1 /**************************************************************************
3 * Copyright 2010 VMware, Inc.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **************************************************************************/
28 #include "util/u_math.h"
29 #include "util/u_memory.h"
30 #include "util/u_prim.h"
31 #include "draw/draw_context.h"
32 #include "draw/draw_gs.h"
33 #include "draw/draw_vbuf.h"
34 #include "draw/draw_vertex.h"
35 #include "draw/draw_pt.h"
36 #include "draw/draw_prim_assembler.h"
37 #include "draw/draw_vs.h"
38 #include "draw/draw_llvm.h"
39 #include "gallivm/lp_bld_init.h"
42 struct llvm_middle_end {
43 struct draw_pt_middle_end base;
44 struct draw_context *draw;
47 struct pt_so_emit *so_emit;
48 struct pt_fetch *fetch;
49 struct pt_post_vs *post_vs;
52 unsigned vertex_data_offset;
57 struct draw_llvm *llvm;
58 struct draw_llvm_variant *current_variant;
63 llvm_middle_end_prepare_gs(struct llvm_middle_end *fpme)
65 struct draw_context *draw = fpme->draw;
66 struct draw_geometry_shader *gs = draw->gs.geometry_shader;
67 struct draw_gs_llvm_variant_key *key;
68 struct draw_gs_llvm_variant *variant = NULL;
69 struct draw_gs_llvm_variant_list_item *li;
70 struct llvm_geometry_shader *shader = llvm_geometry_shader(gs);
71 char store[DRAW_GS_LLVM_MAX_VARIANT_KEY_SIZE];
74 key = draw_gs_llvm_make_variant_key(fpme->llvm, store);
76 /* Search shader's list of variants for the key */
77 li = first_elem(&shader->variants);
78 while (!at_end(&shader->variants, li)) {
79 if (memcmp(&li->base->key, key, shader->variant_key_size) == 0) {
87 /* found the variant, move to head of global list (for LRU) */
88 move_to_head(&fpme->llvm->gs_variants_list,
89 &variant->list_item_global);
92 /* Need to create new variant */
94 /* First check if we've created too many variants. If so, free
95 * 25% of the LRU to avoid using too much memory.
97 if (fpme->llvm->nr_gs_variants >= DRAW_MAX_SHADER_VARIANTS) {
99 * XXX: should we flush here ?
101 for (i = 0; i < DRAW_MAX_SHADER_VARIANTS / 4; i++) {
102 struct draw_gs_llvm_variant_list_item *item;
103 if (is_empty_list(&fpme->llvm->gs_variants_list)) {
106 item = last_elem(&fpme->llvm->gs_variants_list);
109 draw_gs_llvm_destroy_variant(item->base);
113 variant = draw_gs_llvm_create_variant(fpme->llvm, gs->info.num_outputs, key);
116 insert_at_head(&shader->variants, &variant->list_item_local);
117 insert_at_head(&fpme->llvm->gs_variants_list,
118 &variant->list_item_global);
119 fpme->llvm->nr_gs_variants++;
120 shader->variants_cached++;
124 gs->current_variant = variant;
128 * Prepare/validate middle part of the vertex pipeline.
129 * NOTE: if you change this function, also look at the non-LLVM
130 * function fetch_pipeline_prepare() for similar changes.
133 llvm_middle_end_prepare( struct draw_pt_middle_end *middle,
136 unsigned *max_vertices )
138 struct llvm_middle_end *fpme = (struct llvm_middle_end *)middle;
139 struct draw_context *draw = fpme->draw;
140 struct draw_vertex_shader *vs = draw->vs.vertex_shader;
141 struct draw_geometry_shader *gs = draw->gs.geometry_shader;
142 const unsigned out_prim = gs ? gs->output_primitive :
143 u_assembled_prim(in_prim);
145 /* Add one to num_outputs because the pipeline occasionally tags on
146 * an additional texcoord, eg for AA lines.
148 const unsigned nr = MAX2( vs->info.num_inputs,
149 vs->info.num_outputs + 1 );
151 fpme->input_prim = in_prim;
154 /* Always leave room for the vertex header whether we need it or
155 * not. It's hard to get rid of it in particular because of the
156 * viewport code in draw_pt_post_vs.c.
158 fpme->vertex_size = sizeof(struct vertex_header) + nr * 4 * sizeof(float);
161 draw_pt_post_vs_prepare( fpme->post_vs,
166 draw->identity_viewport,
167 draw->rasterizer->clip_halfz,
168 (draw->vs.edgeflag_output ? TRUE : FALSE) );
170 draw_pt_so_emit_prepare( fpme->so_emit, gs == NULL );
172 if (!(opt & PT_PIPELINE)) {
173 draw_pt_emit_prepare( fpme->emit,
177 *max_vertices = MAX2( *max_vertices, 4096 );
180 /* limit max fetches by limiting max_vertices */
181 *max_vertices = 4096;
184 /* return even number */
185 *max_vertices = *max_vertices & ~1;
187 /* Find/create the vertex shader variant */
189 struct draw_llvm_variant_key *key;
190 struct draw_llvm_variant *variant = NULL;
191 struct draw_llvm_variant_list_item *li;
192 struct llvm_vertex_shader *shader = llvm_vertex_shader(vs);
193 char store[DRAW_LLVM_MAX_VARIANT_KEY_SIZE];
196 key = draw_llvm_make_variant_key(fpme->llvm, store);
198 /* Search shader's list of variants for the key */
199 li = first_elem(&shader->variants);
200 while (!at_end(&shader->variants, li)) {
201 if (memcmp(&li->base->key, key, shader->variant_key_size) == 0) {
209 /* found the variant, move to head of global list (for LRU) */
210 move_to_head(&fpme->llvm->vs_variants_list,
211 &variant->list_item_global);
214 /* Need to create new variant */
216 /* First check if we've created too many variants. If so, free
217 * 25% of the LRU to avoid using too much memory.
219 if (fpme->llvm->nr_variants >= DRAW_MAX_SHADER_VARIANTS) {
221 * XXX: should we flush here ?
223 for (i = 0; i < DRAW_MAX_SHADER_VARIANTS / 4; i++) {
224 struct draw_llvm_variant_list_item *item;
225 if (is_empty_list(&fpme->llvm->vs_variants_list)) {
228 item = last_elem(&fpme->llvm->vs_variants_list);
231 draw_llvm_destroy_variant(item->base);
235 variant = draw_llvm_create_variant(fpme->llvm, nr, key);
238 insert_at_head(&shader->variants, &variant->list_item_local);
239 insert_at_head(&fpme->llvm->vs_variants_list,
240 &variant->list_item_global);
241 fpme->llvm->nr_variants++;
242 shader->variants_cached++;
246 fpme->current_variant = variant;
250 llvm_middle_end_prepare_gs(fpme);
256 * Bind/update constant buffer pointers, clip planes and viewport dims.
257 * These are "light weight" parameters which aren't baked into the
258 * generated code. Updating these items is much cheaper than revalidating
259 * and rebuilding the generated pipeline code.
262 llvm_middle_end_bind_parameters(struct draw_pt_middle_end *middle)
264 struct llvm_middle_end *fpme = (struct llvm_middle_end *)middle;
265 struct draw_context *draw = fpme->draw;
268 for (i = 0; i < Elements(fpme->llvm->jit_context.vs_constants); ++i) {
269 fpme->llvm->jit_context.vs_constants[i] = draw->pt.user.vs_constants[i];
271 for (i = 0; i < Elements(fpme->llvm->gs_jit_context.constants); ++i) {
272 fpme->llvm->gs_jit_context.constants[i] = draw->pt.user.gs_constants[i];
275 fpme->llvm->jit_context.planes =
276 (float (*)[DRAW_TOTAL_CLIP_PLANES][4]) draw->pt.user.planes[0];
277 fpme->llvm->gs_jit_context.planes =
278 (float (*)[DRAW_TOTAL_CLIP_PLANES][4]) draw->pt.user.planes[0];
280 fpme->llvm->jit_context.viewport = (float *) draw->viewport.scale;
281 fpme->llvm->gs_jit_context.viewport = (float *) draw->viewport.scale;
285 static void pipeline(struct llvm_middle_end *llvm,
286 const struct draw_vertex_info *vert_info,
287 const struct draw_prim_info *prim_info)
289 if (prim_info->linear)
290 draw_pipeline_run_linear( llvm->draw,
294 draw_pipeline_run( llvm->draw,
299 static void emit(struct pt_emit *emit,
300 const struct draw_vertex_info *vert_info,
301 const struct draw_prim_info *prim_info)
303 if (prim_info->linear) {
304 draw_pt_emit_linear(emit, vert_info, prim_info);
307 draw_pt_emit(emit, vert_info, prim_info);
312 llvm_pipeline_generic( struct draw_pt_middle_end *middle,
313 const struct draw_fetch_info *fetch_info,
314 const struct draw_prim_info *in_prim_info )
316 struct llvm_middle_end *fpme = (struct llvm_middle_end *)middle;
317 struct draw_context *draw = fpme->draw;
318 struct draw_geometry_shader *gshader = draw->gs.geometry_shader;
319 struct draw_prim_info gs_prim_info;
320 struct draw_vertex_info llvm_vert_info;
321 struct draw_vertex_info gs_vert_info;
322 struct draw_vertex_info *vert_info;
323 struct draw_prim_info ia_prim_info;
324 struct draw_vertex_info ia_vert_info;
325 const struct draw_prim_info *prim_info = in_prim_info;
326 boolean free_prim_info = FALSE;
327 unsigned opt = fpme->opt;
328 unsigned clipped = 0;
330 llvm_vert_info.count = fetch_info->count;
331 llvm_vert_info.vertex_size = fpme->vertex_size;
332 llvm_vert_info.stride = fpme->vertex_size;
333 llvm_vert_info.verts =
334 (struct vertex_header *)MALLOC(fpme->vertex_size *
335 align(fetch_info->count, lp_native_vector_width / 32));
336 if (!llvm_vert_info.verts) {
341 if (draw->collect_statistics) {
342 draw->statistics.ia_vertices += fetch_info->count;
343 draw->statistics.ia_primitives +=
344 u_decomposed_prims_for_vertices(prim_info->prim, prim_info->count);
345 draw->statistics.vs_invocations += fetch_info->count;
348 if (fetch_info->linear)
349 clipped = fpme->current_variant->jit_func( &fpme->llvm->jit_context,
350 llvm_vert_info.verts,
351 draw->pt.user.vbuffer,
355 draw->pt.vertex_buffer,
358 clipped = fpme->current_variant->jit_func_elts( &fpme->llvm->jit_context,
359 llvm_vert_info.verts,
360 draw->pt.user.vbuffer,
364 draw->pt.vertex_buffer,
367 /* Finished with fetch and vs:
370 vert_info = &llvm_vert_info;
373 if ((opt & PT_SHADE) && gshader) {
374 struct draw_vertex_shader *vshader = draw->vs.vertex_shader;
375 draw_geometry_shader_run(gshader,
376 draw->pt.user.gs_constants,
377 draw->pt.user.gs_constants_size,
384 FREE(vert_info->verts);
385 vert_info = &gs_vert_info;
386 prim_info = &gs_prim_info;
388 if (draw_prim_assembler_is_required(draw, prim_info, vert_info)) {
389 draw_prim_assembler_run(draw, prim_info, vert_info,
390 &ia_prim_info, &ia_vert_info);
392 if (ia_vert_info.count) {
393 FREE(vert_info->verts);
394 vert_info = &ia_vert_info;
395 prim_info = &ia_prim_info;
396 free_prim_info = TRUE;
400 if (prim_info->count == 0) {
401 debug_printf("GS/IA didn't emit any vertices!\n");
403 FREE(vert_info->verts);
404 if (free_prim_info) {
405 FREE(prim_info->primitive_lengths);
410 /* stream output needs to be done before clipping */
411 draw_pt_so_emit( fpme->so_emit, vert_info, prim_info );
414 * if there's no position, need to stop now, or the latter stages
415 * will try to access non-existent position output.
417 if (draw_current_shader_position_output(draw) != -1) {
418 if ((opt & PT_SHADE) && gshader) {
419 clipped = draw_pt_post_vs_run( fpme->post_vs, vert_info );
425 /* Do we need to run the pipeline? Now will come here if clipped
427 if (opt & PT_PIPELINE) {
428 pipeline( fpme, vert_info, prim_info );
431 emit( fpme->emit, vert_info, prim_info );
434 FREE(vert_info->verts);
435 if (free_prim_info) {
436 FREE(prim_info->primitive_lengths);
441 static void llvm_middle_end_run( struct draw_pt_middle_end *middle,
442 const unsigned *fetch_elts,
443 unsigned fetch_count,
444 const ushort *draw_elts,
446 unsigned prim_flags )
448 struct llvm_middle_end *fpme = (struct llvm_middle_end *)middle;
449 struct draw_fetch_info fetch_info;
450 struct draw_prim_info prim_info;
452 fetch_info.linear = FALSE;
453 fetch_info.start = 0;
454 fetch_info.elts = fetch_elts;
455 fetch_info.count = fetch_count;
457 prim_info.linear = FALSE;
459 prim_info.count = draw_count;
460 prim_info.elts = draw_elts;
461 prim_info.prim = fpme->input_prim;
462 prim_info.flags = prim_flags;
463 prim_info.primitive_count = 1;
464 prim_info.primitive_lengths = &draw_count;
466 llvm_pipeline_generic( middle, &fetch_info, &prim_info );
470 static void llvm_middle_end_linear_run( struct draw_pt_middle_end *middle,
475 struct llvm_middle_end *fpme = (struct llvm_middle_end *)middle;
476 struct draw_fetch_info fetch_info;
477 struct draw_prim_info prim_info;
479 fetch_info.linear = TRUE;
480 fetch_info.start = start;
481 fetch_info.count = count;
482 fetch_info.elts = NULL;
484 prim_info.linear = TRUE;
486 prim_info.count = count;
487 prim_info.elts = NULL;
488 prim_info.prim = fpme->input_prim;
489 prim_info.flags = prim_flags;
490 prim_info.primitive_count = 1;
491 prim_info.primitive_lengths = &count;
493 llvm_pipeline_generic( middle, &fetch_info, &prim_info );
499 llvm_middle_end_linear_run_elts( struct draw_pt_middle_end *middle,
502 const ushort *draw_elts,
504 unsigned prim_flags )
506 struct llvm_middle_end *fpme = (struct llvm_middle_end *)middle;
507 struct draw_fetch_info fetch_info;
508 struct draw_prim_info prim_info;
510 fetch_info.linear = TRUE;
511 fetch_info.start = start;
512 fetch_info.count = count;
513 fetch_info.elts = NULL;
515 prim_info.linear = FALSE;
517 prim_info.count = draw_count;
518 prim_info.elts = draw_elts;
519 prim_info.prim = fpme->input_prim;
520 prim_info.flags = prim_flags;
521 prim_info.primitive_count = 1;
522 prim_info.primitive_lengths = &draw_count;
524 llvm_pipeline_generic( middle, &fetch_info, &prim_info );
531 static void llvm_middle_end_finish( struct draw_pt_middle_end *middle )
536 static void llvm_middle_end_destroy( struct draw_pt_middle_end *middle )
538 struct llvm_middle_end *fpme = (struct llvm_middle_end *)middle;
541 draw_pt_fetch_destroy( fpme->fetch );
544 draw_pt_emit_destroy( fpme->emit );
547 draw_pt_so_emit_destroy( fpme->so_emit );
550 draw_pt_post_vs_destroy( fpme->post_vs );
556 struct draw_pt_middle_end *
557 draw_pt_fetch_pipeline_or_emit_llvm(struct draw_context *draw)
559 struct llvm_middle_end *fpme = 0;
564 fpme = CALLOC_STRUCT( llvm_middle_end );
568 fpme->base.prepare = llvm_middle_end_prepare;
569 fpme->base.bind_parameters = llvm_middle_end_bind_parameters;
570 fpme->base.run = llvm_middle_end_run;
571 fpme->base.run_linear = llvm_middle_end_linear_run;
572 fpme->base.run_linear_elts = llvm_middle_end_linear_run_elts;
573 fpme->base.finish = llvm_middle_end_finish;
574 fpme->base.destroy = llvm_middle_end_destroy;
578 fpme->fetch = draw_pt_fetch_create( draw );
582 fpme->post_vs = draw_pt_post_vs_create( draw );
586 fpme->emit = draw_pt_emit_create( draw );
590 fpme->so_emit = draw_pt_so_emit_create( draw );
594 fpme->llvm = draw->llvm;
598 fpme->current_variant = NULL;
604 llvm_middle_end_destroy( &fpme->base );