1 /**************************************************************************
3 * Copyright 2010 VMware, Inc.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **************************************************************************/
28 #include "util/u_math.h"
29 #include "util/u_memory.h"
30 #include "util/u_prim.h"
31 #include "draw/draw_context.h"
32 #include "draw/draw_gs.h"
33 #include "draw/draw_vbuf.h"
34 #include "draw/draw_vertex.h"
35 #include "draw/draw_pt.h"
36 #include "draw/draw_prim_assembler.h"
37 #include "draw/draw_vs.h"
38 #include "draw/draw_llvm.h"
39 #include "gallivm/lp_bld_init.h"
42 struct llvm_middle_end {
43 struct draw_pt_middle_end base;
44 struct draw_context *draw;
47 struct pt_so_emit *so_emit;
48 struct pt_fetch *fetch;
49 struct pt_post_vs *post_vs;
52 unsigned vertex_data_offset;
57 struct draw_llvm *llvm;
58 struct draw_llvm_variant *current_variant;
63 llvm_middle_end_prepare_gs(struct llvm_middle_end *fpme)
65 struct draw_context *draw = fpme->draw;
66 struct draw_geometry_shader *gs = draw->gs.geometry_shader;
67 struct draw_gs_llvm_variant_key *key;
68 struct draw_gs_llvm_variant *variant = NULL;
69 struct draw_gs_llvm_variant_list_item *li;
70 struct llvm_geometry_shader *shader = llvm_geometry_shader(gs);
71 char store[DRAW_GS_LLVM_MAX_VARIANT_KEY_SIZE];
74 key = draw_gs_llvm_make_variant_key(fpme->llvm, store);
76 /* Search shader's list of variants for the key */
77 li = first_elem(&shader->variants);
78 while (!at_end(&shader->variants, li)) {
79 if (memcmp(&li->base->key, key, shader->variant_key_size) == 0) {
87 /* found the variant, move to head of global list (for LRU) */
88 move_to_head(&fpme->llvm->gs_variants_list,
89 &variant->list_item_global);
92 /* Need to create new variant */
94 /* First check if we've created too many variants. If so, free
95 * 25% of the LRU to avoid using too much memory.
97 if (fpme->llvm->nr_gs_variants >= DRAW_MAX_SHADER_VARIANTS) {
99 * XXX: should we flush here ?
101 for (i = 0; i < DRAW_MAX_SHADER_VARIANTS / 4; i++) {
102 struct draw_gs_llvm_variant_list_item *item;
103 if (is_empty_list(&fpme->llvm->gs_variants_list)) {
106 item = last_elem(&fpme->llvm->gs_variants_list);
109 draw_gs_llvm_destroy_variant(item->base);
113 variant = draw_gs_llvm_create_variant(fpme->llvm, gs->info.num_outputs, key);
116 insert_at_head(&shader->variants, &variant->list_item_local);
117 insert_at_head(&fpme->llvm->gs_variants_list,
118 &variant->list_item_global);
119 fpme->llvm->nr_gs_variants++;
120 shader->variants_cached++;
124 gs->current_variant = variant;
128 * Prepare/validate middle part of the vertex pipeline.
129 * NOTE: if you change this function, also look at the non-LLVM
130 * function fetch_pipeline_prepare() for similar changes.
133 llvm_middle_end_prepare( struct draw_pt_middle_end *middle,
136 unsigned *max_vertices )
138 struct llvm_middle_end *fpme = (struct llvm_middle_end *)middle;
139 struct draw_context *draw = fpme->draw;
140 struct draw_vertex_shader *vs = draw->vs.vertex_shader;
141 struct draw_geometry_shader *gs = draw->gs.geometry_shader;
142 const unsigned out_prim = gs ? gs->output_primitive :
143 u_assembled_prim(in_prim);
144 const unsigned nr = MAX2(vs->info.num_inputs,
145 draw_total_vs_outputs(draw));
146 unsigned point_clip = draw->rasterizer->fill_front == PIPE_POLYGON_MODE_POINT ||
147 out_prim == PIPE_PRIM_POINTS;
149 fpme->input_prim = in_prim;
152 /* Always leave room for the vertex header whether we need it or
153 * not. It's hard to get rid of it in particular because of the
154 * viewport code in draw_pt_post_vs.c.
156 fpme->vertex_size = sizeof(struct vertex_header) + nr * 4 * sizeof(float);
159 draw_pt_post_vs_prepare( fpme->post_vs,
163 point_clip ? draw->guard_band_points_xy :
165 draw->identity_viewport,
166 draw->rasterizer->clip_halfz,
167 (draw->vs.edgeflag_output ? TRUE : FALSE) );
169 draw_pt_so_emit_prepare( fpme->so_emit, gs == NULL );
171 if (!(opt & PT_PIPELINE)) {
172 draw_pt_emit_prepare( fpme->emit,
176 *max_vertices = MAX2( *max_vertices, 4096 );
179 /* limit max fetches by limiting max_vertices */
180 *max_vertices = 4096;
183 /* return even number */
184 *max_vertices = *max_vertices & ~1;
186 /* Find/create the vertex shader variant */
188 struct draw_llvm_variant_key *key;
189 struct draw_llvm_variant *variant = NULL;
190 struct draw_llvm_variant_list_item *li;
191 struct llvm_vertex_shader *shader = llvm_vertex_shader(vs);
192 char store[DRAW_LLVM_MAX_VARIANT_KEY_SIZE];
195 key = draw_llvm_make_variant_key(fpme->llvm, store);
197 /* Search shader's list of variants for the key */
198 li = first_elem(&shader->variants);
199 while (!at_end(&shader->variants, li)) {
200 if (memcmp(&li->base->key, key, shader->variant_key_size) == 0) {
208 /* found the variant, move to head of global list (for LRU) */
209 move_to_head(&fpme->llvm->vs_variants_list,
210 &variant->list_item_global);
213 /* Need to create new variant */
215 /* First check if we've created too many variants. If so, free
216 * 25% of the LRU to avoid using too much memory.
218 if (fpme->llvm->nr_variants >= DRAW_MAX_SHADER_VARIANTS) {
220 * XXX: should we flush here ?
222 for (i = 0; i < DRAW_MAX_SHADER_VARIANTS / 4; i++) {
223 struct draw_llvm_variant_list_item *item;
224 if (is_empty_list(&fpme->llvm->vs_variants_list)) {
227 item = last_elem(&fpme->llvm->vs_variants_list);
230 draw_llvm_destroy_variant(item->base);
234 variant = draw_llvm_create_variant(fpme->llvm, nr, key);
237 insert_at_head(&shader->variants, &variant->list_item_local);
238 insert_at_head(&fpme->llvm->vs_variants_list,
239 &variant->list_item_global);
240 fpme->llvm->nr_variants++;
241 shader->variants_cached++;
245 fpme->current_variant = variant;
249 llvm_middle_end_prepare_gs(fpme);
255 * Bind/update constant buffer pointers, clip planes and viewport dims.
256 * These are "light weight" parameters which aren't baked into the
257 * generated code. Updating these items is much cheaper than revalidating
258 * and rebuilding the generated pipeline code.
261 llvm_middle_end_bind_parameters(struct draw_pt_middle_end *middle)
263 struct llvm_middle_end *fpme = (struct llvm_middle_end *)middle;
264 struct draw_context *draw = fpme->draw;
267 for (i = 0; i < Elements(fpme->llvm->jit_context.vs_constants); ++i) {
269 draw->pt.user.vs_constants_size[i] / (sizeof(float) * 4);
270 fpme->llvm->jit_context.vs_constants[i] = draw->pt.user.vs_constants[i];
271 fpme->llvm->jit_context.num_vs_constants[i] = num_consts;
273 for (i = 0; i < Elements(fpme->llvm->gs_jit_context.constants); ++i) {
275 draw->pt.user.gs_constants_size[i] / (sizeof(float) * 4);
276 fpme->llvm->gs_jit_context.constants[i] = draw->pt.user.gs_constants[i];
277 fpme->llvm->gs_jit_context.num_constants[i] = num_consts;
280 fpme->llvm->jit_context.planes =
281 (float (*)[DRAW_TOTAL_CLIP_PLANES][4]) draw->pt.user.planes[0];
282 fpme->llvm->gs_jit_context.planes =
283 (float (*)[DRAW_TOTAL_CLIP_PLANES][4]) draw->pt.user.planes[0];
285 fpme->llvm->jit_context.viewport = (float *) draw->viewports[0].scale;
286 fpme->llvm->gs_jit_context.viewport = (float *) draw->viewports[0].scale;
290 static void pipeline(struct llvm_middle_end *llvm,
291 const struct draw_vertex_info *vert_info,
292 const struct draw_prim_info *prim_info)
294 if (prim_info->linear)
295 draw_pipeline_run_linear( llvm->draw,
299 draw_pipeline_run( llvm->draw,
304 static void emit(struct pt_emit *emit,
305 const struct draw_vertex_info *vert_info,
306 const struct draw_prim_info *prim_info)
308 if (prim_info->linear) {
309 draw_pt_emit_linear(emit, vert_info, prim_info);
312 draw_pt_emit(emit, vert_info, prim_info);
317 llvm_pipeline_generic( struct draw_pt_middle_end *middle,
318 const struct draw_fetch_info *fetch_info,
319 const struct draw_prim_info *in_prim_info )
321 struct llvm_middle_end *fpme = (struct llvm_middle_end *)middle;
322 struct draw_context *draw = fpme->draw;
323 struct draw_geometry_shader *gshader = draw->gs.geometry_shader;
324 struct draw_prim_info gs_prim_info;
325 struct draw_vertex_info llvm_vert_info;
326 struct draw_vertex_info gs_vert_info;
327 struct draw_vertex_info *vert_info;
328 struct draw_prim_info ia_prim_info;
329 struct draw_vertex_info ia_vert_info;
330 const struct draw_prim_info *prim_info = in_prim_info;
331 boolean free_prim_info = FALSE;
332 unsigned opt = fpme->opt;
333 unsigned clipped = 0;
335 llvm_vert_info.count = fetch_info->count;
336 llvm_vert_info.vertex_size = fpme->vertex_size;
337 llvm_vert_info.stride = fpme->vertex_size;
338 llvm_vert_info.verts =
339 (struct vertex_header *)MALLOC(fpme->vertex_size *
340 align(fetch_info->count, lp_native_vector_width / 32));
341 if (!llvm_vert_info.verts) {
346 if (draw->collect_statistics) {
347 draw->statistics.ia_vertices += prim_info->count;
348 draw->statistics.ia_primitives +=
349 u_decomposed_prims_for_vertices(prim_info->prim, prim_info->count);
350 draw->statistics.vs_invocations += fetch_info->count;
353 if (fetch_info->linear)
354 clipped = fpme->current_variant->jit_func( &fpme->llvm->jit_context,
355 llvm_vert_info.verts,
356 draw->pt.user.vbuffer,
360 draw->pt.vertex_buffer,
364 clipped = fpme->current_variant->jit_func_elts( &fpme->llvm->jit_context,
365 llvm_vert_info.verts,
366 draw->pt.user.vbuffer,
368 draw->pt.user.eltMax,
371 draw->pt.vertex_buffer,
373 draw->pt.user.eltBias);
375 /* Finished with fetch and vs:
378 vert_info = &llvm_vert_info;
381 if ((opt & PT_SHADE) && gshader) {
382 struct draw_vertex_shader *vshader = draw->vs.vertex_shader;
383 draw_geometry_shader_run(gshader,
384 draw->pt.user.gs_constants,
385 draw->pt.user.gs_constants_size,
392 FREE(vert_info->verts);
393 vert_info = &gs_vert_info;
394 prim_info = &gs_prim_info;
396 if (draw_prim_assembler_is_required(draw, prim_info, vert_info)) {
397 draw_prim_assembler_run(draw, prim_info, vert_info,
398 &ia_prim_info, &ia_vert_info);
400 if (ia_vert_info.count) {
401 FREE(vert_info->verts);
402 vert_info = &ia_vert_info;
403 prim_info = &ia_prim_info;
404 free_prim_info = TRUE;
408 if (prim_info->count == 0) {
409 debug_printf("GS/IA didn't emit any vertices!\n");
411 FREE(vert_info->verts);
412 if (free_prim_info) {
413 FREE(prim_info->primitive_lengths);
418 /* stream output needs to be done before clipping */
419 draw_pt_so_emit( fpme->so_emit, vert_info, prim_info );
421 draw_stats_clipper_primitives(draw, prim_info);
424 * if there's no position, need to stop now, or the latter stages
425 * will try to access non-existent position output.
427 if (draw_current_shader_position_output(draw) != -1) {
428 if ((opt & PT_SHADE) && gshader) {
429 clipped = draw_pt_post_vs_run( fpme->post_vs, vert_info, prim_info );
435 /* Do we need to run the pipeline? Now will come here if clipped
437 if (opt & PT_PIPELINE) {
438 pipeline( fpme, vert_info, prim_info );
441 emit( fpme->emit, vert_info, prim_info );
444 FREE(vert_info->verts);
445 if (free_prim_info) {
446 FREE(prim_info->primitive_lengths);
451 static void llvm_middle_end_run( struct draw_pt_middle_end *middle,
452 const unsigned *fetch_elts,
453 unsigned fetch_count,
454 const ushort *draw_elts,
456 unsigned prim_flags )
458 struct llvm_middle_end *fpme = (struct llvm_middle_end *)middle;
459 struct draw_fetch_info fetch_info;
460 struct draw_prim_info prim_info;
462 fetch_info.linear = FALSE;
463 fetch_info.start = 0;
464 fetch_info.elts = fetch_elts;
465 fetch_info.count = fetch_count;
467 prim_info.linear = FALSE;
469 prim_info.count = draw_count;
470 prim_info.elts = draw_elts;
471 prim_info.prim = fpme->input_prim;
472 prim_info.flags = prim_flags;
473 prim_info.primitive_count = 1;
474 prim_info.primitive_lengths = &draw_count;
476 llvm_pipeline_generic( middle, &fetch_info, &prim_info );
480 static void llvm_middle_end_linear_run( struct draw_pt_middle_end *middle,
485 struct llvm_middle_end *fpme = (struct llvm_middle_end *)middle;
486 struct draw_fetch_info fetch_info;
487 struct draw_prim_info prim_info;
489 fetch_info.linear = TRUE;
490 fetch_info.start = start;
491 fetch_info.count = count;
492 fetch_info.elts = NULL;
494 prim_info.linear = TRUE;
496 prim_info.count = count;
497 prim_info.elts = NULL;
498 prim_info.prim = fpme->input_prim;
499 prim_info.flags = prim_flags;
500 prim_info.primitive_count = 1;
501 prim_info.primitive_lengths = &count;
503 llvm_pipeline_generic( middle, &fetch_info, &prim_info );
509 llvm_middle_end_linear_run_elts( struct draw_pt_middle_end *middle,
512 const ushort *draw_elts,
514 unsigned prim_flags )
516 struct llvm_middle_end *fpme = (struct llvm_middle_end *)middle;
517 struct draw_fetch_info fetch_info;
518 struct draw_prim_info prim_info;
520 fetch_info.linear = TRUE;
521 fetch_info.start = start;
522 fetch_info.count = count;
523 fetch_info.elts = NULL;
525 prim_info.linear = FALSE;
527 prim_info.count = draw_count;
528 prim_info.elts = draw_elts;
529 prim_info.prim = fpme->input_prim;
530 prim_info.flags = prim_flags;
531 prim_info.primitive_count = 1;
532 prim_info.primitive_lengths = &draw_count;
534 llvm_pipeline_generic( middle, &fetch_info, &prim_info );
541 static void llvm_middle_end_finish( struct draw_pt_middle_end *middle )
546 static void llvm_middle_end_destroy( struct draw_pt_middle_end *middle )
548 struct llvm_middle_end *fpme = (struct llvm_middle_end *)middle;
551 draw_pt_fetch_destroy( fpme->fetch );
554 draw_pt_emit_destroy( fpme->emit );
557 draw_pt_so_emit_destroy( fpme->so_emit );
560 draw_pt_post_vs_destroy( fpme->post_vs );
566 struct draw_pt_middle_end *
567 draw_pt_fetch_pipeline_or_emit_llvm(struct draw_context *draw)
569 struct llvm_middle_end *fpme = 0;
574 fpme = CALLOC_STRUCT( llvm_middle_end );
578 fpme->base.prepare = llvm_middle_end_prepare;
579 fpme->base.bind_parameters = llvm_middle_end_bind_parameters;
580 fpme->base.run = llvm_middle_end_run;
581 fpme->base.run_linear = llvm_middle_end_linear_run;
582 fpme->base.run_linear_elts = llvm_middle_end_linear_run_elts;
583 fpme->base.finish = llvm_middle_end_finish;
584 fpme->base.destroy = llvm_middle_end_destroy;
588 fpme->fetch = draw_pt_fetch_create( draw );
592 fpme->post_vs = draw_pt_post_vs_create( draw );
596 fpme->emit = draw_pt_emit_create( draw );
600 fpme->so_emit = draw_pt_so_emit_create( draw );
604 fpme->llvm = draw->llvm;
608 fpme->current_variant = NULL;
614 llvm_middle_end_destroy( &fpme->base );