src/gallium/auxiliary/draw/draw_pt_fetch_shade_pipeline_llvm.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2010 VMware, Inc.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  19  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  21  * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
  22  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  23  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  24  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27
  28 #include "util/u_math.h"
  29 #include "util/u_memory.h"
  30 #include "util/u_prim.h"
  31 #include "draw/draw_context.h"
  32 #include "draw/draw_gs.h"
  33 #include "draw/draw_vbuf.h"
  34 #include "draw/draw_vertex.h"
  35 #include "draw/draw_pt.h"
  36 #include "draw/draw_prim_assembler.h"
  37 #include "draw/draw_vs.h"
  38 #include "draw/draw_llvm.h"
  39 #include "gallivm/lp_bld_init.h"
  40
  41
  42 struct llvm_middle_end {
  43    struct draw_pt_middle_end base;
  44    struct draw_context *draw;
  45
  46    struct pt_emit *emit;
  47    struct pt_so_emit *so_emit;
  48    struct pt_fetch *fetch;
  49    struct pt_post_vs *post_vs;
  50
  51
  52    unsigned vertex_data_offset;
  53    unsigned vertex_size;
  54    unsigned input_prim;
  55    unsigned opt;
  56
  57    struct draw_llvm *llvm;
  58    struct draw_llvm_variant *current_variant;
  59 };
  60
  61
  62 static void
  63 llvm_middle_end_prepare_gs(struct llvm_middle_end *fpme)
  64 {
  65    struct draw_context *draw = fpme->draw;
  66    struct draw_geometry_shader *gs = draw->gs.geometry_shader;
  67    struct draw_gs_llvm_variant_key *key;
  68    struct draw_gs_llvm_variant *variant = NULL;
  69    struct draw_gs_llvm_variant_list_item *li;
  70    struct llvm_geometry_shader *shader = llvm_geometry_shader(gs);
  71    char store[DRAW_GS_LLVM_MAX_VARIANT_KEY_SIZE];
  72    unsigned i;
  73
  74    key = draw_gs_llvm_make_variant_key(fpme->llvm, store);
  75
  76    /* Search shader's list of variants for the key */
  77    li = first_elem(&shader->variants);
  78    while (!at_end(&shader->variants, li)) {
  79       if (memcmp(&li->base->key, key, shader->variant_key_size) == 0) {
  80          variant = li->base;
  81          break;
  82       }
  83       li = next_elem(li);
  84    }
  85
  86    if (variant) {
  87       /* found the variant, move to head of global list (for LRU) */
  88       move_to_head(&fpme->llvm->gs_variants_list,
  89                    &variant->list_item_global);
  90    }
  91    else {
  92       /* Need to create new variant */
  93
  94       /* First check if we've created too many variants.  If so, free
  95        * 25% of the LRU to avoid using too much memory.
  96        */
  97       if (fpme->llvm->nr_gs_variants >= DRAW_MAX_SHADER_VARIANTS) {
  98          /*
  99           * XXX: should we flush here ?
 100           */
 101          for (i = 0; i < DRAW_MAX_SHADER_VARIANTS / 4; i++) {
 102             struct draw_gs_llvm_variant_list_item *item;
 103             if (is_empty_list(&fpme->llvm->gs_variants_list)) {
 104                break;
 105             }
 106             item = last_elem(&fpme->llvm->gs_variants_list);
 107             assert(item);
 108             assert(item->base);
 109             draw_gs_llvm_destroy_variant(item->base);
 110          }
 111       }
 112
 113       variant = draw_gs_llvm_create_variant(fpme->llvm, gs->info.num_outputs, key);
 114
 115       if (variant) {
 116          insert_at_head(&shader->variants, &variant->list_item_local);
 117          insert_at_head(&fpme->llvm->gs_variants_list,
 118                         &variant->list_item_global);
 119          fpme->llvm->nr_gs_variants++;
 120          shader->variants_cached++;
 121       }
 122    }
 123
 124    gs->current_variant = variant;
 125 }
 126
 127 /**
 128  * Prepare/validate middle part of the vertex pipeline.
 129  * NOTE: if you change this function, also look at the non-LLVM
 130  * function fetch_pipeline_prepare() for similar changes.
 131  */
 132 static void
 133 llvm_middle_end_prepare( struct draw_pt_middle_end *middle,
 134                          unsigned in_prim,
 135                          unsigned opt,
 136                          unsigned *max_vertices )
 137 {
 138    struct llvm_middle_end *fpme = (struct llvm_middle_end *)middle;
 139    struct draw_context *draw = fpme->draw;
 140    struct draw_vertex_shader *vs = draw->vs.vertex_shader;
 141    struct draw_geometry_shader *gs = draw->gs.geometry_shader;
 142    const unsigned out_prim = gs ? gs->output_primitive :
 143       u_assembled_prim(in_prim);
 144    const unsigned nr = MAX2(vs->info.num_inputs,
 145                             draw_total_vs_outputs(draw));
 146    unsigned point_clip = draw->rasterizer->fill_front == PIPE_POLYGON_MODE_POINT ||
 147                          out_prim == PIPE_PRIM_POINTS;
 148
 149    fpme->input_prim = in_prim;
 150    fpme->opt = opt;
 151
 152    /* Always leave room for the vertex header whether we need it or
 153     * not.  It's hard to get rid of it in particular because of the
 154     * viewport code in draw_pt_post_vs.c.
 155     */
 156    fpme->vertex_size = sizeof(struct vertex_header) + nr * 4 * sizeof(float);
 157
 158
 159    draw_pt_post_vs_prepare( fpme->post_vs,
 160                             draw->clip_xy,
 161                             draw->clip_z,
 162                             draw->clip_user,
 163                             point_clip ? draw->guard_band_points_xy :
 164                                          draw->guard_band_xy,
 165                             draw->identity_viewport,
 166                             draw->rasterizer->clip_halfz,
 167                             (draw->vs.edgeflag_output ? TRUE : FALSE) );
 168
 169    draw_pt_so_emit_prepare( fpme->so_emit, gs == NULL );
 170
 171    if (!(opt & PT_PIPELINE)) {
 172       draw_pt_emit_prepare( fpme->emit,
 173                             out_prim,
 174                             max_vertices );
 175
 176       *max_vertices = MAX2( *max_vertices, 4096 );
 177    }
 178    else {
 179       /* limit max fetches by limiting max_vertices */
 180       *max_vertices = 4096;
 181    }
 182
 183    /* return even number */
 184    *max_vertices = *max_vertices & ~1;
 185
 186    /* Find/create the vertex shader variant */
 187    {
 188       struct draw_llvm_variant_key *key;
 189       struct draw_llvm_variant *variant = NULL;
 190       struct draw_llvm_variant_list_item *li;
 191       struct llvm_vertex_shader *shader = llvm_vertex_shader(vs);
 192       char store[DRAW_LLVM_MAX_VARIANT_KEY_SIZE];
 193       unsigned i;
 194
 195       key = draw_llvm_make_variant_key(fpme->llvm, store);
 196
 197       /* Search shader's list of variants for the key */
 198       li = first_elem(&shader->variants);
 199       while (!at_end(&shader->variants, li)) {
 200          if (memcmp(&li->base->key, key, shader->variant_key_size) == 0) {
 201             variant = li->base;
 202             break;
 203          }
 204          li = next_elem(li);
 205       }
 206
 207       if (variant) {
 208          /* found the variant, move to head of global list (for LRU) */
 209          move_to_head(&fpme->llvm->vs_variants_list,
 210                       &variant->list_item_global);
 211       }
 212       else {
 213          /* Need to create new variant */
 214
 215          /* First check if we've created too many variants.  If so, free
 216           * 25% of the LRU to avoid using too much memory.
 217           */
 218          if (fpme->llvm->nr_variants >= DRAW_MAX_SHADER_VARIANTS) {
 219             /*
 220              * XXX: should we flush here ?
 221              */
 222             for (i = 0; i < DRAW_MAX_SHADER_VARIANTS / 4; i++) {
 223                struct draw_llvm_variant_list_item *item;
 224                if (is_empty_list(&fpme->llvm->vs_variants_list)) {
 225                   break;
 226                }
 227                item = last_elem(&fpme->llvm->vs_variants_list);
 228                assert(item);
 229                assert(item->base);
 230                draw_llvm_destroy_variant(item->base);
 231             }
 232          }
 233
 234          variant = draw_llvm_create_variant(fpme->llvm, nr, key);
 235
 236          if (variant) {
 237             insert_at_head(&shader->variants, &variant->list_item_local);
 238             insert_at_head(&fpme->llvm->vs_variants_list,
 239                            &variant->list_item_global);
 240             fpme->llvm->nr_variants++;
 241             shader->variants_cached++;
 242          }
 243       }
 244
 245       fpme->current_variant = variant;
 246    }
 247
 248    if (gs) {
 249       llvm_middle_end_prepare_gs(fpme);
 250    }
 251 }
 252
 253
 254 /**
 255  * Bind/update constant buffer pointers, clip planes and viewport dims.
 256  * These are "light weight" parameters which aren't baked into the
 257  * generated code.  Updating these items is much cheaper than revalidating
 258  * and rebuilding the generated pipeline code.
 259  */
 260 static void
 261 llvm_middle_end_bind_parameters(struct draw_pt_middle_end *middle)
 262 {
 263    struct llvm_middle_end *fpme = (struct llvm_middle_end *)middle;
 264    struct draw_context *draw = fpme->draw;
 265    unsigned i;
 266
 267    for (i = 0; i < Elements(fpme->llvm->jit_context.vs_constants); ++i) {
 268       int num_consts =
 269          draw->pt.user.vs_constants_size[i] / (sizeof(float) * 4);
 270       fpme->llvm->jit_context.vs_constants[i] = draw->pt.user.vs_constants[i];
 271       fpme->llvm->jit_context.num_vs_constants[i] = num_consts;
 272    }
 273    for (i = 0; i < Elements(fpme->llvm->gs_jit_context.constants); ++i) {
 274       int num_consts =
 275          draw->pt.user.gs_constants_size[i] / (sizeof(float) * 4);
 276       fpme->llvm->gs_jit_context.constants[i] = draw->pt.user.gs_constants[i];
 277       fpme->llvm->gs_jit_context.num_constants[i] = num_consts;
 278    }
 279
 280    fpme->llvm->jit_context.planes =
 281       (float (*)[DRAW_TOTAL_CLIP_PLANES][4]) draw->pt.user.planes[0];
 282    fpme->llvm->gs_jit_context.planes =
 283       (float (*)[DRAW_TOTAL_CLIP_PLANES][4]) draw->pt.user.planes[0];
 284
 285    fpme->llvm->jit_context.viewport = (float *) draw->viewports[0].scale;
 286    fpme->llvm->gs_jit_context.viewport = (float *) draw->viewports[0].scale;
 287 }
 288
 289
 290 static void pipeline(struct llvm_middle_end *llvm,
 291                      const struct draw_vertex_info *vert_info,
 292                      const struct draw_prim_info *prim_info)
 293 {
 294    if (prim_info->linear)
 295       draw_pipeline_run_linear( llvm->draw,
 296                                 vert_info,
 297                                 prim_info);
 298    else
 299       draw_pipeline_run( llvm->draw,
 300                          vert_info,
 301                          prim_info );
 302 }
 303
 304 static void emit(struct pt_emit *emit,
 305                  const struct draw_vertex_info *vert_info,
 306                  const struct draw_prim_info *prim_info)
 307 {
 308    if (prim_info->linear) {
 309       draw_pt_emit_linear(emit, vert_info, prim_info);
 310    }
 311    else {
 312       draw_pt_emit(emit, vert_info, prim_info);
 313    }
 314 }
 315
 316 static void
 317 llvm_pipeline_generic( struct draw_pt_middle_end *middle,
 318                        const struct draw_fetch_info *fetch_info,
 319                        const struct draw_prim_info *in_prim_info )
 320 {
 321    struct llvm_middle_end *fpme = (struct llvm_middle_end *)middle;
 322    struct draw_context *draw = fpme->draw;
 323    struct draw_geometry_shader *gshader = draw->gs.geometry_shader;
 324    struct draw_prim_info gs_prim_info;
 325    struct draw_vertex_info llvm_vert_info;
 326    struct draw_vertex_info gs_vert_info;
 327    struct draw_vertex_info *vert_info;
 328    struct draw_prim_info ia_prim_info;
 329    struct draw_vertex_info ia_vert_info;
 330    const struct draw_prim_info *prim_info = in_prim_info;
 331    boolean free_prim_info = FALSE;
 332    unsigned opt = fpme->opt;
 333    unsigned clipped = 0;
 334
 335    llvm_vert_info.count = fetch_info->count;
 336    llvm_vert_info.vertex_size = fpme->vertex_size;
 337    llvm_vert_info.stride = fpme->vertex_size;
 338    llvm_vert_info.verts =
 339       (struct vertex_header *)MALLOC(fpme->vertex_size *
 340                                      align(fetch_info->count,  lp_native_vector_width / 32));
 341    if (!llvm_vert_info.verts) {
 342       assert(0);
 343       return;
 344    }
 345
 346    if (draw->collect_statistics) {
 347       draw->statistics.ia_vertices += prim_info->count;
 348       draw->statistics.ia_primitives +=
 349          u_decomposed_prims_for_vertices(prim_info->prim, prim_info->count);
 350       draw->statistics.vs_invocations += fetch_info->count;
 351    }
 352
 353    if (fetch_info->linear)
 354       clipped = fpme->current_variant->jit_func( &fpme->llvm->jit_context,
 355                                        llvm_vert_info.verts,
 356                                        draw->pt.user.vbuffer,
 357                                        fetch_info->start,
 358                                        fetch_info->count,
 359                                        fpme->vertex_size,
 360                                        draw->pt.vertex_buffer,
 361                                        draw->instance_id,
 362                                        draw->start_index);
 363    else
 364       clipped = fpme->current_variant->jit_func_elts( &fpme->llvm->jit_context,
 365                                             llvm_vert_info.verts,
 366                                             draw->pt.user.vbuffer,
 367                                             fetch_info->elts,
 368                                             draw->pt.user.eltMax,
 369                                             fetch_info->count,
 370                                             fpme->vertex_size,
 371                                             draw->pt.vertex_buffer,
 372                                             draw->instance_id,
 373                                             draw->pt.user.eltBias);
 374
 375    /* Finished with fetch and vs:
 376     */
 377    fetch_info = NULL;
 378    vert_info = &llvm_vert_info;
 379
 380
 381    if ((opt & PT_SHADE) && gshader) {
 382       struct draw_vertex_shader *vshader = draw->vs.vertex_shader;
 383       draw_geometry_shader_run(gshader,
 384                                draw->pt.user.gs_constants,
 385                                draw->pt.user.gs_constants_size,
 386                                vert_info,
 387                                prim_info,
 388                                &vshader->info,
 389                                &gs_vert_info,
 390                                &gs_prim_info);
 391
 392       FREE(vert_info->verts);
 393       vert_info = &gs_vert_info;
 394       prim_info = &gs_prim_info;
 395    } else {
 396       if (draw_prim_assembler_is_required(draw, prim_info, vert_info)) {
 397          draw_prim_assembler_run(draw, prim_info, vert_info,
 398                                  &ia_prim_info, &ia_vert_info);
 399
 400          if (ia_vert_info.count) {
 401             FREE(vert_info->verts);
 402             vert_info = &ia_vert_info;
 403             prim_info = &ia_prim_info;
 404             free_prim_info = TRUE;
 405          }
 406       }
 407    }
 408    if (prim_info->count == 0) {
 409       debug_printf("GS/IA didn't emit any vertices!\n");
 410
 411       FREE(vert_info->verts);
 412       if (free_prim_info) {
 413          FREE(prim_info->primitive_lengths);
 414       }
 415       return;
 416    }
 417
 418    /* stream output needs to be done before clipping */
 419    draw_pt_so_emit( fpme->so_emit, vert_info, prim_info );
 420
 421    draw_stats_clipper_primitives(draw, prim_info);
 422
 423    /*
 424     * if there's no position, need to stop now, or the latter stages
 425     * will try to access non-existent position output.
 426     */
 427    if (draw_current_shader_position_output(draw) != -1) {
 428       if ((opt & PT_SHADE) && gshader) {
 429          clipped = draw_pt_post_vs_run( fpme->post_vs, vert_info, prim_info );
 430       }
 431       if (clipped) {
 432          opt |= PT_PIPELINE;
 433       }
 434
 435       /* Do we need to run the pipeline? Now will come here if clipped
 436        */
 437       if (opt & PT_PIPELINE) {
 438          pipeline( fpme, vert_info, prim_info );
 439       }
 440       else {
 441          emit( fpme->emit, vert_info, prim_info );
 442       }
 443    }
 444    FREE(vert_info->verts);
 445    if (free_prim_info) {
 446       FREE(prim_info->primitive_lengths);
 447    }
 448 }
 449
 450
 451 static void llvm_middle_end_run( struct draw_pt_middle_end *middle,
 452                                  const unsigned *fetch_elts,
 453                                  unsigned fetch_count,
 454                                  const ushort *draw_elts,
 455                                  unsigned draw_count,
 456                                  unsigned prim_flags )
 457 {
 458    struct llvm_middle_end *fpme = (struct llvm_middle_end *)middle;
 459    struct draw_fetch_info fetch_info;
 460    struct draw_prim_info prim_info;
 461
 462    fetch_info.linear = FALSE;
 463    fetch_info.start = 0;
 464    fetch_info.elts = fetch_elts;
 465    fetch_info.count = fetch_count;
 466
 467    prim_info.linear = FALSE;
 468    prim_info.start = 0;
 469    prim_info.count = draw_count;
 470    prim_info.elts = draw_elts;
 471    prim_info.prim = fpme->input_prim;
 472    prim_info.flags = prim_flags;
 473    prim_info.primitive_count = 1;
 474    prim_info.primitive_lengths = &draw_count;
 475
 476    llvm_pipeline_generic( middle, &fetch_info, &prim_info );
 477 }
 478
 479
 480 static void llvm_middle_end_linear_run( struct draw_pt_middle_end *middle,
 481                                        unsigned start,
 482                                        unsigned count,
 483                                        unsigned prim_flags)
 484 {
 485    struct llvm_middle_end *fpme = (struct llvm_middle_end *)middle;
 486    struct draw_fetch_info fetch_info;
 487    struct draw_prim_info prim_info;
 488
 489    fetch_info.linear = TRUE;
 490    fetch_info.start = start;
 491    fetch_info.count = count;
 492    fetch_info.elts = NULL;
 493
 494    prim_info.linear = TRUE;
 495    prim_info.start = 0;
 496    prim_info.count = count;
 497    prim_info.elts = NULL;
 498    prim_info.prim = fpme->input_prim;
 499    prim_info.flags = prim_flags;
 500    prim_info.primitive_count = 1;
 501    prim_info.primitive_lengths = &count;
 502
 503    llvm_pipeline_generic( middle, &fetch_info, &prim_info );
 504 }
 505
 506
 507
 508 static boolean
 509 llvm_middle_end_linear_run_elts( struct draw_pt_middle_end *middle,
 510                                  unsigned start,
 511                                  unsigned count,
 512                                  const ushort *draw_elts,
 513                                  unsigned draw_count,
 514                                  unsigned prim_flags )
 515 {
 516    struct llvm_middle_end *fpme = (struct llvm_middle_end *)middle;
 517    struct draw_fetch_info fetch_info;
 518    struct draw_prim_info prim_info;
 519
 520    fetch_info.linear = TRUE;
 521    fetch_info.start = start;
 522    fetch_info.count = count;
 523    fetch_info.elts = NULL;
 524
 525    prim_info.linear = FALSE;
 526    prim_info.start = 0;
 527    prim_info.count = draw_count;
 528    prim_info.elts = draw_elts;
 529    prim_info.prim = fpme->input_prim;
 530    prim_info.flags = prim_flags;
 531    prim_info.primitive_count = 1;
 532    prim_info.primitive_lengths = &draw_count;
 533
 534    llvm_pipeline_generic( middle, &fetch_info, &prim_info );
 535
 536    return TRUE;
 537 }
 538
 539
 540
 541 static void llvm_middle_end_finish( struct draw_pt_middle_end *middle )
 542 {
 543    /* nothing to do */
 544 }
 545
 546 static void llvm_middle_end_destroy( struct draw_pt_middle_end *middle )
 547 {
 548    struct llvm_middle_end *fpme = (struct llvm_middle_end *)middle;
 549
 550    if (fpme->fetch)
 551       draw_pt_fetch_destroy( fpme->fetch );
 552
 553    if (fpme->emit)
 554       draw_pt_emit_destroy( fpme->emit );
 555
 556    if (fpme->so_emit)
 557       draw_pt_so_emit_destroy( fpme->so_emit );
 558
 559    if (fpme->post_vs)
 560       draw_pt_post_vs_destroy( fpme->post_vs );
 561
 562    FREE(middle);
 563 }
 564
 565
 566 struct draw_pt_middle_end *
 567 draw_pt_fetch_pipeline_or_emit_llvm(struct draw_context *draw)
 568 {
 569    struct llvm_middle_end *fpme = 0;
 570
 571    if (!draw->llvm)
 572       return NULL;
 573
 574    fpme = CALLOC_STRUCT( llvm_middle_end );
 575    if (!fpme)
 576       goto fail;
 577
 578    fpme->base.prepare         = llvm_middle_end_prepare;
 579    fpme->base.bind_parameters = llvm_middle_end_bind_parameters;
 580    fpme->base.run             = llvm_middle_end_run;
 581    fpme->base.run_linear      = llvm_middle_end_linear_run;
 582    fpme->base.run_linear_elts = llvm_middle_end_linear_run_elts;
 583    fpme->base.finish          = llvm_middle_end_finish;
 584    fpme->base.destroy         = llvm_middle_end_destroy;
 585
 586    fpme->draw = draw;
 587
 588    fpme->fetch = draw_pt_fetch_create( draw );
 589    if (!fpme->fetch)
 590       goto fail;
 591
 592    fpme->post_vs = draw_pt_post_vs_create( draw );
 593    if (!fpme->post_vs)
 594       goto fail;
 595
 596    fpme->emit = draw_pt_emit_create( draw );
 597    if (!fpme->emit)
 598       goto fail;
 599
 600    fpme->so_emit = draw_pt_so_emit_create( draw );
 601    if (!fpme->so_emit)
 602       goto fail;
 603
 604    fpme->llvm = draw->llvm;
 605    if (!fpme->llvm)
 606       goto fail;
 607
 608    fpme->current_variant = NULL;
 609
 610    return &fpme->base;
 611
 612  fail:
 613    if (fpme)
 614       llvm_middle_end_destroy( &fpme->base );
 615
 616    return NULL;
 617 }