src/gallium/auxiliary/draw/draw_llvm.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2010 VMware, Inc.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  19  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  21  * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
  22  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  23  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  24  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27
  28 #include "draw_llvm.h"
  29
  30 #include "draw_context.h"
  31 #include "draw_vs.h"
  32 #include "draw_gs.h"
  33
  34 #include "gallivm/lp_bld_arit.h"
  35 #include "gallivm/lp_bld_arit_overflow.h"
  36 #include "gallivm/lp_bld_logic.h"
  37 #include "gallivm/lp_bld_const.h"
  38 #include "gallivm/lp_bld_swizzle.h"
  39 #include "gallivm/lp_bld_struct.h"
  40 #include "gallivm/lp_bld_type.h"
  41 #include "gallivm/lp_bld_flow.h"
  42 #include "gallivm/lp_bld_debug.h"
  43 #include "gallivm/lp_bld_tgsi.h"
  44 #include "gallivm/lp_bld_printf.h"
  45 #include "gallivm/lp_bld_intr.h"
  46 #include "gallivm/lp_bld_init.h"
  47 #include "gallivm/lp_bld_type.h"
  48 #include "gallivm/lp_bld_pack.h"
  49 #include "gallivm/lp_bld_format.h"
  50
  51 #include "tgsi/tgsi_exec.h"
  52 #include "tgsi/tgsi_dump.h"
  53
  54 #include "util/u_math.h"
  55 #include "util/u_pointer.h"
  56 #include "util/u_string.h"
  57 #include "util/u_simple_list.h"
  58
  59
  60 #define DEBUG_STORE 0
  61
  62
  63 static void
  64 draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *var,
  65                    boolean elts);
  66
  67
  68 struct draw_gs_llvm_iface {
  69    struct lp_build_tgsi_gs_iface base;
  70
  71    struct draw_gs_llvm_variant *variant;
  72    LLVMValueRef input;
  73 };
  74
  75 static INLINE const struct draw_gs_llvm_iface *
  76 draw_gs_llvm_iface(const struct lp_build_tgsi_gs_iface *iface)
  77 {
  78    return (const struct draw_gs_llvm_iface *)iface;
  79 }
  80
  81 /**
  82  * Create LLVM type for draw_vertex_buffer.
  83  */
  84 static LLVMTypeRef
  85 create_jit_dvbuffer_type(struct gallivm_state *gallivm,
  86                          const char *struct_name)
  87 {
  88    LLVMTargetDataRef target = gallivm->target;
  89    LLVMTypeRef dvbuffer_type;
  90    LLVMTypeRef elem_types[DRAW_JIT_DVBUFFER_NUM_FIELDS];
  91    LLVMTypeRef int32_type = LLVMInt32TypeInContext(gallivm->context);
  92
  93    elem_types[DRAW_JIT_DVBUFFER_MAP] =
  94       LLVMPointerType(LLVMIntTypeInContext(gallivm->context, 8), 0);
  95    elem_types[DRAW_JIT_DVBUFFER_SIZE] = int32_type;
  96
  97    dvbuffer_type = LLVMStructTypeInContext(gallivm->context, elem_types,
  98                                            Elements(elem_types), 0);
  99
 100    LP_CHECK_MEMBER_OFFSET(struct draw_vertex_buffer, map,
 101                           target, dvbuffer_type,
 102                           DRAW_JIT_DVBUFFER_MAP);
 103    LP_CHECK_MEMBER_OFFSET(struct draw_vertex_buffer, size,
 104                           target, dvbuffer_type,
 105                           DRAW_JIT_DVBUFFER_SIZE);
 106
 107    return dvbuffer_type;
 108 }
 109
 110 /**
 111  * Create LLVM type for struct draw_jit_texture
 112  */
 113 static LLVMTypeRef
 114 create_jit_texture_type(struct gallivm_state *gallivm, const char *struct_name)
 115 {
 116    LLVMTargetDataRef target = gallivm->target;
 117    LLVMTypeRef texture_type;
 118    LLVMTypeRef elem_types[DRAW_JIT_TEXTURE_NUM_FIELDS];
 119    LLVMTypeRef int32_type = LLVMInt32TypeInContext(gallivm->context);
 120
 121    elem_types[DRAW_JIT_TEXTURE_WIDTH]  =
 122    elem_types[DRAW_JIT_TEXTURE_HEIGHT] =
 123    elem_types[DRAW_JIT_TEXTURE_DEPTH] =
 124    elem_types[DRAW_JIT_TEXTURE_FIRST_LEVEL] =
 125    elem_types[DRAW_JIT_TEXTURE_LAST_LEVEL] = int32_type;
 126    elem_types[DRAW_JIT_TEXTURE_BASE] =
 127       LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0);
 128    elem_types[DRAW_JIT_TEXTURE_ROW_STRIDE] =
 129    elem_types[DRAW_JIT_TEXTURE_IMG_STRIDE] =
 130    elem_types[DRAW_JIT_TEXTURE_MIP_OFFSETS] =
 131       LLVMArrayType(int32_type, PIPE_MAX_TEXTURE_LEVELS);
 132
 133    texture_type = LLVMStructTypeInContext(gallivm->context, elem_types,
 134                                           Elements(elem_types), 0);
 135
 136    LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, width,
 137                           target, texture_type,
 138                           DRAW_JIT_TEXTURE_WIDTH);
 139    LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, height,
 140                           target, texture_type,
 141                           DRAW_JIT_TEXTURE_HEIGHT);
 142    LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, depth,
 143                           target, texture_type,
 144                           DRAW_JIT_TEXTURE_DEPTH);
 145    LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, first_level,
 146                           target, texture_type,
 147                           DRAW_JIT_TEXTURE_FIRST_LEVEL);
 148    LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, last_level,
 149                           target, texture_type,
 150                           DRAW_JIT_TEXTURE_LAST_LEVEL);
 151    LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, base,
 152                           target, texture_type,
 153                           DRAW_JIT_TEXTURE_BASE);
 154    LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, row_stride,
 155                           target, texture_type,
 156                           DRAW_JIT_TEXTURE_ROW_STRIDE);
 157    LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, img_stride,
 158                           target, texture_type,
 159                           DRAW_JIT_TEXTURE_IMG_STRIDE);
 160    LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, mip_offsets,
 161                           target, texture_type,
 162                           DRAW_JIT_TEXTURE_MIP_OFFSETS);
 163
 164    LP_CHECK_STRUCT_SIZE(struct draw_jit_texture, target, texture_type);
 165
 166    return texture_type;
 167 }
 168
 169
 170 /**
 171  * Create LLVM type for struct draw_jit_sampler
 172  */
 173 static LLVMTypeRef
 174 create_jit_sampler_type(struct gallivm_state *gallivm, const char *struct_name)
 175 {
 176    LLVMTargetDataRef target = gallivm->target;
 177    LLVMTypeRef sampler_type;
 178    LLVMTypeRef elem_types[DRAW_JIT_SAMPLER_NUM_FIELDS];
 179
 180    elem_types[DRAW_JIT_SAMPLER_MIN_LOD] =
 181    elem_types[DRAW_JIT_SAMPLER_MAX_LOD] =
 182    elem_types[DRAW_JIT_SAMPLER_LOD_BIAS] = LLVMFloatTypeInContext(gallivm->context);
 183    elem_types[DRAW_JIT_SAMPLER_BORDER_COLOR] =
 184       LLVMArrayType(LLVMFloatTypeInContext(gallivm->context), 4);
 185
 186    sampler_type = LLVMStructTypeInContext(gallivm->context, elem_types,
 187                                           Elements(elem_types), 0);
 188
 189    LP_CHECK_MEMBER_OFFSET(struct draw_jit_sampler, min_lod,
 190                           target, sampler_type,
 191                           DRAW_JIT_SAMPLER_MIN_LOD);
 192    LP_CHECK_MEMBER_OFFSET(struct draw_jit_sampler, max_lod,
 193                           target, sampler_type,
 194                           DRAW_JIT_SAMPLER_MAX_LOD);
 195    LP_CHECK_MEMBER_OFFSET(struct draw_jit_sampler, lod_bias,
 196                           target, sampler_type,
 197                           DRAW_JIT_SAMPLER_LOD_BIAS);
 198    LP_CHECK_MEMBER_OFFSET(struct draw_jit_sampler, border_color,
 199                           target, sampler_type,
 200                           DRAW_JIT_SAMPLER_BORDER_COLOR);
 201
 202    LP_CHECK_STRUCT_SIZE(struct draw_jit_sampler, target, sampler_type);
 203
 204    return sampler_type;
 205 }
 206
 207
 208 /**
 209  * Create LLVM type for struct draw_jit_context
 210  */
 211 static LLVMTypeRef
 212 create_jit_context_type(struct gallivm_state *gallivm,
 213                         LLVMTypeRef texture_type, LLVMTypeRef sampler_type,
 214                         const char *struct_name)
 215 {
 216    LLVMTargetDataRef target = gallivm->target;
 217    LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context);
 218    LLVMTypeRef int_type = LLVMInt32TypeInContext(gallivm->context);
 219    LLVMTypeRef elem_types[DRAW_JIT_CTX_NUM_FIELDS];
 220    LLVMTypeRef context_type;
 221
 222    elem_types[0] = LLVMArrayType(LLVMPointerType(float_type, 0), /* vs_constants */
 223                                  LP_MAX_TGSI_CONST_BUFFERS);
 224    elem_types[1] = LLVMArrayType(int_type, /* num_vs_constants */
 225                                  LP_MAX_TGSI_CONST_BUFFERS);
 226    elem_types[2] = LLVMPointerType(LLVMArrayType(LLVMArrayType(float_type, 4),
 227                                                  DRAW_TOTAL_CLIP_PLANES), 0);
 228    elem_types[3] = LLVMPointerType(float_type, 0); /* viewport */
 229    elem_types[4] = LLVMArrayType(texture_type,
 230                                  PIPE_MAX_SHADER_SAMPLER_VIEWS); /* textures */
 231    elem_types[5] = LLVMArrayType(sampler_type,
 232                                  PIPE_MAX_SAMPLERS); /* samplers */
 233    context_type = LLVMStructTypeInContext(gallivm->context, elem_types,
 234                                           Elements(elem_types), 0);
 235    LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, vs_constants,
 236                           target, context_type, DRAW_JIT_CTX_CONSTANTS);
 237    LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, num_vs_constants,
 238                           target, context_type, DRAW_JIT_CTX_NUM_CONSTANTS);
 239    LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, planes,
 240                           target, context_type, DRAW_JIT_CTX_PLANES);
 241    LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, viewport,
 242                           target, context_type, DRAW_JIT_CTX_VIEWPORT);
 243    LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, textures,
 244                           target, context_type,
 245                           DRAW_JIT_CTX_TEXTURES);
 246    LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, samplers,
 247                           target, context_type,
 248                           DRAW_JIT_CTX_SAMPLERS);
 249    LP_CHECK_STRUCT_SIZE(struct draw_jit_context,
 250                         target, context_type);
 251
 252    return context_type;
 253 }
 254
 255
 256 /**
 257  * Create LLVM type for struct draw_gs_jit_context
 258  */
 259 static LLVMTypeRef
 260 create_gs_jit_context_type(struct gallivm_state *gallivm,
 261                            unsigned vector_length,
 262                            LLVMTypeRef texture_type, LLVMTypeRef sampler_type,
 263                            const char *struct_name)
 264 {
 265    LLVMTargetDataRef target = gallivm->target;
 266    LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context);
 267    LLVMTypeRef int_type = LLVMInt32TypeInContext(gallivm->context);
 268    LLVMTypeRef elem_types[DRAW_GS_JIT_CTX_NUM_FIELDS];
 269    LLVMTypeRef context_type;
 270
 271    elem_types[0] = LLVMArrayType(LLVMPointerType(float_type, 0), /* constants */
 272                                  LP_MAX_TGSI_CONST_BUFFERS);
 273    elem_types[1] = LLVMArrayType(int_type, /* num_constants */
 274                                  LP_MAX_TGSI_CONST_BUFFERS);
 275    elem_types[2] = LLVMPointerType(LLVMArrayType(LLVMArrayType(float_type, 4),
 276                                                  DRAW_TOTAL_CLIP_PLANES), 0);
 277    elem_types[3] = LLVMPointerType(float_type, 0); /* viewport */
 278
 279    elem_types[4] = LLVMArrayType(texture_type,
 280                                  PIPE_MAX_SHADER_SAMPLER_VIEWS); /* textures */
 281    elem_types[5] = LLVMArrayType(sampler_type,
 282                                  PIPE_MAX_SAMPLERS); /* samplers */
 283
 284    elem_types[6] = LLVMPointerType(LLVMPointerType(int_type, 0), 0);
 285    elem_types[7] = LLVMPointerType(LLVMVectorType(int_type,
 286                                                   vector_length), 0);
 287    elem_types[8] = LLVMPointerType(LLVMVectorType(int_type,
 288                                                   vector_length), 0);
 289
 290    context_type = LLVMStructTypeInContext(gallivm->context, elem_types,
 291                                           Elements(elem_types), 0);
 292
 293    LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, constants,
 294                           target, context_type, DRAW_GS_JIT_CTX_CONSTANTS);
 295    LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, num_constants,
 296                           target, context_type, DRAW_GS_JIT_CTX_NUM_CONSTANTS);
 297    LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, planes,
 298                           target, context_type, DRAW_GS_JIT_CTX_PLANES);
 299    LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, viewport,
 300                           target, context_type, DRAW_GS_JIT_CTX_VIEWPORT);
 301    LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, textures,
 302                           target, context_type,
 303                           DRAW_GS_JIT_CTX_TEXTURES);
 304    LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, samplers,
 305                           target, context_type,
 306                           DRAW_GS_JIT_CTX_SAMPLERS);
 307    LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, prim_lengths,
 308                           target, context_type,
 309                           DRAW_GS_JIT_CTX_PRIM_LENGTHS);
 310    LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, emitted_vertices,
 311                           target, context_type,
 312                           DRAW_GS_JIT_CTX_EMITTED_VERTICES);
 313    LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, emitted_prims,
 314                           target, context_type,
 315                           DRAW_GS_JIT_CTX_EMITTED_PRIMS);
 316    LP_CHECK_STRUCT_SIZE(struct draw_gs_jit_context,
 317                         target, context_type);
 318
 319    return context_type;
 320 }
 321
 322
 323 static LLVMTypeRef
 324 create_gs_jit_input_type(struct gallivm_state *gallivm)
 325 {
 326    LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context);
 327    LLVMTypeRef input_array;
 328
 329    input_array = LLVMVectorType(float_type, TGSI_NUM_CHANNELS); /* num primitives */
 330    input_array = LLVMArrayType(input_array, TGSI_NUM_CHANNELS); /* num channels */
 331    input_array = LLVMArrayType(input_array, PIPE_MAX_SHADER_INPUTS); /* num attrs per vertex */
 332    input_array = LLVMPointerType(input_array, 0); /* num vertices per prim */
 333
 334    return input_array;
 335 }
 336
 337 /**
 338  * Create LLVM type for struct pipe_vertex_buffer
 339  */
 340 static LLVMTypeRef
 341 create_jit_vertex_buffer_type(struct gallivm_state *gallivm,
 342                               const char *struct_name)
 343 {
 344    LLVMTargetDataRef target = gallivm->target;
 345    LLVMTypeRef elem_types[4];
 346    LLVMTypeRef vb_type;
 347
 348    elem_types[0] =
 349    elem_types[1] = LLVMInt32TypeInContext(gallivm->context);
 350    elem_types[2] =
 351    elem_types[3] = LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0);
 352
 353    vb_type = LLVMStructTypeInContext(gallivm->context, elem_types,
 354                                      Elements(elem_types), 0);
 355
 356    LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, stride,
 357                           target, vb_type, 0);
 358    LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, buffer_offset,
 359                           target, vb_type, 1);
 360
 361    LP_CHECK_STRUCT_SIZE(struct pipe_vertex_buffer, target, vb_type);
 362
 363    return vb_type;
 364 }
 365
 366
 367 /**
 368  * Create LLVM type for struct vertex_header;
 369  */
 370 static LLVMTypeRef
 371 create_jit_vertex_header(struct gallivm_state *gallivm, int data_elems)
 372 {
 373    LLVMTargetDataRef target = gallivm->target;
 374    LLVMTypeRef elem_types[4];
 375    LLVMTypeRef vertex_header;
 376    char struct_name[24];
 377
 378    util_snprintf(struct_name, 23, "vertex_header%d", data_elems);
 379
 380    elem_types[DRAW_JIT_VERTEX_VERTEX_ID]  = LLVMIntTypeInContext(gallivm->context, 32);
 381    elem_types[DRAW_JIT_VERTEX_CLIP]  = LLVMArrayType(LLVMFloatTypeInContext(gallivm->context), 4);
 382    elem_types[DRAW_JIT_VERTEX_PRE_CLIP_POS]  = LLVMArrayType(LLVMFloatTypeInContext(gallivm->context), 4);
 383    elem_types[DRAW_JIT_VERTEX_DATA]  = LLVMArrayType(elem_types[1], data_elems);
 384
 385    vertex_header = LLVMStructTypeInContext(gallivm->context, elem_types,
 386                                            Elements(elem_types), 0);
 387
 388    /* these are bit-fields and we can't take address of them
 389       LP_CHECK_MEMBER_OFFSET(struct vertex_header, clipmask,
 390       target, vertex_header,
 391       DRAW_JIT_VERTEX_CLIPMASK);
 392       LP_CHECK_MEMBER_OFFSET(struct vertex_header, edgeflag,
 393       target, vertex_header,
 394       DRAW_JIT_VERTEX_EDGEFLAG);
 395       LP_CHECK_MEMBER_OFFSET(struct vertex_header, pad,
 396       target, vertex_header,
 397       DRAW_JIT_VERTEX_PAD);
 398       LP_CHECK_MEMBER_OFFSET(struct vertex_header, vertex_id,
 399       target, vertex_header,
 400       DRAW_JIT_VERTEX_VERTEX_ID);
 401    */
 402    LP_CHECK_MEMBER_OFFSET(struct vertex_header, clip,
 403                           target, vertex_header,
 404                           DRAW_JIT_VERTEX_CLIP);
 405    LP_CHECK_MEMBER_OFFSET(struct vertex_header, pre_clip_pos,
 406                           target, vertex_header,
 407                           DRAW_JIT_VERTEX_PRE_CLIP_POS);
 408    LP_CHECK_MEMBER_OFFSET(struct vertex_header, data,
 409                           target, vertex_header,
 410                           DRAW_JIT_VERTEX_DATA);
 411
 412    assert(LLVMABISizeOfType(target, vertex_header) ==
 413           offsetof(struct vertex_header, data[data_elems]));
 414
 415    return vertex_header;
 416 }
 417
 418
 419 /**
 420  * Create LLVM types for various structures.
 421  */
 422 static void
 423 create_jit_types(struct draw_llvm_variant *variant)
 424 {
 425    struct gallivm_state *gallivm = variant->gallivm;
 426    LLVMTypeRef texture_type, sampler_type, context_type, buffer_type,
 427       vb_type;
 428
 429    texture_type = create_jit_texture_type(gallivm, "texture");
 430    sampler_type = create_jit_sampler_type(gallivm, "sampler");
 431
 432    context_type = create_jit_context_type(gallivm, texture_type, sampler_type,
 433                                           "draw_jit_context");
 434    variant->context_ptr_type = LLVMPointerType(context_type, 0);
 435
 436    buffer_type = create_jit_dvbuffer_type(gallivm, "draw_vertex_buffer");
 437    variant->buffer_ptr_type = LLVMPointerType(buffer_type, 0);
 438
 439    vb_type = create_jit_vertex_buffer_type(gallivm, "pipe_vertex_buffer");
 440    variant->vb_ptr_type = LLVMPointerType(vb_type, 0);
 441 }
 442
 443
 444 static LLVMTypeRef
 445 get_context_ptr_type(struct draw_llvm_variant *variant)
 446 {
 447    if (!variant->context_ptr_type)
 448       create_jit_types(variant);
 449    return variant->context_ptr_type;
 450 }
 451
 452
 453 static LLVMTypeRef
 454 get_buffer_ptr_type(struct draw_llvm_variant *variant)
 455 {
 456    if (!variant->buffer_ptr_type)
 457       create_jit_types(variant);
 458    return variant->buffer_ptr_type;
 459 }
 460
 461
 462 static LLVMTypeRef
 463 get_vb_ptr_type(struct draw_llvm_variant *variant)
 464 {
 465    if (!variant->vb_ptr_type)
 466       create_jit_types(variant);
 467    return variant->vb_ptr_type;
 468 }
 469
 470 static LLVMTypeRef
 471 get_vertex_header_ptr_type(struct draw_llvm_variant *variant)
 472 {
 473    if (!variant->vertex_header_ptr_type)
 474       create_jit_types(variant);
 475    return variant->vertex_header_ptr_type;
 476 }
 477
 478
 479 /**
 480  * Create per-context LLVM info.
 481  */
 482 struct draw_llvm *
 483 draw_llvm_create(struct draw_context *draw)
 484 {
 485    struct draw_llvm *llvm;
 486
 487    llvm = CALLOC_STRUCT( draw_llvm );
 488    if (!llvm)
 489       return NULL;
 490
 491    lp_build_init();
 492
 493    llvm->draw = draw;
 494
 495    llvm->nr_variants = 0;
 496    make_empty_list(&llvm->vs_variants_list);
 497
 498    llvm->nr_gs_variants = 0;
 499    make_empty_list(&llvm->gs_variants_list);
 500
 501    return llvm;
 502 }
 503
 504
 505 /**
 506  * Free per-context LLVM info.
 507  */
 508 void
 509 draw_llvm_destroy(struct draw_llvm *llvm)
 510 {
 511    /* XXX free other draw_llvm data? */
 512    FREE(llvm);
 513 }
 514
 515
 516 /**
 517  * Create LLVM-generated code for a vertex shader.
 518  */
 519 struct draw_llvm_variant *
 520 draw_llvm_create_variant(struct draw_llvm *llvm,
 521                          unsigned num_inputs,
 522                          const struct draw_llvm_variant_key *key)
 523 {
 524    struct draw_llvm_variant *variant;
 525    struct llvm_vertex_shader *shader =
 526       llvm_vertex_shader(llvm->draw->vs.vertex_shader);
 527    LLVMTypeRef vertex_header;
 528    char module_name[64];
 529
 530    variant = MALLOC(sizeof *variant +
 531                     shader->variant_key_size -
 532                     sizeof variant->key);
 533    if (variant == NULL)
 534       return NULL;
 535
 536    variant->llvm = llvm;
 537    variant->shader = shader;
 538
 539    util_snprintf(module_name, sizeof(module_name), "draw_llvm_vs_variant%u",
 540                  variant->shader->variants_cached);
 541
 542    variant->gallivm = gallivm_create(module_name);
 543
 544    create_jit_types(variant);
 545
 546    memcpy(&variant->key, key, shader->variant_key_size);
 547
 548    vertex_header = create_jit_vertex_header(variant->gallivm, num_inputs);
 549
 550    variant->vertex_header_ptr_type = LLVMPointerType(vertex_header, 0);
 551
 552    draw_llvm_generate(llvm, variant, FALSE);  /* linear */
 553    draw_llvm_generate(llvm, variant, TRUE);   /* elts */
 554
 555    gallivm_compile_module(variant->gallivm);
 556
 557    variant->jit_func = (draw_jit_vert_func)
 558          gallivm_jit_function(variant->gallivm, variant->function);
 559
 560    variant->jit_func_elts = (draw_jit_vert_func_elts)
 561          gallivm_jit_function(variant->gallivm, variant->function_elts);
 562
 563    gallivm_free_ir(variant->gallivm);
 564
 565    variant->list_item_global.base = variant;
 566    variant->list_item_local.base = variant;
 567    /*variant->no = */shader->variants_created++;
 568    variant->list_item_global.base = variant;
 569
 570    return variant;
 571 }
 572
 573
 574 static void
 575 generate_vs(struct draw_llvm_variant *variant,
 576             LLVMBuilderRef builder,
 577             struct lp_type vs_type,
 578             LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS],
 579             const LLVMValueRef (*inputs)[TGSI_NUM_CHANNELS],
 580             const struct lp_bld_tgsi_system_values *system_values,
 581             LLVMValueRef context_ptr,
 582             struct lp_build_sampler_soa *draw_sampler,
 583             boolean clamp_vertex_color)
 584 {
 585    struct draw_llvm *llvm = variant->llvm;
 586    const struct tgsi_token *tokens = llvm->draw->vs.vertex_shader->state.tokens;
 587    LLVMValueRef consts_ptr =
 588       draw_jit_context_vs_constants(variant->gallivm, context_ptr);
 589    LLVMValueRef num_consts_ptr =
 590       draw_jit_context_num_vs_constants(variant->gallivm, context_ptr);
 591    struct lp_build_sampler_soa *sampler = 0;
 592
 593    if (gallivm_debug & (GALLIVM_DEBUG_TGSI | GALLIVM_DEBUG_IR)) {
 594       tgsi_dump(tokens, 0);
 595       draw_llvm_dump_variant_key(&variant->key);
 596    }
 597
 598    if (llvm->draw->num_sampler_views && llvm->draw->num_samplers)
 599       sampler = draw_sampler;
 600
 601    lp_build_tgsi_soa(variant->gallivm,
 602                      tokens,
 603                      vs_type,
 604                      NULL /*struct lp_build_mask_context *mask*/,
 605                      consts_ptr,
 606                      num_consts_ptr,
 607                      system_values,
 608                      inputs,
 609                      outputs,
 610                      sampler,
 611                      &llvm->draw->vs.vertex_shader->info,
 612                      NULL);
 613
 614    {
 615       LLVMValueRef out;
 616       unsigned chan, attrib;
 617       struct lp_build_context bld;
 618       struct tgsi_shader_info* info = &llvm->draw->vs.vertex_shader->info;
 619       lp_build_context_init(&bld, variant->gallivm, vs_type);
 620
 621       for (attrib = 0; attrib < info->num_outputs; ++attrib) {
 622          for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
 623             if (outputs[attrib][chan]) {
 624                switch (info->output_semantic_name[attrib]) {
 625                case TGSI_SEMANTIC_COLOR:
 626                case TGSI_SEMANTIC_BCOLOR:
 627                   if (clamp_vertex_color) {
 628                      out = LLVMBuildLoad(builder, outputs[attrib][chan], "");
 629                      out = lp_build_clamp(&bld, out, bld.zero, bld.one);
 630                      LLVMBuildStore(builder, out, outputs[attrib][chan]);
 631                   }
 632                   break;
 633                }
 634             }
 635          }
 636       }
 637    }
 638 }
 639
 640 static void
 641 generate_fetch(struct gallivm_state *gallivm,
 642                struct draw_context *draw,
 643                LLVMValueRef vbuffers_ptr,
 644                LLVMValueRef *res,
 645                struct pipe_vertex_element *velem,
 646                LLVMValueRef vbuf,
 647                LLVMValueRef index,
 648                LLVMValueRef instance_id)
 649 {
 650    const struct util_format_description *format_desc =
 651       util_format_description(velem->src_format);
 652    LLVMValueRef zero = LLVMConstNull(LLVMInt32TypeInContext(gallivm->context));
 653    LLVMBuilderRef builder = gallivm->builder;
 654    LLVMValueRef indices =
 655       LLVMConstInt(LLVMInt64TypeInContext(gallivm->context),
 656                    velem->vertex_buffer_index, 0);
 657    LLVMValueRef vbuffer_ptr = LLVMBuildGEP(builder, vbuffers_ptr,
 658                                            &indices, 1, "");
 659    LLVMValueRef vb_stride = draw_jit_vbuffer_stride(gallivm, vbuf);
 660    LLVMValueRef vb_buffer_offset = draw_jit_vbuffer_offset(gallivm, vbuf);
 661    LLVMValueRef map_ptr = draw_jit_dvbuffer_map(gallivm, vbuffer_ptr);
 662    LLVMValueRef buffer_size = draw_jit_dvbuffer_size(gallivm, vbuffer_ptr);
 663    LLVMValueRef stride;
 664    LLVMValueRef buffer_overflowed;
 665    LLVMValueRef needed_buffer_size;
 666    LLVMValueRef temp_ptr =
 667       lp_build_alloca(gallivm,
 668                       lp_build_vec_type(gallivm, lp_float32_vec4_type()), "");
 669    LLVMValueRef ofbit = NULL;
 670    struct lp_build_if_state if_ctx;
 671
 672    if (velem->instance_divisor) {
 673       /* Index is equal to the start instance plus the number of current
 674        * instance divided by the divisor. In this case we compute it as:
 675        * index = start_instance + (instance_id  / divisor)
 676        */
 677       LLVMValueRef current_instance;
 678       index = lp_build_const_int32(gallivm, draw->start_instance);
 679       current_instance = LLVMBuildUDiv(builder, instance_id,
 680                                        lp_build_const_int32(gallivm, velem->instance_divisor),
 681                                        "instance_divisor");
 682       index = lp_build_uadd_overflow(gallivm, index, current_instance, &ofbit);
 683    }
 684
 685    stride = lp_build_umul_overflow(gallivm, vb_stride, index, &ofbit);
 686    stride = lp_build_uadd_overflow(gallivm, stride, vb_buffer_offset, &ofbit);
 687    stride = lp_build_uadd_overflow(
 688       gallivm, stride,
 689       lp_build_const_int32(gallivm, velem->src_offset), &ofbit);
 690    needed_buffer_size = lp_build_uadd_overflow(
 691       gallivm, stride,
 692       lp_build_const_int32(gallivm,
 693                            util_format_get_blocksize(velem->src_format)),
 694       &ofbit);
 695
 696    buffer_overflowed = LLVMBuildICmp(builder, LLVMIntUGT,
 697                                      needed_buffer_size, buffer_size,
 698                                      "buffer_overflowed");
 699    buffer_overflowed = LLVMBuildOr(builder, buffer_overflowed, ofbit, "");
 700 #if 0
 701    lp_build_printf(gallivm, "vbuf index = %u, vb_stride is %u\n",
 702                    index, vb_stride);
 703    lp_build_printf(gallivm, "   vb_buffer_offset = %u, src_offset is %u\n",
 704                    vb_buffer_offset,
 705                    lp_build_const_int32(gallivm, velem->src_offset));
 706    lp_build_print_value(gallivm, "   blocksize = ",
 707                         lp_build_const_int32(
 708                            gallivm,
 709                            util_format_get_blocksize(velem->src_format)));
 710    lp_build_printf(gallivm, "   instance_id = %u\n", instance_id);
 711    lp_build_printf(gallivm, "   stride = %u\n", stride);
 712    lp_build_printf(gallivm, "   buffer size = %u\n", buffer_size);
 713    lp_build_printf(gallivm, "   needed_buffer_size = %u\n", needed_buffer_size);
 714    lp_build_print_value(gallivm, "   buffer overflowed = ", buffer_overflowed);
 715 #endif
 716
 717    lp_build_if(&if_ctx, gallivm, buffer_overflowed);
 718    {
 719       LLVMValueRef val =
 720          lp_build_const_vec(gallivm, lp_float32_vec4_type(), 0);
 721       LLVMBuildStore(builder, val, temp_ptr);
 722    }
 723    lp_build_else(&if_ctx);
 724    {
 725       LLVMValueRef val;
 726       map_ptr = LLVMBuildGEP(builder, map_ptr, &stride, 1, "");
 727
 728       val = lp_build_fetch_rgba_aos(gallivm,
 729                                     format_desc,
 730                                     lp_float32_vec4_type(),
 731                                     map_ptr,
 732                                     zero, zero, zero);
 733       LLVMBuildStore(builder, val, temp_ptr);
 734    }
 735    lp_build_endif(&if_ctx);
 736
 737    *res = LLVMBuildLoad(builder, temp_ptr, "aos");
 738 }
 739
 740 static void
 741 convert_to_soa(struct gallivm_state *gallivm,
 742                LLVMValueRef (*src_aos)[LP_MAX_VECTOR_WIDTH / 32],
 743                LLVMValueRef (*dst_soa)[TGSI_NUM_CHANNELS],
 744                unsigned num_attribs, const struct lp_type soa_type)
 745 {
 746    unsigned i, j, k;
 747    struct lp_type aos_channel_type = soa_type;
 748
 749    debug_assert(TGSI_NUM_CHANNELS == 4);
 750    debug_assert((soa_type.length % TGSI_NUM_CHANNELS) == 0);
 751
 752    aos_channel_type.length >>= 1;
 753
 754    for (i = 0; i < num_attribs; ++i) {
 755       LLVMValueRef aos_channels[TGSI_NUM_CHANNELS];
 756       unsigned pixels_per_channel = soa_type.length / TGSI_NUM_CHANNELS;
 757
 758       for (j = 0; j < TGSI_NUM_CHANNELS; ++j) {
 759          LLVMValueRef channel[LP_MAX_VECTOR_LENGTH] = { 0 };
 760
 761          assert(pixels_per_channel <= LP_MAX_VECTOR_LENGTH);
 762
 763          for (k = 0; k < pixels_per_channel; ++k) {
 764             channel[k] = src_aos[i][j + TGSI_NUM_CHANNELS * k];
 765          }
 766
 767          aos_channels[j] = lp_build_concat(gallivm, channel, aos_channel_type, pixels_per_channel);
 768       }
 769
 770       lp_build_transpose_aos(gallivm, soa_type, aos_channels, dst_soa[i]);
 771    }
 772 }
 773
 774
 775 static void
 776 store_aos(struct gallivm_state *gallivm,
 777           LLVMValueRef io_ptr,
 778           LLVMValueRef index,
 779           LLVMValueRef value)
 780 {
 781    LLVMTypeRef data_ptr_type = LLVMPointerType(lp_build_vec_type(gallivm, lp_float32_vec4_type()), 0);
 782    LLVMBuilderRef builder = gallivm->builder;
 783    LLVMValueRef data_ptr = draw_jit_header_data(gallivm, io_ptr);
 784    LLVMValueRef indices[3];
 785
 786    indices[0] = lp_build_const_int32(gallivm, 0);
 787    indices[1] = index;
 788    indices[2] = lp_build_const_int32(gallivm, 0);
 789
 790    data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 3, "");
 791    data_ptr = LLVMBuildPointerCast(builder, data_ptr, data_ptr_type, "");
 792
 793 #if DEBUG_STORE
 794    lp_build_printf(gallivm, "    ---- %p storing attribute %d (io = %p)\n", data_ptr, index, io_ptr);
 795 #endif
 796
 797    /* Unaligned store due to the vertex header */
 798    lp_set_store_alignment(LLVMBuildStore(builder, value, data_ptr), sizeof(float));
 799 }
 800
 801 /**
 802  * Adjust the mask to architecture endianess. The mask will the store in struct:
 803  *
 804  * struct vertex_header {
 805  *    unsigned clipmask:DRAW_TOTAL_CLIP_PLANES;
 806  *    unsigned edgeflag:1;
 807  *    unsigned have_clipdist:1;
 808  *    unsigned vertex_id:16;
 809  *    [...]
 810  * }
 811  *
 812  * On little-endian machine nothing needs to done, however on bit-endian machine
 813  * the mask's fields need to be adjusted with the algorithm:
 814  *
 815  * uint32_t reverse (uint32_t x)
 816  * {
 817  *   return (x >> 16) |              // vertex_id
 818  *          ((x & 0x3fff) << 18) |   // clipmask
 819  *          ((x & 0x4000) << 3) |    // have_clipdist
 820  *          ((x & 0x8000) << 1);     // edgeflag
 821  * }
 822  */
 823 static LLVMValueRef
 824 adjust_mask(struct gallivm_state *gallivm,
 825             LLVMValueRef mask)
 826 {
 827 #ifdef PIPE_ARCH_BIG_ENDIAN
 828    LLVMBuilderRef builder = gallivm->builder;
 829    LLVMValueRef vertex_id;
 830    LLVMValueRef clipmask;
 831    LLVMValueRef have_clipdist;
 832    LLVMValueRef edgeflag;
 833
 834    vertex_id = LLVMBuildLShr(builder, mask, lp_build_const_int32(gallivm, 16), "");
 835    clipmask  = LLVMBuildAnd(builder, mask, lp_build_const_int32(gallivm, 0x3fff), "");
 836    clipmask  = LLVMBuildShl(builder, clipmask, lp_build_const_int32(gallivm, 18), "");
 837    have_clipdist = LLVMBuildAnd(builder, mask, lp_build_const_int32(gallivm, 0x4000), "");
 838    have_clipdist = LLVMBuildShl(builder, have_clipdist, lp_build_const_int32(gallivm, 3), "");
 839    edgeflag = LLVMBuildAnd(builder, mask, lp_build_const_int32(gallivm, 0x8000), "");
 840    edgeflag = LLVMBuildShl(builder, edgeflag, lp_build_const_int32(gallivm, 1), "");
 841
 842    mask = LLVMBuildOr(builder, vertex_id, clipmask, "");
 843    mask = LLVMBuildOr(builder, mask, have_clipdist, "");
 844    mask = LLVMBuildOr(builder, mask, edgeflag, "");
 845 #endif
 846    return mask;
 847 }
 848
 849 static void
 850 store_aos_array(struct gallivm_state *gallivm,
 851                 struct lp_type soa_type,
 852                 LLVMValueRef io_ptr,
 853                 LLVMValueRef *indices,
 854                 LLVMValueRef* aos,
 855                 int attrib,
 856                 int num_outputs,
 857                 LLVMValueRef clipmask,
 858                 boolean have_clipdist)
 859 {
 860    LLVMBuilderRef builder = gallivm->builder;
 861    LLVMValueRef attr_index = lp_build_const_int32(gallivm, attrib);
 862    LLVMValueRef inds[LP_MAX_VECTOR_WIDTH / 32];
 863    LLVMValueRef linear_inds[LP_MAX_VECTOR_WIDTH / 32];
 864    LLVMValueRef io_ptrs[LP_MAX_VECTOR_WIDTH / 32];
 865    int vector_length = soa_type.length;
 866    int i;
 867
 868    debug_assert(TGSI_NUM_CHANNELS == 4);
 869
 870    for (i = 0; i < vector_length; i++) {
 871       linear_inds[i] = lp_build_const_int32(gallivm, i);
 872       if (indices) {
 873          inds[i] = indices[i];
 874       } else {
 875          inds[i] = linear_inds[i];
 876       }
 877       io_ptrs[i] = LLVMBuildGEP(builder, io_ptr, &inds[i], 1, "");
 878    }
 879
 880    if (attrib == 0) {
 881       /* store vertex header for each of the n vertices */
 882       LLVMValueRef val, cliptmp;
 883       int vertex_id_pad_edgeflag;
 884
 885       /* If this assertion fails, it means we need to update the bit twidding
 886        * code here.  See struct vertex_header in draw_private.h.
 887        */
 888       assert(DRAW_TOTAL_CLIP_PLANES==14);
 889       /* initialize vertex id:16 = 0xffff, have_clipdist:1 = 0, edgeflag:1 = 1 */
 890       vertex_id_pad_edgeflag = (0xffff << 16) | (1 << DRAW_TOTAL_CLIP_PLANES);
 891       if (have_clipdist)
 892          vertex_id_pad_edgeflag |= 1 << (DRAW_TOTAL_CLIP_PLANES+1);
 893       val = lp_build_const_int_vec(gallivm, lp_int_type(soa_type), vertex_id_pad_edgeflag);
 894       /* OR with the clipmask */
 895       cliptmp = LLVMBuildOr(builder, val, clipmask, "");
 896       for (i = 0; i < vector_length; i++) {
 897          LLVMValueRef id_ptr = draw_jit_header_id(gallivm, io_ptrs[i]);
 898          val = LLVMBuildExtractElement(builder, cliptmp, linear_inds[i], "");
 899          val = adjust_mask(gallivm, val);
 900 #if DEBUG_STORE
 901          lp_build_printf(gallivm, "io = %p, index %d, clipmask = %x\n",
 902                          io_ptrs[i], inds[i], val);
 903 #endif
 904          LLVMBuildStore(builder, val, id_ptr);
 905       }
 906    }
 907
 908    /* store for each of the n vertices */
 909    for (i = 0; i < vector_length; i++) {
 910       store_aos(gallivm, io_ptrs[i], attr_index, aos[i]);
 911    }
 912 }
 913
 914
 915 static void
 916 convert_to_aos(struct gallivm_state *gallivm,
 917                LLVMValueRef io,
 918                LLVMValueRef *indices,
 919                LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS],
 920                LLVMValueRef clipmask,
 921                int num_outputs,
 922                struct lp_type soa_type,
 923                boolean have_clipdist)
 924 {
 925    LLVMBuilderRef builder = gallivm->builder;
 926    unsigned chan, attrib, i;
 927
 928 #if DEBUG_STORE
 929    lp_build_printf(gallivm, "   # storing begin\n");
 930 #endif
 931    for (attrib = 0; attrib < num_outputs; ++attrib) {
 932       LLVMValueRef soa[TGSI_NUM_CHANNELS];
 933       LLVMValueRef aos[LP_MAX_VECTOR_WIDTH / 32];
 934       for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
 935          if (outputs[attrib][chan]) {
 936             LLVMValueRef out = LLVMBuildLoad(builder, outputs[attrib][chan], "");
 937             lp_build_name(out, "output%u.%c", attrib, "xyzw"[chan]);
 938 #if DEBUG_STORE
 939             lp_build_printf(gallivm, "output %d : %d ",
 940                             LLVMConstInt(LLVMInt32TypeInContext(gallivm->context),
 941                                          attrib, 0),
 942                             LLVMConstInt(LLVMInt32TypeInContext(gallivm->context),
 943                                          chan, 0));
 944             lp_build_print_value(gallivm, "val = ", out);
 945             {
 946                LLVMValueRef iv =
 947                   LLVMBuildBitCast(builder, out, lp_build_int_vec_type(gallivm, soa_type), "");
 948
 949                lp_build_print_value(gallivm, "  ival = ", iv);
 950             }
 951 #endif
 952             soa[chan] = out;
 953          }
 954          else {
 955             soa[chan] = 0;
 956          }
 957       }
 958
 959
 960       if (soa_type.length == TGSI_NUM_CHANNELS) {
 961          lp_build_transpose_aos(gallivm, soa_type, soa, aos);
 962       } else {
 963          lp_build_transpose_aos(gallivm, soa_type, soa, soa);
 964
 965          for (i = 0; i < soa_type.length; ++i) {
 966             aos[i] = lp_build_extract_range(gallivm,
 967                                             soa[i % TGSI_NUM_CHANNELS],
 968                                             (i / TGSI_NUM_CHANNELS) * TGSI_NUM_CHANNELS,
 969                                             TGSI_NUM_CHANNELS);
 970          }
 971       }
 972
 973       store_aos_array(gallivm,
 974                       soa_type,
 975                       io, indices,
 976                       aos,
 977                       attrib,
 978                       num_outputs,
 979                       clipmask, have_clipdist);
 980    }
 981 #if DEBUG_STORE
 982    lp_build_printf(gallivm, "   # storing end\n");
 983 #endif
 984 }
 985
 986
 987 /**
 988  * Stores original vertex positions in clip coordinates
 989  */
 990 static void
 991 store_clip(struct gallivm_state *gallivm,
 992            const struct lp_type vs_type,
 993            LLVMValueRef io_ptr,
 994            LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS],
 995            boolean pre_clip_pos, int idx)
 996 {
 997    LLVMBuilderRef builder = gallivm->builder;
 998    LLVMValueRef soa[4];
 999    LLVMValueRef aos[LP_MAX_VECTOR_LENGTH];
1000    LLVMValueRef indices[2];
1001    LLVMValueRef io_ptrs[LP_MAX_VECTOR_WIDTH / 32];
1002    LLVMValueRef inds[LP_MAX_VECTOR_WIDTH / 32];
1003    LLVMValueRef clip_ptrs[LP_MAX_VECTOR_WIDTH / 32];
1004    LLVMTypeRef clip_ptr_type =
1005       LLVMPointerType(LLVMVectorType(LLVMFloatTypeInContext(gallivm->context),
1006                                      4), 0);
1007    int i, j;
1008
1009    indices[0] =
1010    indices[1] = lp_build_const_int32(gallivm, 0);
1011
1012    for (i = 0; i < vs_type.length; i++) {
1013       inds[i] = lp_build_const_int32(gallivm, i);
1014       io_ptrs[i] = LLVMBuildGEP(builder, io_ptr, &inds[i], 1, "");
1015    }
1016
1017    soa[0] = LLVMBuildLoad(builder, outputs[idx][0], ""); /*x0 x1 .. xn*/
1018    soa[1] = LLVMBuildLoad(builder, outputs[idx][1], ""); /*y0 y1 .. yn*/
1019    soa[2] = LLVMBuildLoad(builder, outputs[idx][2], ""); /*z0 z1 .. zn*/
1020    soa[3] = LLVMBuildLoad(builder, outputs[idx][3], ""); /*w0 w1 .. wn*/
1021
1022    if (!pre_clip_pos) {
1023       for (i = 0; i < vs_type.length; i++) {
1024          clip_ptrs[i] = draw_jit_header_clip(gallivm, io_ptrs[i]);
1025       }
1026    } else {
1027       for (i = 0; i < vs_type.length; i++) {
1028          clip_ptrs[i] = draw_jit_header_pre_clip_pos(gallivm, io_ptrs[i]);
1029       }
1030    }
1031
1032    lp_build_transpose_aos(gallivm, vs_type, soa, soa);
1033    for (i = 0; i < vs_type.length; ++i) {
1034       aos[i] = lp_build_extract_range(gallivm,
1035                                       soa[i % TGSI_NUM_CHANNELS],
1036                                       (i / TGSI_NUM_CHANNELS) * TGSI_NUM_CHANNELS,
1037                                       TGSI_NUM_CHANNELS);
1038    }
1039
1040    for (j = 0; j < vs_type.length; j++) {
1041       LLVMValueRef clip_ptr;
1042
1043       clip_ptr = LLVMBuildGEP(builder, clip_ptrs[j], indices, 2, "clipo");
1044       clip_ptr = LLVMBuildPointerCast(builder, clip_ptr, clip_ptr_type, "");
1045
1046       /* Unaligned store */
1047       lp_set_store_alignment(LLVMBuildStore(builder, aos[j], clip_ptr), sizeof(float));
1048    }
1049 }
1050
1051
1052 /**
1053  * Transforms the outputs for viewport mapping
1054  */
1055 static void
1056 generate_viewport(struct draw_llvm_variant *variant,
1057                   LLVMBuilderRef builder,
1058                   struct lp_type vs_type,
1059                   LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS],
1060                   LLVMValueRef context_ptr)
1061 {
1062    int i;
1063    struct gallivm_state *gallivm = variant->gallivm;
1064    struct lp_type f32_type = vs_type;
1065    const unsigned pos = variant->llvm->draw->vs.position_output;
1066    LLVMTypeRef vs_type_llvm = lp_build_vec_type(gallivm, vs_type);
1067    LLVMValueRef out3 = LLVMBuildLoad(builder, outputs[pos][3], ""); /*w0 w1 .. wn*/
1068    LLVMValueRef const1 = lp_build_const_vec(gallivm, f32_type, 1.0);       /*1.0 1.0 1.0 1.0*/
1069    LLVMValueRef vp_ptr = draw_jit_context_viewport(gallivm, context_ptr);
1070
1071    /* for 1/w convention*/
1072    out3 = LLVMBuildFDiv(builder, const1, out3, "");
1073    LLVMBuildStore(builder, out3, outputs[pos][3]);
1074
1075    /* Viewport Mapping */
1076    for (i=0; i<3; i++) {
1077       LLVMValueRef out = LLVMBuildLoad(builder, outputs[pos][i], ""); /*x0 x1 .. xn*/
1078       LLVMValueRef scale;
1079       LLVMValueRef trans;
1080       LLVMValueRef scale_i;
1081       LLVMValueRef trans_i;
1082       LLVMValueRef index;
1083
1084       index = lp_build_const_int32(gallivm, i);
1085       scale_i = LLVMBuildGEP(builder, vp_ptr, &index, 1, "");
1086
1087       index = lp_build_const_int32(gallivm, i+4);
1088       trans_i = LLVMBuildGEP(builder, vp_ptr, &index, 1, "");
1089
1090       scale = lp_build_broadcast(gallivm, vs_type_llvm,
1091                                  LLVMBuildLoad(builder, scale_i, "scale"));
1092       trans = lp_build_broadcast(gallivm, vs_type_llvm,
1093                                  LLVMBuildLoad(builder, trans_i, "trans"));
1094
1095       /* divide by w */
1096       out = LLVMBuildFMul(builder, out, out3, "");
1097       /* mult by scale */
1098       out = LLVMBuildFMul(builder, out, scale, "");
1099       /* add translation */
1100       out = LLVMBuildFAdd(builder, out, trans, "");
1101
1102       /* store transformed outputs */
1103       LLVMBuildStore(builder, out, outputs[pos][i]);
1104    }
1105
1106 }
1107
1108
1109 /**
1110  * Returns clipmask as nxi32 bitmask for the n vertices
1111  */
1112 static LLVMValueRef
1113 generate_clipmask(struct draw_llvm *llvm,
1114                   struct gallivm_state *gallivm,
1115                   struct lp_type vs_type,
1116                   LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS],
1117                   boolean clip_xy,
1118                   boolean clip_z,
1119                   boolean clip_user,
1120                   boolean clip_halfz,
1121                   unsigned ucp_enable,
1122                   LLVMValueRef context_ptr,
1123                   boolean *have_clipdist)
1124 {
1125    LLVMBuilderRef builder = gallivm->builder;
1126    LLVMValueRef mask; /* stores the <nxi32> clipmasks */
1127    LLVMValueRef test, temp;
1128    LLVMValueRef zero, shift;
1129    LLVMValueRef pos_x, pos_y, pos_z, pos_w;
1130    LLVMValueRef cv_x, cv_y, cv_z, cv_w;
1131    LLVMValueRef plane1, planes, plane_ptr, sum;
1132    struct lp_type f32_type = vs_type;
1133    struct lp_type i32_type = lp_int_type(vs_type);
1134    const unsigned pos = llvm->draw->vs.position_output;
1135    const unsigned cv = llvm->draw->vs.clipvertex_output;
1136    int num_written_clipdistance = llvm->draw->vs.vertex_shader->info.num_written_clipdistance;
1137    bool have_cd = false;
1138    unsigned cd[2];
1139
1140    cd[0] = llvm->draw->vs.clipdistance_output[0];
1141    cd[1] = llvm->draw->vs.clipdistance_output[1];
1142
1143    if (cd[0] != pos || cd[1] != pos)
1144       have_cd = true;
1145
1146    if (num_written_clipdistance && !clip_user) {
1147       clip_user = true;
1148       ucp_enable = (1 << num_written_clipdistance) - 1;
1149    }
1150
1151    mask = lp_build_const_int_vec(gallivm, i32_type, 0);
1152    temp = lp_build_const_int_vec(gallivm, i32_type, 0);
1153    zero = lp_build_const_vec(gallivm, f32_type, 0);         /* 0.0f 0.0f 0.0f 0.0f */
1154    shift = lp_build_const_int_vec(gallivm, i32_type, 1);    /* 1 1 1 1 */
1155
1156    /*
1157     * load clipvertex and position from correct locations.
1158     * if they are the same just load them once.
1159     */
1160    pos_x = LLVMBuildLoad(builder, outputs[pos][0], ""); /*x0 x1 .. xn */
1161    pos_y = LLVMBuildLoad(builder, outputs[pos][1], ""); /*y0 y1 .. yn */
1162    pos_z = LLVMBuildLoad(builder, outputs[pos][2], ""); /*z0 z1 .. zn */
1163    pos_w = LLVMBuildLoad(builder, outputs[pos][3], ""); /*w0 w1 .. wn */
1164
1165    if (clip_user && cv != pos) {
1166       cv_x = LLVMBuildLoad(builder, outputs[cv][0], ""); /*x0 x1 .. xn */
1167       cv_y = LLVMBuildLoad(builder, outputs[cv][1], ""); /*y0 y1 .. yn */
1168       cv_z = LLVMBuildLoad(builder, outputs[cv][2], ""); /*z0 z1 .. zn */
1169       cv_w = LLVMBuildLoad(builder, outputs[cv][3], ""); /*w0 w1 .. wn */
1170    } else {
1171       cv_x = pos_x;
1172       cv_y = pos_y;
1173       cv_z = pos_z;
1174       cv_w = pos_w;
1175    }
1176
1177    /* Cliptest, for hardwired planes */
1178    if (clip_xy) {
1179       /* plane 1 */
1180       test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_x , pos_w);
1181       temp = shift;
1182       test = LLVMBuildAnd(builder, test, temp, "");
1183       mask = test;
1184
1185       /* plane 2 */
1186       test = LLVMBuildFAdd(builder, pos_x, pos_w, "");
1187       test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test);
1188       temp = LLVMBuildShl(builder, temp, shift, "");
1189       test = LLVMBuildAnd(builder, test, temp, "");
1190       mask = LLVMBuildOr(builder, mask, test, "");
1191
1192       /* plane 3 */
1193       test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_y, pos_w);
1194       temp = LLVMBuildShl(builder, temp, shift, "");
1195       test = LLVMBuildAnd(builder, test, temp, "");
1196       mask = LLVMBuildOr(builder, mask, test, "");
1197
1198       /* plane 4 */
1199       test = LLVMBuildFAdd(builder, pos_y, pos_w, "");
1200       test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test);
1201       temp = LLVMBuildShl(builder, temp, shift, "");
1202       test = LLVMBuildAnd(builder, test, temp, "");
1203       mask = LLVMBuildOr(builder, mask, test, "");
1204    }
1205
1206    if (clip_z) {
1207       temp = lp_build_const_int_vec(gallivm, i32_type, 16);
1208       if (clip_halfz) {
1209          /* plane 5 */
1210          test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, pos_z);
1211          test = LLVMBuildAnd(builder, test, temp, "");
1212          mask = LLVMBuildOr(builder, mask, test, "");
1213       }
1214       else {
1215          /* plane 5 */
1216          test = LLVMBuildFAdd(builder, pos_z, pos_w, "");
1217          test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test);
1218          test = LLVMBuildAnd(builder, test, temp, "");
1219          mask = LLVMBuildOr(builder, mask, test, "");
1220       }
1221       /* plane 6 */
1222       test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_z, pos_w);
1223       temp = LLVMBuildShl(builder, temp, shift, "");
1224       test = LLVMBuildAnd(builder, test, temp, "");
1225       mask = LLVMBuildOr(builder, mask, test, "");
1226    }
1227
1228    if (clip_user) {
1229       LLVMValueRef planes_ptr = draw_jit_context_planes(gallivm, context_ptr);
1230       LLVMValueRef indices[3];
1231       LLVMValueRef is_nan_or_inf;
1232
1233       /* userclip planes */
1234       while (ucp_enable) {
1235          unsigned plane_idx = ffs(ucp_enable)-1;
1236          ucp_enable &= ~(1 << plane_idx);
1237          plane_idx += 6;
1238
1239          if (have_cd && num_written_clipdistance) {
1240             LLVMValueRef clipdist;
1241             int i;
1242             i = plane_idx - 6;
1243
1244             *have_clipdist = TRUE;
1245             if (i < 4) {
1246                clipdist = LLVMBuildLoad(builder, outputs[cd[0]][i], "");
1247             } else {
1248                clipdist = LLVMBuildLoad(builder, outputs[cd[1]][i-4], "");
1249             }
1250             test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, clipdist);
1251             is_nan_or_inf = lp_build_is_inf_or_nan(gallivm, vs_type, clipdist);
1252             test = LLVMBuildOr(builder, test, is_nan_or_inf, "");
1253             temp = lp_build_const_int_vec(gallivm, i32_type, 1 << plane_idx);
1254             test = LLVMBuildAnd(builder, test, temp, "");
1255             mask = LLVMBuildOr(builder, mask, test, "");
1256          } else {
1257             LLVMTypeRef vs_type_llvm = lp_build_vec_type(gallivm, vs_type);
1258             indices[0] = lp_build_const_int32(gallivm, 0);
1259             indices[1] = lp_build_const_int32(gallivm, plane_idx);
1260
1261             indices[2] = lp_build_const_int32(gallivm, 0);
1262             plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, "");
1263             plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_x");
1264             planes = lp_build_broadcast(gallivm, vs_type_llvm, plane1);
1265             sum = LLVMBuildFMul(builder, planes, cv_x, "");
1266
1267             indices[2] = lp_build_const_int32(gallivm, 1);
1268             plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, "");
1269             plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_y");
1270             planes = lp_build_broadcast(gallivm, vs_type_llvm, plane1);
1271             test = LLVMBuildFMul(builder, planes, cv_y, "");
1272             sum = LLVMBuildFAdd(builder, sum, test, "");
1273
1274             indices[2] = lp_build_const_int32(gallivm, 2);
1275             plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, "");
1276             plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_z");
1277             planes = lp_build_broadcast(gallivm, vs_type_llvm, plane1);
1278             test = LLVMBuildFMul(builder, planes, cv_z, "");
1279             sum = LLVMBuildFAdd(builder, sum, test, "");
1280
1281             indices[2] = lp_build_const_int32(gallivm, 3);
1282             plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, "");
1283             plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_w");
1284             planes = lp_build_broadcast(gallivm, vs_type_llvm, plane1);
1285             test = LLVMBuildFMul(builder, planes, cv_w, "");
1286             sum = LLVMBuildFAdd(builder, sum, test, "");
1287
1288             test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, sum);
1289             temp = lp_build_const_int_vec(gallivm, i32_type, 1 << plane_idx);
1290             test = LLVMBuildAnd(builder, test, temp, "");
1291             mask = LLVMBuildOr(builder, mask, test, "");
1292          }
1293       }
1294    }
1295    return mask;
1296 }
1297
1298
1299 /**
1300  * Returns boolean if any clipping has occurred
1301  * Used zero/non-zero i32 value to represent boolean
1302  */
1303 static LLVMValueRef
1304 clipmask_booli32(struct gallivm_state *gallivm,
1305                  const struct lp_type vs_type,
1306                  LLVMValueRef clipmask_bool_ptr)
1307 {
1308    LLVMBuilderRef builder = gallivm->builder;
1309    LLVMTypeRef int32_type = LLVMInt32TypeInContext(gallivm->context);
1310    LLVMValueRef clipmask_bool = LLVMBuildLoad(builder, clipmask_bool_ptr, "");
1311    LLVMValueRef ret = LLVMConstNull(int32_type);
1312    LLVMValueRef temp;
1313    int i;
1314
1315    /*
1316     * Can do this with log2(vector length) pack instructions and one extract
1317     * (as we don't actually need a or) with sse2 which would be way better.
1318     */
1319    for (i=0; i < vs_type.length; i++) {
1320       temp = LLVMBuildExtractElement(builder, clipmask_bool,
1321                                      lp_build_const_int32(gallivm, i) , "");
1322       ret = LLVMBuildOr(builder, ret, temp, "");
1323    }
1324    return ret;
1325 }
1326
1327 static LLVMValueRef
1328 draw_gs_llvm_fetch_input(const struct lp_build_tgsi_gs_iface *gs_iface,
1329                          struct lp_build_tgsi_context * bld_base,
1330                          boolean is_vindex_indirect,
1331                          LLVMValueRef vertex_index,
1332                          boolean is_aindex_indirect,
1333                          LLVMValueRef attrib_index,
1334                          LLVMValueRef swizzle_index)
1335 {
1336    const struct draw_gs_llvm_iface *gs = draw_gs_llvm_iface(gs_iface);
1337    struct gallivm_state *gallivm = bld_base->base.gallivm;
1338    LLVMBuilderRef builder = gallivm->builder;
1339    LLVMValueRef indices[3];
1340    LLVMValueRef res;
1341    struct lp_type type = bld_base->base.type;
1342
1343    if (is_vindex_indirect || is_aindex_indirect) {
1344       int i;
1345       res = bld_base->base.zero;
1346       for (i = 0; i < type.length; ++i) {
1347          LLVMValueRef idx = lp_build_const_int32(gallivm, i);
1348          LLVMValueRef vert_chan_index = vertex_index;
1349          LLVMValueRef attr_chan_index = attrib_index;
1350          LLVMValueRef channel_vec, value;
1351
1352          if (is_vindex_indirect) {
1353             vert_chan_index = LLVMBuildExtractElement(builder,
1354                                                       vertex_index, idx, "");
1355          }
1356          if (is_aindex_indirect) {
1357             attr_chan_index = LLVMBuildExtractElement(builder,
1358                                                       attrib_index, idx, "");
1359          }
1360
1361          indices[0] = vert_chan_index;
1362          indices[1] = attr_chan_index;
1363          indices[2] = swizzle_index;
1364
1365          channel_vec = LLVMBuildGEP(builder, gs->input, indices, 3, "");
1366          channel_vec = LLVMBuildLoad(builder, channel_vec, "");
1367          value = LLVMBuildExtractElement(builder, channel_vec, idx, "");
1368
1369          res = LLVMBuildInsertElement(builder, res, value, idx, "");
1370       }
1371    } else {
1372       indices[0] = vertex_index;
1373       indices[1] = attrib_index;
1374       indices[2] = swizzle_index;
1375
1376       res = LLVMBuildGEP(builder, gs->input, indices, 3, "");
1377       res = LLVMBuildLoad(builder, res, "");
1378    }
1379
1380    return res;
1381 }
1382
1383 static void
1384 draw_gs_llvm_emit_vertex(const struct lp_build_tgsi_gs_iface *gs_base,
1385                          struct lp_build_tgsi_context * bld_base,
1386                          LLVMValueRef (*outputs)[4],
1387                          LLVMValueRef emitted_vertices_vec)
1388 {
1389    const struct draw_gs_llvm_iface *gs_iface = draw_gs_llvm_iface(gs_base);
1390    struct draw_gs_llvm_variant *variant = gs_iface->variant;
1391    struct gallivm_state *gallivm = variant->gallivm;
1392    LLVMBuilderRef builder = gallivm->builder;
1393    struct lp_type gs_type = bld_base->base.type;
1394    LLVMValueRef clipmask = lp_build_const_int_vec(gallivm,
1395                                                   lp_int_type(gs_type), 0);
1396    LLVMValueRef indices[LP_MAX_VECTOR_LENGTH];
1397    LLVMValueRef next_prim_offset =
1398       lp_build_const_int32(gallivm, variant->shader->base.primitive_boundary);
1399    LLVMValueRef io = variant->io_ptr;
1400    unsigned i;
1401    const struct tgsi_shader_info *gs_info = &variant->shader->base.info;
1402
1403    for (i = 0; i < gs_type.length; ++i) {
1404       LLVMValueRef ind = lp_build_const_int32(gallivm, i);
1405       LLVMValueRef currently_emitted =
1406          LLVMBuildExtractElement(builder, emitted_vertices_vec, ind, "");
1407       indices[i] = LLVMBuildMul(builder, ind, next_prim_offset, "");
1408       indices[i] = LLVMBuildAdd(builder, indices[i], currently_emitted, "");
1409    }
1410
1411    convert_to_aos(gallivm, io, indices,
1412                   outputs, clipmask,
1413                   gs_info->num_outputs, gs_type,
1414                   FALSE);
1415 }
1416
1417 static void
1418 draw_gs_llvm_end_primitive(const struct lp_build_tgsi_gs_iface *gs_base,
1419                            struct lp_build_tgsi_context * bld_base,
1420                            LLVMValueRef verts_per_prim_vec,
1421                            LLVMValueRef emitted_prims_vec)
1422 {
1423    const struct draw_gs_llvm_iface *gs_iface = draw_gs_llvm_iface(gs_base);
1424    struct draw_gs_llvm_variant *variant = gs_iface->variant;
1425    struct gallivm_state *gallivm = variant->gallivm;
1426    LLVMBuilderRef builder = gallivm->builder;
1427    LLVMValueRef prim_lengts_ptr =
1428       draw_gs_jit_prim_lengths(variant->gallivm, variant->context_ptr);
1429    unsigned i;
1430
1431    for (i = 0; i < bld_base->base.type.length; ++i) {
1432       LLVMValueRef ind = lp_build_const_int32(gallivm, i);
1433       LLVMValueRef prims_emitted =
1434          LLVMBuildExtractElement(builder, emitted_prims_vec, ind, "");
1435       LLVMValueRef store_ptr;
1436       LLVMValueRef num_vertices =
1437          LLVMBuildExtractElement(builder, verts_per_prim_vec, ind, "");
1438
1439       store_ptr = LLVMBuildGEP(builder, prim_lengts_ptr, &prims_emitted, 1, "");
1440       store_ptr = LLVMBuildLoad(builder, store_ptr, "");
1441       store_ptr = LLVMBuildGEP(builder, store_ptr, &ind, 1, "");
1442       LLVMBuildStore(builder, num_vertices, store_ptr);
1443    }
1444 }
1445
1446 static void
1447 draw_gs_llvm_epilogue(const struct lp_build_tgsi_gs_iface *gs_base,
1448                       struct lp_build_tgsi_context * bld_base,
1449                       LLVMValueRef total_emitted_vertices_vec,
1450                       LLVMValueRef emitted_prims_vec)
1451 {
1452    const struct draw_gs_llvm_iface *gs_iface = draw_gs_llvm_iface(gs_base);
1453    struct draw_gs_llvm_variant *variant = gs_iface->variant;
1454    struct gallivm_state *gallivm = variant->gallivm;
1455    LLVMBuilderRef builder = gallivm->builder;
1456    LLVMValueRef emitted_verts_ptr =
1457       draw_gs_jit_emitted_vertices(gallivm, variant->context_ptr);
1458    LLVMValueRef emitted_prims_ptr =
1459       draw_gs_jit_emitted_prims(gallivm, variant->context_ptr);
1460    LLVMValueRef zero = lp_build_const_int32(gallivm, 0);
1461
1462    emitted_verts_ptr = LLVMBuildGEP(builder, emitted_verts_ptr, &zero, 0, "");
1463    emitted_prims_ptr = LLVMBuildGEP(builder, emitted_prims_ptr, &zero, 0, "");
1464
1465    LLVMBuildStore(builder, total_emitted_vertices_vec, emitted_verts_ptr);
1466    LLVMBuildStore(builder, emitted_prims_vec, emitted_prims_ptr);
1467 }
1468
1469 static void
1470 draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant,
1471                    boolean elts)
1472 {
1473    struct gallivm_state *gallivm = variant->gallivm;
1474    LLVMContextRef context = gallivm->context;
1475    LLVMTypeRef int32_type = LLVMInt32TypeInContext(context);
1476    LLVMTypeRef arg_types[10];
1477    unsigned num_arg_types =
1478       elts ? Elements(arg_types) : Elements(arg_types) - 1;
1479    LLVMTypeRef func_type;
1480    LLVMValueRef context_ptr;
1481    LLVMBasicBlockRef block;
1482    LLVMBuilderRef builder;
1483    char func_name[64];
1484    struct lp_type vs_type;
1485    LLVMValueRef end, start;
1486    LLVMValueRef count, fetch_elts, fetch_elt_max, fetch_count;
1487    LLVMValueRef vertex_id_offset;
1488    LLVMValueRef stride, step, io_itr;
1489    LLVMValueRef io_ptr, vbuffers_ptr, vb_ptr;
1490    LLVMValueRef zero = lp_build_const_int32(gallivm, 0);
1491    LLVMValueRef one = lp_build_const_int32(gallivm, 1);
1492    struct draw_context *draw = llvm->draw;
1493    const struct tgsi_shader_info *vs_info = &draw->vs.vertex_shader->info;
1494    unsigned i, j;
1495    struct lp_build_context bld;
1496    struct lp_build_loop_state lp_loop;
1497    const int vector_length = lp_native_vector_width / 32;
1498    LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][TGSI_NUM_CHANNELS];
1499    LLVMValueRef fetch_max;
1500    struct lp_build_sampler_soa *sampler = 0;
1501    LLVMValueRef ret, clipmask_bool_ptr;
1502    const struct draw_geometry_shader *gs = draw->gs.geometry_shader;
1503    struct draw_llvm_variant_key *key = &variant->key;
1504    /* If geometry shader is present we need to skip both the viewport
1505     * transformation and clipping otherwise the inputs to the geometry
1506     * shader will be incorrect.
1507     */
1508    const boolean bypass_viewport = gs || key->bypass_viewport;
1509    const boolean enable_cliptest = !gs && (key->clip_xy ||
1510                                            key->clip_z  ||
1511                                            key->clip_user);
1512    LLVMValueRef variant_func;
1513    const unsigned pos = llvm->draw->vs.position_output;
1514    const unsigned cv = llvm->draw->vs.clipvertex_output;
1515    boolean have_clipdist = FALSE;
1516    struct lp_bld_tgsi_system_values system_values;
1517
1518    memset(&system_values, 0, sizeof(system_values));
1519
1520    util_snprintf(func_name, sizeof(func_name), "draw_llvm_vs_variant%u_%s",
1521                  variant->shader->variants_cached, elts ? "elts" : "linear");
1522
1523    i = 0;
1524    arg_types[i++] = get_context_ptr_type(variant);       /* context */
1525    arg_types[i++] = get_vertex_header_ptr_type(variant); /* vertex_header */
1526    arg_types[i++] = get_buffer_ptr_type(variant);        /* vbuffers */
1527    if (elts) {
1528       arg_types[i++] = LLVMPointerType(int32_type, 0);/* fetch_elts  */
1529       arg_types[i++] = int32_type;                  /* fetch_elt_max */
1530    } else
1531       arg_types[i++] = int32_type;                  /* start */
1532    arg_types[i++] = int32_type;                     /* fetch_count / count */
1533    arg_types[i++] = int32_type;                     /* stride */
1534    arg_types[i++] = get_vb_ptr_type(variant);       /* pipe_vertex_buffer's */
1535    arg_types[i++] = int32_type;                     /* instance_id */
1536    arg_types[i++] = int32_type;                     /* vertex_id_offset */
1537
1538    func_type = LLVMFunctionType(int32_type, arg_types, num_arg_types, 0);
1539
1540    variant_func = LLVMAddFunction(gallivm->module, func_name, func_type);
1541
1542    if (elts)
1543       variant->function_elts = variant_func;
1544    else
1545       variant->function = variant_func;
1546
1547    LLVMSetFunctionCallConv(variant_func, LLVMCCallConv);
1548    for (i = 0; i < num_arg_types; ++i)
1549       if (LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind)
1550          LLVMAddAttribute(LLVMGetParam(variant_func, i),
1551                           LLVMNoAliasAttribute);
1552
1553    context_ptr               = LLVMGetParam(variant_func, 0);
1554    io_ptr                    = LLVMGetParam(variant_func, 1);
1555    vbuffers_ptr              = LLVMGetParam(variant_func, 2);
1556    stride                    = LLVMGetParam(variant_func, 5 + (elts ? 1 : 0));
1557    vb_ptr                    = LLVMGetParam(variant_func, 6 + (elts ? 1 : 0));
1558    system_values.instance_id = LLVMGetParam(variant_func, 7 + (elts ? 1 : 0));
1559    vertex_id_offset          = LLVMGetParam(variant_func, 8 + (elts ? 1 : 0));
1560
1561    lp_build_name(context_ptr, "context");
1562    lp_build_name(io_ptr, "io");
1563    lp_build_name(vbuffers_ptr, "vbuffers");
1564    lp_build_name(stride, "stride");
1565    lp_build_name(vb_ptr, "vb");
1566    lp_build_name(system_values.instance_id, "instance_id");
1567    lp_build_name(vertex_id_offset, "vertex_id_offset");
1568
1569    if (elts) {
1570       fetch_elts    = LLVMGetParam(variant_func, 3);
1571       fetch_elt_max = LLVMGetParam(variant_func, 4);
1572       fetch_count   = LLVMGetParam(variant_func, 5);
1573       lp_build_name(fetch_elts, "fetch_elts");
1574       lp_build_name(fetch_elt_max, "fetch_elt_max");
1575       lp_build_name(fetch_count, "fetch_count");
1576       start = count = NULL;
1577    }
1578    else {
1579       start        = LLVMGetParam(variant_func, 3);
1580       count        = LLVMGetParam(variant_func, 4);
1581       lp_build_name(start, "start");
1582       lp_build_name(count, "count");
1583       fetch_elts = fetch_count = NULL;
1584    }
1585
1586    /*
1587     * Function body
1588     */
1589
1590    block = LLVMAppendBasicBlockInContext(gallivm->context, variant_func, "entry");
1591    builder = gallivm->builder;
1592    LLVMPositionBuilderAtEnd(builder, block);
1593
1594    lp_build_context_init(&bld, gallivm, lp_type_int(32));
1595
1596    memset(&vs_type, 0, sizeof vs_type);
1597    vs_type.floating = TRUE; /* floating point values */
1598    vs_type.sign = TRUE;     /* values are signed */
1599    vs_type.norm = FALSE;    /* values are not limited to [0,1] or [-1,1] */
1600    vs_type.width = 32;      /* 32-bit float */
1601    vs_type.length = vector_length;
1602
1603    /* hold temporary "bool" clipmask */
1604    clipmask_bool_ptr = lp_build_alloca(gallivm, lp_build_int_vec_type(gallivm, vs_type), "");
1605    LLVMBuildStore(builder, lp_build_zero(gallivm, lp_int_type(vs_type)), clipmask_bool_ptr);
1606
1607    /* code generated texture sampling */
1608    sampler = draw_llvm_sampler_soa_create(
1609       draw_llvm_variant_key_samplers(key),
1610       context_ptr);
1611
1612    if (elts) {
1613       start = zero;
1614       end = fetch_count;
1615       count = fetch_count;
1616    }
1617    else {
1618       end = lp_build_add(&bld, start, count);
1619    }
1620
1621    step = lp_build_const_int32(gallivm, vector_length);
1622
1623    fetch_max = LLVMBuildSub(builder, end, one, "fetch_max");
1624
1625    lp_build_loop_begin(&lp_loop, gallivm, zero);
1626    {
1627       LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][TGSI_NUM_CHANNELS];
1628       LLVMValueRef aos_attribs[PIPE_MAX_SHADER_INPUTS][LP_MAX_VECTOR_WIDTH / 32] = { { 0 } };
1629       LLVMValueRef io;
1630       LLVMValueRef clipmask;   /* holds the clipmask value */
1631       const LLVMValueRef (*ptr_aos)[TGSI_NUM_CHANNELS];
1632
1633       io_itr = lp_loop.counter;
1634
1635       io = LLVMBuildGEP(builder, io_ptr, &io_itr, 1, "");
1636 #if DEBUG_STORE
1637       lp_build_printf(gallivm, " --- io %d = %p, loop counter %d\n",
1638                       io_itr, io, lp_loop.counter);
1639 #endif
1640       system_values.vertex_id = lp_build_zero(gallivm, lp_type_uint_vec(32, 32*vector_length));
1641       for (i = 0; i < vector_length; ++i) {
1642          LLVMValueRef vert_index =
1643             LLVMBuildAdd(builder,
1644                          lp_loop.counter,
1645                          lp_build_const_int32(gallivm, i), "");
1646          LLVMValueRef true_index =
1647             LLVMBuildAdd(builder, start, vert_index, "");
1648          LLVMValueRef vertex_id;
1649
1650          /* make sure we're not out of bounds which can happen
1651           * if fetch_count % 4 != 0, because on the last iteration
1652           * a few of the 4 vertex fetches will be out of bounds */
1653          true_index = lp_build_min(&bld, true_index, fetch_max);
1654
1655          if (elts) {
1656             LLVMValueRef fetch_ptr;
1657             LLVMValueRef index_overflowed;
1658             LLVMValueRef index_ptr =
1659                lp_build_alloca(
1660                   gallivm,
1661                   lp_build_vec_type(gallivm, lp_type_int(32)), "");
1662             struct lp_build_if_state if_ctx;
1663             index_overflowed = LLVMBuildICmp(builder, LLVMIntUGT,
1664                                              true_index, fetch_elt_max,
1665                                              "index_overflowed");
1666
1667             lp_build_if(&if_ctx, gallivm, index_overflowed);
1668             {
1669                /* Generate maximum possible index so that
1670                 * generate_fetch can treat it just like
1671                 * any other overflow and return zeros.
1672                 * We don't have to worry about the restart
1673                 * primitive index because it has already been
1674                 * handled
1675                 */
1676                LLVMValueRef val =
1677                   lp_build_const_int32(gallivm, 0xffffffff);
1678                LLVMBuildStore(builder, val, index_ptr);
1679             }
1680             lp_build_else(&if_ctx);
1681             {
1682                LLVMValueRef val;
1683                fetch_ptr = LLVMBuildGEP(builder, fetch_elts,
1684                                         &true_index, 1, "");
1685                val = LLVMBuildLoad(builder, fetch_ptr, "");
1686                LLVMBuildStore(builder, val, index_ptr);
1687             }
1688             lp_build_endif(&if_ctx);
1689             true_index = LLVMBuildLoad(builder, index_ptr, "true_index");
1690          }
1691          /* in the paths with elts vertex id has to be unaffected by the
1692           * index bias and because indices inside our elements array have
1693           * already had index bias applied we need to subtract it here to
1694           * get back to the original index.
1695           * in the linear paths vertex id has to be unaffected by the
1696           * original start index and because we abuse the 'start' variable
1697           * to either represent the actual start index or the index at which
1698           * the primitive was split (we split rendering into chunks of at
1699           * most 4095-vertices) we need to back out the original start
1700           * index out of our vertex id here.
1701           */
1702          vertex_id = LLVMBuildSub(builder, true_index, vertex_id_offset, "");
1703
1704          system_values.vertex_id = LLVMBuildInsertElement(
1705             gallivm->builder,
1706             system_values.vertex_id, vertex_id,
1707             lp_build_const_int32(gallivm, i), "");
1708
1709          for (j = 0; j < draw->pt.nr_vertex_elements; ++j) {
1710             struct pipe_vertex_element *velem = &draw->pt.vertex_element[j];
1711             LLVMValueRef vb_index =
1712                lp_build_const_int32(gallivm, velem->vertex_buffer_index);
1713             LLVMValueRef vb = LLVMBuildGEP(builder, vb_ptr, &vb_index, 1, "");
1714             generate_fetch(gallivm, draw, vbuffers_ptr,
1715                            &aos_attribs[j][i], velem, vb, true_index,
1716                            system_values.instance_id);
1717          }
1718       }
1719       convert_to_soa(gallivm, aos_attribs, inputs,
1720                      draw->pt.nr_vertex_elements, vs_type);
1721
1722       ptr_aos = (const LLVMValueRef (*)[TGSI_NUM_CHANNELS]) inputs;
1723       generate_vs(variant,
1724                   builder,
1725                   vs_type,
1726                   outputs,
1727                   ptr_aos,
1728                   &system_values,
1729                   context_ptr,
1730                   sampler,
1731                   key->clamp_vertex_color);
1732
1733       if (pos != -1 && cv != -1) {
1734          /* store original positions in clip before further manipulation */
1735          store_clip(gallivm, vs_type, io, outputs, 0, cv);
1736          store_clip(gallivm, vs_type, io, outputs, 1, pos);
1737
1738          /* do cliptest */
1739          if (enable_cliptest) {
1740             LLVMValueRef temp = LLVMBuildLoad(builder, clipmask_bool_ptr, "");
1741             /* allocate clipmask, assign it integer type */
1742             clipmask = generate_clipmask(llvm,
1743                                          gallivm,
1744                                          vs_type,
1745                                          outputs,
1746                                          key->clip_xy,
1747                                          key->clip_z,
1748                                          key->clip_user,
1749                                          key->clip_halfz,
1750                                          key->ucp_enable,
1751                                          context_ptr, &have_clipdist);
1752             temp = LLVMBuildOr(builder, clipmask, temp, "");
1753             /* store temporary clipping boolean value */
1754             LLVMBuildStore(builder, temp, clipmask_bool_ptr);
1755          }
1756          else {
1757             clipmask = lp_build_const_int_vec(gallivm, lp_int_type(vs_type), 0);
1758          }
1759
1760          /* do viewport mapping */
1761          if (!bypass_viewport) {
1762             generate_viewport(variant, builder, vs_type, outputs, context_ptr);
1763          }
1764       }
1765       else {
1766          clipmask = lp_build_const_int_vec(gallivm, lp_int_type(vs_type), 0);
1767       }
1768
1769       /* store clipmask in vertex header,
1770        * original positions in clip
1771        * and transformed positions in data
1772        */
1773       convert_to_aos(gallivm, io, NULL, outputs, clipmask,
1774                      vs_info->num_outputs, vs_type,
1775                      have_clipdist);
1776    }
1777    lp_build_loop_end_cond(&lp_loop, count, step, LLVMIntUGE);
1778
1779    sampler->destroy(sampler);
1780
1781    /* return clipping boolean value for function */
1782    ret = clipmask_booli32(gallivm, vs_type, clipmask_bool_ptr);
1783
1784    LLVMBuildRet(builder, ret);
1785
1786    gallivm_verify_function(gallivm, variant_func);
1787 }
1788
1789
1790 struct draw_llvm_variant_key *
1791 draw_llvm_make_variant_key(struct draw_llvm *llvm, char *store)
1792 {
1793    unsigned i;
1794    struct draw_llvm_variant_key *key;
1795    struct draw_sampler_static_state *draw_sampler;
1796
1797    key = (struct draw_llvm_variant_key *)store;
1798
1799    key->clamp_vertex_color = llvm->draw->rasterizer->clamp_vertex_color; /**/
1800
1801    /* Presumably all variants of the shader should have the same
1802     * number of vertex elements - ie the number of shader inputs.
1803     * NOTE: we NEED to store the needed number of needed inputs
1804     * here, not the number of provided elements to match keysize
1805     * (and the offset of sampler state in the key).
1806     */
1807    key->nr_vertex_elements = llvm->draw->vs.vertex_shader->info.file_max[TGSI_FILE_INPUT] + 1;
1808    assert(key->nr_vertex_elements <= llvm->draw->pt.nr_vertex_elements);
1809
1810    /* will have to rig this up properly later */
1811    key->clip_xy = llvm->draw->clip_xy;
1812    key->clip_z = llvm->draw->clip_z;
1813    key->clip_user = llvm->draw->clip_user;
1814    key->bypass_viewport = llvm->draw->identity_viewport;
1815    key->clip_halfz = llvm->draw->rasterizer->clip_halfz;
1816    key->need_edgeflags = (llvm->draw->vs.edgeflag_output ? TRUE : FALSE);
1817    key->ucp_enable = llvm->draw->rasterizer->clip_plane_enable;
1818    key->has_gs = llvm->draw->gs.geometry_shader != NULL;
1819    key->num_outputs = draw_total_vs_outputs(llvm->draw);
1820    key->pad1 = 0;
1821
1822    /* All variants of this shader will have the same value for
1823     * nr_samplers.  Not yet trying to compact away holes in the
1824     * sampler array.
1825     */
1826    key->nr_samplers = llvm->draw->vs.vertex_shader->info.file_max[TGSI_FILE_SAMPLER] + 1;
1827    if (llvm->draw->vs.vertex_shader->info.file_max[TGSI_FILE_SAMPLER_VIEW] != -1) {
1828       key->nr_sampler_views =
1829          llvm->draw->vs.vertex_shader->info.file_max[TGSI_FILE_SAMPLER_VIEW] + 1;
1830    }
1831    else {
1832       key->nr_sampler_views = key->nr_samplers;
1833    }
1834
1835    draw_sampler = draw_llvm_variant_key_samplers(key);
1836
1837    memcpy(key->vertex_element,
1838           llvm->draw->pt.vertex_element,
1839           sizeof(struct pipe_vertex_element) * key->nr_vertex_elements);
1840
1841    memset(draw_sampler, 0, MAX2(key->nr_samplers, key->nr_sampler_views) * sizeof *draw_sampler);
1842
1843    for (i = 0 ; i < key->nr_samplers; i++) {
1844       lp_sampler_static_sampler_state(&draw_sampler[i].sampler_state,
1845                                       llvm->draw->samplers[PIPE_SHADER_VERTEX][i]);
1846    }
1847    for (i = 0 ; i < key->nr_sampler_views; i++) {
1848       lp_sampler_static_texture_state(&draw_sampler[i].texture_state,
1849                                       llvm->draw->sampler_views[PIPE_SHADER_VERTEX][i]);
1850    }
1851
1852    return key;
1853 }
1854
1855
1856 void
1857 draw_llvm_dump_variant_key(struct draw_llvm_variant_key *key)
1858 {
1859    unsigned i;
1860    struct draw_sampler_static_state *sampler = draw_llvm_variant_key_samplers(key);
1861
1862    debug_printf("clamp_vertex_color = %u\n", key->clamp_vertex_color);
1863    debug_printf("clip_xy = %u\n", key->clip_xy);
1864    debug_printf("clip_z = %u\n", key->clip_z);
1865    debug_printf("clip_user = %u\n", key->clip_user);
1866    debug_printf("bypass_viewport = %u\n", key->bypass_viewport);
1867    debug_printf("clip_halfz = %u\n", key->clip_halfz);
1868    debug_printf("need_edgeflags = %u\n", key->need_edgeflags);
1869    debug_printf("has_gs = %u\n", key->has_gs);
1870    debug_printf("ucp_enable = %u\n", key->ucp_enable);
1871
1872    for (i = 0 ; i < key->nr_vertex_elements; i++) {
1873       debug_printf("vertex_element[%i].src_offset = %u\n", i, key->vertex_element[i].src_offset);
1874       debug_printf("vertex_element[%i].instance_divisor = %u\n", i, key->vertex_element[i].instance_divisor);
1875       debug_printf("vertex_element[%i].vertex_buffer_index = %u\n", i, key->vertex_element[i].vertex_buffer_index);
1876       debug_printf("vertex_element[%i].src_format = %s\n", i, util_format_name(key->vertex_element[i].src_format));
1877    }
1878
1879    for (i = 0 ; i < key->nr_sampler_views; i++) {
1880       debug_printf("sampler[%i].src_format = %s\n", i, util_format_name(sampler[i].texture_state.format));
1881    }
1882 }
1883
1884
1885 void
1886 draw_llvm_set_mapped_texture(struct draw_context *draw,
1887                              unsigned shader_stage,
1888                              unsigned sview_idx,
1889                              uint32_t width, uint32_t height, uint32_t depth,
1890                              uint32_t first_level, uint32_t last_level,
1891                              const void *base_ptr,
1892                              uint32_t row_stride[PIPE_MAX_TEXTURE_LEVELS],
1893                              uint32_t img_stride[PIPE_MAX_TEXTURE_LEVELS],
1894                              uint32_t mip_offsets[PIPE_MAX_TEXTURE_LEVELS])
1895 {
1896    unsigned j;
1897    struct draw_jit_texture *jit_tex;
1898
1899    assert(shader_stage == PIPE_SHADER_VERTEX ||
1900           shader_stage == PIPE_SHADER_GEOMETRY);
1901
1902    if (shader_stage == PIPE_SHADER_VERTEX) {
1903       assert(sview_idx < Elements(draw->llvm->jit_context.textures));
1904
1905       jit_tex = &draw->llvm->jit_context.textures[sview_idx];
1906    } else if (shader_stage == PIPE_SHADER_GEOMETRY) {
1907       assert(sview_idx < Elements(draw->llvm->gs_jit_context.textures));
1908
1909       jit_tex = &draw->llvm->gs_jit_context.textures[sview_idx];
1910    } else {
1911       assert(0);
1912       return;
1913    }
1914
1915    jit_tex->width = width;
1916    jit_tex->height = height;
1917    jit_tex->depth = depth;
1918    jit_tex->first_level = first_level;
1919    jit_tex->last_level = last_level;
1920    jit_tex->base = base_ptr;
1921
1922    for (j = first_level; j <= last_level; j++) {
1923       jit_tex->mip_offsets[j] = mip_offsets[j];
1924       jit_tex->row_stride[j] = row_stride[j];
1925       jit_tex->img_stride[j] = img_stride[j];
1926    }
1927 }
1928
1929
1930 void
1931 draw_llvm_set_sampler_state(struct draw_context *draw,
1932                             unsigned shader_type)
1933 {
1934    unsigned i;
1935
1936    if (shader_type == PIPE_SHADER_VERTEX) {
1937       for (i = 0; i < draw->num_samplers[PIPE_SHADER_VERTEX]; i++) {
1938          struct draw_jit_sampler *jit_sam = &draw->llvm->jit_context.samplers[i];
1939
1940          if (draw->samplers[i]) {
1941             const struct pipe_sampler_state *s
1942                = draw->samplers[PIPE_SHADER_VERTEX][i];
1943             jit_sam->min_lod = s->min_lod;
1944             jit_sam->max_lod = s->max_lod;
1945             jit_sam->lod_bias = s->lod_bias;
1946             COPY_4V(jit_sam->border_color, s->border_color.f);
1947          }
1948       }
1949    } else if (shader_type == PIPE_SHADER_GEOMETRY) {
1950       for (i = 0; i < draw->num_samplers[PIPE_SHADER_GEOMETRY]; i++) {
1951          struct draw_jit_sampler *jit_sam = &draw->llvm->gs_jit_context.samplers[i];
1952
1953          if (draw->samplers[i]) {
1954             const struct pipe_sampler_state *s
1955                = draw->samplers[PIPE_SHADER_GEOMETRY][i];
1956             jit_sam->min_lod = s->min_lod;
1957             jit_sam->max_lod = s->max_lod;
1958             jit_sam->lod_bias = s->lod_bias;
1959             COPY_4V(jit_sam->border_color, s->border_color.f);
1960          }
1961       }
1962    }
1963 }
1964
1965
1966 void
1967 draw_llvm_destroy_variant(struct draw_llvm_variant *variant)
1968 {
1969    struct draw_llvm *llvm = variant->llvm;
1970
1971    gallivm_destroy(variant->gallivm);
1972
1973    remove_from_list(&variant->list_item_local);
1974    variant->shader->variants_cached--;
1975    remove_from_list(&variant->list_item_global);
1976    llvm->nr_variants--;
1977    FREE(variant);
1978 }
1979
1980
1981 /**
1982  * Create LLVM types for various structures.
1983  */
1984 static void
1985 create_gs_jit_types(struct draw_gs_llvm_variant *var)
1986 {
1987    struct gallivm_state *gallivm = var->gallivm;
1988    LLVMTypeRef texture_type, sampler_type, context_type;
1989
1990    texture_type = create_jit_texture_type(gallivm, "texture");
1991    sampler_type = create_jit_sampler_type(gallivm, "sampler");
1992
1993    context_type = create_gs_jit_context_type(gallivm,
1994                                              var->shader->base.vector_length,
1995                                              texture_type, sampler_type,
1996                                              "draw_gs_jit_context");
1997    var->context_ptr_type = LLVMPointerType(context_type, 0);
1998
1999    var->input_array_type = create_gs_jit_input_type(gallivm);
2000 }
2001
2002 static LLVMTypeRef
2003 get_gs_context_ptr_type(struct draw_gs_llvm_variant *variant)
2004 {
2005    if (!variant->context_ptr_type)
2006       create_gs_jit_types(variant);
2007    return variant->context_ptr_type;
2008 }
2009
2010 static LLVMValueRef
2011 generate_mask_value(struct draw_gs_llvm_variant *variant,
2012                     struct lp_type gs_type)
2013 {
2014    struct gallivm_state *gallivm = variant->gallivm;
2015    LLVMBuilderRef builder = gallivm->builder;
2016    struct lp_type mask_type = lp_int_type(gs_type);
2017    LLVMValueRef num_prims;
2018    LLVMValueRef mask_val = lp_build_const_vec(gallivm, mask_type, 0);
2019    unsigned i;
2020
2021    num_prims = lp_build_broadcast(gallivm, lp_build_vec_type(gallivm, mask_type),
2022                                   variant->num_prims);
2023    for (i = 0; i <= gs_type.length; i++) {
2024       LLVMValueRef idx = lp_build_const_int32(gallivm, i);
2025       mask_val = LLVMBuildInsertElement(builder, mask_val, idx, idx, "");
2026    }
2027    mask_val = lp_build_compare(gallivm, mask_type,
2028                                PIPE_FUNC_GREATER, num_prims, mask_val);
2029
2030    return mask_val;
2031 }
2032
2033 static void
2034 draw_gs_llvm_generate(struct draw_llvm *llvm,
2035                       struct draw_gs_llvm_variant *variant)
2036 {
2037    struct gallivm_state *gallivm = variant->gallivm;
2038    LLVMContextRef context = gallivm->context;
2039    LLVMTypeRef int32_type = LLVMInt32TypeInContext(context);
2040    LLVMTypeRef arg_types[6];
2041    LLVMTypeRef func_type;
2042    LLVMValueRef variant_func;
2043    LLVMValueRef context_ptr;
2044    LLVMValueRef prim_id_ptr;
2045    LLVMBasicBlockRef block;
2046    LLVMBuilderRef builder;
2047    LLVMValueRef io_ptr, input_array, num_prims, mask_val;
2048    struct lp_build_sampler_soa *sampler = 0;
2049    struct lp_build_context bld;
2050    struct lp_bld_tgsi_system_values system_values;
2051    char func_name[64];
2052    struct lp_type gs_type;
2053    unsigned i;
2054    struct draw_gs_llvm_iface gs_iface;
2055    const struct tgsi_token *tokens = variant->shader->base.state.tokens;
2056    LLVMValueRef consts_ptr, num_consts_ptr;
2057    LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][TGSI_NUM_CHANNELS];
2058    struct lp_build_mask_context mask;
2059    const struct tgsi_shader_info *gs_info = &variant->shader->base.info;
2060    unsigned vector_length = variant->shader->base.vector_length;
2061
2062    memset(&system_values, 0, sizeof(system_values));
2063
2064    util_snprintf(func_name, sizeof(func_name), "draw_llvm_gs_variant%u",
2065                  variant->shader->variants_cached);
2066
2067    assert(variant->vertex_header_ptr_type);
2068
2069    arg_types[0] = get_gs_context_ptr_type(variant);    /* context */
2070    arg_types[1] = variant->input_array_type;           /* input */
2071    arg_types[2] = variant->vertex_header_ptr_type;     /* vertex_header */
2072    arg_types[3] = int32_type;                          /* num_prims */
2073    arg_types[4] = int32_type;                          /* instance_id */
2074    arg_types[5] = LLVMPointerType(
2075       LLVMVectorType(int32_type, vector_length), 0);   /* prim_id_ptr */
2076
2077    func_type = LLVMFunctionType(int32_type, arg_types, Elements(arg_types), 0);
2078
2079    variant_func = LLVMAddFunction(gallivm->module, func_name, func_type);
2080
2081    variant->function = variant_func;
2082
2083    LLVMSetFunctionCallConv(variant_func, LLVMCCallConv);
2084
2085    for (i = 0; i < Elements(arg_types); ++i)
2086       if (LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind)
2087          LLVMAddAttribute(LLVMGetParam(variant_func, i),
2088                           LLVMNoAliasAttribute);
2089
2090    context_ptr               = LLVMGetParam(variant_func, 0);
2091    input_array               = LLVMGetParam(variant_func, 1);
2092    io_ptr                    = LLVMGetParam(variant_func, 2);
2093    num_prims                 = LLVMGetParam(variant_func, 3);
2094    system_values.instance_id = LLVMGetParam(variant_func, 4);
2095    prim_id_ptr               = LLVMGetParam(variant_func, 5);
2096
2097    lp_build_name(context_ptr, "context");
2098    lp_build_name(input_array, "input");
2099    lp_build_name(io_ptr, "io");
2100    lp_build_name(num_prims, "num_prims");
2101    lp_build_name(system_values.instance_id, "instance_id");
2102    lp_build_name(prim_id_ptr, "prim_id_ptr");
2103
2104    variant->context_ptr = context_ptr;
2105    variant->io_ptr = io_ptr;
2106    variant->num_prims = num_prims;
2107
2108    gs_iface.base.fetch_input = draw_gs_llvm_fetch_input;
2109    gs_iface.base.emit_vertex = draw_gs_llvm_emit_vertex;
2110    gs_iface.base.end_primitive = draw_gs_llvm_end_primitive;
2111    gs_iface.base.gs_epilogue = draw_gs_llvm_epilogue;
2112    gs_iface.input = input_array;
2113    gs_iface.variant = variant;
2114
2115    /*
2116     * Function body
2117     */
2118
2119    block = LLVMAppendBasicBlockInContext(gallivm->context, variant_func, "entry");
2120    builder = gallivm->builder;
2121    LLVMPositionBuilderAtEnd(builder, block);
2122
2123    lp_build_context_init(&bld, gallivm, lp_type_int(32));
2124
2125    memset(&gs_type, 0, sizeof gs_type);
2126    gs_type.floating = TRUE; /* floating point values */
2127    gs_type.sign = TRUE;     /* values are signed */
2128    gs_type.norm = FALSE;    /* values are not limited to [0,1] or [-1,1] */
2129    gs_type.width = 32;      /* 32-bit float */
2130    gs_type.length = vector_length;
2131
2132    consts_ptr = draw_gs_jit_context_constants(variant->gallivm, context_ptr);
2133    num_consts_ptr =
2134       draw_gs_jit_context_num_constants(variant->gallivm, context_ptr);
2135
2136    /* code generated texture sampling */
2137    sampler = draw_llvm_sampler_soa_create(variant->key.samplers,
2138                                           context_ptr);
2139
2140    mask_val = generate_mask_value(variant, gs_type);
2141    lp_build_mask_begin(&mask, gallivm, gs_type, mask_val);
2142
2143    if (gs_info->uses_primid) {
2144       system_values.prim_id = LLVMBuildLoad(builder, prim_id_ptr, "prim_id");
2145    }
2146
2147    if (gallivm_debug & (GALLIVM_DEBUG_TGSI | GALLIVM_DEBUG_IR)) {
2148       tgsi_dump(tokens, 0);
2149       draw_gs_llvm_dump_variant_key(&variant->key);
2150    }
2151
2152    lp_build_tgsi_soa(variant->gallivm,
2153                      tokens,
2154                      gs_type,
2155                      &mask,
2156                      consts_ptr,
2157                      num_consts_ptr,
2158                      &system_values,
2159                      NULL,
2160                      outputs,
2161                      sampler,
2162                      &llvm->draw->gs.geometry_shader->info,
2163                      (const struct lp_build_tgsi_gs_iface *)&gs_iface);
2164
2165    sampler->destroy(sampler);
2166
2167    lp_build_mask_end(&mask);
2168
2169    LLVMBuildRet(builder, lp_build_zero(gallivm, lp_type_uint(32)));
2170
2171    gallivm_verify_function(gallivm, variant_func);
2172 }
2173
2174
2175 struct draw_gs_llvm_variant *
2176 draw_gs_llvm_create_variant(struct draw_llvm *llvm,
2177                             unsigned num_outputs,
2178                             const struct draw_gs_llvm_variant_key *key)
2179 {
2180    struct draw_gs_llvm_variant *variant;
2181    struct llvm_geometry_shader *shader =
2182       llvm_geometry_shader(llvm->draw->gs.geometry_shader);
2183    LLVMTypeRef vertex_header;
2184    char module_name[64];
2185
2186    variant = MALLOC(sizeof *variant +
2187                     shader->variant_key_size -
2188                     sizeof variant->key);
2189    if (variant == NULL)
2190       return NULL;
2191
2192    variant->llvm = llvm;
2193    variant->shader = shader;
2194
2195    util_snprintf(module_name, sizeof(module_name), "draw_llvm_gs_variant%u",
2196                  variant->shader->variants_cached);
2197
2198    variant->gallivm = gallivm_create(module_name);
2199
2200    create_gs_jit_types(variant);
2201
2202    memcpy(&variant->key, key, shader->variant_key_size);
2203
2204    vertex_header = create_jit_vertex_header(variant->gallivm, num_outputs);
2205
2206    variant->vertex_header_ptr_type = LLVMPointerType(vertex_header, 0);
2207
2208    draw_gs_llvm_generate(llvm, variant);
2209
2210    gallivm_compile_module(variant->gallivm);
2211
2212    variant->jit_func = (draw_gs_jit_func)
2213          gallivm_jit_function(variant->gallivm, variant->function);
2214
2215    gallivm_free_ir(variant->gallivm);
2216
2217    variant->list_item_global.base = variant;
2218    variant->list_item_local.base = variant;
2219    /*variant->no = */shader->variants_created++;
2220    variant->list_item_global.base = variant;
2221
2222    return variant;
2223 }
2224
2225 void
2226 draw_gs_llvm_destroy_variant(struct draw_gs_llvm_variant *variant)
2227 {
2228    struct draw_llvm *llvm = variant->llvm;
2229
2230    gallivm_destroy(variant->gallivm);
2231
2232    remove_from_list(&variant->list_item_local);
2233    variant->shader->variants_cached--;
2234    remove_from_list(&variant->list_item_global);
2235    llvm->nr_gs_variants--;
2236    FREE(variant);
2237 }
2238
2239 struct draw_gs_llvm_variant_key *
2240 draw_gs_llvm_make_variant_key(struct draw_llvm *llvm, char *store)
2241 {
2242    unsigned i;
2243    struct draw_gs_llvm_variant_key *key;
2244    struct draw_sampler_static_state *draw_sampler;
2245
2246    key = (struct draw_gs_llvm_variant_key *)store;
2247
2248    key->num_outputs = draw_total_gs_outputs(llvm->draw);
2249
2250    /* All variants of this shader will have the same value for
2251     * nr_samplers.  Not yet trying to compact away holes in the
2252     * sampler array.
2253     */
2254    key->nr_samplers = llvm->draw->gs.geometry_shader->info.file_max[TGSI_FILE_SAMPLER] + 1;
2255    if (llvm->draw->gs.geometry_shader->info.file_max[TGSI_FILE_SAMPLER_VIEW] != -1) {
2256       key->nr_sampler_views =
2257          llvm->draw->gs.geometry_shader->info.file_max[TGSI_FILE_SAMPLER_VIEW] + 1;
2258    }
2259    else {
2260       key->nr_sampler_views = key->nr_samplers;
2261    }
2262
2263    draw_sampler = key->samplers;
2264
2265    memset(draw_sampler, 0, MAX2(key->nr_samplers, key->nr_sampler_views) * sizeof *draw_sampler);
2266
2267    for (i = 0 ; i < key->nr_samplers; i++) {
2268       lp_sampler_static_sampler_state(&draw_sampler[i].sampler_state,
2269                                       llvm->draw->samplers[PIPE_SHADER_GEOMETRY][i]);
2270    }
2271    for (i = 0 ; i < key->nr_sampler_views; i++) {
2272       lp_sampler_static_texture_state(&draw_sampler[i].texture_state,
2273                                       llvm->draw->sampler_views[PIPE_SHADER_GEOMETRY][i]);
2274    }
2275
2276    return key;
2277 }
2278
2279 void
2280 draw_gs_llvm_dump_variant_key(struct draw_gs_llvm_variant_key *key)
2281 {
2282    unsigned i;
2283    struct draw_sampler_static_state *sampler = key->samplers;
2284
2285    for (i = 0 ; i < key->nr_sampler_views; i++) {
2286       debug_printf("sampler[%i].src_format = %s\n", i,
2287                    util_format_name(sampler[i].texture_state.format));
2288    }
2289 }