src/gallium/auxiliary/draw/draw_llvm.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2010 VMware, Inc.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  19  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  21  * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
  22  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  23  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  24  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27
  28 #include "draw_llvm.h"
  29
  30 #include "draw_context.h"
  31 #include "draw_vs.h"
  32 #include "draw_gs.h"
  33
  34 #include "gallivm/lp_bld_arit.h"
  35 #include "gallivm/lp_bld_logic.h"
  36 #include "gallivm/lp_bld_const.h"
  37 #include "gallivm/lp_bld_swizzle.h"
  38 #include "gallivm/lp_bld_struct.h"
  39 #include "gallivm/lp_bld_type.h"
  40 #include "gallivm/lp_bld_flow.h"
  41 #include "gallivm/lp_bld_debug.h"
  42 #include "gallivm/lp_bld_tgsi.h"
  43 #include "gallivm/lp_bld_printf.h"
  44 #include "gallivm/lp_bld_intr.h"
  45 #include "gallivm/lp_bld_init.h"
  46 #include "gallivm/lp_bld_type.h"
  47 #include "gallivm/lp_bld_pack.h"
  48 #include "gallivm/lp_bld_format.h"
  49
  50 #include "tgsi/tgsi_exec.h"
  51 #include "tgsi/tgsi_dump.h"
  52
  53 #include "util/u_math.h"
  54 #include "util/u_pointer.h"
  55 #include "util/u_string.h"
  56 #include "util/u_simple_list.h"
  57
  58
  59 #define DEBUG_STORE 0
  60
  61
  62 static void
  63 draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *var,
  64                    boolean elts);
  65
  66
  67 struct draw_gs_llvm_iface {
  68    struct lp_build_tgsi_gs_iface base;
  69
  70    struct draw_gs_llvm_variant *variant;
  71    LLVMValueRef input;
  72 };
  73
  74 static INLINE const struct draw_gs_llvm_iface *
  75 draw_gs_llvm_iface(const struct lp_build_tgsi_gs_iface *iface)
  76 {
  77    return (const struct draw_gs_llvm_iface *)iface;
  78 }
  79
  80 /**
  81  * Create LLVM type for struct draw_jit_texture
  82  */
  83 static LLVMTypeRef
  84 create_jit_texture_type(struct gallivm_state *gallivm, const char *struct_name)
  85 {
  86    LLVMTargetDataRef target = gallivm->target;
  87    LLVMTypeRef texture_type;
  88    LLVMTypeRef elem_types[DRAW_JIT_TEXTURE_NUM_FIELDS];
  89    LLVMTypeRef int32_type = LLVMInt32TypeInContext(gallivm->context);
  90
  91    elem_types[DRAW_JIT_TEXTURE_WIDTH]  =
  92    elem_types[DRAW_JIT_TEXTURE_HEIGHT] =
  93    elem_types[DRAW_JIT_TEXTURE_DEPTH] =
  94    elem_types[DRAW_JIT_TEXTURE_FIRST_LEVEL] =
  95    elem_types[DRAW_JIT_TEXTURE_LAST_LEVEL] = int32_type;
  96    elem_types[DRAW_JIT_TEXTURE_BASE] =
  97       LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0);
  98    elem_types[DRAW_JIT_TEXTURE_ROW_STRIDE] =
  99    elem_types[DRAW_JIT_TEXTURE_IMG_STRIDE] =
 100    elem_types[DRAW_JIT_TEXTURE_MIP_OFFSETS] =
 101       LLVMArrayType(int32_type, PIPE_MAX_TEXTURE_LEVELS);
 102
 103    texture_type = LLVMStructTypeInContext(gallivm->context, elem_types,
 104                                           Elements(elem_types), 0);
 105
 106 #if HAVE_LLVM < 0x0300
 107    LLVMAddTypeName(gallivm->module, struct_name, texture_type);
 108
 109    /* Make sure the target's struct layout cache doesn't return
 110     * stale/invalid data.
 111     */
 112    LLVMInvalidateStructLayout(gallivm->target, texture_type);
 113 #endif
 114
 115    LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, width,
 116                           target, texture_type,
 117                           DRAW_JIT_TEXTURE_WIDTH);
 118    LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, height,
 119                           target, texture_type,
 120                           DRAW_JIT_TEXTURE_HEIGHT);
 121    LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, depth,
 122                           target, texture_type,
 123                           DRAW_JIT_TEXTURE_DEPTH);
 124    LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, first_level,
 125                           target, texture_type,
 126                           DRAW_JIT_TEXTURE_FIRST_LEVEL);
 127    LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, last_level,
 128                           target, texture_type,
 129                           DRAW_JIT_TEXTURE_LAST_LEVEL);
 130    LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, base,
 131                           target, texture_type,
 132                           DRAW_JIT_TEXTURE_BASE);
 133    LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, row_stride,
 134                           target, texture_type,
 135                           DRAW_JIT_TEXTURE_ROW_STRIDE);
 136    LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, img_stride,
 137                           target, texture_type,
 138                           DRAW_JIT_TEXTURE_IMG_STRIDE);
 139    LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, mip_offsets,
 140                           target, texture_type,
 141                           DRAW_JIT_TEXTURE_MIP_OFFSETS);
 142
 143    LP_CHECK_STRUCT_SIZE(struct draw_jit_texture, target, texture_type);
 144
 145    return texture_type;
 146 }
 147
 148
 149 /**
 150  * Create LLVM type for struct draw_jit_sampler
 151  */
 152 static LLVMTypeRef
 153 create_jit_sampler_type(struct gallivm_state *gallivm, const char *struct_name)
 154 {
 155    LLVMTargetDataRef target = gallivm->target;
 156    LLVMTypeRef sampler_type;
 157    LLVMTypeRef elem_types[DRAW_JIT_SAMPLER_NUM_FIELDS];
 158
 159    elem_types[DRAW_JIT_SAMPLER_MIN_LOD] =
 160    elem_types[DRAW_JIT_SAMPLER_MAX_LOD] =
 161    elem_types[DRAW_JIT_SAMPLER_LOD_BIAS] = LLVMFloatTypeInContext(gallivm->context);
 162    elem_types[DRAW_JIT_SAMPLER_BORDER_COLOR] =
 163       LLVMArrayType(LLVMFloatTypeInContext(gallivm->context), 4);
 164
 165    sampler_type = LLVMStructTypeInContext(gallivm->context, elem_types,
 166                                           Elements(elem_types), 0);
 167
 168 #if HAVE_LLVM < 0x0300
 169    LLVMAddTypeName(gallivm->module, struct_name, sampler_type);
 170
 171    /* Make sure the target's struct layout cache doesn't return
 172     * stale/invalid data.
 173     */
 174    LLVMInvalidateStructLayout(gallivm->target, sampler_type);
 175 #endif
 176
 177    LP_CHECK_MEMBER_OFFSET(struct draw_jit_sampler, min_lod,
 178                           target, sampler_type,
 179                           DRAW_JIT_SAMPLER_MIN_LOD);
 180    LP_CHECK_MEMBER_OFFSET(struct draw_jit_sampler, max_lod,
 181                           target, sampler_type,
 182                           DRAW_JIT_SAMPLER_MAX_LOD);
 183    LP_CHECK_MEMBER_OFFSET(struct draw_jit_sampler, lod_bias,
 184                           target, sampler_type,
 185                           DRAW_JIT_SAMPLER_LOD_BIAS);
 186    LP_CHECK_MEMBER_OFFSET(struct draw_jit_sampler, border_color,
 187                           target, sampler_type,
 188                           DRAW_JIT_SAMPLER_BORDER_COLOR);
 189
 190    LP_CHECK_STRUCT_SIZE(struct draw_jit_sampler, target, sampler_type);
 191
 192    return sampler_type;
 193 }
 194
 195
 196 /**
 197  * Create LLVM type for struct draw_jit_context
 198  */
 199 static LLVMTypeRef
 200 create_jit_context_type(struct gallivm_state *gallivm,
 201                         LLVMTypeRef texture_type, LLVMTypeRef sampler_type,
 202                         const char *struct_name)
 203 {
 204    LLVMTargetDataRef target = gallivm->target;
 205    LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context);
 206    LLVMTypeRef elem_types[DRAW_JIT_CTX_NUM_FIELDS];
 207    LLVMTypeRef context_type;
 208
 209    elem_types[0] = LLVMArrayType(LLVMPointerType(float_type, 0), /* vs_constants */
 210                                  LP_MAX_TGSI_CONST_BUFFERS);
 211    elem_types[1] = LLVMPointerType(LLVMArrayType(LLVMArrayType(float_type, 4),
 212                                                  DRAW_TOTAL_CLIP_PLANES), 0);
 213    elem_types[2] = LLVMPointerType(float_type, 0); /* viewport */
 214    elem_types[3] = LLVMArrayType(texture_type,
 215                                  PIPE_MAX_SHADER_SAMPLER_VIEWS); /* textures */
 216    elem_types[4] = LLVMArrayType(sampler_type,
 217                                  PIPE_MAX_SAMPLERS); /* samplers */
 218    context_type = LLVMStructTypeInContext(gallivm->context, elem_types,
 219                                           Elements(elem_types), 0);
 220 #if HAVE_LLVM < 0x0300
 221    LLVMAddTypeName(gallivm->module, struct_name, context_type);
 222
 223    LLVMInvalidateStructLayout(gallivm->target, context_type);
 224 #endif
 225
 226    LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, vs_constants,
 227                           target, context_type, DRAW_JIT_CTX_CONSTANTS);
 228    LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, planes,
 229                           target, context_type, DRAW_JIT_CTX_PLANES);
 230    LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, viewport,
 231                           target, context_type, DRAW_JIT_CTX_VIEWPORT);
 232    LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, textures,
 233                           target, context_type,
 234                           DRAW_JIT_CTX_TEXTURES);
 235    LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, samplers,
 236                           target, context_type,
 237                           DRAW_JIT_CTX_SAMPLERS);
 238    LP_CHECK_STRUCT_SIZE(struct draw_jit_context,
 239                         target, context_type);
 240
 241    return context_type;
 242 }
 243
 244
 245 /**
 246  * Create LLVM type for struct draw_gs_jit_context
 247  */
 248 static LLVMTypeRef
 249 create_gs_jit_context_type(struct gallivm_state *gallivm,
 250                            unsigned vector_length,
 251                            LLVMTypeRef texture_type, LLVMTypeRef sampler_type,
 252                            const char *struct_name)
 253 {
 254    LLVMTargetDataRef target = gallivm->target;
 255    LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context);
 256    LLVMTypeRef int_type = LLVMInt32TypeInContext(gallivm->context);
 257    LLVMTypeRef elem_types[DRAW_GS_JIT_CTX_NUM_FIELDS];
 258    LLVMTypeRef context_type;
 259
 260    elem_types[0] = LLVMArrayType(LLVMPointerType(float_type, 0), /* constants */
 261                                  LP_MAX_TGSI_CONST_BUFFERS);
 262    elem_types[1] = LLVMPointerType(LLVMArrayType(LLVMArrayType(float_type, 4),
 263                                                  DRAW_TOTAL_CLIP_PLANES), 0);
 264    elem_types[2] = LLVMPointerType(float_type, 0); /* viewport */
 265
 266    elem_types[3] = LLVMArrayType(texture_type,
 267                                  PIPE_MAX_SHADER_SAMPLER_VIEWS); /* textures */
 268    elem_types[4] = LLVMArrayType(sampler_type,
 269                                  PIPE_MAX_SAMPLERS); /* samplers */
 270
 271    elem_types[5] = LLVMPointerType(LLVMPointerType(int_type, 0), 0);
 272    elem_types[6] = LLVMPointerType(LLVMVectorType(int_type,
 273                                                   vector_length), 0);
 274    elem_types[7] = LLVMPointerType(LLVMVectorType(int_type,
 275                                                   vector_length), 0);
 276
 277    context_type = LLVMStructTypeInContext(gallivm->context, elem_types,
 278                                           Elements(elem_types), 0);
 279 #if HAVE_LLVM < 0x0300
 280    LLVMAddTypeName(gallivm->module, struct_name, context_type);
 281
 282    LLVMInvalidateStructLayout(gallivm->target, context_type);
 283 #endif
 284
 285    LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, constants,
 286                           target, context_type, DRAW_GS_JIT_CTX_CONSTANTS);
 287    LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, planes,
 288                           target, context_type, DRAW_GS_JIT_CTX_PLANES);
 289    LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, viewport,
 290                           target, context_type, DRAW_GS_JIT_CTX_VIEWPORT);
 291    LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, textures,
 292                           target, context_type,
 293                           DRAW_GS_JIT_CTX_TEXTURES);
 294    LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, samplers,
 295                           target, context_type,
 296                           DRAW_GS_JIT_CTX_SAMPLERS);
 297    LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, prim_lengths,
 298                           target, context_type,
 299                           DRAW_GS_JIT_CTX_PRIM_LENGTHS);
 300    LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, emitted_vertices,
 301                           target, context_type,
 302                           DRAW_GS_JIT_CTX_EMITTED_VERTICES);
 303    LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, emitted_prims,
 304                           target, context_type,
 305                           DRAW_GS_JIT_CTX_EMITTED_PRIMS);
 306    LP_CHECK_STRUCT_SIZE(struct draw_gs_jit_context,
 307                         target, context_type);
 308
 309    return context_type;
 310 }
 311
 312
 313 static LLVMTypeRef
 314 create_gs_jit_input_type(struct gallivm_state *gallivm)
 315 {
 316    LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context);
 317    LLVMTypeRef input_array;
 318
 319    input_array = LLVMVectorType(float_type, TGSI_NUM_CHANNELS); /* num primitives */
 320    input_array = LLVMArrayType(input_array, TGSI_NUM_CHANNELS); /* num channels */
 321    input_array = LLVMArrayType(input_array, PIPE_MAX_SHADER_INPUTS); /* num attrs per vertex */
 322    input_array = LLVMPointerType(input_array, 0); /* num vertices per prim */
 323
 324    return input_array;
 325 }
 326
 327 /**
 328  * Create LLVM type for struct pipe_vertex_buffer
 329  */
 330 static LLVMTypeRef
 331 create_jit_vertex_buffer_type(struct gallivm_state *gallivm, const char *struct_name)
 332 {
 333    LLVMTargetDataRef target = gallivm->target;
 334    LLVMTypeRef elem_types[4];
 335    LLVMTypeRef vb_type;
 336
 337    elem_types[0] =
 338    elem_types[1] = LLVMInt32TypeInContext(gallivm->context);
 339    elem_types[2] =
 340    elem_types[3] = LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0); /* vs_constants */
 341
 342    vb_type = LLVMStructTypeInContext(gallivm->context, elem_types,
 343                                      Elements(elem_types), 0);
 344 #if HAVE_LLVM < 0x0300
 345    LLVMAddTypeName(gallivm->module, struct_name, vb_type);
 346
 347    LLVMInvalidateStructLayout(gallivm->target, vb_type);
 348 #endif
 349
 350    LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, stride,
 351                           target, vb_type, 0);
 352    LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, buffer_offset,
 353                           target, vb_type, 1);
 354
 355    LP_CHECK_STRUCT_SIZE(struct pipe_vertex_buffer, target, vb_type);
 356
 357    return vb_type;
 358 }
 359
 360
 361 /**
 362  * Create LLVM type for struct vertex_header;
 363  */
 364 static LLVMTypeRef
 365 create_jit_vertex_header(struct gallivm_state *gallivm, int data_elems)
 366 {
 367    LLVMTargetDataRef target = gallivm->target;
 368    LLVMTypeRef elem_types[4];
 369    LLVMTypeRef vertex_header;
 370    char struct_name[24];
 371
 372    util_snprintf(struct_name, 23, "vertex_header%d", data_elems);
 373
 374    elem_types[DRAW_JIT_VERTEX_VERTEX_ID]  = LLVMIntTypeInContext(gallivm->context, 32);
 375    elem_types[DRAW_JIT_VERTEX_CLIP]  = LLVMArrayType(LLVMFloatTypeInContext(gallivm->context), 4);
 376    elem_types[DRAW_JIT_VERTEX_PRE_CLIP_POS]  = LLVMArrayType(LLVMFloatTypeInContext(gallivm->context), 4);
 377    elem_types[DRAW_JIT_VERTEX_DATA]  = LLVMArrayType(elem_types[1], data_elems);
 378
 379    vertex_header = LLVMStructTypeInContext(gallivm->context, elem_types,
 380                                            Elements(elem_types), 0);
 381 #if HAVE_LLVM < 0x0300
 382    LLVMAddTypeName(gallivm->module, struct_name, vertex_header);
 383
 384    LLVMInvalidateStructLayout(gallivm->target, vertex_header);
 385 #endif
 386
 387    /* these are bit-fields and we can't take address of them
 388       LP_CHECK_MEMBER_OFFSET(struct vertex_header, clipmask,
 389       target, vertex_header,
 390       DRAW_JIT_VERTEX_CLIPMASK);
 391       LP_CHECK_MEMBER_OFFSET(struct vertex_header, edgeflag,
 392       target, vertex_header,
 393       DRAW_JIT_VERTEX_EDGEFLAG);
 394       LP_CHECK_MEMBER_OFFSET(struct vertex_header, pad,
 395       target, vertex_header,
 396       DRAW_JIT_VERTEX_PAD);
 397       LP_CHECK_MEMBER_OFFSET(struct vertex_header, vertex_id,
 398       target, vertex_header,
 399       DRAW_JIT_VERTEX_VERTEX_ID);
 400    */
 401    LP_CHECK_MEMBER_OFFSET(struct vertex_header, clip,
 402                           target, vertex_header,
 403                           DRAW_JIT_VERTEX_CLIP);
 404    LP_CHECK_MEMBER_OFFSET(struct vertex_header, pre_clip_pos,
 405                           target, vertex_header,
 406                           DRAW_JIT_VERTEX_PRE_CLIP_POS);
 407    LP_CHECK_MEMBER_OFFSET(struct vertex_header, data,
 408                           target, vertex_header,
 409                           DRAW_JIT_VERTEX_DATA);
 410
 411    assert(LLVMABISizeOfType(target, vertex_header) ==
 412           offsetof(struct vertex_header, data[data_elems]));
 413
 414    return vertex_header;
 415 }
 416
 417
 418 /**
 419  * Create LLVM types for various structures.
 420  */
 421 static void
 422 create_jit_types(struct draw_llvm_variant *variant)
 423 {
 424    struct gallivm_state *gallivm = variant->gallivm;
 425    LLVMTypeRef texture_type, sampler_type, context_type, buffer_type, vb_type;
 426
 427    texture_type = create_jit_texture_type(gallivm, "texture");
 428    sampler_type = create_jit_sampler_type(gallivm, "sampler");
 429
 430    context_type = create_jit_context_type(gallivm, texture_type, sampler_type,
 431                                           "draw_jit_context");
 432    variant->context_ptr_type = LLVMPointerType(context_type, 0);
 433
 434    buffer_type = LLVMPointerType(LLVMIntTypeInContext(gallivm->context, 8), 0);
 435    variant->buffer_ptr_type = LLVMPointerType(buffer_type, 0);
 436
 437    vb_type = create_jit_vertex_buffer_type(gallivm, "pipe_vertex_buffer");
 438    variant->vb_ptr_type = LLVMPointerType(vb_type, 0);
 439 }
 440
 441
 442 static LLVMTypeRef
 443 get_context_ptr_type(struct draw_llvm_variant *variant)
 444 {
 445    if (!variant->context_ptr_type)
 446       create_jit_types(variant);
 447    return variant->context_ptr_type;
 448 }
 449
 450
 451 static LLVMTypeRef
 452 get_buffer_ptr_type(struct draw_llvm_variant *variant)
 453 {
 454    if (!variant->buffer_ptr_type)
 455       create_jit_types(variant);
 456    return variant->buffer_ptr_type;
 457 }
 458
 459
 460 static LLVMTypeRef
 461 get_vb_ptr_type(struct draw_llvm_variant *variant)
 462 {
 463    if (!variant->vb_ptr_type)
 464       create_jit_types(variant);
 465    return variant->vb_ptr_type;
 466 }
 467
 468 static LLVMTypeRef
 469 get_vertex_header_ptr_type(struct draw_llvm_variant *variant)
 470 {
 471    if (!variant->vertex_header_ptr_type)
 472       create_jit_types(variant);
 473    return variant->vertex_header_ptr_type;
 474 }
 475
 476
 477 /**
 478  * Create per-context LLVM info.
 479  */
 480 struct draw_llvm *
 481 draw_llvm_create(struct draw_context *draw)
 482 {
 483    struct draw_llvm *llvm;
 484
 485    llvm = CALLOC_STRUCT( draw_llvm );
 486    if (!llvm)
 487       return NULL;
 488
 489    lp_build_init();
 490
 491    llvm->draw = draw;
 492
 493    llvm->nr_variants = 0;
 494    make_empty_list(&llvm->vs_variants_list);
 495
 496    llvm->nr_gs_variants = 0;
 497    make_empty_list(&llvm->gs_variants_list);
 498
 499    return llvm;
 500 }
 501
 502
 503 /**
 504  * Free per-context LLVM info.
 505  */
 506 void
 507 draw_llvm_destroy(struct draw_llvm *llvm)
 508 {
 509    /* XXX free other draw_llvm data? */
 510    FREE(llvm);
 511 }
 512
 513
 514 /**
 515  * Create LLVM-generated code for a vertex shader.
 516  */
 517 struct draw_llvm_variant *
 518 draw_llvm_create_variant(struct draw_llvm *llvm,
 519                          unsigned num_inputs,
 520                          const struct draw_llvm_variant_key *key)
 521 {
 522    struct draw_llvm_variant *variant;
 523    struct llvm_vertex_shader *shader =
 524       llvm_vertex_shader(llvm->draw->vs.vertex_shader);
 525    LLVMTypeRef vertex_header;
 526
 527    variant = MALLOC(sizeof *variant +
 528                     shader->variant_key_size -
 529                     sizeof variant->key);
 530    if (variant == NULL)
 531       return NULL;
 532
 533    variant->llvm = llvm;
 534
 535    variant->gallivm = gallivm_create();
 536
 537    create_jit_types(variant);
 538
 539    memcpy(&variant->key, key, shader->variant_key_size);
 540
 541    vertex_header = create_jit_vertex_header(variant->gallivm, num_inputs);
 542
 543    variant->vertex_header_ptr_type = LLVMPointerType(vertex_header, 0);
 544
 545    draw_llvm_generate(llvm, variant, FALSE);  /* linear */
 546    draw_llvm_generate(llvm, variant, TRUE);   /* elts */
 547
 548    gallivm_compile_module(variant->gallivm);
 549
 550    variant->jit_func = (draw_jit_vert_func)
 551          gallivm_jit_function(variant->gallivm, variant->function);
 552
 553    variant->jit_func_elts = (draw_jit_vert_func_elts)
 554          gallivm_jit_function(variant->gallivm, variant->function_elts);
 555
 556    variant->shader = shader;
 557    variant->list_item_global.base = variant;
 558    variant->list_item_local.base = variant;
 559    /*variant->no = */shader->variants_created++;
 560    variant->list_item_global.base = variant;
 561
 562    return variant;
 563 }
 564
 565
 566 static void
 567 generate_vs(struct draw_llvm_variant *variant,
 568             LLVMBuilderRef builder,
 569             struct lp_type vs_type,
 570             LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS],
 571             const LLVMValueRef (*inputs)[TGSI_NUM_CHANNELS],
 572             const struct lp_bld_tgsi_system_values *system_values,
 573             LLVMValueRef context_ptr,
 574             struct lp_build_sampler_soa *draw_sampler,
 575             boolean clamp_vertex_color)
 576 {
 577    struct draw_llvm *llvm = variant->llvm;
 578    const struct tgsi_token *tokens = llvm->draw->vs.vertex_shader->state.tokens;
 579    LLVMValueRef consts_ptr = draw_jit_context_vs_constants(variant->gallivm, context_ptr);
 580    struct lp_build_sampler_soa *sampler = 0;
 581
 582    if (gallivm_debug & (GALLIVM_DEBUG_TGSI | GALLIVM_DEBUG_IR)) {
 583       tgsi_dump(tokens, 0);
 584       draw_llvm_dump_variant_key(&variant->key);
 585    }
 586
 587    if (llvm->draw->num_sampler_views && llvm->draw->num_samplers)
 588       sampler = draw_sampler;
 589
 590    lp_build_tgsi_soa(variant->gallivm,
 591                      tokens,
 592                      vs_type,
 593                      NULL /*struct lp_build_mask_context *mask*/,
 594                      consts_ptr,
 595                      system_values,
 596                      NULL /*pos*/,
 597                      inputs,
 598                      outputs,
 599                      sampler,
 600                      &llvm->draw->vs.vertex_shader->info,
 601                      NULL);
 602
 603    {
 604       LLVMValueRef out;
 605       unsigned chan, attrib;
 606       struct lp_build_context bld;
 607       struct tgsi_shader_info* info = &llvm->draw->vs.vertex_shader->info;
 608       lp_build_context_init(&bld, variant->gallivm, vs_type);
 609
 610       for (attrib = 0; attrib < info->num_outputs; ++attrib) {
 611          for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
 612             if (outputs[attrib][chan]) {
 613                switch (info->output_semantic_name[attrib]) {
 614                case TGSI_SEMANTIC_COLOR:
 615                case TGSI_SEMANTIC_BCOLOR:
 616                   if (clamp_vertex_color) {
 617                      out = LLVMBuildLoad(builder, outputs[attrib][chan], "");
 618                      out = lp_build_clamp(&bld, out, bld.zero, bld.one);
 619                      LLVMBuildStore(builder, out, outputs[attrib][chan]);
 620                   }
 621                   break;
 622                case TGSI_SEMANTIC_FOG:
 623                   if (chan == 1 || chan == 2)
 624                      LLVMBuildStore(builder, bld.zero, outputs[attrib][chan]);
 625                   else if (chan == 3)
 626                      LLVMBuildStore(builder, bld.one, outputs[attrib][chan]);
 627                   break;
 628                }
 629             }
 630          }
 631       }
 632    }
 633 }
 634
 635
 636 static void
 637 generate_fetch(struct gallivm_state *gallivm,
 638                LLVMValueRef vbuffers_ptr,
 639                LLVMValueRef *res,
 640                struct pipe_vertex_element *velem,
 641                LLVMValueRef vbuf,
 642                LLVMValueRef index,
 643                LLVMValueRef instance_id)
 644 {
 645    const struct util_format_description *format_desc = util_format_description(velem->src_format);
 646    LLVMValueRef zero = LLVMConstNull(LLVMInt32TypeInContext(gallivm->context));
 647    LLVMBuilderRef builder = gallivm->builder;
 648    LLVMValueRef indices =
 649       LLVMConstInt(LLVMInt64TypeInContext(gallivm->context),
 650                    velem->vertex_buffer_index, 0);
 651    LLVMValueRef vbuffer_ptr = LLVMBuildGEP(builder, vbuffers_ptr,
 652                                            &indices, 1, "");
 653    LLVMValueRef vb_stride = draw_jit_vbuffer_stride(gallivm, vbuf);
 654    LLVMValueRef vb_buffer_offset = draw_jit_vbuffer_offset(gallivm, vbuf);
 655    LLVMValueRef stride;
 656
 657    if (velem->instance_divisor) {
 658       /* array index = instance_id / instance_divisor */
 659       index = LLVMBuildUDiv(builder, instance_id,
 660                             lp_build_const_int32(gallivm, velem->instance_divisor),
 661                             "instance_divisor");
 662    }
 663
 664    stride = LLVMBuildMul(builder, vb_stride, index, "");
 665
 666    vbuffer_ptr = LLVMBuildLoad(builder, vbuffer_ptr, "vbuffer");
 667
 668    stride = LLVMBuildAdd(builder, stride,
 669                          vb_buffer_offset,
 670                          "");
 671    stride = LLVMBuildAdd(builder, stride,
 672                          lp_build_const_int32(gallivm, velem->src_offset),
 673                          "");
 674
 675 /*   lp_build_printf(gallivm, "vbuf index = %d, stride is %d\n", indices, stride);*/
 676    vbuffer_ptr = LLVMBuildGEP(builder, vbuffer_ptr, &stride, 1, "");
 677
 678    *res = lp_build_fetch_rgba_aos(gallivm,
 679                                   format_desc,
 680                                   lp_float32_vec4_type(),
 681                                   vbuffer_ptr,
 682                                   zero, zero, zero);
 683 }
 684
 685 static void
 686 convert_to_soa(struct gallivm_state *gallivm,
 687                LLVMValueRef (*src_aos)[LP_MAX_VECTOR_WIDTH / 32],
 688                LLVMValueRef (*dst_soa)[TGSI_NUM_CHANNELS],
 689                unsigned num_attribs, const struct lp_type soa_type)
 690 {
 691    unsigned i, j, k;
 692    struct lp_type aos_channel_type = soa_type;
 693
 694    debug_assert(TGSI_NUM_CHANNELS == 4);
 695    debug_assert((soa_type.length % TGSI_NUM_CHANNELS) == 0);
 696
 697    aos_channel_type.length >>= 1;
 698
 699    for (i = 0; i < num_attribs; ++i) {
 700       LLVMValueRef aos_channels[TGSI_NUM_CHANNELS];
 701       unsigned pixels_per_channel = soa_type.length / TGSI_NUM_CHANNELS;
 702
 703       for (j = 0; j < TGSI_NUM_CHANNELS; ++j) {
 704          LLVMValueRef channel[LP_MAX_VECTOR_LENGTH] = { 0 };
 705
 706          assert(pixels_per_channel <= LP_MAX_VECTOR_LENGTH);
 707
 708          for (k = 0; k < pixels_per_channel; ++k) {
 709             channel[k] = src_aos[i][j + TGSI_NUM_CHANNELS * k];
 710          }
 711
 712          aos_channels[j] = lp_build_concat(gallivm, channel, aos_channel_type, pixels_per_channel);
 713       }
 714
 715       lp_build_transpose_aos(gallivm, soa_type, aos_channels, dst_soa[i]);
 716    }
 717 }
 718
 719
 720 static void
 721 store_aos(struct gallivm_state *gallivm,
 722           LLVMValueRef io_ptr,
 723           LLVMValueRef index,
 724           LLVMValueRef value)
 725 {
 726    LLVMTypeRef data_ptr_type = LLVMPointerType(lp_build_vec_type(gallivm, lp_float32_vec4_type()), 0);
 727    LLVMBuilderRef builder = gallivm->builder;
 728    LLVMValueRef data_ptr = draw_jit_header_data(gallivm, io_ptr);
 729    LLVMValueRef indices[3];
 730
 731    indices[0] = lp_build_const_int32(gallivm, 0);
 732    indices[1] = index;
 733    indices[2] = lp_build_const_int32(gallivm, 0);
 734
 735    data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 3, "");
 736    data_ptr = LLVMBuildPointerCast(builder, data_ptr, data_ptr_type, "");
 737
 738 #if DEBUG_STORE
 739    lp_build_printf(gallivm, "    ---- %p storing attribute %d (io = %p)\n", data_ptr, index, io_ptr);
 740 #endif
 741
 742    /* Unaligned store due to the vertex header */
 743    lp_set_store_alignment(LLVMBuildStore(builder, value, data_ptr), sizeof(float));
 744 }
 745
 746 /**
 747  * Adjust the mask to architecture endianess. The mask will the store in struct:
 748  *
 749  * struct vertex_header {
 750  *    unsigned clipmask:DRAW_TOTAL_CLIP_PLANES;
 751  *    unsigned edgeflag:1;
 752  *    unsigned have_clipdist:1;
 753  *    unsigned vertex_id:16;
 754  *    [...]
 755  * }
 756  *
 757  * On little-endian machine nothing needs to done, however on bit-endian machine
 758  * the mask's fields need to be adjusted with the algorithm:
 759  *
 760  * uint32_t reverse (uint32_t x)
 761  * {
 762  *   return (x >> 16) |              // vertex_id
 763  *          ((x & 0x3fff) << 18) |   // clipmask
 764  *          ((x & 0x4000) << 3) |    // have_clipdist
 765  *          ((x & 0x8000) << 1);     // edgeflag
 766  * }
 767  */
 768 static LLVMValueRef
 769 adjust_mask(struct gallivm_state *gallivm,
 770             LLVMValueRef mask)
 771 {
 772 #ifdef PIPE_ARCH_BIG_ENDIAN
 773    LLVMBuilderRef builder = gallivm->builder;
 774    LLVMValueRef vertex_id;
 775    LLVMValueRef clipmask;
 776    LLVMValueRef have_clipdist;
 777    LLVMValueRef edgeflag;
 778
 779    vertex_id = LLVMBuildLShr(builder, mask, lp_build_const_int32(gallivm, 16), "");
 780    clipmask  = LLVMBuildAnd(builder, mask, lp_build_const_int32(gallivm, 0x3fff), "");
 781    clipmask  = LLVMBuildShl(builder, clipmask, lp_build_const_int32(gallivm, 18), "");
 782    have_clipdist = LLVMBuildAnd(builder, mask, lp_build_const_int32(gallivm, 0x4000), "");
 783    have_clipdist = LLVMBuildShl(builder, have_clipdist, lp_build_const_int32(gallivm, 3), "");
 784    edgeflag = LLVMBuildAnd(builder, mask, lp_build_const_int32(gallivm, 0x8000), "");
 785    edgeflag = LLVMBuildShl(builder, edgeflag, lp_build_const_int32(gallivm, 1), "");
 786
 787    mask = LLVMBuildOr(builder, vertex_id, clipmask, "");
 788    mask = LLVMBuildOr(builder, mask, have_clipdist, "");
 789    mask = LLVMBuildOr(builder, mask, edgeflag, "");
 790 #endif
 791    return mask;
 792 }
 793
 794 static void
 795 store_aos_array(struct gallivm_state *gallivm,
 796                 struct lp_type soa_type,
 797                 LLVMValueRef io_ptr,
 798                 LLVMValueRef *indices,
 799                 LLVMValueRef* aos,
 800                 int attrib,
 801                 int num_outputs,
 802                 LLVMValueRef clipmask,
 803                 boolean have_clipdist)
 804 {
 805    LLVMBuilderRef builder = gallivm->builder;
 806    LLVMValueRef attr_index = lp_build_const_int32(gallivm, attrib);
 807    LLVMValueRef inds[LP_MAX_VECTOR_WIDTH / 32];
 808    LLVMValueRef io_ptrs[LP_MAX_VECTOR_WIDTH / 32];
 809    int vector_length = soa_type.length;
 810    int i;
 811
 812    debug_assert(TGSI_NUM_CHANNELS == 4);
 813
 814    for (i = 0; i < vector_length; i++) {
 815       if (indices) {
 816          inds[i] = indices[i];
 817       } else {
 818          inds[i] = lp_build_const_int32(gallivm, i);
 819       }
 820       io_ptrs[i] = LLVMBuildGEP(builder, io_ptr, &inds[i], 1, "");
 821    }
 822
 823    if (attrib == 0) {
 824       /* store vertex header for each of the n vertices */
 825       LLVMValueRef val, cliptmp;
 826       int vertex_id_pad_edgeflag;
 827
 828       /* If this assertion fails, it means we need to update the bit twidding
 829        * code here.  See struct vertex_header in draw_private.h.
 830        */
 831       assert(DRAW_TOTAL_CLIP_PLANES==14);
 832       /* initialize vertex id:16 = 0xffff, have_clipdist:1 = 0, edgeflag:1 = 1 */
 833       vertex_id_pad_edgeflag = (0xffff << 16) | (1 << DRAW_TOTAL_CLIP_PLANES);
 834       if (have_clipdist)
 835          vertex_id_pad_edgeflag |= 1 << (DRAW_TOTAL_CLIP_PLANES+1);
 836       val = lp_build_const_int_vec(gallivm, lp_int_type(soa_type), vertex_id_pad_edgeflag);
 837       /* OR with the clipmask */
 838       cliptmp = LLVMBuildOr(builder, val, clipmask, "");
 839       for (i = 0; i < vector_length; i++) {
 840          LLVMValueRef id_ptr = draw_jit_header_id(gallivm, io_ptrs[i]);
 841          val = LLVMBuildExtractElement(builder, cliptmp, inds[i], "");
 842          val = adjust_mask(gallivm, val);
 843          LLVMBuildStore(builder, val, id_ptr);
 844 #if DEBUG_STORE
 845          lp_build_printf(gallivm, "io = %p, index %d, clipmask = %x\n",
 846                          io_ptrs[i], inds[i], val);
 847 #endif
 848       }
 849    }
 850
 851    /* store for each of the n vertices */
 852    for (i = 0; i < vector_length; i++) {
 853       store_aos(gallivm, io_ptrs[i], attr_index, aos[i]);
 854    }
 855 }
 856
 857
 858 static void
 859 convert_to_aos(struct gallivm_state *gallivm,
 860                LLVMValueRef io,
 861                LLVMValueRef *indices,
 862                LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS],
 863                LLVMValueRef clipmask,
 864                int num_outputs,
 865                struct lp_type soa_type,
 866                boolean have_clipdist)
 867 {
 868    LLVMBuilderRef builder = gallivm->builder;
 869    unsigned chan, attrib, i;
 870
 871 #if DEBUG_STORE
 872    lp_build_printf(gallivm, "   # storing begin\n");
 873 #endif
 874    for (attrib = 0; attrib < num_outputs; ++attrib) {
 875       LLVMValueRef soa[TGSI_NUM_CHANNELS];
 876       LLVMValueRef aos[LP_MAX_VECTOR_WIDTH / 32];
 877       for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
 878          if (outputs[attrib][chan]) {
 879             LLVMValueRef out = LLVMBuildLoad(builder, outputs[attrib][chan], "");
 880             lp_build_name(out, "output%u.%c", attrib, "xyzw"[chan]);
 881 #if DEBUG_STORE
 882             lp_build_printf(gallivm, "output %d : %d ",
 883                             LLVMConstInt(LLVMInt32TypeInContext(gallivm->context),
 884                                          attrib, 0),
 885                             LLVMConstInt(LLVMInt32TypeInContext(gallivm->context),
 886                                          chan, 0));
 887             lp_build_print_value(gallivm, "val = ", out);
 888 #endif
 889             soa[chan] = out;
 890          }
 891          else {
 892             soa[chan] = 0;
 893          }
 894       }
 895
 896
 897       if (soa_type.length == TGSI_NUM_CHANNELS) {
 898          lp_build_transpose_aos(gallivm, soa_type, soa, aos);
 899       } else {
 900          lp_build_transpose_aos(gallivm, soa_type, soa, soa);
 901
 902          for (i = 0; i < soa_type.length; ++i) {
 903             aos[i] = lp_build_extract_range(gallivm,
 904                                             soa[i % TGSI_NUM_CHANNELS],
 905                                             (i / TGSI_NUM_CHANNELS) * TGSI_NUM_CHANNELS,
 906                                             TGSI_NUM_CHANNELS);
 907          }
 908       }
 909
 910       store_aos_array(gallivm,
 911                       soa_type,
 912                       io, indices,
 913                       aos,
 914                       attrib,
 915                       num_outputs,
 916                       clipmask, have_clipdist);
 917    }
 918 #if DEBUG_STORE
 919    lp_build_printf(gallivm, "   # storing end\n");
 920 #endif
 921 }
 922
 923
 924 /**
 925  * Stores original vertex positions in clip coordinates
 926  */
 927 static void
 928 store_clip(struct gallivm_state *gallivm,
 929            const struct lp_type vs_type,
 930            LLVMValueRef io_ptr,
 931            LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS],
 932            boolean pre_clip_pos, int idx)
 933 {
 934    LLVMBuilderRef builder = gallivm->builder;
 935    LLVMValueRef soa[4];
 936    LLVMValueRef aos[LP_MAX_VECTOR_LENGTH];
 937    LLVMValueRef indices[2];
 938    LLVMValueRef io_ptrs[LP_MAX_VECTOR_WIDTH / 32];
 939    LLVMValueRef inds[LP_MAX_VECTOR_WIDTH / 32];
 940    LLVMValueRef clip_ptrs[LP_MAX_VECTOR_WIDTH / 32];
 941    int i, j;
 942
 943    indices[0] =
 944    indices[1] = lp_build_const_int32(gallivm, 0);
 945
 946    for (i = 0; i < vs_type.length; i++) {
 947       inds[i] = lp_build_const_int32(gallivm, i);
 948       io_ptrs[i] = LLVMBuildGEP(builder, io_ptr, &inds[i], 1, "");
 949    }
 950
 951    soa[0] = LLVMBuildLoad(builder, outputs[idx][0], ""); /*x0 x1 .. xn*/
 952    soa[1] = LLVMBuildLoad(builder, outputs[idx][1], ""); /*y0 y1 .. yn*/
 953    soa[2] = LLVMBuildLoad(builder, outputs[idx][2], ""); /*z0 z1 .. zn*/
 954    soa[3] = LLVMBuildLoad(builder, outputs[idx][3], ""); /*w0 w1 .. wn*/
 955
 956    if (!pre_clip_pos) {
 957       for (i = 0; i < vs_type.length; i++) {
 958          clip_ptrs[i] = draw_jit_header_clip(gallivm, io_ptrs[i]);
 959       }
 960    } else {
 961       for (i = 0; i < vs_type.length; i++) {
 962          clip_ptrs[i] = draw_jit_header_pre_clip_pos(gallivm, io_ptrs[i]);
 963       }
 964    }
 965
 966    lp_build_transpose_aos(gallivm, vs_type, soa, soa);
 967    for (i = 0; i < vs_type.length; ++i) {
 968       aos[i] = lp_build_extract_range(gallivm,
 969                                       soa[i % TGSI_NUM_CHANNELS],
 970                                       (i / TGSI_NUM_CHANNELS) * TGSI_NUM_CHANNELS,
 971                                       TGSI_NUM_CHANNELS);
 972    }
 973
 974    for (j = 0; j < vs_type.length; j++) {
 975       LLVMTypeRef  clip_ptr_type = LLVMPointerType(LLVMVectorType(LLVMFloatTypeInContext(gallivm->context), 4), 0);
 976       LLVMValueRef clip_ptr;
 977
 978       clip_ptr = LLVMBuildGEP(builder, clip_ptrs[j], indices, 2, "clipo");
 979       clip_ptr = LLVMBuildPointerCast(builder, clip_ptr, clip_ptr_type, "");
 980
 981       /* Unaligned store */
 982       lp_set_store_alignment(LLVMBuildStore(builder, aos[j], clip_ptr), sizeof(float));
 983    }
 984 }
 985
 986
 987 /**
 988  * Transforms the outputs for viewport mapping
 989  */
 990 static void
 991 generate_viewport(struct draw_llvm_variant *variant,
 992                   LLVMBuilderRef builder,
 993                   struct lp_type vs_type,
 994                   LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS],
 995                   LLVMValueRef context_ptr)
 996 {
 997    int i;
 998    struct gallivm_state *gallivm = variant->gallivm;
 999    struct lp_type f32_type = vs_type;
1000    LLVMTypeRef vs_type_llvm = lp_build_vec_type(gallivm, vs_type);
1001    LLVMValueRef out3 = LLVMBuildLoad(builder, outputs[0][3], ""); /*w0 w1 .. wn*/
1002    LLVMValueRef const1 = lp_build_const_vec(gallivm, f32_type, 1.0);       /*1.0 1.0 1.0 1.0*/
1003    LLVMValueRef vp_ptr = draw_jit_context_viewport(gallivm, context_ptr);
1004
1005    /* for 1/w convention*/
1006    out3 = LLVMBuildFDiv(builder, const1, out3, "");
1007    LLVMBuildStore(builder, out3, outputs[0][3]);
1008
1009    /* Viewport Mapping */
1010    for (i=0; i<3; i++) {
1011       LLVMValueRef out = LLVMBuildLoad(builder, outputs[0][i], ""); /*x0 x1 .. xn*/
1012       LLVMValueRef scale;
1013       LLVMValueRef trans;
1014       LLVMValueRef scale_i;
1015       LLVMValueRef trans_i;
1016       LLVMValueRef index;
1017
1018       index = lp_build_const_int32(gallivm, i);
1019       scale_i = LLVMBuildGEP(builder, vp_ptr, &index, 1, "");
1020
1021       index = lp_build_const_int32(gallivm, i+4);
1022       trans_i = LLVMBuildGEP(builder, vp_ptr, &index, 1, "");
1023
1024       scale = lp_build_broadcast(gallivm, vs_type_llvm,
1025                                  LLVMBuildLoad(builder, scale_i, "scale"));
1026       trans = lp_build_broadcast(gallivm, vs_type_llvm,
1027                                  LLVMBuildLoad(builder, trans_i, "trans"));
1028
1029       /* divide by w */
1030       out = LLVMBuildFMul(builder, out, out3, "");
1031       /* mult by scale */
1032       out = LLVMBuildFMul(builder, out, scale, "");
1033       /* add translation */
1034       out = LLVMBuildFAdd(builder, out, trans, "");
1035
1036       /* store transformed outputs */
1037       LLVMBuildStore(builder, out, outputs[0][i]);
1038    }
1039
1040 }
1041
1042
1043 /**
1044  * Returns clipmask as nxi32 bitmask for the n vertices
1045  */
1046 static LLVMValueRef
1047 generate_clipmask(struct draw_llvm *llvm,
1048                   struct gallivm_state *gallivm,
1049                   struct lp_type vs_type,
1050                   LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS],
1051                   boolean clip_xy,
1052                   boolean clip_z,
1053                   boolean clip_user,
1054                   boolean clip_halfz,
1055                   unsigned ucp_enable,
1056                   LLVMValueRef context_ptr,
1057                   boolean *have_clipdist)
1058 {
1059    LLVMBuilderRef builder = gallivm->builder;
1060    LLVMValueRef mask; /* stores the <nxi32> clipmasks */
1061    LLVMValueRef test, temp;
1062    LLVMValueRef zero, shift;
1063    LLVMValueRef pos_x, pos_y, pos_z, pos_w;
1064    LLVMValueRef cv_x, cv_y, cv_z, cv_w;
1065    LLVMValueRef plane1, planes, plane_ptr, sum;
1066    struct lp_type f32_type = vs_type;
1067    struct lp_type i32_type = lp_int_type(vs_type);
1068    const unsigned pos = draw_current_shader_position_output(llvm->draw);
1069    const unsigned cv = draw_current_shader_clipvertex_output(llvm->draw);
1070    int num_written_clipdistance = llvm->draw->vs.vertex_shader->info.num_written_clipdistance;
1071    bool have_cd = false;
1072    unsigned cd[2];
1073
1074    cd[0] = draw_current_shader_clipdistance_output(llvm->draw, 0);
1075    cd[1] = draw_current_shader_clipdistance_output(llvm->draw, 1);
1076
1077    if (cd[0] != pos || cd[1] != pos)
1078       have_cd = true;
1079
1080    mask = lp_build_const_int_vec(gallivm, i32_type, 0);
1081    temp = lp_build_const_int_vec(gallivm, i32_type, 0);
1082    zero = lp_build_const_vec(gallivm, f32_type, 0);         /* 0.0f 0.0f 0.0f 0.0f */
1083    shift = lp_build_const_int_vec(gallivm, i32_type, 1);    /* 1 1 1 1 */
1084
1085    /*
1086     * load clipvertex and position from correct locations.
1087     * if they are the same just load them once.
1088     */
1089    pos_x = LLVMBuildLoad(builder, outputs[pos][0], ""); /*x0 x1 .. xn */
1090    pos_y = LLVMBuildLoad(builder, outputs[pos][1], ""); /*y0 y1 .. yn */
1091    pos_z = LLVMBuildLoad(builder, outputs[pos][2], ""); /*z0 z1 .. zn */
1092    pos_w = LLVMBuildLoad(builder, outputs[pos][3], ""); /*w0 w1 .. wn */
1093
1094    if (clip_user && cv != pos) {
1095       cv_x = LLVMBuildLoad(builder, outputs[cv][0], ""); /*x0 x1 .. xn */
1096       cv_y = LLVMBuildLoad(builder, outputs[cv][1], ""); /*y0 y1 .. yn */
1097       cv_z = LLVMBuildLoad(builder, outputs[cv][2], ""); /*z0 z1 .. zn */
1098       cv_w = LLVMBuildLoad(builder, outputs[cv][3], ""); /*w0 w1 .. wn */
1099    } else {
1100       cv_x = pos_x;
1101       cv_y = pos_y;
1102       cv_z = pos_z;
1103       cv_w = pos_w;
1104    }
1105
1106    /* Cliptest, for hardwired planes */
1107    if (clip_xy) {
1108       /* plane 1 */
1109       test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_x , pos_w);
1110       temp = shift;
1111       test = LLVMBuildAnd(builder, test, temp, "");
1112       mask = test;
1113
1114       /* plane 2 */
1115       test = LLVMBuildFAdd(builder, pos_x, pos_w, "");
1116       test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test);
1117       temp = LLVMBuildShl(builder, temp, shift, "");
1118       test = LLVMBuildAnd(builder, test, temp, "");
1119       mask = LLVMBuildOr(builder, mask, test, "");
1120
1121       /* plane 3 */
1122       test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_y, pos_w);
1123       temp = LLVMBuildShl(builder, temp, shift, "");
1124       test = LLVMBuildAnd(builder, test, temp, "");
1125       mask = LLVMBuildOr(builder, mask, test, "");
1126
1127       /* plane 4 */
1128       test = LLVMBuildFAdd(builder, pos_y, pos_w, "");
1129       test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test);
1130       temp = LLVMBuildShl(builder, temp, shift, "");
1131       test = LLVMBuildAnd(builder, test, temp, "");
1132       mask = LLVMBuildOr(builder, mask, test, "");
1133    }
1134
1135    if (clip_z) {
1136       temp = lp_build_const_int_vec(gallivm, i32_type, 16);
1137       if (clip_halfz) {
1138          /* plane 5 */
1139          test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, pos_z);
1140          test = LLVMBuildAnd(builder, test, temp, "");
1141          mask = LLVMBuildOr(builder, mask, test, "");
1142       }
1143       else {
1144          /* plane 5 */
1145          test = LLVMBuildFAdd(builder, pos_z, pos_w, "");
1146          test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test);
1147          test = LLVMBuildAnd(builder, test, temp, "");
1148          mask = LLVMBuildOr(builder, mask, test, "");
1149       }
1150       /* plane 6 */
1151       test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_z, pos_w);
1152       temp = LLVMBuildShl(builder, temp, shift, "");
1153       test = LLVMBuildAnd(builder, test, temp, "");
1154       mask = LLVMBuildOr(builder, mask, test, "");
1155    }
1156
1157    if (clip_user) {
1158       LLVMValueRef planes_ptr = draw_jit_context_planes(gallivm, context_ptr);
1159       LLVMValueRef indices[3];
1160
1161       /* userclip planes */
1162       while (ucp_enable) {
1163          unsigned plane_idx = ffs(ucp_enable)-1;
1164          ucp_enable &= ~(1 << plane_idx);
1165          plane_idx += 6;
1166
1167          if (have_cd && num_written_clipdistance) {
1168             LLVMValueRef clipdist;
1169             int i;
1170             i = plane_idx - 6;
1171
1172             *have_clipdist = TRUE;
1173             if (i < 4) {
1174                clipdist = LLVMBuildLoad(builder, outputs[cd[0]][i], "");
1175             } else {
1176                clipdist = LLVMBuildLoad(builder, outputs[cd[1]][i-4], "");
1177             }
1178             test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, clipdist);
1179             temp = lp_build_const_int_vec(gallivm, i32_type, 1 << plane_idx);
1180             test = LLVMBuildAnd(builder, test, temp, "");
1181             mask = LLVMBuildOr(builder, mask, test, "");
1182          } else {
1183             LLVMTypeRef vs_type_llvm = lp_build_vec_type(gallivm, vs_type);
1184             indices[0] = lp_build_const_int32(gallivm, 0);
1185             indices[1] = lp_build_const_int32(gallivm, plane_idx);
1186
1187             indices[2] = lp_build_const_int32(gallivm, 0);
1188             plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, "");
1189             plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_x");
1190             planes = lp_build_broadcast(gallivm, vs_type_llvm, plane1);
1191             sum = LLVMBuildFMul(builder, planes, cv_x, "");
1192
1193             indices[2] = lp_build_const_int32(gallivm, 1);
1194             plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, "");
1195             plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_y");
1196             planes = lp_build_broadcast(gallivm, vs_type_llvm, plane1);
1197             test = LLVMBuildFMul(builder, planes, cv_y, "");
1198             sum = LLVMBuildFAdd(builder, sum, test, "");
1199
1200             indices[2] = lp_build_const_int32(gallivm, 2);
1201             plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, "");
1202             plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_z");
1203             planes = lp_build_broadcast(gallivm, vs_type_llvm, plane1);
1204             test = LLVMBuildFMul(builder, planes, cv_z, "");
1205             sum = LLVMBuildFAdd(builder, sum, test, "");
1206
1207             indices[2] = lp_build_const_int32(gallivm, 3);
1208             plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, "");
1209             plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_w");
1210             planes = lp_build_broadcast(gallivm, vs_type_llvm, plane1);
1211             test = LLVMBuildFMul(builder, planes, cv_w, "");
1212             sum = LLVMBuildFAdd(builder, sum, test, "");
1213
1214             test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, sum);
1215             temp = lp_build_const_int_vec(gallivm, i32_type, 1 << plane_idx);
1216             test = LLVMBuildAnd(builder, test, temp, "");
1217             mask = LLVMBuildOr(builder, mask, test, "");
1218          }
1219       }
1220    }
1221    return mask;
1222 }
1223
1224
1225 /**
1226  * Returns boolean if any clipping has occurred
1227  * Used zero/non-zero i32 value to represent boolean
1228  */
1229 static LLVMValueRef
1230 clipmask_booli32(struct gallivm_state *gallivm,
1231                  const struct lp_type vs_type,
1232                  LLVMValueRef clipmask_bool_ptr)
1233 {
1234    LLVMBuilderRef builder = gallivm->builder;
1235    LLVMTypeRef int32_type = LLVMInt32TypeInContext(gallivm->context);
1236    LLVMValueRef clipmask_bool = LLVMBuildLoad(builder, clipmask_bool_ptr, "");
1237    LLVMValueRef ret = LLVMConstNull(int32_type);
1238    LLVMValueRef temp;
1239    int i;
1240
1241    /*
1242     * Can do this with log2(vector length) pack instructions and one extract
1243     * (as we don't actually need a or) with sse2 which would be way better.
1244     */
1245    for (i=0; i < vs_type.length; i++) {
1246       temp = LLVMBuildExtractElement(builder, clipmask_bool,
1247                                      lp_build_const_int32(gallivm, i) , "");
1248       ret = LLVMBuildOr(builder, ret, temp, "");
1249    }
1250    return ret;
1251 }
1252
1253 static LLVMValueRef
1254 draw_gs_llvm_fetch_input(const struct lp_build_tgsi_gs_iface *gs_iface,
1255                          struct lp_build_tgsi_context * bld_base,
1256                          boolean is_indirect,
1257                          LLVMValueRef vertex_index,
1258                          LLVMValueRef attrib_index,
1259                          LLVMValueRef swizzle_index)
1260 {
1261    const struct draw_gs_llvm_iface *gs = draw_gs_llvm_iface(gs_iface);
1262    struct gallivm_state *gallivm = bld_base->base.gallivm;
1263    LLVMBuilderRef builder = gallivm->builder;
1264    LLVMValueRef indices[3];
1265    LLVMValueRef res;
1266    struct lp_type type = bld_base->base.type;
1267
1268    if (is_indirect) {
1269       int i;
1270       res = bld_base->base.zero;
1271       for (i = 0; i < type.length; ++i) {
1272          LLVMValueRef idx = lp_build_const_int32(gallivm, i);
1273          LLVMValueRef vert_chan_index = LLVMBuildExtractElement(builder,
1274                                                                 vertex_index, idx, "");
1275          LLVMValueRef channel_vec, value;
1276          indices[0] = vert_chan_index;
1277          indices[1] = attrib_index;
1278          indices[2] = swizzle_index;
1279
1280          channel_vec = LLVMBuildGEP(builder, gs->input, indices, 3, "");
1281          channel_vec = LLVMBuildLoad(builder, channel_vec, "");
1282          value = LLVMBuildExtractElement(builder, channel_vec, idx, "");
1283
1284          res = LLVMBuildInsertElement(builder, res, value, idx, "");
1285       }
1286    } else {
1287       indices[0] = vertex_index;
1288       indices[1] = attrib_index;
1289       indices[2] = swizzle_index;
1290
1291       res = LLVMBuildGEP(builder, gs->input, indices, 3, "");
1292       res = LLVMBuildLoad(builder, res, "");
1293    }
1294
1295    return res;
1296 }
1297
1298 static void
1299 draw_gs_llvm_emit_vertex(const struct lp_build_tgsi_gs_iface *gs_base,
1300                          struct lp_build_tgsi_context * bld_base,
1301                          LLVMValueRef (*outputs)[4],
1302                          LLVMValueRef emitted_vertices_vec)
1303 {
1304    const struct draw_gs_llvm_iface *gs_iface = draw_gs_llvm_iface(gs_base);
1305    struct draw_gs_llvm_variant *variant = gs_iface->variant;
1306    struct gallivm_state *gallivm = variant->gallivm;
1307    LLVMBuilderRef builder = gallivm->builder;
1308    struct lp_type gs_type = bld_base->base.type;
1309    LLVMValueRef clipmask = lp_build_const_int_vec(gallivm,
1310                                                   lp_int_type(gs_type), 0);
1311    LLVMValueRef indices[LP_MAX_VECTOR_LENGTH];
1312    LLVMValueRef next_prim_offset =
1313       lp_build_const_int32(gallivm, variant->shader->base.primitive_boundary);
1314    LLVMValueRef io = variant->io_ptr;
1315    unsigned i;
1316    const struct tgsi_shader_info *gs_info = &variant->shader->base.info;
1317
1318    for (i = 0; i < gs_type.length; ++i) {
1319       LLVMValueRef ind = lp_build_const_int32(gallivm, i);
1320       LLVMValueRef currently_emitted =
1321          LLVMBuildExtractElement(builder, emitted_vertices_vec, ind, "");
1322       indices[i] = LLVMBuildMul(builder, ind, next_prim_offset, "");
1323       indices[i] = LLVMBuildAdd(builder, indices[i], currently_emitted, "");
1324    }
1325
1326    convert_to_aos(gallivm, io, indices,
1327                   outputs, clipmask,
1328                   gs_info->num_outputs, gs_type,
1329                   FALSE);
1330 }
1331
1332 static void
1333 draw_gs_llvm_end_primitive(const struct lp_build_tgsi_gs_iface *gs_base,
1334                            struct lp_build_tgsi_context * bld_base,
1335                            LLVMValueRef verts_per_prim_vec,
1336                            LLVMValueRef emitted_prims_vec)
1337 {
1338    const struct draw_gs_llvm_iface *gs_iface = draw_gs_llvm_iface(gs_base);
1339    struct draw_gs_llvm_variant *variant = gs_iface->variant;
1340    struct gallivm_state *gallivm = variant->gallivm;
1341    LLVMBuilderRef builder = gallivm->builder;
1342    LLVMValueRef prim_lengts_ptr =
1343       draw_gs_jit_prim_lengths(variant->gallivm, variant->context_ptr);
1344    unsigned i;
1345
1346    for (i = 0; i < bld_base->base.type.length; ++i) {
1347       LLVMValueRef ind = lp_build_const_int32(gallivm, i);
1348       LLVMValueRef prims_emitted =
1349          LLVMBuildExtractElement(builder, emitted_prims_vec, ind, "");
1350       LLVMValueRef store_ptr;
1351       LLVMValueRef num_vertices =
1352          LLVMBuildExtractElement(builder, verts_per_prim_vec, ind, "");
1353
1354       store_ptr = LLVMBuildGEP(builder, prim_lengts_ptr, &prims_emitted, 1, "");
1355       store_ptr = LLVMBuildLoad(builder, store_ptr, "");
1356       store_ptr = LLVMBuildGEP(builder, store_ptr, &ind, 1, "");
1357       LLVMBuildStore(builder, num_vertices, store_ptr);
1358    }
1359 }
1360
1361 static void
1362 draw_gs_llvm_epilogue(const struct lp_build_tgsi_gs_iface *gs_base,
1363                       struct lp_build_tgsi_context * bld_base,
1364                       LLVMValueRef total_emitted_vertices_vec,
1365                       LLVMValueRef emitted_prims_vec)
1366 {
1367    const struct draw_gs_llvm_iface *gs_iface = draw_gs_llvm_iface(gs_base);
1368    struct draw_gs_llvm_variant *variant = gs_iface->variant;
1369    struct gallivm_state *gallivm = variant->gallivm;
1370    LLVMBuilderRef builder = gallivm->builder;
1371    LLVMValueRef emitted_verts_ptr =
1372       draw_gs_jit_emitted_vertices(gallivm, variant->context_ptr);
1373    LLVMValueRef emitted_prims_ptr =
1374       draw_gs_jit_emitted_prims(gallivm, variant->context_ptr);
1375    LLVMValueRef zero = lp_build_const_int32(gallivm, 0);
1376
1377    emitted_verts_ptr = LLVMBuildGEP(builder, emitted_verts_ptr, &zero, 0, "");
1378    emitted_prims_ptr = LLVMBuildGEP(builder, emitted_prims_ptr, &zero, 0, "");
1379
1380    LLVMBuildStore(builder, total_emitted_vertices_vec, emitted_verts_ptr);
1381    LLVMBuildStore(builder, emitted_prims_vec, emitted_prims_ptr);
1382 }
1383
1384 static void
1385 draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant,
1386                    boolean elts)
1387 {
1388    struct gallivm_state *gallivm = variant->gallivm;
1389    LLVMContextRef context = gallivm->context;
1390    LLVMTypeRef int32_type = LLVMInt32TypeInContext(context);
1391    LLVMTypeRef arg_types[8];
1392    LLVMTypeRef func_type;
1393    LLVMValueRef context_ptr;
1394    LLVMBasicBlockRef block;
1395    LLVMBuilderRef builder;
1396    struct lp_type vs_type;
1397    LLVMValueRef end, start;
1398    LLVMValueRef count, fetch_elts, fetch_count;
1399    LLVMValueRef stride, step, io_itr;
1400    LLVMValueRef io_ptr, vbuffers_ptr, vb_ptr;
1401    LLVMValueRef zero = lp_build_const_int32(gallivm, 0);
1402    LLVMValueRef one = lp_build_const_int32(gallivm, 1);
1403    struct draw_context *draw = llvm->draw;
1404    const struct tgsi_shader_info *vs_info = &draw->vs.vertex_shader->info;
1405    unsigned i, j;
1406    struct lp_build_context bld;
1407    struct lp_build_loop_state lp_loop;
1408    const int vector_length = lp_native_vector_width / 32;
1409    LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][TGSI_NUM_CHANNELS];
1410    LLVMValueRef fetch_max;
1411    struct lp_build_sampler_soa *sampler = 0;
1412    LLVMValueRef ret, clipmask_bool_ptr;
1413    const struct draw_geometry_shader *gs = draw->gs.geometry_shader;
1414    /* If geometry shader is present we need to skip both the viewport
1415     * transformation and clipping otherwise the inputs to the geometry
1416     * shader will be incorrect.
1417     */
1418    const boolean bypass_viewport = gs || variant->key.bypass_viewport;
1419    const boolean enable_cliptest = !gs && (variant->key.clip_xy ||
1420                                            variant->key.clip_z  ||
1421                                            variant->key.clip_user);
1422    LLVMValueRef variant_func;
1423    const unsigned pos = draw_current_shader_position_output(llvm->draw);
1424    const unsigned cv = draw_current_shader_clipvertex_output(llvm->draw);
1425    boolean have_clipdist = FALSE;
1426    struct lp_bld_tgsi_system_values system_values;
1427
1428    memset(&system_values, 0, sizeof(system_values));
1429
1430    arg_types[0] = get_context_ptr_type(variant);       /* context */
1431    arg_types[1] = get_vertex_header_ptr_type(variant); /* vertex_header */
1432    arg_types[2] = get_buffer_ptr_type(variant);        /* vbuffers */
1433    if (elts)
1434       arg_types[3] = LLVMPointerType(int32_type, 0);/* fetch_elts * */
1435    else
1436       arg_types[3] = int32_type;                    /* start */
1437    arg_types[4] = int32_type;                       /* fetch_count / count */
1438    arg_types[5] = int32_type;                       /* stride */
1439    arg_types[6] = get_vb_ptr_type(variant);            /* pipe_vertex_buffer's */
1440    arg_types[7] = int32_type;                       /* instance_id */
1441
1442    func_type = LLVMFunctionType(int32_type, arg_types, Elements(arg_types), 0);
1443
1444    variant_func = LLVMAddFunction(gallivm->module,
1445                                   elts ? "draw_llvm_shader_elts" : "draw_llvm_shader",
1446                                   func_type);
1447
1448    if (elts)
1449       variant->function_elts = variant_func;
1450    else
1451       variant->function = variant_func;
1452
1453    LLVMSetFunctionCallConv(variant_func, LLVMCCallConv);
1454    for (i = 0; i < Elements(arg_types); ++i)
1455       if (LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind)
1456          LLVMAddAttribute(LLVMGetParam(variant_func, i),
1457                           LLVMNoAliasAttribute);
1458
1459    context_ptr               = LLVMGetParam(variant_func, 0);
1460    io_ptr                    = LLVMGetParam(variant_func, 1);
1461    vbuffers_ptr              = LLVMGetParam(variant_func, 2);
1462    stride                    = LLVMGetParam(variant_func, 5);
1463    vb_ptr                    = LLVMGetParam(variant_func, 6);
1464    system_values.instance_id = LLVMGetParam(variant_func, 7);
1465
1466    lp_build_name(context_ptr, "context");
1467    lp_build_name(io_ptr, "io");
1468    lp_build_name(vbuffers_ptr, "vbuffers");
1469    lp_build_name(stride, "stride");
1470    lp_build_name(vb_ptr, "vb");
1471    lp_build_name(system_values.instance_id, "instance_id");
1472
1473    if (elts) {
1474       fetch_elts   = LLVMGetParam(variant_func, 3);
1475       fetch_count  = LLVMGetParam(variant_func, 4);
1476       lp_build_name(fetch_elts, "fetch_elts");
1477       lp_build_name(fetch_count, "fetch_count");
1478       start = count = NULL;
1479    }
1480    else {
1481       start        = LLVMGetParam(variant_func, 3);
1482       count        = LLVMGetParam(variant_func, 4);
1483       lp_build_name(start, "start");
1484       lp_build_name(count, "count");
1485       fetch_elts = fetch_count = NULL;
1486    }
1487
1488    /*
1489     * Function body
1490     */
1491
1492    block = LLVMAppendBasicBlockInContext(gallivm->context, variant_func, "entry");
1493    builder = gallivm->builder;
1494    LLVMPositionBuilderAtEnd(builder, block);
1495
1496    lp_build_context_init(&bld, gallivm, lp_type_int(32));
1497
1498    memset(&vs_type, 0, sizeof vs_type);
1499    vs_type.floating = TRUE; /* floating point values */
1500    vs_type.sign = TRUE;     /* values are signed */
1501    vs_type.norm = FALSE;    /* values are not limited to [0,1] or [-1,1] */
1502    vs_type.width = 32;      /* 32-bit float */
1503    vs_type.length = vector_length;
1504
1505    /* hold temporary "bool" clipmask */
1506    clipmask_bool_ptr = lp_build_alloca(gallivm, lp_build_int_vec_type(gallivm, vs_type), "");
1507    LLVMBuildStore(builder, lp_build_zero(gallivm, lp_int_type(vs_type)), clipmask_bool_ptr);
1508
1509    /* code generated texture sampling */
1510    sampler = draw_llvm_sampler_soa_create(
1511       draw_llvm_variant_key_samplers(&variant->key),
1512       context_ptr);
1513
1514    if (elts) {
1515       start = zero;
1516       end = fetch_count;
1517    }
1518    else {
1519       end = lp_build_add(&bld, start, count);
1520    }
1521
1522    step = lp_build_const_int32(gallivm, vector_length);
1523
1524    fetch_max = LLVMBuildSub(builder, end, one, "fetch_max");
1525
1526    lp_build_loop_begin(&lp_loop, gallivm, start);
1527    {
1528       LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][TGSI_NUM_CHANNELS];
1529       LLVMValueRef aos_attribs[PIPE_MAX_SHADER_INPUTS][LP_MAX_VECTOR_WIDTH / 32] = { { 0 } };
1530       LLVMValueRef io;
1531       LLVMValueRef clipmask;   /* holds the clipmask value */
1532       const LLVMValueRef (*ptr_aos)[TGSI_NUM_CHANNELS];
1533
1534       if (elts)
1535          io_itr = lp_loop.counter;
1536       else
1537          io_itr = LLVMBuildSub(builder, lp_loop.counter, start, "");
1538
1539       io = LLVMBuildGEP(builder, io_ptr, &io_itr, 1, "");
1540 #if DEBUG_STORE
1541       lp_build_printf(gallivm, " --- io %d = %p, loop counter %d\n",
1542                       io_itr, io, lp_loop.counter);
1543 #endif
1544       system_values.vertex_id = lp_build_zero(gallivm, lp_type_uint_vec(32, 32*vector_length));
1545       for (i = 0; i < vector_length; ++i) {
1546          LLVMValueRef true_index =
1547             LLVMBuildAdd(builder,
1548                          lp_loop.counter,
1549                          lp_build_const_int32(gallivm, i), "");
1550
1551          /* make sure we're not out of bounds which can happen
1552           * if fetch_count % 4 != 0, because on the last iteration
1553           * a few of the 4 vertex fetches will be out of bounds */
1554          true_index = lp_build_min(&bld, true_index, fetch_max);
1555
1556          if (elts) {
1557             LLVMValueRef fetch_ptr;
1558             fetch_ptr = LLVMBuildGEP(builder, fetch_elts,
1559                                      &true_index, 1, "");
1560             true_index = LLVMBuildLoad(builder, fetch_ptr, "fetch_elt");
1561          }
1562
1563          system_values.vertex_id = LLVMBuildInsertElement(gallivm->builder,
1564                                                           system_values.vertex_id, true_index,
1565                                                           lp_build_const_int32(gallivm, i), "");
1566          for (j = 0; j < draw->pt.nr_vertex_elements; ++j) {
1567             struct pipe_vertex_element *velem = &draw->pt.vertex_element[j];
1568             LLVMValueRef vb_index =
1569                lp_build_const_int32(gallivm, velem->vertex_buffer_index);
1570             LLVMValueRef vb = LLVMBuildGEP(builder, vb_ptr, &vb_index, 1, "");
1571             generate_fetch(gallivm, vbuffers_ptr,
1572                            &aos_attribs[j][i], velem, vb, true_index,
1573                            system_values.instance_id);
1574          }
1575       }
1576       convert_to_soa(gallivm, aos_attribs, inputs,
1577                      draw->pt.nr_vertex_elements, vs_type);
1578
1579       ptr_aos = (const LLVMValueRef (*)[TGSI_NUM_CHANNELS]) inputs;
1580       generate_vs(variant,
1581                   builder,
1582                   vs_type,
1583                   outputs,
1584                   ptr_aos,
1585                   &system_values,
1586                   context_ptr,
1587                   sampler,
1588                   variant->key.clamp_vertex_color);
1589
1590       if (pos != -1 && cv != -1) {
1591          /* store original positions in clip before further manipulation */
1592          store_clip(gallivm, vs_type, io, outputs, 0, cv);
1593          store_clip(gallivm, vs_type, io, outputs, 1, pos);
1594
1595          /* do cliptest */
1596          if (enable_cliptest) {
1597             LLVMValueRef temp = LLVMBuildLoad(builder, clipmask_bool_ptr, "");
1598             /* allocate clipmask, assign it integer type */
1599             clipmask = generate_clipmask(llvm,
1600                                          gallivm,
1601                                          vs_type,
1602                                          outputs,
1603                                          variant->key.clip_xy,
1604                                          variant->key.clip_z,
1605                                          variant->key.clip_user,
1606                                          variant->key.clip_halfz,
1607                                          variant->key.ucp_enable,
1608                                          context_ptr, &have_clipdist);
1609             temp = LLVMBuildOr(builder, clipmask, temp, "");
1610             /* store temporary clipping boolean value */
1611             LLVMBuildStore(builder, temp, clipmask_bool_ptr);
1612          }
1613          else {
1614             clipmask = lp_build_const_int_vec(gallivm, lp_int_type(vs_type), 0);
1615          }
1616
1617          /* do viewport mapping */
1618          if (!bypass_viewport) {
1619             generate_viewport(variant, builder, vs_type, outputs, context_ptr);
1620          }
1621       }
1622       else {
1623          clipmask = lp_build_const_int_vec(gallivm, lp_int_type(vs_type), 0);
1624       }
1625
1626       /* store clipmask in vertex header,
1627        * original positions in clip
1628        * and transformed positions in data
1629        */
1630       convert_to_aos(gallivm, io, NULL, outputs, clipmask,
1631                      vs_info->num_outputs, vs_type,
1632                      have_clipdist);
1633    }
1634
1635    lp_build_loop_end_cond(&lp_loop, end, step, LLVMIntUGE);
1636
1637    sampler->destroy(sampler);
1638
1639    /* return clipping boolean value for function */
1640    ret = clipmask_booli32(gallivm, vs_type, clipmask_bool_ptr);
1641
1642    LLVMBuildRet(builder, ret);
1643
1644    gallivm_verify_function(gallivm, variant_func);
1645 }
1646
1647
1648 struct draw_llvm_variant_key *
1649 draw_llvm_make_variant_key(struct draw_llvm *llvm, char *store)
1650 {
1651    unsigned i;
1652    struct draw_llvm_variant_key *key;
1653    struct draw_sampler_static_state *draw_sampler;
1654
1655    key = (struct draw_llvm_variant_key *)store;
1656
1657    key->clamp_vertex_color = llvm->draw->rasterizer->clamp_vertex_color; /**/
1658
1659    /* Presumably all variants of the shader should have the same
1660     * number of vertex elements - ie the number of shader inputs.
1661     * NOTE: we NEED to store the needed number of needed inputs
1662     * here, not the number of provided elements to match keysize
1663     * (and the offset of sampler state in the key).
1664     */
1665    key->nr_vertex_elements = llvm->draw->vs.vertex_shader->info.file_max[TGSI_FILE_INPUT] + 1;
1666    assert(key->nr_vertex_elements <= llvm->draw->pt.nr_vertex_elements);
1667
1668    /* will have to rig this up properly later */
1669    key->clip_xy = llvm->draw->clip_xy;
1670    key->clip_z = llvm->draw->clip_z;
1671    key->clip_user = llvm->draw->clip_user;
1672    key->bypass_viewport = llvm->draw->identity_viewport;
1673    key->clip_halfz = !llvm->draw->rasterizer->gl_rasterization_rules;
1674    key->need_edgeflags = (llvm->draw->vs.edgeflag_output ? TRUE : FALSE);
1675    key->ucp_enable = llvm->draw->rasterizer->clip_plane_enable;
1676    key->has_gs = llvm->draw->gs.geometry_shader != NULL;
1677    key->pad1 = 0;
1678
1679    /* All variants of this shader will have the same value for
1680     * nr_samplers.  Not yet trying to compact away holes in the
1681     * sampler array.
1682     */
1683    key->nr_samplers = llvm->draw->vs.vertex_shader->info.file_max[TGSI_FILE_SAMPLER] + 1;
1684    if (llvm->draw->vs.vertex_shader->info.file_max[TGSI_FILE_SAMPLER_VIEW] != -1) {
1685       key->nr_sampler_views =
1686          llvm->draw->vs.vertex_shader->info.file_max[TGSI_FILE_SAMPLER_VIEW] + 1;
1687    }
1688    else {
1689       key->nr_sampler_views = key->nr_samplers;
1690    }
1691
1692    draw_sampler = draw_llvm_variant_key_samplers(key);
1693
1694    memcpy(key->vertex_element,
1695           llvm->draw->pt.vertex_element,
1696           sizeof(struct pipe_vertex_element) * key->nr_vertex_elements);
1697
1698    memset(draw_sampler, 0, MAX2(key->nr_samplers, key->nr_sampler_views) * sizeof *draw_sampler);
1699
1700    for (i = 0 ; i < key->nr_samplers; i++) {
1701       lp_sampler_static_sampler_state(&draw_sampler[i].sampler_state,
1702                                       llvm->draw->samplers[PIPE_SHADER_VERTEX][i]);
1703    }
1704    for (i = 0 ; i < key->nr_sampler_views; i++) {
1705       lp_sampler_static_texture_state(&draw_sampler[i].texture_state,
1706                                       llvm->draw->sampler_views[PIPE_SHADER_VERTEX][i]);
1707    }
1708
1709    return key;
1710 }
1711
1712
1713 void
1714 draw_llvm_dump_variant_key(struct draw_llvm_variant_key *key)
1715 {
1716    unsigned i;
1717    struct draw_sampler_static_state *sampler = draw_llvm_variant_key_samplers(key);
1718
1719    debug_printf("clamp_vertex_color = %u\n", key->clamp_vertex_color);
1720    debug_printf("clip_xy = %u\n", key->clip_xy);
1721    debug_printf("clip_z = %u\n", key->clip_z);
1722    debug_printf("clip_user = %u\n", key->clip_user);
1723    debug_printf("bypass_viewport = %u\n", key->bypass_viewport);
1724    debug_printf("clip_halfz = %u\n", key->clip_halfz);
1725    debug_printf("need_edgeflags = %u\n", key->need_edgeflags);
1726    debug_printf("has_gs = %u\n", key->has_gs);
1727    debug_printf("ucp_enable = %u\n", key->ucp_enable);
1728
1729    for (i = 0 ; i < key->nr_vertex_elements; i++) {
1730       debug_printf("vertex_element[%i].src_offset = %u\n", i, key->vertex_element[i].src_offset);
1731       debug_printf("vertex_element[%i].instance_divisor = %u\n", i, key->vertex_element[i].instance_divisor);
1732       debug_printf("vertex_element[%i].vertex_buffer_index = %u\n", i, key->vertex_element[i].vertex_buffer_index);
1733       debug_printf("vertex_element[%i].src_format = %s\n", i, util_format_name(key->vertex_element[i].src_format));
1734    }
1735
1736    for (i = 0 ; i < key->nr_sampler_views; i++) {
1737       debug_printf("sampler[%i].src_format = %s\n", i, util_format_name(sampler[i].texture_state.format));
1738    }
1739 }
1740
1741
1742 void
1743 draw_llvm_set_mapped_texture(struct draw_context *draw,
1744                              unsigned shader_stage,
1745                              unsigned sview_idx,
1746                              uint32_t width, uint32_t height, uint32_t depth,
1747                              uint32_t first_level, uint32_t last_level,
1748                              const void *base_ptr,
1749                              uint32_t row_stride[PIPE_MAX_TEXTURE_LEVELS],
1750                              uint32_t img_stride[PIPE_MAX_TEXTURE_LEVELS],
1751                              uint32_t mip_offsets[PIPE_MAX_TEXTURE_LEVELS])
1752 {
1753    unsigned j;
1754    struct draw_jit_texture *jit_tex;
1755
1756    assert(shader_stage == PIPE_SHADER_VERTEX ||
1757           shader_stage == PIPE_SHADER_GEOMETRY);
1758
1759    if (shader_stage == PIPE_SHADER_VERTEX) {
1760       assert(sview_idx < Elements(draw->llvm->jit_context.textures));
1761
1762       jit_tex = &draw->llvm->jit_context.textures[sview_idx];
1763    } else if (shader_stage == PIPE_SHADER_GEOMETRY) {
1764       assert(sview_idx < Elements(draw->llvm->gs_jit_context.textures));
1765
1766       jit_tex = &draw->llvm->gs_jit_context.textures[sview_idx];
1767    } else {
1768       assert(0);
1769       return;
1770    }
1771
1772    jit_tex->width = width;
1773    jit_tex->height = height;
1774    jit_tex->depth = depth;
1775    jit_tex->first_level = first_level;
1776    jit_tex->last_level = last_level;
1777    jit_tex->base = base_ptr;
1778
1779    for (j = first_level; j <= last_level; j++) {
1780       jit_tex->mip_offsets[j] = mip_offsets[j];
1781       jit_tex->row_stride[j] = row_stride[j];
1782       jit_tex->img_stride[j] = img_stride[j];
1783    }
1784 }
1785
1786
1787 void
1788 draw_llvm_set_sampler_state(struct draw_context *draw,
1789                             unsigned shader_type)
1790 {
1791    unsigned i;
1792
1793    if (shader_type == PIPE_SHADER_VERTEX) {
1794       for (i = 0; i < draw->num_samplers[PIPE_SHADER_VERTEX]; i++) {
1795          struct draw_jit_sampler *jit_sam = &draw->llvm->jit_context.samplers[i];
1796
1797          if (draw->samplers[i]) {
1798             const struct pipe_sampler_state *s
1799                = draw->samplers[PIPE_SHADER_VERTEX][i];
1800             jit_sam->min_lod = s->min_lod;
1801             jit_sam->max_lod = s->max_lod;
1802             jit_sam->lod_bias = s->lod_bias;
1803             COPY_4V(jit_sam->border_color, s->border_color.f);
1804          }
1805       }
1806    } else if (shader_type == PIPE_SHADER_GEOMETRY) {
1807       for (i = 0; i < draw->num_samplers[PIPE_SHADER_GEOMETRY]; i++) {
1808          struct draw_jit_sampler *jit_sam = &draw->llvm->gs_jit_context.samplers[i];
1809
1810          if (draw->samplers[i]) {
1811             const struct pipe_sampler_state *s
1812                = draw->samplers[PIPE_SHADER_GEOMETRY][i];
1813             jit_sam->min_lod = s->min_lod;
1814             jit_sam->max_lod = s->max_lod;
1815             jit_sam->lod_bias = s->lod_bias;
1816             COPY_4V(jit_sam->border_color, s->border_color.f);
1817          }
1818       }
1819    }
1820 }
1821
1822
1823 void
1824 draw_llvm_destroy_variant(struct draw_llvm_variant *variant)
1825 {
1826    struct draw_llvm *llvm = variant->llvm;
1827
1828    if (variant->function_elts) {
1829       gallivm_free_function(variant->gallivm,
1830                             variant->function_elts, variant->jit_func_elts);
1831    }
1832
1833    if (variant->function) {
1834       gallivm_free_function(variant->gallivm,
1835                             variant->function, variant->jit_func);
1836    }
1837
1838    gallivm_destroy(variant->gallivm);
1839
1840    remove_from_list(&variant->list_item_local);
1841    variant->shader->variants_cached--;
1842    remove_from_list(&variant->list_item_global);
1843    llvm->nr_variants--;
1844    FREE(variant);
1845 }
1846
1847
1848 /**
1849  * Create LLVM types for various structures.
1850  */
1851 static void
1852 create_gs_jit_types(struct draw_gs_llvm_variant *var)
1853 {
1854    struct gallivm_state *gallivm = var->gallivm;
1855    LLVMTypeRef texture_type, sampler_type, context_type;
1856
1857    texture_type = create_jit_texture_type(gallivm, "texture");
1858    sampler_type = create_jit_sampler_type(gallivm, "sampler");
1859
1860    context_type = create_gs_jit_context_type(gallivm,
1861                                              var->shader->base.vector_length,
1862                                              texture_type, sampler_type,
1863                                              "draw_gs_jit_context");
1864    var->context_ptr_type = LLVMPointerType(context_type, 0);
1865
1866    var->input_array_type = create_gs_jit_input_type(gallivm);
1867 }
1868
1869 static LLVMTypeRef
1870 get_gs_context_ptr_type(struct draw_gs_llvm_variant *variant)
1871 {
1872    if (!variant->context_ptr_type)
1873       create_gs_jit_types(variant);
1874    return variant->context_ptr_type;
1875 }
1876
1877 static LLVMValueRef
1878 generate_mask_value(struct draw_gs_llvm_variant *variant,
1879                     struct lp_type gs_type)
1880 {
1881    struct gallivm_state *gallivm = variant->gallivm;
1882    LLVMBuilderRef builder = gallivm->builder;
1883    LLVMValueRef bits[16];
1884    struct lp_type  mask_type = lp_int_type(gs_type);
1885    struct lp_type mask_elem_type = lp_elem_type(mask_type);
1886    LLVMValueRef mask_val = lp_build_const_vec(gallivm,
1887                                               mask_type,
1888                                               0);
1889    unsigned i;
1890
1891    assert(gs_type.length <= Elements(bits));
1892
1893    for (i = gs_type.length; i >= 1; --i) {
1894       int idx = i - 1;
1895       LLVMValueRef ind = lp_build_const_int32(gallivm, i);
1896       bits[idx] = lp_build_compare(gallivm,
1897                                    mask_elem_type, PIPE_FUNC_GEQUAL,
1898                                    variant->num_prims, ind);
1899    }
1900    for (i = 0; i < gs_type.length; ++i) {
1901       LLVMValueRef ind = lp_build_const_int32(gallivm, i);
1902       mask_val = LLVMBuildInsertElement(builder, mask_val, bits[i], ind, "");
1903    }
1904    mask_val = lp_build_compare(gallivm,
1905                                mask_type, PIPE_FUNC_NOTEQUAL,
1906                                mask_val,
1907                                lp_build_const_int_vec(gallivm, mask_type, 0));
1908
1909    return mask_val;
1910 }
1911
1912 static void
1913 draw_gs_llvm_generate(struct draw_llvm *llvm,
1914                       struct draw_gs_llvm_variant *variant)
1915 {
1916    struct gallivm_state *gallivm = variant->gallivm;
1917    LLVMContextRef context = gallivm->context;
1918    LLVMTypeRef int32_type = LLVMInt32TypeInContext(context);
1919    LLVMTypeRef arg_types[6];
1920    LLVMTypeRef func_type;
1921    LLVMValueRef variant_func;
1922    LLVMValueRef context_ptr;
1923    LLVMValueRef prim_id_ptr;
1924    LLVMBasicBlockRef block;
1925    LLVMBuilderRef builder;
1926    LLVMValueRef io_ptr, input_array, num_prims, mask_val;
1927    struct lp_build_sampler_soa *sampler = 0;
1928    struct lp_build_context bld;
1929    struct lp_bld_tgsi_system_values system_values;
1930    struct lp_type gs_type;
1931    unsigned i;
1932    struct draw_gs_llvm_iface gs_iface;
1933    const struct tgsi_token *tokens = variant->shader->base.state.tokens;
1934    LLVMValueRef consts_ptr;
1935    LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][TGSI_NUM_CHANNELS];
1936    struct lp_build_mask_context mask;
1937    const struct tgsi_shader_info *gs_info = &variant->shader->base.info;
1938    unsigned vector_length = variant->shader->base.vector_length;
1939
1940    memset(&system_values, 0, sizeof(system_values));
1941
1942    assert(variant->vertex_header_ptr_type);
1943
1944    arg_types[0] = get_gs_context_ptr_type(variant);    /* context */
1945    arg_types[1] = variant->input_array_type;           /* input */
1946    arg_types[2] = variant->vertex_header_ptr_type;     /* vertex_header */
1947    arg_types[3] = int32_type;                          /* num_prims */
1948    arg_types[4] = int32_type;                          /* instance_id */
1949    arg_types[5] = LLVMPointerType(
1950       LLVMVectorType(int32_type, vector_length), 0);   /* prim_id_ptr */
1951
1952    func_type = LLVMFunctionType(int32_type, arg_types, Elements(arg_types), 0);
1953
1954    variant_func = LLVMAddFunction(gallivm->module, "draw_geometry_shader",
1955                                   func_type);
1956    variant->function = variant_func;
1957
1958    LLVMSetFunctionCallConv(variant_func, LLVMCCallConv);
1959
1960    for (i = 0; i < Elements(arg_types); ++i)
1961       if (LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind)
1962          LLVMAddAttribute(LLVMGetParam(variant_func, i),
1963                           LLVMNoAliasAttribute);
1964
1965    context_ptr               = LLVMGetParam(variant_func, 0);
1966    input_array               = LLVMGetParam(variant_func, 1);
1967    io_ptr                    = LLVMGetParam(variant_func, 2);
1968    num_prims                 = LLVMGetParam(variant_func, 3);
1969    system_values.instance_id = LLVMGetParam(variant_func, 4);
1970    prim_id_ptr               = LLVMGetParam(variant_func, 5);
1971
1972    lp_build_name(context_ptr, "context");
1973    lp_build_name(input_array, "input");
1974    lp_build_name(io_ptr, "io");
1975    lp_build_name(io_ptr, "num_prims");
1976    lp_build_name(system_values.instance_id, "instance_id");
1977    lp_build_name(prim_id_ptr, "prim_id_ptr");
1978
1979    variant->context_ptr = context_ptr;
1980    variant->io_ptr = io_ptr;
1981    variant->num_prims = num_prims;
1982
1983    gs_iface.base.fetch_input = draw_gs_llvm_fetch_input;
1984    gs_iface.base.emit_vertex = draw_gs_llvm_emit_vertex;
1985    gs_iface.base.end_primitive = draw_gs_llvm_end_primitive;
1986    gs_iface.base.gs_epilogue = draw_gs_llvm_epilogue;
1987    gs_iface.input = input_array;
1988    gs_iface.variant = variant;
1989
1990    /*
1991     * Function body
1992     */
1993
1994    block = LLVMAppendBasicBlockInContext(gallivm->context, variant_func, "entry");
1995    builder = gallivm->builder;
1996    LLVMPositionBuilderAtEnd(builder, block);
1997
1998    lp_build_context_init(&bld, gallivm, lp_type_int(32));
1999
2000    memset(&gs_type, 0, sizeof gs_type);
2001    gs_type.floating = TRUE; /* floating point values */
2002    gs_type.sign = TRUE;     /* values are signed */
2003    gs_type.norm = FALSE;    /* values are not limited to [0,1] or [-1,1] */
2004    gs_type.width = 32;      /* 32-bit float */
2005    gs_type.length = vector_length;
2006
2007    consts_ptr = draw_gs_jit_context_constants(variant->gallivm, context_ptr);
2008
2009    /* code generated texture sampling */
2010    sampler = draw_llvm_sampler_soa_create(variant->key.samplers,
2011                                           context_ptr);
2012
2013    mask_val = generate_mask_value(variant, gs_type);
2014    lp_build_mask_begin(&mask, gallivm, gs_type, mask_val);
2015
2016    if (gs_info->uses_primid) {
2017       system_values.prim_id = LLVMBuildLoad(builder, prim_id_ptr, "prim_id");;
2018    }
2019
2020    lp_build_tgsi_soa(variant->gallivm,
2021                      tokens,
2022                      gs_type,
2023                      &mask,
2024                      consts_ptr,
2025                      &system_values,
2026                      NULL /*pos*/,
2027                      NULL,
2028                      outputs,
2029                      sampler,
2030                      &llvm->draw->gs.geometry_shader->info,
2031                      (const struct lp_build_tgsi_gs_iface *)&gs_iface);
2032
2033    sampler->destroy(sampler);
2034
2035    lp_build_mask_end(&mask);
2036
2037    LLVMBuildRet(builder, lp_build_zero(gallivm, lp_type_uint(32)));
2038
2039    gallivm_verify_function(gallivm, variant_func);
2040 }
2041
2042
2043 struct draw_gs_llvm_variant *
2044 draw_gs_llvm_create_variant(struct draw_llvm *llvm,
2045                             unsigned num_outputs,
2046                             const struct draw_gs_llvm_variant_key *key)
2047 {
2048    struct draw_gs_llvm_variant *variant;
2049    struct llvm_geometry_shader *shader =
2050       llvm_geometry_shader(llvm->draw->gs.geometry_shader);
2051    LLVMTypeRef vertex_header;
2052
2053    variant = MALLOC(sizeof *variant +
2054                     shader->variant_key_size -
2055                     sizeof variant->key);
2056    if (variant == NULL)
2057       return NULL;
2058
2059    variant->llvm = llvm;
2060    variant->shader = shader;
2061
2062    variant->gallivm = gallivm_create();
2063
2064    create_gs_jit_types(variant);
2065
2066    memcpy(&variant->key, key, shader->variant_key_size);
2067
2068    vertex_header = create_jit_vertex_header(variant->gallivm, num_outputs);
2069
2070    variant->vertex_header_ptr_type = LLVMPointerType(vertex_header, 0);
2071
2072    draw_gs_llvm_generate(llvm, variant);
2073
2074    gallivm_compile_module(variant->gallivm);
2075
2076    variant->jit_func = (draw_gs_jit_func)
2077          gallivm_jit_function(variant->gallivm, variant->function);
2078
2079    variant->list_item_global.base = variant;
2080    variant->list_item_local.base = variant;
2081    /*variant->no = */shader->variants_created++;
2082    variant->list_item_global.base = variant;
2083
2084    return variant;
2085 }
2086
2087 void
2088 draw_gs_llvm_destroy_variant(struct draw_gs_llvm_variant *variant)
2089 {
2090    struct draw_llvm *llvm = variant->llvm;
2091
2092    if (variant->function) {
2093       gallivm_free_function(variant->gallivm,
2094                             variant->function, variant->jit_func);
2095    }
2096
2097    gallivm_destroy(variant->gallivm);
2098
2099    remove_from_list(&variant->list_item_local);
2100    variant->shader->variants_cached--;
2101    remove_from_list(&variant->list_item_global);
2102    llvm->nr_gs_variants--;
2103    FREE(variant);
2104 }
2105
2106 struct draw_gs_llvm_variant_key *
2107 draw_gs_llvm_make_variant_key(struct draw_llvm *llvm, char *store)
2108 {
2109    unsigned i;
2110    struct draw_gs_llvm_variant_key *key;
2111    struct draw_sampler_static_state *draw_sampler;
2112
2113    key = (struct draw_gs_llvm_variant_key *)store;
2114
2115    /* All variants of this shader will have the same value for
2116     * nr_samplers.  Not yet trying to compact away holes in the
2117     * sampler array.
2118     */
2119    key->nr_samplers = llvm->draw->gs.geometry_shader->info.file_max[TGSI_FILE_SAMPLER] + 1;
2120    if (llvm->draw->gs.geometry_shader->info.file_max[TGSI_FILE_SAMPLER_VIEW] != -1) {
2121       key->nr_sampler_views =
2122          llvm->draw->gs.geometry_shader->info.file_max[TGSI_FILE_SAMPLER_VIEW] + 1;
2123    }
2124    else {
2125       key->nr_sampler_views = key->nr_samplers;
2126    }
2127
2128    draw_sampler = key->samplers;
2129
2130    memset(draw_sampler, 0, MAX2(key->nr_samplers, key->nr_sampler_views) * sizeof *draw_sampler);
2131
2132    for (i = 0 ; i < key->nr_samplers; i++) {
2133       lp_sampler_static_sampler_state(&draw_sampler[i].sampler_state,
2134                                       llvm->draw->samplers[PIPE_SHADER_GEOMETRY][i]);
2135    }
2136    for (i = 0 ; i < key->nr_sampler_views; i++) {
2137       lp_sampler_static_texture_state(&draw_sampler[i].texture_state,
2138                                       llvm->draw->sampler_views[PIPE_SHADER_GEOMETRY][i]);
2139    }
2140
2141    return key;
2142 }
2143
2144 void
2145 draw_gs_llvm_dump_variant_key(struct draw_gs_llvm_variant_key *key)
2146 {
2147    unsigned i;
2148    struct draw_sampler_static_state *sampler = key->samplers;
2149
2150    for (i = 0 ; i < key->nr_sampler_views; i++) {
2151       debug_printf("sampler[%i].src_format = %s\n", i,
2152                    util_format_name(sampler[i].texture_state.format));
2153    }
2154 }