src/gallium/drivers/radeonsi/si_shader.c

   1 /*
   2  * Copyright 2012 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * on the rights to use, copy, modify, merge, publish, distribute, sub
   8  * license, and/or sell copies of the Software, and to permit persons to whom
   9  * the Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  18  * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
  19  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  20  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  21  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *      Tom Stellard <thomas.stellard@amd.com>
  25  *      Michel Dänzer <michel.daenzer@amd.com>
  26  *      Christian König <christian.koenig@amd.com>
  27  */
  28
  29 #include "gallivm/lp_bld_const.h"
  30 #include "gallivm/lp_bld_gather.h"
  31 #include "gallivm/lp_bld_intr.h"
  32 #include "gallivm/lp_bld_logic.h"
  33 #include "gallivm/lp_bld_arit.h"
  34 #include "gallivm/lp_bld_bitarit.h"
  35 #include "gallivm/lp_bld_flow.h"
  36 #include "radeon/r600_cs.h"
  37 #include "radeon/radeon_llvm.h"
  38 #include "radeon/radeon_elf_util.h"
  39 #include "radeon/radeon_llvm_emit.h"
  40 #include "util/u_memory.h"
  41 #include "util/u_pstipple.h"
  42 #include "tgsi/tgsi_parse.h"
  43 #include "tgsi/tgsi_util.h"
  44 #include "tgsi/tgsi_dump.h"
  45
  46 #include "si_pipe.h"
  47 #include "si_shader.h"
  48 #include "sid.h"
  49
  50 #include <errno.h>
  51
  52 static const char *scratch_rsrc_dword0_symbol =
  53         "SCRATCH_RSRC_DWORD0";
  54
  55 static const char *scratch_rsrc_dword1_symbol =
  56         "SCRATCH_RSRC_DWORD1";
  57
  58 struct si_shader_output_values
  59 {
  60         LLVMValueRef values[4];
  61         unsigned name;
  62         unsigned sid;
  63 };
  64
  65 struct si_shader_context
  66 {
  67         struct radeon_llvm_context radeon_bld;
  68         struct si_shader *shader;
  69         struct si_screen *screen;
  70
  71         unsigned type; /* TGSI_PROCESSOR_* specifies the type of shader. */
  72         bool is_gs_copy_shader;
  73         int param_streamout_config;
  74         int param_streamout_write_index;
  75         int param_streamout_offset[4];
  76         int param_vertex_id;
  77         int param_rel_auto_id;
  78         int param_vs_prim_id;
  79         int param_instance_id;
  80         int param_tes_u;
  81         int param_tes_v;
  82         int param_tes_rel_patch_id;
  83         int param_tes_patch_id;
  84         int param_es2gs_offset;
  85
  86         LLVMTargetMachineRef tm;
  87
  88         LLVMValueRef const_md;
  89         LLVMValueRef const_buffers[SI_NUM_CONST_BUFFERS];
  90         LLVMValueRef lds;
  91         LLVMValueRef *constants[SI_NUM_CONST_BUFFERS];
  92         LLVMValueRef sampler_views[SI_NUM_SAMPLERS];
  93         LLVMValueRef sampler_states[SI_NUM_SAMPLERS];
  94         LLVMValueRef fmasks[SI_NUM_USER_SAMPLERS];
  95         LLVMValueRef so_buffers[4];
  96         LLVMValueRef esgs_ring;
  97         LLVMValueRef gsvs_ring[4];
  98         LLVMValueRef gs_next_vertex[4];
  99         LLVMValueRef return_value;
 100
 101         LLVMTypeRef voidt;
 102         LLVMTypeRef i1;
 103         LLVMTypeRef i8;
 104         LLVMTypeRef i32;
 105         LLVMTypeRef i128;
 106         LLVMTypeRef f32;
 107         LLVMTypeRef v16i8;
 108         LLVMTypeRef v4i32;
 109         LLVMTypeRef v4f32;
 110         LLVMTypeRef v8i32;
 111 };
 112
 113 static struct si_shader_context *si_shader_context(
 114         struct lp_build_tgsi_context *bld_base)
 115 {
 116         return (struct si_shader_context *)bld_base;
 117 }
 118
 119 static void si_init_shader_ctx(struct si_shader_context *ctx,
 120                                struct si_screen *sscreen,
 121                                struct si_shader *shader,
 122                                LLVMTargetMachineRef tm,
 123                                struct tgsi_shader_info *info);
 124
 125
 126 #define PERSPECTIVE_BASE 0
 127 #define LINEAR_BASE 9
 128
 129 #define SAMPLE_OFFSET 0
 130 #define CENTER_OFFSET 2
 131 #define CENTROID_OFSET 4
 132
 133 #define USE_SGPR_MAX_SUFFIX_LEN 5
 134 #define CONST_ADDR_SPACE 2
 135 #define LOCAL_ADDR_SPACE 3
 136 #define USER_SGPR_ADDR_SPACE 8
 137
 138
 139 #define SENDMSG_GS 2
 140 #define SENDMSG_GS_DONE 3
 141
 142 #define SENDMSG_GS_OP_NOP      (0 << 4)
 143 #define SENDMSG_GS_OP_CUT      (1 << 4)
 144 #define SENDMSG_GS_OP_EMIT     (2 << 4)
 145 #define SENDMSG_GS_OP_EMIT_CUT (3 << 4)
 146
 147 /**
 148  * Returns a unique index for a semantic name and index. The index must be
 149  * less than 64, so that a 64-bit bitmask of used inputs or outputs can be
 150  * calculated.
 151  */
 152 unsigned si_shader_io_get_unique_index(unsigned semantic_name, unsigned index)
 153 {
 154         switch (semantic_name) {
 155         case TGSI_SEMANTIC_POSITION:
 156                 return 0;
 157         case TGSI_SEMANTIC_PSIZE:
 158                 return 1;
 159         case TGSI_SEMANTIC_CLIPDIST:
 160                 assert(index <= 1);
 161                 return 2 + index;
 162         case TGSI_SEMANTIC_GENERIC:
 163                 if (index <= 63-4)
 164                         return 4 + index;
 165                 else
 166                         /* same explanation as in the default statement,
 167                          * the only user hitting this is st/nine.
 168                          */
 169                         return 0;
 170
 171         /* patch indices are completely separate and thus start from 0 */
 172         case TGSI_SEMANTIC_TESSOUTER:
 173                 return 0;
 174         case TGSI_SEMANTIC_TESSINNER:
 175                 return 1;
 176         case TGSI_SEMANTIC_PATCH:
 177                 return 2 + index;
 178
 179         default:
 180                 /* Don't fail here. The result of this function is only used
 181                  * for LS, TCS, TES, and GS, where legacy GL semantics can't
 182                  * occur, but this function is called for all vertex shaders
 183                  * before it's known whether LS will be compiled or not.
 184                  */
 185                 return 0;
 186         }
 187 }
 188
 189 /**
 190  * Get the value of a shader input parameter and extract a bitfield.
 191  */
 192 static LLVMValueRef unpack_param(struct si_shader_context *ctx,
 193                                  unsigned param, unsigned rshift,
 194                                  unsigned bitwidth)
 195 {
 196         struct gallivm_state *gallivm = &ctx->radeon_bld.gallivm;
 197         LLVMValueRef value = LLVMGetParam(ctx->radeon_bld.main_fn,
 198                                           param);
 199
 200         if (rshift)
 201                 value = LLVMBuildLShr(gallivm->builder, value,
 202                                       lp_build_const_int32(gallivm, rshift), "");
 203
 204         if (rshift + bitwidth < 32) {
 205                 unsigned mask = (1 << bitwidth) - 1;
 206                 value = LLVMBuildAnd(gallivm->builder, value,
 207                                      lp_build_const_int32(gallivm, mask), "");
 208         }
 209
 210         return value;
 211 }
 212
 213 static LLVMValueRef get_rel_patch_id(struct si_shader_context *ctx)
 214 {
 215         switch (ctx->type) {
 216         case TGSI_PROCESSOR_TESS_CTRL:
 217                 return unpack_param(ctx, SI_PARAM_REL_IDS, 0, 8);
 218
 219         case TGSI_PROCESSOR_TESS_EVAL:
 220                 return LLVMGetParam(ctx->radeon_bld.main_fn,
 221                                     ctx->param_tes_rel_patch_id);
 222
 223         default:
 224                 assert(0);
 225                 return NULL;
 226         }
 227 }
 228
 229 /* Tessellation shaders pass outputs to the next shader using LDS.
 230  *
 231  * LS outputs = TCS inputs
 232  * TCS outputs = TES inputs
 233  *
 234  * The LDS layout is:
 235  * - TCS inputs for patch 0
 236  * - TCS inputs for patch 1
 237  * - TCS inputs for patch 2             = get_tcs_in_current_patch_offset (if RelPatchID==2)
 238  * - ...
 239  * - TCS outputs for patch 0            = get_tcs_out_patch0_offset
 240  * - Per-patch TCS outputs for patch 0  = get_tcs_out_patch0_patch_data_offset
 241  * - TCS outputs for patch 1
 242  * - Per-patch TCS outputs for patch 1
 243  * - TCS outputs for patch 2            = get_tcs_out_current_patch_offset (if RelPatchID==2)
 244  * - Per-patch TCS outputs for patch 2  = get_tcs_out_current_patch_data_offset (if RelPatchID==2)
 245  * - ...
 246  *
 247  * All three shaders VS(LS), TCS, TES share the same LDS space.
 248  */
 249
 250 static LLVMValueRef
 251 get_tcs_in_patch_stride(struct si_shader_context *ctx)
 252 {
 253         if (ctx->type == TGSI_PROCESSOR_VERTEX)
 254                 return unpack_param(ctx, SI_PARAM_LS_OUT_LAYOUT, 0, 13);
 255         else if (ctx->type == TGSI_PROCESSOR_TESS_CTRL)
 256                 return unpack_param(ctx, SI_PARAM_TCS_IN_LAYOUT, 0, 13);
 257         else {
 258                 assert(0);
 259                 return NULL;
 260         }
 261 }
 262
 263 static LLVMValueRef
 264 get_tcs_out_patch_stride(struct si_shader_context *ctx)
 265 {
 266         return unpack_param(ctx, SI_PARAM_TCS_OUT_LAYOUT, 0, 13);
 267 }
 268
 269 static LLVMValueRef
 270 get_tcs_out_patch0_offset(struct si_shader_context *ctx)
 271 {
 272         return lp_build_mul_imm(&ctx->radeon_bld.soa.bld_base.uint_bld,
 273                                 unpack_param(ctx,
 274                                              SI_PARAM_TCS_OUT_OFFSETS,
 275                                              0, 16),
 276                                 4);
 277 }
 278
 279 static LLVMValueRef
 280 get_tcs_out_patch0_patch_data_offset(struct si_shader_context *ctx)
 281 {
 282         return lp_build_mul_imm(&ctx->radeon_bld.soa.bld_base.uint_bld,
 283                                 unpack_param(ctx,
 284                                              SI_PARAM_TCS_OUT_OFFSETS,
 285                                              16, 16),
 286                                 4);
 287 }
 288
 289 static LLVMValueRef
 290 get_tcs_in_current_patch_offset(struct si_shader_context *ctx)
 291 {
 292         struct gallivm_state *gallivm = &ctx->radeon_bld.gallivm;
 293         LLVMValueRef patch_stride = get_tcs_in_patch_stride(ctx);
 294         LLVMValueRef rel_patch_id = get_rel_patch_id(ctx);
 295
 296         return LLVMBuildMul(gallivm->builder, patch_stride, rel_patch_id, "");
 297 }
 298
 299 static LLVMValueRef
 300 get_tcs_out_current_patch_offset(struct si_shader_context *ctx)
 301 {
 302         struct gallivm_state *gallivm = &ctx->radeon_bld.gallivm;
 303         LLVMValueRef patch0_offset = get_tcs_out_patch0_offset(ctx);
 304         LLVMValueRef patch_stride = get_tcs_out_patch_stride(ctx);
 305         LLVMValueRef rel_patch_id = get_rel_patch_id(ctx);
 306
 307         return LLVMBuildAdd(gallivm->builder, patch0_offset,
 308                             LLVMBuildMul(gallivm->builder, patch_stride,
 309                                          rel_patch_id, ""),
 310                             "");
 311 }
 312
 313 static LLVMValueRef
 314 get_tcs_out_current_patch_data_offset(struct si_shader_context *ctx)
 315 {
 316         struct gallivm_state *gallivm = &ctx->radeon_bld.gallivm;
 317         LLVMValueRef patch0_patch_data_offset =
 318                 get_tcs_out_patch0_patch_data_offset(ctx);
 319         LLVMValueRef patch_stride = get_tcs_out_patch_stride(ctx);
 320         LLVMValueRef rel_patch_id = get_rel_patch_id(ctx);
 321
 322         return LLVMBuildAdd(gallivm->builder, patch0_patch_data_offset,
 323                             LLVMBuildMul(gallivm->builder, patch_stride,
 324                                          rel_patch_id, ""),
 325                             "");
 326 }
 327
 328 static void build_indexed_store(struct si_shader_context *ctx,
 329                                 LLVMValueRef base_ptr, LLVMValueRef index,
 330                                 LLVMValueRef value)
 331 {
 332         struct lp_build_tgsi_context *bld_base = &ctx->radeon_bld.soa.bld_base;
 333         struct gallivm_state *gallivm = bld_base->base.gallivm;
 334         LLVMValueRef indices[2], pointer;
 335
 336         indices[0] = bld_base->uint_bld.zero;
 337         indices[1] = index;
 338
 339         pointer = LLVMBuildGEP(gallivm->builder, base_ptr, indices, 2, "");
 340         LLVMBuildStore(gallivm->builder, value, pointer);
 341 }
 342
 343 /**
 344  * Build an LLVM bytecode indexed load using LLVMBuildGEP + LLVMBuildLoad.
 345  * It's equivalent to doing a load from &base_ptr[index].
 346  *
 347  * \param base_ptr  Where the array starts.
 348  * \param index     The element index into the array.
 349  */
 350 static LLVMValueRef build_indexed_load(struct si_shader_context *ctx,
 351                                        LLVMValueRef base_ptr, LLVMValueRef index)
 352 {
 353         struct lp_build_tgsi_context *bld_base = &ctx->radeon_bld.soa.bld_base;
 354         struct gallivm_state *gallivm = bld_base->base.gallivm;
 355         LLVMValueRef indices[2], pointer;
 356
 357         indices[0] = bld_base->uint_bld.zero;
 358         indices[1] = index;
 359
 360         pointer = LLVMBuildGEP(gallivm->builder, base_ptr, indices, 2, "");
 361         return LLVMBuildLoad(gallivm->builder, pointer, "");
 362 }
 363
 364 /**
 365  * Do a load from &base_ptr[index], but also add a flag that it's loading
 366  * a constant.
 367  */
 368 static LLVMValueRef build_indexed_load_const(
 369         struct si_shader_context *ctx,
 370         LLVMValueRef base_ptr, LLVMValueRef index)
 371 {
 372         LLVMValueRef result = build_indexed_load(ctx, base_ptr, index);
 373         LLVMSetMetadata(result, 1, ctx->const_md);
 374         return result;
 375 }
 376
 377 static LLVMValueRef get_instance_index_for_fetch(
 378         struct radeon_llvm_context *radeon_bld,
 379         unsigned param_start_instance, unsigned divisor)
 380 {
 381         struct si_shader_context *ctx =
 382                 si_shader_context(&radeon_bld->soa.bld_base);
 383         struct gallivm_state *gallivm = radeon_bld->soa.bld_base.base.gallivm;
 384
 385         LLVMValueRef result = LLVMGetParam(radeon_bld->main_fn,
 386                                            ctx->param_instance_id);
 387
 388         /* The division must be done before START_INSTANCE is added. */
 389         if (divisor > 1)
 390                 result = LLVMBuildUDiv(gallivm->builder, result,
 391                                 lp_build_const_int32(gallivm, divisor), "");
 392
 393         return LLVMBuildAdd(gallivm->builder, result,
 394                             LLVMGetParam(radeon_bld->main_fn, param_start_instance), "");
 395 }
 396
 397 static void declare_input_vs(
 398         struct radeon_llvm_context *radeon_bld,
 399         unsigned input_index,
 400         const struct tgsi_full_declaration *decl)
 401 {
 402         struct lp_build_context *base = &radeon_bld->soa.bld_base.base;
 403         struct gallivm_state *gallivm = base->gallivm;
 404         struct si_shader_context *ctx =
 405                 si_shader_context(&radeon_bld->soa.bld_base);
 406         unsigned divisor =
 407                 ctx->shader->key.vs.prolog.instance_divisors[input_index];
 408
 409         unsigned chan;
 410
 411         LLVMValueRef t_list_ptr;
 412         LLVMValueRef t_offset;
 413         LLVMValueRef t_list;
 414         LLVMValueRef attribute_offset;
 415         LLVMValueRef buffer_index;
 416         LLVMValueRef args[3];
 417         LLVMValueRef input;
 418
 419         /* Load the T list */
 420         t_list_ptr = LLVMGetParam(ctx->radeon_bld.main_fn, SI_PARAM_VERTEX_BUFFERS);
 421
 422         t_offset = lp_build_const_int32(gallivm, input_index);
 423
 424         t_list = build_indexed_load_const(ctx, t_list_ptr, t_offset);
 425
 426         /* Build the attribute offset */
 427         attribute_offset = lp_build_const_int32(gallivm, 0);
 428
 429         if (divisor) {
 430                 /* Build index from instance ID, start instance and divisor */
 431                 ctx->shader->uses_instanceid = true;
 432                 buffer_index = get_instance_index_for_fetch(&ctx->radeon_bld,
 433                                                             SI_PARAM_START_INSTANCE,
 434                                                             divisor);
 435         } else {
 436                 /* Load the buffer index for vertices. */
 437                 LLVMValueRef vertex_id = LLVMGetParam(ctx->radeon_bld.main_fn,
 438                                                       ctx->param_vertex_id);
 439                 LLVMValueRef base_vertex = LLVMGetParam(radeon_bld->main_fn,
 440                                                         SI_PARAM_BASE_VERTEX);
 441                 buffer_index = LLVMBuildAdd(gallivm->builder, base_vertex, vertex_id, "");
 442         }
 443
 444         args[0] = t_list;
 445         args[1] = attribute_offset;
 446         args[2] = buffer_index;
 447         input = lp_build_intrinsic(gallivm->builder,
 448                 "llvm.SI.vs.load.input", ctx->v4f32, args, 3,
 449                 LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
 450
 451         /* Break up the vec4 into individual components */
 452         for (chan = 0; chan < 4; chan++) {
 453                 LLVMValueRef llvm_chan = lp_build_const_int32(gallivm, chan);
 454                 /* XXX: Use a helper function for this.  There is one in
 455                  * tgsi_llvm.c. */
 456                 ctx->radeon_bld.inputs[radeon_llvm_reg_index_soa(input_index, chan)] =
 457                                 LLVMBuildExtractElement(gallivm->builder,
 458                                 input, llvm_chan, "");
 459         }
 460 }
 461
 462 static LLVMValueRef get_primitive_id(struct lp_build_tgsi_context *bld_base,
 463                                      unsigned swizzle)
 464 {
 465         struct si_shader_context *ctx = si_shader_context(bld_base);
 466
 467         if (swizzle > 0)
 468                 return bld_base->uint_bld.zero;
 469
 470         switch (ctx->type) {
 471         case TGSI_PROCESSOR_VERTEX:
 472                 return LLVMGetParam(ctx->radeon_bld.main_fn,
 473                                     ctx->param_vs_prim_id);
 474         case TGSI_PROCESSOR_TESS_CTRL:
 475                 return LLVMGetParam(ctx->radeon_bld.main_fn,
 476                                     SI_PARAM_PATCH_ID);
 477         case TGSI_PROCESSOR_TESS_EVAL:
 478                 return LLVMGetParam(ctx->radeon_bld.main_fn,
 479                                     ctx->param_tes_patch_id);
 480         case TGSI_PROCESSOR_GEOMETRY:
 481                 return LLVMGetParam(ctx->radeon_bld.main_fn,
 482                                     SI_PARAM_PRIMITIVE_ID);
 483         default:
 484                 assert(0);
 485                 return bld_base->uint_bld.zero;
 486         }
 487 }
 488
 489 /**
 490  * Return the value of tgsi_ind_register for indexing.
 491  * This is the indirect index with the constant offset added to it.
 492  */
 493 static LLVMValueRef get_indirect_index(struct si_shader_context *ctx,
 494                                        const struct tgsi_ind_register *ind,
 495                                        int rel_index)
 496 {
 497         struct gallivm_state *gallivm = ctx->radeon_bld.soa.bld_base.base.gallivm;
 498         LLVMValueRef result;
 499
 500         result = ctx->radeon_bld.soa.addr[ind->Index][ind->Swizzle];
 501         result = LLVMBuildLoad(gallivm->builder, result, "");
 502         result = LLVMBuildAdd(gallivm->builder, result,
 503                               lp_build_const_int32(gallivm, rel_index), "");
 504         return result;
 505 }
 506
 507 /**
 508  * Calculate a dword address given an input or output register and a stride.
 509  */
 510 static LLVMValueRef get_dw_address(struct si_shader_context *ctx,
 511                                    const struct tgsi_full_dst_register *dst,
 512                                    const struct tgsi_full_src_register *src,
 513                                    LLVMValueRef vertex_dw_stride,
 514                                    LLVMValueRef base_addr)
 515 {
 516         struct gallivm_state *gallivm = ctx->radeon_bld.soa.bld_base.base.gallivm;
 517         struct tgsi_shader_info *info = &ctx->shader->selector->info;
 518         ubyte *name, *index, *array_first;
 519         int first, param;
 520         struct tgsi_full_dst_register reg;
 521
 522         /* Set the register description. The address computation is the same
 523          * for sources and destinations. */
 524         if (src) {
 525                 reg.Register.File = src->Register.File;
 526                 reg.Register.Index = src->Register.Index;
 527                 reg.Register.Indirect = src->Register.Indirect;
 528                 reg.Register.Dimension = src->Register.Dimension;
 529                 reg.Indirect = src->Indirect;
 530                 reg.Dimension = src->Dimension;
 531                 reg.DimIndirect = src->DimIndirect;
 532         } else
 533                 reg = *dst;
 534
 535         /* If the register is 2-dimensional (e.g. an array of vertices
 536          * in a primitive), calculate the base address of the vertex. */
 537         if (reg.Register.Dimension) {
 538                 LLVMValueRef index;
 539
 540                 if (reg.Dimension.Indirect)
 541                         index = get_indirect_index(ctx, &reg.DimIndirect,
 542                                                    reg.Dimension.Index);
 543                 else
 544                         index = lp_build_const_int32(gallivm, reg.Dimension.Index);
 545
 546                 base_addr = LLVMBuildAdd(gallivm->builder, base_addr,
 547                                          LLVMBuildMul(gallivm->builder, index,
 548                                                       vertex_dw_stride, ""), "");
 549         }
 550
 551         /* Get information about the register. */
 552         if (reg.Register.File == TGSI_FILE_INPUT) {
 553                 name = info->input_semantic_name;
 554                 index = info->input_semantic_index;
 555                 array_first = info->input_array_first;
 556         } else if (reg.Register.File == TGSI_FILE_OUTPUT) {
 557                 name = info->output_semantic_name;
 558                 index = info->output_semantic_index;
 559                 array_first = info->output_array_first;
 560         } else {
 561                 assert(0);
 562                 return NULL;
 563         }
 564
 565         if (reg.Register.Indirect) {
 566                 /* Add the relative address of the element. */
 567                 LLVMValueRef ind_index;
 568
 569                 if (reg.Indirect.ArrayID)
 570                         first = array_first[reg.Indirect.ArrayID];
 571                 else
 572                         first = reg.Register.Index;
 573
 574                 ind_index = get_indirect_index(ctx, &reg.Indirect,
 575                                            reg.Register.Index - first);
 576
 577                 base_addr = LLVMBuildAdd(gallivm->builder, base_addr,
 578                                     LLVMBuildMul(gallivm->builder, ind_index,
 579                                                  lp_build_const_int32(gallivm, 4), ""), "");
 580
 581                 param = si_shader_io_get_unique_index(name[first], index[first]);
 582         } else {
 583                 param = si_shader_io_get_unique_index(name[reg.Register.Index],
 584                                                       index[reg.Register.Index]);
 585         }
 586
 587         /* Add the base address of the element. */
 588         return LLVMBuildAdd(gallivm->builder, base_addr,
 589                             lp_build_const_int32(gallivm, param * 4), "");
 590 }
 591
 592 /**
 593  * Load from LDS.
 594  *
 595  * \param type          output value type
 596  * \param swizzle       offset (typically 0..3); it can be ~0, which loads a vec4
 597  * \param dw_addr       address in dwords
 598  */
 599 static LLVMValueRef lds_load(struct lp_build_tgsi_context *bld_base,
 600                              enum tgsi_opcode_type type, unsigned swizzle,
 601                              LLVMValueRef dw_addr)
 602 {
 603         struct si_shader_context *ctx = si_shader_context(bld_base);
 604         struct gallivm_state *gallivm = bld_base->base.gallivm;
 605         LLVMValueRef value;
 606
 607         if (swizzle == ~0) {
 608                 LLVMValueRef values[TGSI_NUM_CHANNELS];
 609
 610                 for (unsigned chan = 0; chan < TGSI_NUM_CHANNELS; chan++)
 611                         values[chan] = lds_load(bld_base, type, chan, dw_addr);
 612
 613                 return lp_build_gather_values(bld_base->base.gallivm, values,
 614                                               TGSI_NUM_CHANNELS);
 615         }
 616
 617         dw_addr = lp_build_add(&bld_base->uint_bld, dw_addr,
 618                             lp_build_const_int32(gallivm, swizzle));
 619
 620         value = build_indexed_load(ctx, ctx->lds, dw_addr);
 621         if (type == TGSI_TYPE_DOUBLE) {
 622                 LLVMValueRef value2;
 623                 dw_addr = lp_build_add(&bld_base->uint_bld, dw_addr,
 624                                        lp_build_const_int32(gallivm, swizzle + 1));
 625                 value2 = build_indexed_load(ctx, ctx->lds, dw_addr);
 626                 return radeon_llvm_emit_fetch_double(bld_base, value, value2);
 627         }
 628
 629         return LLVMBuildBitCast(gallivm->builder, value,
 630                                 tgsi2llvmtype(bld_base, type), "");
 631 }
 632
 633 /**
 634  * Store to LDS.
 635  *
 636  * \param swizzle       offset (typically 0..3)
 637  * \param dw_addr       address in dwords
 638  * \param value         value to store
 639  */
 640 static void lds_store(struct lp_build_tgsi_context *bld_base,
 641                       unsigned swizzle, LLVMValueRef dw_addr,
 642                       LLVMValueRef value)
 643 {
 644         struct si_shader_context *ctx = si_shader_context(bld_base);
 645         struct gallivm_state *gallivm = bld_base->base.gallivm;
 646
 647         dw_addr = lp_build_add(&bld_base->uint_bld, dw_addr,
 648                             lp_build_const_int32(gallivm, swizzle));
 649
 650         value = LLVMBuildBitCast(gallivm->builder, value, ctx->i32, "");
 651         build_indexed_store(ctx, ctx->lds,
 652                             dw_addr, value);
 653 }
 654
 655 static LLVMValueRef fetch_input_tcs(
 656         struct lp_build_tgsi_context *bld_base,
 657         const struct tgsi_full_src_register *reg,
 658         enum tgsi_opcode_type type, unsigned swizzle)
 659 {
 660         struct si_shader_context *ctx = si_shader_context(bld_base);
 661         LLVMValueRef dw_addr, stride;
 662
 663         stride = unpack_param(ctx, SI_PARAM_TCS_IN_LAYOUT, 13, 8);
 664         dw_addr = get_tcs_in_current_patch_offset(ctx);
 665         dw_addr = get_dw_address(ctx, NULL, reg, stride, dw_addr);
 666
 667         return lds_load(bld_base, type, swizzle, dw_addr);
 668 }
 669
 670 static LLVMValueRef fetch_output_tcs(
 671                 struct lp_build_tgsi_context *bld_base,
 672                 const struct tgsi_full_src_register *reg,
 673                 enum tgsi_opcode_type type, unsigned swizzle)
 674 {
 675         struct si_shader_context *ctx = si_shader_context(bld_base);
 676         LLVMValueRef dw_addr, stride;
 677
 678         if (reg->Register.Dimension) {
 679                 stride = unpack_param(ctx, SI_PARAM_TCS_OUT_LAYOUT, 13, 8);
 680                 dw_addr = get_tcs_out_current_patch_offset(ctx);
 681                 dw_addr = get_dw_address(ctx, NULL, reg, stride, dw_addr);
 682         } else {
 683                 dw_addr = get_tcs_out_current_patch_data_offset(ctx);
 684                 dw_addr = get_dw_address(ctx, NULL, reg, NULL, dw_addr);
 685         }
 686
 687         return lds_load(bld_base, type, swizzle, dw_addr);
 688 }
 689
 690 static LLVMValueRef fetch_input_tes(
 691         struct lp_build_tgsi_context *bld_base,
 692         const struct tgsi_full_src_register *reg,
 693         enum tgsi_opcode_type type, unsigned swizzle)
 694 {
 695         struct si_shader_context *ctx = si_shader_context(bld_base);
 696         LLVMValueRef dw_addr, stride;
 697
 698         if (reg->Register.Dimension) {
 699                 stride = unpack_param(ctx, SI_PARAM_TCS_OUT_LAYOUT, 13, 8);
 700                 dw_addr = get_tcs_out_current_patch_offset(ctx);
 701                 dw_addr = get_dw_address(ctx, NULL, reg, stride, dw_addr);
 702         } else {
 703                 dw_addr = get_tcs_out_current_patch_data_offset(ctx);
 704                 dw_addr = get_dw_address(ctx, NULL, reg, NULL, dw_addr);
 705         }
 706
 707         return lds_load(bld_base, type, swizzle, dw_addr);
 708 }
 709
 710 static void store_output_tcs(struct lp_build_tgsi_context *bld_base,
 711                              const struct tgsi_full_instruction *inst,
 712                              const struct tgsi_opcode_info *info,
 713                              LLVMValueRef dst[4])
 714 {
 715         struct si_shader_context *ctx = si_shader_context(bld_base);
 716         const struct tgsi_full_dst_register *reg = &inst->Dst[0];
 717         unsigned chan_index;
 718         LLVMValueRef dw_addr, stride;
 719
 720         /* Only handle per-patch and per-vertex outputs here.
 721          * Vectors will be lowered to scalars and this function will be called again.
 722          */
 723         if (reg->Register.File != TGSI_FILE_OUTPUT ||
 724             (dst[0] && LLVMGetTypeKind(LLVMTypeOf(dst[0])) == LLVMVectorTypeKind)) {
 725                 radeon_llvm_emit_store(bld_base, inst, info, dst);
 726                 return;
 727         }
 728
 729         if (reg->Register.Dimension) {
 730                 stride = unpack_param(ctx, SI_PARAM_TCS_OUT_LAYOUT, 13, 8);
 731                 dw_addr = get_tcs_out_current_patch_offset(ctx);
 732                 dw_addr = get_dw_address(ctx, reg, NULL, stride, dw_addr);
 733         } else {
 734                 dw_addr = get_tcs_out_current_patch_data_offset(ctx);
 735                 dw_addr = get_dw_address(ctx, reg, NULL, NULL, dw_addr);
 736         }
 737
 738         TGSI_FOR_EACH_DST0_ENABLED_CHANNEL(inst, chan_index) {
 739                 LLVMValueRef value = dst[chan_index];
 740
 741                 if (inst->Instruction.Saturate)
 742                         value = radeon_llvm_saturate(bld_base, value);
 743
 744                 lds_store(bld_base, chan_index, dw_addr, value);
 745         }
 746 }
 747
 748 static LLVMValueRef fetch_input_gs(
 749         struct lp_build_tgsi_context *bld_base,
 750         const struct tgsi_full_src_register *reg,
 751         enum tgsi_opcode_type type,
 752         unsigned swizzle)
 753 {
 754         struct lp_build_context *base = &bld_base->base;
 755         struct si_shader_context *ctx = si_shader_context(bld_base);
 756         struct si_shader *shader = ctx->shader;
 757         struct lp_build_context *uint = &ctx->radeon_bld.soa.bld_base.uint_bld;
 758         struct gallivm_state *gallivm = base->gallivm;
 759         LLVMValueRef vtx_offset;
 760         LLVMValueRef args[9];
 761         unsigned vtx_offset_param;
 762         struct tgsi_shader_info *info = &shader->selector->info;
 763         unsigned semantic_name = info->input_semantic_name[reg->Register.Index];
 764         unsigned semantic_index = info->input_semantic_index[reg->Register.Index];
 765         unsigned param;
 766         LLVMValueRef value;
 767
 768         if (swizzle != ~0 && semantic_name == TGSI_SEMANTIC_PRIMID)
 769                 return get_primitive_id(bld_base, swizzle);
 770
 771         if (!reg->Register.Dimension)
 772                 return NULL;
 773
 774         if (swizzle == ~0) {
 775                 LLVMValueRef values[TGSI_NUM_CHANNELS];
 776                 unsigned chan;
 777                 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
 778                         values[chan] = fetch_input_gs(bld_base, reg, type, chan);
 779                 }
 780                 return lp_build_gather_values(bld_base->base.gallivm, values,
 781                                               TGSI_NUM_CHANNELS);
 782         }
 783
 784         /* Get the vertex offset parameter */
 785         vtx_offset_param = reg->Dimension.Index;
 786         if (vtx_offset_param < 2) {
 787                 vtx_offset_param += SI_PARAM_VTX0_OFFSET;
 788         } else {
 789                 assert(vtx_offset_param < 6);
 790                 vtx_offset_param += SI_PARAM_VTX2_OFFSET - 2;
 791         }
 792         vtx_offset = lp_build_mul_imm(uint,
 793                                       LLVMGetParam(ctx->radeon_bld.main_fn,
 794                                                    vtx_offset_param),
 795                                       4);
 796
 797         param = si_shader_io_get_unique_index(semantic_name, semantic_index);
 798         args[0] = ctx->esgs_ring;
 799         args[1] = vtx_offset;
 800         args[2] = lp_build_const_int32(gallivm, (param * 4 + swizzle) * 256);
 801         args[3] = uint->zero;
 802         args[4] = uint->one;  /* OFFEN */
 803         args[5] = uint->zero; /* IDXEN */
 804         args[6] = uint->one;  /* GLC */
 805         args[7] = uint->zero; /* SLC */
 806         args[8] = uint->zero; /* TFE */
 807
 808         value = lp_build_intrinsic(gallivm->builder,
 809                                    "llvm.SI.buffer.load.dword.i32.i32",
 810                                    ctx->i32, args, 9,
 811                                    LLVMReadOnlyAttribute | LLVMNoUnwindAttribute);
 812         if (type == TGSI_TYPE_DOUBLE) {
 813                 LLVMValueRef value2;
 814                 args[2] = lp_build_const_int32(gallivm, (param * 4 + swizzle + 1) * 256);
 815                 value2 = lp_build_intrinsic(gallivm->builder,
 816                                             "llvm.SI.buffer.load.dword.i32.i32",
 817                                             ctx->i32, args, 9,
 818                                             LLVMReadOnlyAttribute | LLVMNoUnwindAttribute);
 819                 return radeon_llvm_emit_fetch_double(bld_base,
 820                                                      value, value2);
 821         }
 822         return LLVMBuildBitCast(gallivm->builder,
 823                                 value,
 824                                 tgsi2llvmtype(bld_base, type), "");
 825 }
 826
 827 static int lookup_interp_param_index(unsigned interpolate, unsigned location)
 828 {
 829         switch (interpolate) {
 830         case TGSI_INTERPOLATE_CONSTANT:
 831                 return 0;
 832
 833         case TGSI_INTERPOLATE_LINEAR:
 834                 if (location == TGSI_INTERPOLATE_LOC_SAMPLE)
 835                         return SI_PARAM_LINEAR_SAMPLE;
 836                 else if (location == TGSI_INTERPOLATE_LOC_CENTROID)
 837                         return SI_PARAM_LINEAR_CENTROID;
 838                 else
 839                         return SI_PARAM_LINEAR_CENTER;
 840                 break;
 841         case TGSI_INTERPOLATE_COLOR:
 842         case TGSI_INTERPOLATE_PERSPECTIVE:
 843                 if (location == TGSI_INTERPOLATE_LOC_SAMPLE)
 844                         return SI_PARAM_PERSP_SAMPLE;
 845                 else if (location == TGSI_INTERPOLATE_LOC_CENTROID)
 846                         return SI_PARAM_PERSP_CENTROID;
 847                 else
 848                         return SI_PARAM_PERSP_CENTER;
 849                 break;
 850         default:
 851                 fprintf(stderr, "Warning: Unhandled interpolation mode.\n");
 852                 return -1;
 853         }
 854 }
 855
 856 /* This shouldn't be used by explicit INTERP opcodes. */
 857 static unsigned select_interp_param(struct si_shader_context *ctx,
 858                                     unsigned param)
 859 {
 860         if (!ctx->shader->key.ps.prolog.force_persample_interp)
 861                 return param;
 862
 863         /* If the shader doesn't use center/centroid, just return the parameter.
 864          *
 865          * If the shader only uses one set of (i,j), "si_emit_spi_ps_input" can
 866          * switch between center/centroid and sample without shader changes.
 867          */
 868         switch (param) {
 869         case SI_PARAM_PERSP_CENTROID:
 870         case SI_PARAM_PERSP_CENTER:
 871                 return SI_PARAM_PERSP_SAMPLE;
 872
 873         case SI_PARAM_LINEAR_CENTROID:
 874         case SI_PARAM_LINEAR_CENTER:
 875                 return SI_PARAM_LINEAR_SAMPLE;
 876
 877         default:
 878                 return param;
 879         }
 880 }
 881
 882 /**
 883  * Interpolate a fragment shader input.
 884  *
 885  * @param ctx           context
 886  * @param input_index           index of the input in hardware
 887  * @param semantic_name         TGSI_SEMANTIC_*
 888  * @param semantic_index        semantic index
 889  * @param num_interp_inputs     number of all interpolated inputs (= BCOLOR offset)
 890  * @param colors_read_mask      color components read (4 bits for each color, 8 bits in total)
 891  * @param interp_param          interpolation weights (i,j)
 892  * @param prim_mask             SI_PARAM_PRIM_MASK
 893  * @param face                  SI_PARAM_FRONT_FACE
 894  * @param result                the return value (4 components)
 895  */
 896 static void interp_fs_input(struct si_shader_context *ctx,
 897                             unsigned input_index,
 898                             unsigned semantic_name,
 899                             unsigned semantic_index,
 900                             unsigned num_interp_inputs,
 901                             unsigned colors_read_mask,
 902                             LLVMValueRef interp_param,
 903                             LLVMValueRef prim_mask,
 904                             LLVMValueRef face,
 905                             LLVMValueRef result[4])
 906 {
 907         struct lp_build_context *base = &ctx->radeon_bld.soa.bld_base.base;
 908         struct lp_build_context *uint = &ctx->radeon_bld.soa.bld_base.uint_bld;
 909         struct gallivm_state *gallivm = base->gallivm;
 910         const char *intr_name;
 911         LLVMValueRef attr_number;
 912
 913         unsigned chan;
 914
 915         attr_number = lp_build_const_int32(gallivm, input_index);
 916
 917         /* fs.constant returns the param from the middle vertex, so it's not
 918          * really useful for flat shading. It's meant to be used for custom
 919          * interpolation (but the intrinsic can't fetch from the other two
 920          * vertices).
 921          *
 922          * Luckily, it doesn't matter, because we rely on the FLAT_SHADE state
 923          * to do the right thing. The only reason we use fs.constant is that
 924          * fs.interp cannot be used on integers, because they can be equal
 925          * to NaN.
 926          */
 927         intr_name = interp_param ? "llvm.SI.fs.interp" : "llvm.SI.fs.constant";
 928
 929         if (semantic_name == TGSI_SEMANTIC_COLOR &&
 930             ctx->shader->key.ps.prolog.color_two_side) {
 931                 LLVMValueRef args[4];
 932                 LLVMValueRef is_face_positive;
 933                 LLVMValueRef back_attr_number;
 934
 935                 /* If BCOLOR0 is used, BCOLOR1 is at offset "num_inputs + 1",
 936                  * otherwise it's at offset "num_inputs".
 937                  */
 938                 unsigned back_attr_offset = num_interp_inputs;
 939                 if (semantic_index == 1 && colors_read_mask & 0xf)
 940                         back_attr_offset += 1;
 941
 942                 back_attr_number = lp_build_const_int32(gallivm, back_attr_offset);
 943
 944                 is_face_positive = LLVMBuildICmp(gallivm->builder, LLVMIntNE,
 945                                                  face, uint->zero, "");
 946
 947                 args[2] = prim_mask;
 948                 args[3] = interp_param;
 949                 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
 950                         LLVMValueRef llvm_chan = lp_build_const_int32(gallivm, chan);
 951                         LLVMValueRef front, back;
 952
 953                         args[0] = llvm_chan;
 954                         args[1] = attr_number;
 955                         front = lp_build_intrinsic(gallivm->builder, intr_name,
 956                                                 ctx->f32, args, args[3] ? 4 : 3,
 957                                                 LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
 958
 959                         args[1] = back_attr_number;
 960                         back = lp_build_intrinsic(gallivm->builder, intr_name,
 961                                                ctx->f32, args, args[3] ? 4 : 3,
 962                                                LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
 963
 964                         result[chan] = LLVMBuildSelect(gallivm->builder,
 965                                                 is_face_positive,
 966                                                 front,
 967                                                 back,
 968                                                 "");
 969                 }
 970         } else if (semantic_name == TGSI_SEMANTIC_FOG) {
 971                 LLVMValueRef args[4];
 972
 973                 args[0] = uint->zero;
 974                 args[1] = attr_number;
 975                 args[2] = prim_mask;
 976                 args[3] = interp_param;
 977                 result[0] = lp_build_intrinsic(gallivm->builder, intr_name,
 978                                         ctx->f32, args, args[3] ? 4 : 3,
 979                                         LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
 980                 result[1] =
 981                 result[2] = lp_build_const_float(gallivm, 0.0f);
 982                 result[3] = lp_build_const_float(gallivm, 1.0f);
 983         } else {
 984                 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
 985                         LLVMValueRef args[4];
 986                         LLVMValueRef llvm_chan = lp_build_const_int32(gallivm, chan);
 987
 988                         args[0] = llvm_chan;
 989                         args[1] = attr_number;
 990                         args[2] = prim_mask;
 991                         args[3] = interp_param;
 992                         result[chan] = lp_build_intrinsic(gallivm->builder, intr_name,
 993                                                 ctx->f32, args, args[3] ? 4 : 3,
 994                                                 LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
 995                 }
 996         }
 997 }
 998
 999 static void declare_input_fs(
1000         struct radeon_llvm_context *radeon_bld,
1001         unsigned input_index,
1002         const struct tgsi_full_declaration *decl)
1003 {
1004         struct si_shader_context *ctx =
1005                 si_shader_context(&radeon_bld->soa.bld_base);
1006         struct si_shader *shader = ctx->shader;
1007         LLVMValueRef main_fn = radeon_bld->main_fn;
1008         LLVMValueRef interp_param = NULL;
1009         int interp_param_idx;
1010
1011         interp_param_idx = lookup_interp_param_index(decl->Interp.Interpolate,
1012                                                      decl->Interp.Location);
1013         if (interp_param_idx == -1)
1014                 return;
1015         else if (interp_param_idx) {
1016                 interp_param_idx = select_interp_param(ctx,
1017                                                        interp_param_idx);
1018                 interp_param = LLVMGetParam(main_fn, interp_param_idx);
1019         }
1020
1021         interp_fs_input(ctx, input_index, decl->Semantic.Name,
1022                         decl->Semantic.Index, shader->selector->info.num_inputs,
1023                         shader->selector->info.colors_read, interp_param,
1024                         LLVMGetParam(main_fn, SI_PARAM_PRIM_MASK),
1025                         LLVMGetParam(main_fn, SI_PARAM_FRONT_FACE),
1026                         &radeon_bld->inputs[radeon_llvm_reg_index_soa(input_index, 0)]);
1027 }
1028
1029 static LLVMValueRef get_sample_id(struct radeon_llvm_context *radeon_bld)
1030 {
1031         return unpack_param(si_shader_context(&radeon_bld->soa.bld_base),
1032                             SI_PARAM_ANCILLARY, 8, 4);
1033 }
1034
1035 /**
1036  * Load a dword from a constant buffer.
1037  */
1038 static LLVMValueRef buffer_load_const(LLVMBuilderRef builder, LLVMValueRef resource,
1039                                       LLVMValueRef offset, LLVMTypeRef return_type)
1040 {
1041         LLVMValueRef args[2] = {resource, offset};
1042
1043         return lp_build_intrinsic(builder, "llvm.SI.load.const", return_type, args, 2,
1044                                LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
1045 }
1046
1047 static LLVMValueRef load_sample_position(struct radeon_llvm_context *radeon_bld, LLVMValueRef sample_id)
1048 {
1049         struct si_shader_context *ctx =
1050                 si_shader_context(&radeon_bld->soa.bld_base);
1051         struct lp_build_context *uint_bld = &radeon_bld->soa.bld_base.uint_bld;
1052         struct gallivm_state *gallivm = &radeon_bld->gallivm;
1053         LLVMBuilderRef builder = gallivm->builder;
1054         LLVMValueRef desc = LLVMGetParam(ctx->radeon_bld.main_fn, SI_PARAM_CONST_BUFFERS);
1055         LLVMValueRef buf_index = lp_build_const_int32(gallivm, SI_DRIVER_STATE_CONST_BUF);
1056         LLVMValueRef resource = build_indexed_load_const(ctx, desc, buf_index);
1057
1058         /* offset = sample_id * 8  (8 = 2 floats containing samplepos.xy) */
1059         LLVMValueRef offset0 = lp_build_mul_imm(uint_bld, sample_id, 8);
1060         LLVMValueRef offset1 = LLVMBuildAdd(builder, offset0, lp_build_const_int32(gallivm, 4), "");
1061
1062         LLVMValueRef pos[4] = {
1063                 buffer_load_const(builder, resource, offset0, ctx->f32),
1064                 buffer_load_const(builder, resource, offset1, ctx->f32),
1065                 lp_build_const_float(gallivm, 0),
1066                 lp_build_const_float(gallivm, 0)
1067         };
1068
1069         return lp_build_gather_values(gallivm, pos, 4);
1070 }
1071
1072 static void declare_system_value(
1073         struct radeon_llvm_context *radeon_bld,
1074         unsigned index,
1075         const struct tgsi_full_declaration *decl)
1076 {
1077         struct si_shader_context *ctx =
1078                 si_shader_context(&radeon_bld->soa.bld_base);
1079         struct lp_build_context *bld = &radeon_bld->soa.bld_base.base;
1080         struct gallivm_state *gallivm = &radeon_bld->gallivm;
1081         LLVMValueRef value = 0;
1082
1083         switch (decl->Semantic.Name) {
1084         case TGSI_SEMANTIC_INSTANCEID:
1085                 value = LLVMGetParam(radeon_bld->main_fn,
1086                                      ctx->param_instance_id);
1087                 break;
1088
1089         case TGSI_SEMANTIC_VERTEXID:
1090                 value = LLVMBuildAdd(gallivm->builder,
1091                                      LLVMGetParam(radeon_bld->main_fn,
1092                                                   ctx->param_vertex_id),
1093                                      LLVMGetParam(radeon_bld->main_fn,
1094                                                   SI_PARAM_BASE_VERTEX), "");
1095                 break;
1096
1097         case TGSI_SEMANTIC_VERTEXID_NOBASE:
1098                 value = LLVMGetParam(radeon_bld->main_fn,
1099                                      ctx->param_vertex_id);
1100                 break;
1101
1102         case TGSI_SEMANTIC_BASEVERTEX:
1103                 value = LLVMGetParam(radeon_bld->main_fn,
1104                                      SI_PARAM_BASE_VERTEX);
1105                 break;
1106
1107         case TGSI_SEMANTIC_INVOCATIONID:
1108                 if (ctx->type == TGSI_PROCESSOR_TESS_CTRL)
1109                         value = unpack_param(ctx, SI_PARAM_REL_IDS, 8, 5);
1110                 else if (ctx->type == TGSI_PROCESSOR_GEOMETRY)
1111                         value = LLVMGetParam(radeon_bld->main_fn,
1112                                              SI_PARAM_GS_INSTANCE_ID);
1113                 else
1114                         assert(!"INVOCATIONID not implemented");
1115                 break;
1116
1117         case TGSI_SEMANTIC_POSITION:
1118         {
1119                 LLVMValueRef pos[4] = {
1120                         LLVMGetParam(radeon_bld->main_fn, SI_PARAM_POS_X_FLOAT),
1121                         LLVMGetParam(radeon_bld->main_fn, SI_PARAM_POS_Y_FLOAT),
1122                         LLVMGetParam(radeon_bld->main_fn, SI_PARAM_POS_Z_FLOAT),
1123                         lp_build_emit_llvm_unary(&radeon_bld->soa.bld_base, TGSI_OPCODE_RCP,
1124                                                  LLVMGetParam(radeon_bld->main_fn,
1125                                                               SI_PARAM_POS_W_FLOAT)),
1126                 };
1127                 value = lp_build_gather_values(gallivm, pos, 4);
1128                 break;
1129         }
1130
1131         case TGSI_SEMANTIC_FACE:
1132                 value = LLVMGetParam(radeon_bld->main_fn, SI_PARAM_FRONT_FACE);
1133                 break;
1134
1135         case TGSI_SEMANTIC_SAMPLEID:
1136                 value = get_sample_id(radeon_bld);
1137                 break;
1138
1139         case TGSI_SEMANTIC_SAMPLEPOS: {
1140                 LLVMValueRef pos[4] = {
1141                         LLVMGetParam(radeon_bld->main_fn, SI_PARAM_POS_X_FLOAT),
1142                         LLVMGetParam(radeon_bld->main_fn, SI_PARAM_POS_Y_FLOAT),
1143                         lp_build_const_float(gallivm, 0),
1144                         lp_build_const_float(gallivm, 0)
1145                 };
1146                 pos[0] = lp_build_emit_llvm_unary(&radeon_bld->soa.bld_base,
1147                                                   TGSI_OPCODE_FRC, pos[0]);
1148                 pos[1] = lp_build_emit_llvm_unary(&radeon_bld->soa.bld_base,
1149                                                   TGSI_OPCODE_FRC, pos[1]);
1150                 value = lp_build_gather_values(gallivm, pos, 4);
1151                 break;
1152         }
1153
1154         case TGSI_SEMANTIC_SAMPLEMASK:
1155                 /* This can only occur with the OpenGL Core profile, which
1156                  * doesn't support smoothing.
1157                  */
1158                 value = LLVMGetParam(radeon_bld->main_fn, SI_PARAM_SAMPLE_COVERAGE);
1159                 break;
1160
1161         case TGSI_SEMANTIC_TESSCOORD:
1162         {
1163                 LLVMValueRef coord[4] = {
1164                         LLVMGetParam(radeon_bld->main_fn, ctx->param_tes_u),
1165                         LLVMGetParam(radeon_bld->main_fn, ctx->param_tes_v),
1166                         bld->zero,
1167                         bld->zero
1168                 };
1169
1170                 /* For triangles, the vector should be (u, v, 1-u-v). */
1171                 if (ctx->shader->selector->info.properties[TGSI_PROPERTY_TES_PRIM_MODE] ==
1172                     PIPE_PRIM_TRIANGLES)
1173                         coord[2] = lp_build_sub(bld, bld->one,
1174                                                 lp_build_add(bld, coord[0], coord[1]));
1175
1176                 value = lp_build_gather_values(gallivm, coord, 4);
1177                 break;
1178         }
1179
1180         case TGSI_SEMANTIC_VERTICESIN:
1181                 value = unpack_param(ctx, SI_PARAM_TCS_OUT_LAYOUT, 26, 6);
1182                 break;
1183
1184         case TGSI_SEMANTIC_TESSINNER:
1185         case TGSI_SEMANTIC_TESSOUTER:
1186         {
1187                 LLVMValueRef dw_addr;
1188                 int param = si_shader_io_get_unique_index(decl->Semantic.Name, 0);
1189
1190                 dw_addr = get_tcs_out_current_patch_data_offset(ctx);
1191                 dw_addr = LLVMBuildAdd(gallivm->builder, dw_addr,
1192                                        lp_build_const_int32(gallivm, param * 4), "");
1193
1194                 value = lds_load(&radeon_bld->soa.bld_base, TGSI_TYPE_FLOAT,
1195                                  ~0, dw_addr);
1196                 break;
1197         }
1198
1199         case TGSI_SEMANTIC_PRIMID:
1200                 value = get_primitive_id(&radeon_bld->soa.bld_base, 0);
1201                 break;
1202
1203         default:
1204                 assert(!"unknown system value");
1205                 return;
1206         }
1207
1208         radeon_bld->system_values[index] = value;
1209 }
1210
1211 static LLVMValueRef fetch_constant(
1212         struct lp_build_tgsi_context *bld_base,
1213         const struct tgsi_full_src_register *reg,
1214         enum tgsi_opcode_type type,
1215         unsigned swizzle)
1216 {
1217         struct si_shader_context *ctx = si_shader_context(bld_base);
1218         struct lp_build_context *base = &bld_base->base;
1219         const struct tgsi_ind_register *ireg = &reg->Indirect;
1220         unsigned buf, idx;
1221
1222         LLVMValueRef addr, bufp;
1223         LLVMValueRef result;
1224
1225         if (swizzle == LP_CHAN_ALL) {
1226                 unsigned chan;
1227                 LLVMValueRef values[4];
1228                 for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan)
1229                         values[chan] = fetch_constant(bld_base, reg, type, chan);
1230
1231                 return lp_build_gather_values(bld_base->base.gallivm, values, 4);
1232         }
1233
1234         buf = reg->Register.Dimension ? reg->Dimension.Index : 0;
1235         idx = reg->Register.Index * 4 + swizzle;
1236
1237         if (!reg->Register.Indirect && !reg->Dimension.Indirect) {
1238                 if (type != TGSI_TYPE_DOUBLE)
1239                         return bitcast(bld_base, type, ctx->constants[buf][idx]);
1240                 else {
1241                         return radeon_llvm_emit_fetch_double(bld_base,
1242                                                              ctx->constants[buf][idx],
1243                                                              ctx->constants[buf][idx + 1]);
1244                 }
1245         }
1246
1247         if (reg->Register.Dimension && reg->Dimension.Indirect) {
1248                 LLVMValueRef ptr = LLVMGetParam(ctx->radeon_bld.main_fn, SI_PARAM_CONST_BUFFERS);
1249                 LLVMValueRef index;
1250                 index = get_indirect_index(ctx, &reg->DimIndirect,
1251                                                    reg->Dimension.Index);
1252                 bufp = build_indexed_load_const(ctx, ptr, index);
1253         } else
1254                 bufp = ctx->const_buffers[buf];
1255
1256         addr = ctx->radeon_bld.soa.addr[ireg->Index][ireg->Swizzle];
1257         addr = LLVMBuildLoad(base->gallivm->builder, addr, "load addr reg");
1258         addr = lp_build_mul_imm(&bld_base->uint_bld, addr, 16);
1259         addr = lp_build_add(&bld_base->uint_bld, addr,
1260                             lp_build_const_int32(base->gallivm, idx * 4));
1261
1262         result = buffer_load_const(base->gallivm->builder, bufp,
1263                                    addr, ctx->f32);
1264
1265         if (type != TGSI_TYPE_DOUBLE)
1266                 result = bitcast(bld_base, type, result);
1267         else {
1268                 LLVMValueRef addr2, result2;
1269                 addr2 = ctx->radeon_bld.soa.addr[ireg->Index][ireg->Swizzle + 1];
1270                 addr2 = LLVMBuildLoad(base->gallivm->builder, addr2, "load addr reg2");
1271                 addr2 = lp_build_mul_imm(&bld_base->uint_bld, addr2, 16);
1272                 addr2 = lp_build_add(&bld_base->uint_bld, addr2,
1273                                      lp_build_const_int32(base->gallivm, idx * 4));
1274
1275                 result2 = buffer_load_const(base->gallivm->builder, ctx->const_buffers[buf],
1276                                    addr2, ctx->f32);
1277
1278                 result = radeon_llvm_emit_fetch_double(bld_base,
1279                                                        result, result2);
1280         }
1281         return result;
1282 }
1283
1284 /* Upper 16 bits must be zero. */
1285 static LLVMValueRef si_llvm_pack_two_int16(struct gallivm_state *gallivm,
1286                                            LLVMValueRef val[2])
1287 {
1288         return LLVMBuildOr(gallivm->builder, val[0],
1289                            LLVMBuildShl(gallivm->builder, val[1],
1290                                         lp_build_const_int32(gallivm, 16),
1291                                         ""), "");
1292 }
1293
1294 /* Upper 16 bits are ignored and will be dropped. */
1295 static LLVMValueRef si_llvm_pack_two_int32_as_int16(struct gallivm_state *gallivm,
1296                                                     LLVMValueRef val[2])
1297 {
1298         LLVMValueRef v[2] = {
1299                 LLVMBuildAnd(gallivm->builder, val[0],
1300                              lp_build_const_int32(gallivm, 0xffff), ""),
1301                 val[1],
1302         };
1303         return si_llvm_pack_two_int16(gallivm, v);
1304 }
1305
1306 /* Initialize arguments for the shader export intrinsic */
1307 static void si_llvm_init_export_args(struct lp_build_tgsi_context *bld_base,
1308                                      LLVMValueRef *values,
1309                                      unsigned target,
1310                                      LLVMValueRef *args)
1311 {
1312         struct si_shader_context *ctx = si_shader_context(bld_base);
1313         struct lp_build_context *uint =
1314                                 &ctx->radeon_bld.soa.bld_base.uint_bld;
1315         struct lp_build_context *base = &bld_base->base;
1316         struct gallivm_state *gallivm = base->gallivm;
1317         LLVMBuilderRef builder = base->gallivm->builder;
1318         LLVMValueRef val[4];
1319         unsigned spi_shader_col_format = V_028714_SPI_SHADER_32_ABGR;
1320         unsigned chan;
1321         bool is_int8;
1322
1323         /* Default is 0xf. Adjusted below depending on the format. */
1324         args[0] = lp_build_const_int32(base->gallivm, 0xf); /* writemask */
1325
1326         /* Specify whether the EXEC mask represents the valid mask */
1327         args[1] = uint->zero;
1328
1329         /* Specify whether this is the last export */
1330         args[2] = uint->zero;
1331
1332         /* Specify the target we are exporting */
1333         args[3] = lp_build_const_int32(base->gallivm, target);
1334
1335         if (ctx->type == TGSI_PROCESSOR_FRAGMENT) {
1336                 const union si_shader_key *key = &ctx->shader->key;
1337                 unsigned col_formats = key->ps.epilog.spi_shader_col_format;
1338                 int cbuf = target - V_008DFC_SQ_EXP_MRT;
1339
1340                 assert(cbuf >= 0 && cbuf < 8);
1341                 spi_shader_col_format = (col_formats >> (cbuf * 4)) & 0xf;
1342                 is_int8 = (key->ps.epilog.color_is_int8 >> cbuf) & 0x1;
1343         }
1344
1345         args[4] = uint->zero; /* COMPR flag */
1346         args[5] = base->undef;
1347         args[6] = base->undef;
1348         args[7] = base->undef;
1349         args[8] = base->undef;
1350
1351         switch (spi_shader_col_format) {
1352         case V_028714_SPI_SHADER_ZERO:
1353                 args[0] = uint->zero; /* writemask */
1354                 args[3] = lp_build_const_int32(base->gallivm, V_008DFC_SQ_EXP_NULL);
1355                 break;
1356
1357         case V_028714_SPI_SHADER_32_R:
1358                 args[0] = uint->one; /* writemask */
1359                 args[5] = values[0];
1360                 break;
1361
1362         case V_028714_SPI_SHADER_32_GR:
1363                 args[0] = lp_build_const_int32(base->gallivm, 0x3); /* writemask */
1364                 args[5] = values[0];
1365                 args[6] = values[1];
1366                 break;
1367
1368         case V_028714_SPI_SHADER_32_AR:
1369                 args[0] = lp_build_const_int32(base->gallivm, 0x9); /* writemask */
1370                 args[5] = values[0];
1371                 args[8] = values[3];
1372                 break;
1373
1374         case V_028714_SPI_SHADER_FP16_ABGR:
1375                 args[4] = uint->one; /* COMPR flag */
1376
1377                 for (chan = 0; chan < 2; chan++) {
1378                         LLVMValueRef pack_args[2] = {
1379                                 values[2 * chan],
1380                                 values[2 * chan + 1]
1381                         };
1382                         LLVMValueRef packed;
1383
1384                         packed = lp_build_intrinsic(base->gallivm->builder,
1385                                                     "llvm.SI.packf16",
1386                                                     ctx->i32, pack_args, 2,
1387                                                     LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
1388                         args[chan + 5] =
1389                                 LLVMBuildBitCast(base->gallivm->builder,
1390                                                  packed, ctx->f32, "");
1391                 }
1392                 break;
1393
1394         case V_028714_SPI_SHADER_UNORM16_ABGR:
1395                 for (chan = 0; chan < 4; chan++) {
1396                         val[chan] = radeon_llvm_saturate(bld_base, values[chan]);
1397                         val[chan] = LLVMBuildFMul(builder, val[chan],
1398                                                   lp_build_const_float(gallivm, 65535), "");
1399                         val[chan] = LLVMBuildFAdd(builder, val[chan],
1400                                                   lp_build_const_float(gallivm, 0.5), "");
1401                         val[chan] = LLVMBuildFPToUI(builder, val[chan],
1402                                                     ctx->i32, "");
1403                 }
1404
1405                 args[4] = uint->one; /* COMPR flag */
1406                 args[5] = bitcast(bld_base, TGSI_TYPE_FLOAT,
1407                                   si_llvm_pack_two_int16(gallivm, val));
1408                 args[6] = bitcast(bld_base, TGSI_TYPE_FLOAT,
1409                                   si_llvm_pack_two_int16(gallivm, val+2));
1410                 break;
1411
1412         case V_028714_SPI_SHADER_SNORM16_ABGR:
1413                 for (chan = 0; chan < 4; chan++) {
1414                         /* Clamp between [-1, 1]. */
1415                         val[chan] = lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_MIN,
1416                                                               values[chan],
1417                                                               lp_build_const_float(gallivm, 1));
1418                         val[chan] = lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_MAX,
1419                                                               val[chan],
1420                                                               lp_build_const_float(gallivm, -1));
1421                         /* Convert to a signed integer in [-32767, 32767]. */
1422                         val[chan] = LLVMBuildFMul(builder, val[chan],
1423                                                   lp_build_const_float(gallivm, 32767), "");
1424                         /* If positive, add 0.5, else add -0.5. */
1425                         val[chan] = LLVMBuildFAdd(builder, val[chan],
1426                                         LLVMBuildSelect(builder,
1427                                                 LLVMBuildFCmp(builder, LLVMRealOGE,
1428                                                               val[chan], base->zero, ""),
1429                                                 lp_build_const_float(gallivm, 0.5),
1430                                                 lp_build_const_float(gallivm, -0.5), ""), "");
1431                         val[chan] = LLVMBuildFPToSI(builder, val[chan], ctx->i32, "");
1432                 }
1433
1434                 args[4] = uint->one; /* COMPR flag */
1435                 args[5] = bitcast(bld_base, TGSI_TYPE_FLOAT,
1436                                   si_llvm_pack_two_int32_as_int16(gallivm, val));
1437                 args[6] = bitcast(bld_base, TGSI_TYPE_FLOAT,
1438                                   si_llvm_pack_two_int32_as_int16(gallivm, val+2));
1439                 break;
1440
1441         case V_028714_SPI_SHADER_UINT16_ABGR: {
1442                 LLVMValueRef max = lp_build_const_int32(gallivm, is_int8 ?
1443                                                         255 : 65535);
1444                 /* Clamp. */
1445                 for (chan = 0; chan < 4; chan++) {
1446                         val[chan] = bitcast(bld_base, TGSI_TYPE_UNSIGNED, values[chan]);
1447                         val[chan] = lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_UMIN,
1448                                                               val[chan], max);
1449                 }
1450
1451                 args[4] = uint->one; /* COMPR flag */
1452                 args[5] = bitcast(bld_base, TGSI_TYPE_FLOAT,
1453                                   si_llvm_pack_two_int16(gallivm, val));
1454                 args[6] = bitcast(bld_base, TGSI_TYPE_FLOAT,
1455                                   si_llvm_pack_two_int16(gallivm, val+2));
1456                 break;
1457         }
1458
1459         case V_028714_SPI_SHADER_SINT16_ABGR: {
1460                 LLVMValueRef max = lp_build_const_int32(gallivm, is_int8 ?
1461                                                         127 : 32767);
1462                 LLVMValueRef min = lp_build_const_int32(gallivm, is_int8 ?
1463                                                         -128 : -32768);
1464                 /* Clamp. */
1465                 for (chan = 0; chan < 4; chan++) {
1466                         val[chan] = bitcast(bld_base, TGSI_TYPE_UNSIGNED, values[chan]);
1467                         val[chan] = lp_build_emit_llvm_binary(bld_base,
1468                                                               TGSI_OPCODE_IMIN,
1469                                                               val[chan], max);
1470                         val[chan] = lp_build_emit_llvm_binary(bld_base,
1471                                                               TGSI_OPCODE_IMAX,
1472                                                               val[chan], min);
1473                 }
1474
1475                 args[4] = uint->one; /* COMPR flag */
1476                 args[5] = bitcast(bld_base, TGSI_TYPE_FLOAT,
1477                                   si_llvm_pack_two_int32_as_int16(gallivm, val));
1478                 args[6] = bitcast(bld_base, TGSI_TYPE_FLOAT,
1479                                   si_llvm_pack_two_int32_as_int16(gallivm, val+2));
1480                 break;
1481         }
1482
1483         case V_028714_SPI_SHADER_32_ABGR:
1484                 memcpy(&args[5], values, sizeof(values[0]) * 4);
1485                 break;
1486         }
1487 }
1488
1489 static void si_alpha_test(struct lp_build_tgsi_context *bld_base,
1490                           LLVMValueRef alpha)
1491 {
1492         struct si_shader_context *ctx = si_shader_context(bld_base);
1493         struct gallivm_state *gallivm = bld_base->base.gallivm;
1494
1495         if (ctx->shader->key.ps.epilog.alpha_func != PIPE_FUNC_NEVER) {
1496                 LLVMValueRef alpha_ref = LLVMGetParam(ctx->radeon_bld.main_fn,
1497                                 SI_PARAM_ALPHA_REF);
1498
1499                 LLVMValueRef alpha_pass =
1500                         lp_build_cmp(&bld_base->base,
1501                                      ctx->shader->key.ps.epilog.alpha_func,
1502                                      alpha, alpha_ref);
1503                 LLVMValueRef arg =
1504                         lp_build_select(&bld_base->base,
1505                                         alpha_pass,
1506                                         lp_build_const_float(gallivm, 1.0f),
1507                                         lp_build_const_float(gallivm, -1.0f));
1508
1509                 lp_build_intrinsic(gallivm->builder, "llvm.AMDGPU.kill",
1510                                    ctx->voidt, &arg, 1, 0);
1511         } else {
1512                 lp_build_intrinsic(gallivm->builder, "llvm.AMDGPU.kilp",
1513                                    ctx->voidt, NULL, 0, 0);
1514         }
1515 }
1516
1517 static LLVMValueRef si_scale_alpha_by_sample_mask(struct lp_build_tgsi_context *bld_base,
1518                                                   LLVMValueRef alpha,
1519                                                   unsigned samplemask_param)
1520 {
1521         struct si_shader_context *ctx = si_shader_context(bld_base);
1522         struct gallivm_state *gallivm = bld_base->base.gallivm;
1523         LLVMValueRef coverage;
1524
1525         /* alpha = alpha * popcount(coverage) / SI_NUM_SMOOTH_AA_SAMPLES */
1526         coverage = LLVMGetParam(ctx->radeon_bld.main_fn,
1527                                 samplemask_param);
1528         coverage = bitcast(bld_base, TGSI_TYPE_SIGNED, coverage);
1529
1530         coverage = lp_build_intrinsic(gallivm->builder, "llvm.ctpop.i32",
1531                                    ctx->i32,
1532                                    &coverage, 1, LLVMReadNoneAttribute);
1533
1534         coverage = LLVMBuildUIToFP(gallivm->builder, coverage,
1535                                    ctx->f32, "");
1536
1537         coverage = LLVMBuildFMul(gallivm->builder, coverage,
1538                                  lp_build_const_float(gallivm,
1539                                         1.0 / SI_NUM_SMOOTH_AA_SAMPLES), "");
1540
1541         return LLVMBuildFMul(gallivm->builder, alpha, coverage, "");
1542 }
1543
1544 static void si_llvm_emit_clipvertex(struct lp_build_tgsi_context *bld_base,
1545                                     LLVMValueRef (*pos)[9], LLVMValueRef *out_elts)
1546 {
1547         struct si_shader_context *ctx = si_shader_context(bld_base);
1548         struct lp_build_context *base = &bld_base->base;
1549         struct lp_build_context *uint = &ctx->radeon_bld.soa.bld_base.uint_bld;
1550         unsigned reg_index;
1551         unsigned chan;
1552         unsigned const_chan;
1553         LLVMValueRef base_elt;
1554         LLVMValueRef ptr = LLVMGetParam(ctx->radeon_bld.main_fn, SI_PARAM_CONST_BUFFERS);
1555         LLVMValueRef constbuf_index = lp_build_const_int32(base->gallivm, SI_DRIVER_STATE_CONST_BUF);
1556         LLVMValueRef const_resource = build_indexed_load_const(ctx, ptr, constbuf_index);
1557
1558         for (reg_index = 0; reg_index < 2; reg_index ++) {
1559                 LLVMValueRef *args = pos[2 + reg_index];
1560
1561                 args[5] =
1562                 args[6] =
1563                 args[7] =
1564                 args[8] = lp_build_const_float(base->gallivm, 0.0f);
1565
1566                 /* Compute dot products of position and user clip plane vectors */
1567                 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
1568                         for (const_chan = 0; const_chan < TGSI_NUM_CHANNELS; const_chan++) {
1569                                 args[1] = lp_build_const_int32(base->gallivm,
1570                                                                ((reg_index * 4 + chan) * 4 +
1571                                                                 const_chan) * 4);
1572                                 base_elt = buffer_load_const(base->gallivm->builder, const_resource,
1573                                                       args[1], ctx->f32);
1574                                 args[5 + chan] =
1575                                         lp_build_add(base, args[5 + chan],
1576                                                      lp_build_mul(base, base_elt,
1577                                                                   out_elts[const_chan]));
1578                         }
1579                 }
1580
1581                 args[0] = lp_build_const_int32(base->gallivm, 0xf);
1582                 args[1] = uint->zero;
1583                 args[2] = uint->zero;
1584                 args[3] = lp_build_const_int32(base->gallivm,
1585                                                V_008DFC_SQ_EXP_POS + 2 + reg_index);
1586                 args[4] = uint->zero;
1587         }
1588 }
1589
1590 static void si_dump_streamout(struct pipe_stream_output_info *so)
1591 {
1592         unsigned i;
1593
1594         if (so->num_outputs)
1595                 fprintf(stderr, "STREAMOUT\n");
1596
1597         for (i = 0; i < so->num_outputs; i++) {
1598                 unsigned mask = ((1 << so->output[i].num_components) - 1) <<
1599                                 so->output[i].start_component;
1600                 fprintf(stderr, "  %i: BUF%i[%i..%i] <- OUT[%i].%s%s%s%s\n",
1601                         i, so->output[i].output_buffer,
1602                         so->output[i].dst_offset, so->output[i].dst_offset + so->output[i].num_components - 1,
1603                         so->output[i].register_index,
1604                         mask & 1 ? "x" : "",
1605                         mask & 2 ? "y" : "",
1606                         mask & 4 ? "z" : "",
1607                         mask & 8 ? "w" : "");
1608         }
1609 }
1610
1611 /* TBUFFER_STORE_FORMAT_{X,XY,XYZ,XYZW} <- the suffix is selected by num_channels=1..4.
1612  * The type of vdata must be one of i32 (num_channels=1), v2i32 (num_channels=2),
1613  * or v4i32 (num_channels=3,4). */
1614 static void build_tbuffer_store(struct si_shader_context *ctx,
1615                                 LLVMValueRef rsrc,
1616                                 LLVMValueRef vdata,
1617                                 unsigned num_channels,
1618                                 LLVMValueRef vaddr,
1619                                 LLVMValueRef soffset,
1620                                 unsigned inst_offset,
1621                                 unsigned dfmt,
1622                                 unsigned nfmt,
1623                                 unsigned offen,
1624                                 unsigned idxen,
1625                                 unsigned glc,
1626                                 unsigned slc,
1627                                 unsigned tfe)
1628 {
1629         struct gallivm_state *gallivm = &ctx->radeon_bld.gallivm;
1630         LLVMValueRef args[] = {
1631                 rsrc,
1632                 vdata,
1633                 LLVMConstInt(ctx->i32, num_channels, 0),
1634                 vaddr,
1635                 soffset,
1636                 LLVMConstInt(ctx->i32, inst_offset, 0),
1637                 LLVMConstInt(ctx->i32, dfmt, 0),
1638                 LLVMConstInt(ctx->i32, nfmt, 0),
1639                 LLVMConstInt(ctx->i32, offen, 0),
1640                 LLVMConstInt(ctx->i32, idxen, 0),
1641                 LLVMConstInt(ctx->i32, glc, 0),
1642                 LLVMConstInt(ctx->i32, slc, 0),
1643                 LLVMConstInt(ctx->i32, tfe, 0)
1644         };
1645
1646         /* The instruction offset field has 12 bits */
1647         assert(offen || inst_offset < (1 << 12));
1648
1649         /* The intrinsic is overloaded, we need to add a type suffix for overloading to work. */
1650         unsigned func = CLAMP(num_channels, 1, 3) - 1;
1651         const char *types[] = {"i32", "v2i32", "v4i32"};
1652         char name[256];
1653         snprintf(name, sizeof(name), "llvm.SI.tbuffer.store.%s", types[func]);
1654
1655         lp_build_intrinsic(gallivm->builder, name, ctx->voidt,
1656                            args, Elements(args), 0);
1657 }
1658
1659 static void build_tbuffer_store_dwords(struct si_shader_context *ctx,
1660                                      LLVMValueRef rsrc,
1661                                      LLVMValueRef vdata,
1662                                      unsigned num_channels,
1663                                      LLVMValueRef vaddr,
1664                                      LLVMValueRef soffset,
1665                                      unsigned inst_offset)
1666 {
1667         static unsigned dfmt[] = {
1668                 V_008F0C_BUF_DATA_FORMAT_32,
1669                 V_008F0C_BUF_DATA_FORMAT_32_32,
1670                 V_008F0C_BUF_DATA_FORMAT_32_32_32,
1671                 V_008F0C_BUF_DATA_FORMAT_32_32_32_32
1672         };
1673         assert(num_channels >= 1 && num_channels <= 4);
1674
1675         build_tbuffer_store(ctx, rsrc, vdata, num_channels, vaddr, soffset,
1676                             inst_offset, dfmt[num_channels-1],
1677                             V_008F0C_BUF_NUM_FORMAT_UINT, 1, 0, 1, 1, 0);
1678 }
1679
1680 /* On SI, the vertex shader is responsible for writing streamout data
1681  * to buffers. */
1682 static void si_llvm_emit_streamout(struct si_shader_context *ctx,
1683                                    struct si_shader_output_values *outputs,
1684                                    unsigned noutput)
1685 {
1686         struct pipe_stream_output_info *so = &ctx->shader->selector->so;
1687         struct gallivm_state *gallivm = &ctx->radeon_bld.gallivm;
1688         LLVMBuilderRef builder = gallivm->builder;
1689         int i, j;
1690         struct lp_build_if_state if_ctx;
1691
1692         /* Get bits [22:16], i.e. (so_param >> 16) & 127; */
1693         LLVMValueRef so_vtx_count =
1694                 unpack_param(ctx, ctx->param_streamout_config, 16, 7);
1695
1696         LLVMValueRef tid = lp_build_intrinsic(builder, "llvm.SI.tid", ctx->i32,
1697                                            NULL, 0, LLVMReadNoneAttribute);
1698
1699         /* can_emit = tid < so_vtx_count; */
1700         LLVMValueRef can_emit =
1701                 LLVMBuildICmp(builder, LLVMIntULT, tid, so_vtx_count, "");
1702
1703         LLVMValueRef stream_id =
1704                 unpack_param(ctx, ctx->param_streamout_config, 24, 2);
1705
1706         /* Emit the streamout code conditionally. This actually avoids
1707          * out-of-bounds buffer access. The hw tells us via the SGPR
1708          * (so_vtx_count) which threads are allowed to emit streamout data. */
1709         lp_build_if(&if_ctx, gallivm, can_emit);
1710         {
1711                 /* The buffer offset is computed as follows:
1712                  *   ByteOffset = streamout_offset[buffer_id]*4 +
1713                  *                (streamout_write_index + thread_id)*stride[buffer_id] +
1714                  *                attrib_offset
1715                  */
1716
1717                 LLVMValueRef so_write_index =
1718                         LLVMGetParam(ctx->radeon_bld.main_fn,
1719                                      ctx->param_streamout_write_index);
1720
1721                 /* Compute (streamout_write_index + thread_id). */
1722                 so_write_index = LLVMBuildAdd(builder, so_write_index, tid, "");
1723
1724                 /* Compute the write offset for each enabled buffer. */
1725                 LLVMValueRef so_write_offset[4] = {};
1726                 for (i = 0; i < 4; i++) {
1727                         if (!so->stride[i])
1728                                 continue;
1729
1730                         LLVMValueRef so_offset = LLVMGetParam(ctx->radeon_bld.main_fn,
1731                                                               ctx->param_streamout_offset[i]);
1732                         so_offset = LLVMBuildMul(builder, so_offset, LLVMConstInt(ctx->i32, 4, 0), "");
1733
1734                         so_write_offset[i] = LLVMBuildMul(builder, so_write_index,
1735                                                           LLVMConstInt(ctx->i32, so->stride[i]*4, 0), "");
1736                         so_write_offset[i] = LLVMBuildAdd(builder, so_write_offset[i], so_offset, "");
1737                 }
1738
1739                 /* Write streamout data. */
1740                 for (i = 0; i < so->num_outputs; i++) {
1741                         unsigned buf_idx = so->output[i].output_buffer;
1742                         unsigned reg = so->output[i].register_index;
1743                         unsigned start = so->output[i].start_component;
1744                         unsigned num_comps = so->output[i].num_components;
1745                         unsigned stream = so->output[i].stream;
1746                         LLVMValueRef out[4];
1747                         struct lp_build_if_state if_ctx_stream;
1748
1749                         assert(num_comps && num_comps <= 4);
1750                         if (!num_comps || num_comps > 4)
1751                                 continue;
1752
1753                         if (reg >= noutput)
1754                                 continue;
1755
1756                         /* Load the output as int. */
1757                         for (j = 0; j < num_comps; j++) {
1758                                 out[j] = LLVMBuildBitCast(builder,
1759                                                           outputs[reg].values[start+j],
1760                                                 ctx->i32, "");
1761                         }
1762
1763                         /* Pack the output. */
1764                         LLVMValueRef vdata = NULL;
1765
1766                         switch (num_comps) {
1767                         case 1: /* as i32 */
1768                                 vdata = out[0];
1769                                 break;
1770                         case 2: /* as v2i32 */
1771                         case 3: /* as v4i32 (aligned to 4) */
1772                         case 4: /* as v4i32 */
1773                                 vdata = LLVMGetUndef(LLVMVectorType(ctx->i32, util_next_power_of_two(num_comps)));
1774                                 for (j = 0; j < num_comps; j++) {
1775                                         vdata = LLVMBuildInsertElement(builder, vdata, out[j],
1776                                                                        LLVMConstInt(ctx->i32, j, 0), "");
1777                                 }
1778                                 break;
1779                         }
1780
1781                         LLVMValueRef can_emit_stream =
1782                                 LLVMBuildICmp(builder, LLVMIntEQ,
1783                                               stream_id,
1784                                               lp_build_const_int32(gallivm, stream), "");
1785
1786                         lp_build_if(&if_ctx_stream, gallivm, can_emit_stream);
1787                         build_tbuffer_store_dwords(ctx, ctx->so_buffers[buf_idx],
1788                                                    vdata, num_comps,
1789                                                    so_write_offset[buf_idx],
1790                                                    LLVMConstInt(ctx->i32, 0, 0),
1791                                                    so->output[i].dst_offset*4);
1792                         lp_build_endif(&if_ctx_stream);
1793                 }
1794         }
1795         lp_build_endif(&if_ctx);
1796 }
1797
1798
1799 /* Generate export instructions for hardware VS shader stage */
1800 static void si_llvm_export_vs(struct lp_build_tgsi_context *bld_base,
1801                               struct si_shader_output_values *outputs,
1802                               unsigned noutput)
1803 {
1804         struct si_shader_context *ctx = si_shader_context(bld_base);
1805         struct si_shader *shader = ctx->shader;
1806         struct lp_build_context *base = &bld_base->base;
1807         struct lp_build_context *uint =
1808                                 &ctx->radeon_bld.soa.bld_base.uint_bld;
1809         LLVMValueRef args[9];
1810         LLVMValueRef pos_args[4][9] = { { 0 } };
1811         LLVMValueRef psize_value = NULL, edgeflag_value = NULL, layer_value = NULL, viewport_index_value = NULL;
1812         unsigned semantic_name, semantic_index;
1813         unsigned target;
1814         unsigned param_count = 0;
1815         unsigned pos_idx;
1816         int i;
1817
1818         if (outputs && ctx->shader->selector->so.num_outputs) {
1819                 si_llvm_emit_streamout(ctx, outputs, noutput);
1820         }
1821
1822         for (i = 0; i < noutput; i++) {
1823                 semantic_name = outputs[i].name;
1824                 semantic_index = outputs[i].sid;
1825
1826 handle_semantic:
1827                 /* Select the correct target */
1828                 switch(semantic_name) {
1829                 case TGSI_SEMANTIC_PSIZE:
1830                         psize_value = outputs[i].values[0];
1831                         continue;
1832                 case TGSI_SEMANTIC_EDGEFLAG:
1833                         edgeflag_value = outputs[i].values[0];
1834                         continue;
1835                 case TGSI_SEMANTIC_LAYER:
1836                         layer_value = outputs[i].values[0];
1837                         semantic_name = TGSI_SEMANTIC_GENERIC;
1838                         goto handle_semantic;
1839                 case TGSI_SEMANTIC_VIEWPORT_INDEX:
1840                         viewport_index_value = outputs[i].values[0];
1841                         semantic_name = TGSI_SEMANTIC_GENERIC;
1842                         goto handle_semantic;
1843                 case TGSI_SEMANTIC_POSITION:
1844                         target = V_008DFC_SQ_EXP_POS;
1845                         break;
1846                 case TGSI_SEMANTIC_COLOR:
1847                 case TGSI_SEMANTIC_BCOLOR:
1848                         target = V_008DFC_SQ_EXP_PARAM + param_count;
1849                         shader->vs_output_param_offset[i] = param_count;
1850                         param_count++;
1851                         break;
1852                 case TGSI_SEMANTIC_CLIPDIST:
1853                         target = V_008DFC_SQ_EXP_POS + 2 + semantic_index;
1854                         break;
1855                 case TGSI_SEMANTIC_CLIPVERTEX:
1856                         si_llvm_emit_clipvertex(bld_base, pos_args, outputs[i].values);
1857                         continue;
1858                 case TGSI_SEMANTIC_PRIMID:
1859                 case TGSI_SEMANTIC_FOG:
1860                 case TGSI_SEMANTIC_TEXCOORD:
1861                 case TGSI_SEMANTIC_GENERIC:
1862                         target = V_008DFC_SQ_EXP_PARAM + param_count;
1863                         shader->vs_output_param_offset[i] = param_count;
1864                         param_count++;
1865                         break;
1866                 default:
1867                         target = 0;
1868                         fprintf(stderr,
1869                                 "Warning: SI unhandled vs output type:%d\n",
1870                                 semantic_name);
1871                 }
1872
1873                 si_llvm_init_export_args(bld_base, outputs[i].values, target, args);
1874
1875                 if (target >= V_008DFC_SQ_EXP_POS &&
1876                     target <= (V_008DFC_SQ_EXP_POS + 3)) {
1877                         memcpy(pos_args[target - V_008DFC_SQ_EXP_POS],
1878                                args, sizeof(args));
1879                 } else {
1880                         lp_build_intrinsic(base->gallivm->builder,
1881                                            "llvm.SI.export", ctx->voidt,
1882                                            args, 9, 0);
1883                 }
1884
1885                 if (semantic_name == TGSI_SEMANTIC_CLIPDIST) {
1886                         semantic_name = TGSI_SEMANTIC_GENERIC;
1887                         goto handle_semantic;
1888                 }
1889         }
1890
1891         shader->nr_param_exports = param_count;
1892
1893         /* We need to add the position output manually if it's missing. */
1894         if (!pos_args[0][0]) {
1895                 pos_args[0][0] = lp_build_const_int32(base->gallivm, 0xf); /* writemask */
1896                 pos_args[0][1] = uint->zero; /* EXEC mask */
1897                 pos_args[0][2] = uint->zero; /* last export? */
1898                 pos_args[0][3] = lp_build_const_int32(base->gallivm, V_008DFC_SQ_EXP_POS);
1899                 pos_args[0][4] = uint->zero; /* COMPR flag */
1900                 pos_args[0][5] = base->zero; /* X */
1901                 pos_args[0][6] = base->zero; /* Y */
1902                 pos_args[0][7] = base->zero; /* Z */
1903                 pos_args[0][8] = base->one;  /* W */
1904         }
1905
1906         /* Write the misc vector (point size, edgeflag, layer, viewport). */
1907         if (shader->selector->info.writes_psize ||
1908             shader->selector->info.writes_edgeflag ||
1909             shader->selector->info.writes_viewport_index ||
1910             shader->selector->info.writes_layer) {
1911                 pos_args[1][0] = lp_build_const_int32(base->gallivm, /* writemask */
1912                                                       shader->selector->info.writes_psize |
1913                                                       (shader->selector->info.writes_edgeflag << 1) |
1914                                                       (shader->selector->info.writes_layer << 2) |
1915                                                       (shader->selector->info.writes_viewport_index << 3));
1916                 pos_args[1][1] = uint->zero; /* EXEC mask */
1917                 pos_args[1][2] = uint->zero; /* last export? */
1918                 pos_args[1][3] = lp_build_const_int32(base->gallivm, V_008DFC_SQ_EXP_POS + 1);
1919                 pos_args[1][4] = uint->zero; /* COMPR flag */
1920                 pos_args[1][5] = base->zero; /* X */
1921                 pos_args[1][6] = base->zero; /* Y */
1922                 pos_args[1][7] = base->zero; /* Z */
1923                 pos_args[1][8] = base->zero; /* W */
1924
1925                 if (shader->selector->info.writes_psize)
1926                         pos_args[1][5] = psize_value;
1927
1928                 if (shader->selector->info.writes_edgeflag) {
1929                         /* The output is a float, but the hw expects an integer
1930                          * with the first bit containing the edge flag. */
1931                         edgeflag_value = LLVMBuildFPToUI(base->gallivm->builder,
1932                                                          edgeflag_value,
1933                                                          ctx->i32, "");
1934                         edgeflag_value = lp_build_min(&bld_base->int_bld,
1935                                                       edgeflag_value,
1936                                                       bld_base->int_bld.one);
1937
1938                         /* The LLVM intrinsic expects a float. */
1939                         pos_args[1][6] = LLVMBuildBitCast(base->gallivm->builder,
1940                                                           edgeflag_value,
1941                                                           ctx->f32, "");
1942                 }
1943
1944                 if (shader->selector->info.writes_layer)
1945                         pos_args[1][7] = layer_value;
1946
1947                 if (shader->selector->info.writes_viewport_index)
1948                         pos_args[1][8] = viewport_index_value;
1949         }
1950
1951         for (i = 0; i < 4; i++)
1952                 if (pos_args[i][0])
1953                         shader->nr_pos_exports++;
1954
1955         pos_idx = 0;
1956         for (i = 0; i < 4; i++) {
1957                 if (!pos_args[i][0])
1958                         continue;
1959
1960                 /* Specify the target we are exporting */
1961                 pos_args[i][3] = lp_build_const_int32(base->gallivm, V_008DFC_SQ_EXP_POS + pos_idx++);
1962
1963                 if (pos_idx == shader->nr_pos_exports)
1964                         /* Specify that this is the last export */
1965                         pos_args[i][2] = uint->one;
1966
1967                 lp_build_intrinsic(base->gallivm->builder, "llvm.SI.export",
1968                                    ctx->voidt, pos_args[i], 9, 0);
1969         }
1970 }
1971
1972 static void si_write_tess_factors(struct lp_build_tgsi_context *bld_base,
1973                                   LLVMValueRef rel_patch_id,
1974                                   LLVMValueRef invocation_id,
1975                                   LLVMValueRef tcs_out_current_patch_data_offset)
1976 {
1977         struct si_shader_context *ctx = si_shader_context(bld_base);
1978         struct gallivm_state *gallivm = bld_base->base.gallivm;
1979         struct si_shader *shader = ctx->shader;
1980         unsigned tess_inner_index, tess_outer_index;
1981         LLVMValueRef lds_base, lds_inner, lds_outer, byteoffset, buffer;
1982         LLVMValueRef out[6], vec0, vec1, rw_buffers, tf_base;
1983         unsigned stride, outer_comps, inner_comps, i;
1984         struct lp_build_if_state if_ctx;
1985
1986         /* Do this only for invocation 0, because the tess levels are per-patch,
1987          * not per-vertex.
1988          *
1989          * This can't jump, because invocation 0 executes this. It should
1990          * at least mask out the loads and stores for other invocations.
1991          */
1992         lp_build_if(&if_ctx, gallivm,
1993                     LLVMBuildICmp(gallivm->builder, LLVMIntEQ,
1994                                   invocation_id, bld_base->uint_bld.zero, ""));
1995
1996         /* Determine the layout of one tess factor element in the buffer. */
1997         switch (shader->key.tcs.epilog.prim_mode) {
1998         case PIPE_PRIM_LINES:
1999                 stride = 2; /* 2 dwords, 1 vec2 store */
2000                 outer_comps = 2;
2001                 inner_comps = 0;
2002                 break;
2003         case PIPE_PRIM_TRIANGLES:
2004                 stride = 4; /* 4 dwords, 1 vec4 store */
2005                 outer_comps = 3;
2006                 inner_comps = 1;
2007                 break;
2008         case PIPE_PRIM_QUADS:
2009                 stride = 6; /* 6 dwords, 2 stores (vec4 + vec2) */
2010                 outer_comps = 4;
2011                 inner_comps = 2;
2012                 break;
2013         default:
2014                 assert(0);
2015                 return;
2016         }
2017
2018         /* Load tess_inner and tess_outer from LDS.
2019          * Any invocation can write them, so we can't get them from a temporary.
2020          */
2021         tess_inner_index = si_shader_io_get_unique_index(TGSI_SEMANTIC_TESSINNER, 0);
2022         tess_outer_index = si_shader_io_get_unique_index(TGSI_SEMANTIC_TESSOUTER, 0);
2023
2024         lds_base = tcs_out_current_patch_data_offset;
2025         lds_inner = LLVMBuildAdd(gallivm->builder, lds_base,
2026                                  lp_build_const_int32(gallivm,
2027                                                       tess_inner_index * 4), "");
2028         lds_outer = LLVMBuildAdd(gallivm->builder, lds_base,
2029                                  lp_build_const_int32(gallivm,
2030                                                       tess_outer_index * 4), "");
2031
2032         for (i = 0; i < outer_comps; i++)
2033                 out[i] = lds_load(bld_base, TGSI_TYPE_SIGNED, i, lds_outer);
2034         for (i = 0; i < inner_comps; i++)
2035                 out[outer_comps+i] = lds_load(bld_base, TGSI_TYPE_SIGNED, i, lds_inner);
2036
2037         /* Convert the outputs to vectors for stores. */
2038         vec0 = lp_build_gather_values(gallivm, out, MIN2(stride, 4));
2039         vec1 = NULL;
2040
2041         if (stride > 4)
2042                 vec1 = lp_build_gather_values(gallivm, out+4, stride - 4);
2043
2044         /* Get the buffer. */
2045         rw_buffers = LLVMGetParam(ctx->radeon_bld.main_fn,
2046                                   SI_PARAM_RW_BUFFERS);
2047         buffer = build_indexed_load_const(ctx, rw_buffers,
2048                         lp_build_const_int32(gallivm, SI_RING_TESS_FACTOR));
2049
2050         /* Get the offset. */
2051         tf_base = LLVMGetParam(ctx->radeon_bld.main_fn,
2052                                SI_PARAM_TESS_FACTOR_OFFSET);
2053         byteoffset = LLVMBuildMul(gallivm->builder, rel_patch_id,
2054                                   lp_build_const_int32(gallivm, 4 * stride), "");
2055
2056         /* Store the outputs. */
2057         build_tbuffer_store_dwords(ctx, buffer, vec0,
2058                                    MIN2(stride, 4), byteoffset, tf_base, 0);
2059         if (vec1)
2060                 build_tbuffer_store_dwords(ctx, buffer, vec1,
2061                                            stride - 4, byteoffset, tf_base, 16);
2062         lp_build_endif(&if_ctx);
2063 }
2064
2065 /* This only writes the tessellation factor levels. */
2066 static void si_llvm_emit_tcs_epilogue(struct lp_build_tgsi_context *bld_base)
2067 {
2068         struct si_shader_context *ctx = si_shader_context(bld_base);
2069         LLVMValueRef invocation_id;
2070
2071         invocation_id = unpack_param(ctx, SI_PARAM_REL_IDS, 8, 5);
2072
2073         si_write_tess_factors(bld_base,
2074                               get_rel_patch_id(ctx),
2075                               invocation_id,
2076                               get_tcs_out_current_patch_data_offset(ctx));
2077 }
2078
2079 static void si_llvm_emit_ls_epilogue(struct lp_build_tgsi_context *bld_base)
2080 {
2081         struct si_shader_context *ctx = si_shader_context(bld_base);
2082         struct si_shader *shader = ctx->shader;
2083         struct tgsi_shader_info *info = &shader->selector->info;
2084         struct gallivm_state *gallivm = bld_base->base.gallivm;
2085         unsigned i, chan;
2086         LLVMValueRef vertex_id = LLVMGetParam(ctx->radeon_bld.main_fn,
2087                                               ctx->param_rel_auto_id);
2088         LLVMValueRef vertex_dw_stride =
2089                 unpack_param(ctx, SI_PARAM_LS_OUT_LAYOUT, 13, 8);
2090         LLVMValueRef base_dw_addr = LLVMBuildMul(gallivm->builder, vertex_id,
2091                                                  vertex_dw_stride, "");
2092
2093         /* Write outputs to LDS. The next shader (TCS aka HS) will read
2094          * its inputs from it. */
2095         for (i = 0; i < info->num_outputs; i++) {
2096                 LLVMValueRef *out_ptr = ctx->radeon_bld.soa.outputs[i];
2097                 unsigned name = info->output_semantic_name[i];
2098                 unsigned index = info->output_semantic_index[i];
2099                 int param = si_shader_io_get_unique_index(name, index);
2100                 LLVMValueRef dw_addr = LLVMBuildAdd(gallivm->builder, base_dw_addr,
2101                                         lp_build_const_int32(gallivm, param * 4), "");
2102
2103                 for (chan = 0; chan < 4; chan++) {
2104                         lds_store(bld_base, chan, dw_addr,
2105                                   LLVMBuildLoad(gallivm->builder, out_ptr[chan], ""));
2106                 }
2107         }
2108 }
2109
2110 static void si_llvm_emit_es_epilogue(struct lp_build_tgsi_context *bld_base)
2111 {
2112         struct si_shader_context *ctx = si_shader_context(bld_base);
2113         struct gallivm_state *gallivm = bld_base->base.gallivm;
2114         struct si_shader *es = ctx->shader;
2115         struct tgsi_shader_info *info = &es->selector->info;
2116         LLVMValueRef soffset = LLVMGetParam(ctx->radeon_bld.main_fn,
2117                                             ctx->param_es2gs_offset);
2118         unsigned chan;
2119         int i;
2120
2121         for (i = 0; i < info->num_outputs; i++) {
2122                 LLVMValueRef *out_ptr =
2123                         ctx->radeon_bld.soa.outputs[i];
2124                 int param_index;
2125
2126                 if (info->output_semantic_name[i] == TGSI_SEMANTIC_VIEWPORT_INDEX ||
2127                     info->output_semantic_name[i] == TGSI_SEMANTIC_LAYER)
2128                         continue;
2129
2130                 param_index = si_shader_io_get_unique_index(info->output_semantic_name[i],
2131                                                             info->output_semantic_index[i]);
2132
2133                 for (chan = 0; chan < 4; chan++) {
2134                         LLVMValueRef out_val = LLVMBuildLoad(gallivm->builder, out_ptr[chan], "");
2135                         out_val = LLVMBuildBitCast(gallivm->builder, out_val, ctx->i32, "");
2136
2137                         build_tbuffer_store(ctx,
2138                                             ctx->esgs_ring,
2139                                             out_val, 1,
2140                                             LLVMGetUndef(ctx->i32), soffset,
2141                                             (4 * param_index + chan) * 4,
2142                                             V_008F0C_BUF_DATA_FORMAT_32,
2143                                             V_008F0C_BUF_NUM_FORMAT_UINT,
2144                                             0, 0, 1, 1, 0);
2145                 }
2146         }
2147 }
2148
2149 static void si_llvm_emit_gs_epilogue(struct lp_build_tgsi_context *bld_base)
2150 {
2151         struct si_shader_context *ctx = si_shader_context(bld_base);
2152         struct gallivm_state *gallivm = bld_base->base.gallivm;
2153         LLVMValueRef args[2];
2154
2155         args[0] = lp_build_const_int32(gallivm, SENDMSG_GS_OP_NOP | SENDMSG_GS_DONE);
2156         args[1] = LLVMGetParam(ctx->radeon_bld.main_fn, SI_PARAM_GS_WAVE_ID);
2157         lp_build_intrinsic(gallivm->builder, "llvm.SI.sendmsg",
2158                            ctx->voidt, args, 2, LLVMNoUnwindAttribute);
2159 }
2160
2161 static void si_llvm_emit_vs_epilogue(struct lp_build_tgsi_context *bld_base)
2162 {
2163         struct si_shader_context *ctx = si_shader_context(bld_base);
2164         struct gallivm_state *gallivm = bld_base->base.gallivm;
2165         struct tgsi_shader_info *info = &ctx->shader->selector->info;
2166         struct si_shader_output_values *outputs = NULL;
2167         int i,j;
2168
2169         assert(!ctx->is_gs_copy_shader);
2170
2171         outputs = MALLOC((info->num_outputs + 1) * sizeof(outputs[0]));
2172
2173         /* Vertex color clamping.
2174          *
2175          * This uses a state constant loaded in a user data SGPR and
2176          * an IF statement is added that clamps all colors if the constant
2177          * is true.
2178          */
2179         if (ctx->type == TGSI_PROCESSOR_VERTEX) {
2180                 struct lp_build_if_state if_ctx;
2181                 LLVMValueRef cond = NULL;
2182                 LLVMValueRef addr, val;
2183
2184                 for (i = 0; i < info->num_outputs; i++) {
2185                         if (info->output_semantic_name[i] != TGSI_SEMANTIC_COLOR &&
2186                             info->output_semantic_name[i] != TGSI_SEMANTIC_BCOLOR)
2187                                 continue;
2188
2189                         /* We've found a color. */
2190                         if (!cond) {
2191                                 /* The state is in the first bit of the user SGPR. */
2192                                 cond = LLVMGetParam(ctx->radeon_bld.main_fn,
2193                                                     SI_PARAM_VS_STATE_BITS);
2194                                 cond = LLVMBuildTrunc(gallivm->builder, cond,
2195                                                       ctx->i1, "");
2196                                 lp_build_if(&if_ctx, gallivm, cond);
2197                         }
2198
2199                         for (j = 0; j < 4; j++) {
2200                                 addr = ctx->radeon_bld.soa.outputs[i][j];
2201                                 val = LLVMBuildLoad(gallivm->builder, addr, "");
2202                                 val = radeon_llvm_saturate(bld_base, val);
2203                                 LLVMBuildStore(gallivm->builder, val, addr);
2204                         }
2205                 }
2206
2207                 if (cond)
2208                         lp_build_endif(&if_ctx);
2209         }
2210
2211         for (i = 0; i < info->num_outputs; i++) {
2212                 outputs[i].name = info->output_semantic_name[i];
2213                 outputs[i].sid = info->output_semantic_index[i];
2214
2215                 for (j = 0; j < 4; j++)
2216                         outputs[i].values[j] =
2217                                 LLVMBuildLoad(gallivm->builder,
2218                                               ctx->radeon_bld.soa.outputs[i][j],
2219                                               "");
2220         }
2221
2222         /* Export PrimitiveID when PS needs it. */
2223         if (si_vs_exports_prim_id(ctx->shader)) {
2224                 outputs[i].name = TGSI_SEMANTIC_PRIMID;
2225                 outputs[i].sid = 0;
2226                 outputs[i].values[0] = bitcast(bld_base, TGSI_TYPE_FLOAT,
2227                                                get_primitive_id(bld_base, 0));
2228                 outputs[i].values[1] = bld_base->base.undef;
2229                 outputs[i].values[2] = bld_base->base.undef;
2230                 outputs[i].values[3] = bld_base->base.undef;
2231                 i++;
2232         }
2233
2234         si_llvm_export_vs(bld_base, outputs, i);
2235         FREE(outputs);
2236 }
2237
2238 static void si_export_mrt_z(struct lp_build_tgsi_context *bld_base,
2239                            LLVMValueRef depth, LLVMValueRef stencil,
2240                            LLVMValueRef samplemask)
2241 {
2242         struct si_shader_context *ctx = si_shader_context(bld_base);
2243         struct lp_build_context *base = &bld_base->base;
2244         struct lp_build_context *uint = &bld_base->uint_bld;
2245         LLVMValueRef args[9];
2246         unsigned mask = 0;
2247
2248         assert(depth || stencil || samplemask);
2249
2250         args[1] = uint->one; /* whether the EXEC mask is valid */
2251         args[2] = uint->one; /* DONE bit */
2252
2253         /* Specify the target we are exporting */
2254         args[3] = lp_build_const_int32(base->gallivm, V_008DFC_SQ_EXP_MRTZ);
2255
2256         args[4] = uint->zero; /* COMP flag */
2257         args[5] = base->undef; /* R, depth */
2258         args[6] = base->undef; /* G, stencil test value[0:7], stencil op value[8:15] */
2259         args[7] = base->undef; /* B, sample mask */
2260         args[8] = base->undef; /* A, alpha to mask */
2261
2262         if (depth) {
2263                 args[5] = depth;
2264                 mask |= 0x1;
2265         }
2266
2267         if (stencil) {
2268                 args[6] = stencil;
2269                 mask |= 0x2;
2270         }
2271
2272         if (samplemask) {
2273                 args[7] = samplemask;
2274                 mask |= 0x4;
2275         }
2276
2277         /* SI (except OLAND) has a bug that it only looks
2278          * at the X writemask component. */
2279         if (ctx->screen->b.chip_class == SI &&
2280             ctx->screen->b.family != CHIP_OLAND)
2281                 mask |= 0x1;
2282
2283         /* Specify which components to enable */
2284         args[0] = lp_build_const_int32(base->gallivm, mask);
2285
2286         lp_build_intrinsic(base->gallivm->builder, "llvm.SI.export",
2287                            ctx->voidt, args, 9, 0);
2288 }
2289
2290 static void si_export_mrt_color(struct lp_build_tgsi_context *bld_base,
2291                                 LLVMValueRef *color, unsigned index,
2292                                 unsigned samplemask_param,
2293                                 bool is_last)
2294 {
2295         struct si_shader_context *ctx = si_shader_context(bld_base);
2296         struct lp_build_context *base = &bld_base->base;
2297         int i;
2298
2299         /* Clamp color */
2300         if (ctx->shader->key.ps.epilog.clamp_color)
2301                 for (i = 0; i < 4; i++)
2302                         color[i] = radeon_llvm_saturate(bld_base, color[i]);
2303
2304         /* Alpha to one */
2305         if (ctx->shader->key.ps.epilog.alpha_to_one)
2306                 color[3] = base->one;
2307
2308         /* Alpha test */
2309         if (index == 0 &&
2310             ctx->shader->key.ps.epilog.alpha_func != PIPE_FUNC_ALWAYS)
2311                 si_alpha_test(bld_base, color[3]);
2312
2313         /* Line & polygon smoothing */
2314         if (ctx->shader->key.ps.epilog.poly_line_smoothing)
2315                 color[3] = si_scale_alpha_by_sample_mask(bld_base, color[3],
2316                                                          samplemask_param);
2317
2318         /* If last_cbuf > 0, FS_COLOR0_WRITES_ALL_CBUFS is true. */
2319         if (ctx->shader->key.ps.epilog.last_cbuf > 0) {
2320                 LLVMValueRef args[8][9];
2321                 int c, last = -1;
2322
2323                 /* Get the export arguments, also find out what the last one is. */
2324                 for (c = 0; c <= ctx->shader->key.ps.epilog.last_cbuf; c++) {
2325                         si_llvm_init_export_args(bld_base, color,
2326                                                  V_008DFC_SQ_EXP_MRT + c, args[c]);
2327                         if (args[c][0] != bld_base->uint_bld.zero)
2328                                 last = c;
2329                 }
2330
2331                 /* Emit all exports. */
2332                 for (c = 0; c <= ctx->shader->key.ps.epilog.last_cbuf; c++) {
2333                         if (is_last && last == c) {
2334                                 args[c][1] = bld_base->uint_bld.one; /* whether the EXEC mask is valid */
2335                                 args[c][2] = bld_base->uint_bld.one; /* DONE bit */
2336                         } else if (args[c][0] == bld_base->uint_bld.zero)
2337                                 continue; /* unnecessary NULL export */
2338
2339                         lp_build_intrinsic(base->gallivm->builder, "llvm.SI.export",
2340                                            ctx->voidt, args[c], 9, 0);
2341                 }
2342         } else {
2343                 LLVMValueRef args[9];
2344
2345                 /* Export */
2346                 si_llvm_init_export_args(bld_base, color, V_008DFC_SQ_EXP_MRT + index,
2347                                          args);
2348                 if (is_last) {
2349                         args[1] = bld_base->uint_bld.one; /* whether the EXEC mask is valid */
2350                         args[2] = bld_base->uint_bld.one; /* DONE bit */
2351                 } else if (args[0] == bld_base->uint_bld.zero)
2352                         return; /* unnecessary NULL export */
2353
2354                 lp_build_intrinsic(base->gallivm->builder, "llvm.SI.export",
2355                                    ctx->voidt, args, 9, 0);
2356         }
2357 }
2358
2359 static void si_export_null(struct lp_build_tgsi_context *bld_base)
2360 {
2361         struct si_shader_context *ctx = si_shader_context(bld_base);
2362         struct lp_build_context *base = &bld_base->base;
2363         struct lp_build_context *uint = &bld_base->uint_bld;
2364         LLVMValueRef args[9];
2365
2366         args[0] = lp_build_const_int32(base->gallivm, 0x0); /* enabled channels */
2367         args[1] = uint->one; /* whether the EXEC mask is valid */
2368         args[2] = uint->one; /* DONE bit */
2369         args[3] = lp_build_const_int32(base->gallivm, V_008DFC_SQ_EXP_NULL);
2370         args[4] = uint->zero; /* COMPR flag (0 = 32-bit export) */
2371         args[5] = uint->undef; /* R */
2372         args[6] = uint->undef; /* G */
2373         args[7] = uint->undef; /* B */
2374         args[8] = uint->undef; /* A */
2375
2376         lp_build_intrinsic(base->gallivm->builder, "llvm.SI.export",
2377                            ctx->voidt, args, 9, 0);
2378 }
2379
2380 static void si_llvm_emit_fs_epilogue(struct lp_build_tgsi_context *bld_base)
2381 {
2382         struct si_shader_context *ctx = si_shader_context(bld_base);
2383         struct si_shader *shader = ctx->shader;
2384         struct lp_build_context *base = &bld_base->base;
2385         struct tgsi_shader_info *info = &shader->selector->info;
2386         LLVMBuilderRef builder = base->gallivm->builder;
2387         LLVMValueRef depth = NULL, stencil = NULL, samplemask = NULL;
2388         int last_color_export = -1;
2389         int i;
2390
2391         /* Determine the last export. If MRTZ is present, it's always last.
2392          * Otherwise, find the last color export.
2393          */
2394         if (!info->writes_z && !info->writes_stencil && !info->writes_samplemask) {
2395                 unsigned spi_format = shader->key.ps.epilog.spi_shader_col_format;
2396
2397                 /* Don't export NULL and return if alpha-test is enabled. */
2398                 if (shader->key.ps.epilog.alpha_func != PIPE_FUNC_ALWAYS &&
2399                     shader->key.ps.epilog.alpha_func != PIPE_FUNC_NEVER &&
2400                     (spi_format & 0xf) == 0)
2401                         spi_format |= V_028714_SPI_SHADER_32_AR;
2402
2403                 for (i = 0; i < info->num_outputs; i++) {
2404                         unsigned index = info->output_semantic_index[i];
2405
2406                         if (info->output_semantic_name[i] != TGSI_SEMANTIC_COLOR)
2407                                 continue;
2408
2409                         /* If last_cbuf > 0, FS_COLOR0_WRITES_ALL_CBUFS is true. */
2410                         if (shader->key.ps.epilog.last_cbuf > 0) {
2411                                 /* Just set this if any of the colorbuffers are enabled. */
2412                                 if (spi_format &
2413                                     ((1llu << (4 * (shader->key.ps.epilog.last_cbuf + 1))) - 1))
2414                                         last_color_export = i;
2415                                 continue;
2416                         }
2417
2418                         if ((spi_format >> (index * 4)) & 0xf)
2419                                 last_color_export = i;
2420                 }
2421
2422                 /* If there are no outputs, export NULL. */
2423                 if (last_color_export == -1) {
2424                         si_export_null(bld_base);
2425                         return;
2426                 }
2427         }
2428
2429         for (i = 0; i < info->num_outputs; i++) {
2430                 unsigned semantic_name = info->output_semantic_name[i];
2431                 unsigned semantic_index = info->output_semantic_index[i];
2432                 unsigned j;
2433                 LLVMValueRef color[4] = {};
2434
2435                 /* Select the correct target */
2436                 switch (semantic_name) {
2437                 case TGSI_SEMANTIC_POSITION:
2438                         depth = LLVMBuildLoad(builder,
2439                                               ctx->radeon_bld.soa.outputs[i][2], "");
2440                         break;
2441                 case TGSI_SEMANTIC_STENCIL:
2442                         stencil = LLVMBuildLoad(builder,
2443                                                 ctx->radeon_bld.soa.outputs[i][1], "");
2444                         break;
2445                 case TGSI_SEMANTIC_SAMPLEMASK:
2446                         samplemask = LLVMBuildLoad(builder,
2447                                                    ctx->radeon_bld.soa.outputs[i][0], "");
2448                         break;
2449                 case TGSI_SEMANTIC_COLOR:
2450                         for (j = 0; j < 4; j++)
2451                                 color[j] = LLVMBuildLoad(builder,
2452                                                          ctx->radeon_bld.soa.outputs[i][j], "");
2453
2454                         si_export_mrt_color(bld_base, color, semantic_index,
2455                                             SI_PARAM_SAMPLE_COVERAGE,
2456                                             last_color_export == i);
2457                         break;
2458                 default:
2459                         fprintf(stderr,
2460                                 "Warning: SI unhandled fs output type:%d\n",
2461                                 semantic_name);
2462                 }
2463         }
2464
2465         if (depth || stencil || samplemask)
2466                 si_export_mrt_z(bld_base, depth, stencil, samplemask);
2467 }
2468
2469 static void build_tex_intrinsic(const struct lp_build_tgsi_action *action,
2470                                 struct lp_build_tgsi_context *bld_base,
2471                                 struct lp_build_emit_data *emit_data);
2472
2473 static bool tgsi_is_array_sampler(unsigned target)
2474 {
2475         return target == TGSI_TEXTURE_1D_ARRAY ||
2476                target == TGSI_TEXTURE_SHADOW1D_ARRAY ||
2477                target == TGSI_TEXTURE_2D_ARRAY ||
2478                target == TGSI_TEXTURE_SHADOW2D_ARRAY ||
2479                target == TGSI_TEXTURE_CUBE_ARRAY ||
2480                target == TGSI_TEXTURE_SHADOWCUBE_ARRAY ||
2481                target == TGSI_TEXTURE_2D_ARRAY_MSAA;
2482 }
2483
2484 static void set_tex_fetch_args(struct si_shader_context *ctx,
2485                                struct lp_build_emit_data *emit_data,
2486                                unsigned opcode, unsigned target,
2487                                LLVMValueRef res_ptr, LLVMValueRef samp_ptr,
2488                                LLVMValueRef *param, unsigned count,
2489                                unsigned dmask)
2490 {
2491         struct gallivm_state *gallivm = &ctx->radeon_bld.gallivm;
2492         unsigned num_args;
2493         unsigned is_rect = target == TGSI_TEXTURE_RECT;
2494
2495         /* Pad to power of two vector */
2496         while (count < util_next_power_of_two(count))
2497                 param[count++] = LLVMGetUndef(ctx->i32);
2498
2499         /* Texture coordinates. */
2500         if (count > 1)
2501                 emit_data->args[0] = lp_build_gather_values(gallivm, param, count);
2502         else
2503                 emit_data->args[0] = param[0];
2504
2505         /* Resource. */
2506         emit_data->args[1] = res_ptr;
2507         num_args = 2;
2508
2509         if (opcode == TGSI_OPCODE_TXF || opcode == TGSI_OPCODE_TXQ)
2510                 emit_data->dst_type = ctx->v4i32;
2511         else {
2512                 emit_data->dst_type = ctx->v4f32;
2513
2514                 emit_data->args[num_args++] = samp_ptr;
2515         }
2516
2517         emit_data->args[num_args++] = lp_build_const_int32(gallivm, dmask);
2518         emit_data->args[num_args++] = lp_build_const_int32(gallivm, is_rect); /* unorm */
2519         emit_data->args[num_args++] = lp_build_const_int32(gallivm, 0); /* r128 */
2520         emit_data->args[num_args++] = lp_build_const_int32(gallivm,
2521                                         tgsi_is_array_sampler(target)); /* da */
2522         emit_data->args[num_args++] = lp_build_const_int32(gallivm, 0); /* glc */
2523         emit_data->args[num_args++] = lp_build_const_int32(gallivm, 0); /* slc */
2524         emit_data->args[num_args++] = lp_build_const_int32(gallivm, 0); /* tfe */
2525         emit_data->args[num_args++] = lp_build_const_int32(gallivm, 0); /* lwe */
2526
2527         emit_data->arg_count = num_args;
2528 }
2529
2530 static const struct lp_build_tgsi_action tex_action;
2531
2532 enum desc_type {
2533         DESC_IMAGE,
2534         DESC_FMASK,
2535         DESC_SAMPLER
2536 };
2537
2538 static LLVMTypeRef const_array(LLVMTypeRef elem_type, int num_elements)
2539 {
2540         return LLVMPointerType(LLVMArrayType(elem_type, num_elements),
2541                                CONST_ADDR_SPACE);
2542 }
2543
2544 /**
2545  * Load an image view, fmask view. or sampler state descriptor.
2546  */
2547 static LLVMValueRef get_sampler_desc(struct si_shader_context *ctx,
2548                                      LLVMValueRef index, enum desc_type type)
2549 {
2550         struct gallivm_state *gallivm = &ctx->radeon_bld.gallivm;
2551         LLVMBuilderRef builder = gallivm->builder;
2552         LLVMValueRef ptr = LLVMGetParam(ctx->radeon_bld.main_fn,
2553                                         SI_PARAM_SAMPLERS);
2554
2555         switch (type) {
2556         case DESC_IMAGE:
2557                 /* The image is at [0:7]. */
2558                 index = LLVMBuildMul(builder, index, LLVMConstInt(ctx->i32, 2, 0), "");
2559                 break;
2560         case DESC_FMASK:
2561                 /* The FMASK is at [8:15]. */
2562                 index = LLVMBuildMul(builder, index, LLVMConstInt(ctx->i32, 2, 0), "");
2563                 index = LLVMBuildAdd(builder, index, LLVMConstInt(ctx->i32, 1, 0), "");
2564                 break;
2565         case DESC_SAMPLER:
2566                 /* The sampler state is at [12:15]. */
2567                 index = LLVMBuildMul(builder, index, LLVMConstInt(ctx->i32, 4, 0), "");
2568                 index = LLVMBuildAdd(builder, index, LLVMConstInt(ctx->i32, 3, 0), "");
2569                 ptr = LLVMBuildPointerCast(builder, ptr,
2570                                            const_array(ctx->v4i32, 0), "");
2571                 break;
2572         }
2573
2574         return build_indexed_load_const(ctx, ptr, index);
2575 }
2576
2577 static void tex_fetch_ptrs(
2578         struct lp_build_tgsi_context *bld_base,
2579         struct lp_build_emit_data *emit_data,
2580         LLVMValueRef *res_ptr, LLVMValueRef *samp_ptr, LLVMValueRef *fmask_ptr)
2581 {
2582         struct si_shader_context *ctx = si_shader_context(bld_base);
2583         const struct tgsi_full_instruction *inst = emit_data->inst;
2584         unsigned target = inst->Texture.Texture;
2585         unsigned sampler_src;
2586         unsigned sampler_index;
2587
2588         sampler_src = emit_data->inst->Instruction.NumSrcRegs - 1;
2589         sampler_index = emit_data->inst->Src[sampler_src].Register.Index;
2590
2591         if (emit_data->inst->Src[sampler_src].Register.Indirect) {
2592                 const struct tgsi_full_src_register *reg = &emit_data->inst->Src[sampler_src];
2593                 LLVMValueRef ind_index;
2594
2595                 ind_index = get_indirect_index(ctx, &reg->Indirect, reg->Register.Index);
2596
2597                 *res_ptr = get_sampler_desc(ctx, ind_index, DESC_IMAGE);
2598
2599                 if (target == TGSI_TEXTURE_2D_MSAA ||
2600                     target == TGSI_TEXTURE_2D_ARRAY_MSAA) {
2601                         *samp_ptr = NULL;
2602                         *fmask_ptr = get_sampler_desc(ctx, ind_index, DESC_FMASK);
2603                 } else {
2604                         *samp_ptr = get_sampler_desc(ctx, ind_index, DESC_SAMPLER);
2605                         *fmask_ptr = NULL;
2606                 }
2607         } else {
2608                 *res_ptr = ctx->sampler_views[sampler_index];
2609                 *samp_ptr = ctx->sampler_states[sampler_index];
2610                 *fmask_ptr = ctx->fmasks[sampler_index];
2611         }
2612 }
2613
2614 static void tex_fetch_args(
2615         struct lp_build_tgsi_context *bld_base,
2616         struct lp_build_emit_data *emit_data)
2617 {
2618         struct si_shader_context *ctx = si_shader_context(bld_base);
2619         struct gallivm_state *gallivm = bld_base->base.gallivm;
2620         LLVMBuilderRef builder = gallivm->builder;
2621         const struct tgsi_full_instruction *inst = emit_data->inst;
2622         unsigned opcode = inst->Instruction.Opcode;
2623         unsigned target = inst->Texture.Texture;
2624         LLVMValueRef coords[5], derivs[6];
2625         LLVMValueRef address[16];
2626         int ref_pos;
2627         unsigned num_coords = tgsi_util_get_texture_coord_dim(target, &ref_pos);
2628         unsigned count = 0;
2629         unsigned chan;
2630         unsigned num_deriv_channels = 0;
2631         bool has_offset = inst->Texture.NumOffsets > 0;
2632         LLVMValueRef res_ptr, samp_ptr, fmask_ptr = NULL;
2633         unsigned dmask = 0xf;
2634
2635         tex_fetch_ptrs(bld_base, emit_data, &res_ptr, &samp_ptr, &fmask_ptr);
2636
2637         if (opcode == TGSI_OPCODE_TXQ) {
2638                 if (target == TGSI_TEXTURE_BUFFER) {
2639                         /* Read the size from the buffer descriptor directly. */
2640                         LLVMValueRef res = LLVMBuildBitCast(builder, res_ptr, ctx->v8i32, "");
2641                         LLVMValueRef size = LLVMBuildExtractElement(builder, res,
2642                                                         lp_build_const_int32(gallivm, 6), "");
2643
2644                         if (ctx->screen->b.chip_class >= VI) {
2645                                 /* On VI, the descriptor contains the size in bytes,
2646                                  * but TXQ must return the size in elements.
2647                                  * The stride is always non-zero for resources using TXQ.
2648                                  */
2649                                 LLVMValueRef stride =
2650                                         LLVMBuildExtractElement(builder, res,
2651                                                                 lp_build_const_int32(gallivm, 5), "");
2652                                 stride = LLVMBuildLShr(builder, stride,
2653                                                        lp_build_const_int32(gallivm, 16), "");
2654                                 stride = LLVMBuildAnd(builder, stride,
2655                                                       lp_build_const_int32(gallivm, 0x3FFF), "");
2656
2657                                 size = LLVMBuildUDiv(builder, size, stride, "");
2658                         }
2659
2660                         emit_data->args[0] = size;
2661                         return;
2662                 }
2663
2664                 /* Textures - set the mip level. */
2665                 address[count++] = lp_build_emit_fetch(bld_base, inst, 0, TGSI_CHAN_X);
2666
2667                 set_tex_fetch_args(ctx, emit_data, opcode, target, res_ptr,
2668                                    NULL, address, count, 0xf);
2669                 return;
2670         }
2671
2672         if (target == TGSI_TEXTURE_BUFFER) {
2673                 LLVMTypeRef v2i128 = LLVMVectorType(ctx->i128, 2);
2674
2675                 /* Bitcast and truncate v8i32 to v16i8. */
2676                 LLVMValueRef res = res_ptr;
2677                 res = LLVMBuildBitCast(gallivm->builder, res, v2i128, "");
2678                 res = LLVMBuildExtractElement(gallivm->builder, res, bld_base->uint_bld.one, "");
2679                 res = LLVMBuildBitCast(gallivm->builder, res, ctx->v16i8, "");
2680
2681                 emit_data->dst_type = ctx->v4f32;
2682                 emit_data->args[0] = res;
2683                 emit_data->args[1] = bld_base->uint_bld.zero;
2684                 emit_data->args[2] = lp_build_emit_fetch(bld_base, emit_data->inst, 0, TGSI_CHAN_X);
2685                 emit_data->arg_count = 3;
2686                 return;
2687         }
2688
2689         /* Fetch and project texture coordinates */
2690         coords[3] = lp_build_emit_fetch(bld_base, emit_data->inst, 0, TGSI_CHAN_W);
2691         for (chan = 0; chan < 3; chan++ ) {
2692                 coords[chan] = lp_build_emit_fetch(bld_base,
2693                                                    emit_data->inst, 0,
2694                                                    chan);
2695                 if (opcode == TGSI_OPCODE_TXP)
2696                         coords[chan] = lp_build_emit_llvm_binary(bld_base,
2697                                                                  TGSI_OPCODE_DIV,
2698                                                                  coords[chan],
2699                                                                  coords[3]);
2700         }
2701
2702         if (opcode == TGSI_OPCODE_TXP)
2703                 coords[3] = bld_base->base.one;
2704
2705         /* Pack offsets. */
2706         if (has_offset && opcode != TGSI_OPCODE_TXF) {
2707                 /* The offsets are six-bit signed integers packed like this:
2708                  *   X=[5:0], Y=[13:8], and Z=[21:16].
2709                  */
2710                 LLVMValueRef offset[3], pack;
2711
2712                 assert(inst->Texture.NumOffsets == 1);
2713
2714                 for (chan = 0; chan < 3; chan++) {
2715                         offset[chan] = lp_build_emit_fetch_texoffset(bld_base,
2716                                                                      emit_data->inst, 0, chan);
2717                         offset[chan] = LLVMBuildAnd(gallivm->builder, offset[chan],
2718                                                     lp_build_const_int32(gallivm, 0x3f), "");
2719                         if (chan)
2720                                 offset[chan] = LLVMBuildShl(gallivm->builder, offset[chan],
2721                                                             lp_build_const_int32(gallivm, chan*8), "");
2722                 }
2723
2724                 pack = LLVMBuildOr(gallivm->builder, offset[0], offset[1], "");
2725                 pack = LLVMBuildOr(gallivm->builder, pack, offset[2], "");
2726                 address[count++] = pack;
2727         }
2728
2729         /* Pack LOD bias value */
2730         if (opcode == TGSI_OPCODE_TXB)
2731                 address[count++] = coords[3];
2732         if (opcode == TGSI_OPCODE_TXB2)
2733                 address[count++] = lp_build_emit_fetch(bld_base, inst, 1, TGSI_CHAN_X);
2734
2735         /* Pack depth comparison value */
2736         if (tgsi_is_shadow_target(target) && opcode != TGSI_OPCODE_LODQ) {
2737                 if (target == TGSI_TEXTURE_SHADOWCUBE_ARRAY) {
2738                         address[count++] = lp_build_emit_fetch(bld_base, inst, 1, TGSI_CHAN_X);
2739                 } else {
2740                         assert(ref_pos >= 0);
2741                         address[count++] = coords[ref_pos];
2742                 }
2743         }
2744
2745         /* Pack user derivatives */
2746         if (opcode == TGSI_OPCODE_TXD) {
2747                 int param, num_src_deriv_channels;
2748
2749                 switch (target) {
2750                 case TGSI_TEXTURE_3D:
2751                         num_src_deriv_channels = 3;
2752                         num_deriv_channels = 3;
2753                         break;
2754                 case TGSI_TEXTURE_2D:
2755                 case TGSI_TEXTURE_SHADOW2D:
2756                 case TGSI_TEXTURE_RECT:
2757                 case TGSI_TEXTURE_SHADOWRECT:
2758                 case TGSI_TEXTURE_2D_ARRAY:
2759                 case TGSI_TEXTURE_SHADOW2D_ARRAY:
2760                         num_src_deriv_channels = 2;
2761                         num_deriv_channels = 2;
2762                         break;
2763                 case TGSI_TEXTURE_CUBE:
2764                 case TGSI_TEXTURE_SHADOWCUBE:
2765                 case TGSI_TEXTURE_CUBE_ARRAY:
2766                 case TGSI_TEXTURE_SHADOWCUBE_ARRAY:
2767                         /* Cube derivatives will be converted to 2D. */
2768                         num_src_deriv_channels = 3;
2769                         num_deriv_channels = 2;
2770                         break;
2771                 case TGSI_TEXTURE_1D:
2772                 case TGSI_TEXTURE_SHADOW1D:
2773                 case TGSI_TEXTURE_1D_ARRAY:
2774                 case TGSI_TEXTURE_SHADOW1D_ARRAY:
2775                         num_src_deriv_channels = 1;
2776                         num_deriv_channels = 1;
2777                         break;
2778                 default:
2779                         unreachable("invalid target");
2780                 }
2781
2782                 for (param = 0; param < 2; param++)
2783                         for (chan = 0; chan < num_src_deriv_channels; chan++)
2784                                 derivs[param * num_src_deriv_channels + chan] =
2785                                         lp_build_emit_fetch(bld_base, inst, param+1, chan);
2786         }
2787
2788         if (target == TGSI_TEXTURE_CUBE ||
2789             target == TGSI_TEXTURE_CUBE_ARRAY ||
2790             target == TGSI_TEXTURE_SHADOWCUBE ||
2791             target == TGSI_TEXTURE_SHADOWCUBE_ARRAY)
2792                 radeon_llvm_emit_prepare_cube_coords(bld_base, emit_data, coords, derivs);
2793
2794         if (opcode == TGSI_OPCODE_TXD)
2795                 for (int i = 0; i < num_deriv_channels * 2; i++)
2796                         address[count++] = derivs[i];
2797
2798         /* Pack texture coordinates */
2799         address[count++] = coords[0];
2800         if (num_coords > 1)
2801                 address[count++] = coords[1];
2802         if (num_coords > 2)
2803                 address[count++] = coords[2];
2804
2805         /* Pack LOD or sample index */
2806         if (opcode == TGSI_OPCODE_TXL || opcode == TGSI_OPCODE_TXF)
2807                 address[count++] = coords[3];
2808         else if (opcode == TGSI_OPCODE_TXL2)
2809                 address[count++] = lp_build_emit_fetch(bld_base, inst, 1, TGSI_CHAN_X);
2810
2811         if (count > 16) {
2812                 assert(!"Cannot handle more than 16 texture address parameters");
2813                 count = 16;
2814         }
2815
2816         for (chan = 0; chan < count; chan++ ) {
2817                 address[chan] = LLVMBuildBitCast(gallivm->builder,
2818                                                  address[chan], ctx->i32, "");
2819         }
2820
2821         /* Adjust the sample index according to FMASK.
2822          *
2823          * For uncompressed MSAA surfaces, FMASK should return 0x76543210,
2824          * which is the identity mapping. Each nibble says which physical sample
2825          * should be fetched to get that sample.
2826          *
2827          * For example, 0x11111100 means there are only 2 samples stored and
2828          * the second sample covers 3/4 of the pixel. When reading samples 0
2829          * and 1, return physical sample 0 (determined by the first two 0s
2830          * in FMASK), otherwise return physical sample 1.
2831          *
2832          * The sample index should be adjusted as follows:
2833          *   sample_index = (fmask >> (sample_index * 4)) & 0xF;
2834          */
2835         if (target == TGSI_TEXTURE_2D_MSAA ||
2836             target == TGSI_TEXTURE_2D_ARRAY_MSAA) {
2837                 struct lp_build_context *uint_bld = &bld_base->uint_bld;
2838                 struct lp_build_emit_data txf_emit_data = *emit_data;
2839                 LLVMValueRef txf_address[4];
2840                 unsigned txf_count = count;
2841                 struct tgsi_full_instruction inst = {};
2842
2843                 memcpy(txf_address, address, sizeof(txf_address));
2844
2845                 if (target == TGSI_TEXTURE_2D_MSAA) {
2846                         txf_address[2] = bld_base->uint_bld.zero;
2847                 }
2848                 txf_address[3] = bld_base->uint_bld.zero;
2849
2850                 /* Read FMASK using TXF. */
2851                 inst.Instruction.Opcode = TGSI_OPCODE_TXF;
2852                 inst.Texture.Texture = target;
2853                 txf_emit_data.inst = &inst;
2854                 txf_emit_data.chan = 0;
2855                 set_tex_fetch_args(ctx, &txf_emit_data, TGSI_OPCODE_TXF,
2856                                    target, fmask_ptr, NULL,
2857                                    txf_address, txf_count, 0xf);
2858                 build_tex_intrinsic(&tex_action, bld_base, &txf_emit_data);
2859
2860                 /* Initialize some constants. */
2861                 LLVMValueRef four = LLVMConstInt(ctx->i32, 4, 0);
2862                 LLVMValueRef F = LLVMConstInt(ctx->i32, 0xF, 0);
2863
2864                 /* Apply the formula. */
2865                 LLVMValueRef fmask =
2866                         LLVMBuildExtractElement(gallivm->builder,
2867                                                 txf_emit_data.output[0],
2868                                                 uint_bld->zero, "");
2869
2870                 unsigned sample_chan = target == TGSI_TEXTURE_2D_MSAA ? 2 : 3;
2871
2872                 LLVMValueRef sample_index4 =
2873                         LLVMBuildMul(gallivm->builder, address[sample_chan], four, "");
2874
2875                 LLVMValueRef shifted_fmask =
2876                         LLVMBuildLShr(gallivm->builder, fmask, sample_index4, "");
2877
2878                 LLVMValueRef final_sample =
2879                         LLVMBuildAnd(gallivm->builder, shifted_fmask, F, "");
2880
2881                 /* Don't rewrite the sample index if WORD1.DATA_FORMAT of the FMASK
2882                  * resource descriptor is 0 (invalid),
2883                  */
2884                 LLVMValueRef fmask_desc =
2885                         LLVMBuildBitCast(gallivm->builder, fmask_ptr,
2886                                          ctx->v8i32, "");
2887
2888                 LLVMValueRef fmask_word1 =
2889                         LLVMBuildExtractElement(gallivm->builder, fmask_desc,
2890                                                 uint_bld->one, "");
2891
2892                 LLVMValueRef word1_is_nonzero =
2893                         LLVMBuildICmp(gallivm->builder, LLVMIntNE,
2894                                       fmask_word1, uint_bld->zero, "");
2895
2896                 /* Replace the MSAA sample index. */
2897                 address[sample_chan] =
2898                         LLVMBuildSelect(gallivm->builder, word1_is_nonzero,
2899                                         final_sample, address[sample_chan], "");
2900         }
2901
2902         if (opcode == TGSI_OPCODE_TXF) {
2903                 /* add tex offsets */
2904                 if (inst->Texture.NumOffsets) {
2905                         struct lp_build_context *uint_bld = &bld_base->uint_bld;
2906                         struct lp_build_tgsi_soa_context *bld = lp_soa_context(bld_base);
2907                         const struct tgsi_texture_offset *off = inst->TexOffsets;
2908
2909                         assert(inst->Texture.NumOffsets == 1);
2910
2911                         switch (target) {
2912                         case TGSI_TEXTURE_3D:
2913                                 address[2] = lp_build_add(uint_bld, address[2],
2914                                                 bld->immediates[off->Index][off->SwizzleZ]);
2915                                 /* fall through */
2916                         case TGSI_TEXTURE_2D:
2917                         case TGSI_TEXTURE_SHADOW2D:
2918                         case TGSI_TEXTURE_RECT:
2919                         case TGSI_TEXTURE_SHADOWRECT:
2920                         case TGSI_TEXTURE_2D_ARRAY:
2921                         case TGSI_TEXTURE_SHADOW2D_ARRAY:
2922                                 address[1] =
2923                                         lp_build_add(uint_bld, address[1],
2924                                                 bld->immediates[off->Index][off->SwizzleY]);
2925                                 /* fall through */
2926                         case TGSI_TEXTURE_1D:
2927                         case TGSI_TEXTURE_SHADOW1D:
2928                         case TGSI_TEXTURE_1D_ARRAY:
2929                         case TGSI_TEXTURE_SHADOW1D_ARRAY:
2930                                 address[0] =
2931                                         lp_build_add(uint_bld, address[0],
2932                                                 bld->immediates[off->Index][off->SwizzleX]);
2933                                 break;
2934                                 /* texture offsets do not apply to other texture targets */
2935                         }
2936                 }
2937         }
2938
2939         if (opcode == TGSI_OPCODE_TG4) {
2940                 unsigned gather_comp = 0;
2941
2942                 /* DMASK was repurposed for GATHER4. 4 components are always
2943                  * returned and DMASK works like a swizzle - it selects
2944                  * the component to fetch. The only valid DMASK values are
2945                  * 1=red, 2=green, 4=blue, 8=alpha. (e.g. 1 returns
2946                  * (red,red,red,red) etc.) The ISA document doesn't mention
2947                  * this.
2948                  */
2949
2950                 /* Get the component index from src1.x for Gather4. */
2951                 if (!tgsi_is_shadow_target(target)) {
2952                         LLVMValueRef (*imms)[4] = lp_soa_context(bld_base)->immediates;
2953                         LLVMValueRef comp_imm;
2954                         struct tgsi_src_register src1 = inst->Src[1].Register;
2955
2956                         assert(src1.File == TGSI_FILE_IMMEDIATE);
2957
2958                         comp_imm = imms[src1.Index][src1.SwizzleX];
2959                         gather_comp = LLVMConstIntGetZExtValue(comp_imm);
2960                         gather_comp = CLAMP(gather_comp, 0, 3);
2961                 }
2962
2963                 dmask = 1 << gather_comp;
2964         }
2965
2966         set_tex_fetch_args(ctx, emit_data, opcode, target, res_ptr,
2967                            samp_ptr, address, count, dmask);
2968 }
2969
2970 static void build_tex_intrinsic(const struct lp_build_tgsi_action *action,
2971                                 struct lp_build_tgsi_context *bld_base,
2972                                 struct lp_build_emit_data *emit_data)
2973 {
2974         struct lp_build_context *base = &bld_base->base;
2975         unsigned opcode = emit_data->inst->Instruction.Opcode;
2976         unsigned target = emit_data->inst->Texture.Texture;
2977         char intr_name[127];
2978         bool has_offset = emit_data->inst->Texture.NumOffsets > 0;
2979         bool is_shadow = tgsi_is_shadow_target(target);
2980         char type[64];
2981         const char *name = "llvm.SI.image.sample";
2982         const char *infix = "";
2983
2984         if (opcode == TGSI_OPCODE_TXQ && target == TGSI_TEXTURE_BUFFER) {
2985                 /* Just return the buffer size. */
2986                 emit_data->output[emit_data->chan] = emit_data->args[0];
2987                 return;
2988         }
2989
2990         if (target == TGSI_TEXTURE_BUFFER) {
2991                 emit_data->output[emit_data->chan] = lp_build_intrinsic(
2992                         base->gallivm->builder,
2993                         "llvm.SI.vs.load.input", emit_data->dst_type,
2994                         emit_data->args, emit_data->arg_count,
2995                         LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
2996                 return;
2997         }
2998
2999         switch (opcode) {
3000         case TGSI_OPCODE_TXF:
3001                 name = target == TGSI_TEXTURE_2D_MSAA ||
3002                        target == TGSI_TEXTURE_2D_ARRAY_MSAA ?
3003                                "llvm.SI.image.load" :
3004                                "llvm.SI.image.load.mip";
3005                 is_shadow = false;
3006                 has_offset = false;
3007                 break;
3008         case TGSI_OPCODE_TXQ:
3009                 name = "llvm.SI.getresinfo";
3010                 is_shadow = false;
3011                 has_offset = false;
3012                 break;
3013         case TGSI_OPCODE_LODQ:
3014                 name = "llvm.SI.getlod";
3015                 is_shadow = false;
3016                 has_offset = false;
3017                 break;
3018         case TGSI_OPCODE_TEX:
3019         case TGSI_OPCODE_TEX2:
3020         case TGSI_OPCODE_TXP:
3021                 break;
3022         case TGSI_OPCODE_TXB:
3023         case TGSI_OPCODE_TXB2:
3024                 infix = ".b";
3025                 break;
3026         case TGSI_OPCODE_TXL:
3027         case TGSI_OPCODE_TXL2:
3028                 infix = ".l";
3029                 break;
3030         case TGSI_OPCODE_TXD:
3031                 infix = ".d";
3032                 break;
3033         case TGSI_OPCODE_TG4:
3034                 name = "llvm.SI.gather4";
3035                 break;
3036         default:
3037                 assert(0);
3038                 return;
3039         }
3040
3041         if (LLVMGetTypeKind(LLVMTypeOf(emit_data->args[0])) == LLVMVectorTypeKind)
3042                 sprintf(type, ".v%ui32",
3043                         LLVMGetVectorSize(LLVMTypeOf(emit_data->args[0])));
3044         else
3045                 strcpy(type, ".i32");
3046
3047         /* Add the type and suffixes .c, .o if needed. */
3048         sprintf(intr_name, "%s%s%s%s%s",
3049                 name, is_shadow ? ".c" : "", infix,
3050                 has_offset ? ".o" : "", type);
3051
3052         emit_data->output[emit_data->chan] = lp_build_intrinsic(
3053                 base->gallivm->builder, intr_name, emit_data->dst_type,
3054                 emit_data->args, emit_data->arg_count,
3055                 LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
3056
3057         /* Divide the number of layers by 6 to get the number of cubes. */
3058         if (opcode == TGSI_OPCODE_TXQ &&
3059             (target == TGSI_TEXTURE_CUBE_ARRAY ||
3060              target == TGSI_TEXTURE_SHADOWCUBE_ARRAY)) {
3061                 LLVMBuilderRef builder = bld_base->base.gallivm->builder;
3062                 LLVMValueRef two = lp_build_const_int32(bld_base->base.gallivm, 2);
3063                 LLVMValueRef six = lp_build_const_int32(bld_base->base.gallivm, 6);
3064
3065                 LLVMValueRef v4 = emit_data->output[emit_data->chan];
3066                 LLVMValueRef z = LLVMBuildExtractElement(builder, v4, two, "");
3067                 z = LLVMBuildSDiv(builder, z, six, "");
3068
3069                 emit_data->output[emit_data->chan] =
3070                         LLVMBuildInsertElement(builder, v4, z, two, "");
3071         }
3072 }
3073
3074 static void si_llvm_emit_txqs(
3075         const struct lp_build_tgsi_action *action,
3076         struct lp_build_tgsi_context *bld_base,
3077         struct lp_build_emit_data *emit_data)
3078 {
3079         struct si_shader_context *ctx = si_shader_context(bld_base);
3080         struct gallivm_state *gallivm = bld_base->base.gallivm;
3081         LLVMBuilderRef builder = gallivm->builder;
3082         LLVMValueRef res, samples;
3083         LLVMValueRef res_ptr, samp_ptr, fmask_ptr = NULL;
3084
3085         tex_fetch_ptrs(bld_base, emit_data, &res_ptr, &samp_ptr, &fmask_ptr);
3086
3087
3088         /* Read the samples from the descriptor directly. */
3089         res = LLVMBuildBitCast(builder, res_ptr, ctx->v8i32, "");
3090         samples = LLVMBuildExtractElement(
3091                 builder, res,
3092                 lp_build_const_int32(gallivm, 3), "");
3093         samples = LLVMBuildLShr(builder, samples,
3094                                 lp_build_const_int32(gallivm, 16), "");
3095         samples = LLVMBuildAnd(builder, samples,
3096                                lp_build_const_int32(gallivm, 0xf), "");
3097         samples = LLVMBuildShl(builder, lp_build_const_int32(gallivm, 1),
3098                                samples, "");
3099
3100         emit_data->output[emit_data->chan] = samples;
3101 }
3102
3103 /*
3104  * SI implements derivatives using the local data store (LDS)
3105  * All writes to the LDS happen in all executing threads at
3106  * the same time. TID is the Thread ID for the current
3107  * thread and is a value between 0 and 63, representing
3108  * the thread's position in the wavefront.
3109  *
3110  * For the pixel shader threads are grouped into quads of four pixels.
3111  * The TIDs of the pixels of a quad are:
3112  *
3113  *  +------+------+
3114  *  |4n + 0|4n + 1|
3115  *  +------+------+
3116  *  |4n + 2|4n + 3|
3117  *  +------+------+
3118  *
3119  * So, masking the TID with 0xfffffffc yields the TID of the top left pixel
3120  * of the quad, masking with 0xfffffffd yields the TID of the top pixel of
3121  * the current pixel's column, and masking with 0xfffffffe yields the TID
3122  * of the left pixel of the current pixel's row.
3123  *
3124  * Adding 1 yields the TID of the pixel to the right of the left pixel, and
3125  * adding 2 yields the TID of the pixel below the top pixel.
3126  */
3127 /* masks for thread ID. */
3128 #define TID_MASK_TOP_LEFT 0xfffffffc
3129 #define TID_MASK_TOP      0xfffffffd
3130 #define TID_MASK_LEFT     0xfffffffe
3131
3132 static void si_llvm_emit_ddxy(
3133         const struct lp_build_tgsi_action *action,
3134         struct lp_build_tgsi_context *bld_base,
3135         struct lp_build_emit_data *emit_data)
3136 {
3137         struct si_shader_context *ctx = si_shader_context(bld_base);
3138         struct gallivm_state *gallivm = bld_base->base.gallivm;
3139         const struct tgsi_full_instruction *inst = emit_data->inst;
3140         unsigned opcode = inst->Instruction.Opcode;
3141         LLVMValueRef indices[2];
3142         LLVMValueRef store_ptr, load_ptr0, load_ptr1;
3143         LLVMValueRef tl, trbl, result[4];
3144         unsigned swizzle[4];
3145         unsigned c;
3146         int idx;
3147         unsigned mask;
3148
3149         indices[0] = bld_base->uint_bld.zero;
3150         indices[1] = lp_build_intrinsic(gallivm->builder, "llvm.SI.tid", ctx->i32,
3151                                      NULL, 0, LLVMReadNoneAttribute);
3152         store_ptr = LLVMBuildGEP(gallivm->builder, ctx->lds,
3153                                  indices, 2, "");
3154
3155         if (opcode == TGSI_OPCODE_DDX_FINE)
3156                 mask = TID_MASK_LEFT;
3157         else if (opcode == TGSI_OPCODE_DDY_FINE)
3158                 mask = TID_MASK_TOP;
3159         else
3160                 mask = TID_MASK_TOP_LEFT;
3161
3162         indices[1] = LLVMBuildAnd(gallivm->builder, indices[1],
3163                                   lp_build_const_int32(gallivm, mask), "");
3164         load_ptr0 = LLVMBuildGEP(gallivm->builder, ctx->lds,
3165                                  indices, 2, "");
3166
3167         /* for DDX we want to next X pixel, DDY next Y pixel. */
3168         idx = (opcode == TGSI_OPCODE_DDX || opcode == TGSI_OPCODE_DDX_FINE) ? 1 : 2;
3169         indices[1] = LLVMBuildAdd(gallivm->builder, indices[1],
3170                                   lp_build_const_int32(gallivm, idx), "");
3171         load_ptr1 = LLVMBuildGEP(gallivm->builder, ctx->lds,
3172                                  indices, 2, "");
3173
3174         for (c = 0; c < 4; ++c) {
3175                 unsigned i;
3176
3177                 swizzle[c] = tgsi_util_get_full_src_register_swizzle(&inst->Src[0], c);
3178                 for (i = 0; i < c; ++i) {
3179                         if (swizzle[i] == swizzle[c]) {
3180                                 result[c] = result[i];
3181                                 break;
3182                         }
3183                 }
3184                 if (i != c)
3185                         continue;
3186
3187                 LLVMBuildStore(gallivm->builder,
3188                                LLVMBuildBitCast(gallivm->builder,
3189                                                 lp_build_emit_fetch(bld_base, inst, 0, c),
3190                                                 ctx->i32, ""),
3191                                store_ptr);
3192
3193                 tl = LLVMBuildLoad(gallivm->builder, load_ptr0, "");
3194                 tl = LLVMBuildBitCast(gallivm->builder, tl, ctx->f32, "");
3195
3196                 trbl = LLVMBuildLoad(gallivm->builder, load_ptr1, "");
3197                 trbl = LLVMBuildBitCast(gallivm->builder, trbl, ctx->f32, "");
3198
3199                 result[c] = LLVMBuildFSub(gallivm->builder, trbl, tl, "");
3200         }
3201
3202         emit_data->output[0] = lp_build_gather_values(gallivm, result, 4);
3203 }
3204
3205 /*
3206  * this takes an I,J coordinate pair,
3207  * and works out the X and Y derivatives.
3208  * it returns DDX(I), DDX(J), DDY(I), DDY(J).
3209  */
3210 static LLVMValueRef si_llvm_emit_ddxy_interp(
3211         struct lp_build_tgsi_context *bld_base,
3212         LLVMValueRef interp_ij)
3213 {
3214         struct si_shader_context *ctx = si_shader_context(bld_base);
3215         struct gallivm_state *gallivm = bld_base->base.gallivm;
3216         LLVMValueRef indices[2];
3217         LLVMValueRef store_ptr, load_ptr_x, load_ptr_y, load_ptr_ddx, load_ptr_ddy, temp, temp2;
3218         LLVMValueRef tl, tr, bl, result[4];
3219         unsigned c;
3220
3221         indices[0] = bld_base->uint_bld.zero;
3222         indices[1] = lp_build_intrinsic(gallivm->builder, "llvm.SI.tid", ctx->i32,
3223                                         NULL, 0, LLVMReadNoneAttribute);
3224         store_ptr = LLVMBuildGEP(gallivm->builder, ctx->lds,
3225                                  indices, 2, "");
3226
3227         temp = LLVMBuildAnd(gallivm->builder, indices[1],
3228                             lp_build_const_int32(gallivm, TID_MASK_LEFT), "");
3229
3230         temp2 = LLVMBuildAnd(gallivm->builder, indices[1],
3231                              lp_build_const_int32(gallivm, TID_MASK_TOP), "");
3232
3233         indices[1] = temp;
3234         load_ptr_x = LLVMBuildGEP(gallivm->builder, ctx->lds,
3235                                   indices, 2, "");
3236
3237         indices[1] = temp2;
3238         load_ptr_y = LLVMBuildGEP(gallivm->builder, ctx->lds,
3239                                   indices, 2, "");
3240
3241         indices[1] = LLVMBuildAdd(gallivm->builder, temp,
3242                                   lp_build_const_int32(gallivm, 1), "");
3243         load_ptr_ddx = LLVMBuildGEP(gallivm->builder, ctx->lds,
3244                                    indices, 2, "");
3245
3246         indices[1] = LLVMBuildAdd(gallivm->builder, temp2,
3247                                   lp_build_const_int32(gallivm, 2), "");
3248         load_ptr_ddy = LLVMBuildGEP(gallivm->builder, ctx->lds,
3249                                    indices, 2, "");
3250
3251         for (c = 0; c < 2; ++c) {
3252                 LLVMValueRef store_val;
3253                 LLVMValueRef c_ll = lp_build_const_int32(gallivm, c);
3254
3255                 store_val = LLVMBuildExtractElement(gallivm->builder,
3256                                                     interp_ij, c_ll, "");
3257                 LLVMBuildStore(gallivm->builder,
3258                                store_val,
3259                                store_ptr);
3260
3261                 tl = LLVMBuildLoad(gallivm->builder, load_ptr_x, "");
3262                 tl = LLVMBuildBitCast(gallivm->builder, tl, ctx->f32, "");
3263
3264                 tr = LLVMBuildLoad(gallivm->builder, load_ptr_ddx, "");
3265                 tr = LLVMBuildBitCast(gallivm->builder, tr, ctx->f32, "");
3266
3267                 result[c] = LLVMBuildFSub(gallivm->builder, tr, tl, "");
3268
3269                 tl = LLVMBuildLoad(gallivm->builder, load_ptr_y, "");
3270                 tl = LLVMBuildBitCast(gallivm->builder, tl, ctx->f32, "");
3271
3272                 bl = LLVMBuildLoad(gallivm->builder, load_ptr_ddy, "");
3273                 bl = LLVMBuildBitCast(gallivm->builder, bl, ctx->f32, "");
3274
3275                 result[c + 2] = LLVMBuildFSub(gallivm->builder, bl, tl, "");
3276         }
3277
3278         return lp_build_gather_values(gallivm, result, 4);
3279 }
3280
3281 static void interp_fetch_args(
3282         struct lp_build_tgsi_context *bld_base,
3283         struct lp_build_emit_data *emit_data)
3284 {
3285         struct si_shader_context *ctx = si_shader_context(bld_base);
3286         struct gallivm_state *gallivm = bld_base->base.gallivm;
3287         const struct tgsi_full_instruction *inst = emit_data->inst;
3288
3289         if (inst->Instruction.Opcode == TGSI_OPCODE_INTERP_OFFSET) {
3290                 /* offset is in second src, first two channels */
3291                 emit_data->args[0] = lp_build_emit_fetch(bld_base,
3292                                                          emit_data->inst, 1,
3293                                                          TGSI_CHAN_X);
3294                 emit_data->args[1] = lp_build_emit_fetch(bld_base,
3295                                                          emit_data->inst, 1,
3296                                                          TGSI_CHAN_Y);
3297                 emit_data->arg_count = 2;
3298         } else if (inst->Instruction.Opcode == TGSI_OPCODE_INTERP_SAMPLE) {
3299                 LLVMValueRef sample_position;
3300                 LLVMValueRef sample_id;
3301                 LLVMValueRef halfval = lp_build_const_float(gallivm, 0.5f);
3302
3303                 /* fetch sample ID, then fetch its sample position,
3304                  * and place into first two channels.
3305                  */
3306                 sample_id = lp_build_emit_fetch(bld_base,
3307                                                 emit_data->inst, 1, TGSI_CHAN_X);
3308                 sample_id = LLVMBuildBitCast(gallivm->builder, sample_id,
3309                                              ctx->i32, "");
3310                 sample_position = load_sample_position(&ctx->radeon_bld, sample_id);
3311
3312                 emit_data->args[0] = LLVMBuildExtractElement(gallivm->builder,
3313                                                              sample_position,
3314                                                              lp_build_const_int32(gallivm, 0), "");
3315
3316                 emit_data->args[0] = LLVMBuildFSub(gallivm->builder, emit_data->args[0], halfval, "");
3317                 emit_data->args[1] = LLVMBuildExtractElement(gallivm->builder,
3318                                                              sample_position,
3319                                                              lp_build_const_int32(gallivm, 1), "");
3320                 emit_data->args[1] = LLVMBuildFSub(gallivm->builder, emit_data->args[1], halfval, "");
3321                 emit_data->arg_count = 2;
3322         }
3323 }
3324
3325 static void build_interp_intrinsic(const struct lp_build_tgsi_action *action,
3326                                 struct lp_build_tgsi_context *bld_base,
3327                                 struct lp_build_emit_data *emit_data)
3328 {
3329         struct si_shader_context *ctx = si_shader_context(bld_base);
3330         struct si_shader *shader = ctx->shader;
3331         struct gallivm_state *gallivm = bld_base->base.gallivm;
3332         LLVMValueRef interp_param;
3333         const struct tgsi_full_instruction *inst = emit_data->inst;
3334         const char *intr_name;
3335         int input_index = inst->Src[0].Register.Index;
3336         int chan;
3337         int i;
3338         LLVMValueRef attr_number;
3339         LLVMValueRef params = LLVMGetParam(ctx->radeon_bld.main_fn, SI_PARAM_PRIM_MASK);
3340         int interp_param_idx;
3341         unsigned interp = shader->selector->info.input_interpolate[input_index];
3342         unsigned location;
3343
3344         assert(inst->Src[0].Register.File == TGSI_FILE_INPUT);
3345
3346         if (inst->Instruction.Opcode == TGSI_OPCODE_INTERP_OFFSET ||
3347             inst->Instruction.Opcode == TGSI_OPCODE_INTERP_SAMPLE)
3348                 location = TGSI_INTERPOLATE_LOC_CENTER;
3349         else
3350                 location = TGSI_INTERPOLATE_LOC_CENTROID;
3351
3352         interp_param_idx = lookup_interp_param_index(interp, location);
3353         if (interp_param_idx == -1)
3354                 return;
3355         else if (interp_param_idx)
3356                 interp_param = LLVMGetParam(ctx->radeon_bld.main_fn, interp_param_idx);
3357         else
3358                 interp_param = NULL;
3359
3360         attr_number = lp_build_const_int32(gallivm, input_index);
3361
3362         if (inst->Instruction.Opcode == TGSI_OPCODE_INTERP_OFFSET ||
3363             inst->Instruction.Opcode == TGSI_OPCODE_INTERP_SAMPLE) {
3364                 LLVMValueRef ij_out[2];
3365                 LLVMValueRef ddxy_out = si_llvm_emit_ddxy_interp(bld_base, interp_param);
3366
3367                 /*
3368                  * take the I then J parameters, and the DDX/Y for it, and
3369                  * calculate the IJ inputs for the interpolator.
3370                  * temp1 = ddx * offset/sample.x + I;
3371                  * interp_param.I = ddy * offset/sample.y + temp1;
3372                  * temp1 = ddx * offset/sample.x + J;
3373                  * interp_param.J = ddy * offset/sample.y + temp1;
3374                  */
3375                 for (i = 0; i < 2; i++) {
3376                         LLVMValueRef ix_ll = lp_build_const_int32(gallivm, i);
3377                         LLVMValueRef iy_ll = lp_build_const_int32(gallivm, i + 2);
3378                         LLVMValueRef ddx_el = LLVMBuildExtractElement(gallivm->builder,
3379                                                                       ddxy_out, ix_ll, "");
3380                         LLVMValueRef ddy_el = LLVMBuildExtractElement(gallivm->builder,
3381                                                                       ddxy_out, iy_ll, "");
3382                         LLVMValueRef interp_el = LLVMBuildExtractElement(gallivm->builder,
3383                                                                          interp_param, ix_ll, "");
3384                         LLVMValueRef temp1, temp2;
3385
3386                         interp_el = LLVMBuildBitCast(gallivm->builder, interp_el,
3387                                                      ctx->f32, "");
3388
3389                         temp1 = LLVMBuildFMul(gallivm->builder, ddx_el, emit_data->args[0], "");
3390
3391                         temp1 = LLVMBuildFAdd(gallivm->builder, temp1, interp_el, "");
3392
3393                         temp2 = LLVMBuildFMul(gallivm->builder, ddy_el, emit_data->args[1], "");
3394
3395                         temp2 = LLVMBuildFAdd(gallivm->builder, temp2, temp1, "");
3396
3397                         ij_out[i] = LLVMBuildBitCast(gallivm->builder,
3398                                                      temp2, ctx->i32, "");
3399                 }
3400                 interp_param = lp_build_gather_values(bld_base->base.gallivm, ij_out, 2);
3401         }
3402
3403         intr_name = interp_param ? "llvm.SI.fs.interp" : "llvm.SI.fs.constant";
3404         for (chan = 0; chan < 2; chan++) {
3405                 LLVMValueRef args[4];
3406                 LLVMValueRef llvm_chan;
3407                 unsigned schan;
3408
3409                 schan = tgsi_util_get_full_src_register_swizzle(&inst->Src[0], chan);
3410                 llvm_chan = lp_build_const_int32(gallivm, schan);
3411
3412                 args[0] = llvm_chan;
3413                 args[1] = attr_number;
3414                 args[2] = params;
3415                 args[3] = interp_param;
3416
3417                 emit_data->output[chan] =
3418                         lp_build_intrinsic(gallivm->builder, intr_name,
3419                                            ctx->f32, args, args[3] ? 4 : 3,
3420                                            LLVMReadNoneAttribute | LLVMNoUnwindAttribute);
3421         }
3422 }
3423
3424 static unsigned si_llvm_get_stream(struct lp_build_tgsi_context *bld_base,
3425                                        struct lp_build_emit_data *emit_data)
3426 {
3427         LLVMValueRef (*imms)[4] = lp_soa_context(bld_base)->immediates;
3428         struct tgsi_src_register src0 = emit_data->inst->Src[0].Register;
3429         unsigned stream;
3430
3431         assert(src0.File == TGSI_FILE_IMMEDIATE);
3432
3433         stream = LLVMConstIntGetZExtValue(imms[src0.Index][src0.SwizzleX]) & 0x3;
3434         return stream;
3435 }
3436
3437 /* Emit one vertex from the geometry shader */
3438 static void si_llvm_emit_vertex(
3439         const struct lp_build_tgsi_action *action,
3440         struct lp_build_tgsi_context *bld_base,
3441         struct lp_build_emit_data *emit_data)
3442 {
3443         struct si_shader_context *ctx = si_shader_context(bld_base);
3444         struct lp_build_context *uint = &bld_base->uint_bld;
3445         struct si_shader *shader = ctx->shader;
3446         struct tgsi_shader_info *info = &shader->selector->info;
3447         struct gallivm_state *gallivm = bld_base->base.gallivm;
3448         LLVMValueRef soffset = LLVMGetParam(ctx->radeon_bld.main_fn,
3449                                             SI_PARAM_GS2VS_OFFSET);
3450         LLVMValueRef gs_next_vertex;
3451         LLVMValueRef can_emit, kill;
3452         LLVMValueRef args[2];
3453         unsigned chan;
3454         int i;
3455         unsigned stream;
3456
3457         stream = si_llvm_get_stream(bld_base, emit_data);
3458
3459         /* Write vertex attribute values to GSVS ring */
3460         gs_next_vertex = LLVMBuildLoad(gallivm->builder,
3461                                        ctx->gs_next_vertex[stream],
3462                                        "");
3463
3464         /* If this thread has already emitted the declared maximum number of
3465          * vertices, kill it: excessive vertex emissions are not supposed to
3466          * have any effect, and GS threads have no externally observable
3467          * effects other than emitting vertices.
3468          */
3469         can_emit = LLVMBuildICmp(gallivm->builder, LLVMIntULE, gs_next_vertex,
3470                                  lp_build_const_int32(gallivm,
3471                                                       shader->selector->gs_max_out_vertices), "");
3472         kill = lp_build_select(&bld_base->base, can_emit,
3473                                lp_build_const_float(gallivm, 1.0f),
3474                                lp_build_const_float(gallivm, -1.0f));
3475
3476         lp_build_intrinsic(gallivm->builder, "llvm.AMDGPU.kill",
3477                            ctx->voidt, &kill, 1, 0);
3478
3479         for (i = 0; i < info->num_outputs; i++) {
3480                 LLVMValueRef *out_ptr =
3481                         ctx->radeon_bld.soa.outputs[i];
3482
3483                 for (chan = 0; chan < 4; chan++) {
3484                         LLVMValueRef out_val = LLVMBuildLoad(gallivm->builder, out_ptr[chan], "");
3485                         LLVMValueRef voffset =
3486                                 lp_build_const_int32(gallivm, (i * 4 + chan) *
3487                                                      shader->selector->gs_max_out_vertices);
3488
3489                         voffset = lp_build_add(uint, voffset, gs_next_vertex);
3490                         voffset = lp_build_mul_imm(uint, voffset, 4);
3491
3492                         out_val = LLVMBuildBitCast(gallivm->builder, out_val, ctx->i32, "");
3493
3494                         build_tbuffer_store(ctx,
3495                                             ctx->gsvs_ring[stream],
3496                                             out_val, 1,
3497                                             voffset, soffset, 0,
3498                                             V_008F0C_BUF_DATA_FORMAT_32,
3499                                             V_008F0C_BUF_NUM_FORMAT_UINT,
3500                                             1, 0, 1, 1, 0);
3501                 }
3502         }
3503         gs_next_vertex = lp_build_add(uint, gs_next_vertex,
3504                                       lp_build_const_int32(gallivm, 1));
3505
3506         LLVMBuildStore(gallivm->builder, gs_next_vertex, ctx->gs_next_vertex[stream]);
3507
3508         /* Signal vertex emission */
3509         args[0] = lp_build_const_int32(gallivm, SENDMSG_GS_OP_EMIT | SENDMSG_GS | (stream << 8));
3510         args[1] = LLVMGetParam(ctx->radeon_bld.main_fn, SI_PARAM_GS_WAVE_ID);
3511         lp_build_intrinsic(gallivm->builder, "llvm.SI.sendmsg",
3512                            ctx->voidt, args, 2, LLVMNoUnwindAttribute);
3513 }
3514
3515 /* Cut one primitive from the geometry shader */
3516 static void si_llvm_emit_primitive(
3517         const struct lp_build_tgsi_action *action,
3518         struct lp_build_tgsi_context *bld_base,
3519         struct lp_build_emit_data *emit_data)
3520 {
3521         struct si_shader_context *ctx = si_shader_context(bld_base);
3522         struct gallivm_state *gallivm = bld_base->base.gallivm;
3523         LLVMValueRef args[2];
3524         unsigned stream;
3525
3526         /* Signal primitive cut */
3527         stream = si_llvm_get_stream(bld_base, emit_data);
3528         args[0] = lp_build_const_int32(gallivm, SENDMSG_GS_OP_CUT | SENDMSG_GS | (stream << 8));
3529         args[1] = LLVMGetParam(ctx->radeon_bld.main_fn, SI_PARAM_GS_WAVE_ID);
3530         lp_build_intrinsic(gallivm->builder, "llvm.SI.sendmsg",
3531                            ctx->voidt, args, 2, LLVMNoUnwindAttribute);
3532 }
3533
3534 static void si_llvm_emit_barrier(const struct lp_build_tgsi_action *action,
3535                                  struct lp_build_tgsi_context *bld_base,
3536                                  struct lp_build_emit_data *emit_data)
3537 {
3538         struct si_shader_context *ctx = si_shader_context(bld_base);
3539         struct gallivm_state *gallivm = bld_base->base.gallivm;
3540
3541         lp_build_intrinsic(gallivm->builder,
3542                            HAVE_LLVM >= 0x0309 ? "llvm.amdgcn.s.barrier"
3543                                                : "llvm.AMDGPU.barrier.local",
3544                            ctx->voidt, NULL, 0, LLVMNoUnwindAttribute);
3545 }
3546
3547 static const struct lp_build_tgsi_action tex_action = {
3548         .fetch_args = tex_fetch_args,
3549         .emit = build_tex_intrinsic,
3550 };
3551
3552 static const struct lp_build_tgsi_action interp_action = {
3553         .fetch_args = interp_fetch_args,
3554         .emit = build_interp_intrinsic,
3555 };
3556
3557 static void create_meta_data(struct si_shader_context *ctx)
3558 {
3559         struct gallivm_state *gallivm = ctx->radeon_bld.soa.bld_base.base.gallivm;
3560         LLVMValueRef args[3];
3561
3562         args[0] = LLVMMDStringInContext(gallivm->context, "const", 5);
3563         args[1] = 0;
3564         args[2] = lp_build_const_int32(gallivm, 1);
3565
3566         ctx->const_md = LLVMMDNodeInContext(gallivm->context, args, 3);
3567 }
3568
3569 static void declare_streamout_params(struct si_shader_context *ctx,
3570                                      struct pipe_stream_output_info *so,
3571                                      LLVMTypeRef *params, LLVMTypeRef i32,
3572                                      unsigned *num_params)
3573 {
3574         int i;
3575
3576         /* Streamout SGPRs. */
3577         if (so->num_outputs) {
3578                 params[ctx->param_streamout_config = (*num_params)++] = i32;
3579                 params[ctx->param_streamout_write_index = (*num_params)++] = i32;
3580         }
3581         /* A streamout buffer offset is loaded if the stride is non-zero. */
3582         for (i = 0; i < 4; i++) {
3583                 if (!so->stride[i])
3584                         continue;
3585
3586                 params[ctx->param_streamout_offset[i] = (*num_params)++] = i32;
3587         }
3588 }
3589
3590 static unsigned llvm_get_type_size(LLVMTypeRef type)
3591 {
3592         LLVMTypeKind kind = LLVMGetTypeKind(type);
3593
3594         switch (kind) {
3595         case LLVMIntegerTypeKind:
3596                 return LLVMGetIntTypeWidth(type) / 8;
3597         case LLVMFloatTypeKind:
3598                 return 4;
3599         case LLVMPointerTypeKind:
3600                 return 8;
3601         case LLVMVectorTypeKind:
3602                 return LLVMGetVectorSize(type) *
3603                        llvm_get_type_size(LLVMGetElementType(type));
3604         default:
3605                 assert(0);
3606                 return 0;
3607         }
3608 }
3609
3610 static void create_function(struct si_shader_context *ctx)
3611 {
3612         struct lp_build_tgsi_context *bld_base = &ctx->radeon_bld.soa.bld_base;
3613         struct gallivm_state *gallivm = bld_base->base.gallivm;
3614         struct si_shader *shader = ctx->shader;
3615         LLVMTypeRef params[SI_NUM_PARAMS], v2i32, v3i32;
3616         unsigned i, last_array_pointer, last_sgpr, num_params;
3617
3618         v2i32 = LLVMVectorType(ctx->i32, 2);
3619         v3i32 = LLVMVectorType(ctx->i32, 3);
3620
3621         params[SI_PARAM_RW_BUFFERS] = const_array(ctx->v16i8, SI_NUM_RW_BUFFERS);
3622         params[SI_PARAM_CONST_BUFFERS] = const_array(ctx->v16i8, SI_NUM_CONST_BUFFERS);
3623         params[SI_PARAM_SAMPLERS] = const_array(ctx->v8i32, SI_NUM_SAMPLERS);
3624         params[SI_PARAM_UNUSED] = LLVMPointerType(ctx->i32, CONST_ADDR_SPACE);
3625         last_array_pointer = SI_PARAM_UNUSED;
3626
3627         switch (ctx->type) {
3628         case TGSI_PROCESSOR_VERTEX:
3629                 params[SI_PARAM_VERTEX_BUFFERS] = const_array(ctx->v16i8, SI_NUM_VERTEX_BUFFERS);
3630                 last_array_pointer = SI_PARAM_VERTEX_BUFFERS;
3631                 params[SI_PARAM_BASE_VERTEX] = ctx->i32;
3632                 params[SI_PARAM_START_INSTANCE] = ctx->i32;
3633                 num_params = SI_PARAM_START_INSTANCE+1;
3634
3635                 if (shader->key.vs.as_es) {
3636                         params[ctx->param_es2gs_offset = num_params++] = ctx->i32;
3637                 } else if (shader->key.vs.as_ls) {
3638                         params[SI_PARAM_LS_OUT_LAYOUT] = ctx->i32;
3639                         num_params = SI_PARAM_LS_OUT_LAYOUT+1;
3640                 } else {
3641                         if (ctx->is_gs_copy_shader) {
3642                                 last_array_pointer = SI_PARAM_CONST_BUFFERS;
3643                                 num_params = SI_PARAM_CONST_BUFFERS+1;
3644                         } else {
3645                                 params[SI_PARAM_VS_STATE_BITS] = ctx->i32;
3646                                 num_params = SI_PARAM_VS_STATE_BITS+1;
3647                         }
3648
3649                         /* The locations of the other parameters are assigned dynamically. */
3650                         declare_streamout_params(ctx, &shader->selector->so,
3651                                                  params, ctx->i32, &num_params);
3652                 }
3653
3654                 last_sgpr = num_params-1;
3655
3656                 /* VGPRs */
3657                 params[ctx->param_vertex_id = num_params++] = ctx->i32;
3658                 params[ctx->param_rel_auto_id = num_params++] = ctx->i32;
3659                 params[ctx->param_vs_prim_id = num_params++] = ctx->i32;
3660                 params[ctx->param_instance_id = num_params++] = ctx->i32;
3661                 break;
3662
3663         case TGSI_PROCESSOR_TESS_CTRL:
3664                 params[SI_PARAM_TCS_OUT_OFFSETS] = ctx->i32;
3665                 params[SI_PARAM_TCS_OUT_LAYOUT] = ctx->i32;
3666                 params[SI_PARAM_TCS_IN_LAYOUT] = ctx->i32;
3667                 params[SI_PARAM_TESS_FACTOR_OFFSET] = ctx->i32;
3668                 last_sgpr = SI_PARAM_TESS_FACTOR_OFFSET;
3669
3670                 /* VGPRs */
3671                 params[SI_PARAM_PATCH_ID] = ctx->i32;
3672                 params[SI_PARAM_REL_IDS] = ctx->i32;
3673                 num_params = SI_PARAM_REL_IDS+1;
3674                 break;
3675
3676         case TGSI_PROCESSOR_TESS_EVAL:
3677                 params[SI_PARAM_TCS_OUT_OFFSETS] = ctx->i32;
3678                 params[SI_PARAM_TCS_OUT_LAYOUT] = ctx->i32;
3679                 num_params = SI_PARAM_TCS_OUT_LAYOUT+1;
3680
3681                 if (shader->key.tes.as_es) {
3682                         params[ctx->param_es2gs_offset = num_params++] = ctx->i32;
3683                 } else {
3684                         declare_streamout_params(ctx, &shader->selector->so,
3685                                                  params, ctx->i32, &num_params);
3686                 }
3687                 last_sgpr = num_params - 1;
3688
3689                 /* VGPRs */
3690                 params[ctx->param_tes_u = num_params++] = ctx->f32;
3691                 params[ctx->param_tes_v = num_params++] = ctx->f32;
3692                 params[ctx->param_tes_rel_patch_id = num_params++] = ctx->i32;
3693                 params[ctx->param_tes_patch_id = num_params++] = ctx->i32;
3694                 break;
3695
3696         case TGSI_PROCESSOR_GEOMETRY:
3697                 params[SI_PARAM_GS2VS_OFFSET] = ctx->i32;
3698                 params[SI_PARAM_GS_WAVE_ID] = ctx->i32;
3699                 last_sgpr = SI_PARAM_GS_WAVE_ID;
3700
3701                 /* VGPRs */
3702                 params[SI_PARAM_VTX0_OFFSET] = ctx->i32;
3703                 params[SI_PARAM_VTX1_OFFSET] = ctx->i32;
3704                 params[SI_PARAM_PRIMITIVE_ID] = ctx->i32;
3705                 params[SI_PARAM_VTX2_OFFSET] = ctx->i32;
3706                 params[SI_PARAM_VTX3_OFFSET] = ctx->i32;
3707                 params[SI_PARAM_VTX4_OFFSET] = ctx->i32;
3708                 params[SI_PARAM_VTX5_OFFSET] = ctx->i32;
3709                 params[SI_PARAM_GS_INSTANCE_ID] = ctx->i32;
3710                 num_params = SI_PARAM_GS_INSTANCE_ID+1;
3711                 break;
3712
3713         case TGSI_PROCESSOR_FRAGMENT:
3714                 params[SI_PARAM_ALPHA_REF] = ctx->f32;
3715                 params[SI_PARAM_PRIM_MASK] = ctx->i32;
3716                 last_sgpr = SI_PARAM_PRIM_MASK;
3717                 params[SI_PARAM_PERSP_SAMPLE] = v2i32;
3718                 params[SI_PARAM_PERSP_CENTER] = v2i32;
3719                 params[SI_PARAM_PERSP_CENTROID] = v2i32;
3720                 params[SI_PARAM_PERSP_PULL_MODEL] = v3i32;
3721                 params[SI_PARAM_LINEAR_SAMPLE] = v2i32;
3722                 params[SI_PARAM_LINEAR_CENTER] = v2i32;
3723                 params[SI_PARAM_LINEAR_CENTROID] = v2i32;
3724                 params[SI_PARAM_LINE_STIPPLE_TEX] = ctx->f32;
3725                 params[SI_PARAM_POS_X_FLOAT] = ctx->f32;
3726                 params[SI_PARAM_POS_Y_FLOAT] = ctx->f32;
3727                 params[SI_PARAM_POS_Z_FLOAT] = ctx->f32;
3728                 params[SI_PARAM_POS_W_FLOAT] = ctx->f32;
3729                 params[SI_PARAM_FRONT_FACE] = ctx->i32;
3730                 params[SI_PARAM_ANCILLARY] = ctx->i32;
3731                 params[SI_PARAM_SAMPLE_COVERAGE] = ctx->f32;
3732                 params[SI_PARAM_POS_FIXED_PT] = ctx->f32;
3733                 num_params = SI_PARAM_POS_FIXED_PT+1;
3734                 break;
3735
3736         default:
3737                 assert(0 && "unimplemented shader");
3738                 return;
3739         }
3740
3741         assert(num_params <= Elements(params));
3742         radeon_llvm_create_func(&ctx->radeon_bld, NULL, 0,
3743                                 params, num_params);
3744         radeon_llvm_shader_type(ctx->radeon_bld.main_fn, ctx->type);
3745         ctx->return_value = LLVMGetUndef(ctx->radeon_bld.return_type);
3746
3747         shader->num_input_sgprs = 0;
3748         shader->num_input_vgprs = 0;
3749
3750         for (i = 0; i <= last_sgpr; ++i) {
3751                 LLVMValueRef P = LLVMGetParam(ctx->radeon_bld.main_fn, i);
3752
3753                 /* We tell llvm that array inputs are passed by value to allow Sinking pass
3754                  * to move load. Inputs are constant so this is fine. */
3755                 if (i <= last_array_pointer)
3756                         LLVMAddAttribute(P, LLVMByValAttribute);
3757                 else
3758                         LLVMAddAttribute(P, LLVMInRegAttribute);
3759
3760                 shader->num_input_sgprs += llvm_get_type_size(params[i]) / 4;
3761         }
3762
3763         /* Unused fragment shader inputs are eliminated by the compiler,
3764          * so we don't know yet how many there will be.
3765          */
3766         if (ctx->type != TGSI_PROCESSOR_FRAGMENT)
3767                 for (; i < num_params; ++i)
3768                         shader->num_input_vgprs += llvm_get_type_size(params[i]) / 4;
3769
3770         if (bld_base->info &&
3771             (bld_base->info->opcode_count[TGSI_OPCODE_DDX] > 0 ||
3772              bld_base->info->opcode_count[TGSI_OPCODE_DDY] > 0 ||
3773              bld_base->info->opcode_count[TGSI_OPCODE_DDX_FINE] > 0 ||
3774              bld_base->info->opcode_count[TGSI_OPCODE_DDY_FINE] > 0 ||
3775              bld_base->info->opcode_count[TGSI_OPCODE_INTERP_OFFSET] > 0 ||
3776              bld_base->info->opcode_count[TGSI_OPCODE_INTERP_SAMPLE] > 0))
3777                 ctx->lds =
3778                         LLVMAddGlobalInAddressSpace(gallivm->module,
3779                                                     LLVMArrayType(ctx->i32, 64),
3780                                                     "ddxy_lds",
3781                                                     LOCAL_ADDR_SPACE);
3782
3783         if ((ctx->type == TGSI_PROCESSOR_VERTEX && shader->key.vs.as_ls) ||
3784             ctx->type == TGSI_PROCESSOR_TESS_CTRL ||
3785             ctx->type == TGSI_PROCESSOR_TESS_EVAL) {
3786                 /* This is the upper bound, maximum is 32 inputs times 32 vertices */
3787                 unsigned vertex_data_dw_size = 32*32*4;
3788                 unsigned patch_data_dw_size = 32*4;
3789                 /* The formula is: TCS inputs + TCS outputs + TCS patch outputs. */
3790                 unsigned patch_dw_size = vertex_data_dw_size*2 + patch_data_dw_size;
3791                 unsigned lds_dwords = patch_dw_size;
3792
3793                 /* The actual size is computed outside of the shader to reduce
3794                  * the number of shader variants. */
3795                 ctx->lds =
3796                         LLVMAddGlobalInAddressSpace(gallivm->module,
3797                                                     LLVMArrayType(ctx->i32, lds_dwords),
3798                                                     "tess_lds",
3799                                                     LOCAL_ADDR_SPACE);
3800         }
3801 }
3802
3803 static void preload_constants(struct si_shader_context *ctx)
3804 {
3805         struct lp_build_tgsi_context *bld_base = &ctx->radeon_bld.soa.bld_base;
3806         struct gallivm_state *gallivm = bld_base->base.gallivm;
3807         const struct tgsi_shader_info *info = bld_base->info;
3808         unsigned buf;
3809         LLVMValueRef ptr = LLVMGetParam(ctx->radeon_bld.main_fn, SI_PARAM_CONST_BUFFERS);
3810
3811         for (buf = 0; buf < SI_NUM_CONST_BUFFERS; buf++) {
3812                 unsigned i, num_const = info->const_file_max[buf] + 1;
3813
3814                 if (num_const == 0)
3815                         continue;
3816
3817                 /* Allocate space for the constant values */
3818                 ctx->constants[buf] = CALLOC(num_const * 4, sizeof(LLVMValueRef));
3819
3820                 /* Load the resource descriptor */
3821                 ctx->const_buffers[buf] =
3822                         build_indexed_load_const(ctx, ptr, lp_build_const_int32(gallivm, buf));
3823
3824                 /* Load the constants, we rely on the code sinking to do the rest */
3825                 for (i = 0; i < num_const * 4; ++i) {
3826                         ctx->constants[buf][i] =
3827                                 buffer_load_const(gallivm->builder,
3828                                         ctx->const_buffers[buf],
3829                                         lp_build_const_int32(gallivm, i * 4),
3830                                         ctx->f32);
3831                 }
3832         }
3833 }
3834
3835 static void preload_samplers(struct si_shader_context *ctx)
3836 {
3837         struct lp_build_tgsi_context *bld_base = &ctx->radeon_bld.soa.bld_base;
3838         struct gallivm_state *gallivm = bld_base->base.gallivm;
3839         const struct tgsi_shader_info *info = bld_base->info;
3840         unsigned i, num_samplers = info->file_max[TGSI_FILE_SAMPLER] + 1;
3841         LLVMValueRef offset;
3842
3843         if (num_samplers == 0)
3844                 return;
3845
3846         /* Load the resources and samplers, we rely on the code sinking to do the rest */
3847         for (i = 0; i < num_samplers; ++i) {
3848                 /* Resource */
3849                 offset = lp_build_const_int32(gallivm, i);
3850                 ctx->sampler_views[i] =
3851                         get_sampler_desc(ctx, offset, DESC_IMAGE);
3852
3853                 /* FMASK resource */
3854                 if (info->is_msaa_sampler[i])
3855                         ctx->fmasks[i] =
3856                                 get_sampler_desc(ctx, offset, DESC_FMASK);
3857                 else
3858                         ctx->sampler_states[i] =
3859                                 get_sampler_desc(ctx, offset, DESC_SAMPLER);
3860         }
3861 }
3862
3863 static void preload_streamout_buffers(struct si_shader_context *ctx)
3864 {
3865         struct lp_build_tgsi_context *bld_base = &ctx->radeon_bld.soa.bld_base;
3866         struct gallivm_state *gallivm = bld_base->base.gallivm;
3867         unsigned i;
3868
3869         /* Streamout can only be used if the shader is compiled as VS. */
3870         if (!ctx->shader->selector->so.num_outputs ||
3871             (ctx->type == TGSI_PROCESSOR_VERTEX &&
3872              (ctx->shader->key.vs.as_es ||
3873               ctx->shader->key.vs.as_ls)) ||
3874             (ctx->type == TGSI_PROCESSOR_TESS_EVAL &&
3875              ctx->shader->key.tes.as_es))
3876                 return;
3877
3878         LLVMValueRef buf_ptr = LLVMGetParam(ctx->radeon_bld.main_fn,
3879                                             SI_PARAM_RW_BUFFERS);
3880
3881         /* Load the resources, we rely on the code sinking to do the rest */
3882         for (i = 0; i < 4; ++i) {
3883                 if (ctx->shader->selector->so.stride[i]) {
3884                         LLVMValueRef offset = lp_build_const_int32(gallivm,
3885                                                                    SI_SO_BUF_OFFSET + i);
3886
3887                         ctx->so_buffers[i] = build_indexed_load_const(ctx, buf_ptr, offset);
3888                 }
3889         }
3890 }
3891
3892 /**
3893  * Load ESGS and GSVS ring buffer resource descriptors and save the variables
3894  * for later use.
3895  */
3896 static void preload_ring_buffers(struct si_shader_context *ctx)
3897 {
3898         struct gallivm_state *gallivm =
3899                 ctx->radeon_bld.soa.bld_base.base.gallivm;
3900
3901         LLVMValueRef buf_ptr = LLVMGetParam(ctx->radeon_bld.main_fn,
3902                                             SI_PARAM_RW_BUFFERS);
3903
3904         if ((ctx->type == TGSI_PROCESSOR_VERTEX &&
3905              ctx->shader->key.vs.as_es) ||
3906             (ctx->type == TGSI_PROCESSOR_TESS_EVAL &&
3907              ctx->shader->key.tes.as_es) ||
3908             ctx->type == TGSI_PROCESSOR_GEOMETRY) {
3909                 LLVMValueRef offset = lp_build_const_int32(gallivm, SI_RING_ESGS);
3910
3911                 ctx->esgs_ring =
3912                         build_indexed_load_const(ctx, buf_ptr, offset);
3913         }
3914
3915         if (ctx->is_gs_copy_shader) {
3916                 LLVMValueRef offset = lp_build_const_int32(gallivm, SI_RING_GSVS);
3917
3918                 ctx->gsvs_ring[0] =
3919                         build_indexed_load_const(ctx, buf_ptr, offset);
3920         }
3921         if (ctx->type == TGSI_PROCESSOR_GEOMETRY) {
3922                 int i;
3923                 for (i = 0; i < 4; i++) {
3924                         LLVMValueRef offset = lp_build_const_int32(gallivm, SI_RING_GSVS + i);
3925
3926                         ctx->gsvs_ring[i] =
3927                                 build_indexed_load_const(ctx, buf_ptr, offset);
3928                 }
3929         }
3930 }
3931
3932 void si_shader_binary_read_config(struct radeon_shader_binary *binary,
3933                                   struct si_shader_config *conf,
3934                                   unsigned symbol_offset)
3935 {
3936         unsigned i;
3937         const unsigned char *config =
3938                 radeon_shader_binary_config_start(binary, symbol_offset);
3939
3940         /* XXX: We may be able to emit some of these values directly rather than
3941          * extracting fields to be emitted later.
3942          */
3943
3944         for (i = 0; i < binary->config_size_per_symbol; i+= 8) {
3945                 unsigned reg = util_le32_to_cpu(*(uint32_t*)(config + i));
3946                 unsigned value = util_le32_to_cpu(*(uint32_t*)(config + i + 4));
3947                 switch (reg) {
3948                 case R_00B028_SPI_SHADER_PGM_RSRC1_PS:
3949                 case R_00B128_SPI_SHADER_PGM_RSRC1_VS:
3950                 case R_00B228_SPI_SHADER_PGM_RSRC1_GS:
3951                 case R_00B848_COMPUTE_PGM_RSRC1:
3952                         conf->num_sgprs = MAX2(conf->num_sgprs, (G_00B028_SGPRS(value) + 1) * 8);
3953                         conf->num_vgprs = MAX2(conf->num_vgprs, (G_00B028_VGPRS(value) + 1) * 4);
3954                         conf->float_mode =  G_00B028_FLOAT_MODE(value);
3955                         conf->rsrc1 = value;
3956                         break;
3957                 case R_00B02C_SPI_SHADER_PGM_RSRC2_PS:
3958                         conf->lds_size = MAX2(conf->lds_size, G_00B02C_EXTRA_LDS_SIZE(value));
3959                         break;
3960                 case R_00B84C_COMPUTE_PGM_RSRC2:
3961                         conf->lds_size = MAX2(conf->lds_size, G_00B84C_LDS_SIZE(value));
3962                         conf->rsrc2 = value;
3963                         break;
3964                 case R_0286CC_SPI_PS_INPUT_ENA:
3965                         conf->spi_ps_input_ena = value;
3966                         break;
3967                 case R_0286D0_SPI_PS_INPUT_ADDR:
3968                         conf->spi_ps_input_addr = value;
3969                         break;
3970                 case R_0286E8_SPI_TMPRING_SIZE:
3971                 case R_00B860_COMPUTE_TMPRING_SIZE:
3972                         /* WAVESIZE is in units of 256 dwords. */
3973                         conf->scratch_bytes_per_wave =
3974                                 G_00B860_WAVESIZE(value) * 256 * 4 * 1;
3975                         break;
3976                 default:
3977                         {
3978                                 static bool printed;
3979
3980                                 if (!printed) {
3981                                         fprintf(stderr, "Warning: LLVM emitted unknown "
3982                                                 "config register: 0x%x\n", reg);
3983                                         printed = true;
3984                                 }
3985                         }
3986                         break;
3987                 }
3988
3989                 if (!conf->spi_ps_input_addr)
3990                         conf->spi_ps_input_addr = conf->spi_ps_input_ena;
3991         }
3992 }
3993
3994 void si_shader_apply_scratch_relocs(struct si_context *sctx,
3995                         struct si_shader *shader,
3996                         uint64_t scratch_va)
3997 {
3998         unsigned i;
3999         uint32_t scratch_rsrc_dword0 = scratch_va;
4000         uint32_t scratch_rsrc_dword1 =
4001                 S_008F04_BASE_ADDRESS_HI(scratch_va >> 32)
4002                 |  S_008F04_STRIDE(shader->config.scratch_bytes_per_wave / 64);
4003
4004         for (i = 0 ; i < shader->binary.reloc_count; i++) {
4005                 const struct radeon_shader_reloc *reloc =
4006                                         &shader->binary.relocs[i];
4007                 if (!strcmp(scratch_rsrc_dword0_symbol, reloc->name)) {
4008                         util_memcpy_cpu_to_le32(shader->binary.code + reloc->offset,
4009                         &scratch_rsrc_dword0, 4);
4010                 } else if (!strcmp(scratch_rsrc_dword1_symbol, reloc->name)) {
4011                         util_memcpy_cpu_to_le32(shader->binary.code + reloc->offset,
4012                         &scratch_rsrc_dword1, 4);
4013                 }
4014         }
4015 }
4016
4017 int si_shader_binary_upload(struct si_screen *sscreen, struct si_shader *shader)
4018 {
4019         const struct radeon_shader_binary *binary = &shader->binary;
4020         unsigned code_size = binary->code_size + binary->rodata_size;
4021         unsigned char *ptr;
4022
4023         r600_resource_reference(&shader->bo, NULL);
4024         shader->bo = si_resource_create_custom(&sscreen->b.b,
4025                                                PIPE_USAGE_IMMUTABLE,
4026                                                code_size);
4027         if (!shader->bo)
4028                 return -ENOMEM;
4029
4030         ptr = sscreen->b.ws->buffer_map(shader->bo->buf, NULL,
4031                                         PIPE_TRANSFER_READ_WRITE);
4032         util_memcpy_cpu_to_le32(ptr, binary->code, binary->code_size);
4033         if (binary->rodata_size > 0) {
4034                 ptr += binary->code_size;
4035                 util_memcpy_cpu_to_le32(ptr, binary->rodata,
4036                                         binary->rodata_size);
4037         }
4038
4039         sscreen->b.ws->buffer_unmap(shader->bo->buf);
4040         return 0;
4041 }
4042
4043 static void si_shader_dump_disassembly(const struct radeon_shader_binary *binary,
4044                                        struct pipe_debug_callback *debug)
4045 {
4046         char *line, *p;
4047         unsigned i, count;
4048
4049         if (binary->disasm_string) {
4050                 fprintf(stderr, "\nShader Disassembly:\n\n");
4051                 fprintf(stderr, "%s\n", binary->disasm_string);
4052
4053                 if (debug && debug->debug_message) {
4054                         /* Very long debug messages are cut off, so send the
4055                          * disassembly one line at a time. This causes more
4056                          * overhead, but on the plus side it simplifies
4057                          * parsing of resulting logs.
4058                          */
4059                         pipe_debug_message(debug, SHADER_INFO,
4060                                            "Shader Disassembly Begin");
4061
4062                         line = binary->disasm_string;
4063                         while (*line) {
4064                                 p = strchrnul(line, '\n');
4065                                 count = p - line;
4066
4067                                 if (count) {
4068                                         pipe_debug_message(debug, SHADER_INFO,
4069                                                            "%.*s", count, line);
4070                                 }
4071
4072                                 if (!*p)
4073                                         break;
4074                                 line = p + 1;
4075                         }
4076
4077                         pipe_debug_message(debug, SHADER_INFO,
4078                                            "Shader Disassembly End");
4079                 }
4080         } else {
4081                 fprintf(stderr, "SI CODE:\n");
4082                 for (i = 0; i < binary->code_size; i += 4) {
4083                         fprintf(stderr, "@0x%x: %02x%02x%02x%02x\n", i,
4084                                 binary->code[i + 3], binary->code[i + 2],
4085                                 binary->code[i + 1], binary->code[i]);
4086                 }
4087         }
4088 }
4089
4090 static void si_shader_dump_stats(struct si_screen *sscreen,
4091                                  struct si_shader_config *conf,
4092                                  unsigned num_inputs,
4093                                  unsigned code_size,
4094                                  struct pipe_debug_callback *debug,
4095                                  unsigned processor)
4096 {
4097         unsigned lds_increment = sscreen->b.chip_class >= CIK ? 512 : 256;
4098         unsigned lds_per_wave = 0;
4099         unsigned max_simd_waves = 10;
4100
4101         /* Compute LDS usage for PS. */
4102         if (processor == TGSI_PROCESSOR_FRAGMENT) {
4103                 /* The minimum usage per wave is (num_inputs * 36). The maximum
4104                  * usage is (num_inputs * 36 * 16).
4105                  * We can get anything in between and it varies between waves.
4106                  *
4107                  * Other stages don't know the size at compile time or don't
4108                  * allocate LDS per wave, but instead they do it per thread group.
4109                  */
4110                 lds_per_wave = conf->lds_size * lds_increment +
4111                                align(num_inputs * 36, lds_increment);
4112         }
4113
4114         /* Compute the per-SIMD wave counts. */
4115         if (conf->num_sgprs) {
4116                 if (sscreen->b.chip_class >= VI)
4117                         max_simd_waves = MIN2(max_simd_waves, 800 / conf->num_sgprs);
4118                 else
4119                         max_simd_waves = MIN2(max_simd_waves, 512 / conf->num_sgprs);
4120         }
4121
4122         if (conf->num_vgprs)
4123                 max_simd_waves = MIN2(max_simd_waves, 256 / conf->num_vgprs);
4124
4125         /* LDS is 64KB per CU (4 SIMDs), divided into 16KB blocks per SIMD
4126          * that PS can use.
4127          */
4128         if (lds_per_wave)
4129                 max_simd_waves = MIN2(max_simd_waves, 16384 / lds_per_wave);
4130
4131         if (r600_can_dump_shader(&sscreen->b, processor)) {
4132                 if (processor == TGSI_PROCESSOR_FRAGMENT) {
4133                         fprintf(stderr, "*** SHADER CONFIG ***\n"
4134                                 "SPI_PS_INPUT_ADDR = 0x%04x\n"
4135                                 "SPI_PS_INPUT_ENA  = 0x%04x\n",
4136                                 conf->spi_ps_input_addr, conf->spi_ps_input_ena);
4137                 }
4138
4139                 fprintf(stderr, "*** SHADER STATS ***\n"
4140                         "SGPRS: %d\n"
4141                         "VGPRS: %d\n"
4142                         "Code Size: %d bytes\n"
4143                         "LDS: %d blocks\n"
4144                         "Scratch: %d bytes per wave\n"
4145                         "Max Waves: %d\n"
4146                         "********************\n",
4147                         conf->num_sgprs, conf->num_vgprs, code_size,
4148                         conf->lds_size, conf->scratch_bytes_per_wave,
4149                         max_simd_waves);
4150         }
4151
4152         pipe_debug_message(debug, SHADER_INFO,
4153                            "Shader Stats: SGPRS: %d VGPRS: %d Code Size: %d "
4154                            "LDS: %d Scratch: %d Max Waves: %d",
4155                            conf->num_sgprs, conf->num_vgprs, code_size,
4156                            conf->lds_size, conf->scratch_bytes_per_wave,
4157                            max_simd_waves);
4158 }
4159
4160 void si_shader_dump(struct si_screen *sscreen, struct si_shader *shader,
4161                     struct pipe_debug_callback *debug, unsigned processor)
4162 {
4163         if (r600_can_dump_shader(&sscreen->b, processor))
4164                 if (!(sscreen->b.debug_flags & DBG_NO_ASM))
4165                         si_shader_dump_disassembly(&shader->binary, debug);
4166
4167         si_shader_dump_stats(sscreen, &shader->config,
4168                              shader->selector ? shader->selector->info.num_inputs : 0,
4169                              shader->binary.code_size, debug, processor);
4170 }
4171
4172 int si_compile_llvm(struct si_screen *sscreen,
4173                     struct radeon_shader_binary *binary,
4174                     struct si_shader_config *conf,
4175                     LLVMTargetMachineRef tm,
4176                     LLVMModuleRef mod,
4177                     struct pipe_debug_callback *debug,
4178                     unsigned processor,
4179                     const char *name)
4180 {
4181         int r = 0;
4182         unsigned count = p_atomic_inc_return(&sscreen->b.num_compilations);
4183
4184         if (r600_can_dump_shader(&sscreen->b, processor)) {
4185                 fprintf(stderr, "radeonsi: Compiling shader %d\n", count);
4186
4187                 if (!(sscreen->b.debug_flags & (DBG_NO_IR | DBG_PREOPT_IR))) {
4188                         fprintf(stderr, "%s LLVM IR:\n\n", name);
4189                         LLVMDumpModule(mod);
4190                         fprintf(stderr, "\n");
4191                 }
4192         }
4193
4194         if (!si_replace_shader(count, binary)) {
4195                 r = radeon_llvm_compile(mod, binary,
4196                         r600_get_llvm_processor_name(sscreen->b.family), tm,
4197                         debug);
4198                 if (r)
4199                         return r;
4200         }
4201
4202         si_shader_binary_read_config(binary, conf, 0);
4203
4204         /* Enable 64-bit and 16-bit denormals, because there is no performance
4205          * cost.
4206          *
4207          * If denormals are enabled, all floating-point output modifiers are
4208          * ignored.
4209          *
4210          * Don't enable denormals for 32-bit floats, because:
4211          * - Floating-point output modifiers would be ignored by the hw.
4212          * - Some opcodes don't support denormals, such as v_mad_f32. We would
4213          *   have to stop using those.
4214          * - SI & CI would be very slow.
4215          */
4216         conf->float_mode |= V_00B028_FP_64_DENORMS;
4217
4218         FREE(binary->config);
4219         FREE(binary->global_symbol_offsets);
4220         binary->config = NULL;
4221         binary->global_symbol_offsets = NULL;
4222         return r;
4223 }
4224
4225 /* Generate code for the hardware VS shader stage to go with a geometry shader */
4226 static int si_generate_gs_copy_shader(struct si_screen *sscreen,
4227                                       struct si_shader_context *ctx,
4228                                       struct si_shader *gs,
4229                                       struct pipe_debug_callback *debug)
4230 {
4231         struct gallivm_state *gallivm = &ctx->radeon_bld.gallivm;
4232         struct lp_build_tgsi_context *bld_base = &ctx->radeon_bld.soa.bld_base;
4233         struct lp_build_context *uint = &bld_base->uint_bld;
4234         struct si_shader_output_values *outputs;
4235         struct tgsi_shader_info *gsinfo = &gs->selector->info;
4236         LLVMValueRef args[9];
4237         int i, r;
4238
4239         outputs = MALLOC(gsinfo->num_outputs * sizeof(outputs[0]));
4240
4241         si_init_shader_ctx(ctx, sscreen, ctx->shader, ctx->tm, gsinfo);
4242         ctx->type = TGSI_PROCESSOR_VERTEX;
4243         ctx->is_gs_copy_shader = true;
4244
4245         create_meta_data(ctx);
4246         create_function(ctx);
4247         preload_streamout_buffers(ctx);
4248         preload_ring_buffers(ctx);
4249
4250         args[0] = ctx->gsvs_ring[0];
4251         args[1] = lp_build_mul_imm(uint,
4252                                    LLVMGetParam(ctx->radeon_bld.main_fn,
4253                                                 ctx->param_vertex_id),
4254                                    4);
4255         args[3] = uint->zero;
4256         args[4] = uint->one;  /* OFFEN */
4257         args[5] = uint->zero; /* IDXEN */
4258         args[6] = uint->one;  /* GLC */
4259         args[7] = uint->one;  /* SLC */
4260         args[8] = uint->zero; /* TFE */
4261
4262         /* Fetch vertex data from GSVS ring */
4263         for (i = 0; i < gsinfo->num_outputs; ++i) {
4264                 unsigned chan;
4265
4266                 outputs[i].name = gsinfo->output_semantic_name[i];
4267                 outputs[i].sid = gsinfo->output_semantic_index[i];
4268
4269                 for (chan = 0; chan < 4; chan++) {
4270                         args[2] = lp_build_const_int32(gallivm,
4271                                                        (i * 4 + chan) *
4272                                                        gs->selector->gs_max_out_vertices * 16 * 4);
4273
4274                         outputs[i].values[chan] =
4275                                 LLVMBuildBitCast(gallivm->builder,
4276                                                  lp_build_intrinsic(gallivm->builder,
4277                                                                  "llvm.SI.buffer.load.dword.i32.i32",
4278                                                                  ctx->i32, args, 9,
4279                                                                  LLVMReadOnlyAttribute | LLVMNoUnwindAttribute),
4280                                                  ctx->f32, "");
4281                 }
4282         }
4283
4284         si_llvm_export_vs(bld_base, outputs, gsinfo->num_outputs);
4285
4286         LLVMBuildRet(gallivm->builder, ctx->return_value);
4287
4288         /* Dump LLVM IR before any optimization passes */
4289         if (sscreen->b.debug_flags & DBG_PREOPT_IR &&
4290             r600_can_dump_shader(&sscreen->b, TGSI_PROCESSOR_GEOMETRY))
4291                 LLVMDumpModule(bld_base->base.gallivm->module);
4292
4293         radeon_llvm_finalize_module(&ctx->radeon_bld);
4294
4295         r = si_compile_llvm(sscreen, &ctx->shader->binary,
4296                             &ctx->shader->config, ctx->tm,
4297                             bld_base->base.gallivm->module,
4298                             debug, TGSI_PROCESSOR_GEOMETRY,
4299                             "GS Copy Shader");
4300         if (!r) {
4301                 if (r600_can_dump_shader(&sscreen->b, TGSI_PROCESSOR_GEOMETRY))
4302                         fprintf(stderr, "GS Copy Shader:\n");
4303                 si_shader_dump(sscreen, ctx->shader, debug,
4304                                TGSI_PROCESSOR_GEOMETRY);
4305                 r = si_shader_binary_upload(sscreen, ctx->shader);
4306         }
4307
4308         radeon_llvm_dispose(&ctx->radeon_bld);
4309
4310         FREE(outputs);
4311         return r;
4312 }
4313
4314 void si_dump_shader_key(unsigned shader, union si_shader_key *key, FILE *f)
4315 {
4316         int i;
4317
4318         fprintf(f, "SHADER KEY\n");
4319
4320         switch (shader) {
4321         case PIPE_SHADER_VERTEX:
4322                 fprintf(f, "  instance_divisors = {");
4323                 for (i = 0; i < Elements(key->vs.prolog.instance_divisors); i++)
4324                         fprintf(f, !i ? "%u" : ", %u",
4325                                 key->vs.prolog.instance_divisors[i]);
4326                 fprintf(f, "}\n");
4327                 fprintf(f, "  as_es = %u\n", key->vs.as_es);
4328                 fprintf(f, "  as_ls = %u\n", key->vs.as_ls);
4329                 fprintf(f, "  export_prim_id = %u\n", key->vs.epilog.export_prim_id);
4330                 break;
4331
4332         case PIPE_SHADER_TESS_CTRL:
4333                 fprintf(f, "  prim_mode = %u\n", key->tcs.epilog.prim_mode);
4334                 break;
4335
4336         case PIPE_SHADER_TESS_EVAL:
4337                 fprintf(f, "  as_es = %u\n", key->tes.as_es);
4338                 fprintf(f, "  export_prim_id = %u\n", key->tes.epilog.export_prim_id);
4339                 break;
4340
4341         case PIPE_SHADER_GEOMETRY:
4342                 break;
4343
4344         case PIPE_SHADER_FRAGMENT:
4345                 fprintf(f, "  prolog.color_two_side = %u\n", key->ps.prolog.color_two_side);
4346                 fprintf(f, "  prolog.poly_stipple = %u\n", key->ps.prolog.poly_stipple);
4347                 fprintf(f, "  prolog.force_persample_interp = %u\n", key->ps.prolog.force_persample_interp);
4348                 fprintf(f, "  epilog.spi_shader_col_format = 0x%x\n", key->ps.epilog.spi_shader_col_format);
4349                 fprintf(f, "  epilog.color_is_int8 = 0x%X\n", key->ps.epilog.color_is_int8);
4350                 fprintf(f, "  epilog.last_cbuf = %u\n", key->ps.epilog.last_cbuf);
4351                 fprintf(f, "  epilog.alpha_func = %u\n", key->ps.epilog.alpha_func);
4352                 fprintf(f, "  epilog.alpha_to_one = %u\n", key->ps.epilog.alpha_to_one);
4353                 fprintf(f, "  epilog.poly_line_smoothing = %u\n", key->ps.epilog.poly_line_smoothing);
4354                 fprintf(f, "  epilog.clamp_color = %u\n", key->ps.epilog.clamp_color);
4355                 break;
4356
4357         default:
4358                 assert(0);
4359         }
4360 }
4361
4362 static void si_init_shader_ctx(struct si_shader_context *ctx,
4363                                struct si_screen *sscreen,
4364                                struct si_shader *shader,
4365                                LLVMTargetMachineRef tm,
4366                                struct tgsi_shader_info *info)
4367 {
4368         struct lp_build_tgsi_context *bld_base;
4369
4370         memset(ctx, 0, sizeof(*ctx));
4371         radeon_llvm_context_init(&ctx->radeon_bld, "amdgcn--");
4372         ctx->tm = tm;
4373         ctx->screen = sscreen;
4374         if (shader && shader->selector)
4375                 ctx->type = shader->selector->info.processor;
4376         else
4377                 ctx->type = -1;
4378         ctx->shader = shader;
4379
4380         ctx->voidt = LLVMVoidTypeInContext(ctx->radeon_bld.gallivm.context);
4381         ctx->i1 = LLVMInt1TypeInContext(ctx->radeon_bld.gallivm.context);
4382         ctx->i8 = LLVMInt8TypeInContext(ctx->radeon_bld.gallivm.context);
4383         ctx->i32 = LLVMInt32TypeInContext(ctx->radeon_bld.gallivm.context);
4384         ctx->i128 = LLVMIntTypeInContext(ctx->radeon_bld.gallivm.context, 128);
4385         ctx->f32 = LLVMFloatTypeInContext(ctx->radeon_bld.gallivm.context);
4386         ctx->v16i8 = LLVMVectorType(ctx->i8, 16);
4387         ctx->v4i32 = LLVMVectorType(ctx->i32, 4);
4388         ctx->v4f32 = LLVMVectorType(ctx->f32, 4);
4389         ctx->v8i32 = LLVMVectorType(ctx->i32, 8);
4390
4391         bld_base = &ctx->radeon_bld.soa.bld_base;
4392         bld_base->info = info;
4393         bld_base->emit_fetch_funcs[TGSI_FILE_CONSTANT] = fetch_constant;
4394
4395         bld_base->op_actions[TGSI_OPCODE_INTERP_CENTROID] = interp_action;
4396         bld_base->op_actions[TGSI_OPCODE_INTERP_SAMPLE] = interp_action;
4397         bld_base->op_actions[TGSI_OPCODE_INTERP_OFFSET] = interp_action;
4398
4399         bld_base->op_actions[TGSI_OPCODE_TEX] = tex_action;
4400         bld_base->op_actions[TGSI_OPCODE_TEX2] = tex_action;
4401         bld_base->op_actions[TGSI_OPCODE_TXB] = tex_action;
4402         bld_base->op_actions[TGSI_OPCODE_TXB2] = tex_action;
4403         bld_base->op_actions[TGSI_OPCODE_TXD] = tex_action;
4404         bld_base->op_actions[TGSI_OPCODE_TXF] = tex_action;
4405         bld_base->op_actions[TGSI_OPCODE_TXL] = tex_action;
4406         bld_base->op_actions[TGSI_OPCODE_TXL2] = tex_action;
4407         bld_base->op_actions[TGSI_OPCODE_TXP] = tex_action;
4408         bld_base->op_actions[TGSI_OPCODE_TXQ] = tex_action;
4409         bld_base->op_actions[TGSI_OPCODE_TG4] = tex_action;
4410         bld_base->op_actions[TGSI_OPCODE_LODQ] = tex_action;
4411         bld_base->op_actions[TGSI_OPCODE_TXQS].emit = si_llvm_emit_txqs;
4412
4413         bld_base->op_actions[TGSI_OPCODE_DDX].emit = si_llvm_emit_ddxy;
4414         bld_base->op_actions[TGSI_OPCODE_DDY].emit = si_llvm_emit_ddxy;
4415         bld_base->op_actions[TGSI_OPCODE_DDX_FINE].emit = si_llvm_emit_ddxy;
4416         bld_base->op_actions[TGSI_OPCODE_DDY_FINE].emit = si_llvm_emit_ddxy;
4417
4418         bld_base->op_actions[TGSI_OPCODE_EMIT].emit = si_llvm_emit_vertex;
4419         bld_base->op_actions[TGSI_OPCODE_ENDPRIM].emit = si_llvm_emit_primitive;
4420         bld_base->op_actions[TGSI_OPCODE_BARRIER].emit = si_llvm_emit_barrier;
4421
4422         bld_base->op_actions[TGSI_OPCODE_MAX].emit = build_tgsi_intrinsic_nomem;
4423         bld_base->op_actions[TGSI_OPCODE_MAX].intr_name = "llvm.maxnum.f32";
4424         bld_base->op_actions[TGSI_OPCODE_MIN].emit = build_tgsi_intrinsic_nomem;
4425         bld_base->op_actions[TGSI_OPCODE_MIN].intr_name = "llvm.minnum.f32";
4426 }
4427
4428 int si_shader_create(struct si_screen *sscreen, LLVMTargetMachineRef tm,
4429                      struct si_shader *shader,
4430                      struct pipe_debug_callback *debug)
4431 {
4432         struct si_shader_selector *sel = shader->selector;
4433         struct tgsi_token *tokens = sel->tokens;
4434         struct si_shader_context ctx;
4435         struct lp_build_tgsi_context *bld_base;
4436         struct tgsi_shader_info stipple_shader_info;
4437         LLVMModuleRef mod;
4438         int r = 0;
4439         bool poly_stipple = sel->type == PIPE_SHADER_FRAGMENT &&
4440                             shader->key.ps.prolog.poly_stipple;
4441
4442         if (poly_stipple) {
4443                 tokens = util_pstipple_create_fragment_shader(tokens, NULL,
4444                                                 SI_POLY_STIPPLE_SAMPLER,
4445                                                 TGSI_FILE_SYSTEM_VALUE);
4446                 tgsi_scan_shader(tokens, &stipple_shader_info);
4447         }
4448
4449         /* Dump TGSI code before doing TGSI->LLVM conversion in case the
4450          * conversion fails. */
4451         if (r600_can_dump_shader(&sscreen->b, sel->info.processor) &&
4452             !(sscreen->b.debug_flags & DBG_NO_TGSI)) {
4453                 si_dump_shader_key(sel->type, &shader->key, stderr);
4454                 tgsi_dump(tokens, 0);
4455                 si_dump_streamout(&sel->so);
4456         }
4457
4458         si_init_shader_ctx(&ctx, sscreen, shader, tm,
4459                            poly_stipple ? &stipple_shader_info : &sel->info);
4460
4461         shader->uses_instanceid = sel->info.uses_instanceid;
4462
4463         bld_base = &ctx.radeon_bld.soa.bld_base;
4464         ctx.radeon_bld.load_system_value = declare_system_value;
4465
4466         switch (ctx.type) {
4467         case TGSI_PROCESSOR_VERTEX:
4468                 ctx.radeon_bld.load_input = declare_input_vs;
4469                 if (shader->key.vs.as_ls)
4470                         bld_base->emit_epilogue = si_llvm_emit_ls_epilogue;
4471                 else if (shader->key.vs.as_es)
4472                         bld_base->emit_epilogue = si_llvm_emit_es_epilogue;
4473                 else
4474                         bld_base->emit_epilogue = si_llvm_emit_vs_epilogue;
4475                 break;
4476         case TGSI_PROCESSOR_TESS_CTRL:
4477                 bld_base->emit_fetch_funcs[TGSI_FILE_INPUT] = fetch_input_tcs;
4478                 bld_base->emit_fetch_funcs[TGSI_FILE_OUTPUT] = fetch_output_tcs;
4479                 bld_base->emit_store = store_output_tcs;
4480                 bld_base->emit_epilogue = si_llvm_emit_tcs_epilogue;
4481                 break;
4482         case TGSI_PROCESSOR_TESS_EVAL:
4483                 bld_base->emit_fetch_funcs[TGSI_FILE_INPUT] = fetch_input_tes;
4484                 if (shader->key.tes.as_es)
4485                         bld_base->emit_epilogue = si_llvm_emit_es_epilogue;
4486                 else
4487                         bld_base->emit_epilogue = si_llvm_emit_vs_epilogue;
4488                 break;
4489         case TGSI_PROCESSOR_GEOMETRY:
4490                 bld_base->emit_fetch_funcs[TGSI_FILE_INPUT] = fetch_input_gs;
4491                 bld_base->emit_epilogue = si_llvm_emit_gs_epilogue;
4492                 break;
4493         case TGSI_PROCESSOR_FRAGMENT:
4494                 ctx.radeon_bld.load_input = declare_input_fs;
4495                 bld_base->emit_epilogue = si_llvm_emit_fs_epilogue;
4496                 break;
4497         default:
4498                 assert(!"Unsupported shader type");
4499                 return -1;
4500         }
4501
4502         create_meta_data(&ctx);
4503         create_function(&ctx);
4504         preload_constants(&ctx);
4505         preload_samplers(&ctx);
4506         preload_streamout_buffers(&ctx);
4507         preload_ring_buffers(&ctx);
4508
4509         if (ctx.type == TGSI_PROCESSOR_GEOMETRY) {
4510                 int i;
4511                 for (i = 0; i < 4; i++) {
4512                         ctx.gs_next_vertex[i] =
4513                                 lp_build_alloca(bld_base->base.gallivm,
4514                                                 ctx.i32, "");
4515                 }
4516         }
4517
4518         if (!lp_build_tgsi_llvm(bld_base, tokens)) {
4519                 fprintf(stderr, "Failed to translate shader from TGSI to LLVM\n");
4520                 goto out;
4521         }
4522
4523         LLVMBuildRet(bld_base->base.gallivm->builder, ctx.return_value);
4524         mod = bld_base->base.gallivm->module;
4525
4526         /* Dump LLVM IR before any optimization passes */
4527         if (sscreen->b.debug_flags & DBG_PREOPT_IR &&
4528             r600_can_dump_shader(&sscreen->b, ctx.type))
4529                 LLVMDumpModule(mod);
4530
4531         radeon_llvm_finalize_module(&ctx.radeon_bld);
4532
4533         r = si_compile_llvm(sscreen, &shader->binary, &shader->config, tm,
4534                             mod, debug, ctx.type, "TGSI shader");
4535         if (r) {
4536                 fprintf(stderr, "LLVM failed to compile shader\n");
4537                 goto out;
4538         }
4539
4540         si_shader_dump(sscreen, shader, debug, ctx.type);
4541
4542         r = si_shader_binary_upload(sscreen, shader);
4543         if (r) {
4544                 fprintf(stderr, "LLVM failed to upload shader\n");
4545                 goto out;
4546         }
4547
4548         radeon_llvm_dispose(&ctx.radeon_bld);
4549
4550         /* Calculate the number of fragment input VGPRs. */
4551         if (ctx.type == TGSI_PROCESSOR_FRAGMENT) {
4552                 shader->num_input_vgprs = 0;
4553                 shader->face_vgpr_index = -1;
4554
4555                 if (G_0286CC_PERSP_SAMPLE_ENA(shader->config.spi_ps_input_addr))
4556                         shader->num_input_vgprs += 2;
4557                 if (G_0286CC_PERSP_CENTER_ENA(shader->config.spi_ps_input_addr))
4558                         shader->num_input_vgprs += 2;
4559                 if (G_0286CC_PERSP_CENTROID_ENA(shader->config.spi_ps_input_addr))
4560                         shader->num_input_vgprs += 2;
4561                 if (G_0286CC_PERSP_PULL_MODEL_ENA(shader->config.spi_ps_input_addr))
4562                         shader->num_input_vgprs += 3;
4563                 if (G_0286CC_LINEAR_SAMPLE_ENA(shader->config.spi_ps_input_addr))
4564                         shader->num_input_vgprs += 2;
4565                 if (G_0286CC_LINEAR_CENTER_ENA(shader->config.spi_ps_input_addr))
4566                         shader->num_input_vgprs += 2;
4567                 if (G_0286CC_LINEAR_CENTROID_ENA(shader->config.spi_ps_input_addr))
4568                         shader->num_input_vgprs += 2;
4569                 if (G_0286CC_LINE_STIPPLE_TEX_ENA(shader->config.spi_ps_input_addr))
4570                         shader->num_input_vgprs += 1;
4571                 if (G_0286CC_POS_X_FLOAT_ENA(shader->config.spi_ps_input_addr))
4572                         shader->num_input_vgprs += 1;
4573                 if (G_0286CC_POS_Y_FLOAT_ENA(shader->config.spi_ps_input_addr))
4574                         shader->num_input_vgprs += 1;
4575                 if (G_0286CC_POS_Z_FLOAT_ENA(shader->config.spi_ps_input_addr))
4576                         shader->num_input_vgprs += 1;
4577                 if (G_0286CC_POS_W_FLOAT_ENA(shader->config.spi_ps_input_addr))
4578                         shader->num_input_vgprs += 1;
4579                 if (G_0286CC_FRONT_FACE_ENA(shader->config.spi_ps_input_addr)) {
4580                         shader->face_vgpr_index = shader->num_input_vgprs;
4581                         shader->num_input_vgprs += 1;
4582                 }
4583                 if (G_0286CC_ANCILLARY_ENA(shader->config.spi_ps_input_addr))
4584                         shader->num_input_vgprs += 1;
4585                 if (G_0286CC_SAMPLE_COVERAGE_ENA(shader->config.spi_ps_input_addr))
4586                         shader->num_input_vgprs += 1;
4587                 if (G_0286CC_POS_FIXED_PT_ENA(shader->config.spi_ps_input_addr))
4588                         shader->num_input_vgprs += 1;
4589         }
4590
4591         if (ctx.type == TGSI_PROCESSOR_GEOMETRY) {
4592                 shader->gs_copy_shader = CALLOC_STRUCT(si_shader);
4593                 shader->gs_copy_shader->selector = shader->selector;
4594                 ctx.shader = shader->gs_copy_shader;
4595                 if ((r = si_generate_gs_copy_shader(sscreen, &ctx,
4596                                                     shader, debug))) {
4597                         free(shader->gs_copy_shader);
4598                         shader->gs_copy_shader = NULL;
4599                         goto out;
4600                 }
4601         }
4602
4603 out:
4604         for (int i = 0; i < SI_NUM_CONST_BUFFERS; i++)
4605                 FREE(ctx.constants[i]);
4606         if (poly_stipple)
4607                 tgsi_free_tokens(tokens);
4608         return r;
4609 }
4610
4611 void si_shader_destroy(struct si_shader *shader)
4612 {
4613         if (shader->gs_copy_shader) {
4614                 si_shader_destroy(shader->gs_copy_shader);
4615                 FREE(shader->gs_copy_shader);
4616         }
4617
4618         if (shader->scratch_bo)
4619                 r600_resource_reference(&shader->scratch_bo, NULL);
4620
4621         r600_resource_reference(&shader->bo, NULL);
4622
4623         radeon_shader_binary_clean(&shader->binary);
4624 }