src/mesa/drivers/dri/i965/brw_fs_generator.cpp

   1 /*
   2  * Copyright © 2010 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 /** @file brw_fs_generator.cpp
  25  *
  26  * This file supports generating code from the FS LIR to the actual
  27  * native instructions.
  28  */
  29
  30 #include "main/macros.h"
  31 #include "brw_context.h"
  32 #include "brw_eu.h"
  33 #include "brw_fs.h"
  34 #include "brw_cfg.h"
  35
  36 static uint32_t brw_file_from_reg(fs_reg *reg)
  37 {
  38    switch (reg->file) {
  39    case GRF:
  40       return BRW_GENERAL_REGISTER_FILE;
  41    case MRF:
  42       return BRW_MESSAGE_REGISTER_FILE;
  43    case IMM:
  44       return BRW_IMMEDIATE_VALUE;
  45    default:
  46       unreachable("not reached");
  47    }
  48 }
  49
  50 static struct brw_reg
  51 brw_reg_from_fs_reg(fs_reg *reg)
  52 {
  53    struct brw_reg brw_reg;
  54
  55    switch (reg->file) {
  56    case GRF:
  57    case MRF:
  58       if (reg->stride == 0) {
  59          brw_reg = brw_vec1_reg(brw_file_from_reg(reg), reg->reg, 0);
  60       } else if (reg->width < 8) {
  61          brw_reg = brw_vec8_reg(brw_file_from_reg(reg), reg->reg, 0);
  62          brw_reg = stride(brw_reg, reg->width * reg->stride,
  63                           reg->width, reg->stride);
  64       } else {
  65          /* From the Haswell PRM:
  66           *
  67           * VertStride must be used to cross GRF register boundaries. This
  68           * rule implies that elements within a 'Width' cannot cross GRF
  69           * boundaries.
  70           *
  71           * So, for registers with width > 8, we have to use a width of 8
  72           * and trust the compression state to sort out the exec size.
  73           */
  74          brw_reg = brw_vec8_reg(brw_file_from_reg(reg), reg->reg, 0);
  75          brw_reg = stride(brw_reg, 8 * reg->stride, 8, reg->stride);
  76       }
  77
  78       brw_reg = retype(brw_reg, reg->type);
  79       brw_reg = byte_offset(brw_reg, reg->subreg_offset);
  80       break;
  81    case IMM:
  82       switch (reg->type) {
  83       case BRW_REGISTER_TYPE_F:
  84          brw_reg = brw_imm_f(reg->fixed_hw_reg.dw1.f);
  85          break;
  86       case BRW_REGISTER_TYPE_D:
  87          brw_reg = brw_imm_d(reg->fixed_hw_reg.dw1.d);
  88          break;
  89       case BRW_REGISTER_TYPE_UD:
  90          brw_reg = brw_imm_ud(reg->fixed_hw_reg.dw1.ud);
  91          break;
  92       case BRW_REGISTER_TYPE_W:
  93          brw_reg = brw_imm_w(reg->fixed_hw_reg.dw1.d);
  94          break;
  95       case BRW_REGISTER_TYPE_UW:
  96          brw_reg = brw_imm_uw(reg->fixed_hw_reg.dw1.ud);
  97          break;
  98       case BRW_REGISTER_TYPE_VF:
  99          brw_reg = brw_imm_vf(reg->fixed_hw_reg.dw1.ud);
 100          break;
 101       default:
 102          unreachable("not reached");
 103       }
 104       break;
 105    case HW_REG:
 106       assert(reg->type == reg->fixed_hw_reg.type);
 107       brw_reg = reg->fixed_hw_reg;
 108       break;
 109    case BAD_FILE:
 110       /* Probably unused. */
 111       brw_reg = brw_null_reg();
 112       break;
 113    default:
 114       unreachable("not reached");
 115    }
 116    if (reg->abs)
 117       brw_reg = brw_abs(brw_reg);
 118    if (reg->negate)
 119       brw_reg = negate(brw_reg);
 120
 121    return brw_reg;
 122 }
 123
 124 fs_generator::fs_generator(struct brw_context *brw,
 125                            void *mem_ctx,
 126                            const void *key,
 127                            struct brw_stage_prog_data *prog_data,
 128                            struct gl_program *prog,
 129                            unsigned promoted_constants,
 130                            bool runtime_check_aads_emit,
 131                            const char *stage_abbrev)
 132
 133    : brw(brw), devinfo(brw->intelScreen->devinfo), key(key),
 134      prog_data(prog_data),
 135      prog(prog), promoted_constants(promoted_constants),
 136      runtime_check_aads_emit(runtime_check_aads_emit), debug_flag(false),
 137      stage_abbrev(stage_abbrev), mem_ctx(mem_ctx)
 138 {
 139    p = rzalloc(mem_ctx, struct brw_codegen);
 140    brw_init_codegen(devinfo, p, mem_ctx);
 141 }
 142
 143 fs_generator::~fs_generator()
 144 {
 145 }
 146
 147 class ip_record : public exec_node {
 148 public:
 149    DECLARE_RALLOC_CXX_OPERATORS(ip_record)
 150
 151    ip_record(int ip)
 152    {
 153       this->ip = ip;
 154    }
 155
 156    int ip;
 157 };
 158
 159 bool
 160 fs_generator::patch_discard_jumps_to_fb_writes()
 161 {
 162    if (devinfo->gen < 6 || this->discard_halt_patches.is_empty())
 163       return false;
 164
 165    int scale = brw_jump_scale(p->devinfo);
 166
 167    /* There is a somewhat strange undocumented requirement of using
 168     * HALT, according to the simulator.  If some channel has HALTed to
 169     * a particular UIP, then by the end of the program, every channel
 170     * must have HALTed to that UIP.  Furthermore, the tracking is a
 171     * stack, so you can't do the final halt of a UIP after starting
 172     * halting to a new UIP.
 173     *
 174     * Symptoms of not emitting this instruction on actual hardware
 175     * included GPU hangs and sparkly rendering on the piglit discard
 176     * tests.
 177     */
 178    brw_inst *last_halt = gen6_HALT(p);
 179    brw_inst_set_uip(p->devinfo, last_halt, 1 * scale);
 180    brw_inst_set_jip(p->devinfo, last_halt, 1 * scale);
 181
 182    int ip = p->nr_insn;
 183
 184    foreach_in_list(ip_record, patch_ip, &discard_halt_patches) {
 185       brw_inst *patch = &p->store[patch_ip->ip];
 186
 187       assert(brw_inst_opcode(p->devinfo, patch) == BRW_OPCODE_HALT);
 188       /* HALT takes a half-instruction distance from the pre-incremented IP. */
 189       brw_inst_set_uip(p->devinfo, patch, (ip - patch_ip->ip) * scale);
 190    }
 191
 192    this->discard_halt_patches.make_empty();
 193    return true;
 194 }
 195
 196 void
 197 fs_generator::fire_fb_write(fs_inst *inst,
 198                             struct brw_reg payload,
 199                             struct brw_reg implied_header,
 200                             GLuint nr)
 201 {
 202    uint32_t msg_control;
 203
 204    brw_wm_prog_data *prog_data = (brw_wm_prog_data*) this->prog_data;
 205
 206    if (devinfo->gen < 6) {
 207       brw_push_insn_state(p);
 208       brw_set_default_exec_size(p, BRW_EXECUTE_8);
 209       brw_set_default_mask_control(p, BRW_MASK_DISABLE);
 210       brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
 211       brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
 212       brw_MOV(p, offset(payload, 1), brw_vec8_grf(1, 0));
 213       brw_pop_insn_state(p);
 214    }
 215
 216    if (inst->opcode == FS_OPCODE_REP_FB_WRITE)
 217       msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD16_SINGLE_SOURCE_REPLICATED;
 218    else if (prog_data->dual_src_blend) {
 219       if (dispatch_width == 8 || !inst->eot)
 220          msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD8_DUAL_SOURCE_SUBSPAN01;
 221       else
 222          msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD8_DUAL_SOURCE_SUBSPAN23;
 223    } else if (dispatch_width == 16)
 224       msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD16_SINGLE_SOURCE;
 225    else
 226       msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD8_SINGLE_SOURCE_SUBSPAN01;
 227
 228    uint32_t surf_index =
 229       prog_data->binding_table.render_target_start + inst->target;
 230
 231    bool last_render_target = inst->eot ||
 232                              (prog_data->dual_src_blend && dispatch_width == 16);
 233
 234
 235    brw_fb_WRITE(p,
 236                 dispatch_width,
 237                 payload,
 238                 implied_header,
 239                 msg_control,
 240                 surf_index,
 241                 nr,
 242                 0,
 243                 inst->eot,
 244                 last_render_target,
 245                 inst->header_present);
 246
 247    brw_mark_surface_used(&prog_data->base, surf_index);
 248 }
 249
 250 void
 251 fs_generator::generate_fb_write(fs_inst *inst, struct brw_reg payload)
 252 {
 253    brw_wm_prog_data *prog_data = (brw_wm_prog_data*) this->prog_data;
 254    const brw_wm_prog_key * const key = (brw_wm_prog_key * const) this->key;
 255    struct brw_reg implied_header;
 256
 257    if (devinfo->gen < 8 && !devinfo->is_haswell) {
 258       brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
 259    }
 260
 261    if (inst->base_mrf >= 0)
 262       payload = brw_message_reg(inst->base_mrf);
 263
 264    /* Header is 2 regs, g0 and g1 are the contents. g0 will be implied
 265     * move, here's g1.
 266     */
 267    if (inst->header_present) {
 268       brw_push_insn_state(p);
 269       brw_set_default_mask_control(p, BRW_MASK_DISABLE);
 270       brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
 271       brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
 272       brw_set_default_flag_reg(p, 0, 0);
 273
 274       /* On HSW, the GPU will use the predicate on SENDC, unless the header is
 275        * present.
 276        */
 277       if (prog_data->uses_kill) {
 278          struct brw_reg pixel_mask;
 279
 280          if (devinfo->gen >= 6)
 281             pixel_mask = retype(brw_vec1_grf(1, 7), BRW_REGISTER_TYPE_UW);
 282          else
 283             pixel_mask = retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_UW);
 284
 285          brw_MOV(p, pixel_mask, brw_flag_reg(0, 1));
 286       }
 287
 288       if (devinfo->gen >= 6) {
 289          brw_push_insn_state(p);
 290          brw_set_default_exec_size(p, BRW_EXECUTE_16);
 291          brw_set_default_compression_control(p, BRW_COMPRESSION_COMPRESSED);
 292          brw_MOV(p,
 293                  retype(payload, BRW_REGISTER_TYPE_UD),
 294                  retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD));
 295          brw_pop_insn_state(p);
 296
 297          if (inst->target > 0 && key->replicate_alpha) {
 298             /* Set "Source0 Alpha Present to RenderTarget" bit in message
 299              * header.
 300              */
 301             brw_OR(p,
 302                    vec1(retype(payload, BRW_REGISTER_TYPE_UD)),
 303                    vec1(retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD)),
 304                    brw_imm_ud(0x1 << 11));
 305          }
 306
 307          if (inst->target > 0) {
 308             /* Set the render target index for choosing BLEND_STATE. */
 309             brw_MOV(p, retype(vec1(suboffset(payload, 2)),
 310                               BRW_REGISTER_TYPE_UD),
 311                     brw_imm_ud(inst->target));
 312          }
 313
 314          implied_header = brw_null_reg();
 315       } else {
 316          implied_header = retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UW);
 317       }
 318
 319       brw_pop_insn_state(p);
 320    } else {
 321       implied_header = brw_null_reg();
 322    }
 323
 324    if (!runtime_check_aads_emit) {
 325       fire_fb_write(inst, payload, implied_header, inst->mlen);
 326    } else {
 327       /* This can only happen in gen < 6 */
 328       assert(devinfo->gen < 6);
 329
 330       struct brw_reg v1_null_ud = vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_UD));
 331
 332       /* Check runtime bit to detect if we have to send AA data or not */
 333       brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
 334       brw_AND(p,
 335               v1_null_ud,
 336               retype(brw_vec1_grf(1, 6), BRW_REGISTER_TYPE_UD),
 337               brw_imm_ud(1<<26));
 338       brw_inst_set_cond_modifier(p->devinfo, brw_last_inst, BRW_CONDITIONAL_NZ);
 339
 340       int jmp = brw_JMPI(p, brw_imm_ud(0), BRW_PREDICATE_NORMAL) - p->store;
 341       brw_inst_set_exec_size(p->devinfo, brw_last_inst, BRW_EXECUTE_1);
 342       {
 343          /* Don't send AA data */
 344          fire_fb_write(inst, offset(payload, 1), implied_header, inst->mlen-1);
 345       }
 346       brw_land_fwd_jump(p, jmp);
 347       fire_fb_write(inst, payload, implied_header, inst->mlen);
 348    }
 349 }
 350
 351 void
 352 fs_generator::generate_urb_write(fs_inst *inst, struct brw_reg payload)
 353 {
 354    brw_inst *insn;
 355
 356    insn = brw_next_insn(p, BRW_OPCODE_SEND);
 357
 358    brw_set_dest(p, insn, brw_null_reg());
 359    brw_set_src0(p, insn, payload);
 360    brw_set_src1(p, insn, brw_imm_d(0));
 361
 362    brw_inst_set_sfid(p->devinfo, insn, BRW_SFID_URB);
 363    brw_inst_set_urb_opcode(p->devinfo, insn, GEN8_URB_OPCODE_SIMD8_WRITE);
 364
 365    brw_inst_set_mlen(p->devinfo, insn, inst->mlen);
 366    brw_inst_set_rlen(p->devinfo, insn, 0);
 367    brw_inst_set_eot(p->devinfo, insn, inst->eot);
 368    brw_inst_set_header_present(p->devinfo, insn, true);
 369    brw_inst_set_urb_global_offset(p->devinfo, insn, inst->offset);
 370 }
 371
 372 void
 373 fs_generator::generate_blorp_fb_write(fs_inst *inst)
 374 {
 375    brw_fb_WRITE(p,
 376                 16 /* dispatch_width */,
 377                 brw_message_reg(inst->base_mrf),
 378                 brw_reg_from_fs_reg(&inst->src[0]),
 379                 BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD16_SINGLE_SOURCE,
 380                 inst->target,
 381                 inst->mlen,
 382                 0,
 383                 true,
 384                 true,
 385                 inst->header_present);
 386 }
 387
 388 void
 389 fs_generator::generate_linterp(fs_inst *inst,
 390                              struct brw_reg dst, struct brw_reg *src)
 391 {
 392    /* PLN reads:
 393     *                      /   in SIMD16   \
 394     *    -----------------------------------
 395     *   | src1+0 | src1+1 | src1+2 | src1+3 |
 396     *   |-----------------------------------|
 397     *   |(x0, x1)|(y0, y1)|(x2, x3)|(y2, y3)|
 398     *    -----------------------------------
 399     *
 400     * but for the LINE/MAC pair, the LINE reads Xs and the MAC reads Ys:
 401     *
 402     *    -----------------------------------
 403     *   | src1+0 | src1+1 | src1+2 | src1+3 |
 404     *   |-----------------------------------|
 405     *   |(x0, x1)|(y0, y1)|        |        | in SIMD8
 406     *   |-----------------------------------|
 407     *   |(x0, x1)|(x2, x3)|(y0, y1)|(y2, y3)| in SIMD16
 408     *    -----------------------------------
 409     *
 410     * See also: emit_interpolation_setup_gen4().
 411     */
 412    struct brw_reg delta_x = src[0];
 413    struct brw_reg delta_y = offset(src[0], dispatch_width / 8);
 414    struct brw_reg interp = src[1];
 415
 416    if (devinfo->has_pln &&
 417        (devinfo->gen >= 7 || (delta_x.nr & 1) == 0)) {
 418       brw_PLN(p, dst, interp, delta_x);
 419    } else {
 420       brw_LINE(p, brw_null_reg(), interp, delta_x);
 421       brw_MAC(p, dst, suboffset(interp, 1), delta_y);
 422    }
 423 }
 424
 425 void
 426 fs_generator::generate_math_gen6(fs_inst *inst,
 427                                  struct brw_reg dst,
 428                                  struct brw_reg src0,
 429                                  struct brw_reg src1)
 430 {
 431    int op = brw_math_function(inst->opcode);
 432    bool binop = src1.file != BRW_ARCHITECTURE_REGISTER_FILE;
 433
 434    if (dispatch_width == 8) {
 435       gen6_math(p, dst, op, src0, src1);
 436    } else if (dispatch_width == 16) {
 437       brw_push_insn_state(p);
 438       brw_set_default_exec_size(p, BRW_EXECUTE_8);
 439       brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
 440       gen6_math(p, firsthalf(dst), op, firsthalf(src0), firsthalf(src1));
 441       brw_set_default_compression_control(p, BRW_COMPRESSION_2NDHALF);
 442       gen6_math(p, sechalf(dst), op, sechalf(src0),
 443                 binop ? sechalf(src1) : brw_null_reg());
 444       brw_pop_insn_state(p);
 445    }
 446 }
 447
 448 void
 449 fs_generator::generate_math_gen4(fs_inst *inst,
 450                                struct brw_reg dst,
 451                                struct brw_reg src)
 452 {
 453    int op = brw_math_function(inst->opcode);
 454
 455    assert(inst->mlen >= 1);
 456
 457    if (dispatch_width == 8) {
 458       gen4_math(p, dst,
 459                 op,
 460                 inst->base_mrf, src,
 461                 BRW_MATH_PRECISION_FULL);
 462    } else if (dispatch_width == 16) {
 463       brw_set_default_exec_size(p, BRW_EXECUTE_8);
 464       brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
 465       gen4_math(p, firsthalf(dst),
 466                 op,
 467                 inst->base_mrf, firsthalf(src),
 468                 BRW_MATH_PRECISION_FULL);
 469       brw_set_default_compression_control(p, BRW_COMPRESSION_2NDHALF);
 470       gen4_math(p, sechalf(dst),
 471                 op,
 472                 inst->base_mrf + 1, sechalf(src),
 473                 BRW_MATH_PRECISION_FULL);
 474
 475       brw_set_default_compression_control(p, BRW_COMPRESSION_COMPRESSED);
 476    }
 477 }
 478
 479 void
 480 fs_generator::generate_math_g45(fs_inst *inst,
 481                                 struct brw_reg dst,
 482                                 struct brw_reg src)
 483 {
 484    if (inst->opcode == SHADER_OPCODE_POW ||
 485        inst->opcode == SHADER_OPCODE_INT_QUOTIENT ||
 486        inst->opcode == SHADER_OPCODE_INT_REMAINDER) {
 487       generate_math_gen4(inst, dst, src);
 488       return;
 489    }
 490
 491    int op = brw_math_function(inst->opcode);
 492
 493    assert(inst->mlen >= 1);
 494
 495    gen4_math(p, dst,
 496              op,
 497              inst->base_mrf, src,
 498              BRW_MATH_PRECISION_FULL);
 499 }
 500
 501 void
 502 fs_generator::generate_tex(fs_inst *inst, struct brw_reg dst, struct brw_reg src,
 503                            struct brw_reg sampler_index)
 504 {
 505    int msg_type = -1;
 506    int rlen = 4;
 507    uint32_t simd_mode;
 508    uint32_t return_format;
 509    bool is_combined_send = inst->eot;
 510
 511    switch (dst.type) {
 512    case BRW_REGISTER_TYPE_D:
 513       return_format = BRW_SAMPLER_RETURN_FORMAT_SINT32;
 514       break;
 515    case BRW_REGISTER_TYPE_UD:
 516       return_format = BRW_SAMPLER_RETURN_FORMAT_UINT32;
 517       break;
 518    default:
 519       return_format = BRW_SAMPLER_RETURN_FORMAT_FLOAT32;
 520       break;
 521    }
 522
 523    switch (inst->exec_size) {
 524    case 8:
 525       simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD8;
 526       break;
 527    case 16:
 528       simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
 529       break;
 530    default:
 531       unreachable("Invalid width for texture instruction");
 532    }
 533
 534    if (devinfo->gen >= 5) {
 535       switch (inst->opcode) {
 536       case SHADER_OPCODE_TEX:
 537          if (inst->shadow_compare) {
 538             msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_COMPARE;
 539          } else {
 540             msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE;
 541          }
 542          break;
 543       case FS_OPCODE_TXB:
 544          if (inst->shadow_compare) {
 545             msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_BIAS_COMPARE;
 546          } else {
 547             msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_BIAS;
 548          }
 549          break;
 550       case SHADER_OPCODE_TXL:
 551          if (inst->shadow_compare) {
 552             msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LOD_COMPARE;
 553          } else {
 554             msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LOD;
 555          }
 556          break;
 557       case SHADER_OPCODE_TXS:
 558          msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_RESINFO;
 559          break;
 560       case SHADER_OPCODE_TXD:
 561          if (inst->shadow_compare) {
 562             /* Gen7.5+.  Otherwise, lowered by brw_lower_texture_gradients(). */
 563             assert(devinfo->gen >= 8 || devinfo->is_haswell);
 564             msg_type = HSW_SAMPLER_MESSAGE_SAMPLE_DERIV_COMPARE;
 565          } else {
 566             msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_DERIVS;
 567          }
 568          break;
 569       case SHADER_OPCODE_TXF:
 570          msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LD;
 571          break;
 572       case SHADER_OPCODE_TXF_CMS:
 573          if (devinfo->gen >= 7)
 574             msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_LD2DMS;
 575          else
 576             msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LD;
 577          break;
 578       case SHADER_OPCODE_TXF_UMS:
 579          assert(devinfo->gen >= 7);
 580          msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_LD2DSS;
 581          break;
 582       case SHADER_OPCODE_TXF_MCS:
 583          assert(devinfo->gen >= 7);
 584          msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_LD_MCS;
 585          break;
 586       case SHADER_OPCODE_LOD:
 587          msg_type = GEN5_SAMPLER_MESSAGE_LOD;
 588          break;
 589       case SHADER_OPCODE_TG4:
 590          if (inst->shadow_compare) {
 591             assert(devinfo->gen >= 7);
 592             msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_GATHER4_C;
 593          } else {
 594             assert(devinfo->gen >= 6);
 595             msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_GATHER4;
 596          }
 597          break;
 598       case SHADER_OPCODE_TG4_OFFSET:
 599          assert(devinfo->gen >= 7);
 600          if (inst->shadow_compare) {
 601             msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_GATHER4_PO_C;
 602          } else {
 603             msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_GATHER4_PO;
 604          }
 605          break;
 606       default:
 607          unreachable("not reached");
 608       }
 609    } else {
 610       switch (inst->opcode) {
 611       case SHADER_OPCODE_TEX:
 612          /* Note that G45 and older determines shadow compare and dispatch width
 613           * from message length for most messages.
 614           */
 615          if (dispatch_width == 8) {
 616             msg_type = BRW_SAMPLER_MESSAGE_SIMD8_SAMPLE;
 617             if (inst->shadow_compare) {
 618                assert(inst->mlen == 6);
 619             } else {
 620                assert(inst->mlen <= 4);
 621             }
 622          } else {
 623             if (inst->shadow_compare) {
 624                msg_type = BRW_SAMPLER_MESSAGE_SIMD16_SAMPLE_COMPARE;
 625                assert(inst->mlen == 9);
 626             } else {
 627                msg_type = BRW_SAMPLER_MESSAGE_SIMD16_SAMPLE;
 628                assert(inst->mlen <= 7 && inst->mlen % 2 == 1);
 629             }
 630          }
 631          break;
 632       case FS_OPCODE_TXB:
 633          if (inst->shadow_compare) {
 634             assert(dispatch_width == 8);
 635             assert(inst->mlen == 6);
 636             msg_type = BRW_SAMPLER_MESSAGE_SIMD8_SAMPLE_BIAS_COMPARE;
 637          } else {
 638             assert(inst->mlen == 9);
 639             msg_type = BRW_SAMPLER_MESSAGE_SIMD16_SAMPLE_BIAS;
 640             simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
 641          }
 642          break;
 643       case SHADER_OPCODE_TXL:
 644          if (inst->shadow_compare) {
 645             assert(dispatch_width == 8);
 646             assert(inst->mlen == 6);
 647             msg_type = BRW_SAMPLER_MESSAGE_SIMD8_SAMPLE_LOD_COMPARE;
 648          } else {
 649             assert(inst->mlen == 9);
 650             msg_type = BRW_SAMPLER_MESSAGE_SIMD16_SAMPLE_LOD;
 651             simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
 652          }
 653          break;
 654       case SHADER_OPCODE_TXD:
 655          /* There is no sample_d_c message; comparisons are done manually */
 656          assert(dispatch_width == 8);
 657          assert(inst->mlen == 7 || inst->mlen == 10);
 658          msg_type = BRW_SAMPLER_MESSAGE_SIMD8_SAMPLE_GRADIENTS;
 659          break;
 660       case SHADER_OPCODE_TXF:
 661          assert(inst->mlen <= 9 && inst->mlen % 2 == 1);
 662          msg_type = BRW_SAMPLER_MESSAGE_SIMD16_LD;
 663          simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
 664          break;
 665       case SHADER_OPCODE_TXS:
 666          assert(inst->mlen == 3);
 667          msg_type = BRW_SAMPLER_MESSAGE_SIMD16_RESINFO;
 668          simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
 669          break;
 670       default:
 671          unreachable("not reached");
 672       }
 673    }
 674    assert(msg_type != -1);
 675
 676    if (simd_mode == BRW_SAMPLER_SIMD_MODE_SIMD16) {
 677       rlen = 8;
 678       dst = vec16(dst);
 679    }
 680
 681    if (is_combined_send) {
 682       assert(devinfo->gen >= 9 || devinfo->is_cherryview);
 683       rlen = 0;
 684    }
 685
 686    assert(devinfo->gen < 7 || !inst->header_present ||
 687           src.file == BRW_GENERAL_REGISTER_FILE);
 688
 689    assert(sampler_index.type == BRW_REGISTER_TYPE_UD);
 690
 691    /* Load the message header if present.  If there's a texture offset,
 692     * we need to set it up explicitly and load the offset bitfield.
 693     * Otherwise, we can use an implied move from g0 to the first message reg.
 694     */
 695    if (inst->header_present) {
 696       if (devinfo->gen < 6 && !inst->offset) {
 697          /* Set up an implied move from g0 to the MRF. */
 698          src = retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UW);
 699       } else {
 700          struct brw_reg header_reg;
 701
 702          if (devinfo->gen >= 7) {
 703             header_reg = src;
 704          } else {
 705             assert(inst->base_mrf != -1);
 706             header_reg = brw_message_reg(inst->base_mrf);
 707          }
 708
 709          brw_push_insn_state(p);
 710          brw_set_default_exec_size(p, BRW_EXECUTE_8);
 711          brw_set_default_mask_control(p, BRW_MASK_DISABLE);
 712          brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
 713          /* Explicitly set up the message header by copying g0 to the MRF. */
 714          brw_MOV(p, header_reg, brw_vec8_grf(0, 0));
 715
 716          if (inst->offset) {
 717             /* Set the offset bits in DWord 2. */
 718             brw_MOV(p, get_element_ud(header_reg, 2),
 719                        brw_imm_ud(inst->offset));
 720          }
 721
 722          brw_adjust_sampler_state_pointer(p, header_reg, sampler_index);
 723          brw_pop_insn_state(p);
 724       }
 725    }
 726
 727    uint32_t base_binding_table_index = (inst->opcode == SHADER_OPCODE_TG4 ||
 728          inst->opcode == SHADER_OPCODE_TG4_OFFSET)
 729          ? prog_data->binding_table.gather_texture_start
 730          : prog_data->binding_table.texture_start;
 731
 732    if (sampler_index.file == BRW_IMMEDIATE_VALUE) {
 733       uint32_t sampler = sampler_index.dw1.ud;
 734
 735       brw_SAMPLE(p,
 736                  retype(dst, BRW_REGISTER_TYPE_UW),
 737                  inst->base_mrf,
 738                  src,
 739                  sampler + base_binding_table_index,
 740                  sampler % 16,
 741                  msg_type,
 742                  rlen,
 743                  inst->mlen,
 744                  inst->header_present,
 745                  simd_mode,
 746                  return_format);
 747
 748       brw_mark_surface_used(prog_data, sampler + base_binding_table_index);
 749    } else {
 750       /* Non-const sampler index */
 751       /* Note: this clobbers `dst` as a temporary before emitting the send */
 752
 753       struct brw_reg addr = vec1(retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD));
 754       struct brw_reg temp = vec1(retype(dst, BRW_REGISTER_TYPE_UD));
 755
 756       struct brw_reg sampler_reg = vec1(retype(sampler_index, BRW_REGISTER_TYPE_UD));
 757
 758       brw_push_insn_state(p);
 759       brw_set_default_mask_control(p, BRW_MASK_DISABLE);
 760       brw_set_default_access_mode(p, BRW_ALIGN_1);
 761
 762       /* Some care required: `sampler` and `temp` may alias:
 763        *    addr = sampler & 0xff
 764        *    temp = (sampler << 8) & 0xf00
 765        *    addr = addr | temp
 766        */
 767       brw_ADD(p, addr, sampler_reg, brw_imm_ud(base_binding_table_index));
 768       brw_SHL(p, temp, sampler_reg, brw_imm_ud(8u));
 769       brw_AND(p, temp, temp, brw_imm_ud(0x0f00));
 770       brw_AND(p, addr, addr, brw_imm_ud(0x0ff));
 771       brw_OR(p, addr, addr, temp);
 772
 773       brw_pop_insn_state(p);
 774
 775       /* dst = send(offset, a0.0 | <descriptor>) */
 776       brw_inst *insn = brw_send_indirect_message(
 777          p, BRW_SFID_SAMPLER, dst, src, addr);
 778       brw_set_sampler_message(p, insn,
 779                               0 /* surface */,
 780                               0 /* sampler */,
 781                               msg_type,
 782                               rlen,
 783                               inst->mlen /* mlen */,
 784                               inst->header_present /* header */,
 785                               simd_mode,
 786                               return_format);
 787
 788       /* visitor knows more than we do about the surface limit required,
 789        * so has already done marking.
 790        */
 791    }
 792
 793    if (is_combined_send) {
 794       brw_inst_set_eot(p->devinfo, brw_last_inst, true);
 795       brw_inst_set_opcode(p->devinfo, brw_last_inst, BRW_OPCODE_SENDC);
 796    }
 797 }
 798
 799
 800 /* For OPCODE_DDX and OPCODE_DDY, per channel of output we've got input
 801  * looking like:
 802  *
 803  * arg0: ss0.tl ss0.tr ss0.bl ss0.br ss1.tl ss1.tr ss1.bl ss1.br
 804  *
 805  * Ideally, we want to produce:
 806  *
 807  *           DDX                     DDY
 808  * dst: (ss0.tr - ss0.tl)     (ss0.tl - ss0.bl)
 809  *      (ss0.tr - ss0.tl)     (ss0.tr - ss0.br)
 810  *      (ss0.br - ss0.bl)     (ss0.tl - ss0.bl)
 811  *      (ss0.br - ss0.bl)     (ss0.tr - ss0.br)
 812  *      (ss1.tr - ss1.tl)     (ss1.tl - ss1.bl)
 813  *      (ss1.tr - ss1.tl)     (ss1.tr - ss1.br)
 814  *      (ss1.br - ss1.bl)     (ss1.tl - ss1.bl)
 815  *      (ss1.br - ss1.bl)     (ss1.tr - ss1.br)
 816  *
 817  * and add another set of two more subspans if in 16-pixel dispatch mode.
 818  *
 819  * For DDX, it ends up being easy: width = 2, horiz=0 gets us the same result
 820  * for each pair, and vertstride = 2 jumps us 2 elements after processing a
 821  * pair.  But the ideal approximation may impose a huge performance cost on
 822  * sample_d.  On at least Haswell, sample_d instruction does some
 823  * optimizations if the same LOD is used for all pixels in the subspan.
 824  *
 825  * For DDY, we need to use ALIGN16 mode since it's capable of doing the
 826  * appropriate swizzling.
 827  */
 828 void
 829 fs_generator::generate_ddx(enum opcode opcode,
 830                            struct brw_reg dst, struct brw_reg src)
 831 {
 832    unsigned vstride, width;
 833
 834    if (opcode == FS_OPCODE_DDX_FINE) {
 835       /* produce accurate derivatives */
 836       vstride = BRW_VERTICAL_STRIDE_2;
 837       width = BRW_WIDTH_2;
 838    } else {
 839       /* replicate the derivative at the top-left pixel to other pixels */
 840       vstride = BRW_VERTICAL_STRIDE_4;
 841       width = BRW_WIDTH_4;
 842    }
 843
 844    struct brw_reg src0 = brw_reg(src.file, src.nr, 1,
 845                                  src.negate, src.abs,
 846                                  BRW_REGISTER_TYPE_F,
 847                                  vstride,
 848                                  width,
 849                                  BRW_HORIZONTAL_STRIDE_0,
 850                                  BRW_SWIZZLE_XYZW, WRITEMASK_XYZW);
 851    struct brw_reg src1 = brw_reg(src.file, src.nr, 0,
 852                                  src.negate, src.abs,
 853                                  BRW_REGISTER_TYPE_F,
 854                                  vstride,
 855                                  width,
 856                                  BRW_HORIZONTAL_STRIDE_0,
 857                                  BRW_SWIZZLE_XYZW, WRITEMASK_XYZW);
 858    brw_ADD(p, dst, src0, negate(src1));
 859 }
 860
 861 /* The negate_value boolean is used to negate the derivative computation for
 862  * FBOs, since they place the origin at the upper left instead of the lower
 863  * left.
 864  */
 865 void
 866 fs_generator::generate_ddy(enum opcode opcode,
 867                            struct brw_reg dst, struct brw_reg src,
 868                            bool negate_value)
 869 {
 870    if (opcode == FS_OPCODE_DDY_FINE) {
 871       /* From the Ivy Bridge PRM, volume 4 part 3, section 3.3.9 (Register
 872        * Region Restrictions):
 873        *
 874        *     In Align16 access mode, SIMD16 is not allowed for DW operations
 875        *     and SIMD8 is not allowed for DF operations.
 876        *
 877        * In this context, "DW operations" means "operations acting on 32-bit
 878        * values", so it includes operations on floats.
 879        *
 880        * Gen4 has a similar restriction.  From the i965 PRM, section 11.5.3
 881        * (Instruction Compression -> Rules and Restrictions):
 882        *
 883        *     A compressed instruction must be in Align1 access mode. Align16
 884        *     mode instructions cannot be compressed.
 885        *
 886        * Similar text exists in the g45 PRM.
 887        *
 888        * On these platforms, if we're building a SIMD16 shader, we need to
 889        * manually unroll to a pair of SIMD8 instructions.
 890        */
 891       bool unroll_to_simd8 =
 892          (dispatch_width == 16 &&
 893           (devinfo->gen == 4 || (devinfo->gen == 7 && !devinfo->is_haswell)));
 894
 895       /* produce accurate derivatives */
 896       struct brw_reg src0 = brw_reg(src.file, src.nr, 0,
 897                                     src.negate, src.abs,
 898                                     BRW_REGISTER_TYPE_F,
 899                                     BRW_VERTICAL_STRIDE_4,
 900                                     BRW_WIDTH_4,
 901                                     BRW_HORIZONTAL_STRIDE_1,
 902                                     BRW_SWIZZLE_XYXY, WRITEMASK_XYZW);
 903       struct brw_reg src1 = brw_reg(src.file, src.nr, 0,
 904                                     src.negate, src.abs,
 905                                     BRW_REGISTER_TYPE_F,
 906                                     BRW_VERTICAL_STRIDE_4,
 907                                     BRW_WIDTH_4,
 908                                     BRW_HORIZONTAL_STRIDE_1,
 909                                     BRW_SWIZZLE_ZWZW, WRITEMASK_XYZW);
 910       brw_push_insn_state(p);
 911       brw_set_default_access_mode(p, BRW_ALIGN_16);
 912       if (unroll_to_simd8) {
 913          brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
 914          if (negate_value) {
 915             brw_ADD(p, firsthalf(dst), firsthalf(src1), negate(firsthalf(src0)));
 916             brw_set_default_compression_control(p, BRW_COMPRESSION_2NDHALF);
 917             brw_ADD(p, sechalf(dst), sechalf(src1), negate(sechalf(src0)));
 918          } else {
 919             brw_ADD(p, firsthalf(dst), firsthalf(src0), negate(firsthalf(src1)));
 920             brw_set_default_compression_control(p, BRW_COMPRESSION_2NDHALF);
 921             brw_ADD(p, sechalf(dst), sechalf(src0), negate(sechalf(src1)));
 922          }
 923       } else {
 924          if (negate_value)
 925             brw_ADD(p, dst, src1, negate(src0));
 926          else
 927             brw_ADD(p, dst, src0, negate(src1));
 928       }
 929       brw_pop_insn_state(p);
 930    } else {
 931       /* replicate the derivative at the top-left pixel to other pixels */
 932       struct brw_reg src0 = brw_reg(src.file, src.nr, 0,
 933                                     src.negate, src.abs,
 934                                     BRW_REGISTER_TYPE_F,
 935                                     BRW_VERTICAL_STRIDE_4,
 936                                     BRW_WIDTH_4,
 937                                     BRW_HORIZONTAL_STRIDE_0,
 938                                     BRW_SWIZZLE_XYZW, WRITEMASK_XYZW);
 939       struct brw_reg src1 = brw_reg(src.file, src.nr, 2,
 940                                     src.negate, src.abs,
 941                                     BRW_REGISTER_TYPE_F,
 942                                     BRW_VERTICAL_STRIDE_4,
 943                                     BRW_WIDTH_4,
 944                                     BRW_HORIZONTAL_STRIDE_0,
 945                                     BRW_SWIZZLE_XYZW, WRITEMASK_XYZW);
 946       if (negate_value)
 947          brw_ADD(p, dst, src1, negate(src0));
 948       else
 949          brw_ADD(p, dst, src0, negate(src1));
 950    }
 951 }
 952
 953 void
 954 fs_generator::generate_discard_jump(fs_inst *inst)
 955 {
 956    assert(devinfo->gen >= 6);
 957
 958    /* This HALT will be patched up at FB write time to point UIP at the end of
 959     * the program, and at brw_uip_jip() JIP will be set to the end of the
 960     * current block (or the program).
 961     */
 962    this->discard_halt_patches.push_tail(new(mem_ctx) ip_record(p->nr_insn));
 963
 964    brw_push_insn_state(p);
 965    brw_set_default_mask_control(p, BRW_MASK_DISABLE);
 966    gen6_HALT(p);
 967    brw_pop_insn_state(p);
 968 }
 969
 970 void
 971 fs_generator::generate_scratch_write(fs_inst *inst, struct brw_reg src)
 972 {
 973    assert(inst->mlen != 0);
 974
 975    brw_MOV(p,
 976            brw_uvec_mrf(inst->exec_size, (inst->base_mrf + 1), 0),
 977            retype(src, BRW_REGISTER_TYPE_UD));
 978    brw_oword_block_write_scratch(p, brw_message_reg(inst->base_mrf),
 979                                  inst->exec_size / 8, inst->offset);
 980 }
 981
 982 void
 983 fs_generator::generate_scratch_read(fs_inst *inst, struct brw_reg dst)
 984 {
 985    assert(inst->mlen != 0);
 986
 987    brw_oword_block_read_scratch(p, dst, brw_message_reg(inst->base_mrf),
 988                                 inst->exec_size / 8, inst->offset);
 989 }
 990
 991 void
 992 fs_generator::generate_scratch_read_gen7(fs_inst *inst, struct brw_reg dst)
 993 {
 994    gen7_block_read_scratch(p, dst, inst->exec_size / 8, inst->offset);
 995 }
 996
 997 void
 998 fs_generator::generate_uniform_pull_constant_load(fs_inst *inst,
 999                                                   struct brw_reg dst,
1000                                                   struct brw_reg index,
1001                                                   struct brw_reg offset)
1002 {
1003    assert(inst->mlen != 0);
1004
1005    assert(index.file == BRW_IMMEDIATE_VALUE &&
1006           index.type == BRW_REGISTER_TYPE_UD);
1007    uint32_t surf_index = index.dw1.ud;
1008
1009    assert(offset.file == BRW_IMMEDIATE_VALUE &&
1010           offset.type == BRW_REGISTER_TYPE_UD);
1011    uint32_t read_offset = offset.dw1.ud;
1012
1013    brw_oword_block_read(p, dst, brw_message_reg(inst->base_mrf),
1014                         read_offset, surf_index);
1015
1016    brw_mark_surface_used(prog_data, surf_index);
1017 }
1018
1019 void
1020 fs_generator::generate_uniform_pull_constant_load_gen7(fs_inst *inst,
1021                                                        struct brw_reg dst,
1022                                                        struct brw_reg index,
1023                                                        struct brw_reg offset)
1024 {
1025    assert(inst->mlen == 0);
1026    assert(index.type == BRW_REGISTER_TYPE_UD);
1027
1028    assert(offset.file == BRW_GENERAL_REGISTER_FILE);
1029    /* Reference just the dword we need, to avoid angering validate_reg(). */
1030    offset = brw_vec1_grf(offset.nr, 0);
1031
1032    /* We use the SIMD4x2 mode because we want to end up with 4 components in
1033     * the destination loaded consecutively from the same offset (which appears
1034     * in the first component, and the rest are ignored).
1035     */
1036    dst.width = BRW_WIDTH_4;
1037
1038    struct brw_reg src = offset;
1039    bool header_present = false;
1040    int mlen = 1;
1041
1042    if (devinfo->gen >= 9) {
1043       /* Skylake requires a message header in order to use SIMD4x2 mode. */
1044       src = retype(brw_vec4_grf(offset.nr - 1, 0), BRW_REGISTER_TYPE_UD);
1045       mlen = 2;
1046       header_present = true;
1047
1048       brw_push_insn_state(p);
1049       brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1050       brw_MOV(p, vec8(src), retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD));
1051       brw_set_default_access_mode(p, BRW_ALIGN_1);
1052
1053       brw_MOV(p, get_element_ud(src, 2),
1054               brw_imm_ud(GEN9_SAMPLER_SIMD_MODE_EXTENSION_SIMD4X2));
1055       brw_pop_insn_state(p);
1056    }
1057
1058    if (index.file == BRW_IMMEDIATE_VALUE) {
1059
1060       uint32_t surf_index = index.dw1.ud;
1061
1062       brw_push_insn_state(p);
1063       brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
1064       brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1065       brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND);
1066       brw_pop_insn_state(p);
1067
1068       brw_set_dest(p, send, dst);
1069       brw_set_src0(p, send, src);
1070       brw_set_sampler_message(p, send,
1071                               surf_index,
1072                               0, /* LD message ignores sampler unit */
1073                               GEN5_SAMPLER_MESSAGE_SAMPLE_LD,
1074                               1, /* rlen */
1075                               mlen,
1076                               header_present,
1077                               BRW_SAMPLER_SIMD_MODE_SIMD4X2,
1078                               0);
1079
1080       brw_mark_surface_used(prog_data, surf_index);
1081
1082    } else {
1083
1084       struct brw_reg addr = vec1(retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD));
1085
1086       brw_push_insn_state(p);
1087       brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1088       brw_set_default_access_mode(p, BRW_ALIGN_1);
1089
1090       /* a0.0 = surf_index & 0xff */
1091       brw_inst *insn_and = brw_next_insn(p, BRW_OPCODE_AND);
1092       brw_inst_set_exec_size(p->devinfo, insn_and, BRW_EXECUTE_1);
1093       brw_set_dest(p, insn_and, addr);
1094       brw_set_src0(p, insn_and, vec1(retype(index, BRW_REGISTER_TYPE_UD)));
1095       brw_set_src1(p, insn_and, brw_imm_ud(0x0ff));
1096
1097       /* dst = send(payload, a0.0 | <descriptor>) */
1098       brw_inst *insn = brw_send_indirect_message(
1099          p, BRW_SFID_SAMPLER, dst, src, addr);
1100       brw_set_sampler_message(p, insn,
1101                               0,
1102                               0, /* LD message ignores sampler unit */
1103                               GEN5_SAMPLER_MESSAGE_SAMPLE_LD,
1104                               1, /* rlen */
1105                               mlen,
1106                               header_present,
1107                               BRW_SAMPLER_SIMD_MODE_SIMD4X2,
1108                               0);
1109
1110       brw_pop_insn_state(p);
1111
1112       /* visitor knows more than we do about the surface limit required,
1113        * so has already done marking.
1114        */
1115
1116    }
1117 }
1118
1119 void
1120 fs_generator::generate_varying_pull_constant_load(fs_inst *inst,
1121                                                   struct brw_reg dst,
1122                                                   struct brw_reg index,
1123                                                   struct brw_reg offset)
1124 {
1125    assert(devinfo->gen < 7); /* Should use the gen7 variant. */
1126    assert(inst->header_present);
1127    assert(inst->mlen);
1128
1129    assert(index.file == BRW_IMMEDIATE_VALUE &&
1130           index.type == BRW_REGISTER_TYPE_UD);
1131    uint32_t surf_index = index.dw1.ud;
1132
1133    uint32_t simd_mode, rlen, msg_type;
1134    if (dispatch_width == 16) {
1135       simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
1136       rlen = 8;
1137    } else {
1138       simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD8;
1139       rlen = 4;
1140    }
1141
1142    if (devinfo->gen >= 5)
1143       msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LD;
1144    else {
1145       /* We always use the SIMD16 message so that we only have to load U, and
1146        * not V or R.
1147        */
1148       msg_type = BRW_SAMPLER_MESSAGE_SIMD16_LD;
1149       assert(inst->mlen == 3);
1150       assert(inst->regs_written == 8);
1151       rlen = 8;
1152       simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
1153    }
1154
1155    struct brw_reg offset_mrf = retype(brw_message_reg(inst->base_mrf + 1),
1156                                       BRW_REGISTER_TYPE_D);
1157    brw_MOV(p, offset_mrf, offset);
1158
1159    struct brw_reg header = brw_vec8_grf(0, 0);
1160    gen6_resolve_implied_move(p, &header, inst->base_mrf);
1161
1162    brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND);
1163    brw_inst_set_qtr_control(p->devinfo, send, BRW_COMPRESSION_NONE);
1164    brw_set_dest(p, send, retype(dst, BRW_REGISTER_TYPE_UW));
1165    brw_set_src0(p, send, header);
1166    if (devinfo->gen < 6)
1167       brw_inst_set_base_mrf(p->devinfo, send, inst->base_mrf);
1168
1169    /* Our surface is set up as floats, regardless of what actual data is
1170     * stored in it.
1171     */
1172    uint32_t return_format = BRW_SAMPLER_RETURN_FORMAT_FLOAT32;
1173    brw_set_sampler_message(p, send,
1174                            surf_index,
1175                            0, /* sampler (unused) */
1176                            msg_type,
1177                            rlen,
1178                            inst->mlen,
1179                            inst->header_present,
1180                            simd_mode,
1181                            return_format);
1182
1183    brw_mark_surface_used(prog_data, surf_index);
1184 }
1185
1186 void
1187 fs_generator::generate_varying_pull_constant_load_gen7(fs_inst *inst,
1188                                                        struct brw_reg dst,
1189                                                        struct brw_reg index,
1190                                                        struct brw_reg offset)
1191 {
1192    assert(devinfo->gen >= 7);
1193    /* Varying-offset pull constant loads are treated as a normal expression on
1194     * gen7, so the fact that it's a send message is hidden at the IR level.
1195     */
1196    assert(!inst->header_present);
1197    assert(!inst->mlen);
1198    assert(index.type == BRW_REGISTER_TYPE_UD);
1199
1200    uint32_t simd_mode, rlen, mlen;
1201    if (dispatch_width == 16) {
1202       mlen = 2;
1203       rlen = 8;
1204       simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
1205    } else {
1206       mlen = 1;
1207       rlen = 4;
1208       simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD8;
1209    }
1210
1211    if (index.file == BRW_IMMEDIATE_VALUE) {
1212
1213       uint32_t surf_index = index.dw1.ud;
1214
1215       brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND);
1216       brw_set_dest(p, send, retype(dst, BRW_REGISTER_TYPE_UW));
1217       brw_set_src0(p, send, offset);
1218       brw_set_sampler_message(p, send,
1219                               surf_index,
1220                               0, /* LD message ignores sampler unit */
1221                               GEN5_SAMPLER_MESSAGE_SAMPLE_LD,
1222                               rlen,
1223                               mlen,
1224                               false, /* no header */
1225                               simd_mode,
1226                               0);
1227
1228       brw_mark_surface_used(prog_data, surf_index);
1229
1230    } else {
1231
1232       struct brw_reg addr = vec1(retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD));
1233
1234       brw_push_insn_state(p);
1235       brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1236       brw_set_default_access_mode(p, BRW_ALIGN_1);
1237
1238       /* a0.0 = surf_index & 0xff */
1239       brw_inst *insn_and = brw_next_insn(p, BRW_OPCODE_AND);
1240       brw_inst_set_exec_size(p->devinfo, insn_and, BRW_EXECUTE_1);
1241       brw_set_dest(p, insn_and, addr);
1242       brw_set_src0(p, insn_and, vec1(retype(index, BRW_REGISTER_TYPE_UD)));
1243       brw_set_src1(p, insn_and, brw_imm_ud(0x0ff));
1244
1245       brw_pop_insn_state(p);
1246
1247       /* dst = send(offset, a0.0 | <descriptor>) */
1248       brw_inst *insn = brw_send_indirect_message(
1249          p, BRW_SFID_SAMPLER, retype(dst, BRW_REGISTER_TYPE_UW),
1250          offset, addr);
1251       brw_set_sampler_message(p, insn,
1252                               0 /* surface */,
1253                               0 /* sampler */,
1254                               GEN5_SAMPLER_MESSAGE_SAMPLE_LD,
1255                               rlen /* rlen */,
1256                               mlen /* mlen */,
1257                               false /* header */,
1258                               simd_mode,
1259                               0);
1260
1261       /* visitor knows more than we do about the surface limit required,
1262        * so has already done marking.
1263        */
1264    }
1265 }
1266
1267 /**
1268  * Cause the current pixel/sample mask (from R1.7 bits 15:0) to be transferred
1269  * into the flags register (f0.0).
1270  *
1271  * Used only on Gen6 and above.
1272  */
1273 void
1274 fs_generator::generate_mov_dispatch_to_flags(fs_inst *inst)
1275 {
1276    struct brw_reg flags = brw_flag_reg(0, inst->flag_subreg);
1277    struct brw_reg dispatch_mask;
1278
1279    if (devinfo->gen >= 6)
1280       dispatch_mask = retype(brw_vec1_grf(1, 7), BRW_REGISTER_TYPE_UW);
1281    else
1282       dispatch_mask = retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_UW);
1283
1284    brw_push_insn_state(p);
1285    brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1286    brw_MOV(p, flags, dispatch_mask);
1287    brw_pop_insn_state(p);
1288 }
1289
1290 void
1291 fs_generator::generate_pixel_interpolator_query(fs_inst *inst,
1292                                                 struct brw_reg dst,
1293                                                 struct brw_reg src,
1294                                                 struct brw_reg msg_data,
1295                                                 unsigned msg_type)
1296 {
1297    assert(msg_data.file == BRW_IMMEDIATE_VALUE &&
1298           msg_data.type == BRW_REGISTER_TYPE_UD);
1299
1300    brw_pixel_interpolator_query(p,
1301          retype(dst, BRW_REGISTER_TYPE_UW),
1302          src,
1303          inst->pi_noperspective,
1304          msg_type,
1305          msg_data.dw1.ud,
1306          inst->mlen,
1307          inst->regs_written);
1308 }
1309
1310
1311 /**
1312  * Sets the first word of a vgrf for gen7+ simd4x2 uniform pull constant
1313  * sampler LD messages.
1314  *
1315  * We don't want to bake it into the send message's code generation because
1316  * that means we don't get a chance to schedule the instructions.
1317  */
1318 void
1319 fs_generator::generate_set_simd4x2_offset(fs_inst *inst,
1320                                           struct brw_reg dst,
1321                                           struct brw_reg value)
1322 {
1323    assert(value.file == BRW_IMMEDIATE_VALUE);
1324
1325    brw_push_insn_state(p);
1326    brw_set_default_exec_size(p, BRW_EXECUTE_8);
1327    brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
1328    brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1329    brw_MOV(p, retype(brw_vec1_reg(dst.file, dst.nr, 0), value.type), value);
1330    brw_pop_insn_state(p);
1331 }
1332
1333 /* Sets vstride=16, width=8, hstride=2 or vstride=0, width=1, hstride=0
1334  * (when mask is passed as a uniform) of register mask before moving it
1335  * to register dst.
1336  */
1337 void
1338 fs_generator::generate_set_omask(fs_inst *inst,
1339                                  struct brw_reg dst,
1340                                  struct brw_reg mask)
1341 {
1342    bool stride_8_8_1 =
1343     (mask.vstride == BRW_VERTICAL_STRIDE_8 &&
1344      mask.width == BRW_WIDTH_8 &&
1345      mask.hstride == BRW_HORIZONTAL_STRIDE_1);
1346
1347    bool stride_0_1_0 = has_scalar_region(mask);
1348
1349    assert(stride_8_8_1 || stride_0_1_0);
1350    assert(dst.type == BRW_REGISTER_TYPE_UW);
1351
1352    brw_push_insn_state(p);
1353    brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
1354    brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1355
1356    if (stride_8_8_1) {
1357       brw_MOV(p, dst, retype(stride(mask, 16, 8, 2), dst.type));
1358    } else if (stride_0_1_0) {
1359       brw_MOV(p, dst, retype(mask, dst.type));
1360    }
1361    brw_pop_insn_state(p);
1362 }
1363
1364 /* Sets vstride=1, width=4, hstride=0 of register src1 during
1365  * the ADD instruction.
1366  */
1367 void
1368 fs_generator::generate_set_sample_id(fs_inst *inst,
1369                                      struct brw_reg dst,
1370                                      struct brw_reg src0,
1371                                      struct brw_reg src1)
1372 {
1373    assert(dst.type == BRW_REGISTER_TYPE_D ||
1374           dst.type == BRW_REGISTER_TYPE_UD);
1375    assert(src0.type == BRW_REGISTER_TYPE_D ||
1376           src0.type == BRW_REGISTER_TYPE_UD);
1377
1378    brw_push_insn_state(p);
1379    brw_set_default_exec_size(p, BRW_EXECUTE_8);
1380    brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
1381    brw_set_default_mask_control(p, BRW_MASK_DISABLE);
1382    struct brw_reg reg = retype(stride(src1, 1, 4, 0), BRW_REGISTER_TYPE_UW);
1383    if (dispatch_width == 8) {
1384       brw_ADD(p, dst, src0, reg);
1385    } else if (dispatch_width == 16) {
1386       brw_ADD(p, firsthalf(dst), firsthalf(src0), reg);
1387       brw_ADD(p, sechalf(dst), sechalf(src0), suboffset(reg, 2));
1388    }
1389    brw_pop_insn_state(p);
1390 }
1391
1392 void
1393 fs_generator::generate_pack_half_2x16_split(fs_inst *inst,
1394                                             struct brw_reg dst,
1395                                             struct brw_reg x,
1396                                             struct brw_reg y)
1397 {
1398    assert(devinfo->gen >= 7);
1399    assert(dst.type == BRW_REGISTER_TYPE_UD);
1400    assert(x.type == BRW_REGISTER_TYPE_F);
1401    assert(y.type == BRW_REGISTER_TYPE_F);
1402
1403    /* From the Ivybridge PRM, Vol4, Part3, Section 6.27 f32to16:
1404     *
1405     *   Because this instruction does not have a 16-bit floating-point type,
1406     *   the destination data type must be Word (W).
1407     *
1408     *   The destination must be DWord-aligned and specify a horizontal stride
1409     *   (HorzStride) of 2. The 16-bit result is stored in the lower word of
1410     *   each destination channel and the upper word is not modified.
1411     */
1412    struct brw_reg dst_w = spread(retype(dst, BRW_REGISTER_TYPE_W), 2);
1413
1414    /* Give each 32-bit channel of dst the form below, where "." means
1415     * unchanged.
1416     *   0x....hhhh
1417     */
1418    brw_F32TO16(p, dst_w, y);
1419
1420    /* Now the form:
1421     *   0xhhhh0000
1422     */
1423    brw_SHL(p, dst, dst, brw_imm_ud(16u));
1424
1425    /* And, finally the form of packHalf2x16's output:
1426     *   0xhhhhllll
1427     */
1428    brw_F32TO16(p, dst_w, x);
1429 }
1430
1431 void
1432 fs_generator::generate_unpack_half_2x16_split(fs_inst *inst,
1433                                               struct brw_reg dst,
1434                                               struct brw_reg src)
1435 {
1436    assert(devinfo->gen >= 7);
1437    assert(dst.type == BRW_REGISTER_TYPE_F);
1438    assert(src.type == BRW_REGISTER_TYPE_UD);
1439
1440    /* From the Ivybridge PRM, Vol4, Part3, Section 6.26 f16to32:
1441     *
1442     *   Because this instruction does not have a 16-bit floating-point type,
1443     *   the source data type must be Word (W). The destination type must be
1444     *   F (Float).
1445     */
1446    struct brw_reg src_w = spread(retype(src, BRW_REGISTER_TYPE_W), 2);
1447
1448    /* Each channel of src has the form of unpackHalf2x16's input: 0xhhhhllll.
1449     * For the Y case, we wish to access only the upper word; therefore
1450     * a 16-bit subregister offset is needed.
1451     */
1452    assert(inst->opcode == FS_OPCODE_UNPACK_HALF_2x16_SPLIT_X ||
1453           inst->opcode == FS_OPCODE_UNPACK_HALF_2x16_SPLIT_Y);
1454    if (inst->opcode == FS_OPCODE_UNPACK_HALF_2x16_SPLIT_Y)
1455       src_w.subnr += 2;
1456
1457    brw_F16TO32(p, dst, src_w);
1458 }
1459
1460 void
1461 fs_generator::generate_shader_time_add(fs_inst *inst,
1462                                        struct brw_reg payload,
1463                                        struct brw_reg offset,
1464                                        struct brw_reg value)
1465 {
1466    assert(devinfo->gen >= 7);
1467    brw_push_insn_state(p);
1468    brw_set_default_mask_control(p, true);
1469
1470    assert(payload.file == BRW_GENERAL_REGISTER_FILE);
1471    struct brw_reg payload_offset = retype(brw_vec1_grf(payload.nr, 0),
1472                                           offset.type);
1473    struct brw_reg payload_value = retype(brw_vec1_grf(payload.nr + 1, 0),
1474                                          value.type);
1475
1476    assert(offset.file == BRW_IMMEDIATE_VALUE);
1477    if (value.file == BRW_GENERAL_REGISTER_FILE) {
1478       value.width = BRW_WIDTH_1;
1479       value.hstride = BRW_HORIZONTAL_STRIDE_0;
1480       value.vstride = BRW_VERTICAL_STRIDE_0;
1481    } else {
1482       assert(value.file == BRW_IMMEDIATE_VALUE);
1483    }
1484
1485    /* Trying to deal with setup of the params from the IR is crazy in the FS8
1486     * case, and we don't really care about squeezing every bit of performance
1487     * out of this path, so we just emit the MOVs from here.
1488     */
1489    brw_MOV(p, payload_offset, offset);
1490    brw_MOV(p, payload_value, value);
1491    brw_shader_time_add(p, payload,
1492                        prog_data->binding_table.shader_time_start);
1493    brw_pop_insn_state(p);
1494
1495    brw_mark_surface_used(prog_data,
1496                          prog_data->binding_table.shader_time_start);
1497 }
1498
1499 void
1500 fs_generator::generate_untyped_atomic(fs_inst *inst, struct brw_reg dst,
1501                                       struct brw_reg payload,
1502                                       struct brw_reg atomic_op,
1503                                       struct brw_reg surf_index)
1504 {
1505    assert(atomic_op.file == BRW_IMMEDIATE_VALUE &&
1506           atomic_op.type == BRW_REGISTER_TYPE_UD &&
1507           surf_index.file == BRW_IMMEDIATE_VALUE &&
1508           surf_index.type == BRW_REGISTER_TYPE_UD);
1509
1510    brw_untyped_atomic(p, dst, payload,
1511                       atomic_op.dw1.ud, surf_index.dw1.ud,
1512                       inst->mlen, true);
1513
1514    brw_mark_surface_used(prog_data, surf_index.dw1.ud);
1515 }
1516
1517 void
1518 fs_generator::generate_untyped_surface_read(fs_inst *inst, struct brw_reg dst,
1519                                             struct brw_reg payload,
1520                                             struct brw_reg surf_index)
1521 {
1522    assert(surf_index.file == BRW_IMMEDIATE_VALUE &&
1523           surf_index.type == BRW_REGISTER_TYPE_UD);
1524
1525    brw_untyped_surface_read(p, dst, payload, surf_index.dw1.ud, inst->mlen, 1);
1526
1527    brw_mark_surface_used(prog_data, surf_index.dw1.ud);
1528 }
1529
1530 void
1531 fs_generator::enable_debug(const char *shader_name)
1532 {
1533    debug_flag = true;
1534    this->shader_name = shader_name;
1535 }
1536
1537 int
1538 fs_generator::generate_code(const cfg_t *cfg, int dispatch_width)
1539 {
1540    /* align to 64 byte boundary. */
1541    while (p->next_insn_offset % 64)
1542       brw_NOP(p);
1543
1544    this->dispatch_width = dispatch_width;
1545    if (dispatch_width == 16)
1546       brw_set_default_compression_control(p, BRW_COMPRESSION_COMPRESSED);
1547
1548    int start_offset = p->next_insn_offset;
1549    int spill_count = 0, fill_count = 0;
1550    int loop_count = 0;
1551
1552    struct annotation_info annotation;
1553    memset(&annotation, 0, sizeof(annotation));
1554
1555    foreach_block_and_inst (block, fs_inst, inst, cfg) {
1556       struct brw_reg src[3], dst;
1557       unsigned int last_insn_offset = p->next_insn_offset;
1558       bool multiple_instructions_emitted = false;
1559
1560       if (unlikely(debug_flag))
1561          annotate(p->devinfo, &annotation, cfg, inst, p->next_insn_offset);
1562
1563       for (unsigned int i = 0; i < inst->sources; i++) {
1564          src[i] = brw_reg_from_fs_reg(&inst->src[i]);
1565
1566          /* The accumulator result appears to get used for the
1567           * conditional modifier generation.  When negating a UD
1568           * value, there is a 33rd bit generated for the sign in the
1569           * accumulator value, so now you can't check, for example,
1570           * equality with a 32-bit value.  See piglit fs-op-neg-uvec4.
1571           */
1572          assert(!inst->conditional_mod ||
1573                 inst->src[i].type != BRW_REGISTER_TYPE_UD ||
1574                 !inst->src[i].negate);
1575       }
1576       dst = brw_reg_from_fs_reg(&inst->dst);
1577
1578       brw_set_default_predicate_control(p, inst->predicate);
1579       brw_set_default_predicate_inverse(p, inst->predicate_inverse);
1580       brw_set_default_flag_reg(p, 0, inst->flag_subreg);
1581       brw_set_default_saturate(p, inst->saturate);
1582       brw_set_default_mask_control(p, inst->force_writemask_all);
1583       brw_set_default_acc_write_control(p, inst->writes_accumulator);
1584       brw_set_default_exec_size(p, cvt(inst->exec_size) - 1);
1585
1586       switch (inst->exec_size) {
1587       case 1:
1588       case 2:
1589       case 4:
1590          assert(inst->force_writemask_all);
1591          brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
1592          break;
1593       case 8:
1594          if (inst->force_sechalf) {
1595             brw_set_default_compression_control(p, BRW_COMPRESSION_2NDHALF);
1596          } else {
1597             brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
1598          }
1599          break;
1600       case 16:
1601       case 32:
1602          if (type_sz(inst->dst.type) < sizeof(float))
1603             brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
1604          else
1605             brw_set_default_compression_control(p, BRW_COMPRESSION_COMPRESSED);
1606          break;
1607       default:
1608          unreachable("Invalid instruction width");
1609       }
1610
1611       switch (inst->opcode) {
1612       case BRW_OPCODE_MOV:
1613          brw_MOV(p, dst, src[0]);
1614          break;
1615       case BRW_OPCODE_ADD:
1616          brw_ADD(p, dst, src[0], src[1]);
1617          break;
1618       case BRW_OPCODE_MUL:
1619          brw_MUL(p, dst, src[0], src[1]);
1620          break;
1621       case BRW_OPCODE_AVG:
1622          brw_AVG(p, dst, src[0], src[1]);
1623          break;
1624       case BRW_OPCODE_MACH:
1625          brw_MACH(p, dst, src[0], src[1]);
1626          break;
1627
1628       case BRW_OPCODE_LINE:
1629          brw_LINE(p, dst, src[0], src[1]);
1630          break;
1631
1632       case BRW_OPCODE_MAD:
1633          assert(devinfo->gen >= 6);
1634          brw_set_default_access_mode(p, BRW_ALIGN_16);
1635          if (dispatch_width == 16 && !devinfo->supports_simd16_3src) {
1636             brw_set_default_exec_size(p, BRW_EXECUTE_8);
1637             brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
1638             brw_inst *f = brw_MAD(p, firsthalf(dst), firsthalf(src[0]), firsthalf(src[1]), firsthalf(src[2]));
1639             brw_set_default_compression_control(p, BRW_COMPRESSION_2NDHALF);
1640             brw_inst *s = brw_MAD(p, sechalf(dst), sechalf(src[0]), sechalf(src[1]), sechalf(src[2]));
1641             brw_set_default_compression_control(p, BRW_COMPRESSION_COMPRESSED);
1642
1643             if (inst->conditional_mod) {
1644                brw_inst_set_cond_modifier(p->devinfo, f, inst->conditional_mod);
1645                brw_inst_set_cond_modifier(p->devinfo, s, inst->conditional_mod);
1646                multiple_instructions_emitted = true;
1647             }
1648          } else {
1649             brw_MAD(p, dst, src[0], src[1], src[2]);
1650          }
1651          brw_set_default_access_mode(p, BRW_ALIGN_1);
1652          break;
1653
1654       case BRW_OPCODE_LRP:
1655          assert(devinfo->gen >= 6);
1656          brw_set_default_access_mode(p, BRW_ALIGN_16);
1657          if (dispatch_width == 16 && !devinfo->supports_simd16_3src) {
1658             brw_set_default_exec_size(p, BRW_EXECUTE_8);
1659             brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
1660             brw_inst *f = brw_LRP(p, firsthalf(dst), firsthalf(src[0]), firsthalf(src[1]), firsthalf(src[2]));
1661             brw_set_default_compression_control(p, BRW_COMPRESSION_2NDHALF);
1662             brw_inst *s = brw_LRP(p, sechalf(dst), sechalf(src[0]), sechalf(src[1]), sechalf(src[2]));
1663             brw_set_default_compression_control(p, BRW_COMPRESSION_COMPRESSED);
1664
1665             if (inst->conditional_mod) {
1666                brw_inst_set_cond_modifier(p->devinfo, f, inst->conditional_mod);
1667                brw_inst_set_cond_modifier(p->devinfo, s, inst->conditional_mod);
1668                multiple_instructions_emitted = true;
1669             }
1670          } else {
1671             brw_LRP(p, dst, src[0], src[1], src[2]);
1672          }
1673          brw_set_default_access_mode(p, BRW_ALIGN_1);
1674          break;
1675
1676       case BRW_OPCODE_FRC:
1677          brw_FRC(p, dst, src[0]);
1678          break;
1679       case BRW_OPCODE_RNDD:
1680          brw_RNDD(p, dst, src[0]);
1681          break;
1682       case BRW_OPCODE_RNDE:
1683          brw_RNDE(p, dst, src[0]);
1684          break;
1685       case BRW_OPCODE_RNDZ:
1686          brw_RNDZ(p, dst, src[0]);
1687          break;
1688
1689       case BRW_OPCODE_AND:
1690          brw_AND(p, dst, src[0], src[1]);
1691          break;
1692       case BRW_OPCODE_OR:
1693          brw_OR(p, dst, src[0], src[1]);
1694          break;
1695       case BRW_OPCODE_XOR:
1696          brw_XOR(p, dst, src[0], src[1]);
1697          break;
1698       case BRW_OPCODE_NOT:
1699          brw_NOT(p, dst, src[0]);
1700          break;
1701       case BRW_OPCODE_ASR:
1702          brw_ASR(p, dst, src[0], src[1]);
1703          break;
1704       case BRW_OPCODE_SHR:
1705          brw_SHR(p, dst, src[0], src[1]);
1706          break;
1707       case BRW_OPCODE_SHL:
1708          brw_SHL(p, dst, src[0], src[1]);
1709          break;
1710       case BRW_OPCODE_F32TO16:
1711          assert(devinfo->gen >= 7);
1712          brw_F32TO16(p, dst, src[0]);
1713          break;
1714       case BRW_OPCODE_F16TO32:
1715          assert(devinfo->gen >= 7);
1716          brw_F16TO32(p, dst, src[0]);
1717          break;
1718       case BRW_OPCODE_CMP:
1719          /* The Ivybridge/BayTrail WaCMPInstFlagDepClearedEarly workaround says
1720           * that when the destination is a GRF that the dependency-clear bit on
1721           * the flag register is cleared early.
1722           *
1723           * Suggested workarounds are to disable coissuing CMP instructions
1724           * or to split CMP(16) instructions into two CMP(8) instructions.
1725           *
1726           * We choose to split into CMP(8) instructions since disabling
1727           * coissuing would affect CMP instructions not otherwise affected by
1728           * the errata.
1729           */
1730          if (dispatch_width == 16 && devinfo->gen == 7 && !devinfo->is_haswell) {
1731             if (dst.file == BRW_GENERAL_REGISTER_FILE) {
1732                brw_set_default_exec_size(p, BRW_EXECUTE_8);
1733                brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
1734                brw_CMP(p, firsthalf(dst), inst->conditional_mod,
1735                           firsthalf(src[0]), firsthalf(src[1]));
1736                brw_set_default_compression_control(p, BRW_COMPRESSION_2NDHALF);
1737                brw_CMP(p, sechalf(dst), inst->conditional_mod,
1738                           sechalf(src[0]), sechalf(src[1]));
1739                brw_set_default_compression_control(p, BRW_COMPRESSION_COMPRESSED);
1740
1741                multiple_instructions_emitted = true;
1742             } else if (dst.file == BRW_ARCHITECTURE_REGISTER_FILE) {
1743                /* For unknown reasons, the aforementioned workaround is not
1744                 * sufficient. Overriding the type when the destination is the
1745                 * null register is necessary but not sufficient by itself.
1746                 */
1747                assert(dst.nr == BRW_ARF_NULL);
1748                dst.type = BRW_REGISTER_TYPE_D;
1749                brw_CMP(p, dst, inst->conditional_mod, src[0], src[1]);
1750             } else {
1751                unreachable("not reached");
1752             }
1753          } else {
1754             brw_CMP(p, dst, inst->conditional_mod, src[0], src[1]);
1755          }
1756          break;
1757       case BRW_OPCODE_SEL:
1758          brw_SEL(p, dst, src[0], src[1]);
1759          break;
1760       case BRW_OPCODE_BFREV:
1761          assert(devinfo->gen >= 7);
1762          /* BFREV only supports UD type for src and dst. */
1763          brw_BFREV(p, retype(dst, BRW_REGISTER_TYPE_UD),
1764                       retype(src[0], BRW_REGISTER_TYPE_UD));
1765          break;
1766       case BRW_OPCODE_FBH:
1767          assert(devinfo->gen >= 7);
1768          /* FBH only supports UD type for dst. */
1769          brw_FBH(p, retype(dst, BRW_REGISTER_TYPE_UD), src[0]);
1770          break;
1771       case BRW_OPCODE_FBL:
1772          assert(devinfo->gen >= 7);
1773          /* FBL only supports UD type for dst. */
1774          brw_FBL(p, retype(dst, BRW_REGISTER_TYPE_UD), src[0]);
1775          break;
1776       case BRW_OPCODE_CBIT:
1777          assert(devinfo->gen >= 7);
1778          /* CBIT only supports UD type for dst. */
1779          brw_CBIT(p, retype(dst, BRW_REGISTER_TYPE_UD), src[0]);
1780          break;
1781       case BRW_OPCODE_ADDC:
1782          assert(devinfo->gen >= 7);
1783          brw_ADDC(p, dst, src[0], src[1]);
1784          break;
1785       case BRW_OPCODE_SUBB:
1786          assert(devinfo->gen >= 7);
1787          brw_SUBB(p, dst, src[0], src[1]);
1788          break;
1789       case BRW_OPCODE_MAC:
1790          brw_MAC(p, dst, src[0], src[1]);
1791          break;
1792
1793       case BRW_OPCODE_BFE:
1794          assert(devinfo->gen >= 7);
1795          brw_set_default_access_mode(p, BRW_ALIGN_16);
1796          if (dispatch_width == 16 && !devinfo->supports_simd16_3src) {
1797             brw_set_default_exec_size(p, BRW_EXECUTE_8);
1798             brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
1799             brw_BFE(p, firsthalf(dst), firsthalf(src[0]), firsthalf(src[1]), firsthalf(src[2]));
1800             brw_set_default_compression_control(p, BRW_COMPRESSION_2NDHALF);
1801             brw_BFE(p, sechalf(dst), sechalf(src[0]), sechalf(src[1]), sechalf(src[2]));
1802             brw_set_default_compression_control(p, BRW_COMPRESSION_COMPRESSED);
1803          } else {
1804             brw_BFE(p, dst, src[0], src[1], src[2]);
1805          }
1806          brw_set_default_access_mode(p, BRW_ALIGN_1);
1807          break;
1808
1809       case BRW_OPCODE_BFI1:
1810          assert(devinfo->gen >= 7);
1811          /* The Haswell WaForceSIMD8ForBFIInstruction workaround says that we
1812           * should
1813           *
1814           *    "Force BFI instructions to be executed always in SIMD8."
1815           */
1816          if (dispatch_width == 16 && devinfo->is_haswell) {
1817             brw_set_default_exec_size(p, BRW_EXECUTE_8);
1818             brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
1819             brw_BFI1(p, firsthalf(dst), firsthalf(src[0]), firsthalf(src[1]));
1820             brw_set_default_compression_control(p, BRW_COMPRESSION_2NDHALF);
1821             brw_BFI1(p, sechalf(dst), sechalf(src[0]), sechalf(src[1]));
1822             brw_set_default_compression_control(p, BRW_COMPRESSION_COMPRESSED);
1823          } else {
1824             brw_BFI1(p, dst, src[0], src[1]);
1825          }
1826          break;
1827       case BRW_OPCODE_BFI2:
1828          assert(devinfo->gen >= 7);
1829          brw_set_default_access_mode(p, BRW_ALIGN_16);
1830          /* The Haswell WaForceSIMD8ForBFIInstruction workaround says that we
1831           * should
1832           *
1833           *    "Force BFI instructions to be executed always in SIMD8."
1834           *
1835           * Otherwise we would be able to emit compressed instructions like we
1836           * do for the other three-source instructions.
1837           */
1838          if (dispatch_width == 16 &&
1839              (devinfo->is_haswell || !devinfo->supports_simd16_3src)) {
1840             brw_set_default_exec_size(p, BRW_EXECUTE_8);
1841             brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
1842             brw_BFI2(p, firsthalf(dst), firsthalf(src[0]), firsthalf(src[1]), firsthalf(src[2]));
1843             brw_set_default_compression_control(p, BRW_COMPRESSION_2NDHALF);
1844             brw_BFI2(p, sechalf(dst), sechalf(src[0]), sechalf(src[1]), sechalf(src[2]));
1845             brw_set_default_compression_control(p, BRW_COMPRESSION_COMPRESSED);
1846          } else {
1847             brw_BFI2(p, dst, src[0], src[1], src[2]);
1848          }
1849          brw_set_default_access_mode(p, BRW_ALIGN_1);
1850          break;
1851
1852       case BRW_OPCODE_IF:
1853          if (inst->src[0].file != BAD_FILE) {
1854             /* The instruction has an embedded compare (only allowed on gen6) */
1855             assert(devinfo->gen == 6);
1856             gen6_IF(p, inst->conditional_mod, src[0], src[1]);
1857          } else {
1858             brw_IF(p, dispatch_width == 16 ? BRW_EXECUTE_16 : BRW_EXECUTE_8);
1859          }
1860          break;
1861
1862       case BRW_OPCODE_ELSE:
1863          brw_ELSE(p);
1864          break;
1865       case BRW_OPCODE_ENDIF:
1866          brw_ENDIF(p);
1867          break;
1868
1869       case BRW_OPCODE_DO:
1870          brw_DO(p, BRW_EXECUTE_8);
1871          break;
1872
1873       case BRW_OPCODE_BREAK:
1874          brw_BREAK(p);
1875          brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
1876          break;
1877       case BRW_OPCODE_CONTINUE:
1878          brw_CONT(p);
1879          brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
1880          break;
1881
1882       case BRW_OPCODE_WHILE:
1883          brw_WHILE(p);
1884          loop_count++;
1885          break;
1886
1887       case SHADER_OPCODE_RCP:
1888       case SHADER_OPCODE_RSQ:
1889       case SHADER_OPCODE_SQRT:
1890       case SHADER_OPCODE_EXP2:
1891       case SHADER_OPCODE_LOG2:
1892       case SHADER_OPCODE_SIN:
1893       case SHADER_OPCODE_COS:
1894          assert(devinfo->gen < 6 || inst->mlen == 0);
1895          assert(inst->conditional_mod == BRW_CONDITIONAL_NONE);
1896          if (devinfo->gen >= 7) {
1897             gen6_math(p, dst, brw_math_function(inst->opcode), src[0],
1898                       brw_null_reg());
1899          } else if (devinfo->gen == 6) {
1900             generate_math_gen6(inst, dst, src[0], brw_null_reg());
1901          } else if (devinfo->gen == 5 || devinfo->is_g4x) {
1902             generate_math_g45(inst, dst, src[0]);
1903          } else {
1904             generate_math_gen4(inst, dst, src[0]);
1905          }
1906          break;
1907       case SHADER_OPCODE_INT_QUOTIENT:
1908       case SHADER_OPCODE_INT_REMAINDER:
1909       case SHADER_OPCODE_POW:
1910          assert(devinfo->gen < 6 || inst->mlen == 0);
1911          assert(inst->conditional_mod == BRW_CONDITIONAL_NONE);
1912          if (devinfo->gen >= 7 && inst->opcode == SHADER_OPCODE_POW) {
1913             gen6_math(p, dst, brw_math_function(inst->opcode), src[0], src[1]);
1914          } else if (devinfo->gen >= 6) {
1915             generate_math_gen6(inst, dst, src[0], src[1]);
1916          } else {
1917             generate_math_gen4(inst, dst, src[0]);
1918          }
1919          break;
1920       case FS_OPCODE_CINTERP:
1921          brw_MOV(p, dst, src[0]);
1922          break;
1923       case FS_OPCODE_LINTERP:
1924          generate_linterp(inst, dst, src);
1925          break;
1926       case FS_OPCODE_PIXEL_X:
1927          assert(src[0].type == BRW_REGISTER_TYPE_UW);
1928          src[0].subnr = 0 * type_sz(src[0].type);
1929          brw_MOV(p, dst, stride(src[0], 8, 4, 1));
1930          break;
1931       case FS_OPCODE_PIXEL_Y:
1932          assert(src[0].type == BRW_REGISTER_TYPE_UW);
1933          src[0].subnr = 4 * type_sz(src[0].type);
1934          brw_MOV(p, dst, stride(src[0], 8, 4, 1));
1935          break;
1936       case SHADER_OPCODE_TEX:
1937       case FS_OPCODE_TXB:
1938       case SHADER_OPCODE_TXD:
1939       case SHADER_OPCODE_TXF:
1940       case SHADER_OPCODE_TXF_CMS:
1941       case SHADER_OPCODE_TXF_UMS:
1942       case SHADER_OPCODE_TXF_MCS:
1943       case SHADER_OPCODE_TXL:
1944       case SHADER_OPCODE_TXS:
1945       case SHADER_OPCODE_LOD:
1946       case SHADER_OPCODE_TG4:
1947       case SHADER_OPCODE_TG4_OFFSET:
1948          generate_tex(inst, dst, src[0], src[1]);
1949          break;
1950       case FS_OPCODE_DDX_COARSE:
1951       case FS_OPCODE_DDX_FINE:
1952          generate_ddx(inst->opcode, dst, src[0]);
1953          break;
1954       case FS_OPCODE_DDY_COARSE:
1955       case FS_OPCODE_DDY_FINE:
1956          assert(src[1].file == BRW_IMMEDIATE_VALUE);
1957          generate_ddy(inst->opcode, dst, src[0], src[1].dw1.ud);
1958          break;
1959
1960       case SHADER_OPCODE_GEN4_SCRATCH_WRITE:
1961          generate_scratch_write(inst, src[0]);
1962          spill_count++;
1963          break;
1964
1965       case SHADER_OPCODE_GEN4_SCRATCH_READ:
1966          generate_scratch_read(inst, dst);
1967          fill_count++;
1968          break;
1969
1970       case SHADER_OPCODE_GEN7_SCRATCH_READ:
1971          generate_scratch_read_gen7(inst, dst);
1972          fill_count++;
1973          break;
1974
1975       case SHADER_OPCODE_URB_WRITE_SIMD8:
1976          generate_urb_write(inst, src[0]);
1977          break;
1978
1979       case FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD:
1980          generate_uniform_pull_constant_load(inst, dst, src[0], src[1]);
1981          break;
1982
1983       case FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD_GEN7:
1984          generate_uniform_pull_constant_load_gen7(inst, dst, src[0], src[1]);
1985          break;
1986
1987       case FS_OPCODE_VARYING_PULL_CONSTANT_LOAD:
1988          generate_varying_pull_constant_load(inst, dst, src[0], src[1]);
1989          break;
1990
1991       case FS_OPCODE_VARYING_PULL_CONSTANT_LOAD_GEN7:
1992          generate_varying_pull_constant_load_gen7(inst, dst, src[0], src[1]);
1993          break;
1994
1995       case FS_OPCODE_REP_FB_WRITE:
1996       case FS_OPCODE_FB_WRITE:
1997          generate_fb_write(inst, src[0]);
1998          break;
1999
2000       case FS_OPCODE_BLORP_FB_WRITE:
2001          generate_blorp_fb_write(inst);
2002          break;
2003
2004       case FS_OPCODE_MOV_DISPATCH_TO_FLAGS:
2005          generate_mov_dispatch_to_flags(inst);
2006          break;
2007
2008       case FS_OPCODE_DISCARD_JUMP:
2009          generate_discard_jump(inst);
2010          break;
2011
2012       case SHADER_OPCODE_SHADER_TIME_ADD:
2013          generate_shader_time_add(inst, src[0], src[1], src[2]);
2014          break;
2015
2016       case SHADER_OPCODE_UNTYPED_ATOMIC:
2017          generate_untyped_atomic(inst, dst, src[0], src[1], src[2]);
2018          break;
2019
2020       case SHADER_OPCODE_UNTYPED_SURFACE_READ:
2021          generate_untyped_surface_read(inst, dst, src[0], src[1]);
2022          break;
2023
2024       case FS_OPCODE_SET_SIMD4X2_OFFSET:
2025          generate_set_simd4x2_offset(inst, dst, src[0]);
2026          break;
2027
2028       case FS_OPCODE_SET_OMASK:
2029          generate_set_omask(inst, dst, src[0]);
2030          break;
2031
2032       case FS_OPCODE_SET_SAMPLE_ID:
2033          generate_set_sample_id(inst, dst, src[0], src[1]);
2034          break;
2035
2036       case FS_OPCODE_PACK_HALF_2x16_SPLIT:
2037           generate_pack_half_2x16_split(inst, dst, src[0], src[1]);
2038           break;
2039
2040       case FS_OPCODE_UNPACK_HALF_2x16_SPLIT_X:
2041       case FS_OPCODE_UNPACK_HALF_2x16_SPLIT_Y:
2042          generate_unpack_half_2x16_split(inst, dst, src[0]);
2043          break;
2044
2045       case FS_OPCODE_PLACEHOLDER_HALT:
2046          /* This is the place where the final HALT needs to be inserted if
2047           * we've emitted any discards.  If not, this will emit no code.
2048           */
2049          if (!patch_discard_jumps_to_fb_writes()) {
2050             if (unlikely(debug_flag)) {
2051                annotation.ann_count--;
2052             }
2053          }
2054          break;
2055
2056       case FS_OPCODE_INTERPOLATE_AT_CENTROID:
2057          generate_pixel_interpolator_query(inst, dst, src[0], src[1],
2058                                            GEN7_PIXEL_INTERPOLATOR_LOC_CENTROID);
2059          break;
2060
2061       case FS_OPCODE_INTERPOLATE_AT_SAMPLE:
2062          generate_pixel_interpolator_query(inst, dst, src[0], src[1],
2063                                            GEN7_PIXEL_INTERPOLATOR_LOC_SAMPLE);
2064          break;
2065
2066       case FS_OPCODE_INTERPOLATE_AT_SHARED_OFFSET:
2067          generate_pixel_interpolator_query(inst, dst, src[0], src[1],
2068                                            GEN7_PIXEL_INTERPOLATOR_LOC_SHARED_OFFSET);
2069          break;
2070
2071       case FS_OPCODE_INTERPOLATE_AT_PER_SLOT_OFFSET:
2072          generate_pixel_interpolator_query(inst, dst, src[0], src[1],
2073                                            GEN7_PIXEL_INTERPOLATOR_LOC_PER_SLOT_OFFSET);
2074          break;
2075
2076       default:
2077          unreachable("Unsupported opcode");
2078
2079       case SHADER_OPCODE_LOAD_PAYLOAD:
2080          unreachable("Should be lowered by lower_load_payload()");
2081       }
2082
2083       if (multiple_instructions_emitted)
2084          continue;
2085
2086       if (inst->no_dd_clear || inst->no_dd_check || inst->conditional_mod) {
2087          assert(p->next_insn_offset == last_insn_offset + 16 ||
2088                 !"conditional_mod, no_dd_check, or no_dd_clear set for IR "
2089                  "emitting more than 1 instruction");
2090
2091          brw_inst *last = &p->store[last_insn_offset / 16];
2092
2093          if (inst->conditional_mod)
2094             brw_inst_set_cond_modifier(p->devinfo, last, inst->conditional_mod);
2095          brw_inst_set_no_dd_clear(p->devinfo, last, inst->no_dd_clear);
2096          brw_inst_set_no_dd_check(p->devinfo, last, inst->no_dd_check);
2097       }
2098    }
2099
2100    brw_set_uip_jip(p);
2101    annotation_finalize(&annotation, p->next_insn_offset);
2102
2103    int before_size = p->next_insn_offset - start_offset;
2104    brw_compact_instructions(p, start_offset, annotation.ann_count,
2105                             annotation.ann);
2106    int after_size = p->next_insn_offset - start_offset;
2107
2108    if (unlikely(debug_flag)) {
2109       fprintf(stderr, "Native code for %s\n"
2110               "SIMD%d shader: %d instructions. %d loops. %d:%d spills:fills. Promoted %u constants. Compacted %d to %d"
2111               " bytes (%.0f%%)\n",
2112               shader_name, dispatch_width, before_size / 16, loop_count,
2113               spill_count, fill_count, promoted_constants, before_size, after_size,
2114               100.0f * (before_size - after_size) / before_size);
2115
2116       dump_assembly(p->store, annotation.ann_count, annotation.ann,
2117                     p->devinfo, prog);
2118       ralloc_free(annotation.ann);
2119    }
2120
2121    static GLuint msg_id = 0;
2122    _mesa_gl_debug(&brw->ctx, &msg_id,
2123                   MESA_DEBUG_SOURCE_SHADER_COMPILER,
2124                   MESA_DEBUG_TYPE_OTHER,
2125                   MESA_DEBUG_SEVERITY_NOTIFICATION,
2126                   "%s SIMD%d shader: %d inst, %d loops, %d:%d spills:fills, "
2127                   "Promoted %u constants, compacted %d to %d bytes.\n",
2128                   stage_abbrev, dispatch_width, before_size / 16, loop_count,
2129                   spill_count, fill_count, promoted_constants, before_size, after_size);
2130
2131    return start_offset;
2132 }
2133
2134 const unsigned *
2135 fs_generator::get_assembly(unsigned int *assembly_size)
2136 {
2137    return brw_get_program(p, assembly_size);
2138 }