2 * Copyright © 2011 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
29 #include "brw_vec4_builder.h"
30 #include "brw_vec4_live_variables.h"
31 #include "brw_dead_control_flow.h"
32 #include "program/prog_parameter.h"
34 #define MAX_INSTRUCTION (1 << 30)
43 memset(this, 0, sizeof(*this));
45 this->file = BAD_FILE;
48 src_reg::src_reg(enum brw_reg_file file, int nr, const glsl_type *type)
54 if (type && (type->is_scalar() || type->is_vector() || type->is_matrix()))
55 this->swizzle = brw_swizzle_for_size(type->vector_elements);
57 this->swizzle = BRW_SWIZZLE_XYZW;
59 this->type = brw_type_for_base_type(type);
62 /** Generic unset register constructor. */
68 src_reg::src_reg(struct ::brw_reg reg) :
75 src_reg::src_reg(const dst_reg ®) :
78 this->reladdr = reg.reladdr;
79 this->swizzle = brw_swizzle_for_mask(reg.writemask);
85 memset(this, 0, sizeof(*this));
86 this->file = BAD_FILE;
87 this->writemask = WRITEMASK_XYZW;
95 dst_reg::dst_reg(enum brw_reg_file file, int nr)
103 dst_reg::dst_reg(enum brw_reg_file file, int nr, const glsl_type *type,
110 this->type = brw_type_for_base_type(type);
111 this->writemask = writemask;
114 dst_reg::dst_reg(enum brw_reg_file file, int nr, brw_reg_type type,
122 this->writemask = writemask;
125 dst_reg::dst_reg(struct ::brw_reg reg) :
128 this->reg_offset = 0;
129 this->reladdr = NULL;
132 dst_reg::dst_reg(const src_reg ®) :
135 this->writemask = brw_mask_for_swizzle(reg.swizzle);
136 this->reladdr = reg.reladdr;
140 dst_reg::equals(const dst_reg &r) const
142 return (this->backend_reg::equals(r) &&
143 (reladdr == r.reladdr ||
144 (reladdr && r.reladdr && reladdr->equals(*r.reladdr))));
148 vec4_instruction::is_send_from_grf()
151 case SHADER_OPCODE_SHADER_TIME_ADD:
152 case VS_OPCODE_PULL_CONSTANT_LOAD_GEN7:
153 case SHADER_OPCODE_UNTYPED_ATOMIC:
154 case SHADER_OPCODE_UNTYPED_SURFACE_READ:
155 case SHADER_OPCODE_UNTYPED_SURFACE_WRITE:
156 case SHADER_OPCODE_TYPED_ATOMIC:
157 case SHADER_OPCODE_TYPED_SURFACE_READ:
158 case SHADER_OPCODE_TYPED_SURFACE_WRITE:
159 case VEC4_OPCODE_URB_READ:
160 case TCS_OPCODE_URB_WRITE:
161 case TCS_OPCODE_RELEASE_INPUT:
162 case SHADER_OPCODE_BARRIER:
170 * Returns true if this instruction's sources and destinations cannot
171 * safely be the same register.
173 * In most cases, a register can be written over safely by the same
174 * instruction that is its last use. For a single instruction, the
175 * sources are dereferenced before writing of the destination starts
178 * However, there are a few cases where this can be problematic:
180 * - Virtual opcodes that translate to multiple instructions in the
181 * code generator: if src == dst and one instruction writes the
182 * destination before a later instruction reads the source, then
183 * src will have been clobbered.
185 * The register allocator uses this information to set up conflicts between
186 * GRF sources and the destination.
189 vec4_instruction::has_source_and_destination_hazard() const
192 case TCS_OPCODE_SET_INPUT_URB_OFFSETS:
193 case TCS_OPCODE_SET_OUTPUT_URB_OFFSETS:
194 case TES_OPCODE_ADD_INDIRECT_URB_OFFSET:
202 vec4_instruction::regs_read(unsigned arg) const
204 if (src[arg].file == BAD_FILE)
208 case SHADER_OPCODE_SHADER_TIME_ADD:
209 case SHADER_OPCODE_UNTYPED_ATOMIC:
210 case SHADER_OPCODE_UNTYPED_SURFACE_READ:
211 case SHADER_OPCODE_UNTYPED_SURFACE_WRITE:
212 case SHADER_OPCODE_TYPED_ATOMIC:
213 case SHADER_OPCODE_TYPED_SURFACE_READ:
214 case SHADER_OPCODE_TYPED_SURFACE_WRITE:
215 case TCS_OPCODE_URB_WRITE:
216 return arg == 0 ? mlen : 1;
218 case VS_OPCODE_PULL_CONSTANT_LOAD_GEN7:
219 return arg == 1 ? mlen : 1;
227 vec4_instruction::can_do_source_mods(const struct brw_device_info *devinfo)
229 if (devinfo->gen == 6 && is_math())
232 if (is_send_from_grf())
235 if (!backend_instruction::can_do_source_mods())
242 vec4_instruction::can_do_writemask(const struct brw_device_info *devinfo)
245 case SHADER_OPCODE_GEN4_SCRATCH_READ:
246 case VS_OPCODE_PULL_CONSTANT_LOAD:
247 case VS_OPCODE_PULL_CONSTANT_LOAD_GEN7:
248 case VS_OPCODE_SET_SIMD4X2_HEADER_GEN9:
249 case TCS_OPCODE_SET_INPUT_URB_OFFSETS:
250 case TCS_OPCODE_SET_OUTPUT_URB_OFFSETS:
251 case TES_OPCODE_CREATE_INPUT_READ_HEADER:
252 case TES_OPCODE_ADD_INDIRECT_URB_OFFSET:
253 case VEC4_OPCODE_URB_READ:
254 case SHADER_OPCODE_MOV_INDIRECT:
257 /* The MATH instruction on Gen6 only executes in align1 mode, which does
258 * not support writemasking.
260 if (devinfo->gen == 6 && is_math())
271 vec4_instruction::can_change_types() const
273 return dst.type == src[0].type &&
274 !src[0].abs && !src[0].negate && !saturate &&
275 (opcode == BRW_OPCODE_MOV ||
276 (opcode == BRW_OPCODE_SEL &&
277 dst.type == src[1].type &&
278 predicate != BRW_PREDICATE_NONE &&
279 !src[1].abs && !src[1].negate));
283 * Returns how many MRFs an opcode will write over.
285 * Note that this is not the 0 or 1 implied writes in an actual gen
286 * instruction -- the generate_* functions generate additional MOVs
290 vec4_visitor::implied_mrf_writes(vec4_instruction *inst)
292 if (inst->mlen == 0 || inst->is_send_from_grf())
295 switch (inst->opcode) {
296 case SHADER_OPCODE_RCP:
297 case SHADER_OPCODE_RSQ:
298 case SHADER_OPCODE_SQRT:
299 case SHADER_OPCODE_EXP2:
300 case SHADER_OPCODE_LOG2:
301 case SHADER_OPCODE_SIN:
302 case SHADER_OPCODE_COS:
304 case SHADER_OPCODE_INT_QUOTIENT:
305 case SHADER_OPCODE_INT_REMAINDER:
306 case SHADER_OPCODE_POW:
307 case TCS_OPCODE_THREAD_END:
309 case VS_OPCODE_URB_WRITE:
311 case VS_OPCODE_PULL_CONSTANT_LOAD:
313 case SHADER_OPCODE_GEN4_SCRATCH_READ:
315 case SHADER_OPCODE_GEN4_SCRATCH_WRITE:
317 case GS_OPCODE_URB_WRITE:
318 case GS_OPCODE_URB_WRITE_ALLOCATE:
319 case GS_OPCODE_THREAD_END:
321 case GS_OPCODE_FF_SYNC:
323 case TCS_OPCODE_URB_WRITE:
325 case SHADER_OPCODE_SHADER_TIME_ADD:
327 case SHADER_OPCODE_TEX:
328 case SHADER_OPCODE_TXL:
329 case SHADER_OPCODE_TXD:
330 case SHADER_OPCODE_TXF:
331 case SHADER_OPCODE_TXF_CMS:
332 case SHADER_OPCODE_TXF_CMS_W:
333 case SHADER_OPCODE_TXF_MCS:
334 case SHADER_OPCODE_TXS:
335 case SHADER_OPCODE_TG4:
336 case SHADER_OPCODE_TG4_OFFSET:
337 case SHADER_OPCODE_SAMPLEINFO:
338 case VS_OPCODE_GET_BUFFER_SIZE:
339 return inst->header_size;
341 unreachable("not reached");
346 src_reg::equals(const src_reg &r) const
348 return (this->backend_reg::equals(r) &&
349 !reladdr && !r.reladdr);
353 vec4_visitor::opt_vector_float()
355 bool progress = false;
357 foreach_block(block, cfg) {
358 int last_reg = -1, last_reg_offset = -1;
359 enum brw_reg_file last_reg_file = BAD_FILE;
361 uint8_t imm[4] = { 0 };
363 vec4_instruction *imm_inst[4];
364 unsigned writemask = 0;
365 enum brw_reg_type dest_type = BRW_REGISTER_TYPE_F;
367 foreach_inst_in_block_safe(vec4_instruction, inst, block) {
369 enum brw_reg_type need_type;
371 /* Look for unconditional MOVs from an immediate with a partial
372 * writemask. Skip type-conversion MOVs other than integer 0,
373 * where the type doesn't matter. See if the immediate can be
374 * represented as a VF.
376 if (inst->opcode == BRW_OPCODE_MOV &&
377 inst->src[0].file == IMM &&
378 inst->predicate == BRW_PREDICATE_NONE &&
379 inst->dst.writemask != WRITEMASK_XYZW &&
380 (inst->src[0].type == inst->dst.type || inst->src[0].d == 0)) {
382 vf = brw_float_to_vf(inst->src[0].d);
383 need_type = BRW_REGISTER_TYPE_D;
386 vf = brw_float_to_vf(inst->src[0].f);
387 need_type = BRW_REGISTER_TYPE_F;
393 /* If this wasn't a MOV, or the destination register doesn't match,
394 * or we have to switch destination types, then this breaks our
395 * sequence. Combine anything we've accumulated so far.
397 if (last_reg != inst->dst.nr ||
398 last_reg_offset != inst->dst.reg_offset ||
399 last_reg_file != inst->dst.file ||
400 (vf > 0 && dest_type != need_type)) {
402 if (inst_count > 1) {
404 memcpy(&vf, imm, sizeof(vf));
405 vec4_instruction *mov = MOV(imm_inst[0]->dst, brw_imm_vf(vf));
406 mov->dst.type = dest_type;
407 mov->dst.writemask = writemask;
408 inst->insert_before(block, mov);
410 for (int i = 0; i < inst_count; i++) {
411 imm_inst[i]->remove(block);
420 dest_type = BRW_REGISTER_TYPE_F;
422 for (int i = 0; i < 4; i++) {
427 /* Record this instruction's value (if it was representable). */
429 if ((inst->dst.writemask & WRITEMASK_X) != 0)
431 if ((inst->dst.writemask & WRITEMASK_Y) != 0)
433 if ((inst->dst.writemask & WRITEMASK_Z) != 0)
435 if ((inst->dst.writemask & WRITEMASK_W) != 0)
438 writemask |= inst->dst.writemask;
439 imm_inst[inst_count++] = inst;
441 last_reg = inst->dst.nr;
442 last_reg_offset = inst->dst.reg_offset;
443 last_reg_file = inst->dst.file;
445 dest_type = need_type;
451 invalidate_live_intervals();
456 /* Replaces unused channels of a swizzle with channels that are used.
458 * For instance, this pass transforms
460 * mov vgrf4.yz, vgrf5.wxzy
464 * mov vgrf4.yz, vgrf5.xxzx
466 * This eliminates false uses of some channels, letting dead code elimination
467 * remove the instructions that wrote them.
470 vec4_visitor::opt_reduce_swizzle()
472 bool progress = false;
474 foreach_block_and_inst_safe(block, vec4_instruction, inst, cfg) {
475 if (inst->dst.file == BAD_FILE ||
476 inst->dst.file == ARF ||
477 inst->dst.file == FIXED_GRF ||
478 inst->is_send_from_grf())
483 /* Determine which channels of the sources are read. */
484 switch (inst->opcode) {
485 case VEC4_OPCODE_PACK_BYTES:
487 case BRW_OPCODE_DPH: /* FINISHME: DPH reads only three channels of src0,
488 * but all four of src1.
490 swizzle = brw_swizzle_for_size(4);
493 swizzle = brw_swizzle_for_size(3);
496 swizzle = brw_swizzle_for_size(2);
499 swizzle = brw_swizzle_for_mask(inst->dst.writemask);
503 /* Update sources' swizzles. */
504 for (int i = 0; i < 3; i++) {
505 if (inst->src[i].file != VGRF &&
506 inst->src[i].file != ATTR &&
507 inst->src[i].file != UNIFORM)
510 const unsigned new_swizzle =
511 brw_compose_swizzle(swizzle, inst->src[i].swizzle);
512 if (inst->src[i].swizzle != new_swizzle) {
513 inst->src[i].swizzle = new_swizzle;
520 invalidate_live_intervals();
526 vec4_visitor::split_uniform_registers()
528 /* Prior to this, uniforms have been in an array sized according to
529 * the number of vector uniforms present, sparsely filled (so an
530 * aggregate results in reg indices being skipped over). Now we're
531 * going to cut those aggregates up so each .nr index is one
532 * vector. The goal is to make elimination of unused uniform
533 * components easier later.
535 foreach_block_and_inst(block, vec4_instruction, inst, cfg) {
536 for (int i = 0 ; i < 3; i++) {
537 if (inst->src[i].file != UNIFORM)
540 assert(!inst->src[i].reladdr);
542 inst->src[i].nr += inst->src[i].reg_offset;
543 inst->src[i].reg_offset = 0;
549 vec4_visitor::pack_uniform_registers()
551 uint8_t chans_used[this->uniforms];
552 int new_loc[this->uniforms];
553 int new_chan[this->uniforms];
555 memset(chans_used, 0, sizeof(chans_used));
556 memset(new_loc, 0, sizeof(new_loc));
557 memset(new_chan, 0, sizeof(new_chan));
559 /* Find which uniform vectors are actually used by the program. We
560 * expect unused vector elements when we've moved array access out
561 * to pull constants, and from some GLSL code generators like wine.
563 foreach_block_and_inst(block, vec4_instruction, inst, cfg) {
565 switch (inst->opcode) {
566 case VEC4_OPCODE_PACK_BYTES:
578 readmask = inst->dst.writemask;
582 for (int i = 0 ; i < 3; i++) {
583 if (inst->src[i].file != UNIFORM)
586 int reg = inst->src[i].nr;
587 for (int c = 0; c < 4; c++) {
588 if (!(readmask & (1 << c)))
591 chans_used[reg] = MAX2(chans_used[reg],
592 BRW_GET_SWZ(inst->src[i].swizzle, c) + 1);
596 if (inst->opcode == SHADER_OPCODE_MOV_INDIRECT &&
597 inst->src[0].file == UNIFORM) {
598 assert(inst->src[2].file == BRW_IMMEDIATE_VALUE);
599 assert(inst->src[0].subnr == 0);
601 unsigned bytes_read = inst->src[2].ud;
602 assert(bytes_read % 4 == 0);
603 unsigned vec4s_read = DIV_ROUND_UP(bytes_read, 16);
605 /* We just mark every register touched by a MOV_INDIRECT as being
606 * fully used. This ensures that it doesn't broken up piecewise by
607 * the next part of our packing algorithm.
609 int reg = inst->src[0].nr;
610 for (unsigned i = 0; i < vec4s_read; i++)
611 chans_used[reg + i] = 4;
615 int new_uniform_count = 0;
617 /* Now, figure out a packing of the live uniform vectors into our
620 for (int src = 0; src < uniforms; src++) {
621 int size = chans_used[src];
627 /* Find the lowest place we can slot this uniform in. */
628 for (dst = 0; dst < src; dst++) {
629 if (chans_used[dst] + size <= 4)
638 new_chan[src] = chans_used[dst];
640 /* Move the references to the data */
641 for (int j = 0; j < size; j++) {
642 stage_prog_data->param[dst * 4 + new_chan[src] + j] =
643 stage_prog_data->param[src * 4 + j];
646 chans_used[dst] += size;
650 new_uniform_count = MAX2(new_uniform_count, dst + 1);
653 this->uniforms = new_uniform_count;
655 /* Now, update the instructions for our repacked uniforms. */
656 foreach_block_and_inst(block, vec4_instruction, inst, cfg) {
657 for (int i = 0 ; i < 3; i++) {
658 int src = inst->src[i].nr;
660 if (inst->src[i].file != UNIFORM)
663 inst->src[i].nr = new_loc[src];
664 inst->src[i].swizzle += BRW_SWIZZLE4(new_chan[src], new_chan[src],
665 new_chan[src], new_chan[src]);
671 * Does algebraic optimizations (0 * a = 0, 1 * a = a, a + 0 = a).
673 * While GLSL IR also performs this optimization, we end up with it in
674 * our instruction stream for a couple of reasons. One is that we
675 * sometimes generate silly instructions, for example in array access
676 * where we'll generate "ADD offset, index, base" even if base is 0.
677 * The other is that GLSL IR's constant propagation doesn't track the
678 * components of aggregates, so some VS patterns (initialize matrix to
679 * 0, accumulate in vertex blending factors) end up breaking down to
680 * instructions involving 0.
683 vec4_visitor::opt_algebraic()
685 bool progress = false;
687 foreach_block_and_inst(block, vec4_instruction, inst, cfg) {
688 switch (inst->opcode) {
690 if (inst->src[0].file != IMM)
693 if (inst->saturate) {
694 if (inst->dst.type != inst->src[0].type)
695 assert(!"unimplemented: saturate mixed types");
697 if (brw_saturate_immediate(inst->dst.type,
698 &inst->src[0].as_brw_reg())) {
699 inst->saturate = false;
705 case VEC4_OPCODE_UNPACK_UNIFORM:
706 if (inst->src[0].file != UNIFORM) {
707 inst->opcode = BRW_OPCODE_MOV;
713 if (inst->src[1].is_zero()) {
714 inst->opcode = BRW_OPCODE_MOV;
715 inst->src[1] = src_reg();
721 if (inst->src[1].is_zero()) {
722 inst->opcode = BRW_OPCODE_MOV;
723 switch (inst->src[0].type) {
724 case BRW_REGISTER_TYPE_F:
725 inst->src[0] = brw_imm_f(0.0f);
727 case BRW_REGISTER_TYPE_D:
728 inst->src[0] = brw_imm_d(0);
730 case BRW_REGISTER_TYPE_UD:
731 inst->src[0] = brw_imm_ud(0u);
734 unreachable("not reached");
736 inst->src[1] = src_reg();
738 } else if (inst->src[1].is_one()) {
739 inst->opcode = BRW_OPCODE_MOV;
740 inst->src[1] = src_reg();
742 } else if (inst->src[1].is_negative_one()) {
743 inst->opcode = BRW_OPCODE_MOV;
744 inst->src[0].negate = !inst->src[0].negate;
745 inst->src[1] = src_reg();
750 if (inst->conditional_mod == BRW_CONDITIONAL_GE &&
752 inst->src[0].negate &&
753 inst->src[1].is_zero()) {
754 inst->src[0].abs = false;
755 inst->src[0].negate = false;
756 inst->conditional_mod = BRW_CONDITIONAL_Z;
761 case SHADER_OPCODE_BROADCAST:
762 if (is_uniform(inst->src[0]) ||
763 inst->src[1].is_zero()) {
764 inst->opcode = BRW_OPCODE_MOV;
765 inst->src[1] = src_reg();
766 inst->force_writemask_all = true;
777 invalidate_live_intervals();
783 * Only a limited number of hardware registers may be used for push
784 * constants, so this turns access to the overflowed constants into
788 vec4_visitor::move_push_constants_to_pull_constants()
790 int pull_constant_loc[this->uniforms];
792 /* Only allow 32 registers (256 uniform components) as push constants,
793 * which is the limit on gen6.
795 * If changing this value, note the limitation about total_regs in
798 int max_uniform_components = 32 * 8;
799 if (this->uniforms * 4 <= max_uniform_components)
802 /* Make some sort of choice as to which uniforms get sent to pull
803 * constants. We could potentially do something clever here like
804 * look for the most infrequently used uniform vec4s, but leave
807 for (int i = 0; i < this->uniforms * 4; i += 4) {
808 pull_constant_loc[i / 4] = -1;
810 if (i >= max_uniform_components) {
811 const gl_constant_value **values = &stage_prog_data->param[i];
813 /* Try to find an existing copy of this uniform in the pull
814 * constants if it was part of an array access already.
816 for (unsigned int j = 0; j < stage_prog_data->nr_pull_params; j += 4) {
819 for (matches = 0; matches < 4; matches++) {
820 if (stage_prog_data->pull_param[j + matches] != values[matches])
825 pull_constant_loc[i / 4] = j / 4;
830 if (pull_constant_loc[i / 4] == -1) {
831 assert(stage_prog_data->nr_pull_params % 4 == 0);
832 pull_constant_loc[i / 4] = stage_prog_data->nr_pull_params / 4;
834 for (int j = 0; j < 4; j++) {
835 stage_prog_data->pull_param[stage_prog_data->nr_pull_params++] =
842 /* Now actually rewrite usage of the things we've moved to pull
845 foreach_block_and_inst_safe(block, vec4_instruction, inst, cfg) {
846 for (int i = 0 ; i < 3; i++) {
847 if (inst->src[i].file != UNIFORM ||
848 pull_constant_loc[inst->src[i].nr] == -1)
851 int uniform = inst->src[i].nr;
853 dst_reg temp = dst_reg(this, glsl_type::vec4_type);
855 emit_pull_constant_load(block, inst, temp, inst->src[i],
856 pull_constant_loc[uniform], src_reg());
858 inst->src[i].file = temp.file;
859 inst->src[i].nr = temp.nr;
860 inst->src[i].reg_offset = temp.reg_offset;
861 inst->src[i].reladdr = NULL;
865 /* Repack push constants to remove the now-unused ones. */
866 pack_uniform_registers();
869 /* Conditions for which we want to avoid setting the dependency control bits */
871 vec4_visitor::is_dep_ctrl_unsafe(const vec4_instruction *inst)
873 #define IS_DWORD(reg) \
874 (reg.type == BRW_REGISTER_TYPE_UD || \
875 reg.type == BRW_REGISTER_TYPE_D)
877 /* "When source or destination datatype is 64b or operation is integer DWord
878 * multiply, DepCtrl must not be used."
879 * May apply to future SoCs as well.
881 if (devinfo->is_cherryview) {
882 if (inst->opcode == BRW_OPCODE_MUL &&
883 IS_DWORD(inst->src[0]) &&
884 IS_DWORD(inst->src[1]))
889 if (devinfo->gen >= 8) {
890 if (inst->opcode == BRW_OPCODE_F32TO16)
896 * In the presence of send messages, totally interrupt dependency
897 * control. They're long enough that the chance of dependency
898 * control around them just doesn't matter.
901 * From the Ivy Bridge PRM, volume 4 part 3.7, page 80:
902 * When a sequence of NoDDChk and NoDDClr are used, the last instruction that
903 * completes the scoreboard clear must have a non-zero execution mask. This
904 * means, if any kind of predication can change the execution mask or channel
905 * enable of the last instruction, the optimization must be avoided. This is
906 * to avoid instructions being shot down the pipeline when no writes are
910 * Dependency control does not work well over math instructions.
911 * NB: Discovered empirically
913 return (inst->mlen || inst->predicate || inst->is_math());
917 * Sets the dependency control fields on instructions after register
918 * allocation and before the generator is run.
920 * When you have a sequence of instructions like:
922 * DP4 temp.x vertex uniform[0]
923 * DP4 temp.y vertex uniform[0]
924 * DP4 temp.z vertex uniform[0]
925 * DP4 temp.w vertex uniform[0]
927 * The hardware doesn't know that it can actually run the later instructions
928 * while the previous ones are in flight, producing stalls. However, we have
929 * manual fields we can set in the instructions that let it do so.
932 vec4_visitor::opt_set_dependency_control()
934 vec4_instruction *last_grf_write[BRW_MAX_GRF];
935 uint8_t grf_channels_written[BRW_MAX_GRF];
936 vec4_instruction *last_mrf_write[BRW_MAX_GRF];
937 uint8_t mrf_channels_written[BRW_MAX_GRF];
939 assert(prog_data->total_grf ||
940 !"Must be called after register allocation");
942 foreach_block (block, cfg) {
943 memset(last_grf_write, 0, sizeof(last_grf_write));
944 memset(last_mrf_write, 0, sizeof(last_mrf_write));
946 foreach_inst_in_block (vec4_instruction, inst, block) {
947 /* If we read from a register that we were doing dependency control
948 * on, don't do dependency control across the read.
950 for (int i = 0; i < 3; i++) {
951 int reg = inst->src[i].nr + inst->src[i].reg_offset;
952 if (inst->src[i].file == VGRF) {
953 last_grf_write[reg] = NULL;
954 } else if (inst->src[i].file == FIXED_GRF) {
955 memset(last_grf_write, 0, sizeof(last_grf_write));
958 assert(inst->src[i].file != MRF);
961 if (is_dep_ctrl_unsafe(inst)) {
962 memset(last_grf_write, 0, sizeof(last_grf_write));
963 memset(last_mrf_write, 0, sizeof(last_mrf_write));
967 /* Now, see if we can do dependency control for this instruction
968 * against a previous one writing to its destination.
970 int reg = inst->dst.nr + inst->dst.reg_offset;
971 if (inst->dst.file == VGRF || inst->dst.file == FIXED_GRF) {
972 if (last_grf_write[reg] &&
973 !(inst->dst.writemask & grf_channels_written[reg])) {
974 last_grf_write[reg]->no_dd_clear = true;
975 inst->no_dd_check = true;
977 grf_channels_written[reg] = 0;
980 last_grf_write[reg] = inst;
981 grf_channels_written[reg] |= inst->dst.writemask;
982 } else if (inst->dst.file == MRF) {
983 if (last_mrf_write[reg] &&
984 !(inst->dst.writemask & mrf_channels_written[reg])) {
985 last_mrf_write[reg]->no_dd_clear = true;
986 inst->no_dd_check = true;
988 mrf_channels_written[reg] = 0;
991 last_mrf_write[reg] = inst;
992 mrf_channels_written[reg] |= inst->dst.writemask;
999 vec4_instruction::can_reswizzle(const struct brw_device_info *devinfo,
1004 /* Gen6 MATH instructions can not execute in align16 mode, so swizzles
1007 if (devinfo->gen == 6 && is_math() && swizzle != BRW_SWIZZLE_XYZW)
1010 if (!can_do_writemask(devinfo) && dst_writemask != WRITEMASK_XYZW)
1013 /* If this instruction sets anything not referenced by swizzle, then we'd
1014 * totally break it when we reswizzle.
1016 if (dst.writemask & ~swizzle_mask)
1022 for (int i = 0; i < 3; i++) {
1023 if (src[i].is_accumulator())
1031 * For any channels in the swizzle's source that were populated by this
1032 * instruction, rewrite the instruction to put the appropriate result directly
1033 * in those channels.
1035 * e.g. for swizzle=yywx, MUL a.xy b c -> MUL a.yy_x b.yy z.yy_x
1038 vec4_instruction::reswizzle(int dst_writemask, int swizzle)
1040 /* Destination write mask doesn't correspond to source swizzle for the dot
1041 * product and pack_bytes instructions.
1043 if (opcode != BRW_OPCODE_DP4 && opcode != BRW_OPCODE_DPH &&
1044 opcode != BRW_OPCODE_DP3 && opcode != BRW_OPCODE_DP2 &&
1045 opcode != VEC4_OPCODE_PACK_BYTES) {
1046 for (int i = 0; i < 3; i++) {
1047 if (src[i].file == BAD_FILE || src[i].file == IMM)
1050 src[i].swizzle = brw_compose_swizzle(swizzle, src[i].swizzle);
1054 /* Apply the specified swizzle and writemask to the original mask of
1055 * written components.
1057 dst.writemask = dst_writemask &
1058 brw_apply_swizzle_to_mask(swizzle, dst.writemask);
1062 * Tries to reduce extra MOV instructions by taking temporary GRFs that get
1063 * just written and then MOVed into another reg and making the original write
1064 * of the GRF write directly to the final destination instead.
1067 vec4_visitor::opt_register_coalesce()
1069 bool progress = false;
1072 calculate_live_intervals();
1074 foreach_block_and_inst_safe (block, vec4_instruction, inst, cfg) {
1078 if (inst->opcode != BRW_OPCODE_MOV ||
1079 (inst->dst.file != VGRF && inst->dst.file != MRF) ||
1081 inst->src[0].file != VGRF ||
1082 inst->dst.type != inst->src[0].type ||
1083 inst->src[0].abs || inst->src[0].negate || inst->src[0].reladdr)
1086 /* Remove no-op MOVs */
1087 if (inst->dst.file == inst->src[0].file &&
1088 inst->dst.nr == inst->src[0].nr &&
1089 inst->dst.reg_offset == inst->src[0].reg_offset) {
1090 bool is_nop_mov = true;
1092 for (unsigned c = 0; c < 4; c++) {
1093 if ((inst->dst.writemask & (1 << c)) == 0)
1096 if (BRW_GET_SWZ(inst->src[0].swizzle, c) != c) {
1103 inst->remove(block);
1109 bool to_mrf = (inst->dst.file == MRF);
1111 /* Can't coalesce this GRF if someone else was going to
1114 if (var_range_end(var_from_reg(alloc, inst->src[0]), 4) > ip)
1117 /* We need to check interference with the final destination between this
1118 * instruction and the earliest instruction involved in writing the GRF
1119 * we're eliminating. To do that, keep track of which of our source
1120 * channels we've seen initialized.
1122 const unsigned chans_needed =
1123 brw_apply_inv_swizzle_to_mask(inst->src[0].swizzle,
1124 inst->dst.writemask);
1125 unsigned chans_remaining = chans_needed;
1127 /* Now walk up the instruction stream trying to see if we can rewrite
1128 * everything writing to the temporary to write into the destination
1131 vec4_instruction *_scan_inst = (vec4_instruction *)inst->prev;
1132 foreach_inst_in_block_reverse_starting_from(vec4_instruction, scan_inst,
1134 _scan_inst = scan_inst;
1136 if (inst->src[0].in_range(scan_inst->dst, scan_inst->regs_written)) {
1137 /* Found something writing to the reg we want to coalesce away. */
1139 /* SEND instructions can't have MRF as a destination. */
1140 if (scan_inst->mlen)
1143 if (devinfo->gen == 6) {
1144 /* gen6 math instructions must have the destination be
1145 * VGRF, so no compute-to-MRF for them.
1147 if (scan_inst->is_math()) {
1153 /* This doesn't handle saturation on the instruction we
1154 * want to coalesce away if the register types do not match.
1155 * But if scan_inst is a non type-converting 'mov', we can fix
1158 if (inst->saturate &&
1159 inst->dst.type != scan_inst->dst.type &&
1160 !(scan_inst->opcode == BRW_OPCODE_MOV &&
1161 scan_inst->dst.type == scan_inst->src[0].type))
1164 /* If we can't handle the swizzle, bail. */
1165 if (!scan_inst->can_reswizzle(devinfo, inst->dst.writemask,
1166 inst->src[0].swizzle,
1171 /* This doesn't handle coalescing of multiple registers. */
1172 if (scan_inst->regs_written > 1)
1175 /* Mark which channels we found unconditional writes for. */
1176 if (!scan_inst->predicate)
1177 chans_remaining &= ~scan_inst->dst.writemask;
1179 if (chans_remaining == 0)
1183 /* You can't read from an MRF, so if someone else reads our MRF's
1184 * source GRF that we wanted to rewrite, that stops us. If it's a
1185 * GRF we're trying to coalesce to, we don't actually handle
1186 * rewriting sources so bail in that case as well.
1188 bool interfered = false;
1189 for (int i = 0; i < 3; i++) {
1190 if (inst->src[0].in_range(scan_inst->src[i],
1191 scan_inst->regs_read(i)))
1197 /* If somebody else writes the same channels of our destination here,
1198 * we can't coalesce before that.
1200 if (inst->dst.in_range(scan_inst->dst, scan_inst->regs_written) &&
1201 (inst->dst.writemask & scan_inst->dst.writemask) != 0) {
1205 /* Check for reads of the register we're trying to coalesce into. We
1206 * can't go rewriting instructions above that to put some other value
1207 * in the register instead.
1209 if (to_mrf && scan_inst->mlen > 0) {
1210 if (inst->dst.nr >= scan_inst->base_mrf &&
1211 inst->dst.nr < scan_inst->base_mrf + scan_inst->mlen) {
1215 for (int i = 0; i < 3; i++) {
1216 if (inst->dst.in_range(scan_inst->src[i],
1217 scan_inst->regs_read(i)))
1225 if (chans_remaining == 0) {
1226 /* If we've made it here, we have an MOV we want to coalesce out, and
1227 * a scan_inst pointing to the earliest instruction involved in
1228 * computing the value. Now go rewrite the instruction stream
1231 vec4_instruction *scan_inst = _scan_inst;
1232 while (scan_inst != inst) {
1233 if (scan_inst->dst.file == VGRF &&
1234 scan_inst->dst.nr == inst->src[0].nr &&
1235 scan_inst->dst.reg_offset == inst->src[0].reg_offset) {
1236 scan_inst->reswizzle(inst->dst.writemask,
1237 inst->src[0].swizzle);
1238 scan_inst->dst.file = inst->dst.file;
1239 scan_inst->dst.nr = inst->dst.nr;
1240 scan_inst->dst.reg_offset = inst->dst.reg_offset;
1241 if (inst->saturate &&
1242 inst->dst.type != scan_inst->dst.type) {
1243 /* If we have reached this point, scan_inst is a non
1244 * type-converting 'mov' and we can modify its register types
1245 * to match the ones in inst. Otherwise, we could have an
1246 * incorrect saturation result.
1248 scan_inst->dst.type = inst->dst.type;
1249 scan_inst->src[0].type = inst->src[0].type;
1251 scan_inst->saturate |= inst->saturate;
1253 scan_inst = (vec4_instruction *)scan_inst->next;
1255 inst->remove(block);
1261 invalidate_live_intervals();
1267 * Eliminate FIND_LIVE_CHANNEL instructions occurring outside any control
1268 * flow. We could probably do better here with some form of divergence
1272 vec4_visitor::eliminate_find_live_channel()
1274 bool progress = false;
1277 foreach_block_and_inst_safe(block, vec4_instruction, inst, cfg) {
1278 switch (inst->opcode) {
1284 case BRW_OPCODE_ENDIF:
1285 case BRW_OPCODE_WHILE:
1289 case SHADER_OPCODE_FIND_LIVE_CHANNEL:
1291 inst->opcode = BRW_OPCODE_MOV;
1292 inst->src[0] = brw_imm_d(0);
1293 inst->force_writemask_all = true;
1307 * Splits virtual GRFs requesting more than one contiguous physical register.
1309 * We initially create large virtual GRFs for temporary structures, arrays,
1310 * and matrices, so that the dereference visitor functions can add reg_offsets
1311 * to work their way down to the actual member being accessed. But when it
1312 * comes to optimization, we'd like to treat each register as individual
1313 * storage if possible.
1315 * So far, the only thing that might prevent splitting is a send message from
1319 vec4_visitor::split_virtual_grfs()
1321 int num_vars = this->alloc.count;
1322 int new_virtual_grf[num_vars];
1323 bool split_grf[num_vars];
1325 memset(new_virtual_grf, 0, sizeof(new_virtual_grf));
1327 /* Try to split anything > 0 sized. */
1328 for (int i = 0; i < num_vars; i++) {
1329 split_grf[i] = this->alloc.sizes[i] != 1;
1332 /* Check that the instructions are compatible with the registers we're trying
1335 foreach_block_and_inst(block, vec4_instruction, inst, cfg) {
1336 if (inst->dst.file == VGRF && inst->regs_written > 1)
1337 split_grf[inst->dst.nr] = false;
1339 for (int i = 0; i < 3; i++) {
1340 if (inst->src[i].file == VGRF && inst->regs_read(i) > 1)
1341 split_grf[inst->src[i].nr] = false;
1345 /* Allocate new space for split regs. Note that the virtual
1346 * numbers will be contiguous.
1348 for (int i = 0; i < num_vars; i++) {
1352 new_virtual_grf[i] = alloc.allocate(1);
1353 for (unsigned j = 2; j < this->alloc.sizes[i]; j++) {
1354 unsigned reg = alloc.allocate(1);
1355 assert(reg == new_virtual_grf[i] + j - 1);
1358 this->alloc.sizes[i] = 1;
1361 foreach_block_and_inst(block, vec4_instruction, inst, cfg) {
1362 if (inst->dst.file == VGRF && split_grf[inst->dst.nr] &&
1363 inst->dst.reg_offset != 0) {
1364 inst->dst.nr = (new_virtual_grf[inst->dst.nr] +
1365 inst->dst.reg_offset - 1);
1366 inst->dst.reg_offset = 0;
1368 for (int i = 0; i < 3; i++) {
1369 if (inst->src[i].file == VGRF && split_grf[inst->src[i].nr] &&
1370 inst->src[i].reg_offset != 0) {
1371 inst->src[i].nr = (new_virtual_grf[inst->src[i].nr] +
1372 inst->src[i].reg_offset - 1);
1373 inst->src[i].reg_offset = 0;
1377 invalidate_live_intervals();
1381 vec4_visitor::dump_instruction(backend_instruction *be_inst)
1383 dump_instruction(be_inst, stderr);
1387 vec4_visitor::dump_instruction(backend_instruction *be_inst, FILE *file)
1389 vec4_instruction *inst = (vec4_instruction *)be_inst;
1391 if (inst->predicate) {
1392 fprintf(file, "(%cf0.%d%s) ",
1393 inst->predicate_inverse ? '-' : '+',
1395 pred_ctrl_align16[inst->predicate]);
1398 fprintf(file, "%s", brw_instruction_name(devinfo, inst->opcode));
1400 fprintf(file, ".sat");
1401 if (inst->conditional_mod) {
1402 fprintf(file, "%s", conditional_modifier[inst->conditional_mod]);
1403 if (!inst->predicate &&
1404 (devinfo->gen < 5 || (inst->opcode != BRW_OPCODE_SEL &&
1405 inst->opcode != BRW_OPCODE_IF &&
1406 inst->opcode != BRW_OPCODE_WHILE))) {
1407 fprintf(file, ".f0.%d", inst->flag_subreg);
1412 switch (inst->dst.file) {
1414 fprintf(file, "vgrf%d.%d", inst->dst.nr, inst->dst.reg_offset);
1417 fprintf(file, "g%d", inst->dst.nr);
1420 fprintf(file, "m%d", inst->dst.nr);
1423 switch (inst->dst.nr) {
1425 fprintf(file, "null");
1427 case BRW_ARF_ADDRESS:
1428 fprintf(file, "a0.%d", inst->dst.subnr);
1430 case BRW_ARF_ACCUMULATOR:
1431 fprintf(file, "acc%d", inst->dst.subnr);
1434 fprintf(file, "f%d.%d", inst->dst.nr & 0xf, inst->dst.subnr);
1437 fprintf(file, "arf%d.%d", inst->dst.nr & 0xf, inst->dst.subnr);
1440 if (inst->dst.subnr)
1441 fprintf(file, "+%d", inst->dst.subnr);
1444 fprintf(file, "(null)");
1449 unreachable("not reached");
1451 if (inst->dst.writemask != WRITEMASK_XYZW) {
1453 if (inst->dst.writemask & 1)
1455 if (inst->dst.writemask & 2)
1457 if (inst->dst.writemask & 4)
1459 if (inst->dst.writemask & 8)
1462 fprintf(file, ":%s", brw_reg_type_letters(inst->dst.type));
1464 if (inst->src[0].file != BAD_FILE)
1465 fprintf(file, ", ");
1467 for (int i = 0; i < 3 && inst->src[i].file != BAD_FILE; i++) {
1468 if (inst->src[i].negate)
1470 if (inst->src[i].abs)
1472 switch (inst->src[i].file) {
1474 fprintf(file, "vgrf%d", inst->src[i].nr);
1477 fprintf(file, "g%d", inst->src[i].nr);
1480 fprintf(file, "attr%d", inst->src[i].nr);
1483 fprintf(file, "u%d", inst->src[i].nr);
1486 switch (inst->src[i].type) {
1487 case BRW_REGISTER_TYPE_F:
1488 fprintf(file, "%fF", inst->src[i].f);
1490 case BRW_REGISTER_TYPE_D:
1491 fprintf(file, "%dD", inst->src[i].d);
1493 case BRW_REGISTER_TYPE_UD:
1494 fprintf(file, "%uU", inst->src[i].ud);
1496 case BRW_REGISTER_TYPE_VF:
1497 fprintf(file, "[%-gF, %-gF, %-gF, %-gF]",
1498 brw_vf_to_float((inst->src[i].ud >> 0) & 0xff),
1499 brw_vf_to_float((inst->src[i].ud >> 8) & 0xff),
1500 brw_vf_to_float((inst->src[i].ud >> 16) & 0xff),
1501 brw_vf_to_float((inst->src[i].ud >> 24) & 0xff));
1504 fprintf(file, "???");
1509 switch (inst->src[i].nr) {
1511 fprintf(file, "null");
1513 case BRW_ARF_ADDRESS:
1514 fprintf(file, "a0.%d", inst->src[i].subnr);
1516 case BRW_ARF_ACCUMULATOR:
1517 fprintf(file, "acc%d", inst->src[i].subnr);
1520 fprintf(file, "f%d.%d", inst->src[i].nr & 0xf, inst->src[i].subnr);
1523 fprintf(file, "arf%d.%d", inst->src[i].nr & 0xf, inst->src[i].subnr);
1526 if (inst->src[i].subnr)
1527 fprintf(file, "+%d", inst->src[i].subnr);
1530 fprintf(file, "(null)");
1533 unreachable("not reached");
1536 /* Don't print .0; and only VGRFs have reg_offsets and sizes */
1537 if (inst->src[i].reg_offset != 0 &&
1538 inst->src[i].file == VGRF &&
1539 alloc.sizes[inst->src[i].nr] != 1)
1540 fprintf(file, ".%d", inst->src[i].reg_offset);
1542 if (inst->src[i].file != IMM) {
1543 static const char *chans[4] = {"x", "y", "z", "w"};
1545 for (int c = 0; c < 4; c++) {
1546 fprintf(file, "%s", chans[BRW_GET_SWZ(inst->src[i].swizzle, c)]);
1550 if (inst->src[i].abs)
1553 if (inst->src[i].file != IMM) {
1554 fprintf(file, ":%s", brw_reg_type_letters(inst->src[i].type));
1557 if (i < 2 && inst->src[i + 1].file != BAD_FILE)
1558 fprintf(file, ", ");
1561 if (inst->force_writemask_all)
1562 fprintf(file, " NoMask");
1564 fprintf(file, "\n");
1568 static inline struct brw_reg
1569 attribute_to_hw_reg(int attr, bool interleaved)
1572 return stride(brw_vec4_grf(attr / 2, (attr % 2) * 4), 0, 4, 1);
1574 return brw_vec8_grf(attr, 0);
1579 * Replace each register of type ATTR in this->instructions with a reference
1580 * to a fixed HW register.
1582 * If interleaved is true, then each attribute takes up half a register, with
1583 * register N containing attribute 2*N in its first half and attribute 2*N+1
1584 * in its second half (this corresponds to the payload setup used by geometry
1585 * shaders in "single" or "dual instanced" dispatch mode). If interleaved is
1586 * false, then each attribute takes up a whole register, with register N
1587 * containing attribute N (this corresponds to the payload setup used by
1588 * vertex shaders, and by geometry shaders in "dual object" dispatch mode).
1591 vec4_visitor::lower_attributes_to_hw_regs(const int *attribute_map,
1594 foreach_block_and_inst(block, vec4_instruction, inst, cfg) {
1595 for (int i = 0; i < 3; i++) {
1596 if (inst->src[i].file != ATTR)
1599 int grf = attribute_map[inst->src[i].nr + inst->src[i].reg_offset];
1601 /* All attributes used in the shader need to have been assigned a
1602 * hardware register by the caller
1606 struct brw_reg reg = attribute_to_hw_reg(grf, interleaved);
1607 reg.swizzle = inst->src[i].swizzle;
1608 reg.type = inst->src[i].type;
1609 if (inst->src[i].abs)
1611 if (inst->src[i].negate)
1620 vec4_vs_visitor::setup_attributes(int payload_reg)
1623 int attribute_map[VERT_ATTRIB_MAX + 2];
1624 memset(attribute_map, 0, sizeof(attribute_map));
1627 for (int i = 0; i < VERT_ATTRIB_MAX; i++) {
1628 if (vs_prog_data->inputs_read & BITFIELD64_BIT(i)) {
1629 attribute_map[i] = payload_reg + nr_attributes;
1634 /* VertexID is stored by the VF as the last vertex element, but we
1635 * don't represent it with a flag in inputs_read, so we call it
1638 if (vs_prog_data->uses_vertexid || vs_prog_data->uses_instanceid ||
1639 vs_prog_data->uses_basevertex || vs_prog_data->uses_baseinstance) {
1640 attribute_map[VERT_ATTRIB_MAX] = payload_reg + nr_attributes;
1644 if (vs_prog_data->uses_drawid) {
1645 attribute_map[VERT_ATTRIB_MAX + 1] = payload_reg + nr_attributes;
1649 lower_attributes_to_hw_regs(attribute_map, false /* interleaved */);
1651 return payload_reg + vs_prog_data->nr_attributes;
1655 vec4_visitor::setup_uniforms(int reg)
1657 prog_data->base.dispatch_grf_start_reg = reg;
1659 /* The pre-gen6 VS requires that some push constants get loaded no
1660 * matter what, or the GPU would hang.
1662 if (devinfo->gen < 6 && this->uniforms == 0) {
1663 stage_prog_data->param =
1664 reralloc(NULL, stage_prog_data->param, const gl_constant_value *, 4);
1665 for (unsigned int i = 0; i < 4; i++) {
1666 unsigned int slot = this->uniforms * 4 + i;
1667 static gl_constant_value zero = { 0.0 };
1668 stage_prog_data->param[slot] = &zero;
1674 reg += ALIGN(uniforms, 2) / 2;
1677 stage_prog_data->nr_params = this->uniforms * 4;
1679 prog_data->base.curb_read_length =
1680 reg - prog_data->base.dispatch_grf_start_reg;
1686 vec4_vs_visitor::setup_payload(void)
1690 /* The payload always contains important data in g0, which contains
1691 * the URB handles that are passed on to the URB write at the end
1692 * of the thread. So, we always start push constants at g1.
1696 reg = setup_uniforms(reg);
1698 reg = setup_attributes(reg);
1700 this->first_non_payload_grf = reg;
1704 vec4_visitor::lower_minmax()
1706 assert(devinfo->gen < 6);
1708 bool progress = false;
1710 foreach_block_and_inst_safe(block, vec4_instruction, inst, cfg) {
1711 const vec4_builder ibld(this, block, inst);
1713 if (inst->opcode == BRW_OPCODE_SEL &&
1714 inst->predicate == BRW_PREDICATE_NONE) {
1715 /* FIXME: Using CMP doesn't preserve the NaN propagation semantics of
1716 * the original SEL.L/GE instruction
1718 ibld.CMP(ibld.null_reg_d(), inst->src[0], inst->src[1],
1719 inst->conditional_mod);
1720 inst->predicate = BRW_PREDICATE_NORMAL;
1721 inst->conditional_mod = BRW_CONDITIONAL_NONE;
1728 invalidate_live_intervals();
1734 vec4_visitor::get_timestamp()
1736 assert(devinfo->gen >= 7);
1738 src_reg ts = src_reg(brw_reg(BRW_ARCHITECTURE_REGISTER_FILE,
1743 BRW_REGISTER_TYPE_UD,
1744 BRW_VERTICAL_STRIDE_0,
1746 BRW_HORIZONTAL_STRIDE_4,
1750 dst_reg dst = dst_reg(this, glsl_type::uvec4_type);
1752 vec4_instruction *mov = emit(MOV(dst, ts));
1753 /* We want to read the 3 fields we care about (mostly field 0, but also 2)
1754 * even if it's not enabled in the dispatch.
1756 mov->force_writemask_all = true;
1758 return src_reg(dst);
1762 vec4_visitor::emit_shader_time_begin()
1764 current_annotation = "shader time start";
1765 shader_start_time = get_timestamp();
1769 vec4_visitor::emit_shader_time_end()
1771 current_annotation = "shader time end";
1772 src_reg shader_end_time = get_timestamp();
1775 /* Check that there weren't any timestamp reset events (assuming these
1776 * were the only two timestamp reads that happened).
1778 src_reg reset_end = shader_end_time;
1779 reset_end.swizzle = BRW_SWIZZLE_ZZZZ;
1780 vec4_instruction *test = emit(AND(dst_null_ud(), reset_end, brw_imm_ud(1u)));
1781 test->conditional_mod = BRW_CONDITIONAL_Z;
1783 emit(IF(BRW_PREDICATE_NORMAL));
1785 /* Take the current timestamp and get the delta. */
1786 shader_start_time.negate = true;
1787 dst_reg diff = dst_reg(this, glsl_type::uint_type);
1788 emit(ADD(diff, shader_start_time, shader_end_time));
1790 /* If there were no instructions between the two timestamp gets, the diff
1791 * is 2 cycles. Remove that overhead, so I can forget about that when
1792 * trying to determine the time taken for single instructions.
1794 emit(ADD(diff, src_reg(diff), brw_imm_ud(-2u)));
1796 emit_shader_time_write(0, src_reg(diff));
1797 emit_shader_time_write(1, brw_imm_ud(1u));
1798 emit(BRW_OPCODE_ELSE);
1799 emit_shader_time_write(2, brw_imm_ud(1u));
1800 emit(BRW_OPCODE_ENDIF);
1804 vec4_visitor::emit_shader_time_write(int shader_time_subindex, src_reg value)
1807 dst_reg(this, glsl_type::get_array_instance(glsl_type::vec4_type, 2));
1809 dst_reg offset = dst;
1813 offset.type = BRW_REGISTER_TYPE_UD;
1814 int index = shader_time_index * 3 + shader_time_subindex;
1815 emit(MOV(offset, brw_imm_d(index * SHADER_TIME_STRIDE)));
1817 time.type = BRW_REGISTER_TYPE_UD;
1818 emit(MOV(time, value));
1820 vec4_instruction *inst =
1821 emit(SHADER_OPCODE_SHADER_TIME_ADD, dst_reg(), src_reg(dst));
1826 vec4_visitor::convert_to_hw_regs()
1828 foreach_block_and_inst(block, vec4_instruction, inst, cfg) {
1829 for (int i = 0; i < 3; i++) {
1830 struct src_reg &src = inst->src[i];
1834 reg = brw_vec8_grf(src.nr + src.reg_offset, 0);
1835 reg.type = src.type;
1836 reg.swizzle = src.swizzle;
1838 reg.negate = src.negate;
1842 reg = stride(brw_vec4_grf(prog_data->base.dispatch_grf_start_reg +
1843 (src.nr + src.reg_offset) / 2,
1844 ((src.nr + src.reg_offset) % 2) * 4),
1846 reg.type = src.type;
1847 reg.swizzle = src.swizzle;
1849 reg.negate = src.negate;
1851 /* This should have been moved to pull constants. */
1852 assert(!src.reladdr);
1861 /* Probably unused. */
1862 reg = brw_null_reg();
1867 unreachable("not reached");
1873 if (inst->is_3src(devinfo)) {
1874 /* 3-src instructions with scalar sources support arbitrary subnr,
1875 * but don't actually use swizzles. Convert swizzle into subnr.
1877 for (int i = 0; i < 3; i++) {
1878 if (inst->src[i].vstride == BRW_VERTICAL_STRIDE_0) {
1879 assert(brw_is_single_value_swizzle(inst->src[i].swizzle));
1880 inst->src[i].subnr += 4 * BRW_GET_SWZ(inst->src[i].swizzle, 0);
1885 dst_reg &dst = inst->dst;
1888 switch (inst->dst.file) {
1890 reg = brw_vec8_grf(dst.nr + dst.reg_offset, 0);
1891 reg.type = dst.type;
1892 reg.writemask = dst.writemask;
1896 assert(((dst.nr + dst.reg_offset) & ~BRW_MRF_COMPR4) < BRW_MAX_MRF(devinfo->gen));
1897 reg = brw_message_reg(dst.nr + dst.reg_offset);
1898 reg.type = dst.type;
1899 reg.writemask = dst.writemask;
1904 reg = dst.as_brw_reg();
1908 reg = brw_null_reg();
1914 unreachable("not reached");
1924 if (shader_time_index >= 0)
1925 emit_shader_time_begin();
1938 /* Before any optimization, push array accesses out to scratch
1939 * space where we need them to be. This pass may allocate new
1940 * virtual GRFs, so we want to do it early. It also makes sure
1941 * that we have reladdr computations available for CSE, since we'll
1942 * often do repeated subexpressions for those.
1944 move_grf_array_access_to_scratch();
1945 move_uniform_array_access_to_pull_constants();
1947 pack_uniform_registers();
1948 move_push_constants_to_pull_constants();
1949 split_virtual_grfs();
1951 #define OPT(pass, args...) ({ \
1953 bool this_progress = pass(args); \
1955 if (unlikely(INTEL_DEBUG & DEBUG_OPTIMIZER) && this_progress) { \
1956 char filename[64]; \
1957 snprintf(filename, 64, "%s-%s-%02d-%02d-" #pass, \
1958 stage_abbrev, nir->info.name, iteration, pass_num); \
1960 backend_shader::dump_instructions(filename); \
1963 progress = progress || this_progress; \
1968 if (unlikely(INTEL_DEBUG & DEBUG_OPTIMIZER)) {
1970 snprintf(filename, 64, "%s-%s-00-00-start",
1971 stage_abbrev, nir->info.name);
1973 backend_shader::dump_instructions(filename);
1984 OPT(opt_predicated_break, this);
1985 OPT(opt_reduce_swizzle);
1986 OPT(dead_code_eliminate);
1987 OPT(dead_control_flow_eliminate, this);
1988 OPT(opt_copy_propagation);
1989 OPT(opt_cmod_propagation);
1992 OPT(opt_register_coalesce);
1993 OPT(eliminate_find_live_channel);
1998 if (OPT(opt_vector_float)) {
2000 OPT(opt_copy_propagation, false);
2001 OPT(opt_copy_propagation, true);
2002 OPT(dead_code_eliminate);
2005 if (devinfo->gen <= 5 && OPT(lower_minmax)) {
2006 OPT(opt_cmod_propagation);
2008 OPT(opt_copy_propagation);
2009 OPT(dead_code_eliminate);
2017 if (unlikely(INTEL_DEBUG & DEBUG_SPILL_VEC4)) {
2018 /* Debug of register spilling: Go spill everything. */
2019 const int grf_count = alloc.count;
2020 float spill_costs[alloc.count];
2021 bool no_spill[alloc.count];
2022 evaluate_spill_costs(spill_costs, no_spill);
2023 for (int i = 0; i < grf_count; i++) {
2030 bool allocated_without_spills = reg_allocate();
2032 if (!allocated_without_spills) {
2033 compiler->shader_perf_log(log_data,
2034 "%s shader triggered register spilling. "
2035 "Try reducing the number of live vec4 values "
2036 "to improve performance.\n",
2039 while (!reg_allocate()) {
2045 opt_schedule_instructions();
2047 opt_set_dependency_control();
2049 convert_to_hw_regs();
2051 if (last_scratch > 0) {
2052 prog_data->base.total_scratch =
2053 brw_get_scratch_size(last_scratch * REG_SIZE);
2059 } /* namespace brw */
2064 * Compile a vertex shader.
2066 * Returns the final assembly and the program's size.
2069 brw_compile_vs(const struct brw_compiler *compiler, void *log_data,
2071 const struct brw_vs_prog_key *key,
2072 struct brw_vs_prog_data *prog_data,
2073 const nir_shader *src_shader,
2074 gl_clip_plane *clip_planes,
2075 bool use_legacy_snorm_formula,
2076 int shader_time_index,
2077 unsigned *final_assembly_size,
2080 const bool is_scalar = compiler->scalar_stage[MESA_SHADER_VERTEX];
2081 nir_shader *shader = nir_shader_clone(mem_ctx, src_shader);
2082 shader = brw_nir_apply_sampler_key(shader, compiler->devinfo, &key->tex,
2084 brw_nir_lower_vs_inputs(shader, compiler->devinfo, is_scalar,
2085 use_legacy_snorm_formula, key->gl_attrib_wa_flags);
2086 brw_nir_lower_vue_outputs(shader, is_scalar);
2087 shader = brw_postprocess_nir(shader, compiler->devinfo, is_scalar);
2089 const unsigned *assembly = NULL;
2091 unsigned nr_attributes = _mesa_bitcount_64(prog_data->inputs_read);
2093 /* gl_VertexID and gl_InstanceID are system values, but arrive via an
2094 * incoming vertex attribute. So, add an extra slot.
2096 if (shader->info.system_values_read &
2097 (BITFIELD64_BIT(SYSTEM_VALUE_BASE_VERTEX) |
2098 BITFIELD64_BIT(SYSTEM_VALUE_BASE_INSTANCE) |
2099 BITFIELD64_BIT(SYSTEM_VALUE_VERTEX_ID_ZERO_BASE) |
2100 BITFIELD64_BIT(SYSTEM_VALUE_INSTANCE_ID))) {
2104 /* gl_DrawID has its very own vec4 */
2105 if (shader->info.system_values_read & BITFIELD64_BIT(SYSTEM_VALUE_DRAW_ID)) {
2109 unsigned nr_attribute_slots =
2111 _mesa_bitcount_64(shader->info.double_inputs_read);
2113 /* The 3DSTATE_VS documentation lists the lower bound on "Vertex URB Entry
2114 * Read Length" as 1 in vec4 mode, and 0 in SIMD8 mode. Empirically, in
2115 * vec4 mode, the hardware appears to wedge unless we read something.
2118 prog_data->base.urb_read_length =
2119 DIV_ROUND_UP(nr_attribute_slots, 2);
2121 prog_data->base.urb_read_length =
2122 DIV_ROUND_UP(MAX2(nr_attribute_slots, 1), 2);
2124 prog_data->nr_attributes = nr_attributes;
2125 prog_data->nr_attribute_slots = nr_attribute_slots;
2127 /* Since vertex shaders reuse the same VUE entry for inputs and outputs
2128 * (overwriting the original contents), we need to make sure the size is
2129 * the larger of the two.
2131 const unsigned vue_entries =
2132 MAX2(nr_attribute_slots, (unsigned)prog_data->base.vue_map.num_slots);
2134 if (compiler->devinfo->gen == 6)
2135 prog_data->base.urb_entry_size = DIV_ROUND_UP(vue_entries, 8);
2137 prog_data->base.urb_entry_size = DIV_ROUND_UP(vue_entries, 4);
2140 prog_data->base.dispatch_mode = DISPATCH_MODE_SIMD8;
2142 fs_visitor v(compiler, log_data, mem_ctx, key, &prog_data->base.base,
2143 NULL, /* prog; Only used for TEXTURE_RECTANGLE on gen < 8 */
2144 shader, 8, shader_time_index);
2145 if (!v.run_vs(clip_planes)) {
2147 *error_str = ralloc_strdup(mem_ctx, v.fail_msg);
2152 prog_data->base.base.dispatch_grf_start_reg = v.payload.num_regs;
2154 fs_generator g(compiler, log_data, mem_ctx, (void *) key,
2155 &prog_data->base.base, v.promoted_constants,
2156 v.runtime_check_aads_emit, MESA_SHADER_VERTEX);
2157 if (INTEL_DEBUG & DEBUG_VS) {
2158 const char *debug_name =
2159 ralloc_asprintf(mem_ctx, "%s vertex shader %s",
2160 shader->info.label ? shader->info.label : "unnamed",
2163 g.enable_debug(debug_name);
2165 g.generate_code(v.cfg, 8);
2166 assembly = g.get_assembly(final_assembly_size);
2170 prog_data->base.dispatch_mode = DISPATCH_MODE_4X2_DUAL_OBJECT;
2172 vec4_vs_visitor v(compiler, log_data, key, prog_data,
2173 shader, clip_planes, mem_ctx,
2174 shader_time_index, use_legacy_snorm_formula);
2177 *error_str = ralloc_strdup(mem_ctx, v.fail_msg);
2182 assembly = brw_vec4_generate_assembly(compiler, log_data, mem_ctx,
2183 shader, &prog_data->base, v.cfg,
2184 final_assembly_size);