2 * Copyright 2011 Christoph Bumiller
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
33 #include "codegen/unordered_set.h"
34 #include "codegen/nv50_ir_util.h"
35 #include "codegen/nv50_ir_graph.h"
37 #include "codegen/nv50_ir_driver.h"
45 OP_UNION, // unify a new definition and several source values
46 OP_SPLIT, // $r0d -> { $r0, $r1 } ($r0d and $r0/$r1 will be coalesced)
47 OP_MERGE, // opposite of split, e.g. combine 2 32 bit into a 64 bit value
48 OP_CONSTRAINT, // copy values into consecutive registers
49 OP_MOV, // simple copy, no modifiers allowed
52 OP_ADD, // NOTE: add u64 + u32 is legal for targets w/o 64-bit integer adds
59 OP_SAD, // abs(src0 - src1) + src2
70 OP_SAT, // CLAMP(f32, 0.0, 1.0)
75 OP_SET_AND, // dst = (src0 CMP src1) & src2
79 OP_SELP, // dst = src2 ? src0 : src1
80 OP_SLCT, // dst = (src2 CMP 0) ? src0 : src1
87 OP_EXP, // exponential (base M_E)
88 OP_LOG, // natural logarithm
101 OP_BRKPT, // breakpoint (not related to loops)
102 OP_JOINAT, // push control flow convergence point
106 OP_MEMBAR, // memory barrier (mfence, lfence, sfence)
107 OP_VFETCH, // indirection 0 in attribute space, indirection 1 is vertex base
108 OP_PFETCH, // fetch base address of vertex src0 (immediate) [+ src1]
112 OP_EMIT, // emit vertex
113 OP_RESTART, // restart primitive
115 OP_TXB, // texture bias
116 OP_TXL, // texure lod
117 OP_TXF, // texel fetch
118 OP_TXQ, // texture size query
119 OP_TXD, // texture derivatives
120 OP_TXG, // texture gather
121 OP_TXLQ, // texture query lod
122 OP_TEXCSAA, // texture op for coverage sampling
123 OP_TEXPREP, // turn cube map array into 2d array coordinates
124 OP_SULDB, // surface load (raw)
125 OP_SULDP, // surface load (formatted)
126 OP_SUSTB, // surface store (raw)
127 OP_SUSTP, // surface store (formatted)
129 OP_SUREDP, // surface reduction (atomic op)
130 OP_SULEA, // surface load effective address
131 OP_SUBFM, // surface bitfield manipulation
132 OP_SUCLAMP, // clamp surface coordinates
133 OP_SUEAU, // surface effective address
134 OP_MADSP, // special integer multiply-add
135 OP_TEXBAR, // texture dependency barrier
138 OP_RDSV, // read system value
139 OP_WRSV, // write system value
140 OP_PIXLD, // get info about raster object or surfaces
144 OP_POPCNT, // bitcount(src0 & src1)
145 OP_INSBF, // insert first src1[8:15] bits of src0 into src2 at src1[0:7]
146 OP_EXTBF, // place bits [K,K+N) of src0 into dst, src1 = 0xNNKK
147 OP_BFIND, // find highest/lowest set bit
148 OP_PERMT, // dst = bytes from src2,src0 selected by src1 (nvc0's src order)
150 OP_BAR, // execution barrier, sources = { id, thread count, predicate }
151 OP_VADD, // byte/word vector operations
160 OP_CCTL, // cache control
161 OP_SHFL, // warp shuffle
165 // various instruction-specific modifier definitions Instruction::subOp
166 // MOV_FINAL marks a MOV originating from an EXPORT (used for placing TEXBARs)
167 #define NV50_IR_SUBOP_MUL_HIGH 1
168 #define NV50_IR_SUBOP_EMIT_RESTART 1
169 #define NV50_IR_SUBOP_LDC_IL 1
170 #define NV50_IR_SUBOP_LDC_IS 2
171 #define NV50_IR_SUBOP_LDC_ISL 3
172 #define NV50_IR_SUBOP_SHIFT_WRAP 1
173 #define NV50_IR_SUBOP_EMU_PRERET 1
174 #define NV50_IR_SUBOP_TEXBAR(n) n
175 #define NV50_IR_SUBOP_MOV_FINAL 1
176 #define NV50_IR_SUBOP_EXTBF_REV 1
177 #define NV50_IR_SUBOP_BFIND_SAMT 1
178 #define NV50_IR_SUBOP_PERMT_F4E 1
179 #define NV50_IR_SUBOP_PERMT_B4E 2
180 #define NV50_IR_SUBOP_PERMT_RC8 3
181 #define NV50_IR_SUBOP_PERMT_ECL 4
182 #define NV50_IR_SUBOP_PERMT_ECR 5
183 #define NV50_IR_SUBOP_PERMT_RC16 6
184 #define NV50_IR_SUBOP_BAR_SYNC 0
185 #define NV50_IR_SUBOP_BAR_ARRIVE 1
186 #define NV50_IR_SUBOP_BAR_RED_AND 2
187 #define NV50_IR_SUBOP_BAR_RED_OR 3
188 #define NV50_IR_SUBOP_BAR_RED_POPC 4
189 #define NV50_IR_SUBOP_MEMBAR_L 1
190 #define NV50_IR_SUBOP_MEMBAR_S 2
191 #define NV50_IR_SUBOP_MEMBAR_M 3
192 #define NV50_IR_SUBOP_MEMBAR_CTA (0 << 2)
193 #define NV50_IR_SUBOP_MEMBAR_GL (1 << 2)
194 #define NV50_IR_SUBOP_MEMBAR_SYS (2 << 2)
195 #define NV50_IR_SUBOP_MEMBAR_DIR(m) ((m) & 0x3)
196 #define NV50_IR_SUBOP_MEMBAR_SCOPE(m) ((m) & ~0x3)
197 #define NV50_IR_SUBOP_MEMBAR(d,s) \
198 (NV50_IR_SUBOP_MEMBAR_##d | NV50_IR_SUBOP_MEMBAR_##s)
199 #define NV50_IR_SUBOP_ATOM_ADD 0
200 #define NV50_IR_SUBOP_ATOM_MIN 1
201 #define NV50_IR_SUBOP_ATOM_MAX 2
202 #define NV50_IR_SUBOP_ATOM_INC 3
203 #define NV50_IR_SUBOP_ATOM_DEC 4
204 #define NV50_IR_SUBOP_ATOM_AND 5
205 #define NV50_IR_SUBOP_ATOM_OR 6
206 #define NV50_IR_SUBOP_ATOM_XOR 7
207 #define NV50_IR_SUBOP_ATOM_CAS 8
208 #define NV50_IR_SUBOP_ATOM_EXCH 9
209 #define NV50_IR_SUBOP_CCTL_IV 5
210 #define NV50_IR_SUBOP_CCTL_IVALL 6
211 #define NV50_IR_SUBOP_SUST_IGN 0
212 #define NV50_IR_SUBOP_SUST_TRAP 1
213 #define NV50_IR_SUBOP_SUST_SDCL 3
214 #define NV50_IR_SUBOP_SULD_ZERO 0
215 #define NV50_IR_SUBOP_SULD_TRAP 1
216 #define NV50_IR_SUBOP_SULD_SDCL 3
217 #define NV50_IR_SUBOP_SUBFM_3D 1
218 #define NV50_IR_SUBOP_SUCLAMP_2D 0x10
219 #define NV50_IR_SUBOP_SUCLAMP_SD(r, d) (( 0 + (r)) | ((d == 2) ? 0x10 : 0))
220 #define NV50_IR_SUBOP_SUCLAMP_PL(r, d) (( 5 + (r)) | ((d == 2) ? 0x10 : 0))
221 #define NV50_IR_SUBOP_SUCLAMP_BL(r, d) ((10 + (r)) | ((d == 2) ? 0x10 : 0))
222 #define NV50_IR_SUBOP_PIXLD_COUNT 0
223 #define NV50_IR_SUBOP_PIXLD_COVMASK 1
224 #define NV50_IR_SUBOP_PIXLD_COVERED 2
225 #define NV50_IR_SUBOP_PIXLD_OFFSET 3
226 #define NV50_IR_SUBOP_PIXLD_CENT_OFFSET 4
227 #define NV50_IR_SUBOP_PIXLD_SAMPLEID 5
228 #define NV50_IR_SUBOP_SHFL_IDX 0
229 #define NV50_IR_SUBOP_SHFL_UP 1
230 #define NV50_IR_SUBOP_SHFL_DOWN 2
231 #define NV50_IR_SUBOP_SHFL_BFLY 3
232 #define NV50_IR_SUBOP_MADSP_SD 0xffff
233 // Yes, we could represent those with DataType.
234 // Or put the type into operation and have a couple 1000 values in that enum.
235 // This will have to do for now.
236 // The bitfields are supposed to correspond to nve4 ISA.
237 #define NV50_IR_SUBOP_MADSP(a,b,c) (((c) << 8) | ((b) << 4) | (a))
238 #define NV50_IR_SUBOP_V1(d,a,b) (((d) << 10) | ((b) << 5) | (a) | 0x0000)
239 #define NV50_IR_SUBOP_V2(d,a,b) (((d) << 10) | ((b) << 5) | (a) | 0x4000)
240 #define NV50_IR_SUBOP_V4(d,a,b) (((d) << 10) | ((b) << 5) | (a) | 0x8000)
241 #define NV50_IR_SUBOP_Vn(n) ((n) >> 14)
252 TYPE_U64, // 64 bit operations are only lowered after register allocation
264 CC_NEVER = CC_FL, // when used with FILE_FLAGS
267 CC_NOT_P = CC_EQ, // when used with FILE_PREDICATE
295 ROUND_M, // towards -inf
296 ROUND_Z, // towards 0
297 ROUND_P, // towards +inf
298 ROUND_NI, // nearest integer
299 ROUND_MI, // to integer towards -inf
300 ROUND_ZI, // to integer towards 0
301 ROUND_PI, // to integer towards +inf
306 CACHE_CA, // cache at all levels
307 CACHE_WB = CACHE_CA, // cache write back
308 CACHE_CG, // cache at global level
309 CACHE_CS, // cache streaming
310 CACHE_CV, // cache as volatile
311 CACHE_WT = CACHE_CV // cache write-through
318 FILE_PREDICATE, // boolean predicate
319 FILE_FLAGS, // zero/sign/carry/overflow bits
321 LAST_REGISTER_FILE = FILE_ADDRESS,
340 TEX_TARGET_1D_SHADOW,
341 TEX_TARGET_2D_SHADOW,
342 TEX_TARGET_CUBE_SHADOW,
345 TEX_TARGET_2D_MS_ARRAY,
346 TEX_TARGET_CUBE_ARRAY,
347 TEX_TARGET_1D_ARRAY_SHADOW,
348 TEX_TARGET_2D_ARRAY_SHADOW,
350 TEX_TARGET_RECT_SHADOW,
351 TEX_TARGET_CUBE_ARRAY_SHADOW,
363 SV_VERTEX_COUNT, // gl_PatchVerticesIn
400 class CmpInstruction;
401 class TexInstruction;
402 class FlowInstruction;
407 class ImmediateValue;
412 int8_t fileIndex; // signed, may be indirect for CONST[]
413 uint8_t size; // this should match the Instruction type's size
414 DataType type; // mainly for pretty printing
416 uint64_t u64; // immediate values
426 int32_t offset; // offset from 0 (base of address space)
427 int32_t id; // register id (< 0 if virtual/unassigned, in units <= 4)
435 // precedence: NOT after SAT after NEG after ABS
436 #define NV50_IR_MOD_ABS (1 << 0)
437 #define NV50_IR_MOD_NEG (1 << 1)
438 #define NV50_IR_MOD_SAT (1 << 2)
439 #define NV50_IR_MOD_NOT (1 << 3)
440 #define NV50_IR_MOD_NEG_ABS (NV50_IR_MOD_NEG | NV50_IR_MOD_ABS)
442 #define NV50_IR_INTERP_MODE_MASK 0x3
443 #define NV50_IR_INTERP_LINEAR (0 << 0)
444 #define NV50_IR_INTERP_PERSPECTIVE (1 << 0)
445 #define NV50_IR_INTERP_FLAT (2 << 0)
446 #define NV50_IR_INTERP_SC (3 << 0) // what exactly is that ?
447 #define NV50_IR_INTERP_SAMPLE_MASK 0xc
448 #define NV50_IR_INTERP_DEFAULT (0 << 2)
449 #define NV50_IR_INTERP_CENTROID (1 << 2)
450 #define NV50_IR_INTERP_OFFSET (2 << 2)
451 #define NV50_IR_INTERP_SAMPLEID (3 << 2)
453 // do we really want this to be a class ?
457 Modifier() : bits(0) { }
458 Modifier(unsigned int m) : bits(m) { }
459 Modifier(operation op);
461 // @return new Modifier applying a after b (asserts if unrepresentable)
462 Modifier operator*(const Modifier) const;
463 Modifier operator*=(const Modifier m) { *this = *this * m; return *this; }
464 Modifier operator==(const Modifier m) const { return m.bits == bits; }
465 Modifier operator!=(const Modifier m) const { return m.bits != bits; }
467 inline Modifier operator&(const Modifier m) const { return bits & m.bits; }
468 inline Modifier operator|(const Modifier m) const { return bits | m.bits; }
469 inline Modifier operator^(const Modifier m) const { return bits ^ m.bits; }
471 operation getOp() const;
473 inline int neg() const { return (bits & NV50_IR_MOD_NEG) ? 1 : 0; }
474 inline int abs() const { return (bits & NV50_IR_MOD_ABS) ? 1 : 0; }
476 inline operator bool() const { return bits ? true : false; }
478 void applyTo(ImmediateValue &imm) const;
480 int print(char *buf, size_t size) const;
489 ValueRef(Value * = NULL);
490 ValueRef(const ValueRef&);
493 inline bool exists() const { return value != NULL; }
496 void set(const ValueRef&);
497 inline Value *get() const { return value; }
498 inline Value *rep() const;
500 inline Instruction *getInsn() const { return insn; }
501 inline void setInsn(Instruction *inst) { insn = inst; }
503 inline bool isIndirect(int dim) const { return indirect[dim] >= 0; }
504 inline const ValueRef *getIndirect(int dim) const;
506 inline DataFile getFile() const;
507 inline unsigned getSize() const;
509 // SSA: return eventual (traverse MOVs) literal value, if it exists
510 bool getImmediate(ImmediateValue&) const;
514 int8_t indirect[2]; // >= 0 if relative to lvalue in insn->src(indirect[i])
517 bool usedAsPtr; // for printing
527 ValueDef(Value * = NULL);
528 ValueDef(const ValueDef&);
531 inline bool exists() const { return value != NULL; }
533 inline Value *get() const { return value; }
534 inline Value *rep() const;
536 bool mayReplace(const ValueRef &);
537 void replace(const ValueRef &, bool doSet); // replace all uses of the old value
539 inline Instruction *getInsn() const { return insn; }
540 inline void setInsn(Instruction *inst) { insn = inst; }
542 inline DataFile getFile() const;
543 inline unsigned getSize() const;
545 inline void setSSA(LValue *);
546 inline const LValue *preSSA() const;
549 Value *value; // should make this LValue * ...
550 LValue *origin; // pre SSA value
560 virtual Value *clone(ClonePolicy<Function>&) const = 0;
562 virtual int print(char *, size_t, DataType ty = TYPE_NONE) const = 0;
564 virtual bool equals(const Value *, bool strict = false) const;
565 virtual bool interfers(const Value *) const;
566 virtual bool isUniform() const { return true; }
568 inline Value *rep() const { return join; }
570 inline Instruction *getUniqueInsn() const;
571 inline Instruction *getInsn() const; // use when uniqueness is certain
573 inline int refCount() { return uses.size(); }
575 inline LValue *asLValue();
576 inline Symbol *asSym();
577 inline ImmediateValue *asImm();
578 inline const Symbol *asSym() const;
579 inline const ImmediateValue *asImm() const;
581 inline bool inFile(DataFile f) { return reg.file == f; }
583 static inline Value *get(Iterator&);
585 unordered_set<ValueRef *> uses;
586 std::list<ValueDef *> defs;
587 typedef unordered_set<ValueRef *>::iterator UseIterator;
588 typedef unordered_set<ValueRef *>::const_iterator UseCIterator;
589 typedef std::list<ValueDef *>::iterator DefIterator;
590 typedef std::list<ValueDef *>::const_iterator DefCIterator;
595 // TODO: these should be in LValue:
600 class LValue : public Value
603 LValue(Function *, DataFile file);
604 LValue(Function *, LValue *);
607 virtual bool isUniform() const;
609 virtual LValue *clone(ClonePolicy<Function>&) const;
611 virtual int print(char *, size_t, DataType ty = TYPE_NONE) const;
614 unsigned compMask : 8; // compound/component mask
615 unsigned compound : 1; // used by RA, value involved in split/merge
617 unsigned fixedReg : 1; // set & used by RA, earlier just use (id < 0)
618 unsigned noSpill : 1; // do not spill (e.g. if spill temporary already)
621 class Symbol : public Value
624 Symbol(Program *, DataFile file = FILE_MEMORY_CONST, ubyte fileIdx = 0);
627 virtual Symbol *clone(ClonePolicy<Function>&) const;
629 virtual bool equals(const Value *that, bool strict) const;
631 virtual bool isUniform() const;
633 virtual int print(char *, size_t, DataType ty = TYPE_NONE) const;
635 // print with indirect values
636 int print(char *, size_t, Value *, Value *, DataType ty = TYPE_NONE) const;
638 inline void setFile(DataFile file, ubyte fileIndex = 0)
641 reg.fileIndex = fileIndex;
644 inline void setOffset(int32_t offset);
645 inline void setAddress(Symbol *base, int32_t offset);
646 inline void setSV(SVSemantic sv, uint32_t idx = 0);
648 inline const Symbol *getBase() const { return baseSym; }
651 Symbol *baseSym; // array base for Symbols representing array elements
654 class ImmediateValue : public Value
658 ImmediateValue(Program *, uint32_t);
659 ImmediateValue(Program *, float);
660 ImmediateValue(Program *, double);
661 // NOTE: not added to program with
662 ImmediateValue(const ImmediateValue *, DataType ty);
663 ~ImmediateValue() { };
665 virtual ImmediateValue *clone(ClonePolicy<Function>&) const;
667 virtual bool equals(const Value *that, bool strict) const;
669 // these only work if 'type' is valid (we mostly use untyped literals):
670 bool isInteger(const int ival) const; // ival is cast to this' type
671 bool isNegative() const;
676 // for constant folding:
677 ImmediateValue operator+(const ImmediateValue&) const;
678 ImmediateValue operator-(const ImmediateValue&) const;
679 ImmediateValue operator*(const ImmediateValue&) const;
680 ImmediateValue operator/(const ImmediateValue&) const;
682 ImmediateValue& operator=(const ImmediateValue&); // only sets value !
684 bool compare(CondCode cc, float fval) const;
686 virtual int print(char *, size_t, DataType ty = TYPE_NONE) const;
693 Instruction(Function *, operation, DataType);
694 virtual ~Instruction();
696 virtual Instruction *clone(ClonePolicy<Function>&,
697 Instruction * = NULL) const;
699 void setDef(int i, Value *);
700 void setSrc(int s, Value *);
701 void setSrc(int s, const ValueRef&);
702 void swapSources(int a, int b);
703 void moveSources(int s, int delta);
704 bool setIndirect(int s, int dim, Value *);
706 inline ValueRef& src(int s) { return srcs[s]; }
707 inline ValueDef& def(int s) { return defs[s]; }
708 inline const ValueRef& src(int s) const { return srcs[s]; }
709 inline const ValueDef& def(int s) const { return defs[s]; }
711 inline Value *getDef(int d) const { return defs[d].get(); }
712 inline Value *getSrc(int s) const { return srcs[s].get(); }
713 inline Value *getIndirect(int s, int dim) const;
715 inline bool defExists(unsigned d) const
717 return d < defs.size() && defs[d].exists();
719 inline bool srcExists(unsigned s) const
721 return s < srcs.size() && srcs[s].exists();
724 inline bool constrainedDefs() const;
726 bool setPredicate(CondCode ccode, Value *);
727 inline Value *getPredicate() const;
728 bool writesPredicate() const;
729 inline bool isPredicated() const { return predSrc >= 0; }
731 inline void setFlagsSrc(int s, Value *);
732 inline void setFlagsDef(int d, Value *);
733 inline bool usesFlags() const { return flagsSrc >= 0; }
735 unsigned int defCount() const { return defs.size(); };
736 unsigned int defCount(unsigned int mask, bool singleFile = false) const;
737 unsigned int srcCount() const { return srcs.size(); };
738 unsigned int srcCount(unsigned int mask, bool singleFile = false) const;
740 // save & remove / set indirect[0,1] and predicate source
741 void takeExtraSources(int s, Value *[3]);
742 void putExtraSources(int s, Value *[3]);
744 inline void setType(DataType type) { dType = sType = type; }
746 inline void setType(DataType dtype, DataType stype)
752 inline bool isPseudo() const { return op < OP_MOV; }
755 bool isCommutationLegal(const Instruction *) const; // must be adjacent !
756 bool isActionEqual(const Instruction *) const;
757 bool isResultEqual(const Instruction *) const;
761 inline CmpInstruction *asCmp();
762 inline TexInstruction *asTex();
763 inline FlowInstruction *asFlow();
764 inline const TexInstruction *asTex() const;
765 inline const CmpInstruction *asCmp() const;
766 inline const FlowInstruction *asFlow() const;
772 int serial; // CFG order
775 DataType dType; // destination or defining type
776 DataType sType; // source or secondary type
781 uint16_t subOp; // quadop, 1 for mul-high, etc.
783 unsigned encSize : 4; // encoding size in bytes
784 unsigned saturate : 1; // to [0.0f, 1.0f]
785 unsigned join : 1; // converge control flow (use OP_JOIN until end)
786 unsigned fixed : 1; // prevent dead code elimination
787 unsigned terminator : 1; // end of basic block
788 unsigned ftz : 1; // flush denormal to zero
789 unsigned dnz : 1; // denormals, NaN are zero
790 unsigned ipa : 4; // interpolation mode
792 unsigned perPatch : 1;
793 unsigned exit : 1; // terminate program after insn
794 unsigned mask : 4; // for vector ops
796 int8_t postFactor; // MUL/DIV(if < 0) by 1 << postFactor
802 uint32_t sched; // scheduling data (NOTE: maybe move to separate storage)
807 std::deque<ValueDef> defs; // no gaps !
808 std::deque<ValueRef> srcs; // no gaps !
810 // instruction specific methods:
811 // (don't want to subclass, would need more constructors and memory pools)
813 inline void setInterpolate(unsigned int mode) { ipa = mode; }
815 unsigned int getInterpMode() const { return ipa & 0x3; }
816 unsigned int getSampleMode() const { return ipa & 0xc; }
833 class TexInstruction : public Instruction
839 Target(TexTarget targ = TEX_TARGET_2D) : target(targ) { }
841 const char *getName() const { return descTable[target].name; }
842 unsigned int getArgCount() const { return descTable[target].argc; }
843 unsigned int getDim() const { return descTable[target].dim; }
844 int isArray() const { return descTable[target].array ? 1 : 0; }
845 int isCube() const { return descTable[target].cube ? 1 : 0; }
846 int isShadow() const { return descTable[target].shadow ? 1 : 0; }
848 return target == TEX_TARGET_2D_MS || target == TEX_TARGET_2D_MS_ARRAY; }
852 target = TEX_TARGET_2D_ARRAY;
854 target = TEX_TARGET_2D;
858 Target& operator=(TexTarget targ)
860 assert(targ < TEX_TARGET_COUNT);
865 inline bool operator==(TexTarget targ) const { return target == targ; }
866 inline bool operator!=(TexTarget targ) const { return target != targ; }
868 enum TexTarget getEnum() const { return target; }
881 static const struct Desc descTable[TEX_TARGET_COUNT];
884 enum TexTarget target;
888 TexInstruction(Function *, operation);
889 virtual ~TexInstruction();
891 virtual TexInstruction *clone(ClonePolicy<Function>&,
892 Instruction * = NULL) const;
894 inline void setTexture(Target targ, uint8_t r, uint8_t s)
901 void setIndirectR(Value *);
902 void setIndirectS(Value *);
903 inline Value *getIndirectR() const;
904 inline Value *getIndirectS() const;
918 bool liveOnly; // only execute on live pixels of a quad (optimization)
922 int8_t useOffsets; // 0, 1, or 4 for textureGatherOffsets
923 int8_t offset[3]; // only used on nv50
930 ValueRef offset[4][3];
933 class CmpInstruction : public Instruction
936 CmpInstruction(Function *, operation);
938 virtual CmpInstruction *clone(ClonePolicy<Function>&,
939 Instruction * = NULL) const;
941 void setCondition(CondCode cond) { setCond = cond; }
942 CondCode getCondition() const { return setCond; }
948 class FlowInstruction : public Instruction
951 FlowInstruction(Function *, operation, void *target);
953 virtual FlowInstruction *clone(ClonePolicy<Function>&,
954 Instruction * = NULL) const;
957 unsigned allWarp : 1;
958 unsigned absolute : 1;
960 unsigned builtin : 1; // true for calls to emulation code
961 unsigned indirect : 1; // target in src(0)
973 BasicBlock(Function *);
976 BasicBlock *clone(ClonePolicy<Function>&) const;
978 inline int getId() const { return id; }
979 inline unsigned int getInsnCount() const { return numInsns; }
980 inline bool isTerminated() const { return exit && exit->terminator; }
982 bool dominatedBy(BasicBlock *bb);
983 inline bool reachableBy(const BasicBlock *by, const BasicBlock *term);
985 // returns mask of conditional out blocks
986 // e.g. 3 for IF { .. } ELSE { .. } ENDIF, 1 for IF { .. } ENDIF
987 unsigned int initiatesSimpleConditional() const;
990 Function *getFunction() const { return func; }
991 Program *getProgram() const { return program; }
993 Instruction *getEntry() const { return entry; } // first non-phi instruction
994 Instruction *getPhi() const { return phi; }
995 Instruction *getFirst() const { return phi ? phi : entry; }
996 Instruction *getExit() const { return exit; }
998 void insertHead(Instruction *);
999 void insertTail(Instruction *);
1000 void insertBefore(Instruction *, Instruction *);
1001 void insertAfter(Instruction *, Instruction *);
1002 void remove(Instruction *);
1003 void permuteAdjacent(Instruction *, Instruction *);
1005 BasicBlock *idom() const;
1007 // NOTE: currently does not rebuild the dominator tree
1008 BasicBlock *splitBefore(Instruction *, bool attach = true);
1009 BasicBlock *splitAfter(Instruction *, bool attach = true);
1011 DLList& getDF() { return df; }
1012 DLList::Iterator iterDF() { return df.iterator(); }
1014 static inline BasicBlock *get(Iterator&);
1015 static inline BasicBlock *get(Graph::Node *);
1018 Graph::Node cfg; // first edge is branch *taken* (the ELSE branch)
1027 Instruction *joinAt; // for quick reference
1029 bool explicitCont; // loop headers: true if loop contains continue stmts
1039 unsigned int numInsns;
1045 void splitCommon(Instruction *, BasicBlock *, bool attach);
1051 Function(Program *, const char *name, uint32_t label);
1054 static inline Function *get(Graph::Node *node);
1056 inline Program *getProgram() const { return prog; }
1057 inline const char *getName() const { return name; }
1058 inline int getId() const { return id; }
1059 inline uint32_t getLabel() const { return label; }
1062 void printLiveIntervals() const;
1063 void printCFGraph(const char *filePath);
1065 bool setEntry(BasicBlock *);
1066 bool setExit(BasicBlock *);
1068 unsigned int orderInstructions(ArrayList&);
1070 inline void add(BasicBlock *bb, int& id) { allBBlocks.insert(bb, id); }
1071 inline void add(Instruction *insn, int& id) { allInsns.insert(insn, id); }
1072 inline void add(LValue *lval, int& id) { allLValues.insert(lval, id); }
1074 inline LValue *getLValue(int id);
1076 void buildLiveSets();
1077 void buildDefSets();
1078 bool convertToSSA();
1081 std::deque<ValueDef> ins;
1082 std::deque<ValueRef> outs;
1083 std::deque<Value *> clobbers;
1086 Graph::Node *cfgExit;
1088 Graph::Node call; // node in the call graph
1090 BasicBlock **bbArray; // BBs in emission order
1093 unsigned int loopNestingBound;
1101 uint32_t tlsBase; // base address for l[] space (if no stack pointer is used)
1104 ArrayList allBBlocks;
1106 ArrayList allLValues;
1109 void buildLiveSetsPreSSA(BasicBlock *, const int sequence);
1110 void buildDefSetsPreSSA(BasicBlock *bb, const int seq);
1115 const char *const name;
1122 CG_STAGE_SSA, // expected directly before register allocation
1132 TYPE_TESSELLATION_CONTROL,
1133 TYPE_TESSELLATION_EVAL,
1139 Program(Type type, Target *targ);
1144 Type getType() const { return progType; }
1146 inline void add(Function *fn, int& id) { allFuncs.insert(fn, id); }
1147 inline void del(Function *fn, int& id) { allFuncs.remove(id); }
1148 inline void add(Value *rval, int& id) { allRValues.insert(rval, id); }
1150 bool makeFromTGSI(struct nv50_ir_prog_info *);
1151 bool makeFromSM4(struct nv50_ir_prog_info *);
1152 bool convertToSSA();
1153 bool optimizeSSA(int level);
1154 bool optimizePostRA(int level);
1155 bool registerAllocation();
1156 bool emitBinary(struct nv50_ir_prog_info *);
1158 const Target *getTarget() const { return target; }
1161 void emitSymbolTable(struct nv50_ir_prog_info *);
1171 ArrayList allRValues;
1175 uint32_t tlsSize; // size required for FILE_MEMORY_LOCAL
1179 MemoryPool mem_Instruction;
1180 MemoryPool mem_CmpInstruction;
1181 MemoryPool mem_TexInstruction;
1182 MemoryPool mem_FlowInstruction;
1183 MemoryPool mem_LValue;
1184 MemoryPool mem_Symbol;
1185 MemoryPool mem_ImmediateValue;
1190 void *targetPriv; // e.g. to carry information between passes
1192 const struct nv50_ir_prog_info *driver; // for driver configuration
1194 void releaseInstruction(Instruction *);
1195 void releaseValue(Value *);
1198 // TODO: add const version
1202 bool run(Program *, bool ordered = false, bool skipPhi = false);
1203 bool run(Function *, bool ordered = false, bool skipPhi = false);
1206 // return false to continue with next entity on next higher level
1207 virtual bool visit(Function *) { return true; }
1208 virtual bool visit(BasicBlock *) { return true; }
1209 virtual bool visit(Instruction *) { return false; }
1211 bool doRun(Program *, bool ordered, bool skipPhi);
1212 bool doRun(Function *, bool ordered, bool skipPhi);
1220 // =============================================================================
1222 #include "codegen/nv50_ir_inlines.h"
1224 } // namespace nv50_ir
1226 #endif // __NV50_IR_H__