#ifndef sw_SpirvShader_hpp
#define sw_SpirvShader_hpp
-#include "System/Types.hpp"
-#include "Vulkan/VkDebug.hpp"
#include "ShaderCore.hpp"
#include "SpirvID.hpp"
+#include "System/Types.hpp"
+#include "Vulkan/VkDebug.hpp"
+#include "Vulkan/VkConfig.h"
+#include "Device/Config.hpp"
+
+#include <spirv/unified1/spirv.hpp>
+#include <array>
+#include <cstring>
+#include <functional>
#include <string>
#include <vector>
+#include <unordered_set>
#include <unordered_map>
#include <cstdint>
#include <type_traits>
#include <memory>
-#include <spirv/unified1/spirv.hpp>
-#include <Device/Config.hpp>
+
+namespace vk
+{
+ class PipelineLayout;
+} // namespace vk
namespace sw
{
// Forward declarations.
class SpirvRoutine;
+ class GenericValue;
// SIMD contains types that represent multiple scalars packed into a single
// vector data type. Types in the SIMD namespace provide a semantic hint
using Float = rr::Float4;
using Int = rr::Int4;
+ using UInt = rr::UInt4;
}
// Incrementally constructed complex bundle of rvalues
class Intermediate
{
public:
- using Scalar = RValue<SIMD::Float>;
-
- Intermediate(uint32_t size) : contents(new ContentsType[size]), size(size) {}
+ Intermediate(uint32_t size) : scalar(new rr::Value*[size]), size(size) {
+ memset(scalar, 0, sizeof(rr::Value*) * size);
+ }
~Intermediate()
{
- for (auto i = 0u; i < size; i++)
- reinterpret_cast<Scalar *>(&contents[i])->~Scalar();
- delete [] contents;
+ delete[] scalar;
}
- void emplace(uint32_t n, Scalar&& value)
+ void move(uint32_t i, RValue<SIMD::Float> &&scalar) { emplace(i, scalar.value); }
+ void move(uint32_t i, RValue<SIMD::Int> &&scalar) { emplace(i, scalar.value); }
+ void move(uint32_t i, RValue<SIMD::UInt> &&scalar) { emplace(i, scalar.value); }
+
+ void move(uint32_t i, const RValue<SIMD::Float> &scalar) { emplace(i, scalar.value); }
+ void move(uint32_t i, const RValue<SIMD::Int> &scalar) { emplace(i, scalar.value); }
+ void move(uint32_t i, const RValue<SIMD::UInt> &scalar) { emplace(i, scalar.value); }
+
+ void replace(uint32_t i, RValue<SIMD::Float> &&scalar) { replace(i, scalar.value); }
+ void replace(uint32_t i, RValue<SIMD::Int> &&scalar) { replace(i, scalar.value); }
+ void replace(uint32_t i, RValue<SIMD::UInt> &&scalar) { replace(i, scalar.value); }
+
+ void replace(uint32_t i, const RValue<SIMD::Float> &scalar) { replace(i, scalar.value); }
+ void replace(uint32_t i, const RValue<SIMD::Int> &scalar) { replace(i, scalar.value); }
+ void replace(uint32_t i, const RValue<SIMD::UInt> &scalar) { replace(i, scalar.value); }
+
+ // Value retrieval functions.
+ RValue<SIMD::Float> Float(uint32_t i) const
{
- assert(n < size);
- new (&contents[n]) Scalar(value);
+ ASSERT(i < size);
+ ASSERT(scalar[i] != nullptr);
+ return As<SIMD::Float>(scalar[i]); // TODO(b/128539387): RValue<SIMD::Float>(scalar)
}
- void emplace(uint32_t n, const Scalar& value)
+ RValue<SIMD::Int> Int(uint32_t i) const
{
- assert(n < size);
- new (&contents[n]) Scalar(value);
+ ASSERT(i < size);
+ ASSERT(scalar[i] != nullptr);
+ return As<SIMD::Int>(scalar[i]); // TODO(b/128539387): RValue<SIMD::Int>(scalar)
}
- Scalar const & operator[](uint32_t n) const
+ RValue<SIMD::UInt> UInt(uint32_t i) const
{
- assert(n < size);
- return *reinterpret_cast<Scalar const *>(&contents[n]);
+ ASSERT(i < size);
+ ASSERT(scalar[i] != nullptr);
+ return As<SIMD::UInt>(scalar[i]); // TODO(b/128539387): RValue<SIMD::UInt>(scalar)
}
// No copy/move construction or assignment
Intermediate & operator=(Intermediate &&) = delete;
private:
- using ContentsType = std::aligned_storage<sizeof(Scalar), alignof(Scalar)>::type;
+ void emplace(uint32_t i, rr::Value *value)
+ {
+ ASSERT(i < size);
+ ASSERT(scalar[i] == nullptr);
+ scalar[i] = value;
+ }
+
+ void replace(uint32_t i, rr::Value *value)
+ {
+ ASSERT(i < size);
+ scalar[i] = value;
+ }
- ContentsType *contents;
+ rr::Value **const scalar;
uint32_t size;
};
return &iter[n];
}
+ bool operator==(InsnIterator const &other) const
+ {
+ return iter == other.iter;
+ }
+
bool operator!=(InsnIterator const &other) const
{
return iter != other.iter;
return InsnIterator{insns.cend()};
}
- class Type;
- using TypeID = SpirvID<Type>;
-
class Type
{
public:
+ using ID = SpirvID<Type>;
+
+ spv::Op opcode() const { return definition.opcode(); }
+
InsnIterator definition;
spv::StorageClass storageClass = static_cast<spv::StorageClass>(-1);
uint32_t sizeInComponents = 0;
bool isBuiltInBlock = false;
// Inner element type for pointers, arrays, vectors and matrices.
- TypeID element;
+ ID element;
};
- class Object;
- using ObjectID = SpirvID<Object>;
-
class Object
{
public:
+ using ID = SpirvID<Object>;
+
+ spv::Op opcode() const { return definition.opcode(); }
+
InsnIterator definition;
- TypeID type;
- ObjectID pointerBase;
+ Type::ID type;
+ ID pointerBase;
std::unique_ptr<uint32_t[]> constantValue = nullptr;
enum class Kind
{
Unknown, /* for paranoia -- if we get left with an object in this state, the module was broken */
- Variable,
- InterfaceVariable,
- Constant,
- Value,
+ Variable, // TODO: Document
+ InterfaceVariable, // TODO: Document
+ Constant, // Values held by Object::constantValue
+ Value, // Values held by SpirvRoutine::intermediates
+ PhysicalPointer, // Pointer held by SpirvRoutine::physicalPointers
} kind = Kind::Unknown;
};
+ // Block is an interval of SPIR-V instructions, starting with the
+ // opening OpLabel, and ending with a termination instruction.
+ class Block
+ {
+ public:
+ using ID = SpirvID<Block>;
+ using Set = std::unordered_set<ID>;
+
+ // Edge represents the graph edge between two blocks.
+ struct Edge
+ {
+ ID from;
+ ID to;
+
+ bool operator == (const Edge& other) const { return from == other.from && to == other.to; }
+
+ struct Hash
+ {
+ std::size_t operator()(const Edge& edge) const noexcept
+ {
+ return std::hash<uint32_t>()(edge.from.value() * 31 + edge.to.value());
+ }
+ };
+ };
+
+ Block() = default;
+ Block(const Block& other) = default;
+ explicit Block(InsnIterator begin, InsnIterator end);
+
+ /* range-based-for interface */
+ inline InsnIterator begin() const { return begin_; }
+ inline InsnIterator end() const { return end_; }
+
+ enum Kind
+ {
+ Simple, // OpBranch or other simple terminator.
+ StructuredBranchConditional, // OpSelectionMerge + OpBranchConditional
+ UnstructuredBranchConditional, // OpBranchConditional
+ StructuredSwitch, // OpSelectionMerge + OpSwitch
+ UnstructuredSwitch, // OpSwitch
+ Loop, // OpLoopMerge + [OpBranchConditional | OpBranch]
+ };
+
+ Kind kind;
+ InsnIterator mergeInstruction; // Merge instruction.
+ InsnIterator branchInstruction; //
+ ID mergeBlock; // Structured flow merge block.
+ ID continueTarget; // Loop continue block.
+ Set ins; // Blocks that branch into this block.
+ Set outs; // Blocks that this block branches to.
+
+ private:
+ InsnIterator begin_;
+ InsnIterator end_;
+ };
+
struct TypeOrObject {}; // Dummy struct to represent a Type or Object.
// TypeOrObjectID is an identifier that represents a Type or an Object,
- // and supports implicit casting to and from TypeID or ObjectID.
+ // and supports implicit casting to and from Type::ID or Object::ID.
class TypeOrObjectID : public SpirvID<TypeOrObject>
{
public:
using Hash = std::hash<SpirvID<TypeOrObject>>;
inline TypeOrObjectID(uint32_t id) : SpirvID(id) {}
- inline TypeOrObjectID(TypeID id) : SpirvID(id.value()) {}
- inline TypeOrObjectID(ObjectID id) : SpirvID(id.value()) {}
- inline operator TypeID() const { return TypeID(value()); }
- inline operator ObjectID() const { return ObjectID(value()); }
+ inline TypeOrObjectID(Type::ID id) : SpirvID(id.value()) {}
+ inline TypeOrObjectID(Object::ID id) : SpirvID(id.value()) {}
+ inline operator Type::ID() const { return Type::ID(value()); }
+ inline operator Object::ID() const { return Object::ID(value()); }
};
int getSerialID() const
bool NeedsCentroid : 1;
// Compute workgroup dimensions
- int LocalSizeX, LocalSizeY, LocalSizeZ;
+ int WorkgroupSizeX = 1, WorkgroupSizeY = 1, WorkgroupSizeZ = 1;
};
Modes const &getModes() const
int32_t DescriptorSet;
int32_t Binding;
spv::BuiltIn BuiltIn;
+ int32_t Offset;
+ int32_t ArrayStride;
+ int32_t MatrixStride;
bool HasLocation : 1;
bool HasComponent : 1;
bool HasDescriptorSet : 1;
bool NoPerspective : 1;
bool Block : 1;
bool BufferBlock : 1;
+ bool HasOffset : 1;
+ bool HasArrayStride : 1;
+ bool HasMatrixStride : 1;
Decorations()
: Location{-1}, Component{0}, DescriptorSet{-1}, Binding{-1},
BuiltIn{static_cast<spv::BuiltIn>(-1)},
+ Offset{-1}, ArrayStride{-1}, MatrixStride{-1},
HasLocation{false}, HasComponent{false},
HasDescriptorSet{false}, HasBinding{false},
HasBuiltIn{false}, Flat{false}, Centroid{false},
- NoPerspective{false}, Block{false}, BufferBlock{false}
+ NoPerspective{false}, Block{false}, BufferBlock{false},
+ HasOffset{false}, HasArrayStride{false}, HasMatrixStride{false}
{
}
};
std::unordered_map<TypeOrObjectID, Decorations, TypeOrObjectID::Hash> decorations;
- std::unordered_map<TypeID, std::vector<Decorations>> memberDecorations;
+ std::unordered_map<Type::ID, std::vector<Decorations>> memberDecorations;
struct InterfaceComponent
{
struct BuiltinMapping
{
- ObjectID Id;
+ Object::ID Id;
uint32_t FirstComponent;
uint32_t SizeInComponents;
};
std::vector<InterfaceComponent> outputs;
void emitProlog(SpirvRoutine *routine) const;
- void emit(SpirvRoutine *routine) const;
+ void emit(SpirvRoutine *routine, RValue<SIMD::Int> const &activeLaneMask) const;
void emitEpilog(SpirvRoutine *routine) const;
using BuiltInHash = std::hash<std::underlying_type<spv::BuiltIn>::type>;
std::unordered_map<spv::BuiltIn, BuiltinMapping, BuiltInHash> inputBuiltins;
std::unordered_map<spv::BuiltIn, BuiltinMapping, BuiltInHash> outputBuiltins;
- Type const &getType(TypeID id) const
+ Type const &getType(Type::ID id) const
{
auto it = types.find(id);
- assert(it != types.end());
+ ASSERT_MSG(it != types.end(), "Unknown type %d", id.value());
return it->second;
}
- Object const &getObject(ObjectID id) const
+ Object const &getObject(Object::ID id) const
{
auto it = defs.find(id);
- assert(it != defs.end());
+ ASSERT_MSG(it != defs.end(), "Unknown object %d", id.value());
+ return it->second;
+ }
+
+ Block const &getBlock(Block::ID id) const
+ {
+ auto it = blocks.find(id);
+ ASSERT_MSG(it != blocks.end(), "Unknown block %d", id.value());
return it->second;
}
Modes modes;
HandleMap<Type> types;
HandleMap<Object> defs;
+ HandleMap<Block> blocks;
+ Block::ID mainBlockId; // Block of the entry point function.
// DeclareType creates a Type for the given OpTypeX instruction, storing
// it into the types map. It is called from the analysis pass (constructor).
uint32_t ComputeTypeSize(InsnIterator insn);
void ApplyDecorationsForId(Decorations *d, TypeOrObjectID id) const;
- void ApplyDecorationsForIdMember(Decorations *d, TypeID id, uint32_t member) const;
+ void ApplyDecorationsForIdMember(Decorations *d, Type::ID id, uint32_t member) const;
+
+ // Returns true if data in the given storage class is word-interleaved
+ // by each SIMD vector lane, otherwise data is linerally stored.
+ //
+ // A 'lane' is a component of a SIMD vector register.
+ // Given 4 consecutive loads/stores of 4 SIMD vector registers:
+ //
+ // "StorageInterleavedByLane":
+ //
+ // Ptr+0:Reg0.x | Ptr+1:Reg0.y | Ptr+2:Reg0.z | Ptr+3:Reg0.w
+ // --------------+--------------+--------------+--------------
+ // Ptr+4:Reg1.x | Ptr+5:Reg1.y | Ptr+6:Reg1.z | Ptr+7:Reg1.w
+ // --------------+--------------+--------------+--------------
+ // Ptr+8:Reg2.x | Ptr+9:Reg2.y | Ptr+a:Reg2.z | Ptr+b:Reg2.w
+ // --------------+--------------+--------------+--------------
+ // Ptr+c:Reg3.x | Ptr+d:Reg3.y | Ptr+e:Reg3.z | Ptr+f:Reg3.w
+ //
+ // Not "StorageInterleavedByLane":
+ //
+ // Ptr+0:Reg0.x | Ptr+0:Reg0.y | Ptr+0:Reg0.z | Ptr+0:Reg0.w
+ // --------------+--------------+--------------+--------------
+ // Ptr+1:Reg1.x | Ptr+1:Reg1.y | Ptr+1:Reg1.z | Ptr+1:Reg1.w
+ // --------------+--------------+--------------+--------------
+ // Ptr+2:Reg2.x | Ptr+2:Reg2.y | Ptr+2:Reg2.z | Ptr+2:Reg2.w
+ // --------------+--------------+--------------+--------------
+ // Ptr+3:Reg3.x | Ptr+3:Reg3.y | Ptr+3:Reg3.z | Ptr+3:Reg3.w
+ //
+ static bool IsStorageInterleavedByLane(spv::StorageClass storageClass);
template<typename F>
- int VisitInterfaceInner(TypeID id, Decorations d, F f) const;
+ int VisitInterfaceInner(Type::ID id, Decorations d, F f) const;
template<typename F>
- void VisitInterface(ObjectID id, F f) const;
+ void VisitInterface(Object::ID id, F f) const;
- uint32_t GetConstantInt(ObjectID id) const;
+ uint32_t GetConstantInt(Object::ID id) const;
Object& CreateConstant(InsnIterator it);
void ProcessInterfaceVariable(Object &object);
- SIMD::Int WalkAccessChain(ObjectID id, uint32_t numIndexes, uint32_t const *indexIds, SpirvRoutine *routine) const;
- uint32_t WalkLiteralAccessChain(TypeID id, uint32_t numIndexes, uint32_t const *indexes) const;
+ SIMD::Int WalkExplicitLayoutAccessChain(Object::ID id, uint32_t numIndexes, uint32_t const *indexIds, SpirvRoutine *routine) const;
+ SIMD::Int WalkAccessChain(Object::ID id, uint32_t numIndexes, uint32_t const *indexIds, SpirvRoutine *routine) const;
+ uint32_t WalkLiteralAccessChain(Type::ID id, uint32_t numIndexes, uint32_t const *indexes) const;
+
+ // EmitState holds control-flow state for the emit() pass.
+ class EmitState
+ {
+ public:
+ RValue<SIMD::Int> activeLaneMask() const
+ {
+ ASSERT(activeLaneMaskValue != nullptr);
+ return RValue<SIMD::Int>(activeLaneMaskValue);
+ }
+
+ void setActiveLaneMask(RValue<SIMD::Int> mask)
+ {
+ activeLaneMaskValue = mask.value;
+ }
+
+ // Add a new active lane mask edge from the current block to out.
+ // The edge mask value will be (mask AND activeLaneMaskValue).
+ // If multiple active lane masks are added for the same edge, then
+ // they will be ORed together.
+ void addOutputActiveLaneMaskEdge(Block::ID out, RValue<SIMD::Int> mask);
+
+ // Add a new active lane mask for the edge from -> to.
+ // If multiple active lane masks are added for the same edge, then
+ // they will be ORed together.
+ void addActiveLaneMaskEdge(Block::ID from, Block::ID to, RValue<SIMD::Int> mask);
+
+ // Lookup the active lane mask for the edge from -> to.
+ // Asserts if the edge does not exist.
+ RValue<SIMD::Int> getActiveLaneMaskEdge(Block::ID from, Block::ID to);
+
+ SpirvRoutine *routine = nullptr; // The current routine being built.
+ rr::Value *activeLaneMaskValue = nullptr; // The current active lane mask.
+ Block::ID currentBlock; // The current block being built.
+ Block::Set visited; // Blocks already built.
+ std::unordered_map<Block::Edge, RValue<SIMD::Int>, Block::Edge::Hash> edgeActiveLaneMasks;
+ };
+
+ // EmitResult is an enumerator of result values from the Emit functions.
+ enum class EmitResult
+ {
+ Continue, // No termination instructions.
+ Terminator, // Reached a termination instruction.
+ };
+
+ // existsPath returns true if there's a direct or indirect flow from
+ // the 'from' block to the 'to' block.
+ bool existsPath(Block::ID from, Block::ID to) const;
+
+ void EmitBlock(Block::ID id, EmitState *state) const;
+ void EmitInstructions(InsnIterator begin, InsnIterator end, EmitState *state) const;
+ void EmitLoop(EmitState *state) const;
+ EmitResult EmitInstruction(InsnIterator insn, EmitState *state) const;
+
+ // Emit pass instructions:
+ EmitResult EmitVariable(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitLoad(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitStore(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitAccessChain(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitCompositeConstruct(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitCompositeInsert(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitCompositeExtract(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitVectorShuffle(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitVectorTimesScalar(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitVectorExtractDynamic(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitVectorInsertDynamic(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitUnaryOp(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitBinaryOp(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitDot(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitSelect(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitExtendedInstruction(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitAny(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitAll(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitBranch(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitBranchConditional(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitSwitch(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitUnreachable(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitReturn(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitPhi(InsnIterator insn, EmitState *state) const;
+
+ // OpcodeName() returns the name of the opcode op.
+ // If NDEBUG is defined, then OpcodeName() will only return the numerical code.
+ static std::string OpcodeName(spv::Op op);
+ static std::memory_order MemoryOrder(spv::MemorySemanticsMask memorySemantics);
+
+ // Helper as we often need to take dot products as part of doing other things.
+ SIMD::Float Dot(unsigned numComponents, GenericValue const & x, GenericValue const & y) const;
};
class SpirvRoutine
{
public:
+ SpirvRoutine(vk::PipelineLayout const *pipelineLayout);
+
using Value = Array<SIMD::Float>;
- std::unordered_map<SpirvShader::ObjectID, Value> lvalues;
+ vk::PipelineLayout const * const pipelineLayout;
+
+ std::unordered_map<SpirvShader::Object::ID, Value> lvalues;
+
+ std::unordered_map<SpirvShader::Object::ID, Intermediate> intermediates;
- std::unordered_map<SpirvShader::ObjectID, Intermediate> intermediates;
+ std::unordered_map<SpirvShader::Object::ID, Pointer<Byte> > physicalPointers;
Value inputs = Value{MAX_INTERFACE_COMPONENTS};
Value outputs = Value{MAX_INTERFACE_COMPONENTS};
- void createLvalue(SpirvShader::ObjectID id, uint32_t size)
+ std::array<Pointer<Byte>, vk::MAX_BOUND_DESCRIPTOR_SETS> descriptorSets;
+ Pointer<Byte> pushConstants;
+
+ void createLvalue(SpirvShader::Object::ID id, uint32_t size)
{
lvalues.emplace(id, Value(size));
}
- Intermediate& createIntermediate(SpirvShader::ObjectID id, uint32_t size)
+ Intermediate& createIntermediate(SpirvShader::Object::ID id, uint32_t size)
{
auto it = intermediates.emplace(std::piecewise_construct,
std::forward_as_tuple(id),
return it.first->second;
}
- Value& getValue(SpirvShader::ObjectID id)
+ Value& getValue(SpirvShader::Object::ID id)
{
auto it = lvalues.find(id);
- assert(it != lvalues.end());
+ ASSERT_MSG(it != lvalues.end(), "Unknown value %d", id.value());
return it->second;
}
- Intermediate const& getIntermediate(SpirvShader::ObjectID id) const
+ Intermediate const& getIntermediate(SpirvShader::Object::ID id) const
{
auto it = intermediates.find(id);
- assert(it != intermediates.end());
+ ASSERT_MSG(it != intermediates.end(), "Unknown intermediate %d", id.value());
+ return it->second;
+ }
+
+ Pointer<Byte>& getPhysicalPointer(SpirvShader::Object::ID id)
+ {
+ auto it = physicalPointers.find(id);
+ ASSERT_MSG(it != physicalPointers.end(), "Unknown physical pointer %d", id.value());
return it->second;
}
};
Intermediate const *intermediate;
public:
- GenericValue(SpirvShader const *shader, SpirvRoutine const *routine, SpirvShader::ObjectID objId) :
+ GenericValue(SpirvShader const *shader, SpirvRoutine const *routine, SpirvShader::Object::ID objId) :
obj(shader->getObject(objId)),
intermediate(obj.kind == SpirvShader::Object::Kind::Value ? &routine->getIntermediate(objId) : nullptr) {}
- RValue<SIMD::Float> operator[](uint32_t i) const
+ RValue<SIMD::Float> Float(uint32_t i) const
{
- if (intermediate)
- return (*intermediate)[i];
-
+ if (intermediate != nullptr)
+ {
+ return intermediate->Float(i);
+ }
auto constantValue = reinterpret_cast<float *>(obj.constantValue.get());
return RValue<SIMD::Float>(constantValue[i]);
}
+
+ RValue<SIMD::Int> Int(uint32_t i) const
+ {
+ return As<SIMD::Int>(Float(i));
+ }
+
+ RValue<SIMD::UInt> UInt(uint32_t i) const
+ {
+ return As<SIMD::UInt>(Float(i));
+ }
};
}
OSDN Git Service
