#include "System/Types.hpp"
#include "Vulkan/VkDebug.hpp"
#include "Vulkan/VkConfig.h"
+#include "Device/Config.hpp"
+
+#include <spirv/unified1/spirv.hpp>
#include <array>
#include <cstring>
#include <functional>
#include <string>
#include <vector>
+#include <unordered_set>
#include <unordered_map>
#include <cstdint>
#include <type_traits>
#include <memory>
-#include <spirv/unified1/spirv.hpp>
-#include <Device/Config.hpp>
namespace vk
{
class Intermediate
{
public:
- using Scalar = RValue<SIMD::Float>;
-
- Intermediate(uint32_t size) : contents(new ContentsType[size]), size(size) {
-#if !defined(NDEBUG) || defined(DCHECK_ALWAYS_ON)
- memset(contents, 0, sizeof(ContentsType) * size);
-#endif
+ Intermediate(uint32_t size) : scalar(new rr::Value*[size]), size(size) {
+ memset(scalar, 0, sizeof(rr::Value*) * size);
}
~Intermediate()
{
- for (auto i = 0u; i < size; i++)
- reinterpret_cast<Scalar *>(&contents[i])->~Scalar();
- delete [] contents;
+ delete[] scalar;
}
- void emplace(uint32_t n, Scalar&& value)
+ void move(uint32_t i, RValue<SIMD::Float> &&scalar) { emplace(i, scalar.value); }
+ void move(uint32_t i, RValue<SIMD::Int> &&scalar) { emplace(i, scalar.value); }
+ void move(uint32_t i, RValue<SIMD::UInt> &&scalar) { emplace(i, scalar.value); }
+
+ void move(uint32_t i, const RValue<SIMD::Float> &scalar) { emplace(i, scalar.value); }
+ void move(uint32_t i, const RValue<SIMD::Int> &scalar) { emplace(i, scalar.value); }
+ void move(uint32_t i, const RValue<SIMD::UInt> &scalar) { emplace(i, scalar.value); }
+
+ void replace(uint32_t i, RValue<SIMD::Float> &&scalar) { replace(i, scalar.value); }
+ void replace(uint32_t i, RValue<SIMD::Int> &&scalar) { replace(i, scalar.value); }
+ void replace(uint32_t i, RValue<SIMD::UInt> &&scalar) { replace(i, scalar.value); }
+
+ void replace(uint32_t i, const RValue<SIMD::Float> &scalar) { replace(i, scalar.value); }
+ void replace(uint32_t i, const RValue<SIMD::Int> &scalar) { replace(i, scalar.value); }
+ void replace(uint32_t i, const RValue<SIMD::UInt> &scalar) { replace(i, scalar.value); }
+
+ // Value retrieval functions.
+ RValue<SIMD::Float> Float(uint32_t i) const
{
- ASSERT(n < size);
- ASSERT(reinterpret_cast<Scalar const *>(&contents[n])->value == nullptr);
- new (&contents[n]) Scalar(value);
+ ASSERT(i < size);
+ ASSERT(scalar[i] != nullptr);
+ return As<SIMD::Float>(scalar[i]); // TODO(b/128539387): RValue<SIMD::Float>(scalar)
}
- void emplace(uint32_t n, const Scalar& value)
+ RValue<SIMD::Int> Int(uint32_t i) const
{
- ASSERT(n < size);
- ASSERT(reinterpret_cast<Scalar const *>(&contents[n])->value == nullptr);
- new (&contents[n]) Scalar(value);
+ ASSERT(i < size);
+ ASSERT(scalar[i] != nullptr);
+ return As<SIMD::Int>(scalar[i]); // TODO(b/128539387): RValue<SIMD::Int>(scalar)
}
- // Emplace with cast helpers.
- void emplace(uint32_t n, const RValue<SIMD::Int>& value) { emplace(n, As<SIMD::Float>(value)); }
- void emplace(uint32_t n, const RValue<SIMD::UInt>& value) { emplace(n, As<SIMD::Float>(value)); }
-
- // Value retrieval functions.
- RValue<SIMD::Float> Float(uint32_t i) const
+ RValue<SIMD::UInt> UInt(uint32_t i) const
{
ASSERT(i < size);
- auto scalar = reinterpret_cast<Scalar const *>(&contents[i]);
- ASSERT(scalar->value != nullptr);
- return *scalar;
+ ASSERT(scalar[i] != nullptr);
+ return As<SIMD::UInt>(scalar[i]); // TODO(b/128539387): RValue<SIMD::UInt>(scalar)
}
- RValue<SIMD::Int> Int(uint32_t i) const { return As<SIMD::Int>(Float(i)); }
- RValue<SIMD::UInt> UInt(uint32_t i) const { return As<SIMD::UInt>(Float(i)); }
// No copy/move construction or assignment
Intermediate(Intermediate const &) = delete;
Intermediate & operator=(Intermediate &&) = delete;
private:
- using ContentsType = std::aligned_storage<sizeof(Scalar), alignof(Scalar)>::type;
+ void emplace(uint32_t i, rr::Value *value)
+ {
+ ASSERT(i < size);
+ ASSERT(scalar[i] == nullptr);
+ scalar[i] = value;
+ }
- ContentsType *contents;
+ void replace(uint32_t i, rr::Value *value)
+ {
+ ASSERT(i < size);
+ scalar[i] = value;
+ }
+
+ rr::Value **const scalar;
uint32_t size;
};
return &iter[n];
}
+ bool operator==(InsnIterator const &other) const
+ {
+ return iter == other.iter;
+ }
+
bool operator!=(InsnIterator const &other) const
{
return iter != other.iter;
public:
using ID = SpirvID<Type>;
+ spv::Op opcode() const { return definition.opcode(); }
+
InsnIterator definition;
spv::StorageClass storageClass = static_cast<spv::StorageClass>(-1);
uint32_t sizeInComponents = 0;
public:
using ID = SpirvID<Object>;
+ spv::Op opcode() const { return definition.opcode(); }
+
InsnIterator definition;
Type::ID type;
ID pointerBase;
{
public:
using ID = SpirvID<Block>;
+ using Set = std::unordered_set<ID>;
+
+ // Edge represents the graph edge between two blocks.
+ struct Edge
+ {
+ ID from;
+ ID to;
+
+ bool operator == (const Edge& other) const { return from == other.from && to == other.to; }
+
+ struct Hash
+ {
+ std::size_t operator()(const Edge& edge) const noexcept
+ {
+ return std::hash<uint32_t>()(edge.from.value() * 31 + edge.to.value());
+ }
+ };
+ };
Block() = default;
Block(const Block& other) = default;
- explicit Block(InsnIterator begin, InsnIterator end) : begin_(begin), end_(end) {}
+ explicit Block(InsnIterator begin, InsnIterator end);
/* range-based-for interface */
inline InsnIterator begin() const { return begin_; }
inline InsnIterator end() const { return end_; }
+ enum Kind
+ {
+ Simple, // OpBranch or other simple terminator.
+ StructuredBranchConditional, // OpSelectionMerge + OpBranchConditional
+ UnstructuredBranchConditional, // OpBranchConditional
+ StructuredSwitch, // OpSelectionMerge + OpSwitch
+ UnstructuredSwitch, // OpSwitch
+ Loop, // OpLoopMerge + [OpBranchConditional | OpBranch]
+ };
+
+ Kind kind;
+ InsnIterator mergeInstruction; // Merge instruction.
+ InsnIterator branchInstruction; //
+ ID mergeBlock; // Structured flow merge block.
+ ID continueTarget; // Loop continue block.
+ Set ins; // Blocks that branch into this block.
+ Set outs; // Blocks that this block branches to.
+
private:
InsnIterator begin_;
InsnIterator end_;
std::vector<InterfaceComponent> outputs;
void emitProlog(SpirvRoutine *routine) const;
- void emit(SpirvRoutine *routine) const;
+ void emit(SpirvRoutine *routine, RValue<SIMD::Int> const &activeLaneMask) const;
void emitEpilog(SpirvRoutine *routine) const;
using BuiltInHash = std::hash<std::underlying_type<spv::BuiltIn>::type>;
Block const &getBlock(Block::ID id) const
{
auto it = blocks.find(id);
- ASSERT(it != blocks.end());
+ ASSERT_MSG(it != blocks.end(), "Unknown block %d", id.value());
return it->second;
}
HandleMap<Block> blocks;
Block::ID mainBlockId; // Block of the entry point function.
- void EmitBlock(SpirvRoutine *routine, Block const &block) const;
- void EmitInstruction(SpirvRoutine *routine, InsnIterator insn) const;
-
// DeclareType creates a Type for the given OpTypeX instruction, storing
// it into the types map. It is called from the analysis pass (constructor).
void DeclareType(InsnIterator insn);
SIMD::Int WalkAccessChain(Object::ID id, uint32_t numIndexes, uint32_t const *indexIds, SpirvRoutine *routine) const;
uint32_t WalkLiteralAccessChain(Type::ID id, uint32_t numIndexes, uint32_t const *indexes) const;
+ // EmitState holds control-flow state for the emit() pass.
+ class EmitState
+ {
+ public:
+ RValue<SIMD::Int> activeLaneMask() const
+ {
+ ASSERT(activeLaneMaskValue != nullptr);
+ return RValue<SIMD::Int>(activeLaneMaskValue);
+ }
+
+ void setActiveLaneMask(RValue<SIMD::Int> mask)
+ {
+ activeLaneMaskValue = mask.value;
+ }
+
+ // Add a new active lane mask edge from the current block to out.
+ // The edge mask value will be (mask AND activeLaneMaskValue).
+ // If multiple active lane masks are added for the same edge, then
+ // they will be ORed together.
+ void addOutputActiveLaneMaskEdge(Block::ID out, RValue<SIMD::Int> mask);
+
+ // Add a new active lane mask for the edge from -> to.
+ // If multiple active lane masks are added for the same edge, then
+ // they will be ORed together.
+ void addActiveLaneMaskEdge(Block::ID from, Block::ID to, RValue<SIMD::Int> mask);
+
+ // Lookup the active lane mask for the edge from -> to.
+ // Asserts if the edge does not exist.
+ RValue<SIMD::Int> getActiveLaneMaskEdge(Block::ID from, Block::ID to);
+
+ SpirvRoutine *routine = nullptr; // The current routine being built.
+ rr::Value *activeLaneMaskValue = nullptr; // The current active lane mask.
+ Block::ID currentBlock; // The current block being built.
+ Block::Set visited; // Blocks already built.
+ std::unordered_map<Block::Edge, RValue<SIMD::Int>, Block::Edge::Hash> edgeActiveLaneMasks;
+ };
+
+ // EmitResult is an enumerator of result values from the Emit functions.
+ enum class EmitResult
+ {
+ Continue, // No termination instructions.
+ Terminator, // Reached a termination instruction.
+ };
+
+ // existsPath returns true if there's a direct or indirect flow from
+ // the 'from' block to the 'to' block.
+ bool existsPath(Block::ID from, Block::ID to) const;
+
+ void EmitBlock(Block::ID id, EmitState *state) const;
+ void EmitInstructions(InsnIterator begin, InsnIterator end, EmitState *state) const;
+ void EmitLoop(EmitState *state) const;
+ EmitResult EmitInstruction(InsnIterator insn, EmitState *state) const;
+
// Emit pass instructions:
- void EmitVariable(InsnIterator insn, SpirvRoutine *routine) const;
- void EmitLoad(InsnIterator insn, SpirvRoutine *routine) const;
- void EmitStore(InsnIterator insn, SpirvRoutine *routine) const;
- void EmitAccessChain(InsnIterator insn, SpirvRoutine *routine) const;
- void EmitCompositeConstruct(InsnIterator insn, SpirvRoutine *routine) const;
- void EmitCompositeInsert(InsnIterator insn, SpirvRoutine *routine) const;
- void EmitCompositeExtract(InsnIterator insn, SpirvRoutine *routine) const;
- void EmitVectorShuffle(InsnIterator insn, SpirvRoutine *routine) const;
- void EmitVectorTimesScalar(InsnIterator insn, SpirvRoutine *routine) const;
- void EmitVectorExtractDynamic(InsnIterator insn, SpirvRoutine *routine) const;
- void EmitVectorInsertDynamic(InsnIterator insn, SpirvRoutine *routine) const;
- void EmitUnaryOp(InsnIterator insn, SpirvRoutine *routine) const;
- void EmitBinaryOp(InsnIterator insn, SpirvRoutine *routine) const;
- void EmitDot(InsnIterator insn, SpirvRoutine *routine) const;
- void EmitSelect(InsnIterator insn, SpirvRoutine *routine) const;
- void EmitExtendedInstruction(InsnIterator insn, SpirvRoutine *routine) const;
- void EmitAny(InsnIterator insn, SpirvRoutine *routine) const;
- void EmitAll(InsnIterator insn, SpirvRoutine *routine) const;
- void EmitBranch(InsnIterator insn, SpirvRoutine *routine) const;
-
- // OpcodeName returns the name of the opcode op.
- // If NDEBUG is defined, then OpcodeName will only return the numerical code.
+ EmitResult EmitVariable(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitLoad(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitStore(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitAccessChain(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitCompositeConstruct(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitCompositeInsert(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitCompositeExtract(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitVectorShuffle(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitVectorTimesScalar(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitVectorExtractDynamic(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitVectorInsertDynamic(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitUnaryOp(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitBinaryOp(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitDot(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitSelect(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitExtendedInstruction(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitAny(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitAll(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitBranch(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitBranchConditional(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitSwitch(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitUnreachable(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitReturn(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitPhi(InsnIterator insn, EmitState *state) const;
+
+ // OpcodeName() returns the name of the opcode op.
+ // If NDEBUG is defined, then OpcodeName() will only return the numerical code.
static std::string OpcodeName(spv::Op op);
+ static std::memory_order MemoryOrder(spv::MemorySemanticsMask memorySemantics);
// Helper as we often need to take dot products as part of doing other things.
SIMD::Float Dot(unsigned numComponents, GenericValue const & x, GenericValue const & y) const;
Value inputs = Value{MAX_INTERFACE_COMPONENTS};
Value outputs = Value{MAX_INTERFACE_COMPONENTS};
- SIMD::Int activeLaneMask = SIMD::Int(0xFFFFFFFF);
-
std::array<Pointer<Byte>, vk::MAX_BOUND_DESCRIPTOR_SETS> descriptorSets;
Pointer<Byte> pushConstants;
Value& getValue(SpirvShader::Object::ID id)
{
auto it = lvalues.find(id);
- ASSERT(it != lvalues.end());
+ ASSERT_MSG(it != lvalues.end(), "Unknown value %d", id.value());
return it->second;
}
Intermediate const& getIntermediate(SpirvShader::Object::ID id) const
{
auto it = intermediates.find(id);
- ASSERT(it != intermediates.end());
+ ASSERT_MSG(it != intermediates.end(), "Unknown intermediate %d", id.value());
return it->second;
}
Pointer<Byte>& getPhysicalPointer(SpirvShader::Object::ID id)
{
auto it = physicalPointers.find(id);
- assert(it != physicalPointers.end());
+ ASSERT_MSG(it != physicalPointers.end(), "Unknown physical pointer %d", id.value());
return it->second;
}
};
OSDN Git Service
