using namespace mlir;
// TODO: generate these strings using ODS.
+static constexpr const char kMemoryAccessAttrName[] = "memory_access";
+static constexpr const char kSourceMemoryAccessAttrName[] =
+ "source_memory_access";
static constexpr const char kAlignmentAttrName[] = "alignment";
+static constexpr const char kSourceAlignmentAttrName[] = "source_alignment";
static constexpr const char kBranchWeightAttrName[] = "branch_weights";
static constexpr const char kCallee[] = "callee";
static constexpr const char kClusterSize[] = "cluster_size";
return success();
}
+/// Parses optional memory access attributes attached to a memory access
+/// operand/pointer. Specifically, parses the following syntax:
+/// (`[` memory-access `]`)?
+/// where:
+/// memory-access ::= `"None"` | `"Volatile"` | `"Aligned", `
+/// integer-literal | `"NonTemporal"`
static ParseResult parseMemoryAccessAttributes(OpAsmParser &parser,
OperationState &state) {
// Parse an optional list of attributes staring with '['
}
spirv::MemoryAccess memoryAccessAttr;
- if (parseEnumStrAttr(memoryAccessAttr, parser, state)) {
+ if (parseEnumStrAttr(memoryAccessAttr, parser, state,
+ kMemoryAccessAttrName)) {
return failure();
}
return parser.parseRSquare();
}
+// TODO Make sure to merge this and the previous function into one template
+// parameterized by memroy access attribute name and alignment. Doing so now
+// results in VS2017 in producing an internal error (at the call site) that's
+// not detailed enough to understand what is happenning.
+static ParseResult parseSourceMemoryAccessAttributes(OpAsmParser &parser,
+ OperationState &state) {
+ // Parse an optional list of attributes staring with '['
+ if (parser.parseOptionalLSquare()) {
+ // Nothing to do
+ return success();
+ }
+
+ spirv::MemoryAccess memoryAccessAttr;
+ if (parseEnumStrAttr(memoryAccessAttr, parser, state,
+ kSourceMemoryAccessAttrName)) {
+ return failure();
+ }
+
+ if (spirv::bitEnumContains(memoryAccessAttr, spirv::MemoryAccess::Aligned)) {
+ // Parse integer attribute for alignment.
+ Attribute alignmentAttr;
+ Type i32Type = parser.getBuilder().getIntegerType(32);
+ if (parser.parseComma() ||
+ parser.parseAttribute(alignmentAttr, i32Type, kSourceAlignmentAttrName,
+ state.attributes)) {
+ return failure();
+ }
+ }
+ return parser.parseRSquare();
+}
+
template <typename MemoryOpTy>
-static void
-printMemoryAccessAttribute(MemoryOpTy memoryOp, OpAsmPrinter &printer,
- SmallVectorImpl<StringRef> &elidedAttrs) {
+static void printMemoryAccessAttribute(
+ MemoryOpTy memoryOp, OpAsmPrinter &printer,
+ SmallVectorImpl<StringRef> &elidedAttrs,
+ Optional<spirv::MemoryAccess> memoryAccessAtrrValue = None,
+ Optional<llvm::APInt> alignmentAttrValue = None) {
// Print optional memory access attribute.
- if (auto memAccess = memoryOp.memory_access()) {
- elidedAttrs.push_back(spirv::attributeName<spirv::MemoryAccess>());
+ if (auto memAccess = (memoryAccessAtrrValue ? memoryAccessAtrrValue
+ : memoryOp.memory_access())) {
+ elidedAttrs.push_back(kMemoryAccessAttrName);
+
printer << " [\"" << stringifyMemoryAccess(*memAccess) << "\"";
- // Print integer alignment attribute.
- if (auto alignment = memoryOp.alignment()) {
- elidedAttrs.push_back(kAlignmentAttrName);
- printer << ", " << alignment;
+ if (spirv::bitEnumContains(*memAccess, spirv::MemoryAccess::Aligned)) {
+ // Print integer alignment attribute.
+ if (auto alignment = (alignmentAttrValue ? alignmentAttrValue
+ : memoryOp.alignment())) {
+ elidedAttrs.push_back(kAlignmentAttrName);
+ printer << ", " << alignment;
+ }
+ }
+ printer << "]";
+ }
+ elidedAttrs.push_back(spirv::attributeName<spirv::StorageClass>());
+}
+
+// TODO Make sure to merge this and the previous function into one template
+// parameterized by memroy access attribute name and alignment. Doing so now
+// results in VS2017 in producing an internal error (at the call site) that's
+// not detailed enough to understand what is happenning.
+template <typename MemoryOpTy>
+static void printSourceMemoryAccessAttribute(
+ MemoryOpTy memoryOp, OpAsmPrinter &printer,
+ SmallVectorImpl<StringRef> &elidedAttrs,
+ Optional<spirv::MemoryAccess> memoryAccessAtrrValue = None,
+ Optional<llvm::APInt> alignmentAttrValue = None) {
+
+ printer << ", ";
+
+ // Print optional memory access attribute.
+ if (auto memAccess = (memoryAccessAtrrValue ? memoryAccessAtrrValue
+ : memoryOp.memory_access())) {
+ elidedAttrs.push_back(kSourceMemoryAccessAttrName);
+
+ printer << " [\"" << stringifyMemoryAccess(*memAccess) << "\"";
+
+ if (spirv::bitEnumContains(*memAccess, spirv::MemoryAccess::Aligned)) {
+ // Print integer alignment attribute.
+ if (auto alignment = (alignmentAttrValue ? alignmentAttrValue
+ : memoryOp.alignment())) {
+ elidedAttrs.push_back(kSourceAlignmentAttrName);
+ printer << ", " << alignment;
+ }
}
printer << "]";
}
// memory-access attribute is Aligned, then the alignment attribute must be
// present.
auto *op = memoryOp.getOperation();
- auto memAccessAttr = op->getAttr(spirv::attributeName<spirv::MemoryAccess>());
+ auto memAccessAttr = op->getAttr(kMemoryAccessAttrName);
if (!memAccessAttr) {
// Alignment attribute shouldn't be present if memory access attribute is
// not present.
return success();
}
+// TODO Make sure to merge this and the previous function into one template
+// parameterized by memroy access attribute name and alignment. Doing so now
+// results in VS2017 in producing an internal error (at the call site) that's
+// not detailed enough to understand what is happenning.
+template <typename MemoryOpTy>
+static LogicalResult verifySourceMemoryAccessAttribute(MemoryOpTy memoryOp) {
+ // ODS checks for attributes values. Just need to verify that if the
+ // memory-access attribute is Aligned, then the alignment attribute must be
+ // present.
+ auto *op = memoryOp.getOperation();
+ auto memAccessAttr = op->getAttr(kSourceMemoryAccessAttrName);
+ if (!memAccessAttr) {
+ // Alignment attribute shouldn't be present if memory access attribute is
+ // not present.
+ if (op->getAttr(kSourceAlignmentAttrName)) {
+ return memoryOp.emitOpError(
+ "invalid alignment specification without aligned memory access "
+ "specification");
+ }
+ return success();
+ }
+
+ auto memAccessVal = memAccessAttr.template cast<IntegerAttr>();
+ auto memAccess = spirv::symbolizeMemoryAccess(memAccessVal.getInt());
+
+ if (!memAccess) {
+ return memoryOp.emitOpError("invalid memory access specifier: ")
+ << memAccessVal;
+ }
+
+ if (spirv::bitEnumContains(*memAccess, spirv::MemoryAccess::Aligned)) {
+ if (!op->getAttr(kSourceAlignmentAttrName)) {
+ return memoryOp.emitOpError("missing alignment value");
+ }
+ } else {
+ if (op->getAttr(kSourceAlignmentAttrName)) {
+ return memoryOp.emitOpError(
+ "invalid alignment specification with non-aligned memory access "
+ "specification");
+ }
+ }
+ return success();
+}
+
template <typename BarrierOp>
static LogicalResult verifyMemorySemantics(BarrierOp op) {
// According to the SPIR-V specification:
SmallVector<StringRef, 4> elidedAttrs;
printMemoryAccessAttribute(copyMemory, printer, elidedAttrs);
+ printSourceMemoryAccessAttribute(copyMemory, printer, elidedAttrs,
+ copyMemory.source_memory_access(),
+ copyMemory.source_alignment());
printer.printOptionalAttrDict(op->getAttrs(), elidedAttrs);
parser.parseOperand(targetPtrInfo) || parser.parseComma() ||
parseEnumStrAttr(sourceStorageClass, parser) ||
parser.parseOperand(sourcePtrInfo) ||
- parseMemoryAccessAttributes(parser, state) ||
- parser.parseOptionalAttrDict(state.attributes) || parser.parseColon() ||
- parser.parseType(elementType)) {
+ parseMemoryAccessAttributes(parser, state)) {
return failure();
}
+ if (!parser.parseOptionalComma()) {
+ // Parse 2nd memory access attributes.
+ if (parseSourceMemoryAccessAttributes(parser, state)) {
+ return failure();
+ }
+ }
+
+ if (parser.parseColon() || parser.parseType(elementType))
+ return failure();
+
+ if (parser.parseOptionalAttrDict(state.attributes))
+ return failure();
+
auto targetPtrType = spirv::PointerType::get(elementType, targetStorageClass);
auto sourcePtrType = spirv::PointerType::get(elementType, sourceStorageClass);
"both operands must be pointers to the same type");
}
- return verifyMemoryAccessAttribute(copyMemory);
+ if (failed(verifyMemoryAccessAttribute(copyMemory))) {
+ return failure();
+ }
+
+ // TODO - According to the spec:
+ //
+ // If two masks are present, the first applies to Target and cannot include
+ // MakePointerVisible, and the second applies to Source and cannot include
+ // MakePointerAvailable.
+ //
+ // Add such verification here.
+
+ return verifySourceMemoryAccessAttribute(copyMemory);
}
//===----------------------------------------------------------------------===//
return success();
}
+template <>
+LogicalResult
+Deserializer::processOp<spirv::CopyMemoryOp>(ArrayRef<uint32_t> words) {
+ SmallVector<Type, 1> resultTypes;
+ size_t wordIndex = 0;
+ SmallVector<Value, 4> operands;
+ SmallVector<NamedAttribute, 4> attributes;
+
+ if (wordIndex < words.size()) {
+ auto arg = getValue(words[wordIndex]);
+
+ if (!arg) {
+ return emitError(unknownLoc, "unknown result <id> : ")
+ << words[wordIndex];
+ }
+
+ operands.push_back(arg);
+ wordIndex++;
+ }
+
+ if (wordIndex < words.size()) {
+ auto arg = getValue(words[wordIndex]);
+
+ if (!arg) {
+ return emitError(unknownLoc, "unknown result <id> : ")
+ << words[wordIndex];
+ }
+
+ operands.push_back(arg);
+ wordIndex++;
+ }
+
+ bool isAlignedAttr = false;
+
+ if (wordIndex < words.size()) {
+ auto attrValue = words[wordIndex++];
+ attributes.push_back(opBuilder.getNamedAttr(
+ "memory_access", opBuilder.getI32IntegerAttr(attrValue)));
+ isAlignedAttr = (attrValue == 2);
+ }
+
+ if (isAlignedAttr && wordIndex < words.size()) {
+ attributes.push_back(opBuilder.getNamedAttr(
+ "alignment", opBuilder.getI32IntegerAttr(words[wordIndex++])));
+ }
+
+ if (wordIndex < words.size()) {
+ attributes.push_back(opBuilder.getNamedAttr(
+ "source_memory_access",
+ opBuilder.getI32IntegerAttr(words[wordIndex++])));
+ }
+
+ if (wordIndex < words.size()) {
+ attributes.push_back(opBuilder.getNamedAttr(
+ "source_alignment", opBuilder.getI32IntegerAttr(words[wordIndex++])));
+ }
+
+ if (wordIndex != words.size()) {
+ return emitError(unknownLoc,
+ "found more operands than expected when deserializing "
+ "spirv::CopyMemoryOp, only ")
+ << wordIndex << " of " << words.size() << " processed";
+ }
+
+ Location loc = createFileLineColLoc(opBuilder);
+ opBuilder.create<spirv::CopyMemoryOp>(loc, resultTypes, operands, attributes);
+
+ return success();
+}
+
// Pull in auto-generated Deserializer::dispatchToAutogenDeserialization() and
// various Deserializer::processOp<...>() specializations.
#define GET_DESERIALIZATION_FNS
/// Method to serialize an operation in the SPIR-V dialect that is a mirror of
/// an instruction in the SPIR-V spec. This is auto generated if hasOpcode ==
/// 1 and autogenSerialization == 1 in ODS.
- template <typename OpTy> LogicalResult processOp(OpTy op) {
+ template <typename OpTy>
+ LogicalResult processOp(OpTy op) {
return op.emitError("unsupported op serialization");
}
operands);
}
+template <>
+LogicalResult
+Serializer::processOp<spirv::CopyMemoryOp>(spirv::CopyMemoryOp op) {
+ SmallVector<uint32_t, 4> operands;
+ SmallVector<StringRef, 2> elidedAttrs;
+
+ for (Value operand : op.getOperation()->getOperands()) {
+ auto id = getValueID(operand);
+ assert(id && "use before def!");
+ operands.push_back(id);
+ }
+
+ if (auto attr = op.getAttr("memory_access")) {
+ operands.push_back(static_cast<uint32_t>(
+ attr.cast<IntegerAttr>().getValue().getZExtValue()));
+ }
+
+ elidedAttrs.push_back("memory_access");
+
+ if (auto attr = op.getAttr("alignment")) {
+ operands.push_back(static_cast<uint32_t>(
+ attr.cast<IntegerAttr>().getValue().getZExtValue()));
+ }
+
+ elidedAttrs.push_back("alignment");
+
+ if (auto attr = op.getAttr("source_memory_access")) {
+ operands.push_back(static_cast<uint32_t>(
+ attr.cast<IntegerAttr>().getValue().getZExtValue()));
+ }
+
+ elidedAttrs.push_back("source_memory_access");
+
+ if (auto attr = op.getAttr("source_alignment")) {
+ operands.push_back(static_cast<uint32_t>(
+ attr.cast<IntegerAttr>().getValue().getZExtValue()));
+ }
+
+ elidedAttrs.push_back("source_alignment");
+ emitDebugLine(functionBody, op.getLoc());
+ encodeInstructionInto(functionBody, spirv::Opcode::OpCopyMemory, operands);
+
+ return success();
+}
+
// Pull in auto-generated Serializer::dispatchToAutogenSerialization() and
// various Serializer::processOp<...>() specializations.
#define GET_SERIALIZATION_FNS
// -----
-func @copy_memory_incompatible_ptrs() -> () {
+func @copy_memory_incompatible_ptrs() {
%0 = spv.Variable : !spv.ptr<f32, Function>
%1 = spv.Variable : !spv.ptr<i32, Function>
// expected-error @+1 {{both operands must be pointers to the same type}}
// -----
-func @copy_memory_invalid_maa() -> () {
+func @copy_memory_invalid_maa() {
%0 = spv.Variable : !spv.ptr<f32, Function>
%1 = spv.Variable : !spv.ptr<f32, Function>
// expected-error @+1 {{missing alignment value}}
// -----
-func @copy_memory_print_maa() -> () {
+func @copy_memory_invalid_source_maa() {
+ %0 = spv.Variable : !spv.ptr<f32, Function>
+ %1 = spv.Variable : !spv.ptr<f32, Function>
+ // expected-error @+1 {{invalid alignment specification with non-aligned memory access specification}}
+ "spv.CopyMemory"(%0, %1) {source_memory_access=0x0001 : i32, memory_access=0x0002 : i32, source_alignment=8 : i32, alignment=4 : i32} : (!spv.ptr<f32, Function>, !spv.ptr<f32, Function>) -> ()
+ spv.Return
+}
+
+// -----
+
+func @copy_memory_invalid_source_maa2() {
+ %0 = spv.Variable : !spv.ptr<f32, Function>
+ %1 = spv.Variable : !spv.ptr<f32, Function>
+ // expected-error @+1 {{missing alignment value}}
+ "spv.CopyMemory"(%0, %1) {source_memory_access=0x0002 : i32, memory_access=0x0002 : i32, alignment=4 : i32} : (!spv.ptr<f32, Function>, !spv.ptr<f32, Function>) -> ()
+ spv.Return
+}
+
+// -----
+
+func @copy_memory_print_maa() {
%0 = spv.Variable : !spv.ptr<f32, Function>
%1 = spv.Variable : !spv.ptr<f32, Function>
// CHECK: spv.CopyMemory "Function" %{{.*}}, "Function" %{{.*}} ["Aligned", 4] : f32
"spv.CopyMemory"(%0, %1) {memory_access=0x0002 : i32, alignment=4 : i32} : (!spv.ptr<f32, Function>, !spv.ptr<f32, Function>) -> ()
+ // CHECK: spv.CopyMemory "Function" %{{.*}}, "Function" %{{.*}} ["Aligned", 4], ["Volatile"] : f32
+ "spv.CopyMemory"(%0, %1) {source_memory_access=0x0001 : i32, memory_access=0x0002 : i32, alignment=4 : i32} : (!spv.ptr<f32, Function>, !spv.ptr<f32, Function>) -> ()
+
+ // CHECK: spv.CopyMemory "Function" %{{.*}}, "Function" %{{.*}} ["Aligned", 4], ["Aligned", 8] : f32
+ "spv.CopyMemory"(%0, %1) {source_memory_access=0x0002 : i32, memory_access=0x0002 : i32, source_alignment=8 : i32, alignment=4 : i32} : (!spv.ptr<f32, Function>, !spv.ptr<f32, Function>) -> ()
+
spv.Return
}