return res;
}
+template <typename TransferOp>
+LogicalResult getVectorTransferAlignment(LLVMTypeConverter &typeConverter,
+ TransferOp xferOp, unsigned &align) {
+ Type elementTy =
+ typeConverter.convertType(xferOp.getMemRefType().getElementType());
+ if (!elementTy)
+ return failure();
+
+ auto dataLayout = typeConverter.getDialect()->getLLVMModule().getDataLayout();
+ align = dataLayout.getPrefTypeAlignment(
+ elementTy.cast<LLVM::LLVMType>().getUnderlyingType());
+ return success();
+}
+
+static LogicalResult
+replaceTransferOpWithLoadOrStore(ConversionPatternRewriter &rewriter,
+ LLVMTypeConverter &typeConverter, Location loc,
+ TransferReadOp xferOp,
+ ArrayRef<Value> operands, Value dataPtr) {
+ rewriter.replaceOpWithNewOp<LLVM::LoadOp>(xferOp, dataPtr);
+ return success();
+}
+
+static LogicalResult
+replaceTransferOpWithMasked(ConversionPatternRewriter &rewriter,
+ LLVMTypeConverter &typeConverter, Location loc,
+ TransferReadOp xferOp, ArrayRef<Value> operands,
+ Value dataPtr, Value mask) {
+ auto toLLVMTy = [&](Type t) { return typeConverter.convertType(t); };
+ VectorType fillType = xferOp.getVectorType();
+ Value fill = rewriter.create<SplatOp>(loc, fillType, xferOp.padding());
+ fill = rewriter.create<LLVM::DialectCastOp>(loc, toLLVMTy(fillType), fill);
+
+ Type vecTy = typeConverter.convertType(xferOp.getVectorType());
+ if (!vecTy)
+ return failure();
+
+ unsigned align;
+ if (failed(getVectorTransferAlignment(typeConverter, xferOp, align)))
+ return failure();
+
+ rewriter.replaceOpWithNewOp<LLVM::MaskedLoadOp>(
+ xferOp, vecTy, dataPtr, mask, ValueRange{fill},
+ rewriter.getI32IntegerAttr(align));
+ return success();
+}
+
+static LogicalResult
+replaceTransferOpWithLoadOrStore(ConversionPatternRewriter &rewriter,
+ LLVMTypeConverter &typeConverter, Location loc,
+ TransferWriteOp xferOp,
+ ArrayRef<Value> operands, Value dataPtr) {
+ auto adaptor = TransferWriteOpOperandAdaptor(operands);
+ rewriter.replaceOpWithNewOp<LLVM::StoreOp>(xferOp, adaptor.vector(), dataPtr);
+ return success();
+}
+
+static LogicalResult
+replaceTransferOpWithMasked(ConversionPatternRewriter &rewriter,
+ LLVMTypeConverter &typeConverter, Location loc,
+ TransferWriteOp xferOp, ArrayRef<Value> operands,
+ Value dataPtr, Value mask) {
+ unsigned align;
+ if (failed(getVectorTransferAlignment(typeConverter, xferOp, align)))
+ return failure();
+
+ auto adaptor = TransferWriteOpOperandAdaptor(operands);
+ rewriter.replaceOpWithNewOp<LLVM::MaskedStoreOp>(
+ xferOp, adaptor.vector(), dataPtr, mask,
+ rewriter.getI32IntegerAttr(align));
+ return success();
+}
+
+static TransferReadOpOperandAdaptor
+getTransferOpAdapter(TransferReadOp xferOp, ArrayRef<Value> operands) {
+ return TransferReadOpOperandAdaptor(operands);
+}
+
+static TransferWriteOpOperandAdaptor
+getTransferOpAdapter(TransferWriteOp xferOp, ArrayRef<Value> operands) {
+ return TransferWriteOpOperandAdaptor(operands);
+}
+
namespace {
/// Conversion pattern for a vector.matrix_multiply.
}
};
-LogicalResult getLLVMTypeAndAlignment(LLVMTypeConverter &typeConverter,
- Type type, LLVM::LLVMType &llvmType,
- unsigned &align) {
- auto convertedType = typeConverter.convertType(type);
- if (!convertedType)
- return failure();
-
- llvmType = convertedType.template cast<LLVM::LLVMType>();
- auto dataLayout = typeConverter.getDialect()->getLLVMModule().getDataLayout();
- align = dataLayout.getPrefTypeAlignment(llvmType.getUnderlyingType());
- return success();
-}
-
-LogicalResult
-replaceTransferOpWithLoadOrStore(ConversionPatternRewriter &rewriter,
- LLVMTypeConverter &typeConverter, Location loc,
- TransferReadOp xferOp,
- ArrayRef<Value> operands, Value dataPtr) {
- LLVM::LLVMType vecTy;
- unsigned align;
- if (failed(getLLVMTypeAndAlignment(typeConverter, xferOp.getVectorType(),
- vecTy, align)))
- return failure();
- rewriter.replaceOpWithNewOp<LLVM::LoadOp>(xferOp, dataPtr);
- return success();
-}
-
-LogicalResult replaceTransferOpWithMasked(ConversionPatternRewriter &rewriter,
- LLVMTypeConverter &typeConverter,
- Location loc, TransferReadOp xferOp,
- ArrayRef<Value> operands,
- Value dataPtr, Value mask) {
- auto toLLVMTy = [&](Type t) { return typeConverter.convertType(t); };
- VectorType fillType = xferOp.getVectorType();
- Value fill = rewriter.create<SplatOp>(loc, fillType, xferOp.padding());
- fill = rewriter.create<LLVM::DialectCastOp>(loc, toLLVMTy(fillType), fill);
-
- LLVM::LLVMType vecTy;
- unsigned align;
- if (failed(getLLVMTypeAndAlignment(typeConverter, xferOp.getVectorType(),
- vecTy, align)))
- return failure();
-
- rewriter.replaceOpWithNewOp<LLVM::MaskedLoadOp>(
- xferOp, vecTy, dataPtr, mask, ValueRange{fill},
- rewriter.getI32IntegerAttr(align));
- return success();
-}
-
-LogicalResult
-replaceTransferOpWithLoadOrStore(ConversionPatternRewriter &rewriter,
- LLVMTypeConverter &typeConverter, Location loc,
- TransferWriteOp xferOp,
- ArrayRef<Value> operands, Value dataPtr) {
- auto adaptor = TransferWriteOpOperandAdaptor(operands);
- LLVM::LLVMType vecTy;
- unsigned align;
- if (failed(getLLVMTypeAndAlignment(typeConverter, xferOp.getVectorType(),
- vecTy, align)))
- return failure();
- rewriter.replaceOpWithNewOp<LLVM::StoreOp>(xferOp, adaptor.vector(), dataPtr);
- return success();
-}
-
-LogicalResult replaceTransferOpWithMasked(ConversionPatternRewriter &rewriter,
- LLVMTypeConverter &typeConverter,
- Location loc, TransferWriteOp xferOp,
- ArrayRef<Value> operands,
- Value dataPtr, Value mask) {
- auto adaptor = TransferWriteOpOperandAdaptor(operands);
- LLVM::LLVMType vecTy;
- unsigned align;
- if (failed(getLLVMTypeAndAlignment(typeConverter, xferOp.getVectorType(),
- vecTy, align)))
- return failure();
-
- rewriter.replaceOpWithNewOp<LLVM::MaskedStoreOp>(
- xferOp, adaptor.vector(), dataPtr, mask,
- rewriter.getI32IntegerAttr(align));
- return success();
-}
-
-static TransferReadOpOperandAdaptor
-getTransferOpAdapter(TransferReadOp xferOp, ArrayRef<Value> operands) {
- return TransferReadOpOperandAdaptor(operands);
-}
-
-static TransferWriteOpOperandAdaptor
-getTransferOpAdapter(TransferWriteOp xferOp, ArrayRef<Value> operands) {
- return TransferWriteOpOperandAdaptor(operands);
-}
-
-bool isMinorIdentity(AffineMap map, unsigned rank) {
- if (map.getNumResults() < rank)
- return false;
- unsigned startDim = map.getNumDims() - rank;
- for (unsigned i = 0; i < rank; ++i)
- if (map.getResult(i) != getAffineDimExpr(startDim + i, map.getContext()))
- return false;
- return true;
-}
-
/// Conversion pattern that converts a 1-D vector transfer read/write op in a
/// sequence of:
/// 1. Bitcast or addrspacecast to vector form.
if (xferOp.getVectorType().getRank() > 1 ||
llvm::size(xferOp.indices()) == 0)
return failure();
- if (!isMinorIdentity(xferOp.permutation_map(),
- xferOp.getVectorType().getRank()))
+ if (xferOp.permutation_map() !=
+ AffineMap::getMinorIdentityMap(xferOp.permutation_map().getNumInputs(),
+ xferOp.getVectorType().getRank(),
+ op->getContext()))
return failure();
auto toLLVMTy = [&](Type t) { return typeConverter.convertType(t); };
#include <type_traits>
#include "mlir/Conversion/VectorToSCF/VectorToSCF.h"
+
+#include "../PassDetail.h"
#include "mlir/Dialect/Affine/EDSC/Intrinsics.h"
#include "mlir/Dialect/SCF/EDSC/Builders.h"
#include "mlir/Dialect/SCF/EDSC/Intrinsics.h"
#include "mlir/IR/OperationSupport.h"
#include "mlir/IR/PatternMatch.h"
#include "mlir/IR/Types.h"
+#include "mlir/Pass/Pass.h"
+#include "mlir/Transforms/Passes.h"
using namespace mlir;
using namespace mlir::edsc;
}
} // namespace
-
+
/// Analyzes the `transfer` to find an access dimension along the fastest remote
/// MemRef dimension. If such a dimension with coalescing properties is found,
/// `pivs` and `vectorBoundsCapture` are swapped so that the invocation of
}
namespace mlir {
-
+
template <typename TransferOpTy>
VectorTransferRewriter<TransferOpTy>::VectorTransferRewriter(
VectorTransferToSCFOptions options, MLIRContext *context)
} // namespace mlir
+namespace {
+
+struct ConvertVectorToSCFPass
+ : public ConvertVectorToSCFBase<ConvertVectorToSCFPass> {
+ ConvertVectorToSCFPass() = default;
+ ConvertVectorToSCFPass(const ConvertVectorToSCFPass &pass) {}
+ ConvertVectorToSCFPass(const VectorTransferToSCFOptions &options) {
+ this->fullUnroll = options.unroll;
+ }
+
+ void runOnFunction() override {
+ OwningRewritePatternList patterns;
+ auto *context = getFunction().getContext();
+ populateVectorToSCFConversionPatterns(
+ patterns, context, VectorTransferToSCFOptions().setUnroll(fullUnroll));
+ applyPatternsAndFoldGreedily(getFunction(), patterns);
+ }
+};
+
+} // namespace
+
+std::unique_ptr<Pass>
+mlir::createConvertVectorToSCFPass(const VectorTransferToSCFOptions &options) {
+ return std::make_unique<ConvertVectorToSCFPass>(options);
+}
+++ /dev/null
-//===- TestVectorToSCFConversion.cpp - Test VectorTransfers lowering ------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include <type_traits>
-
-#include "mlir/Conversion/VectorToSCF/VectorToSCF.h"
-#include "mlir/IR/PatternMatch.h"
-#include "mlir/Pass/Pass.h"
-#include "mlir/Transforms/Passes.h"
-
-using namespace mlir;
-
-namespace {
-
-struct TestVectorToSCFPass
- : public PassWrapper<TestVectorToSCFPass, FunctionPass> {
- TestVectorToSCFPass() = default;
- TestVectorToSCFPass(const TestVectorToSCFPass &pass) {}
-
- Option<bool> fullUnroll{
- *this, "full-unroll",
- llvm::cl::desc(
- "Perform full unrolling when converting vector transfers to SCF"),
- llvm::cl::init(false)};
-
- void runOnFunction() override {
- OwningRewritePatternList patterns;
- auto *context = &getContext();
- populateVectorToSCFConversionPatterns(
- patterns, context, VectorTransferToSCFOptions().setUnroll(fullUnroll));
- applyPatternsAndFoldGreedily(getFunction(), patterns);
- }
-};
-
-} // end anonymous namespace
-
-namespace mlir {
-void registerTestVectorToSCFPass() {
- PassRegistration<TestVectorToSCFPass> pass(
- "test-convert-vector-to-scf",
- "Converts vector transfer ops to loops over scalars and vector casts");
-}
-} // namespace mlir
OSDN Git Service
