Add support for an empty anchor op string in vectorization. An empty anchor op string is useful after fusion when there are multiple different operations to vectorize.
Depends On D114689
Reviewed By: nicolasvasilache
Differential Revision: https://reviews.llvm.org/D114690
vectorize(StringRef opName,
LinalgTransformationFilter::FilterFunction f = nullptr,
bool vectorizePadding = false) {
- assert(!opName.empty() && "expected an op name");
transformationSequence.emplace_back(std::make_unique<Vectorize>(
opName, linalg::LinalgVectorizationOptions(), f, vectorizePadding));
return *this;
// RUN: mlir-opt %s -test-linalg-codegen-strategy="anchor-func=matmul anchor-op=linalg.matmul tile-sizes=16,32,64 promote promote-full-tile-pad register-tile-sizes=2,4,8 vectorize vectorize-contraction-to=outerproduct split-transfers=true unroll-vector-transfers=false" -split-input-file | FileCheck %s --check-prefix=CHECK-OUTER
// RUN: mlir-opt %s -test-linalg-codegen-strategy="anchor-func=matmul anchor-op=linalg.matmul tile-sizes=16,32,64 tile-interchange=1,2,0 generalize iterator-interchange=0,2,1" -split-input-file | FileCheck %s --check-prefix=CHECK-INTERCHANGE
// RUN: mlir-opt %s -test-linalg-codegen-strategy="anchor-func=matmul anchor-op=linalg.matmul tile-sizes=16,32,64 pad pack-paddings=1,1,0 hoist-paddings=3,3,0" -split-input-file | FileCheck %s --check-prefix=CHECK-PAD
+// RUN: mlir-opt %s -test-linalg-codegen-strategy="anchor-func=matmul anchor-op=linalg.matmul tile-sizes=16,32,64 fuse pad vectorize" -split-input-file | FileCheck %s --check-prefix=CHECK-FUSE
// CHECK-INTRINSIC: func @matmul(
// CHECK-OUTER: func @matmul(
%0 = linalg.matmul ins(%arg0, %arg1: tensor<72x72xf32>, tensor<72x72xf32>) outs(%arg2: tensor<72x72xf32>) -> tensor<72x72xf32>
return %0 : tensor<72x72xf32>
}
+
+// -----
+
+// CHECK-FUSE: func @matmul(
+func @matmul(%arg0: tensor<72x72xf32>, %arg1: tensor<72x72xf32>, %arg2: tensor<72x72xf32>) -> tensor<72x72xf32> {
+
+ // Check the padding and vectorization applies to the fill operation due to the empty anchor op string.
+ // CHECK-FUSE: %[[CST:.*]] = arith.constant dense<0.000000e+00>
+ // CHECK-FUSE: vector.transfer_write %[[CST]]
+ %cst = arith.constant 0.0 : f32
+ %0 = linalg.fill(%cst, %arg0) : f32, tensor<72x72xf32> -> tensor<72x72xf32>
+
+ // Check the matmul is padded and vectorized despite the empty anchor op string.
+ // CHECK-FUSE: vector.outerproduct
+ %1 = linalg.matmul ins(%arg0, %arg1: tensor<72x72xf32>, tensor<72x72xf32>) outs(%0: tensor<72x72xf32>) -> tensor<72x72xf32>
+ return %1 : tensor<72x72xf32>
+}
vector::VectorTransferSplit vectorTransferSplit) {
assert(!anchorOpName.empty());
CodegenStrategy strategy;
- StringRef genericOpName = GenericOp::getOperationName();
strategy
.tileAndFuseIf(fuse && !tileSizes.empty(), anchorOpName,
tilingAndFusionOptions)
.tileIf(!fuse && !tileSizes.empty(), anchorOpName, tilingOptions)
- .promoteIf(promote, anchorOpName,
+ .promoteIf(!fuse && promote, anchorOpName,
LinalgPromotionOptions()
.setAlignment(16)
.setUseFullTileBuffersByDefault(promoteFullTile))
- .tileIf(!registerTileSizes.empty(), anchorOpName, registerTilingOptions)
- .promoteIf(registerPromote, anchorOpName,
+ .tileIf(!fuse && !registerTileSizes.empty(), anchorOpName,
+ registerTilingOptions)
+ .promoteIf(!fuse && registerPromote, anchorOpName,
LinalgPromotionOptions()
.setAlignment(16)
.setUseFullTileBuffersByDefault(registerPromoteFullTile))
- .padIf(pad, anchorOpName, paddingOptions)
- .generalizeIf(generalize, anchorOpName)
+ .padIf(pad, "", paddingOptions)
+ .generalizeIf(generalize, "")
.interchangeIf(!iteratorInterchange.empty(), iteratorInterchange)
- .vectorizeIf(vectorize, generalize ? genericOpName : anchorOpName)
+ .vectorizeIf(vectorize, "")
.vectorLowering(
LinalgVectorLoweringOptions()
.setVectorTransformsOptions(