From: Diego Caballero Date: Wed, 15 Feb 2023 06:01:42 +0000 (+0000) Subject: [mlir][Vector] Enable masking for static shapes X-Git-Tag: upstream/17.0.6~17455 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=1427277eed800335ea211cdb94f10b4976a54231;p=platform%2Fupstream%2Fllvm.git [mlir][Vector] Enable masking for static shapes Support for masking static shapes was already implemented in the past but not enabled so this patch is just removing a pre-condition check and adding some tests with static shapes. Reviewed By: ThomasRaoux Differential Revision: https://reviews.llvm.org/D143937 --- diff --git a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp index fc36477..75d4595 100644 --- a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp +++ b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp @@ -1003,11 +1003,6 @@ mlir::linalg::vectorizeLinalgOpPrecondition(LinalgOp linalgOp, "static sizes"); } - // TODO: Masking is only supported for dynamic shapes so input vector sizes - // must be empty if the op is not dynamic. - if (!linalgOp.hasDynamicShape() && !inputVectorSizes.empty()) - return failure(); - if (linalgOp.hasDynamicShape() && failed(vectorizeDynamicLinalgOpPrecondition(linalgOp))) { LDBG("Dynamically-shaped op failed vectorization pre-conditions\n"); @@ -1092,8 +1087,10 @@ LogicalResult mlir::linalg::vectorize(RewriterBase &rewriter, LinalgOp linalgOp, LLVM_DEBUG(llvm::dbgs() << "\n"); if (failed(vectorizeLinalgOpPrecondition(linalgOp, inputVectorSizes, - vectorizeNDExtract))) + vectorizeNDExtract))) { + LDBG("Vectorization pre-conditions failed\n"); return failure(); + } // Initialize vectorization state. VectorizationState state(rewriter); diff --git a/mlir/test/Dialect/Linalg/vectorization.mlir b/mlir/test/Dialect/Linalg/vectorization.mlir index fb35746..a43fd7a 100644 --- a/mlir/test/Dialect/Linalg/vectorization.mlir +++ b/mlir/test/Dialect/Linalg/vectorization.mlir @@ -2075,3 +2075,111 @@ transform.sequence failures(propagate) { %2 = transform.structured.vectorize %1 } +// ----- + +func.func @vectorize_partial_dynamic_identity(%arg0: tensor<8x?xf32>, + %arg1: tensor<8x?xf32>, + %arg2: tensor<8x?xf32>) -> tensor<8x?xf32> { + %0 = linalg.generic { indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>, + affine_map<(d0, d1) -> (d0, d1)>, + affine_map<(d0, d1) -> (d0, d1)>], + iterator_types = ["parallel", "parallel"] } + ins(%arg0, %arg1 : tensor<8x?xf32>, tensor<8x?xf32>) + outs(%arg2 : tensor<8x?xf32>) { + ^bb(%in0: f32, %in1: f32, %out: f32) : + %0 = arith.addf %in0, %in1 : f32 + linalg.yield %0 : f32 + } -> tensor<8x?xf32> + return %0 : tensor<8x?xf32> +} + +// CHECK-LABEL: func.func @vectorize_partial_dynamic_identity( +// CHECK-SAME: %[[VAL_0:.*]]: tensor<8x?xf32>, %[[VAL_1:.*]]: tensor<8x?xf32>, %[[VAL_2:.*]]: tensor<8x?xf32>) -> tensor<8x?xf32> { +// CHECK: %[[VAL_3:.*]] = arith.constant 1 : index +// CHECK: %[[VAL_4:.*]] = tensor.dim %[[VAL_0]], %[[VAL_3]] : tensor<8x?xf32> +// CHECK: %[[VAL_5:.*]] = arith.constant 0 : index +// CHECK: %[[VAL_6:.*]] = arith.constant 0.000000e+00 : f32 +// CHECK: %[[VAL_7:.*]] = arith.constant 8 : index +// CHECK: %[[VAL_8:.*]] = vector.create_mask %[[VAL_7]], %[[VAL_4]] : vector<8x32xi1> +// CHECK: %[[VAL_9:.*]] = vector.mask %[[VAL_8]] { vector.transfer_read %[[VAL_0]][%[[VAL_5]], %[[VAL_5]]], %[[VAL_6]] {in_bounds = [true, true]} : tensor<8x?xf32>, vector<8x32xf32> } : vector<8x32xi1> -> vector<8x32xf32> +// CHECK: %[[VAL_10:.*]] = arith.constant 0.000000e+00 : f32 +// CHECK: %[[VAL_11:.*]] = vector.mask %[[VAL_8]] { vector.transfer_read %[[VAL_1]][%[[VAL_5]], %[[VAL_5]]], %[[VAL_10]] {in_bounds = [true, true]} : tensor<8x?xf32>, vector<8x32xf32> } : vector<8x32xi1> -> vector<8x32xf32> +// CHECK: %[[VAL_12:.*]] = arith.constant 0.000000e+00 : f32 +// CHECK: %[[VAL_13:.*]] = vector.mask %[[VAL_8]] { vector.transfer_read %[[VAL_2]][%[[VAL_5]], %[[VAL_5]]], %[[VAL_12]] {in_bounds = [true, true]} : tensor<8x?xf32>, vector<8x32xf32> } : vector<8x32xi1> -> vector<8x32xf32> +// CHECK: %[[VAL_14:.*]] = arith.addf %[[VAL_9]], %[[VAL_11]] : vector<8x32xf32> +// CHECK: %[[VAL_15:.*]] = arith.constant 0 : index +// CHECK: %[[VAL_16:.*]] = vector.mask %[[VAL_8]] { vector.transfer_write %[[VAL_14]], %[[VAL_2]][%[[VAL_15]], %[[VAL_15]]] {in_bounds = [true, true]} : vector<8x32xf32>, tensor<8x?xf32> } : vector<8x32xi1> -> tensor<8x?xf32> + + +transform.sequence failures(propagate) { +^bb1(%arg1: !pdl.operation): + %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation + transform.structured.masked_vectorize %0 vector_sizes [8, 32] +} + +// ----- + +func.func @do_not_generate_masks(%arg0: tensor<8x32xf32>, + %arg1: tensor<8x32xf32>, + %arg2: tensor<8x32xf32>) -> tensor<8x32xf32> { + %0 = linalg.generic { indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>, + affine_map<(d0, d1) -> (d0, d1)>, + affine_map<(d0, d1) -> (d0, d1)>], + iterator_types = ["parallel", "parallel"] } + ins(%arg0, %arg1 : tensor<8x32xf32>, tensor<8x32xf32>) + outs(%arg2 : tensor<8x32xf32>) { + ^bb(%in0: f32, %in1: f32, %out: f32) : + %0 = arith.addf %in0, %in1 : f32 + linalg.yield %0 : f32 + } -> tensor<8x32xf32> + return %0 : tensor<8x32xf32> +} + +// CHECK-LABEL: func.func @do_not_generate_masks +// CHECK-NOT: vector.mask + +transform.sequence failures(propagate) { +^bb1(%arg1: !pdl.operation): + %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation + transform.structured.masked_vectorize %0 vector_sizes [8, 32] +} + +// ----- + +func.func @vectorize_static_shape_with_mask(%arg0: tensor<8x30xf32>, + %arg1: tensor<8x30xf32>, + %arg2: tensor<8x30xf32>) -> tensor<8x30xf32> { + %0 = linalg.generic { indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>, + affine_map<(d0, d1) -> (d0, d1)>, + affine_map<(d0, d1) -> (d0, d1)>], + iterator_types = ["parallel", "parallel"] } + ins(%arg0, %arg1 : tensor<8x30xf32>, tensor<8x30xf32>) + outs(%arg2 : tensor<8x30xf32>) { + ^bb(%in0: f32, %in1: f32, %out: f32) : + %0 = arith.addf %in0, %in1 : f32 + linalg.yield %0 : f32 + } -> tensor<8x30xf32> + return %0 : tensor<8x30xf32> +} + +// CHECK-LABEL: func.func @vectorize_static_shape_with_mask( +// CHECK-SAME: %[[VAL_0:.*]]: tensor<8x30xf32>, %[[VAL_1:.*]]: tensor<8x30xf32>, %[[VAL_2:.*]]: tensor<8x30xf32>) -> tensor<8x30xf32> { +// CHECK: %[[VAL_3:.*]] = arith.constant 0 : index +// CHECK: %[[VAL_4:.*]] = arith.constant 0.000000e+00 : f32 +// CHECK: %[[VAL_5:.*]] = arith.constant 8 : index +// CHECK: %[[VAL_6:.*]] = arith.constant 30 : index +// CHECK: %[[VAL_7:.*]] = vector.create_mask %[[VAL_5]], %[[VAL_6]] : vector<8x32xi1> +// CHECK: %[[VAL_8:.*]] = vector.mask %[[VAL_7]] { vector.transfer_read %[[VAL_0]][%[[VAL_3]], %[[VAL_3]]], %[[VAL_4]] {in_bounds = [true, true]} : tensor<8x30xf32>, vector<8x32xf32> } : vector<8x32xi1> -> vector<8x32xf32> +// CHECK: %[[VAL_9:.*]] = arith.constant 0.000000e+00 : f32 +// CHECK: %[[VAL_10:.*]] = vector.mask %[[VAL_7]] { vector.transfer_read %[[VAL_1]][%[[VAL_3]], %[[VAL_3]]], %[[VAL_9]] {in_bounds = [true, true]} : tensor<8x30xf32>, vector<8x32xf32> } : vector<8x32xi1> -> vector<8x32xf32> +// CHECK: %[[VAL_11:.*]] = arith.constant 0.000000e+00 : f32 +// CHECK: %[[VAL_12:.*]] = vector.mask %[[VAL_7]] { vector.transfer_read %[[VAL_2]][%[[VAL_3]], %[[VAL_3]]], %[[VAL_11]] {in_bounds = [true, true]} : tensor<8x30xf32>, vector<8x32xf32> } : vector<8x32xi1> -> vector<8x32xf32> +// CHECK: %[[VAL_13:.*]] = arith.addf %[[VAL_8]], %[[VAL_10]] : vector<8x32xf32> +// CHECK: %[[VAL_14:.*]] = arith.constant 0 : index +// CHECK: %[[VAL_15:.*]] = vector.mask %[[VAL_7]] { vector.transfer_write %[[VAL_13]], %[[VAL_2]][%[[VAL_14]], %[[VAL_14]]] {in_bounds = [true, true]} : vector<8x32xf32>, tensor<8x30xf32> } : vector<8x32xi1> -> tensor<8x30xf32> + +transform.sequence failures(propagate) { +^bb1(%arg1: !pdl.operation): + %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!pdl.operation) -> !pdl.operation + transform.structured.masked_vectorize %0 vector_sizes [8, 32] +}