"""Setup pass pipeline parameters for benchmark functions.
"""
opt = (
- "parallelization-strategy=none"
+ "parallelization-strategy=0"
" vectorization-strategy=0 vl=1 enable-simd-index32=False"
)
pipeline = f"sparse-compiler{{{opt}}}"
struct SparseCompilerOptions
: public PassPipelineOptions<SparseCompilerOptions> {
// These options must be kept in sync with `SparsificationBase`.
-
- PassOptions::Option<enum SparseParallelizationStrategy> parallelization{
+ PassOptions::Option<int32_t> parallelization{
*this, "parallelization-strategy",
- desc("Set the parallelization strategy"),
- init(SparseParallelizationStrategy::kNone)};
- PassOptions::Option<enum SparseVectorizationStrategy> vectorization{
+ desc("Set the parallelization strategy"), init(0)};
+ PassOptions::Option<int32_t> vectorization{
*this, "vectorization-strategy", desc("Set the vectorization strategy"),
- init(SparseVectorizationStrategy::kNone)};
+ init(0)};
PassOptions::Option<int32_t> vectorLength{
*this, "vl", desc("Set the vector length"), init(1)};
PassOptions::Option<bool> enableSIMDIndex32{
/// Projects out the options for `createSparsificationPass`.
SparsificationOptions sparsificationOptions() const {
- return SparsificationOptions(parallelization, vectorization, vectorLength,
- enableSIMDIndex32, enableVLAVectorization);
+ return SparsificationOptions(sparseParallelizationStrategy(parallelization),
+ sparseVectorizationStrategy(vectorization),
+ vectorLength, enableSIMDIndex32,
+ enableVLAVectorization);
}
// These options must be kept in sync with `SparseTensorConversionBase`.
// TODO: support reduction parallelization too?
};
+/// Converts command-line parallelization flag to the strategy enum.
+SparseParallelizationStrategy sparseParallelizationStrategy(int32_t flag);
+
/// Defines a vectorization strategy. Any inner loop is a candidate (full SIMD
/// for parallel loops and horizontal SIMD for reduction loops). A loop is
/// actually vectorized if (1) allowed by the strategy, and (2) the emitted
kAnyStorageInnerLoop
};
+/// Converts command-line vectorization flag to the strategy enum.
+SparseVectorizationStrategy sparseVectorizationStrategy(int32_t flag);
+
/// Options for the Sparsification pass.
struct SparsificationOptions {
SparsificationOptions(SparseParallelizationStrategy p,
"vector::VectorDialect",
];
let options = [
- Option<"parallelization", "parallelization-strategy", "enum SparseParallelizationStrategy",
- "mlir::SparseParallelizationStrategy::kNone",
- "Set the parallelization strategy", [{llvm::cl::values(
- clEnumValN(mlir::SparseParallelizationStrategy::kNone, "none",
- "Turn off sparse parallelization."),
- clEnumValN(mlir::SparseParallelizationStrategy::kDenseOuterLoop,
- "dense-outer-loop",
- "Enable dense outer loop sparse parallelization."),
- clEnumValN(mlir::SparseParallelizationStrategy::kAnyStorageOuterLoop,
- "any-storage-outer-loop",
- "Enable sparse parallelization regardless of storage for the outer loop."),
- clEnumValN(mlir::SparseParallelizationStrategy::kDenseAnyLoop,
- "dense-any-loop",
- "Enable dense parallelization for any loop."),
- clEnumValN(mlir::SparseParallelizationStrategy::kAnyStorageAnyLoop,
- "any-storage-any-loop",
- "Enable sparse parallelization for any storage and loop."))}]>,
- Option<"vectorization", "vectorization-strategy", "enum SparseVectorizationStrategy",
- "mlir::SparseVectorizationStrategy::kNone",
- "Set the vectorization strategy", [{llvm::cl::values(
- clEnumValN(mlir::SparseVectorizationStrategy::kNone, "none",
- "Turn off sparse vectorization."),
- clEnumValN(mlir::SparseVectorizationStrategy::kDenseInnerLoop,
- "dense-inner-loop",
- "Enable vectorization for dense inner loops."),
- clEnumValN(mlir::SparseVectorizationStrategy::kAnyStorageInnerLoop,
- "any-storage-inner-loop",
- "Enable sparse vectorization for inner loops with any storage."))}]>,
+ Option<"parallelization", "parallelization-strategy", "int32_t", "0",
+ "Set the parallelization strategy">,
+ Option<"vectorization", "vectorization-strategy", "int32_t", "0",
+ "Set the vectorization strategy">,
Option<"vectorLength", "vl", "int32_t", "1",
"Set the vector length">,
Option<"enableSIMDIndex32", "enable-simd-index32", "bool", "false",
SparsificationPass() = default;
SparsificationPass(const SparsificationPass &pass) = default;
SparsificationPass(const SparsificationOptions &options) {
- parallelization = options.parallelizationStrategy;
- vectorization = options.vectorizationStrategy;
+ parallelization = static_cast<int32_t>(options.parallelizationStrategy);
+ vectorization = static_cast<int32_t>(options.vectorizationStrategy);
vectorLength = options.vectorLength;
enableSIMDIndex32 = options.enableSIMDIndex32;
enableVLAVectorization = options.enableVLAVectorization;
auto *ctx = &getContext();
RewritePatternSet patterns(ctx);
// Translate strategy flags to strategy options.
- SparsificationOptions options(parallelization, vectorization, vectorLength,
- enableSIMDIndex32, enableVLAVectorization);
+ SparsificationOptions options(
+ sparseParallelizationStrategy(parallelization),
+ sparseVectorizationStrategy(vectorization), vectorLength,
+ enableSIMDIndex32, enableVLAVectorization);
// Apply rewriting.
populateSparsificationPatterns(patterns, options);
vector::populateVectorToVectorCanonicalizationPatterns(patterns);
} // namespace
+SparseParallelizationStrategy
+mlir::sparseParallelizationStrategy(int32_t flag) {
+ switch (flag) {
+ default:
+ return SparseParallelizationStrategy::kNone;
+ case 1:
+ return SparseParallelizationStrategy::kDenseOuterLoop;
+ case 2:
+ return SparseParallelizationStrategy::kAnyStorageOuterLoop;
+ case 3:
+ return SparseParallelizationStrategy::kDenseAnyLoop;
+ case 4:
+ return SparseParallelizationStrategy::kAnyStorageAnyLoop;
+ }
+}
+
+SparseVectorizationStrategy mlir::sparseVectorizationStrategy(int32_t flag) {
+ switch (flag) {
+ default:
+ return SparseVectorizationStrategy::kNone;
+ case 1:
+ return SparseVectorizationStrategy::kDenseInnerLoop;
+ case 2:
+ return SparseVectorizationStrategy::kAnyStorageInnerLoop;
+ }
+}
+
SparseToSparseConversionStrategy
mlir::sparseToSparseConversionStrategy(int32_t flag) {
switch (flag) {
-// RUN: mlir-opt %s -sparsification="parallelization-strategy=none" | \
+// RUN: mlir-opt %s -sparsification="parallelization-strategy=0" | \
// RUN: FileCheck %s --check-prefix=CHECK-PAR0
-// RUN: mlir-opt %s -sparsification="parallelization-strategy=dense-outer-loop" | \
+// RUN: mlir-opt %s -sparsification="parallelization-strategy=1" | \
// RUN: FileCheck %s --check-prefix=CHECK-PAR1
-// RUN: mlir-opt %s -sparsification="parallelization-strategy=any-storage-outer-loop" | \
+// RUN: mlir-opt %s -sparsification="parallelization-strategy=2" | \
// RUN: FileCheck %s --check-prefix=CHECK-PAR2
-// RUN: mlir-opt %s -sparsification="parallelization-strategy=dense-any-loop" | \
+// RUN: mlir-opt %s -sparsification="parallelization-strategy=3" | \
// RUN: FileCheck %s --check-prefix=CHECK-PAR3
-// RUN: mlir-opt %s -sparsification="parallelization-strategy=any-storage-any-loop" | \
+// RUN: mlir-opt %s -sparsification="parallelization-strategy=4" | \
// RUN: FileCheck %s --check-prefix=CHECK-PAR4
#DenseMatrix = #sparse_tensor.encoding<{
-// RUN: mlir-opt %s -sparsification="vectorization-strategy=none vl=16" -cse -split-input-file | \
+// RUN: mlir-opt %s -sparsification="vectorization-strategy=0 vl=16" -cse -split-input-file | \
// RUN: FileCheck %s --check-prefix=CHECK-VEC0
-// RUN: mlir-opt %s -sparsification="vectorization-strategy=dense-inner-loop vl=16" -cse -split-input-file | \
+// RUN: mlir-opt %s -sparsification="vectorization-strategy=1 vl=16" -cse -split-input-file | \
// RUN: FileCheck %s --check-prefix=CHECK-VEC1
-// RUN: mlir-opt %s -sparsification="vectorization-strategy=any-storage-inner-loop vl=16" -cse -split-input-file | \
+// RUN: mlir-opt %s -sparsification="vectorization-strategy=2 vl=16" -cse -split-input-file | \
// RUN: FileCheck %s --check-prefix=CHECK-VEC2
-// RUN: mlir-opt %s -sparsification="vectorization-strategy=any-storage-inner-loop vl=16 enable-simd-index32=true" -cse -split-input-file | \
+// RUN: mlir-opt %s -sparsification="vectorization-strategy=2 vl=16 enable-simd-index32=true" -cse -split-input-file | \
// RUN: FileCheck %s --check-prefix=CHECK-VEC3
-// RUN: mlir-opt %s -sparsification="vectorization-strategy=any-storage-inner-loop vl=4 enable-vla-vectorization=true" -cse -split-input-file | \
+// RUN: mlir-opt %s -sparsification="vectorization-strategy=2 vl=4 enable-vla-vectorization=true" -cse -split-input-file | \
// RUN: FileCheck %s --check-prefix=CHECK-VEC4
#DenseVector = #sparse_tensor.encoding<{ dimLevelType = [ "dense" ] }>
// NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
-// RUN: mlir-opt %s -sparsification="vectorization-strategy=any-storage-inner-loop vl=8" -canonicalize | \
+// RUN: mlir-opt %s -sparsification="vectorization-strategy=2 vl=8" -canonicalize | \
// RUN: FileCheck %s
#SparseMatrix = #sparse_tensor.encoding<{dimLevelType = ["dense","compressed"]}>
// about what constitutes a good test! The CHECK should be
// minimized and named to reflect the test intent.
-// RUN: mlir-opt %s -sparsification="vectorization-strategy=any-storage-inner-loop vl=8" -canonicalize | \
+// RUN: mlir-opt %s -sparsification="vectorization-strategy=2 vl=8" -canonicalize | \
// RUN: FileCheck %s
#SparseVector = #sparse_tensor.encoding<{
-// RUN: mlir-opt %s -sparsification="vectorization-strategy=any-storage-inner-loop vl=16" -scf-for-loop-peeling -canonicalize | \
+// RUN: mlir-opt %s -sparsification="vectorization-strategy=2 vl=16" -scf-for-loop-peeling -canonicalize | \
// RUN: FileCheck %s
#SparseVector = #sparse_tensor.encoding<{
//
// Do the same run, but now with SIMDization as well. This should not change the outcome.
//
-// RUN: mlir-opt %s --sparse-compiler="vectorization-strategy=any-storage-inner-loop vl=2" | \
+// RUN: mlir-opt %s --sparse-compiler="vectorization-strategy=2 vl=2" | \
// RUN: mlir-cpu-runner \
// RUN: -e entry -entry-point-result=void \
// RUN: -shared-libs=%mlir_integration_test_dir/libmlir_c_runner_utils%shlibext | \
//
// Do the same run, but now with SIMDization as well. This should not change the outcome.
//
-// RUN: mlir-opt %s --sparse-compiler="vectorization-strategy=any-storage-inner-loop vl=2" | \
+// RUN: mlir-opt %s --sparse-compiler="vectorization-strategy=2 vl=2" | \
// RUN: mlir-cpu-runner -e entry -entry-point-result=void \
// RUN: -shared-libs=%mlir_integration_test_dir/libmlir_c_runner_utils%shlibext | \
// RUN: FileCheck %s
//
// Do the same run, but now with SIMDization as well. This should not change the outcome.
//
-// RUN: mlir-opt %s --sparse-compiler="vectorization-strategy=any-storage-inner-loop vl=4" | \
+// RUN: mlir-opt %s --sparse-compiler="vectorization-strategy=2 vl=4" | \
// RUN: TENSOR0="%mlir_integration_test_dir/data/test.tns" \
// RUN: mlir-cpu-runner \
// RUN: -e entry -entry-point-result=void \
//
// Do the same run, but now with SIMDization as well. This should not change the outcome.
//
-// RUN: mlir-opt %s --sparse-compiler="vectorization-strategy=any-storage-inner-loop vl=4" | \
+// RUN: mlir-opt %s --sparse-compiler="vectorization-strategy=2 vl=4" | \
// RUN: mlir-cpu-runner -e entry -entry-point-result=void \
// RUN: -shared-libs=%mlir_integration_test_dir/libmlir_c_runner_utils%shlibext | \
// RUN: FileCheck %s
// Do the same run, but now with SIMDization as well. This should not change the outcome.
//
// RUN: mlir-opt %s \
-// RUN: --sparse-compiler="vectorization-strategy=any-storage-inner-loop vl=16 enable-simd-index32" | \
+// RUN: --sparse-compiler="vectorization-strategy=2 vl=16 enable-simd-index32" | \
// RUN: TENSOR0="%mlir_integration_test_dir/data/wide.mtx" \
// RUN: mlir-cpu-runner \
// RUN: -e entry -entry-point-result=void \
//
// Do the same run, but now with SIMDization as well. This should not change the outcome.
//
-// RUN: mlir-opt %s --sparse-compiler="vectorization-strategy=any-storage-inner-loop vl=4" | \
+// RUN: mlir-opt %s --sparse-compiler="vectorization-strategy=2 vl=4" | \
// RUN: TENSOR0="%mlir_integration_test_dir/data/mttkrp_b.tns" \
// RUN: mlir-cpu-runner \
// RUN: -e entry -entry-point-result=void \
//
// Do the same run, but now with SIMDization as well. This should not change the outcome.
//
-// RUN: mlir-opt %s --sparse-compiler="vectorization-strategy=any-storage-inner-loop vl=4" | \
+// RUN: mlir-opt %s --sparse-compiler="vectorization-strategy=2 vl=4" | \
// RUN: TENSOR0="%mlir_integration_test_dir/data/test.mtx" \
// RUN: mlir-cpu-runner \
// RUN: -e entry -entry-point-result=void \
//
// Do the same run, but now with SIMDization as well. This should not change the outcome.
//
-// RUN: mlir-opt %s --sparse-compiler="vectorization-strategy=any-storage-inner-loop vl=2" | \
+// RUN: mlir-opt %s --sparse-compiler="vectorization-strategy=2 vl=2" | \
// RUN: mlir-cpu-runner -e entry -entry-point-result=void \
// RUN: -shared-libs=%mlir_integration_test_dir/libmlir_c_runner_utils%shlibext | \
// RUN: FileCheck %s
//
// Do the same run, but now with SIMDization as well. This should not change the outcome.
//
-// RUN: mlir-opt %s --sparse-compiler="vectorization-strategy=any-storage-inner-loop vl=8" | \
+// RUN: mlir-opt %s --sparse-compiler="vectorization-strategy=2 vl=8" | \
// RUN: mlir-cpu-runner -e entry -entry-point-result=void \
// RUN: -shared-libs=%mlir_integration_test_dir/libmlir_c_runner_utils%shlibext | \
// RUN: FileCheck %s
// Do the same run, but now with SIMDization as well. This should not change the outcome.
//
// RUN: mlir-opt %s \
-// RUN: --sparse-compiler="vectorization-strategy=any-storage-inner-loop vl=4 enable-simd-index32" | \
+// RUN: --sparse-compiler="vectorization-strategy=2 vl=4 enable-simd-index32" | \
// RUN: TENSOR0="%mlir_integration_test_dir/data/test.mtx" \
// RUN: mlir-cpu-runner \
// RUN: -e entry -entry-point-result=void \
//
// Do the same run, but now with SIMDization as well. This should not change the outcome.
//
-// RUN: mlir-opt %s --sparse-compiler="vectorization-strategy=any-storage-inner-loop vl=8" | \
+// RUN: mlir-opt %s --sparse-compiler="vectorization-strategy=2 vl=8" | \
// RUN: mlir-cpu-runner -e entry -entry-point-result=void \
// RUN: -shared-libs=%mlir_integration_test_dir/libmlir_c_runner_utils%shlibext | \
// RUN: FileCheck %s
//
// Do the same run, but now with SIMDization as well. This should not change the outcome.
//
-// RUN: mlir-opt %s --sparse-compiler="vectorization-strategy=any-storage-inner-loop vl=4" | \
+// RUN: mlir-opt %s --sparse-compiler="vectorization-strategy=2 vl=4" | \
// RUN: mlir-cpu-runner \
// RUN: -e entry -entry-point-result=void \
// RUN: -shared-libs=%mlir_integration_test_dir/libmlir_c_runner_utils%shlibext | \
//
// Do the same run, but now with SIMDization as well. This should not change the outcome.
//
-// RUN: mlir-opt %s --sparse-compiler="vectorization-strategy=any-storage-inner-loop vl=2" | \
+// RUN: mlir-opt %s --sparse-compiler="vectorization-strategy=2 vl=2" | \
// RUN: TENSOR0="%mlir_integration_test_dir/data/wide.mtx" \
// RUN: mlir-cpu-runner \
// RUN: -e entry -entry-point-result=void \
//
// Do the same run, but now with SIMDization as well. This should not change the outcome.
//
-// RUN: mlir-opt %s --sparse-compiler="vectorization-strategy=any-storage-inner-loop vl=2" | \
+// RUN: mlir-opt %s --sparse-compiler="vectorization-strategy=2 vl=2" | \
// RUN: TENSOR0="%mlir_integration_test_dir/data/test_symmetric.mtx" \
// RUN: mlir-cpu-runner \
// RUN: -e entry -entry-point-result=void \
ir.AffineMap.get_permutation([0, 1]),
ir.AffineMap.get_permutation([1, 0])
]
- vec_strategy = [
- 'none', 'dense-inner-loop'
- ]
for level in levels:
for ordering in orderings:
for pwidth in [32]:
for iwidth in [32]:
- for vec in vec_strategy:
- for e in [True]:
- vl = 1 if vec == 0 else 16
- attr = st.EncodingAttr.get(level, ordering, pwidth, iwidth)
- opt = (f'parallelization-strategy=none '
- f'vectorization-strategy={vec} '
- f'vl={vl} enable-simd-index32={e}')
- compiler = sparse_compiler.SparseCompiler(
- options=opt, opt_level=0, shared_libs=[support_lib])
- build_compile_and_run_SDDMMM(attr, compiler)
- count = count + 1
+ for par in [0]:
+ for vec in [0, 1]:
+ for e in [True]:
+ vl = 1 if vec == 0 else 16
+ attr = st.EncodingAttr.get(level, ordering, pwidth, iwidth)
+ opt = (f'parallelization-strategy={par} '
+ f'vectorization-strategy={vec} '
+ f'vl={vl} enable-simd-index32={e}')
+ compiler = sparse_compiler.SparseCompiler(
+ options=opt, opt_level=0, shared_libs=[support_lib])
+ build_compile_and_run_SDDMMM(attr, compiler)
+ count = count + 1
# CHECK: Passed 16 tests
print('Passed ', count, 'tests')
# a *single* sparse tensor. Note that we deliberate do not exhaustively
# search the full state space to reduce runtime of the test. It is
# straightforward to adapt the code below to explore more combinations.
+ par = 0
+ vec = 0
vl = 1
e = False
- opt = (f'parallelization-strategy=none '
- f'vectorization-strategy=none '
+ opt = (f'parallelization-strategy={par} '
+ f'vectorization-strategy={vec} '
f'vl={vl} enable-simd-index32={e}')
levels = [[st.DimLevelType.dense, st.DimLevelType.dense],
[st.DimLevelType.dense, st.DimLevelType.compressed],
# CHECK-LABEL: TEST: test_stress
print("\nTEST: test_stress")
with ir.Context() as ctx, ir.Location.unknown():
+ par = 0
+ vec = 0
vl = 1
e = False
sparsification_options = (
- f'parallelization-strategy=none '
- f'vectorization-strategy=none '
+ f'parallelization-strategy={par} '
+ f'vectorization-strategy={vec} '
f'vl={vl} '
f'enable-simd-index32={e}')
compiler = sparse_compiler.SparseCompiler(
projects / platform / upstream / llvm.git / commitdiff