[X86][Costmodel] Load/store i64/f64 Stride=2 VF=4 interleaving costs

The only sched models that for cpu's that support avx2
but not avx512 are: haswell, broadwell, skylake, zen1-3

For load we have:
https://godbolt.org/z/j5co1qWEW - for intels `Block RThroughput: =4.0`; for ryzens, `Block RThroughput: <=2.0`
So pick cost of `4`.

For store we have:
https://godbolt.org/z/j5co1qWEW - for intels `Block RThroughput: =4.0`; for ryzens, `Block RThroughput: <=4.0`
So pick cost of `4`.

I'm directly using the shuffling asm the llc produced,
without any manual fixups that may be needed
to ensure sequential execution.

Reviewed By: RKSimon

Differential Revision: https://reviews.llvm.org/D110837

diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index f80b001..3a1b8a3 100644 (file)
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -5082,7 +5082,7 @@ InstructionCost X86TTIImpl::getInterleavedMemoryOpCostAVX2(
       {2, MVT::v32i32, 16}, // (load 64i32 and) deinterleave into 2 x 32i32
 
       {2, MVT::v2i64, 2}, // (load 4i64 and) deinterleave into 2 x 2i64
-      {2, MVT::v4i64, 6}, // (load 8i64 and) deinterleave into 2 x 4i64
+      {2, MVT::v4i64, 4}, // (load 8i64 and) deinterleave into 2 x 4i64
 
       {3, MVT::v2i8, 10},  // (load 6i8 and) deinterleave into 3 x 2i8
       {3, MVT::v4i8, 4},   // (load 12i8 and) deinterleave into 3 x 4i8
@@ -5132,7 +5132,7 @@ InstructionCost X86TTIImpl::getInterleavedMemoryOpCostAVX2(
       {2, MVT::v32i32, 16}, // interleave 2 x 32i32 into 64i32 (and store)
 
       {2, MVT::v2i64, 2}, // interleave 2 x 2i64 into 4i64 (and store)
-      {2, MVT::v4i64, 6}, // interleave 2 x 4i64 into 8i64 (and store)
+      {2, MVT::v4i64, 4}, // interleave 2 x 4i64 into 8i64 (and store)
 
       {3, MVT::v2i8, 7},   // interleave 3 x 2i8 into 6i8 (and store)
       {3, MVT::v4i8, 8},   // interleave 3 x 4i8 into 12i8 (and store)

diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-f64-stride-2.ll b/llvm/test/Analysis/CostModel/X86/interleaved-load-f64-stride-2.ll
index c69f616..a38e0c9 100644 (file)
--- a/llvm/test/Analysis/CostModel/X86/interleaved-load-f64-stride-2.ll
+++ b/llvm/test/Analysis/CostModel/X86/interleaved-load-f64-stride-2.ll
@@ -25,7 +25,7 @@ target triple = "x86_64-unknown-linux-gnu"
 ;;
 ; AVX2: LV: Found an estimated cost of 1 for VF 1 For instruction:   %v0 = load double, double* %in0, align 8
 ; AVX2: LV: Found an estimated cost of 3 for VF 2 For instruction:   %v0 = load double, double* %in0, align 8
-; AVX2: LV: Found an estimated cost of 8 for VF 4 For instruction:   %v0 = load double, double* %in0, align 8
+; AVX2: LV: Found an estimated cost of 6 for VF 4 For instruction:   %v0 = load double, double* %in0, align 8
 ; AVX2: LV: Found an estimated cost of 32 for VF 8 For instruction:   %v0 = load double, double* %in0, align 8
 ; AVX2: LV: Found an estimated cost of 64 for VF 16 For instruction:   %v0 = load double, double* %in0, align 8
 ;

diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-i64-stride-2.ll b/llvm/test/Analysis/CostModel/X86/interleaved-load-i64-stride-2.ll
index be333a6..98226ae 100644 (file)
--- a/llvm/test/Analysis/CostModel/X86/interleaved-load-i64-stride-2.ll
+++ b/llvm/test/Analysis/CostModel/X86/interleaved-load-i64-stride-2.ll
@@ -25,7 +25,7 @@ target triple = "x86_64-unknown-linux-gnu"
 ;
 ; AVX2: LV: Found an estimated cost of 1 for VF 1 For instruction:   %v0 = load i64, i64* %in0, align 8
 ; AVX2: LV: Found an estimated cost of 3 for VF 2 For instruction:   %v0 = load i64, i64* %in0, align 8
-; AVX2: LV: Found an estimated cost of 8 for VF 4 For instruction:   %v0 = load i64, i64* %in0, align 8
+; AVX2: LV: Found an estimated cost of 6 for VF 4 For instruction:   %v0 = load i64, i64* %in0, align 8
 ; AVX2: LV: Found an estimated cost of 52 for VF 8 For instruction:   %v0 = load i64, i64* %in0, align 8
 ; AVX2: LV: Found an estimated cost of 104 for VF 16 For instruction:   %v0 = load i64, i64* %in0, align 8
 ;

diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-store-double.ll b/llvm/test/Analysis/CostModel/X86/interleaved-load-store-double.ll
deleted file mode 100644 (file)
index 8201e34..0000000
--- a/llvm/test/Analysis/CostModel/X86/interleaved-load-store-double.ll
+++ /dev/null
@@ -1,40 +0,0 @@
-; REQUIRES: asserts
-; RUN: opt -S -loop-vectorize -debug-only=loop-vectorize -mcpu=skylake %s 2>&1 | FileCheck %s
-target datalayout = "e-m:e-p:32:32-f64:32:64-f80:32-n8:16:32-S128"
-target triple = "i386-unknown-linux-gnu"
-
-@doublesrc = common local_unnamed_addr global [120 x double] zeroinitializer, align 4
-@doubledst = common local_unnamed_addr global [120 x double] zeroinitializer, align 4
-
-; Function Attrs: norecurse nounwind
-define void @stride2double(double %k, i32 %width_) {
-entry:
-
-; CHECK: Found an estimated cost of 8 for VF 4 For instruction:   %0 = load double
-; CHECK: Found an estimated cost of 8 for VF 4 For instruction:   store double
-
-  %cmp27 = icmp sgt i32 %width_, 0
-  br i1 %cmp27, label %for.body.lr.ph, label %for.cond.cleanup
-
-for.body.lr.ph:                                   ; preds = %entry
-  br label %for.body
-
-for.cond.cleanup:                                 ; preds = %for.body, %entry
-  ret void
-
-for.body:                                         ; preds = %for.body.lr.ph, %for.body
-  %i.028 = phi i32 [ 0, %for.body.lr.ph ], [ %add16, %for.body ]
-  %arrayidx = getelementptr inbounds [120 x double], [120 x double]* @doublesrc, i32 0, i32 %i.028
-  %0 = load double, double* %arrayidx, align 4
-  %arrayidx2 = getelementptr inbounds [120 x double], [120 x double]* @doubledst, i32 0, i32 %i.028
-  store double %0, double* %arrayidx2, align 4
-  %add4 = add nuw nsw i32 %i.028, 1
-  %arrayidx5 = getelementptr inbounds [120 x double], [120 x double]* @doublesrc, i32 0, i32 %add4
-  %1 = load double, double* %arrayidx5, align 4
-  %arrayidx8 = getelementptr inbounds [120 x double], [120 x double]* @doubledst, i32 0, i32 %add4
-  store double %1, double* %arrayidx8, align 4
-  %add16 = add nuw nsw i32 %i.028, 2
-  %cmp = icmp slt i32 %add16, %width_
-  br i1 %cmp, label %for.body, label %for.cond.cleanup
-}
-

diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-load-store-i64.ll b/llvm/test/Analysis/CostModel/X86/interleaved-load-store-i64.ll
deleted file mode 100644 (file)
index d56bb26..0000000
--- a/llvm/test/Analysis/CostModel/X86/interleaved-load-store-i64.ll
+++ /dev/null
@@ -1,40 +0,0 @@
-; REQUIRES: asserts
-; RUN: opt -S -loop-vectorize -debug-only=loop-vectorize -mcpu=core-avx2 %s 2>&1 | FileCheck %s
-target datalayout = "e-m:e-p:32:32-f64:32:64-f80:32-n8:16:32-S128"
-target triple = "i386-unknown-linux-gnu"
-
-@i64src = common local_unnamed_addr global [120 x i64] zeroinitializer, align 4
-@i64dst = common local_unnamed_addr global [120 x i64] zeroinitializer, align 4
-
-; Function Attrs: norecurse nounwind
-define void @stride2i64(i64 %k, i32 %width_) {
-entry:
-
-; CHECK: Found an estimated cost of 8 for VF 4 For instruction:   %0 = load i64
-; CHECK: Found an estimated cost of 8 for VF 4 For instruction:   store i64
-
-  %cmp27 = icmp sgt i32 %width_, 0
-  br i1 %cmp27, label %for.body.lr.ph, label %for.cond.cleanup
-
-for.body.lr.ph:                                   ; preds = %entry
-  br label %for.body
-
-for.cond.cleanup:                                 ; preds = %for.body, %entry
-  ret void
-
-for.body:
-  %i.028 = phi i32 [ 0, %for.body.lr.ph ], [ %add16, %for.body ]
-  %arrayidx = getelementptr inbounds [120 x i64], [120 x i64]* @i64src, i32 0, i32 %i.028
-  %0 = load i64, i64* %arrayidx, align 4
-  %arrayidx2 = getelementptr inbounds [120 x i64], [120 x i64]* @i64dst, i32 0, i32 %i.028
-  store i64 %0, i64* %arrayidx2, align 4
-  %add4 = add nuw nsw i32 %i.028, 1
-  %arrayidx5 = getelementptr inbounds [120 x i64], [120 x i64]* @i64src, i32 0, i32 %add4
-  %1 = load i64, i64* %arrayidx5, align 4
-  %arrayidx8 = getelementptr inbounds [120 x i64], [120 x i64]* @i64dst, i32 0, i32 %add4
-  store i64 %1, i64* %arrayidx8, align 4
-  %add16 = add nuw nsw i32 %i.028, 2
-  %cmp = icmp slt i32 %add16, %width_
-  br i1 %cmp, label %for.body, label %for.cond.cleanup
-}
-

diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-f64-stride-2.ll b/llvm/test/Analysis/CostModel/X86/interleaved-store-f64-stride-2.ll
index 2f4e880..f42e854 100644 (file)
--- a/llvm/test/Analysis/CostModel/X86/interleaved-store-f64-stride-2.ll
+++ b/llvm/test/Analysis/CostModel/X86/interleaved-store-f64-stride-2.ll
@@ -25,7 +25,7 @@ target triple = "x86_64-unknown-linux-gnu"
 ;
 ; AVX2: LV: Found an estimated cost of 1 for VF 1 For instruction:   store double %v1, double* %out1, align 8
 ; AVX2: LV: Found an estimated cost of 3 for VF 2 For instruction:   store double %v1, double* %out1, align 8
-; AVX2: LV: Found an estimated cost of 8 for VF 4 For instruction:   store double %v1, double* %out1, align 8
+; AVX2: LV: Found an estimated cost of 6 for VF 4 For instruction:   store double %v1, double* %out1, align 8
 ; AVX2: LV: Found an estimated cost of 32 for VF 8 For instruction:   store double %v1, double* %out1, align 8
 ; AVX2: LV: Found an estimated cost of 64 for VF 16 For instruction:   store double %v1, double* %out1, align 8
 ;

diff --git a/llvm/test/Analysis/CostModel/X86/interleaved-store-i64-stride-2.ll b/llvm/test/Analysis/CostModel/X86/interleaved-store-i64-stride-2.ll
index b5c5ab7..3bbbec1 100644 (file)
--- a/llvm/test/Analysis/CostModel/X86/interleaved-store-i64-stride-2.ll
+++ b/llvm/test/Analysis/CostModel/X86/interleaved-store-i64-stride-2.ll
@@ -25,7 +25,7 @@ target triple = "x86_64-unknown-linux-gnu"
 ;
 ; AVX2: LV: Found an estimated cost of 1 for VF 1 For instruction:   store i64 %v1, i64* %out1, align 8
 ; AVX2: LV: Found an estimated cost of 3 for VF 2 For instruction:   store i64 %v1, i64* %out1, align 8
-; AVX2: LV: Found an estimated cost of 8 for VF 4 For instruction:   store i64 %v1, i64* %out1, align 8
+; AVX2: LV: Found an estimated cost of 6 for VF 4 For instruction:   store i64 %v1, i64* %out1, align 8
 ; AVX2: LV: Found an estimated cost of 52 for VF 8 For instruction:   store i64 %v1, i64* %out1, align 8
 ; AVX2: LV: Found an estimated cost of 104 for VF 16 For instruction:   store i64 %v1, i64* %out1, align 8
 ;