From a05e0ff22376f4cbc6707841b3760c6f9f587663 Mon Sep 17 00:00:00 2001 From: Sanjay Patel Date: Fri, 1 Apr 2016 16:27:14 +0000 Subject: [PATCH] [x86] avoid intermediate splat for non-zero memsets (PR27100) Follow-up to D18566 - where we noticed that an intermediate splat was being generated for memsets of non-zero chars. That was because we told getMemsetStores() to use a 32-bit vector element type, and it happily obliged by producing that constant using an integer multiply. The tests that were added in the last patch are now equivalent for AVX1 and AVX2 (no splats, just a vector load), but we have PR27141 to track that splat difference. In the new tests, the splat via shuffling looks ok to me, but there might be some room for improvement depending on uarch there. Note that the SSE1/2 paths are not changed in this patch. That can be a follow-up. This patch should resolve PR27100. Differential Revision: http://reviews.llvm.org/D18676 llvm-svn: 265148 --- llvm/lib/Target/X86/X86ISelLowering.cpp | 14 +-- llvm/test/CodeGen/X86/memset-nonzero.ll | 179 ++++++++++++-------------------- 2 files changed, 74 insertions(+), 119 deletions(-) diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 61f8e67..290f92e 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -2029,12 +2029,14 @@ X86TargetLowering::getOptimalMemOpType(uint64_t Size, (!Subtarget.isUnalignedMem16Slow() || ((DstAlign == 0 || DstAlign >= 16) && (SrcAlign == 0 || SrcAlign >= 16)))) { - if (Size >= 32) { - // FIXME: Check if unaligned 32-byte accesses are slow. - if (Subtarget.hasInt256()) - return MVT::v8i32; - if (Subtarget.hasFp256()) - return MVT::v8f32; + // FIXME: Check if unaligned 32-byte accesses are slow. + if (Size >= 32 && Subtarget.hasAVX()) { + // Although this isn't a well-supported type for AVX1, we'll let + // legalization and shuffle lowering produce the optimal codegen. If we + // choose an optimal type with a vector element larger than a byte, + // getMemsetStores() may create an intermediate splat (using an integer + // multiply) before we splat as a vector. + return MVT::v32i8; } if (Subtarget.hasSSE2()) return MVT::v4i32; diff --git a/llvm/test/CodeGen/X86/memset-nonzero.ll b/llvm/test/CodeGen/X86/memset-nonzero.ll index a193fbb..5d9a615 100644 --- a/llvm/test/CodeGen/X86/memset-nonzero.ll +++ b/llvm/test/CodeGen/X86/memset-nonzero.ll @@ -35,17 +35,11 @@ define void @memset_32_nonzero_bytes(i8* %x) { ; SSE2-NEXT: movq %rax, (%rdi) ; SSE2-NEXT: retq ; -; AVX1-LABEL: memset_32_nonzero_bytes: -; AVX1: vmovaps {{.*#+}} ymm0 = [1.511366e-13,1.511366e-13,1.511366e-13,1.511366e-13,1.511366e-13,1.511366e-13,1.511366e-13,1.511366e-13] -; AVX1-NEXT: vmovups %ymm0, (%rdi) -; AVX1-NEXT: vzeroupper -; AVX1-NEXT: retq -; -; AVX2-LABEL: memset_32_nonzero_bytes: -; AVX2: vbroadcastss {{.*}}(%rip), %ymm0 -; AVX2-NEXT: vmovups %ymm0, (%rdi) -; AVX2-NEXT: vzeroupper -; AVX2-NEXT: retq +; AVX-LABEL: memset_32_nonzero_bytes: +; AVX: vmovaps {{.*#+}} ymm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42] +; AVX-NEXT: vmovups %ymm0, (%rdi) +; AVX-NEXT: vzeroupper +; AVX-NEXT: retq ; %call = tail call i8* @__memset_chk(i8* %x, i32 42, i64 32, i64 -1) ret void @@ -64,19 +58,12 @@ define void @memset_64_nonzero_bytes(i8* %x) { ; SSE2-NEXT: movq %rax, (%rdi) ; SSE2-NEXT: retq ; -; AVX1-LABEL: memset_64_nonzero_bytes: -; AVX1: vmovaps {{.*#+}} ymm0 = [1.511366e-13,1.511366e-13,1.511366e-13,1.511366e-13,1.511366e-13,1.511366e-13,1.511366e-13,1.511366e-13] -; AVX1-NEXT: vmovups %ymm0, 32(%rdi) -; AVX1-NEXT: vmovups %ymm0, (%rdi) -; AVX1-NEXT: vzeroupper -; AVX1-NEXT: retq -; -; AVX2-LABEL: memset_64_nonzero_bytes: -; AVX2: vbroadcastss {{.*}}(%rip), %ymm0 -; AVX2-NEXT: vmovups %ymm0, 32(%rdi) -; AVX2-NEXT: vmovups %ymm0, (%rdi) -; AVX2-NEXT: vzeroupper -; AVX2-NEXT: retq +; AVX-LABEL: memset_64_nonzero_bytes: +; AVX: vmovaps {{.*#+}} ymm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42] +; AVX-NEXT: vmovups %ymm0, 32(%rdi) +; AVX-NEXT: vmovups %ymm0, (%rdi) +; AVX-NEXT: vzeroupper +; AVX-NEXT: retq ; %call = tail call i8* @__memset_chk(i8* %x, i32 42, i64 64, i64 -1) ret void @@ -103,23 +90,14 @@ define void @memset_128_nonzero_bytes(i8* %x) { ; SSE2-NEXT: movq %rax, (%rdi) ; SSE2-NEXT: retq ; -; AVX1-LABEL: memset_128_nonzero_bytes: -; AVX1: vmovaps {{.*#+}} ymm0 = [1.511366e-13,1.511366e-13,1.511366e-13,1.511366e-13,1.511366e-13,1.511366e-13,1.511366e-13,1.511366e-13] -; AVX1-NEXT: vmovups %ymm0, 96(%rdi) -; AVX1-NEXT: vmovups %ymm0, 64(%rdi) -; AVX1-NEXT: vmovups %ymm0, 32(%rdi) -; AVX1-NEXT: vmovups %ymm0, (%rdi) -; AVX1-NEXT: vzeroupper -; AVX1-NEXT: retq -; -; AVX2-LABEL: memset_128_nonzero_bytes: -; AVX2: vbroadcastss {{.*}}(%rip), %ymm0 -; AVX2-NEXT: vmovups %ymm0, 96(%rdi) -; AVX2-NEXT: vmovups %ymm0, 64(%rdi) -; AVX2-NEXT: vmovups %ymm0, 32(%rdi) -; AVX2-NEXT: vmovups %ymm0, (%rdi) -; AVX2-NEXT: vzeroupper -; AVX2-NEXT: retq +; AVX-LABEL: memset_128_nonzero_bytes: +; AVX: vmovaps {{.*#+}} ymm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42] +; AVX-NEXT: vmovups %ymm0, 96(%rdi) +; AVX-NEXT: vmovups %ymm0, 64(%rdi) +; AVX-NEXT: vmovups %ymm0, 32(%rdi) +; AVX-NEXT: vmovups %ymm0, (%rdi) +; AVX-NEXT: vzeroupper +; AVX-NEXT: retq ; %call = tail call i8* @__memset_chk(i8* %x, i32 42, i64 128, i64 -1) ret void @@ -136,31 +114,18 @@ define void @memset_256_nonzero_bytes(i8* %x) { ; SSE2-NEXT: popq %rax ; SSE2-NEXT: retq ; -; AVX1-LABEL: memset_256_nonzero_bytes: -; AVX1: vmovaps {{.*#+}} ymm0 = [1.511366e-13,1.511366e-13,1.511366e-13,1.511366e-13,1.511366e-13,1.511366e-13,1.511366e-13,1.511366e-13] -; AVX1-NEXT: vmovups %ymm0, 224(%rdi) -; AVX1-NEXT: vmovups %ymm0, 192(%rdi) -; AVX1-NEXT: vmovups %ymm0, 160(%rdi) -; AVX1-NEXT: vmovups %ymm0, 128(%rdi) -; AVX1-NEXT: vmovups %ymm0, 96(%rdi) -; AVX1-NEXT: vmovups %ymm0, 64(%rdi) -; AVX1-NEXT: vmovups %ymm0, 32(%rdi) -; AVX1-NEXT: vmovups %ymm0, (%rdi) -; AVX1-NEXT: vzeroupper -; AVX1-NEXT: retq -; -; AVX2-LABEL: memset_256_nonzero_bytes: -; AVX2: vbroadcastss {{.*}}(%rip), %ymm0 -; AVX2-NEXT: vmovups %ymm0, 224(%rdi) -; AVX2-NEXT: vmovups %ymm0, 192(%rdi) -; AVX2-NEXT: vmovups %ymm0, 160(%rdi) -; AVX2-NEXT: vmovups %ymm0, 128(%rdi) -; AVX2-NEXT: vmovups %ymm0, 96(%rdi) -; AVX2-NEXT: vmovups %ymm0, 64(%rdi) -; AVX2-NEXT: vmovups %ymm0, 32(%rdi) -; AVX2-NEXT: vmovups %ymm0, (%rdi) -; AVX2-NEXT: vzeroupper -; AVX2-NEXT: retq +; AVX-LABEL: memset_256_nonzero_bytes: +; AVX: vmovaps {{.*#+}} ymm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42] +; AVX-NEXT: vmovups %ymm0, 224(%rdi) +; AVX-NEXT: vmovups %ymm0, 192(%rdi) +; AVX-NEXT: vmovups %ymm0, 160(%rdi) +; AVX-NEXT: vmovups %ymm0, 128(%rdi) +; AVX-NEXT: vmovups %ymm0, 96(%rdi) +; AVX-NEXT: vmovups %ymm0, 64(%rdi) +; AVX-NEXT: vmovups %ymm0, 32(%rdi) +; AVX-NEXT: vmovups %ymm0, (%rdi) +; AVX-NEXT: vzeroupper +; AVX-NEXT: retq ; %call = tail call i8* @__memset_chk(i8* %x, i32 42, i64 256, i64 -1) ret void @@ -211,21 +176,18 @@ define void @memset_32_nonconst_bytes(i8* %x, i8 %c) { ; SSE2-NEXT: retq ; ; AVX1-LABEL: memset_32_nonconst_bytes: -; AVX1: movzbl %sil, %eax -; AVX1-NEXT: imull $16843009, %eax, %eax # imm = 0x1010101 -; AVX1-NEXT: vmovd %eax, %xmm0 -; AVX1-NEXT: vpermilps {{.*#+}} xmm0 = xmm0[0,0,0,0] +; AVX1: vmovd %esi, %xmm0 +; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpshufb %xmm1, %xmm0, %xmm0 ; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 ; AVX1-NEXT: vmovups %ymm0, (%rdi) ; AVX1-NEXT: vzeroupper ; AVX1-NEXT: retq ; ; AVX2-LABEL: memset_32_nonconst_bytes: -; AVX2: movzbl %sil, %eax -; AVX2-NEXT: imull $16843009, %eax, %eax # imm = 0x1010101 -; AVX2-NEXT: vmovd %eax, %xmm0 -; AVX2-NEXT: vbroadcastss %xmm0, %ymm0 -; AVX2-NEXT: vmovups %ymm0, (%rdi) +; AVX2: vmovd %esi, %xmm0 +; AVX2-NEXT: vpbroadcastb %xmm0, %ymm0 +; AVX2-NEXT: vmovdqu %ymm0, (%rdi) ; AVX2-NEXT: vzeroupper ; AVX2-NEXT: retq ; @@ -249,10 +211,9 @@ define void @memset_64_nonconst_bytes(i8* %x, i8 %c) { ; SSE2-NEXT: retq ; ; AVX1-LABEL: memset_64_nonconst_bytes: -; AVX1: movzbl %sil, %eax -; AVX1-NEXT: imull $16843009, %eax, %eax # imm = 0x1010101 -; AVX1-NEXT: vmovd %eax, %xmm0 -; AVX1-NEXT: vpermilps {{.*#+}} xmm0 = xmm0[0,0,0,0] +; AVX1: vmovd %esi, %xmm0 +; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpshufb %xmm1, %xmm0, %xmm0 ; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 ; AVX1-NEXT: vmovups %ymm0, 32(%rdi) ; AVX1-NEXT: vmovups %ymm0, (%rdi) @@ -260,12 +221,10 @@ define void @memset_64_nonconst_bytes(i8* %x, i8 %c) { ; AVX1-NEXT: retq ; ; AVX2-LABEL: memset_64_nonconst_bytes: -; AVX2: movzbl %sil, %eax -; AVX2-NEXT: imull $16843009, %eax, %eax # imm = 0x1010101 -; AVX2-NEXT: vmovd %eax, %xmm0 -; AVX2-NEXT: vbroadcastss %xmm0, %ymm0 -; AVX2-NEXT: vmovups %ymm0, 32(%rdi) -; AVX2-NEXT: vmovups %ymm0, (%rdi) +; AVX2: vmovd %esi, %xmm0 +; AVX2-NEXT: vpbroadcastb %xmm0, %ymm0 +; AVX2-NEXT: vmovdqu %ymm0, 32(%rdi) +; AVX2-NEXT: vmovdqu %ymm0, (%rdi) ; AVX2-NEXT: vzeroupper ; AVX2-NEXT: retq ; @@ -297,10 +256,9 @@ define void @memset_128_nonconst_bytes(i8* %x, i8 %c) { ; SSE2-NEXT: retq ; ; AVX1-LABEL: memset_128_nonconst_bytes: -; AVX1: movzbl %sil, %eax -; AVX1-NEXT: imull $16843009, %eax, %eax # imm = 0x1010101 -; AVX1-NEXT: vmovd %eax, %xmm0 -; AVX1-NEXT: vpermilps {{.*#+}} xmm0 = xmm0[0,0,0,0] +; AVX1: vmovd %esi, %xmm0 +; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpshufb %xmm1, %xmm0, %xmm0 ; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 ; AVX1-NEXT: vmovups %ymm0, 96(%rdi) ; AVX1-NEXT: vmovups %ymm0, 64(%rdi) @@ -310,14 +268,12 @@ define void @memset_128_nonconst_bytes(i8* %x, i8 %c) { ; AVX1-NEXT: retq ; ; AVX2-LABEL: memset_128_nonconst_bytes: -; AVX2: movzbl %sil, %eax -; AVX2-NEXT: imull $16843009, %eax, %eax # imm = 0x1010101 -; AVX2-NEXT: vmovd %eax, %xmm0 -; AVX2-NEXT: vbroadcastss %xmm0, %ymm0 -; AVX2-NEXT: vmovups %ymm0, 96(%rdi) -; AVX2-NEXT: vmovups %ymm0, 64(%rdi) -; AVX2-NEXT: vmovups %ymm0, 32(%rdi) -; AVX2-NEXT: vmovups %ymm0, (%rdi) +; AVX2: vmovd %esi, %xmm0 +; AVX2-NEXT: vpbroadcastb %xmm0, %ymm0 +; AVX2-NEXT: vmovdqu %ymm0, 96(%rdi) +; AVX2-NEXT: vmovdqu %ymm0, 64(%rdi) +; AVX2-NEXT: vmovdqu %ymm0, 32(%rdi) +; AVX2-NEXT: vmovdqu %ymm0, (%rdi) ; AVX2-NEXT: vzeroupper ; AVX2-NEXT: retq ; @@ -331,10 +287,9 @@ define void @memset_256_nonconst_bytes(i8* %x, i8 %c) { ; SSE2-NEXT: jmp memset # TAILCALL ; ; AVX1-LABEL: memset_256_nonconst_bytes: -; AVX1: movzbl %sil, %eax -; AVX1-NEXT: imull $16843009, %eax, %eax # imm = 0x1010101 -; AVX1-NEXT: vmovd %eax, %xmm0 -; AVX1-NEXT: vpermilps {{.*#+}} xmm0 = xmm0[0,0,0,0] +; AVX1: vmovd %esi, %xmm0 +; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpshufb %xmm1, %xmm0, %xmm0 ; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 ; AVX1-NEXT: vmovups %ymm0, 224(%rdi) ; AVX1-NEXT: vmovups %ymm0, 192(%rdi) @@ -348,18 +303,16 @@ define void @memset_256_nonconst_bytes(i8* %x, i8 %c) { ; AVX1-NEXT: retq ; ; AVX2-LABEL: memset_256_nonconst_bytes: -; AVX2: movzbl %sil, %eax -; AVX2-NEXT: imull $16843009, %eax, %eax # imm = 0x1010101 -; AVX2-NEXT: vmovd %eax, %xmm0 -; AVX2-NEXT: vbroadcastss %xmm0, %ymm0 -; AVX2-NEXT: vmovups %ymm0, 224(%rdi) -; AVX2-NEXT: vmovups %ymm0, 192(%rdi) -; AVX2-NEXT: vmovups %ymm0, 160(%rdi) -; AVX2-NEXT: vmovups %ymm0, 128(%rdi) -; AVX2-NEXT: vmovups %ymm0, 96(%rdi) -; AVX2-NEXT: vmovups %ymm0, 64(%rdi) -; AVX2-NEXT: vmovups %ymm0, 32(%rdi) -; AVX2-NEXT: vmovups %ymm0, (%rdi) +; AVX2: vmovd %esi, %xmm0 +; AVX2-NEXT: vpbroadcastb %xmm0, %ymm0 +; AVX2-NEXT: vmovdqu %ymm0, 224(%rdi) +; AVX2-NEXT: vmovdqu %ymm0, 192(%rdi) +; AVX2-NEXT: vmovdqu %ymm0, 160(%rdi) +; AVX2-NEXT: vmovdqu %ymm0, 128(%rdi) +; AVX2-NEXT: vmovdqu %ymm0, 96(%rdi) +; AVX2-NEXT: vmovdqu %ymm0, 64(%rdi) +; AVX2-NEXT: vmovdqu %ymm0, 32(%rdi) +; AVX2-NEXT: vmovdqu %ymm0, (%rdi) ; AVX2-NEXT: vzeroupper ; AVX2-NEXT: retq ; -- 2.7.4