From 8ae6dad49b4440641073a2ef4001875ebf406352 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Thu, 4 Aug 2016 10:14:39 +0000 Subject: [PATCH] [X86][SSE] Don't decide when to scalarize CTTZ/CTLZ for performance at lowering - this is what cost models are for Improved CTTZ/CTLZ costings will be added shortly llvm-svn: 277713 --- llvm/lib/Target/X86/X86ISelLowering.cpp | 16 +- llvm/test/CodeGen/X86/vec_ctbits.ll | 61 ++- llvm/test/CodeGen/X86/vector-lzcnt-128.ll | 772 +++++++++++++++++++----------- llvm/test/CodeGen/X86/vector-lzcnt-256.ll | 286 +++++++---- llvm/test/CodeGen/X86/vector-tzcnt-128.ll | 453 ++++++++++++------ 5 files changed, 1023 insertions(+), 565 deletions(-) diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index d32fe19..5ca486d 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -785,7 +785,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, setOperationAction(ISD::CTTZ, MVT::v16i8, Custom); setOperationAction(ISD::CTTZ, MVT::v8i16, Custom); setOperationAction(ISD::CTTZ, MVT::v4i32, Custom); - // ISD::CTTZ v2i64 - scalarization is faster. + setOperationAction(ISD::CTTZ, MVT::v2i64, Custom); // Custom lower build_vector, vector_shuffle, and extract_vector_elt. for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32 }) { @@ -880,8 +880,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, setOperationAction(ISD::BITREVERSE, MVT::v16i8, Custom); setOperationAction(ISD::CTLZ, MVT::v16i8, Custom); setOperationAction(ISD::CTLZ, MVT::v8i16, Custom); - // ISD::CTLZ v4i32 - scalarization is faster. - // ISD::CTLZ v2i64 - scalarization is faster. + setOperationAction(ISD::CTLZ, MVT::v4i32, Custom); + setOperationAction(ISD::CTLZ, MVT::v2i64, Custom); } if (!Subtarget.useSoftFloat() && Subtarget.hasSSE41()) { @@ -1025,16 +1025,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) { setOperationAction(ISD::CTPOP, VT, Custom); setOperationAction(ISD::CTTZ, VT, Custom); - } - - // ISD::CTLZ v8i32/v4i64 - scalarization is faster without AVX2 - // as we end up splitting the 256-bit vectors. - for (auto VT : { MVT::v32i8, MVT::v16i16 }) setOperationAction(ISD::CTLZ, VT, Custom); - - if (HasInt256) - for (auto VT : { MVT::v8i32, MVT::v4i64 }) - setOperationAction(ISD::CTLZ, VT, Custom); + } if (Subtarget.hasAnyFMA()) { for (auto VT : { MVT::f32, MVT::f64, MVT::v4f32, MVT::v8f32, diff --git a/llvm/test/CodeGen/X86/vec_ctbits.ll b/llvm/test/CodeGen/X86/vec_ctbits.ll index e151317..23f70db 100644 --- a/llvm/test/CodeGen/X86/vec_ctbits.ll +++ b/llvm/test/CodeGen/X86/vec_ctbits.ll @@ -8,15 +8,26 @@ declare <2 x i64> @llvm.ctpop.v2i64(<2 x i64>) define <2 x i64> @footz(<2 x i64> %a) nounwind { ; CHECK-LABEL: footz: ; CHECK: # BB#0: -; CHECK-NEXT: movd %xmm0, %rax -; CHECK-NEXT: bsfq %rax, %rax -; CHECK-NEXT: movd %rax, %xmm1 -; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] -; CHECK-NEXT: movd %xmm0, %rax -; CHECK-NEXT: bsfq %rax, %rax -; CHECK-NEXT: movd %rax, %xmm0 -; CHECK-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0] -; CHECK-NEXT: movdqa %xmm1, %xmm0 +; CHECK-NEXT: pxor %xmm1, %xmm1 +; CHECK-NEXT: pxor %xmm2, %xmm2 +; CHECK-NEXT: psubq %xmm0, %xmm2 +; CHECK-NEXT: pand %xmm0, %xmm2 +; CHECK-NEXT: psubq {{.*}}(%rip), %xmm2 +; CHECK-NEXT: movdqa %xmm2, %xmm0 +; CHECK-NEXT: psrlq $1, %xmm0 +; CHECK-NEXT: pand {{.*}}(%rip), %xmm0 +; CHECK-NEXT: psubq %xmm0, %xmm2 +; CHECK-NEXT: movdqa {{.*#+}} xmm0 = [3689348814741910323,3689348814741910323] +; CHECK-NEXT: movdqa %xmm2, %xmm3 +; CHECK-NEXT: pand %xmm0, %xmm3 +; CHECK-NEXT: psrlq $2, %xmm2 +; CHECK-NEXT: pand %xmm0, %xmm2 +; CHECK-NEXT: paddq %xmm3, %xmm2 +; CHECK-NEXT: movdqa %xmm2, %xmm0 +; CHECK-NEXT: psrlq $4, %xmm0 +; CHECK-NEXT: paddq %xmm2, %xmm0 +; CHECK-NEXT: pand {{.*}}(%rip), %xmm0 +; CHECK-NEXT: psadbw %xmm1, %xmm0 ; CHECK-NEXT: retq %c = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 true) ret <2 x i64> %c @@ -75,18 +86,26 @@ define <2 x i32> @promtz(<2 x i32> %a) nounwind { ; CHECK-LABEL: promtz: ; CHECK: # BB#0: ; CHECK-NEXT: por {{.*}}(%rip), %xmm0 -; CHECK-NEXT: movd %xmm0, %rax -; CHECK-NEXT: bsfq %rax, %rax -; CHECK-NEXT: movl $64, %ecx -; CHECK-NEXT: cmoveq %rcx, %rax -; CHECK-NEXT: movd %rax, %xmm1 -; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] -; CHECK-NEXT: movd %xmm0, %rax -; CHECK-NEXT: bsfq %rax, %rax -; CHECK-NEXT: cmoveq %rcx, %rax -; CHECK-NEXT: movd %rax, %xmm0 -; CHECK-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0] -; CHECK-NEXT: movdqa %xmm1, %xmm0 +; CHECK-NEXT: pxor %xmm1, %xmm1 +; CHECK-NEXT: pxor %xmm2, %xmm2 +; CHECK-NEXT: psubq %xmm0, %xmm2 +; CHECK-NEXT: pand %xmm0, %xmm2 +; CHECK-NEXT: psubq {{.*}}(%rip), %xmm2 +; CHECK-NEXT: movdqa %xmm2, %xmm0 +; CHECK-NEXT: psrlq $1, %xmm0 +; CHECK-NEXT: pand {{.*}}(%rip), %xmm0 +; CHECK-NEXT: psubq %xmm0, %xmm2 +; CHECK-NEXT: movdqa {{.*#+}} xmm0 = [3689348814741910323,3689348814741910323] +; CHECK-NEXT: movdqa %xmm2, %xmm3 +; CHECK-NEXT: pand %xmm0, %xmm3 +; CHECK-NEXT: psrlq $2, %xmm2 +; CHECK-NEXT: pand %xmm0, %xmm2 +; CHECK-NEXT: paddq %xmm3, %xmm2 +; CHECK-NEXT: movdqa %xmm2, %xmm0 +; CHECK-NEXT: psrlq $4, %xmm0 +; CHECK-NEXT: paddq %xmm2, %xmm0 +; CHECK-NEXT: pand {{.*}}(%rip), %xmm0 +; CHECK-NEXT: psadbw %xmm1, %xmm0 ; CHECK-NEXT: retq %c = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %a, i1 false) ret <2 x i32> %c diff --git a/llvm/test/CodeGen/X86/vector-lzcnt-128.ll b/llvm/test/CodeGen/X86/vector-lzcnt-128.ll index deffdc3..d2c4040 100644 --- a/llvm/test/CodeGen/X86/vector-lzcnt-128.ll +++ b/llvm/test/CodeGen/X86/vector-lzcnt-128.ll @@ -50,52 +50,104 @@ define <2 x i64> @testv2i64(<2 x i64> %in) nounwind { ; ; SSSE3-LABEL: testv2i64: ; SSSE3: # BB#0: -; SSSE3-NEXT: movd %xmm0, %rax -; SSSE3-NEXT: bsrq %rax, %rax -; SSSE3-NEXT: movl $127, %ecx -; SSSE3-NEXT: cmoveq %rcx, %rax -; SSSE3-NEXT: xorq $63, %rax -; SSSE3-NEXT: movd %rax, %xmm1 -; SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] -; SSSE3-NEXT: movd %xmm0, %rax -; SSSE3-NEXT: bsrq %rax, %rax -; SSSE3-NEXT: cmoveq %rcx, %rax -; SSSE3-NEXT: xorq $63, %rax -; SSSE3-NEXT: movd %rax, %xmm0 -; SSSE3-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0] +; SSSE3-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; SSSE3-NEXT: movdqa %xmm0, %xmm1 +; SSSE3-NEXT: pand %xmm2, %xmm1 +; SSSE3-NEXT: movdqa {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0] +; SSSE3-NEXT: movdqa %xmm3, %xmm4 +; SSSE3-NEXT: pshufb %xmm1, %xmm4 +; SSSE3-NEXT: movdqa %xmm0, %xmm1 +; SSSE3-NEXT: psrlw $4, %xmm1 +; SSSE3-NEXT: pand %xmm2, %xmm1 +; SSSE3-NEXT: pxor %xmm2, %xmm2 +; SSSE3-NEXT: pshufb %xmm1, %xmm3 +; SSSE3-NEXT: pcmpeqb %xmm2, %xmm1 +; SSSE3-NEXT: pand %xmm4, %xmm1 +; SSSE3-NEXT: paddb %xmm3, %xmm1 +; SSSE3-NEXT: movdqa %xmm0, %xmm3 +; SSSE3-NEXT: pcmpeqb %xmm2, %xmm3 +; SSSE3-NEXT: psrlw $8, %xmm3 +; SSSE3-NEXT: pand %xmm1, %xmm3 +; SSSE3-NEXT: psrlw $8, %xmm1 +; SSSE3-NEXT: paddw %xmm3, %xmm1 +; SSSE3-NEXT: movdqa %xmm0, %xmm3 +; SSSE3-NEXT: pcmpeqw %xmm2, %xmm3 +; SSSE3-NEXT: psrld $16, %xmm3 +; SSSE3-NEXT: pand %xmm1, %xmm3 +; SSSE3-NEXT: psrld $16, %xmm1 +; SSSE3-NEXT: paddd %xmm3, %xmm1 +; SSSE3-NEXT: pcmpeqd %xmm2, %xmm0 +; SSSE3-NEXT: psrlq $32, %xmm0 +; SSSE3-NEXT: pand %xmm1, %xmm0 +; SSSE3-NEXT: psrlq $32, %xmm1 +; SSSE3-NEXT: paddq %xmm0, %xmm1 ; SSSE3-NEXT: movdqa %xmm1, %xmm0 ; SSSE3-NEXT: retq ; ; SSE41-LABEL: testv2i64: ; SSE41: # BB#0: -; SSE41-NEXT: pextrq $1, %xmm0, %rax -; SSE41-NEXT: bsrq %rax, %rax -; SSE41-NEXT: movl $127, %ecx -; SSE41-NEXT: cmoveq %rcx, %rax -; SSE41-NEXT: xorq $63, %rax -; SSE41-NEXT: movd %rax, %xmm1 -; SSE41-NEXT: movd %xmm0, %rax -; SSE41-NEXT: bsrq %rax, %rax -; SSE41-NEXT: cmoveq %rcx, %rax -; SSE41-NEXT: xorq $63, %rax -; SSE41-NEXT: movd %rax, %xmm0 -; SSE41-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; SSE41-NEXT: movdqa %xmm0, %xmm1 +; SSE41-NEXT: pand %xmm2, %xmm1 +; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0] +; SSE41-NEXT: movdqa %xmm3, %xmm4 +; SSE41-NEXT: pshufb %xmm1, %xmm4 +; SSE41-NEXT: movdqa %xmm0, %xmm1 +; SSE41-NEXT: psrlw $4, %xmm1 +; SSE41-NEXT: pand %xmm2, %xmm1 +; SSE41-NEXT: pxor %xmm2, %xmm2 +; SSE41-NEXT: pshufb %xmm1, %xmm3 +; SSE41-NEXT: pcmpeqb %xmm2, %xmm1 +; SSE41-NEXT: pand %xmm4, %xmm1 +; SSE41-NEXT: paddb %xmm3, %xmm1 +; SSE41-NEXT: movdqa %xmm0, %xmm3 +; SSE41-NEXT: pcmpeqb %xmm2, %xmm3 +; SSE41-NEXT: psrlw $8, %xmm3 +; SSE41-NEXT: pand %xmm1, %xmm3 +; SSE41-NEXT: psrlw $8, %xmm1 +; SSE41-NEXT: paddw %xmm3, %xmm1 +; SSE41-NEXT: movdqa %xmm0, %xmm3 +; SSE41-NEXT: pcmpeqw %xmm2, %xmm3 +; SSE41-NEXT: psrld $16, %xmm3 +; SSE41-NEXT: pand %xmm1, %xmm3 +; SSE41-NEXT: psrld $16, %xmm1 +; SSE41-NEXT: paddd %xmm3, %xmm1 +; SSE41-NEXT: pcmpeqd %xmm2, %xmm0 +; SSE41-NEXT: psrlq $32, %xmm0 +; SSE41-NEXT: pand %xmm1, %xmm0 +; SSE41-NEXT: psrlq $32, %xmm1 +; SSE41-NEXT: paddq %xmm0, %xmm1 +; SSE41-NEXT: movdqa %xmm1, %xmm0 ; SSE41-NEXT: retq ; ; AVX-LABEL: testv2i64: ; AVX: # BB#0: -; AVX-NEXT: vpextrq $1, %xmm0, %rax -; AVX-NEXT: bsrq %rax, %rax -; AVX-NEXT: movl $127, %ecx -; AVX-NEXT: cmoveq %rcx, %rax -; AVX-NEXT: xorq $63, %rax -; AVX-NEXT: vmovq %rax, %xmm1 -; AVX-NEXT: vmovq %xmm0, %rax -; AVX-NEXT: bsrq %rax, %rax -; AVX-NEXT: cmoveq %rcx, %rax -; AVX-NEXT: xorq $63, %rax -; AVX-NEXT: vmovq %rax, %xmm0 -; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; AVX-NEXT: vmovdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX-NEXT: vpand %xmm1, %xmm0, %xmm2 +; AVX-NEXT: vmovdqa {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0] +; AVX-NEXT: vpshufb %xmm2, %xmm3, %xmm2 +; AVX-NEXT: vpsrlw $4, %xmm0, %xmm4 +; AVX-NEXT: vpand %xmm1, %xmm4, %xmm1 +; AVX-NEXT: vpxor %xmm4, %xmm4, %xmm4 +; AVX-NEXT: vpcmpeqb %xmm4, %xmm1, %xmm5 +; AVX-NEXT: vpand %xmm5, %xmm2, %xmm2 +; AVX-NEXT: vpshufb %xmm1, %xmm3, %xmm1 +; AVX-NEXT: vpaddb %xmm1, %xmm2, %xmm1 +; AVX-NEXT: vpcmpeqb %xmm4, %xmm0, %xmm2 +; AVX-NEXT: vpsrlw $8, %xmm2, %xmm2 +; AVX-NEXT: vpand %xmm2, %xmm1, %xmm2 +; AVX-NEXT: vpsrlw $8, %xmm1, %xmm1 +; AVX-NEXT: vpaddw %xmm2, %xmm1, %xmm1 +; AVX-NEXT: vpcmpeqw %xmm4, %xmm0, %xmm2 +; AVX-NEXT: vpsrld $16, %xmm2, %xmm2 +; AVX-NEXT: vpand %xmm2, %xmm1, %xmm2 +; AVX-NEXT: vpsrld $16, %xmm1, %xmm1 +; AVX-NEXT: vpaddd %xmm2, %xmm1, %xmm1 +; AVX-NEXT: vpcmpeqd %xmm4, %xmm0, %xmm0 +; AVX-NEXT: vpsrlq $32, %xmm0, %xmm0 +; AVX-NEXT: vpand %xmm0, %xmm1, %xmm0 +; AVX-NEXT: vpsrlq $32, %xmm1, %xmm1 +; AVX-NEXT: vpaddq %xmm0, %xmm1, %xmm0 ; AVX-NEXT: retq ; ; AVX512VLCD-LABEL: testv2i64: @@ -112,32 +164,38 @@ define <2 x i64> @testv2i64(<2 x i64> %in) nounwind { ; ; X32-SSE-LABEL: testv2i64: ; X32-SSE: # BB#0: -; X32-SSE-NEXT: pushl %esi -; X32-SSE-NEXT: pextrd $2, %xmm0, %eax -; X32-SSE-NEXT: bsrl %eax, %eax -; X32-SSE-NEXT: movl $63, %ecx -; X32-SSE-NEXT: cmovel %ecx, %eax -; X32-SSE-NEXT: xorl $31, %eax -; X32-SSE-NEXT: addl $32, %eax -; X32-SSE-NEXT: pextrd $3, %xmm0, %edx -; X32-SSE-NEXT: bsrl %edx, %esi -; X32-SSE-NEXT: xorl $31, %esi -; X32-SSE-NEXT: testl %edx, %edx -; X32-SSE-NEXT: cmovel %eax, %esi -; X32-SSE-NEXT: movd %esi, %xmm1 -; X32-SSE-NEXT: movd %xmm0, %eax -; X32-SSE-NEXT: bsrl %eax, %eax -; X32-SSE-NEXT: cmovel %ecx, %eax -; X32-SSE-NEXT: xorl $31, %eax -; X32-SSE-NEXT: addl $32, %eax -; X32-SSE-NEXT: pextrd $1, %xmm0, %ecx -; X32-SSE-NEXT: bsrl %ecx, %edx -; X32-SSE-NEXT: xorl $31, %edx -; X32-SSE-NEXT: testl %ecx, %ecx -; X32-SSE-NEXT: cmovel %eax, %edx -; X32-SSE-NEXT: movd %edx, %xmm0 -; X32-SSE-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] -; X32-SSE-NEXT: popl %esi +; X32-SSE-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; X32-SSE-NEXT: movdqa %xmm0, %xmm1 +; X32-SSE-NEXT: pand %xmm2, %xmm1 +; X32-SSE-NEXT: movdqa {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0] +; X32-SSE-NEXT: movdqa %xmm3, %xmm4 +; X32-SSE-NEXT: pshufb %xmm1, %xmm4 +; X32-SSE-NEXT: movdqa %xmm0, %xmm1 +; X32-SSE-NEXT: psrlw $4, %xmm1 +; X32-SSE-NEXT: pand %xmm2, %xmm1 +; X32-SSE-NEXT: pxor %xmm2, %xmm2 +; X32-SSE-NEXT: pshufb %xmm1, %xmm3 +; X32-SSE-NEXT: pcmpeqb %xmm2, %xmm1 +; X32-SSE-NEXT: pand %xmm4, %xmm1 +; X32-SSE-NEXT: paddb %xmm3, %xmm1 +; X32-SSE-NEXT: movdqa %xmm0, %xmm3 +; X32-SSE-NEXT: pcmpeqb %xmm2, %xmm3 +; X32-SSE-NEXT: psrlw $8, %xmm3 +; X32-SSE-NEXT: pand %xmm1, %xmm3 +; X32-SSE-NEXT: psrlw $8, %xmm1 +; X32-SSE-NEXT: paddw %xmm3, %xmm1 +; X32-SSE-NEXT: movdqa %xmm0, %xmm3 +; X32-SSE-NEXT: pcmpeqw %xmm2, %xmm3 +; X32-SSE-NEXT: psrld $16, %xmm3 +; X32-SSE-NEXT: pand %xmm1, %xmm3 +; X32-SSE-NEXT: psrld $16, %xmm1 +; X32-SSE-NEXT: paddd %xmm3, %xmm1 +; X32-SSE-NEXT: pcmpeqd %xmm2, %xmm0 +; X32-SSE-NEXT: psrlq $32, %xmm0 +; X32-SSE-NEXT: pand %xmm1, %xmm0 +; X32-SSE-NEXT: psrlq $32, %xmm1 +; X32-SSE-NEXT: paddq %xmm0, %xmm1 +; X32-SSE-NEXT: movdqa %xmm1, %xmm0 ; X32-SSE-NEXT: retl %out = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %in, i1 0) @@ -177,43 +235,104 @@ define <2 x i64> @testv2i64u(<2 x i64> %in) nounwind { ; ; SSSE3-LABEL: testv2i64u: ; SSSE3: # BB#0: -; SSSE3-NEXT: movd %xmm0, %rax -; SSSE3-NEXT: bsrq %rax, %rax -; SSSE3-NEXT: xorq $63, %rax -; SSSE3-NEXT: movd %rax, %xmm1 -; SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] -; SSSE3-NEXT: movd %xmm0, %rax -; SSSE3-NEXT: bsrq %rax, %rax -; SSSE3-NEXT: xorq $63, %rax -; SSSE3-NEXT: movd %rax, %xmm0 -; SSSE3-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0] +; SSSE3-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; SSSE3-NEXT: movdqa %xmm0, %xmm1 +; SSSE3-NEXT: pand %xmm2, %xmm1 +; SSSE3-NEXT: movdqa {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0] +; SSSE3-NEXT: movdqa %xmm3, %xmm4 +; SSSE3-NEXT: pshufb %xmm1, %xmm4 +; SSSE3-NEXT: movdqa %xmm0, %xmm1 +; SSSE3-NEXT: psrlw $4, %xmm1 +; SSSE3-NEXT: pand %xmm2, %xmm1 +; SSSE3-NEXT: pxor %xmm2, %xmm2 +; SSSE3-NEXT: pshufb %xmm1, %xmm3 +; SSSE3-NEXT: pcmpeqb %xmm2, %xmm1 +; SSSE3-NEXT: pand %xmm4, %xmm1 +; SSSE3-NEXT: paddb %xmm3, %xmm1 +; SSSE3-NEXT: movdqa %xmm0, %xmm3 +; SSSE3-NEXT: pcmpeqb %xmm2, %xmm3 +; SSSE3-NEXT: psrlw $8, %xmm3 +; SSSE3-NEXT: pand %xmm1, %xmm3 +; SSSE3-NEXT: psrlw $8, %xmm1 +; SSSE3-NEXT: paddw %xmm3, %xmm1 +; SSSE3-NEXT: movdqa %xmm0, %xmm3 +; SSSE3-NEXT: pcmpeqw %xmm2, %xmm3 +; SSSE3-NEXT: psrld $16, %xmm3 +; SSSE3-NEXT: pand %xmm1, %xmm3 +; SSSE3-NEXT: psrld $16, %xmm1 +; SSSE3-NEXT: paddd %xmm3, %xmm1 +; SSSE3-NEXT: pcmpeqd %xmm2, %xmm0 +; SSSE3-NEXT: psrlq $32, %xmm0 +; SSSE3-NEXT: pand %xmm1, %xmm0 +; SSSE3-NEXT: psrlq $32, %xmm1 +; SSSE3-NEXT: paddq %xmm0, %xmm1 ; SSSE3-NEXT: movdqa %xmm1, %xmm0 ; SSSE3-NEXT: retq ; ; SSE41-LABEL: testv2i64u: ; SSE41: # BB#0: -; SSE41-NEXT: pextrq $1, %xmm0, %rax -; SSE41-NEXT: bsrq %rax, %rax -; SSE41-NEXT: xorq $63, %rax -; SSE41-NEXT: movd %rax, %xmm1 -; SSE41-NEXT: movd %xmm0, %rax -; SSE41-NEXT: bsrq %rax, %rax -; SSE41-NEXT: xorq $63, %rax -; SSE41-NEXT: movd %rax, %xmm0 -; SSE41-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; SSE41-NEXT: movdqa %xmm0, %xmm1 +; SSE41-NEXT: pand %xmm2, %xmm1 +; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0] +; SSE41-NEXT: movdqa %xmm3, %xmm4 +; SSE41-NEXT: pshufb %xmm1, %xmm4 +; SSE41-NEXT: movdqa %xmm0, %xmm1 +; SSE41-NEXT: psrlw $4, %xmm1 +; SSE41-NEXT: pand %xmm2, %xmm1 +; SSE41-NEXT: pxor %xmm2, %xmm2 +; SSE41-NEXT: pshufb %xmm1, %xmm3 +; SSE41-NEXT: pcmpeqb %xmm2, %xmm1 +; SSE41-NEXT: pand %xmm4, %xmm1 +; SSE41-NEXT: paddb %xmm3, %xmm1 +; SSE41-NEXT: movdqa %xmm0, %xmm3 +; SSE41-NEXT: pcmpeqb %xmm2, %xmm3 +; SSE41-NEXT: psrlw $8, %xmm3 +; SSE41-NEXT: pand %xmm1, %xmm3 +; SSE41-NEXT: psrlw $8, %xmm1 +; SSE41-NEXT: paddw %xmm3, %xmm1 +; SSE41-NEXT: movdqa %xmm0, %xmm3 +; SSE41-NEXT: pcmpeqw %xmm2, %xmm3 +; SSE41-NEXT: psrld $16, %xmm3 +; SSE41-NEXT: pand %xmm1, %xmm3 +; SSE41-NEXT: psrld $16, %xmm1 +; SSE41-NEXT: paddd %xmm3, %xmm1 +; SSE41-NEXT: pcmpeqd %xmm2, %xmm0 +; SSE41-NEXT: psrlq $32, %xmm0 +; SSE41-NEXT: pand %xmm1, %xmm0 +; SSE41-NEXT: psrlq $32, %xmm1 +; SSE41-NEXT: paddq %xmm0, %xmm1 +; SSE41-NEXT: movdqa %xmm1, %xmm0 ; SSE41-NEXT: retq ; ; AVX-LABEL: testv2i64u: ; AVX: # BB#0: -; AVX-NEXT: vpextrq $1, %xmm0, %rax -; AVX-NEXT: bsrq %rax, %rax -; AVX-NEXT: xorq $63, %rax -; AVX-NEXT: vmovq %rax, %xmm1 -; AVX-NEXT: vmovq %xmm0, %rax -; AVX-NEXT: bsrq %rax, %rax -; AVX-NEXT: xorq $63, %rax -; AVX-NEXT: vmovq %rax, %xmm0 -; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; AVX-NEXT: vmovdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX-NEXT: vpand %xmm1, %xmm0, %xmm2 +; AVX-NEXT: vmovdqa {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0] +; AVX-NEXT: vpshufb %xmm2, %xmm3, %xmm2 +; AVX-NEXT: vpsrlw $4, %xmm0, %xmm4 +; AVX-NEXT: vpand %xmm1, %xmm4, %xmm1 +; AVX-NEXT: vpxor %xmm4, %xmm4, %xmm4 +; AVX-NEXT: vpcmpeqb %xmm4, %xmm1, %xmm5 +; AVX-NEXT: vpand %xmm5, %xmm2, %xmm2 +; AVX-NEXT: vpshufb %xmm1, %xmm3, %xmm1 +; AVX-NEXT: vpaddb %xmm1, %xmm2, %xmm1 +; AVX-NEXT: vpcmpeqb %xmm4, %xmm0, %xmm2 +; AVX-NEXT: vpsrlw $8, %xmm2, %xmm2 +; AVX-NEXT: vpand %xmm2, %xmm1, %xmm2 +; AVX-NEXT: vpsrlw $8, %xmm1, %xmm1 +; AVX-NEXT: vpaddw %xmm2, %xmm1, %xmm1 +; AVX-NEXT: vpcmpeqw %xmm4, %xmm0, %xmm2 +; AVX-NEXT: vpsrld $16, %xmm2, %xmm2 +; AVX-NEXT: vpand %xmm2, %xmm1, %xmm2 +; AVX-NEXT: vpsrld $16, %xmm1, %xmm1 +; AVX-NEXT: vpaddd %xmm2, %xmm1, %xmm1 +; AVX-NEXT: vpcmpeqd %xmm4, %xmm0, %xmm0 +; AVX-NEXT: vpsrlq $32, %xmm0, %xmm0 +; AVX-NEXT: vpand %xmm0, %xmm1, %xmm0 +; AVX-NEXT: vpsrlq $32, %xmm1, %xmm1 +; AVX-NEXT: vpaddq %xmm0, %xmm1, %xmm0 ; AVX-NEXT: retq ; ; AVX512VLCD-LABEL: testv2i64u: @@ -230,27 +349,38 @@ define <2 x i64> @testv2i64u(<2 x i64> %in) nounwind { ; ; X32-SSE-LABEL: testv2i64u: ; X32-SSE: # BB#0: -; X32-SSE-NEXT: pextrd $3, %xmm0, %eax -; X32-SSE-NEXT: bsrl %eax, %ecx -; X32-SSE-NEXT: xorl $31, %ecx -; X32-SSE-NEXT: pextrd $2, %xmm0, %edx -; X32-SSE-NEXT: bsrl %edx, %edx -; X32-SSE-NEXT: xorl $31, %edx -; X32-SSE-NEXT: addl $32, %edx -; X32-SSE-NEXT: testl %eax, %eax -; X32-SSE-NEXT: cmovnel %ecx, %edx -; X32-SSE-NEXT: movd %edx, %xmm1 -; X32-SSE-NEXT: pextrd $1, %xmm0, %eax -; X32-SSE-NEXT: bsrl %eax, %ecx -; X32-SSE-NEXT: xorl $31, %ecx -; X32-SSE-NEXT: movd %xmm0, %edx -; X32-SSE-NEXT: bsrl %edx, %edx -; X32-SSE-NEXT: xorl $31, %edx -; X32-SSE-NEXT: addl $32, %edx -; X32-SSE-NEXT: testl %eax, %eax -; X32-SSE-NEXT: cmovnel %ecx, %edx -; X32-SSE-NEXT: movd %edx, %xmm0 -; X32-SSE-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; X32-SSE-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; X32-SSE-NEXT: movdqa %xmm0, %xmm1 +; X32-SSE-NEXT: pand %xmm2, %xmm1 +; X32-SSE-NEXT: movdqa {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0] +; X32-SSE-NEXT: movdqa %xmm3, %xmm4 +; X32-SSE-NEXT: pshufb %xmm1, %xmm4 +; X32-SSE-NEXT: movdqa %xmm0, %xmm1 +; X32-SSE-NEXT: psrlw $4, %xmm1 +; X32-SSE-NEXT: pand %xmm2, %xmm1 +; X32-SSE-NEXT: pxor %xmm2, %xmm2 +; X32-SSE-NEXT: pshufb %xmm1, %xmm3 +; X32-SSE-NEXT: pcmpeqb %xmm2, %xmm1 +; X32-SSE-NEXT: pand %xmm4, %xmm1 +; X32-SSE-NEXT: paddb %xmm3, %xmm1 +; X32-SSE-NEXT: movdqa %xmm0, %xmm3 +; X32-SSE-NEXT: pcmpeqb %xmm2, %xmm3 +; X32-SSE-NEXT: psrlw $8, %xmm3 +; X32-SSE-NEXT: pand %xmm1, %xmm3 +; X32-SSE-NEXT: psrlw $8, %xmm1 +; X32-SSE-NEXT: paddw %xmm3, %xmm1 +; X32-SSE-NEXT: movdqa %xmm0, %xmm3 +; X32-SSE-NEXT: pcmpeqw %xmm2, %xmm3 +; X32-SSE-NEXT: psrld $16, %xmm3 +; X32-SSE-NEXT: pand %xmm1, %xmm3 +; X32-SSE-NEXT: psrld $16, %xmm1 +; X32-SSE-NEXT: paddd %xmm3, %xmm1 +; X32-SSE-NEXT: pcmpeqd %xmm2, %xmm0 +; X32-SSE-NEXT: psrlq $32, %xmm0 +; X32-SSE-NEXT: pand %xmm1, %xmm0 +; X32-SSE-NEXT: psrlq $32, %xmm1 +; X32-SSE-NEXT: paddq %xmm0, %xmm1 +; X32-SSE-NEXT: movdqa %xmm1, %xmm0 ; X32-SSE-NEXT: retl %out = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %in, i1 -1) @@ -324,85 +454,87 @@ define <4 x i32> @testv4i32(<4 x i32> %in) nounwind { ; ; SSSE3-LABEL: testv4i32: ; SSSE3: # BB#0: -; SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3] -; SSSE3-NEXT: movd %xmm1, %eax -; SSSE3-NEXT: bsrl %eax, %eax -; SSSE3-NEXT: movl $63, %ecx -; SSSE3-NEXT: cmovel %ecx, %eax -; SSSE3-NEXT: xorl $31, %eax -; SSSE3-NEXT: movd %eax, %xmm1 -; SSSE3-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3] -; SSSE3-NEXT: movd %xmm2, %eax -; SSSE3-NEXT: bsrl %eax, %eax -; SSSE3-NEXT: cmovel %ecx, %eax -; SSSE3-NEXT: xorl $31, %eax -; SSSE3-NEXT: movd %eax, %xmm2 -; SSSE3-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] -; SSSE3-NEXT: movd %xmm0, %eax -; SSSE3-NEXT: bsrl %eax, %eax -; SSSE3-NEXT: cmovel %ecx, %eax -; SSSE3-NEXT: xorl $31, %eax -; SSSE3-NEXT: movd %eax, %xmm1 -; SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] -; SSSE3-NEXT: movd %xmm0, %eax -; SSSE3-NEXT: bsrl %eax, %eax -; SSSE3-NEXT: cmovel %ecx, %eax -; SSSE3-NEXT: xorl $31, %eax -; SSSE3-NEXT: movd %eax, %xmm0 -; SSSE3-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] -; SSSE3-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] +; SSSE3-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; SSSE3-NEXT: movdqa %xmm0, %xmm1 +; SSSE3-NEXT: pand %xmm2, %xmm1 +; SSSE3-NEXT: movdqa {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0] +; SSSE3-NEXT: movdqa %xmm3, %xmm4 +; SSSE3-NEXT: pshufb %xmm1, %xmm4 +; SSSE3-NEXT: movdqa %xmm0, %xmm1 +; SSSE3-NEXT: psrlw $4, %xmm1 +; SSSE3-NEXT: pand %xmm2, %xmm1 +; SSSE3-NEXT: pxor %xmm2, %xmm2 +; SSSE3-NEXT: pshufb %xmm1, %xmm3 +; SSSE3-NEXT: pcmpeqb %xmm2, %xmm1 +; SSSE3-NEXT: pand %xmm4, %xmm1 +; SSSE3-NEXT: paddb %xmm3, %xmm1 +; SSSE3-NEXT: movdqa %xmm0, %xmm3 +; SSSE3-NEXT: pcmpeqb %xmm2, %xmm3 +; SSSE3-NEXT: psrlw $8, %xmm3 +; SSSE3-NEXT: pand %xmm1, %xmm3 +; SSSE3-NEXT: psrlw $8, %xmm1 +; SSSE3-NEXT: paddw %xmm3, %xmm1 +; SSSE3-NEXT: pcmpeqw %xmm2, %xmm0 +; SSSE3-NEXT: psrld $16, %xmm0 +; SSSE3-NEXT: pand %xmm1, %xmm0 +; SSSE3-NEXT: psrld $16, %xmm1 +; SSSE3-NEXT: paddd %xmm0, %xmm1 ; SSSE3-NEXT: movdqa %xmm1, %xmm0 ; SSSE3-NEXT: retq ; ; SSE41-LABEL: testv4i32: ; SSE41: # BB#0: -; SSE41-NEXT: pextrd $1, %xmm0, %eax -; SSE41-NEXT: bsrl %eax, %eax -; SSE41-NEXT: movl $63, %ecx -; SSE41-NEXT: cmovel %ecx, %eax -; SSE41-NEXT: xorl $31, %eax -; SSE41-NEXT: movd %xmm0, %edx -; SSE41-NEXT: bsrl %edx, %edx -; SSE41-NEXT: cmovel %ecx, %edx -; SSE41-NEXT: xorl $31, %edx -; SSE41-NEXT: movd %edx, %xmm1 -; SSE41-NEXT: pinsrd $1, %eax, %xmm1 -; SSE41-NEXT: pextrd $2, %xmm0, %eax -; SSE41-NEXT: bsrl %eax, %eax -; SSE41-NEXT: cmovel %ecx, %eax -; SSE41-NEXT: xorl $31, %eax -; SSE41-NEXT: pinsrd $2, %eax, %xmm1 -; SSE41-NEXT: pextrd $3, %xmm0, %eax -; SSE41-NEXT: bsrl %eax, %eax -; SSE41-NEXT: cmovel %ecx, %eax -; SSE41-NEXT: xorl $31, %eax -; SSE41-NEXT: pinsrd $3, %eax, %xmm1 +; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; SSE41-NEXT: movdqa %xmm0, %xmm1 +; SSE41-NEXT: pand %xmm2, %xmm1 +; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0] +; SSE41-NEXT: movdqa %xmm3, %xmm4 +; SSE41-NEXT: pshufb %xmm1, %xmm4 +; SSE41-NEXT: movdqa %xmm0, %xmm1 +; SSE41-NEXT: psrlw $4, %xmm1 +; SSE41-NEXT: pand %xmm2, %xmm1 +; SSE41-NEXT: pxor %xmm2, %xmm2 +; SSE41-NEXT: pshufb %xmm1, %xmm3 +; SSE41-NEXT: pcmpeqb %xmm2, %xmm1 +; SSE41-NEXT: pand %xmm4, %xmm1 +; SSE41-NEXT: paddb %xmm3, %xmm1 +; SSE41-NEXT: movdqa %xmm0, %xmm3 +; SSE41-NEXT: pcmpeqb %xmm2, %xmm3 +; SSE41-NEXT: psrlw $8, %xmm3 +; SSE41-NEXT: pand %xmm1, %xmm3 +; SSE41-NEXT: psrlw $8, %xmm1 +; SSE41-NEXT: paddw %xmm3, %xmm1 +; SSE41-NEXT: pcmpeqw %xmm2, %xmm0 +; SSE41-NEXT: psrld $16, %xmm0 +; SSE41-NEXT: pand %xmm1, %xmm0 +; SSE41-NEXT: psrld $16, %xmm1 +; SSE41-NEXT: paddd %xmm0, %xmm1 ; SSE41-NEXT: movdqa %xmm1, %xmm0 ; SSE41-NEXT: retq ; ; AVX-LABEL: testv4i32: ; AVX: # BB#0: -; AVX-NEXT: vpextrd $1, %xmm0, %eax -; AVX-NEXT: bsrl %eax, %eax -; AVX-NEXT: movl $63, %ecx -; AVX-NEXT: cmovel %ecx, %eax -; AVX-NEXT: xorl $31, %eax -; AVX-NEXT: vmovd %xmm0, %edx -; AVX-NEXT: bsrl %edx, %edx -; AVX-NEXT: cmovel %ecx, %edx -; AVX-NEXT: xorl $31, %edx -; AVX-NEXT: vmovd %edx, %xmm1 -; AVX-NEXT: vpinsrd $1, %eax, %xmm1, %xmm1 -; AVX-NEXT: vpextrd $2, %xmm0, %eax -; AVX-NEXT: bsrl %eax, %eax -; AVX-NEXT: cmovel %ecx, %eax -; AVX-NEXT: xorl $31, %eax -; AVX-NEXT: vpinsrd $2, %eax, %xmm1, %xmm1 -; AVX-NEXT: vpextrd $3, %xmm0, %eax -; AVX-NEXT: bsrl %eax, %eax -; AVX-NEXT: cmovel %ecx, %eax -; AVX-NEXT: xorl $31, %eax -; AVX-NEXT: vpinsrd $3, %eax, %xmm1, %xmm0 +; AVX-NEXT: vmovdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX-NEXT: vpand %xmm1, %xmm0, %xmm2 +; AVX-NEXT: vmovdqa {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0] +; AVX-NEXT: vpshufb %xmm2, %xmm3, %xmm2 +; AVX-NEXT: vpsrlw $4, %xmm0, %xmm4 +; AVX-NEXT: vpand %xmm1, %xmm4, %xmm1 +; AVX-NEXT: vpxor %xmm4, %xmm4, %xmm4 +; AVX-NEXT: vpcmpeqb %xmm4, %xmm1, %xmm5 +; AVX-NEXT: vpand %xmm5, %xmm2, %xmm2 +; AVX-NEXT: vpshufb %xmm1, %xmm3, %xmm1 +; AVX-NEXT: vpaddb %xmm1, %xmm2, %xmm1 +; AVX-NEXT: vpcmpeqb %xmm4, %xmm0, %xmm2 +; AVX-NEXT: vpsrlw $8, %xmm2, %xmm2 +; AVX-NEXT: vpand %xmm2, %xmm1, %xmm2 +; AVX-NEXT: vpsrlw $8, %xmm1, %xmm1 +; AVX-NEXT: vpaddw %xmm2, %xmm1, %xmm1 +; AVX-NEXT: vpcmpeqw %xmm4, %xmm0, %xmm0 +; AVX-NEXT: vpsrld $16, %xmm0, %xmm0 +; AVX-NEXT: vpand %xmm0, %xmm1, %xmm0 +; AVX-NEXT: vpsrld $16, %xmm1, %xmm1 +; AVX-NEXT: vpaddd %xmm0, %xmm1, %xmm0 ; AVX-NEXT: retq ; ; AVX512VLCD-LABEL: testv4i32: @@ -419,27 +551,31 @@ define <4 x i32> @testv4i32(<4 x i32> %in) nounwind { ; ; X32-SSE-LABEL: testv4i32: ; X32-SSE: # BB#0: -; X32-SSE-NEXT: pextrd $1, %xmm0, %eax -; X32-SSE-NEXT: bsrl %eax, %eax -; X32-SSE-NEXT: movl $63, %ecx -; X32-SSE-NEXT: cmovel %ecx, %eax -; X32-SSE-NEXT: xorl $31, %eax -; X32-SSE-NEXT: movd %xmm0, %edx -; X32-SSE-NEXT: bsrl %edx, %edx -; X32-SSE-NEXT: cmovel %ecx, %edx -; X32-SSE-NEXT: xorl $31, %edx -; X32-SSE-NEXT: movd %edx, %xmm1 -; X32-SSE-NEXT: pinsrd $1, %eax, %xmm1 -; X32-SSE-NEXT: pextrd $2, %xmm0, %eax -; X32-SSE-NEXT: bsrl %eax, %eax -; X32-SSE-NEXT: cmovel %ecx, %eax -; X32-SSE-NEXT: xorl $31, %eax -; X32-SSE-NEXT: pinsrd $2, %eax, %xmm1 -; X32-SSE-NEXT: pextrd $3, %xmm0, %eax -; X32-SSE-NEXT: bsrl %eax, %eax -; X32-SSE-NEXT: cmovel %ecx, %eax -; X32-SSE-NEXT: xorl $31, %eax -; X32-SSE-NEXT: pinsrd $3, %eax, %xmm1 +; X32-SSE-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; X32-SSE-NEXT: movdqa %xmm0, %xmm1 +; X32-SSE-NEXT: pand %xmm2, %xmm1 +; X32-SSE-NEXT: movdqa {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0] +; X32-SSE-NEXT: movdqa %xmm3, %xmm4 +; X32-SSE-NEXT: pshufb %xmm1, %xmm4 +; X32-SSE-NEXT: movdqa %xmm0, %xmm1 +; X32-SSE-NEXT: psrlw $4, %xmm1 +; X32-SSE-NEXT: pand %xmm2, %xmm1 +; X32-SSE-NEXT: pxor %xmm2, %xmm2 +; X32-SSE-NEXT: pshufb %xmm1, %xmm3 +; X32-SSE-NEXT: pcmpeqb %xmm2, %xmm1 +; X32-SSE-NEXT: pand %xmm4, %xmm1 +; X32-SSE-NEXT: paddb %xmm3, %xmm1 +; X32-SSE-NEXT: movdqa %xmm0, %xmm3 +; X32-SSE-NEXT: pcmpeqb %xmm2, %xmm3 +; X32-SSE-NEXT: psrlw $8, %xmm3 +; X32-SSE-NEXT: pand %xmm1, %xmm3 +; X32-SSE-NEXT: psrlw $8, %xmm1 +; X32-SSE-NEXT: paddw %xmm3, %xmm1 +; X32-SSE-NEXT: pcmpeqw %xmm2, %xmm0 +; X32-SSE-NEXT: psrld $16, %xmm0 +; X32-SSE-NEXT: pand %xmm1, %xmm0 +; X32-SSE-NEXT: psrld $16, %xmm1 +; X32-SSE-NEXT: paddd %xmm0, %xmm1 ; X32-SSE-NEXT: movdqa %xmm1, %xmm0 ; X32-SSE-NEXT: retl @@ -504,70 +640,87 @@ define <4 x i32> @testv4i32u(<4 x i32> %in) nounwind { ; ; SSSE3-LABEL: testv4i32u: ; SSSE3: # BB#0: -; SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,1,2,3] -; SSSE3-NEXT: movd %xmm1, %eax -; SSSE3-NEXT: bsrl %eax, %eax -; SSSE3-NEXT: xorl $31, %eax -; SSSE3-NEXT: movd %eax, %xmm1 -; SSSE3-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3] -; SSSE3-NEXT: movd %xmm2, %eax -; SSSE3-NEXT: bsrl %eax, %eax -; SSSE3-NEXT: xorl $31, %eax -; SSSE3-NEXT: movd %eax, %xmm2 -; SSSE3-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] -; SSSE3-NEXT: movd %xmm0, %eax -; SSSE3-NEXT: bsrl %eax, %eax -; SSSE3-NEXT: xorl $31, %eax -; SSSE3-NEXT: movd %eax, %xmm1 -; SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] -; SSSE3-NEXT: movd %xmm0, %eax -; SSSE3-NEXT: bsrl %eax, %eax -; SSSE3-NEXT: xorl $31, %eax -; SSSE3-NEXT: movd %eax, %xmm0 -; SSSE3-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] -; SSSE3-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] +; SSSE3-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; SSSE3-NEXT: movdqa %xmm0, %xmm1 +; SSSE3-NEXT: pand %xmm2, %xmm1 +; SSSE3-NEXT: movdqa {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0] +; SSSE3-NEXT: movdqa %xmm3, %xmm4 +; SSSE3-NEXT: pshufb %xmm1, %xmm4 +; SSSE3-NEXT: movdqa %xmm0, %xmm1 +; SSSE3-NEXT: psrlw $4, %xmm1 +; SSSE3-NEXT: pand %xmm2, %xmm1 +; SSSE3-NEXT: pxor %xmm2, %xmm2 +; SSSE3-NEXT: pshufb %xmm1, %xmm3 +; SSSE3-NEXT: pcmpeqb %xmm2, %xmm1 +; SSSE3-NEXT: pand %xmm4, %xmm1 +; SSSE3-NEXT: paddb %xmm3, %xmm1 +; SSSE3-NEXT: movdqa %xmm0, %xmm3 +; SSSE3-NEXT: pcmpeqb %xmm2, %xmm3 +; SSSE3-NEXT: psrlw $8, %xmm3 +; SSSE3-NEXT: pand %xmm1, %xmm3 +; SSSE3-NEXT: psrlw $8, %xmm1 +; SSSE3-NEXT: paddw %xmm3, %xmm1 +; SSSE3-NEXT: pcmpeqw %xmm2, %xmm0 +; SSSE3-NEXT: psrld $16, %xmm0 +; SSSE3-NEXT: pand %xmm1, %xmm0 +; SSSE3-NEXT: psrld $16, %xmm1 +; SSSE3-NEXT: paddd %xmm0, %xmm1 ; SSSE3-NEXT: movdqa %xmm1, %xmm0 ; SSSE3-NEXT: retq ; ; SSE41-LABEL: testv4i32u: ; SSE41: # BB#0: -; SSE41-NEXT: pextrd $1, %xmm0, %eax -; SSE41-NEXT: bsrl %eax, %eax -; SSE41-NEXT: xorl $31, %eax -; SSE41-NEXT: movd %xmm0, %ecx -; SSE41-NEXT: bsrl %ecx, %ecx -; SSE41-NEXT: xorl $31, %ecx -; SSE41-NEXT: movd %ecx, %xmm1 -; SSE41-NEXT: pinsrd $1, %eax, %xmm1 -; SSE41-NEXT: pextrd $2, %xmm0, %eax -; SSE41-NEXT: bsrl %eax, %eax -; SSE41-NEXT: xorl $31, %eax -; SSE41-NEXT: pinsrd $2, %eax, %xmm1 -; SSE41-NEXT: pextrd $3, %xmm0, %eax -; SSE41-NEXT: bsrl %eax, %eax -; SSE41-NEXT: xorl $31, %eax -; SSE41-NEXT: pinsrd $3, %eax, %xmm1 +; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; SSE41-NEXT: movdqa %xmm0, %xmm1 +; SSE41-NEXT: pand %xmm2, %xmm1 +; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0] +; SSE41-NEXT: movdqa %xmm3, %xmm4 +; SSE41-NEXT: pshufb %xmm1, %xmm4 +; SSE41-NEXT: movdqa %xmm0, %xmm1 +; SSE41-NEXT: psrlw $4, %xmm1 +; SSE41-NEXT: pand %xmm2, %xmm1 +; SSE41-NEXT: pxor %xmm2, %xmm2 +; SSE41-NEXT: pshufb %xmm1, %xmm3 +; SSE41-NEXT: pcmpeqb %xmm2, %xmm1 +; SSE41-NEXT: pand %xmm4, %xmm1 +; SSE41-NEXT: paddb %xmm3, %xmm1 +; SSE41-NEXT: movdqa %xmm0, %xmm3 +; SSE41-NEXT: pcmpeqb %xmm2, %xmm3 +; SSE41-NEXT: psrlw $8, %xmm3 +; SSE41-NEXT: pand %xmm1, %xmm3 +; SSE41-NEXT: psrlw $8, %xmm1 +; SSE41-NEXT: paddw %xmm3, %xmm1 +; SSE41-NEXT: pcmpeqw %xmm2, %xmm0 +; SSE41-NEXT: psrld $16, %xmm0 +; SSE41-NEXT: pand %xmm1, %xmm0 +; SSE41-NEXT: psrld $16, %xmm1 +; SSE41-NEXT: paddd %xmm0, %xmm1 ; SSE41-NEXT: movdqa %xmm1, %xmm0 ; SSE41-NEXT: retq ; ; AVX-LABEL: testv4i32u: ; AVX: # BB#0: -; AVX-NEXT: vpextrd $1, %xmm0, %eax -; AVX-NEXT: bsrl %eax, %eax -; AVX-NEXT: xorl $31, %eax -; AVX-NEXT: vmovd %xmm0, %ecx -; AVX-NEXT: bsrl %ecx, %ecx -; AVX-NEXT: xorl $31, %ecx -; AVX-NEXT: vmovd %ecx, %xmm1 -; AVX-NEXT: vpinsrd $1, %eax, %xmm1, %xmm1 -; AVX-NEXT: vpextrd $2, %xmm0, %eax -; AVX-NEXT: bsrl %eax, %eax -; AVX-NEXT: xorl $31, %eax -; AVX-NEXT: vpinsrd $2, %eax, %xmm1, %xmm1 -; AVX-NEXT: vpextrd $3, %xmm0, %eax -; AVX-NEXT: bsrl %eax, %eax -; AVX-NEXT: xorl $31, %eax -; AVX-NEXT: vpinsrd $3, %eax, %xmm1, %xmm0 +; AVX-NEXT: vmovdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX-NEXT: vpand %xmm1, %xmm0, %xmm2 +; AVX-NEXT: vmovdqa {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0] +; AVX-NEXT: vpshufb %xmm2, %xmm3, %xmm2 +; AVX-NEXT: vpsrlw $4, %xmm0, %xmm4 +; AVX-NEXT: vpand %xmm1, %xmm4, %xmm1 +; AVX-NEXT: vpxor %xmm4, %xmm4, %xmm4 +; AVX-NEXT: vpcmpeqb %xmm4, %xmm1, %xmm5 +; AVX-NEXT: vpand %xmm5, %xmm2, %xmm2 +; AVX-NEXT: vpshufb %xmm1, %xmm3, %xmm1 +; AVX-NEXT: vpaddb %xmm1, %xmm2, %xmm1 +; AVX-NEXT: vpcmpeqb %xmm4, %xmm0, %xmm2 +; AVX-NEXT: vpsrlw $8, %xmm2, %xmm2 +; AVX-NEXT: vpand %xmm2, %xmm1, %xmm2 +; AVX-NEXT: vpsrlw $8, %xmm1, %xmm1 +; AVX-NEXT: vpaddw %xmm2, %xmm1, %xmm1 +; AVX-NEXT: vpcmpeqw %xmm4, %xmm0, %xmm0 +; AVX-NEXT: vpsrld $16, %xmm0, %xmm0 +; AVX-NEXT: vpand %xmm0, %xmm1, %xmm0 +; AVX-NEXT: vpsrld $16, %xmm1, %xmm1 +; AVX-NEXT: vpaddd %xmm0, %xmm1, %xmm0 ; AVX-NEXT: retq ; ; AVX512VLCD-LABEL: testv4i32u: @@ -584,22 +737,31 @@ define <4 x i32> @testv4i32u(<4 x i32> %in) nounwind { ; ; X32-SSE-LABEL: testv4i32u: ; X32-SSE: # BB#0: -; X32-SSE-NEXT: pextrd $1, %xmm0, %eax -; X32-SSE-NEXT: bsrl %eax, %eax -; X32-SSE-NEXT: xorl $31, %eax -; X32-SSE-NEXT: movd %xmm0, %ecx -; X32-SSE-NEXT: bsrl %ecx, %ecx -; X32-SSE-NEXT: xorl $31, %ecx -; X32-SSE-NEXT: movd %ecx, %xmm1 -; X32-SSE-NEXT: pinsrd $1, %eax, %xmm1 -; X32-SSE-NEXT: pextrd $2, %xmm0, %eax -; X32-SSE-NEXT: bsrl %eax, %eax -; X32-SSE-NEXT: xorl $31, %eax -; X32-SSE-NEXT: pinsrd $2, %eax, %xmm1 -; X32-SSE-NEXT: pextrd $3, %xmm0, %eax -; X32-SSE-NEXT: bsrl %eax, %eax -; X32-SSE-NEXT: xorl $31, %eax -; X32-SSE-NEXT: pinsrd $3, %eax, %xmm1 +; X32-SSE-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; X32-SSE-NEXT: movdqa %xmm0, %xmm1 +; X32-SSE-NEXT: pand %xmm2, %xmm1 +; X32-SSE-NEXT: movdqa {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0] +; X32-SSE-NEXT: movdqa %xmm3, %xmm4 +; X32-SSE-NEXT: pshufb %xmm1, %xmm4 +; X32-SSE-NEXT: movdqa %xmm0, %xmm1 +; X32-SSE-NEXT: psrlw $4, %xmm1 +; X32-SSE-NEXT: pand %xmm2, %xmm1 +; X32-SSE-NEXT: pxor %xmm2, %xmm2 +; X32-SSE-NEXT: pshufb %xmm1, %xmm3 +; X32-SSE-NEXT: pcmpeqb %xmm2, %xmm1 +; X32-SSE-NEXT: pand %xmm4, %xmm1 +; X32-SSE-NEXT: paddb %xmm3, %xmm1 +; X32-SSE-NEXT: movdqa %xmm0, %xmm3 +; X32-SSE-NEXT: pcmpeqb %xmm2, %xmm3 +; X32-SSE-NEXT: psrlw $8, %xmm3 +; X32-SSE-NEXT: pand %xmm1, %xmm3 +; X32-SSE-NEXT: psrlw $8, %xmm1 +; X32-SSE-NEXT: paddw %xmm3, %xmm1 +; X32-SSE-NEXT: pcmpeqw %xmm2, %xmm0 +; X32-SSE-NEXT: psrld $16, %xmm0 +; X32-SSE-NEXT: pand %xmm1, %xmm0 +; X32-SSE-NEXT: psrld $16, %xmm1 +; X32-SSE-NEXT: paddd %xmm0, %xmm1 ; X32-SSE-NEXT: movdqa %xmm1, %xmm0 ; X32-SSE-NEXT: retl @@ -1586,8 +1748,37 @@ define <2 x i64> @foldv2i64() nounwind { ; ; X32-SSE-LABEL: foldv2i64: ; X32-SSE: # BB#0: -; X32-SSE-NEXT: movl $55, %eax -; X32-SSE-NEXT: movd %eax, %xmm0 +; X32-SSE-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; X32-SSE-NEXT: movdqa {{.*#+}} xmm1 = [256,0,4294967295,4294967295] +; X32-SSE-NEXT: movdqa %xmm1, %xmm0 +; X32-SSE-NEXT: pand %xmm2, %xmm0 +; X32-SSE-NEXT: movdqa {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0] +; X32-SSE-NEXT: movdqa %xmm3, %xmm4 +; X32-SSE-NEXT: pshufb %xmm0, %xmm4 +; X32-SSE-NEXT: movdqa %xmm1, %xmm0 +; X32-SSE-NEXT: psrlw $4, %xmm0 +; X32-SSE-NEXT: pand %xmm2, %xmm0 +; X32-SSE-NEXT: pxor %xmm2, %xmm2 +; X32-SSE-NEXT: pshufb %xmm0, %xmm3 +; X32-SSE-NEXT: pcmpeqb %xmm2, %xmm0 +; X32-SSE-NEXT: pand %xmm4, %xmm0 +; X32-SSE-NEXT: paddb %xmm3, %xmm0 +; X32-SSE-NEXT: movdqa %xmm1, %xmm3 +; X32-SSE-NEXT: pcmpeqb %xmm2, %xmm3 +; X32-SSE-NEXT: psrlw $8, %xmm3 +; X32-SSE-NEXT: pand %xmm0, %xmm3 +; X32-SSE-NEXT: psrlw $8, %xmm0 +; X32-SSE-NEXT: paddw %xmm3, %xmm0 +; X32-SSE-NEXT: pcmpeqw %xmm2, %xmm1 +; X32-SSE-NEXT: psrld $16, %xmm1 +; X32-SSE-NEXT: pand %xmm0, %xmm1 +; X32-SSE-NEXT: psrld $16, %xmm0 +; X32-SSE-NEXT: paddd %xmm1, %xmm0 +; X32-SSE-NEXT: movdqa {{.*#+}} xmm1 = [0,4294967295,0,0] +; X32-SSE-NEXT: psrlq $32, %xmm1 +; X32-SSE-NEXT: pand %xmm0, %xmm1 +; X32-SSE-NEXT: psrlq $32, %xmm0 +; X32-SSE-NEXT: paddq %xmm1, %xmm0 ; X32-SSE-NEXT: retl %out = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> , i1 0) ret <2 x i64> %out @@ -1614,8 +1805,37 @@ define <2 x i64> @foldv2i64u() nounwind { ; ; X32-SSE-LABEL: foldv2i64u: ; X32-SSE: # BB#0: -; X32-SSE-NEXT: movl $55, %eax -; X32-SSE-NEXT: movd %eax, %xmm0 +; X32-SSE-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; X32-SSE-NEXT: movdqa {{.*#+}} xmm1 = [256,0,4294967295,4294967295] +; X32-SSE-NEXT: movdqa %xmm1, %xmm0 +; X32-SSE-NEXT: pand %xmm2, %xmm0 +; X32-SSE-NEXT: movdqa {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0] +; X32-SSE-NEXT: movdqa %xmm3, %xmm4 +; X32-SSE-NEXT: pshufb %xmm0, %xmm4 +; X32-SSE-NEXT: movdqa %xmm1, %xmm0 +; X32-SSE-NEXT: psrlw $4, %xmm0 +; X32-SSE-NEXT: pand %xmm2, %xmm0 +; X32-SSE-NEXT: pxor %xmm2, %xmm2 +; X32-SSE-NEXT: pshufb %xmm0, %xmm3 +; X32-SSE-NEXT: pcmpeqb %xmm2, %xmm0 +; X32-SSE-NEXT: pand %xmm4, %xmm0 +; X32-SSE-NEXT: paddb %xmm3, %xmm0 +; X32-SSE-NEXT: movdqa %xmm1, %xmm3 +; X32-SSE-NEXT: pcmpeqb %xmm2, %xmm3 +; X32-SSE-NEXT: psrlw $8, %xmm3 +; X32-SSE-NEXT: pand %xmm0, %xmm3 +; X32-SSE-NEXT: psrlw $8, %xmm0 +; X32-SSE-NEXT: paddw %xmm3, %xmm0 +; X32-SSE-NEXT: pcmpeqw %xmm2, %xmm1 +; X32-SSE-NEXT: psrld $16, %xmm1 +; X32-SSE-NEXT: pand %xmm0, %xmm1 +; X32-SSE-NEXT: psrld $16, %xmm0 +; X32-SSE-NEXT: paddd %xmm1, %xmm0 +; X32-SSE-NEXT: movdqa {{.*#+}} xmm1 = [0,4294967295,0,0] +; X32-SSE-NEXT: psrlq $32, %xmm1 +; X32-SSE-NEXT: pand %xmm0, %xmm1 +; X32-SSE-NEXT: psrlq $32, %xmm0 +; X32-SSE-NEXT: paddq %xmm1, %xmm0 ; X32-SSE-NEXT: retl %out = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> , i1 -1) ret <2 x i64> %out diff --git a/llvm/test/CodeGen/X86/vector-lzcnt-256.ll b/llvm/test/CodeGen/X86/vector-lzcnt-256.ll index e73dbb6..a145621 100644 --- a/llvm/test/CodeGen/X86/vector-lzcnt-256.ll +++ b/llvm/test/CodeGen/X86/vector-lzcnt-256.ll @@ -7,30 +7,57 @@ define <4 x i64> @testv4i64(<4 x i64> %in) nounwind { ; AVX1-LABEL: testv4i64: ; AVX1: # BB#0: -; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 -; AVX1-NEXT: vpextrq $1, %xmm1, %rax -; AVX1-NEXT: bsrq %rax, %rax -; AVX1-NEXT: movl $127, %ecx -; AVX1-NEXT: cmoveq %rcx, %rax -; AVX1-NEXT: vmovq %rax, %xmm2 -; AVX1-NEXT: vmovq %xmm1, %rax -; AVX1-NEXT: bsrq %rax, %rax -; AVX1-NEXT: cmoveq %rcx, %rax -; AVX1-NEXT: vmovq %rax, %xmm1 -; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0] -; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [63,63] -; AVX1-NEXT: vpxor %xmm2, %xmm1, %xmm1 -; AVX1-NEXT: vpextrq $1, %xmm0, %rax -; AVX1-NEXT: bsrq %rax, %rax -; AVX1-NEXT: cmoveq %rcx, %rax -; AVX1-NEXT: vmovq %rax, %xmm3 -; AVX1-NEXT: vmovq %xmm0, %rax -; AVX1-NEXT: bsrq %rax, %rax -; AVX1-NEXT: cmoveq %rcx, %rax -; AVX1-NEXT: vmovq %rax, %xmm0 -; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0] -; AVX1-NEXT: vpxor %xmm2, %xmm0, %xmm0 -; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; AVX1-NEXT: vmovaps {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX1-NEXT: vandps %xmm3, %xmm2, %xmm1 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0] +; AVX1-NEXT: vpshufb %xmm1, %xmm4, %xmm5 +; AVX1-NEXT: vpsrlw $4, %xmm2, %xmm1 +; AVX1-NEXT: vpand %xmm3, %xmm1, %xmm6 +; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpcmpeqb %xmm1, %xmm6, %xmm7 +; AVX1-NEXT: vpand %xmm7, %xmm5, %xmm5 +; AVX1-NEXT: vpshufb %xmm6, %xmm4, %xmm6 +; AVX1-NEXT: vpaddb %xmm6, %xmm5, %xmm5 +; AVX1-NEXT: vpcmpeqb %xmm1, %xmm2, %xmm6 +; AVX1-NEXT: vpsrlw $8, %xmm6, %xmm6 +; AVX1-NEXT: vpand %xmm6, %xmm5, %xmm6 +; AVX1-NEXT: vpsrlw $8, %xmm5, %xmm5 +; AVX1-NEXT: vpaddw %xmm6, %xmm5, %xmm5 +; AVX1-NEXT: vpcmpeqw %xmm1, %xmm2, %xmm6 +; AVX1-NEXT: vpsrld $16, %xmm6, %xmm6 +; AVX1-NEXT: vpand %xmm6, %xmm5, %xmm6 +; AVX1-NEXT: vpsrld $16, %xmm5, %xmm5 +; AVX1-NEXT: vpaddd %xmm6, %xmm5, %xmm5 +; AVX1-NEXT: vpcmpeqd %xmm1, %xmm2, %xmm2 +; AVX1-NEXT: vpsrlq $32, %xmm2, %xmm2 +; AVX1-NEXT: vpand %xmm2, %xmm5, %xmm2 +; AVX1-NEXT: vpsrlq $32, %xmm5, %xmm5 +; AVX1-NEXT: vpaddq %xmm2, %xmm5, %xmm2 +; AVX1-NEXT: vandps %xmm3, %xmm0, %xmm5 +; AVX1-NEXT: vpshufb %xmm5, %xmm4, %xmm5 +; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm6 +; AVX1-NEXT: vpand %xmm3, %xmm6, %xmm3 +; AVX1-NEXT: vpcmpeqb %xmm1, %xmm3, %xmm6 +; AVX1-NEXT: vpand %xmm6, %xmm5, %xmm5 +; AVX1-NEXT: vpshufb %xmm3, %xmm4, %xmm3 +; AVX1-NEXT: vpaddb %xmm3, %xmm5, %xmm3 +; AVX1-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm4 +; AVX1-NEXT: vpsrlw $8, %xmm4, %xmm4 +; AVX1-NEXT: vpand %xmm4, %xmm3, %xmm4 +; AVX1-NEXT: vpsrlw $8, %xmm3, %xmm3 +; AVX1-NEXT: vpaddw %xmm4, %xmm3, %xmm3 +; AVX1-NEXT: vpcmpeqw %xmm1, %xmm0, %xmm4 +; AVX1-NEXT: vpsrld $16, %xmm4, %xmm4 +; AVX1-NEXT: vpand %xmm4, %xmm3, %xmm4 +; AVX1-NEXT: vpsrld $16, %xmm3, %xmm3 +; AVX1-NEXT: vpaddd %xmm4, %xmm3, %xmm3 +; AVX1-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpsrlq $32, %xmm0, %xmm0 +; AVX1-NEXT: vpand %xmm0, %xmm3, %xmm0 +; AVX1-NEXT: vpsrlq $32, %xmm3, %xmm1 +; AVX1-NEXT: vpaddq %xmm0, %xmm1, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 ; AVX1-NEXT: retq ; ; AVX2-LABEL: testv4i64: @@ -82,25 +109,57 @@ define <4 x i64> @testv4i64(<4 x i64> %in) nounwind { define <4 x i64> @testv4i64u(<4 x i64> %in) nounwind { ; AVX1-LABEL: testv4i64u: ; AVX1: # BB#0: -; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 -; AVX1-NEXT: vpextrq $1, %xmm1, %rax -; AVX1-NEXT: bsrq %rax, %rax -; AVX1-NEXT: vmovq %rax, %xmm2 -; AVX1-NEXT: vmovq %xmm1, %rax -; AVX1-NEXT: bsrq %rax, %rax -; AVX1-NEXT: vmovq %rax, %xmm1 -; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0] -; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [63,63] -; AVX1-NEXT: vpxor %xmm2, %xmm1, %xmm1 -; AVX1-NEXT: vpextrq $1, %xmm0, %rax -; AVX1-NEXT: bsrq %rax, %rax -; AVX1-NEXT: vmovq %rax, %xmm3 -; AVX1-NEXT: vmovq %xmm0, %rax -; AVX1-NEXT: bsrq %rax, %rax -; AVX1-NEXT: vmovq %rax, %xmm0 -; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0] -; AVX1-NEXT: vpxor %xmm2, %xmm0, %xmm0 -; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; AVX1-NEXT: vmovaps {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX1-NEXT: vandps %xmm3, %xmm2, %xmm1 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0] +; AVX1-NEXT: vpshufb %xmm1, %xmm4, %xmm5 +; AVX1-NEXT: vpsrlw $4, %xmm2, %xmm1 +; AVX1-NEXT: vpand %xmm3, %xmm1, %xmm6 +; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpcmpeqb %xmm1, %xmm6, %xmm7 +; AVX1-NEXT: vpand %xmm7, %xmm5, %xmm5 +; AVX1-NEXT: vpshufb %xmm6, %xmm4, %xmm6 +; AVX1-NEXT: vpaddb %xmm6, %xmm5, %xmm5 +; AVX1-NEXT: vpcmpeqb %xmm1, %xmm2, %xmm6 +; AVX1-NEXT: vpsrlw $8, %xmm6, %xmm6 +; AVX1-NEXT: vpand %xmm6, %xmm5, %xmm6 +; AVX1-NEXT: vpsrlw $8, %xmm5, %xmm5 +; AVX1-NEXT: vpaddw %xmm6, %xmm5, %xmm5 +; AVX1-NEXT: vpcmpeqw %xmm1, %xmm2, %xmm6 +; AVX1-NEXT: vpsrld $16, %xmm6, %xmm6 +; AVX1-NEXT: vpand %xmm6, %xmm5, %xmm6 +; AVX1-NEXT: vpsrld $16, %xmm5, %xmm5 +; AVX1-NEXT: vpaddd %xmm6, %xmm5, %xmm5 +; AVX1-NEXT: vpcmpeqd %xmm1, %xmm2, %xmm2 +; AVX1-NEXT: vpsrlq $32, %xmm2, %xmm2 +; AVX1-NEXT: vpand %xmm2, %xmm5, %xmm2 +; AVX1-NEXT: vpsrlq $32, %xmm5, %xmm5 +; AVX1-NEXT: vpaddq %xmm2, %xmm5, %xmm2 +; AVX1-NEXT: vandps %xmm3, %xmm0, %xmm5 +; AVX1-NEXT: vpshufb %xmm5, %xmm4, %xmm5 +; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm6 +; AVX1-NEXT: vpand %xmm3, %xmm6, %xmm3 +; AVX1-NEXT: vpcmpeqb %xmm1, %xmm3, %xmm6 +; AVX1-NEXT: vpand %xmm6, %xmm5, %xmm5 +; AVX1-NEXT: vpshufb %xmm3, %xmm4, %xmm3 +; AVX1-NEXT: vpaddb %xmm3, %xmm5, %xmm3 +; AVX1-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm4 +; AVX1-NEXT: vpsrlw $8, %xmm4, %xmm4 +; AVX1-NEXT: vpand %xmm4, %xmm3, %xmm4 +; AVX1-NEXT: vpsrlw $8, %xmm3, %xmm3 +; AVX1-NEXT: vpaddw %xmm4, %xmm3, %xmm3 +; AVX1-NEXT: vpcmpeqw %xmm1, %xmm0, %xmm4 +; AVX1-NEXT: vpsrld $16, %xmm4, %xmm4 +; AVX1-NEXT: vpand %xmm4, %xmm3, %xmm4 +; AVX1-NEXT: vpsrld $16, %xmm3, %xmm3 +; AVX1-NEXT: vpaddd %xmm4, %xmm3, %xmm3 +; AVX1-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpsrlq $32, %xmm0, %xmm0 +; AVX1-NEXT: vpand %xmm0, %xmm3, %xmm0 +; AVX1-NEXT: vpsrlq $32, %xmm3, %xmm1 +; AVX1-NEXT: vpaddq %xmm0, %xmm1, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 ; AVX1-NEXT: retq ; ; AVX2-LABEL: testv4i64u: @@ -153,42 +212,45 @@ define <8 x i32> @testv8i32(<8 x i32> %in) nounwind { ; AVX1-LABEL: testv8i32: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 -; AVX1-NEXT: vpextrd $1, %xmm1, %eax -; AVX1-NEXT: bsrl %eax, %ecx -; AVX1-NEXT: movl $63, %eax -; AVX1-NEXT: cmovel %eax, %ecx -; AVX1-NEXT: vmovd %xmm1, %edx -; AVX1-NEXT: bsrl %edx, %edx -; AVX1-NEXT: cmovel %eax, %edx -; AVX1-NEXT: vmovd %edx, %xmm2 -; AVX1-NEXT: vpinsrd $1, %ecx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrd $2, %xmm1, %ecx -; AVX1-NEXT: bsrl %ecx, %ecx -; AVX1-NEXT: cmovel %eax, %ecx -; AVX1-NEXT: vpinsrd $2, %ecx, %xmm2, %xmm2 -; AVX1-NEXT: vpextrd $3, %xmm1, %ecx -; AVX1-NEXT: bsrl %ecx, %ecx -; AVX1-NEXT: cmovel %eax, %ecx -; AVX1-NEXT: vpinsrd $3, %ecx, %xmm2, %xmm1 -; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [31,31,31,31] -; AVX1-NEXT: vpxor %xmm2, %xmm1, %xmm1 -; AVX1-NEXT: vpextrd $1, %xmm0, %ecx -; AVX1-NEXT: bsrl %ecx, %ecx -; AVX1-NEXT: cmovel %eax, %ecx -; AVX1-NEXT: vmovd %xmm0, %edx -; AVX1-NEXT: bsrl %edx, %edx -; AVX1-NEXT: cmovel %eax, %edx -; AVX1-NEXT: vmovd %edx, %xmm3 -; AVX1-NEXT: vpinsrd $1, %ecx, %xmm3, %xmm3 -; AVX1-NEXT: vpextrd $2, %xmm0, %ecx -; AVX1-NEXT: bsrl %ecx, %ecx -; AVX1-NEXT: cmovel %eax, %ecx -; AVX1-NEXT: vpinsrd $2, %ecx, %xmm3, %xmm3 -; AVX1-NEXT: vpextrd $3, %xmm0, %ecx -; AVX1-NEXT: bsrl %ecx, %ecx -; AVX1-NEXT: cmovel %eax, %ecx -; AVX1-NEXT: vpinsrd $3, %ecx, %xmm3, %xmm0 -; AVX1-NEXT: vpxor %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vmovaps {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX1-NEXT: vandps %xmm2, %xmm1, %xmm3 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0] +; AVX1-NEXT: vpshufb %xmm3, %xmm4, %xmm3 +; AVX1-NEXT: vpsrlw $4, %xmm1, %xmm5 +; AVX1-NEXT: vpand %xmm2, %xmm5, %xmm5 +; AVX1-NEXT: vpxor %xmm6, %xmm6, %xmm6 +; AVX1-NEXT: vpcmpeqb %xmm6, %xmm5, %xmm7 +; AVX1-NEXT: vpand %xmm7, %xmm3, %xmm3 +; AVX1-NEXT: vpshufb %xmm5, %xmm4, %xmm5 +; AVX1-NEXT: vpaddb %xmm5, %xmm3, %xmm3 +; AVX1-NEXT: vpcmpeqb %xmm6, %xmm1, %xmm5 +; AVX1-NEXT: vpsrlw $8, %xmm5, %xmm5 +; AVX1-NEXT: vpand %xmm5, %xmm3, %xmm5 +; AVX1-NEXT: vpsrlw $8, %xmm3, %xmm3 +; AVX1-NEXT: vpaddw %xmm5, %xmm3, %xmm3 +; AVX1-NEXT: vpcmpeqw %xmm6, %xmm1, %xmm1 +; AVX1-NEXT: vpsrld $16, %xmm1, %xmm1 +; AVX1-NEXT: vpand %xmm1, %xmm3, %xmm1 +; AVX1-NEXT: vpsrld $16, %xmm3, %xmm3 +; AVX1-NEXT: vpaddd %xmm1, %xmm3, %xmm1 +; AVX1-NEXT: vandps %xmm2, %xmm0, %xmm3 +; AVX1-NEXT: vpshufb %xmm3, %xmm4, %xmm3 +; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm5 +; AVX1-NEXT: vpand %xmm2, %xmm5, %xmm2 +; AVX1-NEXT: vpcmpeqb %xmm6, %xmm2, %xmm5 +; AVX1-NEXT: vpand %xmm5, %xmm3, %xmm3 +; AVX1-NEXT: vpshufb %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpaddb %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpcmpeqb %xmm6, %xmm0, %xmm3 +; AVX1-NEXT: vpsrlw $8, %xmm3, %xmm3 +; AVX1-NEXT: vpand %xmm3, %xmm2, %xmm3 +; AVX1-NEXT: vpsrlw $8, %xmm2, %xmm2 +; AVX1-NEXT: vpaddw %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vpcmpeqw %xmm6, %xmm0, %xmm0 +; AVX1-NEXT: vpsrld $16, %xmm0, %xmm0 +; AVX1-NEXT: vpand %xmm0, %xmm2, %xmm0 +; AVX1-NEXT: vpsrld $16, %xmm2, %xmm2 +; AVX1-NEXT: vpaddd %xmm0, %xmm2, %xmm0 ; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 ; AVX1-NEXT: retq ; @@ -237,33 +299,45 @@ define <8 x i32> @testv8i32u(<8 x i32> %in) nounwind { ; AVX1-LABEL: testv8i32u: ; AVX1: # BB#0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 -; AVX1-NEXT: vpextrd $1, %xmm1, %eax -; AVX1-NEXT: bsrl %eax, %eax -; AVX1-NEXT: vmovd %xmm1, %ecx -; AVX1-NEXT: bsrl %ecx, %ecx -; AVX1-NEXT: vmovd %ecx, %xmm2 -; AVX1-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrd $2, %xmm1, %eax -; AVX1-NEXT: bsrl %eax, %eax -; AVX1-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2 -; AVX1-NEXT: vpextrd $3, %xmm1, %eax -; AVX1-NEXT: bsrl %eax, %eax -; AVX1-NEXT: vpinsrd $3, %eax, %xmm2, %xmm1 -; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [31,31,31,31] -; AVX1-NEXT: vpxor %xmm2, %xmm1, %xmm1 -; AVX1-NEXT: vpextrd $1, %xmm0, %eax -; AVX1-NEXT: bsrl %eax, %eax -; AVX1-NEXT: vmovd %xmm0, %ecx -; AVX1-NEXT: bsrl %ecx, %ecx -; AVX1-NEXT: vmovd %ecx, %xmm3 -; AVX1-NEXT: vpinsrd $1, %eax, %xmm3, %xmm3 -; AVX1-NEXT: vpextrd $2, %xmm0, %eax -; AVX1-NEXT: bsrl %eax, %eax -; AVX1-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3 -; AVX1-NEXT: vpextrd $3, %xmm0, %eax -; AVX1-NEXT: bsrl %eax, %eax -; AVX1-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0 -; AVX1-NEXT: vpxor %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vmovaps {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX1-NEXT: vandps %xmm2, %xmm1, %xmm3 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0] +; AVX1-NEXT: vpshufb %xmm3, %xmm4, %xmm3 +; AVX1-NEXT: vpsrlw $4, %xmm1, %xmm5 +; AVX1-NEXT: vpand %xmm2, %xmm5, %xmm5 +; AVX1-NEXT: vpxor %xmm6, %xmm6, %xmm6 +; AVX1-NEXT: vpcmpeqb %xmm6, %xmm5, %xmm7 +; AVX1-NEXT: vpand %xmm7, %xmm3, %xmm3 +; AVX1-NEXT: vpshufb %xmm5, %xmm4, %xmm5 +; AVX1-NEXT: vpaddb %xmm5, %xmm3, %xmm3 +; AVX1-NEXT: vpcmpeqb %xmm6, %xmm1, %xmm5 +; AVX1-NEXT: vpsrlw $8, %xmm5, %xmm5 +; AVX1-NEXT: vpand %xmm5, %xmm3, %xmm5 +; AVX1-NEXT: vpsrlw $8, %xmm3, %xmm3 +; AVX1-NEXT: vpaddw %xmm5, %xmm3, %xmm3 +; AVX1-NEXT: vpcmpeqw %xmm6, %xmm1, %xmm1 +; AVX1-NEXT: vpsrld $16, %xmm1, %xmm1 +; AVX1-NEXT: vpand %xmm1, %xmm3, %xmm1 +; AVX1-NEXT: vpsrld $16, %xmm3, %xmm3 +; AVX1-NEXT: vpaddd %xmm1, %xmm3, %xmm1 +; AVX1-NEXT: vandps %xmm2, %xmm0, %xmm3 +; AVX1-NEXT: vpshufb %xmm3, %xmm4, %xmm3 +; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm5 +; AVX1-NEXT: vpand %xmm2, %xmm5, %xmm2 +; AVX1-NEXT: vpcmpeqb %xmm6, %xmm2, %xmm5 +; AVX1-NEXT: vpand %xmm5, %xmm3, %xmm3 +; AVX1-NEXT: vpshufb %xmm2, %xmm4, %xmm2 +; AVX1-NEXT: vpaddb %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vpcmpeqb %xmm6, %xmm0, %xmm3 +; AVX1-NEXT: vpsrlw $8, %xmm3, %xmm3 +; AVX1-NEXT: vpand %xmm3, %xmm2, %xmm3 +; AVX1-NEXT: vpsrlw $8, %xmm2, %xmm2 +; AVX1-NEXT: vpaddw %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vpcmpeqw %xmm6, %xmm0, %xmm0 +; AVX1-NEXT: vpsrld $16, %xmm0, %xmm0 +; AVX1-NEXT: vpand %xmm0, %xmm2, %xmm0 +; AVX1-NEXT: vpsrld $16, %xmm2, %xmm2 +; AVX1-NEXT: vpaddd %xmm0, %xmm2, %xmm0 ; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 ; AVX1-NEXT: retq ; diff --git a/llvm/test/CodeGen/X86/vector-tzcnt-128.ll b/llvm/test/CodeGen/X86/vector-tzcnt-128.ll index 34ac6e47..aa5abab 100644 --- a/llvm/test/CodeGen/X86/vector-tzcnt-128.ll +++ b/llvm/test/CodeGen/X86/vector-tzcnt-128.ll @@ -14,104 +14,178 @@ define <2 x i64> @testv2i64(<2 x i64> %in) nounwind { ; SSE2-LABEL: testv2i64: ; SSE2: # BB#0: -; SSE2-NEXT: movd %xmm0, %rax -; SSE2-NEXT: bsfq %rax, %rax -; SSE2-NEXT: movl $64, %ecx -; SSE2-NEXT: cmoveq %rcx, %rax -; SSE2-NEXT: movd %rax, %xmm1 -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] -; SSE2-NEXT: movd %xmm0, %rax -; SSE2-NEXT: bsfq %rax, %rax -; SSE2-NEXT: cmoveq %rcx, %rax -; SSE2-NEXT: movd %rax, %xmm0 -; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0] -; SSE2-NEXT: movdqa %xmm1, %xmm0 +; SSE2-NEXT: pxor %xmm1, %xmm1 +; SSE2-NEXT: pxor %xmm2, %xmm2 +; SSE2-NEXT: psubq %xmm0, %xmm2 +; SSE2-NEXT: pand %xmm0, %xmm2 +; SSE2-NEXT: psubq {{.*}}(%rip), %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: psrlq $1, %xmm0 +; SSE2-NEXT: pand {{.*}}(%rip), %xmm0 +; SSE2-NEXT: psubq %xmm0, %xmm2 +; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [3689348814741910323,3689348814741910323] +; SSE2-NEXT: movdqa %xmm2, %xmm3 +; SSE2-NEXT: pand %xmm0, %xmm3 +; SSE2-NEXT: psrlq $2, %xmm2 +; SSE2-NEXT: pand %xmm0, %xmm2 +; SSE2-NEXT: paddq %xmm3, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: psrlq $4, %xmm0 +; SSE2-NEXT: paddq %xmm2, %xmm0 +; SSE2-NEXT: pand {{.*}}(%rip), %xmm0 +; SSE2-NEXT: psadbw %xmm1, %xmm0 ; SSE2-NEXT: retq ; ; SSE3-LABEL: testv2i64: ; SSE3: # BB#0: -; SSE3-NEXT: movd %xmm0, %rax -; SSE3-NEXT: bsfq %rax, %rax -; SSE3-NEXT: movl $64, %ecx -; SSE3-NEXT: cmoveq %rcx, %rax -; SSE3-NEXT: movd %rax, %xmm1 -; SSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] -; SSE3-NEXT: movd %xmm0, %rax -; SSE3-NEXT: bsfq %rax, %rax -; SSE3-NEXT: cmoveq %rcx, %rax -; SSE3-NEXT: movd %rax, %xmm0 -; SSE3-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0] -; SSE3-NEXT: movdqa %xmm1, %xmm0 +; SSE3-NEXT: pxor %xmm1, %xmm1 +; SSE3-NEXT: pxor %xmm2, %xmm2 +; SSE3-NEXT: psubq %xmm0, %xmm2 +; SSE3-NEXT: pand %xmm0, %xmm2 +; SSE3-NEXT: psubq {{.*}}(%rip), %xmm2 +; SSE3-NEXT: movdqa %xmm2, %xmm0 +; SSE3-NEXT: psrlq $1, %xmm0 +; SSE3-NEXT: pand {{.*}}(%rip), %xmm0 +; SSE3-NEXT: psubq %xmm0, %xmm2 +; SSE3-NEXT: movdqa {{.*#+}} xmm0 = [3689348814741910323,3689348814741910323] +; SSE3-NEXT: movdqa %xmm2, %xmm3 +; SSE3-NEXT: pand %xmm0, %xmm3 +; SSE3-NEXT: psrlq $2, %xmm2 +; SSE3-NEXT: pand %xmm0, %xmm2 +; SSE3-NEXT: paddq %xmm3, %xmm2 +; SSE3-NEXT: movdqa %xmm2, %xmm0 +; SSE3-NEXT: psrlq $4, %xmm0 +; SSE3-NEXT: paddq %xmm2, %xmm0 +; SSE3-NEXT: pand {{.*}}(%rip), %xmm0 +; SSE3-NEXT: psadbw %xmm1, %xmm0 ; SSE3-NEXT: retq ; ; SSSE3-LABEL: testv2i64: ; SSSE3: # BB#0: -; SSSE3-NEXT: movd %xmm0, %rax -; SSSE3-NEXT: bsfq %rax, %rax -; SSSE3-NEXT: movl $64, %ecx -; SSSE3-NEXT: cmoveq %rcx, %rax -; SSSE3-NEXT: movd %rax, %xmm1 -; SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] -; SSSE3-NEXT: movd %xmm0, %rax -; SSSE3-NEXT: bsfq %rax, %rax -; SSSE3-NEXT: cmoveq %rcx, %rax -; SSSE3-NEXT: movd %rax, %xmm0 -; SSSE3-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0] -; SSSE3-NEXT: movdqa %xmm1, %xmm0 +; SSSE3-NEXT: pxor %xmm1, %xmm1 +; SSSE3-NEXT: pxor %xmm2, %xmm2 +; SSSE3-NEXT: psubq %xmm0, %xmm2 +; SSSE3-NEXT: pand %xmm0, %xmm2 +; SSSE3-NEXT: psubq {{.*}}(%rip), %xmm2 +; SSSE3-NEXT: movdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; SSSE3-NEXT: movdqa %xmm2, %xmm4 +; SSSE3-NEXT: pand %xmm3, %xmm4 +; SSSE3-NEXT: movdqa {{.*#+}} xmm0 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; SSSE3-NEXT: movdqa %xmm0, %xmm5 +; SSSE3-NEXT: pshufb %xmm4, %xmm5 +; SSSE3-NEXT: psrlw $4, %xmm2 +; SSSE3-NEXT: pand %xmm3, %xmm2 +; SSSE3-NEXT: pshufb %xmm2, %xmm0 +; SSSE3-NEXT: paddb %xmm5, %xmm0 +; SSSE3-NEXT: psadbw %xmm1, %xmm0 ; SSSE3-NEXT: retq ; ; SSE41-LABEL: testv2i64: ; SSE41: # BB#0: -; SSE41-NEXT: pextrq $1, %xmm0, %rax -; SSE41-NEXT: bsfq %rax, %rax -; SSE41-NEXT: movl $64, %ecx -; SSE41-NEXT: cmoveq %rcx, %rax -; SSE41-NEXT: movd %rax, %xmm1 -; SSE41-NEXT: movd %xmm0, %rax -; SSE41-NEXT: bsfq %rax, %rax -; SSE41-NEXT: cmoveq %rcx, %rax -; SSE41-NEXT: movd %rax, %xmm0 -; SSE41-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; SSE41-NEXT: pxor %xmm1, %xmm1 +; SSE41-NEXT: pxor %xmm2, %xmm2 +; SSE41-NEXT: psubq %xmm0, %xmm2 +; SSE41-NEXT: pand %xmm0, %xmm2 +; SSE41-NEXT: psubq {{.*}}(%rip), %xmm2 +; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; SSE41-NEXT: movdqa %xmm2, %xmm4 +; SSE41-NEXT: pand %xmm3, %xmm4 +; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; SSE41-NEXT: movdqa %xmm0, %xmm5 +; SSE41-NEXT: pshufb %xmm4, %xmm5 +; SSE41-NEXT: psrlw $4, %xmm2 +; SSE41-NEXT: pand %xmm3, %xmm2 +; SSE41-NEXT: pshufb %xmm2, %xmm0 +; SSE41-NEXT: paddb %xmm5, %xmm0 +; SSE41-NEXT: psadbw %xmm1, %xmm0 ; SSE41-NEXT: retq ; -; AVX-LABEL: testv2i64: -; AVX: # BB#0: -; AVX-NEXT: vpextrq $1, %xmm0, %rax -; AVX-NEXT: bsfq %rax, %rax -; AVX-NEXT: movl $64, %ecx -; AVX-NEXT: cmoveq %rcx, %rax -; AVX-NEXT: vmovq %rax, %xmm1 -; AVX-NEXT: vmovq %xmm0, %rax -; AVX-NEXT: bsfq %rax, %rax -; AVX-NEXT: cmoveq %rcx, %rax -; AVX-NEXT: vmovq %rax, %xmm0 -; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] -; AVX-NEXT: retq +; AVX1-LABEL: testv2i64: +; AVX1: # BB#0: +; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpsubq %xmm0, %xmm1, %xmm2 +; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpsubq {{.*}}(%rip), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm3 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; AVX1-NEXT: vpshufb %xmm3, %xmm4, %xmm3 +; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0 +; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpshufb %xmm0, %xmm4, %xmm0 +; AVX1-NEXT: vpaddb %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: testv2i64: +; AVX2: # BB#0: +; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX2-NEXT: vpsubq %xmm0, %xmm1, %xmm2 +; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm0 +; AVX2-NEXT: vpsubq {{.*}}(%rip), %xmm0, %xmm0 +; AVX2-NEXT: vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm3 +; AVX2-NEXT: vmovdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; AVX2-NEXT: vpshufb %xmm3, %xmm4, %xmm3 +; AVX2-NEXT: vpsrlw $4, %xmm0, %xmm0 +; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm0 +; AVX2-NEXT: vpshufb %xmm0, %xmm4, %xmm0 +; AVX2-NEXT: vpaddb %xmm3, %xmm0, %xmm0 +; AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: retq +; +; AVX512CDVL-LABEL: testv2i64: +; AVX512CDVL: # BB#0: +; AVX512CDVL-NEXT: vpxord %xmm1, %xmm1, %xmm1 +; AVX512CDVL-NEXT: vpsubq %xmm0, %xmm1, %xmm2 +; AVX512CDVL-NEXT: vpandq %xmm2, %xmm0, %xmm0 +; AVX512CDVL-NEXT: vpsubq {{.*}}(%rip), %xmm0, %xmm0 +; AVX512CDVL-NEXT: vmovdqa64 {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX512CDVL-NEXT: vpandq %xmm2, %xmm0, %xmm3 +; AVX512CDVL-NEXT: vmovdqa64 {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; AVX512CDVL-NEXT: vpshufb %xmm3, %xmm4, %xmm3 +; AVX512CDVL-NEXT: vpsrlw $4, %xmm0, %xmm0 +; AVX512CDVL-NEXT: vpandq %xmm2, %xmm0, %xmm0 +; AVX512CDVL-NEXT: vpshufb %xmm0, %xmm4, %xmm0 +; AVX512CDVL-NEXT: vpaddb %xmm3, %xmm0, %xmm0 +; AVX512CDVL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0 +; AVX512CDVL-NEXT: retq +; +; AVX512CD-LABEL: testv2i64: +; AVX512CD: # BB#0: +; AVX512CD-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX512CD-NEXT: vpsubq %xmm0, %xmm1, %xmm2 +; AVX512CD-NEXT: vpand %xmm2, %xmm0, %xmm0 +; AVX512CD-NEXT: vpsubq {{.*}}(%rip), %xmm0, %xmm0 +; AVX512CD-NEXT: vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX512CD-NEXT: vpand %xmm2, %xmm0, %xmm3 +; AVX512CD-NEXT: vmovdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; AVX512CD-NEXT: vpshufb %xmm3, %xmm4, %xmm3 +; AVX512CD-NEXT: vpsrlw $4, %xmm0, %xmm0 +; AVX512CD-NEXT: vpand %xmm2, %xmm0, %xmm0 +; AVX512CD-NEXT: vpshufb %xmm0, %xmm4, %xmm0 +; AVX512CD-NEXT: vpaddb %xmm3, %xmm0, %xmm0 +; AVX512CD-NEXT: vpsadbw %xmm1, %xmm0, %xmm0 +; AVX512CD-NEXT: retq ; ; X32-SSE-LABEL: testv2i64: ; X32-SSE: # BB#0: -; X32-SSE-NEXT: pushl %esi -; X32-SSE-NEXT: pextrd $3, %xmm0, %eax -; X32-SSE-NEXT: bsfl %eax, %eax -; X32-SSE-NEXT: movl $32, %ecx -; X32-SSE-NEXT: cmovel %ecx, %eax -; X32-SSE-NEXT: addl $32, %eax -; X32-SSE-NEXT: pextrd $2, %xmm0, %edx -; X32-SSE-NEXT: bsfl %edx, %esi -; X32-SSE-NEXT: testl %edx, %edx -; X32-SSE-NEXT: cmovel %eax, %esi -; X32-SSE-NEXT: movd %esi, %xmm1 -; X32-SSE-NEXT: pextrd $1, %xmm0, %eax -; X32-SSE-NEXT: bsfl %eax, %eax -; X32-SSE-NEXT: cmovel %ecx, %eax -; X32-SSE-NEXT: addl $32, %eax -; X32-SSE-NEXT: movd %xmm0, %ecx -; X32-SSE-NEXT: bsfl %ecx, %edx -; X32-SSE-NEXT: testl %ecx, %ecx -; X32-SSE-NEXT: cmovel %eax, %edx -; X32-SSE-NEXT: movd %edx, %xmm0 -; X32-SSE-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] -; X32-SSE-NEXT: popl %esi +; X32-SSE-NEXT: pxor %xmm1, %xmm1 +; X32-SSE-NEXT: pxor %xmm2, %xmm2 +; X32-SSE-NEXT: psubq %xmm0, %xmm2 +; X32-SSE-NEXT: pand %xmm0, %xmm2 +; X32-SSE-NEXT: psubq {{\.LCPI.*}}, %xmm2 +; X32-SSE-NEXT: movdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; X32-SSE-NEXT: movdqa %xmm2, %xmm4 +; X32-SSE-NEXT: pand %xmm3, %xmm4 +; X32-SSE-NEXT: movdqa {{.*#+}} xmm0 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; X32-SSE-NEXT: movdqa %xmm0, %xmm5 +; X32-SSE-NEXT: pshufb %xmm4, %xmm5 +; X32-SSE-NEXT: psrlw $4, %xmm2 +; X32-SSE-NEXT: pand %xmm3, %xmm2 +; X32-SSE-NEXT: pshufb %xmm2, %xmm0 +; X32-SSE-NEXT: paddb %xmm5, %xmm0 +; X32-SSE-NEXT: psadbw %xmm1, %xmm0 ; X32-SSE-NEXT: retl %out = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %in, i1 0) ret <2 x i64> %out @@ -120,74 +194,124 @@ define <2 x i64> @testv2i64(<2 x i64> %in) nounwind { define <2 x i64> @testv2i64u(<2 x i64> %in) nounwind { ; SSE2-LABEL: testv2i64u: ; SSE2: # BB#0: -; SSE2-NEXT: movd %xmm0, %rax -; SSE2-NEXT: bsfq %rax, %rax -; SSE2-NEXT: movd %rax, %xmm1 -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] -; SSE2-NEXT: movd %xmm0, %rax -; SSE2-NEXT: bsfq %rax, %rax -; SSE2-NEXT: movd %rax, %xmm0 -; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0] -; SSE2-NEXT: movdqa %xmm1, %xmm0 +; SSE2-NEXT: pxor %xmm1, %xmm1 +; SSE2-NEXT: pxor %xmm2, %xmm2 +; SSE2-NEXT: psubq %xmm0, %xmm2 +; SSE2-NEXT: pand %xmm0, %xmm2 +; SSE2-NEXT: psubq {{.*}}(%rip), %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: psrlq $1, %xmm0 +; SSE2-NEXT: pand {{.*}}(%rip), %xmm0 +; SSE2-NEXT: psubq %xmm0, %xmm2 +; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [3689348814741910323,3689348814741910323] +; SSE2-NEXT: movdqa %xmm2, %xmm3 +; SSE2-NEXT: pand %xmm0, %xmm3 +; SSE2-NEXT: psrlq $2, %xmm2 +; SSE2-NEXT: pand %xmm0, %xmm2 +; SSE2-NEXT: paddq %xmm3, %xmm2 +; SSE2-NEXT: movdqa %xmm2, %xmm0 +; SSE2-NEXT: psrlq $4, %xmm0 +; SSE2-NEXT: paddq %xmm2, %xmm0 +; SSE2-NEXT: pand {{.*}}(%rip), %xmm0 +; SSE2-NEXT: psadbw %xmm1, %xmm0 ; SSE2-NEXT: retq ; ; SSE3-LABEL: testv2i64u: ; SSE3: # BB#0: -; SSE3-NEXT: movd %xmm0, %rax -; SSE3-NEXT: bsfq %rax, %rax -; SSE3-NEXT: movd %rax, %xmm1 -; SSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] -; SSE3-NEXT: movd %xmm0, %rax -; SSE3-NEXT: bsfq %rax, %rax -; SSE3-NEXT: movd %rax, %xmm0 -; SSE3-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0] -; SSE3-NEXT: movdqa %xmm1, %xmm0 +; SSE3-NEXT: pxor %xmm1, %xmm1 +; SSE3-NEXT: pxor %xmm2, %xmm2 +; SSE3-NEXT: psubq %xmm0, %xmm2 +; SSE3-NEXT: pand %xmm0, %xmm2 +; SSE3-NEXT: psubq {{.*}}(%rip), %xmm2 +; SSE3-NEXT: movdqa %xmm2, %xmm0 +; SSE3-NEXT: psrlq $1, %xmm0 +; SSE3-NEXT: pand {{.*}}(%rip), %xmm0 +; SSE3-NEXT: psubq %xmm0, %xmm2 +; SSE3-NEXT: movdqa {{.*#+}} xmm0 = [3689348814741910323,3689348814741910323] +; SSE3-NEXT: movdqa %xmm2, %xmm3 +; SSE3-NEXT: pand %xmm0, %xmm3 +; SSE3-NEXT: psrlq $2, %xmm2 +; SSE3-NEXT: pand %xmm0, %xmm2 +; SSE3-NEXT: paddq %xmm3, %xmm2 +; SSE3-NEXT: movdqa %xmm2, %xmm0 +; SSE3-NEXT: psrlq $4, %xmm0 +; SSE3-NEXT: paddq %xmm2, %xmm0 +; SSE3-NEXT: pand {{.*}}(%rip), %xmm0 +; SSE3-NEXT: psadbw %xmm1, %xmm0 ; SSE3-NEXT: retq ; ; SSSE3-LABEL: testv2i64u: ; SSSE3: # BB#0: -; SSSE3-NEXT: movd %xmm0, %rax -; SSSE3-NEXT: bsfq %rax, %rax -; SSSE3-NEXT: movd %rax, %xmm1 -; SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1] -; SSSE3-NEXT: movd %xmm0, %rax -; SSSE3-NEXT: bsfq %rax, %rax -; SSSE3-NEXT: movd %rax, %xmm0 -; SSSE3-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0] -; SSSE3-NEXT: movdqa %xmm1, %xmm0 +; SSSE3-NEXT: pxor %xmm1, %xmm1 +; SSSE3-NEXT: pxor %xmm2, %xmm2 +; SSSE3-NEXT: psubq %xmm0, %xmm2 +; SSSE3-NEXT: pand %xmm0, %xmm2 +; SSSE3-NEXT: psubq {{.*}}(%rip), %xmm2 +; SSSE3-NEXT: movdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; SSSE3-NEXT: movdqa %xmm2, %xmm4 +; SSSE3-NEXT: pand %xmm3, %xmm4 +; SSSE3-NEXT: movdqa {{.*#+}} xmm0 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; SSSE3-NEXT: movdqa %xmm0, %xmm5 +; SSSE3-NEXT: pshufb %xmm4, %xmm5 +; SSSE3-NEXT: psrlw $4, %xmm2 +; SSSE3-NEXT: pand %xmm3, %xmm2 +; SSSE3-NEXT: pshufb %xmm2, %xmm0 +; SSSE3-NEXT: paddb %xmm5, %xmm0 +; SSSE3-NEXT: psadbw %xmm1, %xmm0 ; SSSE3-NEXT: retq ; ; SSE41-LABEL: testv2i64u: ; SSE41: # BB#0: -; SSE41-NEXT: pextrq $1, %xmm0, %rax -; SSE41-NEXT: bsfq %rax, %rax -; SSE41-NEXT: movd %rax, %xmm1 -; SSE41-NEXT: movd %xmm0, %rax -; SSE41-NEXT: bsfq %rax, %rax -; SSE41-NEXT: movd %rax, %xmm0 -; SSE41-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; SSE41-NEXT: pxor %xmm1, %xmm1 +; SSE41-NEXT: pxor %xmm2, %xmm2 +; SSE41-NEXT: psubq %xmm0, %xmm2 +; SSE41-NEXT: pand %xmm0, %xmm2 +; SSE41-NEXT: psubq {{.*}}(%rip), %xmm2 +; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; SSE41-NEXT: movdqa %xmm2, %xmm4 +; SSE41-NEXT: pand %xmm3, %xmm4 +; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; SSE41-NEXT: movdqa %xmm0, %xmm5 +; SSE41-NEXT: pshufb %xmm4, %xmm5 +; SSE41-NEXT: psrlw $4, %xmm2 +; SSE41-NEXT: pand %xmm3, %xmm2 +; SSE41-NEXT: pshufb %xmm2, %xmm0 +; SSE41-NEXT: paddb %xmm5, %xmm0 +; SSE41-NEXT: psadbw %xmm1, %xmm0 ; SSE41-NEXT: retq ; ; AVX1-LABEL: testv2i64u: ; AVX1: # BB#0: -; AVX1-NEXT: vpextrq $1, %xmm0, %rax -; AVX1-NEXT: bsfq %rax, %rax -; AVX1-NEXT: vmovq %rax, %xmm1 -; AVX1-NEXT: vmovq %xmm0, %rax -; AVX1-NEXT: bsfq %rax, %rax -; AVX1-NEXT: vmovq %rax, %xmm0 -; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpsubq %xmm0, %xmm1, %xmm2 +; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpsubq {{.*}}(%rip), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm3 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; AVX1-NEXT: vpshufb %xmm3, %xmm4, %xmm3 +; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0 +; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpshufb %xmm0, %xmm4, %xmm0 +; AVX1-NEXT: vpaddb %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0 ; AVX1-NEXT: retq ; ; AVX2-LABEL: testv2i64u: ; AVX2: # BB#0: -; AVX2-NEXT: vpextrq $1, %xmm0, %rax -; AVX2-NEXT: bsfq %rax, %rax -; AVX2-NEXT: vmovq %rax, %xmm1 -; AVX2-NEXT: vmovq %xmm0, %rax -; AVX2-NEXT: bsfq %rax, %rax -; AVX2-NEXT: vmovq %rax, %xmm0 -; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX2-NEXT: vpsubq %xmm0, %xmm1, %xmm2 +; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm0 +; AVX2-NEXT: vpsubq {{.*}}(%rip), %xmm0, %xmm0 +; AVX2-NEXT: vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm3 +; AVX2-NEXT: vmovdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; AVX2-NEXT: vpshufb %xmm3, %xmm4, %xmm3 +; AVX2-NEXT: vpsrlw $4, %xmm0, %xmm0 +; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm0 +; AVX2-NEXT: vpshufb %xmm0, %xmm4, %xmm0 +; AVX2-NEXT: vpaddb %xmm3, %xmm0, %xmm0 +; AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0 ; AVX2-NEXT: retq ; ; AVX512CDVL-LABEL: testv2i64u: @@ -212,23 +336,22 @@ define <2 x i64> @testv2i64u(<2 x i64> %in) nounwind { ; ; X32-SSE-LABEL: testv2i64u: ; X32-SSE: # BB#0: -; X32-SSE-NEXT: pextrd $2, %xmm0, %eax -; X32-SSE-NEXT: bsfl %eax, %ecx -; X32-SSE-NEXT: pextrd $3, %xmm0, %edx -; X32-SSE-NEXT: bsfl %edx, %edx -; X32-SSE-NEXT: addl $32, %edx -; X32-SSE-NEXT: testl %eax, %eax -; X32-SSE-NEXT: cmovnel %ecx, %edx -; X32-SSE-NEXT: movd %edx, %xmm1 -; X32-SSE-NEXT: movd %xmm0, %eax -; X32-SSE-NEXT: bsfl %eax, %ecx -; X32-SSE-NEXT: pextrd $1, %xmm0, %edx -; X32-SSE-NEXT: bsfl %edx, %edx -; X32-SSE-NEXT: addl $32, %edx -; X32-SSE-NEXT: testl %eax, %eax -; X32-SSE-NEXT: cmovnel %ecx, %edx -; X32-SSE-NEXT: movd %edx, %xmm0 -; X32-SSE-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; X32-SSE-NEXT: pxor %xmm1, %xmm1 +; X32-SSE-NEXT: pxor %xmm2, %xmm2 +; X32-SSE-NEXT: psubq %xmm0, %xmm2 +; X32-SSE-NEXT: pand %xmm0, %xmm2 +; X32-SSE-NEXT: psubq {{\.LCPI.*}}, %xmm2 +; X32-SSE-NEXT: movdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; X32-SSE-NEXT: movdqa %xmm2, %xmm4 +; X32-SSE-NEXT: pand %xmm3, %xmm4 +; X32-SSE-NEXT: movdqa {{.*#+}} xmm0 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; X32-SSE-NEXT: movdqa %xmm0, %xmm5 +; X32-SSE-NEXT: pshufb %xmm4, %xmm5 +; X32-SSE-NEXT: psrlw $4, %xmm2 +; X32-SSE-NEXT: pand %xmm3, %xmm2 +; X32-SSE-NEXT: pshufb %xmm2, %xmm0 +; X32-SSE-NEXT: paddb %xmm5, %xmm0 +; X32-SSE-NEXT: psadbw %xmm1, %xmm0 ; X32-SSE-NEXT: retl %out = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %in, i1 -1) ret <2 x i64> %out @@ -1400,8 +1523,23 @@ define <2 x i64> @foldv2i64() nounwind { ; ; X32-SSE-LABEL: foldv2i64: ; X32-SSE: # BB#0: -; X32-SSE-NEXT: movl $8, %eax -; X32-SSE-NEXT: movd %eax, %xmm0 +; X32-SSE-NEXT: movdqa {{.*#+}} xmm0 = [256,0,4294967295,4294967295] +; X32-SSE-NEXT: pxor %xmm1, %xmm1 +; X32-SSE-NEXT: pxor %xmm2, %xmm2 +; X32-SSE-NEXT: psubq %xmm0, %xmm2 +; X32-SSE-NEXT: pand %xmm0, %xmm2 +; X32-SSE-NEXT: psubq {{\.LCPI.*}}, %xmm2 +; X32-SSE-NEXT: movdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; X32-SSE-NEXT: movdqa %xmm2, %xmm4 +; X32-SSE-NEXT: pand %xmm3, %xmm4 +; X32-SSE-NEXT: movdqa {{.*#+}} xmm0 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; X32-SSE-NEXT: movdqa %xmm0, %xmm5 +; X32-SSE-NEXT: pshufb %xmm4, %xmm5 +; X32-SSE-NEXT: psrlw $4, %xmm2 +; X32-SSE-NEXT: pand %xmm3, %xmm2 +; X32-SSE-NEXT: pshufb %xmm2, %xmm0 +; X32-SSE-NEXT: paddb %xmm5, %xmm0 +; X32-SSE-NEXT: psadbw %xmm1, %xmm0 ; X32-SSE-NEXT: retl %out = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> , i1 0) ret <2 x i64> %out @@ -1422,8 +1560,23 @@ define <2 x i64> @foldv2i64u() nounwind { ; ; X32-SSE-LABEL: foldv2i64u: ; X32-SSE: # BB#0: -; X32-SSE-NEXT: movl $8, %eax -; X32-SSE-NEXT: movd %eax, %xmm0 +; X32-SSE-NEXT: movdqa {{.*#+}} xmm0 = [256,0,4294967295,4294967295] +; X32-SSE-NEXT: pxor %xmm1, %xmm1 +; X32-SSE-NEXT: pxor %xmm2, %xmm2 +; X32-SSE-NEXT: psubq %xmm0, %xmm2 +; X32-SSE-NEXT: pand %xmm0, %xmm2 +; X32-SSE-NEXT: psubq {{\.LCPI.*}}, %xmm2 +; X32-SSE-NEXT: movdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] +; X32-SSE-NEXT: movdqa %xmm2, %xmm4 +; X32-SSE-NEXT: pand %xmm3, %xmm4 +; X32-SSE-NEXT: movdqa {{.*#+}} xmm0 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4] +; X32-SSE-NEXT: movdqa %xmm0, %xmm5 +; X32-SSE-NEXT: pshufb %xmm4, %xmm5 +; X32-SSE-NEXT: psrlw $4, %xmm2 +; X32-SSE-NEXT: pand %xmm3, %xmm2 +; X32-SSE-NEXT: pshufb %xmm2, %xmm0 +; X32-SSE-NEXT: paddb %xmm5, %xmm0 +; X32-SSE-NEXT: psadbw %xmm1, %xmm0 ; X32-SSE-NEXT: retl %out = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> , i1 -1) ret <2 x i64> %out -- 2.7.4