[X86] Add base SSE2 i686 test coverage to vector bool reduction tests

diff --git a/llvm/test/CodeGen/X86/vector-reduce-and-bool.ll b/llvm/test/CodeGen/X86/vector-reduce-and-bool.ll
index 600326b..db67510 100644 (file)
--- a/llvm/test/CodeGen/X86/vector-reduce-and-bool.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-and-bool.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE,SSE41
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE2,X86-SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE2,X64-SSE,X64-SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE,X64-SSE,SSE41
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1OR2,AVX1
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX1OR2,AVX2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX,AVX512,AVX512F
@@ -11,14 +12,14 @@
 ; Truncate
 ;
 
-define i1 @trunc_v2i64_v2i1(<2 x i64>) {
+define i1 @trunc_v2i64_v2i1(<2 x i64>) nounwind {
 ; SSE2-LABEL: trunc_v2i64_v2i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    psllq $63, %xmm0
 ; SSE2-NEXT:    movmskpd %xmm0, %eax
 ; SSE2-NEXT:    cmpl $3, %eax
 ; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: trunc_v2i64_v2i1:
 ; SSE41:       # %bb.0:
@@ -55,14 +56,14 @@ define i1 @trunc_v2i64_v2i1(<2 x i64>) {
   ret i1 %b
 }
 
-define i1 @trunc_v4i32_v4i1(<4 x i32>) {
+define i1 @trunc_v4i32_v4i1(<4 x i32>) nounwind {
 ; SSE2-LABEL: trunc_v4i32_v4i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pslld $31, %xmm0
 ; SSE2-NEXT:    movmskps %xmm0, %eax
 ; SSE2-NEXT:    xorl $15, %eax
 ; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: trunc_v4i32_v4i1:
 ; SSE41:       # %bb.0:
@@ -99,7 +100,7 @@ define i1 @trunc_v4i32_v4i1(<4 x i32>) {
   ret i1 %b
 }
 
-define i1 @trunc_v8i16_v8i1(<8 x i16>) {
+define i1 @trunc_v8i16_v8i1(<8 x i16>) nounwind {
 ; SSE2-LABEL: trunc_v8i16_v8i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    psllw $7, %xmm0
@@ -107,7 +108,7 @@ define i1 @trunc_v8i16_v8i1(<8 x i16>) {
 ; SSE2-NEXT:    notl %eax
 ; SSE2-NEXT:    testl $21845, %eax # imm = 0x5555
 ; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: trunc_v8i16_v8i1:
 ; SSE41:       # %bb.0:
@@ -144,14 +145,14 @@ define i1 @trunc_v8i16_v8i1(<8 x i16>) {
   ret i1 %b
 }
 
-define i1 @trunc_v16i8_v16i1(<16 x i8>) {
+define i1 @trunc_v16i8_v16i1(<16 x i8>) nounwind {
 ; SSE2-LABEL: trunc_v16i8_v16i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    psllw $7, %xmm0
 ; SSE2-NEXT:    pmovmskb %xmm0, %eax
 ; SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
 ; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: trunc_v16i8_v16i1:
 ; SSE41:       # %bb.0:
@@ -188,7 +189,7 @@ define i1 @trunc_v16i8_v16i1(<16 x i8>) {
   ret i1 %b
 }
 
-define i1 @trunc_v4i64_v4i1(<4 x i64>) {
+define i1 @trunc_v4i64_v4i1(<4 x i64>) nounwind {
 ; SSE2-LABEL: trunc_v4i64_v4i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
@@ -196,7 +197,7 @@ define i1 @trunc_v4i64_v4i1(<4 x i64>) {
 ; SSE2-NEXT:    movmskps %xmm0, %eax
 ; SSE2-NEXT:    cmpl $15, %eax
 ; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: trunc_v4i64_v4i1:
 ; SSE41:       # %bb.0:
@@ -232,7 +233,7 @@ define i1 @trunc_v4i64_v4i1(<4 x i64>) {
   ret i1 %b
 }
 
-define i1 @trunc_v8i32_v8i1(<8 x i32>) {
+define i1 @trunc_v8i32_v8i1(<8 x i32>) nounwind {
 ; SSE2-LABEL: trunc_v8i32_v8i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pand %xmm1, %xmm0
@@ -240,7 +241,7 @@ define i1 @trunc_v8i32_v8i1(<8 x i32>) {
 ; SSE2-NEXT:    movmskps %xmm0, %eax
 ; SSE2-NEXT:    xorl $15, %eax
 ; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: trunc_v8i32_v8i1:
 ; SSE41:       # %bb.0:
@@ -276,7 +277,7 @@ define i1 @trunc_v8i32_v8i1(<8 x i32>) {
   ret i1 %b
 }
 
-define i1 @trunc_v16i16_v16i1(<16 x i16>) {
+define i1 @trunc_v16i16_v16i1(<16 x i16>) nounwind {
 ; SSE2-LABEL: trunc_v16i16_v16i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pand %xmm1, %xmm0
@@ -285,7 +286,7 @@ define i1 @trunc_v16i16_v16i1(<16 x i16>) {
 ; SSE2-NEXT:    notl %eax
 ; SSE2-NEXT:    testl $21845, %eax # imm = 0x5555
 ; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: trunc_v16i16_v16i1:
 ; SSE41:       # %bb.0:
@@ -321,7 +322,7 @@ define i1 @trunc_v16i16_v16i1(<16 x i16>) {
   ret i1 %b
 }
 
-define i1 @trunc_v32i8_v32i1(<32 x i8>) {
+define i1 @trunc_v32i8_v32i1(<32 x i8>) nounwind {
 ; SSE2-LABEL: trunc_v32i8_v32i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pand %xmm1, %xmm0
@@ -329,7 +330,7 @@ define i1 @trunc_v32i8_v32i1(<32 x i8>) {
 ; SSE2-NEXT:    pmovmskb %xmm0, %eax
 ; SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
 ; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: trunc_v32i8_v32i1:
 ; SSE41:       # %bb.0:
@@ -365,26 +366,52 @@ define i1 @trunc_v32i8_v32i1(<32 x i8>) {
   ret i1 %b
 }
 
-define i1 @trunc_v8i64_v8i1(<8 x i64>) {
-; SSE2-LABEL: trunc_v8i64_v8i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[0,2,2,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,1,0,2,4,5,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,1,0,2,4,5,6,7]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; SSE2-NEXT:    movsd {{.*#+}} xmm2 = xmm0[0],xmm2[1]
-; SSE2-NEXT:    psllw $15, %xmm2
-; SSE2-NEXT:    packsswb %xmm2, %xmm2
-; SSE2-NEXT:    pmovmskb %xmm2, %eax
-; SSE2-NEXT:    cmpb $-1, %al
-; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+define i1 @trunc_v8i64_v8i1(<8 x i64>) nounwind {
+; X86-SSE2-LABEL: trunc_v8i64_v8i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; X86-SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X86-SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; X86-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = mem[0,2,2,3]
+; X86-SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,1,0,2,4,5,6,7]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; X86-SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,1,0,2,4,5,6,7]
+; X86-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm2 = xmm0[0],xmm2[1]
+; X86-SSE2-NEXT:    psllw $15, %xmm2
+; X86-SSE2-NEXT:    packsswb %xmm2, %xmm2
+; X86-SSE2-NEXT:    pmovmskb %xmm2, %eax
+; X86-SSE2-NEXT:    cmpb $-1, %al
+; X86-SSE2-NEXT:    sete %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE2-LABEL: trunc_v8i64_v8i1:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; X64-SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X64-SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; X64-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[0,2,2,3]
+; X64-SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,1,0,2,4,5,6,7]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; X64-SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,1,0,2,4,5,6,7]
+; X64-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; X64-SSE2-NEXT:    movsd {{.*#+}} xmm2 = xmm0[0],xmm2[1]
+; X64-SSE2-NEXT:    psllw $15, %xmm2
+; X64-SSE2-NEXT:    packsswb %xmm2, %xmm2
+; X64-SSE2-NEXT:    pmovmskb %xmm2, %eax
+; X64-SSE2-NEXT:    cmpb $-1, %al
+; X64-SSE2-NEXT:    sete %al
+; X64-SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: trunc_v8i64_v8i1:
 ; SSE41:       # %bb.0:
@@ -426,17 +453,34 @@ define i1 @trunc_v8i64_v8i1(<8 x i64>) {
   ret i1 %b
 }
 
-define i1 @trunc_v16i32_v16i1(<16 x i32>) {
-; SSE2-LABEL: trunc_v16i32_v16i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    pslld $31, %xmm0
-; SSE2-NEXT:    movmskps %xmm0, %eax
-; SSE2-NEXT:    xorl $15, %eax
-; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+define i1 @trunc_v16i32_v16i1(<16 x i32>) nounwind {
+; X86-SSE2-LABEL: trunc_v16i32_v16i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    pand %xmm2, %xmm0
+; X86-SSE2-NEXT:    pand 8(%ebp), %xmm1
+; X86-SSE2-NEXT:    pand %xmm0, %xmm1
+; X86-SSE2-NEXT:    pslld $31, %xmm1
+; X86-SSE2-NEXT:    movmskps %xmm1, %eax
+; X86-SSE2-NEXT:    xorl $15, %eax
+; X86-SSE2-NEXT:    sete %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE2-LABEL: trunc_v16i32_v16i1:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    pand %xmm3, %xmm1
+; X64-SSE2-NEXT:    pand %xmm2, %xmm0
+; X64-SSE2-NEXT:    pand %xmm1, %xmm0
+; X64-SSE2-NEXT:    pslld $31, %xmm0
+; X64-SSE2-NEXT:    movmskps %xmm0, %eax
+; X64-SSE2-NEXT:    xorl $15, %eax
+; X64-SSE2-NEXT:    sete %al
+; X64-SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: trunc_v16i32_v16i1:
 ; SSE41:       # %bb.0:
@@ -476,18 +520,36 @@ define i1 @trunc_v16i32_v16i1(<16 x i32>) {
   ret i1 %b
 }
 
-define i1 @trunc_v32i16_v32i1(<32 x i16>) {
-; SSE2-LABEL: trunc_v32i16_v32i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    psllw $7, %xmm0
-; SSE2-NEXT:    pmovmskb %xmm0, %eax
-; SSE2-NEXT:    notl %eax
-; SSE2-NEXT:    testl $21845, %eax # imm = 0x5555
-; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+define i1 @trunc_v32i16_v32i1(<32 x i16>) nounwind {
+; X86-SSE2-LABEL: trunc_v32i16_v32i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    pand %xmm2, %xmm0
+; X86-SSE2-NEXT:    pand 8(%ebp), %xmm1
+; X86-SSE2-NEXT:    pand %xmm0, %xmm1
+; X86-SSE2-NEXT:    psllw $7, %xmm1
+; X86-SSE2-NEXT:    pmovmskb %xmm1, %eax
+; X86-SSE2-NEXT:    notl %eax
+; X86-SSE2-NEXT:    testl $21845, %eax # imm = 0x5555
+; X86-SSE2-NEXT:    sete %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE2-LABEL: trunc_v32i16_v32i1:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    pand %xmm3, %xmm1
+; X64-SSE2-NEXT:    pand %xmm2, %xmm0
+; X64-SSE2-NEXT:    pand %xmm1, %xmm0
+; X64-SSE2-NEXT:    psllw $7, %xmm0
+; X64-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X64-SSE2-NEXT:    notl %eax
+; X64-SSE2-NEXT:    testl $21845, %eax # imm = 0x5555
+; X64-SSE2-NEXT:    sete %al
+; X64-SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: trunc_v32i16_v32i1:
 ; SSE41:       # %bb.0:
@@ -549,17 +611,34 @@ define i1 @trunc_v32i16_v32i1(<32 x i16>) {
   ret i1 %b
 }
 
-define i1 @trunc_v64i8_v64i1(<64 x i8>) {
-; SSE2-LABEL: trunc_v64i8_v64i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    psllw $7, %xmm0
-; SSE2-NEXT:    pmovmskb %xmm0, %eax
-; SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
-; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+define i1 @trunc_v64i8_v64i1(<64 x i8>) nounwind {
+; X86-SSE2-LABEL: trunc_v64i8_v64i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    pand %xmm2, %xmm0
+; X86-SSE2-NEXT:    pand 8(%ebp), %xmm1
+; X86-SSE2-NEXT:    pand %xmm0, %xmm1
+; X86-SSE2-NEXT:    psllw $7, %xmm1
+; X86-SSE2-NEXT:    pmovmskb %xmm1, %eax
+; X86-SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
+; X86-SSE2-NEXT:    sete %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE2-LABEL: trunc_v64i8_v64i1:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    pand %xmm3, %xmm1
+; X64-SSE2-NEXT:    pand %xmm2, %xmm0
+; X64-SSE2-NEXT:    pand %xmm1, %xmm0
+; X64-SSE2-NEXT:    psllw $7, %xmm0
+; X64-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X64-SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
+; X64-SSE2-NEXT:    sete %al
+; X64-SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: trunc_v64i8_v64i1:
 ; SSE41:       # %bb.0:
@@ -625,7 +704,7 @@ define i1 @trunc_v64i8_v64i1(<64 x i8>) {
 ; Comparison With Zero
 ;
 
-define i1 @icmp0_v2i64_v2i1(<2 x i64>) {
+define i1 @icmp0_v2i64_v2i1(<2 x i64>) nounwind {
 ; SSE2-LABEL: icmp0_v2i64_v2i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm1, %xmm1
@@ -633,7 +712,7 @@ define i1 @icmp0_v2i64_v2i1(<2 x i64>) {
 ; SSE2-NEXT:    movmskps %xmm1, %eax
 ; SSE2-NEXT:    xorl $15, %eax
 ; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: icmp0_v2i64_v2i1:
 ; SSE41:       # %bb.0:
@@ -651,7 +730,7 @@ define i1 @icmp0_v2i64_v2i1(<2 x i64>) {
   ret i1 %b
 }
 
-define i1 @icmp0_v4i32_v4i1(<4 x i32>) {
+define i1 @icmp0_v4i32_v4i1(<4 x i32>) nounwind {
 ; SSE2-LABEL: icmp0_v4i32_v4i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm1, %xmm1
@@ -659,7 +738,7 @@ define i1 @icmp0_v4i32_v4i1(<4 x i32>) {
 ; SSE2-NEXT:    movmskps %xmm1, %eax
 ; SSE2-NEXT:    xorl $15, %eax
 ; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: icmp0_v4i32_v4i1:
 ; SSE41:       # %bb.0:
@@ -677,7 +756,7 @@ define i1 @icmp0_v4i32_v4i1(<4 x i32>) {
   ret i1 %b
 }
 
-define i1 @icmp0_v8i16_v8i1(<8 x i16>) {
+define i1 @icmp0_v8i16_v8i1(<8 x i16>) nounwind {
 ; SSE2-LABEL: icmp0_v8i16_v8i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm1, %xmm1
@@ -685,7 +764,7 @@ define i1 @icmp0_v8i16_v8i1(<8 x i16>) {
 ; SSE2-NEXT:    pmovmskb %xmm1, %eax
 ; SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
 ; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: icmp0_v8i16_v8i1:
 ; SSE41:       # %bb.0:
@@ -703,7 +782,7 @@ define i1 @icmp0_v8i16_v8i1(<8 x i16>) {
   ret i1 %b
 }
 
-define i1 @icmp0_v16i8_v16i1(<16 x i8>) {
+define i1 @icmp0_v16i8_v16i1(<16 x i8>) nounwind {
 ; SSE2-LABEL: icmp0_v16i8_v16i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm1, %xmm1
@@ -711,7 +790,7 @@ define i1 @icmp0_v16i8_v16i1(<16 x i8>) {
 ; SSE2-NEXT:    pmovmskb %xmm1, %eax
 ; SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
 ; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: icmp0_v16i8_v16i1:
 ; SSE41:       # %bb.0:
@@ -729,7 +808,7 @@ define i1 @icmp0_v16i8_v16i1(<16 x i8>) {
   ret i1 %b
 }
 
-define i1 @icmp0_v4i64_v4i1(<4 x i64>) {
+define i1 @icmp0_v4i64_v4i1(<4 x i64>) nounwind {
 ; SSE2-LABEL: icmp0_v4i64_v4i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    por %xmm1, %xmm0
@@ -738,7 +817,7 @@ define i1 @icmp0_v4i64_v4i1(<4 x i64>) {
 ; SSE2-NEXT:    movmskps %xmm1, %eax
 ; SSE2-NEXT:    xorl $15, %eax
 ; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: icmp0_v4i64_v4i1:
 ; SSE41:       # %bb.0:
@@ -758,7 +837,7 @@ define i1 @icmp0_v4i64_v4i1(<4 x i64>) {
   ret i1 %b
 }
 
-define i1 @icmp0_v8i32_v8i1(<8 x i32>) {
+define i1 @icmp0_v8i32_v8i1(<8 x i32>) nounwind {
 ; SSE2-LABEL: icmp0_v8i32_v8i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    por %xmm1, %xmm0
@@ -767,7 +846,7 @@ define i1 @icmp0_v8i32_v8i1(<8 x i32>) {
 ; SSE2-NEXT:    movmskps %xmm1, %eax
 ; SSE2-NEXT:    xorl $15, %eax
 ; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: icmp0_v8i32_v8i1:
 ; SSE41:       # %bb.0:
@@ -787,7 +866,7 @@ define i1 @icmp0_v8i32_v8i1(<8 x i32>) {
   ret i1 %b
 }
 
-define i1 @icmp0_v16i16_v16i1(<16 x i16>) {
+define i1 @icmp0_v16i16_v16i1(<16 x i16>) nounwind {
 ; SSE2-LABEL: icmp0_v16i16_v16i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    por %xmm1, %xmm0
@@ -796,7 +875,7 @@ define i1 @icmp0_v16i16_v16i1(<16 x i16>) {
 ; SSE2-NEXT:    pmovmskb %xmm1, %eax
 ; SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
 ; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: icmp0_v16i16_v16i1:
 ; SSE41:       # %bb.0:
@@ -816,7 +895,7 @@ define i1 @icmp0_v16i16_v16i1(<16 x i16>) {
   ret i1 %b
 }
 
-define i1 @icmp0_v32i8_v32i1(<32 x i8>) {
+define i1 @icmp0_v32i8_v32i1(<32 x i8>) nounwind {
 ; SSE2-LABEL: icmp0_v32i8_v32i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    por %xmm1, %xmm0
@@ -825,7 +904,7 @@ define i1 @icmp0_v32i8_v32i1(<32 x i8>) {
 ; SSE2-NEXT:    pmovmskb %xmm1, %eax
 ; SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
 ; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: icmp0_v32i8_v32i1:
 ; SSE41:       # %bb.0:
@@ -845,18 +924,36 @@ define i1 @icmp0_v32i8_v32i1(<32 x i8>) {
   ret i1 %b
 }
 
-define i1 @icmp0_v8i64_v8i1(<8 x i64>) {
-; SSE2-LABEL: icmp0_v8i64_v8i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    por %xmm3, %xmm1
-; SSE2-NEXT:    por %xmm2, %xmm0
-; SSE2-NEXT:    por %xmm1, %xmm0
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE2-NEXT:    movmskps %xmm1, %eax
-; SSE2-NEXT:    xorl $15, %eax
-; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+define i1 @icmp0_v8i64_v8i1(<8 x i64>) nounwind {
+; X86-SSE2-LABEL: icmp0_v8i64_v8i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    por %xmm2, %xmm0
+; X86-SSE2-NEXT:    por 8(%ebp), %xmm1
+; X86-SSE2-NEXT:    por %xmm0, %xmm1
+; X86-SSE2-NEXT:    pxor %xmm0, %xmm0
+; X86-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; X86-SSE2-NEXT:    movmskps %xmm0, %eax
+; X86-SSE2-NEXT:    xorl $15, %eax
+; X86-SSE2-NEXT:    sete %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE2-LABEL: icmp0_v8i64_v8i1:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    por %xmm3, %xmm1
+; X64-SSE2-NEXT:    por %xmm2, %xmm0
+; X64-SSE2-NEXT:    por %xmm1, %xmm0
+; X64-SSE2-NEXT:    pxor %xmm1, %xmm1
+; X64-SSE2-NEXT:    pcmpeqd %xmm0, %xmm1
+; X64-SSE2-NEXT:    movmskps %xmm1, %eax
+; X64-SSE2-NEXT:    xorl $15, %eax
+; X64-SSE2-NEXT:    sete %al
+; X64-SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: icmp0_v8i64_v8i1:
 ; SSE41:       # %bb.0:
@@ -895,18 +992,36 @@ define i1 @icmp0_v8i64_v8i1(<8 x i64>) {
   ret i1 %b
 }
 
-define i1 @icmp0_v16i32_v16i1(<16 x i32>) {
-; SSE2-LABEL: icmp0_v16i32_v16i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    por %xmm3, %xmm1
-; SSE2-NEXT:    por %xmm2, %xmm0
-; SSE2-NEXT:    por %xmm1, %xmm0
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE2-NEXT:    movmskps %xmm1, %eax
-; SSE2-NEXT:    xorl $15, %eax
-; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+define i1 @icmp0_v16i32_v16i1(<16 x i32>) nounwind {
+; X86-SSE2-LABEL: icmp0_v16i32_v16i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    por %xmm2, %xmm0
+; X86-SSE2-NEXT:    por 8(%ebp), %xmm1
+; X86-SSE2-NEXT:    por %xmm0, %xmm1
+; X86-SSE2-NEXT:    pxor %xmm0, %xmm0
+; X86-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; X86-SSE2-NEXT:    movmskps %xmm0, %eax
+; X86-SSE2-NEXT:    xorl $15, %eax
+; X86-SSE2-NEXT:    sete %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE2-LABEL: icmp0_v16i32_v16i1:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    por %xmm3, %xmm1
+; X64-SSE2-NEXT:    por %xmm2, %xmm0
+; X64-SSE2-NEXT:    por %xmm1, %xmm0
+; X64-SSE2-NEXT:    pxor %xmm1, %xmm1
+; X64-SSE2-NEXT:    pcmpeqd %xmm0, %xmm1
+; X64-SSE2-NEXT:    movmskps %xmm1, %eax
+; X64-SSE2-NEXT:    xorl $15, %eax
+; X64-SSE2-NEXT:    sete %al
+; X64-SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: icmp0_v16i32_v16i1:
 ; SSE41:       # %bb.0:
@@ -945,18 +1060,36 @@ define i1 @icmp0_v16i32_v16i1(<16 x i32>) {
   ret i1 %b
 }
 
-define i1 @icmp0_v32i16_v32i1(<32 x i16>) {
-; SSE2-LABEL: icmp0_v32i16_v32i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    por %xmm3, %xmm1
-; SSE2-NEXT:    por %xmm2, %xmm0
-; SSE2-NEXT:    por %xmm1, %xmm0
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm0, %xmm1
-; SSE2-NEXT:    pmovmskb %xmm1, %eax
-; SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
-; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+define i1 @icmp0_v32i16_v32i1(<32 x i16>) nounwind {
+; X86-SSE2-LABEL: icmp0_v32i16_v32i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    por %xmm2, %xmm0
+; X86-SSE2-NEXT:    por 8(%ebp), %xmm1
+; X86-SSE2-NEXT:    por %xmm0, %xmm1
+; X86-SSE2-NEXT:    pxor %xmm0, %xmm0
+; X86-SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
+; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X86-SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
+; X86-SSE2-NEXT:    sete %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE2-LABEL: icmp0_v32i16_v32i1:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    por %xmm3, %xmm1
+; X64-SSE2-NEXT:    por %xmm2, %xmm0
+; X64-SSE2-NEXT:    por %xmm1, %xmm0
+; X64-SSE2-NEXT:    pxor %xmm1, %xmm1
+; X64-SSE2-NEXT:    pcmpeqb %xmm0, %xmm1
+; X64-SSE2-NEXT:    pmovmskb %xmm1, %eax
+; X64-SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
+; X64-SSE2-NEXT:    sete %al
+; X64-SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: icmp0_v32i16_v32i1:
 ; SSE41:       # %bb.0:
@@ -995,18 +1128,36 @@ define i1 @icmp0_v32i16_v32i1(<32 x i16>) {
   ret i1 %b
 }
 
-define i1 @icmp0_v64i8_v64i1(<64 x i8>) {
-; SSE2-LABEL: icmp0_v64i8_v64i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    por %xmm3, %xmm1
-; SSE2-NEXT:    por %xmm2, %xmm0
-; SSE2-NEXT:    por %xmm1, %xmm0
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm0, %xmm1
-; SSE2-NEXT:    pmovmskb %xmm1, %eax
-; SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
-; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+define i1 @icmp0_v64i8_v64i1(<64 x i8>) nounwind {
+; X86-SSE2-LABEL: icmp0_v64i8_v64i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    por %xmm2, %xmm0
+; X86-SSE2-NEXT:    por 8(%ebp), %xmm1
+; X86-SSE2-NEXT:    por %xmm0, %xmm1
+; X86-SSE2-NEXT:    pxor %xmm0, %xmm0
+; X86-SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
+; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X86-SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
+; X86-SSE2-NEXT:    sete %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE2-LABEL: icmp0_v64i8_v64i1:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    por %xmm3, %xmm1
+; X64-SSE2-NEXT:    por %xmm2, %xmm0
+; X64-SSE2-NEXT:    por %xmm1, %xmm0
+; X64-SSE2-NEXT:    pxor %xmm1, %xmm1
+; X64-SSE2-NEXT:    pcmpeqb %xmm0, %xmm1
+; X64-SSE2-NEXT:    pmovmskb %xmm1, %eax
+; X64-SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
+; X64-SSE2-NEXT:    sete %al
+; X64-SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: icmp0_v64i8_v64i1:
 ; SSE41:       # %bb.0:
@@ -1045,7 +1196,7 @@ define i1 @icmp0_v64i8_v64i1(<64 x i8>) {
   ret i1 %b
 }
 
-define i8 @icmp0_v8i1(<8 x i8>) {
+define i8 @icmp0_v8i1(<8 x i8>) nounwind {
 ; SSE2-LABEL: icmp0_v8i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
@@ -1054,7 +1205,7 @@ define i8 @icmp0_v8i1(<8 x i8>) {
 ; SSE2-NEXT:    pmovmskb %xmm0, %eax
 ; SSE2-NEXT:    testl %eax, %eax
 ; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: icmp0_v8i1:
 ; SSE41:       # %bb.0:
@@ -1116,7 +1267,7 @@ define i8 @icmp0_v8i1(<8 x i8>) {
 ; Comparison With All Ones
 ;
 
-define i1 @icmp1_v2i64_v2i1(<2 x i64>) {
+define i1 @icmp1_v2i64_v2i1(<2 x i64>) nounwind {
 ; SSE2-LABEL: icmp1_v2i64_v2i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
@@ -1124,7 +1275,7 @@ define i1 @icmp1_v2i64_v2i1(<2 x i64>) {
 ; SSE2-NEXT:    movmskps %xmm1, %eax
 ; SSE2-NEXT:    xorl $15, %eax
 ; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: icmp1_v2i64_v2i1:
 ; SSE41:       # %bb.0:
@@ -1144,7 +1295,7 @@ define i1 @icmp1_v2i64_v2i1(<2 x i64>) {
   ret i1 %b
 }
 
-define i1 @icmp1_v4i32_v4i1(<4 x i32>) {
+define i1 @icmp1_v4i32_v4i1(<4 x i32>) nounwind {
 ; SSE2-LABEL: icmp1_v4i32_v4i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
@@ -1152,7 +1303,7 @@ define i1 @icmp1_v4i32_v4i1(<4 x i32>) {
 ; SSE2-NEXT:    movmskps %xmm1, %eax
 ; SSE2-NEXT:    xorl $15, %eax
 ; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: icmp1_v4i32_v4i1:
 ; SSE41:       # %bb.0:
@@ -1172,7 +1323,7 @@ define i1 @icmp1_v4i32_v4i1(<4 x i32>) {
   ret i1 %b
 }
 
-define i1 @icmp1_v8i16_v8i1(<8 x i16>) {
+define i1 @icmp1_v8i16_v8i1(<8 x i16>) nounwind {
 ; SSE2-LABEL: icmp1_v8i16_v8i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
@@ -1180,7 +1331,7 @@ define i1 @icmp1_v8i16_v8i1(<8 x i16>) {
 ; SSE2-NEXT:    pmovmskb %xmm1, %eax
 ; SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
 ; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: icmp1_v8i16_v8i1:
 ; SSE41:       # %bb.0:
@@ -1200,7 +1351,7 @@ define i1 @icmp1_v8i16_v8i1(<8 x i16>) {
   ret i1 %b
 }
 
-define i1 @icmp1_v16i8_v16i1(<16 x i8>) {
+define i1 @icmp1_v16i8_v16i1(<16 x i8>) nounwind {
 ; SSE2-LABEL: icmp1_v16i8_v16i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
@@ -1208,7 +1359,7 @@ define i1 @icmp1_v16i8_v16i1(<16 x i8>) {
 ; SSE2-NEXT:    pmovmskb %xmm1, %eax
 ; SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
 ; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: icmp1_v16i8_v16i1:
 ; SSE41:       # %bb.0:
@@ -1228,7 +1379,7 @@ define i1 @icmp1_v16i8_v16i1(<16 x i8>) {
   ret i1 %b
 }
 
-define i1 @icmp1_v4i64_v4i1(<4 x i64>) {
+define i1 @icmp1_v4i64_v4i1(<4 x i64>) nounwind {
 ; SSE2-LABEL: icmp1_v4i64_v4i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pand %xmm1, %xmm0
@@ -1237,7 +1388,7 @@ define i1 @icmp1_v4i64_v4i1(<4 x i64>) {
 ; SSE2-NEXT:    movmskps %xmm1, %eax
 ; SSE2-NEXT:    xorl $15, %eax
 ; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: icmp1_v4i64_v4i1:
 ; SSE41:       # %bb.0:
@@ -1276,7 +1427,7 @@ define i1 @icmp1_v4i64_v4i1(<4 x i64>) {
   ret i1 %b
 }
 
-define i1 @icmp1_v8i32_v8i1(<8 x i32>) {
+define i1 @icmp1_v8i32_v8i1(<8 x i32>) nounwind {
 ; SSE2-LABEL: icmp1_v8i32_v8i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pand %xmm1, %xmm0
@@ -1285,7 +1436,7 @@ define i1 @icmp1_v8i32_v8i1(<8 x i32>) {
 ; SSE2-NEXT:    movmskps %xmm1, %eax
 ; SSE2-NEXT:    xorl $15, %eax
 ; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: icmp1_v8i32_v8i1:
 ; SSE41:       # %bb.0:
@@ -1324,7 +1475,7 @@ define i1 @icmp1_v8i32_v8i1(<8 x i32>) {
   ret i1 %b
 }
 
-define i1 @icmp1_v16i16_v16i1(<16 x i16>) {
+define i1 @icmp1_v16i16_v16i1(<16 x i16>) nounwind {
 ; SSE2-LABEL: icmp1_v16i16_v16i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pand %xmm1, %xmm0
@@ -1333,7 +1484,7 @@ define i1 @icmp1_v16i16_v16i1(<16 x i16>) {
 ; SSE2-NEXT:    pmovmskb %xmm1, %eax
 ; SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
 ; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: icmp1_v16i16_v16i1:
 ; SSE41:       # %bb.0:
@@ -1372,7 +1523,7 @@ define i1 @icmp1_v16i16_v16i1(<16 x i16>) {
   ret i1 %b
 }
 
-define i1 @icmp1_v32i8_v32i1(<32 x i8>) {
+define i1 @icmp1_v32i8_v32i1(<32 x i8>) nounwind {
 ; SSE2-LABEL: icmp1_v32i8_v32i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pand %xmm1, %xmm0
@@ -1381,7 +1532,7 @@ define i1 @icmp1_v32i8_v32i1(<32 x i8>) {
 ; SSE2-NEXT:    pmovmskb %xmm1, %eax
 ; SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
 ; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: icmp1_v32i8_v32i1:
 ; SSE41:       # %bb.0:
@@ -1420,18 +1571,36 @@ define i1 @icmp1_v32i8_v32i1(<32 x i8>) {
   ret i1 %b
 }
 
-define i1 @icmp1_v8i64_v8i1(<8 x i64>) {
-; SSE2-LABEL: icmp1_v8i64_v8i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE2-NEXT:    movmskps %xmm1, %eax
-; SSE2-NEXT:    xorl $15, %eax
-; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+define i1 @icmp1_v8i64_v8i1(<8 x i64>) nounwind {
+; X86-SSE2-LABEL: icmp1_v8i64_v8i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    pand %xmm2, %xmm0
+; X86-SSE2-NEXT:    pand 8(%ebp), %xmm1
+; X86-SSE2-NEXT:    pand %xmm0, %xmm1
+; X86-SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
+; X86-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; X86-SSE2-NEXT:    movmskps %xmm0, %eax
+; X86-SSE2-NEXT:    xorl $15, %eax
+; X86-SSE2-NEXT:    sete %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE2-LABEL: icmp1_v8i64_v8i1:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    pand %xmm3, %xmm1
+; X64-SSE2-NEXT:    pand %xmm2, %xmm0
+; X64-SSE2-NEXT:    pand %xmm1, %xmm0
+; X64-SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
+; X64-SSE2-NEXT:    pcmpeqd %xmm0, %xmm1
+; X64-SSE2-NEXT:    movmskps %xmm1, %eax
+; X64-SSE2-NEXT:    xorl $15, %eax
+; X64-SSE2-NEXT:    sete %al
+; X64-SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: icmp1_v8i64_v8i1:
 ; SSE41:       # %bb.0:
@@ -1475,18 +1644,36 @@ define i1 @icmp1_v8i64_v8i1(<8 x i64>) {
   ret i1 %b
 }
 
-define i1 @icmp1_v16i32_v16i1(<16 x i32>) {
-; SSE2-LABEL: icmp1_v16i32_v16i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE2-NEXT:    movmskps %xmm1, %eax
-; SSE2-NEXT:    xorl $15, %eax
-; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+define i1 @icmp1_v16i32_v16i1(<16 x i32>) nounwind {
+; X86-SSE2-LABEL: icmp1_v16i32_v16i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    pand %xmm2, %xmm0
+; X86-SSE2-NEXT:    pand 8(%ebp), %xmm1
+; X86-SSE2-NEXT:    pand %xmm0, %xmm1
+; X86-SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
+; X86-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; X86-SSE2-NEXT:    movmskps %xmm0, %eax
+; X86-SSE2-NEXT:    xorl $15, %eax
+; X86-SSE2-NEXT:    sete %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE2-LABEL: icmp1_v16i32_v16i1:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    pand %xmm3, %xmm1
+; X64-SSE2-NEXT:    pand %xmm2, %xmm0
+; X64-SSE2-NEXT:    pand %xmm1, %xmm0
+; X64-SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
+; X64-SSE2-NEXT:    pcmpeqd %xmm0, %xmm1
+; X64-SSE2-NEXT:    movmskps %xmm1, %eax
+; X64-SSE2-NEXT:    xorl $15, %eax
+; X64-SSE2-NEXT:    sete %al
+; X64-SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: icmp1_v16i32_v16i1:
 ; SSE41:       # %bb.0:
@@ -1530,18 +1717,36 @@ define i1 @icmp1_v16i32_v16i1(<16 x i32>) {
   ret i1 %b
 }
 
-define i1 @icmp1_v32i16_v32i1(<32 x i16>) {
-; SSE2-LABEL: icmp1_v32i16_v32i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm0, %xmm1
-; SSE2-NEXT:    pmovmskb %xmm1, %eax
-; SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
-; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+define i1 @icmp1_v32i16_v32i1(<32 x i16>) nounwind {
+; X86-SSE2-LABEL: icmp1_v32i16_v32i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    pand %xmm2, %xmm0
+; X86-SSE2-NEXT:    pand 8(%ebp), %xmm1
+; X86-SSE2-NEXT:    pand %xmm0, %xmm1
+; X86-SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
+; X86-SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
+; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X86-SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
+; X86-SSE2-NEXT:    sete %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE2-LABEL: icmp1_v32i16_v32i1:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    pand %xmm3, %xmm1
+; X64-SSE2-NEXT:    pand %xmm2, %xmm0
+; X64-SSE2-NEXT:    pand %xmm1, %xmm0
+; X64-SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
+; X64-SSE2-NEXT:    pcmpeqb %xmm0, %xmm1
+; X64-SSE2-NEXT:    pmovmskb %xmm1, %eax
+; X64-SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
+; X64-SSE2-NEXT:    sete %al
+; X64-SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: icmp1_v32i16_v32i1:
 ; SSE41:       # %bb.0:
@@ -1585,18 +1790,36 @@ define i1 @icmp1_v32i16_v32i1(<32 x i16>) {
   ret i1 %b
 }
 
-define i1 @icmp1_v64i8_v64i1(<64 x i8>) {
-; SSE2-LABEL: icmp1_v64i8_v64i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm0, %xmm1
-; SSE2-NEXT:    pmovmskb %xmm1, %eax
-; SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
-; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+define i1 @icmp1_v64i8_v64i1(<64 x i8>) nounwind {
+; X86-SSE2-LABEL: icmp1_v64i8_v64i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    pand %xmm2, %xmm0
+; X86-SSE2-NEXT:    pand 8(%ebp), %xmm1
+; X86-SSE2-NEXT:    pand %xmm0, %xmm1
+; X86-SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
+; X86-SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
+; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X86-SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
+; X86-SSE2-NEXT:    sete %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE2-LABEL: icmp1_v64i8_v64i1:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    pand %xmm3, %xmm1
+; X64-SSE2-NEXT:    pand %xmm2, %xmm0
+; X64-SSE2-NEXT:    pand %xmm1, %xmm0
+; X64-SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
+; X64-SSE2-NEXT:    pcmpeqb %xmm0, %xmm1
+; X64-SSE2-NEXT:    pmovmskb %xmm1, %eax
+; X64-SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
+; X64-SSE2-NEXT:    sete %al
+; X64-SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: icmp1_v64i8_v64i1:
 ; SSE41:       # %bb.0:
@@ -1640,7 +1863,7 @@ define i1 @icmp1_v64i8_v64i1(<64 x i8>) {
   ret i1 %b
 }
 
-define i8 @icmp1_v8i1(<8 x i8>) {
+define i8 @icmp1_v8i1(<8 x i8>) nounwind {
 ; SSE2-LABEL: icmp1_v8i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
@@ -1649,7 +1872,7 @@ define i8 @icmp1_v8i1(<8 x i8>) {
 ; SSE2-NEXT:    pmovmskb %xmm0, %eax
 ; SSE2-NEXT:    cmpb $-1, %al
 ; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: icmp1_v8i1:
 ; SSE41:       # %bb.0:
@@ -1711,14 +1934,14 @@ define i8 @icmp1_v8i1(<8 x i8>) {
 ; Comparison
 ;
 
-define i1 @icmp_v2i64_v2i1(<2 x i64>, <2 x i64>) {
+define i1 @icmp_v2i64_v2i1(<2 x i64>, <2 x i64>) nounwind {
 ; SSE2-LABEL: icmp_v2i64_v2i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
 ; SSE2-NEXT:    movmskps %xmm0, %eax
 ; SSE2-NEXT:    xorl $15, %eax
 ; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: icmp_v2i64_v2i1:
 ; SSE41:       # %bb.0:
@@ -1738,14 +1961,14 @@ define i1 @icmp_v2i64_v2i1(<2 x i64>, <2 x i64>) {
   ret i1 %b
 }
 
-define i1 @icmp_v4i32_v4i1(<4 x i32>, <4 x i32>) {
+define i1 @icmp_v4i32_v4i1(<4 x i32>, <4 x i32>) nounwind {
 ; SSE2-LABEL: icmp_v4i32_v4i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
 ; SSE2-NEXT:    movmskps %xmm0, %eax
 ; SSE2-NEXT:    xorl $15, %eax
 ; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: icmp_v4i32_v4i1:
 ; SSE41:       # %bb.0:
@@ -1765,14 +1988,14 @@ define i1 @icmp_v4i32_v4i1(<4 x i32>, <4 x i32>) {
   ret i1 %b
 }
 
-define i1 @icmp_v8i16_v8i1(<8 x i16>, <8 x i16>) {
+define i1 @icmp_v8i16_v8i1(<8 x i16>, <8 x i16>) nounwind {
 ; SSE2-LABEL: icmp_v8i16_v8i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
 ; SSE2-NEXT:    pmovmskb %xmm0, %eax
 ; SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
 ; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: icmp_v8i16_v8i1:
 ; SSE41:       # %bb.0:
@@ -1792,14 +2015,14 @@ define i1 @icmp_v8i16_v8i1(<8 x i16>, <8 x i16>) {
   ret i1 %b
 }
 
-define i1 @icmp_v16i8_v16i1(<16 x i8>, <16 x i8>) {
+define i1 @icmp_v16i8_v16i1(<16 x i8>, <16 x i8>) nounwind {
 ; SSE2-LABEL: icmp_v16i8_v16i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
 ; SSE2-NEXT:    pmovmskb %xmm0, %eax
 ; SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
 ; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: icmp_v16i8_v16i1:
 ; SSE41:       # %bb.0:
@@ -1819,16 +2042,32 @@ define i1 @icmp_v16i8_v16i1(<16 x i8>, <16 x i8>) {
   ret i1 %b
 }
 
-define i1 @icmp_v4i64_v4i1(<4 x i64>, <4 x i64>) {
-; SSE2-LABEL: icmp_v4i64_v4i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    movmskps %xmm0, %eax
-; SSE2-NEXT:    xorl $15, %eax
-; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+define i1 @icmp_v4i64_v4i1(<4 x i64>, <4 x i64>) nounwind {
+; X86-SSE2-LABEL: icmp_v4i64_v4i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
+; X86-SSE2-NEXT:    pcmpeqd 8(%ebp), %xmm1
+; X86-SSE2-NEXT:    pand %xmm0, %xmm1
+; X86-SSE2-NEXT:    movmskps %xmm1, %eax
+; X86-SSE2-NEXT:    xorl $15, %eax
+; X86-SSE2-NEXT:    sete %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE2-LABEL: icmp_v4i64_v4i1:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    pcmpeqd %xmm3, %xmm1
+; X64-SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
+; X64-SSE2-NEXT:    pand %xmm1, %xmm0
+; X64-SSE2-NEXT:    movmskps %xmm0, %eax
+; X64-SSE2-NEXT:    xorl $15, %eax
+; X64-SSE2-NEXT:    sete %al
+; X64-SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: icmp_v4i64_v4i1:
 ; SSE41:       # %bb.0:
@@ -1867,16 +2106,32 @@ define i1 @icmp_v4i64_v4i1(<4 x i64>, <4 x i64>) {
   ret i1 %b
 }
 
-define i1 @icmp_v8i32_v8i1(<8 x i32>, <8 x i32>) {
-; SSE2-LABEL: icmp_v8i32_v8i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    movmskps %xmm0, %eax
-; SSE2-NEXT:    xorl $15, %eax
-; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+define i1 @icmp_v8i32_v8i1(<8 x i32>, <8 x i32>) nounwind {
+; X86-SSE2-LABEL: icmp_v8i32_v8i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
+; X86-SSE2-NEXT:    pcmpeqd 8(%ebp), %xmm1
+; X86-SSE2-NEXT:    pand %xmm0, %xmm1
+; X86-SSE2-NEXT:    movmskps %xmm1, %eax
+; X86-SSE2-NEXT:    xorl $15, %eax
+; X86-SSE2-NEXT:    sete %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE2-LABEL: icmp_v8i32_v8i1:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    pcmpeqd %xmm3, %xmm1
+; X64-SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
+; X64-SSE2-NEXT:    pand %xmm1, %xmm0
+; X64-SSE2-NEXT:    movmskps %xmm0, %eax
+; X64-SSE2-NEXT:    xorl $15, %eax
+; X64-SSE2-NEXT:    sete %al
+; X64-SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: icmp_v8i32_v8i1:
 ; SSE41:       # %bb.0:
@@ -1915,16 +2170,32 @@ define i1 @icmp_v8i32_v8i1(<8 x i32>, <8 x i32>) {
   ret i1 %b
 }
 
-define i1 @icmp_v16i16_v16i1(<16 x i16>, <16 x i16>) {
-; SSE2-LABEL: icmp_v16i16_v16i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pcmpeqb %xmm3, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm2, %xmm0
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    pmovmskb %xmm0, %eax
-; SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
-; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+define i1 @icmp_v16i16_v16i1(<16 x i16>, <16 x i16>) nounwind {
+; X86-SSE2-LABEL: icmp_v16i16_v16i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    pcmpeqb %xmm2, %xmm0
+; X86-SSE2-NEXT:    pcmpeqb 8(%ebp), %xmm1
+; X86-SSE2-NEXT:    pand %xmm0, %xmm1
+; X86-SSE2-NEXT:    pmovmskb %xmm1, %eax
+; X86-SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
+; X86-SSE2-NEXT:    sete %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE2-LABEL: icmp_v16i16_v16i1:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    pcmpeqb %xmm3, %xmm1
+; X64-SSE2-NEXT:    pcmpeqb %xmm2, %xmm0
+; X64-SSE2-NEXT:    pand %xmm1, %xmm0
+; X64-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X64-SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
+; X64-SSE2-NEXT:    sete %al
+; X64-SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: icmp_v16i16_v16i1:
 ; SSE41:       # %bb.0:
@@ -1963,16 +2234,32 @@ define i1 @icmp_v16i16_v16i1(<16 x i16>, <16 x i16>) {
   ret i1 %b
 }
 
-define i1 @icmp_v32i8_v32i1(<32 x i8>, <32 x i8>) {
-; SSE2-LABEL: icmp_v32i8_v32i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pcmpeqb %xmm3, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm2, %xmm0
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    pmovmskb %xmm0, %eax
-; SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
-; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+define i1 @icmp_v32i8_v32i1(<32 x i8>, <32 x i8>) nounwind {
+; X86-SSE2-LABEL: icmp_v32i8_v32i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    pcmpeqb %xmm2, %xmm0
+; X86-SSE2-NEXT:    pcmpeqb 8(%ebp), %xmm1
+; X86-SSE2-NEXT:    pand %xmm0, %xmm1
+; X86-SSE2-NEXT:    pmovmskb %xmm1, %eax
+; X86-SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
+; X86-SSE2-NEXT:    sete %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE2-LABEL: icmp_v32i8_v32i1:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    pcmpeqb %xmm3, %xmm1
+; X64-SSE2-NEXT:    pcmpeqb %xmm2, %xmm0
+; X64-SSE2-NEXT:    pand %xmm1, %xmm0
+; X64-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X64-SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
+; X64-SSE2-NEXT:    sete %al
+; X64-SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: icmp_v32i8_v32i1:
 ; SSE41:       # %bb.0:
@@ -2011,20 +2298,41 @@ define i1 @icmp_v32i8_v32i1(<32 x i8>, <32 x i8>) {
   ret i1 %b
 }
 
-define i1 @icmp_v8i64_v8i1(<8 x i64>, <8 x i64>) {
-; SSE2-LABEL: icmp_v8i64_v8i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pcmpeqd %xmm7, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm1
-; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    movmskps %xmm0, %eax
-; SSE2-NEXT:    xorl $15, %eax
-; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+define i1 @icmp_v8i64_v8i1(<8 x i64>, <8 x i64>) nounwind {
+; X86-SSE2-LABEL: icmp_v8i64_v8i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    movdqa 8(%ebp), %xmm3
+; X86-SSE2-NEXT:    pcmpeqd 72(%ebp), %xmm3
+; X86-SSE2-NEXT:    pcmpeqd 40(%ebp), %xmm1
+; X86-SSE2-NEXT:    pand %xmm3, %xmm1
+; X86-SSE2-NEXT:    pcmpeqd 56(%ebp), %xmm2
+; X86-SSE2-NEXT:    pcmpeqd 24(%ebp), %xmm0
+; X86-SSE2-NEXT:    pand %xmm2, %xmm0
+; X86-SSE2-NEXT:    pand %xmm1, %xmm0
+; X86-SSE2-NEXT:    movmskps %xmm0, %eax
+; X86-SSE2-NEXT:    xorl $15, %eax
+; X86-SSE2-NEXT:    sete %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE2-LABEL: icmp_v8i64_v8i1:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    pcmpeqd %xmm7, %xmm3
+; X64-SSE2-NEXT:    pcmpeqd %xmm5, %xmm1
+; X64-SSE2-NEXT:    pand %xmm3, %xmm1
+; X64-SSE2-NEXT:    pcmpeqd %xmm6, %xmm2
+; X64-SSE2-NEXT:    pcmpeqd %xmm4, %xmm0
+; X64-SSE2-NEXT:    pand %xmm2, %xmm0
+; X64-SSE2-NEXT:    pand %xmm1, %xmm0
+; X64-SSE2-NEXT:    movmskps %xmm0, %eax
+; X64-SSE2-NEXT:    xorl $15, %eax
+; X64-SSE2-NEXT:    sete %al
+; X64-SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: icmp_v8i64_v8i1:
 ; SSE41:       # %bb.0:
@@ -2071,20 +2379,41 @@ define i1 @icmp_v8i64_v8i1(<8 x i64>, <8 x i64>) {
   ret i1 %b
 }
 
-define i1 @icmp_v16i32_v16i1(<16 x i32>, <16 x i32>) {
-; SSE2-LABEL: icmp_v16i32_v16i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pcmpeqd %xmm7, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm1
-; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    movmskps %xmm0, %eax
-; SSE2-NEXT:    xorl $15, %eax
-; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+define i1 @icmp_v16i32_v16i1(<16 x i32>, <16 x i32>) nounwind {
+; X86-SSE2-LABEL: icmp_v16i32_v16i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    movdqa 8(%ebp), %xmm3
+; X86-SSE2-NEXT:    pcmpeqd 72(%ebp), %xmm3
+; X86-SSE2-NEXT:    pcmpeqd 40(%ebp), %xmm1
+; X86-SSE2-NEXT:    pand %xmm3, %xmm1
+; X86-SSE2-NEXT:    pcmpeqd 56(%ebp), %xmm2
+; X86-SSE2-NEXT:    pcmpeqd 24(%ebp), %xmm0
+; X86-SSE2-NEXT:    pand %xmm2, %xmm0
+; X86-SSE2-NEXT:    pand %xmm1, %xmm0
+; X86-SSE2-NEXT:    movmskps %xmm0, %eax
+; X86-SSE2-NEXT:    xorl $15, %eax
+; X86-SSE2-NEXT:    sete %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE2-LABEL: icmp_v16i32_v16i1:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    pcmpeqd %xmm7, %xmm3
+; X64-SSE2-NEXT:    pcmpeqd %xmm5, %xmm1
+; X64-SSE2-NEXT:    pand %xmm3, %xmm1
+; X64-SSE2-NEXT:    pcmpeqd %xmm6, %xmm2
+; X64-SSE2-NEXT:    pcmpeqd %xmm4, %xmm0
+; X64-SSE2-NEXT:    pand %xmm2, %xmm0
+; X64-SSE2-NEXT:    pand %xmm1, %xmm0
+; X64-SSE2-NEXT:    movmskps %xmm0, %eax
+; X64-SSE2-NEXT:    xorl $15, %eax
+; X64-SSE2-NEXT:    sete %al
+; X64-SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: icmp_v16i32_v16i1:
 ; SSE41:       # %bb.0:
@@ -2131,20 +2460,41 @@ define i1 @icmp_v16i32_v16i1(<16 x i32>, <16 x i32>) {
   ret i1 %b
 }
 
-define i1 @icmp_v32i16_v32i1(<32 x i16>, <32 x i16>) {
-; SSE2-LABEL: icmp_v32i16_v32i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pcmpeqb %xmm7, %xmm3
-; SSE2-NEXT:    pcmpeqb %xmm5, %xmm1
-; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm6, %xmm2
-; SSE2-NEXT:    pcmpeqb %xmm4, %xmm0
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    pmovmskb %xmm0, %eax
-; SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
-; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+define i1 @icmp_v32i16_v32i1(<32 x i16>, <32 x i16>) nounwind {
+; X86-SSE2-LABEL: icmp_v32i16_v32i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    movdqa 8(%ebp), %xmm3
+; X86-SSE2-NEXT:    pcmpeqb 72(%ebp), %xmm3
+; X86-SSE2-NEXT:    pcmpeqb 40(%ebp), %xmm1
+; X86-SSE2-NEXT:    pand %xmm3, %xmm1
+; X86-SSE2-NEXT:    pcmpeqb 56(%ebp), %xmm2
+; X86-SSE2-NEXT:    pcmpeqb 24(%ebp), %xmm0
+; X86-SSE2-NEXT:    pand %xmm2, %xmm0
+; X86-SSE2-NEXT:    pand %xmm1, %xmm0
+; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X86-SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
+; X86-SSE2-NEXT:    sete %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE2-LABEL: icmp_v32i16_v32i1:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    pcmpeqb %xmm7, %xmm3
+; X64-SSE2-NEXT:    pcmpeqb %xmm5, %xmm1
+; X64-SSE2-NEXT:    pand %xmm3, %xmm1
+; X64-SSE2-NEXT:    pcmpeqb %xmm6, %xmm2
+; X64-SSE2-NEXT:    pcmpeqb %xmm4, %xmm0
+; X64-SSE2-NEXT:    pand %xmm2, %xmm0
+; X64-SSE2-NEXT:    pand %xmm1, %xmm0
+; X64-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X64-SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
+; X64-SSE2-NEXT:    sete %al
+; X64-SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: icmp_v32i16_v32i1:
 ; SSE41:       # %bb.0:
@@ -2191,20 +2541,41 @@ define i1 @icmp_v32i16_v32i1(<32 x i16>, <32 x i16>) {
   ret i1 %b
 }
 
-define i1 @icmp_v64i8_v64i1(<64 x i8>, <64 x i8>) {
-; SSE2-LABEL: icmp_v64i8_v64i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pcmpeqb %xmm7, %xmm3
-; SSE2-NEXT:    pcmpeqb %xmm5, %xmm1
-; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    pcmpeqb %xmm6, %xmm2
-; SSE2-NEXT:    pcmpeqb %xmm4, %xmm0
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    pmovmskb %xmm0, %eax
-; SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
-; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
+define i1 @icmp_v64i8_v64i1(<64 x i8>, <64 x i8>) nounwind {
+; X86-SSE2-LABEL: icmp_v64i8_v64i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    movdqa 8(%ebp), %xmm3
+; X86-SSE2-NEXT:    pcmpeqb 72(%ebp), %xmm3
+; X86-SSE2-NEXT:    pcmpeqb 40(%ebp), %xmm1
+; X86-SSE2-NEXT:    pand %xmm3, %xmm1
+; X86-SSE2-NEXT:    pcmpeqb 56(%ebp), %xmm2
+; X86-SSE2-NEXT:    pcmpeqb 24(%ebp), %xmm0
+; X86-SSE2-NEXT:    pand %xmm2, %xmm0
+; X86-SSE2-NEXT:    pand %xmm1, %xmm0
+; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X86-SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
+; X86-SSE2-NEXT:    sete %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE2-LABEL: icmp_v64i8_v64i1:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    pcmpeqb %xmm7, %xmm3
+; X64-SSE2-NEXT:    pcmpeqb %xmm5, %xmm1
+; X64-SSE2-NEXT:    pand %xmm3, %xmm1
+; X64-SSE2-NEXT:    pcmpeqb %xmm6, %xmm2
+; X64-SSE2-NEXT:    pcmpeqb %xmm4, %xmm0
+; X64-SSE2-NEXT:    pand %xmm2, %xmm0
+; X64-SSE2-NEXT:    pand %xmm1, %xmm0
+; X64-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X64-SSE2-NEXT:    xorl $65535, %eax # imm = 0xFFFF
+; X64-SSE2-NEXT:    sete %al
+; X64-SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: icmp_v64i8_v64i1:
 ; SSE41:       # %bb.0:
@@ -2259,3 +2630,4 @@ declare i1 @llvm.vector.reduce.and.v32i1(<32 x i1>)
 declare i1 @llvm.vector.reduce.and.v64i1(<64 x i1>)
 ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
 ; SSE: {{.*}}
+; X64-SSE: {{.*}}

diff --git a/llvm/test/CodeGen/X86/vector-reduce-or-bool.ll b/llvm/test/CodeGen/X86/vector-reduce-or-bool.ll
index bb633a3..c5cb55a 100644 (file)
--- a/llvm/test/CodeGen/X86/vector-reduce-or-bool.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-or-bool.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE,SSE41
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE2,X86-SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE2,X64-SSE,X64-SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE,X64-SSE,SSE41
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1OR2,AVX1
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX1OR2,AVX2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX,AVX512,AVX512F
@@ -11,14 +12,14 @@
 ; Truncate
 ;
 
-define i1 @trunc_v2i64_v2i1(<2 x i64>) {
+define i1 @trunc_v2i64_v2i1(<2 x i64>) nounwind {
 ; SSE2-LABEL: trunc_v2i64_v2i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    psllq $63, %xmm0
 ; SSE2-NEXT:    movmskpd %xmm0, %eax
 ; SSE2-NEXT:    testl %eax, %eax
 ; SSE2-NEXT:    setne %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: trunc_v2i64_v2i1:
 ; SSE41:       # %bb.0:
@@ -55,14 +56,14 @@ define i1 @trunc_v2i64_v2i1(<2 x i64>) {
   ret i1 %b
 }
 
-define i1 @trunc_v4i32_v4i1(<4 x i32>) {
+define i1 @trunc_v4i32_v4i1(<4 x i32>) nounwind {
 ; SSE2-LABEL: trunc_v4i32_v4i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pslld $31, %xmm0
 ; SSE2-NEXT:    movmskps %xmm0, %eax
 ; SSE2-NEXT:    testl %eax, %eax
 ; SSE2-NEXT:    setne %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: trunc_v4i32_v4i1:
 ; SSE41:       # %bb.0:
@@ -99,14 +100,14 @@ define i1 @trunc_v4i32_v4i1(<4 x i32>) {
   ret i1 %b
 }
 
-define i1 @trunc_v8i16_v8i1(<8 x i16>) {
+define i1 @trunc_v8i16_v8i1(<8 x i16>) nounwind {
 ; SSE2-LABEL: trunc_v8i16_v8i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    psllw $7, %xmm0
 ; SSE2-NEXT:    pmovmskb %xmm0, %eax
 ; SSE2-NEXT:    testl $21845, %eax # imm = 0x5555
 ; SSE2-NEXT:    setne %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: trunc_v8i16_v8i1:
 ; SSE41:       # %bb.0:
@@ -143,14 +144,14 @@ define i1 @trunc_v8i16_v8i1(<8 x i16>) {
   ret i1 %b
 }
 
-define i1 @trunc_v16i8_v16i1(<16 x i8>) {
+define i1 @trunc_v16i8_v16i1(<16 x i8>) nounwind {
 ; SSE2-LABEL: trunc_v16i8_v16i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    psllw $7, %xmm0
 ; SSE2-NEXT:    pmovmskb %xmm0, %eax
 ; SSE2-NEXT:    testl %eax, %eax
 ; SSE2-NEXT:    setne %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: trunc_v16i8_v16i1:
 ; SSE41:       # %bb.0:
@@ -187,7 +188,7 @@ define i1 @trunc_v16i8_v16i1(<16 x i8>) {
   ret i1 %b
 }
 
-define i1 @trunc_v4i64_v4i1(<4 x i64>) {
+define i1 @trunc_v4i64_v4i1(<4 x i64>) nounwind {
 ; SSE2-LABEL: trunc_v4i64_v4i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
@@ -195,7 +196,7 @@ define i1 @trunc_v4i64_v4i1(<4 x i64>) {
 ; SSE2-NEXT:    movmskps %xmm0, %eax
 ; SSE2-NEXT:    testl %eax, %eax
 ; SSE2-NEXT:    setne %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: trunc_v4i64_v4i1:
 ; SSE41:       # %bb.0:
@@ -231,7 +232,7 @@ define i1 @trunc_v4i64_v4i1(<4 x i64>) {
   ret i1 %b
 }
 
-define i1 @trunc_v8i32_v8i1(<8 x i32>) {
+define i1 @trunc_v8i32_v8i1(<8 x i32>) nounwind {
 ; SSE2-LABEL: trunc_v8i32_v8i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    por %xmm1, %xmm0
@@ -239,7 +240,7 @@ define i1 @trunc_v8i32_v8i1(<8 x i32>) {
 ; SSE2-NEXT:    movmskps %xmm0, %eax
 ; SSE2-NEXT:    testl %eax, %eax
 ; SSE2-NEXT:    setne %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: trunc_v8i32_v8i1:
 ; SSE41:       # %bb.0:
@@ -275,7 +276,7 @@ define i1 @trunc_v8i32_v8i1(<8 x i32>) {
   ret i1 %b
 }
 
-define i1 @trunc_v16i16_v16i1(<16 x i16>) {
+define i1 @trunc_v16i16_v16i1(<16 x i16>) nounwind {
 ; SSE2-LABEL: trunc_v16i16_v16i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    por %xmm1, %xmm0
@@ -283,7 +284,7 @@ define i1 @trunc_v16i16_v16i1(<16 x i16>) {
 ; SSE2-NEXT:    pmovmskb %xmm0, %eax
 ; SSE2-NEXT:    testl $21845, %eax # imm = 0x5555
 ; SSE2-NEXT:    setne %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: trunc_v16i16_v16i1:
 ; SSE41:       # %bb.0:
@@ -319,7 +320,7 @@ define i1 @trunc_v16i16_v16i1(<16 x i16>) {
   ret i1 %b
 }
 
-define i1 @trunc_v32i8_v32i1(<32 x i8>) {
+define i1 @trunc_v32i8_v32i1(<32 x i8>) nounwind {
 ; SSE2-LABEL: trunc_v32i8_v32i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    por %xmm1, %xmm0
@@ -327,7 +328,7 @@ define i1 @trunc_v32i8_v32i1(<32 x i8>) {
 ; SSE2-NEXT:    pmovmskb %xmm0, %eax
 ; SSE2-NEXT:    testl %eax, %eax
 ; SSE2-NEXT:    setne %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: trunc_v32i8_v32i1:
 ; SSE41:       # %bb.0:
@@ -363,25 +364,50 @@ define i1 @trunc_v32i8_v32i1(<32 x i8>) {
   ret i1 %b
 }
 
-define i1 @trunc_v8i64_v8i1(<8 x i64>) {
-; SSE2-LABEL: trunc_v8i64_v8i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[0,2,2,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,1,0,2,4,5,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,1,0,2,4,5,6,7]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; SSE2-NEXT:    movsd {{.*#+}} xmm2 = xmm0[0],xmm2[1]
-; SSE2-NEXT:    psllw $15, %xmm2
-; SSE2-NEXT:    pmovmskb %xmm2, %eax
-; SSE2-NEXT:    testl $43690, %eax # imm = 0xAAAA
-; SSE2-NEXT:    setne %al
-; SSE2-NEXT:    retq
+define i1 @trunc_v8i64_v8i1(<8 x i64>) nounwind {
+; X86-SSE2-LABEL: trunc_v8i64_v8i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; X86-SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X86-SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; X86-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = mem[0,2,2,3]
+; X86-SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,1,0,2,4,5,6,7]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; X86-SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,1,0,2,4,5,6,7]
+; X86-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm2 = xmm0[0],xmm2[1]
+; X86-SSE2-NEXT:    psllw $15, %xmm2
+; X86-SSE2-NEXT:    pmovmskb %xmm2, %eax
+; X86-SSE2-NEXT:    testl $43690, %eax # imm = 0xAAAA
+; X86-SSE2-NEXT:    setne %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE2-LABEL: trunc_v8i64_v8i1:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; X64-SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X64-SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; X64-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[0,2,2,3]
+; X64-SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,1,0,2,4,5,6,7]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; X64-SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,1,0,2,4,5,6,7]
+; X64-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; X64-SSE2-NEXT:    movsd {{.*#+}} xmm2 = xmm0[0],xmm2[1]
+; X64-SSE2-NEXT:    psllw $15, %xmm2
+; X64-SSE2-NEXT:    pmovmskb %xmm2, %eax
+; X64-SSE2-NEXT:    testl $43690, %eax # imm = 0xAAAA
+; X64-SSE2-NEXT:    setne %al
+; X64-SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: trunc_v8i64_v8i1:
 ; SSE41:       # %bb.0:
@@ -422,17 +448,34 @@ define i1 @trunc_v8i64_v8i1(<8 x i64>) {
   ret i1 %b
 }
 
-define i1 @trunc_v16i32_v16i1(<16 x i32>) {
-; SSE2-LABEL: trunc_v16i32_v16i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    por %xmm3, %xmm1
-; SSE2-NEXT:    por %xmm2, %xmm0
-; SSE2-NEXT:    por %xmm1, %xmm0
-; SSE2-NEXT:    pslld $31, %xmm0
-; SSE2-NEXT:    movmskps %xmm0, %eax
-; SSE2-NEXT:    testl %eax, %eax
-; SSE2-NEXT:    setne %al
-; SSE2-NEXT:    retq
+define i1 @trunc_v16i32_v16i1(<16 x i32>) nounwind {
+; X86-SSE2-LABEL: trunc_v16i32_v16i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    por %xmm2, %xmm0
+; X86-SSE2-NEXT:    por 8(%ebp), %xmm1
+; X86-SSE2-NEXT:    por %xmm0, %xmm1
+; X86-SSE2-NEXT:    pslld $31, %xmm1
+; X86-SSE2-NEXT:    movmskps %xmm1, %eax
+; X86-SSE2-NEXT:    testl %eax, %eax
+; X86-SSE2-NEXT:    setne %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE2-LABEL: trunc_v16i32_v16i1:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    por %xmm3, %xmm1
+; X64-SSE2-NEXT:    por %xmm2, %xmm0
+; X64-SSE2-NEXT:    por %xmm1, %xmm0
+; X64-SSE2-NEXT:    pslld $31, %xmm0
+; X64-SSE2-NEXT:    movmskps %xmm0, %eax
+; X64-SSE2-NEXT:    testl %eax, %eax
+; X64-SSE2-NEXT:    setne %al
+; X64-SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: trunc_v16i32_v16i1:
 ; SSE41:       # %bb.0:
@@ -472,17 +515,34 @@ define i1 @trunc_v16i32_v16i1(<16 x i32>) {
   ret i1 %b
 }
 
-define i1 @trunc_v32i16_v32i1(<32 x i16>) {
-; SSE2-LABEL: trunc_v32i16_v32i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    por %xmm3, %xmm1
-; SSE2-NEXT:    por %xmm2, %xmm0
-; SSE2-NEXT:    por %xmm1, %xmm0
-; SSE2-NEXT:    psllw $7, %xmm0
-; SSE2-NEXT:    pmovmskb %xmm0, %eax
-; SSE2-NEXT:    testl $21845, %eax # imm = 0x5555
-; SSE2-NEXT:    setne %al
-; SSE2-NEXT:    retq
+define i1 @trunc_v32i16_v32i1(<32 x i16>) nounwind {
+; X86-SSE2-LABEL: trunc_v32i16_v32i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    por %xmm2, %xmm0
+; X86-SSE2-NEXT:    por 8(%ebp), %xmm1
+; X86-SSE2-NEXT:    por %xmm0, %xmm1
+; X86-SSE2-NEXT:    psllw $7, %xmm1
+; X86-SSE2-NEXT:    pmovmskb %xmm1, %eax
+; X86-SSE2-NEXT:    testl $21845, %eax # imm = 0x5555
+; X86-SSE2-NEXT:    setne %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE2-LABEL: trunc_v32i16_v32i1:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    por %xmm3, %xmm1
+; X64-SSE2-NEXT:    por %xmm2, %xmm0
+; X64-SSE2-NEXT:    por %xmm1, %xmm0
+; X64-SSE2-NEXT:    psllw $7, %xmm0
+; X64-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X64-SSE2-NEXT:    testl $21845, %eax # imm = 0x5555
+; X64-SSE2-NEXT:    setne %al
+; X64-SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: trunc_v32i16_v32i1:
 ; SSE41:       # %bb.0:
@@ -522,17 +582,34 @@ define i1 @trunc_v32i16_v32i1(<32 x i16>) {
   ret i1 %b
 }
 
-define i1 @trunc_v64i8_v64i1(<64 x i8>) {
-; SSE2-LABEL: trunc_v64i8_v64i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    por %xmm3, %xmm1
-; SSE2-NEXT:    por %xmm2, %xmm0
-; SSE2-NEXT:    por %xmm1, %xmm0
-; SSE2-NEXT:    psllw $7, %xmm0
-; SSE2-NEXT:    pmovmskb %xmm0, %eax
-; SSE2-NEXT:    testl %eax, %eax
-; SSE2-NEXT:    setne %al
-; SSE2-NEXT:    retq
+define i1 @trunc_v64i8_v64i1(<64 x i8>) nounwind {
+; X86-SSE2-LABEL: trunc_v64i8_v64i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    por %xmm2, %xmm0
+; X86-SSE2-NEXT:    por 8(%ebp), %xmm1
+; X86-SSE2-NEXT:    por %xmm0, %xmm1
+; X86-SSE2-NEXT:    psllw $7, %xmm1
+; X86-SSE2-NEXT:    pmovmskb %xmm1, %eax
+; X86-SSE2-NEXT:    testl %eax, %eax
+; X86-SSE2-NEXT:    setne %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE2-LABEL: trunc_v64i8_v64i1:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    por %xmm3, %xmm1
+; X64-SSE2-NEXT:    por %xmm2, %xmm0
+; X64-SSE2-NEXT:    por %xmm1, %xmm0
+; X64-SSE2-NEXT:    psllw $7, %xmm0
+; X64-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X64-SSE2-NEXT:    testl %eax, %eax
+; X64-SSE2-NEXT:    setne %al
+; X64-SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: trunc_v64i8_v64i1:
 ; SSE41:       # %bb.0:
@@ -576,7 +653,7 @@ define i1 @trunc_v64i8_v64i1(<64 x i8>) {
 ; Comparison With Zero
 ;
 
-define i1 @icmp0_v2i64_v2i1(<2 x i64>) {
+define i1 @icmp0_v2i64_v2i1(<2 x i64>) nounwind {
 ; SSE2-LABEL: icmp0_v2i64_v2i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm1, %xmm1
@@ -586,7 +663,7 @@ define i1 @icmp0_v2i64_v2i1(<2 x i64>) {
 ; SSE2-NEXT:    movmskpd %xmm0, %eax
 ; SSE2-NEXT:    testl %eax, %eax
 ; SSE2-NEXT:    setne %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: icmp0_v2i64_v2i1:
 ; SSE41:       # %bb.0:
@@ -637,7 +714,7 @@ define i1 @icmp0_v2i64_v2i1(<2 x i64>) {
   ret i1 %b
 }
 
-define i1 @icmp0_v4i32_v4i1(<4 x i32>) {
+define i1 @icmp0_v4i32_v4i1(<4 x i32>) nounwind {
 ; SSE-LABEL: icmp0_v4i32_v4i1:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pxor %xmm1, %xmm1
@@ -645,7 +722,7 @@ define i1 @icmp0_v4i32_v4i1(<4 x i32>) {
 ; SSE-NEXT:    movmskps %xmm1, %eax
 ; SSE-NEXT:    testl %eax, %eax
 ; SSE-NEXT:    setne %al
-; SSE-NEXT:    retq
+; SSE-NEXT:    ret{{[l|q]}}
 ;
 ; AVX1OR2-LABEL: icmp0_v4i32_v4i1:
 ; AVX1OR2:       # %bb.0:
@@ -687,7 +764,7 @@ define i1 @icmp0_v4i32_v4i1(<4 x i32>) {
   ret i1 %b
 }
 
-define i1 @icmp0_v8i16_v8i1(<8 x i16>) {
+define i1 @icmp0_v8i16_v8i1(<8 x i16>) nounwind {
 ; SSE-LABEL: icmp0_v8i16_v8i1:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pxor %xmm1, %xmm1
@@ -695,7 +772,7 @@ define i1 @icmp0_v8i16_v8i1(<8 x i16>) {
 ; SSE-NEXT:    pmovmskb %xmm1, %eax
 ; SSE-NEXT:    testl %eax, %eax
 ; SSE-NEXT:    setne %al
-; SSE-NEXT:    retq
+; SSE-NEXT:    ret{{[l|q]}}
 ;
 ; AVX1OR2-LABEL: icmp0_v8i16_v8i1:
 ; AVX1OR2:       # %bb.0:
@@ -739,7 +816,7 @@ define i1 @icmp0_v8i16_v8i1(<8 x i16>) {
   ret i1 %b
 }
 
-define i1 @icmp0_v16i8_v16i1(<16 x i8>) {
+define i1 @icmp0_v16i8_v16i1(<16 x i8>) nounwind {
 ; SSE-LABEL: icmp0_v16i8_v16i1:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pxor %xmm1, %xmm1
@@ -747,7 +824,7 @@ define i1 @icmp0_v16i8_v16i1(<16 x i8>) {
 ; SSE-NEXT:    pmovmskb %xmm1, %eax
 ; SSE-NEXT:    testl %eax, %eax
 ; SSE-NEXT:    setne %al
-; SSE-NEXT:    retq
+; SSE-NEXT:    ret{{[l|q]}}
 ;
 ; AVX1OR2-LABEL: icmp0_v16i8_v16i1:
 ; AVX1OR2:       # %bb.0:
@@ -787,7 +864,7 @@ define i1 @icmp0_v16i8_v16i1(<16 x i8>) {
   ret i1 %b
 }
 
-define i1 @icmp0_v4i64_v4i1(<4 x i64>) {
+define i1 @icmp0_v4i64_v4i1(<4 x i64>) nounwind {
 ; SSE2-LABEL: icmp0_v4i64_v4i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
@@ -801,7 +878,7 @@ define i1 @icmp0_v4i64_v4i1(<4 x i64>) {
 ; SSE2-NEXT:    movmskps %xmm1, %eax
 ; SSE2-NEXT:    testl %eax, %eax
 ; SSE2-NEXT:    setne %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: icmp0_v4i64_v4i1:
 ; SSE41:       # %bb.0:
@@ -868,7 +945,7 @@ define i1 @icmp0_v4i64_v4i1(<4 x i64>) {
   ret i1 %b
 }
 
-define i1 @icmp0_v8i32_v8i1(<8 x i32>) {
+define i1 @icmp0_v8i32_v8i1(<8 x i32>) nounwind {
 ; SSE-LABEL: icmp0_v8i32_v8i1:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pxor %xmm2, %xmm2
@@ -878,7 +955,7 @@ define i1 @icmp0_v8i32_v8i1(<8 x i32>) {
 ; SSE-NEXT:    pmovmskb %xmm0, %eax
 ; SSE-NEXT:    testl %eax, %eax
 ; SSE-NEXT:    setne %al
-; SSE-NEXT:    retq
+; SSE-NEXT:    ret{{[l|q]}}
 ;
 ; AVX1-LABEL: icmp0_v8i32_v8i1:
 ; AVX1:       # %bb.0:
@@ -934,7 +1011,7 @@ define i1 @icmp0_v8i32_v8i1(<8 x i32>) {
   ret i1 %b
 }
 
-define i1 @icmp0_v16i16_v16i1(<16 x i16>) {
+define i1 @icmp0_v16i16_v16i1(<16 x i16>) nounwind {
 ; SSE-LABEL: icmp0_v16i16_v16i1:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pxor %xmm2, %xmm2
@@ -944,7 +1021,7 @@ define i1 @icmp0_v16i16_v16i1(<16 x i16>) {
 ; SSE-NEXT:    pmovmskb %xmm0, %eax
 ; SSE-NEXT:    testl %eax, %eax
 ; SSE-NEXT:    setne %al
-; SSE-NEXT:    retq
+; SSE-NEXT:    ret{{[l|q]}}
 ;
 ; AVX1-LABEL: icmp0_v16i16_v16i1:
 ; AVX1:       # %bb.0:
@@ -1001,7 +1078,7 @@ define i1 @icmp0_v16i16_v16i1(<16 x i16>) {
   ret i1 %b
 }
 
-define i1 @icmp0_v32i8_v32i1(<32 x i8>) {
+define i1 @icmp0_v32i8_v32i1(<32 x i8>) nounwind {
 ; SSE-LABEL: icmp0_v32i8_v32i1:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pxor %xmm2, %xmm2
@@ -1011,7 +1088,7 @@ define i1 @icmp0_v32i8_v32i1(<32 x i8>) {
 ; SSE-NEXT:    pmovmskb %xmm0, %eax
 ; SSE-NEXT:    testl %eax, %eax
 ; SSE-NEXT:    setne %al
-; SSE-NEXT:    retq
+; SSE-NEXT:    ret{{[l|q]}}
 ;
 ; AVX1-LABEL: icmp0_v32i8_v32i1:
 ; AVX1:       # %bb.0:
@@ -1067,29 +1144,58 @@ define i1 @icmp0_v32i8_v32i1(<32 x i8>) {
   ret i1 %b
 }
 
-define i1 @icmp0_v8i64_v8i1(<8 x i64>) {
-; SSE2-LABEL: icmp0_v8i64_v8i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,0,3,2]
-; SSE2-NEXT:    pand %xmm3, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,0,3,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    packssdw %xmm5, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,0,3,2]
-; SSE2-NEXT:    pand %xmm1, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,0,3,2]
-; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    packssdw %xmm2, %xmm1
-; SSE2-NEXT:    packssdw %xmm3, %xmm1
-; SSE2-NEXT:    pmovmskb %xmm1, %eax
-; SSE2-NEXT:    testl $43690, %eax # imm = 0xAAAA
-; SSE2-NEXT:    setne %al
-; SSE2-NEXT:    retq
+define i1 @icmp0_v8i64_v8i1(<8 x i64>) nounwind {
+; X86-SSE2-LABEL: icmp0_v8i64_v8i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    pxor %xmm3, %xmm3
+; X86-SSE2-NEXT:    pcmpeqd %xmm3, %xmm1
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,0,3,2]
+; X86-SSE2-NEXT:    pand %xmm1, %xmm4
+; X86-SSE2-NEXT:    pcmpeqd %xmm3, %xmm0
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,0,3,2]
+; X86-SSE2-NEXT:    pand %xmm0, %xmm1
+; X86-SSE2-NEXT:    packssdw %xmm4, %xmm1
+; X86-SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,0,3,2]
+; X86-SSE2-NEXT:    pand %xmm2, %xmm0
+; X86-SSE2-NEXT:    pcmpeqd 8(%ebp), %xmm3
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,0,3,2]
+; X86-SSE2-NEXT:    pand %xmm3, %xmm2
+; X86-SSE2-NEXT:    packssdw %xmm2, %xmm0
+; X86-SSE2-NEXT:    packssdw %xmm0, %xmm1
+; X86-SSE2-NEXT:    pmovmskb %xmm1, %eax
+; X86-SSE2-NEXT:    testl $43690, %eax # imm = 0xAAAA
+; X86-SSE2-NEXT:    setne %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE2-LABEL: icmp0_v8i64_v8i1:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    pxor %xmm4, %xmm4
+; X64-SSE2-NEXT:    pcmpeqd %xmm4, %xmm3
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,0,3,2]
+; X64-SSE2-NEXT:    pand %xmm3, %xmm5
+; X64-SSE2-NEXT:    pcmpeqd %xmm4, %xmm2
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,0,3,2]
+; X64-SSE2-NEXT:    pand %xmm2, %xmm3
+; X64-SSE2-NEXT:    packssdw %xmm5, %xmm3
+; X64-SSE2-NEXT:    pcmpeqd %xmm4, %xmm1
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,0,3,2]
+; X64-SSE2-NEXT:    pand %xmm1, %xmm2
+; X64-SSE2-NEXT:    pcmpeqd %xmm4, %xmm0
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,0,3,2]
+; X64-SSE2-NEXT:    pand %xmm0, %xmm1
+; X64-SSE2-NEXT:    packssdw %xmm2, %xmm1
+; X64-SSE2-NEXT:    packssdw %xmm3, %xmm1
+; X64-SSE2-NEXT:    pmovmskb %xmm1, %eax
+; X64-SSE2-NEXT:    testl $43690, %eax # imm = 0xAAAA
+; X64-SSE2-NEXT:    setne %al
+; X64-SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: icmp0_v8i64_v8i1:
 ; SSE41:       # %bb.0:
@@ -1165,21 +1271,42 @@ define i1 @icmp0_v8i64_v8i1(<8 x i64>) {
   ret i1 %b
 }
 
-define i1 @icmp0_v16i32_v16i1(<16 x i32>) {
-; SSE-LABEL: icmp0_v16i32_v16i1:
-; SSE:       # %bb.0:
-; SSE-NEXT:    pxor %xmm4, %xmm4
-; SSE-NEXT:    pcmpeqd %xmm4, %xmm3
-; SSE-NEXT:    pcmpeqd %xmm4, %xmm2
-; SSE-NEXT:    packssdw %xmm3, %xmm2
-; SSE-NEXT:    pcmpeqd %xmm4, %xmm1
-; SSE-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSE-NEXT:    packssdw %xmm1, %xmm0
-; SSE-NEXT:    packsswb %xmm2, %xmm0
-; SSE-NEXT:    pmovmskb %xmm0, %eax
-; SSE-NEXT:    testl %eax, %eax
-; SSE-NEXT:    setne %al
-; SSE-NEXT:    retq
+define i1 @icmp0_v16i32_v16i1(<16 x i32>) nounwind {
+; X86-SSE2-LABEL: icmp0_v16i32_v16i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    pxor %xmm3, %xmm3
+; X86-SSE2-NEXT:    pcmpeqd %xmm3, %xmm1
+; X86-SSE2-NEXT:    pcmpeqd %xmm3, %xmm0
+; X86-SSE2-NEXT:    packssdw %xmm1, %xmm0
+; X86-SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
+; X86-SSE2-NEXT:    pcmpeqd 8(%ebp), %xmm3
+; X86-SSE2-NEXT:    packssdw %xmm3, %xmm2
+; X86-SSE2-NEXT:    packsswb %xmm2, %xmm0
+; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X86-SSE2-NEXT:    testl %eax, %eax
+; X86-SSE2-NEXT:    setne %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE-LABEL: icmp0_v16i32_v16i1:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pxor %xmm4, %xmm4
+; X64-SSE-NEXT:    pcmpeqd %xmm4, %xmm3
+; X64-SSE-NEXT:    pcmpeqd %xmm4, %xmm2
+; X64-SSE-NEXT:    packssdw %xmm3, %xmm2
+; X64-SSE-NEXT:    pcmpeqd %xmm4, %xmm1
+; X64-SSE-NEXT:    pcmpeqd %xmm4, %xmm0
+; X64-SSE-NEXT:    packssdw %xmm1, %xmm0
+; X64-SSE-NEXT:    packsswb %xmm2, %xmm0
+; X64-SSE-NEXT:    pmovmskb %xmm0, %eax
+; X64-SSE-NEXT:    testl %eax, %eax
+; X64-SSE-NEXT:    setne %al
+; X64-SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: icmp0_v16i32_v16i1:
 ; AVX1:       # %bb.0:
@@ -1223,21 +1350,42 @@ define i1 @icmp0_v16i32_v16i1(<16 x i32>) {
   ret i1 %b
 }
 
-define i1 @icmp0_v32i16_v32i1(<32 x i16>) {
-; SSE-LABEL: icmp0_v32i16_v32i1:
-; SSE:       # %bb.0:
-; SSE-NEXT:    pxor %xmm4, %xmm4
-; SSE-NEXT:    pcmpeqw %xmm4, %xmm3
-; SSE-NEXT:    pcmpeqw %xmm4, %xmm2
-; SSE-NEXT:    packsswb %xmm3, %xmm2
-; SSE-NEXT:    pcmpeqw %xmm4, %xmm1
-; SSE-NEXT:    pcmpeqw %xmm4, %xmm0
-; SSE-NEXT:    packsswb %xmm1, %xmm0
-; SSE-NEXT:    por %xmm2, %xmm0
-; SSE-NEXT:    pmovmskb %xmm0, %eax
-; SSE-NEXT:    testl %eax, %eax
-; SSE-NEXT:    setne %al
-; SSE-NEXT:    retq
+define i1 @icmp0_v32i16_v32i1(<32 x i16>) nounwind {
+; X86-SSE2-LABEL: icmp0_v32i16_v32i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    pxor %xmm3, %xmm3
+; X86-SSE2-NEXT:    pcmpeqw %xmm3, %xmm1
+; X86-SSE2-NEXT:    pcmpeqw %xmm3, %xmm0
+; X86-SSE2-NEXT:    packsswb %xmm1, %xmm0
+; X86-SSE2-NEXT:    pcmpeqw %xmm3, %xmm2
+; X86-SSE2-NEXT:    pcmpeqw 8(%ebp), %xmm3
+; X86-SSE2-NEXT:    packsswb %xmm3, %xmm2
+; X86-SSE2-NEXT:    por %xmm0, %xmm2
+; X86-SSE2-NEXT:    pmovmskb %xmm2, %eax
+; X86-SSE2-NEXT:    testl %eax, %eax
+; X86-SSE2-NEXT:    setne %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE-LABEL: icmp0_v32i16_v32i1:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pxor %xmm4, %xmm4
+; X64-SSE-NEXT:    pcmpeqw %xmm4, %xmm3
+; X64-SSE-NEXT:    pcmpeqw %xmm4, %xmm2
+; X64-SSE-NEXT:    packsswb %xmm3, %xmm2
+; X64-SSE-NEXT:    pcmpeqw %xmm4, %xmm1
+; X64-SSE-NEXT:    pcmpeqw %xmm4, %xmm0
+; X64-SSE-NEXT:    packsswb %xmm1, %xmm0
+; X64-SSE-NEXT:    por %xmm2, %xmm0
+; X64-SSE-NEXT:    pmovmskb %xmm0, %eax
+; X64-SSE-NEXT:    testl %eax, %eax
+; X64-SSE-NEXT:    setne %al
+; X64-SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: icmp0_v32i16_v32i1:
 ; AVX1:       # %bb.0:
@@ -1303,21 +1451,42 @@ define i1 @icmp0_v32i16_v32i1(<32 x i16>) {
   ret i1 %b
 }
 
-define i1 @icmp0_v64i8_v64i1(<64 x i8>) {
-; SSE-LABEL: icmp0_v64i8_v64i1:
-; SSE:       # %bb.0:
-; SSE-NEXT:    pxor %xmm4, %xmm4
-; SSE-NEXT:    pcmpeqb %xmm4, %xmm2
-; SSE-NEXT:    pcmpeqb %xmm4, %xmm0
-; SSE-NEXT:    por %xmm2, %xmm0
-; SSE-NEXT:    pcmpeqb %xmm4, %xmm3
-; SSE-NEXT:    pcmpeqb %xmm4, %xmm1
-; SSE-NEXT:    por %xmm3, %xmm1
-; SSE-NEXT:    por %xmm0, %xmm1
-; SSE-NEXT:    pmovmskb %xmm1, %eax
-; SSE-NEXT:    testl %eax, %eax
-; SSE-NEXT:    setne %al
-; SSE-NEXT:    retq
+define i1 @icmp0_v64i8_v64i1(<64 x i8>) nounwind {
+; X86-SSE2-LABEL: icmp0_v64i8_v64i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    pxor %xmm3, %xmm3
+; X86-SSE2-NEXT:    pcmpeqb %xmm3, %xmm1
+; X86-SSE2-NEXT:    pcmpeqb %xmm3, %xmm2
+; X86-SSE2-NEXT:    pcmpeqb %xmm3, %xmm0
+; X86-SSE2-NEXT:    por %xmm2, %xmm0
+; X86-SSE2-NEXT:    pcmpeqb 8(%ebp), %xmm3
+; X86-SSE2-NEXT:    por %xmm1, %xmm3
+; X86-SSE2-NEXT:    por %xmm0, %xmm3
+; X86-SSE2-NEXT:    pmovmskb %xmm3, %eax
+; X86-SSE2-NEXT:    testl %eax, %eax
+; X86-SSE2-NEXT:    setne %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE-LABEL: icmp0_v64i8_v64i1:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pxor %xmm4, %xmm4
+; X64-SSE-NEXT:    pcmpeqb %xmm4, %xmm2
+; X64-SSE-NEXT:    pcmpeqb %xmm4, %xmm0
+; X64-SSE-NEXT:    por %xmm2, %xmm0
+; X64-SSE-NEXT:    pcmpeqb %xmm4, %xmm3
+; X64-SSE-NEXT:    pcmpeqb %xmm4, %xmm1
+; X64-SSE-NEXT:    por %xmm3, %xmm1
+; X64-SSE-NEXT:    por %xmm0, %xmm1
+; X64-SSE-NEXT:    pmovmskb %xmm1, %eax
+; X64-SSE-NEXT:    testl %eax, %eax
+; X64-SSE-NEXT:    setne %al
+; X64-SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: icmp0_v64i8_v64i1:
 ; AVX1:       # %bb.0:
@@ -1385,7 +1554,7 @@ define i1 @icmp0_v64i8_v64i1(<64 x i8>) {
 ; Comparison
 ;
 
-define i1 @icmp_v2i64_v2i1(<2 x i64>, <2 x i64>) {
+define i1 @icmp_v2i64_v2i1(<2 x i64>, <2 x i64>) nounwind {
 ; SSE2-LABEL: icmp_v2i64_v2i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
@@ -1394,7 +1563,7 @@ define i1 @icmp_v2i64_v2i1(<2 x i64>, <2 x i64>) {
 ; SSE2-NEXT:    movmskpd %xmm1, %eax
 ; SSE2-NEXT:    testl %eax, %eax
 ; SSE2-NEXT:    setne %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: icmp_v2i64_v2i1:
 ; SSE41:       # %bb.0:
@@ -1445,14 +1614,14 @@ define i1 @icmp_v2i64_v2i1(<2 x i64>, <2 x i64>) {
   ret i1 %b
 }
 
-define i1 @icmp_v4i32_v4i1(<4 x i32>, <4 x i32>) {
+define i1 @icmp_v4i32_v4i1(<4 x i32>, <4 x i32>) nounwind {
 ; SSE-LABEL: icmp_v4i32_v4i1:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pcmpeqd %xmm1, %xmm0
 ; SSE-NEXT:    movmskps %xmm0, %eax
 ; SSE-NEXT:    testl %eax, %eax
 ; SSE-NEXT:    setne %al
-; SSE-NEXT:    retq
+; SSE-NEXT:    ret{{[l|q]}}
 ;
 ; AVX1OR2-LABEL: icmp_v4i32_v4i1:
 ; AVX1OR2:       # %bb.0:
@@ -1495,14 +1664,14 @@ define i1 @icmp_v4i32_v4i1(<4 x i32>, <4 x i32>) {
   ret i1 %b
 }
 
-define i1 @icmp_v8i16_v8i1(<8 x i16>, <8 x i16>) {
+define i1 @icmp_v8i16_v8i1(<8 x i16>, <8 x i16>) nounwind {
 ; SSE-LABEL: icmp_v8i16_v8i1:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pcmpeqw %xmm1, %xmm0
 ; SSE-NEXT:    pmovmskb %xmm0, %eax
 ; SSE-NEXT:    testl %eax, %eax
 ; SSE-NEXT:    setne %al
-; SSE-NEXT:    retq
+; SSE-NEXT:    ret{{[l|q]}}
 ;
 ; AVX1OR2-LABEL: icmp_v8i16_v8i1:
 ; AVX1OR2:       # %bb.0:
@@ -1545,14 +1714,14 @@ define i1 @icmp_v8i16_v8i1(<8 x i16>, <8 x i16>) {
   ret i1 %b
 }
 
-define i1 @icmp_v16i8_v16i1(<16 x i8>, <16 x i8>) {
+define i1 @icmp_v16i8_v16i1(<16 x i8>, <16 x i8>) nounwind {
 ; SSE-LABEL: icmp_v16i8_v16i1:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pcmpeqb %xmm1, %xmm0
 ; SSE-NEXT:    pmovmskb %xmm0, %eax
 ; SSE-NEXT:    testl %eax, %eax
 ; SSE-NEXT:    setne %al
-; SSE-NEXT:    retq
+; SSE-NEXT:    ret{{[l|q]}}
 ;
 ; AVX1OR2-LABEL: icmp_v16i8_v16i1:
 ; AVX1OR2:       # %bb.0:
@@ -1591,20 +1760,40 @@ define i1 @icmp_v16i8_v16i1(<16 x i8>, <16 x i8>) {
   ret i1 %b
 }
 
-define i1 @icmp_v4i64_v4i1(<4 x i64>, <4 x i64>) {
-; SSE2-LABEL: icmp_v4i64_v4i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,0,3,2]
-; SSE2-NEXT:    pand %xmm1, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,0,3,2]
-; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    packssdw %xmm3, %xmm1
-; SSE2-NEXT:    movmskps %xmm1, %eax
-; SSE2-NEXT:    testl %eax, %eax
-; SSE2-NEXT:    setne %al
-; SSE2-NEXT:    retq
+define i1 @icmp_v4i64_v4i1(<4 x i64>, <4 x i64>) nounwind {
+; X86-SSE2-LABEL: icmp_v4i64_v4i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,0,3,2]
+; X86-SSE2-NEXT:    pand %xmm0, %xmm2
+; X86-SSE2-NEXT:    pcmpeqd 8(%ebp), %xmm1
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,0,3,2]
+; X86-SSE2-NEXT:    pand %xmm1, %xmm0
+; X86-SSE2-NEXT:    packssdw %xmm0, %xmm2
+; X86-SSE2-NEXT:    movmskps %xmm2, %eax
+; X86-SSE2-NEXT:    testl %eax, %eax
+; X86-SSE2-NEXT:    setne %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE2-LABEL: icmp_v4i64_v4i1:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    pcmpeqd %xmm3, %xmm1
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,0,3,2]
+; X64-SSE2-NEXT:    pand %xmm1, %xmm3
+; X64-SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,0,3,2]
+; X64-SSE2-NEXT:    pand %xmm0, %xmm1
+; X64-SSE2-NEXT:    packssdw %xmm3, %xmm1
+; X64-SSE2-NEXT:    movmskps %xmm1, %eax
+; X64-SSE2-NEXT:    testl %eax, %eax
+; X64-SSE2-NEXT:    setne %al
+; X64-SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: icmp_v4i64_v4i1:
 ; SSE41:       # %bb.0:
@@ -1671,16 +1860,32 @@ define i1 @icmp_v4i64_v4i1(<4 x i64>, <4 x i64>) {
   ret i1 %b
 }
 
-define i1 @icmp_v8i32_v8i1(<8 x i32>, <8 x i32>) {
-; SSE-LABEL: icmp_v8i32_v8i1:
-; SSE:       # %bb.0:
-; SSE-NEXT:    pcmpeqd %xmm3, %xmm1
-; SSE-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE-NEXT:    packssdw %xmm1, %xmm0
-; SSE-NEXT:    pmovmskb %xmm0, %eax
-; SSE-NEXT:    testl %eax, %eax
-; SSE-NEXT:    setne %al
-; SSE-NEXT:    retq
+define i1 @icmp_v8i32_v8i1(<8 x i32>, <8 x i32>) nounwind {
+; X86-SSE2-LABEL: icmp_v8i32_v8i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
+; X86-SSE2-NEXT:    pcmpeqd 8(%ebp), %xmm1
+; X86-SSE2-NEXT:    packssdw %xmm1, %xmm0
+; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X86-SSE2-NEXT:    testl %eax, %eax
+; X86-SSE2-NEXT:    setne %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE-LABEL: icmp_v8i32_v8i1:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pcmpeqd %xmm3, %xmm1
+; X64-SSE-NEXT:    pcmpeqd %xmm2, %xmm0
+; X64-SSE-NEXT:    packssdw %xmm1, %xmm0
+; X64-SSE-NEXT:    pmovmskb %xmm0, %eax
+; X64-SSE-NEXT:    testl %eax, %eax
+; X64-SSE-NEXT:    setne %al
+; X64-SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: icmp_v8i32_v8i1:
 ; AVX1:       # %bb.0:
@@ -1737,16 +1942,32 @@ define i1 @icmp_v8i32_v8i1(<8 x i32>, <8 x i32>) {
   ret i1 %b
 }
 
-define i1 @icmp_v16i16_v16i1(<16 x i16>, <16 x i16>) {
-; SSE-LABEL: icmp_v16i16_v16i1:
-; SSE:       # %bb.0:
-; SSE-NEXT:    pcmpeqw %xmm3, %xmm1
-; SSE-NEXT:    pcmpeqw %xmm2, %xmm0
-; SSE-NEXT:    packsswb %xmm1, %xmm0
-; SSE-NEXT:    pmovmskb %xmm0, %eax
-; SSE-NEXT:    testl %eax, %eax
-; SSE-NEXT:    setne %al
-; SSE-NEXT:    retq
+define i1 @icmp_v16i16_v16i1(<16 x i16>, <16 x i16>) nounwind {
+; X86-SSE2-LABEL: icmp_v16i16_v16i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    pcmpeqw %xmm2, %xmm0
+; X86-SSE2-NEXT:    pcmpeqw 8(%ebp), %xmm1
+; X86-SSE2-NEXT:    packsswb %xmm1, %xmm0
+; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X86-SSE2-NEXT:    testl %eax, %eax
+; X86-SSE2-NEXT:    setne %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE-LABEL: icmp_v16i16_v16i1:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pcmpeqw %xmm3, %xmm1
+; X64-SSE-NEXT:    pcmpeqw %xmm2, %xmm0
+; X64-SSE-NEXT:    packsswb %xmm1, %xmm0
+; X64-SSE-NEXT:    pmovmskb %xmm0, %eax
+; X64-SSE-NEXT:    testl %eax, %eax
+; X64-SSE-NEXT:    setne %al
+; X64-SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: icmp_v16i16_v16i1:
 ; AVX1:       # %bb.0:
@@ -1802,16 +2023,32 @@ define i1 @icmp_v16i16_v16i1(<16 x i16>, <16 x i16>) {
   ret i1 %b
 }
 
-define i1 @icmp_v32i8_v32i1(<32 x i8>, <32 x i8>) {
-; SSE-LABEL: icmp_v32i8_v32i1:
-; SSE:       # %bb.0:
-; SSE-NEXT:    pcmpeqb %xmm3, %xmm1
-; SSE-NEXT:    pcmpeqb %xmm2, %xmm0
-; SSE-NEXT:    por %xmm1, %xmm0
-; SSE-NEXT:    pmovmskb %xmm0, %eax
-; SSE-NEXT:    testl %eax, %eax
-; SSE-NEXT:    setne %al
-; SSE-NEXT:    retq
+define i1 @icmp_v32i8_v32i1(<32 x i8>, <32 x i8>) nounwind {
+; X86-SSE2-LABEL: icmp_v32i8_v32i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    pcmpeqb %xmm2, %xmm0
+; X86-SSE2-NEXT:    pcmpeqb 8(%ebp), %xmm1
+; X86-SSE2-NEXT:    por %xmm0, %xmm1
+; X86-SSE2-NEXT:    pmovmskb %xmm1, %eax
+; X86-SSE2-NEXT:    testl %eax, %eax
+; X86-SSE2-NEXT:    setne %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE-LABEL: icmp_v32i8_v32i1:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pcmpeqb %xmm3, %xmm1
+; X64-SSE-NEXT:    pcmpeqb %xmm2, %xmm0
+; X64-SSE-NEXT:    por %xmm1, %xmm0
+; X64-SSE-NEXT:    pmovmskb %xmm0, %eax
+; X64-SSE-NEXT:    testl %eax, %eax
+; X64-SSE-NEXT:    setne %al
+; X64-SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: icmp_v32i8_v32i1:
 ; AVX1:       # %bb.0:
@@ -1866,28 +2103,57 @@ define i1 @icmp_v32i8_v32i1(<32 x i8>, <32 x i8>) {
   ret i1 %b
 }
 
-define i1 @icmp_v8i64_v8i1(<8 x i64>, <8 x i64>) {
-; SSE2-LABEL: icmp_v8i64_v8i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pcmpeqd %xmm7, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[1,0,3,2]
-; SSE2-NEXT:    pand %xmm3, %xmm7
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,0,3,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    packssdw %xmm7, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,0,3,2]
-; SSE2-NEXT:    pand %xmm1, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,0,3,2]
-; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    packssdw %xmm2, %xmm1
-; SSE2-NEXT:    packssdw %xmm3, %xmm1
-; SSE2-NEXT:    pmovmskb %xmm1, %eax
-; SSE2-NEXT:    testl $43690, %eax # imm = 0xAAAA
-; SSE2-NEXT:    setne %al
-; SSE2-NEXT:    retq
+define i1 @icmp_v8i64_v8i1(<8 x i64>, <8 x i64>) nounwind {
+; X86-SSE2-LABEL: icmp_v8i64_v8i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    movdqa 8(%ebp), %xmm3
+; X86-SSE2-NEXT:    pcmpeqd 72(%ebp), %xmm3
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,0,3,2]
+; X86-SSE2-NEXT:    pand %xmm3, %xmm4
+; X86-SSE2-NEXT:    pcmpeqd 56(%ebp), %xmm2
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,0,3,2]
+; X86-SSE2-NEXT:    pand %xmm2, %xmm3
+; X86-SSE2-NEXT:    packssdw %xmm4, %xmm3
+; X86-SSE2-NEXT:    pcmpeqd 40(%ebp), %xmm1
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,0,3,2]
+; X86-SSE2-NEXT:    pand %xmm1, %xmm2
+; X86-SSE2-NEXT:    pcmpeqd 24(%ebp), %xmm0
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,0,3,2]
+; X86-SSE2-NEXT:    pand %xmm0, %xmm1
+; X86-SSE2-NEXT:    packssdw %xmm2, %xmm1
+; X86-SSE2-NEXT:    packssdw %xmm3, %xmm1
+; X86-SSE2-NEXT:    pmovmskb %xmm1, %eax
+; X86-SSE2-NEXT:    testl $43690, %eax # imm = 0xAAAA
+; X86-SSE2-NEXT:    setne %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE2-LABEL: icmp_v8i64_v8i1:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    pcmpeqd %xmm7, %xmm3
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[1,0,3,2]
+; X64-SSE2-NEXT:    pand %xmm3, %xmm7
+; X64-SSE2-NEXT:    pcmpeqd %xmm6, %xmm2
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,0,3,2]
+; X64-SSE2-NEXT:    pand %xmm2, %xmm3
+; X64-SSE2-NEXT:    packssdw %xmm7, %xmm3
+; X64-SSE2-NEXT:    pcmpeqd %xmm5, %xmm1
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,0,3,2]
+; X64-SSE2-NEXT:    pand %xmm1, %xmm2
+; X64-SSE2-NEXT:    pcmpeqd %xmm4, %xmm0
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,0,3,2]
+; X64-SSE2-NEXT:    pand %xmm0, %xmm1
+; X64-SSE2-NEXT:    packssdw %xmm2, %xmm1
+; X64-SSE2-NEXT:    packssdw %xmm3, %xmm1
+; X64-SSE2-NEXT:    pmovmskb %xmm1, %eax
+; X64-SSE2-NEXT:    testl $43690, %eax # imm = 0xAAAA
+; X64-SSE2-NEXT:    setne %al
+; X64-SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: icmp_v8i64_v8i1:
 ; SSE41:       # %bb.0:
@@ -1962,20 +2228,41 @@ define i1 @icmp_v8i64_v8i1(<8 x i64>, <8 x i64>) {
   ret i1 %b
 }
 
-define i1 @icmp_v16i32_v16i1(<16 x i32>, <16 x i32>) {
-; SSE-LABEL: icmp_v16i32_v16i1:
-; SSE:       # %bb.0:
-; SSE-NEXT:    pcmpeqd %xmm7, %xmm3
-; SSE-NEXT:    pcmpeqd %xmm6, %xmm2
-; SSE-NEXT:    packssdw %xmm3, %xmm2
-; SSE-NEXT:    pcmpeqd %xmm5, %xmm1
-; SSE-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSE-NEXT:    packssdw %xmm1, %xmm0
-; SSE-NEXT:    packsswb %xmm2, %xmm0
-; SSE-NEXT:    pmovmskb %xmm0, %eax
-; SSE-NEXT:    testl %eax, %eax
-; SSE-NEXT:    setne %al
-; SSE-NEXT:    retq
+define i1 @icmp_v16i32_v16i1(<16 x i32>, <16 x i32>) nounwind {
+; X86-SSE2-LABEL: icmp_v16i32_v16i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    movdqa 8(%ebp), %xmm3
+; X86-SSE2-NEXT:    pcmpeqd 72(%ebp), %xmm3
+; X86-SSE2-NEXT:    pcmpeqd 56(%ebp), %xmm2
+; X86-SSE2-NEXT:    packssdw %xmm3, %xmm2
+; X86-SSE2-NEXT:    pcmpeqd 40(%ebp), %xmm1
+; X86-SSE2-NEXT:    pcmpeqd 24(%ebp), %xmm0
+; X86-SSE2-NEXT:    packssdw %xmm1, %xmm0
+; X86-SSE2-NEXT:    packsswb %xmm2, %xmm0
+; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X86-SSE2-NEXT:    testl %eax, %eax
+; X86-SSE2-NEXT:    setne %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE-LABEL: icmp_v16i32_v16i1:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pcmpeqd %xmm7, %xmm3
+; X64-SSE-NEXT:    pcmpeqd %xmm6, %xmm2
+; X64-SSE-NEXT:    packssdw %xmm3, %xmm2
+; X64-SSE-NEXT:    pcmpeqd %xmm5, %xmm1
+; X64-SSE-NEXT:    pcmpeqd %xmm4, %xmm0
+; X64-SSE-NEXT:    packssdw %xmm1, %xmm0
+; X64-SSE-NEXT:    packsswb %xmm2, %xmm0
+; X64-SSE-NEXT:    pmovmskb %xmm0, %eax
+; X64-SSE-NEXT:    testl %eax, %eax
+; X64-SSE-NEXT:    setne %al
+; X64-SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: icmp_v16i32_v16i1:
 ; AVX1:       # %bb.0:
@@ -2019,20 +2306,41 @@ define i1 @icmp_v16i32_v16i1(<16 x i32>, <16 x i32>) {
   ret i1 %b
 }
 
-define i1 @icmp_v32i16_v32i1(<32 x i16>, <32 x i16>) {
-; SSE-LABEL: icmp_v32i16_v32i1:
-; SSE:       # %bb.0:
-; SSE-NEXT:    pcmpeqw %xmm7, %xmm3
-; SSE-NEXT:    pcmpeqw %xmm6, %xmm2
-; SSE-NEXT:    packsswb %xmm3, %xmm2
-; SSE-NEXT:    pcmpeqw %xmm5, %xmm1
-; SSE-NEXT:    pcmpeqw %xmm4, %xmm0
-; SSE-NEXT:    packsswb %xmm1, %xmm0
-; SSE-NEXT:    por %xmm2, %xmm0
-; SSE-NEXT:    pmovmskb %xmm0, %eax
-; SSE-NEXT:    testl %eax, %eax
-; SSE-NEXT:    setne %al
-; SSE-NEXT:    retq
+define i1 @icmp_v32i16_v32i1(<32 x i16>, <32 x i16>) nounwind {
+; X86-SSE2-LABEL: icmp_v32i16_v32i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    movdqa 8(%ebp), %xmm3
+; X86-SSE2-NEXT:    pcmpeqw 72(%ebp), %xmm3
+; X86-SSE2-NEXT:    pcmpeqw 56(%ebp), %xmm2
+; X86-SSE2-NEXT:    packsswb %xmm3, %xmm2
+; X86-SSE2-NEXT:    pcmpeqw 40(%ebp), %xmm1
+; X86-SSE2-NEXT:    pcmpeqw 24(%ebp), %xmm0
+; X86-SSE2-NEXT:    packsswb %xmm1, %xmm0
+; X86-SSE2-NEXT:    por %xmm2, %xmm0
+; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X86-SSE2-NEXT:    testl %eax, %eax
+; X86-SSE2-NEXT:    setne %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE-LABEL: icmp_v32i16_v32i1:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pcmpeqw %xmm7, %xmm3
+; X64-SSE-NEXT:    pcmpeqw %xmm6, %xmm2
+; X64-SSE-NEXT:    packsswb %xmm3, %xmm2
+; X64-SSE-NEXT:    pcmpeqw %xmm5, %xmm1
+; X64-SSE-NEXT:    pcmpeqw %xmm4, %xmm0
+; X64-SSE-NEXT:    packsswb %xmm1, %xmm0
+; X64-SSE-NEXT:    por %xmm2, %xmm0
+; X64-SSE-NEXT:    pmovmskb %xmm0, %eax
+; X64-SSE-NEXT:    testl %eax, %eax
+; X64-SSE-NEXT:    setne %al
+; X64-SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: icmp_v32i16_v32i1:
 ; AVX1:       # %bb.0:
@@ -2098,20 +2406,41 @@ define i1 @icmp_v32i16_v32i1(<32 x i16>, <32 x i16>) {
   ret i1 %b
 }
 
-define i1 @icmp_v64i8_v64i1(<64 x i8>, <64 x i8>) {
-; SSE-LABEL: icmp_v64i8_v64i1:
-; SSE:       # %bb.0:
-; SSE-NEXT:    pcmpeqb %xmm6, %xmm2
-; SSE-NEXT:    pcmpeqb %xmm4, %xmm0
-; SSE-NEXT:    por %xmm2, %xmm0
-; SSE-NEXT:    pcmpeqb %xmm7, %xmm3
-; SSE-NEXT:    pcmpeqb %xmm5, %xmm1
-; SSE-NEXT:    por %xmm3, %xmm1
-; SSE-NEXT:    por %xmm0, %xmm1
-; SSE-NEXT:    pmovmskb %xmm1, %eax
-; SSE-NEXT:    testl %eax, %eax
-; SSE-NEXT:    setne %al
-; SSE-NEXT:    retq
+define i1 @icmp_v64i8_v64i1(<64 x i8>, <64 x i8>) nounwind {
+; X86-SSE2-LABEL: icmp_v64i8_v64i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    movdqa 8(%ebp), %xmm3
+; X86-SSE2-NEXT:    pcmpeqb 56(%ebp), %xmm2
+; X86-SSE2-NEXT:    pcmpeqb 24(%ebp), %xmm0
+; X86-SSE2-NEXT:    por %xmm2, %xmm0
+; X86-SSE2-NEXT:    pcmpeqb 72(%ebp), %xmm3
+; X86-SSE2-NEXT:    pcmpeqb 40(%ebp), %xmm1
+; X86-SSE2-NEXT:    por %xmm3, %xmm1
+; X86-SSE2-NEXT:    por %xmm0, %xmm1
+; X86-SSE2-NEXT:    pmovmskb %xmm1, %eax
+; X86-SSE2-NEXT:    testl %eax, %eax
+; X86-SSE2-NEXT:    setne %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE-LABEL: icmp_v64i8_v64i1:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pcmpeqb %xmm6, %xmm2
+; X64-SSE-NEXT:    pcmpeqb %xmm4, %xmm0
+; X64-SSE-NEXT:    por %xmm2, %xmm0
+; X64-SSE-NEXT:    pcmpeqb %xmm7, %xmm3
+; X64-SSE-NEXT:    pcmpeqb %xmm5, %xmm1
+; X64-SSE-NEXT:    por %xmm3, %xmm1
+; X64-SSE-NEXT:    por %xmm0, %xmm1
+; X64-SSE-NEXT:    pmovmskb %xmm1, %eax
+; X64-SSE-NEXT:    testl %eax, %eax
+; X64-SSE-NEXT:    setne %al
+; X64-SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: icmp_v64i8_v64i1:
 ; AVX1:       # %bb.0:

diff --git a/llvm/test/CodeGen/X86/vector-reduce-xor-bool.ll b/llvm/test/CodeGen/X86/vector-reduce-xor-bool.ll
index 5f8c066..161f639 100644 (file)
--- a/llvm/test/CodeGen/X86/vector-reduce-xor-bool.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-xor-bool.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE,SSE41
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE2,X86-SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE2,X64-SSE,X64-SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE,X64-SSE,SSE41
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512,AVX512F
@@ -11,14 +12,14 @@
 ; Truncate
 ;
 
-define i1 @trunc_v2i64_v2i1(<2 x i64>) {
+define i1 @trunc_v2i64_v2i1(<2 x i64>) nounwind {
 ; SSE-LABEL: trunc_v2i64_v2i1:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    psllq $63, %xmm0
 ; SSE-NEXT:    movmskpd %xmm0, %eax
 ; SSE-NEXT:    testb %al, %al
 ; SSE-NEXT:    setnp %al
-; SSE-NEXT:    retq
+; SSE-NEXT:    ret{{[l|q]}}
 ;
 ; AVX-LABEL: trunc_v2i64_v2i1:
 ; AVX:       # %bb.0:
@@ -61,14 +62,14 @@ define i1 @trunc_v2i64_v2i1(<2 x i64>) {
   ret i1 %b
 }
 
-define i1 @trunc_v4i32_v4i1(<4 x i32>) {
+define i1 @trunc_v4i32_v4i1(<4 x i32>) nounwind {
 ; SSE-LABEL: trunc_v4i32_v4i1:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pslld $31, %xmm0
 ; SSE-NEXT:    movmskps %xmm0, %eax
 ; SSE-NEXT:    testb %al, %al
 ; SSE-NEXT:    setnp %al
-; SSE-NEXT:    retq
+; SSE-NEXT:    ret{{[l|q]}}
 ;
 ; AVX-LABEL: trunc_v4i32_v4i1:
 ; AVX:       # %bb.0:
@@ -111,7 +112,7 @@ define i1 @trunc_v4i32_v4i1(<4 x i32>) {
   ret i1 %b
 }
 
-define i1 @trunc_v8i16_v8i1(<8 x i16>) {
+define i1 @trunc_v8i16_v8i1(<8 x i16>) nounwind {
 ; SSE-LABEL: trunc_v8i16_v8i1:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    psllw $15, %xmm0
@@ -119,7 +120,7 @@ define i1 @trunc_v8i16_v8i1(<8 x i16>) {
 ; SSE-NEXT:    pmovmskb %xmm0, %eax
 ; SSE-NEXT:    testb %al, %al
 ; SSE-NEXT:    setnp %al
-; SSE-NEXT:    retq
+; SSE-NEXT:    ret{{[l|q]}}
 ;
 ; AVX-LABEL: trunc_v8i16_v8i1:
 ; AVX:       # %bb.0:
@@ -164,14 +165,14 @@ define i1 @trunc_v8i16_v8i1(<8 x i16>) {
   ret i1 %b
 }
 
-define i1 @trunc_v16i8_v16i1(<16 x i8>) {
+define i1 @trunc_v16i8_v16i1(<16 x i8>) nounwind {
 ; SSE-LABEL: trunc_v16i8_v16i1:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    psllw $7, %xmm0
 ; SSE-NEXT:    pmovmskb %xmm0, %eax
 ; SSE-NEXT:    xorb %ah, %al
 ; SSE-NEXT:    setnp %al
-; SSE-NEXT:    retq
+; SSE-NEXT:    ret{{[l|q]}}
 ;
 ; AVX-LABEL: trunc_v16i8_v16i1:
 ; AVX:       # %bb.0:
@@ -193,7 +194,7 @@ define i1 @trunc_v16i8_v16i1(<16 x i8>) {
   ret i1 %b
 }
 
-define i1 @trunc_v4i64_v4i1(<4 x i64>) {
+define i1 @trunc_v4i64_v4i1(<4 x i64>) nounwind {
 ; SSE-LABEL: trunc_v4i64_v4i1:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
@@ -201,7 +202,7 @@ define i1 @trunc_v4i64_v4i1(<4 x i64>) {
 ; SSE-NEXT:    movmskps %xmm0, %eax
 ; SSE-NEXT:    testb %al, %al
 ; SSE-NEXT:    setnp %al
-; SSE-NEXT:    retq
+; SSE-NEXT:    ret{{[l|q]}}
 ;
 ; AVX1-LABEL: trunc_v4i64_v4i1:
 ; AVX1:       # %bb.0:
@@ -257,7 +258,7 @@ define i1 @trunc_v4i64_v4i1(<4 x i64>) {
   ret i1 %b
 }
 
-define i1 @trunc_v8i32_v8i1(<8 x i32>) {
+define i1 @trunc_v8i32_v8i1(<8 x i32>) nounwind {
 ; SSE2-LABEL: trunc_v8i32_v8i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pslld $16, %xmm1
@@ -270,7 +271,7 @@ define i1 @trunc_v8i32_v8i1(<8 x i32>) {
 ; SSE2-NEXT:    pmovmskb %xmm0, %eax
 ; SSE2-NEXT:    testb %al, %al
 ; SSE2-NEXT:    setnp %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: trunc_v8i32_v8i1:
 ; SSE41:       # %bb.0:
@@ -340,7 +341,7 @@ define i1 @trunc_v8i32_v8i1(<8 x i32>) {
   ret i1 %b
 }
 
-define i1 @trunc_v16i16_v16i1(<16 x i16>) {
+define i1 @trunc_v16i16_v16i1(<16 x i16>) nounwind {
 ; SSE-LABEL: trunc_v16i16_v16i1:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
@@ -351,7 +352,7 @@ define i1 @trunc_v16i16_v16i1(<16 x i16>) {
 ; SSE-NEXT:    pmovmskb %xmm0, %eax
 ; SSE-NEXT:    xorb %ah, %al
 ; SSE-NEXT:    setnp %al
-; SSE-NEXT:    retq
+; SSE-NEXT:    ret{{[l|q]}}
 ;
 ; AVX1-LABEL: trunc_v16i16_v16i1:
 ; AVX1:       # %bb.0:
@@ -418,7 +419,7 @@ define i1 @trunc_v16i16_v16i1(<16 x i16>) {
   ret i1 %b
 }
 
-define i1 @trunc_v32i8_v32i1(<32 x i8>) {
+define i1 @trunc_v32i8_v32i1(<32 x i8>) nounwind {
 ; SSE-LABEL: trunc_v32i8_v32i1:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pxor %xmm1, %xmm0
@@ -426,7 +427,7 @@ define i1 @trunc_v32i8_v32i1(<32 x i8>) {
 ; SSE-NEXT:    pmovmskb %xmm0, %eax
 ; SSE-NEXT:    xorb %ah, %al
 ; SSE-NEXT:    setnp %al
-; SSE-NEXT:    retq
+; SSE-NEXT:    ret{{[l|q]}}
 ;
 ; AVX1-LABEL: trunc_v32i8_v32i1:
 ; AVX1:       # %bb.0:
@@ -499,26 +500,52 @@ define i1 @trunc_v32i8_v32i1(<32 x i8>) {
   ret i1 %b
 }
 
-define i1 @trunc_v8i64_v8i1(<8 x i64>) {
-; SSE2-LABEL: trunc_v8i64_v8i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[0,2,2,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,1,0,2,4,5,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,1,0,2,4,5,6,7]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; SSE2-NEXT:    movsd {{.*#+}} xmm2 = xmm0[0],xmm2[1]
-; SSE2-NEXT:    psllw $15, %xmm2
-; SSE2-NEXT:    packsswb %xmm2, %xmm2
-; SSE2-NEXT:    pmovmskb %xmm2, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    setnp %al
-; SSE2-NEXT:    retq
+define i1 @trunc_v8i64_v8i1(<8 x i64>) nounwind {
+; X86-SSE2-LABEL: trunc_v8i64_v8i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; X86-SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X86-SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; X86-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = mem[0,2,2,3]
+; X86-SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,1,0,2,4,5,6,7]
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; X86-SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,1,0,2,4,5,6,7]
+; X86-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm2 = xmm0[0],xmm2[1]
+; X86-SSE2-NEXT:    psllw $15, %xmm2
+; X86-SSE2-NEXT:    packsswb %xmm2, %xmm2
+; X86-SSE2-NEXT:    pmovmskb %xmm2, %eax
+; X86-SSE2-NEXT:    testb %al, %al
+; X86-SSE2-NEXT:    setnp %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE2-LABEL: trunc_v8i64_v8i1:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; X64-SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X64-SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; X64-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[0,2,2,3]
+; X64-SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,1,0,2,4,5,6,7]
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; X64-SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,1,0,2,4,5,6,7]
+; X64-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; X64-SSE2-NEXT:    movsd {{.*#+}} xmm2 = xmm0[0],xmm2[1]
+; X64-SSE2-NEXT:    psllw $15, %xmm2
+; X64-SSE2-NEXT:    packsswb %xmm2, %xmm2
+; X64-SSE2-NEXT:    pmovmskb %xmm2, %eax
+; X64-SSE2-NEXT:    testb %al, %al
+; X64-SSE2-NEXT:    setnp %al
+; X64-SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: trunc_v8i64_v8i1:
 ; SSE41:       # %bb.0:
@@ -598,22 +625,44 @@ define i1 @trunc_v8i64_v8i1(<8 x i64>) {
   ret i1 %b
 }
 
-define i1 @trunc_v16i32_v16i1(<16 x i32>) {
-; SSE2-LABEL: trunc_v16i32_v16i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0]
-; SSE2-NEXT:    pand %xmm4, %xmm3
-; SSE2-NEXT:    pand %xmm4, %xmm2
-; SSE2-NEXT:    packuswb %xmm3, %xmm2
-; SSE2-NEXT:    pand %xmm4, %xmm1
-; SSE2-NEXT:    pand %xmm4, %xmm0
-; SSE2-NEXT:    packuswb %xmm1, %xmm0
-; SSE2-NEXT:    packuswb %xmm2, %xmm0
-; SSE2-NEXT:    psllw $7, %xmm0
-; SSE2-NEXT:    pmovmskb %xmm0, %eax
-; SSE2-NEXT:    xorb %ah, %al
-; SSE2-NEXT:    setnp %al
-; SSE2-NEXT:    retq
+define i1 @trunc_v16i32_v16i1(<16 x i32>) nounwind {
+; X86-SSE2-LABEL: trunc_v16i32_v16i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0]
+; X86-SSE2-NEXT:    pand %xmm3, %xmm1
+; X86-SSE2-NEXT:    pand %xmm3, %xmm0
+; X86-SSE2-NEXT:    packuswb %xmm1, %xmm0
+; X86-SSE2-NEXT:    pand %xmm3, %xmm2
+; X86-SSE2-NEXT:    pand 8(%ebp), %xmm3
+; X86-SSE2-NEXT:    packuswb %xmm3, %xmm2
+; X86-SSE2-NEXT:    packuswb %xmm2, %xmm0
+; X86-SSE2-NEXT:    psllw $7, %xmm0
+; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X86-SSE2-NEXT:    xorb %ah, %al
+; X86-SSE2-NEXT:    setnp %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE2-LABEL: trunc_v16i32_v16i1:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0]
+; X64-SSE2-NEXT:    pand %xmm4, %xmm3
+; X64-SSE2-NEXT:    pand %xmm4, %xmm2
+; X64-SSE2-NEXT:    packuswb %xmm3, %xmm2
+; X64-SSE2-NEXT:    pand %xmm4, %xmm1
+; X64-SSE2-NEXT:    pand %xmm4, %xmm0
+; X64-SSE2-NEXT:    packuswb %xmm1, %xmm0
+; X64-SSE2-NEXT:    packuswb %xmm2, %xmm0
+; X64-SSE2-NEXT:    psllw $7, %xmm0
+; X64-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X64-SSE2-NEXT:    xorb %ah, %al
+; X64-SSE2-NEXT:    setnp %al
+; X64-SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: trunc_v16i32_v16i1:
 ; SSE41:       # %bb.0:
@@ -704,22 +753,44 @@ define i1 @trunc_v16i32_v16i1(<16 x i32>) {
   ret i1 %b
 }
 
-define i1 @trunc_v32i16_v32i1(<32 x i16>) {
-; SSE-LABEL: trunc_v32i16_v32i1:
-; SSE:       # %bb.0:
-; SSE-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
-; SSE-NEXT:    pand %xmm4, %xmm3
-; SSE-NEXT:    pand %xmm4, %xmm2
-; SSE-NEXT:    packuswb %xmm3, %xmm2
-; SSE-NEXT:    pand %xmm4, %xmm1
-; SSE-NEXT:    pand %xmm4, %xmm0
-; SSE-NEXT:    packuswb %xmm1, %xmm0
-; SSE-NEXT:    pxor %xmm2, %xmm0
-; SSE-NEXT:    psllw $7, %xmm0
-; SSE-NEXT:    pmovmskb %xmm0, %eax
-; SSE-NEXT:    xorb %ah, %al
-; SSE-NEXT:    setnp %al
-; SSE-NEXT:    retq
+define i1 @trunc_v32i16_v32i1(<32 x i16>) nounwind {
+; X86-SSE2-LABEL: trunc_v32i16_v32i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
+; X86-SSE2-NEXT:    pand %xmm3, %xmm1
+; X86-SSE2-NEXT:    pand %xmm3, %xmm0
+; X86-SSE2-NEXT:    packuswb %xmm1, %xmm0
+; X86-SSE2-NEXT:    pand %xmm3, %xmm2
+; X86-SSE2-NEXT:    pand 8(%ebp), %xmm3
+; X86-SSE2-NEXT:    packuswb %xmm3, %xmm2
+; X86-SSE2-NEXT:    pxor %xmm0, %xmm2
+; X86-SSE2-NEXT:    psllw $7, %xmm2
+; X86-SSE2-NEXT:    pmovmskb %xmm2, %eax
+; X86-SSE2-NEXT:    xorb %ah, %al
+; X86-SSE2-NEXT:    setnp %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE-LABEL: trunc_v32i16_v32i1:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
+; X64-SSE-NEXT:    pand %xmm4, %xmm3
+; X64-SSE-NEXT:    pand %xmm4, %xmm2
+; X64-SSE-NEXT:    packuswb %xmm3, %xmm2
+; X64-SSE-NEXT:    pand %xmm4, %xmm1
+; X64-SSE-NEXT:    pand %xmm4, %xmm0
+; X64-SSE-NEXT:    packuswb %xmm1, %xmm0
+; X64-SSE-NEXT:    pxor %xmm2, %xmm0
+; X64-SSE-NEXT:    psllw $7, %xmm0
+; X64-SSE-NEXT:    pmovmskb %xmm0, %eax
+; X64-SSE-NEXT:    xorb %ah, %al
+; X64-SSE-NEXT:    setnp %al
+; X64-SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_v32i16_v32i1:
 ; AVX1:       # %bb.0:
@@ -801,17 +872,34 @@ define i1 @trunc_v32i16_v32i1(<32 x i16>) {
   ret i1 %b
 }
 
-define i1 @trunc_v64i8_v64i1(<64 x i8>) {
-; SSE-LABEL: trunc_v64i8_v64i1:
-; SSE:       # %bb.0:
-; SSE-NEXT:    pxor %xmm3, %xmm1
-; SSE-NEXT:    pxor %xmm2, %xmm0
-; SSE-NEXT:    pxor %xmm1, %xmm0
-; SSE-NEXT:    psllw $7, %xmm0
-; SSE-NEXT:    pmovmskb %xmm0, %eax
-; SSE-NEXT:    xorb %ah, %al
-; SSE-NEXT:    setnp %al
-; SSE-NEXT:    retq
+define i1 @trunc_v64i8_v64i1(<64 x i8>) nounwind {
+; X86-SSE2-LABEL: trunc_v64i8_v64i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    pxor %xmm2, %xmm0
+; X86-SSE2-NEXT:    pxor 8(%ebp), %xmm1
+; X86-SSE2-NEXT:    pxor %xmm0, %xmm1
+; X86-SSE2-NEXT:    psllw $7, %xmm1
+; X86-SSE2-NEXT:    pmovmskb %xmm1, %eax
+; X86-SSE2-NEXT:    xorb %ah, %al
+; X86-SSE2-NEXT:    setnp %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE-LABEL: trunc_v64i8_v64i1:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pxor %xmm3, %xmm1
+; X64-SSE-NEXT:    pxor %xmm2, %xmm0
+; X64-SSE-NEXT:    pxor %xmm1, %xmm0
+; X64-SSE-NEXT:    psllw $7, %xmm0
+; X64-SSE-NEXT:    pmovmskb %xmm0, %eax
+; X64-SSE-NEXT:    xorb %ah, %al
+; X64-SSE-NEXT:    setnp %al
+; X64-SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_v64i8_v64i1:
 ; AVX1:       # %bb.0:
@@ -902,7 +990,7 @@ define i1 @trunc_v64i8_v64i1(<64 x i8>) {
 ; Comparison With Zero
 ;
 
-define i1 @icmp0_v2i64_v2i1(<2 x i64>) {
+define i1 @icmp0_v2i64_v2i1(<2 x i64>) nounwind {
 ; SSE2-LABEL: icmp0_v2i64_v2i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm1, %xmm1
@@ -912,7 +1000,7 @@ define i1 @icmp0_v2i64_v2i1(<2 x i64>) {
 ; SSE2-NEXT:    movmskpd %xmm0, %eax
 ; SSE2-NEXT:    testb %al, %al
 ; SSE2-NEXT:    setnp %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: icmp0_v2i64_v2i1:
 ; SSE41:       # %bb.0:
@@ -964,7 +1052,7 @@ define i1 @icmp0_v2i64_v2i1(<2 x i64>) {
   ret i1 %b
 }
 
-define i1 @icmp0_v4i32_v4i1(<4 x i32>) {
+define i1 @icmp0_v4i32_v4i1(<4 x i32>) nounwind {
 ; SSE-LABEL: icmp0_v4i32_v4i1:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pxor %xmm1, %xmm1
@@ -972,7 +1060,7 @@ define i1 @icmp0_v4i32_v4i1(<4 x i32>) {
 ; SSE-NEXT:    movmskps %xmm1, %eax
 ; SSE-NEXT:    testb %al, %al
 ; SSE-NEXT:    setnp %al
-; SSE-NEXT:    retq
+; SSE-NEXT:    ret{{[l|q]}}
 ;
 ; AVX-LABEL: icmp0_v4i32_v4i1:
 ; AVX:       # %bb.0:
@@ -1015,7 +1103,7 @@ define i1 @icmp0_v4i32_v4i1(<4 x i32>) {
   ret i1 %b
 }
 
-define i1 @icmp0_v8i16_v8i1(<8 x i16>) {
+define i1 @icmp0_v8i16_v8i1(<8 x i16>) nounwind {
 ; SSE-LABEL: icmp0_v8i16_v8i1:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pxor %xmm1, %xmm1
@@ -1024,7 +1112,7 @@ define i1 @icmp0_v8i16_v8i1(<8 x i16>) {
 ; SSE-NEXT:    pmovmskb %xmm1, %eax
 ; SSE-NEXT:    testb %al, %al
 ; SSE-NEXT:    setnp %al
-; SSE-NEXT:    retq
+; SSE-NEXT:    ret{{[l|q]}}
 ;
 ; AVX-LABEL: icmp0_v8i16_v8i1:
 ; AVX:       # %bb.0:
@@ -1070,7 +1158,7 @@ define i1 @icmp0_v8i16_v8i1(<8 x i16>) {
   ret i1 %b
 }
 
-define i1 @icmp0_v16i8_v16i1(<16 x i8>) {
+define i1 @icmp0_v16i8_v16i1(<16 x i8>) nounwind {
 ; SSE-LABEL: icmp0_v16i8_v16i1:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pxor %xmm1, %xmm1
@@ -1078,7 +1166,7 @@ define i1 @icmp0_v16i8_v16i1(<16 x i8>) {
 ; SSE-NEXT:    pmovmskb %xmm1, %eax
 ; SSE-NEXT:    xorb %ah, %al
 ; SSE-NEXT:    setnp %al
-; SSE-NEXT:    retq
+; SSE-NEXT:    ret{{[l|q]}}
 ;
 ; AVX-LABEL: icmp0_v16i8_v16i1:
 ; AVX:       # %bb.0:
@@ -1124,7 +1212,7 @@ define i1 @icmp0_v16i8_v16i1(<16 x i8>) {
   ret i1 %b
 }
 
-define i1 @icmp0_v4i64_v4i1(<4 x i64>) {
+define i1 @icmp0_v4i64_v4i1(<4 x i64>) nounwind {
 ; SSE2-LABEL: icmp0_v4i64_v4i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
@@ -1138,7 +1226,7 @@ define i1 @icmp0_v4i64_v4i1(<4 x i64>) {
 ; SSE2-NEXT:    movmskps %xmm1, %eax
 ; SSE2-NEXT:    testb %al, %al
 ; SSE2-NEXT:    setnp %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: icmp0_v4i64_v4i1:
 ; SSE41:       # %bb.0:
@@ -1207,7 +1295,7 @@ define i1 @icmp0_v4i64_v4i1(<4 x i64>) {
   ret i1 %b
 }
 
-define i1 @icmp0_v8i32_v8i1(<8 x i32>) {
+define i1 @icmp0_v8i32_v8i1(<8 x i32>) nounwind {
 ; SSE-LABEL: icmp0_v8i32_v8i1:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pxor %xmm2, %xmm2
@@ -1218,7 +1306,7 @@ define i1 @icmp0_v8i32_v8i1(<8 x i32>) {
 ; SSE-NEXT:    pmovmskb %xmm0, %eax
 ; SSE-NEXT:    testb %al, %al
 ; SSE-NEXT:    setnp %al
-; SSE-NEXT:    retq
+; SSE-NEXT:    ret{{[l|q]}}
 ;
 ; AVX1-LABEL: icmp0_v8i32_v8i1:
 ; AVX1:       # %bb.0:
@@ -1276,7 +1364,7 @@ define i1 @icmp0_v8i32_v8i1(<8 x i32>) {
   ret i1 %b
 }
 
-define i1 @icmp0_v16i16_v16i1(<16 x i16>) {
+define i1 @icmp0_v16i16_v16i1(<16 x i16>) nounwind {
 ; SSE-LABEL: icmp0_v16i16_v16i1:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pxor %xmm2, %xmm2
@@ -1286,7 +1374,7 @@ define i1 @icmp0_v16i16_v16i1(<16 x i16>) {
 ; SSE-NEXT:    pmovmskb %xmm0, %eax
 ; SSE-NEXT:    xorb %ah, %al
 ; SSE-NEXT:    setnp %al
-; SSE-NEXT:    retq
+; SSE-NEXT:    ret{{[l|q]}}
 ;
 ; AVX1-LABEL: icmp0_v16i16_v16i1:
 ; AVX1:       # %bb.0:
@@ -1354,7 +1442,7 @@ define i1 @icmp0_v16i16_v16i1(<16 x i16>) {
   ret i1 %b
 }
 
-define i1 @icmp0_v32i8_v32i1(<32 x i8>) {
+define i1 @icmp0_v32i8_v32i1(<32 x i8>) nounwind {
 ; SSE-LABEL: icmp0_v32i8_v32i1:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pxor %xmm2, %xmm2
@@ -1364,7 +1452,7 @@ define i1 @icmp0_v32i8_v32i1(<32 x i8>) {
 ; SSE-NEXT:    pmovmskb %xmm0, %eax
 ; SSE-NEXT:    xorb %ah, %al
 ; SSE-NEXT:    setnp %al
-; SSE-NEXT:    retq
+; SSE-NEXT:    ret{{[l|q]}}
 ;
 ; AVX1-LABEL: icmp0_v32i8_v32i1:
 ; AVX1:       # %bb.0:
@@ -1442,30 +1530,60 @@ define i1 @icmp0_v32i8_v32i1(<32 x i8>) {
   ret i1 %b
 }
 
-define i1 @icmp0_v8i64_v8i1(<8 x i64>) {
-; SSE2-LABEL: icmp0_v8i64_v8i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,0,3,2]
-; SSE2-NEXT:    pand %xmm3, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,0,3,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    packssdw %xmm5, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,0,3,2]
-; SSE2-NEXT:    pand %xmm1, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,0,3,2]
-; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    packssdw %xmm2, %xmm1
-; SSE2-NEXT:    packssdw %xmm3, %xmm1
-; SSE2-NEXT:    packsswb %xmm1, %xmm1
-; SSE2-NEXT:    pmovmskb %xmm1, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    setnp %al
-; SSE2-NEXT:    retq
+define i1 @icmp0_v8i64_v8i1(<8 x i64>) nounwind {
+; X86-SSE2-LABEL: icmp0_v8i64_v8i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    pxor %xmm3, %xmm3
+; X86-SSE2-NEXT:    pcmpeqd %xmm3, %xmm1
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,0,3,2]
+; X86-SSE2-NEXT:    pand %xmm1, %xmm4
+; X86-SSE2-NEXT:    pcmpeqd %xmm3, %xmm0
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,0,3,2]
+; X86-SSE2-NEXT:    pand %xmm0, %xmm1
+; X86-SSE2-NEXT:    packssdw %xmm4, %xmm1
+; X86-SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,0,3,2]
+; X86-SSE2-NEXT:    pand %xmm2, %xmm0
+; X86-SSE2-NEXT:    pcmpeqd 8(%ebp), %xmm3
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,0,3,2]
+; X86-SSE2-NEXT:    pand %xmm3, %xmm2
+; X86-SSE2-NEXT:    packssdw %xmm2, %xmm0
+; X86-SSE2-NEXT:    packssdw %xmm0, %xmm1
+; X86-SSE2-NEXT:    packsswb %xmm1, %xmm1
+; X86-SSE2-NEXT:    pmovmskb %xmm1, %eax
+; X86-SSE2-NEXT:    testb %al, %al
+; X86-SSE2-NEXT:    setnp %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE2-LABEL: icmp0_v8i64_v8i1:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    pxor %xmm4, %xmm4
+; X64-SSE2-NEXT:    pcmpeqd %xmm4, %xmm3
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,0,3,2]
+; X64-SSE2-NEXT:    pand %xmm3, %xmm5
+; X64-SSE2-NEXT:    pcmpeqd %xmm4, %xmm2
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,0,3,2]
+; X64-SSE2-NEXT:    pand %xmm2, %xmm3
+; X64-SSE2-NEXT:    packssdw %xmm5, %xmm3
+; X64-SSE2-NEXT:    pcmpeqd %xmm4, %xmm1
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,0,3,2]
+; X64-SSE2-NEXT:    pand %xmm1, %xmm2
+; X64-SSE2-NEXT:    pcmpeqd %xmm4, %xmm0
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,0,3,2]
+; X64-SSE2-NEXT:    pand %xmm0, %xmm1
+; X64-SSE2-NEXT:    packssdw %xmm2, %xmm1
+; X64-SSE2-NEXT:    packssdw %xmm3, %xmm1
+; X64-SSE2-NEXT:    packsswb %xmm1, %xmm1
+; X64-SSE2-NEXT:    pmovmskb %xmm1, %eax
+; X64-SSE2-NEXT:    testb %al, %al
+; X64-SSE2-NEXT:    setnp %al
+; X64-SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: icmp0_v8i64_v8i1:
 ; SSE41:       # %bb.0:
@@ -1545,21 +1663,42 @@ define i1 @icmp0_v8i64_v8i1(<8 x i64>) {
   ret i1 %b
 }
 
-define i1 @icmp0_v16i32_v16i1(<16 x i32>) {
-; SSE-LABEL: icmp0_v16i32_v16i1:
-; SSE:       # %bb.0:
-; SSE-NEXT:    pxor %xmm4, %xmm4
-; SSE-NEXT:    pcmpeqd %xmm4, %xmm3
-; SSE-NEXT:    pcmpeqd %xmm4, %xmm2
-; SSE-NEXT:    packssdw %xmm3, %xmm2
-; SSE-NEXT:    pcmpeqd %xmm4, %xmm1
-; SSE-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSE-NEXT:    packssdw %xmm1, %xmm0
-; SSE-NEXT:    packsswb %xmm2, %xmm0
-; SSE-NEXT:    pmovmskb %xmm0, %eax
-; SSE-NEXT:    xorb %ah, %al
-; SSE-NEXT:    setnp %al
-; SSE-NEXT:    retq
+define i1 @icmp0_v16i32_v16i1(<16 x i32>) nounwind {
+; X86-SSE2-LABEL: icmp0_v16i32_v16i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    pxor %xmm3, %xmm3
+; X86-SSE2-NEXT:    pcmpeqd %xmm3, %xmm1
+; X86-SSE2-NEXT:    pcmpeqd %xmm3, %xmm0
+; X86-SSE2-NEXT:    packssdw %xmm1, %xmm0
+; X86-SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
+; X86-SSE2-NEXT:    pcmpeqd 8(%ebp), %xmm3
+; X86-SSE2-NEXT:    packssdw %xmm3, %xmm2
+; X86-SSE2-NEXT:    packsswb %xmm2, %xmm0
+; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X86-SSE2-NEXT:    xorb %ah, %al
+; X86-SSE2-NEXT:    setnp %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE-LABEL: icmp0_v16i32_v16i1:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pxor %xmm4, %xmm4
+; X64-SSE-NEXT:    pcmpeqd %xmm4, %xmm3
+; X64-SSE-NEXT:    pcmpeqd %xmm4, %xmm2
+; X64-SSE-NEXT:    packssdw %xmm3, %xmm2
+; X64-SSE-NEXT:    pcmpeqd %xmm4, %xmm1
+; X64-SSE-NEXT:    pcmpeqd %xmm4, %xmm0
+; X64-SSE-NEXT:    packssdw %xmm1, %xmm0
+; X64-SSE-NEXT:    packsswb %xmm2, %xmm0
+; X64-SSE-NEXT:    pmovmskb %xmm0, %eax
+; X64-SSE-NEXT:    xorb %ah, %al
+; X64-SSE-NEXT:    setnp %al
+; X64-SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: icmp0_v16i32_v16i1:
 ; AVX1:       # %bb.0:
@@ -1631,21 +1770,42 @@ define i1 @icmp0_v16i32_v16i1(<16 x i32>) {
   ret i1 %b
 }
 
-define i1 @icmp0_v32i16_v32i1(<32 x i16>) {
-; SSE-LABEL: icmp0_v32i16_v32i1:
-; SSE:       # %bb.0:
-; SSE-NEXT:    pxor %xmm4, %xmm4
-; SSE-NEXT:    pcmpeqw %xmm4, %xmm1
-; SSE-NEXT:    pcmpeqw %xmm4, %xmm0
-; SSE-NEXT:    packsswb %xmm1, %xmm0
-; SSE-NEXT:    pcmpeqw %xmm4, %xmm3
-; SSE-NEXT:    pcmpeqw %xmm4, %xmm2
-; SSE-NEXT:    packsswb %xmm3, %xmm2
-; SSE-NEXT:    pxor %xmm0, %xmm2
-; SSE-NEXT:    pmovmskb %xmm2, %eax
-; SSE-NEXT:    xorb %ah, %al
-; SSE-NEXT:    setnp %al
-; SSE-NEXT:    retq
+define i1 @icmp0_v32i16_v32i1(<32 x i16>) nounwind {
+; X86-SSE2-LABEL: icmp0_v32i16_v32i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    pxor %xmm3, %xmm3
+; X86-SSE2-NEXT:    pcmpeqw %xmm3, %xmm2
+; X86-SSE2-NEXT:    pcmpeqw %xmm3, %xmm1
+; X86-SSE2-NEXT:    pcmpeqw %xmm3, %xmm0
+; X86-SSE2-NEXT:    packsswb %xmm1, %xmm0
+; X86-SSE2-NEXT:    pcmpeqw 8(%ebp), %xmm3
+; X86-SSE2-NEXT:    packsswb %xmm3, %xmm2
+; X86-SSE2-NEXT:    pxor %xmm0, %xmm2
+; X86-SSE2-NEXT:    pmovmskb %xmm2, %eax
+; X86-SSE2-NEXT:    xorb %ah, %al
+; X86-SSE2-NEXT:    setnp %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE-LABEL: icmp0_v32i16_v32i1:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pxor %xmm4, %xmm4
+; X64-SSE-NEXT:    pcmpeqw %xmm4, %xmm1
+; X64-SSE-NEXT:    pcmpeqw %xmm4, %xmm0
+; X64-SSE-NEXT:    packsswb %xmm1, %xmm0
+; X64-SSE-NEXT:    pcmpeqw %xmm4, %xmm3
+; X64-SSE-NEXT:    pcmpeqw %xmm4, %xmm2
+; X64-SSE-NEXT:    packsswb %xmm3, %xmm2
+; X64-SSE-NEXT:    pxor %xmm0, %xmm2
+; X64-SSE-NEXT:    pmovmskb %xmm2, %eax
+; X64-SSE-NEXT:    xorb %ah, %al
+; X64-SSE-NEXT:    setnp %al
+; X64-SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: icmp0_v32i16_v32i1:
 ; AVX1:       # %bb.0:
@@ -1731,21 +1891,42 @@ define i1 @icmp0_v32i16_v32i1(<32 x i16>) {
   ret i1 %b
 }
 
-define i1 @icmp0_v64i8_v64i1(<64 x i8>) {
-; SSE-LABEL: icmp0_v64i8_v64i1:
-; SSE:       # %bb.0:
-; SSE-NEXT:    pxor %xmm4, %xmm4
-; SSE-NEXT:    pcmpeqb %xmm4, %xmm2
-; SSE-NEXT:    pcmpeqb %xmm4, %xmm0
-; SSE-NEXT:    pxor %xmm2, %xmm0
-; SSE-NEXT:    pcmpeqb %xmm4, %xmm3
-; SSE-NEXT:    pcmpeqb %xmm4, %xmm1
-; SSE-NEXT:    pxor %xmm3, %xmm1
-; SSE-NEXT:    pxor %xmm0, %xmm1
-; SSE-NEXT:    pmovmskb %xmm1, %eax
-; SSE-NEXT:    xorb %ah, %al
-; SSE-NEXT:    setnp %al
-; SSE-NEXT:    retq
+define i1 @icmp0_v64i8_v64i1(<64 x i8>) nounwind {
+; X86-SSE2-LABEL: icmp0_v64i8_v64i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    pxor %xmm3, %xmm3
+; X86-SSE2-NEXT:    pcmpeqb %xmm3, %xmm1
+; X86-SSE2-NEXT:    pcmpeqb %xmm3, %xmm2
+; X86-SSE2-NEXT:    pcmpeqb %xmm3, %xmm0
+; X86-SSE2-NEXT:    pxor %xmm2, %xmm0
+; X86-SSE2-NEXT:    pcmpeqb 8(%ebp), %xmm3
+; X86-SSE2-NEXT:    pxor %xmm1, %xmm3
+; X86-SSE2-NEXT:    pxor %xmm0, %xmm3
+; X86-SSE2-NEXT:    pmovmskb %xmm3, %eax
+; X86-SSE2-NEXT:    xorb %ah, %al
+; X86-SSE2-NEXT:    setnp %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE-LABEL: icmp0_v64i8_v64i1:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pxor %xmm4, %xmm4
+; X64-SSE-NEXT:    pcmpeqb %xmm4, %xmm2
+; X64-SSE-NEXT:    pcmpeqb %xmm4, %xmm0
+; X64-SSE-NEXT:    pxor %xmm2, %xmm0
+; X64-SSE-NEXT:    pcmpeqb %xmm4, %xmm3
+; X64-SSE-NEXT:    pcmpeqb %xmm4, %xmm1
+; X64-SSE-NEXT:    pxor %xmm3, %xmm1
+; X64-SSE-NEXT:    pxor %xmm0, %xmm1
+; X64-SSE-NEXT:    pmovmskb %xmm1, %eax
+; X64-SSE-NEXT:    xorb %ah, %al
+; X64-SSE-NEXT:    setnp %al
+; X64-SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: icmp0_v64i8_v64i1:
 ; AVX1:       # %bb.0:
@@ -1844,7 +2025,7 @@ define i1 @icmp0_v64i8_v64i1(<64 x i8>) {
 ; Comparison
 ;
 
-define i1 @icmp_v2i64_v2i1(<2 x i64>, <2 x i64>) {
+define i1 @icmp_v2i64_v2i1(<2 x i64>, <2 x i64>) nounwind {
 ; SSE2-LABEL: icmp_v2i64_v2i1:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
@@ -1853,7 +2034,7 @@ define i1 @icmp_v2i64_v2i1(<2 x i64>, <2 x i64>) {
 ; SSE2-NEXT:    movmskpd %xmm1, %eax
 ; SSE2-NEXT:    testb %al, %al
 ; SSE2-NEXT:    setnp %al
-; SSE2-NEXT:    retq
+; SSE2-NEXT:    ret{{[l|q]}}
 ;
 ; SSE41-LABEL: icmp_v2i64_v2i1:
 ; SSE41:       # %bb.0:
@@ -1905,14 +2086,14 @@ define i1 @icmp_v2i64_v2i1(<2 x i64>, <2 x i64>) {
   ret i1 %b
 }
 
-define i1 @icmp_v4i32_v4i1(<4 x i32>, <4 x i32>) {
+define i1 @icmp_v4i32_v4i1(<4 x i32>, <4 x i32>) nounwind {
 ; SSE-LABEL: icmp_v4i32_v4i1:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pcmpeqd %xmm1, %xmm0
 ; SSE-NEXT:    movmskps %xmm0, %eax
 ; SSE-NEXT:    testb %al, %al
 ; SSE-NEXT:    setnp %al
-; SSE-NEXT:    retq
+; SSE-NEXT:    ret{{[l|q]}}
 ;
 ; AVX-LABEL: icmp_v4i32_v4i1:
 ; AVX:       # %bb.0:
@@ -1956,7 +2137,7 @@ define i1 @icmp_v4i32_v4i1(<4 x i32>, <4 x i32>) {
   ret i1 %b
 }
 
-define i1 @icmp_v8i16_v8i1(<8 x i16>, <8 x i16>) {
+define i1 @icmp_v8i16_v8i1(<8 x i16>, <8 x i16>) nounwind {
 ; SSE-LABEL: icmp_v8i16_v8i1:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pcmpeqw %xmm1, %xmm0
@@ -1964,7 +2145,7 @@ define i1 @icmp_v8i16_v8i1(<8 x i16>, <8 x i16>) {
 ; SSE-NEXT:    pmovmskb %xmm0, %eax
 ; SSE-NEXT:    testb %al, %al
 ; SSE-NEXT:    setnp %al
-; SSE-NEXT:    retq
+; SSE-NEXT:    ret{{[l|q]}}
 ;
 ; AVX-LABEL: icmp_v8i16_v8i1:
 ; AVX:       # %bb.0:
@@ -2009,14 +2190,14 @@ define i1 @icmp_v8i16_v8i1(<8 x i16>, <8 x i16>) {
   ret i1 %b
 }
 
-define i1 @icmp_v16i8_v16i1(<16 x i8>, <16 x i8>) {
+define i1 @icmp_v16i8_v16i1(<16 x i8>, <16 x i8>) nounwind {
 ; SSE-LABEL: icmp_v16i8_v16i1:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pcmpeqb %xmm1, %xmm0
 ; SSE-NEXT:    pmovmskb %xmm0, %eax
 ; SSE-NEXT:    xorb %ah, %al
 ; SSE-NEXT:    setnp %al
-; SSE-NEXT:    retq
+; SSE-NEXT:    ret{{[l|q]}}
 ;
 ; AVX-LABEL: icmp_v16i8_v16i1:
 ; AVX:       # %bb.0:
@@ -2061,20 +2242,40 @@ define i1 @icmp_v16i8_v16i1(<16 x i8>, <16 x i8>) {
   ret i1 %b
 }
 
-define i1 @icmp_v4i64_v4i1(<4 x i64>, <4 x i64>) {
-; SSE2-LABEL: icmp_v4i64_v4i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,0,3,2]
-; SSE2-NEXT:    pand %xmm1, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,0,3,2]
-; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    packssdw %xmm3, %xmm1
-; SSE2-NEXT:    movmskps %xmm1, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    setnp %al
-; SSE2-NEXT:    retq
+define i1 @icmp_v4i64_v4i1(<4 x i64>, <4 x i64>) nounwind {
+; X86-SSE2-LABEL: icmp_v4i64_v4i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,0,3,2]
+; X86-SSE2-NEXT:    pand %xmm0, %xmm2
+; X86-SSE2-NEXT:    pcmpeqd 8(%ebp), %xmm1
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,0,3,2]
+; X86-SSE2-NEXT:    pand %xmm1, %xmm0
+; X86-SSE2-NEXT:    packssdw %xmm0, %xmm2
+; X86-SSE2-NEXT:    movmskps %xmm2, %eax
+; X86-SSE2-NEXT:    testb %al, %al
+; X86-SSE2-NEXT:    setnp %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE2-LABEL: icmp_v4i64_v4i1:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    pcmpeqd %xmm3, %xmm1
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,0,3,2]
+; X64-SSE2-NEXT:    pand %xmm1, %xmm3
+; X64-SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,0,3,2]
+; X64-SSE2-NEXT:    pand %xmm0, %xmm1
+; X64-SSE2-NEXT:    packssdw %xmm3, %xmm1
+; X64-SSE2-NEXT:    movmskps %xmm1, %eax
+; X64-SSE2-NEXT:    testb %al, %al
+; X64-SSE2-NEXT:    setnp %al
+; X64-SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: icmp_v4i64_v4i1:
 ; SSE41:       # %bb.0:
@@ -2143,17 +2344,34 @@ define i1 @icmp_v4i64_v4i1(<4 x i64>, <4 x i64>) {
   ret i1 %b
 }
 
-define i1 @icmp_v8i32_v8i1(<8 x i32>, <8 x i32>) {
-; SSE-LABEL: icmp_v8i32_v8i1:
-; SSE:       # %bb.0:
-; SSE-NEXT:    pcmpeqd %xmm3, %xmm1
-; SSE-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE-NEXT:    packssdw %xmm1, %xmm0
-; SSE-NEXT:    packsswb %xmm0, %xmm0
-; SSE-NEXT:    pmovmskb %xmm0, %eax
-; SSE-NEXT:    testb %al, %al
-; SSE-NEXT:    setnp %al
-; SSE-NEXT:    retq
+define i1 @icmp_v8i32_v8i1(<8 x i32>, <8 x i32>) nounwind {
+; X86-SSE2-LABEL: icmp_v8i32_v8i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
+; X86-SSE2-NEXT:    pcmpeqd 8(%ebp), %xmm1
+; X86-SSE2-NEXT:    packssdw %xmm1, %xmm0
+; X86-SSE2-NEXT:    packsswb %xmm0, %xmm0
+; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X86-SSE2-NEXT:    testb %al, %al
+; X86-SSE2-NEXT:    setnp %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE-LABEL: icmp_v8i32_v8i1:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pcmpeqd %xmm3, %xmm1
+; X64-SSE-NEXT:    pcmpeqd %xmm2, %xmm0
+; X64-SSE-NEXT:    packssdw %xmm1, %xmm0
+; X64-SSE-NEXT:    packsswb %xmm0, %xmm0
+; X64-SSE-NEXT:    pmovmskb %xmm0, %eax
+; X64-SSE-NEXT:    testb %al, %al
+; X64-SSE-NEXT:    setnp %al
+; X64-SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: icmp_v8i32_v8i1:
 ; AVX1:       # %bb.0:
@@ -2212,16 +2430,32 @@ define i1 @icmp_v8i32_v8i1(<8 x i32>, <8 x i32>) {
   ret i1 %b
 }
 
-define i1 @icmp_v16i16_v16i1(<16 x i16>, <16 x i16>) {
-; SSE-LABEL: icmp_v16i16_v16i1:
-; SSE:       # %bb.0:
-; SSE-NEXT:    pcmpeqw %xmm3, %xmm1
-; SSE-NEXT:    pcmpeqw %xmm2, %xmm0
-; SSE-NEXT:    packsswb %xmm1, %xmm0
-; SSE-NEXT:    pmovmskb %xmm0, %eax
-; SSE-NEXT:    xorb %ah, %al
-; SSE-NEXT:    setnp %al
-; SSE-NEXT:    retq
+define i1 @icmp_v16i16_v16i1(<16 x i16>, <16 x i16>) nounwind {
+; X86-SSE2-LABEL: icmp_v16i16_v16i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    pcmpeqw %xmm2, %xmm0
+; X86-SSE2-NEXT:    pcmpeqw 8(%ebp), %xmm1
+; X86-SSE2-NEXT:    packsswb %xmm1, %xmm0
+; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X86-SSE2-NEXT:    xorb %ah, %al
+; X86-SSE2-NEXT:    setnp %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE-LABEL: icmp_v16i16_v16i1:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pcmpeqw %xmm3, %xmm1
+; X64-SSE-NEXT:    pcmpeqw %xmm2, %xmm0
+; X64-SSE-NEXT:    packsswb %xmm1, %xmm0
+; X64-SSE-NEXT:    pmovmskb %xmm0, %eax
+; X64-SSE-NEXT:    xorb %ah, %al
+; X64-SSE-NEXT:    setnp %al
+; X64-SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: icmp_v16i16_v16i1:
 ; AVX1:       # %bb.0:
@@ -2288,16 +2522,32 @@ define i1 @icmp_v16i16_v16i1(<16 x i16>, <16 x i16>) {
   ret i1 %b
 }
 
-define i1 @icmp_v32i8_v32i1(<32 x i8>, <32 x i8>) {
-; SSE-LABEL: icmp_v32i8_v32i1:
-; SSE:       # %bb.0:
-; SSE-NEXT:    pcmpeqb %xmm3, %xmm1
-; SSE-NEXT:    pcmpeqb %xmm2, %xmm0
-; SSE-NEXT:    pxor %xmm1, %xmm0
-; SSE-NEXT:    pmovmskb %xmm0, %eax
-; SSE-NEXT:    xorb %ah, %al
-; SSE-NEXT:    setnp %al
-; SSE-NEXT:    retq
+define i1 @icmp_v32i8_v32i1(<32 x i8>, <32 x i8>) nounwind {
+; X86-SSE2-LABEL: icmp_v32i8_v32i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    pcmpeqb %xmm2, %xmm0
+; X86-SSE2-NEXT:    pcmpeqb 8(%ebp), %xmm1
+; X86-SSE2-NEXT:    pxor %xmm0, %xmm1
+; X86-SSE2-NEXT:    pmovmskb %xmm1, %eax
+; X86-SSE2-NEXT:    xorb %ah, %al
+; X86-SSE2-NEXT:    setnp %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE-LABEL: icmp_v32i8_v32i1:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pcmpeqb %xmm3, %xmm1
+; X64-SSE-NEXT:    pcmpeqb %xmm2, %xmm0
+; X64-SSE-NEXT:    pxor %xmm1, %xmm0
+; X64-SSE-NEXT:    pmovmskb %xmm0, %eax
+; X64-SSE-NEXT:    xorb %ah, %al
+; X64-SSE-NEXT:    setnp %al
+; X64-SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: icmp_v32i8_v32i1:
 ; AVX1:       # %bb.0:
@@ -2374,29 +2624,59 @@ define i1 @icmp_v32i8_v32i1(<32 x i8>, <32 x i8>) {
   ret i1 %b
 }
 
-define i1 @icmp_v8i64_v8i1(<8 x i64>, <8 x i64>) {
-; SSE2-LABEL: icmp_v8i64_v8i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pcmpeqd %xmm7, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[1,0,3,2]
-; SSE2-NEXT:    pand %xmm3, %xmm7
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,0,3,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    packssdw %xmm7, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,0,3,2]
-; SSE2-NEXT:    pand %xmm1, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,0,3,2]
-; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    packssdw %xmm2, %xmm1
-; SSE2-NEXT:    packssdw %xmm3, %xmm1
-; SSE2-NEXT:    packsswb %xmm1, %xmm1
-; SSE2-NEXT:    pmovmskb %xmm1, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    setnp %al
-; SSE2-NEXT:    retq
+define i1 @icmp_v8i64_v8i1(<8 x i64>, <8 x i64>) nounwind {
+; X86-SSE2-LABEL: icmp_v8i64_v8i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    movdqa 8(%ebp), %xmm3
+; X86-SSE2-NEXT:    pcmpeqd 72(%ebp), %xmm3
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,0,3,2]
+; X86-SSE2-NEXT:    pand %xmm3, %xmm4
+; X86-SSE2-NEXT:    pcmpeqd 56(%ebp), %xmm2
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,0,3,2]
+; X86-SSE2-NEXT:    pand %xmm2, %xmm3
+; X86-SSE2-NEXT:    packssdw %xmm4, %xmm3
+; X86-SSE2-NEXT:    pcmpeqd 40(%ebp), %xmm1
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,0,3,2]
+; X86-SSE2-NEXT:    pand %xmm1, %xmm2
+; X86-SSE2-NEXT:    pcmpeqd 24(%ebp), %xmm0
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,0,3,2]
+; X86-SSE2-NEXT:    pand %xmm0, %xmm1
+; X86-SSE2-NEXT:    packssdw %xmm2, %xmm1
+; X86-SSE2-NEXT:    packssdw %xmm3, %xmm1
+; X86-SSE2-NEXT:    packsswb %xmm1, %xmm1
+; X86-SSE2-NEXT:    pmovmskb %xmm1, %eax
+; X86-SSE2-NEXT:    testb %al, %al
+; X86-SSE2-NEXT:    setnp %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE2-LABEL: icmp_v8i64_v8i1:
+; X64-SSE2:       # %bb.0:
+; X64-SSE2-NEXT:    pcmpeqd %xmm7, %xmm3
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[1,0,3,2]
+; X64-SSE2-NEXT:    pand %xmm3, %xmm7
+; X64-SSE2-NEXT:    pcmpeqd %xmm6, %xmm2
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,0,3,2]
+; X64-SSE2-NEXT:    pand %xmm2, %xmm3
+; X64-SSE2-NEXT:    packssdw %xmm7, %xmm3
+; X64-SSE2-NEXT:    pcmpeqd %xmm5, %xmm1
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,0,3,2]
+; X64-SSE2-NEXT:    pand %xmm1, %xmm2
+; X64-SSE2-NEXT:    pcmpeqd %xmm4, %xmm0
+; X64-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,0,3,2]
+; X64-SSE2-NEXT:    pand %xmm0, %xmm1
+; X64-SSE2-NEXT:    packssdw %xmm2, %xmm1
+; X64-SSE2-NEXT:    packssdw %xmm3, %xmm1
+; X64-SSE2-NEXT:    packsswb %xmm1, %xmm1
+; X64-SSE2-NEXT:    pmovmskb %xmm1, %eax
+; X64-SSE2-NEXT:    testb %al, %al
+; X64-SSE2-NEXT:    setnp %al
+; X64-SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: icmp_v8i64_v8i1:
 ; SSE41:       # %bb.0:
@@ -2475,20 +2755,41 @@ define i1 @icmp_v8i64_v8i1(<8 x i64>, <8 x i64>) {
   ret i1 %b
 }
 
-define i1 @icmp_v16i32_v16i1(<16 x i32>, <16 x i32>) {
-; SSE-LABEL: icmp_v16i32_v16i1:
-; SSE:       # %bb.0:
-; SSE-NEXT:    pcmpeqd %xmm7, %xmm3
-; SSE-NEXT:    pcmpeqd %xmm6, %xmm2
-; SSE-NEXT:    packssdw %xmm3, %xmm2
-; SSE-NEXT:    pcmpeqd %xmm5, %xmm1
-; SSE-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSE-NEXT:    packssdw %xmm1, %xmm0
-; SSE-NEXT:    packsswb %xmm2, %xmm0
-; SSE-NEXT:    pmovmskb %xmm0, %eax
-; SSE-NEXT:    xorb %ah, %al
-; SSE-NEXT:    setnp %al
-; SSE-NEXT:    retq
+define i1 @icmp_v16i32_v16i1(<16 x i32>, <16 x i32>) nounwind {
+; X86-SSE2-LABEL: icmp_v16i32_v16i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    movdqa 8(%ebp), %xmm3
+; X86-SSE2-NEXT:    pcmpeqd 72(%ebp), %xmm3
+; X86-SSE2-NEXT:    pcmpeqd 56(%ebp), %xmm2
+; X86-SSE2-NEXT:    packssdw %xmm3, %xmm2
+; X86-SSE2-NEXT:    pcmpeqd 40(%ebp), %xmm1
+; X86-SSE2-NEXT:    pcmpeqd 24(%ebp), %xmm0
+; X86-SSE2-NEXT:    packssdw %xmm1, %xmm0
+; X86-SSE2-NEXT:    packsswb %xmm2, %xmm0
+; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X86-SSE2-NEXT:    xorb %ah, %al
+; X86-SSE2-NEXT:    setnp %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE-LABEL: icmp_v16i32_v16i1:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pcmpeqd %xmm7, %xmm3
+; X64-SSE-NEXT:    pcmpeqd %xmm6, %xmm2
+; X64-SSE-NEXT:    packssdw %xmm3, %xmm2
+; X64-SSE-NEXT:    pcmpeqd %xmm5, %xmm1
+; X64-SSE-NEXT:    pcmpeqd %xmm4, %xmm0
+; X64-SSE-NEXT:    packssdw %xmm1, %xmm0
+; X64-SSE-NEXT:    packsswb %xmm2, %xmm0
+; X64-SSE-NEXT:    pmovmskb %xmm0, %eax
+; X64-SSE-NEXT:    xorb %ah, %al
+; X64-SSE-NEXT:    setnp %al
+; X64-SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: icmp_v16i32_v16i1:
 ; AVX1:       # %bb.0:
@@ -2560,20 +2861,41 @@ define i1 @icmp_v16i32_v16i1(<16 x i32>, <16 x i32>) {
   ret i1 %b
 }
 
-define i1 @icmp_v32i16_v32i1(<32 x i16>, <32 x i16>) {
-; SSE-LABEL: icmp_v32i16_v32i1:
-; SSE:       # %bb.0:
-; SSE-NEXT:    pcmpeqw %xmm5, %xmm1
-; SSE-NEXT:    pcmpeqw %xmm4, %xmm0
-; SSE-NEXT:    packsswb %xmm1, %xmm0
-; SSE-NEXT:    pcmpeqw %xmm7, %xmm3
-; SSE-NEXT:    pcmpeqw %xmm6, %xmm2
-; SSE-NEXT:    packsswb %xmm3, %xmm2
-; SSE-NEXT:    pxor %xmm0, %xmm2
-; SSE-NEXT:    pmovmskb %xmm2, %eax
-; SSE-NEXT:    xorb %ah, %al
-; SSE-NEXT:    setnp %al
-; SSE-NEXT:    retq
+define i1 @icmp_v32i16_v32i1(<32 x i16>, <32 x i16>) nounwind {
+; X86-SSE2-LABEL: icmp_v32i16_v32i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    movdqa 8(%ebp), %xmm3
+; X86-SSE2-NEXT:    pcmpeqw 40(%ebp), %xmm1
+; X86-SSE2-NEXT:    pcmpeqw 24(%ebp), %xmm0
+; X86-SSE2-NEXT:    packsswb %xmm1, %xmm0
+; X86-SSE2-NEXT:    pcmpeqw 72(%ebp), %xmm3
+; X86-SSE2-NEXT:    pcmpeqw 56(%ebp), %xmm2
+; X86-SSE2-NEXT:    packsswb %xmm3, %xmm2
+; X86-SSE2-NEXT:    pxor %xmm0, %xmm2
+; X86-SSE2-NEXT:    pmovmskb %xmm2, %eax
+; X86-SSE2-NEXT:    xorb %ah, %al
+; X86-SSE2-NEXT:    setnp %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE-LABEL: icmp_v32i16_v32i1:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pcmpeqw %xmm5, %xmm1
+; X64-SSE-NEXT:    pcmpeqw %xmm4, %xmm0
+; X64-SSE-NEXT:    packsswb %xmm1, %xmm0
+; X64-SSE-NEXT:    pcmpeqw %xmm7, %xmm3
+; X64-SSE-NEXT:    pcmpeqw %xmm6, %xmm2
+; X64-SSE-NEXT:    packsswb %xmm3, %xmm2
+; X64-SSE-NEXT:    pxor %xmm0, %xmm2
+; X64-SSE-NEXT:    pmovmskb %xmm2, %eax
+; X64-SSE-NEXT:    xorb %ah, %al
+; X64-SSE-NEXT:    setnp %al
+; X64-SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: icmp_v32i16_v32i1:
 ; AVX1:       # %bb.0:
@@ -2659,20 +2981,41 @@ define i1 @icmp_v32i16_v32i1(<32 x i16>, <32 x i16>) {
   ret i1 %b
 }
 
-define i1 @icmp_v64i8_v64i1(<64 x i8>, <64 x i8>) {
-; SSE-LABEL: icmp_v64i8_v64i1:
-; SSE:       # %bb.0:
-; SSE-NEXT:    pcmpeqb %xmm6, %xmm2
-; SSE-NEXT:    pcmpeqb %xmm4, %xmm0
-; SSE-NEXT:    pxor %xmm2, %xmm0
-; SSE-NEXT:    pcmpeqb %xmm7, %xmm3
-; SSE-NEXT:    pcmpeqb %xmm5, %xmm1
-; SSE-NEXT:    pxor %xmm3, %xmm1
-; SSE-NEXT:    pxor %xmm0, %xmm1
-; SSE-NEXT:    pmovmskb %xmm1, %eax
-; SSE-NEXT:    xorb %ah, %al
-; SSE-NEXT:    setnp %al
-; SSE-NEXT:    retq
+define i1 @icmp_v64i8_v64i1(<64 x i8>, <64 x i8>) nounwind {
+; X86-SSE2-LABEL: icmp_v64i8_v64i1:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    movdqa 8(%ebp), %xmm3
+; X86-SSE2-NEXT:    pcmpeqb 56(%ebp), %xmm2
+; X86-SSE2-NEXT:    pcmpeqb 24(%ebp), %xmm0
+; X86-SSE2-NEXT:    pxor %xmm2, %xmm0
+; X86-SSE2-NEXT:    pcmpeqb 72(%ebp), %xmm3
+; X86-SSE2-NEXT:    pcmpeqb 40(%ebp), %xmm1
+; X86-SSE2-NEXT:    pxor %xmm3, %xmm1
+; X86-SSE2-NEXT:    pxor %xmm0, %xmm1
+; X86-SSE2-NEXT:    pmovmskb %xmm1, %eax
+; X86-SSE2-NEXT:    xorb %ah, %al
+; X86-SSE2-NEXT:    setnp %al
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE-LABEL: icmp_v64i8_v64i1:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    pcmpeqb %xmm6, %xmm2
+; X64-SSE-NEXT:    pcmpeqb %xmm4, %xmm0
+; X64-SSE-NEXT:    pxor %xmm2, %xmm0
+; X64-SSE-NEXT:    pcmpeqb %xmm7, %xmm3
+; X64-SSE-NEXT:    pcmpeqb %xmm5, %xmm1
+; X64-SSE-NEXT:    pxor %xmm3, %xmm1
+; X64-SSE-NEXT:    pxor %xmm0, %xmm1
+; X64-SSE-NEXT:    pmovmskb %xmm1, %eax
+; X64-SSE-NEXT:    xorb %ah, %al
+; X64-SSE-NEXT:    setnp %al
+; X64-SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: icmp_v64i8_v64i1:
 ; AVX1:       # %bb.0: