From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Sat, 10 Oct 2020 18:09:58 +0000 (+0100)
Subject: [PowerPC] ReplaceNodeResults - bail on funnel shifts and let generic legalizers deal... 
X-Git-Tag: llvmorg-13-init~9623
X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=2c3e4a21f93d5cd93321db08add991f992a64f18;p=platform%2Fupstream%2Fllvm.git

[PowerPC] ReplaceNodeResults - bail on funnel shifts and let generic legalizers deal with it

Fixes regression raised on D88834 for 32-bit triple + 64-bit cpu cases (which apparently is a thing).
---

diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 9e2dda33..2c2bc8a 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -11040,6 +11040,10 @@ void PPCTargetLowering::ReplaceNodeResults(SDNode *N,
       Results.push_back(Lowered);
     return;
   }
+  case ISD::FSHL:
+  case ISD::FSHR:
+    // Don't handle funnel shifts here.
+    return;
   case ISD::BITCAST:
     // Don't handle bitcast here.
     return;
diff --git a/llvm/test/CodeGen/PowerPC/funnel-shift-rot.ll b/llvm/test/CodeGen/PowerPC/funnel-shift-rot.ll
index d96b851..c44304d 100644
--- a/llvm/test/CodeGen/PowerPC/funnel-shift-rot.ll
+++ b/llvm/test/CodeGen/PowerPC/funnel-shift-rot.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=ppc32-- | FileCheck %s --check-prefixes=CHECK,CHECK32
+; RUN: llc < %s -mtriple=ppc32-- | FileCheck %s --check-prefixes=CHECK,CHECK32,CHECK32_32
+; RUN: llc < %s -mtriple=ppc32-- -mcpu=ppc64 | FileCheck %s --check-prefixes=CHECK,CHECK32,CHECK32_64
 ; RUN: llc < %s -mtriple=powerpc64le-- | FileCheck %s --check-prefixes=CHECK,CHECK64
 
 declare i8 @llvm.fshl.i8(i8, i8, i8)
@@ -84,30 +85,55 @@ define i32 @rotl_i32(i32 %x, i32 %z) {
 }
 
 define i64 @rotl_i64(i64 %x, i64 %z) {
-; CHECK32-LABEL: rotl_i64:
-; CHECK32:       # %bb.0:
-; CHECK32-NEXT:    clrlwi 5, 6, 26
-; CHECK32-NEXT:    subfic 8, 5, 32
-; CHECK32-NEXT:    neg 6, 6
-; CHECK32-NEXT:    slw 7, 3, 5
-; CHECK32-NEXT:    addi 9, 5, -32
-; CHECK32-NEXT:    srw 8, 4, 8
-; CHECK32-NEXT:    clrlwi 6, 6, 26
-; CHECK32-NEXT:    slw 9, 4, 9
-; CHECK32-NEXT:    or 7, 7, 8
-; CHECK32-NEXT:    subfic 8, 6, 32
-; CHECK32-NEXT:    or 7, 7, 9
-; CHECK32-NEXT:    addi 9, 6, -32
-; CHECK32-NEXT:    slw 8, 3, 8
-; CHECK32-NEXT:    srw 9, 3, 9
-; CHECK32-NEXT:    srw 3, 3, 6
-; CHECK32-NEXT:    srw 6, 4, 6
-; CHECK32-NEXT:    or 6, 6, 8
-; CHECK32-NEXT:    or 6, 6, 9
-; CHECK32-NEXT:    slw 4, 4, 5
-; CHECK32-NEXT:    or 3, 7, 3
-; CHECK32-NEXT:    or 4, 4, 6
-; CHECK32-NEXT:    blr
+; CHECK32_32-LABEL: rotl_i64:
+; CHECK32_32:       # %bb.0:
+; CHECK32_32-NEXT:    clrlwi 5, 6, 26
+; CHECK32_32-NEXT:    subfic 8, 5, 32
+; CHECK32_32-NEXT:    neg 6, 6
+; CHECK32_32-NEXT:    slw 7, 3, 5
+; CHECK32_32-NEXT:    addi 9, 5, -32
+; CHECK32_32-NEXT:    srw 8, 4, 8
+; CHECK32_32-NEXT:    clrlwi 6, 6, 26
+; CHECK32_32-NEXT:    slw 9, 4, 9
+; CHECK32_32-NEXT:    or 7, 7, 8
+; CHECK32_32-NEXT:    subfic 8, 6, 32
+; CHECK32_32-NEXT:    or 7, 7, 9
+; CHECK32_32-NEXT:    addi 9, 6, -32
+; CHECK32_32-NEXT:    slw 8, 3, 8
+; CHECK32_32-NEXT:    srw 9, 3, 9
+; CHECK32_32-NEXT:    srw 3, 3, 6
+; CHECK32_32-NEXT:    srw 6, 4, 6
+; CHECK32_32-NEXT:    or 6, 6, 8
+; CHECK32_32-NEXT:    or 6, 6, 9
+; CHECK32_32-NEXT:    slw 4, 4, 5
+; CHECK32_32-NEXT:    or 3, 7, 3
+; CHECK32_32-NEXT:    or 4, 4, 6
+; CHECK32_32-NEXT:    blr
+;
+; CHECK32_64-LABEL: rotl_i64:
+; CHECK32_64:       # %bb.0:
+; CHECK32_64-NEXT:    clrlwi 5, 6, 26
+; CHECK32_64-NEXT:    neg 6, 6
+; CHECK32_64-NEXT:    subfic 8, 5, 32
+; CHECK32_64-NEXT:    slw 7, 3, 5
+; CHECK32_64-NEXT:    clrlwi 6, 6, 26
+; CHECK32_64-NEXT:    srw 8, 4, 8
+; CHECK32_64-NEXT:    addi 9, 5, -32
+; CHECK32_64-NEXT:    or 7, 7, 8
+; CHECK32_64-NEXT:    subfic 8, 6, 32
+; CHECK32_64-NEXT:    slw 5, 4, 5
+; CHECK32_64-NEXT:    slw 9, 4, 9
+; CHECK32_64-NEXT:    srw 10, 3, 6
+; CHECK32_64-NEXT:    srw 4, 4, 6
+; CHECK32_64-NEXT:    addi 6, 6, -32
+; CHECK32_64-NEXT:    slw 8, 3, 8
+; CHECK32_64-NEXT:    srw 3, 3, 6
+; CHECK32_64-NEXT:    or 4, 4, 8
+; CHECK32_64-NEXT:    or 6, 7, 9
+; CHECK32_64-NEXT:    or 4, 4, 3
+; CHECK32_64-NEXT:    or 3, 6, 10
+; CHECK32_64-NEXT:    or 4, 5, 4
+; CHECK32_64-NEXT:    blr
 ;
 ; CHECK64-LABEL: rotl_i64:
 ; CHECK64:       # %bb.0:
@@ -120,13 +146,18 @@ define i64 @rotl_i64(i64 %x, i64 %z) {
 ; Vector rotate.
 
 define <4 x i32> @rotl_v4i32(<4 x i32> %x, <4 x i32> %z) {
-; CHECK32-LABEL: rotl_v4i32:
-; CHECK32:       # %bb.0:
-; CHECK32-NEXT:    rotlw 3, 3, 7
-; CHECK32-NEXT:    rotlw 4, 4, 8
-; CHECK32-NEXT:    rotlw 5, 5, 9
-; CHECK32-NEXT:    rotlw 6, 6, 10
-; CHECK32-NEXT:    blr
+; CHECK32_32-LABEL: rotl_v4i32:
+; CHECK32_32:       # %bb.0:
+; CHECK32_32-NEXT:    rotlw 3, 3, 7
+; CHECK32_32-NEXT:    rotlw 4, 4, 8
+; CHECK32_32-NEXT:    rotlw 5, 5, 9
+; CHECK32_32-NEXT:    rotlw 6, 6, 10
+; CHECK32_32-NEXT:    blr
+;
+; CHECK32_64-LABEL: rotl_v4i32:
+; CHECK32_64:       # %bb.0:
+; CHECK32_64-NEXT:    vrlw 2, 2, 3
+; CHECK32_64-NEXT:    blr
 ;
 ; CHECK64-LABEL: rotl_v4i32:
 ; CHECK64:       # %bb.0:
@@ -139,13 +170,19 @@ define <4 x i32> @rotl_v4i32(<4 x i32> %x, <4 x i32> %z) {
 ; Vector rotate by constant splat amount.
 
 define <4 x i32> @rotl_v4i32_const_shift(<4 x i32> %x) {
-; CHECK32-LABEL: rotl_v4i32_const_shift:
-; CHECK32:       # %bb.0:
-; CHECK32-NEXT:    rotlwi 3, 3, 3
-; CHECK32-NEXT:    rotlwi 4, 4, 3
-; CHECK32-NEXT:    rotlwi 5, 5, 3
-; CHECK32-NEXT:    rotlwi 6, 6, 3
-; CHECK32-NEXT:    blr
+; CHECK32_32-LABEL: rotl_v4i32_const_shift:
+; CHECK32_32:       # %bb.0:
+; CHECK32_32-NEXT:    rotlwi 3, 3, 3
+; CHECK32_32-NEXT:    rotlwi 4, 4, 3
+; CHECK32_32-NEXT:    rotlwi 5, 5, 3
+; CHECK32_32-NEXT:    rotlwi 6, 6, 3
+; CHECK32_32-NEXT:    blr
+;
+; CHECK32_64-LABEL: rotl_v4i32_const_shift:
+; CHECK32_64:       # %bb.0:
+; CHECK32_64-NEXT:    vspltisw 3, 3
+; CHECK32_64-NEXT:    vrlw 2, 2, 3
+; CHECK32_64-NEXT:    blr
 ;
 ; CHECK64-LABEL: rotl_v4i32_const_shift:
 ; CHECK64:       # %bb.0:
@@ -217,30 +254,55 @@ define i32 @rotr_i32(i32 %x, i32 %z) {
 }
 
 define i64 @rotr_i64(i64 %x, i64 %z) {
-; CHECK32-LABEL: rotr_i64:
-; CHECK32:       # %bb.0:
-; CHECK32-NEXT:    clrlwi 5, 6, 26
-; CHECK32-NEXT:    subfic 8, 5, 32
-; CHECK32-NEXT:    neg 6, 6
-; CHECK32-NEXT:    srw 7, 4, 5
-; CHECK32-NEXT:    addi 9, 5, -32
-; CHECK32-NEXT:    slw 8, 3, 8
-; CHECK32-NEXT:    clrlwi 6, 6, 26
-; CHECK32-NEXT:    srw 9, 3, 9
-; CHECK32-NEXT:    or 7, 7, 8
-; CHECK32-NEXT:    subfic 8, 6, 32
-; CHECK32-NEXT:    or 7, 7, 9
-; CHECK32-NEXT:    addi 9, 6, -32
-; CHECK32-NEXT:    srw 8, 4, 8
-; CHECK32-NEXT:    slw 9, 4, 9
-; CHECK32-NEXT:    slw 4, 4, 6
-; CHECK32-NEXT:    slw 6, 3, 6
-; CHECK32-NEXT:    or 6, 6, 8
-; CHECK32-NEXT:    or 6, 6, 9
-; CHECK32-NEXT:    srw 3, 3, 5
-; CHECK32-NEXT:    or 4, 7, 4
-; CHECK32-NEXT:    or 3, 3, 6
-; CHECK32-NEXT:    blr
+; CHECK32_32-LABEL: rotr_i64:
+; CHECK32_32:       # %bb.0:
+; CHECK32_32-NEXT:    clrlwi 5, 6, 26
+; CHECK32_32-NEXT:    subfic 8, 5, 32
+; CHECK32_32-NEXT:    neg 6, 6
+; CHECK32_32-NEXT:    srw 7, 4, 5
+; CHECK32_32-NEXT:    addi 9, 5, -32
+; CHECK32_32-NEXT:    slw 8, 3, 8
+; CHECK32_32-NEXT:    clrlwi 6, 6, 26
+; CHECK32_32-NEXT:    srw 9, 3, 9
+; CHECK32_32-NEXT:    or 7, 7, 8
+; CHECK32_32-NEXT:    subfic 8, 6, 32
+; CHECK32_32-NEXT:    or 7, 7, 9
+; CHECK32_32-NEXT:    addi 9, 6, -32
+; CHECK32_32-NEXT:    srw 8, 4, 8
+; CHECK32_32-NEXT:    slw 9, 4, 9
+; CHECK32_32-NEXT:    slw 4, 4, 6
+; CHECK32_32-NEXT:    slw 6, 3, 6
+; CHECK32_32-NEXT:    or 6, 6, 8
+; CHECK32_32-NEXT:    or 6, 6, 9
+; CHECK32_32-NEXT:    srw 3, 3, 5
+; CHECK32_32-NEXT:    or 4, 7, 4
+; CHECK32_32-NEXT:    or 3, 3, 6
+; CHECK32_32-NEXT:    blr
+;
+; CHECK32_64-LABEL: rotr_i64:
+; CHECK32_64:       # %bb.0:
+; CHECK32_64-NEXT:    clrlwi 5, 6, 26
+; CHECK32_64-NEXT:    neg 6, 6
+; CHECK32_64-NEXT:    subfic 8, 5, 32
+; CHECK32_64-NEXT:    srw 7, 4, 5
+; CHECK32_64-NEXT:    clrlwi 6, 6, 26
+; CHECK32_64-NEXT:    slw 8, 3, 8
+; CHECK32_64-NEXT:    addi 9, 5, -32
+; CHECK32_64-NEXT:    or 7, 7, 8
+; CHECK32_64-NEXT:    subfic 8, 6, 32
+; CHECK32_64-NEXT:    srw 5, 3, 5
+; CHECK32_64-NEXT:    srw 9, 3, 9
+; CHECK32_64-NEXT:    slw 10, 4, 6
+; CHECK32_64-NEXT:    slw 3, 3, 6
+; CHECK32_64-NEXT:    addi 6, 6, -32
+; CHECK32_64-NEXT:    srw 8, 4, 8
+; CHECK32_64-NEXT:    slw 4, 4, 6
+; CHECK32_64-NEXT:    or 3, 3, 8
+; CHECK32_64-NEXT:    or 6, 7, 9
+; CHECK32_64-NEXT:    or 3, 3, 4
+; CHECK32_64-NEXT:    or 4, 6, 10
+; CHECK32_64-NEXT:    or 3, 5, 3
+; CHECK32_64-NEXT:    blr
 ;
 ; CHECK64-LABEL: rotr_i64:
 ; CHECK64:       # %bb.0:
@@ -254,17 +316,24 @@ define i64 @rotr_i64(i64 %x, i64 %z) {
 ; Vector rotate.
 
 define <4 x i32> @rotr_v4i32(<4 x i32> %x, <4 x i32> %z) {
-; CHECK32-LABEL: rotr_v4i32:
-; CHECK32:       # %bb.0:
-; CHECK32-NEXT:    neg 7, 7
-; CHECK32-NEXT:    neg 8, 8
-; CHECK32-NEXT:    neg 9, 9
-; CHECK32-NEXT:    neg 10, 10
-; CHECK32-NEXT:    rotlw 3, 3, 7
-; CHECK32-NEXT:    rotlw 4, 4, 8
-; CHECK32-NEXT:    rotlw 5, 5, 9
-; CHECK32-NEXT:    rotlw 6, 6, 10
-; CHECK32-NEXT:    blr
+; CHECK32_32-LABEL: rotr_v4i32:
+; CHECK32_32:       # %bb.0:
+; CHECK32_32-NEXT:    neg 7, 7
+; CHECK32_32-NEXT:    neg 8, 8
+; CHECK32_32-NEXT:    neg 9, 9
+; CHECK32_32-NEXT:    neg 10, 10
+; CHECK32_32-NEXT:    rotlw 3, 3, 7
+; CHECK32_32-NEXT:    rotlw 4, 4, 8
+; CHECK32_32-NEXT:    rotlw 5, 5, 9
+; CHECK32_32-NEXT:    rotlw 6, 6, 10
+; CHECK32_32-NEXT:    blr
+;
+; CHECK32_64-LABEL: rotr_v4i32:
+; CHECK32_64:       # %bb.0:
+; CHECK32_64-NEXT:    vxor 4, 4, 4
+; CHECK32_64-NEXT:    vsubuwm 3, 4, 3
+; CHECK32_64-NEXT:    vrlw 2, 2, 3
+; CHECK32_64-NEXT:    blr
 ;
 ; CHECK64-LABEL: rotr_v4i32:
 ; CHECK64:       # %bb.0:
@@ -279,13 +348,21 @@ define <4 x i32> @rotr_v4i32(<4 x i32> %x, <4 x i32> %z) {
 ; Vector rotate by constant splat amount.
 
 define <4 x i32> @rotr_v4i32_const_shift(<4 x i32> %x) {
-; CHECK32-LABEL: rotr_v4i32_const_shift:
-; CHECK32:       # %bb.0:
-; CHECK32-NEXT:    rotlwi 3, 3, 29
-; CHECK32-NEXT:    rotlwi 4, 4, 29
-; CHECK32-NEXT:    rotlwi 5, 5, 29
-; CHECK32-NEXT:    rotlwi 6, 6, 29
-; CHECK32-NEXT:    blr
+; CHECK32_32-LABEL: rotr_v4i32_const_shift:
+; CHECK32_32:       # %bb.0:
+; CHECK32_32-NEXT:    rotlwi 3, 3, 29
+; CHECK32_32-NEXT:    rotlwi 4, 4, 29
+; CHECK32_32-NEXT:    rotlwi 5, 5, 29
+; CHECK32_32-NEXT:    rotlwi 6, 6, 29
+; CHECK32_32-NEXT:    blr
+;
+; CHECK32_64-LABEL: rotr_v4i32_const_shift:
+; CHECK32_64:       # %bb.0:
+; CHECK32_64-NEXT:    vspltisw 3, -16
+; CHECK32_64-NEXT:    vspltisw 4, 13
+; CHECK32_64-NEXT:    vsubuwm 3, 4, 3
+; CHECK32_64-NEXT:    vrlw 2, 2, 3
+; CHECK32_64-NEXT:    blr
 ;
 ; CHECK64-LABEL: rotr_v4i32_const_shift:
 ; CHECK64:       # %bb.0:
diff --git a/llvm/test/CodeGen/PowerPC/funnel-shift.ll b/llvm/test/CodeGen/PowerPC/funnel-shift.ll
index dd28b47..e938444 100644
--- a/llvm/test/CodeGen/PowerPC/funnel-shift.ll
+++ b/llvm/test/CodeGen/PowerPC/funnel-shift.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=ppc32-- | FileCheck %s --check-prefixes=CHECK,CHECK32
+; RUN: llc < %s -mtriple=ppc32-- | FileCheck %s --check-prefixes=CHECK,CHECK32,CHECK32_32
+; RUN: llc < %s -mtriple=ppc32-- -mcpu=ppc64 | FileCheck %s --check-prefixes=CHECK,CHECK32,CHECK32_64
 ; RUN: llc < %s -mtriple=powerpc64le-- | FileCheck %s --check-prefixes=CHECK,CHECK64
 
 declare i8 @llvm.fshl.i8(i8, i8, i8)
@@ -39,33 +40,61 @@ define i32 @fshl_i32(i32 %x, i32 %y, i32 %z) {
 }
 
 define i64 @fshl_i64(i64 %x, i64 %y, i64 %z) {
-; CHECK32-LABEL: fshl_i64:
-; CHECK32:       # %bb.0:
-; CHECK32-NEXT:    clrlwi 7, 8, 26
-; CHECK32-NEXT:    not 8, 8
-; CHECK32-NEXT:    rotlwi 6, 6, 31
-; CHECK32-NEXT:    subfic 10, 7, 32
-; CHECK32-NEXT:    srwi 9, 5, 1
-; CHECK32-NEXT:    slw 3, 3, 7
-; CHECK32-NEXT:    clrlwi 8, 8, 26
-; CHECK32-NEXT:    rlwimi 6, 5, 31, 0, 0
-; CHECK32-NEXT:    srw 5, 4, 10
-; CHECK32-NEXT:    srw 10, 9, 8
-; CHECK32-NEXT:    srw 6, 6, 8
-; CHECK32-NEXT:    or 3, 3, 5
-; CHECK32-NEXT:    subfic 5, 8, 32
-; CHECK32-NEXT:    addi 8, 8, -32
-; CHECK32-NEXT:    slw 5, 9, 5
-; CHECK32-NEXT:    srw 8, 9, 8
-; CHECK32-NEXT:    addi 9, 7, -32
-; CHECK32-NEXT:    slw 9, 4, 9
-; CHECK32-NEXT:    or 5, 6, 5
-; CHECK32-NEXT:    or 3, 3, 9
-; CHECK32-NEXT:    or 5, 5, 8
-; CHECK32-NEXT:    slw 4, 4, 7
-; CHECK32-NEXT:    or 3, 3, 10
-; CHECK32-NEXT:    or 4, 4, 5
-; CHECK32-NEXT:    blr
+; CHECK32_32-LABEL: fshl_i64:
+; CHECK32_32:       # %bb.0:
+; CHECK32_32-NEXT:    clrlwi 7, 8, 26
+; CHECK32_32-NEXT:    not 8, 8
+; CHECK32_32-NEXT:    rotlwi 6, 6, 31
+; CHECK32_32-NEXT:    subfic 10, 7, 32
+; CHECK32_32-NEXT:    srwi 9, 5, 1
+; CHECK32_32-NEXT:    slw 3, 3, 7
+; CHECK32_32-NEXT:    clrlwi 8, 8, 26
+; CHECK32_32-NEXT:    rlwimi 6, 5, 31, 0, 0
+; CHECK32_32-NEXT:    srw 5, 4, 10
+; CHECK32_32-NEXT:    srw 10, 9, 8
+; CHECK32_32-NEXT:    srw 6, 6, 8
+; CHECK32_32-NEXT:    or 3, 3, 5
+; CHECK32_32-NEXT:    subfic 5, 8, 32
+; CHECK32_32-NEXT:    addi 8, 8, -32
+; CHECK32_32-NEXT:    slw 5, 9, 5
+; CHECK32_32-NEXT:    srw 8, 9, 8
+; CHECK32_32-NEXT:    addi 9, 7, -32
+; CHECK32_32-NEXT:    slw 9, 4, 9
+; CHECK32_32-NEXT:    or 5, 6, 5
+; CHECK32_32-NEXT:    or 3, 3, 9
+; CHECK32_32-NEXT:    or 5, 5, 8
+; CHECK32_32-NEXT:    slw 4, 4, 7
+; CHECK32_32-NEXT:    or 3, 3, 10
+; CHECK32_32-NEXT:    or 4, 4, 5
+; CHECK32_32-NEXT:    blr
+;
+; CHECK32_64-LABEL: fshl_i64:
+; CHECK32_64:       # %bb.0:
+; CHECK32_64-NEXT:    clrlwi 7, 8, 26
+; CHECK32_64-NEXT:    not 8, 8
+; CHECK32_64-NEXT:    subfic 9, 7, 32
+; CHECK32_64-NEXT:    rotlwi 6, 6, 31
+; CHECK32_64-NEXT:    slw 3, 3, 7
+; CHECK32_64-NEXT:    clrlwi 8, 8, 26
+; CHECK32_64-NEXT:    srw 9, 4, 9
+; CHECK32_64-NEXT:    rlwimi 6, 5, 31, 0, 0
+; CHECK32_64-NEXT:    srwi 5, 5, 1
+; CHECK32_64-NEXT:    addi 10, 7, -32
+; CHECK32_64-NEXT:    or 3, 3, 9
+; CHECK32_64-NEXT:    subfic 9, 8, 32
+; CHECK32_64-NEXT:    slw 7, 4, 7
+; CHECK32_64-NEXT:    slw 4, 4, 10
+; CHECK32_64-NEXT:    srw 10, 5, 8
+; CHECK32_64-NEXT:    srw 6, 6, 8
+; CHECK32_64-NEXT:    addi 8, 8, -32
+; CHECK32_64-NEXT:    slw 9, 5, 9
+; CHECK32_64-NEXT:    srw 5, 5, 8
+; CHECK32_64-NEXT:    or 6, 6, 9
+; CHECK32_64-NEXT:    or 3, 3, 4
+; CHECK32_64-NEXT:    or 4, 6, 5
+; CHECK32_64-NEXT:    or 3, 3, 10
+; CHECK32_64-NEXT:    or 4, 7, 4
+; CHECK32_64-NEXT:    blr
 ;
 ; CHECK64-LABEL: fshl_i64:
 ; CHECK64:       # %bb.0:
@@ -82,62 +111,119 @@ define i64 @fshl_i64(i64 %x, i64 %y, i64 %z) {
 ; Verify that weird types are minimally supported.
 declare i37 @llvm.fshl.i37(i37, i37, i37)
 define i37 @fshl_i37(i37 %x, i37 %y, i37 %z) {
-; CHECK32-LABEL: fshl_i37:
-; CHECK32:       # %bb.0:
-; CHECK32-NEXT:    mflr 0
-; CHECK32-NEXT:    stw 0, 4(1)
-; CHECK32-NEXT:    stwu 1, -32(1)
-; CHECK32-NEXT:    .cfi_def_cfa_offset 32
-; CHECK32-NEXT:    .cfi_offset lr, 4
-; CHECK32-NEXT:    .cfi_offset r27, -20
-; CHECK32-NEXT:    .cfi_offset r28, -16
-; CHECK32-NEXT:    .cfi_offset r29, -12
-; CHECK32-NEXT:    .cfi_offset r30, -8
-; CHECK32-NEXT:    stw 27, 12(1) # 4-byte Folded Spill
-; CHECK32-NEXT:    mr 27, 3
-; CHECK32-NEXT:    stw 28, 16(1) # 4-byte Folded Spill
-; CHECK32-NEXT:    mr 28, 4
-; CHECK32-NEXT:    stw 29, 20(1) # 4-byte Folded Spill
-; CHECK32-NEXT:    mr 29, 5
-; CHECK32-NEXT:    stw 30, 24(1) # 4-byte Folded Spill
-; CHECK32-NEXT:    mr 30, 6
-; CHECK32-NEXT:    mr 3, 7
-; CHECK32-NEXT:    mr 4, 8
-; CHECK32-NEXT:    li 5, 0
-; CHECK32-NEXT:    li 6, 37
-; CHECK32-NEXT:    bl __umoddi3
-; CHECK32-NEXT:    clrlwi 6, 4, 26
-; CHECK32-NEXT:    srwi 3, 30, 6
-; CHECK32-NEXT:    not 4, 4
-; CHECK32-NEXT:    subfic 8, 6, 32
-; CHECK32-NEXT:    slwi 5, 30, 26
-; CHECK32-NEXT:    rlwimi 3, 29, 26, 1, 5
-; CHECK32-NEXT:    slw 7, 27, 6
-; CHECK32-NEXT:    clrlwi 4, 4, 26
-; CHECK32-NEXT:    srw 8, 28, 8
-; CHECK32-NEXT:    srw 9, 3, 4
-; CHECK32-NEXT:    srw 5, 5, 4
-; CHECK32-NEXT:    or 7, 7, 8
-; CHECK32-NEXT:    subfic 8, 4, 32
-; CHECK32-NEXT:    addi 4, 4, -32
-; CHECK32-NEXT:    slw 8, 3, 8
-; CHECK32-NEXT:    srw 4, 3, 4
-; CHECK32-NEXT:    addi 3, 6, -32
-; CHECK32-NEXT:    slw 3, 28, 3
-; CHECK32-NEXT:    or 5, 5, 8
-; CHECK32-NEXT:    or 3, 7, 3
-; CHECK32-NEXT:    or 4, 5, 4
-; CHECK32-NEXT:    slw 5, 28, 6
-; CHECK32-NEXT:    or 3, 3, 9
-; CHECK32-NEXT:    or 4, 5, 4
-; CHECK32-NEXT:    lwz 30, 24(1) # 4-byte Folded Reload
-; CHECK32-NEXT:    lwz 29, 20(1) # 4-byte Folded Reload
-; CHECK32-NEXT:    lwz 28, 16(1) # 4-byte Folded Reload
-; CHECK32-NEXT:    lwz 27, 12(1) # 4-byte Folded Reload
-; CHECK32-NEXT:    lwz 0, 36(1)
-; CHECK32-NEXT:    addi 1, 1, 32
-; CHECK32-NEXT:    mtlr 0
-; CHECK32-NEXT:    blr
+; CHECK32_32-LABEL: fshl_i37:
+; CHECK32_32:       # %bb.0:
+; CHECK32_32-NEXT:    mflr 0
+; CHECK32_32-NEXT:    stw 0, 4(1)
+; CHECK32_32-NEXT:    stwu 1, -32(1)
+; CHECK32_32-NEXT:    .cfi_def_cfa_offset 32
+; CHECK32_32-NEXT:    .cfi_offset lr, 4
+; CHECK32_32-NEXT:    .cfi_offset r27, -20
+; CHECK32_32-NEXT:    .cfi_offset r28, -16
+; CHECK32_32-NEXT:    .cfi_offset r29, -12
+; CHECK32_32-NEXT:    .cfi_offset r30, -8
+; CHECK32_32-NEXT:    stw 27, 12(1) # 4-byte Folded Spill
+; CHECK32_32-NEXT:    mr 27, 3
+; CHECK32_32-NEXT:    stw 28, 16(1) # 4-byte Folded Spill
+; CHECK32_32-NEXT:    mr 28, 4
+; CHECK32_32-NEXT:    stw 29, 20(1) # 4-byte Folded Spill
+; CHECK32_32-NEXT:    mr 29, 5
+; CHECK32_32-NEXT:    stw 30, 24(1) # 4-byte Folded Spill
+; CHECK32_32-NEXT:    mr 30, 6
+; CHECK32_32-NEXT:    mr 3, 7
+; CHECK32_32-NEXT:    mr 4, 8
+; CHECK32_32-NEXT:    li 5, 0
+; CHECK32_32-NEXT:    li 6, 37
+; CHECK32_32-NEXT:    bl __umoddi3
+; CHECK32_32-NEXT:    clrlwi 6, 4, 26
+; CHECK32_32-NEXT:    srwi 3, 30, 6
+; CHECK32_32-NEXT:    not 4, 4
+; CHECK32_32-NEXT:    subfic 8, 6, 32
+; CHECK32_32-NEXT:    slwi 5, 30, 26
+; CHECK32_32-NEXT:    rlwimi 3, 29, 26, 1, 5
+; CHECK32_32-NEXT:    slw 7, 27, 6
+; CHECK32_32-NEXT:    clrlwi 4, 4, 26
+; CHECK32_32-NEXT:    srw 8, 28, 8
+; CHECK32_32-NEXT:    srw 9, 3, 4
+; CHECK32_32-NEXT:    srw 5, 5, 4
+; CHECK32_32-NEXT:    or 7, 7, 8
+; CHECK32_32-NEXT:    subfic 8, 4, 32
+; CHECK32_32-NEXT:    addi 4, 4, -32
+; CHECK32_32-NEXT:    slw 8, 3, 8
+; CHECK32_32-NEXT:    srw 4, 3, 4
+; CHECK32_32-NEXT:    addi 3, 6, -32
+; CHECK32_32-NEXT:    slw 3, 28, 3
+; CHECK32_32-NEXT:    or 5, 5, 8
+; CHECK32_32-NEXT:    or 3, 7, 3
+; CHECK32_32-NEXT:    or 4, 5, 4
+; CHECK32_32-NEXT:    slw 5, 28, 6
+; CHECK32_32-NEXT:    or 3, 3, 9
+; CHECK32_32-NEXT:    or 4, 5, 4
+; CHECK32_32-NEXT:    lwz 30, 24(1) # 4-byte Folded Reload
+; CHECK32_32-NEXT:    lwz 29, 20(1) # 4-byte Folded Reload
+; CHECK32_32-NEXT:    lwz 28, 16(1) # 4-byte Folded Reload
+; CHECK32_32-NEXT:    lwz 27, 12(1) # 4-byte Folded Reload
+; CHECK32_32-NEXT:    lwz 0, 36(1)
+; CHECK32_32-NEXT:    addi 1, 1, 32
+; CHECK32_32-NEXT:    mtlr 0
+; CHECK32_32-NEXT:    blr
+;
+; CHECK32_64-LABEL: fshl_i37:
+; CHECK32_64:       # %bb.0:
+; CHECK32_64-NEXT:    mflr 0
+; CHECK32_64-NEXT:    stw 0, 4(1)
+; CHECK32_64-NEXT:    stwu 1, -32(1)
+; CHECK32_64-NEXT:    .cfi_def_cfa_offset 32
+; CHECK32_64-NEXT:    .cfi_offset lr, 4
+; CHECK32_64-NEXT:    .cfi_offset r27, -20
+; CHECK32_64-NEXT:    .cfi_offset r28, -16
+; CHECK32_64-NEXT:    .cfi_offset r29, -12
+; CHECK32_64-NEXT:    .cfi_offset r30, -8
+; CHECK32_64-NEXT:    stw 27, 12(1) # 4-byte Folded Spill
+; CHECK32_64-NEXT:    mr 27, 3
+; CHECK32_64-NEXT:    mr 3, 7
+; CHECK32_64-NEXT:    stw 28, 16(1) # 4-byte Folded Spill
+; CHECK32_64-NEXT:    mr 28, 4
+; CHECK32_64-NEXT:    mr 4, 8
+; CHECK32_64-NEXT:    stw 29, 20(1) # 4-byte Folded Spill
+; CHECK32_64-NEXT:    mr 29, 5
+; CHECK32_64-NEXT:    li 5, 0
+; CHECK32_64-NEXT:    stw 30, 24(1) # 4-byte Folded Spill
+; CHECK32_64-NEXT:    mr 30, 6
+; CHECK32_64-NEXT:    li 6, 37
+; CHECK32_64-NEXT:    bl __umoddi3
+; CHECK32_64-NEXT:    clrlwi 6, 4, 26
+; CHECK32_64-NEXT:    not 4, 4
+; CHECK32_64-NEXT:    subfic 8, 6, 32
+; CHECK32_64-NEXT:    srwi 3, 30, 6
+; CHECK32_64-NEXT:    slw 7, 27, 6
+; CHECK32_64-NEXT:    clrlwi 4, 4, 26
+; CHECK32_64-NEXT:    lwz 27, 12(1) # 4-byte Folded Reload
+; CHECK32_64-NEXT:    srw 8, 28, 8
+; CHECK32_64-NEXT:    rlwimi 3, 29, 26, 1, 5
+; CHECK32_64-NEXT:    lwz 29, 20(1) # 4-byte Folded Reload
+; CHECK32_64-NEXT:    slwi 5, 30, 26
+; CHECK32_64-NEXT:    or 7, 7, 8
+; CHECK32_64-NEXT:    subfic 8, 4, 32
+; CHECK32_64-NEXT:    lwz 30, 24(1) # 4-byte Folded Reload
+; CHECK32_64-NEXT:    addi 9, 6, -32
+; CHECK32_64-NEXT:    srw 10, 3, 4
+; CHECK32_64-NEXT:    srw 5, 5, 4
+; CHECK32_64-NEXT:    addi 4, 4, -32
+; CHECK32_64-NEXT:    slw 8, 3, 8
+; CHECK32_64-NEXT:    slw 9, 28, 9
+; CHECK32_64-NEXT:    srw 3, 3, 4
+; CHECK32_64-NEXT:    or 4, 5, 8
+; CHECK32_64-NEXT:    slw 6, 28, 6
+; CHECK32_64-NEXT:    or 5, 7, 9
+; CHECK32_64-NEXT:    lwz 28, 16(1) # 4-byte Folded Reload
+; CHECK32_64-NEXT:    or 4, 4, 3
+; CHECK32_64-NEXT:    or 3, 5, 10
+; CHECK32_64-NEXT:    lwz 0, 36(1)
+; CHECK32_64-NEXT:    or 4, 6, 4
+; CHECK32_64-NEXT:    addi 1, 1, 32
+; CHECK32_64-NEXT:    mtlr 0
+; CHECK32_64-NEXT:    blr
 ;
 ; CHECK64-LABEL: fshl_i37:
 ; CHECK64:       # %bb.0:
@@ -259,33 +345,61 @@ define i32 @fshr_i32(i32 %x, i32 %y, i32 %z) {
 }
 
 define i64 @fshr_i64(i64 %x, i64 %y, i64 %z) {
-; CHECK32-LABEL: fshr_i64:
-; CHECK32:       # %bb.0:
-; CHECK32-NEXT:    clrlwi 7, 8, 26
-; CHECK32-NEXT:    slwi 9, 4, 1
-; CHECK32-NEXT:    not 8, 8
-; CHECK32-NEXT:    rotlwi 4, 4, 1
-; CHECK32-NEXT:    subfic 10, 7, 32
-; CHECK32-NEXT:    srw 6, 6, 7
-; CHECK32-NEXT:    clrlwi 8, 8, 26
-; CHECK32-NEXT:    rlwimi 4, 3, 1, 0, 30
-; CHECK32-NEXT:    slw 3, 5, 10
-; CHECK32-NEXT:    slw 10, 9, 8
-; CHECK32-NEXT:    slw 4, 4, 8
-; CHECK32-NEXT:    or 3, 6, 3
-; CHECK32-NEXT:    subfic 6, 8, 32
-; CHECK32-NEXT:    addi 8, 8, -32
-; CHECK32-NEXT:    srw 6, 9, 6
-; CHECK32-NEXT:    slw 8, 9, 8
-; CHECK32-NEXT:    addi 9, 7, -32
-; CHECK32-NEXT:    srw 9, 5, 9
-; CHECK32-NEXT:    or 3, 3, 9
-; CHECK32-NEXT:    or 6, 4, 6
-; CHECK32-NEXT:    or 4, 10, 3
-; CHECK32-NEXT:    or 3, 6, 8
-; CHECK32-NEXT:    srw 5, 5, 7
-; CHECK32-NEXT:    or 3, 3, 5
-; CHECK32-NEXT:    blr
+; CHECK32_32-LABEL: fshr_i64:
+; CHECK32_32:       # %bb.0:
+; CHECK32_32-NEXT:    clrlwi 7, 8, 26
+; CHECK32_32-NEXT:    slwi 9, 4, 1
+; CHECK32_32-NEXT:    not 8, 8
+; CHECK32_32-NEXT:    rotlwi 4, 4, 1
+; CHECK32_32-NEXT:    subfic 10, 7, 32
+; CHECK32_32-NEXT:    srw 6, 6, 7
+; CHECK32_32-NEXT:    clrlwi 8, 8, 26
+; CHECK32_32-NEXT:    rlwimi 4, 3, 1, 0, 30
+; CHECK32_32-NEXT:    slw 3, 5, 10
+; CHECK32_32-NEXT:    slw 10, 9, 8
+; CHECK32_32-NEXT:    slw 4, 4, 8
+; CHECK32_32-NEXT:    or 3, 6, 3
+; CHECK32_32-NEXT:    subfic 6, 8, 32
+; CHECK32_32-NEXT:    addi 8, 8, -32
+; CHECK32_32-NEXT:    srw 6, 9, 6
+; CHECK32_32-NEXT:    slw 8, 9, 8
+; CHECK32_32-NEXT:    addi 9, 7, -32
+; CHECK32_32-NEXT:    srw 9, 5, 9
+; CHECK32_32-NEXT:    or 3, 3, 9
+; CHECK32_32-NEXT:    or 6, 4, 6
+; CHECK32_32-NEXT:    or 4, 10, 3
+; CHECK32_32-NEXT:    or 3, 6, 8
+; CHECK32_32-NEXT:    srw 5, 5, 7
+; CHECK32_32-NEXT:    or 3, 3, 5
+; CHECK32_32-NEXT:    blr
+;
+; CHECK32_64-LABEL: fshr_i64:
+; CHECK32_64:       # %bb.0:
+; CHECK32_64-NEXT:    rotlwi 7, 4, 1
+; CHECK32_64-NEXT:    slwi 4, 4, 1
+; CHECK32_64-NEXT:    rlwimi 7, 3, 1, 0, 30
+; CHECK32_64-NEXT:    clrlwi 3, 8, 26
+; CHECK32_64-NEXT:    not 8, 8
+; CHECK32_64-NEXT:    subfic 9, 3, 32
+; CHECK32_64-NEXT:    srw 6, 6, 3
+; CHECK32_64-NEXT:    clrlwi 8, 8, 26
+; CHECK32_64-NEXT:    slw 9, 5, 9
+; CHECK32_64-NEXT:    addi 10, 3, -32
+; CHECK32_64-NEXT:    or 6, 6, 9
+; CHECK32_64-NEXT:    subfic 9, 8, 32
+; CHECK32_64-NEXT:    srw 3, 5, 3
+; CHECK32_64-NEXT:    srw 5, 5, 10
+; CHECK32_64-NEXT:    slw 10, 4, 8
+; CHECK32_64-NEXT:    slw 7, 7, 8
+; CHECK32_64-NEXT:    addi 8, 8, -32
+; CHECK32_64-NEXT:    srw 9, 4, 9
+; CHECK32_64-NEXT:    slw 4, 4, 8
+; CHECK32_64-NEXT:    or 7, 7, 9
+; CHECK32_64-NEXT:    or 5, 6, 5
+; CHECK32_64-NEXT:    or 6, 7, 4
+; CHECK32_64-NEXT:    or 4, 10, 5
+; CHECK32_64-NEXT:    or 3, 6, 3
+; CHECK32_64-NEXT:    blr
 ;
 ; CHECK64-LABEL: fshr_i64:
 ; CHECK64:       # %bb.0:
@@ -302,66 +416,127 @@ define i64 @fshr_i64(i64 %x, i64 %y, i64 %z) {
 ; Verify that weird types are minimally supported.
 declare i37 @llvm.fshr.i37(i37, i37, i37)
 define i37 @fshr_i37(i37 %x, i37 %y, i37 %z) {
-; CHECK32-LABEL: fshr_i37:
-; CHECK32:       # %bb.0:
-; CHECK32-NEXT:    mflr 0
-; CHECK32-NEXT:    stw 0, 4(1)
-; CHECK32-NEXT:    stwu 1, -32(1)
-; CHECK32-NEXT:    .cfi_def_cfa_offset 32
-; CHECK32-NEXT:    .cfi_offset lr, 4
-; CHECK32-NEXT:    .cfi_offset r27, -20
-; CHECK32-NEXT:    .cfi_offset r28, -16
-; CHECK32-NEXT:    .cfi_offset r29, -12
-; CHECK32-NEXT:    .cfi_offset r30, -8
-; CHECK32-NEXT:    stw 27, 12(1) # 4-byte Folded Spill
-; CHECK32-NEXT:    mr 27, 3
-; CHECK32-NEXT:    stw 28, 16(1) # 4-byte Folded Spill
-; CHECK32-NEXT:    mr 28, 4
-; CHECK32-NEXT:    stw 29, 20(1) # 4-byte Folded Spill
-; CHECK32-NEXT:    mr 29, 5
-; CHECK32-NEXT:    stw 30, 24(1) # 4-byte Folded Spill
-; CHECK32-NEXT:    mr 30, 6
-; CHECK32-NEXT:    mr 3, 7
-; CHECK32-NEXT:    mr 4, 8
-; CHECK32-NEXT:    li 5, 0
-; CHECK32-NEXT:    li 6, 37
-; CHECK32-NEXT:    bl __umoddi3
-; CHECK32-NEXT:    addi 4, 4, 27
-; CHECK32-NEXT:    rotlwi 5, 30, 27
-; CHECK32-NEXT:    clrlwi 8, 4, 26
-; CHECK32-NEXT:    slwi 3, 30, 27
-; CHECK32-NEXT:    rotlwi 7, 28, 1
-; CHECK32-NEXT:    rlwimi 5, 29, 27, 0, 4
-; CHECK32-NEXT:    not 4, 4
-; CHECK32-NEXT:    subfic 9, 8, 32
-; CHECK32-NEXT:    slwi 6, 28, 1
-; CHECK32-NEXT:    rlwimi 7, 27, 1, 0, 30
-; CHECK32-NEXT:    srw 3, 3, 8
-; CHECK32-NEXT:    clrlwi 4, 4, 26
-; CHECK32-NEXT:    slw 9, 5, 9
-; CHECK32-NEXT:    slw 10, 6, 4
-; CHECK32-NEXT:    slw 7, 7, 4
-; CHECK32-NEXT:    or 3, 3, 9
-; CHECK32-NEXT:    subfic 9, 4, 32
-; CHECK32-NEXT:    addi 4, 4, -32
-; CHECK32-NEXT:    srw 9, 6, 9
-; CHECK32-NEXT:    slw 6, 6, 4
-; CHECK32-NEXT:    addi 4, 8, -32
-; CHECK32-NEXT:    srw 4, 5, 4
-; CHECK32-NEXT:    or 3, 3, 4
-; CHECK32-NEXT:    or 7, 7, 9
-; CHECK32-NEXT:    or 4, 10, 3
-; CHECK32-NEXT:    or 3, 7, 6
-; CHECK32-NEXT:    srw 5, 5, 8
-; CHECK32-NEXT:    or 3, 3, 5
-; CHECK32-NEXT:    lwz 30, 24(1) # 4-byte Folded Reload
-; CHECK32-NEXT:    lwz 29, 20(1) # 4-byte Folded Reload
-; CHECK32-NEXT:    lwz 28, 16(1) # 4-byte Folded Reload
-; CHECK32-NEXT:    lwz 27, 12(1) # 4-byte Folded Reload
-; CHECK32-NEXT:    lwz 0, 36(1)
-; CHECK32-NEXT:    addi 1, 1, 32
-; CHECK32-NEXT:    mtlr 0
-; CHECK32-NEXT:    blr
+; CHECK32_32-LABEL: fshr_i37:
+; CHECK32_32:       # %bb.0:
+; CHECK32_32-NEXT:    mflr 0
+; CHECK32_32-NEXT:    stw 0, 4(1)
+; CHECK32_32-NEXT:    stwu 1, -32(1)
+; CHECK32_32-NEXT:    .cfi_def_cfa_offset 32
+; CHECK32_32-NEXT:    .cfi_offset lr, 4
+; CHECK32_32-NEXT:    .cfi_offset r27, -20
+; CHECK32_32-NEXT:    .cfi_offset r28, -16
+; CHECK32_32-NEXT:    .cfi_offset r29, -12
+; CHECK32_32-NEXT:    .cfi_offset r30, -8
+; CHECK32_32-NEXT:    stw 27, 12(1) # 4-byte Folded Spill
+; CHECK32_32-NEXT:    mr 27, 3
+; CHECK32_32-NEXT:    stw 28, 16(1) # 4-byte Folded Spill
+; CHECK32_32-NEXT:    mr 28, 4
+; CHECK32_32-NEXT:    stw 29, 20(1) # 4-byte Folded Spill
+; CHECK32_32-NEXT:    mr 29, 5
+; CHECK32_32-NEXT:    stw 30, 24(1) # 4-byte Folded Spill
+; CHECK32_32-NEXT:    mr 30, 6
+; CHECK32_32-NEXT:    mr 3, 7
+; CHECK32_32-NEXT:    mr 4, 8
+; CHECK32_32-NEXT:    li 5, 0
+; CHECK32_32-NEXT:    li 6, 37
+; CHECK32_32-NEXT:    bl __umoddi3
+; CHECK32_32-NEXT:    addi 4, 4, 27
+; CHECK32_32-NEXT:    rotlwi 5, 30, 27
+; CHECK32_32-NEXT:    clrlwi 8, 4, 26
+; CHECK32_32-NEXT:    slwi 3, 30, 27
+; CHECK32_32-NEXT:    rotlwi 7, 28, 1
+; CHECK32_32-NEXT:    rlwimi 5, 29, 27, 0, 4
+; CHECK32_32-NEXT:    not 4, 4
+; CHECK32_32-NEXT:    subfic 9, 8, 32
+; CHECK32_32-NEXT:    slwi 6, 28, 1
+; CHECK32_32-NEXT:    rlwimi 7, 27, 1, 0, 30
+; CHECK32_32-NEXT:    srw 3, 3, 8
+; CHECK32_32-NEXT:    clrlwi 4, 4, 26
+; CHECK32_32-NEXT:    slw 9, 5, 9
+; CHECK32_32-NEXT:    slw 10, 6, 4
+; CHECK32_32-NEXT:    slw 7, 7, 4
+; CHECK32_32-NEXT:    or 3, 3, 9
+; CHECK32_32-NEXT:    subfic 9, 4, 32
+; CHECK32_32-NEXT:    addi 4, 4, -32
+; CHECK32_32-NEXT:    srw 9, 6, 9
+; CHECK32_32-NEXT:    slw 6, 6, 4
+; CHECK32_32-NEXT:    addi 4, 8, -32
+; CHECK32_32-NEXT:    srw 4, 5, 4
+; CHECK32_32-NEXT:    or 3, 3, 4
+; CHECK32_32-NEXT:    or 7, 7, 9
+; CHECK32_32-NEXT:    or 4, 10, 3
+; CHECK32_32-NEXT:    or 3, 7, 6
+; CHECK32_32-NEXT:    srw 5, 5, 8
+; CHECK32_32-NEXT:    or 3, 3, 5
+; CHECK32_32-NEXT:    lwz 30, 24(1) # 4-byte Folded Reload
+; CHECK32_32-NEXT:    lwz 29, 20(1) # 4-byte Folded Reload
+; CHECK32_32-NEXT:    lwz 28, 16(1) # 4-byte Folded Reload
+; CHECK32_32-NEXT:    lwz 27, 12(1) # 4-byte Folded Reload
+; CHECK32_32-NEXT:    lwz 0, 36(1)
+; CHECK32_32-NEXT:    addi 1, 1, 32
+; CHECK32_32-NEXT:    mtlr 0
+; CHECK32_32-NEXT:    blr
+;
+; CHECK32_64-LABEL: fshr_i37:
+; CHECK32_64:       # %bb.0:
+; CHECK32_64-NEXT:    mflr 0
+; CHECK32_64-NEXT:    stw 0, 4(1)
+; CHECK32_64-NEXT:    stwu 1, -32(1)
+; CHECK32_64-NEXT:    .cfi_def_cfa_offset 32
+; CHECK32_64-NEXT:    .cfi_offset lr, 4
+; CHECK32_64-NEXT:    .cfi_offset r27, -20
+; CHECK32_64-NEXT:    .cfi_offset r28, -16
+; CHECK32_64-NEXT:    .cfi_offset r29, -12
+; CHECK32_64-NEXT:    .cfi_offset r30, -8
+; CHECK32_64-NEXT:    stw 27, 12(1) # 4-byte Folded Spill
+; CHECK32_64-NEXT:    mr 27, 3
+; CHECK32_64-NEXT:    mr 3, 7
+; CHECK32_64-NEXT:    stw 28, 16(1) # 4-byte Folded Spill
+; CHECK32_64-NEXT:    mr 28, 4
+; CHECK32_64-NEXT:    mr 4, 8
+; CHECK32_64-NEXT:    stw 29, 20(1) # 4-byte Folded Spill
+; CHECK32_64-NEXT:    mr 29, 5
+; CHECK32_64-NEXT:    li 5, 0
+; CHECK32_64-NEXT:    stw 30, 24(1) # 4-byte Folded Spill
+; CHECK32_64-NEXT:    mr 30, 6
+; CHECK32_64-NEXT:    li 6, 37
+; CHECK32_64-NEXT:    bl __umoddi3
+; CHECK32_64-NEXT:    addi 4, 4, 27
+; CHECK32_64-NEXT:    rotlwi 3, 30, 27
+; CHECK32_64-NEXT:    clrlwi 8, 4, 26
+; CHECK32_64-NEXT:    rlwimi 3, 29, 27, 0, 4
+; CHECK32_64-NEXT:    lwz 29, 20(1) # 4-byte Folded Reload
+; CHECK32_64-NEXT:    slwi 6, 30, 27
+; CHECK32_64-NEXT:    lwz 30, 24(1) # 4-byte Folded Reload
+; CHECK32_64-NEXT:    not 4, 4
+; CHECK32_64-NEXT:    subfic 9, 8, 32
+; CHECK32_64-NEXT:    rotlwi 5, 28, 1
+; CHECK32_64-NEXT:    srw 6, 6, 8
+; CHECK32_64-NEXT:    clrlwi 4, 4, 26
+; CHECK32_64-NEXT:    slw 9, 3, 9
+; CHECK32_64-NEXT:    rlwimi 5, 27, 1, 0, 30
+; CHECK32_64-NEXT:    slwi 7, 28, 1
+; CHECK32_64-NEXT:    lwz 28, 16(1) # 4-byte Folded Reload
+; CHECK32_64-NEXT:    addi 10, 8, -32
+; CHECK32_64-NEXT:    lwz 27, 12(1) # 4-byte Folded Reload
+; CHECK32_64-NEXT:    or 6, 6, 9
+; CHECK32_64-NEXT:    subfic 9, 4, 32
+; CHECK32_64-NEXT:    srw 8, 3, 8
+; CHECK32_64-NEXT:    srw 3, 3, 10
+; CHECK32_64-NEXT:    lwz 0, 36(1)
+; CHECK32_64-NEXT:    slw 10, 7, 4
+; CHECK32_64-NEXT:    slw 5, 5, 4
+; CHECK32_64-NEXT:    addi 4, 4, -32
+; CHECK32_64-NEXT:    srw 9, 7, 9
+; CHECK32_64-NEXT:    slw 4, 7, 4
+; CHECK32_64-NEXT:    or 5, 5, 9
+; CHECK32_64-NEXT:    or 3, 6, 3
+; CHECK32_64-NEXT:    or 5, 5, 4
+; CHECK32_64-NEXT:    or 4, 10, 3
+; CHECK32_64-NEXT:    or 3, 5, 8
+; CHECK32_64-NEXT:    addi 1, 1, 32
+; CHECK32_64-NEXT:    mtlr 0
+; CHECK32_64-NEXT:    blr
 ;
 ; CHECK64-LABEL: fshr_i37:
 ; CHECK64:       # %bb.0:
@@ -484,13 +659,18 @@ define <4 x i32> @fshl_v4i32_shift_by_bitwidth(<4 x i32> %x, <4 x i32> %y) {
 }
 
 define <4 x i32> @fshr_v4i32_shift_by_bitwidth(<4 x i32> %x, <4 x i32> %y) {
-; CHECK32-LABEL: fshr_v4i32_shift_by_bitwidth:
-; CHECK32:       # %bb.0:
-; CHECK32-NEXT:    mr 6, 10
-; CHECK32-NEXT:    mr 5, 9
-; CHECK32-NEXT:    mr 4, 8
-; CHECK32-NEXT:    mr 3, 7
-; CHECK32-NEXT:    blr
+; CHECK32_32-LABEL: fshr_v4i32_shift_by_bitwidth:
+; CHECK32_32:       # %bb.0:
+; CHECK32_32-NEXT:    mr 6, 10
+; CHECK32_32-NEXT:    mr 5, 9
+; CHECK32_32-NEXT:    mr 4, 8
+; CHECK32_32-NEXT:    mr 3, 7
+; CHECK32_32-NEXT:    blr
+;
+; CHECK32_64-LABEL: fshr_v4i32_shift_by_bitwidth:
+; CHECK32_64:       # %bb.0:
+; CHECK32_64-NEXT:    vmr 2, 3
+; CHECK32_64-NEXT:    blr
 ;
 ; CHECK64-LABEL: fshr_v4i32_shift_by_bitwidth:
 ; CHECK64:       # %bb.0: