predicates.md (gpc_reg_operand): Don't allow all hard registers numbered greater...
[platform/upstream/gcc.git] / gcc / config / rs6000 / rs6000.md
index 91bdb7d..e156e14 100644 (file)
@@ -1,5 +1,5 @@
 ;; Machine description for IBM RISC System 6000 (POWER) for GNU C compiler
-;; Copyright (C) 1990-2014 Free Software Foundation, Inc.
+;; Copyright (C) 1990-2015 Free Software Foundation, Inc.
 ;; Contributed by Richard Kenner (kenner@vlsi1.ultra.nyu.edu)
 
 ;; This file is part of GCC.
@@ -69,6 +69,7 @@
    UNSPEC_PROBE_STACK          ; probe stack memory reference
    UNSPEC_TOCPTR               ; address of a word pointing to the TOC
    UNSPEC_TOC                  ; address of the TOC (more-or-less)
+   UNSPEC_TOCSLOT              ; offset from r1 of toc pointer save slot
    UNSPEC_MOVSI_GOT
    UNSPEC_MV_CR_OV             ; move_from_CR_ov_bit
    UNSPEC_FCTIWZ
    UNSPEC_LFIWAX
    UNSPEC_LFIWZX
    UNSPEC_FCTIWUZ
+   UNSPEC_NOP
    UNSPEC_GRP_END_NOP
    UNSPEC_P8V_FMRGOW
    UNSPEC_P8V_MTVSRWZ
    load,store,fpload,fpstore,vecload,vecstore,
    cmp,
    branch,jmpreg,mfjmpr,mtjmpr,trap,isync,sync,load_l,store_c,
-   compare,
    cr_logical,delayed_cr,mfcr,mfcrf,mtcr,
    fpcompare,fp,fpsimple,dmul,sdiv,ddiv,ssqrt,dsqrt,
    brinc,
 ;; If this instruction is microcoded on the CELL processor
 ; The default for load extended, the recorded instructions and rotate/shifts by a variable is always microcoded
 (define_attr "cell_micro" "not,conditional,always"
-  (if_then_else (ior (eq_attr "type" "compare")
-                    (and (eq_attr "type" "shift,exts,mul")
+  (if_then_else (ior (and (eq_attr "type" "shift,exts,mul")
                          (eq_attr "dot" "yes"))
                     (and (eq_attr "type" "load")
                          (eq_attr "sign_extend" "yes"))
 (define_mode_attr f32_sv [(SF "stxsspx %x1,%y0")  (SD "stxsiwzx %x1,%y0")])
 
 ; Definitions for 32-bit fpr direct move
+; At present, the decimal modes are not allowed in the traditional altivec
+; registers, so restrict the constraints to just the traditional FPRs.
 (define_mode_attr f32_dm [(SF "wn") (SD "wh")])
 
+; Definitions for 32-bit VSX
+(define_mode_attr f32_vsx [(SF "ww") (SD "wn")])
+
+; Definitions for 32-bit use of altivec registers
+(define_mode_attr f32_av  [(SF "wu") (SD "wn")])
+
+; Definitions for 64-bit VSX
+(define_mode_attr f64_vsx [(DF "ws") (DD "wn")])
+
+; Definitions for 64-bit direct move
+(define_mode_attr f64_dm  [(DF "wk") (DD "wh")])
+
+; Definitions for 64-bit use of altivec registers
+(define_mode_attr f64_av  [(DF "wv") (DD "wn")])
+
 ; These modes do not fit in integer registers in 32-bit mode.
 ; but on e500v2, the gpr are 64 bit registers
 (define_mode_iterator DIFD [DI (DF "!TARGET_E500_DOUBLE") DD])
 ; SF/DF constraint for arithmetic on VSX registers
 (define_mode_attr Fv           [(SF "wy") (DF "ws")])
 
+; SF/DF constraint for arithmetic on altivec registers
+(define_mode_attr Fa           [(SF "wu") (DF "wv")])
+
 ; s/d suffix for things like fp_addsub_s/fp_addsub_d
 (define_mode_attr Fs           [(SF "s")  (DF "d")])
 
 
 ; Logical operators.
 (define_code_iterator iorxor [ior xor])
-(define_code_attr iorxor [(ior "ior") (xor "xor")])
-(define_code_attr IORXOR [(ior "IOR") (xor "XOR")])
 
 ; Signed/unsigned variants of ops.
 (define_code_iterator any_extend [sign_extend zero_extend])
 ;; either.
 
 ;; Mode attribute for boolean operation register constraints for output
-(define_mode_attr BOOL_REGS_OUTPUT     [(TI    "&r,r,r,wa,v")
+(define_mode_attr BOOL_REGS_OUTPUT     [(TI    "&r,r,r,wt,v")
                                         (PTI   "&r,r,r")
                                         (V16QI "wa,v,&?r,?r,?r")
                                         (V8HI  "wa,v,&?r,?r,?r")
                                         (V1TI  "wa,v,&?r,?r,?r")])
 
 ;; Mode attribute for boolean operation register constraints for operand1
-(define_mode_attr BOOL_REGS_OP1                [(TI    "r,0,r,wa,v")
+(define_mode_attr BOOL_REGS_OP1                [(TI    "r,0,r,wt,v")
                                         (PTI   "r,0,r")
                                         (V16QI "wa,v,r,0,r")
                                         (V8HI  "wa,v,r,0,r")
                                         (V1TI  "wa,v,r,0,r")])
 
 ;; Mode attribute for boolean operation register constraints for operand2
-(define_mode_attr BOOL_REGS_OP2                [(TI    "r,r,0,wa,v")
+(define_mode_attr BOOL_REGS_OP2                [(TI    "r,r,0,wt,v")
                                         (PTI   "r,r,0")
                                         (V16QI "wa,v,r,r,0")
                                         (V8HI  "wa,v,r,r,0")
 ;; Mode attribute for boolean operation register constraints for operand1
 ;; for one_cmpl.  To simplify things, we repeat the constraint where 0
 ;; is used for operand1 or operand2
-(define_mode_attr BOOL_REGS_UNARY      [(TI    "r,0,0,wa,v")
+(define_mode_attr BOOL_REGS_UNARY      [(TI    "r,0,0,wt,v")
                                         (PTI   "r,0,0")
                                         (V16QI "wa,v,r,0,0")
                                         (V8HI  "wa,v,r,0,0")
                                         (V2DI  "wa,v,r,0,0")
                                         (V2DF  "wa,v,r,0,0")
                                         (V1TI  "wa,v,r,0,0")])
+
+;; Reload iterator for creating the function to allocate a base register to
+;; supplement addressing modes.
+(define_mode_iterator RELOAD [V16QI V8HI V4SI V2DI V4SF V2DF V1TI
+                             SF SD SI DF DD DI TI PTI])
+
 \f
 ;; Start with fixed-point load and store insns.  Here we put only the more
 ;; complex forms.  Basic data transfer is done later.
   cr0 = gen_rtx_REG (CCmode, CR0_REGNO);
   emit_insn (gen_dlmzb (scratch_dlmzb, word1, word2, cr0));
   cond = gen_rtx_NE (VOIDmode, cr0, const0_rtx);
-  emit_jump_insn (gen_rtx_SET (VOIDmode,
-                               pc_rtx,
+  emit_jump_insn (gen_rtx_SET (pc_rtx,
                                gen_rtx_IF_THEN_ELSE (VOIDmode,
                                                      cond,
                                                      gen_rtx_LABEL_REF
                                                         end_label),
                                                      pc_rtx)));
   emit_insn (gen_addsi3 (scratch_string, scratch_string, GEN_INT (8)));
-  emit_jump_insn (gen_rtx_SET (VOIDmode,
-                               pc_rtx,
+  emit_jump_insn (gen_rtx_SET (pc_rtx,
                                gen_rtx_LABEL_REF (VOIDmode, loop_label)));
   emit_barrier ();
   emit_label (end_label);
                  (match_operand:SDI 2 "reg_or_add_cint_operand" "")))]
   ""
 {
-  if (<MODE>mode == DImode && ! TARGET_POWERPC64)
+  if (<MODE>mode == DImode && !TARGET_POWERPC64)
     {
-      if (non_short_cint_operand (operands[2], DImode))
-       FAIL;
+      rtx lo0 = gen_lowpart (SImode, operands[0]);
+      rtx lo1 = gen_lowpart (SImode, operands[1]);
+      rtx lo2 = gen_lowpart (SImode, operands[2]);
+      rtx hi0 = gen_highpart (SImode, operands[0]);
+      rtx hi1 = gen_highpart (SImode, operands[1]);
+      rtx hi2 = gen_highpart_mode (SImode, DImode, operands[2]);
+
+      if (!reg_or_short_operand (lo2, SImode))
+       lo2 = force_reg (SImode, lo2);
+      if (!adde_operand (hi2, SImode))
+       hi2 = force_reg (SImode, hi2);
+
+      emit_insn (gen_addsi3_carry (lo0, lo1, lo2));
+      emit_insn (gen_addsi3_carry_in (hi0, hi1, hi2));
+      DONE;
     }
-  else if (GET_CODE (operands[2]) == CONST_INT
-          && ! add_operand (operands[2], <MODE>mode))
+
+  if (CONST_INT_P (operands[2]) && !add_operand (operands[2], <MODE>mode))
     {
       rtx tmp = ((!can_create_pseudo_p ()
                  || rtx_equal_p (operands[0], operands[1]))
     }
 })
 
-;; Discourage ai/addic because of carry but provide it in an alternative
-;; allowing register zero as source.
-(define_insn "*add<mode>3_internal1"
-  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r,r,?r,r")
-       (plus:GPR (match_operand:GPR 1 "gpc_reg_operand" "%r,b,r,b")
-                 (match_operand:GPR 2 "add_operand" "r,I,I,L")))]
-  "!DECIMAL_FLOAT_MODE_P (GET_MODE (operands[0])) && !DECIMAL_FLOAT_MODE_P (GET_MODE (operands[1]))"
+(define_insn "*add<mode>3"
+  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r,r,r")
+       (plus:GPR (match_operand:GPR 1 "gpc_reg_operand" "%r,b,b")
+                 (match_operand:GPR 2 "add_operand" "r,I,L")))]
+  ""
   "@
    add %0,%1,%2
    addi %0,%1,%2
-   addic %0,%1,%2
    addis %0,%1,%v2"
   [(set_attr "type" "add")])
 
   "addis %0,%1,ha16(%2)"
   [(set_attr "type" "add")])
 
-(define_insn "*add<mode>3_internal2"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC (plus:P (match_operand:P 1 "gpc_reg_operand" "%r,r,r,r")
-                           (match_operand:P 2 "reg_or_short_operand" "r,I,r,I"))
+(define_insn_and_split "*add<mode>3_dot"
+  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
+       (compare:CC (plus:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r")
+                             (match_operand:GPR 2 "gpc_reg_operand" "r,r"))
                    (const_int 0)))
-   (clobber (match_scratch:P 3 "=r,r,r,r"))]
-  ""
+   (clobber (match_scratch:GPR 0 "=r,r"))]
+  "<MODE>mode == Pmode"
   "@
-   add. %3,%1,%2
-   addic. %3,%1,%2
-   #
+   add. %0,%1,%2
    #"
-  [(set_attr "type" "add,compare,compare,compare")
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
+  [(set (match_dup 0)
+       (plus:GPR (match_dup 1)
+                (match_dup 2)))
+   (set (match_dup 3)
+       (compare:CC (match_dup 0)
+                   (const_int 0)))]
+  ""
+  [(set_attr "type" "add")
    (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
+   (set_attr "length" "4,8")])
 
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_cr0_operand" "")
-       (compare:CC (plus:GPR (match_operand:GPR 1 "gpc_reg_operand" "")
-                             (match_operand:GPR 2 "reg_or_short_operand" ""))
+(define_insn_and_split "*add<mode>3_dot2"
+  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
+       (compare:CC (plus:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r")
+                             (match_operand:GPR 2 "gpc_reg_operand" "r,r"))
                    (const_int 0)))
-   (clobber (match_scratch:GPR 3 ""))]
-  "reload_completed"
-  [(set (match_dup 3)
+   (set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
        (plus:GPR (match_dup 1)
-                (match_dup 2)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
+                 (match_dup 2)))]
+  "<MODE>mode == Pmode"
+  "@
+   add. %0,%1,%2
+   #"
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
+  [(set (match_dup 0)
+       (plus:GPR (match_dup 1)
+                 (match_dup 2)))
+   (set (match_dup 3)
+       (compare:CC (match_dup 0)
                    (const_int 0)))]
-  "")
+  ""
+  [(set_attr "type" "add")
+   (set_attr "dot" "yes")
+   (set_attr "length" "4,8")])
 
-(define_insn "*add<mode>3_internal3"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC (plus:P (match_operand:P 1 "gpc_reg_operand" "%r,r,r,r")
-                           (match_operand:P 2 "reg_or_short_operand" "r,I,r,I"))
+(define_insn_and_split "*add<mode>3_imm_dot"
+  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
+       (compare:CC (plus:GPR (match_operand:GPR 1 "gpc_reg_operand" "%r,b")
+                             (match_operand:GPR 2 "short_cint_operand" "I,I"))
                    (const_int 0)))
-   (set (match_operand:P 0 "gpc_reg_operand" "=r,r,r,r")
-       (plus:P (match_dup 1)
-               (match_dup 2)))]
-  ""
+   (clobber (match_scratch:GPR 0 "=r,r"))
+   (clobber (reg:GPR CA_REGNO))]
+  "<MODE>mode == Pmode"
   "@
-   add. %0,%1,%2
    addic. %0,%1,%2
-   #
    #"
-  [(set_attr "type" "add,compare,compare,compare")
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
+  [(set (match_dup 0)
+       (plus:GPR (match_dup 1)
+                 (match_dup 2)))
+   (set (match_dup 3)
+       (compare:CC (match_dup 0)
+                   (const_int 0)))]
+  ""
+  [(set_attr "type" "add")
    (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
+   (set_attr "length" "4,8")])
 
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_cr0_operand" "")
-       (compare:CC (plus:P (match_operand:P 1 "gpc_reg_operand" "")
-                           (match_operand:P 2 "reg_or_short_operand" ""))
+(define_insn_and_split "*add<mode>3_imm_dot2"
+  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
+       (compare:CC (plus:GPR (match_operand:GPR 1 "gpc_reg_operand" "%r,b")
+                             (match_operand:GPR 2 "short_cint_operand" "I,I"))
                    (const_int 0)))
-   (set (match_operand:P 0 "gpc_reg_operand" "")
-       (plus:P (match_dup 1) (match_dup 2)))]
-  "reload_completed"
+   (set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
+       (plus:GPR (match_dup 1)
+                 (match_dup 2)))
+   (clobber (reg:GPR CA_REGNO))]
+  "<MODE>mode == Pmode"
+  "@
+   addic. %0,%1,%2
+   #"
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
   [(set (match_dup 0)
-       (plus:P (match_dup 1)
-               (match_dup 2)))
+       (plus:GPR (match_dup 1)
+                 (match_dup 2)))
    (set (match_dup 3)
        (compare:CC (match_dup 0)
                    (const_int 0)))]
-  "")
+  ""
+  [(set_attr "type" "add")
+   (set_attr "dot" "yes")
+   (set_attr "length" "4,8")])
 
 ;; Split an add that we can't do in one insn into two insns, each of which
 ;; does one 16-bit part.  This is used by combine.  Note that the low-order
     FAIL;
 })
 
+
+(define_insn "add<mode>3_carry"
+  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
+       (plus:P (match_operand:P 1 "gpc_reg_operand" "r")
+               (match_operand:P 2 "reg_or_short_operand" "rI")))
+   (set (reg:P CA_REGNO)
+       (ltu:P (plus:P (match_dup 1)
+                      (match_dup 2))
+              (match_dup 1)))]
+  ""
+  "add%I2c %0,%1,%2"
+  [(set_attr "type" "add")])
+
+(define_insn "*add<mode>3_imm_carry_pos"
+  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
+       (plus:P (match_operand:P 1 "gpc_reg_operand" "r")
+               (match_operand:P 2 "short_cint_operand" "n")))
+   (set (reg:P CA_REGNO)
+       (geu:P (match_dup 1)
+              (match_operand:P 3 "const_int_operand" "n")))]
+  "INTVAL (operands[2]) > 0
+   && INTVAL (operands[2]) + INTVAL (operands[3]) == 0"
+  "addic %0,%1,%2"
+  [(set_attr "type" "add")])
+
+(define_insn "*add<mode>3_imm_carry_0"
+  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
+       (match_operand:P 1 "gpc_reg_operand" "r"))
+   (set (reg:P CA_REGNO)
+       (const_int 0))]
+  ""
+  "addic %0,%1,0"
+  [(set_attr "type" "add")])
+
+(define_insn "*add<mode>3_imm_carry_m1"
+  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
+       (plus:P (match_operand:P 1 "gpc_reg_operand" "r")
+               (const_int -1)))
+   (set (reg:P CA_REGNO)
+       (ne:P (match_dup 1)
+             (const_int 0)))]
+  ""
+  "addic %0,%1,-1"
+  [(set_attr "type" "add")])
+
+(define_insn "*add<mode>3_imm_carry_neg"
+  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
+       (plus:P (match_operand:P 1 "gpc_reg_operand" "r")
+               (match_operand:P 2 "short_cint_operand" "n")))
+   (set (reg:P CA_REGNO)
+       (gtu:P (match_dup 1)
+              (match_operand:P 3 "const_int_operand" "n")))]
+  "INTVAL (operands[2]) < 0
+   && INTVAL (operands[2]) + INTVAL (operands[3]) == -1"
+  "addic %0,%1,%2"
+  [(set_attr "type" "add")])
+
+
+(define_expand "add<mode>3_carry_in"
+  [(parallel [
+     (set (match_operand:GPR 0 "gpc_reg_operand")
+         (plus:GPR (plus:GPR (match_operand:GPR 1 "gpc_reg_operand")
+                             (match_operand:GPR 2 "adde_operand"))
+                   (reg:GPR CA_REGNO)))
+     (clobber (reg:GPR CA_REGNO))])]
+  ""
+{
+  if (operands[2] == const0_rtx)
+    {
+      emit_insn (gen_add<mode>3_carry_in_0 (operands[0], operands[1]));
+      DONE;
+    }
+  if (operands[2] == constm1_rtx)
+    {
+      emit_insn (gen_add<mode>3_carry_in_m1 (operands[0], operands[1]));
+      DONE;
+    }
+})
+
+(define_insn "*add<mode>3_carry_in_internal"
+  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
+       (plus:GPR (plus:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")
+                           (match_operand:GPR 2 "gpc_reg_operand" "r"))
+                 (reg:GPR CA_REGNO)))
+   (clobber (reg:GPR CA_REGNO))]
+  ""
+  "adde %0,%1,%2"
+  [(set_attr "type" "add")])
+
+(define_insn "add<mode>3_carry_in_0"
+  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
+       (plus:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")
+                 (reg:GPR CA_REGNO)))
+   (clobber (reg:GPR CA_REGNO))]
+  ""
+  "addze %0,%1"
+  [(set_attr "type" "add")])
+
+(define_insn "add<mode>3_carry_in_m1"
+  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
+       (plus:GPR (plus:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")
+                           (reg:GPR CA_REGNO))
+                 (const_int -1)))
+   (clobber (reg:GPR CA_REGNO))]
+  ""
+  "addme %0,%1"
+  [(set_attr "type" "add")])
+
+
 (define_expand "one_cmpl<mode>2"
   [(set (match_operand:SDI 0 "gpc_reg_operand" "")
        (not:SDI (match_operand:SDI 1 "gpc_reg_operand" "")))]
                   (match_operand:SDI 2 "gpc_reg_operand" "")))]
   ""
 {
-  if (short_cint_operand (operands[1], <MODE>mode)
-      && !(<MODE>mode == DImode && !TARGET_POWERPC64))
+  if (<MODE>mode == DImode && !TARGET_POWERPC64)
+    {
+      rtx lo0 = gen_lowpart (SImode, operands[0]);
+      rtx lo1 = gen_lowpart (SImode, operands[1]);
+      rtx lo2 = gen_lowpart (SImode, operands[2]);
+      rtx hi0 = gen_highpart (SImode, operands[0]);
+      rtx hi1 = gen_highpart_mode (SImode, DImode, operands[1]);
+      rtx hi2 = gen_highpart (SImode, operands[2]);
+
+      if (!reg_or_short_operand (lo1, SImode))
+       lo1 = force_reg (SImode, lo1);
+      if (!adde_operand (hi1, SImode))
+       hi1 = force_reg (SImode, hi1);
+
+      emit_insn (gen_subfsi3_carry (lo0, lo2, lo1));
+      emit_insn (gen_subfsi3_carry_in (hi0, hi2, hi1));
+      DONE;
+    }
+
+  if (short_cint_operand (operands[1], <MODE>mode))
     {
       emit_insn (gen_subf<mode>3_imm (operands[0], operands[2], operands[1]));
       DONE;
   [(set_attr "type" "add")])
 
 
-(define_expand "neg<mode>2"
-  [(set (match_operand:SDI 0 "gpc_reg_operand" "")
-       (neg:SDI (match_operand:SDI 1 "gpc_reg_operand" "")))]
+(define_insn "subf<mode>3_carry"
+  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
+       (minus:P (match_operand:P 2 "reg_or_short_operand" "rI")
+                (match_operand:P 1 "gpc_reg_operand" "r")))
+   (set (reg:P CA_REGNO)
+       (leu:P (match_dup 1)
+              (match_dup 2)))]
   ""
-  "")
+  "subf%I2c %0,%1,%2"
+  [(set_attr "type" "add")])
+
+(define_insn "*subf<mode>3_imm_carry_0"
+  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
+       (neg:P (match_operand:P 1 "gpc_reg_operand" "r")))
+   (set (reg:P CA_REGNO)
+       (eq:P (match_dup 1)
+             (const_int 0)))]
+  ""
+  "subfic %0,%1,0"
+  [(set_attr "type" "add")])
+
+(define_insn "*subf<mode>3_imm_carry_m1"
+  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
+       (not:P (match_operand:P 1 "gpc_reg_operand" "r")))
+   (set (reg:P CA_REGNO)
+       (const_int 1))]
+  ""
+  "subfic %0,%1,-1"
+  [(set_attr "type" "add")])
+
+
+(define_expand "subf<mode>3_carry_in"
+  [(parallel [
+     (set (match_operand:GPR 0 "gpc_reg_operand")
+         (plus:GPR (plus:GPR (not:GPR (match_operand:GPR 1 "gpc_reg_operand"))
+                             (reg:GPR CA_REGNO))
+                   (match_operand:GPR 2 "adde_operand")))
+     (clobber (reg:GPR CA_REGNO))])]
+  ""
+{
+  if (operands[2] == const0_rtx)
+    {
+      emit_insn (gen_subf<mode>3_carry_in_0 (operands[0], operands[1]));
+      DONE;
+    }
+  if (operands[2] == constm1_rtx)
+    {
+      emit_insn (gen_subf<mode>3_carry_in_m1 (operands[0], operands[1]));
+      DONE;
+    }
+})
+
+(define_insn "*subf<mode>3_carry_in_internal"
+  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
+       (plus:GPR (plus:GPR (not:GPR (match_operand:GPR 1 "gpc_reg_operand" "r"))
+                           (reg:GPR CA_REGNO))
+                 (match_operand:GPR 2 "gpc_reg_operand" "r")))
+   (clobber (reg:GPR CA_REGNO))]
+  ""
+  "subfe %0,%1,%2"
+  [(set_attr "type" "add")])
+
+(define_insn "subf<mode>3_carry_in_0"
+  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
+       (plus:GPR (not:GPR (match_operand:GPR 1 "gpc_reg_operand" "r"))
+                 (reg:GPR CA_REGNO)))
+   (clobber (reg:GPR CA_REGNO))]
+  ""
+  "subfze %0,%1"
+  [(set_attr "type" "add")])
+
+(define_insn "subf<mode>3_carry_in_m1"
+  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
+       (plus:GPR (minus:GPR (reg:GPR CA_REGNO)
+                            (match_operand:GPR 1 "gpc_reg_operand" "r"))
+                 (const_int -2)))
+   (clobber (reg:GPR CA_REGNO))]
+  ""
+  "subfme %0,%1"
+  [(set_attr "type" "add")])
+
+(define_insn "subf<mode>3_carry_in_xx"
+  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
+       (plus:GPR (reg:GPR CA_REGNO)
+                 (const_int -1)))
+   (clobber (reg:GPR CA_REGNO))]
+  ""
+  "subfe %0,%0,%0"
+  [(set_attr "type" "add")])
+
 
-(define_insn "*neg<mode>2"
+(define_insn "neg<mode>2"
   [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
        (neg:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")))]
   ""
   [(set_attr "length" "4,4,12")
    (set_attr "type" "load,store,*")])
 
-;; We are always BITS_BIG_ENDIAN, so the (const_int 16) below is
-;; correct for -mlittle as well as -mbig.
 (define_split
   [(set (match_operand:HI 0 "gpc_reg_operand" "")
        (bswap:HI (match_operand:HI 1 "gpc_reg_operand" "")))
    (clobber (match_operand:SI 2 "gpc_reg_operand" ""))]
   "reload_completed"
   [(set (match_dup 3)
-       (zero_extract:SI (match_dup 4)
-                        (const_int 8)
-                        (const_int 16)))
+       (and:SI (lshiftrt:SI (match_dup 4)
+                            (const_int 8))
+               (const_int 255)))
    (set (match_dup 2)
        (and:SI (ashift:SI (match_dup 4)
                           (const_int 8))
                   (bswap:DI
                    (match_operand:DI 1 "reg_or_mem_operand" "")))
              (clobber (match_scratch:DI 2 ""))
-             (clobber (match_scratch:DI 3 ""))
-             (clobber (match_scratch:DI 4 ""))])]
+             (clobber (match_scratch:DI 3 ""))])]
   ""
 {
   if (!REG_P (operands[0]) && !REG_P (operands[1]))
 
 ;; Power7/cell has ldbrx/stdbrx, so use it directly
 (define_insn "*bswapdi2_ldbrx"
-  [(set (match_operand:DI 0 "reg_or_mem_operand" "=&r,Z,??&r")
+  [(set (match_operand:DI 0 "reg_or_mem_operand" "=r,Z,&r")
        (bswap:DI (match_operand:DI 1 "reg_or_mem_operand" "Z,r,r")))
    (clobber (match_scratch:DI 2 "=X,X,&r"))
-   (clobber (match_scratch:DI 3 "=X,X,&r"))
-   (clobber (match_scratch:DI 4 "=X,X,&r"))]
+   (clobber (match_scratch:DI 3 "=X,X,&r"))]
   "TARGET_POWERPC64 && TARGET_LDBRX
    && (REG_P (operands[0]) || REG_P (operands[1]))"
   "@
 
 ;; Non-power7/cell, fall back to use lwbrx/stwbrx
 (define_insn "*bswapdi2_64bit"
-  [(set (match_operand:DI 0 "reg_or_mem_operand" "=&r,Z,&r")
+  [(set (match_operand:DI 0 "reg_or_mem_operand" "=r,Z,&r")
        (bswap:DI (match_operand:DI 1 "reg_or_mem_operand" "Z,r,r")))
    (clobber (match_scratch:DI 2 "=&b,&b,&r"))
-   (clobber (match_scratch:DI 3 "=&r,&r,&r"))
-   (clobber (match_scratch:DI 4 "=&r,X,&r"))]
+   (clobber (match_scratch:DI 3 "=&r,&r,&r"))]
   "TARGET_POWERPC64 && !TARGET_LDBRX
    && (REG_P (operands[0]) || REG_P (operands[1]))
    && !(MEM_P (operands[0]) && MEM_VOLATILE_P (operands[0]))
   [(set (match_operand:DI 0 "gpc_reg_operand" "")
        (bswap:DI (match_operand:DI 1 "indexed_or_indirect_operand" "")))
    (clobber (match_operand:DI 2 "gpc_reg_operand" ""))
-   (clobber (match_operand:DI 3 "gpc_reg_operand" ""))
-   (clobber (match_operand:DI 4 "gpc_reg_operand" ""))]
+   (clobber (match_operand:DI 3 "gpc_reg_operand" ""))]
   "TARGET_POWERPC64 && !TARGET_LDBRX && reload_completed"
   [(const_int 0)]
   "
   rtx src    = operands[1];
   rtx op2    = operands[2];
   rtx op3    = operands[3];
-  rtx op4    = operands[4];
   rtx op3_32 = simplify_gen_subreg (SImode, op3, DImode,
                                    BYTES_BIG_ENDIAN ? 4 : 0);
-  rtx op4_32 = simplify_gen_subreg (SImode, op4, DImode,
-                                   BYTES_BIG_ENDIAN ? 4 : 0);
+  rtx dest_32 = simplify_gen_subreg (SImode, dest, DImode,
+                                    BYTES_BIG_ENDIAN ? 4 : 0);
   rtx addr1;
   rtx addr2;
-  rtx word_high;
-  rtx word_low;
+  rtx word1;
+  rtx word2;
 
   addr1 = XEXP (src, 0);
   if (GET_CODE (addr1) == PLUS)
       addr2 = gen_rtx_PLUS (Pmode, op2, addr1);
     }
 
+  word1 = change_address (src, SImode, addr1);
+  word2 = change_address (src, SImode, addr2);
+
   if (BYTES_BIG_ENDIAN)
     {
-      word_high = change_address (src, SImode, addr1);
-      word_low  = change_address (src, SImode, addr2);
+      emit_insn (gen_bswapsi2 (op3_32, word2));
+      emit_insn (gen_bswapsi2 (dest_32, word1));
     }
   else
     {
-      word_high = change_address (src, SImode, addr2);
-      word_low  = change_address (src, SImode, addr1);
+      emit_insn (gen_bswapsi2 (op3_32, word1));
+      emit_insn (gen_bswapsi2 (dest_32, word2));
     }
 
-  emit_insn (gen_bswapsi2 (op3_32, word_low));
-  emit_insn (gen_bswapsi2 (op4_32, word_high));
-  emit_insn (gen_ashldi3 (dest, op3, GEN_INT (32)));
-  emit_insn (gen_iordi3 (dest, dest, op4));
+  emit_insn (gen_ashldi3 (op3, op3, GEN_INT (32)));
+  emit_insn (gen_iordi3 (dest, dest, op3));
   DONE;
 }")
 
   [(set (match_operand:DI 0 "indexed_or_indirect_operand" "")
        (bswap:DI (match_operand:DI 1 "gpc_reg_operand" "")))
    (clobber (match_operand:DI 2 "gpc_reg_operand" ""))
-   (clobber (match_operand:DI 3 "gpc_reg_operand" ""))
-   (clobber (match_operand:DI 4 "" ""))]
+   (clobber (match_operand:DI 3 "gpc_reg_operand" ""))]
   "TARGET_POWERPC64 && !TARGET_LDBRX && reload_completed"
   [(const_int 0)]
   "
                                    BYTES_BIG_ENDIAN ? 4 : 0);
   rtx addr1;
   rtx addr2;
-  rtx word_high;
-  rtx word_low;
+  rtx word1;
+  rtx word2;
 
   addr1 = XEXP (dest, 0);
   if (GET_CODE (addr1) == PLUS)
       addr2 = gen_rtx_PLUS (Pmode, op2, addr1);
     }
 
+  word1 = change_address (dest, SImode, addr1);
+  word2 = change_address (dest, SImode, addr2);
+
   emit_insn (gen_lshrdi3 (op3, src, GEN_INT (32)));
+
   if (BYTES_BIG_ENDIAN)
     {
-      word_high = change_address (dest, SImode, addr1);
-      word_low  = change_address (dest, SImode, addr2);
+      emit_insn (gen_bswapsi2 (word1, src_si));
+      emit_insn (gen_bswapsi2 (word2, op3_si));
     }
   else
     {
-      word_high = change_address (dest, SImode, addr2);
-      word_low  = change_address (dest, SImode, addr1);
+      emit_insn (gen_bswapsi2 (word2, src_si));
+      emit_insn (gen_bswapsi2 (word1, op3_si));
     }
-  emit_insn (gen_bswapsi2 (word_high, src_si));
-  emit_insn (gen_bswapsi2 (word_low, op3_si));
   DONE;
 }")
 
   [(set (match_operand:DI 0 "gpc_reg_operand" "")
        (bswap:DI (match_operand:DI 1 "gpc_reg_operand" "")))
    (clobber (match_operand:DI 2 "gpc_reg_operand" ""))
-   (clobber (match_operand:DI 3 "gpc_reg_operand" ""))
-   (clobber (match_operand:DI 4 "" ""))]
+   (clobber (match_operand:DI 3 "gpc_reg_operand" ""))]
   "TARGET_POWERPC64 && reload_completed"
   [(const_int 0)]
   "
 }")
 
 (define_insn "bswapdi2_32bit"
-  [(set (match_operand:DI 0 "reg_or_mem_operand" "=&r,Z,&r")
+  [(set (match_operand:DI 0 "reg_or_mem_operand" "=r,Z,?&r")
        (bswap:DI (match_operand:DI 1 "reg_or_mem_operand" "Z,r,r")))
    (clobber (match_scratch:SI 2 "=&b,&b,X"))]
   "!TARGET_POWERPC64 && (REG_P (operands[0]) || REG_P (operands[1]))"
   if (GET_CODE (addr1) == PLUS)
     {
       emit_insn (gen_add3_insn (op2, XEXP (addr1, 0), GEN_INT (4)));
-      if (TARGET_AVOID_XFORM)
+      if (TARGET_AVOID_XFORM
+         || REGNO (XEXP (addr1, 1)) == REGNO (dest2))
        {
          emit_insn (gen_add3_insn (op2, XEXP (addr1, 1), op2));
          addr2 = op2;
       else
        addr2 = gen_rtx_PLUS (SImode, op2, XEXP (addr1, 1));
     }
-  else if (TARGET_AVOID_XFORM)
+  else if (TARGET_AVOID_XFORM
+          || REGNO (addr1) == REGNO (dest2))
     {
       emit_insn (gen_add3_insn (op2, addr1, GEN_INT (4)));
       addr2 = op2;
   word2 = change_address (src, SImode, addr2);
 
   emit_insn (gen_bswapsi2 (dest2, word1));
+  /* The REGNO (dest2) tests above ensure that addr2 has not been trashed,
+     thus allowing us to omit an early clobber on the output.  */
   emit_insn (gen_bswapsi2 (dest1, word2));
   DONE;
 }")
    (set_attr "size" "<bits>")])
 
 
-;; For powers of two we can do srai/aze for divide and then adjust for
+;; For powers of two we can do sra[wd]i/addze for divide and then adjust for
 ;; modulus.  If it isn't a power of two, force operands into register and do
 ;; a normal divide.
 (define_expand "div<mode>3"
                 (match_operand:GPR 2 "reg_or_cint_operand" "")))]
   ""
 {
-  if (GET_CODE (operands[2]) != CONST_INT
-      || INTVAL (operands[2]) <= 0
-      || exact_log2 (INTVAL (operands[2])) < 0)
-    operands[2] = force_reg (<MODE>mode, operands[2]);
+  if (CONST_INT_P (operands[2])
+      && INTVAL (operands[2]) > 0
+      && exact_log2 (INTVAL (operands[2])) >= 0)
+    {
+      emit_insn (gen_div<mode>3_sra (operands[0], operands[1], operands[2]));
+      DONE;
+    }
+
+  operands[2] = force_reg (<MODE>mode, operands[2]);
 })
 
 (define_insn "*div<mode>3"
   [(set_attr "type" "div")
    (set_attr "size" "<bits>")])
 
-(define_expand "mod<mode>3"
-  [(use (match_operand:GPR 0 "gpc_reg_operand" ""))
-   (use (match_operand:GPR 1 "gpc_reg_operand" ""))
-   (use (match_operand:GPR 2 "reg_or_cint_operand" ""))]
-  ""
-  "
-{
-  int i;
-  rtx temp1;
-  rtx temp2;
-
-  if (GET_CODE (operands[2]) != CONST_INT
-      || INTVAL (operands[2]) <= 0
-      || (i = exact_log2 (INTVAL (operands[2]))) < 0)
-    FAIL;
-
-  temp1 = gen_reg_rtx (<MODE>mode);
-  temp2 = gen_reg_rtx (<MODE>mode);
-
-  emit_insn (gen_div<mode>3 (temp1, operands[1], operands[2]));
-  emit_insn (gen_ashl<mode>3 (temp2, temp1, GEN_INT (i)));
-  emit_insn (gen_sub<mode>3 (operands[0], operands[1], temp2));
-  DONE;
-}")
-
-(define_insn ""
+(define_insn "div<mode>3_sra"
   [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
        (div:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")
-                (match_operand:GPR 2 "exact_log2_cint_operand" "N")))]
+                (match_operand:GPR 2 "exact_log2_cint_operand" "N")))
+   (clobber (reg:GPR CA_REGNO))]
   ""
   "sra<wd>i %0,%1,%p2\;addze %0,%0"
   [(set_attr "type" "two")
    (set_attr "length" "8")])
 
-(define_insn ""
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC (div:P (match_operand:P 1 "gpc_reg_operand" "r,r")
-                          (match_operand:P 2 "exact_log2_cint_operand" "N,N"))
+(define_insn_and_split "*div<mode>3_sra_dot"
+  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
+       (compare:CC (div:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r")
+                            (match_operand:GPR 2 "exact_log2_cint_operand" "N,N"))
                    (const_int 0)))
-   (clobber (match_scratch:P 3 "=r,r"))]
-  ""
+   (clobber (match_scratch:GPR 0 "=r,r"))
+   (clobber (reg:GPR CA_REGNO))]
+  "<MODE>mode == Pmode"
   "@
-   sra<wd>i %3,%1,%p2\;addze. %3,%3
+   sra<wd>i %0,%1,%p2\;addze. %0,%0
    #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "8,12")
-   (set_attr "cell_micro" "not")])
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_cr0_operand" "")
-       (compare:CC (div:GPR (match_operand:GPR 1 "gpc_reg_operand" "")
-                            (match_operand:GPR 2 "exact_log2_cint_operand"
-                             ""))
-                   (const_int 0)))
-   (clobber (match_scratch:GPR 3 ""))]
-  "reload_completed"
-  [(set (match_dup 3)
-       (div:<MODE> (match_dup 1) (match_dup 2)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
+  [(parallel [(set (match_dup 0)
+                  (div:GPR (match_dup 1)
+                           (match_dup 2)))
+             (clobber (reg:GPR CA_REGNO))])
+   (set (match_dup 3)
+       (compare:CC (match_dup 0)
                    (const_int 0)))]
-  "")
+  ""
+  [(set_attr "type" "two")
+   (set_attr "length" "8,12")
+   (set_attr "cell_micro" "not")])
 
-(define_insn ""
+(define_insn_and_split "*div<mode>3_sra_dot2"
   [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
-       (compare:CC (div:P (match_operand:P 1 "gpc_reg_operand" "r,r")
-                          (match_operand:P 2 "exact_log2_cint_operand" "N,N"))
+       (compare:CC (div:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r")
+                            (match_operand:GPR 2 "exact_log2_cint_operand" "N,N"))
                    (const_int 0)))
-   (set (match_operand:P 0 "gpc_reg_operand" "=r,r")
-       (div:P (match_dup 1) (match_dup 2)))]
-  ""
+   (set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
+       (div:GPR (match_dup 1)
+                (match_dup 2)))
+   (clobber (reg:GPR CA_REGNO))]
+  "<MODE>mode == Pmode"
   "@
    sra<wd>i %0,%1,%p2\;addze. %0,%0
    #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "8,12")
-   (set_attr "cell_micro" "not")])
-
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_cr0_operand" "")
-       (compare:CC (div:GPR (match_operand:GPR 1 "gpc_reg_operand" "")
-                            (match_operand:GPR 2 "exact_log2_cint_operand"
-                             ""))
-                   (const_int 0)))
-   (set (match_operand:GPR 0 "gpc_reg_operand" "")
-       (div:GPR (match_dup 1) (match_dup 2)))]
-  "reload_completed"
-  [(set (match_dup 0)
-       (div:<MODE> (match_dup 1) (match_dup 2)))
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
+  [(parallel [(set (match_dup 0)
+                  (div:GPR (match_dup 1)
+                           (match_dup 2)))
+             (clobber (reg:GPR CA_REGNO))])
    (set (match_dup 3)
        (compare:CC (match_dup 0)
                    (const_int 0)))]
-  "")
+  ""
+  [(set_attr "type" "two")
+   (set_attr "length" "8,12")
+   (set_attr "cell_micro" "not")])
+
+(define_expand "mod<mode>3"
+  [(use (match_operand:GPR 0 "gpc_reg_operand" ""))
+   (use (match_operand:GPR 1 "gpc_reg_operand" ""))
+   (use (match_operand:GPR 2 "reg_or_cint_operand" ""))]
+  ""
+{
+  int i;
+  rtx temp1;
+  rtx temp2;
+
+  if (GET_CODE (operands[2]) != CONST_INT
+      || INTVAL (operands[2]) <= 0
+      || (i = exact_log2 (INTVAL (operands[2]))) < 0)
+    FAIL;
+
+  temp1 = gen_reg_rtx (<MODE>mode);
+  temp2 = gen_reg_rtx (<MODE>mode);
+
+  emit_insn (gen_div<mode>3 (temp1, operands[1], operands[2]));
+  emit_insn (gen_ashl<mode>3 (temp2, temp1, GEN_INT (i)));
+  emit_insn (gen_sub<mode>3 (operands[0], operands[1], temp2));
+  DONE;
+})
 \f
 ;; Logical instructions
 ;; The logical instructions are mostly combined by using match_operator,
   if (logical_const_operand (operands[2], <MODE>mode)
       && !any_mask_operand (operands[2], <MODE>mode))
     {
-      emit_insn (gen_and<mode>3_imm (operands[0], operands[1], operands[2]));
-      DONE;
+      if (rs6000_gen_cell_microcode)
+       {
+         emit_insn (gen_and<mode>3_imm (operands[0], operands[1], operands[2]));
+         DONE;
+       }
+      else
+       operands[2] = force_reg (<MODE>mode, operands[2]);
     }
 
   if ((<MODE>mode == DImode && !and64_2_operand (operands[2], <MODE>mode))
   [(set_attr "length" "8")])
 
 
-(define_expand "<iorxor><mode>3"
+(define_expand "<code><mode>3"
   [(set (match_operand:SDI 0 "gpc_reg_operand" "")
        (iorxor:SDI (match_operand:SDI 1 "gpc_reg_operand" "")
                    (match_operand:SDI 2 "reg_or_cint_operand" "")))]
 {
   if (<MODE>mode == DImode && !TARGET_POWERPC64)
     {
-      rs6000_split_logical (operands, <IORXOR>, false, false, false);
+      rs6000_split_logical (operands, <CODE>, false, false, false);
       DONE;
     }
 
       HOST_WIDE_INT lo = value & 0xffff;
       HOST_WIDE_INT hi = value - lo;
 
-      emit_insn (gen_<iorxor><mode>3 (tmp, operands[1], GEN_INT (hi)));
-      emit_insn (gen_<iorxor><mode>3 (operands[0], tmp, GEN_INT (lo)));
+      emit_insn (gen_<code><mode>3 (tmp, operands[1], GEN_INT (hi)));
+      emit_insn (gen_<code><mode>3 (operands[0], tmp, GEN_INT (lo)));
       DONE;
     }
 
     emit_insn (gen_extzvdi_internal (operands[0], operands[1], operands[2],
                                     operands[3]));
   else
-    emit_insn (gen_extzvsi_internal (operands[0], operands[1], operands[2],
-                                    operands[3]));
+    FAIL;
+
   DONE;
 }")
 
-(define_insn "extzvsi_internal"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-       (zero_extract:SI (match_operand:SI 1 "gpc_reg_operand" "r")
-                        (match_operand:SI 2 "const_int_operand" "i")
-                        (match_operand:SI 3 "const_int_operand" "i")))]
-  ""
-  "*
-{
-  int start = INTVAL (operands[3]) & 31;
-  int size = INTVAL (operands[2]) & 31;
-
-  if (start + size >= 32)
-    operands[3] = const0_rtx;
-  else
-    operands[3] = GEN_INT (start + size);
-  return \"rlwinm %0,%1,%3,%s2,31\";
-}"
-  [(set_attr "type" "shift")])
-
-(define_insn "*extzvsi_internal1"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC (zero_extract:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                        (match_operand:SI 2 "const_int_operand" "i,i")
-                        (match_operand:SI 3 "const_int_operand" "i,i"))
-                   (const_int 0)))
-   (clobber (match_scratch:SI 4 "=r,r"))]
-  ""
-  "*
-{
-  int start = INTVAL (operands[3]) & 31;
-  int size = INTVAL (operands[2]) & 31;
-
-  /* Force split for non-cc0 compare.  */
-  if (which_alternative == 1)
-     return \"#\";
-
-  /* If the bit-field being tested fits in the upper or lower half of a
-     word, it is possible to use andiu. or andil. to test it.  This is
-     useful because the condition register set-use delay is smaller for
-     andi[ul]. than for rlinm.  This doesn't work when the starting bit
-     position is 0 because the LT and GT bits may be set wrong.  */
-
-  if ((start > 0 && start + size <= 16) || start >= 16)
-    {
-      operands[3] = GEN_INT (((1 << (16 - (start & 15)))
-                             - (1 << (16 - (start & 15) - size))));
-      if (start < 16)
-       return \"andis. %4,%1,%3\";
-      else
-       return \"andi. %4,%1,%3\";
-    }
-
-  if (start + size >= 32)
-    operands[3] = const0_rtx;
-  else
-    operands[3] = GEN_INT (start + size);
-  return \"rlwinm. %4,%1,%3,%s2,31\";
-}"
-  [(set_attr "type" "shift")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extract:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                        (match_operand:SI 2 "const_int_operand" "")
-                        (match_operand:SI 3 "const_int_operand" ""))
-                   (const_int 0)))
-   (clobber (match_scratch:SI 4 ""))]
-  "reload_completed"
-  [(set (match_dup 4)
-       (zero_extract:SI (match_dup 1) (match_dup 2)
-                        (match_dup 3)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 4)
-                   (const_int 0)))]
-  "")
-
-(define_insn "*extzvsi_internal2"
-  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
-       (compare:CC (zero_extract:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                        (match_operand:SI 2 "const_int_operand" "i,i")
-                        (match_operand:SI 3 "const_int_operand" "i,i"))
-                   (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
-       (zero_extract:SI (match_dup 1) (match_dup 2) (match_dup 3)))]
-  ""
-  "*
-{
-  int start = INTVAL (operands[3]) & 31;
-  int size = INTVAL (operands[2]) & 31;
-
-  /* Force split for non-cc0 compare.  */
-  if (which_alternative == 1)
-     return \"#\";
-
-  /* Since we are using the output value, we can't ignore any need for
-     a shift.  The bit-field must end at the LSB.  */
-  if (start >= 16 && start + size == 32)
-    {
-      operands[3] = GEN_INT ((1 << size) - 1);
-      return \"andi. %0,%1,%3\";
-    }
-
-  if (start + size >= 32)
-    operands[3] = const0_rtx;
-  else
-    operands[3] = GEN_INT (start + size);
-  return \"rlwinm. %0,%1,%3,%s2,31\";
-}"
-  [(set_attr "type" "shift")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 4 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extract:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                        (match_operand:SI 2 "const_int_operand" "")
-                        (match_operand:SI 3 "const_int_operand" ""))
-                   (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (zero_extract:SI (match_dup 1) (match_dup 2) (match_dup 3)))]
-  "reload_completed"
-  [(set (match_dup 0)
-       (zero_extract:SI (match_dup 1) (match_dup 2) (match_dup 3)))
-   (set (match_dup 4)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
-
 (define_insn "extzvdi_internal"
   [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
        (zero_extract:DI (match_operand:DI 1 "gpc_reg_operand" "r")
    (set_attr "dot" "yes")])
 
 (define_insn "*extzvdi_internal2"
-  [(set (match_operand:CC 4 "gpc_reg_operand" "=x")
+  [(set (match_operand:CC 4 "cc_reg_operand" "=x")
        (compare:CC (zero_extract:DI (match_operand:DI 1 "gpc_reg_operand" "r")
                         (match_operand:SI 2 "const_int_operand" "i")
                         (match_operand:SI 3 "const_int_operand" "i"))
    (set_attr "length" "4,8")])
 
 
-(define_insn "*rotlsi3_internal4"
+(define_insn "*rotlsi3_mask"
   [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (and:SI (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r")
                           (match_operand:SI 2 "reg_or_cint_operand" "rn"))
   [(set_attr "type" "shift")
    (set_attr "maybe_var_shift" "yes")])
 
-(define_insn "*rotlsi3_internal5"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC (and:SI
-                    (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                               (match_operand:SI 2 "reg_or_cint_operand" "rn,rn"))
-                    (match_operand:SI 3 "mask_operand" "n,n"))
-                   (const_int 0)))
-   (clobber (match_scratch:SI 4 "=r,r"))]
-  ""
+(define_insn_and_split "*rotlsi3_mask_dot"
+  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
+       (compare:CC
+        (and:SI (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
+                           (match_operand:SI 2 "reg_or_cint_operand" "rn,rn"))
+                (match_operand:SI 3 "mask_operand" "n,n"))
+        (const_int 0)))
+   (clobber (match_scratch:SI 0 "=r,r"))]
+  "rs6000_gen_cell_microcode
+   && (TARGET_32BIT || UINTVAL (operands[3]) <= 0x7fffffff)"
   "@
-   rlw%I2nm. %4,%1,%h2,%m3,%M3
+   rlw%I2nm. %0,%1,%h2,%m3,%M3
    #"
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[4], CCmode)"
+  [(set (match_dup 0)
+       (and:SI (rotate:SI (match_dup 1)
+                          (match_dup 2))
+               (match_dup 3)))
+   (set (match_dup 4)
+       (compare:CC (match_dup 0)
+                   (const_int 0)))]
+  ""
   [(set_attr "type" "shift")
    (set_attr "maybe_var_shift" "yes")
    (set_attr "dot" "yes")
    (set_attr "length" "4,8")])
 
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (and:SI
-                    (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                               (match_operand:SI 2 "reg_or_cint_operand" ""))
-                    (match_operand:SI 3 "mask_operand" ""))
-                   (const_int 0)))
-   (clobber (match_scratch:SI 4 ""))]
-  "reload_completed"
-  [(set (match_dup 4)
-       (and:SI (rotate:SI (match_dup 1)
-                               (match_dup 2))
-                    (match_dup 3)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 4)
-                   (const_int 0)))]
-  "")
-
-(define_insn "*rotlsi3_internal6"
+(define_insn_and_split "*rotlsi3_mask_dot2"
   [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
-       (compare:CC (and:SI
-                    (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                               (match_operand:SI 2 "reg_or_cint_operand" "rn,rn"))
-                    (match_operand:SI 3 "mask_operand" "n,n"))
-                   (const_int 0)))
+       (compare:CC
+        (and:SI (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
+                           (match_operand:SI 2 "reg_or_cint_operand" "rn,rn"))
+                (match_operand:SI 3 "mask_operand" "n,n"))
+        (const_int 0)))
    (set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
-       (and:SI (rotate:SI (match_dup 1) (match_dup 2)) (match_dup 3)))]
-  ""
+       (and:SI (rotate:SI (match_dup 1)
+                          (match_dup 2))
+               (match_dup 3)))]
+  "rs6000_gen_cell_microcode
+   && (TARGET_32BIT || UINTVAL (operands[3]) <= 0x7fffffff)"
   "@
    rlw%I2nm. %0,%1,%h2,%m3,%M3
    #"
-  [(set_attr "type" "shift")
-   (set_attr "maybe_var_shift" "yes")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 4 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (and:SI
-                    (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                               (match_operand:SI 2 "reg_or_cint_operand" ""))
-                    (match_operand:SI 3 "mask_operand" ""))
-                   (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (and:SI (rotate:SI (match_dup 1) (match_dup 2)) (match_dup 3)))]
-  "reload_completed"
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[4], CCmode)"
   [(set (match_dup 0)
-       (and:SI (rotate:SI (match_dup 1) (match_dup 2)) (match_dup 3)))
+       (and:SI (rotate:SI (match_dup 1)
+                          (match_dup 2))
+               (match_dup 3)))
    (set (match_dup 4)
        (compare:CC (match_dup 0)
                    (const_int 0)))]
-  "")
-
-(define_insn "*rotlsi3_internal7le"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-       (zero_extend:SI
-        (subreg:QI
-         (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r")
-                    (match_operand:SI 2 "reg_or_cint_operand" "ri")) 0)))]
-  "!BYTES_BIG_ENDIAN"
-  "rlw%I2nm %0,%1,%h2,0xff"
-  [(set (attr "cell_micro")
-     (if_then_else (match_operand:SI 2 "const_int_operand" "")
-       (const_string "not")
-       (const_string "always")))
-   (set_attr "type" "shift")])
-
-(define_insn "*rotlsi3_internal7be"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-       (zero_extend:SI
-        (subreg:QI
-         (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r")
-                    (match_operand:SI 2 "reg_or_cint_operand" "ri")) 3)))]
-  "BYTES_BIG_ENDIAN"
-  "rlw%I2nm %0,%1,%h2,0xff"
-  [(set (attr "cell_micro")
-     (if_then_else (match_operand:SI 2 "const_int_operand" "")
-       (const_string "not")
-       (const_string "always")))
-   (set_attr "type" "shift")])
-
-(define_insn "*rotlsi3_internal8le"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC (zero_extend:SI
-                    (subreg:QI
-                     (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                                (match_operand:SI 2 "reg_or_cint_operand" "rn,rn")) 0))
-                   (const_int 0)))
-   (clobber (match_scratch:SI 3 "=r,r"))]
-  "!BYTES_BIG_ENDIAN"
-  "@
-   rlw%I2nm. %3,%1,%h2,0xff
-   #"
+  ""
   [(set_attr "type" "shift")
    (set_attr "maybe_var_shift" "yes")
    (set_attr "dot" "yes")
    (set_attr "length" "4,8")])
 
-(define_insn "*rotlsi3_internal8be"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC (zero_extend:SI
-                    (subreg:QI
-                     (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                                (match_operand:SI 2 "reg_or_cint_operand" "rn,rn")) 3))
+
+(define_insn "ashl<mode>3"
+  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
+       (ashift:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")
+                   (match_operand:SI 2 "reg_or_cint_operand" "rn")))]
+  ""
+  "sl<wd>%I2 %0,%1,%<hH>2"
+  [(set_attr "type" "shift")
+   (set_attr "maybe_var_shift" "yes")])
+
+(define_insn "*ashlsi3_64"
+  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
+       (zero_extend:DI
+           (ashift:SI (match_operand:SI 1 "gpc_reg_operand" "r")
+                      (match_operand:SI 2 "reg_or_cint_operand" "rn"))))]
+  "TARGET_POWERPC64"
+  "slw%I2 %0,%1,%h2"
+  [(set_attr "type" "shift")
+   (set_attr "maybe_var_shift" "yes")])
+
+(define_insn_and_split "*ashl<mode>3_dot"
+  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
+       (compare:CC (ashift:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r")
+                               (match_operand:SI 2 "reg_or_cint_operand" "rn,rn"))
                    (const_int 0)))
-   (clobber (match_scratch:SI 3 "=r,r"))]
-  "BYTES_BIG_ENDIAN"
+   (clobber (match_scratch:GPR 0 "=r,r"))]
+  "<MODE>mode == Pmode && rs6000_gen_cell_microcode"
   "@
-   rlw%I2nm. %3,%1,%h2,0xff
+   sl<wd>%I2. %0,%1,%<hH>2
    #"
-  [(set_attr "type" "shift")
-   (set_attr "maybe_var_shift" "yes")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:SI
-                    (subreg:QI
-                     (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                                (match_operand:SI 2 "reg_or_cint_operand" "")) 0))
-                   (const_int 0)))
-   (clobber (match_scratch:SI 3 ""))]
-  "!BYTES_BIG_ENDIAN && reload_completed"
-  [(set (match_dup 3)
-       (zero_extend:SI (subreg:QI
-                     (rotate:SI (match_dup 1)
-                                (match_dup 2)) 0)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:SI
-                    (subreg:QI
-                     (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                                (match_operand:SI 2 "reg_or_cint_operand" "")) 3))
-                   (const_int 0)))
-   (clobber (match_scratch:SI 3 ""))]
-  "BYTES_BIG_ENDIAN && reload_completed"
-  [(set (match_dup 3)
-       (zero_extend:SI (subreg:QI
-                     (rotate:SI (match_dup 1)
-                                (match_dup 2)) 3)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
-
-(define_insn "*rotlsi3_internal9le"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
-       (compare:CC (zero_extend:SI
-                    (subreg:QI
-                     (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                                (match_operand:SI 2 "reg_or_cint_operand" "rn,rn")) 0))
-                   (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
-       (zero_extend:SI (subreg:QI (rotate:SI (match_dup 1) (match_dup 2)) 0)))]
-  "!BYTES_BIG_ENDIAN"
-  "@
-   rlw%I2nm. %0,%1,%h2,0xff
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "maybe_var_shift" "yes")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_insn "*rotlsi3_internal9be"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
-       (compare:CC (zero_extend:SI
-                    (subreg:QI
-                     (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                                (match_operand:SI 2 "reg_or_cint_operand" "rn,rn")) 3))
-                   (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
-       (zero_extend:SI (subreg:QI (rotate:SI (match_dup 1) (match_dup 2)) 3)))]
-  "BYTES_BIG_ENDIAN"
-  "@
-   rlw%I2nm. %0,%1,%h2,0xff
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "maybe_var_shift" "yes")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:SI
-                    (subreg:QI
-                     (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                                (match_operand:SI 2 "reg_or_cint_operand" "")) 0))
-                   (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (zero_extend:SI (subreg:QI (rotate:SI (match_dup 1) (match_dup 2)) 0)))]
-  "!BYTES_BIG_ENDIAN && reload_completed"
-  [(set (match_dup 0)
-       (zero_extend:SI (subreg:QI (rotate:SI (match_dup 1) (match_dup 2)) 0)))
-   (set (match_dup 3)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
-
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:SI
-                    (subreg:QI
-                     (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                                (match_operand:SI 2 "reg_or_cint_operand" "")) 3))
-                   (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (zero_extend:SI (subreg:QI (rotate:SI (match_dup 1) (match_dup 2)) 3)))]
-  "BYTES_BIG_ENDIAN && reload_completed"
-  [(set (match_dup 0)
-       (zero_extend:SI (subreg:QI (rotate:SI (match_dup 1) (match_dup 2)) 3)))
-   (set (match_dup 3)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
-
-(define_insn "*rotlsi3_internal10le"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-       (zero_extend:SI
-        (subreg:HI
-         (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r")
-                    (match_operand:SI 2 "reg_or_cint_operand" "rn")) 0)))]
-  "!BYTES_BIG_ENDIAN"
-  "rlw%I2nm %0,%1,%h2,0xffff"
-  [(set_attr "type" "shift")
-   (set_attr "maybe_var_shift" "yes")])
-
-(define_insn "*rotlsi3_internal10be"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-       (zero_extend:SI
-        (subreg:HI
-         (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r")
-                    (match_operand:SI 2 "reg_or_cint_operand" "rn")) 2)))]
-  "BYTES_BIG_ENDIAN"
-  "rlw%I2nm %0,%1,%h2,0xffff"
-  [(set_attr "type" "shift")
-   (set_attr "maybe_var_shift" "yes")])
-
-(define_insn "*rotlsi3_internal11le"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC (zero_extend:SI
-                    (subreg:HI
-                     (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                                (match_operand:SI 2 "reg_or_cint_operand" "rn,rn")) 0))
-                   (const_int 0)))
-   (clobber (match_scratch:SI 3 "=r,r"))]
-  "!BYTES_BIG_ENDIAN"
-  "@
-   rlw%I2nm. %3,%1,%h2,0xffff
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "maybe_var_shift" "yes")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_insn "*rotlsi3_internal11be"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC (zero_extend:SI
-                    (subreg:HI
-                     (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                                (match_operand:SI 2 "reg_or_cint_operand" "rn,rn")) 2))
-                   (const_int 0)))
-   (clobber (match_scratch:SI 3 "=r,r"))]
-  "BYTES_BIG_ENDIAN"
-  "@
-   rlw%I2nm. %3,%1,%h2,0xffff
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "maybe_var_shift" "yes")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:SI
-                    (subreg:HI
-                     (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                                (match_operand:SI 2 "reg_or_cint_operand" "")) 0))
-                   (const_int 0)))
-   (clobber (match_scratch:SI 3 ""))]
-  "!BYTES_BIG_ENDIAN && reload_completed"
-  [(set (match_dup 3)
-       (zero_extend:SI (subreg:HI
-                     (rotate:SI (match_dup 1)
-                                (match_dup 2)) 0)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:SI
-                    (subreg:HI
-                     (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                                (match_operand:SI 2 "reg_or_cint_operand" "")) 2))
-                   (const_int 0)))
-   (clobber (match_scratch:SI 3 ""))]
-  "BYTES_BIG_ENDIAN && reload_completed"
-  [(set (match_dup 3)
-       (zero_extend:SI (subreg:HI
-                     (rotate:SI (match_dup 1)
-                                (match_dup 2)) 2)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
-
-(define_insn "*rotlsi3_internal12le"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
-       (compare:CC (zero_extend:SI
-                    (subreg:HI
-                     (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                                (match_operand:SI 2 "reg_or_cint_operand" "rn,rn")) 0))
-                   (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
-       (zero_extend:SI (subreg:HI (rotate:SI (match_dup 1) (match_dup 2)) 0)))]
-  "!BYTES_BIG_ENDIAN"
-  "@
-   rlw%I2nm. %0,%1,%h2,0xffff
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "maybe_var_shift" "yes")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_insn "*rotlsi3_internal12be"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
-       (compare:CC (zero_extend:SI
-                    (subreg:HI
-                     (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                                (match_operand:SI 2 "reg_or_cint_operand" "rn,rn")) 2))
-                   (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
-       (zero_extend:SI (subreg:HI (rotate:SI (match_dup 1) (match_dup 2)) 2)))]
-  "BYTES_BIG_ENDIAN"
-  "@
-   rlw%I2nm. %0,%1,%h2,0xffff
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "maybe_var_shift" "yes")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:SI
-                    (subreg:HI
-                     (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                                (match_operand:SI 2 "reg_or_cint_operand" "")) 0))
-                   (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (zero_extend:SI (subreg:HI (rotate:SI (match_dup 1) (match_dup 2)) 0)))]
-  "!BYTES_BIG_ENDIAN && reload_completed"
-  [(set (match_dup 0)
-       (zero_extend:SI (subreg:HI (rotate:SI (match_dup 1) (match_dup 2)) 0)))
-   (set (match_dup 3)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
-
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:SI
-                    (subreg:HI
-                     (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                                (match_operand:SI 2 "reg_or_cint_operand" "")) 2))
-                   (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (zero_extend:SI (subreg:HI (rotate:SI (match_dup 1) (match_dup 2)) 2)))]
-  "BYTES_BIG_ENDIAN && reload_completed"
-  [(set (match_dup 0)
-       (zero_extend:SI (subreg:HI (rotate:SI (match_dup 1) (match_dup 2)) 2)))
-   (set (match_dup 3)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
-
-
-(define_insn "ashl<mode>3"
-  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
-       (ashift:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")
-                   (match_operand:SI 2 "reg_or_cint_operand" "rn")))]
-  ""
-  "sl<wd>%I2 %0,%1,%<hH>2"
-  [(set_attr "type" "shift")
-   (set_attr "maybe_var_shift" "yes")])
-
-(define_insn "*ashlsi3_64"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
-       (zero_extend:DI
-           (ashift:SI (match_operand:SI 1 "gpc_reg_operand" "r")
-                      (match_operand:SI 2 "reg_or_cint_operand" "rn"))))]
-  "TARGET_POWERPC64"
-  "slw%I2 %0,%1,%h2"
-  [(set_attr "type" "shift")
-   (set_attr "maybe_var_shift" "yes")])
-
-(define_insn_and_split "*ashl<mode>3_dot"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
-       (compare:CC (ashift:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r")
-                               (match_operand:SI 2 "reg_or_cint_operand" "rn,rn"))
-                   (const_int 0)))
-   (clobber (match_scratch:GPR 0 "=r,r"))]
-  "<MODE>mode == Pmode && rs6000_gen_cell_microcode"
-  "@
-   sl<wd>%I2. %0,%1,%<hH>2
-   #"
-  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
-  [(set (match_dup 0)
-       (ashift:GPR (match_dup 1)
-                   (match_dup 2)))
-   (set (match_dup 3)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  ""
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
+  [(set (match_dup 0)
+       (ashift:GPR (match_dup 1)
+                   (match_dup 2)))
+   (set (match_dup 3)
+       (compare:CC (match_dup 0)
+                   (const_int 0)))]
+  ""
   [(set_attr "type" "shift")
    (set_attr "maybe_var_shift" "yes")
    (set_attr "dot" "yes")
    (set_attr "length" "4,8")])
 
 
-(define_insn "rlwinm"
+(define_insn "*ashlsi3_imm_mask"
   [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (and:SI (ashift:SI (match_operand:SI 1 "gpc_reg_operand" "r")
                           (match_operand:SI 2 "const_int_operand" "i"))
   "rlwinm %0,%1,%h2,%m3,%M3"
   [(set_attr "type" "shift")])
 
-(define_insn ""
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
+(define_insn_and_split "*ashlsi3_imm_mask_dot"
+  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
        (compare:CC
         (and:SI (ashift:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
                            (match_operand:SI 2 "const_int_operand" "i,i"))
                 (match_operand:SI 3 "mask_operand" "n,n"))
         (const_int 0)))
-   (clobber (match_scratch:SI 4 "=r,r"))]
-  "includes_lshift_p (operands[2], operands[3])"
+   (clobber (match_scratch:SI 0 "=r,r"))]
+  "rs6000_gen_cell_microcode
+   && (TARGET_32BIT || UINTVAL (operands[3]) <= 0x7fffffff)
+   && includes_lshift_p (operands[2], operands[3])"
   "@
-   rlwinm. %4,%1,%h2,%m3,%M3
+   rlwinm. %0,%1,%h2,%m3,%M3
    #"
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[4], CCmode)"
+  [(set (match_dup 0)
+       (and:SI (ashift:SI (match_dup 1)
+                          (match_dup 2))
+               (match_dup 3)))
+   (set (match_dup 4)
+       (compare:CC (match_dup 0)
+                   (const_int 0)))]
+  ""
   [(set_attr "type" "shift")
    (set_attr "dot" "yes")
    (set_attr "length" "4,8")])
 
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC
-        (and:SI (ashift:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                           (match_operand:SI 2 "const_int_operand" ""))
-                (match_operand:SI 3 "mask_operand" ""))
-        (const_int 0)))
-   (clobber (match_scratch:SI 4 ""))]
-  "includes_lshift_p (operands[2], operands[3]) && reload_completed"
-  [(set (match_dup 4)
-       (and:SI (ashift:SI (match_dup 1) (match_dup 2))
-                (match_dup 3)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 4)
-                   (const_int 0)))]
-  "")
-
-(define_insn ""
+(define_insn_and_split "*ashlsi3_imm_mask_dot2"
   [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
        (compare:CC
         (and:SI (ashift:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
                 (match_operand:SI 3 "mask_operand" "n,n"))
         (const_int 0)))
    (set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
-       (and:SI (ashift:SI (match_dup 1) (match_dup 2)) (match_dup 3)))]
-  "includes_lshift_p (operands[2], operands[3])"
+       (and:SI (ashift:SI (match_dup 1)
+                          (match_dup 2))
+               (match_dup 3)))]
+  "rs6000_gen_cell_microcode
+   && (TARGET_32BIT || UINTVAL (operands[3]) <= 0x7fffffff)
+   && includes_lshift_p (operands[2], operands[3])"
   "@
    rlwinm. %0,%1,%h2,%m3,%M3
    #"
-  [(set_attr "type" "shift")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 4 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC
-        (and:SI (ashift:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                           (match_operand:SI 2 "const_int_operand" ""))
-                (match_operand:SI 3 "mask_operand" ""))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (and:SI (ashift:SI (match_dup 1) (match_dup 2)) (match_dup 3)))]
-  "includes_lshift_p (operands[2], operands[3]) && reload_completed"
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[4], CCmode)"
   [(set (match_dup 0)
-       (and:SI (ashift:SI (match_dup 1) (match_dup 2)) (match_dup 3)))
+       (and:SI (ashift:SI (match_dup 1)
+                          (match_dup 2))
+               (match_dup 3)))
    (set (match_dup 4)
        (compare:CC (match_dup 0)
                    (const_int 0)))]
-  "")
+  ""
+  [(set_attr "type" "shift")
+   (set_attr "dot" "yes")
+   (set_attr "length" "4,8")])
 
 
 (define_insn "lshr<mode>3"
    (set_attr "length" "4,8")])
 
 
-(define_insn ""
+(define_insn "*lshrsi3_imm_mask"
   [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (and:SI (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r")
                             (match_operand:SI 2 "const_int_operand" "i"))
   "rlwinm %0,%1,%s2,%m3,%M3"
   [(set_attr "type" "shift")])
 
-(define_insn ""
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
+(define_insn_and_split "*lshrsi3_imm_mask_dot"
+  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
        (compare:CC
         (and:SI (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
                              (match_operand:SI 2 "const_int_operand" "i,i"))
                 (match_operand:SI 3 "mask_operand" "n,n"))
         (const_int 0)))
-   (clobber (match_scratch:SI 4 "=r,r"))]
-  "includes_rshift_p (operands[2], operands[3])"
+   (clobber (match_scratch:SI 0 "=r,r"))]
+  "rs6000_gen_cell_microcode
+   && (TARGET_32BIT || UINTVAL (operands[3]) <= 0x7fffffff)
+   && includes_rshift_p (operands[2], operands[3])"
   "@
-   rlwinm. %4,%1,%s2,%m3,%M3
+   rlwinm. %0,%1,%s2,%m3,%M3
    #"
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[4], CCmode)"
+  [(set (match_dup 0)
+       (and:SI (lshiftrt:SI (match_dup 1)
+                            (match_dup 2))
+               (match_dup 3)))
+   (set (match_dup 4)
+       (compare:CC (match_dup 0)
+                   (const_int 0)))]
+  ""
   [(set_attr "type" "shift")
    (set_attr "dot" "yes")
    (set_attr "length" "4,8")])
 
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC
-        (and:SI (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                             (match_operand:SI 2 "const_int_operand" ""))
-                (match_operand:SI 3 "mask_operand" ""))
-        (const_int 0)))
-   (clobber (match_scratch:SI 4 ""))]
-  "includes_rshift_p (operands[2], operands[3]) && reload_completed"
-  [(set (match_dup 4)
-       (and:SI (lshiftrt:SI (match_dup 1) (match_dup 2))
-                (match_dup 3)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 4)
-                   (const_int 0)))]
-  "")
-
-(define_insn ""
+(define_insn_and_split "*lshrsi3_imm_mask_dot2"
   [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
        (compare:CC
         (and:SI (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
                 (match_operand:SI 3 "mask_operand" "n,n"))
         (const_int 0)))
    (set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
-       (and:SI (lshiftrt:SI (match_dup 1) (match_dup 2)) (match_dup 3)))]
-  "includes_rshift_p (operands[2], operands[3])"
+       (and:SI (lshiftrt:SI (match_dup 1)
+                            (match_dup 2))
+               (match_dup 3)))]
+  "rs6000_gen_cell_microcode
+   && (TARGET_32BIT || UINTVAL (operands[3]) <= 0x7fffffff)
+   && includes_rshift_p (operands[2], operands[3])"
   "@
    rlwinm. %0,%1,%s2,%m3,%M3
    #"
-  [(set_attr "type" "shift")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 4 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC
-        (and:SI (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                             (match_operand:SI 2 "const_int_operand" ""))
-                (match_operand:SI 3 "mask_operand" ""))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (and:SI (lshiftrt:SI (match_dup 1) (match_dup 2)) (match_dup 3)))]
-  "includes_rshift_p (operands[2], operands[3]) && reload_completed"
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[4], CCmode)"
   [(set (match_dup 0)
-       (and:SI (lshiftrt:SI (match_dup 1) (match_dup 2)) (match_dup 3)))
+       (and:SI (lshiftrt:SI (match_dup 1)
+                            (match_dup 2))
+               (match_dup 3)))
    (set (match_dup 4)
        (compare:CC (match_dup 0)
                    (const_int 0)))]
-  "")
-
-(define_insn "*lshiftrt_internal1le"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-       (zero_extend:SI
-        (subreg:QI
-         (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r")
-                      (match_operand:SI 2 "const_int_operand" "i")) 0)))]
-  "!BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (255))"
-  "rlwinm %0,%1,%s2,0xff"
-  [(set_attr "type" "shift")])
-
-(define_insn "*lshiftrt_internal1be"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-       (zero_extend:SI
-        (subreg:QI
-         (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r")
-                      (match_operand:SI 2 "const_int_operand" "i")) 3)))]
-  "BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (255))"
-  "rlwinm %0,%1,%s2,0xff"
-  [(set_attr "type" "shift")])
-
-(define_insn "*lshiftrt_internal2le"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (zero_extend:SI
-         (subreg:QI
-          (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                       (match_operand:SI 2 "const_int_operand" "i,i")) 0))
-        (const_int 0)))
-   (clobber (match_scratch:SI 3 "=r,r"))]
-  "!BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (255))"
-  "@
-   rlwinm. %3,%1,%s2,0xff
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_insn "*lshiftrt_internal2be"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (zero_extend:SI
-         (subreg:QI
-          (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                       (match_operand:SI 2 "const_int_operand" "i,i")) 3))
-        (const_int 0)))
-   (clobber (match_scratch:SI 3 "=r,r"))]
-  "BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (255))"
-  "@
-   rlwinm. %3,%1,%s2,0xff
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC
-        (zero_extend:SI
-         (subreg:QI
-          (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                       (match_operand:SI 2 "const_int_operand" "")) 0))
-        (const_int 0)))
-   (clobber (match_scratch:SI 3 ""))]
-  "!BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (255)) && reload_completed"
-  [(set (match_dup 3)
-       (zero_extend:SI (subreg:QI
-          (lshiftrt:SI (match_dup 1)
-                       (match_dup 2)) 0)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC
-        (zero_extend:SI
-         (subreg:QI
-          (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                       (match_operand:SI 2 "const_int_operand" "")) 3))
-        (const_int 0)))
-   (clobber (match_scratch:SI 3 ""))]
-  "BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (255)) && reload_completed"
-  [(set (match_dup 3)
-       (zero_extend:SI (subreg:QI
-          (lshiftrt:SI (match_dup 1)
-                       (match_dup 2)) 3)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
-
-(define_insn "*lshiftrt_internal3le"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (zero_extend:SI
-         (subreg:QI
-          (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                       (match_operand:SI 2 "const_int_operand" "i,i")) 0))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
-       (zero_extend:SI (subreg:QI (lshiftrt:SI (match_dup 1) (match_dup 2)) 0)))]
-  "!BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (255))"
-  "@
-   rlwinm. %0,%1,%s2,0xff
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_insn "*lshiftrt_internal3be"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (zero_extend:SI
-         (subreg:QI
-          (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                       (match_operand:SI 2 "const_int_operand" "i,i")) 3))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
-       (zero_extend:SI (subreg:QI (lshiftrt:SI (match_dup 1) (match_dup 2)) 3)))]
-  "BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (255))"
-  "@
-   rlwinm. %0,%1,%s2,0xff
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC
-        (zero_extend:SI
-         (subreg:QI
-          (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                       (match_operand:SI 2 "const_int_operand" "")) 0))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (zero_extend:SI (subreg:QI (lshiftrt:SI (match_dup 1) (match_dup 2)) 0)))]
-  "!BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (255)) && reload_completed"
-  [(set (match_dup 0)
-       (zero_extend:SI (subreg:QI (lshiftrt:SI (match_dup 1) (match_dup 2)) 0)))
-   (set (match_dup 3)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
-
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC
-        (zero_extend:SI
-         (subreg:QI
-          (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                       (match_operand:SI 2 "const_int_operand" "")) 3))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (zero_extend:SI (subreg:QI (lshiftrt:SI (match_dup 1) (match_dup 2)) 3)))]
-  "BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (255)) && reload_completed"
-  [(set (match_dup 0)
-       (zero_extend:SI (subreg:QI (lshiftrt:SI (match_dup 1) (match_dup 2)) 3)))
-   (set (match_dup 3)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
-
-(define_insn "*lshiftrt_internal4le"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-       (zero_extend:SI
-        (subreg:HI
-         (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r")
-                      (match_operand:SI 2 "const_int_operand" "i")) 0)))]
-  "!BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (65535))"
-  "rlwinm %0,%1,%s2,0xffff"
-  [(set_attr "type" "shift")])
-
-(define_insn "*lshiftrt_internal4be"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-       (zero_extend:SI
-        (subreg:HI
-         (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r")
-                      (match_operand:SI 2 "const_int_operand" "i")) 2)))]
-  "BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (65535))"
-  "rlwinm %0,%1,%s2,0xffff"
-  [(set_attr "type" "shift")])
-
-(define_insn "*lshiftrt_internal5le"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (zero_extend:SI
-         (subreg:HI
-          (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                       (match_operand:SI 2 "const_int_operand" "i,i")) 0))
-        (const_int 0)))
-   (clobber (match_scratch:SI 3 "=r,r"))]
-  "!BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (65535))"
-  "@
-   rlwinm. %3,%1,%s2,0xffff
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_insn "*lshiftrt_internal5be"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (zero_extend:SI
-         (subreg:HI
-          (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                       (match_operand:SI 2 "const_int_operand" "i,i")) 2))
-        (const_int 0)))
-   (clobber (match_scratch:SI 3 "=r,r"))]
-  "BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (65535))"
-  "@
-   rlwinm. %3,%1,%s2,0xffff
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC
-        (zero_extend:SI
-         (subreg:HI
-          (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                       (match_operand:SI 2 "const_int_operand" "")) 0))
-        (const_int 0)))
-   (clobber (match_scratch:SI 3 ""))]
-  "!BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (65535)) && reload_completed"
-  [(set (match_dup 3)
-       (zero_extend:SI (subreg:HI
-          (lshiftrt:SI (match_dup 1)
-                       (match_dup 2)) 0)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC
-        (zero_extend:SI
-         (subreg:HI
-          (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                       (match_operand:SI 2 "const_int_operand" "")) 2))
-        (const_int 0)))
-   (clobber (match_scratch:SI 3 ""))]
-  "BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (65535)) && reload_completed"
-  [(set (match_dup 3)
-       (zero_extend:SI (subreg:HI
-          (lshiftrt:SI (match_dup 1)
-                       (match_dup 2)) 2)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
-
-(define_insn "*lshiftrt_internal5le"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (zero_extend:SI
-         (subreg:HI
-          (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                       (match_operand:SI 2 "const_int_operand" "i,i")) 0))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
-       (zero_extend:SI (subreg:HI (lshiftrt:SI (match_dup 1) (match_dup 2)) 0)))]
-  "!BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (65535))"
-  "@
-   rlwinm. %0,%1,%s2,0xffff
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_insn "*lshiftrt_internal5be"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (zero_extend:SI
-         (subreg:HI
-          (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                       (match_operand:SI 2 "const_int_operand" "i,i")) 2))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
-       (zero_extend:SI (subreg:HI (lshiftrt:SI (match_dup 1) (match_dup 2)) 2)))]
-  "BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (65535))"
-  "@
-   rlwinm. %0,%1,%s2,0xffff
-   #"
+  ""
   [(set_attr "type" "shift")
    (set_attr "dot" "yes")
    (set_attr "length" "4,8")])
 
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC
-        (zero_extend:SI
-         (subreg:HI
-          (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                       (match_operand:SI 2 "const_int_operand" "")) 0))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (zero_extend:SI (subreg:HI (lshiftrt:SI (match_dup 1) (match_dup 2)) 0)))]
-  "!BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (65535)) && reload_completed"
-  [(set (match_dup 0)
-       (zero_extend:SI (subreg:HI (lshiftrt:SI (match_dup 1) (match_dup 2)) 0)))
-   (set (match_dup 3)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
-
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC
-        (zero_extend:SI
-         (subreg:HI
-          (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                       (match_operand:SI 2 "const_int_operand" "")) 2))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (zero_extend:SI (subreg:HI (lshiftrt:SI (match_dup 1) (match_dup 2)) 2)))]
-  "BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (65535)) && reload_completed"
-  [(set (match_dup 0)
-       (zero_extend:SI (subreg:HI (lshiftrt:SI (match_dup 1) (match_dup 2)) 2)))
-   (set (match_dup 3)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
-
 
 (define_expand "ashr<mode>3"
   [(parallel [(set (match_operand:GPR 0 "gpc_reg_operand" "")
   "")
 
 (define_insn_and_split "*extendsfdf2_fpr"
-  [(set (match_operand:DF 0 "gpc_reg_operand" "=d,?d,d,ws,?ws,wv")
+  [(set (match_operand:DF 0 "gpc_reg_operand" "=d,?d,d,ws,?ws,wu")
        (float_extend:DF (match_operand:SF 1 "reg_or_mem_operand" "0,f,m,0,wy,Z")))]
   "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT"
   "@
    fmr %0,%1
    lfs%U1%X1 %0,%1
    #
-   xxlor %x0,%x1,%x1
+   xscpsgndp %x0,%x1,%x1
    lxsspx %x0,%y1"
   "&& reload_completed && REG_P (operands[1]) && REGNO (operands[0]) == REGNO (operands[1])"
   [(const_int 0)]
   emit_note (NOTE_INSN_DELETED);
   DONE;
 }
-  [(set_attr "type" "fp,fp,fpload,fp,vecsimple,fpload")])
+  [(set_attr "type" "fp,fp,fpload,fp,fp,fpload")])
 
 (define_expand "truncdfsf2"
   [(set (match_operand:SF 0 "gpc_reg_operand" "")
   "")
 
 (define_insn "*truncdfsf2_fpr"
-  [(set (match_operand:SF 0 "gpc_reg_operand" "=f")
-       (float_truncate:SF (match_operand:DF 1 "gpc_reg_operand" "d")))]
+  [(set (match_operand:SF 0 "gpc_reg_operand" "=f,wy")
+       (float_truncate:SF (match_operand:DF 1 "gpc_reg_operand" "d,ws")))]
   "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT"
-  "frsp %0,%1"
+  "@
+   frsp %0,%1
+   xsrsp %x0,%x1"
   [(set_attr "type" "fp")])
 
 ;; This expander is here to avoid FLOAT_WORDS_BIGENDIAN tests in
   "TARGET_<MODE>_FPR && TARGET_CMPB"
   "@
    fcpsgn %0,%2,%1
-   xscpsgn<Fvsx> %x0,%x2,%x1"
+   xscpsgndp %x0,%x2,%x1"
   [(set_attr "type" "fp")])
 
 ;; For MIN, MAX, and conditional move, we use DEFINE_EXPAND's that involve a
 ; not be needed and also in case the insns are deleted as dead code.
 
 (define_insn_and_split "floatsi<mode>2_lfiwax"
-  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=d")
+  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=<Fv>")
        (float:SFDF (match_operand:SI 1 "nonimmediate_operand" "r")))
-   (clobber (match_scratch:DI 2 "=d"))]
+   (clobber (match_scratch:DI 2 "=wj"))]
   "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT && TARGET_LFIWAX
    && <SI_CONVERT_FP> && can_create_pseudo_p ()"
   "#"
    (set_attr "type" "fpload")])
 
 (define_insn_and_split "floatsi<mode>2_lfiwax_mem"
-  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=d,<rreg2>")
+  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=<Ff>,<Fa>")
        (float:SFDF
         (sign_extend:DI
          (match_operand:SI 1 "indexed_or_indirect_operand" "Z,Z"))))
   [(set_attr "type" "fpload,fpload,mftgpr")])
 
 (define_insn_and_split "floatunssi<mode>2_lfiwzx"
-  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=d")
+  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=<Fv>")
        (unsigned_float:SFDF (match_operand:SI 1 "nonimmediate_operand" "r")))
-   (clobber (match_scratch:DI 2 "=d"))]
+   (clobber (match_scratch:DI 2 "=wj"))]
   "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT && TARGET_LFIWZX
    && <SI_CONVERT_FP>"
   "#"
    (set_attr "type" "fpload")])
 
 (define_insn_and_split "floatunssi<mode>2_lfiwzx_mem"
-  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=d,<rreg2>")
+  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=<Ff>,<Fa>")
        (unsigned_float:SFDF
         (zero_extend:DI
          (match_operand:SI 1 "indexed_or_indirect_operand" "Z,Z"))))
   highword = adjust_address (operands[4], SImode, 0);
   lowword = adjust_address (operands[4], SImode, 4);
   if (! WORDS_BIG_ENDIAN)
-    {
-      rtx tmp;
-      tmp = highword; highword = lowword; lowword = tmp;
-    }
+    std::swap (lowword, highword);
 
   emit_insn (gen_xorsi3 (operands[6], operands[1],
                         GEN_INT (~ (HOST_WIDE_INT) 0x7fffffff)));
   highword = adjust_address (operands[4], SImode, 0);
   lowword = adjust_address (operands[4], SImode, 4);
   if (! WORDS_BIG_ENDIAN)
-    {
-      rtx tmp;
-      tmp = highword; highword = lowword; lowword = tmp;
-    }
+    std::swap (lowword, highword);
 
   emit_move_insn (lowword, operands[1]);
   emit_move_insn (highword, operands[2]);
   "")
 
 (define_insn "*fix_trunc<mode>di2_fctidz"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=d")
-       (fix:DI (match_operand:SFDF 1 "gpc_reg_operand" "d")))]
+  [(set (match_operand:DI 0 "gpc_reg_operand" "=d,wi")
+       (fix:DI (match_operand:SFDF 1 "gpc_reg_operand" "<Ff>,<Fa>")))]
   "TARGET_HARD_FLOAT && TARGET_DOUBLE_FLOAT && TARGET_FPRS
-    && TARGET_FCFID && !VECTOR_UNIT_VSX_P (<MODE>mode)"
-  "fctidz %0,%1"
+    && TARGET_FCFID"
+  "@
+   fctidz %0,%1
+   xscvdpsxds %x0,%x1"
   [(set_attr "type" "fp")])
 
 (define_expand "fixuns_trunc<mode>si2"
   "")
 
 (define_insn "*fixuns_trunc<mode>di2_fctiduz"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=d")
-       (unsigned_fix:DI (match_operand:SFDF 1 "gpc_reg_operand" "d")))]
+  [(set (match_operand:DI 0 "gpc_reg_operand" "=d,wi")
+       (unsigned_fix:DI (match_operand:SFDF 1 "gpc_reg_operand" "<Ff>,<Fa>")))]
   "TARGET_HARD_FLOAT && TARGET_DOUBLE_FLOAT && TARGET_FPRS
-    && TARGET_FCTIDUZ && !VECTOR_UNIT_VSX_P (<MODE>mode)"
-  "fctiduz %0,%1"
+    && TARGET_FCTIDUZ"
+  "@
+   fctiduz %0,%1
+   xscvdpuxds %x0,%x1"
   [(set_attr "type" "fp")])
 
 ; Here, we use (set (reg) (unspec:DI [(fix:SI ...)] UNSPEC_FCTIWZ))
 ; because the first makes it clear that operand 0 is not live
 ; before the instruction.
 (define_insn "fctiwz_<mode>"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=d")
-       (unspec:DI [(fix:SI (match_operand:SFDF 1 "gpc_reg_operand" "d"))]
+  [(set (match_operand:DI 0 "gpc_reg_operand" "=d,wi")
+       (unspec:DI [(fix:SI (match_operand:SFDF 1 "gpc_reg_operand" "<Ff>,<Fv>"))]
                   UNSPEC_FCTIWZ))]
   "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT"
-  "fctiwz %0,%1"
+  "@
+   fctiwz %0,%1
+   xscvdpsxws %x0,%x1"
   [(set_attr "type" "fp")])
 
 (define_insn "fctiwuz_<mode>"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=d")
+  [(set (match_operand:DI 0 "gpc_reg_operand" "=d,wi")
        (unspec:DI [(unsigned_fix:SI
-                    (match_operand:SFDF 1 "gpc_reg_operand" "<rreg2>"))]
+                    (match_operand:SFDF 1 "gpc_reg_operand" "<Ff>,<Fv>"))]
                   UNSPEC_FCTIWUZ))]
   "TARGET_HARD_FLOAT && TARGET_FPRS && <TARGET_FLOAT> && TARGET_FCTIWUZ"
-  "fctiwuz %0,%1"
+  "@
+   fctiwuz %0,%1
+   xscvdpuxws %x0,%x1"
   [(set_attr "type" "fp")])
 
 ;; Only optimize (float (fix x)) -> frz if we are in fast-math mode, since
 ;; since the friz instruction does not truncate the value if the floating
 ;; point value is < LONG_MIN or > LONG_MAX.
 (define_insn "*friz"
-  [(set (match_operand:DF 0 "gpc_reg_operand" "=d")
-       (float:DF (fix:DI (match_operand:DF 1 "gpc_reg_operand" "d"))))]
+  [(set (match_operand:DF 0 "gpc_reg_operand" "=d,ws")
+       (float:DF (fix:DI (match_operand:DF 1 "gpc_reg_operand" "d,ws"))))]
   "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT && TARGET_FPRND
-   && !VECTOR_UNIT_VSX_P (DFmode) && flag_unsafe_math_optimizations
-   && !flag_trapping_math && TARGET_FRIZ"
-  "friz %0,%1"
+   && flag_unsafe_math_optimizations && !flag_trapping_math && TARGET_FRIZ"
+  "@
+   friz %0,%1
+   xsrdpiz %x0,%x1"
   [(set_attr "type" "fp")])
 
 ;; Since FCTIWZ doesn't sign extend the upper bits, we have to do a store and a
   "")
 
 (define_insn "*floatdidf2_fpr"
-  [(set (match_operand:DF 0 "gpc_reg_operand" "=d")
-       (float:DF (match_operand:DI 1 "gpc_reg_operand" "d")))]
-  "TARGET_FCFID && TARGET_HARD_FLOAT && TARGET_DOUBLE_FLOAT && TARGET_FPRS
-   && !VECTOR_UNIT_VSX_P (DFmode)"
-  "fcfid %0,%1"
+  [(set (match_operand:DF 0 "gpc_reg_operand" "=d,ws")
+       (float:DF (match_operand:DI 1 "gpc_reg_operand" "d,wi")))]
+  "TARGET_FCFID && TARGET_HARD_FLOAT && TARGET_DOUBLE_FLOAT && TARGET_FPRS"
+  "@
+   fcfid %0,%1
+   xscvsxddp %x0,%x1"
   [(set_attr "type" "fp")])
 
 ; Allow the combiner to merge source memory operands to the conversion so that
 ; hit.  We will split after reload to avoid the trip through the GPRs
 
 (define_insn_and_split "*floatdidf2_mem"
-  [(set (match_operand:DF 0 "gpc_reg_operand" "=d")
-       (float:DF (match_operand:DI 1 "memory_operand" "m")))
-   (clobber (match_scratch:DI 2 "=d"))]
+  [(set (match_operand:DF 0 "gpc_reg_operand" "=d,ws")
+       (float:DF (match_operand:DI 1 "memory_operand" "m,Z")))
+   (clobber (match_scratch:DI 2 "=d,wi"))]
   "TARGET_HARD_FLOAT && TARGET_DOUBLE_FLOAT && TARGET_FPRS && TARGET_FCFID"
   "#"
   "&& reload_completed"
   [(set (match_operand:DF 0 "gpc_reg_operand" "")
        (unsigned_float:DF
         (match_operand:DI 1 "gpc_reg_operand" "")))]
-  "TARGET_HARD_FLOAT && (TARGET_FCFIDU || VECTOR_UNIT_VSX_P (DFmode))"
+  "TARGET_HARD_FLOAT && TARGET_FCFIDU"
   "")
 
 (define_insn "*floatunsdidf2_fcfidu"
-  [(set (match_operand:DF 0 "gpc_reg_operand" "=d")
-       (unsigned_float:DF (match_operand:DI 1 "gpc_reg_operand" "d")))]
-  "TARGET_HARD_FLOAT && TARGET_FCFIDU && !VECTOR_UNIT_VSX_P (DFmode)"
-  "fcfidu %0,%1"
+  [(set (match_operand:DF 0 "gpc_reg_operand" "=d,ws")
+       (unsigned_float:DF (match_operand:DI 1 "gpc_reg_operand" "d,wi")))]
+  "TARGET_HARD_FLOAT && TARGET_FCFIDU"
+  "@
+   fcfidu %0,%1
+   xscvuxddp %x0,%x1"
   [(set_attr "type" "fp")
    (set_attr "length" "4")])
 
 (define_insn_and_split "*floatunsdidf2_mem"
-  [(set (match_operand:DF 0 "gpc_reg_operand" "=d")
-       (unsigned_float:DF (match_operand:DI 1 "memory_operand" "m")))
-   (clobber (match_scratch:DI 2 "=d"))]
+  [(set (match_operand:DF 0 "gpc_reg_operand" "=d,ws")
+       (unsigned_float:DF (match_operand:DI 1 "memory_operand" "m,Z")))
+   (clobber (match_scratch:DI 2 "=d,wi"))]
   "TARGET_HARD_FLOAT && (TARGET_FCFIDU || VECTOR_UNIT_VSX_P (DFmode))"
   "#"
   "&& reload_completed"
 }")
 
 (define_insn "floatdisf2_fcfids"
-  [(set (match_operand:SF 0 "gpc_reg_operand" "=f")
-       (float:SF (match_operand:DI 1 "gpc_reg_operand" "d")))]
+  [(set (match_operand:SF 0 "gpc_reg_operand" "=f,wy")
+       (float:SF (match_operand:DI 1 "gpc_reg_operand" "d,wi")))]
   "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_SINGLE_FLOAT
    && TARGET_DOUBLE_FLOAT && TARGET_FCFIDS"
-  "fcfids %0,%1"
+  "@
+   fcfids %0,%1
+   xscvsxdsp %x0,%x1"
   [(set_attr "type" "fp")])
 
 (define_insn_and_split "*floatdisf2_mem"
-  [(set (match_operand:SF 0 "gpc_reg_operand" "=f")
-       (float:SF (match_operand:DI 1 "memory_operand" "m")))
-   (clobber (match_scratch:DI 2 "=f"))]
+  [(set (match_operand:SF 0 "gpc_reg_operand" "=f,wy,wy")
+       (float:SF (match_operand:DI 1 "memory_operand" "m,m,Z")))
+   (clobber (match_scratch:DI 2 "=d,d,wi"))]
   "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_SINGLE_FLOAT
    && TARGET_DOUBLE_FLOAT && TARGET_FCFIDS"
   "#"
   [(set (match_operand:SF 0 "gpc_reg_operand" "=f")
         (float:SF (match_operand:DI 1 "gpc_reg_operand" "d")))
    (clobber (match_scratch:DF 2 "=d"))]
-  "TARGET_FCFID && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_SINGLE_FLOAT"
+  "TARGET_FCFID && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_SINGLE_FLOAT
+   && !TARGET_FCFIDS"
   "#"
   "&& reload_completed"
   [(set (match_dup 2)
                           (label_ref (match_operand:DI 2 "" ""))
                           (pc)))
    (set (match_dup 0) (match_dup 1))]
-  "TARGET_POWERPC64 && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_SINGLE_FLOAT"
+  "TARGET_POWERPC64 && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_SINGLE_FLOAT
+   && !TARGET_FCFIDS"
   "
 {
   operands[3] = gen_reg_rtx (DImode);
   "")
 
 (define_insn "floatunsdisf2_fcfidus"
-  [(set (match_operand:SF 0 "gpc_reg_operand" "=f")
-        (unsigned_float:SF (match_operand:DI 1 "gpc_reg_operand" "d")))]
+  [(set (match_operand:SF 0 "gpc_reg_operand" "=f,wu")
+        (unsigned_float:SF (match_operand:DI 1 "gpc_reg_operand" "d,wi")))]
   "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_SINGLE_FLOAT
    && TARGET_DOUBLE_FLOAT && TARGET_FCFIDUS"
-  "fcfidus %0,%1"
+  "@
+   fcfidus %0,%1
+   xscvuxdsp %x0,%x1"
   [(set_attr "type" "fp")])
 
 (define_insn_and_split "*floatunsdisf2_mem"
-  [(set (match_operand:SF 0 "gpc_reg_operand" "=f")
-       (unsigned_float:SF (match_operand:DI 1 "memory_operand" "m")))
-   (clobber (match_scratch:DI 2 "=f"))]
+  [(set (match_operand:SF 0 "gpc_reg_operand" "=f,wy,wy")
+       (unsigned_float:SF (match_operand:DI 1 "memory_operand" "m,m,Z")))
+   (clobber (match_scratch:DI 2 "=d,d,wi"))]
   "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_SINGLE_FLOAT
    && TARGET_DOUBLE_FLOAT && TARGET_FCFIDUS"
   "#"
 ;; (for example, having an input in 7,8 and an output in 6,7).  We
 ;; also allow for the output being the same as one of the inputs.
 
-(define_insn "addti3"
-  [(set (match_operand:TI 0 "gpc_reg_operand" "=&r,&r,r,r")
-       (plus:TI (match_operand:TI 1 "gpc_reg_operand" "%r,r,0,0")
-                (match_operand:TI 2 "reg_or_short_operand" "r,I,r,I")))]
+(define_expand "addti3"
+  [(set (match_operand:TI 0 "gpc_reg_operand" "")
+       (plus:TI (match_operand:TI 1 "gpc_reg_operand" "")
+                (match_operand:TI 2 "reg_or_short_operand" "")))]
   "TARGET_64BIT"
 {
-  if (WORDS_BIG_ENDIAN)
-    return (GET_CODE (operands[2])) != CONST_INT
-           ? \"addc %L0,%L1,%L2\;adde %0,%1,%2\"
-           : \"addic %L0,%L1,%2\;add%G2e %0,%1\";
-  else
-    return (GET_CODE (operands[2])) != CONST_INT
-           ? \"addc %0,%1,%2\;adde %L0,%L1,%L2\"
-           : \"addic %0,%1,%2\;add%G2e %L0,%L1\";
-}
-  [(set_attr "type" "two")
-   (set_attr "length" "8")])
+  rtx lo0 = gen_lowpart (DImode, operands[0]);
+  rtx lo1 = gen_lowpart (DImode, operands[1]);
+  rtx lo2 = gen_lowpart (DImode, operands[2]);
+  rtx hi0 = gen_highpart (DImode, operands[0]);
+  rtx hi1 = gen_highpart (DImode, operands[1]);
+  rtx hi2 = gen_highpart_mode (DImode, TImode, operands[2]);
+
+  if (!reg_or_short_operand (lo2, DImode))
+    lo2 = force_reg (DImode, lo2);
+  if (!adde_operand (hi2, DImode))
+    hi2 = force_reg (DImode, hi2);
 
-(define_insn "subti3"
-  [(set (match_operand:TI 0 "gpc_reg_operand" "=&r,&r,r,r,r")
-       (minus:TI (match_operand:TI 1 "reg_or_short_operand" "r,I,0,r,I")
-                 (match_operand:TI 2 "gpc_reg_operand" "r,r,r,0,0")))]
+  emit_insn (gen_adddi3_carry (lo0, lo1, lo2));
+  emit_insn (gen_adddi3_carry_in (hi0, hi1, hi2));
+  DONE;
+})
+
+(define_expand "subti3"
+  [(set (match_operand:TI 0 "gpc_reg_operand" "")
+       (minus:TI (match_operand:TI 1 "reg_or_short_operand" "")
+                 (match_operand:TI 2 "gpc_reg_operand" "")))]
   "TARGET_64BIT"
 {
-  if (WORDS_BIG_ENDIAN)
-    return (GET_CODE (operands[1]) != CONST_INT)
-           ? \"subfc %L0,%L2,%L1\;subfe %0,%2,%1\"
-           : \"subfic %L0,%L2,%1\;subf%G1e %0,%2\";
-  else
-    return (GET_CODE (operands[1]) != CONST_INT)
-           ? \"subfc %0,%2,%1\;subfe %L0,%L2,%L1\"
-           : \"subfic %0,%2,%1\;subf%G1e %L0,%L2\";
-}
-  [(set_attr "type" "two")
-   (set_attr "length" "8")])
+  rtx lo0 = gen_lowpart (DImode, operands[0]);
+  rtx lo1 = gen_lowpart (DImode, operands[1]);
+  rtx lo2 = gen_lowpart (DImode, operands[2]);
+  rtx hi0 = gen_highpart (DImode, operands[0]);
+  rtx hi1 = gen_highpart_mode (DImode, TImode, operands[1]);
+  rtx hi2 = gen_highpart (DImode, operands[2]);
 
+  if (!reg_or_short_operand (lo1, DImode))
+    lo1 = force_reg (DImode, lo1);
+  if (!adde_operand (hi1, DImode))
+    hi1 = force_reg (DImode, hi1);
 
-;; Define the DImode operations that can be done in a small number
-;; of instructions.  The & constraints are to prevent the register
-;; allocator from allocating registers that overlap with the inputs
-;; (for example, having an input in 7,8 and an output in 6,7).  We
-;; also allow for the output being the same as one of the inputs.
+  emit_insn (gen_subfdi3_carry (lo0, lo2, lo1));
+  emit_insn (gen_subfdi3_carry_in (hi0, hi2, hi1));
+  DONE;
+})
 
-(define_insn "*adddi3_noppc64"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=&r,&r,r,r")
-       (plus:DI (match_operand:DI 1 "gpc_reg_operand" "%r,r,0,0")
-                (match_operand:DI 2 "reg_or_short_operand" "r,I,r,I")))]
-  "! TARGET_POWERPC64"
-  "*
-{
-  if (WORDS_BIG_ENDIAN)
-    return (GET_CODE (operands[2])) != CONST_INT
-           ? \"addc %L0,%L1,%L2\;adde %0,%1,%2\"
-           : \"addic %L0,%L1,%2\;add%G2e %0,%1\";
-  else
-    return (GET_CODE (operands[2])) != CONST_INT
-           ? \"addc %0,%1,%2\;adde %L0,%L1,%L2\"
-           : \"addic %0,%1,%2\;add%G2e %L0,%L1\";
-}"
-  [(set_attr "type" "two")
-   (set_attr "length" "8")])
 
-(define_insn "*subdi3_noppc64"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=&r,&r,r,r,r")
-       (minus:DI (match_operand:DI 1 "reg_or_short_operand" "r,I,0,r,I")
-                 (match_operand:DI 2 "gpc_reg_operand" "r,r,r,0,0")))]
-  "! TARGET_POWERPC64"
-  "*
-{
-  if (WORDS_BIG_ENDIAN)
-    return (GET_CODE (operands[1]) != CONST_INT)
-           ? \"subfc %L0,%L2,%L1\;subfe %0,%2,%1\"
-           : \"subfic %L0,%L2,%1\;subf%G1e %0,%2\";
-  else
-    return (GET_CODE (operands[1]) != CONST_INT)
-           ? \"subfc %0,%2,%1\;subfe %L0,%L2,%L1\"
-           : \"subfic %0,%2,%1\;subf%G1e %L0,%L2\";
-}"
-  [(set_attr "type" "two")
-   (set_attr "length" "8")])
-
-(define_insn "*negdi2_noppc64"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=&r,r")
-       (neg:DI (match_operand:DI 1 "gpc_reg_operand" "r,0")))]
-  "! TARGET_POWERPC64"
-  "*
-{
-  return (WORDS_BIG_ENDIAN)
-    ? \"subfic %L0,%L1,0\;subfze %0,%1\"
-    : \"subfic %0,%1,0\;subfze %L0,%L1\";
-}"
-  [(set_attr "type" "two")
-   (set_attr "length" "8")])
-
-
-;; Shift by a variable amount is too complex to be worth open-coding.  We
-;; just handle shifts by constants.
-(define_insn "ashrdi3_no_power"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=&r,&r")
-       (ashiftrt:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
-                    (match_operand:SI 2 "const_int_operand" "M,i")))
-   (clobber (reg:SI CA_REGNO))]
-  "!TARGET_POWERPC64"
+;; Shift by a variable amount is too complex to be worth open-coding.  We
+;; just handle shifts by constants.
+(define_insn "ashrdi3_no_power"
+  [(set (match_operand:DI 0 "gpc_reg_operand" "=&r,&r")
+       (ashiftrt:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
+                    (match_operand:SI 2 "const_int_operand" "M,i")))
+   (clobber (reg:SI CA_REGNO))]
+  "!TARGET_POWERPC64"
 {
   switch (which_alternative)
     {
                    (const_int 0)))]
   "")
 
-(define_insn "*rotldi3_internal7le"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
-       (zero_extend:DI
-        (subreg:QI
-         (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r")
-                    (match_operand:DI 2 "reg_or_cint_operand" "rn")) 0)))]
-  "TARGET_POWERPC64 && !BYTES_BIG_ENDIAN"
-  "rld%I2cl %0,%1,%H2,56"
-  [(set_attr "type" "shift")
-   (set_attr "maybe_var_shift" "yes")])
 
-(define_insn "*rotldi3_internal7be"
+(define_insn "*ashldi3_internal4"
   [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
-       (zero_extend:DI
-        (subreg:QI
-         (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r")
-                    (match_operand:DI 2 "reg_or_cint_operand" "rn")) 7)))]
-  "TARGET_POWERPC64 && BYTES_BIG_ENDIAN"
-  "rld%I2cl %0,%1,%H2,56"
-  [(set_attr "type" "shift")
-   (set_attr "maybe_var_shift" "yes")])
-
-(define_insn "*rotldi3_internal8le"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC (zero_extend:DI
-                    (subreg:QI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
-                                (match_operand:DI 2 "reg_or_cint_operand" "rn,rn")) 0))
-                   (const_int 0)))
-   (clobber (match_scratch:DI 3 "=r,r"))]
-  "TARGET_64BIT && !BYTES_BIG_ENDIAN"
-  "@
-   rld%I2cl. %3,%1,%H2,56
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "maybe_var_shift" "yes")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
+       (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "r")
+                          (match_operand:SI 2 "const_int_operand" "i"))
+               (match_operand:DI 3 "const_int_operand" "n")))]
+  "TARGET_POWERPC64 && includes_rldic_lshift_p (operands[2], operands[3])"
+  "rldic %0,%1,%H2,%W3"
+  [(set_attr "type" "shift")])
 
-(define_insn "*rotldi3_internal8be"
+(define_insn "ashldi3_internal5"
   [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC (zero_extend:DI
-                    (subreg:QI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
-                                (match_operand:DI 2 "reg_or_cint_operand" "rn,rn")) 7))
-                   (const_int 0)))
-   (clobber (match_scratch:DI 3 "=r,r"))]
-  "TARGET_64BIT && BYTES_BIG_ENDIAN"
+       (compare:CC
+        (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
+                           (match_operand:SI 2 "const_int_operand" "i,i"))
+                (match_operand:DI 3 "const_int_operand" "n,n"))
+        (const_int 0)))
+   (clobber (match_scratch:DI 4 "=r,r"))]
+  "TARGET_64BIT && includes_rldic_lshift_p (operands[2], operands[3])"
   "@
-   rld%I2cl. %3,%1,%H2,56
+   rldic. %4,%1,%H2,%W3
    #"
   [(set_attr "type" "shift")
-   (set_attr "maybe_var_shift" "yes")
    (set_attr "dot" "yes")
    (set_attr "length" "4,8")])
 
 (define_split
   [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:DI
-                    (subreg:QI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                                (match_operand:DI 2 "reg_or_cint_operand" "")) 0))
-                   (const_int 0)))
-   (clobber (match_scratch:DI 3 ""))]
-  "TARGET_POWERPC64 && !BYTES_BIG_ENDIAN && reload_completed"
-  [(set (match_dup 3)
-       (zero_extend:DI (subreg:QI
-                     (rotate:DI (match_dup 1)
-                                (match_dup 2)) 0)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:DI
-                    (subreg:QI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                                (match_operand:DI 2 "reg_or_cint_operand" "")) 7))
-                   (const_int 0)))
-   (clobber (match_scratch:DI 3 ""))]
-  "TARGET_POWERPC64 && BYTES_BIG_ENDIAN && reload_completed"
-  [(set (match_dup 3)
-       (zero_extend:DI (subreg:QI
-                     (rotate:DI (match_dup 1)
-                                (match_dup 2)) 7)))
+       (compare:CC
+        (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "")
+                           (match_operand:SI 2 "const_int_operand" ""))
+                (match_operand:DI 3 "const_int_operand" ""))
+        (const_int 0)))
+   (clobber (match_scratch:DI 4 ""))]
+  "TARGET_POWERPC64 && reload_completed
+   && includes_rldic_lshift_p (operands[2], operands[3])"
+  [(set (match_dup 4)
+       (and:DI (ashift:DI (match_dup 1) (match_dup 2))
+               (match_dup 3)))
    (set (match_dup 0)
-       (compare:CC (match_dup 3)
+       (compare:CC (match_dup 4)
                    (const_int 0)))]
   "")
 
-(define_insn "*rotldi3_internal9le"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
-       (compare:CC (zero_extend:DI
-                    (subreg:QI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
-                                (match_operand:DI 2 "reg_or_cint_operand" "rn,rn")) 0))
-                   (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
-       (zero_extend:DI (subreg:QI (rotate:DI (match_dup 1) (match_dup 2)) 0)))]
-  "TARGET_64BIT && !BYTES_BIG_ENDIAN"
-  "@
-   rld%I2cl. %0,%1,%H2,56
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "maybe_var_shift" "yes")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_insn "*rotldi3_internal9be"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
-       (compare:CC (zero_extend:DI
-                    (subreg:QI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
-                                (match_operand:DI 2 "reg_or_cint_operand" "rn,rn")) 7))
-                   (const_int 0)))
+(define_insn "*ashldi3_internal6"
+  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
+       (compare:CC
+        (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
+                           (match_operand:SI 2 "const_int_operand" "i,i"))
+                   (match_operand:DI 3 "const_int_operand" "n,n"))
+        (const_int 0)))
    (set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
-       (zero_extend:DI (subreg:QI (rotate:DI (match_dup 1) (match_dup 2)) 7)))]
-  "TARGET_64BIT && BYTES_BIG_ENDIAN"
+       (and:DI (ashift:DI (match_dup 1) (match_dup 2)) (match_dup 3)))]
+  "TARGET_64BIT && includes_rldic_lshift_p (operands[2], operands[3])"
   "@
-   rld%I2cl. %0,%1,%H2,56
+   rldic. %0,%1,%H2,%W3
    #"
   [(set_attr "type" "shift")
-   (set_attr "maybe_var_shift" "yes")
    (set_attr "dot" "yes")
    (set_attr "length" "4,8")])
 
 (define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:DI
-                    (subreg:QI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                                (match_operand:DI 2 "reg_or_cint_operand" "")) 0))
-                   (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "")
-       (zero_extend:DI (subreg:QI (rotate:DI (match_dup 1) (match_dup 2)) 0)))]
-  "TARGET_POWERPC64 && !BYTES_BIG_ENDIAN && reload_completed"
-  [(set (match_dup 0)
-       (zero_extend:DI (subreg:QI (rotate:DI (match_dup 1) (match_dup 2)) 0)))
-   (set (match_dup 3)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
-
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:DI
-                    (subreg:QI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                                (match_operand:DI 2 "reg_or_cint_operand" "")) 7))
-                   (const_int 0)))
+  [(set (match_operand:CC 4 "cc_reg_not_micro_cr0_operand" "")
+       (compare:CC
+        (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "")
+                           (match_operand:SI 2 "const_int_operand" ""))
+                (match_operand:DI 3 "const_int_operand" ""))
+        (const_int 0)))
    (set (match_operand:DI 0 "gpc_reg_operand" "")
-       (zero_extend:DI (subreg:QI (rotate:DI (match_dup 1) (match_dup 2)) 7)))]
-  "TARGET_POWERPC64 && BYTES_BIG_ENDIAN && reload_completed"
+       (and:DI (ashift:DI (match_dup 1) (match_dup 2)) (match_dup 3)))]
+  "TARGET_POWERPC64 && reload_completed
+   && includes_rldic_lshift_p (operands[2], operands[3])"
   [(set (match_dup 0)
-       (zero_extend:DI (subreg:QI (rotate:DI (match_dup 1) (match_dup 2)) 7)))
-   (set (match_dup 3)
+       (and:DI (ashift:DI (match_dup 1) (match_dup 2))
+               (match_dup 3)))
+   (set (match_dup 4)
        (compare:CC (match_dup 0)
                    (const_int 0)))]
   "")
 
-(define_insn "*rotldi3_internal10le"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
-       (zero_extend:DI
-        (subreg:HI
-         (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r")
-                    (match_operand:DI 2 "reg_or_cint_operand" "rn")) 0)))]
-  "TARGET_POWERPC64 && !BYTES_BIG_ENDIAN"
-  "rld%I2cl %0,%1,%H2,48"
-  [(set_attr "type" "shift")
-   (set_attr "maybe_var_shift" "yes")])
-
-(define_insn "*rotldi3_internal10be"
+(define_insn "*ashldi3_internal7"
   [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
-       (zero_extend:DI
-        (subreg:HI
-         (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r")
-                    (match_operand:DI 2 "reg_or_cint_operand" "rn")) 6)))]
-  "TARGET_POWERPC64 && BYTES_BIG_ENDIAN"
-  "rld%I2cl %0,%1,%H2,48"
-  [(set_attr "type" "shift")
-   (set_attr "maybe_var_shift" "yes")])
-
-(define_insn "*rotldi3_internal11le"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC (zero_extend:DI
-                    (subreg:HI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
-                                (match_operand:DI 2 "reg_or_cint_operand" "rn,rn")) 0))
-                   (const_int 0)))
-   (clobber (match_scratch:DI 3 "=r,r"))]
-  "TARGET_64BIT && !BYTES_BIG_ENDIAN"
-  "@
-   rld%I2cl. %3,%1,%H2,48
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "maybe_var_shift" "yes")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
+       (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "r")
+                          (match_operand:SI 2 "const_int_operand" "i"))
+               (match_operand:DI 3 "mask64_operand" "n")))]
+  "TARGET_POWERPC64 && includes_rldicr_lshift_p (operands[2], operands[3])"
+  "rldicr %0,%1,%H2,%S3"
+  [(set_attr "type" "shift")])
 
-(define_insn "*rotldi3_internal11be"
+(define_insn "ashldi3_internal8"
   [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC (zero_extend:DI
-                    (subreg:HI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
-                                (match_operand:DI 2 "reg_or_cint_operand" "rn,rn")) 6))
-                   (const_int 0)))
-   (clobber (match_scratch:DI 3 "=r,r"))]
-  "TARGET_64BIT && BYTES_BIG_ENDIAN"
+       (compare:CC
+        (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
+                           (match_operand:SI 2 "const_int_operand" "i,i"))
+                (match_operand:DI 3 "mask64_operand" "n,n"))
+        (const_int 0)))
+   (clobber (match_scratch:DI 4 "=r,r"))]
+  "TARGET_64BIT && includes_rldicr_lshift_p (operands[2], operands[3])"
   "@
-   rld%I2cl. %3,%1,%H2,48
+   rldicr. %4,%1,%H2,%S3
    #"
   [(set_attr "type" "shift")
-   (set_attr "maybe_var_shift" "yes")
    (set_attr "dot" "yes")
    (set_attr "length" "4,8")])
 
 (define_split
   [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:DI
-                    (subreg:HI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                                (match_operand:DI 2 "reg_or_cint_operand" "")) 0))
-                   (const_int 0)))
-   (clobber (match_scratch:DI 3 ""))]
-  "TARGET_POWERPC64 && !BYTES_BIG_ENDIAN && reload_completed"
-  [(set (match_dup 3)
-       (zero_extend:DI (subreg:HI
-                     (rotate:DI (match_dup 1)
-                                (match_dup 2)) 0)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:DI
-                    (subreg:HI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                                (match_operand:DI 2 "reg_or_cint_operand" "")) 6))
-                   (const_int 0)))
-   (clobber (match_scratch:DI 3 ""))]
-  "TARGET_POWERPC64 && BYTES_BIG_ENDIAN && reload_completed"
-  [(set (match_dup 3)
-       (zero_extend:DI (subreg:HI
-                     (rotate:DI (match_dup 1)
-                                (match_dup 2)) 6)))
+       (compare:CC
+        (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "")
+                           (match_operand:SI 2 "const_int_operand" ""))
+                (match_operand:DI 3 "mask64_operand" ""))
+        (const_int 0)))
+   (clobber (match_scratch:DI 4 ""))]
+  "TARGET_POWERPC64 && reload_completed
+   && includes_rldicr_lshift_p (operands[2], operands[3])"
+  [(set (match_dup 4)
+       (and:DI (ashift:DI (match_dup 1) (match_dup 2))
+               (match_dup 3)))
    (set (match_dup 0)
-       (compare:CC (match_dup 3)
+       (compare:CC (match_dup 4)
                    (const_int 0)))]
   "")
 
-(define_insn "*rotldi3_internal12le"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
-       (compare:CC (zero_extend:DI
-                    (subreg:HI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
-                                (match_operand:DI 2 "reg_or_cint_operand" "rn,rn")) 0))
-                   (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
-       (zero_extend:DI (subreg:HI (rotate:DI (match_dup 1) (match_dup 2)) 0)))]
-  "TARGET_64BIT && !BYTES_BIG_ENDIAN"
-  "@
-   rld%I2cl. %0,%1,%H2,48
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "maybe_var_shift" "yes")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_insn "*rotldi3_internal12be"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
-       (compare:CC (zero_extend:DI
-                    (subreg:HI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
-                                (match_operand:DI 2 "reg_or_cint_operand" "rn,rn")) 6))
-                   (const_int 0)))
+(define_insn "*ashldi3_internal9"
+  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
+       (compare:CC
+        (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
+                           (match_operand:SI 2 "const_int_operand" "i,i"))
+                   (match_operand:DI 3 "mask64_operand" "n,n"))
+        (const_int 0)))
    (set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
-       (zero_extend:DI (subreg:HI (rotate:DI (match_dup 1) (match_dup 2)) 6)))]
-  "TARGET_64BIT && BYTES_BIG_ENDIAN"
+       (and:DI (ashift:DI (match_dup 1) (match_dup 2)) (match_dup 3)))]
+  "TARGET_64BIT && includes_rldicr_lshift_p (operands[2], operands[3])"
   "@
-   rld%I2cl. %0,%1,%H2,48
+   rldicr. %0,%1,%H2,%S3
    #"
   [(set_attr "type" "shift")
-   (set_attr "maybe_var_shift" "yes")
    (set_attr "dot" "yes")
    (set_attr "length" "4,8")])
 
 (define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:DI
-                    (subreg:HI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                                (match_operand:DI 2 "reg_or_cint_operand" "")) 0))
-                   (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "")
-       (zero_extend:DI (subreg:HI (rotate:DI (match_dup 1) (match_dup 2)) 0)))]
-  "TARGET_POWERPC64 && !BYTES_BIG_ENDIAN && reload_completed"
-  [(set (match_dup 0)
-       (zero_extend:DI (subreg:HI (rotate:DI (match_dup 1) (match_dup 2)) 0)))
-   (set (match_dup 3)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
-
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:DI
-                    (subreg:HI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                                (match_operand:DI 2 "reg_or_cint_operand" "")) 6))
-                   (const_int 0)))
+  [(set (match_operand:CC 4 "cc_reg_not_micro_cr0_operand" "")
+       (compare:CC
+        (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "")
+                           (match_operand:SI 2 "const_int_operand" ""))
+                (match_operand:DI 3 "mask64_operand" ""))
+        (const_int 0)))
    (set (match_operand:DI 0 "gpc_reg_operand" "")
-       (zero_extend:DI (subreg:HI (rotate:DI (match_dup 1) (match_dup 2)) 6)))]
-  "TARGET_POWERPC64 && BYTES_BIG_ENDIAN && reload_completed"
+       (and:DI (ashift:DI (match_dup 1) (match_dup 2)) (match_dup 3)))]
+  "TARGET_POWERPC64 && reload_completed
+   && includes_rldicr_lshift_p (operands[2], operands[3])"
   [(set (match_dup 0)
-       (zero_extend:DI (subreg:HI (rotate:DI (match_dup 1) (match_dup 2)) 6)))
-   (set (match_dup 3)
+       (and:DI (ashift:DI (match_dup 1) (match_dup 2))
+               (match_dup 3)))
+   (set (match_dup 4)
        (compare:CC (match_dup 0)
                    (const_int 0)))]
   "")
 
-(define_insn "*rotldi3_internal13le"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
-       (zero_extend:DI
-        (subreg:SI
-         (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r")
-                    (match_operand:DI 2 "reg_or_cint_operand" "rn")) 0)))]
-  "TARGET_POWERPC64 && !BYTES_BIG_ENDIAN"
-  "rld%I2cl %0,%1,%H2,32"
-  [(set_attr "type" "shift")
-   (set_attr "maybe_var_shift" "yes")])
 
-(define_insn "*rotldi3_internal13be"
+(define_insn_and_split "*anddi3_2rld"
   [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
-       (zero_extend:DI
-        (subreg:SI
-         (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r")
-                    (match_operand:DI 2 "reg_or_cint_operand" "rn")) 4)))]
-  "TARGET_POWERPC64 && BYTES_BIG_ENDIAN"
-  "rld%I2cl %0,%1,%H2,32"
-  [(set_attr "type" "shift")
-   (set_attr "maybe_var_shift" "yes")])
-
-(define_insn "*rotldi3_internal14le"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC (zero_extend:DI
-                    (subreg:SI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
-                                (match_operand:DI 2 "reg_or_cint_operand" "rn,rn")) 0))
-                   (const_int 0)))
-   (clobber (match_scratch:DI 3 "=r,r"))]
-  "TARGET_64BIT && !BYTES_BIG_ENDIAN"
-  "@
-   rld%I2cl. %3,%1,%H2,32
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "maybe_var_shift" "yes")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_insn "*rotldi3_internal14be"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC (zero_extend:DI
-                    (subreg:SI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
-                                (match_operand:DI 2 "reg_or_cint_operand" "rn,rn")) 4))
-                   (const_int 0)))
-   (clobber (match_scratch:DI 3 "=r,r"))]
-  "TARGET_64BIT && BYTES_BIG_ENDIAN"
-  "@
-   rld%I2cl. %3,%1,%H2,32
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "maybe_var_shift" "yes")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:DI
-                    (subreg:SI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                                (match_operand:DI 2 "reg_or_cint_operand" "")) 0))
-                   (const_int 0)))
-   (clobber (match_scratch:DI 3 ""))]
-  "TARGET_POWERPC64 && !BYTES_BIG_ENDIAN && reload_completed"
-  [(set (match_dup 3)
-       (zero_extend:DI (subreg:SI
-                     (rotate:DI (match_dup 1)
-                                (match_dup 2)) 0)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:DI
-                    (subreg:SI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                                (match_operand:DI 2 "reg_or_cint_operand" "")) 4))
-                   (const_int 0)))
-   (clobber (match_scratch:DI 3 ""))]
-  "TARGET_POWERPC64 && BYTES_BIG_ENDIAN && reload_completed"
-  [(set (match_dup 3)
-       (zero_extend:DI (subreg:SI
-                     (rotate:DI (match_dup 1)
-                                (match_dup 2)) 4)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
-
-(define_insn "*rotldi3_internal15le"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
-       (compare:CC (zero_extend:DI
-                    (subreg:SI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
-                                (match_operand:DI 2 "reg_or_cint_operand" "rn,rn")) 0))
-                   (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
-       (zero_extend:DI (subreg:SI (rotate:DI (match_dup 1) (match_dup 2)) 0)))]
-  "TARGET_64BIT && !BYTES_BIG_ENDIAN"
-  "@
-   rld%I2cl. %0,%1,%H2,32
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "maybe_var_shift" "yes")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_insn "*rotldi3_internal15be"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
-       (compare:CC (zero_extend:DI
-                    (subreg:SI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
-                                (match_operand:DI 2 "reg_or_cint_operand" "rn,rn")) 4))
-                   (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
-       (zero_extend:DI (subreg:SI (rotate:DI (match_dup 1) (match_dup 2)) 4)))]
-  "TARGET_64BIT && BYTES_BIG_ENDIAN"
-  "@
-   rld%I2cl. %0,%1,%H2,32
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "maybe_var_shift" "yes")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:DI
-                    (subreg:SI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                                (match_operand:DI 2 "reg_or_cint_operand" "")) 0))
-                   (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "")
-       (zero_extend:DI (subreg:SI (rotate:DI (match_dup 1) (match_dup 2)) 0)))]
-  "TARGET_POWERPC64 && !BYTES_BIG_ENDIAN && reload_completed"
-  [(set (match_dup 0)
-       (zero_extend:DI (subreg:SI (rotate:DI (match_dup 1) (match_dup 2)) 0)))
-   (set (match_dup 3)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
-
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:DI
-                    (subreg:SI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                                (match_operand:DI 2 "reg_or_cint_operand" "")) 4))
-                   (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "")
-       (zero_extend:DI (subreg:SI (rotate:DI (match_dup 1) (match_dup 2)) 4)))]
-  "TARGET_POWERPC64 && BYTES_BIG_ENDIAN && reload_completed"
-  [(set (match_dup 0)
-       (zero_extend:DI (subreg:SI (rotate:DI (match_dup 1) (match_dup 2)) 4)))
-   (set (match_dup 3)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
-
-(define_insn "*ashldi3_internal4"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
-       (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "r")
-                          (match_operand:SI 2 "const_int_operand" "i"))
-               (match_operand:DI 3 "const_int_operand" "n")))]
-  "TARGET_POWERPC64 && includes_rldic_lshift_p (operands[2], operands[3])"
-  "rldic %0,%1,%H2,%W3"
-  [(set_attr "type" "shift")])
-
-(define_insn "ashldi3_internal5"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
-                           (match_operand:SI 2 "const_int_operand" "i,i"))
-                (match_operand:DI 3 "const_int_operand" "n,n"))
-        (const_int 0)))
-   (clobber (match_scratch:DI 4 "=r,r"))]
-  "TARGET_64BIT && includes_rldic_lshift_p (operands[2], operands[3])"
-  "@
-   rldic. %4,%1,%H2,%W3
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC
-        (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                           (match_operand:SI 2 "const_int_operand" ""))
-                (match_operand:DI 3 "const_int_operand" ""))
-        (const_int 0)))
-   (clobber (match_scratch:DI 4 ""))]
-  "TARGET_POWERPC64 && reload_completed
-   && includes_rldic_lshift_p (operands[2], operands[3])"
-  [(set (match_dup 4)
-       (and:DI (ashift:DI (match_dup 1) (match_dup 2))
-               (match_dup 3)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 4)
-                   (const_int 0)))]
-  "")
-
-(define_insn "*ashldi3_internal6"
-  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
-                           (match_operand:SI 2 "const_int_operand" "i,i"))
-                   (match_operand:DI 3 "const_int_operand" "n,n"))
-        (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
-       (and:DI (ashift:DI (match_dup 1) (match_dup 2)) (match_dup 3)))]
-  "TARGET_64BIT && includes_rldic_lshift_p (operands[2], operands[3])"
-  "@
-   rldic. %0,%1,%H2,%W3
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 4 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC
-        (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                           (match_operand:SI 2 "const_int_operand" ""))
-                (match_operand:DI 3 "const_int_operand" ""))
-        (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "")
-       (and:DI (ashift:DI (match_dup 1) (match_dup 2)) (match_dup 3)))]
-  "TARGET_POWERPC64 && reload_completed
-   && includes_rldic_lshift_p (operands[2], operands[3])"
-  [(set (match_dup 0)
-       (and:DI (ashift:DI (match_dup 1) (match_dup 2))
-               (match_dup 3)))
-   (set (match_dup 4)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
-
-(define_insn "*ashldi3_internal7"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
-       (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "r")
-                          (match_operand:SI 2 "const_int_operand" "i"))
-               (match_operand:DI 3 "mask64_operand" "n")))]
-  "TARGET_POWERPC64 && includes_rldicr_lshift_p (operands[2], operands[3])"
-  "rldicr %0,%1,%H2,%S3"
-  [(set_attr "type" "shift")])
-
-(define_insn "ashldi3_internal8"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
-                           (match_operand:SI 2 "const_int_operand" "i,i"))
-                (match_operand:DI 3 "mask64_operand" "n,n"))
-        (const_int 0)))
-   (clobber (match_scratch:DI 4 "=r,r"))]
-  "TARGET_64BIT && includes_rldicr_lshift_p (operands[2], operands[3])"
-  "@
-   rldicr. %4,%1,%H2,%S3
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC
-        (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                           (match_operand:SI 2 "const_int_operand" ""))
-                (match_operand:DI 3 "mask64_operand" ""))
-        (const_int 0)))
-   (clobber (match_scratch:DI 4 ""))]
-  "TARGET_POWERPC64 && reload_completed
-   && includes_rldicr_lshift_p (operands[2], operands[3])"
-  [(set (match_dup 4)
-       (and:DI (ashift:DI (match_dup 1) (match_dup 2))
-               (match_dup 3)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 4)
-                   (const_int 0)))]
-  "")
-
-(define_insn "*ashldi3_internal9"
-  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
-                           (match_operand:SI 2 "const_int_operand" "i,i"))
-                   (match_operand:DI 3 "mask64_operand" "n,n"))
-        (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
-       (and:DI (ashift:DI (match_dup 1) (match_dup 2)) (match_dup 3)))]
-  "TARGET_64BIT && includes_rldicr_lshift_p (operands[2], operands[3])"
-  "@
-   rldicr. %0,%1,%H2,%S3
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 4 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC
-        (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                           (match_operand:SI 2 "const_int_operand" ""))
-                (match_operand:DI 3 "mask64_operand" ""))
-        (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "")
-       (and:DI (ashift:DI (match_dup 1) (match_dup 2)) (match_dup 3)))]
-  "TARGET_POWERPC64 && reload_completed
-   && includes_rldicr_lshift_p (operands[2], operands[3])"
-  [(set (match_dup 0)
-       (and:DI (ashift:DI (match_dup 1) (match_dup 2))
-               (match_dup 3)))
-   (set (match_dup 4)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
-
-
-(define_insn_and_split "*anddi3_2rld"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
-       (and:DI (match_operand:DI 1 "gpc_reg_operand" "%r")
-               (match_operand:DI 2 "and_2rld_operand" "n")))]
-  "TARGET_POWERPC64"
-  "#"
-  ""
-  [(set (match_dup 0)
-       (and:DI (rotate:DI (match_dup 1)
-                          (match_dup 4))
-               (match_dup 5)))
-   (set (match_dup 0)
-       (and:DI (rotate:DI (match_dup 0)
-                          (match_dup 6))
-               (match_dup 7)))]
-{
-  build_mask64_2_operands (operands[2], &operands[4]);
-}
-  [(set_attr "length" "8")])
+       (and:DI (match_operand:DI 1 "gpc_reg_operand" "%r")
+               (match_operand:DI 2 "and_2rld_operand" "n")))]
+  "TARGET_POWERPC64"
+  "#"
+  ""
+  [(set (match_dup 0)
+       (and:DI (rotate:DI (match_dup 1)
+                          (match_dup 4))
+               (match_dup 5)))
+   (set (match_dup 0)
+       (and:DI (rotate:DI (match_dup 0)
+                          (match_dup 6))
+               (match_dup 7)))]
+{
+  build_mask64_2_operands (operands[2], &operands[4]);
+}
+  [(set_attr "length" "8")])
 
 (define_insn_and_split "*anddi3_2rld_dot"
   [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
 {
   build_mask64_2_operands (operands[2], &operands[4]);
 }
-  [(set_attr "type" "compare")
+  [(set_attr "type" "two")
    (set_attr "dot" "yes")
    (set_attr "length" "8,12")])
 
 {
   build_mask64_2_operands (operands[2], &operands[4]);
 }
-  [(set_attr "type" "compare")
+  [(set_attr "type" "two")
    (set_attr "dot" "yes")
    (set_attr "length" "8,12")])
 \f
   [(set (match_operand:BOOL_128 0 "vlogical_operand" "=<BOOL_REGS_OUTPUT>")
        (match_operator:BOOL_128 3 "boolean_operator"
         [(not:BOOL_128
-          (match_operand:BOOL_128 2 "vlogical_operand" "<BOOL_REGS_OP1>"))
-         (match_operand:BOOL_128 1 "vlogical_operand" "<BOOL_REGS_OP2>")]))]
+          (match_operand:BOOL_128 2 "vlogical_operand" "<BOOL_REGS_OP2>"))
+         (match_operand:BOOL_128 1 "vlogical_operand" "<BOOL_REGS_OP1>")]))]
   "TARGET_P8_VECTOR || (GET_CODE (operands[3]) == AND)"
 {
   if (TARGET_VSX && vsx_register_operand (operands[0], <MODE>mode))
    && reload_completed && int_reg_operand (operands[0], <MODE>mode)"
   [(const_int 0)]
 {
-  rs6000_split_logical (operands, GET_CODE (operands[3]), false, true, false);
+  rs6000_split_logical (operands, GET_CODE (operands[3]), false, false, true);
   DONE;
 }
   [(set (attr "type")
   [(set (match_operand:TI2 0 "int_reg_operand" "=&r,r,r")
        (match_operator:TI2 3 "boolean_operator"
         [(not:TI2
-          (match_operand:TI2 1 "int_reg_operand" "r,0,r"))
-         (match_operand:TI2 2 "int_reg_operand" "r,r,0")]))]
+          (match_operand:TI2 2 "int_reg_operand" "r,0,r"))
+         (match_operand:TI2 1 "int_reg_operand" "r,r,0")]))]
   "!TARGET_P8_VECTOR && (GET_CODE (operands[3]) != AND)"
   "#"
   "reload_completed && !TARGET_P8_VECTOR && (GET_CODE (operands[3]) != AND)"
   [(const_int 0)]
 {
-  rs6000_split_logical (operands, GET_CODE (operands[3]), false, true, false);
+  rs6000_split_logical (operands, GET_CODE (operands[3]), false, false, true);
   DONE;
 }
   [(set_attr "type" "integer")
 }")
 
 (define_insn "mov<mode>_hardfloat"
-  [(set (match_operand:FMOVE32 0 "nonimmediate_operand" "=!r,!r,m,f,wa,wa,<f32_lr>,<f32_sm>,wu,Z,?<f32_dm>,?r,*c*l,!r,*h,!r,!r")
-       (match_operand:FMOVE32 1 "input_operand" "r,m,r,f,wa,j,<f32_lm>,<f32_sr>,Z,wu,r,<f32_dm>,r,h,0,G,Fn"))]
+  [(set (match_operand:FMOVE32 0 "nonimmediate_operand" "=!r,!r,m,f,<f32_vsx>,<f32_vsx>,!r,<f32_lr>,<f32_sm>,<f32_av>,Z,?<f32_dm>,?r,*c*l,!r,*h")
+       (match_operand:FMOVE32 1 "input_operand" "r,m,r,f,<f32_vsx>,j,j,<f32_lm>,<f32_sr>,Z,<f32_av>,r,<f32_dm>,r,h,0"))]
   "(gpc_reg_operand (operands[0], <MODE>mode)
    || gpc_reg_operand (operands[1], <MODE>mode))
    && (TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_SINGLE_FLOAT)"
    lwz%U1%X1 %0,%1
    stw%U0%X0 %1,%0
    fmr %0,%1
-   xxlor %x0,%x1,%x1
+   xscpsgndp %x0,%x1,%x1
    xxlxor %x0,%x0,%x0
+   li %0,0
    <f32_li>
    <f32_si>
    <f32_lv>
    mfvsrwz %0,%x1
    mt%0 %1
    mf%1 %0
-   nop
-   #
-   #"
-  [(set_attr "type" "*,load,store,fp,vecsimple,vecsimple,fpload,fpstore,fpload,fpstore,mftgpr,mffgpr,mtjmpr,mfjmpr,*,*,*")
-   (set_attr "length" "4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,8")])
+   nop"
+  [(set_attr "type" "*,load,store,fp,fp,vecsimple,integer,fpload,fpstore,fpload,fpstore,mftgpr,mffgpr,mtjmpr,mfjmpr,*")
+   (set_attr "length" "4")])
 
 (define_insn "*mov<mode>_softfloat"
   [(set (match_operand:FMOVE32 0 "nonimmediate_operand" "=r,cl,r,r,m,r,r,r,r,*h")
 ;; since the D-form version of the memory instructions does not need a GPR for
 ;; reloading.
 
+;; If we have FPR registers, rs6000_emit_move has moved all constants to memory,
+;; except for 0.0 which can be created on VSX with an xor instruction.
+
 (define_insn "*mov<mode>_hardfloat32"
-  [(set (match_operand:FMOVE64 0 "nonimmediate_operand" "=m,d,d,wv,Z,wa,wa,Y,r,!r,!r,!r,!r")
-       (match_operand:FMOVE64 1 "input_operand" "d,m,d,Z,wv,wa,j,r,Y,r,G,H,F"))]
+  [(set (match_operand:FMOVE64 0 "nonimmediate_operand" "=m,d,d,<f64_av>,Z,<f64_vsx>,<f64_vsx>,!r,Y,r,!r")
+       (match_operand:FMOVE64 1 "input_operand" "d,m,d,Z,<f64_av>,<f64_vsx>,j,j,r,Y,r"))]
   "! TARGET_POWERPC64 && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT 
    && (gpc_reg_operand (operands[0], <MODE>mode)
        || gpc_reg_operand (operands[1], <MODE>mode))"
    #
    #
    #
-   #
-   #
    #"
-  [(set_attr "type" "fpstore,fpload,fp,fpload,fpstore,vecsimple,vecsimple,store,load,two,fp,fp,*")
-   (set_attr "length" "4,4,4,4,4,4,4,8,8,8,8,12,16")])
+  [(set_attr "type" "fpstore,fpload,fp,fpload,fpstore,vecsimple,vecsimple,two,store,load,two")
+   (set_attr "length" "4,4,4,4,4,4,4,8,8,8,8")])
 
 (define_insn "*mov<mode>_softfloat32"
   [(set (match_operand:FMOVE64 0 "nonimmediate_operand" "=Y,r,r,r,r,r")
 ; ld/std require word-aligned displacements -> 'Y' constraint.
 ; List Y->r and r->Y before r->r for reload.
 (define_insn "*mov<mode>_hardfloat64"
-  [(set (match_operand:FMOVE64 0 "nonimmediate_operand" "=m,d,d,wv,Z,wa,wa,Y,r,!r,*c*l,!r,*h,!r,!r,!r,r,wg,r,wk")
-       (match_operand:FMOVE64 1 "input_operand" "d,m,d,Z,wv,wa,j,r,Y,r,r,h,0,G,H,F,wg,r,wk,r"))]
+  [(set (match_operand:FMOVE64 0 "nonimmediate_operand" "=m,d,d,<f64_av>,Z,<f64_vsx>,<f64_vsx>,!r,Y,r,!r,*c*l,!r,*h,r,wg,r,<f64_dm>")
+       (match_operand:FMOVE64 1 "input_operand" "d,m,d,Z,<f64_av>,<f64_vsx>,j,j,r,Y,r,r,h,0,wg,r,<f64_dm>,r"))]
   "TARGET_POWERPC64 && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT
    && (gpc_reg_operand (operands[0], <MODE>mode)
        || gpc_reg_operand (operands[1], <MODE>mode))"
    stxsd%U0x %x1,%y0
    xxlor %x0,%x1,%x1
    xxlxor %x0,%x0,%x0
+   li %0,0
    std%U0%X0 %1,%0
    ld%U1%X1 %0,%1
    mr %0,%1
    mt%0 %1
    mf%1 %0
    nop
-   #
-   #
-   #
    mftgpr %0,%1
    mffgpr %0,%1
    mfvsrd %0,%x1
    mtvsrd %x0,%1"
-  [(set_attr "type" "fpstore,fpload,fp,fpload,fpstore,vecsimple,vecsimple,store,load,*,mtjmpr,mfjmpr,*,*,*,*,mftgpr,mffgpr,mftgpr,mffgpr")
-   (set_attr "length" "4,4,4,4,4,4,4,4,4,4,4,4,4,8,12,16,4,4,4,4")])
+  [(set_attr "type" "fpstore,fpload,fp,fpload,fpstore,vecsimple,vecsimple,integer,store,load,*,mtjmpr,mfjmpr,*,mftgpr,mffgpr,mftgpr,mffgpr")
+   (set_attr "length" "4")])
 
 (define_insn "*mov<mode>_softfloat64"
   [(set (match_operand:FMOVE64 0 "nonimmediate_operand" "=Y,r,r,cl,r,r,r,r,*h")
 ;; problematical.  Don't allow direct move for this case.
 
 (define_insn_and_split "*mov<mode>_64bit_dm"
-  [(set (match_operand:FMOVE128 0 "nonimmediate_operand" "=m,d,d,Y,r,r,r,wm")
-       (match_operand:FMOVE128 1 "input_operand" "d,m,d,r,YGHF,r,wm,r"))]
+  [(set (match_operand:FMOVE128 0 "nonimmediate_operand" "=m,d,d,ws,Y,r,r,r,wm")
+       (match_operand:FMOVE128 1 "input_operand" "d,m,d,j,r,jY,r,wm,r"))]
   "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_POWERPC64
    && (<MODE>mode != TDmode || WORDS_BIG_ENDIAN)
    && (gpc_reg_operand (operands[0], <MODE>mode)
   "&& reload_completed"
   [(pc)]
 { rs6000_split_multireg_move (operands[0], operands[1]); DONE; }
-  [(set_attr "length" "8,8,8,12,12,8,8,8")])
+  [(set_attr "length" "8,8,8,8,12,12,8,8,8")])
 
 (define_insn_and_split "*movtd_64bit_nodm"
-  [(set (match_operand:TD 0 "nonimmediate_operand" "=m,d,d,Y,r,r")
-       (match_operand:TD 1 "input_operand" "d,m,d,r,YGHF,r"))]
+  [(set (match_operand:TD 0 "nonimmediate_operand" "=m,d,d,ws,Y,r,r")
+       (match_operand:TD 1 "input_operand" "d,m,d,j,r,jY,r"))]
   "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_POWERPC64 && !WORDS_BIG_ENDIAN
    && (gpc_reg_operand (operands[0], TDmode)
        || gpc_reg_operand (operands[1], TDmode))"
   "&& reload_completed"
   [(pc)]
 { rs6000_split_multireg_move (operands[0], operands[1]); DONE; }
-  [(set_attr "length" "8,8,8,12,12,8")])
+  [(set_attr "length" "8,8,8,8,12,12,8")])
 
 (define_insn_and_split "*mov<mode>_32bit"
-  [(set (match_operand:FMOVE128 0 "nonimmediate_operand" "=m,d,d,Y,r,r")
-       (match_operand:FMOVE128 1 "input_operand" "d,m,d,r,YGHF,r"))]
+  [(set (match_operand:FMOVE128 0 "nonimmediate_operand" "=m,d,d,ws,Y,r,r")
+       (match_operand:FMOVE128 1 "input_operand" "d,m,d,j,r,jY,r"))]
   "TARGET_HARD_FLOAT && TARGET_FPRS && !TARGET_POWERPC64
    && (gpc_reg_operand (operands[0], <MODE>mode)
        || gpc_reg_operand (operands[1], <MODE>mode))"
   "&& reload_completed"
   [(pc)]
 { rs6000_split_multireg_move (operands[0], operands[1]); DONE; }
-  [(set_attr "length" "8,8,8,20,20,16")])
+  [(set_attr "length" "8,8,8,8,20,20,16")])
 
 (define_insn_and_split "*mov<mode>_softfloat"
   [(set (match_operand:FMOVE128 0 "rs6000_nonimmediate_operand" "=Y,r,r")
    && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT 
    && TARGET_LONG_DOUBLE_128"
 {
-  operands[2] = CONST0_RTX (DFmode);
-  /* Generate GOT reference early for SVR4 PIC.  */
-  if (DEFAULT_ABI == ABI_V4 && flag_pic)
-    operands[2] = validize_mem (force_const_mem (DFmode, operands[2]));
+  /* VSX can create 0.0 directly, otherwise let rs6000_emit_move create
+     the proper constant.  */
+  if (TARGET_VSX)
+    operands[2] = CONST0_RTX (DFmode);
+  else
+    {
+      operands[2] = gen_reg_rtx (DFmode);
+      rs6000_emit_move (operands[2], CONST0_RTX (DFmode), DFmode);
+    }
 })
 
 (define_insn_and_split "*extenddftf2_internal"
-  [(set (match_operand:TF 0 "nonimmediate_operand" "=m,Y,d,&d,r")
-       (float_extend:TF (match_operand:DF 1 "input_operand" "d,r,md,md,rmGHF")))
-   (use (match_operand:DF 2 "zero_reg_mem_operand" "d,r,m,d,n"))]
+  [(set (match_operand:TF 0 "nonimmediate_operand" "=m,Y,ws,d,&d")
+       (float_extend:TF (match_operand:DF 1 "input_operand" "d,r,md,md,md")))
+   (use (match_operand:DF 2 "zero_reg_mem_operand" "d,r,j,m,d"))]
   "!TARGET_IEEEQUAD
    && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT 
    && TARGET_LONG_DOUBLE_128"
 })
 
 \f
+;; Reload patterns for various types using the vector registers.  We may need
+;; an additional base register to convert the reg+offset addressing to reg+reg
+;; for vector registers and reg+reg or (reg+reg)&(-16) addressing to just an
+;; index register for gpr registers.
+(define_expand "reload_<RELOAD:mode>_<P:mptrsize>_store"
+  [(parallel [(match_operand:RELOAD 0 "memory_operand" "m")
+              (match_operand:RELOAD 1 "gpc_reg_operand" "wa")
+              (match_operand:P 2 "register_operand" "=b")])]
+  "<P:tptrsize>"
+{
+  rs6000_secondary_reload_inner (operands[1], operands[0], operands[2], true);
+  DONE;
+})
+
+(define_expand "reload_<RELOAD:mode>_<P:mptrsize>_load"
+  [(parallel [(match_operand:RELOAD 0 "gpc_reg_operand" "wa")
+              (match_operand:RELOAD 1 "memory_operand" "m")
+              (match_operand:P 2 "register_operand" "=b")])]
+  "<P:tptrsize>"
+{
+  rs6000_secondary_reload_inner (operands[0], operands[1], operands[2], false);
+  DONE;
+})
+
+
+;; Reload sometimes tries to move the address to a GPR, and can generate
+;; invalid RTL for addresses involving AND -16.  Allow addresses involving
+;; reg+reg, reg+small constant, or just reg, all wrapped in an AND -16.
+
+(define_insn_and_split "*vec_reload_and_plus_<mptrsize>"
+  [(set (match_operand:P 0 "gpc_reg_operand" "=b")
+       (and:P (plus:P (match_operand:P 1 "gpc_reg_operand" "r")
+                      (match_operand:P 2 "reg_or_cint_operand" "rI"))
+              (const_int -16)))]
+  "TARGET_ALTIVEC && (reload_in_progress || reload_completed)"
+  "#"
+  "&& reload_completed"
+  [(set (match_dup 0)
+       (plus:P (match_dup 1)
+               (match_dup 2)))
+   (set (match_dup 0)
+       (and:P (match_dup 0)
+              (const_int -16)))])
+\f
 ;; Power8 merge instructions to allow direct move to/from floating point
 ;; registers in 32-bit mode.  We use TF mode to get two registers to move the
 ;; individual 32-bit parts across.  Subreg doesn't work too well on the TF
   emit_insn (gen_p8_mtvsrd_1 (tmp, gpr_hi_reg));
   emit_insn (gen_p8_mtvsrd_2 (tmp, gpr_lo_reg));
   emit_insn (gen_p8_xxpermdi_<mode> (dest, tmp));
+  DONE;
 }
   [(set_attr "length" "12")
    (set_attr "type" "three")])
   emit_insn (gen_p8_mfvsrd_3_<mode> (gpr_hi_reg, src));
   emit_insn (gen_vsx_xxpermdi_<mode> (tmp, src, src, GEN_INT (3)));
   emit_insn (gen_p8_mfvsrd_3_<mode> (gpr_lo_reg, tmp));
+  DONE;
 }
   [(set_attr "length" "12")
    (set_attr "type" "three")])
 
   for (i = 0; i < count; i++)
     XVECEXP (operands[3], 0, i)
-      = gen_rtx_SET (VOIDmode, gen_rtx_REG (SImode, regno + i),
+      = gen_rtx_SET (gen_rtx_REG (SImode, regno + i),
                     adjust_address_nv (op1, SImode, i * 4));
 }")
 
   op0 = replace_equiv_address (operands[0], to);
 
   XVECEXP (operands[3], 0, 0)
-    = gen_rtx_SET (VOIDmode, adjust_address_nv (op0, SImode, 0), operands[1]);
+    = gen_rtx_SET (adjust_address_nv (op0, SImode, 0), operands[1]);
   XVECEXP (operands[3], 0, 1) = gen_rtx_CLOBBER (VOIDmode,
                                                 gen_rtx_SCRATCH (SImode));
 
   for (i = 1; i < count; i++)
     XVECEXP (operands[3], 0, i + 1)
-      = gen_rtx_SET (VOIDmode,
-                    adjust_address_nv (op0, SImode, i * 4),
+      = gen_rtx_SET (adjust_address_nv (op0, SImode, i * 4),
                     gen_rtx_REG (SImode, regno + i));
 }")
 
 ;; sequences, using get_attr_length here will smash the operands
 ;; array.  Neither is there an early_cobbler_p predicate.
 ;; Disallow subregs for E500 so we don't munge frob_di_df_2.
+;; Also this optimization interferes with scalars going into
+;; altivec registers (the code does reloading through the FPRs).
 (define_peephole2
   [(set (match_operand:DF 0 "gpc_reg_operand" "")
        (match_operand:DF 1 "any_operand" ""))
    (set (match_operand:DF 2 "gpc_reg_operand" "")
        (match_dup 0))]
   "!(TARGET_E500_DOUBLE && GET_CODE (operands[2]) == SUBREG)
+   && !TARGET_UPPER_REGS_DF
    && peep2_reg_dead_p (2, operands[0])"
   [(set (match_dup 2) (match_dup 1))])
 
        (match_operand:SF 1 "any_operand" ""))
    (set (match_operand:SF 2 "gpc_reg_operand" "")
        (match_dup 0))]
-  "peep2_reg_dead_p (2, operands[0])"
+  "!TARGET_UPPER_REGS_SF
+   && peep2_reg_dead_p (2, operands[0])"
   [(set (match_dup 2) (match_dup 1))])
 
 \f
   operands[3] = gen_frame_mem (Pmode, operands[0]);
   operands[4] = gen_frame_mem (Pmode, operands[1]);
   p = rtvec_alloc (1);
-  RTVEC_ELT (p, 0) = gen_rtx_SET (VOIDmode,
-                                 gen_frame_mem (BLKmode, operands[0]),
+  RTVEC_ELT (p, 0) = gen_rtx_SET (gen_frame_mem (BLKmode, operands[0]),
                                  const0_rtx);
   operands[5] = gen_rtx_PARALLEL (VOIDmode, p);
 }")
   operands[4] = adjust_address_nv (operands[1], Pmode, units_per_word);
   operands[5] = gen_frame_mem (Pmode, operands[3]);
   p = rtvec_alloc (1);
-  RTVEC_ELT (p, 0) = gen_rtx_SET (VOIDmode,
-                                 gen_frame_mem (BLKmode, operands[0]),
+  RTVEC_ELT (p, 0) = gen_rtx_SET (gen_frame_mem (BLKmode, operands[0]),
                                  const0_rtx);
   operands[6] = gen_rtx_PARALLEL (VOIDmode, p);
 }")
    "addis %0,%1+%3@u(%2)")
 
 (define_insn "*largetoc_low"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
-        (lo_sum:DI (match_operand:DI 1 "gpc_reg_operand" "b,!*r")
+  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
+        (lo_sum:DI (match_operand:DI 1 "gpc_reg_operand" "b")
                   (match_operand:DI 2 "" "")))]
    "TARGET_ELF && TARGET_CMODEL != CMODEL_SMALL"
-   "@
-    addi %0,%1,%2@l
-    addic %0,%1,%2@l")
+   "addi %0,%1,%2@l")
 
 (define_insn "*largetoc_low_aix<mode>"
   [(set (match_operand:P 0 "gpc_reg_operand" "=r")
    "la %0,%2@l(%1)")
 
 (define_insn_and_split "*tocref<mode>"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=b*r")
+  [(set (match_operand:P 0 "gpc_reg_operand" "=b")
        (match_operand:P 1 "small_toc_ref" "R"))]
    "TARGET_TOC"
    "la %0,%a1"
   "lis %0,%1@ha")
 
 (define_insn "elf_low"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
-       (lo_sum:SI (match_operand:SI 1 "gpc_reg_operand" "b,!*r")
+  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
+       (lo_sum:SI (match_operand:SI 1 "gpc_reg_operand" "b")
                   (match_operand 2 "" "")))]
    "TARGET_ELF && ! TARGET_64BIT"
-   "@
-    la %0,%2@l(%1)
-    addic %0,%1,%K2")
+   "la %0,%2@l(%1)")
 \f
 ;; Call and call_value insns
 (define_expand "call"
 ;; Call to indirect functions with the AIX abi using a 3 word descriptor.
 ;; Operand0 is the addresss of the function to call
 ;; Operand2 is the location in the function descriptor to load r2 from
-;; Operand3 is the stack location to hold the current TOC pointer
+;; Operand3 is the offset of the stack location holding the current TOC pointer
 
 (define_insn "*call_indirect_aix<mode>"
   [(call (mem:SI (match_operand:P 0 "register_operand" "c,*l"))
         (match_operand 1 "" "g,g"))
    (use (match_operand:P 2 "memory_operand" "<ptrm>,<ptrm>"))
-   (set (reg:P TOC_REGNUM) (match_operand:P 3 "memory_operand" "<ptrm>,<ptrm>"))
+   (set (reg:P TOC_REGNUM) (unspec [(match_operand:P 3 "const_int_operand" "n,n")] UNSPEC_TOCSLOT))
    (clobber (reg:P LR_REGNO))]
   "DEFAULT_ABI == ABI_AIX"
-  "<ptrload> 2,%2\;b%T0l\;<ptrload> 2,%3"
+  "<ptrload> 2,%2\;b%T0l\;<ptrload> 2,%3(1)"
   [(set_attr "type" "jmpreg")
    (set_attr "length" "12")])
 
        (call (mem:SI (match_operand:P 1 "register_operand" "c,*l"))
              (match_operand 2 "" "g,g")))
    (use (match_operand:P 3 "memory_operand" "<ptrm>,<ptrm>"))
-   (set (reg:P TOC_REGNUM) (match_operand:P 4 "memory_operand" "<ptrm>,<ptrm>"))
+   (set (reg:P TOC_REGNUM) (unspec [(match_operand:P 4 "const_int_operand" "n,n")] UNSPEC_TOCSLOT))
    (clobber (reg:P LR_REGNO))]
   "DEFAULT_ABI == ABI_AIX"
-  "<ptrload> 2,%3\;b%T1l\;<ptrload> 2,%4"
+  "<ptrload> 2,%3\;b%T1l\;<ptrload> 2,%4(1)"
   [(set_attr "type" "jmpreg")
    (set_attr "length" "12")])
 
 ;; Call to indirect functions with the ELFv2 ABI.
 ;; Operand0 is the addresss of the function to call
-;; Operand2 is the stack location to hold the current TOC pointer
+;; Operand2 is the offset of the stack location holding the current TOC pointer
 
 (define_insn "*call_indirect_elfv2<mode>"
   [(call (mem:SI (match_operand:P 0 "register_operand" "c,*l"))
         (match_operand 1 "" "g,g"))
-   (set (reg:P TOC_REGNUM) (match_operand:P 2 "memory_operand" "<ptrm>,<ptrm>"))
+   (set (reg:P TOC_REGNUM) (unspec [(match_operand:P 2 "const_int_operand" "n,n")] UNSPEC_TOCSLOT))
    (clobber (reg:P LR_REGNO))]
   "DEFAULT_ABI == ABI_ELFv2"
-  "b%T0l\;<ptrload> 2,%2"
+  "b%T0l\;<ptrload> 2,%2(1)"
   [(set_attr "type" "jmpreg")
    (set_attr "length" "8")])
 
   [(set (match_operand 0 "" "")
        (call (mem:SI (match_operand:P 1 "register_operand" "c,*l"))
              (match_operand 2 "" "g,g")))
-   (set (reg:P TOC_REGNUM) (match_operand:P 3 "memory_operand" "<ptrm>,<ptrm>"))
+   (set (reg:P TOC_REGNUM) (unspec [(match_operand:P 3 "const_int_operand" "n,n")] UNSPEC_TOCSLOT))
    (clobber (reg:P LR_REGNO))]
   "DEFAULT_ABI == ABI_ELFv2"
-  "b%T1l\;<ptrload> 2,%3"
+  "b%T1l\;<ptrload> 2,%3(1)"
   [(set_attr "type" "jmpreg")
    (set_attr "length" "8")])
 
   ""
   "")
 
-(define_expand "probe_stack"
-  [(set (match_operand 0 "memory_operand" "=m")
-        (unspec [(const_int 0)] UNSPEC_PROBE_STACK))]
+(define_expand "probe_stack_address"
+  [(use (match_operand 0 "address_operand"))]
   ""
 {
+  operands[0] = gen_rtx_MEM (Pmode, operands[0]);
+  MEM_VOLATILE_P (operands[0]) = 1;
+
   if (TARGET_64BIT)
     emit_insn (gen_probe_stack_di (operands[0]));
   else
   DONE;
 }")
 
-(define_expand "cstore<mode>4"
-  [(use (match_operator 1 "rs6000_cbranch_operator"
-         [(match_operand:GPR 2 "gpc_reg_operand" "")
-          (match_operand:GPR 3 "reg_or_short_operand" "")]))
-   (clobber (match_operand:SI 0 "register_operand"))]
+(define_expand "cstore<mode>4_unsigned"
+  [(use (match_operator 1 "unsigned_comparison_operator"
+         [(match_operand:P 2 "gpc_reg_operand" "")
+          (match_operand:P 3 "reg_or_short_operand" "")]))
+   (clobber (match_operand:P 0 "register_operand"))]
   ""
-  "
 {
-  /* Take care of the possibility that operands[3] might be negative but
-     this might be a logical operation.  That insn doesn't exist.  */
-  if (GET_CODE (operands[3]) == CONST_INT
-      && INTVAL (operands[3]) < 0)
+  enum rtx_code cond_code = GET_CODE (operands[1]);
+
+  rtx op0 = operands[0];
+  rtx op1 = operands[2];
+  rtx op2 = operands[3];
+
+  if (cond_code == GEU || cond_code == LTU)
     {
-      operands[3] = force_reg (<MODE>mode, operands[3]);
-      operands[1] = gen_rtx_fmt_ee (GET_CODE (operands[1]),
-                                   GET_MODE (operands[1]),
-                                   operands[2], operands[3]);
+      cond_code = swap_condition (cond_code);
+      op1 = operands[3];
+      op2 = operands[2];
     }
 
-  /* For SNE, we would prefer that the xor/abs sequence be used for integers.
-     For SEQ, likewise, except that comparisons with zero should be done
-     with an scc insns.  However, due to the order that combine see the
-     resulting insns, we must, in fact, allow SEQ for integers.  Fail in
-     the cases we don't want to handle or are best handled by portable
-     code.  */
-  if (GET_CODE (operands[1]) == NE)
-    FAIL;
-  if ((GET_CODE (operands[1]) == LT || GET_CODE (operands[1]) == LE
-       || GET_CODE (operands[1]) == GT || GET_CODE (operands[1]) == GE)
-      && operands[3] == const0_rtx)
-    FAIL;
-  rs6000_emit_sCOND (<MODE>mode, operands);
+  if (!gpc_reg_operand (op1, <MODE>mode))
+    op1 = force_reg (<MODE>mode, op1);
+  if (!reg_or_short_operand (op2, <MODE>mode))
+    op2 = force_reg (<MODE>mode, op2);
+
+  rtx tmp = gen_reg_rtx (<MODE>mode);
+  rtx tmp2 = gen_reg_rtx (<MODE>mode);
+
+  emit_insn (gen_subf<mode>3_carry (tmp, op1, op2));
+  emit_insn (gen_subf<mode>3_carry_in_xx (tmp2));
+
+  if (cond_code == LEU)
+    emit_insn (gen_add<mode>3 (op0, tmp2, const1_rtx));
+  else
+    emit_insn (gen_neg<mode>2 (op0, tmp2));
+
   DONE;
-}")
+})
+
+(define_expand "cstore<mode>4_signed_imm"
+  [(use (match_operator 1 "signed_comparison_operator"
+         [(match_operand:GPR 2 "gpc_reg_operand")
+          (match_operand:GPR 3 "immediate_operand")]))
+   (clobber (match_operand:GPR 0 "register_operand"))]
+  ""
+{
+  bool invert = false;
+
+  enum rtx_code cond_code = GET_CODE (operands[1]);
+
+  rtx op0 = operands[0];
+  rtx op1 = operands[2];
+  HOST_WIDE_INT val = INTVAL (operands[3]);
+
+  if (cond_code == GE || cond_code == GT)
+    {
+      cond_code = reverse_condition (cond_code);
+      invert = true;
+    }
+
+  if (cond_code == LE)
+    val++;
+
+  rtx tmp = gen_reg_rtx (<MODE>mode);
+  emit_insn (gen_add<mode>3 (tmp, op1, GEN_INT (-val)));
+  rtx x = gen_reg_rtx (<MODE>mode);
+  if (val < 0)
+    emit_insn (gen_and<mode>3 (x, op1, tmp));
+  else
+    emit_insn (gen_ior<mode>3 (x, op1, tmp));
+
+  if (invert)
+    {
+      rtx tmp = gen_reg_rtx (<MODE>mode);
+      emit_insn (gen_one_cmpl<mode>2 (tmp, x));
+      x = tmp;
+    }
+
+  int sh = GET_MODE_BITSIZE (<MODE>mode) - 1;
+  emit_insn (gen_lshr<mode>3 (op0, x, GEN_INT (sh)));
+
+  DONE;
+})
+
+(define_expand "cstore<mode>4_unsigned_imm"
+  [(use (match_operator 1 "unsigned_comparison_operator"
+         [(match_operand:GPR 2 "gpc_reg_operand")
+          (match_operand:GPR 3 "immediate_operand")]))
+   (clobber (match_operand:GPR 0 "register_operand"))]
+  ""
+{
+  bool invert = false;
+
+  enum rtx_code cond_code = GET_CODE (operands[1]);
+
+  rtx op0 = operands[0];
+  rtx op1 = operands[2];
+  HOST_WIDE_INT val = INTVAL (operands[3]);
+
+  if (cond_code == GEU || cond_code == GTU)
+    {
+      cond_code = reverse_condition (cond_code);
+      invert = true;
+    }
+
+  if (cond_code == LEU)
+    val++;
+
+  rtx tmp = gen_reg_rtx (<MODE>mode);
+  rtx tmp2 = gen_reg_rtx (<MODE>mode);
+  emit_insn (gen_add<mode>3 (tmp, op1, GEN_INT (-val)));
+  emit_insn (gen_one_cmpl<mode>2 (tmp2, op1));
+  rtx x = gen_reg_rtx (<MODE>mode);
+  if (val < 0)
+    emit_insn (gen_ior<mode>3 (x, tmp, tmp2));
+  else
+    emit_insn (gen_and<mode>3 (x, tmp, tmp2));
+
+  if (invert)
+    {
+      rtx tmp = gen_reg_rtx (<MODE>mode);
+      emit_insn (gen_one_cmpl<mode>2 (tmp, x));
+      x = tmp;
+    }
+
+  int sh = GET_MODE_BITSIZE (<MODE>mode) - 1;
+  emit_insn (gen_lshr<mode>3 (op0, x, GEN_INT (sh)));
+
+  DONE;
+})
+
+(define_expand "cstore<mode>4"
+  [(use (match_operator 1 "rs6000_cbranch_operator"
+         [(match_operand:GPR 2 "gpc_reg_operand")
+          (match_operand:GPR 3 "reg_or_short_operand")]))
+   (clobber (match_operand:GPR 0 "register_operand"))]
+  ""
+{
+  /* Use ISEL if the user asked for it.  */
+  if (TARGET_ISEL)
+    rs6000_emit_sISEL (<MODE>mode, operands);
+
+  /* Expanding EQ and NE directly to some machine instructions does not help
+     but does hurt combine.  So don't.  */
+  else if (GET_CODE (operands[1]) == EQ)
+    emit_insn (gen_eq<mode>3 (operands[0], operands[2], operands[3]));
+  else if (<MODE>mode == Pmode
+          && GET_CODE (operands[1]) == NE)
+    emit_insn (gen_ne<mode>3 (operands[0], operands[2], operands[3]));
+  else if (GET_CODE (operands[1]) == NE)
+    {
+      rtx tmp = gen_reg_rtx (<MODE>mode);
+      emit_insn (gen_eq<mode>3 (tmp, operands[2], operands[3]));
+      emit_insn (gen_xor<mode>3 (operands[0], tmp, const1_rtx));
+    }
+
+  /* Expanding the unsigned comparisons however helps a lot: all the neg_ltu
+     etc. combinations magically work out just right.  */
+  else if (<MODE>mode == Pmode
+          && unsigned_comparison_operator (operands[1], VOIDmode))
+    emit_insn (gen_cstore<mode>4_unsigned (operands[0], operands[1],
+                                          operands[2], operands[3]));
+
+  /* For signed comparisons against a constant, we can do some simple
+     bit-twiddling.  */
+  else if (signed_comparison_operator (operands[1], VOIDmode)
+          && CONST_INT_P (operands[3]))
+    emit_insn (gen_cstore<mode>4_signed_imm (operands[0], operands[1],
+                                            operands[2], operands[3]));
+
+  /* And similarly for unsigned comparisons.  */
+  else if (unsigned_comparison_operator (operands[1], VOIDmode)
+          && CONST_INT_P (operands[3]))
+    emit_insn (gen_cstore<mode>4_unsigned_imm (operands[0], operands[1],
+                                              operands[2], operands[3]));
+
+  /* Everything else, use the mfcr brute force.  */
+  else
+    rs6000_emit_sCOND (<MODE>mode, operands);
+
+  DONE;
+})
 
 (define_expand "cstore<mode>4"
   [(use (match_operator 1 "rs6000_cbranch_operator"
           (match_operand:FP 3 "gpc_reg_operand" "")]))
    (clobber (match_operand:SI 0 "register_operand"))]
   ""
-  "
 {
   rs6000_emit_sCOND (<MODE>mode, operands);
   DONE;
-}")
+})
 
 
 (define_expand "stack_protect_set"
                       (match_operand 7 "" "")
                       (match_operand 8 "" "")))]
   "peep2_reg_dead_p (3, operands[0])
-   && peep2_reg_dead_p (4, operands[4])"
+   && peep2_reg_dead_p (4, operands[4])
+   && REGNO (operands[0]) != REGNO (operands[5])"
  [(set (match_dup 0) (xor:SI (match_dup 5) (match_dup 9)))
   (set (match_dup 4) (compare:CC (match_dup 0) (match_dup 10)))
   (set (pc) (if_then_else (match_dup 6) (match_dup 7) (match_dup 8)))]
   else
     count = 32 - (put_bit - is_bit);
 
-  operands[5] = GEN_INT (count);
-  operands[6] = GEN_INT (put_bit);
-
-  return \"mfcr %4%Q2\;rlwinm. %4,%4,%5,%6,%6\";
-}"
-  [(set_attr "type" "shift")
-   (set_attr "dot" "yes")
-   (set_attr "length" "8,16")])
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC
-        (ashift:SI (match_operator:SI 1 "scc_comparison_operator"
-                                      [(match_operand 2 "cc_reg_operand" "")
-                                       (const_int 0)])
-                   (match_operand:SI 3 "const_int_operand" ""))
-        (const_int 0)))
-   (set (match_operand:SI 4 "gpc_reg_operand" "")
-       (ashift:SI (match_op_dup 1 [(match_dup 2) (const_int 0)])
-                  (match_dup 3)))]
-  "reload_completed"
-  [(set (match_dup 4)
-       (ashift:SI (match_op_dup 1 [(match_dup 2) (const_int 0)])
-                  (match_dup 3)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 4)
-                   (const_int 0)))]
-  "")
-
-;; There is a 3 cycle delay between consecutive mfcr instructions
-;; so it is useful to combine 2 scc instructions to use only one mfcr.
-
-(define_peephole
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-       (match_operator:SI 1 "scc_comparison_operator"
-                          [(match_operand 2 "cc_reg_operand" "y")
-                           (const_int 0)]))
-   (set (match_operand:SI 3 "gpc_reg_operand" "=r")
-       (match_operator:SI 4 "scc_comparison_operator"
-                          [(match_operand 5 "cc_reg_operand" "y")
-                           (const_int 0)]))]
-  "REGNO (operands[2]) != REGNO (operands[5])"
-  "mfcr %3\;rlwinm %0,%3,%J1,1\;rlwinm %3,%3,%J4,1"
-  [(set_attr "type" "mfcr")
-   (set_attr "length" "12")])
-
-(define_peephole
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
-       (match_operator:DI 1 "scc_comparison_operator"
-                          [(match_operand 2 "cc_reg_operand" "y")
-                           (const_int 0)]))
-   (set (match_operand:DI 3 "gpc_reg_operand" "=r")
-       (match_operator:DI 4 "scc_comparison_operator"
-                          [(match_operand 5 "cc_reg_operand" "y")
-                           (const_int 0)]))]
-  "TARGET_POWERPC64 && REGNO (operands[2]) != REGNO (operands[5])"
-  "mfcr %3\;rlwinm %0,%3,%J1,1\;rlwinm %3,%3,%J4,1"
-  [(set_attr "type" "mfcr")
-   (set_attr "length" "12")])
-
-;; There are some scc insns that can be done directly, without a compare.
-;; These are faster because they don't involve the communications between
-;; the FXU and branch units.   In fact, we will be replacing all of the
-;; integer scc insns here or in the portable methods in emit_store_flag.
-;;
-;; Also support (neg (scc ..)) since that construct is used to replace
-;; branches, (plus (scc ..) ..) since that construct is common and
-;; takes no more insns than scc, and (and (neg (scc ..)) ..) in the
-;; cases where it is no more expensive than (neg (scc ..)).
-
-;; Have reload force a constant into a register for the simple insns that
-;; otherwise won't accept constants.  We do this because it is faster than
-;; the cmp/mfcr sequence we would otherwise generate.
-
-(define_mode_attr scc_eq_op2 [(SI "rKLI")
-                             (DI "rKJI")])
-
-(define_insn_and_split "*eq<mode>"
-  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
-       (eq:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")
-               (match_operand:GPR 2 "scc_eq_operand" "<scc_eq_op2>")))]
-  ""
-  "#"
-  ""
-  [(set (match_dup 0)
-       (clz:GPR (match_dup 3)))
-   (set (match_dup 0)
-       (lshiftrt:GPR (match_dup 0) (match_dup 4)))]
-  {
-    if (GET_CODE (operands[2]) != CONST_INT || INTVAL (operands[2]) != 0)
-      {
-       /* Use output operand as intermediate.  */
-       operands[3] = operands[0];
-
-       if (logical_operand (operands[2], <MODE>mode))
-         emit_insn (gen_rtx_SET (VOIDmode, operands[3],
-                                 gen_rtx_XOR (<MODE>mode,
-                                              operands[1], operands[2])));
-       else
-         emit_insn (gen_rtx_SET (VOIDmode, operands[3],
-                                 gen_rtx_PLUS (<MODE>mode, operands[1],
-                                               negate_rtx (<MODE>mode,
-                                                           operands[2]))));
-      }
-    else
-      operands[3] = operands[1];
-
-    operands[4] = GEN_INT (exact_log2 (GET_MODE_BITSIZE (<MODE>mode)));
-  })
-
-(define_insn_and_split "*eq<mode>_compare"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=y")
-       (compare:CC
-        (eq:P (match_operand:P 1 "gpc_reg_operand" "=r")
-              (match_operand:P 2 "scc_eq_operand" "<scc_eq_op2>"))
-        (const_int 0)))
-   (set (match_operand:P 0 "gpc_reg_operand" "=r")
-       (eq:P (match_dup 1) (match_dup 2)))]
-  "optimize_size"
-  "#"
-  "optimize_size"
-  [(set (match_dup 0)
-       (clz:P (match_dup 4)))
-   (parallel [(set (match_dup 3)
-                  (compare:CC (lshiftrt:P (match_dup 0) (match_dup 5))
-                              (const_int 0)))
-             (set (match_dup 0)
-                  (lshiftrt:P (match_dup 0) (match_dup 5)))])]
-  {
-    if (GET_CODE (operands[2]) != CONST_INT || INTVAL (operands[2]) != 0)
-      {
-       /* Use output operand as intermediate.  */
-       operands[4] = operands[0];
-
-       if (logical_operand (operands[2], <MODE>mode))
-         emit_insn (gen_rtx_SET (VOIDmode, operands[4],
-                                 gen_rtx_XOR (<MODE>mode,
-                                              operands[1], operands[2])));
-       else
-         emit_insn (gen_rtx_SET (VOIDmode, operands[4],
-                                 gen_rtx_PLUS (<MODE>mode, operands[1],
-                                               negate_rtx (<MODE>mode,
-                                                           operands[2]))));
-      }
-    else
-      operands[4] = operands[1];
-
-    operands[5] = GEN_INT (exact_log2 (GET_MODE_BITSIZE (<MODE>mode)));
-  })
-
-;; We have insns of the form shown by the first define_insn below.  If
-;; there is something inside the comparison operation, we must split it.
-(define_split
-  [(set (match_operand:SI 0 "gpc_reg_operand" "")
-       (plus:SI (match_operator 1 "comparison_operator"
-                                [(match_operand:SI 2 "" "")
-                                 (match_operand:SI 3
-                                                   "reg_or_cint_operand" "")])
-                (match_operand:SI 4 "gpc_reg_operand" "")))
-   (clobber (match_operand:SI 5 "register_operand" ""))]
-  "! gpc_reg_operand (operands[2], SImode)"
-  [(set (match_dup 5) (match_dup 2))
-   (set (match_dup 0) (plus:SI (match_op_dup 1 [(match_dup 5) (match_dup 3)])
-                              (match_dup 4)))])
-
-(define_insn "*plus_eqsi"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=&r,&r,&r,&r,&r")
-       (plus:SI (eq:SI (match_operand:SI 1 "gpc_reg_operand" "%r,r,r,r,r")
-                       (match_operand:SI 2 "scc_eq_operand" "r,O,K,L,I"))
-                (match_operand:SI 3 "gpc_reg_operand" "r,r,r,r,r")))]
-  "TARGET_32BIT"
-  "@
-   xor %0,%1,%2\;subfic %0,%0,0\;addze %0,%3
-   subfic %0,%1,0\;addze %0,%3
-   xori %0,%1,%b2\;subfic %0,%0,0\;addze %0,%3
-   xoris %0,%1,%u2\;subfic %0,%0,0\;addze %0,%3
-   subfic %0,%1,%2\;subfic %0,%0,0\;addze %0,%3"
-  [(set_attr "type" "three,two,three,three,three")
-   (set_attr "length" "12,8,12,12,12")])
-
-(define_insn "*compare_plus_eqsi"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,x,x,x,x,?y,?y,?y,?y,?y")
-       (compare:CC
-        (plus:SI
-         (eq:SI (match_operand:SI 1 "gpc_reg_operand" "%r,r,r,r,r,r,r,r,r,r")
-                (match_operand:SI 2 "scc_eq_operand" "r,O,K,L,I,r,O,K,L,I"))
-         (match_operand:SI 3 "gpc_reg_operand" "r,r,r,r,r,r,r,r,r,r"))
-        (const_int 0)))
-   (clobber (match_scratch:SI 4 "=&r,&r,&r,&r,&r,&r,&r,&r,&r,&r"))]
-  "TARGET_32BIT && optimize_size"
-  "@
-   xor %4,%1,%2\;subfic %4,%4,0\;addze. %4,%3
-   subfic %4,%1,0\;addze. %4,%3
-   xori %4,%1,%b2\;subfic %4,%4,0\;addze. %4,%3
-   xoris %4,%1,%u2\;subfic %4,%4,0\;addze. %4,%3
-   subfic %4,%1,%2\;subfic %4,%4,0\;addze. %4,%3
-   #
-   #
-   #
-   #
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "12,8,12,12,12,16,12,16,16,16")])
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_cr0_operand" "")
-       (compare:CC
-        (plus:SI
-         (eq:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                (match_operand:SI 2 "scc_eq_operand" ""))
-         (match_operand:SI 3 "gpc_reg_operand" ""))
-        (const_int 0)))
-   (clobber (match_scratch:SI 4 ""))]
-  "TARGET_32BIT && optimize_size && reload_completed"
-  [(set (match_dup 4)
-       (plus:SI (eq:SI (match_dup 1)
-                (match_dup 2))
-         (match_dup 3)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 4)
-                   (const_int 0)))]
-  "")
-
-(define_insn "*plus_eqsi_compare"
-  [(set (match_operand:CC 4 "cc_reg_operand" "=x,x,x,x,x,?y,?y,?y,?y,?y")
-       (compare:CC
-        (plus:SI
-         (eq:SI (match_operand:SI 1 "gpc_reg_operand" "%r,r,r,r,r,r,r,r,r,r")
-                (match_operand:SI 2 "scc_eq_operand" "r,O,K,L,I,r,O,K,L,I"))
-         (match_operand:SI 3 "gpc_reg_operand" "r,r,r,r,r,r,r,r,r,r"))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=&r,&r,&r,&r,&r,&r,&r,&r,&r,&r")
-       (plus:SI (eq:SI (match_dup 1) (match_dup 2)) (match_dup 3)))]
-  "TARGET_32BIT && optimize_size"
-  "@
-   xor %0,%1,%2\;subfic %0,%0,0\;addze. %0,%3
-   subfic %0,%1,0\;addze. %0,%3
-   xori %0,%1,%b2\;subfic %0,%0,0\;addze. %0,%3
-   xoris %0,%1,%u2\;subfic %0,%0,0\;addze. %0,%3
-   subfic %0,%1,%2\;subfic %0,%0,0\;addze. %0,%3
-   #
-   #
-   #
-   #
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "12,8,12,12,12,16,12,16,16,16")])
-
-(define_split
-  [(set (match_operand:CC 4 "cc_reg_not_cr0_operand" "")
-       (compare:CC
-        (plus:SI
-         (eq:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                (match_operand:SI 2 "scc_eq_operand" ""))
-         (match_operand:SI 3 "gpc_reg_operand" ""))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (plus:SI (eq:SI (match_dup 1) (match_dup 2)) (match_dup 3)))]
-  "TARGET_32BIT && optimize_size && reload_completed"
-  [(set (match_dup 0)
-       (plus:SI (eq:SI (match_dup 1) (match_dup 2)) (match_dup 3)))
-   (set (match_dup 4)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
-
-(define_insn "*neg_eq0<mode>"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
-       (neg:P (eq:P (match_operand:P 1 "gpc_reg_operand" "r")
-                    (const_int 0))))]
-  ""
-  "addic %0,%1,-1\;subfe %0,%0,%0"
-  [(set_attr "type" "two")
-   (set_attr "length" "8")])
-
-(define_insn_and_split "*neg_eq<mode>"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
-       (neg:P (eq:P (match_operand:P 1 "gpc_reg_operand" "%r")
-                    (match_operand:P 2 "scc_eq_operand" "<scc_eq_op2>"))))]
-  ""
-  "#"
-  ""
-  [(set (match_dup 0) (neg:P (eq:P (match_dup 3) (const_int 0))))]
-  {
-    if (GET_CODE (operands[2]) != CONST_INT || INTVAL (operands[2]) != 0)
-      {
-       /* Use output operand as intermediate.  */
-       operands[3] = operands[0];
-
-       if (logical_operand (operands[2], <MODE>mode))
-         emit_insn (gen_rtx_SET (VOIDmode, operands[3],
-                                 gen_rtx_XOR (<MODE>mode,
-                                              operands[1], operands[2])));
-       else
-         emit_insn (gen_rtx_SET (VOIDmode, operands[3],
-                                 gen_rtx_PLUS (<MODE>mode, operands[1],
-                                               negate_rtx (<MODE>mode,
-                                                           operands[2]))));
-      }
-    else
-      operands[3] = operands[1];
-  })
-
-(define_insn "*ne0_<mode>"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
-       (ne:P (match_operand:P 1 "gpc_reg_operand" "r")
-             (const_int 0)))
-   (clobber (match_scratch:P 2 "=&r"))]
-  "!(TARGET_32BIT && TARGET_ISEL)"
-  "addic %2,%1,-1\;subfe %0,%2,%1"
-  [(set_attr "type" "two")
-   (set_attr "length" "8")])
-
-(define_insn "*plus_ne0_<mode>"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
-       (plus:P (ne:P (match_operand:P 1 "gpc_reg_operand" "r")
-                     (const_int 0))
-               (match_operand:P 2 "gpc_reg_operand" "r")))
-   (clobber (match_scratch:P 3 "=&r"))]
-  ""
-  "addic %3,%1,-1\;addze %0,%2"
-  [(set_attr "type" "two")
-   (set_attr "length" "8")])
-
-(define_insn "*compare_plus_ne0_<mode>"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC (plus:P (ne:P (match_operand:P 1 "gpc_reg_operand" "r,r")
-                                 (const_int 0))
-                           (match_operand:P 2 "gpc_reg_operand" "r,r"))
-                   (const_int 0)))
-   (clobber (match_scratch:P 3 "=&r,&r"))
-   (clobber (match_scratch:P 4 "=X,&r"))]
-  ""
-  "@
-   addic %3,%1,-1\;addze. %3,%2
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "8,12")])
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (ne:P (match_operand:SI 1 "gpc_reg_operand" "")
-                         (const_int 0))
-                   (neg:P (match_operand:P 2 "gpc_reg_operand" ""))))
-   (clobber (match_scratch:P 3 ""))
-   (clobber (match_scratch:P 4 ""))]
-  "reload_completed"
-  [(parallel [(set (match_dup 3)
-                  (plus:P (ne:P (match_dup 1)
-                                (const_int 0))
-                          (match_dup 2)))
-              (clobber (match_dup 4))])
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
-
-; For combine.
-(define_insn "*compare_plus_ne0_<mode>_1"
-  [(set (match_operand:CCEQ 0 "cc_reg_operand" "=x,?y")
-       (compare:CCEQ (ne:P (match_operand:P 1 "gpc_reg_operand" "r,r")
-                           (const_int 0))
-                     (neg:P (match_operand:P 2 "gpc_reg_operand" "r,r"))))
-   (clobber (match_scratch:P 3 "=&r,&r"))
-   (clobber (match_scratch:P 4 "=X,&r"))]
-  ""
-  "@
-   addic %3,%1,-1\;addze. %3,%2
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "8,12")])
-
-(define_split
-  [(set (match_operand:CCEQ 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CCEQ (ne:P (match_operand:SI 1 "gpc_reg_operand" "")
-                           (const_int 0))
-                     (neg:P (match_operand:P 2 "gpc_reg_operand" ""))))
-   (clobber (match_scratch:P 3 ""))
-   (clobber (match_scratch:P 4 ""))]
-  "reload_completed"
-  [(parallel [(set (match_dup 3)
-                  (plus:P (ne:P (match_dup 1)
-                                (const_int 0))
-                          (match_dup 2)))
-              (clobber (match_dup 4))])
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
-
-(define_insn "*plus_ne0_<mode>_compare"
-  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (plus:P (ne:P (match_operand:P 1 "gpc_reg_operand" "r,r")
-                      (const_int 0))
-                (match_operand:P 2 "gpc_reg_operand" "r,r"))
-        (const_int 0)))
-   (set (match_operand:P 0 "gpc_reg_operand" "=r,r")
-       (plus:P (ne:P (match_dup 1)
-                     (const_int 0))
-               (match_dup 2)))
-   (clobber (match_scratch:P 3 "=&r,&r"))]
-  ""
-  "@
-   addic %3,%1,-1\;addze. %0,%2
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "8,12")])
-
-(define_split
-  [(set (match_operand:CC 4 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC
-        (plus:P (ne:P (match_operand:P 1 "gpc_reg_operand" "")
-                      (const_int 0))
-                (match_operand:P 2 "gpc_reg_operand" ""))
-        (const_int 0)))
-   (set (match_operand:P 0 "gpc_reg_operand" "")
-       (plus:P (ne:P (match_dup 1)
-                     (const_int 0))
-               (match_dup 2)))
-   (clobber (match_scratch:P 3 ""))]
-  "reload_completed"
-  [(parallel [(set (match_dup 0)
-                  (plus:P (ne:P (match_dup 1)
-                                (const_int 0))
-                          (match_dup 2)))
-             (clobber (match_dup 3))])
-   (set (match_dup 4)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
-
-(define_insn "*leu<mode>"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
-       (leu:P (match_operand:P 1 "gpc_reg_operand" "r")
-              (match_operand:P 2 "reg_or_short_operand" "rI")))]
-  ""
-  "subf%I2c %0,%1,%2\;li %0,0\;adde %0,%0,%0"
-  [(set_attr "type" "three")
-   (set_attr "length" "12")])
-
-(define_insn "*leu<mode>_compare"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (leu:P (match_operand:P 1 "gpc_reg_operand" "r,r")
-               (match_operand:P 2 "reg_or_short_operand" "rI,rI"))
-        (const_int 0)))
-   (set (match_operand:P 0 "gpc_reg_operand" "=r,r")
-       (leu:P (match_dup 1) (match_dup 2)))]
-  ""
-  "@
-   subf%I2c %0,%1,%2\;li %0,0\;adde. %0,%0,%0
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "12,16")])
-
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_cr0_operand" "")
-       (compare:CC
-        (leu:P (match_operand:P 1 "gpc_reg_operand" "")
-               (match_operand:P 2 "reg_or_short_operand" ""))
-        (const_int 0)))
-   (set (match_operand:P 0 "gpc_reg_operand" "")
-       (leu:P (match_dup 1) (match_dup 2)))]
-  "reload_completed"
-  [(set (match_dup 0)
-       (leu:P (match_dup 1) (match_dup 2)))
-   (set (match_dup 3)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
-
-(define_insn "*plus_leu<mode>"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=&r")
-       (plus:P (leu:P (match_operand:P 1 "gpc_reg_operand" "r")
-                      (match_operand:P 2 "reg_or_short_operand" "rI"))
-               (match_operand:P 3 "gpc_reg_operand" "r")))]
-  ""
-  "subf%I2c %0,%1,%2\;addze %0,%3"
-  [(set_attr "type" "two")
-   (set_attr "length" "8")])
-
-(define_insn ""
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (plus:SI (leu:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                         (match_operand:SI 2 "reg_or_short_operand" "rI,rI"))
-                 (match_operand:SI 3 "gpc_reg_operand" "r,r"))
-        (const_int 0)))
-   (clobber (match_scratch:SI 4 "=&r,&r"))]
-  "TARGET_32BIT"
-  "@
-   subf%I2c %4,%1,%2\;addze. %4,%3
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "8,12")])
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_cr0_operand" "")
-       (compare:CC
-        (plus:SI (leu:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                         (match_operand:SI 2 "reg_or_short_operand" ""))
-                 (match_operand:SI 3 "gpc_reg_operand" ""))
-        (const_int 0)))
-   (clobber (match_scratch:SI 4 ""))]
-  "TARGET_32BIT && reload_completed"
-  [(set (match_dup 4)
-       (plus:SI (leu:SI (match_dup 1) (match_dup 2))
-                 (match_dup 3)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 4)
-                   (const_int 0)))]
-  "")
-
-(define_insn ""
-  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (plus:SI (leu:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                         (match_operand:SI 2 "reg_or_short_operand" "rI,rI"))
-                 (match_operand:SI 3 "gpc_reg_operand" "r,r"))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=&r,&r")
-       (plus:SI (leu:SI (match_dup 1) (match_dup 2)) (match_dup 3)))]
-  "TARGET_32BIT"
-  "@
-   subf%I2c %0,%1,%2\;addze. %0,%3
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "8,12")])
-
-(define_split
-  [(set (match_operand:CC 4 "cc_reg_not_cr0_operand" "")
-       (compare:CC
-        (plus:SI (leu:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                         (match_operand:SI 2 "reg_or_short_operand" ""))
-                 (match_operand:SI 3 "gpc_reg_operand" ""))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (plus:SI (leu:SI (match_dup 1) (match_dup 2)) (match_dup 3)))]
-  "TARGET_32BIT && reload_completed"
-  [(set (match_dup 0)
-       (plus:SI (leu:SI (match_dup 1) (match_dup 2)) (match_dup 3)))
-   (set (match_dup 4)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
-
-(define_insn "*neg_leu<mode>"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
-       (neg:P (leu:P (match_operand:P 1 "gpc_reg_operand" "r")
-                     (match_operand:P 2 "reg_or_short_operand" "rI"))))]
-  ""
-  "subf%I2c %0,%1,%2\;subfe %0,%0,%0\;nand %0,%0,%0"
-   [(set_attr "type" "three")
-    (set_attr "length" "12")])
-
-(define_insn "*and_neg_leu<mode>"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=&r")
-       (and:P (neg:P
-                (leu:P (match_operand:P 1 "gpc_reg_operand" "r")
-                       (match_operand:P 2 "reg_or_short_operand" "rI")))
-               (match_operand:P 3 "gpc_reg_operand" "r")))]
-  ""
-  "subf%I2c %0,%1,%2\;subfe %0,%0,%0\;andc %0,%3,%0"
-  [(set_attr "type" "three")
-   (set_attr "length" "12")])
-
-(define_insn ""
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (and:SI (neg:SI
-                 (leu:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                         (match_operand:SI 2 "reg_or_short_operand" "rI,rI")))
-                (match_operand:SI 3 "gpc_reg_operand" "r,r"))
-        (const_int 0)))
-   (clobber (match_scratch:SI 4 "=&r,&r"))]
-  "TARGET_32BIT"
-  "@
-   subf%I2c %4,%1,%2\;subfe %4,%4,%4\;andc. %4,%3,%4
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "12,16")])
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_cr0_operand" "")
-       (compare:CC
-        (and:SI (neg:SI
-                 (leu:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                         (match_operand:SI 2 "reg_or_short_operand" "")))
-                (match_operand:SI 3 "gpc_reg_operand" ""))
-        (const_int 0)))
-   (clobber (match_scratch:SI 4 ""))]
-  "TARGET_32BIT && reload_completed"
-  [(set (match_dup 4)
-       (and:SI (neg:SI (leu:SI (match_dup 1) (match_dup 2)))
-               (match_dup 3)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 4)
-                   (const_int 0)))]
-  "")
-
-(define_insn ""
-  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (and:SI (neg:SI
-                 (leu:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                         (match_operand:SI 2 "reg_or_short_operand" "rI,rI")))
-                (match_operand:SI 3 "gpc_reg_operand" "r,r"))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=&r,&r")
-       (and:SI (neg:SI (leu:SI (match_dup 1) (match_dup 2))) (match_dup 3)))]
-  "TARGET_32BIT"
-  "@
-   subf%I2c %0,%1,%2\;subfe %0,%0,%0\;andc. %0,%3,%0
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "12,16")])
-
-(define_split
-  [(set (match_operand:CC 4 "cc_reg_not_cr0_operand" "")
-       (compare:CC
-        (and:SI (neg:SI
-                 (leu:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                         (match_operand:SI 2 "reg_or_short_operand" "")))
-                (match_operand:SI 3 "gpc_reg_operand" ""))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (and:SI (neg:SI (leu:SI (match_dup 1) (match_dup 2))) (match_dup 3)))]
-  "TARGET_32BIT && reload_completed"
-  [(set (match_dup 0)
-       (and:SI (neg:SI (leu:SI (match_dup 1) (match_dup 2)))
-               (match_dup 3)))
-   (set (match_dup 4)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
-
-(define_insn_and_split "*ltu<mode>"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=r,r")
-       (ltu:P (match_operand:P 1 "gpc_reg_operand" "r,r")
-              (match_operand:P 2 "reg_or_neg_short_operand" "r,P")))]
-  ""
-  "#"
-  ""
-  [(set (match_dup 0) (neg:P (ltu:P (match_dup 1) (match_dup 2))))
-   (set (match_dup 0) (neg:P (match_dup 0)))]
-  "")
-
-(define_insn_and_split "*ltu<mode>_compare"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC
-        (ltu:P (match_operand:P 1 "gpc_reg_operand" "r,r,r,r")
-               (match_operand:P 2 "reg_or_neg_short_operand" "r,P,r,P"))
-        (const_int 0)))
-   (set (match_operand:P 0 "gpc_reg_operand" "=r,r,r,r")
-       (ltu:P (match_dup 1) (match_dup 2)))]
-  ""
-  "#"
-  ""
-  [(set (match_dup 0) (neg:P (ltu:P (match_dup 1) (match_dup 2))))
-   (parallel [(set (match_dup 3)
-                  (compare:CC (neg:P (match_dup 0)) (const_int 0)))
-             (set (match_dup 0) (neg:P (match_dup 0)))])]
-  "")
-
-(define_insn_and_split "*plus_ltu<mode>"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=&r,r")
-       (plus:P (ltu:P (match_operand:P 1 "gpc_reg_operand" "r,r")
-                      (match_operand:P 2 "reg_or_neg_short_operand" "r,P"))
-               (match_operand:P 3 "gpc_reg_operand" "r,r")))]
-  ""
-  "#"
-  "&& !reg_overlap_mentioned_p (operands[0], operands[3])"
-  [(set (match_dup 0) (neg:P (ltu:P (match_dup 1) (match_dup 2))))
-   (set (match_dup 0) (minus:P (match_dup 3) (match_dup 0)))]
-  "")
-
-(define_insn_and_split "*plus_ltu<mode>_1"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=&r,r")
-       (plus:P (ltu:P (match_operand:P 1 "gpc_reg_operand" "r,r")
-                      (match_operand:P 2 "reg_or_neg_short_operand" "r,P"))
-               (match_operand:P 3 "short_cint_operand" "I,I")))]
-  ""
-  "#"
-  "&& !reg_overlap_mentioned_p (operands[0], operands[3])"
-  [(set (match_dup 0) (neg:P (ltu:P (match_dup 1) (match_dup 2))))
-   (parallel [(set (match_dup 0) (minus:P (match_dup 3) (match_dup 0)))
-             (clobber (reg:P CA_REGNO))])]
-  "")
-
-(define_insn_and_split "*plus_ltu<mode>_compare"
-  [(set (match_operand:CC 4 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC
-        (plus:P (ltu:P (match_operand:P 1 "gpc_reg_operand" "r,r,r,r")
-                       (match_operand:P 2 "reg_or_neg_short_operand" "r,P,r,P"))
-                (match_operand:P 3 "gpc_reg_operand" "r,r,r,r"))
-        (const_int 0)))
-   (set (match_operand:P 0 "gpc_reg_operand" "=&r,&r,&r,&r")
-       (plus:P (ltu:P (match_dup 1) (match_dup 2)) (match_dup 3)))]
-  ""
-  "#"
-  "&& !reg_overlap_mentioned_p (operands[0], operands[3])"
-  [(set (match_dup 0) (neg:P (ltu:P (match_dup 1) (match_dup 2))))
-   (parallel [(set (match_dup 4)
-                  (compare:CC (minus:P (match_dup 3) (match_dup 0))
-                              (const_int 0)))
-             (set (match_dup 0) (minus:P (match_dup 3) (match_dup 0)))])]
-  "")
-
-(define_insn "*neg_ltu<mode>"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=r,r")
-       (neg:P (ltu:P (match_operand:P 1 "gpc_reg_operand" "r,r")
-                     (match_operand:P 2 "reg_or_neg_short_operand" "r,P"))))]
-  ""
-  "@
-   subfc %0,%2,%1\;subfe %0,%0,%0
-   addic %0,%1,%n2\;subfe %0,%0,%0"
-  [(set_attr "type" "two")
-   (set_attr "length" "8")])
-
-(define_insn "*geu<mode>"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=r,r")
-       (geu:P (match_operand:P 1 "gpc_reg_operand" "r,r")
-              (match_operand:P 2 "reg_or_neg_short_operand" "r,P")))]
-  ""
-  "@
-   subfc %0,%2,%1\;li %0,0\;adde %0,%0,%0
-   addic %0,%1,%n2\;li %0,0\;adde %0,%0,%0"
-  [(set_attr "type" "three")
-   (set_attr "length" "12")])
-
-(define_insn "*geu<mode>_compare"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC
-        (geu:P (match_operand:P 1 "gpc_reg_operand" "r,r,r,r")
-               (match_operand:P 2 "reg_or_neg_short_operand" "r,P,r,P"))
-        (const_int 0)))
-   (set (match_operand:P 0 "gpc_reg_operand" "=r,r,r,r")
-       (geu:P (match_dup 1) (match_dup 2)))]
-  ""
-  "@
-   subfc %0,%2,%1\;li %0,0\;adde. %0,%0,%0
-   addic %0,%1,%n2\;li %0,0\;adde. %0,%0,%0
-   #
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "12,12,16,16")])
-
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC
-        (geu:P (match_operand:P 1 "gpc_reg_operand" "")
-               (match_operand:P 2 "reg_or_neg_short_operand" ""))
-        (const_int 0)))
-   (set (match_operand:P 0 "gpc_reg_operand" "")
-       (geu:P (match_dup 1) (match_dup 2)))]
-  "reload_completed"
-  [(set (match_dup 0)
-       (geu:P (match_dup 1) (match_dup 2)))
-   (set (match_dup 3)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
-
-(define_insn "*plus_geu<mode>"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=&r,&r")
-       (plus:P (geu:P (match_operand:P 1 "gpc_reg_operand" "r,r")
-                      (match_operand:P 2 "reg_or_neg_short_operand" "r,P"))
-               (match_operand:P 3 "gpc_reg_operand" "r,r")))]
-  ""
-  "@
-   subfc %0,%2,%1\;addze %0,%3
-   addic %0,%1,%n2\;addze %0,%3"
-  [(set_attr "type" "two")
-   (set_attr "length" "8")])
-
-(define_insn ""
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC
-        (plus:SI (geu:SI (match_operand:SI 1 "gpc_reg_operand" "r,r,r,r")
-                         (match_operand:SI 2 "reg_or_neg_short_operand" "r,P,r,P"))
-                 (match_operand:SI 3 "gpc_reg_operand" "r,r,r,r"))
-        (const_int 0)))
-   (clobber (match_scratch:SI 4 "=&r,&r,&r,&r"))]
-  "TARGET_32BIT"
-  "@
-   subfc %4,%2,%1\;addze. %4,%3
-   addic %4,%1,%n2\;addze. %4,%3
-   #
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "8,8,12,12")])
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_cr0_operand" "")
-       (compare:CC
-        (plus:SI (geu:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                         (match_operand:SI 2 "reg_or_neg_short_operand" ""))
-                 (match_operand:SI 3 "gpc_reg_operand" ""))
-        (const_int 0)))
-   (clobber (match_scratch:SI 4 ""))]
-  "TARGET_32BIT && reload_completed"
-  [(set (match_dup 4)
-       (plus:SI (geu:SI (match_dup 1) (match_dup 2))
-                 (match_dup 3)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 4)
-                   (const_int 0)))]
-  "")
-
-(define_insn ""
-  [(set (match_operand:CC 4 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC
-        (plus:SI (geu:SI (match_operand:SI 1 "gpc_reg_operand" "r,r,r,r")
-                         (match_operand:SI 2 "reg_or_neg_short_operand" "r,P,r,P"))
-                 (match_operand:SI 3 "gpc_reg_operand" "r,r,r,r"))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=&r,&r,&r,&r")
-       (plus:SI (geu:SI (match_dup 1) (match_dup 2)) (match_dup 3)))]
-  "TARGET_32BIT"
-  "@
-   subfc %0,%2,%1\;addze. %0,%3
-   addic %0,%1,%n2\;addze. %0,%3
-   #
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "8,8,12,12")])
-
-(define_split
-  [(set (match_operand:CC 4 "cc_reg_not_cr0_operand" "")
-       (compare:CC
-        (plus:SI (geu:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                         (match_operand:SI 2 "reg_or_neg_short_operand" ""))
-                 (match_operand:SI 3 "gpc_reg_operand" ""))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (plus:SI (geu:SI (match_dup 1) (match_dup 2)) (match_dup 3)))]
-  "TARGET_32BIT && reload_completed"
-  [(set (match_dup 0)
-       (plus:SI (geu:SI (match_dup 1) (match_dup 2)) (match_dup 3)))
-   (set (match_dup 4)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
-
-(define_insn "*neg_geu<mode>"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=r,r")
-       (neg:P (geu:P (match_operand:P 1 "gpc_reg_operand" "r,r")
-                     (match_operand:P 2 "reg_or_short_operand" "r,I"))))]
-  ""
-  "@
-   subfc %0,%2,%1\;subfe %0,%0,%0\;nand %0,%0,%0
-   subfic %0,%1,-1\;add%I2c %0,%0,%2\;subfe %0,%0,%0"
-  [(set_attr "type" "three")
-   (set_attr "length" "12")])
-
-(define_insn "*and_neg_geu<mode>"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=&r,&r")
-       (and:P (neg:P
-                (geu:P (match_operand:P 1 "gpc_reg_operand" "r,r")
-                       (match_operand:P 2 "reg_or_neg_short_operand" "r,P")))
-               (match_operand:P 3 "gpc_reg_operand" "r,r")))]
-  ""
-  "@
-   subfc %0,%2,%1\;subfe %0,%0,%0\;andc %0,%3,%0
-   addic %0,%1,%n2\;subfe %0,%0,%0\;andc %0,%3,%0"
-  [(set_attr "type" "three")
-   (set_attr "length" "12")])
-
-(define_insn ""
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC
-        (and:SI (neg:SI
-                 (geu:SI (match_operand:SI 1 "gpc_reg_operand" "r,r,r,r")
-                         (match_operand:SI 2 "reg_or_neg_short_operand" "r,P,r,P")))
-                (match_operand:SI 3 "gpc_reg_operand" "r,r,r,r"))
-        (const_int 0)))
-   (clobber (match_scratch:SI 4 "=&r,&r,&r,&r"))]
-  "TARGET_32BIT"
-  "@
-   subfc %4,%2,%1\;subfe %4,%4,%4\;andc. %4,%3,%4
-   addic %4,%1,%n2\;subfe %4,%4,%4\;andc. %4,%3,%4
-   #
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "12,12,16,16")])
-
+  operands[5] = GEN_INT (count);
+  operands[6] = GEN_INT (put_bit);
+
+  return \"mfcr %4%Q2\;rlwinm. %4,%4,%5,%6,%6\";
+}"
+  [(set_attr "type" "shift")
+   (set_attr "dot" "yes")
+   (set_attr "length" "8,16")])
+
 (define_split
-  [(set (match_operand:CC 0 "cc_reg_not_cr0_operand" "")
+  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
        (compare:CC
-        (and:SI (neg:SI
-                 (geu:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                         (match_operand:SI 2 "reg_or_neg_short_operand" "")))
-                (match_operand:SI 3 "gpc_reg_operand" ""))
+        (ashift:SI (match_operator:SI 1 "scc_comparison_operator"
+                                      [(match_operand 2 "cc_reg_operand" "")
+                                       (const_int 0)])
+                   (match_operand:SI 3 "const_int_operand" ""))
         (const_int 0)))
-   (clobber (match_scratch:SI 4 ""))]
-  "TARGET_32BIT && reload_completed"
+   (set (match_operand:SI 4 "gpc_reg_operand" "")
+       (ashift:SI (match_op_dup 1 [(match_dup 2) (const_int 0)])
+                  (match_dup 3)))]
+  "reload_completed"
   [(set (match_dup 4)
-       (and:SI (neg:SI (geu:SI (match_dup 1) (match_dup 2)))
-               (match_dup 3)))
+       (ashift:SI (match_op_dup 1 [(match_dup 2) (const_int 0)])
+                  (match_dup 3)))
    (set (match_dup 0)
        (compare:CC (match_dup 4)
                    (const_int 0)))]
   "")
 
-(define_insn ""
-  [(set (match_operand:CC 4 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC
-        (and:SI (neg:SI
-                 (geu:SI (match_operand:SI 1 "gpc_reg_operand" "r,r,r,r")
-                         (match_operand:SI 2 "reg_or_neg_short_operand" "r,P,r,P")))
-                (match_operand:SI 3 "gpc_reg_operand" "r,r,r,r"))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=&r,&r,&r,&r")
-       (and:SI (neg:SI (geu:SI (match_dup 1) (match_dup 2))) (match_dup 3)))]
-  "TARGET_32BIT"
-  "@
-   subfc %0,%2,%1\;subfe %0,%0,%0\;andc. %0,%3,%0
-   addic %0,%1,%n2\;subfe %0,%0,%0\;andc. %0,%3,%0
-   #
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "12,12,16,16")])
-
-(define_split
-  [(set (match_operand:CC 4 "cc_reg_not_cr0_operand" "")
-       (compare:CC
-        (and:SI (neg:SI
-                 (geu:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                         (match_operand:SI 2 "reg_or_neg_short_operand" "")))
-                (match_operand:SI 3 "gpc_reg_operand" ""))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (and:SI (neg:SI (geu:SI (match_dup 1) (match_dup 2))) (match_dup 3)))]
-  "TARGET_32BIT && reload_completed"
-  [(set (match_dup 0)
-       (and:SI (neg:SI (geu:SI (match_dup 1) (match_dup 2))) (match_dup 3)))
-   (set (match_dup 4)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
+;; There is a 3 cycle delay between consecutive mfcr instructions
+;; so it is useful to combine 2 scc instructions to use only one mfcr.
 
-(define_insn "*plus_gt0<mode>"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=&r")
-       (plus:P (gt:P (match_operand:P 1 "gpc_reg_operand" "r")
-                     (const_int 0))
-                (match_operand:P 2 "gpc_reg_operand" "r")))]
-  ""
-  "addc %0,%1,%1\;subfe %0,%1,%0\;addze %0,%2"
-  [(set_attr "type" "three")
+(define_peephole
+  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
+       (match_operator:SI 1 "scc_comparison_operator"
+                          [(match_operand 2 "cc_reg_operand" "y")
+                           (const_int 0)]))
+   (set (match_operand:SI 3 "gpc_reg_operand" "=r")
+       (match_operator:SI 4 "scc_comparison_operator"
+                          [(match_operand 5 "cc_reg_operand" "y")
+                           (const_int 0)]))]
+  "REGNO (operands[2]) != REGNO (operands[5])"
+  "mfcr %3\;rlwinm %0,%3,%J1,1\;rlwinm %3,%3,%J4,1"
+  [(set_attr "type" "mfcr")
    (set_attr "length" "12")])
 
-(define_insn ""
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (plus:SI (gt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                        (const_int 0))
-                 (match_operand:SI 2 "gpc_reg_operand" "r,r"))
-        (const_int 0)))
-   (clobber (match_scratch:SI 3 "=&r,&r"))]
-  "TARGET_32BIT"
-  "@
-   addc %3,%1,%1\;subfe %3,%1,%3\;addze. %3,%2
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "12,16")])
+(define_peephole
+  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
+       (match_operator:DI 1 "scc_comparison_operator"
+                          [(match_operand 2 "cc_reg_operand" "y")
+                           (const_int 0)]))
+   (set (match_operand:DI 3 "gpc_reg_operand" "=r")
+       (match_operator:DI 4 "scc_comparison_operator"
+                          [(match_operand 5 "cc_reg_operand" "y")
+                           (const_int 0)]))]
+  "TARGET_POWERPC64 && REGNO (operands[2]) != REGNO (operands[5])"
+  "mfcr %3\;rlwinm %0,%3,%J1,1\;rlwinm %3,%3,%J4,1"
+  [(set_attr "type" "mfcr")
+   (set_attr "length" "12")])
 
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_cr0_operand" "")
-       (compare:CC
-        (plus:SI (gt:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                        (const_int 0))
-                 (match_operand:SI 2 "gpc_reg_operand" ""))
-        (const_int 0)))
-   (clobber (match_scratch:SI 3 ""))]
-  "TARGET_32BIT && reload_completed"
-  [(set (match_dup 3)
-       (plus:SI (gt:SI (match_dup 1) (const_int 0))
-                 (match_dup 2)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
 
-(define_insn ""
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (plus:DI (gt:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
-                        (const_int 0))
-                 (match_operand:DI 2 "gpc_reg_operand" "r,r"))
-        (const_int 0)))
-   (clobber (match_scratch:DI 3 "=&r,&r"))]
-  "TARGET_64BIT"
-  "@
-   addc %3,%1,%1\;subfe %3,%1,%3\;addze. %3,%2
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "12,16")])
+(define_mode_attr scc_eq_op2 [(SI "rKLI")
+                             (DI "rKJI")])
 
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC
-        (plus:DI (gt:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                        (const_int 0))
-                 (match_operand:DI 2 "gpc_reg_operand" ""))
-        (const_int 0)))
-   (clobber (match_scratch:DI 3 ""))]
-  "TARGET_64BIT && reload_completed"
-  [(set (match_dup 3)
-       (plus:DI (gt:DI (match_dup 1) (const_int 0))
-                (match_dup 2)))
+(define_insn_and_split "eq<mode>3"
+  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
+       (eq:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")
+               (match_operand:GPR 2 "scc_eq_operand" "<scc_eq_op2>")))
+   (clobber (match_scratch:GPR 3 "=r"))
+   (clobber (match_scratch:GPR 4 "=r"))]
+  ""
+  "#"
+  ""
+  [(set (match_dup 4)
+       (clz:GPR (match_dup 3)))
    (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
+       (lshiftrt:GPR (match_dup 4)
+                     (match_dup 5)))]
+{
+  operands[3] = rs6000_emit_eqne (<MODE>mode,
+                                 operands[1], operands[2], operands[3]);
 
-(define_insn ""
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (plus:SI (gt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                        (const_int 0))
-                 (match_operand:SI 2 "gpc_reg_operand" "r,r"))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=&r,&r")
-       (plus:SI (gt:SI (match_dup 1) (const_int 0)) (match_dup 2)))]
-  "TARGET_32BIT"
-  "@
-   addc %0,%1,%1\;subfe %0,%1,%0\;addze. %0,%2
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "12,16")])
+  if (GET_CODE (operands[4]) == SCRATCH)
+    operands[4] = gen_reg_rtx (<MODE>mode);
 
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_cr0_operand" "")
-       (compare:CC
-        (plus:SI (gt:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                        (const_int 0))
-                 (match_operand:SI 2 "gpc_reg_operand" ""))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (plus:SI (gt:SI (match_dup 1) (const_int 0)) (match_dup 2)))]
-  "TARGET_32BIT && reload_completed"
-  [(set (match_dup 0)
-       (plus:SI (gt:SI (match_dup 1) (const_int 0)) (match_dup 2)))
-   (set (match_dup 3)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
+  operands[5] = GEN_INT (exact_log2 (GET_MODE_BITSIZE (<MODE>mode)));
+}
+  [(set (attr "length")
+       (if_then_else (match_test "operands[2] == const0_rtx")
+                     (const_string "8")
+                     (const_string "12")))])
 
-(define_insn ""
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (plus:DI (gt:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
-                        (const_int 0))
-                 (match_operand:DI 2 "gpc_reg_operand" "r,r"))
-        (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "=&r,&r")
-       (plus:DI (gt:DI (match_dup 1) (const_int 0)) (match_dup 2)))]
-  "TARGET_64BIT"
-  "@
-   addc %0,%1,%1\;subfe %0,%1,%0\;addze. %0,%2
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "12,16")])
+(define_insn_and_split "ne<mode>3"
+  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
+       (ne:P (match_operand:P 1 "gpc_reg_operand" "r")
+             (match_operand:P 2 "scc_eq_operand" "<scc_eq_op2>")))
+   (clobber (match_scratch:P 3 "=r"))
+   (clobber (match_scratch:P 4 "=r"))
+   (clobber (reg:P CA_REGNO))]
+  "!TARGET_ISEL"
+  "#"
+  ""
+  [(parallel [(set (match_dup 4)
+                  (plus:P (match_dup 3)
+                          (const_int -1)))
+             (set (reg:P CA_REGNO)
+                  (ne:P (match_dup 3)
+                        (const_int 0)))])
+   (parallel [(set (match_dup 0)
+                  (plus:P (plus:P (not:P (match_dup 4))
+                                  (reg:P CA_REGNO))
+                          (match_dup 3)))
+             (clobber (reg:P CA_REGNO))])]
+{
+  operands[3] = rs6000_emit_eqne (<MODE>mode,
+                                 operands[1], operands[2], operands[3]);
 
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC
-        (plus:DI (gt:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                        (const_int 0))
-                 (match_operand:DI 2 "gpc_reg_operand" ""))
-        (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "")
-       (plus:DI (gt:DI (match_dup 1) (const_int 0)) (match_dup 2)))]
-  "TARGET_64BIT && reload_completed"
-  [(set (match_dup 0)
-       (plus:DI (gt:DI (match_dup 1) (const_int 0)) (match_dup 2)))
-   (set (match_dup 3)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
+  if (GET_CODE (operands[4]) == SCRATCH)
+    operands[4] = gen_reg_rtx (<MODE>mode);
+}
+  [(set (attr "length")
+       (if_then_else (match_test "operands[2] == const0_rtx")
+                     (const_string "8")
+                     (const_string "12")))])
 
-(define_insn_and_split "*gtu<mode>"
+(define_insn_and_split "*neg_eq_<mode>"
   [(set (match_operand:P 0 "gpc_reg_operand" "=r")
-       (gtu:P (match_operand:P 1 "gpc_reg_operand" "r")
-              (match_operand:P 2 "reg_or_short_operand" "rI")))]
+       (neg:P (eq:P (match_operand:P 1 "gpc_reg_operand" "r")
+                    (match_operand:P 2 "scc_eq_operand" "<scc_eq_op2>"))))
+   (clobber (match_scratch:P 3 "=r"))
+   (clobber (match_scratch:P 4 "=r"))
+   (clobber (reg:P CA_REGNO))]
   ""
   "#"
   ""
-  [(set (match_dup 0) (neg:P (gtu:P (match_dup 1) (match_dup 2))))
-   (set (match_dup 0) (neg:P (match_dup 0)))]
-  "")
+  [(parallel [(set (match_dup 4)
+                  (plus:P (match_dup 3)
+                          (const_int -1)))
+             (set (reg:P CA_REGNO)
+                  (ne:P (match_dup 3)
+                        (const_int 0)))])
+   (parallel [(set (match_dup 0)
+                  (plus:P (reg:P CA_REGNO)
+                          (const_int -1)))
+             (clobber (reg:P CA_REGNO))])]
+{
+  operands[3] = rs6000_emit_eqne (<MODE>mode,
+                                 operands[1], operands[2], operands[3]);
 
-(define_insn_and_split "*gtu<mode>_compare"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (gtu:P (match_operand:P 1 "gpc_reg_operand" "r,r")
-                (match_operand:P 2 "reg_or_short_operand" "rI,rI"))
-        (const_int 0)))
-   (set (match_operand:P 0 "gpc_reg_operand" "=r,r")
-       (gtu:P (match_dup 1) (match_dup 2)))]
+  if (GET_CODE (operands[4]) == SCRATCH)
+    operands[4] = gen_reg_rtx (<MODE>mode);
+}
+  [(set (attr "length")
+       (if_then_else (match_test "operands[2] == const0_rtx")
+                     (const_string "8")
+                     (const_string "12")))])
+
+(define_insn_and_split "*neg_ne_<mode>"
+  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
+       (neg:P (ne:P (match_operand:P 1 "gpc_reg_operand" "r")
+                    (match_operand:P 2 "scc_eq_operand" "<scc_eq_op2>"))))
+   (clobber (match_scratch:P 3 "=r"))
+   (clobber (match_scratch:P 4 "=r"))
+   (clobber (reg:P CA_REGNO))]
   ""
   "#"
   ""
-  [(set (match_dup 0) (neg:P (gtu:P (match_dup 1) (match_dup 2))))
-   (parallel [(set (match_dup 3)
-                  (compare:CC (neg:P (match_dup 0)) (const_int 0)))
-             (set (match_dup 0) (neg:P (match_dup 0)))])]
-  "")
+  [(parallel [(set (match_dup 4)
+                  (neg:P (match_dup 3)))
+             (set (reg:P CA_REGNO)
+                  (eq:P (match_dup 3)
+                        (const_int 0)))])
+   (parallel [(set (match_dup 0)
+                  (plus:P (reg:P CA_REGNO)
+                          (const_int -1)))
+             (clobber (reg:P CA_REGNO))])]
+{
+  operands[3] = rs6000_emit_eqne (<MODE>mode,
+                                 operands[1], operands[2], operands[3]);
+
+  if (GET_CODE (operands[4]) == SCRATCH)
+    operands[4] = gen_reg_rtx (<MODE>mode);
+}
+  [(set (attr "length")
+       (if_then_else (match_test "operands[2] == const0_rtx")
+                     (const_string "8")
+                     (const_string "12")))])
 
-(define_insn_and_split "*plus_gtu<mode>"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=&r")
-        (plus:P (gtu:P (match_operand:P 1 "gpc_reg_operand" "r")
-                      (match_operand:P 2 "reg_or_short_operand" "rI"))
-               (match_operand:P 3 "gpc_reg_operand" "r")))]
+(define_insn_and_split "*plus_eq_<mode>"
+  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
+       (plus:P (eq:P (match_operand:P 1 "gpc_reg_operand" "r")
+                     (match_operand:P 2 "scc_eq_operand" "<scc_eq_op2>"))
+               (match_operand:P 3 "gpc_reg_operand" "r")))
+   (clobber (match_scratch:P 4 "=r"))
+   (clobber (match_scratch:P 5 "=r"))
+   (clobber (reg:P CA_REGNO))]
   ""
   "#"
-  "&& !reg_overlap_mentioned_p (operands[0], operands[3])"
-  [(set (match_dup 0) (neg:P (gtu:P (match_dup 1) (match_dup 2))))
-   (set (match_dup 0) (minus:P (match_dup 3) (match_dup 0)))]
-  "")
+  ""
+  [(parallel [(set (match_dup 5)
+                  (neg:P (match_dup 4)))
+             (set (reg:P CA_REGNO)
+                  (eq:P (match_dup 4)
+                        (const_int 0)))])
+   (parallel [(set (match_dup 0)
+                  (plus:P (match_dup 3)
+                          (reg:P CA_REGNO)))
+             (clobber (reg:P CA_REGNO))])]
+{
+  operands[4] = rs6000_emit_eqne (<MODE>mode,
+                                 operands[1], operands[2], operands[4]);
+
+  if (GET_CODE (operands[5]) == SCRATCH)
+    operands[5] = gen_reg_rtx (<MODE>mode);
+}
+  [(set (attr "length")
+       (if_then_else (match_test "operands[2] == const0_rtx")
+                     (const_string "8")
+                     (const_string "12")))])
 
-(define_insn_and_split "*plus_gtu<mode>_1"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=&r")
-        (plus:P (gtu:P (match_operand:P 1 "gpc_reg_operand" "r")
-                      (match_operand:P 2 "reg_or_short_operand" "rI"))
-               (match_operand:P 3 "short_cint_operand" "I")))]
+(define_insn_and_split "*plus_ne_<mode>"
+  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
+       (plus:P (ne:P (match_operand:P 1 "gpc_reg_operand" "r")
+                     (match_operand:P 2 "scc_eq_operand" "<scc_eq_op2>"))
+               (match_operand:P 3 "gpc_reg_operand" "r")))
+   (clobber (match_scratch:P 4 "=r"))
+   (clobber (match_scratch:P 5 "=r"))
+   (clobber (reg:P CA_REGNO))]
   ""
   "#"
-  "&& !reg_overlap_mentioned_p (operands[0], operands[3])"
-  [(set (match_dup 0) (neg:P (gtu:P (match_dup 1) (match_dup 2))))
-   (parallel [(set (match_dup 0) (minus:P (match_dup 3) (match_dup 0)))
+  ""
+  [(parallel [(set (match_dup 5)
+                  (plus:P (match_dup 4)
+                          (const_int -1)))
+             (set (reg:P CA_REGNO)
+                  (ne:P (match_dup 4)
+                        (const_int 0)))])
+   (parallel [(set (match_dup 0)
+                  (plus:P (match_dup 3)
+                          (reg:P CA_REGNO)))
              (clobber (reg:P CA_REGNO))])]
-  "")
+{
+  operands[4] = rs6000_emit_eqne (<MODE>mode,
+                                 operands[1], operands[2], operands[4]);
 
-(define_insn_and_split "*plus_gtu<mode>_compare"
-  [(set (match_operand:CC 4 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC
-        (plus:P (gtu:P (match_operand:P 1 "gpc_reg_operand" "r,r,r,r")
-                       (match_operand:P 2 "reg_or_short_operand" "I,r,I,r"))
-                (match_operand:P 3 "gpc_reg_operand" "r,r,r,r"))
-        (const_int 0)))
-   (set (match_operand:P 0 "gpc_reg_operand" "=&r,&r,&r,&r")
-       (plus:P (gtu:P (match_dup 1) (match_dup 2)) (match_dup 3)))]
+  if (GET_CODE (operands[5]) == SCRATCH)
+    operands[5] = gen_reg_rtx (<MODE>mode);
+}
+  [(set (attr "length")
+       (if_then_else (match_test "operands[2] == const0_rtx")
+                     (const_string "8")
+                     (const_string "12")))])
+
+(define_insn_and_split "*minus_eq_<mode>"
+  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
+       (minus:P (match_operand:P 3 "gpc_reg_operand" "r")
+                (eq:P (match_operand:P 1 "gpc_reg_operand" "r")
+                      (match_operand:P 2 "scc_eq_operand" "<scc_eq_op2>"))))
+   (clobber (match_scratch:P 4 "=r"))
+   (clobber (match_scratch:P 5 "=r"))
+   (clobber (reg:P CA_REGNO))]
   ""
   "#"
-  "&& !reg_overlap_mentioned_p (operands[0], operands[3])"
-  [(set (match_dup 0) (neg:P (gtu:P (match_dup 1) (match_dup 2))))
-   (parallel [(set (match_dup 4)
-                  (compare:CC (minus:P (match_dup 3) (match_dup 0))
-                              (const_int 0)))
-             (set (match_dup 0) (minus:P (match_dup 3) (match_dup 0)))])]
-  "")
+  ""
+  [(parallel [(set (match_dup 5)
+                  (plus:P (match_dup 4)
+                          (const_int -1)))
+             (set (reg:P CA_REGNO)
+                  (ne:P (match_dup 4)
+                        (const_int 0)))])
+   (parallel [(set (match_dup 0)
+                  (plus:P (plus:P (match_dup 3)
+                                  (reg:P CA_REGNO))
+                          (const_int -1)))
+             (clobber (reg:P CA_REGNO))])]
+{
+  operands[4] = rs6000_emit_eqne (<MODE>mode,
+                                 operands[1], operands[2], operands[4]);
+
+  if (GET_CODE (operands[5]) == SCRATCH)
+    operands[5] = gen_reg_rtx (<MODE>mode);
+}
+  [(set (attr "length")
+       (if_then_else (match_test "operands[2] == const0_rtx")
+                     (const_string "8")
+                     (const_string "12")))])
 
-(define_insn "*neg_gtu<mode>"
+(define_insn_and_split "*minus_ne_<mode>"
   [(set (match_operand:P 0 "gpc_reg_operand" "=r")
-       (neg:P (gtu:P (match_operand:P 1 "gpc_reg_operand" "r")
-                     (match_operand:P 2 "reg_or_short_operand" "rI"))))]
+       (minus:P (match_operand:P 3 "gpc_reg_operand" "r")
+                (ne:P (match_operand:P 1 "gpc_reg_operand" "r")
+                      (match_operand:P 2 "scc_eq_operand" "<scc_eq_op2>"))))
+   (clobber (match_scratch:P 4 "=r"))
+   (clobber (match_scratch:P 5 "=r"))
+   (clobber (reg:P CA_REGNO))]
   ""
-  "subf%I2c %0,%1,%2\;subfe %0,%0,%0"
-  [(set_attr "type" "two")
-   (set_attr "length" "8")])
+  "#"
+  ""
+  [(parallel [(set (match_dup 5)
+                  (neg:P (match_dup 4)))
+             (set (reg:P CA_REGNO)
+                  (eq:P (match_dup 4)
+                        (const_int 0)))])
+   (parallel [(set (match_dup 0)
+                  (plus:P (plus:P (match_dup 3)
+                                  (reg:P CA_REGNO))
+                          (const_int -1)))
+             (clobber (reg:P CA_REGNO))])]
+{
+  operands[4] = rs6000_emit_eqne (<MODE>mode,
+                                 operands[1], operands[2], operands[4]);
+
+  if (GET_CODE (operands[5]) == SCRATCH)
+    operands[5] = gen_reg_rtx (<MODE>mode);
+}
+  [(set (attr "length")
+       (if_then_else (match_test "operands[2] == const0_rtx")
+                     (const_string "8")
+                     (const_string "12")))])
+
+(define_insn_and_split "*eqsi3_ext<mode>"
+  [(set (match_operand:EXTSI 0 "gpc_reg_operand" "=r")
+       (eq:EXTSI (match_operand:SI 1 "gpc_reg_operand" "r")
+                 (match_operand:SI 2 "scc_eq_operand" "rKLI")))
+   (clobber (match_scratch:SI 3 "=r"))
+   (clobber (match_scratch:SI 4 "=r"))]
+  ""
+  "#"
+  ""
+  [(set (match_dup 4)
+       (clz:SI (match_dup 3)))
+   (set (match_dup 0)
+       (zero_extend:EXTSI
+         (lshiftrt:SI (match_dup 4)
+                      (const_int 5))))]
+{
+  operands[3] = rs6000_emit_eqne (SImode,
+                                 operands[1], operands[2], operands[3]);
+
+  if (GET_CODE (operands[4]) == SCRATCH)
+    operands[4] = gen_reg_rtx (SImode);
+}
+  [(set (attr "length")
+       (if_then_else (match_test "operands[2] == const0_rtx")
+                     (const_string "8")
+                     (const_string "12")))])
+
+(define_insn_and_split "*nesi3_ext<mode>"
+  [(set (match_operand:EXTSI 0 "gpc_reg_operand" "=r")
+       (ne:EXTSI (match_operand:SI 1 "gpc_reg_operand" "r")
+                 (match_operand:SI 2 "scc_eq_operand" "rKLI")))
+   (clobber (match_scratch:SI 3 "=r"))
+   (clobber (match_scratch:SI 4 "=r"))
+   (clobber (match_scratch:EXTSI 5 "=r"))]
+  ""
+  "#"
+  ""
+  [(set (match_dup 4)
+       (clz:SI (match_dup 3)))
+   (set (match_dup 5)
+       (zero_extend:EXTSI
+         (lshiftrt:SI (match_dup 4)
+                      (const_int 5))))
+   (set (match_dup 0)
+       (xor:EXTSI (match_dup 5)
+                  (const_int 1)))]
+{
+  operands[3] = rs6000_emit_eqne (SImode,
+                                 operands[1], operands[2], operands[3]);
 
+  if (GET_CODE (operands[4]) == SCRATCH)
+    operands[4] = gen_reg_rtx (SImode);
+  if (GET_CODE (operands[5]) == SCRATCH)
+    operands[5] = gen_reg_rtx (<MODE>mode);
+}
+  [(set (attr "length")
+       (if_then_else (match_test "operands[2] == const0_rtx")
+                     (const_string "12")
+                     (const_string "16")))])
 \f
 ;; Define both directions of branch and return.  If we need a reload
 ;; register, we'd rather use CR0 since it is much easier to copy a
   [(set_attr "type" "jmpreg")])
 
 (define_insn "nop"
-  [(const_int 0)]
+  [(unspec [(const_int 0)] UNSPEC_NOP)]
   ""
   "nop")
 
 
 (define_insn "*ctr<mode>_internal1"
   [(set (pc)
-       (if_then_else (ne (match_operand:P 1 "register_operand" "c,*r,*r,*r")
+       (if_then_else (ne (match_operand:P 1 "register_operand" "c,*b,*b,*b")
                          (const_int 1))
                      (label_ref (match_operand 0 "" ""))
                      (pc)))
     return \"bdz $+8\;b %l0\";
 }"
   [(set_attr "type" "branch")
-   (set_attr "length" "*,12,16,16")])
+   (set_attr "length" "*,16,20,20")])
 
 (define_insn "*ctr<mode>_internal2"
   [(set (pc)
-       (if_then_else (ne (match_operand:P 1 "register_operand" "c,*r,*r,*r")
+       (if_then_else (ne (match_operand:P 1 "register_operand" "c,*b,*b,*b")
                          (const_int 1))
                      (pc)
                      (label_ref (match_operand 0 "" ""))))
     return \"bdnz $+8\;b %l0\";
 }"
   [(set_attr "type" "branch")
-   (set_attr "length" "*,12,16,16")])
+   (set_attr "length" "*,16,20,20")])
 
 ;; Similar but use EQ
 
 (define_insn "*ctr<mode>_internal5"
   [(set (pc)
-       (if_then_else (eq (match_operand:P 1 "register_operand" "c,*r,*r,*r")
+       (if_then_else (eq (match_operand:P 1 "register_operand" "c,*b,*b,*b")
                          (const_int 1))
                      (label_ref (match_operand 0 "" ""))
                      (pc)))
     return \"bdnz $+8\;b %l0\";
 }"
   [(set_attr "type" "branch")
-   (set_attr "length" "*,12,16,16")])
+   (set_attr "length" "*,16,20,20")])
 
 (define_insn "*ctr<mode>_internal6"
   [(set (pc)
-       (if_then_else (eq (match_operand:P 1 "register_operand" "c,*r,*r,*r")
+       (if_then_else (eq (match_operand:P 1 "register_operand" "c,*b,*b,*b")
                          (const_int 1))
                      (pc)
                      (label_ref (match_operand 0 "" ""))))
     return \"bdz $+8\;b %l0\";
 }"
   [(set_attr "type" "branch")
-   (set_attr "length" "*,12,16,16")])
+   (set_attr "length" "*,16,20,20")])
 
 ;; Now the splitters if we could not allocate the CTR register
 
    (clobber (match_scratch:CC 3 ""))
    (clobber (match_scratch:P 4 ""))]
   "reload_completed"
-  [(parallel [(set (match_dup 3)
-                  (compare:CC (plus:P (match_dup 1)
-                                       (const_int -1))
-                              (const_int 0)))
-             (set (match_dup 0)
-                  (plus:P (match_dup 1)
-                           (const_int -1)))])
+  [(set (match_dup 3)
+       (compare:CC (match_dup 1)
+                   (const_int 1)))
+   (set (match_dup 0)
+       (plus:P (match_dup 1)
+               (const_int -1)))
    (set (pc) (if_then_else (match_dup 7)
                           (match_dup 5)
                           (match_dup 6)))]
    (clobber (match_scratch:CC 3 ""))
    (clobber (match_scratch:P 4 ""))]
   "reload_completed && ! gpc_reg_operand (operands[0], SImode)"
-  [(parallel [(set (match_dup 3)
-                  (compare:CC (plus:P (match_dup 1)
-                                       (const_int -1))
-                              (const_int 0)))
-             (set (match_dup 4)
-                  (plus:P (match_dup 1)
-                           (const_int -1)))])
+  [(set (match_dup 3)
+       (compare:CC (match_dup 1)
+                   (const_int 1)))
+   (set (match_dup 4)
+       (plus:P (match_dup 1)
+               (const_int -1)))
    (set (match_dup 0)
        (match_dup 4))
    (set (pc) (if_then_else (match_dup 7)