predicates.md (gpc_reg_operand): Don't allow all hard registers numbered greater...
[platform/upstream/gcc.git] / gcc / config / rs6000 / rs6000.md
index 54df18a..e156e14 100644 (file)
@@ -1,5 +1,5 @@
 ;; Machine description for IBM RISC System 6000 (POWER) for GNU C compiler
-;; Copyright (C) 1990-2014 Free Software Foundation, Inc.
+;; Copyright (C) 1990-2015 Free Software Foundation, Inc.
 ;; Contributed by Richard Kenner (kenner@vlsi1.ultra.nyu.edu)
 
 ;; This file is part of GCC.
@@ -56,6 +56,8 @@
    (TFHAR_REGNO                        114)
    (TFIAR_REGNO                        115)
    (TEXASR_REGNO               116)
+   (FIRST_SPE_HIGH_REGNO       117)
+   (LAST_SPE_HIGH_REGNO                148)
   ])
 
 ;;
@@ -67,6 +69,7 @@
    UNSPEC_PROBE_STACK          ; probe stack memory reference
    UNSPEC_TOCPTR               ; address of a word pointing to the TOC
    UNSPEC_TOC                  ; address of the TOC (more-or-less)
+   UNSPEC_TOCSLOT              ; offset from r1 of toc pointer save slot
    UNSPEC_MOVSI_GOT
    UNSPEC_MV_CR_OV             ; move_from_CR_ov_bit
    UNSPEC_FCTIWZ
    UNSPEC_LFIWAX
    UNSPEC_LFIWZX
    UNSPEC_FCTIWUZ
+   UNSPEC_NOP
    UNSPEC_GRP_END_NOP
    UNSPEC_P8V_FMRGOW
    UNSPEC_P8V_MTVSRWZ
    UNSPEC_UNPACK_128BIT
    UNSPEC_PACK_128BIT
    UNSPEC_LSQ
+   UNSPEC_FUSION_GPR
   ])
 
 ;;
    load,store,fpload,fpstore,vecload,vecstore,
    cmp,
    branch,jmpreg,mfjmpr,mtjmpr,trap,isync,sync,load_l,store_c,
-   compare,
    cr_logical,delayed_cr,mfcr,mfcrf,mtcr,
    fpcompare,fp,fpsimple,dmul,sdiv,ddiv,ssqrt,dsqrt,
    brinc,
 (define_attr "size" "8,16,32,64" (const_string "32"))
 
 ;; Is this instruction record form ("dot", signed compare to 0, writing CR0)?
-;; This is used for add, logical, shift, mul.
+;; This is used for add, logical, shift, exts, mul.
 (define_attr "dot" "no,yes" (const_string "no"))
 
 ;; Does this instruction sign-extend its result?
                (const_string "yes")
                (const_string "no")))
 
+;; Is this instruction using operands[2] as shift amount, and can that be a
+;; register?
+;; This is used for shift insns.
+(define_attr "maybe_var_shift" "no,yes" (const_string "no"))
+
 ;; Is this instruction using a shift amount from a register?
 ;; This is used for shift insns.
-(define_attr "var_shift" "no,yes" (const_string "no"))
+(define_attr "var_shift" "no,yes"
+  (if_then_else (and (eq_attr "type" "shift")
+                    (eq_attr "maybe_var_shift" "yes"))
+               (if_then_else (match_operand 2 "gpc_reg_operand")
+                             (const_string "yes")
+                             (const_string "no"))
+               (const_string "no")))
 
 ;; Define floating point instruction sub-types for use with Xfpu.md
 (define_attr "fp_type" "fp_default,fp_addsub_s,fp_addsub_d,fp_mul_s,fp_mul_d,fp_div_s,fp_div_d,fp_maddsub_s,fp_maddsub_d,fp_sqrt_s,fp_sqrt_d" (const_string "fp_default"))
 ;; If this instruction is microcoded on the CELL processor
 ; The default for load extended, the recorded instructions and rotate/shifts by a variable is always microcoded
 (define_attr "cell_micro" "not,conditional,always"
-  (if_then_else (ior (eq_attr "type" "compare")
-                    (and (eq_attr "type" "shift,mul")
+  (if_then_else (ior (and (eq_attr "type" "shift,exts,mul")
                          (eq_attr "dot" "yes"))
                     (and (eq_attr "type" "load")
                          (eq_attr "sign_extend" "yes"))
 ; Any supported integer mode that fits in one register.
 (define_mode_iterator INT1 [QI HI SI (DI "TARGET_POWERPC64")])
 
-; extend modes for DImode
-(define_mode_iterator QHSI [QI HI SI])
+; Everything we can extend QImode to.
+(define_mode_iterator EXTQI [HI SI (DI "TARGET_POWERPC64")])
+
+; Everything we can extend HImode to.
+(define_mode_iterator EXTHI [SI (DI "TARGET_POWERPC64")])
+
+; Everything we can extend SImode to.
+(define_mode_iterator EXTSI [(DI "TARGET_POWERPC64")])
 
 ; QImode or HImode for small atomic ops
 (define_mode_iterator QHI [QI HI])
 (define_mode_attr f32_sv [(SF "stxsspx %x1,%y0")  (SD "stxsiwzx %x1,%y0")])
 
 ; Definitions for 32-bit fpr direct move
-(define_mode_attr f32_dm [(SF "wn") (SD "wm")])
+; At present, the decimal modes are not allowed in the traditional altivec
+; registers, so restrict the constraints to just the traditional FPRs.
+(define_mode_attr f32_dm [(SF "wn") (SD "wh")])
+
+; Definitions for 32-bit VSX
+(define_mode_attr f32_vsx [(SF "ww") (SD "wn")])
+
+; Definitions for 32-bit use of altivec registers
+(define_mode_attr f32_av  [(SF "wu") (SD "wn")])
+
+; Definitions for 64-bit VSX
+(define_mode_attr f64_vsx [(DF "ws") (DD "wn")])
+
+; Definitions for 64-bit direct move
+(define_mode_attr f64_dm  [(DF "wk") (DD "wh")])
+
+; Definitions for 64-bit use of altivec registers
+(define_mode_attr f64_av  [(DF "wv") (DD "wn")])
 
 ; These modes do not fit in integer registers in 32-bit mode.
 ; but on e500v2, the gpr are 64 bit registers
 ; SF/DF constraint for arithmetic on VSX registers
 (define_mode_attr Fv           [(SF "wy") (DF "ws")])
 
+; SF/DF constraint for arithmetic on altivec registers
+(define_mode_attr Fa           [(SF "wu") (DF "wv")])
+
 ; s/d suffix for things like fp_addsub_s/fp_addsub_d
 (define_mode_attr Fs           [(SF "s")  (DF "d")])
 
                               (simple_return "1")])
 (define_code_attr return_str [(return "") (simple_return "simple_")])
 
+; Logical operators.
+(define_code_iterator iorxor [ior xor])
+
+; Signed/unsigned variants of ops.
+(define_code_iterator any_extend [sign_extend zero_extend])
+(define_code_attr u [(sign_extend "") (zero_extend "u")])
+(define_code_attr su [(sign_extend "s") (zero_extend "u")])
+
 ; Various instructions that come in SI and DI forms.
 ; A generic w/d attribute, for things like cmpw/cmpd.
 (define_mode_attr wd [(QI    "b")
 ;; ISEL/ISEL64 target selection
 (define_mode_attr sel [(SI "") (DI "64")])
 
+;; Bitmask for shift instructions
+(define_mode_attr hH [(SI "h") (DI "H")])
+
+;; A mode twice the size of the given mode
+(define_mode_attr dmode [(SI "di") (DI "ti")])
+(define_mode_attr DMODE [(SI "DI") (DI "TI")])
+
 ;; Suffix for reload patterns
 (define_mode_attr ptrsize [(SI "32bit")
                           (DI "64bit")])
 ;; either.
 
 ;; Mode attribute for boolean operation register constraints for output
-(define_mode_attr BOOL_REGS_OUTPUT     [(TI    "&r,r,r,wa,v")
+(define_mode_attr BOOL_REGS_OUTPUT     [(TI    "&r,r,r,wt,v")
                                         (PTI   "&r,r,r")
                                         (V16QI "wa,v,&?r,?r,?r")
                                         (V8HI  "wa,v,&?r,?r,?r")
                                         (V1TI  "wa,v,&?r,?r,?r")])
 
 ;; Mode attribute for boolean operation register constraints for operand1
-(define_mode_attr BOOL_REGS_OP1                [(TI    "r,0,r,wa,v")
+(define_mode_attr BOOL_REGS_OP1                [(TI    "r,0,r,wt,v")
                                         (PTI   "r,0,r")
                                         (V16QI "wa,v,r,0,r")
                                         (V8HI  "wa,v,r,0,r")
                                         (V1TI  "wa,v,r,0,r")])
 
 ;; Mode attribute for boolean operation register constraints for operand2
-(define_mode_attr BOOL_REGS_OP2                [(TI    "r,r,0,wa,v")
+(define_mode_attr BOOL_REGS_OP2                [(TI    "r,r,0,wt,v")
                                         (PTI   "r,r,0")
                                         (V16QI "wa,v,r,r,0")
                                         (V8HI  "wa,v,r,r,0")
 ;; Mode attribute for boolean operation register constraints for operand1
 ;; for one_cmpl.  To simplify things, we repeat the constraint where 0
 ;; is used for operand1 or operand2
-(define_mode_attr BOOL_REGS_UNARY      [(TI    "r,0,0,wa,v")
+(define_mode_attr BOOL_REGS_UNARY      [(TI    "r,0,0,wt,v")
                                         (PTI   "r,0,0")
                                         (V16QI "wa,v,r,0,0")
                                         (V8HI  "wa,v,r,0,0")
                                         (V2DF  "wa,v,r,0,0")
                                         (V1TI  "wa,v,r,0,0")])
 
-;; Mode attribute for the clobber of CC0 for AND expansion.
-;; For the 128-bit types, we never do AND immediate, but we need to
-;; get the correct number of X's for the number of operands.
-(define_mode_attr BOOL_REGS_AND_CR0    [(TI    "X,X,X,X,X")
-                                        (PTI   "X,X,X")
-                                        (V16QI "X,X,X,X,X")
-                                        (V8HI  "X,X,X,X,X")
-                                        (V4SI  "X,X,X,X,X")
-                                        (V4SF  "X,X,X,X,X")
-                                        (V2DI  "X,X,X,X,X")
-                                        (V2DF  "X,X,X,X,X")
-                                        (V1TI  "X,X,X,X,X")])
+;; Reload iterator for creating the function to allocate a base register to
+;; supplement addressing modes.
+(define_mode_iterator RELOAD [V16QI V8HI V4SI V2DI V4SF V2DF V1TI
+                             SF SD SI DF DD DI TI PTI])
+
 \f
 ;; Start with fixed-point load and store insns.  Here we put only the more
 ;; complex forms.  Basic data transfer is done later.
 
-(define_expand "zero_extend<mode>di2"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "")
-       (zero_extend:DI (match_operand:QHSI 1 "gpc_reg_operand" "")))]
-  "TARGET_POWERPC64"
-  "")
-
-(define_insn "*zero_extend<mode>di2_internal1"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
-       (zero_extend:DI (match_operand:QHSI 1 "reg_or_mem_operand" "m,r")))]
-  "TARGET_POWERPC64 && (<MODE>mode != SImode || !TARGET_LFIWZX)"
+(define_insn "zero_extendqi<mode>2"
+  [(set (match_operand:EXTQI 0 "gpc_reg_operand" "=r,r")
+       (zero_extend:EXTQI (match_operand:QI 1 "reg_or_mem_operand" "m,r")))]
+  ""
   "@
-   l<wd>z%U1%X1 %0,%1
-   rldicl %0,%1,0,<dbits>"
+   lbz%U1%X1 %0,%1
+   rlwinm %0,%1,0,0xff"
   [(set_attr "type" "load,shift")])
 
-(define_insn "*zero_extend<mode>di2_internal2"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC (zero_extend:DI (match_operand:QHSI 1 "gpc_reg_operand" "r,r"))
+(define_insn_and_split "*zero_extendqi<mode>2_dot"
+  [(set (match_operand:CC 2 "cc_reg_operand" "=x,?y")
+       (compare:CC (zero_extend:EXTQI (match_operand:QI 1 "gpc_reg_operand" "r,r"))
                    (const_int 0)))
-   (clobber (match_scratch:DI 2 "=r,r"))]
-  "TARGET_64BIT"
+   (clobber (match_scratch:EXTQI 0 "=r,r"))]
+  "rs6000_gen_cell_microcode"
   "@
-   rldicl. %2,%1,0,<dbits>
+   andi. %0,%1,0xff
    #"
-  [(set_attr "type" "shift")
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[2], CCmode)"
+  [(set (match_dup 0)
+       (zero_extend:EXTQI (match_dup 1)))
+   (set (match_dup 2)
+       (compare:CC (match_dup 0)
+                   (const_int 0)))]
+  ""
+  [(set_attr "type" "logical")
    (set_attr "dot" "yes")
    (set_attr "length" "4,8")])
 
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:DI (match_operand:QHSI 1 "gpc_reg_operand" ""))
+(define_insn_and_split "*zero_extendqi<mode>2_dot2"
+  [(set (match_operand:CC 2 "cc_reg_operand" "=x,?y")
+       (compare:CC (zero_extend:EXTQI (match_operand:QI 1 "gpc_reg_operand" "r,r"))
                    (const_int 0)))
-   (clobber (match_scratch:DI 2 ""))]
-  "TARGET_POWERPC64 && reload_completed"
-  [(set (match_dup 2)
-       (zero_extend:DI (match_dup 1)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 2)
+   (set (match_operand:EXTQI 0 "gpc_reg_operand" "=r,r")
+       (zero_extend:EXTQI (match_dup 1)))]
+  "rs6000_gen_cell_microcode"
+  "@
+   andi. %0,%1,0xff
+   #"
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[2], CCmode)"
+  [(set (match_dup 0)
+       (zero_extend:EXTQI (match_dup 1)))
+   (set (match_dup 2)
+       (compare:CC (match_dup 0)
                    (const_int 0)))]
-  "")
+  ""
+  [(set_attr "type" "logical")
+   (set_attr "dot" "yes")
+   (set_attr "length" "4,8")])
+
+
+(define_insn "zero_extendhi<mode>2"
+  [(set (match_operand:EXTHI 0 "gpc_reg_operand" "=r,r")
+       (zero_extend:EXTHI (match_operand:HI 1 "reg_or_mem_operand" "m,r")))]
+  ""
+  "@
+   lhz%U1%X1 %0,%1
+   rlwinm %0,%1,0,0xffff"
+  [(set_attr "type" "load,shift")])
 
-(define_insn "*zero_extend<mode>di2_internal3"
+(define_insn_and_split "*zero_extendhi<mode>2_dot"
   [(set (match_operand:CC 2 "cc_reg_operand" "=x,?y")
-       (compare:CC (zero_extend:DI (match_operand:QHSI 1 "gpc_reg_operand" "r,r"))
+       (compare:CC (zero_extend:EXTHI (match_operand:HI 1 "gpc_reg_operand" "r,r"))
                    (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
-       (zero_extend:DI (match_dup 1)))]
-  "TARGET_64BIT"
+   (clobber (match_scratch:EXTHI 0 "=r,r"))]
+  "rs6000_gen_cell_microcode"
   "@
-   rldicl. %0,%1,0,<dbits>
+   andi. %0,%1,0xffff
    #"
-  [(set_attr "type" "shift")
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[2], CCmode)"
+  [(set (match_dup 0)
+       (zero_extend:EXTHI (match_dup 1)))
+   (set (match_dup 2)
+       (compare:CC (match_dup 0)
+                   (const_int 0)))]
+  ""
+  [(set_attr "type" "logical")
    (set_attr "dot" "yes")
    (set_attr "length" "4,8")])
 
-(define_split
-  [(set (match_operand:CC 2 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:DI (match_operand:QHSI 1 "gpc_reg_operand" ""))
+(define_insn_and_split "*zero_extendhi<mode>2_dot2"
+  [(set (match_operand:CC 2 "cc_reg_operand" "=x,?y")
+       (compare:CC (zero_extend:EXTHI (match_operand:HI 1 "gpc_reg_operand" "r,r"))
                    (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "")
-       (zero_extend:DI (match_dup 1)))]
-  "TARGET_POWERPC64 && reload_completed"
+   (set (match_operand:EXTHI 0 "gpc_reg_operand" "=r,r")
+       (zero_extend:EXTHI (match_dup 1)))]
+  "rs6000_gen_cell_microcode"
+  "@
+   andi. %0,%1,0xffff
+   #"
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[2], CCmode)"
   [(set (match_dup 0)
-       (zero_extend:DI (match_dup 1)))
+       (zero_extend:EXTHI (match_dup 1)))
    (set (match_dup 2)
        (compare:CC (match_dup 0)
                    (const_int 0)))]
-  "")
+  ""
+  [(set_attr "type" "logical")
+   (set_attr "dot" "yes")
+   (set_attr "length" "4,8")])
+
 
-(define_insn "*zero_extendsidi2_lfiwzx"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r,r,??wm,!wz,!wu")
-       (zero_extend:DI (match_operand:SI 1 "reg_or_mem_operand" "m,r,r,Z,Z")))]
-  "TARGET_POWERPC64 && TARGET_LFIWZX"
+(define_insn "zero_extendsi<mode>2"
+  [(set (match_operand:EXTSI 0 "gpc_reg_operand" "=r,r,??wj,!wz,!wu")
+       (zero_extend:EXTSI (match_operand:SI 1 "reg_or_mem_operand" "m,r,r,Z,Z")))]
+  ""
   "@
    lwz%U1%X1 %0,%1
    rldicl %0,%1,0,32
    lxsiwzx %x0,%y1"
   [(set_attr "type" "load,shift,mffgpr,fpload,fpload")])
 
-(define_insn "extendqidi2"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
-       (sign_extend:DI (match_operand:QI 1 "gpc_reg_operand" "r")))]
-  "TARGET_POWERPC64"
-  "extsb %0,%1"
-  [(set_attr "type" "exts")])
-
-(define_insn ""
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC (sign_extend:DI (match_operand:QI 1 "gpc_reg_operand" "r,r"))
+(define_insn_and_split "*zero_extendsi<mode>2_dot"
+  [(set (match_operand:CC 2 "cc_reg_operand" "=x,?y")
+       (compare:CC (zero_extend:EXTSI (match_operand:SI 1 "gpc_reg_operand" "r,r"))
                    (const_int 0)))
-   (clobber (match_scratch:DI 2 "=r,r"))]
-  "TARGET_64BIT"
+   (clobber (match_scratch:EXTSI 0 "=r,r"))]
+  "rs6000_gen_cell_microcode"
   "@
-   extsb. %2,%1
+   rldicl. %0,%1,0,32
    #"
-  [(set_attr "type" "compare")
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[2], CCmode)"
+  [(set (match_dup 0)
+       (zero_extend:DI (match_dup 1)))
+   (set (match_dup 2)
+       (compare:CC (match_dup 0)
+                   (const_int 0)))]
+  ""
+  [(set_attr "type" "shift")
+   (set_attr "dot" "yes")
    (set_attr "length" "4,8")])
 
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (sign_extend:DI (match_operand:QI 1 "gpc_reg_operand" ""))
+(define_insn_and_split "*zero_extendsi<mode>2_dot2"
+  [(set (match_operand:CC 2 "cc_reg_operand" "=x,?y")
+       (compare:CC (zero_extend:EXTSI (match_operand:SI 1 "gpc_reg_operand" "r,r"))
                    (const_int 0)))
-   (clobber (match_scratch:DI 2 ""))]
-  "TARGET_POWERPC64 && reload_completed"
-  [(set (match_dup 2)
-       (sign_extend:DI (match_dup 1)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 2)
+   (set (match_operand:EXTSI 0 "gpc_reg_operand" "=r,r")
+       (zero_extend:EXTSI (match_dup 1)))]
+  "rs6000_gen_cell_microcode"
+  "@
+   rldicl. %0,%1,0,32
+   #"
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[2], CCmode)"
+  [(set (match_dup 0)
+       (zero_extend:EXTSI (match_dup 1)))
+   (set (match_dup 2)
+       (compare:CC (match_dup 0)
                    (const_int 0)))]
-  "")
+  ""
+  [(set_attr "type" "shift")
+   (set_attr "dot" "yes")
+   (set_attr "length" "4,8")])
 
-(define_insn ""
+
+(define_insn "extendqi<mode>2"
+  [(set (match_operand:EXTQI 0 "gpc_reg_operand" "=r")
+       (sign_extend:EXTQI (match_operand:QI 1 "gpc_reg_operand" "r")))]
+  ""
+  "extsb %0,%1"
+  [(set_attr "type" "exts")])
+
+(define_insn_and_split "*extendqi<mode>2_dot"
   [(set (match_operand:CC 2 "cc_reg_operand" "=x,?y")
-       (compare:CC (sign_extend:DI (match_operand:QI 1 "gpc_reg_operand" "r,r"))
+       (compare:CC (sign_extend:EXTQI (match_operand:QI 1 "gpc_reg_operand" "r,r"))
                    (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
-       (sign_extend:DI (match_dup 1)))]
-  "TARGET_64BIT"
+   (clobber (match_scratch:EXTQI 0 "=r,r"))]
+  "rs6000_gen_cell_microcode"
   "@
    extsb. %0,%1
    #"
-  [(set_attr "type" "compare")
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[2], CCmode)"
+  [(set (match_dup 0)
+       (sign_extend:EXTQI (match_dup 1)))
+   (set (match_dup 2)
+       (compare:CC (match_dup 0)
+                   (const_int 0)))]
+  ""
+  [(set_attr "type" "exts")
+   (set_attr "dot" "yes")
    (set_attr "length" "4,8")])
 
-(define_split
-  [(set (match_operand:CC 2 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (sign_extend:DI (match_operand:QI 1 "gpc_reg_operand" ""))
+(define_insn_and_split "*extendqi<mode>2_dot2"
+  [(set (match_operand:CC 2 "cc_reg_operand" "=x,?y")
+       (compare:CC (sign_extend:EXTQI (match_operand:QI 1 "gpc_reg_operand" "r,r"))
                    (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "")
-       (sign_extend:DI (match_dup 1)))]
-  "TARGET_POWERPC64 && reload_completed"
+   (set (match_operand:EXTQI 0 "gpc_reg_operand" "=r,r")
+       (sign_extend:EXTQI (match_dup 1)))]
+  "rs6000_gen_cell_microcode"
+  "@
+   extsb. %0,%1
+   #"
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[2], CCmode)"
   [(set (match_dup 0)
-       (sign_extend:DI (match_dup 1)))
+       (sign_extend:EXTQI (match_dup 1)))
    (set (match_dup 2)
        (compare:CC (match_dup 0)
                    (const_int 0)))]
-  "")
+  ""
+  [(set_attr "type" "exts")
+   (set_attr "dot" "yes")
+   (set_attr "length" "4,8")])
 
-(define_expand "extendhidi2"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "")
-       (sign_extend:DI (match_operand:HI 1 "gpc_reg_operand" "")))]
-  "TARGET_POWERPC64"
+
+(define_expand "extendhi<mode>2"
+  [(set (match_operand:EXTHI 0 "gpc_reg_operand" "")
+       (sign_extend:EXTHI (match_operand:HI 1 "gpc_reg_operand" "")))]
+  ""
   "")
 
-(define_insn ""
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
-       (sign_extend:DI (match_operand:HI 1 "reg_or_mem_operand" "m,r")))]
-  "TARGET_POWERPC64 && rs6000_gen_cell_microcode"
+(define_insn "*extendhi<mode>2"
+  [(set (match_operand:EXTHI 0 "gpc_reg_operand" "=r,r")
+       (sign_extend:EXTHI (match_operand:HI 1 "reg_or_mem_operand" "m,r")))]
+  "rs6000_gen_cell_microcode"
   "@
    lha%U1%X1 %0,%1
    extsh %0,%1"
   [(set_attr "type" "load,exts")
    (set_attr "sign_extend" "yes")])
 
-(define_insn ""
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
-       (sign_extend:DI (match_operand:HI 1 "gpc_reg_operand" "r")))]
-  "TARGET_POWERPC64 && !rs6000_gen_cell_microcode"
+(define_insn "*extendhi<mode>2_noload"
+  [(set (match_operand:EXTHI 0 "gpc_reg_operand" "=r")
+        (sign_extend:EXTHI (match_operand:HI 1 "gpc_reg_operand" "r")))]
+  "!rs6000_gen_cell_microcode"
   "extsh %0,%1"
   [(set_attr "type" "exts")])
 
-(define_insn ""
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC (sign_extend:DI (match_operand:HI 1 "gpc_reg_operand" "r,r"))
+(define_insn_and_split "*extendhi<mode>2_dot"
+  [(set (match_operand:CC 2 "cc_reg_operand" "=x,?y")
+       (compare:CC (sign_extend:EXTHI (match_operand:HI 1 "gpc_reg_operand" "r,r"))
                    (const_int 0)))
-   (clobber (match_scratch:DI 2 "=r,r"))]
-  "TARGET_64BIT"
+   (clobber (match_scratch:EXTHI 0 "=r,r"))]
+  "rs6000_gen_cell_microcode"
   "@
-   extsh. %2,%1
+   extsh. %0,%1
    #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (sign_extend:DI (match_operand:HI 1 "gpc_reg_operand" ""))
-                   (const_int 0)))
-   (clobber (match_scratch:DI 2 ""))]
-  "TARGET_POWERPC64 && reload_completed"
-  [(set (match_dup 2)
-       (sign_extend:DI (match_dup 1)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 2)
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[2], CCmode)"
+  [(set (match_dup 0)
+       (sign_extend:EXTHI (match_dup 1)))
+   (set (match_dup 2)
+       (compare:CC (match_dup 0)
                    (const_int 0)))]
-  "")
+  ""
+  [(set_attr "type" "exts")
+   (set_attr "dot" "yes")
+   (set_attr "length" "4,8")])
 
-(define_insn ""
+(define_insn_and_split "*extendhi<mode>2_dot2"
   [(set (match_operand:CC 2 "cc_reg_operand" "=x,?y")
-       (compare:CC (sign_extend:DI (match_operand:HI 1 "gpc_reg_operand" "r,r"))
+       (compare:CC (sign_extend:EXTHI (match_operand:HI 1 "gpc_reg_operand" "r,r"))
                    (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
-       (sign_extend:DI (match_dup 1)))]
-  "TARGET_64BIT"
+   (set (match_operand:EXTHI 0 "gpc_reg_operand" "=r,r")
+       (sign_extend:EXTHI (match_dup 1)))]
+  "rs6000_gen_cell_microcode"
   "@
    extsh. %0,%1
    #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 2 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (sign_extend:DI (match_operand:HI 1 "gpc_reg_operand" ""))
-                   (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "")
-       (sign_extend:DI (match_dup 1)))]
-  "TARGET_POWERPC64 && reload_completed"
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[2], CCmode)"
   [(set (match_dup 0)
-       (sign_extend:DI (match_dup 1)))
+       (sign_extend:EXTHI (match_dup 1)))
    (set (match_dup 2)
        (compare:CC (match_dup 0)
                    (const_int 0)))]
-  "")
+  ""
+  [(set_attr "type" "exts")
+   (set_attr "dot" "yes")
+   (set_attr "length" "4,8")])
 
-(define_expand "extendsidi2"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "")
-       (sign_extend:DI (match_operand:SI 1 "gpc_reg_operand" "")))]
-  "TARGET_POWERPC64"
-  "")
 
-(define_insn "*extendsidi2_lfiwax"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r,r,??wm,!wl,!wu")
-       (sign_extend:DI (match_operand:SI 1 "lwa_operand" "Y,r,r,Z,Z")))]
-  "TARGET_POWERPC64 && TARGET_LFIWAX"
+(define_insn "extendsi<mode>2"
+  [(set (match_operand:EXTSI 0 "gpc_reg_operand" "=r,r,??wj,!wl,!wu")
+       (sign_extend:EXTSI (match_operand:SI 1 "lwa_operand" "Y,r,r,Z,Z")))]
+  ""
   "@
    lwa%U1%X1 %0,%1
    extsw %0,%1
   [(set_attr "type" "load,exts,mffgpr,fpload,fpload")
    (set_attr "sign_extend" "yes")])
 
-(define_insn "*extendsidi2_nocell"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
-       (sign_extend:DI (match_operand:SI 1 "lwa_operand" "Y,r")))]
-  "TARGET_POWERPC64 && rs6000_gen_cell_microcode && !TARGET_LFIWAX"
-  "@
-   lwa%U1%X1 %0,%1
-   extsw %0,%1"
-  [(set_attr "type" "load,exts")
-   (set_attr "sign_extend" "yes")])
-
-(define_insn "*extendsidi2_nocell"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
-       (sign_extend:DI (match_operand:SI 1 "gpc_reg_operand" "r")))]
-  "TARGET_POWERPC64 && !rs6000_gen_cell_microcode"
-  "extsw %0,%1"
-  [(set_attr "type" "exts")])
-
-(define_insn ""
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC (sign_extend:DI (match_operand:SI 1 "gpc_reg_operand" "r,r"))
+(define_insn_and_split "*extendsi<mode>2_dot"
+  [(set (match_operand:CC 2 "cc_reg_operand" "=x,?y")
+       (compare:CC (sign_extend:EXTSI (match_operand:SI 1 "gpc_reg_operand" "r,r"))
                    (const_int 0)))
-   (clobber (match_scratch:DI 2 "=r,r"))]
-  "TARGET_64BIT"
+   (clobber (match_scratch:EXTSI 0 "=r,r"))]
+  "rs6000_gen_cell_microcode"
   "@
-   extsw. %2,%1
+   extsw. %0,%1
    #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (sign_extend:DI (match_operand:SI 1 "gpc_reg_operand" ""))
-                   (const_int 0)))
-   (clobber (match_scratch:DI 2 ""))]
-  "TARGET_POWERPC64 && reload_completed"
-  [(set (match_dup 2)
-       (sign_extend:DI (match_dup 1)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 2)
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[2], CCmode)"
+  [(set (match_dup 0)
+       (sign_extend:EXTSI (match_dup 1)))
+   (set (match_dup 2)
+       (compare:CC (match_dup 0)
                    (const_int 0)))]
-  "")
+  ""
+  [(set_attr "type" "exts")
+   (set_attr "dot" "yes")
+   (set_attr "length" "4,8")])
 
-(define_insn ""
+(define_insn_and_split "*extendsi<mode>2_dot2"
   [(set (match_operand:CC 2 "cc_reg_operand" "=x,?y")
-       (compare:CC (sign_extend:DI (match_operand:SI 1 "gpc_reg_operand" "r,r"))
+       (compare:CC (sign_extend:EXTSI (match_operand:SI 1 "gpc_reg_operand" "r,r"))
                    (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
-       (sign_extend:DI (match_dup 1)))]
-  "TARGET_64BIT"
+   (set (match_operand:EXTSI 0 "gpc_reg_operand" "=r,r")
+       (sign_extend:EXTSI (match_dup 1)))]
+  "rs6000_gen_cell_microcode"
   "@
    extsw. %0,%1
    #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 2 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (sign_extend:DI (match_operand:SI 1 "gpc_reg_operand" ""))
-                   (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "")
-       (sign_extend:DI (match_dup 1)))]
-  "TARGET_POWERPC64 && reload_completed"
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[2], CCmode)"
   [(set (match_dup 0)
-       (sign_extend:DI (match_dup 1)))
+       (sign_extend:EXTSI (match_dup 1)))
    (set (match_dup 2)
        (compare:CC (match_dup 0)
                    (const_int 0)))]
-  "")
-
-(define_expand "zero_extendqisi2"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "")
-       (zero_extend:SI (match_operand:QI 1 "gpc_reg_operand" "")))]
   ""
-  "")
-
-(define_insn ""
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
-       (zero_extend:SI (match_operand:QI 1 "reg_or_mem_operand" "m,r")))]
-  ""
-  "@
-   lbz%U1%X1 %0,%1
-   rlwinm %0,%1,0,0xff"
-  [(set_attr "type" "load,shift")])
-
-(define_insn ""
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC (zero_extend:SI (match_operand:QI 1 "gpc_reg_operand" "r,r"))
-                   (const_int 0)))
-   (clobber (match_scratch:SI 2 "=r,r"))]
-  ""
-  "@
-   andi. %2,%1,0xff
-   #"
-  [(set_attr "type" "logical,compare")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:SI (match_operand:QI 1 "gpc_reg_operand" ""))
-                   (const_int 0)))
-   (clobber (match_scratch:SI 2 ""))]
-  "reload_completed"
-  [(set (match_dup 2)
-       (zero_extend:SI (match_dup 1)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 2)
-                   (const_int 0)))]
-  "")
-
-(define_insn ""
-  [(set (match_operand:CC 2 "cc_reg_operand" "=x,?y")
-       (compare:CC (zero_extend:SI (match_operand:QI 1 "gpc_reg_operand" "r,r"))
-                   (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
-       (zero_extend:SI (match_dup 1)))]
-  ""
-  "@
-   andi. %0,%1,0xff
-   #"
-  [(set_attr "type" "logical,compare")
+  [(set_attr "type" "exts")
    (set_attr "dot" "yes")
    (set_attr "length" "4,8")])
+\f
+;; IBM 405, 440, 464 and 476 half-word multiplication operations.
 
-(define_split
-  [(set (match_operand:CC 2 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:SI (match_operand:QI 1 "gpc_reg_operand" ""))
-                   (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (zero_extend:SI (match_dup 1)))]
-  "reload_completed"
-  [(set (match_dup 0)
-       (zero_extend:SI (match_dup 1)))
-   (set (match_dup 2)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
+(define_insn "*macchwc"
+  [(set (match_operand:CC 3 "cc_reg_operand" "=x")
+        (compare:CC (plus:SI (mult:SI (ashiftrt:SI
+                                       (match_operand:SI 2 "gpc_reg_operand" "r")
+                                       (const_int 16))
+                                      (sign_extend:SI
+                                       (match_operand:HI 1 "gpc_reg_operand" "r")))
+                             (match_operand:SI 4 "gpc_reg_operand" "0"))
+                    (const_int 0)))
+   (set (match_operand:SI 0 "gpc_reg_operand" "=r")
+        (plus:SI (mult:SI (ashiftrt:SI
+                           (match_dup 2)
+                           (const_int 16))
+                          (sign_extend:SI
+                           (match_dup 1)))
+                 (match_dup 4)))]
+  "TARGET_MULHW"
+  "macchw. %0,%1,%2"
+  [(set_attr "type" "halfmul")])
 
-(define_insn "extendqisi2"
+(define_insn "*macchw"
   [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-       (sign_extend:SI (match_operand:QI 1 "gpc_reg_operand" "r")))]
-  ""
-  "extsb %0,%1"
-  [(set_attr "type" "exts")])
+        (plus:SI (mult:SI (ashiftrt:SI
+                           (match_operand:SI 2 "gpc_reg_operand" "r")
+                           (const_int 16))
+                          (sign_extend:SI
+                           (match_operand:HI 1 "gpc_reg_operand" "r")))
+                 (match_operand:SI 3 "gpc_reg_operand" "0")))]
+  "TARGET_MULHW"
+  "macchw %0,%1,%2"
+  [(set_attr "type" "halfmul")])
 
-(define_insn ""
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC (sign_extend:SI (match_operand:QI 1 "gpc_reg_operand" "r,r"))
-                   (const_int 0)))
-   (clobber (match_scratch:SI 2 "=r,r"))]
-  ""
-  "@
-   extsb. %2,%1
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "4,8")])
+(define_insn "*macchwuc"
+  [(set (match_operand:CC 3 "cc_reg_operand" "=x")
+        (compare:CC (plus:SI (mult:SI (lshiftrt:SI
+                                       (match_operand:SI 2 "gpc_reg_operand" "r")
+                                       (const_int 16))
+                                      (zero_extend:SI
+                                       (match_operand:HI 1 "gpc_reg_operand" "r")))
+                             (match_operand:SI 4 "gpc_reg_operand" "0"))
+                    (const_int 0)))
+   (set (match_operand:SI 0 "gpc_reg_operand" "=r")
+        (plus:SI (mult:SI (lshiftrt:SI
+                           (match_dup 2)
+                           (const_int 16))
+                          (zero_extend:SI
+                           (match_dup 1)))
+                 (match_dup 4)))]
+  "TARGET_MULHW"
+  "macchwu. %0,%1,%2"
+  [(set_attr "type" "halfmul")])
 
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (sign_extend:SI (match_operand:QI 1 "gpc_reg_operand" ""))
-                   (const_int 0)))
-   (clobber (match_scratch:SI 2 ""))]
-  "reload_completed"
-  [(set (match_dup 2)
-       (sign_extend:SI (match_dup 1)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 2)
-                   (const_int 0)))]
-  "")
+(define_insn "*macchwu"
+  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
+        (plus:SI (mult:SI (lshiftrt:SI
+                           (match_operand:SI 2 "gpc_reg_operand" "r")
+                           (const_int 16))
+                          (zero_extend:SI
+                           (match_operand:HI 1 "gpc_reg_operand" "r")))
+                 (match_operand:SI 3 "gpc_reg_operand" "0")))]
+  "TARGET_MULHW"
+  "macchwu %0,%1,%2"
+  [(set_attr "type" "halfmul")])
 
-(define_insn ""
-  [(set (match_operand:CC 2 "cc_reg_operand" "=x,?y")
-       (compare:CC (sign_extend:SI (match_operand:QI 1 "gpc_reg_operand" "r,r"))
-                   (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
-       (sign_extend:SI (match_dup 1)))]
-  ""
-  "@
-   extsb. %0,%1
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "4,8")])
+(define_insn "*machhwc"
+  [(set (match_operand:CC 3 "cc_reg_operand" "=x")
+        (compare:CC (plus:SI (mult:SI (ashiftrt:SI
+                                       (match_operand:SI 1 "gpc_reg_operand" "%r")
+                                       (const_int 16))
+                                      (ashiftrt:SI
+                                       (match_operand:SI 2 "gpc_reg_operand" "r")
+                                       (const_int 16)))
+                             (match_operand:SI 4 "gpc_reg_operand" "0"))
+                    (const_int 0)))
+   (set (match_operand:SI 0 "gpc_reg_operand" "=r")
+        (plus:SI (mult:SI (ashiftrt:SI
+                           (match_dup 1)
+                           (const_int 16))
+                          (ashiftrt:SI
+                           (match_dup 2)
+                           (const_int 16)))
+                 (match_dup 4)))]
+  "TARGET_MULHW"
+  "machhw. %0,%1,%2"
+  [(set_attr "type" "halfmul")])
 
-(define_split
-  [(set (match_operand:CC 2 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (sign_extend:SI (match_operand:QI 1 "gpc_reg_operand" ""))
-                   (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (sign_extend:SI (match_dup 1)))]
-  "reload_completed"
-  [(set (match_dup 0)
-       (sign_extend:SI (match_dup 1)))
-   (set (match_dup 2)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
+(define_insn "*machhw"
+  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
+        (plus:SI (mult:SI (ashiftrt:SI
+                           (match_operand:SI 1 "gpc_reg_operand" "%r")
+                           (const_int 16))
+                          (ashiftrt:SI
+                           (match_operand:SI 2 "gpc_reg_operand" "r")
+                           (const_int 16)))
+                 (match_operand:SI 3 "gpc_reg_operand" "0")))]
+  "TARGET_MULHW"
+  "machhw %0,%1,%2"
+  [(set_attr "type" "halfmul")])
 
-(define_insn ""
-  [(set (match_operand:HI 0 "gpc_reg_operand" "=r,r")
-       (zero_extend:HI (match_operand:QI 1 "reg_or_mem_operand" "m,r")))]
-  ""
-  "@
-   lbz%U1%X1 %0,%1
-   rlwinm %0,%1,0,0xff"
-  [(set_attr "type" "load,shift")])
+(define_insn "*machhwuc"
+  [(set (match_operand:CC 3 "cc_reg_operand" "=x")
+        (compare:CC (plus:SI (mult:SI (lshiftrt:SI
+                                       (match_operand:SI 1 "gpc_reg_operand" "%r")
+                                       (const_int 16))
+                                      (lshiftrt:SI
+                                       (match_operand:SI 2 "gpc_reg_operand" "r")
+                                       (const_int 16)))
+                             (match_operand:SI 4 "gpc_reg_operand" "0"))
+                    (const_int 0)))
+   (set (match_operand:SI 0 "gpc_reg_operand" "=r")
+        (plus:SI (mult:SI (lshiftrt:SI
+                           (match_dup 1)
+                           (const_int 16))
+                          (lshiftrt:SI
+                           (match_dup 2)
+                           (const_int 16)))
+                 (match_dup 4)))]
+  "TARGET_MULHW"
+  "machhwu. %0,%1,%2"
+  [(set_attr "type" "halfmul")])
 
-(define_insn ""
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC (zero_extend:HI (match_operand:QI 1 "gpc_reg_operand" "r,r"))
-                   (const_int 0)))
-   (clobber (match_scratch:HI 2 "=r,r"))]
-  ""
-  "@
-   andi. %2,%1,0xff
-   #"
-  [(set_attr "type" "logical,compare")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
+(define_insn "*machhwu"
+  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
+        (plus:SI (mult:SI (lshiftrt:SI
+                           (match_operand:SI 1 "gpc_reg_operand" "%r")
+                           (const_int 16))
+                          (lshiftrt:SI
+                           (match_operand:SI 2 "gpc_reg_operand" "r")
+                           (const_int 16)))
+                 (match_operand:SI 3 "gpc_reg_operand" "0")))]
+  "TARGET_MULHW"
+  "machhwu %0,%1,%2"
+  [(set_attr "type" "halfmul")])
 
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:HI (match_operand:QI 1 "gpc_reg_operand" ""))
-                   (const_int 0)))
-   (clobber (match_scratch:HI 2 ""))]
-  "reload_completed"
-  [(set (match_dup 2)
-       (zero_extend:HI (match_dup 1)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 2)
-                   (const_int 0)))]
-  "")
+(define_insn "*maclhwc"
+  [(set (match_operand:CC 3 "cc_reg_operand" "=x")
+        (compare:CC (plus:SI (mult:SI (sign_extend:SI
+                                       (match_operand:HI 1 "gpc_reg_operand" "%r"))
+                                      (sign_extend:SI
+                                       (match_operand:HI 2 "gpc_reg_operand" "r")))
+                             (match_operand:SI 4 "gpc_reg_operand" "0"))
+                    (const_int 0)))
+   (set (match_operand:SI 0 "gpc_reg_operand" "=r")
+        (plus:SI (mult:SI (sign_extend:SI
+                           (match_dup 1))
+                          (sign_extend:SI
+                           (match_dup 2)))
+                 (match_dup 4)))]
+  "TARGET_MULHW"
+  "maclhw. %0,%1,%2"
+  [(set_attr "type" "halfmul")])
 
-(define_insn ""
-  [(set (match_operand:CC 2 "cc_reg_operand" "=x,?y")
-       (compare:CC (zero_extend:HI (match_operand:QI 1 "gpc_reg_operand" "r,r"))
-                   (const_int 0)))
-   (set (match_operand:HI 0 "gpc_reg_operand" "=r,r")
-       (zero_extend:HI (match_dup 1)))]
-  ""
-  "@
-   andi. %0,%1,0xff
-   #"
-  [(set_attr "type" "logical,compare")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
+(define_insn "*maclhw"
+  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
+        (plus:SI (mult:SI (sign_extend:SI
+                           (match_operand:HI 1 "gpc_reg_operand" "%r"))
+                          (sign_extend:SI
+                           (match_operand:HI 2 "gpc_reg_operand" "r")))
+                 (match_operand:SI 3 "gpc_reg_operand" "0")))]
+  "TARGET_MULHW"
+  "maclhw %0,%1,%2"
+  [(set_attr "type" "halfmul")])
 
-(define_split
-  [(set (match_operand:CC 2 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:HI (match_operand:QI 1 "gpc_reg_operand" ""))
-                   (const_int 0)))
-   (set (match_operand:HI 0 "gpc_reg_operand" "")
-       (zero_extend:HI (match_dup 1)))]
-  "reload_completed"
-  [(set (match_dup 0)
-       (zero_extend:HI (match_dup 1)))
-   (set (match_dup 2)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
-
-(define_insn "extendqihi2"
-  [(set (match_operand:HI 0 "gpc_reg_operand" "=r")
-       (sign_extend:HI (match_operand:QI 1 "gpc_reg_operand" "r")))]
-  ""
-  "extsb %0,%1"
-  [(set_attr "type" "exts")])
-
-(define_insn ""
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC (sign_extend:HI (match_operand:QI 1 "gpc_reg_operand" "r,r"))
-                   (const_int 0)))
-   (clobber (match_scratch:HI 2 "=r,r"))]
-  ""
-  "@
-   extsb. %2,%1
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (sign_extend:HI (match_operand:QI 1 "gpc_reg_operand" ""))
-                   (const_int 0)))
-   (clobber (match_scratch:HI 2 ""))]
-  "reload_completed"
-  [(set (match_dup 2)
-       (sign_extend:HI (match_dup 1)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 2)
-                   (const_int 0)))]
-  "")
-
-(define_insn ""
-  [(set (match_operand:CC 2 "cc_reg_operand" "=x,?y")
-       (compare:CC (sign_extend:HI (match_operand:QI 1 "gpc_reg_operand" "r,r"))
-                   (const_int 0)))
-   (set (match_operand:HI 0 "gpc_reg_operand" "=r,r")
-       (sign_extend:HI (match_dup 1)))]
-  ""
-  "@
-   extsb. %0,%1
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 2 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (sign_extend:HI (match_operand:QI 1 "gpc_reg_operand" ""))
-                   (const_int 0)))
-   (set (match_operand:HI 0 "gpc_reg_operand" "")
-       (sign_extend:HI (match_dup 1)))]
-  "reload_completed"
-  [(set (match_dup 0)
-       (sign_extend:HI (match_dup 1)))
-   (set (match_dup 2)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
-
-(define_expand "zero_extendhisi2"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "")
-       (zero_extend:SI (match_operand:HI 1 "gpc_reg_operand" "")))]
-  ""
-  "")
-
-(define_insn ""
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
-       (zero_extend:SI (match_operand:HI 1 "reg_or_mem_operand" "m,r")))]
-  ""
-  "@
-   lhz%U1%X1 %0,%1
-   rlwinm %0,%1,0,0xffff"
-  [(set_attr "type" "load,shift")])
-
-(define_insn ""
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC (zero_extend:SI (match_operand:HI 1 "gpc_reg_operand" "r,r"))
-                   (const_int 0)))
-   (clobber (match_scratch:SI 2 "=r,r"))]
-  ""
-  "@
-   andi. %2,%1,0xffff
-   #"
-  [(set_attr "type" "logical,compare")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:SI (match_operand:HI 1 "gpc_reg_operand" ""))
-                   (const_int 0)))
-   (clobber (match_scratch:SI 2 ""))]
-  "reload_completed"
-  [(set (match_dup 2)
-       (zero_extend:SI (match_dup 1)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 2)
-                   (const_int 0)))]
-  "")
-
-(define_insn ""
-  [(set (match_operand:CC 2 "cc_reg_operand" "=x,?y")
-       (compare:CC (zero_extend:SI (match_operand:HI 1 "gpc_reg_operand" "r,r"))
-                   (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
-       (zero_extend:SI (match_dup 1)))]
-  ""
-  "@
-   andi. %0,%1,0xffff
-   #"
-  [(set_attr "type" "logical,compare")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 2 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:SI (match_operand:HI 1 "gpc_reg_operand" ""))
-                   (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (zero_extend:SI (match_dup 1)))]
-  "reload_completed"
-  [(set (match_dup 0)
-       (zero_extend:SI (match_dup 1)))
-   (set (match_dup 2)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
-
-(define_expand "extendhisi2"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "")
-       (sign_extend:SI (match_operand:HI 1 "gpc_reg_operand" "")))]
-  ""
-  "")
-
-(define_insn ""
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
-       (sign_extend:SI (match_operand:HI 1 "reg_or_mem_operand" "m,r")))]
-  "rs6000_gen_cell_microcode"
-  "@
-   lha%U1%X1 %0,%1
-   extsh %0,%1"
-  [(set_attr "type" "load,exts")
-   (set_attr "sign_extend" "yes")])
-
-(define_insn ""
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-        (sign_extend:SI (match_operand:HI 1 "gpc_reg_operand" "r")))]
-  "!rs6000_gen_cell_microcode"
-  "extsh %0,%1"
-  [(set_attr "type" "exts")])
-
-(define_insn ""
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC (sign_extend:SI (match_operand:HI 1 "gpc_reg_operand" "r,r"))
-                   (const_int 0)))
-   (clobber (match_scratch:SI 2 "=r,r"))]
-  ""
-  "@
-   extsh. %2,%1
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (sign_extend:SI (match_operand:HI 1 "gpc_reg_operand" ""))
-                   (const_int 0)))
-   (clobber (match_scratch:SI 2 ""))]
-  "reload_completed"
-  [(set (match_dup 2)
-       (sign_extend:SI (match_dup 1)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 2)
-                   (const_int 0)))]
-  "")
-
-(define_insn ""
-  [(set (match_operand:CC 2 "cc_reg_operand" "=x,?y")
-       (compare:CC (sign_extend:SI (match_operand:HI 1 "gpc_reg_operand" "r,r"))
-                   (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
-       (sign_extend:SI (match_dup 1)))]
-  ""
-  "@
-   extsh. %0,%1
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 2 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (sign_extend:SI (match_operand:HI 1 "gpc_reg_operand" ""))
-                   (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (sign_extend:SI (match_dup 1)))]
-  "reload_completed"
-  [(set (match_dup 0)
-       (sign_extend:SI (match_dup 1)))
-   (set (match_dup 2)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
-\f
-;; IBM 405, 440, 464 and 476 half-word multiplication operations.
-
-(define_insn "*macchwc"
+(define_insn "*maclhwuc"
   [(set (match_operand:CC 3 "cc_reg_operand" "=x")
-        (compare:CC (plus:SI (mult:SI (ashiftrt:SI
-                                       (match_operand:SI 2 "gpc_reg_operand" "r")
-                                       (const_int 16))
-                                      (sign_extend:SI
-                                       (match_operand:HI 1 "gpc_reg_operand" "r")))
+        (compare:CC (plus:SI (mult:SI (zero_extend:SI
+                                       (match_operand:HI 1 "gpc_reg_operand" "%r"))
+                                      (zero_extend:SI
+                                       (match_operand:HI 2 "gpc_reg_operand" "r")))
                              (match_operand:SI 4 "gpc_reg_operand" "0"))
                     (const_int 0)))
    (set (match_operand:SI 0 "gpc_reg_operand" "=r")
-        (plus:SI (mult:SI (ashiftrt:SI
-                           (match_dup 2)
-                           (const_int 16))
-                          (sign_extend:SI
-                           (match_dup 1)))
+        (plus:SI (mult:SI (zero_extend:SI
+                           (match_dup 1))
+                          (zero_extend:SI
+                           (match_dup 2)))
                  (match_dup 4)))]
   "TARGET_MULHW"
-  "macchw. %0,%1,%2"
+  "maclhwu. %0,%1,%2"
   [(set_attr "type" "halfmul")])
 
-(define_insn "*macchw"
+(define_insn "*maclhwu"
   [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-        (plus:SI (mult:SI (ashiftrt:SI
-                           (match_operand:SI 2 "gpc_reg_operand" "r")
-                           (const_int 16))
-                          (sign_extend:SI
-                           (match_operand:HI 1 "gpc_reg_operand" "r")))
+        (plus:SI (mult:SI (zero_extend:SI
+                           (match_operand:HI 1 "gpc_reg_operand" "%r"))
+                          (zero_extend:SI
+                           (match_operand:HI 2 "gpc_reg_operand" "r")))
                  (match_operand:SI 3 "gpc_reg_operand" "0")))]
   "TARGET_MULHW"
-  "macchw %0,%1,%2"
+  "maclhwu %0,%1,%2"
   [(set_attr "type" "halfmul")])
 
-(define_insn "*macchwuc"
+(define_insn "*nmacchwc"
   [(set (match_operand:CC 3 "cc_reg_operand" "=x")
-        (compare:CC (plus:SI (mult:SI (lshiftrt:SI
-                                       (match_operand:SI 2 "gpc_reg_operand" "r")
-                                       (const_int 16))
-                                      (zero_extend:SI
-                                       (match_operand:HI 1 "gpc_reg_operand" "r")))
-                             (match_operand:SI 4 "gpc_reg_operand" "0"))
+        (compare:CC (minus:SI (match_operand:SI 4 "gpc_reg_operand" "0")
+                              (mult:SI (ashiftrt:SI
+                                        (match_operand:SI 2 "gpc_reg_operand" "r")
+                                        (const_int 16))
+                                       (sign_extend:SI
+                                        (match_operand:HI 1 "gpc_reg_operand" "r"))))
                     (const_int 0)))
    (set (match_operand:SI 0 "gpc_reg_operand" "=r")
-        (plus:SI (mult:SI (lshiftrt:SI
-                           (match_dup 2)
-                           (const_int 16))
-                          (zero_extend:SI
-                           (match_dup 1)))
-                 (match_dup 4)))]
+        (minus:SI (match_dup 4)
+                  (mult:SI (ashiftrt:SI
+                            (match_dup 2)
+                            (const_int 16))
+                           (sign_extend:SI
+                            (match_dup 1)))))]
   "TARGET_MULHW"
-  "macchwu. %0,%1,%2"
+  "nmacchw. %0,%1,%2"
   [(set_attr "type" "halfmul")])
 
-(define_insn "*macchwu"
+(define_insn "*nmacchw"
   [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-        (plus:SI (mult:SI (lshiftrt:SI
-                           (match_operand:SI 2 "gpc_reg_operand" "r")
-                           (const_int 16))
-                          (zero_extend:SI
-                           (match_operand:HI 1 "gpc_reg_operand" "r")))
-                 (match_operand:SI 3 "gpc_reg_operand" "0")))]
+        (minus:SI (match_operand:SI 3 "gpc_reg_operand" "0")
+                  (mult:SI (ashiftrt:SI
+                            (match_operand:SI 2 "gpc_reg_operand" "r")
+                            (const_int 16))
+                           (sign_extend:SI
+                            (match_operand:HI 1 "gpc_reg_operand" "r")))))]
   "TARGET_MULHW"
-  "macchwu %0,%1,%2"
+  "nmacchw %0,%1,%2"
   [(set_attr "type" "halfmul")])
 
-(define_insn "*machhwc"
+(define_insn "*nmachhwc"
   [(set (match_operand:CC 3 "cc_reg_operand" "=x")
-        (compare:CC (plus:SI (mult:SI (ashiftrt:SI
-                                       (match_operand:SI 1 "gpc_reg_operand" "%r")
-                                       (const_int 16))
-                                      (ashiftrt:SI
-                                       (match_operand:SI 2 "gpc_reg_operand" "r")
-                                       (const_int 16)))
-                             (match_operand:SI 4 "gpc_reg_operand" "0"))
-                    (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=r")
-        (plus:SI (mult:SI (ashiftrt:SI
-                           (match_dup 1)
-                           (const_int 16))
-                          (ashiftrt:SI
-                           (match_dup 2)
-                           (const_int 16)))
-                 (match_dup 4)))]
-  "TARGET_MULHW"
-  "machhw. %0,%1,%2"
-  [(set_attr "type" "halfmul")])
-
-(define_insn "*machhw"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-        (plus:SI (mult:SI (ashiftrt:SI
-                           (match_operand:SI 1 "gpc_reg_operand" "%r")
-                           (const_int 16))
-                          (ashiftrt:SI
-                           (match_operand:SI 2 "gpc_reg_operand" "r")
-                           (const_int 16)))
-                 (match_operand:SI 3 "gpc_reg_operand" "0")))]
-  "TARGET_MULHW"
-  "machhw %0,%1,%2"
-  [(set_attr "type" "halfmul")])
-
-(define_insn "*machhwuc"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x")
-        (compare:CC (plus:SI (mult:SI (lshiftrt:SI
-                                       (match_operand:SI 1 "gpc_reg_operand" "%r")
-                                       (const_int 16))
-                                      (lshiftrt:SI
-                                       (match_operand:SI 2 "gpc_reg_operand" "r")
-                                       (const_int 16)))
-                             (match_operand:SI 4 "gpc_reg_operand" "0"))
-                    (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=r")
-        (plus:SI (mult:SI (lshiftrt:SI
-                           (match_dup 1)
-                           (const_int 16))
-                          (lshiftrt:SI
-                           (match_dup 2)
-                           (const_int 16)))
-                 (match_dup 4)))]
-  "TARGET_MULHW"
-  "machhwu. %0,%1,%2"
-  [(set_attr "type" "halfmul")])
-
-(define_insn "*machhwu"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-        (plus:SI (mult:SI (lshiftrt:SI
-                           (match_operand:SI 1 "gpc_reg_operand" "%r")
-                           (const_int 16))
-                          (lshiftrt:SI
-                           (match_operand:SI 2 "gpc_reg_operand" "r")
-                           (const_int 16)))
-                 (match_operand:SI 3 "gpc_reg_operand" "0")))]
-  "TARGET_MULHW"
-  "machhwu %0,%1,%2"
-  [(set_attr "type" "halfmul")])
-
-(define_insn "*maclhwc"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x")
-        (compare:CC (plus:SI (mult:SI (sign_extend:SI
-                                       (match_operand:HI 1 "gpc_reg_operand" "%r"))
-                                      (sign_extend:SI
-                                       (match_operand:HI 2 "gpc_reg_operand" "r")))
-                             (match_operand:SI 4 "gpc_reg_operand" "0"))
-                    (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=r")
-        (plus:SI (mult:SI (sign_extend:SI
-                           (match_dup 1))
-                          (sign_extend:SI
-                           (match_dup 2)))
-                 (match_dup 4)))]
-  "TARGET_MULHW"
-  "maclhw. %0,%1,%2"
-  [(set_attr "type" "halfmul")])
-
-(define_insn "*maclhw"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-        (plus:SI (mult:SI (sign_extend:SI
-                           (match_operand:HI 1 "gpc_reg_operand" "%r"))
-                          (sign_extend:SI
-                           (match_operand:HI 2 "gpc_reg_operand" "r")))
-                 (match_operand:SI 3 "gpc_reg_operand" "0")))]
-  "TARGET_MULHW"
-  "maclhw %0,%1,%2"
-  [(set_attr "type" "halfmul")])
-
-(define_insn "*maclhwuc"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x")
-        (compare:CC (plus:SI (mult:SI (zero_extend:SI
-                                       (match_operand:HI 1 "gpc_reg_operand" "%r"))
-                                      (zero_extend:SI
-                                       (match_operand:HI 2 "gpc_reg_operand" "r")))
-                             (match_operand:SI 4 "gpc_reg_operand" "0"))
-                    (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=r")
-        (plus:SI (mult:SI (zero_extend:SI
-                           (match_dup 1))
-                          (zero_extend:SI
-                           (match_dup 2)))
-                 (match_dup 4)))]
-  "TARGET_MULHW"
-  "maclhwu. %0,%1,%2"
-  [(set_attr "type" "halfmul")])
-
-(define_insn "*maclhwu"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-        (plus:SI (mult:SI (zero_extend:SI
-                           (match_operand:HI 1 "gpc_reg_operand" "%r"))
-                          (zero_extend:SI
-                           (match_operand:HI 2 "gpc_reg_operand" "r")))
-                 (match_operand:SI 3 "gpc_reg_operand" "0")))]
-  "TARGET_MULHW"
-  "maclhwu %0,%1,%2"
-  [(set_attr "type" "halfmul")])
-
-(define_insn "*nmacchwc"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x")
-        (compare:CC (minus:SI (match_operand:SI 4 "gpc_reg_operand" "0")
-                              (mult:SI (ashiftrt:SI
-                                        (match_operand:SI 2 "gpc_reg_operand" "r")
-                                        (const_int 16))
-                                       (sign_extend:SI
-                                        (match_operand:HI 1 "gpc_reg_operand" "r"))))
-                    (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=r")
-        (minus:SI (match_dup 4)
-                  (mult:SI (ashiftrt:SI
-                            (match_dup 2)
-                            (const_int 16))
-                           (sign_extend:SI
-                            (match_dup 1)))))]
-  "TARGET_MULHW"
-  "nmacchw. %0,%1,%2"
-  [(set_attr "type" "halfmul")])
-
-(define_insn "*nmacchw"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-        (minus:SI (match_operand:SI 3 "gpc_reg_operand" "0")
-                  (mult:SI (ashiftrt:SI
-                            (match_operand:SI 2 "gpc_reg_operand" "r")
-                            (const_int 16))
-                           (sign_extend:SI
-                            (match_operand:HI 1 "gpc_reg_operand" "r")))))]
-  "TARGET_MULHW"
-  "nmacchw %0,%1,%2"
-  [(set_attr "type" "halfmul")])
-
-(define_insn "*nmachhwc"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x")
-        (compare:CC (minus:SI (match_operand:SI 4 "gpc_reg_operand" "0")
-                              (mult:SI (ashiftrt:SI
-                                        (match_operand:SI 1 "gpc_reg_operand" "%r")
-                                        (const_int 16))
-                                       (ashiftrt:SI
-                                        (match_operand:SI 2 "gpc_reg_operand" "r")
-                                        (const_int 16))))
+        (compare:CC (minus:SI (match_operand:SI 4 "gpc_reg_operand" "0")
+                              (mult:SI (ashiftrt:SI
+                                        (match_operand:SI 1 "gpc_reg_operand" "%r")
+                                        (const_int 16))
+                                       (ashiftrt:SI
+                                        (match_operand:SI 2 "gpc_reg_operand" "r")
+                                        (const_int 16))))
                     (const_int 0)))
    (set (match_operand:SI 0 "gpc_reg_operand" "=r")
         (minus:SI (match_dup 4)
   cr0 = gen_rtx_REG (CCmode, CR0_REGNO);
   emit_insn (gen_dlmzb (scratch_dlmzb, word1, word2, cr0));
   cond = gen_rtx_NE (VOIDmode, cr0, const0_rtx);
-  emit_jump_insn (gen_rtx_SET (VOIDmode,
-                               pc_rtx,
+  emit_jump_insn (gen_rtx_SET (pc_rtx,
                                gen_rtx_IF_THEN_ELSE (VOIDmode,
                                                      cond,
                                                      gen_rtx_LABEL_REF
                                                         end_label),
                                                      pc_rtx)));
   emit_insn (gen_addsi3 (scratch_string, scratch_string, GEN_INT (8)));
-  emit_jump_insn (gen_rtx_SET (VOIDmode,
-                               pc_rtx,
+  emit_jump_insn (gen_rtx_SET (pc_rtx,
                                gen_rtx_LABEL_REF (VOIDmode, loop_label)));
   emit_barrier ();
   emit_label (end_label);
   emit_insn (gen_addsi3 (scratch_string, scratch_string, scratch_dlmzb));
   emit_insn (gen_subsi3 (result, scratch_string, addr));
-  emit_insn (gen_subsi3 (result, result, const1_rtx));
+  emit_insn (gen_addsi3 (result, result, constm1_rtx));
   DONE;
 })
 \f
                  (match_operand:SDI 2 "reg_or_add_cint_operand" "")))]
   ""
 {
-  if (<MODE>mode == DImode && ! TARGET_POWERPC64)
+  if (<MODE>mode == DImode && !TARGET_POWERPC64)
     {
-      if (non_short_cint_operand (operands[2], DImode))
-       FAIL;
+      rtx lo0 = gen_lowpart (SImode, operands[0]);
+      rtx lo1 = gen_lowpart (SImode, operands[1]);
+      rtx lo2 = gen_lowpart (SImode, operands[2]);
+      rtx hi0 = gen_highpart (SImode, operands[0]);
+      rtx hi1 = gen_highpart (SImode, operands[1]);
+      rtx hi2 = gen_highpart_mode (SImode, DImode, operands[2]);
+
+      if (!reg_or_short_operand (lo2, SImode))
+       lo2 = force_reg (SImode, lo2);
+      if (!adde_operand (hi2, SImode))
+       hi2 = force_reg (SImode, hi2);
+
+      emit_insn (gen_addsi3_carry (lo0, lo1, lo2));
+      emit_insn (gen_addsi3_carry_in (hi0, hi1, hi2));
+      DONE;
     }
-  else if (GET_CODE (operands[2]) == CONST_INT
-          && ! add_operand (operands[2], <MODE>mode))
+
+  if (CONST_INT_P (operands[2]) && !add_operand (operands[2], <MODE>mode))
     {
       rtx tmp = ((!can_create_pseudo_p ()
                  || rtx_equal_p (operands[0], operands[1]))
     }
 })
 
-;; Discourage ai/addic because of carry but provide it in an alternative
-;; allowing register zero as source.
-(define_insn "*add<mode>3_internal1"
-  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r,r,?r,r")
-       (plus:GPR (match_operand:GPR 1 "gpc_reg_operand" "%r,b,r,b")
-                 (match_operand:GPR 2 "add_operand" "r,I,I,L")))]
-  "!DECIMAL_FLOAT_MODE_P (GET_MODE (operands[0])) && !DECIMAL_FLOAT_MODE_P (GET_MODE (operands[1]))"
+(define_insn "*add<mode>3"
+  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r,r,r")
+       (plus:GPR (match_operand:GPR 1 "gpc_reg_operand" "%r,b,b")
+                 (match_operand:GPR 2 "add_operand" "r,I,L")))]
+  ""
   "@
    add %0,%1,%2
    addi %0,%1,%2
-   addic %0,%1,%2
    addis %0,%1,%v2"
   [(set_attr "type" "add")])
 
   "addis %0,%1,ha16(%2)"
   [(set_attr "type" "add")])
 
-(define_insn "*add<mode>3_internal2"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC (plus:P (match_operand:P 1 "gpc_reg_operand" "%r,r,r,r")
-                           (match_operand:P 2 "reg_or_short_operand" "r,I,r,I"))
+(define_insn_and_split "*add<mode>3_dot"
+  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
+       (compare:CC (plus:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r")
+                             (match_operand:GPR 2 "gpc_reg_operand" "r,r"))
                    (const_int 0)))
-   (clobber (match_scratch:P 3 "=r,r,r,r"))]
-  ""
+   (clobber (match_scratch:GPR 0 "=r,r"))]
+  "<MODE>mode == Pmode"
   "@
-   add. %3,%1,%2
-   addic. %3,%1,%2
-   #
+   add. %0,%1,%2
    #"
-  [(set_attr "type" "add,compare,compare,compare")
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
+  [(set (match_dup 0)
+       (plus:GPR (match_dup 1)
+                (match_dup 2)))
+   (set (match_dup 3)
+       (compare:CC (match_dup 0)
+                   (const_int 0)))]
+  ""
+  [(set_attr "type" "add")
    (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
+   (set_attr "length" "4,8")])
 
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_cr0_operand" "")
-       (compare:CC (plus:GPR (match_operand:GPR 1 "gpc_reg_operand" "")
-                             (match_operand:GPR 2 "reg_or_short_operand" ""))
+(define_insn_and_split "*add<mode>3_dot2"
+  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
+       (compare:CC (plus:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r")
+                             (match_operand:GPR 2 "gpc_reg_operand" "r,r"))
                    (const_int 0)))
-   (clobber (match_scratch:GPR 3 ""))]
-  "reload_completed"
-  [(set (match_dup 3)
+   (set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
        (plus:GPR (match_dup 1)
-                (match_dup 2)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
+                 (match_dup 2)))]
+  "<MODE>mode == Pmode"
+  "@
+   add. %0,%1,%2
+   #"
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
+  [(set (match_dup 0)
+       (plus:GPR (match_dup 1)
+                 (match_dup 2)))
+   (set (match_dup 3)
+       (compare:CC (match_dup 0)
                    (const_int 0)))]
-  "")
+  ""
+  [(set_attr "type" "add")
+   (set_attr "dot" "yes")
+   (set_attr "length" "4,8")])
 
-(define_insn "*add<mode>3_internal3"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC (plus:P (match_operand:P 1 "gpc_reg_operand" "%r,r,r,r")
-                           (match_operand:P 2 "reg_or_short_operand" "r,I,r,I"))
+(define_insn_and_split "*add<mode>3_imm_dot"
+  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
+       (compare:CC (plus:GPR (match_operand:GPR 1 "gpc_reg_operand" "%r,b")
+                             (match_operand:GPR 2 "short_cint_operand" "I,I"))
                    (const_int 0)))
-   (set (match_operand:P 0 "gpc_reg_operand" "=r,r,r,r")
-       (plus:P (match_dup 1)
-               (match_dup 2)))]
-  ""
+   (clobber (match_scratch:GPR 0 "=r,r"))
+   (clobber (reg:GPR CA_REGNO))]
+  "<MODE>mode == Pmode"
   "@
-   add. %0,%1,%2
    addic. %0,%1,%2
-   #
    #"
-  [(set_attr "type" "add,compare,compare,compare")
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
+  [(set (match_dup 0)
+       (plus:GPR (match_dup 1)
+                 (match_dup 2)))
+   (set (match_dup 3)
+       (compare:CC (match_dup 0)
+                   (const_int 0)))]
+  ""
+  [(set_attr "type" "add")
    (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
+   (set_attr "length" "4,8")])
 
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_cr0_operand" "")
-       (compare:CC (plus:P (match_operand:P 1 "gpc_reg_operand" "")
-                           (match_operand:P 2 "reg_or_short_operand" ""))
+(define_insn_and_split "*add<mode>3_imm_dot2"
+  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
+       (compare:CC (plus:GPR (match_operand:GPR 1 "gpc_reg_operand" "%r,b")
+                             (match_operand:GPR 2 "short_cint_operand" "I,I"))
                    (const_int 0)))
-   (set (match_operand:P 0 "gpc_reg_operand" "")
-       (plus:P (match_dup 1) (match_dup 2)))]
-  "reload_completed"
+   (set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
+       (plus:GPR (match_dup 1)
+                 (match_dup 2)))
+   (clobber (reg:GPR CA_REGNO))]
+  "<MODE>mode == Pmode"
+  "@
+   addic. %0,%1,%2
+   #"
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
   [(set (match_dup 0)
-       (plus:P (match_dup 1)
-               (match_dup 2)))
+       (plus:GPR (match_dup 1)
+                 (match_dup 2)))
    (set (match_dup 3)
        (compare:CC (match_dup 0)
                    (const_int 0)))]
-  "")
+  ""
+  [(set_attr "type" "add")
+   (set_attr "dot" "yes")
+   (set_attr "length" "4,8")])
 
 ;; Split an add that we can't do in one insn into two insns, each of which
 ;; does one 16-bit part.  This is used by combine.  Note that the low-order
     FAIL;
 })
 
-(define_expand "one_cmpl<mode>2"
-  [(set (match_operand:SDI 0 "gpc_reg_operand" "")
-       (not:SDI (match_operand:SDI 1 "gpc_reg_operand" "")))]
+
+(define_insn "add<mode>3_carry"
+  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
+       (plus:P (match_operand:P 1 "gpc_reg_operand" "r")
+               (match_operand:P 2 "reg_or_short_operand" "rI")))
+   (set (reg:P CA_REGNO)
+       (ltu:P (plus:P (match_dup 1)
+                      (match_dup 2))
+              (match_dup 1)))]
+  ""
+  "add%I2c %0,%1,%2"
+  [(set_attr "type" "add")])
+
+(define_insn "*add<mode>3_imm_carry_pos"
+  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
+       (plus:P (match_operand:P 1 "gpc_reg_operand" "r")
+               (match_operand:P 2 "short_cint_operand" "n")))
+   (set (reg:P CA_REGNO)
+       (geu:P (match_dup 1)
+              (match_operand:P 3 "const_int_operand" "n")))]
+  "INTVAL (operands[2]) > 0
+   && INTVAL (operands[2]) + INTVAL (operands[3]) == 0"
+  "addic %0,%1,%2"
+  [(set_attr "type" "add")])
+
+(define_insn "*add<mode>3_imm_carry_0"
+  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
+       (match_operand:P 1 "gpc_reg_operand" "r"))
+   (set (reg:P CA_REGNO)
+       (const_int 0))]
+  ""
+  "addic %0,%1,0"
+  [(set_attr "type" "add")])
+
+(define_insn "*add<mode>3_imm_carry_m1"
+  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
+       (plus:P (match_operand:P 1 "gpc_reg_operand" "r")
+               (const_int -1)))
+   (set (reg:P CA_REGNO)
+       (ne:P (match_dup 1)
+             (const_int 0)))]
+  ""
+  "addic %0,%1,-1"
+  [(set_attr "type" "add")])
+
+(define_insn "*add<mode>3_imm_carry_neg"
+  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
+       (plus:P (match_operand:P 1 "gpc_reg_operand" "r")
+               (match_operand:P 2 "short_cint_operand" "n")))
+   (set (reg:P CA_REGNO)
+       (gtu:P (match_dup 1)
+              (match_operand:P 3 "const_int_operand" "n")))]
+  "INTVAL (operands[2]) < 0
+   && INTVAL (operands[2]) + INTVAL (operands[3]) == -1"
+  "addic %0,%1,%2"
+  [(set_attr "type" "add")])
+
+
+(define_expand "add<mode>3_carry_in"
+  [(parallel [
+     (set (match_operand:GPR 0 "gpc_reg_operand")
+         (plus:GPR (plus:GPR (match_operand:GPR 1 "gpc_reg_operand")
+                             (match_operand:GPR 2 "adde_operand"))
+                   (reg:GPR CA_REGNO)))
+     (clobber (reg:GPR CA_REGNO))])]
   ""
 {
-  if (<MODE>mode == DImode && !TARGET_POWERPC64)
+  if (operands[2] == const0_rtx)
     {
-      rs6000_split_logical (operands, NOT, false, false, false, NULL_RTX);
+      emit_insn (gen_add<mode>3_carry_in_0 (operands[0], operands[1]));
+      DONE;
+    }
+  if (operands[2] == constm1_rtx)
+    {
+      emit_insn (gen_add<mode>3_carry_in_m1 (operands[0], operands[1]));
+      DONE;
+    }
+})
+
+(define_insn "*add<mode>3_carry_in_internal"
+  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
+       (plus:GPR (plus:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")
+                           (match_operand:GPR 2 "gpc_reg_operand" "r"))
+                 (reg:GPR CA_REGNO)))
+   (clobber (reg:GPR CA_REGNO))]
+  ""
+  "adde %0,%1,%2"
+  [(set_attr "type" "add")])
+
+(define_insn "add<mode>3_carry_in_0"
+  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
+       (plus:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")
+                 (reg:GPR CA_REGNO)))
+   (clobber (reg:GPR CA_REGNO))]
+  ""
+  "addze %0,%1"
+  [(set_attr "type" "add")])
+
+(define_insn "add<mode>3_carry_in_m1"
+  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
+       (plus:GPR (plus:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")
+                           (reg:GPR CA_REGNO))
+                 (const_int -1)))
+   (clobber (reg:GPR CA_REGNO))]
+  ""
+  "addme %0,%1"
+  [(set_attr "type" "add")])
+
+
+(define_expand "one_cmpl<mode>2"
+  [(set (match_operand:SDI 0 "gpc_reg_operand" "")
+       (not:SDI (match_operand:SDI 1 "gpc_reg_operand" "")))]
+  ""
+{
+  if (<MODE>mode == DImode && !TARGET_POWERPC64)
+    {
+      rs6000_split_logical (operands, NOT, false, false, false);
       DONE;
     }
 })
   [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
        (not:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")))]
   ""
-  "nor %0,%1,%1")
+  "not %0,%1")
 
-(define_insn ""
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC (not:P (match_operand:P 1 "gpc_reg_operand" "r,r"))
+(define_insn_and_split "*one_cmpl<mode>2_dot"
+  [(set (match_operand:CC 2 "cc_reg_operand" "=x,?y")
+       (compare:CC (not:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r"))
                    (const_int 0)))
-   (clobber (match_scratch:P 2 "=r,r"))]
-  ""
+   (clobber (match_scratch:GPR 0 "=r,r"))]
+  "<MODE>mode == Pmode && rs6000_gen_cell_microcode"
   "@
-   nor. %2,%1,%1
+   not. %0,%1
    #"
-  [(set_attr "type" "logical,compare")
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[2], CCmode)"
+  [(set (match_dup 0)
+       (not:GPR (match_dup 1)))
+   (set (match_dup 2)
+       (compare:CC (match_dup 0)
+                   (const_int 0)))]
+  ""
+  [(set_attr "type" "logical")
    (set_attr "dot" "yes")
    (set_attr "length" "4,8")])
 
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (not:P (match_operand:P 1 "gpc_reg_operand" ""))
-                   (const_int 0)))
-   (clobber (match_scratch:P 2 ""))]
-  "reload_completed"
-  [(set (match_dup 2)
-       (not:P (match_dup 1)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 2)
-                   (const_int 0)))]
-  "")
-
-(define_insn ""
+(define_insn_and_split "*one_cmpl<mode>2_dot2"
   [(set (match_operand:CC 2 "cc_reg_operand" "=x,?y")
-       (compare:CC (not:P (match_operand:P 1 "gpc_reg_operand" "r,r"))
+       (compare:CC (not:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r"))
                    (const_int 0)))
-   (set (match_operand:P 0 "gpc_reg_operand" "=r,r")
-       (not:P (match_dup 1)))]
-  ""
+   (set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
+       (not:GPR (match_dup 1)))]
+  "<MODE>mode == Pmode && rs6000_gen_cell_microcode"
   "@
-   nor. %0,%1,%1
+   not. %0,%1
    #"
-  [(set_attr "type" "logical,compare")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 2 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (not:P (match_operand:P 1 "gpc_reg_operand" ""))
-                   (const_int 0)))
-   (set (match_operand:P 0 "gpc_reg_operand" "")
-       (not:P (match_dup 1)))]
-  "reload_completed"
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[2], CCmode)"
   [(set (match_dup 0)
-       (not:P (match_dup 1)))
+       (not:GPR (match_dup 1)))
    (set (match_dup 2)
        (compare:CC (match_dup 0)
                    (const_int 0)))]
-  "")
+  ""
+  [(set_attr "type" "logical")
+   (set_attr "dot" "yes")
+   (set_attr "length" "4,8")])
 
-(define_insn ""
-  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
-       (minus:GPR (match_operand:GPR 1 "reg_or_short_operand" "r,I")
-                  (match_operand:GPR 2 "gpc_reg_operand" "r,r")))]
+
+(define_expand "sub<mode>3"
+  [(set (match_operand:SDI 0 "gpc_reg_operand" "")
+       (minus:SDI (match_operand:SDI 1 "reg_or_short_operand" "")
+                  (match_operand:SDI 2 "gpc_reg_operand" "")))]
   ""
-  "@
-   subf %0,%2,%1
-   subfic %0,%2,%1"
+{
+  if (<MODE>mode == DImode && !TARGET_POWERPC64)
+    {
+      rtx lo0 = gen_lowpart (SImode, operands[0]);
+      rtx lo1 = gen_lowpart (SImode, operands[1]);
+      rtx lo2 = gen_lowpart (SImode, operands[2]);
+      rtx hi0 = gen_highpart (SImode, operands[0]);
+      rtx hi1 = gen_highpart_mode (SImode, DImode, operands[1]);
+      rtx hi2 = gen_highpart (SImode, operands[2]);
+
+      if (!reg_or_short_operand (lo1, SImode))
+       lo1 = force_reg (SImode, lo1);
+      if (!adde_operand (hi1, SImode))
+       hi1 = force_reg (SImode, hi1);
+
+      emit_insn (gen_subfsi3_carry (lo0, lo2, lo1));
+      emit_insn (gen_subfsi3_carry_in (hi0, hi2, hi1));
+      DONE;
+    }
+
+  if (short_cint_operand (operands[1], <MODE>mode))
+    {
+      emit_insn (gen_subf<mode>3_imm (operands[0], operands[2], operands[1]));
+      DONE;
+    }
+})
+
+(define_insn "*subf<mode>3"
+  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
+       (minus:GPR (match_operand:GPR 2 "gpc_reg_operand" "r")
+                  (match_operand:GPR 1 "gpc_reg_operand" "r")))]
+  ""
+  "subf %0,%1,%2"
   [(set_attr "type" "add")])
 
-(define_insn ""
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC (minus:P (match_operand:P 1 "gpc_reg_operand" "r,r")
-                            (match_operand:P 2 "gpc_reg_operand" "r,r"))
+(define_insn_and_split "*subf<mode>3_dot"
+  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
+       (compare:CC (minus:GPR (match_operand:GPR 2 "gpc_reg_operand" "r,r")
+                              (match_operand:GPR 1 "gpc_reg_operand" "r,r"))
                    (const_int 0)))
-   (clobber (match_scratch:P 3 "=r,r"))]
-  ""
+   (clobber (match_scratch:GPR 0 "=r,r"))]
+  "<MODE>mode == Pmode"
   "@
-   subf. %3,%2,%1
+   subf. %0,%1,%2
    #"
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
+  [(set (match_dup 0)
+       (minus:GPR (match_dup 2)
+                  (match_dup 1)))
+   (set (match_dup 3)
+       (compare:CC (match_dup 0)
+                   (const_int 0)))]
+  ""
   [(set_attr "type" "add")
    (set_attr "dot" "yes")
    (set_attr "length" "4,8")])
 
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_cr0_operand" "")
-       (compare:CC (minus:P (match_operand:P 1 "gpc_reg_operand" "")
-                            (match_operand:P 2 "gpc_reg_operand" ""))
-                   (const_int 0)))
-   (clobber (match_scratch:P 3 ""))]
-  "reload_completed"
-  [(set (match_dup 3)
-       (minus:P (match_dup 1)
-                 (match_dup 2)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
-
-(define_insn ""
+(define_insn_and_split "*subf<mode>3_dot2"
   [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
-       (compare:CC (minus:P (match_operand:P 1 "gpc_reg_operand" "r,r")
-                            (match_operand:P 2 "gpc_reg_operand" "r,r"))
+       (compare:CC (minus:GPR (match_operand:GPR 2 "gpc_reg_operand" "r,r")
+                              (match_operand:GPR 1 "gpc_reg_operand" "r,r"))
                    (const_int 0)))
-   (set (match_operand:P 0 "gpc_reg_operand" "=r,r")
-       (minus:P (match_dup 1)
-                 (match_dup 2)))]
-  ""
+   (set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
+       (minus:GPR (match_dup 2)
+                  (match_dup 1)))]
+  "<MODE>mode == Pmode"
   "@
-   subf. %0,%2,%1
+   subf. %0,%1,%2
    #"
-  [(set_attr "type" "add")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_cr0_operand" "")
-       (compare:CC (minus:P (match_operand:P 1 "gpc_reg_operand" "")
-                            (match_operand:P 2 "gpc_reg_operand" ""))
-                   (const_int 0)))
-   (set (match_operand:P 0 "gpc_reg_operand" "")
-       (minus:P (match_dup 1)
-                 (match_dup 2)))]
-  "reload_completed"
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
   [(set (match_dup 0)
-       (minus:P (match_dup 1)
-                 (match_dup 2)))
+       (minus:GPR (match_dup 2)
+                  (match_dup 1)))
    (set (match_dup 3)
        (compare:CC (match_dup 0)
                    (const_int 0)))]
-  "")
+  ""
+  [(set_attr "type" "add")
+   (set_attr "dot" "yes")
+   (set_attr "length" "4,8")])
 
-(define_expand "sub<mode>3"
-  [(set (match_operand:SDI 0 "gpc_reg_operand" "")
-       (minus:SDI (match_operand:SDI 1 "reg_or_short_operand" "")
-                  (match_operand:SDI 2 "reg_or_sub_cint_operand" "")))]
+(define_insn "subf<mode>3_imm"
+  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
+       (minus:GPR (match_operand:GPR 2 "short_cint_operand" "I")
+                  (match_operand:GPR 1 "gpc_reg_operand" "r")))
+   (clobber (reg:GPR CA_REGNO))]
+  ""
+  "subfic %0,%1,%2"
+  [(set_attr "type" "add")])
+
+
+(define_insn "subf<mode>3_carry"
+  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
+       (minus:P (match_operand:P 2 "reg_or_short_operand" "rI")
+                (match_operand:P 1 "gpc_reg_operand" "r")))
+   (set (reg:P CA_REGNO)
+       (leu:P (match_dup 1)
+              (match_dup 2)))]
+  ""
+  "subf%I2c %0,%1,%2"
+  [(set_attr "type" "add")])
+
+(define_insn "*subf<mode>3_imm_carry_0"
+  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
+       (neg:P (match_operand:P 1 "gpc_reg_operand" "r")))
+   (set (reg:P CA_REGNO)
+       (eq:P (match_dup 1)
+             (const_int 0)))]
+  ""
+  "subfic %0,%1,0"
+  [(set_attr "type" "add")])
+
+(define_insn "*subf<mode>3_imm_carry_m1"
+  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
+       (not:P (match_operand:P 1 "gpc_reg_operand" "r")))
+   (set (reg:P CA_REGNO)
+       (const_int 1))]
+  ""
+  "subfic %0,%1,-1"
+  [(set_attr "type" "add")])
+
+
+(define_expand "subf<mode>3_carry_in"
+  [(parallel [
+     (set (match_operand:GPR 0 "gpc_reg_operand")
+         (plus:GPR (plus:GPR (not:GPR (match_operand:GPR 1 "gpc_reg_operand"))
+                             (reg:GPR CA_REGNO))
+                   (match_operand:GPR 2 "adde_operand")))
+     (clobber (reg:GPR CA_REGNO))])]
   ""
-  "
 {
-  if (GET_CODE (operands[2]) == CONST_INT)
+  if (operands[2] == const0_rtx)
     {
-      emit_insn (gen_add<mode>3 (operands[0], operands[1],
-                                negate_rtx (<MODE>mode, operands[2])));
+      emit_insn (gen_subf<mode>3_carry_in_0 (operands[0], operands[1]));
       DONE;
     }
-}")
+  if (operands[2] == constm1_rtx)
+    {
+      emit_insn (gen_subf<mode>3_carry_in_m1 (operands[0], operands[1]));
+      DONE;
+    }
+})
 
-(define_expand "neg<mode>2"
-  [(set (match_operand:SDI 0 "gpc_reg_operand" "")
-       (neg:SDI (match_operand:SDI 1 "gpc_reg_operand" "")))]
+(define_insn "*subf<mode>3_carry_in_internal"
+  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
+       (plus:GPR (plus:GPR (not:GPR (match_operand:GPR 1 "gpc_reg_operand" "r"))
+                           (reg:GPR CA_REGNO))
+                 (match_operand:GPR 2 "gpc_reg_operand" "r")))
+   (clobber (reg:GPR CA_REGNO))]
   ""
-  "")
+  "subfe %0,%1,%2"
+  [(set_attr "type" "add")])
+
+(define_insn "subf<mode>3_carry_in_0"
+  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
+       (plus:GPR (not:GPR (match_operand:GPR 1 "gpc_reg_operand" "r"))
+                 (reg:GPR CA_REGNO)))
+   (clobber (reg:GPR CA_REGNO))]
+  ""
+  "subfze %0,%1"
+  [(set_attr "type" "add")])
+
+(define_insn "subf<mode>3_carry_in_m1"
+  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
+       (plus:GPR (minus:GPR (reg:GPR CA_REGNO)
+                            (match_operand:GPR 1 "gpc_reg_operand" "r"))
+                 (const_int -2)))
+   (clobber (reg:GPR CA_REGNO))]
+  ""
+  "subfme %0,%1"
+  [(set_attr "type" "add")])
+
+(define_insn "subf<mode>3_carry_in_xx"
+  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
+       (plus:GPR (reg:GPR CA_REGNO)
+                 (const_int -1)))
+   (clobber (reg:GPR CA_REGNO))]
+  ""
+  "subfe %0,%0,%0"
+  [(set_attr "type" "add")])
+
 
-(define_insn "*neg<mode>2_internal"
+(define_insn "neg<mode>2"
   [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
        (neg:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")))]
   ""
   "neg %0,%1"
   [(set_attr "type" "add")])
 
-(define_insn ""
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC (neg:P (match_operand:P 1 "gpc_reg_operand" "r,r"))
+(define_insn_and_split "*neg<mode>2_dot"
+  [(set (match_operand:CC 2 "cc_reg_operand" "=x,?y")
+       (compare:CC (neg:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r"))
                    (const_int 0)))
-   (clobber (match_scratch:P 2 "=r,r"))]
-  ""
+   (clobber (match_scratch:GPR 0 "=r,r"))]
+  "<MODE>mode == Pmode"
   "@
-   neg. %2,%1
+   neg. %0,%1
    #"
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[2], CCmode)"
+  [(set (match_dup 0)
+       (neg:GPR (match_dup 1)))
+   (set (match_dup 2)
+       (compare:CC (match_dup 0)
+                   (const_int 0)))]
+  ""
   [(set_attr "type" "add")
    (set_attr "dot" "yes")
    (set_attr "length" "4,8")])
 
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_cr0_operand" "")
-       (compare:CC (neg:P (match_operand:P 1 "gpc_reg_operand" ""))
-                   (const_int 0)))
-   (clobber (match_scratch:P 2 ""))]
-  "reload_completed"
-  [(set (match_dup 2)
-       (neg:P (match_dup 1)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 2)
-                   (const_int 0)))]
-  "")
-
-(define_insn ""
+(define_insn_and_split "*neg<mode>2_dot2"
   [(set (match_operand:CC 2 "cc_reg_operand" "=x,?y")
-       (compare:CC (neg:P (match_operand:P 1 "gpc_reg_operand" "r,r"))
+       (compare:CC (neg:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r"))
                    (const_int 0)))
-   (set (match_operand:P 0 "gpc_reg_operand" "=r,r")
-       (neg:P (match_dup 1)))]
-  ""
+   (set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
+       (neg:GPR (match_dup 1)))]
+  "<MODE>mode == Pmode"
   "@
    neg. %0,%1
    #"
-  [(set_attr "type" "add")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 2 "cc_reg_not_cr0_operand" "")
-       (compare:CC (neg:P (match_operand:P 1 "gpc_reg_operand" ""))
-                   (const_int 0)))
-   (set (match_operand:P 0 "gpc_reg_operand" "")
-       (neg:P (match_dup 1)))]
-  "reload_completed"
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[2], CCmode)"
   [(set (match_dup 0)
-       (neg:P (match_dup 1)))
+       (neg:GPR (match_dup 1)))
    (set (match_dup 2)
        (compare:CC (match_dup 0)
                    (const_int 0)))]
-  "")
+  ""
+  [(set_attr "type" "add")
+   (set_attr "dot" "yes")
+   (set_attr "length" "4,8")])
+
 
 (define_insn "clz<mode>2"
   [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
 (define_expand "ctz<mode>2"
   [(set (match_dup 2)
        (neg:GPR (match_operand:GPR 1 "gpc_reg_operand" "")))
-   (parallel [(set (match_dup 3) (and:GPR (match_dup 1)
-                                         (match_dup 2)))
-             (clobber (scratch:CC))])
-   (set (match_dup 4) (clz:GPR (match_dup 3)))
-   (set (match_operand:GPR 0 "gpc_reg_operand" "")
-       (minus:GPR (match_dup 5) (match_dup 4)))]
+   (set (match_dup 3)
+       (and:GPR (match_dup 1)
+                (match_dup 2)))
+   (set (match_dup 4)
+       (clz:GPR (match_dup 3)))
+   (parallel [(set (match_operand:GPR 0 "gpc_reg_operand" "")
+                  (minus:GPR (match_dup 5)
+                             (match_dup 4)))
+             (clobber (reg:GPR CA_REGNO))])]
   ""
-  {
-     operands[2] = gen_reg_rtx (<MODE>mode);
-     operands[3] = gen_reg_rtx (<MODE>mode);
-     operands[4] = gen_reg_rtx (<MODE>mode);
-     operands[5] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - 1);
-  })
+{
+  operands[2] = gen_reg_rtx (<MODE>mode);
+  operands[3] = gen_reg_rtx (<MODE>mode);
+  operands[4] = gen_reg_rtx (<MODE>mode);
+  operands[5] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - 1);
+})
 
 (define_expand "ffs<mode>2"
   [(set (match_dup 2)
        (neg:GPR (match_operand:GPR 1 "gpc_reg_operand" "")))
-   (parallel [(set (match_dup 3) (and:GPR (match_dup 1)
-                                         (match_dup 2)))
-             (clobber (scratch:CC))])
-   (set (match_dup 4) (clz:GPR (match_dup 3)))
-   (set (match_operand:GPR 0 "gpc_reg_operand" "")
-       (minus:GPR (match_dup 5) (match_dup 4)))]
+   (set (match_dup 3)
+       (and:GPR (match_dup 1)
+                (match_dup 2)))
+   (set (match_dup 4)
+       (clz:GPR (match_dup 3)))
+   (parallel [(set (match_operand:GPR 0 "gpc_reg_operand" "")
+                  (minus:GPR (match_dup 5)
+                             (match_dup 4)))
+             (clobber (reg:GPR CA_REGNO))])]
   ""
-  {
-     operands[2] = gen_reg_rtx (<MODE>mode);
-     operands[3] = gen_reg_rtx (<MODE>mode);
-     operands[4] = gen_reg_rtx (<MODE>mode);
-     operands[5] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode));
-  })
+{
+  operands[2] = gen_reg_rtx (<MODE>mode);
+  operands[3] = gen_reg_rtx (<MODE>mode);
+  operands[4] = gen_reg_rtx (<MODE>mode);
+  operands[5] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode));
+})
+
+
+(define_expand "popcount<mode>2"
+  [(set (match_operand:GPR 0 "gpc_reg_operand" "")
+       (popcount:GPR (match_operand:GPR 1 "gpc_reg_operand" "")))]
+  "TARGET_POPCNTB || TARGET_POPCNTD"
+{
+  rs6000_emit_popcount (operands[0], operands[1]);
+  DONE;
+})
 
 (define_insn "popcntb<mode>2"
   [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
-        (unspec:GPR [(match_operand:GPR 1 "gpc_reg_operand" "r")]
-                     UNSPEC_POPCNTB))]
+       (unspec:GPR [(match_operand:GPR 1 "gpc_reg_operand" "r")]
+                   UNSPEC_POPCNTB))]
   "TARGET_POPCNTB"
   "popcntb %0,%1"
-  [(set_attr "length" "4")
-   (set_attr "type" "popcnt")])
+  [(set_attr "type" "popcnt")])
 
 (define_insn "popcntd<mode>2"
   [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
        (popcount:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")))]
   "TARGET_POPCNTD"
   "popcnt<wd> %0,%1"
-  [(set_attr "length" "4")
-   (set_attr "type" "popcnt")])
+  [(set_attr "type" "popcnt")])
 
-(define_expand "popcount<mode>2"
+
+(define_expand "parity<mode>2"
   [(set (match_operand:GPR 0 "gpc_reg_operand" "")
-       (popcount:GPR (match_operand:GPR 1 "gpc_reg_operand" "")))]
-  "TARGET_POPCNTB || TARGET_POPCNTD"
-  {
-    rs6000_emit_popcount (operands[0], operands[1]);
-    DONE;
-  })
+       (parity:GPR (match_operand:GPR 1 "gpc_reg_operand" "")))]
+  "TARGET_POPCNTB"
+{
+  rs6000_emit_parity (operands[0], operands[1]);
+  DONE;
+})
 
 (define_insn "parity<mode>2_cmpb"
   [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
        (unspec:GPR [(match_operand:GPR 1 "gpc_reg_operand" "r")] UNSPEC_PARITY))]
   "TARGET_CMPB && TARGET_POPCNTB"
   "prty<wd> %0,%1"
-  [(set_attr "length" "4")
-   (set_attr "type" "popcnt")])
+  [(set_attr "type" "popcnt")])
 
-(define_expand "parity<mode>2"
-  [(set (match_operand:GPR 0 "gpc_reg_operand" "")
-       (parity:GPR (match_operand:GPR 1 "gpc_reg_operand" "")))]
-  "TARGET_POPCNTB"
-  {
-    rs6000_emit_parity (operands[0], operands[1]);
-    DONE;
-  })
 
 ;; Since the hardware zeros the upper part of the register, save generating the
 ;; AND immediate if we are converting to unsigned
   [(set_attr "length" "4,4,12")
    (set_attr "type" "load,store,*")])
 
-;; We are always BITS_BIG_ENDIAN, so the (const_int 16) below is
-;; correct for -mlittle as well as -mbig.
 (define_split
   [(set (match_operand:HI 0 "gpc_reg_operand" "")
        (bswap:HI (match_operand:HI 1 "gpc_reg_operand" "")))
    (clobber (match_operand:SI 2 "gpc_reg_operand" ""))]
   "reload_completed"
   [(set (match_dup 3)
-       (zero_extract:SI (match_dup 4)
-                        (const_int 8)
-                        (const_int 16)))
+       (and:SI (lshiftrt:SI (match_dup 4)
+                            (const_int 8))
+               (const_int 255)))
    (set (match_dup 2)
        (and:SI (ashift:SI (match_dup 4)
                           (const_int 8))
                   (bswap:DI
                    (match_operand:DI 1 "reg_or_mem_operand" "")))
              (clobber (match_scratch:DI 2 ""))
-             (clobber (match_scratch:DI 3 ""))
-             (clobber (match_scratch:DI 4 ""))])]
+             (clobber (match_scratch:DI 3 ""))])]
   ""
 {
   if (!REG_P (operands[0]) && !REG_P (operands[1]))
 
 ;; Power7/cell has ldbrx/stdbrx, so use it directly
 (define_insn "*bswapdi2_ldbrx"
-  [(set (match_operand:DI 0 "reg_or_mem_operand" "=&r,Z,??&r")
+  [(set (match_operand:DI 0 "reg_or_mem_operand" "=r,Z,&r")
        (bswap:DI (match_operand:DI 1 "reg_or_mem_operand" "Z,r,r")))
    (clobber (match_scratch:DI 2 "=X,X,&r"))
-   (clobber (match_scratch:DI 3 "=X,X,&r"))
-   (clobber (match_scratch:DI 4 "=X,X,&r"))]
+   (clobber (match_scratch:DI 3 "=X,X,&r"))]
   "TARGET_POWERPC64 && TARGET_LDBRX
    && (REG_P (operands[0]) || REG_P (operands[1]))"
   "@
 
 ;; Non-power7/cell, fall back to use lwbrx/stwbrx
 (define_insn "*bswapdi2_64bit"
-  [(set (match_operand:DI 0 "reg_or_mem_operand" "=&r,Z,&r")
+  [(set (match_operand:DI 0 "reg_or_mem_operand" "=r,Z,&r")
        (bswap:DI (match_operand:DI 1 "reg_or_mem_operand" "Z,r,r")))
    (clobber (match_scratch:DI 2 "=&b,&b,&r"))
-   (clobber (match_scratch:DI 3 "=&r,&r,&r"))
-   (clobber (match_scratch:DI 4 "=&r,X,&r"))]
+   (clobber (match_scratch:DI 3 "=&r,&r,&r"))]
   "TARGET_POWERPC64 && !TARGET_LDBRX
    && (REG_P (operands[0]) || REG_P (operands[1]))
    && !(MEM_P (operands[0]) && MEM_VOLATILE_P (operands[0]))
   [(set (match_operand:DI 0 "gpc_reg_operand" "")
        (bswap:DI (match_operand:DI 1 "indexed_or_indirect_operand" "")))
    (clobber (match_operand:DI 2 "gpc_reg_operand" ""))
-   (clobber (match_operand:DI 3 "gpc_reg_operand" ""))
-   (clobber (match_operand:DI 4 "gpc_reg_operand" ""))]
+   (clobber (match_operand:DI 3 "gpc_reg_operand" ""))]
   "TARGET_POWERPC64 && !TARGET_LDBRX && reload_completed"
   [(const_int 0)]
   "
   rtx src    = operands[1];
   rtx op2    = operands[2];
   rtx op3    = operands[3];
-  rtx op4    = operands[4];
   rtx op3_32 = simplify_gen_subreg (SImode, op3, DImode,
                                    BYTES_BIG_ENDIAN ? 4 : 0);
-  rtx op4_32 = simplify_gen_subreg (SImode, op4, DImode,
-                                   BYTES_BIG_ENDIAN ? 4 : 0);
+  rtx dest_32 = simplify_gen_subreg (SImode, dest, DImode,
+                                    BYTES_BIG_ENDIAN ? 4 : 0);
   rtx addr1;
   rtx addr2;
-  rtx word_high;
-  rtx word_low;
+  rtx word1;
+  rtx word2;
 
   addr1 = XEXP (src, 0);
   if (GET_CODE (addr1) == PLUS)
       addr2 = gen_rtx_PLUS (Pmode, op2, addr1);
     }
 
+  word1 = change_address (src, SImode, addr1);
+  word2 = change_address (src, SImode, addr2);
+
   if (BYTES_BIG_ENDIAN)
     {
-      word_high = change_address (src, SImode, addr1);
-      word_low  = change_address (src, SImode, addr2);
+      emit_insn (gen_bswapsi2 (op3_32, word2));
+      emit_insn (gen_bswapsi2 (dest_32, word1));
     }
   else
     {
-      word_high = change_address (src, SImode, addr2);
-      word_low  = change_address (src, SImode, addr1);
+      emit_insn (gen_bswapsi2 (op3_32, word1));
+      emit_insn (gen_bswapsi2 (dest_32, word2));
     }
 
-  emit_insn (gen_bswapsi2 (op3_32, word_low));
-  emit_insn (gen_bswapsi2 (op4_32, word_high));
-  emit_insn (gen_ashldi3 (dest, op3, GEN_INT (32)));
-  emit_insn (gen_iordi3 (dest, dest, op4));
-}")
+  emit_insn (gen_ashldi3 (op3, op3, GEN_INT (32)));
+  emit_insn (gen_iordi3 (dest, dest, op3));
+  DONE;
+}")
 
 (define_split
   [(set (match_operand:DI 0 "indexed_or_indirect_operand" "")
        (bswap:DI (match_operand:DI 1 "gpc_reg_operand" "")))
    (clobber (match_operand:DI 2 "gpc_reg_operand" ""))
-   (clobber (match_operand:DI 3 "gpc_reg_operand" ""))
-   (clobber (match_operand:DI 4 "" ""))]
+   (clobber (match_operand:DI 3 "gpc_reg_operand" ""))]
   "TARGET_POWERPC64 && !TARGET_LDBRX && reload_completed"
   [(const_int 0)]
   "
                                    BYTES_BIG_ENDIAN ? 4 : 0);
   rtx addr1;
   rtx addr2;
-  rtx word_high;
-  rtx word_low;
+  rtx word1;
+  rtx word2;
 
   addr1 = XEXP (dest, 0);
   if (GET_CODE (addr1) == PLUS)
       addr2 = gen_rtx_PLUS (Pmode, op2, addr1);
     }
 
+  word1 = change_address (dest, SImode, addr1);
+  word2 = change_address (dest, SImode, addr2);
+
   emit_insn (gen_lshrdi3 (op3, src, GEN_INT (32)));
+
   if (BYTES_BIG_ENDIAN)
     {
-      word_high = change_address (dest, SImode, addr1);
-      word_low  = change_address (dest, SImode, addr2);
+      emit_insn (gen_bswapsi2 (word1, src_si));
+      emit_insn (gen_bswapsi2 (word2, op3_si));
     }
   else
     {
-      word_high = change_address (dest, SImode, addr2);
-      word_low  = change_address (dest, SImode, addr1);
+      emit_insn (gen_bswapsi2 (word2, src_si));
+      emit_insn (gen_bswapsi2 (word1, op3_si));
     }
-  emit_insn (gen_bswapsi2 (word_high, src_si));
-  emit_insn (gen_bswapsi2 (word_low, op3_si));
+  DONE;
 }")
 
 (define_split
   [(set (match_operand:DI 0 "gpc_reg_operand" "")
        (bswap:DI (match_operand:DI 1 "gpc_reg_operand" "")))
    (clobber (match_operand:DI 2 "gpc_reg_operand" ""))
-   (clobber (match_operand:DI 3 "gpc_reg_operand" ""))
-   (clobber (match_operand:DI 4 "" ""))]
+   (clobber (match_operand:DI 3 "gpc_reg_operand" ""))]
   "TARGET_POWERPC64 && reload_completed"
   [(const_int 0)]
   "
   emit_insn (gen_bswapsi2 (op3_si, op2_si));
   emit_insn (gen_ashldi3 (dest, dest, GEN_INT (32)));
   emit_insn (gen_iordi3 (dest, dest, op3));
+  DONE;
 }")
 
 (define_insn "bswapdi2_32bit"
-  [(set (match_operand:DI 0 "reg_or_mem_operand" "=&r,Z,&r")
+  [(set (match_operand:DI 0 "reg_or_mem_operand" "=r,Z,?&r")
        (bswap:DI (match_operand:DI 1 "reg_or_mem_operand" "Z,r,r")))
    (clobber (match_scratch:SI 2 "=&b,&b,X"))]
   "!TARGET_POWERPC64 && (REG_P (operands[0]) || REG_P (operands[1]))"
   if (GET_CODE (addr1) == PLUS)
     {
       emit_insn (gen_add3_insn (op2, XEXP (addr1, 0), GEN_INT (4)));
-      if (TARGET_AVOID_XFORM)
+      if (TARGET_AVOID_XFORM
+         || REGNO (XEXP (addr1, 1)) == REGNO (dest2))
        {
          emit_insn (gen_add3_insn (op2, XEXP (addr1, 1), op2));
          addr2 = op2;
       else
        addr2 = gen_rtx_PLUS (SImode, op2, XEXP (addr1, 1));
     }
-  else if (TARGET_AVOID_XFORM)
+  else if (TARGET_AVOID_XFORM
+          || REGNO (addr1) == REGNO (dest2))
     {
       emit_insn (gen_add3_insn (op2, addr1, GEN_INT (4)));
       addr2 = op2;
   word2 = change_address (src, SImode, addr2);
 
   emit_insn (gen_bswapsi2 (dest2, word1));
+  /* The REGNO (dest2) tests above ensure that addr2 has not been trashed,
+     thus allowing us to omit an early clobber on the output.  */
   emit_insn (gen_bswapsi2 (dest1, word2));
+  DONE;
 }")
 
 (define_split
 
   emit_insn (gen_bswapsi2 (word2, src1));
   emit_insn (gen_bswapsi2 (word1, src2));
+  DONE;
 }")
 
 (define_split
 
   emit_insn (gen_bswapsi2 (dest1, src2));
   emit_insn (gen_bswapsi2 (dest2, src1));
+  DONE;
 }")
 
-(define_insn "mulsi3"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
-       (mult:SI (match_operand:SI 1 "gpc_reg_operand" "%r,r")
-                (match_operand:SI 2 "reg_or_short_operand" "r,I")))]
+
+(define_insn "mul<mode>3"
+  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
+       (mult:GPR (match_operand:GPR 1 "gpc_reg_operand" "%r,r")
+                 (match_operand:GPR 2 "reg_or_short_operand" "r,I")))]
   ""
   "@
-   mullw %0,%1,%2
+   mull<wd> %0,%1,%2
    mulli %0,%1,%2"
    [(set_attr "type" "mul")
     (set (attr "size")
-      (cond [(match_operand:SI 2 "s8bit_cint_operand" "")
+      (cond [(match_operand:GPR 2 "s8bit_cint_operand" "")
                (const_string "8")
-             (match_operand:SI 2 "short_cint_operand" "")
+             (match_operand:GPR 2 "short_cint_operand" "")
                (const_string "16")]
-       (const_string "32")))])
+       (const_string "<bits>")))])
 
-(define_insn "*mulsi3_internal1"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC (mult:SI (match_operand:SI 1 "gpc_reg_operand" "%r,r")
-                            (match_operand:SI 2 "gpc_reg_operand" "r,r"))
+(define_insn_and_split "*mul<mode>3_dot"
+  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
+       (compare:CC (mult:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r")
+                             (match_operand:GPR 2 "gpc_reg_operand" "r,r"))
                    (const_int 0)))
-   (clobber (match_scratch:SI 3 "=r,r"))]
-  "TARGET_32BIT"
+   (clobber (match_scratch:GPR 0 "=r,r"))]
+  "<MODE>mode == Pmode && rs6000_gen_cell_microcode"
   "@
-   mullw. %3,%1,%2
+   mull<wd>. %0,%1,%2
    #"
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
+  [(set (match_dup 0)
+       (mult:GPR (match_dup 1)
+                 (match_dup 2)))
+   (set (match_dup 3)
+       (compare:CC (match_dup 0)
+                   (const_int 0)))]
+  ""
   [(set_attr "type" "mul")
+   (set_attr "size" "<bits>")
    (set_attr "dot" "yes")
    (set_attr "length" "4,8")])
 
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (mult:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                            (match_operand:SI 2 "gpc_reg_operand" ""))
-                   (const_int 0)))
-   (clobber (match_scratch:SI 3 ""))]
-  "TARGET_32BIT && reload_completed"
-  [(set (match_dup 3)
-       (mult:SI (match_dup 1) (match_dup 2)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
-
-(define_insn "*mulsi3_internal2"
+(define_insn_and_split "*mul<mode>3_dot2"
   [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
-       (compare:CC (mult:SI (match_operand:SI 1 "gpc_reg_operand" "%r,r")
-                            (match_operand:SI 2 "gpc_reg_operand" "r,r"))
+       (compare:CC (mult:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r")
+                             (match_operand:GPR 2 "gpc_reg_operand" "r,r"))
                    (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
-       (mult:SI (match_dup 1) (match_dup 2)))]
-  "TARGET_32BIT"
+   (set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
+       (mult:GPR (match_dup 1)
+                 (match_dup 2)))]
+  "<MODE>mode == Pmode && rs6000_gen_cell_microcode"
   "@
-   mullw. %0,%1,%2
+   mull<wd>. %0,%1,%2
    #"
-  [(set_attr "type" "mul")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (mult:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                            (match_operand:SI 2 "gpc_reg_operand" ""))
-                   (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (mult:SI (match_dup 1) (match_dup 2)))]
-  "TARGET_32BIT && reload_completed"
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
   [(set (match_dup 0)
-       (mult:SI (match_dup 1) (match_dup 2)))
+       (mult:GPR (match_dup 1)
+                 (match_dup 2)))
    (set (match_dup 3)
        (compare:CC (match_dup 0)
                    (const_int 0)))]
-  "")
+  ""
+  [(set_attr "type" "mul")
+   (set_attr "size" "<bits>")
+   (set_attr "dot" "yes")
+   (set_attr "length" "4,8")])
+
+
+(define_expand "<su>mul<mode>3_highpart"
+  [(set (match_operand:GPR 0 "gpc_reg_operand")
+       (subreg:GPR
+         (mult:<DMODE> (any_extend:<DMODE>
+                         (match_operand:GPR 1 "gpc_reg_operand"))
+                       (any_extend:<DMODE>
+                         (match_operand:GPR 2 "gpc_reg_operand")))
+        0))]
+  ""
+{
+  if (<MODE>mode == SImode && TARGET_POWERPC64)
+    {
+      emit_insn (gen_<su>mulsi3_highpart_64 (operands[0], operands[1],
+                                            operands[2]));
+      DONE;
+    }
+
+  if (!WORDS_BIG_ENDIAN)
+    {
+      emit_insn (gen_<su>mul<mode>3_highpart_le (operands[0], operands[1],
+                                                operands[2]));
+      DONE;
+    }
+})
+
+(define_insn "*<su>mul<mode>3_highpart"
+  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
+       (subreg:GPR
+         (mult:<DMODE> (any_extend:<DMODE>
+                         (match_operand:GPR 1 "gpc_reg_operand" "r"))
+                       (any_extend:<DMODE>
+                         (match_operand:GPR 2 "gpc_reg_operand" "r")))
+        0))]
+  "WORDS_BIG_ENDIAN && !(<MODE>mode == SImode && TARGET_POWERPC64)"
+  "mulh<wd><u> %0,%1,%2"
+  [(set_attr "type" "mul")
+   (set_attr "size" "<bits>")])
+
+(define_insn "<su>mulsi3_highpart_le"
+  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
+       (subreg:SI
+         (mult:DI (any_extend:DI
+                    (match_operand:SI 1 "gpc_reg_operand" "r"))
+                  (any_extend:DI
+                    (match_operand:SI 2 "gpc_reg_operand" "r")))
+        4))]
+  "!WORDS_BIG_ENDIAN && !TARGET_POWERPC64"
+  "mulhw<u> %0,%1,%2"
+  [(set_attr "type" "mul")])
+
+(define_insn "<su>muldi3_highpart_le"
+  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
+       (subreg:DI
+         (mult:TI (any_extend:TI
+                    (match_operand:DI 1 "gpc_reg_operand" "r"))
+                  (any_extend:TI
+                    (match_operand:DI 2 "gpc_reg_operand" "r")))
+        8))]
+  "!WORDS_BIG_ENDIAN && TARGET_POWERPC64"
+  "mulhd<u> %0,%1,%2"
+  [(set_attr "type" "mul")
+   (set_attr "size" "64")])
+
+(define_insn "<su>mulsi3_highpart_64"
+  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
+       (truncate:SI
+         (lshiftrt:DI
+           (mult:DI (any_extend:DI
+                      (match_operand:SI 1 "gpc_reg_operand" "r"))
+                    (any_extend:DI
+                      (match_operand:SI 2 "gpc_reg_operand" "r")))
+           (const_int 32))))]
+  "TARGET_POWERPC64"
+  "mulhw<u> %0,%1,%2"
+  [(set_attr "type" "mul")])
+
+(define_expand "<u>mul<mode><dmode>3"
+  [(set (match_operand:<DMODE> 0 "gpc_reg_operand")
+       (mult:<DMODE> (any_extend:<DMODE>
+                       (match_operand:GPR 1 "gpc_reg_operand"))
+                     (any_extend:<DMODE>
+                       (match_operand:GPR 2 "gpc_reg_operand"))))]
+  "!(<MODE>mode == SImode && TARGET_POWERPC64)"
+{
+  rtx l = gen_reg_rtx (<MODE>mode);
+  rtx h = gen_reg_rtx (<MODE>mode);
+  emit_insn (gen_mul<mode>3 (l, operands[1], operands[2]));
+  emit_insn (gen_<su>mul<mode>3_highpart (h, operands[1], operands[2]));
+  emit_move_insn (gen_lowpart (<MODE>mode, operands[0]), l);
+  emit_move_insn (gen_highpart (<MODE>mode, operands[0]), h);
+  DONE;
+})
 
 
 (define_insn "udiv<mode>3"
    (set_attr "size" "<bits>")])
 
 
-;; For powers of two we can do srai/aze for divide and then adjust for
+;; For powers of two we can do sra[wd]i/addze for divide and then adjust for
 ;; modulus.  If it isn't a power of two, force operands into register and do
 ;; a normal divide.
 (define_expand "div<mode>3"
                 (match_operand:GPR 2 "reg_or_cint_operand" "")))]
   ""
 {
-  if (GET_CODE (operands[2]) != CONST_INT
-      || INTVAL (operands[2]) <= 0
-      || exact_log2 (INTVAL (operands[2])) < 0)
-    operands[2] = force_reg (<MODE>mode, operands[2]);
+  if (CONST_INT_P (operands[2])
+      && INTVAL (operands[2]) > 0
+      && exact_log2 (INTVAL (operands[2])) >= 0)
+    {
+      emit_insn (gen_div<mode>3_sra (operands[0], operands[1], operands[2]));
+      DONE;
+    }
+
+  operands[2] = force_reg (<MODE>mode, operands[2]);
 })
 
 (define_insn "*div<mode>3"
   [(set_attr "type" "div")
    (set_attr "size" "<bits>")])
 
+(define_insn "div<mode>3_sra"
+  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
+       (div:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")
+                (match_operand:GPR 2 "exact_log2_cint_operand" "N")))
+   (clobber (reg:GPR CA_REGNO))]
+  ""
+  "sra<wd>i %0,%1,%p2\;addze %0,%0"
+  [(set_attr "type" "two")
+   (set_attr "length" "8")])
+
+(define_insn_and_split "*div<mode>3_sra_dot"
+  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
+       (compare:CC (div:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r")
+                            (match_operand:GPR 2 "exact_log2_cint_operand" "N,N"))
+                   (const_int 0)))
+   (clobber (match_scratch:GPR 0 "=r,r"))
+   (clobber (reg:GPR CA_REGNO))]
+  "<MODE>mode == Pmode"
+  "@
+   sra<wd>i %0,%1,%p2\;addze. %0,%0
+   #"
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
+  [(parallel [(set (match_dup 0)
+                  (div:GPR (match_dup 1)
+                           (match_dup 2)))
+             (clobber (reg:GPR CA_REGNO))])
+   (set (match_dup 3)
+       (compare:CC (match_dup 0)
+                   (const_int 0)))]
+  ""
+  [(set_attr "type" "two")
+   (set_attr "length" "8,12")
+   (set_attr "cell_micro" "not")])
+
+(define_insn_and_split "*div<mode>3_sra_dot2"
+  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
+       (compare:CC (div:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r")
+                            (match_operand:GPR 2 "exact_log2_cint_operand" "N,N"))
+                   (const_int 0)))
+   (set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
+       (div:GPR (match_dup 1)
+                (match_dup 2)))
+   (clobber (reg:GPR CA_REGNO))]
+  "<MODE>mode == Pmode"
+  "@
+   sra<wd>i %0,%1,%p2\;addze. %0,%0
+   #"
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
+  [(parallel [(set (match_dup 0)
+                  (div:GPR (match_dup 1)
+                           (match_dup 2)))
+             (clobber (reg:GPR CA_REGNO))])
+   (set (match_dup 3)
+       (compare:CC (match_dup 0)
+                   (const_int 0)))]
+  ""
+  [(set_attr "type" "two")
+   (set_attr "length" "8,12")
+   (set_attr "cell_micro" "not")])
+
 (define_expand "mod<mode>3"
   [(use (match_operand:GPR 0 "gpc_reg_operand" ""))
    (use (match_operand:GPR 1 "gpc_reg_operand" ""))
    (use (match_operand:GPR 2 "reg_or_cint_operand" ""))]
   ""
-  "
 {
   int i;
   rtx temp1;
   emit_insn (gen_ashl<mode>3 (temp2, temp1, GEN_INT (i)));
   emit_insn (gen_sub<mode>3 (operands[0], operands[1], temp2));
   DONE;
-}")
+})
+\f
+;; Logical instructions
+;; The logical instructions are mostly combined by using match_operator,
+;; but the plain AND insns are somewhat different because there is no
+;; plain 'andi' (only 'andi.'), no plain 'andis', and there are all
+;; those rotate-and-mask operations.  Thus, the AND insns come first.
 
-(define_insn ""
-  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
-       (div:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")
-                (match_operand:GPR 2 "exact_log2_cint_operand" "N")))]
+(define_expand "and<mode>3"
+  [(set (match_operand:SDI 0 "gpc_reg_operand" "")
+       (and:SDI (match_operand:SDI 1 "gpc_reg_operand" "")
+                (match_operand:SDI 2 "reg_or_cint_operand" "")))]
   ""
-  "sra<wd>i %0,%1,%p2\;addze %0,%0"
-  [(set_attr "type" "two")
-   (set_attr "length" "8")])
+{
+  if (<MODE>mode == DImode && !TARGET_POWERPC64)
+    {
+      rs6000_split_logical (operands, AND, false, false, false);
+      DONE;
+    }
 
-(define_insn ""
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC (div:P (match_operand:P 1 "gpc_reg_operand" "r,r")
-                          (match_operand:P 2 "exact_log2_cint_operand" "N,N"))
+  if (logical_const_operand (operands[2], <MODE>mode)
+      && !any_mask_operand (operands[2], <MODE>mode))
+    {
+      if (rs6000_gen_cell_microcode)
+       {
+         emit_insn (gen_and<mode>3_imm (operands[0], operands[1], operands[2]));
+         DONE;
+       }
+      else
+       operands[2] = force_reg (<MODE>mode, operands[2]);
+    }
+
+  if ((<MODE>mode == DImode && !and64_2_operand (operands[2], <MODE>mode))
+      || (<MODE>mode != DImode && !and_operand (operands[2], <MODE>mode)))
+    operands[2] = force_reg (<MODE>mode, operands[2]);
+})
+
+
+(define_insn "and<mode>3_imm"
+  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
+       (and:GPR (match_operand:GPR 1 "gpc_reg_operand" "%r")
+                (match_operand:GPR 2 "logical_const_operand" "n")))
+   (clobber (match_scratch:CC 3 "=x"))]
+  "rs6000_gen_cell_microcode
+   && !any_mask_operand (operands[2], <MODE>mode)"
+  "andi%e2. %0,%1,%u2"
+  [(set_attr "type" "logical")
+   (set_attr "dot" "yes")])
+
+(define_insn_and_split "*and<mode>3_imm_dot"
+  [(set (match_operand:CC 3 "cc_reg_operand" "=x,??y")
+       (compare:CC (and:GPR (match_operand:GPR 1 "gpc_reg_operand" "%r,r")
+                            (match_operand:GPR 2 "logical_const_operand" "n,n"))
                    (const_int 0)))
-   (clobber (match_scratch:P 3 "=r,r"))]
-  ""
+   (clobber (match_scratch:GPR 0 "=r,r"))
+   (clobber (match_scratch:CC 4 "=X,x"))]
+  "(<MODE>mode == Pmode || UINTVAL (operands[2]) <= 0x7fffffff)
+   && rs6000_gen_cell_microcode
+   && !any_mask_operand (operands[2], <MODE>mode)"
   "@
-   sra<wd>i %3,%1,%p2\;addze. %3,%3
+   andi%e2. %0,%1,%u2
    #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "8,12")
-   (set_attr "cell_micro" "not")])
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_cr0_operand" "")
-       (compare:CC (div:GPR (match_operand:GPR 1 "gpc_reg_operand" "")
-                            (match_operand:GPR 2 "exact_log2_cint_operand"
-                             ""))
-                   (const_int 0)))
-   (clobber (match_scratch:GPR 3 ""))]
-  "reload_completed"
-  [(set (match_dup 3)
-       (div:<MODE> (match_dup 1) (match_dup 2)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
+  [(parallel [(set (match_dup 0)
+                  (and:GPR (match_dup 1)
+                           (match_dup 2)))
+             (clobber (match_dup 4))])
+   (set (match_dup 3)
+       (compare:CC (match_dup 0)
                    (const_int 0)))]
-  "")
+  ""
+  [(set_attr "type" "logical")
+   (set_attr "dot" "yes")
+   (set_attr "length" "4,8")])
 
-(define_insn ""
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
-       (compare:CC (div:P (match_operand:P 1 "gpc_reg_operand" "r,r")
-                          (match_operand:P 2 "exact_log2_cint_operand" "N,N"))
+(define_insn_and_split "*and<mode>3_imm_dot2"
+  [(set (match_operand:CC 3 "cc_reg_operand" "=x,??y")
+       (compare:CC (and:GPR (match_operand:GPR 1 "gpc_reg_operand" "%r,r")
+                            (match_operand:GPR 2 "logical_const_operand" "n,n"))
                    (const_int 0)))
-   (set (match_operand:P 0 "gpc_reg_operand" "=r,r")
-       (div:P (match_dup 1) (match_dup 2)))]
-  ""
+   (set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
+       (and:GPR (match_dup 1)
+                (match_dup 2)))
+   (clobber (match_scratch:CC 4 "=X,x"))]
+  "(<MODE>mode == Pmode || UINTVAL (operands[2]) <= 0x7fffffff)
+   && rs6000_gen_cell_microcode
+   && !any_mask_operand (operands[2], <MODE>mode)"
   "@
-   sra<wd>i %0,%1,%p2\;addze. %0,%0
+   andi%e2. %0,%1,%u2
    #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "8,12")
-   (set_attr "cell_micro" "not")])
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
+  [(parallel [(set (match_dup 0)
+                  (and:GPR (match_dup 1)
+                           (match_dup 2)))
+             (clobber (match_dup 4))])
+   (set (match_dup 3)
+       (compare:CC (match_dup 0)
+                   (const_int 0)))]
+  ""
+  [(set_attr "type" "logical")
+   (set_attr "dot" "yes")
+   (set_attr "length" "4,8")])
 
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_cr0_operand" "")
-       (compare:CC (div:GPR (match_operand:GPR 1 "gpc_reg_operand" "")
-                            (match_operand:GPR 2 "exact_log2_cint_operand"
-                             ""))
+(define_insn_and_split "*and<mode>3_imm_mask_dot"
+  [(set (match_operand:CC 3 "cc_reg_operand" "=x,??y")
+       (compare:CC (and:GPR (match_operand:GPR 1 "gpc_reg_operand" "%r,r")
+                            (match_operand:GPR 2 "logical_const_operand" "n,n"))
                    (const_int 0)))
-   (set (match_operand:GPR 0 "gpc_reg_operand" "")
-       (div:GPR (match_dup 1) (match_dup 2)))]
-  "reload_completed"
+   (clobber (match_scratch:GPR 0 "=r,r"))]
+  "(<MODE>mode == Pmode || UINTVAL (operands[2]) <= 0x7fffffff)
+   && rs6000_gen_cell_microcode
+   && any_mask_operand (operands[2], <MODE>mode)"
+  "@
+   andi%e2. %0,%1,%u2
+   #"
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
   [(set (match_dup 0)
-       (div:<MODE> (match_dup 1) (match_dup 2)))
+       (and:GPR (match_dup 1)
+                (match_dup 2)))
    (set (match_dup 3)
        (compare:CC (match_dup 0)
                    (const_int 0)))]
-  "")
-\f
-;; Logical instructions
-;; The logical instructions are mostly combined by using match_operator,
-;; but the plain AND insns are somewhat different because there is no
-;; plain 'andi' (only 'andi.'), no plain 'andis', and there are all
-;; those rotate-and-mask operations.  Thus, the AND insns come first.
-
-(define_expand "andsi3"
-  [(parallel
-    [(set (match_operand:SI 0 "gpc_reg_operand" "")
-         (and:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                 (match_operand:SI 2 "and_operand" "")))
-     (clobber (match_scratch:CC 3 ""))])]
   ""
-  "")
+  [(set_attr "type" "logical")
+   (set_attr "dot" "yes")
+   (set_attr "length" "4,8")])
 
-(define_insn "andsi3_mc"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r,r,r,r")
-       (and:SI (match_operand:SI 1 "gpc_reg_operand" "%r,r,r,r")
-               (match_operand:SI 2 "and_operand" "?r,T,K,L")))
-   (clobber (match_scratch:CC 3 "=X,X,x,x"))]
-  "rs6000_gen_cell_microcode"
+(define_insn_and_split "*and<mode>3_imm_mask_dot2"
+  [(set (match_operand:CC 3 "cc_reg_operand" "=x,??y")
+       (compare:CC (and:GPR (match_operand:GPR 1 "gpc_reg_operand" "%r,r")
+                            (match_operand:GPR 2 "logical_const_operand" "n,n"))
+                   (const_int 0)))
+   (set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
+       (and:GPR (match_dup 1)
+                (match_dup 2)))]
+  "(<MODE>mode == Pmode || UINTVAL (operands[2]) <= 0x7fffffff)
+   && rs6000_gen_cell_microcode
+   && any_mask_operand (operands[2], <MODE>mode)"
   "@
-   and %0,%1,%2
-   rlwinm %0,%1,0,%m2,%M2
-   andi. %0,%1,%b2
-   andis. %0,%1,%u2"
-  [(set_attr "type" "*,shift,logical,logical")
-   (set_attr "dot" "no,no,yes,yes")])
+   andi%e2. %0,%1,%u2
+   #"
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
+  [(set (match_dup 0)
+       (and:GPR (match_dup 1)
+                (match_dup 2)))
+   (set (match_dup 3)
+       (compare:CC (match_dup 0)
+                   (const_int 0)))]
+  ""
+  [(set_attr "type" "logical")
+   (set_attr "dot" "yes")
+   (set_attr "length" "4,8")])
 
-(define_insn "andsi3_nomc"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
-       (and:SI (match_operand:SI 1 "gpc_reg_operand" "%r,r")
-               (match_operand:SI 2 "and_operand" "?r,T")))
-   (clobber (match_scratch:CC 3 "=X,X"))]
-  "!rs6000_gen_cell_microcode"
-  "@
-   and %0,%1,%2
-   rlwinm %0,%1,0,%m2,%M2"
-  [(set_attr "type" "logical,shift")])
 
-(define_insn "andsi3_internal0_nomc"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
-        (and:SI (match_operand:SI 1 "gpc_reg_operand" "%r,r")
-                (match_operand:SI 2 "and_operand" "?r,T")))]
-  "!rs6000_gen_cell_microcode"
+(define_insn "*and<mode>3_mask"
+  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
+       (and:GPR (match_operand:GPR 1 "gpc_reg_operand" "%r,r")
+                (match_operand:GPR 2 "any_mask_operand" "S,T")))]
+  ""
   "@
-   and %0,%1,%2
+   rldic%B2 %0,%1,0,%S2
    rlwinm %0,%1,0,%m2,%M2"
-  [(set_attr "type" "logical,shift")])
-
-
-;; Note to set cr's other than cr0 we do the and immediate and then
-;; the test again -- this avoids a mfcr which on the higher end
-;; machines causes an execution serialization
+  [(set_attr "type" "shift")])
 
-(define_insn "*andsi3_internal2_mc"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,x,x,x,?y,??y,??y,?y")
-       (compare:CC (and:SI (match_operand:SI 1 "gpc_reg_operand" "%r,r,r,r,r,r,r,r")
-                           (match_operand:SI 2 "and_operand" "r,K,L,T,r,K,L,T"))
+(define_insn_and_split "*and<mode>3_mask_dot"
+  [(set (match_operand:CC 3 "cc_reg_operand" "=x,x,?y,?y")
+       (compare:CC (and:GPR (match_operand:GPR 1 "gpc_reg_operand" "%r,r,r,r")
+                            (match_operand:GPR 2 "any_mask_operand" "S,T,S,T"))
                    (const_int 0)))
-   (clobber (match_scratch:SI 3 "=r,r,r,r,r,r,r,r"))
-   (clobber (match_scratch:CC 4 "=X,X,X,X,X,x,x,X"))]
-  "TARGET_32BIT && rs6000_gen_cell_microcode"
-  "@
-   and. %3,%1,%2
-   andi. %3,%1,%b2
-   andis. %3,%1,%u2
-   rlwinm. %3,%1,0,%m2,%M2
-   #
-   #
+   (clobber (match_scratch:GPR 0 "=r,r,r,r"))]
+  "(<MODE>mode == Pmode || UINTVAL (operands[2]) <= 0x7fffffff)
+   && rs6000_gen_cell_microcode
+   && !logical_const_operand (operands[2], <MODE>mode)"
+  "@
+   rldic%B2. %0,%1,0,%S2
+   rlwinm. %0,%1,0,%m2,%M2
    #
    #"
-  [(set_attr "type" "logical,logical,logical,shift,\
-                    compare,compare,compare,compare")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,4,4,4,8,8,8,8")])
-
-(define_insn "*andsi3_internal3_mc"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,x,x,x,?y,??y,??y,?y")
-       (compare:CC (and:SI (match_operand:SI 1 "gpc_reg_operand" "%r,r,r,r,r,r,r,r")
-                           (match_operand:SI 2 "and_operand" "r,K,L,T,r,K,L,T"))
-                   (const_int 0)))
-   (clobber (match_scratch:SI 3 "=r,r,r,r,r,r,r,r"))
-   (clobber (match_scratch:CC 4 "=X,X,X,X,X,x,x,X"))]
-  "TARGET_64BIT && rs6000_gen_cell_microcode"
-  "@
-   #
-   andi. %3,%1,%b2
-   andis. %3,%1,%u2
-   rlwinm. %3,%1,0,%m2,%M2
-   #
-   #
-   #
-   #"
-  [(set_attr "type" "compare,logical,logical,shift,compare,\
-                    compare,compare,compare")
-   (set_attr "dot" "yes")
-   (set_attr "length" "8,4,4,4,8,8,8,8")])
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (and:GPR (match_operand:GPR 1 "gpc_reg_operand" "")
-                            (match_operand:GPR 2 "and_operand" ""))
-                   (const_int 0)))
-   (clobber (match_scratch:GPR 3 ""))
-   (clobber (match_scratch:CC 4 ""))]
-  "reload_completed"
-  [(parallel [(set (match_dup 3)
-                  (and:<MODE> (match_dup 1)
-                              (match_dup 2)))
-             (clobber (match_dup 4))])
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
-
-;; We don't have a 32 bit "and. rt,ra,rb" for ppc64.  cr is set from the
-;; whole 64 bit reg, and we don't know what is in the high 32 bits.
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_operand" "")
-       (compare:CC (and:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                           (match_operand:SI 2 "gpc_reg_operand" ""))
-                   (const_int 0)))
-   (clobber (match_scratch:SI 3 ""))
-   (clobber (match_scratch:CC 4 ""))]
-  "TARGET_POWERPC64 && reload_completed"
-  [(parallel [(set (match_dup 3)
-                  (and:SI (match_dup 1)
-                          (match_dup 2)))
-             (clobber (match_dup 4))])
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
+  [(set (match_dup 0)
+       (and:GPR (match_dup 1)
+                (match_dup 2)))
+   (set (match_dup 3)
+       (compare:CC (match_dup 0)
                    (const_int 0)))]
-  "")
-
-(define_insn "*andsi3_internal4"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,x,x,x,?y,??y,??y,?y")
-       (compare:CC (and:SI (match_operand:SI 1 "gpc_reg_operand" "%r,r,r,r,r,r,r,r")
-                           (match_operand:SI 2 "and_operand" "r,K,L,T,r,K,L,T"))
-                   (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r,r,r,r,r,r,r")
-       (and:SI (match_dup 1)
-               (match_dup 2)))
-   (clobber (match_scratch:CC 4 "=X,X,X,X,X,x,x,X"))]
-  "TARGET_32BIT && rs6000_gen_cell_microcode"
-  "@
-   and. %0,%1,%2
-   andi. %0,%1,%b2
-   andis. %0,%1,%u2
-   rlwinm. %0,%1,0,%m2,%M2
-   #
-   #
-   #
-   #"
-  [(set_attr "type" "logical,logical,logical,shift,\
-                    compare,compare,compare,compare")
+  ""
+  [(set_attr "type" "shift")
    (set_attr "dot" "yes")
-   (set_attr "length" "4,4,4,4,8,8,8,8")])
+   (set_attr "length" "4,4,8,8")])
 
-(define_insn "*andsi3_internal5_mc"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,x,x,x,?y,??y,??y,?y")
-       (compare:CC (and:SI (match_operand:SI 1 "gpc_reg_operand" "%r,r,r,r,r,r,r,r")
-                           (match_operand:SI 2 "and_operand" "r,K,L,T,r,K,L,T"))
+(define_insn_and_split "*and<mode>3_mask_dot2"
+  [(set (match_operand:CC 3 "cc_reg_operand" "=x,x,?y,?y")
+       (compare:CC (and:GPR (match_operand:GPR 1 "gpc_reg_operand" "%r,r,r,r")
+                            (match_operand:GPR 2 "any_mask_operand" "S,T,S,T"))
                    (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r,r,r,r,r,r,r")
-       (and:SI (match_dup 1)
-               (match_dup 2)))
-   (clobber (match_scratch:CC 4 "=X,X,X,X,X,x,x,X"))]
-  "TARGET_64BIT && rs6000_gen_cell_microcode"
+   (set (match_operand:GPR 0 "gpc_reg_operand" "=r,r,r,r")
+       (and:GPR (match_dup 1)
+                (match_dup 2)))]
+  "(<MODE>mode == Pmode || UINTVAL (operands[2]) <= 0x7fffffff)
+   && rs6000_gen_cell_microcode
+   && !logical_const_operand (operands[2], <MODE>mode)"
   "@
-   #
-   andi. %0,%1,%b2
-   andis. %0,%1,%u2
+   rldic%B2. %0,%1,0,%S2
    rlwinm. %0,%1,0,%m2,%M2
    #
-   #
-   #
    #"
-  [(set_attr "type" "compare,logical,logical,shift,compare,\
-                    compare,compare,compare")
-   (set_attr "dot" "yes")
-   (set_attr "length" "8,4,4,4,8,8,8,8")])
-
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (and:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                           (match_operand:SI 2 "and_operand" ""))
-                   (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (and:SI (match_dup 1)
-               (match_dup 2)))
-   (clobber (match_scratch:CC 4 ""))]
-  "reload_completed"
-  [(parallel [(set (match_dup 0)
-                  (and:SI (match_dup 1)
-                          (match_dup 2)))
-             (clobber (match_dup 4))])
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
+  [(set (match_dup 0)
+       (and:GPR (match_dup 1)
+                (match_dup 2)))
    (set (match_dup 3)
        (compare:CC (match_dup 0)
                    (const_int 0)))]
-  "")
+  ""
+  [(set_attr "type" "shift")
+   (set_attr "dot" "yes")
+   (set_attr "length" "4,4,8,8")])
+
+
+
+(define_insn "andsi3_internal0_nomc"
+  [(set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
+        (and:SI (match_operand:SI 1 "gpc_reg_operand" "%r,r")
+                (match_operand:SI 2 "and_operand" "?r,T")))]
+  "!rs6000_gen_cell_microcode"
+  "@
+   and %0,%1,%2
+   rlwinm %0,%1,0,%m2,%M2"
+  [(set_attr "type" "logical,shift")])
 
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_operand" "")
-       (compare:CC (and:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                           (match_operand:SI 2 "gpc_reg_operand" ""))
-                   (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (and:SI (match_dup 1)
-               (match_dup 2)))
-   (clobber (match_scratch:CC 4 ""))]
-  "TARGET_POWERPC64 && reload_completed"
-  [(parallel [(set (match_dup 0)
-                  (and:SI (match_dup 1)
-                          (match_dup 2)))
-             (clobber (match_dup 4))])
-   (set (match_dup 3)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
 
 ;; Handle the PowerPC64 rlwinm corner case
 
 }"
   [(set_attr "length" "8")])
 
-(define_expand "iorsi3"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "")
-       (ior:SI (match_operand:SI 1 "gpc_reg_operand" "")
-               (match_operand:SI 2 "reg_or_logical_cint_operand" "")))]
+
+(define_expand "<code><mode>3"
+  [(set (match_operand:SDI 0 "gpc_reg_operand" "")
+       (iorxor:SDI (match_operand:SDI 1 "gpc_reg_operand" "")
+                   (match_operand:SDI 2 "reg_or_cint_operand" "")))]
   ""
-  "
 {
-  if (GET_CODE (operands[2]) == CONST_INT
-      && ! logical_operand (operands[2], SImode))
+  if (<MODE>mode == DImode && !TARGET_POWERPC64)
+    {
+      rs6000_split_logical (operands, <CODE>, false, false, false);
+      DONE;
+    }
+
+  if (non_logical_cint_operand (operands[2], <MODE>mode))
     {
-      HOST_WIDE_INT value = INTVAL (operands[2]);
       rtx tmp = ((!can_create_pseudo_p ()
                  || rtx_equal_p (operands[0], operands[1]))
-                ? operands[0] : gen_reg_rtx (SImode));
+                ? operands[0] : gen_reg_rtx (<MODE>mode));
+
+      HOST_WIDE_INT value = INTVAL (operands[2]);
+      HOST_WIDE_INT lo = value & 0xffff;
+      HOST_WIDE_INT hi = value - lo;
 
-      emit_insn (gen_iorsi3 (tmp, operands[1],
-                            GEN_INT (value & (~ (HOST_WIDE_INT) 0xffff))));
-      emit_insn (gen_iorsi3 (operands[0], tmp, GEN_INT (value & 0xffff)));
+      emit_insn (gen_<code><mode>3 (tmp, operands[1], GEN_INT (hi)));
+      emit_insn (gen_<code><mode>3 (operands[0], tmp, GEN_INT (lo)));
       DONE;
     }
-}")
 
-(define_expand "xorsi3"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "")
-       (xor:SI (match_operand:SI 1 "gpc_reg_operand" "")
-               (match_operand:SI 2 "reg_or_logical_cint_operand" "")))]
+  if (!reg_or_logical_cint_operand (operands[2], <MODE>mode))
+    operands[2] = force_reg (<MODE>mode, operands[2]);
+})
+
+(define_split
+  [(set (match_operand:GPR 0 "gpc_reg_operand" "")
+       (iorxor:GPR (match_operand:GPR 1 "gpc_reg_operand" "")
+                   (match_operand:GPR 2 "non_logical_cint_operand" "")))]
   ""
-  "
+  [(set (match_dup 3)
+       (iorxor:GPR (match_dup 1)
+                   (match_dup 4)))
+   (set (match_dup 0)
+       (iorxor:GPR (match_dup 3)
+                   (match_dup 5)))]
 {
-  if (GET_CODE (operands[2]) == CONST_INT
-      && ! logical_operand (operands[2], SImode))
-    {
-      HOST_WIDE_INT value = INTVAL (operands[2]);
-      rtx tmp = ((!can_create_pseudo_p ()
+  operands[3] = ((!can_create_pseudo_p ()
                  || rtx_equal_p (operands[0], operands[1]))
-                ? operands[0] : gen_reg_rtx (SImode));
+                ? operands[0] : gen_reg_rtx (<MODE>mode));
 
-      emit_insn (gen_xorsi3 (tmp, operands[1],
-                            GEN_INT (value & (~ (HOST_WIDE_INT) 0xffff))));
-      emit_insn (gen_xorsi3 (operands[0], tmp, GEN_INT (value & 0xffff)));
-      DONE;
-    }
-}")
+  HOST_WIDE_INT value = INTVAL (operands[2]);
+  HOST_WIDE_INT lo = value & 0xffff;
+  HOST_WIDE_INT hi = value - lo;
+
+  operands[4] = GEN_INT (hi);
+  operands[5] = GEN_INT (lo);
+})
 
-(define_insn "*boolsi3_internal1"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r,r,r")
-       (match_operator:SI 3 "boolean_or_operator"
-        [(match_operand:SI 1 "gpc_reg_operand" "%r,r,r")
-         (match_operand:SI 2 "logical_operand" "r,K,L")]))]
+(define_insn "*bool<mode>3_imm"
+  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
+       (match_operator:GPR 3 "boolean_or_operator"
+        [(match_operand:GPR 1 "gpc_reg_operand" "%r")
+         (match_operand:GPR 2 "logical_const_operand" "n")]))]
   ""
-  "@
-   %q3 %0,%1,%2
-   %q3i %0,%1,%b2
-   %q3is %0,%1,%u2")
+  "%q3i%e2 %0,%1,%u2"
+  [(set_attr "type" "logical")])
 
-(define_insn "*boolsi3_internal2"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC (match_operator:SI 4 "boolean_or_operator"
-        [(match_operand:SI 1 "gpc_reg_operand" "%r,r")
-         (match_operand:SI 2 "gpc_reg_operand" "r,r")])
+(define_insn "*bool<mode>3"
+  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
+       (match_operator:GPR 3 "boolean_operator"
+        [(match_operand:GPR 1 "gpc_reg_operand" "r")
+         (match_operand:GPR 2 "gpc_reg_operand" "r")]))]
+  ""
+  "%q3 %0,%1,%2"
+  [(set_attr "type" "logical")])
+
+(define_insn_and_split "*bool<mode>3_dot"
+  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
+       (compare:CC (match_operator:GPR 3 "boolean_operator"
+        [(match_operand:GPR 1 "gpc_reg_operand" "r,r")
+         (match_operand:GPR 2 "gpc_reg_operand" "r,r")])
         (const_int 0)))
-   (clobber (match_scratch:SI 3 "=r,r"))]
-  "TARGET_32BIT"
+   (clobber (match_scratch:GPR 0 "=r,r"))]
+  "<MODE>mode == Pmode && rs6000_gen_cell_microcode"
   "@
-   %q4. %3,%1,%2
+   %q3. %0,%1,%2
    #"
-  [(set_attr "type" "logical,compare")
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[4], CCmode)"
+  [(set (match_dup 0)
+       (match_dup 3))
+   (set (match_dup 4)
+       (compare:CC (match_dup 0)
+                   (const_int 0)))]
+  ""
+  [(set_attr "type" "logical")
    (set_attr "dot" "yes")
    (set_attr "length" "4,8")])
 
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (match_operator:SI 4 "boolean_operator"
-        [(match_operand:SI 1 "gpc_reg_operand" "")
-         (match_operand:SI 2 "gpc_reg_operand" "")])
-        (const_int 0)))
-   (clobber (match_scratch:SI 3 ""))]
-  "TARGET_32BIT && reload_completed"
-  [(set (match_dup 3) (match_dup 4))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
-
-(define_insn "*boolsi3_internal3"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
-       (compare:CC (match_operator:SI 4 "boolean_operator"
-        [(match_operand:SI 1 "gpc_reg_operand" "%r,r")
-         (match_operand:SI 2 "gpc_reg_operand" "r,r")])
+(define_insn_and_split "*bool<mode>3_dot2"
+  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
+       (compare:CC (match_operator:GPR 3 "boolean_operator"
+        [(match_operand:GPR 1 "gpc_reg_operand" "r,r")
+         (match_operand:GPR 2 "gpc_reg_operand" "r,r")])
         (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
-       (match_dup 4))]
-  "TARGET_32BIT"
+   (set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
+       (match_dup 3))]
+  "<MODE>mode == Pmode && rs6000_gen_cell_microcode"
   "@
-   %q4. %0,%1,%2
+   %q3. %0,%1,%2
    #"
-  [(set_attr "type" "logical,compare")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (match_operator:SI 4 "boolean_operator"
-        [(match_operand:SI 1 "gpc_reg_operand" "")
-         (match_operand:SI 2 "gpc_reg_operand" "")])
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (match_dup 4))]
-  "TARGET_32BIT && reload_completed"
-  [(set (match_dup 0) (match_dup 4))
-   (set (match_dup 3)
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[4], CCmode)"
+  [(set (match_dup 0)
+       (match_dup 3))
+   (set (match_dup 4)
        (compare:CC (match_dup 0)
                    (const_int 0)))]
-  "")
-
-;; Split a logical operation that we can't do in one insn into two insns,
-;; each of which does one 16-bit part.  This is used by combine.
+  ""
+  [(set_attr "type" "logical")
+   (set_attr "dot" "yes")
+   (set_attr "length" "4,8")])
 
-(define_split
-  [(set (match_operand:SI 0 "gpc_reg_operand" "")
-       (match_operator:SI 3 "boolean_or_operator"
-        [(match_operand:SI 1 "gpc_reg_operand" "")
-         (match_operand:SI 2 "non_logical_cint_operand" "")]))]
-  ""
-  [(set (match_dup 0) (match_dup 4))
-   (set (match_dup 0) (match_dup 5))]
-"
-{
-  rtx i;
-  i = GEN_INT (INTVAL (operands[2]) & (~ (HOST_WIDE_INT) 0xffff));
-  operands[4] = gen_rtx_fmt_ee (GET_CODE (operands[3]), SImode,
-                               operands[1], i);
-  i = GEN_INT (INTVAL (operands[2]) & 0xffff);
-  operands[5] = gen_rtx_fmt_ee (GET_CODE (operands[3]), SImode,
-                               operands[0], i);
-}")
 
-(define_insn "*boolcsi3_internal1"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-       (match_operator:SI 3 "boolean_operator"
-        [(not:SI (match_operand:SI 1 "gpc_reg_operand" "r"))
-         (match_operand:SI 2 "gpc_reg_operand" "r")]))]
+(define_insn "*boolc<mode>3"
+  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
+       (match_operator:GPR 3 "boolean_operator"
+        [(not:GPR (match_operand:GPR 2 "gpc_reg_operand" "r"))
+         (match_operand:GPR 1 "gpc_reg_operand" "r")]))]
   ""
-  "%q3 %0,%2,%1")
+  "%q3 %0,%1,%2"
+  [(set_attr "type" "logical")])
 
-(define_insn "*boolcsi3_internal2"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC (match_operator:SI 4 "boolean_operator"
-        [(not:SI (match_operand:SI 1 "gpc_reg_operand" "r,r"))
-         (match_operand:SI 2 "gpc_reg_operand" "r,r")])
+(define_insn_and_split "*boolc<mode>3_dot"
+  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
+       (compare:CC (match_operator:GPR 3 "boolean_operator"
+        [(not:GPR (match_operand:GPR 2 "gpc_reg_operand" "r,r"))
+         (match_operand:GPR 1 "gpc_reg_operand" "r,r")])
         (const_int 0)))
-   (clobber (match_scratch:SI 3 "=r,r"))]
-  "TARGET_32BIT"
+   (clobber (match_scratch:GPR 0 "=r,r"))]
+  "<MODE>mode == Pmode && rs6000_gen_cell_microcode"
   "@
-   %q4. %3,%2,%1
+   %q3. %0,%1,%2
    #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (match_operator:SI 4 "boolean_operator"
-        [(not:SI (match_operand:SI 1 "gpc_reg_operand" ""))
-         (match_operand:SI 2 "gpc_reg_operand" "")])
-        (const_int 0)))
-   (clobber (match_scratch:SI 3 ""))]
-  "TARGET_32BIT && reload_completed"
-  [(set (match_dup 3) (match_dup 4))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[4], CCmode)"
+  [(set (match_dup 0)
+       (match_dup 3))
+   (set (match_dup 4)
+       (compare:CC (match_dup 0)
                    (const_int 0)))]
-  "")
+  ""
+  [(set_attr "type" "logical")
+   (set_attr "dot" "yes")
+   (set_attr "length" "4,8")])
 
-(define_insn "*boolcsi3_internal3"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
-       (compare:CC (match_operator:SI 4 "boolean_operator"
-        [(not:SI (match_operand:SI 1 "gpc_reg_operand" "%r,r"))
-         (match_operand:SI 2 "gpc_reg_operand" "r,r")])
+(define_insn_and_split "*boolc<mode>3_dot2"
+  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
+       (compare:CC (match_operator:GPR 3 "boolean_operator"
+        [(not:GPR (match_operand:GPR 2 "gpc_reg_operand" "r,r"))
+         (match_operand:GPR 1 "gpc_reg_operand" "r,r")])
         (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
-       (match_dup 4))]
-  "TARGET_32BIT"
+   (set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
+       (match_dup 3))]
+  "<MODE>mode == Pmode && rs6000_gen_cell_microcode"
   "@
-   %q4. %0,%2,%1
+   %q3. %0,%1,%2
    #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (match_operator:SI 4 "boolean_operator"
-        [(not:SI (match_operand:SI 1 "gpc_reg_operand" ""))
-         (match_operand:SI 2 "gpc_reg_operand" "")])
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (match_dup 4))]
-  "TARGET_32BIT && reload_completed"
-  [(set (match_dup 0) (match_dup 4))
-   (set (match_dup 3)
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[4], CCmode)"
+  [(set (match_dup 0)
+       (match_dup 3))
+   (set (match_dup 4)
        (compare:CC (match_dup 0)
                    (const_int 0)))]
-  "")
+  ""
+  [(set_attr "type" "logical")
+   (set_attr "dot" "yes")
+   (set_attr "length" "4,8")])
 
-(define_insn "*boolccsi3_internal1"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-       (match_operator:SI 3 "boolean_operator"
-        [(not:SI (match_operand:SI 1 "gpc_reg_operand" "r"))
-         (not:SI (match_operand:SI 2 "gpc_reg_operand" "r"))]))]
+
+(define_insn "*boolcc<mode>3"
+  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
+       (match_operator:GPR 3 "boolean_operator"
+        [(not:GPR (match_operand:GPR 1 "gpc_reg_operand" "r"))
+         (not:GPR (match_operand:GPR 2 "gpc_reg_operand" "r"))]))]
   ""
-  "%q3 %0,%1,%2")
+  "%q3 %0,%1,%2"
+  [(set_attr "type" "logical")])
 
-(define_insn "*boolccsi3_internal2"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC (match_operator:SI 4 "boolean_operator"
-        [(not:SI (match_operand:SI 1 "gpc_reg_operand" "r,r"))
-         (not:SI (match_operand:SI 2 "gpc_reg_operand" "r,r"))])
+(define_insn_and_split "*boolcc<mode>3_dot"
+  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
+       (compare:CC (match_operator:GPR 3 "boolean_operator"
+        [(not:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r"))
+         (not:GPR (match_operand:GPR 2 "gpc_reg_operand" "r,r"))])
         (const_int 0)))
-   (clobber (match_scratch:SI 3 "=r,r"))]
-  "TARGET_32BIT"
+   (clobber (match_scratch:GPR 0 "=r,r"))]
+  "<MODE>mode == Pmode && rs6000_gen_cell_microcode"
   "@
-   %q4. %3,%1,%2
+   %q3. %0,%1,%2
    #"
-  [(set_attr "type" "logical,compare")
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[4], CCmode)"
+  [(set (match_dup 0)
+       (match_dup 3))
+   (set (match_dup 4)
+       (compare:CC (match_dup 0)
+                   (const_int 0)))]
+  ""
+  [(set_attr "type" "logical")
    (set_attr "dot" "yes")
    (set_attr "length" "4,8")])
 
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (match_operator:SI 4 "boolean_operator"
-        [(not:SI (match_operand:SI 1 "gpc_reg_operand" ""))
-         (not:SI (match_operand:SI 2 "gpc_reg_operand" ""))])
-        (const_int 0)))
-   (clobber (match_scratch:SI 3 ""))]
-  "TARGET_32BIT && reload_completed"
-  [(set (match_dup 3) (match_dup 4))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
-
-(define_insn "*boolccsi3_internal3"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
-       (compare:CC (match_operator:SI 4 "boolean_operator"
-        [(not:SI (match_operand:SI 1 "gpc_reg_operand" "%r,r"))
-         (not:SI (match_operand:SI 2 "gpc_reg_operand" "r,r"))])
+(define_insn_and_split "*boolcc<mode>3_dot2"
+  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
+       (compare:CC (match_operator:GPR 3 "boolean_operator"
+        [(not:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r"))
+         (not:GPR (match_operand:GPR 2 "gpc_reg_operand" "r,r"))])
         (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
-       (match_dup 4))]
-  "TARGET_32BIT"
+   (set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
+       (match_dup 3))]
+  "<MODE>mode == Pmode && rs6000_gen_cell_microcode"
   "@
-   %q4. %0,%1,%2
+   %q3. %0,%1,%2
    #"
-  [(set_attr "type" "logical,compare")
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[4], CCmode)"
+  [(set (match_dup 0)
+       (match_dup 3))
+   (set (match_dup 4)
+       (compare:CC (match_dup 0)
+                   (const_int 0)))]
+  ""
+  [(set_attr "type" "logical")
    (set_attr "dot" "yes")
    (set_attr "length" "4,8")])
 
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (match_operator:SI 4 "boolean_operator"
-        [(not:SI (match_operand:SI 1 "gpc_reg_operand" ""))
-         (not:SI (match_operand:SI 2 "gpc_reg_operand" ""))])
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (match_dup 4))]
-  "TARGET_32BIT && reload_completed"
-  [(set (match_dup 0) (match_dup 4))
-   (set (match_dup 3)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
+
+;; TODO: Should have dots of this as well.
+(define_insn "*eqv<mode>3"
+  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
+       (not:GPR (xor:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")
+                         (match_operand:GPR 2 "gpc_reg_operand" "r"))))]
+  ""
+  "eqv %0,%1,%2"
+  [(set_attr "type" "logical")])
 \f
 ;; Rotate and shift insns, in all their variants.  These support shifts,
 ;; field inserts and extracts, and various combinations thereof.
     emit_insn (gen_extzvdi_internal (operands[0], operands[1], operands[2],
                                     operands[3]));
   else
-    emit_insn (gen_extzvsi_internal (operands[0], operands[1], operands[2],
-                                    operands[3]));
+    FAIL;
+
   DONE;
 }")
 
-(define_insn "extzvsi_internal"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-       (zero_extract:SI (match_operand:SI 1 "gpc_reg_operand" "r")
-                        (match_operand:SI 2 "const_int_operand" "i")
-                        (match_operand:SI 3 "const_int_operand" "i")))]
-  ""
-  "*
-{
-  int start = INTVAL (operands[3]) & 31;
-  int size = INTVAL (operands[2]) & 31;
-
-  if (start + size >= 32)
-    operands[3] = const0_rtx;
-  else
-    operands[3] = GEN_INT (start + size);
-  return \"rlwinm %0,%1,%3,%s2,31\";
-}"
-  [(set_attr "type" "shift")])
-
-(define_insn "*extzvsi_internal1"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC (zero_extract:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                        (match_operand:SI 2 "const_int_operand" "i,i")
-                        (match_operand:SI 3 "const_int_operand" "i,i"))
-                   (const_int 0)))
-   (clobber (match_scratch:SI 4 "=r,r"))]
-  ""
-  "*
-{
-  int start = INTVAL (operands[3]) & 31;
-  int size = INTVAL (operands[2]) & 31;
-
-  /* Force split for non-cc0 compare.  */
-  if (which_alternative == 1)
-     return \"#\";
-
-  /* If the bit-field being tested fits in the upper or lower half of a
-     word, it is possible to use andiu. or andil. to test it.  This is
-     useful because the condition register set-use delay is smaller for
-     andi[ul]. than for rlinm.  This doesn't work when the starting bit
-     position is 0 because the LT and GT bits may be set wrong.  */
-
-  if ((start > 0 && start + size <= 16) || start >= 16)
-    {
-      operands[3] = GEN_INT (((1 << (16 - (start & 15)))
-                             - (1 << (16 - (start & 15) - size))));
-      if (start < 16)
-       return \"andis. %4,%1,%3\";
-      else
-       return \"andi. %4,%1,%3\";
-    }
-
-  if (start + size >= 32)
-    operands[3] = const0_rtx;
-  else
-    operands[3] = GEN_INT (start + size);
-  return \"rlwinm. %4,%1,%3,%s2,31\";
-}"
-  [(set_attr "type" "shift")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extract:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                        (match_operand:SI 2 "const_int_operand" "")
-                        (match_operand:SI 3 "const_int_operand" ""))
-                   (const_int 0)))
-   (clobber (match_scratch:SI 4 ""))]
-  "reload_completed"
-  [(set (match_dup 4)
-       (zero_extract:SI (match_dup 1) (match_dup 2)
-                        (match_dup 3)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 4)
-                   (const_int 0)))]
-  "")
-
-(define_insn "*extzvsi_internal2"
-  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
-       (compare:CC (zero_extract:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                        (match_operand:SI 2 "const_int_operand" "i,i")
-                        (match_operand:SI 3 "const_int_operand" "i,i"))
-                   (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
-       (zero_extract:SI (match_dup 1) (match_dup 2) (match_dup 3)))]
-  ""
-  "*
-{
-  int start = INTVAL (operands[3]) & 31;
-  int size = INTVAL (operands[2]) & 31;
-
-  /* Force split for non-cc0 compare.  */
-  if (which_alternative == 1)
-     return \"#\";
-
-  /* Since we are using the output value, we can't ignore any need for
-     a shift.  The bit-field must end at the LSB.  */
-  if (start >= 16 && start + size == 32)
-    {
-      operands[3] = GEN_INT ((1 << size) - 1);
-      return \"andi. %0,%1,%3\";
-    }
-
-  if (start + size >= 32)
-    operands[3] = const0_rtx;
-  else
-    operands[3] = GEN_INT (start + size);
-  return \"rlwinm. %0,%1,%3,%s2,31\";
-}"
-  [(set_attr "type" "shift")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 4 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extract:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                        (match_operand:SI 2 "const_int_operand" "")
-                        (match_operand:SI 3 "const_int_operand" ""))
-                   (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (zero_extract:SI (match_dup 1) (match_dup 2) (match_dup 3)))]
-  "reload_completed"
-  [(set (match_dup 0)
-       (zero_extract:SI (match_dup 1) (match_dup 2) (match_dup 3)))
-   (set (match_dup 4)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
-
 (define_insn "extzvdi_internal"
   [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
        (zero_extract:DI (match_operand:DI 1 "gpc_reg_operand" "r")
    (set_attr "dot" "yes")])
 
 (define_insn "*extzvdi_internal2"
-  [(set (match_operand:CC 4 "gpc_reg_operand" "=x")
+  [(set (match_operand:CC 4 "cc_reg_operand" "=x")
        (compare:CC (zero_extract:DI (match_operand:DI 1 "gpc_reg_operand" "r")
                         (match_operand:SI 2 "const_int_operand" "i")
                         (match_operand:SI 3 "const_int_operand" "i"))
   [(set_attr "type" "shift")
    (set_attr "dot" "yes")])
 
-(define_insn "rotlsi3"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
-       (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                  (match_operand:SI 2 "reg_or_cint_operand" "r,i")))]
+
+(define_insn "rotl<mode>3"
+  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
+       (rotate:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")
+                   (match_operand:SI 2 "reg_or_cint_operand" "rn")))]
   ""
-  "@
-   rlwnm %0,%1,%2,0xffffffff
-   rlwinm %0,%1,%h2,0xffffffff"
+  "rotl<wd>%I2 %0,%1,%<hH>2"
   [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no")])
+   (set_attr "maybe_var_shift" "yes")])
 
 (define_insn "*rotlsi3_64"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
-       (zero_extend:DI
-           (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                      (match_operand:SI 2 "reg_or_cint_operand" "r,i"))))]
-  "TARGET_64BIT"
-  "@
-   rlwnm %0,%1,%2,0xffffffff
-   rlwinm %0,%1,%h2,0xffffffff"
+  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
+       (zero_extend:DI
+           (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r")
+                      (match_operand:SI 2 "reg_or_cint_operand" "rn"))))]
+  "TARGET_POWERPC64"
+  "rotlw%I2 %0,%1,%h2"
   [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no")])
+   (set_attr "maybe_var_shift" "yes")])
 
-(define_insn "*rotlsi3_internal2"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r,r,r,r")
-                              (match_operand:SI 2 "reg_or_cint_operand" "r,i,r,i"))
+(define_insn_and_split "*rotl<mode>3_dot"
+  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
+       (compare:CC (rotate:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r")
+                               (match_operand:SI 2 "reg_or_cint_operand" "rn,rn"))
                    (const_int 0)))
-   (clobber (match_scratch:SI 3 "=r,r,r,r"))]
-  ""
+   (clobber (match_scratch:GPR 0 "=r,r"))]
+  "<MODE>mode == Pmode && rs6000_gen_cell_microcode"
   "@
-   rlwnm. %3,%1,%2,0xffffffff
-   rlwinm. %3,%1,%h2,0xffffffff
-   #
+   rotl<wd>%I2. %0,%1,%<hH>2
    #"
-  [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no,yes,no")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                              (match_operand:SI 2 "reg_or_cint_operand" ""))
-                   (const_int 0)))
-   (clobber (match_scratch:SI 3 ""))]
-  "reload_completed"
-  [(set (match_dup 3)
-       (rotate:SI (match_dup 1) (match_dup 2)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
+  [(set (match_dup 0)
+       (rotate:GPR (match_dup 1)
+                   (match_dup 2)))
+   (set (match_dup 3)
+       (compare:CC (match_dup 0)
                    (const_int 0)))]
-  "")
-
-(define_insn "*rotlsi3_internal3"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r,r,r,r")
-                              (match_operand:SI 2 "reg_or_cint_operand" "r,i,r,i"))
-                   (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r,r,r")
-       (rotate:SI (match_dup 1) (match_dup 2)))]
   ""
-  "@
-   rlwnm. %0,%1,%2,0xffffffff
-   rlwinm. %0,%1,%h2,0xffffffff
-   #
-   #"
   [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no,yes,no")
+   (set_attr "maybe_var_shift" "yes")
    (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
+   (set_attr "length" "4,8")])
 
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                              (match_operand:SI 2 "reg_or_cint_operand" ""))
+(define_insn_and_split "*rotl<mode>3_dot2"
+  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
+       (compare:CC (rotate:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r")
+                               (match_operand:SI 2 "reg_or_cint_operand" "rn,rn"))
                    (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (rotate:SI (match_dup 1) (match_dup 2)))]
-  "reload_completed"
+   (set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
+       (rotate:GPR (match_dup 1)
+                   (match_dup 2)))]
+  "<MODE>mode == Pmode && rs6000_gen_cell_microcode"
+  "@
+   rotl<wd>%I2. %0,%1,%<hH>2
+   #"
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
   [(set (match_dup 0)
-       (rotate:SI (match_dup 1) (match_dup 2)))
+       (rotate:GPR (match_dup 1)
+                   (match_dup 2)))
    (set (match_dup 3)
        (compare:CC (match_dup 0)
                    (const_int 0)))]
-  "")
-
-(define_insn "*rotlsi3_internal4"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
-       (and:SI (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                          (match_operand:SI 2 "reg_or_cint_operand" "r,i"))
-               (match_operand:SI 3 "mask_operand" "n,n")))]
-  ""
-  "@
-   rlwnm %0,%1,%2,%m3,%M3
-   rlwinm %0,%1,%h2,%m3,%M3"
-  [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no")])
-
-(define_insn "*rotlsi3_internal5"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC (and:SI
-                    (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r,r,r,r")
-                               (match_operand:SI 2 "reg_or_cint_operand" "r,i,r,i"))
-                    (match_operand:SI 3 "mask_operand" "n,n,n,n"))
-                   (const_int 0)))
-   (clobber (match_scratch:SI 4 "=r,r,r,r"))]
   ""
-  "@
-   rlwnm. %4,%1,%2,%m3,%M3
-   rlwinm. %4,%1,%h2,%m3,%M3
-   #
-   #"
   [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no,yes,no")
+   (set_attr "maybe_var_shift" "yes")
    (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
+   (set_attr "length" "4,8")])
 
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (and:SI
-                    (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                               (match_operand:SI 2 "reg_or_cint_operand" ""))
-                    (match_operand:SI 3 "mask_operand" ""))
-                   (const_int 0)))
-   (clobber (match_scratch:SI 4 ""))]
-  "reload_completed"
-  [(set (match_dup 4)
-       (and:SI (rotate:SI (match_dup 1)
-                               (match_dup 2))
-                    (match_dup 3)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 4)
-                   (const_int 0)))]
-  "")
 
-(define_insn "*rotlsi3_internal6"
-  [(set (match_operand:CC 4 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC (and:SI
-                    (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r,r,r,r")
-                               (match_operand:SI 2 "reg_or_cint_operand" "r,i,r,i"))
-                    (match_operand:SI 3 "mask_operand" "n,n,n,n"))
-                   (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r,r,r")
-       (and:SI (rotate:SI (match_dup 1) (match_dup 2)) (match_dup 3)))]
+(define_insn "*rotlsi3_mask"
+  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
+       (and:SI (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r")
+                          (match_operand:SI 2 "reg_or_cint_operand" "rn"))
+               (match_operand:SI 3 "mask_operand" "n")))]
   ""
-  "@
-   rlwnm. %0,%1,%2,%m3,%M3
-   rlwinm. %0,%1,%h2,%m3,%M3
-   #
-   #"
+  "rlw%I2nm %0,%1,%h2,%m3,%M3"
   [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no,yes,no")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
+   (set_attr "maybe_var_shift" "yes")])
 
-(define_split
-  [(set (match_operand:CC 4 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (and:SI
-                    (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                               (match_operand:SI 2 "reg_or_cint_operand" ""))
-                    (match_operand:SI 3 "mask_operand" ""))
-                   (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (and:SI (rotate:SI (match_dup 1) (match_dup 2)) (match_dup 3)))]
-  "reload_completed"
+(define_insn_and_split "*rotlsi3_mask_dot"
+  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
+       (compare:CC
+        (and:SI (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
+                           (match_operand:SI 2 "reg_or_cint_operand" "rn,rn"))
+                (match_operand:SI 3 "mask_operand" "n,n"))
+        (const_int 0)))
+   (clobber (match_scratch:SI 0 "=r,r"))]
+  "rs6000_gen_cell_microcode
+   && (TARGET_32BIT || UINTVAL (operands[3]) <= 0x7fffffff)"
+  "@
+   rlw%I2nm. %0,%1,%h2,%m3,%M3
+   #"
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[4], CCmode)"
   [(set (match_dup 0)
-       (and:SI (rotate:SI (match_dup 1) (match_dup 2)) (match_dup 3)))
+       (and:SI (rotate:SI (match_dup 1)
+                          (match_dup 2))
+               (match_dup 3)))
    (set (match_dup 4)
        (compare:CC (match_dup 0)
                    (const_int 0)))]
-  "")
-
-(define_insn "*rotlsi3_internal7le"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-       (zero_extend:SI
-        (subreg:QI
-         (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r")
-                    (match_operand:SI 2 "reg_or_cint_operand" "ri")) 0)))]
-  "!BYTES_BIG_ENDIAN"
-  "rlw%I2nm %0,%1,%h2,0xff"
-  [(set (attr "cell_micro")
-     (if_then_else (match_operand:SI 2 "const_int_operand" "")
-       (const_string "not")
-       (const_string "always")))
-   (set_attr "type" "shift")])
-
-(define_insn "*rotlsi3_internal7be"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-       (zero_extend:SI
-        (subreg:QI
-         (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r")
-                    (match_operand:SI 2 "reg_or_cint_operand" "ri")) 3)))]
-  "BYTES_BIG_ENDIAN"
-  "rlw%I2nm %0,%1,%h2,0xff"
-  [(set (attr "cell_micro")
-     (if_then_else (match_operand:SI 2 "const_int_operand" "")
-       (const_string "not")
-       (const_string "always")))
-   (set_attr "type" "shift")])
-
-(define_insn "*rotlsi3_internal8le"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC (zero_extend:SI
-                    (subreg:QI
-                     (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r,r,r,r")
-                                (match_operand:SI 2 "reg_or_cint_operand" "r,i,r,i")) 0))
-                   (const_int 0)))
-   (clobber (match_scratch:SI 3 "=r,r,r,r"))]
-  "!BYTES_BIG_ENDIAN"
-  "@
-   rlwnm. %3,%1,%2,0xff
-   rlwinm. %3,%1,%h2,0xff
-   #
-   #"
+  ""
   [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no,yes,no")
+   (set_attr "maybe_var_shift" "yes")
    (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
+   (set_attr "length" "4,8")])
 
-(define_insn "*rotlsi3_internal8be"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC (zero_extend:SI
-                    (subreg:QI
-                     (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r,r,r,r")
-                                (match_operand:SI 2 "reg_or_cint_operand" "r,i,r,i")) 3))
-                   (const_int 0)))
-   (clobber (match_scratch:SI 3 "=r,r,r,r"))]
-  "BYTES_BIG_ENDIAN"
+(define_insn_and_split "*rotlsi3_mask_dot2"
+  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
+       (compare:CC
+        (and:SI (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
+                           (match_operand:SI 2 "reg_or_cint_operand" "rn,rn"))
+                (match_operand:SI 3 "mask_operand" "n,n"))
+        (const_int 0)))
+   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
+       (and:SI (rotate:SI (match_dup 1)
+                          (match_dup 2))
+               (match_dup 3)))]
+  "rs6000_gen_cell_microcode
+   && (TARGET_32BIT || UINTVAL (operands[3]) <= 0x7fffffff)"
   "@
-   rlwnm. %3,%1,%2,0xff
-   rlwinm. %3,%1,%h2,0xff
-   #
+   rlw%I2nm. %0,%1,%h2,%m3,%M3
    #"
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[4], CCmode)"
+  [(set (match_dup 0)
+       (and:SI (rotate:SI (match_dup 1)
+                          (match_dup 2))
+               (match_dup 3)))
+   (set (match_dup 4)
+       (compare:CC (match_dup 0)
+                   (const_int 0)))]
+  ""
   [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no,yes,no")
+   (set_attr "maybe_var_shift" "yes")
    (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
+   (set_attr "length" "4,8")])
 
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:SI
-                    (subreg:QI
-                     (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                                (match_operand:SI 2 "reg_or_cint_operand" "")) 0))
-                   (const_int 0)))
-   (clobber (match_scratch:SI 3 ""))]
-  "!BYTES_BIG_ENDIAN && reload_completed"
-  [(set (match_dup 3)
-       (zero_extend:SI (subreg:QI
-                     (rotate:SI (match_dup 1)
-                                (match_dup 2)) 0)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
 
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:SI
-                    (subreg:QI
-                     (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                                (match_operand:SI 2 "reg_or_cint_operand" "")) 3))
-                   (const_int 0)))
-   (clobber (match_scratch:SI 3 ""))]
-  "BYTES_BIG_ENDIAN && reload_completed"
-  [(set (match_dup 3)
-       (zero_extend:SI (subreg:QI
-                     (rotate:SI (match_dup 1)
-                                (match_dup 2)) 3)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
+(define_insn "ashl<mode>3"
+  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
+       (ashift:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")
+                   (match_operand:SI 2 "reg_or_cint_operand" "rn")))]
+  ""
+  "sl<wd>%I2 %0,%1,%<hH>2"
+  [(set_attr "type" "shift")
+   (set_attr "maybe_var_shift" "yes")])
 
-(define_insn "*rotlsi3_internal9le"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC (zero_extend:SI
-                    (subreg:QI
-                     (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r,r,r,r")
-                                (match_operand:SI 2 "reg_or_cint_operand" "r,i,r,i")) 0))
-                   (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r,r,r")
-       (zero_extend:SI (subreg:QI (rotate:SI (match_dup 1) (match_dup 2)) 0)))]
-  "!BYTES_BIG_ENDIAN"
-  "@
-   rlwnm. %0,%1,%2,0xff
-   rlwinm. %0,%1,%h2,0xff
-   #
-   #"
+(define_insn "*ashlsi3_64"
+  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
+       (zero_extend:DI
+           (ashift:SI (match_operand:SI 1 "gpc_reg_operand" "r")
+                      (match_operand:SI 2 "reg_or_cint_operand" "rn"))))]
+  "TARGET_POWERPC64"
+  "slw%I2 %0,%1,%h2"
   [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no,yes,no")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
+   (set_attr "maybe_var_shift" "yes")])
 
-(define_insn "*rotlsi3_internal9be"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC (zero_extend:SI
-                    (subreg:QI
-                     (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r,r,r,r")
-                                (match_operand:SI 2 "reg_or_cint_operand" "r,i,r,i")) 3))
+(define_insn_and_split "*ashl<mode>3_dot"
+  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
+       (compare:CC (ashift:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r")
+                               (match_operand:SI 2 "reg_or_cint_operand" "rn,rn"))
                    (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r,r,r")
-       (zero_extend:SI (subreg:QI (rotate:SI (match_dup 1) (match_dup 2)) 3)))]
-  "BYTES_BIG_ENDIAN"
+   (clobber (match_scratch:GPR 0 "=r,r"))]
+  "<MODE>mode == Pmode && rs6000_gen_cell_microcode"
   "@
-   rlwnm. %0,%1,%2,0xff
-   rlwinm. %0,%1,%h2,0xff
-   #
+   sl<wd>%I2. %0,%1,%<hH>2
    #"
-  [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no,yes,no")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
-
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:SI
-                    (subreg:QI
-                     (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                                (match_operand:SI 2 "reg_or_cint_operand" "")) 0))
-                   (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (zero_extend:SI (subreg:QI (rotate:SI (match_dup 1) (match_dup 2)) 0)))]
-  "!BYTES_BIG_ENDIAN && reload_completed"
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
   [(set (match_dup 0)
-       (zero_extend:SI (subreg:QI (rotate:SI (match_dup 1) (match_dup 2)) 0)))
+       (ashift:GPR (match_dup 1)
+                   (match_dup 2)))
    (set (match_dup 3)
        (compare:CC (match_dup 0)
                    (const_int 0)))]
-  "")
+  ""
+  [(set_attr "type" "shift")
+   (set_attr "maybe_var_shift" "yes")
+   (set_attr "dot" "yes")
+   (set_attr "length" "4,8")])
 
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:SI
-                    (subreg:QI
-                     (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                                (match_operand:SI 2 "reg_or_cint_operand" "")) 3))
+(define_insn_and_split "*ashl<mode>3_dot2"
+  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
+       (compare:CC (ashift:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r")
+                               (match_operand:SI 2 "reg_or_cint_operand" "rn,rn"))
                    (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (zero_extend:SI (subreg:QI (rotate:SI (match_dup 1) (match_dup 2)) 3)))]
-  "BYTES_BIG_ENDIAN && reload_completed"
+   (set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
+       (ashift:GPR (match_dup 1)
+                   (match_dup 2)))]
+  "<MODE>mode == Pmode && rs6000_gen_cell_microcode"
+  "@
+   sl<wd>%I2. %0,%1,%<hH>2
+   #"
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
   [(set (match_dup 0)
-       (zero_extend:SI (subreg:QI (rotate:SI (match_dup 1) (match_dup 2)) 3)))
+       (ashift:GPR (match_dup 1)
+                   (match_dup 2)))
    (set (match_dup 3)
        (compare:CC (match_dup 0)
                    (const_int 0)))]
-  "")
-
-(define_insn "*rotlsi3_internal10le"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
-       (zero_extend:SI
-        (subreg:HI
-         (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                    (match_operand:SI 2 "reg_or_cint_operand" "r,i")) 0)))]
-  "!BYTES_BIG_ENDIAN"
-  "@
-   rlwnm %0,%1,%2,0xffff
-   rlwinm %0,%1,%h2,0xffff"
+  ""
   [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no")])
-
-(define_insn "*rotlsi3_internal10be"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
-       (zero_extend:SI
-        (subreg:HI
-         (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                    (match_operand:SI 2 "reg_or_cint_operand" "r,i")) 2)))]
-  "BYTES_BIG_ENDIAN"
-  "@
-   rlwnm %0,%1,%2,0xffff
-   rlwinm %0,%1,%h2,0xffff"
-  [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no")])
-
-(define_insn "*rotlsi3_internal11le"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC (zero_extend:SI
-                    (subreg:HI
-                     (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r,r,r,r")
-                                (match_operand:SI 2 "reg_or_cint_operand" "r,i,r,i")) 0))
-                   (const_int 0)))
-   (clobber (match_scratch:SI 3 "=r,r,r,r"))]
-  "!BYTES_BIG_ENDIAN"
-  "@
-   rlwnm. %3,%1,%2,0xffff
-   rlwinm. %3,%1,%h2,0xffff
-   #
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no,yes,no")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
-
-(define_insn "*rotlsi3_internal11be"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC (zero_extend:SI
-                    (subreg:HI
-                     (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r,r,r,r")
-                                (match_operand:SI 2 "reg_or_cint_operand" "r,i,r,i")) 2))
-                   (const_int 0)))
-   (clobber (match_scratch:SI 3 "=r,r,r,r"))]
-  "BYTES_BIG_ENDIAN"
-  "@
-   rlwnm. %3,%1,%2,0xffff
-   rlwinm. %3,%1,%h2,0xffff
-   #
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no,yes,no")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:SI
-                    (subreg:HI
-                     (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                                (match_operand:SI 2 "reg_or_cint_operand" "")) 0))
-                   (const_int 0)))
-   (clobber (match_scratch:SI 3 ""))]
-  "!BYTES_BIG_ENDIAN && reload_completed"
-  [(set (match_dup 3)
-       (zero_extend:SI (subreg:HI
-                     (rotate:SI (match_dup 1)
-                                (match_dup 2)) 0)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:SI
-                    (subreg:HI
-                     (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                                (match_operand:SI 2 "reg_or_cint_operand" "")) 2))
-                   (const_int 0)))
-   (clobber (match_scratch:SI 3 ""))]
-  "BYTES_BIG_ENDIAN && reload_completed"
-  [(set (match_dup 3)
-       (zero_extend:SI (subreg:HI
-                     (rotate:SI (match_dup 1)
-                                (match_dup 2)) 2)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
-
-(define_insn "*rotlsi3_internal12le"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC (zero_extend:SI
-                    (subreg:HI
-                     (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r,r,r,r")
-                                (match_operand:SI 2 "reg_or_cint_operand" "r,i,r,i")) 0))
-                   (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r,r,r")
-       (zero_extend:SI (subreg:HI (rotate:SI (match_dup 1) (match_dup 2)) 0)))]
-  "!BYTES_BIG_ENDIAN"
-  "@
-   rlwnm. %0,%1,%2,0xffff
-   rlwinm. %0,%1,%h2,0xffff
-   #
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no,yes,no")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
-
-(define_insn "*rotlsi3_internal12be"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC (zero_extend:SI
-                    (subreg:HI
-                     (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "r,r,r,r")
-                                (match_operand:SI 2 "reg_or_cint_operand" "r,i,r,i")) 2))
-                   (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r,r,r")
-       (zero_extend:SI (subreg:HI (rotate:SI (match_dup 1) (match_dup 2)) 2)))]
-  "BYTES_BIG_ENDIAN"
-  "@
-   rlwnm. %0,%1,%2,0xffff
-   rlwinm. %0,%1,%h2,0xffff
-   #
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no,yes,no")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
-
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:SI
-                    (subreg:HI
-                     (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                                (match_operand:SI 2 "reg_or_cint_operand" "")) 0))
-                   (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (zero_extend:SI (subreg:HI (rotate:SI (match_dup 1) (match_dup 2)) 0)))]
-  "!BYTES_BIG_ENDIAN && reload_completed"
-  [(set (match_dup 0)
-       (zero_extend:SI (subreg:HI (rotate:SI (match_dup 1) (match_dup 2)) 0)))
-   (set (match_dup 3)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
-
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:SI
-                    (subreg:HI
-                     (rotate:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                                (match_operand:SI 2 "reg_or_cint_operand" "")) 2))
-                   (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (zero_extend:SI (subreg:HI (rotate:SI (match_dup 1) (match_dup 2)) 2)))]
-  "BYTES_BIG_ENDIAN && reload_completed"
-  [(set (match_dup 0)
-       (zero_extend:SI (subreg:HI (rotate:SI (match_dup 1) (match_dup 2)) 2)))
-   (set (match_dup 3)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
-
-(define_insn "ashlsi3"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
-       (ashift:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                  (match_operand:SI 2 "reg_or_cint_operand" "r,i")))]
-  ""
-  "@
-   slw %0,%1,%2
-   slwi %0,%1,%h2"
-  [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no")])
-
-(define_insn "*ashlsi3_64"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
-       (zero_extend:DI
-           (ashift:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                      (match_operand:SI 2 "reg_or_cint_operand" "r,i"))))]
-  "TARGET_POWERPC64"
-  "@
-   slw %0,%1,%2
-   slwi %0,%1,%h2"
-  [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no")])
-
-(define_insn ""
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC (ashift:SI (match_operand:SI 1 "gpc_reg_operand" "r,r,r,r")
-                              (match_operand:SI 2 "reg_or_cint_operand" "r,i,r,i"))
-                   (const_int 0)))
-   (clobber (match_scratch:SI 3 "=r,r,r,r"))]
-  "TARGET_32BIT"
-  "@
-   slw. %3,%1,%2
-   slwi. %3,%1,%h2
-   #
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no,yes,no")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_cr0_operand" "")
-       (compare:CC (ashift:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                              (match_operand:SI 2 "reg_or_cint_operand" ""))
-                   (const_int 0)))
-   (clobber (match_scratch:SI 3 ""))]
-  "TARGET_32BIT && reload_completed"
-  [(set (match_dup 3)
-       (ashift:SI (match_dup 1) (match_dup 2)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
-
-(define_insn ""
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC (ashift:SI (match_operand:SI 1 "gpc_reg_operand" "r,r,r,r")
-                              (match_operand:SI 2 "reg_or_cint_operand" "r,i,r,i"))
-                   (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r,r,r")
-       (ashift:SI (match_dup 1) (match_dup 2)))]
-  "TARGET_32BIT"
-  "@
-   slw. %0,%1,%2
-   slwi. %0,%1,%h2
-   #
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no,yes,no")
+   (set_attr "maybe_var_shift" "yes")
    (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
+   (set_attr "length" "4,8")])
 
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_cr0_operand" "")
-       (compare:CC (ashift:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                              (match_operand:SI 2 "reg_or_cint_operand" ""))
-                   (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (ashift:SI (match_dup 1) (match_dup 2)))]
-  "TARGET_32BIT && reload_completed"
-  [(set (match_dup 0)
-       (ashift:SI (match_dup 1) (match_dup 2)))
-   (set (match_dup 3)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
 
-(define_insn "rlwinm"
+(define_insn "*ashlsi3_imm_mask"
   [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (and:SI (ashift:SI (match_operand:SI 1 "gpc_reg_operand" "r")
                           (match_operand:SI 2 "const_int_operand" "i"))
   "rlwinm %0,%1,%h2,%m3,%M3"
   [(set_attr "type" "shift")])
 
-(define_insn ""
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
+(define_insn_and_split "*ashlsi3_imm_mask_dot"
+  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
        (compare:CC
         (and:SI (ashift:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
                            (match_operand:SI 2 "const_int_operand" "i,i"))
                 (match_operand:SI 3 "mask_operand" "n,n"))
         (const_int 0)))
-   (clobber (match_scratch:SI 4 "=r,r"))]
-  "includes_lshift_p (operands[2], operands[3])"
+   (clobber (match_scratch:SI 0 "=r,r"))]
+  "rs6000_gen_cell_microcode
+   && (TARGET_32BIT || UINTVAL (operands[3]) <= 0x7fffffff)
+   && includes_lshift_p (operands[2], operands[3])"
   "@
-   rlwinm. %4,%1,%h2,%m3,%M3
+   rlwinm. %0,%1,%h2,%m3,%M3
    #"
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[4], CCmode)"
+  [(set (match_dup 0)
+       (and:SI (ashift:SI (match_dup 1)
+                          (match_dup 2))
+               (match_dup 3)))
+   (set (match_dup 4)
+       (compare:CC (match_dup 0)
+                   (const_int 0)))]
+  ""
   [(set_attr "type" "shift")
    (set_attr "dot" "yes")
    (set_attr "length" "4,8")])
 
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC
-        (and:SI (ashift:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                           (match_operand:SI 2 "const_int_operand" ""))
-                (match_operand:SI 3 "mask_operand" ""))
-        (const_int 0)))
-   (clobber (match_scratch:SI 4 ""))]
-  "includes_lshift_p (operands[2], operands[3]) && reload_completed"
-  [(set (match_dup 4)
-       (and:SI (ashift:SI (match_dup 1) (match_dup 2))
-                (match_dup 3)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 4)
-                   (const_int 0)))]
-  "")
-
-(define_insn ""
+(define_insn_and_split "*ashlsi3_imm_mask_dot2"
   [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
        (compare:CC
         (and:SI (ashift:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
                 (match_operand:SI 3 "mask_operand" "n,n"))
         (const_int 0)))
    (set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
-       (and:SI (ashift:SI (match_dup 1) (match_dup 2)) (match_dup 3)))]
-  "includes_lshift_p (operands[2], operands[3])"
+       (and:SI (ashift:SI (match_dup 1)
+                          (match_dup 2))
+               (match_dup 3)))]
+  "rs6000_gen_cell_microcode
+   && (TARGET_32BIT || UINTVAL (operands[3]) <= 0x7fffffff)
+   && includes_lshift_p (operands[2], operands[3])"
   "@
    rlwinm. %0,%1,%h2,%m3,%M3
    #"
-  [(set_attr "type" "shift")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 4 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC
-        (and:SI (ashift:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                           (match_operand:SI 2 "const_int_operand" ""))
-                (match_operand:SI 3 "mask_operand" ""))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (and:SI (ashift:SI (match_dup 1) (match_dup 2)) (match_dup 3)))]
-  "includes_lshift_p (operands[2], operands[3]) && reload_completed"
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[4], CCmode)"
   [(set (match_dup 0)
-       (and:SI (ashift:SI (match_dup 1) (match_dup 2)) (match_dup 3)))
+       (and:SI (ashift:SI (match_dup 1)
+                          (match_dup 2))
+               (match_dup 3)))
    (set (match_dup 4)
        (compare:CC (match_dup 0)
                    (const_int 0)))]
-  "")
+  ""
+  [(set_attr "type" "shift")
+   (set_attr "dot" "yes")
+   (set_attr "length" "4,8")])
 
-(define_insn "lshrsi3"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r,r,r")
-       (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r,r")
-                    (match_operand:SI 2 "reg_or_cint_operand" "O,r,i")))]
+
+(define_insn "lshr<mode>3"
+  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
+       (lshiftrt:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")
+                     (match_operand:SI 2 "reg_or_cint_operand" "rn")))]
   ""
-  "@
-  mr %0,%1
-  srw %0,%1,%2
-  srwi %0,%1,%h2"
-  [(set_attr "type" "integer,shift,shift")
-   (set_attr "var_shift" "no,yes,no")])
+  "sr<wd>%I2 %0,%1,%<hH>2"
+  [(set_attr "type" "shift")
+   (set_attr "maybe_var_shift" "yes")])
 
 (define_insn "*lshrsi3_64"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
-       (zero_extend:DI
-           (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                        (match_operand:SI 2 "reg_or_cint_operand" "r,i"))))]
+  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
+       (zero_extend:DI
+           (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r")
+                        (match_operand:SI 2 "reg_or_cint_operand" "rn"))))]
   "TARGET_POWERPC64"
-  "@
-  srw %0,%1,%2
-  srwi %0,%1,%h2"
+  "srw%I2 %0,%1,%h2"
   [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no")])
+   (set_attr "maybe_var_shift" "yes")])
 
-(define_insn ""
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,x,x,?y,?y,?y")
-       (compare:CC (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r,r,r,r,r")
-                                (match_operand:SI 2 "reg_or_cint_operand" "O,r,i,O,r,i"))
+(define_insn_and_split "*lshr<mode>3_dot"
+  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
+       (compare:CC (lshiftrt:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r")
+                                 (match_operand:SI 2 "reg_or_cint_operand" "rn,rn"))
                    (const_int 0)))
-   (clobber (match_scratch:SI 3 "=X,r,r,X,r,r"))]
-  "TARGET_32BIT"
+   (clobber (match_scratch:GPR 0 "=r,r"))]
+  "<MODE>mode == Pmode && rs6000_gen_cell_microcode"
   "@
-   mr. %1,%1
-   srw. %3,%1,%2
-   srwi. %3,%1,%h2
-   #
-   #
+   sr<wd>%I2. %0,%1,%<hH>2
    #"
-  [(set_attr "type" "logical,shift,shift,shift,shift,shift")
-   (set_attr "var_shift" "no,yes,no,no,yes,no")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,4,4,8,8,8")])
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_cr0_operand" "")
-       (compare:CC (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                                (match_operand:SI 2 "reg_or_cint_operand" ""))
-                   (const_int 0)))
-   (clobber (match_scratch:SI 3 ""))]
-  "TARGET_32BIT && reload_completed"
-  [(set (match_dup 3)
-       (lshiftrt:SI (match_dup 1) (match_dup 2)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
+  [(set (match_dup 0)
+       (lshiftrt:GPR (match_dup 1)
+                     (match_dup 2)))
+   (set (match_dup 3)
+       (compare:CC (match_dup 0)
                    (const_int 0)))]
-  "")
+  ""
+  [(set_attr "type" "shift")
+   (set_attr "maybe_var_shift" "yes")
+   (set_attr "dot" "yes")
+   (set_attr "length" "4,8")])
 
-(define_insn ""
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,x,x,?y,?y,?y")
-       (compare:CC (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r,r,r,r,r")
-                                (match_operand:SI 2 "reg_or_cint_operand" "O,r,i,O,r,i"))
+(define_insn_and_split "*lshr<mode>3_dot2"
+  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
+       (compare:CC (lshiftrt:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r")
+                                 (match_operand:SI 2 "reg_or_cint_operand" "rn,rn"))
                    (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r,r,r,r,r")
-       (lshiftrt:SI (match_dup 1) (match_dup 2)))]
-  "TARGET_32BIT"
+   (set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
+       (lshiftrt:GPR (match_dup 1)
+                     (match_dup 2)))]
+  "<MODE>mode == Pmode && rs6000_gen_cell_microcode"
   "@
-   mr. %0,%1
-   srw. %0,%1,%2
-   srwi. %0,%1,%h2
-   #
-   #
+   sr<wd>%I2. %0,%1,%<hH>2
    #"
-  [(set_attr "type" "logical,shift,shift,shift,shift,shift")
-   (set_attr "var_shift" "no,yes,no,no,yes,no")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,4,4,8,8,8")])
-
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_cr0_operand" "")
-       (compare:CC (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                                (match_operand:SI 2 "reg_or_cint_operand" ""))
-                   (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (lshiftrt:SI (match_dup 1) (match_dup 2)))]
-  "TARGET_32BIT && reload_completed"
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
   [(set (match_dup 0)
-       (lshiftrt:SI (match_dup 1) (match_dup 2)))
+       (lshiftrt:GPR (match_dup 1)
+                     (match_dup 2)))
    (set (match_dup 3)
        (compare:CC (match_dup 0)
                    (const_int 0)))]
-  "")
+  ""
+  [(set_attr "type" "shift")
+   (set_attr "maybe_var_shift" "yes")
+   (set_attr "dot" "yes")
+   (set_attr "length" "4,8")])
 
-(define_insn ""
+
+(define_insn "*lshrsi3_imm_mask"
   [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
        (and:SI (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r")
                             (match_operand:SI 2 "const_int_operand" "i"))
   "rlwinm %0,%1,%s2,%m3,%M3"
   [(set_attr "type" "shift")])
 
-(define_insn ""
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
+(define_insn_and_split "*lshrsi3_imm_mask_dot"
+  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
        (compare:CC
         (and:SI (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
                              (match_operand:SI 2 "const_int_operand" "i,i"))
                 (match_operand:SI 3 "mask_operand" "n,n"))
         (const_int 0)))
-   (clobber (match_scratch:SI 4 "=r,r"))]
-  "includes_rshift_p (operands[2], operands[3])"
+   (clobber (match_scratch:SI 0 "=r,r"))]
+  "rs6000_gen_cell_microcode
+   && (TARGET_32BIT || UINTVAL (operands[3]) <= 0x7fffffff)
+   && includes_rshift_p (operands[2], operands[3])"
   "@
-   rlwinm. %4,%1,%s2,%m3,%M3
+   rlwinm. %0,%1,%s2,%m3,%M3
    #"
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[4], CCmode)"
+  [(set (match_dup 0)
+       (and:SI (lshiftrt:SI (match_dup 1)
+                            (match_dup 2))
+               (match_dup 3)))
+   (set (match_dup 4)
+       (compare:CC (match_dup 0)
+                   (const_int 0)))]
+  ""
   [(set_attr "type" "shift")
    (set_attr "dot" "yes")
    (set_attr "length" "4,8")])
 
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC
-        (and:SI (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                             (match_operand:SI 2 "const_int_operand" ""))
-                (match_operand:SI 3 "mask_operand" ""))
-        (const_int 0)))
-   (clobber (match_scratch:SI 4 ""))]
-  "includes_rshift_p (operands[2], operands[3]) && reload_completed"
-  [(set (match_dup 4)
-       (and:SI (lshiftrt:SI (match_dup 1) (match_dup 2))
-                (match_dup 3)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 4)
-                   (const_int 0)))]
-  "")
-
-(define_insn ""
+(define_insn_and_split "*lshrsi3_imm_mask_dot2"
   [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
        (compare:CC
         (and:SI (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
                 (match_operand:SI 3 "mask_operand" "n,n"))
         (const_int 0)))
    (set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
-       (and:SI (lshiftrt:SI (match_dup 1) (match_dup 2)) (match_dup 3)))]
-  "includes_rshift_p (operands[2], operands[3])"
+       (and:SI (lshiftrt:SI (match_dup 1)
+                            (match_dup 2))
+               (match_dup 3)))]
+  "rs6000_gen_cell_microcode
+   && (TARGET_32BIT || UINTVAL (operands[3]) <= 0x7fffffff)
+   && includes_rshift_p (operands[2], operands[3])"
   "@
    rlwinm. %0,%1,%s2,%m3,%M3
    #"
-  [(set_attr "type" "shift")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 4 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC
-        (and:SI (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                             (match_operand:SI 2 "const_int_operand" ""))
-                (match_operand:SI 3 "mask_operand" ""))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (and:SI (lshiftrt:SI (match_dup 1) (match_dup 2)) (match_dup 3)))]
-  "includes_rshift_p (operands[2], operands[3]) && reload_completed"
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[4], CCmode)"
   [(set (match_dup 0)
-       (and:SI (lshiftrt:SI (match_dup 1) (match_dup 2)) (match_dup 3)))
+       (and:SI (lshiftrt:SI (match_dup 1)
+                            (match_dup 2))
+               (match_dup 3)))
    (set (match_dup 4)
        (compare:CC (match_dup 0)
                    (const_int 0)))]
-  "")
+  ""
+  [(set_attr "type" "shift")
+   (set_attr "dot" "yes")
+   (set_attr "length" "4,8")])
 
-(define_insn "*lshiftrt_internal1le"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-       (zero_extend:SI
-        (subreg:QI
-         (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r")
-                      (match_operand:SI 2 "const_int_operand" "i")) 0)))]
-  "!BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (255))"
-  "rlwinm %0,%1,%s2,0xff"
-  [(set_attr "type" "shift")])
 
-(define_insn "*lshiftrt_internal1be"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-       (zero_extend:SI
-        (subreg:QI
-         (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r")
-                      (match_operand:SI 2 "const_int_operand" "i")) 3)))]
-  "BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (255))"
-  "rlwinm %0,%1,%s2,0xff"
-  [(set_attr "type" "shift")])
+(define_expand "ashr<mode>3"
+  [(parallel [(set (match_operand:GPR 0 "gpc_reg_operand" "")
+                  (ashiftrt:GPR (match_operand:GPR 1 "gpc_reg_operand" "")
+                                (match_operand:SI 2 "reg_or_cint_operand" "")))
+             (clobber (reg:GPR CA_REGNO))])]
+  ""
+{
+  /* The generic code does not generate optimal code for the low word
+     (it should be a rlwimi and a rot).  Until we have target code to
+     solve this generically, keep this expander.  */
 
-(define_insn "*lshiftrt_internal2le"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (zero_extend:SI
-         (subreg:QI
-          (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                       (match_operand:SI 2 "const_int_operand" "i,i")) 0))
-        (const_int 0)))
-   (clobber (match_scratch:SI 3 "=r,r"))]
-  "!BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (255))"
-  "@
-   rlwinm. %3,%1,%s2,0xff
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
+  if (<MODE>mode == DImode && !TARGET_POWERPC64)
+    {
+      if (CONST_INT_P (operands[2]))
+       {
+         emit_insn (gen_ashrdi3_no_power (operands[0], operands[1], operands[2]));
+         DONE;
+       }
+      else
+       FAIL;
+    }
+})
 
-(define_insn "*lshiftrt_internal2be"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (zero_extend:SI
-         (subreg:QI
-          (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                       (match_operand:SI 2 "const_int_operand" "i,i")) 3))
-        (const_int 0)))
-   (clobber (match_scratch:SI 3 "=r,r"))]
-  "BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (255))"
-  "@
-   rlwinm. %3,%1,%s2,0xff
-   #"
+(define_insn "*ashr<mode>3"
+  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
+       (ashiftrt:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")
+                     (match_operand:SI 2 "reg_or_cint_operand" "rn")))
+   (clobber (reg:GPR CA_REGNO))]
+  ""
+  "sra<wd>%I2 %0,%1,%<hH>2"
   [(set_attr "type" "shift")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC
-        (zero_extend:SI
-         (subreg:QI
-          (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                       (match_operand:SI 2 "const_int_operand" "")) 0))
-        (const_int 0)))
-   (clobber (match_scratch:SI 3 ""))]
-  "!BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (255)) && reload_completed"
-  [(set (match_dup 3)
-       (zero_extend:SI (subreg:QI
-          (lshiftrt:SI (match_dup 1)
-                       (match_dup 2)) 0)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC
-        (zero_extend:SI
-         (subreg:QI
-          (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                       (match_operand:SI 2 "const_int_operand" "")) 3))
-        (const_int 0)))
-   (clobber (match_scratch:SI 3 ""))]
-  "BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (255)) && reload_completed"
-  [(set (match_dup 3)
-       (zero_extend:SI (subreg:QI
-          (lshiftrt:SI (match_dup 1)
-                       (match_dup 2)) 3)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
+   (set_attr "maybe_var_shift" "yes")])
 
-(define_insn "*lshiftrt_internal3le"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (zero_extend:SI
-         (subreg:QI
-          (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                       (match_operand:SI 2 "const_int_operand" "i,i")) 0))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
-       (zero_extend:SI (subreg:QI (lshiftrt:SI (match_dup 1) (match_dup 2)) 0)))]
-  "!BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (255))"
-  "@
-   rlwinm. %0,%1,%s2,0xff
-   #"
+(define_insn "*ashrsi3_64"
+  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
+       (sign_extend:DI
+           (ashiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r")
+                        (match_operand:SI 2 "reg_or_cint_operand" "rn"))))
+   (clobber (reg:SI CA_REGNO))]
+  "TARGET_POWERPC64"
+  "sraw%I2 %0,%1,%h2"
   [(set_attr "type" "shift")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
+   (set_attr "maybe_var_shift" "yes")])
 
-(define_insn "*lshiftrt_internal3be"
+(define_insn_and_split "*ashr<mode>3_dot"
   [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (zero_extend:SI
-         (subreg:QI
-          (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                       (match_operand:SI 2 "const_int_operand" "i,i")) 3))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
-       (zero_extend:SI (subreg:QI (lshiftrt:SI (match_dup 1) (match_dup 2)) 3)))]
-  "BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (255))"
+       (compare:CC (ashiftrt:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r")
+                                 (match_operand:SI 2 "reg_or_cint_operand" "rn,rn"))
+                   (const_int 0)))
+   (clobber (match_scratch:GPR 0 "=r,r"))
+   (clobber (reg:GPR CA_REGNO))]
+  "<MODE>mode == Pmode && rs6000_gen_cell_microcode"
   "@
-   rlwinm. %0,%1,%s2,0xff
+   sra<wd>%I2. %0,%1,%<hH>2
    #"
-  [(set_attr "type" "shift")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC
-        (zero_extend:SI
-         (subreg:QI
-          (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                       (match_operand:SI 2 "const_int_operand" "")) 0))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (zero_extend:SI (subreg:QI (lshiftrt:SI (match_dup 1) (match_dup 2)) 0)))]
-  "!BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (255)) && reload_completed"
-  [(set (match_dup 0)
-       (zero_extend:SI (subreg:QI (lshiftrt:SI (match_dup 1) (match_dup 2)) 0)))
-   (set (match_dup 3)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
-
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC
-        (zero_extend:SI
-         (subreg:QI
-          (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                       (match_operand:SI 2 "const_int_operand" "")) 3))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (zero_extend:SI (subreg:QI (lshiftrt:SI (match_dup 1) (match_dup 2)) 3)))]
-  "BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (255)) && reload_completed"
-  [(set (match_dup 0)
-       (zero_extend:SI (subreg:QI (lshiftrt:SI (match_dup 1) (match_dup 2)) 3)))
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
+  [(parallel [(set (match_dup 0)
+                  (ashiftrt:GPR (match_dup 1)
+                                (match_dup 2)))
+             (clobber (reg:GPR CA_REGNO))])
    (set (match_dup 3)
        (compare:CC (match_dup 0)
                    (const_int 0)))]
-  "")
-
-(define_insn "*lshiftrt_internal4le"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-       (zero_extend:SI
-        (subreg:HI
-         (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r")
-                      (match_operand:SI 2 "const_int_operand" "i")) 0)))]
-  "!BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (65535))"
-  "rlwinm %0,%1,%s2,0xffff"
-  [(set_attr "type" "shift")])
-
-(define_insn "*lshiftrt_internal4be"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-       (zero_extend:SI
-        (subreg:HI
-         (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r")
-                      (match_operand:SI 2 "const_int_operand" "i")) 2)))]
-  "BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (65535))"
-  "rlwinm %0,%1,%s2,0xffff"
-  [(set_attr "type" "shift")])
-
-(define_insn "*lshiftrt_internal5le"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (zero_extend:SI
-         (subreg:HI
-          (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                       (match_operand:SI 2 "const_int_operand" "i,i")) 0))
-        (const_int 0)))
-   (clobber (match_scratch:SI 3 "=r,r"))]
-  "!BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (65535))"
-  "@
-   rlwinm. %3,%1,%s2,0xffff
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_insn "*lshiftrt_internal5be"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (zero_extend:SI
-         (subreg:HI
-          (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                       (match_operand:SI 2 "const_int_operand" "i,i")) 2))
-        (const_int 0)))
-   (clobber (match_scratch:SI 3 "=r,r"))]
-  "BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (65535))"
-  "@
-   rlwinm. %3,%1,%s2,0xffff
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC
-        (zero_extend:SI
-         (subreg:HI
-          (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                       (match_operand:SI 2 "const_int_operand" "")) 0))
-        (const_int 0)))
-   (clobber (match_scratch:SI 3 ""))]
-  "!BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (65535)) && reload_completed"
-  [(set (match_dup 3)
-       (zero_extend:SI (subreg:HI
-          (lshiftrt:SI (match_dup 1)
-                       (match_dup 2)) 0)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC
-        (zero_extend:SI
-         (subreg:HI
-          (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                       (match_operand:SI 2 "const_int_operand" "")) 2))
-        (const_int 0)))
-   (clobber (match_scratch:SI 3 ""))]
-  "BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (65535)) && reload_completed"
-  [(set (match_dup 3)
-       (zero_extend:SI (subreg:HI
-          (lshiftrt:SI (match_dup 1)
-                       (match_dup 2)) 2)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
-
-(define_insn "*lshiftrt_internal5le"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (zero_extend:SI
-         (subreg:HI
-          (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                       (match_operand:SI 2 "const_int_operand" "i,i")) 0))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
-       (zero_extend:SI (subreg:HI (lshiftrt:SI (match_dup 1) (match_dup 2)) 0)))]
-  "!BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (65535))"
-  "@
-   rlwinm. %0,%1,%s2,0xffff
-   #"
+  ""
   [(set_attr "type" "shift")
+   (set_attr "maybe_var_shift" "yes")
    (set_attr "dot" "yes")
    (set_attr "length" "4,8")])
 
-(define_insn "*lshiftrt_internal5be"
+(define_insn_and_split "*ashr<mode>3_dot2"
   [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (zero_extend:SI
-         (subreg:HI
-          (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                       (match_operand:SI 2 "const_int_operand" "i,i")) 2))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
-       (zero_extend:SI (subreg:HI (lshiftrt:SI (match_dup 1) (match_dup 2)) 2)))]
-  "BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (65535))"
+       (compare:CC (ashiftrt:GPR (match_operand:GPR 1 "gpc_reg_operand" "r,r")
+                                 (match_operand:SI 2 "reg_or_cint_operand" "rn,rn"))
+                   (const_int 0)))
+   (set (match_operand:GPR 0 "gpc_reg_operand" "=r,r")
+       (ashiftrt:GPR (match_dup 1)
+                     (match_dup 2)))
+   (clobber (reg:GPR CA_REGNO))]
+  "<MODE>mode == Pmode && rs6000_gen_cell_microcode"
   "@
-   rlwinm. %0,%1,%s2,0xffff
+   sra<wd>%I2. %0,%1,%<hH>2
    #"
-  [(set_attr "type" "shift")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC
-        (zero_extend:SI
-         (subreg:HI
-          (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                       (match_operand:SI 2 "const_int_operand" "")) 0))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (zero_extend:SI (subreg:HI (lshiftrt:SI (match_dup 1) (match_dup 2)) 0)))]
-  "!BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (65535)) && reload_completed"
-  [(set (match_dup 0)
-       (zero_extend:SI (subreg:HI (lshiftrt:SI (match_dup 1) (match_dup 2)) 0)))
-   (set (match_dup 3)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
-
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC
-        (zero_extend:SI
-         (subreg:HI
-          (lshiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                       (match_operand:SI 2 "const_int_operand" "")) 2))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (zero_extend:SI (subreg:HI (lshiftrt:SI (match_dup 1) (match_dup 2)) 2)))]
-  "BYTES_BIG_ENDIAN && includes_rshift_p (operands[2], GEN_INT (65535)) && reload_completed"
-  [(set (match_dup 0)
-       (zero_extend:SI (subreg:HI (lshiftrt:SI (match_dup 1) (match_dup 2)) 2)))
+  "&& reload_completed && cc_reg_not_cr0_operand (operands[3], CCmode)"
+  [(parallel [(set (match_dup 0)
+                  (ashiftrt:GPR (match_dup 1)
+                                (match_dup 2)))
+             (clobber (reg:GPR CA_REGNO))])
    (set (match_dup 3)
        (compare:CC (match_dup 0)
                    (const_int 0)))]
-  "")
-
-(define_insn "ashrsi3"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
-       (ashiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                    (match_operand:SI 2 "reg_or_cint_operand" "r,i")))]
-  ""
-  "@
-   sraw %0,%1,%2
-   srawi %0,%1,%h2"
-  [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no")])
-
-(define_insn "*ashrsi3_64"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
-       (sign_extend:DI
-           (ashiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                        (match_operand:SI 2 "reg_or_cint_operand" "r,i"))))]
-  "TARGET_POWERPC64"
-  "@
-   sraw %0,%1,%2
-   srawi %0,%1,%h2"
-  [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no")])
-
-(define_insn ""
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC (ashiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r,r,r")
-                                (match_operand:SI 2 "reg_or_cint_operand" "r,i,r,i"))
-                   (const_int 0)))
-   (clobber (match_scratch:SI 3 "=r,r,r,r"))]
-  ""
-  "@
-   sraw. %3,%1,%2
-   srawi. %3,%1,%h2
-   #
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no,yes,no")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (ashiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                                (match_operand:SI 2 "reg_or_cint_operand" ""))
-                   (const_int 0)))
-   (clobber (match_scratch:SI 3 ""))]
-  "reload_completed"
-  [(set (match_dup 3)
-       (ashiftrt:SI (match_dup 1) (match_dup 2)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
-
-(define_insn ""
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC (ashiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r,r,r")
-                                (match_operand:SI 2 "reg_or_cint_operand" "r,i,r,i"))
-                   (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=r,r,r,r")
-       (ashiftrt:SI (match_dup 1) (match_dup 2)))]
   ""
-  "@
-   sraw. %0,%1,%2
-   srawi. %0,%1,%h2
-   #
-   #"
   [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no,yes,no")
+   (set_attr "maybe_var_shift" "yes")
    (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
+   (set_attr "length" "4,8")])
 \f
 ;; Builtins to replace a division to generate FRE reciprocal estimate
 ;; instructions and the necessary fixup instructions
   DONE;
 })
 \f
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (ashiftrt:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                                (match_operand:SI 2 "reg_or_cint_operand" ""))
-                   (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (ashiftrt:SI (match_dup 1) (match_dup 2)))]
-  "reload_completed"
-  [(set (match_dup 0)
-       (ashiftrt:SI (match_dup 1) (match_dup 2)))
-   (set (match_dup 3)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
-
-\f
 ;; Floating-point insns, excluding normal data motion.  We combine the SF/DF
 ;; modes here, and also add in conditional vsx/power8-vector support to access
 ;; values in the traditional Altivec registers if the appropriate
   "")
 
 (define_insn_and_split "*extendsfdf2_fpr"
-  [(set (match_operand:DF 0 "gpc_reg_operand" "=d,?d,d,ws,?ws,wv")
+  [(set (match_operand:DF 0 "gpc_reg_operand" "=d,?d,d,ws,?ws,wu")
        (float_extend:DF (match_operand:SF 1 "reg_or_mem_operand" "0,f,m,0,wy,Z")))]
   "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT"
   "@
    fmr %0,%1
    lfs%U1%X1 %0,%1
    #
-   xxlor %x0,%x1,%x1
+   xscpsgndp %x0,%x1,%x1
    lxsspx %x0,%y1"
   "&& reload_completed && REG_P (operands[1]) && REGNO (operands[0]) == REGNO (operands[1])"
   [(const_int 0)]
   emit_note (NOTE_INSN_DELETED);
   DONE;
 }
-  [(set_attr "type" "fp,fp,fpload,fp,vecsimple,fpload")])
+  [(set_attr "type" "fp,fp,fpload,fp,fp,fpload")])
 
 (define_expand "truncdfsf2"
   [(set (match_operand:SF 0 "gpc_reg_operand" "")
   "")
 
 (define_insn "*truncdfsf2_fpr"
-  [(set (match_operand:SF 0 "gpc_reg_operand" "=f")
-       (float_truncate:SF (match_operand:DF 1 "gpc_reg_operand" "d")))]
+  [(set (match_operand:SF 0 "gpc_reg_operand" "=f,wy")
+       (float_truncate:SF (match_operand:DF 1 "gpc_reg_operand" "d,ws")))]
   "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT"
-  "frsp %0,%1"
+  "@
+   frsp %0,%1
+   xsrsp %x0,%x1"
   [(set_attr "type" "fp")])
 
 ;; This expander is here to avoid FLOAT_WORDS_BIGENDIAN tests in
   "TARGET_<MODE>_FPR && TARGET_CMPB"
   "@
    fcpsgn %0,%2,%1
-   xscpsgn<Fvsx> %x0,%x2,%x1"
+   xscpsgndp %x0,%x2,%x1"
   [(set_attr "type" "fp")])
 
 ;; For MIN, MAX, and conditional move, we use DEFINE_EXPAND's that involve a
 ; We don't define lfiwax/lfiwzx with the normal definition, because we
 ; don't want to support putting SImode in FPR registers.
 (define_insn "lfiwax"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=d,wm,!wm")
+  [(set (match_operand:DI 0 "gpc_reg_operand" "=d,wj,!wj")
        (unspec:DI [(match_operand:SI 1 "reg_or_indexed_operand" "Z,Z,r")]
                   UNSPEC_LFIWAX))]
   "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT && TARGET_LFIWAX"
 ; not be needed and also in case the insns are deleted as dead code.
 
 (define_insn_and_split "floatsi<mode>2_lfiwax"
-  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=d")
+  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=<Fv>")
        (float:SFDF (match_operand:SI 1 "nonimmediate_operand" "r")))
-   (clobber (match_scratch:DI 2 "=d"))]
+   (clobber (match_scratch:DI 2 "=wj"))]
   "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT && TARGET_LFIWAX
    && <SI_CONVERT_FP> && can_create_pseudo_p ()"
   "#"
    (set_attr "type" "fpload")])
 
 (define_insn_and_split "floatsi<mode>2_lfiwax_mem"
-  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=d,<rreg2>")
+  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=<Ff>,<Fa>")
        (float:SFDF
         (sign_extend:DI
-         (match_operand:SI 1 "memory_operand" "Z,Z"))))
+         (match_operand:SI 1 "indexed_or_indirect_operand" "Z,Z"))))
    (clobber (match_scratch:DI 2 "=0,d"))]
   "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT && TARGET_LFIWAX
    && <SI_CONVERT_FP>"
    (set_attr "type" "fpload")])
 
 (define_insn "lfiwzx"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=d,wm,!wm")
+  [(set (match_operand:DI 0 "gpc_reg_operand" "=d,wj,!wj")
        (unspec:DI [(match_operand:SI 1 "reg_or_indexed_operand" "Z,Z,r")]
                   UNSPEC_LFIWZX))]
   "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT && TARGET_LFIWZX"
   [(set_attr "type" "fpload,fpload,mftgpr")])
 
 (define_insn_and_split "floatunssi<mode>2_lfiwzx"
-  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=d")
+  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=<Fv>")
        (unsigned_float:SFDF (match_operand:SI 1 "nonimmediate_operand" "r")))
-   (clobber (match_scratch:DI 2 "=d"))]
+   (clobber (match_scratch:DI 2 "=wj"))]
   "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT && TARGET_LFIWZX
    && <SI_CONVERT_FP>"
   "#"
    (set_attr "type" "fpload")])
 
 (define_insn_and_split "floatunssi<mode>2_lfiwzx_mem"
-  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=d,<rreg2>")
+  [(set (match_operand:SFDF 0 "gpc_reg_operand" "=<Ff>,<Fa>")
        (unsigned_float:SFDF
         (zero_extend:DI
-         (match_operand:SI 1 "memory_operand" "Z,Z"))))
+         (match_operand:SI 1 "indexed_or_indirect_operand" "Z,Z"))))
    (clobber (match_scratch:DI 2 "=0,d"))]
   "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT && TARGET_LFIWZX
    && <SI_CONVERT_FP>"
   highword = adjust_address (operands[4], SImode, 0);
   lowword = adjust_address (operands[4], SImode, 4);
   if (! WORDS_BIG_ENDIAN)
-    {
-      rtx tmp;
-      tmp = highword; highword = lowword; lowword = tmp;
-    }
+    std::swap (lowword, highword);
 
   emit_insn (gen_xorsi3 (operands[6], operands[1],
                         GEN_INT (~ (HOST_WIDE_INT) 0x7fffffff)));
   highword = adjust_address (operands[4], SImode, 0);
   lowword = adjust_address (operands[4], SImode, 4);
   if (! WORDS_BIG_ENDIAN)
-    {
-      rtx tmp;
-      tmp = highword; highword = lowword; lowword = tmp;
-    }
+    std::swap (lowword, highword);
 
   emit_move_insn (lowword, operands[1]);
   emit_move_insn (highword, operands[2]);
   "")
 
 (define_insn "*fix_trunc<mode>di2_fctidz"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=d")
-       (fix:DI (match_operand:SFDF 1 "gpc_reg_operand" "d")))]
+  [(set (match_operand:DI 0 "gpc_reg_operand" "=d,wi")
+       (fix:DI (match_operand:SFDF 1 "gpc_reg_operand" "<Ff>,<Fa>")))]
   "TARGET_HARD_FLOAT && TARGET_DOUBLE_FLOAT && TARGET_FPRS
-    && TARGET_FCFID && !VECTOR_UNIT_VSX_P (<MODE>mode)"
-  "fctidz %0,%1"
+    && TARGET_FCFID"
+  "@
+   fctidz %0,%1
+   xscvdpsxds %x0,%x1"
   [(set_attr "type" "fp")])
 
 (define_expand "fixuns_trunc<mode>si2"
   "")
 
 (define_insn "*fixuns_trunc<mode>di2_fctiduz"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=d")
-       (unsigned_fix:DI (match_operand:SFDF 1 "gpc_reg_operand" "d")))]
+  [(set (match_operand:DI 0 "gpc_reg_operand" "=d,wi")
+       (unsigned_fix:DI (match_operand:SFDF 1 "gpc_reg_operand" "<Ff>,<Fa>")))]
   "TARGET_HARD_FLOAT && TARGET_DOUBLE_FLOAT && TARGET_FPRS
-    && TARGET_FCTIDUZ && !VECTOR_UNIT_VSX_P (<MODE>mode)"
-  "fctiduz %0,%1"
+    && TARGET_FCTIDUZ"
+  "@
+   fctiduz %0,%1
+   xscvdpuxds %x0,%x1"
   [(set_attr "type" "fp")])
 
 ; Here, we use (set (reg) (unspec:DI [(fix:SI ...)] UNSPEC_FCTIWZ))
 ; because the first makes it clear that operand 0 is not live
 ; before the instruction.
 (define_insn "fctiwz_<mode>"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=d")
-       (unspec:DI [(fix:SI (match_operand:SFDF 1 "gpc_reg_operand" "d"))]
+  [(set (match_operand:DI 0 "gpc_reg_operand" "=d,wi")
+       (unspec:DI [(fix:SI (match_operand:SFDF 1 "gpc_reg_operand" "<Ff>,<Fv>"))]
                   UNSPEC_FCTIWZ))]
   "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT"
-  "fctiwz %0,%1"
+  "@
+   fctiwz %0,%1
+   xscvdpsxws %x0,%x1"
   [(set_attr "type" "fp")])
 
 (define_insn "fctiwuz_<mode>"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=d")
+  [(set (match_operand:DI 0 "gpc_reg_operand" "=d,wi")
        (unspec:DI [(unsigned_fix:SI
-                    (match_operand:SFDF 1 "gpc_reg_operand" "<rreg2>"))]
+                    (match_operand:SFDF 1 "gpc_reg_operand" "<Ff>,<Fv>"))]
                   UNSPEC_FCTIWUZ))]
   "TARGET_HARD_FLOAT && TARGET_FPRS && <TARGET_FLOAT> && TARGET_FCTIWUZ"
-  "fctiwuz %0,%1"
+  "@
+   fctiwuz %0,%1
+   xscvdpuxws %x0,%x1"
   [(set_attr "type" "fp")])
 
 ;; Only optimize (float (fix x)) -> frz if we are in fast-math mode, since
 ;; since the friz instruction does not truncate the value if the floating
 ;; point value is < LONG_MIN or > LONG_MAX.
 (define_insn "*friz"
-  [(set (match_operand:DF 0 "gpc_reg_operand" "=d")
-       (float:DF (fix:DI (match_operand:DF 1 "gpc_reg_operand" "d"))))]
+  [(set (match_operand:DF 0 "gpc_reg_operand" "=d,ws")
+       (float:DF (fix:DI (match_operand:DF 1 "gpc_reg_operand" "d,ws"))))]
   "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT && TARGET_FPRND
-   && !VECTOR_UNIT_VSX_P (DFmode) && flag_unsafe_math_optimizations
-   && !flag_trapping_math && TARGET_FRIZ"
-  "friz %0,%1"
+   && flag_unsafe_math_optimizations && !flag_trapping_math && TARGET_FRIZ"
+  "@
+   friz %0,%1
+   xsrdpiz %x0,%x1"
   [(set_attr "type" "fp")])
 
 ;; Since FCTIWZ doesn't sign extend the upper bits, we have to do a store and a
   "")
 
 (define_insn "*floatdidf2_fpr"
-  [(set (match_operand:DF 0 "gpc_reg_operand" "=d")
-       (float:DF (match_operand:DI 1 "gpc_reg_operand" "d")))]
-  "TARGET_FCFID && TARGET_HARD_FLOAT && TARGET_DOUBLE_FLOAT && TARGET_FPRS
-   && !VECTOR_UNIT_VSX_P (DFmode)"
-  "fcfid %0,%1"
+  [(set (match_operand:DF 0 "gpc_reg_operand" "=d,ws")
+       (float:DF (match_operand:DI 1 "gpc_reg_operand" "d,wi")))]
+  "TARGET_FCFID && TARGET_HARD_FLOAT && TARGET_DOUBLE_FLOAT && TARGET_FPRS"
+  "@
+   fcfid %0,%1
+   xscvsxddp %x0,%x1"
   [(set_attr "type" "fp")])
 
 ; Allow the combiner to merge source memory operands to the conversion so that
 ; hit.  We will split after reload to avoid the trip through the GPRs
 
 (define_insn_and_split "*floatdidf2_mem"
-  [(set (match_operand:DF 0 "gpc_reg_operand" "=d")
-       (float:DF (match_operand:DI 1 "memory_operand" "m")))
-   (clobber (match_scratch:DI 2 "=d"))]
+  [(set (match_operand:DF 0 "gpc_reg_operand" "=d,ws")
+       (float:DF (match_operand:DI 1 "memory_operand" "m,Z")))
+   (clobber (match_scratch:DI 2 "=d,wi"))]
   "TARGET_HARD_FLOAT && TARGET_DOUBLE_FLOAT && TARGET_FPRS && TARGET_FCFID"
   "#"
   "&& reload_completed"
   [(set (match_operand:DF 0 "gpc_reg_operand" "")
        (unsigned_float:DF
         (match_operand:DI 1 "gpc_reg_operand" "")))]
-  "TARGET_HARD_FLOAT && (TARGET_FCFIDU || VECTOR_UNIT_VSX_P (DFmode))"
+  "TARGET_HARD_FLOAT && TARGET_FCFIDU"
   "")
 
 (define_insn "*floatunsdidf2_fcfidu"
-  [(set (match_operand:DF 0 "gpc_reg_operand" "=d")
-       (unsigned_float:DF (match_operand:DI 1 "gpc_reg_operand" "d")))]
-  "TARGET_HARD_FLOAT && TARGET_FCFIDU && !VECTOR_UNIT_VSX_P (DFmode)"
-  "fcfidu %0,%1"
+  [(set (match_operand:DF 0 "gpc_reg_operand" "=d,ws")
+       (unsigned_float:DF (match_operand:DI 1 "gpc_reg_operand" "d,wi")))]
+  "TARGET_HARD_FLOAT && TARGET_FCFIDU"
+  "@
+   fcfidu %0,%1
+   xscvuxddp %x0,%x1"
   [(set_attr "type" "fp")
    (set_attr "length" "4")])
 
 (define_insn_and_split "*floatunsdidf2_mem"
-  [(set (match_operand:DF 0 "gpc_reg_operand" "=d")
-       (unsigned_float:DF (match_operand:DI 1 "memory_operand" "m")))
-   (clobber (match_scratch:DI 2 "=d"))]
+  [(set (match_operand:DF 0 "gpc_reg_operand" "=d,ws")
+       (unsigned_float:DF (match_operand:DI 1 "memory_operand" "m,Z")))
+   (clobber (match_scratch:DI 2 "=d,wi"))]
   "TARGET_HARD_FLOAT && (TARGET_FCFIDU || VECTOR_UNIT_VSX_P (DFmode))"
   "#"
   "&& reload_completed"
 }")
 
 (define_insn "floatdisf2_fcfids"
-  [(set (match_operand:SF 0 "gpc_reg_operand" "=f")
-       (float:SF (match_operand:DI 1 "gpc_reg_operand" "d")))]
+  [(set (match_operand:SF 0 "gpc_reg_operand" "=f,wy")
+       (float:SF (match_operand:DI 1 "gpc_reg_operand" "d,wi")))]
   "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_SINGLE_FLOAT
    && TARGET_DOUBLE_FLOAT && TARGET_FCFIDS"
-  "fcfids %0,%1"
+  "@
+   fcfids %0,%1
+   xscvsxdsp %x0,%x1"
   [(set_attr "type" "fp")])
 
 (define_insn_and_split "*floatdisf2_mem"
-  [(set (match_operand:SF 0 "gpc_reg_operand" "=f")
-       (float:SF (match_operand:DI 1 "memory_operand" "m")))
-   (clobber (match_scratch:DI 2 "=f"))]
+  [(set (match_operand:SF 0 "gpc_reg_operand" "=f,wy,wy")
+       (float:SF (match_operand:DI 1 "memory_operand" "m,m,Z")))
+   (clobber (match_scratch:DI 2 "=d,d,wi"))]
   "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_SINGLE_FLOAT
    && TARGET_DOUBLE_FLOAT && TARGET_FCFIDS"
   "#"
   [(set (match_operand:SF 0 "gpc_reg_operand" "=f")
         (float:SF (match_operand:DI 1 "gpc_reg_operand" "d")))
    (clobber (match_scratch:DF 2 "=d"))]
-  "TARGET_FCFID && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_SINGLE_FLOAT"
+  "TARGET_FCFID && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_SINGLE_FLOAT
+   && !TARGET_FCFIDS"
   "#"
   "&& reload_completed"
   [(set (match_dup 2)
 ;; by a bit that won't be lost at that stage, but is below the SFmode
 ;; rounding position.
 (define_expand "floatdisf2_internal2"
-  [(set (match_dup 3) (ashiftrt:DI (match_operand:DI 1 "" "")
-                                  (const_int 53)))
-   (parallel [(set (match_operand:DI 0 "" "") (and:DI (match_dup 1)
-                                                     (const_int 2047)))
-             (clobber (scratch:CC))])
+  [(parallel [(set (match_dup 3) (ashiftrt:DI (match_operand:DI 1 "" "")
+                                             (const_int 53)))
+             (clobber (reg:DI CA_REGNO))])
+   (set (match_operand:DI 0 "" "") (and:DI (match_dup 1)
+                                          (const_int 2047)))
    (set (match_dup 3) (plus:DI (match_dup 3)
                               (const_int 1)))
    (set (match_dup 0) (plus:DI (match_dup 0)
                                     (const_int 2)))
    (set (match_dup 0) (ior:DI (match_dup 0)
                              (match_dup 1)))
-   (parallel [(set (match_dup 0) (and:DI (match_dup 0)
-                                        (const_int -2048)))
-             (clobber (scratch:CC))])
+   (set (match_dup 0) (and:DI (match_dup 0)
+                             (const_int -2048)))
    (set (pc) (if_then_else (geu (match_dup 4) (const_int 0))
                           (label_ref (match_operand:DI 2 "" ""))
                           (pc)))
    (set (match_dup 0) (match_dup 1))]
-  "TARGET_POWERPC64 && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_SINGLE_FLOAT"
+  "TARGET_POWERPC64 && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_SINGLE_FLOAT
+   && !TARGET_FCFIDS"
   "
 {
   operands[3] = gen_reg_rtx (DImode);
   "")
 
 (define_insn "floatunsdisf2_fcfidus"
-  [(set (match_operand:SF 0 "gpc_reg_operand" "=f")
-        (unsigned_float:SF (match_operand:DI 1 "gpc_reg_operand" "d")))]
+  [(set (match_operand:SF 0 "gpc_reg_operand" "=f,wu")
+        (unsigned_float:SF (match_operand:DI 1 "gpc_reg_operand" "d,wi")))]
   "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_SINGLE_FLOAT
    && TARGET_DOUBLE_FLOAT && TARGET_FCFIDUS"
-  "fcfidus %0,%1"
+  "@
+   fcfidus %0,%1
+   xscvuxdsp %x0,%x1"
   [(set_attr "type" "fp")])
 
 (define_insn_and_split "*floatunsdisf2_mem"
-  [(set (match_operand:SF 0 "gpc_reg_operand" "=f")
-       (unsigned_float:SF (match_operand:DI 1 "memory_operand" "m")))
-   (clobber (match_scratch:DI 2 "=f"))]
+  [(set (match_operand:SF 0 "gpc_reg_operand" "=f,wy,wy")
+       (unsigned_float:SF (match_operand:DI 1 "memory_operand" "m,m,Z")))
+   (clobber (match_scratch:DI 2 "=d,d,wi"))]
   "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_SINGLE_FLOAT
    && TARGET_DOUBLE_FLOAT && TARGET_FCFIDUS"
   "#"
 ;; (for example, having an input in 7,8 and an output in 6,7).  We
 ;; also allow for the output being the same as one of the inputs.
 
-(define_insn "addti3"
-  [(set (match_operand:TI 0 "gpc_reg_operand" "=&r,&r,r,r")
-       (plus:TI (match_operand:TI 1 "gpc_reg_operand" "%r,r,0,0")
-                (match_operand:TI 2 "reg_or_short_operand" "r,I,r,I")))]
+(define_expand "addti3"
+  [(set (match_operand:TI 0 "gpc_reg_operand" "")
+       (plus:TI (match_operand:TI 1 "gpc_reg_operand" "")
+                (match_operand:TI 2 "reg_or_short_operand" "")))]
   "TARGET_64BIT"
 {
-  if (WORDS_BIG_ENDIAN)
-    return (GET_CODE (operands[2])) != CONST_INT
-           ? \"addc %L0,%L1,%L2\;adde %0,%1,%2\"
-           : \"addic %L0,%L1,%2\;add%G2e %0,%1\";
-  else
-    return (GET_CODE (operands[2])) != CONST_INT
-           ? \"addc %0,%1,%2\;adde %L0,%L1,%L2\"
-           : \"addic %0,%1,%2\;add%G2e %L0,%L1\";
-}
-  [(set_attr "type" "two")
-   (set_attr "length" "8")])
-
-(define_insn "subti3"
-  [(set (match_operand:TI 0 "gpc_reg_operand" "=&r,&r,r,r,r")
-       (minus:TI (match_operand:TI 1 "reg_or_short_operand" "r,I,0,r,I")
-                 (match_operand:TI 2 "gpc_reg_operand" "r,r,r,0,0")))]
-  "TARGET_64BIT"
-{
-  if (WORDS_BIG_ENDIAN)
-    return (GET_CODE (operands[1]) != CONST_INT)
-           ? \"subfc %L0,%L2,%L1\;subfe %0,%2,%1\"
-           : \"subfic %L0,%L2,%1\;subf%G1e %0,%2\";
-  else
-    return (GET_CODE (operands[1]) != CONST_INT)
-           ? \"subfc %0,%2,%1\;subfe %L0,%L2,%L1\"
-           : \"subfic %0,%2,%1\;subf%G1e %L0,%L2\";
-}
-  [(set_attr "type" "two")
-   (set_attr "length" "8")])
-
-
-;; Define the DImode operations that can be done in a small number
-;; of instructions.  The & constraints are to prevent the register
-;; allocator from allocating registers that overlap with the inputs
-;; (for example, having an input in 7,8 and an output in 6,7).  We
-;; also allow for the output being the same as one of the inputs.
-
-(define_insn "*adddi3_noppc64"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=&r,&r,r,r")
-       (plus:DI (match_operand:DI 1 "gpc_reg_operand" "%r,r,0,0")
-                (match_operand:DI 2 "reg_or_short_operand" "r,I,r,I")))]
-  "! TARGET_POWERPC64"
-  "*
-{
-  if (WORDS_BIG_ENDIAN)
-    return (GET_CODE (operands[2])) != CONST_INT
-           ? \"addc %L0,%L1,%L2\;adde %0,%1,%2\"
-           : \"addic %L0,%L1,%2\;add%G2e %0,%1\";
-  else
-    return (GET_CODE (operands[2])) != CONST_INT
-           ? \"addc %0,%1,%2\;adde %L0,%L1,%L2\"
-           : \"addic %0,%1,%2\;add%G2e %L0,%L1\";
-}"
-  [(set_attr "type" "two")
-   (set_attr "length" "8")])
-
-(define_insn "*subdi3_noppc64"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=&r,&r,r,r,r")
-       (minus:DI (match_operand:DI 1 "reg_or_short_operand" "r,I,0,r,I")
-                 (match_operand:DI 2 "gpc_reg_operand" "r,r,r,0,0")))]
-  "! TARGET_POWERPC64"
-  "*
-{
-  if (WORDS_BIG_ENDIAN)
-    return (GET_CODE (operands[1]) != CONST_INT)
-           ? \"subfc %L0,%L2,%L1\;subfe %0,%2,%1\"
-           : \"subfic %L0,%L2,%1\;subf%G1e %0,%2\";
-  else
-    return (GET_CODE (operands[1]) != CONST_INT)
-           ? \"subfc %0,%2,%1\;subfe %L0,%L2,%L1\"
-           : \"subfic %0,%2,%1\;subf%G1e %L0,%L2\";
-}"
-  [(set_attr "type" "two")
-   (set_attr "length" "8")])
-
-(define_insn "*negdi2_noppc64"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=&r,r")
-       (neg:DI (match_operand:DI 1 "gpc_reg_operand" "r,0")))]
-  "! TARGET_POWERPC64"
-  "*
-{
-  return (WORDS_BIG_ENDIAN)
-    ? \"subfic %L0,%L1,0\;subfze %0,%1\"
-    : \"subfic %0,%1,0\;subfze %L0,%L1\";
-}"
-  [(set_attr "type" "two")
-   (set_attr "length" "8")])
+  rtx lo0 = gen_lowpart (DImode, operands[0]);
+  rtx lo1 = gen_lowpart (DImode, operands[1]);
+  rtx lo2 = gen_lowpart (DImode, operands[2]);
+  rtx hi0 = gen_highpart (DImode, operands[0]);
+  rtx hi1 = gen_highpart (DImode, operands[1]);
+  rtx hi2 = gen_highpart_mode (DImode, TImode, operands[2]);
 
-(define_insn "mulsidi3"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=&r")
-       (mult:DI (sign_extend:DI (match_operand:SI 1 "gpc_reg_operand" "%r"))
-                (sign_extend:DI (match_operand:SI 2 "gpc_reg_operand" "r"))))]
-  "! TARGET_POWERPC64"
-{
-  return (WORDS_BIG_ENDIAN)
-    ? \"mulhw %0,%1,%2\;mullw %L0,%1,%2\"
-    : \"mulhw %L0,%1,%2\;mullw %0,%1,%2\";
-}
-  [(set_attr "type" "mul")
-   (set_attr "length" "8")])
+  if (!reg_or_short_operand (lo2, DImode))
+    lo2 = force_reg (DImode, lo2);
+  if (!adde_operand (hi2, DImode))
+    hi2 = force_reg (DImode, hi2);
 
-(define_split
-  [(set (match_operand:DI 0 "gpc_reg_operand" "")
-       (mult:DI (sign_extend:DI (match_operand:SI 1 "gpc_reg_operand" ""))
-                (sign_extend:DI (match_operand:SI 2 "gpc_reg_operand" ""))))]
-  "! TARGET_POWERPC64 && reload_completed"
-  [(set (match_dup 3)
-       (truncate:SI
-        (lshiftrt:DI (mult:DI (sign_extend:DI (match_dup 1))
-                              (sign_extend:DI (match_dup 2)))
-                     (const_int 32))))
-   (set (match_dup 4)
-       (mult:SI (match_dup 1)
-                (match_dup 2)))]
-  "
-{
-  int endian = (WORDS_BIG_ENDIAN == 0);
-  operands[3] = operand_subword (operands[0], endian, 0, DImode);
-  operands[4] = operand_subword (operands[0], 1 - endian, 0, DImode);
-}")
+  emit_insn (gen_adddi3_carry (lo0, lo1, lo2));
+  emit_insn (gen_adddi3_carry_in (hi0, hi1, hi2));
+  DONE;
+})
 
-(define_insn "umulsidi3"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=&r")
-       (mult:DI (zero_extend:DI (match_operand:SI 1 "gpc_reg_operand" "%r"))
-                (zero_extend:DI (match_operand:SI 2 "gpc_reg_operand" "r"))))]
-  "! TARGET_POWERPC64"
-  "*
+(define_expand "subti3"
+  [(set (match_operand:TI 0 "gpc_reg_operand" "")
+       (minus:TI (match_operand:TI 1 "reg_or_short_operand" "")
+                 (match_operand:TI 2 "gpc_reg_operand" "")))]
+  "TARGET_64BIT"
 {
-  return (WORDS_BIG_ENDIAN)
-    ? \"mulhwu %0,%1,%2\;mullw %L0,%1,%2\"
-    : \"mulhwu %L0,%1,%2\;mullw %0,%1,%2\";
-}"
-  [(set_attr "type" "mul")
-   (set_attr "length" "8")])
+  rtx lo0 = gen_lowpart (DImode, operands[0]);
+  rtx lo1 = gen_lowpart (DImode, operands[1]);
+  rtx lo2 = gen_lowpart (DImode, operands[2]);
+  rtx hi0 = gen_highpart (DImode, operands[0]);
+  rtx hi1 = gen_highpart_mode (DImode, TImode, operands[1]);
+  rtx hi2 = gen_highpart (DImode, operands[2]);
 
-(define_split
-  [(set (match_operand:DI 0 "gpc_reg_operand" "")
-       (mult:DI (zero_extend:DI (match_operand:SI 1 "gpc_reg_operand" ""))
-                (zero_extend:DI (match_operand:SI 2 "gpc_reg_operand" ""))))]
-  "! TARGET_POWERPC64 && reload_completed"
-  [(set (match_dup 3)
-       (truncate:SI
-        (lshiftrt:DI (mult:DI (zero_extend:DI (match_dup 1))
-                              (zero_extend:DI (match_dup 2)))
-                     (const_int 32))))
-   (set (match_dup 4)
-       (mult:SI (match_dup 1)
-                (match_dup 2)))]
-  "
-{
-  int endian = (WORDS_BIG_ENDIAN == 0);
-  operands[3] = operand_subword (operands[0], endian, 0, DImode);
-  operands[4] = operand_subword (operands[0], 1 - endian, 0, DImode);
-}")
+  if (!reg_or_short_operand (lo1, DImode))
+    lo1 = force_reg (DImode, lo1);
+  if (!adde_operand (hi1, DImode))
+    hi1 = force_reg (DImode, hi1);
 
-(define_insn "smulsi3_highpart"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-       (truncate:SI
-        (lshiftrt:DI (mult:DI (sign_extend:DI
-                               (match_operand:SI 1 "gpc_reg_operand" "%r"))
-                              (sign_extend:DI
-                               (match_operand:SI 2 "gpc_reg_operand" "r")))
-                     (const_int 32))))]
-  ""
-  "mulhw %0,%1,%2"
-  [(set_attr "type" "mul")])
+  emit_insn (gen_subfdi3_carry (lo0, lo2, lo1));
+  emit_insn (gen_subfdi3_carry_in (hi0, hi2, hi1));
+  DONE;
+})
 
-(define_insn "umulsi3_highpart"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-       (truncate:SI
-        (lshiftrt:DI (mult:DI (zero_extend:DI
-                               (match_operand:SI 1 "gpc_reg_operand" "%r"))
-                              (zero_extend:DI
-                               (match_operand:SI 2 "gpc_reg_operand" "r")))
-                     (const_int 32))))]
-  ""
-  "mulhwu %0,%1,%2"
-  [(set_attr "type" "mul")])
 
 ;; Shift by a variable amount is too complex to be worth open-coding.  We
 ;; just handle shifts by constants.
 (define_insn "ashrdi3_no_power"
   [(set (match_operand:DI 0 "gpc_reg_operand" "=&r,&r")
        (ashiftrt:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
-                    (match_operand:SI 2 "const_int_operand" "M,i")))]
+                    (match_operand:SI 2 "const_int_operand" "M,i")))
+   (clobber (reg:SI CA_REGNO))]
   "!TARGET_POWERPC64"
-  "*
 {
   switch (which_alternative)
     {
       else
        return \"srwi %0,%1,%h2\;insrwi %0,%L1,%h2,0\;srawi %L0,%L1,%h2\";
     }
-}"
+}
   [(set_attr "type" "two,three")
    (set_attr "length" "8,12")])
 
 \f
 ;; PowerPC64 DImode operations.
 
-(define_insn "muldi3"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
-        (mult:DI (match_operand:DI 1 "gpc_reg_operand" "%r,r")
-                 (match_operand:DI 2 "reg_or_short_operand" "r,I")))]
-  "TARGET_POWERPC64"
-  "@
-   mulld %0,%1,%2
-   mulli %0,%1,%2"
-   [(set_attr "type" "mul")
-    (set (attr "size")
-      (cond [(match_operand:SI 2 "s8bit_cint_operand" "")
-               (const_string "8")
-            (match_operand:SI 2 "short_cint_operand" "")
-               (const_string "16")]
-       (const_string "64")))])
-
-(define_insn "*muldi3_internal1"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC (mult:DI (match_operand:DI 1 "gpc_reg_operand" "%r,r")
-                            (match_operand:DI 2 "gpc_reg_operand" "r,r"))
-                   (const_int 0)))
-   (clobber (match_scratch:DI 3 "=r,r"))]
-  "TARGET_POWERPC64"
-  "@
-   mulld. %3,%1,%2
-   #"
-  [(set_attr "type" "mul")
-   (set_attr "size" "64")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_cr0_operand" "")
-       (compare:CC (mult:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                            (match_operand:DI 2 "gpc_reg_operand" ""))
-                   (const_int 0)))
-   (clobber (match_scratch:DI 3 ""))]
-  "TARGET_POWERPC64 && reload_completed"
-  [(set (match_dup 3)
-       (mult:DI (match_dup 1) (match_dup 2)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
-
-(define_insn "*muldi3_internal2"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
-       (compare:CC (mult:DI (match_operand:DI 1 "gpc_reg_operand" "%r,r")
-                            (match_operand:DI 2 "gpc_reg_operand" "r,r"))
-                   (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
-       (mult:DI (match_dup 1) (match_dup 2)))]
-  "TARGET_POWERPC64"
-  "@
-   mulld. %0,%1,%2
-   #"
-  [(set_attr "type" "mul")
-   (set_attr "size" "64")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (mult:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                            (match_operand:DI 2 "gpc_reg_operand" ""))
-                   (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "")
-       (mult:DI (match_dup 1) (match_dup 2)))]
-  "TARGET_POWERPC64 && reload_completed"
-  [(set (match_dup 0)
-       (mult:DI (match_dup 1) (match_dup 2)))
-   (set (match_dup 3)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
-
-(define_insn "smuldi3_highpart"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
-       (truncate:DI
-        (lshiftrt:TI (mult:TI (sign_extend:TI
-                               (match_operand:DI 1 "gpc_reg_operand" "%r"))
-                              (sign_extend:TI
-                               (match_operand:DI 2 "gpc_reg_operand" "r")))
-                     (const_int 64))))]
-  "TARGET_POWERPC64"
-  "mulhd %0,%1,%2"
-  [(set_attr "type" "mul")
-   (set_attr "size" "64")])
-
-(define_insn "umuldi3_highpart"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
-       (truncate:DI
-        (lshiftrt:TI (mult:TI (zero_extend:TI
-                               (match_operand:DI 1 "gpc_reg_operand" "%r"))
-                              (zero_extend:TI
-                               (match_operand:DI 2 "gpc_reg_operand" "r")))
-                     (const_int 64))))]
-  "TARGET_POWERPC64"
-  "mulhdu %0,%1,%2"
-  [(set_attr "type" "mul")
-   (set_attr "size" "64")])
-
-(define_expand "mulditi3"
-  [(set (match_operand:TI 0 "gpc_reg_operand")
-       (mult:TI (sign_extend:TI (match_operand:DI 1 "gpc_reg_operand"))
-                (sign_extend:TI (match_operand:DI 2 "gpc_reg_operand"))))]
-  "TARGET_POWERPC64"
-{
-  rtx l = gen_reg_rtx (DImode), h = gen_reg_rtx (DImode);
-  emit_insn (gen_muldi3 (l, operands[1], operands[2]));
-  emit_insn (gen_smuldi3_highpart (h, operands[1], operands[2]));
-  emit_move_insn (gen_lowpart (DImode, operands[0]), l);
-  emit_move_insn (gen_highpart (DImode, operands[0]), h);
-  DONE;
-})
-
-(define_expand "umulditi3"
-  [(set (match_operand:TI 0 "gpc_reg_operand")
-       (mult:TI (zero_extend:TI (match_operand:DI 1 "gpc_reg_operand"))
-                (zero_extend:TI (match_operand:DI 2 "gpc_reg_operand"))))]
-  "TARGET_POWERPC64"
-{
-  rtx l = gen_reg_rtx (DImode), h = gen_reg_rtx (DImode);
-  emit_insn (gen_muldi3 (l, operands[1], operands[2]));
-  emit_insn (gen_umuldi3_highpart (h, operands[1], operands[2]));
-  emit_move_insn (gen_lowpart (DImode, operands[0]), l);
-  emit_move_insn (gen_highpart (DImode, operands[0]), h);
-  DONE;
-})
-
-(define_insn "rotldi3"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
-       (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
-                  (match_operand:DI 2 "reg_or_cint_operand" "r,i")))]
-  "TARGET_POWERPC64"
-  "@
-   rldcl %0,%1,%2,0
-   rldicl %0,%1,%H2,0"
-  [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no")])
-
-(define_insn "*rotldi3_internal2"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r,r,r")
-                              (match_operand:DI 2 "reg_or_cint_operand" "r,i,r,i"))
-                   (const_int 0)))
-   (clobber (match_scratch:DI 3 "=r,r,r,r"))]
-  "TARGET_64BIT"
-  "@
-   rldcl. %3,%1,%2,0
-   rldicl. %3,%1,%H2,0
-   #
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no,yes,no")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                              (match_operand:DI 2 "reg_or_cint_operand" ""))
-                   (const_int 0)))
-   (clobber (match_scratch:DI 3 ""))]
-  "TARGET_POWERPC64 && reload_completed"
-  [(set (match_dup 3)
-       (rotate:DI (match_dup 1) (match_dup 2)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
-
-(define_insn "*rotldi3_internal3"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r,r,r")
-                              (match_operand:DI 2 "reg_or_cint_operand" "r,i,r,i"))
-                   (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r,r,r")
-       (rotate:DI (match_dup 1) (match_dup 2)))]
-  "TARGET_64BIT"
-  "@
-   rldcl. %0,%1,%2,0
-   rldicl. %0,%1,%H2,0
-   #
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no,yes,no")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
-
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                              (match_operand:DI 2 "reg_or_cint_operand" ""))
-                   (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "")
-       (rotate:DI (match_dup 1) (match_dup 2)))]
-  "TARGET_POWERPC64 && reload_completed"
-  [(set (match_dup 0)
-       (rotate:DI (match_dup 1) (match_dup 2)))
-   (set (match_dup 3)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
-
 (define_insn "*rotldi3_internal4"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
-       (and:DI (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
-                          (match_operand:DI 2 "reg_or_cint_operand" "r,i"))
-               (match_operand:DI 3 "mask64_operand" "n,n")))]
+  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
+       (and:DI (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r")
+                          (match_operand:DI 2 "reg_or_cint_operand" "rn"))
+               (match_operand:DI 3 "mask64_operand" "n")))]
   "TARGET_POWERPC64"
-  "@
-   rldc%B3 %0,%1,%2,%S3
-   rldic%B3 %0,%1,%H2,%S3"
+  "rld%I2c%B3 %0,%1,%H2,%S3"
   [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no")])
+   (set_attr "maybe_var_shift" "yes")])
 
 (define_insn "*rotldi3_internal5"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,x,?y,?y")
+  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
        (compare:CC (and:DI
-                    (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r,r,r")
-                               (match_operand:DI 2 "reg_or_cint_operand" "r,i,r,i"))
-                    (match_operand:DI 3 "mask64_operand" "n,n,n,n"))
+                    (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
+                               (match_operand:DI 2 "reg_or_cint_operand" "rn,rn"))
+                    (match_operand:DI 3 "mask64_operand" "n,n"))
                    (const_int 0)))
-   (clobber (match_scratch:DI 4 "=r,r,r,r"))]
+   (clobber (match_scratch:DI 4 "=r,r"))]
   "TARGET_64BIT"
   "@
-   rldc%B3. %4,%1,%2,%S3
-   rldic%B3. %4,%1,%H2,%S3
-   #
+   rld%I2c%B3. %4,%1,%H2,%S3
    #"
   [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no,yes,no")
+   (set_attr "maybe_var_shift" "yes")
    (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
+   (set_attr "length" "4,8")])
 
 (define_split
   [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
   "")
 
 (define_insn "*rotldi3_internal6"
-  [(set (match_operand:CC 4 "cc_reg_operand" "=x,x,?y,?y")
+  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
        (compare:CC (and:DI
-                    (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r,r,r")
-                               (match_operand:DI 2 "reg_or_cint_operand" "r,i,r,i"))
-                    (match_operand:DI 3 "mask64_operand" "n,n,n,n"))
+                    (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
+                               (match_operand:DI 2 "reg_or_cint_operand" "rn,rn"))
+                    (match_operand:DI 3 "mask64_operand" "n,n"))
                    (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r,r,r")
+   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
        (and:DI (rotate:DI (match_dup 1) (match_dup 2)) (match_dup 3)))]
   "TARGET_64BIT"
   "@
-   rldc%B3. %0,%1,%2,%S3
-   rldic%B3. %0,%1,%H2,%S3
-   #
+   rld%I2c%B3. %0,%1,%H2,%S3
    #"
   [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no,yes,no")
+   (set_attr "maybe_var_shift" "yes")
    (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
+   (set_attr "length" "4,8")])
 
 (define_split
   [(set (match_operand:CC 4 "cc_reg_not_micro_cr0_operand" "")
                    (const_int 0)))]
   "")
 
-(define_insn "*rotldi3_internal7le"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
-       (zero_extend:DI
-        (subreg:QI
-         (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
-                    (match_operand:DI 2 "reg_or_cint_operand" "r,i")) 0)))]
-  "TARGET_POWERPC64 && !BYTES_BIG_ENDIAN"
-  "@
-   rldcl %0,%1,%2,56
-   rldicl %0,%1,%H2,56"
-  [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no")])
 
-(define_insn "*rotldi3_internal7be"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
-       (zero_extend:DI
-        (subreg:QI
-         (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
-                    (match_operand:DI 2 "reg_or_cint_operand" "r,i")) 7)))]
-  "TARGET_POWERPC64 && BYTES_BIG_ENDIAN"
-  "@
-   rldcl %0,%1,%2,56
-   rldicl %0,%1,%H2,56"
-  [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no")])
-
-(define_insn "*rotldi3_internal8le"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC (zero_extend:DI
-                    (subreg:QI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r,r,r")
-                                (match_operand:DI 2 "reg_or_cint_operand" "r,i,r,i")) 0))
-                   (const_int 0)))
-   (clobber (match_scratch:DI 3 "=r,r,r,r"))]
-  "TARGET_64BIT && !BYTES_BIG_ENDIAN"
-  "@
-   rldcl. %3,%1,%2,56
-   rldicl. %3,%1,%H2,56
-   #
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no,yes,no")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
+(define_insn "*ashldi3_internal4"
+  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
+       (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "r")
+                          (match_operand:SI 2 "const_int_operand" "i"))
+               (match_operand:DI 3 "const_int_operand" "n")))]
+  "TARGET_POWERPC64 && includes_rldic_lshift_p (operands[2], operands[3])"
+  "rldic %0,%1,%H2,%W3"
+  [(set_attr "type" "shift")])
 
-(define_insn "*rotldi3_internal8be"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC (zero_extend:DI
-                    (subreg:QI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r,r,r")
-                                (match_operand:DI 2 "reg_or_cint_operand" "r,i,r,i")) 7))
-                   (const_int 0)))
-   (clobber (match_scratch:DI 3 "=r,r,r,r"))]
-  "TARGET_64BIT && BYTES_BIG_ENDIAN"
+(define_insn "ashldi3_internal5"
+  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
+       (compare:CC
+        (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
+                           (match_operand:SI 2 "const_int_operand" "i,i"))
+                (match_operand:DI 3 "const_int_operand" "n,n"))
+        (const_int 0)))
+   (clobber (match_scratch:DI 4 "=r,r"))]
+  "TARGET_64BIT && includes_rldic_lshift_p (operands[2], operands[3])"
   "@
-   rldcl. %3,%1,%2,56
-   rldicl. %3,%1,%H2,56
-   #
+   rldic. %4,%1,%H2,%W3
    #"
   [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no,yes,no")
    (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
+   (set_attr "length" "4,8")])
 
 (define_split
   [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:DI
-                    (subreg:QI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                                (match_operand:DI 2 "reg_or_cint_operand" "")) 0))
-                   (const_int 0)))
-   (clobber (match_scratch:DI 3 ""))]
-  "TARGET_POWERPC64 && !BYTES_BIG_ENDIAN && reload_completed"
-  [(set (match_dup 3)
-       (zero_extend:DI (subreg:QI
-                     (rotate:DI (match_dup 1)
-                                (match_dup 2)) 0)))
+       (compare:CC
+        (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "")
+                           (match_operand:SI 2 "const_int_operand" ""))
+                (match_operand:DI 3 "const_int_operand" ""))
+        (const_int 0)))
+   (clobber (match_scratch:DI 4 ""))]
+  "TARGET_POWERPC64 && reload_completed
+   && includes_rldic_lshift_p (operands[2], operands[3])"
+  [(set (match_dup 4)
+       (and:DI (ashift:DI (match_dup 1) (match_dup 2))
+               (match_dup 3)))
    (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:DI
-                    (subreg:QI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                                (match_operand:DI 2 "reg_or_cint_operand" "")) 7))
-                   (const_int 0)))
-   (clobber (match_scratch:DI 3 ""))]
-  "TARGET_POWERPC64 && BYTES_BIG_ENDIAN && reload_completed"
-  [(set (match_dup 3)
-       (zero_extend:DI (subreg:QI
-                     (rotate:DI (match_dup 1)
-                                (match_dup 2)) 7)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
+       (compare:CC (match_dup 4)
                    (const_int 0)))]
   "")
 
-(define_insn "*rotldi3_internal9le"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC (zero_extend:DI
-                    (subreg:QI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r,r,r")
-                                (match_operand:DI 2 "reg_or_cint_operand" "r,i,r,i")) 0))
-                   (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r,r,r")
-       (zero_extend:DI (subreg:QI (rotate:DI (match_dup 1) (match_dup 2)) 0)))]
-  "TARGET_64BIT && !BYTES_BIG_ENDIAN"
-  "@
-   rldcl. %0,%1,%2,56
-   rldicl. %0,%1,%H2,56
-   #
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no,yes,no")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
-
-(define_insn "*rotldi3_internal9be"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC (zero_extend:DI
-                    (subreg:QI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r,r,r")
-                                (match_operand:DI 2 "reg_or_cint_operand" "r,i,r,i")) 7))
-                   (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r,r,r")
-       (zero_extend:DI (subreg:QI (rotate:DI (match_dup 1) (match_dup 2)) 7)))]
-  "TARGET_64BIT && BYTES_BIG_ENDIAN"
+(define_insn "*ashldi3_internal6"
+  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
+       (compare:CC
+        (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
+                           (match_operand:SI 2 "const_int_operand" "i,i"))
+                   (match_operand:DI 3 "const_int_operand" "n,n"))
+        (const_int 0)))
+   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
+       (and:DI (ashift:DI (match_dup 1) (match_dup 2)) (match_dup 3)))]
+  "TARGET_64BIT && includes_rldic_lshift_p (operands[2], operands[3])"
   "@
-   rldcl. %0,%1,%2,56
-   rldicl. %0,%1,%H2,56
-   #
+   rldic. %0,%1,%H2,%W3
    #"
   [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no,yes,no")
    (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
-
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:DI
-                    (subreg:QI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                                (match_operand:DI 2 "reg_or_cint_operand" "")) 0))
-                   (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "")
-       (zero_extend:DI (subreg:QI (rotate:DI (match_dup 1) (match_dup 2)) 0)))]
-  "TARGET_POWERPC64 && !BYTES_BIG_ENDIAN && reload_completed"
-  [(set (match_dup 0)
-       (zero_extend:DI (subreg:QI (rotate:DI (match_dup 1) (match_dup 2)) 0)))
-   (set (match_dup 3)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
+   (set_attr "length" "4,8")])
 
 (define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:DI
-                    (subreg:QI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                                (match_operand:DI 2 "reg_or_cint_operand" "")) 7))
-                   (const_int 0)))
+  [(set (match_operand:CC 4 "cc_reg_not_micro_cr0_operand" "")
+       (compare:CC
+        (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "")
+                           (match_operand:SI 2 "const_int_operand" ""))
+                (match_operand:DI 3 "const_int_operand" ""))
+        (const_int 0)))
    (set (match_operand:DI 0 "gpc_reg_operand" "")
-       (zero_extend:DI (subreg:QI (rotate:DI (match_dup 1) (match_dup 2)) 7)))]
-  "TARGET_POWERPC64 && BYTES_BIG_ENDIAN && reload_completed"
+       (and:DI (ashift:DI (match_dup 1) (match_dup 2)) (match_dup 3)))]
+  "TARGET_POWERPC64 && reload_completed
+   && includes_rldic_lshift_p (operands[2], operands[3])"
   [(set (match_dup 0)
-       (zero_extend:DI (subreg:QI (rotate:DI (match_dup 1) (match_dup 2)) 7)))
-   (set (match_dup 3)
+       (and:DI (ashift:DI (match_dup 1) (match_dup 2))
+               (match_dup 3)))
+   (set (match_dup 4)
        (compare:CC (match_dup 0)
                    (const_int 0)))]
   "")
 
-(define_insn "*rotldi3_internal10le"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
-       (zero_extend:DI
-        (subreg:HI
-         (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
-                    (match_operand:DI 2 "reg_or_cint_operand" "r,i")) 0)))]
-  "TARGET_POWERPC64 && !BYTES_BIG_ENDIAN"
-  "@
-   rldcl %0,%1,%2,48
-   rldicl %0,%1,%H2,48"
-  [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no")])
-
-(define_insn "*rotldi3_internal10be"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
-       (zero_extend:DI
-        (subreg:HI
-         (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
-                    (match_operand:DI 2 "reg_or_cint_operand" "r,i")) 6)))]
-  "TARGET_POWERPC64 && BYTES_BIG_ENDIAN"
-  "@
-   rldcl %0,%1,%2,48
-   rldicl %0,%1,%H2,48"
-  [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no")])
-
-(define_insn "*rotldi3_internal11le"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC (zero_extend:DI
-                    (subreg:HI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r,r,r")
-                                (match_operand:DI 2 "reg_or_cint_operand" "r,i,r,i")) 0))
-                   (const_int 0)))
-   (clobber (match_scratch:DI 3 "=r,r,r,r"))]
-  "TARGET_64BIT && !BYTES_BIG_ENDIAN"
-  "@
-   rldcl. %3,%1,%2,48
-   rldicl. %3,%1,%H2,48
-   #
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no,yes,no")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
+(define_insn "*ashldi3_internal7"
+  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
+       (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "r")
+                          (match_operand:SI 2 "const_int_operand" "i"))
+               (match_operand:DI 3 "mask64_operand" "n")))]
+  "TARGET_POWERPC64 && includes_rldicr_lshift_p (operands[2], operands[3])"
+  "rldicr %0,%1,%H2,%S3"
+  [(set_attr "type" "shift")])
 
-(define_insn "*rotldi3_internal11be"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC (zero_extend:DI
-                    (subreg:HI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r,r,r")
-                                (match_operand:DI 2 "reg_or_cint_operand" "r,i,r,i")) 6))
-                   (const_int 0)))
-   (clobber (match_scratch:DI 3 "=r,r,r,r"))]
-  "TARGET_64BIT && BYTES_BIG_ENDIAN"
+(define_insn "ashldi3_internal8"
+  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
+       (compare:CC
+        (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
+                           (match_operand:SI 2 "const_int_operand" "i,i"))
+                (match_operand:DI 3 "mask64_operand" "n,n"))
+        (const_int 0)))
+   (clobber (match_scratch:DI 4 "=r,r"))]
+  "TARGET_64BIT && includes_rldicr_lshift_p (operands[2], operands[3])"
   "@
-   rldcl. %3,%1,%2,48
-   rldicl. %3,%1,%H2,48
-   #
+   rldicr. %4,%1,%H2,%S3
    #"
   [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no,yes,no")
    (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:DI
-                    (subreg:HI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                                (match_operand:DI 2 "reg_or_cint_operand" "")) 0))
-                   (const_int 0)))
-   (clobber (match_scratch:DI 3 ""))]
-  "TARGET_POWERPC64 && !BYTES_BIG_ENDIAN && reload_completed"
-  [(set (match_dup 3)
-       (zero_extend:DI (subreg:HI
-                     (rotate:DI (match_dup 1)
-                                (match_dup 2)) 0)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
+   (set_attr "length" "4,8")])
 
 (define_split
   [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:DI
-                    (subreg:HI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                                (match_operand:DI 2 "reg_or_cint_operand" "")) 6))
-                   (const_int 0)))
-   (clobber (match_scratch:DI 3 ""))]
-  "TARGET_POWERPC64 && BYTES_BIG_ENDIAN && reload_completed"
-  [(set (match_dup 3)
-       (zero_extend:DI (subreg:HI
-                     (rotate:DI (match_dup 1)
-                                (match_dup 2)) 6)))
+       (compare:CC
+        (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "")
+                           (match_operand:SI 2 "const_int_operand" ""))
+                (match_operand:DI 3 "mask64_operand" ""))
+        (const_int 0)))
+   (clobber (match_scratch:DI 4 ""))]
+  "TARGET_POWERPC64 && reload_completed
+   && includes_rldicr_lshift_p (operands[2], operands[3])"
+  [(set (match_dup 4)
+       (and:DI (ashift:DI (match_dup 1) (match_dup 2))
+               (match_dup 3)))
    (set (match_dup 0)
-       (compare:CC (match_dup 3)
+       (compare:CC (match_dup 4)
                    (const_int 0)))]
   "")
 
-(define_insn "*rotldi3_internal12le"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC (zero_extend:DI
-                    (subreg:HI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r,r,r")
-                                (match_operand:DI 2 "reg_or_cint_operand" "r,i,r,i")) 0))
-                   (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r,r,r")
-       (zero_extend:DI (subreg:HI (rotate:DI (match_dup 1) (match_dup 2)) 0)))]
-  "TARGET_64BIT && !BYTES_BIG_ENDIAN"
-  "@
-   rldcl. %0,%1,%2,48
-   rldicl. %0,%1,%H2,48
-   #
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no,yes,no")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
-
-(define_insn "*rotldi3_internal12be"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC (zero_extend:DI
-                    (subreg:HI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r,r,r")
-                                (match_operand:DI 2 "reg_or_cint_operand" "r,i,r,i")) 6))
-                   (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r,r,r")
-       (zero_extend:DI (subreg:HI (rotate:DI (match_dup 1) (match_dup 2)) 6)))]
-  "TARGET_64BIT && BYTES_BIG_ENDIAN"
+(define_insn "*ashldi3_internal9"
+  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
+       (compare:CC
+        (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
+                           (match_operand:SI 2 "const_int_operand" "i,i"))
+                   (match_operand:DI 3 "mask64_operand" "n,n"))
+        (const_int 0)))
+   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
+       (and:DI (ashift:DI (match_dup 1) (match_dup 2)) (match_dup 3)))]
+  "TARGET_64BIT && includes_rldicr_lshift_p (operands[2], operands[3])"
   "@
-   rldcl. %0,%1,%2,48
-   rldicl. %0,%1,%H2,48
-   #
+   rldicr. %0,%1,%H2,%S3
    #"
   [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no,yes,no")
    (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
+   (set_attr "length" "4,8")])
 
 (define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:DI
-                    (subreg:HI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                                (match_operand:DI 2 "reg_or_cint_operand" "")) 0))
-                   (const_int 0)))
+  [(set (match_operand:CC 4 "cc_reg_not_micro_cr0_operand" "")
+       (compare:CC
+        (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "")
+                           (match_operand:SI 2 "const_int_operand" ""))
+                (match_operand:DI 3 "mask64_operand" ""))
+        (const_int 0)))
    (set (match_operand:DI 0 "gpc_reg_operand" "")
-       (zero_extend:DI (subreg:HI (rotate:DI (match_dup 1) (match_dup 2)) 0)))]
-  "TARGET_POWERPC64 && !BYTES_BIG_ENDIAN && reload_completed"
+       (and:DI (ashift:DI (match_dup 1) (match_dup 2)) (match_dup 3)))]
+  "TARGET_POWERPC64 && reload_completed
+   && includes_rldicr_lshift_p (operands[2], operands[3])"
   [(set (match_dup 0)
-       (zero_extend:DI (subreg:HI (rotate:DI (match_dup 1) (match_dup 2)) 0)))
-   (set (match_dup 3)
+       (and:DI (ashift:DI (match_dup 1) (match_dup 2))
+               (match_dup 3)))
+   (set (match_dup 4)
        (compare:CC (match_dup 0)
                    (const_int 0)))]
   "")
 
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:DI
-                    (subreg:HI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                                (match_operand:DI 2 "reg_or_cint_operand" "")) 6))
-                   (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "")
-       (zero_extend:DI (subreg:HI (rotate:DI (match_dup 1) (match_dup 2)) 6)))]
-  "TARGET_POWERPC64 && BYTES_BIG_ENDIAN && reload_completed"
-  [(set (match_dup 0)
-       (zero_extend:DI (subreg:HI (rotate:DI (match_dup 1) (match_dup 2)) 6)))
-   (set (match_dup 3)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
 
-(define_insn "*rotldi3_internal13le"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
-       (zero_extend:DI
-        (subreg:SI
-         (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
-                    (match_operand:DI 2 "reg_or_cint_operand" "r,i")) 0)))]
-  "TARGET_POWERPC64 && !BYTES_BIG_ENDIAN"
-  "@
-   rldcl %0,%1,%2,32
-   rldicl %0,%1,%H2,32"
-  [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no")])
+(define_insn_and_split "*anddi3_2rld"
+  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
+       (and:DI (match_operand:DI 1 "gpc_reg_operand" "%r")
+               (match_operand:DI 2 "and_2rld_operand" "n")))]
+  "TARGET_POWERPC64"
+  "#"
+  ""
+  [(set (match_dup 0)
+       (and:DI (rotate:DI (match_dup 1)
+                          (match_dup 4))
+               (match_dup 5)))
+   (set (match_dup 0)
+       (and:DI (rotate:DI (match_dup 0)
+                          (match_dup 6))
+               (match_dup 7)))]
+{
+  build_mask64_2_operands (operands[2], &operands[4]);
+}
+  [(set_attr "length" "8")])
 
-(define_insn "*rotldi3_internal13be"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
-       (zero_extend:DI
-        (subreg:SI
-         (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
-                    (match_operand:DI 2 "reg_or_cint_operand" "r,i")) 4)))]
-  "TARGET_POWERPC64 && BYTES_BIG_ENDIAN"
-  "@
-   rldcl %0,%1,%2,32
-   rldicl %0,%1,%H2,32"
-  [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no")])
-
-(define_insn "*rotldi3_internal14le"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC (zero_extend:DI
-                    (subreg:SI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r,r,r")
-                                (match_operand:DI 2 "reg_or_cint_operand" "r,i,r,i")) 0))
+(define_insn_and_split "*anddi3_2rld_dot"
+  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
+       (compare:CC (and:DI (match_operand:DI 1 "gpc_reg_operand" "%r,r")
+                           (match_operand:DI 2 "and_2rld_operand" "n,n"))
                    (const_int 0)))
-   (clobber (match_scratch:DI 3 "=r,r,r,r"))]
-  "TARGET_64BIT && !BYTES_BIG_ENDIAN"
+   (clobber (match_scratch:DI 0 "=r,r"))]
+  "TARGET_64BIT && rs6000_gen_cell_microcode"
   "@
-   rldcl. %3,%1,%2,32
-   rldicl. %3,%1,%H2,32
    #
    #"
-  [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no,yes,no")
+  "&& reload_completed"
+  [(set (match_dup 0)
+       (and:DI (rotate:DI (match_dup 1)
+                          (match_dup 4))
+               (match_dup 5)))
+   (parallel [(set (match_dup 3)
+                  (compare:CC (and:DI (rotate:DI (match_dup 0)
+                                                 (match_dup 6))
+                                      (match_dup 7))
+                              (const_int 0)))
+             (clobber (match_dup 0))])]
+{
+  build_mask64_2_operands (operands[2], &operands[4]);
+}
+  [(set_attr "type" "two")
    (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
+   (set_attr "length" "8,12")])
 
-(define_insn "*rotldi3_internal14be"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC (zero_extend:DI
-                    (subreg:SI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r,r,r")
-                                (match_operand:DI 2 "reg_or_cint_operand" "r,i,r,i")) 4))
+(define_insn_and_split "*anddi3_2rld_dot2"
+  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
+       (compare:CC (and:DI (match_operand:DI 1 "gpc_reg_operand" "%r,r")
+                           (match_operand:DI 2 "and_2rld_operand" "n,n"))
                    (const_int 0)))
-   (clobber (match_scratch:DI 3 "=r,r,r,r"))]
-  "TARGET_64BIT && BYTES_BIG_ENDIAN"
+   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
+       (and:DI (match_dup 1)
+               (match_dup 2)))]
+  "TARGET_64BIT && rs6000_gen_cell_microcode"
   "@
-   rldcl. %3,%1,%2,32
-   rldicl. %3,%1,%H2,32
    #
    #"
-  [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no,yes,no")
+  "&& reload_completed"
+  [(set (match_dup 0)
+       (and:DI (rotate:DI (match_dup 1)
+                          (match_dup 4))
+               (match_dup 5)))
+   (parallel [(set (match_dup 3)
+                  (compare:CC (and:DI (rotate:DI (match_dup 0)
+                                                 (match_dup 6))
+                                      (match_dup 7))
+                              (const_int 0)))
+             (set (match_dup 0)
+                  (and:DI (rotate:DI (match_dup 0)
+                                     (match_dup 6))
+                          (match_dup 7)))])]
+{
+  build_mask64_2_operands (operands[2], &operands[4]);
+}
+  [(set_attr "type" "two")
    (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
+   (set_attr "length" "8,12")])
+\f
+;; 128-bit logical operations expanders
 
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:DI
-                    (subreg:SI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                                (match_operand:DI 2 "reg_or_cint_operand" "")) 0))
-                   (const_int 0)))
-   (clobber (match_scratch:DI 3 ""))]
-  "TARGET_POWERPC64 && !BYTES_BIG_ENDIAN && reload_completed"
-  [(set (match_dup 3)
-       (zero_extend:DI (subreg:SI
-                     (rotate:DI (match_dup 1)
-                                (match_dup 2)) 0)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
+(define_expand "and<mode>3"
+  [(set (match_operand:BOOL_128 0 "vlogical_operand" "")
+       (and:BOOL_128 (match_operand:BOOL_128 1 "vlogical_operand" "")
+                     (match_operand:BOOL_128 2 "vlogical_operand" "")))]
+  ""
   "")
 
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:DI
-                    (subreg:SI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                                (match_operand:DI 2 "reg_or_cint_operand" "")) 4))
-                   (const_int 0)))
-   (clobber (match_scratch:DI 3 ""))]
-  "TARGET_POWERPC64 && BYTES_BIG_ENDIAN && reload_completed"
-  [(set (match_dup 3)
-       (zero_extend:DI (subreg:SI
-                     (rotate:DI (match_dup 1)
-                                (match_dup 2)) 4)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
+(define_expand "ior<mode>3"
+  [(set (match_operand:BOOL_128 0 "vlogical_operand" "")
+        (ior:BOOL_128 (match_operand:BOOL_128 1 "vlogical_operand" "")
+                     (match_operand:BOOL_128 2 "vlogical_operand" "")))]
+  ""
   "")
 
-(define_insn "*rotldi3_internal15le"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC (zero_extend:DI
-                    (subreg:SI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r,r,r")
-                                (match_operand:DI 2 "reg_or_cint_operand" "r,i,r,i")) 0))
-                   (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r,r,r")
-       (zero_extend:DI (subreg:SI (rotate:DI (match_dup 1) (match_dup 2)) 0)))]
-  "TARGET_64BIT && !BYTES_BIG_ENDIAN"
-  "@
-   rldcl. %0,%1,%2,32
-   rldicl. %0,%1,%H2,32
-   #
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no,yes,no")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
-
-(define_insn "*rotldi3_internal15be"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC (zero_extend:DI
-                    (subreg:SI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "r,r,r,r")
-                                (match_operand:DI 2 "reg_or_cint_operand" "r,i,r,i")) 4))
-                   (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r,r,r")
-       (zero_extend:DI (subreg:SI (rotate:DI (match_dup 1) (match_dup 2)) 4)))]
-  "TARGET_64BIT && BYTES_BIG_ENDIAN"
-  "@
-   rldcl. %0,%1,%2,32
-   rldicl. %0,%1,%H2,32
-   #
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no,yes,no")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
-
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:DI
-                    (subreg:SI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                                (match_operand:DI 2 "reg_or_cint_operand" "")) 0))
-                   (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "")
-       (zero_extend:DI (subreg:SI (rotate:DI (match_dup 1) (match_dup 2)) 0)))]
-  "TARGET_POWERPC64 && !BYTES_BIG_ENDIAN && reload_completed"
-  [(set (match_dup 0)
-       (zero_extend:DI (subreg:SI (rotate:DI (match_dup 1) (match_dup 2)) 0)))
-   (set (match_dup 3)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
+(define_expand "xor<mode>3"
+  [(set (match_operand:BOOL_128 0 "vlogical_operand" "")
+        (xor:BOOL_128 (match_operand:BOOL_128 1 "vlogical_operand" "")
+                     (match_operand:BOOL_128 2 "vlogical_operand" "")))]
+  ""
   "")
 
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (zero_extend:DI
-                    (subreg:SI
-                     (rotate:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                                (match_operand:DI 2 "reg_or_cint_operand" "")) 4))
-                   (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "")
-       (zero_extend:DI (subreg:SI (rotate:DI (match_dup 1) (match_dup 2)) 4)))]
-  "TARGET_POWERPC64 && BYTES_BIG_ENDIAN && reload_completed"
-  [(set (match_dup 0)
-       (zero_extend:DI (subreg:SI (rotate:DI (match_dup 1) (match_dup 2)) 4)))
-   (set (match_dup 3)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
+(define_expand "one_cmpl<mode>2"
+  [(set (match_operand:BOOL_128 0 "vlogical_operand" "")
+        (not:BOOL_128 (match_operand:BOOL_128 1 "vlogical_operand" "")))]
+  ""
   "")
 
-(define_expand "ashldi3"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "")
-       (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                  (match_operand:SI 2 "reg_or_cint_operand" "")))]
-  "TARGET_POWERPC64"
+(define_expand "nor<mode>3"
+  [(set (match_operand:BOOL_128 0 "vlogical_operand" "")
+       (and:BOOL_128
+        (not:BOOL_128 (match_operand:BOOL_128 1 "vlogical_operand" ""))
+        (not:BOOL_128 (match_operand:BOOL_128 2 "vlogical_operand" ""))))]
+  ""
   "")
 
-(define_insn "*ashldi3_internal1"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
-       (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
-                  (match_operand:SI 2 "reg_or_cint_operand" "r,i")))]
-  "TARGET_POWERPC64"
-  "@
-   sld %0,%1,%2
-   sldi %0,%1,%H2"
-  [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no")])
-
-(define_insn "*ashldi3_internal2"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "r,r,r,r")
-                              (match_operand:SI 2 "reg_or_cint_operand" "r,i,r,i"))
-                   (const_int 0)))
-   (clobber (match_scratch:DI 3 "=r,r,r,r"))]
-  "TARGET_64BIT"
-  "@
-   sld. %3,%1,%2
-   sldi. %3,%1,%H2
-   #
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no,yes,no")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
+(define_expand "andc<mode>3"
+  [(set (match_operand:BOOL_128 0 "vlogical_operand" "")
+        (and:BOOL_128
+        (not:BOOL_128 (match_operand:BOOL_128 2 "vlogical_operand" ""))
+        (match_operand:BOOL_128 1 "vlogical_operand" "")))]
+  ""
+  "")
 
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_cr0_operand" "")
-       (compare:CC (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                              (match_operand:SI 2 "reg_or_cint_operand" ""))
-                   (const_int 0)))
-   (clobber (match_scratch:DI 3 ""))]
-  "TARGET_POWERPC64 && reload_completed"
-  [(set (match_dup 3)
-       (ashift:DI (match_dup 1) (match_dup 2)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
+;; Power8 vector logical instructions.
+(define_expand "eqv<mode>3"
+  [(set (match_operand:BOOL_128 0 "vlogical_operand" "")
+       (not:BOOL_128
+        (xor:BOOL_128 (match_operand:BOOL_128 1 "vlogical_operand" "")
+                      (match_operand:BOOL_128 2 "vlogical_operand" ""))))]
+  "<MODE>mode == TImode || <MODE>mode == PTImode || TARGET_P8_VECTOR"
   "")
 
-(define_insn "*ashldi3_internal3"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "r,r,r,r")
-                              (match_operand:SI 2 "reg_or_cint_operand" "r,i,r,i"))
-                   (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r,r,r")
-       (ashift:DI (match_dup 1) (match_dup 2)))]
-  "TARGET_64BIT"
-  "@
-   sld. %0,%1,%2
-   sldi. %0,%1,%H2
-   #
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no,yes,no")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
+;; Rewrite nand into canonical form
+(define_expand "nand<mode>3"
+  [(set (match_operand:BOOL_128 0 "vlogical_operand" "")
+       (ior:BOOL_128
+        (not:BOOL_128 (match_operand:BOOL_128 1 "vlogical_operand" ""))
+        (not:BOOL_128 (match_operand:BOOL_128 2 "vlogical_operand" ""))))]
+  "<MODE>mode == TImode || <MODE>mode == PTImode || TARGET_P8_VECTOR"
+  "")
 
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_cr0_operand" "")
-       (compare:CC (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                              (match_operand:SI 2 "reg_or_cint_operand" ""))
-                   (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "")
-       (ashift:DI (match_dup 1) (match_dup 2)))]
-  "TARGET_POWERPC64 && reload_completed"
-  [(set (match_dup 0)
-       (ashift:DI (match_dup 1) (match_dup 2)))
-   (set (match_dup 3)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
+;; The canonical form is to have the negated element first, so we need to
+;; reverse arguments.
+(define_expand "orc<mode>3"
+  [(set (match_operand:BOOL_128 0 "vlogical_operand" "")
+       (ior:BOOL_128
+        (not:BOOL_128 (match_operand:BOOL_128 2 "vlogical_operand" ""))
+        (match_operand:BOOL_128 1 "vlogical_operand" "")))]
+  "<MODE>mode == TImode || <MODE>mode == PTImode || TARGET_P8_VECTOR"
   "")
 
-(define_insn "*ashldi3_internal4"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
-       (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "r")
-                          (match_operand:SI 2 "const_int_operand" "i"))
-               (match_operand:DI 3 "const_int_operand" "n")))]
-  "TARGET_POWERPC64 && includes_rldic_lshift_p (operands[2], operands[3])"
-  "rldic %0,%1,%H2,%W3"
-  [(set_attr "type" "shift")])
+;; 128-bit logical operations insns and split operations
+(define_insn_and_split "*and<mode>3_internal"
+  [(set (match_operand:BOOL_128 0 "vlogical_operand" "=<BOOL_REGS_OUTPUT>")
+        (and:BOOL_128
+        (match_operand:BOOL_128 1 "vlogical_operand" "%<BOOL_REGS_OP1>")
+        (match_operand:BOOL_128 2 "vlogical_operand" "<BOOL_REGS_OP2>")))]
+  ""
+{
+  if (TARGET_VSX && vsx_register_operand (operands[0], <MODE>mode))
+    return "xxland %x0,%x1,%x2";
 
-(define_insn "ashldi3_internal5"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
-                           (match_operand:SI 2 "const_int_operand" "i,i"))
-                (match_operand:DI 3 "const_int_operand" "n,n"))
-        (const_int 0)))
-   (clobber (match_scratch:DI 4 "=r,r"))]
-  "TARGET_64BIT && includes_rldic_lshift_p (operands[2], operands[3])"
-  "@
-   rldic. %4,%1,%H2,%W3
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
+  if (TARGET_ALTIVEC && altivec_register_operand (operands[0], <MODE>mode))
+    return "vand %0,%1,%2";
 
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC
-        (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                           (match_operand:SI 2 "const_int_operand" ""))
-                (match_operand:DI 3 "const_int_operand" ""))
-        (const_int 0)))
-   (clobber (match_scratch:DI 4 ""))]
-  "TARGET_POWERPC64 && reload_completed
-   && includes_rldic_lshift_p (operands[2], operands[3])"
-  [(set (match_dup 4)
-       (and:DI (ashift:DI (match_dup 1) (match_dup 2))
-               (match_dup 3)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 4)
-                   (const_int 0)))]
-  "")
+  return "#";
+}
+  "reload_completed && int_reg_operand (operands[0], <MODE>mode)"
+  [(const_int 0)]
+{
+  rs6000_split_logical (operands, AND, false, false, false);
+  DONE;
+}
+  [(set (attr "type")
+      (if_then_else
+       (match_test "vsx_register_operand (operands[0], <MODE>mode)")
+       (const_string "vecsimple")
+       (const_string "integer")))
+   (set (attr "length")
+      (if_then_else
+       (match_test "vsx_register_operand (operands[0], <MODE>mode)")
+       (const_string "4")
+       (if_then_else
+        (match_test "TARGET_POWERPC64")
+        (const_string "8")
+        (const_string "16"))))])
 
-(define_insn "*ashldi3_internal6"
-  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
-                           (match_operand:SI 2 "const_int_operand" "i,i"))
-                   (match_operand:DI 3 "const_int_operand" "n,n"))
-        (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
-       (and:DI (ashift:DI (match_dup 1) (match_dup 2)) (match_dup 3)))]
-  "TARGET_64BIT && includes_rldic_lshift_p (operands[2], operands[3])"
-  "@
-   rldic. %0,%1,%H2,%W3
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
+;; 128-bit IOR/XOR
+(define_insn_and_split "*bool<mode>3_internal"
+  [(set (match_operand:BOOL_128 0 "vlogical_operand" "=<BOOL_REGS_OUTPUT>")
+       (match_operator:BOOL_128 3 "boolean_or_operator"
+        [(match_operand:BOOL_128 1 "vlogical_operand" "%<BOOL_REGS_OP1>")
+         (match_operand:BOOL_128 2 "vlogical_operand" "<BOOL_REGS_OP2>")]))]
+  ""
+{
+  if (TARGET_VSX && vsx_register_operand (operands[0], <MODE>mode))
+    return "xxl%q3 %x0,%x1,%x2";
 
-(define_split
-  [(set (match_operand:CC 4 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC
-        (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                           (match_operand:SI 2 "const_int_operand" ""))
-                (match_operand:DI 3 "const_int_operand" ""))
-        (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "")
-       (and:DI (ashift:DI (match_dup 1) (match_dup 2)) (match_dup 3)))]
-  "TARGET_POWERPC64 && reload_completed
-   && includes_rldic_lshift_p (operands[2], operands[3])"
-  [(set (match_dup 0)
-       (and:DI (ashift:DI (match_dup 1) (match_dup 2))
-               (match_dup 3)))
-   (set (match_dup 4)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
+  if (TARGET_ALTIVEC && altivec_register_operand (operands[0], <MODE>mode))
+    return "v%q3 %0,%1,%2";
 
-(define_insn "*ashldi3_internal7"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
-       (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "r")
-                          (match_operand:SI 2 "const_int_operand" "i"))
-               (match_operand:DI 3 "mask64_operand" "n")))]
-  "TARGET_POWERPC64 && includes_rldicr_lshift_p (operands[2], operands[3])"
-  "rldicr %0,%1,%H2,%S3"
-  [(set_attr "type" "shift")])
+  return "#";
+}
+  "reload_completed && int_reg_operand (operands[0], <MODE>mode)"
+  [(const_int 0)]
+{
+  rs6000_split_logical (operands, GET_CODE (operands[3]), false, false, false);
+  DONE;
+}
+  [(set (attr "type")
+      (if_then_else
+       (match_test "vsx_register_operand (operands[0], <MODE>mode)")
+       (const_string "vecsimple")
+       (const_string "integer")))
+   (set (attr "length")
+      (if_then_else
+       (match_test "vsx_register_operand (operands[0], <MODE>mode)")
+       (const_string "4")
+       (if_then_else
+        (match_test "TARGET_POWERPC64")
+        (const_string "8")
+        (const_string "16"))))])
 
-(define_insn "ashldi3_internal8"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
-                           (match_operand:SI 2 "const_int_operand" "i,i"))
-                (match_operand:DI 3 "mask64_operand" "n,n"))
-        (const_int 0)))
-   (clobber (match_scratch:DI 4 "=r,r"))]
-  "TARGET_64BIT && includes_rldicr_lshift_p (operands[2], operands[3])"
-  "@
-   rldicr. %4,%1,%H2,%S3
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC
-        (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                           (match_operand:SI 2 "const_int_operand" ""))
-                (match_operand:DI 3 "mask64_operand" ""))
-        (const_int 0)))
-   (clobber (match_scratch:DI 4 ""))]
-  "TARGET_POWERPC64 && reload_completed
-   && includes_rldicr_lshift_p (operands[2], operands[3])"
-  [(set (match_dup 4)
-       (and:DI (ashift:DI (match_dup 1) (match_dup 2))
-               (match_dup 3)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 4)
-                   (const_int 0)))]
-  "")
+;; 128-bit ANDC/ORC
+(define_insn_and_split "*boolc<mode>3_internal1"
+  [(set (match_operand:BOOL_128 0 "vlogical_operand" "=<BOOL_REGS_OUTPUT>")
+       (match_operator:BOOL_128 3 "boolean_operator"
+        [(not:BOOL_128
+          (match_operand:BOOL_128 2 "vlogical_operand" "<BOOL_REGS_OP2>"))
+         (match_operand:BOOL_128 1 "vlogical_operand" "<BOOL_REGS_OP1>")]))]
+  "TARGET_P8_VECTOR || (GET_CODE (operands[3]) == AND)"
+{
+  if (TARGET_VSX && vsx_register_operand (operands[0], <MODE>mode))
+    return "xxl%q3 %x0,%x1,%x2";
 
-(define_insn "*ashldi3_internal9"
-  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
-                           (match_operand:SI 2 "const_int_operand" "i,i"))
-                   (match_operand:DI 3 "mask64_operand" "n,n"))
-        (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
-       (and:DI (ashift:DI (match_dup 1) (match_dup 2)) (match_dup 3)))]
-  "TARGET_64BIT && includes_rldicr_lshift_p (operands[2], operands[3])"
-  "@
-   rldicr. %0,%1,%H2,%S3
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
+  if (TARGET_ALTIVEC && altivec_register_operand (operands[0], <MODE>mode))
+    return "v%q3 %0,%1,%2";
 
-(define_split
-  [(set (match_operand:CC 4 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC
-        (and:DI (ashift:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                           (match_operand:SI 2 "const_int_operand" ""))
-                (match_operand:DI 3 "mask64_operand" ""))
-        (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "")
-       (and:DI (ashift:DI (match_dup 1) (match_dup 2)) (match_dup 3)))]
-  "TARGET_POWERPC64 && reload_completed
-   && includes_rldicr_lshift_p (operands[2], operands[3])"
-  [(set (match_dup 0)
-       (and:DI (ashift:DI (match_dup 1) (match_dup 2))
-               (match_dup 3)))
-   (set (match_dup 4)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
+  return "#";
+}
+  "(TARGET_P8_VECTOR || (GET_CODE (operands[3]) == AND))
+   && reload_completed && int_reg_operand (operands[0], <MODE>mode)"
+  [(const_int 0)]
+{
+  rs6000_split_logical (operands, GET_CODE (operands[3]), false, false, true);
+  DONE;
+}
+  [(set (attr "type")
+      (if_then_else
+       (match_test "vsx_register_operand (operands[0], <MODE>mode)")
+       (const_string "vecsimple")
+       (const_string "integer")))
+   (set (attr "length")
+      (if_then_else
+       (match_test "vsx_register_operand (operands[0], <MODE>mode)")
+       (const_string "4")
+       (if_then_else
+        (match_test "TARGET_POWERPC64")
+        (const_string "8")
+        (const_string "16"))))])
 
-(define_expand "lshrdi3"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "")
-       (lshiftrt:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                    (match_operand:SI 2 "reg_or_cint_operand" "")))]
-  "TARGET_POWERPC64"
-  "")
+(define_insn_and_split "*boolc<mode>3_internal2"
+  [(set (match_operand:TI2 0 "int_reg_operand" "=&r,r,r")
+       (match_operator:TI2 3 "boolean_operator"
+        [(not:TI2
+          (match_operand:TI2 2 "int_reg_operand" "r,0,r"))
+         (match_operand:TI2 1 "int_reg_operand" "r,r,0")]))]
+  "!TARGET_P8_VECTOR && (GET_CODE (operands[3]) != AND)"
+  "#"
+  "reload_completed && !TARGET_P8_VECTOR && (GET_CODE (operands[3]) != AND)"
+  [(const_int 0)]
+{
+  rs6000_split_logical (operands, GET_CODE (operands[3]), false, false, true);
+  DONE;
+}
+  [(set_attr "type" "integer")
+   (set (attr "length")
+       (if_then_else
+        (match_test "TARGET_POWERPC64")
+        (const_string "8")
+        (const_string "16")))])
 
-(define_insn "*lshrdi3_internal1"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
-       (lshiftrt:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
-                    (match_operand:SI 2 "reg_or_cint_operand" "r,i")))]
-  "TARGET_POWERPC64"
-  "@
-   srd %0,%1,%2
-   srdi %0,%1,%H2"
-  [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no")])
+;; 128-bit NAND/NOR
+(define_insn_and_split "*boolcc<mode>3_internal1"
+  [(set (match_operand:BOOL_128 0 "vlogical_operand" "=<BOOL_REGS_OUTPUT>")
+       (match_operator:BOOL_128 3 "boolean_operator"
+        [(not:BOOL_128
+          (match_operand:BOOL_128 1 "vlogical_operand" "<BOOL_REGS_OP1>"))
+         (not:BOOL_128
+          (match_operand:BOOL_128 2 "vlogical_operand" "<BOOL_REGS_OP2>"))]))]
+  "TARGET_P8_VECTOR || (GET_CODE (operands[3]) == AND)"
+{
+  if (TARGET_VSX && vsx_register_operand (operands[0], <MODE>mode))
+    return "xxl%q3 %x0,%x1,%x2";
 
-(define_insn "*lshrdi3_internal2"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC (lshiftrt:DI (match_operand:DI 1 "gpc_reg_operand" "r,r,r,r")
-                                (match_operand:SI 2 "reg_or_cint_operand" "r,i,r,i"))
-                   (const_int 0)))
-   (clobber (match_scratch:DI 3 "=r,r,r,r"))]
-  "TARGET_64BIT "
-  "@
-   srd. %3,%1,%2
-   srdi. %3,%1,%H2
-   #
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no,yes,no")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
+  if (TARGET_ALTIVEC && altivec_register_operand (operands[0], <MODE>mode))
+    return "v%q3 %0,%1,%2";
 
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (lshiftrt:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                                (match_operand:SI 2 "reg_or_cint_operand" ""))
-                   (const_int 0)))
-   (clobber (match_scratch:DI 3 ""))]
-  "TARGET_POWERPC64 && reload_completed"
-  [(set (match_dup 3)
-       (lshiftrt:DI (match_dup 1) (match_dup 2)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
+  return "#";
+}
+  "(TARGET_P8_VECTOR || (GET_CODE (operands[3]) == AND))
+   && reload_completed && int_reg_operand (operands[0], <MODE>mode)"
+  [(const_int 0)]
+{
+  rs6000_split_logical (operands, GET_CODE (operands[3]), false, true, true);
+  DONE;
+}
+  [(set (attr "type")
+      (if_then_else
+       (match_test "vsx_register_operand (operands[0], <MODE>mode)")
+       (const_string "vecsimple")
+       (const_string "integer")))
+   (set (attr "length")
+      (if_then_else
+       (match_test "vsx_register_operand (operands[0], <MODE>mode)")
+       (const_string "4")
+       (if_then_else
+        (match_test "TARGET_POWERPC64")
+        (const_string "8")
+        (const_string "16"))))])
 
-(define_insn "*lshrdi3_internal3"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC (lshiftrt:DI (match_operand:DI 1 "gpc_reg_operand" "r,r,r,r")
-                                (match_operand:SI 2 "reg_or_cint_operand" "r,i,r,i"))
-                   (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r,r,r")
-       (lshiftrt:DI (match_dup 1) (match_dup 2)))]
-  "TARGET_64BIT"
-  "@
-   srd. %0,%1,%2
-   srdi. %0,%1,%H2
-   #
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no,yes,no")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
+(define_insn_and_split "*boolcc<mode>3_internal2"
+  [(set (match_operand:TI2 0 "int_reg_operand" "=&r,r,r")
+       (match_operator:TI2 3 "boolean_operator"
+        [(not:TI2
+          (match_operand:TI2 1 "int_reg_operand" "r,0,r"))
+         (not:TI2
+          (match_operand:TI2 2 "int_reg_operand" "r,r,0"))]))]
+  "!TARGET_P8_VECTOR && (GET_CODE (operands[3]) != AND)"
+  "#"
+  "reload_completed && !TARGET_P8_VECTOR && (GET_CODE (operands[3]) != AND)"
+  [(const_int 0)]
+{
+  rs6000_split_logical (operands, GET_CODE (operands[3]), false, true, true);
+  DONE;
+}
+  [(set_attr "type" "integer")
+   (set (attr "length")
+       (if_then_else
+        (match_test "TARGET_POWERPC64")
+        (const_string "8")
+        (const_string "16")))])
 
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (lshiftrt:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                                (match_operand:SI 2 "reg_or_cint_operand" ""))
-                   (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "")
-       (lshiftrt:DI (match_dup 1) (match_dup 2)))]
-  "TARGET_POWERPC64 && reload_completed"
-  [(set (match_dup 0)
-       (lshiftrt:DI (match_dup 1) (match_dup 2)))
-   (set (match_dup 3)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
 
-(define_expand "ashrdi3"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "")
-       (ashiftrt:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                    (match_operand:SI 2 "reg_or_cint_operand" "")))]
-  ""
-  "
+;; 128-bit EQV
+(define_insn_and_split "*eqv<mode>3_internal1"
+  [(set (match_operand:BOOL_128 0 "vlogical_operand" "=<BOOL_REGS_OUTPUT>")
+       (not:BOOL_128
+        (xor:BOOL_128
+         (match_operand:BOOL_128 1 "vlogical_operand" "<BOOL_REGS_OP1>")
+         (match_operand:BOOL_128 2 "vlogical_operand" "<BOOL_REGS_OP2>"))))]
+  "TARGET_P8_VECTOR"
 {
-  if (TARGET_POWERPC64)
-    ;
-  else if (GET_CODE (operands[2]) == CONST_INT)
-    {
-      emit_insn (gen_ashrdi3_no_power (operands[0], operands[1], operands[2]));
-      DONE;
-    }
-  else
-    FAIL;
-}")
+  if (vsx_register_operand (operands[0], <MODE>mode))
+    return "xxleqv %x0,%x1,%x2";
 
-(define_insn "*ashrdi3_internal1"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
-       (ashiftrt:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
-                    (match_operand:SI 2 "reg_or_cint_operand" "r,i")))]
-  "TARGET_POWERPC64"
-  "@
-   srad %0,%1,%2
-   sradi %0,%1,%H2"
-  [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no")])
+  return "#";
+}
+  "TARGET_P8_VECTOR && reload_completed
+   && int_reg_operand (operands[0], <MODE>mode)"
+  [(const_int 0)]
+{
+  rs6000_split_logical (operands, XOR, true, false, false);
+  DONE;
+}
+  [(set (attr "type")
+      (if_then_else
+       (match_test "vsx_register_operand (operands[0], <MODE>mode)")
+       (const_string "vecsimple")
+       (const_string "integer")))
+   (set (attr "length")
+      (if_then_else
+       (match_test "vsx_register_operand (operands[0], <MODE>mode)")
+       (const_string "4")
+       (if_then_else
+        (match_test "TARGET_POWERPC64")
+        (const_string "8")
+        (const_string "16"))))])
 
-(define_insn "*ashrdi3_internal2"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC (ashiftrt:DI (match_operand:DI 1 "gpc_reg_operand" "r,r,r,r")
-                                (match_operand:SI 2 "reg_or_cint_operand" "r,i,r,i"))
-                   (const_int 0)))
-   (clobber (match_scratch:DI 3 "=r,r,r,r"))]
-  "TARGET_64BIT"
-  "@
-   srad. %3,%1,%2
-   sradi. %3,%1,%H2
-   #
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no,yes,no")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
+(define_insn_and_split "*eqv<mode>3_internal2"
+  [(set (match_operand:TI2 0 "int_reg_operand" "=&r,r,r")
+       (not:TI2
+        (xor:TI2
+         (match_operand:TI2 1 "int_reg_operand" "r,0,r")
+         (match_operand:TI2 2 "int_reg_operand" "r,r,0"))))]
+  "!TARGET_P8_VECTOR"
+  "#"
+  "reload_completed && !TARGET_P8_VECTOR"
+  [(const_int 0)]
+{
+  rs6000_split_logical (operands, XOR, true, false, false);
+  DONE;
+}
+  [(set_attr "type" "integer")
+   (set (attr "length")
+       (if_then_else
+        (match_test "TARGET_POWERPC64")
+        (const_string "8")
+        (const_string "16")))])
 
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_cr0_operand" "")
-       (compare:CC (ashiftrt:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                                (match_operand:SI 2 "reg_or_cint_operand" ""))
-                   (const_int 0)))
-   (clobber (match_scratch:DI 3 ""))]
-  "TARGET_POWERPC64 && reload_completed"
-  [(set (match_dup 3)
-       (ashiftrt:DI (match_dup 1) (match_dup 2)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
+;; 128-bit one's complement
+(define_insn_and_split "*one_cmpl<mode>3_internal"
+  [(set (match_operand:BOOL_128 0 "vlogical_operand" "=<BOOL_REGS_OUTPUT>")
+       (not:BOOL_128
+         (match_operand:BOOL_128 1 "vlogical_operand" "<BOOL_REGS_UNARY>")))]
+  ""
+{
+  if (TARGET_VSX && vsx_register_operand (operands[0], <MODE>mode))
+    return "xxlnor %x0,%x1,%x1";
 
-(define_insn "*ashrdi3_internal3"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC (ashiftrt:DI (match_operand:DI 1 "gpc_reg_operand" "r,r,r,r")
-                                (match_operand:SI 2 "reg_or_cint_operand" "r,i,r,i"))
-                   (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r,r,r")
-       (ashiftrt:DI (match_dup 1) (match_dup 2)))]
-  "TARGET_64BIT"
-  "@
-   srad. %0,%1,%2
-   sradi. %0,%1,%H2
-   #
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "var_shift" "yes,no,yes,no")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8,8")])
+  if (TARGET_ALTIVEC && altivec_register_operand (operands[0], <MODE>mode))
+    return "vnor %0,%1,%1";
 
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_cr0_operand" "")
-       (compare:CC (ashiftrt:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                                (match_operand:SI 2 "reg_or_cint_operand" ""))
-                   (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "")
-       (ashiftrt:DI (match_dup 1) (match_dup 2)))]
-  "TARGET_POWERPC64 && reload_completed"
-  [(set (match_dup 0)
-       (ashiftrt:DI (match_dup 1) (match_dup 2)))
-   (set (match_dup 3)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
+  return "#";
+}
+  "reload_completed && int_reg_operand (operands[0], <MODE>mode)"
+  [(const_int 0)]
+{
+  rs6000_split_logical (operands, NOT, false, false, false);
+  DONE;
+}
+  [(set (attr "type")
+      (if_then_else
+       (match_test "vsx_register_operand (operands[0], <MODE>mode)")
+       (const_string "vecsimple")
+       (const_string "integer")))
+   (set (attr "length")
+      (if_then_else
+       (match_test "vsx_register_operand (operands[0], <MODE>mode)")
+       (const_string "4")
+       (if_then_else
+        (match_test "TARGET_POWERPC64")
+        (const_string "8")
+        (const_string "16"))))])
 
-(define_expand "anddi3"
-  [(parallel
-    [(set (match_operand:DI 0 "gpc_reg_operand" "")
-         (and:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                 (match_operand:DI 2 "reg_or_cint_operand" "")))
-     (clobber (match_scratch:CC 3 ""))])]
-  ""
+\f
+;; Now define ways of moving data around.
+
+;; Set up a register with a value from the GOT table
+
+(define_expand "movsi_got"
+  [(set (match_operand:SI 0 "gpc_reg_operand" "")
+       (unspec:SI [(match_operand:SI 1 "got_operand" "")
+                   (match_dup 2)] UNSPEC_MOVSI_GOT))]
+  "DEFAULT_ABI == ABI_V4 && flag_pic == 1"
+  "
 {
-  if (!TARGET_POWERPC64)
+  if (GET_CODE (operands[1]) == CONST)
     {
-      rtx cc = gen_rtx_SCRATCH (CCmode);
-      rs6000_split_logical (operands, AND, false, false, false, cc);
-      DONE;
+      rtx offset = const0_rtx;
+      HOST_WIDE_INT value;
+
+      operands[1] = eliminate_constant_term (XEXP (operands[1], 0), &offset);
+      value = INTVAL (offset);
+      if (value != 0)
+       {
+         rtx tmp = (!can_create_pseudo_p ()
+                    ? operands[0]
+                    : gen_reg_rtx (Pmode));
+         emit_insn (gen_movsi_got (tmp, operands[1]));
+         emit_insn (gen_addsi3 (operands[0], tmp, offset));
+         DONE;
+       }
     }
-  else if (!and64_2_operand (operands[2], DImode))
-    operands[2] = force_reg (DImode, operands[2]);
-})
 
-(define_insn "anddi3_mc"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r,r,r,r,r,r")
-       (and:DI (match_operand:DI 1 "gpc_reg_operand" "%r,r,r,r,r,r")
-               (match_operand:DI 2 "and64_2_operand" "?r,S,T,K,J,t")))
-   (clobber (match_scratch:CC 3 "=X,X,X,x,x,X"))]
-  "TARGET_POWERPC64 && rs6000_gen_cell_microcode"
-  "@
-   and %0,%1,%2
-   rldic%B2 %0,%1,0,%S2
-   rlwinm %0,%1,0,%m2,%M2
-   andi. %0,%1,%b2
-   andis. %0,%1,%u2
-   #"
-  [(set_attr "type" "*,shift,shift,logical,logical,*")
-   (set_attr "dot" "no,no,no,yes,yes,no")
-   (set_attr "length" "4,4,4,4,4,8")])
+  operands[2] = rs6000_got_register (operands[1]);
+}")
 
-(define_insn "anddi3_nomc"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r,r,r,r")
-       (and:DI (match_operand:DI 1 "gpc_reg_operand" "%r,r,r,r")
-               (match_operand:DI 2 "and64_2_operand" "?r,S,T,t")))
-   (clobber (match_scratch:CC 3 "=X,X,X,X"))]
-  "TARGET_POWERPC64 && !rs6000_gen_cell_microcode"
-  "@
-   and %0,%1,%2
-   rldic%B2 %0,%1,0,%S2
-   rlwinm %0,%1,0,%m2,%M2
-   #"
-  [(set_attr "type" "*,shift,shift,*")
-   (set_attr "length" "4,4,4,8")])
+(define_insn "*movsi_got_internal"
+  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
+       (unspec:SI [(match_operand:SI 1 "got_no_const_operand" "")
+                   (match_operand:SI 2 "gpc_reg_operand" "b")]
+                  UNSPEC_MOVSI_GOT))]
+  "DEFAULT_ABI == ABI_V4 && flag_pic == 1"
+  "lwz %0,%a1@got(%2)"
+  [(set_attr "type" "load")])
 
+;; Used by sched, shorten_branches and final when the GOT pseudo reg
+;; didn't get allocated to a hard register.
 (define_split
-  [(set (match_operand:DI 0 "gpc_reg_operand" "")
-       (and:DI (match_operand:DI 1 "gpc_reg_operand" "")
-               (match_operand:DI 2 "mask64_2_operand" "")))
-   (clobber (match_scratch:CC 3 ""))]
-  "TARGET_POWERPC64
-    && (fixed_regs[CR0_REGNO] || !logical_operand (operands[2], DImode))
-    && !mask_operand (operands[2], DImode)
-    && !mask64_operand (operands[2], DImode)"
-  [(set (match_dup 0)
-       (and:DI (rotate:DI (match_dup 1)
-                          (match_dup 4))
-               (match_dup 5)))
-   (set (match_dup 0)
-       (and:DI (rotate:DI (match_dup 0)
-                          (match_dup 6))
-               (match_dup 7)))]
-{
-  build_mask64_2_operands (operands[2], &operands[4]);
-})
+  [(set (match_operand:SI 0 "gpc_reg_operand" "")
+       (unspec:SI [(match_operand:SI 1 "got_no_const_operand" "")
+                   (match_operand:SI 2 "memory_operand" "")]
+                  UNSPEC_MOVSI_GOT))]
+  "DEFAULT_ABI == ABI_V4
+    && flag_pic == 1
+    && (reload_in_progress || reload_completed)"
+  [(set (match_dup 0) (match_dup 2))
+   (set (match_dup 0) (unspec:SI [(match_dup 1)(match_dup 0)]
+                                UNSPEC_MOVSI_GOT))]
+  "")
 
-(define_insn "*anddi3_internal2_mc"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,x,x,x,x,x,?y,?y,?y,??y,??y,?y")
-       (compare:CC (and:DI (match_operand:DI 1 "gpc_reg_operand" "%r,r,r,r,r,r,r,r,r,r,r,r")
-                           (match_operand:DI 2 "and64_2_operand" "r,S,T,K,J,t,r,S,T,K,J,t"))
-                   (const_int 0)))
-   (clobber (match_scratch:DI 3 "=r,r,r,r,r,r,r,r,r,r,r,r"))
-   (clobber (match_scratch:CC 4 "=X,X,X,X,X,X,X,X,X,x,x,X"))]
-  "TARGET_64BIT && rs6000_gen_cell_microcode"
+;; For SI, we special-case integers that can't be loaded in one insn.  We
+;; do the load 16-bits at a time.  We could do this by loading from memory,
+;; and this is even supposed to be faster, but it is simpler not to get
+;; integers in the TOC.
+(define_insn "movsi_low"
+  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
+        (mem:SI (lo_sum:SI (match_operand:SI 1 "gpc_reg_operand" "b")
+                           (match_operand 2 "" ""))))]
+  "TARGET_MACHO && ! TARGET_64BIT"
+  "lwz %0,lo16(%2)(%1)"
+  [(set_attr "type" "load")
+   (set_attr "length" "4")])
+
+(define_insn "*movsi_internal1"
+  [(set (match_operand:SI 0 "rs6000_nonimmediate_operand" "=r,r,r,m,r,r,r,r,*c*l,*h,*h")
+       (match_operand:SI 1 "input_operand" "r,U,m,r,I,L,n,*h,r,r,0"))]
+  "!TARGET_SINGLE_FPU &&
+   (gpc_reg_operand (operands[0], SImode) || gpc_reg_operand (operands[1], SImode))"
   "@
-   and. %3,%1,%2
-   rldic%B2. %3,%1,0,%S2
-   rlwinm. %3,%1,0,%m2,%M2
-   andi. %3,%1,%b2
-   andis. %3,%1,%u2
-   #
-   #
-   #
-   #
+   mr %0,%1
+   la %0,%a1
+   lwz%U1%X1 %0,%1
+   stw%U0%X0 %1,%0
+   li %0,%1
+   lis %0,%v1
    #
+   mf%1 %0
+   mt%0 %1
+   mt%0 %1
+   nop"
+  [(set_attr "type" "*,*,load,store,*,*,*,mfjmpr,mtjmpr,*,*")
+   (set_attr "length" "4,4,4,4,4,4,8,4,4,4,4")])
+
+(define_insn "*movsi_internal1_single"
+  [(set (match_operand:SI 0 "rs6000_nonimmediate_operand" "=r,r,r,m,r,r,r,r,*c*l,*h,*h,m,*f")
+        (match_operand:SI 1 "input_operand" "r,U,m,r,I,L,n,*h,r,r,0,f,m"))]
+  "TARGET_SINGLE_FPU &&
+   (gpc_reg_operand (operands[0], SImode) || gpc_reg_operand (operands[1], SImode))"
+  "@
+   mr %0,%1
+   la %0,%a1
+   lwz%U1%X1 %0,%1
+   stw%U0%X0 %1,%0
+   li %0,%1
+   lis %0,%v1
    #
-   #"
-  [(set_attr "type" "logical,shift,shift,logical,\
-                    logical,compare,compare,compare,compare,compare,\
-                    compare,compare")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,4,4,4,4,8,8,8,8,8,8,12")])
+   mf%1 %0
+   mt%0 %1
+   mt%0 %1
+   nop
+   stfs%U0%X0 %1,%0
+   lfs%U1%X1 %0,%1"
+  [(set_attr "type" "*,*,load,store,*,*,*,mfjmpr,mtjmpr,*,*,fpstore,fpload")
+   (set_attr "length" "4,4,4,4,4,4,8,4,4,4,4,4,4")])
+
+;; Split a load of a large constant into the appropriate two-insn
+;; sequence.
 
 (define_split
-  [(set (match_operand:CC 0 "cc_reg_operand" "")
-        (compare:CC (and:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                            (match_operand:DI 2 "mask64_2_operand" ""))
-                    (const_int 0)))
-   (clobber (match_scratch:DI 3 ""))
-   (clobber (match_scratch:CC 4 ""))]
-  "TARGET_64BIT && reload_completed
-    && (fixed_regs[CR0_REGNO] || !logical_operand (operands[2], DImode))
-    && !mask_operand (operands[2], DImode)
-    && !mask64_operand (operands[2], DImode)"
-  [(set (match_dup 3)
-       (and:DI (rotate:DI (match_dup 1)
-                          (match_dup 5))
-               (match_dup 6)))
-   (parallel [(set (match_dup 0)
-                  (compare:CC (and:DI (rotate:DI (match_dup 3)
-                                                 (match_dup 7))
-                                      (match_dup 8))
-                              (const_int 0)))
-             (clobber (match_dup 3))])]
+  [(set (match_operand:SI 0 "gpc_reg_operand" "")
+       (match_operand:SI 1 "const_int_operand" ""))]
+  "(unsigned HOST_WIDE_INT) (INTVAL (operands[1]) + 0x8000) >= 0x10000
+   && (INTVAL (operands[1]) & 0xffff) != 0"
+  [(set (match_dup 0)
+       (match_dup 2))
+   (set (match_dup 0)
+       (ior:SI (match_dup 0)
+               (match_dup 3)))]
   "
 {
-  build_mask64_2_operands (operands[2], &operands[5]);
+  if (rs6000_emit_set_const (operands[0], operands[1]))
+    DONE;
+  else
+    FAIL;
 }")
 
-(define_insn "*anddi3_internal3_mc"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,x,x,x,x,x,?y,?y,?y,??y,??y,?y")
-       (compare:CC (and:DI (match_operand:DI 1 "gpc_reg_operand" "%r,r,r,r,r,r,r,r,r,r,r,r")
-                           (match_operand:DI 2 "and64_2_operand" "r,S,T,K,J,t,r,S,T,K,J,t"))
+(define_insn "*mov<mode>_internal2"
+  [(set (match_operand:CC 2 "cc_reg_operand" "=y,x,?y")
+       (compare:CC (match_operand:P 1 "gpc_reg_operand" "0,r,r")
                    (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r,r,r,r,r,r,r,r,r,r,r")
-       (and:DI (match_dup 1) (match_dup 2)))
-   (clobber (match_scratch:CC 4 "=X,X,X,X,X,X,X,X,X,x,x,X"))]
-  "TARGET_64BIT && rs6000_gen_cell_microcode"
+   (set (match_operand:P 0 "gpc_reg_operand" "=r,r,r") (match_dup 1))]
+  ""
   "@
-   and. %0,%1,%2
-   rldic%B2. %0,%1,0,%S2
-   rlwinm. %0,%1,0,%m2,%M2
-   andi. %0,%1,%b2
-   andis. %0,%1,%u2
-   #
-   #
-   #
-   #
-   #
-   #
+   cmp<wd>i %2,%0,0
+   mr. %0,%1
    #"
-  [(set_attr "type" "logical,shift,shift,logical,\
-                    logical,compare,compare,compare,compare,compare,\
-                    compare,compare")
+  [(set_attr "type" "cmp,logical,cmp")
    (set_attr "dot" "yes")
-   (set_attr "length" "4,4,4,4,4,8,8,8,8,8,8,12")])
+   (set_attr "length" "4,4,8")])
 
 (define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (and:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                           (match_operand:DI 2 "and64_2_operand" ""))
+  [(set (match_operand:CC 2 "cc_reg_not_micro_cr0_operand" "")
+       (compare:CC (match_operand:P 1 "gpc_reg_operand" "")
                    (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "")
-       (and:DI (match_dup 1) (match_dup 2)))
-   (clobber (match_scratch:CC 4 ""))]
-  "TARGET_64BIT && reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (and:DI (match_dup 1) (match_dup 2)))
-              (clobber (match_dup 4))])
-   (set (match_dup 3)
+   (set (match_operand:P 0 "gpc_reg_operand" "") (match_dup 1))]
+  "reload_completed"
+  [(set (match_dup 0) (match_dup 1))
+   (set (match_dup 2)
        (compare:CC (match_dup 0)
                    (const_int 0)))]
   "")
+\f
+(define_insn "*movhi_internal"
+  [(set (match_operand:HI 0 "nonimmediate_operand" "=r,r,m,r,r,*c*l,*h")
+       (match_operand:HI 1 "input_operand" "r,m,r,i,*h,r,0"))]
+  "gpc_reg_operand (operands[0], HImode)
+   || gpc_reg_operand (operands[1], HImode)"
+  "@
+   mr %0,%1
+   lhz%U1%X1 %0,%1
+   sth%U0%X0 %1,%0
+   li %0,%w1
+   mf%1 %0
+   mt%0 %1
+   nop"
+  [(set_attr "type" "*,load,store,*,mfjmpr,mtjmpr,*")])
 
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_operand" "")
-        (compare:CC (and:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                            (match_operand:DI 2 "mask64_2_operand" ""))
-                    (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "")
-       (and:DI (match_dup 1) (match_dup 2)))
-   (clobber (match_scratch:CC 4 ""))]
-  "TARGET_64BIT && reload_completed
-    && (fixed_regs[CR0_REGNO] || !logical_operand (operands[2], DImode))
-    && !mask_operand (operands[2], DImode)
-    && !mask64_operand (operands[2], DImode)"
-  [(set (match_dup 0)
-       (and:DI (rotate:DI (match_dup 1)
-                          (match_dup 5))
-               (match_dup 6)))
-   (parallel [(set (match_dup 3)
-                  (compare:CC (and:DI (rotate:DI (match_dup 0)
-                                                 (match_dup 7))
-                                      (match_dup 8))
-                              (const_int 0)))
-             (set (match_dup 0)
-                  (and:DI (rotate:DI (match_dup 0)
-                                     (match_dup 7))
-                          (match_dup 8)))])]
-  "
-{
-  build_mask64_2_operands (operands[2], &operands[5]);
-}")
+(define_expand "mov<mode>"
+  [(set (match_operand:INT 0 "general_operand" "")
+       (match_operand:INT 1 "any_operand" ""))]
+  ""
+  "{ rs6000_emit_move (operands[0], operands[1], <MODE>mode); DONE; }")
 
-(define_expand "iordi3"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "")
-       (ior:DI (match_operand:DI 1 "gpc_reg_operand" "")
-               (match_operand:DI 2 "reg_or_cint_operand" "")))]
+(define_insn "*movqi_internal"
+  [(set (match_operand:QI 0 "nonimmediate_operand" "=r,r,m,r,r,*c*l,*h")
+       (match_operand:QI 1 "input_operand" "r,m,r,i,*h,r,0"))]
+  "gpc_reg_operand (operands[0], QImode)
+   || gpc_reg_operand (operands[1], QImode)"
+  "@
+   mr %0,%1
+   lbz%U1%X1 %0,%1
+   stb%U0%X0 %1,%0
+   li %0,%1
+   mf%1 %0
+   mt%0 %1
+   nop"
+  [(set_attr "type" "*,load,store,*,mfjmpr,mtjmpr,*")])
+\f
+;; Here is how to move condition codes around.  When we store CC data in
+;; an integer register or memory, we store just the high-order 4 bits.
+;; This lets us not shift in the most common case of CR0.
+(define_expand "movcc"
+  [(set (match_operand:CC 0 "nonimmediate_operand" "")
+       (match_operand:CC 1 "nonimmediate_operand" ""))]
   ""
-{
-  if (!TARGET_POWERPC64)
-    {
-      rs6000_split_logical (operands, IOR, false, false, false, NULL_RTX);
-      DONE;
-    }
-  else if (!reg_or_logical_cint_operand (operands[2], DImode))
-    operands[2] = force_reg (DImode, operands[2]);
-  else if (non_logical_cint_operand (operands[2], DImode))
-    {
-      HOST_WIDE_INT value;
-      rtx tmp = ((!can_create_pseudo_p ()
-                 || rtx_equal_p (operands[0], operands[1]))
-                ? operands[0] : gen_reg_rtx (DImode));
+  "")
 
-      value = INTVAL (operands[2]);
-      emit_insn (gen_iordi3 (tmp, operands[1],
-                            GEN_INT (value & (~ (HOST_WIDE_INT) 0xffff))));
+(define_insn "*movcc_internal1"
+  [(set (match_operand:CC 0 "nonimmediate_operand" "=y,x,?y,y,r,r,r,r,r,cl,r,m")
+       (match_operand:CC 1 "general_operand" "y,r,r,O,x,y,r,I,h,r,m,r"))]
+  "register_operand (operands[0], CCmode)
+   || register_operand (operands[1], CCmode)"
+  "@
+   mcrf %0,%1
+   mtcrf 128,%1
+   rlwinm %1,%1,%F0,0xffffffff\;mtcrf %R0,%1\;rlwinm %1,%1,%f0,0xffffffff
+   crxor %0,%0,%0
+   mfcr %0%Q1
+   mfcr %0%Q1\;rlwinm %0,%0,%f1,0xf0000000
+   mr %0,%1
+   li %0,%1
+   mf%1 %0
+   mt%0 %1
+   lwz%U1%X1 %0,%1
+   stw%U0%X0 %1,%0"
+  [(set (attr "type")
+     (cond [(eq_attr "alternative" "0,3")
+               (const_string "cr_logical")
+           (eq_attr "alternative" "1,2")
+               (const_string "mtcr")
+           (eq_attr "alternative" "6,7")
+               (const_string "integer")
+           (eq_attr "alternative" "8")
+               (const_string "mfjmpr")
+           (eq_attr "alternative" "9")
+               (const_string "mtjmpr")
+           (eq_attr "alternative" "10")
+               (const_string "load")
+           (eq_attr "alternative" "11")
+               (const_string "store")
+           (match_test "TARGET_MFCRF")
+               (const_string "mfcrf")
+          ]
+       (const_string "mfcr")))
+   (set_attr "length" "4,4,12,4,4,8,4,4,4,4,4,4")])
+\f
+;; For floating-point, we normally deal with the floating-point registers
+;; unless -msoft-float is used.  The sole exception is that parameter passing
+;; can produce floating-point values in fixed-point registers.  Unless the
+;; value is a simple constant or already in memory, we deal with this by
+;; allocating memory and copying the value explicitly via that memory location.
 
-      emit_insn (gen_iordi3 (operands[0], tmp, GEN_INT (value & 0xffff)));
-      DONE;
-    }
-})
+;; Move 32-bit binary/decimal floating point
+(define_expand "mov<mode>"
+  [(set (match_operand:FMOVE32 0 "nonimmediate_operand" "")
+       (match_operand:FMOVE32 1 "any_operand" ""))]
+  "<fmove_ok>"
+  "{ rs6000_emit_move (operands[0], operands[1], <MODE>mode); DONE; }")
 
-(define_expand "xordi3"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "")
-       (xor:DI (match_operand:DI 1 "gpc_reg_operand" "")
-               (match_operand:DI 2 "reg_or_cint_operand" "")))]
-  ""
+(define_split
+  [(set (match_operand:FMOVE32 0 "gpc_reg_operand" "")
+       (match_operand:FMOVE32 1 "const_double_operand" ""))]
+  "reload_completed
+   && ((GET_CODE (operands[0]) == REG && REGNO (operands[0]) <= 31)
+       || (GET_CODE (operands[0]) == SUBREG
+          && GET_CODE (SUBREG_REG (operands[0])) == REG
+          && REGNO (SUBREG_REG (operands[0])) <= 31))"
+  [(set (match_dup 2) (match_dup 3))]
+  "
 {
-  if (!TARGET_POWERPC64)
-    {
-      rs6000_split_logical (operands, XOR, false, false, false, NULL_RTX);
-      DONE;
-    }
-  else if (!reg_or_logical_cint_operand (operands[2], DImode))
-    operands[2] = force_reg (DImode, operands[2]);
-  if (non_logical_cint_operand (operands[2], DImode))
-    {
-      HOST_WIDE_INT value;
-      rtx tmp = ((!can_create_pseudo_p ()
-                 || rtx_equal_p (operands[0], operands[1]))
-                ? operands[0] : gen_reg_rtx (DImode));
+  long l;
+  REAL_VALUE_TYPE rv;
 
-      value = INTVAL (operands[2]);
-      emit_insn (gen_xordi3 (tmp, operands[1],
-                            GEN_INT (value & (~ (HOST_WIDE_INT) 0xffff))));
+  REAL_VALUE_FROM_CONST_DOUBLE (rv, operands[1]);
+  <real_value_to_target> (rv, l);
 
-      emit_insn (gen_xordi3 (operands[0], tmp, GEN_INT (value & 0xffff)));
-      DONE;
-    }
-})
+  if (! TARGET_POWERPC64)
+    operands[2] = operand_subword (operands[0], 0, 0, <MODE>mode);
+  else
+    operands[2] = gen_lowpart (SImode, operands[0]);
 
-(define_insn "*booldi3_internal1"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r,r,r")
-       (match_operator:DI 3 "boolean_or_operator"
-        [(match_operand:DI 1 "gpc_reg_operand" "%r,r,r")
-         (match_operand:DI 2 "logical_operand" "r,K,JF")]))]
-  "TARGET_POWERPC64"
-  "@
-   %q3 %0,%1,%2
-   %q3i %0,%1,%b2
-   %q3is %0,%1,%u2")
+  operands[3] = gen_int_mode (l, SImode);
+}")
 
-(define_insn "*booldi3_internal2"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC (match_operator:DI 4 "boolean_or_operator"
-        [(match_operand:DI 1 "gpc_reg_operand" "%r,r")
-         (match_operand:DI 2 "gpc_reg_operand" "r,r")])
-        (const_int 0)))
-   (clobber (match_scratch:DI 3 "=r,r"))]
-  "TARGET_64BIT"
+(define_insn "mov<mode>_hardfloat"
+  [(set (match_operand:FMOVE32 0 "nonimmediate_operand" "=!r,!r,m,f,<f32_vsx>,<f32_vsx>,!r,<f32_lr>,<f32_sm>,<f32_av>,Z,?<f32_dm>,?r,*c*l,!r,*h")
+       (match_operand:FMOVE32 1 "input_operand" "r,m,r,f,<f32_vsx>,j,j,<f32_lm>,<f32_sr>,Z,<f32_av>,r,<f32_dm>,r,h,0"))]
+  "(gpc_reg_operand (operands[0], <MODE>mode)
+   || gpc_reg_operand (operands[1], <MODE>mode))
+   && (TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_SINGLE_FLOAT)"
   "@
-   %q4. %3,%1,%2
-   #"
-  [(set_attr "type" "logical,compare")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (match_operator:DI 4 "boolean_operator"
-        [(match_operand:DI 1 "gpc_reg_operand" "")
-         (match_operand:DI 2 "gpc_reg_operand" "")])
-        (const_int 0)))
-   (clobber (match_scratch:DI 3 ""))]
-  "TARGET_POWERPC64 && reload_completed"
-  [(set (match_dup 3) (match_dup 4))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
+   mr %0,%1
+   lwz%U1%X1 %0,%1
+   stw%U0%X0 %1,%0
+   fmr %0,%1
+   xscpsgndp %x0,%x1,%x1
+   xxlxor %x0,%x0,%x0
+   li %0,0
+   <f32_li>
+   <f32_si>
+   <f32_lv>
+   <f32_sv>
+   mtvsrwz %x0,%1
+   mfvsrwz %0,%x1
+   mt%0 %1
+   mf%1 %0
+   nop"
+  [(set_attr "type" "*,load,store,fp,fp,vecsimple,integer,fpload,fpstore,fpload,fpstore,mftgpr,mffgpr,mtjmpr,mfjmpr,*")
+   (set_attr "length" "4")])
 
-(define_insn "*booldi3_internal3"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
-       (compare:CC (match_operator:DI 4 "boolean_or_operator"
-        [(match_operand:DI 1 "gpc_reg_operand" "%r,r")
-         (match_operand:DI 2 "gpc_reg_operand" "r,r")])
-        (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
-       (match_dup 4))]
-  "TARGET_64BIT"
+(define_insn "*mov<mode>_softfloat"
+  [(set (match_operand:FMOVE32 0 "nonimmediate_operand" "=r,cl,r,r,m,r,r,r,r,*h")
+       (match_operand:FMOVE32 1 "input_operand" "r,r,h,m,r,I,L,G,Fn,0"))]
+  "(gpc_reg_operand (operands[0], <MODE>mode)
+   || gpc_reg_operand (operands[1], <MODE>mode))
+   && (TARGET_SOFT_FLOAT || !TARGET_FPRS)"
   "@
-   %q4. %0,%1,%2
-   #"
-  [(set_attr "type" "logical,compare")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
+   mr %0,%1
+   mt%0 %1
+   mf%1 %0
+   lwz%U1%X1 %0,%1
+   stw%U0%X0 %1,%0
+   li %0,%1
+   lis %0,%v1
+   #
+   #
+   nop"
+  [(set_attr "type" "*,mtjmpr,mfjmpr,load,store,*,*,*,*,*")
+   (set_attr "length" "4,4,4,4,4,4,4,4,8,4")])
+
+\f
+;; Move 64-bit binary/decimal floating point
+(define_expand "mov<mode>"
+  [(set (match_operand:FMOVE64 0 "nonimmediate_operand" "")
+       (match_operand:FMOVE64 1 "any_operand" ""))]
+  ""
+  "{ rs6000_emit_move (operands[0], operands[1], <MODE>mode); DONE; }")
 
 (define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (match_operator:DI 4 "boolean_operator"
-        [(match_operand:DI 1 "gpc_reg_operand" "")
-         (match_operand:DI 2 "gpc_reg_operand" "")])
-        (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "")
-       (match_dup 4))]
-  "TARGET_POWERPC64 && reload_completed"
-  [(set (match_dup 0) (match_dup 4))
-   (set (match_dup 3)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
+  [(set (match_operand:FMOVE64 0 "gpc_reg_operand" "")
+       (match_operand:FMOVE64 1 "const_int_operand" ""))]
+  "! TARGET_POWERPC64 && reload_completed
+   && ((GET_CODE (operands[0]) == REG && REGNO (operands[0]) <= 31)
+       || (GET_CODE (operands[0]) == SUBREG
+          && GET_CODE (SUBREG_REG (operands[0])) == REG
+          && REGNO (SUBREG_REG (operands[0])) <= 31))"
+  [(set (match_dup 2) (match_dup 4))
+   (set (match_dup 3) (match_dup 1))]
+  "
+{
+  int endian = (WORDS_BIG_ENDIAN == 0);
+  HOST_WIDE_INT value = INTVAL (operands[1]);
 
-;; Split a logical operation that we can't do in one insn into two insns,
-;; each of which does one 16-bit part.  This is used by combine.
+  operands[2] = operand_subword (operands[0], endian, 0, <MODE>mode);
+  operands[3] = operand_subword (operands[0], 1 - endian, 0, <MODE>mode);
+  operands[4] = GEN_INT (value >> 32);
+  operands[1] = GEN_INT (((value & 0xffffffff) ^ 0x80000000) - 0x80000000);
+}")
 
 (define_split
-  [(set (match_operand:DI 0 "gpc_reg_operand" "")
-       (match_operator:DI 3 "boolean_or_operator"
-        [(match_operand:DI 1 "gpc_reg_operand" "")
-         (match_operand:DI 2 "non_logical_cint_operand" "")]))]
-  "TARGET_POWERPC64"
-  [(set (match_dup 0) (match_dup 4))
-   (set (match_dup 0) (match_dup 5))]
-"
-{
-  rtx i3,i4;
-
-  i3 = GEN_INT (INTVAL (operands[2]) & (~ (HOST_WIDE_INT) 0xffff));
-  i4 = GEN_INT (INTVAL (operands[2]) & 0xffff);
-  operands[4] = gen_rtx_fmt_ee (GET_CODE (operands[3]), DImode,
-                               operands[1], i3);
-  operands[5] = gen_rtx_fmt_ee (GET_CODE (operands[3]), DImode,
-                               operands[0], i4);
-}")
+  [(set (match_operand:FMOVE64 0 "gpc_reg_operand" "")
+       (match_operand:FMOVE64 1 "const_double_operand" ""))]
+  "! TARGET_POWERPC64 && reload_completed
+   && ((GET_CODE (operands[0]) == REG && REGNO (operands[0]) <= 31)
+       || (GET_CODE (operands[0]) == SUBREG
+          && GET_CODE (SUBREG_REG (operands[0])) == REG
+          && REGNO (SUBREG_REG (operands[0])) <= 31))"
+  [(set (match_dup 2) (match_dup 4))
+   (set (match_dup 3) (match_dup 5))]
+  "
+{
+  int endian = (WORDS_BIG_ENDIAN == 0);
+  long l[2];
+  REAL_VALUE_TYPE rv;
 
-(define_insn "*boolcdi3_internal1"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
-       (match_operator:DI 3 "boolean_operator"
-        [(not:DI (match_operand:DI 1 "gpc_reg_operand" "r"))
-         (match_operand:DI 2 "gpc_reg_operand" "r")]))]
-  "TARGET_POWERPC64"
-  "%q3 %0,%2,%1")
+  REAL_VALUE_FROM_CONST_DOUBLE (rv, operands[1]);
+  <real_value_to_target> (rv, l);
 
-(define_insn "*boolcdi3_internal2"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC (match_operator:DI 4 "boolean_operator"
-        [(not:DI (match_operand:DI 1 "gpc_reg_operand" "r,r"))
-         (match_operand:DI 2 "gpc_reg_operand" "r,r")])
-        (const_int 0)))
-   (clobber (match_scratch:DI 3 "=r,r"))]
-  "TARGET_64BIT"
-  "@
-   %q4. %3,%2,%1
-   #"
-  [(set_attr "type" "logical,compare")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
+  operands[2] = operand_subword (operands[0], endian, 0, <MODE>mode);
+  operands[3] = operand_subword (operands[0], 1 - endian, 0, <MODE>mode);
+  operands[4] = gen_int_mode (l[endian], SImode);
+  operands[5] = gen_int_mode (l[1 - endian], SImode);
+}")
 
 (define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (match_operator:DI 4 "boolean_operator"
-        [(not:DI (match_operand:DI 1 "gpc_reg_operand" ""))
-         (match_operand:DI 2 "gpc_reg_operand" "")])
-        (const_int 0)))
-   (clobber (match_scratch:DI 3 ""))]
-  "TARGET_POWERPC64 && reload_completed"
-  [(set (match_dup 3) (match_dup 4))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
+  [(set (match_operand:FMOVE64 0 "gpc_reg_operand" "")
+       (match_operand:FMOVE64 1 "const_double_operand" ""))]
+  "TARGET_POWERPC64 && reload_completed
+   && ((GET_CODE (operands[0]) == REG && REGNO (operands[0]) <= 31)
+       || (GET_CODE (operands[0]) == SUBREG
+          && GET_CODE (SUBREG_REG (operands[0])) == REG
+          && REGNO (SUBREG_REG (operands[0])) <= 31))"
+  [(set (match_dup 2) (match_dup 3))]
+  "
+{
+  int endian = (WORDS_BIG_ENDIAN == 0);
+  long l[2];
+  REAL_VALUE_TYPE rv;
+  HOST_WIDE_INT val;
 
-(define_insn "*boolcdi3_internal3"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
-       (compare:CC (match_operator:DI 4 "boolean_operator"
-        [(not:DI (match_operand:DI 1 "gpc_reg_operand" "%r,r"))
-         (match_operand:DI 2 "gpc_reg_operand" "r,r")])
-        (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
-       (match_dup 4))]
-  "TARGET_64BIT"
-  "@
-   %q4. %0,%2,%1
-   #"
-  [(set_attr "type" "logical,compare")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
+  REAL_VALUE_FROM_CONST_DOUBLE (rv, operands[1]);
+  <real_value_to_target> (rv, l);
 
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (match_operator:DI 4 "boolean_operator"
-        [(not:DI (match_operand:DI 1 "gpc_reg_operand" ""))
-         (match_operand:DI 2 "gpc_reg_operand" "")])
-        (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "")
-       (match_dup 4))]
-  "TARGET_POWERPC64 && reload_completed"
-  [(set (match_dup 0) (match_dup 4))
-   (set (match_dup 3)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
+  operands[2] = gen_lowpart (DImode, operands[0]);
+  /* HIGHPART is lower memory address when WORDS_BIG_ENDIAN.  */
+  val = ((HOST_WIDE_INT)(unsigned long)l[endian] << 32
+         | ((HOST_WIDE_INT)(unsigned long)l[1 - endian]));
 
-(define_insn "*boolccdi3_internal1"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
-       (match_operator:DI 3 "boolean_operator"
-        [(not:DI (match_operand:DI 1 "gpc_reg_operand" "r"))
-         (not:DI (match_operand:DI 2 "gpc_reg_operand" "r"))]))]
-  "TARGET_POWERPC64"
-  "%q3 %0,%1,%2")
+  operands[3] = gen_int_mode (val, DImode);
+}")
 
-(define_insn "*boolccdi3_internal2"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC (match_operator:DI 4 "boolean_operator"
-        [(not:DI (match_operand:DI 1 "gpc_reg_operand" "r,r"))
-         (not:DI (match_operand:DI 2 "gpc_reg_operand" "r,r"))])
-        (const_int 0)))
-   (clobber (match_scratch:DI 3 "=r,r"))]
-  "TARGET_64BIT"
-  "@
-   %q4. %3,%1,%2
-   #"
-  [(set_attr "type" "logical,compare")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
+;; Don't have reload use general registers to load a constant.  It is
+;; less efficient than loading the constant into an FP register, since
+;; it will probably be used there.
 
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (match_operator:DI 4 "boolean_operator"
-        [(not:DI (match_operand:DI 1 "gpc_reg_operand" ""))
-         (not:DI (match_operand:DI 2 "gpc_reg_operand" ""))])
-        (const_int 0)))
-   (clobber (match_scratch:DI 3 ""))]
-  "TARGET_POWERPC64 && reload_completed"
-  [(set (match_dup 3) (match_dup 4))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
+;; The move constraints are ordered to prefer floating point registers before
+;; general purpose registers to avoid doing a store and a load to get the value
+;; into a floating point register when it is needed for a floating point
+;; operation.  Prefer traditional floating point registers over VSX registers,
+;; since the D-form version of the memory instructions does not need a GPR for
+;; reloading.
 
-(define_insn "*boolccdi3_internal3"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
-       (compare:CC (match_operator:DI 4 "boolean_operator"
-        [(not:DI (match_operand:DI 1 "gpc_reg_operand" "%r,r"))
-         (not:DI (match_operand:DI 2 "gpc_reg_operand" "r,r"))])
-        (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
-       (match_dup 4))]
-  "TARGET_64BIT"
+;; If we have FPR registers, rs6000_emit_move has moved all constants to memory,
+;; except for 0.0 which can be created on VSX with an xor instruction.
+
+(define_insn "*mov<mode>_hardfloat32"
+  [(set (match_operand:FMOVE64 0 "nonimmediate_operand" "=m,d,d,<f64_av>,Z,<f64_vsx>,<f64_vsx>,!r,Y,r,!r")
+       (match_operand:FMOVE64 1 "input_operand" "d,m,d,Z,<f64_av>,<f64_vsx>,j,j,r,Y,r"))]
+  "! TARGET_POWERPC64 && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT 
+   && (gpc_reg_operand (operands[0], <MODE>mode)
+       || gpc_reg_operand (operands[1], <MODE>mode))"
   "@
-   %q4. %0,%1,%2
+   stfd%U0%X0 %1,%0
+   lfd%U1%X1 %0,%1
+   fmr %0,%1
+   lxsd%U1x %x0,%y1
+   stxsd%U0x %x1,%y0
+   xxlor %x0,%x1,%x1
+   xxlxor %x0,%x0,%x0
+   #
+   #
+   #
    #"
-  [(set_attr "type" "logical,compare")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,8")])
+  [(set_attr "type" "fpstore,fpload,fp,fpload,fpstore,vecsimple,vecsimple,two,store,load,two")
+   (set_attr "length" "4,4,4,4,4,4,4,8,8,8,8")])
 
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (match_operator:DI 4 "boolean_operator"
-        [(not:DI (match_operand:DI 1 "gpc_reg_operand" ""))
-         (not:DI (match_operand:DI 2 "gpc_reg_operand" ""))])
-        (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "")
-       (match_dup 4))]
-  "TARGET_POWERPC64 && reload_completed"
-  [(set (match_dup 0) (match_dup 4))
-   (set (match_dup 3)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
+(define_insn "*mov<mode>_softfloat32"
+  [(set (match_operand:FMOVE64 0 "nonimmediate_operand" "=Y,r,r,r,r,r")
+       (match_operand:FMOVE64 1 "input_operand" "r,Y,r,G,H,F"))]
+  "! TARGET_POWERPC64 
+   && ((TARGET_FPRS && TARGET_SINGLE_FLOAT) 
+       || TARGET_SOFT_FLOAT || TARGET_E500_SINGLE
+       || (<MODE>mode == DDmode && TARGET_E500_DOUBLE))
+   && (gpc_reg_operand (operands[0], <MODE>mode)
+       || gpc_reg_operand (operands[1], <MODE>mode))"
+  "#"
+  [(set_attr "type" "store,load,two,*,*,*")
+   (set_attr "length" "8,8,8,8,12,16")])
 
-;; Eqv operation.
-(define_insn "*eqv<mode>3"
-  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
-       (not:GPR
-        (xor:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")
-                 (match_operand:GPR 2 "gpc_reg_operand" "r"))))]
-  ""
-  "eqv %0,%1,%2"
-  [(set_attr "type" "integer")
+; ld/std require word-aligned displacements -> 'Y' constraint.
+; List Y->r and r->Y before r->r for reload.
+(define_insn "*mov<mode>_hardfloat64"
+  [(set (match_operand:FMOVE64 0 "nonimmediate_operand" "=m,d,d,<f64_av>,Z,<f64_vsx>,<f64_vsx>,!r,Y,r,!r,*c*l,!r,*h,r,wg,r,<f64_dm>")
+       (match_operand:FMOVE64 1 "input_operand" "d,m,d,Z,<f64_av>,<f64_vsx>,j,j,r,Y,r,r,h,0,wg,r,<f64_dm>,r"))]
+  "TARGET_POWERPC64 && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT
+   && (gpc_reg_operand (operands[0], <MODE>mode)
+       || gpc_reg_operand (operands[1], <MODE>mode))"
+  "@
+   stfd%U0%X0 %1,%0
+   lfd%U1%X1 %0,%1
+   fmr %0,%1
+   lxsd%U1x %x0,%y1
+   stxsd%U0x %x1,%y0
+   xxlor %x0,%x1,%x1
+   xxlxor %x0,%x0,%x0
+   li %0,0
+   std%U0%X0 %1,%0
+   ld%U1%X1 %0,%1
+   mr %0,%1
+   mt%0 %1
+   mf%1 %0
+   nop
+   mftgpr %0,%1
+   mffgpr %0,%1
+   mfvsrd %0,%x1
+   mtvsrd %x0,%1"
+  [(set_attr "type" "fpstore,fpload,fp,fpload,fpstore,vecsimple,vecsimple,integer,store,load,*,mtjmpr,mfjmpr,*,mftgpr,mffgpr,mftgpr,mffgpr")
    (set_attr "length" "4")])
 
+(define_insn "*mov<mode>_softfloat64"
+  [(set (match_operand:FMOVE64 0 "nonimmediate_operand" "=Y,r,r,cl,r,r,r,r,*h")
+       (match_operand:FMOVE64 1 "input_operand" "r,Y,r,r,h,G,H,F,0"))]
+  "TARGET_POWERPC64 && (TARGET_SOFT_FLOAT || !TARGET_FPRS)
+   && (gpc_reg_operand (operands[0], <MODE>mode)
+       || gpc_reg_operand (operands[1], <MODE>mode))"
+  "@
+   std%U0%X0 %1,%0
+   ld%U1%X1 %0,%1
+   mr %0,%1
+   mt%0 %1
+   mf%1 %0
+   #
+   #
+   #
+   nop"
+  [(set_attr "type" "store,load,*,mtjmpr,mfjmpr,*,*,*,*")
+   (set_attr "length" "4,4,4,4,4,8,12,16,4")])
 \f
-;; 128-bit logical operations expanders
-
-(define_expand "and<mode>3"
-  [(parallel [(set (match_operand:BOOL_128 0 "vlogical_operand" "")
-                  (and:BOOL_128
-                   (match_operand:BOOL_128 1 "vlogical_operand" "")
-                   (match_operand:BOOL_128 2 "vlogical_operand" "")))
-             (clobber (match_scratch:CC 3 ""))])]
+(define_expand "mov<mode>"
+  [(set (match_operand:FMOVE128 0 "general_operand" "")
+       (match_operand:FMOVE128 1 "any_operand" ""))]
   ""
-  "")
+  "{ rs6000_emit_move (operands[0], operands[1], <MODE>mode); DONE; }")
 
-(define_expand "ior<mode>3"
-  [(set (match_operand:BOOL_128 0 "vlogical_operand" "")
-        (ior:BOOL_128 (match_operand:BOOL_128 1 "vlogical_operand" "")
-                     (match_operand:BOOL_128 2 "vlogical_operand" "")))]
-  ""
-  "")
+;; It's important to list Y->r and r->Y before r->r because otherwise
+;; reload, given m->r, will try to pick r->r and reload it, which
+;; doesn't make progress.
 
-(define_expand "xor<mode>3"
-  [(set (match_operand:BOOL_128 0 "vlogical_operand" "")
-        (xor:BOOL_128 (match_operand:BOOL_128 1 "vlogical_operand" "")
-                     (match_operand:BOOL_128 2 "vlogical_operand" "")))]
-  ""
-  "")
+;; We can't split little endian direct moves of TDmode, because the words are
+;; not swapped like they are for TImode or TFmode.  Subregs therefore are
+;; problematical.  Don't allow direct move for this case.
 
-(define_expand "one_cmpl<mode>2"
-  [(set (match_operand:BOOL_128 0 "vlogical_operand" "")
-        (not:BOOL_128 (match_operand:BOOL_128 1 "vlogical_operand" "")))]
-  ""
-  "")
+(define_insn_and_split "*mov<mode>_64bit_dm"
+  [(set (match_operand:FMOVE128 0 "nonimmediate_operand" "=m,d,d,ws,Y,r,r,r,wm")
+       (match_operand:FMOVE128 1 "input_operand" "d,m,d,j,r,jY,r,wm,r"))]
+  "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_POWERPC64
+   && (<MODE>mode != TDmode || WORDS_BIG_ENDIAN)
+   && (gpc_reg_operand (operands[0], <MODE>mode)
+       || gpc_reg_operand (operands[1], <MODE>mode))"
+  "#"
+  "&& reload_completed"
+  [(pc)]
+{ rs6000_split_multireg_move (operands[0], operands[1]); DONE; }
+  [(set_attr "length" "8,8,8,8,12,12,8,8,8")])
 
-(define_expand "nor<mode>3"
-  [(set (match_operand:BOOL_128 0 "vlogical_operand" "")
-       (and:BOOL_128
-        (not:BOOL_128 (match_operand:BOOL_128 1 "vlogical_operand" ""))
-        (not:BOOL_128 (match_operand:BOOL_128 2 "vlogical_operand" ""))))]
-  ""
-  "")
+(define_insn_and_split "*movtd_64bit_nodm"
+  [(set (match_operand:TD 0 "nonimmediate_operand" "=m,d,d,ws,Y,r,r")
+       (match_operand:TD 1 "input_operand" "d,m,d,j,r,jY,r"))]
+  "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_POWERPC64 && !WORDS_BIG_ENDIAN
+   && (gpc_reg_operand (operands[0], TDmode)
+       || gpc_reg_operand (operands[1], TDmode))"
+  "#"
+  "&& reload_completed"
+  [(pc)]
+{ rs6000_split_multireg_move (operands[0], operands[1]); DONE; }
+  [(set_attr "length" "8,8,8,8,12,12,8")])
 
-(define_expand "andc<mode>3"
-  [(set (match_operand:BOOL_128 0 "vlogical_operand" "")
-        (and:BOOL_128
-        (not:BOOL_128 (match_operand:BOOL_128 2 "vlogical_operand" ""))
-        (match_operand:BOOL_128 1 "vlogical_operand" "")))]
-  ""
-  "")
-
-;; Power8 vector logical instructions.
-(define_expand "eqv<mode>3"
-  [(set (match_operand:BOOL_128 0 "vlogical_operand" "")
-       (not:BOOL_128
-        (xor:BOOL_128 (match_operand:BOOL_128 1 "vlogical_operand" "")
-                      (match_operand:BOOL_128 2 "vlogical_operand" ""))))]
-  "<MODE>mode == TImode || <MODE>mode == PTImode || TARGET_P8_VECTOR"
-  "")
-
-;; Rewrite nand into canonical form
-(define_expand "nand<mode>3"
-  [(set (match_operand:BOOL_128 0 "vlogical_operand" "")
-       (ior:BOOL_128
-        (not:BOOL_128 (match_operand:BOOL_128 1 "vlogical_operand" ""))
-        (not:BOOL_128 (match_operand:BOOL_128 2 "vlogical_operand" ""))))]
-  "<MODE>mode == TImode || <MODE>mode == PTImode || TARGET_P8_VECTOR"
-  "")
+(define_insn_and_split "*mov<mode>_32bit"
+  [(set (match_operand:FMOVE128 0 "nonimmediate_operand" "=m,d,d,ws,Y,r,r")
+       (match_operand:FMOVE128 1 "input_operand" "d,m,d,j,r,jY,r"))]
+  "TARGET_HARD_FLOAT && TARGET_FPRS && !TARGET_POWERPC64
+   && (gpc_reg_operand (operands[0], <MODE>mode)
+       || gpc_reg_operand (operands[1], <MODE>mode))"
+  "#"
+  "&& reload_completed"
+  [(pc)]
+{ rs6000_split_multireg_move (operands[0], operands[1]); DONE; }
+  [(set_attr "length" "8,8,8,8,20,20,16")])
 
-;; The canonical form is to have the negated element first, so we need to
-;; reverse arguments.
-(define_expand "orc<mode>3"
-  [(set (match_operand:BOOL_128 0 "vlogical_operand" "")
-       (ior:BOOL_128
-        (not:BOOL_128 (match_operand:BOOL_128 2 "vlogical_operand" ""))
-        (match_operand:BOOL_128 1 "vlogical_operand" "")))]
-  "<MODE>mode == TImode || <MODE>mode == PTImode || TARGET_P8_VECTOR"
-  "")
+(define_insn_and_split "*mov<mode>_softfloat"
+  [(set (match_operand:FMOVE128 0 "rs6000_nonimmediate_operand" "=Y,r,r")
+       (match_operand:FMOVE128 1 "input_operand" "r,YGHF,r"))]
+  "(TARGET_SOFT_FLOAT || !TARGET_FPRS)
+   && (gpc_reg_operand (operands[0], <MODE>mode)
+       || gpc_reg_operand (operands[1], <MODE>mode))"
+  "#"
+  "&& reload_completed"
+  [(pc)]
+{ rs6000_split_multireg_move (operands[0], operands[1]); DONE; }
+  [(set_attr "length" "20,20,16")])
 
-;; 128-bit logical operations insns and split operations
-(define_insn_and_split "*and<mode>3_internal"
-  [(set (match_operand:BOOL_128 0 "vlogical_operand" "=<BOOL_REGS_OUTPUT>")
-        (and:BOOL_128
-        (match_operand:BOOL_128 1 "vlogical_operand" "%<BOOL_REGS_OP1>")
-        (match_operand:BOOL_128 2 "vlogical_operand" "<BOOL_REGS_OP2>")))
-   (clobber (match_scratch:CC 3 "<BOOL_REGS_AND_CR0>"))]
-  ""
+(define_expand "extenddftf2"
+  [(set (match_operand:TF 0 "nonimmediate_operand" "")
+       (float_extend:TF (match_operand:DF 1 "input_operand" "")))]
+  "!TARGET_IEEEQUAD
+   && TARGET_HARD_FLOAT
+   && (TARGET_FPRS || TARGET_E500_DOUBLE)
+   && TARGET_LONG_DOUBLE_128"
 {
-  if (TARGET_VSX && vsx_register_operand (operands[0], <MODE>mode))
-    return "xxland %x0,%x1,%x2";
+  if (TARGET_E500_DOUBLE)
+    emit_insn (gen_spe_extenddftf2 (operands[0], operands[1]));
+  else
+    emit_insn (gen_extenddftf2_fprs (operands[0], operands[1]));
+  DONE;
+})
 
-  if (TARGET_ALTIVEC && altivec_register_operand (operands[0], <MODE>mode))
-    return "vand %0,%1,%2";
+(define_expand "extenddftf2_fprs"
+  [(parallel [(set (match_operand:TF 0 "nonimmediate_operand" "")
+                  (float_extend:TF (match_operand:DF 1 "input_operand" "")))
+             (use (match_dup 2))])]
+  "!TARGET_IEEEQUAD
+   && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT 
+   && TARGET_LONG_DOUBLE_128"
+{
+  /* VSX can create 0.0 directly, otherwise let rs6000_emit_move create
+     the proper constant.  */
+  if (TARGET_VSX)
+    operands[2] = CONST0_RTX (DFmode);
+  else
+    {
+      operands[2] = gen_reg_rtx (DFmode);
+      rs6000_emit_move (operands[2], CONST0_RTX (DFmode), DFmode);
+    }
+})
 
-  return "#";
-}
-  "reload_completed && int_reg_operand (operands[0], <MODE>mode)"
-  [(const_int 0)]
+(define_insn_and_split "*extenddftf2_internal"
+  [(set (match_operand:TF 0 "nonimmediate_operand" "=m,Y,ws,d,&d")
+       (float_extend:TF (match_operand:DF 1 "input_operand" "d,r,md,md,md")))
+   (use (match_operand:DF 2 "zero_reg_mem_operand" "d,r,j,m,d"))]
+  "!TARGET_IEEEQUAD
+   && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT 
+   && TARGET_LONG_DOUBLE_128"
+  "#"
+  "&& reload_completed"
+  [(pc)]
 {
-  rs6000_split_logical (operands, AND, false, false, false, operands[3]);
+  const int lo_word = LONG_DOUBLE_LARGE_FIRST ? GET_MODE_SIZE (DFmode) : 0;
+  const int hi_word = LONG_DOUBLE_LARGE_FIRST ? 0 : GET_MODE_SIZE (DFmode);
+  emit_move_insn (simplify_gen_subreg (DFmode, operands[0], TFmode, hi_word),
+                 operands[1]);
+  emit_move_insn (simplify_gen_subreg (DFmode, operands[0], TFmode, lo_word),
+                 operands[2]);
   DONE;
-}
-  [(set (attr "type")
-      (if_then_else
-       (match_test "vsx_register_operand (operands[0], <MODE>mode)")
-       (const_string "vecsimple")
-       (const_string "integer")))
-   (set (attr "length")
-      (if_then_else
-       (match_test "vsx_register_operand (operands[0], <MODE>mode)")
-       (const_string "4")
-       (if_then_else
-        (match_test "TARGET_POWERPC64")
-        (const_string "8")
-        (const_string "16"))))])
+})
 
-;; 128-bit IOR/XOR
-(define_insn_and_split "*bool<mode>3_internal"
-  [(set (match_operand:BOOL_128 0 "vlogical_operand" "=<BOOL_REGS_OUTPUT>")
-       (match_operator:BOOL_128 3 "boolean_or_operator"
-        [(match_operand:BOOL_128 1 "vlogical_operand" "%<BOOL_REGS_OP1>")
-         (match_operand:BOOL_128 2 "vlogical_operand" "<BOOL_REGS_OP2>")]))]
-  ""
+(define_expand "extendsftf2"
+  [(set (match_operand:TF 0 "nonimmediate_operand" "")
+       (float_extend:TF (match_operand:SF 1 "gpc_reg_operand" "")))]
+  "!TARGET_IEEEQUAD
+   && TARGET_HARD_FLOAT
+   && (TARGET_FPRS || TARGET_E500_DOUBLE)
+   && TARGET_LONG_DOUBLE_128"
 {
-  if (TARGET_VSX && vsx_register_operand (operands[0], <MODE>mode))
-    return "xxl%q3 %x0,%x1,%x2";
+  rtx tmp = gen_reg_rtx (DFmode);
+  emit_insn (gen_extendsfdf2 (tmp, operands[1]));
+  emit_insn (gen_extenddftf2 (operands[0], tmp));
+  DONE;
+})
 
-  if (TARGET_ALTIVEC && altivec_register_operand (operands[0], <MODE>mode))
-    return "v%q3 %0,%1,%2";
+(define_expand "trunctfdf2"
+  [(set (match_operand:DF 0 "gpc_reg_operand" "")
+       (float_truncate:DF (match_operand:TF 1 "gpc_reg_operand" "")))]
+  "!TARGET_IEEEQUAD
+   && TARGET_HARD_FLOAT
+   && (TARGET_FPRS || TARGET_E500_DOUBLE)
+   && TARGET_LONG_DOUBLE_128"
+  "")
 
-  return "#";
-}
-  "reload_completed && int_reg_operand (operands[0], <MODE>mode)"
+(define_insn_and_split "trunctfdf2_internal1"
+  [(set (match_operand:DF 0 "gpc_reg_operand" "=d,?d")
+       (float_truncate:DF (match_operand:TF 1 "gpc_reg_operand" "0,d")))]
+  "!TARGET_IEEEQUAD && !TARGET_XL_COMPAT
+   && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_LONG_DOUBLE_128"
+  "@
+   #
+   fmr %0,%1"
+  "&& reload_completed && REGNO (operands[0]) == REGNO (operands[1])"
   [(const_int 0)]
 {
-  rs6000_split_logical (operands, GET_CODE (operands[3]), false, false, false,
-                       NULL_RTX);
+  emit_note (NOTE_INSN_DELETED);
   DONE;
 }
-  [(set (attr "type")
-      (if_then_else
-       (match_test "vsx_register_operand (operands[0], <MODE>mode)")
-       (const_string "vecsimple")
-       (const_string "integer")))
-   (set (attr "length")
-      (if_then_else
-       (match_test "vsx_register_operand (operands[0], <MODE>mode)")
-       (const_string "4")
-       (if_then_else
-        (match_test "TARGET_POWERPC64")
-        (const_string "8")
-        (const_string "16"))))])
-
-;; 128-bit ANDC/ORC
-(define_insn_and_split "*boolc<mode>3_internal1"
-  [(set (match_operand:BOOL_128 0 "vlogical_operand" "=<BOOL_REGS_OUTPUT>")
-       (match_operator:BOOL_128 3 "boolean_operator"
-        [(not:BOOL_128
-          (match_operand:BOOL_128 2 "vlogical_operand" "<BOOL_REGS_OP1>"))
-         (match_operand:BOOL_128 1 "vlogical_operand" "<BOOL_REGS_OP2>")]))]
-  "TARGET_P8_VECTOR || (GET_CODE (operands[3]) == AND)"
-{
-  if (TARGET_VSX && vsx_register_operand (operands[0], <MODE>mode))
-    return "xxl%q3 %x0,%x1,%x2";
+  [(set_attr "type" "fp")])
 
-  if (TARGET_ALTIVEC && altivec_register_operand (operands[0], <MODE>mode))
-    return "v%q3 %0,%1,%2";
+(define_insn "trunctfdf2_internal2"
+  [(set (match_operand:DF 0 "gpc_reg_operand" "=d")
+       (float_truncate:DF (match_operand:TF 1 "gpc_reg_operand" "d")))]
+  "!TARGET_IEEEQUAD && TARGET_XL_COMPAT
+   && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT 
+   && TARGET_LONG_DOUBLE_128"
+  "fadd %0,%1,%L1"
+  [(set_attr "type" "fp")
+   (set_attr "fp_type" "fp_addsub_d")])
 
-  return "#";
-}
-  "(TARGET_P8_VECTOR || (GET_CODE (operands[3]) == AND))
-   && reload_completed && int_reg_operand (operands[0], <MODE>mode)"
-  [(const_int 0)]
+(define_expand "trunctfsf2"
+  [(set (match_operand:SF 0 "gpc_reg_operand" "")
+       (float_truncate:SF (match_operand:TF 1 "gpc_reg_operand" "")))]
+  "!TARGET_IEEEQUAD
+   && TARGET_HARD_FLOAT
+   && (TARGET_FPRS || TARGET_E500_DOUBLE)
+   && TARGET_LONG_DOUBLE_128"
 {
-  rs6000_split_logical (operands, GET_CODE (operands[3]), false, true, false,
-                       NULL_RTX);
+  if (TARGET_E500_DOUBLE)
+    emit_insn (gen_spe_trunctfsf2 (operands[0], operands[1]));
+  else
+    emit_insn (gen_trunctfsf2_fprs (operands[0], operands[1]));
   DONE;
-}
-  [(set (attr "type")
-      (if_then_else
-       (match_test "vsx_register_operand (operands[0], <MODE>mode)")
-       (const_string "vecsimple")
-       (const_string "integer")))
-   (set (attr "length")
-      (if_then_else
-       (match_test "vsx_register_operand (operands[0], <MODE>mode)")
-       (const_string "4")
-       (if_then_else
-        (match_test "TARGET_POWERPC64")
-        (const_string "8")
-        (const_string "16"))))])
+})
 
-(define_insn_and_split "*boolc<mode>3_internal2"
-  [(set (match_operand:TI2 0 "int_reg_operand" "=&r,r,r")
-       (match_operator:TI2 3 "boolean_operator"
-        [(not:TI2
-          (match_operand:TI2 1 "int_reg_operand" "r,0,r"))
-         (match_operand:TI2 2 "int_reg_operand" "r,r,0")]))]
-  "!TARGET_P8_VECTOR && (GET_CODE (operands[3]) != AND)"
+(define_insn_and_split "trunctfsf2_fprs"
+  [(set (match_operand:SF 0 "gpc_reg_operand" "=f")
+       (float_truncate:SF (match_operand:TF 1 "gpc_reg_operand" "d")))
+   (clobber (match_scratch:DF 2 "=d"))]
+  "!TARGET_IEEEQUAD
+   && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_SINGLE_FLOAT 
+   && TARGET_LONG_DOUBLE_128"
   "#"
-  "reload_completed && !TARGET_P8_VECTOR && (GET_CODE (operands[3]) != AND)"
-  [(const_int 0)]
+  "&& reload_completed"
+  [(set (match_dup 2)
+       (float_truncate:DF (match_dup 1)))
+   (set (match_dup 0)
+       (float_truncate:SF (match_dup 2)))]
+  "")
+
+(define_expand "floatsitf2"
+  [(set (match_operand:TF 0 "gpc_reg_operand" "")
+        (float:TF (match_operand:SI 1 "gpc_reg_operand" "")))]
+  "!TARGET_IEEEQUAD
+   && TARGET_HARD_FLOAT
+   && (TARGET_FPRS || TARGET_E500_DOUBLE)
+   && TARGET_LONG_DOUBLE_128"
 {
-  rs6000_split_logical (operands, GET_CODE (operands[3]), false, true, false,
-                       NULL_RTX);
+  rtx tmp = gen_reg_rtx (DFmode);
+  expand_float (tmp, operands[1], false);
+  emit_insn (gen_extenddftf2 (operands[0], tmp));
   DONE;
-}
-  [(set_attr "type" "integer")
-   (set (attr "length")
-       (if_then_else
-        (match_test "TARGET_POWERPC64")
-        (const_string "8")
-        (const_string "16")))])
-
-;; 128-bit NAND/NOR
-(define_insn_and_split "*boolcc<mode>3_internal1"
-  [(set (match_operand:BOOL_128 0 "vlogical_operand" "=<BOOL_REGS_OUTPUT>")
-       (match_operator:BOOL_128 3 "boolean_operator"
-        [(not:BOOL_128
-          (match_operand:BOOL_128 1 "vlogical_operand" "<BOOL_REGS_OP1>"))
-         (not:BOOL_128
-          (match_operand:BOOL_128 2 "vlogical_operand" "<BOOL_REGS_OP2>"))]))]
-  "TARGET_P8_VECTOR || (GET_CODE (operands[3]) == AND)"
-{
-  if (TARGET_VSX && vsx_register_operand (operands[0], <MODE>mode))
-    return "xxl%q3 %x0,%x1,%x2";
+})
 
-  if (TARGET_ALTIVEC && altivec_register_operand (operands[0], <MODE>mode))
-    return "v%q3 %0,%1,%2";
+; fadd, but rounding towards zero.
+; This is probably not the optimal code sequence.
+(define_insn "fix_trunc_helper"
+  [(set (match_operand:DF 0 "gpc_reg_operand" "=d")
+       (unspec:DF [(match_operand:TF 1 "gpc_reg_operand" "d")]
+                  UNSPEC_FIX_TRUNC_TF))
+   (clobber (match_operand:DF 2 "gpc_reg_operand" "=&d"))]
+  "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT"
+  "mffs %2\n\tmtfsb1 31\n\tmtfsb0 30\n\tfadd %0,%1,%L1\n\tmtfsf 1,%2"
+  [(set_attr "type" "fp")
+   (set_attr "length" "20")])
 
-  return "#";
-}
-  "(TARGET_P8_VECTOR || (GET_CODE (operands[3]) == AND))
-   && reload_completed && int_reg_operand (operands[0], <MODE>mode)"
-  [(const_int 0)]
+(define_expand "fix_trunctfsi2"
+  [(set (match_operand:SI 0 "gpc_reg_operand" "")
+       (fix:SI (match_operand:TF 1 "gpc_reg_operand" "")))]
+  "!TARGET_IEEEQUAD && TARGET_HARD_FLOAT
+   && (TARGET_FPRS || TARGET_E500_DOUBLE) && TARGET_LONG_DOUBLE_128"
 {
-  rs6000_split_logical (operands, GET_CODE (operands[3]), false, true, true,
-                       NULL_RTX);
+  if (TARGET_E500_DOUBLE)
+    emit_insn (gen_spe_fix_trunctfsi2 (operands[0], operands[1]));
+  else
+    emit_insn (gen_fix_trunctfsi2_fprs (operands[0], operands[1]));
   DONE;
-}
-  [(set (attr "type")
-      (if_then_else
-       (match_test "vsx_register_operand (operands[0], <MODE>mode)")
-       (const_string "vecsimple")
-       (const_string "integer")))
-   (set (attr "length")
-      (if_then_else
-       (match_test "vsx_register_operand (operands[0], <MODE>mode)")
-       (const_string "4")
-       (if_then_else
-        (match_test "TARGET_POWERPC64")
-        (const_string "8")
-        (const_string "16"))))])
+})
 
-(define_insn_and_split "*boolcc<mode>3_internal2"
-  [(set (match_operand:TI2 0 "int_reg_operand" "=&r,r,r")
-       (match_operator:TI2 3 "boolean_operator"
-        [(not:TI2
-          (match_operand:TI2 1 "int_reg_operand" "r,0,r"))
-         (not:TI2
-          (match_operand:TI2 2 "int_reg_operand" "r,r,0"))]))]
-  "!TARGET_P8_VECTOR && (GET_CODE (operands[3]) != AND)"
+(define_expand "fix_trunctfsi2_fprs"
+  [(parallel [(set (match_operand:SI 0 "gpc_reg_operand" "")
+                  (fix:SI (match_operand:TF 1 "gpc_reg_operand" "")))
+             (clobber (match_dup 2))
+             (clobber (match_dup 3))
+             (clobber (match_dup 4))
+             (clobber (match_dup 5))])]
+  "!TARGET_IEEEQUAD
+   && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_LONG_DOUBLE_128"
+{
+  operands[2] = gen_reg_rtx (DFmode);
+  operands[3] = gen_reg_rtx (DFmode);
+  operands[4] = gen_reg_rtx (DImode);
+  operands[5] = assign_stack_temp (DImode, GET_MODE_SIZE (DImode));
+})
+
+(define_insn_and_split "*fix_trunctfsi2_internal"
+  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
+        (fix:SI (match_operand:TF 1 "gpc_reg_operand" "d")))
+   (clobber (match_operand:DF 2 "gpc_reg_operand" "=d"))
+   (clobber (match_operand:DF 3 "gpc_reg_operand" "=&d"))
+   (clobber (match_operand:DI 4 "gpc_reg_operand" "=d"))
+   (clobber (match_operand:DI 5 "offsettable_mem_operand" "=o"))]
+  "!TARGET_IEEEQUAD
+   && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_LONG_DOUBLE_128"
   "#"
-  "reload_completed && !TARGET_P8_VECTOR && (GET_CODE (operands[3]) != AND)"
-  [(const_int 0)]
+  ""
+  [(pc)]
 {
-  rs6000_split_logical (operands, GET_CODE (operands[3]), false, true, true,
-                       NULL_RTX);
+  rtx lowword;
+  emit_insn (gen_fix_trunc_helper (operands[2], operands[1], operands[3]));
+
+  gcc_assert (MEM_P (operands[5]));
+  lowword = adjust_address (operands[5], SImode, WORDS_BIG_ENDIAN ? 4 : 0);
+
+  emit_insn (gen_fctiwz_df (operands[4], operands[2]));
+  emit_move_insn (operands[5], operands[4]);
+  emit_move_insn (operands[0], lowword);
   DONE;
-}
-  [(set_attr "type" "integer")
-   (set (attr "length")
-       (if_then_else
-        (match_test "TARGET_POWERPC64")
-        (const_string "8")
-        (const_string "16")))])
+})
 
+(define_expand "negtf2"
+  [(set (match_operand:TF 0 "gpc_reg_operand" "")
+       (neg:TF (match_operand:TF 1 "gpc_reg_operand" "")))]
+  "!TARGET_IEEEQUAD
+   && TARGET_HARD_FLOAT
+   && (TARGET_FPRS || TARGET_E500_DOUBLE)
+   && TARGET_LONG_DOUBLE_128"
+  "")
 
-;; 128-bit EQV
-(define_insn_and_split "*eqv<mode>3_internal1"
-  [(set (match_operand:BOOL_128 0 "vlogical_operand" "=<BOOL_REGS_OUTPUT>")
-       (not:BOOL_128
-        (xor:BOOL_128
-         (match_operand:BOOL_128 1 "vlogical_operand" "<BOOL_REGS_OP1>")
-         (match_operand:BOOL_128 2 "vlogical_operand" "<BOOL_REGS_OP2>"))))]
-  "TARGET_P8_VECTOR"
+(define_insn "negtf2_internal"
+  [(set (match_operand:TF 0 "gpc_reg_operand" "=d")
+       (neg:TF (match_operand:TF 1 "gpc_reg_operand" "d")))]
+  "!TARGET_IEEEQUAD
+   && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_LONG_DOUBLE_128"
+  "*
 {
-  if (vsx_register_operand (operands[0], <MODE>mode))
-    return "xxleqv %x0,%x1,%x2";
+  if (REGNO (operands[0]) == REGNO (operands[1]) + 1)
+    return \"fneg %L0,%L1\;fneg %0,%1\";
+  else
+    return \"fneg %0,%1\;fneg %L0,%L1\";
+}"
+  [(set_attr "type" "fp")
+   (set_attr "length" "8")])
 
-  return "#";
-}
-  "TARGET_P8_VECTOR && reload_completed
-   && int_reg_operand (operands[0], <MODE>mode)"
-  [(const_int 0)]
+(define_expand "abstf2"
+  [(set (match_operand:TF 0 "gpc_reg_operand" "")
+       (abs:TF (match_operand:TF 1 "gpc_reg_operand" "")))]
+  "!TARGET_IEEEQUAD
+   && TARGET_HARD_FLOAT
+   && (TARGET_FPRS || TARGET_E500_DOUBLE)
+   && TARGET_LONG_DOUBLE_128"
+  "
 {
-  rs6000_split_logical (operands, XOR, true, false, false, NULL_RTX);
+  rtx label = gen_label_rtx ();
+  if (TARGET_E500_DOUBLE)
+    {
+      if (flag_finite_math_only && !flag_trapping_math)
+       emit_insn (gen_spe_abstf2_tst (operands[0], operands[1], label));
+      else
+       emit_insn (gen_spe_abstf2_cmp (operands[0], operands[1], label));
+    }
+  else
+    emit_insn (gen_abstf2_internal (operands[0], operands[1], label));
+  emit_label (label);
   DONE;
-}
-  [(set (attr "type")
-      (if_then_else
-       (match_test "vsx_register_operand (operands[0], <MODE>mode)")
-       (const_string "vecsimple")
-       (const_string "integer")))
-   (set (attr "length")
-      (if_then_else
-       (match_test "vsx_register_operand (operands[0], <MODE>mode)")
-       (const_string "4")
-       (if_then_else
-        (match_test "TARGET_POWERPC64")
-        (const_string "8")
-        (const_string "16"))))])
+}")
 
-(define_insn_and_split "*eqv<mode>3_internal2"
-  [(set (match_operand:TI2 0 "int_reg_operand" "=&r,r,r")
-       (not:TI2
-        (xor:TI2
-         (match_operand:TI2 1 "int_reg_operand" "r,0,r")
-         (match_operand:TI2 2 "int_reg_operand" "r,r,0"))))]
-  "!TARGET_P8_VECTOR"
-  "#"
-  "reload_completed && !TARGET_P8_VECTOR"
-  [(const_int 0)]
+(define_expand "abstf2_internal"
+  [(set (match_operand:TF 0 "gpc_reg_operand" "")
+       (match_operand:TF 1 "gpc_reg_operand" ""))
+   (set (match_dup 3) (match_dup 5))
+   (set (match_dup 5) (abs:DF (match_dup 5)))
+   (set (match_dup 4) (compare:CCFP (match_dup 3) (match_dup 5)))
+   (set (pc) (if_then_else (eq (match_dup 4) (const_int 0))
+                          (label_ref (match_operand 2 "" ""))
+                          (pc)))
+   (set (match_dup 6) (neg:DF (match_dup 6)))]
+  "!TARGET_IEEEQUAD
+   && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT 
+   && TARGET_LONG_DOUBLE_128"
+  "
 {
-  rs6000_split_logical (operands, XOR, true, false, false, NULL_RTX);
-  DONE;
-}
-  [(set_attr "type" "integer")
-   (set (attr "length")
-       (if_then_else
-        (match_test "TARGET_POWERPC64")
-        (const_string "8")
-        (const_string "16")))])
+  const int hi_word = LONG_DOUBLE_LARGE_FIRST ? 0 : GET_MODE_SIZE (DFmode);
+  const int lo_word = LONG_DOUBLE_LARGE_FIRST ? GET_MODE_SIZE (DFmode) : 0;
+  operands[3] = gen_reg_rtx (DFmode);
+  operands[4] = gen_reg_rtx (CCFPmode);
+  operands[5] = simplify_gen_subreg (DFmode, operands[0], TFmode, hi_word);
+  operands[6] = simplify_gen_subreg (DFmode, operands[0], TFmode, lo_word);
+}")
+\f
+;; Reload helper functions used by rs6000_secondary_reload.  The patterns all
+;; must have 3 arguments, and scratch register constraint must be a single
+;; constraint.
 
-;; 128-bit one's complement
-(define_insn_and_split "*one_cmpl<mode>3_internal"
-  [(set (match_operand:BOOL_128 0 "vlogical_operand" "=<BOOL_REGS_OUTPUT>")
-       (not:BOOL_128
-         (match_operand:BOOL_128 1 "vlogical_operand" "<BOOL_REGS_UNARY>")))]
+;; Reload patterns to support gpr load/store with misaligned mem.
+;; and multiple gpr load/store at offset >= 0xfffc
+(define_expand "reload_<mode>_store"
+  [(parallel [(match_operand 0 "memory_operand" "=m")
+              (match_operand 1 "gpc_reg_operand" "r")
+              (match_operand:GPR 2 "register_operand" "=&b")])]
   ""
 {
-  if (TARGET_VSX && vsx_register_operand (operands[0], <MODE>mode))
-    return "xxlnor %x0,%x1,%x1";
+  rs6000_secondary_reload_gpr (operands[1], operands[0], operands[2], true);
+  DONE;
+})
 
-  if (TARGET_ALTIVEC && altivec_register_operand (operands[0], <MODE>mode))
-    return "vnor %0,%1,%1";
+(define_expand "reload_<mode>_load"
+  [(parallel [(match_operand 0 "gpc_reg_operand" "=r")
+              (match_operand 1 "memory_operand" "m")
+              (match_operand:GPR 2 "register_operand" "=b")])]
+  ""
+{
+  rs6000_secondary_reload_gpr (operands[0], operands[1], operands[2], false);
+  DONE;
+})
 
-  return "#";
-}
-  "reload_completed && int_reg_operand (operands[0], <MODE>mode)"
-  [(const_int 0)]
+\f
+;; Reload patterns for various types using the vector registers.  We may need
+;; an additional base register to convert the reg+offset addressing to reg+reg
+;; for vector registers and reg+reg or (reg+reg)&(-16) addressing to just an
+;; index register for gpr registers.
+(define_expand "reload_<RELOAD:mode>_<P:mptrsize>_store"
+  [(parallel [(match_operand:RELOAD 0 "memory_operand" "m")
+              (match_operand:RELOAD 1 "gpc_reg_operand" "wa")
+              (match_operand:P 2 "register_operand" "=b")])]
+  "<P:tptrsize>"
+{
+  rs6000_secondary_reload_inner (operands[1], operands[0], operands[2], true);
+  DONE;
+})
+
+(define_expand "reload_<RELOAD:mode>_<P:mptrsize>_load"
+  [(parallel [(match_operand:RELOAD 0 "gpc_reg_operand" "wa")
+              (match_operand:RELOAD 1 "memory_operand" "m")
+              (match_operand:P 2 "register_operand" "=b")])]
+  "<P:tptrsize>"
 {
-  rs6000_split_logical (operands, NOT, false, false, false, NULL_RTX);
+  rs6000_secondary_reload_inner (operands[0], operands[1], operands[2], false);
   DONE;
-}
-  [(set (attr "type")
-      (if_then_else
-       (match_test "vsx_register_operand (operands[0], <MODE>mode)")
-       (const_string "vecsimple")
-       (const_string "integer")))
-   (set (attr "length")
-      (if_then_else
-       (match_test "vsx_register_operand (operands[0], <MODE>mode)")
-       (const_string "4")
-       (if_then_else
-        (match_test "TARGET_POWERPC64")
-        (const_string "8")
-        (const_string "16"))))])
+})
+
 
+;; Reload sometimes tries to move the address to a GPR, and can generate
+;; invalid RTL for addresses involving AND -16.  Allow addresses involving
+;; reg+reg, reg+small constant, or just reg, all wrapped in an AND -16.
+
+(define_insn_and_split "*vec_reload_and_plus_<mptrsize>"
+  [(set (match_operand:P 0 "gpc_reg_operand" "=b")
+       (and:P (plus:P (match_operand:P 1 "gpc_reg_operand" "r")
+                      (match_operand:P 2 "reg_or_cint_operand" "rI"))
+              (const_int -16)))]
+  "TARGET_ALTIVEC && (reload_in_progress || reload_completed)"
+  "#"
+  "&& reload_completed"
+  [(set (match_dup 0)
+       (plus:P (match_dup 1)
+               (match_dup 2)))
+   (set (match_dup 0)
+       (and:P (match_dup 0)
+              (const_int -16)))])
 \f
-;; Now define ways of moving data around.
+;; Power8 merge instructions to allow direct move to/from floating point
+;; registers in 32-bit mode.  We use TF mode to get two registers to move the
+;; individual 32-bit parts across.  Subreg doesn't work too well on the TF
+;; value, since it is allocated in reload and not all of the flow information
+;; is setup for it.  We have two patterns to do the two moves between gprs and
+;; fprs.  There isn't a dependancy between the two, but we could potentially
+;; schedule other instructions between the two instructions.  TFmode is
+;; currently limited to traditional FPR registers.  If/when this is changed, we
+;; will need to revist %L to make sure it works with VSX registers, or add an
+;; %x version of %L.
 
-;; Set up a register with a value from the GOT table
+(define_insn "p8_fmrgow_<mode>"
+  [(set (match_operand:FMOVE64X 0 "register_operand" "=d")
+       (unspec:FMOVE64X [(match_operand:TF 1 "register_operand" "d")]
+                        UNSPEC_P8V_FMRGOW))]
+  "!TARGET_POWERPC64 && TARGET_DIRECT_MOVE"
+  "fmrgow %0,%1,%L1"
+  [(set_attr "type" "vecperm")])
 
-(define_expand "movsi_got"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "")
-       (unspec:SI [(match_operand:SI 1 "got_operand" "")
-                   (match_dup 2)] UNSPEC_MOVSI_GOT))]
-  "DEFAULT_ABI == ABI_V4 && flag_pic == 1"
-  "
-{
-  if (GET_CODE (operands[1]) == CONST)
-    {
-      rtx offset = const0_rtx;
-      HOST_WIDE_INT value;
+(define_insn "p8_mtvsrwz_1"
+  [(set (match_operand:TF 0 "register_operand" "=d")
+       (unspec:TF [(match_operand:SI 1 "register_operand" "r")]
+                  UNSPEC_P8V_MTVSRWZ))]
+  "!TARGET_POWERPC64 && TARGET_DIRECT_MOVE"
+  "mtvsrwz %x0,%1"
+  [(set_attr "type" "mftgpr")])
 
-      operands[1] = eliminate_constant_term (XEXP (operands[1], 0), &offset);
-      value = INTVAL (offset);
-      if (value != 0)
-       {
-         rtx tmp = (!can_create_pseudo_p ()
-                    ? operands[0]
-                    : gen_reg_rtx (Pmode));
-         emit_insn (gen_movsi_got (tmp, operands[1]));
-         emit_insn (gen_addsi3 (operands[0], tmp, offset));
-         DONE;
-       }
-    }
+(define_insn "p8_mtvsrwz_2"
+  [(set (match_operand:TF 0 "register_operand" "+d")
+       (unspec:TF [(match_dup 0)
+                   (match_operand:SI 1 "register_operand" "r")]
+                  UNSPEC_P8V_MTVSRWZ))]
+  "!TARGET_POWERPC64 && TARGET_DIRECT_MOVE"
+  "mtvsrwz %L0,%1"
+  [(set_attr "type" "mftgpr")])
 
-  operands[2] = rs6000_got_register (operands[1]);
-}")
+(define_insn_and_split "reload_fpr_from_gpr<mode>"
+  [(set (match_operand:FMOVE64X 0 "register_operand" "=ws")
+       (unspec:FMOVE64X [(match_operand:FMOVE64X 1 "register_operand" "r")]
+                        UNSPEC_P8V_RELOAD_FROM_GPR))
+   (clobber (match_operand:TF 2 "register_operand" "=d"))]
+  "!TARGET_POWERPC64 && TARGET_DIRECT_MOVE"
+  "#"
+  "&& reload_completed"
+  [(const_int 0)]
+{
+  rtx dest = operands[0];
+  rtx src = operands[1];
+  rtx tmp = operands[2];
+  rtx gpr_hi_reg = gen_highpart (SImode, src);
+  rtx gpr_lo_reg = gen_lowpart (SImode, src);
 
-(define_insn "*movsi_got_internal"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-       (unspec:SI [(match_operand:SI 1 "got_no_const_operand" "")
-                   (match_operand:SI 2 "gpc_reg_operand" "b")]
-                  UNSPEC_MOVSI_GOT))]
-  "DEFAULT_ABI == ABI_V4 && flag_pic == 1"
-  "lwz %0,%a1@got(%2)"
-  [(set_attr "type" "load")])
+  emit_insn (gen_p8_mtvsrwz_1 (tmp, gpr_hi_reg));
+  emit_insn (gen_p8_mtvsrwz_2 (tmp, gpr_lo_reg));
+  emit_insn (gen_p8_fmrgow_<mode> (dest, tmp));
+  DONE;
+}
+  [(set_attr "length" "12")
+   (set_attr "type" "three")])
 
-;; Used by sched, shorten_branches and final when the GOT pseudo reg
-;; didn't get allocated to a hard register.
-(define_split
-  [(set (match_operand:SI 0 "gpc_reg_operand" "")
-       (unspec:SI [(match_operand:SI 1 "got_no_const_operand" "")
-                   (match_operand:SI 2 "memory_operand" "")]
-                  UNSPEC_MOVSI_GOT))]
-  "DEFAULT_ABI == ABI_V4
-    && flag_pic == 1
-    && (reload_in_progress || reload_completed)"
-  [(set (match_dup 0) (match_dup 2))
-   (set (match_dup 0) (unspec:SI [(match_dup 1)(match_dup 0)]
-                                UNSPEC_MOVSI_GOT))]
-  "")
+;; Move 128 bit values from GPRs to VSX registers in 64-bit mode
+(define_insn "p8_mtvsrd_1"
+  [(set (match_operand:TF 0 "register_operand" "=ws")
+       (unspec:TF [(match_operand:DI 1 "register_operand" "r")]
+                  UNSPEC_P8V_MTVSRD))]
+  "TARGET_POWERPC64 && TARGET_DIRECT_MOVE"
+  "mtvsrd %0,%1"
+  [(set_attr "type" "mftgpr")])
 
-;; For SI, we special-case integers that can't be loaded in one insn.  We
-;; do the load 16-bits at a time.  We could do this by loading from memory,
-;; and this is even supposed to be faster, but it is simpler not to get
-;; integers in the TOC.
-(define_insn "movsi_low"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-        (mem:SI (lo_sum:SI (match_operand:SI 1 "gpc_reg_operand" "b")
-                           (match_operand 2 "" ""))))]
-  "TARGET_MACHO && ! TARGET_64BIT"
-  "lwz %0,lo16(%2)(%1)"
-  [(set_attr "type" "load")
-   (set_attr "length" "4")])
+(define_insn "p8_mtvsrd_2"
+  [(set (match_operand:TF 0 "register_operand" "+ws")
+       (unspec:TF [(match_dup 0)
+                   (match_operand:DI 1 "register_operand" "r")]
+                  UNSPEC_P8V_MTVSRD))]
+  "TARGET_POWERPC64 && TARGET_DIRECT_MOVE"
+  "mtvsrd %L0,%1"
+  [(set_attr "type" "mftgpr")])
 
-(define_insn "*movsi_internal1"
-  [(set (match_operand:SI 0 "rs6000_nonimmediate_operand" "=r,r,r,m,r,r,r,r,*c*l,*h,*h")
-       (match_operand:SI 1 "input_operand" "r,U,m,r,I,L,n,*h,r,r,0"))]
-  "!TARGET_SINGLE_FPU &&
-   (gpc_reg_operand (operands[0], SImode) || gpc_reg_operand (operands[1], SImode))"
-  "@
-   mr %0,%1
-   la %0,%a1
-   lwz%U1%X1 %0,%1
-   stw%U0%X0 %1,%0
-   li %0,%1
-   lis %0,%v1
-   #
-   mf%1 %0
-   mt%0 %1
-   mt%0 %1
-   nop"
-  [(set_attr "type" "*,*,load,store,*,*,*,mfjmpr,mtjmpr,*,*")
-   (set_attr "length" "4,4,4,4,4,4,8,4,4,4,4")])
+(define_insn "p8_xxpermdi_<mode>"
+  [(set (match_operand:FMOVE128_GPR 0 "register_operand" "=wa")
+       (unspec:FMOVE128_GPR [(match_operand:TF 1 "register_operand" "ws")]
+                            UNSPEC_P8V_XXPERMDI))]
+  "TARGET_POWERPC64 && TARGET_DIRECT_MOVE"
+  "xxpermdi %x0,%1,%L1,0"
+  [(set_attr "type" "vecperm")])
 
-(define_insn "*movsi_internal1_single"
-  [(set (match_operand:SI 0 "rs6000_nonimmediate_operand" "=r,r,r,m,r,r,r,r,*c*l,*h,*h,m,*f")
-        (match_operand:SI 1 "input_operand" "r,U,m,r,I,L,n,*h,r,r,0,f,m"))]
-  "TARGET_SINGLE_FPU &&
-   (gpc_reg_operand (operands[0], SImode) || gpc_reg_operand (operands[1], SImode))"
-  "@
-   mr %0,%1
-   la %0,%a1
-   lwz%U1%X1 %0,%1
-   stw%U0%X0 %1,%0
-   li %0,%1
-   lis %0,%v1
-   #
-   mf%1 %0
-   mt%0 %1
-   mt%0 %1
-   nop
-   stfs%U0%X0 %1,%0
-   lfs%U1%X1 %0,%1"
-  [(set_attr "type" "*,*,load,store,*,*,*,mfjmpr,mtjmpr,*,*,fpstore,fpload")
-   (set_attr "length" "4,4,4,4,4,4,8,4,4,4,4,4,4")])
+(define_insn_and_split "reload_vsx_from_gpr<mode>"
+  [(set (match_operand:FMOVE128_GPR 0 "register_operand" "=wa")
+       (unspec:FMOVE128_GPR
+        [(match_operand:FMOVE128_GPR 1 "register_operand" "r")]
+        UNSPEC_P8V_RELOAD_FROM_GPR))
+   (clobber (match_operand:TF 2 "register_operand" "=ws"))]
+  "TARGET_POWERPC64 && TARGET_DIRECT_MOVE"
+  "#"
+  "&& reload_completed"
+  [(const_int 0)]
+{
+  rtx dest = operands[0];
+  rtx src = operands[1];
+  rtx tmp = operands[2];
+  rtx gpr_hi_reg = gen_highpart (DImode, src);
+  rtx gpr_lo_reg = gen_lowpart (DImode, src);
 
-;; Split a load of a large constant into the appropriate two-insn
-;; sequence.
+  emit_insn (gen_p8_mtvsrd_1 (tmp, gpr_hi_reg));
+  emit_insn (gen_p8_mtvsrd_2 (tmp, gpr_lo_reg));
+  emit_insn (gen_p8_xxpermdi_<mode> (dest, tmp));
+  DONE;
+}
+  [(set_attr "length" "12")
+   (set_attr "type" "three")])
 
 (define_split
-  [(set (match_operand:SI 0 "gpc_reg_operand" "")
-       (match_operand:SI 1 "const_int_operand" ""))]
-  "(unsigned HOST_WIDE_INT) (INTVAL (operands[1]) + 0x8000) >= 0x10000
-   && (INTVAL (operands[1]) & 0xffff) != 0"
-  [(set (match_dup 0)
-       (match_dup 2))
-   (set (match_dup 0)
-       (ior:SI (match_dup 0)
-               (match_dup 3)))]
-  "
-{ rtx tem = rs6000_emit_set_const (operands[0], SImode, operands[1], 2);
-
-  if (tem == operands[0])
-    DONE;
-  else
-    FAIL;
-}")
+  [(set (match_operand:FMOVE128_GPR 0 "nonimmediate_operand" "")
+       (match_operand:FMOVE128_GPR 1 "input_operand" ""))]
+  "reload_completed
+   && (int_reg_operand (operands[0], <MODE>mode)
+       || int_reg_operand (operands[1], <MODE>mode))"
+  [(pc)]
+{ rs6000_split_multireg_move (operands[0], operands[1]); DONE; })
 
-(define_insn "*mov<mode>_internal2"
-  [(set (match_operand:CC 2 "cc_reg_operand" "=y,x,?y")
-       (compare:CC (match_operand:P 1 "gpc_reg_operand" "0,r,r")
-                   (const_int 0)))
-   (set (match_operand:P 0 "gpc_reg_operand" "=r,r,r") (match_dup 1))]
-  ""
-  "@
-   cmp<wd>i %2,%0,0
-   mr. %0,%1
-   #"
-  [(set_attr "type" "cmp,logical,cmp")
-   (set_attr "dot" "yes")
-   (set_attr "length" "4,4,8")])
+;; Move SFmode to a VSX from a GPR register.  Because scalar floating point
+;; type is stored internally as double precision in the VSX registers, we have
+;; to convert it from the vector format.
 
-(define_split
-  [(set (match_operand:CC 2 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (match_operand:P 1 "gpc_reg_operand" "")
-                   (const_int 0)))
-   (set (match_operand:P 0 "gpc_reg_operand" "") (match_dup 1))]
-  "reload_completed"
-  [(set (match_dup 0) (match_dup 1))
-   (set (match_dup 2)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
-\f
-(define_insn "*movhi_internal"
-  [(set (match_operand:HI 0 "nonimmediate_operand" "=r,r,m,r,r,*c*l,*h")
-       (match_operand:HI 1 "input_operand" "r,m,r,i,*h,r,0"))]
-  "gpc_reg_operand (operands[0], HImode)
-   || gpc_reg_operand (operands[1], HImode)"
-  "@
-   mr %0,%1
-   lhz%U1%X1 %0,%1
-   sth%U0%X0 %1,%0
-   li %0,%w1
-   mf%1 %0
-   mt%0 %1
-   nop"
-  [(set_attr "type" "*,load,store,*,mfjmpr,mtjmpr,*")])
+(define_insn_and_split "reload_vsx_from_gprsf"
+  [(set (match_operand:SF 0 "register_operand" "=wa")
+       (unspec:SF [(match_operand:SF 1 "register_operand" "r")]
+                  UNSPEC_P8V_RELOAD_FROM_GPR))
+   (clobber (match_operand:DI 2 "register_operand" "=r"))]
+  "TARGET_POWERPC64 && TARGET_DIRECT_MOVE"
+  "#"
+  "&& reload_completed"
+  [(const_int 0)]
+{
+  rtx op0 = operands[0];
+  rtx op1 = operands[1];
+  rtx op2 = operands[2];
+  /* Also use the destination register to hold the unconverted DImode value.
+     This is conceptually a separate value from OP0, so we use gen_rtx_REG
+     rather than simplify_gen_subreg.  */
+  rtx op0_di = gen_rtx_REG (DImode, REGNO (op0));
+  rtx op1_di = simplify_gen_subreg (DImode, op1, SFmode, 0);
 
-(define_expand "mov<mode>"
-  [(set (match_operand:INT 0 "general_operand" "")
-       (match_operand:INT 1 "any_operand" ""))]
-  ""
-  "{ rs6000_emit_move (operands[0], operands[1], <MODE>mode); DONE; }")
+  /* Move SF value to upper 32-bits for xscvspdpn.  */
+  emit_insn (gen_ashldi3 (op2, op1_di, GEN_INT (32)));
+  emit_move_insn (op0_di, op2);
+  emit_insn (gen_vsx_xscvspdpn_directmove (op0, op0_di));
+  DONE;
+}
+  [(set_attr "length" "8")
+   (set_attr "type" "two")])
 
-(define_insn "*movqi_internal"
-  [(set (match_operand:QI 0 "nonimmediate_operand" "=r,r,m,r,r,*c*l,*h")
-       (match_operand:QI 1 "input_operand" "r,m,r,i,*h,r,0"))]
-  "gpc_reg_operand (operands[0], QImode)
-   || gpc_reg_operand (operands[1], QImode)"
-  "@
-   mr %0,%1
-   lbz%U1%X1 %0,%1
-   stb%U0%X0 %1,%0
-   li %0,%1
-   mf%1 %0
-   mt%0 %1
-   nop"
-  [(set_attr "type" "*,load,store,*,mfjmpr,mtjmpr,*")])
-\f
-;; Here is how to move condition codes around.  When we store CC data in
-;; an integer register or memory, we store just the high-order 4 bits.
-;; This lets us not shift in the most common case of CR0.
-(define_expand "movcc"
-  [(set (match_operand:CC 0 "nonimmediate_operand" "")
-       (match_operand:CC 1 "nonimmediate_operand" ""))]
-  ""
-  "")
+;; Move 128 bit values from VSX registers to GPRs in 64-bit mode by doing a
+;; normal 64-bit move, followed by an xxpermdi to get the bottom 64-bit value,
+;; and then doing a move of that.
+(define_insn "p8_mfvsrd_3_<mode>"
+  [(set (match_operand:DF 0 "register_operand" "=r")
+       (unspec:DF [(match_operand:FMOVE128_GPR 1 "register_operand" "wa")]
+                  UNSPEC_P8V_RELOAD_FROM_VSX))]
+  "TARGET_POWERPC64 && TARGET_DIRECT_MOVE"
+  "mfvsrd %0,%x1"
+  [(set_attr "type" "mftgpr")])
 
-(define_insn "*movcc_internal1"
-  [(set (match_operand:CC 0 "nonimmediate_operand" "=y,x,?y,y,r,r,r,r,r,cl,r,m")
-       (match_operand:CC 1 "general_operand" "y,r,r,O,x,y,r,I,h,r,m,r"))]
-  "register_operand (operands[0], CCmode)
-   || register_operand (operands[1], CCmode)"
-  "@
-   mcrf %0,%1
-   mtcrf 128,%1
-   rlwinm %1,%1,%F0,0xffffffff\;mtcrf %R0,%1\;rlwinm %1,%1,%f0,0xffffffff
-   crxor %0,%0,%0
-   mfcr %0%Q1
-   mfcr %0%Q1\;rlwinm %0,%0,%f1,0xf0000000
-   mr %0,%1
-   li %0,%1
-   mf%1 %0
-   mt%0 %1
-   lwz%U1%X1 %0,%1
-   stw%U0%X0 %1,%0"
-  [(set (attr "type")
-     (cond [(eq_attr "alternative" "0,3")
-               (const_string "cr_logical")
-           (eq_attr "alternative" "1,2")
-               (const_string "mtcr")
-           (eq_attr "alternative" "6,7")
-               (const_string "integer")
-           (eq_attr "alternative" "8")
-               (const_string "mfjmpr")
-           (eq_attr "alternative" "9")
-               (const_string "mtjmpr")
-           (eq_attr "alternative" "10")
-               (const_string "load")
-           (eq_attr "alternative" "11")
-               (const_string "store")
-           (match_test "TARGET_MFCRF")
-               (const_string "mfcrf")
-          ]
-       (const_string "mfcr")))
-   (set_attr "length" "4,4,12,4,4,8,4,4,4,4,4,4")])
-\f
-;; For floating-point, we normally deal with the floating-point registers
-;; unless -msoft-float is used.  The sole exception is that parameter passing
-;; can produce floating-point values in fixed-point registers.  Unless the
-;; value is a simple constant or already in memory, we deal with this by
-;; allocating memory and copying the value explicitly via that memory location.
+(define_insn_and_split "reload_gpr_from_vsx<mode>"
+  [(set (match_operand:FMOVE128_GPR 0 "register_operand" "=r")
+       (unspec:FMOVE128_GPR
+        [(match_operand:FMOVE128_GPR 1 "register_operand" "wa")]
+        UNSPEC_P8V_RELOAD_FROM_VSX))
+   (clobber (match_operand:FMOVE128_GPR 2 "register_operand" "=wa"))]
+  "TARGET_POWERPC64 && TARGET_DIRECT_MOVE"
+  "#"
+  "&& reload_completed"
+  [(const_int 0)]
+{
+  rtx dest = operands[0];
+  rtx src = operands[1];
+  rtx tmp = operands[2];
+  rtx gpr_hi_reg = gen_highpart (DFmode, dest);
+  rtx gpr_lo_reg = gen_lowpart (DFmode, dest);
 
-;; Move 32-bit binary/decimal floating point
-(define_expand "mov<mode>"
-  [(set (match_operand:FMOVE32 0 "nonimmediate_operand" "")
-       (match_operand:FMOVE32 1 "any_operand" ""))]
-  "<fmove_ok>"
-  "{ rs6000_emit_move (operands[0], operands[1], <MODE>mode); DONE; }")
+  emit_insn (gen_p8_mfvsrd_3_<mode> (gpr_hi_reg, src));
+  emit_insn (gen_vsx_xxpermdi_<mode> (tmp, src, src, GEN_INT (3)));
+  emit_insn (gen_p8_mfvsrd_3_<mode> (gpr_lo_reg, tmp));
+  DONE;
+}
+  [(set_attr "length" "12")
+   (set_attr "type" "three")])
 
-(define_split
-  [(set (match_operand:FMOVE32 0 "gpc_reg_operand" "")
-       (match_operand:FMOVE32 1 "const_double_operand" ""))]
-  "reload_completed
-   && ((GET_CODE (operands[0]) == REG && REGNO (operands[0]) <= 31)
-       || (GET_CODE (operands[0]) == SUBREG
-          && GET_CODE (SUBREG_REG (operands[0])) == REG
-          && REGNO (SUBREG_REG (operands[0])) <= 31))"
-  [(set (match_dup 2) (match_dup 3))]
-  "
+;; Move SFmode to a GPR from a VSX register.  Because scalar floating point
+;; type is stored internally as double precision, we have to convert it to the
+;; vector format.
+
+(define_insn_and_split "reload_gpr_from_vsxsf"
+  [(set (match_operand:SF 0 "register_operand" "=r")
+       (unspec:SF [(match_operand:SF 1 "register_operand" "wa")]
+                  UNSPEC_P8V_RELOAD_FROM_VSX))
+   (clobber (match_operand:V4SF 2 "register_operand" "=wa"))]
+  "TARGET_POWERPC64 && TARGET_DIRECT_MOVE"
+  "#"
+  "&& reload_completed"
+  [(const_int 0)]
 {
-  long l;
-  REAL_VALUE_TYPE rv;
+  rtx op0 = operands[0];
+  rtx op1 = operands[1];
+  rtx op2 = operands[2];
+  rtx diop0 = simplify_gen_subreg (DImode, op0, SFmode, 0);
 
-  REAL_VALUE_FROM_CONST_DOUBLE (rv, operands[1]);
-  <real_value_to_target> (rv, l);
+  emit_insn (gen_vsx_xscvdpspn_scalar (op2, op1));
+  emit_insn (gen_p8_mfvsrd_4_disf (diop0, op2));
+  emit_insn (gen_lshrdi3 (diop0, diop0, GEN_INT (32)));
+  DONE;
+}
+  [(set_attr "length" "12")
+   (set_attr "type" "three")])
 
-  if (! TARGET_POWERPC64)
-    operands[2] = operand_subword (operands[0], 0, 0, <MODE>mode);
-  else
-    operands[2] = gen_lowpart (SImode, operands[0]);
+(define_insn "p8_mfvsrd_4_disf"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+       (unspec:DI [(match_operand:V4SF 1 "register_operand" "wa")]
+                  UNSPEC_P8V_RELOAD_FROM_VSX))]
+  "TARGET_POWERPC64 && TARGET_DIRECT_MOVE"
+  "mfvsrd %0,%x1"
+  [(set_attr "type" "mftgpr")])
 
-  operands[3] = gen_int_mode (l, SImode);
-}")
+\f
+;; Next come the multi-word integer load and store and the load and store
+;; multiple insns.
 
-(define_insn "mov<mode>_hardfloat"
-  [(set (match_operand:FMOVE32 0 "nonimmediate_operand" "=!r,!r,m,f,wa,wa,<f32_lr>,<f32_sm>,wu,Z,?<f32_dm>,?r,*c*l,!r,*h,!r,!r")
-       (match_operand:FMOVE32 1 "input_operand" "r,m,r,f,wa,j,<f32_lm>,<f32_sr>,Z,wu,r,<f32_dm>,r,h,0,G,Fn"))]
-  "(gpc_reg_operand (operands[0], <MODE>mode)
-   || gpc_reg_operand (operands[1], <MODE>mode))
-   && (TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_SINGLE_FLOAT)"
+;; List r->r after r->Y, otherwise reload will try to reload a
+;; non-offsettable address by using r->r which won't make progress.
+;; Use of fprs is disparaged slightly otherwise reload prefers to reload
+;; a gpr into a fpr instead of reloading an invalid 'Y' address
+(define_insn "*movdi_internal32"
+  [(set (match_operand:DI 0 "rs6000_nonimmediate_operand" "=Y,r,r,?m,?*d,?*d,r")
+       (match_operand:DI 1 "input_operand" "r,Y,r,d,m,d,IJKnGHF"))]
+  "! TARGET_POWERPC64
+   && (gpc_reg_operand (operands[0], DImode)
+       || gpc_reg_operand (operands[1], DImode))"
   "@
-   mr %0,%1
-   lwz%U1%X1 %0,%1
-   stw%U0%X0 %1,%0
+   #
+   #
+   #
+   stfd%U0%X0 %1,%0
+   lfd%U1%X1 %0,%1
    fmr %0,%1
-   xxlor %x0,%x1,%x1
-   xxlxor %x0,%x0,%x0
-   <f32_li>
-   <f32_si>
-   <f32_lv>
-   <f32_sv>
-   mtvsrwz %x0,%1
-   mfvsrwz %0,%x1
-   mt%0 %1
-   mf%1 %0
-   nop
-   #
    #"
-  [(set_attr "type" "*,load,store,fp,vecsimple,vecsimple,fpload,fpstore,fpload,fpstore,mftgpr,mffgpr,mtjmpr,mfjmpr,*,*,*")
-   (set_attr "length" "4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,8")])
-
-(define_insn "*mov<mode>_softfloat"
-  [(set (match_operand:FMOVE32 0 "nonimmediate_operand" "=r,cl,r,r,m,r,r,r,r,*h")
-       (match_operand:FMOVE32 1 "input_operand" "r,r,h,m,r,I,L,G,Fn,0"))]
-  "(gpc_reg_operand (operands[0], <MODE>mode)
-   || gpc_reg_operand (operands[1], <MODE>mode))
-   && (TARGET_SOFT_FLOAT || !TARGET_FPRS)"
-  "@
-   mr %0,%1
-   mt%0 %1
-   mf%1 %0
-   lwz%U1%X1 %0,%1
-   stw%U0%X0 %1,%0
-   li %0,%1
-   lis %0,%v1
-   #
-   #
-   nop"
-  [(set_attr "type" "*,mtjmpr,mfjmpr,load,store,*,*,*,*,*")
-   (set_attr "length" "4,4,4,4,4,4,4,4,8,4")])
-
-\f
-;; Move 64-bit binary/decimal floating point
-(define_expand "mov<mode>"
-  [(set (match_operand:FMOVE64 0 "nonimmediate_operand" "")
-       (match_operand:FMOVE64 1 "any_operand" ""))]
-  ""
-  "{ rs6000_emit_move (operands[0], operands[1], <MODE>mode); DONE; }")
+  [(set_attr "type" "store,load,*,fpstore,fpload,fp,*")])
 
 (define_split
-  [(set (match_operand:FMOVE64 0 "gpc_reg_operand" "")
-       (match_operand:FMOVE64 1 "const_int_operand" ""))]
+  [(set (match_operand:DI 0 "gpc_reg_operand" "")
+       (match_operand:DI 1 "const_int_operand" ""))]
   "! TARGET_POWERPC64 && reload_completed
-   && ((GET_CODE (operands[0]) == REG && REGNO (operands[0]) <= 31)
-       || (GET_CODE (operands[0]) == SUBREG
-          && GET_CODE (SUBREG_REG (operands[0])) == REG
-          && REGNO (SUBREG_REG (operands[0])) <= 31))"
+   && gpr_or_gpr_p (operands[0], operands[1])
+   && !direct_move_p (operands[0], operands[1])"
   [(set (match_dup 2) (match_dup 4))
    (set (match_dup 3) (match_dup 1))]
   "
 {
-  int endian = (WORDS_BIG_ENDIAN == 0);
   HOST_WIDE_INT value = INTVAL (operands[1]);
-
-  operands[2] = operand_subword (operands[0], endian, 0, <MODE>mode);
-  operands[3] = operand_subword (operands[0], 1 - endian, 0, <MODE>mode);
+  operands[2] = operand_subword_force (operands[0], WORDS_BIG_ENDIAN == 0,
+                                      DImode);
+  operands[3] = operand_subword_force (operands[0], WORDS_BIG_ENDIAN != 0,
+                                      DImode);
   operands[4] = GEN_INT (value >> 32);
   operands[1] = GEN_INT (((value & 0xffffffff) ^ 0x80000000) - 0x80000000);
 }")
 
 (define_split
-  [(set (match_operand:FMOVE64 0 "gpc_reg_operand" "")
-       (match_operand:FMOVE64 1 "const_double_operand" ""))]
-  "! TARGET_POWERPC64 && reload_completed
-   && ((GET_CODE (operands[0]) == REG && REGNO (operands[0]) <= 31)
-       || (GET_CODE (operands[0]) == SUBREG
-          && GET_CODE (SUBREG_REG (operands[0])) == REG
-          && REGNO (SUBREG_REG (operands[0])) <= 31))"
-  [(set (match_dup 2) (match_dup 4))
-   (set (match_dup 3) (match_dup 5))]
-  "
-{
-  int endian = (WORDS_BIG_ENDIAN == 0);
-  long l[2];
-  REAL_VALUE_TYPE rv;
-
-  REAL_VALUE_FROM_CONST_DOUBLE (rv, operands[1]);
-  <real_value_to_target> (rv, l);
-
-  operands[2] = operand_subword (operands[0], endian, 0, <MODE>mode);
-  operands[3] = operand_subword (operands[0], 1 - endian, 0, <MODE>mode);
-  operands[4] = gen_int_mode (l[endian], SImode);
-  operands[5] = gen_int_mode (l[1 - endian], SImode);
-}")
-
-(define_split
-  [(set (match_operand:FMOVE64 0 "gpc_reg_operand" "")
-       (match_operand:FMOVE64 1 "const_double_operand" ""))]
-  "TARGET_POWERPC64 && reload_completed
-   && ((GET_CODE (operands[0]) == REG && REGNO (operands[0]) <= 31)
-       || (GET_CODE (operands[0]) == SUBREG
-          && GET_CODE (SUBREG_REG (operands[0])) == REG
-          && REGNO (SUBREG_REG (operands[0])) <= 31))"
-  [(set (match_dup 2) (match_dup 3))]
-  "
-{
-  int endian = (WORDS_BIG_ENDIAN == 0);
-  long l[2];
-  REAL_VALUE_TYPE rv;
-  HOST_WIDE_INT val;
-
-  REAL_VALUE_FROM_CONST_DOUBLE (rv, operands[1]);
-  <real_value_to_target> (rv, l);
-
-  operands[2] = gen_lowpart (DImode, operands[0]);
-  /* HIGHPART is lower memory address when WORDS_BIG_ENDIAN.  */
-  val = ((HOST_WIDE_INT)(unsigned long)l[endian] << 32
-         | ((HOST_WIDE_INT)(unsigned long)l[1 - endian]));
-
-  operands[3] = gen_int_mode (val, DImode);
-}")
-
-;; Don't have reload use general registers to load a constant.  It is
-;; less efficient than loading the constant into an FP register, since
-;; it will probably be used there.
-
-;; The move constraints are ordered to prefer floating point registers before
-;; general purpose registers to avoid doing a store and a load to get the value
-;; into a floating point register when it is needed for a floating point
-;; operation.  Prefer traditional floating point registers over VSX registers,
-;; since the D-form version of the memory instructions does not need a GPR for
-;; reloading.
+  [(set (match_operand:DIFD 0 "rs6000_nonimmediate_operand" "")
+        (match_operand:DIFD 1 "input_operand" ""))]
+  "reload_completed && !TARGET_POWERPC64
+   && gpr_or_gpr_p (operands[0], operands[1])
+   && !direct_move_p (operands[0], operands[1])"
+  [(pc)]
+{ rs6000_split_multireg_move (operands[0], operands[1]); DONE; })
 
-(define_insn "*mov<mode>_hardfloat32"
-  [(set (match_operand:FMOVE64 0 "nonimmediate_operand" "=m,d,d,wv,Z,wa,wa,Y,r,!r,!r,!r,!r")
-       (match_operand:FMOVE64 1 "input_operand" "d,m,d,Z,wv,wa,j,r,Y,r,G,H,F"))]
-  "! TARGET_POWERPC64 && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT 
-   && (gpc_reg_operand (operands[0], <MODE>mode)
-       || gpc_reg_operand (operands[1], <MODE>mode))"
+(define_insn "*movdi_internal64"
+  [(set (match_operand:DI 0 "nonimmediate_operand" "=Y,r,r,r,r,r,?m,?*d,?*d,r,*h,*h,r,?*wg,r,?*wj,?*wi")
+       (match_operand:DI 1 "input_operand" "r,Y,r,I,L,nF,d,m,d,*h,r,0,*wg,r,*wj,r,O"))]
+  "TARGET_POWERPC64
+   && (gpc_reg_operand (operands[0], DImode)
+       || gpc_reg_operand (operands[1], DImode))"
   "@
-   stfd%U0%X0 %1,%0
-   lfd%U1%X1 %0,%1
-   fmr %0,%1
-   lxsd%U1x %x0,%y1
-   stxsd%U0x %x1,%y0
-   xxlor %x0,%x1,%x1
-   xxlxor %x0,%x0,%x0
-   #
-   #
-   #
-   #
+   std%U0%X0 %1,%0
+   ld%U1%X1 %0,%1
+   mr %0,%1
+   li %0,%1
+   lis %0,%v1
    #
-   #"
-  [(set_attr "type" "fpstore,fpload,fp,fpload,fpstore,vecsimple,vecsimple,store,load,two,fp,fp,*")
-   (set_attr "length" "4,4,4,4,4,4,4,8,8,8,8,12,16")])
-
-(define_insn "*mov<mode>_softfloat32"
-  [(set (match_operand:FMOVE64 0 "nonimmediate_operand" "=Y,r,r,r,r,r")
-       (match_operand:FMOVE64 1 "input_operand" "r,Y,r,G,H,F"))]
-  "! TARGET_POWERPC64 
-   && ((TARGET_FPRS && TARGET_SINGLE_FLOAT) 
-       || TARGET_SOFT_FLOAT || TARGET_E500_SINGLE
-       || (<MODE>mode == DDmode && TARGET_E500_DOUBLE))
-   && (gpc_reg_operand (operands[0], <MODE>mode)
-       || gpc_reg_operand (operands[1], <MODE>mode))"
-  "#"
-  [(set_attr "type" "store,load,two,*,*,*")
-   (set_attr "length" "8,8,8,8,12,16")])
-
-; ld/std require word-aligned displacements -> 'Y' constraint.
-; List Y->r and r->Y before r->r for reload.
-(define_insn "*mov<mode>_hardfloat64"
-  [(set (match_operand:FMOVE64 0 "nonimmediate_operand" "=m,d,d,wv,Z,wa,wa,Y,r,!r,*c*l,!r,*h,!r,!r,!r,r,wg,r,wm")
-       (match_operand:FMOVE64 1 "input_operand" "d,m,d,Z,wv,wa,j,r,Y,r,r,h,0,G,H,F,wg,r,wm,r"))]
-  "TARGET_POWERPC64 && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT
-   && (gpc_reg_operand (operands[0], <MODE>mode)
-       || gpc_reg_operand (operands[1], <MODE>mode))"
-  "@
    stfd%U0%X0 %1,%0
    lfd%U1%X1 %0,%1
    fmr %0,%1
-   lxsd%U1x %x0,%y1
-   stxsd%U0x %x1,%y0
-   xxlor %x0,%x1,%x1
-   xxlxor %x0,%x0,%x0
-   std%U0%X0 %1,%0
-   ld%U1%X1 %0,%1
-   mr %0,%1
-   mt%0 %1
    mf%1 %0
+   mt%0 %1
    nop
-   #
-   #
-   #
    mftgpr %0,%1
    mffgpr %0,%1
    mfvsrd %0,%x1
-   mtvsrd %x0,%1"
-  [(set_attr "type" "fpstore,fpload,fp,fpload,fpstore,vecsimple,vecsimple,store,load,*,mtjmpr,mfjmpr,*,*,*,*,mftgpr,mffgpr,mftgpr,mffgpr")
-   (set_attr "length" "4,4,4,4,4,4,4,4,4,4,4,4,4,8,12,16,4,4,4,4")])
-
-(define_insn "*mov<mode>_softfloat64"
-  [(set (match_operand:FMOVE64 0 "nonimmediate_operand" "=Y,r,r,cl,r,r,r,r,*h")
-       (match_operand:FMOVE64 1 "input_operand" "r,Y,r,r,h,G,H,F,0"))]
-  "TARGET_POWERPC64 && (TARGET_SOFT_FLOAT || !TARGET_FPRS)
-   && (gpc_reg_operand (operands[0], <MODE>mode)
-       || gpc_reg_operand (operands[1], <MODE>mode))"
-  "@
-   std%U0%X0 %1,%0
-   ld%U1%X1 %0,%1
-   mr %0,%1
-   mt%0 %1
-   mf%1 %0
-   #
-   #
-   #
-   nop"
-  [(set_attr "type" "store,load,*,mtjmpr,mfjmpr,*,*,*,*")
-   (set_attr "length" "4,4,4,4,4,8,12,16,4")])
-\f
-(define_expand "mov<mode>"
-  [(set (match_operand:FMOVE128 0 "general_operand" "")
-       (match_operand:FMOVE128 1 "any_operand" ""))]
-  ""
-  "{ rs6000_emit_move (operands[0], operands[1], <MODE>mode); DONE; }")
-
-;; It's important to list Y->r and r->Y before r->r because otherwise
-;; reload, given m->r, will try to pick r->r and reload it, which
-;; doesn't make progress.
+   mtvsrd %x0,%1
+   xxlxor %x0,%x0,%x0"
+  [(set_attr "type" "store,load,*,*,*,*,fpstore,fpload,fp,mfjmpr,mtjmpr,*,mftgpr,mffgpr,mftgpr,mffgpr,vecsimple")
+   (set_attr "length" "4,4,4,4,4,20,4,4,4,4,4,4,4,4,4,4,4")])
 
-;; We can't split little endian direct moves of TDmode, because the words are
-;; not swapped like they are for TImode or TFmode.  Subregs therefore are
-;; problematical.  Don't allow direct move for this case.
+;; Generate all one-bits and clear left or right.
+;; Use (and:DI (rotate:DI ...)) to avoid anddi3 unnecessary clobber.
+(define_split
+  [(set (match_operand:DI 0 "gpc_reg_operand" "")
+       (match_operand:DI 1 "mask64_operand" ""))]
+  "TARGET_POWERPC64 && num_insns_constant (operands[1], DImode) > 1"
+  [(set (match_dup 0) (const_int -1))
+   (set (match_dup 0)
+       (and:DI (rotate:DI (match_dup 0)
+                          (const_int 0))
+               (match_dup 1)))]
+  "")
 
-(define_insn_and_split "*mov<mode>_64bit_dm"
-  [(set (match_operand:FMOVE128 0 "nonimmediate_operand" "=m,d,d,Y,r,r,r,wm")
-       (match_operand:FMOVE128 1 "input_operand" "d,m,d,r,YGHF,r,wm,r"))]
-  "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_POWERPC64
-   && (<MODE>mode != TDmode || WORDS_BIG_ENDIAN)
-   && (gpc_reg_operand (operands[0], <MODE>mode)
-       || gpc_reg_operand (operands[1], <MODE>mode))"
-  "#"
-  "&& reload_completed"
-  [(pc)]
-{ rs6000_split_multireg_move (operands[0], operands[1]); DONE; }
-  [(set_attr "length" "8,8,8,12,12,8,8,8")])
+;; Split a load of a large constant into the appropriate five-instruction
+;; sequence.  Handle anything in a constant number of insns.
+;; When non-easy constants can go in the TOC, this should use
+;; easy_fp_constant predicate.
+(define_split
+  [(set (match_operand:DI 0 "gpc_reg_operand" "")
+       (match_operand:DI 1 "const_int_operand" ""))]
+  "TARGET_POWERPC64 && num_insns_constant (operands[1], DImode) > 1"
+  [(set (match_dup 0) (match_dup 2))
+   (set (match_dup 0) (plus:DI (match_dup 0) (match_dup 3)))]
+  "
+{
+  if (rs6000_emit_set_const (operands[0], operands[1]))
+    DONE;
+  else
+    FAIL;
+}")
 
-(define_insn_and_split "*movtd_64bit_nodm"
-  [(set (match_operand:TD 0 "nonimmediate_operand" "=m,d,d,Y,r,r")
-       (match_operand:TD 1 "input_operand" "d,m,d,r,YGHF,r"))]
-  "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_POWERPC64 && !WORDS_BIG_ENDIAN
-   && (gpc_reg_operand (operands[0], TDmode)
-       || gpc_reg_operand (operands[1], TDmode))"
-  "#"
-  "&& reload_completed"
-  [(pc)]
-{ rs6000_split_multireg_move (operands[0], operands[1]); DONE; }
-  [(set_attr "length" "8,8,8,12,12,8")])
+(define_split
+  [(set (match_operand:DI 0 "gpc_reg_operand" "")
+       (match_operand:DI 1 "const_scalar_int_operand" ""))]
+  "TARGET_POWERPC64 && num_insns_constant (operands[1], DImode) > 1"
+  [(set (match_dup 0) (match_dup 2))
+   (set (match_dup 0) (plus:DI (match_dup 0) (match_dup 3)))]
+  "
+{
+  if (rs6000_emit_set_const (operands[0], operands[1]))
+    DONE;
+  else
+    FAIL;
+}")
+\f
+;; TImode/PTImode is similar, except that we usually want to compute the
+;; address into a register and use lsi/stsi (the exception is during reload).
 
-(define_insn_and_split "*mov<mode>_32bit"
-  [(set (match_operand:FMOVE128 0 "nonimmediate_operand" "=m,d,d,Y,r,r")
-       (match_operand:FMOVE128 1 "input_operand" "d,m,d,r,YGHF,r"))]
-  "TARGET_HARD_FLOAT && TARGET_FPRS && !TARGET_POWERPC64
+(define_insn "*mov<mode>_string"
+  [(set (match_operand:TI2 0 "reg_or_mem_operand" "=Q,Y,????r,????r,????r,r")
+       (match_operand:TI2 1 "input_operand" "r,r,Q,Y,r,n"))]
+  "! TARGET_POWERPC64
+   && (<MODE>mode != TImode || VECTOR_MEM_NONE_P (TImode))
    && (gpc_reg_operand (operands[0], <MODE>mode)
        || gpc_reg_operand (operands[1], <MODE>mode))"
-  "#"
-  "&& reload_completed"
-  [(pc)]
-{ rs6000_split_multireg_move (operands[0], operands[1]); DONE; }
-  [(set_attr "length" "8,8,8,20,20,16")])
+  "*
+{
+  switch (which_alternative)
+    {
+    default:
+      gcc_unreachable ();
+    case 0:
+      if (TARGET_STRING)
+        return \"stswi %1,%P0,16\";
+    case 1:
+      return \"#\";
+    case 2:
+      /* If the address is not used in the output, we can use lsi.  Otherwise,
+        fall through to generating four loads.  */
+      if (TARGET_STRING
+          && ! reg_overlap_mentioned_p (operands[0], operands[1]))
+       return \"lswi %0,%P1,16\";
+      /* ... fall through ...  */
+    case 3:
+    case 4:
+    case 5:
+      return \"#\";
+    }
+}"
+  [(set_attr "type" "store,store,load,load,*,*")
+   (set_attr "update" "yes")
+   (set_attr "indexed" "yes")
+   (set (attr "cell_micro") (if_then_else (match_test "TARGET_STRING")
+                                         (const_string "always")
+                                         (const_string "conditional")))])
 
-(define_insn_and_split "*mov<mode>_softfloat"
-  [(set (match_operand:FMOVE128 0 "rs6000_nonimmediate_operand" "=Y,r,r")
-       (match_operand:FMOVE128 1 "input_operand" "r,YGHF,r"))]
-  "(TARGET_SOFT_FLOAT || !TARGET_FPRS)
+(define_insn "*mov<mode>_ppc64"
+  [(set (match_operand:TI2 0 "nonimmediate_operand" "=wQ,Y,r,r,r,r")
+       (match_operand:TI2 1 "input_operand" "r,r,wQ,Y,r,n"))]
+  "(TARGET_POWERPC64 && VECTOR_MEM_NONE_P (<MODE>mode)
    && (gpc_reg_operand (operands[0], <MODE>mode)
-       || gpc_reg_operand (operands[1], <MODE>mode))"
-  "#"
-  "&& reload_completed"
-  [(pc)]
-{ rs6000_split_multireg_move (operands[0], operands[1]); DONE; }
-  [(set_attr "length" "20,20,16")])
-
-(define_expand "extenddftf2"
-  [(set (match_operand:TF 0 "nonimmediate_operand" "")
-       (float_extend:TF (match_operand:DF 1 "input_operand" "")))]
-  "!TARGET_IEEEQUAD
-   && TARGET_HARD_FLOAT
-   && (TARGET_FPRS || TARGET_E500_DOUBLE)
-   && TARGET_LONG_DOUBLE_128"
+       || gpc_reg_operand (operands[1], <MODE>mode)))"
 {
-  if (TARGET_E500_DOUBLE)
-    emit_insn (gen_spe_extenddftf2 (operands[0], operands[1]));
-  else
-    emit_insn (gen_extenddftf2_fprs (operands[0], operands[1]));
-  DONE;
-})
+  return rs6000_output_move_128bit (operands);
+}
+  [(set_attr "type" "store,store,load,load,*,*")
+   (set_attr "length" "8")])
 
-(define_expand "extenddftf2_fprs"
-  [(parallel [(set (match_operand:TF 0 "nonimmediate_operand" "")
-                  (float_extend:TF (match_operand:DF 1 "input_operand" "")))
-             (use (match_dup 2))])]
-  "!TARGET_IEEEQUAD
-   && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT 
-   && TARGET_LONG_DOUBLE_128"
+(define_split
+  [(set (match_operand:TI2 0 "int_reg_operand" "")
+       (match_operand:TI2 1 "const_scalar_int_operand" ""))]
+  "TARGET_POWERPC64
+   && (VECTOR_MEM_NONE_P (<MODE>mode)
+       || (reload_completed && INT_REGNO_P (REGNO (operands[0]))))"
+  [(set (match_dup 2) (match_dup 4))
+   (set (match_dup 3) (match_dup 5))]
+  "
 {
-  operands[2] = CONST0_RTX (DFmode);
-  /* Generate GOT reference early for SVR4 PIC.  */
-  if (DEFAULT_ABI == ABI_V4 && flag_pic)
-    operands[2] = validize_mem (force_const_mem (DFmode, operands[2]));
-})
+  operands[2] = operand_subword_force (operands[0], WORDS_BIG_ENDIAN == 0,
+                                      <MODE>mode);
+  operands[3] = operand_subword_force (operands[0], WORDS_BIG_ENDIAN != 0,
+                                      <MODE>mode);
+  if (CONST_WIDE_INT_P (operands[1]))
+    {
+      operands[4] = GEN_INT (CONST_WIDE_INT_ELT (operands[1], 1));
+      operands[5] = GEN_INT (CONST_WIDE_INT_ELT (operands[1], 0));
+    }
+  else if (CONST_INT_P (operands[1]))
+    {
+      operands[4] = GEN_INT (- (INTVAL (operands[1]) < 0));
+      operands[5] = operands[1];
+    }
+  else
+    FAIL;
+}")
 
-(define_insn_and_split "*extenddftf2_internal"
-  [(set (match_operand:TF 0 "nonimmediate_operand" "=m,Y,d,&d,r")
-       (float_extend:TF (match_operand:DF 1 "input_operand" "d,r,md,md,rmGHF")))
-   (use (match_operand:DF 2 "zero_reg_mem_operand" "d,r,m,d,n"))]
-  "!TARGET_IEEEQUAD
-   && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT 
-   && TARGET_LONG_DOUBLE_128"
-  "#"
-  "&& reload_completed"
+(define_split
+  [(set (match_operand:TI2 0 "nonimmediate_operand" "")
+        (match_operand:TI2 1 "input_operand" ""))]
+  "reload_completed
+   && gpr_or_gpr_p (operands[0], operands[1])
+   && !direct_move_p (operands[0], operands[1])
+   && !quad_load_store_p (operands[0], operands[1])"
   [(pc)]
+{ rs6000_split_multireg_move (operands[0], operands[1]); DONE; })
+\f
+(define_expand "load_multiple"
+  [(match_par_dup 3 [(set (match_operand:SI 0 "" "")
+                         (match_operand:SI 1 "" ""))
+                    (use (match_operand:SI 2 "" ""))])]
+  "TARGET_STRING && !TARGET_POWERPC64"
+  "
 {
-  const int lo_word = LONG_DOUBLE_LARGE_FIRST ? GET_MODE_SIZE (DFmode) : 0;
-  const int hi_word = LONG_DOUBLE_LARGE_FIRST ? 0 : GET_MODE_SIZE (DFmode);
-  emit_move_insn (simplify_gen_subreg (DFmode, operands[0], TFmode, hi_word),
-                 operands[1]);
-  emit_move_insn (simplify_gen_subreg (DFmode, operands[0], TFmode, lo_word),
-                 operands[2]);
-  DONE;
-})
-
-(define_expand "extendsftf2"
-  [(set (match_operand:TF 0 "nonimmediate_operand" "")
-       (float_extend:TF (match_operand:SF 1 "gpc_reg_operand" "")))]
-  "!TARGET_IEEEQUAD
-   && TARGET_HARD_FLOAT
-   && (TARGET_FPRS || TARGET_E500_DOUBLE)
-   && TARGET_LONG_DOUBLE_128"
-{
-  rtx tmp = gen_reg_rtx (DFmode);
-  emit_insn (gen_extendsfdf2 (tmp, operands[1]));
-  emit_insn (gen_extenddftf2 (operands[0], tmp));
-  DONE;
-})
+  int regno;
+  int count;
+  rtx op1;
+  int i;
 
-(define_expand "trunctfdf2"
-  [(set (match_operand:DF 0 "gpc_reg_operand" "")
-       (float_truncate:DF (match_operand:TF 1 "gpc_reg_operand" "")))]
-  "!TARGET_IEEEQUAD
-   && TARGET_HARD_FLOAT
-   && (TARGET_FPRS || TARGET_E500_DOUBLE)
-   && TARGET_LONG_DOUBLE_128"
-  "")
+  /* Support only loading a constant number of fixed-point registers from
+     memory and only bother with this if more than two; the machine
+     doesn't support more than eight.  */
+  if (GET_CODE (operands[2]) != CONST_INT
+      || INTVAL (operands[2]) <= 2
+      || INTVAL (operands[2]) > 8
+      || GET_CODE (operands[1]) != MEM
+      || GET_CODE (operands[0]) != REG
+      || REGNO (operands[0]) >= 32)
+    FAIL;
 
-(define_insn_and_split "trunctfdf2_internal1"
-  [(set (match_operand:DF 0 "gpc_reg_operand" "=d,?d")
-       (float_truncate:DF (match_operand:TF 1 "gpc_reg_operand" "0,d")))]
-  "!TARGET_IEEEQUAD && !TARGET_XL_COMPAT
-   && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_LONG_DOUBLE_128"
-  "@
-   #
-   fmr %0,%1"
-  "&& reload_completed && REGNO (operands[0]) == REGNO (operands[1])"
-  [(const_int 0)]
-{
-  emit_note (NOTE_INSN_DELETED);
-  DONE;
-}
-  [(set_attr "type" "fp")])
+  count = INTVAL (operands[2]);
+  regno = REGNO (operands[0]);
 
-(define_insn "trunctfdf2_internal2"
-  [(set (match_operand:DF 0 "gpc_reg_operand" "=d")
-       (float_truncate:DF (match_operand:TF 1 "gpc_reg_operand" "d")))]
-  "!TARGET_IEEEQUAD && TARGET_XL_COMPAT
-   && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT 
-   && TARGET_LONG_DOUBLE_128"
-  "fadd %0,%1,%L1"
-  [(set_attr "type" "fp")
-   (set_attr "fp_type" "fp_addsub_d")])
+  operands[3] = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (count));
+  op1 = replace_equiv_address (operands[1],
+                              force_reg (SImode, XEXP (operands[1], 0)));
 
-(define_expand "trunctfsf2"
-  [(set (match_operand:SF 0 "gpc_reg_operand" "")
-       (float_truncate:SF (match_operand:TF 1 "gpc_reg_operand" "")))]
-  "!TARGET_IEEEQUAD
-   && TARGET_HARD_FLOAT
-   && (TARGET_FPRS || TARGET_E500_DOUBLE)
-   && TARGET_LONG_DOUBLE_128"
-{
-  if (TARGET_E500_DOUBLE)
-    emit_insn (gen_spe_trunctfsf2 (operands[0], operands[1]));
-  else
-    emit_insn (gen_trunctfsf2_fprs (operands[0], operands[1]));
-  DONE;
-})
+  for (i = 0; i < count; i++)
+    XVECEXP (operands[3], 0, i)
+      = gen_rtx_SET (gen_rtx_REG (SImode, regno + i),
+                    adjust_address_nv (op1, SImode, i * 4));
+}")
 
-(define_insn_and_split "trunctfsf2_fprs"
-  [(set (match_operand:SF 0 "gpc_reg_operand" "=f")
-       (float_truncate:SF (match_operand:TF 1 "gpc_reg_operand" "d")))
-   (clobber (match_scratch:DF 2 "=d"))]
-  "!TARGET_IEEEQUAD
-   && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_SINGLE_FLOAT 
-   && TARGET_LONG_DOUBLE_128"
-  "#"
-  "&& reload_completed"
-  [(set (match_dup 2)
-       (float_truncate:DF (match_dup 1)))
-   (set (match_dup 0)
-       (float_truncate:SF (match_dup 2)))]
-  "")
-
-(define_expand "floatsitf2"
-  [(set (match_operand:TF 0 "gpc_reg_operand" "")
-        (float:TF (match_operand:SI 1 "gpc_reg_operand" "")))]
-  "!TARGET_IEEEQUAD
-   && TARGET_HARD_FLOAT
-   && (TARGET_FPRS || TARGET_E500_DOUBLE)
-   && TARGET_LONG_DOUBLE_128"
-{
-  rtx tmp = gen_reg_rtx (DFmode);
-  expand_float (tmp, operands[1], false);
-  emit_insn (gen_extenddftf2 (operands[0], tmp));
-  DONE;
-})
-
-; fadd, but rounding towards zero.
-; This is probably not the optimal code sequence.
-(define_insn "fix_trunc_helper"
-  [(set (match_operand:DF 0 "gpc_reg_operand" "=d")
-       (unspec:DF [(match_operand:TF 1 "gpc_reg_operand" "d")]
-                  UNSPEC_FIX_TRUNC_TF))
-   (clobber (match_operand:DF 2 "gpc_reg_operand" "=&d"))]
-  "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT"
-  "mffs %2\n\tmtfsb1 31\n\tmtfsb0 30\n\tfadd %0,%1,%L1\n\tmtfsf 1,%2"
-  [(set_attr "type" "fp")
-   (set_attr "length" "20")])
-
-(define_expand "fix_trunctfsi2"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "")
-       (fix:SI (match_operand:TF 1 "gpc_reg_operand" "")))]
-  "!TARGET_IEEEQUAD && TARGET_HARD_FLOAT
-   && (TARGET_FPRS || TARGET_E500_DOUBLE) && TARGET_LONG_DOUBLE_128"
-{
-  if (TARGET_E500_DOUBLE)
-    emit_insn (gen_spe_fix_trunctfsi2 (operands[0], operands[1]));
-  else
-    emit_insn (gen_fix_trunctfsi2_fprs (operands[0], operands[1]));
-  DONE;
-})
-
-(define_expand "fix_trunctfsi2_fprs"
-  [(parallel [(set (match_operand:SI 0 "gpc_reg_operand" "")
-                  (fix:SI (match_operand:TF 1 "gpc_reg_operand" "")))
-             (clobber (match_dup 2))
-             (clobber (match_dup 3))
-             (clobber (match_dup 4))
-             (clobber (match_dup 5))])]
-  "!TARGET_IEEEQUAD
-   && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_LONG_DOUBLE_128"
-{
-  operands[2] = gen_reg_rtx (DFmode);
-  operands[3] = gen_reg_rtx (DFmode);
-  operands[4] = gen_reg_rtx (DImode);
-  operands[5] = assign_stack_temp (DImode, GET_MODE_SIZE (DImode));
-})
-
-(define_insn_and_split "*fix_trunctfsi2_internal"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-        (fix:SI (match_operand:TF 1 "gpc_reg_operand" "d")))
-   (clobber (match_operand:DF 2 "gpc_reg_operand" "=d"))
-   (clobber (match_operand:DF 3 "gpc_reg_operand" "=&d"))
-   (clobber (match_operand:DI 4 "gpc_reg_operand" "=d"))
-   (clobber (match_operand:DI 5 "offsettable_mem_operand" "=o"))]
-  "!TARGET_IEEEQUAD
-   && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_LONG_DOUBLE_128"
-  "#"
-  ""
-  [(pc)]
-{
-  rtx lowword;
-  emit_insn (gen_fix_trunc_helper (operands[2], operands[1], operands[3]));
-
-  gcc_assert (MEM_P (operands[5]));
-  lowword = adjust_address (operands[5], SImode, WORDS_BIG_ENDIAN ? 4 : 0);
-
-  emit_insn (gen_fctiwz_df (operands[4], operands[2]));
-  emit_move_insn (operands[5], operands[4]);
-  emit_move_insn (operands[0], lowword);
-  DONE;
-})
-
-(define_expand "negtf2"
-  [(set (match_operand:TF 0 "gpc_reg_operand" "")
-       (neg:TF (match_operand:TF 1 "gpc_reg_operand" "")))]
-  "!TARGET_IEEEQUAD
-   && TARGET_HARD_FLOAT
-   && (TARGET_FPRS || TARGET_E500_DOUBLE)
-   && TARGET_LONG_DOUBLE_128"
-  "")
-
-(define_insn "negtf2_internal"
-  [(set (match_operand:TF 0 "gpc_reg_operand" "=d")
-       (neg:TF (match_operand:TF 1 "gpc_reg_operand" "d")))]
-  "!TARGET_IEEEQUAD
-   && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_LONG_DOUBLE_128"
+(define_insn "*ldmsi8"
+  [(match_parallel 0 "load_multiple_operation"
+    [(set (match_operand:SI 2 "gpc_reg_operand" "")
+          (mem:SI (match_operand:SI 1 "gpc_reg_operand" "b")))
+     (set (match_operand:SI 3 "gpc_reg_operand" "")
+          (mem:SI (plus:SI (match_dup 1) (const_int 4))))
+     (set (match_operand:SI 4 "gpc_reg_operand" "")
+          (mem:SI (plus:SI (match_dup 1) (const_int 8))))
+     (set (match_operand:SI 5 "gpc_reg_operand" "")
+          (mem:SI (plus:SI (match_dup 1) (const_int 12))))
+     (set (match_operand:SI 6 "gpc_reg_operand" "")
+          (mem:SI (plus:SI (match_dup 1) (const_int 16))))
+     (set (match_operand:SI 7 "gpc_reg_operand" "")
+          (mem:SI (plus:SI (match_dup 1) (const_int 20))))
+     (set (match_operand:SI 8 "gpc_reg_operand" "")
+          (mem:SI (plus:SI (match_dup 1) (const_int 24))))
+     (set (match_operand:SI 9 "gpc_reg_operand" "")
+          (mem:SI (plus:SI (match_dup 1) (const_int 28))))])]
+  "TARGET_STRING && XVECLEN (operands[0], 0) == 8"
   "*
-{
-  if (REGNO (operands[0]) == REGNO (operands[1]) + 1)
-    return \"fneg %L0,%L1\;fneg %0,%1\";
-  else
-    return \"fneg %0,%1\;fneg %L0,%L1\";
-}"
-  [(set_attr "type" "fp")
-   (set_attr "length" "8")])
+{ return rs6000_output_load_multiple (operands); }"
+  [(set_attr "type" "load")
+   (set_attr "update" "yes")
+   (set_attr "indexed" "yes")
+   (set_attr "length" "32")])
 
-(define_expand "abstf2"
-  [(set (match_operand:TF 0 "gpc_reg_operand" "")
-       (abs:TF (match_operand:TF 1 "gpc_reg_operand" "")))]
-  "!TARGET_IEEEQUAD
-   && TARGET_HARD_FLOAT
-   && (TARGET_FPRS || TARGET_E500_DOUBLE)
-   && TARGET_LONG_DOUBLE_128"
-  "
-{
-  rtx label = gen_label_rtx ();
-  if (TARGET_E500_DOUBLE)
-    {
-      if (flag_finite_math_only && !flag_trapping_math)
-       emit_insn (gen_spe_abstf2_tst (operands[0], operands[1], label));
-      else
-       emit_insn (gen_spe_abstf2_cmp (operands[0], operands[1], label));
-    }
-  else
-    emit_insn (gen_abstf2_internal (operands[0], operands[1], label));
-  emit_label (label);
-  DONE;
-}")
+(define_insn "*ldmsi7"
+  [(match_parallel 0 "load_multiple_operation"
+    [(set (match_operand:SI 2 "gpc_reg_operand" "")
+          (mem:SI (match_operand:SI 1 "gpc_reg_operand" "b")))
+     (set (match_operand:SI 3 "gpc_reg_operand" "")
+          (mem:SI (plus:SI (match_dup 1) (const_int 4))))
+     (set (match_operand:SI 4 "gpc_reg_operand" "")
+          (mem:SI (plus:SI (match_dup 1) (const_int 8))))
+     (set (match_operand:SI 5 "gpc_reg_operand" "")
+          (mem:SI (plus:SI (match_dup 1) (const_int 12))))
+     (set (match_operand:SI 6 "gpc_reg_operand" "")
+          (mem:SI (plus:SI (match_dup 1) (const_int 16))))
+     (set (match_operand:SI 7 "gpc_reg_operand" "")
+          (mem:SI (plus:SI (match_dup 1) (const_int 20))))
+     (set (match_operand:SI 8 "gpc_reg_operand" "")
+          (mem:SI (plus:SI (match_dup 1) (const_int 24))))])]
+  "TARGET_STRING && XVECLEN (operands[0], 0) == 7"
+  "*
+{ return rs6000_output_load_multiple (operands); }"
+  [(set_attr "type" "load")
+   (set_attr "update" "yes")
+   (set_attr "indexed" "yes")
+   (set_attr "length" "32")])
 
-(define_expand "abstf2_internal"
-  [(set (match_operand:TF 0 "gpc_reg_operand" "")
-       (match_operand:TF 1 "gpc_reg_operand" ""))
-   (set (match_dup 3) (match_dup 5))
-   (set (match_dup 5) (abs:DF (match_dup 5)))
-   (set (match_dup 4) (compare:CCFP (match_dup 3) (match_dup 5)))
-   (set (pc) (if_then_else (eq (match_dup 4) (const_int 0))
-                          (label_ref (match_operand 2 "" ""))
-                          (pc)))
-   (set (match_dup 6) (neg:DF (match_dup 6)))]
-  "!TARGET_IEEEQUAD
-   && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT 
-   && TARGET_LONG_DOUBLE_128"
-  "
-{
-  const int hi_word = LONG_DOUBLE_LARGE_FIRST ? 0 : GET_MODE_SIZE (DFmode);
-  const int lo_word = LONG_DOUBLE_LARGE_FIRST ? GET_MODE_SIZE (DFmode) : 0;
-  operands[3] = gen_reg_rtx (DFmode);
-  operands[4] = gen_reg_rtx (CCFPmode);
-  operands[5] = simplify_gen_subreg (DFmode, operands[0], TFmode, hi_word);
-  operands[6] = simplify_gen_subreg (DFmode, operands[0], TFmode, lo_word);
-}")
-\f
-;; Reload helper functions used by rs6000_secondary_reload.  The patterns all
-;; must have 3 arguments, and scratch register constraint must be a single
-;; constraint.
+(define_insn "*ldmsi6"
+  [(match_parallel 0 "load_multiple_operation"
+    [(set (match_operand:SI 2 "gpc_reg_operand" "")
+          (mem:SI (match_operand:SI 1 "gpc_reg_operand" "b")))
+     (set (match_operand:SI 3 "gpc_reg_operand" "")
+          (mem:SI (plus:SI (match_dup 1) (const_int 4))))
+     (set (match_operand:SI 4 "gpc_reg_operand" "")
+          (mem:SI (plus:SI (match_dup 1) (const_int 8))))
+     (set (match_operand:SI 5 "gpc_reg_operand" "")
+          (mem:SI (plus:SI (match_dup 1) (const_int 12))))
+     (set (match_operand:SI 6 "gpc_reg_operand" "")
+          (mem:SI (plus:SI (match_dup 1) (const_int 16))))
+     (set (match_operand:SI 7 "gpc_reg_operand" "")
+          (mem:SI (plus:SI (match_dup 1) (const_int 20))))])]
+  "TARGET_STRING && XVECLEN (operands[0], 0) == 6"
+  "*
+{ return rs6000_output_load_multiple (operands); }"
+  [(set_attr "type" "load")
+   (set_attr "update" "yes")
+   (set_attr "indexed" "yes")
+   (set_attr "length" "32")])
 
-;; Reload patterns to support gpr load/store with misaligned mem.
-;; and multiple gpr load/store at offset >= 0xfffc
-(define_expand "reload_<mode>_store"
-  [(parallel [(match_operand 0 "memory_operand" "=m")
-              (match_operand 1 "gpc_reg_operand" "r")
-              (match_operand:GPR 2 "register_operand" "=&b")])]
-  ""
-{
-  rs6000_secondary_reload_gpr (operands[1], operands[0], operands[2], true);
-  DONE;
-})
+(define_insn "*ldmsi5"
+  [(match_parallel 0 "load_multiple_operation"
+    [(set (match_operand:SI 2 "gpc_reg_operand" "")
+          (mem:SI (match_operand:SI 1 "gpc_reg_operand" "b")))
+     (set (match_operand:SI 3 "gpc_reg_operand" "")
+          (mem:SI (plus:SI (match_dup 1) (const_int 4))))
+     (set (match_operand:SI 4 "gpc_reg_operand" "")
+          (mem:SI (plus:SI (match_dup 1) (const_int 8))))
+     (set (match_operand:SI 5 "gpc_reg_operand" "")
+          (mem:SI (plus:SI (match_dup 1) (const_int 12))))
+     (set (match_operand:SI 6 "gpc_reg_operand" "")
+          (mem:SI (plus:SI (match_dup 1) (const_int 16))))])]
+  "TARGET_STRING && XVECLEN (operands[0], 0) == 5"
+  "*
+{ return rs6000_output_load_multiple (operands); }"
+  [(set_attr "type" "load")
+   (set_attr "update" "yes")
+   (set_attr "indexed" "yes")
+   (set_attr "length" "32")])
 
-(define_expand "reload_<mode>_load"
-  [(parallel [(match_operand 0 "gpc_reg_operand" "=r")
-              (match_operand 1 "memory_operand" "m")
-              (match_operand:GPR 2 "register_operand" "=b")])]
-  ""
-{
-  rs6000_secondary_reload_gpr (operands[0], operands[1], operands[2], false);
-  DONE;
-})
-
-\f
-;; Power8 merge instructions to allow direct move to/from floating point
-;; registers in 32-bit mode.  We use TF mode to get two registers to move the
-;; individual 32-bit parts across.  Subreg doesn't work too well on the TF
-;; value, since it is allocated in reload and not all of the flow information
-;; is setup for it.  We have two patterns to do the two moves between gprs and
-;; fprs.  There isn't a dependancy between the two, but we could potentially
-;; schedule other instructions between the two instructions.  TFmode is
-;; currently limited to traditional FPR registers.  If/when this is changed, we
-;; will need to revist %L to make sure it works with VSX registers, or add an
-;; %x version of %L.
-
-(define_insn "p8_fmrgow_<mode>"
-  [(set (match_operand:FMOVE64X 0 "register_operand" "=d")
-       (unspec:FMOVE64X [(match_operand:TF 1 "register_operand" "d")]
-                        UNSPEC_P8V_FMRGOW))]
-  "!TARGET_POWERPC64 && TARGET_DIRECT_MOVE"
-  "fmrgow %0,%1,%L1"
-  [(set_attr "type" "vecperm")])
-
-(define_insn "p8_mtvsrwz_1"
-  [(set (match_operand:TF 0 "register_operand" "=d")
-       (unspec:TF [(match_operand:SI 1 "register_operand" "r")]
-                  UNSPEC_P8V_MTVSRWZ))]
-  "!TARGET_POWERPC64 && TARGET_DIRECT_MOVE"
-  "mtvsrwz %x0,%1"
-  [(set_attr "type" "mftgpr")])
-
-(define_insn "p8_mtvsrwz_2"
-  [(set (match_operand:TF 0 "register_operand" "+d")
-       (unspec:TF [(match_dup 0)
-                   (match_operand:SI 1 "register_operand" "r")]
-                  UNSPEC_P8V_MTVSRWZ))]
-  "!TARGET_POWERPC64 && TARGET_DIRECT_MOVE"
-  "mtvsrwz %L0,%1"
-  [(set_attr "type" "mftgpr")])
-
-(define_insn_and_split "reload_fpr_from_gpr<mode>"
-  [(set (match_operand:FMOVE64X 0 "register_operand" "=ws")
-       (unspec:FMOVE64X [(match_operand:FMOVE64X 1 "register_operand" "r")]
-                        UNSPEC_P8V_RELOAD_FROM_GPR))
-   (clobber (match_operand:TF 2 "register_operand" "=d"))]
-  "!TARGET_POWERPC64 && TARGET_DIRECT_MOVE"
-  "#"
-  "&& reload_completed"
-  [(const_int 0)]
-{
-  rtx dest = operands[0];
-  rtx src = operands[1];
-  rtx tmp = operands[2];
-  rtx gpr_hi_reg = gen_highpart (SImode, src);
-  rtx gpr_lo_reg = gen_lowpart (SImode, src);
-
-  emit_insn (gen_p8_mtvsrwz_1 (tmp, gpr_hi_reg));
-  emit_insn (gen_p8_mtvsrwz_2 (tmp, gpr_lo_reg));
-  emit_insn (gen_p8_fmrgow_<mode> (dest, tmp));
-  DONE;
-}
-  [(set_attr "length" "12")
-   (set_attr "type" "three")])
-
-;; Move 128 bit values from GPRs to VSX registers in 64-bit mode
-(define_insn "p8_mtvsrd_1"
-  [(set (match_operand:TF 0 "register_operand" "=ws")
-       (unspec:TF [(match_operand:DI 1 "register_operand" "r")]
-                  UNSPEC_P8V_MTVSRD))]
-  "TARGET_POWERPC64 && TARGET_DIRECT_MOVE"
-  "mtvsrd %0,%1"
-  [(set_attr "type" "mftgpr")])
-
-(define_insn "p8_mtvsrd_2"
-  [(set (match_operand:TF 0 "register_operand" "+ws")
-       (unspec:TF [(match_dup 0)
-                   (match_operand:DI 1 "register_operand" "r")]
-                  UNSPEC_P8V_MTVSRD))]
-  "TARGET_POWERPC64 && TARGET_DIRECT_MOVE"
-  "mtvsrd %L0,%1"
-  [(set_attr "type" "mftgpr")])
-
-(define_insn "p8_xxpermdi_<mode>"
-  [(set (match_operand:FMOVE128_GPR 0 "register_operand" "=wa")
-       (unspec:FMOVE128_GPR [(match_operand:TF 1 "register_operand" "ws")]
-                            UNSPEC_P8V_XXPERMDI))]
-  "TARGET_POWERPC64 && TARGET_DIRECT_MOVE"
-  "xxpermdi %x0,%1,%L1,0"
-  [(set_attr "type" "vecperm")])
-
-(define_insn_and_split "reload_vsx_from_gpr<mode>"
-  [(set (match_operand:FMOVE128_GPR 0 "register_operand" "=wa")
-       (unspec:FMOVE128_GPR
-        [(match_operand:FMOVE128_GPR 1 "register_operand" "r")]
-        UNSPEC_P8V_RELOAD_FROM_GPR))
-   (clobber (match_operand:TF 2 "register_operand" "=ws"))]
-  "TARGET_POWERPC64 && TARGET_DIRECT_MOVE"
-  "#"
-  "&& reload_completed"
-  [(const_int 0)]
-{
-  rtx dest = operands[0];
-  rtx src = operands[1];
-  rtx tmp = operands[2];
-  rtx gpr_hi_reg = gen_highpart (DImode, src);
-  rtx gpr_lo_reg = gen_lowpart (DImode, src);
-
-  emit_insn (gen_p8_mtvsrd_1 (tmp, gpr_hi_reg));
-  emit_insn (gen_p8_mtvsrd_2 (tmp, gpr_lo_reg));
-  emit_insn (gen_p8_xxpermdi_<mode> (dest, tmp));
-}
-  [(set_attr "length" "12")
-   (set_attr "type" "three")])
-
-(define_split
-  [(set (match_operand:FMOVE128_GPR 0 "nonimmediate_operand" "")
-       (match_operand:FMOVE128_GPR 1 "input_operand" ""))]
-  "reload_completed
-   && (int_reg_operand (operands[0], <MODE>mode)
-       || int_reg_operand (operands[1], <MODE>mode))"
-  [(pc)]
-{ rs6000_split_multireg_move (operands[0], operands[1]); DONE; })
-
-;; Move SFmode to a VSX from a GPR register.  Because scalar floating point
-;; type is stored internally as double precision in the VSX registers, we have
-;; to convert it from the vector format.
-
-(define_insn_and_split "reload_vsx_from_gprsf"
-  [(set (match_operand:SF 0 "register_operand" "=wa")
-       (unspec:SF [(match_operand:SF 1 "register_operand" "r")]
-                  UNSPEC_P8V_RELOAD_FROM_GPR))
-   (clobber (match_operand:DI 2 "register_operand" "=r"))]
-  "TARGET_POWERPC64 && TARGET_DIRECT_MOVE"
-  "#"
-  "&& reload_completed"
-  [(const_int 0)]
-{
-  rtx op0 = operands[0];
-  rtx op1 = operands[1];
-  rtx op2 = operands[2];
-  /* Also use the destination register to hold the unconverted DImode value.
-     This is conceptually a separate value from OP0, so we use gen_rtx_REG
-     rather than simplify_gen_subreg.  */
-  rtx op0_di = gen_rtx_REG (DImode, REGNO (op0));
-  rtx op1_di = simplify_gen_subreg (DImode, op1, SFmode, 0);
-
-  /* Move SF value to upper 32-bits for xscvspdpn.  */
-  emit_insn (gen_ashldi3 (op2, op1_di, GEN_INT (32)));
-  emit_move_insn (op0_di, op2);
-  emit_insn (gen_vsx_xscvspdpn_directmove (op0, op0_di));
-  DONE;
-}
-  [(set_attr "length" "8")
-   (set_attr "type" "two")])
-
-;; Move 128 bit values from VSX registers to GPRs in 64-bit mode by doing a
-;; normal 64-bit move, followed by an xxpermdi to get the bottom 64-bit value,
-;; and then doing a move of that.
-(define_insn "p8_mfvsrd_3_<mode>"
-  [(set (match_operand:DF 0 "register_operand" "=r")
-       (unspec:DF [(match_operand:FMOVE128_GPR 1 "register_operand" "wa")]
-                  UNSPEC_P8V_RELOAD_FROM_VSX))]
-  "TARGET_POWERPC64 && TARGET_DIRECT_MOVE"
-  "mfvsrd %0,%x1"
-  [(set_attr "type" "mftgpr")])
-
-(define_insn_and_split "reload_gpr_from_vsx<mode>"
-  [(set (match_operand:FMOVE128_GPR 0 "register_operand" "=r")
-       (unspec:FMOVE128_GPR
-        [(match_operand:FMOVE128_GPR 1 "register_operand" "wa")]
-        UNSPEC_P8V_RELOAD_FROM_VSX))
-   (clobber (match_operand:FMOVE128_GPR 2 "register_operand" "=wa"))]
-  "TARGET_POWERPC64 && TARGET_DIRECT_MOVE"
-  "#"
-  "&& reload_completed"
-  [(const_int 0)]
-{
-  rtx dest = operands[0];
-  rtx src = operands[1];
-  rtx tmp = operands[2];
-  rtx gpr_hi_reg = gen_highpart (DFmode, dest);
-  rtx gpr_lo_reg = gen_lowpart (DFmode, dest);
-
-  emit_insn (gen_p8_mfvsrd_3_<mode> (gpr_hi_reg, src));
-  emit_insn (gen_vsx_xxpermdi_<mode> (tmp, src, src, GEN_INT (3)));
-  emit_insn (gen_p8_mfvsrd_3_<mode> (gpr_lo_reg, tmp));
-}
-  [(set_attr "length" "12")
-   (set_attr "type" "three")])
-
-;; Move SFmode to a GPR from a VSX register.  Because scalar floating point
-;; type is stored internally as double precision, we have to convert it to the
-;; vector format.
-
-(define_insn_and_split "reload_gpr_from_vsxsf"
-  [(set (match_operand:SF 0 "register_operand" "=r")
-       (unspec:SF [(match_operand:SF 1 "register_operand" "wa")]
-                  UNSPEC_P8V_RELOAD_FROM_VSX))
-   (clobber (match_operand:V4SF 2 "register_operand" "=wa"))]
-  "TARGET_POWERPC64 && TARGET_DIRECT_MOVE"
-  "#"
-  "&& reload_completed"
-  [(const_int 0)]
-{
-  rtx op0 = operands[0];
-  rtx op1 = operands[1];
-  rtx op2 = operands[2];
-  rtx diop0 = simplify_gen_subreg (DImode, op0, SFmode, 0);
-
-  emit_insn (gen_vsx_xscvdpspn_scalar (op2, op1));
-  emit_insn (gen_p8_mfvsrd_4_disf (diop0, op2));
-  emit_insn (gen_lshrdi3 (diop0, diop0, GEN_INT (32)));
-  DONE;
-}
-  [(set_attr "length" "12")
-   (set_attr "type" "three")])
-
-(define_insn "p8_mfvsrd_4_disf"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-       (unspec:DI [(match_operand:V4SF 1 "register_operand" "wa")]
-                  UNSPEC_P8V_RELOAD_FROM_VSX))]
-  "TARGET_POWERPC64 && TARGET_DIRECT_MOVE"
-  "mfvsrd %0,%x1"
-  [(set_attr "type" "mftgpr")])
-
-\f
-;; Next come the multi-word integer load and store and the load and store
-;; multiple insns.
-
-;; List r->r after r->Y, otherwise reload will try to reload a
-;; non-offsettable address by using r->r which won't make progress.
-;; Use of fprs is disparaged slightly otherwise reload prefers to reload
-;; a gpr into a fpr instead of reloading an invalid 'Y' address
-(define_insn "*movdi_internal32"
-  [(set (match_operand:DI 0 "rs6000_nonimmediate_operand" "=Y,r,r,?m,?*d,?*d,r")
-       (match_operand:DI 1 "input_operand" "r,Y,r,d,m,d,IJKnGHF"))]
-  "! TARGET_POWERPC64
-   && (gpc_reg_operand (operands[0], DImode)
-       || gpc_reg_operand (operands[1], DImode))"
-  "@
-   #
-   #
-   #
-   stfd%U0%X0 %1,%0
-   lfd%U1%X1 %0,%1
-   fmr %0,%1
-   #"
-  [(set_attr "type" "store,load,*,fpstore,fpload,fp,*")])
-
-(define_split
-  [(set (match_operand:DI 0 "gpc_reg_operand" "")
-       (match_operand:DI 1 "const_int_operand" ""))]
-  "! TARGET_POWERPC64 && reload_completed
-   && gpr_or_gpr_p (operands[0], operands[1])
-   && !direct_move_p (operands[0], operands[1])"
-  [(set (match_dup 2) (match_dup 4))
-   (set (match_dup 3) (match_dup 1))]
-  "
-{
-  HOST_WIDE_INT value = INTVAL (operands[1]);
-  operands[2] = operand_subword_force (operands[0], WORDS_BIG_ENDIAN == 0,
-                                      DImode);
-  operands[3] = operand_subword_force (operands[0], WORDS_BIG_ENDIAN != 0,
-                                      DImode);
-  operands[4] = GEN_INT (value >> 32);
-  operands[1] = GEN_INT (((value & 0xffffffff) ^ 0x80000000) - 0x80000000);
-}")
-
-(define_split
-  [(set (match_operand:DIFD 0 "rs6000_nonimmediate_operand" "")
-        (match_operand:DIFD 1 "input_operand" ""))]
-  "reload_completed && !TARGET_POWERPC64
-   && gpr_or_gpr_p (operands[0], operands[1])
-   && !direct_move_p (operands[0], operands[1])"
-  [(pc)]
-{ rs6000_split_multireg_move (operands[0], operands[1]); DONE; })
-
-(define_insn "*movdi_internal64"
-  [(set (match_operand:DI 0 "nonimmediate_operand" "=Y,r,r,r,r,r,?m,?*d,?*d,r,*h,*h,r,?*wg,r,?*wm")
-       (match_operand:DI 1 "input_operand" "r,Y,r,I,L,nF,d,m,d,*h,r,0,*wg,r,*wm,r"))]
-  "TARGET_POWERPC64
-   && (gpc_reg_operand (operands[0], DImode)
-       || gpc_reg_operand (operands[1], DImode))"
-  "@
-   std%U0%X0 %1,%0
-   ld%U1%X1 %0,%1
-   mr %0,%1
-   li %0,%1
-   lis %0,%v1
-   #
-   stfd%U0%X0 %1,%0
-   lfd%U1%X1 %0,%1
-   fmr %0,%1
-   mf%1 %0
-   mt%0 %1
-   nop
-   mftgpr %0,%1
-   mffgpr %0,%1
-   mfvsrd %0,%x1
-   mtvsrd %x0,%1"
-  [(set_attr "type" "store,load,*,*,*,*,fpstore,fpload,fp,mfjmpr,mtjmpr,*,mftgpr,mffgpr,mftgpr,mffgpr")
-   (set_attr "length" "4,4,4,4,4,20,4,4,4,4,4,4,4,4,4,4")])
-
-;; Generate all one-bits and clear left or right.
-;; Use (and:DI (rotate:DI ...)) to avoid anddi3 unnecessary clobber.
-(define_split
-  [(set (match_operand:DI 0 "gpc_reg_operand" "")
-       (match_operand:DI 1 "mask64_operand" ""))]
-  "TARGET_POWERPC64 && num_insns_constant (operands[1], DImode) > 1"
-  [(set (match_dup 0) (const_int -1))
-   (set (match_dup 0)
-       (and:DI (rotate:DI (match_dup 0)
-                          (const_int 0))
-               (match_dup 1)))]
-  "")
-
-;; Split a load of a large constant into the appropriate five-instruction
-;; sequence.  Handle anything in a constant number of insns.
-;; When non-easy constants can go in the TOC, this should use
-;; easy_fp_constant predicate.
-(define_split
-  [(set (match_operand:DI 0 "gpc_reg_operand" "")
-       (match_operand:DI 1 "const_int_operand" ""))]
-  "TARGET_POWERPC64 && num_insns_constant (operands[1], DImode) > 1"
-  [(set (match_dup 0) (match_dup 2))
-   (set (match_dup 0) (plus:DI (match_dup 0) (match_dup 3)))]
-  "
-{ rtx tem = rs6000_emit_set_const (operands[0], DImode, operands[1], 5);
-
-  if (tem == operands[0])
-    DONE;
-  else
-    FAIL;
-}")
-
-(define_split
-  [(set (match_operand:DI 0 "gpc_reg_operand" "")
-       (match_operand:DI 1 "const_scalar_int_operand" ""))]
-  "TARGET_POWERPC64 && num_insns_constant (operands[1], DImode) > 1"
-  [(set (match_dup 0) (match_dup 2))
-   (set (match_dup 0) (plus:DI (match_dup 0) (match_dup 3)))]
-  "
-{ rtx tem = rs6000_emit_set_const (operands[0], DImode, operands[1], 5);
-
-  if (tem == operands[0])
-    DONE;
-  else
-    FAIL;
-}")
-\f
-;; TImode/PTImode is similar, except that we usually want to compute the
-;; address into a register and use lsi/stsi (the exception is during reload).
-
-(define_insn "*mov<mode>_string"
-  [(set (match_operand:TI2 0 "reg_or_mem_operand" "=Q,Y,????r,????r,????r,r")
-       (match_operand:TI2 1 "input_operand" "r,r,Q,Y,r,n"))]
-  "! TARGET_POWERPC64
-   && (<MODE>mode != TImode || VECTOR_MEM_NONE_P (TImode))
-   && (gpc_reg_operand (operands[0], <MODE>mode)
-       || gpc_reg_operand (operands[1], <MODE>mode))"
-  "*
-{
-  switch (which_alternative)
-    {
-    default:
-      gcc_unreachable ();
-    case 0:
-      if (TARGET_STRING)
-        return \"stswi %1,%P0,16\";
-    case 1:
-      return \"#\";
-    case 2:
-      /* If the address is not used in the output, we can use lsi.  Otherwise,
-        fall through to generating four loads.  */
-      if (TARGET_STRING
-          && ! reg_overlap_mentioned_p (operands[0], operands[1]))
-       return \"lswi %0,%P1,16\";
-      /* ... fall through ...  */
-    case 3:
-    case 4:
-    case 5:
-      return \"#\";
-    }
-}"
-  [(set_attr "type" "store,store,load,load,*,*")
-   (set_attr "update" "yes")
-   (set_attr "indexed" "yes")
-   (set (attr "cell_micro") (if_then_else (match_test "TARGET_STRING")
-                                         (const_string "always")
-                                         (const_string "conditional")))])
-
-(define_insn "*mov<mode>_ppc64"
-  [(set (match_operand:TI2 0 "nonimmediate_operand" "=wQ,Y,r,r,r,r")
-       (match_operand:TI2 1 "input_operand" "r,r,wQ,Y,r,n"))]
-  "(TARGET_POWERPC64 && VECTOR_MEM_NONE_P (<MODE>mode)
-   && (gpc_reg_operand (operands[0], <MODE>mode)
-       || gpc_reg_operand (operands[1], <MODE>mode)))"
-{
-  return rs6000_output_move_128bit (operands);
-}
-  [(set_attr "type" "store,store,load,load,*,*")
-   (set_attr "length" "8")])
-
-(define_split
-  [(set (match_operand:TI2 0 "int_reg_operand" "")
-       (match_operand:TI2 1 "const_scalar_int_operand" ""))]
-  "TARGET_POWERPC64
-   && (VECTOR_MEM_NONE_P (<MODE>mode)
-       || (reload_completed && INT_REGNO_P (REGNO (operands[0]))))"
-  [(set (match_dup 2) (match_dup 4))
-   (set (match_dup 3) (match_dup 5))]
-  "
-{
-  operands[2] = operand_subword_force (operands[0], WORDS_BIG_ENDIAN == 0,
-                                      <MODE>mode);
-  operands[3] = operand_subword_force (operands[0], WORDS_BIG_ENDIAN != 0,
-                                      <MODE>mode);
-  if (CONST_WIDE_INT_P (operands[1]))
-    {
-      operands[4] = GEN_INT (CONST_WIDE_INT_ELT (operands[1], 1));
-      operands[5] = GEN_INT (CONST_WIDE_INT_ELT (operands[1], 0));
-    }
-  else if (CONST_INT_P (operands[1]))
-    {
-      operands[4] = GEN_INT (- (INTVAL (operands[1]) < 0));
-      operands[5] = operands[1];
-    }
-  else
-    FAIL;
-}")
-
-(define_split
-  [(set (match_operand:TI2 0 "nonimmediate_operand" "")
-        (match_operand:TI2 1 "input_operand" ""))]
-  "reload_completed
-   && gpr_or_gpr_p (operands[0], operands[1])
-   && !direct_move_p (operands[0], operands[1])
-   && !quad_load_store_p (operands[0], operands[1])"
-  [(pc)]
-{ rs6000_split_multireg_move (operands[0], operands[1]); DONE; })
-\f
-(define_expand "load_multiple"
-  [(match_par_dup 3 [(set (match_operand:SI 0 "" "")
-                         (match_operand:SI 1 "" ""))
-                    (use (match_operand:SI 2 "" ""))])]
-  "TARGET_STRING && !TARGET_POWERPC64"
-  "
-{
-  int regno;
-  int count;
-  rtx op1;
-  int i;
-
-  /* Support only loading a constant number of fixed-point registers from
-     memory and only bother with this if more than two; the machine
-     doesn't support more than eight.  */
-  if (GET_CODE (operands[2]) != CONST_INT
-      || INTVAL (operands[2]) <= 2
-      || INTVAL (operands[2]) > 8
-      || GET_CODE (operands[1]) != MEM
-      || GET_CODE (operands[0]) != REG
-      || REGNO (operands[0]) >= 32)
-    FAIL;
-
-  count = INTVAL (operands[2]);
-  regno = REGNO (operands[0]);
-
-  operands[3] = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (count));
-  op1 = replace_equiv_address (operands[1],
-                              force_reg (SImode, XEXP (operands[1], 0)));
-
-  for (i = 0; i < count; i++)
-    XVECEXP (operands[3], 0, i)
-      = gen_rtx_SET (VOIDmode, gen_rtx_REG (SImode, regno + i),
-                    adjust_address_nv (op1, SImode, i * 4));
-}")
-
-(define_insn "*ldmsi8"
-  [(match_parallel 0 "load_multiple_operation"
-    [(set (match_operand:SI 2 "gpc_reg_operand" "")
-          (mem:SI (match_operand:SI 1 "gpc_reg_operand" "b")))
-     (set (match_operand:SI 3 "gpc_reg_operand" "")
-          (mem:SI (plus:SI (match_dup 1) (const_int 4))))
-     (set (match_operand:SI 4 "gpc_reg_operand" "")
-          (mem:SI (plus:SI (match_dup 1) (const_int 8))))
-     (set (match_operand:SI 5 "gpc_reg_operand" "")
-          (mem:SI (plus:SI (match_dup 1) (const_int 12))))
-     (set (match_operand:SI 6 "gpc_reg_operand" "")
-          (mem:SI (plus:SI (match_dup 1) (const_int 16))))
-     (set (match_operand:SI 7 "gpc_reg_operand" "")
-          (mem:SI (plus:SI (match_dup 1) (const_int 20))))
-     (set (match_operand:SI 8 "gpc_reg_operand" "")
-          (mem:SI (plus:SI (match_dup 1) (const_int 24))))
-     (set (match_operand:SI 9 "gpc_reg_operand" "")
-          (mem:SI (plus:SI (match_dup 1) (const_int 28))))])]
-  "TARGET_STRING && XVECLEN (operands[0], 0) == 8"
-  "*
-{ return rs6000_output_load_multiple (operands); }"
-  [(set_attr "type" "load")
-   (set_attr "update" "yes")
-   (set_attr "indexed" "yes")
-   (set_attr "length" "32")])
-
-(define_insn "*ldmsi7"
-  [(match_parallel 0 "load_multiple_operation"
-    [(set (match_operand:SI 2 "gpc_reg_operand" "")
-          (mem:SI (match_operand:SI 1 "gpc_reg_operand" "b")))
-     (set (match_operand:SI 3 "gpc_reg_operand" "")
-          (mem:SI (plus:SI (match_dup 1) (const_int 4))))
-     (set (match_operand:SI 4 "gpc_reg_operand" "")
-          (mem:SI (plus:SI (match_dup 1) (const_int 8))))
-     (set (match_operand:SI 5 "gpc_reg_operand" "")
-          (mem:SI (plus:SI (match_dup 1) (const_int 12))))
-     (set (match_operand:SI 6 "gpc_reg_operand" "")
-          (mem:SI (plus:SI (match_dup 1) (const_int 16))))
-     (set (match_operand:SI 7 "gpc_reg_operand" "")
-          (mem:SI (plus:SI (match_dup 1) (const_int 20))))
-     (set (match_operand:SI 8 "gpc_reg_operand" "")
-          (mem:SI (plus:SI (match_dup 1) (const_int 24))))])]
-  "TARGET_STRING && XVECLEN (operands[0], 0) == 7"
-  "*
-{ return rs6000_output_load_multiple (operands); }"
-  [(set_attr "type" "load")
-   (set_attr "update" "yes")
-   (set_attr "indexed" "yes")
-   (set_attr "length" "32")])
-
-(define_insn "*ldmsi6"
-  [(match_parallel 0 "load_multiple_operation"
-    [(set (match_operand:SI 2 "gpc_reg_operand" "")
-          (mem:SI (match_operand:SI 1 "gpc_reg_operand" "b")))
-     (set (match_operand:SI 3 "gpc_reg_operand" "")
-          (mem:SI (plus:SI (match_dup 1) (const_int 4))))
-     (set (match_operand:SI 4 "gpc_reg_operand" "")
-          (mem:SI (plus:SI (match_dup 1) (const_int 8))))
-     (set (match_operand:SI 5 "gpc_reg_operand" "")
-          (mem:SI (plus:SI (match_dup 1) (const_int 12))))
-     (set (match_operand:SI 6 "gpc_reg_operand" "")
-          (mem:SI (plus:SI (match_dup 1) (const_int 16))))
-     (set (match_operand:SI 7 "gpc_reg_operand" "")
-          (mem:SI (plus:SI (match_dup 1) (const_int 20))))])]
-  "TARGET_STRING && XVECLEN (operands[0], 0) == 6"
-  "*
-{ return rs6000_output_load_multiple (operands); }"
-  [(set_attr "type" "load")
-   (set_attr "update" "yes")
-   (set_attr "indexed" "yes")
-   (set_attr "length" "32")])
-
-(define_insn "*ldmsi5"
-  [(match_parallel 0 "load_multiple_operation"
-    [(set (match_operand:SI 2 "gpc_reg_operand" "")
-          (mem:SI (match_operand:SI 1 "gpc_reg_operand" "b")))
-     (set (match_operand:SI 3 "gpc_reg_operand" "")
-          (mem:SI (plus:SI (match_dup 1) (const_int 4))))
-     (set (match_operand:SI 4 "gpc_reg_operand" "")
-          (mem:SI (plus:SI (match_dup 1) (const_int 8))))
-     (set (match_operand:SI 5 "gpc_reg_operand" "")
-          (mem:SI (plus:SI (match_dup 1) (const_int 12))))
-     (set (match_operand:SI 6 "gpc_reg_operand" "")
-          (mem:SI (plus:SI (match_dup 1) (const_int 16))))])]
-  "TARGET_STRING && XVECLEN (operands[0], 0) == 5"
-  "*
-{ return rs6000_output_load_multiple (operands); }"
-  [(set_attr "type" "load")
-   (set_attr "update" "yes")
-   (set_attr "indexed" "yes")
-   (set_attr "length" "32")])
-
-(define_insn "*ldmsi4"
-  [(match_parallel 0 "load_multiple_operation"
-    [(set (match_operand:SI 2 "gpc_reg_operand" "")
-          (mem:SI (match_operand:SI 1 "gpc_reg_operand" "b")))
-     (set (match_operand:SI 3 "gpc_reg_operand" "")
-          (mem:SI (plus:SI (match_dup 1) (const_int 4))))
-     (set (match_operand:SI 4 "gpc_reg_operand" "")
-          (mem:SI (plus:SI (match_dup 1) (const_int 8))))
-     (set (match_operand:SI 5 "gpc_reg_operand" "")
-          (mem:SI (plus:SI (match_dup 1) (const_int 12))))])]
-  "TARGET_STRING && XVECLEN (operands[0], 0) == 4"
-  "*
-{ return rs6000_output_load_multiple (operands); }"
-  [(set_attr "type" "load")
-   (set_attr "update" "yes")
-   (set_attr "indexed" "yes")
-   (set_attr "length" "32")])
-
-(define_insn "*ldmsi3"
-  [(match_parallel 0 "load_multiple_operation"
-    [(set (match_operand:SI 2 "gpc_reg_operand" "")
-          (mem:SI (match_operand:SI 1 "gpc_reg_operand" "b")))
-     (set (match_operand:SI 3 "gpc_reg_operand" "")
-          (mem:SI (plus:SI (match_dup 1) (const_int 4))))
-     (set (match_operand:SI 4 "gpc_reg_operand" "")
-          (mem:SI (plus:SI (match_dup 1) (const_int 8))))])]
-  "TARGET_STRING && XVECLEN (operands[0], 0) == 3"
-  "*
-{ return rs6000_output_load_multiple (operands); }"
-  [(set_attr "type" "load")
-   (set_attr "update" "yes")
-   (set_attr "indexed" "yes")
-   (set_attr "length" "32")])
-
-(define_expand "store_multiple"
-  [(match_par_dup 3 [(set (match_operand:SI 0 "" "")
-                         (match_operand:SI 1 "" ""))
-                    (clobber (scratch:SI))
-                    (use (match_operand:SI 2 "" ""))])]
-  "TARGET_STRING && !TARGET_POWERPC64"
-  "
-{
-  int regno;
-  int count;
-  rtx to;
-  rtx op0;
-  int i;
-
-  /* Support only storing a constant number of fixed-point registers to
-     memory and only bother with this if more than two; the machine
-     doesn't support more than eight.  */
-  if (GET_CODE (operands[2]) != CONST_INT
-      || INTVAL (operands[2]) <= 2
-      || INTVAL (operands[2]) > 8
-      || GET_CODE (operands[0]) != MEM
-      || GET_CODE (operands[1]) != REG
-      || REGNO (operands[1]) >= 32)
-    FAIL;
-
-  count = INTVAL (operands[2]);
-  regno = REGNO (operands[1]);
-
-  operands[3] = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (count + 1));
-  to = force_reg (SImode, XEXP (operands[0], 0));
-  op0 = replace_equiv_address (operands[0], to);
-
-  XVECEXP (operands[3], 0, 0)
-    = gen_rtx_SET (VOIDmode, adjust_address_nv (op0, SImode, 0), operands[1]);
-  XVECEXP (operands[3], 0, 1) = gen_rtx_CLOBBER (VOIDmode,
-                                                gen_rtx_SCRATCH (SImode));
-
-  for (i = 1; i < count; i++)
-    XVECEXP (operands[3], 0, i + 1)
-      = gen_rtx_SET (VOIDmode,
-                    adjust_address_nv (op0, SImode, i * 4),
-                    gen_rtx_REG (SImode, regno + i));
-}")
-
-(define_insn "*stmsi8"
-  [(match_parallel 0 "store_multiple_operation"
-    [(set (mem:SI (match_operand:SI 1 "gpc_reg_operand" "b"))
-         (match_operand:SI 2 "gpc_reg_operand" "r"))
-     (clobber (match_scratch:SI 3 "=X"))
-     (set (mem:SI (plus:SI (match_dup 1) (const_int 4)))
-         (match_operand:SI 4 "gpc_reg_operand" "r"))
-     (set (mem:SI (plus:SI (match_dup 1) (const_int 8)))
-         (match_operand:SI 5 "gpc_reg_operand" "r"))
-     (set (mem:SI (plus:SI (match_dup 1) (const_int 12)))
-         (match_operand:SI 6 "gpc_reg_operand" "r"))
-     (set (mem:SI (plus:SI (match_dup 1) (const_int 16)))
-         (match_operand:SI 7 "gpc_reg_operand" "r"))
-     (set (mem:SI (plus:SI (match_dup 1) (const_int 20)))
-         (match_operand:SI 8 "gpc_reg_operand" "r"))
-     (set (mem:SI (plus:SI (match_dup 1) (const_int 24)))
-         (match_operand:SI 9 "gpc_reg_operand" "r"))
-     (set (mem:SI (plus:SI (match_dup 1) (const_int 28)))
-         (match_operand:SI 10 "gpc_reg_operand" "r"))])]
-  "TARGET_STRING && XVECLEN (operands[0], 0) == 9"
-  "stswi %2,%1,%O0"
-  [(set_attr "type" "store")
-   (set_attr "update" "yes")
-   (set_attr "indexed" "yes")
-   (set_attr "cell_micro" "always")])
-
-(define_insn "*stmsi7"
-  [(match_parallel 0 "store_multiple_operation"
-    [(set (mem:SI (match_operand:SI 1 "gpc_reg_operand" "b"))
-         (match_operand:SI 2 "gpc_reg_operand" "r"))
-     (clobber (match_scratch:SI 3 "=X"))
-     (set (mem:SI (plus:SI (match_dup 1) (const_int 4)))
-         (match_operand:SI 4 "gpc_reg_operand" "r"))
-     (set (mem:SI (plus:SI (match_dup 1) (const_int 8)))
-         (match_operand:SI 5 "gpc_reg_operand" "r"))
-     (set (mem:SI (plus:SI (match_dup 1) (const_int 12)))
-         (match_operand:SI 6 "gpc_reg_operand" "r"))
-     (set (mem:SI (plus:SI (match_dup 1) (const_int 16)))
-         (match_operand:SI 7 "gpc_reg_operand" "r"))
-     (set (mem:SI (plus:SI (match_dup 1) (const_int 20)))
-         (match_operand:SI 8 "gpc_reg_operand" "r"))
-     (set (mem:SI (plus:SI (match_dup 1) (const_int 24)))
-         (match_operand:SI 9 "gpc_reg_operand" "r"))])]
-  "TARGET_STRING && XVECLEN (operands[0], 0) == 8"
-  "stswi %2,%1,%O0"
-  [(set_attr "type" "store")
-   (set_attr "update" "yes")
-   (set_attr "indexed" "yes")
-   (set_attr "cell_micro" "always")])
-
-(define_insn "*stmsi6"
-  [(match_parallel 0 "store_multiple_operation"
-    [(set (mem:SI (match_operand:SI 1 "gpc_reg_operand" "b"))
-         (match_operand:SI 2 "gpc_reg_operand" "r"))
-     (clobber (match_scratch:SI 3 "=X"))
-     (set (mem:SI (plus:SI (match_dup 1) (const_int 4)))
-         (match_operand:SI 4 "gpc_reg_operand" "r"))
-     (set (mem:SI (plus:SI (match_dup 1) (const_int 8)))
-         (match_operand:SI 5 "gpc_reg_operand" "r"))
-     (set (mem:SI (plus:SI (match_dup 1) (const_int 12)))
-         (match_operand:SI 6 "gpc_reg_operand" "r"))
-     (set (mem:SI (plus:SI (match_dup 1) (const_int 16)))
-         (match_operand:SI 7 "gpc_reg_operand" "r"))
-     (set (mem:SI (plus:SI (match_dup 1) (const_int 20)))
-         (match_operand:SI 8 "gpc_reg_operand" "r"))])]
-  "TARGET_STRING && XVECLEN (operands[0], 0) == 7"
-  "stswi %2,%1,%O0"
-  [(set_attr "type" "store")
-   (set_attr "update" "yes")
-   (set_attr "indexed" "yes")
-   (set_attr "cell_micro" "always")])
-
-(define_insn "*stmsi5"
-  [(match_parallel 0 "store_multiple_operation"
-    [(set (mem:SI (match_operand:SI 1 "gpc_reg_operand" "b"))
-         (match_operand:SI 2 "gpc_reg_operand" "r"))
-     (clobber (match_scratch:SI 3 "=X"))
-     (set (mem:SI (plus:SI (match_dup 1) (const_int 4)))
-         (match_operand:SI 4 "gpc_reg_operand" "r"))
-     (set (mem:SI (plus:SI (match_dup 1) (const_int 8)))
-         (match_operand:SI 5 "gpc_reg_operand" "r"))
-     (set (mem:SI (plus:SI (match_dup 1) (const_int 12)))
-         (match_operand:SI 6 "gpc_reg_operand" "r"))
-     (set (mem:SI (plus:SI (match_dup 1) (const_int 16)))
-         (match_operand:SI 7 "gpc_reg_operand" "r"))])]
-  "TARGET_STRING && XVECLEN (operands[0], 0) == 6"
-  "stswi %2,%1,%O0"
-  [(set_attr "type" "store")
-   (set_attr "update" "yes")
-   (set_attr "indexed" "yes")
-   (set_attr "cell_micro" "always")])
-
-(define_insn "*stmsi4"
-  [(match_parallel 0 "store_multiple_operation"
-    [(set (mem:SI (match_operand:SI 1 "gpc_reg_operand" "b"))
-         (match_operand:SI 2 "gpc_reg_operand" "r"))
-     (clobber (match_scratch:SI 3 "=X"))
-     (set (mem:SI (plus:SI (match_dup 1) (const_int 4)))
-         (match_operand:SI 4 "gpc_reg_operand" "r"))
-     (set (mem:SI (plus:SI (match_dup 1) (const_int 8)))
-         (match_operand:SI 5 "gpc_reg_operand" "r"))
-     (set (mem:SI (plus:SI (match_dup 1) (const_int 12)))
-         (match_operand:SI 6 "gpc_reg_operand" "r"))])]
-  "TARGET_STRING && XVECLEN (operands[0], 0) == 5"
-  "stswi %2,%1,%O0"
-  [(set_attr "type" "store")
-   (set_attr "update" "yes")
-   (set_attr "indexed" "yes")
-   (set_attr "cell_micro" "always")])
-
-(define_insn "*stmsi3"
-  [(match_parallel 0 "store_multiple_operation"
-    [(set (mem:SI (match_operand:SI 1 "gpc_reg_operand" "b"))
-         (match_operand:SI 2 "gpc_reg_operand" "r"))
-     (clobber (match_scratch:SI 3 "=X"))
-     (set (mem:SI (plus:SI (match_dup 1) (const_int 4)))
-         (match_operand:SI 4 "gpc_reg_operand" "r"))
-     (set (mem:SI (plus:SI (match_dup 1) (const_int 8)))
-         (match_operand:SI 5 "gpc_reg_operand" "r"))])]
-  "TARGET_STRING && XVECLEN (operands[0], 0) == 4"
-  "stswi %2,%1,%O0"
-  [(set_attr "type" "store")
-   (set_attr "update" "yes")
-   (set_attr "indexed" "yes")
-   (set_attr "cell_micro" "always")])
-\f
-(define_expand "setmemsi"
-  [(parallel [(set (match_operand:BLK 0 "" "")
-                  (match_operand 2 "const_int_operand" ""))
-             (use (match_operand:SI 1 "" ""))
-             (use (match_operand:SI 3 "" ""))])]
-  ""
-  "
-{
-  /* If value to set is not zero, use the library routine.  */
-  if (operands[2] != const0_rtx)
-    FAIL;
-
-  if (expand_block_clear (operands))
-    DONE;
-  else
-    FAIL;
-}")
-
-;; String/block move insn.
-;; Argument 0 is the destination
-;; Argument 1 is the source
-;; Argument 2 is the length
-;; Argument 3 is the alignment
-
-(define_expand "movmemsi"
-  [(parallel [(set (match_operand:BLK 0 "" "")
-                  (match_operand:BLK 1 "" ""))
-             (use (match_operand:SI 2 "" ""))
-             (use (match_operand:SI 3 "" ""))])]
-  ""
-  "
-{
-  if (expand_block_move (operands))
-    DONE;
-  else
-    FAIL;
-}")
-
-;; Move up to 32 bytes at a time.  The fixed registers are needed because the
-;; register allocator doesn't have a clue about allocating 8 word registers.
-;; rD/rS = r5 is preferred, efficient form.
-(define_expand "movmemsi_8reg"
-  [(parallel [(set (match_operand 0 "" "")
-                  (match_operand 1 "" ""))
-             (use (match_operand 2 "" ""))
-             (use (match_operand 3 "" ""))
-             (clobber (reg:SI  5))
-             (clobber (reg:SI  6))
-             (clobber (reg:SI  7))
-             (clobber (reg:SI  8))
-             (clobber (reg:SI  9))
-             (clobber (reg:SI 10))
-             (clobber (reg:SI 11))
-             (clobber (reg:SI 12))
-             (clobber (match_scratch:SI 4 ""))])]
-  "TARGET_STRING"
-  "")
-
-(define_insn ""
-  [(set (mem:BLK (match_operand:P 0 "gpc_reg_operand" "b"))
-       (mem:BLK (match_operand:P 1 "gpc_reg_operand" "b")))
-   (use (match_operand:SI 2 "immediate_operand" "i"))
-   (use (match_operand:SI 3 "immediate_operand" "i"))
-   (clobber (match_operand:SI 4 "gpc_reg_operand" "=&r"))
-   (clobber (reg:SI  6))
-   (clobber (reg:SI  7))
-   (clobber (reg:SI  8))
-   (clobber (reg:SI  9))
-   (clobber (reg:SI 10))
-   (clobber (reg:SI 11))
-   (clobber (reg:SI 12))
-   (clobber (match_scratch:SI 5 "=X"))]
-  "TARGET_STRING
-   && ((INTVAL (operands[2]) > 24 && INTVAL (operands[2]) < 32)
-       || INTVAL (operands[2]) == 0)
-   && (REGNO (operands[0]) < 5 || REGNO (operands[0]) > 12)
-   && (REGNO (operands[1]) < 5 || REGNO (operands[1]) > 12)
-   && REGNO (operands[4]) == 5"
-  "lswi %4,%1,%2\;stswi %4,%0,%2"
-  [(set_attr "type" "store")
-   (set_attr "update" "yes")
-   (set_attr "indexed" "yes")
-   (set_attr "cell_micro" "always")
-   (set_attr "length" "8")])
-
-;; Move up to 24 bytes at a time.  The fixed registers are needed because the
-;; register allocator doesn't have a clue about allocating 6 word registers.
-;; rD/rS = r5 is preferred, efficient form.
-(define_expand "movmemsi_6reg"
-  [(parallel [(set (match_operand 0 "" "")
-                  (match_operand 1 "" ""))
-             (use (match_operand 2 "" ""))
-             (use (match_operand 3 "" ""))
-             (clobber (reg:SI  5))
-             (clobber (reg:SI  6))
-             (clobber (reg:SI  7))
-             (clobber (reg:SI  8))
-             (clobber (reg:SI  9))
-             (clobber (reg:SI 10))
-             (clobber (match_scratch:SI 4 ""))])]
-  "TARGET_STRING"
-  "")
-
-(define_insn ""
-  [(set (mem:BLK (match_operand:P 0 "gpc_reg_operand" "b"))
-       (mem:BLK (match_operand:P 1 "gpc_reg_operand" "b")))
-   (use (match_operand:SI 2 "immediate_operand" "i"))
-   (use (match_operand:SI 3 "immediate_operand" "i"))
-   (clobber (match_operand:SI 4 "gpc_reg_operand" "=&r"))
-   (clobber (reg:SI  6))
-   (clobber (reg:SI  7))
-   (clobber (reg:SI  8))
-   (clobber (reg:SI  9))
-   (clobber (reg:SI 10))
-   (clobber (match_scratch:SI 5 "=X"))]
-  "TARGET_STRING
-   && INTVAL (operands[2]) > 16 && INTVAL (operands[2]) <= 32
-   && (REGNO (operands[0]) < 5 || REGNO (operands[0]) > 10)
-   && (REGNO (operands[1]) < 5 || REGNO (operands[1]) > 10)
-   && REGNO (operands[4]) == 5"
-  "lswi %4,%1,%2\;stswi %4,%0,%2"
-  [(set_attr "type" "store")
-   (set_attr "update" "yes")
-   (set_attr "indexed" "yes")
-   (set_attr "cell_micro" "always")
-   (set_attr "length" "8")])
-
-;; Move up to 16 bytes at a time, using 4 fixed registers to avoid spill
-;; problems with TImode.
-;; rD/rS = r5 is preferred, efficient form.
-(define_expand "movmemsi_4reg"
-  [(parallel [(set (match_operand 0 "" "")
-                  (match_operand 1 "" ""))
-             (use (match_operand 2 "" ""))
-             (use (match_operand 3 "" ""))
-             (clobber (reg:SI 5))
-             (clobber (reg:SI 6))
-             (clobber (reg:SI 7))
-             (clobber (reg:SI 8))
-             (clobber (match_scratch:SI 4 ""))])]
-  "TARGET_STRING"
-  "")
-
-(define_insn ""
-  [(set (mem:BLK (match_operand:P 0 "gpc_reg_operand" "b"))
-       (mem:BLK (match_operand:P 1 "gpc_reg_operand" "b")))
-   (use (match_operand:SI 2 "immediate_operand" "i"))
-   (use (match_operand:SI 3 "immediate_operand" "i"))
-   (clobber (match_operand:SI 4 "gpc_reg_operand" "=&r"))
-   (clobber (reg:SI 6))
-   (clobber (reg:SI 7))
-   (clobber (reg:SI 8))
-   (clobber (match_scratch:SI 5 "=X"))]
-  "TARGET_STRING
-   && INTVAL (operands[2]) > 8 && INTVAL (operands[2]) <= 16
-   && (REGNO (operands[0]) < 5 || REGNO (operands[0]) > 8)
-   && (REGNO (operands[1]) < 5 || REGNO (operands[1]) > 8)
-   && REGNO (operands[4]) == 5"
-  "lswi %4,%1,%2\;stswi %4,%0,%2"
-  [(set_attr "type" "store")
-   (set_attr "update" "yes")
-   (set_attr "indexed" "yes")
-   (set_attr "cell_micro" "always")
-   (set_attr "length" "8")])
-
-;; Move up to 8 bytes at a time.
-(define_expand "movmemsi_2reg"
-  [(parallel [(set (match_operand 0 "" "")
-                  (match_operand 1 "" ""))
-             (use (match_operand 2 "" ""))
-             (use (match_operand 3 "" ""))
-             (clobber (match_scratch:DI 4 ""))
-             (clobber (match_scratch:SI 5 ""))])]
-  "TARGET_STRING && ! TARGET_POWERPC64"
-  "")
-
-(define_insn ""
-  [(set (mem:BLK (match_operand:SI 0 "gpc_reg_operand" "b"))
-       (mem:BLK (match_operand:SI 1 "gpc_reg_operand" "b")))
-   (use (match_operand:SI 2 "immediate_operand" "i"))
-   (use (match_operand:SI 3 "immediate_operand" "i"))
-   (clobber (match_scratch:DI 4 "=&r"))
-   (clobber (match_scratch:SI 5 "=X"))]
-  "TARGET_STRING && ! TARGET_POWERPC64
-   && INTVAL (operands[2]) > 4 && INTVAL (operands[2]) <= 8"
-  "lswi %4,%1,%2\;stswi %4,%0,%2"
-  [(set_attr "type" "store")
-   (set_attr "update" "yes")
-   (set_attr "indexed" "yes")
-   (set_attr "cell_micro" "always")
-   (set_attr "length" "8")])
-
-;; Move up to 4 bytes at a time.
-(define_expand "movmemsi_1reg"
-  [(parallel [(set (match_operand 0 "" "")
-                  (match_operand 1 "" ""))
-             (use (match_operand 2 "" ""))
-             (use (match_operand 3 "" ""))
-             (clobber (match_scratch:SI 4 ""))
-             (clobber (match_scratch:SI 5 ""))])]
-  "TARGET_STRING"
-  "")
-
-(define_insn ""
-  [(set (mem:BLK (match_operand:P 0 "gpc_reg_operand" "b"))
-       (mem:BLK (match_operand:P 1 "gpc_reg_operand" "b")))
-   (use (match_operand:SI 2 "immediate_operand" "i"))
-   (use (match_operand:SI 3 "immediate_operand" "i"))
-   (clobber (match_scratch:SI 4 "=&r"))
-   (clobber (match_scratch:SI 5 "=X"))]
-  "TARGET_STRING && INTVAL (operands[2]) > 0 && INTVAL (operands[2]) <= 4"
-  "lswi %4,%1,%2\;stswi %4,%0,%2"
-  [(set_attr "type" "store")
-   (set_attr "update" "yes")
-   (set_attr "indexed" "yes")
-   (set_attr "cell_micro" "always")
-   (set_attr "length" "8")])
-\f
-;; Define insns that do load or store with update.  Some of these we can
-;; get by using pre-decrement or pre-increment, but the hardware can also
-;; do cases where the increment is not the size of the object.
-;;
-;; In all these cases, we use operands 0 and 1 for the register being
-;; incremented because those are the operands that local-alloc will
-;; tie and these are the pair most likely to be tieable (and the ones
-;; that will benefit the most).
-
-(define_insn "*movdi_update1"
-  [(set (match_operand:DI 3 "gpc_reg_operand" "=r,r")
-       (mem:DI (plus:DI (match_operand:DI 1 "gpc_reg_operand" "0,0")
-                        (match_operand:DI 2 "reg_or_aligned_short_operand" "r,I"))))
-   (set (match_operand:DI 0 "gpc_reg_operand" "=b,b")
-       (plus:DI (match_dup 1) (match_dup 2)))]
-  "TARGET_POWERPC64 && TARGET_UPDATE
-   && (!avoiding_indexed_address_p (DImode)
-       || !gpc_reg_operand (operands[2], DImode))"
-  "@
-   ldux %3,%0,%2
-   ldu %3,%2(%0)"
-  [(set_attr "type" "load")
-   (set_attr "update" "yes")
-   (set_attr "indexed" "yes,no")])
-
-(define_insn "movdi_<mode>_update"
-  [(set (mem:DI (plus:P (match_operand:P 1 "gpc_reg_operand" "0,0")
-                        (match_operand:P 2 "reg_or_aligned_short_operand" "r,I")))
-       (match_operand:DI 3 "gpc_reg_operand" "r,r"))
-   (set (match_operand:P 0 "gpc_reg_operand" "=b,b")
-       (plus:P (match_dup 1) (match_dup 2)))]
-  "TARGET_POWERPC64 && TARGET_UPDATE
-   && (!avoiding_indexed_address_p (Pmode)
-       || !gpc_reg_operand (operands[2], Pmode)
-       || (REG_P (operands[0])
-          && REGNO (operands[0]) == STACK_POINTER_REGNUM))"
-  "@
-   stdux %3,%0,%2
-   stdu %3,%2(%0)"
-  [(set_attr "type" "store")
-   (set_attr "update" "yes")
-   (set_attr "indexed" "yes,no")])
-
-;; This pattern is only conditional on TARGET_POWERPC64, as it is
-;; needed for stack allocation, even if the user passes -mno-update.
-(define_insn "movdi_<mode>_update_stack"
-  [(set (mem:DI (plus:P (match_operand:P 1 "gpc_reg_operand" "0,0")
-                        (match_operand:P 2 "reg_or_aligned_short_operand" "r,I")))
-       (match_operand:DI 3 "gpc_reg_operand" "r,r"))
-   (set (match_operand:P 0 "gpc_reg_operand" "=b,b")
-       (plus:P (match_dup 1) (match_dup 2)))]
-  "TARGET_POWERPC64"
-  "@
-   stdux %3,%0,%2
-   stdu %3,%2(%0)"
-  [(set_attr "type" "store")
-   (set_attr "update" "yes")
-   (set_attr "indexed" "yes,no")])
-
-(define_insn "*movsi_update1"
-  [(set (match_operand:SI 3 "gpc_reg_operand" "=r,r")
-       (mem:SI (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
-                        (match_operand:SI 2 "reg_or_short_operand" "r,I"))))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
-       (plus:SI (match_dup 1) (match_dup 2)))]
-  "TARGET_UPDATE
-   && (!avoiding_indexed_address_p (SImode)
-       || !gpc_reg_operand (operands[2], SImode))"
-  "@
-   lwzux %3,%0,%2
-   lwzu %3,%2(%0)"
-  [(set_attr "type" "load")
-   (set_attr "update" "yes")
-   (set_attr "indexed" "yes,no")])
-
-(define_insn "*movsi_update2"
-  [(set (match_operand:DI 3 "gpc_reg_operand" "=r")
-       (sign_extend:DI
-        (mem:SI (plus:DI (match_operand:DI 1 "gpc_reg_operand" "0")
-                         (match_operand:DI 2 "gpc_reg_operand" "r")))))
-   (set (match_operand:DI 0 "gpc_reg_operand" "=b")
-       (plus:DI (match_dup 1) (match_dup 2)))]
-  "TARGET_POWERPC64 && rs6000_gen_cell_microcode
-   && !avoiding_indexed_address_p (DImode)"
-  "lwaux %3,%0,%2"
-  [(set_attr "type" "load")
-   (set_attr "sign_extend" "yes")
-   (set_attr "update" "yes")
-   (set_attr "indexed" "yes")])
-
-(define_insn "movsi_update"
-  [(set (mem:SI (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
-                        (match_operand:SI 2 "reg_or_short_operand" "r,I")))
-       (match_operand:SI 3 "gpc_reg_operand" "r,r"))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
-       (plus:SI (match_dup 1) (match_dup 2)))]
-  "TARGET_UPDATE
-   && (!avoiding_indexed_address_p (SImode)
-       || !gpc_reg_operand (operands[2], SImode)
-       || (REG_P (operands[0])
-          && REGNO (operands[0]) == STACK_POINTER_REGNUM))"
-  "@
-   stwux %3,%0,%2
-   stwu %3,%2(%0)"
-  [(set_attr "type" "store")
-   (set_attr "update" "yes")
-   (set_attr "indexed" "yes,no")])
-
-;; This is an unconditional pattern; needed for stack allocation, even
-;; if the user passes -mno-update.
-(define_insn "movsi_update_stack"
-  [(set (mem:SI (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
-                        (match_operand:SI 2 "reg_or_short_operand" "r,I")))
-       (match_operand:SI 3 "gpc_reg_operand" "r,r"))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
-       (plus:SI (match_dup 1) (match_dup 2)))]
-  ""
-  "@
-   stwux %3,%0,%2
-   stwu %3,%2(%0)"
-  [(set_attr "type" "store")
-   (set_attr "update" "yes")
-   (set_attr "indexed" "yes,no")])
-
-(define_insn "*movhi_update1"
-  [(set (match_operand:HI 3 "gpc_reg_operand" "=r,r")
-       (mem:HI (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
-                        (match_operand:SI 2 "reg_or_short_operand" "r,I"))))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
-       (plus:SI (match_dup 1) (match_dup 2)))]
-  "TARGET_UPDATE
-   && (!avoiding_indexed_address_p (SImode)
-       || !gpc_reg_operand (operands[2], SImode))"
-  "@
-   lhzux %3,%0,%2
-   lhzu %3,%2(%0)"
-  [(set_attr "type" "load")
-   (set_attr "update" "yes")
-   (set_attr "indexed" "yes,no")])
-
-(define_insn "*movhi_update2"
-  [(set (match_operand:SI 3 "gpc_reg_operand" "=r,r")
-       (zero_extend:SI
-        (mem:HI (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
-                         (match_operand:SI 2 "reg_or_short_operand" "r,I")))))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
-       (plus:SI (match_dup 1) (match_dup 2)))]
-  "TARGET_UPDATE
-   && (!avoiding_indexed_address_p (SImode)
-       || !gpc_reg_operand (operands[2], SImode))"
-  "@
-   lhzux %3,%0,%2
-   lhzu %3,%2(%0)"
-  [(set_attr "type" "load")
-   (set_attr "update" "yes")
-   (set_attr "indexed" "yes,no")])
-
-(define_insn "*movhi_update3"
-  [(set (match_operand:SI 3 "gpc_reg_operand" "=r,r")
-       (sign_extend:SI
-        (mem:HI (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
-                         (match_operand:SI 2 "reg_or_short_operand" "r,I")))))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
-       (plus:SI (match_dup 1) (match_dup 2)))]
-  "TARGET_UPDATE && rs6000_gen_cell_microcode
-   && (!avoiding_indexed_address_p (SImode)
-       || !gpc_reg_operand (operands[2], SImode))"
-  "@
-   lhaux %3,%0,%2
-   lhau %3,%2(%0)"
-  [(set_attr "type" "load")
-   (set_attr "sign_extend" "yes")
-   (set_attr "update" "yes")
-   (set_attr "indexed" "yes,no")])
-
-(define_insn "*movhi_update4"
-  [(set (mem:HI (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
-                        (match_operand:SI 2 "reg_or_short_operand" "r,I")))
-       (match_operand:HI 3 "gpc_reg_operand" "r,r"))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
-       (plus:SI (match_dup 1) (match_dup 2)))]
-  "TARGET_UPDATE
-   && (!avoiding_indexed_address_p (SImode)
-       || !gpc_reg_operand (operands[2], SImode))"
-  "@
-   sthux %3,%0,%2
-   sthu %3,%2(%0)"
-  [(set_attr "type" "store")
-   (set_attr "update" "yes")
-   (set_attr "indexed" "yes,no")])
-
-(define_insn "*movqi_update1"
-  [(set (match_operand:QI 3 "gpc_reg_operand" "=r,r")
-       (mem:QI (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
-                        (match_operand:SI 2 "reg_or_short_operand" "r,I"))))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
-       (plus:SI (match_dup 1) (match_dup 2)))]
-  "TARGET_UPDATE
-   && (!avoiding_indexed_address_p (SImode)
-       || !gpc_reg_operand (operands[2], SImode))"
-  "@
-   lbzux %3,%0,%2
-   lbzu %3,%2(%0)"
-  [(set_attr "type" "load")
-   (set_attr "update" "yes")
-   (set_attr "indexed" "yes,no")])
-
-(define_insn "*movqi_update2"
-  [(set (match_operand:SI 3 "gpc_reg_operand" "=r,r")
-       (zero_extend:SI
-        (mem:QI (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
-                         (match_operand:SI 2 "reg_or_short_operand" "r,I")))))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
-       (plus:SI (match_dup 1) (match_dup 2)))]
-  "TARGET_UPDATE
-   && (!avoiding_indexed_address_p (SImode)
-       || !gpc_reg_operand (operands[2], SImode))"
-  "@
-   lbzux %3,%0,%2
-   lbzu %3,%2(%0)"
-  [(set_attr "type" "load")
-   (set_attr "update" "yes")
-   (set_attr "indexed" "yes,no")])
-
-(define_insn "*movqi_update3"
-  [(set (mem:QI (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
-                        (match_operand:SI 2 "reg_or_short_operand" "r,I")))
-       (match_operand:QI 3 "gpc_reg_operand" "r,r"))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
-       (plus:SI (match_dup 1) (match_dup 2)))]
-  "TARGET_UPDATE
-   && (!avoiding_indexed_address_p (SImode)
-       || !gpc_reg_operand (operands[2], SImode))"
-  "@
-   stbux %3,%0,%2
-   stbu %3,%2(%0)"
-  [(set_attr "type" "store")
-   (set_attr "update" "yes")
-   (set_attr "indexed" "yes,no")])
-
-(define_insn "*movsf_update1"
-  [(set (match_operand:SF 3 "gpc_reg_operand" "=f,f")
-       (mem:SF (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
-                        (match_operand:SI 2 "reg_or_short_operand" "r,I"))))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
-       (plus:SI (match_dup 1) (match_dup 2)))]
-  "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_SINGLE_FLOAT && TARGET_UPDATE
-   && (!avoiding_indexed_address_p (SImode)
-       || !gpc_reg_operand (operands[2], SImode))"
-  "@
-   lfsux %3,%0,%2
-   lfsu %3,%2(%0)"
-  [(set_attr "type" "fpload")
-   (set_attr "update" "yes")
-   (set_attr "indexed" "yes,no")])
-
-(define_insn "*movsf_update2"
-  [(set (mem:SF (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
-                        (match_operand:SI 2 "reg_or_short_operand" "r,I")))
-       (match_operand:SF 3 "gpc_reg_operand" "f,f"))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
-       (plus:SI (match_dup 1) (match_dup 2)))]
-  "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_SINGLE_FLOAT && TARGET_UPDATE
-   && (!avoiding_indexed_address_p (SImode)
-       || !gpc_reg_operand (operands[2], SImode))"
-  "@
-   stfsux %3,%0,%2
-   stfsu %3,%2(%0)"
-  [(set_attr "type" "fpstore")
-   (set_attr "update" "yes")
-   (set_attr "indexed" "yes,no")])
-
-(define_insn "*movsf_update3"
-  [(set (match_operand:SF 3 "gpc_reg_operand" "=r,r")
-       (mem:SF (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
-                        (match_operand:SI 2 "reg_or_short_operand" "r,I"))))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
-       (plus:SI (match_dup 1) (match_dup 2)))]
-  "(TARGET_SOFT_FLOAT || !TARGET_FPRS) && TARGET_UPDATE
-   && (!avoiding_indexed_address_p (SImode)
-       || !gpc_reg_operand (operands[2], SImode))"
-  "@
-   lwzux %3,%0,%2
-   lwzu %3,%2(%0)"
-  [(set_attr "type" "load")
-   (set_attr "update" "yes")
-   (set_attr "indexed" "yes,no")])
-
-(define_insn "*movsf_update4"
-  [(set (mem:SF (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
-                        (match_operand:SI 2 "reg_or_short_operand" "r,I")))
-       (match_operand:SF 3 "gpc_reg_operand" "r,r"))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
-       (plus:SI (match_dup 1) (match_dup 2)))]
-  "(TARGET_SOFT_FLOAT || !TARGET_FPRS) && TARGET_UPDATE
-   && (!avoiding_indexed_address_p (SImode)
-       || !gpc_reg_operand (operands[2], SImode))"
-  "@
-   stwux %3,%0,%2
-   stwu %3,%2(%0)"
-  [(set_attr "type" "store")
-   (set_attr "update" "yes")
-   (set_attr "indexed" "yes,no")])
-
-(define_insn "*movdf_update1"
-  [(set (match_operand:DF 3 "gpc_reg_operand" "=d,d")
-       (mem:DF (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
-                        (match_operand:SI 2 "reg_or_short_operand" "r,I"))))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
-       (plus:SI (match_dup 1) (match_dup 2)))]
-  "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT && TARGET_UPDATE
-   && (!avoiding_indexed_address_p (SImode)
-       || !gpc_reg_operand (operands[2], SImode))"
-  "@
-   lfdux %3,%0,%2
-   lfdu %3,%2(%0)"
-  [(set_attr "type" "fpload")
-   (set_attr "update" "yes")
-   (set_attr "indexed" "yes,no")])
-
-(define_insn "*movdf_update2"
-  [(set (mem:DF (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
-                        (match_operand:SI 2 "reg_or_short_operand" "r,I")))
-       (match_operand:DF 3 "gpc_reg_operand" "d,d"))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
-       (plus:SI (match_dup 1) (match_dup 2)))]
-  "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT && TARGET_UPDATE
-   && (!avoiding_indexed_address_p (SImode)
-       || !gpc_reg_operand (operands[2], SImode))"
-  "@
-   stfdux %3,%0,%2
-   stfdu %3,%2(%0)"
-  [(set_attr "type" "fpstore")
-   (set_attr "update" "yes")
-   (set_attr "indexed" "yes,no")])
-
-
-;; After inserting conditional returns we can sometimes have
-;; unnecessary register moves.  Unfortunately we cannot have a
-;; modeless peephole here, because some single SImode sets have early
-;; clobber outputs.  Although those sets expand to multi-ppc-insn
-;; sequences, using get_attr_length here will smash the operands
-;; array.  Neither is there an early_cobbler_p predicate.
-;; Disallow subregs for E500 so we don't munge frob_di_df_2.
-(define_peephole2
-  [(set (match_operand:DF 0 "gpc_reg_operand" "")
-       (match_operand:DF 1 "any_operand" ""))
-   (set (match_operand:DF 2 "gpc_reg_operand" "")
-       (match_dup 0))]
-  "!(TARGET_E500_DOUBLE && GET_CODE (operands[2]) == SUBREG)
-   && peep2_reg_dead_p (2, operands[0])"
-  [(set (match_dup 2) (match_dup 1))])
-
-(define_peephole2
-  [(set (match_operand:SF 0 "gpc_reg_operand" "")
-       (match_operand:SF 1 "any_operand" ""))
-   (set (match_operand:SF 2 "gpc_reg_operand" "")
-       (match_dup 0))]
-  "peep2_reg_dead_p (2, operands[0])"
-  [(set (match_dup 2) (match_dup 1))])
-
-\f
-;; TLS support.
-
-;; Mode attributes for different ABIs.
-(define_mode_iterator TLSmode [(SI "! TARGET_64BIT") (DI "TARGET_64BIT")])
-(define_mode_attr tls_abi_suffix [(SI "32") (DI "64")])
-(define_mode_attr tls_sysv_suffix [(SI "si") (DI "di")])
-(define_mode_attr tls_insn_suffix [(SI "wz") (DI "d")])
-
-(define_insn_and_split "tls_gd_aix<TLSmode:tls_abi_suffix>"
-  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
-        (call (mem:TLSmode (match_operand:TLSmode 3 "symbol_ref_operand" "s"))
-             (match_operand 4 "" "g")))
-   (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
-                   (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
-                  UNSPEC_TLSGD)
-   (clobber (reg:SI LR_REGNO))]
-  "HAVE_AS_TLS && (DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2)"
-{
-  if (TARGET_CMODEL != CMODEL_SMALL)
-    return "addis %0,%1,%2@got@tlsgd@ha\;addi %0,%0,%2@got@tlsgd@l\;"
-          "bl %z3\;nop";
-  else
-    return "addi %0,%1,%2@got@tlsgd\;bl %z3\;nop";
-}
-  "&& TARGET_TLS_MARKERS"
-  [(set (match_dup 0)
-       (unspec:TLSmode [(match_dup 1)
-                        (match_dup 2)]
-                       UNSPEC_TLSGD))
-   (parallel [(set (match_dup 0)
-                  (call (mem:TLSmode (match_dup 3))
-                        (match_dup 4)))
-             (unspec:TLSmode [(match_dup 2)] UNSPEC_TLSGD)
-             (clobber (reg:SI LR_REGNO))])]
-  ""
-  [(set_attr "type" "two")
-   (set (attr "length")
-     (if_then_else (ne (symbol_ref "TARGET_CMODEL") (symbol_ref "CMODEL_SMALL"))
-                  (const_int 16)
-                  (const_int 12)))])
-
-(define_insn_and_split "tls_gd_sysv<TLSmode:tls_sysv_suffix>"
-  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
-        (call (mem:TLSmode (match_operand:TLSmode 3 "symbol_ref_operand" "s"))
-             (match_operand 4 "" "g")))
-   (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
-                   (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
-                  UNSPEC_TLSGD)
-   (clobber (reg:SI LR_REGNO))]
-  "HAVE_AS_TLS && DEFAULT_ABI == ABI_V4"
-{
-  if (flag_pic)
-    {
-      if (TARGET_SECURE_PLT && flag_pic == 2)
-       return "addi %0,%1,%2@got@tlsgd\;bl %z3+32768@plt";
-      else
-       return "addi %0,%1,%2@got@tlsgd\;bl %z3@plt";
-    }
-  else
-    return "addi %0,%1,%2@got@tlsgd\;bl %z3";
-}
-  "&& TARGET_TLS_MARKERS"
-  [(set (match_dup 0)
-       (unspec:TLSmode [(match_dup 1)
-                        (match_dup 2)]
-                       UNSPEC_TLSGD))
-   (parallel [(set (match_dup 0)
-                  (call (mem:TLSmode (match_dup 3))
-                        (match_dup 4)))
-             (unspec:TLSmode [(match_dup 2)] UNSPEC_TLSGD)
-             (clobber (reg:SI LR_REGNO))])]
-  ""
-  [(set_attr "type" "two")
-   (set_attr "length" "8")])
-
-(define_insn_and_split "*tls_gd<TLSmode:tls_abi_suffix>"
-  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
-       (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
-                        (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
-                       UNSPEC_TLSGD))]
-  "HAVE_AS_TLS && TARGET_TLS_MARKERS"
-  "addi %0,%1,%2@got@tlsgd"
-  "&& TARGET_CMODEL != CMODEL_SMALL"
-  [(set (match_dup 3)
-       (high:TLSmode
-           (unspec:TLSmode [(match_dup 1) (match_dup 2)] UNSPEC_TLSGD)))
-   (set (match_dup 0)
-       (lo_sum:TLSmode (match_dup 3)
-           (unspec:TLSmode [(match_dup 1) (match_dup 2)] UNSPEC_TLSGD)))]
-  "
-{
-  operands[3] = gen_reg_rtx (TARGET_64BIT ? DImode : SImode);
-}"
-  [(set (attr "length")
-     (if_then_else (ne (symbol_ref "TARGET_CMODEL") (symbol_ref "CMODEL_SMALL"))
-                  (const_int 8)
-                  (const_int 4)))])
-
-(define_insn "*tls_gd_high<TLSmode:tls_abi_suffix>"
-  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
-     (high:TLSmode
-       (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
-                       (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
-                      UNSPEC_TLSGD)))]
-  "HAVE_AS_TLS && TARGET_TLS_MARKERS && TARGET_CMODEL != CMODEL_SMALL"
-  "addis %0,%1,%2@got@tlsgd@ha"
-  [(set_attr "length" "4")])
-
-(define_insn "*tls_gd_low<TLSmode:tls_abi_suffix>"
-  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
-     (lo_sum:TLSmode (match_operand:TLSmode 1 "gpc_reg_operand" "b")
-       (unspec:TLSmode [(match_operand:TLSmode 3 "gpc_reg_operand" "b")
-                       (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
-                      UNSPEC_TLSGD)))]
-  "HAVE_AS_TLS && TARGET_TLS_MARKERS && TARGET_CMODEL != CMODEL_SMALL"
-  "addi %0,%1,%2@got@tlsgd@l"
-  [(set_attr "length" "4")])
-
-(define_insn "*tls_gd_call_aix<TLSmode:tls_abi_suffix>"
-  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
-        (call (mem:TLSmode (match_operand:TLSmode 1 "symbol_ref_operand" "s"))
-             (match_operand 2 "" "g")))
-   (unspec:TLSmode [(match_operand:TLSmode 3 "rs6000_tls_symbol_ref" "")]
-                  UNSPEC_TLSGD)
-   (clobber (reg:SI LR_REGNO))]
-  "HAVE_AS_TLS && TARGET_TLS_MARKERS
-   && (DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2)"
-  "bl %z1(%3@tlsgd)\;nop"
-  [(set_attr "type" "branch")
-   (set_attr "length" "8")])
-
-(define_insn "*tls_gd_call_sysv<TLSmode:tls_abi_suffix>"
-  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
-        (call (mem:TLSmode (match_operand:TLSmode 1 "symbol_ref_operand" "s"))
-             (match_operand 2 "" "g")))
-   (unspec:TLSmode [(match_operand:TLSmode 3 "rs6000_tls_symbol_ref" "")]
-                  UNSPEC_TLSGD)
-   (clobber (reg:SI LR_REGNO))]
-  "HAVE_AS_TLS && DEFAULT_ABI == ABI_V4 && TARGET_TLS_MARKERS"
-{
-  if (flag_pic)
-    {
-      if (TARGET_SECURE_PLT && flag_pic == 2)
-       return "bl %z1+32768(%3@tlsgd)@plt";
-      return "bl %z1(%3@tlsgd)@plt";
-    }
-  return "bl %z1(%3@tlsgd)";
-}
-  [(set_attr "type" "branch")
-   (set_attr "length" "4")])
-
-(define_insn_and_split "tls_ld_aix<TLSmode:tls_abi_suffix>"
-  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
-        (call (mem:TLSmode (match_operand:TLSmode 2 "symbol_ref_operand" "s"))
-             (match_operand 3 "" "g")))
-   (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")]
-                  UNSPEC_TLSLD)
-   (clobber (reg:SI LR_REGNO))]
-  "HAVE_AS_TLS && (DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2)"
-{
-  if (TARGET_CMODEL != CMODEL_SMALL)
-    return "addis %0,%1,%&@got@tlsld@ha\;addi %0,%0,%&@got@tlsld@l\;"
-          "bl %z2\;nop";
-  else
-    return "addi %0,%1,%&@got@tlsld\;bl %z2\;nop";
-}
-  "&& TARGET_TLS_MARKERS"
-  [(set (match_dup 0)
-       (unspec:TLSmode [(match_dup 1)]
-                       UNSPEC_TLSLD))
-   (parallel [(set (match_dup 0)
-                  (call (mem:TLSmode (match_dup 2))
-                        (match_dup 3)))
-             (unspec:TLSmode [(const_int 0)] UNSPEC_TLSLD)
-             (clobber (reg:SI LR_REGNO))])]
-  ""
-  [(set_attr "type" "two")
-   (set (attr "length")
-     (if_then_else (ne (symbol_ref "TARGET_CMODEL") (symbol_ref "CMODEL_SMALL"))
-                  (const_int 16)
-                  (const_int 12)))])
-
-(define_insn_and_split "tls_ld_sysv<TLSmode:tls_sysv_suffix>"
-  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
-        (call (mem:TLSmode (match_operand:TLSmode 2 "symbol_ref_operand" "s"))
-             (match_operand 3 "" "g")))
-   (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")]
-                  UNSPEC_TLSLD)
-   (clobber (reg:SI LR_REGNO))]
-  "HAVE_AS_TLS && DEFAULT_ABI == ABI_V4"
-{
-  if (flag_pic)
-    {
-      if (TARGET_SECURE_PLT && flag_pic == 2)
-       return "addi %0,%1,%&@got@tlsld\;bl %z2+32768@plt";
-      else
-       return "addi %0,%1,%&@got@tlsld\;bl %z2@plt";
-    }
-  else
-    return "addi %0,%1,%&@got@tlsld\;bl %z2";
-}
-  "&& TARGET_TLS_MARKERS"
-  [(set (match_dup 0)
-       (unspec:TLSmode [(match_dup 1)]
-                       UNSPEC_TLSLD))
-   (parallel [(set (match_dup 0)
-                  (call (mem:TLSmode (match_dup 2))
-                        (match_dup 3)))
-             (unspec:TLSmode [(const_int 0)] UNSPEC_TLSLD)
-             (clobber (reg:SI LR_REGNO))])]
-  ""
-  [(set_attr "length" "8")])
-
-(define_insn_and_split "*tls_ld<TLSmode:tls_abi_suffix>"
-  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
-       (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")]
-                       UNSPEC_TLSLD))]
-  "HAVE_AS_TLS && TARGET_TLS_MARKERS"
-  "addi %0,%1,%&@got@tlsld"
-  "&& TARGET_CMODEL != CMODEL_SMALL"
-  [(set (match_dup 2)
-       (high:TLSmode
-           (unspec:TLSmode [(const_int 0) (match_dup 1)] UNSPEC_TLSLD)))
-   (set (match_dup 0)
-       (lo_sum:TLSmode (match_dup 2)
-           (unspec:TLSmode [(const_int 0) (match_dup 1)] UNSPEC_TLSLD)))]
-  "
-{
-  operands[2] = gen_reg_rtx (TARGET_64BIT ? DImode : SImode);
-}"
-  [(set (attr "length")
-     (if_then_else (ne (symbol_ref "TARGET_CMODEL") (symbol_ref "CMODEL_SMALL"))
-                  (const_int 8)
-                  (const_int 4)))])
-
-(define_insn "*tls_ld_high<TLSmode:tls_abi_suffix>"
-  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
-     (high:TLSmode
-       (unspec:TLSmode [(const_int 0)
-                       (match_operand:TLSmode 1 "gpc_reg_operand" "b")]
-                      UNSPEC_TLSLD)))]
-  "HAVE_AS_TLS && TARGET_TLS_MARKERS && TARGET_CMODEL != CMODEL_SMALL"
-  "addis %0,%1,%&@got@tlsld@ha"
-  [(set_attr "length" "4")])
-
-(define_insn "*tls_ld_low<TLSmode:tls_abi_suffix>"
-  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
-     (lo_sum:TLSmode (match_operand:TLSmode 1 "gpc_reg_operand" "b")
-       (unspec:TLSmode [(const_int 0)
-                        (match_operand:TLSmode 2 "gpc_reg_operand" "b")]
-                       UNSPEC_TLSLD)))]
-  "HAVE_AS_TLS && TARGET_TLS_MARKERS && TARGET_CMODEL != CMODEL_SMALL"
-  "addi %0,%1,%&@got@tlsld@l"
-  [(set_attr "length" "4")])
-
-(define_insn "*tls_ld_call_aix<TLSmode:tls_abi_suffix>"
-  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
-        (call (mem:TLSmode (match_operand:TLSmode 1 "symbol_ref_operand" "s"))
-             (match_operand 2 "" "g")))
-   (unspec:TLSmode [(const_int 0)] UNSPEC_TLSLD)
-   (clobber (reg:SI LR_REGNO))]
-  "HAVE_AS_TLS && TARGET_TLS_MARKERS
-   && (DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2)"
-  "bl %z1(%&@tlsld)\;nop"
-  [(set_attr "type" "branch")
-   (set_attr "length" "8")])
-
-(define_insn "*tls_ld_call_sysv<TLSmode:tls_abi_suffix>"
-  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
-        (call (mem:TLSmode (match_operand:TLSmode 1 "symbol_ref_operand" "s"))
-             (match_operand 2 "" "g")))
-   (unspec:TLSmode [(const_int 0)] UNSPEC_TLSLD)
-   (clobber (reg:SI LR_REGNO))]
-  "HAVE_AS_TLS && DEFAULT_ABI == ABI_V4 && TARGET_TLS_MARKERS"
-{
-  if (flag_pic)
-    {
-      if (TARGET_SECURE_PLT && flag_pic == 2)
-       return "bl %z1+32768(%&@tlsld)@plt";
-      return "bl %z1(%&@tlsld)@plt";
-    }
-  return "bl %z1(%&@tlsld)";
-}
-  [(set_attr "type" "branch")
-   (set_attr "length" "4")])
-
-(define_insn "tls_dtprel_<TLSmode:tls_abi_suffix>"
-  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=r")
-       (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
-                        (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
-                       UNSPEC_TLSDTPREL))]
-  "HAVE_AS_TLS"
-  "addi %0,%1,%2@dtprel")
-
-(define_insn "tls_dtprel_ha_<TLSmode:tls_abi_suffix>"
-  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=r")
-       (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
-                        (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
-                       UNSPEC_TLSDTPRELHA))]
-  "HAVE_AS_TLS"
-  "addis %0,%1,%2@dtprel@ha")
-
-(define_insn "tls_dtprel_lo_<TLSmode:tls_abi_suffix>"
-  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=r")
-       (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
-                        (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
-                       UNSPEC_TLSDTPRELLO))]
-  "HAVE_AS_TLS"
-  "addi %0,%1,%2@dtprel@l")
-
-(define_insn_and_split "tls_got_dtprel_<TLSmode:tls_abi_suffix>"
-  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=r")
-       (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
-                        (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
-                       UNSPEC_TLSGOTDTPREL))]
-  "HAVE_AS_TLS"
-  "l<TLSmode:tls_insn_suffix> %0,%2@got@dtprel(%1)"
-  "&& TARGET_CMODEL != CMODEL_SMALL"
-  [(set (match_dup 3)
-       (high:TLSmode
-           (unspec:TLSmode [(match_dup 1) (match_dup 2)] UNSPEC_TLSGOTDTPREL)))
-   (set (match_dup 0)
-       (lo_sum:TLSmode (match_dup 3)
-           (unspec:TLSmode [(match_dup 1) (match_dup 2)] UNSPEC_TLSGOTDTPREL)))]
-  "
-{
-  operands[3] = gen_reg_rtx (TARGET_64BIT ? DImode : SImode);
-}"
-  [(set (attr "length")
-     (if_then_else (ne (symbol_ref "TARGET_CMODEL") (symbol_ref "CMODEL_SMALL"))
-                  (const_int 8)
-                  (const_int 4)))])
-
-(define_insn "*tls_got_dtprel_high<TLSmode:tls_abi_suffix>"
-  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
-     (high:TLSmode
-       (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
-                       (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
-                      UNSPEC_TLSGOTDTPREL)))]
-  "HAVE_AS_TLS && TARGET_CMODEL != CMODEL_SMALL"
-  "addis %0,%1,%2@got@dtprel@ha"
-  [(set_attr "length" "4")])
-
-(define_insn "*tls_got_dtprel_low<TLSmode:tls_abi_suffix>"
-  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=r")
-     (lo_sum:TLSmode (match_operand:TLSmode 1 "gpc_reg_operand" "b")
-        (unspec:TLSmode [(match_operand:TLSmode 3 "gpc_reg_operand" "b")
-                         (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
-                        UNSPEC_TLSGOTDTPREL)))]
-  "HAVE_AS_TLS && TARGET_CMODEL != CMODEL_SMALL"
-  "l<TLSmode:tls_insn_suffix> %0,%2@got@dtprel@l(%1)"
-  [(set_attr "length" "4")])
-
-(define_insn "tls_tprel_<TLSmode:tls_abi_suffix>"
-  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=r")
-       (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
-                        (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
-                       UNSPEC_TLSTPREL))]
-  "HAVE_AS_TLS"
-  "addi %0,%1,%2@tprel")
-
-(define_insn "tls_tprel_ha_<TLSmode:tls_abi_suffix>"
-  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=r")
-       (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
-                        (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
-                       UNSPEC_TLSTPRELHA))]
-  "HAVE_AS_TLS"
-  "addis %0,%1,%2@tprel@ha")
+(define_insn "*ldmsi4"
+  [(match_parallel 0 "load_multiple_operation"
+    [(set (match_operand:SI 2 "gpc_reg_operand" "")
+          (mem:SI (match_operand:SI 1 "gpc_reg_operand" "b")))
+     (set (match_operand:SI 3 "gpc_reg_operand" "")
+          (mem:SI (plus:SI (match_dup 1) (const_int 4))))
+     (set (match_operand:SI 4 "gpc_reg_operand" "")
+          (mem:SI (plus:SI (match_dup 1) (const_int 8))))
+     (set (match_operand:SI 5 "gpc_reg_operand" "")
+          (mem:SI (plus:SI (match_dup 1) (const_int 12))))])]
+  "TARGET_STRING && XVECLEN (operands[0], 0) == 4"
+  "*
+{ return rs6000_output_load_multiple (operands); }"
+  [(set_attr "type" "load")
+   (set_attr "update" "yes")
+   (set_attr "indexed" "yes")
+   (set_attr "length" "32")])
 
-(define_insn "tls_tprel_lo_<TLSmode:tls_abi_suffix>"
-  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=r")
-       (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
-                        (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
-                       UNSPEC_TLSTPRELLO))]
-  "HAVE_AS_TLS"
-  "addi %0,%1,%2@tprel@l")
+(define_insn "*ldmsi3"
+  [(match_parallel 0 "load_multiple_operation"
+    [(set (match_operand:SI 2 "gpc_reg_operand" "")
+          (mem:SI (match_operand:SI 1 "gpc_reg_operand" "b")))
+     (set (match_operand:SI 3 "gpc_reg_operand" "")
+          (mem:SI (plus:SI (match_dup 1) (const_int 4))))
+     (set (match_operand:SI 4 "gpc_reg_operand" "")
+          (mem:SI (plus:SI (match_dup 1) (const_int 8))))])]
+  "TARGET_STRING && XVECLEN (operands[0], 0) == 3"
+  "*
+{ return rs6000_output_load_multiple (operands); }"
+  [(set_attr "type" "load")
+   (set_attr "update" "yes")
+   (set_attr "indexed" "yes")
+   (set_attr "length" "32")])
 
-;; "b" output constraint here and on tls_tls input to support linker tls
-;; optimization.  The linker may edit the instructions emitted by a
-;; tls_got_tprel/tls_tls pair to addis,addi.
-(define_insn_and_split "tls_got_tprel_<TLSmode:tls_abi_suffix>"
-  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
-       (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
-                        (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
-                       UNSPEC_TLSGOTTPREL))]
-  "HAVE_AS_TLS"
-  "l<TLSmode:tls_insn_suffix> %0,%2@got@tprel(%1)"
-  "&& TARGET_CMODEL != CMODEL_SMALL"
-  [(set (match_dup 3)
-       (high:TLSmode
-           (unspec:TLSmode [(match_dup 1) (match_dup 2)] UNSPEC_TLSGOTTPREL)))
-   (set (match_dup 0)
-       (lo_sum:TLSmode (match_dup 3)
-           (unspec:TLSmode [(match_dup 1) (match_dup 2)] UNSPEC_TLSGOTTPREL)))]
+(define_expand "store_multiple"
+  [(match_par_dup 3 [(set (match_operand:SI 0 "" "")
+                         (match_operand:SI 1 "" ""))
+                    (clobber (scratch:SI))
+                    (use (match_operand:SI 2 "" ""))])]
+  "TARGET_STRING && !TARGET_POWERPC64"
   "
 {
-  operands[3] = gen_reg_rtx (TARGET_64BIT ? DImode : SImode);
-}"
-  [(set (attr "length")
-     (if_then_else (ne (symbol_ref "TARGET_CMODEL") (symbol_ref "CMODEL_SMALL"))
-                  (const_int 8)
-                  (const_int 4)))])
+  int regno;
+  int count;
+  rtx to;
+  rtx op0;
+  int i;
 
-(define_insn "*tls_got_tprel_high<TLSmode:tls_abi_suffix>"
-  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
-     (high:TLSmode
-       (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
-                       (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
-                      UNSPEC_TLSGOTTPREL)))]
-  "HAVE_AS_TLS && TARGET_CMODEL != CMODEL_SMALL"
-  "addis %0,%1,%2@got@tprel@ha"
-  [(set_attr "length" "4")])
+  /* Support only storing a constant number of fixed-point registers to
+     memory and only bother with this if more than two; the machine
+     doesn't support more than eight.  */
+  if (GET_CODE (operands[2]) != CONST_INT
+      || INTVAL (operands[2]) <= 2
+      || INTVAL (operands[2]) > 8
+      || GET_CODE (operands[0]) != MEM
+      || GET_CODE (operands[1]) != REG
+      || REGNO (operands[1]) >= 32)
+    FAIL;
 
-(define_insn "*tls_got_tprel_low<TLSmode:tls_abi_suffix>"
-  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=r")
-     (lo_sum:TLSmode (match_operand:TLSmode 1 "gpc_reg_operand" "b")
-        (unspec:TLSmode [(match_operand:TLSmode 3 "gpc_reg_operand" "b")
-                         (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
-                        UNSPEC_TLSGOTTPREL)))]
-  "HAVE_AS_TLS && TARGET_CMODEL != CMODEL_SMALL"
-  "l<TLSmode:tls_insn_suffix> %0,%2@got@tprel@l(%1)"
-  [(set_attr "length" "4")])
+  count = INTVAL (operands[2]);
+  regno = REGNO (operands[1]);
 
-(define_insn "tls_tls_<TLSmode:tls_abi_suffix>"
-  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=r")
-       (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
-                        (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
-                       UNSPEC_TLSTLS))]
-  "TARGET_ELF && HAVE_AS_TLS"
-  "add %0,%1,%2@tls")
+  operands[3] = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (count + 1));
+  to = force_reg (SImode, XEXP (operands[0], 0));
+  op0 = replace_equiv_address (operands[0], to);
 
-(define_expand "tls_get_tpointer"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "")
-       (unspec:SI [(const_int 0)] UNSPEC_TLSTLS))]
-  "TARGET_XCOFF && HAVE_AS_TLS"
-  "
-{
-  emit_insn (gen_tls_get_tpointer_internal ());
-  emit_move_insn (operands[0], gen_rtx_REG (SImode, 3));
-  DONE;
+  XVECEXP (operands[3], 0, 0)
+    = gen_rtx_SET (adjust_address_nv (op0, SImode, 0), operands[1]);
+  XVECEXP (operands[3], 0, 1) = gen_rtx_CLOBBER (VOIDmode,
+                                                gen_rtx_SCRATCH (SImode));
+
+  for (i = 1; i < count; i++)
+    XVECEXP (operands[3], 0, i + 1)
+      = gen_rtx_SET (adjust_address_nv (op0, SImode, i * 4),
+                    gen_rtx_REG (SImode, regno + i));
 }")
 
-(define_insn "tls_get_tpointer_internal"
-  [(set (reg:SI 3)
-       (unspec:SI [(const_int 0)] UNSPEC_TLSTLS))
-   (clobber (reg:SI LR_REGNO))]
-  "TARGET_XCOFF && HAVE_AS_TLS"
-  "bla __get_tpointer")
+(define_insn "*stmsi8"
+  [(match_parallel 0 "store_multiple_operation"
+    [(set (mem:SI (match_operand:SI 1 "gpc_reg_operand" "b"))
+         (match_operand:SI 2 "gpc_reg_operand" "r"))
+     (clobber (match_scratch:SI 3 "=X"))
+     (set (mem:SI (plus:SI (match_dup 1) (const_int 4)))
+         (match_operand:SI 4 "gpc_reg_operand" "r"))
+     (set (mem:SI (plus:SI (match_dup 1) (const_int 8)))
+         (match_operand:SI 5 "gpc_reg_operand" "r"))
+     (set (mem:SI (plus:SI (match_dup 1) (const_int 12)))
+         (match_operand:SI 6 "gpc_reg_operand" "r"))
+     (set (mem:SI (plus:SI (match_dup 1) (const_int 16)))
+         (match_operand:SI 7 "gpc_reg_operand" "r"))
+     (set (mem:SI (plus:SI (match_dup 1) (const_int 20)))
+         (match_operand:SI 8 "gpc_reg_operand" "r"))
+     (set (mem:SI (plus:SI (match_dup 1) (const_int 24)))
+         (match_operand:SI 9 "gpc_reg_operand" "r"))
+     (set (mem:SI (plus:SI (match_dup 1) (const_int 28)))
+         (match_operand:SI 10 "gpc_reg_operand" "r"))])]
+  "TARGET_STRING && XVECLEN (operands[0], 0) == 9"
+  "stswi %2,%1,%O0"
+  [(set_attr "type" "store")
+   (set_attr "update" "yes")
+   (set_attr "indexed" "yes")
+   (set_attr "cell_micro" "always")])
 
-(define_expand "tls_get_addr<mode>"
-  [(set (match_operand:P 0 "gpc_reg_operand" "")
-       (unspec:P [(match_operand:P 1 "gpc_reg_operand" "")
-                   (match_operand:P 2 "gpc_reg_operand" "")] UNSPEC_TLSTLS))]
-  "TARGET_XCOFF && HAVE_AS_TLS"
+(define_insn "*stmsi7"
+  [(match_parallel 0 "store_multiple_operation"
+    [(set (mem:SI (match_operand:SI 1 "gpc_reg_operand" "b"))
+         (match_operand:SI 2 "gpc_reg_operand" "r"))
+     (clobber (match_scratch:SI 3 "=X"))
+     (set (mem:SI (plus:SI (match_dup 1) (const_int 4)))
+         (match_operand:SI 4 "gpc_reg_operand" "r"))
+     (set (mem:SI (plus:SI (match_dup 1) (const_int 8)))
+         (match_operand:SI 5 "gpc_reg_operand" "r"))
+     (set (mem:SI (plus:SI (match_dup 1) (const_int 12)))
+         (match_operand:SI 6 "gpc_reg_operand" "r"))
+     (set (mem:SI (plus:SI (match_dup 1) (const_int 16)))
+         (match_operand:SI 7 "gpc_reg_operand" "r"))
+     (set (mem:SI (plus:SI (match_dup 1) (const_int 20)))
+         (match_operand:SI 8 "gpc_reg_operand" "r"))
+     (set (mem:SI (plus:SI (match_dup 1) (const_int 24)))
+         (match_operand:SI 9 "gpc_reg_operand" "r"))])]
+  "TARGET_STRING && XVECLEN (operands[0], 0) == 8"
+  "stswi %2,%1,%O0"
+  [(set_attr "type" "store")
+   (set_attr "update" "yes")
+   (set_attr "indexed" "yes")
+   (set_attr "cell_micro" "always")])
+
+(define_insn "*stmsi6"
+  [(match_parallel 0 "store_multiple_operation"
+    [(set (mem:SI (match_operand:SI 1 "gpc_reg_operand" "b"))
+         (match_operand:SI 2 "gpc_reg_operand" "r"))
+     (clobber (match_scratch:SI 3 "=X"))
+     (set (mem:SI (plus:SI (match_dup 1) (const_int 4)))
+         (match_operand:SI 4 "gpc_reg_operand" "r"))
+     (set (mem:SI (plus:SI (match_dup 1) (const_int 8)))
+         (match_operand:SI 5 "gpc_reg_operand" "r"))
+     (set (mem:SI (plus:SI (match_dup 1) (const_int 12)))
+         (match_operand:SI 6 "gpc_reg_operand" "r"))
+     (set (mem:SI (plus:SI (match_dup 1) (const_int 16)))
+         (match_operand:SI 7 "gpc_reg_operand" "r"))
+     (set (mem:SI (plus:SI (match_dup 1) (const_int 20)))
+         (match_operand:SI 8 "gpc_reg_operand" "r"))])]
+  "TARGET_STRING && XVECLEN (operands[0], 0) == 7"
+  "stswi %2,%1,%O0"
+  [(set_attr "type" "store")
+   (set_attr "update" "yes")
+   (set_attr "indexed" "yes")
+   (set_attr "cell_micro" "always")])
+
+(define_insn "*stmsi5"
+  [(match_parallel 0 "store_multiple_operation"
+    [(set (mem:SI (match_operand:SI 1 "gpc_reg_operand" "b"))
+         (match_operand:SI 2 "gpc_reg_operand" "r"))
+     (clobber (match_scratch:SI 3 "=X"))
+     (set (mem:SI (plus:SI (match_dup 1) (const_int 4)))
+         (match_operand:SI 4 "gpc_reg_operand" "r"))
+     (set (mem:SI (plus:SI (match_dup 1) (const_int 8)))
+         (match_operand:SI 5 "gpc_reg_operand" "r"))
+     (set (mem:SI (plus:SI (match_dup 1) (const_int 12)))
+         (match_operand:SI 6 "gpc_reg_operand" "r"))
+     (set (mem:SI (plus:SI (match_dup 1) (const_int 16)))
+         (match_operand:SI 7 "gpc_reg_operand" "r"))])]
+  "TARGET_STRING && XVECLEN (operands[0], 0) == 6"
+  "stswi %2,%1,%O0"
+  [(set_attr "type" "store")
+   (set_attr "update" "yes")
+   (set_attr "indexed" "yes")
+   (set_attr "cell_micro" "always")])
+
+(define_insn "*stmsi4"
+  [(match_parallel 0 "store_multiple_operation"
+    [(set (mem:SI (match_operand:SI 1 "gpc_reg_operand" "b"))
+         (match_operand:SI 2 "gpc_reg_operand" "r"))
+     (clobber (match_scratch:SI 3 "=X"))
+     (set (mem:SI (plus:SI (match_dup 1) (const_int 4)))
+         (match_operand:SI 4 "gpc_reg_operand" "r"))
+     (set (mem:SI (plus:SI (match_dup 1) (const_int 8)))
+         (match_operand:SI 5 "gpc_reg_operand" "r"))
+     (set (mem:SI (plus:SI (match_dup 1) (const_int 12)))
+         (match_operand:SI 6 "gpc_reg_operand" "r"))])]
+  "TARGET_STRING && XVECLEN (operands[0], 0) == 5"
+  "stswi %2,%1,%O0"
+  [(set_attr "type" "store")
+   (set_attr "update" "yes")
+   (set_attr "indexed" "yes")
+   (set_attr "cell_micro" "always")])
+
+(define_insn "*stmsi3"
+  [(match_parallel 0 "store_multiple_operation"
+    [(set (mem:SI (match_operand:SI 1 "gpc_reg_operand" "b"))
+         (match_operand:SI 2 "gpc_reg_operand" "r"))
+     (clobber (match_scratch:SI 3 "=X"))
+     (set (mem:SI (plus:SI (match_dup 1) (const_int 4)))
+         (match_operand:SI 4 "gpc_reg_operand" "r"))
+     (set (mem:SI (plus:SI (match_dup 1) (const_int 8)))
+         (match_operand:SI 5 "gpc_reg_operand" "r"))])]
+  "TARGET_STRING && XVECLEN (operands[0], 0) == 4"
+  "stswi %2,%1,%O0"
+  [(set_attr "type" "store")
+   (set_attr "update" "yes")
+   (set_attr "indexed" "yes")
+   (set_attr "cell_micro" "always")])
+\f
+(define_expand "setmemsi"
+  [(parallel [(set (match_operand:BLK 0 "" "")
+                  (match_operand 2 "const_int_operand" ""))
+             (use (match_operand:SI 1 "" ""))
+             (use (match_operand:SI 3 "" ""))])]
+  ""
   "
 {
-  emit_move_insn (gen_rtx_REG (Pmode, 3), operands[1]);
-  emit_move_insn (gen_rtx_REG (Pmode, 4), operands[2]);
-  emit_insn (gen_tls_get_addr_internal<mode> ());
-  emit_move_insn (operands[0], gen_rtx_REG (Pmode, 3));
-  DONE;
+  /* If value to set is not zero, use the library routine.  */
+  if (operands[2] != const0_rtx)
+    FAIL;
+
+  if (expand_block_clear (operands))
+    DONE;
+  else
+    FAIL;
 }")
 
-(define_insn "tls_get_addr_internal<mode>"
-  [(set (reg:P 3)
-       (unspec:P [(reg:P 3) (reg:P 4)] UNSPEC_TLSTLS))
-   (clobber (reg:P 0))
-   (clobber (reg:P 4))
-   (clobber (reg:P 5))
-   (clobber (reg:P 11))
-   (clobber (reg:CC CR0_REGNO))
-   (clobber (reg:P LR_REGNO))]
-  "TARGET_XCOFF && HAVE_AS_TLS"
-  "bla __tls_get_addr")
-\f
-;; Next come insns related to the calling sequence.
-;;
-;; First, an insn to allocate new stack space for dynamic use (e.g., alloca).
-;; We move the back-chain and decrement the stack pointer.
+;; String/block move insn.
+;; Argument 0 is the destination
+;; Argument 1 is the source
+;; Argument 2 is the length
+;; Argument 3 is the alignment
 
-(define_expand "allocate_stack"
-  [(set (match_operand 0 "gpc_reg_operand" "")
-       (minus (reg 1) (match_operand 1 "reg_or_short_operand" "")))
-   (set (reg 1)
-       (minus (reg 1) (match_dup 1)))]
+(define_expand "movmemsi"
+  [(parallel [(set (match_operand:BLK 0 "" "")
+                  (match_operand:BLK 1 "" ""))
+             (use (match_operand:SI 2 "" ""))
+             (use (match_operand:SI 3 "" ""))])]
   ""
   "
-{ rtx chain = gen_reg_rtx (Pmode);
-  rtx stack_bot = gen_rtx_MEM (Pmode, stack_pointer_rtx);
-  rtx neg_op0;
-  rtx insn, par, set, mem;
-
-  emit_move_insn (chain, stack_bot);
-
-  /* Check stack bounds if necessary.  */
-  if (crtl->limit_stack)
-    {
-      rtx available;
-      available = expand_binop (Pmode, sub_optab,
-                               stack_pointer_rtx, stack_limit_rtx,
-                               NULL_RTX, 1, OPTAB_WIDEN);
-      emit_insn (gen_cond_trap (LTU, available, operands[1], const0_rtx));
-    }
-
-  if (GET_CODE (operands[1]) != CONST_INT
-      || INTVAL (operands[1]) < -32767
-      || INTVAL (operands[1]) > 32768)
-    {
-      neg_op0 = gen_reg_rtx (Pmode);
-      if (TARGET_32BIT)
-       emit_insn (gen_negsi2 (neg_op0, operands[1]));
-      else
-       emit_insn (gen_negdi2 (neg_op0, operands[1]));
-    }
+{
+  if (expand_block_move (operands))
+    DONE;
   else
-    neg_op0 = GEN_INT (- INTVAL (operands[1]));
-
-  insn = emit_insn ((* ((TARGET_32BIT) ? gen_movsi_update_stack
-                                      : gen_movdi_di_update_stack))
-                       (stack_pointer_rtx, stack_pointer_rtx, neg_op0,
-                        chain));
-  /* Since we didn't use gen_frame_mem to generate the MEM, grab
-     it now and set the alias set/attributes. The above gen_*_update
-     calls will generate a PARALLEL with the MEM set being the first
-     operation. */
-  par = PATTERN (insn);
-  gcc_assert (GET_CODE (par) == PARALLEL);
-  set = XVECEXP (par, 0, 0);
-  gcc_assert (GET_CODE (set) == SET);
-  mem = SET_DEST (set);
-  gcc_assert (MEM_P (mem));
-  MEM_NOTRAP_P (mem) = 1;
-  set_mem_alias_set (mem, get_frame_alias_set ());
-
-  emit_move_insn (operands[0], virtual_stack_dynamic_rtx);
-  DONE;
+    FAIL;
 }")
 
-;; These patterns say how to save and restore the stack pointer.  We need not
-;; save the stack pointer at function level since we are careful to
-;; preserve the backchain.  At block level, we have to restore the backchain
-;; when we restore the stack pointer.
-;;
-;; For nonlocal gotos, we must save both the stack pointer and its
-;; backchain and restore both.  Note that in the nonlocal case, the
-;; save area is a memory location.
+;; Move up to 32 bytes at a time.  The fixed registers are needed because the
+;; register allocator doesn't have a clue about allocating 8 word registers.
+;; rD/rS = r5 is preferred, efficient form.
+(define_expand "movmemsi_8reg"
+  [(parallel [(set (match_operand 0 "" "")
+                  (match_operand 1 "" ""))
+             (use (match_operand 2 "" ""))
+             (use (match_operand 3 "" ""))
+             (clobber (reg:SI  5))
+             (clobber (reg:SI  6))
+             (clobber (reg:SI  7))
+             (clobber (reg:SI  8))
+             (clobber (reg:SI  9))
+             (clobber (reg:SI 10))
+             (clobber (reg:SI 11))
+             (clobber (reg:SI 12))
+             (clobber (match_scratch:SI 4 ""))])]
+  "TARGET_STRING"
+  "")
 
-(define_expand "save_stack_function"
-  [(match_operand 0 "any_operand" "")
-   (match_operand 1 "any_operand" "")]
-  ""
-  "DONE;")
+(define_insn ""
+  [(set (mem:BLK (match_operand:P 0 "gpc_reg_operand" "b"))
+       (mem:BLK (match_operand:P 1 "gpc_reg_operand" "b")))
+   (use (match_operand:SI 2 "immediate_operand" "i"))
+   (use (match_operand:SI 3 "immediate_operand" "i"))
+   (clobber (match_operand:SI 4 "gpc_reg_operand" "=&r"))
+   (clobber (reg:SI  6))
+   (clobber (reg:SI  7))
+   (clobber (reg:SI  8))
+   (clobber (reg:SI  9))
+   (clobber (reg:SI 10))
+   (clobber (reg:SI 11))
+   (clobber (reg:SI 12))
+   (clobber (match_scratch:SI 5 "=X"))]
+  "TARGET_STRING
+   && ((INTVAL (operands[2]) > 24 && INTVAL (operands[2]) < 32)
+       || INTVAL (operands[2]) == 0)
+   && (REGNO (operands[0]) < 5 || REGNO (operands[0]) > 12)
+   && (REGNO (operands[1]) < 5 || REGNO (operands[1]) > 12)
+   && REGNO (operands[4]) == 5"
+  "lswi %4,%1,%2\;stswi %4,%0,%2"
+  [(set_attr "type" "store")
+   (set_attr "update" "yes")
+   (set_attr "indexed" "yes")
+   (set_attr "cell_micro" "always")
+   (set_attr "length" "8")])
 
-(define_expand "restore_stack_function"
-  [(match_operand 0 "any_operand" "")
-   (match_operand 1 "any_operand" "")]
-  ""
-  "DONE;")
+;; Move up to 24 bytes at a time.  The fixed registers are needed because the
+;; register allocator doesn't have a clue about allocating 6 word registers.
+;; rD/rS = r5 is preferred, efficient form.
+(define_expand "movmemsi_6reg"
+  [(parallel [(set (match_operand 0 "" "")
+                  (match_operand 1 "" ""))
+             (use (match_operand 2 "" ""))
+             (use (match_operand 3 "" ""))
+             (clobber (reg:SI  5))
+             (clobber (reg:SI  6))
+             (clobber (reg:SI  7))
+             (clobber (reg:SI  8))
+             (clobber (reg:SI  9))
+             (clobber (reg:SI 10))
+             (clobber (match_scratch:SI 4 ""))])]
+  "TARGET_STRING"
+  "")
 
-;; Adjust stack pointer (op0) to a new value (op1).
-;; First copy old stack backchain to new location, and ensure that the
-;; scheduler won't reorder the sp assignment before the backchain write.
-(define_expand "restore_stack_block"
-  [(set (match_dup 2) (match_dup 3))
-   (set (match_dup 4) (match_dup 2))
-   (match_dup 5)
-   (set (match_operand 0 "register_operand" "")
-       (match_operand 1 "register_operand" ""))]
-  ""
-  "
-{
-  rtvec p;
+(define_insn ""
+  [(set (mem:BLK (match_operand:P 0 "gpc_reg_operand" "b"))
+       (mem:BLK (match_operand:P 1 "gpc_reg_operand" "b")))
+   (use (match_operand:SI 2 "immediate_operand" "i"))
+   (use (match_operand:SI 3 "immediate_operand" "i"))
+   (clobber (match_operand:SI 4 "gpc_reg_operand" "=&r"))
+   (clobber (reg:SI  6))
+   (clobber (reg:SI  7))
+   (clobber (reg:SI  8))
+   (clobber (reg:SI  9))
+   (clobber (reg:SI 10))
+   (clobber (match_scratch:SI 5 "=X"))]
+  "TARGET_STRING
+   && INTVAL (operands[2]) > 16 && INTVAL (operands[2]) <= 32
+   && (REGNO (operands[0]) < 5 || REGNO (operands[0]) > 10)
+   && (REGNO (operands[1]) < 5 || REGNO (operands[1]) > 10)
+   && REGNO (operands[4]) == 5"
+  "lswi %4,%1,%2\;stswi %4,%0,%2"
+  [(set_attr "type" "store")
+   (set_attr "update" "yes")
+   (set_attr "indexed" "yes")
+   (set_attr "cell_micro" "always")
+   (set_attr "length" "8")])
 
-  operands[1] = force_reg (Pmode, operands[1]);
-  operands[2] = gen_reg_rtx (Pmode);
-  operands[3] = gen_frame_mem (Pmode, operands[0]);
-  operands[4] = gen_frame_mem (Pmode, operands[1]);
-  p = rtvec_alloc (1);
-  RTVEC_ELT (p, 0) = gen_rtx_SET (VOIDmode,
-                                 gen_frame_mem (BLKmode, operands[0]),
-                                 const0_rtx);
-  operands[5] = gen_rtx_PARALLEL (VOIDmode, p);
-}")
+;; Move up to 16 bytes at a time, using 4 fixed registers to avoid spill
+;; problems with TImode.
+;; rD/rS = r5 is preferred, efficient form.
+(define_expand "movmemsi_4reg"
+  [(parallel [(set (match_operand 0 "" "")
+                  (match_operand 1 "" ""))
+             (use (match_operand 2 "" ""))
+             (use (match_operand 3 "" ""))
+             (clobber (reg:SI 5))
+             (clobber (reg:SI 6))
+             (clobber (reg:SI 7))
+             (clobber (reg:SI 8))
+             (clobber (match_scratch:SI 4 ""))])]
+  "TARGET_STRING"
+  "")
+
+(define_insn ""
+  [(set (mem:BLK (match_operand:P 0 "gpc_reg_operand" "b"))
+       (mem:BLK (match_operand:P 1 "gpc_reg_operand" "b")))
+   (use (match_operand:SI 2 "immediate_operand" "i"))
+   (use (match_operand:SI 3 "immediate_operand" "i"))
+   (clobber (match_operand:SI 4 "gpc_reg_operand" "=&r"))
+   (clobber (reg:SI 6))
+   (clobber (reg:SI 7))
+   (clobber (reg:SI 8))
+   (clobber (match_scratch:SI 5 "=X"))]
+  "TARGET_STRING
+   && INTVAL (operands[2]) > 8 && INTVAL (operands[2]) <= 16
+   && (REGNO (operands[0]) < 5 || REGNO (operands[0]) > 8)
+   && (REGNO (operands[1]) < 5 || REGNO (operands[1]) > 8)
+   && REGNO (operands[4]) == 5"
+  "lswi %4,%1,%2\;stswi %4,%0,%2"
+  [(set_attr "type" "store")
+   (set_attr "update" "yes")
+   (set_attr "indexed" "yes")
+   (set_attr "cell_micro" "always")
+   (set_attr "length" "8")])
 
-(define_expand "save_stack_nonlocal"
-  [(set (match_dup 3) (match_dup 4))
-   (set (match_operand 0 "memory_operand" "") (match_dup 3))
-   (set (match_dup 2) (match_operand 1 "register_operand" ""))]
-  ""
-  "
-{
-  int units_per_word = (TARGET_32BIT) ? 4 : 8;
+;; Move up to 8 bytes at a time.
+(define_expand "movmemsi_2reg"
+  [(parallel [(set (match_operand 0 "" "")
+                  (match_operand 1 "" ""))
+             (use (match_operand 2 "" ""))
+             (use (match_operand 3 "" ""))
+             (clobber (match_scratch:DI 4 ""))
+             (clobber (match_scratch:SI 5 ""))])]
+  "TARGET_STRING && ! TARGET_POWERPC64"
+  "")
 
-  /* Copy the backchain to the first word, sp to the second.  */
-  operands[0] = adjust_address_nv (operands[0], Pmode, 0);
-  operands[2] = adjust_address_nv (operands[0], Pmode, units_per_word);
-  operands[3] = gen_reg_rtx (Pmode);
-  operands[4] = gen_frame_mem (Pmode, operands[1]);
-}")
+(define_insn ""
+  [(set (mem:BLK (match_operand:SI 0 "gpc_reg_operand" "b"))
+       (mem:BLK (match_operand:SI 1 "gpc_reg_operand" "b")))
+   (use (match_operand:SI 2 "immediate_operand" "i"))
+   (use (match_operand:SI 3 "immediate_operand" "i"))
+   (clobber (match_scratch:DI 4 "=&r"))
+   (clobber (match_scratch:SI 5 "=X"))]
+  "TARGET_STRING && ! TARGET_POWERPC64
+   && INTVAL (operands[2]) > 4 && INTVAL (operands[2]) <= 8"
+  "lswi %4,%1,%2\;stswi %4,%0,%2"
+  [(set_attr "type" "store")
+   (set_attr "update" "yes")
+   (set_attr "indexed" "yes")
+   (set_attr "cell_micro" "always")
+   (set_attr "length" "8")])
 
-(define_expand "restore_stack_nonlocal"
-  [(set (match_dup 2) (match_operand 1 "memory_operand" ""))
-   (set (match_dup 3) (match_dup 4))
-   (set (match_dup 5) (match_dup 2))
-   (match_dup 6)
-   (set (match_operand 0 "register_operand" "") (match_dup 3))]
-  ""
-  "
-{
-  int units_per_word = (TARGET_32BIT) ? 4 : 8;
-  rtvec p;
+;; Move up to 4 bytes at a time.
+(define_expand "movmemsi_1reg"
+  [(parallel [(set (match_operand 0 "" "")
+                  (match_operand 1 "" ""))
+             (use (match_operand 2 "" ""))
+             (use (match_operand 3 "" ""))
+             (clobber (match_scratch:SI 4 ""))
+             (clobber (match_scratch:SI 5 ""))])]
+  "TARGET_STRING"
+  "")
 
-  /* Restore the backchain from the first word, sp from the second.  */
-  operands[2] = gen_reg_rtx (Pmode);
-  operands[3] = gen_reg_rtx (Pmode);
-  operands[1] = adjust_address_nv (operands[1], Pmode, 0);
-  operands[4] = adjust_address_nv (operands[1], Pmode, units_per_word);
-  operands[5] = gen_frame_mem (Pmode, operands[3]);
-  p = rtvec_alloc (1);
-  RTVEC_ELT (p, 0) = gen_rtx_SET (VOIDmode,
-                                 gen_frame_mem (BLKmode, operands[0]),
-                                 const0_rtx);
-  operands[6] = gen_rtx_PARALLEL (VOIDmode, p);
-}")
+(define_insn ""
+  [(set (mem:BLK (match_operand:P 0 "gpc_reg_operand" "b"))
+       (mem:BLK (match_operand:P 1 "gpc_reg_operand" "b")))
+   (use (match_operand:SI 2 "immediate_operand" "i"))
+   (use (match_operand:SI 3 "immediate_operand" "i"))
+   (clobber (match_scratch:SI 4 "=&r"))
+   (clobber (match_scratch:SI 5 "=X"))]
+  "TARGET_STRING && INTVAL (operands[2]) > 0 && INTVAL (operands[2]) <= 4"
+  "lswi %4,%1,%2\;stswi %4,%0,%2"
+  [(set_attr "type" "store")
+   (set_attr "update" "yes")
+   (set_attr "indexed" "yes")
+   (set_attr "cell_micro" "always")
+   (set_attr "length" "8")])
 \f
-;; TOC register handling.
-
-;; Code to initialize the TOC register...
-
-(define_insn "load_toc_aix_si"
-  [(parallel [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-                  (unspec:SI [(const_int 0)] UNSPEC_TOC))
-             (use (reg:SI 2))])]
-  "(DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2) && TARGET_32BIT"
-  "*
-{
-  char buf[30];
-  ASM_GENERATE_INTERNAL_LABEL (buf, \"LCTOC\", 1);
-  operands[1] = gen_rtx_SYMBOL_REF (Pmode, ggc_strdup (buf));
-  operands[2] = gen_rtx_REG (Pmode, 2);
-  return \"lwz %0,%1(%2)\";
-}"
-  [(set_attr "type" "load")
-   (set_attr "update" "no")
-   (set_attr "indexed" "no")])
+;; Define insns that do load or store with update.  Some of these we can
+;; get by using pre-decrement or pre-increment, but the hardware can also
+;; do cases where the increment is not the size of the object.
+;;
+;; In all these cases, we use operands 0 and 1 for the register being
+;; incremented because those are the operands that local-alloc will
+;; tie and these are the pair most likely to be tieable (and the ones
+;; that will benefit the most).
 
-(define_insn "load_toc_aix_di"
-  [(parallel [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
-                  (unspec:DI [(const_int 0)] UNSPEC_TOC))
-             (use (reg:DI 2))])]
-  "(DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2) && TARGET_64BIT"
-  "*
-{
-  char buf[30];
-#ifdef TARGET_RELOCATABLE
-  ASM_GENERATE_INTERNAL_LABEL (buf, \"LCTOC\",
-                              !TARGET_MINIMAL_TOC || TARGET_RELOCATABLE);
-#else
-  ASM_GENERATE_INTERNAL_LABEL (buf, \"LCTOC\", 1);
-#endif
-  if (TARGET_ELF)
-    strcat (buf, \"@toc\");
-  operands[1] = gen_rtx_SYMBOL_REF (Pmode, ggc_strdup (buf));
-  operands[2] = gen_rtx_REG (Pmode, 2);
-  return \"ld %0,%1(%2)\";
-}"
+(define_insn "*movdi_update1"
+  [(set (match_operand:DI 3 "gpc_reg_operand" "=r,r")
+       (mem:DI (plus:DI (match_operand:DI 1 "gpc_reg_operand" "0,0")
+                        (match_operand:DI 2 "reg_or_aligned_short_operand" "r,I"))))
+   (set (match_operand:DI 0 "gpc_reg_operand" "=b,b")
+       (plus:DI (match_dup 1) (match_dup 2)))]
+  "TARGET_POWERPC64 && TARGET_UPDATE
+   && (!avoiding_indexed_address_p (DImode)
+       || !gpc_reg_operand (operands[2], DImode))"
+  "@
+   ldux %3,%0,%2
+   ldu %3,%2(%0)"
   [(set_attr "type" "load")
-   (set_attr "update" "no")
-   (set_attr "indexed" "no")])
-
-(define_insn "load_toc_v4_pic_si"
-  [(set (reg:SI LR_REGNO)
-       (unspec:SI [(const_int 0)] UNSPEC_TOC))]
-  "DEFAULT_ABI == ABI_V4 && flag_pic == 1 && TARGET_32BIT"
-  "bl _GLOBAL_OFFSET_TABLE_@local-4"
-  [(set_attr "type" "branch")
-   (set_attr "length" "4")])
-
-(define_expand "load_toc_v4_PIC_1"
-  [(parallel [(set (reg:SI LR_REGNO)
-                  (match_operand:SI 0 "immediate_operand" "s"))
-             (use (unspec [(match_dup 0)] UNSPEC_TOC))])]
-  "TARGET_ELF && DEFAULT_ABI == ABI_V4
-   && (flag_pic == 2 || (flag_pic && TARGET_SECURE_PLT))"
-  "")
+   (set_attr "update" "yes")
+   (set_attr "indexed" "yes,no")])
 
-(define_insn "load_toc_v4_PIC_1_normal"
-  [(set (reg:SI LR_REGNO)
-       (match_operand:SI 0 "immediate_operand" "s"))
-   (use (unspec [(match_dup 0)] UNSPEC_TOC))]
-  "!TARGET_LINK_STACK && TARGET_ELF && DEFAULT_ABI == ABI_V4
-   && (flag_pic == 2 || (flag_pic && TARGET_SECURE_PLT))"
-  "bcl 20,31,%0\\n%0:"
-  [(set_attr "type" "branch")
-   (set_attr "length" "4")])
+(define_insn "movdi_<mode>_update"
+  [(set (mem:DI (plus:P (match_operand:P 1 "gpc_reg_operand" "0,0")
+                        (match_operand:P 2 "reg_or_aligned_short_operand" "r,I")))
+       (match_operand:DI 3 "gpc_reg_operand" "r,r"))
+   (set (match_operand:P 0 "gpc_reg_operand" "=b,b")
+       (plus:P (match_dup 1) (match_dup 2)))]
+  "TARGET_POWERPC64 && TARGET_UPDATE
+   && (!avoiding_indexed_address_p (Pmode)
+       || !gpc_reg_operand (operands[2], Pmode)
+       || (REG_P (operands[0])
+          && REGNO (operands[0]) == STACK_POINTER_REGNUM))"
+  "@
+   stdux %3,%0,%2
+   stdu %3,%2(%0)"
+  [(set_attr "type" "store")
+   (set_attr "update" "yes")
+   (set_attr "indexed" "yes,no")])
 
-(define_insn "load_toc_v4_PIC_1_476"
-  [(set (reg:SI LR_REGNO)
-       (match_operand:SI 0 "immediate_operand" "s"))
-   (use (unspec [(match_dup 0)] UNSPEC_TOC))]
-  "TARGET_LINK_STACK && TARGET_ELF && DEFAULT_ABI == ABI_V4
-   && (flag_pic == 2 || (flag_pic && TARGET_SECURE_PLT))"
-  "*
-{
-  char name[32];
-  static char templ[32];
+;; This pattern is only conditional on TARGET_POWERPC64, as it is
+;; needed for stack allocation, even if the user passes -mno-update.
+(define_insn "movdi_<mode>_update_stack"
+  [(set (mem:DI (plus:P (match_operand:P 1 "gpc_reg_operand" "0,0")
+                        (match_operand:P 2 "reg_or_aligned_short_operand" "r,I")))
+       (match_operand:DI 3 "gpc_reg_operand" "r,r"))
+   (set (match_operand:P 0 "gpc_reg_operand" "=b,b")
+       (plus:P (match_dup 1) (match_dup 2)))]
+  "TARGET_POWERPC64"
+  "@
+   stdux %3,%0,%2
+   stdu %3,%2(%0)"
+  [(set_attr "type" "store")
+   (set_attr "update" "yes")
+   (set_attr "indexed" "yes,no")])
 
-  get_ppc476_thunk_name (name);
-  sprintf (templ, \"bl %s\\n%%0:\", name);
-  return templ;
-}"
-  [(set_attr "type" "branch")
-   (set_attr "length" "4")])
+(define_insn "*movsi_update1"
+  [(set (match_operand:SI 3 "gpc_reg_operand" "=r,r")
+       (mem:SI (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
+                        (match_operand:SI 2 "reg_or_short_operand" "r,I"))))
+   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
+       (plus:SI (match_dup 1) (match_dup 2)))]
+  "TARGET_UPDATE
+   && (!avoiding_indexed_address_p (SImode)
+       || !gpc_reg_operand (operands[2], SImode))"
+  "@
+   lwzux %3,%0,%2
+   lwzu %3,%2(%0)"
+  [(set_attr "type" "load")
+   (set_attr "update" "yes")
+   (set_attr "indexed" "yes,no")])
 
-(define_expand "load_toc_v4_PIC_1b"
-  [(parallel [(set (reg:SI LR_REGNO)
-                  (unspec:SI [(match_operand:SI 0 "immediate_operand" "s")
-                              (label_ref (match_operand 1 "" ""))]
-                          UNSPEC_TOCPTR))
-             (match_dup 1)])]
-  "TARGET_ELF && DEFAULT_ABI == ABI_V4 && flag_pic == 2"
-  "")
+(define_insn "*movsi_update2"
+  [(set (match_operand:DI 3 "gpc_reg_operand" "=r")
+       (sign_extend:DI
+        (mem:SI (plus:DI (match_operand:DI 1 "gpc_reg_operand" "0")
+                         (match_operand:DI 2 "gpc_reg_operand" "r")))))
+   (set (match_operand:DI 0 "gpc_reg_operand" "=b")
+       (plus:DI (match_dup 1) (match_dup 2)))]
+  "TARGET_POWERPC64 && rs6000_gen_cell_microcode
+   && !avoiding_indexed_address_p (DImode)"
+  "lwaux %3,%0,%2"
+  [(set_attr "type" "load")
+   (set_attr "sign_extend" "yes")
+   (set_attr "update" "yes")
+   (set_attr "indexed" "yes")])
 
-(define_insn "load_toc_v4_PIC_1b_normal"
-  [(set (reg:SI LR_REGNO)
-       (unspec:SI [(match_operand:SI 0 "immediate_operand" "s")
-                   (label_ref (match_operand 1 "" ""))]
-               UNSPEC_TOCPTR))
-   (match_dup 1)]
-  "!TARGET_LINK_STACK && TARGET_ELF && DEFAULT_ABI == ABI_V4 && flag_pic == 2"
-  "bcl 20,31,$+8\;.long %0-$"
-  [(set_attr "type" "branch")
-   (set_attr "length" "8")])
+(define_insn "movsi_update"
+  [(set (mem:SI (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
+                        (match_operand:SI 2 "reg_or_short_operand" "r,I")))
+       (match_operand:SI 3 "gpc_reg_operand" "r,r"))
+   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
+       (plus:SI (match_dup 1) (match_dup 2)))]
+  "TARGET_UPDATE
+   && (!avoiding_indexed_address_p (SImode)
+       || !gpc_reg_operand (operands[2], SImode)
+       || (REG_P (operands[0])
+          && REGNO (operands[0]) == STACK_POINTER_REGNUM))"
+  "@
+   stwux %3,%0,%2
+   stwu %3,%2(%0)"
+  [(set_attr "type" "store")
+   (set_attr "update" "yes")
+   (set_attr "indexed" "yes,no")])
 
-(define_insn "load_toc_v4_PIC_1b_476"
-  [(set (reg:SI LR_REGNO)
-       (unspec:SI [(match_operand:SI 0 "immediate_operand" "s")
-                   (label_ref (match_operand 1 "" ""))]
-               UNSPEC_TOCPTR))
-   (match_dup 1)]
-  "TARGET_LINK_STACK && TARGET_ELF && DEFAULT_ABI == ABI_V4 && flag_pic == 2"
-  "*
-{
-  char name[32];
-  static char templ[32];
+;; This is an unconditional pattern; needed for stack allocation, even
+;; if the user passes -mno-update.
+(define_insn "movsi_update_stack"
+  [(set (mem:SI (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
+                        (match_operand:SI 2 "reg_or_short_operand" "r,I")))
+       (match_operand:SI 3 "gpc_reg_operand" "r,r"))
+   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
+       (plus:SI (match_dup 1) (match_dup 2)))]
+  ""
+  "@
+   stwux %3,%0,%2
+   stwu %3,%2(%0)"
+  [(set_attr "type" "store")
+   (set_attr "update" "yes")
+   (set_attr "indexed" "yes,no")])
 
-  get_ppc476_thunk_name (name);
-  sprintf (templ, \"bl %s\\n\\tb $+8\\n\\t.long %%0-$\", name);
-  return templ;
-}"
-  [(set_attr "type" "branch")
-   (set_attr "length" "16")])
+(define_insn "*movhi_update1"
+  [(set (match_operand:HI 3 "gpc_reg_operand" "=r,r")
+       (mem:HI (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
+                        (match_operand:SI 2 "reg_or_short_operand" "r,I"))))
+   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
+       (plus:SI (match_dup 1) (match_dup 2)))]
+  "TARGET_UPDATE
+   && (!avoiding_indexed_address_p (SImode)
+       || !gpc_reg_operand (operands[2], SImode))"
+  "@
+   lhzux %3,%0,%2
+   lhzu %3,%2(%0)"
+  [(set_attr "type" "load")
+   (set_attr "update" "yes")
+   (set_attr "indexed" "yes,no")])
 
-(define_insn "load_toc_v4_PIC_2"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-       (mem:SI (plus:SI (match_operand:SI 1 "gpc_reg_operand" "b")
-                  (minus:SI (match_operand:SI 2 "immediate_operand" "s")
-                            (match_operand:SI 3 "immediate_operand" "s")))))]
-  "TARGET_ELF && DEFAULT_ABI == ABI_V4 && flag_pic == 2"
-  "lwz %0,%2-%3(%1)"
-  [(set_attr "type" "load")])
+(define_insn "*movhi_update2"
+  [(set (match_operand:SI 3 "gpc_reg_operand" "=r,r")
+       (zero_extend:SI
+        (mem:HI (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
+                         (match_operand:SI 2 "reg_or_short_operand" "r,I")))))
+   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
+       (plus:SI (match_dup 1) (match_dup 2)))]
+  "TARGET_UPDATE
+   && (!avoiding_indexed_address_p (SImode)
+       || !gpc_reg_operand (operands[2], SImode))"
+  "@
+   lhzux %3,%0,%2
+   lhzu %3,%2(%0)"
+  [(set_attr "type" "load")
+   (set_attr "update" "yes")
+   (set_attr "indexed" "yes,no")])
 
-(define_insn "load_toc_v4_PIC_3b"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-       (plus:SI (match_operand:SI 1 "gpc_reg_operand" "b")
-                (high:SI
-                  (minus:SI (match_operand:SI 2 "symbol_ref_operand" "s")
-                            (match_operand:SI 3 "symbol_ref_operand" "s")))))]
-  "TARGET_ELF && TARGET_SECURE_PLT && DEFAULT_ABI == ABI_V4 && flag_pic"
-  "addis %0,%1,%2-%3@ha")
+(define_insn "*movhi_update3"
+  [(set (match_operand:SI 3 "gpc_reg_operand" "=r,r")
+       (sign_extend:SI
+        (mem:HI (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
+                         (match_operand:SI 2 "reg_or_short_operand" "r,I")))))
+   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
+       (plus:SI (match_dup 1) (match_dup 2)))]
+  "TARGET_UPDATE && rs6000_gen_cell_microcode
+   && (!avoiding_indexed_address_p (SImode)
+       || !gpc_reg_operand (operands[2], SImode))"
+  "@
+   lhaux %3,%0,%2
+   lhau %3,%2(%0)"
+  [(set_attr "type" "load")
+   (set_attr "sign_extend" "yes")
+   (set_attr "update" "yes")
+   (set_attr "indexed" "yes,no")])
 
-(define_insn "load_toc_v4_PIC_3c"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-       (lo_sum:SI (match_operand:SI 1 "gpc_reg_operand" "b")
-                  (minus:SI (match_operand:SI 2 "symbol_ref_operand" "s")
-                            (match_operand:SI 3 "symbol_ref_operand" "s"))))]
-  "TARGET_ELF && TARGET_SECURE_PLT && DEFAULT_ABI == ABI_V4 && flag_pic"
-  "addi %0,%1,%2-%3@l")
+(define_insn "*movhi_update4"
+  [(set (mem:HI (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
+                        (match_operand:SI 2 "reg_or_short_operand" "r,I")))
+       (match_operand:HI 3 "gpc_reg_operand" "r,r"))
+   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
+       (plus:SI (match_dup 1) (match_dup 2)))]
+  "TARGET_UPDATE
+   && (!avoiding_indexed_address_p (SImode)
+       || !gpc_reg_operand (operands[2], SImode))"
+  "@
+   sthux %3,%0,%2
+   sthu %3,%2(%0)"
+  [(set_attr "type" "store")
+   (set_attr "update" "yes")
+   (set_attr "indexed" "yes,no")])
 
-;; If the TOC is shared over a translation unit, as happens with all
-;; the kinds of PIC that we support, we need to restore the TOC
-;; pointer only when jumping over units of translation.
-;; On Darwin, we need to reload the picbase.
+(define_insn "*movqi_update1"
+  [(set (match_operand:QI 3 "gpc_reg_operand" "=r,r")
+       (mem:QI (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
+                        (match_operand:SI 2 "reg_or_short_operand" "r,I"))))
+   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
+       (plus:SI (match_dup 1) (match_dup 2)))]
+  "TARGET_UPDATE
+   && (!avoiding_indexed_address_p (SImode)
+       || !gpc_reg_operand (operands[2], SImode))"
+  "@
+   lbzux %3,%0,%2
+   lbzu %3,%2(%0)"
+  [(set_attr "type" "load")
+   (set_attr "update" "yes")
+   (set_attr "indexed" "yes,no")])
 
-(define_expand "builtin_setjmp_receiver"
-  [(use (label_ref (match_operand 0 "" "")))]
-  "(DEFAULT_ABI == ABI_V4 && flag_pic == 1)
-   || (TARGET_TOC && TARGET_MINIMAL_TOC)
-   || (DEFAULT_ABI == ABI_DARWIN && flag_pic)"
-  "
-{
-#if TARGET_MACHO
-  if (DEFAULT_ABI == ABI_DARWIN)
-    {
-      rtx picrtx = gen_rtx_SYMBOL_REF (Pmode, MACHOPIC_FUNCTION_BASE_NAME);
-      rtx picreg = gen_rtx_REG (Pmode, RS6000_PIC_OFFSET_TABLE_REGNUM);
-      rtx tmplabrtx;
-      char tmplab[20];
+(define_insn "*movqi_update2"
+  [(set (match_operand:SI 3 "gpc_reg_operand" "=r,r")
+       (zero_extend:SI
+        (mem:QI (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
+                         (match_operand:SI 2 "reg_or_short_operand" "r,I")))))
+   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
+       (plus:SI (match_dup 1) (match_dup 2)))]
+  "TARGET_UPDATE
+   && (!avoiding_indexed_address_p (SImode)
+       || !gpc_reg_operand (operands[2], SImode))"
+  "@
+   lbzux %3,%0,%2
+   lbzu %3,%2(%0)"
+  [(set_attr "type" "load")
+   (set_attr "update" "yes")
+   (set_attr "indexed" "yes,no")])
 
-      crtl->uses_pic_offset_table = 1;
-      ASM_GENERATE_INTERNAL_LABEL(tmplab, \"LSJR\",
-                                 CODE_LABEL_NUMBER (operands[0]));
-      tmplabrtx = gen_rtx_SYMBOL_REF (Pmode, ggc_strdup (tmplab));
+(define_insn "*movqi_update3"
+  [(set (mem:QI (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
+                        (match_operand:SI 2 "reg_or_short_operand" "r,I")))
+       (match_operand:QI 3 "gpc_reg_operand" "r,r"))
+   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
+       (plus:SI (match_dup 1) (match_dup 2)))]
+  "TARGET_UPDATE
+   && (!avoiding_indexed_address_p (SImode)
+       || !gpc_reg_operand (operands[2], SImode))"
+  "@
+   stbux %3,%0,%2
+   stbu %3,%2(%0)"
+  [(set_attr "type" "store")
+   (set_attr "update" "yes")
+   (set_attr "indexed" "yes,no")])
 
-      emit_insn (gen_load_macho_picbase (tmplabrtx));
-      emit_move_insn (picreg, gen_rtx_REG (Pmode, LR_REGNO));
-      emit_insn (gen_macho_correct_pic (picreg, picreg, picrtx, tmplabrtx));
-    }
-  else
-#endif
-    rs6000_emit_load_toc_table (FALSE);
-  DONE;
-}")
+(define_insn "*movsf_update1"
+  [(set (match_operand:SF 3 "gpc_reg_operand" "=f,f")
+       (mem:SF (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
+                        (match_operand:SI 2 "reg_or_short_operand" "r,I"))))
+   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
+       (plus:SI (match_dup 1) (match_dup 2)))]
+  "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_SINGLE_FLOAT && TARGET_UPDATE
+   && (!avoiding_indexed_address_p (SImode)
+       || !gpc_reg_operand (operands[2], SImode))"
+  "@
+   lfsux %3,%0,%2
+   lfsu %3,%2(%0)"
+  [(set_attr "type" "fpload")
+   (set_attr "update" "yes")
+   (set_attr "indexed" "yes,no")])
 
-;; Largetoc support
-(define_insn "*largetoc_high"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=b*r")
-        (high:DI
-         (unspec [(match_operand:DI 1 "" "")
-                  (match_operand:DI 2 "gpc_reg_operand" "b")]
-                 UNSPEC_TOCREL)))]
-   "TARGET_ELF && TARGET_CMODEL != CMODEL_SMALL"
-   "addis %0,%2,%1@toc@ha")
+(define_insn "*movsf_update2"
+  [(set (mem:SF (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
+                        (match_operand:SI 2 "reg_or_short_operand" "r,I")))
+       (match_operand:SF 3 "gpc_reg_operand" "f,f"))
+   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
+       (plus:SI (match_dup 1) (match_dup 2)))]
+  "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_SINGLE_FLOAT && TARGET_UPDATE
+   && (!avoiding_indexed_address_p (SImode)
+       || !gpc_reg_operand (operands[2], SImode))"
+  "@
+   stfsux %3,%0,%2
+   stfsu %3,%2(%0)"
+  [(set_attr "type" "fpstore")
+   (set_attr "update" "yes")
+   (set_attr "indexed" "yes,no")])
 
-(define_insn "*largetoc_high_aix<mode>"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=b*r")
-        (high:P
-         (unspec [(match_operand:P 1 "" "")
-                  (match_operand:P 2 "gpc_reg_operand" "b")]
-                 UNSPEC_TOCREL)))]
-   "TARGET_XCOFF && TARGET_CMODEL != CMODEL_SMALL"
-   "addis %0,%1@u(%2)")
+(define_insn "*movsf_update3"
+  [(set (match_operand:SF 3 "gpc_reg_operand" "=r,r")
+       (mem:SF (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
+                        (match_operand:SI 2 "reg_or_short_operand" "r,I"))))
+   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
+       (plus:SI (match_dup 1) (match_dup 2)))]
+  "(TARGET_SOFT_FLOAT || !TARGET_FPRS) && TARGET_UPDATE
+   && (!avoiding_indexed_address_p (SImode)
+       || !gpc_reg_operand (operands[2], SImode))"
+  "@
+   lwzux %3,%0,%2
+   lwzu %3,%2(%0)"
+  [(set_attr "type" "load")
+   (set_attr "update" "yes")
+   (set_attr "indexed" "yes,no")])
 
-(define_insn "*largetoc_high_plus"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=b*r")
-        (high:DI
-         (plus:DI
-           (unspec [(match_operand:DI 1 "" "")
-                    (match_operand:DI 2 "gpc_reg_operand" "b")]
-                   UNSPEC_TOCREL)
-           (match_operand:DI 3 "add_cint_operand" "n"))))]
-   "TARGET_ELF && TARGET_CMODEL != CMODEL_SMALL"
-   "addis %0,%2,%1+%3@toc@ha")
+(define_insn "*movsf_update4"
+  [(set (mem:SF (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
+                        (match_operand:SI 2 "reg_or_short_operand" "r,I")))
+       (match_operand:SF 3 "gpc_reg_operand" "r,r"))
+   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
+       (plus:SI (match_dup 1) (match_dup 2)))]
+  "(TARGET_SOFT_FLOAT || !TARGET_FPRS) && TARGET_UPDATE
+   && (!avoiding_indexed_address_p (SImode)
+       || !gpc_reg_operand (operands[2], SImode))"
+  "@
+   stwux %3,%0,%2
+   stwu %3,%2(%0)"
+  [(set_attr "type" "store")
+   (set_attr "update" "yes")
+   (set_attr "indexed" "yes,no")])
 
-(define_insn "*largetoc_high_plus_aix<mode>"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=b*r")
-        (high:P
-         (plus:P
-           (unspec [(match_operand:P 1 "" "")
-                    (match_operand:P 2 "gpc_reg_operand" "b")]
-                   UNSPEC_TOCREL)
-           (match_operand:P 3 "add_cint_operand" "n"))))]
-   "TARGET_XCOFF && TARGET_CMODEL != CMODEL_SMALL"
-   "addis %0,%1+%3@u(%2)")
+(define_insn "*movdf_update1"
+  [(set (match_operand:DF 3 "gpc_reg_operand" "=d,d")
+       (mem:DF (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
+                        (match_operand:SI 2 "reg_or_short_operand" "r,I"))))
+   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
+       (plus:SI (match_dup 1) (match_dup 2)))]
+  "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT && TARGET_UPDATE
+   && (!avoiding_indexed_address_p (SImode)
+       || !gpc_reg_operand (operands[2], SImode))"
+  "@
+   lfdux %3,%0,%2
+   lfdu %3,%2(%0)"
+  [(set_attr "type" "fpload")
+   (set_attr "update" "yes")
+   (set_attr "indexed" "yes,no")])
 
-(define_insn "*largetoc_low"
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r,r")
-        (lo_sum:DI (match_operand:DI 1 "gpc_reg_operand" "b,!*r")
-                  (match_operand:DI 2 "" "")))]
-   "TARGET_ELF && TARGET_CMODEL != CMODEL_SMALL"
-   "@
-    addi %0,%1,%2@l
-    addic %0,%1,%2@l")
+(define_insn "*movdf_update2"
+  [(set (mem:DF (plus:SI (match_operand:SI 1 "gpc_reg_operand" "0,0")
+                        (match_operand:SI 2 "reg_or_short_operand" "r,I")))
+       (match_operand:DF 3 "gpc_reg_operand" "d,d"))
+   (set (match_operand:SI 0 "gpc_reg_operand" "=b,b")
+       (plus:SI (match_dup 1) (match_dup 2)))]
+  "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT && TARGET_UPDATE
+   && (!avoiding_indexed_address_p (SImode)
+       || !gpc_reg_operand (operands[2], SImode))"
+  "@
+   stfdux %3,%0,%2
+   stfdu %3,%2(%0)"
+  [(set_attr "type" "fpstore")
+   (set_attr "update" "yes")
+   (set_attr "indexed" "yes,no")])
 
-(define_insn "*largetoc_low_aix<mode>"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
-        (lo_sum:P (match_operand:P 1 "gpc_reg_operand" "b")
-                  (match_operand:P 2 "" "")))]
-   "TARGET_XCOFF && TARGET_CMODEL != CMODEL_SMALL"
-   "la %0,%2@l(%1)")
 
-(define_insn_and_split "*tocref<mode>"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=b*r")
-       (match_operand:P 1 "small_toc_ref" "R"))]
-   "TARGET_TOC"
-   "la %0,%a1"
-   "&& TARGET_CMODEL != CMODEL_SMALL && reload_completed"
-  [(set (match_dup 0) (high:P (match_dup 1)))
-   (set (match_dup 0) (lo_sum:P (match_dup 0) (match_dup 1)))])
+;; After inserting conditional returns we can sometimes have
+;; unnecessary register moves.  Unfortunately we cannot have a
+;; modeless peephole here, because some single SImode sets have early
+;; clobber outputs.  Although those sets expand to multi-ppc-insn
+;; sequences, using get_attr_length here will smash the operands
+;; array.  Neither is there an early_cobbler_p predicate.
+;; Disallow subregs for E500 so we don't munge frob_di_df_2.
+;; Also this optimization interferes with scalars going into
+;; altivec registers (the code does reloading through the FPRs).
+(define_peephole2
+  [(set (match_operand:DF 0 "gpc_reg_operand" "")
+       (match_operand:DF 1 "any_operand" ""))
+   (set (match_operand:DF 2 "gpc_reg_operand" "")
+       (match_dup 0))]
+  "!(TARGET_E500_DOUBLE && GET_CODE (operands[2]) == SUBREG)
+   && !TARGET_UPPER_REGS_DF
+   && peep2_reg_dead_p (2, operands[0])"
+  [(set (match_dup 2) (match_dup 1))])
 
-;; Elf specific ways of loading addresses for non-PIC code.
-;; The output of this could be r0, but we make a very strong
-;; preference for a base register because it will usually
-;; be needed there.
-(define_insn "elf_high"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=b*r")
-       (high:SI (match_operand 1 "" "")))]
-  "TARGET_ELF && ! TARGET_64BIT"
-  "lis %0,%1@ha")
+(define_peephole2
+  [(set (match_operand:SF 0 "gpc_reg_operand" "")
+       (match_operand:SF 1 "any_operand" ""))
+   (set (match_operand:SF 2 "gpc_reg_operand" "")
+       (match_dup 0))]
+  "!TARGET_UPPER_REGS_SF
+   && peep2_reg_dead_p (2, operands[0])"
+  [(set (match_dup 2) (match_dup 1))])
 
-(define_insn "elf_low"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r,r")
-       (lo_sum:SI (match_operand:SI 1 "gpc_reg_operand" "b,!*r")
-                  (match_operand 2 "" "")))]
-   "TARGET_ELF && ! TARGET_64BIT"
-   "@
-    la %0,%2@l(%1)
-    addic %0,%1,%K2")
 \f
-;; Call and call_value insns
-(define_expand "call"
-  [(parallel [(call (mem:SI (match_operand 0 "address_operand" ""))
-                   (match_operand 1 "" ""))
-             (use (match_operand 2 "" ""))
-             (clobber (reg:SI LR_REGNO))])]
-  ""
-  "
-{
-#if TARGET_MACHO
-  if (MACHOPIC_INDIRECT)
-    operands[0] = machopic_indirect_call_target (operands[0]);
-#endif
-
-  gcc_assert (GET_CODE (operands[0]) == MEM);
-  gcc_assert (GET_CODE (operands[1]) == CONST_INT);
+;; TLS support.
 
-  operands[0] = XEXP (operands[0], 0);
+;; Mode attributes for different ABIs.
+(define_mode_iterator TLSmode [(SI "! TARGET_64BIT") (DI "TARGET_64BIT")])
+(define_mode_attr tls_abi_suffix [(SI "32") (DI "64")])
+(define_mode_attr tls_sysv_suffix [(SI "si") (DI "di")])
+(define_mode_attr tls_insn_suffix [(SI "wz") (DI "d")])
 
-  if (DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2)
-    {
-      rs6000_call_aix (NULL_RTX, operands[0], operands[1], operands[2]);
-      DONE;
-    }
+(define_insn_and_split "tls_gd_aix<TLSmode:tls_abi_suffix>"
+  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
+        (call (mem:TLSmode (match_operand:TLSmode 3 "symbol_ref_operand" "s"))
+             (match_operand 4 "" "g")))
+   (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
+                   (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
+                  UNSPEC_TLSGD)
+   (clobber (reg:SI LR_REGNO))]
+  "HAVE_AS_TLS && (DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2)"
+{
+  if (TARGET_CMODEL != CMODEL_SMALL)
+    return "addis %0,%1,%2@got@tlsgd@ha\;addi %0,%0,%2@got@tlsgd@l\;"
+          "bl %z3\;nop";
+  else
+    return "addi %0,%1,%2@got@tlsgd\;bl %z3\;nop";
+}
+  "&& TARGET_TLS_MARKERS"
+  [(set (match_dup 0)
+       (unspec:TLSmode [(match_dup 1)
+                        (match_dup 2)]
+                       UNSPEC_TLSGD))
+   (parallel [(set (match_dup 0)
+                  (call (mem:TLSmode (match_dup 3))
+                        (match_dup 4)))
+             (unspec:TLSmode [(match_dup 2)] UNSPEC_TLSGD)
+             (clobber (reg:SI LR_REGNO))])]
+  ""
+  [(set_attr "type" "two")
+   (set (attr "length")
+     (if_then_else (ne (symbol_ref "TARGET_CMODEL") (symbol_ref "CMODEL_SMALL"))
+                  (const_int 16)
+                  (const_int 12)))])
 
-  if (GET_CODE (operands[0]) != SYMBOL_REF
-      || (DEFAULT_ABI != ABI_DARWIN && (INTVAL (operands[2]) & CALL_LONG) != 0))
+(define_insn_and_split "tls_gd_sysv<TLSmode:tls_sysv_suffix>"
+  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
+        (call (mem:TLSmode (match_operand:TLSmode 3 "symbol_ref_operand" "s"))
+             (match_operand 4 "" "g")))
+   (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
+                   (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
+                  UNSPEC_TLSGD)
+   (clobber (reg:SI LR_REGNO))]
+  "HAVE_AS_TLS && DEFAULT_ABI == ABI_V4"
+{
+  if (flag_pic)
     {
-      if (INTVAL (operands[2]) & CALL_LONG)
-       operands[0] = rs6000_longcall_ref (operands[0]);
-
-      switch (DEFAULT_ABI)
-        {
-       case ABI_V4:
-       case ABI_DARWIN:
-         operands[0] = force_reg (Pmode, operands[0]);
-         break;
-
-       default:
-         gcc_unreachable ();
-       }
+      if (TARGET_SECURE_PLT && flag_pic == 2)
+       return "addi %0,%1,%2@got@tlsgd\;bl %z3+32768@plt";
+      else
+       return "addi %0,%1,%2@got@tlsgd\;bl %z3@plt";
     }
-}")
-
-(define_expand "call_value"
-  [(parallel [(set (match_operand 0 "" "")
-                  (call (mem:SI (match_operand 1 "address_operand" ""))
-                        (match_operand 2 "" "")))
-             (use (match_operand 3 "" ""))
+  else
+    return "addi %0,%1,%2@got@tlsgd\;bl %z3";
+}
+  "&& TARGET_TLS_MARKERS"
+  [(set (match_dup 0)
+       (unspec:TLSmode [(match_dup 1)
+                        (match_dup 2)]
+                       UNSPEC_TLSGD))
+   (parallel [(set (match_dup 0)
+                  (call (mem:TLSmode (match_dup 3))
+                        (match_dup 4)))
+             (unspec:TLSmode [(match_dup 2)] UNSPEC_TLSGD)
              (clobber (reg:SI LR_REGNO))])]
   ""
+  [(set_attr "type" "two")
+   (set_attr "length" "8")])
+
+(define_insn_and_split "*tls_gd<TLSmode:tls_abi_suffix>"
+  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
+       (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
+                        (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
+                       UNSPEC_TLSGD))]
+  "HAVE_AS_TLS && TARGET_TLS_MARKERS"
+  "addi %0,%1,%2@got@tlsgd"
+  "&& TARGET_CMODEL != CMODEL_SMALL"
+  [(set (match_dup 3)
+       (high:TLSmode
+           (unspec:TLSmode [(match_dup 1) (match_dup 2)] UNSPEC_TLSGD)))
+   (set (match_dup 0)
+       (lo_sum:TLSmode (match_dup 3)
+           (unspec:TLSmode [(match_dup 1) (match_dup 2)] UNSPEC_TLSGD)))]
   "
 {
-#if TARGET_MACHO
-  if (MACHOPIC_INDIRECT)
-    operands[1] = machopic_indirect_call_target (operands[1]);
-#endif
+  operands[3] = gen_reg_rtx (TARGET_64BIT ? DImode : SImode);
+}"
+  [(set (attr "length")
+     (if_then_else (ne (symbol_ref "TARGET_CMODEL") (symbol_ref "CMODEL_SMALL"))
+                  (const_int 8)
+                  (const_int 4)))])
 
-  gcc_assert (GET_CODE (operands[1]) == MEM);
-  gcc_assert (GET_CODE (operands[2]) == CONST_INT);
+(define_insn "*tls_gd_high<TLSmode:tls_abi_suffix>"
+  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
+     (high:TLSmode
+       (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
+                       (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
+                      UNSPEC_TLSGD)))]
+  "HAVE_AS_TLS && TARGET_TLS_MARKERS && TARGET_CMODEL != CMODEL_SMALL"
+  "addis %0,%1,%2@got@tlsgd@ha"
+  [(set_attr "length" "4")])
 
-  operands[1] = XEXP (operands[1], 0);
+(define_insn "*tls_gd_low<TLSmode:tls_abi_suffix>"
+  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
+     (lo_sum:TLSmode (match_operand:TLSmode 1 "gpc_reg_operand" "b")
+       (unspec:TLSmode [(match_operand:TLSmode 3 "gpc_reg_operand" "b")
+                       (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
+                      UNSPEC_TLSGD)))]
+  "HAVE_AS_TLS && TARGET_TLS_MARKERS && TARGET_CMODEL != CMODEL_SMALL"
+  "addi %0,%1,%2@got@tlsgd@l"
+  [(set_attr "length" "4")])
 
-  if (DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2)
-    {
-      rs6000_call_aix (operands[0], operands[1], operands[2], operands[3]);
-      DONE;
-    }
+(define_insn "*tls_gd_call_aix<TLSmode:tls_abi_suffix>"
+  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
+        (call (mem:TLSmode (match_operand:TLSmode 1 "symbol_ref_operand" "s"))
+             (match_operand 2 "" "g")))
+   (unspec:TLSmode [(match_operand:TLSmode 3 "rs6000_tls_symbol_ref" "")]
+                  UNSPEC_TLSGD)
+   (clobber (reg:SI LR_REGNO))]
+  "HAVE_AS_TLS && TARGET_TLS_MARKERS
+   && (DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2)"
+  "bl %z1(%3@tlsgd)\;nop"
+  [(set_attr "type" "branch")
+   (set_attr "length" "8")])
 
-  if (GET_CODE (operands[1]) != SYMBOL_REF
-      || (DEFAULT_ABI != ABI_DARWIN && (INTVAL (operands[3]) & CALL_LONG) != 0))
+(define_insn "*tls_gd_call_sysv<TLSmode:tls_abi_suffix>"
+  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
+        (call (mem:TLSmode (match_operand:TLSmode 1 "symbol_ref_operand" "s"))
+             (match_operand 2 "" "g")))
+   (unspec:TLSmode [(match_operand:TLSmode 3 "rs6000_tls_symbol_ref" "")]
+                  UNSPEC_TLSGD)
+   (clobber (reg:SI LR_REGNO))]
+  "HAVE_AS_TLS && DEFAULT_ABI == ABI_V4 && TARGET_TLS_MARKERS"
+{
+  if (flag_pic)
     {
-      if (INTVAL (operands[3]) & CALL_LONG)
-       operands[1] = rs6000_longcall_ref (operands[1]);
-
-      switch (DEFAULT_ABI)
-        {
-       case ABI_V4:
-       case ABI_DARWIN:
-         operands[1] = force_reg (Pmode, operands[1]);
-         break;
-
-       default:
-         gcc_unreachable ();
-       }
+      if (TARGET_SECURE_PLT && flag_pic == 2)
+       return "bl %z1+32768(%3@tlsgd)@plt";
+      return "bl %z1(%3@tlsgd)@plt";
     }
-}")
-
-;; Call to function in current module.  No TOC pointer reload needed.
-;; Operand2 is nonzero if we are using the V.4 calling sequence and
-;; either the function was not prototyped, or it was prototyped as a
-;; variable argument function.  It is > 0 if FP registers were passed
-;; and < 0 if they were not.
+  return "bl %z1(%3@tlsgd)";
+}
+  [(set_attr "type" "branch")
+   (set_attr "length" "4")])
 
-(define_insn "*call_local32"
-  [(call (mem:SI (match_operand:SI 0 "current_file_function_operand" "s,s"))
-        (match_operand 1 "" "g,g"))
-   (use (match_operand:SI 2 "immediate_operand" "O,n"))
+(define_insn_and_split "tls_ld_aix<TLSmode:tls_abi_suffix>"
+  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
+        (call (mem:TLSmode (match_operand:TLSmode 2 "symbol_ref_operand" "s"))
+             (match_operand 3 "" "g")))
+   (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")]
+                  UNSPEC_TLSLD)
    (clobber (reg:SI LR_REGNO))]
-  "(INTVAL (operands[2]) & CALL_LONG) == 0"
-  "*
+  "HAVE_AS_TLS && (DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2)"
 {
-  if (INTVAL (operands[2]) & CALL_V4_SET_FP_ARGS)
-    output_asm_insn (\"crxor 6,6,6\", operands);
+  if (TARGET_CMODEL != CMODEL_SMALL)
+    return "addis %0,%1,%&@got@tlsld@ha\;addi %0,%0,%&@got@tlsld@l\;"
+          "bl %z2\;nop";
+  else
+    return "addi %0,%1,%&@got@tlsld\;bl %z2\;nop";
+}
+  "&& TARGET_TLS_MARKERS"
+  [(set (match_dup 0)
+       (unspec:TLSmode [(match_dup 1)]
+                       UNSPEC_TLSLD))
+   (parallel [(set (match_dup 0)
+                  (call (mem:TLSmode (match_dup 2))
+                        (match_dup 3)))
+             (unspec:TLSmode [(const_int 0)] UNSPEC_TLSLD)
+             (clobber (reg:SI LR_REGNO))])]
+  ""
+  [(set_attr "type" "two")
+   (set (attr "length")
+     (if_then_else (ne (symbol_ref "TARGET_CMODEL") (symbol_ref "CMODEL_SMALL"))
+                  (const_int 16)
+                  (const_int 12)))])
 
-  else if (INTVAL (operands[2]) & CALL_V4_CLEAR_FP_ARGS)
-    output_asm_insn (\"creqv 6,6,6\", operands);
+(define_insn_and_split "tls_ld_sysv<TLSmode:tls_sysv_suffix>"
+  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
+        (call (mem:TLSmode (match_operand:TLSmode 2 "symbol_ref_operand" "s"))
+             (match_operand 3 "" "g")))
+   (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")]
+                  UNSPEC_TLSLD)
+   (clobber (reg:SI LR_REGNO))]
+  "HAVE_AS_TLS && DEFAULT_ABI == ABI_V4"
+{
+  if (flag_pic)
+    {
+      if (TARGET_SECURE_PLT && flag_pic == 2)
+       return "addi %0,%1,%&@got@tlsld\;bl %z2+32768@plt";
+      else
+       return "addi %0,%1,%&@got@tlsld\;bl %z2@plt";
+    }
+  else
+    return "addi %0,%1,%&@got@tlsld\;bl %z2";
+}
+  "&& TARGET_TLS_MARKERS"
+  [(set (match_dup 0)
+       (unspec:TLSmode [(match_dup 1)]
+                       UNSPEC_TLSLD))
+   (parallel [(set (match_dup 0)
+                  (call (mem:TLSmode (match_dup 2))
+                        (match_dup 3)))
+             (unspec:TLSmode [(const_int 0)] UNSPEC_TLSLD)
+             (clobber (reg:SI LR_REGNO))])]
+  ""
+  [(set_attr "length" "8")])
 
-  return (DEFAULT_ABI == ABI_V4 && flag_pic) ? \"bl %z0@local\" : \"bl %z0\";
+(define_insn_and_split "*tls_ld<TLSmode:tls_abi_suffix>"
+  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
+       (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")]
+                       UNSPEC_TLSLD))]
+  "HAVE_AS_TLS && TARGET_TLS_MARKERS"
+  "addi %0,%1,%&@got@tlsld"
+  "&& TARGET_CMODEL != CMODEL_SMALL"
+  [(set (match_dup 2)
+       (high:TLSmode
+           (unspec:TLSmode [(const_int 0) (match_dup 1)] UNSPEC_TLSLD)))
+   (set (match_dup 0)
+       (lo_sum:TLSmode (match_dup 2)
+           (unspec:TLSmode [(const_int 0) (match_dup 1)] UNSPEC_TLSLD)))]
+  "
+{
+  operands[2] = gen_reg_rtx (TARGET_64BIT ? DImode : SImode);
 }"
+  [(set (attr "length")
+     (if_then_else (ne (symbol_ref "TARGET_CMODEL") (symbol_ref "CMODEL_SMALL"))
+                  (const_int 8)
+                  (const_int 4)))])
+
+(define_insn "*tls_ld_high<TLSmode:tls_abi_suffix>"
+  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
+     (high:TLSmode
+       (unspec:TLSmode [(const_int 0)
+                       (match_operand:TLSmode 1 "gpc_reg_operand" "b")]
+                      UNSPEC_TLSLD)))]
+  "HAVE_AS_TLS && TARGET_TLS_MARKERS && TARGET_CMODEL != CMODEL_SMALL"
+  "addis %0,%1,%&@got@tlsld@ha"
+  [(set_attr "length" "4")])
+
+(define_insn "*tls_ld_low<TLSmode:tls_abi_suffix>"
+  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
+     (lo_sum:TLSmode (match_operand:TLSmode 1 "gpc_reg_operand" "b")
+       (unspec:TLSmode [(const_int 0)
+                        (match_operand:TLSmode 2 "gpc_reg_operand" "b")]
+                       UNSPEC_TLSLD)))]
+  "HAVE_AS_TLS && TARGET_TLS_MARKERS && TARGET_CMODEL != CMODEL_SMALL"
+  "addi %0,%1,%&@got@tlsld@l"
+  [(set_attr "length" "4")])
+
+(define_insn "*tls_ld_call_aix<TLSmode:tls_abi_suffix>"
+  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
+        (call (mem:TLSmode (match_operand:TLSmode 1 "symbol_ref_operand" "s"))
+             (match_operand 2 "" "g")))
+   (unspec:TLSmode [(const_int 0)] UNSPEC_TLSLD)
+   (clobber (reg:SI LR_REGNO))]
+  "HAVE_AS_TLS && TARGET_TLS_MARKERS
+   && (DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2)"
+  "bl %z1(%&@tlsld)\;nop"
   [(set_attr "type" "branch")
-   (set_attr "length" "4,8")])
+   (set_attr "length" "8")])
 
-(define_insn "*call_local64"
-  [(call (mem:SI (match_operand:DI 0 "current_file_function_operand" "s,s"))
-        (match_operand 1 "" "g,g"))
-   (use (match_operand:SI 2 "immediate_operand" "O,n"))
+(define_insn "*tls_ld_call_sysv<TLSmode:tls_abi_suffix>"
+  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
+        (call (mem:TLSmode (match_operand:TLSmode 1 "symbol_ref_operand" "s"))
+             (match_operand 2 "" "g")))
+   (unspec:TLSmode [(const_int 0)] UNSPEC_TLSLD)
    (clobber (reg:SI LR_REGNO))]
-  "TARGET_64BIT && (INTVAL (operands[2]) & CALL_LONG) == 0"
-  "*
+  "HAVE_AS_TLS && DEFAULT_ABI == ABI_V4 && TARGET_TLS_MARKERS"
 {
-  if (INTVAL (operands[2]) & CALL_V4_SET_FP_ARGS)
-    output_asm_insn (\"crxor 6,6,6\", operands);
+  if (flag_pic)
+    {
+      if (TARGET_SECURE_PLT && flag_pic == 2)
+       return "bl %z1+32768(%&@tlsld)@plt";
+      return "bl %z1(%&@tlsld)@plt";
+    }
+  return "bl %z1(%&@tlsld)";
+}
+  [(set_attr "type" "branch")
+   (set_attr "length" "4")])
+
+(define_insn "tls_dtprel_<TLSmode:tls_abi_suffix>"
+  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=r")
+       (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
+                        (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
+                       UNSPEC_TLSDTPREL))]
+  "HAVE_AS_TLS"
+  "addi %0,%1,%2@dtprel")
 
-  else if (INTVAL (operands[2]) & CALL_V4_CLEAR_FP_ARGS)
-    output_asm_insn (\"creqv 6,6,6\", operands);
+(define_insn "tls_dtprel_ha_<TLSmode:tls_abi_suffix>"
+  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=r")
+       (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
+                        (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
+                       UNSPEC_TLSDTPRELHA))]
+  "HAVE_AS_TLS"
+  "addis %0,%1,%2@dtprel@ha")
 
-  return (DEFAULT_ABI == ABI_V4 && flag_pic) ? \"bl %z0@local\" : \"bl %z0\";
-}"
-  [(set_attr "type" "branch")
-   (set_attr "length" "4,8")])
+(define_insn "tls_dtprel_lo_<TLSmode:tls_abi_suffix>"
+  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=r")
+       (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
+                        (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
+                       UNSPEC_TLSDTPRELLO))]
+  "HAVE_AS_TLS"
+  "addi %0,%1,%2@dtprel@l")
 
-(define_insn "*call_value_local32"
-  [(set (match_operand 0 "" "")
-       (call (mem:SI (match_operand:SI 1 "current_file_function_operand" "s,s"))
-             (match_operand 2 "" "g,g")))
-   (use (match_operand:SI 3 "immediate_operand" "O,n"))
-   (clobber (reg:SI LR_REGNO))]
-  "(INTVAL (operands[3]) & CALL_LONG) == 0"
-  "*
+(define_insn_and_split "tls_got_dtprel_<TLSmode:tls_abi_suffix>"
+  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=r")
+       (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
+                        (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
+                       UNSPEC_TLSGOTDTPREL))]
+  "HAVE_AS_TLS"
+  "l<TLSmode:tls_insn_suffix> %0,%2@got@dtprel(%1)"
+  "&& TARGET_CMODEL != CMODEL_SMALL"
+  [(set (match_dup 3)
+       (high:TLSmode
+           (unspec:TLSmode [(match_dup 1) (match_dup 2)] UNSPEC_TLSGOTDTPREL)))
+   (set (match_dup 0)
+       (lo_sum:TLSmode (match_dup 3)
+           (unspec:TLSmode [(match_dup 1) (match_dup 2)] UNSPEC_TLSGOTDTPREL)))]
+  "
 {
-  if (INTVAL (operands[3]) & CALL_V4_SET_FP_ARGS)
-    output_asm_insn (\"crxor 6,6,6\", operands);
+  operands[3] = gen_reg_rtx (TARGET_64BIT ? DImode : SImode);
+}"
+  [(set (attr "length")
+     (if_then_else (ne (symbol_ref "TARGET_CMODEL") (symbol_ref "CMODEL_SMALL"))
+                  (const_int 8)
+                  (const_int 4)))])
 
-  else if (INTVAL (operands[3]) & CALL_V4_CLEAR_FP_ARGS)
-    output_asm_insn (\"creqv 6,6,6\", operands);
+(define_insn "*tls_got_dtprel_high<TLSmode:tls_abi_suffix>"
+  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
+     (high:TLSmode
+       (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
+                       (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
+                      UNSPEC_TLSGOTDTPREL)))]
+  "HAVE_AS_TLS && TARGET_CMODEL != CMODEL_SMALL"
+  "addis %0,%1,%2@got@dtprel@ha"
+  [(set_attr "length" "4")])
 
-  return (DEFAULT_ABI == ABI_V4 && flag_pic) ? \"bl %z1@local\" : \"bl %z1\";
-}"
-  [(set_attr "type" "branch")
-   (set_attr "length" "4,8")])
+(define_insn "*tls_got_dtprel_low<TLSmode:tls_abi_suffix>"
+  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=r")
+     (lo_sum:TLSmode (match_operand:TLSmode 1 "gpc_reg_operand" "b")
+        (unspec:TLSmode [(match_operand:TLSmode 3 "gpc_reg_operand" "b")
+                         (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
+                        UNSPEC_TLSGOTDTPREL)))]
+  "HAVE_AS_TLS && TARGET_CMODEL != CMODEL_SMALL"
+  "l<TLSmode:tls_insn_suffix> %0,%2@got@dtprel@l(%1)"
+  [(set_attr "length" "4")])
 
+(define_insn "tls_tprel_<TLSmode:tls_abi_suffix>"
+  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=r")
+       (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
+                        (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
+                       UNSPEC_TLSTPREL))]
+  "HAVE_AS_TLS"
+  "addi %0,%1,%2@tprel")
 
-(define_insn "*call_value_local64"
-  [(set (match_operand 0 "" "")
-       (call (mem:SI (match_operand:DI 1 "current_file_function_operand" "s,s"))
-             (match_operand 2 "" "g,g")))
-   (use (match_operand:SI 3 "immediate_operand" "O,n"))
-   (clobber (reg:SI LR_REGNO))]
-  "TARGET_64BIT && (INTVAL (operands[3]) & CALL_LONG) == 0"
-  "*
-{
-  if (INTVAL (operands[3]) & CALL_V4_SET_FP_ARGS)
-    output_asm_insn (\"crxor 6,6,6\", operands);
+(define_insn "tls_tprel_ha_<TLSmode:tls_abi_suffix>"
+  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=r")
+       (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
+                        (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
+                       UNSPEC_TLSTPRELHA))]
+  "HAVE_AS_TLS"
+  "addis %0,%1,%2@tprel@ha")
 
-  else if (INTVAL (operands[3]) & CALL_V4_CLEAR_FP_ARGS)
-    output_asm_insn (\"creqv 6,6,6\", operands);
+(define_insn "tls_tprel_lo_<TLSmode:tls_abi_suffix>"
+  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=r")
+       (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
+                        (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
+                       UNSPEC_TLSTPRELLO))]
+  "HAVE_AS_TLS"
+  "addi %0,%1,%2@tprel@l")
 
-  return (DEFAULT_ABI == ABI_V4 && flag_pic) ? \"bl %z1@local\" : \"bl %z1\";
+;; "b" output constraint here and on tls_tls input to support linker tls
+;; optimization.  The linker may edit the instructions emitted by a
+;; tls_got_tprel/tls_tls pair to addis,addi.
+(define_insn_and_split "tls_got_tprel_<TLSmode:tls_abi_suffix>"
+  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
+       (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
+                        (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
+                       UNSPEC_TLSGOTTPREL))]
+  "HAVE_AS_TLS"
+  "l<TLSmode:tls_insn_suffix> %0,%2@got@tprel(%1)"
+  "&& TARGET_CMODEL != CMODEL_SMALL"
+  [(set (match_dup 3)
+       (high:TLSmode
+           (unspec:TLSmode [(match_dup 1) (match_dup 2)] UNSPEC_TLSGOTTPREL)))
+   (set (match_dup 0)
+       (lo_sum:TLSmode (match_dup 3)
+           (unspec:TLSmode [(match_dup 1) (match_dup 2)] UNSPEC_TLSGOTTPREL)))]
+  "
+{
+  operands[3] = gen_reg_rtx (TARGET_64BIT ? DImode : SImode);
 }"
-  [(set_attr "type" "branch")
-   (set_attr "length" "4,8")])
+  [(set (attr "length")
+     (if_then_else (ne (symbol_ref "TARGET_CMODEL") (symbol_ref "CMODEL_SMALL"))
+                  (const_int 8)
+                  (const_int 4)))])
 
+(define_insn "*tls_got_tprel_high<TLSmode:tls_abi_suffix>"
+  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=b")
+     (high:TLSmode
+       (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
+                       (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
+                      UNSPEC_TLSGOTTPREL)))]
+  "HAVE_AS_TLS && TARGET_CMODEL != CMODEL_SMALL"
+  "addis %0,%1,%2@got@tprel@ha"
+  [(set_attr "length" "4")])
 
-;; A function pointer under System V is just a normal pointer
-;; operands[0] is the function pointer
-;; operands[1] is the stack size to clean up
-;; operands[2] is the value FUNCTION_ARG returns for the VOID argument
-;; which indicates how to set cr1
+(define_insn "*tls_got_tprel_low<TLSmode:tls_abi_suffix>"
+  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=r")
+     (lo_sum:TLSmode (match_operand:TLSmode 1 "gpc_reg_operand" "b")
+        (unspec:TLSmode [(match_operand:TLSmode 3 "gpc_reg_operand" "b")
+                         (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
+                        UNSPEC_TLSGOTTPREL)))]
+  "HAVE_AS_TLS && TARGET_CMODEL != CMODEL_SMALL"
+  "l<TLSmode:tls_insn_suffix> %0,%2@got@tprel@l(%1)"
+  [(set_attr "length" "4")])
 
-(define_insn "*call_indirect_nonlocal_sysv<mode>"
-  [(call (mem:SI (match_operand:P 0 "register_operand" "c,*l,c,*l"))
-        (match_operand 1 "" "g,g,g,g"))
-   (use (match_operand:SI 2 "immediate_operand" "O,O,n,n"))
+(define_insn "tls_tls_<TLSmode:tls_abi_suffix>"
+  [(set (match_operand:TLSmode 0 "gpc_reg_operand" "=r")
+       (unspec:TLSmode [(match_operand:TLSmode 1 "gpc_reg_operand" "b")
+                        (match_operand:TLSmode 2 "rs6000_tls_symbol_ref" "")]
+                       UNSPEC_TLSTLS))]
+  "TARGET_ELF && HAVE_AS_TLS"
+  "add %0,%1,%2@tls")
+
+(define_expand "tls_get_tpointer"
+  [(set (match_operand:SI 0 "gpc_reg_operand" "")
+       (unspec:SI [(const_int 0)] UNSPEC_TLSTLS))]
+  "TARGET_XCOFF && HAVE_AS_TLS"
+  "
+{
+  emit_insn (gen_tls_get_tpointer_internal ());
+  emit_move_insn (operands[0], gen_rtx_REG (SImode, 3));
+  DONE;
+}")
+
+(define_insn "tls_get_tpointer_internal"
+  [(set (reg:SI 3)
+       (unspec:SI [(const_int 0)] UNSPEC_TLSTLS))
    (clobber (reg:SI LR_REGNO))]
-  "DEFAULT_ABI == ABI_V4
-   || DEFAULT_ABI == ABI_DARWIN"
+  "TARGET_XCOFF && HAVE_AS_TLS"
+  "bla __get_tpointer")
+
+(define_expand "tls_get_addr<mode>"
+  [(set (match_operand:P 0 "gpc_reg_operand" "")
+       (unspec:P [(match_operand:P 1 "gpc_reg_operand" "")
+                   (match_operand:P 2 "gpc_reg_operand" "")] UNSPEC_TLSTLS))]
+  "TARGET_XCOFF && HAVE_AS_TLS"
+  "
 {
-  if (INTVAL (operands[2]) & CALL_V4_SET_FP_ARGS)
-    output_asm_insn ("crxor 6,6,6", operands);
+  emit_move_insn (gen_rtx_REG (Pmode, 3), operands[1]);
+  emit_move_insn (gen_rtx_REG (Pmode, 4), operands[2]);
+  emit_insn (gen_tls_get_addr_internal<mode> ());
+  emit_move_insn (operands[0], gen_rtx_REG (Pmode, 3));
+  DONE;
+}")
 
-  else if (INTVAL (operands[2]) & CALL_V4_CLEAR_FP_ARGS)
-    output_asm_insn ("creqv 6,6,6", operands);
+(define_insn "tls_get_addr_internal<mode>"
+  [(set (reg:P 3)
+       (unspec:P [(reg:P 3) (reg:P 4)] UNSPEC_TLSTLS))
+   (clobber (reg:P 0))
+   (clobber (reg:P 4))
+   (clobber (reg:P 5))
+   (clobber (reg:P 11))
+   (clobber (reg:CC CR0_REGNO))
+   (clobber (reg:P LR_REGNO))]
+  "TARGET_XCOFF && HAVE_AS_TLS"
+  "bla __tls_get_addr")
+\f
+;; Next come insns related to the calling sequence.
+;;
+;; First, an insn to allocate new stack space for dynamic use (e.g., alloca).
+;; We move the back-chain and decrement the stack pointer.
 
-  return "b%T0l";
-}
-  [(set_attr "type" "jmpreg,jmpreg,jmpreg,jmpreg")
-   (set_attr "length" "4,4,8,8")])
+(define_expand "allocate_stack"
+  [(set (match_operand 0 "gpc_reg_operand" "")
+       (minus (reg 1) (match_operand 1 "reg_or_short_operand" "")))
+   (set (reg 1)
+       (minus (reg 1) (match_dup 1)))]
+  ""
+  "
+{ rtx chain = gen_reg_rtx (Pmode);
+  rtx stack_bot = gen_rtx_MEM (Pmode, stack_pointer_rtx);
+  rtx neg_op0;
+  rtx insn, par, set, mem;
 
-(define_insn_and_split "*call_nonlocal_sysv<mode>"
-  [(call (mem:SI (match_operand:P 0 "symbol_ref_operand" "s,s"))
-        (match_operand 1 "" "g,g"))
-   (use (match_operand:SI 2 "immediate_operand" "O,n"))
-   (clobber (reg:SI LR_REGNO))]
-  "(DEFAULT_ABI == ABI_DARWIN
-   || (DEFAULT_ABI == ABI_V4
-       && (INTVAL (operands[2]) & CALL_LONG) == 0))"
-{
-  if (INTVAL (operands[2]) & CALL_V4_SET_FP_ARGS)
-    output_asm_insn ("crxor 6,6,6", operands);
+  emit_move_insn (chain, stack_bot);
 
-  else if (INTVAL (operands[2]) & CALL_V4_CLEAR_FP_ARGS)
-    output_asm_insn ("creqv 6,6,6", operands);
+  /* Check stack bounds if necessary.  */
+  if (crtl->limit_stack)
+    {
+      rtx available;
+      available = expand_binop (Pmode, sub_optab,
+                               stack_pointer_rtx, stack_limit_rtx,
+                               NULL_RTX, 1, OPTAB_WIDEN);
+      emit_insn (gen_cond_trap (LTU, available, operands[1], const0_rtx));
+    }
 
-#if TARGET_MACHO
-  return output_call(insn, operands, 0, 2);
-#else
-  if (DEFAULT_ABI == ABI_V4 && flag_pic)
+  if (GET_CODE (operands[1]) != CONST_INT
+      || INTVAL (operands[1]) < -32767
+      || INTVAL (operands[1]) > 32768)
     {
-      gcc_assert (!TARGET_SECURE_PLT);
-      return "bl %z0@plt";
+      neg_op0 = gen_reg_rtx (Pmode);
+      if (TARGET_32BIT)
+       emit_insn (gen_negsi2 (neg_op0, operands[1]));
+      else
+       emit_insn (gen_negdi2 (neg_op0, operands[1]));
     }
   else
-    return "bl %z0";
-#endif
-}
-  "DEFAULT_ABI == ABI_V4
-   && TARGET_SECURE_PLT && flag_pic && !SYMBOL_REF_LOCAL_P (operands[0])
-   && (INTVAL (operands[2]) & CALL_LONG) == 0"
-  [(parallel [(call (mem:SI (match_dup 0))
-                   (match_dup 1))
-             (use (match_dup 2))
-             (use (match_dup 3))
-             (clobber (reg:SI LR_REGNO))])]
-{
-  operands[3] = pic_offset_table_rtx;
-}
-  [(set_attr "type" "branch,branch")
-   (set_attr "length" "4,8")])
-
-(define_insn "*call_nonlocal_sysv_secure<mode>"
-  [(call (mem:SI (match_operand:P 0 "symbol_ref_operand" "s,s"))
-        (match_operand 1 "" "g,g"))
-   (use (match_operand:SI 2 "immediate_operand" "O,n"))
-   (use (match_operand:SI 3 "register_operand" "r,r"))
-   (clobber (reg:SI LR_REGNO))]
-  "(DEFAULT_ABI == ABI_V4
-    && TARGET_SECURE_PLT && flag_pic && !SYMBOL_REF_LOCAL_P (operands[0])
-    && (INTVAL (operands[2]) & CALL_LONG) == 0)"
-{
-  if (INTVAL (operands[2]) & CALL_V4_SET_FP_ARGS)
-    output_asm_insn ("crxor 6,6,6", operands);
+    neg_op0 = GEN_INT (- INTVAL (operands[1]));
 
-  else if (INTVAL (operands[2]) & CALL_V4_CLEAR_FP_ARGS)
-    output_asm_insn ("creqv 6,6,6", operands);
+  insn = emit_insn ((* ((TARGET_32BIT) ? gen_movsi_update_stack
+                                      : gen_movdi_di_update_stack))
+                       (stack_pointer_rtx, stack_pointer_rtx, neg_op0,
+                        chain));
+  /* Since we didn't use gen_frame_mem to generate the MEM, grab
+     it now and set the alias set/attributes. The above gen_*_update
+     calls will generate a PARALLEL with the MEM set being the first
+     operation. */
+  par = PATTERN (insn);
+  gcc_assert (GET_CODE (par) == PARALLEL);
+  set = XVECEXP (par, 0, 0);
+  gcc_assert (GET_CODE (set) == SET);
+  mem = SET_DEST (set);
+  gcc_assert (MEM_P (mem));
+  MEM_NOTRAP_P (mem) = 1;
+  set_mem_alias_set (mem, get_frame_alias_set ());
 
-  if (flag_pic == 2)
-    /* The magic 32768 offset here and in the other sysv call insns
-       corresponds to the offset of r30 in .got2, as given by LCTOC1.
-       See sysv4.h:toc_section.  */
-    return "bl %z0+32768@plt";
-  else
-    return "bl %z0@plt";
-}
-  [(set_attr "type" "branch,branch")
-   (set_attr "length" "4,8")])
+  emit_move_insn (operands[0], virtual_stack_dynamic_rtx);
+  DONE;
+}")
 
-(define_insn "*call_value_indirect_nonlocal_sysv<mode>"
-  [(set (match_operand 0 "" "")
-       (call (mem:SI (match_operand:P 1 "register_operand" "c,*l,c,*l"))
-             (match_operand 2 "" "g,g,g,g")))
-   (use (match_operand:SI 3 "immediate_operand" "O,O,n,n"))
-   (clobber (reg:SI LR_REGNO))]
-  "DEFAULT_ABI == ABI_V4
-   || DEFAULT_ABI == ABI_DARWIN"
-{
-  if (INTVAL (operands[3]) & CALL_V4_SET_FP_ARGS)
-    output_asm_insn ("crxor 6,6,6", operands);
+;; These patterns say how to save and restore the stack pointer.  We need not
+;; save the stack pointer at function level since we are careful to
+;; preserve the backchain.  At block level, we have to restore the backchain
+;; when we restore the stack pointer.
+;;
+;; For nonlocal gotos, we must save both the stack pointer and its
+;; backchain and restore both.  Note that in the nonlocal case, the
+;; save area is a memory location.
 
-  else if (INTVAL (operands[3]) & CALL_V4_CLEAR_FP_ARGS)
-    output_asm_insn ("creqv 6,6,6", operands);
+(define_expand "save_stack_function"
+  [(match_operand 0 "any_operand" "")
+   (match_operand 1 "any_operand" "")]
+  ""
+  "DONE;")
 
-  return "b%T1l";
-}
-  [(set_attr "type" "jmpreg,jmpreg,jmpreg,jmpreg")
-   (set_attr "length" "4,4,8,8")])
+(define_expand "restore_stack_function"
+  [(match_operand 0 "any_operand" "")
+   (match_operand 1 "any_operand" "")]
+  ""
+  "DONE;")
 
-(define_insn_and_split "*call_value_nonlocal_sysv<mode>"
-  [(set (match_operand 0 "" "")
-       (call (mem:SI (match_operand:P 1 "symbol_ref_operand" "s,s"))
-             (match_operand 2 "" "g,g")))
-   (use (match_operand:SI 3 "immediate_operand" "O,n"))
-   (clobber (reg:SI LR_REGNO))]
-  "(DEFAULT_ABI == ABI_DARWIN
-   || (DEFAULT_ABI == ABI_V4
-       && (INTVAL (operands[3]) & CALL_LONG) == 0))"
+;; Adjust stack pointer (op0) to a new value (op1).
+;; First copy old stack backchain to new location, and ensure that the
+;; scheduler won't reorder the sp assignment before the backchain write.
+(define_expand "restore_stack_block"
+  [(set (match_dup 2) (match_dup 3))
+   (set (match_dup 4) (match_dup 2))
+   (match_dup 5)
+   (set (match_operand 0 "register_operand" "")
+       (match_operand 1 "register_operand" ""))]
+  ""
+  "
 {
-  if (INTVAL (operands[3]) & CALL_V4_SET_FP_ARGS)
-    output_asm_insn ("crxor 6,6,6", operands);
+  rtvec p;
 
-  else if (INTVAL (operands[3]) & CALL_V4_CLEAR_FP_ARGS)
-    output_asm_insn ("creqv 6,6,6", operands);
+  operands[1] = force_reg (Pmode, operands[1]);
+  operands[2] = gen_reg_rtx (Pmode);
+  operands[3] = gen_frame_mem (Pmode, operands[0]);
+  operands[4] = gen_frame_mem (Pmode, operands[1]);
+  p = rtvec_alloc (1);
+  RTVEC_ELT (p, 0) = gen_rtx_SET (gen_frame_mem (BLKmode, operands[0]),
+                                 const0_rtx);
+  operands[5] = gen_rtx_PARALLEL (VOIDmode, p);
+}")
 
-#if TARGET_MACHO
-  return output_call(insn, operands, 1, 3);
-#else
-  if (DEFAULT_ABI == ABI_V4 && flag_pic)
-    {
-      gcc_assert (!TARGET_SECURE_PLT);
-      return "bl %z1@plt";
-    }
-  else
-    return "bl %z1";
-#endif
-}
-  "DEFAULT_ABI == ABI_V4
-   && TARGET_SECURE_PLT && flag_pic && !SYMBOL_REF_LOCAL_P (operands[1])
-   && (INTVAL (operands[3]) & CALL_LONG) == 0"
-  [(parallel [(set (match_dup 0)
-                  (call (mem:SI (match_dup 1))
-                        (match_dup 2)))
-             (use (match_dup 3))
-             (use (match_dup 4))
-             (clobber (reg:SI LR_REGNO))])]
+(define_expand "save_stack_nonlocal"
+  [(set (match_dup 3) (match_dup 4))
+   (set (match_operand 0 "memory_operand" "") (match_dup 3))
+   (set (match_dup 2) (match_operand 1 "register_operand" ""))]
+  ""
+  "
 {
-  operands[4] = pic_offset_table_rtx;
-}
-  [(set_attr "type" "branch,branch")
-   (set_attr "length" "4,8")])
+  int units_per_word = (TARGET_32BIT) ? 4 : 8;
 
-(define_insn "*call_value_nonlocal_sysv_secure<mode>"
-  [(set (match_operand 0 "" "")
-       (call (mem:SI (match_operand:P 1 "symbol_ref_operand" "s,s"))
-             (match_operand 2 "" "g,g")))
-   (use (match_operand:SI 3 "immediate_operand" "O,n"))
-   (use (match_operand:SI 4 "register_operand" "r,r"))
-   (clobber (reg:SI LR_REGNO))]
-  "(DEFAULT_ABI == ABI_V4
-    && TARGET_SECURE_PLT && flag_pic && !SYMBOL_REF_LOCAL_P (operands[1])
-    && (INTVAL (operands[3]) & CALL_LONG) == 0)"
+  /* Copy the backchain to the first word, sp to the second.  */
+  operands[0] = adjust_address_nv (operands[0], Pmode, 0);
+  operands[2] = adjust_address_nv (operands[0], Pmode, units_per_word);
+  operands[3] = gen_reg_rtx (Pmode);
+  operands[4] = gen_frame_mem (Pmode, operands[1]);
+}")
+
+(define_expand "restore_stack_nonlocal"
+  [(set (match_dup 2) (match_operand 1 "memory_operand" ""))
+   (set (match_dup 3) (match_dup 4))
+   (set (match_dup 5) (match_dup 2))
+   (match_dup 6)
+   (set (match_operand 0 "register_operand" "") (match_dup 3))]
+  ""
+  "
 {
-  if (INTVAL (operands[3]) & CALL_V4_SET_FP_ARGS)
-    output_asm_insn ("crxor 6,6,6", operands);
+  int units_per_word = (TARGET_32BIT) ? 4 : 8;
+  rtvec p;
 
-  else if (INTVAL (operands[3]) & CALL_V4_CLEAR_FP_ARGS)
-    output_asm_insn ("creqv 6,6,6", operands);
+  /* Restore the backchain from the first word, sp from the second.  */
+  operands[2] = gen_reg_rtx (Pmode);
+  operands[3] = gen_reg_rtx (Pmode);
+  operands[1] = adjust_address_nv (operands[1], Pmode, 0);
+  operands[4] = adjust_address_nv (operands[1], Pmode, units_per_word);
+  operands[5] = gen_frame_mem (Pmode, operands[3]);
+  p = rtvec_alloc (1);
+  RTVEC_ELT (p, 0) = gen_rtx_SET (gen_frame_mem (BLKmode, operands[0]),
+                                 const0_rtx);
+  operands[6] = gen_rtx_PARALLEL (VOIDmode, p);
+}")
+\f
+;; TOC register handling.
 
-  if (flag_pic == 2)
-    return "bl %z1+32768@plt";
-  else
-    return "bl %z1@plt";
-}
-  [(set_attr "type" "branch,branch")
-   (set_attr "length" "4,8")])
+;; Code to initialize the TOC register...
+
+(define_insn "load_toc_aix_si"
+  [(parallel [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
+                  (unspec:SI [(const_int 0)] UNSPEC_TOC))
+             (use (reg:SI 2))])]
+  "(DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2) && TARGET_32BIT"
+  "*
+{
+  char buf[30];
+  ASM_GENERATE_INTERNAL_LABEL (buf, \"LCTOC\", 1);
+  operands[1] = gen_rtx_SYMBOL_REF (Pmode, ggc_strdup (buf));
+  operands[2] = gen_rtx_REG (Pmode, 2);
+  return \"lwz %0,%1(%2)\";
+}"
+  [(set_attr "type" "load")
+   (set_attr "update" "no")
+   (set_attr "indexed" "no")])
+
+(define_insn "load_toc_aix_di"
+  [(parallel [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
+                  (unspec:DI [(const_int 0)] UNSPEC_TOC))
+             (use (reg:DI 2))])]
+  "(DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2) && TARGET_64BIT"
+  "*
+{
+  char buf[30];
+#ifdef TARGET_RELOCATABLE
+  ASM_GENERATE_INTERNAL_LABEL (buf, \"LCTOC\",
+                              !TARGET_MINIMAL_TOC || TARGET_RELOCATABLE);
+#else
+  ASM_GENERATE_INTERNAL_LABEL (buf, \"LCTOC\", 1);
+#endif
+  if (TARGET_ELF)
+    strcat (buf, \"@toc\");
+  operands[1] = gen_rtx_SYMBOL_REF (Pmode, ggc_strdup (buf));
+  operands[2] = gen_rtx_REG (Pmode, 2);
+  return \"ld %0,%1(%2)\";
+}"
+  [(set_attr "type" "load")
+   (set_attr "update" "no")
+   (set_attr "indexed" "no")])
 
+(define_insn "load_toc_v4_pic_si"
+  [(set (reg:SI LR_REGNO)
+       (unspec:SI [(const_int 0)] UNSPEC_TOC))]
+  "DEFAULT_ABI == ABI_V4 && flag_pic == 1 && TARGET_32BIT"
+  "bl _GLOBAL_OFFSET_TABLE_@local-4"
+  [(set_attr "type" "branch")
+   (set_attr "length" "4")])
 
-;; Call to AIX abi function in the same module.
+(define_expand "load_toc_v4_PIC_1"
+  [(parallel [(set (reg:SI LR_REGNO)
+                  (match_operand:SI 0 "immediate_operand" "s"))
+             (use (unspec [(match_dup 0)] UNSPEC_TOC))])]
+  "TARGET_ELF && DEFAULT_ABI == ABI_V4
+   && (flag_pic == 2 || (flag_pic && TARGET_SECURE_PLT))"
+  "")
 
-(define_insn "*call_local_aix<mode>"
-  [(call (mem:SI (match_operand:P 0 "current_file_function_operand" "s"))
-        (match_operand 1 "" "g"))
-   (clobber (reg:P LR_REGNO))]
-  "DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2"
-  "bl %z0"
+(define_insn "load_toc_v4_PIC_1_normal"
+  [(set (reg:SI LR_REGNO)
+       (match_operand:SI 0 "immediate_operand" "s"))
+   (use (unspec [(match_dup 0)] UNSPEC_TOC))]
+  "!TARGET_LINK_STACK && TARGET_ELF && DEFAULT_ABI == ABI_V4
+   && (flag_pic == 2 || (flag_pic && TARGET_SECURE_PLT))"
+  "bcl 20,31,%0\\n%0:"
   [(set_attr "type" "branch")
    (set_attr "length" "4")])
 
-(define_insn "*call_value_local_aix<mode>"
-  [(set (match_operand 0 "" "")
-       (call (mem:SI (match_operand:P 1 "current_file_function_operand" "s"))
-             (match_operand 2 "" "g")))
-   (clobber (reg:P LR_REGNO))]
-  "DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2"
-  "bl %z1"
+(define_insn "load_toc_v4_PIC_1_476"
+  [(set (reg:SI LR_REGNO)
+       (match_operand:SI 0 "immediate_operand" "s"))
+   (use (unspec [(match_dup 0)] UNSPEC_TOC))]
+  "TARGET_LINK_STACK && TARGET_ELF && DEFAULT_ABI == ABI_V4
+   && (flag_pic == 2 || (flag_pic && TARGET_SECURE_PLT))"
+  "*
+{
+  char name[32];
+  static char templ[32];
+
+  get_ppc476_thunk_name (name);
+  sprintf (templ, \"bl %s\\n%%0:\", name);
+  return templ;
+}"
   [(set_attr "type" "branch")
    (set_attr "length" "4")])
 
-;; Call to AIX abi function which may be in another module.
-;; Restore the TOC pointer (r2) after the call.
+(define_expand "load_toc_v4_PIC_1b"
+  [(parallel [(set (reg:SI LR_REGNO)
+                  (unspec:SI [(match_operand:SI 0 "immediate_operand" "s")
+                              (label_ref (match_operand 1 "" ""))]
+                          UNSPEC_TOCPTR))
+             (match_dup 1)])]
+  "TARGET_ELF && DEFAULT_ABI == ABI_V4 && flag_pic == 2"
+  "")
 
-(define_insn "*call_nonlocal_aix<mode>"
-  [(call (mem:SI (match_operand:P 0 "symbol_ref_operand" "s"))
-        (match_operand 1 "" "g"))
-   (clobber (reg:P LR_REGNO))]
-  "DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2"
-  "bl %z0\;nop"
+(define_insn "load_toc_v4_PIC_1b_normal"
+  [(set (reg:SI LR_REGNO)
+       (unspec:SI [(match_operand:SI 0 "immediate_operand" "s")
+                   (label_ref (match_operand 1 "" ""))]
+               UNSPEC_TOCPTR))
+   (match_dup 1)]
+  "!TARGET_LINK_STACK && TARGET_ELF && DEFAULT_ABI == ABI_V4 && flag_pic == 2"
+  "bcl 20,31,$+8\;.long %0-$"
   [(set_attr "type" "branch")
    (set_attr "length" "8")])
 
-(define_insn "*call_value_nonlocal_aix<mode>"
-  [(set (match_operand 0 "" "")
-       (call (mem:SI (match_operand:P 1 "symbol_ref_operand" "s"))
-             (match_operand 2 "" "g")))
-   (clobber (reg:P LR_REGNO))]
-  "DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2"
-  "bl %z1\;nop"
+(define_insn "load_toc_v4_PIC_1b_476"
+  [(set (reg:SI LR_REGNO)
+       (unspec:SI [(match_operand:SI 0 "immediate_operand" "s")
+                   (label_ref (match_operand 1 "" ""))]
+               UNSPEC_TOCPTR))
+   (match_dup 1)]
+  "TARGET_LINK_STACK && TARGET_ELF && DEFAULT_ABI == ABI_V4 && flag_pic == 2"
+  "*
+{
+  char name[32];
+  static char templ[32];
+
+  get_ppc476_thunk_name (name);
+  sprintf (templ, \"bl %s\\n\\tb $+8\\n\\t.long %%0-$\", name);
+  return templ;
+}"
   [(set_attr "type" "branch")
-   (set_attr "length" "8")])
+   (set_attr "length" "16")])
 
-;; Call to indirect functions with the AIX abi using a 3 word descriptor.
-;; Operand0 is the addresss of the function to call
-;; Operand2 is the location in the function descriptor to load r2 from
-;; Operand3 is the stack location to hold the current TOC pointer
+(define_insn "load_toc_v4_PIC_2"
+  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
+       (mem:SI (plus:SI (match_operand:SI 1 "gpc_reg_operand" "b")
+                  (minus:SI (match_operand:SI 2 "immediate_operand" "s")
+                            (match_operand:SI 3 "immediate_operand" "s")))))]
+  "TARGET_ELF && DEFAULT_ABI == ABI_V4 && flag_pic == 2"
+  "lwz %0,%2-%3(%1)"
+  [(set_attr "type" "load")])
 
-(define_insn "*call_indirect_aix<mode>"
-  [(call (mem:SI (match_operand:P 0 "register_operand" "c,*l"))
-        (match_operand 1 "" "g,g"))
-   (use (match_operand:P 2 "memory_operand" "<ptrm>,<ptrm>"))
-   (set (reg:P TOC_REGNUM) (match_operand:P 3 "memory_operand" "<ptrm>,<ptrm>"))
-   (clobber (reg:P LR_REGNO))]
-  "DEFAULT_ABI == ABI_AIX"
-  "<ptrload> 2,%2\;b%T0l\;<ptrload> 2,%3"
-  [(set_attr "type" "jmpreg")
-   (set_attr "length" "12")])
+(define_insn "load_toc_v4_PIC_3b"
+  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
+       (plus:SI (match_operand:SI 1 "gpc_reg_operand" "b")
+                (high:SI
+                  (minus:SI (match_operand:SI 2 "symbol_ref_operand" "s")
+                            (match_operand:SI 3 "symbol_ref_operand" "s")))))]
+  "TARGET_ELF && TARGET_SECURE_PLT && DEFAULT_ABI == ABI_V4 && flag_pic"
+  "addis %0,%1,%2-%3@ha")
 
-(define_insn "*call_value_indirect_aix<mode>"
-  [(set (match_operand 0 "" "")
-       (call (mem:SI (match_operand:P 1 "register_operand" "c,*l"))
-             (match_operand 2 "" "g,g")))
-   (use (match_operand:P 3 "memory_operand" "<ptrm>,<ptrm>"))
-   (set (reg:P TOC_REGNUM) (match_operand:P 4 "memory_operand" "<ptrm>,<ptrm>"))
-   (clobber (reg:P LR_REGNO))]
-  "DEFAULT_ABI == ABI_AIX"
-  "<ptrload> 2,%3\;b%T1l\;<ptrload> 2,%4"
-  [(set_attr "type" "jmpreg")
-   (set_attr "length" "12")])
+(define_insn "load_toc_v4_PIC_3c"
+  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
+       (lo_sum:SI (match_operand:SI 1 "gpc_reg_operand" "b")
+                  (minus:SI (match_operand:SI 2 "symbol_ref_operand" "s")
+                            (match_operand:SI 3 "symbol_ref_operand" "s"))))]
+  "TARGET_ELF && TARGET_SECURE_PLT && DEFAULT_ABI == ABI_V4 && flag_pic"
+  "addi %0,%1,%2-%3@l")
 
-;; Call to indirect functions with the ELFv2 ABI.
-;; Operand0 is the addresss of the function to call
-;; Operand2 is the stack location to hold the current TOC pointer
+;; If the TOC is shared over a translation unit, as happens with all
+;; the kinds of PIC that we support, we need to restore the TOC
+;; pointer only when jumping over units of translation.
+;; On Darwin, we need to reload the picbase.
 
-(define_insn "*call_indirect_elfv2<mode>"
-  [(call (mem:SI (match_operand:P 0 "register_operand" "c,*l"))
-        (match_operand 1 "" "g,g"))
-   (set (reg:P TOC_REGNUM) (match_operand:P 2 "memory_operand" "<ptrm>,<ptrm>"))
-   (clobber (reg:P LR_REGNO))]
-  "DEFAULT_ABI == ABI_ELFv2"
-  "b%T0l\;<ptrload> 2,%2"
-  [(set_attr "type" "jmpreg")
-   (set_attr "length" "8")])
+(define_expand "builtin_setjmp_receiver"
+  [(use (label_ref (match_operand 0 "" "")))]
+  "(DEFAULT_ABI == ABI_V4 && flag_pic == 1)
+   || (TARGET_TOC && TARGET_MINIMAL_TOC)
+   || (DEFAULT_ABI == ABI_DARWIN && flag_pic)"
+  "
+{
+#if TARGET_MACHO
+  if (DEFAULT_ABI == ABI_DARWIN)
+    {
+      rtx picrtx = gen_rtx_SYMBOL_REF (Pmode, MACHOPIC_FUNCTION_BASE_NAME);
+      rtx picreg = gen_rtx_REG (Pmode, RS6000_PIC_OFFSET_TABLE_REGNUM);
+      rtx tmplabrtx;
+      char tmplab[20];
 
-(define_insn "*call_value_indirect_elfv2<mode>"
-  [(set (match_operand 0 "" "")
-       (call (mem:SI (match_operand:P 1 "register_operand" "c,*l"))
-             (match_operand 2 "" "g,g")))
-   (set (reg:P TOC_REGNUM) (match_operand:P 3 "memory_operand" "<ptrm>,<ptrm>"))
-   (clobber (reg:P LR_REGNO))]
-  "DEFAULT_ABI == ABI_ELFv2"
-  "b%T1l\;<ptrload> 2,%3"
-  [(set_attr "type" "jmpreg")
-   (set_attr "length" "8")])
+      crtl->uses_pic_offset_table = 1;
+      ASM_GENERATE_INTERNAL_LABEL(tmplab, \"LSJR\",
+                                 CODE_LABEL_NUMBER (operands[0]));
+      tmplabrtx = gen_rtx_SYMBOL_REF (Pmode, ggc_strdup (tmplab));
 
+      emit_insn (gen_load_macho_picbase (tmplabrtx));
+      emit_move_insn (picreg, gen_rtx_REG (Pmode, LR_REGNO));
+      emit_insn (gen_macho_correct_pic (picreg, picreg, picrtx, tmplabrtx));
+    }
+  else
+#endif
+    rs6000_emit_load_toc_table (FALSE);
+  DONE;
+}")
 
-;; Call subroutine returning any type.
-(define_expand "untyped_call"
-  [(parallel [(call (match_operand 0 "" "")
-                   (const_int 0))
-             (match_operand 1 "" "")
-             (match_operand 2 "" "")])]
-  ""
-  "
-{
-  int i;
+;; Largetoc support
+(define_insn "*largetoc_high"
+  [(set (match_operand:DI 0 "gpc_reg_operand" "=b*r")
+        (high:DI
+         (unspec [(match_operand:DI 1 "" "")
+                  (match_operand:DI 2 "gpc_reg_operand" "b")]
+                 UNSPEC_TOCREL)))]
+   "TARGET_ELF && TARGET_CMODEL != CMODEL_SMALL"
+   "addis %0,%2,%1@toc@ha")
+
+(define_insn "*largetoc_high_aix<mode>"
+  [(set (match_operand:P 0 "gpc_reg_operand" "=b*r")
+        (high:P
+         (unspec [(match_operand:P 1 "" "")
+                  (match_operand:P 2 "gpc_reg_operand" "b")]
+                 UNSPEC_TOCREL)))]
+   "TARGET_XCOFF && TARGET_CMODEL != CMODEL_SMALL"
+   "addis %0,%1@u(%2)")
+
+(define_insn "*largetoc_high_plus"
+  [(set (match_operand:DI 0 "gpc_reg_operand" "=b*r")
+        (high:DI
+         (plus:DI
+           (unspec [(match_operand:DI 1 "" "")
+                    (match_operand:DI 2 "gpc_reg_operand" "b")]
+                   UNSPEC_TOCREL)
+           (match_operand:DI 3 "add_cint_operand" "n"))))]
+   "TARGET_ELF && TARGET_CMODEL != CMODEL_SMALL"
+   "addis %0,%2,%1+%3@toc@ha")
+
+(define_insn "*largetoc_high_plus_aix<mode>"
+  [(set (match_operand:P 0 "gpc_reg_operand" "=b*r")
+        (high:P
+         (plus:P
+           (unspec [(match_operand:P 1 "" "")
+                    (match_operand:P 2 "gpc_reg_operand" "b")]
+                   UNSPEC_TOCREL)
+           (match_operand:P 3 "add_cint_operand" "n"))))]
+   "TARGET_XCOFF && TARGET_CMODEL != CMODEL_SMALL"
+   "addis %0,%1+%3@u(%2)")
 
-  emit_call_insn (GEN_CALL (operands[0], const0_rtx, const0_rtx, const0_rtx));
+(define_insn "*largetoc_low"
+  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
+        (lo_sum:DI (match_operand:DI 1 "gpc_reg_operand" "b")
+                  (match_operand:DI 2 "" "")))]
+   "TARGET_ELF && TARGET_CMODEL != CMODEL_SMALL"
+   "addi %0,%1,%2@l")
 
-  for (i = 0; i < XVECLEN (operands[2], 0); i++)
-    {
-      rtx set = XVECEXP (operands[2], 0, i);
-      emit_move_insn (SET_DEST (set), SET_SRC (set));
-    }
+(define_insn "*largetoc_low_aix<mode>"
+  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
+        (lo_sum:P (match_operand:P 1 "gpc_reg_operand" "b")
+                  (match_operand:P 2 "" "")))]
+   "TARGET_XCOFF && TARGET_CMODEL != CMODEL_SMALL"
+   "la %0,%2@l(%1)")
 
-  /* The optimizer does not know that the call sets the function value
-     registers we stored in the result block.  We avoid problems by
-     claiming that all hard registers are used and clobbered at this
-     point.  */
-  emit_insn (gen_blockage ());
+(define_insn_and_split "*tocref<mode>"
+  [(set (match_operand:P 0 "gpc_reg_operand" "=b")
+       (match_operand:P 1 "small_toc_ref" "R"))]
+   "TARGET_TOC"
+   "la %0,%a1"
+   "&& TARGET_CMODEL != CMODEL_SMALL && reload_completed"
+  [(set (match_dup 0) (high:P (match_dup 1)))
+   (set (match_dup 0) (lo_sum:P (match_dup 0) (match_dup 1)))])
 
-  DONE;
-}")
+;; Elf specific ways of loading addresses for non-PIC code.
+;; The output of this could be r0, but we make a very strong
+;; preference for a base register because it will usually
+;; be needed there.
+(define_insn "elf_high"
+  [(set (match_operand:SI 0 "gpc_reg_operand" "=b*r")
+       (high:SI (match_operand 1 "" "")))]
+  "TARGET_ELF && ! TARGET_64BIT"
+  "lis %0,%1@ha")
 
-;; sibling call patterns
-(define_expand "sibcall"
+(define_insn "elf_low"
+  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
+       (lo_sum:SI (match_operand:SI 1 "gpc_reg_operand" "b")
+                  (match_operand 2 "" "")))]
+   "TARGET_ELF && ! TARGET_64BIT"
+   "la %0,%2@l(%1)")
+\f
+;; Call and call_value insns
+(define_expand "call"
   [(parallel [(call (mem:SI (match_operand 0 "address_operand" ""))
                    (match_operand 1 "" ""))
              (use (match_operand 2 "" ""))
-             (use (reg:SI LR_REGNO))
-             (simple_return)])]
+             (clobber (reg:SI LR_REGNO))])]
   ""
   "
 {
 
   if (DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2)
     {
-      rs6000_sibcall_aix (NULL_RTX, operands[0], operands[1], operands[2]);
+      rs6000_call_aix (NULL_RTX, operands[0], operands[1], operands[2]);
       DONE;
     }
+
+  if (GET_CODE (operands[0]) != SYMBOL_REF
+      || (DEFAULT_ABI != ABI_DARWIN && (INTVAL (operands[2]) & CALL_LONG) != 0))
+    {
+      if (INTVAL (operands[2]) & CALL_LONG)
+       operands[0] = rs6000_longcall_ref (operands[0]);
+
+      switch (DEFAULT_ABI)
+        {
+       case ABI_V4:
+       case ABI_DARWIN:
+         operands[0] = force_reg (Pmode, operands[0]);
+         break;
+
+       default:
+         gcc_unreachable ();
+       }
+    }
 }")
 
-(define_expand "sibcall_value"
-  [(parallel [(set (match_operand 0 "register_operand" "")
-               (call (mem:SI (match_operand 1 "address_operand" ""))
-                     (match_operand 2 "" "")))
+(define_expand "call_value"
+  [(parallel [(set (match_operand 0 "" "")
+                  (call (mem:SI (match_operand 1 "address_operand" ""))
+                        (match_operand 2 "" "")))
              (use (match_operand 3 "" ""))
-             (use (reg:SI LR_REGNO))
-             (simple_return)])]
+             (clobber (reg:SI LR_REGNO))])]
   ""
   "
 {
 
   if (DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2)
     {
-      rs6000_sibcall_aix (operands[0], operands[1], operands[2], operands[3]);
+      rs6000_call_aix (operands[0], operands[1], operands[2], operands[3]);
       DONE;
     }
+
+  if (GET_CODE (operands[1]) != SYMBOL_REF
+      || (DEFAULT_ABI != ABI_DARWIN && (INTVAL (operands[3]) & CALL_LONG) != 0))
+    {
+      if (INTVAL (operands[3]) & CALL_LONG)
+       operands[1] = rs6000_longcall_ref (operands[1]);
+
+      switch (DEFAULT_ABI)
+        {
+       case ABI_V4:
+       case ABI_DARWIN:
+         operands[1] = force_reg (Pmode, operands[1]);
+         break;
+
+       default:
+         gcc_unreachable ();
+       }
+    }
 }")
 
-;; this and similar patterns must be marked as using LR, otherwise
-;; dataflow will try to delete the store into it.  This is true
-;; even when the actual reg to jump to is in CTR, when LR was
-;; saved and restored around the PIC-setting BCL.
-(define_insn "*sibcall_local32"
+;; Call to function in current module.  No TOC pointer reload needed.
+;; Operand2 is nonzero if we are using the V.4 calling sequence and
+;; either the function was not prototyped, or it was prototyped as a
+;; variable argument function.  It is > 0 if FP registers were passed
+;; and < 0 if they were not.
+
+(define_insn "*call_local32"
   [(call (mem:SI (match_operand:SI 0 "current_file_function_operand" "s,s"))
         (match_operand 1 "" "g,g"))
    (use (match_operand:SI 2 "immediate_operand" "O,n"))
-   (use (reg:SI LR_REGNO))
-   (simple_return)]
+   (clobber (reg:SI LR_REGNO))]
   "(INTVAL (operands[2]) & CALL_LONG) == 0"
   "*
 {
   else if (INTVAL (operands[2]) & CALL_V4_CLEAR_FP_ARGS)
     output_asm_insn (\"creqv 6,6,6\", operands);
 
-  return (DEFAULT_ABI == ABI_V4 && flag_pic) ? \"b %z0@local\" : \"b %z0\";
+  return (DEFAULT_ABI == ABI_V4 && flag_pic) ? \"bl %z0@local\" : \"bl %z0\";
 }"
   [(set_attr "type" "branch")
    (set_attr "length" "4,8")])
 
-(define_insn "*sibcall_local64"
+(define_insn "*call_local64"
   [(call (mem:SI (match_operand:DI 0 "current_file_function_operand" "s,s"))
         (match_operand 1 "" "g,g"))
    (use (match_operand:SI 2 "immediate_operand" "O,n"))
-   (use (reg:SI LR_REGNO))
-   (simple_return)]
+   (clobber (reg:SI LR_REGNO))]
   "TARGET_64BIT && (INTVAL (operands[2]) & CALL_LONG) == 0"
   "*
 {
   if (INTVAL (operands[2]) & CALL_V4_SET_FP_ARGS)
-    output_asm_insn (\"crxor 6,6,6\", operands);
+    output_asm_insn (\"crxor 6,6,6\", operands);
+
+  else if (INTVAL (operands[2]) & CALL_V4_CLEAR_FP_ARGS)
+    output_asm_insn (\"creqv 6,6,6\", operands);
+
+  return (DEFAULT_ABI == ABI_V4 && flag_pic) ? \"bl %z0@local\" : \"bl %z0\";
+}"
+  [(set_attr "type" "branch")
+   (set_attr "length" "4,8")])
+
+(define_insn "*call_value_local32"
+  [(set (match_operand 0 "" "")
+       (call (mem:SI (match_operand:SI 1 "current_file_function_operand" "s,s"))
+             (match_operand 2 "" "g,g")))
+   (use (match_operand:SI 3 "immediate_operand" "O,n"))
+   (clobber (reg:SI LR_REGNO))]
+  "(INTVAL (operands[3]) & CALL_LONG) == 0"
+  "*
+{
+  if (INTVAL (operands[3]) & CALL_V4_SET_FP_ARGS)
+    output_asm_insn (\"crxor 6,6,6\", operands);
+
+  else if (INTVAL (operands[3]) & CALL_V4_CLEAR_FP_ARGS)
+    output_asm_insn (\"creqv 6,6,6\", operands);
+
+  return (DEFAULT_ABI == ABI_V4 && flag_pic) ? \"bl %z1@local\" : \"bl %z1\";
+}"
+  [(set_attr "type" "branch")
+   (set_attr "length" "4,8")])
+
+
+(define_insn "*call_value_local64"
+  [(set (match_operand 0 "" "")
+       (call (mem:SI (match_operand:DI 1 "current_file_function_operand" "s,s"))
+             (match_operand 2 "" "g,g")))
+   (use (match_operand:SI 3 "immediate_operand" "O,n"))
+   (clobber (reg:SI LR_REGNO))]
+  "TARGET_64BIT && (INTVAL (operands[3]) & CALL_LONG) == 0"
+  "*
+{
+  if (INTVAL (operands[3]) & CALL_V4_SET_FP_ARGS)
+    output_asm_insn (\"crxor 6,6,6\", operands);
+
+  else if (INTVAL (operands[3]) & CALL_V4_CLEAR_FP_ARGS)
+    output_asm_insn (\"creqv 6,6,6\", operands);
+
+  return (DEFAULT_ABI == ABI_V4 && flag_pic) ? \"bl %z1@local\" : \"bl %z1\";
+}"
+  [(set_attr "type" "branch")
+   (set_attr "length" "4,8")])
+
+
+;; A function pointer under System V is just a normal pointer
+;; operands[0] is the function pointer
+;; operands[1] is the stack size to clean up
+;; operands[2] is the value FUNCTION_ARG returns for the VOID argument
+;; which indicates how to set cr1
+
+(define_insn "*call_indirect_nonlocal_sysv<mode>"
+  [(call (mem:SI (match_operand:P 0 "register_operand" "c,*l,c,*l"))
+        (match_operand 1 "" "g,g,g,g"))
+   (use (match_operand:SI 2 "immediate_operand" "O,O,n,n"))
+   (clobber (reg:SI LR_REGNO))]
+  "DEFAULT_ABI == ABI_V4
+   || DEFAULT_ABI == ABI_DARWIN"
+{
+  if (INTVAL (operands[2]) & CALL_V4_SET_FP_ARGS)
+    output_asm_insn ("crxor 6,6,6", operands);
+
+  else if (INTVAL (operands[2]) & CALL_V4_CLEAR_FP_ARGS)
+    output_asm_insn ("creqv 6,6,6", operands);
+
+  return "b%T0l";
+}
+  [(set_attr "type" "jmpreg,jmpreg,jmpreg,jmpreg")
+   (set_attr "length" "4,4,8,8")])
+
+(define_insn_and_split "*call_nonlocal_sysv<mode>"
+  [(call (mem:SI (match_operand:P 0 "symbol_ref_operand" "s,s"))
+        (match_operand 1 "" "g,g"))
+   (use (match_operand:SI 2 "immediate_operand" "O,n"))
+   (clobber (reg:SI LR_REGNO))]
+  "(DEFAULT_ABI == ABI_DARWIN
+   || (DEFAULT_ABI == ABI_V4
+       && (INTVAL (operands[2]) & CALL_LONG) == 0))"
+{
+  if (INTVAL (operands[2]) & CALL_V4_SET_FP_ARGS)
+    output_asm_insn ("crxor 6,6,6", operands);
+
+  else if (INTVAL (operands[2]) & CALL_V4_CLEAR_FP_ARGS)
+    output_asm_insn ("creqv 6,6,6", operands);
+
+#if TARGET_MACHO
+  return output_call(insn, operands, 0, 2);
+#else
+  if (DEFAULT_ABI == ABI_V4 && flag_pic)
+    {
+      gcc_assert (!TARGET_SECURE_PLT);
+      return "bl %z0@plt";
+    }
+  else
+    return "bl %z0";
+#endif
+}
+  "DEFAULT_ABI == ABI_V4
+   && TARGET_SECURE_PLT && flag_pic && !SYMBOL_REF_LOCAL_P (operands[0])
+   && (INTVAL (operands[2]) & CALL_LONG) == 0"
+  [(parallel [(call (mem:SI (match_dup 0))
+                   (match_dup 1))
+             (use (match_dup 2))
+             (use (match_dup 3))
+             (clobber (reg:SI LR_REGNO))])]
+{
+  operands[3] = pic_offset_table_rtx;
+}
+  [(set_attr "type" "branch,branch")
+   (set_attr "length" "4,8")])
+
+(define_insn "*call_nonlocal_sysv_secure<mode>"
+  [(call (mem:SI (match_operand:P 0 "symbol_ref_operand" "s,s"))
+        (match_operand 1 "" "g,g"))
+   (use (match_operand:SI 2 "immediate_operand" "O,n"))
+   (use (match_operand:SI 3 "register_operand" "r,r"))
+   (clobber (reg:SI LR_REGNO))]
+  "(DEFAULT_ABI == ABI_V4
+    && TARGET_SECURE_PLT && flag_pic && !SYMBOL_REF_LOCAL_P (operands[0])
+    && (INTVAL (operands[2]) & CALL_LONG) == 0)"
+{
+  if (INTVAL (operands[2]) & CALL_V4_SET_FP_ARGS)
+    output_asm_insn ("crxor 6,6,6", operands);
 
   else if (INTVAL (operands[2]) & CALL_V4_CLEAR_FP_ARGS)
-    output_asm_insn (\"creqv 6,6,6\", operands);
+    output_asm_insn ("creqv 6,6,6", operands);
 
-  return (DEFAULT_ABI == ABI_V4 && flag_pic) ? \"b %z0@local\" : \"b %z0\";
-}"
-  [(set_attr "type" "branch")
+  if (flag_pic == 2)
+    /* The magic 32768 offset here and in the other sysv call insns
+       corresponds to the offset of r30 in .got2, as given by LCTOC1.
+       See sysv4.h:toc_section.  */
+    return "bl %z0+32768@plt";
+  else
+    return "bl %z0@plt";
+}
+  [(set_attr "type" "branch,branch")
    (set_attr "length" "4,8")])
 
-(define_insn "*sibcall_value_local32"
+(define_insn "*call_value_indirect_nonlocal_sysv<mode>"
   [(set (match_operand 0 "" "")
-       (call (mem:SI (match_operand:SI 1 "current_file_function_operand" "s,s"))
-             (match_operand 2 "" "g,g")))
-   (use (match_operand:SI 3 "immediate_operand" "O,n"))
-   (use (reg:SI LR_REGNO))
-   (simple_return)]
-  "(INTVAL (operands[3]) & CALL_LONG) == 0"
-  "*
+       (call (mem:SI (match_operand:P 1 "register_operand" "c,*l,c,*l"))
+             (match_operand 2 "" "g,g,g,g")))
+   (use (match_operand:SI 3 "immediate_operand" "O,O,n,n"))
+   (clobber (reg:SI LR_REGNO))]
+  "DEFAULT_ABI == ABI_V4
+   || DEFAULT_ABI == ABI_DARWIN"
 {
   if (INTVAL (operands[3]) & CALL_V4_SET_FP_ARGS)
-    output_asm_insn (\"crxor 6,6,6\", operands);
+    output_asm_insn ("crxor 6,6,6", operands);
 
   else if (INTVAL (operands[3]) & CALL_V4_CLEAR_FP_ARGS)
-    output_asm_insn (\"creqv 6,6,6\", operands);
+    output_asm_insn ("creqv 6,6,6", operands);
 
-  return (DEFAULT_ABI == ABI_V4 && flag_pic) ? \"b %z1@local\" : \"b %z1\";
-}"
-  [(set_attr "type" "branch")
-   (set_attr "length" "4,8")])
+  return "b%T1l";
+}
+  [(set_attr "type" "jmpreg,jmpreg,jmpreg,jmpreg")
+   (set_attr "length" "4,4,8,8")])
 
-(define_insn "*sibcall_value_local64"
+(define_insn_and_split "*call_value_nonlocal_sysv<mode>"
   [(set (match_operand 0 "" "")
-       (call (mem:SI (match_operand:DI 1 "current_file_function_operand" "s,s"))
+       (call (mem:SI (match_operand:P 1 "symbol_ref_operand" "s,s"))
              (match_operand 2 "" "g,g")))
    (use (match_operand:SI 3 "immediate_operand" "O,n"))
-   (use (reg:SI LR_REGNO))
-   (simple_return)]
-  "TARGET_64BIT && (INTVAL (operands[3]) & CALL_LONG) == 0"
-  "*
+   (clobber (reg:SI LR_REGNO))]
+  "(DEFAULT_ABI == ABI_DARWIN
+   || (DEFAULT_ABI == ABI_V4
+       && (INTVAL (operands[3]) & CALL_LONG) == 0))"
 {
   if (INTVAL (operands[3]) & CALL_V4_SET_FP_ARGS)
-    output_asm_insn (\"crxor 6,6,6\", operands);
+    output_asm_insn ("crxor 6,6,6", operands);
 
   else if (INTVAL (operands[3]) & CALL_V4_CLEAR_FP_ARGS)
-    output_asm_insn (\"creqv 6,6,6\", operands);
-
-  return (DEFAULT_ABI == ABI_V4 && flag_pic) ? \"b %z1@local\" : \"b %z1\";
-}"
-  [(set_attr "type" "branch")
-   (set_attr "length" "4,8")])
-
-(define_insn "*sibcall_nonlocal_sysv<mode>"
-  [(call (mem:SI (match_operand:P 0 "call_operand" "s,s,c,c"))
-        (match_operand 1 "" ""))
-   (use (match_operand 2 "immediate_operand" "O,n,O,n"))
-   (use (reg:SI LR_REGNO))
-   (simple_return)]
-  "(DEFAULT_ABI == ABI_DARWIN
-    || DEFAULT_ABI == ABI_V4)
-   && (INTVAL (operands[2]) & CALL_LONG) == 0"
-  "*
-{
-  if (INTVAL (operands[2]) & CALL_V4_SET_FP_ARGS)
-    output_asm_insn (\"crxor 6,6,6\", operands);
-
-  else if (INTVAL (operands[2]) & CALL_V4_CLEAR_FP_ARGS)
-    output_asm_insn (\"creqv 6,6,6\", operands);
+    output_asm_insn ("creqv 6,6,6", operands);
 
-  if (which_alternative >= 2)
-    return \"b%T0\";
-  else if (DEFAULT_ABI == ABI_V4 && flag_pic)
+#if TARGET_MACHO
+  return output_call(insn, operands, 1, 3);
+#else
+  if (DEFAULT_ABI == ABI_V4 && flag_pic)
     {
       gcc_assert (!TARGET_SECURE_PLT);
-      return \"b %z0@plt\";
+      return "bl %z1@plt";
     }
   else
-    return \"b %z0\";
-}"
-  [(set_attr "type" "branch")
-   (set_attr "length" "4,8,4,8")])
+    return "bl %z1";
+#endif
+}
+  "DEFAULT_ABI == ABI_V4
+   && TARGET_SECURE_PLT && flag_pic && !SYMBOL_REF_LOCAL_P (operands[1])
+   && (INTVAL (operands[3]) & CALL_LONG) == 0"
+  [(parallel [(set (match_dup 0)
+                  (call (mem:SI (match_dup 1))
+                        (match_dup 2)))
+             (use (match_dup 3))
+             (use (match_dup 4))
+             (clobber (reg:SI LR_REGNO))])]
+{
+  operands[4] = pic_offset_table_rtx;
+}
+  [(set_attr "type" "branch,branch")
+   (set_attr "length" "4,8")])
 
-(define_insn "*sibcall_value_nonlocal_sysv<mode>"
+(define_insn "*call_value_nonlocal_sysv_secure<mode>"
   [(set (match_operand 0 "" "")
-       (call (mem:SI (match_operand:P 1 "call_operand" "s,s,c,c"))
-             (match_operand 2 "" "")))
-   (use (match_operand:SI 3 "immediate_operand" "O,n,O,n"))
-   (use (reg:SI LR_REGNO))
-   (simple_return)]
-  "(DEFAULT_ABI == ABI_DARWIN
-    || DEFAULT_ABI == ABI_V4)
-   && (INTVAL (operands[3]) & CALL_LONG) == 0"
-  "*
+       (call (mem:SI (match_operand:P 1 "symbol_ref_operand" "s,s"))
+             (match_operand 2 "" "g,g")))
+   (use (match_operand:SI 3 "immediate_operand" "O,n"))
+   (use (match_operand:SI 4 "register_operand" "r,r"))
+   (clobber (reg:SI LR_REGNO))]
+  "(DEFAULT_ABI == ABI_V4
+    && TARGET_SECURE_PLT && flag_pic && !SYMBOL_REF_LOCAL_P (operands[1])
+    && (INTVAL (operands[3]) & CALL_LONG) == 0)"
 {
   if (INTVAL (operands[3]) & CALL_V4_SET_FP_ARGS)
-    output_asm_insn (\"crxor 6,6,6\", operands);
+    output_asm_insn ("crxor 6,6,6", operands);
 
   else if (INTVAL (operands[3]) & CALL_V4_CLEAR_FP_ARGS)
-    output_asm_insn (\"creqv 6,6,6\", operands);
+    output_asm_insn ("creqv 6,6,6", operands);
 
-  if (which_alternative >= 2)
-    return \"b%T1\";
-  else if (DEFAULT_ABI == ABI_V4 && flag_pic)
-    {
-      gcc_assert (!TARGET_SECURE_PLT);
-      return \"b %z1@plt\";
-    }
+  if (flag_pic == 2)
+    return "bl %z1+32768@plt";
   else
-    return \"b %z1\";
-}"
-  [(set_attr "type" "branch")
-   (set_attr "length" "4,8,4,8")])
+    return "bl %z1@plt";
+}
+  [(set_attr "type" "branch,branch")
+   (set_attr "length" "4,8")])
 
-;; AIX ABI sibling call patterns.
 
-(define_insn "*sibcall_aix<mode>"
-  [(call (mem:SI (match_operand:P 0 "call_operand" "s,c"))
-        (match_operand 1 "" "g,g"))
-   (simple_return)]
+;; Call to AIX abi function in the same module.
+
+(define_insn "*call_local_aix<mode>"
+  [(call (mem:SI (match_operand:P 0 "current_file_function_operand" "s"))
+        (match_operand 1 "" "g"))
+   (clobber (reg:P LR_REGNO))]
   "DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2"
-  "@
-   b %z0
-   b%T0"
+  "bl %z0"
   [(set_attr "type" "branch")
    (set_attr "length" "4")])
 
-(define_insn "*sibcall_value_aix<mode>"
+(define_insn "*call_value_local_aix<mode>"
   [(set (match_operand 0 "" "")
-       (call (mem:SI (match_operand:P 1 "call_operand" "s,c"))
-             (match_operand 2 "" "g,g")))
-   (simple_return)]
+       (call (mem:SI (match_operand:P 1 "current_file_function_operand" "s"))
+             (match_operand 2 "" "g")))
+   (clobber (reg:P LR_REGNO))]
   "DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2"
-  "@
-   b %z1
-   b%T1"
+  "bl %z1"
   [(set_attr "type" "branch")
    (set_attr "length" "4")])
 
-(define_expand "sibcall_epilogue"
-  [(use (const_int 0))]
-  ""
-{
-  if (!TARGET_SCHED_PROLOG)
-    emit_insn (gen_blockage ());
-  rs6000_emit_epilogue (TRUE);
-  DONE;
-})
+;; Call to AIX abi function which may be in another module.
+;; Restore the TOC pointer (r2) after the call.
 
-;; UNSPEC_VOLATILE is considered to use and clobber all hard registers and
-;; all of memory.  This blocks insns from being moved across this point.
+(define_insn "*call_nonlocal_aix<mode>"
+  [(call (mem:SI (match_operand:P 0 "symbol_ref_operand" "s"))
+        (match_operand 1 "" "g"))
+   (clobber (reg:P LR_REGNO))]
+  "DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2"
+  "bl %z0\;nop"
+  [(set_attr "type" "branch")
+   (set_attr "length" "8")])
 
-(define_insn "blockage"
-  [(unspec_volatile [(const_int 0)] UNSPECV_BLOCK)]
-  ""
-  "")
+(define_insn "*call_value_nonlocal_aix<mode>"
+  [(set (match_operand 0 "" "")
+       (call (mem:SI (match_operand:P 1 "symbol_ref_operand" "s"))
+             (match_operand 2 "" "g")))
+   (clobber (reg:P LR_REGNO))]
+  "DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2"
+  "bl %z1\;nop"
+  [(set_attr "type" "branch")
+   (set_attr "length" "8")])
 
-(define_expand "probe_stack"
-  [(set (match_operand 0 "memory_operand" "=m")
-        (unspec [(const_int 0)] UNSPEC_PROBE_STACK))]
-  ""
-{
-  if (TARGET_64BIT)
-    emit_insn (gen_probe_stack_di (operands[0]));
-  else
-    emit_insn (gen_probe_stack_si (operands[0]));
-  DONE;
-})
+;; Call to indirect functions with the AIX abi using a 3 word descriptor.
+;; Operand0 is the addresss of the function to call
+;; Operand2 is the location in the function descriptor to load r2 from
+;; Operand3 is the offset of the stack location holding the current TOC pointer
 
-(define_insn "probe_stack_<mode>"
-  [(set (match_operand:P 0 "memory_operand" "=m")
-        (unspec:P [(const_int 0)] UNSPEC_PROBE_STACK))]
-  ""
-{
-  operands[1] = gen_rtx_REG (Pmode, 0);
-  return "st<wd>%U0%X0 %1,%0";
-}
-  [(set_attr "type" "store")
-   (set (attr "update")
-       (if_then_else (match_operand 0 "update_address_mem")
-                     (const_string "yes")
-                     (const_string "no")))
-   (set (attr "indexed")
-       (if_then_else (match_operand 0 "indexed_address_mem")
-                     (const_string "yes")
-                     (const_string "no")))
-   (set_attr "length" "4")])
+(define_insn "*call_indirect_aix<mode>"
+  [(call (mem:SI (match_operand:P 0 "register_operand" "c,*l"))
+        (match_operand 1 "" "g,g"))
+   (use (match_operand:P 2 "memory_operand" "<ptrm>,<ptrm>"))
+   (set (reg:P TOC_REGNUM) (unspec [(match_operand:P 3 "const_int_operand" "n,n")] UNSPEC_TOCSLOT))
+   (clobber (reg:P LR_REGNO))]
+  "DEFAULT_ABI == ABI_AIX"
+  "<ptrload> 2,%2\;b%T0l\;<ptrload> 2,%3(1)"
+  [(set_attr "type" "jmpreg")
+   (set_attr "length" "12")])
+
+(define_insn "*call_value_indirect_aix<mode>"
+  [(set (match_operand 0 "" "")
+       (call (mem:SI (match_operand:P 1 "register_operand" "c,*l"))
+             (match_operand 2 "" "g,g")))
+   (use (match_operand:P 3 "memory_operand" "<ptrm>,<ptrm>"))
+   (set (reg:P TOC_REGNUM) (unspec [(match_operand:P 4 "const_int_operand" "n,n")] UNSPEC_TOCSLOT))
+   (clobber (reg:P LR_REGNO))]
+  "DEFAULT_ABI == ABI_AIX"
+  "<ptrload> 2,%3\;b%T1l\;<ptrload> 2,%4(1)"
+  [(set_attr "type" "jmpreg")
+   (set_attr "length" "12")])
 
-(define_insn "probe_stack_range<P:mode>"
-  [(set (match_operand:P 0 "register_operand" "=r")
-       (unspec_volatile:P [(match_operand:P 1 "register_operand" "0")
-                           (match_operand:P 2 "register_operand" "r")]
-                          UNSPECV_PROBE_STACK_RANGE))]
-  ""
-  "* return output_probe_stack_range (operands[0], operands[2]);"
-  [(set_attr "type" "three")])
-\f
-;; Compare insns are next.  Note that the RS/6000 has two types of compares,
-;; signed & unsigned, and one type of branch.
-;;
-;; Start with the DEFINE_EXPANDs to generate the rtl for compares, scc
-;; insns, and branches.
+;; Call to indirect functions with the ELFv2 ABI.
+;; Operand0 is the addresss of the function to call
+;; Operand2 is the offset of the stack location holding the current TOC pointer
 
-(define_expand "cbranch<mode>4"
-  [(use (match_operator 0 "rs6000_cbranch_operator"
-         [(match_operand:GPR 1 "gpc_reg_operand" "")
-          (match_operand:GPR 2 "reg_or_short_operand" "")]))
-   (use (match_operand 3 ""))]
-  ""
-  "
-{
-  /* Take care of the possibility that operands[2] might be negative but
-     this might be a logical operation.  That insn doesn't exist.  */
-  if (GET_CODE (operands[2]) == CONST_INT
-      && INTVAL (operands[2]) < 0)
-    {
-      operands[2] = force_reg (<MODE>mode, operands[2]);
-      operands[0] = gen_rtx_fmt_ee (GET_CODE (operands[0]),
-                                   GET_MODE (operands[0]),
-                                   operands[1], operands[2]);
-   }
+(define_insn "*call_indirect_elfv2<mode>"
+  [(call (mem:SI (match_operand:P 0 "register_operand" "c,*l"))
+        (match_operand 1 "" "g,g"))
+   (set (reg:P TOC_REGNUM) (unspec [(match_operand:P 2 "const_int_operand" "n,n")] UNSPEC_TOCSLOT))
+   (clobber (reg:P LR_REGNO))]
+  "DEFAULT_ABI == ABI_ELFv2"
+  "b%T0l\;<ptrload> 2,%2(1)"
+  [(set_attr "type" "jmpreg")
+   (set_attr "length" "8")])
 
-  rs6000_emit_cbranch (<MODE>mode, operands);
-  DONE;
-}")
+(define_insn "*call_value_indirect_elfv2<mode>"
+  [(set (match_operand 0 "" "")
+       (call (mem:SI (match_operand:P 1 "register_operand" "c,*l"))
+             (match_operand 2 "" "g,g")))
+   (set (reg:P TOC_REGNUM) (unspec [(match_operand:P 3 "const_int_operand" "n,n")] UNSPEC_TOCSLOT))
+   (clobber (reg:P LR_REGNO))]
+  "DEFAULT_ABI == ABI_ELFv2"
+  "b%T1l\;<ptrload> 2,%3(1)"
+  [(set_attr "type" "jmpreg")
+   (set_attr "length" "8")])
 
-(define_expand "cbranch<mode>4"
-  [(use (match_operator 0 "rs6000_cbranch_operator"
-         [(match_operand:FP 1 "gpc_reg_operand" "")
-          (match_operand:FP 2 "gpc_reg_operand" "")]))
-   (use (match_operand 3 ""))]
-  ""
-  "
-{
-  rs6000_emit_cbranch (<MODE>mode, operands);
-  DONE;
-}")
 
-(define_expand "cstore<mode>4"
-  [(use (match_operator 1 "rs6000_cbranch_operator"
-         [(match_operand:GPR 2 "gpc_reg_operand" "")
-          (match_operand:GPR 3 "reg_or_short_operand" "")]))
-   (clobber (match_operand:SI 0 "register_operand"))]
+;; Call subroutine returning any type.
+(define_expand "untyped_call"
+  [(parallel [(call (match_operand 0 "" "")
+                   (const_int 0))
+             (match_operand 1 "" "")
+             (match_operand 2 "" "")])]
   ""
   "
 {
-  /* Take care of the possibility that operands[3] might be negative but
-     this might be a logical operation.  That insn doesn't exist.  */
-  if (GET_CODE (operands[3]) == CONST_INT
-      && INTVAL (operands[3]) < 0)
+  int i;
+
+  emit_call_insn (GEN_CALL (operands[0], const0_rtx, const0_rtx, const0_rtx));
+
+  for (i = 0; i < XVECLEN (operands[2], 0); i++)
     {
-      operands[3] = force_reg (<MODE>mode, operands[3]);
-      operands[1] = gen_rtx_fmt_ee (GET_CODE (operands[1]),
-                                   GET_MODE (operands[1]),
-                                   operands[2], operands[3]);
+      rtx set = XVECEXP (operands[2], 0, i);
+      emit_move_insn (SET_DEST (set), SET_SRC (set));
     }
 
-  /* For SNE, we would prefer that the xor/abs sequence be used for integers.
-     For SEQ, likewise, except that comparisons with zero should be done
-     with an scc insns.  However, due to the order that combine see the
-     resulting insns, we must, in fact, allow SEQ for integers.  Fail in
-     the cases we don't want to handle or are best handled by portable
-     code.  */
-  if (GET_CODE (operands[1]) == NE)
-    FAIL;
-  if ((GET_CODE (operands[1]) == LT || GET_CODE (operands[1]) == LE
-       || GET_CODE (operands[1]) == GT || GET_CODE (operands[1]) == GE)
-      && operands[3] == const0_rtx)
-    FAIL;
-  rs6000_emit_sCOND (<MODE>mode, operands);
-  DONE;
-}")
+  /* The optimizer does not know that the call sets the function value
+     registers we stored in the result block.  We avoid problems by
+     claiming that all hard registers are used and clobbered at this
+     point.  */
+  emit_insn (gen_blockage ());
 
-(define_expand "cstore<mode>4"
-  [(use (match_operator 1 "rs6000_cbranch_operator"
-         [(match_operand:FP 2 "gpc_reg_operand" "")
-          (match_operand:FP 3 "gpc_reg_operand" "")]))
-   (clobber (match_operand:SI 0 "register_operand"))]
-  ""
-  "
-{
-  rs6000_emit_sCOND (<MODE>mode, operands);
   DONE;
 }")
 
-
-(define_expand "stack_protect_set"
-  [(match_operand 0 "memory_operand" "")
-   (match_operand 1 "memory_operand" "")]
-  ""
-{
-#ifdef TARGET_THREAD_SSP_OFFSET
-  rtx tlsreg = gen_rtx_REG (Pmode, TARGET_64BIT ? 13 : 2);
-  rtx addr = gen_rtx_PLUS (Pmode, tlsreg, GEN_INT (TARGET_THREAD_SSP_OFFSET));
-  operands[1] = gen_rtx_MEM (Pmode, addr);
-#endif
-  if (TARGET_64BIT)
-    emit_insn (gen_stack_protect_setdi (operands[0], operands[1]));
-  else
-    emit_insn (gen_stack_protect_setsi (operands[0], operands[1]));
-  DONE;
-})
-
-(define_insn "stack_protect_setsi"
-  [(set (match_operand:SI 0 "memory_operand" "=m")
-       (unspec:SI [(match_operand:SI 1 "memory_operand" "m")] UNSPEC_SP_SET))
-   (set (match_scratch:SI 2 "=&r") (const_int 0))]
-  "TARGET_32BIT"
-  "lwz%U1%X1 %2,%1\;stw%U0%X0 %2,%0\;li %2,0"
-  [(set_attr "type" "three")
-   (set_attr "length" "12")])
-
-(define_insn "stack_protect_setdi"
-  [(set (match_operand:DI 0 "memory_operand" "=Y")
-       (unspec:DI [(match_operand:DI 1 "memory_operand" "Y")] UNSPEC_SP_SET))
-   (set (match_scratch:DI 2 "=&r") (const_int 0))]
-  "TARGET_64BIT"
-  "ld%U1%X1 %2,%1\;std%U0%X0 %2,%0\;li %2,0"
-  [(set_attr "type" "three")
-   (set_attr "length" "12")])
-
-(define_expand "stack_protect_test"
-  [(match_operand 0 "memory_operand" "")
-   (match_operand 1 "memory_operand" "")
-   (match_operand 2 "" "")]
+;; sibling call patterns
+(define_expand "sibcall"
+  [(parallel [(call (mem:SI (match_operand 0 "address_operand" ""))
+                   (match_operand 1 "" ""))
+             (use (match_operand 2 "" ""))
+             (use (reg:SI LR_REGNO))
+             (simple_return)])]
   ""
+  "
 {
-  rtx test, op0, op1;
-#ifdef TARGET_THREAD_SSP_OFFSET
-  rtx tlsreg = gen_rtx_REG (Pmode, TARGET_64BIT ? 13 : 2);
-  rtx addr = gen_rtx_PLUS (Pmode, tlsreg, GEN_INT (TARGET_THREAD_SSP_OFFSET));
-  operands[1] = gen_rtx_MEM (Pmode, addr);
+#if TARGET_MACHO
+  if (MACHOPIC_INDIRECT)
+    operands[0] = machopic_indirect_call_target (operands[0]);
 #endif
-  op0 = operands[0];
-  op1 = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, operands[1]), UNSPEC_SP_TEST);
-  test = gen_rtx_EQ (VOIDmode, op0, op1);
-  emit_jump_insn (gen_cbranchsi4 (test, op0, op1, operands[2]));
-  DONE;
-})
-
-(define_insn "stack_protect_testsi"
-  [(set (match_operand:CCEQ 0 "cc_reg_operand" "=x,?y")
-        (unspec:CCEQ [(match_operand:SI 1 "memory_operand" "m,m")
-                     (match_operand:SI 2 "memory_operand" "m,m")]
-                    UNSPEC_SP_TEST))
-   (set (match_scratch:SI 4 "=r,r") (const_int 0))
-   (clobber (match_scratch:SI 3 "=&r,&r"))]
-  "TARGET_32BIT"
-  "@
-   lwz%U1%X1 %3,%1\;lwz%U2%X2 %4,%2\;xor. %3,%3,%4\;li %4,0
-   lwz%U1%X1 %3,%1\;lwz%U2%X2 %4,%2\;cmplw %0,%3,%4\;li %3,0\;li %4,0"
-  [(set_attr "length" "16,20")])
-
-(define_insn "stack_protect_testdi"
-  [(set (match_operand:CCEQ 0 "cc_reg_operand" "=x,?y")
-        (unspec:CCEQ [(match_operand:DI 1 "memory_operand" "Y,Y")
-                     (match_operand:DI 2 "memory_operand" "Y,Y")]
-                    UNSPEC_SP_TEST))
-   (set (match_scratch:DI 4 "=r,r") (const_int 0))
-   (clobber (match_scratch:DI 3 "=&r,&r"))]
-  "TARGET_64BIT"
-  "@
-   ld%U1%X1 %3,%1\;ld%U2%X2 %4,%2\;xor. %3,%3,%4\;li %4,0
-   ld%U1%X1 %3,%1\;ld%U2%X2 %4,%2\;cmpld %0,%3,%4\;li %3,0\;li %4,0"
-  [(set_attr "length" "16,20")])
-
-\f
-;; Here are the actual compare insns.
-(define_insn "*cmp<mode>_internal1"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=y")
-       (compare:CC (match_operand:GPR 1 "gpc_reg_operand" "r")
-                   (match_operand:GPR 2 "reg_or_short_operand" "rI")))]
-  ""
-  "cmp<wd>%I2 %0,%1,%2"
-  [(set_attr "type" "cmp")])
-
-;; If we are comparing a register for equality with a large constant,
-;; we can do this with an XOR followed by a compare.  But this is profitable
-;; only if the large constant is only used for the comparison (and in this
-;; case we already have a register to reuse as scratch).
-;;
-;; For 64-bit registers, we could only do so if the constant's bit 15 is clear:
-;; otherwise we'd need to XOR with FFFFFFFF????0000 which is not available.
 
-(define_peephole2
-  [(set (match_operand:SI 0 "register_operand")
-        (match_operand:SI 1 "logical_const_operand" ""))
-   (set (match_dup 0) (match_operator:SI 3 "boolean_or_operator"
-                      [(match_dup 0)
-                       (match_operand:SI 2 "logical_const_operand" "")]))
-   (set (match_operand:CC 4 "cc_reg_operand" "")
-        (compare:CC (match_operand:SI 5 "gpc_reg_operand" "")
-                    (match_dup 0)))
-   (set (pc)
-        (if_then_else (match_operator 6 "equality_operator"
-                       [(match_dup 4) (const_int 0)])
-                      (match_operand 7 "" "")
-                      (match_operand 8 "" "")))]
-  "peep2_reg_dead_p (3, operands[0])
-   && peep2_reg_dead_p (4, operands[4])"
- [(set (match_dup 0) (xor:SI (match_dup 5) (match_dup 9)))
-  (set (match_dup 4) (compare:CC (match_dup 0) (match_dup 10)))
-  (set (pc) (if_then_else (match_dup 6) (match_dup 7) (match_dup 8)))]
-{
-  /* Get the constant we are comparing against, and see what it looks like
-     when sign-extended from 16 to 32 bits.  Then see what constant we could
-     XOR with SEXTC to get the sign-extended value.  */
-  rtx cnst = simplify_const_binary_operation (GET_CODE (operands[3]),
-                                             SImode,
-                                             operands[1], operands[2]);
-  HOST_WIDE_INT c = INTVAL (cnst);
-  HOST_WIDE_INT sextc = ((c & 0xffff) ^ 0x8000) - 0x8000;
-  HOST_WIDE_INT xorv = c ^ sextc;
+  gcc_assert (GET_CODE (operands[0]) == MEM);
+  gcc_assert (GET_CODE (operands[1]) == CONST_INT);
 
-  operands[9] = GEN_INT (xorv);
-  operands[10] = GEN_INT (sextc);
-})
+  operands[0] = XEXP (operands[0], 0);
 
-(define_insn "*cmpsi_internal2"
-  [(set (match_operand:CCUNS 0 "cc_reg_operand" "=y")
-       (compare:CCUNS (match_operand:SI 1 "gpc_reg_operand" "r")
-                      (match_operand:SI 2 "reg_or_u_short_operand" "rK")))]
-  ""
-  "cmplw%I2 %0,%1,%b2"
-  [(set_attr "type" "cmp")])
+  if (DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2)
+    {
+      rs6000_sibcall_aix (NULL_RTX, operands[0], operands[1], operands[2]);
+      DONE;
+    }
+}")
 
-(define_insn "*cmpdi_internal2"
-  [(set (match_operand:CCUNS 0 "cc_reg_operand" "=y")
-       (compare:CCUNS (match_operand:DI 1 "gpc_reg_operand" "r")
-                      (match_operand:DI 2 "reg_or_u_short_operand" "rK")))]
+(define_expand "sibcall_value"
+  [(parallel [(set (match_operand 0 "register_operand" "")
+               (call (mem:SI (match_operand 1 "address_operand" ""))
+                     (match_operand 2 "" "")))
+             (use (match_operand 3 "" ""))
+             (use (reg:SI LR_REGNO))
+             (simple_return)])]
   ""
-  "cmpld%I2 %0,%1,%b2"
-  [(set_attr "type" "cmp")])
+  "
+{
+#if TARGET_MACHO
+  if (MACHOPIC_INDIRECT)
+    operands[1] = machopic_indirect_call_target (operands[1]);
+#endif
 
-;; The following two insns don't exist as single insns, but if we provide
-;; them, we can swap an add and compare, which will enable us to overlap more
-;; of the required delay between a compare and branch.  We generate code for
-;; them by splitting.
+  gcc_assert (GET_CODE (operands[1]) == MEM);
+  gcc_assert (GET_CODE (operands[2]) == CONST_INT);
 
-(define_insn ""
-  [(set (match_operand:CC 3 "cc_reg_operand" "=y")
-       (compare:CC (match_operand:SI 1 "gpc_reg_operand" "r")
-                   (match_operand:SI 2 "short_cint_operand" "i")))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=r")
-       (plus:SI (match_dup 1) (match_operand:SI 4 "short_cint_operand" "i")))]
-  ""
-  "#"
-  [(set_attr "length" "8")])
+  operands[1] = XEXP (operands[1], 0);
 
-(define_insn ""
-  [(set (match_operand:CCUNS 3 "cc_reg_operand" "=y")
-       (compare:CCUNS (match_operand:SI 1 "gpc_reg_operand" "r")
-                      (match_operand:SI 2 "u_short_cint_operand" "i")))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=r")
-       (plus:SI (match_dup 1) (match_operand:SI 4 "short_cint_operand" "i")))]
-  ""
-  "#"
-  [(set_attr "length" "8")])
+  if (DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2)
+    {
+      rs6000_sibcall_aix (operands[0], operands[1], operands[2], operands[3]);
+      DONE;
+    }
+}")
 
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_operand" "")
-       (compare:CC (match_operand:SI 1 "gpc_reg_operand" "")
-                   (match_operand:SI 2 "short_cint_operand" "")))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (plus:SI (match_dup 1) (match_operand:SI 4 "short_cint_operand" "")))]
-  ""
-  [(set (match_dup 3) (compare:CC (match_dup 1) (match_dup 2)))
-   (set (match_dup 0) (plus:SI (match_dup 1) (match_dup 4)))])
+;; this and similar patterns must be marked as using LR, otherwise
+;; dataflow will try to delete the store into it.  This is true
+;; even when the actual reg to jump to is in CTR, when LR was
+;; saved and restored around the PIC-setting BCL.
+(define_insn "*sibcall_local32"
+  [(call (mem:SI (match_operand:SI 0 "current_file_function_operand" "s,s"))
+        (match_operand 1 "" "g,g"))
+   (use (match_operand:SI 2 "immediate_operand" "O,n"))
+   (use (reg:SI LR_REGNO))
+   (simple_return)]
+  "(INTVAL (operands[2]) & CALL_LONG) == 0"
+  "*
+{
+  if (INTVAL (operands[2]) & CALL_V4_SET_FP_ARGS)
+    output_asm_insn (\"crxor 6,6,6\", operands);
 
-(define_split
-  [(set (match_operand:CCUNS 3 "cc_reg_operand" "")
-       (compare:CCUNS (match_operand:SI 1 "gpc_reg_operand" "")
-                      (match_operand:SI 2 "u_short_cint_operand" "")))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (plus:SI (match_dup 1) (match_operand:SI 4 "short_cint_operand" "")))]
-  ""
-  [(set (match_dup 3) (compare:CCUNS (match_dup 1) (match_dup 2)))
-   (set (match_dup 0) (plus:SI (match_dup 1) (match_dup 4)))])
+  else if (INTVAL (operands[2]) & CALL_V4_CLEAR_FP_ARGS)
+    output_asm_insn (\"creqv 6,6,6\", operands);
 
-;; Only need to compare second words if first words equal
-(define_insn "*cmptf_internal1"
-  [(set (match_operand:CCFP 0 "cc_reg_operand" "=y")
-       (compare:CCFP (match_operand:TF 1 "gpc_reg_operand" "d")
-                     (match_operand:TF 2 "gpc_reg_operand" "d")))]
-  "!TARGET_IEEEQUAD && !TARGET_XL_COMPAT
-   && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT && TARGET_LONG_DOUBLE_128"
-  "fcmpu %0,%1,%2\;bne %0,$+8\;fcmpu %0,%L1,%L2"
-  [(set_attr "type" "fpcompare")
-   (set_attr "length" "12")])
+  return (DEFAULT_ABI == ABI_V4 && flag_pic) ? \"b %z0@local\" : \"b %z0\";
+}"
+  [(set_attr "type" "branch")
+   (set_attr "length" "4,8")])
 
-(define_insn_and_split "*cmptf_internal2"
-  [(set (match_operand:CCFP 0 "cc_reg_operand" "=y")
-       (compare:CCFP (match_operand:TF 1 "gpc_reg_operand" "d")
-                     (match_operand:TF 2 "gpc_reg_operand" "d")))
-    (clobber (match_scratch:DF 3 "=d"))
-    (clobber (match_scratch:DF 4 "=d"))
-    (clobber (match_scratch:DF 5 "=d"))
-    (clobber (match_scratch:DF 6 "=d"))
-    (clobber (match_scratch:DF 7 "=d"))
-    (clobber (match_scratch:DF 8 "=d"))
-    (clobber (match_scratch:DF 9 "=d"))
-    (clobber (match_scratch:DF 10 "=d"))
-    (clobber (match_scratch:GPR 11 "=b"))]
-  "!TARGET_IEEEQUAD && TARGET_XL_COMPAT
-   && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT && TARGET_LONG_DOUBLE_128"
-  "#"
-  "&& reload_completed"
-  [(set (match_dup 3) (match_dup 14))
-   (set (match_dup 4) (match_dup 15))
-   (set (match_dup 9) (abs:DF (match_dup 5)))
-   (set (match_dup 0) (compare:CCFP (match_dup 9) (match_dup 3)))
-   (set (pc) (if_then_else (ne (match_dup 0) (const_int 0))
-                          (label_ref (match_dup 12))
-                          (pc)))
-   (set (match_dup 0) (compare:CCFP (match_dup 5) (match_dup 7)))
-   (set (pc) (label_ref (match_dup 13)))
-   (match_dup 12)
-   (set (match_dup 10) (minus:DF (match_dup 5) (match_dup 7)))
-   (set (match_dup 9) (minus:DF (match_dup 6) (match_dup 8)))
-   (set (match_dup 9) (plus:DF (match_dup 10) (match_dup 9)))
-   (set (match_dup 0) (compare:CCFP (match_dup 9) (match_dup 4)))
-   (match_dup 13)]
+(define_insn "*sibcall_local64"
+  [(call (mem:SI (match_operand:DI 0 "current_file_function_operand" "s,s"))
+        (match_operand 1 "" "g,g"))
+   (use (match_operand:SI 2 "immediate_operand" "O,n"))
+   (use (reg:SI LR_REGNO))
+   (simple_return)]
+  "TARGET_64BIT && (INTVAL (operands[2]) & CALL_LONG) == 0"
+  "*
 {
-  REAL_VALUE_TYPE rv;
-  const int lo_word = LONG_DOUBLE_LARGE_FIRST ? GET_MODE_SIZE (DFmode) : 0;
-  const int hi_word = LONG_DOUBLE_LARGE_FIRST ? 0 : GET_MODE_SIZE (DFmode);
+  if (INTVAL (operands[2]) & CALL_V4_SET_FP_ARGS)
+    output_asm_insn (\"crxor 6,6,6\", operands);
 
-  operands[5] = simplify_gen_subreg (DFmode, operands[1], TFmode, hi_word);
-  operands[6] = simplify_gen_subreg (DFmode, operands[1], TFmode, lo_word);
-  operands[7] = simplify_gen_subreg (DFmode, operands[2], TFmode, hi_word);
-  operands[8] = simplify_gen_subreg (DFmode, operands[2], TFmode, lo_word);
-  operands[12] = gen_label_rtx ();
-  operands[13] = gen_label_rtx ();
-  real_inf (&rv);
-  operands[14] = force_const_mem (DFmode,
-                                 CONST_DOUBLE_FROM_REAL_VALUE (rv, DFmode));
-  operands[15] = force_const_mem (DFmode,
-                                 CONST_DOUBLE_FROM_REAL_VALUE (dconst0,
-                                                               DFmode));
-  if (TARGET_TOC)
-    {
-      rtx tocref;
-      tocref = create_TOC_reference (XEXP (operands[14], 0), operands[11]);
-      operands[14] = gen_const_mem (DFmode, tocref);
-      tocref = create_TOC_reference (XEXP (operands[15], 0), operands[11]);
-      operands[15] = gen_const_mem (DFmode, tocref);
-      set_mem_alias_set (operands[14], get_TOC_alias_set ());
-      set_mem_alias_set (operands[15], get_TOC_alias_set ());
-    }
-})
-\f
-;; Now we have the scc insns.  We can do some combinations because of the
-;; way the machine works.
-;;
-;; Note that this is probably faster if we can put an insn between the
-;; mfcr and rlinm, but this is tricky.  Let's leave it for now.  In most
-;; cases the insns below which don't use an intermediate CR field will
-;; be used instead.
-(define_insn ""
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-       (match_operator:SI 1 "scc_comparison_operator"
-                          [(match_operand 2 "cc_reg_operand" "y")
-                           (const_int 0)]))]
-  ""
-  "mfcr %0%Q2\;rlwinm %0,%0,%J1,1"
-  [(set (attr "type")
-     (cond [(match_test "TARGET_MFCRF")
-               (const_string "mfcrf")
-          ]
-       (const_string "mfcr")))
-   (set_attr "length" "8")])
+  else if (INTVAL (operands[2]) & CALL_V4_CLEAR_FP_ARGS)
+    output_asm_insn (\"creqv 6,6,6\", operands);
 
-;; Same as above, but get the GT bit.
-(define_insn "move_from_CR_gt_bit"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-       (unspec:SI [(match_operand 1 "cc_reg_operand" "y")] UNSPEC_MV_CR_GT))]
-  "TARGET_HARD_FLOAT && !TARGET_FPRS"
-  "mfcr %0\;rlwinm %0,%0,%D1,31,31"
-  [(set_attr "type" "mfcr")
-   (set_attr "length" "8")])
+  return (DEFAULT_ABI == ABI_V4 && flag_pic) ? \"b %z0@local\" : \"b %z0\";
+}"
+  [(set_attr "type" "branch")
+   (set_attr "length" "4,8")])
 
-;; Same as above, but get the OV/ORDERED bit.
-(define_insn "move_from_CR_ov_bit"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-       (unspec:SI [(match_operand:CC 1 "cc_reg_operand" "y")]
-                  UNSPEC_MV_CR_OV))]
-  "TARGET_ISEL"
-  "mfcr %0\;rlwinm %0,%0,%t1,1"
-  [(set_attr "type" "mfcr")
-   (set_attr "length" "8")])
+(define_insn "*sibcall_value_local32"
+  [(set (match_operand 0 "" "")
+       (call (mem:SI (match_operand:SI 1 "current_file_function_operand" "s,s"))
+             (match_operand 2 "" "g,g")))
+   (use (match_operand:SI 3 "immediate_operand" "O,n"))
+   (use (reg:SI LR_REGNO))
+   (simple_return)]
+  "(INTVAL (operands[3]) & CALL_LONG) == 0"
+  "*
+{
+  if (INTVAL (operands[3]) & CALL_V4_SET_FP_ARGS)
+    output_asm_insn (\"crxor 6,6,6\", operands);
 
-(define_insn ""
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
-       (match_operator:DI 1 "scc_comparison_operator"
-                          [(match_operand 2 "cc_reg_operand" "y")
-                           (const_int 0)]))]
-  "TARGET_POWERPC64"
-  "mfcr %0%Q2\;rlwinm %0,%0,%J1,1"
-  [(set (attr "type")
-     (cond [(match_test "TARGET_MFCRF")
-               (const_string "mfcrf")
-          ]
-       (const_string "mfcr")))
-   (set_attr "length" "8")])
+  else if (INTVAL (operands[3]) & CALL_V4_CLEAR_FP_ARGS)
+    output_asm_insn (\"creqv 6,6,6\", operands);
+
+  return (DEFAULT_ABI == ABI_V4 && flag_pic) ? \"b %z1@local\" : \"b %z1\";
+}"
+  [(set_attr "type" "branch")
+   (set_attr "length" "4,8")])
+
+(define_insn "*sibcall_value_local64"
+  [(set (match_operand 0 "" "")
+       (call (mem:SI (match_operand:DI 1 "current_file_function_operand" "s,s"))
+             (match_operand 2 "" "g,g")))
+   (use (match_operand:SI 3 "immediate_operand" "O,n"))
+   (use (reg:SI LR_REGNO))
+   (simple_return)]
+  "TARGET_64BIT && (INTVAL (operands[3]) & CALL_LONG) == 0"
+  "*
+{
+  if (INTVAL (operands[3]) & CALL_V4_SET_FP_ARGS)
+    output_asm_insn (\"crxor 6,6,6\", operands);
 
-(define_insn ""
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC (match_operator:SI 1 "scc_comparison_operator"
-                                      [(match_operand 2 "cc_reg_operand" "y,y")
-                                       (const_int 0)])
-                   (const_int 0)))
-   (set (match_operand:SI 3 "gpc_reg_operand" "=r,r")
-       (match_op_dup 1 [(match_dup 2) (const_int 0)]))]
-  "TARGET_32BIT"
-  "@
-   mfcr %3%Q2\;rlwinm. %3,%3,%J1,1
-   #"
-  [(set_attr "type" "shift")
-   (set_attr "dot" "yes")
-   (set_attr "length" "8,16")])
+  else if (INTVAL (operands[3]) & CALL_V4_CLEAR_FP_ARGS)
+    output_asm_insn (\"creqv 6,6,6\", operands);
 
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_cr0_operand" "")
-       (compare:CC (match_operator:SI 1 "scc_comparison_operator"
-                                      [(match_operand 2 "cc_reg_operand" "")
-                                       (const_int 0)])
-                   (const_int 0)))
-   (set (match_operand:SI 3 "gpc_reg_operand" "")
-       (match_op_dup 1 [(match_dup 2) (const_int 0)]))]
-  "TARGET_32BIT && reload_completed"
-  [(set (match_dup 3)
-       (match_op_dup 1 [(match_dup 2) (const_int 0)]))
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
+  return (DEFAULT_ABI == ABI_V4 && flag_pic) ? \"b %z1@local\" : \"b %z1\";
+}"
+  [(set_attr "type" "branch")
+   (set_attr "length" "4,8")])
 
-(define_insn ""
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-       (ashift:SI (match_operator:SI 1 "scc_comparison_operator"
-                                     [(match_operand 2 "cc_reg_operand" "y")
-                                      (const_int 0)])
-                  (match_operand:SI 3 "const_int_operand" "n")))]
-  ""
+(define_insn "*sibcall_nonlocal_sysv<mode>"
+  [(call (mem:SI (match_operand:P 0 "call_operand" "s,s,c,c"))
+        (match_operand 1 "" ""))
+   (use (match_operand 2 "immediate_operand" "O,n,O,n"))
+   (use (reg:SI LR_REGNO))
+   (simple_return)]
+  "(DEFAULT_ABI == ABI_DARWIN
+    || DEFAULT_ABI == ABI_V4)
+   && (INTVAL (operands[2]) & CALL_LONG) == 0"
   "*
 {
-  int is_bit = ccr_bit (operands[1], 1);
-  int put_bit = 31 - (INTVAL (operands[3]) & 31);
-  int count;
-
-  if (is_bit >= put_bit)
-    count = is_bit - put_bit;
-  else
-    count = 32 - (put_bit - is_bit);
+  if (INTVAL (operands[2]) & CALL_V4_SET_FP_ARGS)
+    output_asm_insn (\"crxor 6,6,6\", operands);
 
-  operands[4] = GEN_INT (count);
-  operands[5] = GEN_INT (put_bit);
+  else if (INTVAL (operands[2]) & CALL_V4_CLEAR_FP_ARGS)
+    output_asm_insn (\"creqv 6,6,6\", operands);
 
-  return \"mfcr %0%Q2\;rlwinm %0,%0,%4,%5,%5\";
+  if (which_alternative >= 2)
+    return \"b%T0\";
+  else if (DEFAULT_ABI == ABI_V4 && flag_pic)
+    {
+      gcc_assert (!TARGET_SECURE_PLT);
+      return \"b %z0@plt\";
+    }
+  else
+    return \"b %z0\";
 }"
-  [(set (attr "type")
-     (cond [(match_test "TARGET_MFCRF")
-               (const_string "mfcrf")
-          ]
-       (const_string "mfcr")))
-   (set_attr "length" "8")])
+  [(set_attr "type" "branch")
+   (set_attr "length" "4,8,4,8")])
 
-(define_insn ""
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (ashift:SI (match_operator:SI 1 "scc_comparison_operator"
-                                      [(match_operand 2 "cc_reg_operand" "y,y")
-                                       (const_int 0)])
-                   (match_operand:SI 3 "const_int_operand" "n,n"))
-        (const_int 0)))
-   (set (match_operand:SI 4 "gpc_reg_operand" "=r,r")
-       (ashift:SI (match_op_dup 1 [(match_dup 2) (const_int 0)])
-                  (match_dup 3)))]
-  ""
+(define_insn "*sibcall_value_nonlocal_sysv<mode>"
+  [(set (match_operand 0 "" "")
+       (call (mem:SI (match_operand:P 1 "call_operand" "s,s,c,c"))
+             (match_operand 2 "" "")))
+   (use (match_operand:SI 3 "immediate_operand" "O,n,O,n"))
+   (use (reg:SI LR_REGNO))
+   (simple_return)]
+  "(DEFAULT_ABI == ABI_DARWIN
+    || DEFAULT_ABI == ABI_V4)
+   && (INTVAL (operands[3]) & CALL_LONG) == 0"
   "*
 {
-  int is_bit = ccr_bit (operands[1], 1);
-  int put_bit = 31 - (INTVAL (operands[3]) & 31);
-  int count;
+  if (INTVAL (operands[3]) & CALL_V4_SET_FP_ARGS)
+    output_asm_insn (\"crxor 6,6,6\", operands);
 
-  /* Force split for non-cc0 compare.  */
-  if (which_alternative == 1)
-     return \"#\";
+  else if (INTVAL (operands[3]) & CALL_V4_CLEAR_FP_ARGS)
+    output_asm_insn (\"creqv 6,6,6\", operands);
 
-  if (is_bit >= put_bit)
-    count = is_bit - put_bit;
+  if (which_alternative >= 2)
+    return \"b%T1\";
+  else if (DEFAULT_ABI == ABI_V4 && flag_pic)
+    {
+      gcc_assert (!TARGET_SECURE_PLT);
+      return \"b %z1@plt\";
+    }
   else
-    count = 32 - (put_bit - is_bit);
+    return \"b %z1\";
+}"
+  [(set_attr "type" "branch")
+   (set_attr "length" "4,8,4,8")])
 
-  operands[5] = GEN_INT (count);
-  operands[6] = GEN_INT (put_bit);
+;; AIX ABI sibling call patterns.
 
-  return \"mfcr %4%Q2\;rlwinm. %4,%4,%5,%6,%6\";
-}"
-  [(set_attr "type" "shift")
-   (set_attr "dot" "yes")
-   (set_attr "length" "8,16")])
+(define_insn "*sibcall_aix<mode>"
+  [(call (mem:SI (match_operand:P 0 "call_operand" "s,c"))
+        (match_operand 1 "" "g,g"))
+   (simple_return)]
+  "DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2"
+  "@
+   b %z0
+   b%T0"
+  [(set_attr "type" "branch")
+   (set_attr "length" "4")])
 
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC
-        (ashift:SI (match_operator:SI 1 "scc_comparison_operator"
-                                      [(match_operand 2 "cc_reg_operand" "")
-                                       (const_int 0)])
-                   (match_operand:SI 3 "const_int_operand" ""))
-        (const_int 0)))
-   (set (match_operand:SI 4 "gpc_reg_operand" "")
-       (ashift:SI (match_op_dup 1 [(match_dup 2) (const_int 0)])
-                  (match_dup 3)))]
-  "reload_completed"
-  [(set (match_dup 4)
-       (ashift:SI (match_op_dup 1 [(match_dup 2) (const_int 0)])
-                  (match_dup 3)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 4)
-                   (const_int 0)))]
+(define_insn "*sibcall_value_aix<mode>"
+  [(set (match_operand 0 "" "")
+       (call (mem:SI (match_operand:P 1 "call_operand" "s,c"))
+             (match_operand 2 "" "g,g")))
+   (simple_return)]
+  "DEFAULT_ABI == ABI_AIX || DEFAULT_ABI == ABI_ELFv2"
+  "@
+   b %z1
+   b%T1"
+  [(set_attr "type" "branch")
+   (set_attr "length" "4")])
+
+(define_expand "sibcall_epilogue"
+  [(use (const_int 0))]
+  ""
+{
+  if (!TARGET_SCHED_PROLOG)
+    emit_insn (gen_blockage ());
+  rs6000_emit_epilogue (TRUE);
+  DONE;
+})
+
+;; UNSPEC_VOLATILE is considered to use and clobber all hard registers and
+;; all of memory.  This blocks insns from being moved across this point.
+
+(define_insn "blockage"
+  [(unspec_volatile [(const_int 0)] UNSPECV_BLOCK)]
+  ""
   "")
 
-;; There is a 3 cycle delay between consecutive mfcr instructions
-;; so it is useful to combine 2 scc instructions to use only one mfcr.
+(define_expand "probe_stack_address"
+  [(use (match_operand 0 "address_operand"))]
+  ""
+{
+  operands[0] = gen_rtx_MEM (Pmode, operands[0]);
+  MEM_VOLATILE_P (operands[0]) = 1;
 
-(define_peephole
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
-       (match_operator:SI 1 "scc_comparison_operator"
-                          [(match_operand 2 "cc_reg_operand" "y")
-                           (const_int 0)]))
-   (set (match_operand:SI 3 "gpc_reg_operand" "=r")
-       (match_operator:SI 4 "scc_comparison_operator"
-                          [(match_operand 5 "cc_reg_operand" "y")
-                           (const_int 0)]))]
-  "REGNO (operands[2]) != REGNO (operands[5])"
-  "mfcr %3\;rlwinm %0,%3,%J1,1\;rlwinm %3,%3,%J4,1"
-  [(set_attr "type" "mfcr")
-   (set_attr "length" "12")])
+  if (TARGET_64BIT)
+    emit_insn (gen_probe_stack_di (operands[0]));
+  else
+    emit_insn (gen_probe_stack_si (operands[0]));
+  DONE;
+})
 
-(define_peephole
-  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
-       (match_operator:DI 1 "scc_comparison_operator"
-                          [(match_operand 2 "cc_reg_operand" "y")
-                           (const_int 0)]))
-   (set (match_operand:DI 3 "gpc_reg_operand" "=r")
-       (match_operator:DI 4 "scc_comparison_operator"
-                          [(match_operand 5 "cc_reg_operand" "y")
-                           (const_int 0)]))]
-  "TARGET_POWERPC64 && REGNO (operands[2]) != REGNO (operands[5])"
-  "mfcr %3\;rlwinm %0,%3,%J1,1\;rlwinm %3,%3,%J4,1"
-  [(set_attr "type" "mfcr")
-   (set_attr "length" "12")])
+(define_insn "probe_stack_<mode>"
+  [(set (match_operand:P 0 "memory_operand" "=m")
+        (unspec:P [(const_int 0)] UNSPEC_PROBE_STACK))]
+  ""
+{
+  operands[1] = gen_rtx_REG (Pmode, 0);
+  return "st<wd>%U0%X0 %1,%0";
+}
+  [(set_attr "type" "store")
+   (set (attr "update")
+       (if_then_else (match_operand 0 "update_address_mem")
+                     (const_string "yes")
+                     (const_string "no")))
+   (set (attr "indexed")
+       (if_then_else (match_operand 0 "indexed_address_mem")
+                     (const_string "yes")
+                     (const_string "no")))
+   (set_attr "length" "4")])
 
-;; There are some scc insns that can be done directly, without a compare.
-;; These are faster because they don't involve the communications between
-;; the FXU and branch units.   In fact, we will be replacing all of the
-;; integer scc insns here or in the portable methods in emit_store_flag.
+(define_insn "probe_stack_range<P:mode>"
+  [(set (match_operand:P 0 "register_operand" "=r")
+       (unspec_volatile:P [(match_operand:P 1 "register_operand" "0")
+                           (match_operand:P 2 "register_operand" "r")]
+                          UNSPECV_PROBE_STACK_RANGE))]
+  ""
+  "* return output_probe_stack_range (operands[0], operands[2]);"
+  [(set_attr "type" "three")])
+\f
+;; Compare insns are next.  Note that the RS/6000 has two types of compares,
+;; signed & unsigned, and one type of branch.
 ;;
-;; Also support (neg (scc ..)) since that construct is used to replace
-;; branches, (plus (scc ..) ..) since that construct is common and
-;; takes no more insns than scc, and (and (neg (scc ..)) ..) in the
-;; cases where it is no more expensive than (neg (scc ..)).
+;; Start with the DEFINE_EXPANDs to generate the rtl for compares, scc
+;; insns, and branches.
 
-;; Have reload force a constant into a register for the simple insns that
-;; otherwise won't accept constants.  We do this because it is faster than
-;; the cmp/mfcr sequence we would otherwise generate.
+(define_expand "cbranch<mode>4"
+  [(use (match_operator 0 "rs6000_cbranch_operator"
+         [(match_operand:GPR 1 "gpc_reg_operand" "")
+          (match_operand:GPR 2 "reg_or_short_operand" "")]))
+   (use (match_operand 3 ""))]
+  ""
+  "
+{
+  /* Take care of the possibility that operands[2] might be negative but
+     this might be a logical operation.  That insn doesn't exist.  */
+  if (GET_CODE (operands[2]) == CONST_INT
+      && INTVAL (operands[2]) < 0)
+    {
+      operands[2] = force_reg (<MODE>mode, operands[2]);
+      operands[0] = gen_rtx_fmt_ee (GET_CODE (operands[0]),
+                                   GET_MODE (operands[0]),
+                                   operands[1], operands[2]);
+   }
 
-(define_mode_attr scc_eq_op2 [(SI "rKLI")
-                             (DI "rKJI")])
+  rs6000_emit_cbranch (<MODE>mode, operands);
+  DONE;
+}")
 
-(define_insn_and_split "*eq<mode>"
-  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
-       (eq:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")
-               (match_operand:GPR 2 "scc_eq_operand" "<scc_eq_op2>")))]
-  ""
-  "#"
+(define_expand "cbranch<mode>4"
+  [(use (match_operator 0 "rs6000_cbranch_operator"
+         [(match_operand:FP 1 "gpc_reg_operand" "")
+          (match_operand:FP 2 "gpc_reg_operand" "")]))
+   (use (match_operand 3 ""))]
   ""
-  [(set (match_dup 0)
-       (clz:GPR (match_dup 3)))
-   (set (match_dup 0)
-       (lshiftrt:GPR (match_dup 0) (match_dup 4)))]
-  {
-    if (GET_CODE (operands[2]) != CONST_INT || INTVAL (operands[2]) != 0)
-      {
-       /* Use output operand as intermediate.  */
-       operands[3] = operands[0];
-
-       if (logical_operand (operands[2], <MODE>mode))
-         emit_insn (gen_rtx_SET (VOIDmode, operands[3],
-                                 gen_rtx_XOR (<MODE>mode,
-                                              operands[1], operands[2])));
-       else
-         emit_insn (gen_rtx_SET (VOIDmode, operands[3],
-                                 gen_rtx_PLUS (<MODE>mode, operands[1],
-                                               negate_rtx (<MODE>mode,
-                                                           operands[2]))));
-      }
-    else
-      operands[3] = operands[1];
+  "
+{
+  rs6000_emit_cbranch (<MODE>mode, operands);
+  DONE;
+}")
 
-    operands[4] = GEN_INT (exact_log2 (GET_MODE_BITSIZE (<MODE>mode)));
-  })
+(define_expand "cstore<mode>4_unsigned"
+  [(use (match_operator 1 "unsigned_comparison_operator"
+         [(match_operand:P 2 "gpc_reg_operand" "")
+          (match_operand:P 3 "reg_or_short_operand" "")]))
+   (clobber (match_operand:P 0 "register_operand"))]
+  ""
+{
+  enum rtx_code cond_code = GET_CODE (operands[1]);
 
-(define_insn_and_split "*eq<mode>_compare"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=y")
-       (compare:CC
-        (eq:P (match_operand:P 1 "gpc_reg_operand" "=r")
-              (match_operand:P 2 "scc_eq_operand" "<scc_eq_op2>"))
-        (const_int 0)))
-   (set (match_operand:P 0 "gpc_reg_operand" "=r")
-       (eq:P (match_dup 1) (match_dup 2)))]
-  "optimize_size"
-  "#"
-  "optimize_size"
-  [(set (match_dup 0)
-       (clz:P (match_dup 4)))
-   (parallel [(set (match_dup 3)
-                  (compare:CC (lshiftrt:P (match_dup 0) (match_dup 5))
-                              (const_int 0)))
-             (set (match_dup 0)
-                  (lshiftrt:P (match_dup 0) (match_dup 5)))])]
-  {
-    if (GET_CODE (operands[2]) != CONST_INT || INTVAL (operands[2]) != 0)
-      {
-       /* Use output operand as intermediate.  */
-       operands[4] = operands[0];
-
-       if (logical_operand (operands[2], <MODE>mode))
-         emit_insn (gen_rtx_SET (VOIDmode, operands[4],
-                                 gen_rtx_XOR (<MODE>mode,
-                                              operands[1], operands[2])));
-       else
-         emit_insn (gen_rtx_SET (VOIDmode, operands[4],
-                                 gen_rtx_PLUS (<MODE>mode, operands[1],
-                                               negate_rtx (<MODE>mode,
-                                                           operands[2]))));
-      }
-    else
-      operands[4] = operands[1];
+  rtx op0 = operands[0];
+  rtx op1 = operands[2];
+  rtx op2 = operands[3];
 
-    operands[5] = GEN_INT (exact_log2 (GET_MODE_BITSIZE (<MODE>mode)));
-  })
+  if (cond_code == GEU || cond_code == LTU)
+    {
+      cond_code = swap_condition (cond_code);
+      op1 = operands[3];
+      op2 = operands[2];
+    }
 
-;; We have insns of the form shown by the first define_insn below.  If
-;; there is something inside the comparison operation, we must split it.
-(define_split
-  [(set (match_operand:SI 0 "gpc_reg_operand" "")
-       (plus:SI (match_operator 1 "comparison_operator"
-                                [(match_operand:SI 2 "" "")
-                                 (match_operand:SI 3
-                                                   "reg_or_cint_operand" "")])
-                (match_operand:SI 4 "gpc_reg_operand" "")))
-   (clobber (match_operand:SI 5 "register_operand" ""))]
-  "! gpc_reg_operand (operands[2], SImode)"
-  [(set (match_dup 5) (match_dup 2))
-   (set (match_dup 0) (plus:SI (match_op_dup 1 [(match_dup 5) (match_dup 3)])
-                              (match_dup 4)))])
-
-(define_insn "*plus_eqsi"
-  [(set (match_operand:SI 0 "gpc_reg_operand" "=&r,&r,&r,&r,&r")
-       (plus:SI (eq:SI (match_operand:SI 1 "gpc_reg_operand" "%r,r,r,r,r")
-                       (match_operand:SI 2 "scc_eq_operand" "r,O,K,L,I"))
-                (match_operand:SI 3 "gpc_reg_operand" "r,r,r,r,r")))]
-  "TARGET_32BIT"
-  "@
-   xor %0,%1,%2\;subfic %0,%0,0\;addze %0,%3
-   subfic %0,%1,0\;addze %0,%3
-   xori %0,%1,%b2\;subfic %0,%0,0\;addze %0,%3
-   xoris %0,%1,%u2\;subfic %0,%0,0\;addze %0,%3
-   subfic %0,%1,%2\;subfic %0,%0,0\;addze %0,%3"
-  [(set_attr "type" "three,two,three,three,three")
-   (set_attr "length" "12,8,12,12,12")])
+  if (!gpc_reg_operand (op1, <MODE>mode))
+    op1 = force_reg (<MODE>mode, op1);
+  if (!reg_or_short_operand (op2, <MODE>mode))
+    op2 = force_reg (<MODE>mode, op2);
 
-(define_insn "*compare_plus_eqsi"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,x,x,x,x,?y,?y,?y,?y,?y")
-       (compare:CC
-        (plus:SI
-         (eq:SI (match_operand:SI 1 "gpc_reg_operand" "%r,r,r,r,r,r,r,r,r,r")
-                (match_operand:SI 2 "scc_eq_operand" "r,O,K,L,I,r,O,K,L,I"))
-         (match_operand:SI 3 "gpc_reg_operand" "r,r,r,r,r,r,r,r,r,r"))
-        (const_int 0)))
-   (clobber (match_scratch:SI 4 "=&r,&r,&r,&r,&r,&r,&r,&r,&r,&r"))]
-  "TARGET_32BIT && optimize_size"
-  "@
-   xor %4,%1,%2\;subfic %4,%4,0\;addze. %4,%3
-   subfic %4,%1,0\;addze. %4,%3
-   xori %4,%1,%b2\;subfic %4,%4,0\;addze. %4,%3
-   xoris %4,%1,%u2\;subfic %4,%4,0\;addze. %4,%3
-   subfic %4,%1,%2\;subfic %4,%4,0\;addze. %4,%3
-   #
-   #
-   #
-   #
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "12,8,12,12,12,16,12,16,16,16")])
+  rtx tmp = gen_reg_rtx (<MODE>mode);
+  rtx tmp2 = gen_reg_rtx (<MODE>mode);
 
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_cr0_operand" "")
-       (compare:CC
-        (plus:SI
-         (eq:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                (match_operand:SI 2 "scc_eq_operand" ""))
-         (match_operand:SI 3 "gpc_reg_operand" ""))
-        (const_int 0)))
-   (clobber (match_scratch:SI 4 ""))]
-  "TARGET_32BIT && optimize_size && reload_completed"
-  [(set (match_dup 4)
-       (plus:SI (eq:SI (match_dup 1)
-                (match_dup 2))
-         (match_dup 3)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 4)
-                   (const_int 0)))]
-  "")
+  emit_insn (gen_subf<mode>3_carry (tmp, op1, op2));
+  emit_insn (gen_subf<mode>3_carry_in_xx (tmp2));
 
-(define_insn "*plus_eqsi_compare"
-  [(set (match_operand:CC 4 "cc_reg_operand" "=x,x,x,x,x,?y,?y,?y,?y,?y")
-       (compare:CC
-        (plus:SI
-         (eq:SI (match_operand:SI 1 "gpc_reg_operand" "%r,r,r,r,r,r,r,r,r,r")
-                (match_operand:SI 2 "scc_eq_operand" "r,O,K,L,I,r,O,K,L,I"))
-         (match_operand:SI 3 "gpc_reg_operand" "r,r,r,r,r,r,r,r,r,r"))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=&r,&r,&r,&r,&r,&r,&r,&r,&r,&r")
-       (plus:SI (eq:SI (match_dup 1) (match_dup 2)) (match_dup 3)))]
-  "TARGET_32BIT && optimize_size"
-  "@
-   xor %0,%1,%2\;subfic %0,%0,0\;addze. %0,%3
-   subfic %0,%1,0\;addze. %0,%3
-   xori %0,%1,%b2\;subfic %0,%0,0\;addze. %0,%3
-   xoris %0,%1,%u2\;subfic %0,%0,0\;addze. %0,%3
-   subfic %0,%1,%2\;subfic %0,%0,0\;addze. %0,%3
-   #
-   #
-   #
-   #
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "12,8,12,12,12,16,12,16,16,16")])
+  if (cond_code == LEU)
+    emit_insn (gen_add<mode>3 (op0, tmp2, const1_rtx));
+  else
+    emit_insn (gen_neg<mode>2 (op0, tmp2));
 
-(define_split
-  [(set (match_operand:CC 4 "cc_reg_not_cr0_operand" "")
-       (compare:CC
-        (plus:SI
-         (eq:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                (match_operand:SI 2 "scc_eq_operand" ""))
-         (match_operand:SI 3 "gpc_reg_operand" ""))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (plus:SI (eq:SI (match_dup 1) (match_dup 2)) (match_dup 3)))]
-  "TARGET_32BIT && optimize_size && reload_completed"
-  [(set (match_dup 0)
-       (plus:SI (eq:SI (match_dup 1) (match_dup 2)) (match_dup 3)))
-   (set (match_dup 4)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
+  DONE;
+})
 
-(define_insn "*neg_eq0<mode>"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
-       (neg:P (eq:P (match_operand:P 1 "gpc_reg_operand" "r")
-                    (const_int 0))))]
+(define_expand "cstore<mode>4_signed_imm"
+  [(use (match_operator 1 "signed_comparison_operator"
+         [(match_operand:GPR 2 "gpc_reg_operand")
+          (match_operand:GPR 3 "immediate_operand")]))
+   (clobber (match_operand:GPR 0 "register_operand"))]
   ""
-  "addic %0,%1,-1\;subfe %0,%0,%0"
-  [(set_attr "type" "two")
-   (set_attr "length" "8")])
+{
+  bool invert = false;
 
-(define_insn_and_split "*neg_eq<mode>"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
-       (neg:P (eq:P (match_operand:P 1 "gpc_reg_operand" "%r")
-                    (match_operand:P 2 "scc_eq_operand" "<scc_eq_op2>"))))]
-  ""
-  "#"
-  ""
-  [(set (match_dup 0) (neg:P (eq:P (match_dup 3) (const_int 0))))]
-  {
-    if (GET_CODE (operands[2]) != CONST_INT || INTVAL (operands[2]) != 0)
-      {
-       /* Use output operand as intermediate.  */
-       operands[3] = operands[0];
-
-       if (logical_operand (operands[2], <MODE>mode))
-         emit_insn (gen_rtx_SET (VOIDmode, operands[3],
-                                 gen_rtx_XOR (<MODE>mode,
-                                              operands[1], operands[2])));
-       else
-         emit_insn (gen_rtx_SET (VOIDmode, operands[3],
-                                 gen_rtx_PLUS (<MODE>mode, operands[1],
-                                               negate_rtx (<MODE>mode,
-                                                           operands[2]))));
-      }
-    else
-      operands[3] = operands[1];
-  })
+  enum rtx_code cond_code = GET_CODE (operands[1]);
 
-(define_insn "*ne0_<mode>"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
-       (ne:P (match_operand:P 1 "gpc_reg_operand" "r")
-             (const_int 0)))
-   (clobber (match_scratch:P 2 "=&r"))]
-  "!(TARGET_32BIT && TARGET_ISEL)"
-  "addic %2,%1,-1\;subfe %0,%2,%1"
-  [(set_attr "type" "two")
-   (set_attr "length" "8")])
+  rtx op0 = operands[0];
+  rtx op1 = operands[2];
+  HOST_WIDE_INT val = INTVAL (operands[3]);
 
-(define_insn "*plus_ne0_<mode>"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
-       (plus:P (ne:P (match_operand:P 1 "gpc_reg_operand" "r")
-                     (const_int 0))
-               (match_operand:P 2 "gpc_reg_operand" "r")))
-   (clobber (match_scratch:P 3 "=&r"))]
-  ""
-  "addic %3,%1,-1\;addze %0,%2"
-  [(set_attr "type" "two")
-   (set_attr "length" "8")])
+  if (cond_code == GE || cond_code == GT)
+    {
+      cond_code = reverse_condition (cond_code);
+      invert = true;
+    }
 
-(define_insn "*compare_plus_ne0_<mode>"
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC (plus:P (ne:P (match_operand:P 1 "gpc_reg_operand" "r,r")
-                                 (const_int 0))
-                           (match_operand:P 2 "gpc_reg_operand" "r,r"))
-                   (const_int 0)))
-   (clobber (match_scratch:P 3 "=&r,&r"))
-   (clobber (match_scratch:P 4 "=X,&r"))]
-  ""
-  "@
-   addic %3,%1,-1\;addze. %3,%2
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "8,12")])
+  if (cond_code == LE)
+    val++;
 
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC (ne:P (match_operand:SI 1 "gpc_reg_operand" "")
-                         (const_int 0))
-                   (neg:P (match_operand:P 2 "gpc_reg_operand" ""))))
-   (clobber (match_scratch:P 3 ""))
-   (clobber (match_scratch:P 4 ""))]
-  "reload_completed"
-  [(parallel [(set (match_dup 3)
-                  (plus:P (ne:P (match_dup 1)
-                                (const_int 0))
-                          (match_dup 2)))
-              (clobber (match_dup 4))])
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
+  rtx tmp = gen_reg_rtx (<MODE>mode);
+  emit_insn (gen_add<mode>3 (tmp, op1, GEN_INT (-val)));
+  rtx x = gen_reg_rtx (<MODE>mode);
+  if (val < 0)
+    emit_insn (gen_and<mode>3 (x, op1, tmp));
+  else
+    emit_insn (gen_ior<mode>3 (x, op1, tmp));
 
-; For combine.
-(define_insn "*compare_plus_ne0_<mode>_1"
-  [(set (match_operand:CCEQ 0 "cc_reg_operand" "=x,?y")
-       (compare:CCEQ (ne:P (match_operand:P 1 "gpc_reg_operand" "r,r")
-                           (const_int 0))
-                     (neg:P (match_operand:P 2 "gpc_reg_operand" "r,r"))))
-   (clobber (match_scratch:P 3 "=&r,&r"))
-   (clobber (match_scratch:P 4 "=X,&r"))]
-  ""
-  "@
-   addic %3,%1,-1\;addze. %3,%2
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "8,12")])
+  if (invert)
+    {
+      rtx tmp = gen_reg_rtx (<MODE>mode);
+      emit_insn (gen_one_cmpl<mode>2 (tmp, x));
+      x = tmp;
+    }
 
-(define_split
-  [(set (match_operand:CCEQ 0 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CCEQ (ne:P (match_operand:SI 1 "gpc_reg_operand" "")
-                           (const_int 0))
-                     (neg:P (match_operand:P 2 "gpc_reg_operand" ""))))
-   (clobber (match_scratch:P 3 ""))
-   (clobber (match_scratch:P 4 ""))]
-  "reload_completed"
-  [(parallel [(set (match_dup 3)
-                  (plus:P (ne:P (match_dup 1)
-                                (const_int 0))
-                          (match_dup 2)))
-              (clobber (match_dup 4))])
-   (set (match_dup 0)
-       (compare:CC (match_dup 3)
-                   (const_int 0)))]
-  "")
+  int sh = GET_MODE_BITSIZE (<MODE>mode) - 1;
+  emit_insn (gen_lshr<mode>3 (op0, x, GEN_INT (sh)));
 
-(define_insn "*plus_ne0_<mode>_compare"
-  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (plus:P (ne:P (match_operand:P 1 "gpc_reg_operand" "r,r")
-                      (const_int 0))
-                (match_operand:P 2 "gpc_reg_operand" "r,r"))
-        (const_int 0)))
-   (set (match_operand:P 0 "gpc_reg_operand" "=r,r")
-       (plus:P (ne:P (match_dup 1)
-                     (const_int 0))
-               (match_dup 2)))
-   (clobber (match_scratch:P 3 "=&r,&r"))]
+  DONE;
+})
+
+(define_expand "cstore<mode>4_unsigned_imm"
+  [(use (match_operator 1 "unsigned_comparison_operator"
+         [(match_operand:GPR 2 "gpc_reg_operand")
+          (match_operand:GPR 3 "immediate_operand")]))
+   (clobber (match_operand:GPR 0 "register_operand"))]
   ""
-  "@
-   addic %3,%1,-1\;addze. %0,%2
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "8,12")])
+{
+  bool invert = false;
 
-(define_split
-  [(set (match_operand:CC 4 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC
-        (plus:P (ne:P (match_operand:P 1 "gpc_reg_operand" "")
-                      (const_int 0))
-                (match_operand:P 2 "gpc_reg_operand" ""))
-        (const_int 0)))
-   (set (match_operand:P 0 "gpc_reg_operand" "")
-       (plus:P (ne:P (match_dup 1)
-                     (const_int 0))
-               (match_dup 2)))
-   (clobber (match_scratch:P 3 ""))]
-  "reload_completed"
-  [(parallel [(set (match_dup 0)
-                  (plus:P (ne:P (match_dup 1)
-                                (const_int 0))
-                          (match_dup 2)))
-             (clobber (match_dup 3))])
-   (set (match_dup 4)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
+  enum rtx_code cond_code = GET_CODE (operands[1]);
 
-(define_insn "*leu<mode>"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
-       (leu:P (match_operand:P 1 "gpc_reg_operand" "r")
-              (match_operand:P 2 "reg_or_short_operand" "rI")))]
-  ""
-  "subf%I2c %0,%1,%2\;li %0,0\;adde %0,%0,%0"
-  [(set_attr "type" "three")
-   (set_attr "length" "12")])
+  rtx op0 = operands[0];
+  rtx op1 = operands[2];
+  HOST_WIDE_INT val = INTVAL (operands[3]);
 
-(define_insn "*leu<mode>_compare"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (leu:P (match_operand:P 1 "gpc_reg_operand" "r,r")
-               (match_operand:P 2 "reg_or_short_operand" "rI,rI"))
-        (const_int 0)))
-   (set (match_operand:P 0 "gpc_reg_operand" "=r,r")
-       (leu:P (match_dup 1) (match_dup 2)))]
-  ""
-  "@
-   subf%I2c %0,%1,%2\;li %0,0\;adde. %0,%0,%0
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "12,16")])
+  if (cond_code == GEU || cond_code == GTU)
+    {
+      cond_code = reverse_condition (cond_code);
+      invert = true;
+    }
 
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_cr0_operand" "")
-       (compare:CC
-        (leu:P (match_operand:P 1 "gpc_reg_operand" "")
-               (match_operand:P 2 "reg_or_short_operand" ""))
-        (const_int 0)))
-   (set (match_operand:P 0 "gpc_reg_operand" "")
-       (leu:P (match_dup 1) (match_dup 2)))]
-  "reload_completed"
-  [(set (match_dup 0)
-       (leu:P (match_dup 1) (match_dup 2)))
-   (set (match_dup 3)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
+  if (cond_code == LEU)
+    val++;
 
-(define_insn "*plus_leu<mode>"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=&r")
-       (plus:P (leu:P (match_operand:P 1 "gpc_reg_operand" "r")
-                      (match_operand:P 2 "reg_or_short_operand" "rI"))
-               (match_operand:P 3 "gpc_reg_operand" "r")))]
-  ""
-  "subf%I2c %0,%1,%2\;addze %0,%3"
-  [(set_attr "type" "two")
-   (set_attr "length" "8")])
+  rtx tmp = gen_reg_rtx (<MODE>mode);
+  rtx tmp2 = gen_reg_rtx (<MODE>mode);
+  emit_insn (gen_add<mode>3 (tmp, op1, GEN_INT (-val)));
+  emit_insn (gen_one_cmpl<mode>2 (tmp2, op1));
+  rtx x = gen_reg_rtx (<MODE>mode);
+  if (val < 0)
+    emit_insn (gen_ior<mode>3 (x, tmp, tmp2));
+  else
+    emit_insn (gen_and<mode>3 (x, tmp, tmp2));
 
-(define_insn ""
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (plus:SI (leu:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                         (match_operand:SI 2 "reg_or_short_operand" "rI,rI"))
-                 (match_operand:SI 3 "gpc_reg_operand" "r,r"))
-        (const_int 0)))
-   (clobber (match_scratch:SI 4 "=&r,&r"))]
-  "TARGET_32BIT"
-  "@
-   subf%I2c %4,%1,%2\;addze. %4,%3
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "8,12")])
+  if (invert)
+    {
+      rtx tmp = gen_reg_rtx (<MODE>mode);
+      emit_insn (gen_one_cmpl<mode>2 (tmp, x));
+      x = tmp;
+    }
 
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_cr0_operand" "")
-       (compare:CC
-        (plus:SI (leu:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                         (match_operand:SI 2 "reg_or_short_operand" ""))
-                 (match_operand:SI 3 "gpc_reg_operand" ""))
-        (const_int 0)))
-   (clobber (match_scratch:SI 4 ""))]
-  "TARGET_32BIT && reload_completed"
-  [(set (match_dup 4)
-       (plus:SI (leu:SI (match_dup 1) (match_dup 2))
-                 (match_dup 3)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 4)
-                   (const_int 0)))]
-  "")
+  int sh = GET_MODE_BITSIZE (<MODE>mode) - 1;
+  emit_insn (gen_lshr<mode>3 (op0, x, GEN_INT (sh)));
 
-(define_insn ""
-  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (plus:SI (leu:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                         (match_operand:SI 2 "reg_or_short_operand" "rI,rI"))
-                 (match_operand:SI 3 "gpc_reg_operand" "r,r"))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=&r,&r")
-       (plus:SI (leu:SI (match_dup 1) (match_dup 2)) (match_dup 3)))]
-  "TARGET_32BIT"
-  "@
-   subf%I2c %0,%1,%2\;addze. %0,%3
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "8,12")])
+  DONE;
+})
 
-(define_split
-  [(set (match_operand:CC 4 "cc_reg_not_cr0_operand" "")
-       (compare:CC
-        (plus:SI (leu:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                         (match_operand:SI 2 "reg_or_short_operand" ""))
-                 (match_operand:SI 3 "gpc_reg_operand" ""))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (plus:SI (leu:SI (match_dup 1) (match_dup 2)) (match_dup 3)))]
-  "TARGET_32BIT && reload_completed"
-  [(set (match_dup 0)
-       (plus:SI (leu:SI (match_dup 1) (match_dup 2)) (match_dup 3)))
-   (set (match_dup 4)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
+(define_expand "cstore<mode>4"
+  [(use (match_operator 1 "rs6000_cbranch_operator"
+         [(match_operand:GPR 2 "gpc_reg_operand")
+          (match_operand:GPR 3 "reg_or_short_operand")]))
+   (clobber (match_operand:GPR 0 "register_operand"))]
+  ""
+{
+  /* Use ISEL if the user asked for it.  */
+  if (TARGET_ISEL)
+    rs6000_emit_sISEL (<MODE>mode, operands);
+
+  /* Expanding EQ and NE directly to some machine instructions does not help
+     but does hurt combine.  So don't.  */
+  else if (GET_CODE (operands[1]) == EQ)
+    emit_insn (gen_eq<mode>3 (operands[0], operands[2], operands[3]));
+  else if (<MODE>mode == Pmode
+          && GET_CODE (operands[1]) == NE)
+    emit_insn (gen_ne<mode>3 (operands[0], operands[2], operands[3]));
+  else if (GET_CODE (operands[1]) == NE)
+    {
+      rtx tmp = gen_reg_rtx (<MODE>mode);
+      emit_insn (gen_eq<mode>3 (tmp, operands[2], operands[3]));
+      emit_insn (gen_xor<mode>3 (operands[0], tmp, const1_rtx));
+    }
 
-(define_insn "*neg_leu<mode>"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
-       (neg:P (leu:P (match_operand:P 1 "gpc_reg_operand" "r")
-                     (match_operand:P 2 "reg_or_short_operand" "rI"))))]
+  /* Expanding the unsigned comparisons however helps a lot: all the neg_ltu
+     etc. combinations magically work out just right.  */
+  else if (<MODE>mode == Pmode
+          && unsigned_comparison_operator (operands[1], VOIDmode))
+    emit_insn (gen_cstore<mode>4_unsigned (operands[0], operands[1],
+                                          operands[2], operands[3]));
+
+  /* For signed comparisons against a constant, we can do some simple
+     bit-twiddling.  */
+  else if (signed_comparison_operator (operands[1], VOIDmode)
+          && CONST_INT_P (operands[3]))
+    emit_insn (gen_cstore<mode>4_signed_imm (operands[0], operands[1],
+                                            operands[2], operands[3]));
+
+  /* And similarly for unsigned comparisons.  */
+  else if (unsigned_comparison_operator (operands[1], VOIDmode)
+          && CONST_INT_P (operands[3]))
+    emit_insn (gen_cstore<mode>4_unsigned_imm (operands[0], operands[1],
+                                              operands[2], operands[3]));
+
+  /* Everything else, use the mfcr brute force.  */
+  else
+    rs6000_emit_sCOND (<MODE>mode, operands);
+
+  DONE;
+})
+
+(define_expand "cstore<mode>4"
+  [(use (match_operator 1 "rs6000_cbranch_operator"
+         [(match_operand:FP 2 "gpc_reg_operand" "")
+          (match_operand:FP 3 "gpc_reg_operand" "")]))
+   (clobber (match_operand:SI 0 "register_operand"))]
   ""
-  "subf%I2c %0,%1,%2\;subfe %0,%0,%0\;nand %0,%0,%0"
-   [(set_attr "type" "three")
-    (set_attr "length" "12")])
+{
+  rs6000_emit_sCOND (<MODE>mode, operands);
+  DONE;
+})
+
 
-(define_insn "*and_neg_leu<mode>"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=&r")
-       (and:P (neg:P
-                (leu:P (match_operand:P 1 "gpc_reg_operand" "r")
-                       (match_operand:P 2 "reg_or_short_operand" "rI")))
-               (match_operand:P 3 "gpc_reg_operand" "r")))]
+(define_expand "stack_protect_set"
+  [(match_operand 0 "memory_operand" "")
+   (match_operand 1 "memory_operand" "")]
   ""
-  "subf%I2c %0,%1,%2\;subfe %0,%0,%0\;andc %0,%3,%0"
+{
+#ifdef TARGET_THREAD_SSP_OFFSET
+  rtx tlsreg = gen_rtx_REG (Pmode, TARGET_64BIT ? 13 : 2);
+  rtx addr = gen_rtx_PLUS (Pmode, tlsreg, GEN_INT (TARGET_THREAD_SSP_OFFSET));
+  operands[1] = gen_rtx_MEM (Pmode, addr);
+#endif
+  if (TARGET_64BIT)
+    emit_insn (gen_stack_protect_setdi (operands[0], operands[1]));
+  else
+    emit_insn (gen_stack_protect_setsi (operands[0], operands[1]));
+  DONE;
+})
+
+(define_insn "stack_protect_setsi"
+  [(set (match_operand:SI 0 "memory_operand" "=m")
+       (unspec:SI [(match_operand:SI 1 "memory_operand" "m")] UNSPEC_SP_SET))
+   (set (match_scratch:SI 2 "=&r") (const_int 0))]
+  "TARGET_32BIT"
+  "lwz%U1%X1 %2,%1\;stw%U0%X0 %2,%0\;li %2,0"
   [(set_attr "type" "three")
    (set_attr "length" "12")])
 
-(define_insn ""
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (and:SI (neg:SI
-                 (leu:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                         (match_operand:SI 2 "reg_or_short_operand" "rI,rI")))
-                (match_operand:SI 3 "gpc_reg_operand" "r,r"))
-        (const_int 0)))
-   (clobber (match_scratch:SI 4 "=&r,&r"))]
-  "TARGET_32BIT"
-  "@
-   subf%I2c %4,%1,%2\;subfe %4,%4,%4\;andc. %4,%3,%4
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "12,16")])
+(define_insn "stack_protect_setdi"
+  [(set (match_operand:DI 0 "memory_operand" "=Y")
+       (unspec:DI [(match_operand:DI 1 "memory_operand" "Y")] UNSPEC_SP_SET))
+   (set (match_scratch:DI 2 "=&r") (const_int 0))]
+  "TARGET_64BIT"
+  "ld%U1%X1 %2,%1\;std%U0%X0 %2,%0\;li %2,0"
+  [(set_attr "type" "three")
+   (set_attr "length" "12")])
 
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_cr0_operand" "")
-       (compare:CC
-        (and:SI (neg:SI
-                 (leu:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                         (match_operand:SI 2 "reg_or_short_operand" "")))
-                (match_operand:SI 3 "gpc_reg_operand" ""))
-        (const_int 0)))
-   (clobber (match_scratch:SI 4 ""))]
-  "TARGET_32BIT && reload_completed"
-  [(set (match_dup 4)
-       (and:SI (neg:SI (leu:SI (match_dup 1) (match_dup 2)))
-               (match_dup 3)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 4)
-                   (const_int 0)))]
-  "")
+(define_expand "stack_protect_test"
+  [(match_operand 0 "memory_operand" "")
+   (match_operand 1 "memory_operand" "")
+   (match_operand 2 "" "")]
+  ""
+{
+  rtx test, op0, op1;
+#ifdef TARGET_THREAD_SSP_OFFSET
+  rtx tlsreg = gen_rtx_REG (Pmode, TARGET_64BIT ? 13 : 2);
+  rtx addr = gen_rtx_PLUS (Pmode, tlsreg, GEN_INT (TARGET_THREAD_SSP_OFFSET));
+  operands[1] = gen_rtx_MEM (Pmode, addr);
+#endif
+  op0 = operands[0];
+  op1 = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, operands[1]), UNSPEC_SP_TEST);
+  test = gen_rtx_EQ (VOIDmode, op0, op1);
+  emit_jump_insn (gen_cbranchsi4 (test, op0, op1, operands[2]));
+  DONE;
+})
 
-(define_insn ""
-  [(set (match_operand:CC 4 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (and:SI (neg:SI
-                 (leu:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                         (match_operand:SI 2 "reg_or_short_operand" "rI,rI")))
-                (match_operand:SI 3 "gpc_reg_operand" "r,r"))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=&r,&r")
-       (and:SI (neg:SI (leu:SI (match_dup 1) (match_dup 2))) (match_dup 3)))]
+(define_insn "stack_protect_testsi"
+  [(set (match_operand:CCEQ 0 "cc_reg_operand" "=x,?y")
+        (unspec:CCEQ [(match_operand:SI 1 "memory_operand" "m,m")
+                     (match_operand:SI 2 "memory_operand" "m,m")]
+                    UNSPEC_SP_TEST))
+   (set (match_scratch:SI 4 "=r,r") (const_int 0))
+   (clobber (match_scratch:SI 3 "=&r,&r"))]
   "TARGET_32BIT"
   "@
-   subf%I2c %0,%1,%2\;subfe %0,%0,%0\;andc. %0,%3,%0
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "12,16")])
-
-(define_split
-  [(set (match_operand:CC 4 "cc_reg_not_cr0_operand" "")
-       (compare:CC
-        (and:SI (neg:SI
-                 (leu:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                         (match_operand:SI 2 "reg_or_short_operand" "")))
-                (match_operand:SI 3 "gpc_reg_operand" ""))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (and:SI (neg:SI (leu:SI (match_dup 1) (match_dup 2))) (match_dup 3)))]
-  "TARGET_32BIT && reload_completed"
-  [(set (match_dup 0)
-       (and:SI (neg:SI (leu:SI (match_dup 1) (match_dup 2)))
-               (match_dup 3)))
-   (set (match_dup 4)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
+   lwz%U1%X1 %3,%1\;lwz%U2%X2 %4,%2\;xor. %3,%3,%4\;li %4,0
+   lwz%U1%X1 %3,%1\;lwz%U2%X2 %4,%2\;cmplw %0,%3,%4\;li %3,0\;li %4,0"
+  [(set_attr "length" "16,20")])
 
-(define_insn_and_split "*ltu<mode>"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=r,r")
-       (ltu:P (match_operand:P 1 "gpc_reg_operand" "r,r")
-              (match_operand:P 2 "reg_or_neg_short_operand" "r,P")))]
-  ""
-  "#"
-  ""
-  [(set (match_dup 0) (neg:P (ltu:P (match_dup 1) (match_dup 2))))
-   (set (match_dup 0) (neg:P (match_dup 0)))]
-  "")
+(define_insn "stack_protect_testdi"
+  [(set (match_operand:CCEQ 0 "cc_reg_operand" "=x,?y")
+        (unspec:CCEQ [(match_operand:DI 1 "memory_operand" "Y,Y")
+                     (match_operand:DI 2 "memory_operand" "Y,Y")]
+                    UNSPEC_SP_TEST))
+   (set (match_scratch:DI 4 "=r,r") (const_int 0))
+   (clobber (match_scratch:DI 3 "=&r,&r"))]
+  "TARGET_64BIT"
+  "@
+   ld%U1%X1 %3,%1\;ld%U2%X2 %4,%2\;xor. %3,%3,%4\;li %4,0
+   ld%U1%X1 %3,%1\;ld%U2%X2 %4,%2\;cmpld %0,%3,%4\;li %3,0\;li %4,0"
+  [(set_attr "length" "16,20")])
 
-(define_insn_and_split "*ltu<mode>_compare"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC
-        (ltu:P (match_operand:P 1 "gpc_reg_operand" "r,r,r,r")
-               (match_operand:P 2 "reg_or_neg_short_operand" "r,P,r,P"))
-        (const_int 0)))
-   (set (match_operand:P 0 "gpc_reg_operand" "=r,r,r,r")
-       (ltu:P (match_dup 1) (match_dup 2)))]
-  ""
-  "#"
+\f
+;; Here are the actual compare insns.
+(define_insn "*cmp<mode>_internal1"
+  [(set (match_operand:CC 0 "cc_reg_operand" "=y")
+       (compare:CC (match_operand:GPR 1 "gpc_reg_operand" "r")
+                   (match_operand:GPR 2 "reg_or_short_operand" "rI")))]
   ""
-  [(set (match_dup 0) (neg:P (ltu:P (match_dup 1) (match_dup 2))))
-   (parallel [(set (match_dup 3)
-                  (compare:CC (neg:P (match_dup 0)) (const_int 0)))
-             (set (match_dup 0) (neg:P (match_dup 0)))])]
-  "")
+  "cmp<wd>%I2 %0,%1,%2"
+  [(set_attr "type" "cmp")])
 
-(define_insn_and_split "*plus_ltu<mode>"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=&r,r")
-       (plus:P (ltu:P (match_operand:P 1 "gpc_reg_operand" "r,r")
-                      (match_operand:P 2 "reg_or_neg_short_operand" "r,P"))
-               (match_operand:P 3 "reg_or_short_operand" "rI,rI")))]
-  ""
-  "#"
-  "&& !reg_overlap_mentioned_p (operands[0], operands[3])"
-  [(set (match_dup 0) (neg:P (ltu:P (match_dup 1) (match_dup 2))))
-   (set (match_dup 0) (minus:P (match_dup 3) (match_dup 0)))]
-  "")
+;; If we are comparing a register for equality with a large constant,
+;; we can do this with an XOR followed by a compare.  But this is profitable
+;; only if the large constant is only used for the comparison (and in this
+;; case we already have a register to reuse as scratch).
+;;
+;; For 64-bit registers, we could only do so if the constant's bit 15 is clear:
+;; otherwise we'd need to XOR with FFFFFFFF????0000 which is not available.
 
-(define_insn_and_split "*plus_ltu<mode>_compare"
-  [(set (match_operand:CC 4 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC
-        (plus:P (ltu:P (match_operand:P 1 "gpc_reg_operand" "r,r,r,r")
-                       (match_operand:P 2 "reg_or_neg_short_operand" "r,P,r,P"))
-                (match_operand:P 3 "gpc_reg_operand" "r,r,r,r"))
-        (const_int 0)))
-   (set (match_operand:P 0 "gpc_reg_operand" "=&r,&r,&r,&r")
-       (plus:P (ltu:P (match_dup 1) (match_dup 2)) (match_dup 3)))]
-  ""
-  "#"
-  "&& !reg_overlap_mentioned_p (operands[0], operands[3])"
-  [(set (match_dup 0) (neg:P (ltu:P (match_dup 1) (match_dup 2))))
-   (parallel [(set (match_dup 4)
-                  (compare:CC (minus:P (match_dup 3) (match_dup 0))
-                              (const_int 0)))
-             (set (match_dup 0) (minus:P (match_dup 3) (match_dup 0)))])]
-  "")
+(define_peephole2
+  [(set (match_operand:SI 0 "register_operand")
+        (match_operand:SI 1 "logical_const_operand" ""))
+   (set (match_dup 0) (match_operator:SI 3 "boolean_or_operator"
+                      [(match_dup 0)
+                       (match_operand:SI 2 "logical_const_operand" "")]))
+   (set (match_operand:CC 4 "cc_reg_operand" "")
+        (compare:CC (match_operand:SI 5 "gpc_reg_operand" "")
+                    (match_dup 0)))
+   (set (pc)
+        (if_then_else (match_operator 6 "equality_operator"
+                       [(match_dup 4) (const_int 0)])
+                      (match_operand 7 "" "")
+                      (match_operand 8 "" "")))]
+  "peep2_reg_dead_p (3, operands[0])
+   && peep2_reg_dead_p (4, operands[4])
+   && REGNO (operands[0]) != REGNO (operands[5])"
+ [(set (match_dup 0) (xor:SI (match_dup 5) (match_dup 9)))
+  (set (match_dup 4) (compare:CC (match_dup 0) (match_dup 10)))
+  (set (pc) (if_then_else (match_dup 6) (match_dup 7) (match_dup 8)))]
+{
+  /* Get the constant we are comparing against, and see what it looks like
+     when sign-extended from 16 to 32 bits.  Then see what constant we could
+     XOR with SEXTC to get the sign-extended value.  */
+  rtx cnst = simplify_const_binary_operation (GET_CODE (operands[3]),
+                                             SImode,
+                                             operands[1], operands[2]);
+  HOST_WIDE_INT c = INTVAL (cnst);
+  HOST_WIDE_INT sextc = ((c & 0xffff) ^ 0x8000) - 0x8000;
+  HOST_WIDE_INT xorv = c ^ sextc;
 
-(define_insn "*neg_ltu<mode>"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=r,r")
-       (neg:P (ltu:P (match_operand:P 1 "gpc_reg_operand" "r,r")
-                     (match_operand:P 2 "reg_or_neg_short_operand" "r,P"))))]
-  ""
-  "@
-   subfc %0,%2,%1\;subfe %0,%0,%0
-   addic %0,%1,%n2\;subfe %0,%0,%0"
-  [(set_attr "type" "two")
-   (set_attr "length" "8")])
+  operands[9] = GEN_INT (xorv);
+  operands[10] = GEN_INT (sextc);
+})
 
-(define_insn "*geu<mode>"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=r,r")
-       (geu:P (match_operand:P 1 "gpc_reg_operand" "r,r")
-              (match_operand:P 2 "reg_or_neg_short_operand" "r,P")))]
+(define_insn "*cmpsi_internal2"
+  [(set (match_operand:CCUNS 0 "cc_reg_operand" "=y")
+       (compare:CCUNS (match_operand:SI 1 "gpc_reg_operand" "r")
+                      (match_operand:SI 2 "reg_or_u_short_operand" "rK")))]
   ""
-  "@
-   subfc %0,%2,%1\;li %0,0\;adde %0,%0,%0
-   addic %0,%1,%n2\;li %0,0\;adde %0,%0,%0"
-  [(set_attr "type" "three")
-   (set_attr "length" "12")])
+  "cmplw%I2 %0,%1,%b2"
+  [(set_attr "type" "cmp")])
 
-(define_insn "*geu<mode>_compare"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC
-        (geu:P (match_operand:P 1 "gpc_reg_operand" "r,r,r,r")
-               (match_operand:P 2 "reg_or_neg_short_operand" "r,P,r,P"))
-        (const_int 0)))
-   (set (match_operand:P 0 "gpc_reg_operand" "=r,r,r,r")
-       (geu:P (match_dup 1) (match_dup 2)))]
+(define_insn "*cmpdi_internal2"
+  [(set (match_operand:CCUNS 0 "cc_reg_operand" "=y")
+       (compare:CCUNS (match_operand:DI 1 "gpc_reg_operand" "r")
+                      (match_operand:DI 2 "reg_or_u_short_operand" "rK")))]
   ""
-  "@
-   subfc %0,%2,%1\;li %0,0\;adde. %0,%0,%0
-   addic %0,%1,%n2\;li %0,0\;adde. %0,%0,%0
-   #
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "12,12,16,16")])
-
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC
-        (geu:P (match_operand:P 1 "gpc_reg_operand" "")
-               (match_operand:P 2 "reg_or_neg_short_operand" ""))
-        (const_int 0)))
-   (set (match_operand:P 0 "gpc_reg_operand" "")
-       (geu:P (match_dup 1) (match_dup 2)))]
-  "reload_completed"
-  [(set (match_dup 0)
-       (geu:P (match_dup 1) (match_dup 2)))
-   (set (match_dup 3)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
+  "cmpld%I2 %0,%1,%b2"
+  [(set_attr "type" "cmp")])
 
-(define_insn "*plus_geu<mode>"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=&r,&r")
-       (plus:P (geu:P (match_operand:P 1 "gpc_reg_operand" "r,r")
-                      (match_operand:P 2 "reg_or_neg_short_operand" "r,P"))
-               (match_operand:P 3 "gpc_reg_operand" "r,r")))]
-  ""
-  "@
-   subfc %0,%2,%1\;addze %0,%3
-   addic %0,%1,%n2\;addze %0,%3"
-  [(set_attr "type" "two")
-   (set_attr "length" "8")])
+;; The following two insns don't exist as single insns, but if we provide
+;; them, we can swap an add and compare, which will enable us to overlap more
+;; of the required delay between a compare and branch.  We generate code for
+;; them by splitting.
 
 (define_insn ""
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC
-        (plus:SI (geu:SI (match_operand:SI 1 "gpc_reg_operand" "r,r,r,r")
-                         (match_operand:SI 2 "reg_or_neg_short_operand" "r,P,r,P"))
-                 (match_operand:SI 3 "gpc_reg_operand" "r,r,r,r"))
-        (const_int 0)))
-   (clobber (match_scratch:SI 4 "=&r,&r,&r,&r"))]
-  "TARGET_32BIT"
-  "@
-   subfc %4,%2,%1\;addze. %4,%3
-   addic %4,%1,%n2\;addze. %4,%3
-   #
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "8,8,12,12")])
-
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_cr0_operand" "")
-       (compare:CC
-        (plus:SI (geu:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                         (match_operand:SI 2 "reg_or_neg_short_operand" ""))
-                 (match_operand:SI 3 "gpc_reg_operand" ""))
-        (const_int 0)))
-   (clobber (match_scratch:SI 4 ""))]
-  "TARGET_32BIT && reload_completed"
-  [(set (match_dup 4)
-       (plus:SI (geu:SI (match_dup 1) (match_dup 2))
-                 (match_dup 3)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 4)
-                   (const_int 0)))]
-  "")
+  [(set (match_operand:CC 3 "cc_reg_operand" "=y")
+       (compare:CC (match_operand:SI 1 "gpc_reg_operand" "r")
+                   (match_operand:SI 2 "short_cint_operand" "i")))
+   (set (match_operand:SI 0 "gpc_reg_operand" "=r")
+       (plus:SI (match_dup 1) (match_operand:SI 4 "short_cint_operand" "i")))]
+  ""
+  "#"
+  [(set_attr "length" "8")])
 
 (define_insn ""
-  [(set (match_operand:CC 4 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC
-        (plus:SI (geu:SI (match_operand:SI 1 "gpc_reg_operand" "r,r,r,r")
-                         (match_operand:SI 2 "reg_or_neg_short_operand" "r,P,r,P"))
-                 (match_operand:SI 3 "gpc_reg_operand" "r,r,r,r"))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=&r,&r,&r,&r")
-       (plus:SI (geu:SI (match_dup 1) (match_dup 2)) (match_dup 3)))]
-  "TARGET_32BIT"
-  "@
-   subfc %0,%2,%1\;addze. %0,%3
-   addic %0,%1,%n2\;addze. %0,%3
-   #
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "8,8,12,12")])
+  [(set (match_operand:CCUNS 3 "cc_reg_operand" "=y")
+       (compare:CCUNS (match_operand:SI 1 "gpc_reg_operand" "r")
+                      (match_operand:SI 2 "u_short_cint_operand" "i")))
+   (set (match_operand:SI 0 "gpc_reg_operand" "=r")
+       (plus:SI (match_dup 1) (match_operand:SI 4 "short_cint_operand" "i")))]
+  ""
+  "#"
+  [(set_attr "length" "8")])
 
 (define_split
-  [(set (match_operand:CC 4 "cc_reg_not_cr0_operand" "")
-       (compare:CC
-        (plus:SI (geu:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                         (match_operand:SI 2 "reg_or_neg_short_operand" ""))
-                 (match_operand:SI 3 "gpc_reg_operand" ""))
-        (const_int 0)))
+  [(set (match_operand:CC 3 "cc_reg_operand" "")
+       (compare:CC (match_operand:SI 1 "gpc_reg_operand" "")
+                   (match_operand:SI 2 "short_cint_operand" "")))
    (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (plus:SI (geu:SI (match_dup 1) (match_dup 2)) (match_dup 3)))]
-  "TARGET_32BIT && reload_completed"
-  [(set (match_dup 0)
-       (plus:SI (geu:SI (match_dup 1) (match_dup 2)) (match_dup 3)))
-   (set (match_dup 4)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
-
-(define_insn "*neg_geu<mode>"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=r,r")
-       (neg:P (geu:P (match_operand:P 1 "gpc_reg_operand" "r,r")
-                     (match_operand:P 2 "reg_or_short_operand" "r,I"))))]
+       (plus:SI (match_dup 1) (match_operand:SI 4 "short_cint_operand" "")))]
   ""
-  "@
-   subfc %0,%2,%1\;subfe %0,%0,%0\;nand %0,%0,%0
-   subfic %0,%1,-1\;add%I2c %0,%0,%2\;subfe %0,%0,%0"
-  [(set_attr "type" "three")
-   (set_attr "length" "12")])
+  [(set (match_dup 3) (compare:CC (match_dup 1) (match_dup 2)))
+   (set (match_dup 0) (plus:SI (match_dup 1) (match_dup 4)))])
 
-(define_insn "*and_neg_geu<mode>"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=&r,&r")
-       (and:P (neg:P
-                (geu:P (match_operand:P 1 "gpc_reg_operand" "r,r")
-                       (match_operand:P 2 "reg_or_neg_short_operand" "r,P")))
-               (match_operand:P 3 "gpc_reg_operand" "r,r")))]
+(define_split
+  [(set (match_operand:CCUNS 3 "cc_reg_operand" "")
+       (compare:CCUNS (match_operand:SI 1 "gpc_reg_operand" "")
+                      (match_operand:SI 2 "u_short_cint_operand" "")))
+   (set (match_operand:SI 0 "gpc_reg_operand" "")
+       (plus:SI (match_dup 1) (match_operand:SI 4 "short_cint_operand" "")))]
   ""
-  "@
-   subfc %0,%2,%1\;subfe %0,%0,%0\;andc %0,%3,%0
-   addic %0,%1,%n2\;subfe %0,%0,%0\;andc %0,%3,%0"
-  [(set_attr "type" "three")
-   (set_attr "length" "12")])
+  [(set (match_dup 3) (compare:CCUNS (match_dup 1) (match_dup 2)))
+   (set (match_dup 0) (plus:SI (match_dup 1) (match_dup 4)))])
 
-(define_insn ""
-  [(set (match_operand:CC 0 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC
-        (and:SI (neg:SI
-                 (geu:SI (match_operand:SI 1 "gpc_reg_operand" "r,r,r,r")
-                         (match_operand:SI 2 "reg_or_neg_short_operand" "r,P,r,P")))
-                (match_operand:SI 3 "gpc_reg_operand" "r,r,r,r"))
-        (const_int 0)))
-   (clobber (match_scratch:SI 4 "=&r,&r,&r,&r"))]
-  "TARGET_32BIT"
-  "@
-   subfc %4,%2,%1\;subfe %4,%4,%4\;andc. %4,%3,%4
-   addic %4,%1,%n2\;subfe %4,%4,%4\;andc. %4,%3,%4
-   #
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "12,12,16,16")])
+;; Only need to compare second words if first words equal
+(define_insn "*cmptf_internal1"
+  [(set (match_operand:CCFP 0 "cc_reg_operand" "=y")
+       (compare:CCFP (match_operand:TF 1 "gpc_reg_operand" "d")
+                     (match_operand:TF 2 "gpc_reg_operand" "d")))]
+  "!TARGET_IEEEQUAD && !TARGET_XL_COMPAT
+   && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT && TARGET_LONG_DOUBLE_128"
+  "fcmpu %0,%1,%2\;bne %0,$+8\;fcmpu %0,%L1,%L2"
+  [(set_attr "type" "fpcompare")
+   (set_attr "length" "12")])
 
-(define_split
-  [(set (match_operand:CC 0 "cc_reg_not_cr0_operand" "")
-       (compare:CC
-        (and:SI (neg:SI
-                 (geu:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                         (match_operand:SI 2 "reg_or_neg_short_operand" "")))
-                (match_operand:SI 3 "gpc_reg_operand" ""))
-        (const_int 0)))
-   (clobber (match_scratch:SI 4 ""))]
-  "TARGET_32BIT && reload_completed"
-  [(set (match_dup 4)
-       (and:SI (neg:SI (geu:SI (match_dup 1) (match_dup 2)))
-               (match_dup 3)))
-   (set (match_dup 0)
-       (compare:CC (match_dup 4)
-                   (const_int 0)))]
-  "")
+(define_insn_and_split "*cmptf_internal2"
+  [(set (match_operand:CCFP 0 "cc_reg_operand" "=y")
+       (compare:CCFP (match_operand:TF 1 "gpc_reg_operand" "d")
+                     (match_operand:TF 2 "gpc_reg_operand" "d")))
+    (clobber (match_scratch:DF 3 "=d"))
+    (clobber (match_scratch:DF 4 "=d"))
+    (clobber (match_scratch:DF 5 "=d"))
+    (clobber (match_scratch:DF 6 "=d"))
+    (clobber (match_scratch:DF 7 "=d"))
+    (clobber (match_scratch:DF 8 "=d"))
+    (clobber (match_scratch:DF 9 "=d"))
+    (clobber (match_scratch:DF 10 "=d"))
+    (clobber (match_scratch:GPR 11 "=b"))]
+  "!TARGET_IEEEQUAD && TARGET_XL_COMPAT
+   && TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT && TARGET_LONG_DOUBLE_128"
+  "#"
+  "&& reload_completed"
+  [(set (match_dup 3) (match_dup 14))
+   (set (match_dup 4) (match_dup 15))
+   (set (match_dup 9) (abs:DF (match_dup 5)))
+   (set (match_dup 0) (compare:CCFP (match_dup 9) (match_dup 3)))
+   (set (pc) (if_then_else (ne (match_dup 0) (const_int 0))
+                          (label_ref (match_dup 12))
+                          (pc)))
+   (set (match_dup 0) (compare:CCFP (match_dup 5) (match_dup 7)))
+   (set (pc) (label_ref (match_dup 13)))
+   (match_dup 12)
+   (set (match_dup 10) (minus:DF (match_dup 5) (match_dup 7)))
+   (set (match_dup 9) (minus:DF (match_dup 6) (match_dup 8)))
+   (set (match_dup 9) (plus:DF (match_dup 10) (match_dup 9)))
+   (set (match_dup 0) (compare:CCFP (match_dup 9) (match_dup 4)))
+   (match_dup 13)]
+{
+  REAL_VALUE_TYPE rv;
+  const int lo_word = LONG_DOUBLE_LARGE_FIRST ? GET_MODE_SIZE (DFmode) : 0;
+  const int hi_word = LONG_DOUBLE_LARGE_FIRST ? 0 : GET_MODE_SIZE (DFmode);
 
+  operands[5] = simplify_gen_subreg (DFmode, operands[1], TFmode, hi_word);
+  operands[6] = simplify_gen_subreg (DFmode, operands[1], TFmode, lo_word);
+  operands[7] = simplify_gen_subreg (DFmode, operands[2], TFmode, hi_word);
+  operands[8] = simplify_gen_subreg (DFmode, operands[2], TFmode, lo_word);
+  operands[12] = gen_label_rtx ();
+  operands[13] = gen_label_rtx ();
+  real_inf (&rv);
+  operands[14] = force_const_mem (DFmode,
+                                 CONST_DOUBLE_FROM_REAL_VALUE (rv, DFmode));
+  operands[15] = force_const_mem (DFmode,
+                                 CONST_DOUBLE_FROM_REAL_VALUE (dconst0,
+                                                               DFmode));
+  if (TARGET_TOC)
+    {
+      rtx tocref;
+      tocref = create_TOC_reference (XEXP (operands[14], 0), operands[11]);
+      operands[14] = gen_const_mem (DFmode, tocref);
+      tocref = create_TOC_reference (XEXP (operands[15], 0), operands[11]);
+      operands[15] = gen_const_mem (DFmode, tocref);
+      set_mem_alias_set (operands[14], get_TOC_alias_set ());
+      set_mem_alias_set (operands[15], get_TOC_alias_set ());
+    }
+})
+\f
+;; Now we have the scc insns.  We can do some combinations because of the
+;; way the machine works.
+;;
+;; Note that this is probably faster if we can put an insn between the
+;; mfcr and rlinm, but this is tricky.  Let's leave it for now.  In most
+;; cases the insns below which don't use an intermediate CR field will
+;; be used instead.
 (define_insn ""
-  [(set (match_operand:CC 4 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC
-        (and:SI (neg:SI
-                 (geu:SI (match_operand:SI 1 "gpc_reg_operand" "r,r,r,r")
-                         (match_operand:SI 2 "reg_or_neg_short_operand" "r,P,r,P")))
-                (match_operand:SI 3 "gpc_reg_operand" "r,r,r,r"))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=&r,&r,&r,&r")
-       (and:SI (neg:SI (geu:SI (match_dup 1) (match_dup 2))) (match_dup 3)))]
-  "TARGET_32BIT"
-  "@
-   subfc %0,%2,%1\;subfe %0,%0,%0\;andc. %0,%3,%0
-   addic %0,%1,%n2\;subfe %0,%0,%0\;andc. %0,%3,%0
-   #
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "12,12,16,16")])
+  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
+       (match_operator:SI 1 "scc_comparison_operator"
+                          [(match_operand 2 "cc_reg_operand" "y")
+                           (const_int 0)]))]
+  ""
+  "mfcr %0%Q2\;rlwinm %0,%0,%J1,1"
+  [(set (attr "type")
+     (cond [(match_test "TARGET_MFCRF")
+               (const_string "mfcrf")
+          ]
+       (const_string "mfcr")))
+   (set_attr "length" "8")])
 
-(define_split
-  [(set (match_operand:CC 4 "cc_reg_not_cr0_operand" "")
-       (compare:CC
-        (and:SI (neg:SI
-                 (geu:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                         (match_operand:SI 2 "reg_or_neg_short_operand" "")))
-                (match_operand:SI 3 "gpc_reg_operand" ""))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (and:SI (neg:SI (geu:SI (match_dup 1) (match_dup 2))) (match_dup 3)))]
-  "TARGET_32BIT && reload_completed"
-  [(set (match_dup 0)
-       (and:SI (neg:SI (geu:SI (match_dup 1) (match_dup 2))) (match_dup 3)))
-   (set (match_dup 4)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
+;; Same as above, but get the GT bit.
+(define_insn "move_from_CR_gt_bit"
+  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
+       (unspec:SI [(match_operand 1 "cc_reg_operand" "y")] UNSPEC_MV_CR_GT))]
+  "TARGET_HARD_FLOAT && !TARGET_FPRS"
+  "mfcr %0\;rlwinm %0,%0,%D1,31,31"
+  [(set_attr "type" "mfcr")
+   (set_attr "length" "8")])
 
-(define_insn "*plus_gt0<mode>"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=&r")
-       (plus:P (gt:P (match_operand:P 1 "gpc_reg_operand" "r")
-                     (const_int 0))
-                (match_operand:P 2 "gpc_reg_operand" "r")))]
-  ""
-  "addc %0,%1,%1\;subfe %0,%1,%0\;addze %0,%2"
-  [(set_attr "type" "three")
-   (set_attr "length" "12")])
+;; Same as above, but get the OV/ORDERED bit.
+(define_insn "move_from_CR_ov_bit"
+  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
+       (unspec:SI [(match_operand:CC 1 "cc_reg_operand" "y")]
+                  UNSPEC_MV_CR_OV))]
+  "TARGET_ISEL"
+  "mfcr %0\;rlwinm %0,%0,%t1,1"
+  [(set_attr "type" "mfcr")
+   (set_attr "length" "8")])
+
+(define_insn ""
+  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
+       (match_operator:DI 1 "scc_comparison_operator"
+                          [(match_operand 2 "cc_reg_operand" "y")
+                           (const_int 0)]))]
+  "TARGET_POWERPC64"
+  "mfcr %0%Q2\;rlwinm %0,%0,%J1,1"
+  [(set (attr "type")
+     (cond [(match_test "TARGET_MFCRF")
+               (const_string "mfcrf")
+          ]
+       (const_string "mfcr")))
+   (set_attr "length" "8")])
 
 (define_insn ""
   [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (plus:SI (gt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                        (const_int 0))
-                 (match_operand:SI 2 "gpc_reg_operand" "r,r"))
-        (const_int 0)))
-   (clobber (match_scratch:SI 3 "=&r,&r"))]
+       (compare:CC (match_operator:SI 1 "scc_comparison_operator"
+                                      [(match_operand 2 "cc_reg_operand" "y,y")
+                                       (const_int 0)])
+                   (const_int 0)))
+   (set (match_operand:SI 3 "gpc_reg_operand" "=r,r")
+       (match_op_dup 1 [(match_dup 2) (const_int 0)]))]
   "TARGET_32BIT"
   "@
-   addc %3,%1,%1\;subfe %3,%1,%3\;addze. %3,%2
+   mfcr %3%Q2\;rlwinm. %3,%3,%J1,1
    #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "12,16")])
+  [(set_attr "type" "shift")
+   (set_attr "dot" "yes")
+   (set_attr "length" "8,16")])
 
 (define_split
   [(set (match_operand:CC 0 "cc_reg_not_cr0_operand" "")
-       (compare:CC
-        (plus:SI (gt:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                        (const_int 0))
-                 (match_operand:SI 2 "gpc_reg_operand" ""))
-        (const_int 0)))
-   (clobber (match_scratch:SI 3 ""))]
+       (compare:CC (match_operator:SI 1 "scc_comparison_operator"
+                                      [(match_operand 2 "cc_reg_operand" "")
+                                       (const_int 0)])
+                   (const_int 0)))
+   (set (match_operand:SI 3 "gpc_reg_operand" "")
+       (match_op_dup 1 [(match_dup 2) (const_int 0)]))]
   "TARGET_32BIT && reload_completed"
   [(set (match_dup 3)
-       (plus:SI (gt:SI (match_dup 1) (const_int 0))
-                 (match_dup 2)))
+       (match_op_dup 1 [(match_dup 2) (const_int 0)]))
    (set (match_dup 0)
        (compare:CC (match_dup 3)
                    (const_int 0)))]
   "")
 
 (define_insn ""
+  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
+       (ashift:SI (match_operator:SI 1 "scc_comparison_operator"
+                                     [(match_operand 2 "cc_reg_operand" "y")
+                                      (const_int 0)])
+                  (match_operand:SI 3 "const_int_operand" "n")))]
+  ""
+  "*
+{
+  int is_bit = ccr_bit (operands[1], 1);
+  int put_bit = 31 - (INTVAL (operands[3]) & 31);
+  int count;
+
+  if (is_bit >= put_bit)
+    count = is_bit - put_bit;
+  else
+    count = 32 - (put_bit - is_bit);
+
+  operands[4] = GEN_INT (count);
+  operands[5] = GEN_INT (put_bit);
+
+  return \"mfcr %0%Q2\;rlwinm %0,%0,%4,%5,%5\";
+}"
+  [(set (attr "type")
+     (cond [(match_test "TARGET_MFCRF")
+               (const_string "mfcrf")
+          ]
+       (const_string "mfcr")))
+   (set_attr "length" "8")])
+
+(define_insn ""
   [(set (match_operand:CC 0 "cc_reg_operand" "=x,?y")
        (compare:CC
-        (plus:DI (gt:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
-                        (const_int 0))
-                 (match_operand:DI 2 "gpc_reg_operand" "r,r"))
+        (ashift:SI (match_operator:SI 1 "scc_comparison_operator"
+                                      [(match_operand 2 "cc_reg_operand" "y,y")
+                                       (const_int 0)])
+                   (match_operand:SI 3 "const_int_operand" "n,n"))
         (const_int 0)))
-   (clobber (match_scratch:DI 3 "=&r,&r"))]
-  "TARGET_64BIT"
-  "@
-   addc %3,%1,%1\;subfe %3,%1,%3\;addze. %3,%2
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "12,16")])
+   (set (match_operand:SI 4 "gpc_reg_operand" "=r,r")
+       (ashift:SI (match_op_dup 1 [(match_dup 2) (const_int 0)])
+                  (match_dup 3)))]
+  ""
+  "*
+{
+  int is_bit = ccr_bit (operands[1], 1);
+  int put_bit = 31 - (INTVAL (operands[3]) & 31);
+  int count;
+
+  /* Force split for non-cc0 compare.  */
+  if (which_alternative == 1)
+     return \"#\";
+
+  if (is_bit >= put_bit)
+    count = is_bit - put_bit;
+  else
+    count = 32 - (put_bit - is_bit);
+
+  operands[5] = GEN_INT (count);
+  operands[6] = GEN_INT (put_bit);
+
+  return \"mfcr %4%Q2\;rlwinm. %4,%4,%5,%6,%6\";
+}"
+  [(set_attr "type" "shift")
+   (set_attr "dot" "yes")
+   (set_attr "length" "8,16")])
 
 (define_split
   [(set (match_operand:CC 0 "cc_reg_not_micro_cr0_operand" "")
        (compare:CC
-        (plus:DI (gt:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                        (const_int 0))
-                 (match_operand:DI 2 "gpc_reg_operand" ""))
+        (ashift:SI (match_operator:SI 1 "scc_comparison_operator"
+                                      [(match_operand 2 "cc_reg_operand" "")
+                                       (const_int 0)])
+                   (match_operand:SI 3 "const_int_operand" ""))
         (const_int 0)))
-   (clobber (match_scratch:DI 3 ""))]
-  "TARGET_64BIT && reload_completed"
-  [(set (match_dup 3)
-       (plus:DI (gt:DI (match_dup 1) (const_int 0))
-                (match_dup 2)))
+   (set (match_operand:SI 4 "gpc_reg_operand" "")
+       (ashift:SI (match_op_dup 1 [(match_dup 2) (const_int 0)])
+                  (match_dup 3)))]
+  "reload_completed"
+  [(set (match_dup 4)
+       (ashift:SI (match_op_dup 1 [(match_dup 2) (const_int 0)])
+                  (match_dup 3)))
    (set (match_dup 0)
-       (compare:CC (match_dup 3)
+       (compare:CC (match_dup 4)
                    (const_int 0)))]
   "")
 
-(define_insn ""
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (plus:SI (gt:SI (match_operand:SI 1 "gpc_reg_operand" "r,r")
-                        (const_int 0))
-                 (match_operand:SI 2 "gpc_reg_operand" "r,r"))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "=&r,&r")
-       (plus:SI (gt:SI (match_dup 1) (const_int 0)) (match_dup 2)))]
-  "TARGET_32BIT"
-  "@
-   addc %0,%1,%1\;subfe %0,%1,%0\;addze. %0,%2
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "12,16")])
+;; There is a 3 cycle delay between consecutive mfcr instructions
+;; so it is useful to combine 2 scc instructions to use only one mfcr.
 
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_cr0_operand" "")
-       (compare:CC
-        (plus:SI (gt:SI (match_operand:SI 1 "gpc_reg_operand" "")
-                        (const_int 0))
-                 (match_operand:SI 2 "gpc_reg_operand" ""))
-        (const_int 0)))
-   (set (match_operand:SI 0 "gpc_reg_operand" "")
-       (plus:SI (gt:SI (match_dup 1) (const_int 0)) (match_dup 2)))]
-  "TARGET_32BIT && reload_completed"
-  [(set (match_dup 0)
-       (plus:SI (gt:SI (match_dup 1) (const_int 0)) (match_dup 2)))
-   (set (match_dup 3)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
+(define_peephole
+  [(set (match_operand:SI 0 "gpc_reg_operand" "=r")
+       (match_operator:SI 1 "scc_comparison_operator"
+                          [(match_operand 2 "cc_reg_operand" "y")
+                           (const_int 0)]))
+   (set (match_operand:SI 3 "gpc_reg_operand" "=r")
+       (match_operator:SI 4 "scc_comparison_operator"
+                          [(match_operand 5 "cc_reg_operand" "y")
+                           (const_int 0)]))]
+  "REGNO (operands[2]) != REGNO (operands[5])"
+  "mfcr %3\;rlwinm %0,%3,%J1,1\;rlwinm %3,%3,%J4,1"
+  [(set_attr "type" "mfcr")
+   (set_attr "length" "12")])
 
-(define_insn ""
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (plus:DI (gt:DI (match_operand:DI 1 "gpc_reg_operand" "r,r")
-                        (const_int 0))
-                 (match_operand:DI 2 "gpc_reg_operand" "r,r"))
-        (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "=&r,&r")
-       (plus:DI (gt:DI (match_dup 1) (const_int 0)) (match_dup 2)))]
-  "TARGET_64BIT"
-  "@
-   addc %0,%1,%1\;subfe %0,%1,%0\;addze. %0,%2
-   #"
-  [(set_attr "type" "compare")
-   (set_attr "length" "12,16")])
+(define_peephole
+  [(set (match_operand:DI 0 "gpc_reg_operand" "=r")
+       (match_operator:DI 1 "scc_comparison_operator"
+                          [(match_operand 2 "cc_reg_operand" "y")
+                           (const_int 0)]))
+   (set (match_operand:DI 3 "gpc_reg_operand" "=r")
+       (match_operator:DI 4 "scc_comparison_operator"
+                          [(match_operand 5 "cc_reg_operand" "y")
+                           (const_int 0)]))]
+  "TARGET_POWERPC64 && REGNO (operands[2]) != REGNO (operands[5])"
+  "mfcr %3\;rlwinm %0,%3,%J1,1\;rlwinm %3,%3,%J4,1"
+  [(set_attr "type" "mfcr")
+   (set_attr "length" "12")])
 
-(define_split
-  [(set (match_operand:CC 3 "cc_reg_not_micro_cr0_operand" "")
-       (compare:CC
-        (plus:DI (gt:DI (match_operand:DI 1 "gpc_reg_operand" "")
-                        (const_int 0))
-                 (match_operand:DI 2 "gpc_reg_operand" ""))
-        (const_int 0)))
-   (set (match_operand:DI 0 "gpc_reg_operand" "")
-       (plus:DI (gt:DI (match_dup 1) (const_int 0)) (match_dup 2)))]
-  "TARGET_64BIT && reload_completed"
-  [(set (match_dup 0)
-       (plus:DI (gt:DI (match_dup 1) (const_int 0)) (match_dup 2)))
-   (set (match_dup 3)
-       (compare:CC (match_dup 0)
-                   (const_int 0)))]
-  "")
 
-(define_insn_and_split "*gtu<mode>"
+(define_mode_attr scc_eq_op2 [(SI "rKLI")
+                             (DI "rKJI")])
+
+(define_insn_and_split "eq<mode>3"
+  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
+       (eq:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")
+               (match_operand:GPR 2 "scc_eq_operand" "<scc_eq_op2>")))
+   (clobber (match_scratch:GPR 3 "=r"))
+   (clobber (match_scratch:GPR 4 "=r"))]
+  ""
+  "#"
+  ""
+  [(set (match_dup 4)
+       (clz:GPR (match_dup 3)))
+   (set (match_dup 0)
+       (lshiftrt:GPR (match_dup 4)
+                     (match_dup 5)))]
+{
+  operands[3] = rs6000_emit_eqne (<MODE>mode,
+                                 operands[1], operands[2], operands[3]);
+
+  if (GET_CODE (operands[4]) == SCRATCH)
+    operands[4] = gen_reg_rtx (<MODE>mode);
+
+  operands[5] = GEN_INT (exact_log2 (GET_MODE_BITSIZE (<MODE>mode)));
+}
+  [(set (attr "length")
+       (if_then_else (match_test "operands[2] == const0_rtx")
+                     (const_string "8")
+                     (const_string "12")))])
+
+(define_insn_and_split "ne<mode>3"
+  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
+       (ne:P (match_operand:P 1 "gpc_reg_operand" "r")
+             (match_operand:P 2 "scc_eq_operand" "<scc_eq_op2>")))
+   (clobber (match_scratch:P 3 "=r"))
+   (clobber (match_scratch:P 4 "=r"))
+   (clobber (reg:P CA_REGNO))]
+  "!TARGET_ISEL"
+  "#"
+  ""
+  [(parallel [(set (match_dup 4)
+                  (plus:P (match_dup 3)
+                          (const_int -1)))
+             (set (reg:P CA_REGNO)
+                  (ne:P (match_dup 3)
+                        (const_int 0)))])
+   (parallel [(set (match_dup 0)
+                  (plus:P (plus:P (not:P (match_dup 4))
+                                  (reg:P CA_REGNO))
+                          (match_dup 3)))
+             (clobber (reg:P CA_REGNO))])]
+{
+  operands[3] = rs6000_emit_eqne (<MODE>mode,
+                                 operands[1], operands[2], operands[3]);
+
+  if (GET_CODE (operands[4]) == SCRATCH)
+    operands[4] = gen_reg_rtx (<MODE>mode);
+}
+  [(set (attr "length")
+       (if_then_else (match_test "operands[2] == const0_rtx")
+                     (const_string "8")
+                     (const_string "12")))])
+
+(define_insn_and_split "*neg_eq_<mode>"
   [(set (match_operand:P 0 "gpc_reg_operand" "=r")
-       (gtu:P (match_operand:P 1 "gpc_reg_operand" "r")
-              (match_operand:P 2 "reg_or_short_operand" "rI")))]
+       (neg:P (eq:P (match_operand:P 1 "gpc_reg_operand" "r")
+                    (match_operand:P 2 "scc_eq_operand" "<scc_eq_op2>"))))
+   (clobber (match_scratch:P 3 "=r"))
+   (clobber (match_scratch:P 4 "=r"))
+   (clobber (reg:P CA_REGNO))]
   ""
   "#"
   ""
-  [(set (match_dup 0) (neg:P (gtu:P (match_dup 1) (match_dup 2))))
-   (set (match_dup 0) (neg:P (match_dup 0)))]
-  "")
+  [(parallel [(set (match_dup 4)
+                  (plus:P (match_dup 3)
+                          (const_int -1)))
+             (set (reg:P CA_REGNO)
+                  (ne:P (match_dup 3)
+                        (const_int 0)))])
+   (parallel [(set (match_dup 0)
+                  (plus:P (reg:P CA_REGNO)
+                          (const_int -1)))
+             (clobber (reg:P CA_REGNO))])]
+{
+  operands[3] = rs6000_emit_eqne (<MODE>mode,
+                                 operands[1], operands[2], operands[3]);
 
-(define_insn_and_split "*gtu<mode>_compare"
-  [(set (match_operand:CC 3 "cc_reg_operand" "=x,?y")
-       (compare:CC
-        (gtu:P (match_operand:P 1 "gpc_reg_operand" "r,r")
-                (match_operand:P 2 "reg_or_short_operand" "rI,rI"))
-        (const_int 0)))
-   (set (match_operand:P 0 "gpc_reg_operand" "=r,r")
-       (gtu:P (match_dup 1) (match_dup 2)))]
+  if (GET_CODE (operands[4]) == SCRATCH)
+    operands[4] = gen_reg_rtx (<MODE>mode);
+}
+  [(set (attr "length")
+       (if_then_else (match_test "operands[2] == const0_rtx")
+                     (const_string "8")
+                     (const_string "12")))])
+
+(define_insn_and_split "*neg_ne_<mode>"
+  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
+       (neg:P (ne:P (match_operand:P 1 "gpc_reg_operand" "r")
+                    (match_operand:P 2 "scc_eq_operand" "<scc_eq_op2>"))))
+   (clobber (match_scratch:P 3 "=r"))
+   (clobber (match_scratch:P 4 "=r"))
+   (clobber (reg:P CA_REGNO))]
   ""
   "#"
   ""
-  [(set (match_dup 0) (neg:P (gtu:P (match_dup 1) (match_dup 2))))
-   (parallel [(set (match_dup 3)
-                  (compare:CC (neg:P (match_dup 0)) (const_int 0)))
-             (set (match_dup 0) (neg:P (match_dup 0)))])]
-  "")
+  [(parallel [(set (match_dup 4)
+                  (neg:P (match_dup 3)))
+             (set (reg:P CA_REGNO)
+                  (eq:P (match_dup 3)
+                        (const_int 0)))])
+   (parallel [(set (match_dup 0)
+                  (plus:P (reg:P CA_REGNO)
+                          (const_int -1)))
+             (clobber (reg:P CA_REGNO))])]
+{
+  operands[3] = rs6000_emit_eqne (<MODE>mode,
+                                 operands[1], operands[2], operands[3]);
+
+  if (GET_CODE (operands[4]) == SCRATCH)
+    operands[4] = gen_reg_rtx (<MODE>mode);
+}
+  [(set (attr "length")
+       (if_then_else (match_test "operands[2] == const0_rtx")
+                     (const_string "8")
+                     (const_string "12")))])
 
-(define_insn_and_split "*plus_gtu<mode>"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=&r")
-        (plus:P (gtu:P (match_operand:P 1 "gpc_reg_operand" "r")
-                      (match_operand:P 2 "reg_or_short_operand" "rI"))
-               (match_operand:P 3 "reg_or_short_operand" "rI")))]
+(define_insn_and_split "*plus_eq_<mode>"
+  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
+       (plus:P (eq:P (match_operand:P 1 "gpc_reg_operand" "r")
+                     (match_operand:P 2 "scc_eq_operand" "<scc_eq_op2>"))
+               (match_operand:P 3 "gpc_reg_operand" "r")))
+   (clobber (match_scratch:P 4 "=r"))
+   (clobber (match_scratch:P 5 "=r"))
+   (clobber (reg:P CA_REGNO))]
   ""
   "#"
-  "&& !reg_overlap_mentioned_p (operands[0], operands[3])"
-  [(set (match_dup 0) (neg:P (gtu:P (match_dup 1) (match_dup 2))))
-   (set (match_dup 0) (minus:P (match_dup 3) (match_dup 0)))]
-  "")
+  ""
+  [(parallel [(set (match_dup 5)
+                  (neg:P (match_dup 4)))
+             (set (reg:P CA_REGNO)
+                  (eq:P (match_dup 4)
+                        (const_int 0)))])
+   (parallel [(set (match_dup 0)
+                  (plus:P (match_dup 3)
+                          (reg:P CA_REGNO)))
+             (clobber (reg:P CA_REGNO))])]
+{
+  operands[4] = rs6000_emit_eqne (<MODE>mode,
+                                 operands[1], operands[2], operands[4]);
 
-(define_insn_and_split "*plus_gtu<mode>_compare"
-  [(set (match_operand:CC 4 "cc_reg_operand" "=x,x,?y,?y")
-       (compare:CC
-        (plus:P (gtu:P (match_operand:P 1 "gpc_reg_operand" "r,r,r,r")
-                       (match_operand:P 2 "reg_or_short_operand" "I,r,I,r"))
-                (match_operand:P 3 "gpc_reg_operand" "r,r,r,r"))
-        (const_int 0)))
-   (set (match_operand:P 0 "gpc_reg_operand" "=&r,&r,&r,&r")
-       (plus:P (gtu:P (match_dup 1) (match_dup 2)) (match_dup 3)))]
+  if (GET_CODE (operands[5]) == SCRATCH)
+    operands[5] = gen_reg_rtx (<MODE>mode);
+}
+  [(set (attr "length")
+       (if_then_else (match_test "operands[2] == const0_rtx")
+                     (const_string "8")
+                     (const_string "12")))])
+
+(define_insn_and_split "*plus_ne_<mode>"
+  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
+       (plus:P (ne:P (match_operand:P 1 "gpc_reg_operand" "r")
+                     (match_operand:P 2 "scc_eq_operand" "<scc_eq_op2>"))
+               (match_operand:P 3 "gpc_reg_operand" "r")))
+   (clobber (match_scratch:P 4 "=r"))
+   (clobber (match_scratch:P 5 "=r"))
+   (clobber (reg:P CA_REGNO))]
   ""
   "#"
-  "&& !reg_overlap_mentioned_p (operands[0], operands[3])"
-  [(set (match_dup 0) (neg:P (gtu:P (match_dup 1) (match_dup 2))))
-   (parallel [(set (match_dup 4)
-                  (compare:CC (minus:P (match_dup 3) (match_dup 0))
-                              (const_int 0)))
-             (set (match_dup 0) (minus:P (match_dup 3) (match_dup 0)))])]
-  "")
+  ""
+  [(parallel [(set (match_dup 5)
+                  (plus:P (match_dup 4)
+                          (const_int -1)))
+             (set (reg:P CA_REGNO)
+                  (ne:P (match_dup 4)
+                        (const_int 0)))])
+   (parallel [(set (match_dup 0)
+                  (plus:P (match_dup 3)
+                          (reg:P CA_REGNO)))
+             (clobber (reg:P CA_REGNO))])]
+{
+  operands[4] = rs6000_emit_eqne (<MODE>mode,
+                                 operands[1], operands[2], operands[4]);
+
+  if (GET_CODE (operands[5]) == SCRATCH)
+    operands[5] = gen_reg_rtx (<MODE>mode);
+}
+  [(set (attr "length")
+       (if_then_else (match_test "operands[2] == const0_rtx")
+                     (const_string "8")
+                     (const_string "12")))])
 
-(define_insn "*neg_gtu<mode>"
+(define_insn_and_split "*minus_eq_<mode>"
   [(set (match_operand:P 0 "gpc_reg_operand" "=r")
-       (neg:P (gtu:P (match_operand:P 1 "gpc_reg_operand" "r")
-                     (match_operand:P 2 "reg_or_short_operand" "rI"))))]
+       (minus:P (match_operand:P 3 "gpc_reg_operand" "r")
+                (eq:P (match_operand:P 1 "gpc_reg_operand" "r")
+                      (match_operand:P 2 "scc_eq_operand" "<scc_eq_op2>"))))
+   (clobber (match_scratch:P 4 "=r"))
+   (clobber (match_scratch:P 5 "=r"))
+   (clobber (reg:P CA_REGNO))]
   ""
-  "subf%I2c %0,%1,%2\;subfe %0,%0,%0"
-  [(set_attr "type" "two")
-   (set_attr "length" "8")])
+  "#"
+  ""
+  [(parallel [(set (match_dup 5)
+                  (plus:P (match_dup 4)
+                          (const_int -1)))
+             (set (reg:P CA_REGNO)
+                  (ne:P (match_dup 4)
+                        (const_int 0)))])
+   (parallel [(set (match_dup 0)
+                  (plus:P (plus:P (match_dup 3)
+                                  (reg:P CA_REGNO))
+                          (const_int -1)))
+             (clobber (reg:P CA_REGNO))])]
+{
+  operands[4] = rs6000_emit_eqne (<MODE>mode,
+                                 operands[1], operands[2], operands[4]);
+
+  if (GET_CODE (operands[5]) == SCRATCH)
+    operands[5] = gen_reg_rtx (<MODE>mode);
+}
+  [(set (attr "length")
+       (if_then_else (match_test "operands[2] == const0_rtx")
+                     (const_string "8")
+                     (const_string "12")))])
+
+(define_insn_and_split "*minus_ne_<mode>"
+  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
+       (minus:P (match_operand:P 3 "gpc_reg_operand" "r")
+                (ne:P (match_operand:P 1 "gpc_reg_operand" "r")
+                      (match_operand:P 2 "scc_eq_operand" "<scc_eq_op2>"))))
+   (clobber (match_scratch:P 4 "=r"))
+   (clobber (match_scratch:P 5 "=r"))
+   (clobber (reg:P CA_REGNO))]
+  ""
+  "#"
+  ""
+  [(parallel [(set (match_dup 5)
+                  (neg:P (match_dup 4)))
+             (set (reg:P CA_REGNO)
+                  (eq:P (match_dup 4)
+                        (const_int 0)))])
+   (parallel [(set (match_dup 0)
+                  (plus:P (plus:P (match_dup 3)
+                                  (reg:P CA_REGNO))
+                          (const_int -1)))
+             (clobber (reg:P CA_REGNO))])]
+{
+  operands[4] = rs6000_emit_eqne (<MODE>mode,
+                                 operands[1], operands[2], operands[4]);
+
+  if (GET_CODE (operands[5]) == SCRATCH)
+    operands[5] = gen_reg_rtx (<MODE>mode);
+}
+  [(set (attr "length")
+       (if_then_else (match_test "operands[2] == const0_rtx")
+                     (const_string "8")
+                     (const_string "12")))])
+
+(define_insn_and_split "*eqsi3_ext<mode>"
+  [(set (match_operand:EXTSI 0 "gpc_reg_operand" "=r")
+       (eq:EXTSI (match_operand:SI 1 "gpc_reg_operand" "r")
+                 (match_operand:SI 2 "scc_eq_operand" "rKLI")))
+   (clobber (match_scratch:SI 3 "=r"))
+   (clobber (match_scratch:SI 4 "=r"))]
+  ""
+  "#"
+  ""
+  [(set (match_dup 4)
+       (clz:SI (match_dup 3)))
+   (set (match_dup 0)
+       (zero_extend:EXTSI
+         (lshiftrt:SI (match_dup 4)
+                      (const_int 5))))]
+{
+  operands[3] = rs6000_emit_eqne (SImode,
+                                 operands[1], operands[2], operands[3]);
 
+  if (GET_CODE (operands[4]) == SCRATCH)
+    operands[4] = gen_reg_rtx (SImode);
+}
+  [(set (attr "length")
+       (if_then_else (match_test "operands[2] == const0_rtx")
+                     (const_string "8")
+                     (const_string "12")))])
+
+(define_insn_and_split "*nesi3_ext<mode>"
+  [(set (match_operand:EXTSI 0 "gpc_reg_operand" "=r")
+       (ne:EXTSI (match_operand:SI 1 "gpc_reg_operand" "r")
+                 (match_operand:SI 2 "scc_eq_operand" "rKLI")))
+   (clobber (match_scratch:SI 3 "=r"))
+   (clobber (match_scratch:SI 4 "=r"))
+   (clobber (match_scratch:EXTSI 5 "=r"))]
+  ""
+  "#"
+  ""
+  [(set (match_dup 4)
+       (clz:SI (match_dup 3)))
+   (set (match_dup 5)
+       (zero_extend:EXTSI
+         (lshiftrt:SI (match_dup 4)
+                      (const_int 5))))
+   (set (match_dup 0)
+       (xor:EXTSI (match_dup 5)
+                  (const_int 1)))]
+{
+  operands[3] = rs6000_emit_eqne (SImode,
+                                 operands[1], operands[2], operands[3]);
+
+  if (GET_CODE (operands[4]) == SCRATCH)
+    operands[4] = gen_reg_rtx (SImode);
+  if (GET_CODE (operands[5]) == SCRATCH)
+    operands[5] = gen_reg_rtx (<MODE>mode);
+}
+  [(set (attr "length")
+       (if_then_else (match_test "operands[2] == const0_rtx")
+                     (const_string "12")
+                     (const_string "16")))])
 \f
 ;; Define both directions of branch and return.  If we need a reload
 ;; register, we'd rather use CR0 since it is much easier to copy a
   [(set_attr "type" "jmpreg")])
 
 (define_insn "nop"
-  [(const_int 0)]
+  [(unspec [(const_int 0)] UNSPEC_NOP)]
   ""
   "nop")
 
 
 (define_insn "*ctr<mode>_internal1"
   [(set (pc)
-       (if_then_else (ne (match_operand:P 1 "register_operand" "c,*r,*r,*r")
+       (if_then_else (ne (match_operand:P 1 "register_operand" "c,*b,*b,*b")
                          (const_int 1))
                      (label_ref (match_operand 0 "" ""))
                      (pc)))
     return \"bdz $+8\;b %l0\";
 }"
   [(set_attr "type" "branch")
-   (set_attr "length" "*,12,16,16")])
+   (set_attr "length" "*,16,20,20")])
 
 (define_insn "*ctr<mode>_internal2"
   [(set (pc)
-       (if_then_else (ne (match_operand:P 1 "register_operand" "c,*r,*r,*r")
+       (if_then_else (ne (match_operand:P 1 "register_operand" "c,*b,*b,*b")
                          (const_int 1))
                      (pc)
                      (label_ref (match_operand 0 "" ""))))
     return \"bdnz $+8\;b %l0\";
 }"
   [(set_attr "type" "branch")
-   (set_attr "length" "*,12,16,16")])
+   (set_attr "length" "*,16,20,20")])
 
 ;; Similar but use EQ
 
 (define_insn "*ctr<mode>_internal5"
   [(set (pc)
-       (if_then_else (eq (match_operand:P 1 "register_operand" "c,*r,*r,*r")
+       (if_then_else (eq (match_operand:P 1 "register_operand" "c,*b,*b,*b")
                          (const_int 1))
                      (label_ref (match_operand 0 "" ""))
                      (pc)))
     return \"bdnz $+8\;b %l0\";
 }"
   [(set_attr "type" "branch")
-   (set_attr "length" "*,12,16,16")])
+   (set_attr "length" "*,16,20,20")])
 
 (define_insn "*ctr<mode>_internal6"
   [(set (pc)
-       (if_then_else (eq (match_operand:P 1 "register_operand" "c,*r,*r,*r")
+       (if_then_else (eq (match_operand:P 1 "register_operand" "c,*b,*b,*b")
                          (const_int 1))
                      (pc)
                      (label_ref (match_operand 0 "" ""))))
     return \"bdz $+8\;b %l0\";
 }"
   [(set_attr "type" "branch")
-   (set_attr "length" "*,12,16,16")])
+   (set_attr "length" "*,16,20,20")])
 
 ;; Now the splitters if we could not allocate the CTR register
 
    (clobber (match_scratch:CC 3 ""))
    (clobber (match_scratch:P 4 ""))]
   "reload_completed"
-  [(parallel [(set (match_dup 3)
-                  (compare:CC (plus:P (match_dup 1)
-                                       (const_int -1))
-                              (const_int 0)))
-             (set (match_dup 0)
-                  (plus:P (match_dup 1)
-                           (const_int -1)))])
+  [(set (match_dup 3)
+       (compare:CC (match_dup 1)
+                   (const_int 1)))
+   (set (match_dup 0)
+       (plus:P (match_dup 1)
+               (const_int -1)))
    (set (pc) (if_then_else (match_dup 7)
                           (match_dup 5)
                           (match_dup 6)))]
    (clobber (match_scratch:CC 3 ""))
    (clobber (match_scratch:P 4 ""))]
   "reload_completed && ! gpc_reg_operand (operands[0], SImode)"
-  [(parallel [(set (match_dup 3)
-                  (compare:CC (plus:P (match_dup 1)
-                                       (const_int -1))
-                              (const_int 0)))
-             (set (match_dup 4)
-                  (plus:P (match_dup 1)
-                           (const_int -1)))])
+  [(set (match_dup 3)
+       (compare:CC (match_dup 1)
+                   (const_int 1)))
+   (set (match_dup 4)
+       (plus:P (match_dup 1)
+               (const_int -1)))
    (set (match_dup 0)
        (match_dup 4))
    (set (pc) (if_then_else (match_dup 7)
   [(set_attr "length" "20")])
 
 (define_insn "rs6000_mftb_<mode>"
-  [(set (match_operand:P 0 "gpc_reg_operand" "=r")
-        (unspec_volatile:P [(const_int 0)] UNSPECV_MFTB))]
+  [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
+        (unspec_volatile:GPR [(const_int 0)] UNSPECV_MFTB))]
   ""
 {
   if (TARGET_MFCRF)
 ;; a GPR.  The addis instruction must be adjacent to the load, and use the same
 ;; register that is being loaded.  The fused ops must be physically adjacent.
 
-;; We use define_peephole for the actual addis/load, and the register used to
-;; hold the addis value must be the same as the register being loaded.  We use
-;; define_peephole2 to change the register used for addis to be the register
-;; being loaded, since we can look at whether it is dead after the load insn.
-
-(define_peephole
-  [(set (match_operand:P 0 "base_reg_operand" "")
-       (match_operand:P 1 "fusion_gpr_addis" ""))
-   (set (match_operand:INT1 2 "base_reg_operand" "")
-       (match_operand:INT1 3 "fusion_gpr_mem_load" ""))]
-  "TARGET_P8_FUSION && fusion_gpr_load_p (operands, false)"
-{
-  return emit_fusion_gpr_load (operands);
-}
-  [(set_attr "type" "load")
-   (set_attr "length" "8")])
+;; Find cases where the addis that feeds into a load instruction is either used
+;; once or is the same as the target register, and replace it with the fusion
+;; insn
 
 (define_peephole2
   [(set (match_operand:P 0 "base_reg_operand" "")
    (set (match_operand:INT1 2 "base_reg_operand" "")
        (match_operand:INT1 3 "fusion_gpr_mem_load" ""))]
   "TARGET_P8_FUSION
-   && (REGNO (operands[0]) != REGNO (operands[2])
-       || GET_CODE (operands[3]) == SIGN_EXTEND)
-   && fusion_gpr_load_p (operands, true)"
+   && fusion_gpr_load_p (operands[0], operands[1], operands[2],
+                        operands[3])"
   [(const_int 0)]
 {
   expand_fusion_gpr_load (operands);
   DONE;
 })
 
+;; Fusion insn, created by the define_peephole2 above (and eventually by
+;; reload)
+
+(define_insn "fusion_gpr_load_<mode>"
+  [(set (match_operand:INT1 0 "base_reg_operand" "=&b")
+       (unspec:INT1 [(match_operand:INT1 1 "fusion_gpr_mem_combo" "")]
+                    UNSPEC_FUSION_GPR))]
+  "TARGET_P8_FUSION"
+{
+  return emit_fusion_gpr_load (operands[0], operands[1]);
+}
+  [(set_attr "type" "load")
+   (set_attr "length" "8")])
+
 \f
 ;; Miscellaneous ISA 2.06 (power7) instructions
 (define_insn "addg6s"
   ""
   "")
 
-;; The Advance Toolchain 7.0-3 added private builtins: __builtin_longdouble_dw0
-;; and __builtin_longdouble_dw1 to optimize glibc.  Add support for these
-;; builtins here.
-
-(define_expand "unpacktf_0"
-  [(set (match_operand:DF 0 "nonimmediate_operand" "")
-       (unspec:DF [(match_operand:TF 1 "register_operand" "")
-                   (const_int 0)]
-        UNSPEC_UNPACK_128BIT))]
-  ""
-  "")
-
-(define_expand "unpacktf_1"
-  [(set (match_operand:DF 0 "nonimmediate_operand" "")
-       (unspec:DF [(match_operand:TF 1 "register_operand" "")
-                   (const_int 1)]
-        UNSPEC_UNPACK_128BIT))]
-  ""
-  "")
-
 (define_insn_and_split "unpack<mode>_dm"
   [(set (match_operand:<FP128_64> 0 "nonimmediate_operand" "=d,m,d,r,m")
        (unspec:<FP128_64>