gcc/config/aarch64/aarch64-simd.md

   1 ;; Machine description for AArch64 AdvSIMD architecture.
   2 ;; Copyright (C) 2011-2014 Free Software Foundation, Inc.
   3 ;; Contributed by ARM Ltd.
   4 ;;
   5 ;; This file is part of GCC.
   6 ;;
   7 ;; GCC is free software; you can redistribute it and/or modify it
   8 ;; under the terms of the GNU General Public License as published by
   9 ;; the Free Software Foundation; either version 3, or (at your option)
  10 ;; any later version.
  11 ;;
  12 ;; GCC is distributed in the hope that it will be useful, but
  13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
  14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 ;; General Public License for more details.
  16 ;;
  17 ;; You should have received a copy of the GNU General Public License
  18 ;; along with GCC; see the file COPYING3.  If not see
  19 ;; <http://www.gnu.org/licenses/>.
  20
  21 (define_expand "mov<mode>"
  22   [(set (match_operand:VALL 0 "nonimmediate_operand" "")
  23         (match_operand:VALL 1 "general_operand" ""))]
  24   "TARGET_SIMD"
  25   "
  26     if (GET_CODE (operands[0]) == MEM)
  27       operands[1] = force_reg (<MODE>mode, operands[1]);
  28   "
  29 )
  30
  31 (define_expand "movmisalign<mode>"
  32   [(set (match_operand:VALL 0 "nonimmediate_operand" "")
  33         (match_operand:VALL 1 "general_operand" ""))]
  34   "TARGET_SIMD"
  35 {
  36   /* This pattern is not permitted to fail during expansion: if both arguments
  37      are non-registers (e.g. memory := constant, which can be created by the
  38      auto-vectorizer), force operand 1 into a register.  */
  39   if (!register_operand (operands[0], <MODE>mode)
  40       && !register_operand (operands[1], <MODE>mode))
  41     operands[1] = force_reg (<MODE>mode, operands[1]);
  42 })
  43
  44 (define_insn "aarch64_simd_dup<mode>"
  45   [(set (match_operand:VDQ 0 "register_operand" "=w, w")
  46         (vec_duplicate:VDQ (match_operand:<VEL> 1 "register_operand" "r, w")))]
  47   "TARGET_SIMD"
  48   "@
  49    dup\\t%0.<Vtype>, %<vw>1
  50    dup\\t%0.<Vtype>, %1.<Vetype>[0]"
  51   [(set_attr "type" "neon_from_gp<q>, neon_dup<q>")]
  52 )
  53
  54 (define_insn "aarch64_simd_dup<mode>"
  55   [(set (match_operand:VDQF 0 "register_operand" "=w")
  56         (vec_duplicate:VDQF (match_operand:<VEL> 1 "register_operand" "w")))]
  57   "TARGET_SIMD"
  58   "dup\\t%0.<Vtype>, %1.<Vetype>[0]"
  59   [(set_attr "type" "neon_dup<q>")]
  60 )
  61
  62 (define_insn "aarch64_dup_lane<mode>"
  63   [(set (match_operand:VALL 0 "register_operand" "=w")
  64         (vec_duplicate:VALL
  65           (vec_select:<VEL>
  66             (match_operand:VALL 1 "register_operand" "w")
  67             (parallel [(match_operand:SI 2 "immediate_operand" "i")])
  68           )))]
  69   "TARGET_SIMD"
  70   {
  71     operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
  72     return "dup\\t%0.<Vtype>, %1.<Vetype>[%2]";
  73   }
  74   [(set_attr "type" "neon_dup<q>")]
  75 )
  76
  77 (define_insn "aarch64_dup_lane_<vswap_width_name><mode>"
  78   [(set (match_operand:VALL 0 "register_operand" "=w")
  79         (vec_duplicate:VALL
  80           (vec_select:<VEL>
  81             (match_operand:<VSWAP_WIDTH> 1 "register_operand" "w")
  82             (parallel [(match_operand:SI 2 "immediate_operand" "i")])
  83           )))]
  84   "TARGET_SIMD"
  85   {
  86     operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
  87                                           INTVAL (operands[2])));
  88     return "dup\\t%0.<Vtype>, %1.<Vetype>[%2]";
  89   }
  90   [(set_attr "type" "neon_dup<q>")]
  91 )
  92
  93 (define_insn "*aarch64_simd_mov<mode>"
  94   [(set (match_operand:VD 0 "nonimmediate_operand"
  95                 "=w, m,  w, ?r, ?w, ?r, w")
  96         (match_operand:VD 1 "general_operand"
  97                 "m,  w,  w,  w,  r,  r, Dn"))]
  98   "TARGET_SIMD
  99    && (register_operand (operands[0], <MODE>mode)
 100        || register_operand (operands[1], <MODE>mode))"
 101 {
 102    switch (which_alternative)
 103      {
 104      case 0: return "ldr\\t%d0, %1";
 105      case 1: return "str\\t%d1, %0";
 106      case 2: return "orr\t%0.<Vbtype>, %1.<Vbtype>, %1.<Vbtype>";
 107      case 3: return "umov\t%0, %1.d[0]";
 108      case 4: return "ins\t%0.d[0], %1";
 109      case 5: return "mov\t%0, %1";
 110      case 6:
 111         return aarch64_output_simd_mov_immediate (operands[1],
 112                                                   <MODE>mode, 64);
 113      default: gcc_unreachable ();
 114      }
 115 }
 116   [(set_attr "type" "neon_load1_1reg<q>, neon_store1_1reg<q>,\
 117                      neon_logic<q>, neon_to_gp<q>, neon_from_gp<q>,\
 118                      mov_reg, neon_move<q>")]
 119 )
 120
 121 (define_insn "*aarch64_simd_mov<mode>"
 122   [(set (match_operand:VQ 0 "nonimmediate_operand"
 123                 "=w, m,  w, ?r, ?w, ?r, w")
 124         (match_operand:VQ 1 "general_operand"
 125                 "m,  w,  w,  w,  r,  r, Dn"))]
 126   "TARGET_SIMD
 127    && (register_operand (operands[0], <MODE>mode)
 128        || register_operand (operands[1], <MODE>mode))"
 129 {
 130   switch (which_alternative)
 131     {
 132     case 0:
 133         return "ldr\\t%q0, %1";
 134     case 1:
 135         return "str\\t%q1, %0";
 136     case 2:
 137         return "orr\t%0.<Vbtype>, %1.<Vbtype>, %1.<Vbtype>";
 138     case 3:
 139     case 4:
 140     case 5:
 141         return "#";
 142     case 6:
 143         return aarch64_output_simd_mov_immediate (operands[1], <MODE>mode, 128);
 144     default:
 145         gcc_unreachable ();
 146     }
 147 }
 148   [(set_attr "type" "neon_load1_1reg<q>, neon_store1_1reg<q>,\
 149                      neon_logic<q>, multiple, multiple, multiple,\
 150                      neon_move<q>")
 151    (set_attr "length" "4,4,4,8,8,8,4")]
 152 )
 153
 154 (define_split
 155   [(set (match_operand:VQ 0 "register_operand" "")
 156       (match_operand:VQ 1 "register_operand" ""))]
 157   "TARGET_SIMD && reload_completed
 158    && GP_REGNUM_P (REGNO (operands[0]))
 159    && GP_REGNUM_P (REGNO (operands[1]))"
 160   [(set (match_dup 0) (match_dup 1))
 161    (set (match_dup 2) (match_dup 3))]
 162 {
 163   int rdest = REGNO (operands[0]);
 164   int rsrc = REGNO (operands[1]);
 165   rtx dest[2], src[2];
 166
 167   dest[0] = gen_rtx_REG (DImode, rdest);
 168   src[0] = gen_rtx_REG (DImode, rsrc);
 169   dest[1] = gen_rtx_REG (DImode, rdest + 1);
 170   src[1] = gen_rtx_REG (DImode, rsrc + 1);
 171
 172   aarch64_simd_disambiguate_copy (operands, dest, src, 2);
 173 })
 174
 175 (define_split
 176   [(set (match_operand:VQ 0 "register_operand" "")
 177         (match_operand:VQ 1 "register_operand" ""))]
 178   "TARGET_SIMD && reload_completed
 179    && ((FP_REGNUM_P (REGNO (operands[0])) && GP_REGNUM_P (REGNO (operands[1])))
 180        || (GP_REGNUM_P (REGNO (operands[0])) && FP_REGNUM_P (REGNO (operands[1]))))"
 181   [(const_int 0)]
 182 {
 183   aarch64_split_simd_move (operands[0], operands[1]);
 184   DONE;
 185 })
 186
 187 (define_expand "aarch64_split_simd_mov<mode>"
 188   [(set (match_operand:VQ 0)
 189         (match_operand:VQ 1))]
 190   "TARGET_SIMD"
 191   {
 192     rtx dst = operands[0];
 193     rtx src = operands[1];
 194
 195     if (GP_REGNUM_P (REGNO (src)))
 196       {
 197         rtx src_low_part = gen_lowpart (<VHALF>mode, src);
 198         rtx src_high_part = gen_highpart (<VHALF>mode, src);
 199
 200         emit_insn
 201           (gen_move_lo_quad_<mode> (dst, src_low_part));
 202         emit_insn
 203           (gen_move_hi_quad_<mode> (dst, src_high_part));
 204       }
 205
 206     else
 207       {
 208         rtx dst_low_part = gen_lowpart (<VHALF>mode, dst);
 209         rtx dst_high_part = gen_highpart (<VHALF>mode, dst);
 210         rtx lo = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
 211         rtx hi = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
 212
 213         emit_insn
 214           (gen_aarch64_simd_mov_from_<mode>low (dst_low_part, src, lo));
 215         emit_insn
 216           (gen_aarch64_simd_mov_from_<mode>high (dst_high_part, src, hi));
 217       }
 218     DONE;
 219   }
 220 )
 221
 222 (define_insn "aarch64_simd_mov_from_<mode>low"
 223   [(set (match_operand:<VHALF> 0 "register_operand" "=r")
 224         (vec_select:<VHALF>
 225           (match_operand:VQ 1 "register_operand" "w")
 226           (match_operand:VQ 2 "vect_par_cnst_lo_half" "")))]
 227   "TARGET_SIMD && reload_completed"
 228   "umov\t%0, %1.d[0]"
 229   [(set_attr "type" "neon_to_gp<q>")
 230    (set_attr "length" "4")
 231   ])
 232
 233 (define_insn "aarch64_simd_mov_from_<mode>high"
 234   [(set (match_operand:<VHALF> 0 "register_operand" "=r")
 235         (vec_select:<VHALF>
 236           (match_operand:VQ 1 "register_operand" "w")
 237           (match_operand:VQ 2 "vect_par_cnst_hi_half" "")))]
 238   "TARGET_SIMD && reload_completed"
 239   "umov\t%0, %1.d[1]"
 240   [(set_attr "type" "neon_to_gp<q>")
 241    (set_attr "length" "4")
 242   ])
 243
 244 (define_insn "orn<mode>3"
 245  [(set (match_operand:VDQ 0 "register_operand" "=w")
 246        (ior:VDQ (not:VDQ (match_operand:VDQ 1 "register_operand" "w"))
 247                 (match_operand:VDQ 2 "register_operand" "w")))]
 248  "TARGET_SIMD"
 249  "orn\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
 250   [(set_attr "type" "neon_logic<q>")]
 251 )
 252
 253 (define_insn "bic<mode>3"
 254  [(set (match_operand:VDQ 0 "register_operand" "=w")
 255        (and:VDQ (not:VDQ (match_operand:VDQ 1 "register_operand" "w"))
 256                 (match_operand:VDQ 2 "register_operand" "w")))]
 257  "TARGET_SIMD"
 258  "bic\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
 259   [(set_attr "type" "neon_logic<q>")]
 260 )
 261
 262 (define_insn "add<mode>3"
 263   [(set (match_operand:VDQ 0 "register_operand" "=w")
 264         (plus:VDQ (match_operand:VDQ 1 "register_operand" "w")
 265                   (match_operand:VDQ 2 "register_operand" "w")))]
 266   "TARGET_SIMD"
 267   "add\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
 268   [(set_attr "type" "neon_add<q>")]
 269 )
 270
 271 (define_insn "sub<mode>3"
 272   [(set (match_operand:VDQ 0 "register_operand" "=w")
 273         (minus:VDQ (match_operand:VDQ 1 "register_operand" "w")
 274                    (match_operand:VDQ 2 "register_operand" "w")))]
 275   "TARGET_SIMD"
 276   "sub\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
 277   [(set_attr "type" "neon_sub<q>")]
 278 )
 279
 280 (define_insn "mul<mode>3"
 281   [(set (match_operand:VDQM 0 "register_operand" "=w")
 282         (mult:VDQM (match_operand:VDQM 1 "register_operand" "w")
 283                    (match_operand:VDQM 2 "register_operand" "w")))]
 284   "TARGET_SIMD"
 285   "mul\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
 286   [(set_attr "type" "neon_mul_<Vetype><q>")]
 287 )
 288
 289 (define_insn "bswap<mode>"
 290   [(set (match_operand:VDQHSD 0 "register_operand" "=w")
 291         (bswap:VDQHSD (match_operand:VDQHSD 1 "register_operand" "w")))]
 292   "TARGET_SIMD"
 293   "rev<Vrevsuff>\\t%0.<Vbtype>, %1.<Vbtype>"
 294   [(set_attr "type" "neon_rev<q>")]
 295 )
 296
 297 (define_insn "aarch64_rbit<mode>"
 298   [(set (match_operand:VB 0 "register_operand" "=w")
 299         (unspec:VB [(match_operand:VB 1 "register_operand" "w")]
 300                    UNSPEC_RBIT))]
 301   "TARGET_SIMD"
 302   "rbit\\t%0.<Vbtype>, %1.<Vbtype>"
 303   [(set_attr "type" "neon_rbit")]
 304 )
 305
 306 (define_insn "*aarch64_mul3_elt<mode>"
 307  [(set (match_operand:VMUL 0 "register_operand" "=w")
 308     (mult:VMUL
 309       (vec_duplicate:VMUL
 310           (vec_select:<VEL>
 311             (match_operand:VMUL 1 "register_operand" "<h_con>")
 312             (parallel [(match_operand:SI 2 "immediate_operand")])))
 313       (match_operand:VMUL 3 "register_operand" "w")))]
 314   "TARGET_SIMD"
 315   {
 316     operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
 317     return "<f>mul\\t%0.<Vtype>, %3.<Vtype>, %1.<Vetype>[%2]";
 318   }
 319   [(set_attr "type" "neon<fp>_mul_<Vetype>_scalar<q>")]
 320 )
 321
 322 (define_insn "*aarch64_mul3_elt_<vswap_width_name><mode>"
 323   [(set (match_operand:VMUL_CHANGE_NLANES 0 "register_operand" "=w")
 324      (mult:VMUL_CHANGE_NLANES
 325        (vec_duplicate:VMUL_CHANGE_NLANES
 326           (vec_select:<VEL>
 327             (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
 328             (parallel [(match_operand:SI 2 "immediate_operand")])))
 329       (match_operand:VMUL_CHANGE_NLANES 3 "register_operand" "w")))]
 330   "TARGET_SIMD"
 331   {
 332     operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
 333                                           INTVAL (operands[2])));
 334     return "<f>mul\\t%0.<Vtype>, %3.<Vtype>, %1.<Vetype>[%2]";
 335   }
 336   [(set_attr "type" "neon<fp>_mul_<Vetype>_scalar<q>")]
 337 )
 338
 339 (define_insn "*aarch64_mul3_elt_to_128df"
 340   [(set (match_operand:V2DF 0 "register_operand" "=w")
 341      (mult:V2DF
 342        (vec_duplicate:V2DF
 343          (match_operand:DF 2 "register_operand" "w"))
 344       (match_operand:V2DF 1 "register_operand" "w")))]
 345   "TARGET_SIMD"
 346   "fmul\\t%0.2d, %1.2d, %2.d[0]"
 347   [(set_attr "type" "neon_fp_mul_d_scalar_q")]
 348 )
 349
 350 (define_insn "*aarch64_mul3_elt_to_64v2df"
 351   [(set (match_operand:DF 0 "register_operand" "=w")
 352      (mult:DF
 353        (vec_select:DF
 354          (match_operand:V2DF 1 "register_operand" "w")
 355          (parallel [(match_operand:SI 2 "immediate_operand")]))
 356        (match_operand:DF 3 "register_operand" "w")))]
 357   "TARGET_SIMD"
 358   {
 359     operands[2] = GEN_INT (ENDIAN_LANE_N (V2DFmode, INTVAL (operands[2])));
 360     return "fmul\\t%0.2d, %3.2d, %1.d[%2]";
 361   }
 362   [(set_attr "type" "neon_fp_mul_d_scalar_q")]
 363 )
 364
 365 (define_insn "neg<mode>2"
 366   [(set (match_operand:VDQ 0 "register_operand" "=w")
 367         (neg:VDQ (match_operand:VDQ 1 "register_operand" "w")))]
 368   "TARGET_SIMD"
 369   "neg\t%0.<Vtype>, %1.<Vtype>"
 370   [(set_attr "type" "neon_neg<q>")]
 371 )
 372
 373 (define_insn "abs<mode>2"
 374   [(set (match_operand:VDQ 0 "register_operand" "=w")
 375         (abs:VDQ (match_operand:VDQ 1 "register_operand" "w")))]
 376   "TARGET_SIMD"
 377   "abs\t%0.<Vtype>, %1.<Vtype>"
 378   [(set_attr "type" "neon_abs<q>")]
 379 )
 380
 381 (define_insn "abd<mode>_3"
 382   [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
 383         (abs:VDQ_BHSI (minus:VDQ_BHSI
 384                        (match_operand:VDQ_BHSI 1 "register_operand" "w")
 385                        (match_operand:VDQ_BHSI 2 "register_operand" "w"))))]
 386   "TARGET_SIMD"
 387   "sabd\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
 388   [(set_attr "type" "neon_abd<q>")]
 389 )
 390
 391 (define_insn "aba<mode>_3"
 392   [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
 393         (plus:VDQ_BHSI (abs:VDQ_BHSI (minus:VDQ_BHSI
 394                          (match_operand:VDQ_BHSI 1 "register_operand" "w")
 395                          (match_operand:VDQ_BHSI 2 "register_operand" "w")))
 396                        (match_operand:VDQ_BHSI 3 "register_operand" "0")))]
 397   "TARGET_SIMD"
 398   "saba\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
 399   [(set_attr "type" "neon_arith_acc<q>")]
 400 )
 401
 402 (define_insn "fabd<mode>_3"
 403   [(set (match_operand:VDQF 0 "register_operand" "=w")
 404         (abs:VDQF (minus:VDQF
 405                    (match_operand:VDQF 1 "register_operand" "w")
 406                    (match_operand:VDQF 2 "register_operand" "w"))))]
 407   "TARGET_SIMD"
 408   "fabd\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
 409   [(set_attr "type" "neon_fp_abd_<Vetype><q>")]
 410 )
 411
 412 (define_insn "*fabd_scalar<mode>3"
 413   [(set (match_operand:GPF 0 "register_operand" "=w")
 414         (abs:GPF (minus:GPF
 415                  (match_operand:GPF 1 "register_operand" "w")
 416                  (match_operand:GPF 2 "register_operand" "w"))))]
 417   "TARGET_SIMD"
 418   "fabd\t%<s>0, %<s>1, %<s>2"
 419   [(set_attr "type" "neon_fp_abd_<Vetype><q>")]
 420 )
 421
 422 (define_insn "and<mode>3"
 423   [(set (match_operand:VDQ 0 "register_operand" "=w")
 424         (and:VDQ (match_operand:VDQ 1 "register_operand" "w")
 425                  (match_operand:VDQ 2 "register_operand" "w")))]
 426   "TARGET_SIMD"
 427   "and\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
 428   [(set_attr "type" "neon_logic<q>")]
 429 )
 430
 431 (define_insn "ior<mode>3"
 432   [(set (match_operand:VDQ 0 "register_operand" "=w")
 433         (ior:VDQ (match_operand:VDQ 1 "register_operand" "w")
 434                  (match_operand:VDQ 2 "register_operand" "w")))]
 435   "TARGET_SIMD"
 436   "orr\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
 437   [(set_attr "type" "neon_logic<q>")]
 438 )
 439
 440 (define_insn "xor<mode>3"
 441   [(set (match_operand:VDQ 0 "register_operand" "=w")
 442         (xor:VDQ (match_operand:VDQ 1 "register_operand" "w")
 443                  (match_operand:VDQ 2 "register_operand" "w")))]
 444   "TARGET_SIMD"
 445   "eor\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
 446   [(set_attr "type" "neon_logic<q>")]
 447 )
 448
 449 (define_insn "one_cmpl<mode>2"
 450   [(set (match_operand:VDQ 0 "register_operand" "=w")
 451         (not:VDQ (match_operand:VDQ 1 "register_operand" "w")))]
 452   "TARGET_SIMD"
 453   "not\t%0.<Vbtype>, %1.<Vbtype>"
 454   [(set_attr "type" "neon_logic<q>")]
 455 )
 456
 457 (define_insn "aarch64_simd_vec_set<mode>"
 458   [(set (match_operand:VQ_S 0 "register_operand" "=w,w")
 459         (vec_merge:VQ_S
 460             (vec_duplicate:VQ_S
 461                 (match_operand:<VEL> 1 "register_operand" "r,w"))
 462             (match_operand:VQ_S 3 "register_operand" "0,0")
 463             (match_operand:SI 2 "immediate_operand" "i,i")))]
 464   "TARGET_SIMD"
 465   {
 466    int elt = ENDIAN_LANE_N (<MODE>mode, exact_log2 (INTVAL (operands[2])));
 467    operands[2] = GEN_INT ((HOST_WIDE_INT) 1 << elt);
 468    switch (which_alternative)
 469      {
 470      case 0:
 471         return "ins\\t%0.<Vetype>[%p2], %w1";
 472      case 1:
 473         return "ins\\t%0.<Vetype>[%p2], %1.<Vetype>[0]";
 474      default:
 475         gcc_unreachable ();
 476      }
 477   }
 478   [(set_attr "type" "neon_from_gp<q>, neon_ins<q>")]
 479 )
 480
 481 (define_insn "aarch64_simd_lshr<mode>"
 482  [(set (match_operand:VDQ 0 "register_operand" "=w")
 483        (lshiftrt:VDQ (match_operand:VDQ 1 "register_operand" "w")
 484                      (match_operand:VDQ  2 "aarch64_simd_rshift_imm" "Dr")))]
 485  "TARGET_SIMD"
 486  "ushr\t%0.<Vtype>, %1.<Vtype>, %2"
 487   [(set_attr "type" "neon_shift_imm<q>")]
 488 )
 489
 490 (define_insn "aarch64_simd_ashr<mode>"
 491  [(set (match_operand:VDQ 0 "register_operand" "=w")
 492        (ashiftrt:VDQ (match_operand:VDQ 1 "register_operand" "w")
 493                      (match_operand:VDQ  2 "aarch64_simd_rshift_imm" "Dr")))]
 494  "TARGET_SIMD"
 495  "sshr\t%0.<Vtype>, %1.<Vtype>, %2"
 496   [(set_attr "type" "neon_shift_imm<q>")]
 497 )
 498
 499 (define_insn "aarch64_simd_imm_shl<mode>"
 500  [(set (match_operand:VDQ 0 "register_operand" "=w")
 501        (ashift:VDQ (match_operand:VDQ 1 "register_operand" "w")
 502                    (match_operand:VDQ  2 "aarch64_simd_lshift_imm" "Dl")))]
 503  "TARGET_SIMD"
 504   "shl\t%0.<Vtype>, %1.<Vtype>, %2"
 505   [(set_attr "type" "neon_shift_imm<q>")]
 506 )
 507
 508 (define_insn "aarch64_simd_reg_sshl<mode>"
 509  [(set (match_operand:VDQ 0 "register_operand" "=w")
 510        (ashift:VDQ (match_operand:VDQ 1 "register_operand" "w")
 511                    (match_operand:VDQ 2 "register_operand" "w")))]
 512  "TARGET_SIMD"
 513  "sshl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
 514   [(set_attr "type" "neon_shift_reg<q>")]
 515 )
 516
 517 (define_insn "aarch64_simd_reg_shl<mode>_unsigned"
 518  [(set (match_operand:VDQ 0 "register_operand" "=w")
 519        (unspec:VDQ [(match_operand:VDQ 1 "register_operand" "w")
 520                     (match_operand:VDQ 2 "register_operand" "w")]
 521                    UNSPEC_ASHIFT_UNSIGNED))]
 522  "TARGET_SIMD"
 523  "ushl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
 524   [(set_attr "type" "neon_shift_reg<q>")]
 525 )
 526
 527 (define_insn "aarch64_simd_reg_shl<mode>_signed"
 528  [(set (match_operand:VDQ 0 "register_operand" "=w")
 529        (unspec:VDQ [(match_operand:VDQ 1 "register_operand" "w")
 530                     (match_operand:VDQ 2 "register_operand" "w")]
 531                    UNSPEC_ASHIFT_SIGNED))]
 532  "TARGET_SIMD"
 533  "sshl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
 534   [(set_attr "type" "neon_shift_reg<q>")]
 535 )
 536
 537 (define_expand "ashl<mode>3"
 538   [(match_operand:VDQ 0 "register_operand" "")
 539    (match_operand:VDQ 1 "register_operand" "")
 540    (match_operand:SI  2 "general_operand" "")]
 541  "TARGET_SIMD"
 542 {
 543   int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
 544   int shift_amount;
 545
 546   if (CONST_INT_P (operands[2]))
 547     {
 548       shift_amount = INTVAL (operands[2]);
 549       if (shift_amount >= 0 && shift_amount < bit_width)
 550         {
 551           rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
 552                                                        shift_amount);
 553           emit_insn (gen_aarch64_simd_imm_shl<mode> (operands[0],
 554                                                      operands[1],
 555                                                      tmp));
 556           DONE;
 557         }
 558       else
 559         {
 560           operands[2] = force_reg (SImode, operands[2]);
 561         }
 562     }
 563   else if (MEM_P (operands[2]))
 564     {
 565       operands[2] = force_reg (SImode, operands[2]);
 566     }
 567
 568   if (REG_P (operands[2]))
 569     {
 570       rtx tmp = gen_reg_rtx (<MODE>mode);
 571       emit_insn (gen_aarch64_simd_dup<mode> (tmp,
 572                                              convert_to_mode (<VEL>mode,
 573                                                               operands[2],
 574                                                               0)));
 575       emit_insn (gen_aarch64_simd_reg_sshl<mode> (operands[0], operands[1],
 576                                                   tmp));
 577       DONE;
 578     }
 579   else
 580     FAIL;
 581 }
 582 )
 583
 584 (define_expand "lshr<mode>3"
 585   [(match_operand:VDQ 0 "register_operand" "")
 586    (match_operand:VDQ 1 "register_operand" "")
 587    (match_operand:SI  2 "general_operand" "")]
 588  "TARGET_SIMD"
 589 {
 590   int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
 591   int shift_amount;
 592
 593   if (CONST_INT_P (operands[2]))
 594     {
 595       shift_amount = INTVAL (operands[2]);
 596       if (shift_amount > 0 && shift_amount <= bit_width)
 597         {
 598           rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
 599                                                        shift_amount);
 600           emit_insn (gen_aarch64_simd_lshr<mode> (operands[0],
 601                                                   operands[1],
 602                                                   tmp));
 603           DONE;
 604         }
 605       else
 606         operands[2] = force_reg (SImode, operands[2]);
 607     }
 608   else if (MEM_P (operands[2]))
 609     {
 610       operands[2] = force_reg (SImode, operands[2]);
 611     }
 612
 613   if (REG_P (operands[2]))
 614     {
 615       rtx tmp = gen_reg_rtx (SImode);
 616       rtx tmp1 = gen_reg_rtx (<MODE>mode);
 617       emit_insn (gen_negsi2 (tmp, operands[2]));
 618       emit_insn (gen_aarch64_simd_dup<mode> (tmp1,
 619                                              convert_to_mode (<VEL>mode,
 620                                                               tmp, 0)));
 621       emit_insn (gen_aarch64_simd_reg_shl<mode>_unsigned (operands[0],
 622                                                           operands[1],
 623                                                           tmp1));
 624       DONE;
 625     }
 626   else
 627     FAIL;
 628 }
 629 )
 630
 631 (define_expand "ashr<mode>3"
 632   [(match_operand:VDQ 0 "register_operand" "")
 633    (match_operand:VDQ 1 "register_operand" "")
 634    (match_operand:SI  2 "general_operand" "")]
 635  "TARGET_SIMD"
 636 {
 637   int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
 638   int shift_amount;
 639
 640   if (CONST_INT_P (operands[2]))
 641     {
 642       shift_amount = INTVAL (operands[2]);
 643       if (shift_amount > 0 && shift_amount <= bit_width)
 644         {
 645           rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
 646                                                        shift_amount);
 647           emit_insn (gen_aarch64_simd_ashr<mode> (operands[0],
 648                                                   operands[1],
 649                                                   tmp));
 650           DONE;
 651         }
 652       else
 653         operands[2] = force_reg (SImode, operands[2]);
 654     }
 655   else if (MEM_P (operands[2]))
 656     {
 657       operands[2] = force_reg (SImode, operands[2]);
 658     }
 659
 660   if (REG_P (operands[2]))
 661     {
 662       rtx tmp = gen_reg_rtx (SImode);
 663       rtx tmp1 = gen_reg_rtx (<MODE>mode);
 664       emit_insn (gen_negsi2 (tmp, operands[2]));
 665       emit_insn (gen_aarch64_simd_dup<mode> (tmp1,
 666                                              convert_to_mode (<VEL>mode,
 667                                                               tmp, 0)));
 668       emit_insn (gen_aarch64_simd_reg_shl<mode>_signed (operands[0],
 669                                                         operands[1],
 670                                                         tmp1));
 671       DONE;
 672     }
 673   else
 674     FAIL;
 675 }
 676 )
 677
 678 (define_expand "vashl<mode>3"
 679  [(match_operand:VDQ 0 "register_operand" "")
 680   (match_operand:VDQ 1 "register_operand" "")
 681   (match_operand:VDQ 2 "register_operand" "")]
 682  "TARGET_SIMD"
 683 {
 684   emit_insn (gen_aarch64_simd_reg_sshl<mode> (operands[0], operands[1],
 685                                               operands[2]));
 686   DONE;
 687 })
 688
 689 ;; Using mode VQ_S as there is no V2DImode neg!
 690 ;; Negating individual lanes most certainly offsets the
 691 ;; gain from vectorization.
 692 (define_expand "vashr<mode>3"
 693  [(match_operand:VQ_S 0 "register_operand" "")
 694   (match_operand:VQ_S 1 "register_operand" "")
 695   (match_operand:VQ_S 2 "register_operand" "")]
 696  "TARGET_SIMD"
 697 {
 698   rtx neg = gen_reg_rtx (<MODE>mode);
 699   emit (gen_neg<mode>2 (neg, operands[2]));
 700   emit_insn (gen_aarch64_simd_reg_shl<mode>_signed (operands[0], operands[1],
 701                                                     neg));
 702   DONE;
 703 })
 704
 705 ;; DI vector shift
 706 (define_expand "aarch64_ashr_simddi"
 707   [(match_operand:DI 0 "register_operand" "=w")
 708    (match_operand:DI 1 "register_operand" "w")
 709    (match_operand:SI 2 "aarch64_shift_imm64_di" "")]
 710   "TARGET_SIMD"
 711   {
 712     if (INTVAL (operands[2]) == 64)
 713       emit_insn (gen_aarch64_sshr_simddi (operands[0], operands[1]));
 714     else
 715       emit_insn (gen_ashrdi3 (operands[0], operands[1], operands[2]));
 716     DONE;
 717   }
 718 )
 719
 720 ;; SIMD shift by 64.  This pattern is a special case as standard pattern does
 721 ;; not handle NEON shifts by 64.
 722 (define_insn "aarch64_sshr_simddi"
 723   [(set (match_operand:DI 0 "register_operand" "=w")
 724         (unspec:DI
 725           [(match_operand:DI 1 "register_operand" "w")] UNSPEC_SSHR64))]
 726   "TARGET_SIMD"
 727   "sshr\t%d0, %d1, 64"
 728   [(set_attr "type" "neon_shift_imm")]
 729 )
 730
 731 (define_expand "vlshr<mode>3"
 732  [(match_operand:VQ_S 0 "register_operand" "")
 733   (match_operand:VQ_S 1 "register_operand" "")
 734   (match_operand:VQ_S 2 "register_operand" "")]
 735  "TARGET_SIMD"
 736 {
 737   rtx neg = gen_reg_rtx (<MODE>mode);
 738   emit (gen_neg<mode>2 (neg, operands[2]));
 739   emit_insn (gen_aarch64_simd_reg_shl<mode>_unsigned (operands[0], operands[1],
 740                                                       neg));
 741   DONE;
 742 })
 743
 744 (define_expand "aarch64_lshr_simddi"
 745   [(match_operand:DI 0 "register_operand" "=w")
 746    (match_operand:DI 1 "register_operand" "w")
 747    (match_operand:SI 2 "aarch64_shift_imm64_di" "")]
 748   "TARGET_SIMD"
 749   {
 750     if (INTVAL (operands[2]) == 64)
 751       emit_insn (gen_aarch64_ushr_simddi (operands[0], operands[1]));
 752     else
 753       emit_insn (gen_lshrdi3 (operands[0], operands[1], operands[2]));
 754     DONE;
 755   }
 756 )
 757
 758 ;; SIMD shift by 64.  This pattern is a special case as standard pattern does
 759 ;; not handle NEON shifts by 64.
 760 (define_insn "aarch64_ushr_simddi"
 761   [(set (match_operand:DI 0 "register_operand" "=w")
 762         (unspec:DI
 763           [(match_operand:DI 1 "register_operand" "w")] UNSPEC_USHR64))]
 764   "TARGET_SIMD"
 765   "ushr\t%d0, %d1, 64"
 766   [(set_attr "type" "neon_shift_imm")]
 767 )
 768
 769 (define_expand "vec_set<mode>"
 770   [(match_operand:VQ_S 0 "register_operand")
 771    (match_operand:<VEL> 1 "register_operand")
 772    (match_operand:SI 2 "immediate_operand")]
 773   "TARGET_SIMD"
 774   {
 775     HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
 776     emit_insn (gen_aarch64_simd_vec_set<mode> (operands[0], operands[1],
 777                                             GEN_INT (elem), operands[0]));
 778     DONE;
 779   }
 780 )
 781
 782 (define_insn "aarch64_simd_vec_setv2di"
 783   [(set (match_operand:V2DI 0 "register_operand" "=w,w")
 784         (vec_merge:V2DI
 785             (vec_duplicate:V2DI
 786                 (match_operand:DI 1 "register_operand" "r,w"))
 787             (match_operand:V2DI 3 "register_operand" "0,0")
 788             (match_operand:SI 2 "immediate_operand" "i,i")))]
 789   "TARGET_SIMD"
 790   {
 791     int elt = ENDIAN_LANE_N (V2DImode, exact_log2 (INTVAL (operands[2])));
 792     operands[2] = GEN_INT ((HOST_WIDE_INT) 1 << elt);
 793     switch (which_alternative)
 794       {
 795       case 0:
 796         return "ins\\t%0.d[%p2], %1";
 797       case 1:
 798         return "ins\\t%0.d[%p2], %1.d[0]";
 799       default:
 800         gcc_unreachable ();
 801       }
 802   }
 803   [(set_attr "type" "neon_from_gp, neon_ins_q")]
 804 )
 805
 806 (define_expand "vec_setv2di"
 807   [(match_operand:V2DI 0 "register_operand")
 808    (match_operand:DI 1 "register_operand")
 809    (match_operand:SI 2 "immediate_operand")]
 810   "TARGET_SIMD"
 811   {
 812     HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
 813     emit_insn (gen_aarch64_simd_vec_setv2di (operands[0], operands[1],
 814                                           GEN_INT (elem), operands[0]));
 815     DONE;
 816   }
 817 )
 818
 819 (define_insn "aarch64_simd_vec_set<mode>"
 820   [(set (match_operand:VDQF 0 "register_operand" "=w")
 821         (vec_merge:VDQF
 822             (vec_duplicate:VDQF
 823                 (match_operand:<VEL> 1 "register_operand" "w"))
 824             (match_operand:VDQF 3 "register_operand" "0")
 825             (match_operand:SI 2 "immediate_operand" "i")))]
 826   "TARGET_SIMD"
 827   {
 828     int elt = ENDIAN_LANE_N (<MODE>mode, exact_log2 (INTVAL (operands[2])));
 829
 830     operands[2] = GEN_INT ((HOST_WIDE_INT)1 << elt);
 831     return "ins\t%0.<Vetype>[%p2], %1.<Vetype>[0]";
 832   }
 833   [(set_attr "type" "neon_ins<q>")]
 834 )
 835
 836 (define_expand "vec_set<mode>"
 837   [(match_operand:VDQF 0 "register_operand" "+w")
 838    (match_operand:<VEL> 1 "register_operand" "w")
 839    (match_operand:SI 2 "immediate_operand" "")]
 840   "TARGET_SIMD"
 841   {
 842     HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
 843     emit_insn (gen_aarch64_simd_vec_set<mode> (operands[0], operands[1],
 844                                           GEN_INT (elem), operands[0]));
 845     DONE;
 846   }
 847 )
 848
 849
 850 (define_insn "aarch64_mla<mode>"
 851  [(set (match_operand:VQ_S 0 "register_operand" "=w")
 852        (plus:VQ_S (mult:VQ_S (match_operand:VQ_S 2 "register_operand" "w")
 853                              (match_operand:VQ_S 3 "register_operand" "w"))
 854                   (match_operand:VQ_S 1 "register_operand" "0")))]
 855  "TARGET_SIMD"
 856  "mla\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
 857   [(set_attr "type" "neon_mla_<Vetype><q>")]
 858 )
 859
 860 (define_insn "*aarch64_mla_elt<mode>"
 861  [(set (match_operand:VDQHS 0 "register_operand" "=w")
 862        (plus:VDQHS
 863          (mult:VDQHS
 864            (vec_duplicate:VDQHS
 865               (vec_select:<VEL>
 866                 (match_operand:VDQHS 1 "register_operand" "<h_con>")
 867                   (parallel [(match_operand:SI 2 "immediate_operand")])))
 868            (match_operand:VDQHS 3 "register_operand" "w"))
 869          (match_operand:VDQHS 4 "register_operand" "0")))]
 870  "TARGET_SIMD"
 871   {
 872     operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
 873     return "mla\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
 874   }
 875   [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
 876 )
 877
 878 (define_insn "*aarch64_mla_elt_<vswap_width_name><mode>"
 879  [(set (match_operand:VDQHS 0 "register_operand" "=w")
 880        (plus:VDQHS
 881          (mult:VDQHS
 882            (vec_duplicate:VDQHS
 883               (vec_select:<VEL>
 884                 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
 885                   (parallel [(match_operand:SI 2 "immediate_operand")])))
 886            (match_operand:VDQHS 3 "register_operand" "w"))
 887          (match_operand:VDQHS 4 "register_operand" "0")))]
 888  "TARGET_SIMD"
 889   {
 890     operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
 891                                           INTVAL (operands[2])));
 892     return "mla\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
 893   }
 894   [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
 895 )
 896
 897 (define_insn "aarch64_mls<mode>"
 898  [(set (match_operand:VQ_S 0 "register_operand" "=w")
 899        (minus:VQ_S (match_operand:VQ_S 1 "register_operand" "0")
 900                    (mult:VQ_S (match_operand:VQ_S 2 "register_operand" "w")
 901                               (match_operand:VQ_S 3 "register_operand" "w"))))]
 902  "TARGET_SIMD"
 903  "mls\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
 904   [(set_attr "type" "neon_mla_<Vetype><q>")]
 905 )
 906
 907 (define_insn "*aarch64_mls_elt<mode>"
 908  [(set (match_operand:VDQHS 0 "register_operand" "=w")
 909        (minus:VDQHS
 910          (match_operand:VDQHS 4 "register_operand" "0")
 911          (mult:VDQHS
 912            (vec_duplicate:VDQHS
 913               (vec_select:<VEL>
 914                 (match_operand:VDQHS 1 "register_operand" "<h_con>")
 915                   (parallel [(match_operand:SI 2 "immediate_operand")])))
 916            (match_operand:VDQHS 3 "register_operand" "w"))))]
 917  "TARGET_SIMD"
 918   {
 919     operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
 920     return "mls\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
 921   }
 922   [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
 923 )
 924
 925 (define_insn "*aarch64_mls_elt_<vswap_width_name><mode>"
 926  [(set (match_operand:VDQHS 0 "register_operand" "=w")
 927        (minus:VDQHS
 928          (match_operand:VDQHS 4 "register_operand" "0")
 929          (mult:VDQHS
 930            (vec_duplicate:VDQHS
 931               (vec_select:<VEL>
 932                 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
 933                   (parallel [(match_operand:SI 2 "immediate_operand")])))
 934            (match_operand:VDQHS 3 "register_operand" "w"))))]
 935  "TARGET_SIMD"
 936   {
 937     operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
 938                                           INTVAL (operands[2])));
 939     return "mls\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
 940   }
 941   [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
 942 )
 943
 944 ;; Max/Min operations.
 945 (define_insn "<su><maxmin><mode>3"
 946  [(set (match_operand:VQ_S 0 "register_operand" "=w")
 947        (MAXMIN:VQ_S (match_operand:VQ_S 1 "register_operand" "w")
 948                     (match_operand:VQ_S 2 "register_operand" "w")))]
 949  "TARGET_SIMD"
 950  "<su><maxmin>\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
 951   [(set_attr "type" "neon_minmax<q>")]
 952 )
 953
 954 ;; vec_concat gives a new vector with the low elements from operand 1, and
 955 ;; the high elements from operand 2.  That is to say, given op1 = { a, b }
 956 ;; op2 = { c, d }, vec_concat (op1, op2) = { a, b, c, d }.
 957 ;; What that means, is that the RTL descriptions of the below patterns
 958 ;; need to change depending on endianness.
 959
 960 ;; Move to the low architectural bits of the register.
 961 ;; On little-endian this is { operand, zeroes }
 962 ;; On big-endian this is { zeroes, operand }
 963
 964 (define_insn "move_lo_quad_internal_<mode>"
 965   [(set (match_operand:VQ_NO2E 0 "register_operand" "=w,w,w")
 966         (vec_concat:VQ_NO2E
 967           (match_operand:<VHALF> 1 "register_operand" "w,r,r")
 968           (vec_duplicate:<VHALF> (const_int 0))))]
 969   "TARGET_SIMD && !BYTES_BIG_ENDIAN"
 970   "@
 971    dup\\t%d0, %1.d[0]
 972    fmov\\t%d0, %1
 973    dup\\t%d0, %1"
 974   [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
 975    (set_attr "simd" "yes,*,yes")
 976    (set_attr "fp" "*,yes,*")
 977    (set_attr "length" "4")]
 978 )
 979
 980 (define_insn "move_lo_quad_internal_<mode>"
 981   [(set (match_operand:VQ_2E 0 "register_operand" "=w,w,w")
 982         (vec_concat:VQ_2E
 983           (match_operand:<VHALF> 1 "register_operand" "w,r,r")
 984           (const_int 0)))]
 985   "TARGET_SIMD && !BYTES_BIG_ENDIAN"
 986   "@
 987    dup\\t%d0, %1.d[0]
 988    fmov\\t%d0, %1
 989    dup\\t%d0, %1"
 990   [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
 991    (set_attr "simd" "yes,*,yes")
 992    (set_attr "fp" "*,yes,*")
 993    (set_attr "length" "4")]
 994 )
 995
 996 (define_insn "move_lo_quad_internal_be_<mode>"
 997   [(set (match_operand:VQ_NO2E 0 "register_operand" "=w,w,w")
 998         (vec_concat:VQ_NO2E
 999           (vec_duplicate:<VHALF> (const_int 0))
1000           (match_operand:<VHALF> 1 "register_operand" "w,r,r")))]
1001   "TARGET_SIMD && BYTES_BIG_ENDIAN"
1002   "@
1003    dup\\t%d0, %1.d[0]
1004    fmov\\t%d0, %1
1005    dup\\t%d0, %1"
1006   [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
1007    (set_attr "simd" "yes,*,yes")
1008    (set_attr "fp" "*,yes,*")
1009    (set_attr "length" "4")]
1010 )
1011
1012 (define_insn "move_lo_quad_internal_be_<mode>"
1013   [(set (match_operand:VQ_2E 0 "register_operand" "=w,w,w")
1014         (vec_concat:VQ_2E
1015           (const_int 0)
1016           (match_operand:<VHALF> 1 "register_operand" "w,r,r")))]
1017   "TARGET_SIMD && BYTES_BIG_ENDIAN"
1018   "@
1019    dup\\t%d0, %1.d[0]
1020    fmov\\t%d0, %1
1021    dup\\t%d0, %1"
1022   [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
1023    (set_attr "simd" "yes,*,yes")
1024    (set_attr "fp" "*,yes,*")
1025    (set_attr "length" "4")]
1026 )
1027
1028 (define_expand "move_lo_quad_<mode>"
1029   [(match_operand:VQ 0 "register_operand")
1030    (match_operand:VQ 1 "register_operand")]
1031   "TARGET_SIMD"
1032 {
1033   if (BYTES_BIG_ENDIAN)
1034     emit_insn (gen_move_lo_quad_internal_be_<mode> (operands[0], operands[1]));
1035   else
1036     emit_insn (gen_move_lo_quad_internal_<mode> (operands[0], operands[1]));
1037   DONE;
1038 }
1039 )
1040
1041 ;; Move operand1 to the high architectural bits of the register, keeping
1042 ;; the low architectural bits of operand2.
1043 ;; For little-endian this is { operand2, operand1 }
1044 ;; For big-endian this is { operand1, operand2 }
1045
1046 (define_insn "aarch64_simd_move_hi_quad_<mode>"
1047   [(set (match_operand:VQ 0 "register_operand" "+w,w")
1048         (vec_concat:VQ
1049           (vec_select:<VHALF>
1050                 (match_dup 0)
1051                 (match_operand:VQ 2 "vect_par_cnst_lo_half" ""))
1052           (match_operand:<VHALF> 1 "register_operand" "w,r")))]
1053   "TARGET_SIMD && !BYTES_BIG_ENDIAN"
1054   "@
1055    ins\\t%0.d[1], %1.d[0]
1056    ins\\t%0.d[1], %1"
1057   [(set_attr "type" "neon_ins")]
1058 )
1059
1060 (define_insn "aarch64_simd_move_hi_quad_be_<mode>"
1061   [(set (match_operand:VQ 0 "register_operand" "+w,w")
1062         (vec_concat:VQ
1063           (match_operand:<VHALF> 1 "register_operand" "w,r")
1064           (vec_select:<VHALF>
1065                 (match_dup 0)
1066                 (match_operand:VQ 2 "vect_par_cnst_lo_half" ""))))]
1067   "TARGET_SIMD && BYTES_BIG_ENDIAN"
1068   "@
1069    ins\\t%0.d[1], %1.d[0]
1070    ins\\t%0.d[1], %1"
1071   [(set_attr "type" "neon_ins")]
1072 )
1073
1074 (define_expand "move_hi_quad_<mode>"
1075  [(match_operand:VQ 0 "register_operand" "")
1076   (match_operand:<VHALF> 1 "register_operand" "")]
1077  "TARGET_SIMD"
1078 {
1079   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1080   if (BYTES_BIG_ENDIAN)
1081     emit_insn (gen_aarch64_simd_move_hi_quad_be_<mode> (operands[0],
1082                     operands[1], p));
1083   else
1084     emit_insn (gen_aarch64_simd_move_hi_quad_<mode> (operands[0],
1085                     operands[1], p));
1086   DONE;
1087 })
1088
1089 ;; Narrowing operations.
1090
1091 ;; For doubles.
1092 (define_insn "aarch64_simd_vec_pack_trunc_<mode>"
1093  [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
1094        (truncate:<VNARROWQ> (match_operand:VQN 1 "register_operand" "w")))]
1095  "TARGET_SIMD"
1096  "xtn\\t%0.<Vntype>, %1.<Vtype>"
1097   [(set_attr "type" "neon_shift_imm_narrow_q")]
1098 )
1099
1100 (define_expand "vec_pack_trunc_<mode>"
1101  [(match_operand:<VNARROWD> 0 "register_operand" "")
1102   (match_operand:VDN 1 "register_operand" "")
1103   (match_operand:VDN 2 "register_operand" "")]
1104  "TARGET_SIMD"
1105 {
1106   rtx tempreg = gen_reg_rtx (<VDBL>mode);
1107   int lo = BYTES_BIG_ENDIAN ? 2 : 1;
1108   int hi = BYTES_BIG_ENDIAN ? 1 : 2;
1109
1110   emit_insn (gen_move_lo_quad_<Vdbl> (tempreg, operands[lo]));
1111   emit_insn (gen_move_hi_quad_<Vdbl> (tempreg, operands[hi]));
1112   emit_insn (gen_aarch64_simd_vec_pack_trunc_<Vdbl> (operands[0], tempreg));
1113   DONE;
1114 })
1115
1116 ;; For quads.
1117
1118 (define_insn "vec_pack_trunc_<mode>"
1119  [(set (match_operand:<VNARROWQ2> 0 "register_operand" "=&w")
1120        (vec_concat:<VNARROWQ2>
1121          (truncate:<VNARROWQ> (match_operand:VQN 1 "register_operand" "w"))
1122          (truncate:<VNARROWQ> (match_operand:VQN 2 "register_operand" "w"))))]
1123  "TARGET_SIMD"
1124  {
1125    if (BYTES_BIG_ENDIAN)
1126      return "xtn\\t%0.<Vntype>, %2.<Vtype>\;xtn2\\t%0.<V2ntype>, %1.<Vtype>";
1127    else
1128      return "xtn\\t%0.<Vntype>, %1.<Vtype>\;xtn2\\t%0.<V2ntype>, %2.<Vtype>";
1129  }
1130   [(set_attr "type" "multiple")
1131    (set_attr "length" "8")]
1132 )
1133
1134 ;; Widening operations.
1135
1136 (define_insn "aarch64_simd_vec_unpack<su>_lo_<mode>"
1137   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1138         (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1139                                (match_operand:VQW 1 "register_operand" "w")
1140                                (match_operand:VQW 2 "vect_par_cnst_lo_half" "")
1141                             )))]
1142   "TARGET_SIMD"
1143   "<su>shll %0.<Vwtype>, %1.<Vhalftype>, 0"
1144   [(set_attr "type" "neon_shift_imm_long")]
1145 )
1146
1147 (define_insn "aarch64_simd_vec_unpack<su>_hi_<mode>"
1148   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1149         (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1150                                (match_operand:VQW 1 "register_operand" "w")
1151                                (match_operand:VQW 2 "vect_par_cnst_hi_half" "")
1152                             )))]
1153   "TARGET_SIMD"
1154   "<su>shll2 %0.<Vwtype>, %1.<Vtype>, 0"
1155   [(set_attr "type" "neon_shift_imm_long")]
1156 )
1157
1158 (define_expand "vec_unpack<su>_hi_<mode>"
1159   [(match_operand:<VWIDE> 0 "register_operand" "")
1160    (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand"))]
1161   "TARGET_SIMD"
1162   {
1163     rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
1164     emit_insn (gen_aarch64_simd_vec_unpack<su>_hi_<mode> (operands[0],
1165                                                           operands[1], p));
1166     DONE;
1167   }
1168 )
1169
1170 (define_expand "vec_unpack<su>_lo_<mode>"
1171   [(match_operand:<VWIDE> 0 "register_operand" "")
1172    (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))]
1173   "TARGET_SIMD"
1174   {
1175     rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1176     emit_insn (gen_aarch64_simd_vec_unpack<su>_lo_<mode> (operands[0],
1177                                                           operands[1], p));
1178     DONE;
1179   }
1180 )
1181
1182 ;; Widening arithmetic.
1183
1184 (define_insn "*aarch64_<su>mlal_lo<mode>"
1185   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1186         (plus:<VWIDE>
1187           (mult:<VWIDE>
1188               (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1189                  (match_operand:VQW 2 "register_operand" "w")
1190                  (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1191               (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1192                  (match_operand:VQW 4 "register_operand" "w")
1193                  (match_dup 3))))
1194           (match_operand:<VWIDE> 1 "register_operand" "0")))]
1195   "TARGET_SIMD"
1196   "<su>mlal\t%0.<Vwtype>, %2.<Vhalftype>, %4.<Vhalftype>"
1197   [(set_attr "type" "neon_mla_<Vetype>_long")]
1198 )
1199
1200 (define_insn "*aarch64_<su>mlal_hi<mode>"
1201   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1202         (plus:<VWIDE>
1203           (mult:<VWIDE>
1204               (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1205                  (match_operand:VQW 2 "register_operand" "w")
1206                  (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1207               (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1208                  (match_operand:VQW 4 "register_operand" "w")
1209                  (match_dup 3))))
1210           (match_operand:<VWIDE> 1 "register_operand" "0")))]
1211   "TARGET_SIMD"
1212   "<su>mlal2\t%0.<Vwtype>, %2.<Vtype>, %4.<Vtype>"
1213   [(set_attr "type" "neon_mla_<Vetype>_long")]
1214 )
1215
1216 (define_insn "*aarch64_<su>mlsl_lo<mode>"
1217   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1218         (minus:<VWIDE>
1219           (match_operand:<VWIDE> 1 "register_operand" "0")
1220           (mult:<VWIDE>
1221               (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1222                  (match_operand:VQW 2 "register_operand" "w")
1223                  (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1224               (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1225                  (match_operand:VQW 4 "register_operand" "w")
1226                  (match_dup 3))))))]
1227   "TARGET_SIMD"
1228   "<su>mlsl\t%0.<Vwtype>, %2.<Vhalftype>, %4.<Vhalftype>"
1229   [(set_attr "type" "neon_mla_<Vetype>_long")]
1230 )
1231
1232 (define_insn "*aarch64_<su>mlsl_hi<mode>"
1233   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1234         (minus:<VWIDE>
1235           (match_operand:<VWIDE> 1 "register_operand" "0")
1236           (mult:<VWIDE>
1237               (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1238                  (match_operand:VQW 2 "register_operand" "w")
1239                  (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1240               (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1241                  (match_operand:VQW 4 "register_operand" "w")
1242                  (match_dup 3))))))]
1243   "TARGET_SIMD"
1244   "<su>mlsl2\t%0.<Vwtype>, %2.<Vtype>, %4.<Vtype>"
1245   [(set_attr "type" "neon_mla_<Vetype>_long")]
1246 )
1247
1248 (define_insn "*aarch64_<su>mlal<mode>"
1249   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1250         (plus:<VWIDE>
1251           (mult:<VWIDE>
1252             (ANY_EXTEND:<VWIDE>
1253               (match_operand:VDW 1 "register_operand" "w"))
1254             (ANY_EXTEND:<VWIDE>
1255               (match_operand:VDW 2 "register_operand" "w")))
1256           (match_operand:<VWIDE> 3 "register_operand" "0")))]
1257   "TARGET_SIMD"
1258   "<su>mlal\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
1259   [(set_attr "type" "neon_mla_<Vetype>_long")]
1260 )
1261
1262 (define_insn "*aarch64_<su>mlsl<mode>"
1263   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1264         (minus:<VWIDE>
1265           (match_operand:<VWIDE> 1 "register_operand" "0")
1266           (mult:<VWIDE>
1267             (ANY_EXTEND:<VWIDE>
1268               (match_operand:VDW 2 "register_operand" "w"))
1269             (ANY_EXTEND:<VWIDE>
1270               (match_operand:VDW 3 "register_operand" "w")))))]
1271   "TARGET_SIMD"
1272   "<su>mlsl\t%0.<Vwtype>, %2.<Vtype>, %3.<Vtype>"
1273   [(set_attr "type" "neon_mla_<Vetype>_long")]
1274 )
1275
1276 (define_insn "aarch64_simd_vec_<su>mult_lo_<mode>"
1277  [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1278        (mult:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1279                            (match_operand:VQW 1 "register_operand" "w")
1280                            (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1281                      (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1282                            (match_operand:VQW 2 "register_operand" "w")
1283                            (match_dup 3)))))]
1284   "TARGET_SIMD"
1285   "<su>mull\\t%0.<Vwtype>, %1.<Vhalftype>, %2.<Vhalftype>"
1286   [(set_attr "type" "neon_mul_<Vetype>_long")]
1287 )
1288
1289 (define_expand "vec_widen_<su>mult_lo_<mode>"
1290   [(match_operand:<VWIDE> 0 "register_operand" "")
1291    (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))
1292    (ANY_EXTEND:<VWIDE> (match_operand:VQW 2 "register_operand" ""))]
1293  "TARGET_SIMD"
1294  {
1295    rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1296    emit_insn (gen_aarch64_simd_vec_<su>mult_lo_<mode> (operands[0],
1297                                                        operands[1],
1298                                                        operands[2], p));
1299    DONE;
1300  }
1301 )
1302
1303 (define_insn "aarch64_simd_vec_<su>mult_hi_<mode>"
1304  [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1305       (mult:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1306                             (match_operand:VQW 1 "register_operand" "w")
1307                             (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1308                     (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1309                             (match_operand:VQW 2 "register_operand" "w")
1310                             (match_dup 3)))))]
1311   "TARGET_SIMD"
1312   "<su>mull2\\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
1313   [(set_attr "type" "neon_mul_<Vetype>_long")]
1314 )
1315
1316 (define_expand "vec_widen_<su>mult_hi_<mode>"
1317   [(match_operand:<VWIDE> 0 "register_operand" "")
1318    (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))
1319    (ANY_EXTEND:<VWIDE> (match_operand:VQW 2 "register_operand" ""))]
1320  "TARGET_SIMD"
1321  {
1322    rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
1323    emit_insn (gen_aarch64_simd_vec_<su>mult_hi_<mode> (operands[0],
1324                                                        operands[1],
1325                                                        operands[2], p));
1326    DONE;
1327
1328  }
1329 )
1330
1331 ;; FP vector operations.
1332 ;; AArch64 AdvSIMD supports single-precision (32-bit) and
1333 ;; double-precision (64-bit) floating-point data types and arithmetic as
1334 ;; defined by the IEEE 754-2008 standard.  This makes them vectorizable
1335 ;; without the need for -ffast-math or -funsafe-math-optimizations.
1336 ;;
1337 ;; Floating-point operations can raise an exception.  Vectorizing such
1338 ;; operations are safe because of reasons explained below.
1339 ;;
1340 ;; ARMv8 permits an extension to enable trapped floating-point
1341 ;; exception handling, however this is an optional feature.  In the
1342 ;; event of a floating-point exception being raised by vectorised
1343 ;; code then:
1344 ;; 1.  If trapped floating-point exceptions are available, then a trap
1345 ;;     will be taken when any lane raises an enabled exception.  A trap
1346 ;;     handler may determine which lane raised the exception.
1347 ;; 2.  Alternatively a sticky exception flag is set in the
1348 ;;     floating-point status register (FPSR).  Software may explicitly
1349 ;;     test the exception flags, in which case the tests will either
1350 ;;     prevent vectorisation, allowing precise identification of the
1351 ;;     failing operation, or if tested outside of vectorisable regions
1352 ;;     then the specific operation and lane are not of interest.
1353
1354 ;; FP arithmetic operations.
1355
1356 (define_insn "add<mode>3"
1357  [(set (match_operand:VDQF 0 "register_operand" "=w")
1358        (plus:VDQF (match_operand:VDQF 1 "register_operand" "w")
1359                   (match_operand:VDQF 2 "register_operand" "w")))]
1360  "TARGET_SIMD"
1361  "fadd\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1362   [(set_attr "type" "neon_fp_addsub_<Vetype><q>")]
1363 )
1364
1365 (define_insn "sub<mode>3"
1366  [(set (match_operand:VDQF 0 "register_operand" "=w")
1367        (minus:VDQF (match_operand:VDQF 1 "register_operand" "w")
1368                    (match_operand:VDQF 2 "register_operand" "w")))]
1369  "TARGET_SIMD"
1370  "fsub\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1371   [(set_attr "type" "neon_fp_addsub_<Vetype><q>")]
1372 )
1373
1374 (define_insn "mul<mode>3"
1375  [(set (match_operand:VDQF 0 "register_operand" "=w")
1376        (mult:VDQF (match_operand:VDQF 1 "register_operand" "w")
1377                   (match_operand:VDQF 2 "register_operand" "w")))]
1378  "TARGET_SIMD"
1379  "fmul\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1380   [(set_attr "type" "neon_fp_mul_<Vetype><q>")]
1381 )
1382
1383 (define_insn "div<mode>3"
1384  [(set (match_operand:VDQF 0 "register_operand" "=w")
1385        (div:VDQF (match_operand:VDQF 1 "register_operand" "w")
1386                  (match_operand:VDQF 2 "register_operand" "w")))]
1387  "TARGET_SIMD"
1388  "fdiv\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1389   [(set_attr "type" "neon_fp_div_<Vetype><q>")]
1390 )
1391
1392 (define_insn "neg<mode>2"
1393  [(set (match_operand:VDQF 0 "register_operand" "=w")
1394        (neg:VDQF (match_operand:VDQF 1 "register_operand" "w")))]
1395  "TARGET_SIMD"
1396  "fneg\\t%0.<Vtype>, %1.<Vtype>"
1397   [(set_attr "type" "neon_fp_neg_<Vetype><q>")]
1398 )
1399
1400 (define_insn "abs<mode>2"
1401  [(set (match_operand:VDQF 0 "register_operand" "=w")
1402        (abs:VDQF (match_operand:VDQF 1 "register_operand" "w")))]
1403  "TARGET_SIMD"
1404  "fabs\\t%0.<Vtype>, %1.<Vtype>"
1405   [(set_attr "type" "neon_fp_abs_<Vetype><q>")]
1406 )
1407
1408 (define_insn "fma<mode>4"
1409   [(set (match_operand:VDQF 0 "register_operand" "=w")
1410        (fma:VDQF (match_operand:VDQF 1 "register_operand" "w")
1411                 (match_operand:VDQF 2 "register_operand" "w")
1412                 (match_operand:VDQF 3 "register_operand" "0")))]
1413   "TARGET_SIMD"
1414  "fmla\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1415   [(set_attr "type" "neon_fp_mla_<Vetype><q>")]
1416 )
1417
1418 (define_insn "*aarch64_fma4_elt<mode>"
1419   [(set (match_operand:VDQF 0 "register_operand" "=w")
1420     (fma:VDQF
1421       (vec_duplicate:VDQF
1422         (vec_select:<VEL>
1423           (match_operand:VDQF 1 "register_operand" "<h_con>")
1424           (parallel [(match_operand:SI 2 "immediate_operand")])))
1425       (match_operand:VDQF 3 "register_operand" "w")
1426       (match_operand:VDQF 4 "register_operand" "0")))]
1427   "TARGET_SIMD"
1428   {
1429     operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
1430     return "fmla\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1431   }
1432   [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1433 )
1434
1435 (define_insn "*aarch64_fma4_elt_<vswap_width_name><mode>"
1436   [(set (match_operand:VDQSF 0 "register_operand" "=w")
1437     (fma:VDQSF
1438       (vec_duplicate:VDQSF
1439         (vec_select:<VEL>
1440           (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
1441           (parallel [(match_operand:SI 2 "immediate_operand")])))
1442       (match_operand:VDQSF 3 "register_operand" "w")
1443       (match_operand:VDQSF 4 "register_operand" "0")))]
1444   "TARGET_SIMD"
1445   {
1446     operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
1447                                           INTVAL (operands[2])));
1448     return "fmla\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1449   }
1450   [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1451 )
1452
1453 (define_insn "*aarch64_fma4_elt_to_128df"
1454   [(set (match_operand:V2DF 0 "register_operand" "=w")
1455     (fma:V2DF
1456       (vec_duplicate:V2DF
1457           (match_operand:DF 1 "register_operand" "w"))
1458       (match_operand:V2DF 2 "register_operand" "w")
1459       (match_operand:V2DF 3 "register_operand" "0")))]
1460   "TARGET_SIMD"
1461   "fmla\\t%0.2d, %2.2d, %1.2d[0]"
1462   [(set_attr "type" "neon_fp_mla_d_scalar_q")]
1463 )
1464
1465 (define_insn "*aarch64_fma4_elt_to_64v2df"
1466   [(set (match_operand:DF 0 "register_operand" "=w")
1467     (fma:DF
1468         (vec_select:DF
1469           (match_operand:V2DF 1 "register_operand" "w")
1470           (parallel [(match_operand:SI 2 "immediate_operand")]))
1471       (match_operand:DF 3 "register_operand" "w")
1472       (match_operand:DF 4 "register_operand" "0")))]
1473   "TARGET_SIMD"
1474   {
1475     operands[2] = GEN_INT (ENDIAN_LANE_N (V2DFmode, INTVAL (operands[2])));
1476     return "fmla\\t%0.2d, %3.2d, %1.2d[%2]";
1477   }
1478   [(set_attr "type" "neon_fp_mla_d_scalar_q")]
1479 )
1480
1481 (define_insn "fnma<mode>4"
1482   [(set (match_operand:VDQF 0 "register_operand" "=w")
1483         (fma:VDQF
1484           (match_operand:VDQF 1 "register_operand" "w")
1485           (neg:VDQF
1486             (match_operand:VDQF 2 "register_operand" "w"))
1487           (match_operand:VDQF 3 "register_operand" "0")))]
1488   "TARGET_SIMD"
1489  "fmls\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1490   [(set_attr "type" "neon_fp_mla_<Vetype><q>")]
1491 )
1492
1493 (define_insn "*aarch64_fnma4_elt<mode>"
1494   [(set (match_operand:VDQF 0 "register_operand" "=w")
1495     (fma:VDQF
1496       (neg:VDQF
1497         (match_operand:VDQF 3 "register_operand" "w"))
1498       (vec_duplicate:VDQF
1499         (vec_select:<VEL>
1500           (match_operand:VDQF 1 "register_operand" "<h_con>")
1501           (parallel [(match_operand:SI 2 "immediate_operand")])))
1502       (match_operand:VDQF 4 "register_operand" "0")))]
1503   "TARGET_SIMD"
1504   {
1505     operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
1506     return "fmls\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1507   }
1508   [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1509 )
1510
1511 (define_insn "*aarch64_fnma4_elt_<vswap_width_name><mode>"
1512   [(set (match_operand:VDQSF 0 "register_operand" "=w")
1513     (fma:VDQSF
1514       (neg:VDQSF
1515         (match_operand:VDQSF 3 "register_operand" "w"))
1516       (vec_duplicate:VDQSF
1517         (vec_select:<VEL>
1518           (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
1519           (parallel [(match_operand:SI 2 "immediate_operand")])))
1520       (match_operand:VDQSF 4 "register_operand" "0")))]
1521   "TARGET_SIMD"
1522   {
1523     operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
1524                                           INTVAL (operands[2])));
1525     return "fmls\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1526   }
1527   [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1528 )
1529
1530 (define_insn "*aarch64_fnma4_elt_to_128df"
1531   [(set (match_operand:V2DF 0 "register_operand" "=w")
1532     (fma:V2DF
1533       (neg:V2DF
1534         (match_operand:V2DF 2 "register_operand" "w"))
1535       (vec_duplicate:V2DF
1536         (match_operand:DF 1 "register_operand" "w"))
1537       (match_operand:V2DF 3 "register_operand" "0")))]
1538   "TARGET_SIMD"
1539   "fmls\\t%0.2d, %2.2d, %1.2d[0]"
1540   [(set_attr "type" "neon_fp_mla_d_scalar_q")]
1541 )
1542
1543 (define_insn "*aarch64_fnma4_elt_to_64v2df"
1544   [(set (match_operand:DF 0 "register_operand" "=w")
1545     (fma:DF
1546       (vec_select:DF
1547         (match_operand:V2DF 1 "register_operand" "w")
1548         (parallel [(match_operand:SI 2 "immediate_operand")]))
1549       (neg:DF
1550         (match_operand:DF 3 "register_operand" "w"))
1551       (match_operand:DF 4 "register_operand" "0")))]
1552   "TARGET_SIMD"
1553   {
1554     operands[2] = GEN_INT (ENDIAN_LANE_N (V2DFmode, INTVAL (operands[2])));
1555     return "fmls\\t%0.2d, %3.2d, %1.2d[%2]";
1556   }
1557   [(set_attr "type" "neon_fp_mla_d_scalar_q")]
1558 )
1559
1560 ;; Vector versions of the floating-point frint patterns.
1561 ;; Expands to btrunc, ceil, floor, nearbyint, rint, round, frintn.
1562 (define_insn "<frint_pattern><mode>2"
1563   [(set (match_operand:VDQF 0 "register_operand" "=w")
1564         (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")]
1565                       FRINT))]
1566   "TARGET_SIMD"
1567   "frint<frint_suffix>\\t%0.<Vtype>, %1.<Vtype>"
1568   [(set_attr "type" "neon_fp_round_<Vetype><q>")]
1569 )
1570
1571 ;; Vector versions of the fcvt standard patterns.
1572 ;; Expands to lbtrunc, lround, lceil, lfloor
1573 (define_insn "l<fcvt_pattern><su_optab><VDQF:mode><fcvt_target>2"
1574   [(set (match_operand:<FCVT_TARGET> 0 "register_operand" "=w")
1575         (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1576                                [(match_operand:VDQF 1 "register_operand" "w")]
1577                                FCVT)))]
1578   "TARGET_SIMD"
1579   "fcvt<frint_suffix><su>\\t%0.<Vtype>, %1.<Vtype>"
1580   [(set_attr "type" "neon_fp_to_int_<Vetype><q>")]
1581 )
1582
1583 (define_expand "<optab><VDQF:mode><fcvt_target>2"
1584   [(set (match_operand:<FCVT_TARGET> 0 "register_operand")
1585         (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1586                                [(match_operand:VDQF 1 "register_operand")]
1587                                UNSPEC_FRINTZ)))]
1588   "TARGET_SIMD"
1589   {})
1590
1591 (define_expand "<fix_trunc_optab><VDQF:mode><fcvt_target>2"
1592   [(set (match_operand:<FCVT_TARGET> 0 "register_operand")
1593         (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1594                                [(match_operand:VDQF 1 "register_operand")]
1595                                UNSPEC_FRINTZ)))]
1596   "TARGET_SIMD"
1597   {})
1598
1599 (define_expand "ftrunc<VDQF:mode>2"
1600   [(set (match_operand:VDQF 0 "register_operand")
1601         (unspec:VDQF [(match_operand:VDQF 1 "register_operand")]
1602                       UNSPEC_FRINTZ))]
1603   "TARGET_SIMD"
1604   {})
1605
1606 (define_insn "<optab><fcvt_target><VDQF:mode>2"
1607   [(set (match_operand:VDQF 0 "register_operand" "=w")
1608         (FLOATUORS:VDQF
1609           (match_operand:<FCVT_TARGET> 1 "register_operand" "w")))]
1610   "TARGET_SIMD"
1611   "<su_optab>cvtf\\t%0.<Vtype>, %1.<Vtype>"
1612   [(set_attr "type" "neon_int_to_fp_<Vetype><q>")]
1613 )
1614
1615 ;; Conversions between vectors of floats and doubles.
1616 ;; Contains a mix of patterns to match standard pattern names
1617 ;; and those for intrinsics.
1618
1619 ;; Float widening operations.
1620
1621 (define_insn "vec_unpacks_lo_v4sf"
1622   [(set (match_operand:V2DF 0 "register_operand" "=w")
1623         (float_extend:V2DF
1624           (vec_select:V2SF
1625             (match_operand:V4SF 1 "register_operand" "w")
1626             (parallel [(const_int 0) (const_int 1)])
1627           )))]
1628   "TARGET_SIMD"
1629   "fcvtl\\t%0.2d, %1.2s"
1630   [(set_attr "type" "neon_fp_cvt_widen_s")]
1631 )
1632
1633 (define_insn "aarch64_float_extend_lo_v2df"
1634   [(set (match_operand:V2DF 0 "register_operand" "=w")
1635         (float_extend:V2DF
1636           (match_operand:V2SF 1 "register_operand" "w")))]
1637   "TARGET_SIMD"
1638   "fcvtl\\t%0.2d, %1.2s"
1639   [(set_attr "type" "neon_fp_cvt_widen_s")]
1640 )
1641
1642 (define_insn "vec_unpacks_hi_v4sf"
1643   [(set (match_operand:V2DF 0 "register_operand" "=w")
1644         (float_extend:V2DF
1645           (vec_select:V2SF
1646             (match_operand:V4SF 1 "register_operand" "w")
1647             (parallel [(const_int 2) (const_int 3)])
1648           )))]
1649   "TARGET_SIMD"
1650   "fcvtl2\\t%0.2d, %1.4s"
1651   [(set_attr "type" "neon_fp_cvt_widen_s")]
1652 )
1653
1654 ;; Float narrowing operations.
1655
1656 (define_insn "aarch64_float_truncate_lo_v2sf"
1657   [(set (match_operand:V2SF 0 "register_operand" "=w")
1658       (float_truncate:V2SF
1659         (match_operand:V2DF 1 "register_operand" "w")))]
1660   "TARGET_SIMD"
1661   "fcvtn\\t%0.2s, %1.2d"
1662   [(set_attr "type" "neon_fp_cvt_narrow_d_q")]
1663 )
1664
1665 (define_insn "aarch64_float_truncate_hi_v4sf"
1666   [(set (match_operand:V4SF 0 "register_operand" "=w")
1667     (vec_concat:V4SF
1668       (match_operand:V2SF 1 "register_operand" "0")
1669       (float_truncate:V2SF
1670         (match_operand:V2DF 2 "register_operand" "w"))))]
1671   "TARGET_SIMD"
1672   "fcvtn2\\t%0.4s, %2.2d"
1673   [(set_attr "type" "neon_fp_cvt_narrow_d_q")]
1674 )
1675
1676 (define_expand "vec_pack_trunc_v2df"
1677   [(set (match_operand:V4SF 0 "register_operand")
1678       (vec_concat:V4SF
1679         (float_truncate:V2SF
1680             (match_operand:V2DF 1 "register_operand"))
1681         (float_truncate:V2SF
1682             (match_operand:V2DF 2 "register_operand"))
1683           ))]
1684   "TARGET_SIMD"
1685   {
1686     rtx tmp = gen_reg_rtx (V2SFmode);
1687     int lo = BYTES_BIG_ENDIAN ? 2 : 1;
1688     int hi = BYTES_BIG_ENDIAN ? 1 : 2;
1689
1690     emit_insn (gen_aarch64_float_truncate_lo_v2sf (tmp, operands[lo]));
1691     emit_insn (gen_aarch64_float_truncate_hi_v4sf (operands[0],
1692                                                    tmp, operands[hi]));
1693     DONE;
1694   }
1695 )
1696
1697 (define_expand "vec_pack_trunc_df"
1698   [(set (match_operand:V2SF 0 "register_operand")
1699       (vec_concat:V2SF
1700         (float_truncate:SF
1701             (match_operand:DF 1 "register_operand"))
1702         (float_truncate:SF
1703             (match_operand:DF 2 "register_operand"))
1704           ))]
1705   "TARGET_SIMD"
1706   {
1707     rtx tmp = gen_reg_rtx (V2SFmode);
1708     int lo = BYTES_BIG_ENDIAN ? 2 : 1;
1709     int hi = BYTES_BIG_ENDIAN ? 1 : 2;
1710
1711     emit_insn (gen_move_lo_quad_v2df (tmp, operands[lo]));
1712     emit_insn (gen_move_hi_quad_v2df (tmp, operands[hi]));
1713     emit_insn (gen_aarch64_float_truncate_lo_v2sf (operands[0], tmp));
1714     DONE;
1715   }
1716 )
1717
1718 (define_insn "aarch64_vmls<mode>"
1719   [(set (match_operand:VDQF 0 "register_operand" "=w")
1720        (minus:VDQF (match_operand:VDQF 1 "register_operand" "0")
1721                    (mult:VDQF (match_operand:VDQF 2 "register_operand" "w")
1722                               (match_operand:VDQF 3 "register_operand" "w"))))]
1723   "TARGET_SIMD"
1724  "fmls\\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
1725   [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1726 )
1727
1728 ;; FP Max/Min
1729 ;; Max/Min are introduced by idiom recognition by GCC's mid-end.  An
1730 ;; expression like:
1731 ;;      a = (b < c) ? b : c;
1732 ;; is idiom-matched as MIN_EXPR<b,c> only if -ffinite-math-only is enabled
1733 ;; either explicitly or indirectly via -ffast-math.
1734 ;;
1735 ;; MIN_EXPR and MAX_EXPR eventually map to 'smin' and 'smax' in RTL.
1736 ;; The 'smax' and 'smin' RTL standard pattern names do not specify which
1737 ;; operand will be returned when both operands are zero (i.e. they may not
1738 ;; honour signed zeroes), or when either operand is NaN.  Therefore GCC
1739 ;; only introduces MIN_EXPR/MAX_EXPR in fast math mode or when not honouring
1740 ;; NaNs.
1741
1742 (define_insn "<su><maxmin><mode>3"
1743   [(set (match_operand:VDQF 0 "register_operand" "=w")
1744         (FMAXMIN:VDQF (match_operand:VDQF 1 "register_operand" "w")
1745                    (match_operand:VDQF 2 "register_operand" "w")))]
1746   "TARGET_SIMD"
1747   "f<maxmin>nm\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1748   [(set_attr "type" "neon_fp_minmax_<Vetype><q>")]
1749 )
1750
1751 (define_insn "<maxmin_uns><mode>3"
1752   [(set (match_operand:VDQF 0 "register_operand" "=w")
1753        (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")
1754                      (match_operand:VDQF 2 "register_operand" "w")]
1755                     FMAXMIN_UNS))]
1756   "TARGET_SIMD"
1757   "<maxmin_uns_op>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1758   [(set_attr "type" "neon_fp_minmax_<Vetype><q>")]
1759 )
1760
1761 ;; 'across lanes' add.
1762
1763 (define_insn "reduc_<sur>plus_<mode>"
1764  [(set (match_operand:VDQV 0 "register_operand" "=w")
1765        (unspec:VDQV [(match_operand:VDQV 1 "register_operand" "w")]
1766                     SUADDV))]
1767  "TARGET_SIMD"
1768  "add<VDQV:vp>\\t%<Vetype>0, %1.<Vtype>"
1769   [(set_attr "type" "neon_reduc_add<q>")]
1770 )
1771
1772 (define_insn "reduc_<sur>plus_v2si"
1773  [(set (match_operand:V2SI 0 "register_operand" "=w")
1774        (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")]
1775                     SUADDV))]
1776  "TARGET_SIMD"
1777  "addp\\t%0.2s, %1.2s, %1.2s"
1778   [(set_attr "type" "neon_reduc_add")]
1779 )
1780
1781 (define_insn "reduc_splus_<mode>"
1782  [(set (match_operand:V2F 0 "register_operand" "=w")
1783        (unspec:V2F [(match_operand:V2F 1 "register_operand" "w")]
1784                    UNSPEC_FADDV))]
1785  "TARGET_SIMD"
1786  "faddp\\t%<Vetype>0, %1.<Vtype>"
1787   [(set_attr "type" "neon_fp_reduc_add_<Vetype><q>")]
1788 )
1789
1790 (define_insn "aarch64_addpv4sf"
1791  [(set (match_operand:V4SF 0 "register_operand" "=w")
1792        (unspec:V4SF [(match_operand:V4SF 1 "register_operand" "w")]
1793                     UNSPEC_FADDV))]
1794  "TARGET_SIMD"
1795  "faddp\\t%0.4s, %1.4s, %1.4s"
1796   [(set_attr "type" "neon_fp_reduc_add_s_q")]
1797 )
1798
1799 (define_expand "reduc_splus_v4sf"
1800  [(set (match_operand:V4SF 0 "register_operand")
1801        (unspec:V4SF [(match_operand:V4SF 1 "register_operand")]
1802                     UNSPEC_FADDV))]
1803  "TARGET_SIMD"
1804 {
1805   emit_insn (gen_aarch64_addpv4sf (operands[0], operands[1]));
1806   emit_insn (gen_aarch64_addpv4sf (operands[0], operands[0]));
1807   DONE;
1808 })
1809
1810 (define_insn "clz<mode>2"
1811  [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
1812        (clz:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w")))]
1813  "TARGET_SIMD"
1814  "clz\\t%0.<Vtype>, %1.<Vtype>"
1815   [(set_attr "type" "neon_cls<q>")]
1816 )
1817
1818 ;; 'across lanes' max and min ops.
1819
1820 (define_insn "reduc_<maxmin_uns>_<mode>"
1821  [(set (match_operand:VDQV_S 0 "register_operand" "=w")
1822        (unspec:VDQV_S [(match_operand:VDQV_S 1 "register_operand" "w")]
1823                     MAXMINV))]
1824  "TARGET_SIMD"
1825  "<maxmin_uns_op>v\\t%<Vetype>0, %1.<Vtype>"
1826   [(set_attr "type" "neon_reduc_minmax<q>")]
1827 )
1828
1829 (define_insn "reduc_<maxmin_uns>_v2si"
1830  [(set (match_operand:V2SI 0 "register_operand" "=w")
1831        (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")]
1832                     MAXMINV))]
1833  "TARGET_SIMD"
1834  "<maxmin_uns_op>p\\t%0.2s, %1.2s, %1.2s"
1835   [(set_attr "type" "neon_reduc_minmax")]
1836 )
1837
1838 (define_insn "reduc_<maxmin_uns>_<mode>"
1839  [(set (match_operand:V2F 0 "register_operand" "=w")
1840        (unspec:V2F [(match_operand:V2F 1 "register_operand" "w")]
1841                     FMAXMINV))]
1842  "TARGET_SIMD"
1843  "<maxmin_uns_op>p\\t%<Vetype>0, %1.<Vtype>"
1844   [(set_attr "type" "neon_fp_reduc_minmax_<Vetype><q>")]
1845 )
1846
1847 (define_insn "reduc_<maxmin_uns>_v4sf"
1848  [(set (match_operand:V4SF 0 "register_operand" "=w")
1849        (unspec:V4SF [(match_operand:V4SF 1 "register_operand" "w")]
1850                     FMAXMINV))]
1851  "TARGET_SIMD"
1852  "<maxmin_uns_op>v\\t%s0, %1.4s"
1853   [(set_attr "type" "neon_fp_reduc_minmax_s_q")]
1854 )
1855
1856 ;; aarch64_simd_bsl may compile to any of bsl/bif/bit depending on register
1857 ;; allocation.
1858 ;; Operand 1 is the mask, operands 2 and 3 are the bitfields from which
1859 ;; to select.
1860 ;;
1861 ;; Thus our BSL is of the form:
1862 ;;   op0 = bsl (mask, op2, op3)
1863 ;; We can use any of:
1864 ;;
1865 ;;   if (op0 = mask)
1866 ;;     bsl mask, op1, op2
1867 ;;   if (op0 = op1) (so 1-bits in mask choose bits from op2, else op0)
1868 ;;     bit op0, op2, mask
1869 ;;   if (op0 = op2) (so 0-bits in mask choose bits from op1, else op0)
1870 ;;     bif op0, op1, mask
1871
1872 (define_insn "aarch64_simd_bsl<mode>_internal"
1873   [(set (match_operand:VSDQ_I_DI 0 "register_operand"           "=w,w,w")
1874         (ior:VSDQ_I_DI
1875            (and:VSDQ_I_DI
1876              (not:<V_cmp_result>
1877                (match_operand:<V_cmp_result> 1 "register_operand"       " 0,w,w"))
1878              (match_operand:VSDQ_I_DI 3 "register_operand"      " w,0,w"))
1879            (and:VSDQ_I_DI
1880              (match_dup:<V_cmp_result> 1)
1881              (match_operand:VSDQ_I_DI 2 "register_operand"      " w,w,0"))
1882         ))]
1883   "TARGET_SIMD"
1884   "@
1885   bsl\\t%0.<Vbtype>, %2.<Vbtype>, %3.<Vbtype>
1886   bit\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>
1887   bif\\t%0.<Vbtype>, %3.<Vbtype>, %1.<Vbtype>"
1888   [(set_attr "type" "neon_bsl<q>")]
1889 )
1890
1891 (define_expand "aarch64_simd_bsl<mode>"
1892   [(match_operand:VALLDIF 0 "register_operand")
1893    (match_operand:<V_cmp_result> 1 "register_operand")
1894    (match_operand:VALLDIF 2 "register_operand")
1895    (match_operand:VALLDIF 3 "register_operand")]
1896  "TARGET_SIMD"
1897 {
1898   /* We can't alias operands together if they have different modes.  */
1899   rtx tmp = operands[0];
1900   if (FLOAT_MODE_P (<MODE>mode))
1901     {
1902       operands[2] = gen_lowpart (<V_cmp_result>mode, operands[2]);
1903       operands[3] = gen_lowpart (<V_cmp_result>mode, operands[3]);
1904       tmp = gen_reg_rtx (<V_cmp_result>mode);
1905     }
1906   operands[1] = gen_lowpart (<V_cmp_result>mode, operands[1]);
1907   emit_insn (gen_aarch64_simd_bsl<v_cmp_result>_internal (tmp,
1908                                                           operands[1],
1909                                                           operands[2],
1910                                                           operands[3]));
1911   if (tmp != operands[0])
1912     emit_move_insn (operands[0], gen_lowpart (<MODE>mode, tmp));
1913
1914   DONE;
1915 })
1916
1917 (define_expand "aarch64_vcond_internal<mode><mode>"
1918   [(set (match_operand:VDQ 0 "register_operand")
1919         (if_then_else:VDQ
1920           (match_operator 3 "comparison_operator"
1921             [(match_operand:VDQ 4 "register_operand")
1922              (match_operand:VDQ 5 "nonmemory_operand")])
1923           (match_operand:VDQ 1 "nonmemory_operand")
1924           (match_operand:VDQ 2 "nonmemory_operand")))]
1925   "TARGET_SIMD"
1926 {
1927   rtx op1 = operands[1];
1928   rtx op2 = operands[2];
1929   rtx mask = gen_reg_rtx (<MODE>mode);
1930   enum rtx_code code = GET_CODE (operands[3]);
1931
1932   /* Switching OP1 and OP2 is necessary for NE (to output a cmeq insn),
1933      and desirable for other comparisons if it results in FOO ? -1 : 0
1934      (this allows direct use of the comparison result without a bsl).  */
1935   if (code == NE
1936       || (code != EQ
1937           && op1 == CONST0_RTX (<V_cmp_result>mode)
1938           && op2 == CONSTM1_RTX (<V_cmp_result>mode)))
1939     {
1940       op1 = operands[2];
1941       op2 = operands[1];
1942       switch (code)
1943         {
1944         case LE: code = GT; break;
1945         case LT: code = GE; break;
1946         case GE: code = LT; break;
1947         case GT: code = LE; break;
1948         /* No case EQ.  */
1949         case NE: code = EQ; break;
1950         case LTU: code = GEU; break;
1951         case LEU: code = GTU; break;
1952         case GTU: code = LEU; break;
1953         case GEU: code = LTU; break;
1954         default: gcc_unreachable ();
1955         }
1956     }
1957
1958   /* Make sure we can handle the last operand.  */
1959   switch (code)
1960     {
1961     case NE:
1962       /* Normalized to EQ above.  */
1963       gcc_unreachable ();
1964
1965     case LE:
1966     case LT:
1967     case GE:
1968     case GT:
1969     case EQ:
1970       /* These instructions have a form taking an immediate zero.  */
1971       if (operands[5] == CONST0_RTX (<MODE>mode))
1972         break;
1973       /* Fall through, as may need to load into register.  */
1974     default:
1975       if (!REG_P (operands[5]))
1976         operands[5] = force_reg (<MODE>mode, operands[5]);
1977       break;
1978     }
1979
1980   switch (code)
1981     {
1982     case LT:
1983       emit_insn (gen_aarch64_cmlt<mode> (mask, operands[4], operands[5]));
1984       break;
1985
1986     case GE:
1987       emit_insn (gen_aarch64_cmge<mode> (mask, operands[4], operands[5]));
1988       break;
1989
1990     case LE:
1991       emit_insn (gen_aarch64_cmle<mode> (mask, operands[4], operands[5]));
1992       break;
1993
1994     case GT:
1995       emit_insn (gen_aarch64_cmgt<mode> (mask, operands[4], operands[5]));
1996       break;
1997
1998     case LTU:
1999       emit_insn (gen_aarch64_cmgtu<mode> (mask, operands[5], operands[4]));
2000       break;
2001
2002     case GEU:
2003       emit_insn (gen_aarch64_cmgeu<mode> (mask, operands[4], operands[5]));
2004       break;
2005
2006     case LEU:
2007       emit_insn (gen_aarch64_cmgeu<mode> (mask, operands[5], operands[4]));
2008       break;
2009
2010     case GTU:
2011       emit_insn (gen_aarch64_cmgtu<mode> (mask, operands[4], operands[5]));
2012       break;
2013
2014     /* NE has been normalized to EQ above.  */
2015     case EQ:
2016       emit_insn (gen_aarch64_cmeq<mode> (mask, operands[4], operands[5]));
2017       break;
2018
2019     default:
2020       gcc_unreachable ();
2021     }
2022
2023     /* If we have (a = (b CMP c) ? -1 : 0);
2024        Then we can simply move the generated mask.  */
2025
2026     if (op1 == CONSTM1_RTX (<V_cmp_result>mode)
2027         && op2 == CONST0_RTX (<V_cmp_result>mode))
2028       emit_move_insn (operands[0], mask);
2029     else
2030       {
2031         if (!REG_P (op1))
2032           op1 = force_reg (<MODE>mode, op1);
2033         if (!REG_P (op2))
2034           op2 = force_reg (<MODE>mode, op2);
2035         emit_insn (gen_aarch64_simd_bsl<mode> (operands[0], mask,
2036                                                op1, op2));
2037       }
2038
2039   DONE;
2040 })
2041
2042 (define_expand "aarch64_vcond_internal<VDQF_COND:mode><VDQF:mode>"
2043   [(set (match_operand:VDQF_COND 0 "register_operand")
2044         (if_then_else:VDQF
2045           (match_operator 3 "comparison_operator"
2046             [(match_operand:VDQF 4 "register_operand")
2047              (match_operand:VDQF 5 "nonmemory_operand")])
2048           (match_operand:VDQF_COND 1 "nonmemory_operand")
2049           (match_operand:VDQF_COND 2 "nonmemory_operand")))]
2050   "TARGET_SIMD"
2051 {
2052   int inverse = 0;
2053   int use_zero_form = 0;
2054   int swap_bsl_operands = 0;
2055   rtx op1 = operands[1];
2056   rtx op2 = operands[2];
2057   rtx mask = gen_reg_rtx (<VDQF_COND:V_cmp_result>mode);
2058   rtx tmp = gen_reg_rtx (<VDQF_COND:V_cmp_result>mode);
2059
2060   rtx (*base_comparison) (rtx, rtx, rtx);
2061   rtx (*complimentary_comparison) (rtx, rtx, rtx);
2062
2063   switch (GET_CODE (operands[3]))
2064     {
2065     case GE:
2066     case GT:
2067     case LE:
2068     case LT:
2069     case EQ:
2070       if (operands[5] == CONST0_RTX (<MODE>mode))
2071         {
2072           use_zero_form = 1;
2073           break;
2074         }
2075       /* Fall through.  */
2076     default:
2077       if (!REG_P (operands[5]))
2078         operands[5] = force_reg (<VDQF:MODE>mode, operands[5]);
2079     }
2080
2081   switch (GET_CODE (operands[3]))
2082     {
2083     case LT:
2084     case UNLT:
2085       inverse = 1;
2086       /* Fall through.  */
2087     case GE:
2088     case UNGE:
2089     case ORDERED:
2090     case UNORDERED:
2091       base_comparison = gen_aarch64_cmge<VDQF:mode>;
2092       complimentary_comparison = gen_aarch64_cmgt<VDQF:mode>;
2093       break;
2094     case LE:
2095     case UNLE:
2096       inverse = 1;
2097       /* Fall through.  */
2098     case GT:
2099     case UNGT:
2100       base_comparison = gen_aarch64_cmgt<VDQF:mode>;
2101       complimentary_comparison = gen_aarch64_cmge<VDQF:mode>;
2102       break;
2103     case EQ:
2104     case NE:
2105     case UNEQ:
2106       base_comparison = gen_aarch64_cmeq<VDQF:mode>;
2107       complimentary_comparison = gen_aarch64_cmeq<VDQF:mode>;
2108       break;
2109     default:
2110       gcc_unreachable ();
2111     }
2112
2113   switch (GET_CODE (operands[3]))
2114     {
2115     case LT:
2116     case LE:
2117     case GT:
2118     case GE:
2119     case EQ:
2120       /* The easy case.  Here we emit one of FCMGE, FCMGT or FCMEQ.
2121          As a LT b <=> b GE a && a LE b <=> b GT a.  Our transformations are:
2122          a GE b -> a GE b
2123          a GT b -> a GT b
2124          a LE b -> b GE a
2125          a LT b -> b GT a
2126          a EQ b -> a EQ b
2127          Note that there also exist direct comparison against 0 forms,
2128          so catch those as a special case.  */
2129       if (use_zero_form)
2130         {
2131           inverse = 0;
2132           switch (GET_CODE (operands[3]))
2133             {
2134             case LT:
2135               base_comparison = gen_aarch64_cmlt<VDQF:mode>;
2136               break;
2137             case LE:
2138               base_comparison = gen_aarch64_cmle<VDQF:mode>;
2139               break;
2140             default:
2141               /* Do nothing, other zero form cases already have the correct
2142                  base_comparison.  */
2143               break;
2144             }
2145         }
2146
2147       if (!inverse)
2148         emit_insn (base_comparison (mask, operands[4], operands[5]));
2149       else
2150         emit_insn (complimentary_comparison (mask, operands[5], operands[4]));
2151       break;
2152     case UNLT:
2153     case UNLE:
2154     case UNGT:
2155     case UNGE:
2156     case NE:
2157       /* FCM returns false for lanes which are unordered, so if we use
2158          the inverse of the comparison we actually want to emit, then
2159          swap the operands to BSL, we will end up with the correct result.
2160          Note that a NE NaN and NaN NE b are true for all a, b.
2161
2162          Our transformations are:
2163          a GE b -> !(b GT a)
2164          a GT b -> !(b GE a)
2165          a LE b -> !(a GT b)
2166          a LT b -> !(a GE b)
2167          a NE b -> !(a EQ b)  */
2168
2169       if (inverse)
2170         emit_insn (base_comparison (mask, operands[4], operands[5]));
2171       else
2172         emit_insn (complimentary_comparison (mask, operands[5], operands[4]));
2173
2174       swap_bsl_operands = 1;
2175       break;
2176     case UNEQ:
2177       /* We check (a > b ||  b > a).  combining these comparisons give us
2178          true iff !(a != b && a ORDERED b), swapping the operands to BSL
2179          will then give us (a == b ||  a UNORDERED b) as intended.  */
2180
2181       emit_insn (gen_aarch64_cmgt<VDQF:mode> (mask, operands[4], operands[5]));
2182       emit_insn (gen_aarch64_cmgt<VDQF:mode> (tmp, operands[5], operands[4]));
2183       emit_insn (gen_ior<VDQF_COND:v_cmp_result>3 (mask, mask, tmp));
2184       swap_bsl_operands = 1;
2185       break;
2186     case UNORDERED:
2187        /* Operands are ORDERED iff (a > b || b >= a).
2188          Swapping the operands to BSL will give the UNORDERED case.  */
2189      swap_bsl_operands = 1;
2190      /* Fall through.  */
2191     case ORDERED:
2192       emit_insn (gen_aarch64_cmgt<VDQF:mode> (tmp, operands[4], operands[5]));
2193       emit_insn (gen_aarch64_cmge<VDQF:mode> (mask, operands[5], operands[4]));
2194       emit_insn (gen_ior<VDQF_COND:v_cmp_result>3 (mask, mask, tmp));
2195       break;
2196     default:
2197       gcc_unreachable ();
2198     }
2199
2200   if (swap_bsl_operands)
2201     {
2202       op1 = operands[2];
2203       op2 = operands[1];
2204     }
2205
2206     /* If we have (a = (b CMP c) ? -1 : 0);
2207        Then we can simply move the generated mask.  */
2208
2209     if (op1 == CONSTM1_RTX (<VDQF_COND:V_cmp_result>mode)
2210         && op2 == CONST0_RTX (<VDQF_COND:V_cmp_result>mode))
2211       emit_move_insn (operands[0], mask);
2212     else
2213       {
2214         if (!REG_P (op1))
2215           op1 = force_reg (<VDQF_COND:MODE>mode, op1);
2216         if (!REG_P (op2))
2217           op2 = force_reg (<VDQF_COND:MODE>mode, op2);
2218         emit_insn (gen_aarch64_simd_bsl<VDQF_COND:mode> (operands[0], mask,
2219                                                op1, op2));
2220       }
2221
2222   DONE;
2223 })
2224
2225 (define_expand "vcond<mode><mode>"
2226   [(set (match_operand:VALL 0 "register_operand")
2227         (if_then_else:VALL
2228           (match_operator 3 "comparison_operator"
2229             [(match_operand:VALL 4 "register_operand")
2230              (match_operand:VALL 5 "nonmemory_operand")])
2231           (match_operand:VALL 1 "nonmemory_operand")
2232           (match_operand:VALL 2 "nonmemory_operand")))]
2233   "TARGET_SIMD"
2234 {
2235   emit_insn (gen_aarch64_vcond_internal<mode><mode> (operands[0], operands[1],
2236                                                operands[2], operands[3],
2237                                                operands[4], operands[5]));
2238   DONE;
2239 })
2240
2241 (define_expand "vcond<v_cmp_result><mode>"
2242   [(set (match_operand:<V_cmp_result> 0 "register_operand")
2243         (if_then_else:<V_cmp_result>
2244           (match_operator 3 "comparison_operator"
2245             [(match_operand:VDQF 4 "register_operand")
2246              (match_operand:VDQF 5 "nonmemory_operand")])
2247           (match_operand:<V_cmp_result> 1 "nonmemory_operand")
2248           (match_operand:<V_cmp_result> 2 "nonmemory_operand")))]
2249   "TARGET_SIMD"
2250 {
2251   emit_insn (gen_aarch64_vcond_internal<v_cmp_result><mode> (
2252                                                 operands[0], operands[1],
2253                                                 operands[2], operands[3],
2254                                                 operands[4], operands[5]));
2255   DONE;
2256 })
2257
2258 (define_expand "vcondu<mode><mode>"
2259   [(set (match_operand:VDQ 0 "register_operand")
2260         (if_then_else:VDQ
2261           (match_operator 3 "comparison_operator"
2262             [(match_operand:VDQ 4 "register_operand")
2263              (match_operand:VDQ 5 "nonmemory_operand")])
2264           (match_operand:VDQ 1 "nonmemory_operand")
2265           (match_operand:VDQ 2 "nonmemory_operand")))]
2266   "TARGET_SIMD"
2267 {
2268   emit_insn (gen_aarch64_vcond_internal<mode><mode> (operands[0], operands[1],
2269                                                operands[2], operands[3],
2270                                                operands[4], operands[5]));
2271   DONE;
2272 })
2273
2274 ;; Patterns for AArch64 SIMD Intrinsics.
2275
2276 (define_expand "aarch64_create<mode>"
2277   [(match_operand:VD_RE 0 "register_operand" "")
2278    (match_operand:DI 1 "general_operand" "")]
2279   "TARGET_SIMD"
2280 {
2281   rtx src = gen_lowpart (<MODE>mode, operands[1]);
2282   emit_move_insn (operands[0], src);
2283   DONE;
2284 })
2285
2286 ;; Lane extraction with sign extension to general purpose register.
2287 (define_insn "*aarch64_get_lane_extend<GPI:mode><VDQQH:mode>"
2288   [(set (match_operand:GPI 0 "register_operand" "=r")
2289         (sign_extend:GPI
2290           (vec_select:<VEL>
2291             (match_operand:VDQQH 1 "register_operand" "w")
2292             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2293   "TARGET_SIMD"
2294   {
2295     operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
2296     return "smov\\t%<GPI:w>0, %1.<VDQQH:Vetype>[%2]";
2297   }
2298   [(set_attr "type" "neon_to_gp<q>")]
2299 )
2300
2301 (define_insn "*aarch64_get_lane_zero_extendsi<mode>"
2302   [(set (match_operand:SI 0 "register_operand" "=r")
2303         (zero_extend:SI
2304           (vec_select:<VEL>
2305             (match_operand:VDQQH 1 "register_operand" "w")
2306             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2307   "TARGET_SIMD"
2308   {
2309     operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
2310     return "umov\\t%w0, %1.<Vetype>[%2]";
2311   }
2312   [(set_attr "type" "neon_to_gp<q>")]
2313 )
2314
2315 (define_expand "aarch64_be_checked_get_lane<mode>"
2316   [(match_operand:<VEL> 0 "aarch64_simd_nonimmediate_operand")
2317    (match_operand:VALL 1 "register_operand")
2318    (match_operand:SI 2 "immediate_operand")]
2319   "TARGET_SIMD"
2320   {
2321     operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
2322     emit_insn (gen_aarch64_get_lane<mode> (operands[0],
2323                                            operands[1],
2324                                            operands[2]));
2325     DONE;
2326   }
2327 )
2328
2329 ;; Lane extraction of a value, neither sign nor zero extension
2330 ;; is guaranteed so upper bits should be considered undefined.
2331 (define_insn "aarch64_get_lane<mode>"
2332   [(set (match_operand:<VEL> 0 "aarch64_simd_nonimmediate_operand" "=r, w, Utv")
2333         (vec_select:<VEL>
2334           (match_operand:VALL 1 "register_operand" "w, w, w")
2335           (parallel [(match_operand:SI 2 "immediate_operand" "i, i, i")])))]
2336   "TARGET_SIMD"
2337   {
2338     operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
2339     switch (which_alternative)
2340       {
2341         case 0:
2342           return "umov\\t%<vwcore>0, %1.<Vetype>[%2]";
2343         case 1:
2344           return "dup\\t%<Vetype>0, %1.<Vetype>[%2]";
2345         case 2:
2346           return "st1\\t{%1.<Vetype>}[%2], %0";
2347         default:
2348           gcc_unreachable ();
2349       }
2350   }
2351   [(set_attr "type" "neon_to_gp<q>, neon_dup<q>, neon_store1_one_lane<q>")]
2352 )
2353
2354 (define_expand "aarch64_get_lanedi"
2355   [(match_operand:DI 0 "register_operand")
2356    (match_operand:DI 1 "register_operand")
2357    (match_operand:SI 2 "immediate_operand")]
2358   "TARGET_SIMD"
2359 {
2360   aarch64_simd_lane_bounds (operands[2], 0, 1);
2361   emit_move_insn (operands[0], operands[1]);
2362   DONE;
2363 })
2364
2365 (define_expand "aarch64_reinterpretv8qi<mode>"
2366   [(match_operand:V8QI 0 "register_operand" "")
2367    (match_operand:VDC 1 "register_operand" "")]
2368   "TARGET_SIMD"
2369 {
2370   aarch64_simd_reinterpret (operands[0], operands[1]);
2371   DONE;
2372 })
2373
2374 (define_expand "aarch64_reinterpretv4hi<mode>"
2375   [(match_operand:V4HI 0 "register_operand" "")
2376    (match_operand:VDC 1 "register_operand" "")]
2377   "TARGET_SIMD"
2378 {
2379   aarch64_simd_reinterpret (operands[0], operands[1]);
2380   DONE;
2381 })
2382
2383 (define_expand "aarch64_reinterpretv2si<mode>"
2384   [(match_operand:V2SI 0 "register_operand" "")
2385    (match_operand:VDC 1 "register_operand" "")]
2386   "TARGET_SIMD"
2387 {
2388   aarch64_simd_reinterpret (operands[0], operands[1]);
2389   DONE;
2390 })
2391
2392 (define_expand "aarch64_reinterpretv2sf<mode>"
2393   [(match_operand:V2SF 0 "register_operand" "")
2394    (match_operand:VDC 1 "register_operand" "")]
2395   "TARGET_SIMD"
2396 {
2397   aarch64_simd_reinterpret (operands[0], operands[1]);
2398   DONE;
2399 })
2400
2401 (define_expand "aarch64_reinterpretdi<mode>"
2402   [(match_operand:DI 0 "register_operand" "")
2403    (match_operand:VD_RE 1 "register_operand" "")]
2404   "TARGET_SIMD"
2405 {
2406   aarch64_simd_reinterpret (operands[0], operands[1]);
2407   DONE;
2408 })
2409
2410 (define_expand "aarch64_reinterpretdf<mode>"
2411   [(match_operand:DF 0 "register_operand" "")
2412    (match_operand:VD_RE 1 "register_operand" "")]
2413   "TARGET_SIMD"
2414 {
2415   aarch64_simd_reinterpret (operands[0], operands[1]);
2416   DONE;
2417 })
2418
2419 (define_expand "aarch64_reinterpretv16qi<mode>"
2420   [(match_operand:V16QI 0 "register_operand" "")
2421    (match_operand:VQ 1 "register_operand" "")]
2422   "TARGET_SIMD"
2423 {
2424   aarch64_simd_reinterpret (operands[0], operands[1]);
2425   DONE;
2426 })
2427
2428 (define_expand "aarch64_reinterpretv8hi<mode>"
2429   [(match_operand:V8HI 0 "register_operand" "")
2430    (match_operand:VQ 1 "register_operand" "")]
2431   "TARGET_SIMD"
2432 {
2433   aarch64_simd_reinterpret (operands[0], operands[1]);
2434   DONE;
2435 })
2436
2437 (define_expand "aarch64_reinterpretv4si<mode>"
2438   [(match_operand:V4SI 0 "register_operand" "")
2439    (match_operand:VQ 1 "register_operand" "")]
2440   "TARGET_SIMD"
2441 {
2442   aarch64_simd_reinterpret (operands[0], operands[1]);
2443   DONE;
2444 })
2445
2446 (define_expand "aarch64_reinterpretv4sf<mode>"
2447   [(match_operand:V4SF 0 "register_operand" "")
2448    (match_operand:VQ 1 "register_operand" "")]
2449   "TARGET_SIMD"
2450 {
2451   aarch64_simd_reinterpret (operands[0], operands[1]);
2452   DONE;
2453 })
2454
2455 (define_expand "aarch64_reinterpretv2di<mode>"
2456   [(match_operand:V2DI 0 "register_operand" "")
2457    (match_operand:VQ 1 "register_operand" "")]
2458   "TARGET_SIMD"
2459 {
2460   aarch64_simd_reinterpret (operands[0], operands[1]);
2461   DONE;
2462 })
2463
2464 (define_expand "aarch64_reinterpretv2df<mode>"
2465   [(match_operand:V2DF 0 "register_operand" "")
2466    (match_operand:VQ 1 "register_operand" "")]
2467   "TARGET_SIMD"
2468 {
2469   aarch64_simd_reinterpret (operands[0], operands[1]);
2470   DONE;
2471 })
2472
2473 ;; In this insn, operand 1 should be low, and operand 2 the high part of the
2474 ;; dest vector.
2475
2476 (define_insn "*aarch64_combinez<mode>"
2477   [(set (match_operand:<VDBL> 0 "register_operand" "=&w")
2478         (vec_concat:<VDBL>
2479            (match_operand:VDIC 1 "register_operand" "w")
2480            (match_operand:VDIC 2 "aarch64_simd_imm_zero" "Dz")))]
2481   "TARGET_SIMD && !BYTES_BIG_ENDIAN"
2482   "mov\\t%0.8b, %1.8b"
2483   [(set_attr "type" "neon_move<q>")]
2484 )
2485
2486 (define_insn "*aarch64_combinez_be<mode>"
2487   [(set (match_operand:<VDBL> 0 "register_operand" "=&w")
2488         (vec_concat:<VDBL>
2489            (match_operand:VDIC 2 "aarch64_simd_imm_zero" "Dz")
2490            (match_operand:VDIC 1 "register_operand" "w")))]
2491   "TARGET_SIMD && BYTES_BIG_ENDIAN"
2492   "mov\\t%0.8b, %1.8b"
2493   [(set_attr "type" "neon_move<q>")]
2494 )
2495
2496 (define_expand "aarch64_combine<mode>"
2497   [(match_operand:<VDBL> 0 "register_operand")
2498    (match_operand:VDC 1 "register_operand")
2499    (match_operand:VDC 2 "register_operand")]
2500   "TARGET_SIMD"
2501 {
2502   rtx op1, op2;
2503   if (BYTES_BIG_ENDIAN)
2504     {
2505       op1 = operands[2];
2506       op2 = operands[1];
2507     }
2508   else
2509     {
2510       op1 = operands[1];
2511       op2 = operands[2];
2512     }
2513   emit_insn (gen_aarch64_combine_internal<mode> (operands[0], op1, op2));
2514   DONE;
2515 }
2516 )
2517
2518 (define_insn_and_split "aarch64_combine_internal<mode>"
2519   [(set (match_operand:<VDBL> 0 "register_operand" "=&w")
2520         (vec_concat:<VDBL> (match_operand:VDC 1 "register_operand" "w")
2521                            (match_operand:VDC 2 "register_operand" "w")))]
2522   "TARGET_SIMD"
2523   "#"
2524   "&& reload_completed"
2525   [(const_int 0)]
2526 {
2527   if (BYTES_BIG_ENDIAN)
2528     aarch64_split_simd_combine (operands[0], operands[2], operands[1]);
2529   else
2530     aarch64_split_simd_combine (operands[0], operands[1], operands[2]);
2531   DONE;
2532 }
2533 [(set_attr "type" "multiple")]
2534 )
2535
2536 (define_expand "aarch64_simd_combine<mode>"
2537   [(match_operand:<VDBL> 0 "register_operand")
2538    (match_operand:VDC 1 "register_operand")
2539    (match_operand:VDC 2 "register_operand")]
2540   "TARGET_SIMD"
2541   {
2542     emit_insn (gen_move_lo_quad_<Vdbl> (operands[0], operands[1]));
2543     emit_insn (gen_move_hi_quad_<Vdbl> (operands[0], operands[2]));
2544     DONE;
2545   }
2546 [(set_attr "type" "multiple")]
2547 )
2548
2549 ;; <su><addsub>l<q>.
2550
2551 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>_hi_internal"
2552  [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2553        (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2554                            (match_operand:VQW 1 "register_operand" "w")
2555                            (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
2556                        (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2557                            (match_operand:VQW 2 "register_operand" "w")
2558                            (match_dup 3)))))]
2559   "TARGET_SIMD"
2560   "<ANY_EXTEND:su><ADDSUB:optab>l2\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
2561   [(set_attr "type" "neon_<ADDSUB:optab>_long")]
2562 )
2563
2564 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>_lo_internal"
2565  [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2566        (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2567                            (match_operand:VQW 1 "register_operand" "w")
2568                            (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
2569                        (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2570                            (match_operand:VQW 2 "register_operand" "w")
2571                            (match_dup 3)))))]
2572   "TARGET_SIMD"
2573   "<ANY_EXTEND:su><ADDSUB:optab>l\t%0.<Vwtype>, %1.<Vhalftype>, %2.<Vhalftype>"
2574   [(set_attr "type" "neon_<ADDSUB:optab>_long")]
2575 )
2576
2577
2578 (define_expand "aarch64_saddl2<mode>"
2579   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2580    (match_operand:VQW 1 "register_operand" "w")
2581    (match_operand:VQW 2 "register_operand" "w")]
2582   "TARGET_SIMD"
2583 {
2584   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2585   emit_insn (gen_aarch64_saddl<mode>_hi_internal (operands[0], operands[1],
2586                                                   operands[2], p));
2587   DONE;
2588 })
2589
2590 (define_expand "aarch64_uaddl2<mode>"
2591   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2592    (match_operand:VQW 1 "register_operand" "w")
2593    (match_operand:VQW 2 "register_operand" "w")]
2594   "TARGET_SIMD"
2595 {
2596   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2597   emit_insn (gen_aarch64_uaddl<mode>_hi_internal (operands[0], operands[1],
2598                                                   operands[2], p));
2599   DONE;
2600 })
2601
2602 (define_expand "aarch64_ssubl2<mode>"
2603   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2604    (match_operand:VQW 1 "register_operand" "w")
2605    (match_operand:VQW 2 "register_operand" "w")]
2606   "TARGET_SIMD"
2607 {
2608   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2609   emit_insn (gen_aarch64_ssubl<mode>_hi_internal (operands[0], operands[1],
2610                                                 operands[2], p));
2611   DONE;
2612 })
2613
2614 (define_expand "aarch64_usubl2<mode>"
2615   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2616    (match_operand:VQW 1 "register_operand" "w")
2617    (match_operand:VQW 2 "register_operand" "w")]
2618   "TARGET_SIMD"
2619 {
2620   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2621   emit_insn (gen_aarch64_usubl<mode>_hi_internal (operands[0], operands[1],
2622                                                 operands[2], p));
2623   DONE;
2624 })
2625
2626 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>"
2627  [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2628        (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE>
2629                            (match_operand:VDW 1 "register_operand" "w"))
2630                        (ANY_EXTEND:<VWIDE>
2631                            (match_operand:VDW 2 "register_operand" "w"))))]
2632   "TARGET_SIMD"
2633   "<ANY_EXTEND:su><ADDSUB:optab>l %0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
2634   [(set_attr "type" "neon_<ADDSUB:optab>_long")]
2635 )
2636
2637 ;; <su><addsub>w<q>.
2638
2639 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>w<mode>"
2640   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2641         (ADDSUB:<VWIDE> (match_operand:<VWIDE> 1 "register_operand" "w")
2642                         (ANY_EXTEND:<VWIDE>
2643                           (match_operand:VDW 2 "register_operand" "w"))))]
2644   "TARGET_SIMD"
2645   "<ANY_EXTEND:su><ADDSUB:optab>w\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vtype>"
2646   [(set_attr "type" "neon_<ADDSUB:optab>_widen")]
2647 )
2648
2649 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>w2<mode>_internal"
2650   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2651         (ADDSUB:<VWIDE> (match_operand:<VWIDE> 1 "register_operand" "w")
2652                         (ANY_EXTEND:<VWIDE>
2653                           (vec_select:<VHALF>
2654                            (match_operand:VQW 2 "register_operand" "w")
2655                            (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))))]
2656   "TARGET_SIMD"
2657   "<ANY_EXTEND:su><ADDSUB:optab>w2\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vtype>"
2658   [(set_attr "type" "neon_<ADDSUB:optab>_widen")]
2659 )
2660
2661 (define_expand "aarch64_saddw2<mode>"
2662   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2663    (match_operand:<VWIDE> 1 "register_operand" "w")
2664    (match_operand:VQW 2 "register_operand" "w")]
2665   "TARGET_SIMD"
2666 {
2667   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2668   emit_insn (gen_aarch64_saddw2<mode>_internal (operands[0], operands[1],
2669                                                 operands[2], p));
2670   DONE;
2671 })
2672
2673 (define_expand "aarch64_uaddw2<mode>"
2674   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2675    (match_operand:<VWIDE> 1 "register_operand" "w")
2676    (match_operand:VQW 2 "register_operand" "w")]
2677   "TARGET_SIMD"
2678 {
2679   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2680   emit_insn (gen_aarch64_uaddw2<mode>_internal (operands[0], operands[1],
2681                                                 operands[2], p));
2682   DONE;
2683 })
2684
2685
2686 (define_expand "aarch64_ssubw2<mode>"
2687   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2688    (match_operand:<VWIDE> 1 "register_operand" "w")
2689    (match_operand:VQW 2 "register_operand" "w")]
2690   "TARGET_SIMD"
2691 {
2692   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2693   emit_insn (gen_aarch64_ssubw2<mode>_internal (operands[0], operands[1],
2694                                                 operands[2], p));
2695   DONE;
2696 })
2697
2698 (define_expand "aarch64_usubw2<mode>"
2699   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2700    (match_operand:<VWIDE> 1 "register_operand" "w")
2701    (match_operand:VQW 2 "register_operand" "w")]
2702   "TARGET_SIMD"
2703 {
2704   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2705   emit_insn (gen_aarch64_usubw2<mode>_internal (operands[0], operands[1],
2706                                                 operands[2], p));
2707   DONE;
2708 })
2709
2710 ;; <su><r>h<addsub>.
2711
2712 (define_insn "aarch64_<sur>h<addsub><mode>"
2713   [(set (match_operand:VQ_S 0 "register_operand" "=w")
2714         (unspec:VQ_S [(match_operand:VQ_S 1 "register_operand" "w")
2715                       (match_operand:VQ_S 2 "register_operand" "w")]
2716                      HADDSUB))]
2717   "TARGET_SIMD"
2718   "<sur>h<addsub>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
2719   [(set_attr "type" "neon_<addsub>_halve<q>")]
2720 )
2721
2722 ;; <r><addsub>hn<q>.
2723
2724 (define_insn "aarch64_<sur><addsub>hn<mode>"
2725   [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
2726         (unspec:<VNARROWQ> [(match_operand:VQN 1 "register_operand" "w")
2727                             (match_operand:VQN 2 "register_operand" "w")]
2728                            ADDSUBHN))]
2729   "TARGET_SIMD"
2730   "<sur><addsub>hn\\t%0.<Vntype>, %1.<Vtype>, %2.<Vtype>"
2731   [(set_attr "type" "neon_<addsub>_halve_narrow_q")]
2732 )
2733
2734 (define_insn "aarch64_<sur><addsub>hn2<mode>"
2735   [(set (match_operand:<VNARROWQ2> 0 "register_operand" "=w")
2736         (unspec:<VNARROWQ2> [(match_operand:<VNARROWQ> 1 "register_operand" "0")
2737                              (match_operand:VQN 2 "register_operand" "w")
2738                              (match_operand:VQN 3 "register_operand" "w")]
2739                             ADDSUBHN2))]
2740   "TARGET_SIMD"
2741   "<sur><addsub>hn2\\t%0.<V2ntype>, %2.<Vtype>, %3.<Vtype>"
2742   [(set_attr "type" "neon_<addsub>_halve_narrow_q")]
2743 )
2744
2745 ;; pmul.
2746
2747 (define_insn "aarch64_pmul<mode>"
2748   [(set (match_operand:VB 0 "register_operand" "=w")
2749         (unspec:VB [(match_operand:VB 1 "register_operand" "w")
2750                     (match_operand:VB 2 "register_operand" "w")]
2751                    UNSPEC_PMUL))]
2752  "TARGET_SIMD"
2753  "pmul\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
2754   [(set_attr "type" "neon_mul_<Vetype><q>")]
2755 )
2756
2757 ;; <su>q<addsub>
2758
2759 (define_insn "aarch64_<su_optab><optab><mode>"
2760   [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
2761         (BINQOPS:VSDQ_I (match_operand:VSDQ_I 1 "register_operand" "w")
2762                           (match_operand:VSDQ_I 2 "register_operand" "w")))]
2763   "TARGET_SIMD"
2764   "<su_optab><optab>\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
2765   [(set_attr "type" "neon_<optab><q>")]
2766 )
2767
2768 ;; suqadd and usqadd
2769
2770 (define_insn "aarch64_<sur>qadd<mode>"
2771   [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
2772         (unspec:VSDQ_I [(match_operand:VSDQ_I 1 "register_operand" "0")
2773                         (match_operand:VSDQ_I 2 "register_operand" "w")]
2774                        USSUQADD))]
2775   "TARGET_SIMD"
2776   "<sur>qadd\\t%<v>0<Vmtype>, %<v>2<Vmtype>"
2777   [(set_attr "type" "neon_qadd<q>")]
2778 )
2779
2780 ;; sqmovun
2781
2782 (define_insn "aarch64_sqmovun<mode>"
2783   [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
2784         (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")]
2785                             UNSPEC_SQXTUN))]
2786    "TARGET_SIMD"
2787    "sqxtun\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>"
2788    [(set_attr "type" "neon_sat_shift_imm_narrow_q")]
2789  )
2790
2791 ;; sqmovn and uqmovn
2792
2793 (define_insn "aarch64_<sur>qmovn<mode>"
2794   [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
2795         (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")]
2796                             SUQMOVN))]
2797   "TARGET_SIMD"
2798   "<sur>qxtn\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>"
2799    [(set_attr "type" "neon_sat_shift_imm_narrow_q")]
2800  )
2801
2802 ;; <su>q<absneg>
2803
2804 (define_insn "aarch64_s<optab><mode>"
2805   [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
2806         (UNQOPS:VSDQ_I
2807           (match_operand:VSDQ_I 1 "register_operand" "w")))]
2808   "TARGET_SIMD"
2809   "s<optab>\\t%<v>0<Vmtype>, %<v>1<Vmtype>"
2810   [(set_attr "type" "neon_<optab><q>")]
2811 )
2812
2813 ;; sq<r>dmulh.
2814
2815 (define_insn "aarch64_sq<r>dmulh<mode>"
2816   [(set (match_operand:VSDQ_HSI 0 "register_operand" "=w")
2817         (unspec:VSDQ_HSI
2818           [(match_operand:VSDQ_HSI 1 "register_operand" "w")
2819            (match_operand:VSDQ_HSI 2 "register_operand" "w")]
2820          VQDMULH))]
2821   "TARGET_SIMD"
2822   "sq<r>dmulh\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
2823   [(set_attr "type" "neon_sat_mul_<Vetype><q>")]
2824 )
2825
2826 ;; sq<r>dmulh_lane
2827
2828 (define_expand "aarch64_sqdmulh_lane<mode>"
2829   [(match_operand:VDQHS 0 "register_operand" "")
2830    (match_operand:VDQHS 1 "register_operand" "")
2831    (match_operand:<VCOND> 2 "register_operand" "")
2832    (match_operand:SI 3 "immediate_operand" "")]
2833   "TARGET_SIMD"
2834   {
2835      aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCOND>mode));
2836      operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
2837      emit_insn (gen_aarch64_sqdmulh_lane<mode>_internal (operands[0],
2838                                                          operands[1],
2839                                                          operands[2],
2840                                                          operands[3]));
2841      DONE;
2842   }
2843 )
2844
2845 (define_expand "aarch64_sqrdmulh_lane<mode>"
2846   [(match_operand:VDQHS 0 "register_operand" "")
2847    (match_operand:VDQHS 1 "register_operand" "")
2848    (match_operand:<VCOND> 2 "register_operand" "")
2849    (match_operand:SI 3 "immediate_operand" "")]
2850   "TARGET_SIMD"
2851   {
2852      aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCOND>mode));
2853      operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
2854      emit_insn (gen_aarch64_sqrdmulh_lane<mode>_internal (operands[0],
2855                                                           operands[1],
2856                                                           operands[2],
2857                                                           operands[3]));
2858      DONE;
2859   }
2860 )
2861
2862 (define_insn "aarch64_sq<r>dmulh_lane<mode>_internal"
2863   [(set (match_operand:VDQHS 0 "register_operand" "=w")
2864         (unspec:VDQHS
2865           [(match_operand:VDQHS 1 "register_operand" "w")
2866            (vec_select:<VEL>
2867              (match_operand:<VCOND> 2 "register_operand" "<vwx>")
2868              (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
2869          VQDMULH))]
2870   "TARGET_SIMD"
2871   "*
2872    aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCOND>mode));
2873    operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
2874    return \"sq<r>dmulh\\t%0.<Vtype>, %1.<Vtype>, %2.<Vetype>[%3]\";"
2875   [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
2876 )
2877
2878 (define_expand "aarch64_sqdmulh_laneq<mode>"
2879   [(match_operand:VDQHS 0 "register_operand" "")
2880    (match_operand:VDQHS 1 "register_operand" "")
2881    (match_operand:<VCONQ> 2 "register_operand" "")
2882    (match_operand:SI 3 "immediate_operand" "")]
2883   "TARGET_SIMD"
2884   {
2885      aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCONQ>mode));
2886      operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
2887      emit_insn (gen_aarch64_sqdmulh_laneq<mode>_internal (operands[0],
2888                                                           operands[1],
2889                                                           operands[2],
2890                                                           operands[3]));
2891      DONE;
2892    }
2893 )
2894
2895 (define_expand "aarch64_sqrdmulh_laneq<mode>"
2896   [(match_operand:VDQHS 0 "register_operand" "")
2897    (match_operand:VDQHS 1 "register_operand" "")
2898    (match_operand:<VCONQ> 2 "register_operand" "")
2899    (match_operand:SI 3 "immediate_operand" "")]
2900   "TARGET_SIMD"
2901   {
2902      aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCONQ>mode));
2903      operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
2904      emit_insn (gen_aarch64_sqrdmulh_laneq<mode>_internal (operands[0],
2905                                                            operands[1],
2906                                                            operands[2],
2907                                                            operands[3]));
2908      DONE;
2909    }
2910 )
2911
2912 (define_insn "aarch64_sq<r>dmulh_laneq<mode>_internal"
2913   [(set (match_operand:VDQHS 0 "register_operand" "=w")
2914         (unspec:VDQHS
2915           [(match_operand:VDQHS 1 "register_operand" "w")
2916            (vec_select:<VEL>
2917              (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
2918              (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
2919          VQDMULH))]
2920   "TARGET_SIMD"
2921   "*
2922    operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
2923    return \"sq<r>dmulh\\t%0.<Vtype>, %1.<Vtype>, %2.<Vetype>[%3]\";"
2924   [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
2925 )
2926
2927 (define_expand "aarch64_sqdmulh_lane<mode>"
2928   [(match_operand:SD_HSI 0 "register_operand" "")
2929    (match_operand:SD_HSI 1 "register_operand" "")
2930    (match_operand:<VCOND> 2 "register_operand" "")
2931    (match_operand:SI 3 "immediate_operand" "")]
2932   "TARGET_SIMD"
2933   {
2934     aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCOND>mode));
2935     operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
2936     emit_insn (gen_aarch64_sqdmulh_lane<mode>_internal (operands[0],
2937                                                         operands[1],
2938                                                         operands[2],
2939                                                         operands[3]));
2940     DONE;
2941   }
2942 )
2943
2944 (define_expand "aarch64_sqrdmulh_lane<mode>"
2945   [(match_operand:SD_HSI 0 "register_operand" "")
2946    (match_operand:SD_HSI 1 "register_operand" "")
2947    (match_operand:<VCOND> 2 "register_operand" "")
2948    (match_operand:SI 3 "immediate_operand" "")]
2949   "TARGET_SIMD"
2950   {
2951     aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCOND>mode));
2952     operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
2953     emit_insn (gen_aarch64_sqrdmulh_lane<mode>_internal (operands[0],
2954                                                          operands[1],
2955                                                          operands[2],
2956                                                          operands[3]));
2957     DONE;
2958   }
2959 )
2960
2961 (define_insn "aarch64_sq<r>dmulh_lane<mode>_internal"
2962   [(set (match_operand:SD_HSI 0 "register_operand" "=w")
2963         (unspec:SD_HSI
2964           [(match_operand:SD_HSI 1 "register_operand" "w")
2965            (vec_select:<VEL>
2966              (match_operand:<VCOND> 2 "register_operand" "<vwx>")
2967              (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
2968          VQDMULH))]
2969   "TARGET_SIMD"
2970   "*
2971    operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
2972    return \"sq<r>dmulh\\t%<v>0, %<v>1, %2.<v>[%3]\";"
2973   [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
2974 )
2975
2976 ;; vqdml[sa]l
2977
2978 (define_insn "aarch64_sqdml<SBINQOPS:as>l<mode>"
2979   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2980         (SBINQOPS:<VWIDE>
2981           (match_operand:<VWIDE> 1 "register_operand" "0")
2982           (ss_ashift:<VWIDE>
2983               (mult:<VWIDE>
2984                 (sign_extend:<VWIDE>
2985                       (match_operand:VSD_HSI 2 "register_operand" "w"))
2986                 (sign_extend:<VWIDE>
2987                       (match_operand:VSD_HSI 3 "register_operand" "w")))
2988               (const_int 1))))]
2989   "TARGET_SIMD"
2990   "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %<v>3<Vmtype>"
2991   [(set_attr "type" "neon_sat_mla_<Vetype>_long")]
2992 )
2993
2994 ;; vqdml[sa]l_lane
2995
2996 (define_insn "aarch64_sqdml<SBINQOPS:as>l_lane<mode>_internal"
2997   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2998         (SBINQOPS:<VWIDE>
2999           (match_operand:<VWIDE> 1 "register_operand" "0")
3000           (ss_ashift:<VWIDE>
3001             (mult:<VWIDE>
3002               (sign_extend:<VWIDE>
3003                 (match_operand:VD_HSI 2 "register_operand" "w"))
3004               (sign_extend:<VWIDE>
3005                 (vec_duplicate:VD_HSI
3006                   (vec_select:<VEL>
3007                     (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3008                     (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
3009               ))
3010             (const_int 1))))]
3011   "TARGET_SIMD"
3012   {
3013     operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3014     return
3015       "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3016   }
3017   [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3018 )
3019
3020 (define_insn "aarch64_sqdml<SBINQOPS:as>l_laneq<mode>_internal"
3021   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3022         (SBINQOPS:<VWIDE>
3023           (match_operand:<VWIDE> 1 "register_operand" "0")
3024           (ss_ashift:<VWIDE>
3025             (mult:<VWIDE>
3026               (sign_extend:<VWIDE>
3027                 (match_operand:VD_HSI 2 "register_operand" "w"))
3028               (sign_extend:<VWIDE>
3029                 (vec_duplicate:VD_HSI
3030                   (vec_select:<VEL>
3031                     (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3032                     (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
3033               ))
3034             (const_int 1))))]
3035   "TARGET_SIMD"
3036   {
3037     operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3038     return
3039       "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3040   }
3041   [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3042 )
3043
3044 (define_insn "aarch64_sqdml<SBINQOPS:as>l_lane<mode>_internal"
3045   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3046         (SBINQOPS:<VWIDE>
3047           (match_operand:<VWIDE> 1 "register_operand" "0")
3048           (ss_ashift:<VWIDE>
3049             (mult:<VWIDE>
3050               (sign_extend:<VWIDE>
3051                 (match_operand:SD_HSI 2 "register_operand" "w"))
3052               (sign_extend:<VWIDE>
3053                 (vec_select:<VEL>
3054                   (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3055                   (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
3056               )
3057             (const_int 1))))]
3058   "TARGET_SIMD"
3059   {
3060     operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3061     return
3062       "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3063   }
3064   [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3065 )
3066
3067 (define_insn "aarch64_sqdml<SBINQOPS:as>l_laneq<mode>_internal"
3068   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3069         (SBINQOPS:<VWIDE>
3070           (match_operand:<VWIDE> 1 "register_operand" "0")
3071           (ss_ashift:<VWIDE>
3072             (mult:<VWIDE>
3073               (sign_extend:<VWIDE>
3074                 (match_operand:SD_HSI 2 "register_operand" "w"))
3075               (sign_extend:<VWIDE>
3076                 (vec_select:<VEL>
3077                   (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3078                   (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
3079               )
3080             (const_int 1))))]
3081   "TARGET_SIMD"
3082   {
3083     operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3084     return
3085       "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3086   }
3087   [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3088 )
3089
3090 (define_expand "aarch64_sqdmlal_lane<mode>"
3091   [(match_operand:<VWIDE> 0 "register_operand" "=w")
3092    (match_operand:<VWIDE> 1 "register_operand" "0")
3093    (match_operand:VSD_HSI 2 "register_operand" "w")
3094    (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3095    (match_operand:SI 4 "immediate_operand" "i")]
3096   "TARGET_SIMD"
3097 {
3098   aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCOND>mode));
3099   operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3100   emit_insn (gen_aarch64_sqdmlal_lane<mode>_internal (operands[0], operands[1],
3101                                                       operands[2], operands[3],
3102                                                       operands[4]));
3103   DONE;
3104 })
3105
3106 (define_expand "aarch64_sqdmlal_laneq<mode>"
3107   [(match_operand:<VWIDE> 0 "register_operand" "=w")
3108    (match_operand:<VWIDE> 1 "register_operand" "0")
3109    (match_operand:VSD_HSI 2 "register_operand" "w")
3110    (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3111    (match_operand:SI 4 "immediate_operand" "i")]
3112   "TARGET_SIMD"
3113 {
3114   aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCONQ>mode));
3115   operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3116   emit_insn (gen_aarch64_sqdmlal_laneq<mode>_internal (operands[0], operands[1],
3117                                                       operands[2], operands[3],
3118                                                       operands[4]));
3119   DONE;
3120 })
3121
3122 (define_expand "aarch64_sqdmlsl_lane<mode>"
3123   [(match_operand:<VWIDE> 0 "register_operand" "=w")
3124    (match_operand:<VWIDE> 1 "register_operand" "0")
3125    (match_operand:VSD_HSI 2 "register_operand" "w")
3126    (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3127    (match_operand:SI 4 "immediate_operand" "i")]
3128   "TARGET_SIMD"
3129 {
3130   aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCOND>mode));
3131   operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3132   emit_insn (gen_aarch64_sqdmlsl_lane<mode>_internal (operands[0], operands[1],
3133                                                       operands[2], operands[3],
3134                                                       operands[4]));
3135   DONE;
3136 })
3137
3138 (define_expand "aarch64_sqdmlsl_laneq<mode>"
3139   [(match_operand:<VWIDE> 0 "register_operand" "=w")
3140    (match_operand:<VWIDE> 1 "register_operand" "0")
3141    (match_operand:VSD_HSI 2 "register_operand" "w")
3142    (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3143    (match_operand:SI 4 "immediate_operand" "i")]
3144   "TARGET_SIMD"
3145 {
3146   aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCONQ>mode));
3147   operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3148   emit_insn (gen_aarch64_sqdmlsl_laneq<mode>_internal (operands[0], operands[1],
3149                                                       operands[2], operands[3],
3150                                                       operands[4]));
3151   DONE;
3152 })
3153
3154 ;; vqdml[sa]l_n
3155
3156 (define_insn "aarch64_sqdml<SBINQOPS:as>l_n<mode>"
3157   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3158         (SBINQOPS:<VWIDE>
3159           (match_operand:<VWIDE> 1 "register_operand" "0")
3160           (ss_ashift:<VWIDE>
3161               (mult:<VWIDE>
3162                 (sign_extend:<VWIDE>
3163                       (match_operand:VD_HSI 2 "register_operand" "w"))
3164                 (sign_extend:<VWIDE>
3165                   (vec_duplicate:VD_HSI
3166                     (match_operand:<VEL> 3 "register_operand" "<vwx>"))))
3167               (const_int 1))))]
3168   "TARGET_SIMD"
3169   "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[0]"
3170   [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3171 )
3172
3173 ;; sqdml[as]l2
3174
3175 (define_insn "aarch64_sqdml<SBINQOPS:as>l2<mode>_internal"
3176   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3177         (SBINQOPS:<VWIDE>
3178          (match_operand:<VWIDE> 1 "register_operand" "0")
3179          (ss_ashift:<VWIDE>
3180              (mult:<VWIDE>
3181                (sign_extend:<VWIDE>
3182                  (vec_select:<VHALF>
3183                      (match_operand:VQ_HSI 2 "register_operand" "w")
3184                      (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3185                (sign_extend:<VWIDE>
3186                  (vec_select:<VHALF>
3187                      (match_operand:VQ_HSI 3 "register_operand" "w")
3188                      (match_dup 4))))
3189              (const_int 1))))]
3190   "TARGET_SIMD"
3191   "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %<v>3<Vmtype>"
3192   [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3193 )
3194
3195 (define_expand "aarch64_sqdmlal2<mode>"
3196   [(match_operand:<VWIDE> 0 "register_operand" "=w")
3197    (match_operand:<VWIDE> 1 "register_operand" "w")
3198    (match_operand:VQ_HSI 2 "register_operand" "w")
3199    (match_operand:VQ_HSI 3 "register_operand" "w")]
3200   "TARGET_SIMD"
3201 {
3202   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3203   emit_insn (gen_aarch64_sqdmlal2<mode>_internal (operands[0], operands[1],
3204                                                   operands[2], operands[3], p));
3205   DONE;
3206 })
3207
3208 (define_expand "aarch64_sqdmlsl2<mode>"
3209   [(match_operand:<VWIDE> 0 "register_operand" "=w")
3210    (match_operand:<VWIDE> 1 "register_operand" "w")
3211    (match_operand:VQ_HSI 2 "register_operand" "w")
3212    (match_operand:VQ_HSI 3 "register_operand" "w")]
3213   "TARGET_SIMD"
3214 {
3215   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3216   emit_insn (gen_aarch64_sqdmlsl2<mode>_internal (operands[0], operands[1],
3217                                                   operands[2], operands[3], p));
3218   DONE;
3219 })
3220
3221 ;; vqdml[sa]l2_lane
3222
3223 (define_insn "aarch64_sqdml<SBINQOPS:as>l2_lane<mode>_internal"
3224   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3225         (SBINQOPS:<VWIDE>
3226           (match_operand:<VWIDE> 1 "register_operand" "0")
3227           (ss_ashift:<VWIDE>
3228               (mult:<VWIDE>
3229                 (sign_extend:<VWIDE>
3230                   (vec_select:<VHALF>
3231                     (match_operand:VQ_HSI 2 "register_operand" "w")
3232                     (match_operand:VQ_HSI 5 "vect_par_cnst_hi_half" "")))
3233                 (sign_extend:<VWIDE>
3234                   (vec_duplicate:<VHALF>
3235                     (vec_select:<VEL>
3236                       (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3237                       (parallel [(match_operand:SI 4 "immediate_operand" "i")])
3238                     ))))
3239               (const_int 1))))]
3240   "TARGET_SIMD"
3241   {
3242     operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3243     return
3244      "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3245   }
3246   [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3247 )
3248
3249 (define_insn "aarch64_sqdml<SBINQOPS:as>l2_laneq<mode>_internal"
3250   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3251         (SBINQOPS:<VWIDE>
3252           (match_operand:<VWIDE> 1 "register_operand" "0")
3253           (ss_ashift:<VWIDE>
3254               (mult:<VWIDE>
3255                 (sign_extend:<VWIDE>
3256                   (vec_select:<VHALF>
3257                     (match_operand:VQ_HSI 2 "register_operand" "w")
3258                     (match_operand:VQ_HSI 5 "vect_par_cnst_hi_half" "")))
3259                 (sign_extend:<VWIDE>
3260                   (vec_duplicate:<VHALF>
3261                     (vec_select:<VEL>
3262                       (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3263                       (parallel [(match_operand:SI 4 "immediate_operand" "i")])
3264                     ))))
3265               (const_int 1))))]
3266   "TARGET_SIMD"
3267   {
3268     operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3269     return
3270      "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3271   }
3272   [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3273 )
3274
3275 (define_expand "aarch64_sqdmlal2_lane<mode>"
3276   [(match_operand:<VWIDE> 0 "register_operand" "=w")
3277    (match_operand:<VWIDE> 1 "register_operand" "w")
3278    (match_operand:VQ_HSI 2 "register_operand" "w")
3279    (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3280    (match_operand:SI 4 "immediate_operand" "i")]
3281   "TARGET_SIMD"
3282 {
3283   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3284   aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCOND>mode));
3285   operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3286   emit_insn (gen_aarch64_sqdmlal2_lane<mode>_internal (operands[0], operands[1],
3287                                                        operands[2], operands[3],
3288                                                        operands[4], p));
3289   DONE;
3290 })
3291
3292 (define_expand "aarch64_sqdmlal2_laneq<mode>"
3293   [(match_operand:<VWIDE> 0 "register_operand" "=w")
3294    (match_operand:<VWIDE> 1 "register_operand" "w")
3295    (match_operand:VQ_HSI 2 "register_operand" "w")
3296    (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3297    (match_operand:SI 4 "immediate_operand" "i")]
3298   "TARGET_SIMD"
3299 {
3300   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3301   aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCONQ>mode));
3302   operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3303   emit_insn (gen_aarch64_sqdmlal2_laneq<mode>_internal (operands[0], operands[1],
3304                                                        operands[2], operands[3],
3305                                                        operands[4], p));
3306   DONE;
3307 })
3308
3309 (define_expand "aarch64_sqdmlsl2_lane<mode>"
3310   [(match_operand:<VWIDE> 0 "register_operand" "=w")
3311    (match_operand:<VWIDE> 1 "register_operand" "w")
3312    (match_operand:VQ_HSI 2 "register_operand" "w")
3313    (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3314    (match_operand:SI 4 "immediate_operand" "i")]
3315   "TARGET_SIMD"
3316 {
3317   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3318   aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCOND>mode));
3319   operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3320   emit_insn (gen_aarch64_sqdmlsl2_lane<mode>_internal (operands[0], operands[1],
3321                                                        operands[2], operands[3],
3322                                                        operands[4], p));
3323   DONE;
3324 })
3325
3326 (define_expand "aarch64_sqdmlsl2_laneq<mode>"
3327   [(match_operand:<VWIDE> 0 "register_operand" "=w")
3328    (match_operand:<VWIDE> 1 "register_operand" "w")
3329    (match_operand:VQ_HSI 2 "register_operand" "w")
3330    (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3331    (match_operand:SI 4 "immediate_operand" "i")]
3332   "TARGET_SIMD"
3333 {
3334   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3335   aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCONQ>mode));
3336   operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3337   emit_insn (gen_aarch64_sqdmlsl2_laneq<mode>_internal (operands[0], operands[1],
3338                                                        operands[2], operands[3],
3339                                                        operands[4], p));
3340   DONE;
3341 })
3342
3343 (define_insn "aarch64_sqdml<SBINQOPS:as>l2_n<mode>_internal"
3344   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3345         (SBINQOPS:<VWIDE>
3346           (match_operand:<VWIDE> 1 "register_operand" "0")
3347           (ss_ashift:<VWIDE>
3348             (mult:<VWIDE>
3349               (sign_extend:<VWIDE>
3350                 (vec_select:<VHALF>
3351                   (match_operand:VQ_HSI 2 "register_operand" "w")
3352                   (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3353               (sign_extend:<VWIDE>
3354                 (vec_duplicate:<VHALF>
3355                   (match_operand:<VEL> 3 "register_operand" "<vwx>"))))
3356             (const_int 1))))]
3357   "TARGET_SIMD"
3358   "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[0]"
3359   [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3360 )
3361
3362 (define_expand "aarch64_sqdmlal2_n<mode>"
3363   [(match_operand:<VWIDE> 0 "register_operand" "=w")
3364    (match_operand:<VWIDE> 1 "register_operand" "w")
3365    (match_operand:VQ_HSI 2 "register_operand" "w")
3366    (match_operand:<VEL> 3 "register_operand" "w")]
3367   "TARGET_SIMD"
3368 {
3369   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3370   emit_insn (gen_aarch64_sqdmlal2_n<mode>_internal (operands[0], operands[1],
3371                                                     operands[2], operands[3],
3372                                                     p));
3373   DONE;
3374 })
3375
3376 (define_expand "aarch64_sqdmlsl2_n<mode>"
3377   [(match_operand:<VWIDE> 0 "register_operand" "=w")
3378    (match_operand:<VWIDE> 1 "register_operand" "w")
3379    (match_operand:VQ_HSI 2 "register_operand" "w")
3380    (match_operand:<VEL> 3 "register_operand" "w")]
3381   "TARGET_SIMD"
3382 {
3383   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3384   emit_insn (gen_aarch64_sqdmlsl2_n<mode>_internal (operands[0], operands[1],
3385                                                     operands[2], operands[3],
3386                                                     p));
3387   DONE;
3388 })
3389
3390 ;; vqdmull
3391
3392 (define_insn "aarch64_sqdmull<mode>"
3393   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3394         (ss_ashift:<VWIDE>
3395              (mult:<VWIDE>
3396                (sign_extend:<VWIDE>
3397                      (match_operand:VSD_HSI 1 "register_operand" "w"))
3398                (sign_extend:<VWIDE>
3399                      (match_operand:VSD_HSI 2 "register_operand" "w")))
3400              (const_int 1)))]
3401   "TARGET_SIMD"
3402   "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3403   [(set_attr "type" "neon_sat_mul_<Vetype>_long")]
3404 )
3405
3406 ;; vqdmull_lane
3407
3408 (define_insn "aarch64_sqdmull_lane<mode>_internal"
3409   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3410         (ss_ashift:<VWIDE>
3411              (mult:<VWIDE>
3412                (sign_extend:<VWIDE>
3413                  (match_operand:VD_HSI 1 "register_operand" "w"))
3414                (sign_extend:<VWIDE>
3415                  (vec_duplicate:VD_HSI
3416                    (vec_select:<VEL>
3417                      (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3418                      (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3419                ))
3420              (const_int 1)))]
3421   "TARGET_SIMD"
3422   {
3423     operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3424     return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3425   }
3426   [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3427 )
3428
3429 (define_insn "aarch64_sqdmull_laneq<mode>_internal"
3430   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3431         (ss_ashift:<VWIDE>
3432              (mult:<VWIDE>
3433                (sign_extend:<VWIDE>
3434                  (match_operand:VD_HSI 1 "register_operand" "w"))
3435                (sign_extend:<VWIDE>
3436                  (vec_duplicate:VD_HSI
3437                    (vec_select:<VEL>
3438                      (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3439                      (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3440                ))
3441              (const_int 1)))]
3442   "TARGET_SIMD"
3443   {
3444     operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3445     return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3446   }
3447   [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3448 )
3449
3450 (define_insn "aarch64_sqdmull_lane<mode>_internal"
3451   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3452         (ss_ashift:<VWIDE>
3453              (mult:<VWIDE>
3454                (sign_extend:<VWIDE>
3455                  (match_operand:SD_HSI 1 "register_operand" "w"))
3456                (sign_extend:<VWIDE>
3457                  (vec_select:<VEL>
3458                    (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3459                    (parallel [(match_operand:SI 3 "immediate_operand" "i")]))
3460                ))
3461              (const_int 1)))]
3462   "TARGET_SIMD"
3463   {
3464     operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3465     return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3466   }
3467   [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3468 )
3469
3470 (define_insn "aarch64_sqdmull_laneq<mode>_internal"
3471   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3472         (ss_ashift:<VWIDE>
3473              (mult:<VWIDE>
3474                (sign_extend:<VWIDE>
3475                  (match_operand:SD_HSI 1 "register_operand" "w"))
3476                (sign_extend:<VWIDE>
3477                  (vec_select:<VEL>
3478                    (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3479                    (parallel [(match_operand:SI 3 "immediate_operand" "i")]))
3480                ))
3481              (const_int 1)))]
3482   "TARGET_SIMD"
3483   {
3484     operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3485     return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3486   }
3487   [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3488 )
3489
3490 (define_expand "aarch64_sqdmull_lane<mode>"
3491   [(match_operand:<VWIDE> 0 "register_operand" "=w")
3492    (match_operand:VSD_HSI 1 "register_operand" "w")
3493    (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3494    (match_operand:SI 3 "immediate_operand" "i")]
3495   "TARGET_SIMD"
3496 {
3497   aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCOND>mode));
3498   operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3499   emit_insn (gen_aarch64_sqdmull_lane<mode>_internal (operands[0], operands[1],
3500                                                       operands[2], operands[3]));
3501   DONE;
3502 })
3503
3504 (define_expand "aarch64_sqdmull_laneq<mode>"
3505   [(match_operand:<VWIDE> 0 "register_operand" "=w")
3506    (match_operand:VD_HSI 1 "register_operand" "w")
3507    (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3508    (match_operand:SI 3 "immediate_operand" "i")]
3509   "TARGET_SIMD"
3510 {
3511   aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCONQ>mode));
3512   operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3513   emit_insn (gen_aarch64_sqdmull_laneq<mode>_internal
3514                (operands[0], operands[1], operands[2], operands[3]));
3515   DONE;
3516 })
3517
3518 ;; vqdmull_n
3519
3520 (define_insn "aarch64_sqdmull_n<mode>"
3521   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3522         (ss_ashift:<VWIDE>
3523              (mult:<VWIDE>
3524                (sign_extend:<VWIDE>
3525                  (match_operand:VD_HSI 1 "register_operand" "w"))
3526                (sign_extend:<VWIDE>
3527                  (vec_duplicate:VD_HSI
3528                    (match_operand:<VEL> 2 "register_operand" "<vwx>")))
3529                )
3530              (const_int 1)))]
3531   "TARGET_SIMD"
3532   "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[0]"
3533   [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3534 )
3535
3536 ;; vqdmull2
3537
3538
3539
3540 (define_insn "aarch64_sqdmull2<mode>_internal"
3541   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3542         (ss_ashift:<VWIDE>
3543              (mult:<VWIDE>
3544                (sign_extend:<VWIDE>
3545                  (vec_select:<VHALF>
3546                    (match_operand:VQ_HSI 1 "register_operand" "w")
3547                    (match_operand:VQ_HSI 3 "vect_par_cnst_hi_half" "")))
3548                (sign_extend:<VWIDE>
3549                  (vec_select:<VHALF>
3550                    (match_operand:VQ_HSI 2 "register_operand" "w")
3551                    (match_dup 3)))
3552                )
3553              (const_int 1)))]
3554   "TARGET_SIMD"
3555   "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3556   [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3557 )
3558
3559 (define_expand "aarch64_sqdmull2<mode>"
3560   [(match_operand:<VWIDE> 0 "register_operand" "=w")
3561    (match_operand:VQ_HSI 1 "register_operand" "w")
3562    (match_operand:VQ_HSI 2 "register_operand" "w")]
3563   "TARGET_SIMD"
3564 {
3565   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3566   emit_insn (gen_aarch64_sqdmull2<mode>_internal (operands[0], operands[1],
3567                                                   operands[2], p));
3568   DONE;
3569 })
3570
3571 ;; vqdmull2_lane
3572
3573 (define_insn "aarch64_sqdmull2_lane<mode>_internal"
3574   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3575         (ss_ashift:<VWIDE>
3576              (mult:<VWIDE>
3577                (sign_extend:<VWIDE>
3578                  (vec_select:<VHALF>
3579                    (match_operand:VQ_HSI 1 "register_operand" "w")
3580                    (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3581                (sign_extend:<VWIDE>
3582                  (vec_duplicate:<VHALF>
3583                    (vec_select:<VEL>
3584                      (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3585                      (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3586                ))
3587              (const_int 1)))]
3588   "TARGET_SIMD"
3589   {
3590     operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3591     return "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3592   }
3593   [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3594 )
3595
3596 (define_insn "aarch64_sqdmull2_laneq<mode>_internal"
3597   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3598         (ss_ashift:<VWIDE>
3599              (mult:<VWIDE>
3600                (sign_extend:<VWIDE>
3601                  (vec_select:<VHALF>
3602                    (match_operand:VQ_HSI 1 "register_operand" "w")
3603                    (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3604                (sign_extend:<VWIDE>
3605                  (vec_duplicate:<VHALF>
3606                    (vec_select:<VEL>
3607                      (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3608                      (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3609                ))
3610              (const_int 1)))]
3611   "TARGET_SIMD"
3612   {
3613     operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3614     return "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3615   }
3616   [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3617 )
3618
3619 (define_expand "aarch64_sqdmull2_lane<mode>"
3620   [(match_operand:<VWIDE> 0 "register_operand" "=w")
3621    (match_operand:VQ_HSI 1 "register_operand" "w")
3622    (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3623    (match_operand:SI 3 "immediate_operand" "i")]
3624   "TARGET_SIMD"
3625 {
3626   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3627   aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCOND>mode));
3628   operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3629   emit_insn (gen_aarch64_sqdmull2_lane<mode>_internal (operands[0], operands[1],
3630                                                        operands[2], operands[3],
3631                                                        p));
3632   DONE;
3633 })
3634
3635 (define_expand "aarch64_sqdmull2_laneq<mode>"
3636   [(match_operand:<VWIDE> 0 "register_operand" "=w")
3637    (match_operand:VQ_HSI 1 "register_operand" "w")
3638    (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3639    (match_operand:SI 3 "immediate_operand" "i")]
3640   "TARGET_SIMD"
3641 {
3642   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3643   aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCONQ>mode));
3644   operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3645   emit_insn (gen_aarch64_sqdmull2_laneq<mode>_internal (operands[0], operands[1],
3646                                                        operands[2], operands[3],
3647                                                        p));
3648   DONE;
3649 })
3650
3651 ;; vqdmull2_n
3652
3653 (define_insn "aarch64_sqdmull2_n<mode>_internal"
3654   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3655         (ss_ashift:<VWIDE>
3656              (mult:<VWIDE>
3657                (sign_extend:<VWIDE>
3658                  (vec_select:<VHALF>
3659                    (match_operand:VQ_HSI 1 "register_operand" "w")
3660                    (match_operand:VQ_HSI 3 "vect_par_cnst_hi_half" "")))
3661                (sign_extend:<VWIDE>
3662                  (vec_duplicate:<VHALF>
3663                    (match_operand:<VEL> 2 "register_operand" "<vwx>")))
3664                )
3665              (const_int 1)))]
3666   "TARGET_SIMD"
3667   "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[0]"
3668   [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3669 )
3670
3671 (define_expand "aarch64_sqdmull2_n<mode>"
3672   [(match_operand:<VWIDE> 0 "register_operand" "=w")
3673    (match_operand:VQ_HSI 1 "register_operand" "w")
3674    (match_operand:<VEL> 2 "register_operand" "w")]
3675   "TARGET_SIMD"
3676 {
3677   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3678   emit_insn (gen_aarch64_sqdmull2_n<mode>_internal (operands[0], operands[1],
3679                                                     operands[2], p));
3680   DONE;
3681 })
3682
3683 ;; vshl
3684
3685 (define_insn "aarch64_<sur>shl<mode>"
3686   [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3687         (unspec:VSDQ_I_DI
3688           [(match_operand:VSDQ_I_DI 1 "register_operand" "w")
3689            (match_operand:VSDQ_I_DI 2 "register_operand" "w")]
3690          VSHL))]
3691   "TARGET_SIMD"
3692   "<sur>shl\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>";
3693   [(set_attr "type" "neon_shift_reg<q>")]
3694 )
3695
3696
3697 ;; vqshl
3698
3699 (define_insn "aarch64_<sur>q<r>shl<mode>"
3700   [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
3701         (unspec:VSDQ_I
3702           [(match_operand:VSDQ_I 1 "register_operand" "w")
3703            (match_operand:VSDQ_I 2 "register_operand" "w")]
3704          VQSHL))]
3705   "TARGET_SIMD"
3706   "<sur>q<r>shl\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>";
3707   [(set_attr "type" "neon_sat_shift_reg<q>")]
3708 )
3709
3710 ;; vshll_n
3711
3712 (define_insn "aarch64_<sur>shll_n<mode>"
3713   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3714         (unspec:<VWIDE> [(match_operand:VDW 1 "register_operand" "w")
3715                          (match_operand:SI 2 "immediate_operand" "i")]
3716                          VSHLL))]
3717   "TARGET_SIMD"
3718   "*
3719   int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3720   aarch64_simd_const_bounds (operands[2], 0, bit_width + 1);
3721   if (INTVAL (operands[2]) == bit_width)
3722   {
3723     return \"shll\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3724   }
3725   else {
3726     return \"<sur>shll\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3727   }"
3728   [(set_attr "type" "neon_shift_imm_long")]
3729 )
3730
3731 ;; vshll_high_n
3732
3733 (define_insn "aarch64_<sur>shll2_n<mode>"
3734   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3735         (unspec:<VWIDE> [(match_operand:VQW 1 "register_operand" "w")
3736                          (match_operand:SI 2 "immediate_operand" "i")]
3737                          VSHLL))]
3738   "TARGET_SIMD"
3739   "*
3740   int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3741   aarch64_simd_const_bounds (operands[2], 0, bit_width + 1);
3742   if (INTVAL (operands[2]) == bit_width)
3743   {
3744     return \"shll2\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3745   }
3746   else {
3747     return \"<sur>shll2\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3748   }"
3749   [(set_attr "type" "neon_shift_imm_long")]
3750 )
3751
3752 ;; vrshr_n
3753
3754 (define_insn "aarch64_<sur>shr_n<mode>"
3755   [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3756         (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "w")
3757                            (match_operand:SI 2 "immediate_operand" "i")]
3758                           VRSHR_N))]
3759   "TARGET_SIMD"
3760   "*
3761   int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3762   aarch64_simd_const_bounds (operands[2], 1, bit_width + 1);
3763   return \"<sur>shr\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2\";"
3764   [(set_attr "type" "neon_sat_shift_imm<q>")]
3765 )
3766
3767 ;; v(r)sra_n
3768
3769 (define_insn "aarch64_<sur>sra_n<mode>"
3770   [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3771         (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "0")
3772                        (match_operand:VSDQ_I_DI 2 "register_operand" "w")
3773                        (match_operand:SI 3 "immediate_operand" "i")]
3774                       VSRA))]
3775   "TARGET_SIMD"
3776   "*
3777   int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3778   aarch64_simd_const_bounds (operands[3], 1, bit_width + 1);
3779   return \"<sur>sra\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %3\";"
3780   [(set_attr "type" "neon_shift_acc<q>")]
3781 )
3782
3783 ;; vs<lr>i_n
3784
3785 (define_insn "aarch64_<sur>s<lr>i_n<mode>"
3786   [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3787         (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "0")
3788                        (match_operand:VSDQ_I_DI 2 "register_operand" "w")
3789                        (match_operand:SI 3 "immediate_operand" "i")]
3790                       VSLRI))]
3791   "TARGET_SIMD"
3792   "*
3793   int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3794   aarch64_simd_const_bounds (operands[3], 1 - <VSLRI:offsetlr>,
3795                              bit_width - <VSLRI:offsetlr> + 1);
3796   return \"s<lr>i\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %3\";"
3797   [(set_attr "type" "neon_shift_imm<q>")]
3798 )
3799
3800 ;; vqshl(u)
3801
3802 (define_insn "aarch64_<sur>qshl<u>_n<mode>"
3803   [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
3804         (unspec:VSDQ_I [(match_operand:VSDQ_I 1 "register_operand" "w")
3805                        (match_operand:SI 2 "immediate_operand" "i")]
3806                       VQSHL_N))]
3807   "TARGET_SIMD"
3808   "*
3809   int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3810   aarch64_simd_const_bounds (operands[2], 0, bit_width);
3811   return \"<sur>qshl<u>\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2\";"
3812   [(set_attr "type" "neon_sat_shift_imm<q>")]
3813 )
3814
3815
3816 ;; vq(r)shr(u)n_n
3817
3818 (define_insn "aarch64_<sur>q<r>shr<u>n_n<mode>"
3819   [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
3820         (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")
3821                             (match_operand:SI 2 "immediate_operand" "i")]
3822                            VQSHRN_N))]
3823   "TARGET_SIMD"
3824   "*
3825   int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3826   aarch64_simd_const_bounds (operands[2], 1, bit_width + 1);
3827   return \"<sur>q<r>shr<u>n\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>, %2\";"
3828   [(set_attr "type" "neon_sat_shift_imm_narrow_q")]
3829 )
3830
3831
3832 ;; cm(eq|ge|gt|lt|le)
3833 ;; Note, we have constraints for Dz and Z as different expanders
3834 ;; have different ideas of what should be passed to this pattern.
3835
3836 (define_insn "aarch64_cm<optab><mode>"
3837   [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w,w")
3838         (neg:<V_cmp_result>
3839           (COMPARISONS:<V_cmp_result>
3840             (match_operand:VDQ 1 "register_operand" "w,w")
3841             (match_operand:VDQ 2 "aarch64_simd_reg_or_zero" "w,ZDz")
3842           )))]
3843   "TARGET_SIMD"
3844   "@
3845   cm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>
3846   cm<optab>\t%<v>0<Vmtype>, %<v>1<Vmtype>, #0"
3847   [(set_attr "type" "neon_compare<q>, neon_compare_zero<q>")]
3848 )
3849
3850 (define_insn_and_split "aarch64_cm<optab>di"
3851   [(set (match_operand:DI 0 "register_operand" "=w,w,r")
3852         (neg:DI
3853           (COMPARISONS:DI
3854             (match_operand:DI 1 "register_operand" "w,w,r")
3855             (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,ZDz,r")
3856           )))
3857      (clobber (reg:CC CC_REGNUM))]
3858   "TARGET_SIMD"
3859   "#"
3860   "reload_completed"
3861   [(set (match_operand:DI 0 "register_operand")
3862         (neg:DI
3863           (COMPARISONS:DI
3864             (match_operand:DI 1 "register_operand")
3865             (match_operand:DI 2 "aarch64_simd_reg_or_zero")
3866           )))]
3867   {
3868     /* If we are in the general purpose register file,
3869        we split to a sequence of comparison and store.  */
3870     if (GP_REGNUM_P (REGNO (operands[0]))
3871         && GP_REGNUM_P (REGNO (operands[1])))
3872       {
3873         enum machine_mode mode = SELECT_CC_MODE (<CMP>, operands[1], operands[2]);
3874         rtx cc_reg = aarch64_gen_compare_reg (<CMP>, operands[1], operands[2]);
3875         rtx comparison = gen_rtx_<CMP> (mode, operands[1], operands[2]);
3876         emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
3877         DONE;
3878       }
3879     /* Otherwise, we expand to a similar pattern which does not
3880        clobber CC_REGNUM.  */
3881   }
3882   [(set_attr "type" "neon_compare, neon_compare_zero, multiple")]
3883 )
3884
3885 (define_insn "*aarch64_cm<optab>di"
3886   [(set (match_operand:DI 0 "register_operand" "=w,w")
3887         (neg:DI
3888           (COMPARISONS:DI
3889             (match_operand:DI 1 "register_operand" "w,w")
3890             (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,ZDz")
3891           )))]
3892   "TARGET_SIMD && reload_completed"
3893   "@
3894   cm<n_optab>\t%d0, %d<cmp_1>, %d<cmp_2>
3895   cm<optab>\t%d0, %d1, #0"
3896   [(set_attr "type" "neon_compare, neon_compare_zero")]
3897 )
3898
3899 ;; cm(hs|hi)
3900
3901 (define_insn "aarch64_cm<optab><mode>"
3902   [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w")
3903         (neg:<V_cmp_result>
3904           (UCOMPARISONS:<V_cmp_result>
3905             (match_operand:VDQ 1 "register_operand" "w")
3906             (match_operand:VDQ 2 "register_operand" "w")
3907           )))]
3908   "TARGET_SIMD"
3909   "cm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>"
3910   [(set_attr "type" "neon_compare<q>")]
3911 )
3912
3913 (define_insn_and_split "aarch64_cm<optab>di"
3914   [(set (match_operand:DI 0 "register_operand" "=w,r")
3915         (neg:DI
3916           (UCOMPARISONS:DI
3917             (match_operand:DI 1 "register_operand" "w,r")
3918             (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,r")
3919           )))
3920     (clobber (reg:CC CC_REGNUM))]
3921   "TARGET_SIMD"
3922   "#"
3923   "reload_completed"
3924   [(set (match_operand:DI 0 "register_operand")
3925         (neg:DI
3926           (UCOMPARISONS:DI
3927             (match_operand:DI 1 "register_operand")
3928             (match_operand:DI 2 "aarch64_simd_reg_or_zero")
3929           )))]
3930   {
3931     /* If we are in the general purpose register file,
3932        we split to a sequence of comparison and store.  */
3933     if (GP_REGNUM_P (REGNO (operands[0]))
3934         && GP_REGNUM_P (REGNO (operands[1])))
3935       {
3936         enum machine_mode mode = CCmode;
3937         rtx cc_reg = aarch64_gen_compare_reg (<CMP>, operands[1], operands[2]);
3938         rtx comparison = gen_rtx_<CMP> (mode, operands[1], operands[2]);
3939         emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
3940         DONE;
3941       }
3942     /* Otherwise, we expand to a similar pattern which does not
3943        clobber CC_REGNUM.  */
3944   }
3945   [(set_attr "type" "neon_compare,multiple")]
3946 )
3947
3948 (define_insn "*aarch64_cm<optab>di"
3949   [(set (match_operand:DI 0 "register_operand" "=w")
3950         (neg:DI
3951           (UCOMPARISONS:DI
3952             (match_operand:DI 1 "register_operand" "w")
3953             (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w")
3954           )))]
3955   "TARGET_SIMD && reload_completed"
3956   "cm<n_optab>\t%d0, %d<cmp_1>, %d<cmp_2>"
3957   [(set_attr "type" "neon_compare")]
3958 )
3959
3960 ;; cmtst
3961
3962 ;; Although neg (ne (and x y) 0) is the natural way of expressing a cmtst,
3963 ;; we don't have any insns using ne, and aarch64_vcond_internal outputs
3964 ;; not (neg (eq (and x y) 0))
3965 ;; which is rewritten by simplify_rtx as
3966 ;; plus (eq (and x y) 0) -1.
3967
3968 (define_insn "aarch64_cmtst<mode>"
3969   [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w")
3970         (plus:<V_cmp_result>
3971           (eq:<V_cmp_result>
3972             (and:VDQ
3973               (match_operand:VDQ 1 "register_operand" "w")
3974               (match_operand:VDQ 2 "register_operand" "w"))
3975             (match_operand:VDQ 3 "aarch64_simd_imm_zero"))
3976           (match_operand:<V_cmp_result> 4 "aarch64_simd_imm_minus_one")))
3977   ]
3978   "TARGET_SIMD"
3979   "cmtst\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3980   [(set_attr "type" "neon_tst<q>")]
3981 )
3982
3983 (define_insn_and_split "aarch64_cmtstdi"
3984   [(set (match_operand:DI 0 "register_operand" "=w,r")
3985         (neg:DI
3986           (ne:DI
3987             (and:DI
3988               (match_operand:DI 1 "register_operand" "w,r")
3989               (match_operand:DI 2 "register_operand" "w,r"))
3990             (const_int 0))))
3991     (clobber (reg:CC CC_REGNUM))]
3992   "TARGET_SIMD"
3993   "#"
3994   "reload_completed"
3995   [(set (match_operand:DI 0 "register_operand")
3996         (neg:DI
3997           (ne:DI
3998             (and:DI
3999               (match_operand:DI 1 "register_operand")
4000               (match_operand:DI 2 "register_operand"))
4001             (const_int 0))))]
4002   {
4003     /* If we are in the general purpose register file,
4004        we split to a sequence of comparison and store.  */
4005     if (GP_REGNUM_P (REGNO (operands[0]))
4006         && GP_REGNUM_P (REGNO (operands[1])))
4007       {
4008         rtx and_tree = gen_rtx_AND (DImode, operands[1], operands[2]);
4009         enum machine_mode mode = SELECT_CC_MODE (NE, and_tree, const0_rtx);
4010         rtx cc_reg = aarch64_gen_compare_reg (NE, and_tree, const0_rtx);
4011         rtx comparison = gen_rtx_NE (mode, and_tree, const0_rtx);
4012         emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
4013         DONE;
4014       }
4015     /* Otherwise, we expand to a similar pattern which does not
4016        clobber CC_REGNUM.  */
4017   }
4018   [(set_attr "type" "neon_tst,multiple")]
4019 )
4020
4021 (define_insn "*aarch64_cmtstdi"
4022   [(set (match_operand:DI 0 "register_operand" "=w")
4023         (neg:DI
4024           (ne:DI
4025             (and:DI
4026               (match_operand:DI 1 "register_operand" "w")
4027               (match_operand:DI 2 "register_operand" "w"))
4028             (const_int 0))))]
4029   "TARGET_SIMD"
4030   "cmtst\t%d0, %d1, %d2"
4031   [(set_attr "type" "neon_tst")]
4032 )
4033
4034 ;; fcm(eq|ge|gt|le|lt)
4035
4036 (define_insn "aarch64_cm<optab><mode>"
4037   [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w,w")
4038         (neg:<V_cmp_result>
4039           (COMPARISONS:<V_cmp_result>
4040             (match_operand:VALLF 1 "register_operand" "w,w")
4041             (match_operand:VALLF 2 "aarch64_simd_reg_or_zero" "w,YDz")
4042           )))]
4043   "TARGET_SIMD"
4044   "@
4045   fcm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>
4046   fcm<optab>\t%<v>0<Vmtype>, %<v>1<Vmtype>, 0"
4047   [(set_attr "type" "neon_fp_compare_<Vetype><q>")]
4048 )
4049
4050 ;; fac(ge|gt)
4051 ;; Note we can also handle what would be fac(le|lt) by
4052 ;; generating fac(ge|gt).
4053
4054 (define_insn "*aarch64_fac<optab><mode>"
4055   [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w")
4056         (neg:<V_cmp_result>
4057           (FAC_COMPARISONS:<V_cmp_result>
4058             (abs:VALLF (match_operand:VALLF 1 "register_operand" "w"))
4059             (abs:VALLF (match_operand:VALLF 2 "register_operand" "w"))
4060   )))]
4061   "TARGET_SIMD"
4062   "fac<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>"
4063   [(set_attr "type" "neon_fp_compare_<Vetype><q>")]
4064 )
4065
4066 ;; addp
4067
4068 (define_insn "aarch64_addp<mode>"
4069   [(set (match_operand:VD_BHSI 0 "register_operand" "=w")
4070         (unspec:VD_BHSI
4071           [(match_operand:VD_BHSI 1 "register_operand" "w")
4072            (match_operand:VD_BHSI 2 "register_operand" "w")]
4073           UNSPEC_ADDP))]
4074   "TARGET_SIMD"
4075   "addp\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
4076   [(set_attr "type" "neon_reduc_add<q>")]
4077 )
4078
4079 (define_insn "aarch64_addpdi"
4080   [(set (match_operand:DI 0 "register_operand" "=w")
4081         (unspec:DI
4082           [(match_operand:V2DI 1 "register_operand" "w")]
4083           UNSPEC_ADDP))]
4084   "TARGET_SIMD"
4085   "addp\t%d0, %1.2d"
4086   [(set_attr "type" "neon_reduc_add")]
4087 )
4088
4089 ;; sqrt
4090
4091 (define_insn "sqrt<mode>2"
4092   [(set (match_operand:VDQF 0 "register_operand" "=w")
4093         (sqrt:VDQF (match_operand:VDQF 1 "register_operand" "w")))]
4094   "TARGET_SIMD"
4095   "fsqrt\\t%0.<Vtype>, %1.<Vtype>"
4096   [(set_attr "type" "neon_fp_sqrt_<Vetype><q>")]
4097 )
4098
4099 ;; Patterns for vector struct loads and stores.
4100
4101 (define_insn "vec_load_lanesoi<mode>"
4102   [(set (match_operand:OI 0 "register_operand" "=w")
4103         (unspec:OI [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")
4104                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4105                    UNSPEC_LD2))]
4106   "TARGET_SIMD"
4107   "ld2\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
4108   [(set_attr "type" "neon_load2_2reg<q>")]
4109 )
4110
4111 (define_insn "aarch64_simd_ld2r<mode>"
4112   [(set (match_operand:OI 0 "register_operand" "=w")
4113        (unspec:OI [(match_operand:<V_TWO_ELEM> 1 "aarch64_simd_struct_operand" "Utv")
4114                    (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
4115                   UNSPEC_LD2_DUP))]
4116   "TARGET_SIMD"
4117   "ld2r\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
4118   [(set_attr "type" "neon_load2_all_lanes<q>")]
4119 )
4120
4121 (define_insn "vec_store_lanesoi<mode>"
4122   [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
4123         (unspec:OI [(match_operand:OI 1 "register_operand" "w")
4124                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4125                    UNSPEC_ST2))]
4126   "TARGET_SIMD"
4127   "st2\\t{%S1.<Vtype> - %T1.<Vtype>}, %0"
4128   [(set_attr "type" "neon_store2_2reg<q>")]
4129 )
4130
4131 (define_insn "vec_store_lanesoi_lane<mode>"
4132   [(set (match_operand:<V_TWO_ELEM> 0 "aarch64_simd_struct_operand" "=Utv")
4133         (unspec:<V_TWO_ELEM> [(match_operand:OI 1 "register_operand" "w")
4134                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4135                     (match_operand:SI 2 "immediate_operand" "i")]
4136                    UNSPEC_ST2_LANE))]
4137   "TARGET_SIMD"
4138   "st2\\t{%S1.<Vetype> - %T1.<Vetype>}[%2], %0"
4139   [(set_attr "type" "neon_store3_one_lane<q>")]
4140 )
4141
4142 (define_insn "vec_load_lanesci<mode>"
4143   [(set (match_operand:CI 0 "register_operand" "=w")
4144         (unspec:CI [(match_operand:CI 1 "aarch64_simd_struct_operand" "Utv")
4145                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4146                    UNSPEC_LD3))]
4147   "TARGET_SIMD"
4148   "ld3\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
4149   [(set_attr "type" "neon_load3_3reg<q>")]
4150 )
4151
4152 (define_insn "aarch64_simd_ld3r<mode>"
4153   [(set (match_operand:CI 0 "register_operand" "=w")
4154        (unspec:CI [(match_operand:<V_THREE_ELEM> 1 "aarch64_simd_struct_operand" "Utv")
4155                    (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
4156                   UNSPEC_LD3_DUP))]
4157   "TARGET_SIMD"
4158   "ld3r\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
4159   [(set_attr "type" "neon_load3_all_lanes<q>")]
4160 )
4161
4162 (define_insn "vec_store_lanesci<mode>"
4163   [(set (match_operand:CI 0 "aarch64_simd_struct_operand" "=Utv")
4164         (unspec:CI [(match_operand:CI 1 "register_operand" "w")
4165                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4166                    UNSPEC_ST3))]
4167   "TARGET_SIMD"
4168   "st3\\t{%S1.<Vtype> - %U1.<Vtype>}, %0"
4169   [(set_attr "type" "neon_store3_3reg<q>")]
4170 )
4171
4172 (define_insn "vec_store_lanesci_lane<mode>"
4173   [(set (match_operand:<V_THREE_ELEM> 0 "aarch64_simd_struct_operand" "=Utv")
4174         (unspec:<V_THREE_ELEM> [(match_operand:CI 1 "register_operand" "w")
4175                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4176                     (match_operand:SI 2 "immediate_operand" "i")]
4177                    UNSPEC_ST3_LANE))]
4178   "TARGET_SIMD"
4179   "st3\\t{%S1.<Vetype> - %U1.<Vetype>}[%2], %0"
4180   [(set_attr "type" "neon_store3_one_lane<q>")]
4181 )
4182
4183 (define_insn "vec_load_lanesxi<mode>"
4184   [(set (match_operand:XI 0 "register_operand" "=w")
4185         (unspec:XI [(match_operand:XI 1 "aarch64_simd_struct_operand" "Utv")
4186                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4187                    UNSPEC_LD4))]
4188   "TARGET_SIMD"
4189   "ld4\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
4190   [(set_attr "type" "neon_load4_4reg<q>")]
4191 )
4192
4193 (define_insn "aarch64_simd_ld4r<mode>"
4194   [(set (match_operand:XI 0 "register_operand" "=w")
4195        (unspec:XI [(match_operand:<V_FOUR_ELEM> 1 "aarch64_simd_struct_operand" "Utv")
4196                    (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
4197                   UNSPEC_LD4_DUP))]
4198   "TARGET_SIMD"
4199   "ld4r\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
4200   [(set_attr "type" "neon_load4_all_lanes<q>")]
4201 )
4202
4203 (define_insn "vec_store_lanesxi<mode>"
4204   [(set (match_operand:XI 0 "aarch64_simd_struct_operand" "=Utv")
4205         (unspec:XI [(match_operand:XI 1 "register_operand" "w")
4206                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4207                    UNSPEC_ST4))]
4208   "TARGET_SIMD"
4209   "st4\\t{%S1.<Vtype> - %V1.<Vtype>}, %0"
4210   [(set_attr "type" "neon_store4_4reg<q>")]
4211 )
4212
4213 (define_insn "vec_store_lanesxi_lane<mode>"
4214   [(set (match_operand:<V_FOUR_ELEM> 0 "aarch64_simd_struct_operand" "=Utv")
4215         (unspec:<V_FOUR_ELEM> [(match_operand:XI 1 "register_operand" "w")
4216                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4217                     (match_operand:SI 2 "immediate_operand" "i")]
4218                    UNSPEC_ST4_LANE))]
4219   "TARGET_SIMD"
4220   "st4\\t{%S1.<Vetype> - %V1.<Vetype>}[%2], %0"
4221   [(set_attr "type" "neon_store4_one_lane<q>")]
4222 )
4223
4224 ;; Reload patterns for AdvSIMD register list operands.
4225
4226 (define_expand "mov<mode>"
4227   [(set (match_operand:VSTRUCT 0 "aarch64_simd_nonimmediate_operand" "")
4228         (match_operand:VSTRUCT 1 "aarch64_simd_general_operand" ""))]
4229   "TARGET_SIMD"
4230 {
4231   if (can_create_pseudo_p ())
4232     {
4233       if (GET_CODE (operands[0]) != REG)
4234         operands[1] = force_reg (<MODE>mode, operands[1]);
4235     }
4236 })
4237
4238 (define_insn "*aarch64_mov<mode>"
4239   [(set (match_operand:VSTRUCT 0 "aarch64_simd_nonimmediate_operand" "=w,Utv,w")
4240         (match_operand:VSTRUCT 1 "aarch64_simd_general_operand" " w,w,Utv"))]
4241   "TARGET_SIMD
4242    && (register_operand (operands[0], <MODE>mode)
4243        || register_operand (operands[1], <MODE>mode))"
4244
4245 {
4246   switch (which_alternative)
4247     {
4248     case 0: return "#";
4249     case 1: return "st1\\t{%S1.16b - %<Vendreg>1.16b}, %0";
4250     case 2: return "ld1\\t{%S0.16b - %<Vendreg>0.16b}, %1";
4251     default: gcc_unreachable ();
4252     }
4253 }
4254   [(set_attr "type" "neon_move,neon_store<nregs>_<nregs>reg_q,\
4255                      neon_load<nregs>_<nregs>reg_q")
4256    (set (attr "length") (symbol_ref "aarch64_simd_attr_length_move (insn)"))]
4257 )
4258
4259 (define_insn "aarch64_be_ld1<mode>"
4260   [(set (match_operand:VALLDI 0 "register_operand" "=w")
4261         (unspec:VALLDI [(match_operand:VALLDI 1 "aarch64_simd_struct_operand" "Utv")]
4262         UNSPEC_LD1))]
4263   "TARGET_SIMD"
4264   "ld1\\t{%0<Vmtype>}, %1"
4265   [(set_attr "type" "neon_load1_1reg<q>")]
4266 )
4267
4268 (define_insn "aarch64_be_st1<mode>"
4269   [(set (match_operand:VALLDI 0 "aarch64_simd_struct_operand" "=Utv")
4270         (unspec:VALLDI [(match_operand:VALLDI 1 "register_operand" "w")]
4271         UNSPEC_ST1))]
4272   "TARGET_SIMD"
4273   "st1\\t{%1<Vmtype>}, %0"
4274   [(set_attr "type" "neon_store1_1reg<q>")]
4275 )
4276
4277 (define_split
4278   [(set (match_operand:OI 0 "register_operand" "")
4279         (match_operand:OI 1 "register_operand" ""))]
4280   "TARGET_SIMD && reload_completed"
4281   [(set (match_dup 0) (match_dup 1))
4282    (set (match_dup 2) (match_dup 3))]
4283 {
4284   int rdest = REGNO (operands[0]);
4285   int rsrc = REGNO (operands[1]);
4286   rtx dest[2], src[2];
4287
4288   dest[0] = gen_rtx_REG (TFmode, rdest);
4289   src[0] = gen_rtx_REG (TFmode, rsrc);
4290   dest[1] = gen_rtx_REG (TFmode, rdest + 1);
4291   src[1] = gen_rtx_REG (TFmode, rsrc + 1);
4292
4293   aarch64_simd_disambiguate_copy (operands, dest, src, 2);
4294 })
4295
4296 (define_split
4297   [(set (match_operand:CI 0 "register_operand" "")
4298         (match_operand:CI 1 "register_operand" ""))]
4299   "TARGET_SIMD && reload_completed"
4300   [(set (match_dup 0) (match_dup 1))
4301    (set (match_dup 2) (match_dup 3))
4302    (set (match_dup 4) (match_dup 5))]
4303 {
4304   int rdest = REGNO (operands[0]);
4305   int rsrc = REGNO (operands[1]);
4306   rtx dest[3], src[3];
4307
4308   dest[0] = gen_rtx_REG (TFmode, rdest);
4309   src[0] = gen_rtx_REG (TFmode, rsrc);
4310   dest[1] = gen_rtx_REG (TFmode, rdest + 1);
4311   src[1] = gen_rtx_REG (TFmode, rsrc + 1);
4312   dest[2] = gen_rtx_REG (TFmode, rdest + 2);
4313   src[2] = gen_rtx_REG (TFmode, rsrc + 2);
4314
4315   aarch64_simd_disambiguate_copy (operands, dest, src, 3);
4316 })
4317
4318 (define_split
4319   [(set (match_operand:XI 0 "register_operand" "")
4320         (match_operand:XI 1 "register_operand" ""))]
4321   "TARGET_SIMD && reload_completed"
4322   [(set (match_dup 0) (match_dup 1))
4323    (set (match_dup 2) (match_dup 3))
4324    (set (match_dup 4) (match_dup 5))
4325    (set (match_dup 6) (match_dup 7))]
4326 {
4327   int rdest = REGNO (operands[0]);
4328   int rsrc = REGNO (operands[1]);
4329   rtx dest[4], src[4];
4330
4331   dest[0] = gen_rtx_REG (TFmode, rdest);
4332   src[0] = gen_rtx_REG (TFmode, rsrc);
4333   dest[1] = gen_rtx_REG (TFmode, rdest + 1);
4334   src[1] = gen_rtx_REG (TFmode, rsrc + 1);
4335   dest[2] = gen_rtx_REG (TFmode, rdest + 2);
4336   src[2] = gen_rtx_REG (TFmode, rsrc + 2);
4337   dest[3] = gen_rtx_REG (TFmode, rdest + 3);
4338   src[3] = gen_rtx_REG (TFmode, rsrc + 3);
4339
4340   aarch64_simd_disambiguate_copy (operands, dest, src, 4);
4341 })
4342
4343 (define_expand "aarch64_ld2r<mode>"
4344   [(match_operand:OI 0 "register_operand" "=w")
4345    (match_operand:DI 1 "register_operand" "w")
4346    (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4347   "TARGET_SIMD"
4348 {
4349   enum machine_mode mode = <V_TWO_ELEM>mode;
4350   rtx mem = gen_rtx_MEM (mode, operands[1]);
4351
4352   emit_insn (gen_aarch64_simd_ld2r<mode> (operands[0], mem));
4353   DONE;
4354 })
4355
4356 (define_expand "aarch64_ld3r<mode>"
4357   [(match_operand:CI 0 "register_operand" "=w")
4358    (match_operand:DI 1 "register_operand" "w")
4359    (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4360   "TARGET_SIMD"
4361 {
4362   enum machine_mode mode = <V_THREE_ELEM>mode;
4363   rtx mem = gen_rtx_MEM (mode, operands[1]);
4364
4365   emit_insn (gen_aarch64_simd_ld3r<mode> (operands[0], mem));
4366   DONE;
4367 })
4368
4369 (define_expand "aarch64_ld4r<mode>"
4370   [(match_operand:XI 0 "register_operand" "=w")
4371    (match_operand:DI 1 "register_operand" "w")
4372    (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4373   "TARGET_SIMD"
4374 {
4375   enum machine_mode mode = <V_FOUR_ELEM>mode;
4376   rtx mem = gen_rtx_MEM (mode, operands[1]);
4377
4378   emit_insn (gen_aarch64_simd_ld4r<mode> (operands[0],mem));
4379   DONE;
4380 })
4381
4382 (define_insn "aarch64_ld2<mode>_dreg"
4383   [(set (match_operand:OI 0 "register_operand" "=w")
4384         (subreg:OI
4385           (vec_concat:<VRL2>
4386             (vec_concat:<VDBL>
4387              (unspec:VD [(match_operand:TI 1 "aarch64_simd_struct_operand" "Utv")]
4388                         UNSPEC_LD2)
4389              (vec_duplicate:VD (const_int 0)))
4390             (vec_concat:<VDBL>
4391              (unspec:VD [(match_dup 1)]
4392                         UNSPEC_LD2)
4393              (vec_duplicate:VD (const_int 0)))) 0))]
4394   "TARGET_SIMD"
4395   "ld2\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
4396   [(set_attr "type" "neon_load2_2reg<q>")]
4397 )
4398
4399 (define_insn "aarch64_ld2<mode>_dreg"
4400   [(set (match_operand:OI 0 "register_operand" "=w")
4401         (subreg:OI
4402           (vec_concat:<VRL2>
4403             (vec_concat:<VDBL>
4404              (unspec:DX [(match_operand:TI 1 "aarch64_simd_struct_operand" "Utv")]
4405                         UNSPEC_LD2)
4406              (const_int 0))
4407             (vec_concat:<VDBL>
4408              (unspec:DX [(match_dup 1)]
4409                         UNSPEC_LD2)
4410              (const_int 0))) 0))]
4411   "TARGET_SIMD"
4412   "ld1\\t{%S0.1d - %T0.1d}, %1"
4413   [(set_attr "type" "neon_load1_2reg<q>")]
4414 )
4415
4416 (define_insn "aarch64_ld3<mode>_dreg"
4417   [(set (match_operand:CI 0 "register_operand" "=w")
4418         (subreg:CI
4419          (vec_concat:<VRL3>
4420           (vec_concat:<VRL2>
4421             (vec_concat:<VDBL>
4422              (unspec:VD [(match_operand:EI 1 "aarch64_simd_struct_operand" "Utv")]
4423                         UNSPEC_LD3)
4424              (vec_duplicate:VD (const_int 0)))
4425             (vec_concat:<VDBL>
4426              (unspec:VD [(match_dup 1)]
4427                         UNSPEC_LD3)
4428              (vec_duplicate:VD (const_int 0))))
4429           (vec_concat:<VDBL>
4430              (unspec:VD [(match_dup 1)]
4431                         UNSPEC_LD3)
4432              (vec_duplicate:VD (const_int 0)))) 0))]
4433   "TARGET_SIMD"
4434   "ld3\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
4435   [(set_attr "type" "neon_load3_3reg<q>")]
4436 )
4437
4438 (define_insn "aarch64_ld3<mode>_dreg"
4439   [(set (match_operand:CI 0 "register_operand" "=w")
4440         (subreg:CI
4441          (vec_concat:<VRL3>
4442           (vec_concat:<VRL2>
4443             (vec_concat:<VDBL>
4444              (unspec:DX [(match_operand:EI 1 "aarch64_simd_struct_operand" "Utv")]
4445                         UNSPEC_LD3)
4446              (const_int 0))
4447             (vec_concat:<VDBL>
4448              (unspec:DX [(match_dup 1)]
4449                         UNSPEC_LD3)
4450              (const_int 0)))
4451           (vec_concat:<VDBL>
4452              (unspec:DX [(match_dup 1)]
4453                         UNSPEC_LD3)
4454              (const_int 0))) 0))]
4455   "TARGET_SIMD"
4456   "ld1\\t{%S0.1d - %U0.1d}, %1"
4457   [(set_attr "type" "neon_load1_3reg<q>")]
4458 )
4459
4460 (define_insn "aarch64_ld4<mode>_dreg"
4461   [(set (match_operand:XI 0 "register_operand" "=w")
4462         (subreg:XI
4463          (vec_concat:<VRL4>
4464            (vec_concat:<VRL2>
4465              (vec_concat:<VDBL>
4466                (unspec:VD [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")]
4467                           UNSPEC_LD4)
4468                (vec_duplicate:VD (const_int 0)))
4469               (vec_concat:<VDBL>
4470                 (unspec:VD [(match_dup 1)]
4471                         UNSPEC_LD4)
4472                 (vec_duplicate:VD (const_int 0))))
4473            (vec_concat:<VRL2>
4474              (vec_concat:<VDBL>
4475                (unspec:VD [(match_dup 1)]
4476                         UNSPEC_LD4)
4477                (vec_duplicate:VD (const_int 0)))
4478              (vec_concat:<VDBL>
4479                (unspec:VD [(match_dup 1)]
4480                         UNSPEC_LD4)
4481                (vec_duplicate:VD (const_int 0))))) 0))]
4482   "TARGET_SIMD"
4483   "ld4\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
4484   [(set_attr "type" "neon_load4_4reg<q>")]
4485 )
4486
4487 (define_insn "aarch64_ld4<mode>_dreg"
4488   [(set (match_operand:XI 0 "register_operand" "=w")
4489         (subreg:XI
4490          (vec_concat:<VRL4>
4491            (vec_concat:<VRL2>
4492              (vec_concat:<VDBL>
4493                (unspec:DX [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")]
4494                           UNSPEC_LD4)
4495                (const_int 0))
4496               (vec_concat:<VDBL>
4497                 (unspec:DX [(match_dup 1)]
4498                         UNSPEC_LD4)
4499                 (const_int 0)))
4500            (vec_concat:<VRL2>
4501              (vec_concat:<VDBL>
4502                (unspec:DX [(match_dup 1)]
4503                         UNSPEC_LD4)
4504                (const_int 0))
4505              (vec_concat:<VDBL>
4506                (unspec:DX [(match_dup 1)]
4507                         UNSPEC_LD4)
4508                (const_int 0)))) 0))]
4509   "TARGET_SIMD"
4510   "ld1\\t{%S0.1d - %V0.1d}, %1"
4511   [(set_attr "type" "neon_load1_4reg<q>")]
4512 )
4513
4514 (define_expand "aarch64_ld<VSTRUCT:nregs><VDC:mode>"
4515  [(match_operand:VSTRUCT 0 "register_operand" "=w")
4516   (match_operand:DI 1 "register_operand" "r")
4517   (unspec:VDC [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4518   "TARGET_SIMD"
4519 {
4520   enum machine_mode mode = <VSTRUCT:VSTRUCT_DREG>mode;
4521   rtx mem = gen_rtx_MEM (mode, operands[1]);
4522
4523   emit_insn (gen_aarch64_ld<VSTRUCT:nregs><VDC:mode>_dreg (operands[0], mem));
4524   DONE;
4525 })
4526
4527 (define_expand "aarch64_ld1<VALL:mode>"
4528  [(match_operand:VALL 0 "register_operand")
4529   (match_operand:DI 1 "register_operand")]
4530   "TARGET_SIMD"
4531 {
4532   enum machine_mode mode = <VALL:MODE>mode;
4533   rtx mem = gen_rtx_MEM (mode, operands[1]);
4534
4535   if (BYTES_BIG_ENDIAN)
4536     emit_insn (gen_aarch64_be_ld1<VALL:mode> (operands[0], mem));
4537   else
4538     emit_move_insn (operands[0], mem);
4539   DONE;
4540 })
4541
4542 (define_expand "aarch64_ld<VSTRUCT:nregs><VQ:mode>"
4543  [(match_operand:VSTRUCT 0 "register_operand" "=w")
4544   (match_operand:DI 1 "register_operand" "r")
4545   (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4546   "TARGET_SIMD"
4547 {
4548   enum machine_mode mode = <VSTRUCT:MODE>mode;
4549   rtx mem = gen_rtx_MEM (mode, operands[1]);
4550
4551   emit_insn (gen_vec_load_lanes<VSTRUCT:mode><VQ:mode> (operands[0], mem));
4552   DONE;
4553 })
4554
4555 ;; Expanders for builtins to extract vector registers from large
4556 ;; opaque integer modes.
4557
4558 ;; D-register list.
4559
4560 (define_expand "aarch64_get_dreg<VSTRUCT:mode><VDC:mode>"
4561  [(match_operand:VDC 0 "register_operand" "=w")
4562   (match_operand:VSTRUCT 1 "register_operand" "w")
4563   (match_operand:SI 2 "immediate_operand" "i")]
4564   "TARGET_SIMD"
4565 {
4566   int part = INTVAL (operands[2]);
4567   rtx temp = gen_reg_rtx (<VDC:VDBL>mode);
4568   int offset = part * 16;
4569
4570   emit_move_insn (temp, gen_rtx_SUBREG (<VDC:VDBL>mode, operands[1], offset));
4571   emit_move_insn (operands[0], gen_lowpart (<VDC:MODE>mode, temp));
4572   DONE;
4573 })
4574
4575 ;; Q-register list.
4576
4577 (define_expand "aarch64_get_qreg<VSTRUCT:mode><VQ:mode>"
4578  [(match_operand:VQ 0 "register_operand" "=w")
4579   (match_operand:VSTRUCT 1 "register_operand" "w")
4580   (match_operand:SI 2 "immediate_operand" "i")]
4581   "TARGET_SIMD"
4582 {
4583   int part = INTVAL (operands[2]);
4584   int offset = part * 16;
4585
4586   emit_move_insn (operands[0],
4587                   gen_rtx_SUBREG (<VQ:MODE>mode, operands[1], offset));
4588   DONE;
4589 })
4590
4591 ;; Permuted-store expanders for neon intrinsics.
4592
4593 ;; Permute instructions
4594
4595 ;; vec_perm support
4596
4597 (define_expand "vec_perm_const<mode>"
4598   [(match_operand:VALL 0 "register_operand")
4599    (match_operand:VALL 1 "register_operand")
4600    (match_operand:VALL 2 "register_operand")
4601    (match_operand:<V_cmp_result> 3)]
4602   "TARGET_SIMD"
4603 {
4604   if (aarch64_expand_vec_perm_const (operands[0], operands[1],
4605                                      operands[2], operands[3]))
4606     DONE;
4607   else
4608     FAIL;
4609 })
4610
4611 (define_expand "vec_perm<mode>"
4612   [(match_operand:VB 0 "register_operand")
4613    (match_operand:VB 1 "register_operand")
4614    (match_operand:VB 2 "register_operand")
4615    (match_operand:VB 3 "register_operand")]
4616   "TARGET_SIMD"
4617 {
4618   aarch64_expand_vec_perm (operands[0], operands[1],
4619                            operands[2], operands[3]);
4620   DONE;
4621 })
4622
4623 (define_insn "aarch64_tbl1<mode>"
4624   [(set (match_operand:VB 0 "register_operand" "=w")
4625         (unspec:VB [(match_operand:V16QI 1 "register_operand" "w")
4626                     (match_operand:VB 2 "register_operand" "w")]
4627                    UNSPEC_TBL))]
4628   "TARGET_SIMD"
4629   "tbl\\t%0.<Vtype>, {%1.16b}, %2.<Vtype>"
4630   [(set_attr "type" "neon_tbl1<q>")]
4631 )
4632
4633 ;; Two source registers.
4634
4635 (define_insn "aarch64_tbl2v16qi"
4636   [(set (match_operand:V16QI 0 "register_operand" "=w")
4637         (unspec:V16QI [(match_operand:OI 1 "register_operand" "w")
4638                        (match_operand:V16QI 2 "register_operand" "w")]
4639                       UNSPEC_TBL))]
4640   "TARGET_SIMD"
4641   "tbl\\t%0.16b, {%S1.16b - %T1.16b}, %2.16b"
4642   [(set_attr "type" "neon_tbl2_q")]
4643 )
4644
4645 (define_insn_and_split "aarch64_combinev16qi"
4646   [(set (match_operand:OI 0 "register_operand" "=w")
4647         (unspec:OI [(match_operand:V16QI 1 "register_operand" "w")
4648                     (match_operand:V16QI 2 "register_operand" "w")]
4649                    UNSPEC_CONCAT))]
4650   "TARGET_SIMD"
4651   "#"
4652   "&& reload_completed"
4653   [(const_int 0)]
4654 {
4655   aarch64_split_combinev16qi (operands);
4656   DONE;
4657 }
4658 [(set_attr "type" "multiple")]
4659 )
4660
4661 (define_insn "aarch64_<PERMUTE:perm_insn><PERMUTE:perm_hilo><mode>"
4662   [(set (match_operand:VALL 0 "register_operand" "=w")
4663         (unspec:VALL [(match_operand:VALL 1 "register_operand" "w")
4664                       (match_operand:VALL 2 "register_operand" "w")]
4665                        PERMUTE))]
4666   "TARGET_SIMD"
4667   "<PERMUTE:perm_insn><PERMUTE:perm_hilo>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
4668   [(set_attr "type" "neon_permute<q>")]
4669 )
4670
4671 ;; Note immediate (third) operand is lane index not byte index.
4672 (define_insn "aarch64_ext<mode>"
4673   [(set (match_operand:VALL 0 "register_operand" "=w")
4674         (unspec:VALL [(match_operand:VALL 1 "register_operand" "w")
4675                       (match_operand:VALL 2 "register_operand" "w")
4676                       (match_operand:SI 3 "immediate_operand" "i")]
4677                      UNSPEC_EXT))]
4678   "TARGET_SIMD"
4679 {
4680   operands[3] = GEN_INT (INTVAL (operands[3])
4681       * GET_MODE_SIZE (GET_MODE_INNER (<MODE>mode)));
4682   return "ext\\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>, #%3";
4683 }
4684   [(set_attr "type" "neon_ext<q>")]
4685 )
4686
4687 ;; This exists solely to check the arguments to the corresponding __builtin.
4688 ;; Used where we want an error for out-of-range indices which would otherwise
4689 ;; be silently wrapped (e.g. the mask to a __builtin_shuffle).
4690 (define_expand "aarch64_im_lane_boundsi"
4691   [(match_operand:SI 0 "immediate_operand" "i")
4692    (match_operand:SI 1 "immediate_operand" "i")]
4693   "TARGET_SIMD"
4694 {
4695   aarch64_simd_lane_bounds (operands[0], 0, INTVAL (operands[1]));
4696   DONE;
4697 }
4698 )
4699
4700 (define_insn "aarch64_rev<REVERSE:rev_op><mode>"
4701   [(set (match_operand:VALL 0 "register_operand" "=w")
4702         (unspec:VALL [(match_operand:VALL 1 "register_operand" "w")]
4703                     REVERSE))]
4704   "TARGET_SIMD"
4705   "rev<REVERSE:rev_op>\\t%0.<Vtype>, %1.<Vtype>"
4706   [(set_attr "type" "neon_rev<q>")]
4707 )
4708
4709 (define_insn "aarch64_st2<mode>_dreg"
4710   [(set (match_operand:TI 0 "aarch64_simd_struct_operand" "=Utv")
4711         (unspec:TI [(match_operand:OI 1 "register_operand" "w")
4712                     (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4713                    UNSPEC_ST2))]
4714   "TARGET_SIMD"
4715   "st2\\t{%S1.<Vtype> - %T1.<Vtype>}, %0"
4716   [(set_attr "type" "neon_store2_2reg")]
4717 )
4718
4719 (define_insn "aarch64_st2<mode>_dreg"
4720   [(set (match_operand:TI 0 "aarch64_simd_struct_operand" "=Utv")
4721         (unspec:TI [(match_operand:OI 1 "register_operand" "w")
4722                     (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4723                    UNSPEC_ST2))]
4724   "TARGET_SIMD"
4725   "st1\\t{%S1.1d - %T1.1d}, %0"
4726   [(set_attr "type" "neon_store1_2reg")]
4727 )
4728
4729 (define_insn "aarch64_st3<mode>_dreg"
4730   [(set (match_operand:EI 0 "aarch64_simd_struct_operand" "=Utv")
4731         (unspec:EI [(match_operand:CI 1 "register_operand" "w")
4732                     (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4733                    UNSPEC_ST3))]
4734   "TARGET_SIMD"
4735   "st3\\t{%S1.<Vtype> - %U1.<Vtype>}, %0"
4736   [(set_attr "type" "neon_store3_3reg")]
4737 )
4738
4739 (define_insn "aarch64_st3<mode>_dreg"
4740   [(set (match_operand:EI 0 "aarch64_simd_struct_operand" "=Utv")
4741         (unspec:EI [(match_operand:CI 1 "register_operand" "w")
4742                     (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4743                    UNSPEC_ST3))]
4744   "TARGET_SIMD"
4745   "st1\\t{%S1.1d - %U1.1d}, %0"
4746   [(set_attr "type" "neon_store1_3reg")]
4747 )
4748
4749 (define_insn "aarch64_st4<mode>_dreg"
4750   [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
4751         (unspec:OI [(match_operand:XI 1 "register_operand" "w")
4752                     (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4753                    UNSPEC_ST4))]
4754   "TARGET_SIMD"
4755   "st4\\t{%S1.<Vtype> - %V1.<Vtype>}, %0"
4756   [(set_attr "type" "neon_store4_4reg")]
4757 )
4758
4759 (define_insn "aarch64_st4<mode>_dreg"
4760   [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
4761         (unspec:OI [(match_operand:XI 1 "register_operand" "w")
4762                     (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4763                    UNSPEC_ST4))]
4764   "TARGET_SIMD"
4765   "st1\\t{%S1.1d - %V1.1d}, %0"
4766   [(set_attr "type" "neon_store1_4reg")]
4767 )
4768
4769 (define_expand "aarch64_st<VSTRUCT:nregs><VDC:mode>"
4770  [(match_operand:DI 0 "register_operand" "r")
4771   (match_operand:VSTRUCT 1 "register_operand" "w")
4772   (unspec:VDC [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4773   "TARGET_SIMD"
4774 {
4775   enum machine_mode mode = <VSTRUCT:VSTRUCT_DREG>mode;
4776   rtx mem = gen_rtx_MEM (mode, operands[0]);
4777
4778   emit_insn (gen_aarch64_st<VSTRUCT:nregs><VDC:mode>_dreg (mem, operands[1]));
4779   DONE;
4780 })
4781
4782 (define_expand "aarch64_st<VSTRUCT:nregs><VQ:mode>"
4783  [(match_operand:DI 0 "register_operand" "r")
4784   (match_operand:VSTRUCT 1 "register_operand" "w")
4785   (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4786   "TARGET_SIMD"
4787 {
4788   enum machine_mode mode = <VSTRUCT:MODE>mode;
4789   rtx mem = gen_rtx_MEM (mode, operands[0]);
4790
4791   emit_insn (gen_vec_store_lanes<VSTRUCT:mode><VQ:mode> (mem, operands[1]));
4792   DONE;
4793 })
4794
4795 (define_expand "aarch64_st2_lane<VQ:mode>"
4796  [(match_operand:DI 0 "register_operand" "r")
4797   (match_operand:OI 1 "register_operand" "w")
4798   (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4799   (match_operand:SI 2 "immediate_operand")]
4800   "TARGET_SIMD"
4801 {
4802   enum machine_mode mode = <V_TWO_ELEM>mode;
4803   rtx mem = gen_rtx_MEM (mode, operands[0]);
4804   operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
4805
4806   emit_insn (gen_vec_store_lanesoi_lane<VQ:mode> (mem,
4807                                                   operands[1],
4808                                                   operands[2]));
4809   DONE;
4810 })
4811
4812 (define_expand "aarch64_st3_lane<VQ:mode>"
4813  [(match_operand:DI 0 "register_operand" "r")
4814   (match_operand:CI 1 "register_operand" "w")
4815   (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4816   (match_operand:SI 2 "immediate_operand")]
4817   "TARGET_SIMD"
4818 {
4819   enum machine_mode mode = <V_THREE_ELEM>mode;
4820   rtx mem = gen_rtx_MEM (mode, operands[0]);
4821   operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
4822
4823   emit_insn (gen_vec_store_lanesci_lane<VQ:mode> (mem,
4824                                                   operands[1],
4825                                                   operands[2]));
4826   DONE;
4827 })
4828
4829 (define_expand "aarch64_st4_lane<VQ:mode>"
4830  [(match_operand:DI 0 "register_operand" "r")
4831   (match_operand:XI 1 "register_operand" "w")
4832   (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4833   (match_operand:SI 2 "immediate_operand")]
4834   "TARGET_SIMD"
4835 {
4836   enum machine_mode mode = <V_FOUR_ELEM>mode;
4837   rtx mem = gen_rtx_MEM (mode, operands[0]);
4838   operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
4839
4840   emit_insn (gen_vec_store_lanesxi_lane<VQ:mode> (mem,
4841                                                   operands[1],
4842                                                   operands[2]));
4843   DONE;
4844 })
4845
4846 (define_expand "aarch64_st1<VALL:mode>"
4847  [(match_operand:DI 0 "register_operand")
4848   (match_operand:VALL 1 "register_operand")]
4849   "TARGET_SIMD"
4850 {
4851   enum machine_mode mode = <VALL:MODE>mode;
4852   rtx mem = gen_rtx_MEM (mode, operands[0]);
4853
4854   if (BYTES_BIG_ENDIAN)
4855     emit_insn (gen_aarch64_be_st1<VALL:mode> (mem, operands[1]));
4856   else
4857     emit_move_insn (mem, operands[1]);
4858   DONE;
4859 })
4860
4861 ;; Expander for builtins to insert vector registers into large
4862 ;; opaque integer modes.
4863
4864 ;; Q-register list.  We don't need a D-reg inserter as we zero
4865 ;; extend them in arm_neon.h and insert the resulting Q-regs.
4866
4867 (define_expand "aarch64_set_qreg<VSTRUCT:mode><VQ:mode>"
4868  [(match_operand:VSTRUCT 0 "register_operand" "+w")
4869   (match_operand:VSTRUCT 1 "register_operand" "0")
4870   (match_operand:VQ 2 "register_operand" "w")
4871   (match_operand:SI 3 "immediate_operand" "i")]
4872   "TARGET_SIMD"
4873 {
4874   int part = INTVAL (operands[3]);
4875   int offset = part * 16;
4876
4877   emit_move_insn (operands[0], operands[1]);
4878   emit_move_insn (gen_rtx_SUBREG (<VQ:MODE>mode, operands[0], offset),
4879                   operands[2]);
4880   DONE;
4881 })
4882
4883 ;; Standard pattern name vec_init<mode>.
4884
4885 (define_expand "vec_init<mode>"
4886   [(match_operand:VALL 0 "register_operand" "")
4887    (match_operand 1 "" "")]
4888   "TARGET_SIMD"
4889 {
4890   aarch64_expand_vector_init (operands[0], operands[1]);
4891   DONE;
4892 })
4893
4894 (define_insn "*aarch64_simd_ld1r<mode>"
4895   [(set (match_operand:VALL 0 "register_operand" "=w")
4896         (vec_duplicate:VALL
4897           (match_operand:<VEL> 1 "aarch64_simd_struct_operand" "Utv")))]
4898   "TARGET_SIMD"
4899   "ld1r\\t{%0.<Vtype>}, %1"
4900   [(set_attr "type" "neon_load1_all_lanes")]
4901 )
4902
4903 (define_insn "aarch64_frecpe<mode>"
4904   [(set (match_operand:VDQF 0 "register_operand" "=w")
4905         (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")]
4906                     UNSPEC_FRECPE))]
4907   "TARGET_SIMD"
4908   "frecpe\\t%0.<Vtype>, %1.<Vtype>"
4909   [(set_attr "type" "neon_fp_recpe_<Vetype><q>")]
4910 )
4911
4912 (define_insn "aarch64_frecp<FRECP:frecp_suffix><mode>"
4913   [(set (match_operand:GPF 0 "register_operand" "=w")
4914         (unspec:GPF [(match_operand:GPF 1 "register_operand" "w")]
4915                     FRECP))]
4916   "TARGET_SIMD"
4917   "frecp<FRECP:frecp_suffix>\\t%<s>0, %<s>1"
4918   [(set_attr "type" "neon_fp_recp<FRECP:frecp_suffix>_<GPF:Vetype><GPF:q>")]
4919 )
4920
4921 (define_insn "aarch64_frecps<mode>"
4922   [(set (match_operand:VALLF 0 "register_operand" "=w")
4923         (unspec:VALLF [(match_operand:VALLF 1 "register_operand" "w")
4924                      (match_operand:VALLF 2 "register_operand" "w")]
4925                     UNSPEC_FRECPS))]
4926   "TARGET_SIMD"
4927   "frecps\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
4928   [(set_attr "type" "neon_fp_recps_<Vetype><q>")]
4929 )
4930
4931 ;; Standard pattern name vec_extract<mode>.
4932
4933 (define_expand "vec_extract<mode>"
4934   [(match_operand:<VEL> 0 "aarch64_simd_nonimmediate_operand" "")
4935    (match_operand:VALL 1 "register_operand" "")
4936    (match_operand:SI 2 "immediate_operand" "")]
4937   "TARGET_SIMD"
4938 {
4939     emit_insn
4940       (gen_aarch64_get_lane<mode> (operands[0], operands[1], operands[2]));
4941     DONE;
4942 })
4943
4944 ;; aes
4945
4946 (define_insn "aarch64_crypto_aes<aes_op>v16qi"
4947   [(set (match_operand:V16QI 0 "register_operand" "=w")
4948         (unspec:V16QI [(match_operand:V16QI 1 "register_operand" "0")
4949                        (match_operand:V16QI 2 "register_operand" "w")]
4950          CRYPTO_AES))]
4951   "TARGET_SIMD && TARGET_CRYPTO"
4952   "aes<aes_op>\\t%0.16b, %2.16b"
4953   [(set_attr "type" "crypto_aese")]
4954 )
4955
4956 (define_insn "aarch64_crypto_aes<aesmc_op>v16qi"
4957   [(set (match_operand:V16QI 0 "register_operand" "=w")
4958         (unspec:V16QI [(match_operand:V16QI 1 "register_operand" "w")]
4959          CRYPTO_AESMC))]
4960   "TARGET_SIMD && TARGET_CRYPTO"
4961   "aes<aesmc_op>\\t%0.16b, %1.16b"
4962   [(set_attr "type" "crypto_aesmc")]
4963 )
4964
4965 ;; sha1
4966
4967 (define_insn "aarch64_crypto_sha1hsi"
4968   [(set (match_operand:SI 0 "register_operand" "=w")
4969         (unspec:SI [(match_operand:SI 1
4970                        "register_operand" "w")]
4971          UNSPEC_SHA1H))]
4972   "TARGET_SIMD && TARGET_CRYPTO"
4973   "sha1h\\t%s0, %s1"
4974   [(set_attr "type" "crypto_sha1_fast")]
4975 )
4976
4977 (define_insn "aarch64_crypto_sha1su1v4si"
4978   [(set (match_operand:V4SI 0 "register_operand" "=w")
4979         (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
4980                       (match_operand:V4SI 2 "register_operand" "w")]
4981          UNSPEC_SHA1SU1))]
4982   "TARGET_SIMD && TARGET_CRYPTO"
4983   "sha1su1\\t%0.4s, %2.4s"
4984   [(set_attr "type" "crypto_sha1_fast")]
4985 )
4986
4987 (define_insn "aarch64_crypto_sha1<sha1_op>v4si"
4988   [(set (match_operand:V4SI 0 "register_operand" "=w")
4989         (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
4990                       (match_operand:SI 2 "register_operand" "w")
4991                       (match_operand:V4SI 3 "register_operand" "w")]
4992          CRYPTO_SHA1))]
4993   "TARGET_SIMD && TARGET_CRYPTO"
4994   "sha1<sha1_op>\\t%q0, %s2, %3.4s"
4995   [(set_attr "type" "crypto_sha1_slow")]
4996 )
4997
4998 (define_insn "aarch64_crypto_sha1su0v4si"
4999   [(set (match_operand:V4SI 0 "register_operand" "=w")
5000         (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5001                       (match_operand:V4SI 2 "register_operand" "w")
5002                       (match_operand:V4SI 3 "register_operand" "w")]
5003          UNSPEC_SHA1SU0))]
5004   "TARGET_SIMD && TARGET_CRYPTO"
5005   "sha1su0\\t%0.4s, %2.4s, %3.4s"
5006   [(set_attr "type" "crypto_sha1_xor")]
5007 )
5008
5009 ;; sha256
5010
5011 (define_insn "aarch64_crypto_sha256h<sha256_op>v4si"
5012   [(set (match_operand:V4SI 0 "register_operand" "=w")
5013         (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5014                       (match_operand:V4SI 2 "register_operand" "w")
5015                       (match_operand:V4SI 3 "register_operand" "w")]
5016          CRYPTO_SHA256))]
5017   "TARGET_SIMD && TARGET_CRYPTO"
5018   "sha256h<sha256_op>\\t%q0, %q2, %3.4s"
5019   [(set_attr "type" "crypto_sha256_slow")]
5020 )
5021
5022 (define_insn "aarch64_crypto_sha256su0v4si"
5023   [(set (match_operand:V4SI 0 "register_operand" "=w")
5024         (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5025                       (match_operand:V4SI 2 "register_operand" "w")]
5026          UNSPEC_SHA256SU0))]
5027   "TARGET_SIMD &&TARGET_CRYPTO"
5028   "sha256su0\\t%0.4s, %2.4s"
5029   [(set_attr "type" "crypto_sha256_fast")]
5030 )
5031
5032 (define_insn "aarch64_crypto_sha256su1v4si"
5033   [(set (match_operand:V4SI 0 "register_operand" "=w")
5034         (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5035                       (match_operand:V4SI 2 "register_operand" "w")
5036                       (match_operand:V4SI 3 "register_operand" "w")]
5037          UNSPEC_SHA256SU1))]
5038   "TARGET_SIMD &&TARGET_CRYPTO"
5039   "sha256su1\\t%0.4s, %2.4s, %3.4s"
5040   [(set_attr "type" "crypto_sha256_slow")]
5041 )
5042
5043 ;; pmull
5044
5045 (define_insn "aarch64_crypto_pmulldi"
5046   [(set (match_operand:TI 0 "register_operand" "=w")
5047         (unspec:TI  [(match_operand:DI 1 "register_operand" "w")
5048                      (match_operand:DI 2 "register_operand" "w")]
5049                     UNSPEC_PMULL))]
5050  "TARGET_SIMD && TARGET_CRYPTO"
5051  "pmull\\t%0.1q, %1.1d, %2.1d"
5052   [(set_attr "type" "neon_mul_d_long")]
5053 )
5054
5055 (define_insn "aarch64_crypto_pmullv2di"
5056  [(set (match_operand:TI 0 "register_operand" "=w")
5057        (unspec:TI [(match_operand:V2DI 1 "register_operand" "w")
5058                    (match_operand:V2DI 2 "register_operand" "w")]
5059                   UNSPEC_PMULL2))]
5060   "TARGET_SIMD && TARGET_CRYPTO"
5061   "pmull2\\t%0.1q, %1.2d, %2.2d"
5062   [(set_attr "type" "neon_mul_d_long")]
5063 )