src/jit/simd.cpp

   1 // Licensed to the .NET Foundation under one or more agreements.
   2 // The .NET Foundation licenses this file to you under the MIT license.
   3 // See the LICENSE file in the project root for more information.
   4
   5 //
   6 //   SIMD Support
   7 //
   8 // IMPORTANT NOTES AND CAVEATS:
   9 //
  10 // This implementation is preliminary, and may change dramatically.
  11 //
  12 // New JIT types, TYP_SIMDxx, are introduced, and the SIMD intrinsics are created as GT_SIMD nodes.
  13 // Nodes of SIMD types will be typed as TYP_SIMD* (e.g. TYP_SIMD8, TYP_SIMD16, etc.).
  14 //
  15 // Note that currently the "reference implementation" is the same as the runtime dll.  As such, it is currently
  16 // providing implementations for those methods not currently supported by the JIT as intrinsics.
  17 //
  18 // These are currently recognized using string compares, in order to provide an implementation in the JIT
  19 // without taking a dependency on the VM.
  20 // Furthermore, in the CTP, in order to limit the impact of doing these string compares
  21 // against assembly names, we only look for the SIMDVector assembly if we are compiling a class constructor.  This
  22 // makes it somewhat more "pay for play" but is a significant usability compromise.
  23 // This has been addressed for RTM by doing the assembly recognition in the VM.
  24 // --------------------------------------------------------------------------------------
  25
  26 #include "jitpch.h"
  27 #include "simd.h"
  28
  29 #ifdef _MSC_VER
  30 #pragma hdrstop
  31 #endif
  32
  33 #ifndef LEGACY_BACKEND // This file is ONLY used for the RyuJIT backend that uses the linear scan register allocator.
  34
  35 #ifdef FEATURE_SIMD
  36
  37 // Intrinsic Id to intrinsic info map
  38 const SIMDIntrinsicInfo simdIntrinsicInfoArray[] = {
  39 #define SIMD_INTRINSIC(mname, inst, id, name, retType, argCount, arg1, arg2, arg3, t1, t2, t3, t4, t5, t6, t7, t8, t9, \
  40                        t10)                                                                                            \
  41     {SIMDIntrinsic##id, mname, inst, retType, argCount, arg1, arg2, arg3, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10},
  42 #include "simdintrinsiclist.h"
  43 };
  44
  45 //------------------------------------------------------------------------
  46 // getSIMDVectorLength: Get the length (number of elements of base type) of
  47 //                      SIMD Vector given its size and base (element) type.
  48 //
  49 // Arguments:
  50 //    simdSize   - size of the SIMD vector
  51 //    baseType   - type of the elements of the SIMD vector
  52 //
  53 // static
  54 int Compiler::getSIMDVectorLength(unsigned simdSize, var_types baseType)
  55 {
  56     return simdSize / genTypeSize(baseType);
  57 }
  58
  59 //------------------------------------------------------------------------
  60 // Get the length (number of elements of base type) of SIMD Vector given by typeHnd.
  61 //
  62 // Arguments:
  63 //    typeHnd  - type handle of the SIMD vector
  64 //
  65 int Compiler::getSIMDVectorLength(CORINFO_CLASS_HANDLE typeHnd)
  66 {
  67     unsigned  sizeBytes = 0;
  68     var_types baseType  = getBaseTypeAndSizeOfSIMDType(typeHnd, &sizeBytes);
  69     return getSIMDVectorLength(sizeBytes, baseType);
  70 }
  71
  72 //------------------------------------------------------------------------
  73 // Get the preferred alignment of SIMD vector type for better performance.
  74 //
  75 // Arguments:
  76 //    typeHnd  - type handle of the SIMD vector
  77 //
  78 int Compiler::getSIMDTypeAlignment(var_types simdType)
  79 {
  80 #ifdef _TARGET_XARCH_
  81     // Fixed length vectors have the following alignment preference
  82     // Vector2   = 8 byte alignment
  83     // Vector3/4 = 16-byte alignment
  84     unsigned size = genTypeSize(simdType);
  85
  86     // preferred alignment for SSE2 128-bit vectors is 16-bytes
  87     if (size == 8)
  88     {
  89         return 8;
  90     }
  91     else if (size <= 16)
  92     {
  93         assert((size == 12) || (size == 16));
  94         return 16;
  95     }
  96     else
  97     {
  98         assert(size == 32);
  99         return 32;
 100     }
 101 #elif defined(_TARGET_ARM64_)
 102     return 16;
 103 #else
 104     assert(!"getSIMDTypeAlignment() unimplemented on target arch");
 105     unreached();
 106 #endif
 107 }
 108
 109 //----------------------------------------------------------------------------------
 110 // Return the base type and size of SIMD vector type given its type handle.
 111 //
 112 // Arguments:
 113 //    typeHnd   - The handle of the type we're interested in.
 114 //    sizeBytes - out param
 115 //
 116 // Return Value:
 117 //    base type of SIMD vector.
 118 //    sizeBytes if non-null is set to size in bytes.
 119 //
 120 // TODO-Throughput: current implementation parses class name to find base type. Change
 121 //         this when we implement  SIMD intrinsic identification for the final
 122 //         product.
 123 //
 124 var_types Compiler::getBaseTypeAndSizeOfSIMDType(CORINFO_CLASS_HANDLE typeHnd, unsigned* sizeBytes /*= nullptr */)
 125 {
 126     assert(featureSIMD);
 127     if (typeHnd == nullptr)
 128     {
 129         return TYP_UNKNOWN;
 130     }
 131
 132     // fast path search using cached type handles of important types
 133     var_types simdBaseType = TYP_UNKNOWN;
 134     unsigned  size         = 0;
 135
 136     // TODO - Optimize SIMD type recognition by IntrinsicAttribute
 137     if (isSIMDClass(typeHnd))
 138     {
 139         // The most likely to be used type handles are looked up first followed by
 140         // less likely to be used type handles
 141         if (typeHnd == SIMDFloatHandle)
 142         {
 143             simdBaseType = TYP_FLOAT;
 144             JITDUMP("  Known type SIMD Vector<Float>\n");
 145         }
 146         else if (typeHnd == SIMDIntHandle)
 147         {
 148             simdBaseType = TYP_INT;
 149             JITDUMP("  Known type SIMD Vector<Int>\n");
 150         }
 151         else if (typeHnd == SIMDVector2Handle)
 152         {
 153             simdBaseType = TYP_FLOAT;
 154             size         = 2 * genTypeSize(TYP_FLOAT);
 155             assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE));
 156             JITDUMP("  Known type Vector2\n");
 157         }
 158         else if (typeHnd == SIMDVector3Handle)
 159         {
 160             simdBaseType = TYP_FLOAT;
 161             size         = 3 * genTypeSize(TYP_FLOAT);
 162             assert(size == info.compCompHnd->getClassSize(typeHnd));
 163             JITDUMP("  Known type Vector3\n");
 164         }
 165         else if (typeHnd == SIMDVector4Handle)
 166         {
 167             simdBaseType = TYP_FLOAT;
 168             size         = 4 * genTypeSize(TYP_FLOAT);
 169             assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE));
 170             JITDUMP("  Known type Vector4\n");
 171         }
 172         else if (typeHnd == SIMDVectorHandle)
 173         {
 174             JITDUMP("  Known type Vector\n");
 175         }
 176         else if (typeHnd == SIMDUShortHandle)
 177         {
 178             simdBaseType = TYP_USHORT;
 179             JITDUMP("  Known type SIMD Vector<ushort>\n");
 180         }
 181         else if (typeHnd == SIMDUByteHandle)
 182         {
 183             simdBaseType = TYP_UBYTE;
 184             JITDUMP("  Known type SIMD Vector<ubyte>\n");
 185         }
 186         else if (typeHnd == SIMDDoubleHandle)
 187         {
 188             simdBaseType = TYP_DOUBLE;
 189             JITDUMP("  Known type SIMD Vector<Double>\n");
 190         }
 191         else if (typeHnd == SIMDLongHandle)
 192         {
 193             simdBaseType = TYP_LONG;
 194             JITDUMP("  Known type SIMD Vector<Long>\n");
 195         }
 196         else if (typeHnd == SIMDShortHandle)
 197         {
 198             simdBaseType = TYP_SHORT;
 199             JITDUMP("  Known type SIMD Vector<short>\n");
 200         }
 201         else if (typeHnd == SIMDByteHandle)
 202         {
 203             simdBaseType = TYP_BYTE;
 204             JITDUMP("  Known type SIMD Vector<byte>\n");
 205         }
 206         else if (typeHnd == SIMDUIntHandle)
 207         {
 208             simdBaseType = TYP_UINT;
 209             JITDUMP("  Known type SIMD Vector<uint>\n");
 210         }
 211         else if (typeHnd == SIMDULongHandle)
 212         {
 213             simdBaseType = TYP_ULONG;
 214             JITDUMP("  Known type SIMD Vector<ulong>\n");
 215         }
 216
 217         // slow path search
 218         if (simdBaseType == TYP_UNKNOWN)
 219         {
 220             // Doesn't match with any of the cached type handles.
 221             // Obtain base type by parsing fully qualified class name.
 222             //
 223             // TODO-Throughput: implement product shipping solution to query base type.
 224             WCHAR  className[256] = {0};
 225             WCHAR* pbuf           = &className[0];
 226             int    len            = _countof(className);
 227             info.compCompHnd->appendClassName(&pbuf, &len, typeHnd, TRUE, FALSE, FALSE);
 228             noway_assert(pbuf < &className[256]);
 229             JITDUMP("SIMD Candidate Type %S\n", className);
 230
 231             if (wcsncmp(className, W("System.Numerics."), 16) == 0)
 232             {
 233                 if (wcsncmp(&(className[16]), W("Vector`1["), 9) == 0)
 234                 {
 235                     if (wcsncmp(&(className[25]), W("System.Single"), 13) == 0)
 236                     {
 237                         SIMDFloatHandle = typeHnd;
 238                         simdBaseType    = TYP_FLOAT;
 239                         JITDUMP("  Found type SIMD Vector<Float>\n");
 240                     }
 241                     else if (wcsncmp(&(className[25]), W("System.Int32"), 12) == 0)
 242                     {
 243                         SIMDIntHandle = typeHnd;
 244                         simdBaseType  = TYP_INT;
 245                         JITDUMP("  Found type SIMD Vector<Int>\n");
 246                     }
 247                     else if (wcsncmp(&(className[25]), W("System.UInt16"), 13) == 0)
 248                     {
 249                         SIMDUShortHandle = typeHnd;
 250                         simdBaseType     = TYP_USHORT;
 251                         JITDUMP("  Found type SIMD Vector<ushort>\n");
 252                     }
 253                     else if (wcsncmp(&(className[25]), W("System.Byte"), 11) == 0)
 254                     {
 255                         SIMDUByteHandle = typeHnd;
 256                         simdBaseType    = TYP_UBYTE;
 257                         JITDUMP("  Found type SIMD Vector<ubyte>\n");
 258                     }
 259                     else if (wcsncmp(&(className[25]), W("System.Double"), 13) == 0)
 260                     {
 261                         SIMDDoubleHandle = typeHnd;
 262                         simdBaseType     = TYP_DOUBLE;
 263                         JITDUMP("  Found type SIMD Vector<Double>\n");
 264                     }
 265                     else if (wcsncmp(&(className[25]), W("System.Int64"), 12) == 0)
 266                     {
 267                         SIMDLongHandle = typeHnd;
 268                         simdBaseType   = TYP_LONG;
 269                         JITDUMP("  Found type SIMD Vector<Long>\n");
 270                     }
 271                     else if (wcsncmp(&(className[25]), W("System.Int16"), 12) == 0)
 272                     {
 273                         SIMDShortHandle = typeHnd;
 274                         simdBaseType    = TYP_SHORT;
 275                         JITDUMP("  Found type SIMD Vector<short>\n");
 276                     }
 277                     else if (wcsncmp(&(className[25]), W("System.SByte"), 12) == 0)
 278                     {
 279                         SIMDByteHandle = typeHnd;
 280                         simdBaseType   = TYP_BYTE;
 281                         JITDUMP("  Found type SIMD Vector<byte>\n");
 282                     }
 283                     else if (wcsncmp(&(className[25]), W("System.UInt32"), 13) == 0)
 284                     {
 285                         SIMDUIntHandle = typeHnd;
 286                         simdBaseType   = TYP_UINT;
 287                         JITDUMP("  Found type SIMD Vector<uint>\n");
 288                     }
 289                     else if (wcsncmp(&(className[25]), W("System.UInt64"), 13) == 0)
 290                     {
 291                         SIMDULongHandle = typeHnd;
 292                         simdBaseType    = TYP_ULONG;
 293                         JITDUMP("  Found type SIMD Vector<ulong>\n");
 294                     }
 295                     else
 296                     {
 297                         JITDUMP("  Unknown SIMD Vector<T>\n");
 298                     }
 299                 }
 300                 else if (wcsncmp(&(className[16]), W("Vector2"), 8) == 0)
 301                 {
 302                     SIMDVector2Handle = typeHnd;
 303
 304                     simdBaseType = TYP_FLOAT;
 305                     size         = 2 * genTypeSize(TYP_FLOAT);
 306                     assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE));
 307                     JITDUMP(" Found Vector2\n");
 308                 }
 309                 else if (wcsncmp(&(className[16]), W("Vector3"), 8) == 0)
 310                 {
 311                     SIMDVector3Handle = typeHnd;
 312
 313                     simdBaseType = TYP_FLOAT;
 314                     size         = 3 * genTypeSize(TYP_FLOAT);
 315                     assert(size == info.compCompHnd->getClassSize(typeHnd));
 316                     JITDUMP(" Found Vector3\n");
 317                 }
 318                 else if (wcsncmp(&(className[16]), W("Vector4"), 8) == 0)
 319                 {
 320                     SIMDVector4Handle = typeHnd;
 321
 322                     simdBaseType = TYP_FLOAT;
 323                     size         = 4 * genTypeSize(TYP_FLOAT);
 324                     assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE));
 325                     JITDUMP(" Found Vector4\n");
 326                 }
 327                 else if (wcsncmp(&(className[16]), W("Vector"), 6) == 0)
 328                 {
 329                     SIMDVectorHandle = typeHnd;
 330                     JITDUMP(" Found type Vector\n");
 331                 }
 332                 else
 333                 {
 334                     JITDUMP("  Unknown SIMD Type\n");
 335                 }
 336             }
 337         }
 338         if (simdBaseType != TYP_UNKNOWN && sizeBytes != nullptr)
 339         {
 340             // If not a fixed size vector then its size is same as SIMD vector
 341             // register length in bytes
 342             if (size == 0)
 343             {
 344                 size = getSIMDVectorRegisterByteLength();
 345             }
 346
 347             *sizeBytes = size;
 348             setUsesSIMDTypes(true);
 349         }
 350     }
 351 #ifdef FEATURE_HW_INTRINSICS
 352     else if (isIntrinsicType(typeHnd))
 353     {
 354         const size_t Vector64SizeBytes  = 64 / 8;
 355         const size_t Vector128SizeBytes = 128 / 8;
 356         const size_t Vector256SizeBytes = 256 / 8;
 357
 358 #if defined(_TARGET_XARCH_)
 359         static_assert_no_msg(YMM_REGSIZE_BYTES == Vector256SizeBytes);
 360         static_assert_no_msg(XMM_REGSIZE_BYTES == Vector128SizeBytes);
 361
 362         if (typeHnd == Vector256FloatHandle)
 363         {
 364             simdBaseType = TYP_FLOAT;
 365             size         = Vector256SizeBytes;
 366             JITDUMP("  Known type Vector256<float>\n");
 367         }
 368         else if (typeHnd == Vector256DoubleHandle)
 369         {
 370             simdBaseType = TYP_DOUBLE;
 371             size         = Vector256SizeBytes;
 372             JITDUMP("  Known type Vector256<double>\n");
 373         }
 374         else if (typeHnd == Vector256IntHandle)
 375         {
 376             simdBaseType = TYP_INT;
 377             size         = Vector256SizeBytes;
 378             JITDUMP("  Known type Vector256<int>\n");
 379         }
 380         else if (typeHnd == Vector256UIntHandle)
 381         {
 382             simdBaseType = TYP_UINT;
 383             size         = Vector256SizeBytes;
 384             JITDUMP("  Known type Vector256<uint>\n");
 385         }
 386         else if (typeHnd == Vector256ShortHandle)
 387         {
 388             simdBaseType = TYP_SHORT;
 389             size         = Vector256SizeBytes;
 390             JITDUMP("  Known type Vector256<short>\n");
 391         }
 392         else if (typeHnd == Vector256UShortHandle)
 393         {
 394             simdBaseType = TYP_USHORT;
 395             size         = Vector256SizeBytes;
 396             JITDUMP("  Known type Vector256<ushort>\n");
 397         }
 398         else if (typeHnd == Vector256ByteHandle)
 399         {
 400             simdBaseType = TYP_BYTE;
 401             size         = Vector256SizeBytes;
 402             JITDUMP("  Known type Vector256<sbyte>\n");
 403         }
 404         else if (typeHnd == Vector256UByteHandle)
 405         {
 406             simdBaseType = TYP_UBYTE;
 407             size         = Vector256SizeBytes;
 408             JITDUMP("  Known type Vector256<byte>\n");
 409         }
 410         else if (typeHnd == Vector256LongHandle)
 411         {
 412             simdBaseType = TYP_LONG;
 413             size         = Vector256SizeBytes;
 414             JITDUMP("  Known type Vector256<long>\n");
 415         }
 416         else if (typeHnd == Vector256ULongHandle)
 417         {
 418             simdBaseType = TYP_ULONG;
 419             size         = Vector256SizeBytes;
 420             JITDUMP("  Known type Vector256<ulong>\n");
 421         }
 422         else if (typeHnd == Vector256FloatHandle)
 423         {
 424             simdBaseType = TYP_FLOAT;
 425             size         = Vector256SizeBytes;
 426             JITDUMP("  Known type Vector256<float>\n");
 427         }
 428         else
 429 #endif // defined(_TARGET_XARCH)
 430             if (typeHnd == Vector128DoubleHandle)
 431         {
 432             simdBaseType = TYP_DOUBLE;
 433             size         = Vector128SizeBytes;
 434             JITDUMP("  Known type Vector128<double>\n");
 435         }
 436         else if (typeHnd == Vector128IntHandle)
 437         {
 438             simdBaseType = TYP_INT;
 439             size         = Vector128SizeBytes;
 440             JITDUMP("  Known type Vector128<int>\n");
 441         }
 442         else if (typeHnd == Vector128UIntHandle)
 443         {
 444             simdBaseType = TYP_UINT;
 445             size         = Vector128SizeBytes;
 446             JITDUMP("  Known type Vector128<uint>\n");
 447         }
 448         else if (typeHnd == Vector128ShortHandle)
 449         {
 450             simdBaseType = TYP_SHORT;
 451             size         = Vector128SizeBytes;
 452             JITDUMP("  Known type Vector128<short>\n");
 453         }
 454         else if (typeHnd == Vector128UShortHandle)
 455         {
 456             simdBaseType = TYP_USHORT;
 457             size         = Vector128SizeBytes;
 458             JITDUMP("  Known type Vector128<ushort>\n");
 459         }
 460         else if (typeHnd == Vector128ByteHandle)
 461         {
 462             simdBaseType = TYP_BYTE;
 463             size         = Vector128SizeBytes;
 464             JITDUMP("  Known type Vector128<sbyte>\n");
 465         }
 466         else if (typeHnd == Vector128UByteHandle)
 467         {
 468             simdBaseType = TYP_UBYTE;
 469             size         = Vector128SizeBytes;
 470             JITDUMP("  Known type Vector128<byte>\n");
 471         }
 472         else if (typeHnd == Vector128LongHandle)
 473         {
 474             simdBaseType = TYP_LONG;
 475             size         = Vector128SizeBytes;
 476             JITDUMP("  Known type Vector128<long>\n");
 477         }
 478         else if (typeHnd == Vector128ULongHandle)
 479         {
 480             simdBaseType = TYP_ULONG;
 481             size         = Vector128SizeBytes;
 482             JITDUMP("  Known type Vector128<ulong>\n");
 483         }
 484 #if defined(_TARGET_ARM64_)
 485         else if (typeHnd == Vector64IntHandle)
 486         {
 487             simdBaseType = TYP_INT;
 488             size         = Vector64SizeBytes;
 489             JITDUMP("  Known type Vector64<int>\n");
 490         }
 491         else if (typeHnd == Vector64UIntHandle)
 492         {
 493             simdBaseType = TYP_UINT;
 494             size         = Vector64SizeBytes;
 495             JITDUMP("  Known type Vector64<uint>\n");
 496         }
 497         else if (typeHnd == Vector64ShortHandle)
 498         {
 499             simdBaseType = TYP_SHORT;
 500             size         = Vector64SizeBytes;
 501             JITDUMP("  Known type Vector64<short>\n");
 502         }
 503         else if (typeHnd == Vector64UShortHandle)
 504         {
 505             simdBaseType = TYP_USHORT;
 506             size         = Vector64SizeBytes;
 507             JITDUMP("  Known type Vector64<ushort>\n");
 508         }
 509         else if (typeHnd == Vector64ByteHandle)
 510         {
 511             simdBaseType = TYP_BYTE;
 512             size         = Vector64SizeBytes;
 513             JITDUMP("  Known type Vector64<sbyte>\n");
 514         }
 515         else if (typeHnd == Vector64UByteHandle)
 516         {
 517             simdBaseType = TYP_UBYTE;
 518             size         = Vector64SizeBytes;
 519             JITDUMP("  Known type Vector64<byte>\n");
 520         }
 521 #endif // defined(_TARGET_ARM64_)
 522
 523         // slow path search
 524         if (simdBaseType == TYP_UNKNOWN)
 525         {
 526             // Doesn't match with any of the cached type handles.
 527             const char*          className   = getClassNameFromMetadata(typeHnd, nullptr);
 528             CORINFO_CLASS_HANDLE baseTypeHnd = getTypeInstantiationArgument(typeHnd, 0);
 529
 530             if (baseTypeHnd != nullptr)
 531             {
 532                 CorInfoType type = info.compCompHnd->getTypeForPrimitiveNumericClass(baseTypeHnd);
 533
 534                 JITDUMP("HW Intrinsic SIMD Candidate Type %s with Base Type %s\n", className,
 535                         getClassNameFromMetadata(baseTypeHnd, nullptr));
 536
 537 #if defined(_TARGET_XARCH_)
 538                 if (strcmp(className, "Vector256`1") == 0)
 539                 {
 540                     size = Vector256SizeBytes;
 541                     switch (type)
 542                     {
 543                         case CORINFO_TYPE_FLOAT:
 544                             Vector256FloatHandle = typeHnd;
 545                             simdBaseType         = TYP_FLOAT;
 546                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<float>\n");
 547                             break;
 548                         case CORINFO_TYPE_DOUBLE:
 549                             Vector256DoubleHandle = typeHnd;
 550                             simdBaseType          = TYP_DOUBLE;
 551                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<double>\n");
 552                             break;
 553                         case CORINFO_TYPE_INT:
 554                             Vector256IntHandle = typeHnd;
 555                             simdBaseType       = TYP_INT;
 556                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<int>\n");
 557                             break;
 558                         case CORINFO_TYPE_UINT:
 559                             Vector256UIntHandle = typeHnd;
 560                             simdBaseType        = TYP_UINT;
 561                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<uint>\n");
 562                             break;
 563                         case CORINFO_TYPE_SHORT:
 564                             Vector256ShortHandle = typeHnd;
 565                             simdBaseType         = TYP_SHORT;
 566                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<short>\n");
 567                             break;
 568                         case CORINFO_TYPE_USHORT:
 569                             Vector256UShortHandle = typeHnd;
 570                             simdBaseType          = TYP_USHORT;
 571                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<ushort>\n");
 572                             break;
 573                         case CORINFO_TYPE_LONG:
 574                             Vector256LongHandle = typeHnd;
 575                             simdBaseType        = TYP_LONG;
 576                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<long>\n");
 577                             break;
 578                         case CORINFO_TYPE_ULONG:
 579                             Vector256ULongHandle = typeHnd;
 580                             simdBaseType         = TYP_ULONG;
 581                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<ulong>\n");
 582                             break;
 583                         case CORINFO_TYPE_UBYTE:
 584                             Vector256UByteHandle = typeHnd;
 585                             simdBaseType         = TYP_UBYTE;
 586                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<byte>\n");
 587                             break;
 588                         case CORINFO_TYPE_BYTE:
 589                             Vector256ByteHandle = typeHnd;
 590                             simdBaseType        = TYP_BYTE;
 591                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<sbyte>\n");
 592                             break;
 593
 594                         default:
 595                             JITDUMP("  Unknown Hardware Intrinsic SIMD Type Vector256<T>\n");
 596                     }
 597                 }
 598                 else
 599 #endif // defined(_TARGET_XARCH_)
 600                     if (strcmp(className, "Vector128`1") == 0)
 601                 {
 602                     size = Vector128SizeBytes;
 603                     switch (type)
 604                     {
 605                         case CORINFO_TYPE_FLOAT:
 606                             Vector128FloatHandle = typeHnd;
 607                             simdBaseType         = TYP_FLOAT;
 608                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<float>\n");
 609                             break;
 610                         case CORINFO_TYPE_DOUBLE:
 611                             Vector128DoubleHandle = typeHnd;
 612                             simdBaseType          = TYP_DOUBLE;
 613                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<double>\n");
 614                             break;
 615                         case CORINFO_TYPE_INT:
 616                             Vector128IntHandle = typeHnd;
 617                             simdBaseType       = TYP_INT;
 618                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<int>\n");
 619                             break;
 620                         case CORINFO_TYPE_UINT:
 621                             Vector128UIntHandle = typeHnd;
 622                             simdBaseType        = TYP_UINT;
 623                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<uint>\n");
 624                             break;
 625                         case CORINFO_TYPE_SHORT:
 626                             Vector128ShortHandle = typeHnd;
 627                             simdBaseType         = TYP_SHORT;
 628                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<short>\n");
 629                             break;
 630                         case CORINFO_TYPE_USHORT:
 631                             Vector128UShortHandle = typeHnd;
 632                             simdBaseType          = TYP_USHORT;
 633                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<ushort>\n");
 634                             break;
 635                         case CORINFO_TYPE_LONG:
 636                             Vector128LongHandle = typeHnd;
 637                             simdBaseType        = TYP_LONG;
 638                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<long>\n");
 639                             break;
 640                         case CORINFO_TYPE_ULONG:
 641                             Vector128ULongHandle = typeHnd;
 642                             simdBaseType         = TYP_ULONG;
 643                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<ulong>\n");
 644                             break;
 645                         case CORINFO_TYPE_UBYTE:
 646                             Vector128UByteHandle = typeHnd;
 647                             simdBaseType         = TYP_UBYTE;
 648                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<byte>\n");
 649                             break;
 650                         case CORINFO_TYPE_BYTE:
 651                             Vector128ByteHandle = typeHnd;
 652                             simdBaseType        = TYP_BYTE;
 653                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<sbyte>\n");
 654                             break;
 655
 656                         default:
 657                             JITDUMP("  Unknown Hardware Intrinsic SIMD Type Vector128<T>\n");
 658                     }
 659                 }
 660 #if defined(_TARGET_ARM64_)
 661                 else if (strcmp(className, "Vector64`1") == 0)
 662                 {
 663                     size = Vector64SizeBytes;
 664                     switch (type)
 665                     {
 666                         case CORINFO_TYPE_FLOAT:
 667                             Vector64FloatHandle = typeHnd;
 668                             simdBaseType        = TYP_FLOAT;
 669                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<float>\n");
 670                             break;
 671                         case CORINFO_TYPE_INT:
 672                             Vector64IntHandle = typeHnd;
 673                             simdBaseType      = TYP_INT;
 674                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<int>\n");
 675                             break;
 676                         case CORINFO_TYPE_UINT:
 677                             Vector64UIntHandle = typeHnd;
 678                             simdBaseType       = TYP_UINT;
 679                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<uint>\n");
 680                             break;
 681                         case CORINFO_TYPE_SHORT:
 682                             Vector64ShortHandle = typeHnd;
 683                             simdBaseType        = TYP_SHORT;
 684                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<short>\n");
 685                             break;
 686                         case CORINFO_TYPE_USHORT:
 687                             Vector64UShortHandle = typeHnd;
 688                             simdBaseType         = TYP_USHORT;
 689                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<ushort>\n");
 690                             break;
 691                         case CORINFO_TYPE_UBYTE:
 692                             Vector64UByteHandle = typeHnd;
 693                             simdBaseType        = TYP_UBYTE;
 694                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<byte>\n");
 695                             break;
 696                         case CORINFO_TYPE_BYTE:
 697                             Vector64ByteHandle = typeHnd;
 698                             simdBaseType       = TYP_BYTE;
 699                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<sbyte>\n");
 700                             break;
 701
 702                         default:
 703                             JITDUMP("  Unknown Hardware Intrinsic SIMD Type Vector64<T>\n");
 704                     }
 705                 }
 706 #endif // defined(_TARGET_ARM64_)
 707             }
 708         }
 709
 710         if (sizeBytes != nullptr)
 711         {
 712             *sizeBytes = size;
 713         }
 714
 715         if (simdBaseType != TYP_UNKNOWN)
 716         {
 717             setUsesSIMDTypes(true);
 718         }
 719     }
 720 #endif // FEATURE_HW_INTRINSICS
 721
 722     return simdBaseType;
 723 }
 724
 725 //--------------------------------------------------------------------------------------
 726 // getSIMDIntrinsicInfo: get SIMD intrinsic info given the method handle.
 727 //
 728 // Arguments:
 729 //    inOutTypeHnd    - The handle of the type on which the method is invoked.  This is an in-out param.
 730 //    methodHnd       - The handle of the method we're interested in.
 731 //    sig             - method signature info
 732 //    isNewObj        - whether this call represents a newboj constructor call
 733 //    argCount        - argument count - out pram
 734 //    baseType        - base type of the intrinsic - out param
 735 //    sizeBytes       - size of SIMD vector type on which the method is invoked - out param
 736 //
 737 // Return Value:
 738 //    SIMDIntrinsicInfo struct initialized corresponding to methodHnd.
 739 //    Sets SIMDIntrinsicInfo.id to SIMDIntrinsicInvalid if methodHnd doesn't correspond
 740 //    to any SIMD intrinsic.  Also, sets the out params inOutTypeHnd, argCount, baseType and
 741 //    sizeBytes.
 742 //
 743 //    Note that VectorMath class doesn't have a base type and first argument of the method
 744 //    determines the SIMD vector type on which intrinsic is invoked. In such a case inOutTypeHnd
 745 //    is modified by this routine.
 746 //
 747 // TODO-Throughput: The current implementation is based on method name string parsing.
 748 //         Although we now have type identification from the VM, the parsing of intrinsic names
 749 //         could be made more efficient.
 750 //
 751 const SIMDIntrinsicInfo* Compiler::getSIMDIntrinsicInfo(CORINFO_CLASS_HANDLE* inOutTypeHnd,
 752                                                         CORINFO_METHOD_HANDLE methodHnd,
 753                                                         CORINFO_SIG_INFO*     sig,
 754                                                         bool                  isNewObj,
 755                                                         unsigned*             argCount,
 756                                                         var_types*            baseType,
 757                                                         unsigned*             sizeBytes)
 758 {
 759     assert(featureSIMD);
 760     assert(baseType != nullptr);
 761     assert(sizeBytes != nullptr);
 762
 763     // get baseType and size of the type
 764     CORINFO_CLASS_HANDLE typeHnd = *inOutTypeHnd;
 765     *baseType                    = getBaseTypeAndSizeOfSIMDType(typeHnd, sizeBytes);
 766
 767     if (typeHnd == SIMDVectorHandle)
 768     {
 769         // All of the supported intrinsics on this static class take a first argument that's a vector,
 770         // which determines the baseType.
 771         // The exception is the IsHardwareAccelerated property, which is handled as a special case.
 772         assert(*baseType == TYP_UNKNOWN);
 773         if (sig->numArgs == 0)
 774         {
 775             const SIMDIntrinsicInfo* hwAccelIntrinsicInfo = &(simdIntrinsicInfoArray[SIMDIntrinsicHWAccel]);
 776             if ((strcmp(eeGetMethodName(methodHnd, nullptr), hwAccelIntrinsicInfo->methodName) == 0) &&
 777                 JITtype2varType(sig->retType) == hwAccelIntrinsicInfo->retType)
 778             {
 779                 // Sanity check
 780                 assert(hwAccelIntrinsicInfo->argCount == 0 && hwAccelIntrinsicInfo->isInstMethod == false);
 781                 return hwAccelIntrinsicInfo;
 782             }
 783             return nullptr;
 784         }
 785         else
 786         {
 787             typeHnd       = info.compCompHnd->getArgClass(sig, sig->args);
 788             *inOutTypeHnd = typeHnd;
 789             *baseType     = getBaseTypeAndSizeOfSIMDType(typeHnd, sizeBytes);
 790         }
 791     }
 792
 793     if (*baseType == TYP_UNKNOWN)
 794     {
 795         JITDUMP("NOT a SIMD Intrinsic: unsupported baseType\n");
 796         return nullptr;
 797     }
 798
 799     // account for implicit "this" arg
 800     *argCount = sig->numArgs;
 801     if (sig->hasThis())
 802     {
 803         *argCount += 1;
 804     }
 805
 806     // Get the Intrinsic Id by parsing method name.
 807     //
 808     // TODO-Throughput: replace sequential search by binary search by arranging entries
 809     // sorted by method name.
 810     SIMDIntrinsicID intrinsicId = SIMDIntrinsicInvalid;
 811     const char*     methodName  = eeGetMethodName(methodHnd, nullptr);
 812     for (int i = SIMDIntrinsicNone + 1; i < SIMDIntrinsicInvalid; ++i)
 813     {
 814         if (strcmp(methodName, simdIntrinsicInfoArray[i].methodName) == 0)
 815         {
 816             // Found an entry for the method; further check whether it is one of
 817             // the supported base types.
 818             bool found = false;
 819             for (int j = 0; j < SIMD_INTRINSIC_MAX_BASETYPE_COUNT; ++j)
 820             {
 821                 // Convention: if there are fewer base types supported than MAX_BASETYPE_COUNT,
 822                 // the end of the list is marked by TYP_UNDEF.
 823                 if (simdIntrinsicInfoArray[i].supportedBaseTypes[j] == TYP_UNDEF)
 824                 {
 825                     break;
 826                 }
 827
 828                 if (simdIntrinsicInfoArray[i].supportedBaseTypes[j] == *baseType)
 829                 {
 830                     found = true;
 831                     break;
 832                 }
 833             }
 834
 835             if (!found)
 836             {
 837                 continue;
 838             }
 839
 840             // Now, check the arguments.
 841             unsigned int fixedArgCnt    = simdIntrinsicInfoArray[i].argCount;
 842             unsigned int expectedArgCnt = fixedArgCnt;
 843
 844             // First handle SIMDIntrinsicInitN, where the arg count depends on the type.
 845             // The listed arg types include the vector and the first two init values, which is the expected number
 846             // for Vector2.  For other cases, we'll check their types here.
 847             if (*argCount > expectedArgCnt)
 848             {
 849                 if (i == SIMDIntrinsicInitN)
 850                 {
 851                     if (*argCount == 3 && typeHnd == SIMDVector2Handle)
 852                     {
 853                         expectedArgCnt = 3;
 854                     }
 855                     else if (*argCount == 4 && typeHnd == SIMDVector3Handle)
 856                     {
 857                         expectedArgCnt = 4;
 858                     }
 859                     else if (*argCount == 5 && typeHnd == SIMDVector4Handle)
 860                     {
 861                         expectedArgCnt = 5;
 862                     }
 863                 }
 864                 else if (i == SIMDIntrinsicInitFixed)
 865                 {
 866                     if (*argCount == 4 && typeHnd == SIMDVector4Handle)
 867                     {
 868                         expectedArgCnt = 4;
 869                     }
 870                 }
 871             }
 872             if (*argCount != expectedArgCnt)
 873             {
 874                 continue;
 875             }
 876
 877             // Validate the types of individual args passed are what is expected of.
 878             // If any of the types don't match with what is expected, don't consider
 879             // as an intrinsic.  This will make an older JIT with SIMD capabilities
 880             // resilient to breaking changes to SIMD managed API.
 881             //
 882             // Note that from IL type stack, args get popped in right to left order
 883             // whereas args get listed in method signatures in left to right order.
 884
 885             int stackIndex = (expectedArgCnt - 1);
 886
 887             // Track the arguments from the signature - we currently only use this to distinguish
 888             // integral and pointer types, both of which will by TYP_I_IMPL on the importer stack.
 889             CORINFO_ARG_LIST_HANDLE argLst = sig->args;
 890
 891             CORINFO_CLASS_HANDLE argClass;
 892             for (unsigned int argIndex = 0; found == true && argIndex < expectedArgCnt; argIndex++)
 893             {
 894                 bool isThisPtr = ((argIndex == 0) && sig->hasThis());
 895
 896                 // In case of "newobj SIMDVector<T>(T val)", thisPtr won't be present on type stack.
 897                 // We don't check anything in that case.
 898                 if (!isThisPtr || !isNewObj)
 899                 {
 900                     GenTree*  arg     = impStackTop(stackIndex).val;
 901                     var_types argType = arg->TypeGet();
 902
 903                     var_types expectedArgType;
 904                     if (argIndex < fixedArgCnt)
 905                     {
 906                         // Convention:
 907                         //   - intrinsicInfo.argType[i] == TYP_UNDEF - intrinsic doesn't have a valid arg at position i
 908                         //   - intrinsicInfo.argType[i] == TYP_UNKNOWN - arg type should be same as basetype
 909                         // Note that we pop the args off in reverse order.
 910                         expectedArgType = simdIntrinsicInfoArray[i].argType[argIndex];
 911                         assert(expectedArgType != TYP_UNDEF);
 912                         if (expectedArgType == TYP_UNKNOWN)
 913                         {
 914                             // The type of the argument will be genActualType(*baseType).
 915                             expectedArgType = genActualType(*baseType);
 916                             argType         = genActualType(argType);
 917                         }
 918                     }
 919                     else
 920                     {
 921                         expectedArgType = *baseType;
 922                     }
 923
 924                     if (!isThisPtr && argType == TYP_I_IMPL)
 925                     {
 926                         // The reference implementation has a constructor that takes a pointer.
 927                         // We don't want to recognize that one.  This requires us to look at the CorInfoType
 928                         // in order to distinguish a signature with a pointer argument from one with an
 929                         // integer argument of pointer size, both of which will be TYP_I_IMPL on the stack.
 930                         // TODO-Review: This seems quite fragile.  We should consider beefing up the checking
 931                         // here.
 932                         CorInfoType corType = strip(info.compCompHnd->getArgType(sig, argLst, &argClass));
 933                         if (corType == CORINFO_TYPE_PTR)
 934                         {
 935                             found = false;
 936                         }
 937                     }
 938
 939                     if (varTypeIsSIMD(argType))
 940                     {
 941                         argType = TYP_STRUCT;
 942                     }
 943                     if (argType != expectedArgType)
 944                     {
 945                         found = false;
 946                     }
 947                 }
 948                 if (argIndex != 0 || !sig->hasThis())
 949                 {
 950                     argLst = info.compCompHnd->getArgNext(argLst);
 951                 }
 952                 stackIndex--;
 953             }
 954
 955             // Cross check return type and static vs. instance is what we are expecting.
 956             // If not, don't consider it as an intrinsic.
 957             // Note that ret type of TYP_UNKNOWN means that it is not known apriori and must be same as baseType
 958             if (found)
 959             {
 960                 var_types expectedRetType = simdIntrinsicInfoArray[i].retType;
 961                 if (expectedRetType == TYP_UNKNOWN)
 962                 {
 963                     // JIT maps uint/ulong type vars to TYP_INT/TYP_LONG.
 964                     expectedRetType =
 965                         (*baseType == TYP_UINT || *baseType == TYP_ULONG) ? genActualType(*baseType) : *baseType;
 966                 }
 967
 968                 if (JITtype2varType(sig->retType) != expectedRetType ||
 969                     sig->hasThis() != simdIntrinsicInfoArray[i].isInstMethod)
 970                 {
 971                     found = false;
 972                 }
 973             }
 974
 975             if (found)
 976             {
 977                 intrinsicId = (SIMDIntrinsicID)i;
 978                 break;
 979             }
 980         }
 981     }
 982
 983     if (intrinsicId != SIMDIntrinsicInvalid)
 984     {
 985         JITDUMP("Method %s maps to SIMD intrinsic %s\n", methodName, simdIntrinsicNames[intrinsicId]);
 986         return &simdIntrinsicInfoArray[intrinsicId];
 987     }
 988     else
 989     {
 990         JITDUMP("Method %s is NOT a SIMD intrinsic\n", methodName);
 991     }
 992
 993     return nullptr;
 994 }
 995
 996 // Pops and returns GenTree node from importer's type stack.
 997 // Normalizes TYP_STRUCT value in case of GT_CALL, GT_RET_EXPR and arg nodes.
 998 //
 999 // Arguments:
1000 //    type        -  the type of value that the caller expects to be popped off the stack.
1001 //    expectAddr  -  if true indicates we are expecting type stack entry to be a TYP_BYREF.
1002 //
1003 // Notes:
1004 //    If the popped value is a struct, and the expected type is a simd type, it will be set
1005 //    to that type, otherwise it will assert if the type being popped is not the expected type.
1006
1007 GenTree* Compiler::impSIMDPopStack(var_types type, bool expectAddr)
1008 {
1009     StackEntry se   = impPopStack();
1010     typeInfo   ti   = se.seTypeInfo;
1011     GenTree*   tree = se.val;
1012
1013     // If expectAddr is true implies what we have on stack is address and we need
1014     // SIMD type struct that it points to.
1015     if (expectAddr)
1016     {
1017         assert(tree->TypeGet() == TYP_BYREF);
1018         if (tree->OperGet() == GT_ADDR)
1019         {
1020             tree = tree->gtGetOp1();
1021         }
1022         else
1023         {
1024             tree = gtNewOperNode(GT_IND, type, tree);
1025         }
1026     }
1027
1028     bool isParam = false;
1029
1030     // If we have a ldobj of a SIMD local we need to transform it.
1031     if (tree->OperGet() == GT_OBJ)
1032     {
1033         GenTree* addr = tree->gtOp.gtOp1;
1034         if ((addr->OperGet() == GT_ADDR) && isSIMDTypeLocal(addr->gtOp.gtOp1))
1035         {
1036             tree = addr->gtOp.gtOp1;
1037         }
1038     }
1039
1040     if (tree->OperGet() == GT_LCL_VAR)
1041     {
1042         unsigned   lclNum    = tree->AsLclVarCommon()->GetLclNum();
1043         LclVarDsc* lclVarDsc = &lvaTable[lclNum];
1044         isParam              = lclVarDsc->lvIsParam;
1045     }
1046
1047     // normalize TYP_STRUCT value
1048     if (varTypeIsStruct(tree) && ((tree->OperGet() == GT_RET_EXPR) || (tree->OperGet() == GT_CALL) || isParam))
1049     {
1050         assert(ti.IsType(TI_STRUCT));
1051         CORINFO_CLASS_HANDLE structType = ti.GetClassHandleForValueClass();
1052         tree                            = impNormStructVal(tree, structType, (unsigned)CHECK_SPILL_ALL);
1053     }
1054
1055     // Now set the type of the tree to the specialized SIMD struct type, if applicable.
1056     if (genActualType(tree->gtType) != genActualType(type))
1057     {
1058         assert(tree->gtType == TYP_STRUCT);
1059         tree->gtType = type;
1060     }
1061     else if (tree->gtType == TYP_BYREF)
1062     {
1063         assert(tree->IsLocal() || (tree->OperGet() == GT_RET_EXPR) || (tree->OperGet() == GT_CALL) ||
1064                ((tree->gtOper == GT_ADDR) && varTypeIsSIMD(tree->gtGetOp1())));
1065     }
1066
1067     return tree;
1068 }
1069
1070 // impSIMDGetFixed: Create a GT_SIMD tree for a Get property of SIMD vector with a fixed index.
1071 //
1072 // Arguments:
1073 //    baseType - The base (element) type of the SIMD vector.
1074 //    simdSize - The total size in bytes of the SIMD vector.
1075 //    index    - The index of the field to get.
1076 //
1077 // Return Value:
1078 //    Returns a GT_SIMD node with the SIMDIntrinsicGetItem intrinsic id.
1079 //
1080 GenTreeSIMD* Compiler::impSIMDGetFixed(var_types simdType, var_types baseType, unsigned simdSize, int index)
1081 {
1082     assert(simdSize >= ((index + 1) * genTypeSize(baseType)));
1083
1084     // op1 is a SIMD source.
1085     GenTree* op1 = impSIMDPopStack(simdType, true);
1086
1087     GenTree*     op2      = gtNewIconNode(index);
1088     GenTreeSIMD* simdTree = gtNewSIMDNode(baseType, op1, op2, SIMDIntrinsicGetItem, baseType, simdSize);
1089     return simdTree;
1090 }
1091
1092 #ifdef _TARGET_XARCH_
1093 // impSIMDLongRelOpEqual: transforms operands and returns the SIMD intrinsic to be applied on
1094 // transformed operands to obtain == comparison result.
1095 //
1096 // Argumens:
1097 //    typeHnd  -  type handle of SIMD vector
1098 //    size     -  SIMD vector size
1099 //    op1      -  in-out parameter; first operand
1100 //    op2      -  in-out parameter; second operand
1101 //
1102 // Return Value:
1103 //    Modifies in-out params op1, op2 and returns intrinsic ID to be applied to modified operands
1104 //
1105 SIMDIntrinsicID Compiler::impSIMDLongRelOpEqual(CORINFO_CLASS_HANDLE typeHnd,
1106                                                 unsigned             size,
1107                                                 GenTree**            pOp1,
1108                                                 GenTree**            pOp2)
1109 {
1110     var_types simdType = (*pOp1)->TypeGet();
1111     assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1112
1113     // There is no direct SSE2 support for comparing TYP_LONG vectors.
1114     // These have to be implemented in terms of TYP_INT vector comparison operations.
1115     //
1116     // Equality(v1, v2):
1117     // tmp = (v1 == v2) i.e. compare for equality as if v1 and v2 are vector<int>
1118     // result = BitwiseAnd(t, shuffle(t, (2, 3, 0, 1)))
1119     // Shuffle is meant to swap the comparison results of low-32-bits and high 32-bits of respective long elements.
1120
1121     // Compare vector<long> as if they were vector<int> and assign the result to a temp
1122     GenTree* compResult = gtNewSIMDNode(simdType, *pOp1, *pOp2, SIMDIntrinsicEqual, TYP_INT, size);
1123     unsigned lclNum     = lvaGrabTemp(true DEBUGARG("SIMD Long =="));
1124     lvaSetStruct(lclNum, typeHnd, false);
1125     GenTree* tmp = gtNewLclvNode(lclNum, simdType);
1126     GenTree* asg = gtNewTempAssign(lclNum, compResult);
1127
1128     // op1 = GT_COMMA(tmp=compResult, tmp)
1129     // op2 = Shuffle(tmp, 0xB1)
1130     // IntrinsicId = BitwiseAnd
1131     *pOp1 = gtNewOperNode(GT_COMMA, simdType, asg, tmp);
1132     *pOp2 = gtNewSIMDNode(simdType, gtNewLclvNode(lclNum, simdType), gtNewIconNode(SHUFFLE_ZWXY, TYP_INT),
1133                           SIMDIntrinsicShuffleSSE2, TYP_INT, size);
1134     return SIMDIntrinsicBitwiseAnd;
1135 }
1136
1137 // impSIMDLongRelOpGreaterThan: transforms operands and returns the SIMD intrinsic to be applied on
1138 // transformed operands to obtain > comparison result.
1139 //
1140 // Argumens:
1141 //    typeHnd  -  type handle of SIMD vector
1142 //    size     -  SIMD vector size
1143 //    pOp1     -  in-out parameter; first operand
1144 //    pOp2     -  in-out parameter; second operand
1145 //
1146 // Return Value:
1147 //    Modifies in-out params pOp1, pOp2 and returns intrinsic ID to be applied to modified operands
1148 //
1149 SIMDIntrinsicID Compiler::impSIMDLongRelOpGreaterThan(CORINFO_CLASS_HANDLE typeHnd,
1150                                                       unsigned             size,
1151                                                       GenTree**            pOp1,
1152                                                       GenTree**            pOp2)
1153 {
1154     var_types simdType = (*pOp1)->TypeGet();
1155     assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1156
1157     // GreaterThan(v1, v2) where v1 and v2 are vector long.
1158     // Let us consider the case of single long element comparison.
1159     // say L1 = (x1, y1) and L2 = (x2, y2) where x1, y1, x2, and y2 are 32-bit integers that comprise the longs L1 and
1160     // L2.
1161     //
1162     // GreaterThan(L1, L2) can be expressed in terms of > relationship between 32-bit integers that comprise L1 and L2
1163     // as
1164     //                    =  (x1, y1) > (x2, y2)
1165     //                    =  (x1 > x2) || [(x1 == x2) && (y1 > y2)]   - eq (1)
1166     //
1167     // t = (v1 > v2)  32-bit signed comparison
1168     // u = (v1 == v2) 32-bit sized element equality
1169     // v = (v1 > v2)  32-bit unsigned comparison
1170     //
1171     // z = shuffle(t, (3, 3, 1, 1))  - This corresponds to (x1 > x2) in eq(1) above
1172     // t1 = Shuffle(v, (2, 2, 0, 0)) - This corresponds to (y1 > y2) in eq(1) above
1173     // u1 = Shuffle(u, (3, 3, 1, 1)) - This corresponds to (x1 == x2) in eq(1) above
1174     // w = And(t1, u1)               - This corresponds to [(x1 == x2) && (y1 > y2)] in eq(1) above
1175     // Result = BitwiseOr(z, w)
1176
1177     // Since op1 and op2 gets used multiple times, make sure side effects are computed.
1178     GenTree* dupOp1    = nullptr;
1179     GenTree* dupOp2    = nullptr;
1180     GenTree* dupDupOp1 = nullptr;
1181     GenTree* dupDupOp2 = nullptr;
1182
1183     if (((*pOp1)->gtFlags & GTF_SIDE_EFFECT) != 0)
1184     {
1185         dupOp1    = fgInsertCommaFormTemp(pOp1, typeHnd);
1186         dupDupOp1 = gtNewLclvNode(dupOp1->AsLclVarCommon()->GetLclNum(), simdType);
1187     }
1188     else
1189     {
1190         dupOp1    = gtCloneExpr(*pOp1);
1191         dupDupOp1 = gtCloneExpr(*pOp1);
1192     }
1193
1194     if (((*pOp2)->gtFlags & GTF_SIDE_EFFECT) != 0)
1195     {
1196         dupOp2    = fgInsertCommaFormTemp(pOp2, typeHnd);
1197         dupDupOp2 = gtNewLclvNode(dupOp2->AsLclVarCommon()->GetLclNum(), simdType);
1198     }
1199     else
1200     {
1201         dupOp2    = gtCloneExpr(*pOp2);
1202         dupDupOp2 = gtCloneExpr(*pOp2);
1203     }
1204
1205     assert(dupDupOp1 != nullptr && dupDupOp2 != nullptr);
1206     assert(dupOp1 != nullptr && dupOp2 != nullptr);
1207     assert(*pOp1 != nullptr && *pOp2 != nullptr);
1208
1209     // v1GreaterThanv2Signed - signed 32-bit comparison
1210     GenTree* v1GreaterThanv2Signed = gtNewSIMDNode(simdType, *pOp1, *pOp2, SIMDIntrinsicGreaterThan, TYP_INT, size);
1211
1212     // v1Equalsv2 - 32-bit equality
1213     GenTree* v1Equalsv2 = gtNewSIMDNode(simdType, dupOp1, dupOp2, SIMDIntrinsicEqual, TYP_INT, size);
1214
1215     // v1GreaterThanv2Unsigned - unsigned 32-bit comparison
1216     var_types       tempBaseType = TYP_UINT;
1217     SIMDIntrinsicID sid = impSIMDRelOp(SIMDIntrinsicGreaterThan, typeHnd, size, &tempBaseType, &dupDupOp1, &dupDupOp2);
1218     GenTree*        v1GreaterThanv2Unsigned = gtNewSIMDNode(simdType, dupDupOp1, dupDupOp2, sid, tempBaseType, size);
1219
1220     GenTree* z = gtNewSIMDNode(simdType, v1GreaterThanv2Signed, gtNewIconNode(SHUFFLE_WWYY, TYP_INT),
1221                                SIMDIntrinsicShuffleSSE2, TYP_FLOAT, size);
1222     GenTree* t1 = gtNewSIMDNode(simdType, v1GreaterThanv2Unsigned, gtNewIconNode(SHUFFLE_ZZXX, TYP_INT),
1223                                 SIMDIntrinsicShuffleSSE2, TYP_FLOAT, size);
1224     GenTree* u1 = gtNewSIMDNode(simdType, v1Equalsv2, gtNewIconNode(SHUFFLE_WWYY, TYP_INT), SIMDIntrinsicShuffleSSE2,
1225                                 TYP_FLOAT, size);
1226     GenTree* w = gtNewSIMDNode(simdType, u1, t1, SIMDIntrinsicBitwiseAnd, TYP_INT, size);
1227
1228     *pOp1 = z;
1229     *pOp2 = w;
1230     return SIMDIntrinsicBitwiseOr;
1231 }
1232
1233 // impSIMDLongRelOpGreaterThanOrEqual: transforms operands and returns the SIMD intrinsic to be applied on
1234 // transformed operands to obtain >= comparison result.
1235 //
1236 // Argumens:
1237 //    typeHnd  -  type handle of SIMD vector
1238 //    size     -  SIMD vector size
1239 //    pOp1      -  in-out parameter; first operand
1240 //    pOp2      -  in-out parameter; second operand
1241 //
1242 // Return Value:
1243 //    Modifies in-out params pOp1, pOp2 and returns intrinsic ID to be applied to modified operands
1244 //
1245 SIMDIntrinsicID Compiler::impSIMDLongRelOpGreaterThanOrEqual(CORINFO_CLASS_HANDLE typeHnd,
1246                                                              unsigned             size,
1247                                                              GenTree**            pOp1,
1248                                                              GenTree**            pOp2)
1249 {
1250     var_types simdType = (*pOp1)->TypeGet();
1251     assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1252
1253     // expand this to (a == b) | (a > b)
1254     GenTree* dupOp1 = nullptr;
1255     GenTree* dupOp2 = nullptr;
1256
1257     if (((*pOp1)->gtFlags & GTF_SIDE_EFFECT) != 0)
1258     {
1259         dupOp1 = fgInsertCommaFormTemp(pOp1, typeHnd);
1260     }
1261     else
1262     {
1263         dupOp1 = gtCloneExpr(*pOp1);
1264     }
1265
1266     if (((*pOp2)->gtFlags & GTF_SIDE_EFFECT) != 0)
1267     {
1268         dupOp2 = fgInsertCommaFormTemp(pOp2, typeHnd);
1269     }
1270     else
1271     {
1272         dupOp2 = gtCloneExpr(*pOp2);
1273     }
1274
1275     assert(dupOp1 != nullptr && dupOp2 != nullptr);
1276     assert(*pOp1 != nullptr && *pOp2 != nullptr);
1277
1278     // (a==b)
1279     SIMDIntrinsicID id = impSIMDLongRelOpEqual(typeHnd, size, pOp1, pOp2);
1280     *pOp1              = gtNewSIMDNode(simdType, *pOp1, *pOp2, id, TYP_LONG, size);
1281
1282     // (a > b)
1283     id    = impSIMDLongRelOpGreaterThan(typeHnd, size, &dupOp1, &dupOp2);
1284     *pOp2 = gtNewSIMDNode(simdType, dupOp1, dupOp2, id, TYP_LONG, size);
1285
1286     return SIMDIntrinsicBitwiseOr;
1287 }
1288
1289 // impSIMDInt32OrSmallIntRelOpGreaterThanOrEqual: transforms operands and returns the SIMD intrinsic to be applied on
1290 // transformed operands to obtain >= comparison result in case of integer base type vectors
1291 //
1292 // Argumens:
1293 //    typeHnd  -  type handle of SIMD vector
1294 //    size     -  SIMD vector size
1295 //    baseType -  base type of SIMD vector
1296 //    pOp1      -  in-out parameter; first operand
1297 //    pOp2      -  in-out parameter; second operand
1298 //
1299 // Return Value:
1300 //    Modifies in-out params pOp1, pOp2 and returns intrinsic ID to be applied to modified operands
1301 //
1302 SIMDIntrinsicID Compiler::impSIMDIntegralRelOpGreaterThanOrEqual(
1303     CORINFO_CLASS_HANDLE typeHnd, unsigned size, var_types baseType, GenTree** pOp1, GenTree** pOp2)
1304 {
1305     var_types simdType = (*pOp1)->TypeGet();
1306     assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1307
1308     // This routine should be used only for integer base type vectors
1309     assert(varTypeIsIntegral(baseType));
1310     if ((getSIMDSupportLevel() == SIMD_SSE2_Supported) && ((baseType == TYP_LONG) || baseType == TYP_UBYTE))
1311     {
1312         return impSIMDLongRelOpGreaterThanOrEqual(typeHnd, size, pOp1, pOp2);
1313     }
1314
1315     // expand this to (a == b) | (a > b)
1316     GenTree* dupOp1 = nullptr;
1317     GenTree* dupOp2 = nullptr;
1318
1319     if (((*pOp1)->gtFlags & GTF_SIDE_EFFECT) != 0)
1320     {
1321         dupOp1 = fgInsertCommaFormTemp(pOp1, typeHnd);
1322     }
1323     else
1324     {
1325         dupOp1 = gtCloneExpr(*pOp1);
1326     }
1327
1328     if (((*pOp2)->gtFlags & GTF_SIDE_EFFECT) != 0)
1329     {
1330         dupOp2 = fgInsertCommaFormTemp(pOp2, typeHnd);
1331     }
1332     else
1333     {
1334         dupOp2 = gtCloneExpr(*pOp2);
1335     }
1336
1337     assert(dupOp1 != nullptr && dupOp2 != nullptr);
1338     assert(*pOp1 != nullptr && *pOp2 != nullptr);
1339
1340     // (a==b)
1341     *pOp1 = gtNewSIMDNode(simdType, *pOp1, *pOp2, SIMDIntrinsicEqual, baseType, size);
1342
1343     // (a > b)
1344     *pOp2 = gtNewSIMDNode(simdType, dupOp1, dupOp2, SIMDIntrinsicGreaterThan, baseType, size);
1345
1346     return SIMDIntrinsicBitwiseOr;
1347 }
1348 #endif // _TARGET_XARCH_
1349
1350 // Transforms operands and returns the SIMD intrinsic to be applied on
1351 // transformed operands to obtain given relop result.
1352 //
1353 // Argumens:
1354 //    relOpIntrinsicId - Relational operator SIMD intrinsic
1355 //    typeHnd          - type handle of SIMD vector
1356 //    size             -  SIMD vector size
1357 //    inOutBaseType    - base type of SIMD vector
1358 //    pOp1             -  in-out parameter; first operand
1359 //    pOp2             -  in-out parameter; second operand
1360 //
1361 // Return Value:
1362 //    Modifies in-out params pOp1, pOp2, inOutBaseType and returns intrinsic ID to be applied to modified operands
1363 //
1364 SIMDIntrinsicID Compiler::impSIMDRelOp(SIMDIntrinsicID      relOpIntrinsicId,
1365                                        CORINFO_CLASS_HANDLE typeHnd,
1366                                        unsigned             size,
1367                                        var_types*           inOutBaseType,
1368                                        GenTree**            pOp1,
1369                                        GenTree**            pOp2)
1370 {
1371     var_types simdType = (*pOp1)->TypeGet();
1372     assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1373
1374     assert(isRelOpSIMDIntrinsic(relOpIntrinsicId));
1375
1376     SIMDIntrinsicID intrinsicID = relOpIntrinsicId;
1377 #ifdef _TARGET_XARCH_
1378     var_types baseType = *inOutBaseType;
1379
1380     if (varTypeIsFloating(baseType))
1381     {
1382         // SSE2/AVX doesn't support > and >= on vector float/double.
1383         // Therefore, we need to use < and <= with swapped operands
1384         if (relOpIntrinsicId == SIMDIntrinsicGreaterThan || relOpIntrinsicId == SIMDIntrinsicGreaterThanOrEqual)
1385         {
1386             GenTree* tmp = *pOp1;
1387             *pOp1        = *pOp2;
1388             *pOp2        = tmp;
1389
1390             intrinsicID =
1391                 (relOpIntrinsicId == SIMDIntrinsicGreaterThan) ? SIMDIntrinsicLessThan : SIMDIntrinsicLessThanOrEqual;
1392         }
1393     }
1394     else if (varTypeIsIntegral(baseType))
1395     {
1396         // SSE/AVX doesn't support < and <= on integer base type vectors.
1397         // Therefore, we need to use > and >= with swapped operands.
1398         if (intrinsicID == SIMDIntrinsicLessThan || intrinsicID == SIMDIntrinsicLessThanOrEqual)
1399         {
1400             GenTree* tmp = *pOp1;
1401             *pOp1        = *pOp2;
1402             *pOp2        = tmp;
1403
1404             intrinsicID = (relOpIntrinsicId == SIMDIntrinsicLessThan) ? SIMDIntrinsicGreaterThan
1405                                                                       : SIMDIntrinsicGreaterThanOrEqual;
1406         }
1407
1408         if ((getSIMDSupportLevel() == SIMD_SSE2_Supported) && baseType == TYP_LONG)
1409         {
1410             // There is no direct SSE2 support for comparing TYP_LONG vectors.
1411             // These have to be implemented interms of TYP_INT vector comparison operations.
1412             if (intrinsicID == SIMDIntrinsicEqual)
1413             {
1414                 intrinsicID = impSIMDLongRelOpEqual(typeHnd, size, pOp1, pOp2);
1415             }
1416             else if (intrinsicID == SIMDIntrinsicGreaterThan)
1417             {
1418                 intrinsicID = impSIMDLongRelOpGreaterThan(typeHnd, size, pOp1, pOp2);
1419             }
1420             else if (intrinsicID == SIMDIntrinsicGreaterThanOrEqual)
1421             {
1422                 intrinsicID = impSIMDLongRelOpGreaterThanOrEqual(typeHnd, size, pOp1, pOp2);
1423             }
1424             else
1425             {
1426                 unreached();
1427             }
1428         }
1429         // SSE2 and AVX direct support for signed comparison of int32, int16 and int8 types
1430         else if (!varTypeIsUnsigned(baseType))
1431         {
1432             if (intrinsicID == SIMDIntrinsicGreaterThanOrEqual)
1433             {
1434                 intrinsicID = impSIMDIntegralRelOpGreaterThanOrEqual(typeHnd, size, baseType, pOp1, pOp2);
1435             }
1436         }
1437         else // unsigned
1438         {
1439             // Vector<byte>, Vector<ushort>, Vector<uint> and Vector<ulong>:
1440             // SSE2 supports > for signed comparison. Therefore, to use it for
1441             // comparing unsigned numbers, we subtract a constant from both the
1442             // operands such that the result fits within the corresponding signed
1443             // type.  The resulting signed numbers are compared using SSE2 signed
1444             // comparison.
1445             //
1446             // Vector<byte>: constant to be subtracted is 2^7
1447             // Vector<ushort> constant to be subtracted is 2^15
1448             // Vector<uint> constant to be subtracted is 2^31
1449             // Vector<ulong> constant to be subtracted is 2^63
1450             //
1451             // We need to treat op1 and op2 as signed for comparison purpose after
1452             // the transformation.
1453             __int64 constVal = 0;
1454             switch (baseType)
1455             {
1456                 case TYP_UBYTE:
1457                     constVal       = 0x80808080;
1458                     *inOutBaseType = TYP_BYTE;
1459                     break;
1460                 case TYP_USHORT:
1461                     constVal       = 0x80008000;
1462                     *inOutBaseType = TYP_SHORT;
1463                     break;
1464                 case TYP_UINT:
1465                     constVal       = 0x80000000;
1466                     *inOutBaseType = TYP_INT;
1467                     break;
1468                 case TYP_ULONG:
1469                     constVal       = 0x8000000000000000LL;
1470                     *inOutBaseType = TYP_LONG;
1471                     break;
1472                 default:
1473                     unreached();
1474                     break;
1475             }
1476             assert(constVal != 0);
1477
1478             // This transformation is not required for equality.
1479             if (intrinsicID != SIMDIntrinsicEqual)
1480             {
1481                 // For constructing const vector use either long or int base type.
1482                 var_types tempBaseType;
1483                 GenTree*  initVal;
1484                 if (baseType == TYP_ULONG)
1485                 {
1486                     tempBaseType = TYP_LONG;
1487                     initVal      = gtNewLconNode(constVal);
1488                 }
1489                 else
1490                 {
1491                     tempBaseType = TYP_INT;
1492                     initVal      = gtNewIconNode((ssize_t)constVal);
1493                 }
1494                 initVal->gtType      = tempBaseType;
1495                 GenTree* constVector = gtNewSIMDNode(simdType, initVal, nullptr, SIMDIntrinsicInit, tempBaseType, size);
1496
1497                 // Assign constVector to a temp, since we intend to use it more than once
1498                 // TODO-CQ: We have quite a few such constant vectors constructed during
1499                 // the importation of SIMD intrinsics.  Make sure that we have a single
1500                 // temp per distinct constant per method.
1501                 GenTree* tmp = fgInsertCommaFormTemp(&constVector, typeHnd);
1502
1503                 // op1 = op1 - constVector
1504                 // op2 = op2 - constVector
1505                 *pOp1 = gtNewSIMDNode(simdType, *pOp1, constVector, SIMDIntrinsicSub, baseType, size);
1506                 *pOp2 = gtNewSIMDNode(simdType, *pOp2, tmp, SIMDIntrinsicSub, baseType, size);
1507             }
1508
1509             return impSIMDRelOp(intrinsicID, typeHnd, size, inOutBaseType, pOp1, pOp2);
1510         }
1511     }
1512 #elif defined(_TARGET_ARM64_)
1513     // TODO-ARM64-CQ handle comparisons against zero
1514
1515     // _TARGET_ARM64_ doesn't support < and <= on register register comparisons
1516     // Therefore, we need to use > and >= with swapped operands.
1517     if (intrinsicID == SIMDIntrinsicLessThan || intrinsicID == SIMDIntrinsicLessThanOrEqual)
1518     {
1519         GenTree* tmp = *pOp1;
1520         *pOp1        = *pOp2;
1521         *pOp2        = tmp;
1522
1523         intrinsicID =
1524             (intrinsicID == SIMDIntrinsicLessThan) ? SIMDIntrinsicGreaterThan : SIMDIntrinsicGreaterThanOrEqual;
1525     }
1526 #else  // !_TARGET_XARCH_
1527     assert(!"impSIMDRelOp() unimplemented on target arch");
1528     unreached();
1529 #endif // !_TARGET_XARCH_
1530
1531     return intrinsicID;
1532 }
1533
1534 //-------------------------------------------------------------------------
1535 // impSIMDAbs: creates GT_SIMD node to compute Abs value of a given vector.
1536 //
1537 // Arguments:
1538 //    typeHnd     -  type handle of SIMD vector
1539 //    baseType    -  base type of vector
1540 //    size        -  vector size in bytes
1541 //    op1         -  operand of Abs intrinsic
1542 //
1543 GenTree* Compiler::impSIMDAbs(CORINFO_CLASS_HANDLE typeHnd, var_types baseType, unsigned size, GenTree* op1)
1544 {
1545     assert(varTypeIsSIMD(op1));
1546
1547     var_types simdType = op1->TypeGet();
1548     GenTree*  retVal   = nullptr;
1549
1550 #ifdef _TARGET_XARCH_
1551     // When there is no direct support, Abs(v) could be computed
1552     // on integer vectors as follows:
1553     //     BitVector = v < vector.Zero
1554     //     result = ConditionalSelect(BitVector, vector.Zero - v, v)
1555
1556     bool useConditionalSelect = false;
1557     if (getSIMDSupportLevel() == SIMD_SSE2_Supported)
1558     {
1559         // SSE2 doesn't support abs on signed integer type vectors.
1560         if (baseType == TYP_LONG || baseType == TYP_INT || baseType == TYP_SHORT || baseType == TYP_BYTE)
1561         {
1562             useConditionalSelect = true;
1563         }
1564     }
1565     else
1566     {
1567         assert(getSIMDSupportLevel() >= SIMD_SSE4_Supported);
1568         if (baseType == TYP_LONG)
1569         {
1570             // SSE4/AVX2 don't support abs on long type vector.
1571             useConditionalSelect = true;
1572         }
1573     }
1574
1575     if (useConditionalSelect)
1576     {
1577         // This works only on integer vectors not on float/double vectors.
1578         assert(varTypeIsIntegral(baseType));
1579
1580         GenTree* op1Assign;
1581         unsigned op1LclNum;
1582
1583         if (op1->OperGet() == GT_LCL_VAR)
1584         {
1585             op1LclNum = op1->gtLclVarCommon.gtLclNum;
1586             op1Assign = nullptr;
1587         }
1588         else
1589         {
1590             op1LclNum = lvaGrabTemp(true DEBUGARG("SIMD Abs op1"));
1591             lvaSetStruct(op1LclNum, typeHnd, false);
1592             op1Assign = gtNewTempAssign(op1LclNum, op1);
1593             op1       = gtNewLclvNode(op1LclNum, op1->TypeGet());
1594         }
1595
1596         // Assign Vector.Zero to a temp since it is needed more than once
1597         GenTree* vecZero       = gtNewSIMDVectorZero(simdType, baseType, size);
1598         unsigned vecZeroLclNum = lvaGrabTemp(true DEBUGARG("SIMD Abs VecZero"));
1599         lvaSetStruct(vecZeroLclNum, typeHnd, false);
1600         GenTree* vecZeroAssign = gtNewTempAssign(vecZeroLclNum, vecZero);
1601
1602         // Construct BitVector = v < vector.Zero
1603         GenTree*        bitVecOp1     = op1;
1604         GenTree*        bitVecOp2     = gtNewLclvNode(vecZeroLclNum, vecZero->TypeGet());
1605         var_types       relOpBaseType = baseType;
1606         SIMDIntrinsicID relOpIntrinsic =
1607             impSIMDRelOp(SIMDIntrinsicLessThan, typeHnd, size, &relOpBaseType, &bitVecOp1, &bitVecOp2);
1608         GenTree* bitVec       = gtNewSIMDNode(simdType, bitVecOp1, bitVecOp2, relOpIntrinsic, relOpBaseType, size);
1609         unsigned bitVecLclNum = lvaGrabTemp(true DEBUGARG("SIMD Abs bitVec"));
1610         lvaSetStruct(bitVecLclNum, typeHnd, false);
1611         GenTree* bitVecAssign = gtNewTempAssign(bitVecLclNum, bitVec);
1612         bitVec                = gtNewLclvNode(bitVecLclNum, bitVec->TypeGet());
1613
1614         // Construct condSelectOp1 = vector.Zero - v
1615         GenTree* subOp1 = gtNewLclvNode(vecZeroLclNum, vecZero->TypeGet());
1616         GenTree* subOp2 = gtNewLclvNode(op1LclNum, op1->TypeGet());
1617         GenTree* negVec = gtNewSIMDNode(simdType, subOp1, subOp2, SIMDIntrinsicSub, baseType, size);
1618
1619         // Construct ConditionalSelect(bitVec, vector.Zero - v, v)
1620         GenTree* vec = gtNewLclvNode(op1LclNum, op1->TypeGet());
1621         retVal       = impSIMDSelect(typeHnd, baseType, size, bitVec, negVec, vec);
1622
1623         // Prepend bitVec assignment to retVal.
1624         // retVal = (tmp2 = v < tmp1), CondSelect(tmp2, tmp1 - v, v)
1625         retVal = gtNewOperNode(GT_COMMA, simdType, bitVecAssign, retVal);
1626
1627         // Prepend vecZero assignment to retVal.
1628         // retVal =  (tmp1 = vector.Zero), (tmp2 = v < tmp1), CondSelect(tmp2, tmp1 - v, v)
1629         retVal = gtNewOperNode(GT_COMMA, simdType, vecZeroAssign, retVal);
1630
1631         // If op1 was assigned to a temp, prepend that to retVal.
1632         if (op1Assign != nullptr)
1633         {
1634             // retVal = (v=op1), (tmp1 = vector.Zero), (tmp2 = v < tmp1), CondSelect(tmp2, tmp1 - v, v)
1635             retVal = gtNewOperNode(GT_COMMA, simdType, op1Assign, retVal);
1636         }
1637     }
1638     else if (varTypeIsFloating(baseType))
1639     {
1640         // Abs(vf) = vf & new SIMDVector<float>(0x7fffffff);
1641         // Abs(vd) = vf & new SIMDVector<double>(0x7fffffffffffffff);
1642         GenTree* bitMask = nullptr;
1643         if (baseType == TYP_FLOAT)
1644         {
1645             float f;
1646             static_assert_no_msg(sizeof(float) == sizeof(int));
1647             *((int*)&f) = 0x7fffffff;
1648             bitMask     = gtNewDconNode(f);
1649         }
1650         else if (baseType == TYP_DOUBLE)
1651         {
1652             double d;
1653             static_assert_no_msg(sizeof(double) == sizeof(__int64));
1654             *((__int64*)&d) = 0x7fffffffffffffffLL;
1655             bitMask         = gtNewDconNode(d);
1656         }
1657
1658         assert(bitMask != nullptr);
1659         bitMask->gtType        = baseType;
1660         GenTree* bitMaskVector = gtNewSIMDNode(simdType, bitMask, SIMDIntrinsicInit, baseType, size);
1661         retVal                 = gtNewSIMDNode(simdType, op1, bitMaskVector, SIMDIntrinsicBitwiseAnd, baseType, size);
1662     }
1663     else if (baseType == TYP_USHORT || baseType == TYP_UBYTE || baseType == TYP_UINT || baseType == TYP_ULONG)
1664     {
1665         // Abs is a no-op on unsigned integer type vectors
1666         retVal = op1;
1667     }
1668     else
1669     {
1670         assert(getSIMDSupportLevel() >= SIMD_SSE4_Supported);
1671         assert(baseType != TYP_LONG);
1672
1673         retVal = gtNewSIMDNode(simdType, op1, SIMDIntrinsicAbs, baseType, size);
1674     }
1675 #elif defined(_TARGET_ARM64_)
1676     if (varTypeIsUnsigned(baseType))
1677     {
1678         // Abs is a no-op on unsigned integer type vectors
1679         retVal = op1;
1680     }
1681     else
1682     {
1683         retVal = gtNewSIMDNode(simdType, op1, SIMDIntrinsicAbs, baseType, size);
1684     }
1685 #else  // !defined(_TARGET_XARCH)_ && !defined(_TARGET_ARM64_)
1686     assert(!"Abs intrinsic on non-xarch target not implemented");
1687 #endif // !_TARGET_XARCH_
1688
1689     return retVal;
1690 }
1691
1692 // Creates a GT_SIMD tree for Select operation
1693 //
1694 // Argumens:
1695 //    typeHnd          -  type handle of SIMD vector
1696 //    baseType         -  base type of SIMD vector
1697 //    size             -  SIMD vector size
1698 //    op1              -  first operand = Condition vector vc
1699 //    op2              -  second operand = va
1700 //    op3              -  third operand = vb
1701 //
1702 // Return Value:
1703 //    Returns GT_SIMD tree that computes Select(vc, va, vb)
1704 //
1705 GenTree* Compiler::impSIMDSelect(
1706     CORINFO_CLASS_HANDLE typeHnd, var_types baseType, unsigned size, GenTree* op1, GenTree* op2, GenTree* op3)
1707 {
1708     assert(varTypeIsSIMD(op1));
1709     var_types simdType = op1->TypeGet();
1710     assert(op2->TypeGet() == simdType);
1711     assert(op3->TypeGet() == simdType);
1712
1713     // TODO-ARM64-CQ Support generating select instruction for SIMD
1714
1715     // Select(BitVector vc, va, vb) = (va & vc) | (vb & !vc)
1716     // Select(op1, op2, op3)        = (op2 & op1) | (op3 & !op1)
1717     //                              = SIMDIntrinsicBitwiseOr(SIMDIntrinsicBitwiseAnd(op2, op1),
1718     //                                                       SIMDIntrinsicBitwiseAndNot(op3, op1))
1719     //
1720     // If Op1 has side effect, create an assignment to a temp
1721     GenTree* tmp = op1;
1722     GenTree* asg = nullptr;
1723     if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0)
1724     {
1725         unsigned lclNum = lvaGrabTemp(true DEBUGARG("SIMD Select"));
1726         lvaSetStruct(lclNum, typeHnd, false);
1727         tmp = gtNewLclvNode(lclNum, op1->TypeGet());
1728         asg = gtNewTempAssign(lclNum, op1);
1729     }
1730
1731     GenTree* andExpr = gtNewSIMDNode(simdType, op2, tmp, SIMDIntrinsicBitwiseAnd, baseType, size);
1732     GenTree* dupOp1  = gtCloneExpr(tmp);
1733     assert(dupOp1 != nullptr);
1734 #ifdef _TARGET_ARM64_
1735     // ARM64 implements SIMDIntrinsicBitwiseAndNot as Left & ~Right
1736     GenTree* andNotExpr = gtNewSIMDNode(simdType, op3, dupOp1, SIMDIntrinsicBitwiseAndNot, baseType, size);
1737 #else
1738     // XARCH implements SIMDIntrinsicBitwiseAndNot as ~Left & Right
1739     GenTree* andNotExpr = gtNewSIMDNode(simdType, dupOp1, op3, SIMDIntrinsicBitwiseAndNot, baseType, size);
1740 #endif
1741     GenTree* simdTree = gtNewSIMDNode(simdType, andExpr, andNotExpr, SIMDIntrinsicBitwiseOr, baseType, size);
1742
1743     // If asg not null, create a GT_COMMA tree.
1744     if (asg != nullptr)
1745     {
1746         simdTree = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), asg, simdTree);
1747     }
1748
1749     return simdTree;
1750 }
1751
1752 // Creates a GT_SIMD tree for Min/Max operation
1753 //
1754 // Argumens:
1755 //    IntrinsicId      -  SIMD intrinsic Id, either Min or Max
1756 //    typeHnd          -  type handle of SIMD vector
1757 //    baseType         -  base type of SIMD vector
1758 //    size             -  SIMD vector size
1759 //    op1              -  first operand = va
1760 //    op2              -  second operand = vb
1761 //
1762 // Return Value:
1763 //    Returns GT_SIMD tree that computes Max(va, vb)
1764 //
1765 GenTree* Compiler::impSIMDMinMax(SIMDIntrinsicID      intrinsicId,
1766                                  CORINFO_CLASS_HANDLE typeHnd,
1767                                  var_types            baseType,
1768                                  unsigned             size,
1769                                  GenTree*             op1,
1770                                  GenTree*             op2)
1771 {
1772     assert(intrinsicId == SIMDIntrinsicMin || intrinsicId == SIMDIntrinsicMax);
1773     assert(varTypeIsSIMD(op1));
1774     var_types simdType = op1->TypeGet();
1775     assert(op2->TypeGet() == simdType);
1776
1777 #if defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
1778     GenTree* simdTree = nullptr;
1779
1780 #ifdef _TARGET_XARCH_
1781     // SSE2 has direct support for float/double/signed word/unsigned byte.
1782     // SSE4.1 has direct support for int32/uint32/signed byte/unsigned word.
1783     // For other integer types we compute min/max as follows
1784     //
1785     // int32/uint32 (SSE2)
1786     // int64/uint64 (SSE2&SSE4):
1787     //       compResult        = (op1 < op2) in case of Min
1788     //                           (op1 > op2) in case of Max
1789     //       Min/Max(op1, op2) = Select(compResult, op1, op2)
1790     //
1791     // unsigned word (SSE2):
1792     //        op1 = op1 - 2^15  ; to make it fit within a signed word
1793     //        op2 = op2 - 2^15  ; to make it fit within a signed word
1794     //        result = SSE2 signed word Min/Max(op1, op2)
1795     //        result = result + 2^15  ; readjust it back
1796     //
1797     // signed byte (SSE2):
1798     //        op1 = op1 + 2^7  ; to make it unsigned
1799     //        op1 = op1 + 2^7  ; to make it unsigned
1800     //        result = SSE2 unsigned byte Min/Max(op1, op2)
1801     //        result = result - 2^15 ; readjust it back
1802
1803     if (varTypeIsFloating(baseType) || baseType == TYP_SHORT || baseType == TYP_UBYTE ||
1804         (getSIMDSupportLevel() >= SIMD_SSE4_Supported &&
1805          (baseType == TYP_BYTE || baseType == TYP_INT || baseType == TYP_UINT || baseType == TYP_USHORT)))
1806     {
1807         // SSE2 or SSE4.1 has direct support
1808         simdTree = gtNewSIMDNode(simdType, op1, op2, intrinsicId, baseType, size);
1809     }
1810     else if (baseType == TYP_USHORT || baseType == TYP_BYTE)
1811     {
1812         assert(getSIMDSupportLevel() == SIMD_SSE2_Supported);
1813         int             constVal;
1814         SIMDIntrinsicID operIntrinsic;
1815         SIMDIntrinsicID adjustIntrinsic;
1816         var_types       minMaxOperBaseType;
1817         if (baseType == TYP_USHORT)
1818         {
1819             constVal           = 0x80008000;
1820             operIntrinsic      = SIMDIntrinsicSub;
1821             adjustIntrinsic    = SIMDIntrinsicAdd;
1822             minMaxOperBaseType = TYP_SHORT;
1823         }
1824         else
1825         {
1826             assert(baseType == TYP_BYTE);
1827             constVal           = 0x80808080;
1828             operIntrinsic      = SIMDIntrinsicAdd;
1829             adjustIntrinsic    = SIMDIntrinsicSub;
1830             minMaxOperBaseType = TYP_UBYTE;
1831         }
1832
1833         GenTree* initVal     = gtNewIconNode(constVal);
1834         GenTree* constVector = gtNewSIMDNode(simdType, initVal, nullptr, SIMDIntrinsicInit, TYP_INT, size);
1835
1836         // Assign constVector to a temp, since we intend to use it more than once
1837         // TODO-CQ: We have quite a few such constant vectors constructed during
1838         // the importation of SIMD intrinsics.  Make sure that we have a single
1839         // temp per distinct constant per method.
1840         GenTree* tmp = fgInsertCommaFormTemp(&constVector, typeHnd);
1841
1842         // op1 = op1 - constVector
1843         // op2 = op2 - constVector
1844         op1 = gtNewSIMDNode(simdType, op1, constVector, operIntrinsic, baseType, size);
1845         op2 = gtNewSIMDNode(simdType, op2, tmp, operIntrinsic, baseType, size);
1846
1847         // compute min/max of op1 and op2 considering them as if minMaxOperBaseType
1848         simdTree = gtNewSIMDNode(simdType, op1, op2, intrinsicId, minMaxOperBaseType, size);
1849
1850         // re-adjust the value by adding or subtracting constVector
1851         tmp      = gtNewLclvNode(tmp->AsLclVarCommon()->GetLclNum(), tmp->TypeGet());
1852         simdTree = gtNewSIMDNode(simdType, simdTree, tmp, adjustIntrinsic, baseType, size);
1853     }
1854 #elif defined(_TARGET_ARM64_)
1855     // Arm64 has direct support for all types except int64/uint64
1856     // For which we compute min/max as follows
1857     //
1858     // int64/uint64
1859     //       compResult        = (op1 < op2) in case of Min
1860     //                           (op1 > op2) in case of Max
1861     //       Min/Max(op1, op2) = Select(compResult, op1, op2)
1862     if (baseType != TYP_ULONG && baseType != TYP_LONG)
1863     {
1864         simdTree = gtNewSIMDNode(simdType, op1, op2, intrinsicId, baseType, size);
1865     }
1866 #endif
1867     else
1868     {
1869         GenTree* dupOp1    = nullptr;
1870         GenTree* dupOp2    = nullptr;
1871         GenTree* op1Assign = nullptr;
1872         GenTree* op2Assign = nullptr;
1873         unsigned op1LclNum;
1874         unsigned op2LclNum;
1875
1876         if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0)
1877         {
1878             op1LclNum = lvaGrabTemp(true DEBUGARG("SIMD Min/Max"));
1879             dupOp1    = gtNewLclvNode(op1LclNum, op1->TypeGet());
1880             lvaSetStruct(op1LclNum, typeHnd, false);
1881             op1Assign = gtNewTempAssign(op1LclNum, op1);
1882             op1       = gtNewLclvNode(op1LclNum, op1->TypeGet());
1883         }
1884         else
1885         {
1886             dupOp1 = gtCloneExpr(op1);
1887         }
1888
1889         if ((op2->gtFlags & GTF_SIDE_EFFECT) != 0)
1890         {
1891             op2LclNum = lvaGrabTemp(true DEBUGARG("SIMD Min/Max"));
1892             dupOp2    = gtNewLclvNode(op2LclNum, op2->TypeGet());
1893             lvaSetStruct(op2LclNum, typeHnd, false);
1894             op2Assign = gtNewTempAssign(op2LclNum, op2);
1895             op2       = gtNewLclvNode(op2LclNum, op2->TypeGet());
1896         }
1897         else
1898         {
1899             dupOp2 = gtCloneExpr(op2);
1900         }
1901
1902         SIMDIntrinsicID relOpIntrinsic =
1903             (intrinsicId == SIMDIntrinsicMin) ? SIMDIntrinsicLessThan : SIMDIntrinsicGreaterThan;
1904         var_types relOpBaseType = baseType;
1905
1906         // compResult = op1 relOp op2
1907         // simdTree = Select(compResult, op1, op2);
1908         assert(dupOp1 != nullptr);
1909         assert(dupOp2 != nullptr);
1910         relOpIntrinsic            = impSIMDRelOp(relOpIntrinsic, typeHnd, size, &relOpBaseType, &dupOp1, &dupOp2);
1911         GenTree* compResult       = gtNewSIMDNode(simdType, dupOp1, dupOp2, relOpIntrinsic, relOpBaseType, size);
1912         unsigned compResultLclNum = lvaGrabTemp(true DEBUGARG("SIMD Min/Max"));
1913         lvaSetStruct(compResultLclNum, typeHnd, false);
1914         GenTree* compResultAssign = gtNewTempAssign(compResultLclNum, compResult);
1915         compResult                = gtNewLclvNode(compResultLclNum, compResult->TypeGet());
1916         simdTree                  = impSIMDSelect(typeHnd, baseType, size, compResult, op1, op2);
1917         simdTree                  = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), compResultAssign, simdTree);
1918
1919         // Now create comma trees if we have created assignments of op1/op2 to temps
1920         if (op2Assign != nullptr)
1921         {
1922             simdTree = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), op2Assign, simdTree);
1923         }
1924
1925         if (op1Assign != nullptr)
1926         {
1927             simdTree = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), op1Assign, simdTree);
1928         }
1929     }
1930
1931     assert(simdTree != nullptr);
1932     return simdTree;
1933 #else  // !(defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_))
1934     assert(!"impSIMDMinMax() unimplemented on target arch");
1935     unreached();
1936 #endif // !(defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_))
1937 }
1938
1939 //------------------------------------------------------------------------
1940 // getOp1ForConstructor: Get the op1 for a constructor call.
1941 //
1942 // Arguments:
1943 //    opcode     - the opcode being handled (needed to identify the CEE_NEWOBJ case)
1944 //    newobjThis - For CEE_NEWOBJ, this is the temp grabbed for the allocated uninitalized object.
1945 //    clsHnd    - The handle of the class of the method.
1946 //
1947 // Return Value:
1948 //    The tree node representing the object to be initialized with the constructor.
1949 //
1950 // Notes:
1951 //    This method handles the differences between the CEE_NEWOBJ and constructor cases.
1952 //
1953 GenTree* Compiler::getOp1ForConstructor(OPCODE opcode, GenTree* newobjThis, CORINFO_CLASS_HANDLE clsHnd)
1954 {
1955     GenTree* op1;
1956     if (opcode == CEE_NEWOBJ)
1957     {
1958         op1 = newobjThis;
1959         assert(newobjThis->gtOper == GT_ADDR && newobjThis->gtOp.gtOp1->gtOper == GT_LCL_VAR);
1960
1961         // push newobj result on type stack
1962         unsigned tmp = op1->gtOp.gtOp1->gtLclVarCommon.gtLclNum;
1963         impPushOnStack(gtNewLclvNode(tmp, lvaGetRealType(tmp)), verMakeTypeInfo(clsHnd).NormaliseForStack());
1964     }
1965     else
1966     {
1967         op1 = impSIMDPopStack(TYP_BYREF);
1968     }
1969     assert(op1->TypeGet() == TYP_BYREF);
1970     return op1;
1971 }
1972
1973 //-------------------------------------------------------------------
1974 // Set the flag that indicates that the lclVar referenced by this tree
1975 // is used in a SIMD intrinsic.
1976 // Arguments:
1977 //      tree - GenTree*
1978
1979 void Compiler::setLclRelatedToSIMDIntrinsic(GenTree* tree)
1980 {
1981     assert(tree->OperIsLocal());
1982     unsigned   lclNum                = tree->AsLclVarCommon()->GetLclNum();
1983     LclVarDsc* lclVarDsc             = &lvaTable[lclNum];
1984     lclVarDsc->lvUsedInSIMDIntrinsic = true;
1985 }
1986
1987 //-------------------------------------------------------------
1988 // Check if two field nodes reference at the same memory location.
1989 // Notice that this check is just based on pattern matching.
1990 // Arguments:
1991 //      op1 - GenTree*.
1992 //      op2 - GenTree*.
1993 // Return Value:
1994 //    If op1's parents node and op2's parents node are at the same location, return true. Otherwise, return false
1995
1996 bool areFieldsParentsLocatedSame(GenTree* op1, GenTree* op2)
1997 {
1998     assert(op1->OperGet() == GT_FIELD);
1999     assert(op2->OperGet() == GT_FIELD);
2000
2001     GenTree* op1ObjRef = op1->gtField.gtFldObj;
2002     GenTree* op2ObjRef = op2->gtField.gtFldObj;
2003     while (op1ObjRef != nullptr && op2ObjRef != nullptr)
2004     {
2005
2006         if (op1ObjRef->OperGet() != op2ObjRef->OperGet())
2007         {
2008             break;
2009         }
2010         else if (op1ObjRef->OperGet() == GT_ADDR)
2011         {
2012             op1ObjRef = op1ObjRef->gtOp.gtOp1;
2013             op2ObjRef = op2ObjRef->gtOp.gtOp1;
2014         }
2015
2016         if (op1ObjRef->OperIsLocal() && op2ObjRef->OperIsLocal() &&
2017             op1ObjRef->AsLclVarCommon()->GetLclNum() == op2ObjRef->AsLclVarCommon()->GetLclNum())
2018         {
2019             return true;
2020         }
2021         else if (op1ObjRef->OperGet() == GT_FIELD && op2ObjRef->OperGet() == GT_FIELD &&
2022                  op1ObjRef->gtField.gtFldHnd == op2ObjRef->gtField.gtFldHnd)
2023         {
2024             op1ObjRef = op1ObjRef->gtField.gtFldObj;
2025             op2ObjRef = op2ObjRef->gtField.gtFldObj;
2026             continue;
2027         }
2028         else
2029         {
2030             break;
2031         }
2032     }
2033
2034     return false;
2035 }
2036
2037 //----------------------------------------------------------------------
2038 // Check whether two field are contiguous
2039 // Arguments:
2040 //      first - GenTree*. The Type of the node should be TYP_FLOAT
2041 //      second - GenTree*. The Type of the node should be TYP_FLOAT
2042 // Return Value:
2043 //      if the first field is located before second field, and they are located contiguously,
2044 //      then return true. Otherwise, return false.
2045
2046 bool Compiler::areFieldsContiguous(GenTree* first, GenTree* second)
2047 {
2048     assert(first->OperGet() == GT_FIELD);
2049     assert(second->OperGet() == GT_FIELD);
2050     assert(first->gtType == TYP_FLOAT);
2051     assert(second->gtType == TYP_FLOAT);
2052
2053     var_types firstFieldType  = first->gtType;
2054     var_types secondFieldType = second->gtType;
2055
2056     unsigned firstFieldEndOffset = first->gtField.gtFldOffset + genTypeSize(firstFieldType);
2057     unsigned secondFieldOffset   = second->gtField.gtFldOffset;
2058     if (firstFieldEndOffset == secondFieldOffset && firstFieldType == secondFieldType &&
2059         areFieldsParentsLocatedSame(first, second))
2060     {
2061         return true;
2062     }
2063
2064     return false;
2065 }
2066
2067 //-------------------------------------------------------------------------------
2068 // Check whether two array element nodes are located contiguously or not.
2069 // Arguments:
2070 //      op1 - GenTree*.
2071 //      op2 - GenTree*.
2072 // Return Value:
2073 //      if the array element op1 is located before array element op2, and they are contiguous,
2074 //      then return true. Otherwise, return false.
2075 // TODO-CQ:
2076 //      Right this can only check array element with const number as index. In future,
2077 //      we should consider to allow this function to check the index using expression.
2078
2079 bool Compiler::areArrayElementsContiguous(GenTree* op1, GenTree* op2)
2080 {
2081     noway_assert(op1->gtOper == GT_INDEX);
2082     noway_assert(op2->gtOper == GT_INDEX);
2083     GenTreeIndex* op1Index = op1->AsIndex();
2084     GenTreeIndex* op2Index = op2->AsIndex();
2085
2086     GenTree* op1ArrayRef = op1Index->Arr();
2087     GenTree* op2ArrayRef = op2Index->Arr();
2088     assert(op1ArrayRef->TypeGet() == TYP_REF);
2089     assert(op2ArrayRef->TypeGet() == TYP_REF);
2090
2091     GenTree* op1IndexNode = op1Index->Index();
2092     GenTree* op2IndexNode = op2Index->Index();
2093     if ((op1IndexNode->OperGet() == GT_CNS_INT && op2IndexNode->OperGet() == GT_CNS_INT) &&
2094         op1IndexNode->gtIntCon.gtIconVal + 1 == op2IndexNode->gtIntCon.gtIconVal)
2095     {
2096         if (op1ArrayRef->OperGet() == GT_FIELD && op2ArrayRef->OperGet() == GT_FIELD &&
2097             areFieldsParentsLocatedSame(op1ArrayRef, op2ArrayRef))
2098         {
2099             return true;
2100         }
2101         else if (op1ArrayRef->OperIsLocal() && op2ArrayRef->OperIsLocal() &&
2102                  op1ArrayRef->AsLclVarCommon()->GetLclNum() == op2ArrayRef->AsLclVarCommon()->GetLclNum())
2103         {
2104             return true;
2105         }
2106     }
2107     return false;
2108 }
2109
2110 //-------------------------------------------------------------------------------
2111 // Check whether two argument nodes are contiguous or not.
2112 // Arguments:
2113 //      op1 - GenTree*.
2114 //      op2 - GenTree*.
2115 // Return Value:
2116 //      if the argument node op1 is located before argument node op2, and they are located contiguously,
2117 //      then return true. Otherwise, return false.
2118 // TODO-CQ:
2119 //      Right now this can only check field and array. In future we should add more cases.
2120 //
2121
2122 bool Compiler::areArgumentsContiguous(GenTree* op1, GenTree* op2)
2123 {
2124     if (op1->OperGet() == GT_INDEX && op2->OperGet() == GT_INDEX)
2125     {
2126         return areArrayElementsContiguous(op1, op2);
2127     }
2128     else if (op1->OperGet() == GT_FIELD && op2->OperGet() == GT_FIELD)
2129     {
2130         return areFieldsContiguous(op1, op2);
2131     }
2132     return false;
2133 }
2134
2135 //--------------------------------------------------------------------------------------------------------
2136 // createAddressNodeForSIMDInit: Generate the address node(GT_LEA) if we want to intialize vector2, vector3 or vector4
2137 // from first argument's address.
2138 //
2139 // Arguments:
2140 //      tree - GenTree*. This the tree node which is used to get the address for indir.
2141 //      simdsize - unsigned. This the simd vector size.
2142 //      arrayElementsCount - unsigned. This is used for generating the boundary check for array.
2143 //
2144 // Return value:
2145 //      return the address node.
2146 //
2147 // TODO-CQ:
2148 //      1. Currently just support for GT_FIELD and GT_INDEX, because we can only verify the GT_INDEX node or GT_Field
2149 //         are located contiguously or not. In future we should support more cases.
2150 //      2. Though it happens to just work fine front-end phases are not aware of GT_LEA node.  Therefore, convert these
2151 //         to use GT_ADDR.
2152 GenTree* Compiler::createAddressNodeForSIMDInit(GenTree* tree, unsigned simdSize)
2153 {
2154     assert(tree->OperGet() == GT_FIELD || tree->OperGet() == GT_INDEX);
2155     GenTree*  byrefNode  = nullptr;
2156     GenTree*  startIndex = nullptr;
2157     unsigned  offset     = 0;
2158     var_types baseType   = tree->gtType;
2159
2160     if (tree->OperGet() == GT_FIELD)
2161     {
2162         GenTree* objRef = tree->gtField.gtFldObj;
2163         if (objRef != nullptr && objRef->gtOper == GT_ADDR)
2164         {
2165             GenTree* obj = objRef->gtOp.gtOp1;
2166
2167             // If the field is directly from a struct, then in this case,
2168             // we should set this struct's lvUsedInSIMDIntrinsic as true,
2169             // so that this sturct won't be promoted.
2170             // e.g. s.x x is a field, and s is a struct, then we should set the s's lvUsedInSIMDIntrinsic as true.
2171             // so that s won't be promoted.
2172             // Notice that if we have a case like s1.s2.x. s1 s2 are struct, and x is a field, then it is possible that
2173             // s1 can be promoted, so that s2 can be promoted. The reason for that is if we don't allow s1 to be
2174             // promoted, then this will affect the other optimizations which are depend on s1's struct promotion.
2175             // TODO-CQ:
2176             //  In future, we should optimize this case so that if there is a nested field like s1.s2.x and s1.s2.x's
2177             //  address is used for initializing the vector, then s1 can be promoted but s2 can't.
2178             if (varTypeIsSIMD(obj) && obj->OperIsLocal())
2179             {
2180                 setLclRelatedToSIMDIntrinsic(obj);
2181             }
2182         }
2183
2184         byrefNode = gtCloneExpr(tree->gtField.gtFldObj);
2185         assert(byrefNode != nullptr);
2186         offset = tree->gtField.gtFldOffset;
2187     }
2188     else if (tree->OperGet() == GT_INDEX)
2189     {
2190
2191         GenTree* index = tree->AsIndex()->Index();
2192         assert(index->OperGet() == GT_CNS_INT);
2193
2194         GenTree* checkIndexExpr = nullptr;
2195         unsigned indexVal       = (unsigned)(index->gtIntCon.gtIconVal);
2196         offset                  = indexVal * genTypeSize(tree->TypeGet());
2197         GenTree* arrayRef       = tree->AsIndex()->Arr();
2198
2199         // Generate the boundary check exception.
2200         // The length for boundary check should be the maximum index number which should be
2201         // (first argument's index number) + (how many array arguments we have) - 1
2202         // = indexVal + arrayElementsCount - 1
2203         unsigned arrayElementsCount  = simdSize / genTypeSize(baseType);
2204         checkIndexExpr               = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, indexVal + arrayElementsCount - 1);
2205         GenTreeArrLen*    arrLen     = gtNewArrLen(TYP_INT, arrayRef, (int)offsetof(CORINFO_Array, length));
2206         GenTreeBoundsChk* arrBndsChk = new (this, GT_ARR_BOUNDS_CHECK)
2207             GenTreeBoundsChk(GT_ARR_BOUNDS_CHECK, TYP_VOID, checkIndexExpr, arrLen, SCK_RNGCHK_FAIL);
2208
2209         offset += offsetof(CORINFO_Array, u1Elems);
2210         byrefNode = gtNewOperNode(GT_COMMA, arrayRef->TypeGet(), arrBndsChk, gtCloneExpr(arrayRef));
2211     }
2212     else
2213     {
2214         unreached();
2215     }
2216     GenTree* address =
2217         new (this, GT_LEA) GenTreeAddrMode(TYP_BYREF, byrefNode, startIndex, genTypeSize(tree->TypeGet()), offset);
2218     return address;
2219 }
2220
2221 //-------------------------------------------------------------------------------
2222 // impMarkContiguousSIMDFieldAssignments: Try to identify if there are contiguous
2223 // assignments from SIMD field to memory. If there are, then mark the related
2224 // lclvar so that it won't be promoted.
2225 //
2226 // Arguments:
2227 //      stmt - GenTree*. Input statement node.
2228
2229 void Compiler::impMarkContiguousSIMDFieldAssignments(GenTree* stmt)
2230 {
2231     if (!featureSIMD || opts.MinOpts())
2232     {
2233         return;
2234     }
2235     GenTree* expr = stmt->gtStmt.gtStmtExpr;
2236     if (expr->OperGet() == GT_ASG && expr->TypeGet() == TYP_FLOAT)
2237     {
2238         GenTree*  curDst            = expr->gtOp.gtOp1;
2239         GenTree*  curSrc            = expr->gtOp.gtOp2;
2240         unsigned  index             = 0;
2241         var_types baseType          = TYP_UNKNOWN;
2242         unsigned  simdSize          = 0;
2243         GenTree*  srcSimdStructNode = getSIMDStructFromField(curSrc, &baseType, &index, &simdSize, true);
2244         if (srcSimdStructNode == nullptr || baseType != TYP_FLOAT)
2245         {
2246             fgPreviousCandidateSIMDFieldAsgStmt = nullptr;
2247         }
2248         else if (index == 0 && isSIMDTypeLocal(srcSimdStructNode))
2249         {
2250             fgPreviousCandidateSIMDFieldAsgStmt = stmt;
2251         }
2252         else if (fgPreviousCandidateSIMDFieldAsgStmt != nullptr)
2253         {
2254             assert(index > 0);
2255             GenTree* prevAsgExpr = fgPreviousCandidateSIMDFieldAsgStmt->gtStmt.gtStmtExpr;
2256             GenTree* prevDst     = prevAsgExpr->gtOp.gtOp1;
2257             GenTree* prevSrc     = prevAsgExpr->gtOp.gtOp2;
2258             if (!areArgumentsContiguous(prevDst, curDst) || !areArgumentsContiguous(prevSrc, curSrc))
2259             {
2260                 fgPreviousCandidateSIMDFieldAsgStmt = nullptr;
2261             }
2262             else
2263             {
2264                 if (index == (simdSize / genTypeSize(baseType) - 1))
2265                 {
2266                     // Successfully found the pattern, mark the lclvar as UsedInSIMDIntrinsic
2267                     if (srcSimdStructNode->OperIsLocal())
2268                     {
2269                         setLclRelatedToSIMDIntrinsic(srcSimdStructNode);
2270                     }
2271
2272                     if (curDst->OperGet() == GT_FIELD)
2273                     {
2274                         GenTree* objRef = curDst->gtField.gtFldObj;
2275                         if (objRef != nullptr && objRef->gtOper == GT_ADDR)
2276                         {
2277                             GenTree* obj = objRef->gtOp.gtOp1;
2278                             if (varTypeIsStruct(obj) && obj->OperIsLocal())
2279                             {
2280                                 setLclRelatedToSIMDIntrinsic(obj);
2281                             }
2282                         }
2283                     }
2284                 }
2285                 else
2286                 {
2287                     fgPreviousCandidateSIMDFieldAsgStmt = stmt;
2288                 }
2289             }
2290         }
2291     }
2292     else
2293     {
2294         fgPreviousCandidateSIMDFieldAsgStmt = nullptr;
2295     }
2296 }
2297
2298 //------------------------------------------------------------------------
2299 // impSIMDIntrinsic: Check method to see if it is a SIMD method
2300 //
2301 // Arguments:
2302 //    opcode     - the opcode being handled (needed to identify the CEE_NEWOBJ case)
2303 //    newobjThis - For CEE_NEWOBJ, this is the temp grabbed for the allocated uninitalized object.
2304 //    clsHnd     - The handle of the class of the method.
2305 //    method     - The handle of the method.
2306 //    sig        - The call signature for the method.
2307 //    memberRef  - The memberRef token for the method reference.
2308 //
2309 // Return Value:
2310 //    If clsHnd is a known SIMD type, and 'method' is one of the methods that are
2311 //    implemented as an intrinsic in the JIT, then return the tree that implements
2312 //    it.
2313 //
2314 GenTree* Compiler::impSIMDIntrinsic(OPCODE                opcode,
2315                                     GenTree*              newobjThis,
2316                                     CORINFO_CLASS_HANDLE  clsHnd,
2317                                     CORINFO_METHOD_HANDLE methodHnd,
2318                                     CORINFO_SIG_INFO*     sig,
2319                                     int                   memberRef)
2320 {
2321     assert(featureSIMD);
2322
2323     if (!isSIMDClass(clsHnd))
2324     {
2325         return nullptr;
2326     }
2327
2328     // Get base type and intrinsic Id
2329     var_types                baseType = TYP_UNKNOWN;
2330     unsigned                 size     = 0;
2331     unsigned                 argCount = 0;
2332     const SIMDIntrinsicInfo* intrinsicInfo =
2333         getSIMDIntrinsicInfo(&clsHnd, methodHnd, sig, (opcode == CEE_NEWOBJ), &argCount, &baseType, &size);
2334     if (intrinsicInfo == nullptr || intrinsicInfo->id == SIMDIntrinsicInvalid)
2335     {
2336         return nullptr;
2337     }
2338
2339     SIMDIntrinsicID simdIntrinsicID = intrinsicInfo->id;
2340     var_types       simdType;
2341     if (baseType != TYP_UNKNOWN)
2342     {
2343         simdType = getSIMDTypeForSize(size);
2344     }
2345     else
2346     {
2347         assert(simdIntrinsicID == SIMDIntrinsicHWAccel);
2348         simdType = TYP_UNKNOWN;
2349     }
2350     bool      instMethod = intrinsicInfo->isInstMethod;
2351     var_types callType   = JITtype2varType(sig->retType);
2352     if (callType == TYP_STRUCT)
2353     {
2354         // Note that here we are assuming that, if the call returns a struct, that it is the same size as the
2355         // struct on which the method is declared. This is currently true for all methods on Vector types,
2356         // but if this ever changes, we will need to determine the callType from the signature.
2357         assert(info.compCompHnd->getClassSize(sig->retTypeClass) == genTypeSize(simdType));
2358         callType = simdType;
2359     }
2360
2361     GenTree* simdTree   = nullptr;
2362     GenTree* op1        = nullptr;
2363     GenTree* op2        = nullptr;
2364     GenTree* op3        = nullptr;
2365     GenTree* retVal     = nullptr;
2366     GenTree* copyBlkDst = nullptr;
2367     bool     doCopyBlk  = false;
2368
2369     switch (simdIntrinsicID)
2370     {
2371         case SIMDIntrinsicGetCount:
2372         {
2373             int            length       = getSIMDVectorLength(clsHnd);
2374             GenTreeIntCon* intConstTree = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, length);
2375             retVal                      = intConstTree;
2376
2377             intConstTree->gtFlags |= GTF_ICON_SIMD_COUNT;
2378         }
2379         break;
2380
2381         case SIMDIntrinsicGetZero:
2382             retVal = gtNewSIMDVectorZero(simdType, baseType, size);
2383             break;
2384
2385         case SIMDIntrinsicGetOne:
2386             retVal = gtNewSIMDVectorOne(simdType, baseType, size);
2387             break;
2388
2389         case SIMDIntrinsicGetAllOnes:
2390         {
2391             // Equivalent to (Vector<T>) new Vector<int>(0xffffffff);
2392             GenTree* initVal = gtNewIconNode(0xffffffff, TYP_INT);
2393             simdTree         = gtNewSIMDNode(simdType, initVal, nullptr, SIMDIntrinsicInit, TYP_INT, size);
2394             if (baseType != TYP_INT)
2395             {
2396                 // cast it to required baseType if different from TYP_INT
2397                 simdTree = gtNewSIMDNode(simdType, simdTree, nullptr, SIMDIntrinsicCast, baseType, size);
2398             }
2399             retVal = simdTree;
2400         }
2401         break;
2402
2403         case SIMDIntrinsicInit:
2404         case SIMDIntrinsicInitN:
2405         {
2406             // SIMDIntrinsicInit:
2407             //    op2 - the initializer value
2408             //    op1 - byref of vector
2409             //
2410             // SIMDIntrinsicInitN
2411             //    op2 - list of initializer values stitched into a list
2412             //    op1 - byref of vector
2413             bool initFromFirstArgIndir = false;
2414             if (simdIntrinsicID == SIMDIntrinsicInit)
2415             {
2416                 op2 = impSIMDPopStack(baseType);
2417             }
2418             else
2419             {
2420                 assert(simdIntrinsicID == SIMDIntrinsicInitN);
2421                 assert(baseType == TYP_FLOAT);
2422
2423                 unsigned initCount    = argCount - 1;
2424                 unsigned elementCount = getSIMDVectorLength(size, baseType);
2425                 noway_assert(initCount == elementCount);
2426                 GenTree* nextArg = op2;
2427
2428                 // Build a GT_LIST with the N values.
2429                 // We must maintain left-to-right order of the args, but we will pop
2430                 // them off in reverse order (the Nth arg was pushed onto the stack last).
2431
2432                 GenTree* list              = nullptr;
2433                 GenTree* firstArg          = nullptr;
2434                 GenTree* prevArg           = nullptr;
2435                 int      offset            = 0;
2436                 bool     areArgsContiguous = true;
2437                 for (unsigned i = 0; i < initCount; i++)
2438                 {
2439                     GenTree* nextArg = impSIMDPopStack(baseType);
2440                     if (areArgsContiguous)
2441                     {
2442                         GenTree* curArg = nextArg;
2443                         firstArg        = curArg;
2444
2445                         if (prevArg != nullptr)
2446                         {
2447                             // Recall that we are popping the args off the stack in reverse order.
2448                             areArgsContiguous = areArgumentsContiguous(curArg, prevArg);
2449                         }
2450                         prevArg = curArg;
2451                     }
2452
2453                     list = new (this, GT_LIST) GenTreeOp(GT_LIST, baseType, nextArg, list);
2454                 }
2455
2456                 if (areArgsContiguous && baseType == TYP_FLOAT)
2457                 {
2458                     // Since Vector2, Vector3 and Vector4's arguments type are only float,
2459                     // we intialize the vector from first argument address, only when
2460                     // the baseType is TYP_FLOAT and the arguments are located contiguously in memory
2461                     initFromFirstArgIndir = true;
2462                     GenTree*  op2Address  = createAddressNodeForSIMDInit(firstArg, size);
2463                     var_types simdType    = getSIMDTypeForSize(size);
2464                     op2                   = gtNewOperNode(GT_IND, simdType, op2Address);
2465                 }
2466                 else
2467                 {
2468                     op2 = list;
2469                 }
2470             }
2471
2472             op1 = getOp1ForConstructor(opcode, newobjThis, clsHnd);
2473
2474             assert(op1->TypeGet() == TYP_BYREF);
2475             assert(genActualType(op2->TypeGet()) == genActualType(baseType) || initFromFirstArgIndir);
2476
2477             // For integral base types of size less than TYP_INT, expand the initializer
2478             // to fill size of TYP_INT bytes.
2479             if (varTypeIsSmallInt(baseType))
2480             {
2481                 // This case should occur only for Init intrinsic.
2482                 assert(simdIntrinsicID == SIMDIntrinsicInit);
2483
2484                 unsigned baseSize = genTypeSize(baseType);
2485                 int      multiplier;
2486                 if (baseSize == 1)
2487                 {
2488                     multiplier = 0x01010101;
2489                 }
2490                 else
2491                 {
2492                     assert(baseSize == 2);
2493                     multiplier = 0x00010001;
2494                 }
2495
2496                 GenTree* t1 = nullptr;
2497                 if (baseType == TYP_BYTE)
2498                 {
2499                     // What we have is a signed byte initializer,
2500                     // which when loaded to a reg will get sign extended to TYP_INT.
2501                     // But what we need is the initializer without sign extended or
2502                     // rather zero extended to 32-bits.
2503                     t1 = gtNewOperNode(GT_AND, TYP_INT, op2, gtNewIconNode(0xff, TYP_INT));
2504                 }
2505                 else if (baseType == TYP_SHORT)
2506                 {
2507                     // What we have is a signed short initializer,
2508                     // which when loaded to a reg will get sign extended to TYP_INT.
2509                     // But what we need is the initializer without sign extended or
2510                     // rather zero extended to 32-bits.
2511                     t1 = gtNewOperNode(GT_AND, TYP_INT, op2, gtNewIconNode(0xffff, TYP_INT));
2512                 }
2513                 else
2514                 {
2515                     assert(baseType == TYP_UBYTE || baseType == TYP_USHORT);
2516                     t1 = gtNewCastNode(TYP_INT, op2, false, TYP_INT);
2517                 }
2518
2519                 assert(t1 != nullptr);
2520                 GenTree* t2 = gtNewIconNode(multiplier, TYP_INT);
2521                 op2         = gtNewOperNode(GT_MUL, TYP_INT, t1, t2);
2522
2523                 // Construct a vector of TYP_INT with the new initializer and cast it back to vector of baseType
2524                 simdTree = gtNewSIMDNode(simdType, op2, nullptr, simdIntrinsicID, TYP_INT, size);
2525                 simdTree = gtNewSIMDNode(simdType, simdTree, nullptr, SIMDIntrinsicCast, baseType, size);
2526             }
2527             else
2528             {
2529
2530                 if (initFromFirstArgIndir)
2531                 {
2532                     simdTree = op2;
2533                     if (op1->gtOp.gtOp1->OperIsLocal())
2534                     {
2535                         // label the dst struct's lclvar is used for SIMD intrinsic,
2536                         // so that this dst struct won't be promoted.
2537                         setLclRelatedToSIMDIntrinsic(op1->gtOp.gtOp1);
2538                     }
2539                 }
2540                 else
2541                 {
2542                     simdTree = gtNewSIMDNode(simdType, op2, nullptr, simdIntrinsicID, baseType, size);
2543                 }
2544             }
2545
2546             copyBlkDst = op1;
2547             doCopyBlk  = true;
2548         }
2549         break;
2550
2551         case SIMDIntrinsicInitArray:
2552         case SIMDIntrinsicInitArrayX:
2553         case SIMDIntrinsicCopyToArray:
2554         case SIMDIntrinsicCopyToArrayX:
2555         {
2556             // op3 - index into array in case of SIMDIntrinsicCopyToArrayX and SIMDIntrinsicInitArrayX
2557             // op2 - array itself
2558             // op1 - byref to vector struct
2559
2560             unsigned int vectorLength = getSIMDVectorLength(size, baseType);
2561             // (This constructor takes only the zero-based arrays.)
2562             // We will add one or two bounds checks:
2563             // 1. If we have an index, we must do a check on that first.
2564             //    We can't combine it with the index + vectorLength check because
2565             //    a. It might be negative, and b. It may need to raise a different exception
2566             //    (captured as SCK_ARG_RNG_EXCPN for CopyTo and SCK_RNGCHK_FAIL for Init).
2567             // 2. We need to generate a check (SCK_ARG_EXCPN for CopyTo and SCK_RNGCHK_FAIL for Init)
2568             //    for the last array element we will access.
2569             //    We'll either check against (vectorLength - 1) or (index + vectorLength - 1).
2570
2571             GenTree* checkIndexExpr = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, vectorLength - 1);
2572
2573             // Get the index into the array.  If it has been provided, it will be on the
2574             // top of the stack.  Otherwise, it is null.
2575             if (argCount == 3)
2576             {
2577                 op3 = impSIMDPopStack(TYP_INT);
2578                 if (op3->IsIntegralConst(0))
2579                 {
2580                     op3 = nullptr;
2581                 }
2582             }
2583             else
2584             {
2585                 // TODO-CQ: Here, or elsewhere, check for the pattern where op2 is a newly constructed array, and
2586                 // change this to the InitN form.
2587                 // op3 = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, 0);
2588                 op3 = nullptr;
2589             }
2590
2591             // Clone the array for use in the bounds check.
2592             op2 = impSIMDPopStack(TYP_REF);
2593             assert(op2->TypeGet() == TYP_REF);
2594             GenTree* arrayRefForArgChk = op2;
2595             GenTree* argRngChk         = nullptr;
2596             GenTree* asg               = nullptr;
2597             if ((arrayRefForArgChk->gtFlags & GTF_SIDE_EFFECT) != 0)
2598             {
2599                 op2 = fgInsertCommaFormTemp(&arrayRefForArgChk);
2600             }
2601             else
2602             {
2603                 op2 = gtCloneExpr(arrayRefForArgChk);
2604             }
2605             assert(op2 != nullptr);
2606
2607             if (op3 != nullptr)
2608             {
2609                 SpecialCodeKind op3CheckKind;
2610                 if (simdIntrinsicID == SIMDIntrinsicInitArrayX)
2611                 {
2612                     op3CheckKind = SCK_RNGCHK_FAIL;
2613                 }
2614                 else
2615                 {
2616                     assert(simdIntrinsicID == SIMDIntrinsicCopyToArrayX);
2617                     op3CheckKind = SCK_ARG_RNG_EXCPN;
2618                 }
2619                 // We need to use the original expression on this, which is the first check.
2620                 GenTree* arrayRefForArgRngChk = arrayRefForArgChk;
2621                 // Then we clone the clone we just made for the next check.
2622                 arrayRefForArgChk = gtCloneExpr(op2);
2623                 // We know we MUST have had a cloneable expression.
2624                 assert(arrayRefForArgChk != nullptr);
2625                 GenTree* index = op3;
2626                 if ((index->gtFlags & GTF_SIDE_EFFECT) != 0)
2627                 {
2628                     op3 = fgInsertCommaFormTemp(&index);
2629                 }
2630                 else
2631                 {
2632                     op3 = gtCloneExpr(index);
2633                 }
2634
2635                 GenTreeArrLen* arrLen =
2636                     gtNewArrLen(TYP_INT, arrayRefForArgRngChk, (int)offsetof(CORINFO_Array, length));
2637                 argRngChk = new (this, GT_ARR_BOUNDS_CHECK)
2638                     GenTreeBoundsChk(GT_ARR_BOUNDS_CHECK, TYP_VOID, index, arrLen, op3CheckKind);
2639                 // Now, clone op3 to create another node for the argChk
2640                 GenTree* index2 = gtCloneExpr(op3);
2641                 assert(index != nullptr);
2642                 checkIndexExpr = gtNewOperNode(GT_ADD, TYP_INT, index2, checkIndexExpr);
2643             }
2644
2645             // Insert a bounds check for index + offset - 1.
2646             // This must be a "normal" array.
2647             SpecialCodeKind op2CheckKind;
2648             if (simdIntrinsicID == SIMDIntrinsicInitArray || simdIntrinsicID == SIMDIntrinsicInitArrayX)
2649             {
2650                 op2CheckKind = SCK_RNGCHK_FAIL;
2651             }
2652             else
2653             {
2654                 op2CheckKind = SCK_ARG_EXCPN;
2655             }
2656             GenTreeArrLen*    arrLen = gtNewArrLen(TYP_INT, arrayRefForArgChk, (int)offsetof(CORINFO_Array, length));
2657             GenTreeBoundsChk* argChk = new (this, GT_ARR_BOUNDS_CHECK)
2658                 GenTreeBoundsChk(GT_ARR_BOUNDS_CHECK, TYP_VOID, checkIndexExpr, arrLen, op2CheckKind);
2659
2660             // Create a GT_COMMA tree for the bounds check(s).
2661             op2 = gtNewOperNode(GT_COMMA, op2->TypeGet(), argChk, op2);
2662             if (argRngChk != nullptr)
2663             {
2664                 op2 = gtNewOperNode(GT_COMMA, op2->TypeGet(), argRngChk, op2);
2665             }
2666
2667             if (simdIntrinsicID == SIMDIntrinsicInitArray || simdIntrinsicID == SIMDIntrinsicInitArrayX)
2668             {
2669                 op1        = getOp1ForConstructor(opcode, newobjThis, clsHnd);
2670                 simdTree   = gtNewSIMDNode(simdType, op2, op3, SIMDIntrinsicInitArray, baseType, size);
2671                 copyBlkDst = op1;
2672                 doCopyBlk  = true;
2673             }
2674             else
2675             {
2676                 assert(simdIntrinsicID == SIMDIntrinsicCopyToArray || simdIntrinsicID == SIMDIntrinsicCopyToArrayX);
2677                 op1 = impSIMDPopStack(simdType, instMethod);
2678                 assert(op1->TypeGet() == simdType);
2679
2680                 // copy vector (op1) to array (op2) starting at index (op3)
2681                 simdTree = op1;
2682
2683                 // TODO-Cleanup: Though it happens to just work fine front-end phases are not aware of GT_LEA node.
2684                 // Therefore, convert these to use GT_ADDR .
2685                 copyBlkDst = new (this, GT_LEA)
2686                     GenTreeAddrMode(TYP_BYREF, op2, op3, genTypeSize(baseType), offsetof(CORINFO_Array, u1Elems));
2687                 doCopyBlk = true;
2688             }
2689         }
2690         break;
2691
2692         case SIMDIntrinsicInitFixed:
2693         {
2694             // We are initializing a fixed-length vector VLarge with a smaller fixed-length vector VSmall, plus 1 or 2
2695             // additional floats.
2696             //    op4 (optional) - float value for VLarge.W, if VLarge is Vector4, and VSmall is Vector2
2697             //    op3 - float value for VLarge.Z or VLarge.W
2698             //    op2 - VSmall
2699             //    op1 - byref of VLarge
2700             assert(baseType == TYP_FLOAT);
2701             unsigned elementByteCount = 4;
2702
2703             GenTree* op4 = nullptr;
2704             if (argCount == 4)
2705             {
2706                 op4 = impSIMDPopStack(TYP_FLOAT);
2707                 assert(op4->TypeGet() == TYP_FLOAT);
2708             }
2709             op3 = impSIMDPopStack(TYP_FLOAT);
2710             assert(op3->TypeGet() == TYP_FLOAT);
2711             // The input vector will either be TYP_SIMD8 or TYP_SIMD12.
2712             var_types smallSIMDType = TYP_SIMD8;
2713             if ((op4 == nullptr) && (simdType == TYP_SIMD16))
2714             {
2715                 smallSIMDType = TYP_SIMD12;
2716             }
2717             op2 = impSIMDPopStack(smallSIMDType);
2718             op1 = getOp1ForConstructor(opcode, newobjThis, clsHnd);
2719
2720             // We are going to redefine the operands so that:
2721             // - op3 is the value that's going into the Z position, or null if it's a Vector4 constructor with a single
2722             // operand, and
2723             // - op4 is the W position value, or null if this is a Vector3 constructor.
2724             if (size == 16 && argCount == 3)
2725             {
2726                 op4 = op3;
2727                 op3 = nullptr;
2728             }
2729
2730             simdTree = op2;
2731             if (op3 != nullptr)
2732             {
2733                 simdTree = gtNewSIMDNode(simdType, simdTree, op3, SIMDIntrinsicSetZ, baseType, size);
2734             }
2735             if (op4 != nullptr)
2736             {
2737                 simdTree = gtNewSIMDNode(simdType, simdTree, op4, SIMDIntrinsicSetW, baseType, size);
2738             }
2739
2740             copyBlkDst = op1;
2741             doCopyBlk  = true;
2742         }
2743         break;
2744
2745         case SIMDIntrinsicOpEquality:
2746         case SIMDIntrinsicInstEquals:
2747         {
2748             op2 = impSIMDPopStack(simdType);
2749             op1 = impSIMDPopStack(simdType, instMethod);
2750
2751             assert(op1->TypeGet() == simdType);
2752             assert(op2->TypeGet() == simdType);
2753
2754             simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, SIMDIntrinsicOpEquality, baseType, size);
2755             if (simdType == TYP_SIMD12)
2756             {
2757                 simdTree->gtFlags |= GTF_SIMD12_OP;
2758             }
2759             retVal = simdTree;
2760         }
2761         break;
2762
2763         case SIMDIntrinsicOpInEquality:
2764         {
2765             // op1 is the first operand
2766             // op2 is the second operand
2767             op2      = impSIMDPopStack(simdType);
2768             op1      = impSIMDPopStack(simdType, instMethod);
2769             simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, SIMDIntrinsicOpInEquality, baseType, size);
2770             if (simdType == TYP_SIMD12)
2771             {
2772                 simdTree->gtFlags |= GTF_SIMD12_OP;
2773             }
2774             retVal = simdTree;
2775         }
2776         break;
2777
2778         case SIMDIntrinsicEqual:
2779         case SIMDIntrinsicLessThan:
2780         case SIMDIntrinsicLessThanOrEqual:
2781         case SIMDIntrinsicGreaterThan:
2782         case SIMDIntrinsicGreaterThanOrEqual:
2783         {
2784             op2 = impSIMDPopStack(simdType);
2785             op1 = impSIMDPopStack(simdType, instMethod);
2786
2787             SIMDIntrinsicID intrinsicID = impSIMDRelOp(simdIntrinsicID, clsHnd, size, &baseType, &op1, &op2);
2788             simdTree                    = gtNewSIMDNode(genActualType(callType), op1, op2, intrinsicID, baseType, size);
2789             retVal                      = simdTree;
2790         }
2791         break;
2792
2793         case SIMDIntrinsicAdd:
2794         case SIMDIntrinsicSub:
2795         case SIMDIntrinsicMul:
2796         case SIMDIntrinsicDiv:
2797         case SIMDIntrinsicBitwiseAnd:
2798         case SIMDIntrinsicBitwiseAndNot:
2799         case SIMDIntrinsicBitwiseOr:
2800         case SIMDIntrinsicBitwiseXor:
2801         {
2802 #if defined(DEBUG)
2803             // check for the cases where we don't support intrinsics.
2804             // This check should be done before we make modifications to type stack.
2805             // Note that this is more of a double safety check for robustness since
2806             // we expect getSIMDIntrinsicInfo() to have filtered out intrinsics on
2807             // unsupported base types. If getSIMdIntrinsicInfo() doesn't filter due
2808             // to some bug, assert in chk/dbg will fire.
2809             if (!varTypeIsFloating(baseType))
2810             {
2811                 if (simdIntrinsicID == SIMDIntrinsicMul)
2812                 {
2813 #if defined(_TARGET_XARCH_)
2814                     if ((baseType != TYP_INT) && (baseType != TYP_SHORT))
2815                     {
2816                         // TODO-CQ: implement mul on these integer vectors.
2817                         // Note that SSE2 has no direct support for these vectors.
2818                         assert(!"Mul not supported on long/ulong/uint/small int vectors\n");
2819                         return nullptr;
2820                     }
2821 #endif // _TARGET_XARCH_
2822 #if defined(_TARGET_ARM64_)
2823                     if ((baseType == TYP_ULONG) && (baseType == TYP_LONG))
2824                     {
2825                         // TODO-CQ: implement mul on these integer vectors.
2826                         // Note that ARM64 has no direct support for these vectors.
2827                         assert(!"Mul not supported on long/ulong vectors\n");
2828                         return nullptr;
2829                     }
2830 #endif // _TARGET_ARM64_
2831                 }
2832 #if defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
2833                 // common to all integer type vectors
2834                 if (simdIntrinsicID == SIMDIntrinsicDiv)
2835                 {
2836                     // SSE2 doesn't support div on non-floating point vectors.
2837                     assert(!"Div not supported on integer type vectors\n");
2838                     return nullptr;
2839                 }
2840 #endif // defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
2841             }
2842 #endif // DEBUG
2843
2844             // op1 is the first operand; if instance method, op1 is "this" arg
2845             // op2 is the second operand
2846             op2 = impSIMDPopStack(simdType);
2847             op1 = impSIMDPopStack(simdType, instMethod);
2848
2849             simdTree = gtNewSIMDNode(simdType, op1, op2, simdIntrinsicID, baseType, size);
2850             retVal   = simdTree;
2851         }
2852         break;
2853
2854         case SIMDIntrinsicSelect:
2855         {
2856             // op3 is a SIMD variable that is the second source
2857             // op2 is a SIMD variable that is the first source
2858             // op1 is a SIMD variable which is the bit mask.
2859             op3 = impSIMDPopStack(simdType);
2860             op2 = impSIMDPopStack(simdType);
2861             op1 = impSIMDPopStack(simdType);
2862
2863             retVal = impSIMDSelect(clsHnd, baseType, size, op1, op2, op3);
2864         }
2865         break;
2866
2867         case SIMDIntrinsicMin:
2868         case SIMDIntrinsicMax:
2869         {
2870             // op1 is the first operand; if instance method, op1 is "this" arg
2871             // op2 is the second operand
2872             op2 = impSIMDPopStack(simdType);
2873             op1 = impSIMDPopStack(simdType, instMethod);
2874
2875             retVal = impSIMDMinMax(simdIntrinsicID, clsHnd, baseType, size, op1, op2);
2876         }
2877         break;
2878
2879         case SIMDIntrinsicGetItem:
2880         {
2881             // op1 is a SIMD variable that is "this" arg
2882             // op2 is an index of TYP_INT
2883             op2              = impSIMDPopStack(TYP_INT);
2884             op1              = impSIMDPopStack(simdType, instMethod);
2885             int vectorLength = getSIMDVectorLength(size, baseType);
2886             if (!op2->IsCnsIntOrI() || op2->AsIntCon()->gtIconVal >= vectorLength || op2->AsIntCon()->gtIconVal < 0)
2887             {
2888                 // We need to bounds-check the length of the vector.
2889                 // For that purpose, we need to clone the index expression.
2890                 GenTree* index = op2;
2891                 if ((index->gtFlags & GTF_SIDE_EFFECT) != 0)
2892                 {
2893                     op2 = fgInsertCommaFormTemp(&index);
2894                 }
2895                 else
2896                 {
2897                     op2 = gtCloneExpr(index);
2898                 }
2899
2900                 GenTree*          lengthNode = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, vectorLength);
2901                 GenTreeBoundsChk* simdChk =
2902                     new (this, GT_SIMD_CHK) GenTreeBoundsChk(GT_SIMD_CHK, TYP_VOID, index, lengthNode, SCK_RNGCHK_FAIL);
2903
2904                 // Create a GT_COMMA tree for the bounds check.
2905                 op2 = gtNewOperNode(GT_COMMA, op2->TypeGet(), simdChk, op2);
2906             }
2907
2908             assert(op1->TypeGet() == simdType);
2909             assert(op2->TypeGet() == TYP_INT);
2910
2911             simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, simdIntrinsicID, baseType, size);
2912             retVal   = simdTree;
2913         }
2914         break;
2915
2916         case SIMDIntrinsicDotProduct:
2917         {
2918 #if defined(_TARGET_XARCH_)
2919             // Right now dot product is supported only for float/double vectors and
2920             // int vectors on SSE4/AVX.
2921             if (!varTypeIsFloating(baseType) && !(baseType == TYP_INT && getSIMDSupportLevel() >= SIMD_SSE4_Supported))
2922             {
2923                 return nullptr;
2924             }
2925 #endif // _TARGET_XARCH_
2926
2927             // op1 is a SIMD variable that is the first source and also "this" arg.
2928             // op2 is a SIMD variable which is the second source.
2929             op2 = impSIMDPopStack(simdType);
2930             op1 = impSIMDPopStack(simdType, instMethod);
2931
2932             simdTree = gtNewSIMDNode(baseType, op1, op2, simdIntrinsicID, baseType, size);
2933             if (simdType == TYP_SIMD12)
2934             {
2935                 simdTree->gtFlags |= GTF_SIMD12_OP;
2936             }
2937             retVal = simdTree;
2938         }
2939         break;
2940
2941         case SIMDIntrinsicSqrt:
2942         {
2943 #if (defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)) && defined(DEBUG)
2944             // SSE/AVX/ARM64 doesn't support sqrt on integer type vectors and hence
2945             // should never be seen as an intrinsic here. See SIMDIntrinsicList.h
2946             // for supported base types for this intrinsic.
2947             if (!varTypeIsFloating(baseType))
2948             {
2949                 assert(!"Sqrt not supported on integer vectors\n");
2950                 return nullptr;
2951             }
2952 #endif // (defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)) && defined(DEBUG)
2953
2954             op1 = impSIMDPopStack(simdType);
2955
2956             retVal = gtNewSIMDNode(genActualType(callType), op1, nullptr, simdIntrinsicID, baseType, size);
2957         }
2958         break;
2959
2960         case SIMDIntrinsicAbs:
2961             op1    = impSIMDPopStack(simdType);
2962             retVal = impSIMDAbs(clsHnd, baseType, size, op1);
2963             break;
2964
2965         case SIMDIntrinsicGetW:
2966             retVal = impSIMDGetFixed(simdType, baseType, size, 3);
2967             break;
2968
2969         case SIMDIntrinsicGetZ:
2970             retVal = impSIMDGetFixed(simdType, baseType, size, 2);
2971             break;
2972
2973         case SIMDIntrinsicGetY:
2974             retVal = impSIMDGetFixed(simdType, baseType, size, 1);
2975             break;
2976
2977         case SIMDIntrinsicGetX:
2978             retVal = impSIMDGetFixed(simdType, baseType, size, 0);
2979             break;
2980
2981         case SIMDIntrinsicSetW:
2982         case SIMDIntrinsicSetZ:
2983         case SIMDIntrinsicSetY:
2984         case SIMDIntrinsicSetX:
2985         {
2986             // op2 is the value to be set at indexTemp position
2987             // op1 is SIMD vector that is going to be modified, which is a byref
2988
2989             // If op1 has a side-effect, then don't make it an intrinsic.
2990             // It would be in-efficient to read the entire vector into xmm reg,
2991             // modify it and write back entire xmm reg.
2992             //
2993             // TODO-CQ: revisit this later.
2994             op1 = impStackTop(1).val;
2995             if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0)
2996             {
2997                 return nullptr;
2998             }
2999
3000             op2 = impSIMDPopStack(baseType);
3001             op1 = impSIMDPopStack(simdType, instMethod);
3002
3003             GenTree* src = gtCloneExpr(op1);
3004             assert(src != nullptr);
3005             simdTree = gtNewSIMDNode(simdType, src, op2, simdIntrinsicID, baseType, size);
3006
3007             copyBlkDst = gtNewOperNode(GT_ADDR, TYP_BYREF, op1);
3008             doCopyBlk  = true;
3009         }
3010         break;
3011
3012         // Unary operators that take and return a Vector.
3013         case SIMDIntrinsicCast:
3014         case SIMDIntrinsicConvertToSingle:
3015         case SIMDIntrinsicConvertToDouble:
3016         case SIMDIntrinsicConvertToInt32:
3017         {
3018             op1 = impSIMDPopStack(simdType, instMethod);
3019
3020             simdTree = gtNewSIMDNode(simdType, op1, nullptr, simdIntrinsicID, baseType, size);
3021             retVal   = simdTree;
3022         }
3023         break;
3024
3025         case SIMDIntrinsicConvertToInt64:
3026         {
3027 #ifdef _TARGET_64BIT_
3028             op1 = impSIMDPopStack(simdType, instMethod);
3029
3030             simdTree = gtNewSIMDNode(simdType, op1, nullptr, simdIntrinsicID, baseType, size);
3031             retVal   = simdTree;
3032 #else
3033             JITDUMP("SIMD Conversion to Int64 is not supported on this platform\n");
3034             return nullptr;
3035 #endif
3036         }
3037         break;
3038
3039         case SIMDIntrinsicNarrow:
3040         {
3041             assert(!instMethod);
3042             op2 = impSIMDPopStack(simdType);
3043             op1 = impSIMDPopStack(simdType);
3044             // op1 and op2 are two input Vector<T>.
3045             simdTree = gtNewSIMDNode(simdType, op1, op2, simdIntrinsicID, baseType, size);
3046             retVal   = simdTree;
3047         }
3048         break;
3049
3050         case SIMDIntrinsicWiden:
3051         {
3052             GenTree* dstAddrHi = impSIMDPopStack(TYP_BYREF);
3053             GenTree* dstAddrLo = impSIMDPopStack(TYP_BYREF);
3054             op1                = impSIMDPopStack(simdType);
3055             GenTree* dupOp1    = fgInsertCommaFormTemp(&op1, gtGetStructHandleForSIMD(simdType, baseType));
3056
3057             // Widen the lower half and assign it to dstAddrLo.
3058             simdTree = gtNewSIMDNode(simdType, op1, nullptr, SIMDIntrinsicWidenLo, baseType, size);
3059             GenTree* loDest =
3060                 new (this, GT_BLK) GenTreeBlk(GT_BLK, simdType, dstAddrLo, getSIMDTypeSizeInBytes(clsHnd));
3061             GenTree* loAsg = gtNewBlkOpNode(loDest, simdTree, getSIMDTypeSizeInBytes(clsHnd),
3062                                             false, // not volatile
3063                                             true); // copyBlock
3064             loAsg->gtFlags |= ((simdTree->gtFlags | dstAddrLo->gtFlags) & GTF_ALL_EFFECT);
3065
3066             // Widen the upper half and assign it to dstAddrHi.
3067             simdTree = gtNewSIMDNode(simdType, dupOp1, nullptr, SIMDIntrinsicWidenHi, baseType, size);
3068             GenTree* hiDest =
3069                 new (this, GT_BLK) GenTreeBlk(GT_BLK, simdType, dstAddrHi, getSIMDTypeSizeInBytes(clsHnd));
3070             GenTree* hiAsg = gtNewBlkOpNode(hiDest, simdTree, getSIMDTypeSizeInBytes(clsHnd),
3071                                             false, // not volatile
3072                                             true); // copyBlock
3073             hiAsg->gtFlags |= ((simdTree->gtFlags | dstAddrHi->gtFlags) & GTF_ALL_EFFECT);
3074
3075             retVal = gtNewOperNode(GT_COMMA, simdType, loAsg, hiAsg);
3076         }
3077         break;
3078
3079         case SIMDIntrinsicHWAccel:
3080         {
3081             GenTreeIntCon* intConstTree = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, 1);
3082             retVal                      = intConstTree;
3083         }
3084         break;
3085
3086         default:
3087             assert(!"Unimplemented SIMD Intrinsic");
3088             return nullptr;
3089     }
3090
3091 #if defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
3092     // XArch/Arm64: also indicate that we use floating point registers.
3093     // The need for setting this here is that a method may not have SIMD
3094     // type lclvars, but might be exercising SIMD intrinsics on fields of
3095     // SIMD type.
3096     //
3097     // e.g.  public Vector<float> ComplexVecFloat::sqabs() { return this.r * this.r + this.i * this.i; }
3098     compFloatingPointUsed = true;
3099 #endif // defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
3100
3101     // At this point, we have a tree that we are going to store into a destination.
3102     // TODO-1stClassStructs: This should be a simple store or assignment, and should not require
3103     // GTF_ALL_EFFECT for the dest. This is currently emulating the previous behavior of
3104     // block ops.
3105     if (doCopyBlk)
3106     {
3107         GenTree* dest = new (this, GT_BLK) GenTreeBlk(GT_BLK, simdType, copyBlkDst, getSIMDTypeSizeInBytes(clsHnd));
3108         dest->gtFlags |= GTF_GLOB_REF;
3109         retVal = gtNewBlkOpNode(dest, simdTree, getSIMDTypeSizeInBytes(clsHnd),
3110                                 false, // not volatile
3111                                 true); // copyBlock
3112         retVal->gtFlags |= ((simdTree->gtFlags | copyBlkDst->gtFlags) & GTF_ALL_EFFECT);
3113     }
3114
3115     return retVal;
3116 }
3117
3118 #endif // FEATURE_SIMD
3119
3120 #endif // !LEGACY_BACKEND