src/jit/simd.cpp

   1 // Licensed to the .NET Foundation under one or more agreements.
   2 // The .NET Foundation licenses this file to you under the MIT license.
   3 // See the LICENSE file in the project root for more information.
   4
   5 //
   6 //   SIMD Support
   7 //
   8 // IMPORTANT NOTES AND CAVEATS:
   9 //
  10 // This implementation is preliminary, and may change dramatically.
  11 //
  12 // New JIT types, TYP_SIMDxx, are introduced, and the SIMD intrinsics are created as GT_SIMD nodes.
  13 // Nodes of SIMD types will be typed as TYP_SIMD* (e.g. TYP_SIMD8, TYP_SIMD16, etc.).
  14 //
  15 // Note that currently the "reference implementation" is the same as the runtime dll.  As such, it is currently
  16 // providing implementations for those methods not currently supported by the JIT as intrinsics.
  17 //
  18 // These are currently recognized using string compares, in order to provide an implementation in the JIT
  19 // without taking a dependency on the VM.
  20 // Furthermore, in the CTP, in order to limit the impact of doing these string compares
  21 // against assembly names, we only look for the SIMDVector assembly if we are compiling a class constructor.  This
  22 // makes it somewhat more "pay for play" but is a significant usability compromise.
  23 // This has been addressed for RTM by doing the assembly recognition in the VM.
  24 // --------------------------------------------------------------------------------------
  25
  26 #include "jitpch.h"
  27 #include "simd.h"
  28
  29 #ifdef _MSC_VER
  30 #pragma hdrstop
  31 #endif
  32
  33 #ifndef LEGACY_BACKEND // This file is ONLY used for the RyuJIT backend that uses the linear scan register allocator.
  34
  35 #ifdef FEATURE_SIMD
  36
  37 // Intrinsic Id to intrinsic info map
  38 const SIMDIntrinsicInfo simdIntrinsicInfoArray[] = {
  39 #define SIMD_INTRINSIC(mname, inst, id, name, retType, argCount, arg1, arg2, arg3, t1, t2, t3, t4, t5, t6, t7, t8, t9, \
  40                        t10)                                                                                            \
  41     {SIMDIntrinsic##id, mname, inst, retType, argCount, arg1, arg2, arg3, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10},
  42 #include "simdintrinsiclist.h"
  43 };
  44
  45 //------------------------------------------------------------------------
  46 // getSIMDVectorLength: Get the length (number of elements of base type) of
  47 //                      SIMD Vector given its size and base (element) type.
  48 //
  49 // Arguments:
  50 //    simdSize   - size of the SIMD vector
  51 //    baseType   - type of the elements of the SIMD vector
  52 //
  53 // static
  54 int Compiler::getSIMDVectorLength(unsigned simdSize, var_types baseType)
  55 {
  56     return simdSize / genTypeSize(baseType);
  57 }
  58
  59 //------------------------------------------------------------------------
  60 // Get the length (number of elements of base type) of SIMD Vector given by typeHnd.
  61 //
  62 // Arguments:
  63 //    typeHnd  - type handle of the SIMD vector
  64 //
  65 int Compiler::getSIMDVectorLength(CORINFO_CLASS_HANDLE typeHnd)
  66 {
  67     unsigned  sizeBytes = 0;
  68     var_types baseType  = getBaseTypeAndSizeOfSIMDType(typeHnd, &sizeBytes);
  69     return getSIMDVectorLength(sizeBytes, baseType);
  70 }
  71
  72 //------------------------------------------------------------------------
  73 // Get the preferred alignment of SIMD vector type for better performance.
  74 //
  75 // Arguments:
  76 //    typeHnd  - type handle of the SIMD vector
  77 //
  78 int Compiler::getSIMDTypeAlignment(var_types simdType)
  79 {
  80 #ifdef _TARGET_XARCH_
  81     // Fixed length vectors have the following alignment preference
  82     // Vector2   = 8 byte alignment
  83     // Vector3/4 = 16-byte alignment
  84     unsigned size = genTypeSize(simdType);
  85
  86     // preferred alignment for SSE2 128-bit vectors is 16-bytes
  87     if (size == 8)
  88     {
  89         return 8;
  90     }
  91     else if (size <= 16)
  92     {
  93         assert((size == 12) || (size == 16));
  94         return 16;
  95     }
  96     else
  97     {
  98         assert(size == 32);
  99         return 32;
 100     }
 101 #elif defined(_TARGET_ARM64_)
 102     return 16;
 103 #else
 104     assert(!"getSIMDTypeAlignment() unimplemented on target arch");
 105     unreached();
 106 #endif
 107 }
 108
 109 //----------------------------------------------------------------------------------
 110 // Return the base type and size of SIMD vector type given its type handle.
 111 //
 112 // Arguments:
 113 //    typeHnd   - The handle of the type we're interested in.
 114 //    sizeBytes - out param
 115 //
 116 // Return Value:
 117 //    base type of SIMD vector.
 118 //    sizeBytes if non-null is set to size in bytes.
 119 //
 120 // TODO-Throughput: current implementation parses class name to find base type. Change
 121 //         this when we implement  SIMD intrinsic identification for the final
 122 //         product.
 123 //
 124 var_types Compiler::getBaseTypeAndSizeOfSIMDType(CORINFO_CLASS_HANDLE typeHnd, unsigned* sizeBytes /*= nullptr */)
 125 {
 126     assert(featureSIMD);
 127     if (typeHnd == nullptr)
 128     {
 129         return TYP_UNKNOWN;
 130     }
 131
 132     // fast path search using cached type handles of important types
 133     var_types simdBaseType = TYP_UNKNOWN;
 134     unsigned  size         = 0;
 135
 136     // TODO - Optimize SIMD type recognition by IntrinsicAttribute
 137     if (isSIMDClass(typeHnd))
 138     {
 139         // The most likely to be used type handles are looked up first followed by
 140         // less likely to be used type handles
 141         if (typeHnd == SIMDFloatHandle)
 142         {
 143             simdBaseType = TYP_FLOAT;
 144             JITDUMP("  Known type SIMD Vector<Float>\n");
 145         }
 146         else if (typeHnd == SIMDIntHandle)
 147         {
 148             simdBaseType = TYP_INT;
 149             JITDUMP("  Known type SIMD Vector<Int>\n");
 150         }
 151         else if (typeHnd == SIMDVector2Handle)
 152         {
 153             simdBaseType = TYP_FLOAT;
 154             size         = 2 * genTypeSize(TYP_FLOAT);
 155             assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE));
 156             JITDUMP("  Known type Vector2\n");
 157         }
 158         else if (typeHnd == SIMDVector3Handle)
 159         {
 160             simdBaseType = TYP_FLOAT;
 161             size         = 3 * genTypeSize(TYP_FLOAT);
 162             assert(size == info.compCompHnd->getClassSize(typeHnd));
 163             JITDUMP("  Known type Vector3\n");
 164         }
 165         else if (typeHnd == SIMDVector4Handle)
 166         {
 167             simdBaseType = TYP_FLOAT;
 168             size         = 4 * genTypeSize(TYP_FLOAT);
 169             assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE));
 170             JITDUMP("  Known type Vector4\n");
 171         }
 172         else if (typeHnd == SIMDVectorHandle)
 173         {
 174             JITDUMP("  Known type Vector\n");
 175         }
 176         else if (typeHnd == SIMDUShortHandle)
 177         {
 178             simdBaseType = TYP_USHORT;
 179             JITDUMP("  Known type SIMD Vector<ushort>\n");
 180         }
 181         else if (typeHnd == SIMDUByteHandle)
 182         {
 183             simdBaseType = TYP_UBYTE;
 184             JITDUMP("  Known type SIMD Vector<ubyte>\n");
 185         }
 186         else if (typeHnd == SIMDDoubleHandle)
 187         {
 188             simdBaseType = TYP_DOUBLE;
 189             JITDUMP("  Known type SIMD Vector<Double>\n");
 190         }
 191         else if (typeHnd == SIMDLongHandle)
 192         {
 193             simdBaseType = TYP_LONG;
 194             JITDUMP("  Known type SIMD Vector<Long>\n");
 195         }
 196         else if (typeHnd == SIMDShortHandle)
 197         {
 198             simdBaseType = TYP_SHORT;
 199             JITDUMP("  Known type SIMD Vector<short>\n");
 200         }
 201         else if (typeHnd == SIMDByteHandle)
 202         {
 203             simdBaseType = TYP_BYTE;
 204             JITDUMP("  Known type SIMD Vector<byte>\n");
 205         }
 206         else if (typeHnd == SIMDUIntHandle)
 207         {
 208             simdBaseType = TYP_UINT;
 209             JITDUMP("  Known type SIMD Vector<uint>\n");
 210         }
 211         else if (typeHnd == SIMDULongHandle)
 212         {
 213             simdBaseType = TYP_ULONG;
 214             JITDUMP("  Known type SIMD Vector<ulong>\n");
 215         }
 216
 217         // slow path search
 218         if (simdBaseType == TYP_UNKNOWN)
 219         {
 220             // Doesn't match with any of the cached type handles.
 221             // Obtain base type by parsing fully qualified class name.
 222             //
 223             // TODO-Throughput: implement product shipping solution to query base type.
 224             WCHAR  className[256] = {0};
 225             WCHAR* pbuf           = &className[0];
 226             int    len            = _countof(className);
 227             info.compCompHnd->appendClassName(&pbuf, &len, typeHnd, TRUE, FALSE, FALSE);
 228             noway_assert(pbuf < &className[256]);
 229             JITDUMP("SIMD Candidate Type %S\n", className);
 230
 231             if (wcsncmp(className, W("System.Numerics."), 16) == 0)
 232             {
 233                 if (wcsncmp(&(className[16]), W("Vector`1["), 9) == 0)
 234                 {
 235                     if (wcsncmp(&(className[25]), W("System.Single"), 13) == 0)
 236                     {
 237                         SIMDFloatHandle = typeHnd;
 238                         simdBaseType    = TYP_FLOAT;
 239                         JITDUMP("  Found type SIMD Vector<Float>\n");
 240                     }
 241                     else if (wcsncmp(&(className[25]), W("System.Int32"), 12) == 0)
 242                     {
 243                         SIMDIntHandle = typeHnd;
 244                         simdBaseType  = TYP_INT;
 245                         JITDUMP("  Found type SIMD Vector<Int>\n");
 246                     }
 247                     else if (wcsncmp(&(className[25]), W("System.UInt16"), 13) == 0)
 248                     {
 249                         SIMDUShortHandle = typeHnd;
 250                         simdBaseType     = TYP_USHORT;
 251                         JITDUMP("  Found type SIMD Vector<ushort>\n");
 252                     }
 253                     else if (wcsncmp(&(className[25]), W("System.Byte"), 11) == 0)
 254                     {
 255                         SIMDUByteHandle = typeHnd;
 256                         simdBaseType    = TYP_UBYTE;
 257                         JITDUMP("  Found type SIMD Vector<ubyte>\n");
 258                     }
 259                     else if (wcsncmp(&(className[25]), W("System.Double"), 13) == 0)
 260                     {
 261                         SIMDDoubleHandle = typeHnd;
 262                         simdBaseType     = TYP_DOUBLE;
 263                         JITDUMP("  Found type SIMD Vector<Double>\n");
 264                     }
 265                     else if (wcsncmp(&(className[25]), W("System.Int64"), 12) == 0)
 266                     {
 267                         SIMDLongHandle = typeHnd;
 268                         simdBaseType   = TYP_LONG;
 269                         JITDUMP("  Found type SIMD Vector<Long>\n");
 270                     }
 271                     else if (wcsncmp(&(className[25]), W("System.Int16"), 12) == 0)
 272                     {
 273                         SIMDShortHandle = typeHnd;
 274                         simdBaseType    = TYP_SHORT;
 275                         JITDUMP("  Found type SIMD Vector<short>\n");
 276                     }
 277                     else if (wcsncmp(&(className[25]), W("System.SByte"), 12) == 0)
 278                     {
 279                         SIMDByteHandle = typeHnd;
 280                         simdBaseType   = TYP_BYTE;
 281                         JITDUMP("  Found type SIMD Vector<byte>\n");
 282                     }
 283                     else if (wcsncmp(&(className[25]), W("System.UInt32"), 13) == 0)
 284                     {
 285                         SIMDUIntHandle = typeHnd;
 286                         simdBaseType   = TYP_UINT;
 287                         JITDUMP("  Found type SIMD Vector<uint>\n");
 288                     }
 289                     else if (wcsncmp(&(className[25]), W("System.UInt64"), 13) == 0)
 290                     {
 291                         SIMDULongHandle = typeHnd;
 292                         simdBaseType    = TYP_ULONG;
 293                         JITDUMP("  Found type SIMD Vector<ulong>\n");
 294                     }
 295                     else
 296                     {
 297                         JITDUMP("  Unknown SIMD Vector<T>\n");
 298                     }
 299                 }
 300                 else if (wcsncmp(&(className[16]), W("Vector2"), 8) == 0)
 301                 {
 302                     SIMDVector2Handle = typeHnd;
 303
 304                     simdBaseType = TYP_FLOAT;
 305                     size         = 2 * genTypeSize(TYP_FLOAT);
 306                     assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE));
 307                     JITDUMP(" Found Vector2\n");
 308                 }
 309                 else if (wcsncmp(&(className[16]), W("Vector3"), 8) == 0)
 310                 {
 311                     SIMDVector3Handle = typeHnd;
 312
 313                     simdBaseType = TYP_FLOAT;
 314                     size         = 3 * genTypeSize(TYP_FLOAT);
 315                     assert(size == info.compCompHnd->getClassSize(typeHnd));
 316                     JITDUMP(" Found Vector3\n");
 317                 }
 318                 else if (wcsncmp(&(className[16]), W("Vector4"), 8) == 0)
 319                 {
 320                     SIMDVector4Handle = typeHnd;
 321
 322                     simdBaseType = TYP_FLOAT;
 323                     size         = 4 * genTypeSize(TYP_FLOAT);
 324                     assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE));
 325                     JITDUMP(" Found Vector4\n");
 326                 }
 327                 else if (wcsncmp(&(className[16]), W("Vector"), 6) == 0)
 328                 {
 329                     SIMDVectorHandle = typeHnd;
 330                     JITDUMP(" Found type Vector\n");
 331                 }
 332                 else
 333                 {
 334                     JITDUMP("  Unknown SIMD Type\n");
 335                 }
 336             }
 337         }
 338         if (simdBaseType != TYP_UNKNOWN && sizeBytes != nullptr)
 339         {
 340             // If not a fixed size vector then its size is same as SIMD vector
 341             // register length in bytes
 342             if (size == 0)
 343             {
 344                 size = getSIMDVectorRegisterByteLength();
 345             }
 346
 347             *sizeBytes = size;
 348             setUsesSIMDTypes(true);
 349         }
 350     }
 351 #ifdef FEATURE_HW_INTRINSICS
 352     else if (isIntrinsicType(typeHnd))
 353     {
 354         const size_t Vector64SizeBytes  = 64 / 8;
 355         const size_t Vector128SizeBytes = 128 / 8;
 356         const size_t Vector256SizeBytes = 256 / 8;
 357
 358 #if defined(_TARGET_XARCH_)
 359         static_assert_no_msg(YMM_REGSIZE_BYTES == Vector256SizeBytes);
 360         static_assert_no_msg(XMM_REGSIZE_BYTES == Vector128SizeBytes);
 361
 362         if (typeHnd == Vector256FloatHandle)
 363         {
 364             simdBaseType = TYP_FLOAT;
 365             size         = Vector256SizeBytes;
 366             JITDUMP("  Known type Vector256<float>\n");
 367         }
 368         else if (typeHnd == Vector256DoubleHandle)
 369         {
 370             simdBaseType = TYP_DOUBLE;
 371             size         = Vector256SizeBytes;
 372             JITDUMP("  Known type Vector256<double>\n");
 373         }
 374         else if (typeHnd == Vector256IntHandle)
 375         {
 376             simdBaseType = TYP_INT;
 377             size         = Vector256SizeBytes;
 378             JITDUMP("  Known type Vector256<int>\n");
 379         }
 380         else if (typeHnd == Vector256UIntHandle)
 381         {
 382             simdBaseType = TYP_UINT;
 383             size         = Vector256SizeBytes;
 384             JITDUMP("  Known type Vector256<uint>\n");
 385         }
 386         else if (typeHnd == Vector256ShortHandle)
 387         {
 388             simdBaseType = TYP_SHORT;
 389             size         = Vector256SizeBytes;
 390             JITDUMP("  Known type Vector256<short>\n");
 391         }
 392         else if (typeHnd == Vector256UShortHandle)
 393         {
 394             simdBaseType = TYP_USHORT;
 395             size         = Vector256SizeBytes;
 396             JITDUMP("  Known type Vector256<ushort>\n");
 397         }
 398         else if (typeHnd == Vector256ByteHandle)
 399         {
 400             simdBaseType = TYP_BYTE;
 401             size         = Vector256SizeBytes;
 402             JITDUMP("  Known type Vector256<sbyte>\n");
 403         }
 404         else if (typeHnd == Vector256UByteHandle)
 405         {
 406             simdBaseType = TYP_UBYTE;
 407             size         = Vector256SizeBytes;
 408             JITDUMP("  Known type Vector256<byte>\n");
 409         }
 410         else if (typeHnd == Vector256LongHandle)
 411         {
 412             simdBaseType = TYP_LONG;
 413             size         = Vector256SizeBytes;
 414             JITDUMP("  Known type Vector256<long>\n");
 415         }
 416         else if (typeHnd == Vector256ULongHandle)
 417         {
 418             simdBaseType = TYP_ULONG;
 419             size         = Vector256SizeBytes;
 420             JITDUMP("  Known type Vector256<ulong>\n");
 421         }
 422         else if (typeHnd == Vector256FloatHandle)
 423         {
 424             simdBaseType = TYP_FLOAT;
 425             size         = Vector256SizeBytes;
 426             JITDUMP("  Known type Vector256<float>\n");
 427         }
 428         else
 429 #endif // defined(_TARGET_XARCH)
 430             if (typeHnd == Vector128DoubleHandle)
 431         {
 432             simdBaseType = TYP_DOUBLE;
 433             size         = Vector128SizeBytes;
 434             JITDUMP("  Known type Vector128<double>\n");
 435         }
 436         else if (typeHnd == Vector128IntHandle)
 437         {
 438             simdBaseType = TYP_INT;
 439             size         = Vector128SizeBytes;
 440             JITDUMP("  Known type Vector128<int>\n");
 441         }
 442         else if (typeHnd == Vector128UIntHandle)
 443         {
 444             simdBaseType = TYP_UINT;
 445             size         = Vector128SizeBytes;
 446             JITDUMP("  Known type Vector128<uint>\n");
 447         }
 448         else if (typeHnd == Vector128ShortHandle)
 449         {
 450             simdBaseType = TYP_SHORT;
 451             size         = Vector128SizeBytes;
 452             JITDUMP("  Known type Vector128<short>\n");
 453         }
 454         else if (typeHnd == Vector128UShortHandle)
 455         {
 456             simdBaseType = TYP_USHORT;
 457             size         = Vector128SizeBytes;
 458             JITDUMP("  Known type Vector128<ushort>\n");
 459         }
 460         else if (typeHnd == Vector128ByteHandle)
 461         {
 462             simdBaseType = TYP_BYTE;
 463             size         = Vector128SizeBytes;
 464             JITDUMP("  Known type Vector128<sbyte>\n");
 465         }
 466         else if (typeHnd == Vector128UByteHandle)
 467         {
 468             simdBaseType = TYP_UBYTE;
 469             size         = Vector128SizeBytes;
 470             JITDUMP("  Known type Vector128<byte>\n");
 471         }
 472         else if (typeHnd == Vector128LongHandle)
 473         {
 474             simdBaseType = TYP_LONG;
 475             size         = Vector128SizeBytes;
 476             JITDUMP("  Known type Vector128<long>\n");
 477         }
 478         else if (typeHnd == Vector128ULongHandle)
 479         {
 480             simdBaseType = TYP_ULONG;
 481             size         = Vector128SizeBytes;
 482             JITDUMP("  Known type Vector128<ulong>\n");
 483         }
 484 #if defined(_TARGET_ARM64_)
 485         else if (typeHnd == Vector64DoubleHandle)
 486         {
 487             simdBaseType = TYP_DOUBLE;
 488             size         = Vector64SizeBytes;
 489             JITDUMP("  Known type Vector64<double>\n");
 490         }
 491         else if (typeHnd == Vector64IntHandle)
 492         {
 493             simdBaseType = TYP_INT;
 494             size         = Vector64SizeBytes;
 495             JITDUMP("  Known type Vector64<int>\n");
 496         }
 497         else if (typeHnd == Vector64UIntHandle)
 498         {
 499             simdBaseType = TYP_UINT;
 500             size         = Vector64SizeBytes;
 501             JITDUMP("  Known type Vector64<uint>\n");
 502         }
 503         else if (typeHnd == Vector64ShortHandle)
 504         {
 505             simdBaseType = TYP_SHORT;
 506             size         = Vector64SizeBytes;
 507             JITDUMP("  Known type Vector64<short>\n");
 508         }
 509         else if (typeHnd == Vector64UShortHandle)
 510         {
 511             simdBaseType = TYP_USHORT;
 512             size         = Vector64SizeBytes;
 513             JITDUMP("  Known type Vector64<ushort>\n");
 514         }
 515         else if (typeHnd == Vector64ByteHandle)
 516         {
 517             simdBaseType = TYP_BYTE;
 518             size         = Vector64SizeBytes;
 519             JITDUMP("  Known type Vector64<sbyte>\n");
 520         }
 521         else if (typeHnd == Vector64UByteHandle)
 522         {
 523             simdBaseType = TYP_UBYTE;
 524             size         = Vector64SizeBytes;
 525             JITDUMP("  Known type Vector64<byte>\n");
 526         }
 527         else if (typeHnd == Vector64LongHandle)
 528         {
 529             simdBaseType = TYP_LONG;
 530             size         = Vector64SizeBytes;
 531             JITDUMP("  Known type Vector64<long>\n");
 532         }
 533         else if (typeHnd == Vector64ULongHandle)
 534         {
 535             simdBaseType = TYP_ULONG;
 536             size         = Vector64SizeBytes;
 537             JITDUMP("  Known type Vector64<ulong>\n");
 538         }
 539 #endif // defined(_TARGET_ARM64_)
 540
 541         // slow path search
 542         if (simdBaseType == TYP_UNKNOWN)
 543         {
 544             // Doesn't match with any of the cached type handles.
 545             const char*          className   = getClassNameFromMetadata(typeHnd, nullptr);
 546             CORINFO_CLASS_HANDLE baseTypeHnd = getTypeInstantiationArgument(typeHnd, 0);
 547
 548             if (baseTypeHnd != nullptr)
 549             {
 550                 CorInfoType type = info.compCompHnd->getTypeForPrimitiveNumericClass(baseTypeHnd);
 551
 552                 JITDUMP("HW Intrinsic SIMD Candidate Type %s with Base Type %s\n", className,
 553                         getClassNameFromMetadata(baseTypeHnd, nullptr));
 554
 555 #if defined(_TARGET_XARCH_)
 556                 if (strcmp(className, "Vector256`1") == 0)
 557                 {
 558                     size = Vector256SizeBytes;
 559                     switch (type)
 560                     {
 561                         case CORINFO_TYPE_FLOAT:
 562                             Vector256FloatHandle = typeHnd;
 563                             simdBaseType         = TYP_FLOAT;
 564                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<float>\n");
 565                             break;
 566                         case CORINFO_TYPE_DOUBLE:
 567                             Vector256DoubleHandle = typeHnd;
 568                             simdBaseType          = TYP_DOUBLE;
 569                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<double>\n");
 570                             break;
 571                         case CORINFO_TYPE_INT:
 572                             Vector256IntHandle = typeHnd;
 573                             simdBaseType       = TYP_INT;
 574                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<int>\n");
 575                             break;
 576                         case CORINFO_TYPE_UINT:
 577                             Vector256UIntHandle = typeHnd;
 578                             simdBaseType        = TYP_UINT;
 579                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<uint>\n");
 580                             break;
 581                         case CORINFO_TYPE_SHORT:
 582                             Vector256ShortHandle = typeHnd;
 583                             simdBaseType         = TYP_SHORT;
 584                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<short>\n");
 585                             break;
 586                         case CORINFO_TYPE_USHORT:
 587                             Vector256UShortHandle = typeHnd;
 588                             simdBaseType          = TYP_USHORT;
 589                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<ushort>\n");
 590                             break;
 591                         case CORINFO_TYPE_LONG:
 592                             Vector256LongHandle = typeHnd;
 593                             simdBaseType        = TYP_LONG;
 594                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<long>\n");
 595                             break;
 596                         case CORINFO_TYPE_ULONG:
 597                             Vector256ULongHandle = typeHnd;
 598                             simdBaseType         = TYP_ULONG;
 599                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<ulong>\n");
 600                             break;
 601                         case CORINFO_TYPE_UBYTE:
 602                             Vector256UByteHandle = typeHnd;
 603                             simdBaseType         = TYP_UBYTE;
 604                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<byte>\n");
 605                             break;
 606                         case CORINFO_TYPE_BYTE:
 607                             Vector256ByteHandle = typeHnd;
 608                             simdBaseType        = TYP_BYTE;
 609                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<sbyte>\n");
 610                             break;
 611
 612                         default:
 613                             JITDUMP("  Unknown Hardware Intrinsic SIMD Type Vector256<T>\n");
 614                     }
 615                 }
 616                 else
 617 #endif // defined(_TARGET_XARCH_)
 618                     if (strcmp(className, "Vector128`1") == 0)
 619                 {
 620                     size = Vector128SizeBytes;
 621                     switch (type)
 622                     {
 623                         case CORINFO_TYPE_FLOAT:
 624                             Vector128FloatHandle = typeHnd;
 625                             simdBaseType         = TYP_FLOAT;
 626                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<float>\n");
 627                             break;
 628                         case CORINFO_TYPE_DOUBLE:
 629                             Vector128DoubleHandle = typeHnd;
 630                             simdBaseType          = TYP_DOUBLE;
 631                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<double>\n");
 632                             break;
 633                         case CORINFO_TYPE_INT:
 634                             Vector128IntHandle = typeHnd;
 635                             simdBaseType       = TYP_INT;
 636                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<int>\n");
 637                             break;
 638                         case CORINFO_TYPE_UINT:
 639                             Vector128UIntHandle = typeHnd;
 640                             simdBaseType        = TYP_UINT;
 641                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<uint>\n");
 642                             break;
 643                         case CORINFO_TYPE_SHORT:
 644                             Vector128ShortHandle = typeHnd;
 645                             simdBaseType         = TYP_SHORT;
 646                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<short>\n");
 647                             break;
 648                         case CORINFO_TYPE_USHORT:
 649                             Vector128UShortHandle = typeHnd;
 650                             simdBaseType          = TYP_USHORT;
 651                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<ushort>\n");
 652                             break;
 653                         case CORINFO_TYPE_LONG:
 654                             Vector128LongHandle = typeHnd;
 655                             simdBaseType        = TYP_LONG;
 656                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<long>\n");
 657                             break;
 658                         case CORINFO_TYPE_ULONG:
 659                             Vector128ULongHandle = typeHnd;
 660                             simdBaseType         = TYP_ULONG;
 661                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<ulong>\n");
 662                             break;
 663                         case CORINFO_TYPE_UBYTE:
 664                             Vector128UByteHandle = typeHnd;
 665                             simdBaseType         = TYP_UBYTE;
 666                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<byte>\n");
 667                             break;
 668                         case CORINFO_TYPE_BYTE:
 669                             Vector128ByteHandle = typeHnd;
 670                             simdBaseType        = TYP_BYTE;
 671                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<sbyte>\n");
 672                             break;
 673
 674                         default:
 675                             JITDUMP("  Unknown Hardware Intrinsic SIMD Type Vector128<T>\n");
 676                     }
 677                 }
 678 #if defined(_TARGET_ARM64_)
 679                 else if (strcmp(className, "Vector64`1") == 0)
 680                 {
 681                     size = Vector64SizeBytes;
 682                     switch (type)
 683                     {
 684                         case CORINFO_TYPE_FLOAT:
 685                             Vector64FloatHandle = typeHnd;
 686                             simdBaseType        = TYP_FLOAT;
 687                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<float>\n");
 688                             break;
 689                         case CORINFO_TYPE_DOUBLE:
 690                             Vector64DoubleHandle = typeHnd;
 691                             simdBaseType         = TYP_DOUBLE;
 692                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<double>\n");
 693                             break;
 694                         case CORINFO_TYPE_INT:
 695                             Vector64IntHandle = typeHnd;
 696                             simdBaseType      = TYP_INT;
 697                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<int>\n");
 698                             break;
 699                         case CORINFO_TYPE_UINT:
 700                             Vector64UIntHandle = typeHnd;
 701                             simdBaseType       = TYP_UINT;
 702                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<uint>\n");
 703                             break;
 704                         case CORINFO_TYPE_SHORT:
 705                             Vector64ShortHandle = typeHnd;
 706                             simdBaseType        = TYP_SHORT;
 707                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<short>\n");
 708                             break;
 709                         case CORINFO_TYPE_USHORT:
 710                             Vector64UShortHandle = typeHnd;
 711                             simdBaseType         = TYP_USHORT;
 712                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<ushort>\n");
 713                             break;
 714                         case CORINFO_TYPE_LONG:
 715                             Vector64LongHandle = typeHnd;
 716                             simdBaseType       = TYP_LONG;
 717                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<long>\n");
 718                             break;
 719                         case CORINFO_TYPE_ULONG:
 720                             Vector64ULongHandle = typeHnd;
 721                             simdBaseType        = TYP_ULONG;
 722                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<ulong>\n");
 723                             break;
 724                         case CORINFO_TYPE_UBYTE:
 725                             Vector64UByteHandle = typeHnd;
 726                             simdBaseType        = TYP_UBYTE;
 727                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<byte>\n");
 728                             break;
 729                         case CORINFO_TYPE_BYTE:
 730                             Vector64ByteHandle = typeHnd;
 731                             simdBaseType       = TYP_BYTE;
 732                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<sbyte>\n");
 733                             break;
 734
 735                         default:
 736                             JITDUMP("  Unknown Hardware Intrinsic SIMD Type Vector64<T>\n");
 737                     }
 738                 }
 739 #endif // defined(_TARGET_ARM64_)
 740             }
 741         }
 742
 743         if (simdBaseType != TYP_UNKNOWN && sizeBytes != nullptr)
 744         {
 745             *sizeBytes = size;
 746             setUsesSIMDTypes(true);
 747         }
 748     }
 749 #endif // FEATURE_HW_INTRINSICS
 750
 751     return simdBaseType;
 752 }
 753
 754 //--------------------------------------------------------------------------------------
 755 // getSIMDIntrinsicInfo: get SIMD intrinsic info given the method handle.
 756 //
 757 // Arguments:
 758 //    inOutTypeHnd    - The handle of the type on which the method is invoked.  This is an in-out param.
 759 //    methodHnd       - The handle of the method we're interested in.
 760 //    sig             - method signature info
 761 //    isNewObj        - whether this call represents a newboj constructor call
 762 //    argCount        - argument count - out pram
 763 //    baseType        - base type of the intrinsic - out param
 764 //    sizeBytes       - size of SIMD vector type on which the method is invoked - out param
 765 //
 766 // Return Value:
 767 //    SIMDIntrinsicInfo struct initialized corresponding to methodHnd.
 768 //    Sets SIMDIntrinsicInfo.id to SIMDIntrinsicInvalid if methodHnd doesn't correspond
 769 //    to any SIMD intrinsic.  Also, sets the out params inOutTypeHnd, argCount, baseType and
 770 //    sizeBytes.
 771 //
 772 //    Note that VectorMath class doesn't have a base type and first argument of the method
 773 //    determines the SIMD vector type on which intrinsic is invoked. In such a case inOutTypeHnd
 774 //    is modified by this routine.
 775 //
 776 // TODO-Throughput: The current implementation is based on method name string parsing.
 777 //         Although we now have type identification from the VM, the parsing of intrinsic names
 778 //         could be made more efficient.
 779 //
 780 const SIMDIntrinsicInfo* Compiler::getSIMDIntrinsicInfo(CORINFO_CLASS_HANDLE* inOutTypeHnd,
 781                                                         CORINFO_METHOD_HANDLE methodHnd,
 782                                                         CORINFO_SIG_INFO*     sig,
 783                                                         bool                  isNewObj,
 784                                                         unsigned*             argCount,
 785                                                         var_types*            baseType,
 786                                                         unsigned*             sizeBytes)
 787 {
 788     assert(featureSIMD);
 789     assert(baseType != nullptr);
 790     assert(sizeBytes != nullptr);
 791
 792     // get baseType and size of the type
 793     CORINFO_CLASS_HANDLE typeHnd = *inOutTypeHnd;
 794     *baseType                    = getBaseTypeAndSizeOfSIMDType(typeHnd, sizeBytes);
 795
 796     if (typeHnd == SIMDVectorHandle)
 797     {
 798         // All of the supported intrinsics on this static class take a first argument that's a vector,
 799         // which determines the baseType.
 800         // The exception is the IsHardwareAccelerated property, which is handled as a special case.
 801         assert(*baseType == TYP_UNKNOWN);
 802         if (sig->numArgs == 0)
 803         {
 804             const SIMDIntrinsicInfo* hwAccelIntrinsicInfo = &(simdIntrinsicInfoArray[SIMDIntrinsicHWAccel]);
 805             if ((strcmp(eeGetMethodName(methodHnd, nullptr), hwAccelIntrinsicInfo->methodName) == 0) &&
 806                 JITtype2varType(sig->retType) == hwAccelIntrinsicInfo->retType)
 807             {
 808                 // Sanity check
 809                 assert(hwAccelIntrinsicInfo->argCount == 0 && hwAccelIntrinsicInfo->isInstMethod == false);
 810                 return hwAccelIntrinsicInfo;
 811             }
 812             return nullptr;
 813         }
 814         else
 815         {
 816             typeHnd       = info.compCompHnd->getArgClass(sig, sig->args);
 817             *inOutTypeHnd = typeHnd;
 818             *baseType     = getBaseTypeAndSizeOfSIMDType(typeHnd, sizeBytes);
 819         }
 820     }
 821
 822     if (*baseType == TYP_UNKNOWN)
 823     {
 824         JITDUMP("NOT a SIMD Intrinsic: unsupported baseType\n");
 825         return nullptr;
 826     }
 827
 828     // account for implicit "this" arg
 829     *argCount = sig->numArgs;
 830     if (sig->hasThis())
 831     {
 832         *argCount += 1;
 833     }
 834
 835     // Get the Intrinsic Id by parsing method name.
 836     //
 837     // TODO-Throughput: replace sequential search by binary search by arranging entries
 838     // sorted by method name.
 839     SIMDIntrinsicID intrinsicId = SIMDIntrinsicInvalid;
 840     const char*     methodName  = eeGetMethodName(methodHnd, nullptr);
 841     for (int i = SIMDIntrinsicNone + 1; i < SIMDIntrinsicInvalid; ++i)
 842     {
 843         if (strcmp(methodName, simdIntrinsicInfoArray[i].methodName) == 0)
 844         {
 845             // Found an entry for the method; further check whether it is one of
 846             // the supported base types.
 847             bool found = false;
 848             for (int j = 0; j < SIMD_INTRINSIC_MAX_BASETYPE_COUNT; ++j)
 849             {
 850                 // Convention: if there are fewer base types supported than MAX_BASETYPE_COUNT,
 851                 // the end of the list is marked by TYP_UNDEF.
 852                 if (simdIntrinsicInfoArray[i].supportedBaseTypes[j] == TYP_UNDEF)
 853                 {
 854                     break;
 855                 }
 856
 857                 if (simdIntrinsicInfoArray[i].supportedBaseTypes[j] == *baseType)
 858                 {
 859                     found = true;
 860                     break;
 861                 }
 862             }
 863
 864             if (!found)
 865             {
 866                 continue;
 867             }
 868
 869             // Now, check the arguments.
 870             unsigned int fixedArgCnt    = simdIntrinsicInfoArray[i].argCount;
 871             unsigned int expectedArgCnt = fixedArgCnt;
 872
 873             // First handle SIMDIntrinsicInitN, where the arg count depends on the type.
 874             // The listed arg types include the vector and the first two init values, which is the expected number
 875             // for Vector2.  For other cases, we'll check their types here.
 876             if (*argCount > expectedArgCnt)
 877             {
 878                 if (i == SIMDIntrinsicInitN)
 879                 {
 880                     if (*argCount == 3 && typeHnd == SIMDVector2Handle)
 881                     {
 882                         expectedArgCnt = 3;
 883                     }
 884                     else if (*argCount == 4 && typeHnd == SIMDVector3Handle)
 885                     {
 886                         expectedArgCnt = 4;
 887                     }
 888                     else if (*argCount == 5 && typeHnd == SIMDVector4Handle)
 889                     {
 890                         expectedArgCnt = 5;
 891                     }
 892                 }
 893                 else if (i == SIMDIntrinsicInitFixed)
 894                 {
 895                     if (*argCount == 4 && typeHnd == SIMDVector4Handle)
 896                     {
 897                         expectedArgCnt = 4;
 898                     }
 899                 }
 900             }
 901             if (*argCount != expectedArgCnt)
 902             {
 903                 continue;
 904             }
 905
 906             // Validate the types of individual args passed are what is expected of.
 907             // If any of the types don't match with what is expected, don't consider
 908             // as an intrinsic.  This will make an older JIT with SIMD capabilities
 909             // resilient to breaking changes to SIMD managed API.
 910             //
 911             // Note that from IL type stack, args get popped in right to left order
 912             // whereas args get listed in method signatures in left to right order.
 913
 914             int stackIndex = (expectedArgCnt - 1);
 915
 916             // Track the arguments from the signature - we currently only use this to distinguish
 917             // integral and pointer types, both of which will by TYP_I_IMPL on the importer stack.
 918             CORINFO_ARG_LIST_HANDLE argLst = sig->args;
 919
 920             CORINFO_CLASS_HANDLE argClass;
 921             for (unsigned int argIndex = 0; found == true && argIndex < expectedArgCnt; argIndex++)
 922             {
 923                 bool isThisPtr = ((argIndex == 0) && sig->hasThis());
 924
 925                 // In case of "newobj SIMDVector<T>(T val)", thisPtr won't be present on type stack.
 926                 // We don't check anything in that case.
 927                 if (!isThisPtr || !isNewObj)
 928                 {
 929                     GenTreePtr arg     = impStackTop(stackIndex).val;
 930                     var_types  argType = arg->TypeGet();
 931
 932                     var_types expectedArgType;
 933                     if (argIndex < fixedArgCnt)
 934                     {
 935                         // Convention:
 936                         //   - intrinsicInfo.argType[i] == TYP_UNDEF - intrinsic doesn't have a valid arg at position i
 937                         //   - intrinsicInfo.argType[i] == TYP_UNKNOWN - arg type should be same as basetype
 938                         // Note that we pop the args off in reverse order.
 939                         expectedArgType = simdIntrinsicInfoArray[i].argType[argIndex];
 940                         assert(expectedArgType != TYP_UNDEF);
 941                         if (expectedArgType == TYP_UNKNOWN)
 942                         {
 943                             // The type of the argument will be genActualType(*baseType).
 944                             expectedArgType = genActualType(*baseType);
 945                             argType         = genActualType(argType);
 946                         }
 947                     }
 948                     else
 949                     {
 950                         expectedArgType = *baseType;
 951                     }
 952
 953                     if (!isThisPtr && argType == TYP_I_IMPL)
 954                     {
 955                         // The reference implementation has a constructor that takes a pointer.
 956                         // We don't want to recognize that one.  This requires us to look at the CorInfoType
 957                         // in order to distinguish a signature with a pointer argument from one with an
 958                         // integer argument of pointer size, both of which will be TYP_I_IMPL on the stack.
 959                         // TODO-Review: This seems quite fragile.  We should consider beefing up the checking
 960                         // here.
 961                         CorInfoType corType = strip(info.compCompHnd->getArgType(sig, argLst, &argClass));
 962                         if (corType == CORINFO_TYPE_PTR)
 963                         {
 964                             found = false;
 965                         }
 966                     }
 967
 968                     if (varTypeIsSIMD(argType))
 969                     {
 970                         argType = TYP_STRUCT;
 971                     }
 972                     if (argType != expectedArgType)
 973                     {
 974                         found = false;
 975                     }
 976                 }
 977                 if (argIndex != 0 || !sig->hasThis())
 978                 {
 979                     argLst = info.compCompHnd->getArgNext(argLst);
 980                 }
 981                 stackIndex--;
 982             }
 983
 984             // Cross check return type and static vs. instance is what we are expecting.
 985             // If not, don't consider it as an intrinsic.
 986             // Note that ret type of TYP_UNKNOWN means that it is not known apriori and must be same as baseType
 987             if (found)
 988             {
 989                 var_types expectedRetType = simdIntrinsicInfoArray[i].retType;
 990                 if (expectedRetType == TYP_UNKNOWN)
 991                 {
 992                     // JIT maps uint/ulong type vars to TYP_INT/TYP_LONG.
 993                     expectedRetType =
 994                         (*baseType == TYP_UINT || *baseType == TYP_ULONG) ? genActualType(*baseType) : *baseType;
 995                 }
 996
 997                 if (JITtype2varType(sig->retType) != expectedRetType ||
 998                     sig->hasThis() != simdIntrinsicInfoArray[i].isInstMethod)
 999                 {
1000                     found = false;
1001                 }
1002             }
1003
1004             if (found)
1005             {
1006                 intrinsicId = (SIMDIntrinsicID)i;
1007                 break;
1008             }
1009         }
1010     }
1011
1012     if (intrinsicId != SIMDIntrinsicInvalid)
1013     {
1014         JITDUMP("Method %s maps to SIMD intrinsic %s\n", methodName, simdIntrinsicNames[intrinsicId]);
1015         return &simdIntrinsicInfoArray[intrinsicId];
1016     }
1017     else
1018     {
1019         JITDUMP("Method %s is NOT a SIMD intrinsic\n", methodName);
1020     }
1021
1022     return nullptr;
1023 }
1024
1025 // Pops and returns GenTree node from importer's type stack.
1026 // Normalizes TYP_STRUCT value in case of GT_CALL, GT_RET_EXPR and arg nodes.
1027 //
1028 // Arguments:
1029 //    type        -  the type of value that the caller expects to be popped off the stack.
1030 //    expectAddr  -  if true indicates we are expecting type stack entry to be a TYP_BYREF.
1031 //
1032 // Notes:
1033 //    If the popped value is a struct, and the expected type is a simd type, it will be set
1034 //    to that type, otherwise it will assert if the type being popped is not the expected type.
1035
1036 GenTreePtr Compiler::impSIMDPopStack(var_types type, bool expectAddr)
1037 {
1038     StackEntry se   = impPopStack();
1039     typeInfo   ti   = se.seTypeInfo;
1040     GenTreePtr tree = se.val;
1041
1042     // If expectAddr is true implies what we have on stack is address and we need
1043     // SIMD type struct that it points to.
1044     if (expectAddr)
1045     {
1046         assert(tree->TypeGet() == TYP_BYREF);
1047         if (tree->OperGet() == GT_ADDR)
1048         {
1049             tree = tree->gtGetOp1();
1050         }
1051         else
1052         {
1053             tree = gtNewOperNode(GT_IND, type, tree);
1054         }
1055     }
1056
1057     bool isParam = false;
1058
1059     // If we have a ldobj of a SIMD local we need to transform it.
1060     if (tree->OperGet() == GT_OBJ)
1061     {
1062         GenTree* addr = tree->gtOp.gtOp1;
1063         if ((addr->OperGet() == GT_ADDR) && isSIMDTypeLocal(addr->gtOp.gtOp1))
1064         {
1065             tree = addr->gtOp.gtOp1;
1066         }
1067     }
1068
1069     if (tree->OperGet() == GT_LCL_VAR)
1070     {
1071         unsigned   lclNum    = tree->AsLclVarCommon()->GetLclNum();
1072         LclVarDsc* lclVarDsc = &lvaTable[lclNum];
1073         isParam              = lclVarDsc->lvIsParam;
1074     }
1075
1076     // normalize TYP_STRUCT value
1077     if (varTypeIsStruct(tree) && ((tree->OperGet() == GT_RET_EXPR) || (tree->OperGet() == GT_CALL) || isParam))
1078     {
1079         assert(ti.IsType(TI_STRUCT));
1080         CORINFO_CLASS_HANDLE structType = ti.GetClassHandleForValueClass();
1081         tree                            = impNormStructVal(tree, structType, (unsigned)CHECK_SPILL_ALL);
1082     }
1083
1084     // Now set the type of the tree to the specialized SIMD struct type, if applicable.
1085     if (genActualType(tree->gtType) != genActualType(type))
1086     {
1087         assert(tree->gtType == TYP_STRUCT);
1088         tree->gtType = type;
1089     }
1090     else if (tree->gtType == TYP_BYREF)
1091     {
1092         assert(tree->IsLocal() || (tree->OperGet() == GT_RET_EXPR) || (tree->OperGet() == GT_CALL) ||
1093                ((tree->gtOper == GT_ADDR) && varTypeIsSIMD(tree->gtGetOp1())));
1094     }
1095
1096     return tree;
1097 }
1098
1099 // impSIMDGetFixed: Create a GT_SIMD tree for a Get property of SIMD vector with a fixed index.
1100 //
1101 // Arguments:
1102 //    baseType - The base (element) type of the SIMD vector.
1103 //    simdSize - The total size in bytes of the SIMD vector.
1104 //    index    - The index of the field to get.
1105 //
1106 // Return Value:
1107 //    Returns a GT_SIMD node with the SIMDIntrinsicGetItem intrinsic id.
1108 //
1109 GenTreeSIMD* Compiler::impSIMDGetFixed(var_types simdType, var_types baseType, unsigned simdSize, int index)
1110 {
1111     assert(simdSize >= ((index + 1) * genTypeSize(baseType)));
1112
1113     // op1 is a SIMD source.
1114     GenTree* op1 = impSIMDPopStack(simdType, true);
1115
1116     GenTree*     op2      = gtNewIconNode(index);
1117     GenTreeSIMD* simdTree = gtNewSIMDNode(baseType, op1, op2, SIMDIntrinsicGetItem, baseType, simdSize);
1118     return simdTree;
1119 }
1120
1121 #ifdef _TARGET_XARCH_
1122 // impSIMDLongRelOpEqual: transforms operands and returns the SIMD intrinsic to be applied on
1123 // transformed operands to obtain == comparison result.
1124 //
1125 // Argumens:
1126 //    typeHnd  -  type handle of SIMD vector
1127 //    size     -  SIMD vector size
1128 //    op1      -  in-out parameter; first operand
1129 //    op2      -  in-out parameter; second operand
1130 //
1131 // Return Value:
1132 //    Modifies in-out params op1, op2 and returns intrinsic ID to be applied to modified operands
1133 //
1134 SIMDIntrinsicID Compiler::impSIMDLongRelOpEqual(CORINFO_CLASS_HANDLE typeHnd,
1135                                                 unsigned             size,
1136                                                 GenTree**            pOp1,
1137                                                 GenTree**            pOp2)
1138 {
1139     var_types simdType = (*pOp1)->TypeGet();
1140     assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1141
1142     // There is no direct SSE2 support for comparing TYP_LONG vectors.
1143     // These have to be implemented in terms of TYP_INT vector comparison operations.
1144     //
1145     // Equality(v1, v2):
1146     // tmp = (v1 == v2) i.e. compare for equality as if v1 and v2 are vector<int>
1147     // result = BitwiseAnd(t, shuffle(t, (2, 3, 0, 1)))
1148     // Shuffle is meant to swap the comparison results of low-32-bits and high 32-bits of respective long elements.
1149
1150     // Compare vector<long> as if they were vector<int> and assign the result to a temp
1151     GenTree* compResult = gtNewSIMDNode(simdType, *pOp1, *pOp2, SIMDIntrinsicEqual, TYP_INT, size);
1152     unsigned lclNum     = lvaGrabTemp(true DEBUGARG("SIMD Long =="));
1153     lvaSetStruct(lclNum, typeHnd, false);
1154     GenTree* tmp = gtNewLclvNode(lclNum, simdType);
1155     GenTree* asg = gtNewTempAssign(lclNum, compResult);
1156
1157     // op1 = GT_COMMA(tmp=compResult, tmp)
1158     // op2 = Shuffle(tmp, 0xB1)
1159     // IntrinsicId = BitwiseAnd
1160     *pOp1 = gtNewOperNode(GT_COMMA, simdType, asg, tmp);
1161     *pOp2 = gtNewSIMDNode(simdType, gtNewLclvNode(lclNum, simdType), gtNewIconNode(SHUFFLE_ZWXY, TYP_INT),
1162                           SIMDIntrinsicShuffleSSE2, TYP_INT, size);
1163     return SIMDIntrinsicBitwiseAnd;
1164 }
1165
1166 // impSIMDLongRelOpGreaterThan: transforms operands and returns the SIMD intrinsic to be applied on
1167 // transformed operands to obtain > comparison result.
1168 //
1169 // Argumens:
1170 //    typeHnd  -  type handle of SIMD vector
1171 //    size     -  SIMD vector size
1172 //    pOp1     -  in-out parameter; first operand
1173 //    pOp2     -  in-out parameter; second operand
1174 //
1175 // Return Value:
1176 //    Modifies in-out params pOp1, pOp2 and returns intrinsic ID to be applied to modified operands
1177 //
1178 SIMDIntrinsicID Compiler::impSIMDLongRelOpGreaterThan(CORINFO_CLASS_HANDLE typeHnd,
1179                                                       unsigned             size,
1180                                                       GenTree**            pOp1,
1181                                                       GenTree**            pOp2)
1182 {
1183     var_types simdType = (*pOp1)->TypeGet();
1184     assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1185
1186     // GreaterThan(v1, v2) where v1 and v2 are vector long.
1187     // Let us consider the case of single long element comparison.
1188     // say L1 = (x1, y1) and L2 = (x2, y2) where x1, y1, x2, and y2 are 32-bit integers that comprise the longs L1 and
1189     // L2.
1190     //
1191     // GreaterThan(L1, L2) can be expressed in terms of > relationship between 32-bit integers that comprise L1 and L2
1192     // as
1193     //                    =  (x1, y1) > (x2, y2)
1194     //                    =  (x1 > x2) || [(x1 == x2) && (y1 > y2)]   - eq (1)
1195     //
1196     // t = (v1 > v2)  32-bit signed comparison
1197     // u = (v1 == v2) 32-bit sized element equality
1198     // v = (v1 > v2)  32-bit unsigned comparison
1199     //
1200     // z = shuffle(t, (3, 3, 1, 1))  - This corresponds to (x1 > x2) in eq(1) above
1201     // t1 = Shuffle(v, (2, 2, 0, 0)) - This corresponds to (y1 > y2) in eq(1) above
1202     // u1 = Shuffle(u, (3, 3, 1, 1)) - This corresponds to (x1 == x2) in eq(1) above
1203     // w = And(t1, u1)               - This corresponds to [(x1 == x2) && (y1 > y2)] in eq(1) above
1204     // Result = BitwiseOr(z, w)
1205
1206     // Since op1 and op2 gets used multiple times, make sure side effects are computed.
1207     GenTree* dupOp1    = nullptr;
1208     GenTree* dupOp2    = nullptr;
1209     GenTree* dupDupOp1 = nullptr;
1210     GenTree* dupDupOp2 = nullptr;
1211
1212     if (((*pOp1)->gtFlags & GTF_SIDE_EFFECT) != 0)
1213     {
1214         dupOp1    = fgInsertCommaFormTemp(pOp1, typeHnd);
1215         dupDupOp1 = gtNewLclvNode(dupOp1->AsLclVarCommon()->GetLclNum(), simdType);
1216     }
1217     else
1218     {
1219         dupOp1    = gtCloneExpr(*pOp1);
1220         dupDupOp1 = gtCloneExpr(*pOp1);
1221     }
1222
1223     if (((*pOp2)->gtFlags & GTF_SIDE_EFFECT) != 0)
1224     {
1225         dupOp2    = fgInsertCommaFormTemp(pOp2, typeHnd);
1226         dupDupOp2 = gtNewLclvNode(dupOp2->AsLclVarCommon()->GetLclNum(), simdType);
1227     }
1228     else
1229     {
1230         dupOp2    = gtCloneExpr(*pOp2);
1231         dupDupOp2 = gtCloneExpr(*pOp2);
1232     }
1233
1234     assert(dupDupOp1 != nullptr && dupDupOp2 != nullptr);
1235     assert(dupOp1 != nullptr && dupOp2 != nullptr);
1236     assert(*pOp1 != nullptr && *pOp2 != nullptr);
1237
1238     // v1GreaterThanv2Signed - signed 32-bit comparison
1239     GenTree* v1GreaterThanv2Signed = gtNewSIMDNode(simdType, *pOp1, *pOp2, SIMDIntrinsicGreaterThan, TYP_INT, size);
1240
1241     // v1Equalsv2 - 32-bit equality
1242     GenTree* v1Equalsv2 = gtNewSIMDNode(simdType, dupOp1, dupOp2, SIMDIntrinsicEqual, TYP_INT, size);
1243
1244     // v1GreaterThanv2Unsigned - unsigned 32-bit comparison
1245     var_types       tempBaseType = TYP_UINT;
1246     SIMDIntrinsicID sid = impSIMDRelOp(SIMDIntrinsicGreaterThan, typeHnd, size, &tempBaseType, &dupDupOp1, &dupDupOp2);
1247     GenTree*        v1GreaterThanv2Unsigned = gtNewSIMDNode(simdType, dupDupOp1, dupDupOp2, sid, tempBaseType, size);
1248
1249     GenTree* z = gtNewSIMDNode(simdType, v1GreaterThanv2Signed, gtNewIconNode(SHUFFLE_WWYY, TYP_INT),
1250                                SIMDIntrinsicShuffleSSE2, TYP_FLOAT, size);
1251     GenTree* t1 = gtNewSIMDNode(simdType, v1GreaterThanv2Unsigned, gtNewIconNode(SHUFFLE_ZZXX, TYP_INT),
1252                                 SIMDIntrinsicShuffleSSE2, TYP_FLOAT, size);
1253     GenTree* u1 = gtNewSIMDNode(simdType, v1Equalsv2, gtNewIconNode(SHUFFLE_WWYY, TYP_INT), SIMDIntrinsicShuffleSSE2,
1254                                 TYP_FLOAT, size);
1255     GenTree* w = gtNewSIMDNode(simdType, u1, t1, SIMDIntrinsicBitwiseAnd, TYP_INT, size);
1256
1257     *pOp1 = z;
1258     *pOp2 = w;
1259     return SIMDIntrinsicBitwiseOr;
1260 }
1261
1262 // impSIMDLongRelOpGreaterThanOrEqual: transforms operands and returns the SIMD intrinsic to be applied on
1263 // transformed operands to obtain >= comparison result.
1264 //
1265 // Argumens:
1266 //    typeHnd  -  type handle of SIMD vector
1267 //    size     -  SIMD vector size
1268 //    pOp1      -  in-out parameter; first operand
1269 //    pOp2      -  in-out parameter; second operand
1270 //
1271 // Return Value:
1272 //    Modifies in-out params pOp1, pOp2 and returns intrinsic ID to be applied to modified operands
1273 //
1274 SIMDIntrinsicID Compiler::impSIMDLongRelOpGreaterThanOrEqual(CORINFO_CLASS_HANDLE typeHnd,
1275                                                              unsigned             size,
1276                                                              GenTree**            pOp1,
1277                                                              GenTree**            pOp2)
1278 {
1279     var_types simdType = (*pOp1)->TypeGet();
1280     assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1281
1282     // expand this to (a == b) | (a > b)
1283     GenTree* dupOp1 = nullptr;
1284     GenTree* dupOp2 = nullptr;
1285
1286     if (((*pOp1)->gtFlags & GTF_SIDE_EFFECT) != 0)
1287     {
1288         dupOp1 = fgInsertCommaFormTemp(pOp1, typeHnd);
1289     }
1290     else
1291     {
1292         dupOp1 = gtCloneExpr(*pOp1);
1293     }
1294
1295     if (((*pOp2)->gtFlags & GTF_SIDE_EFFECT) != 0)
1296     {
1297         dupOp2 = fgInsertCommaFormTemp(pOp2, typeHnd);
1298     }
1299     else
1300     {
1301         dupOp2 = gtCloneExpr(*pOp2);
1302     }
1303
1304     assert(dupOp1 != nullptr && dupOp2 != nullptr);
1305     assert(*pOp1 != nullptr && *pOp2 != nullptr);
1306
1307     // (a==b)
1308     SIMDIntrinsicID id = impSIMDLongRelOpEqual(typeHnd, size, pOp1, pOp2);
1309     *pOp1              = gtNewSIMDNode(simdType, *pOp1, *pOp2, id, TYP_LONG, size);
1310
1311     // (a > b)
1312     id    = impSIMDLongRelOpGreaterThan(typeHnd, size, &dupOp1, &dupOp2);
1313     *pOp2 = gtNewSIMDNode(simdType, dupOp1, dupOp2, id, TYP_LONG, size);
1314
1315     return SIMDIntrinsicBitwiseOr;
1316 }
1317
1318 // impSIMDInt32OrSmallIntRelOpGreaterThanOrEqual: transforms operands and returns the SIMD intrinsic to be applied on
1319 // transformed operands to obtain >= comparison result in case of integer base type vectors
1320 //
1321 // Argumens:
1322 //    typeHnd  -  type handle of SIMD vector
1323 //    size     -  SIMD vector size
1324 //    baseType -  base type of SIMD vector
1325 //    pOp1      -  in-out parameter; first operand
1326 //    pOp2      -  in-out parameter; second operand
1327 //
1328 // Return Value:
1329 //    Modifies in-out params pOp1, pOp2 and returns intrinsic ID to be applied to modified operands
1330 //
1331 SIMDIntrinsicID Compiler::impSIMDIntegralRelOpGreaterThanOrEqual(
1332     CORINFO_CLASS_HANDLE typeHnd, unsigned size, var_types baseType, GenTree** pOp1, GenTree** pOp2)
1333 {
1334     var_types simdType = (*pOp1)->TypeGet();
1335     assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1336
1337     // This routine should be used only for integer base type vectors
1338     assert(varTypeIsIntegral(baseType));
1339     if ((getSIMDSupportLevel() == SIMD_SSE2_Supported) && ((baseType == TYP_LONG) || baseType == TYP_UBYTE))
1340     {
1341         return impSIMDLongRelOpGreaterThanOrEqual(typeHnd, size, pOp1, pOp2);
1342     }
1343
1344     // expand this to (a == b) | (a > b)
1345     GenTree* dupOp1 = nullptr;
1346     GenTree* dupOp2 = nullptr;
1347
1348     if (((*pOp1)->gtFlags & GTF_SIDE_EFFECT) != 0)
1349     {
1350         dupOp1 = fgInsertCommaFormTemp(pOp1, typeHnd);
1351     }
1352     else
1353     {
1354         dupOp1 = gtCloneExpr(*pOp1);
1355     }
1356
1357     if (((*pOp2)->gtFlags & GTF_SIDE_EFFECT) != 0)
1358     {
1359         dupOp2 = fgInsertCommaFormTemp(pOp2, typeHnd);
1360     }
1361     else
1362     {
1363         dupOp2 = gtCloneExpr(*pOp2);
1364     }
1365
1366     assert(dupOp1 != nullptr && dupOp2 != nullptr);
1367     assert(*pOp1 != nullptr && *pOp2 != nullptr);
1368
1369     // (a==b)
1370     *pOp1 = gtNewSIMDNode(simdType, *pOp1, *pOp2, SIMDIntrinsicEqual, baseType, size);
1371
1372     // (a > b)
1373     *pOp2 = gtNewSIMDNode(simdType, dupOp1, dupOp2, SIMDIntrinsicGreaterThan, baseType, size);
1374
1375     return SIMDIntrinsicBitwiseOr;
1376 }
1377 #endif // _TARGET_XARCH_
1378
1379 // Transforms operands and returns the SIMD intrinsic to be applied on
1380 // transformed operands to obtain given relop result.
1381 //
1382 // Argumens:
1383 //    relOpIntrinsicId - Relational operator SIMD intrinsic
1384 //    typeHnd          - type handle of SIMD vector
1385 //    size             -  SIMD vector size
1386 //    inOutBaseType    - base type of SIMD vector
1387 //    pOp1             -  in-out parameter; first operand
1388 //    pOp2             -  in-out parameter; second operand
1389 //
1390 // Return Value:
1391 //    Modifies in-out params pOp1, pOp2, inOutBaseType and returns intrinsic ID to be applied to modified operands
1392 //
1393 SIMDIntrinsicID Compiler::impSIMDRelOp(SIMDIntrinsicID      relOpIntrinsicId,
1394                                        CORINFO_CLASS_HANDLE typeHnd,
1395                                        unsigned             size,
1396                                        var_types*           inOutBaseType,
1397                                        GenTree**            pOp1,
1398                                        GenTree**            pOp2)
1399 {
1400     var_types simdType = (*pOp1)->TypeGet();
1401     assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1402
1403     assert(isRelOpSIMDIntrinsic(relOpIntrinsicId));
1404
1405     SIMDIntrinsicID intrinsicID = relOpIntrinsicId;
1406 #ifdef _TARGET_XARCH_
1407     var_types baseType = *inOutBaseType;
1408
1409     if (varTypeIsFloating(baseType))
1410     {
1411         // SSE2/AVX doesn't support > and >= on vector float/double.
1412         // Therefore, we need to use < and <= with swapped operands
1413         if (relOpIntrinsicId == SIMDIntrinsicGreaterThan || relOpIntrinsicId == SIMDIntrinsicGreaterThanOrEqual)
1414         {
1415             GenTree* tmp = *pOp1;
1416             *pOp1        = *pOp2;
1417             *pOp2        = tmp;
1418
1419             intrinsicID =
1420                 (relOpIntrinsicId == SIMDIntrinsicGreaterThan) ? SIMDIntrinsicLessThan : SIMDIntrinsicLessThanOrEqual;
1421         }
1422     }
1423     else if (varTypeIsIntegral(baseType))
1424     {
1425         // SSE/AVX doesn't support < and <= on integer base type vectors.
1426         // Therefore, we need to use > and >= with swapped operands.
1427         if (intrinsicID == SIMDIntrinsicLessThan || intrinsicID == SIMDIntrinsicLessThanOrEqual)
1428         {
1429             GenTree* tmp = *pOp1;
1430             *pOp1        = *pOp2;
1431             *pOp2        = tmp;
1432
1433             intrinsicID = (relOpIntrinsicId == SIMDIntrinsicLessThan) ? SIMDIntrinsicGreaterThan
1434                                                                       : SIMDIntrinsicGreaterThanOrEqual;
1435         }
1436
1437         if ((getSIMDSupportLevel() == SIMD_SSE2_Supported) && baseType == TYP_LONG)
1438         {
1439             // There is no direct SSE2 support for comparing TYP_LONG vectors.
1440             // These have to be implemented interms of TYP_INT vector comparison operations.
1441             if (intrinsicID == SIMDIntrinsicEqual)
1442             {
1443                 intrinsicID = impSIMDLongRelOpEqual(typeHnd, size, pOp1, pOp2);
1444             }
1445             else if (intrinsicID == SIMDIntrinsicGreaterThan)
1446             {
1447                 intrinsicID = impSIMDLongRelOpGreaterThan(typeHnd, size, pOp1, pOp2);
1448             }
1449             else if (intrinsicID == SIMDIntrinsicGreaterThanOrEqual)
1450             {
1451                 intrinsicID = impSIMDLongRelOpGreaterThanOrEqual(typeHnd, size, pOp1, pOp2);
1452             }
1453             else
1454             {
1455                 unreached();
1456             }
1457         }
1458         // SSE2 and AVX direct support for signed comparison of int32, int16 and int8 types
1459         else if (!varTypeIsUnsigned(baseType))
1460         {
1461             if (intrinsicID == SIMDIntrinsicGreaterThanOrEqual)
1462             {
1463                 intrinsicID = impSIMDIntegralRelOpGreaterThanOrEqual(typeHnd, size, baseType, pOp1, pOp2);
1464             }
1465         }
1466         else // unsigned
1467         {
1468             // Vector<byte>, Vector<ushort>, Vector<uint> and Vector<ulong>:
1469             // SSE2 supports > for signed comparison. Therefore, to use it for
1470             // comparing unsigned numbers, we subtract a constant from both the
1471             // operands such that the result fits within the corresponding signed
1472             // type.  The resulting signed numbers are compared using SSE2 signed
1473             // comparison.
1474             //
1475             // Vector<byte>: constant to be subtracted is 2^7
1476             // Vector<ushort> constant to be subtracted is 2^15
1477             // Vector<uint> constant to be subtracted is 2^31
1478             // Vector<ulong> constant to be subtracted is 2^63
1479             //
1480             // We need to treat op1 and op2 as signed for comparison purpose after
1481             // the transformation.
1482             __int64 constVal = 0;
1483             switch (baseType)
1484             {
1485                 case TYP_UBYTE:
1486                     constVal       = 0x80808080;
1487                     *inOutBaseType = TYP_BYTE;
1488                     break;
1489                 case TYP_USHORT:
1490                     constVal       = 0x80008000;
1491                     *inOutBaseType = TYP_SHORT;
1492                     break;
1493                 case TYP_UINT:
1494                     constVal       = 0x80000000;
1495                     *inOutBaseType = TYP_INT;
1496                     break;
1497                 case TYP_ULONG:
1498                     constVal       = 0x8000000000000000LL;
1499                     *inOutBaseType = TYP_LONG;
1500                     break;
1501                 default:
1502                     unreached();
1503                     break;
1504             }
1505             assert(constVal != 0);
1506
1507             // This transformation is not required for equality.
1508             if (intrinsicID != SIMDIntrinsicEqual)
1509             {
1510                 // For constructing const vector use either long or int base type.
1511                 var_types tempBaseType;
1512                 GenTree*  initVal;
1513                 if (baseType == TYP_ULONG)
1514                 {
1515                     tempBaseType = TYP_LONG;
1516                     initVal      = gtNewLconNode(constVal);
1517                 }
1518                 else
1519                 {
1520                     tempBaseType = TYP_INT;
1521                     initVal      = gtNewIconNode((ssize_t)constVal);
1522                 }
1523                 initVal->gtType      = tempBaseType;
1524                 GenTree* constVector = gtNewSIMDNode(simdType, initVal, nullptr, SIMDIntrinsicInit, tempBaseType, size);
1525
1526                 // Assign constVector to a temp, since we intend to use it more than once
1527                 // TODO-CQ: We have quite a few such constant vectors constructed during
1528                 // the importation of SIMD intrinsics.  Make sure that we have a single
1529                 // temp per distinct constant per method.
1530                 GenTree* tmp = fgInsertCommaFormTemp(&constVector, typeHnd);
1531
1532                 // op1 = op1 - constVector
1533                 // op2 = op2 - constVector
1534                 *pOp1 = gtNewSIMDNode(simdType, *pOp1, constVector, SIMDIntrinsicSub, baseType, size);
1535                 *pOp2 = gtNewSIMDNode(simdType, *pOp2, tmp, SIMDIntrinsicSub, baseType, size);
1536             }
1537
1538             return impSIMDRelOp(intrinsicID, typeHnd, size, inOutBaseType, pOp1, pOp2);
1539         }
1540     }
1541 #elif defined(_TARGET_ARM64_)
1542     // TODO-ARM64-CQ handle comparisons against zero
1543
1544     // _TARGET_ARM64_ doesn't support < and <= on register register comparisons
1545     // Therefore, we need to use > and >= with swapped operands.
1546     if (intrinsicID == SIMDIntrinsicLessThan || intrinsicID == SIMDIntrinsicLessThanOrEqual)
1547     {
1548         GenTree* tmp = *pOp1;
1549         *pOp1        = *pOp2;
1550         *pOp2        = tmp;
1551
1552         intrinsicID =
1553             (intrinsicID == SIMDIntrinsicLessThan) ? SIMDIntrinsicGreaterThan : SIMDIntrinsicGreaterThanOrEqual;
1554     }
1555 #else  // !_TARGET_XARCH_
1556     assert(!"impSIMDRelOp() unimplemented on target arch");
1557     unreached();
1558 #endif // !_TARGET_XARCH_
1559
1560     return intrinsicID;
1561 }
1562
1563 //-------------------------------------------------------------------------
1564 // impSIMDAbs: creates GT_SIMD node to compute Abs value of a given vector.
1565 //
1566 // Arguments:
1567 //    typeHnd     -  type handle of SIMD vector
1568 //    baseType    -  base type of vector
1569 //    size        -  vector size in bytes
1570 //    op1         -  operand of Abs intrinsic
1571 //
1572 GenTreePtr Compiler::impSIMDAbs(CORINFO_CLASS_HANDLE typeHnd, var_types baseType, unsigned size, GenTree* op1)
1573 {
1574     assert(varTypeIsSIMD(op1));
1575
1576     var_types  simdType = op1->TypeGet();
1577     GenTreePtr retVal   = nullptr;
1578
1579 #ifdef _TARGET_XARCH_
1580     // When there is no direct support, Abs(v) could be computed
1581     // on integer vectors as follows:
1582     //     BitVector = v < vector.Zero
1583     //     result = ConditionalSelect(BitVector, vector.Zero - v, v)
1584
1585     bool useConditionalSelect = false;
1586     if (getSIMDSupportLevel() == SIMD_SSE2_Supported)
1587     {
1588         // SSE2 doesn't support abs on signed integer type vectors.
1589         if (baseType == TYP_LONG || baseType == TYP_INT || baseType == TYP_SHORT || baseType == TYP_BYTE)
1590         {
1591             useConditionalSelect = true;
1592         }
1593     }
1594     else
1595     {
1596         assert(getSIMDSupportLevel() >= SIMD_SSE4_Supported);
1597         if (baseType == TYP_LONG)
1598         {
1599             // SSE4/AVX2 don't support abs on long type vector.
1600             useConditionalSelect = true;
1601         }
1602     }
1603
1604     if (useConditionalSelect)
1605     {
1606         // This works only on integer vectors not on float/double vectors.
1607         assert(varTypeIsIntegral(baseType));
1608
1609         GenTreePtr op1Assign;
1610         unsigned   op1LclNum;
1611
1612         if (op1->OperGet() == GT_LCL_VAR)
1613         {
1614             op1LclNum = op1->gtLclVarCommon.gtLclNum;
1615             op1Assign = nullptr;
1616         }
1617         else
1618         {
1619             op1LclNum = lvaGrabTemp(true DEBUGARG("SIMD Abs op1"));
1620             lvaSetStruct(op1LclNum, typeHnd, false);
1621             op1Assign = gtNewTempAssign(op1LclNum, op1);
1622             op1       = gtNewLclvNode(op1LclNum, op1->TypeGet());
1623         }
1624
1625         // Assign Vector.Zero to a temp since it is needed more than once
1626         GenTreePtr vecZero       = gtNewSIMDVectorZero(simdType, baseType, size);
1627         unsigned   vecZeroLclNum = lvaGrabTemp(true DEBUGARG("SIMD Abs VecZero"));
1628         lvaSetStruct(vecZeroLclNum, typeHnd, false);
1629         GenTreePtr vecZeroAssign = gtNewTempAssign(vecZeroLclNum, vecZero);
1630
1631         // Construct BitVector = v < vector.Zero
1632         GenTreePtr      bitVecOp1     = op1;
1633         GenTreePtr      bitVecOp2     = gtNewLclvNode(vecZeroLclNum, vecZero->TypeGet());
1634         var_types       relOpBaseType = baseType;
1635         SIMDIntrinsicID relOpIntrinsic =
1636             impSIMDRelOp(SIMDIntrinsicLessThan, typeHnd, size, &relOpBaseType, &bitVecOp1, &bitVecOp2);
1637         GenTreePtr bitVec       = gtNewSIMDNode(simdType, bitVecOp1, bitVecOp2, relOpIntrinsic, relOpBaseType, size);
1638         unsigned   bitVecLclNum = lvaGrabTemp(true DEBUGARG("SIMD Abs bitVec"));
1639         lvaSetStruct(bitVecLclNum, typeHnd, false);
1640         GenTreePtr bitVecAssign = gtNewTempAssign(bitVecLclNum, bitVec);
1641         bitVec                  = gtNewLclvNode(bitVecLclNum, bitVec->TypeGet());
1642
1643         // Construct condSelectOp1 = vector.Zero - v
1644         GenTreePtr subOp1 = gtNewLclvNode(vecZeroLclNum, vecZero->TypeGet());
1645         GenTreePtr subOp2 = gtNewLclvNode(op1LclNum, op1->TypeGet());
1646         GenTreePtr negVec = gtNewSIMDNode(simdType, subOp1, subOp2, SIMDIntrinsicSub, baseType, size);
1647
1648         // Construct ConditionalSelect(bitVec, vector.Zero - v, v)
1649         GenTreePtr vec = gtNewLclvNode(op1LclNum, op1->TypeGet());
1650         retVal         = impSIMDSelect(typeHnd, baseType, size, bitVec, negVec, vec);
1651
1652         // Prepend bitVec assignment to retVal.
1653         // retVal = (tmp2 = v < tmp1), CondSelect(tmp2, tmp1 - v, v)
1654         retVal = gtNewOperNode(GT_COMMA, simdType, bitVecAssign, retVal);
1655
1656         // Prepend vecZero assignment to retVal.
1657         // retVal =  (tmp1 = vector.Zero), (tmp2 = v < tmp1), CondSelect(tmp2, tmp1 - v, v)
1658         retVal = gtNewOperNode(GT_COMMA, simdType, vecZeroAssign, retVal);
1659
1660         // If op1 was assigned to a temp, prepend that to retVal.
1661         if (op1Assign != nullptr)
1662         {
1663             // retVal = (v=op1), (tmp1 = vector.Zero), (tmp2 = v < tmp1), CondSelect(tmp2, tmp1 - v, v)
1664             retVal = gtNewOperNode(GT_COMMA, simdType, op1Assign, retVal);
1665         }
1666     }
1667     else if (varTypeIsFloating(baseType))
1668     {
1669         // Abs(vf) = vf & new SIMDVector<float>(0x7fffffff);
1670         // Abs(vd) = vf & new SIMDVector<double>(0x7fffffffffffffff);
1671         GenTree* bitMask = nullptr;
1672         if (baseType == TYP_FLOAT)
1673         {
1674             float f;
1675             static_assert_no_msg(sizeof(float) == sizeof(int));
1676             *((int*)&f) = 0x7fffffff;
1677             bitMask     = gtNewDconNode(f);
1678         }
1679         else if (baseType == TYP_DOUBLE)
1680         {
1681             double d;
1682             static_assert_no_msg(sizeof(double) == sizeof(__int64));
1683             *((__int64*)&d) = 0x7fffffffffffffffLL;
1684             bitMask         = gtNewDconNode(d);
1685         }
1686
1687         assert(bitMask != nullptr);
1688         bitMask->gtType        = baseType;
1689         GenTree* bitMaskVector = gtNewSIMDNode(simdType, bitMask, SIMDIntrinsicInit, baseType, size);
1690         retVal                 = gtNewSIMDNode(simdType, op1, bitMaskVector, SIMDIntrinsicBitwiseAnd, baseType, size);
1691     }
1692     else if (baseType == TYP_USHORT || baseType == TYP_UBYTE || baseType == TYP_UINT || baseType == TYP_ULONG)
1693     {
1694         // Abs is a no-op on unsigned integer type vectors
1695         retVal = op1;
1696     }
1697     else
1698     {
1699         assert(getSIMDSupportLevel() >= SIMD_SSE4_Supported);
1700         assert(baseType != TYP_LONG);
1701
1702         retVal = gtNewSIMDNode(simdType, op1, SIMDIntrinsicAbs, baseType, size);
1703     }
1704 #elif defined(_TARGET_ARM64_)
1705     if (varTypeIsUnsigned(baseType))
1706     {
1707         // Abs is a no-op on unsigned integer type vectors
1708         retVal = op1;
1709     }
1710     else
1711     {
1712         retVal = gtNewSIMDNode(simdType, op1, SIMDIntrinsicAbs, baseType, size);
1713     }
1714 #else  // !defined(_TARGET_XARCH)_ && !defined(_TARGET_ARM64_)
1715     assert(!"Abs intrinsic on non-xarch target not implemented");
1716 #endif // !_TARGET_XARCH_
1717
1718     return retVal;
1719 }
1720
1721 // Creates a GT_SIMD tree for Select operation
1722 //
1723 // Argumens:
1724 //    typeHnd          -  type handle of SIMD vector
1725 //    baseType         -  base type of SIMD vector
1726 //    size             -  SIMD vector size
1727 //    op1              -  first operand = Condition vector vc
1728 //    op2              -  second operand = va
1729 //    op3              -  third operand = vb
1730 //
1731 // Return Value:
1732 //    Returns GT_SIMD tree that computes Select(vc, va, vb)
1733 //
1734 GenTreePtr Compiler::impSIMDSelect(
1735     CORINFO_CLASS_HANDLE typeHnd, var_types baseType, unsigned size, GenTree* op1, GenTree* op2, GenTree* op3)
1736 {
1737     assert(varTypeIsSIMD(op1));
1738     var_types simdType = op1->TypeGet();
1739     assert(op2->TypeGet() == simdType);
1740     assert(op3->TypeGet() == simdType);
1741
1742     // TODO-ARM64-CQ Support generating select instruction for SIMD
1743
1744     // Select(BitVector vc, va, vb) = (va & vc) | (vb & !vc)
1745     // Select(op1, op2, op3)        = (op2 & op1) | (op3 & !op1)
1746     //                              = SIMDIntrinsicBitwiseOr(SIMDIntrinsicBitwiseAnd(op2, op1),
1747     //                                                       SIMDIntrinsicBitwiseAndNot(op3, op1))
1748     //
1749     // If Op1 has side effect, create an assignment to a temp
1750     GenTree* tmp = op1;
1751     GenTree* asg = nullptr;
1752     if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0)
1753     {
1754         unsigned lclNum = lvaGrabTemp(true DEBUGARG("SIMD Select"));
1755         lvaSetStruct(lclNum, typeHnd, false);
1756         tmp = gtNewLclvNode(lclNum, op1->TypeGet());
1757         asg = gtNewTempAssign(lclNum, op1);
1758     }
1759
1760     GenTree* andExpr = gtNewSIMDNode(simdType, op2, tmp, SIMDIntrinsicBitwiseAnd, baseType, size);
1761     GenTree* dupOp1  = gtCloneExpr(tmp);
1762     assert(dupOp1 != nullptr);
1763 #ifdef _TARGET_ARM64_
1764     // ARM64 implements SIMDIntrinsicBitwiseAndNot as Left & ~Right
1765     GenTree* andNotExpr = gtNewSIMDNode(simdType, op3, dupOp1, SIMDIntrinsicBitwiseAndNot, baseType, size);
1766 #else
1767     // XARCH implements SIMDIntrinsicBitwiseAndNot as ~Left & Right
1768     GenTree* andNotExpr = gtNewSIMDNode(simdType, dupOp1, op3, SIMDIntrinsicBitwiseAndNot, baseType, size);
1769 #endif
1770     GenTree* simdTree = gtNewSIMDNode(simdType, andExpr, andNotExpr, SIMDIntrinsicBitwiseOr, baseType, size);
1771
1772     // If asg not null, create a GT_COMMA tree.
1773     if (asg != nullptr)
1774     {
1775         simdTree = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), asg, simdTree);
1776     }
1777
1778     return simdTree;
1779 }
1780
1781 // Creates a GT_SIMD tree for Min/Max operation
1782 //
1783 // Argumens:
1784 //    IntrinsicId      -  SIMD intrinsic Id, either Min or Max
1785 //    typeHnd          -  type handle of SIMD vector
1786 //    baseType         -  base type of SIMD vector
1787 //    size             -  SIMD vector size
1788 //    op1              -  first operand = va
1789 //    op2              -  second operand = vb
1790 //
1791 // Return Value:
1792 //    Returns GT_SIMD tree that computes Max(va, vb)
1793 //
1794 GenTreePtr Compiler::impSIMDMinMax(SIMDIntrinsicID      intrinsicId,
1795                                    CORINFO_CLASS_HANDLE typeHnd,
1796                                    var_types            baseType,
1797                                    unsigned             size,
1798                                    GenTree*             op1,
1799                                    GenTree*             op2)
1800 {
1801     assert(intrinsicId == SIMDIntrinsicMin || intrinsicId == SIMDIntrinsicMax);
1802     assert(varTypeIsSIMD(op1));
1803     var_types simdType = op1->TypeGet();
1804     assert(op2->TypeGet() == simdType);
1805
1806 #if defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
1807     GenTree* simdTree = nullptr;
1808
1809 #ifdef _TARGET_XARCH_
1810     // SSE2 has direct support for float/double/signed word/unsigned byte.
1811     // SSE4.1 has direct support for int32/uint32/signed byte/unsigned word.
1812     // For other integer types we compute min/max as follows
1813     //
1814     // int32/uint32 (SSE2)
1815     // int64/uint64 (SSE2&SSE4):
1816     //       compResult        = (op1 < op2) in case of Min
1817     //                           (op1 > op2) in case of Max
1818     //       Min/Max(op1, op2) = Select(compResult, op1, op2)
1819     //
1820     // unsigned word (SSE2):
1821     //        op1 = op1 - 2^15  ; to make it fit within a signed word
1822     //        op2 = op2 - 2^15  ; to make it fit within a signed word
1823     //        result = SSE2 signed word Min/Max(op1, op2)
1824     //        result = result + 2^15  ; readjust it back
1825     //
1826     // signed byte (SSE2):
1827     //        op1 = op1 + 2^7  ; to make it unsigned
1828     //        op1 = op1 + 2^7  ; to make it unsigned
1829     //        result = SSE2 unsigned byte Min/Max(op1, op2)
1830     //        result = result - 2^15 ; readjust it back
1831
1832     if (varTypeIsFloating(baseType) || baseType == TYP_SHORT || baseType == TYP_UBYTE ||
1833         (getSIMDSupportLevel() >= SIMD_SSE4_Supported &&
1834          (baseType == TYP_BYTE || baseType == TYP_INT || baseType == TYP_UINT || baseType == TYP_USHORT)))
1835     {
1836         // SSE2 or SSE4.1 has direct support
1837         simdTree = gtNewSIMDNode(simdType, op1, op2, intrinsicId, baseType, size);
1838     }
1839     else if (baseType == TYP_USHORT || baseType == TYP_BYTE)
1840     {
1841         assert(getSIMDSupportLevel() == SIMD_SSE2_Supported);
1842         int             constVal;
1843         SIMDIntrinsicID operIntrinsic;
1844         SIMDIntrinsicID adjustIntrinsic;
1845         var_types       minMaxOperBaseType;
1846         if (baseType == TYP_USHORT)
1847         {
1848             constVal           = 0x80008000;
1849             operIntrinsic      = SIMDIntrinsicSub;
1850             adjustIntrinsic    = SIMDIntrinsicAdd;
1851             minMaxOperBaseType = TYP_SHORT;
1852         }
1853         else
1854         {
1855             assert(baseType == TYP_BYTE);
1856             constVal           = 0x80808080;
1857             operIntrinsic      = SIMDIntrinsicAdd;
1858             adjustIntrinsic    = SIMDIntrinsicSub;
1859             minMaxOperBaseType = TYP_UBYTE;
1860         }
1861
1862         GenTree* initVal     = gtNewIconNode(constVal);
1863         GenTree* constVector = gtNewSIMDNode(simdType, initVal, nullptr, SIMDIntrinsicInit, TYP_INT, size);
1864
1865         // Assign constVector to a temp, since we intend to use it more than once
1866         // TODO-CQ: We have quite a few such constant vectors constructed during
1867         // the importation of SIMD intrinsics.  Make sure that we have a single
1868         // temp per distinct constant per method.
1869         GenTree* tmp = fgInsertCommaFormTemp(&constVector, typeHnd);
1870
1871         // op1 = op1 - constVector
1872         // op2 = op2 - constVector
1873         op1 = gtNewSIMDNode(simdType, op1, constVector, operIntrinsic, baseType, size);
1874         op2 = gtNewSIMDNode(simdType, op2, tmp, operIntrinsic, baseType, size);
1875
1876         // compute min/max of op1 and op2 considering them as if minMaxOperBaseType
1877         simdTree = gtNewSIMDNode(simdType, op1, op2, intrinsicId, minMaxOperBaseType, size);
1878
1879         // re-adjust the value by adding or subtracting constVector
1880         tmp      = gtNewLclvNode(tmp->AsLclVarCommon()->GetLclNum(), tmp->TypeGet());
1881         simdTree = gtNewSIMDNode(simdType, simdTree, tmp, adjustIntrinsic, baseType, size);
1882     }
1883 #elif defined(_TARGET_ARM64_)
1884     // Arm64 has direct support for all types except int64/uint64
1885     // For which we compute min/max as follows
1886     //
1887     // int64/uint64
1888     //       compResult        = (op1 < op2) in case of Min
1889     //                           (op1 > op2) in case of Max
1890     //       Min/Max(op1, op2) = Select(compResult, op1, op2)
1891     if (baseType != TYP_ULONG && baseType != TYP_LONG)
1892     {
1893         simdTree = gtNewSIMDNode(simdType, op1, op2, intrinsicId, baseType, size);
1894     }
1895 #endif
1896     else
1897     {
1898         GenTree* dupOp1    = nullptr;
1899         GenTree* dupOp2    = nullptr;
1900         GenTree* op1Assign = nullptr;
1901         GenTree* op2Assign = nullptr;
1902         unsigned op1LclNum;
1903         unsigned op2LclNum;
1904
1905         if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0)
1906         {
1907             op1LclNum = lvaGrabTemp(true DEBUGARG("SIMD Min/Max"));
1908             dupOp1    = gtNewLclvNode(op1LclNum, op1->TypeGet());
1909             lvaSetStruct(op1LclNum, typeHnd, false);
1910             op1Assign = gtNewTempAssign(op1LclNum, op1);
1911             op1       = gtNewLclvNode(op1LclNum, op1->TypeGet());
1912         }
1913         else
1914         {
1915             dupOp1 = gtCloneExpr(op1);
1916         }
1917
1918         if ((op2->gtFlags & GTF_SIDE_EFFECT) != 0)
1919         {
1920             op2LclNum = lvaGrabTemp(true DEBUGARG("SIMD Min/Max"));
1921             dupOp2    = gtNewLclvNode(op2LclNum, op2->TypeGet());
1922             lvaSetStruct(op2LclNum, typeHnd, false);
1923             op2Assign = gtNewTempAssign(op2LclNum, op2);
1924             op2       = gtNewLclvNode(op2LclNum, op2->TypeGet());
1925         }
1926         else
1927         {
1928             dupOp2 = gtCloneExpr(op2);
1929         }
1930
1931         SIMDIntrinsicID relOpIntrinsic =
1932             (intrinsicId == SIMDIntrinsicMin) ? SIMDIntrinsicLessThan : SIMDIntrinsicGreaterThan;
1933         var_types relOpBaseType = baseType;
1934
1935         // compResult = op1 relOp op2
1936         // simdTree = Select(compResult, op1, op2);
1937         assert(dupOp1 != nullptr);
1938         assert(dupOp2 != nullptr);
1939         relOpIntrinsic            = impSIMDRelOp(relOpIntrinsic, typeHnd, size, &relOpBaseType, &dupOp1, &dupOp2);
1940         GenTree* compResult       = gtNewSIMDNode(simdType, dupOp1, dupOp2, relOpIntrinsic, relOpBaseType, size);
1941         unsigned compResultLclNum = lvaGrabTemp(true DEBUGARG("SIMD Min/Max"));
1942         lvaSetStruct(compResultLclNum, typeHnd, false);
1943         GenTree* compResultAssign = gtNewTempAssign(compResultLclNum, compResult);
1944         compResult                = gtNewLclvNode(compResultLclNum, compResult->TypeGet());
1945         simdTree                  = impSIMDSelect(typeHnd, baseType, size, compResult, op1, op2);
1946         simdTree                  = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), compResultAssign, simdTree);
1947
1948         // Now create comma trees if we have created assignments of op1/op2 to temps
1949         if (op2Assign != nullptr)
1950         {
1951             simdTree = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), op2Assign, simdTree);
1952         }
1953
1954         if (op1Assign != nullptr)
1955         {
1956             simdTree = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), op1Assign, simdTree);
1957         }
1958     }
1959
1960     assert(simdTree != nullptr);
1961     return simdTree;
1962 #else  // !(defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_))
1963     assert(!"impSIMDMinMax() unimplemented on target arch");
1964     unreached();
1965 #endif // !(defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_))
1966 }
1967
1968 //------------------------------------------------------------------------
1969 // getOp1ForConstructor: Get the op1 for a constructor call.
1970 //
1971 // Arguments:
1972 //    opcode     - the opcode being handled (needed to identify the CEE_NEWOBJ case)
1973 //    newobjThis - For CEE_NEWOBJ, this is the temp grabbed for the allocated uninitalized object.
1974 //    clsHnd    - The handle of the class of the method.
1975 //
1976 // Return Value:
1977 //    The tree node representing the object to be initialized with the constructor.
1978 //
1979 // Notes:
1980 //    This method handles the differences between the CEE_NEWOBJ and constructor cases.
1981 //
1982 GenTreePtr Compiler::getOp1ForConstructor(OPCODE opcode, GenTreePtr newobjThis, CORINFO_CLASS_HANDLE clsHnd)
1983 {
1984     GenTree* op1;
1985     if (opcode == CEE_NEWOBJ)
1986     {
1987         op1 = newobjThis;
1988         assert(newobjThis->gtOper == GT_ADDR && newobjThis->gtOp.gtOp1->gtOper == GT_LCL_VAR);
1989
1990         // push newobj result on type stack
1991         unsigned tmp = op1->gtOp.gtOp1->gtLclVarCommon.gtLclNum;
1992         impPushOnStack(gtNewLclvNode(tmp, lvaGetRealType(tmp)), verMakeTypeInfo(clsHnd).NormaliseForStack());
1993     }
1994     else
1995     {
1996         op1 = impSIMDPopStack(TYP_BYREF);
1997     }
1998     assert(op1->TypeGet() == TYP_BYREF);
1999     return op1;
2000 }
2001
2002 //-------------------------------------------------------------------
2003 // Set the flag that indicates that the lclVar referenced by this tree
2004 // is used in a SIMD intrinsic.
2005 // Arguments:
2006 //      tree - GenTreePtr
2007
2008 void Compiler::setLclRelatedToSIMDIntrinsic(GenTreePtr tree)
2009 {
2010     assert(tree->OperIsLocal());
2011     unsigned   lclNum                = tree->AsLclVarCommon()->GetLclNum();
2012     LclVarDsc* lclVarDsc             = &lvaTable[lclNum];
2013     lclVarDsc->lvUsedInSIMDIntrinsic = true;
2014 }
2015
2016 //-------------------------------------------------------------
2017 // Check if two field nodes reference at the same memory location.
2018 // Notice that this check is just based on pattern matching.
2019 // Arguments:
2020 //      op1 - GenTreePtr.
2021 //      op2 - GenTreePtr.
2022 // Return Value:
2023 //    If op1's parents node and op2's parents node are at the same location, return true. Otherwise, return false
2024
2025 bool areFieldsParentsLocatedSame(GenTreePtr op1, GenTreePtr op2)
2026 {
2027     assert(op1->OperGet() == GT_FIELD);
2028     assert(op2->OperGet() == GT_FIELD);
2029
2030     GenTreePtr op1ObjRef = op1->gtField.gtFldObj;
2031     GenTreePtr op2ObjRef = op2->gtField.gtFldObj;
2032     while (op1ObjRef != nullptr && op2ObjRef != nullptr)
2033     {
2034
2035         if (op1ObjRef->OperGet() != op2ObjRef->OperGet())
2036         {
2037             break;
2038         }
2039         else if (op1ObjRef->OperGet() == GT_ADDR)
2040         {
2041             op1ObjRef = op1ObjRef->gtOp.gtOp1;
2042             op2ObjRef = op2ObjRef->gtOp.gtOp1;
2043         }
2044
2045         if (op1ObjRef->OperIsLocal() && op2ObjRef->OperIsLocal() &&
2046             op1ObjRef->AsLclVarCommon()->GetLclNum() == op2ObjRef->AsLclVarCommon()->GetLclNum())
2047         {
2048             return true;
2049         }
2050         else if (op1ObjRef->OperGet() == GT_FIELD && op2ObjRef->OperGet() == GT_FIELD &&
2051                  op1ObjRef->gtField.gtFldHnd == op2ObjRef->gtField.gtFldHnd)
2052         {
2053             op1ObjRef = op1ObjRef->gtField.gtFldObj;
2054             op2ObjRef = op2ObjRef->gtField.gtFldObj;
2055             continue;
2056         }
2057         else
2058         {
2059             break;
2060         }
2061     }
2062
2063     return false;
2064 }
2065
2066 //----------------------------------------------------------------------
2067 // Check whether two field are contiguous
2068 // Arguments:
2069 //      first - GenTreePtr. The Type of the node should be TYP_FLOAT
2070 //      second - GenTreePtr. The Type of the node should be TYP_FLOAT
2071 // Return Value:
2072 //      if the first field is located before second field, and they are located contiguously,
2073 //      then return true. Otherwise, return false.
2074
2075 bool Compiler::areFieldsContiguous(GenTreePtr first, GenTreePtr second)
2076 {
2077     assert(first->OperGet() == GT_FIELD);
2078     assert(second->OperGet() == GT_FIELD);
2079     assert(first->gtType == TYP_FLOAT);
2080     assert(second->gtType == TYP_FLOAT);
2081
2082     var_types firstFieldType  = first->gtType;
2083     var_types secondFieldType = second->gtType;
2084
2085     unsigned firstFieldEndOffset = first->gtField.gtFldOffset + genTypeSize(firstFieldType);
2086     unsigned secondFieldOffset   = second->gtField.gtFldOffset;
2087     if (firstFieldEndOffset == secondFieldOffset && firstFieldType == secondFieldType &&
2088         areFieldsParentsLocatedSame(first, second))
2089     {
2090         return true;
2091     }
2092
2093     return false;
2094 }
2095
2096 //-------------------------------------------------------------------------------
2097 // Check whether two array element nodes are located contiguously or not.
2098 // Arguments:
2099 //      op1 - GenTreePtr.
2100 //      op2 - GenTreePtr.
2101 // Return Value:
2102 //      if the array element op1 is located before array element op2, and they are contiguous,
2103 //      then return true. Otherwise, return false.
2104 // TODO-CQ:
2105 //      Right this can only check array element with const number as index. In future,
2106 //      we should consider to allow this function to check the index using expression.
2107
2108 bool Compiler::areArrayElementsContiguous(GenTreePtr op1, GenTreePtr op2)
2109 {
2110     noway_assert(op1->gtOper == GT_INDEX);
2111     noway_assert(op2->gtOper == GT_INDEX);
2112     GenTreeIndex* op1Index = op1->AsIndex();
2113     GenTreeIndex* op2Index = op2->AsIndex();
2114
2115     GenTreePtr op1ArrayRef = op1Index->Arr();
2116     GenTreePtr op2ArrayRef = op2Index->Arr();
2117     assert(op1ArrayRef->TypeGet() == TYP_REF);
2118     assert(op2ArrayRef->TypeGet() == TYP_REF);
2119
2120     GenTreePtr op1IndexNode = op1Index->Index();
2121     GenTreePtr op2IndexNode = op2Index->Index();
2122     if ((op1IndexNode->OperGet() == GT_CNS_INT && op2IndexNode->OperGet() == GT_CNS_INT) &&
2123         op1IndexNode->gtIntCon.gtIconVal + 1 == op2IndexNode->gtIntCon.gtIconVal)
2124     {
2125         if (op1ArrayRef->OperGet() == GT_FIELD && op2ArrayRef->OperGet() == GT_FIELD &&
2126             areFieldsParentsLocatedSame(op1ArrayRef, op2ArrayRef))
2127         {
2128             return true;
2129         }
2130         else if (op1ArrayRef->OperIsLocal() && op2ArrayRef->OperIsLocal() &&
2131                  op1ArrayRef->AsLclVarCommon()->GetLclNum() == op2ArrayRef->AsLclVarCommon()->GetLclNum())
2132         {
2133             return true;
2134         }
2135     }
2136     return false;
2137 }
2138
2139 //-------------------------------------------------------------------------------
2140 // Check whether two argument nodes are contiguous or not.
2141 // Arguments:
2142 //      op1 - GenTreePtr.
2143 //      op2 - GenTreePtr.
2144 // Return Value:
2145 //      if the argument node op1 is located before argument node op2, and they are located contiguously,
2146 //      then return true. Otherwise, return false.
2147 // TODO-CQ:
2148 //      Right now this can only check field and array. In future we should add more cases.
2149 //
2150
2151 bool Compiler::areArgumentsContiguous(GenTreePtr op1, GenTreePtr op2)
2152 {
2153     if (op1->OperGet() == GT_INDEX && op2->OperGet() == GT_INDEX)
2154     {
2155         return areArrayElementsContiguous(op1, op2);
2156     }
2157     else if (op1->OperGet() == GT_FIELD && op2->OperGet() == GT_FIELD)
2158     {
2159         return areFieldsContiguous(op1, op2);
2160     }
2161     return false;
2162 }
2163
2164 //--------------------------------------------------------------------------------------------------------
2165 // createAddressNodeForSIMDInit: Generate the address node(GT_LEA) if we want to intialize vector2, vector3 or vector4
2166 // from first argument's address.
2167 //
2168 // Arguments:
2169 //      tree - GenTreePtr. This the tree node which is used to get the address for indir.
2170 //      simdsize - unsigned. This the simd vector size.
2171 //      arrayElementsCount - unsigned. This is used for generating the boundary check for array.
2172 //
2173 // Return value:
2174 //      return the address node.
2175 //
2176 // TODO-CQ:
2177 //      1. Currently just support for GT_FIELD and GT_INDEX, because we can only verify the GT_INDEX node or GT_Field
2178 //         are located contiguously or not. In future we should support more cases.
2179 //      2. Though it happens to just work fine front-end phases are not aware of GT_LEA node.  Therefore, convert these
2180 //         to use GT_ADDR.
2181 GenTreePtr Compiler::createAddressNodeForSIMDInit(GenTreePtr tree, unsigned simdSize)
2182 {
2183     assert(tree->OperGet() == GT_FIELD || tree->OperGet() == GT_INDEX);
2184     GenTreePtr byrefNode  = nullptr;
2185     GenTreePtr startIndex = nullptr;
2186     unsigned   offset     = 0;
2187     var_types  baseType   = tree->gtType;
2188
2189     if (tree->OperGet() == GT_FIELD)
2190     {
2191         GenTreePtr objRef = tree->gtField.gtFldObj;
2192         if (objRef != nullptr && objRef->gtOper == GT_ADDR)
2193         {
2194             GenTreePtr obj = objRef->gtOp.gtOp1;
2195
2196             // If the field is directly from a struct, then in this case,
2197             // we should set this struct's lvUsedInSIMDIntrinsic as true,
2198             // so that this sturct won't be promoted.
2199             // e.g. s.x x is a field, and s is a struct, then we should set the s's lvUsedInSIMDIntrinsic as true.
2200             // so that s won't be promoted.
2201             // Notice that if we have a case like s1.s2.x. s1 s2 are struct, and x is a field, then it is possible that
2202             // s1 can be promoted, so that s2 can be promoted. The reason for that is if we don't allow s1 to be
2203             // promoted, then this will affect the other optimizations which are depend on s1's struct promotion.
2204             // TODO-CQ:
2205             //  In future, we should optimize this case so that if there is a nested field like s1.s2.x and s1.s2.x's
2206             //  address is used for initializing the vector, then s1 can be promoted but s2 can't.
2207             if (varTypeIsSIMD(obj) && obj->OperIsLocal())
2208             {
2209                 setLclRelatedToSIMDIntrinsic(obj);
2210             }
2211         }
2212
2213         byrefNode = gtCloneExpr(tree->gtField.gtFldObj);
2214         assert(byrefNode != nullptr);
2215         offset = tree->gtField.gtFldOffset;
2216     }
2217     else if (tree->OperGet() == GT_INDEX)
2218     {
2219
2220         GenTreePtr index = tree->AsIndex()->Index();
2221         assert(index->OperGet() == GT_CNS_INT);
2222
2223         GenTreePtr checkIndexExpr = nullptr;
2224         unsigned   indexVal       = (unsigned)(index->gtIntCon.gtIconVal);
2225         offset                    = indexVal * genTypeSize(tree->TypeGet());
2226         GenTreePtr arrayRef       = tree->AsIndex()->Arr();
2227
2228         // Generate the boundary check exception.
2229         // The length for boundary check should be the maximum index number which should be
2230         // (first argument's index number) + (how many array arguments we have) - 1
2231         // = indexVal + arrayElementsCount - 1
2232         unsigned arrayElementsCount  = simdSize / genTypeSize(baseType);
2233         checkIndexExpr               = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, indexVal + arrayElementsCount - 1);
2234         GenTreeArrLen*    arrLen     = gtNewArrLen(TYP_INT, arrayRef, (int)offsetof(CORINFO_Array, length));
2235         GenTreeBoundsChk* arrBndsChk = new (this, GT_ARR_BOUNDS_CHECK)
2236             GenTreeBoundsChk(GT_ARR_BOUNDS_CHECK, TYP_VOID, checkIndexExpr, arrLen, SCK_RNGCHK_FAIL);
2237
2238         offset += offsetof(CORINFO_Array, u1Elems);
2239         byrefNode = gtNewOperNode(GT_COMMA, arrayRef->TypeGet(), arrBndsChk, gtCloneExpr(arrayRef));
2240     }
2241     else
2242     {
2243         unreached();
2244     }
2245     GenTreePtr address =
2246         new (this, GT_LEA) GenTreeAddrMode(TYP_BYREF, byrefNode, startIndex, genTypeSize(tree->TypeGet()), offset);
2247     return address;
2248 }
2249
2250 //-------------------------------------------------------------------------------
2251 // impMarkContiguousSIMDFieldAssignments: Try to identify if there are contiguous
2252 // assignments from SIMD field to memory. If there are, then mark the related
2253 // lclvar so that it won't be promoted.
2254 //
2255 // Arguments:
2256 //      stmt - GenTreePtr. Input statement node.
2257
2258 void Compiler::impMarkContiguousSIMDFieldAssignments(GenTreePtr stmt)
2259 {
2260     if (!featureSIMD || opts.MinOpts())
2261     {
2262         return;
2263     }
2264     GenTreePtr expr = stmt->gtStmt.gtStmtExpr;
2265     if (expr->OperGet() == GT_ASG && expr->TypeGet() == TYP_FLOAT)
2266     {
2267         GenTreePtr curDst            = expr->gtOp.gtOp1;
2268         GenTreePtr curSrc            = expr->gtOp.gtOp2;
2269         unsigned   index             = 0;
2270         var_types  baseType          = TYP_UNKNOWN;
2271         unsigned   simdSize          = 0;
2272         GenTreePtr srcSimdStructNode = getSIMDStructFromField(curSrc, &baseType, &index, &simdSize, true);
2273         if (srcSimdStructNode == nullptr || baseType != TYP_FLOAT)
2274         {
2275             fgPreviousCandidateSIMDFieldAsgStmt = nullptr;
2276         }
2277         else if (index == 0 && isSIMDTypeLocal(srcSimdStructNode))
2278         {
2279             fgPreviousCandidateSIMDFieldAsgStmt = stmt;
2280         }
2281         else if (fgPreviousCandidateSIMDFieldAsgStmt != nullptr)
2282         {
2283             assert(index > 0);
2284             GenTreePtr prevAsgExpr = fgPreviousCandidateSIMDFieldAsgStmt->gtStmt.gtStmtExpr;
2285             GenTreePtr prevDst     = prevAsgExpr->gtOp.gtOp1;
2286             GenTreePtr prevSrc     = prevAsgExpr->gtOp.gtOp2;
2287             if (!areArgumentsContiguous(prevDst, curDst) || !areArgumentsContiguous(prevSrc, curSrc))
2288             {
2289                 fgPreviousCandidateSIMDFieldAsgStmt = nullptr;
2290             }
2291             else
2292             {
2293                 if (index == (simdSize / genTypeSize(baseType) - 1))
2294                 {
2295                     // Successfully found the pattern, mark the lclvar as UsedInSIMDIntrinsic
2296                     if (srcSimdStructNode->OperIsLocal())
2297                     {
2298                         setLclRelatedToSIMDIntrinsic(srcSimdStructNode);
2299                     }
2300
2301                     if (curDst->OperGet() == GT_FIELD)
2302                     {
2303                         GenTreePtr objRef = curDst->gtField.gtFldObj;
2304                         if (objRef != nullptr && objRef->gtOper == GT_ADDR)
2305                         {
2306                             GenTreePtr obj = objRef->gtOp.gtOp1;
2307                             if (varTypeIsStruct(obj) && obj->OperIsLocal())
2308                             {
2309                                 setLclRelatedToSIMDIntrinsic(obj);
2310                             }
2311                         }
2312                     }
2313                 }
2314                 else
2315                 {
2316                     fgPreviousCandidateSIMDFieldAsgStmt = stmt;
2317                 }
2318             }
2319         }
2320     }
2321     else
2322     {
2323         fgPreviousCandidateSIMDFieldAsgStmt = nullptr;
2324     }
2325 }
2326
2327 //------------------------------------------------------------------------
2328 // impSIMDIntrinsic: Check method to see if it is a SIMD method
2329 //
2330 // Arguments:
2331 //    opcode     - the opcode being handled (needed to identify the CEE_NEWOBJ case)
2332 //    newobjThis - For CEE_NEWOBJ, this is the temp grabbed for the allocated uninitalized object.
2333 //    clsHnd     - The handle of the class of the method.
2334 //    method     - The handle of the method.
2335 //    sig        - The call signature for the method.
2336 //    memberRef  - The memberRef token for the method reference.
2337 //
2338 // Return Value:
2339 //    If clsHnd is a known SIMD type, and 'method' is one of the methods that are
2340 //    implemented as an intrinsic in the JIT, then return the tree that implements
2341 //    it.
2342 //
2343 GenTreePtr Compiler::impSIMDIntrinsic(OPCODE                opcode,
2344                                       GenTreePtr            newobjThis,
2345                                       CORINFO_CLASS_HANDLE  clsHnd,
2346                                       CORINFO_METHOD_HANDLE methodHnd,
2347                                       CORINFO_SIG_INFO*     sig,
2348                                       int                   memberRef)
2349 {
2350     assert(featureSIMD);
2351
2352     if (!isSIMDClass(clsHnd))
2353     {
2354         return nullptr;
2355     }
2356
2357     // Get base type and intrinsic Id
2358     var_types                baseType = TYP_UNKNOWN;
2359     unsigned                 size     = 0;
2360     unsigned                 argCount = 0;
2361     const SIMDIntrinsicInfo* intrinsicInfo =
2362         getSIMDIntrinsicInfo(&clsHnd, methodHnd, sig, (opcode == CEE_NEWOBJ), &argCount, &baseType, &size);
2363     if (intrinsicInfo == nullptr || intrinsicInfo->id == SIMDIntrinsicInvalid)
2364     {
2365         return nullptr;
2366     }
2367
2368     SIMDIntrinsicID simdIntrinsicID = intrinsicInfo->id;
2369     var_types       simdType;
2370     if (baseType != TYP_UNKNOWN)
2371     {
2372         simdType = getSIMDTypeForSize(size);
2373     }
2374     else
2375     {
2376         assert(simdIntrinsicID == SIMDIntrinsicHWAccel);
2377         simdType = TYP_UNKNOWN;
2378     }
2379     bool      instMethod = intrinsicInfo->isInstMethod;
2380     var_types callType   = JITtype2varType(sig->retType);
2381     if (callType == TYP_STRUCT)
2382     {
2383         // Note that here we are assuming that, if the call returns a struct, that it is the same size as the
2384         // struct on which the method is declared. This is currently true for all methods on Vector types,
2385         // but if this ever changes, we will need to determine the callType from the signature.
2386         assert(info.compCompHnd->getClassSize(sig->retTypeClass) == genTypeSize(simdType));
2387         callType = simdType;
2388     }
2389
2390     GenTree* simdTree   = nullptr;
2391     GenTree* op1        = nullptr;
2392     GenTree* op2        = nullptr;
2393     GenTree* op3        = nullptr;
2394     GenTree* retVal     = nullptr;
2395     GenTree* copyBlkDst = nullptr;
2396     bool     doCopyBlk  = false;
2397
2398     switch (simdIntrinsicID)
2399     {
2400         case SIMDIntrinsicGetCount:
2401         {
2402             int            length       = getSIMDVectorLength(clsHnd);
2403             GenTreeIntCon* intConstTree = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, length);
2404             retVal                      = intConstTree;
2405
2406             intConstTree->gtFlags |= GTF_ICON_SIMD_COUNT;
2407         }
2408         break;
2409
2410         case SIMDIntrinsicGetZero:
2411             retVal = gtNewSIMDVectorZero(simdType, baseType, size);
2412             break;
2413
2414         case SIMDIntrinsicGetOne:
2415             retVal = gtNewSIMDVectorOne(simdType, baseType, size);
2416             break;
2417
2418         case SIMDIntrinsicGetAllOnes:
2419         {
2420             // Equivalent to (Vector<T>) new Vector<int>(0xffffffff);
2421             GenTree* initVal = gtNewIconNode(0xffffffff, TYP_INT);
2422             simdTree         = gtNewSIMDNode(simdType, initVal, nullptr, SIMDIntrinsicInit, TYP_INT, size);
2423             if (baseType != TYP_INT)
2424             {
2425                 // cast it to required baseType if different from TYP_INT
2426                 simdTree = gtNewSIMDNode(simdType, simdTree, nullptr, SIMDIntrinsicCast, baseType, size);
2427             }
2428             retVal = simdTree;
2429         }
2430         break;
2431
2432         case SIMDIntrinsicInit:
2433         case SIMDIntrinsicInitN:
2434         {
2435             // SIMDIntrinsicInit:
2436             //    op2 - the initializer value
2437             //    op1 - byref of vector
2438             //
2439             // SIMDIntrinsicInitN
2440             //    op2 - list of initializer values stitched into a list
2441             //    op1 - byref of vector
2442             bool initFromFirstArgIndir = false;
2443             if (simdIntrinsicID == SIMDIntrinsicInit)
2444             {
2445                 op2 = impSIMDPopStack(baseType);
2446             }
2447             else
2448             {
2449                 assert(simdIntrinsicID == SIMDIntrinsicInitN);
2450                 assert(baseType == TYP_FLOAT);
2451
2452                 unsigned initCount    = argCount - 1;
2453                 unsigned elementCount = getSIMDVectorLength(size, baseType);
2454                 noway_assert(initCount == elementCount);
2455                 GenTree* nextArg = op2;
2456
2457                 // Build a GT_LIST with the N values.
2458                 // We must maintain left-to-right order of the args, but we will pop
2459                 // them off in reverse order (the Nth arg was pushed onto the stack last).
2460
2461                 GenTree*   list              = nullptr;
2462                 GenTreePtr firstArg          = nullptr;
2463                 GenTreePtr prevArg           = nullptr;
2464                 int        offset            = 0;
2465                 bool       areArgsContiguous = true;
2466                 for (unsigned i = 0; i < initCount; i++)
2467                 {
2468                     GenTree* nextArg = impSIMDPopStack(baseType);
2469                     if (areArgsContiguous)
2470                     {
2471                         GenTreePtr curArg = nextArg;
2472                         firstArg          = curArg;
2473
2474                         if (prevArg != nullptr)
2475                         {
2476                             // Recall that we are popping the args off the stack in reverse order.
2477                             areArgsContiguous = areArgumentsContiguous(curArg, prevArg);
2478                         }
2479                         prevArg = curArg;
2480                     }
2481
2482                     list = new (this, GT_LIST) GenTreeOp(GT_LIST, baseType, nextArg, list);
2483                 }
2484
2485                 if (areArgsContiguous && baseType == TYP_FLOAT)
2486                 {
2487                     // Since Vector2, Vector3 and Vector4's arguments type are only float,
2488                     // we intialize the vector from first argument address, only when
2489                     // the baseType is TYP_FLOAT and the arguments are located contiguously in memory
2490                     initFromFirstArgIndir = true;
2491                     GenTreePtr op2Address = createAddressNodeForSIMDInit(firstArg, size);
2492                     var_types  simdType   = getSIMDTypeForSize(size);
2493                     op2                   = gtNewOperNode(GT_IND, simdType, op2Address);
2494                 }
2495                 else
2496                 {
2497                     op2 = list;
2498                 }
2499             }
2500
2501             op1 = getOp1ForConstructor(opcode, newobjThis, clsHnd);
2502
2503             assert(op1->TypeGet() == TYP_BYREF);
2504             assert(genActualType(op2->TypeGet()) == genActualType(baseType) || initFromFirstArgIndir);
2505
2506             // For integral base types of size less than TYP_INT, expand the initializer
2507             // to fill size of TYP_INT bytes.
2508             if (varTypeIsSmallInt(baseType))
2509             {
2510                 // This case should occur only for Init intrinsic.
2511                 assert(simdIntrinsicID == SIMDIntrinsicInit);
2512
2513                 unsigned baseSize = genTypeSize(baseType);
2514                 int      multiplier;
2515                 if (baseSize == 1)
2516                 {
2517                     multiplier = 0x01010101;
2518                 }
2519                 else
2520                 {
2521                     assert(baseSize == 2);
2522                     multiplier = 0x00010001;
2523                 }
2524
2525                 GenTree* t1 = nullptr;
2526                 if (baseType == TYP_BYTE)
2527                 {
2528                     // What we have is a signed byte initializer,
2529                     // which when loaded to a reg will get sign extended to TYP_INT.
2530                     // But what we need is the initializer without sign extended or
2531                     // rather zero extended to 32-bits.
2532                     t1 = gtNewOperNode(GT_AND, TYP_INT, op2, gtNewIconNode(0xff, TYP_INT));
2533                 }
2534                 else if (baseType == TYP_SHORT)
2535                 {
2536                     // What we have is a signed short initializer,
2537                     // which when loaded to a reg will get sign extended to TYP_INT.
2538                     // But what we need is the initializer without sign extended or
2539                     // rather zero extended to 32-bits.
2540                     t1 = gtNewOperNode(GT_AND, TYP_INT, op2, gtNewIconNode(0xffff, TYP_INT));
2541                 }
2542                 else
2543                 {
2544                     assert(baseType == TYP_UBYTE || baseType == TYP_USHORT);
2545                     t1 = gtNewCastNode(TYP_INT, op2, TYP_INT);
2546                 }
2547
2548                 assert(t1 != nullptr);
2549                 GenTree* t2 = gtNewIconNode(multiplier, TYP_INT);
2550                 op2         = gtNewOperNode(GT_MUL, TYP_INT, t1, t2);
2551
2552                 // Construct a vector of TYP_INT with the new initializer and cast it back to vector of baseType
2553                 simdTree = gtNewSIMDNode(simdType, op2, nullptr, simdIntrinsicID, TYP_INT, size);
2554                 simdTree = gtNewSIMDNode(simdType, simdTree, nullptr, SIMDIntrinsicCast, baseType, size);
2555             }
2556             else
2557             {
2558
2559                 if (initFromFirstArgIndir)
2560                 {
2561                     simdTree = op2;
2562                     if (op1->gtOp.gtOp1->OperIsLocal())
2563                     {
2564                         // label the dst struct's lclvar is used for SIMD intrinsic,
2565                         // so that this dst struct won't be promoted.
2566                         setLclRelatedToSIMDIntrinsic(op1->gtOp.gtOp1);
2567                     }
2568                 }
2569                 else
2570                 {
2571                     simdTree = gtNewSIMDNode(simdType, op2, nullptr, simdIntrinsicID, baseType, size);
2572                 }
2573             }
2574
2575             copyBlkDst = op1;
2576             doCopyBlk  = true;
2577         }
2578         break;
2579
2580         case SIMDIntrinsicInitArray:
2581         case SIMDIntrinsicInitArrayX:
2582         case SIMDIntrinsicCopyToArray:
2583         case SIMDIntrinsicCopyToArrayX:
2584         {
2585             // op3 - index into array in case of SIMDIntrinsicCopyToArrayX and SIMDIntrinsicInitArrayX
2586             // op2 - array itself
2587             // op1 - byref to vector struct
2588
2589             unsigned int vectorLength = getSIMDVectorLength(size, baseType);
2590             // (This constructor takes only the zero-based arrays.)
2591             // We will add one or two bounds checks:
2592             // 1. If we have an index, we must do a check on that first.
2593             //    We can't combine it with the index + vectorLength check because
2594             //    a. It might be negative, and b. It may need to raise a different exception
2595             //    (captured as SCK_ARG_RNG_EXCPN for CopyTo and SCK_RNGCHK_FAIL for Init).
2596             // 2. We need to generate a check (SCK_ARG_EXCPN for CopyTo and SCK_RNGCHK_FAIL for Init)
2597             //    for the last array element we will access.
2598             //    We'll either check against (vectorLength - 1) or (index + vectorLength - 1).
2599
2600             GenTree* checkIndexExpr = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, vectorLength - 1);
2601
2602             // Get the index into the array.  If it has been provided, it will be on the
2603             // top of the stack.  Otherwise, it is null.
2604             if (argCount == 3)
2605             {
2606                 op3 = impSIMDPopStack(TYP_INT);
2607                 if (op3->IsIntegralConst(0))
2608                 {
2609                     op3 = nullptr;
2610                 }
2611             }
2612             else
2613             {
2614                 // TODO-CQ: Here, or elsewhere, check for the pattern where op2 is a newly constructed array, and
2615                 // change this to the InitN form.
2616                 // op3 = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, 0);
2617                 op3 = nullptr;
2618             }
2619
2620             // Clone the array for use in the bounds check.
2621             op2 = impSIMDPopStack(TYP_REF);
2622             assert(op2->TypeGet() == TYP_REF);
2623             GenTree* arrayRefForArgChk = op2;
2624             GenTree* argRngChk         = nullptr;
2625             GenTree* asg               = nullptr;
2626             if ((arrayRefForArgChk->gtFlags & GTF_SIDE_EFFECT) != 0)
2627             {
2628                 op2 = fgInsertCommaFormTemp(&arrayRefForArgChk);
2629             }
2630             else
2631             {
2632                 op2 = gtCloneExpr(arrayRefForArgChk);
2633             }
2634             assert(op2 != nullptr);
2635
2636             if (op3 != nullptr)
2637             {
2638                 SpecialCodeKind op3CheckKind;
2639                 if (simdIntrinsicID == SIMDIntrinsicInitArrayX)
2640                 {
2641                     op3CheckKind = SCK_RNGCHK_FAIL;
2642                 }
2643                 else
2644                 {
2645                     assert(simdIntrinsicID == SIMDIntrinsicCopyToArrayX);
2646                     op3CheckKind = SCK_ARG_RNG_EXCPN;
2647                 }
2648                 // We need to use the original expression on this, which is the first check.
2649                 GenTree* arrayRefForArgRngChk = arrayRefForArgChk;
2650                 // Then we clone the clone we just made for the next check.
2651                 arrayRefForArgChk = gtCloneExpr(op2);
2652                 // We know we MUST have had a cloneable expression.
2653                 assert(arrayRefForArgChk != nullptr);
2654                 GenTree* index = op3;
2655                 if ((index->gtFlags & GTF_SIDE_EFFECT) != 0)
2656                 {
2657                     op3 = fgInsertCommaFormTemp(&index);
2658                 }
2659                 else
2660                 {
2661                     op3 = gtCloneExpr(index);
2662                 }
2663
2664                 GenTreeArrLen* arrLen =
2665                     gtNewArrLen(TYP_INT, arrayRefForArgRngChk, (int)offsetof(CORINFO_Array, length));
2666                 argRngChk = new (this, GT_ARR_BOUNDS_CHECK)
2667                     GenTreeBoundsChk(GT_ARR_BOUNDS_CHECK, TYP_VOID, index, arrLen, op3CheckKind);
2668                 // Now, clone op3 to create another node for the argChk
2669                 GenTree* index2 = gtCloneExpr(op3);
2670                 assert(index != nullptr);
2671                 checkIndexExpr = gtNewOperNode(GT_ADD, TYP_INT, index2, checkIndexExpr);
2672             }
2673
2674             // Insert a bounds check for index + offset - 1.
2675             // This must be a "normal" array.
2676             SpecialCodeKind op2CheckKind;
2677             if (simdIntrinsicID == SIMDIntrinsicInitArray || simdIntrinsicID == SIMDIntrinsicInitArrayX)
2678             {
2679                 op2CheckKind = SCK_RNGCHK_FAIL;
2680             }
2681             else
2682             {
2683                 op2CheckKind = SCK_ARG_EXCPN;
2684             }
2685             GenTreeArrLen*    arrLen = gtNewArrLen(TYP_INT, arrayRefForArgChk, (int)offsetof(CORINFO_Array, length));
2686             GenTreeBoundsChk* argChk = new (this, GT_ARR_BOUNDS_CHECK)
2687                 GenTreeBoundsChk(GT_ARR_BOUNDS_CHECK, TYP_VOID, checkIndexExpr, arrLen, op2CheckKind);
2688
2689             // Create a GT_COMMA tree for the bounds check(s).
2690             op2 = gtNewOperNode(GT_COMMA, op2->TypeGet(), argChk, op2);
2691             if (argRngChk != nullptr)
2692             {
2693                 op2 = gtNewOperNode(GT_COMMA, op2->TypeGet(), argRngChk, op2);
2694             }
2695
2696             if (simdIntrinsicID == SIMDIntrinsicInitArray || simdIntrinsicID == SIMDIntrinsicInitArrayX)
2697             {
2698                 op1        = getOp1ForConstructor(opcode, newobjThis, clsHnd);
2699                 simdTree   = gtNewSIMDNode(simdType, op2, op3, SIMDIntrinsicInitArray, baseType, size);
2700                 copyBlkDst = op1;
2701                 doCopyBlk  = true;
2702             }
2703             else
2704             {
2705                 assert(simdIntrinsicID == SIMDIntrinsicCopyToArray || simdIntrinsicID == SIMDIntrinsicCopyToArrayX);
2706                 op1 = impSIMDPopStack(simdType, instMethod);
2707                 assert(op1->TypeGet() == simdType);
2708
2709                 // copy vector (op1) to array (op2) starting at index (op3)
2710                 simdTree = op1;
2711
2712                 // TODO-Cleanup: Though it happens to just work fine front-end phases are not aware of GT_LEA node.
2713                 // Therefore, convert these to use GT_ADDR .
2714                 copyBlkDst = new (this, GT_LEA)
2715                     GenTreeAddrMode(TYP_BYREF, op2, op3, genTypeSize(baseType), offsetof(CORINFO_Array, u1Elems));
2716                 doCopyBlk = true;
2717             }
2718         }
2719         break;
2720
2721         case SIMDIntrinsicInitFixed:
2722         {
2723             // We are initializing a fixed-length vector VLarge with a smaller fixed-length vector VSmall, plus 1 or 2
2724             // additional floats.
2725             //    op4 (optional) - float value for VLarge.W, if VLarge is Vector4, and VSmall is Vector2
2726             //    op3 - float value for VLarge.Z or VLarge.W
2727             //    op2 - VSmall
2728             //    op1 - byref of VLarge
2729             assert(baseType == TYP_FLOAT);
2730             unsigned elementByteCount = 4;
2731
2732             GenTree* op4 = nullptr;
2733             if (argCount == 4)
2734             {
2735                 op4 = impSIMDPopStack(TYP_FLOAT);
2736                 assert(op4->TypeGet() == TYP_FLOAT);
2737             }
2738             op3 = impSIMDPopStack(TYP_FLOAT);
2739             assert(op3->TypeGet() == TYP_FLOAT);
2740             // The input vector will either be TYP_SIMD8 or TYP_SIMD12.
2741             var_types smallSIMDType = TYP_SIMD8;
2742             if ((op4 == nullptr) && (simdType == TYP_SIMD16))
2743             {
2744                 smallSIMDType = TYP_SIMD12;
2745             }
2746             op2 = impSIMDPopStack(smallSIMDType);
2747             op1 = getOp1ForConstructor(opcode, newobjThis, clsHnd);
2748
2749             // We are going to redefine the operands so that:
2750             // - op3 is the value that's going into the Z position, or null if it's a Vector4 constructor with a single
2751             // operand, and
2752             // - op4 is the W position value, or null if this is a Vector3 constructor.
2753             if (size == 16 && argCount == 3)
2754             {
2755                 op4 = op3;
2756                 op3 = nullptr;
2757             }
2758
2759             simdTree = op2;
2760             if (op3 != nullptr)
2761             {
2762                 simdTree = gtNewSIMDNode(simdType, simdTree, op3, SIMDIntrinsicSetZ, baseType, size);
2763             }
2764             if (op4 != nullptr)
2765             {
2766                 simdTree = gtNewSIMDNode(simdType, simdTree, op4, SIMDIntrinsicSetW, baseType, size);
2767             }
2768
2769             copyBlkDst = op1;
2770             doCopyBlk  = true;
2771         }
2772         break;
2773
2774         case SIMDIntrinsicOpEquality:
2775         case SIMDIntrinsicInstEquals:
2776         {
2777             op2 = impSIMDPopStack(simdType);
2778             op1 = impSIMDPopStack(simdType, instMethod);
2779
2780             assert(op1->TypeGet() == simdType);
2781             assert(op2->TypeGet() == simdType);
2782
2783             simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, SIMDIntrinsicOpEquality, baseType, size);
2784             if (simdType == TYP_SIMD12)
2785             {
2786                 simdTree->gtFlags |= GTF_SIMD12_OP;
2787             }
2788             retVal = simdTree;
2789         }
2790         break;
2791
2792         case SIMDIntrinsicOpInEquality:
2793         {
2794             // op1 is the first operand
2795             // op2 is the second operand
2796             op2      = impSIMDPopStack(simdType);
2797             op1      = impSIMDPopStack(simdType, instMethod);
2798             simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, SIMDIntrinsicOpInEquality, baseType, size);
2799             if (simdType == TYP_SIMD12)
2800             {
2801                 simdTree->gtFlags |= GTF_SIMD12_OP;
2802             }
2803             retVal = simdTree;
2804         }
2805         break;
2806
2807         case SIMDIntrinsicEqual:
2808         case SIMDIntrinsicLessThan:
2809         case SIMDIntrinsicLessThanOrEqual:
2810         case SIMDIntrinsicGreaterThan:
2811         case SIMDIntrinsicGreaterThanOrEqual:
2812         {
2813             op2 = impSIMDPopStack(simdType);
2814             op1 = impSIMDPopStack(simdType, instMethod);
2815
2816             SIMDIntrinsicID intrinsicID = impSIMDRelOp(simdIntrinsicID, clsHnd, size, &baseType, &op1, &op2);
2817             simdTree                    = gtNewSIMDNode(genActualType(callType), op1, op2, intrinsicID, baseType, size);
2818             retVal                      = simdTree;
2819         }
2820         break;
2821
2822         case SIMDIntrinsicAdd:
2823         case SIMDIntrinsicSub:
2824         case SIMDIntrinsicMul:
2825         case SIMDIntrinsicDiv:
2826         case SIMDIntrinsicBitwiseAnd:
2827         case SIMDIntrinsicBitwiseAndNot:
2828         case SIMDIntrinsicBitwiseOr:
2829         case SIMDIntrinsicBitwiseXor:
2830         {
2831 #if defined(DEBUG)
2832             // check for the cases where we don't support intrinsics.
2833             // This check should be done before we make modifications to type stack.
2834             // Note that this is more of a double safety check for robustness since
2835             // we expect getSIMDIntrinsicInfo() to have filtered out intrinsics on
2836             // unsupported base types. If getSIMdIntrinsicInfo() doesn't filter due
2837             // to some bug, assert in chk/dbg will fire.
2838             if (!varTypeIsFloating(baseType))
2839             {
2840                 if (simdIntrinsicID == SIMDIntrinsicMul)
2841                 {
2842 #if defined(_TARGET_XARCH_)
2843                     if ((baseType != TYP_INT) && (baseType != TYP_SHORT))
2844                     {
2845                         // TODO-CQ: implement mul on these integer vectors.
2846                         // Note that SSE2 has no direct support for these vectors.
2847                         assert(!"Mul not supported on long/ulong/uint/small int vectors\n");
2848                         return nullptr;
2849                     }
2850 #endif // _TARGET_XARCH_
2851 #if defined(_TARGET_ARM64_)
2852                     if ((baseType == TYP_ULONG) && (baseType == TYP_LONG))
2853                     {
2854                         // TODO-CQ: implement mul on these integer vectors.
2855                         // Note that ARM64 has no direct support for these vectors.
2856                         assert(!"Mul not supported on long/ulong vectors\n");
2857                         return nullptr;
2858                     }
2859 #endif // _TARGET_ARM64_
2860                 }
2861 #if defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
2862                 // common to all integer type vectors
2863                 if (simdIntrinsicID == SIMDIntrinsicDiv)
2864                 {
2865                     // SSE2 doesn't support div on non-floating point vectors.
2866                     assert(!"Div not supported on integer type vectors\n");
2867                     return nullptr;
2868                 }
2869 #endif // defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
2870             }
2871 #endif // DEBUG
2872
2873             // op1 is the first operand; if instance method, op1 is "this" arg
2874             // op2 is the second operand
2875             op2 = impSIMDPopStack(simdType);
2876             op1 = impSIMDPopStack(simdType, instMethod);
2877
2878             simdTree = gtNewSIMDNode(simdType, op1, op2, simdIntrinsicID, baseType, size);
2879             retVal   = simdTree;
2880         }
2881         break;
2882
2883         case SIMDIntrinsicSelect:
2884         {
2885             // op3 is a SIMD variable that is the second source
2886             // op2 is a SIMD variable that is the first source
2887             // op1 is a SIMD variable which is the bit mask.
2888             op3 = impSIMDPopStack(simdType);
2889             op2 = impSIMDPopStack(simdType);
2890             op1 = impSIMDPopStack(simdType);
2891
2892             retVal = impSIMDSelect(clsHnd, baseType, size, op1, op2, op3);
2893         }
2894         break;
2895
2896         case SIMDIntrinsicMin:
2897         case SIMDIntrinsicMax:
2898         {
2899             // op1 is the first operand; if instance method, op1 is "this" arg
2900             // op2 is the second operand
2901             op2 = impSIMDPopStack(simdType);
2902             op1 = impSIMDPopStack(simdType, instMethod);
2903
2904             retVal = impSIMDMinMax(simdIntrinsicID, clsHnd, baseType, size, op1, op2);
2905         }
2906         break;
2907
2908         case SIMDIntrinsicGetItem:
2909         {
2910             // op1 is a SIMD variable that is "this" arg
2911             // op2 is an index of TYP_INT
2912             op2              = impSIMDPopStack(TYP_INT);
2913             op1              = impSIMDPopStack(simdType, instMethod);
2914             int vectorLength = getSIMDVectorLength(size, baseType);
2915             if (!op2->IsCnsIntOrI() || op2->AsIntCon()->gtIconVal >= vectorLength || op2->AsIntCon()->gtIconVal < 0)
2916             {
2917                 // We need to bounds-check the length of the vector.
2918                 // For that purpose, we need to clone the index expression.
2919                 GenTree* index = op2;
2920                 if ((index->gtFlags & GTF_SIDE_EFFECT) != 0)
2921                 {
2922                     op2 = fgInsertCommaFormTemp(&index);
2923                 }
2924                 else
2925                 {
2926                     op2 = gtCloneExpr(index);
2927                 }
2928
2929                 GenTree*          lengthNode = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, vectorLength);
2930                 GenTreeBoundsChk* simdChk =
2931                     new (this, GT_SIMD_CHK) GenTreeBoundsChk(GT_SIMD_CHK, TYP_VOID, index, lengthNode, SCK_RNGCHK_FAIL);
2932
2933                 // Create a GT_COMMA tree for the bounds check.
2934                 op2 = gtNewOperNode(GT_COMMA, op2->TypeGet(), simdChk, op2);
2935             }
2936
2937             assert(op1->TypeGet() == simdType);
2938             assert(op2->TypeGet() == TYP_INT);
2939
2940             simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, simdIntrinsicID, baseType, size);
2941             retVal   = simdTree;
2942         }
2943         break;
2944
2945         case SIMDIntrinsicDotProduct:
2946         {
2947 #if defined(_TARGET_XARCH_)
2948             // Right now dot product is supported only for float/double vectors and
2949             // int vectors on SSE4/AVX.
2950             if (!varTypeIsFloating(baseType) && !(baseType == TYP_INT && getSIMDSupportLevel() >= SIMD_SSE4_Supported))
2951             {
2952                 return nullptr;
2953             }
2954 #endif // _TARGET_XARCH_
2955
2956             // op1 is a SIMD variable that is the first source and also "this" arg.
2957             // op2 is a SIMD variable which is the second source.
2958             op2 = impSIMDPopStack(simdType);
2959             op1 = impSIMDPopStack(simdType, instMethod);
2960
2961             simdTree = gtNewSIMDNode(baseType, op1, op2, simdIntrinsicID, baseType, size);
2962             if (simdType == TYP_SIMD12)
2963             {
2964                 simdTree->gtFlags |= GTF_SIMD12_OP;
2965             }
2966             retVal = simdTree;
2967         }
2968         break;
2969
2970         case SIMDIntrinsicSqrt:
2971         {
2972 #if (defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)) && defined(DEBUG)
2973             // SSE/AVX/ARM64 doesn't support sqrt on integer type vectors and hence
2974             // should never be seen as an intrinsic here. See SIMDIntrinsicList.h
2975             // for supported base types for this intrinsic.
2976             if (!varTypeIsFloating(baseType))
2977             {
2978                 assert(!"Sqrt not supported on integer vectors\n");
2979                 return nullptr;
2980             }
2981 #endif // (defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)) && defined(DEBUG)
2982
2983             op1 = impSIMDPopStack(simdType);
2984
2985             retVal = gtNewSIMDNode(genActualType(callType), op1, nullptr, simdIntrinsicID, baseType, size);
2986         }
2987         break;
2988
2989         case SIMDIntrinsicAbs:
2990             op1    = impSIMDPopStack(simdType);
2991             retVal = impSIMDAbs(clsHnd, baseType, size, op1);
2992             break;
2993
2994         case SIMDIntrinsicGetW:
2995             retVal = impSIMDGetFixed(simdType, baseType, size, 3);
2996             break;
2997
2998         case SIMDIntrinsicGetZ:
2999             retVal = impSIMDGetFixed(simdType, baseType, size, 2);
3000             break;
3001
3002         case SIMDIntrinsicGetY:
3003             retVal = impSIMDGetFixed(simdType, baseType, size, 1);
3004             break;
3005
3006         case SIMDIntrinsicGetX:
3007             retVal = impSIMDGetFixed(simdType, baseType, size, 0);
3008             break;
3009
3010         case SIMDIntrinsicSetW:
3011         case SIMDIntrinsicSetZ:
3012         case SIMDIntrinsicSetY:
3013         case SIMDIntrinsicSetX:
3014         {
3015             // op2 is the value to be set at indexTemp position
3016             // op1 is SIMD vector that is going to be modified, which is a byref
3017
3018             // If op1 has a side-effect, then don't make it an intrinsic.
3019             // It would be in-efficient to read the entire vector into xmm reg,
3020             // modify it and write back entire xmm reg.
3021             //
3022             // TODO-CQ: revisit this later.
3023             op1 = impStackTop(1).val;
3024             if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0)
3025             {
3026                 return nullptr;
3027             }
3028
3029             op2 = impSIMDPopStack(baseType);
3030             op1 = impSIMDPopStack(simdType, instMethod);
3031
3032             GenTree* src = gtCloneExpr(op1);
3033             assert(src != nullptr);
3034             simdTree = gtNewSIMDNode(simdType, src, op2, simdIntrinsicID, baseType, size);
3035
3036             copyBlkDst = gtNewOperNode(GT_ADDR, TYP_BYREF, op1);
3037             doCopyBlk  = true;
3038         }
3039         break;
3040
3041         // Unary operators that take and return a Vector.
3042         case SIMDIntrinsicCast:
3043         case SIMDIntrinsicConvertToSingle:
3044         case SIMDIntrinsicConvertToDouble:
3045         case SIMDIntrinsicConvertToInt32:
3046         case SIMDIntrinsicConvertToUInt32:
3047         {
3048             op1 = impSIMDPopStack(simdType, instMethod);
3049
3050             simdTree = gtNewSIMDNode(simdType, op1, nullptr, simdIntrinsicID, baseType, size);
3051             retVal   = simdTree;
3052         }
3053         break;
3054
3055         case SIMDIntrinsicConvertToInt64:
3056         case SIMDIntrinsicConvertToUInt64:
3057         {
3058 #ifdef _TARGET_64BIT_
3059             op1 = impSIMDPopStack(simdType, instMethod);
3060
3061             simdTree = gtNewSIMDNode(simdType, op1, nullptr, simdIntrinsicID, baseType, size);
3062             retVal   = simdTree;
3063 #else
3064             JITDUMP("SIMD Conversion to Int64/UInt64 is not supported on this platform\n");
3065             return nullptr;
3066 #endif
3067         }
3068         break;
3069
3070         case SIMDIntrinsicNarrow:
3071         {
3072             assert(!instMethod);
3073             op2 = impSIMDPopStack(simdType);
3074             op1 = impSIMDPopStack(simdType);
3075             // op1 and op2 are two input Vector<T>.
3076             simdTree = gtNewSIMDNode(simdType, op1, op2, simdIntrinsicID, baseType, size);
3077             retVal   = simdTree;
3078         }
3079         break;
3080
3081         case SIMDIntrinsicWiden:
3082         {
3083             GenTree* dstAddrHi = impSIMDPopStack(TYP_BYREF);
3084             GenTree* dstAddrLo = impSIMDPopStack(TYP_BYREF);
3085             op1                = impSIMDPopStack(simdType);
3086             GenTree* dupOp1    = fgInsertCommaFormTemp(&op1, gtGetStructHandleForSIMD(simdType, baseType));
3087
3088             // Widen the lower half and assign it to dstAddrLo.
3089             simdTree = gtNewSIMDNode(simdType, op1, nullptr, SIMDIntrinsicWidenLo, baseType, size);
3090             GenTree* loDest =
3091                 new (this, GT_BLK) GenTreeBlk(GT_BLK, simdType, dstAddrLo, getSIMDTypeSizeInBytes(clsHnd));
3092             GenTree* loAsg = gtNewBlkOpNode(loDest, simdTree, getSIMDTypeSizeInBytes(clsHnd),
3093                                             false, // not volatile
3094                                             true); // copyBlock
3095             loAsg->gtFlags |= ((simdTree->gtFlags | dstAddrLo->gtFlags) & GTF_ALL_EFFECT);
3096
3097             // Widen the upper half and assign it to dstAddrHi.
3098             simdTree = gtNewSIMDNode(simdType, dupOp1, nullptr, SIMDIntrinsicWidenHi, baseType, size);
3099             GenTree* hiDest =
3100                 new (this, GT_BLK) GenTreeBlk(GT_BLK, simdType, dstAddrHi, getSIMDTypeSizeInBytes(clsHnd));
3101             GenTree* hiAsg = gtNewBlkOpNode(hiDest, simdTree, getSIMDTypeSizeInBytes(clsHnd),
3102                                             false, // not volatile
3103                                             true); // copyBlock
3104             hiAsg->gtFlags |= ((simdTree->gtFlags | dstAddrHi->gtFlags) & GTF_ALL_EFFECT);
3105
3106             retVal = gtNewOperNode(GT_COMMA, simdType, loAsg, hiAsg);
3107         }
3108         break;
3109
3110         case SIMDIntrinsicHWAccel:
3111         {
3112             GenTreeIntCon* intConstTree = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, 1);
3113             retVal                      = intConstTree;
3114         }
3115         break;
3116
3117         default:
3118             assert(!"Unimplemented SIMD Intrinsic");
3119             return nullptr;
3120     }
3121
3122 #if defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
3123     // XArch/Arm64: also indicate that we use floating point registers.
3124     // The need for setting this here is that a method may not have SIMD
3125     // type lclvars, but might be exercising SIMD intrinsics on fields of
3126     // SIMD type.
3127     //
3128     // e.g.  public Vector<float> ComplexVecFloat::sqabs() { return this.r * this.r + this.i * this.i; }
3129     compFloatingPointUsed = true;
3130 #endif // defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
3131
3132     // At this point, we have a tree that we are going to store into a destination.
3133     // TODO-1stClassStructs: This should be a simple store or assignment, and should not require
3134     // GTF_ALL_EFFECT for the dest. This is currently emulating the previous behavior of
3135     // block ops.
3136     if (doCopyBlk)
3137     {
3138         GenTree* dest = new (this, GT_BLK) GenTreeBlk(GT_BLK, simdType, copyBlkDst, getSIMDTypeSizeInBytes(clsHnd));
3139         dest->gtFlags |= GTF_GLOB_REF;
3140         retVal = gtNewBlkOpNode(dest, simdTree, getSIMDTypeSizeInBytes(clsHnd),
3141                                 false, // not volatile
3142                                 true); // copyBlock
3143         retVal->gtFlags |= ((simdTree->gtFlags | copyBlkDst->gtFlags) & GTF_ALL_EFFECT);
3144     }
3145
3146     return retVal;
3147 }
3148
3149 #endif // FEATURE_SIMD
3150
3151 #endif // !LEGACY_BACKEND