src/jit/simd.cpp

   1 // Licensed to the .NET Foundation under one or more agreements.
   2 // The .NET Foundation licenses this file to you under the MIT license.
   3 // See the LICENSE file in the project root for more information.
   4
   5 //
   6 //   SIMD Support
   7 //
   8 // IMPORTANT NOTES AND CAVEATS:
   9 //
  10 // This implementation is preliminary, and may change dramatically.
  11 //
  12 // New JIT types, TYP_SIMDxx, are introduced, and the SIMD intrinsics are created as GT_SIMD nodes.
  13 // Nodes of SIMD types will be typed as TYP_SIMD* (e.g. TYP_SIMD8, TYP_SIMD16, etc.).
  14 //
  15 // Note that currently the "reference implementation" is the same as the runtime dll.  As such, it is currently
  16 // providing implementations for those methods not currently supported by the JIT as intrinsics.
  17 //
  18 // These are currently recognized using string compares, in order to provide an implementation in the JIT
  19 // without taking a dependency on the VM.
  20 // Furthermore, in the CTP, in order to limit the impact of doing these string compares
  21 // against assembly names, we only look for the SIMDVector assembly if we are compiling a class constructor.  This
  22 // makes it somewhat more "pay for play" but is a significant usability compromise.
  23 // This has been addressed for RTM by doing the assembly recognition in the VM.
  24 // --------------------------------------------------------------------------------------
  25
  26 #include "jitpch.h"
  27 #include "simd.h"
  28
  29 #ifdef _MSC_VER
  30 #pragma hdrstop
  31 #endif
  32
  33 #ifdef FEATURE_SIMD
  34
  35 // Intrinsic Id to intrinsic info map
  36 const SIMDIntrinsicInfo simdIntrinsicInfoArray[] = {
  37 #define SIMD_INTRINSIC(mname, inst, id, name, retType, argCount, arg1, arg2, arg3, t1, t2, t3, t4, t5, t6, t7, t8, t9, \
  38                        t10)                                                                                            \
  39     {SIMDIntrinsic##id, mname, inst, retType, argCount, arg1, arg2, arg3, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10},
  40 #include "simdintrinsiclist.h"
  41 };
  42
  43 //------------------------------------------------------------------------
  44 // getSIMDVectorLength: Get the length (number of elements of base type) of
  45 //                      SIMD Vector given its size and base (element) type.
  46 //
  47 // Arguments:
  48 //    simdSize   - size of the SIMD vector
  49 //    baseType   - type of the elements of the SIMD vector
  50 //
  51 // static
  52 int Compiler::getSIMDVectorLength(unsigned simdSize, var_types baseType)
  53 {
  54     return simdSize / genTypeSize(baseType);
  55 }
  56
  57 //------------------------------------------------------------------------
  58 // Get the length (number of elements of base type) of SIMD Vector given by typeHnd.
  59 //
  60 // Arguments:
  61 //    typeHnd  - type handle of the SIMD vector
  62 //
  63 int Compiler::getSIMDVectorLength(CORINFO_CLASS_HANDLE typeHnd)
  64 {
  65     unsigned  sizeBytes = 0;
  66     var_types baseType  = getBaseTypeAndSizeOfSIMDType(typeHnd, &sizeBytes);
  67     return getSIMDVectorLength(sizeBytes, baseType);
  68 }
  69
  70 //------------------------------------------------------------------------
  71 // Get the preferred alignment of SIMD vector type for better performance.
  72 //
  73 // Arguments:
  74 //    typeHnd  - type handle of the SIMD vector
  75 //
  76 int Compiler::getSIMDTypeAlignment(var_types simdType)
  77 {
  78 #ifdef _TARGET_XARCH_
  79     // Fixed length vectors have the following alignment preference
  80     // Vector2   = 8 byte alignment
  81     // Vector3/4 = 16-byte alignment
  82     unsigned size = genTypeSize(simdType);
  83
  84     // preferred alignment for SSE2 128-bit vectors is 16-bytes
  85     if (size == 8)
  86     {
  87         return 8;
  88     }
  89     else if (size <= 16)
  90     {
  91         assert((size == 12) || (size == 16));
  92         return 16;
  93     }
  94     else
  95     {
  96         assert(size == 32);
  97         return 32;
  98     }
  99 #elif defined(_TARGET_ARM64_)
 100     return 16;
 101 #else
 102     assert(!"getSIMDTypeAlignment() unimplemented on target arch");
 103     unreached();
 104 #endif
 105 }
 106
 107 //----------------------------------------------------------------------------------
 108 // Return the base type and size of SIMD vector type given its type handle.
 109 //
 110 // Arguments:
 111 //    typeHnd   - The handle of the type we're interested in.
 112 //    sizeBytes - out param
 113 //
 114 // Return Value:
 115 //    base type of SIMD vector.
 116 //    sizeBytes if non-null is set to size in bytes.
 117 //
 118 // TODO-Throughput: current implementation parses class name to find base type. Change
 119 //         this when we implement  SIMD intrinsic identification for the final
 120 //         product.
 121 //
 122 var_types Compiler::getBaseTypeAndSizeOfSIMDType(CORINFO_CLASS_HANDLE typeHnd, unsigned* sizeBytes /*= nullptr */)
 123 {
 124     assert(featureSIMD);
 125     if (typeHnd == nullptr)
 126     {
 127         return TYP_UNKNOWN;
 128     }
 129
 130     // fast path search using cached type handles of important types
 131     var_types simdBaseType = TYP_UNKNOWN;
 132     unsigned  size         = 0;
 133
 134     // TODO - Optimize SIMD type recognition by IntrinsicAttribute
 135     if (isSIMDClass(typeHnd))
 136     {
 137         // The most likely to be used type handles are looked up first followed by
 138         // less likely to be used type handles
 139         if (typeHnd == SIMDFloatHandle)
 140         {
 141             simdBaseType = TYP_FLOAT;
 142             JITDUMP("  Known type SIMD Vector<Float>\n");
 143         }
 144         else if (typeHnd == SIMDIntHandle)
 145         {
 146             simdBaseType = TYP_INT;
 147             JITDUMP("  Known type SIMD Vector<Int>\n");
 148         }
 149         else if (typeHnd == SIMDVector2Handle)
 150         {
 151             simdBaseType = TYP_FLOAT;
 152             size         = 2 * genTypeSize(TYP_FLOAT);
 153             assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE));
 154             JITDUMP("  Known type Vector2\n");
 155         }
 156         else if (typeHnd == SIMDVector3Handle)
 157         {
 158             simdBaseType = TYP_FLOAT;
 159             size         = 3 * genTypeSize(TYP_FLOAT);
 160             assert(size == info.compCompHnd->getClassSize(typeHnd));
 161             JITDUMP("  Known type Vector3\n");
 162         }
 163         else if (typeHnd == SIMDVector4Handle)
 164         {
 165             simdBaseType = TYP_FLOAT;
 166             size         = 4 * genTypeSize(TYP_FLOAT);
 167             assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE));
 168             JITDUMP("  Known type Vector4\n");
 169         }
 170         else if (typeHnd == SIMDVectorHandle)
 171         {
 172             JITDUMP("  Known type Vector\n");
 173         }
 174         else if (typeHnd == SIMDUShortHandle)
 175         {
 176             simdBaseType = TYP_USHORT;
 177             JITDUMP("  Known type SIMD Vector<ushort>\n");
 178         }
 179         else if (typeHnd == SIMDUByteHandle)
 180         {
 181             simdBaseType = TYP_UBYTE;
 182             JITDUMP("  Known type SIMD Vector<ubyte>\n");
 183         }
 184         else if (typeHnd == SIMDDoubleHandle)
 185         {
 186             simdBaseType = TYP_DOUBLE;
 187             JITDUMP("  Known type SIMD Vector<Double>\n");
 188         }
 189         else if (typeHnd == SIMDLongHandle)
 190         {
 191             simdBaseType = TYP_LONG;
 192             JITDUMP("  Known type SIMD Vector<Long>\n");
 193         }
 194         else if (typeHnd == SIMDShortHandle)
 195         {
 196             simdBaseType = TYP_SHORT;
 197             JITDUMP("  Known type SIMD Vector<short>\n");
 198         }
 199         else if (typeHnd == SIMDByteHandle)
 200         {
 201             simdBaseType = TYP_BYTE;
 202             JITDUMP("  Known type SIMD Vector<byte>\n");
 203         }
 204         else if (typeHnd == SIMDUIntHandle)
 205         {
 206             simdBaseType = TYP_UINT;
 207             JITDUMP("  Known type SIMD Vector<uint>\n");
 208         }
 209         else if (typeHnd == SIMDULongHandle)
 210         {
 211             simdBaseType = TYP_ULONG;
 212             JITDUMP("  Known type SIMD Vector<ulong>\n");
 213         }
 214
 215         // slow path search
 216         if (simdBaseType == TYP_UNKNOWN)
 217         {
 218             // Doesn't match with any of the cached type handles.
 219             // Obtain base type by parsing fully qualified class name.
 220             //
 221             // TODO-Throughput: implement product shipping solution to query base type.
 222             WCHAR  className[256] = {0};
 223             WCHAR* pbuf           = &className[0];
 224             int    len            = _countof(className);
 225             info.compCompHnd->appendClassName(&pbuf, &len, typeHnd, TRUE, FALSE, FALSE);
 226             noway_assert(pbuf < &className[256]);
 227             JITDUMP("SIMD Candidate Type %S\n", className);
 228
 229             if (wcsncmp(className, W("System.Numerics."), 16) == 0)
 230             {
 231                 if (wcsncmp(&(className[16]), W("Vector`1["), 9) == 0)
 232                 {
 233                     if (wcsncmp(&(className[25]), W("System.Single"), 13) == 0)
 234                     {
 235                         SIMDFloatHandle = typeHnd;
 236                         simdBaseType    = TYP_FLOAT;
 237                         JITDUMP("  Found type SIMD Vector<Float>\n");
 238                     }
 239                     else if (wcsncmp(&(className[25]), W("System.Int32"), 12) == 0)
 240                     {
 241                         SIMDIntHandle = typeHnd;
 242                         simdBaseType  = TYP_INT;
 243                         JITDUMP("  Found type SIMD Vector<Int>\n");
 244                     }
 245                     else if (wcsncmp(&(className[25]), W("System.UInt16"), 13) == 0)
 246                     {
 247                         SIMDUShortHandle = typeHnd;
 248                         simdBaseType     = TYP_USHORT;
 249                         JITDUMP("  Found type SIMD Vector<ushort>\n");
 250                     }
 251                     else if (wcsncmp(&(className[25]), W("System.Byte"), 11) == 0)
 252                     {
 253                         SIMDUByteHandle = typeHnd;
 254                         simdBaseType    = TYP_UBYTE;
 255                         JITDUMP("  Found type SIMD Vector<ubyte>\n");
 256                     }
 257                     else if (wcsncmp(&(className[25]), W("System.Double"), 13) == 0)
 258                     {
 259                         SIMDDoubleHandle = typeHnd;
 260                         simdBaseType     = TYP_DOUBLE;
 261                         JITDUMP("  Found type SIMD Vector<Double>\n");
 262                     }
 263                     else if (wcsncmp(&(className[25]), W("System.Int64"), 12) == 0)
 264                     {
 265                         SIMDLongHandle = typeHnd;
 266                         simdBaseType   = TYP_LONG;
 267                         JITDUMP("  Found type SIMD Vector<Long>\n");
 268                     }
 269                     else if (wcsncmp(&(className[25]), W("System.Int16"), 12) == 0)
 270                     {
 271                         SIMDShortHandle = typeHnd;
 272                         simdBaseType    = TYP_SHORT;
 273                         JITDUMP("  Found type SIMD Vector<short>\n");
 274                     }
 275                     else if (wcsncmp(&(className[25]), W("System.SByte"), 12) == 0)
 276                     {
 277                         SIMDByteHandle = typeHnd;
 278                         simdBaseType   = TYP_BYTE;
 279                         JITDUMP("  Found type SIMD Vector<byte>\n");
 280                     }
 281                     else if (wcsncmp(&(className[25]), W("System.UInt32"), 13) == 0)
 282                     {
 283                         SIMDUIntHandle = typeHnd;
 284                         simdBaseType   = TYP_UINT;
 285                         JITDUMP("  Found type SIMD Vector<uint>\n");
 286                     }
 287                     else if (wcsncmp(&(className[25]), W("System.UInt64"), 13) == 0)
 288                     {
 289                         SIMDULongHandle = typeHnd;
 290                         simdBaseType    = TYP_ULONG;
 291                         JITDUMP("  Found type SIMD Vector<ulong>\n");
 292                     }
 293                     else
 294                     {
 295                         JITDUMP("  Unknown SIMD Vector<T>\n");
 296                     }
 297                 }
 298                 else if (wcsncmp(&(className[16]), W("Vector2"), 8) == 0)
 299                 {
 300                     SIMDVector2Handle = typeHnd;
 301
 302                     simdBaseType = TYP_FLOAT;
 303                     size         = 2 * genTypeSize(TYP_FLOAT);
 304                     assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE));
 305                     JITDUMP(" Found Vector2\n");
 306                 }
 307                 else if (wcsncmp(&(className[16]), W("Vector3"), 8) == 0)
 308                 {
 309                     SIMDVector3Handle = typeHnd;
 310
 311                     simdBaseType = TYP_FLOAT;
 312                     size         = 3 * genTypeSize(TYP_FLOAT);
 313                     assert(size == info.compCompHnd->getClassSize(typeHnd));
 314                     JITDUMP(" Found Vector3\n");
 315                 }
 316                 else if (wcsncmp(&(className[16]), W("Vector4"), 8) == 0)
 317                 {
 318                     SIMDVector4Handle = typeHnd;
 319
 320                     simdBaseType = TYP_FLOAT;
 321                     size         = 4 * genTypeSize(TYP_FLOAT);
 322                     assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE));
 323                     JITDUMP(" Found Vector4\n");
 324                 }
 325                 else if (wcsncmp(&(className[16]), W("Vector"), 6) == 0)
 326                 {
 327                     SIMDVectorHandle = typeHnd;
 328                     JITDUMP(" Found type Vector\n");
 329                 }
 330                 else
 331                 {
 332                     JITDUMP("  Unknown SIMD Type\n");
 333                 }
 334             }
 335         }
 336         if (simdBaseType != TYP_UNKNOWN && sizeBytes != nullptr)
 337         {
 338             // If not a fixed size vector then its size is same as SIMD vector
 339             // register length in bytes
 340             if (size == 0)
 341             {
 342                 size = getSIMDVectorRegisterByteLength();
 343             }
 344
 345             *sizeBytes = size;
 346             setUsesSIMDTypes(true);
 347         }
 348     }
 349 #ifdef FEATURE_HW_INTRINSICS
 350     else if (isIntrinsicType(typeHnd))
 351     {
 352         const size_t Vector64SizeBytes  = 64 / 8;
 353         const size_t Vector128SizeBytes = 128 / 8;
 354         const size_t Vector256SizeBytes = 256 / 8;
 355
 356 #if defined(_TARGET_XARCH_)
 357         static_assert_no_msg(YMM_REGSIZE_BYTES == Vector256SizeBytes);
 358         static_assert_no_msg(XMM_REGSIZE_BYTES == Vector128SizeBytes);
 359
 360         if (typeHnd == Vector256FloatHandle)
 361         {
 362             simdBaseType = TYP_FLOAT;
 363             size         = Vector256SizeBytes;
 364             JITDUMP("  Known type Vector256<float>\n");
 365         }
 366         else if (typeHnd == Vector256DoubleHandle)
 367         {
 368             simdBaseType = TYP_DOUBLE;
 369             size         = Vector256SizeBytes;
 370             JITDUMP("  Known type Vector256<double>\n");
 371         }
 372         else if (typeHnd == Vector256IntHandle)
 373         {
 374             simdBaseType = TYP_INT;
 375             size         = Vector256SizeBytes;
 376             JITDUMP("  Known type Vector256<int>\n");
 377         }
 378         else if (typeHnd == Vector256UIntHandle)
 379         {
 380             simdBaseType = TYP_UINT;
 381             size         = Vector256SizeBytes;
 382             JITDUMP("  Known type Vector256<uint>\n");
 383         }
 384         else if (typeHnd == Vector256ShortHandle)
 385         {
 386             simdBaseType = TYP_SHORT;
 387             size         = Vector256SizeBytes;
 388             JITDUMP("  Known type Vector256<short>\n");
 389         }
 390         else if (typeHnd == Vector256UShortHandle)
 391         {
 392             simdBaseType = TYP_USHORT;
 393             size         = Vector256SizeBytes;
 394             JITDUMP("  Known type Vector256<ushort>\n");
 395         }
 396         else if (typeHnd == Vector256ByteHandle)
 397         {
 398             simdBaseType = TYP_BYTE;
 399             size         = Vector256SizeBytes;
 400             JITDUMP("  Known type Vector256<sbyte>\n");
 401         }
 402         else if (typeHnd == Vector256UByteHandle)
 403         {
 404             simdBaseType = TYP_UBYTE;
 405             size         = Vector256SizeBytes;
 406             JITDUMP("  Known type Vector256<byte>\n");
 407         }
 408         else if (typeHnd == Vector256LongHandle)
 409         {
 410             simdBaseType = TYP_LONG;
 411             size         = Vector256SizeBytes;
 412             JITDUMP("  Known type Vector256<long>\n");
 413         }
 414         else if (typeHnd == Vector256ULongHandle)
 415         {
 416             simdBaseType = TYP_ULONG;
 417             size         = Vector256SizeBytes;
 418             JITDUMP("  Known type Vector256<ulong>\n");
 419         }
 420         else if (typeHnd == Vector256FloatHandle)
 421         {
 422             simdBaseType = TYP_FLOAT;
 423             size         = Vector256SizeBytes;
 424             JITDUMP("  Known type Vector256<float>\n");
 425         }
 426         else
 427 #endif // defined(_TARGET_XARCH)
 428             if (typeHnd == Vector128DoubleHandle)
 429         {
 430             simdBaseType = TYP_DOUBLE;
 431             size         = Vector128SizeBytes;
 432             JITDUMP("  Known type Vector128<double>\n");
 433         }
 434         else if (typeHnd == Vector128IntHandle)
 435         {
 436             simdBaseType = TYP_INT;
 437             size         = Vector128SizeBytes;
 438             JITDUMP("  Known type Vector128<int>\n");
 439         }
 440         else if (typeHnd == Vector128UIntHandle)
 441         {
 442             simdBaseType = TYP_UINT;
 443             size         = Vector128SizeBytes;
 444             JITDUMP("  Known type Vector128<uint>\n");
 445         }
 446         else if (typeHnd == Vector128ShortHandle)
 447         {
 448             simdBaseType = TYP_SHORT;
 449             size         = Vector128SizeBytes;
 450             JITDUMP("  Known type Vector128<short>\n");
 451         }
 452         else if (typeHnd == Vector128UShortHandle)
 453         {
 454             simdBaseType = TYP_USHORT;
 455             size         = Vector128SizeBytes;
 456             JITDUMP("  Known type Vector128<ushort>\n");
 457         }
 458         else if (typeHnd == Vector128ByteHandle)
 459         {
 460             simdBaseType = TYP_BYTE;
 461             size         = Vector128SizeBytes;
 462             JITDUMP("  Known type Vector128<sbyte>\n");
 463         }
 464         else if (typeHnd == Vector128UByteHandle)
 465         {
 466             simdBaseType = TYP_UBYTE;
 467             size         = Vector128SizeBytes;
 468             JITDUMP("  Known type Vector128<byte>\n");
 469         }
 470         else if (typeHnd == Vector128LongHandle)
 471         {
 472             simdBaseType = TYP_LONG;
 473             size         = Vector128SizeBytes;
 474             JITDUMP("  Known type Vector128<long>\n");
 475         }
 476         else if (typeHnd == Vector128ULongHandle)
 477         {
 478             simdBaseType = TYP_ULONG;
 479             size         = Vector128SizeBytes;
 480             JITDUMP("  Known type Vector128<ulong>\n");
 481         }
 482 #if defined(_TARGET_ARM64_)
 483         else if (typeHnd == Vector64IntHandle)
 484         {
 485             simdBaseType = TYP_INT;
 486             size         = Vector64SizeBytes;
 487             JITDUMP("  Known type Vector64<int>\n");
 488         }
 489         else if (typeHnd == Vector64UIntHandle)
 490         {
 491             simdBaseType = TYP_UINT;
 492             size         = Vector64SizeBytes;
 493             JITDUMP("  Known type Vector64<uint>\n");
 494         }
 495         else if (typeHnd == Vector64ShortHandle)
 496         {
 497             simdBaseType = TYP_SHORT;
 498             size         = Vector64SizeBytes;
 499             JITDUMP("  Known type Vector64<short>\n");
 500         }
 501         else if (typeHnd == Vector64UShortHandle)
 502         {
 503             simdBaseType = TYP_USHORT;
 504             size         = Vector64SizeBytes;
 505             JITDUMP("  Known type Vector64<ushort>\n");
 506         }
 507         else if (typeHnd == Vector64ByteHandle)
 508         {
 509             simdBaseType = TYP_BYTE;
 510             size         = Vector64SizeBytes;
 511             JITDUMP("  Known type Vector64<sbyte>\n");
 512         }
 513         else if (typeHnd == Vector64UByteHandle)
 514         {
 515             simdBaseType = TYP_UBYTE;
 516             size         = Vector64SizeBytes;
 517             JITDUMP("  Known type Vector64<byte>\n");
 518         }
 519 #endif // defined(_TARGET_ARM64_)
 520
 521         // slow path search
 522         if (simdBaseType == TYP_UNKNOWN)
 523         {
 524             // Doesn't match with any of the cached type handles.
 525             const char*          className   = getClassNameFromMetadata(typeHnd, nullptr);
 526             CORINFO_CLASS_HANDLE baseTypeHnd = getTypeInstantiationArgument(typeHnd, 0);
 527
 528             if (baseTypeHnd != nullptr)
 529             {
 530                 CorInfoType type = info.compCompHnd->getTypeForPrimitiveNumericClass(baseTypeHnd);
 531
 532                 JITDUMP("HW Intrinsic SIMD Candidate Type %s with Base Type %s\n", className,
 533                         getClassNameFromMetadata(baseTypeHnd, nullptr));
 534
 535 #if defined(_TARGET_XARCH_)
 536                 if (strcmp(className, "Vector256`1") == 0)
 537                 {
 538                     size = Vector256SizeBytes;
 539                     switch (type)
 540                     {
 541                         case CORINFO_TYPE_FLOAT:
 542                             Vector256FloatHandle = typeHnd;
 543                             simdBaseType         = TYP_FLOAT;
 544                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<float>\n");
 545                             break;
 546                         case CORINFO_TYPE_DOUBLE:
 547                             Vector256DoubleHandle = typeHnd;
 548                             simdBaseType          = TYP_DOUBLE;
 549                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<double>\n");
 550                             break;
 551                         case CORINFO_TYPE_INT:
 552                             Vector256IntHandle = typeHnd;
 553                             simdBaseType       = TYP_INT;
 554                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<int>\n");
 555                             break;
 556                         case CORINFO_TYPE_UINT:
 557                             Vector256UIntHandle = typeHnd;
 558                             simdBaseType        = TYP_UINT;
 559                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<uint>\n");
 560                             break;
 561                         case CORINFO_TYPE_SHORT:
 562                             Vector256ShortHandle = typeHnd;
 563                             simdBaseType         = TYP_SHORT;
 564                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<short>\n");
 565                             break;
 566                         case CORINFO_TYPE_USHORT:
 567                             Vector256UShortHandle = typeHnd;
 568                             simdBaseType          = TYP_USHORT;
 569                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<ushort>\n");
 570                             break;
 571                         case CORINFO_TYPE_LONG:
 572                             Vector256LongHandle = typeHnd;
 573                             simdBaseType        = TYP_LONG;
 574                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<long>\n");
 575                             break;
 576                         case CORINFO_TYPE_ULONG:
 577                             Vector256ULongHandle = typeHnd;
 578                             simdBaseType         = TYP_ULONG;
 579                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<ulong>\n");
 580                             break;
 581                         case CORINFO_TYPE_UBYTE:
 582                             Vector256UByteHandle = typeHnd;
 583                             simdBaseType         = TYP_UBYTE;
 584                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<byte>\n");
 585                             break;
 586                         case CORINFO_TYPE_BYTE:
 587                             Vector256ByteHandle = typeHnd;
 588                             simdBaseType        = TYP_BYTE;
 589                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<sbyte>\n");
 590                             break;
 591
 592                         default:
 593                             JITDUMP("  Unknown Hardware Intrinsic SIMD Type Vector256<T>\n");
 594                     }
 595                 }
 596                 else
 597 #endif // defined(_TARGET_XARCH_)
 598                     if (strcmp(className, "Vector128`1") == 0)
 599                 {
 600                     size = Vector128SizeBytes;
 601                     switch (type)
 602                     {
 603                         case CORINFO_TYPE_FLOAT:
 604                             Vector128FloatHandle = typeHnd;
 605                             simdBaseType         = TYP_FLOAT;
 606                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<float>\n");
 607                             break;
 608                         case CORINFO_TYPE_DOUBLE:
 609                             Vector128DoubleHandle = typeHnd;
 610                             simdBaseType          = TYP_DOUBLE;
 611                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<double>\n");
 612                             break;
 613                         case CORINFO_TYPE_INT:
 614                             Vector128IntHandle = typeHnd;
 615                             simdBaseType       = TYP_INT;
 616                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<int>\n");
 617                             break;
 618                         case CORINFO_TYPE_UINT:
 619                             Vector128UIntHandle = typeHnd;
 620                             simdBaseType        = TYP_UINT;
 621                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<uint>\n");
 622                             break;
 623                         case CORINFO_TYPE_SHORT:
 624                             Vector128ShortHandle = typeHnd;
 625                             simdBaseType         = TYP_SHORT;
 626                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<short>\n");
 627                             break;
 628                         case CORINFO_TYPE_USHORT:
 629                             Vector128UShortHandle = typeHnd;
 630                             simdBaseType          = TYP_USHORT;
 631                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<ushort>\n");
 632                             break;
 633                         case CORINFO_TYPE_LONG:
 634                             Vector128LongHandle = typeHnd;
 635                             simdBaseType        = TYP_LONG;
 636                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<long>\n");
 637                             break;
 638                         case CORINFO_TYPE_ULONG:
 639                             Vector128ULongHandle = typeHnd;
 640                             simdBaseType         = TYP_ULONG;
 641                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<ulong>\n");
 642                             break;
 643                         case CORINFO_TYPE_UBYTE:
 644                             Vector128UByteHandle = typeHnd;
 645                             simdBaseType         = TYP_UBYTE;
 646                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<byte>\n");
 647                             break;
 648                         case CORINFO_TYPE_BYTE:
 649                             Vector128ByteHandle = typeHnd;
 650                             simdBaseType        = TYP_BYTE;
 651                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<sbyte>\n");
 652                             break;
 653
 654                         default:
 655                             JITDUMP("  Unknown Hardware Intrinsic SIMD Type Vector128<T>\n");
 656                     }
 657                 }
 658 #if defined(_TARGET_ARM64_)
 659                 else if (strcmp(className, "Vector64`1") == 0)
 660                 {
 661                     size = Vector64SizeBytes;
 662                     switch (type)
 663                     {
 664                         case CORINFO_TYPE_FLOAT:
 665                             Vector64FloatHandle = typeHnd;
 666                             simdBaseType        = TYP_FLOAT;
 667                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<float>\n");
 668                             break;
 669                         case CORINFO_TYPE_INT:
 670                             Vector64IntHandle = typeHnd;
 671                             simdBaseType      = TYP_INT;
 672                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<int>\n");
 673                             break;
 674                         case CORINFO_TYPE_UINT:
 675                             Vector64UIntHandle = typeHnd;
 676                             simdBaseType       = TYP_UINT;
 677                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<uint>\n");
 678                             break;
 679                         case CORINFO_TYPE_SHORT:
 680                             Vector64ShortHandle = typeHnd;
 681                             simdBaseType        = TYP_SHORT;
 682                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<short>\n");
 683                             break;
 684                         case CORINFO_TYPE_USHORT:
 685                             Vector64UShortHandle = typeHnd;
 686                             simdBaseType         = TYP_USHORT;
 687                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<ushort>\n");
 688                             break;
 689                         case CORINFO_TYPE_UBYTE:
 690                             Vector64UByteHandle = typeHnd;
 691                             simdBaseType        = TYP_UBYTE;
 692                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<byte>\n");
 693                             break;
 694                         case CORINFO_TYPE_BYTE:
 695                             Vector64ByteHandle = typeHnd;
 696                             simdBaseType       = TYP_BYTE;
 697                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<sbyte>\n");
 698                             break;
 699
 700                         default:
 701                             JITDUMP("  Unknown Hardware Intrinsic SIMD Type Vector64<T>\n");
 702                     }
 703                 }
 704 #endif // defined(_TARGET_ARM64_)
 705             }
 706         }
 707
 708         if (sizeBytes != nullptr)
 709         {
 710             *sizeBytes = size;
 711         }
 712
 713         if (simdBaseType != TYP_UNKNOWN)
 714         {
 715             setUsesSIMDTypes(true);
 716         }
 717     }
 718 #endif // FEATURE_HW_INTRINSICS
 719
 720     return simdBaseType;
 721 }
 722
 723 //--------------------------------------------------------------------------------------
 724 // getSIMDIntrinsicInfo: get SIMD intrinsic info given the method handle.
 725 //
 726 // Arguments:
 727 //    inOutTypeHnd    - The handle of the type on which the method is invoked.  This is an in-out param.
 728 //    methodHnd       - The handle of the method we're interested in.
 729 //    sig             - method signature info
 730 //    isNewObj        - whether this call represents a newboj constructor call
 731 //    argCount        - argument count - out pram
 732 //    baseType        - base type of the intrinsic - out param
 733 //    sizeBytes       - size of SIMD vector type on which the method is invoked - out param
 734 //
 735 // Return Value:
 736 //    SIMDIntrinsicInfo struct initialized corresponding to methodHnd.
 737 //    Sets SIMDIntrinsicInfo.id to SIMDIntrinsicInvalid if methodHnd doesn't correspond
 738 //    to any SIMD intrinsic.  Also, sets the out params inOutTypeHnd, argCount, baseType and
 739 //    sizeBytes.
 740 //
 741 //    Note that VectorMath class doesn't have a base type and first argument of the method
 742 //    determines the SIMD vector type on which intrinsic is invoked. In such a case inOutTypeHnd
 743 //    is modified by this routine.
 744 //
 745 // TODO-Throughput: The current implementation is based on method name string parsing.
 746 //         Although we now have type identification from the VM, the parsing of intrinsic names
 747 //         could be made more efficient.
 748 //
 749 const SIMDIntrinsicInfo* Compiler::getSIMDIntrinsicInfo(CORINFO_CLASS_HANDLE* inOutTypeHnd,
 750                                                         CORINFO_METHOD_HANDLE methodHnd,
 751                                                         CORINFO_SIG_INFO*     sig,
 752                                                         bool                  isNewObj,
 753                                                         unsigned*             argCount,
 754                                                         var_types*            baseType,
 755                                                         unsigned*             sizeBytes)
 756 {
 757     assert(featureSIMD);
 758     assert(baseType != nullptr);
 759     assert(sizeBytes != nullptr);
 760
 761     // get baseType and size of the type
 762     CORINFO_CLASS_HANDLE typeHnd = *inOutTypeHnd;
 763     *baseType                    = getBaseTypeAndSizeOfSIMDType(typeHnd, sizeBytes);
 764
 765     if (typeHnd == SIMDVectorHandle)
 766     {
 767         // All of the supported intrinsics on this static class take a first argument that's a vector,
 768         // which determines the baseType.
 769         // The exception is the IsHardwareAccelerated property, which is handled as a special case.
 770         assert(*baseType == TYP_UNKNOWN);
 771         if (sig->numArgs == 0)
 772         {
 773             const SIMDIntrinsicInfo* hwAccelIntrinsicInfo = &(simdIntrinsicInfoArray[SIMDIntrinsicHWAccel]);
 774             if ((strcmp(eeGetMethodName(methodHnd, nullptr), hwAccelIntrinsicInfo->methodName) == 0) &&
 775                 JITtype2varType(sig->retType) == hwAccelIntrinsicInfo->retType)
 776             {
 777                 // Sanity check
 778                 assert(hwAccelIntrinsicInfo->argCount == 0 && hwAccelIntrinsicInfo->isInstMethod == false);
 779                 return hwAccelIntrinsicInfo;
 780             }
 781             return nullptr;
 782         }
 783         else
 784         {
 785             typeHnd       = info.compCompHnd->getArgClass(sig, sig->args);
 786             *inOutTypeHnd = typeHnd;
 787             *baseType     = getBaseTypeAndSizeOfSIMDType(typeHnd, sizeBytes);
 788         }
 789     }
 790
 791     if (*baseType == TYP_UNKNOWN)
 792     {
 793         JITDUMP("NOT a SIMD Intrinsic: unsupported baseType\n");
 794         return nullptr;
 795     }
 796
 797     // account for implicit "this" arg
 798     *argCount = sig->numArgs;
 799     if (sig->hasThis())
 800     {
 801         *argCount += 1;
 802     }
 803
 804     // Get the Intrinsic Id by parsing method name.
 805     //
 806     // TODO-Throughput: replace sequential search by binary search by arranging entries
 807     // sorted by method name.
 808     SIMDIntrinsicID intrinsicId = SIMDIntrinsicInvalid;
 809     const char*     methodName  = eeGetMethodName(methodHnd, nullptr);
 810     for (int i = SIMDIntrinsicNone + 1; i < SIMDIntrinsicInvalid; ++i)
 811     {
 812         if (strcmp(methodName, simdIntrinsicInfoArray[i].methodName) == 0)
 813         {
 814             // Found an entry for the method; further check whether it is one of
 815             // the supported base types.
 816             bool found = false;
 817             for (int j = 0; j < SIMD_INTRINSIC_MAX_BASETYPE_COUNT; ++j)
 818             {
 819                 // Convention: if there are fewer base types supported than MAX_BASETYPE_COUNT,
 820                 // the end of the list is marked by TYP_UNDEF.
 821                 if (simdIntrinsicInfoArray[i].supportedBaseTypes[j] == TYP_UNDEF)
 822                 {
 823                     break;
 824                 }
 825
 826                 if (simdIntrinsicInfoArray[i].supportedBaseTypes[j] == *baseType)
 827                 {
 828                     found = true;
 829                     break;
 830                 }
 831             }
 832
 833             if (!found)
 834             {
 835                 continue;
 836             }
 837
 838             // Now, check the arguments.
 839             unsigned int fixedArgCnt    = simdIntrinsicInfoArray[i].argCount;
 840             unsigned int expectedArgCnt = fixedArgCnt;
 841
 842             // First handle SIMDIntrinsicInitN, where the arg count depends on the type.
 843             // The listed arg types include the vector and the first two init values, which is the expected number
 844             // for Vector2.  For other cases, we'll check their types here.
 845             if (*argCount > expectedArgCnt)
 846             {
 847                 if (i == SIMDIntrinsicInitN)
 848                 {
 849                     if (*argCount == 3 && typeHnd == SIMDVector2Handle)
 850                     {
 851                         expectedArgCnt = 3;
 852                     }
 853                     else if (*argCount == 4 && typeHnd == SIMDVector3Handle)
 854                     {
 855                         expectedArgCnt = 4;
 856                     }
 857                     else if (*argCount == 5 && typeHnd == SIMDVector4Handle)
 858                     {
 859                         expectedArgCnt = 5;
 860                     }
 861                 }
 862                 else if (i == SIMDIntrinsicInitFixed)
 863                 {
 864                     if (*argCount == 4 && typeHnd == SIMDVector4Handle)
 865                     {
 866                         expectedArgCnt = 4;
 867                     }
 868                 }
 869             }
 870             if (*argCount != expectedArgCnt)
 871             {
 872                 continue;
 873             }
 874
 875             // Validate the types of individual args passed are what is expected of.
 876             // If any of the types don't match with what is expected, don't consider
 877             // as an intrinsic.  This will make an older JIT with SIMD capabilities
 878             // resilient to breaking changes to SIMD managed API.
 879             //
 880             // Note that from IL type stack, args get popped in right to left order
 881             // whereas args get listed in method signatures in left to right order.
 882
 883             int stackIndex = (expectedArgCnt - 1);
 884
 885             // Track the arguments from the signature - we currently only use this to distinguish
 886             // integral and pointer types, both of which will by TYP_I_IMPL on the importer stack.
 887             CORINFO_ARG_LIST_HANDLE argLst = sig->args;
 888
 889             CORINFO_CLASS_HANDLE argClass;
 890             for (unsigned int argIndex = 0; found == true && argIndex < expectedArgCnt; argIndex++)
 891             {
 892                 bool isThisPtr = ((argIndex == 0) && sig->hasThis());
 893
 894                 // In case of "newobj SIMDVector<T>(T val)", thisPtr won't be present on type stack.
 895                 // We don't check anything in that case.
 896                 if (!isThisPtr || !isNewObj)
 897                 {
 898                     GenTree*  arg     = impStackTop(stackIndex).val;
 899                     var_types argType = arg->TypeGet();
 900
 901                     var_types expectedArgType;
 902                     if (argIndex < fixedArgCnt)
 903                     {
 904                         // Convention:
 905                         //   - intrinsicInfo.argType[i] == TYP_UNDEF - intrinsic doesn't have a valid arg at position i
 906                         //   - intrinsicInfo.argType[i] == TYP_UNKNOWN - arg type should be same as basetype
 907                         // Note that we pop the args off in reverse order.
 908                         expectedArgType = simdIntrinsicInfoArray[i].argType[argIndex];
 909                         assert(expectedArgType != TYP_UNDEF);
 910                         if (expectedArgType == TYP_UNKNOWN)
 911                         {
 912                             // The type of the argument will be genActualType(*baseType).
 913                             expectedArgType = genActualType(*baseType);
 914                             argType         = genActualType(argType);
 915                         }
 916                     }
 917                     else
 918                     {
 919                         expectedArgType = *baseType;
 920                     }
 921
 922                     if (!isThisPtr && argType == TYP_I_IMPL)
 923                     {
 924                         // The reference implementation has a constructor that takes a pointer.
 925                         // We don't want to recognize that one.  This requires us to look at the CorInfoType
 926                         // in order to distinguish a signature with a pointer argument from one with an
 927                         // integer argument of pointer size, both of which will be TYP_I_IMPL on the stack.
 928                         // TODO-Review: This seems quite fragile.  We should consider beefing up the checking
 929                         // here.
 930                         CorInfoType corType = strip(info.compCompHnd->getArgType(sig, argLst, &argClass));
 931                         if (corType == CORINFO_TYPE_PTR)
 932                         {
 933                             found = false;
 934                         }
 935                     }
 936
 937                     if (varTypeIsSIMD(argType))
 938                     {
 939                         argType = TYP_STRUCT;
 940                     }
 941                     if (argType != expectedArgType)
 942                     {
 943                         found = false;
 944                     }
 945                 }
 946                 if (argIndex != 0 || !sig->hasThis())
 947                 {
 948                     argLst = info.compCompHnd->getArgNext(argLst);
 949                 }
 950                 stackIndex--;
 951             }
 952
 953             // Cross check return type and static vs. instance is what we are expecting.
 954             // If not, don't consider it as an intrinsic.
 955             // Note that ret type of TYP_UNKNOWN means that it is not known apriori and must be same as baseType
 956             if (found)
 957             {
 958                 var_types expectedRetType = simdIntrinsicInfoArray[i].retType;
 959                 if (expectedRetType == TYP_UNKNOWN)
 960                 {
 961                     // JIT maps uint/ulong type vars to TYP_INT/TYP_LONG.
 962                     expectedRetType =
 963                         (*baseType == TYP_UINT || *baseType == TYP_ULONG) ? genActualType(*baseType) : *baseType;
 964                 }
 965
 966                 if (JITtype2varType(sig->retType) != expectedRetType ||
 967                     sig->hasThis() != simdIntrinsicInfoArray[i].isInstMethod)
 968                 {
 969                     found = false;
 970                 }
 971             }
 972
 973             if (found)
 974             {
 975                 intrinsicId = (SIMDIntrinsicID)i;
 976                 break;
 977             }
 978         }
 979     }
 980
 981     if (intrinsicId != SIMDIntrinsicInvalid)
 982     {
 983         JITDUMP("Method %s maps to SIMD intrinsic %s\n", methodName, simdIntrinsicNames[intrinsicId]);
 984         return &simdIntrinsicInfoArray[intrinsicId];
 985     }
 986     else
 987     {
 988         JITDUMP("Method %s is NOT a SIMD intrinsic\n", methodName);
 989     }
 990
 991     return nullptr;
 992 }
 993
 994 // Pops and returns GenTree node from importer's type stack.
 995 // Normalizes TYP_STRUCT value in case of GT_CALL, GT_RET_EXPR and arg nodes.
 996 //
 997 // Arguments:
 998 //    type        -  the type of value that the caller expects to be popped off the stack.
 999 //    expectAddr  -  if true indicates we are expecting type stack entry to be a TYP_BYREF.
1000 //    structType  -  the class handle to use when normalizing if it is not the same as the stack entry class handle;
1001 //                   this can happen for certain scenarios, such as folding away a static cast, where we want the
1002 //                   value popped to have the type that would have been returned.
1003 //
1004 // Notes:
1005 //    If the popped value is a struct, and the expected type is a simd type, it will be set
1006 //    to that type, otherwise it will assert if the type being popped is not the expected type.
1007
1008 GenTree* Compiler::impSIMDPopStack(var_types type, bool expectAddr, CORINFO_CLASS_HANDLE structType)
1009 {
1010     StackEntry se   = impPopStack();
1011     typeInfo   ti   = se.seTypeInfo;
1012     GenTree*   tree = se.val;
1013
1014     // If expectAddr is true implies what we have on stack is address and we need
1015     // SIMD type struct that it points to.
1016     if (expectAddr)
1017     {
1018         assert(tree->TypeGet() == TYP_BYREF);
1019         if (tree->OperGet() == GT_ADDR)
1020         {
1021             tree = tree->gtGetOp1();
1022         }
1023         else
1024         {
1025             tree = gtNewOperNode(GT_IND, type, tree);
1026         }
1027     }
1028
1029     bool isParam = false;
1030
1031     // If we have a ldobj of a SIMD local we need to transform it.
1032     if (tree->OperGet() == GT_OBJ)
1033     {
1034         GenTree* addr = tree->gtOp.gtOp1;
1035         if ((addr->OperGet() == GT_ADDR) && isSIMDTypeLocal(addr->gtOp.gtOp1))
1036         {
1037             tree = addr->gtOp.gtOp1;
1038         }
1039     }
1040
1041     if (tree->OperGet() == GT_LCL_VAR)
1042     {
1043         unsigned   lclNum    = tree->AsLclVarCommon()->GetLclNum();
1044         LclVarDsc* lclVarDsc = &lvaTable[lclNum];
1045         isParam              = lclVarDsc->lvIsParam;
1046     }
1047
1048     // normalize TYP_STRUCT value
1049     if (varTypeIsStruct(tree) && ((tree->OperGet() == GT_RET_EXPR) || (tree->OperGet() == GT_CALL) || isParam))
1050     {
1051         assert(ti.IsType(TI_STRUCT));
1052
1053         if (structType == nullptr)
1054         {
1055             structType = ti.GetClassHandleForValueClass();
1056         }
1057
1058         tree = impNormStructVal(tree, structType, (unsigned)CHECK_SPILL_ALL);
1059     }
1060
1061     // Now set the type of the tree to the specialized SIMD struct type, if applicable.
1062     if (genActualType(tree->gtType) != genActualType(type))
1063     {
1064         assert(tree->gtType == TYP_STRUCT);
1065         tree->gtType = type;
1066     }
1067     else if (tree->gtType == TYP_BYREF)
1068     {
1069         assert(tree->IsLocal() || (tree->OperGet() == GT_RET_EXPR) || (tree->OperGet() == GT_CALL) ||
1070                ((tree->gtOper == GT_ADDR) && varTypeIsSIMD(tree->gtGetOp1())));
1071     }
1072
1073     return tree;
1074 }
1075
1076 // impSIMDGetFixed: Create a GT_SIMD tree for a Get property of SIMD vector with a fixed index.
1077 //
1078 // Arguments:
1079 //    baseType - The base (element) type of the SIMD vector.
1080 //    simdSize - The total size in bytes of the SIMD vector.
1081 //    index    - The index of the field to get.
1082 //
1083 // Return Value:
1084 //    Returns a GT_SIMD node with the SIMDIntrinsicGetItem intrinsic id.
1085 //
1086 GenTreeSIMD* Compiler::impSIMDGetFixed(var_types simdType, var_types baseType, unsigned simdSize, int index)
1087 {
1088     assert(simdSize >= ((index + 1) * genTypeSize(baseType)));
1089
1090     // op1 is a SIMD source.
1091     GenTree* op1 = impSIMDPopStack(simdType, true);
1092
1093     GenTree*     op2      = gtNewIconNode(index);
1094     GenTreeSIMD* simdTree = gtNewSIMDNode(baseType, op1, op2, SIMDIntrinsicGetItem, baseType, simdSize);
1095     return simdTree;
1096 }
1097
1098 #ifdef _TARGET_XARCH_
1099 // impSIMDLongRelOpEqual: transforms operands and returns the SIMD intrinsic to be applied on
1100 // transformed operands to obtain == comparison result.
1101 //
1102 // Argumens:
1103 //    typeHnd  -  type handle of SIMD vector
1104 //    size     -  SIMD vector size
1105 //    op1      -  in-out parameter; first operand
1106 //    op2      -  in-out parameter; second operand
1107 //
1108 // Return Value:
1109 //    Modifies in-out params op1, op2 and returns intrinsic ID to be applied to modified operands
1110 //
1111 SIMDIntrinsicID Compiler::impSIMDLongRelOpEqual(CORINFO_CLASS_HANDLE typeHnd,
1112                                                 unsigned             size,
1113                                                 GenTree**            pOp1,
1114                                                 GenTree**            pOp2)
1115 {
1116     var_types simdType = (*pOp1)->TypeGet();
1117     assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1118
1119     // There is no direct SSE2 support for comparing TYP_LONG vectors.
1120     // These have to be implemented in terms of TYP_INT vector comparison operations.
1121     //
1122     // Equality(v1, v2):
1123     // tmp = (v1 == v2) i.e. compare for equality as if v1 and v2 are vector<int>
1124     // result = BitwiseAnd(t, shuffle(t, (2, 3, 0, 1)))
1125     // Shuffle is meant to swap the comparison results of low-32-bits and high 32-bits of respective long elements.
1126
1127     // Compare vector<long> as if they were vector<int> and assign the result to a temp
1128     GenTree* compResult = gtNewSIMDNode(simdType, *pOp1, *pOp2, SIMDIntrinsicEqual, TYP_INT, size);
1129     unsigned lclNum     = lvaGrabTemp(true DEBUGARG("SIMD Long =="));
1130     lvaSetStruct(lclNum, typeHnd, false);
1131     GenTree* tmp = gtNewLclvNode(lclNum, simdType);
1132     GenTree* asg = gtNewTempAssign(lclNum, compResult);
1133
1134     // op1 = GT_COMMA(tmp=compResult, tmp)
1135     // op2 = Shuffle(tmp, 0xB1)
1136     // IntrinsicId = BitwiseAnd
1137     *pOp1 = gtNewOperNode(GT_COMMA, simdType, asg, tmp);
1138     *pOp2 = gtNewSIMDNode(simdType, gtNewLclvNode(lclNum, simdType), gtNewIconNode(SHUFFLE_ZWXY, TYP_INT),
1139                           SIMDIntrinsicShuffleSSE2, TYP_INT, size);
1140     return SIMDIntrinsicBitwiseAnd;
1141 }
1142
1143 // impSIMDLongRelOpGreaterThan: transforms operands and returns the SIMD intrinsic to be applied on
1144 // transformed operands to obtain > comparison result.
1145 //
1146 // Argumens:
1147 //    typeHnd  -  type handle of SIMD vector
1148 //    size     -  SIMD vector size
1149 //    pOp1     -  in-out parameter; first operand
1150 //    pOp2     -  in-out parameter; second operand
1151 //
1152 // Return Value:
1153 //    Modifies in-out params pOp1, pOp2 and returns intrinsic ID to be applied to modified operands
1154 //
1155 SIMDIntrinsicID Compiler::impSIMDLongRelOpGreaterThan(CORINFO_CLASS_HANDLE typeHnd,
1156                                                       unsigned             size,
1157                                                       GenTree**            pOp1,
1158                                                       GenTree**            pOp2)
1159 {
1160     var_types simdType = (*pOp1)->TypeGet();
1161     assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1162
1163     // GreaterThan(v1, v2) where v1 and v2 are vector long.
1164     // Let us consider the case of single long element comparison.
1165     // say L1 = (x1, y1) and L2 = (x2, y2) where x1, y1, x2, and y2 are 32-bit integers that comprise the longs L1 and
1166     // L2.
1167     //
1168     // GreaterThan(L1, L2) can be expressed in terms of > relationship between 32-bit integers that comprise L1 and L2
1169     // as
1170     //                    =  (x1, y1) > (x2, y2)
1171     //                    =  (x1 > x2) || [(x1 == x2) && (y1 > y2)]   - eq (1)
1172     //
1173     // t = (v1 > v2)  32-bit signed comparison
1174     // u = (v1 == v2) 32-bit sized element equality
1175     // v = (v1 > v2)  32-bit unsigned comparison
1176     //
1177     // z = shuffle(t, (3, 3, 1, 1))  - This corresponds to (x1 > x2) in eq(1) above
1178     // t1 = Shuffle(v, (2, 2, 0, 0)) - This corresponds to (y1 > y2) in eq(1) above
1179     // u1 = Shuffle(u, (3, 3, 1, 1)) - This corresponds to (x1 == x2) in eq(1) above
1180     // w = And(t1, u1)               - This corresponds to [(x1 == x2) && (y1 > y2)] in eq(1) above
1181     // Result = BitwiseOr(z, w)
1182
1183     // Since op1 and op2 gets used multiple times, make sure side effects are computed.
1184     GenTree* dupOp1    = nullptr;
1185     GenTree* dupOp2    = nullptr;
1186     GenTree* dupDupOp1 = nullptr;
1187     GenTree* dupDupOp2 = nullptr;
1188
1189     if (((*pOp1)->gtFlags & GTF_SIDE_EFFECT) != 0)
1190     {
1191         dupOp1    = fgInsertCommaFormTemp(pOp1, typeHnd);
1192         dupDupOp1 = gtNewLclvNode(dupOp1->AsLclVarCommon()->GetLclNum(), simdType);
1193     }
1194     else
1195     {
1196         dupOp1    = gtCloneExpr(*pOp1);
1197         dupDupOp1 = gtCloneExpr(*pOp1);
1198     }
1199
1200     if (((*pOp2)->gtFlags & GTF_SIDE_EFFECT) != 0)
1201     {
1202         dupOp2    = fgInsertCommaFormTemp(pOp2, typeHnd);
1203         dupDupOp2 = gtNewLclvNode(dupOp2->AsLclVarCommon()->GetLclNum(), simdType);
1204     }
1205     else
1206     {
1207         dupOp2    = gtCloneExpr(*pOp2);
1208         dupDupOp2 = gtCloneExpr(*pOp2);
1209     }
1210
1211     assert(dupDupOp1 != nullptr && dupDupOp2 != nullptr);
1212     assert(dupOp1 != nullptr && dupOp2 != nullptr);
1213     assert(*pOp1 != nullptr && *pOp2 != nullptr);
1214
1215     // v1GreaterThanv2Signed - signed 32-bit comparison
1216     GenTree* v1GreaterThanv2Signed = gtNewSIMDNode(simdType, *pOp1, *pOp2, SIMDIntrinsicGreaterThan, TYP_INT, size);
1217
1218     // v1Equalsv2 - 32-bit equality
1219     GenTree* v1Equalsv2 = gtNewSIMDNode(simdType, dupOp1, dupOp2, SIMDIntrinsicEqual, TYP_INT, size);
1220
1221     // v1GreaterThanv2Unsigned - unsigned 32-bit comparison
1222     var_types       tempBaseType = TYP_UINT;
1223     SIMDIntrinsicID sid = impSIMDRelOp(SIMDIntrinsicGreaterThan, typeHnd, size, &tempBaseType, &dupDupOp1, &dupDupOp2);
1224     GenTree*        v1GreaterThanv2Unsigned = gtNewSIMDNode(simdType, dupDupOp1, dupDupOp2, sid, tempBaseType, size);
1225
1226     GenTree* z = gtNewSIMDNode(simdType, v1GreaterThanv2Signed, gtNewIconNode(SHUFFLE_WWYY, TYP_INT),
1227                                SIMDIntrinsicShuffleSSE2, TYP_FLOAT, size);
1228     GenTree* t1 = gtNewSIMDNode(simdType, v1GreaterThanv2Unsigned, gtNewIconNode(SHUFFLE_ZZXX, TYP_INT),
1229                                 SIMDIntrinsicShuffleSSE2, TYP_FLOAT, size);
1230     GenTree* u1 = gtNewSIMDNode(simdType, v1Equalsv2, gtNewIconNode(SHUFFLE_WWYY, TYP_INT), SIMDIntrinsicShuffleSSE2,
1231                                 TYP_FLOAT, size);
1232     GenTree* w = gtNewSIMDNode(simdType, u1, t1, SIMDIntrinsicBitwiseAnd, TYP_INT, size);
1233
1234     *pOp1 = z;
1235     *pOp2 = w;
1236     return SIMDIntrinsicBitwiseOr;
1237 }
1238
1239 // impSIMDLongRelOpGreaterThanOrEqual: transforms operands and returns the SIMD intrinsic to be applied on
1240 // transformed operands to obtain >= comparison result.
1241 //
1242 // Argumens:
1243 //    typeHnd  -  type handle of SIMD vector
1244 //    size     -  SIMD vector size
1245 //    pOp1      -  in-out parameter; first operand
1246 //    pOp2      -  in-out parameter; second operand
1247 //
1248 // Return Value:
1249 //    Modifies in-out params pOp1, pOp2 and returns intrinsic ID to be applied to modified operands
1250 //
1251 SIMDIntrinsicID Compiler::impSIMDLongRelOpGreaterThanOrEqual(CORINFO_CLASS_HANDLE typeHnd,
1252                                                              unsigned             size,
1253                                                              GenTree**            pOp1,
1254                                                              GenTree**            pOp2)
1255 {
1256     var_types simdType = (*pOp1)->TypeGet();
1257     assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1258
1259     // expand this to (a == b) | (a > b)
1260     GenTree* dupOp1 = nullptr;
1261     GenTree* dupOp2 = nullptr;
1262
1263     if (((*pOp1)->gtFlags & GTF_SIDE_EFFECT) != 0)
1264     {
1265         dupOp1 = fgInsertCommaFormTemp(pOp1, typeHnd);
1266     }
1267     else
1268     {
1269         dupOp1 = gtCloneExpr(*pOp1);
1270     }
1271
1272     if (((*pOp2)->gtFlags & GTF_SIDE_EFFECT) != 0)
1273     {
1274         dupOp2 = fgInsertCommaFormTemp(pOp2, typeHnd);
1275     }
1276     else
1277     {
1278         dupOp2 = gtCloneExpr(*pOp2);
1279     }
1280
1281     assert(dupOp1 != nullptr && dupOp2 != nullptr);
1282     assert(*pOp1 != nullptr && *pOp2 != nullptr);
1283
1284     // (a==b)
1285     SIMDIntrinsicID id = impSIMDLongRelOpEqual(typeHnd, size, pOp1, pOp2);
1286     *pOp1              = gtNewSIMDNode(simdType, *pOp1, *pOp2, id, TYP_LONG, size);
1287
1288     // (a > b)
1289     id    = impSIMDLongRelOpGreaterThan(typeHnd, size, &dupOp1, &dupOp2);
1290     *pOp2 = gtNewSIMDNode(simdType, dupOp1, dupOp2, id, TYP_LONG, size);
1291
1292     return SIMDIntrinsicBitwiseOr;
1293 }
1294
1295 // impSIMDInt32OrSmallIntRelOpGreaterThanOrEqual: transforms operands and returns the SIMD intrinsic to be applied on
1296 // transformed operands to obtain >= comparison result in case of integer base type vectors
1297 //
1298 // Argumens:
1299 //    typeHnd  -  type handle of SIMD vector
1300 //    size     -  SIMD vector size
1301 //    baseType -  base type of SIMD vector
1302 //    pOp1      -  in-out parameter; first operand
1303 //    pOp2      -  in-out parameter; second operand
1304 //
1305 // Return Value:
1306 //    Modifies in-out params pOp1, pOp2 and returns intrinsic ID to be applied to modified operands
1307 //
1308 SIMDIntrinsicID Compiler::impSIMDIntegralRelOpGreaterThanOrEqual(
1309     CORINFO_CLASS_HANDLE typeHnd, unsigned size, var_types baseType, GenTree** pOp1, GenTree** pOp2)
1310 {
1311     var_types simdType = (*pOp1)->TypeGet();
1312     assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1313
1314     // This routine should be used only for integer base type vectors
1315     assert(varTypeIsIntegral(baseType));
1316     if ((getSIMDSupportLevel() == SIMD_SSE2_Supported) && ((baseType == TYP_LONG) || baseType == TYP_UBYTE))
1317     {
1318         return impSIMDLongRelOpGreaterThanOrEqual(typeHnd, size, pOp1, pOp2);
1319     }
1320
1321     // expand this to (a == b) | (a > b)
1322     GenTree* dupOp1 = nullptr;
1323     GenTree* dupOp2 = nullptr;
1324
1325     if (((*pOp1)->gtFlags & GTF_SIDE_EFFECT) != 0)
1326     {
1327         dupOp1 = fgInsertCommaFormTemp(pOp1, typeHnd);
1328     }
1329     else
1330     {
1331         dupOp1 = gtCloneExpr(*pOp1);
1332     }
1333
1334     if (((*pOp2)->gtFlags & GTF_SIDE_EFFECT) != 0)
1335     {
1336         dupOp2 = fgInsertCommaFormTemp(pOp2, typeHnd);
1337     }
1338     else
1339     {
1340         dupOp2 = gtCloneExpr(*pOp2);
1341     }
1342
1343     assert(dupOp1 != nullptr && dupOp2 != nullptr);
1344     assert(*pOp1 != nullptr && *pOp2 != nullptr);
1345
1346     // (a==b)
1347     *pOp1 = gtNewSIMDNode(simdType, *pOp1, *pOp2, SIMDIntrinsicEqual, baseType, size);
1348
1349     // (a > b)
1350     *pOp2 = gtNewSIMDNode(simdType, dupOp1, dupOp2, SIMDIntrinsicGreaterThan, baseType, size);
1351
1352     return SIMDIntrinsicBitwiseOr;
1353 }
1354 #endif // _TARGET_XARCH_
1355
1356 // Transforms operands and returns the SIMD intrinsic to be applied on
1357 // transformed operands to obtain given relop result.
1358 //
1359 // Argumens:
1360 //    relOpIntrinsicId - Relational operator SIMD intrinsic
1361 //    typeHnd          - type handle of SIMD vector
1362 //    size             -  SIMD vector size
1363 //    inOutBaseType    - base type of SIMD vector
1364 //    pOp1             -  in-out parameter; first operand
1365 //    pOp2             -  in-out parameter; second operand
1366 //
1367 // Return Value:
1368 //    Modifies in-out params pOp1, pOp2, inOutBaseType and returns intrinsic ID to be applied to modified operands
1369 //
1370 SIMDIntrinsicID Compiler::impSIMDRelOp(SIMDIntrinsicID      relOpIntrinsicId,
1371                                        CORINFO_CLASS_HANDLE typeHnd,
1372                                        unsigned             size,
1373                                        var_types*           inOutBaseType,
1374                                        GenTree**            pOp1,
1375                                        GenTree**            pOp2)
1376 {
1377     var_types simdType = (*pOp1)->TypeGet();
1378     assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1379
1380     assert(isRelOpSIMDIntrinsic(relOpIntrinsicId));
1381
1382     SIMDIntrinsicID intrinsicID = relOpIntrinsicId;
1383 #ifdef _TARGET_XARCH_
1384     var_types baseType = *inOutBaseType;
1385
1386     if (varTypeIsFloating(baseType))
1387     {
1388         // SSE2/AVX doesn't support > and >= on vector float/double.
1389         // Therefore, we need to use < and <= with swapped operands
1390         if (relOpIntrinsicId == SIMDIntrinsicGreaterThan || relOpIntrinsicId == SIMDIntrinsicGreaterThanOrEqual)
1391         {
1392             GenTree* tmp = *pOp1;
1393             *pOp1        = *pOp2;
1394             *pOp2        = tmp;
1395
1396             intrinsicID =
1397                 (relOpIntrinsicId == SIMDIntrinsicGreaterThan) ? SIMDIntrinsicLessThan : SIMDIntrinsicLessThanOrEqual;
1398         }
1399     }
1400     else if (varTypeIsIntegral(baseType))
1401     {
1402         // SSE/AVX doesn't support < and <= on integer base type vectors.
1403         // Therefore, we need to use > and >= with swapped operands.
1404         if (intrinsicID == SIMDIntrinsicLessThan || intrinsicID == SIMDIntrinsicLessThanOrEqual)
1405         {
1406             GenTree* tmp = *pOp1;
1407             *pOp1        = *pOp2;
1408             *pOp2        = tmp;
1409
1410             intrinsicID = (relOpIntrinsicId == SIMDIntrinsicLessThan) ? SIMDIntrinsicGreaterThan
1411                                                                       : SIMDIntrinsicGreaterThanOrEqual;
1412         }
1413
1414         if ((getSIMDSupportLevel() == SIMD_SSE2_Supported) && baseType == TYP_LONG)
1415         {
1416             // There is no direct SSE2 support for comparing TYP_LONG vectors.
1417             // These have to be implemented interms of TYP_INT vector comparison operations.
1418             if (intrinsicID == SIMDIntrinsicEqual)
1419             {
1420                 intrinsicID = impSIMDLongRelOpEqual(typeHnd, size, pOp1, pOp2);
1421             }
1422             else if (intrinsicID == SIMDIntrinsicGreaterThan)
1423             {
1424                 intrinsicID = impSIMDLongRelOpGreaterThan(typeHnd, size, pOp1, pOp2);
1425             }
1426             else if (intrinsicID == SIMDIntrinsicGreaterThanOrEqual)
1427             {
1428                 intrinsicID = impSIMDLongRelOpGreaterThanOrEqual(typeHnd, size, pOp1, pOp2);
1429             }
1430             else
1431             {
1432                 unreached();
1433             }
1434         }
1435         // SSE2 and AVX direct support for signed comparison of int32, int16 and int8 types
1436         else if (!varTypeIsUnsigned(baseType))
1437         {
1438             if (intrinsicID == SIMDIntrinsicGreaterThanOrEqual)
1439             {
1440                 intrinsicID = impSIMDIntegralRelOpGreaterThanOrEqual(typeHnd, size, baseType, pOp1, pOp2);
1441             }
1442         }
1443         else // unsigned
1444         {
1445             // Vector<byte>, Vector<ushort>, Vector<uint> and Vector<ulong>:
1446             // SSE2 supports > for signed comparison. Therefore, to use it for
1447             // comparing unsigned numbers, we subtract a constant from both the
1448             // operands such that the result fits within the corresponding signed
1449             // type.  The resulting signed numbers are compared using SSE2 signed
1450             // comparison.
1451             //
1452             // Vector<byte>: constant to be subtracted is 2^7
1453             // Vector<ushort> constant to be subtracted is 2^15
1454             // Vector<uint> constant to be subtracted is 2^31
1455             // Vector<ulong> constant to be subtracted is 2^63
1456             //
1457             // We need to treat op1 and op2 as signed for comparison purpose after
1458             // the transformation.
1459             __int64 constVal = 0;
1460             switch (baseType)
1461             {
1462                 case TYP_UBYTE:
1463                     constVal       = 0x80808080;
1464                     *inOutBaseType = TYP_BYTE;
1465                     break;
1466                 case TYP_USHORT:
1467                     constVal       = 0x80008000;
1468                     *inOutBaseType = TYP_SHORT;
1469                     break;
1470                 case TYP_UINT:
1471                     constVal       = 0x80000000;
1472                     *inOutBaseType = TYP_INT;
1473                     break;
1474                 case TYP_ULONG:
1475                     constVal       = 0x8000000000000000LL;
1476                     *inOutBaseType = TYP_LONG;
1477                     break;
1478                 default:
1479                     unreached();
1480                     break;
1481             }
1482             assert(constVal != 0);
1483
1484             // This transformation is not required for equality.
1485             if (intrinsicID != SIMDIntrinsicEqual)
1486             {
1487                 // For constructing const vector use either long or int base type.
1488                 var_types tempBaseType;
1489                 GenTree*  initVal;
1490                 if (baseType == TYP_ULONG)
1491                 {
1492                     tempBaseType = TYP_LONG;
1493                     initVal      = gtNewLconNode(constVal);
1494                 }
1495                 else
1496                 {
1497                     tempBaseType = TYP_INT;
1498                     initVal      = gtNewIconNode((ssize_t)constVal);
1499                 }
1500                 initVal->gtType      = tempBaseType;
1501                 GenTree* constVector = gtNewSIMDNode(simdType, initVal, nullptr, SIMDIntrinsicInit, tempBaseType, size);
1502
1503                 // Assign constVector to a temp, since we intend to use it more than once
1504                 // TODO-CQ: We have quite a few such constant vectors constructed during
1505                 // the importation of SIMD intrinsics.  Make sure that we have a single
1506                 // temp per distinct constant per method.
1507                 GenTree* tmp = fgInsertCommaFormTemp(&constVector, typeHnd);
1508
1509                 // op1 = op1 - constVector
1510                 // op2 = op2 - constVector
1511                 *pOp1 = gtNewSIMDNode(simdType, *pOp1, constVector, SIMDIntrinsicSub, baseType, size);
1512                 *pOp2 = gtNewSIMDNode(simdType, *pOp2, tmp, SIMDIntrinsicSub, baseType, size);
1513             }
1514
1515             return impSIMDRelOp(intrinsicID, typeHnd, size, inOutBaseType, pOp1, pOp2);
1516         }
1517     }
1518 #elif defined(_TARGET_ARM64_)
1519     // TODO-ARM64-CQ handle comparisons against zero
1520
1521     // _TARGET_ARM64_ doesn't support < and <= on register register comparisons
1522     // Therefore, we need to use > and >= with swapped operands.
1523     if (intrinsicID == SIMDIntrinsicLessThan || intrinsicID == SIMDIntrinsicLessThanOrEqual)
1524     {
1525         GenTree* tmp = *pOp1;
1526         *pOp1        = *pOp2;
1527         *pOp2        = tmp;
1528
1529         intrinsicID =
1530             (intrinsicID == SIMDIntrinsicLessThan) ? SIMDIntrinsicGreaterThan : SIMDIntrinsicGreaterThanOrEqual;
1531     }
1532 #else  // !_TARGET_XARCH_
1533     assert(!"impSIMDRelOp() unimplemented on target arch");
1534     unreached();
1535 #endif // !_TARGET_XARCH_
1536
1537     return intrinsicID;
1538 }
1539
1540 //-------------------------------------------------------------------------
1541 // impSIMDAbs: creates GT_SIMD node to compute Abs value of a given vector.
1542 //
1543 // Arguments:
1544 //    typeHnd     -  type handle of SIMD vector
1545 //    baseType    -  base type of vector
1546 //    size        -  vector size in bytes
1547 //    op1         -  operand of Abs intrinsic
1548 //
1549 GenTree* Compiler::impSIMDAbs(CORINFO_CLASS_HANDLE typeHnd, var_types baseType, unsigned size, GenTree* op1)
1550 {
1551     assert(varTypeIsSIMD(op1));
1552
1553     var_types simdType = op1->TypeGet();
1554     GenTree*  retVal   = nullptr;
1555
1556 #ifdef _TARGET_XARCH_
1557     // When there is no direct support, Abs(v) could be computed
1558     // on integer vectors as follows:
1559     //     BitVector = v < vector.Zero
1560     //     result = ConditionalSelect(BitVector, vector.Zero - v, v)
1561
1562     bool useConditionalSelect = false;
1563     if (getSIMDSupportLevel() == SIMD_SSE2_Supported)
1564     {
1565         // SSE2 doesn't support abs on signed integer type vectors.
1566         if (baseType == TYP_LONG || baseType == TYP_INT || baseType == TYP_SHORT || baseType == TYP_BYTE)
1567         {
1568             useConditionalSelect = true;
1569         }
1570     }
1571     else
1572     {
1573         assert(getSIMDSupportLevel() >= SIMD_SSE4_Supported);
1574         if (baseType == TYP_LONG)
1575         {
1576             // SSE4/AVX2 don't support abs on long type vector.
1577             useConditionalSelect = true;
1578         }
1579     }
1580
1581     if (useConditionalSelect)
1582     {
1583         // This works only on integer vectors not on float/double vectors.
1584         assert(varTypeIsIntegral(baseType));
1585
1586         GenTree* op1Assign;
1587         unsigned op1LclNum;
1588
1589         if (op1->OperGet() == GT_LCL_VAR)
1590         {
1591             op1LclNum = op1->gtLclVarCommon.gtLclNum;
1592             op1Assign = nullptr;
1593         }
1594         else
1595         {
1596             op1LclNum = lvaGrabTemp(true DEBUGARG("SIMD Abs op1"));
1597             lvaSetStruct(op1LclNum, typeHnd, false);
1598             op1Assign = gtNewTempAssign(op1LclNum, op1);
1599             op1       = gtNewLclvNode(op1LclNum, op1->TypeGet());
1600         }
1601
1602         // Assign Vector.Zero to a temp since it is needed more than once
1603         GenTree* vecZero       = gtNewSIMDVectorZero(simdType, baseType, size);
1604         unsigned vecZeroLclNum = lvaGrabTemp(true DEBUGARG("SIMD Abs VecZero"));
1605         lvaSetStruct(vecZeroLclNum, typeHnd, false);
1606         GenTree* vecZeroAssign = gtNewTempAssign(vecZeroLclNum, vecZero);
1607
1608         // Construct BitVector = v < vector.Zero
1609         GenTree*        bitVecOp1     = op1;
1610         GenTree*        bitVecOp2     = gtNewLclvNode(vecZeroLclNum, vecZero->TypeGet());
1611         var_types       relOpBaseType = baseType;
1612         SIMDIntrinsicID relOpIntrinsic =
1613             impSIMDRelOp(SIMDIntrinsicLessThan, typeHnd, size, &relOpBaseType, &bitVecOp1, &bitVecOp2);
1614         GenTree* bitVec       = gtNewSIMDNode(simdType, bitVecOp1, bitVecOp2, relOpIntrinsic, relOpBaseType, size);
1615         unsigned bitVecLclNum = lvaGrabTemp(true DEBUGARG("SIMD Abs bitVec"));
1616         lvaSetStruct(bitVecLclNum, typeHnd, false);
1617         GenTree* bitVecAssign = gtNewTempAssign(bitVecLclNum, bitVec);
1618         bitVec                = gtNewLclvNode(bitVecLclNum, bitVec->TypeGet());
1619
1620         // Construct condSelectOp1 = vector.Zero - v
1621         GenTree* subOp1 = gtNewLclvNode(vecZeroLclNum, vecZero->TypeGet());
1622         GenTree* subOp2 = gtNewLclvNode(op1LclNum, op1->TypeGet());
1623         GenTree* negVec = gtNewSIMDNode(simdType, subOp1, subOp2, SIMDIntrinsicSub, baseType, size);
1624
1625         // Construct ConditionalSelect(bitVec, vector.Zero - v, v)
1626         GenTree* vec = gtNewLclvNode(op1LclNum, op1->TypeGet());
1627         retVal       = impSIMDSelect(typeHnd, baseType, size, bitVec, negVec, vec);
1628
1629         // Prepend bitVec assignment to retVal.
1630         // retVal = (tmp2 = v < tmp1), CondSelect(tmp2, tmp1 - v, v)
1631         retVal = gtNewOperNode(GT_COMMA, simdType, bitVecAssign, retVal);
1632
1633         // Prepend vecZero assignment to retVal.
1634         // retVal =  (tmp1 = vector.Zero), (tmp2 = v < tmp1), CondSelect(tmp2, tmp1 - v, v)
1635         retVal = gtNewOperNode(GT_COMMA, simdType, vecZeroAssign, retVal);
1636
1637         // If op1 was assigned to a temp, prepend that to retVal.
1638         if (op1Assign != nullptr)
1639         {
1640             // retVal = (v=op1), (tmp1 = vector.Zero), (tmp2 = v < tmp1), CondSelect(tmp2, tmp1 - v, v)
1641             retVal = gtNewOperNode(GT_COMMA, simdType, op1Assign, retVal);
1642         }
1643     }
1644     else if (varTypeIsFloating(baseType))
1645     {
1646         // Abs(vf) = vf & new SIMDVector<float>(0x7fffffff);
1647         // Abs(vd) = vf & new SIMDVector<double>(0x7fffffffffffffff);
1648         GenTree* bitMask = nullptr;
1649         if (baseType == TYP_FLOAT)
1650         {
1651             float f;
1652             static_assert_no_msg(sizeof(float) == sizeof(int));
1653             *((int*)&f) = 0x7fffffff;
1654             bitMask     = gtNewDconNode(f);
1655         }
1656         else if (baseType == TYP_DOUBLE)
1657         {
1658             double d;
1659             static_assert_no_msg(sizeof(double) == sizeof(__int64));
1660             *((__int64*)&d) = 0x7fffffffffffffffLL;
1661             bitMask         = gtNewDconNode(d);
1662         }
1663
1664         assert(bitMask != nullptr);
1665         bitMask->gtType        = baseType;
1666         GenTree* bitMaskVector = gtNewSIMDNode(simdType, bitMask, SIMDIntrinsicInit, baseType, size);
1667         retVal                 = gtNewSIMDNode(simdType, op1, bitMaskVector, SIMDIntrinsicBitwiseAnd, baseType, size);
1668     }
1669     else if (baseType == TYP_USHORT || baseType == TYP_UBYTE || baseType == TYP_UINT || baseType == TYP_ULONG)
1670     {
1671         // Abs is a no-op on unsigned integer type vectors
1672         retVal = op1;
1673     }
1674     else
1675     {
1676         assert(getSIMDSupportLevel() >= SIMD_SSE4_Supported);
1677         assert(baseType != TYP_LONG);
1678
1679         retVal = gtNewSIMDNode(simdType, op1, SIMDIntrinsicAbs, baseType, size);
1680     }
1681 #elif defined(_TARGET_ARM64_)
1682     if (varTypeIsUnsigned(baseType))
1683     {
1684         // Abs is a no-op on unsigned integer type vectors
1685         retVal = op1;
1686     }
1687     else
1688     {
1689         retVal = gtNewSIMDNode(simdType, op1, SIMDIntrinsicAbs, baseType, size);
1690     }
1691 #else  // !defined(_TARGET_XARCH)_ && !defined(_TARGET_ARM64_)
1692     assert(!"Abs intrinsic on non-xarch target not implemented");
1693 #endif // !_TARGET_XARCH_
1694
1695     return retVal;
1696 }
1697
1698 // Creates a GT_SIMD tree for Select operation
1699 //
1700 // Argumens:
1701 //    typeHnd          -  type handle of SIMD vector
1702 //    baseType         -  base type of SIMD vector
1703 //    size             -  SIMD vector size
1704 //    op1              -  first operand = Condition vector vc
1705 //    op2              -  second operand = va
1706 //    op3              -  third operand = vb
1707 //
1708 // Return Value:
1709 //    Returns GT_SIMD tree that computes Select(vc, va, vb)
1710 //
1711 GenTree* Compiler::impSIMDSelect(
1712     CORINFO_CLASS_HANDLE typeHnd, var_types baseType, unsigned size, GenTree* op1, GenTree* op2, GenTree* op3)
1713 {
1714     assert(varTypeIsSIMD(op1));
1715     var_types simdType = op1->TypeGet();
1716     assert(op2->TypeGet() == simdType);
1717     assert(op3->TypeGet() == simdType);
1718
1719     // TODO-ARM64-CQ Support generating select instruction for SIMD
1720
1721     // Select(BitVector vc, va, vb) = (va & vc) | (vb & !vc)
1722     // Select(op1, op2, op3)        = (op2 & op1) | (op3 & !op1)
1723     //                              = SIMDIntrinsicBitwiseOr(SIMDIntrinsicBitwiseAnd(op2, op1),
1724     //                                                       SIMDIntrinsicBitwiseAndNot(op3, op1))
1725     //
1726     // If Op1 has side effect, create an assignment to a temp
1727     GenTree* tmp = op1;
1728     GenTree* asg = nullptr;
1729     if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0)
1730     {
1731         unsigned lclNum = lvaGrabTemp(true DEBUGARG("SIMD Select"));
1732         lvaSetStruct(lclNum, typeHnd, false);
1733         tmp = gtNewLclvNode(lclNum, op1->TypeGet());
1734         asg = gtNewTempAssign(lclNum, op1);
1735     }
1736
1737     GenTree* andExpr = gtNewSIMDNode(simdType, op2, tmp, SIMDIntrinsicBitwiseAnd, baseType, size);
1738     GenTree* dupOp1  = gtCloneExpr(tmp);
1739     assert(dupOp1 != nullptr);
1740 #ifdef _TARGET_ARM64_
1741     // ARM64 implements SIMDIntrinsicBitwiseAndNot as Left & ~Right
1742     GenTree* andNotExpr = gtNewSIMDNode(simdType, op3, dupOp1, SIMDIntrinsicBitwiseAndNot, baseType, size);
1743 #else
1744     // XARCH implements SIMDIntrinsicBitwiseAndNot as ~Left & Right
1745     GenTree* andNotExpr = gtNewSIMDNode(simdType, dupOp1, op3, SIMDIntrinsicBitwiseAndNot, baseType, size);
1746 #endif
1747     GenTree* simdTree = gtNewSIMDNode(simdType, andExpr, andNotExpr, SIMDIntrinsicBitwiseOr, baseType, size);
1748
1749     // If asg not null, create a GT_COMMA tree.
1750     if (asg != nullptr)
1751     {
1752         simdTree = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), asg, simdTree);
1753     }
1754
1755     return simdTree;
1756 }
1757
1758 // Creates a GT_SIMD tree for Min/Max operation
1759 //
1760 // Argumens:
1761 //    IntrinsicId      -  SIMD intrinsic Id, either Min or Max
1762 //    typeHnd          -  type handle of SIMD vector
1763 //    baseType         -  base type of SIMD vector
1764 //    size             -  SIMD vector size
1765 //    op1              -  first operand = va
1766 //    op2              -  second operand = vb
1767 //
1768 // Return Value:
1769 //    Returns GT_SIMD tree that computes Max(va, vb)
1770 //
1771 GenTree* Compiler::impSIMDMinMax(SIMDIntrinsicID      intrinsicId,
1772                                  CORINFO_CLASS_HANDLE typeHnd,
1773                                  var_types            baseType,
1774                                  unsigned             size,
1775                                  GenTree*             op1,
1776                                  GenTree*             op2)
1777 {
1778     assert(intrinsicId == SIMDIntrinsicMin || intrinsicId == SIMDIntrinsicMax);
1779     assert(varTypeIsSIMD(op1));
1780     var_types simdType = op1->TypeGet();
1781     assert(op2->TypeGet() == simdType);
1782
1783 #if defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
1784     GenTree* simdTree = nullptr;
1785
1786 #ifdef _TARGET_XARCH_
1787     // SSE2 has direct support for float/double/signed word/unsigned byte.
1788     // SSE4.1 has direct support for int32/uint32/signed byte/unsigned word.
1789     // For other integer types we compute min/max as follows
1790     //
1791     // int32/uint32 (SSE2)
1792     // int64/uint64 (SSE2&SSE4):
1793     //       compResult        = (op1 < op2) in case of Min
1794     //                           (op1 > op2) in case of Max
1795     //       Min/Max(op1, op2) = Select(compResult, op1, op2)
1796     //
1797     // unsigned word (SSE2):
1798     //        op1 = op1 - 2^15  ; to make it fit within a signed word
1799     //        op2 = op2 - 2^15  ; to make it fit within a signed word
1800     //        result = SSE2 signed word Min/Max(op1, op2)
1801     //        result = result + 2^15  ; readjust it back
1802     //
1803     // signed byte (SSE2):
1804     //        op1 = op1 + 2^7  ; to make it unsigned
1805     //        op1 = op1 + 2^7  ; to make it unsigned
1806     //        result = SSE2 unsigned byte Min/Max(op1, op2)
1807     //        result = result - 2^15 ; readjust it back
1808
1809     if (varTypeIsFloating(baseType) || baseType == TYP_SHORT || baseType == TYP_UBYTE ||
1810         (getSIMDSupportLevel() >= SIMD_SSE4_Supported &&
1811          (baseType == TYP_BYTE || baseType == TYP_INT || baseType == TYP_UINT || baseType == TYP_USHORT)))
1812     {
1813         // SSE2 or SSE4.1 has direct support
1814         simdTree = gtNewSIMDNode(simdType, op1, op2, intrinsicId, baseType, size);
1815     }
1816     else if (baseType == TYP_USHORT || baseType == TYP_BYTE)
1817     {
1818         assert(getSIMDSupportLevel() == SIMD_SSE2_Supported);
1819         int             constVal;
1820         SIMDIntrinsicID operIntrinsic;
1821         SIMDIntrinsicID adjustIntrinsic;
1822         var_types       minMaxOperBaseType;
1823         if (baseType == TYP_USHORT)
1824         {
1825             constVal           = 0x80008000;
1826             operIntrinsic      = SIMDIntrinsicSub;
1827             adjustIntrinsic    = SIMDIntrinsicAdd;
1828             minMaxOperBaseType = TYP_SHORT;
1829         }
1830         else
1831         {
1832             assert(baseType == TYP_BYTE);
1833             constVal           = 0x80808080;
1834             operIntrinsic      = SIMDIntrinsicAdd;
1835             adjustIntrinsic    = SIMDIntrinsicSub;
1836             minMaxOperBaseType = TYP_UBYTE;
1837         }
1838
1839         GenTree* initVal     = gtNewIconNode(constVal);
1840         GenTree* constVector = gtNewSIMDNode(simdType, initVal, nullptr, SIMDIntrinsicInit, TYP_INT, size);
1841
1842         // Assign constVector to a temp, since we intend to use it more than once
1843         // TODO-CQ: We have quite a few such constant vectors constructed during
1844         // the importation of SIMD intrinsics.  Make sure that we have a single
1845         // temp per distinct constant per method.
1846         GenTree* tmp = fgInsertCommaFormTemp(&constVector, typeHnd);
1847
1848         // op1 = op1 - constVector
1849         // op2 = op2 - constVector
1850         op1 = gtNewSIMDNode(simdType, op1, constVector, operIntrinsic, baseType, size);
1851         op2 = gtNewSIMDNode(simdType, op2, tmp, operIntrinsic, baseType, size);
1852
1853         // compute min/max of op1 and op2 considering them as if minMaxOperBaseType
1854         simdTree = gtNewSIMDNode(simdType, op1, op2, intrinsicId, minMaxOperBaseType, size);
1855
1856         // re-adjust the value by adding or subtracting constVector
1857         tmp      = gtNewLclvNode(tmp->AsLclVarCommon()->GetLclNum(), tmp->TypeGet());
1858         simdTree = gtNewSIMDNode(simdType, simdTree, tmp, adjustIntrinsic, baseType, size);
1859     }
1860 #elif defined(_TARGET_ARM64_)
1861     // Arm64 has direct support for all types except int64/uint64
1862     // For which we compute min/max as follows
1863     //
1864     // int64/uint64
1865     //       compResult        = (op1 < op2) in case of Min
1866     //                           (op1 > op2) in case of Max
1867     //       Min/Max(op1, op2) = Select(compResult, op1, op2)
1868     if (baseType != TYP_ULONG && baseType != TYP_LONG)
1869     {
1870         simdTree = gtNewSIMDNode(simdType, op1, op2, intrinsicId, baseType, size);
1871     }
1872 #endif
1873     else
1874     {
1875         GenTree* dupOp1    = nullptr;
1876         GenTree* dupOp2    = nullptr;
1877         GenTree* op1Assign = nullptr;
1878         GenTree* op2Assign = nullptr;
1879         unsigned op1LclNum;
1880         unsigned op2LclNum;
1881
1882         if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0)
1883         {
1884             op1LclNum = lvaGrabTemp(true DEBUGARG("SIMD Min/Max"));
1885             dupOp1    = gtNewLclvNode(op1LclNum, op1->TypeGet());
1886             lvaSetStruct(op1LclNum, typeHnd, false);
1887             op1Assign = gtNewTempAssign(op1LclNum, op1);
1888             op1       = gtNewLclvNode(op1LclNum, op1->TypeGet());
1889         }
1890         else
1891         {
1892             dupOp1 = gtCloneExpr(op1);
1893         }
1894
1895         if ((op2->gtFlags & GTF_SIDE_EFFECT) != 0)
1896         {
1897             op2LclNum = lvaGrabTemp(true DEBUGARG("SIMD Min/Max"));
1898             dupOp2    = gtNewLclvNode(op2LclNum, op2->TypeGet());
1899             lvaSetStruct(op2LclNum, typeHnd, false);
1900             op2Assign = gtNewTempAssign(op2LclNum, op2);
1901             op2       = gtNewLclvNode(op2LclNum, op2->TypeGet());
1902         }
1903         else
1904         {
1905             dupOp2 = gtCloneExpr(op2);
1906         }
1907
1908         SIMDIntrinsicID relOpIntrinsic =
1909             (intrinsicId == SIMDIntrinsicMin) ? SIMDIntrinsicLessThan : SIMDIntrinsicGreaterThan;
1910         var_types relOpBaseType = baseType;
1911
1912         // compResult = op1 relOp op2
1913         // simdTree = Select(compResult, op1, op2);
1914         assert(dupOp1 != nullptr);
1915         assert(dupOp2 != nullptr);
1916         relOpIntrinsic            = impSIMDRelOp(relOpIntrinsic, typeHnd, size, &relOpBaseType, &dupOp1, &dupOp2);
1917         GenTree* compResult       = gtNewSIMDNode(simdType, dupOp1, dupOp2, relOpIntrinsic, relOpBaseType, size);
1918         unsigned compResultLclNum = lvaGrabTemp(true DEBUGARG("SIMD Min/Max"));
1919         lvaSetStruct(compResultLclNum, typeHnd, false);
1920         GenTree* compResultAssign = gtNewTempAssign(compResultLclNum, compResult);
1921         compResult                = gtNewLclvNode(compResultLclNum, compResult->TypeGet());
1922         simdTree                  = impSIMDSelect(typeHnd, baseType, size, compResult, op1, op2);
1923         simdTree                  = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), compResultAssign, simdTree);
1924
1925         // Now create comma trees if we have created assignments of op1/op2 to temps
1926         if (op2Assign != nullptr)
1927         {
1928             simdTree = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), op2Assign, simdTree);
1929         }
1930
1931         if (op1Assign != nullptr)
1932         {
1933             simdTree = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), op1Assign, simdTree);
1934         }
1935     }
1936
1937     assert(simdTree != nullptr);
1938     return simdTree;
1939 #else  // !(defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_))
1940     assert(!"impSIMDMinMax() unimplemented on target arch");
1941     unreached();
1942 #endif // !(defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_))
1943 }
1944
1945 //------------------------------------------------------------------------
1946 // getOp1ForConstructor: Get the op1 for a constructor call.
1947 //
1948 // Arguments:
1949 //    opcode     - the opcode being handled (needed to identify the CEE_NEWOBJ case)
1950 //    newobjThis - For CEE_NEWOBJ, this is the temp grabbed for the allocated uninitalized object.
1951 //    clsHnd    - The handle of the class of the method.
1952 //
1953 // Return Value:
1954 //    The tree node representing the object to be initialized with the constructor.
1955 //
1956 // Notes:
1957 //    This method handles the differences between the CEE_NEWOBJ and constructor cases.
1958 //
1959 GenTree* Compiler::getOp1ForConstructor(OPCODE opcode, GenTree* newobjThis, CORINFO_CLASS_HANDLE clsHnd)
1960 {
1961     GenTree* op1;
1962     if (opcode == CEE_NEWOBJ)
1963     {
1964         op1 = newobjThis;
1965         assert(newobjThis->gtOper == GT_ADDR && newobjThis->gtOp.gtOp1->gtOper == GT_LCL_VAR);
1966
1967         // push newobj result on type stack
1968         unsigned tmp = op1->gtOp.gtOp1->gtLclVarCommon.gtLclNum;
1969         impPushOnStack(gtNewLclvNode(tmp, lvaGetRealType(tmp)), verMakeTypeInfo(clsHnd).NormaliseForStack());
1970     }
1971     else
1972     {
1973         op1 = impSIMDPopStack(TYP_BYREF);
1974     }
1975     assert(op1->TypeGet() == TYP_BYREF);
1976     return op1;
1977 }
1978
1979 //-------------------------------------------------------------------
1980 // Set the flag that indicates that the lclVar referenced by this tree
1981 // is used in a SIMD intrinsic.
1982 // Arguments:
1983 //      tree - GenTree*
1984
1985 void Compiler::setLclRelatedToSIMDIntrinsic(GenTree* tree)
1986 {
1987     assert(tree->OperIsLocal());
1988     unsigned   lclNum                = tree->AsLclVarCommon()->GetLclNum();
1989     LclVarDsc* lclVarDsc             = &lvaTable[lclNum];
1990     lclVarDsc->lvUsedInSIMDIntrinsic = true;
1991 }
1992
1993 //-------------------------------------------------------------
1994 // Check if two field nodes reference at the same memory location.
1995 // Notice that this check is just based on pattern matching.
1996 // Arguments:
1997 //      op1 - GenTree*.
1998 //      op2 - GenTree*.
1999 // Return Value:
2000 //    If op1's parents node and op2's parents node are at the same location, return true. Otherwise, return false
2001
2002 bool areFieldsParentsLocatedSame(GenTree* op1, GenTree* op2)
2003 {
2004     assert(op1->OperGet() == GT_FIELD);
2005     assert(op2->OperGet() == GT_FIELD);
2006
2007     GenTree* op1ObjRef = op1->gtField.gtFldObj;
2008     GenTree* op2ObjRef = op2->gtField.gtFldObj;
2009     while (op1ObjRef != nullptr && op2ObjRef != nullptr)
2010     {
2011
2012         if (op1ObjRef->OperGet() != op2ObjRef->OperGet())
2013         {
2014             break;
2015         }
2016         else if (op1ObjRef->OperGet() == GT_ADDR)
2017         {
2018             op1ObjRef = op1ObjRef->gtOp.gtOp1;
2019             op2ObjRef = op2ObjRef->gtOp.gtOp1;
2020         }
2021
2022         if (op1ObjRef->OperIsLocal() && op2ObjRef->OperIsLocal() &&
2023             op1ObjRef->AsLclVarCommon()->GetLclNum() == op2ObjRef->AsLclVarCommon()->GetLclNum())
2024         {
2025             return true;
2026         }
2027         else if (op1ObjRef->OperGet() == GT_FIELD && op2ObjRef->OperGet() == GT_FIELD &&
2028                  op1ObjRef->gtField.gtFldHnd == op2ObjRef->gtField.gtFldHnd)
2029         {
2030             op1ObjRef = op1ObjRef->gtField.gtFldObj;
2031             op2ObjRef = op2ObjRef->gtField.gtFldObj;
2032             continue;
2033         }
2034         else
2035         {
2036             break;
2037         }
2038     }
2039
2040     return false;
2041 }
2042
2043 //----------------------------------------------------------------------
2044 // Check whether two field are contiguous
2045 // Arguments:
2046 //      first - GenTree*. The Type of the node should be TYP_FLOAT
2047 //      second - GenTree*. The Type of the node should be TYP_FLOAT
2048 // Return Value:
2049 //      if the first field is located before second field, and they are located contiguously,
2050 //      then return true. Otherwise, return false.
2051
2052 bool Compiler::areFieldsContiguous(GenTree* first, GenTree* second)
2053 {
2054     assert(first->OperGet() == GT_FIELD);
2055     assert(second->OperGet() == GT_FIELD);
2056     assert(first->gtType == TYP_FLOAT);
2057     assert(second->gtType == TYP_FLOAT);
2058
2059     var_types firstFieldType  = first->gtType;
2060     var_types secondFieldType = second->gtType;
2061
2062     unsigned firstFieldEndOffset = first->gtField.gtFldOffset + genTypeSize(firstFieldType);
2063     unsigned secondFieldOffset   = second->gtField.gtFldOffset;
2064     if (firstFieldEndOffset == secondFieldOffset && firstFieldType == secondFieldType &&
2065         areFieldsParentsLocatedSame(first, second))
2066     {
2067         return true;
2068     }
2069
2070     return false;
2071 }
2072
2073 //-------------------------------------------------------------------------------
2074 // Check whether two array element nodes are located contiguously or not.
2075 // Arguments:
2076 //      op1 - GenTree*.
2077 //      op2 - GenTree*.
2078 // Return Value:
2079 //      if the array element op1 is located before array element op2, and they are contiguous,
2080 //      then return true. Otherwise, return false.
2081 // TODO-CQ:
2082 //      Right this can only check array element with const number as index. In future,
2083 //      we should consider to allow this function to check the index using expression.
2084
2085 bool Compiler::areArrayElementsContiguous(GenTree* op1, GenTree* op2)
2086 {
2087     noway_assert(op1->gtOper == GT_INDEX);
2088     noway_assert(op2->gtOper == GT_INDEX);
2089     GenTreeIndex* op1Index = op1->AsIndex();
2090     GenTreeIndex* op2Index = op2->AsIndex();
2091
2092     GenTree* op1ArrayRef = op1Index->Arr();
2093     GenTree* op2ArrayRef = op2Index->Arr();
2094     assert(op1ArrayRef->TypeGet() == TYP_REF);
2095     assert(op2ArrayRef->TypeGet() == TYP_REF);
2096
2097     GenTree* op1IndexNode = op1Index->Index();
2098     GenTree* op2IndexNode = op2Index->Index();
2099     if ((op1IndexNode->OperGet() == GT_CNS_INT && op2IndexNode->OperGet() == GT_CNS_INT) &&
2100         op1IndexNode->gtIntCon.gtIconVal + 1 == op2IndexNode->gtIntCon.gtIconVal)
2101     {
2102         if (op1ArrayRef->OperGet() == GT_FIELD && op2ArrayRef->OperGet() == GT_FIELD &&
2103             areFieldsParentsLocatedSame(op1ArrayRef, op2ArrayRef))
2104         {
2105             return true;
2106         }
2107         else if (op1ArrayRef->OperIsLocal() && op2ArrayRef->OperIsLocal() &&
2108                  op1ArrayRef->AsLclVarCommon()->GetLclNum() == op2ArrayRef->AsLclVarCommon()->GetLclNum())
2109         {
2110             return true;
2111         }
2112     }
2113     return false;
2114 }
2115
2116 //-------------------------------------------------------------------------------
2117 // Check whether two argument nodes are contiguous or not.
2118 // Arguments:
2119 //      op1 - GenTree*.
2120 //      op2 - GenTree*.
2121 // Return Value:
2122 //      if the argument node op1 is located before argument node op2, and they are located contiguously,
2123 //      then return true. Otherwise, return false.
2124 // TODO-CQ:
2125 //      Right now this can only check field and array. In future we should add more cases.
2126 //
2127
2128 bool Compiler::areArgumentsContiguous(GenTree* op1, GenTree* op2)
2129 {
2130     if (op1->OperGet() == GT_INDEX && op2->OperGet() == GT_INDEX)
2131     {
2132         return areArrayElementsContiguous(op1, op2);
2133     }
2134     else if (op1->OperGet() == GT_FIELD && op2->OperGet() == GT_FIELD)
2135     {
2136         return areFieldsContiguous(op1, op2);
2137     }
2138     return false;
2139 }
2140
2141 //--------------------------------------------------------------------------------------------------------
2142 // createAddressNodeForSIMDInit: Generate the address node(GT_LEA) if we want to intialize vector2, vector3 or vector4
2143 // from first argument's address.
2144 //
2145 // Arguments:
2146 //      tree - GenTree*. This the tree node which is used to get the address for indir.
2147 //      simdsize - unsigned. This the simd vector size.
2148 //      arrayElementsCount - unsigned. This is used for generating the boundary check for array.
2149 //
2150 // Return value:
2151 //      return the address node.
2152 //
2153 // TODO-CQ:
2154 //      1. Currently just support for GT_FIELD and GT_INDEX, because we can only verify the GT_INDEX node or GT_Field
2155 //         are located contiguously or not. In future we should support more cases.
2156 //      2. Though it happens to just work fine front-end phases are not aware of GT_LEA node.  Therefore, convert these
2157 //         to use GT_ADDR.
2158 GenTree* Compiler::createAddressNodeForSIMDInit(GenTree* tree, unsigned simdSize)
2159 {
2160     assert(tree->OperGet() == GT_FIELD || tree->OperGet() == GT_INDEX);
2161     GenTree*  byrefNode  = nullptr;
2162     GenTree*  startIndex = nullptr;
2163     unsigned  offset     = 0;
2164     var_types baseType   = tree->gtType;
2165
2166     if (tree->OperGet() == GT_FIELD)
2167     {
2168         GenTree* objRef = tree->gtField.gtFldObj;
2169         if (objRef != nullptr && objRef->gtOper == GT_ADDR)
2170         {
2171             GenTree* obj = objRef->gtOp.gtOp1;
2172
2173             // If the field is directly from a struct, then in this case,
2174             // we should set this struct's lvUsedInSIMDIntrinsic as true,
2175             // so that this sturct won't be promoted.
2176             // e.g. s.x x is a field, and s is a struct, then we should set the s's lvUsedInSIMDIntrinsic as true.
2177             // so that s won't be promoted.
2178             // Notice that if we have a case like s1.s2.x. s1 s2 are struct, and x is a field, then it is possible that
2179             // s1 can be promoted, so that s2 can be promoted. The reason for that is if we don't allow s1 to be
2180             // promoted, then this will affect the other optimizations which are depend on s1's struct promotion.
2181             // TODO-CQ:
2182             //  In future, we should optimize this case so that if there is a nested field like s1.s2.x and s1.s2.x's
2183             //  address is used for initializing the vector, then s1 can be promoted but s2 can't.
2184             if (varTypeIsSIMD(obj) && obj->OperIsLocal())
2185             {
2186                 setLclRelatedToSIMDIntrinsic(obj);
2187             }
2188         }
2189
2190         byrefNode = gtCloneExpr(tree->gtField.gtFldObj);
2191         assert(byrefNode != nullptr);
2192         offset = tree->gtField.gtFldOffset;
2193     }
2194     else if (tree->OperGet() == GT_INDEX)
2195     {
2196
2197         GenTree* index = tree->AsIndex()->Index();
2198         assert(index->OperGet() == GT_CNS_INT);
2199
2200         GenTree* checkIndexExpr = nullptr;
2201         unsigned indexVal       = (unsigned)(index->gtIntCon.gtIconVal);
2202         offset                  = indexVal * genTypeSize(tree->TypeGet());
2203         GenTree* arrayRef       = tree->AsIndex()->Arr();
2204
2205         // Generate the boundary check exception.
2206         // The length for boundary check should be the maximum index number which should be
2207         // (first argument's index number) + (how many array arguments we have) - 1
2208         // = indexVal + arrayElementsCount - 1
2209         unsigned arrayElementsCount  = simdSize / genTypeSize(baseType);
2210         checkIndexExpr               = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, indexVal + arrayElementsCount - 1);
2211         GenTreeArrLen*    arrLen     = gtNewArrLen(TYP_INT, arrayRef, (int)OFFSETOF__CORINFO_Array__length);
2212         GenTreeBoundsChk* arrBndsChk = new (this, GT_ARR_BOUNDS_CHECK)
2213             GenTreeBoundsChk(GT_ARR_BOUNDS_CHECK, TYP_VOID, checkIndexExpr, arrLen, SCK_RNGCHK_FAIL);
2214
2215         offset += OFFSETOF__CORINFO_Array__data;
2216         byrefNode = gtNewOperNode(GT_COMMA, arrayRef->TypeGet(), arrBndsChk, gtCloneExpr(arrayRef));
2217     }
2218     else
2219     {
2220         unreached();
2221     }
2222     GenTree* address =
2223         new (this, GT_LEA) GenTreeAddrMode(TYP_BYREF, byrefNode, startIndex, genTypeSize(tree->TypeGet()), offset);
2224     return address;
2225 }
2226
2227 //-------------------------------------------------------------------------------
2228 // impMarkContiguousSIMDFieldAssignments: Try to identify if there are contiguous
2229 // assignments from SIMD field to memory. If there are, then mark the related
2230 // lclvar so that it won't be promoted.
2231 //
2232 // Arguments:
2233 //      stmt - GenTree*. Input statement node.
2234
2235 void Compiler::impMarkContiguousSIMDFieldAssignments(GenTree* stmt)
2236 {
2237     if (!featureSIMD || opts.MinOpts())
2238     {
2239         return;
2240     }
2241     GenTree* expr = stmt->gtStmt.gtStmtExpr;
2242     if (expr->OperGet() == GT_ASG && expr->TypeGet() == TYP_FLOAT)
2243     {
2244         GenTree*  curDst            = expr->gtOp.gtOp1;
2245         GenTree*  curSrc            = expr->gtOp.gtOp2;
2246         unsigned  index             = 0;
2247         var_types baseType          = TYP_UNKNOWN;
2248         unsigned  simdSize          = 0;
2249         GenTree*  srcSimdStructNode = getSIMDStructFromField(curSrc, &baseType, &index, &simdSize, true);
2250         if (srcSimdStructNode == nullptr || baseType != TYP_FLOAT)
2251         {
2252             fgPreviousCandidateSIMDFieldAsgStmt = nullptr;
2253         }
2254         else if (index == 0 && isSIMDTypeLocal(srcSimdStructNode))
2255         {
2256             fgPreviousCandidateSIMDFieldAsgStmt = stmt;
2257         }
2258         else if (fgPreviousCandidateSIMDFieldAsgStmt != nullptr)
2259         {
2260             assert(index > 0);
2261             GenTree* prevAsgExpr = fgPreviousCandidateSIMDFieldAsgStmt->gtStmt.gtStmtExpr;
2262             GenTree* prevDst     = prevAsgExpr->gtOp.gtOp1;
2263             GenTree* prevSrc     = prevAsgExpr->gtOp.gtOp2;
2264             if (!areArgumentsContiguous(prevDst, curDst) || !areArgumentsContiguous(prevSrc, curSrc))
2265             {
2266                 fgPreviousCandidateSIMDFieldAsgStmt = nullptr;
2267             }
2268             else
2269             {
2270                 if (index == (simdSize / genTypeSize(baseType) - 1))
2271                 {
2272                     // Successfully found the pattern, mark the lclvar as UsedInSIMDIntrinsic
2273                     if (srcSimdStructNode->OperIsLocal())
2274                     {
2275                         setLclRelatedToSIMDIntrinsic(srcSimdStructNode);
2276                     }
2277
2278                     if (curDst->OperGet() == GT_FIELD)
2279                     {
2280                         GenTree* objRef = curDst->gtField.gtFldObj;
2281                         if (objRef != nullptr && objRef->gtOper == GT_ADDR)
2282                         {
2283                             GenTree* obj = objRef->gtOp.gtOp1;
2284                             if (varTypeIsStruct(obj) && obj->OperIsLocal())
2285                             {
2286                                 setLclRelatedToSIMDIntrinsic(obj);
2287                             }
2288                         }
2289                     }
2290                 }
2291                 else
2292                 {
2293                     fgPreviousCandidateSIMDFieldAsgStmt = stmt;
2294                 }
2295             }
2296         }
2297     }
2298     else
2299     {
2300         fgPreviousCandidateSIMDFieldAsgStmt = nullptr;
2301     }
2302 }
2303
2304 //------------------------------------------------------------------------
2305 // impSIMDIntrinsic: Check method to see if it is a SIMD method
2306 //
2307 // Arguments:
2308 //    opcode     - the opcode being handled (needed to identify the CEE_NEWOBJ case)
2309 //    newobjThis - For CEE_NEWOBJ, this is the temp grabbed for the allocated uninitalized object.
2310 //    clsHnd     - The handle of the class of the method.
2311 //    method     - The handle of the method.
2312 //    sig        - The call signature for the method.
2313 //    memberRef  - The memberRef token for the method reference.
2314 //
2315 // Return Value:
2316 //    If clsHnd is a known SIMD type, and 'method' is one of the methods that are
2317 //    implemented as an intrinsic in the JIT, then return the tree that implements
2318 //    it.
2319 //
2320 GenTree* Compiler::impSIMDIntrinsic(OPCODE                opcode,
2321                                     GenTree*              newobjThis,
2322                                     CORINFO_CLASS_HANDLE  clsHnd,
2323                                     CORINFO_METHOD_HANDLE methodHnd,
2324                                     CORINFO_SIG_INFO*     sig,
2325                                     int                   memberRef)
2326 {
2327     assert(featureSIMD);
2328
2329     if (!isSIMDClass(clsHnd))
2330     {
2331         return nullptr;
2332     }
2333
2334     // Get base type and intrinsic Id
2335     var_types                baseType = TYP_UNKNOWN;
2336     unsigned                 size     = 0;
2337     unsigned                 argCount = 0;
2338     const SIMDIntrinsicInfo* intrinsicInfo =
2339         getSIMDIntrinsicInfo(&clsHnd, methodHnd, sig, (opcode == CEE_NEWOBJ), &argCount, &baseType, &size);
2340     if (intrinsicInfo == nullptr || intrinsicInfo->id == SIMDIntrinsicInvalid)
2341     {
2342         return nullptr;
2343     }
2344
2345     SIMDIntrinsicID simdIntrinsicID = intrinsicInfo->id;
2346     var_types       simdType;
2347     if (baseType != TYP_UNKNOWN)
2348     {
2349         simdType = getSIMDTypeForSize(size);
2350     }
2351     else
2352     {
2353         assert(simdIntrinsicID == SIMDIntrinsicHWAccel);
2354         simdType = TYP_UNKNOWN;
2355     }
2356     bool      instMethod = intrinsicInfo->isInstMethod;
2357     var_types callType   = JITtype2varType(sig->retType);
2358     if (callType == TYP_STRUCT)
2359     {
2360         // Note that here we are assuming that, if the call returns a struct, that it is the same size as the
2361         // struct on which the method is declared. This is currently true for all methods on Vector types,
2362         // but if this ever changes, we will need to determine the callType from the signature.
2363         assert(info.compCompHnd->getClassSize(sig->retTypeClass) == genTypeSize(simdType));
2364         callType = simdType;
2365     }
2366
2367     GenTree* simdTree   = nullptr;
2368     GenTree* op1        = nullptr;
2369     GenTree* op2        = nullptr;
2370     GenTree* op3        = nullptr;
2371     GenTree* retVal     = nullptr;
2372     GenTree* copyBlkDst = nullptr;
2373     bool     doCopyBlk  = false;
2374
2375     switch (simdIntrinsicID)
2376     {
2377         case SIMDIntrinsicGetCount:
2378         {
2379             int            length       = getSIMDVectorLength(clsHnd);
2380             GenTreeIntCon* intConstTree = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, length);
2381             retVal                      = intConstTree;
2382
2383             intConstTree->gtFlags |= GTF_ICON_SIMD_COUNT;
2384         }
2385         break;
2386
2387         case SIMDIntrinsicGetZero:
2388             retVal = gtNewSIMDVectorZero(simdType, baseType, size);
2389             break;
2390
2391         case SIMDIntrinsicGetOne:
2392             retVal = gtNewSIMDVectorOne(simdType, baseType, size);
2393             break;
2394
2395         case SIMDIntrinsicGetAllOnes:
2396         {
2397             // Equivalent to (Vector<T>) new Vector<int>(0xffffffff);
2398             GenTree* initVal = gtNewIconNode(0xffffffff, TYP_INT);
2399             simdTree         = gtNewSIMDNode(simdType, initVal, nullptr, SIMDIntrinsicInit, TYP_INT, size);
2400             if (baseType != TYP_INT)
2401             {
2402                 // cast it to required baseType if different from TYP_INT
2403                 simdTree = gtNewSIMDNode(simdType, simdTree, nullptr, SIMDIntrinsicCast, baseType, size);
2404             }
2405             retVal = simdTree;
2406         }
2407         break;
2408
2409         case SIMDIntrinsicInit:
2410         case SIMDIntrinsicInitN:
2411         {
2412             // SIMDIntrinsicInit:
2413             //    op2 - the initializer value
2414             //    op1 - byref of vector
2415             //
2416             // SIMDIntrinsicInitN
2417             //    op2 - list of initializer values stitched into a list
2418             //    op1 - byref of vector
2419             bool initFromFirstArgIndir = false;
2420             if (simdIntrinsicID == SIMDIntrinsicInit)
2421             {
2422                 op2 = impSIMDPopStack(baseType);
2423             }
2424             else
2425             {
2426                 assert(simdIntrinsicID == SIMDIntrinsicInitN);
2427                 assert(baseType == TYP_FLOAT);
2428
2429                 unsigned initCount    = argCount - 1;
2430                 unsigned elementCount = getSIMDVectorLength(size, baseType);
2431                 noway_assert(initCount == elementCount);
2432                 GenTree* nextArg = op2;
2433
2434                 // Build a GT_LIST with the N values.
2435                 // We must maintain left-to-right order of the args, but we will pop
2436                 // them off in reverse order (the Nth arg was pushed onto the stack last).
2437
2438                 GenTree* list              = nullptr;
2439                 GenTree* firstArg          = nullptr;
2440                 GenTree* prevArg           = nullptr;
2441                 int      offset            = 0;
2442                 bool     areArgsContiguous = true;
2443                 for (unsigned i = 0; i < initCount; i++)
2444                 {
2445                     GenTree* nextArg = impSIMDPopStack(baseType);
2446                     if (areArgsContiguous)
2447                     {
2448                         GenTree* curArg = nextArg;
2449                         firstArg        = curArg;
2450
2451                         if (prevArg != nullptr)
2452                         {
2453                             // Recall that we are popping the args off the stack in reverse order.
2454                             areArgsContiguous = areArgumentsContiguous(curArg, prevArg);
2455                         }
2456                         prevArg = curArg;
2457                     }
2458
2459                     list = new (this, GT_LIST) GenTreeOp(GT_LIST, baseType, nextArg, list);
2460                 }
2461
2462                 if (areArgsContiguous && baseType == TYP_FLOAT)
2463                 {
2464                     // Since Vector2, Vector3 and Vector4's arguments type are only float,
2465                     // we intialize the vector from first argument address, only when
2466                     // the baseType is TYP_FLOAT and the arguments are located contiguously in memory
2467                     initFromFirstArgIndir = true;
2468                     GenTree*  op2Address  = createAddressNodeForSIMDInit(firstArg, size);
2469                     var_types simdType    = getSIMDTypeForSize(size);
2470                     op2                   = gtNewOperNode(GT_IND, simdType, op2Address);
2471                 }
2472                 else
2473                 {
2474                     op2 = list;
2475                 }
2476             }
2477
2478             op1 = getOp1ForConstructor(opcode, newobjThis, clsHnd);
2479
2480             assert(op1->TypeGet() == TYP_BYREF);
2481             assert(genActualType(op2->TypeGet()) == genActualType(baseType) || initFromFirstArgIndir);
2482
2483             // For integral base types of size less than TYP_INT, expand the initializer
2484             // to fill size of TYP_INT bytes.
2485             if (varTypeIsSmallInt(baseType))
2486             {
2487                 // This case should occur only for Init intrinsic.
2488                 assert(simdIntrinsicID == SIMDIntrinsicInit);
2489
2490                 unsigned baseSize = genTypeSize(baseType);
2491                 int      multiplier;
2492                 if (baseSize == 1)
2493                 {
2494                     multiplier = 0x01010101;
2495                 }
2496                 else
2497                 {
2498                     assert(baseSize == 2);
2499                     multiplier = 0x00010001;
2500                 }
2501
2502                 GenTree* t1 = nullptr;
2503                 if (baseType == TYP_BYTE)
2504                 {
2505                     // What we have is a signed byte initializer,
2506                     // which when loaded to a reg will get sign extended to TYP_INT.
2507                     // But what we need is the initializer without sign extended or
2508                     // rather zero extended to 32-bits.
2509                     t1 = gtNewOperNode(GT_AND, TYP_INT, op2, gtNewIconNode(0xff, TYP_INT));
2510                 }
2511                 else if (baseType == TYP_SHORT)
2512                 {
2513                     // What we have is a signed short initializer,
2514                     // which when loaded to a reg will get sign extended to TYP_INT.
2515                     // But what we need is the initializer without sign extended or
2516                     // rather zero extended to 32-bits.
2517                     t1 = gtNewOperNode(GT_AND, TYP_INT, op2, gtNewIconNode(0xffff, TYP_INT));
2518                 }
2519                 else
2520                 {
2521                     assert(baseType == TYP_UBYTE || baseType == TYP_USHORT);
2522                     t1 = gtNewCastNode(TYP_INT, op2, false, TYP_INT);
2523                 }
2524
2525                 assert(t1 != nullptr);
2526                 GenTree* t2 = gtNewIconNode(multiplier, TYP_INT);
2527                 op2         = gtNewOperNode(GT_MUL, TYP_INT, t1, t2);
2528
2529                 // Construct a vector of TYP_INT with the new initializer and cast it back to vector of baseType
2530                 simdTree = gtNewSIMDNode(simdType, op2, nullptr, simdIntrinsicID, TYP_INT, size);
2531                 simdTree = gtNewSIMDNode(simdType, simdTree, nullptr, SIMDIntrinsicCast, baseType, size);
2532             }
2533             else
2534             {
2535
2536                 if (initFromFirstArgIndir)
2537                 {
2538                     simdTree = op2;
2539                     if (op1->gtOp.gtOp1->OperIsLocal())
2540                     {
2541                         // label the dst struct's lclvar is used for SIMD intrinsic,
2542                         // so that this dst struct won't be promoted.
2543                         setLclRelatedToSIMDIntrinsic(op1->gtOp.gtOp1);
2544                     }
2545                 }
2546                 else
2547                 {
2548                     simdTree = gtNewSIMDNode(simdType, op2, nullptr, simdIntrinsicID, baseType, size);
2549                 }
2550             }
2551
2552             copyBlkDst = op1;
2553             doCopyBlk  = true;
2554         }
2555         break;
2556
2557         case SIMDIntrinsicInitArray:
2558         case SIMDIntrinsicInitArrayX:
2559         case SIMDIntrinsicCopyToArray:
2560         case SIMDIntrinsicCopyToArrayX:
2561         {
2562             // op3 - index into array in case of SIMDIntrinsicCopyToArrayX and SIMDIntrinsicInitArrayX
2563             // op2 - array itself
2564             // op1 - byref to vector struct
2565
2566             unsigned int vectorLength = getSIMDVectorLength(size, baseType);
2567             // (This constructor takes only the zero-based arrays.)
2568             // We will add one or two bounds checks:
2569             // 1. If we have an index, we must do a check on that first.
2570             //    We can't combine it with the index + vectorLength check because
2571             //    a. It might be negative, and b. It may need to raise a different exception
2572             //    (captured as SCK_ARG_RNG_EXCPN for CopyTo and SCK_RNGCHK_FAIL for Init).
2573             // 2. We need to generate a check (SCK_ARG_EXCPN for CopyTo and SCK_RNGCHK_FAIL for Init)
2574             //    for the last array element we will access.
2575             //    We'll either check against (vectorLength - 1) or (index + vectorLength - 1).
2576
2577             GenTree* checkIndexExpr = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, vectorLength - 1);
2578
2579             // Get the index into the array.  If it has been provided, it will be on the
2580             // top of the stack.  Otherwise, it is null.
2581             if (argCount == 3)
2582             {
2583                 op3 = impSIMDPopStack(TYP_INT);
2584                 if (op3->IsIntegralConst(0))
2585                 {
2586                     op3 = nullptr;
2587                 }
2588             }
2589             else
2590             {
2591                 // TODO-CQ: Here, or elsewhere, check for the pattern where op2 is a newly constructed array, and
2592                 // change this to the InitN form.
2593                 // op3 = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, 0);
2594                 op3 = nullptr;
2595             }
2596
2597             // Clone the array for use in the bounds check.
2598             op2 = impSIMDPopStack(TYP_REF);
2599             assert(op2->TypeGet() == TYP_REF);
2600             GenTree* arrayRefForArgChk = op2;
2601             GenTree* argRngChk         = nullptr;
2602             GenTree* asg               = nullptr;
2603             if ((arrayRefForArgChk->gtFlags & GTF_SIDE_EFFECT) != 0)
2604             {
2605                 op2 = fgInsertCommaFormTemp(&arrayRefForArgChk);
2606             }
2607             else
2608             {
2609                 op2 = gtCloneExpr(arrayRefForArgChk);
2610             }
2611             assert(op2 != nullptr);
2612
2613             if (op3 != nullptr)
2614             {
2615                 SpecialCodeKind op3CheckKind;
2616                 if (simdIntrinsicID == SIMDIntrinsicInitArrayX)
2617                 {
2618                     op3CheckKind = SCK_RNGCHK_FAIL;
2619                 }
2620                 else
2621                 {
2622                     assert(simdIntrinsicID == SIMDIntrinsicCopyToArrayX);
2623                     op3CheckKind = SCK_ARG_RNG_EXCPN;
2624                 }
2625                 // We need to use the original expression on this, which is the first check.
2626                 GenTree* arrayRefForArgRngChk = arrayRefForArgChk;
2627                 // Then we clone the clone we just made for the next check.
2628                 arrayRefForArgChk = gtCloneExpr(op2);
2629                 // We know we MUST have had a cloneable expression.
2630                 assert(arrayRefForArgChk != nullptr);
2631                 GenTree* index = op3;
2632                 if ((index->gtFlags & GTF_SIDE_EFFECT) != 0)
2633                 {
2634                     op3 = fgInsertCommaFormTemp(&index);
2635                 }
2636                 else
2637                 {
2638                     op3 = gtCloneExpr(index);
2639                 }
2640
2641                 GenTreeArrLen* arrLen =
2642                     gtNewArrLen(TYP_INT, arrayRefForArgRngChk, (int)OFFSETOF__CORINFO_Array__length);
2643                 argRngChk = new (this, GT_ARR_BOUNDS_CHECK)
2644                     GenTreeBoundsChk(GT_ARR_BOUNDS_CHECK, TYP_VOID, index, arrLen, op3CheckKind);
2645                 // Now, clone op3 to create another node for the argChk
2646                 GenTree* index2 = gtCloneExpr(op3);
2647                 assert(index != nullptr);
2648                 checkIndexExpr = gtNewOperNode(GT_ADD, TYP_INT, index2, checkIndexExpr);
2649             }
2650
2651             // Insert a bounds check for index + offset - 1.
2652             // This must be a "normal" array.
2653             SpecialCodeKind op2CheckKind;
2654             if (simdIntrinsicID == SIMDIntrinsicInitArray || simdIntrinsicID == SIMDIntrinsicInitArrayX)
2655             {
2656                 op2CheckKind = SCK_RNGCHK_FAIL;
2657             }
2658             else
2659             {
2660                 op2CheckKind = SCK_ARG_EXCPN;
2661             }
2662             GenTreeArrLen*    arrLen = gtNewArrLen(TYP_INT, arrayRefForArgChk, (int)OFFSETOF__CORINFO_Array__length);
2663             GenTreeBoundsChk* argChk = new (this, GT_ARR_BOUNDS_CHECK)
2664                 GenTreeBoundsChk(GT_ARR_BOUNDS_CHECK, TYP_VOID, checkIndexExpr, arrLen, op2CheckKind);
2665
2666             // Create a GT_COMMA tree for the bounds check(s).
2667             op2 = gtNewOperNode(GT_COMMA, op2->TypeGet(), argChk, op2);
2668             if (argRngChk != nullptr)
2669             {
2670                 op2 = gtNewOperNode(GT_COMMA, op2->TypeGet(), argRngChk, op2);
2671             }
2672
2673             if (simdIntrinsicID == SIMDIntrinsicInitArray || simdIntrinsicID == SIMDIntrinsicInitArrayX)
2674             {
2675                 op1        = getOp1ForConstructor(opcode, newobjThis, clsHnd);
2676                 simdTree   = gtNewSIMDNode(simdType, op2, op3, SIMDIntrinsicInitArray, baseType, size);
2677                 copyBlkDst = op1;
2678                 doCopyBlk  = true;
2679             }
2680             else
2681             {
2682                 assert(simdIntrinsicID == SIMDIntrinsicCopyToArray || simdIntrinsicID == SIMDIntrinsicCopyToArrayX);
2683                 op1 = impSIMDPopStack(simdType, instMethod);
2684                 assert(op1->TypeGet() == simdType);
2685
2686                 // copy vector (op1) to array (op2) starting at index (op3)
2687                 simdTree = op1;
2688
2689                 // TODO-Cleanup: Though it happens to just work fine front-end phases are not aware of GT_LEA node.
2690                 // Therefore, convert these to use GT_ADDR .
2691                 copyBlkDst = new (this, GT_LEA)
2692                     GenTreeAddrMode(TYP_BYREF, op2, op3, genTypeSize(baseType), OFFSETOF__CORINFO_Array__data);
2693                 doCopyBlk = true;
2694             }
2695         }
2696         break;
2697
2698         case SIMDIntrinsicInitFixed:
2699         {
2700             // We are initializing a fixed-length vector VLarge with a smaller fixed-length vector VSmall, plus 1 or 2
2701             // additional floats.
2702             //    op4 (optional) - float value for VLarge.W, if VLarge is Vector4, and VSmall is Vector2
2703             //    op3 - float value for VLarge.Z or VLarge.W
2704             //    op2 - VSmall
2705             //    op1 - byref of VLarge
2706             assert(baseType == TYP_FLOAT);
2707             unsigned elementByteCount = 4;
2708
2709             GenTree* op4 = nullptr;
2710             if (argCount == 4)
2711             {
2712                 op4 = impSIMDPopStack(TYP_FLOAT);
2713                 assert(op4->TypeGet() == TYP_FLOAT);
2714             }
2715             op3 = impSIMDPopStack(TYP_FLOAT);
2716             assert(op3->TypeGet() == TYP_FLOAT);
2717             // The input vector will either be TYP_SIMD8 or TYP_SIMD12.
2718             var_types smallSIMDType = TYP_SIMD8;
2719             if ((op4 == nullptr) && (simdType == TYP_SIMD16))
2720             {
2721                 smallSIMDType = TYP_SIMD12;
2722             }
2723             op2 = impSIMDPopStack(smallSIMDType);
2724             op1 = getOp1ForConstructor(opcode, newobjThis, clsHnd);
2725
2726             // We are going to redefine the operands so that:
2727             // - op3 is the value that's going into the Z position, or null if it's a Vector4 constructor with a single
2728             // operand, and
2729             // - op4 is the W position value, or null if this is a Vector3 constructor.
2730             if (size == 16 && argCount == 3)
2731             {
2732                 op4 = op3;
2733                 op3 = nullptr;
2734             }
2735
2736             simdTree = op2;
2737             if (op3 != nullptr)
2738             {
2739                 simdTree = gtNewSIMDNode(simdType, simdTree, op3, SIMDIntrinsicSetZ, baseType, size);
2740             }
2741             if (op4 != nullptr)
2742             {
2743                 simdTree = gtNewSIMDNode(simdType, simdTree, op4, SIMDIntrinsicSetW, baseType, size);
2744             }
2745
2746             copyBlkDst = op1;
2747             doCopyBlk  = true;
2748         }
2749         break;
2750
2751         case SIMDIntrinsicOpEquality:
2752         case SIMDIntrinsicInstEquals:
2753         {
2754             op2 = impSIMDPopStack(simdType);
2755             op1 = impSIMDPopStack(simdType, instMethod);
2756
2757             assert(op1->TypeGet() == simdType);
2758             assert(op2->TypeGet() == simdType);
2759
2760             simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, SIMDIntrinsicOpEquality, baseType, size);
2761             if (simdType == TYP_SIMD12)
2762             {
2763                 simdTree->gtFlags |= GTF_SIMD12_OP;
2764             }
2765             retVal = simdTree;
2766         }
2767         break;
2768
2769         case SIMDIntrinsicOpInEquality:
2770         {
2771             // op1 is the first operand
2772             // op2 is the second operand
2773             op2      = impSIMDPopStack(simdType);
2774             op1      = impSIMDPopStack(simdType, instMethod);
2775             simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, SIMDIntrinsicOpInEquality, baseType, size);
2776             if (simdType == TYP_SIMD12)
2777             {
2778                 simdTree->gtFlags |= GTF_SIMD12_OP;
2779             }
2780             retVal = simdTree;
2781         }
2782         break;
2783
2784         case SIMDIntrinsicEqual:
2785         case SIMDIntrinsicLessThan:
2786         case SIMDIntrinsicLessThanOrEqual:
2787         case SIMDIntrinsicGreaterThan:
2788         case SIMDIntrinsicGreaterThanOrEqual:
2789         {
2790             op2 = impSIMDPopStack(simdType);
2791             op1 = impSIMDPopStack(simdType, instMethod);
2792
2793             SIMDIntrinsicID intrinsicID = impSIMDRelOp(simdIntrinsicID, clsHnd, size, &baseType, &op1, &op2);
2794             simdTree                    = gtNewSIMDNode(genActualType(callType), op1, op2, intrinsicID, baseType, size);
2795             retVal                      = simdTree;
2796         }
2797         break;
2798
2799         case SIMDIntrinsicAdd:
2800         case SIMDIntrinsicSub:
2801         case SIMDIntrinsicMul:
2802         case SIMDIntrinsicDiv:
2803         case SIMDIntrinsicBitwiseAnd:
2804         case SIMDIntrinsicBitwiseAndNot:
2805         case SIMDIntrinsicBitwiseOr:
2806         case SIMDIntrinsicBitwiseXor:
2807         {
2808 #if defined(DEBUG)
2809             // check for the cases where we don't support intrinsics.
2810             // This check should be done before we make modifications to type stack.
2811             // Note that this is more of a double safety check for robustness since
2812             // we expect getSIMDIntrinsicInfo() to have filtered out intrinsics on
2813             // unsupported base types. If getSIMdIntrinsicInfo() doesn't filter due
2814             // to some bug, assert in chk/dbg will fire.
2815             if (!varTypeIsFloating(baseType))
2816             {
2817                 if (simdIntrinsicID == SIMDIntrinsicMul)
2818                 {
2819 #if defined(_TARGET_XARCH_)
2820                     if ((baseType != TYP_INT) && (baseType != TYP_SHORT))
2821                     {
2822                         // TODO-CQ: implement mul on these integer vectors.
2823                         // Note that SSE2 has no direct support for these vectors.
2824                         assert(!"Mul not supported on long/ulong/uint/small int vectors\n");
2825                         return nullptr;
2826                     }
2827 #endif // _TARGET_XARCH_
2828 #if defined(_TARGET_ARM64_)
2829                     if ((baseType == TYP_ULONG) && (baseType == TYP_LONG))
2830                     {
2831                         // TODO-CQ: implement mul on these integer vectors.
2832                         // Note that ARM64 has no direct support for these vectors.
2833                         assert(!"Mul not supported on long/ulong vectors\n");
2834                         return nullptr;
2835                     }
2836 #endif // _TARGET_ARM64_
2837                 }
2838 #if defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
2839                 // common to all integer type vectors
2840                 if (simdIntrinsicID == SIMDIntrinsicDiv)
2841                 {
2842                     // SSE2 doesn't support div on non-floating point vectors.
2843                     assert(!"Div not supported on integer type vectors\n");
2844                     return nullptr;
2845                 }
2846 #endif // defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
2847             }
2848 #endif // DEBUG
2849
2850             // op1 is the first operand; if instance method, op1 is "this" arg
2851             // op2 is the second operand
2852             op2 = impSIMDPopStack(simdType);
2853             op1 = impSIMDPopStack(simdType, instMethod);
2854
2855             simdTree = gtNewSIMDNode(simdType, op1, op2, simdIntrinsicID, baseType, size);
2856             retVal   = simdTree;
2857         }
2858         break;
2859
2860         case SIMDIntrinsicSelect:
2861         {
2862             // op3 is a SIMD variable that is the second source
2863             // op2 is a SIMD variable that is the first source
2864             // op1 is a SIMD variable which is the bit mask.
2865             op3 = impSIMDPopStack(simdType);
2866             op2 = impSIMDPopStack(simdType);
2867             op1 = impSIMDPopStack(simdType);
2868
2869             retVal = impSIMDSelect(clsHnd, baseType, size, op1, op2, op3);
2870         }
2871         break;
2872
2873         case SIMDIntrinsicMin:
2874         case SIMDIntrinsicMax:
2875         {
2876             // op1 is the first operand; if instance method, op1 is "this" arg
2877             // op2 is the second operand
2878             op2 = impSIMDPopStack(simdType);
2879             op1 = impSIMDPopStack(simdType, instMethod);
2880
2881             retVal = impSIMDMinMax(simdIntrinsicID, clsHnd, baseType, size, op1, op2);
2882         }
2883         break;
2884
2885         case SIMDIntrinsicGetItem:
2886         {
2887             // op1 is a SIMD variable that is "this" arg
2888             // op2 is an index of TYP_INT
2889             op2              = impSIMDPopStack(TYP_INT);
2890             op1              = impSIMDPopStack(simdType, instMethod);
2891             int vectorLength = getSIMDVectorLength(size, baseType);
2892             if (!op2->IsCnsIntOrI() || op2->AsIntCon()->gtIconVal >= vectorLength || op2->AsIntCon()->gtIconVal < 0)
2893             {
2894                 // We need to bounds-check the length of the vector.
2895                 // For that purpose, we need to clone the index expression.
2896                 GenTree* index = op2;
2897                 if ((index->gtFlags & GTF_SIDE_EFFECT) != 0)
2898                 {
2899                     op2 = fgInsertCommaFormTemp(&index);
2900                 }
2901                 else
2902                 {
2903                     op2 = gtCloneExpr(index);
2904                 }
2905
2906                 GenTree*          lengthNode = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, vectorLength);
2907                 GenTreeBoundsChk* simdChk =
2908                     new (this, GT_SIMD_CHK) GenTreeBoundsChk(GT_SIMD_CHK, TYP_VOID, index, lengthNode, SCK_RNGCHK_FAIL);
2909
2910                 // Create a GT_COMMA tree for the bounds check.
2911                 op2 = gtNewOperNode(GT_COMMA, op2->TypeGet(), simdChk, op2);
2912             }
2913
2914             assert(op1->TypeGet() == simdType);
2915             assert(op2->TypeGet() == TYP_INT);
2916
2917             simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, simdIntrinsicID, baseType, size);
2918             retVal   = simdTree;
2919         }
2920         break;
2921
2922         case SIMDIntrinsicDotProduct:
2923         {
2924 #if defined(_TARGET_XARCH_)
2925             // Right now dot product is supported only for float/double vectors and
2926             // int vectors on SSE4/AVX.
2927             if (!varTypeIsFloating(baseType) && !(baseType == TYP_INT && getSIMDSupportLevel() >= SIMD_SSE4_Supported))
2928             {
2929                 return nullptr;
2930             }
2931 #endif // _TARGET_XARCH_
2932
2933             // op1 is a SIMD variable that is the first source and also "this" arg.
2934             // op2 is a SIMD variable which is the second source.
2935             op2 = impSIMDPopStack(simdType);
2936             op1 = impSIMDPopStack(simdType, instMethod);
2937
2938             simdTree = gtNewSIMDNode(baseType, op1, op2, simdIntrinsicID, baseType, size);
2939             if (simdType == TYP_SIMD12)
2940             {
2941                 simdTree->gtFlags |= GTF_SIMD12_OP;
2942             }
2943             retVal = simdTree;
2944         }
2945         break;
2946
2947         case SIMDIntrinsicSqrt:
2948         {
2949 #if (defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)) && defined(DEBUG)
2950             // SSE/AVX/ARM64 doesn't support sqrt on integer type vectors and hence
2951             // should never be seen as an intrinsic here. See SIMDIntrinsicList.h
2952             // for supported base types for this intrinsic.
2953             if (!varTypeIsFloating(baseType))
2954             {
2955                 assert(!"Sqrt not supported on integer vectors\n");
2956                 return nullptr;
2957             }
2958 #endif // (defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)) && defined(DEBUG)
2959
2960             op1 = impSIMDPopStack(simdType);
2961
2962             retVal = gtNewSIMDNode(genActualType(callType), op1, nullptr, simdIntrinsicID, baseType, size);
2963         }
2964         break;
2965
2966         case SIMDIntrinsicAbs:
2967             op1    = impSIMDPopStack(simdType);
2968             retVal = impSIMDAbs(clsHnd, baseType, size, op1);
2969             break;
2970
2971         case SIMDIntrinsicGetW:
2972             retVal = impSIMDGetFixed(simdType, baseType, size, 3);
2973             break;
2974
2975         case SIMDIntrinsicGetZ:
2976             retVal = impSIMDGetFixed(simdType, baseType, size, 2);
2977             break;
2978
2979         case SIMDIntrinsicGetY:
2980             retVal = impSIMDGetFixed(simdType, baseType, size, 1);
2981             break;
2982
2983         case SIMDIntrinsicGetX:
2984             retVal = impSIMDGetFixed(simdType, baseType, size, 0);
2985             break;
2986
2987         case SIMDIntrinsicSetW:
2988         case SIMDIntrinsicSetZ:
2989         case SIMDIntrinsicSetY:
2990         case SIMDIntrinsicSetX:
2991         {
2992             // op2 is the value to be set at indexTemp position
2993             // op1 is SIMD vector that is going to be modified, which is a byref
2994
2995             // If op1 has a side-effect, then don't make it an intrinsic.
2996             // It would be in-efficient to read the entire vector into xmm reg,
2997             // modify it and write back entire xmm reg.
2998             //
2999             // TODO-CQ: revisit this later.
3000             op1 = impStackTop(1).val;
3001             if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0)
3002             {
3003                 return nullptr;
3004             }
3005
3006             op2 = impSIMDPopStack(baseType);
3007             op1 = impSIMDPopStack(simdType, instMethod);
3008
3009             GenTree* src = gtCloneExpr(op1);
3010             assert(src != nullptr);
3011             simdTree = gtNewSIMDNode(simdType, src, op2, simdIntrinsicID, baseType, size);
3012
3013             copyBlkDst = gtNewOperNode(GT_ADDR, TYP_BYREF, op1);
3014             doCopyBlk  = true;
3015         }
3016         break;
3017
3018         // Unary operators that take and return a Vector.
3019         case SIMDIntrinsicCast:
3020         case SIMDIntrinsicConvertToSingle:
3021         case SIMDIntrinsicConvertToDouble:
3022         case SIMDIntrinsicConvertToInt32:
3023         {
3024             op1 = impSIMDPopStack(simdType, instMethod);
3025
3026             simdTree = gtNewSIMDNode(simdType, op1, nullptr, simdIntrinsicID, baseType, size);
3027             retVal   = simdTree;
3028         }
3029         break;
3030
3031         case SIMDIntrinsicConvertToInt64:
3032         {
3033 #ifdef _TARGET_64BIT_
3034             op1 = impSIMDPopStack(simdType, instMethod);
3035
3036             simdTree = gtNewSIMDNode(simdType, op1, nullptr, simdIntrinsicID, baseType, size);
3037             retVal   = simdTree;
3038 #else
3039             JITDUMP("SIMD Conversion to Int64 is not supported on this platform\n");
3040             return nullptr;
3041 #endif
3042         }
3043         break;
3044
3045         case SIMDIntrinsicNarrow:
3046         {
3047             assert(!instMethod);
3048             op2 = impSIMDPopStack(simdType);
3049             op1 = impSIMDPopStack(simdType);
3050             // op1 and op2 are two input Vector<T>.
3051             simdTree = gtNewSIMDNode(simdType, op1, op2, simdIntrinsicID, baseType, size);
3052             retVal   = simdTree;
3053         }
3054         break;
3055
3056         case SIMDIntrinsicWiden:
3057         {
3058             GenTree* dstAddrHi = impSIMDPopStack(TYP_BYREF);
3059             GenTree* dstAddrLo = impSIMDPopStack(TYP_BYREF);
3060             op1                = impSIMDPopStack(simdType);
3061             GenTree* dupOp1    = fgInsertCommaFormTemp(&op1, gtGetStructHandleForSIMD(simdType, baseType));
3062
3063             // Widen the lower half and assign it to dstAddrLo.
3064             simdTree = gtNewSIMDNode(simdType, op1, nullptr, SIMDIntrinsicWidenLo, baseType, size);
3065             GenTree* loDest =
3066                 new (this, GT_BLK) GenTreeBlk(GT_BLK, simdType, dstAddrLo, getSIMDTypeSizeInBytes(clsHnd));
3067             GenTree* loAsg = gtNewBlkOpNode(loDest, simdTree, getSIMDTypeSizeInBytes(clsHnd),
3068                                             false, // not volatile
3069                                             true); // copyBlock
3070             loAsg->gtFlags |= ((simdTree->gtFlags | dstAddrLo->gtFlags) & GTF_ALL_EFFECT);
3071
3072             // Widen the upper half and assign it to dstAddrHi.
3073             simdTree = gtNewSIMDNode(simdType, dupOp1, nullptr, SIMDIntrinsicWidenHi, baseType, size);
3074             GenTree* hiDest =
3075                 new (this, GT_BLK) GenTreeBlk(GT_BLK, simdType, dstAddrHi, getSIMDTypeSizeInBytes(clsHnd));
3076             GenTree* hiAsg = gtNewBlkOpNode(hiDest, simdTree, getSIMDTypeSizeInBytes(clsHnd),
3077                                             false, // not volatile
3078                                             true); // copyBlock
3079             hiAsg->gtFlags |= ((simdTree->gtFlags | dstAddrHi->gtFlags) & GTF_ALL_EFFECT);
3080
3081             retVal = gtNewOperNode(GT_COMMA, simdType, loAsg, hiAsg);
3082         }
3083         break;
3084
3085         case SIMDIntrinsicHWAccel:
3086         {
3087             GenTreeIntCon* intConstTree = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, 1);
3088             retVal                      = intConstTree;
3089         }
3090         break;
3091
3092         default:
3093             assert(!"Unimplemented SIMD Intrinsic");
3094             return nullptr;
3095     }
3096
3097 #if defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
3098     // XArch/Arm64: also indicate that we use floating point registers.
3099     // The need for setting this here is that a method may not have SIMD
3100     // type lclvars, but might be exercising SIMD intrinsics on fields of
3101     // SIMD type.
3102     //
3103     // e.g.  public Vector<float> ComplexVecFloat::sqabs() { return this.r * this.r + this.i * this.i; }
3104     compFloatingPointUsed = true;
3105 #endif // defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
3106
3107     // At this point, we have a tree that we are going to store into a destination.
3108     // TODO-1stClassStructs: This should be a simple store or assignment, and should not require
3109     // GTF_ALL_EFFECT for the dest. This is currently emulating the previous behavior of
3110     // block ops.
3111     if (doCopyBlk)
3112     {
3113         GenTree* dest = new (this, GT_BLK) GenTreeBlk(GT_BLK, simdType, copyBlkDst, getSIMDTypeSizeInBytes(clsHnd));
3114         dest->gtFlags |= GTF_GLOB_REF;
3115         retVal = gtNewBlkOpNode(dest, simdTree, getSIMDTypeSizeInBytes(clsHnd),
3116                                 false, // not volatile
3117                                 true); // copyBlock
3118         retVal->gtFlags |= ((simdTree->gtFlags | copyBlkDst->gtFlags) & GTF_ALL_EFFECT);
3119     }
3120
3121     return retVal;
3122 }
3123
3124 #endif // FEATURE_SIMD