src/jit/simd.cpp

   1 // Licensed to the .NET Foundation under one or more agreements.
   2 // The .NET Foundation licenses this file to you under the MIT license.
   3 // See the LICENSE file in the project root for more information.
   4
   5 //
   6 //   SIMD Support
   7 //
   8 // IMPORTANT NOTES AND CAVEATS:
   9 //
  10 // This implementation is preliminary, and may change dramatically.
  11 //
  12 // New JIT types, TYP_SIMDxx, are introduced, and the SIMD intrinsics are created as GT_SIMD nodes.
  13 // Nodes of SIMD types will be typed as TYP_SIMD* (e.g. TYP_SIMD8, TYP_SIMD16, etc.).
  14 //
  15 // Note that currently the "reference implementation" is the same as the runtime dll.  As such, it is currently
  16 // providing implementations for those methods not currently supported by the JIT as intrinsics.
  17 //
  18 // These are currently recognized using string compares, in order to provide an implementation in the JIT
  19 // without taking a dependency on the VM.
  20 // Furthermore, in the CTP, in order to limit the impact of doing these string compares
  21 // against assembly names, we only look for the SIMDVector assembly if we are compiling a class constructor.  This
  22 // makes it somewhat more "pay for play" but is a significant usability compromise.
  23 // This has been addressed for RTM by doing the assembly recognition in the VM.
  24 // --------------------------------------------------------------------------------------
  25
  26 #include "jitpch.h"
  27 #include "simd.h"
  28
  29 #ifdef _MSC_VER
  30 #pragma hdrstop
  31 #endif
  32
  33 #ifdef FEATURE_SIMD
  34
  35 // Intrinsic Id to intrinsic info map
  36 const SIMDIntrinsicInfo simdIntrinsicInfoArray[] = {
  37 #define SIMD_INTRINSIC(mname, inst, id, name, retType, argCount, arg1, arg2, arg3, t1, t2, t3, t4, t5, t6, t7, t8, t9, \
  38                        t10)                                                                                            \
  39     {SIMDIntrinsic##id, mname, inst, retType, argCount, arg1, arg2, arg3, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10},
  40 #include "simdintrinsiclist.h"
  41 };
  42
  43 //------------------------------------------------------------------------
  44 // getSIMDVectorLength: Get the length (number of elements of base type) of
  45 //                      SIMD Vector given its size and base (element) type.
  46 //
  47 // Arguments:
  48 //    simdSize   - size of the SIMD vector
  49 //    baseType   - type of the elements of the SIMD vector
  50 //
  51 // static
  52 int Compiler::getSIMDVectorLength(unsigned simdSize, var_types baseType)
  53 {
  54     return simdSize / genTypeSize(baseType);
  55 }
  56
  57 //------------------------------------------------------------------------
  58 // Get the length (number of elements of base type) of SIMD Vector given by typeHnd.
  59 //
  60 // Arguments:
  61 //    typeHnd  - type handle of the SIMD vector
  62 //
  63 int Compiler::getSIMDVectorLength(CORINFO_CLASS_HANDLE typeHnd)
  64 {
  65     unsigned  sizeBytes = 0;
  66     var_types baseType  = getBaseTypeAndSizeOfSIMDType(typeHnd, &sizeBytes);
  67     return getSIMDVectorLength(sizeBytes, baseType);
  68 }
  69
  70 //------------------------------------------------------------------------
  71 // Get the preferred alignment of SIMD vector type for better performance.
  72 //
  73 // Arguments:
  74 //    typeHnd  - type handle of the SIMD vector
  75 //
  76 int Compiler::getSIMDTypeAlignment(var_types simdType)
  77 {
  78 #ifdef _TARGET_XARCH_
  79     // Fixed length vectors have the following alignment preference
  80     // Vector2   = 8 byte alignment
  81     // Vector3/4 = 16-byte alignment
  82     unsigned size = genTypeSize(simdType);
  83
  84     // preferred alignment for SSE2 128-bit vectors is 16-bytes
  85     if (size == 8)
  86     {
  87         return 8;
  88     }
  89     else if (size <= 16)
  90     {
  91         assert((size == 12) || (size == 16));
  92         return 16;
  93     }
  94     else
  95     {
  96         assert(size == 32);
  97         return 32;
  98     }
  99 #elif defined(_TARGET_ARM64_)
 100     return 16;
 101 #else
 102     assert(!"getSIMDTypeAlignment() unimplemented on target arch");
 103     unreached();
 104 #endif
 105 }
 106
 107 //----------------------------------------------------------------------------------
 108 // Return the base type and size of SIMD vector type given its type handle.
 109 //
 110 // Arguments:
 111 //    typeHnd   - The handle of the type we're interested in.
 112 //    sizeBytes - out param
 113 //
 114 // Return Value:
 115 //    base type of SIMD vector.
 116 //    sizeBytes if non-null is set to size in bytes.
 117 //
 118 // TODO-Throughput: current implementation parses class name to find base type. Change
 119 //         this when we implement  SIMD intrinsic identification for the final
 120 //         product.
 121 //
 122 var_types Compiler::getBaseTypeAndSizeOfSIMDType(CORINFO_CLASS_HANDLE typeHnd, unsigned* sizeBytes /*= nullptr */)
 123 {
 124     assert(featureSIMD);
 125
 126     if (m_simdHandleCache == nullptr)
 127     {
 128         if (impInlineInfo == nullptr)
 129         {
 130             m_simdHandleCache = new (this, CMK_Generic) SIMDHandlesCache();
 131         }
 132         else
 133         {
 134             // Steal the inliner compiler's cache (create it if not available).
 135
 136             if (impInlineInfo->InlineRoot->m_simdHandleCache == nullptr)
 137             {
 138                 impInlineInfo->InlineRoot->m_simdHandleCache = new (this, CMK_Generic) SIMDHandlesCache();
 139             }
 140
 141             m_simdHandleCache = impInlineInfo->InlineRoot->m_simdHandleCache;
 142         }
 143     }
 144
 145     if (typeHnd == nullptr)
 146     {
 147         return TYP_UNKNOWN;
 148     }
 149
 150     // fast path search using cached type handles of important types
 151     var_types simdBaseType = TYP_UNKNOWN;
 152     unsigned  size         = 0;
 153
 154     // TODO - Optimize SIMD type recognition by IntrinsicAttribute
 155     if (isSIMDClass(typeHnd))
 156     {
 157         // The most likely to be used type handles are looked up first followed by
 158         // less likely to be used type handles
 159         if (typeHnd == m_simdHandleCache->SIMDFloatHandle)
 160         {
 161             simdBaseType = TYP_FLOAT;
 162             JITDUMP("  Known type SIMD Vector<Float>\n");
 163         }
 164         else if (typeHnd == m_simdHandleCache->SIMDIntHandle)
 165         {
 166             simdBaseType = TYP_INT;
 167             JITDUMP("  Known type SIMD Vector<Int>\n");
 168         }
 169         else if (typeHnd == m_simdHandleCache->SIMDVector2Handle)
 170         {
 171             simdBaseType = TYP_FLOAT;
 172             size         = 2 * genTypeSize(TYP_FLOAT);
 173             assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE));
 174             JITDUMP("  Known type Vector2\n");
 175         }
 176         else if (typeHnd == m_simdHandleCache->SIMDVector3Handle)
 177         {
 178             simdBaseType = TYP_FLOAT;
 179             size         = 3 * genTypeSize(TYP_FLOAT);
 180             assert(size == info.compCompHnd->getClassSize(typeHnd));
 181             JITDUMP("  Known type Vector3\n");
 182         }
 183         else if (typeHnd == m_simdHandleCache->SIMDVector4Handle)
 184         {
 185             simdBaseType = TYP_FLOAT;
 186             size         = 4 * genTypeSize(TYP_FLOAT);
 187             assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE));
 188             JITDUMP("  Known type Vector4\n");
 189         }
 190         else if (typeHnd == m_simdHandleCache->SIMDVectorHandle)
 191         {
 192             JITDUMP("  Known type Vector\n");
 193         }
 194         else if (typeHnd == m_simdHandleCache->SIMDUShortHandle)
 195         {
 196             simdBaseType = TYP_USHORT;
 197             JITDUMP("  Known type SIMD Vector<ushort>\n");
 198         }
 199         else if (typeHnd == m_simdHandleCache->SIMDUByteHandle)
 200         {
 201             simdBaseType = TYP_UBYTE;
 202             JITDUMP("  Known type SIMD Vector<ubyte>\n");
 203         }
 204         else if (typeHnd == m_simdHandleCache->SIMDDoubleHandle)
 205         {
 206             simdBaseType = TYP_DOUBLE;
 207             JITDUMP("  Known type SIMD Vector<Double>\n");
 208         }
 209         else if (typeHnd == m_simdHandleCache->SIMDLongHandle)
 210         {
 211             simdBaseType = TYP_LONG;
 212             JITDUMP("  Known type SIMD Vector<Long>\n");
 213         }
 214         else if (typeHnd == m_simdHandleCache->SIMDShortHandle)
 215         {
 216             simdBaseType = TYP_SHORT;
 217             JITDUMP("  Known type SIMD Vector<short>\n");
 218         }
 219         else if (typeHnd == m_simdHandleCache->SIMDByteHandle)
 220         {
 221             simdBaseType = TYP_BYTE;
 222             JITDUMP("  Known type SIMD Vector<byte>\n");
 223         }
 224         else if (typeHnd == m_simdHandleCache->SIMDUIntHandle)
 225         {
 226             simdBaseType = TYP_UINT;
 227             JITDUMP("  Known type SIMD Vector<uint>\n");
 228         }
 229         else if (typeHnd == m_simdHandleCache->SIMDULongHandle)
 230         {
 231             simdBaseType = TYP_ULONG;
 232             JITDUMP("  Known type SIMD Vector<ulong>\n");
 233         }
 234
 235         // slow path search
 236         if (simdBaseType == TYP_UNKNOWN)
 237         {
 238             // Doesn't match with any of the cached type handles.
 239             // Obtain base type by parsing fully qualified class name.
 240             //
 241             // TODO-Throughput: implement product shipping solution to query base type.
 242             WCHAR  className[256] = {0};
 243             WCHAR* pbuf           = &className[0];
 244             int    len            = _countof(className);
 245             info.compCompHnd->appendClassName(&pbuf, &len, typeHnd, TRUE, FALSE, FALSE);
 246             noway_assert(pbuf < &className[256]);
 247             JITDUMP("SIMD Candidate Type %S\n", className);
 248
 249             if (wcsncmp(className, W("System.Numerics."), 16) == 0)
 250             {
 251                 if (wcsncmp(&(className[16]), W("Vector`1["), 9) == 0)
 252                 {
 253                     if (wcsncmp(&(className[25]), W("System.Single"), 13) == 0)
 254                     {
 255                         m_simdHandleCache->SIMDFloatHandle = typeHnd;
 256                         simdBaseType                       = TYP_FLOAT;
 257                         JITDUMP("  Found type SIMD Vector<Float>\n");
 258                     }
 259                     else if (wcsncmp(&(className[25]), W("System.Int32"), 12) == 0)
 260                     {
 261                         m_simdHandleCache->SIMDIntHandle = typeHnd;
 262                         simdBaseType                     = TYP_INT;
 263                         JITDUMP("  Found type SIMD Vector<Int>\n");
 264                     }
 265                     else if (wcsncmp(&(className[25]), W("System.UInt16"), 13) == 0)
 266                     {
 267                         m_simdHandleCache->SIMDUShortHandle = typeHnd;
 268                         simdBaseType                        = TYP_USHORT;
 269                         JITDUMP("  Found type SIMD Vector<ushort>\n");
 270                     }
 271                     else if (wcsncmp(&(className[25]), W("System.Byte"), 11) == 0)
 272                     {
 273                         m_simdHandleCache->SIMDUByteHandle = typeHnd;
 274                         simdBaseType                       = TYP_UBYTE;
 275                         JITDUMP("  Found type SIMD Vector<ubyte>\n");
 276                     }
 277                     else if (wcsncmp(&(className[25]), W("System.Double"), 13) == 0)
 278                     {
 279                         m_simdHandleCache->SIMDDoubleHandle = typeHnd;
 280                         simdBaseType                        = TYP_DOUBLE;
 281                         JITDUMP("  Found type SIMD Vector<Double>\n");
 282                     }
 283                     else if (wcsncmp(&(className[25]), W("System.Int64"), 12) == 0)
 284                     {
 285                         m_simdHandleCache->SIMDLongHandle = typeHnd;
 286                         simdBaseType                      = TYP_LONG;
 287                         JITDUMP("  Found type SIMD Vector<Long>\n");
 288                     }
 289                     else if (wcsncmp(&(className[25]), W("System.Int16"), 12) == 0)
 290                     {
 291                         m_simdHandleCache->SIMDShortHandle = typeHnd;
 292                         simdBaseType                       = TYP_SHORT;
 293                         JITDUMP("  Found type SIMD Vector<short>\n");
 294                     }
 295                     else if (wcsncmp(&(className[25]), W("System.SByte"), 12) == 0)
 296                     {
 297                         m_simdHandleCache->SIMDByteHandle = typeHnd;
 298                         simdBaseType                      = TYP_BYTE;
 299                         JITDUMP("  Found type SIMD Vector<byte>\n");
 300                     }
 301                     else if (wcsncmp(&(className[25]), W("System.UInt32"), 13) == 0)
 302                     {
 303                         m_simdHandleCache->SIMDUIntHandle = typeHnd;
 304                         simdBaseType                      = TYP_UINT;
 305                         JITDUMP("  Found type SIMD Vector<uint>\n");
 306                     }
 307                     else if (wcsncmp(&(className[25]), W("System.UInt64"), 13) == 0)
 308                     {
 309                         m_simdHandleCache->SIMDULongHandle = typeHnd;
 310                         simdBaseType                       = TYP_ULONG;
 311                         JITDUMP("  Found type SIMD Vector<ulong>\n");
 312                     }
 313                     else
 314                     {
 315                         JITDUMP("  Unknown SIMD Vector<T>\n");
 316                     }
 317                 }
 318                 else if (wcsncmp(&(className[16]), W("Vector2"), 8) == 0)
 319                 {
 320                     m_simdHandleCache->SIMDVector2Handle = typeHnd;
 321
 322                     simdBaseType = TYP_FLOAT;
 323                     size         = 2 * genTypeSize(TYP_FLOAT);
 324                     assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE));
 325                     JITDUMP(" Found Vector2\n");
 326                 }
 327                 else if (wcsncmp(&(className[16]), W("Vector3"), 8) == 0)
 328                 {
 329                     m_simdHandleCache->SIMDVector3Handle = typeHnd;
 330
 331                     simdBaseType = TYP_FLOAT;
 332                     size         = 3 * genTypeSize(TYP_FLOAT);
 333                     assert(size == info.compCompHnd->getClassSize(typeHnd));
 334                     JITDUMP(" Found Vector3\n");
 335                 }
 336                 else if (wcsncmp(&(className[16]), W("Vector4"), 8) == 0)
 337                 {
 338                     m_simdHandleCache->SIMDVector4Handle = typeHnd;
 339
 340                     simdBaseType = TYP_FLOAT;
 341                     size         = 4 * genTypeSize(TYP_FLOAT);
 342                     assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE));
 343                     JITDUMP(" Found Vector4\n");
 344                 }
 345                 else if (wcsncmp(&(className[16]), W("Vector"), 6) == 0)
 346                 {
 347                     m_simdHandleCache->SIMDVectorHandle = typeHnd;
 348                     JITDUMP(" Found type Vector\n");
 349                 }
 350                 else
 351                 {
 352                     JITDUMP("  Unknown SIMD Type\n");
 353                 }
 354             }
 355         }
 356         if (simdBaseType != TYP_UNKNOWN && sizeBytes != nullptr)
 357         {
 358             // If not a fixed size vector then its size is same as SIMD vector
 359             // register length in bytes
 360             if (size == 0)
 361             {
 362                 size = getSIMDVectorRegisterByteLength();
 363             }
 364
 365             *sizeBytes = size;
 366             setUsesSIMDTypes(true);
 367         }
 368     }
 369 #ifdef FEATURE_HW_INTRINSICS
 370     else if (isIntrinsicType(typeHnd))
 371     {
 372         const size_t Vector64SizeBytes  = 64 / 8;
 373         const size_t Vector128SizeBytes = 128 / 8;
 374         const size_t Vector256SizeBytes = 256 / 8;
 375
 376 #if defined(_TARGET_XARCH_)
 377         static_assert_no_msg(YMM_REGSIZE_BYTES == Vector256SizeBytes);
 378         static_assert_no_msg(XMM_REGSIZE_BYTES == Vector128SizeBytes);
 379
 380         if (typeHnd == m_simdHandleCache->Vector256FloatHandle)
 381         {
 382             simdBaseType = TYP_FLOAT;
 383             size         = Vector256SizeBytes;
 384             JITDUMP("  Known type Vector256<float>\n");
 385         }
 386         else if (typeHnd == m_simdHandleCache->Vector256DoubleHandle)
 387         {
 388             simdBaseType = TYP_DOUBLE;
 389             size         = Vector256SizeBytes;
 390             JITDUMP("  Known type Vector256<double>\n");
 391         }
 392         else if (typeHnd == m_simdHandleCache->Vector256IntHandle)
 393         {
 394             simdBaseType = TYP_INT;
 395             size         = Vector256SizeBytes;
 396             JITDUMP("  Known type Vector256<int>\n");
 397         }
 398         else if (typeHnd == m_simdHandleCache->Vector256UIntHandle)
 399         {
 400             simdBaseType = TYP_UINT;
 401             size         = Vector256SizeBytes;
 402             JITDUMP("  Known type Vector256<uint>\n");
 403         }
 404         else if (typeHnd == m_simdHandleCache->Vector256ShortHandle)
 405         {
 406             simdBaseType = TYP_SHORT;
 407             size         = Vector256SizeBytes;
 408             JITDUMP("  Known type Vector256<short>\n");
 409         }
 410         else if (typeHnd == m_simdHandleCache->Vector256UShortHandle)
 411         {
 412             simdBaseType = TYP_USHORT;
 413             size         = Vector256SizeBytes;
 414             JITDUMP("  Known type Vector256<ushort>\n");
 415         }
 416         else if (typeHnd == m_simdHandleCache->Vector256ByteHandle)
 417         {
 418             simdBaseType = TYP_BYTE;
 419             size         = Vector256SizeBytes;
 420             JITDUMP("  Known type Vector256<sbyte>\n");
 421         }
 422         else if (typeHnd == m_simdHandleCache->Vector256UByteHandle)
 423         {
 424             simdBaseType = TYP_UBYTE;
 425             size         = Vector256SizeBytes;
 426             JITDUMP("  Known type Vector256<byte>\n");
 427         }
 428         else if (typeHnd == m_simdHandleCache->Vector256LongHandle)
 429         {
 430             simdBaseType = TYP_LONG;
 431             size         = Vector256SizeBytes;
 432             JITDUMP("  Known type Vector256<long>\n");
 433         }
 434         else if (typeHnd == m_simdHandleCache->Vector256ULongHandle)
 435         {
 436             simdBaseType = TYP_ULONG;
 437             size         = Vector256SizeBytes;
 438             JITDUMP("  Known type Vector256<ulong>\n");
 439         }
 440         else
 441 #endif // defined(_TARGET_XARCH)
 442             if (typeHnd == m_simdHandleCache->Vector128FloatHandle)
 443         {
 444             simdBaseType = TYP_FLOAT;
 445             size         = Vector128SizeBytes;
 446             JITDUMP("  Known type Vector128<float>\n");
 447         }
 448         else if (typeHnd == m_simdHandleCache->Vector128DoubleHandle)
 449         {
 450             simdBaseType = TYP_DOUBLE;
 451             size         = Vector128SizeBytes;
 452             JITDUMP("  Known type Vector128<double>\n");
 453         }
 454         else if (typeHnd == m_simdHandleCache->Vector128IntHandle)
 455         {
 456             simdBaseType = TYP_INT;
 457             size         = Vector128SizeBytes;
 458             JITDUMP("  Known type Vector128<int>\n");
 459         }
 460         else if (typeHnd == m_simdHandleCache->Vector128UIntHandle)
 461         {
 462             simdBaseType = TYP_UINT;
 463             size         = Vector128SizeBytes;
 464             JITDUMP("  Known type Vector128<uint>\n");
 465         }
 466         else if (typeHnd == m_simdHandleCache->Vector128ShortHandle)
 467         {
 468             simdBaseType = TYP_SHORT;
 469             size         = Vector128SizeBytes;
 470             JITDUMP("  Known type Vector128<short>\n");
 471         }
 472         else if (typeHnd == m_simdHandleCache->Vector128UShortHandle)
 473         {
 474             simdBaseType = TYP_USHORT;
 475             size         = Vector128SizeBytes;
 476             JITDUMP("  Known type Vector128<ushort>\n");
 477         }
 478         else if (typeHnd == m_simdHandleCache->Vector128ByteHandle)
 479         {
 480             simdBaseType = TYP_BYTE;
 481             size         = Vector128SizeBytes;
 482             JITDUMP("  Known type Vector128<sbyte>\n");
 483         }
 484         else if (typeHnd == m_simdHandleCache->Vector128UByteHandle)
 485         {
 486             simdBaseType = TYP_UBYTE;
 487             size         = Vector128SizeBytes;
 488             JITDUMP("  Known type Vector128<byte>\n");
 489         }
 490         else if (typeHnd == m_simdHandleCache->Vector128LongHandle)
 491         {
 492             simdBaseType = TYP_LONG;
 493             size         = Vector128SizeBytes;
 494             JITDUMP("  Known type Vector128<long>\n");
 495         }
 496         else if (typeHnd == m_simdHandleCache->Vector128ULongHandle)
 497         {
 498             simdBaseType = TYP_ULONG;
 499             size         = Vector128SizeBytes;
 500             JITDUMP("  Known type Vector128<ulong>\n");
 501         }
 502         else
 503 #if defined(_TARGET_ARM64_)
 504             if (typeHnd == m_simdHandleCache->Vector64FloatHandle)
 505         {
 506             simdBaseType = TYP_FLOAT;
 507             size         = Vector64SizeBytes;
 508             JITDUMP("  Known type Vector64<float>\n");
 509         }
 510         else if (typeHnd == m_simdHandleCache->Vector64IntHandle)
 511         {
 512             simdBaseType = TYP_INT;
 513             size         = Vector64SizeBytes;
 514             JITDUMP("  Known type Vector64<int>\n");
 515         }
 516         else if (typeHnd == m_simdHandleCache->Vector64UIntHandle)
 517         {
 518             simdBaseType = TYP_UINT;
 519             size         = Vector64SizeBytes;
 520             JITDUMP("  Known type Vector64<uint>\n");
 521         }
 522         else if (typeHnd == m_simdHandleCache->Vector64ShortHandle)
 523         {
 524             simdBaseType = TYP_SHORT;
 525             size         = Vector64SizeBytes;
 526             JITDUMP("  Known type Vector64<short>\n");
 527         }
 528         else if (typeHnd == m_simdHandleCache->Vector64UShortHandle)
 529         {
 530             simdBaseType = TYP_USHORT;
 531             size         = Vector64SizeBytes;
 532             JITDUMP("  Known type Vector64<ushort>\n");
 533         }
 534         else if (typeHnd == m_simdHandleCache->Vector64ByteHandle)
 535         {
 536             simdBaseType = TYP_BYTE;
 537             size         = Vector64SizeBytes;
 538             JITDUMP("  Known type Vector64<sbyte>\n");
 539         }
 540         else if (typeHnd == m_simdHandleCache->Vector64UByteHandle)
 541         {
 542             simdBaseType = TYP_UBYTE;
 543             size         = Vector64SizeBytes;
 544             JITDUMP("  Known type Vector64<byte>\n");
 545         }
 546 #endif // defined(_TARGET_ARM64_)
 547
 548         // slow path search
 549         if (simdBaseType == TYP_UNKNOWN)
 550         {
 551             // Doesn't match with any of the cached type handles.
 552             const char*          className   = getClassNameFromMetadata(typeHnd, nullptr);
 553             CORINFO_CLASS_HANDLE baseTypeHnd = getTypeInstantiationArgument(typeHnd, 0);
 554
 555             if (baseTypeHnd != nullptr)
 556             {
 557                 CorInfoType type = info.compCompHnd->getTypeForPrimitiveNumericClass(baseTypeHnd);
 558
 559                 JITDUMP("HW Intrinsic SIMD Candidate Type %s with Base Type %s\n", className,
 560                         getClassNameFromMetadata(baseTypeHnd, nullptr));
 561
 562 #if defined(_TARGET_XARCH_)
 563                 if (strcmp(className, "Vector256`1") == 0)
 564                 {
 565                     size = Vector256SizeBytes;
 566                     switch (type)
 567                     {
 568                         case CORINFO_TYPE_FLOAT:
 569                             m_simdHandleCache->Vector256FloatHandle = typeHnd;
 570                             simdBaseType                            = TYP_FLOAT;
 571                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<float>\n");
 572                             break;
 573                         case CORINFO_TYPE_DOUBLE:
 574                             m_simdHandleCache->Vector256DoubleHandle = typeHnd;
 575                             simdBaseType                             = TYP_DOUBLE;
 576                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<double>\n");
 577                             break;
 578                         case CORINFO_TYPE_INT:
 579                             m_simdHandleCache->Vector256IntHandle = typeHnd;
 580                             simdBaseType                          = TYP_INT;
 581                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<int>\n");
 582                             break;
 583                         case CORINFO_TYPE_UINT:
 584                             m_simdHandleCache->Vector256UIntHandle = typeHnd;
 585                             simdBaseType                           = TYP_UINT;
 586                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<uint>\n");
 587                             break;
 588                         case CORINFO_TYPE_SHORT:
 589                             m_simdHandleCache->Vector256ShortHandle = typeHnd;
 590                             simdBaseType                            = TYP_SHORT;
 591                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<short>\n");
 592                             break;
 593                         case CORINFO_TYPE_USHORT:
 594                             m_simdHandleCache->Vector256UShortHandle = typeHnd;
 595                             simdBaseType                             = TYP_USHORT;
 596                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<ushort>\n");
 597                             break;
 598                         case CORINFO_TYPE_LONG:
 599                             m_simdHandleCache->Vector256LongHandle = typeHnd;
 600                             simdBaseType                           = TYP_LONG;
 601                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<long>\n");
 602                             break;
 603                         case CORINFO_TYPE_ULONG:
 604                             m_simdHandleCache->Vector256ULongHandle = typeHnd;
 605                             simdBaseType                            = TYP_ULONG;
 606                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<ulong>\n");
 607                             break;
 608                         case CORINFO_TYPE_UBYTE:
 609                             m_simdHandleCache->Vector256UByteHandle = typeHnd;
 610                             simdBaseType                            = TYP_UBYTE;
 611                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<byte>\n");
 612                             break;
 613                         case CORINFO_TYPE_BYTE:
 614                             m_simdHandleCache->Vector256ByteHandle = typeHnd;
 615                             simdBaseType                           = TYP_BYTE;
 616                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<sbyte>\n");
 617                             break;
 618
 619                         default:
 620                             JITDUMP("  Unknown Hardware Intrinsic SIMD Type Vector256<T>\n");
 621                     }
 622                 }
 623                 else
 624 #endif // defined(_TARGET_XARCH_)
 625                     if (strcmp(className, "Vector128`1") == 0)
 626                 {
 627                     size = Vector128SizeBytes;
 628                     switch (type)
 629                     {
 630                         case CORINFO_TYPE_FLOAT:
 631                             m_simdHandleCache->Vector128FloatHandle = typeHnd;
 632                             simdBaseType                            = TYP_FLOAT;
 633                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<float>\n");
 634                             break;
 635                         case CORINFO_TYPE_DOUBLE:
 636                             m_simdHandleCache->Vector128DoubleHandle = typeHnd;
 637                             simdBaseType                             = TYP_DOUBLE;
 638                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<double>\n");
 639                             break;
 640                         case CORINFO_TYPE_INT:
 641                             m_simdHandleCache->Vector128IntHandle = typeHnd;
 642                             simdBaseType                          = TYP_INT;
 643                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<int>\n");
 644                             break;
 645                         case CORINFO_TYPE_UINT:
 646                             m_simdHandleCache->Vector128UIntHandle = typeHnd;
 647                             simdBaseType                           = TYP_UINT;
 648                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<uint>\n");
 649                             break;
 650                         case CORINFO_TYPE_SHORT:
 651                             m_simdHandleCache->Vector128ShortHandle = typeHnd;
 652                             simdBaseType                            = TYP_SHORT;
 653                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<short>\n");
 654                             break;
 655                         case CORINFO_TYPE_USHORT:
 656                             m_simdHandleCache->Vector128UShortHandle = typeHnd;
 657                             simdBaseType                             = TYP_USHORT;
 658                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<ushort>\n");
 659                             break;
 660                         case CORINFO_TYPE_LONG:
 661                             m_simdHandleCache->Vector128LongHandle = typeHnd;
 662                             simdBaseType                           = TYP_LONG;
 663                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<long>\n");
 664                             break;
 665                         case CORINFO_TYPE_ULONG:
 666                             m_simdHandleCache->Vector128ULongHandle = typeHnd;
 667                             simdBaseType                            = TYP_ULONG;
 668                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<ulong>\n");
 669                             break;
 670                         case CORINFO_TYPE_UBYTE:
 671                             m_simdHandleCache->Vector128UByteHandle = typeHnd;
 672                             simdBaseType                            = TYP_UBYTE;
 673                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<byte>\n");
 674                             break;
 675                         case CORINFO_TYPE_BYTE:
 676                             m_simdHandleCache->Vector128ByteHandle = typeHnd;
 677                             simdBaseType                           = TYP_BYTE;
 678                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<sbyte>\n");
 679                             break;
 680
 681                         default:
 682                             JITDUMP("  Unknown Hardware Intrinsic SIMD Type Vector128<T>\n");
 683                     }
 684                 }
 685 #if defined(_TARGET_ARM64_)
 686                 else if (strcmp(className, "Vector64`1") == 0)
 687                 {
 688                     size = Vector64SizeBytes;
 689                     switch (type)
 690                     {
 691                         case CORINFO_TYPE_FLOAT:
 692                             m_simdHandleCache->Vector64FloatHandle = typeHnd;
 693                             simdBaseType                           = TYP_FLOAT;
 694                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<float>\n");
 695                             break;
 696                         case CORINFO_TYPE_INT:
 697                             m_simdHandleCache->Vector64IntHandle = typeHnd;
 698                             simdBaseType                         = TYP_INT;
 699                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<int>\n");
 700                             break;
 701                         case CORINFO_TYPE_UINT:
 702                             m_simdHandleCache->Vector64UIntHandle = typeHnd;
 703                             simdBaseType                          = TYP_UINT;
 704                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<uint>\n");
 705                             break;
 706                         case CORINFO_TYPE_SHORT:
 707                             m_simdHandleCache->Vector64ShortHandle = typeHnd;
 708                             simdBaseType                           = TYP_SHORT;
 709                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<short>\n");
 710                             break;
 711                         case CORINFO_TYPE_USHORT:
 712                             m_simdHandleCache->Vector64UShortHandle = typeHnd;
 713                             simdBaseType                            = TYP_USHORT;
 714                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<ushort>\n");
 715                             break;
 716                         case CORINFO_TYPE_UBYTE:
 717                             m_simdHandleCache->Vector64UByteHandle = typeHnd;
 718                             simdBaseType                           = TYP_UBYTE;
 719                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<byte>\n");
 720                             break;
 721                         case CORINFO_TYPE_BYTE:
 722                             m_simdHandleCache->Vector64ByteHandle = typeHnd;
 723                             simdBaseType                          = TYP_BYTE;
 724                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<sbyte>\n");
 725                             break;
 726
 727                         default:
 728                             JITDUMP("  Unknown Hardware Intrinsic SIMD Type Vector64<T>\n");
 729                     }
 730                 }
 731 #endif // defined(_TARGET_ARM64_)
 732             }
 733         }
 734
 735         if (sizeBytes != nullptr)
 736         {
 737             *sizeBytes = size;
 738         }
 739
 740         if (simdBaseType != TYP_UNKNOWN)
 741         {
 742             setUsesSIMDTypes(true);
 743         }
 744     }
 745 #endif // FEATURE_HW_INTRINSICS
 746
 747     return simdBaseType;
 748 }
 749
 750 //--------------------------------------------------------------------------------------
 751 // getSIMDIntrinsicInfo: get SIMD intrinsic info given the method handle.
 752 //
 753 // Arguments:
 754 //    inOutTypeHnd    - The handle of the type on which the method is invoked.  This is an in-out param.
 755 //    methodHnd       - The handle of the method we're interested in.
 756 //    sig             - method signature info
 757 //    isNewObj        - whether this call represents a newboj constructor call
 758 //    argCount        - argument count - out pram
 759 //    baseType        - base type of the intrinsic - out param
 760 //    sizeBytes       - size of SIMD vector type on which the method is invoked - out param
 761 //
 762 // Return Value:
 763 //    SIMDIntrinsicInfo struct initialized corresponding to methodHnd.
 764 //    Sets SIMDIntrinsicInfo.id to SIMDIntrinsicInvalid if methodHnd doesn't correspond
 765 //    to any SIMD intrinsic.  Also, sets the out params inOutTypeHnd, argCount, baseType and
 766 //    sizeBytes.
 767 //
 768 //    Note that VectorMath class doesn't have a base type and first argument of the method
 769 //    determines the SIMD vector type on which intrinsic is invoked. In such a case inOutTypeHnd
 770 //    is modified by this routine.
 771 //
 772 // TODO-Throughput: The current implementation is based on method name string parsing.
 773 //         Although we now have type identification from the VM, the parsing of intrinsic names
 774 //         could be made more efficient.
 775 //
 776 const SIMDIntrinsicInfo* Compiler::getSIMDIntrinsicInfo(CORINFO_CLASS_HANDLE* inOutTypeHnd,
 777                                                         CORINFO_METHOD_HANDLE methodHnd,
 778                                                         CORINFO_SIG_INFO*     sig,
 779                                                         bool                  isNewObj,
 780                                                         unsigned*             argCount,
 781                                                         var_types*            baseType,
 782                                                         unsigned*             sizeBytes)
 783 {
 784     assert(featureSIMD);
 785     assert(baseType != nullptr);
 786     assert(sizeBytes != nullptr);
 787
 788     // get baseType and size of the type
 789     CORINFO_CLASS_HANDLE typeHnd = *inOutTypeHnd;
 790     *baseType                    = getBaseTypeAndSizeOfSIMDType(typeHnd, sizeBytes);
 791
 792     if (typeHnd == m_simdHandleCache->SIMDVectorHandle)
 793     {
 794         // All of the supported intrinsics on this static class take a first argument that's a vector,
 795         // which determines the baseType.
 796         // The exception is the IsHardwareAccelerated property, which is handled as a special case.
 797         assert(*baseType == TYP_UNKNOWN);
 798         if (sig->numArgs == 0)
 799         {
 800             const SIMDIntrinsicInfo* hwAccelIntrinsicInfo = &(simdIntrinsicInfoArray[SIMDIntrinsicHWAccel]);
 801             if ((strcmp(eeGetMethodName(methodHnd, nullptr), hwAccelIntrinsicInfo->methodName) == 0) &&
 802                 JITtype2varType(sig->retType) == hwAccelIntrinsicInfo->retType)
 803             {
 804                 // Sanity check
 805                 assert(hwAccelIntrinsicInfo->argCount == 0 && hwAccelIntrinsicInfo->isInstMethod == false);
 806                 return hwAccelIntrinsicInfo;
 807             }
 808             return nullptr;
 809         }
 810         else
 811         {
 812             typeHnd       = info.compCompHnd->getArgClass(sig, sig->args);
 813             *inOutTypeHnd = typeHnd;
 814             *baseType     = getBaseTypeAndSizeOfSIMDType(typeHnd, sizeBytes);
 815         }
 816     }
 817
 818     if (*baseType == TYP_UNKNOWN)
 819     {
 820         JITDUMP("NOT a SIMD Intrinsic: unsupported baseType\n");
 821         return nullptr;
 822     }
 823
 824     // account for implicit "this" arg
 825     *argCount = sig->numArgs;
 826     if (sig->hasThis())
 827     {
 828         *argCount += 1;
 829     }
 830
 831     // Get the Intrinsic Id by parsing method name.
 832     //
 833     // TODO-Throughput: replace sequential search by binary search by arranging entries
 834     // sorted by method name.
 835     SIMDIntrinsicID intrinsicId = SIMDIntrinsicInvalid;
 836     const char*     methodName  = eeGetMethodName(methodHnd, nullptr);
 837     for (int i = SIMDIntrinsicNone + 1; i < SIMDIntrinsicInvalid; ++i)
 838     {
 839         if (strcmp(methodName, simdIntrinsicInfoArray[i].methodName) == 0)
 840         {
 841             // Found an entry for the method; further check whether it is one of
 842             // the supported base types.
 843             bool found = false;
 844             for (int j = 0; j < SIMD_INTRINSIC_MAX_BASETYPE_COUNT; ++j)
 845             {
 846                 // Convention: if there are fewer base types supported than MAX_BASETYPE_COUNT,
 847                 // the end of the list is marked by TYP_UNDEF.
 848                 if (simdIntrinsicInfoArray[i].supportedBaseTypes[j] == TYP_UNDEF)
 849                 {
 850                     break;
 851                 }
 852
 853                 if (simdIntrinsicInfoArray[i].supportedBaseTypes[j] == *baseType)
 854                 {
 855                     found = true;
 856                     break;
 857                 }
 858             }
 859
 860             if (!found)
 861             {
 862                 continue;
 863             }
 864
 865             // Now, check the arguments.
 866             unsigned int fixedArgCnt    = simdIntrinsicInfoArray[i].argCount;
 867             unsigned int expectedArgCnt = fixedArgCnt;
 868
 869             // First handle SIMDIntrinsicInitN, where the arg count depends on the type.
 870             // The listed arg types include the vector and the first two init values, which is the expected number
 871             // for Vector2.  For other cases, we'll check their types here.
 872             if (*argCount > expectedArgCnt)
 873             {
 874                 if (i == SIMDIntrinsicInitN)
 875                 {
 876                     if (*argCount == 3 && typeHnd == m_simdHandleCache->SIMDVector2Handle)
 877                     {
 878                         expectedArgCnt = 3;
 879                     }
 880                     else if (*argCount == 4 && typeHnd == m_simdHandleCache->SIMDVector3Handle)
 881                     {
 882                         expectedArgCnt = 4;
 883                     }
 884                     else if (*argCount == 5 && typeHnd == m_simdHandleCache->SIMDVector4Handle)
 885                     {
 886                         expectedArgCnt = 5;
 887                     }
 888                 }
 889                 else if (i == SIMDIntrinsicInitFixed)
 890                 {
 891                     if (*argCount == 4 && typeHnd == m_simdHandleCache->SIMDVector4Handle)
 892                     {
 893                         expectedArgCnt = 4;
 894                     }
 895                 }
 896             }
 897             if (*argCount != expectedArgCnt)
 898             {
 899                 continue;
 900             }
 901
 902             // Validate the types of individual args passed are what is expected of.
 903             // If any of the types don't match with what is expected, don't consider
 904             // as an intrinsic.  This will make an older JIT with SIMD capabilities
 905             // resilient to breaking changes to SIMD managed API.
 906             //
 907             // Note that from IL type stack, args get popped in right to left order
 908             // whereas args get listed in method signatures in left to right order.
 909
 910             int stackIndex = (expectedArgCnt - 1);
 911
 912             // Track the arguments from the signature - we currently only use this to distinguish
 913             // integral and pointer types, both of which will by TYP_I_IMPL on the importer stack.
 914             CORINFO_ARG_LIST_HANDLE argLst = sig->args;
 915
 916             CORINFO_CLASS_HANDLE argClass;
 917             for (unsigned int argIndex = 0; found == true && argIndex < expectedArgCnt; argIndex++)
 918             {
 919                 bool isThisPtr = ((argIndex == 0) && sig->hasThis());
 920
 921                 // In case of "newobj SIMDVector<T>(T val)", thisPtr won't be present on type stack.
 922                 // We don't check anything in that case.
 923                 if (!isThisPtr || !isNewObj)
 924                 {
 925                     GenTree*  arg     = impStackTop(stackIndex).val;
 926                     var_types argType = arg->TypeGet();
 927
 928                     var_types expectedArgType;
 929                     if (argIndex < fixedArgCnt)
 930                     {
 931                         // Convention:
 932                         //   - intrinsicInfo.argType[i] == TYP_UNDEF - intrinsic doesn't have a valid arg at position i
 933                         //   - intrinsicInfo.argType[i] == TYP_UNKNOWN - arg type should be same as basetype
 934                         // Note that we pop the args off in reverse order.
 935                         expectedArgType = simdIntrinsicInfoArray[i].argType[argIndex];
 936                         assert(expectedArgType != TYP_UNDEF);
 937                         if (expectedArgType == TYP_UNKNOWN)
 938                         {
 939                             // The type of the argument will be genActualType(*baseType).
 940                             expectedArgType = genActualType(*baseType);
 941                             argType         = genActualType(argType);
 942                         }
 943                     }
 944                     else
 945                     {
 946                         expectedArgType = *baseType;
 947                     }
 948
 949                     if (!isThisPtr && argType == TYP_I_IMPL)
 950                     {
 951                         // The reference implementation has a constructor that takes a pointer.
 952                         // We don't want to recognize that one.  This requires us to look at the CorInfoType
 953                         // in order to distinguish a signature with a pointer argument from one with an
 954                         // integer argument of pointer size, both of which will be TYP_I_IMPL on the stack.
 955                         // TODO-Review: This seems quite fragile.  We should consider beefing up the checking
 956                         // here.
 957                         CorInfoType corType = strip(info.compCompHnd->getArgType(sig, argLst, &argClass));
 958                         if (corType == CORINFO_TYPE_PTR)
 959                         {
 960                             found = false;
 961                         }
 962                     }
 963
 964                     if (varTypeIsSIMD(argType))
 965                     {
 966                         argType = TYP_STRUCT;
 967                     }
 968                     if (argType != expectedArgType)
 969                     {
 970                         found = false;
 971                     }
 972                 }
 973                 if (argIndex != 0 || !sig->hasThis())
 974                 {
 975                     argLst = info.compCompHnd->getArgNext(argLst);
 976                 }
 977                 stackIndex--;
 978             }
 979
 980             // Cross check return type and static vs. instance is what we are expecting.
 981             // If not, don't consider it as an intrinsic.
 982             // Note that ret type of TYP_UNKNOWN means that it is not known apriori and must be same as baseType
 983             if (found)
 984             {
 985                 var_types expectedRetType = simdIntrinsicInfoArray[i].retType;
 986                 if (expectedRetType == TYP_UNKNOWN)
 987                 {
 988                     // JIT maps uint/ulong type vars to TYP_INT/TYP_LONG.
 989                     expectedRetType =
 990                         (*baseType == TYP_UINT || *baseType == TYP_ULONG) ? genActualType(*baseType) : *baseType;
 991                 }
 992
 993                 if (JITtype2varType(sig->retType) != expectedRetType ||
 994                     sig->hasThis() != simdIntrinsicInfoArray[i].isInstMethod)
 995                 {
 996                     found = false;
 997                 }
 998             }
 999
1000             if (found)
1001             {
1002                 intrinsicId = (SIMDIntrinsicID)i;
1003                 break;
1004             }
1005         }
1006     }
1007
1008     if (intrinsicId != SIMDIntrinsicInvalid)
1009     {
1010         JITDUMP("Method %s maps to SIMD intrinsic %s\n", methodName, simdIntrinsicNames[intrinsicId]);
1011         return &simdIntrinsicInfoArray[intrinsicId];
1012     }
1013     else
1014     {
1015         JITDUMP("Method %s is NOT a SIMD intrinsic\n", methodName);
1016     }
1017
1018     return nullptr;
1019 }
1020
1021 // Pops and returns GenTree node from importer's type stack.
1022 // Normalizes TYP_STRUCT value in case of GT_CALL, GT_RET_EXPR and arg nodes.
1023 //
1024 // Arguments:
1025 //    type        -  the type of value that the caller expects to be popped off the stack.
1026 //    expectAddr  -  if true indicates we are expecting type stack entry to be a TYP_BYREF.
1027 //    structType  -  the class handle to use when normalizing if it is not the same as the stack entry class handle;
1028 //                   this can happen for certain scenarios, such as folding away a static cast, where we want the
1029 //                   value popped to have the type that would have been returned.
1030 //
1031 // Notes:
1032 //    If the popped value is a struct, and the expected type is a simd type, it will be set
1033 //    to that type, otherwise it will assert if the type being popped is not the expected type.
1034
1035 GenTree* Compiler::impSIMDPopStack(var_types type, bool expectAddr, CORINFO_CLASS_HANDLE structType)
1036 {
1037     StackEntry se   = impPopStack();
1038     typeInfo   ti   = se.seTypeInfo;
1039     GenTree*   tree = se.val;
1040
1041     // If expectAddr is true implies what we have on stack is address and we need
1042     // SIMD type struct that it points to.
1043     if (expectAddr)
1044     {
1045         assert(tree->TypeGet() == TYP_BYREF);
1046         if (tree->OperGet() == GT_ADDR)
1047         {
1048             tree = tree->gtGetOp1();
1049         }
1050         else
1051         {
1052             tree = gtNewOperNode(GT_IND, type, tree);
1053         }
1054     }
1055
1056     bool isParam = false;
1057
1058     // If we have a ldobj of a SIMD local we need to transform it.
1059     if (tree->OperGet() == GT_OBJ)
1060     {
1061         GenTree* addr = tree->gtOp.gtOp1;
1062         if ((addr->OperGet() == GT_ADDR) && isSIMDTypeLocal(addr->gtOp.gtOp1))
1063         {
1064             tree = addr->gtOp.gtOp1;
1065         }
1066     }
1067
1068     if (tree->OperGet() == GT_LCL_VAR)
1069     {
1070         unsigned   lclNum    = tree->AsLclVarCommon()->GetLclNum();
1071         LclVarDsc* lclVarDsc = &lvaTable[lclNum];
1072         isParam              = lclVarDsc->lvIsParam;
1073     }
1074
1075     // normalize TYP_STRUCT value
1076     if (varTypeIsStruct(tree) && ((tree->OperGet() == GT_RET_EXPR) || (tree->OperGet() == GT_CALL) || isParam))
1077     {
1078         assert(ti.IsType(TI_STRUCT));
1079
1080         if (structType == nullptr)
1081         {
1082             structType = ti.GetClassHandleForValueClass();
1083         }
1084
1085         tree = impNormStructVal(tree, structType, (unsigned)CHECK_SPILL_ALL);
1086     }
1087
1088     // Now set the type of the tree to the specialized SIMD struct type, if applicable.
1089     if (genActualType(tree->gtType) != genActualType(type))
1090     {
1091         assert(tree->gtType == TYP_STRUCT);
1092         tree->gtType = type;
1093     }
1094     else if (tree->gtType == TYP_BYREF)
1095     {
1096         assert(tree->IsLocal() || (tree->OperGet() == GT_RET_EXPR) || (tree->OperGet() == GT_CALL) ||
1097                ((tree->gtOper == GT_ADDR) && varTypeIsSIMD(tree->gtGetOp1())));
1098     }
1099
1100     return tree;
1101 }
1102
1103 // impSIMDGetFixed: Create a GT_SIMD tree for a Get property of SIMD vector with a fixed index.
1104 //
1105 // Arguments:
1106 //    baseType - The base (element) type of the SIMD vector.
1107 //    simdSize - The total size in bytes of the SIMD vector.
1108 //    index    - The index of the field to get.
1109 //
1110 // Return Value:
1111 //    Returns a GT_SIMD node with the SIMDIntrinsicGetItem intrinsic id.
1112 //
1113 GenTreeSIMD* Compiler::impSIMDGetFixed(var_types simdType, var_types baseType, unsigned simdSize, int index)
1114 {
1115     assert(simdSize >= ((index + 1) * genTypeSize(baseType)));
1116
1117     // op1 is a SIMD source.
1118     GenTree* op1 = impSIMDPopStack(simdType, true);
1119
1120     GenTree*     op2      = gtNewIconNode(index);
1121     GenTreeSIMD* simdTree = gtNewSIMDNode(baseType, op1, op2, SIMDIntrinsicGetItem, baseType, simdSize);
1122     return simdTree;
1123 }
1124
1125 #ifdef _TARGET_XARCH_
1126 // impSIMDLongRelOpEqual: transforms operands and returns the SIMD intrinsic to be applied on
1127 // transformed operands to obtain == comparison result.
1128 //
1129 // Arguments:
1130 //    typeHnd  -  type handle of SIMD vector
1131 //    size     -  SIMD vector size
1132 //    op1      -  in-out parameter; first operand
1133 //    op2      -  in-out parameter; second operand
1134 //
1135 // Return Value:
1136 //    Modifies in-out params op1, op2 and returns intrinsic ID to be applied to modified operands
1137 //
1138 SIMDIntrinsicID Compiler::impSIMDLongRelOpEqual(CORINFO_CLASS_HANDLE typeHnd,
1139                                                 unsigned             size,
1140                                                 GenTree**            pOp1,
1141                                                 GenTree**            pOp2)
1142 {
1143     var_types simdType = (*pOp1)->TypeGet();
1144     assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1145
1146     // There is no direct SSE2 support for comparing TYP_LONG vectors.
1147     // These have to be implemented in terms of TYP_INT vector comparison operations.
1148     //
1149     // Equality(v1, v2):
1150     // tmp = (v1 == v2) i.e. compare for equality as if v1 and v2 are vector<int>
1151     // result = BitwiseAnd(t, shuffle(t, (2, 3, 0, 1)))
1152     // Shuffle is meant to swap the comparison results of low-32-bits and high 32-bits of respective long elements.
1153
1154     // Compare vector<long> as if they were vector<int> and assign the result to a temp
1155     GenTree* compResult = gtNewSIMDNode(simdType, *pOp1, *pOp2, SIMDIntrinsicEqual, TYP_INT, size);
1156     unsigned lclNum     = lvaGrabTemp(true DEBUGARG("SIMD Long =="));
1157     lvaSetStruct(lclNum, typeHnd, false);
1158     GenTree* tmp = gtNewLclvNode(lclNum, simdType);
1159     GenTree* asg = gtNewTempAssign(lclNum, compResult);
1160
1161     // op1 = GT_COMMA(tmp=compResult, tmp)
1162     // op2 = Shuffle(tmp, 0xB1)
1163     // IntrinsicId = BitwiseAnd
1164     *pOp1 = gtNewOperNode(GT_COMMA, simdType, asg, tmp);
1165     *pOp2 = gtNewSIMDNode(simdType, gtNewLclvNode(lclNum, simdType), gtNewIconNode(SHUFFLE_ZWXY, TYP_INT),
1166                           SIMDIntrinsicShuffleSSE2, TYP_INT, size);
1167     return SIMDIntrinsicBitwiseAnd;
1168 }
1169
1170 // impSIMDLongRelOpGreaterThan: transforms operands and returns the SIMD intrinsic to be applied on
1171 // transformed operands to obtain > comparison result.
1172 //
1173 // Arguments:
1174 //    typeHnd  -  type handle of SIMD vector
1175 //    size     -  SIMD vector size
1176 //    pOp1     -  in-out parameter; first operand
1177 //    pOp2     -  in-out parameter; second operand
1178 //
1179 // Return Value:
1180 //    Modifies in-out params pOp1, pOp2 and returns intrinsic ID to be applied to modified operands
1181 //
1182 SIMDIntrinsicID Compiler::impSIMDLongRelOpGreaterThan(CORINFO_CLASS_HANDLE typeHnd,
1183                                                       unsigned             size,
1184                                                       GenTree**            pOp1,
1185                                                       GenTree**            pOp2)
1186 {
1187     var_types simdType = (*pOp1)->TypeGet();
1188     assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1189
1190     // GreaterThan(v1, v2) where v1 and v2 are vector long.
1191     // Let us consider the case of single long element comparison.
1192     // say L1 = (x1, y1) and L2 = (x2, y2) where x1, y1, x2, and y2 are 32-bit integers that comprise the longs L1 and
1193     // L2.
1194     //
1195     // GreaterThan(L1, L2) can be expressed in terms of > relationship between 32-bit integers that comprise L1 and L2
1196     // as
1197     //                    =  (x1, y1) > (x2, y2)
1198     //                    =  (x1 > x2) || [(x1 == x2) && (y1 > y2)]   - eq (1)
1199     //
1200     // t = (v1 > v2)  32-bit signed comparison
1201     // u = (v1 == v2) 32-bit sized element equality
1202     // v = (v1 > v2)  32-bit unsigned comparison
1203     //
1204     // z = shuffle(t, (3, 3, 1, 1))  - This corresponds to (x1 > x2) in eq(1) above
1205     // t1 = Shuffle(v, (2, 2, 0, 0)) - This corresponds to (y1 > y2) in eq(1) above
1206     // u1 = Shuffle(u, (3, 3, 1, 1)) - This corresponds to (x1 == x2) in eq(1) above
1207     // w = And(t1, u1)               - This corresponds to [(x1 == x2) && (y1 > y2)] in eq(1) above
1208     // Result = BitwiseOr(z, w)
1209
1210     // Since op1 and op2 gets used multiple times, make sure side effects are computed.
1211     GenTree* dupOp1    = nullptr;
1212     GenTree* dupOp2    = nullptr;
1213     GenTree* dupDupOp1 = nullptr;
1214     GenTree* dupDupOp2 = nullptr;
1215
1216     if (((*pOp1)->gtFlags & GTF_SIDE_EFFECT) != 0)
1217     {
1218         dupOp1    = fgInsertCommaFormTemp(pOp1, typeHnd);
1219         dupDupOp1 = gtNewLclvNode(dupOp1->AsLclVarCommon()->GetLclNum(), simdType);
1220     }
1221     else
1222     {
1223         dupOp1    = gtCloneExpr(*pOp1);
1224         dupDupOp1 = gtCloneExpr(*pOp1);
1225     }
1226
1227     if (((*pOp2)->gtFlags & GTF_SIDE_EFFECT) != 0)
1228     {
1229         dupOp2    = fgInsertCommaFormTemp(pOp2, typeHnd);
1230         dupDupOp2 = gtNewLclvNode(dupOp2->AsLclVarCommon()->GetLclNum(), simdType);
1231     }
1232     else
1233     {
1234         dupOp2    = gtCloneExpr(*pOp2);
1235         dupDupOp2 = gtCloneExpr(*pOp2);
1236     }
1237
1238     assert(dupDupOp1 != nullptr && dupDupOp2 != nullptr);
1239     assert(dupOp1 != nullptr && dupOp2 != nullptr);
1240     assert(*pOp1 != nullptr && *pOp2 != nullptr);
1241
1242     // v1GreaterThanv2Signed - signed 32-bit comparison
1243     GenTree* v1GreaterThanv2Signed = gtNewSIMDNode(simdType, *pOp1, *pOp2, SIMDIntrinsicGreaterThan, TYP_INT, size);
1244
1245     // v1Equalsv2 - 32-bit equality
1246     GenTree* v1Equalsv2 = gtNewSIMDNode(simdType, dupOp1, dupOp2, SIMDIntrinsicEqual, TYP_INT, size);
1247
1248     // v1GreaterThanv2Unsigned - unsigned 32-bit comparison
1249     var_types       tempBaseType = TYP_UINT;
1250     SIMDIntrinsicID sid = impSIMDRelOp(SIMDIntrinsicGreaterThan, typeHnd, size, &tempBaseType, &dupDupOp1, &dupDupOp2);
1251     GenTree*        v1GreaterThanv2Unsigned = gtNewSIMDNode(simdType, dupDupOp1, dupDupOp2, sid, tempBaseType, size);
1252
1253     GenTree* z = gtNewSIMDNode(simdType, v1GreaterThanv2Signed, gtNewIconNode(SHUFFLE_WWYY, TYP_INT),
1254                                SIMDIntrinsicShuffleSSE2, TYP_FLOAT, size);
1255     GenTree* t1 = gtNewSIMDNode(simdType, v1GreaterThanv2Unsigned, gtNewIconNode(SHUFFLE_ZZXX, TYP_INT),
1256                                 SIMDIntrinsicShuffleSSE2, TYP_FLOAT, size);
1257     GenTree* u1 = gtNewSIMDNode(simdType, v1Equalsv2, gtNewIconNode(SHUFFLE_WWYY, TYP_INT), SIMDIntrinsicShuffleSSE2,
1258                                 TYP_FLOAT, size);
1259     GenTree* w = gtNewSIMDNode(simdType, u1, t1, SIMDIntrinsicBitwiseAnd, TYP_INT, size);
1260
1261     *pOp1 = z;
1262     *pOp2 = w;
1263     return SIMDIntrinsicBitwiseOr;
1264 }
1265
1266 // impSIMDLongRelOpGreaterThanOrEqual: transforms operands and returns the SIMD intrinsic to be applied on
1267 // transformed operands to obtain >= comparison result.
1268 //
1269 // Arguments:
1270 //    typeHnd  -  type handle of SIMD vector
1271 //    size     -  SIMD vector size
1272 //    pOp1      -  in-out parameter; first operand
1273 //    pOp2      -  in-out parameter; second operand
1274 //
1275 // Return Value:
1276 //    Modifies in-out params pOp1, pOp2 and returns intrinsic ID to be applied to modified operands
1277 //
1278 SIMDIntrinsicID Compiler::impSIMDLongRelOpGreaterThanOrEqual(CORINFO_CLASS_HANDLE typeHnd,
1279                                                              unsigned             size,
1280                                                              GenTree**            pOp1,
1281                                                              GenTree**            pOp2)
1282 {
1283     var_types simdType = (*pOp1)->TypeGet();
1284     assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1285
1286     // expand this to (a == b) | (a > b)
1287     GenTree* dupOp1 = nullptr;
1288     GenTree* dupOp2 = nullptr;
1289
1290     if (((*pOp1)->gtFlags & GTF_SIDE_EFFECT) != 0)
1291     {
1292         dupOp1 = fgInsertCommaFormTemp(pOp1, typeHnd);
1293     }
1294     else
1295     {
1296         dupOp1 = gtCloneExpr(*pOp1);
1297     }
1298
1299     if (((*pOp2)->gtFlags & GTF_SIDE_EFFECT) != 0)
1300     {
1301         dupOp2 = fgInsertCommaFormTemp(pOp2, typeHnd);
1302     }
1303     else
1304     {
1305         dupOp2 = gtCloneExpr(*pOp2);
1306     }
1307
1308     assert(dupOp1 != nullptr && dupOp2 != nullptr);
1309     assert(*pOp1 != nullptr && *pOp2 != nullptr);
1310
1311     // (a==b)
1312     SIMDIntrinsicID id = impSIMDLongRelOpEqual(typeHnd, size, pOp1, pOp2);
1313     *pOp1              = gtNewSIMDNode(simdType, *pOp1, *pOp2, id, TYP_LONG, size);
1314
1315     // (a > b)
1316     id    = impSIMDLongRelOpGreaterThan(typeHnd, size, &dupOp1, &dupOp2);
1317     *pOp2 = gtNewSIMDNode(simdType, dupOp1, dupOp2, id, TYP_LONG, size);
1318
1319     return SIMDIntrinsicBitwiseOr;
1320 }
1321
1322 // impSIMDInt32OrSmallIntRelOpGreaterThanOrEqual: transforms operands and returns the SIMD intrinsic to be applied on
1323 // transformed operands to obtain >= comparison result in case of integer base type vectors
1324 //
1325 // Arguments:
1326 //    typeHnd  -  type handle of SIMD vector
1327 //    size     -  SIMD vector size
1328 //    baseType -  base type of SIMD vector
1329 //    pOp1      -  in-out parameter; first operand
1330 //    pOp2      -  in-out parameter; second operand
1331 //
1332 // Return Value:
1333 //    Modifies in-out params pOp1, pOp2 and returns intrinsic ID to be applied to modified operands
1334 //
1335 SIMDIntrinsicID Compiler::impSIMDIntegralRelOpGreaterThanOrEqual(
1336     CORINFO_CLASS_HANDLE typeHnd, unsigned size, var_types baseType, GenTree** pOp1, GenTree** pOp2)
1337 {
1338     var_types simdType = (*pOp1)->TypeGet();
1339     assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1340
1341     // This routine should be used only for integer base type vectors
1342     assert(varTypeIsIntegral(baseType));
1343     if ((getSIMDSupportLevel() == SIMD_SSE2_Supported) && ((baseType == TYP_LONG) || baseType == TYP_UBYTE))
1344     {
1345         return impSIMDLongRelOpGreaterThanOrEqual(typeHnd, size, pOp1, pOp2);
1346     }
1347
1348     // expand this to (a == b) | (a > b)
1349     GenTree* dupOp1 = nullptr;
1350     GenTree* dupOp2 = nullptr;
1351
1352     if (((*pOp1)->gtFlags & GTF_SIDE_EFFECT) != 0)
1353     {
1354         dupOp1 = fgInsertCommaFormTemp(pOp1, typeHnd);
1355     }
1356     else
1357     {
1358         dupOp1 = gtCloneExpr(*pOp1);
1359     }
1360
1361     if (((*pOp2)->gtFlags & GTF_SIDE_EFFECT) != 0)
1362     {
1363         dupOp2 = fgInsertCommaFormTemp(pOp2, typeHnd);
1364     }
1365     else
1366     {
1367         dupOp2 = gtCloneExpr(*pOp2);
1368     }
1369
1370     assert(dupOp1 != nullptr && dupOp2 != nullptr);
1371     assert(*pOp1 != nullptr && *pOp2 != nullptr);
1372
1373     // (a==b)
1374     *pOp1 = gtNewSIMDNode(simdType, *pOp1, *pOp2, SIMDIntrinsicEqual, baseType, size);
1375
1376     // (a > b)
1377     *pOp2 = gtNewSIMDNode(simdType, dupOp1, dupOp2, SIMDIntrinsicGreaterThan, baseType, size);
1378
1379     return SIMDIntrinsicBitwiseOr;
1380 }
1381 #endif // _TARGET_XARCH_
1382
1383 // Transforms operands and returns the SIMD intrinsic to be applied on
1384 // transformed operands to obtain given relop result.
1385 //
1386 // Arguments:
1387 //    relOpIntrinsicId - Relational operator SIMD intrinsic
1388 //    typeHnd          - type handle of SIMD vector
1389 //    size             -  SIMD vector size
1390 //    inOutBaseType    - base type of SIMD vector
1391 //    pOp1             -  in-out parameter; first operand
1392 //    pOp2             -  in-out parameter; second operand
1393 //
1394 // Return Value:
1395 //    Modifies in-out params pOp1, pOp2, inOutBaseType and returns intrinsic ID to be applied to modified operands
1396 //
1397 SIMDIntrinsicID Compiler::impSIMDRelOp(SIMDIntrinsicID      relOpIntrinsicId,
1398                                        CORINFO_CLASS_HANDLE typeHnd,
1399                                        unsigned             size,
1400                                        var_types*           inOutBaseType,
1401                                        GenTree**            pOp1,
1402                                        GenTree**            pOp2)
1403 {
1404     var_types simdType = (*pOp1)->TypeGet();
1405     assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1406
1407     assert(isRelOpSIMDIntrinsic(relOpIntrinsicId));
1408
1409     SIMDIntrinsicID intrinsicID = relOpIntrinsicId;
1410 #ifdef _TARGET_XARCH_
1411     var_types baseType = *inOutBaseType;
1412
1413     if (varTypeIsFloating(baseType))
1414     {
1415         // SSE2/AVX doesn't support > and >= on vector float/double.
1416         // Therefore, we need to use < and <= with swapped operands
1417         if (relOpIntrinsicId == SIMDIntrinsicGreaterThan || relOpIntrinsicId == SIMDIntrinsicGreaterThanOrEqual)
1418         {
1419             GenTree* tmp = *pOp1;
1420             *pOp1        = *pOp2;
1421             *pOp2        = tmp;
1422
1423             intrinsicID =
1424                 (relOpIntrinsicId == SIMDIntrinsicGreaterThan) ? SIMDIntrinsicLessThan : SIMDIntrinsicLessThanOrEqual;
1425         }
1426     }
1427     else if (varTypeIsIntegral(baseType))
1428     {
1429         // SSE/AVX doesn't support < and <= on integer base type vectors.
1430         // Therefore, we need to use > and >= with swapped operands.
1431         if (intrinsicID == SIMDIntrinsicLessThan || intrinsicID == SIMDIntrinsicLessThanOrEqual)
1432         {
1433             GenTree* tmp = *pOp1;
1434             *pOp1        = *pOp2;
1435             *pOp2        = tmp;
1436
1437             intrinsicID = (relOpIntrinsicId == SIMDIntrinsicLessThan) ? SIMDIntrinsicGreaterThan
1438                                                                       : SIMDIntrinsicGreaterThanOrEqual;
1439         }
1440
1441         if ((getSIMDSupportLevel() == SIMD_SSE2_Supported) && baseType == TYP_LONG)
1442         {
1443             // There is no direct SSE2 support for comparing TYP_LONG vectors.
1444             // These have to be implemented interms of TYP_INT vector comparison operations.
1445             if (intrinsicID == SIMDIntrinsicEqual)
1446             {
1447                 intrinsicID = impSIMDLongRelOpEqual(typeHnd, size, pOp1, pOp2);
1448             }
1449             else if (intrinsicID == SIMDIntrinsicGreaterThan)
1450             {
1451                 intrinsicID = impSIMDLongRelOpGreaterThan(typeHnd, size, pOp1, pOp2);
1452             }
1453             else if (intrinsicID == SIMDIntrinsicGreaterThanOrEqual)
1454             {
1455                 intrinsicID = impSIMDLongRelOpGreaterThanOrEqual(typeHnd, size, pOp1, pOp2);
1456             }
1457             else
1458             {
1459                 unreached();
1460             }
1461         }
1462         // SSE2 and AVX direct support for signed comparison of int32, int16 and int8 types
1463         else if (!varTypeIsUnsigned(baseType))
1464         {
1465             if (intrinsicID == SIMDIntrinsicGreaterThanOrEqual)
1466             {
1467                 intrinsicID = impSIMDIntegralRelOpGreaterThanOrEqual(typeHnd, size, baseType, pOp1, pOp2);
1468             }
1469         }
1470         else // unsigned
1471         {
1472             // Vector<byte>, Vector<ushort>, Vector<uint> and Vector<ulong>:
1473             // SSE2 supports > for signed comparison. Therefore, to use it for
1474             // comparing unsigned numbers, we subtract a constant from both the
1475             // operands such that the result fits within the corresponding signed
1476             // type.  The resulting signed numbers are compared using SSE2 signed
1477             // comparison.
1478             //
1479             // Vector<byte>: constant to be subtracted is 2^7
1480             // Vector<ushort> constant to be subtracted is 2^15
1481             // Vector<uint> constant to be subtracted is 2^31
1482             // Vector<ulong> constant to be subtracted is 2^63
1483             //
1484             // We need to treat op1 and op2 as signed for comparison purpose after
1485             // the transformation.
1486             __int64 constVal = 0;
1487             switch (baseType)
1488             {
1489                 case TYP_UBYTE:
1490                     constVal       = 0x80808080;
1491                     *inOutBaseType = TYP_BYTE;
1492                     break;
1493                 case TYP_USHORT:
1494                     constVal       = 0x80008000;
1495                     *inOutBaseType = TYP_SHORT;
1496                     break;
1497                 case TYP_UINT:
1498                     constVal       = 0x80000000;
1499                     *inOutBaseType = TYP_INT;
1500                     break;
1501                 case TYP_ULONG:
1502                     constVal       = 0x8000000000000000LL;
1503                     *inOutBaseType = TYP_LONG;
1504                     break;
1505                 default:
1506                     unreached();
1507                     break;
1508             }
1509             assert(constVal != 0);
1510
1511             // This transformation is not required for equality.
1512             if (intrinsicID != SIMDIntrinsicEqual)
1513             {
1514                 // For constructing const vector use either long or int base type.
1515                 var_types tempBaseType;
1516                 GenTree*  initVal;
1517                 if (baseType == TYP_ULONG)
1518                 {
1519                     tempBaseType = TYP_LONG;
1520                     initVal      = gtNewLconNode(constVal);
1521                 }
1522                 else
1523                 {
1524                     tempBaseType = TYP_INT;
1525                     initVal      = gtNewIconNode((ssize_t)constVal);
1526                 }
1527                 initVal->gtType      = tempBaseType;
1528                 GenTree* constVector = gtNewSIMDNode(simdType, initVal, nullptr, SIMDIntrinsicInit, tempBaseType, size);
1529
1530                 // Assign constVector to a temp, since we intend to use it more than once
1531                 // TODO-CQ: We have quite a few such constant vectors constructed during
1532                 // the importation of SIMD intrinsics.  Make sure that we have a single
1533                 // temp per distinct constant per method.
1534                 GenTree* tmp = fgInsertCommaFormTemp(&constVector, typeHnd);
1535
1536                 // op1 = op1 - constVector
1537                 // op2 = op2 - constVector
1538                 *pOp1 = gtNewSIMDNode(simdType, *pOp1, constVector, SIMDIntrinsicSub, baseType, size);
1539                 *pOp2 = gtNewSIMDNode(simdType, *pOp2, tmp, SIMDIntrinsicSub, baseType, size);
1540             }
1541
1542             return impSIMDRelOp(intrinsicID, typeHnd, size, inOutBaseType, pOp1, pOp2);
1543         }
1544     }
1545 #elif defined(_TARGET_ARM64_)
1546     // TODO-ARM64-CQ handle comparisons against zero
1547
1548     // _TARGET_ARM64_ doesn't support < and <= on register register comparisons
1549     // Therefore, we need to use > and >= with swapped operands.
1550     if (intrinsicID == SIMDIntrinsicLessThan || intrinsicID == SIMDIntrinsicLessThanOrEqual)
1551     {
1552         GenTree* tmp = *pOp1;
1553         *pOp1        = *pOp2;
1554         *pOp2        = tmp;
1555
1556         intrinsicID =
1557             (intrinsicID == SIMDIntrinsicLessThan) ? SIMDIntrinsicGreaterThan : SIMDIntrinsicGreaterThanOrEqual;
1558     }
1559 #else  // !_TARGET_XARCH_
1560     assert(!"impSIMDRelOp() unimplemented on target arch");
1561     unreached();
1562 #endif // !_TARGET_XARCH_
1563
1564     return intrinsicID;
1565 }
1566
1567 //-------------------------------------------------------------------------
1568 // impSIMDAbs: creates GT_SIMD node to compute Abs value of a given vector.
1569 //
1570 // Arguments:
1571 //    typeHnd     -  type handle of SIMD vector
1572 //    baseType    -  base type of vector
1573 //    size        -  vector size in bytes
1574 //    op1         -  operand of Abs intrinsic
1575 //
1576 GenTree* Compiler::impSIMDAbs(CORINFO_CLASS_HANDLE typeHnd, var_types baseType, unsigned size, GenTree* op1)
1577 {
1578     assert(varTypeIsSIMD(op1));
1579
1580     var_types simdType = op1->TypeGet();
1581     GenTree*  retVal   = nullptr;
1582
1583 #ifdef _TARGET_XARCH_
1584     // When there is no direct support, Abs(v) could be computed
1585     // on integer vectors as follows:
1586     //     BitVector = v < vector.Zero
1587     //     result = ConditionalSelect(BitVector, vector.Zero - v, v)
1588
1589     bool useConditionalSelect = false;
1590     if (getSIMDSupportLevel() == SIMD_SSE2_Supported)
1591     {
1592         // SSE2 doesn't support abs on signed integer type vectors.
1593         if (baseType == TYP_LONG || baseType == TYP_INT || baseType == TYP_SHORT || baseType == TYP_BYTE)
1594         {
1595             useConditionalSelect = true;
1596         }
1597     }
1598     else
1599     {
1600         assert(getSIMDSupportLevel() >= SIMD_SSE4_Supported);
1601         if (baseType == TYP_LONG)
1602         {
1603             // SSE4/AVX2 don't support abs on long type vector.
1604             useConditionalSelect = true;
1605         }
1606     }
1607
1608     if (useConditionalSelect)
1609     {
1610         // This works only on integer vectors not on float/double vectors.
1611         assert(varTypeIsIntegral(baseType));
1612
1613         GenTree* op1Assign;
1614         unsigned op1LclNum;
1615
1616         if (op1->OperGet() == GT_LCL_VAR)
1617         {
1618             op1LclNum = op1->gtLclVarCommon.gtLclNum;
1619             op1Assign = nullptr;
1620         }
1621         else
1622         {
1623             op1LclNum = lvaGrabTemp(true DEBUGARG("SIMD Abs op1"));
1624             lvaSetStruct(op1LclNum, typeHnd, false);
1625             op1Assign = gtNewTempAssign(op1LclNum, op1);
1626             op1       = gtNewLclvNode(op1LclNum, op1->TypeGet());
1627         }
1628
1629         // Assign Vector.Zero to a temp since it is needed more than once
1630         GenTree* vecZero       = gtNewSIMDVectorZero(simdType, baseType, size);
1631         unsigned vecZeroLclNum = lvaGrabTemp(true DEBUGARG("SIMD Abs VecZero"));
1632         lvaSetStruct(vecZeroLclNum, typeHnd, false);
1633         GenTree* vecZeroAssign = gtNewTempAssign(vecZeroLclNum, vecZero);
1634
1635         // Construct BitVector = v < vector.Zero
1636         GenTree*        bitVecOp1     = op1;
1637         GenTree*        bitVecOp2     = gtNewLclvNode(vecZeroLclNum, vecZero->TypeGet());
1638         var_types       relOpBaseType = baseType;
1639         SIMDIntrinsicID relOpIntrinsic =
1640             impSIMDRelOp(SIMDIntrinsicLessThan, typeHnd, size, &relOpBaseType, &bitVecOp1, &bitVecOp2);
1641         GenTree* bitVec       = gtNewSIMDNode(simdType, bitVecOp1, bitVecOp2, relOpIntrinsic, relOpBaseType, size);
1642         unsigned bitVecLclNum = lvaGrabTemp(true DEBUGARG("SIMD Abs bitVec"));
1643         lvaSetStruct(bitVecLclNum, typeHnd, false);
1644         GenTree* bitVecAssign = gtNewTempAssign(bitVecLclNum, bitVec);
1645         bitVec                = gtNewLclvNode(bitVecLclNum, bitVec->TypeGet());
1646
1647         // Construct condSelectOp1 = vector.Zero - v
1648         GenTree* subOp1 = gtNewLclvNode(vecZeroLclNum, vecZero->TypeGet());
1649         GenTree* subOp2 = gtNewLclvNode(op1LclNum, op1->TypeGet());
1650         GenTree* negVec = gtNewSIMDNode(simdType, subOp1, subOp2, SIMDIntrinsicSub, baseType, size);
1651
1652         // Construct ConditionalSelect(bitVec, vector.Zero - v, v)
1653         GenTree* vec = gtNewLclvNode(op1LclNum, op1->TypeGet());
1654         retVal       = impSIMDSelect(typeHnd, baseType, size, bitVec, negVec, vec);
1655
1656         // Prepend bitVec assignment to retVal.
1657         // retVal = (tmp2 = v < tmp1), CondSelect(tmp2, tmp1 - v, v)
1658         retVal = gtNewOperNode(GT_COMMA, simdType, bitVecAssign, retVal);
1659
1660         // Prepend vecZero assignment to retVal.
1661         // retVal =  (tmp1 = vector.Zero), (tmp2 = v < tmp1), CondSelect(tmp2, tmp1 - v, v)
1662         retVal = gtNewOperNode(GT_COMMA, simdType, vecZeroAssign, retVal);
1663
1664         // If op1 was assigned to a temp, prepend that to retVal.
1665         if (op1Assign != nullptr)
1666         {
1667             // retVal = (v=op1), (tmp1 = vector.Zero), (tmp2 = v < tmp1), CondSelect(tmp2, tmp1 - v, v)
1668             retVal = gtNewOperNode(GT_COMMA, simdType, op1Assign, retVal);
1669         }
1670     }
1671     else if (varTypeIsFloating(baseType))
1672     {
1673         // Abs(vf) = vf & new SIMDVector<float>(0x7fffffff);
1674         // Abs(vd) = vf & new SIMDVector<double>(0x7fffffffffffffff);
1675         GenTree* bitMask = nullptr;
1676         if (baseType == TYP_FLOAT)
1677         {
1678             float f;
1679             static_assert_no_msg(sizeof(float) == sizeof(int));
1680             *((int*)&f) = 0x7fffffff;
1681             bitMask     = gtNewDconNode(f);
1682         }
1683         else if (baseType == TYP_DOUBLE)
1684         {
1685             double d;
1686             static_assert_no_msg(sizeof(double) == sizeof(__int64));
1687             *((__int64*)&d) = 0x7fffffffffffffffLL;
1688             bitMask         = gtNewDconNode(d);
1689         }
1690
1691         assert(bitMask != nullptr);
1692         bitMask->gtType        = baseType;
1693         GenTree* bitMaskVector = gtNewSIMDNode(simdType, bitMask, SIMDIntrinsicInit, baseType, size);
1694         retVal                 = gtNewSIMDNode(simdType, op1, bitMaskVector, SIMDIntrinsicBitwiseAnd, baseType, size);
1695     }
1696     else if (baseType == TYP_USHORT || baseType == TYP_UBYTE || baseType == TYP_UINT || baseType == TYP_ULONG)
1697     {
1698         // Abs is a no-op on unsigned integer type vectors
1699         retVal = op1;
1700     }
1701     else
1702     {
1703         assert(getSIMDSupportLevel() >= SIMD_SSE4_Supported);
1704         assert(baseType != TYP_LONG);
1705
1706         retVal = gtNewSIMDNode(simdType, op1, SIMDIntrinsicAbs, baseType, size);
1707     }
1708 #elif defined(_TARGET_ARM64_)
1709     if (varTypeIsUnsigned(baseType))
1710     {
1711         // Abs is a no-op on unsigned integer type vectors
1712         retVal = op1;
1713     }
1714     else
1715     {
1716         retVal = gtNewSIMDNode(simdType, op1, SIMDIntrinsicAbs, baseType, size);
1717     }
1718 #else  // !defined(_TARGET_XARCH)_ && !defined(_TARGET_ARM64_)
1719     assert(!"Abs intrinsic on non-xarch target not implemented");
1720 #endif // !_TARGET_XARCH_
1721
1722     return retVal;
1723 }
1724
1725 // Creates a GT_SIMD tree for Select operation
1726 //
1727 // Arguments:
1728 //    typeHnd          -  type handle of SIMD vector
1729 //    baseType         -  base type of SIMD vector
1730 //    size             -  SIMD vector size
1731 //    op1              -  first operand = Condition vector vc
1732 //    op2              -  second operand = va
1733 //    op3              -  third operand = vb
1734 //
1735 // Return Value:
1736 //    Returns GT_SIMD tree that computes Select(vc, va, vb)
1737 //
1738 GenTree* Compiler::impSIMDSelect(
1739     CORINFO_CLASS_HANDLE typeHnd, var_types baseType, unsigned size, GenTree* op1, GenTree* op2, GenTree* op3)
1740 {
1741     assert(varTypeIsSIMD(op1));
1742     var_types simdType = op1->TypeGet();
1743     assert(op2->TypeGet() == simdType);
1744     assert(op3->TypeGet() == simdType);
1745
1746     // TODO-ARM64-CQ Support generating select instruction for SIMD
1747
1748     // Select(BitVector vc, va, vb) = (va & vc) | (vb & !vc)
1749     // Select(op1, op2, op3)        = (op2 & op1) | (op3 & !op1)
1750     //                              = SIMDIntrinsicBitwiseOr(SIMDIntrinsicBitwiseAnd(op2, op1),
1751     //                                                       SIMDIntrinsicBitwiseAndNot(op3, op1))
1752     //
1753     // If Op1 has side effect, create an assignment to a temp
1754     GenTree* tmp = op1;
1755     GenTree* asg = nullptr;
1756     if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0)
1757     {
1758         unsigned lclNum = lvaGrabTemp(true DEBUGARG("SIMD Select"));
1759         lvaSetStruct(lclNum, typeHnd, false);
1760         tmp = gtNewLclvNode(lclNum, op1->TypeGet());
1761         asg = gtNewTempAssign(lclNum, op1);
1762     }
1763
1764     GenTree* andExpr = gtNewSIMDNode(simdType, op2, tmp, SIMDIntrinsicBitwiseAnd, baseType, size);
1765     GenTree* dupOp1  = gtCloneExpr(tmp);
1766     assert(dupOp1 != nullptr);
1767 #ifdef _TARGET_ARM64_
1768     // ARM64 implements SIMDIntrinsicBitwiseAndNot as Left & ~Right
1769     GenTree* andNotExpr = gtNewSIMDNode(simdType, op3, dupOp1, SIMDIntrinsicBitwiseAndNot, baseType, size);
1770 #else
1771     // XARCH implements SIMDIntrinsicBitwiseAndNot as ~Left & Right
1772     GenTree* andNotExpr = gtNewSIMDNode(simdType, dupOp1, op3, SIMDIntrinsicBitwiseAndNot, baseType, size);
1773 #endif
1774     GenTree* simdTree = gtNewSIMDNode(simdType, andExpr, andNotExpr, SIMDIntrinsicBitwiseOr, baseType, size);
1775
1776     // If asg not null, create a GT_COMMA tree.
1777     if (asg != nullptr)
1778     {
1779         simdTree = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), asg, simdTree);
1780     }
1781
1782     return simdTree;
1783 }
1784
1785 // Creates a GT_SIMD tree for Min/Max operation
1786 //
1787 // Arguments:
1788 //    IntrinsicId      -  SIMD intrinsic Id, either Min or Max
1789 //    typeHnd          -  type handle of SIMD vector
1790 //    baseType         -  base type of SIMD vector
1791 //    size             -  SIMD vector size
1792 //    op1              -  first operand = va
1793 //    op2              -  second operand = vb
1794 //
1795 // Return Value:
1796 //    Returns GT_SIMD tree that computes Max(va, vb)
1797 //
1798 GenTree* Compiler::impSIMDMinMax(SIMDIntrinsicID      intrinsicId,
1799                                  CORINFO_CLASS_HANDLE typeHnd,
1800                                  var_types            baseType,
1801                                  unsigned             size,
1802                                  GenTree*             op1,
1803                                  GenTree*             op2)
1804 {
1805     assert(intrinsicId == SIMDIntrinsicMin || intrinsicId == SIMDIntrinsicMax);
1806     assert(varTypeIsSIMD(op1));
1807     var_types simdType = op1->TypeGet();
1808     assert(op2->TypeGet() == simdType);
1809
1810 #if defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
1811     GenTree* simdTree = nullptr;
1812
1813 #ifdef _TARGET_XARCH_
1814     // SSE2 has direct support for float/double/signed word/unsigned byte.
1815     // SSE4.1 has direct support for int32/uint32/signed byte/unsigned word.
1816     // For other integer types we compute min/max as follows
1817     //
1818     // int32/uint32 (SSE2)
1819     // int64/uint64 (SSE2&SSE4):
1820     //       compResult        = (op1 < op2) in case of Min
1821     //                           (op1 > op2) in case of Max
1822     //       Min/Max(op1, op2) = Select(compResult, op1, op2)
1823     //
1824     // unsigned word (SSE2):
1825     //        op1 = op1 - 2^15  ; to make it fit within a signed word
1826     //        op2 = op2 - 2^15  ; to make it fit within a signed word
1827     //        result = SSE2 signed word Min/Max(op1, op2)
1828     //        result = result + 2^15  ; readjust it back
1829     //
1830     // signed byte (SSE2):
1831     //        op1 = op1 + 2^7  ; to make it unsigned
1832     //        op1 = op1 + 2^7  ; to make it unsigned
1833     //        result = SSE2 unsigned byte Min/Max(op1, op2)
1834     //        result = result - 2^15 ; readjust it back
1835
1836     if (varTypeIsFloating(baseType) || baseType == TYP_SHORT || baseType == TYP_UBYTE ||
1837         (getSIMDSupportLevel() >= SIMD_SSE4_Supported &&
1838          (baseType == TYP_BYTE || baseType == TYP_INT || baseType == TYP_UINT || baseType == TYP_USHORT)))
1839     {
1840         // SSE2 or SSE4.1 has direct support
1841         simdTree = gtNewSIMDNode(simdType, op1, op2, intrinsicId, baseType, size);
1842     }
1843     else if (baseType == TYP_USHORT || baseType == TYP_BYTE)
1844     {
1845         assert(getSIMDSupportLevel() == SIMD_SSE2_Supported);
1846         int             constVal;
1847         SIMDIntrinsicID operIntrinsic;
1848         SIMDIntrinsicID adjustIntrinsic;
1849         var_types       minMaxOperBaseType;
1850         if (baseType == TYP_USHORT)
1851         {
1852             constVal           = 0x80008000;
1853             operIntrinsic      = SIMDIntrinsicSub;
1854             adjustIntrinsic    = SIMDIntrinsicAdd;
1855             minMaxOperBaseType = TYP_SHORT;
1856         }
1857         else
1858         {
1859             assert(baseType == TYP_BYTE);
1860             constVal           = 0x80808080;
1861             operIntrinsic      = SIMDIntrinsicAdd;
1862             adjustIntrinsic    = SIMDIntrinsicSub;
1863             minMaxOperBaseType = TYP_UBYTE;
1864         }
1865
1866         GenTree* initVal     = gtNewIconNode(constVal);
1867         GenTree* constVector = gtNewSIMDNode(simdType, initVal, nullptr, SIMDIntrinsicInit, TYP_INT, size);
1868
1869         // Assign constVector to a temp, since we intend to use it more than once
1870         // TODO-CQ: We have quite a few such constant vectors constructed during
1871         // the importation of SIMD intrinsics.  Make sure that we have a single
1872         // temp per distinct constant per method.
1873         GenTree* tmp = fgInsertCommaFormTemp(&constVector, typeHnd);
1874
1875         // op1 = op1 - constVector
1876         // op2 = op2 - constVector
1877         op1 = gtNewSIMDNode(simdType, op1, constVector, operIntrinsic, baseType, size);
1878         op2 = gtNewSIMDNode(simdType, op2, tmp, operIntrinsic, baseType, size);
1879
1880         // compute min/max of op1 and op2 considering them as if minMaxOperBaseType
1881         simdTree = gtNewSIMDNode(simdType, op1, op2, intrinsicId, minMaxOperBaseType, size);
1882
1883         // re-adjust the value by adding or subtracting constVector
1884         tmp      = gtNewLclvNode(tmp->AsLclVarCommon()->GetLclNum(), tmp->TypeGet());
1885         simdTree = gtNewSIMDNode(simdType, simdTree, tmp, adjustIntrinsic, baseType, size);
1886     }
1887 #elif defined(_TARGET_ARM64_)
1888     // Arm64 has direct support for all types except int64/uint64
1889     // For which we compute min/max as follows
1890     //
1891     // int64/uint64
1892     //       compResult        = (op1 < op2) in case of Min
1893     //                           (op1 > op2) in case of Max
1894     //       Min/Max(op1, op2) = Select(compResult, op1, op2)
1895     if (baseType != TYP_ULONG && baseType != TYP_LONG)
1896     {
1897         simdTree = gtNewSIMDNode(simdType, op1, op2, intrinsicId, baseType, size);
1898     }
1899 #endif
1900     else
1901     {
1902         GenTree* dupOp1    = nullptr;
1903         GenTree* dupOp2    = nullptr;
1904         GenTree* op1Assign = nullptr;
1905         GenTree* op2Assign = nullptr;
1906         unsigned op1LclNum;
1907         unsigned op2LclNum;
1908
1909         if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0)
1910         {
1911             op1LclNum = lvaGrabTemp(true DEBUGARG("SIMD Min/Max"));
1912             dupOp1    = gtNewLclvNode(op1LclNum, op1->TypeGet());
1913             lvaSetStruct(op1LclNum, typeHnd, false);
1914             op1Assign = gtNewTempAssign(op1LclNum, op1);
1915             op1       = gtNewLclvNode(op1LclNum, op1->TypeGet());
1916         }
1917         else
1918         {
1919             dupOp1 = gtCloneExpr(op1);
1920         }
1921
1922         if ((op2->gtFlags & GTF_SIDE_EFFECT) != 0)
1923         {
1924             op2LclNum = lvaGrabTemp(true DEBUGARG("SIMD Min/Max"));
1925             dupOp2    = gtNewLclvNode(op2LclNum, op2->TypeGet());
1926             lvaSetStruct(op2LclNum, typeHnd, false);
1927             op2Assign = gtNewTempAssign(op2LclNum, op2);
1928             op2       = gtNewLclvNode(op2LclNum, op2->TypeGet());
1929         }
1930         else
1931         {
1932             dupOp2 = gtCloneExpr(op2);
1933         }
1934
1935         SIMDIntrinsicID relOpIntrinsic =
1936             (intrinsicId == SIMDIntrinsicMin) ? SIMDIntrinsicLessThan : SIMDIntrinsicGreaterThan;
1937         var_types relOpBaseType = baseType;
1938
1939         // compResult = op1 relOp op2
1940         // simdTree = Select(compResult, op1, op2);
1941         assert(dupOp1 != nullptr);
1942         assert(dupOp2 != nullptr);
1943         relOpIntrinsic            = impSIMDRelOp(relOpIntrinsic, typeHnd, size, &relOpBaseType, &dupOp1, &dupOp2);
1944         GenTree* compResult       = gtNewSIMDNode(simdType, dupOp1, dupOp2, relOpIntrinsic, relOpBaseType, size);
1945         unsigned compResultLclNum = lvaGrabTemp(true DEBUGARG("SIMD Min/Max"));
1946         lvaSetStruct(compResultLclNum, typeHnd, false);
1947         GenTree* compResultAssign = gtNewTempAssign(compResultLclNum, compResult);
1948         compResult                = gtNewLclvNode(compResultLclNum, compResult->TypeGet());
1949         simdTree                  = impSIMDSelect(typeHnd, baseType, size, compResult, op1, op2);
1950         simdTree                  = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), compResultAssign, simdTree);
1951
1952         // Now create comma trees if we have created assignments of op1/op2 to temps
1953         if (op2Assign != nullptr)
1954         {
1955             simdTree = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), op2Assign, simdTree);
1956         }
1957
1958         if (op1Assign != nullptr)
1959         {
1960             simdTree = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), op1Assign, simdTree);
1961         }
1962     }
1963
1964     assert(simdTree != nullptr);
1965     return simdTree;
1966 #else  // !(defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_))
1967     assert(!"impSIMDMinMax() unimplemented on target arch");
1968     unreached();
1969 #endif // !(defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_))
1970 }
1971
1972 //------------------------------------------------------------------------
1973 // getOp1ForConstructor: Get the op1 for a constructor call.
1974 //
1975 // Arguments:
1976 //    opcode     - the opcode being handled (needed to identify the CEE_NEWOBJ case)
1977 //    newobjThis - For CEE_NEWOBJ, this is the temp grabbed for the allocated uninitalized object.
1978 //    clsHnd    - The handle of the class of the method.
1979 //
1980 // Return Value:
1981 //    The tree node representing the object to be initialized with the constructor.
1982 //
1983 // Notes:
1984 //    This method handles the differences between the CEE_NEWOBJ and constructor cases.
1985 //
1986 GenTree* Compiler::getOp1ForConstructor(OPCODE opcode, GenTree* newobjThis, CORINFO_CLASS_HANDLE clsHnd)
1987 {
1988     GenTree* op1;
1989     if (opcode == CEE_NEWOBJ)
1990     {
1991         op1 = newobjThis;
1992         assert(newobjThis->gtOper == GT_ADDR && newobjThis->gtOp.gtOp1->gtOper == GT_LCL_VAR);
1993
1994         // push newobj result on type stack
1995         unsigned tmp = op1->gtOp.gtOp1->gtLclVarCommon.gtLclNum;
1996         impPushOnStack(gtNewLclvNode(tmp, lvaGetRealType(tmp)), verMakeTypeInfo(clsHnd).NormaliseForStack());
1997     }
1998     else
1999     {
2000         op1 = impSIMDPopStack(TYP_BYREF);
2001     }
2002     assert(op1->TypeGet() == TYP_BYREF);
2003     return op1;
2004 }
2005
2006 //-------------------------------------------------------------------
2007 // Set the flag that indicates that the lclVar referenced by this tree
2008 // is used in a SIMD intrinsic.
2009 // Arguments:
2010 //      tree - GenTree*
2011
2012 void Compiler::setLclRelatedToSIMDIntrinsic(GenTree* tree)
2013 {
2014     assert(tree->OperIsLocal());
2015     unsigned   lclNum                = tree->AsLclVarCommon()->GetLclNum();
2016     LclVarDsc* lclVarDsc             = &lvaTable[lclNum];
2017     lclVarDsc->lvUsedInSIMDIntrinsic = true;
2018 }
2019
2020 //-------------------------------------------------------------
2021 // Check if two field nodes reference at the same memory location.
2022 // Notice that this check is just based on pattern matching.
2023 // Arguments:
2024 //      op1 - GenTree*.
2025 //      op2 - GenTree*.
2026 // Return Value:
2027 //    If op1's parents node and op2's parents node are at the same location, return true. Otherwise, return false
2028
2029 bool areFieldsParentsLocatedSame(GenTree* op1, GenTree* op2)
2030 {
2031     assert(op1->OperGet() == GT_FIELD);
2032     assert(op2->OperGet() == GT_FIELD);
2033
2034     GenTree* op1ObjRef = op1->gtField.gtFldObj;
2035     GenTree* op2ObjRef = op2->gtField.gtFldObj;
2036     while (op1ObjRef != nullptr && op2ObjRef != nullptr)
2037     {
2038
2039         if (op1ObjRef->OperGet() != op2ObjRef->OperGet())
2040         {
2041             break;
2042         }
2043         else if (op1ObjRef->OperGet() == GT_ADDR)
2044         {
2045             op1ObjRef = op1ObjRef->gtOp.gtOp1;
2046             op2ObjRef = op2ObjRef->gtOp.gtOp1;
2047         }
2048
2049         if (op1ObjRef->OperIsLocal() && op2ObjRef->OperIsLocal() &&
2050             op1ObjRef->AsLclVarCommon()->GetLclNum() == op2ObjRef->AsLclVarCommon()->GetLclNum())
2051         {
2052             return true;
2053         }
2054         else if (op1ObjRef->OperGet() == GT_FIELD && op2ObjRef->OperGet() == GT_FIELD &&
2055                  op1ObjRef->gtField.gtFldHnd == op2ObjRef->gtField.gtFldHnd)
2056         {
2057             op1ObjRef = op1ObjRef->gtField.gtFldObj;
2058             op2ObjRef = op2ObjRef->gtField.gtFldObj;
2059             continue;
2060         }
2061         else
2062         {
2063             break;
2064         }
2065     }
2066
2067     return false;
2068 }
2069
2070 //----------------------------------------------------------------------
2071 // Check whether two field are contiguous
2072 // Arguments:
2073 //      first - GenTree*. The Type of the node should be TYP_FLOAT
2074 //      second - GenTree*. The Type of the node should be TYP_FLOAT
2075 // Return Value:
2076 //      if the first field is located before second field, and they are located contiguously,
2077 //      then return true. Otherwise, return false.
2078
2079 bool Compiler::areFieldsContiguous(GenTree* first, GenTree* second)
2080 {
2081     assert(first->OperGet() == GT_FIELD);
2082     assert(second->OperGet() == GT_FIELD);
2083     assert(first->gtType == TYP_FLOAT);
2084     assert(second->gtType == TYP_FLOAT);
2085
2086     var_types firstFieldType  = first->gtType;
2087     var_types secondFieldType = second->gtType;
2088
2089     unsigned firstFieldEndOffset = first->gtField.gtFldOffset + genTypeSize(firstFieldType);
2090     unsigned secondFieldOffset   = second->gtField.gtFldOffset;
2091     if (firstFieldEndOffset == secondFieldOffset && firstFieldType == secondFieldType &&
2092         areFieldsParentsLocatedSame(first, second))
2093     {
2094         return true;
2095     }
2096
2097     return false;
2098 }
2099
2100 //-------------------------------------------------------------------------------
2101 // Check whether two array element nodes are located contiguously or not.
2102 // Arguments:
2103 //      op1 - GenTree*.
2104 //      op2 - GenTree*.
2105 // Return Value:
2106 //      if the array element op1 is located before array element op2, and they are contiguous,
2107 //      then return true. Otherwise, return false.
2108 // TODO-CQ:
2109 //      Right this can only check array element with const number as index. In future,
2110 //      we should consider to allow this function to check the index using expression.
2111
2112 bool Compiler::areArrayElementsContiguous(GenTree* op1, GenTree* op2)
2113 {
2114     noway_assert(op1->gtOper == GT_INDEX);
2115     noway_assert(op2->gtOper == GT_INDEX);
2116     GenTreeIndex* op1Index = op1->AsIndex();
2117     GenTreeIndex* op2Index = op2->AsIndex();
2118
2119     GenTree* op1ArrayRef = op1Index->Arr();
2120     GenTree* op2ArrayRef = op2Index->Arr();
2121     assert(op1ArrayRef->TypeGet() == TYP_REF);
2122     assert(op2ArrayRef->TypeGet() == TYP_REF);
2123
2124     GenTree* op1IndexNode = op1Index->Index();
2125     GenTree* op2IndexNode = op2Index->Index();
2126     if ((op1IndexNode->OperGet() == GT_CNS_INT && op2IndexNode->OperGet() == GT_CNS_INT) &&
2127         op1IndexNode->gtIntCon.gtIconVal + 1 == op2IndexNode->gtIntCon.gtIconVal)
2128     {
2129         if (op1ArrayRef->OperGet() == GT_FIELD && op2ArrayRef->OperGet() == GT_FIELD &&
2130             areFieldsParentsLocatedSame(op1ArrayRef, op2ArrayRef))
2131         {
2132             return true;
2133         }
2134         else if (op1ArrayRef->OperIsLocal() && op2ArrayRef->OperIsLocal() &&
2135                  op1ArrayRef->AsLclVarCommon()->GetLclNum() == op2ArrayRef->AsLclVarCommon()->GetLclNum())
2136         {
2137             return true;
2138         }
2139     }
2140     return false;
2141 }
2142
2143 //-------------------------------------------------------------------------------
2144 // Check whether two argument nodes are contiguous or not.
2145 // Arguments:
2146 //      op1 - GenTree*.
2147 //      op2 - GenTree*.
2148 // Return Value:
2149 //      if the argument node op1 is located before argument node op2, and they are located contiguously,
2150 //      then return true. Otherwise, return false.
2151 // TODO-CQ:
2152 //      Right now this can only check field and array. In future we should add more cases.
2153 //
2154
2155 bool Compiler::areArgumentsContiguous(GenTree* op1, GenTree* op2)
2156 {
2157     if (op1->OperGet() == GT_INDEX && op2->OperGet() == GT_INDEX)
2158     {
2159         return areArrayElementsContiguous(op1, op2);
2160     }
2161     else if (op1->OperGet() == GT_FIELD && op2->OperGet() == GT_FIELD)
2162     {
2163         return areFieldsContiguous(op1, op2);
2164     }
2165     return false;
2166 }
2167
2168 //--------------------------------------------------------------------------------------------------------
2169 // createAddressNodeForSIMDInit: Generate the address node(GT_LEA) if we want to intialize vector2, vector3 or vector4
2170 // from first argument's address.
2171 //
2172 // Arguments:
2173 //      tree - GenTree*. This the tree node which is used to get the address for indir.
2174 //      simdsize - unsigned. This the simd vector size.
2175 //      arrayElementsCount - unsigned. This is used for generating the boundary check for array.
2176 //
2177 // Return value:
2178 //      return the address node.
2179 //
2180 // TODO-CQ:
2181 //      1. Currently just support for GT_FIELD and GT_INDEX, because we can only verify the GT_INDEX node or GT_Field
2182 //         are located contiguously or not. In future we should support more cases.
2183 //      2. Though it happens to just work fine front-end phases are not aware of GT_LEA node.  Therefore, convert these
2184 //         to use GT_ADDR.
2185 GenTree* Compiler::createAddressNodeForSIMDInit(GenTree* tree, unsigned simdSize)
2186 {
2187     assert(tree->OperGet() == GT_FIELD || tree->OperGet() == GT_INDEX);
2188     GenTree*  byrefNode  = nullptr;
2189     GenTree*  startIndex = nullptr;
2190     unsigned  offset     = 0;
2191     var_types baseType   = tree->gtType;
2192
2193     if (tree->OperGet() == GT_FIELD)
2194     {
2195         GenTree* objRef = tree->gtField.gtFldObj;
2196         if (objRef != nullptr && objRef->gtOper == GT_ADDR)
2197         {
2198             GenTree* obj = objRef->gtOp.gtOp1;
2199
2200             // If the field is directly from a struct, then in this case,
2201             // we should set this struct's lvUsedInSIMDIntrinsic as true,
2202             // so that this sturct won't be promoted.
2203             // e.g. s.x x is a field, and s is a struct, then we should set the s's lvUsedInSIMDIntrinsic as true.
2204             // so that s won't be promoted.
2205             // Notice that if we have a case like s1.s2.x. s1 s2 are struct, and x is a field, then it is possible that
2206             // s1 can be promoted, so that s2 can be promoted. The reason for that is if we don't allow s1 to be
2207             // promoted, then this will affect the other optimizations which are depend on s1's struct promotion.
2208             // TODO-CQ:
2209             //  In future, we should optimize this case so that if there is a nested field like s1.s2.x and s1.s2.x's
2210             //  address is used for initializing the vector, then s1 can be promoted but s2 can't.
2211             if (varTypeIsSIMD(obj) && obj->OperIsLocal())
2212             {
2213                 setLclRelatedToSIMDIntrinsic(obj);
2214             }
2215         }
2216
2217         byrefNode = gtCloneExpr(tree->gtField.gtFldObj);
2218         assert(byrefNode != nullptr);
2219         offset = tree->gtField.gtFldOffset;
2220     }
2221     else if (tree->OperGet() == GT_INDEX)
2222     {
2223
2224         GenTree* index = tree->AsIndex()->Index();
2225         assert(index->OperGet() == GT_CNS_INT);
2226
2227         GenTree* checkIndexExpr = nullptr;
2228         unsigned indexVal       = (unsigned)(index->gtIntCon.gtIconVal);
2229         offset                  = indexVal * genTypeSize(tree->TypeGet());
2230         GenTree* arrayRef       = tree->AsIndex()->Arr();
2231
2232         // Generate the boundary check exception.
2233         // The length for boundary check should be the maximum index number which should be
2234         // (first argument's index number) + (how many array arguments we have) - 1
2235         // = indexVal + arrayElementsCount - 1
2236         unsigned arrayElementsCount  = simdSize / genTypeSize(baseType);
2237         checkIndexExpr               = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, indexVal + arrayElementsCount - 1);
2238         GenTreeArrLen*    arrLen     = gtNewArrLen(TYP_INT, arrayRef, (int)OFFSETOF__CORINFO_Array__length);
2239         GenTreeBoundsChk* arrBndsChk = new (this, GT_ARR_BOUNDS_CHECK)
2240             GenTreeBoundsChk(GT_ARR_BOUNDS_CHECK, TYP_VOID, checkIndexExpr, arrLen, SCK_RNGCHK_FAIL);
2241
2242         offset += OFFSETOF__CORINFO_Array__data;
2243         byrefNode = gtNewOperNode(GT_COMMA, arrayRef->TypeGet(), arrBndsChk, gtCloneExpr(arrayRef));
2244     }
2245     else
2246     {
2247         unreached();
2248     }
2249     GenTree* address =
2250         new (this, GT_LEA) GenTreeAddrMode(TYP_BYREF, byrefNode, startIndex, genTypeSize(tree->TypeGet()), offset);
2251     return address;
2252 }
2253
2254 //-------------------------------------------------------------------------------
2255 // impMarkContiguousSIMDFieldAssignments: Try to identify if there are contiguous
2256 // assignments from SIMD field to memory. If there are, then mark the related
2257 // lclvar so that it won't be promoted.
2258 //
2259 // Arguments:
2260 //      stmt - GenTree*. Input statement node.
2261
2262 void Compiler::impMarkContiguousSIMDFieldAssignments(GenTree* stmt)
2263 {
2264     if (!featureSIMD || opts.MinOpts())
2265     {
2266         return;
2267     }
2268     GenTree* expr = stmt->gtStmt.gtStmtExpr;
2269     if (expr->OperGet() == GT_ASG && expr->TypeGet() == TYP_FLOAT)
2270     {
2271         GenTree*  curDst            = expr->gtOp.gtOp1;
2272         GenTree*  curSrc            = expr->gtOp.gtOp2;
2273         unsigned  index             = 0;
2274         var_types baseType          = TYP_UNKNOWN;
2275         unsigned  simdSize          = 0;
2276         GenTree*  srcSimdStructNode = getSIMDStructFromField(curSrc, &baseType, &index, &simdSize, true);
2277         if (srcSimdStructNode == nullptr || baseType != TYP_FLOAT)
2278         {
2279             fgPreviousCandidateSIMDFieldAsgStmt = nullptr;
2280         }
2281         else if (index == 0 && isSIMDTypeLocal(srcSimdStructNode))
2282         {
2283             fgPreviousCandidateSIMDFieldAsgStmt = stmt;
2284         }
2285         else if (fgPreviousCandidateSIMDFieldAsgStmt != nullptr)
2286         {
2287             assert(index > 0);
2288             GenTree* prevAsgExpr = fgPreviousCandidateSIMDFieldAsgStmt->gtStmt.gtStmtExpr;
2289             GenTree* prevDst     = prevAsgExpr->gtOp.gtOp1;
2290             GenTree* prevSrc     = prevAsgExpr->gtOp.gtOp2;
2291             if (!areArgumentsContiguous(prevDst, curDst) || !areArgumentsContiguous(prevSrc, curSrc))
2292             {
2293                 fgPreviousCandidateSIMDFieldAsgStmt = nullptr;
2294             }
2295             else
2296             {
2297                 if (index == (simdSize / genTypeSize(baseType) - 1))
2298                 {
2299                     // Successfully found the pattern, mark the lclvar as UsedInSIMDIntrinsic
2300                     if (srcSimdStructNode->OperIsLocal())
2301                     {
2302                         setLclRelatedToSIMDIntrinsic(srcSimdStructNode);
2303                     }
2304
2305                     if (curDst->OperGet() == GT_FIELD)
2306                     {
2307                         GenTree* objRef = curDst->gtField.gtFldObj;
2308                         if (objRef != nullptr && objRef->gtOper == GT_ADDR)
2309                         {
2310                             GenTree* obj = objRef->gtOp.gtOp1;
2311                             if (varTypeIsStruct(obj) && obj->OperIsLocal())
2312                             {
2313                                 setLclRelatedToSIMDIntrinsic(obj);
2314                             }
2315                         }
2316                     }
2317                 }
2318                 else
2319                 {
2320                     fgPreviousCandidateSIMDFieldAsgStmt = stmt;
2321                 }
2322             }
2323         }
2324     }
2325     else
2326     {
2327         fgPreviousCandidateSIMDFieldAsgStmt = nullptr;
2328     }
2329 }
2330
2331 //------------------------------------------------------------------------
2332 // impSIMDIntrinsic: Check method to see if it is a SIMD method
2333 //
2334 // Arguments:
2335 //    opcode     - the opcode being handled (needed to identify the CEE_NEWOBJ case)
2336 //    newobjThis - For CEE_NEWOBJ, this is the temp grabbed for the allocated uninitalized object.
2337 //    clsHnd     - The handle of the class of the method.
2338 //    method     - The handle of the method.
2339 //    sig        - The call signature for the method.
2340 //    memberRef  - The memberRef token for the method reference.
2341 //
2342 // Return Value:
2343 //    If clsHnd is a known SIMD type, and 'method' is one of the methods that are
2344 //    implemented as an intrinsic in the JIT, then return the tree that implements
2345 //    it.
2346 //
2347 GenTree* Compiler::impSIMDIntrinsic(OPCODE                opcode,
2348                                     GenTree*              newobjThis,
2349                                     CORINFO_CLASS_HANDLE  clsHnd,
2350                                     CORINFO_METHOD_HANDLE methodHnd,
2351                                     CORINFO_SIG_INFO*     sig,
2352                                     unsigned              methodFlags,
2353                                     int                   memberRef)
2354 {
2355     assert(featureSIMD);
2356
2357     // Exit early if we are not in one of the SIMD types.
2358     if (!isSIMDClass(clsHnd))
2359     {
2360         return nullptr;
2361     }
2362
2363 #ifdef FEATURE_CORECLR
2364     // For coreclr, we also exit early if the method is not a JIT Intrinsic (which requires the [Intrinsic] attribute).
2365     if ((methodFlags & CORINFO_FLG_JIT_INTRINSIC) == 0)
2366     {
2367         return nullptr;
2368     }
2369 #endif // FEATURE_CORECLR
2370
2371     // Get base type and intrinsic Id
2372     var_types                baseType = TYP_UNKNOWN;
2373     unsigned                 size     = 0;
2374     unsigned                 argCount = 0;
2375     const SIMDIntrinsicInfo* intrinsicInfo =
2376         getSIMDIntrinsicInfo(&clsHnd, methodHnd, sig, (opcode == CEE_NEWOBJ), &argCount, &baseType, &size);
2377     if (intrinsicInfo == nullptr || intrinsicInfo->id == SIMDIntrinsicInvalid)
2378     {
2379         return nullptr;
2380     }
2381
2382     SIMDIntrinsicID simdIntrinsicID = intrinsicInfo->id;
2383     var_types       simdType;
2384     if (baseType != TYP_UNKNOWN)
2385     {
2386         simdType = getSIMDTypeForSize(size);
2387     }
2388     else
2389     {
2390         assert(simdIntrinsicID == SIMDIntrinsicHWAccel);
2391         simdType = TYP_UNKNOWN;
2392     }
2393     bool      instMethod = intrinsicInfo->isInstMethod;
2394     var_types callType   = JITtype2varType(sig->retType);
2395     if (callType == TYP_STRUCT)
2396     {
2397         // Note that here we are assuming that, if the call returns a struct, that it is the same size as the
2398         // struct on which the method is declared. This is currently true for all methods on Vector types,
2399         // but if this ever changes, we will need to determine the callType from the signature.
2400         assert(info.compCompHnd->getClassSize(sig->retTypeClass) == genTypeSize(simdType));
2401         callType = simdType;
2402     }
2403
2404     GenTree* simdTree   = nullptr;
2405     GenTree* op1        = nullptr;
2406     GenTree* op2        = nullptr;
2407     GenTree* op3        = nullptr;
2408     GenTree* retVal     = nullptr;
2409     GenTree* copyBlkDst = nullptr;
2410     bool     doCopyBlk  = false;
2411
2412     switch (simdIntrinsicID)
2413     {
2414         case SIMDIntrinsicGetCount:
2415         {
2416             int            length       = getSIMDVectorLength(clsHnd);
2417             GenTreeIntCon* intConstTree = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, length);
2418             retVal                      = intConstTree;
2419
2420             intConstTree->gtFlags |= GTF_ICON_SIMD_COUNT;
2421         }
2422         break;
2423
2424         case SIMDIntrinsicGetZero:
2425             retVal = gtNewSIMDVectorZero(simdType, baseType, size);
2426             break;
2427
2428         case SIMDIntrinsicGetOne:
2429             retVal = gtNewSIMDVectorOne(simdType, baseType, size);
2430             break;
2431
2432         case SIMDIntrinsicGetAllOnes:
2433         {
2434             // Equivalent to (Vector<T>) new Vector<int>(0xffffffff);
2435             GenTree* initVal = gtNewIconNode(0xffffffff, TYP_INT);
2436             simdTree         = gtNewSIMDNode(simdType, initVal, nullptr, SIMDIntrinsicInit, TYP_INT, size);
2437             if (baseType != TYP_INT)
2438             {
2439                 // cast it to required baseType if different from TYP_INT
2440                 simdTree = gtNewSIMDNode(simdType, simdTree, nullptr, SIMDIntrinsicCast, baseType, size);
2441             }
2442             retVal = simdTree;
2443         }
2444         break;
2445
2446         case SIMDIntrinsicInit:
2447         case SIMDIntrinsicInitN:
2448         {
2449             // SIMDIntrinsicInit:
2450             //    op2 - the initializer value
2451             //    op1 - byref of vector
2452             //
2453             // SIMDIntrinsicInitN
2454             //    op2 - list of initializer values stitched into a list
2455             //    op1 - byref of vector
2456             bool initFromFirstArgIndir = false;
2457             if (simdIntrinsicID == SIMDIntrinsicInit)
2458             {
2459                 op2 = impSIMDPopStack(baseType);
2460             }
2461             else
2462             {
2463                 assert(simdIntrinsicID == SIMDIntrinsicInitN);
2464                 assert(baseType == TYP_FLOAT);
2465
2466                 unsigned initCount    = argCount - 1;
2467                 unsigned elementCount = getSIMDVectorLength(size, baseType);
2468                 noway_assert(initCount == elementCount);
2469                 GenTree* nextArg = op2;
2470
2471                 // Build a GT_LIST with the N values.
2472                 // We must maintain left-to-right order of the args, but we will pop
2473                 // them off in reverse order (the Nth arg was pushed onto the stack last).
2474
2475                 GenTree* list              = nullptr;
2476                 GenTree* firstArg          = nullptr;
2477                 GenTree* prevArg           = nullptr;
2478                 int      offset            = 0;
2479                 bool     areArgsContiguous = true;
2480                 for (unsigned i = 0; i < initCount; i++)
2481                 {
2482                     GenTree* nextArg = impSIMDPopStack(baseType);
2483                     if (areArgsContiguous)
2484                     {
2485                         GenTree* curArg = nextArg;
2486                         firstArg        = curArg;
2487
2488                         if (prevArg != nullptr)
2489                         {
2490                             // Recall that we are popping the args off the stack in reverse order.
2491                             areArgsContiguous = areArgumentsContiguous(curArg, prevArg);
2492                         }
2493                         prevArg = curArg;
2494                     }
2495
2496                     list = new (this, GT_LIST) GenTreeOp(GT_LIST, baseType, nextArg, list);
2497                 }
2498
2499                 if (areArgsContiguous && baseType == TYP_FLOAT)
2500                 {
2501                     // Since Vector2, Vector3 and Vector4's arguments type are only float,
2502                     // we intialize the vector from first argument address, only when
2503                     // the baseType is TYP_FLOAT and the arguments are located contiguously in memory
2504                     initFromFirstArgIndir = true;
2505                     GenTree*  op2Address  = createAddressNodeForSIMDInit(firstArg, size);
2506                     var_types simdType    = getSIMDTypeForSize(size);
2507                     op2                   = gtNewOperNode(GT_IND, simdType, op2Address);
2508                 }
2509                 else
2510                 {
2511                     op2 = list;
2512                 }
2513             }
2514
2515             op1 = getOp1ForConstructor(opcode, newobjThis, clsHnd);
2516
2517             assert(op1->TypeGet() == TYP_BYREF);
2518             assert(genActualType(op2->TypeGet()) == genActualType(baseType) || initFromFirstArgIndir);
2519
2520             // For integral base types of size less than TYP_INT, expand the initializer
2521             // to fill size of TYP_INT bytes.
2522             if (varTypeIsSmallInt(baseType))
2523             {
2524                 // This case should occur only for Init intrinsic.
2525                 assert(simdIntrinsicID == SIMDIntrinsicInit);
2526
2527                 unsigned baseSize = genTypeSize(baseType);
2528                 int      multiplier;
2529                 if (baseSize == 1)
2530                 {
2531                     multiplier = 0x01010101;
2532                 }
2533                 else
2534                 {
2535                     assert(baseSize == 2);
2536                     multiplier = 0x00010001;
2537                 }
2538
2539                 GenTree* t1 = nullptr;
2540                 if (baseType == TYP_BYTE)
2541                 {
2542                     // What we have is a signed byte initializer,
2543                     // which when loaded to a reg will get sign extended to TYP_INT.
2544                     // But what we need is the initializer without sign extended or
2545                     // rather zero extended to 32-bits.
2546                     t1 = gtNewOperNode(GT_AND, TYP_INT, op2, gtNewIconNode(0xff, TYP_INT));
2547                 }
2548                 else if (baseType == TYP_SHORT)
2549                 {
2550                     // What we have is a signed short initializer,
2551                     // which when loaded to a reg will get sign extended to TYP_INT.
2552                     // But what we need is the initializer without sign extended or
2553                     // rather zero extended to 32-bits.
2554                     t1 = gtNewOperNode(GT_AND, TYP_INT, op2, gtNewIconNode(0xffff, TYP_INT));
2555                 }
2556                 else
2557                 {
2558                     assert(baseType == TYP_UBYTE || baseType == TYP_USHORT);
2559                     t1 = gtNewCastNode(TYP_INT, op2, false, TYP_INT);
2560                 }
2561
2562                 assert(t1 != nullptr);
2563                 GenTree* t2 = gtNewIconNode(multiplier, TYP_INT);
2564                 op2         = gtNewOperNode(GT_MUL, TYP_INT, t1, t2);
2565
2566                 // Construct a vector of TYP_INT with the new initializer and cast it back to vector of baseType
2567                 simdTree = gtNewSIMDNode(simdType, op2, nullptr, simdIntrinsicID, TYP_INT, size);
2568                 simdTree = gtNewSIMDNode(simdType, simdTree, nullptr, SIMDIntrinsicCast, baseType, size);
2569             }
2570             else
2571             {
2572
2573                 if (initFromFirstArgIndir)
2574                 {
2575                     simdTree = op2;
2576                     if (op1->gtOp.gtOp1->OperIsLocal())
2577                     {
2578                         // label the dst struct's lclvar is used for SIMD intrinsic,
2579                         // so that this dst struct won't be promoted.
2580                         setLclRelatedToSIMDIntrinsic(op1->gtOp.gtOp1);
2581                     }
2582                 }
2583                 else
2584                 {
2585                     simdTree = gtNewSIMDNode(simdType, op2, nullptr, simdIntrinsicID, baseType, size);
2586                 }
2587             }
2588
2589             copyBlkDst = op1;
2590             doCopyBlk  = true;
2591         }
2592         break;
2593
2594         case SIMDIntrinsicInitArray:
2595         case SIMDIntrinsicInitArrayX:
2596         case SIMDIntrinsicCopyToArray:
2597         case SIMDIntrinsicCopyToArrayX:
2598         {
2599             // op3 - index into array in case of SIMDIntrinsicCopyToArrayX and SIMDIntrinsicInitArrayX
2600             // op2 - array itself
2601             // op1 - byref to vector struct
2602
2603             unsigned int vectorLength = getSIMDVectorLength(size, baseType);
2604             // (This constructor takes only the zero-based arrays.)
2605             // We will add one or two bounds checks:
2606             // 1. If we have an index, we must do a check on that first.
2607             //    We can't combine it with the index + vectorLength check because
2608             //    a. It might be negative, and b. It may need to raise a different exception
2609             //    (captured as SCK_ARG_RNG_EXCPN for CopyTo and SCK_RNGCHK_FAIL for Init).
2610             // 2. We need to generate a check (SCK_ARG_EXCPN for CopyTo and SCK_RNGCHK_FAIL for Init)
2611             //    for the last array element we will access.
2612             //    We'll either check against (vectorLength - 1) or (index + vectorLength - 1).
2613
2614             GenTree* checkIndexExpr = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, vectorLength - 1);
2615
2616             // Get the index into the array.  If it has been provided, it will be on the
2617             // top of the stack.  Otherwise, it is null.
2618             if (argCount == 3)
2619             {
2620                 op3 = impSIMDPopStack(TYP_INT);
2621                 if (op3->IsIntegralConst(0))
2622                 {
2623                     op3 = nullptr;
2624                 }
2625             }
2626             else
2627             {
2628                 // TODO-CQ: Here, or elsewhere, check for the pattern where op2 is a newly constructed array, and
2629                 // change this to the InitN form.
2630                 // op3 = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, 0);
2631                 op3 = nullptr;
2632             }
2633
2634             // Clone the array for use in the bounds check.
2635             op2 = impSIMDPopStack(TYP_REF);
2636             assert(op2->TypeGet() == TYP_REF);
2637             GenTree* arrayRefForArgChk = op2;
2638             GenTree* argRngChk         = nullptr;
2639             GenTree* asg               = nullptr;
2640             if ((arrayRefForArgChk->gtFlags & GTF_SIDE_EFFECT) != 0)
2641             {
2642                 op2 = fgInsertCommaFormTemp(&arrayRefForArgChk);
2643             }
2644             else
2645             {
2646                 op2 = gtCloneExpr(arrayRefForArgChk);
2647             }
2648             assert(op2 != nullptr);
2649
2650             if (op3 != nullptr)
2651             {
2652                 SpecialCodeKind op3CheckKind;
2653                 if (simdIntrinsicID == SIMDIntrinsicInitArrayX)
2654                 {
2655                     op3CheckKind = SCK_RNGCHK_FAIL;
2656                 }
2657                 else
2658                 {
2659                     assert(simdIntrinsicID == SIMDIntrinsicCopyToArrayX);
2660                     op3CheckKind = SCK_ARG_RNG_EXCPN;
2661                 }
2662                 // We need to use the original expression on this, which is the first check.
2663                 GenTree* arrayRefForArgRngChk = arrayRefForArgChk;
2664                 // Then we clone the clone we just made for the next check.
2665                 arrayRefForArgChk = gtCloneExpr(op2);
2666                 // We know we MUST have had a cloneable expression.
2667                 assert(arrayRefForArgChk != nullptr);
2668                 GenTree* index = op3;
2669                 if ((index->gtFlags & GTF_SIDE_EFFECT) != 0)
2670                 {
2671                     op3 = fgInsertCommaFormTemp(&index);
2672                 }
2673                 else
2674                 {
2675                     op3 = gtCloneExpr(index);
2676                 }
2677
2678                 GenTreeArrLen* arrLen =
2679                     gtNewArrLen(TYP_INT, arrayRefForArgRngChk, (int)OFFSETOF__CORINFO_Array__length);
2680                 argRngChk = new (this, GT_ARR_BOUNDS_CHECK)
2681                     GenTreeBoundsChk(GT_ARR_BOUNDS_CHECK, TYP_VOID, index, arrLen, op3CheckKind);
2682                 // Now, clone op3 to create another node for the argChk
2683                 GenTree* index2 = gtCloneExpr(op3);
2684                 assert(index != nullptr);
2685                 checkIndexExpr = gtNewOperNode(GT_ADD, TYP_INT, index2, checkIndexExpr);
2686             }
2687
2688             // Insert a bounds check for index + offset - 1.
2689             // This must be a "normal" array.
2690             SpecialCodeKind op2CheckKind;
2691             if (simdIntrinsicID == SIMDIntrinsicInitArray || simdIntrinsicID == SIMDIntrinsicInitArrayX)
2692             {
2693                 op2CheckKind = SCK_RNGCHK_FAIL;
2694             }
2695             else
2696             {
2697                 op2CheckKind = SCK_ARG_EXCPN;
2698             }
2699             GenTreeArrLen*    arrLen = gtNewArrLen(TYP_INT, arrayRefForArgChk, (int)OFFSETOF__CORINFO_Array__length);
2700             GenTreeBoundsChk* argChk = new (this, GT_ARR_BOUNDS_CHECK)
2701                 GenTreeBoundsChk(GT_ARR_BOUNDS_CHECK, TYP_VOID, checkIndexExpr, arrLen, op2CheckKind);
2702
2703             // Create a GT_COMMA tree for the bounds check(s).
2704             op2 = gtNewOperNode(GT_COMMA, op2->TypeGet(), argChk, op2);
2705             if (argRngChk != nullptr)
2706             {
2707                 op2 = gtNewOperNode(GT_COMMA, op2->TypeGet(), argRngChk, op2);
2708             }
2709
2710             if (simdIntrinsicID == SIMDIntrinsicInitArray || simdIntrinsicID == SIMDIntrinsicInitArrayX)
2711             {
2712                 op1        = getOp1ForConstructor(opcode, newobjThis, clsHnd);
2713                 simdTree   = gtNewSIMDNode(simdType, op2, op3, SIMDIntrinsicInitArray, baseType, size);
2714                 copyBlkDst = op1;
2715                 doCopyBlk  = true;
2716             }
2717             else
2718             {
2719                 assert(simdIntrinsicID == SIMDIntrinsicCopyToArray || simdIntrinsicID == SIMDIntrinsicCopyToArrayX);
2720                 op1 = impSIMDPopStack(simdType, instMethod);
2721                 assert(op1->TypeGet() == simdType);
2722
2723                 // copy vector (op1) to array (op2) starting at index (op3)
2724                 simdTree = op1;
2725
2726                 // TODO-Cleanup: Though it happens to just work fine front-end phases are not aware of GT_LEA node.
2727                 // Therefore, convert these to use GT_ADDR .
2728                 copyBlkDst = new (this, GT_LEA)
2729                     GenTreeAddrMode(TYP_BYREF, op2, op3, genTypeSize(baseType), OFFSETOF__CORINFO_Array__data);
2730                 doCopyBlk = true;
2731             }
2732         }
2733         break;
2734
2735         case SIMDIntrinsicInitFixed:
2736         {
2737             // We are initializing a fixed-length vector VLarge with a smaller fixed-length vector VSmall, plus 1 or 2
2738             // additional floats.
2739             //    op4 (optional) - float value for VLarge.W, if VLarge is Vector4, and VSmall is Vector2
2740             //    op3 - float value for VLarge.Z or VLarge.W
2741             //    op2 - VSmall
2742             //    op1 - byref of VLarge
2743             assert(baseType == TYP_FLOAT);
2744             unsigned elementByteCount = 4;
2745
2746             GenTree* op4 = nullptr;
2747             if (argCount == 4)
2748             {
2749                 op4 = impSIMDPopStack(TYP_FLOAT);
2750                 assert(op4->TypeGet() == TYP_FLOAT);
2751             }
2752             op3 = impSIMDPopStack(TYP_FLOAT);
2753             assert(op3->TypeGet() == TYP_FLOAT);
2754             // The input vector will either be TYP_SIMD8 or TYP_SIMD12.
2755             var_types smallSIMDType = TYP_SIMD8;
2756             if ((op4 == nullptr) && (simdType == TYP_SIMD16))
2757             {
2758                 smallSIMDType = TYP_SIMD12;
2759             }
2760             op2 = impSIMDPopStack(smallSIMDType);
2761             op1 = getOp1ForConstructor(opcode, newobjThis, clsHnd);
2762
2763             // We are going to redefine the operands so that:
2764             // - op3 is the value that's going into the Z position, or null if it's a Vector4 constructor with a single
2765             // operand, and
2766             // - op4 is the W position value, or null if this is a Vector3 constructor.
2767             if (size == 16 && argCount == 3)
2768             {
2769                 op4 = op3;
2770                 op3 = nullptr;
2771             }
2772
2773             simdTree = op2;
2774             if (op3 != nullptr)
2775             {
2776                 simdTree = gtNewSIMDNode(simdType, simdTree, op3, SIMDIntrinsicSetZ, baseType, size);
2777             }
2778             if (op4 != nullptr)
2779             {
2780                 simdTree = gtNewSIMDNode(simdType, simdTree, op4, SIMDIntrinsicSetW, baseType, size);
2781             }
2782
2783             copyBlkDst = op1;
2784             doCopyBlk  = true;
2785         }
2786         break;
2787
2788         case SIMDIntrinsicOpEquality:
2789         case SIMDIntrinsicInstEquals:
2790         {
2791             op2 = impSIMDPopStack(simdType);
2792             op1 = impSIMDPopStack(simdType, instMethod);
2793
2794             assert(op1->TypeGet() == simdType);
2795             assert(op2->TypeGet() == simdType);
2796
2797             simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, SIMDIntrinsicOpEquality, baseType, size);
2798             if (simdType == TYP_SIMD12)
2799             {
2800                 simdTree->gtFlags |= GTF_SIMD12_OP;
2801             }
2802             retVal = simdTree;
2803         }
2804         break;
2805
2806         case SIMDIntrinsicOpInEquality:
2807         {
2808             // op1 is the first operand
2809             // op2 is the second operand
2810             op2      = impSIMDPopStack(simdType);
2811             op1      = impSIMDPopStack(simdType, instMethod);
2812             simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, SIMDIntrinsicOpInEquality, baseType, size);
2813             if (simdType == TYP_SIMD12)
2814             {
2815                 simdTree->gtFlags |= GTF_SIMD12_OP;
2816             }
2817             retVal = simdTree;
2818         }
2819         break;
2820
2821         case SIMDIntrinsicEqual:
2822         case SIMDIntrinsicLessThan:
2823         case SIMDIntrinsicLessThanOrEqual:
2824         case SIMDIntrinsicGreaterThan:
2825         case SIMDIntrinsicGreaterThanOrEqual:
2826         {
2827             op2 = impSIMDPopStack(simdType);
2828             op1 = impSIMDPopStack(simdType, instMethod);
2829
2830             SIMDIntrinsicID intrinsicID = impSIMDRelOp(simdIntrinsicID, clsHnd, size, &baseType, &op1, &op2);
2831             simdTree                    = gtNewSIMDNode(genActualType(callType), op1, op2, intrinsicID, baseType, size);
2832             retVal                      = simdTree;
2833         }
2834         break;
2835
2836         case SIMDIntrinsicAdd:
2837         case SIMDIntrinsicSub:
2838         case SIMDIntrinsicMul:
2839         case SIMDIntrinsicDiv:
2840         case SIMDIntrinsicBitwiseAnd:
2841         case SIMDIntrinsicBitwiseAndNot:
2842         case SIMDIntrinsicBitwiseOr:
2843         case SIMDIntrinsicBitwiseXor:
2844         {
2845 #if defined(DEBUG)
2846             // check for the cases where we don't support intrinsics.
2847             // This check should be done before we make modifications to type stack.
2848             // Note that this is more of a double safety check for robustness since
2849             // we expect getSIMDIntrinsicInfo() to have filtered out intrinsics on
2850             // unsupported base types. If getSIMdIntrinsicInfo() doesn't filter due
2851             // to some bug, assert in chk/dbg will fire.
2852             if (!varTypeIsFloating(baseType))
2853             {
2854                 if (simdIntrinsicID == SIMDIntrinsicMul)
2855                 {
2856 #if defined(_TARGET_XARCH_)
2857                     if ((baseType != TYP_INT) && (baseType != TYP_SHORT))
2858                     {
2859                         // TODO-CQ: implement mul on these integer vectors.
2860                         // Note that SSE2 has no direct support for these vectors.
2861                         assert(!"Mul not supported on long/ulong/uint/small int vectors\n");
2862                         return nullptr;
2863                     }
2864 #endif // _TARGET_XARCH_
2865 #if defined(_TARGET_ARM64_)
2866                     if ((baseType == TYP_ULONG) && (baseType == TYP_LONG))
2867                     {
2868                         // TODO-CQ: implement mul on these integer vectors.
2869                         // Note that ARM64 has no direct support for these vectors.
2870                         assert(!"Mul not supported on long/ulong vectors\n");
2871                         return nullptr;
2872                     }
2873 #endif // _TARGET_ARM64_
2874                 }
2875 #if defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
2876                 // common to all integer type vectors
2877                 if (simdIntrinsicID == SIMDIntrinsicDiv)
2878                 {
2879                     // SSE2 doesn't support div on non-floating point vectors.
2880                     assert(!"Div not supported on integer type vectors\n");
2881                     return nullptr;
2882                 }
2883 #endif // defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
2884             }
2885 #endif // DEBUG
2886
2887             // op1 is the first operand; if instance method, op1 is "this" arg
2888             // op2 is the second operand
2889             op2 = impSIMDPopStack(simdType);
2890             op1 = impSIMDPopStack(simdType, instMethod);
2891
2892 #ifdef _TARGET_XARCH_
2893             if (simdIntrinsicID == SIMDIntrinsicBitwiseAndNot)
2894             {
2895                 // XARCH implements SIMDIntrinsicBitwiseAndNot as ~op1 & op2, while the
2896                 // software implementation does op1 & ~op2, so we need to swap the operands
2897
2898                 GenTree* tmp = op2;
2899                 op2          = op1;
2900                 op1          = tmp;
2901             }
2902 #endif // _TARGET_XARCH_
2903
2904             simdTree = gtNewSIMDNode(simdType, op1, op2, simdIntrinsicID, baseType, size);
2905             retVal   = simdTree;
2906         }
2907         break;
2908
2909         case SIMDIntrinsicSelect:
2910         {
2911             // op3 is a SIMD variable that is the second source
2912             // op2 is a SIMD variable that is the first source
2913             // op1 is a SIMD variable which is the bit mask.
2914             op3 = impSIMDPopStack(simdType);
2915             op2 = impSIMDPopStack(simdType);
2916             op1 = impSIMDPopStack(simdType);
2917
2918             retVal = impSIMDSelect(clsHnd, baseType, size, op1, op2, op3);
2919         }
2920         break;
2921
2922         case SIMDIntrinsicMin:
2923         case SIMDIntrinsicMax:
2924         {
2925             // op1 is the first operand; if instance method, op1 is "this" arg
2926             // op2 is the second operand
2927             op2 = impSIMDPopStack(simdType);
2928             op1 = impSIMDPopStack(simdType, instMethod);
2929
2930             retVal = impSIMDMinMax(simdIntrinsicID, clsHnd, baseType, size, op1, op2);
2931         }
2932         break;
2933
2934         case SIMDIntrinsicGetItem:
2935         {
2936             // op1 is a SIMD variable that is "this" arg
2937             // op2 is an index of TYP_INT
2938             op2              = impSIMDPopStack(TYP_INT);
2939             op1              = impSIMDPopStack(simdType, instMethod);
2940             int vectorLength = getSIMDVectorLength(size, baseType);
2941             if (!op2->IsCnsIntOrI() || op2->AsIntCon()->gtIconVal >= vectorLength || op2->AsIntCon()->gtIconVal < 0)
2942             {
2943                 // We need to bounds-check the length of the vector.
2944                 // For that purpose, we need to clone the index expression.
2945                 GenTree* index = op2;
2946                 if ((index->gtFlags & GTF_SIDE_EFFECT) != 0)
2947                 {
2948                     op2 = fgInsertCommaFormTemp(&index);
2949                 }
2950                 else
2951                 {
2952                     op2 = gtCloneExpr(index);
2953                 }
2954
2955                 GenTree*          lengthNode = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, vectorLength);
2956                 GenTreeBoundsChk* simdChk =
2957                     new (this, GT_SIMD_CHK) GenTreeBoundsChk(GT_SIMD_CHK, TYP_VOID, index, lengthNode, SCK_RNGCHK_FAIL);
2958
2959                 // Create a GT_COMMA tree for the bounds check.
2960                 op2 = gtNewOperNode(GT_COMMA, op2->TypeGet(), simdChk, op2);
2961             }
2962
2963             assert(op1->TypeGet() == simdType);
2964             assert(op2->TypeGet() == TYP_INT);
2965
2966             simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, simdIntrinsicID, baseType, size);
2967             retVal   = simdTree;
2968         }
2969         break;
2970
2971         case SIMDIntrinsicDotProduct:
2972         {
2973 #if defined(_TARGET_XARCH_)
2974             // Right now dot product is supported only for float/double vectors and
2975             // int vectors on SSE4/AVX.
2976             if (!varTypeIsFloating(baseType) && !(baseType == TYP_INT && getSIMDSupportLevel() >= SIMD_SSE4_Supported))
2977             {
2978                 return nullptr;
2979             }
2980 #endif // _TARGET_XARCH_
2981
2982             // op1 is a SIMD variable that is the first source and also "this" arg.
2983             // op2 is a SIMD variable which is the second source.
2984             op2 = impSIMDPopStack(simdType);
2985             op1 = impSIMDPopStack(simdType, instMethod);
2986
2987             simdTree = gtNewSIMDNode(baseType, op1, op2, simdIntrinsicID, baseType, size);
2988             if (simdType == TYP_SIMD12)
2989             {
2990                 simdTree->gtFlags |= GTF_SIMD12_OP;
2991             }
2992             retVal = simdTree;
2993         }
2994         break;
2995
2996         case SIMDIntrinsicSqrt:
2997         {
2998 #if (defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)) && defined(DEBUG)
2999             // SSE/AVX/ARM64 doesn't support sqrt on integer type vectors and hence
3000             // should never be seen as an intrinsic here. See SIMDIntrinsicList.h
3001             // for supported base types for this intrinsic.
3002             if (!varTypeIsFloating(baseType))
3003             {
3004                 assert(!"Sqrt not supported on integer vectors\n");
3005                 return nullptr;
3006             }
3007 #endif // (defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)) && defined(DEBUG)
3008
3009             op1 = impSIMDPopStack(simdType);
3010
3011             retVal = gtNewSIMDNode(genActualType(callType), op1, nullptr, simdIntrinsicID, baseType, size);
3012         }
3013         break;
3014
3015         case SIMDIntrinsicAbs:
3016             op1    = impSIMDPopStack(simdType);
3017             retVal = impSIMDAbs(clsHnd, baseType, size, op1);
3018             break;
3019
3020         case SIMDIntrinsicGetW:
3021             retVal = impSIMDGetFixed(simdType, baseType, size, 3);
3022             break;
3023
3024         case SIMDIntrinsicGetZ:
3025             retVal = impSIMDGetFixed(simdType, baseType, size, 2);
3026             break;
3027
3028         case SIMDIntrinsicGetY:
3029             retVal = impSIMDGetFixed(simdType, baseType, size, 1);
3030             break;
3031
3032         case SIMDIntrinsicGetX:
3033             retVal = impSIMDGetFixed(simdType, baseType, size, 0);
3034             break;
3035
3036         case SIMDIntrinsicSetW:
3037         case SIMDIntrinsicSetZ:
3038         case SIMDIntrinsicSetY:
3039         case SIMDIntrinsicSetX:
3040         {
3041             // op2 is the value to be set at indexTemp position
3042             // op1 is SIMD vector that is going to be modified, which is a byref
3043
3044             // If op1 has a side-effect, then don't make it an intrinsic.
3045             // It would be in-efficient to read the entire vector into xmm reg,
3046             // modify it and write back entire xmm reg.
3047             //
3048             // TODO-CQ: revisit this later.
3049             op1 = impStackTop(1).val;
3050             if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0)
3051             {
3052                 return nullptr;
3053             }
3054
3055             op2 = impSIMDPopStack(baseType);
3056             op1 = impSIMDPopStack(simdType, instMethod);
3057
3058             GenTree* src = gtCloneExpr(op1);
3059             assert(src != nullptr);
3060             simdTree = gtNewSIMDNode(simdType, src, op2, simdIntrinsicID, baseType, size);
3061
3062             copyBlkDst = gtNewOperNode(GT_ADDR, TYP_BYREF, op1);
3063             doCopyBlk  = true;
3064         }
3065         break;
3066
3067         // Unary operators that take and return a Vector.
3068         case SIMDIntrinsicCast:
3069         case SIMDIntrinsicConvertToSingle:
3070         case SIMDIntrinsicConvertToDouble:
3071         case SIMDIntrinsicConvertToInt32:
3072         {
3073             op1 = impSIMDPopStack(simdType, instMethod);
3074
3075             simdTree = gtNewSIMDNode(simdType, op1, nullptr, simdIntrinsicID, baseType, size);
3076             retVal   = simdTree;
3077         }
3078         break;
3079
3080         case SIMDIntrinsicConvertToInt64:
3081         {
3082 #ifdef _TARGET_64BIT_
3083             op1 = impSIMDPopStack(simdType, instMethod);
3084
3085             simdTree = gtNewSIMDNode(simdType, op1, nullptr, simdIntrinsicID, baseType, size);
3086             retVal   = simdTree;
3087 #else
3088             JITDUMP("SIMD Conversion to Int64 is not supported on this platform\n");
3089             return nullptr;
3090 #endif
3091         }
3092         break;
3093
3094         case SIMDIntrinsicNarrow:
3095         {
3096             assert(!instMethod);
3097             op2 = impSIMDPopStack(simdType);
3098             op1 = impSIMDPopStack(simdType);
3099             // op1 and op2 are two input Vector<T>.
3100             simdTree = gtNewSIMDNode(simdType, op1, op2, simdIntrinsicID, baseType, size);
3101             retVal   = simdTree;
3102         }
3103         break;
3104
3105         case SIMDIntrinsicWiden:
3106         {
3107             GenTree* dstAddrHi = impSIMDPopStack(TYP_BYREF);
3108             GenTree* dstAddrLo = impSIMDPopStack(TYP_BYREF);
3109             op1                = impSIMDPopStack(simdType);
3110             GenTree* dupOp1    = fgInsertCommaFormTemp(&op1, gtGetStructHandleForSIMD(simdType, baseType));
3111
3112             // Widen the lower half and assign it to dstAddrLo.
3113             simdTree = gtNewSIMDNode(simdType, op1, nullptr, SIMDIntrinsicWidenLo, baseType, size);
3114             GenTree* loDest =
3115                 new (this, GT_BLK) GenTreeBlk(GT_BLK, simdType, dstAddrLo, getSIMDTypeSizeInBytes(clsHnd));
3116             GenTree* loAsg = gtNewBlkOpNode(loDest, simdTree, getSIMDTypeSizeInBytes(clsHnd),
3117                                             false, // not volatile
3118                                             true); // copyBlock
3119             loAsg->gtFlags |= ((simdTree->gtFlags | dstAddrLo->gtFlags) & GTF_ALL_EFFECT);
3120
3121             // Widen the upper half and assign it to dstAddrHi.
3122             simdTree = gtNewSIMDNode(simdType, dupOp1, nullptr, SIMDIntrinsicWidenHi, baseType, size);
3123             GenTree* hiDest =
3124                 new (this, GT_BLK) GenTreeBlk(GT_BLK, simdType, dstAddrHi, getSIMDTypeSizeInBytes(clsHnd));
3125             GenTree* hiAsg = gtNewBlkOpNode(hiDest, simdTree, getSIMDTypeSizeInBytes(clsHnd),
3126                                             false, // not volatile
3127                                             true); // copyBlock
3128             hiAsg->gtFlags |= ((simdTree->gtFlags | dstAddrHi->gtFlags) & GTF_ALL_EFFECT);
3129
3130             retVal = gtNewOperNode(GT_COMMA, simdType, loAsg, hiAsg);
3131         }
3132         break;
3133
3134         case SIMDIntrinsicHWAccel:
3135         {
3136             GenTreeIntCon* intConstTree = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, 1);
3137             retVal                      = intConstTree;
3138         }
3139         break;
3140
3141         default:
3142             assert(!"Unimplemented SIMD Intrinsic");
3143             return nullptr;
3144     }
3145
3146 #if defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
3147     // XArch/Arm64: also indicate that we use floating point registers.
3148     // The need for setting this here is that a method may not have SIMD
3149     // type lclvars, but might be exercising SIMD intrinsics on fields of
3150     // SIMD type.
3151     //
3152     // e.g.  public Vector<float> ComplexVecFloat::sqabs() { return this.r * this.r + this.i * this.i; }
3153     compFloatingPointUsed = true;
3154 #endif // defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
3155
3156     // At this point, we have a tree that we are going to store into a destination.
3157     // TODO-1stClassStructs: This should be a simple store or assignment, and should not require
3158     // GTF_ALL_EFFECT for the dest. This is currently emulating the previous behavior of
3159     // block ops.
3160     if (doCopyBlk)
3161     {
3162         GenTree* dest = new (this, GT_BLK) GenTreeBlk(GT_BLK, simdType, copyBlkDst, getSIMDTypeSizeInBytes(clsHnd));
3163         dest->gtFlags |= GTF_GLOB_REF;
3164         retVal = gtNewBlkOpNode(dest, simdTree, getSIMDTypeSizeInBytes(clsHnd),
3165                                 false, // not volatile
3166                                 true); // copyBlock
3167         retVal->gtFlags |= ((simdTree->gtFlags | copyBlkDst->gtFlags) & GTF_ALL_EFFECT);
3168     }
3169
3170     return retVal;
3171 }
3172
3173 #endif // FEATURE_SIMD