src/jit/simd.cpp

   1 // Licensed to the .NET Foundation under one or more agreements.
   2 // The .NET Foundation licenses this file to you under the MIT license.
   3 // See the LICENSE file in the project root for more information.
   4
   5 //
   6 //   SIMD Support
   7 //
   8 // IMPORTANT NOTES AND CAVEATS:
   9 //
  10 // This implementation is preliminary, and may change dramatically.
  11 //
  12 // New JIT types, TYP_SIMDxx, are introduced, and the SIMD intrinsics are created as GT_SIMD nodes.
  13 // Nodes of SIMD types will be typed as TYP_SIMD* (e.g. TYP_SIMD8, TYP_SIMD16, etc.).
  14 //
  15 // Note that currently the "reference implementation" is the same as the runtime dll.  As such, it is currently
  16 // providing implementations for those methods not currently supported by the JIT as intrinsics.
  17 //
  18 // These are currently recognized using string compares, in order to provide an implementation in the JIT
  19 // without taking a dependency on the VM.
  20 // Furthermore, in the CTP, in order to limit the impact of doing these string compares
  21 // against assembly names, we only look for the SIMDVector assembly if we are compiling a class constructor.  This
  22 // makes it somewhat more "pay for play" but is a significant usability compromise.
  23 // This has been addressed for RTM by doing the assembly recognition in the VM.
  24 // --------------------------------------------------------------------------------------
  25
  26 #include "jitpch.h"
  27 #include "simd.h"
  28
  29 #ifdef _MSC_VER
  30 #pragma hdrstop
  31 #endif
  32
  33 #ifdef FEATURE_SIMD
  34
  35 // Intrinsic Id to intrinsic info map
  36 const SIMDIntrinsicInfo simdIntrinsicInfoArray[] = {
  37 #define SIMD_INTRINSIC(mname, inst, id, name, retType, argCount, arg1, arg2, arg3, t1, t2, t3, t4, t5, t6, t7, t8, t9, \
  38                        t10)                                                                                            \
  39     {SIMDIntrinsic##id, mname, inst, retType, argCount, arg1, arg2, arg3, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10},
  40 #include "simdintrinsiclist.h"
  41 };
  42
  43 //------------------------------------------------------------------------
  44 // getSIMDVectorLength: Get the length (number of elements of base type) of
  45 //                      SIMD Vector given its size and base (element) type.
  46 //
  47 // Arguments:
  48 //    simdSize   - size of the SIMD vector
  49 //    baseType   - type of the elements of the SIMD vector
  50 //
  51 // static
  52 int Compiler::getSIMDVectorLength(unsigned simdSize, var_types baseType)
  53 {
  54     return simdSize / genTypeSize(baseType);
  55 }
  56
  57 //------------------------------------------------------------------------
  58 // Get the length (number of elements of base type) of SIMD Vector given by typeHnd.
  59 //
  60 // Arguments:
  61 //    typeHnd  - type handle of the SIMD vector
  62 //
  63 int Compiler::getSIMDVectorLength(CORINFO_CLASS_HANDLE typeHnd)
  64 {
  65     unsigned  sizeBytes = 0;
  66     var_types baseType  = getBaseTypeAndSizeOfSIMDType(typeHnd, &sizeBytes);
  67     return getSIMDVectorLength(sizeBytes, baseType);
  68 }
  69
  70 //------------------------------------------------------------------------
  71 // Get the preferred alignment of SIMD vector type for better performance.
  72 //
  73 // Arguments:
  74 //    typeHnd  - type handle of the SIMD vector
  75 //
  76 int Compiler::getSIMDTypeAlignment(var_types simdType)
  77 {
  78 #ifdef _TARGET_XARCH_
  79     // Fixed length vectors have the following alignment preference
  80     // Vector2   = 8 byte alignment
  81     // Vector3/4 = 16-byte alignment
  82     unsigned size = genTypeSize(simdType);
  83
  84     // preferred alignment for SSE2 128-bit vectors is 16-bytes
  85     if (size == 8)
  86     {
  87         return 8;
  88     }
  89     else if (size <= 16)
  90     {
  91         assert((size == 12) || (size == 16));
  92         return 16;
  93     }
  94     else
  95     {
  96         assert(size == 32);
  97         return 32;
  98     }
  99 #elif defined(_TARGET_ARM64_)
 100     return 16;
 101 #else
 102     assert(!"getSIMDTypeAlignment() unimplemented on target arch");
 103     unreached();
 104 #endif
 105 }
 106
 107 //----------------------------------------------------------------------------------
 108 // Return the base type and size of SIMD vector type given its type handle.
 109 //
 110 // Arguments:
 111 //    typeHnd   - The handle of the type we're interested in.
 112 //    sizeBytes - out param
 113 //
 114 // Return Value:
 115 //    base type of SIMD vector.
 116 //    sizeBytes if non-null is set to size in bytes.
 117 //
 118 // TODO-Throughput: current implementation parses class name to find base type. Change
 119 //         this when we implement  SIMD intrinsic identification for the final
 120 //         product.
 121 //
 122 var_types Compiler::getBaseTypeAndSizeOfSIMDType(CORINFO_CLASS_HANDLE typeHnd, unsigned* sizeBytes /*= nullptr */)
 123 {
 124     assert(featureSIMD);
 125
 126     if (m_simdHandleCache == nullptr)
 127     {
 128         if (impInlineInfo == nullptr)
 129         {
 130             m_simdHandleCache = new (this, CMK_Generic) SIMDHandlesCache();
 131         }
 132         else
 133         {
 134             // Steal the inliner compiler's cache (create it if not available).
 135
 136             if (impInlineInfo->InlineRoot->m_simdHandleCache == nullptr)
 137             {
 138                 impInlineInfo->InlineRoot->m_simdHandleCache = new (this, CMK_Generic) SIMDHandlesCache();
 139             }
 140
 141             m_simdHandleCache = impInlineInfo->InlineRoot->m_simdHandleCache;
 142         }
 143     }
 144
 145     if (typeHnd == nullptr)
 146     {
 147         return TYP_UNKNOWN;
 148     }
 149
 150     // fast path search using cached type handles of important types
 151     var_types simdBaseType = TYP_UNKNOWN;
 152     unsigned  size         = 0;
 153
 154     // TODO - Optimize SIMD type recognition by IntrinsicAttribute
 155     if (isSIMDClass(typeHnd))
 156     {
 157         // The most likely to be used type handles are looked up first followed by
 158         // less likely to be used type handles
 159         if (typeHnd == m_simdHandleCache->SIMDFloatHandle)
 160         {
 161             simdBaseType = TYP_FLOAT;
 162             JITDUMP("  Known type SIMD Vector<Float>\n");
 163         }
 164         else if (typeHnd == m_simdHandleCache->SIMDIntHandle)
 165         {
 166             simdBaseType = TYP_INT;
 167             JITDUMP("  Known type SIMD Vector<Int>\n");
 168         }
 169         else if (typeHnd == m_simdHandleCache->SIMDVector2Handle)
 170         {
 171             simdBaseType = TYP_FLOAT;
 172             size         = 2 * genTypeSize(TYP_FLOAT);
 173             assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE));
 174             JITDUMP("  Known type Vector2\n");
 175         }
 176         else if (typeHnd == m_simdHandleCache->SIMDVector3Handle)
 177         {
 178             simdBaseType = TYP_FLOAT;
 179             size         = 3 * genTypeSize(TYP_FLOAT);
 180             assert(size == info.compCompHnd->getClassSize(typeHnd));
 181             JITDUMP("  Known type Vector3\n");
 182         }
 183         else if (typeHnd == m_simdHandleCache->SIMDVector4Handle)
 184         {
 185             simdBaseType = TYP_FLOAT;
 186             size         = 4 * genTypeSize(TYP_FLOAT);
 187             assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE));
 188             JITDUMP("  Known type Vector4\n");
 189         }
 190         else if (typeHnd == m_simdHandleCache->SIMDVectorHandle)
 191         {
 192             JITDUMP("  Known type Vector\n");
 193         }
 194         else if (typeHnd == m_simdHandleCache->SIMDUShortHandle)
 195         {
 196             simdBaseType = TYP_USHORT;
 197             JITDUMP("  Known type SIMD Vector<ushort>\n");
 198         }
 199         else if (typeHnd == m_simdHandleCache->SIMDUByteHandle)
 200         {
 201             simdBaseType = TYP_UBYTE;
 202             JITDUMP("  Known type SIMD Vector<ubyte>\n");
 203         }
 204         else if (typeHnd == m_simdHandleCache->SIMDDoubleHandle)
 205         {
 206             simdBaseType = TYP_DOUBLE;
 207             JITDUMP("  Known type SIMD Vector<Double>\n");
 208         }
 209         else if (typeHnd == m_simdHandleCache->SIMDLongHandle)
 210         {
 211             simdBaseType = TYP_LONG;
 212             JITDUMP("  Known type SIMD Vector<Long>\n");
 213         }
 214         else if (typeHnd == m_simdHandleCache->SIMDShortHandle)
 215         {
 216             simdBaseType = TYP_SHORT;
 217             JITDUMP("  Known type SIMD Vector<short>\n");
 218         }
 219         else if (typeHnd == m_simdHandleCache->SIMDByteHandle)
 220         {
 221             simdBaseType = TYP_BYTE;
 222             JITDUMP("  Known type SIMD Vector<byte>\n");
 223         }
 224         else if (typeHnd == m_simdHandleCache->SIMDUIntHandle)
 225         {
 226             simdBaseType = TYP_UINT;
 227             JITDUMP("  Known type SIMD Vector<uint>\n");
 228         }
 229         else if (typeHnd == m_simdHandleCache->SIMDULongHandle)
 230         {
 231             simdBaseType = TYP_ULONG;
 232             JITDUMP("  Known type SIMD Vector<ulong>\n");
 233         }
 234
 235         // slow path search
 236         if (simdBaseType == TYP_UNKNOWN)
 237         {
 238             // Doesn't match with any of the cached type handles.
 239             // Obtain base type by parsing fully qualified class name.
 240             //
 241             // TODO-Throughput: implement product shipping solution to query base type.
 242             WCHAR  className[256] = {0};
 243             WCHAR* pbuf           = &className[0];
 244             int    len            = _countof(className);
 245             info.compCompHnd->appendClassName(&pbuf, &len, typeHnd, TRUE, FALSE, FALSE);
 246             noway_assert(pbuf < &className[256]);
 247             JITDUMP("SIMD Candidate Type %S\n", className);
 248
 249             if (wcsncmp(className, W("System.Numerics."), 16) == 0)
 250             {
 251                 if (wcsncmp(&(className[16]), W("Vector`1["), 9) == 0)
 252                 {
 253                     if (wcsncmp(&(className[25]), W("System.Single"), 13) == 0)
 254                     {
 255                         m_simdHandleCache->SIMDFloatHandle = typeHnd;
 256                         simdBaseType                       = TYP_FLOAT;
 257                         JITDUMP("  Found type SIMD Vector<Float>\n");
 258                     }
 259                     else if (wcsncmp(&(className[25]), W("System.Int32"), 12) == 0)
 260                     {
 261                         m_simdHandleCache->SIMDIntHandle = typeHnd;
 262                         simdBaseType                     = TYP_INT;
 263                         JITDUMP("  Found type SIMD Vector<Int>\n");
 264                     }
 265                     else if (wcsncmp(&(className[25]), W("System.UInt16"), 13) == 0)
 266                     {
 267                         m_simdHandleCache->SIMDUShortHandle = typeHnd;
 268                         simdBaseType                        = TYP_USHORT;
 269                         JITDUMP("  Found type SIMD Vector<ushort>\n");
 270                     }
 271                     else if (wcsncmp(&(className[25]), W("System.Byte"), 11) == 0)
 272                     {
 273                         m_simdHandleCache->SIMDUByteHandle = typeHnd;
 274                         simdBaseType                       = TYP_UBYTE;
 275                         JITDUMP("  Found type SIMD Vector<ubyte>\n");
 276                     }
 277                     else if (wcsncmp(&(className[25]), W("System.Double"), 13) == 0)
 278                     {
 279                         m_simdHandleCache->SIMDDoubleHandle = typeHnd;
 280                         simdBaseType                        = TYP_DOUBLE;
 281                         JITDUMP("  Found type SIMD Vector<Double>\n");
 282                     }
 283                     else if (wcsncmp(&(className[25]), W("System.Int64"), 12) == 0)
 284                     {
 285                         m_simdHandleCache->SIMDLongHandle = typeHnd;
 286                         simdBaseType                      = TYP_LONG;
 287                         JITDUMP("  Found type SIMD Vector<Long>\n");
 288                     }
 289                     else if (wcsncmp(&(className[25]), W("System.Int16"), 12) == 0)
 290                     {
 291                         m_simdHandleCache->SIMDShortHandle = typeHnd;
 292                         simdBaseType                       = TYP_SHORT;
 293                         JITDUMP("  Found type SIMD Vector<short>\n");
 294                     }
 295                     else if (wcsncmp(&(className[25]), W("System.SByte"), 12) == 0)
 296                     {
 297                         m_simdHandleCache->SIMDByteHandle = typeHnd;
 298                         simdBaseType                      = TYP_BYTE;
 299                         JITDUMP("  Found type SIMD Vector<byte>\n");
 300                     }
 301                     else if (wcsncmp(&(className[25]), W("System.UInt32"), 13) == 0)
 302                     {
 303                         m_simdHandleCache->SIMDUIntHandle = typeHnd;
 304                         simdBaseType                      = TYP_UINT;
 305                         JITDUMP("  Found type SIMD Vector<uint>\n");
 306                     }
 307                     else if (wcsncmp(&(className[25]), W("System.UInt64"), 13) == 0)
 308                     {
 309                         m_simdHandleCache->SIMDULongHandle = typeHnd;
 310                         simdBaseType                       = TYP_ULONG;
 311                         JITDUMP("  Found type SIMD Vector<ulong>\n");
 312                     }
 313                     else
 314                     {
 315                         JITDUMP("  Unknown SIMD Vector<T>\n");
 316                     }
 317                 }
 318                 else if (wcsncmp(&(className[16]), W("Vector2"), 8) == 0)
 319                 {
 320                     m_simdHandleCache->SIMDVector2Handle = typeHnd;
 321
 322                     simdBaseType = TYP_FLOAT;
 323                     size         = 2 * genTypeSize(TYP_FLOAT);
 324                     assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE));
 325                     JITDUMP(" Found Vector2\n");
 326                 }
 327                 else if (wcsncmp(&(className[16]), W("Vector3"), 8) == 0)
 328                 {
 329                     m_simdHandleCache->SIMDVector3Handle = typeHnd;
 330
 331                     simdBaseType = TYP_FLOAT;
 332                     size         = 3 * genTypeSize(TYP_FLOAT);
 333                     assert(size == info.compCompHnd->getClassSize(typeHnd));
 334                     JITDUMP(" Found Vector3\n");
 335                 }
 336                 else if (wcsncmp(&(className[16]), W("Vector4"), 8) == 0)
 337                 {
 338                     m_simdHandleCache->SIMDVector4Handle = typeHnd;
 339
 340                     simdBaseType = TYP_FLOAT;
 341                     size         = 4 * genTypeSize(TYP_FLOAT);
 342                     assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE));
 343                     JITDUMP(" Found Vector4\n");
 344                 }
 345                 else if (wcsncmp(&(className[16]), W("Vector"), 6) == 0)
 346                 {
 347                     m_simdHandleCache->SIMDVectorHandle = typeHnd;
 348                     JITDUMP(" Found type Vector\n");
 349                 }
 350                 else
 351                 {
 352                     JITDUMP("  Unknown SIMD Type\n");
 353                 }
 354             }
 355         }
 356         if (simdBaseType != TYP_UNKNOWN && sizeBytes != nullptr)
 357         {
 358             // If not a fixed size vector then its size is same as SIMD vector
 359             // register length in bytes
 360             if (size == 0)
 361             {
 362                 size = getSIMDVectorRegisterByteLength();
 363             }
 364
 365             *sizeBytes = size;
 366             setUsesSIMDTypes(true);
 367         }
 368     }
 369 #ifdef FEATURE_HW_INTRINSICS
 370     else if (isIntrinsicType(typeHnd))
 371     {
 372         const size_t Vector64SizeBytes  = 64 / 8;
 373         const size_t Vector128SizeBytes = 128 / 8;
 374         const size_t Vector256SizeBytes = 256 / 8;
 375
 376 #if defined(_TARGET_XARCH_)
 377         static_assert_no_msg(YMM_REGSIZE_BYTES == Vector256SizeBytes);
 378         static_assert_no_msg(XMM_REGSIZE_BYTES == Vector128SizeBytes);
 379
 380         if (typeHnd == m_simdHandleCache->Vector256FloatHandle)
 381         {
 382             simdBaseType = TYP_FLOAT;
 383             size         = Vector256SizeBytes;
 384             JITDUMP("  Known type Vector256<float>\n");
 385         }
 386         else if (typeHnd == m_simdHandleCache->Vector256DoubleHandle)
 387         {
 388             simdBaseType = TYP_DOUBLE;
 389             size         = Vector256SizeBytes;
 390             JITDUMP("  Known type Vector256<double>\n");
 391         }
 392         else if (typeHnd == m_simdHandleCache->Vector256IntHandle)
 393         {
 394             simdBaseType = TYP_INT;
 395             size         = Vector256SizeBytes;
 396             JITDUMP("  Known type Vector256<int>\n");
 397         }
 398         else if (typeHnd == m_simdHandleCache->Vector256UIntHandle)
 399         {
 400             simdBaseType = TYP_UINT;
 401             size         = Vector256SizeBytes;
 402             JITDUMP("  Known type Vector256<uint>\n");
 403         }
 404         else if (typeHnd == m_simdHandleCache->Vector256ShortHandle)
 405         {
 406             simdBaseType = TYP_SHORT;
 407             size         = Vector256SizeBytes;
 408             JITDUMP("  Known type Vector256<short>\n");
 409         }
 410         else if (typeHnd == m_simdHandleCache->Vector256UShortHandle)
 411         {
 412             simdBaseType = TYP_USHORT;
 413             size         = Vector256SizeBytes;
 414             JITDUMP("  Known type Vector256<ushort>\n");
 415         }
 416         else if (typeHnd == m_simdHandleCache->Vector256ByteHandle)
 417         {
 418             simdBaseType = TYP_BYTE;
 419             size         = Vector256SizeBytes;
 420             JITDUMP("  Known type Vector256<sbyte>\n");
 421         }
 422         else if (typeHnd == m_simdHandleCache->Vector256UByteHandle)
 423         {
 424             simdBaseType = TYP_UBYTE;
 425             size         = Vector256SizeBytes;
 426             JITDUMP("  Known type Vector256<byte>\n");
 427         }
 428         else if (typeHnd == m_simdHandleCache->Vector256LongHandle)
 429         {
 430             simdBaseType = TYP_LONG;
 431             size         = Vector256SizeBytes;
 432             JITDUMP("  Known type Vector256<long>\n");
 433         }
 434         else if (typeHnd == m_simdHandleCache->Vector256ULongHandle)
 435         {
 436             simdBaseType = TYP_ULONG;
 437             size         = Vector256SizeBytes;
 438             JITDUMP("  Known type Vector256<ulong>\n");
 439         }
 440         else
 441 #endif // defined(_TARGET_XARCH)
 442             if (typeHnd == m_simdHandleCache->Vector128FloatHandle)
 443         {
 444             simdBaseType = TYP_FLOAT;
 445             size         = Vector128SizeBytes;
 446             JITDUMP("  Known type Vector128<float>\n");
 447         }
 448         else if (typeHnd == m_simdHandleCache->Vector128DoubleHandle)
 449         {
 450             simdBaseType = TYP_DOUBLE;
 451             size         = Vector128SizeBytes;
 452             JITDUMP("  Known type Vector128<double>\n");
 453         }
 454         else if (typeHnd == m_simdHandleCache->Vector128IntHandle)
 455         {
 456             simdBaseType = TYP_INT;
 457             size         = Vector128SizeBytes;
 458             JITDUMP("  Known type Vector128<int>\n");
 459         }
 460         else if (typeHnd == m_simdHandleCache->Vector128UIntHandle)
 461         {
 462             simdBaseType = TYP_UINT;
 463             size         = Vector128SizeBytes;
 464             JITDUMP("  Known type Vector128<uint>\n");
 465         }
 466         else if (typeHnd == m_simdHandleCache->Vector128ShortHandle)
 467         {
 468             simdBaseType = TYP_SHORT;
 469             size         = Vector128SizeBytes;
 470             JITDUMP("  Known type Vector128<short>\n");
 471         }
 472         else if (typeHnd == m_simdHandleCache->Vector128UShortHandle)
 473         {
 474             simdBaseType = TYP_USHORT;
 475             size         = Vector128SizeBytes;
 476             JITDUMP("  Known type Vector128<ushort>\n");
 477         }
 478         else if (typeHnd == m_simdHandleCache->Vector128ByteHandle)
 479         {
 480             simdBaseType = TYP_BYTE;
 481             size         = Vector128SizeBytes;
 482             JITDUMP("  Known type Vector128<sbyte>\n");
 483         }
 484         else if (typeHnd == m_simdHandleCache->Vector128UByteHandle)
 485         {
 486             simdBaseType = TYP_UBYTE;
 487             size         = Vector128SizeBytes;
 488             JITDUMP("  Known type Vector128<byte>\n");
 489         }
 490         else if (typeHnd == m_simdHandleCache->Vector128LongHandle)
 491         {
 492             simdBaseType = TYP_LONG;
 493             size         = Vector128SizeBytes;
 494             JITDUMP("  Known type Vector128<long>\n");
 495         }
 496         else if (typeHnd == m_simdHandleCache->Vector128ULongHandle)
 497         {
 498             simdBaseType = TYP_ULONG;
 499             size         = Vector128SizeBytes;
 500             JITDUMP("  Known type Vector128<ulong>\n");
 501         }
 502         else
 503 #if defined(_TARGET_ARM64_)
 504             if (typeHnd == m_simdHandleCache->Vector64FloatHandle)
 505         {
 506             simdBaseType = TYP_FLOAT;
 507             size         = Vector64SizeBytes;
 508             JITDUMP("  Known type Vector64<float>\n");
 509         }
 510         else if (typeHnd == m_simdHandleCache->Vector64IntHandle)
 511         {
 512             simdBaseType = TYP_INT;
 513             size         = Vector64SizeBytes;
 514             JITDUMP("  Known type Vector64<int>\n");
 515         }
 516         else if (typeHnd == m_simdHandleCache->Vector64UIntHandle)
 517         {
 518             simdBaseType = TYP_UINT;
 519             size         = Vector64SizeBytes;
 520             JITDUMP("  Known type Vector64<uint>\n");
 521         }
 522         else if (typeHnd == m_simdHandleCache->Vector64ShortHandle)
 523         {
 524             simdBaseType = TYP_SHORT;
 525             size         = Vector64SizeBytes;
 526             JITDUMP("  Known type Vector64<short>\n");
 527         }
 528         else if (typeHnd == m_simdHandleCache->Vector64UShortHandle)
 529         {
 530             simdBaseType = TYP_USHORT;
 531             size         = Vector64SizeBytes;
 532             JITDUMP("  Known type Vector64<ushort>\n");
 533         }
 534         else if (typeHnd == m_simdHandleCache->Vector64ByteHandle)
 535         {
 536             simdBaseType = TYP_BYTE;
 537             size         = Vector64SizeBytes;
 538             JITDUMP("  Known type Vector64<sbyte>\n");
 539         }
 540         else if (typeHnd == m_simdHandleCache->Vector64UByteHandle)
 541         {
 542             simdBaseType = TYP_UBYTE;
 543             size         = Vector64SizeBytes;
 544             JITDUMP("  Known type Vector64<byte>\n");
 545         }
 546 #endif // defined(_TARGET_ARM64_)
 547
 548         // slow path search
 549         if (simdBaseType == TYP_UNKNOWN)
 550         {
 551             // Doesn't match with any of the cached type handles.
 552             const char*          className   = getClassNameFromMetadata(typeHnd, nullptr);
 553             CORINFO_CLASS_HANDLE baseTypeHnd = getTypeInstantiationArgument(typeHnd, 0);
 554
 555             if (baseTypeHnd != nullptr)
 556             {
 557                 CorInfoType type = info.compCompHnd->getTypeForPrimitiveNumericClass(baseTypeHnd);
 558
 559                 JITDUMP("HW Intrinsic SIMD Candidate Type %s with Base Type %s\n", className,
 560                         getClassNameFromMetadata(baseTypeHnd, nullptr));
 561
 562 #if defined(_TARGET_XARCH_)
 563                 if (strcmp(className, "Vector256`1") == 0)
 564                 {
 565                     size = Vector256SizeBytes;
 566                     switch (type)
 567                     {
 568                         case CORINFO_TYPE_FLOAT:
 569                             m_simdHandleCache->Vector256FloatHandle = typeHnd;
 570                             simdBaseType                            = TYP_FLOAT;
 571                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<float>\n");
 572                             break;
 573                         case CORINFO_TYPE_DOUBLE:
 574                             m_simdHandleCache->Vector256DoubleHandle = typeHnd;
 575                             simdBaseType                             = TYP_DOUBLE;
 576                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<double>\n");
 577                             break;
 578                         case CORINFO_TYPE_INT:
 579                             m_simdHandleCache->Vector256IntHandle = typeHnd;
 580                             simdBaseType                          = TYP_INT;
 581                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<int>\n");
 582                             break;
 583                         case CORINFO_TYPE_UINT:
 584                             m_simdHandleCache->Vector256UIntHandle = typeHnd;
 585                             simdBaseType                           = TYP_UINT;
 586                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<uint>\n");
 587                             break;
 588                         case CORINFO_TYPE_SHORT:
 589                             m_simdHandleCache->Vector256ShortHandle = typeHnd;
 590                             simdBaseType                            = TYP_SHORT;
 591                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<short>\n");
 592                             break;
 593                         case CORINFO_TYPE_USHORT:
 594                             m_simdHandleCache->Vector256UShortHandle = typeHnd;
 595                             simdBaseType                             = TYP_USHORT;
 596                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<ushort>\n");
 597                             break;
 598                         case CORINFO_TYPE_LONG:
 599                             m_simdHandleCache->Vector256LongHandle = typeHnd;
 600                             simdBaseType                           = TYP_LONG;
 601                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<long>\n");
 602                             break;
 603                         case CORINFO_TYPE_ULONG:
 604                             m_simdHandleCache->Vector256ULongHandle = typeHnd;
 605                             simdBaseType                            = TYP_ULONG;
 606                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<ulong>\n");
 607                             break;
 608                         case CORINFO_TYPE_UBYTE:
 609                             m_simdHandleCache->Vector256UByteHandle = typeHnd;
 610                             simdBaseType                            = TYP_UBYTE;
 611                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<byte>\n");
 612                             break;
 613                         case CORINFO_TYPE_BYTE:
 614                             m_simdHandleCache->Vector256ByteHandle = typeHnd;
 615                             simdBaseType                           = TYP_BYTE;
 616                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<sbyte>\n");
 617                             break;
 618
 619                         default:
 620                             JITDUMP("  Unknown Hardware Intrinsic SIMD Type Vector256<T>\n");
 621                     }
 622                 }
 623                 else
 624 #endif // defined(_TARGET_XARCH_)
 625                     if (strcmp(className, "Vector128`1") == 0)
 626                 {
 627                     size = Vector128SizeBytes;
 628                     switch (type)
 629                     {
 630                         case CORINFO_TYPE_FLOAT:
 631                             m_simdHandleCache->Vector128FloatHandle = typeHnd;
 632                             simdBaseType                            = TYP_FLOAT;
 633                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<float>\n");
 634                             break;
 635                         case CORINFO_TYPE_DOUBLE:
 636                             m_simdHandleCache->Vector128DoubleHandle = typeHnd;
 637                             simdBaseType                             = TYP_DOUBLE;
 638                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<double>\n");
 639                             break;
 640                         case CORINFO_TYPE_INT:
 641                             m_simdHandleCache->Vector128IntHandle = typeHnd;
 642                             simdBaseType                          = TYP_INT;
 643                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<int>\n");
 644                             break;
 645                         case CORINFO_TYPE_UINT:
 646                             m_simdHandleCache->Vector128UIntHandle = typeHnd;
 647                             simdBaseType                           = TYP_UINT;
 648                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<uint>\n");
 649                             break;
 650                         case CORINFO_TYPE_SHORT:
 651                             m_simdHandleCache->Vector128ShortHandle = typeHnd;
 652                             simdBaseType                            = TYP_SHORT;
 653                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<short>\n");
 654                             break;
 655                         case CORINFO_TYPE_USHORT:
 656                             m_simdHandleCache->Vector128UShortHandle = typeHnd;
 657                             simdBaseType                             = TYP_USHORT;
 658                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<ushort>\n");
 659                             break;
 660                         case CORINFO_TYPE_LONG:
 661                             m_simdHandleCache->Vector128LongHandle = typeHnd;
 662                             simdBaseType                           = TYP_LONG;
 663                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<long>\n");
 664                             break;
 665                         case CORINFO_TYPE_ULONG:
 666                             m_simdHandleCache->Vector128ULongHandle = typeHnd;
 667                             simdBaseType                            = TYP_ULONG;
 668                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<ulong>\n");
 669                             break;
 670                         case CORINFO_TYPE_UBYTE:
 671                             m_simdHandleCache->Vector128UByteHandle = typeHnd;
 672                             simdBaseType                            = TYP_UBYTE;
 673                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<byte>\n");
 674                             break;
 675                         case CORINFO_TYPE_BYTE:
 676                             m_simdHandleCache->Vector128ByteHandle = typeHnd;
 677                             simdBaseType                           = TYP_BYTE;
 678                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<sbyte>\n");
 679                             break;
 680
 681                         default:
 682                             JITDUMP("  Unknown Hardware Intrinsic SIMD Type Vector128<T>\n");
 683                     }
 684                 }
 685 #if defined(_TARGET_ARM64_)
 686                 else if (strcmp(className, "Vector64`1") == 0)
 687                 {
 688                     size = Vector64SizeBytes;
 689                     switch (type)
 690                     {
 691                         case CORINFO_TYPE_FLOAT:
 692                             m_simdHandleCache->Vector64FloatHandle = typeHnd;
 693                             simdBaseType                           = TYP_FLOAT;
 694                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<float>\n");
 695                             break;
 696                         case CORINFO_TYPE_INT:
 697                             m_simdHandleCache->Vector64IntHandle = typeHnd;
 698                             simdBaseType                         = TYP_INT;
 699                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<int>\n");
 700                             break;
 701                         case CORINFO_TYPE_UINT:
 702                             m_simdHandleCache->Vector64UIntHandle = typeHnd;
 703                             simdBaseType                          = TYP_UINT;
 704                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<uint>\n");
 705                             break;
 706                         case CORINFO_TYPE_SHORT:
 707                             m_simdHandleCache->Vector64ShortHandle = typeHnd;
 708                             simdBaseType                           = TYP_SHORT;
 709                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<short>\n");
 710                             break;
 711                         case CORINFO_TYPE_USHORT:
 712                             m_simdHandleCache->Vector64UShortHandle = typeHnd;
 713                             simdBaseType                            = TYP_USHORT;
 714                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<ushort>\n");
 715                             break;
 716                         case CORINFO_TYPE_UBYTE:
 717                             m_simdHandleCache->Vector64UByteHandle = typeHnd;
 718                             simdBaseType                           = TYP_UBYTE;
 719                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<byte>\n");
 720                             break;
 721                         case CORINFO_TYPE_BYTE:
 722                             m_simdHandleCache->Vector64ByteHandle = typeHnd;
 723                             simdBaseType                          = TYP_BYTE;
 724                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<sbyte>\n");
 725                             break;
 726
 727                         default:
 728                             JITDUMP("  Unknown Hardware Intrinsic SIMD Type Vector64<T>\n");
 729                     }
 730                 }
 731 #endif // defined(_TARGET_ARM64_)
 732             }
 733         }
 734
 735         if (sizeBytes != nullptr)
 736         {
 737             *sizeBytes = size;
 738         }
 739
 740         if (simdBaseType != TYP_UNKNOWN)
 741         {
 742             setUsesSIMDTypes(true);
 743         }
 744     }
 745 #endif // FEATURE_HW_INTRINSICS
 746
 747     return simdBaseType;
 748 }
 749
 750 //--------------------------------------------------------------------------------------
 751 // getSIMDIntrinsicInfo: get SIMD intrinsic info given the method handle.
 752 //
 753 // Arguments:
 754 //    inOutTypeHnd    - The handle of the type on which the method is invoked.  This is an in-out param.
 755 //    methodHnd       - The handle of the method we're interested in.
 756 //    sig             - method signature info
 757 //    isNewObj        - whether this call represents a newboj constructor call
 758 //    argCount        - argument count - out pram
 759 //    baseType        - base type of the intrinsic - out param
 760 //    sizeBytes       - size of SIMD vector type on which the method is invoked - out param
 761 //
 762 // Return Value:
 763 //    SIMDIntrinsicInfo struct initialized corresponding to methodHnd.
 764 //    Sets SIMDIntrinsicInfo.id to SIMDIntrinsicInvalid if methodHnd doesn't correspond
 765 //    to any SIMD intrinsic.  Also, sets the out params inOutTypeHnd, argCount, baseType and
 766 //    sizeBytes.
 767 //
 768 //    Note that VectorMath class doesn't have a base type and first argument of the method
 769 //    determines the SIMD vector type on which intrinsic is invoked. In such a case inOutTypeHnd
 770 //    is modified by this routine.
 771 //
 772 // TODO-Throughput: The current implementation is based on method name string parsing.
 773 //         Although we now have type identification from the VM, the parsing of intrinsic names
 774 //         could be made more efficient.
 775 //
 776 const SIMDIntrinsicInfo* Compiler::getSIMDIntrinsicInfo(CORINFO_CLASS_HANDLE* inOutTypeHnd,
 777                                                         CORINFO_METHOD_HANDLE methodHnd,
 778                                                         CORINFO_SIG_INFO*     sig,
 779                                                         bool                  isNewObj,
 780                                                         unsigned*             argCount,
 781                                                         var_types*            baseType,
 782                                                         unsigned*             sizeBytes)
 783 {
 784     assert(featureSIMD);
 785     assert(baseType != nullptr);
 786     assert(sizeBytes != nullptr);
 787
 788     // get baseType and size of the type
 789     CORINFO_CLASS_HANDLE typeHnd = *inOutTypeHnd;
 790     *baseType                    = getBaseTypeAndSizeOfSIMDType(typeHnd, sizeBytes);
 791
 792     if (typeHnd == m_simdHandleCache->SIMDVectorHandle)
 793     {
 794         // All of the supported intrinsics on this static class take a first argument that's a vector,
 795         // which determines the baseType.
 796         // The exception is the IsHardwareAccelerated property, which is handled as a special case.
 797         assert(*baseType == TYP_UNKNOWN);
 798         if (sig->numArgs == 0)
 799         {
 800             const SIMDIntrinsicInfo* hwAccelIntrinsicInfo = &(simdIntrinsicInfoArray[SIMDIntrinsicHWAccel]);
 801             if ((strcmp(eeGetMethodName(methodHnd, nullptr), hwAccelIntrinsicInfo->methodName) == 0) &&
 802                 JITtype2varType(sig->retType) == hwAccelIntrinsicInfo->retType)
 803             {
 804                 // Sanity check
 805                 assert(hwAccelIntrinsicInfo->argCount == 0 && hwAccelIntrinsicInfo->isInstMethod == false);
 806                 return hwAccelIntrinsicInfo;
 807             }
 808             return nullptr;
 809         }
 810         else
 811         {
 812             typeHnd       = info.compCompHnd->getArgClass(sig, sig->args);
 813             *inOutTypeHnd = typeHnd;
 814             *baseType     = getBaseTypeAndSizeOfSIMDType(typeHnd, sizeBytes);
 815         }
 816     }
 817
 818     if (*baseType == TYP_UNKNOWN)
 819     {
 820         JITDUMP("NOT a SIMD Intrinsic: unsupported baseType\n");
 821         return nullptr;
 822     }
 823
 824     // account for implicit "this" arg
 825     *argCount = sig->numArgs;
 826     if (sig->hasThis())
 827     {
 828         *argCount += 1;
 829     }
 830
 831     // Get the Intrinsic Id by parsing method name.
 832     //
 833     // TODO-Throughput: replace sequential search by binary search by arranging entries
 834     // sorted by method name.
 835     SIMDIntrinsicID intrinsicId = SIMDIntrinsicInvalid;
 836     const char*     methodName  = eeGetMethodName(methodHnd, nullptr);
 837     for (int i = SIMDIntrinsicNone + 1; i < SIMDIntrinsicInvalid; ++i)
 838     {
 839         if (strcmp(methodName, simdIntrinsicInfoArray[i].methodName) == 0)
 840         {
 841             // Found an entry for the method; further check whether it is one of
 842             // the supported base types.
 843             bool found = false;
 844             for (int j = 0; j < SIMD_INTRINSIC_MAX_BASETYPE_COUNT; ++j)
 845             {
 846                 // Convention: if there are fewer base types supported than MAX_BASETYPE_COUNT,
 847                 // the end of the list is marked by TYP_UNDEF.
 848                 if (simdIntrinsicInfoArray[i].supportedBaseTypes[j] == TYP_UNDEF)
 849                 {
 850                     break;
 851                 }
 852
 853                 if (simdIntrinsicInfoArray[i].supportedBaseTypes[j] == *baseType)
 854                 {
 855                     found = true;
 856                     break;
 857                 }
 858             }
 859
 860             if (!found)
 861             {
 862                 continue;
 863             }
 864
 865             // Now, check the arguments.
 866             unsigned int fixedArgCnt    = simdIntrinsicInfoArray[i].argCount;
 867             unsigned int expectedArgCnt = fixedArgCnt;
 868
 869             // First handle SIMDIntrinsicInitN, where the arg count depends on the type.
 870             // The listed arg types include the vector and the first two init values, which is the expected number
 871             // for Vector2.  For other cases, we'll check their types here.
 872             if (*argCount > expectedArgCnt)
 873             {
 874                 if (i == SIMDIntrinsicInitN)
 875                 {
 876                     if (*argCount == 3 && typeHnd == m_simdHandleCache->SIMDVector2Handle)
 877                     {
 878                         expectedArgCnt = 3;
 879                     }
 880                     else if (*argCount == 4 && typeHnd == m_simdHandleCache->SIMDVector3Handle)
 881                     {
 882                         expectedArgCnt = 4;
 883                     }
 884                     else if (*argCount == 5 && typeHnd == m_simdHandleCache->SIMDVector4Handle)
 885                     {
 886                         expectedArgCnt = 5;
 887                     }
 888                 }
 889                 else if (i == SIMDIntrinsicInitFixed)
 890                 {
 891                     if (*argCount == 4 && typeHnd == m_simdHandleCache->SIMDVector4Handle)
 892                     {
 893                         expectedArgCnt = 4;
 894                     }
 895                 }
 896             }
 897             if (*argCount != expectedArgCnt)
 898             {
 899                 continue;
 900             }
 901
 902             // Validate the types of individual args passed are what is expected of.
 903             // If any of the types don't match with what is expected, don't consider
 904             // as an intrinsic.  This will make an older JIT with SIMD capabilities
 905             // resilient to breaking changes to SIMD managed API.
 906             //
 907             // Note that from IL type stack, args get popped in right to left order
 908             // whereas args get listed in method signatures in left to right order.
 909
 910             int stackIndex = (expectedArgCnt - 1);
 911
 912             // Track the arguments from the signature - we currently only use this to distinguish
 913             // integral and pointer types, both of which will by TYP_I_IMPL on the importer stack.
 914             CORINFO_ARG_LIST_HANDLE argLst = sig->args;
 915
 916             CORINFO_CLASS_HANDLE argClass;
 917             for (unsigned int argIndex = 0; found == true && argIndex < expectedArgCnt; argIndex++)
 918             {
 919                 bool isThisPtr = ((argIndex == 0) && sig->hasThis());
 920
 921                 // In case of "newobj SIMDVector<T>(T val)", thisPtr won't be present on type stack.
 922                 // We don't check anything in that case.
 923                 if (!isThisPtr || !isNewObj)
 924                 {
 925                     GenTree*  arg     = impStackTop(stackIndex).val;
 926                     var_types argType = arg->TypeGet();
 927
 928                     var_types expectedArgType;
 929                     if (argIndex < fixedArgCnt)
 930                     {
 931                         // Convention:
 932                         //   - intrinsicInfo.argType[i] == TYP_UNDEF - intrinsic doesn't have a valid arg at position i
 933                         //   - intrinsicInfo.argType[i] == TYP_UNKNOWN - arg type should be same as basetype
 934                         // Note that we pop the args off in reverse order.
 935                         expectedArgType = simdIntrinsicInfoArray[i].argType[argIndex];
 936                         assert(expectedArgType != TYP_UNDEF);
 937                         if (expectedArgType == TYP_UNKNOWN)
 938                         {
 939                             // The type of the argument will be genActualType(*baseType).
 940                             expectedArgType = genActualType(*baseType);
 941                             argType         = genActualType(argType);
 942                         }
 943                     }
 944                     else
 945                     {
 946                         expectedArgType = *baseType;
 947                     }
 948
 949                     if (!isThisPtr && argType == TYP_I_IMPL)
 950                     {
 951                         // The reference implementation has a constructor that takes a pointer.
 952                         // We don't want to recognize that one.  This requires us to look at the CorInfoType
 953                         // in order to distinguish a signature with a pointer argument from one with an
 954                         // integer argument of pointer size, both of which will be TYP_I_IMPL on the stack.
 955                         // TODO-Review: This seems quite fragile.  We should consider beefing up the checking
 956                         // here.
 957                         CorInfoType corType = strip(info.compCompHnd->getArgType(sig, argLst, &argClass));
 958                         if (corType == CORINFO_TYPE_PTR)
 959                         {
 960                             found = false;
 961                         }
 962                     }
 963
 964                     if (varTypeIsSIMD(argType))
 965                     {
 966                         argType = TYP_STRUCT;
 967                     }
 968                     if (argType != expectedArgType)
 969                     {
 970                         found = false;
 971                     }
 972                 }
 973                 if (argIndex != 0 || !sig->hasThis())
 974                 {
 975                     argLst = info.compCompHnd->getArgNext(argLst);
 976                 }
 977                 stackIndex--;
 978             }
 979
 980             // Cross check return type and static vs. instance is what we are expecting.
 981             // If not, don't consider it as an intrinsic.
 982             // Note that ret type of TYP_UNKNOWN means that it is not known apriori and must be same as baseType
 983             if (found)
 984             {
 985                 var_types expectedRetType = simdIntrinsicInfoArray[i].retType;
 986                 if (expectedRetType == TYP_UNKNOWN)
 987                 {
 988                     // JIT maps uint/ulong type vars to TYP_INT/TYP_LONG.
 989                     expectedRetType =
 990                         (*baseType == TYP_UINT || *baseType == TYP_ULONG) ? genActualType(*baseType) : *baseType;
 991                 }
 992
 993                 if (JITtype2varType(sig->retType) != expectedRetType ||
 994                     sig->hasThis() != simdIntrinsicInfoArray[i].isInstMethod)
 995                 {
 996                     found = false;
 997                 }
 998             }
 999
1000             if (found)
1001             {
1002                 intrinsicId = (SIMDIntrinsicID)i;
1003                 break;
1004             }
1005         }
1006     }
1007
1008     if (intrinsicId != SIMDIntrinsicInvalid)
1009     {
1010         JITDUMP("Method %s maps to SIMD intrinsic %s\n", methodName, simdIntrinsicNames[intrinsicId]);
1011         return &simdIntrinsicInfoArray[intrinsicId];
1012     }
1013     else
1014     {
1015         JITDUMP("Method %s is NOT a SIMD intrinsic\n", methodName);
1016     }
1017
1018     return nullptr;
1019 }
1020
1021 // Pops and returns GenTree node from importer's type stack.
1022 // Normalizes TYP_STRUCT value in case of GT_CALL, GT_RET_EXPR and arg nodes.
1023 //
1024 // Arguments:
1025 //    type         -  the type of value that the caller expects to be popped off the stack.
1026 //    expectAddr   -  if true indicates we are expecting type stack entry to be a TYP_BYREF.
1027 //    structHandle -  the class handle to use when normalizing if it is not the same as the stack entry class handle;
1028 //                    this can happen for certain scenarios, such as folding away a static cast, where we want the
1029 //                    value popped to have the type that would have been returned.
1030 //
1031 // Notes:
1032 //    If the popped value is a struct, and the expected type is a simd type, it will be set
1033 //    to that type, otherwise it will assert if the type being popped is not the expected type.
1034
1035 GenTree* Compiler::impSIMDPopStack(var_types type, bool expectAddr, CORINFO_CLASS_HANDLE structHandle)
1036 {
1037     StackEntry se   = impPopStack();
1038     typeInfo   ti   = se.seTypeInfo;
1039     GenTree*   tree = se.val;
1040
1041     // If expectAddr is true implies what we have on stack is address and we need
1042     // SIMD type struct that it points to.
1043     if (expectAddr)
1044     {
1045         assert(tree->TypeGet() == TYP_BYREF);
1046         if (tree->OperGet() == GT_ADDR)
1047         {
1048             tree = tree->gtGetOp1();
1049         }
1050         else
1051         {
1052             tree = gtNewOperNode(GT_IND, type, tree);
1053         }
1054     }
1055
1056     bool isParam = false;
1057
1058     // If we are popping a struct type it must have a matching handle if one is specified.
1059     // - If we have an existing 'OBJ' and 'structHandle' is specified, we will change its
1060     //   handle if it doesn't match.
1061     //   This can happen when we have a retyping of a vector that doesn't translate to any
1062     //   actual IR.
1063     // - (If it's not an OBJ and it's used in a parameter context where it is required,
1064     //   impNormStructVal will add one).
1065     //
1066     if (tree->OperGet() == GT_OBJ)
1067     {
1068         if ((structHandle != NO_CLASS_HANDLE) && (tree->AsObj()->gtClass != structHandle))
1069         {
1070             // In this case we need to retain the GT_OBJ to retype the value.
1071             tree->AsObj()->gtClass = structHandle;
1072         }
1073         else
1074         {
1075             GenTree* addr = tree->gtOp.gtOp1;
1076             if ((addr->OperGet() == GT_ADDR) && isSIMDTypeLocal(addr->gtOp.gtOp1))
1077             {
1078                 tree = addr->gtOp.gtOp1;
1079             }
1080         }
1081     }
1082
1083     if (tree->OperGet() == GT_LCL_VAR)
1084     {
1085         unsigned   lclNum    = tree->AsLclVarCommon()->GetLclNum();
1086         LclVarDsc* lclVarDsc = &lvaTable[lclNum];
1087         isParam              = lclVarDsc->lvIsParam;
1088     }
1089
1090     // normalize TYP_STRUCT value
1091     if (varTypeIsStruct(tree) && ((tree->OperGet() == GT_RET_EXPR) || (tree->OperGet() == GT_CALL) || isParam))
1092     {
1093         assert(ti.IsType(TI_STRUCT));
1094
1095         if (structHandle == nullptr)
1096         {
1097             structHandle = ti.GetClassHandleForValueClass();
1098         }
1099
1100         tree = impNormStructVal(tree, structHandle, (unsigned)CHECK_SPILL_ALL);
1101     }
1102
1103     // Now set the type of the tree to the specialized SIMD struct type, if applicable.
1104     if (genActualType(tree->gtType) != genActualType(type))
1105     {
1106         assert(tree->gtType == TYP_STRUCT);
1107         tree->gtType = type;
1108     }
1109     else if (tree->gtType == TYP_BYREF)
1110     {
1111         assert(tree->IsLocal() || (tree->OperGet() == GT_RET_EXPR) || (tree->OperGet() == GT_CALL) ||
1112                ((tree->gtOper == GT_ADDR) && varTypeIsSIMD(tree->gtGetOp1())));
1113     }
1114
1115     return tree;
1116 }
1117
1118 // impSIMDGetFixed: Create a GT_SIMD tree for a Get property of SIMD vector with a fixed index.
1119 //
1120 // Arguments:
1121 //    baseType - The base (element) type of the SIMD vector.
1122 //    simdSize - The total size in bytes of the SIMD vector.
1123 //    index    - The index of the field to get.
1124 //
1125 // Return Value:
1126 //    Returns a GT_SIMD node with the SIMDIntrinsicGetItem intrinsic id.
1127 //
1128 GenTreeSIMD* Compiler::impSIMDGetFixed(var_types simdType, var_types baseType, unsigned simdSize, int index)
1129 {
1130     assert(simdSize >= ((index + 1) * genTypeSize(baseType)));
1131
1132     // op1 is a SIMD source.
1133     GenTree* op1 = impSIMDPopStack(simdType, true);
1134
1135     GenTree*     op2      = gtNewIconNode(index);
1136     GenTreeSIMD* simdTree = gtNewSIMDNode(baseType, op1, op2, SIMDIntrinsicGetItem, baseType, simdSize);
1137     return simdTree;
1138 }
1139
1140 #ifdef _TARGET_XARCH_
1141 // impSIMDLongRelOpEqual: transforms operands and returns the SIMD intrinsic to be applied on
1142 // transformed operands to obtain == comparison result.
1143 //
1144 // Arguments:
1145 //    typeHnd  -  type handle of SIMD vector
1146 //    size     -  SIMD vector size
1147 //    op1      -  in-out parameter; first operand
1148 //    op2      -  in-out parameter; second operand
1149 //
1150 // Return Value:
1151 //    Modifies in-out params op1, op2 and returns intrinsic ID to be applied to modified operands
1152 //
1153 SIMDIntrinsicID Compiler::impSIMDLongRelOpEqual(CORINFO_CLASS_HANDLE typeHnd,
1154                                                 unsigned             size,
1155                                                 GenTree**            pOp1,
1156                                                 GenTree**            pOp2)
1157 {
1158     var_types simdType = (*pOp1)->TypeGet();
1159     assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1160
1161     // There is no direct SSE2 support for comparing TYP_LONG vectors.
1162     // These have to be implemented in terms of TYP_INT vector comparison operations.
1163     //
1164     // Equality(v1, v2):
1165     // tmp = (v1 == v2) i.e. compare for equality as if v1 and v2 are vector<int>
1166     // result = BitwiseAnd(t, shuffle(t, (2, 3, 0, 1)))
1167     // Shuffle is meant to swap the comparison results of low-32-bits and high 32-bits of respective long elements.
1168
1169     // Compare vector<long> as if they were vector<int> and assign the result to a temp
1170     GenTree* compResult = gtNewSIMDNode(simdType, *pOp1, *pOp2, SIMDIntrinsicEqual, TYP_INT, size);
1171     unsigned lclNum     = lvaGrabTemp(true DEBUGARG("SIMD Long =="));
1172     lvaSetStruct(lclNum, typeHnd, false);
1173     GenTree* tmp = gtNewLclvNode(lclNum, simdType);
1174     GenTree* asg = gtNewTempAssign(lclNum, compResult);
1175
1176     // op1 = GT_COMMA(tmp=compResult, tmp)
1177     // op2 = Shuffle(tmp, 0xB1)
1178     // IntrinsicId = BitwiseAnd
1179     *pOp1 = gtNewOperNode(GT_COMMA, simdType, asg, tmp);
1180     *pOp2 = gtNewSIMDNode(simdType, gtNewLclvNode(lclNum, simdType), gtNewIconNode(SHUFFLE_ZWXY, TYP_INT),
1181                           SIMDIntrinsicShuffleSSE2, TYP_INT, size);
1182     return SIMDIntrinsicBitwiseAnd;
1183 }
1184
1185 // impSIMDLongRelOpGreaterThan: transforms operands and returns the SIMD intrinsic to be applied on
1186 // transformed operands to obtain > comparison result.
1187 //
1188 // Arguments:
1189 //    typeHnd  -  type handle of SIMD vector
1190 //    size     -  SIMD vector size
1191 //    pOp1     -  in-out parameter; first operand
1192 //    pOp2     -  in-out parameter; second operand
1193 //
1194 // Return Value:
1195 //    Modifies in-out params pOp1, pOp2 and returns intrinsic ID to be applied to modified operands
1196 //
1197 SIMDIntrinsicID Compiler::impSIMDLongRelOpGreaterThan(CORINFO_CLASS_HANDLE typeHnd,
1198                                                       unsigned             size,
1199                                                       GenTree**            pOp1,
1200                                                       GenTree**            pOp2)
1201 {
1202     var_types simdType = (*pOp1)->TypeGet();
1203     assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1204
1205     // GreaterThan(v1, v2) where v1 and v2 are vector long.
1206     // Let us consider the case of single long element comparison.
1207     // say L1 = (x1, y1) and L2 = (x2, y2) where x1, y1, x2, and y2 are 32-bit integers that comprise the longs L1 and
1208     // L2.
1209     //
1210     // GreaterThan(L1, L2) can be expressed in terms of > relationship between 32-bit integers that comprise L1 and L2
1211     // as
1212     //                    =  (x1, y1) > (x2, y2)
1213     //                    =  (x1 > x2) || [(x1 == x2) && (y1 > y2)]   - eq (1)
1214     //
1215     // t = (v1 > v2)  32-bit signed comparison
1216     // u = (v1 == v2) 32-bit sized element equality
1217     // v = (v1 > v2)  32-bit unsigned comparison
1218     //
1219     // z = shuffle(t, (3, 3, 1, 1))  - This corresponds to (x1 > x2) in eq(1) above
1220     // t1 = Shuffle(v, (2, 2, 0, 0)) - This corresponds to (y1 > y2) in eq(1) above
1221     // u1 = Shuffle(u, (3, 3, 1, 1)) - This corresponds to (x1 == x2) in eq(1) above
1222     // w = And(t1, u1)               - This corresponds to [(x1 == x2) && (y1 > y2)] in eq(1) above
1223     // Result = BitwiseOr(z, w)
1224
1225     // Since op1 and op2 gets used multiple times, make sure side effects are computed.
1226     GenTree* dupOp1    = nullptr;
1227     GenTree* dupOp2    = nullptr;
1228     GenTree* dupDupOp1 = nullptr;
1229     GenTree* dupDupOp2 = nullptr;
1230
1231     if (((*pOp1)->gtFlags & GTF_SIDE_EFFECT) != 0)
1232     {
1233         dupOp1    = fgInsertCommaFormTemp(pOp1, typeHnd);
1234         dupDupOp1 = gtNewLclvNode(dupOp1->AsLclVarCommon()->GetLclNum(), simdType);
1235     }
1236     else
1237     {
1238         dupOp1    = gtCloneExpr(*pOp1);
1239         dupDupOp1 = gtCloneExpr(*pOp1);
1240     }
1241
1242     if (((*pOp2)->gtFlags & GTF_SIDE_EFFECT) != 0)
1243     {
1244         dupOp2    = fgInsertCommaFormTemp(pOp2, typeHnd);
1245         dupDupOp2 = gtNewLclvNode(dupOp2->AsLclVarCommon()->GetLclNum(), simdType);
1246     }
1247     else
1248     {
1249         dupOp2    = gtCloneExpr(*pOp2);
1250         dupDupOp2 = gtCloneExpr(*pOp2);
1251     }
1252
1253     assert(dupDupOp1 != nullptr && dupDupOp2 != nullptr);
1254     assert(dupOp1 != nullptr && dupOp2 != nullptr);
1255     assert(*pOp1 != nullptr && *pOp2 != nullptr);
1256
1257     // v1GreaterThanv2Signed - signed 32-bit comparison
1258     GenTree* v1GreaterThanv2Signed = gtNewSIMDNode(simdType, *pOp1, *pOp2, SIMDIntrinsicGreaterThan, TYP_INT, size);
1259
1260     // v1Equalsv2 - 32-bit equality
1261     GenTree* v1Equalsv2 = gtNewSIMDNode(simdType, dupOp1, dupOp2, SIMDIntrinsicEqual, TYP_INT, size);
1262
1263     // v1GreaterThanv2Unsigned - unsigned 32-bit comparison
1264     var_types       tempBaseType = TYP_UINT;
1265     SIMDIntrinsicID sid = impSIMDRelOp(SIMDIntrinsicGreaterThan, typeHnd, size, &tempBaseType, &dupDupOp1, &dupDupOp2);
1266     GenTree*        v1GreaterThanv2Unsigned = gtNewSIMDNode(simdType, dupDupOp1, dupDupOp2, sid, tempBaseType, size);
1267
1268     GenTree* z = gtNewSIMDNode(simdType, v1GreaterThanv2Signed, gtNewIconNode(SHUFFLE_WWYY, TYP_INT),
1269                                SIMDIntrinsicShuffleSSE2, TYP_FLOAT, size);
1270     GenTree* t1 = gtNewSIMDNode(simdType, v1GreaterThanv2Unsigned, gtNewIconNode(SHUFFLE_ZZXX, TYP_INT),
1271                                 SIMDIntrinsicShuffleSSE2, TYP_FLOAT, size);
1272     GenTree* u1 = gtNewSIMDNode(simdType, v1Equalsv2, gtNewIconNode(SHUFFLE_WWYY, TYP_INT), SIMDIntrinsicShuffleSSE2,
1273                                 TYP_FLOAT, size);
1274     GenTree* w = gtNewSIMDNode(simdType, u1, t1, SIMDIntrinsicBitwiseAnd, TYP_INT, size);
1275
1276     *pOp1 = z;
1277     *pOp2 = w;
1278     return SIMDIntrinsicBitwiseOr;
1279 }
1280
1281 // impSIMDLongRelOpGreaterThanOrEqual: transforms operands and returns the SIMD intrinsic to be applied on
1282 // transformed operands to obtain >= comparison result.
1283 //
1284 // Arguments:
1285 //    typeHnd  -  type handle of SIMD vector
1286 //    size     -  SIMD vector size
1287 //    pOp1      -  in-out parameter; first operand
1288 //    pOp2      -  in-out parameter; second operand
1289 //
1290 // Return Value:
1291 //    Modifies in-out params pOp1, pOp2 and returns intrinsic ID to be applied to modified operands
1292 //
1293 SIMDIntrinsicID Compiler::impSIMDLongRelOpGreaterThanOrEqual(CORINFO_CLASS_HANDLE typeHnd,
1294                                                              unsigned             size,
1295                                                              GenTree**            pOp1,
1296                                                              GenTree**            pOp2)
1297 {
1298     var_types simdType = (*pOp1)->TypeGet();
1299     assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1300
1301     // expand this to (a == b) | (a > b)
1302     GenTree* dupOp1 = nullptr;
1303     GenTree* dupOp2 = nullptr;
1304
1305     if (((*pOp1)->gtFlags & GTF_SIDE_EFFECT) != 0)
1306     {
1307         dupOp1 = fgInsertCommaFormTemp(pOp1, typeHnd);
1308     }
1309     else
1310     {
1311         dupOp1 = gtCloneExpr(*pOp1);
1312     }
1313
1314     if (((*pOp2)->gtFlags & GTF_SIDE_EFFECT) != 0)
1315     {
1316         dupOp2 = fgInsertCommaFormTemp(pOp2, typeHnd);
1317     }
1318     else
1319     {
1320         dupOp2 = gtCloneExpr(*pOp2);
1321     }
1322
1323     assert(dupOp1 != nullptr && dupOp2 != nullptr);
1324     assert(*pOp1 != nullptr && *pOp2 != nullptr);
1325
1326     // (a==b)
1327     SIMDIntrinsicID id = impSIMDLongRelOpEqual(typeHnd, size, pOp1, pOp2);
1328     *pOp1              = gtNewSIMDNode(simdType, *pOp1, *pOp2, id, TYP_LONG, size);
1329
1330     // (a > b)
1331     id    = impSIMDLongRelOpGreaterThan(typeHnd, size, &dupOp1, &dupOp2);
1332     *pOp2 = gtNewSIMDNode(simdType, dupOp1, dupOp2, id, TYP_LONG, size);
1333
1334     return SIMDIntrinsicBitwiseOr;
1335 }
1336
1337 // impSIMDInt32OrSmallIntRelOpGreaterThanOrEqual: transforms operands and returns the SIMD intrinsic to be applied on
1338 // transformed operands to obtain >= comparison result in case of integer base type vectors
1339 //
1340 // Arguments:
1341 //    typeHnd  -  type handle of SIMD vector
1342 //    size     -  SIMD vector size
1343 //    baseType -  base type of SIMD vector
1344 //    pOp1      -  in-out parameter; first operand
1345 //    pOp2      -  in-out parameter; second operand
1346 //
1347 // Return Value:
1348 //    Modifies in-out params pOp1, pOp2 and returns intrinsic ID to be applied to modified operands
1349 //
1350 SIMDIntrinsicID Compiler::impSIMDIntegralRelOpGreaterThanOrEqual(
1351     CORINFO_CLASS_HANDLE typeHnd, unsigned size, var_types baseType, GenTree** pOp1, GenTree** pOp2)
1352 {
1353     var_types simdType = (*pOp1)->TypeGet();
1354     assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1355
1356     // This routine should be used only for integer base type vectors
1357     assert(varTypeIsIntegral(baseType));
1358     if ((getSIMDSupportLevel() == SIMD_SSE2_Supported) && ((baseType == TYP_LONG) || baseType == TYP_UBYTE))
1359     {
1360         return impSIMDLongRelOpGreaterThanOrEqual(typeHnd, size, pOp1, pOp2);
1361     }
1362
1363     // expand this to (a == b) | (a > b)
1364     GenTree* dupOp1 = nullptr;
1365     GenTree* dupOp2 = nullptr;
1366
1367     if (((*pOp1)->gtFlags & GTF_SIDE_EFFECT) != 0)
1368     {
1369         dupOp1 = fgInsertCommaFormTemp(pOp1, typeHnd);
1370     }
1371     else
1372     {
1373         dupOp1 = gtCloneExpr(*pOp1);
1374     }
1375
1376     if (((*pOp2)->gtFlags & GTF_SIDE_EFFECT) != 0)
1377     {
1378         dupOp2 = fgInsertCommaFormTemp(pOp2, typeHnd);
1379     }
1380     else
1381     {
1382         dupOp2 = gtCloneExpr(*pOp2);
1383     }
1384
1385     assert(dupOp1 != nullptr && dupOp2 != nullptr);
1386     assert(*pOp1 != nullptr && *pOp2 != nullptr);
1387
1388     // (a==b)
1389     *pOp1 = gtNewSIMDNode(simdType, *pOp1, *pOp2, SIMDIntrinsicEqual, baseType, size);
1390
1391     // (a > b)
1392     *pOp2 = gtNewSIMDNode(simdType, dupOp1, dupOp2, SIMDIntrinsicGreaterThan, baseType, size);
1393
1394     return SIMDIntrinsicBitwiseOr;
1395 }
1396 #endif // _TARGET_XARCH_
1397
1398 // Transforms operands and returns the SIMD intrinsic to be applied on
1399 // transformed operands to obtain given relop result.
1400 //
1401 // Arguments:
1402 //    relOpIntrinsicId - Relational operator SIMD intrinsic
1403 //    typeHnd          - type handle of SIMD vector
1404 //    size             -  SIMD vector size
1405 //    inOutBaseType    - base type of SIMD vector
1406 //    pOp1             -  in-out parameter; first operand
1407 //    pOp2             -  in-out parameter; second operand
1408 //
1409 // Return Value:
1410 //    Modifies in-out params pOp1, pOp2, inOutBaseType and returns intrinsic ID to be applied to modified operands
1411 //
1412 SIMDIntrinsicID Compiler::impSIMDRelOp(SIMDIntrinsicID      relOpIntrinsicId,
1413                                        CORINFO_CLASS_HANDLE typeHnd,
1414                                        unsigned             size,
1415                                        var_types*           inOutBaseType,
1416                                        GenTree**            pOp1,
1417                                        GenTree**            pOp2)
1418 {
1419     var_types simdType = (*pOp1)->TypeGet();
1420     assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1421
1422     assert(isRelOpSIMDIntrinsic(relOpIntrinsicId));
1423
1424     SIMDIntrinsicID intrinsicID = relOpIntrinsicId;
1425 #ifdef _TARGET_XARCH_
1426     var_types baseType = *inOutBaseType;
1427
1428     if (varTypeIsFloating(baseType))
1429     {
1430         // SSE2/AVX doesn't support > and >= on vector float/double.
1431         // Therefore, we need to use < and <= with swapped operands
1432         if (relOpIntrinsicId == SIMDIntrinsicGreaterThan || relOpIntrinsicId == SIMDIntrinsicGreaterThanOrEqual)
1433         {
1434             GenTree* tmp = *pOp1;
1435             *pOp1        = *pOp2;
1436             *pOp2        = tmp;
1437
1438             intrinsicID =
1439                 (relOpIntrinsicId == SIMDIntrinsicGreaterThan) ? SIMDIntrinsicLessThan : SIMDIntrinsicLessThanOrEqual;
1440         }
1441     }
1442     else if (varTypeIsIntegral(baseType))
1443     {
1444         // SSE/AVX doesn't support < and <= on integer base type vectors.
1445         // Therefore, we need to use > and >= with swapped operands.
1446         if (intrinsicID == SIMDIntrinsicLessThan || intrinsicID == SIMDIntrinsicLessThanOrEqual)
1447         {
1448             GenTree* tmp = *pOp1;
1449             *pOp1        = *pOp2;
1450             *pOp2        = tmp;
1451
1452             intrinsicID = (relOpIntrinsicId == SIMDIntrinsicLessThan) ? SIMDIntrinsicGreaterThan
1453                                                                       : SIMDIntrinsicGreaterThanOrEqual;
1454         }
1455
1456         if ((getSIMDSupportLevel() == SIMD_SSE2_Supported) && baseType == TYP_LONG)
1457         {
1458             // There is no direct SSE2 support for comparing TYP_LONG vectors.
1459             // These have to be implemented interms of TYP_INT vector comparison operations.
1460             if (intrinsicID == SIMDIntrinsicEqual)
1461             {
1462                 intrinsicID = impSIMDLongRelOpEqual(typeHnd, size, pOp1, pOp2);
1463             }
1464             else if (intrinsicID == SIMDIntrinsicGreaterThan)
1465             {
1466                 intrinsicID = impSIMDLongRelOpGreaterThan(typeHnd, size, pOp1, pOp2);
1467             }
1468             else if (intrinsicID == SIMDIntrinsicGreaterThanOrEqual)
1469             {
1470                 intrinsicID = impSIMDLongRelOpGreaterThanOrEqual(typeHnd, size, pOp1, pOp2);
1471             }
1472             else
1473             {
1474                 unreached();
1475             }
1476         }
1477         // SSE2 and AVX direct support for signed comparison of int32, int16 and int8 types
1478         else if (!varTypeIsUnsigned(baseType))
1479         {
1480             if (intrinsicID == SIMDIntrinsicGreaterThanOrEqual)
1481             {
1482                 intrinsicID = impSIMDIntegralRelOpGreaterThanOrEqual(typeHnd, size, baseType, pOp1, pOp2);
1483             }
1484         }
1485         else // unsigned
1486         {
1487             // Vector<byte>, Vector<ushort>, Vector<uint> and Vector<ulong>:
1488             // SSE2 supports > for signed comparison. Therefore, to use it for
1489             // comparing unsigned numbers, we subtract a constant from both the
1490             // operands such that the result fits within the corresponding signed
1491             // type.  The resulting signed numbers are compared using SSE2 signed
1492             // comparison.
1493             //
1494             // Vector<byte>: constant to be subtracted is 2^7
1495             // Vector<ushort> constant to be subtracted is 2^15
1496             // Vector<uint> constant to be subtracted is 2^31
1497             // Vector<ulong> constant to be subtracted is 2^63
1498             //
1499             // We need to treat op1 and op2 as signed for comparison purpose after
1500             // the transformation.
1501             __int64 constVal = 0;
1502             switch (baseType)
1503             {
1504                 case TYP_UBYTE:
1505                     constVal       = 0x80808080;
1506                     *inOutBaseType = TYP_BYTE;
1507                     break;
1508                 case TYP_USHORT:
1509                     constVal       = 0x80008000;
1510                     *inOutBaseType = TYP_SHORT;
1511                     break;
1512                 case TYP_UINT:
1513                     constVal       = 0x80000000;
1514                     *inOutBaseType = TYP_INT;
1515                     break;
1516                 case TYP_ULONG:
1517                     constVal       = 0x8000000000000000LL;
1518                     *inOutBaseType = TYP_LONG;
1519                     break;
1520                 default:
1521                     unreached();
1522                     break;
1523             }
1524             assert(constVal != 0);
1525
1526             // This transformation is not required for equality.
1527             if (intrinsicID != SIMDIntrinsicEqual)
1528             {
1529                 // For constructing const vector use either long or int base type.
1530                 var_types tempBaseType;
1531                 GenTree*  initVal;
1532                 if (baseType == TYP_ULONG)
1533                 {
1534                     tempBaseType = TYP_LONG;
1535                     initVal      = gtNewLconNode(constVal);
1536                 }
1537                 else
1538                 {
1539                     tempBaseType = TYP_INT;
1540                     initVal      = gtNewIconNode((ssize_t)constVal);
1541                 }
1542                 initVal->gtType      = tempBaseType;
1543                 GenTree* constVector = gtNewSIMDNode(simdType, initVal, nullptr, SIMDIntrinsicInit, tempBaseType, size);
1544
1545                 // Assign constVector to a temp, since we intend to use it more than once
1546                 // TODO-CQ: We have quite a few such constant vectors constructed during
1547                 // the importation of SIMD intrinsics.  Make sure that we have a single
1548                 // temp per distinct constant per method.
1549                 GenTree* tmp = fgInsertCommaFormTemp(&constVector, typeHnd);
1550
1551                 // op1 = op1 - constVector
1552                 // op2 = op2 - constVector
1553                 *pOp1 = gtNewSIMDNode(simdType, *pOp1, constVector, SIMDIntrinsicSub, baseType, size);
1554                 *pOp2 = gtNewSIMDNode(simdType, *pOp2, tmp, SIMDIntrinsicSub, baseType, size);
1555             }
1556
1557             return impSIMDRelOp(intrinsicID, typeHnd, size, inOutBaseType, pOp1, pOp2);
1558         }
1559     }
1560 #elif defined(_TARGET_ARM64_)
1561     // TODO-ARM64-CQ handle comparisons against zero
1562
1563     // _TARGET_ARM64_ doesn't support < and <= on register register comparisons
1564     // Therefore, we need to use > and >= with swapped operands.
1565     if (intrinsicID == SIMDIntrinsicLessThan || intrinsicID == SIMDIntrinsicLessThanOrEqual)
1566     {
1567         GenTree* tmp = *pOp1;
1568         *pOp1        = *pOp2;
1569         *pOp2        = tmp;
1570
1571         intrinsicID =
1572             (intrinsicID == SIMDIntrinsicLessThan) ? SIMDIntrinsicGreaterThan : SIMDIntrinsicGreaterThanOrEqual;
1573     }
1574 #else  // !_TARGET_XARCH_
1575     assert(!"impSIMDRelOp() unimplemented on target arch");
1576     unreached();
1577 #endif // !_TARGET_XARCH_
1578
1579     return intrinsicID;
1580 }
1581
1582 //-------------------------------------------------------------------------
1583 // impSIMDAbs: creates GT_SIMD node to compute Abs value of a given vector.
1584 //
1585 // Arguments:
1586 //    typeHnd     -  type handle of SIMD vector
1587 //    baseType    -  base type of vector
1588 //    size        -  vector size in bytes
1589 //    op1         -  operand of Abs intrinsic
1590 //
1591 GenTree* Compiler::impSIMDAbs(CORINFO_CLASS_HANDLE typeHnd, var_types baseType, unsigned size, GenTree* op1)
1592 {
1593     assert(varTypeIsSIMD(op1));
1594
1595     var_types simdType = op1->TypeGet();
1596     GenTree*  retVal   = nullptr;
1597
1598 #ifdef _TARGET_XARCH_
1599     // When there is no direct support, Abs(v) could be computed
1600     // on integer vectors as follows:
1601     //     BitVector = v < vector.Zero
1602     //     result = ConditionalSelect(BitVector, vector.Zero - v, v)
1603
1604     bool useConditionalSelect = false;
1605     if (getSIMDSupportLevel() == SIMD_SSE2_Supported)
1606     {
1607         // SSE2 doesn't support abs on signed integer type vectors.
1608         if (baseType == TYP_LONG || baseType == TYP_INT || baseType == TYP_SHORT || baseType == TYP_BYTE)
1609         {
1610             useConditionalSelect = true;
1611         }
1612     }
1613     else
1614     {
1615         assert(getSIMDSupportLevel() >= SIMD_SSE4_Supported);
1616         if (baseType == TYP_LONG)
1617         {
1618             // SSE4/AVX2 don't support abs on long type vector.
1619             useConditionalSelect = true;
1620         }
1621     }
1622
1623     if (useConditionalSelect)
1624     {
1625         // This works only on integer vectors not on float/double vectors.
1626         assert(varTypeIsIntegral(baseType));
1627
1628         GenTree* op1Assign;
1629         unsigned op1LclNum;
1630
1631         if (op1->OperGet() == GT_LCL_VAR)
1632         {
1633             op1LclNum = op1->gtLclVarCommon.gtLclNum;
1634             op1Assign = nullptr;
1635         }
1636         else
1637         {
1638             op1LclNum = lvaGrabTemp(true DEBUGARG("SIMD Abs op1"));
1639             lvaSetStruct(op1LclNum, typeHnd, false);
1640             op1Assign = gtNewTempAssign(op1LclNum, op1);
1641             op1       = gtNewLclvNode(op1LclNum, op1->TypeGet());
1642         }
1643
1644         // Assign Vector.Zero to a temp since it is needed more than once
1645         GenTree* vecZero       = gtNewSIMDVectorZero(simdType, baseType, size);
1646         unsigned vecZeroLclNum = lvaGrabTemp(true DEBUGARG("SIMD Abs VecZero"));
1647         lvaSetStruct(vecZeroLclNum, typeHnd, false);
1648         GenTree* vecZeroAssign = gtNewTempAssign(vecZeroLclNum, vecZero);
1649
1650         // Construct BitVector = v < vector.Zero
1651         GenTree*        bitVecOp1     = op1;
1652         GenTree*        bitVecOp2     = gtNewLclvNode(vecZeroLclNum, vecZero->TypeGet());
1653         var_types       relOpBaseType = baseType;
1654         SIMDIntrinsicID relOpIntrinsic =
1655             impSIMDRelOp(SIMDIntrinsicLessThan, typeHnd, size, &relOpBaseType, &bitVecOp1, &bitVecOp2);
1656         GenTree* bitVec       = gtNewSIMDNode(simdType, bitVecOp1, bitVecOp2, relOpIntrinsic, relOpBaseType, size);
1657         unsigned bitVecLclNum = lvaGrabTemp(true DEBUGARG("SIMD Abs bitVec"));
1658         lvaSetStruct(bitVecLclNum, typeHnd, false);
1659         GenTree* bitVecAssign = gtNewTempAssign(bitVecLclNum, bitVec);
1660         bitVec                = gtNewLclvNode(bitVecLclNum, bitVec->TypeGet());
1661
1662         // Construct condSelectOp1 = vector.Zero - v
1663         GenTree* subOp1 = gtNewLclvNode(vecZeroLclNum, vecZero->TypeGet());
1664         GenTree* subOp2 = gtNewLclvNode(op1LclNum, op1->TypeGet());
1665         GenTree* negVec = gtNewSIMDNode(simdType, subOp1, subOp2, SIMDIntrinsicSub, baseType, size);
1666
1667         // Construct ConditionalSelect(bitVec, vector.Zero - v, v)
1668         GenTree* vec = gtNewLclvNode(op1LclNum, op1->TypeGet());
1669         retVal       = impSIMDSelect(typeHnd, baseType, size, bitVec, negVec, vec);
1670
1671         // Prepend bitVec assignment to retVal.
1672         // retVal = (tmp2 = v < tmp1), CondSelect(tmp2, tmp1 - v, v)
1673         retVal = gtNewOperNode(GT_COMMA, simdType, bitVecAssign, retVal);
1674
1675         // Prepend vecZero assignment to retVal.
1676         // retVal =  (tmp1 = vector.Zero), (tmp2 = v < tmp1), CondSelect(tmp2, tmp1 - v, v)
1677         retVal = gtNewOperNode(GT_COMMA, simdType, vecZeroAssign, retVal);
1678
1679         // If op1 was assigned to a temp, prepend that to retVal.
1680         if (op1Assign != nullptr)
1681         {
1682             // retVal = (v=op1), (tmp1 = vector.Zero), (tmp2 = v < tmp1), CondSelect(tmp2, tmp1 - v, v)
1683             retVal = gtNewOperNode(GT_COMMA, simdType, op1Assign, retVal);
1684         }
1685     }
1686     else if (varTypeIsFloating(baseType))
1687     {
1688         // Abs(vf) = vf & new SIMDVector<float>(0x7fffffff);
1689         // Abs(vd) = vf & new SIMDVector<double>(0x7fffffffffffffff);
1690         GenTree* bitMask = nullptr;
1691         if (baseType == TYP_FLOAT)
1692         {
1693             float f;
1694             static_assert_no_msg(sizeof(float) == sizeof(int));
1695             *((int*)&f) = 0x7fffffff;
1696             bitMask     = gtNewDconNode(f);
1697         }
1698         else if (baseType == TYP_DOUBLE)
1699         {
1700             double d;
1701             static_assert_no_msg(sizeof(double) == sizeof(__int64));
1702             *((__int64*)&d) = 0x7fffffffffffffffLL;
1703             bitMask         = gtNewDconNode(d);
1704         }
1705
1706         assert(bitMask != nullptr);
1707         bitMask->gtType        = baseType;
1708         GenTree* bitMaskVector = gtNewSIMDNode(simdType, bitMask, SIMDIntrinsicInit, baseType, size);
1709         retVal                 = gtNewSIMDNode(simdType, op1, bitMaskVector, SIMDIntrinsicBitwiseAnd, baseType, size);
1710     }
1711     else if (baseType == TYP_USHORT || baseType == TYP_UBYTE || baseType == TYP_UINT || baseType == TYP_ULONG)
1712     {
1713         // Abs is a no-op on unsigned integer type vectors
1714         retVal = op1;
1715     }
1716     else
1717     {
1718         assert(getSIMDSupportLevel() >= SIMD_SSE4_Supported);
1719         assert(baseType != TYP_LONG);
1720
1721         retVal = gtNewSIMDNode(simdType, op1, SIMDIntrinsicAbs, baseType, size);
1722     }
1723 #elif defined(_TARGET_ARM64_)
1724     if (varTypeIsUnsigned(baseType))
1725     {
1726         // Abs is a no-op on unsigned integer type vectors
1727         retVal = op1;
1728     }
1729     else
1730     {
1731         retVal = gtNewSIMDNode(simdType, op1, SIMDIntrinsicAbs, baseType, size);
1732     }
1733 #else  // !defined(_TARGET_XARCH)_ && !defined(_TARGET_ARM64_)
1734     assert(!"Abs intrinsic on non-xarch target not implemented");
1735 #endif // !_TARGET_XARCH_
1736
1737     return retVal;
1738 }
1739
1740 // Creates a GT_SIMD tree for Select operation
1741 //
1742 // Arguments:
1743 //    typeHnd          -  type handle of SIMD vector
1744 //    baseType         -  base type of SIMD vector
1745 //    size             -  SIMD vector size
1746 //    op1              -  first operand = Condition vector vc
1747 //    op2              -  second operand = va
1748 //    op3              -  third operand = vb
1749 //
1750 // Return Value:
1751 //    Returns GT_SIMD tree that computes Select(vc, va, vb)
1752 //
1753 GenTree* Compiler::impSIMDSelect(
1754     CORINFO_CLASS_HANDLE typeHnd, var_types baseType, unsigned size, GenTree* op1, GenTree* op2, GenTree* op3)
1755 {
1756     assert(varTypeIsSIMD(op1));
1757     var_types simdType = op1->TypeGet();
1758     assert(op2->TypeGet() == simdType);
1759     assert(op3->TypeGet() == simdType);
1760
1761     // TODO-ARM64-CQ Support generating select instruction for SIMD
1762
1763     // Select(BitVector vc, va, vb) = (va & vc) | (vb & !vc)
1764     // Select(op1, op2, op3)        = (op2 & op1) | (op3 & !op1)
1765     //                              = SIMDIntrinsicBitwiseOr(SIMDIntrinsicBitwiseAnd(op2, op1),
1766     //                                                       SIMDIntrinsicBitwiseAndNot(op3, op1))
1767     //
1768     // If Op1 has side effect, create an assignment to a temp
1769     GenTree* tmp = op1;
1770     GenTree* asg = nullptr;
1771     if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0)
1772     {
1773         unsigned lclNum = lvaGrabTemp(true DEBUGARG("SIMD Select"));
1774         lvaSetStruct(lclNum, typeHnd, false);
1775         tmp = gtNewLclvNode(lclNum, op1->TypeGet());
1776         asg = gtNewTempAssign(lclNum, op1);
1777     }
1778
1779     GenTree* andExpr = gtNewSIMDNode(simdType, op2, tmp, SIMDIntrinsicBitwiseAnd, baseType, size);
1780     GenTree* dupOp1  = gtCloneExpr(tmp);
1781     assert(dupOp1 != nullptr);
1782 #ifdef _TARGET_ARM64_
1783     // ARM64 implements SIMDIntrinsicBitwiseAndNot as Left & ~Right
1784     GenTree* andNotExpr = gtNewSIMDNode(simdType, op3, dupOp1, SIMDIntrinsicBitwiseAndNot, baseType, size);
1785 #else
1786     // XARCH implements SIMDIntrinsicBitwiseAndNot as ~Left & Right
1787     GenTree* andNotExpr = gtNewSIMDNode(simdType, dupOp1, op3, SIMDIntrinsicBitwiseAndNot, baseType, size);
1788 #endif
1789     GenTree* simdTree = gtNewSIMDNode(simdType, andExpr, andNotExpr, SIMDIntrinsicBitwiseOr, baseType, size);
1790
1791     // If asg not null, create a GT_COMMA tree.
1792     if (asg != nullptr)
1793     {
1794         simdTree = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), asg, simdTree);
1795     }
1796
1797     return simdTree;
1798 }
1799
1800 // Creates a GT_SIMD tree for Min/Max operation
1801 //
1802 // Arguments:
1803 //    IntrinsicId      -  SIMD intrinsic Id, either Min or Max
1804 //    typeHnd          -  type handle of SIMD vector
1805 //    baseType         -  base type of SIMD vector
1806 //    size             -  SIMD vector size
1807 //    op1              -  first operand = va
1808 //    op2              -  second operand = vb
1809 //
1810 // Return Value:
1811 //    Returns GT_SIMD tree that computes Max(va, vb)
1812 //
1813 GenTree* Compiler::impSIMDMinMax(SIMDIntrinsicID      intrinsicId,
1814                                  CORINFO_CLASS_HANDLE typeHnd,
1815                                  var_types            baseType,
1816                                  unsigned             size,
1817                                  GenTree*             op1,
1818                                  GenTree*             op2)
1819 {
1820     assert(intrinsicId == SIMDIntrinsicMin || intrinsicId == SIMDIntrinsicMax);
1821     assert(varTypeIsSIMD(op1));
1822     var_types simdType = op1->TypeGet();
1823     assert(op2->TypeGet() == simdType);
1824
1825 #if defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
1826     GenTree* simdTree = nullptr;
1827
1828 #ifdef _TARGET_XARCH_
1829     // SSE2 has direct support for float/double/signed word/unsigned byte.
1830     // SSE4.1 has direct support for int32/uint32/signed byte/unsigned word.
1831     // For other integer types we compute min/max as follows
1832     //
1833     // int32/uint32 (SSE2)
1834     // int64/uint64 (SSE2&SSE4):
1835     //       compResult        = (op1 < op2) in case of Min
1836     //                           (op1 > op2) in case of Max
1837     //       Min/Max(op1, op2) = Select(compResult, op1, op2)
1838     //
1839     // unsigned word (SSE2):
1840     //        op1 = op1 - 2^15  ; to make it fit within a signed word
1841     //        op2 = op2 - 2^15  ; to make it fit within a signed word
1842     //        result = SSE2 signed word Min/Max(op1, op2)
1843     //        result = result + 2^15  ; readjust it back
1844     //
1845     // signed byte (SSE2):
1846     //        op1 = op1 + 2^7  ; to make it unsigned
1847     //        op1 = op1 + 2^7  ; to make it unsigned
1848     //        result = SSE2 unsigned byte Min/Max(op1, op2)
1849     //        result = result - 2^15 ; readjust it back
1850
1851     if (varTypeIsFloating(baseType) || baseType == TYP_SHORT || baseType == TYP_UBYTE ||
1852         (getSIMDSupportLevel() >= SIMD_SSE4_Supported &&
1853          (baseType == TYP_BYTE || baseType == TYP_INT || baseType == TYP_UINT || baseType == TYP_USHORT)))
1854     {
1855         // SSE2 or SSE4.1 has direct support
1856         simdTree = gtNewSIMDNode(simdType, op1, op2, intrinsicId, baseType, size);
1857     }
1858     else if (baseType == TYP_USHORT || baseType == TYP_BYTE)
1859     {
1860         assert(getSIMDSupportLevel() == SIMD_SSE2_Supported);
1861         int             constVal;
1862         SIMDIntrinsicID operIntrinsic;
1863         SIMDIntrinsicID adjustIntrinsic;
1864         var_types       minMaxOperBaseType;
1865         if (baseType == TYP_USHORT)
1866         {
1867             constVal           = 0x80008000;
1868             operIntrinsic      = SIMDIntrinsicSub;
1869             adjustIntrinsic    = SIMDIntrinsicAdd;
1870             minMaxOperBaseType = TYP_SHORT;
1871         }
1872         else
1873         {
1874             assert(baseType == TYP_BYTE);
1875             constVal           = 0x80808080;
1876             operIntrinsic      = SIMDIntrinsicAdd;
1877             adjustIntrinsic    = SIMDIntrinsicSub;
1878             minMaxOperBaseType = TYP_UBYTE;
1879         }
1880
1881         GenTree* initVal     = gtNewIconNode(constVal);
1882         GenTree* constVector = gtNewSIMDNode(simdType, initVal, nullptr, SIMDIntrinsicInit, TYP_INT, size);
1883
1884         // Assign constVector to a temp, since we intend to use it more than once
1885         // TODO-CQ: We have quite a few such constant vectors constructed during
1886         // the importation of SIMD intrinsics.  Make sure that we have a single
1887         // temp per distinct constant per method.
1888         GenTree* tmp = fgInsertCommaFormTemp(&constVector, typeHnd);
1889
1890         // op1 = op1 - constVector
1891         // op2 = op2 - constVector
1892         op1 = gtNewSIMDNode(simdType, op1, constVector, operIntrinsic, baseType, size);
1893         op2 = gtNewSIMDNode(simdType, op2, tmp, operIntrinsic, baseType, size);
1894
1895         // compute min/max of op1 and op2 considering them as if minMaxOperBaseType
1896         simdTree = gtNewSIMDNode(simdType, op1, op2, intrinsicId, minMaxOperBaseType, size);
1897
1898         // re-adjust the value by adding or subtracting constVector
1899         tmp      = gtNewLclvNode(tmp->AsLclVarCommon()->GetLclNum(), tmp->TypeGet());
1900         simdTree = gtNewSIMDNode(simdType, simdTree, tmp, adjustIntrinsic, baseType, size);
1901     }
1902 #elif defined(_TARGET_ARM64_)
1903     // Arm64 has direct support for all types except int64/uint64
1904     // For which we compute min/max as follows
1905     //
1906     // int64/uint64
1907     //       compResult        = (op1 < op2) in case of Min
1908     //                           (op1 > op2) in case of Max
1909     //       Min/Max(op1, op2) = Select(compResult, op1, op2)
1910     if (baseType != TYP_ULONG && baseType != TYP_LONG)
1911     {
1912         simdTree = gtNewSIMDNode(simdType, op1, op2, intrinsicId, baseType, size);
1913     }
1914 #endif
1915     else
1916     {
1917         GenTree* dupOp1    = nullptr;
1918         GenTree* dupOp2    = nullptr;
1919         GenTree* op1Assign = nullptr;
1920         GenTree* op2Assign = nullptr;
1921         unsigned op1LclNum;
1922         unsigned op2LclNum;
1923
1924         if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0)
1925         {
1926             op1LclNum = lvaGrabTemp(true DEBUGARG("SIMD Min/Max"));
1927             dupOp1    = gtNewLclvNode(op1LclNum, op1->TypeGet());
1928             lvaSetStruct(op1LclNum, typeHnd, false);
1929             op1Assign = gtNewTempAssign(op1LclNum, op1);
1930             op1       = gtNewLclvNode(op1LclNum, op1->TypeGet());
1931         }
1932         else
1933         {
1934             dupOp1 = gtCloneExpr(op1);
1935         }
1936
1937         if ((op2->gtFlags & GTF_SIDE_EFFECT) != 0)
1938         {
1939             op2LclNum = lvaGrabTemp(true DEBUGARG("SIMD Min/Max"));
1940             dupOp2    = gtNewLclvNode(op2LclNum, op2->TypeGet());
1941             lvaSetStruct(op2LclNum, typeHnd, false);
1942             op2Assign = gtNewTempAssign(op2LclNum, op2);
1943             op2       = gtNewLclvNode(op2LclNum, op2->TypeGet());
1944         }
1945         else
1946         {
1947             dupOp2 = gtCloneExpr(op2);
1948         }
1949
1950         SIMDIntrinsicID relOpIntrinsic =
1951             (intrinsicId == SIMDIntrinsicMin) ? SIMDIntrinsicLessThan : SIMDIntrinsicGreaterThan;
1952         var_types relOpBaseType = baseType;
1953
1954         // compResult = op1 relOp op2
1955         // simdTree = Select(compResult, op1, op2);
1956         assert(dupOp1 != nullptr);
1957         assert(dupOp2 != nullptr);
1958         relOpIntrinsic            = impSIMDRelOp(relOpIntrinsic, typeHnd, size, &relOpBaseType, &dupOp1, &dupOp2);
1959         GenTree* compResult       = gtNewSIMDNode(simdType, dupOp1, dupOp2, relOpIntrinsic, relOpBaseType, size);
1960         unsigned compResultLclNum = lvaGrabTemp(true DEBUGARG("SIMD Min/Max"));
1961         lvaSetStruct(compResultLclNum, typeHnd, false);
1962         GenTree* compResultAssign = gtNewTempAssign(compResultLclNum, compResult);
1963         compResult                = gtNewLclvNode(compResultLclNum, compResult->TypeGet());
1964         simdTree                  = impSIMDSelect(typeHnd, baseType, size, compResult, op1, op2);
1965         simdTree                  = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), compResultAssign, simdTree);
1966
1967         // Now create comma trees if we have created assignments of op1/op2 to temps
1968         if (op2Assign != nullptr)
1969         {
1970             simdTree = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), op2Assign, simdTree);
1971         }
1972
1973         if (op1Assign != nullptr)
1974         {
1975             simdTree = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), op1Assign, simdTree);
1976         }
1977     }
1978
1979     assert(simdTree != nullptr);
1980     return simdTree;
1981 #else  // !(defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_))
1982     assert(!"impSIMDMinMax() unimplemented on target arch");
1983     unreached();
1984 #endif // !(defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_))
1985 }
1986
1987 //------------------------------------------------------------------------
1988 // getOp1ForConstructor: Get the op1 for a constructor call.
1989 //
1990 // Arguments:
1991 //    opcode     - the opcode being handled (needed to identify the CEE_NEWOBJ case)
1992 //    newobjThis - For CEE_NEWOBJ, this is the temp grabbed for the allocated uninitalized object.
1993 //    clsHnd    - The handle of the class of the method.
1994 //
1995 // Return Value:
1996 //    The tree node representing the object to be initialized with the constructor.
1997 //
1998 // Notes:
1999 //    This method handles the differences between the CEE_NEWOBJ and constructor cases.
2000 //
2001 GenTree* Compiler::getOp1ForConstructor(OPCODE opcode, GenTree* newobjThis, CORINFO_CLASS_HANDLE clsHnd)
2002 {
2003     GenTree* op1;
2004     if (opcode == CEE_NEWOBJ)
2005     {
2006         op1 = newobjThis;
2007         assert(newobjThis->gtOper == GT_ADDR && newobjThis->gtOp.gtOp1->gtOper == GT_LCL_VAR);
2008
2009         // push newobj result on type stack
2010         unsigned tmp = op1->gtOp.gtOp1->gtLclVarCommon.gtLclNum;
2011         impPushOnStack(gtNewLclvNode(tmp, lvaGetRealType(tmp)), verMakeTypeInfo(clsHnd).NormaliseForStack());
2012     }
2013     else
2014     {
2015         op1 = impSIMDPopStack(TYP_BYREF);
2016     }
2017     assert(op1->TypeGet() == TYP_BYREF);
2018     return op1;
2019 }
2020
2021 //-------------------------------------------------------------------
2022 // Set the flag that indicates that the lclVar referenced by this tree
2023 // is used in a SIMD intrinsic.
2024 // Arguments:
2025 //      tree - GenTree*
2026
2027 void Compiler::setLclRelatedToSIMDIntrinsic(GenTree* tree)
2028 {
2029     assert(tree->OperIsLocal());
2030     unsigned   lclNum                = tree->AsLclVarCommon()->GetLclNum();
2031     LclVarDsc* lclVarDsc             = &lvaTable[lclNum];
2032     lclVarDsc->lvUsedInSIMDIntrinsic = true;
2033 }
2034
2035 //-------------------------------------------------------------
2036 // Check if two field nodes reference at the same memory location.
2037 // Notice that this check is just based on pattern matching.
2038 // Arguments:
2039 //      op1 - GenTree*.
2040 //      op2 - GenTree*.
2041 // Return Value:
2042 //    If op1's parents node and op2's parents node are at the same location, return true. Otherwise, return false
2043
2044 bool areFieldsParentsLocatedSame(GenTree* op1, GenTree* op2)
2045 {
2046     assert(op1->OperGet() == GT_FIELD);
2047     assert(op2->OperGet() == GT_FIELD);
2048
2049     GenTree* op1ObjRef = op1->gtField.gtFldObj;
2050     GenTree* op2ObjRef = op2->gtField.gtFldObj;
2051     while (op1ObjRef != nullptr && op2ObjRef != nullptr)
2052     {
2053
2054         if (op1ObjRef->OperGet() != op2ObjRef->OperGet())
2055         {
2056             break;
2057         }
2058         else if (op1ObjRef->OperGet() == GT_ADDR)
2059         {
2060             op1ObjRef = op1ObjRef->gtOp.gtOp1;
2061             op2ObjRef = op2ObjRef->gtOp.gtOp1;
2062         }
2063
2064         if (op1ObjRef->OperIsLocal() && op2ObjRef->OperIsLocal() &&
2065             op1ObjRef->AsLclVarCommon()->GetLclNum() == op2ObjRef->AsLclVarCommon()->GetLclNum())
2066         {
2067             return true;
2068         }
2069         else if (op1ObjRef->OperGet() == GT_FIELD && op2ObjRef->OperGet() == GT_FIELD &&
2070                  op1ObjRef->gtField.gtFldHnd == op2ObjRef->gtField.gtFldHnd)
2071         {
2072             op1ObjRef = op1ObjRef->gtField.gtFldObj;
2073             op2ObjRef = op2ObjRef->gtField.gtFldObj;
2074             continue;
2075         }
2076         else
2077         {
2078             break;
2079         }
2080     }
2081
2082     return false;
2083 }
2084
2085 //----------------------------------------------------------------------
2086 // Check whether two field are contiguous
2087 // Arguments:
2088 //      first - GenTree*. The Type of the node should be TYP_FLOAT
2089 //      second - GenTree*. The Type of the node should be TYP_FLOAT
2090 // Return Value:
2091 //      if the first field is located before second field, and they are located contiguously,
2092 //      then return true. Otherwise, return false.
2093
2094 bool Compiler::areFieldsContiguous(GenTree* first, GenTree* second)
2095 {
2096     assert(first->OperGet() == GT_FIELD);
2097     assert(second->OperGet() == GT_FIELD);
2098     assert(first->gtType == TYP_FLOAT);
2099     assert(second->gtType == TYP_FLOAT);
2100
2101     var_types firstFieldType  = first->gtType;
2102     var_types secondFieldType = second->gtType;
2103
2104     unsigned firstFieldEndOffset = first->gtField.gtFldOffset + genTypeSize(firstFieldType);
2105     unsigned secondFieldOffset   = second->gtField.gtFldOffset;
2106     if (firstFieldEndOffset == secondFieldOffset && firstFieldType == secondFieldType &&
2107         areFieldsParentsLocatedSame(first, second))
2108     {
2109         return true;
2110     }
2111
2112     return false;
2113 }
2114
2115 //-------------------------------------------------------------------------------
2116 // Check whether two array element nodes are located contiguously or not.
2117 // Arguments:
2118 //      op1 - GenTree*.
2119 //      op2 - GenTree*.
2120 // Return Value:
2121 //      if the array element op1 is located before array element op2, and they are contiguous,
2122 //      then return true. Otherwise, return false.
2123 // TODO-CQ:
2124 //      Right this can only check array element with const number as index. In future,
2125 //      we should consider to allow this function to check the index using expression.
2126
2127 bool Compiler::areArrayElementsContiguous(GenTree* op1, GenTree* op2)
2128 {
2129     noway_assert(op1->gtOper == GT_INDEX);
2130     noway_assert(op2->gtOper == GT_INDEX);
2131     GenTreeIndex* op1Index = op1->AsIndex();
2132     GenTreeIndex* op2Index = op2->AsIndex();
2133
2134     GenTree* op1ArrayRef = op1Index->Arr();
2135     GenTree* op2ArrayRef = op2Index->Arr();
2136     assert(op1ArrayRef->TypeGet() == TYP_REF);
2137     assert(op2ArrayRef->TypeGet() == TYP_REF);
2138
2139     GenTree* op1IndexNode = op1Index->Index();
2140     GenTree* op2IndexNode = op2Index->Index();
2141     if ((op1IndexNode->OperGet() == GT_CNS_INT && op2IndexNode->OperGet() == GT_CNS_INT) &&
2142         op1IndexNode->gtIntCon.gtIconVal + 1 == op2IndexNode->gtIntCon.gtIconVal)
2143     {
2144         if (op1ArrayRef->OperGet() == GT_FIELD && op2ArrayRef->OperGet() == GT_FIELD &&
2145             areFieldsParentsLocatedSame(op1ArrayRef, op2ArrayRef))
2146         {
2147             return true;
2148         }
2149         else if (op1ArrayRef->OperIsLocal() && op2ArrayRef->OperIsLocal() &&
2150                  op1ArrayRef->AsLclVarCommon()->GetLclNum() == op2ArrayRef->AsLclVarCommon()->GetLclNum())
2151         {
2152             return true;
2153         }
2154     }
2155     return false;
2156 }
2157
2158 //-------------------------------------------------------------------------------
2159 // Check whether two argument nodes are contiguous or not.
2160 // Arguments:
2161 //      op1 - GenTree*.
2162 //      op2 - GenTree*.
2163 // Return Value:
2164 //      if the argument node op1 is located before argument node op2, and they are located contiguously,
2165 //      then return true. Otherwise, return false.
2166 // TODO-CQ:
2167 //      Right now this can only check field and array. In future we should add more cases.
2168 //
2169
2170 bool Compiler::areArgumentsContiguous(GenTree* op1, GenTree* op2)
2171 {
2172     if (op1->OperGet() == GT_INDEX && op2->OperGet() == GT_INDEX)
2173     {
2174         return areArrayElementsContiguous(op1, op2);
2175     }
2176     else if (op1->OperGet() == GT_FIELD && op2->OperGet() == GT_FIELD)
2177     {
2178         return areFieldsContiguous(op1, op2);
2179     }
2180     return false;
2181 }
2182
2183 //--------------------------------------------------------------------------------------------------------
2184 // createAddressNodeForSIMDInit: Generate the address node(GT_LEA) if we want to intialize vector2, vector3 or vector4
2185 // from first argument's address.
2186 //
2187 // Arguments:
2188 //      tree - GenTree*. This the tree node which is used to get the address for indir.
2189 //      simdsize - unsigned. This the simd vector size.
2190 //      arrayElementsCount - unsigned. This is used for generating the boundary check for array.
2191 //
2192 // Return value:
2193 //      return the address node.
2194 //
2195 // TODO-CQ:
2196 //      1. Currently just support for GT_FIELD and GT_INDEX, because we can only verify the GT_INDEX node or GT_Field
2197 //         are located contiguously or not. In future we should support more cases.
2198 //      2. Though it happens to just work fine front-end phases are not aware of GT_LEA node.  Therefore, convert these
2199 //         to use GT_ADDR.
2200 GenTree* Compiler::createAddressNodeForSIMDInit(GenTree* tree, unsigned simdSize)
2201 {
2202     assert(tree->OperGet() == GT_FIELD || tree->OperGet() == GT_INDEX);
2203     GenTree*  byrefNode  = nullptr;
2204     GenTree*  startIndex = nullptr;
2205     unsigned  offset     = 0;
2206     var_types baseType   = tree->gtType;
2207
2208     if (tree->OperGet() == GT_FIELD)
2209     {
2210         GenTree* objRef = tree->gtField.gtFldObj;
2211         if (objRef != nullptr && objRef->gtOper == GT_ADDR)
2212         {
2213             GenTree* obj = objRef->gtOp.gtOp1;
2214
2215             // If the field is directly from a struct, then in this case,
2216             // we should set this struct's lvUsedInSIMDIntrinsic as true,
2217             // so that this sturct won't be promoted.
2218             // e.g. s.x x is a field, and s is a struct, then we should set the s's lvUsedInSIMDIntrinsic as true.
2219             // so that s won't be promoted.
2220             // Notice that if we have a case like s1.s2.x. s1 s2 are struct, and x is a field, then it is possible that
2221             // s1 can be promoted, so that s2 can be promoted. The reason for that is if we don't allow s1 to be
2222             // promoted, then this will affect the other optimizations which are depend on s1's struct promotion.
2223             // TODO-CQ:
2224             //  In future, we should optimize this case so that if there is a nested field like s1.s2.x and s1.s2.x's
2225             //  address is used for initializing the vector, then s1 can be promoted but s2 can't.
2226             if (varTypeIsSIMD(obj) && obj->OperIsLocal())
2227             {
2228                 setLclRelatedToSIMDIntrinsic(obj);
2229             }
2230         }
2231
2232         byrefNode = gtCloneExpr(tree->gtField.gtFldObj);
2233         assert(byrefNode != nullptr);
2234         offset = tree->gtField.gtFldOffset;
2235     }
2236     else if (tree->OperGet() == GT_INDEX)
2237     {
2238
2239         GenTree* index = tree->AsIndex()->Index();
2240         assert(index->OperGet() == GT_CNS_INT);
2241
2242         GenTree* checkIndexExpr = nullptr;
2243         unsigned indexVal       = (unsigned)(index->gtIntCon.gtIconVal);
2244         offset                  = indexVal * genTypeSize(tree->TypeGet());
2245         GenTree* arrayRef       = tree->AsIndex()->Arr();
2246
2247         // Generate the boundary check exception.
2248         // The length for boundary check should be the maximum index number which should be
2249         // (first argument's index number) + (how many array arguments we have) - 1
2250         // = indexVal + arrayElementsCount - 1
2251         unsigned arrayElementsCount  = simdSize / genTypeSize(baseType);
2252         checkIndexExpr               = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, indexVal + arrayElementsCount - 1);
2253         GenTreeArrLen*    arrLen     = gtNewArrLen(TYP_INT, arrayRef, (int)OFFSETOF__CORINFO_Array__length);
2254         GenTreeBoundsChk* arrBndsChk = new (this, GT_ARR_BOUNDS_CHECK)
2255             GenTreeBoundsChk(GT_ARR_BOUNDS_CHECK, TYP_VOID, checkIndexExpr, arrLen, SCK_RNGCHK_FAIL);
2256
2257         offset += OFFSETOF__CORINFO_Array__data;
2258         byrefNode = gtNewOperNode(GT_COMMA, arrayRef->TypeGet(), arrBndsChk, gtCloneExpr(arrayRef));
2259     }
2260     else
2261     {
2262         unreached();
2263     }
2264     GenTree* address =
2265         new (this, GT_LEA) GenTreeAddrMode(TYP_BYREF, byrefNode, startIndex, genTypeSize(tree->TypeGet()), offset);
2266     return address;
2267 }
2268
2269 //-------------------------------------------------------------------------------
2270 // impMarkContiguousSIMDFieldAssignments: Try to identify if there are contiguous
2271 // assignments from SIMD field to memory. If there are, then mark the related
2272 // lclvar so that it won't be promoted.
2273 //
2274 // Arguments:
2275 //      stmt - GenTree*. Input statement node.
2276
2277 void Compiler::impMarkContiguousSIMDFieldAssignments(GenTree* stmt)
2278 {
2279     if (!featureSIMD || opts.OptimizationDisabled())
2280     {
2281         return;
2282     }
2283     GenTree* expr = stmt->gtStmt.gtStmtExpr;
2284     if (expr->OperGet() == GT_ASG && expr->TypeGet() == TYP_FLOAT)
2285     {
2286         GenTree*  curDst            = expr->gtOp.gtOp1;
2287         GenTree*  curSrc            = expr->gtOp.gtOp2;
2288         unsigned  index             = 0;
2289         var_types baseType          = TYP_UNKNOWN;
2290         unsigned  simdSize          = 0;
2291         GenTree*  srcSimdStructNode = getSIMDStructFromField(curSrc, &baseType, &index, &simdSize, true);
2292         if (srcSimdStructNode == nullptr || baseType != TYP_FLOAT)
2293         {
2294             fgPreviousCandidateSIMDFieldAsgStmt = nullptr;
2295         }
2296         else if (index == 0 && isSIMDTypeLocal(srcSimdStructNode))
2297         {
2298             fgPreviousCandidateSIMDFieldAsgStmt = stmt;
2299         }
2300         else if (fgPreviousCandidateSIMDFieldAsgStmt != nullptr)
2301         {
2302             assert(index > 0);
2303             GenTree* prevAsgExpr = fgPreviousCandidateSIMDFieldAsgStmt->gtStmt.gtStmtExpr;
2304             GenTree* prevDst     = prevAsgExpr->gtOp.gtOp1;
2305             GenTree* prevSrc     = prevAsgExpr->gtOp.gtOp2;
2306             if (!areArgumentsContiguous(prevDst, curDst) || !areArgumentsContiguous(prevSrc, curSrc))
2307             {
2308                 fgPreviousCandidateSIMDFieldAsgStmt = nullptr;
2309             }
2310             else
2311             {
2312                 if (index == (simdSize / genTypeSize(baseType) - 1))
2313                 {
2314                     // Successfully found the pattern, mark the lclvar as UsedInSIMDIntrinsic
2315                     if (srcSimdStructNode->OperIsLocal())
2316                     {
2317                         setLclRelatedToSIMDIntrinsic(srcSimdStructNode);
2318                     }
2319
2320                     if (curDst->OperGet() == GT_FIELD)
2321                     {
2322                         GenTree* objRef = curDst->gtField.gtFldObj;
2323                         if (objRef != nullptr && objRef->gtOper == GT_ADDR)
2324                         {
2325                             GenTree* obj = objRef->gtOp.gtOp1;
2326                             if (varTypeIsStruct(obj) && obj->OperIsLocal())
2327                             {
2328                                 setLclRelatedToSIMDIntrinsic(obj);
2329                             }
2330                         }
2331                     }
2332                 }
2333                 else
2334                 {
2335                     fgPreviousCandidateSIMDFieldAsgStmt = stmt;
2336                 }
2337             }
2338         }
2339     }
2340     else
2341     {
2342         fgPreviousCandidateSIMDFieldAsgStmt = nullptr;
2343     }
2344 }
2345
2346 //------------------------------------------------------------------------
2347 // impSIMDIntrinsic: Check method to see if it is a SIMD method
2348 //
2349 // Arguments:
2350 //    opcode     - the opcode being handled (needed to identify the CEE_NEWOBJ case)
2351 //    newobjThis - For CEE_NEWOBJ, this is the temp grabbed for the allocated uninitalized object.
2352 //    clsHnd     - The handle of the class of the method.
2353 //    method     - The handle of the method.
2354 //    sig        - The call signature for the method.
2355 //    memberRef  - The memberRef token for the method reference.
2356 //
2357 // Return Value:
2358 //    If clsHnd is a known SIMD type, and 'method' is one of the methods that are
2359 //    implemented as an intrinsic in the JIT, then return the tree that implements
2360 //    it.
2361 //
2362 GenTree* Compiler::impSIMDIntrinsic(OPCODE                opcode,
2363                                     GenTree*              newobjThis,
2364                                     CORINFO_CLASS_HANDLE  clsHnd,
2365                                     CORINFO_METHOD_HANDLE methodHnd,
2366                                     CORINFO_SIG_INFO*     sig,
2367                                     unsigned              methodFlags,
2368                                     int                   memberRef)
2369 {
2370     assert(featureSIMD);
2371
2372     // Exit early if we are not in one of the SIMD types.
2373     if (!isSIMDClass(clsHnd))
2374     {
2375         return nullptr;
2376     }
2377
2378 #ifdef FEATURE_CORECLR
2379     // For coreclr, we also exit early if the method is not a JIT Intrinsic (which requires the [Intrinsic] attribute).
2380     if ((methodFlags & CORINFO_FLG_JIT_INTRINSIC) == 0)
2381     {
2382         return nullptr;
2383     }
2384 #endif // FEATURE_CORECLR
2385
2386     // Get base type and intrinsic Id
2387     var_types                baseType = TYP_UNKNOWN;
2388     unsigned                 size     = 0;
2389     unsigned                 argCount = 0;
2390     const SIMDIntrinsicInfo* intrinsicInfo =
2391         getSIMDIntrinsicInfo(&clsHnd, methodHnd, sig, (opcode == CEE_NEWOBJ), &argCount, &baseType, &size);
2392     if (intrinsicInfo == nullptr || intrinsicInfo->id == SIMDIntrinsicInvalid)
2393     {
2394         return nullptr;
2395     }
2396
2397     SIMDIntrinsicID simdIntrinsicID = intrinsicInfo->id;
2398     var_types       simdType;
2399     if (baseType != TYP_UNKNOWN)
2400     {
2401         simdType = getSIMDTypeForSize(size);
2402     }
2403     else
2404     {
2405         assert(simdIntrinsicID == SIMDIntrinsicHWAccel);
2406         simdType = TYP_UNKNOWN;
2407     }
2408     bool      instMethod = intrinsicInfo->isInstMethod;
2409     var_types callType   = JITtype2varType(sig->retType);
2410     if (callType == TYP_STRUCT)
2411     {
2412         // Note that here we are assuming that, if the call returns a struct, that it is the same size as the
2413         // struct on which the method is declared. This is currently true for all methods on Vector types,
2414         // but if this ever changes, we will need to determine the callType from the signature.
2415         assert(info.compCompHnd->getClassSize(sig->retTypeClass) == genTypeSize(simdType));
2416         callType = simdType;
2417     }
2418
2419     GenTree* simdTree   = nullptr;
2420     GenTree* op1        = nullptr;
2421     GenTree* op2        = nullptr;
2422     GenTree* op3        = nullptr;
2423     GenTree* retVal     = nullptr;
2424     GenTree* copyBlkDst = nullptr;
2425     bool     doCopyBlk  = false;
2426
2427     switch (simdIntrinsicID)
2428     {
2429         case SIMDIntrinsicGetCount:
2430         {
2431             int            length       = getSIMDVectorLength(clsHnd);
2432             GenTreeIntCon* intConstTree = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, length);
2433             retVal                      = intConstTree;
2434
2435             intConstTree->gtFlags |= GTF_ICON_SIMD_COUNT;
2436         }
2437         break;
2438
2439         case SIMDIntrinsicGetZero:
2440             retVal = gtNewSIMDVectorZero(simdType, baseType, size);
2441             break;
2442
2443         case SIMDIntrinsicGetOne:
2444             retVal = gtNewSIMDVectorOne(simdType, baseType, size);
2445             break;
2446
2447         case SIMDIntrinsicGetAllOnes:
2448         {
2449             // Equivalent to (Vector<T>) new Vector<int>(0xffffffff);
2450             GenTree* initVal = gtNewIconNode(0xffffffff, TYP_INT);
2451             simdTree         = gtNewSIMDNode(simdType, initVal, nullptr, SIMDIntrinsicInit, TYP_INT, size);
2452             if (baseType != TYP_INT)
2453             {
2454                 // cast it to required baseType if different from TYP_INT
2455                 simdTree = gtNewSIMDNode(simdType, simdTree, nullptr, SIMDIntrinsicCast, baseType, size);
2456             }
2457             retVal = simdTree;
2458         }
2459         break;
2460
2461         case SIMDIntrinsicInit:
2462         case SIMDIntrinsicInitN:
2463         {
2464             // SIMDIntrinsicInit:
2465             //    op2 - the initializer value
2466             //    op1 - byref of vector
2467             //
2468             // SIMDIntrinsicInitN
2469             //    op2 - list of initializer values stitched into a list
2470             //    op1 - byref of vector
2471             bool initFromFirstArgIndir = false;
2472             if (simdIntrinsicID == SIMDIntrinsicInit)
2473             {
2474                 op2 = impSIMDPopStack(baseType);
2475             }
2476             else
2477             {
2478                 assert(simdIntrinsicID == SIMDIntrinsicInitN);
2479                 assert(baseType == TYP_FLOAT);
2480
2481                 unsigned initCount    = argCount - 1;
2482                 unsigned elementCount = getSIMDVectorLength(size, baseType);
2483                 noway_assert(initCount == elementCount);
2484
2485                 // Build a GT_LIST with the N values.
2486                 // We must maintain left-to-right order of the args, but we will pop
2487                 // them off in reverse order (the Nth arg was pushed onto the stack last).
2488
2489                 GenTree* list              = nullptr;
2490                 GenTree* firstArg          = nullptr;
2491                 GenTree* prevArg           = nullptr;
2492                 bool     areArgsContiguous = true;
2493                 for (unsigned i = 0; i < initCount; i++)
2494                 {
2495                     GenTree* nextArg = impSIMDPopStack(baseType);
2496                     if (areArgsContiguous)
2497                     {
2498                         GenTree* curArg = nextArg;
2499                         firstArg        = curArg;
2500
2501                         if (prevArg != nullptr)
2502                         {
2503                             // Recall that we are popping the args off the stack in reverse order.
2504                             areArgsContiguous = areArgumentsContiguous(curArg, prevArg);
2505                         }
2506                         prevArg = curArg;
2507                     }
2508
2509                     list = new (this, GT_LIST) GenTreeOp(GT_LIST, baseType, nextArg, list);
2510                 }
2511
2512                 if (areArgsContiguous && baseType == TYP_FLOAT)
2513                 {
2514                     // Since Vector2, Vector3 and Vector4's arguments type are only float,
2515                     // we intialize the vector from first argument address, only when
2516                     // the baseType is TYP_FLOAT and the arguments are located contiguously in memory
2517                     initFromFirstArgIndir = true;
2518                     GenTree*  op2Address  = createAddressNodeForSIMDInit(firstArg, size);
2519                     var_types simdType    = getSIMDTypeForSize(size);
2520                     op2                   = gtNewOperNode(GT_IND, simdType, op2Address);
2521                 }
2522                 else
2523                 {
2524                     op2 = list;
2525                 }
2526             }
2527
2528             op1 = getOp1ForConstructor(opcode, newobjThis, clsHnd);
2529
2530             assert(op1->TypeGet() == TYP_BYREF);
2531             assert(genActualType(op2->TypeGet()) == genActualType(baseType) || initFromFirstArgIndir);
2532
2533             // For integral base types of size less than TYP_INT, expand the initializer
2534             // to fill size of TYP_INT bytes.
2535             if (varTypeIsSmallInt(baseType))
2536             {
2537                 // This case should occur only for Init intrinsic.
2538                 assert(simdIntrinsicID == SIMDIntrinsicInit);
2539
2540                 unsigned baseSize = genTypeSize(baseType);
2541                 int      multiplier;
2542                 if (baseSize == 1)
2543                 {
2544                     multiplier = 0x01010101;
2545                 }
2546                 else
2547                 {
2548                     assert(baseSize == 2);
2549                     multiplier = 0x00010001;
2550                 }
2551
2552                 GenTree* t1 = nullptr;
2553                 if (baseType == TYP_BYTE)
2554                 {
2555                     // What we have is a signed byte initializer,
2556                     // which when loaded to a reg will get sign extended to TYP_INT.
2557                     // But what we need is the initializer without sign extended or
2558                     // rather zero extended to 32-bits.
2559                     t1 = gtNewOperNode(GT_AND, TYP_INT, op2, gtNewIconNode(0xff, TYP_INT));
2560                 }
2561                 else if (baseType == TYP_SHORT)
2562                 {
2563                     // What we have is a signed short initializer,
2564                     // which when loaded to a reg will get sign extended to TYP_INT.
2565                     // But what we need is the initializer without sign extended or
2566                     // rather zero extended to 32-bits.
2567                     t1 = gtNewOperNode(GT_AND, TYP_INT, op2, gtNewIconNode(0xffff, TYP_INT));
2568                 }
2569                 else
2570                 {
2571                     assert(baseType == TYP_UBYTE || baseType == TYP_USHORT);
2572                     t1 = gtNewCastNode(TYP_INT, op2, false, TYP_INT);
2573                 }
2574
2575                 assert(t1 != nullptr);
2576                 GenTree* t2 = gtNewIconNode(multiplier, TYP_INT);
2577                 op2         = gtNewOperNode(GT_MUL, TYP_INT, t1, t2);
2578
2579                 // Construct a vector of TYP_INT with the new initializer and cast it back to vector of baseType
2580                 simdTree = gtNewSIMDNode(simdType, op2, nullptr, simdIntrinsicID, TYP_INT, size);
2581                 simdTree = gtNewSIMDNode(simdType, simdTree, nullptr, SIMDIntrinsicCast, baseType, size);
2582             }
2583             else
2584             {
2585
2586                 if (initFromFirstArgIndir)
2587                 {
2588                     simdTree = op2;
2589                     if (op1->gtOp.gtOp1->OperIsLocal())
2590                     {
2591                         // label the dst struct's lclvar is used for SIMD intrinsic,
2592                         // so that this dst struct won't be promoted.
2593                         setLclRelatedToSIMDIntrinsic(op1->gtOp.gtOp1);
2594                     }
2595                 }
2596                 else
2597                 {
2598                     simdTree = gtNewSIMDNode(simdType, op2, nullptr, simdIntrinsicID, baseType, size);
2599                 }
2600             }
2601
2602             copyBlkDst = op1;
2603             doCopyBlk  = true;
2604         }
2605         break;
2606
2607         case SIMDIntrinsicInitArray:
2608         case SIMDIntrinsicInitArrayX:
2609         case SIMDIntrinsicCopyToArray:
2610         case SIMDIntrinsicCopyToArrayX:
2611         {
2612             // op3 - index into array in case of SIMDIntrinsicCopyToArrayX and SIMDIntrinsicInitArrayX
2613             // op2 - array itself
2614             // op1 - byref to vector struct
2615
2616             unsigned int vectorLength = getSIMDVectorLength(size, baseType);
2617             // (This constructor takes only the zero-based arrays.)
2618             // We will add one or two bounds checks:
2619             // 1. If we have an index, we must do a check on that first.
2620             //    We can't combine it with the index + vectorLength check because
2621             //    a. It might be negative, and b. It may need to raise a different exception
2622             //    (captured as SCK_ARG_RNG_EXCPN for CopyTo and SCK_RNGCHK_FAIL for Init).
2623             // 2. We need to generate a check (SCK_ARG_EXCPN for CopyTo and SCK_RNGCHK_FAIL for Init)
2624             //    for the last array element we will access.
2625             //    We'll either check against (vectorLength - 1) or (index + vectorLength - 1).
2626
2627             GenTree* checkIndexExpr = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, vectorLength - 1);
2628
2629             // Get the index into the array.  If it has been provided, it will be on the
2630             // top of the stack.  Otherwise, it is null.
2631             if (argCount == 3)
2632             {
2633                 op3 = impSIMDPopStack(TYP_INT);
2634                 if (op3->IsIntegralConst(0))
2635                 {
2636                     op3 = nullptr;
2637                 }
2638             }
2639             else
2640             {
2641                 // TODO-CQ: Here, or elsewhere, check for the pattern where op2 is a newly constructed array, and
2642                 // change this to the InitN form.
2643                 // op3 = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, 0);
2644                 op3 = nullptr;
2645             }
2646
2647             // Clone the array for use in the bounds check.
2648             op2 = impSIMDPopStack(TYP_REF);
2649             assert(op2->TypeGet() == TYP_REF);
2650             GenTree* arrayRefForArgChk = op2;
2651             GenTree* argRngChk         = nullptr;
2652             if ((arrayRefForArgChk->gtFlags & GTF_SIDE_EFFECT) != 0)
2653             {
2654                 op2 = fgInsertCommaFormTemp(&arrayRefForArgChk);
2655             }
2656             else
2657             {
2658                 op2 = gtCloneExpr(arrayRefForArgChk);
2659             }
2660             assert(op2 != nullptr);
2661
2662             if (op3 != nullptr)
2663             {
2664                 SpecialCodeKind op3CheckKind;
2665                 if (simdIntrinsicID == SIMDIntrinsicInitArrayX)
2666                 {
2667                     op3CheckKind = SCK_RNGCHK_FAIL;
2668                 }
2669                 else
2670                 {
2671                     assert(simdIntrinsicID == SIMDIntrinsicCopyToArrayX);
2672                     op3CheckKind = SCK_ARG_RNG_EXCPN;
2673                 }
2674                 // We need to use the original expression on this, which is the first check.
2675                 GenTree* arrayRefForArgRngChk = arrayRefForArgChk;
2676                 // Then we clone the clone we just made for the next check.
2677                 arrayRefForArgChk = gtCloneExpr(op2);
2678                 // We know we MUST have had a cloneable expression.
2679                 assert(arrayRefForArgChk != nullptr);
2680                 GenTree* index = op3;
2681                 if ((index->gtFlags & GTF_SIDE_EFFECT) != 0)
2682                 {
2683                     op3 = fgInsertCommaFormTemp(&index);
2684                 }
2685                 else
2686                 {
2687                     op3 = gtCloneExpr(index);
2688                 }
2689
2690                 GenTreeArrLen* arrLen =
2691                     gtNewArrLen(TYP_INT, arrayRefForArgRngChk, (int)OFFSETOF__CORINFO_Array__length);
2692                 argRngChk = new (this, GT_ARR_BOUNDS_CHECK)
2693                     GenTreeBoundsChk(GT_ARR_BOUNDS_CHECK, TYP_VOID, index, arrLen, op3CheckKind);
2694                 // Now, clone op3 to create another node for the argChk
2695                 GenTree* index2 = gtCloneExpr(op3);
2696                 assert(index != nullptr);
2697                 checkIndexExpr = gtNewOperNode(GT_ADD, TYP_INT, index2, checkIndexExpr);
2698             }
2699
2700             // Insert a bounds check for index + offset - 1.
2701             // This must be a "normal" array.
2702             SpecialCodeKind op2CheckKind;
2703             if (simdIntrinsicID == SIMDIntrinsicInitArray || simdIntrinsicID == SIMDIntrinsicInitArrayX)
2704             {
2705                 op2CheckKind = SCK_RNGCHK_FAIL;
2706             }
2707             else
2708             {
2709                 op2CheckKind = SCK_ARG_EXCPN;
2710             }
2711             GenTreeArrLen*    arrLen = gtNewArrLen(TYP_INT, arrayRefForArgChk, (int)OFFSETOF__CORINFO_Array__length);
2712             GenTreeBoundsChk* argChk = new (this, GT_ARR_BOUNDS_CHECK)
2713                 GenTreeBoundsChk(GT_ARR_BOUNDS_CHECK, TYP_VOID, checkIndexExpr, arrLen, op2CheckKind);
2714
2715             // Create a GT_COMMA tree for the bounds check(s).
2716             op2 = gtNewOperNode(GT_COMMA, op2->TypeGet(), argChk, op2);
2717             if (argRngChk != nullptr)
2718             {
2719                 op2 = gtNewOperNode(GT_COMMA, op2->TypeGet(), argRngChk, op2);
2720             }
2721
2722             if (simdIntrinsicID == SIMDIntrinsicInitArray || simdIntrinsicID == SIMDIntrinsicInitArrayX)
2723             {
2724                 op1        = getOp1ForConstructor(opcode, newobjThis, clsHnd);
2725                 simdTree   = gtNewSIMDNode(simdType, op2, op3, SIMDIntrinsicInitArray, baseType, size);
2726                 copyBlkDst = op1;
2727                 doCopyBlk  = true;
2728             }
2729             else
2730             {
2731                 assert(simdIntrinsicID == SIMDIntrinsicCopyToArray || simdIntrinsicID == SIMDIntrinsicCopyToArrayX);
2732                 op1 = impSIMDPopStack(simdType, instMethod);
2733                 assert(op1->TypeGet() == simdType);
2734
2735                 // copy vector (op1) to array (op2) starting at index (op3)
2736                 simdTree = op1;
2737
2738                 // TODO-Cleanup: Though it happens to just work fine front-end phases are not aware of GT_LEA node.
2739                 // Therefore, convert these to use GT_ADDR .
2740                 copyBlkDst = new (this, GT_LEA)
2741                     GenTreeAddrMode(TYP_BYREF, op2, op3, genTypeSize(baseType), OFFSETOF__CORINFO_Array__data);
2742                 doCopyBlk = true;
2743             }
2744         }
2745         break;
2746
2747         case SIMDIntrinsicInitFixed:
2748         {
2749             // We are initializing a fixed-length vector VLarge with a smaller fixed-length vector VSmall, plus 1 or 2
2750             // additional floats.
2751             //    op4 (optional) - float value for VLarge.W, if VLarge is Vector4, and VSmall is Vector2
2752             //    op3 - float value for VLarge.Z or VLarge.W
2753             //    op2 - VSmall
2754             //    op1 - byref of VLarge
2755             assert(baseType == TYP_FLOAT);
2756
2757             GenTree* op4 = nullptr;
2758             if (argCount == 4)
2759             {
2760                 op4 = impSIMDPopStack(TYP_FLOAT);
2761                 assert(op4->TypeGet() == TYP_FLOAT);
2762             }
2763             op3 = impSIMDPopStack(TYP_FLOAT);
2764             assert(op3->TypeGet() == TYP_FLOAT);
2765             // The input vector will either be TYP_SIMD8 or TYP_SIMD12.
2766             var_types smallSIMDType = TYP_SIMD8;
2767             if ((op4 == nullptr) && (simdType == TYP_SIMD16))
2768             {
2769                 smallSIMDType = TYP_SIMD12;
2770             }
2771             op2 = impSIMDPopStack(smallSIMDType);
2772             op1 = getOp1ForConstructor(opcode, newobjThis, clsHnd);
2773
2774             // We are going to redefine the operands so that:
2775             // - op3 is the value that's going into the Z position, or null if it's a Vector4 constructor with a single
2776             // operand, and
2777             // - op4 is the W position value, or null if this is a Vector3 constructor.
2778             if (size == 16 && argCount == 3)
2779             {
2780                 op4 = op3;
2781                 op3 = nullptr;
2782             }
2783
2784             simdTree = op2;
2785             if (op3 != nullptr)
2786             {
2787                 simdTree = gtNewSIMDNode(simdType, simdTree, op3, SIMDIntrinsicSetZ, baseType, size);
2788             }
2789             if (op4 != nullptr)
2790             {
2791                 simdTree = gtNewSIMDNode(simdType, simdTree, op4, SIMDIntrinsicSetW, baseType, size);
2792             }
2793
2794             copyBlkDst = op1;
2795             doCopyBlk  = true;
2796         }
2797         break;
2798
2799         case SIMDIntrinsicOpEquality:
2800         case SIMDIntrinsicInstEquals:
2801         {
2802             op2 = impSIMDPopStack(simdType);
2803             op1 = impSIMDPopStack(simdType, instMethod);
2804
2805             assert(op1->TypeGet() == simdType);
2806             assert(op2->TypeGet() == simdType);
2807
2808             simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, SIMDIntrinsicOpEquality, baseType, size);
2809             if (simdType == TYP_SIMD12)
2810             {
2811                 simdTree->gtFlags |= GTF_SIMD12_OP;
2812             }
2813             retVal = simdTree;
2814         }
2815         break;
2816
2817         case SIMDIntrinsicOpInEquality:
2818         {
2819             // op1 is the first operand
2820             // op2 is the second operand
2821             op2      = impSIMDPopStack(simdType);
2822             op1      = impSIMDPopStack(simdType, instMethod);
2823             simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, SIMDIntrinsicOpInEquality, baseType, size);
2824             if (simdType == TYP_SIMD12)
2825             {
2826                 simdTree->gtFlags |= GTF_SIMD12_OP;
2827             }
2828             retVal = simdTree;
2829         }
2830         break;
2831
2832         case SIMDIntrinsicEqual:
2833         case SIMDIntrinsicLessThan:
2834         case SIMDIntrinsicLessThanOrEqual:
2835         case SIMDIntrinsicGreaterThan:
2836         case SIMDIntrinsicGreaterThanOrEqual:
2837         {
2838             op2 = impSIMDPopStack(simdType);
2839             op1 = impSIMDPopStack(simdType, instMethod);
2840
2841             SIMDIntrinsicID intrinsicID = impSIMDRelOp(simdIntrinsicID, clsHnd, size, &baseType, &op1, &op2);
2842             simdTree                    = gtNewSIMDNode(genActualType(callType), op1, op2, intrinsicID, baseType, size);
2843             retVal                      = simdTree;
2844         }
2845         break;
2846
2847         case SIMDIntrinsicAdd:
2848         case SIMDIntrinsicSub:
2849         case SIMDIntrinsicMul:
2850         case SIMDIntrinsicDiv:
2851         case SIMDIntrinsicBitwiseAnd:
2852         case SIMDIntrinsicBitwiseAndNot:
2853         case SIMDIntrinsicBitwiseOr:
2854         case SIMDIntrinsicBitwiseXor:
2855         {
2856 #if defined(DEBUG)
2857             // check for the cases where we don't support intrinsics.
2858             // This check should be done before we make modifications to type stack.
2859             // Note that this is more of a double safety check for robustness since
2860             // we expect getSIMDIntrinsicInfo() to have filtered out intrinsics on
2861             // unsupported base types. If getSIMdIntrinsicInfo() doesn't filter due
2862             // to some bug, assert in chk/dbg will fire.
2863             if (!varTypeIsFloating(baseType))
2864             {
2865                 if (simdIntrinsicID == SIMDIntrinsicMul)
2866                 {
2867 #if defined(_TARGET_XARCH_)
2868                     if ((baseType != TYP_INT) && (baseType != TYP_SHORT))
2869                     {
2870                         // TODO-CQ: implement mul on these integer vectors.
2871                         // Note that SSE2 has no direct support for these vectors.
2872                         assert(!"Mul not supported on long/ulong/uint/small int vectors\n");
2873                         return nullptr;
2874                     }
2875 #endif // _TARGET_XARCH_
2876 #if defined(_TARGET_ARM64_)
2877                     if ((baseType == TYP_ULONG) && (baseType == TYP_LONG))
2878                     {
2879                         // TODO-CQ: implement mul on these integer vectors.
2880                         // Note that ARM64 has no direct support for these vectors.
2881                         assert(!"Mul not supported on long/ulong vectors\n");
2882                         return nullptr;
2883                     }
2884 #endif // _TARGET_ARM64_
2885                 }
2886 #if defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
2887                 // common to all integer type vectors
2888                 if (simdIntrinsicID == SIMDIntrinsicDiv)
2889                 {
2890                     // SSE2 doesn't support div on non-floating point vectors.
2891                     assert(!"Div not supported on integer type vectors\n");
2892                     return nullptr;
2893                 }
2894 #endif // defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
2895             }
2896 #endif // DEBUG
2897
2898             // op1 is the first operand; if instance method, op1 is "this" arg
2899             // op2 is the second operand
2900             op2 = impSIMDPopStack(simdType);
2901             op1 = impSIMDPopStack(simdType, instMethod);
2902
2903 #ifdef _TARGET_XARCH_
2904             if (simdIntrinsicID == SIMDIntrinsicBitwiseAndNot)
2905             {
2906                 // XARCH implements SIMDIntrinsicBitwiseAndNot as ~op1 & op2, while the
2907                 // software implementation does op1 & ~op2, so we need to swap the operands
2908
2909                 GenTree* tmp = op2;
2910                 op2          = op1;
2911                 op1          = tmp;
2912             }
2913 #endif // _TARGET_XARCH_
2914
2915             simdTree = gtNewSIMDNode(simdType, op1, op2, simdIntrinsicID, baseType, size);
2916             retVal   = simdTree;
2917         }
2918         break;
2919
2920         case SIMDIntrinsicSelect:
2921         {
2922             // op3 is a SIMD variable that is the second source
2923             // op2 is a SIMD variable that is the first source
2924             // op1 is a SIMD variable which is the bit mask.
2925             op3 = impSIMDPopStack(simdType);
2926             op2 = impSIMDPopStack(simdType);
2927             op1 = impSIMDPopStack(simdType);
2928
2929             retVal = impSIMDSelect(clsHnd, baseType, size, op1, op2, op3);
2930         }
2931         break;
2932
2933         case SIMDIntrinsicMin:
2934         case SIMDIntrinsicMax:
2935         {
2936             // op1 is the first operand; if instance method, op1 is "this" arg
2937             // op2 is the second operand
2938             op2 = impSIMDPopStack(simdType);
2939             op1 = impSIMDPopStack(simdType, instMethod);
2940
2941             retVal = impSIMDMinMax(simdIntrinsicID, clsHnd, baseType, size, op1, op2);
2942         }
2943         break;
2944
2945         case SIMDIntrinsicGetItem:
2946         {
2947             // op1 is a SIMD variable that is "this" arg
2948             // op2 is an index of TYP_INT
2949             op2              = impSIMDPopStack(TYP_INT);
2950             op1              = impSIMDPopStack(simdType, instMethod);
2951             int vectorLength = getSIMDVectorLength(size, baseType);
2952             if (!op2->IsCnsIntOrI() || op2->AsIntCon()->gtIconVal >= vectorLength || op2->AsIntCon()->gtIconVal < 0)
2953             {
2954                 // We need to bounds-check the length of the vector.
2955                 // For that purpose, we need to clone the index expression.
2956                 GenTree* index = op2;
2957                 if ((index->gtFlags & GTF_SIDE_EFFECT) != 0)
2958                 {
2959                     op2 = fgInsertCommaFormTemp(&index);
2960                 }
2961                 else
2962                 {
2963                     op2 = gtCloneExpr(index);
2964                 }
2965
2966                 GenTree*          lengthNode = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, vectorLength);
2967                 GenTreeBoundsChk* simdChk =
2968                     new (this, GT_SIMD_CHK) GenTreeBoundsChk(GT_SIMD_CHK, TYP_VOID, index, lengthNode, SCK_RNGCHK_FAIL);
2969
2970                 // Create a GT_COMMA tree for the bounds check.
2971                 op2 = gtNewOperNode(GT_COMMA, op2->TypeGet(), simdChk, op2);
2972             }
2973
2974             assert(op1->TypeGet() == simdType);
2975             assert(op2->TypeGet() == TYP_INT);
2976
2977             simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, simdIntrinsicID, baseType, size);
2978             retVal   = simdTree;
2979         }
2980         break;
2981
2982         case SIMDIntrinsicDotProduct:
2983         {
2984 #if defined(_TARGET_XARCH_)
2985             // Right now dot product is supported only for float/double vectors and
2986             // int vectors on SSE4/AVX.
2987             if (!varTypeIsFloating(baseType) && !(baseType == TYP_INT && getSIMDSupportLevel() >= SIMD_SSE4_Supported))
2988             {
2989                 return nullptr;
2990             }
2991 #endif // _TARGET_XARCH_
2992
2993             // op1 is a SIMD variable that is the first source and also "this" arg.
2994             // op2 is a SIMD variable which is the second source.
2995             op2 = impSIMDPopStack(simdType);
2996             op1 = impSIMDPopStack(simdType, instMethod);
2997
2998             simdTree = gtNewSIMDNode(baseType, op1, op2, simdIntrinsicID, baseType, size);
2999             if (simdType == TYP_SIMD12)
3000             {
3001                 simdTree->gtFlags |= GTF_SIMD12_OP;
3002             }
3003             retVal = simdTree;
3004         }
3005         break;
3006
3007         case SIMDIntrinsicSqrt:
3008         {
3009 #if (defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)) && defined(DEBUG)
3010             // SSE/AVX/ARM64 doesn't support sqrt on integer type vectors and hence
3011             // should never be seen as an intrinsic here. See SIMDIntrinsicList.h
3012             // for supported base types for this intrinsic.
3013             if (!varTypeIsFloating(baseType))
3014             {
3015                 assert(!"Sqrt not supported on integer vectors\n");
3016                 return nullptr;
3017             }
3018 #endif // (defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)) && defined(DEBUG)
3019
3020             op1 = impSIMDPopStack(simdType);
3021
3022             retVal = gtNewSIMDNode(genActualType(callType), op1, nullptr, simdIntrinsicID, baseType, size);
3023         }
3024         break;
3025
3026         case SIMDIntrinsicAbs:
3027             op1    = impSIMDPopStack(simdType);
3028             retVal = impSIMDAbs(clsHnd, baseType, size, op1);
3029             break;
3030
3031         case SIMDIntrinsicGetW:
3032             retVal = impSIMDGetFixed(simdType, baseType, size, 3);
3033             break;
3034
3035         case SIMDIntrinsicGetZ:
3036             retVal = impSIMDGetFixed(simdType, baseType, size, 2);
3037             break;
3038
3039         case SIMDIntrinsicGetY:
3040             retVal = impSIMDGetFixed(simdType, baseType, size, 1);
3041             break;
3042
3043         case SIMDIntrinsicGetX:
3044             retVal = impSIMDGetFixed(simdType, baseType, size, 0);
3045             break;
3046
3047         case SIMDIntrinsicSetW:
3048         case SIMDIntrinsicSetZ:
3049         case SIMDIntrinsicSetY:
3050         case SIMDIntrinsicSetX:
3051         {
3052             // op2 is the value to be set at indexTemp position
3053             // op1 is SIMD vector that is going to be modified, which is a byref
3054
3055             // If op1 has a side-effect, then don't make it an intrinsic.
3056             // It would be in-efficient to read the entire vector into xmm reg,
3057             // modify it and write back entire xmm reg.
3058             //
3059             // TODO-CQ: revisit this later.
3060             op1 = impStackTop(1).val;
3061             if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0)
3062             {
3063                 return nullptr;
3064             }
3065
3066             op2 = impSIMDPopStack(baseType);
3067             op1 = impSIMDPopStack(simdType, instMethod);
3068
3069             GenTree* src = gtCloneExpr(op1);
3070             assert(src != nullptr);
3071             simdTree = gtNewSIMDNode(simdType, src, op2, simdIntrinsicID, baseType, size);
3072
3073             copyBlkDst = gtNewOperNode(GT_ADDR, TYP_BYREF, op1);
3074             doCopyBlk  = true;
3075         }
3076         break;
3077
3078         // Unary operators that take and return a Vector.
3079         case SIMDIntrinsicCast:
3080         case SIMDIntrinsicConvertToSingle:
3081         case SIMDIntrinsicConvertToDouble:
3082         case SIMDIntrinsicConvertToInt32:
3083         {
3084             op1 = impSIMDPopStack(simdType, instMethod);
3085
3086             simdTree = gtNewSIMDNode(simdType, op1, nullptr, simdIntrinsicID, baseType, size);
3087             retVal   = simdTree;
3088         }
3089         break;
3090
3091         case SIMDIntrinsicConvertToInt64:
3092         {
3093 #ifdef _TARGET_64BIT_
3094             op1 = impSIMDPopStack(simdType, instMethod);
3095
3096             simdTree = gtNewSIMDNode(simdType, op1, nullptr, simdIntrinsicID, baseType, size);
3097             retVal   = simdTree;
3098 #else
3099             JITDUMP("SIMD Conversion to Int64 is not supported on this platform\n");
3100             return nullptr;
3101 #endif
3102         }
3103         break;
3104
3105         case SIMDIntrinsicNarrow:
3106         {
3107             assert(!instMethod);
3108             op2 = impSIMDPopStack(simdType);
3109             op1 = impSIMDPopStack(simdType);
3110             // op1 and op2 are two input Vector<T>.
3111             simdTree = gtNewSIMDNode(simdType, op1, op2, simdIntrinsicID, baseType, size);
3112             retVal   = simdTree;
3113         }
3114         break;
3115
3116         case SIMDIntrinsicWiden:
3117         {
3118             GenTree* dstAddrHi = impSIMDPopStack(TYP_BYREF);
3119             GenTree* dstAddrLo = impSIMDPopStack(TYP_BYREF);
3120             op1                = impSIMDPopStack(simdType);
3121             // op1 must have a valid class handle; the following method will assert it.
3122             CORINFO_CLASS_HANDLE op1Handle = gtGetStructHandle(op1);
3123             GenTree*             dupOp1    = fgInsertCommaFormTemp(&op1, op1Handle);
3124
3125             // Widen the lower half and assign it to dstAddrLo.
3126             simdTree = gtNewSIMDNode(simdType, op1, nullptr, SIMDIntrinsicWidenLo, baseType, size);
3127             GenTree* loDest =
3128                 new (this, GT_BLK) GenTreeBlk(GT_BLK, simdType, dstAddrLo, getSIMDTypeSizeInBytes(clsHnd));
3129             GenTree* loAsg = gtNewBlkOpNode(loDest, simdTree, getSIMDTypeSizeInBytes(clsHnd),
3130                                             false, // not volatile
3131                                             true); // copyBlock
3132             loAsg->gtFlags |= ((simdTree->gtFlags | dstAddrLo->gtFlags) & GTF_ALL_EFFECT);
3133
3134             // Widen the upper half and assign it to dstAddrHi.
3135             simdTree = gtNewSIMDNode(simdType, dupOp1, nullptr, SIMDIntrinsicWidenHi, baseType, size);
3136             GenTree* hiDest =
3137                 new (this, GT_BLK) GenTreeBlk(GT_BLK, simdType, dstAddrHi, getSIMDTypeSizeInBytes(clsHnd));
3138             GenTree* hiAsg = gtNewBlkOpNode(hiDest, simdTree, getSIMDTypeSizeInBytes(clsHnd),
3139                                             false, // not volatile
3140                                             true); // copyBlock
3141             hiAsg->gtFlags |= ((simdTree->gtFlags | dstAddrHi->gtFlags) & GTF_ALL_EFFECT);
3142
3143             retVal = gtNewOperNode(GT_COMMA, simdType, loAsg, hiAsg);
3144         }
3145         break;
3146
3147         case SIMDIntrinsicHWAccel:
3148         {
3149             GenTreeIntCon* intConstTree = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, 1);
3150             retVal                      = intConstTree;
3151         }
3152         break;
3153
3154         default:
3155             assert(!"Unimplemented SIMD Intrinsic");
3156             return nullptr;
3157     }
3158
3159 #if defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
3160     // XArch/Arm64: also indicate that we use floating point registers.
3161     // The need for setting this here is that a method may not have SIMD
3162     // type lclvars, but might be exercising SIMD intrinsics on fields of
3163     // SIMD type.
3164     //
3165     // e.g.  public Vector<float> ComplexVecFloat::sqabs() { return this.r * this.r + this.i * this.i; }
3166     compFloatingPointUsed = true;
3167 #endif // defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
3168
3169     // At this point, we have a tree that we are going to store into a destination.
3170     // TODO-1stClassStructs: This should be a simple store or assignment, and should not require
3171     // GTF_ALL_EFFECT for the dest. This is currently emulating the previous behavior of
3172     // block ops.
3173     if (doCopyBlk)
3174     {
3175         GenTree* dest = new (this, GT_BLK) GenTreeBlk(GT_BLK, simdType, copyBlkDst, getSIMDTypeSizeInBytes(clsHnd));
3176         dest->gtFlags |= GTF_GLOB_REF;
3177         retVal = gtNewBlkOpNode(dest, simdTree, getSIMDTypeSizeInBytes(clsHnd),
3178                                 false, // not volatile
3179                                 true); // copyBlock
3180         retVal->gtFlags |= ((simdTree->gtFlags | copyBlkDst->gtFlags) & GTF_ALL_EFFECT);
3181     }
3182
3183     return retVal;
3184 }
3185
3186 #endif // FEATURE_SIMD