src/jit/simd.cpp

   1 // Licensed to the .NET Foundation under one or more agreements.
   2 // The .NET Foundation licenses this file to you under the MIT license.
   3 // See the LICENSE file in the project root for more information.
   4
   5 //
   6 //   SIMD Support
   7 //
   8 // IMPORTANT NOTES AND CAVEATS:
   9 //
  10 // This implementation is preliminary, and may change dramatically.
  11 //
  12 // New JIT types, TYP_SIMDxx, are introduced, and the SIMD intrinsics are created as GT_SIMD nodes.
  13 // Nodes of SIMD types will be typed as TYP_SIMD* (e.g. TYP_SIMD8, TYP_SIMD16, etc.).
  14 //
  15 // Note that currently the "reference implementation" is the same as the runtime dll.  As such, it is currently
  16 // providing implementations for those methods not currently supported by the JIT as intrinsics.
  17 //
  18 // These are currently recognized using string compares, in order to provide an implementation in the JIT
  19 // without taking a dependency on the VM.
  20 // Furthermore, in the CTP, in order to limit the impact of doing these string compares
  21 // against assembly names, we only look for the SIMDVector assembly if we are compiling a class constructor.  This
  22 // makes it somewhat more "pay for play" but is a significant usability compromise.
  23 // This has been addressed for RTM by doing the assembly recognition in the VM.
  24 // --------------------------------------------------------------------------------------
  25
  26 #include "jitpch.h"
  27 #include "simd.h"
  28
  29 #ifdef _MSC_VER
  30 #pragma hdrstop
  31 #endif
  32
  33 #ifdef FEATURE_SIMD
  34
  35 // Intrinsic Id to intrinsic info map
  36 const SIMDIntrinsicInfo simdIntrinsicInfoArray[] = {
  37 #define SIMD_INTRINSIC(mname, inst, id, name, retType, argCount, arg1, arg2, arg3, t1, t2, t3, t4, t5, t6, t7, t8, t9, \
  38                        t10)                                                                                            \
  39     {SIMDIntrinsic##id, mname, inst, retType, argCount, arg1, arg2, arg3, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10},
  40 #include "simdintrinsiclist.h"
  41 };
  42
  43 //------------------------------------------------------------------------
  44 // getSIMDVectorLength: Get the length (number of elements of base type) of
  45 //                      SIMD Vector given its size and base (element) type.
  46 //
  47 // Arguments:
  48 //    simdSize   - size of the SIMD vector
  49 //    baseType   - type of the elements of the SIMD vector
  50 //
  51 // static
  52 int Compiler::getSIMDVectorLength(unsigned simdSize, var_types baseType)
  53 {
  54     return simdSize / genTypeSize(baseType);
  55 }
  56
  57 //------------------------------------------------------------------------
  58 // Get the length (number of elements of base type) of SIMD Vector given by typeHnd.
  59 //
  60 // Arguments:
  61 //    typeHnd  - type handle of the SIMD vector
  62 //
  63 int Compiler::getSIMDVectorLength(CORINFO_CLASS_HANDLE typeHnd)
  64 {
  65     unsigned  sizeBytes = 0;
  66     var_types baseType  = getBaseTypeAndSizeOfSIMDType(typeHnd, &sizeBytes);
  67     return getSIMDVectorLength(sizeBytes, baseType);
  68 }
  69
  70 //------------------------------------------------------------------------
  71 // Get the preferred alignment of SIMD vector type for better performance.
  72 //
  73 // Arguments:
  74 //    typeHnd  - type handle of the SIMD vector
  75 //
  76 int Compiler::getSIMDTypeAlignment(var_types simdType)
  77 {
  78 #ifdef _TARGET_XARCH_
  79     // Fixed length vectors have the following alignment preference
  80     // Vector2   = 8 byte alignment
  81     // Vector3/4 = 16-byte alignment
  82     unsigned size = genTypeSize(simdType);
  83
  84     // preferred alignment for SSE2 128-bit vectors is 16-bytes
  85     if (size == 8)
  86     {
  87         return 8;
  88     }
  89     else if (size <= 16)
  90     {
  91         assert((size == 12) || (size == 16));
  92         return 16;
  93     }
  94     else
  95     {
  96         assert(size == 32);
  97         return 32;
  98     }
  99 #elif defined(_TARGET_ARM64_)
 100     return 16;
 101 #else
 102     assert(!"getSIMDTypeAlignment() unimplemented on target arch");
 103     unreached();
 104 #endif
 105 }
 106
 107 //----------------------------------------------------------------------------------
 108 // Return the base type and size of SIMD vector type given its type handle.
 109 //
 110 // Arguments:
 111 //    typeHnd   - The handle of the type we're interested in.
 112 //    sizeBytes - out param
 113 //
 114 // Return Value:
 115 //    base type of SIMD vector.
 116 //    sizeBytes if non-null is set to size in bytes.
 117 //
 118 // TODO-Throughput: current implementation parses class name to find base type. Change
 119 //         this when we implement  SIMD intrinsic identification for the final
 120 //         product.
 121 //
 122 var_types Compiler::getBaseTypeAndSizeOfSIMDType(CORINFO_CLASS_HANDLE typeHnd, unsigned* sizeBytes /*= nullptr */)
 123 {
 124     assert(featureSIMD);
 125
 126     if (m_simdHandleCache == nullptr)
 127     {
 128         if (impInlineInfo == nullptr)
 129         {
 130             m_simdHandleCache = new (this, CMK_Generic) SIMDHandlesCache();
 131         }
 132         else
 133         {
 134             // Steal the inliner compiler's cache (create it if not available).
 135
 136             if (impInlineInfo->InlineRoot->m_simdHandleCache == nullptr)
 137             {
 138                 impInlineInfo->InlineRoot->m_simdHandleCache = new (this, CMK_Generic) SIMDHandlesCache();
 139             }
 140
 141             m_simdHandleCache = impInlineInfo->InlineRoot->m_simdHandleCache;
 142         }
 143     }
 144
 145     if (typeHnd == nullptr)
 146     {
 147         return TYP_UNKNOWN;
 148     }
 149
 150     // fast path search using cached type handles of important types
 151     var_types simdBaseType = TYP_UNKNOWN;
 152     unsigned  size         = 0;
 153
 154     // TODO - Optimize SIMD type recognition by IntrinsicAttribute
 155     if (isSIMDClass(typeHnd))
 156     {
 157         // The most likely to be used type handles are looked up first followed by
 158         // less likely to be used type handles
 159         if (typeHnd == m_simdHandleCache->SIMDFloatHandle)
 160         {
 161             simdBaseType = TYP_FLOAT;
 162             JITDUMP("  Known type SIMD Vector<Float>\n");
 163         }
 164         else if (typeHnd == m_simdHandleCache->SIMDIntHandle)
 165         {
 166             simdBaseType = TYP_INT;
 167             JITDUMP("  Known type SIMD Vector<Int>\n");
 168         }
 169         else if (typeHnd == m_simdHandleCache->SIMDVector2Handle)
 170         {
 171             simdBaseType = TYP_FLOAT;
 172             size         = 2 * genTypeSize(TYP_FLOAT);
 173             assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE));
 174             JITDUMP("  Known type Vector2\n");
 175         }
 176         else if (typeHnd == m_simdHandleCache->SIMDVector3Handle)
 177         {
 178             simdBaseType = TYP_FLOAT;
 179             size         = 3 * genTypeSize(TYP_FLOAT);
 180             assert(size == info.compCompHnd->getClassSize(typeHnd));
 181             JITDUMP("  Known type Vector3\n");
 182         }
 183         else if (typeHnd == m_simdHandleCache->SIMDVector4Handle)
 184         {
 185             simdBaseType = TYP_FLOAT;
 186             size         = 4 * genTypeSize(TYP_FLOAT);
 187             assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE));
 188             JITDUMP("  Known type Vector4\n");
 189         }
 190         else if (typeHnd == m_simdHandleCache->SIMDVectorHandle)
 191         {
 192             JITDUMP("  Known type Vector\n");
 193         }
 194         else if (typeHnd == m_simdHandleCache->SIMDUShortHandle)
 195         {
 196             simdBaseType = TYP_USHORT;
 197             JITDUMP("  Known type SIMD Vector<ushort>\n");
 198         }
 199         else if (typeHnd == m_simdHandleCache->SIMDUByteHandle)
 200         {
 201             simdBaseType = TYP_UBYTE;
 202             JITDUMP("  Known type SIMD Vector<ubyte>\n");
 203         }
 204         else if (typeHnd == m_simdHandleCache->SIMDDoubleHandle)
 205         {
 206             simdBaseType = TYP_DOUBLE;
 207             JITDUMP("  Known type SIMD Vector<Double>\n");
 208         }
 209         else if (typeHnd == m_simdHandleCache->SIMDLongHandle)
 210         {
 211             simdBaseType = TYP_LONG;
 212             JITDUMP("  Known type SIMD Vector<Long>\n");
 213         }
 214         else if (typeHnd == m_simdHandleCache->SIMDShortHandle)
 215         {
 216             simdBaseType = TYP_SHORT;
 217             JITDUMP("  Known type SIMD Vector<short>\n");
 218         }
 219         else if (typeHnd == m_simdHandleCache->SIMDByteHandle)
 220         {
 221             simdBaseType = TYP_BYTE;
 222             JITDUMP("  Known type SIMD Vector<byte>\n");
 223         }
 224         else if (typeHnd == m_simdHandleCache->SIMDUIntHandle)
 225         {
 226             simdBaseType = TYP_UINT;
 227             JITDUMP("  Known type SIMD Vector<uint>\n");
 228         }
 229         else if (typeHnd == m_simdHandleCache->SIMDULongHandle)
 230         {
 231             simdBaseType = TYP_ULONG;
 232             JITDUMP("  Known type SIMD Vector<ulong>\n");
 233         }
 234
 235         // slow path search
 236         if (simdBaseType == TYP_UNKNOWN)
 237         {
 238             // Doesn't match with any of the cached type handles.
 239             // Obtain base type by parsing fully qualified class name.
 240             //
 241             // TODO-Throughput: implement product shipping solution to query base type.
 242             WCHAR  className[256] = {0};
 243             WCHAR* pbuf           = &className[0];
 244             int    len            = _countof(className);
 245             info.compCompHnd->appendClassName(&pbuf, &len, typeHnd, TRUE, FALSE, FALSE);
 246             noway_assert(pbuf < &className[256]);
 247             JITDUMP("SIMD Candidate Type %S\n", className);
 248
 249             if (wcsncmp(className, W("System.Numerics."), 16) == 0)
 250             {
 251                 if (wcsncmp(&(className[16]), W("Vector`1["), 9) == 0)
 252                 {
 253                     if (wcsncmp(&(className[25]), W("System.Single"), 13) == 0)
 254                     {
 255                         m_simdHandleCache->SIMDFloatHandle = typeHnd;
 256                         simdBaseType                       = TYP_FLOAT;
 257                         JITDUMP("  Found type SIMD Vector<Float>\n");
 258                     }
 259                     else if (wcsncmp(&(className[25]), W("System.Int32"), 12) == 0)
 260                     {
 261                         m_simdHandleCache->SIMDIntHandle = typeHnd;
 262                         simdBaseType                     = TYP_INT;
 263                         JITDUMP("  Found type SIMD Vector<Int>\n");
 264                     }
 265                     else if (wcsncmp(&(className[25]), W("System.UInt16"), 13) == 0)
 266                     {
 267                         m_simdHandleCache->SIMDUShortHandle = typeHnd;
 268                         simdBaseType                        = TYP_USHORT;
 269                         JITDUMP("  Found type SIMD Vector<ushort>\n");
 270                     }
 271                     else if (wcsncmp(&(className[25]), W("System.Byte"), 11) == 0)
 272                     {
 273                         m_simdHandleCache->SIMDUByteHandle = typeHnd;
 274                         simdBaseType                       = TYP_UBYTE;
 275                         JITDUMP("  Found type SIMD Vector<ubyte>\n");
 276                     }
 277                     else if (wcsncmp(&(className[25]), W("System.Double"), 13) == 0)
 278                     {
 279                         m_simdHandleCache->SIMDDoubleHandle = typeHnd;
 280                         simdBaseType                        = TYP_DOUBLE;
 281                         JITDUMP("  Found type SIMD Vector<Double>\n");
 282                     }
 283                     else if (wcsncmp(&(className[25]), W("System.Int64"), 12) == 0)
 284                     {
 285                         m_simdHandleCache->SIMDLongHandle = typeHnd;
 286                         simdBaseType                      = TYP_LONG;
 287                         JITDUMP("  Found type SIMD Vector<Long>\n");
 288                     }
 289                     else if (wcsncmp(&(className[25]), W("System.Int16"), 12) == 0)
 290                     {
 291                         m_simdHandleCache->SIMDShortHandle = typeHnd;
 292                         simdBaseType                       = TYP_SHORT;
 293                         JITDUMP("  Found type SIMD Vector<short>\n");
 294                     }
 295                     else if (wcsncmp(&(className[25]), W("System.SByte"), 12) == 0)
 296                     {
 297                         m_simdHandleCache->SIMDByteHandle = typeHnd;
 298                         simdBaseType                      = TYP_BYTE;
 299                         JITDUMP("  Found type SIMD Vector<byte>\n");
 300                     }
 301                     else if (wcsncmp(&(className[25]), W("System.UInt32"), 13) == 0)
 302                     {
 303                         m_simdHandleCache->SIMDUIntHandle = typeHnd;
 304                         simdBaseType                      = TYP_UINT;
 305                         JITDUMP("  Found type SIMD Vector<uint>\n");
 306                     }
 307                     else if (wcsncmp(&(className[25]), W("System.UInt64"), 13) == 0)
 308                     {
 309                         m_simdHandleCache->SIMDULongHandle = typeHnd;
 310                         simdBaseType                       = TYP_ULONG;
 311                         JITDUMP("  Found type SIMD Vector<ulong>\n");
 312                     }
 313                     else
 314                     {
 315                         JITDUMP("  Unknown SIMD Vector<T>\n");
 316                     }
 317                 }
 318                 else if (wcsncmp(&(className[16]), W("Vector2"), 8) == 0)
 319                 {
 320                     m_simdHandleCache->SIMDVector2Handle = typeHnd;
 321
 322                     simdBaseType = TYP_FLOAT;
 323                     size         = 2 * genTypeSize(TYP_FLOAT);
 324                     assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE));
 325                     JITDUMP(" Found Vector2\n");
 326                 }
 327                 else if (wcsncmp(&(className[16]), W("Vector3"), 8) == 0)
 328                 {
 329                     m_simdHandleCache->SIMDVector3Handle = typeHnd;
 330
 331                     simdBaseType = TYP_FLOAT;
 332                     size         = 3 * genTypeSize(TYP_FLOAT);
 333                     assert(size == info.compCompHnd->getClassSize(typeHnd));
 334                     JITDUMP(" Found Vector3\n");
 335                 }
 336                 else if (wcsncmp(&(className[16]), W("Vector4"), 8) == 0)
 337                 {
 338                     m_simdHandleCache->SIMDVector4Handle = typeHnd;
 339
 340                     simdBaseType = TYP_FLOAT;
 341                     size         = 4 * genTypeSize(TYP_FLOAT);
 342                     assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE));
 343                     JITDUMP(" Found Vector4\n");
 344                 }
 345                 else if (wcsncmp(&(className[16]), W("Vector"), 6) == 0)
 346                 {
 347                     m_simdHandleCache->SIMDVectorHandle = typeHnd;
 348                     JITDUMP(" Found type Vector\n");
 349                 }
 350                 else
 351                 {
 352                     JITDUMP("  Unknown SIMD Type\n");
 353                 }
 354             }
 355         }
 356         if (simdBaseType != TYP_UNKNOWN && sizeBytes != nullptr)
 357         {
 358             // If not a fixed size vector then its size is same as SIMD vector
 359             // register length in bytes
 360             if (size == 0)
 361             {
 362                 size = getSIMDVectorRegisterByteLength();
 363             }
 364
 365             *sizeBytes = size;
 366             setUsesSIMDTypes(true);
 367         }
 368     }
 369 #ifdef FEATURE_HW_INTRINSICS
 370     else if (isIntrinsicType(typeHnd))
 371     {
 372         const size_t Vector64SizeBytes  = 64 / 8;
 373         const size_t Vector128SizeBytes = 128 / 8;
 374         const size_t Vector256SizeBytes = 256 / 8;
 375
 376 #if defined(_TARGET_XARCH_)
 377         static_assert_no_msg(YMM_REGSIZE_BYTES == Vector256SizeBytes);
 378         static_assert_no_msg(XMM_REGSIZE_BYTES == Vector128SizeBytes);
 379
 380         if (typeHnd == m_simdHandleCache->Vector256FloatHandle)
 381         {
 382             simdBaseType = TYP_FLOAT;
 383             size         = Vector256SizeBytes;
 384             JITDUMP("  Known type Vector256<float>\n");
 385         }
 386         else if (typeHnd == m_simdHandleCache->Vector256DoubleHandle)
 387         {
 388             simdBaseType = TYP_DOUBLE;
 389             size         = Vector256SizeBytes;
 390             JITDUMP("  Known type Vector256<double>\n");
 391         }
 392         else if (typeHnd == m_simdHandleCache->Vector256IntHandle)
 393         {
 394             simdBaseType = TYP_INT;
 395             size         = Vector256SizeBytes;
 396             JITDUMP("  Known type Vector256<int>\n");
 397         }
 398         else if (typeHnd == m_simdHandleCache->Vector256UIntHandle)
 399         {
 400             simdBaseType = TYP_UINT;
 401             size         = Vector256SizeBytes;
 402             JITDUMP("  Known type Vector256<uint>\n");
 403         }
 404         else if (typeHnd == m_simdHandleCache->Vector256ShortHandle)
 405         {
 406             simdBaseType = TYP_SHORT;
 407             size         = Vector256SizeBytes;
 408             JITDUMP("  Known type Vector256<short>\n");
 409         }
 410         else if (typeHnd == m_simdHandleCache->Vector256UShortHandle)
 411         {
 412             simdBaseType = TYP_USHORT;
 413             size         = Vector256SizeBytes;
 414             JITDUMP("  Known type Vector256<ushort>\n");
 415         }
 416         else if (typeHnd == m_simdHandleCache->Vector256ByteHandle)
 417         {
 418             simdBaseType = TYP_BYTE;
 419             size         = Vector256SizeBytes;
 420             JITDUMP("  Known type Vector256<sbyte>\n");
 421         }
 422         else if (typeHnd == m_simdHandleCache->Vector256UByteHandle)
 423         {
 424             simdBaseType = TYP_UBYTE;
 425             size         = Vector256SizeBytes;
 426             JITDUMP("  Known type Vector256<byte>\n");
 427         }
 428         else if (typeHnd == m_simdHandleCache->Vector256LongHandle)
 429         {
 430             simdBaseType = TYP_LONG;
 431             size         = Vector256SizeBytes;
 432             JITDUMP("  Known type Vector256<long>\n");
 433         }
 434         else if (typeHnd == m_simdHandleCache->Vector256ULongHandle)
 435         {
 436             simdBaseType = TYP_ULONG;
 437             size         = Vector256SizeBytes;
 438             JITDUMP("  Known type Vector256<ulong>\n");
 439         }
 440         else if (typeHnd == m_simdHandleCache->Vector256FloatHandle)
 441         {
 442             simdBaseType = TYP_FLOAT;
 443             size         = Vector256SizeBytes;
 444             JITDUMP("  Known type Vector256<float>\n");
 445         }
 446         else
 447 #endif // defined(_TARGET_XARCH)
 448             if (typeHnd == m_simdHandleCache->Vector128DoubleHandle)
 449         {
 450             simdBaseType = TYP_DOUBLE;
 451             size         = Vector128SizeBytes;
 452             JITDUMP("  Known type Vector128<double>\n");
 453         }
 454         else if (typeHnd == m_simdHandleCache->Vector128IntHandle)
 455         {
 456             simdBaseType = TYP_INT;
 457             size         = Vector128SizeBytes;
 458             JITDUMP("  Known type Vector128<int>\n");
 459         }
 460         else if (typeHnd == m_simdHandleCache->Vector128UIntHandle)
 461         {
 462             simdBaseType = TYP_UINT;
 463             size         = Vector128SizeBytes;
 464             JITDUMP("  Known type Vector128<uint>\n");
 465         }
 466         else if (typeHnd == m_simdHandleCache->Vector128ShortHandle)
 467         {
 468             simdBaseType = TYP_SHORT;
 469             size         = Vector128SizeBytes;
 470             JITDUMP("  Known type Vector128<short>\n");
 471         }
 472         else if (typeHnd == m_simdHandleCache->Vector128UShortHandle)
 473         {
 474             simdBaseType = TYP_USHORT;
 475             size         = Vector128SizeBytes;
 476             JITDUMP("  Known type Vector128<ushort>\n");
 477         }
 478         else if (typeHnd == m_simdHandleCache->Vector128ByteHandle)
 479         {
 480             simdBaseType = TYP_BYTE;
 481             size         = Vector128SizeBytes;
 482             JITDUMP("  Known type Vector128<sbyte>\n");
 483         }
 484         else if (typeHnd == m_simdHandleCache->Vector128UByteHandle)
 485         {
 486             simdBaseType = TYP_UBYTE;
 487             size         = Vector128SizeBytes;
 488             JITDUMP("  Known type Vector128<byte>\n");
 489         }
 490         else if (typeHnd == m_simdHandleCache->Vector128LongHandle)
 491         {
 492             simdBaseType = TYP_LONG;
 493             size         = Vector128SizeBytes;
 494             JITDUMP("  Known type Vector128<long>\n");
 495         }
 496         else if (typeHnd == m_simdHandleCache->Vector128ULongHandle)
 497         {
 498             simdBaseType = TYP_ULONG;
 499             size         = Vector128SizeBytes;
 500             JITDUMP("  Known type Vector128<ulong>\n");
 501         }
 502 #if defined(_TARGET_ARM64_)
 503         else if (typeHnd == m_simdHandleCache->Vector64IntHandle)
 504         {
 505             simdBaseType = TYP_INT;
 506             size         = Vector64SizeBytes;
 507             JITDUMP("  Known type Vector64<int>\n");
 508         }
 509         else if (typeHnd == m_simdHandleCache->Vector64UIntHandle)
 510         {
 511             simdBaseType = TYP_UINT;
 512             size         = Vector64SizeBytes;
 513             JITDUMP("  Known type Vector64<uint>\n");
 514         }
 515         else if (typeHnd == m_simdHandleCache->Vector64ShortHandle)
 516         {
 517             simdBaseType = TYP_SHORT;
 518             size         = Vector64SizeBytes;
 519             JITDUMP("  Known type Vector64<short>\n");
 520         }
 521         else if (typeHnd == m_simdHandleCache->Vector64UShortHandle)
 522         {
 523             simdBaseType = TYP_USHORT;
 524             size         = Vector64SizeBytes;
 525             JITDUMP("  Known type Vector64<ushort>\n");
 526         }
 527         else if (typeHnd == m_simdHandleCache->Vector64ByteHandle)
 528         {
 529             simdBaseType = TYP_BYTE;
 530             size         = Vector64SizeBytes;
 531             JITDUMP("  Known type Vector64<sbyte>\n");
 532         }
 533         else if (typeHnd == m_simdHandleCache->Vector64UByteHandle)
 534         {
 535             simdBaseType = TYP_UBYTE;
 536             size         = Vector64SizeBytes;
 537             JITDUMP("  Known type Vector64<byte>\n");
 538         }
 539 #endif // defined(_TARGET_ARM64_)
 540
 541         // slow path search
 542         if (simdBaseType == TYP_UNKNOWN)
 543         {
 544             // Doesn't match with any of the cached type handles.
 545             const char*          className   = getClassNameFromMetadata(typeHnd, nullptr);
 546             CORINFO_CLASS_HANDLE baseTypeHnd = getTypeInstantiationArgument(typeHnd, 0);
 547
 548             if (baseTypeHnd != nullptr)
 549             {
 550                 CorInfoType type = info.compCompHnd->getTypeForPrimitiveNumericClass(baseTypeHnd);
 551
 552                 JITDUMP("HW Intrinsic SIMD Candidate Type %s with Base Type %s\n", className,
 553                         getClassNameFromMetadata(baseTypeHnd, nullptr));
 554
 555 #if defined(_TARGET_XARCH_)
 556                 if (strcmp(className, "Vector256`1") == 0)
 557                 {
 558                     size = Vector256SizeBytes;
 559                     switch (type)
 560                     {
 561                         case CORINFO_TYPE_FLOAT:
 562                             m_simdHandleCache->Vector256FloatHandle = typeHnd;
 563                             simdBaseType                            = TYP_FLOAT;
 564                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<float>\n");
 565                             break;
 566                         case CORINFO_TYPE_DOUBLE:
 567                             m_simdHandleCache->Vector256DoubleHandle = typeHnd;
 568                             simdBaseType                             = TYP_DOUBLE;
 569                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<double>\n");
 570                             break;
 571                         case CORINFO_TYPE_INT:
 572                             m_simdHandleCache->Vector256IntHandle = typeHnd;
 573                             simdBaseType                          = TYP_INT;
 574                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<int>\n");
 575                             break;
 576                         case CORINFO_TYPE_UINT:
 577                             m_simdHandleCache->Vector256UIntHandle = typeHnd;
 578                             simdBaseType                           = TYP_UINT;
 579                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<uint>\n");
 580                             break;
 581                         case CORINFO_TYPE_SHORT:
 582                             m_simdHandleCache->Vector256ShortHandle = typeHnd;
 583                             simdBaseType                            = TYP_SHORT;
 584                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<short>\n");
 585                             break;
 586                         case CORINFO_TYPE_USHORT:
 587                             m_simdHandleCache->Vector256UShortHandle = typeHnd;
 588                             simdBaseType                             = TYP_USHORT;
 589                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<ushort>\n");
 590                             break;
 591                         case CORINFO_TYPE_LONG:
 592                             m_simdHandleCache->Vector256LongHandle = typeHnd;
 593                             simdBaseType                           = TYP_LONG;
 594                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<long>\n");
 595                             break;
 596                         case CORINFO_TYPE_ULONG:
 597                             m_simdHandleCache->Vector256ULongHandle = typeHnd;
 598                             simdBaseType                            = TYP_ULONG;
 599                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<ulong>\n");
 600                             break;
 601                         case CORINFO_TYPE_UBYTE:
 602                             m_simdHandleCache->Vector256UByteHandle = typeHnd;
 603                             simdBaseType                            = TYP_UBYTE;
 604                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<byte>\n");
 605                             break;
 606                         case CORINFO_TYPE_BYTE:
 607                             m_simdHandleCache->Vector256ByteHandle = typeHnd;
 608                             simdBaseType                           = TYP_BYTE;
 609                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector256<sbyte>\n");
 610                             break;
 611
 612                         default:
 613                             JITDUMP("  Unknown Hardware Intrinsic SIMD Type Vector256<T>\n");
 614                     }
 615                 }
 616                 else
 617 #endif // defined(_TARGET_XARCH_)
 618                     if (strcmp(className, "Vector128`1") == 0)
 619                 {
 620                     size = Vector128SizeBytes;
 621                     switch (type)
 622                     {
 623                         case CORINFO_TYPE_FLOAT:
 624                             m_simdHandleCache->Vector128FloatHandle = typeHnd;
 625                             simdBaseType                            = TYP_FLOAT;
 626                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<float>\n");
 627                             break;
 628                         case CORINFO_TYPE_DOUBLE:
 629                             m_simdHandleCache->Vector128DoubleHandle = typeHnd;
 630                             simdBaseType                             = TYP_DOUBLE;
 631                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<double>\n");
 632                             break;
 633                         case CORINFO_TYPE_INT:
 634                             m_simdHandleCache->Vector128IntHandle = typeHnd;
 635                             simdBaseType                          = TYP_INT;
 636                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<int>\n");
 637                             break;
 638                         case CORINFO_TYPE_UINT:
 639                             m_simdHandleCache->Vector128UIntHandle = typeHnd;
 640                             simdBaseType                           = TYP_UINT;
 641                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<uint>\n");
 642                             break;
 643                         case CORINFO_TYPE_SHORT:
 644                             m_simdHandleCache->Vector128ShortHandle = typeHnd;
 645                             simdBaseType                            = TYP_SHORT;
 646                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<short>\n");
 647                             break;
 648                         case CORINFO_TYPE_USHORT:
 649                             m_simdHandleCache->Vector128UShortHandle = typeHnd;
 650                             simdBaseType                             = TYP_USHORT;
 651                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<ushort>\n");
 652                             break;
 653                         case CORINFO_TYPE_LONG:
 654                             m_simdHandleCache->Vector128LongHandle = typeHnd;
 655                             simdBaseType                           = TYP_LONG;
 656                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<long>\n");
 657                             break;
 658                         case CORINFO_TYPE_ULONG:
 659                             m_simdHandleCache->Vector128ULongHandle = typeHnd;
 660                             simdBaseType                            = TYP_ULONG;
 661                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<ulong>\n");
 662                             break;
 663                         case CORINFO_TYPE_UBYTE:
 664                             m_simdHandleCache->Vector128UByteHandle = typeHnd;
 665                             simdBaseType                            = TYP_UBYTE;
 666                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<byte>\n");
 667                             break;
 668                         case CORINFO_TYPE_BYTE:
 669                             m_simdHandleCache->Vector128ByteHandle = typeHnd;
 670                             simdBaseType                           = TYP_BYTE;
 671                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector128<sbyte>\n");
 672                             break;
 673
 674                         default:
 675                             JITDUMP("  Unknown Hardware Intrinsic SIMD Type Vector128<T>\n");
 676                     }
 677                 }
 678 #if defined(_TARGET_ARM64_)
 679                 else if (strcmp(className, "Vector64`1") == 0)
 680                 {
 681                     size = Vector64SizeBytes;
 682                     switch (type)
 683                     {
 684                         case CORINFO_TYPE_FLOAT:
 685                             m_simdHandleCache->Vector64FloatHandle = typeHnd;
 686                             simdBaseType                           = TYP_FLOAT;
 687                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<float>\n");
 688                             break;
 689                         case CORINFO_TYPE_INT:
 690                             m_simdHandleCache->Vector64IntHandle = typeHnd;
 691                             simdBaseType                         = TYP_INT;
 692                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<int>\n");
 693                             break;
 694                         case CORINFO_TYPE_UINT:
 695                             m_simdHandleCache->Vector64UIntHandle = typeHnd;
 696                             simdBaseType                          = TYP_UINT;
 697                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<uint>\n");
 698                             break;
 699                         case CORINFO_TYPE_SHORT:
 700                             m_simdHandleCache->Vector64ShortHandle = typeHnd;
 701                             simdBaseType                           = TYP_SHORT;
 702                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<short>\n");
 703                             break;
 704                         case CORINFO_TYPE_USHORT:
 705                             m_simdHandleCache->Vector64UShortHandle = typeHnd;
 706                             simdBaseType                            = TYP_USHORT;
 707                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<ushort>\n");
 708                             break;
 709                         case CORINFO_TYPE_UBYTE:
 710                             m_simdHandleCache->Vector64UByteHandle = typeHnd;
 711                             simdBaseType                           = TYP_UBYTE;
 712                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<byte>\n");
 713                             break;
 714                         case CORINFO_TYPE_BYTE:
 715                             m_simdHandleCache->Vector64ByteHandle = typeHnd;
 716                             simdBaseType                          = TYP_BYTE;
 717                             JITDUMP("  Found type Hardware Intrinsic SIMD Vector64<sbyte>\n");
 718                             break;
 719
 720                         default:
 721                             JITDUMP("  Unknown Hardware Intrinsic SIMD Type Vector64<T>\n");
 722                     }
 723                 }
 724 #endif // defined(_TARGET_ARM64_)
 725             }
 726         }
 727
 728         if (sizeBytes != nullptr)
 729         {
 730             *sizeBytes = size;
 731         }
 732
 733         if (simdBaseType != TYP_UNKNOWN)
 734         {
 735             setUsesSIMDTypes(true);
 736         }
 737     }
 738 #endif // FEATURE_HW_INTRINSICS
 739
 740     return simdBaseType;
 741 }
 742
 743 //--------------------------------------------------------------------------------------
 744 // getSIMDIntrinsicInfo: get SIMD intrinsic info given the method handle.
 745 //
 746 // Arguments:
 747 //    inOutTypeHnd    - The handle of the type on which the method is invoked.  This is an in-out param.
 748 //    methodHnd       - The handle of the method we're interested in.
 749 //    sig             - method signature info
 750 //    isNewObj        - whether this call represents a newboj constructor call
 751 //    argCount        - argument count - out pram
 752 //    baseType        - base type of the intrinsic - out param
 753 //    sizeBytes       - size of SIMD vector type on which the method is invoked - out param
 754 //
 755 // Return Value:
 756 //    SIMDIntrinsicInfo struct initialized corresponding to methodHnd.
 757 //    Sets SIMDIntrinsicInfo.id to SIMDIntrinsicInvalid if methodHnd doesn't correspond
 758 //    to any SIMD intrinsic.  Also, sets the out params inOutTypeHnd, argCount, baseType and
 759 //    sizeBytes.
 760 //
 761 //    Note that VectorMath class doesn't have a base type and first argument of the method
 762 //    determines the SIMD vector type on which intrinsic is invoked. In such a case inOutTypeHnd
 763 //    is modified by this routine.
 764 //
 765 // TODO-Throughput: The current implementation is based on method name string parsing.
 766 //         Although we now have type identification from the VM, the parsing of intrinsic names
 767 //         could be made more efficient.
 768 //
 769 const SIMDIntrinsicInfo* Compiler::getSIMDIntrinsicInfo(CORINFO_CLASS_HANDLE* inOutTypeHnd,
 770                                                         CORINFO_METHOD_HANDLE methodHnd,
 771                                                         CORINFO_SIG_INFO*     sig,
 772                                                         bool                  isNewObj,
 773                                                         unsigned*             argCount,
 774                                                         var_types*            baseType,
 775                                                         unsigned*             sizeBytes)
 776 {
 777     assert(featureSIMD);
 778     assert(baseType != nullptr);
 779     assert(sizeBytes != nullptr);
 780
 781     // get baseType and size of the type
 782     CORINFO_CLASS_HANDLE typeHnd = *inOutTypeHnd;
 783     *baseType                    = getBaseTypeAndSizeOfSIMDType(typeHnd, sizeBytes);
 784
 785     if (typeHnd == m_simdHandleCache->SIMDVectorHandle)
 786     {
 787         // All of the supported intrinsics on this static class take a first argument that's a vector,
 788         // which determines the baseType.
 789         // The exception is the IsHardwareAccelerated property, which is handled as a special case.
 790         assert(*baseType == TYP_UNKNOWN);
 791         if (sig->numArgs == 0)
 792         {
 793             const SIMDIntrinsicInfo* hwAccelIntrinsicInfo = &(simdIntrinsicInfoArray[SIMDIntrinsicHWAccel]);
 794             if ((strcmp(eeGetMethodName(methodHnd, nullptr), hwAccelIntrinsicInfo->methodName) == 0) &&
 795                 JITtype2varType(sig->retType) == hwAccelIntrinsicInfo->retType)
 796             {
 797                 // Sanity check
 798                 assert(hwAccelIntrinsicInfo->argCount == 0 && hwAccelIntrinsicInfo->isInstMethod == false);
 799                 return hwAccelIntrinsicInfo;
 800             }
 801             return nullptr;
 802         }
 803         else
 804         {
 805             typeHnd       = info.compCompHnd->getArgClass(sig, sig->args);
 806             *inOutTypeHnd = typeHnd;
 807             *baseType     = getBaseTypeAndSizeOfSIMDType(typeHnd, sizeBytes);
 808         }
 809     }
 810
 811     if (*baseType == TYP_UNKNOWN)
 812     {
 813         JITDUMP("NOT a SIMD Intrinsic: unsupported baseType\n");
 814         return nullptr;
 815     }
 816
 817     // account for implicit "this" arg
 818     *argCount = sig->numArgs;
 819     if (sig->hasThis())
 820     {
 821         *argCount += 1;
 822     }
 823
 824     // Get the Intrinsic Id by parsing method name.
 825     //
 826     // TODO-Throughput: replace sequential search by binary search by arranging entries
 827     // sorted by method name.
 828     SIMDIntrinsicID intrinsicId = SIMDIntrinsicInvalid;
 829     const char*     methodName  = eeGetMethodName(methodHnd, nullptr);
 830     for (int i = SIMDIntrinsicNone + 1; i < SIMDIntrinsicInvalid; ++i)
 831     {
 832         if (strcmp(methodName, simdIntrinsicInfoArray[i].methodName) == 0)
 833         {
 834             // Found an entry for the method; further check whether it is one of
 835             // the supported base types.
 836             bool found = false;
 837             for (int j = 0; j < SIMD_INTRINSIC_MAX_BASETYPE_COUNT; ++j)
 838             {
 839                 // Convention: if there are fewer base types supported than MAX_BASETYPE_COUNT,
 840                 // the end of the list is marked by TYP_UNDEF.
 841                 if (simdIntrinsicInfoArray[i].supportedBaseTypes[j] == TYP_UNDEF)
 842                 {
 843                     break;
 844                 }
 845
 846                 if (simdIntrinsicInfoArray[i].supportedBaseTypes[j] == *baseType)
 847                 {
 848                     found = true;
 849                     break;
 850                 }
 851             }
 852
 853             if (!found)
 854             {
 855                 continue;
 856             }
 857
 858             // Now, check the arguments.
 859             unsigned int fixedArgCnt    = simdIntrinsicInfoArray[i].argCount;
 860             unsigned int expectedArgCnt = fixedArgCnt;
 861
 862             // First handle SIMDIntrinsicInitN, where the arg count depends on the type.
 863             // The listed arg types include the vector and the first two init values, which is the expected number
 864             // for Vector2.  For other cases, we'll check their types here.
 865             if (*argCount > expectedArgCnt)
 866             {
 867                 if (i == SIMDIntrinsicInitN)
 868                 {
 869                     if (*argCount == 3 && typeHnd == m_simdHandleCache->SIMDVector2Handle)
 870                     {
 871                         expectedArgCnt = 3;
 872                     }
 873                     else if (*argCount == 4 && typeHnd == m_simdHandleCache->SIMDVector3Handle)
 874                     {
 875                         expectedArgCnt = 4;
 876                     }
 877                     else if (*argCount == 5 && typeHnd == m_simdHandleCache->SIMDVector4Handle)
 878                     {
 879                         expectedArgCnt = 5;
 880                     }
 881                 }
 882                 else if (i == SIMDIntrinsicInitFixed)
 883                 {
 884                     if (*argCount == 4 && typeHnd == m_simdHandleCache->SIMDVector4Handle)
 885                     {
 886                         expectedArgCnt = 4;
 887                     }
 888                 }
 889             }
 890             if (*argCount != expectedArgCnt)
 891             {
 892                 continue;
 893             }
 894
 895             // Validate the types of individual args passed are what is expected of.
 896             // If any of the types don't match with what is expected, don't consider
 897             // as an intrinsic.  This will make an older JIT with SIMD capabilities
 898             // resilient to breaking changes to SIMD managed API.
 899             //
 900             // Note that from IL type stack, args get popped in right to left order
 901             // whereas args get listed in method signatures in left to right order.
 902
 903             int stackIndex = (expectedArgCnt - 1);
 904
 905             // Track the arguments from the signature - we currently only use this to distinguish
 906             // integral and pointer types, both of which will by TYP_I_IMPL on the importer stack.
 907             CORINFO_ARG_LIST_HANDLE argLst = sig->args;
 908
 909             CORINFO_CLASS_HANDLE argClass;
 910             for (unsigned int argIndex = 0; found == true && argIndex < expectedArgCnt; argIndex++)
 911             {
 912                 bool isThisPtr = ((argIndex == 0) && sig->hasThis());
 913
 914                 // In case of "newobj SIMDVector<T>(T val)", thisPtr won't be present on type stack.
 915                 // We don't check anything in that case.
 916                 if (!isThisPtr || !isNewObj)
 917                 {
 918                     GenTree*  arg     = impStackTop(stackIndex).val;
 919                     var_types argType = arg->TypeGet();
 920
 921                     var_types expectedArgType;
 922                     if (argIndex < fixedArgCnt)
 923                     {
 924                         // Convention:
 925                         //   - intrinsicInfo.argType[i] == TYP_UNDEF - intrinsic doesn't have a valid arg at position i
 926                         //   - intrinsicInfo.argType[i] == TYP_UNKNOWN - arg type should be same as basetype
 927                         // Note that we pop the args off in reverse order.
 928                         expectedArgType = simdIntrinsicInfoArray[i].argType[argIndex];
 929                         assert(expectedArgType != TYP_UNDEF);
 930                         if (expectedArgType == TYP_UNKNOWN)
 931                         {
 932                             // The type of the argument will be genActualType(*baseType).
 933                             expectedArgType = genActualType(*baseType);
 934                             argType         = genActualType(argType);
 935                         }
 936                     }
 937                     else
 938                     {
 939                         expectedArgType = *baseType;
 940                     }
 941
 942                     if (!isThisPtr && argType == TYP_I_IMPL)
 943                     {
 944                         // The reference implementation has a constructor that takes a pointer.
 945                         // We don't want to recognize that one.  This requires us to look at the CorInfoType
 946                         // in order to distinguish a signature with a pointer argument from one with an
 947                         // integer argument of pointer size, both of which will be TYP_I_IMPL on the stack.
 948                         // TODO-Review: This seems quite fragile.  We should consider beefing up the checking
 949                         // here.
 950                         CorInfoType corType = strip(info.compCompHnd->getArgType(sig, argLst, &argClass));
 951                         if (corType == CORINFO_TYPE_PTR)
 952                         {
 953                             found = false;
 954                         }
 955                     }
 956
 957                     if (varTypeIsSIMD(argType))
 958                     {
 959                         argType = TYP_STRUCT;
 960                     }
 961                     if (argType != expectedArgType)
 962                     {
 963                         found = false;
 964                     }
 965                 }
 966                 if (argIndex != 0 || !sig->hasThis())
 967                 {
 968                     argLst = info.compCompHnd->getArgNext(argLst);
 969                 }
 970                 stackIndex--;
 971             }
 972
 973             // Cross check return type and static vs. instance is what we are expecting.
 974             // If not, don't consider it as an intrinsic.
 975             // Note that ret type of TYP_UNKNOWN means that it is not known apriori and must be same as baseType
 976             if (found)
 977             {
 978                 var_types expectedRetType = simdIntrinsicInfoArray[i].retType;
 979                 if (expectedRetType == TYP_UNKNOWN)
 980                 {
 981                     // JIT maps uint/ulong type vars to TYP_INT/TYP_LONG.
 982                     expectedRetType =
 983                         (*baseType == TYP_UINT || *baseType == TYP_ULONG) ? genActualType(*baseType) : *baseType;
 984                 }
 985
 986                 if (JITtype2varType(sig->retType) != expectedRetType ||
 987                     sig->hasThis() != simdIntrinsicInfoArray[i].isInstMethod)
 988                 {
 989                     found = false;
 990                 }
 991             }
 992
 993             if (found)
 994             {
 995                 intrinsicId = (SIMDIntrinsicID)i;
 996                 break;
 997             }
 998         }
 999     }
1000
1001     if (intrinsicId != SIMDIntrinsicInvalid)
1002     {
1003         JITDUMP("Method %s maps to SIMD intrinsic %s\n", methodName, simdIntrinsicNames[intrinsicId]);
1004         return &simdIntrinsicInfoArray[intrinsicId];
1005     }
1006     else
1007     {
1008         JITDUMP("Method %s is NOT a SIMD intrinsic\n", methodName);
1009     }
1010
1011     return nullptr;
1012 }
1013
1014 // Pops and returns GenTree node from importer's type stack.
1015 // Normalizes TYP_STRUCT value in case of GT_CALL, GT_RET_EXPR and arg nodes.
1016 //
1017 // Arguments:
1018 //    type        -  the type of value that the caller expects to be popped off the stack.
1019 //    expectAddr  -  if true indicates we are expecting type stack entry to be a TYP_BYREF.
1020 //
1021 // Notes:
1022 //    If the popped value is a struct, and the expected type is a simd type, it will be set
1023 //    to that type, otherwise it will assert if the type being popped is not the expected type.
1024
1025 GenTree* Compiler::impSIMDPopStack(var_types type, bool expectAddr)
1026 {
1027     StackEntry se   = impPopStack();
1028     typeInfo   ti   = se.seTypeInfo;
1029     GenTree*   tree = se.val;
1030
1031     // If expectAddr is true implies what we have on stack is address and we need
1032     // SIMD type struct that it points to.
1033     if (expectAddr)
1034     {
1035         assert(tree->TypeGet() == TYP_BYREF);
1036         if (tree->OperGet() == GT_ADDR)
1037         {
1038             tree = tree->gtGetOp1();
1039         }
1040         else
1041         {
1042             tree = gtNewOperNode(GT_IND, type, tree);
1043         }
1044     }
1045
1046     bool isParam = false;
1047
1048     // If we have a ldobj of a SIMD local we need to transform it.
1049     if (tree->OperGet() == GT_OBJ)
1050     {
1051         GenTree* addr = tree->gtOp.gtOp1;
1052         if ((addr->OperGet() == GT_ADDR) && isSIMDTypeLocal(addr->gtOp.gtOp1))
1053         {
1054             tree = addr->gtOp.gtOp1;
1055         }
1056     }
1057
1058     if (tree->OperGet() == GT_LCL_VAR)
1059     {
1060         unsigned   lclNum    = tree->AsLclVarCommon()->GetLclNum();
1061         LclVarDsc* lclVarDsc = &lvaTable[lclNum];
1062         isParam              = lclVarDsc->lvIsParam;
1063     }
1064
1065     // normalize TYP_STRUCT value
1066     if (varTypeIsStruct(tree) && ((tree->OperGet() == GT_RET_EXPR) || (tree->OperGet() == GT_CALL) || isParam))
1067     {
1068         assert(ti.IsType(TI_STRUCT));
1069         CORINFO_CLASS_HANDLE structType = ti.GetClassHandleForValueClass();
1070         tree                            = impNormStructVal(tree, structType, (unsigned)CHECK_SPILL_ALL);
1071     }
1072
1073     // Now set the type of the tree to the specialized SIMD struct type, if applicable.
1074     if (genActualType(tree->gtType) != genActualType(type))
1075     {
1076         assert(tree->gtType == TYP_STRUCT);
1077         tree->gtType = type;
1078     }
1079     else if (tree->gtType == TYP_BYREF)
1080     {
1081         assert(tree->IsLocal() || (tree->OperGet() == GT_RET_EXPR) || (tree->OperGet() == GT_CALL) ||
1082                ((tree->gtOper == GT_ADDR) && varTypeIsSIMD(tree->gtGetOp1())));
1083     }
1084
1085     return tree;
1086 }
1087
1088 // impSIMDGetFixed: Create a GT_SIMD tree for a Get property of SIMD vector with a fixed index.
1089 //
1090 // Arguments:
1091 //    baseType - The base (element) type of the SIMD vector.
1092 //    simdSize - The total size in bytes of the SIMD vector.
1093 //    index    - The index of the field to get.
1094 //
1095 // Return Value:
1096 //    Returns a GT_SIMD node with the SIMDIntrinsicGetItem intrinsic id.
1097 //
1098 GenTreeSIMD* Compiler::impSIMDGetFixed(var_types simdType, var_types baseType, unsigned simdSize, int index)
1099 {
1100     assert(simdSize >= ((index + 1) * genTypeSize(baseType)));
1101
1102     // op1 is a SIMD source.
1103     GenTree* op1 = impSIMDPopStack(simdType, true);
1104
1105     GenTree*     op2      = gtNewIconNode(index);
1106     GenTreeSIMD* simdTree = gtNewSIMDNode(baseType, op1, op2, SIMDIntrinsicGetItem, baseType, simdSize);
1107     return simdTree;
1108 }
1109
1110 #ifdef _TARGET_XARCH_
1111 // impSIMDLongRelOpEqual: transforms operands and returns the SIMD intrinsic to be applied on
1112 // transformed operands to obtain == comparison result.
1113 //
1114 // Argumens:
1115 //    typeHnd  -  type handle of SIMD vector
1116 //    size     -  SIMD vector size
1117 //    op1      -  in-out parameter; first operand
1118 //    op2      -  in-out parameter; second operand
1119 //
1120 // Return Value:
1121 //    Modifies in-out params op1, op2 and returns intrinsic ID to be applied to modified operands
1122 //
1123 SIMDIntrinsicID Compiler::impSIMDLongRelOpEqual(CORINFO_CLASS_HANDLE typeHnd,
1124                                                 unsigned             size,
1125                                                 GenTree**            pOp1,
1126                                                 GenTree**            pOp2)
1127 {
1128     var_types simdType = (*pOp1)->TypeGet();
1129     assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1130
1131     // There is no direct SSE2 support for comparing TYP_LONG vectors.
1132     // These have to be implemented in terms of TYP_INT vector comparison operations.
1133     //
1134     // Equality(v1, v2):
1135     // tmp = (v1 == v2) i.e. compare for equality as if v1 and v2 are vector<int>
1136     // result = BitwiseAnd(t, shuffle(t, (2, 3, 0, 1)))
1137     // Shuffle is meant to swap the comparison results of low-32-bits and high 32-bits of respective long elements.
1138
1139     // Compare vector<long> as if they were vector<int> and assign the result to a temp
1140     GenTree* compResult = gtNewSIMDNode(simdType, *pOp1, *pOp2, SIMDIntrinsicEqual, TYP_INT, size);
1141     unsigned lclNum     = lvaGrabTemp(true DEBUGARG("SIMD Long =="));
1142     lvaSetStruct(lclNum, typeHnd, false);
1143     GenTree* tmp = gtNewLclvNode(lclNum, simdType);
1144     GenTree* asg = gtNewTempAssign(lclNum, compResult);
1145
1146     // op1 = GT_COMMA(tmp=compResult, tmp)
1147     // op2 = Shuffle(tmp, 0xB1)
1148     // IntrinsicId = BitwiseAnd
1149     *pOp1 = gtNewOperNode(GT_COMMA, simdType, asg, tmp);
1150     *pOp2 = gtNewSIMDNode(simdType, gtNewLclvNode(lclNum, simdType), gtNewIconNode(SHUFFLE_ZWXY, TYP_INT),
1151                           SIMDIntrinsicShuffleSSE2, TYP_INT, size);
1152     return SIMDIntrinsicBitwiseAnd;
1153 }
1154
1155 // impSIMDLongRelOpGreaterThan: transforms operands and returns the SIMD intrinsic to be applied on
1156 // transformed operands to obtain > comparison result.
1157 //
1158 // Argumens:
1159 //    typeHnd  -  type handle of SIMD vector
1160 //    size     -  SIMD vector size
1161 //    pOp1     -  in-out parameter; first operand
1162 //    pOp2     -  in-out parameter; second operand
1163 //
1164 // Return Value:
1165 //    Modifies in-out params pOp1, pOp2 and returns intrinsic ID to be applied to modified operands
1166 //
1167 SIMDIntrinsicID Compiler::impSIMDLongRelOpGreaterThan(CORINFO_CLASS_HANDLE typeHnd,
1168                                                       unsigned             size,
1169                                                       GenTree**            pOp1,
1170                                                       GenTree**            pOp2)
1171 {
1172     var_types simdType = (*pOp1)->TypeGet();
1173     assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1174
1175     // GreaterThan(v1, v2) where v1 and v2 are vector long.
1176     // Let us consider the case of single long element comparison.
1177     // say L1 = (x1, y1) and L2 = (x2, y2) where x1, y1, x2, and y2 are 32-bit integers that comprise the longs L1 and
1178     // L2.
1179     //
1180     // GreaterThan(L1, L2) can be expressed in terms of > relationship between 32-bit integers that comprise L1 and L2
1181     // as
1182     //                    =  (x1, y1) > (x2, y2)
1183     //                    =  (x1 > x2) || [(x1 == x2) && (y1 > y2)]   - eq (1)
1184     //
1185     // t = (v1 > v2)  32-bit signed comparison
1186     // u = (v1 == v2) 32-bit sized element equality
1187     // v = (v1 > v2)  32-bit unsigned comparison
1188     //
1189     // z = shuffle(t, (3, 3, 1, 1))  - This corresponds to (x1 > x2) in eq(1) above
1190     // t1 = Shuffle(v, (2, 2, 0, 0)) - This corresponds to (y1 > y2) in eq(1) above
1191     // u1 = Shuffle(u, (3, 3, 1, 1)) - This corresponds to (x1 == x2) in eq(1) above
1192     // w = And(t1, u1)               - This corresponds to [(x1 == x2) && (y1 > y2)] in eq(1) above
1193     // Result = BitwiseOr(z, w)
1194
1195     // Since op1 and op2 gets used multiple times, make sure side effects are computed.
1196     GenTree* dupOp1    = nullptr;
1197     GenTree* dupOp2    = nullptr;
1198     GenTree* dupDupOp1 = nullptr;
1199     GenTree* dupDupOp2 = nullptr;
1200
1201     if (((*pOp1)->gtFlags & GTF_SIDE_EFFECT) != 0)
1202     {
1203         dupOp1    = fgInsertCommaFormTemp(pOp1, typeHnd);
1204         dupDupOp1 = gtNewLclvNode(dupOp1->AsLclVarCommon()->GetLclNum(), simdType);
1205     }
1206     else
1207     {
1208         dupOp1    = gtCloneExpr(*pOp1);
1209         dupDupOp1 = gtCloneExpr(*pOp1);
1210     }
1211
1212     if (((*pOp2)->gtFlags & GTF_SIDE_EFFECT) != 0)
1213     {
1214         dupOp2    = fgInsertCommaFormTemp(pOp2, typeHnd);
1215         dupDupOp2 = gtNewLclvNode(dupOp2->AsLclVarCommon()->GetLclNum(), simdType);
1216     }
1217     else
1218     {
1219         dupOp2    = gtCloneExpr(*pOp2);
1220         dupDupOp2 = gtCloneExpr(*pOp2);
1221     }
1222
1223     assert(dupDupOp1 != nullptr && dupDupOp2 != nullptr);
1224     assert(dupOp1 != nullptr && dupOp2 != nullptr);
1225     assert(*pOp1 != nullptr && *pOp2 != nullptr);
1226
1227     // v1GreaterThanv2Signed - signed 32-bit comparison
1228     GenTree* v1GreaterThanv2Signed = gtNewSIMDNode(simdType, *pOp1, *pOp2, SIMDIntrinsicGreaterThan, TYP_INT, size);
1229
1230     // v1Equalsv2 - 32-bit equality
1231     GenTree* v1Equalsv2 = gtNewSIMDNode(simdType, dupOp1, dupOp2, SIMDIntrinsicEqual, TYP_INT, size);
1232
1233     // v1GreaterThanv2Unsigned - unsigned 32-bit comparison
1234     var_types       tempBaseType = TYP_UINT;
1235     SIMDIntrinsicID sid = impSIMDRelOp(SIMDIntrinsicGreaterThan, typeHnd, size, &tempBaseType, &dupDupOp1, &dupDupOp2);
1236     GenTree*        v1GreaterThanv2Unsigned = gtNewSIMDNode(simdType, dupDupOp1, dupDupOp2, sid, tempBaseType, size);
1237
1238     GenTree* z = gtNewSIMDNode(simdType, v1GreaterThanv2Signed, gtNewIconNode(SHUFFLE_WWYY, TYP_INT),
1239                                SIMDIntrinsicShuffleSSE2, TYP_FLOAT, size);
1240     GenTree* t1 = gtNewSIMDNode(simdType, v1GreaterThanv2Unsigned, gtNewIconNode(SHUFFLE_ZZXX, TYP_INT),
1241                                 SIMDIntrinsicShuffleSSE2, TYP_FLOAT, size);
1242     GenTree* u1 = gtNewSIMDNode(simdType, v1Equalsv2, gtNewIconNode(SHUFFLE_WWYY, TYP_INT), SIMDIntrinsicShuffleSSE2,
1243                                 TYP_FLOAT, size);
1244     GenTree* w = gtNewSIMDNode(simdType, u1, t1, SIMDIntrinsicBitwiseAnd, TYP_INT, size);
1245
1246     *pOp1 = z;
1247     *pOp2 = w;
1248     return SIMDIntrinsicBitwiseOr;
1249 }
1250
1251 // impSIMDLongRelOpGreaterThanOrEqual: transforms operands and returns the SIMD intrinsic to be applied on
1252 // transformed operands to obtain >= comparison result.
1253 //
1254 // Argumens:
1255 //    typeHnd  -  type handle of SIMD vector
1256 //    size     -  SIMD vector size
1257 //    pOp1      -  in-out parameter; first operand
1258 //    pOp2      -  in-out parameter; second operand
1259 //
1260 // Return Value:
1261 //    Modifies in-out params pOp1, pOp2 and returns intrinsic ID to be applied to modified operands
1262 //
1263 SIMDIntrinsicID Compiler::impSIMDLongRelOpGreaterThanOrEqual(CORINFO_CLASS_HANDLE typeHnd,
1264                                                              unsigned             size,
1265                                                              GenTree**            pOp1,
1266                                                              GenTree**            pOp2)
1267 {
1268     var_types simdType = (*pOp1)->TypeGet();
1269     assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1270
1271     // expand this to (a == b) | (a > b)
1272     GenTree* dupOp1 = nullptr;
1273     GenTree* dupOp2 = nullptr;
1274
1275     if (((*pOp1)->gtFlags & GTF_SIDE_EFFECT) != 0)
1276     {
1277         dupOp1 = fgInsertCommaFormTemp(pOp1, typeHnd);
1278     }
1279     else
1280     {
1281         dupOp1 = gtCloneExpr(*pOp1);
1282     }
1283
1284     if (((*pOp2)->gtFlags & GTF_SIDE_EFFECT) != 0)
1285     {
1286         dupOp2 = fgInsertCommaFormTemp(pOp2, typeHnd);
1287     }
1288     else
1289     {
1290         dupOp2 = gtCloneExpr(*pOp2);
1291     }
1292
1293     assert(dupOp1 != nullptr && dupOp2 != nullptr);
1294     assert(*pOp1 != nullptr && *pOp2 != nullptr);
1295
1296     // (a==b)
1297     SIMDIntrinsicID id = impSIMDLongRelOpEqual(typeHnd, size, pOp1, pOp2);
1298     *pOp1              = gtNewSIMDNode(simdType, *pOp1, *pOp2, id, TYP_LONG, size);
1299
1300     // (a > b)
1301     id    = impSIMDLongRelOpGreaterThan(typeHnd, size, &dupOp1, &dupOp2);
1302     *pOp2 = gtNewSIMDNode(simdType, dupOp1, dupOp2, id, TYP_LONG, size);
1303
1304     return SIMDIntrinsicBitwiseOr;
1305 }
1306
1307 // impSIMDInt32OrSmallIntRelOpGreaterThanOrEqual: transforms operands and returns the SIMD intrinsic to be applied on
1308 // transformed operands to obtain >= comparison result in case of integer base type vectors
1309 //
1310 // Argumens:
1311 //    typeHnd  -  type handle of SIMD vector
1312 //    size     -  SIMD vector size
1313 //    baseType -  base type of SIMD vector
1314 //    pOp1      -  in-out parameter; first operand
1315 //    pOp2      -  in-out parameter; second operand
1316 //
1317 // Return Value:
1318 //    Modifies in-out params pOp1, pOp2 and returns intrinsic ID to be applied to modified operands
1319 //
1320 SIMDIntrinsicID Compiler::impSIMDIntegralRelOpGreaterThanOrEqual(
1321     CORINFO_CLASS_HANDLE typeHnd, unsigned size, var_types baseType, GenTree** pOp1, GenTree** pOp2)
1322 {
1323     var_types simdType = (*pOp1)->TypeGet();
1324     assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1325
1326     // This routine should be used only for integer base type vectors
1327     assert(varTypeIsIntegral(baseType));
1328     if ((getSIMDSupportLevel() == SIMD_SSE2_Supported) && ((baseType == TYP_LONG) || baseType == TYP_UBYTE))
1329     {
1330         return impSIMDLongRelOpGreaterThanOrEqual(typeHnd, size, pOp1, pOp2);
1331     }
1332
1333     // expand this to (a == b) | (a > b)
1334     GenTree* dupOp1 = nullptr;
1335     GenTree* dupOp2 = nullptr;
1336
1337     if (((*pOp1)->gtFlags & GTF_SIDE_EFFECT) != 0)
1338     {
1339         dupOp1 = fgInsertCommaFormTemp(pOp1, typeHnd);
1340     }
1341     else
1342     {
1343         dupOp1 = gtCloneExpr(*pOp1);
1344     }
1345
1346     if (((*pOp2)->gtFlags & GTF_SIDE_EFFECT) != 0)
1347     {
1348         dupOp2 = fgInsertCommaFormTemp(pOp2, typeHnd);
1349     }
1350     else
1351     {
1352         dupOp2 = gtCloneExpr(*pOp2);
1353     }
1354
1355     assert(dupOp1 != nullptr && dupOp2 != nullptr);
1356     assert(*pOp1 != nullptr && *pOp2 != nullptr);
1357
1358     // (a==b)
1359     *pOp1 = gtNewSIMDNode(simdType, *pOp1, *pOp2, SIMDIntrinsicEqual, baseType, size);
1360
1361     // (a > b)
1362     *pOp2 = gtNewSIMDNode(simdType, dupOp1, dupOp2, SIMDIntrinsicGreaterThan, baseType, size);
1363
1364     return SIMDIntrinsicBitwiseOr;
1365 }
1366 #endif // _TARGET_XARCH_
1367
1368 // Transforms operands and returns the SIMD intrinsic to be applied on
1369 // transformed operands to obtain given relop result.
1370 //
1371 // Argumens:
1372 //    relOpIntrinsicId - Relational operator SIMD intrinsic
1373 //    typeHnd          - type handle of SIMD vector
1374 //    size             -  SIMD vector size
1375 //    inOutBaseType    - base type of SIMD vector
1376 //    pOp1             -  in-out parameter; first operand
1377 //    pOp2             -  in-out parameter; second operand
1378 //
1379 // Return Value:
1380 //    Modifies in-out params pOp1, pOp2, inOutBaseType and returns intrinsic ID to be applied to modified operands
1381 //
1382 SIMDIntrinsicID Compiler::impSIMDRelOp(SIMDIntrinsicID      relOpIntrinsicId,
1383                                        CORINFO_CLASS_HANDLE typeHnd,
1384                                        unsigned             size,
1385                                        var_types*           inOutBaseType,
1386                                        GenTree**            pOp1,
1387                                        GenTree**            pOp2)
1388 {
1389     var_types simdType = (*pOp1)->TypeGet();
1390     assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1391
1392     assert(isRelOpSIMDIntrinsic(relOpIntrinsicId));
1393
1394     SIMDIntrinsicID intrinsicID = relOpIntrinsicId;
1395 #ifdef _TARGET_XARCH_
1396     var_types baseType = *inOutBaseType;
1397
1398     if (varTypeIsFloating(baseType))
1399     {
1400         // SSE2/AVX doesn't support > and >= on vector float/double.
1401         // Therefore, we need to use < and <= with swapped operands
1402         if (relOpIntrinsicId == SIMDIntrinsicGreaterThan || relOpIntrinsicId == SIMDIntrinsicGreaterThanOrEqual)
1403         {
1404             GenTree* tmp = *pOp1;
1405             *pOp1        = *pOp2;
1406             *pOp2        = tmp;
1407
1408             intrinsicID =
1409                 (relOpIntrinsicId == SIMDIntrinsicGreaterThan) ? SIMDIntrinsicLessThan : SIMDIntrinsicLessThanOrEqual;
1410         }
1411     }
1412     else if (varTypeIsIntegral(baseType))
1413     {
1414         // SSE/AVX doesn't support < and <= on integer base type vectors.
1415         // Therefore, we need to use > and >= with swapped operands.
1416         if (intrinsicID == SIMDIntrinsicLessThan || intrinsicID == SIMDIntrinsicLessThanOrEqual)
1417         {
1418             GenTree* tmp = *pOp1;
1419             *pOp1        = *pOp2;
1420             *pOp2        = tmp;
1421
1422             intrinsicID = (relOpIntrinsicId == SIMDIntrinsicLessThan) ? SIMDIntrinsicGreaterThan
1423                                                                       : SIMDIntrinsicGreaterThanOrEqual;
1424         }
1425
1426         if ((getSIMDSupportLevel() == SIMD_SSE2_Supported) && baseType == TYP_LONG)
1427         {
1428             // There is no direct SSE2 support for comparing TYP_LONG vectors.
1429             // These have to be implemented interms of TYP_INT vector comparison operations.
1430             if (intrinsicID == SIMDIntrinsicEqual)
1431             {
1432                 intrinsicID = impSIMDLongRelOpEqual(typeHnd, size, pOp1, pOp2);
1433             }
1434             else if (intrinsicID == SIMDIntrinsicGreaterThan)
1435             {
1436                 intrinsicID = impSIMDLongRelOpGreaterThan(typeHnd, size, pOp1, pOp2);
1437             }
1438             else if (intrinsicID == SIMDIntrinsicGreaterThanOrEqual)
1439             {
1440                 intrinsicID = impSIMDLongRelOpGreaterThanOrEqual(typeHnd, size, pOp1, pOp2);
1441             }
1442             else
1443             {
1444                 unreached();
1445             }
1446         }
1447         // SSE2 and AVX direct support for signed comparison of int32, int16 and int8 types
1448         else if (!varTypeIsUnsigned(baseType))
1449         {
1450             if (intrinsicID == SIMDIntrinsicGreaterThanOrEqual)
1451             {
1452                 intrinsicID = impSIMDIntegralRelOpGreaterThanOrEqual(typeHnd, size, baseType, pOp1, pOp2);
1453             }
1454         }
1455         else // unsigned
1456         {
1457             // Vector<byte>, Vector<ushort>, Vector<uint> and Vector<ulong>:
1458             // SSE2 supports > for signed comparison. Therefore, to use it for
1459             // comparing unsigned numbers, we subtract a constant from both the
1460             // operands such that the result fits within the corresponding signed
1461             // type.  The resulting signed numbers are compared using SSE2 signed
1462             // comparison.
1463             //
1464             // Vector<byte>: constant to be subtracted is 2^7
1465             // Vector<ushort> constant to be subtracted is 2^15
1466             // Vector<uint> constant to be subtracted is 2^31
1467             // Vector<ulong> constant to be subtracted is 2^63
1468             //
1469             // We need to treat op1 and op2 as signed for comparison purpose after
1470             // the transformation.
1471             __int64 constVal = 0;
1472             switch (baseType)
1473             {
1474                 case TYP_UBYTE:
1475                     constVal       = 0x80808080;
1476                     *inOutBaseType = TYP_BYTE;
1477                     break;
1478                 case TYP_USHORT:
1479                     constVal       = 0x80008000;
1480                     *inOutBaseType = TYP_SHORT;
1481                     break;
1482                 case TYP_UINT:
1483                     constVal       = 0x80000000;
1484                     *inOutBaseType = TYP_INT;
1485                     break;
1486                 case TYP_ULONG:
1487                     constVal       = 0x8000000000000000LL;
1488                     *inOutBaseType = TYP_LONG;
1489                     break;
1490                 default:
1491                     unreached();
1492                     break;
1493             }
1494             assert(constVal != 0);
1495
1496             // This transformation is not required for equality.
1497             if (intrinsicID != SIMDIntrinsicEqual)
1498             {
1499                 // For constructing const vector use either long or int base type.
1500                 var_types tempBaseType;
1501                 GenTree*  initVal;
1502                 if (baseType == TYP_ULONG)
1503                 {
1504                     tempBaseType = TYP_LONG;
1505                     initVal      = gtNewLconNode(constVal);
1506                 }
1507                 else
1508                 {
1509                     tempBaseType = TYP_INT;
1510                     initVal      = gtNewIconNode((ssize_t)constVal);
1511                 }
1512                 initVal->gtType      = tempBaseType;
1513                 GenTree* constVector = gtNewSIMDNode(simdType, initVal, nullptr, SIMDIntrinsicInit, tempBaseType, size);
1514
1515                 // Assign constVector to a temp, since we intend to use it more than once
1516                 // TODO-CQ: We have quite a few such constant vectors constructed during
1517                 // the importation of SIMD intrinsics.  Make sure that we have a single
1518                 // temp per distinct constant per method.
1519                 GenTree* tmp = fgInsertCommaFormTemp(&constVector, typeHnd);
1520
1521                 // op1 = op1 - constVector
1522                 // op2 = op2 - constVector
1523                 *pOp1 = gtNewSIMDNode(simdType, *pOp1, constVector, SIMDIntrinsicSub, baseType, size);
1524                 *pOp2 = gtNewSIMDNode(simdType, *pOp2, tmp, SIMDIntrinsicSub, baseType, size);
1525             }
1526
1527             return impSIMDRelOp(intrinsicID, typeHnd, size, inOutBaseType, pOp1, pOp2);
1528         }
1529     }
1530 #elif defined(_TARGET_ARM64_)
1531     // TODO-ARM64-CQ handle comparisons against zero
1532
1533     // _TARGET_ARM64_ doesn't support < and <= on register register comparisons
1534     // Therefore, we need to use > and >= with swapped operands.
1535     if (intrinsicID == SIMDIntrinsicLessThan || intrinsicID == SIMDIntrinsicLessThanOrEqual)
1536     {
1537         GenTree* tmp = *pOp1;
1538         *pOp1        = *pOp2;
1539         *pOp2        = tmp;
1540
1541         intrinsicID =
1542             (intrinsicID == SIMDIntrinsicLessThan) ? SIMDIntrinsicGreaterThan : SIMDIntrinsicGreaterThanOrEqual;
1543     }
1544 #else  // !_TARGET_XARCH_
1545     assert(!"impSIMDRelOp() unimplemented on target arch");
1546     unreached();
1547 #endif // !_TARGET_XARCH_
1548
1549     return intrinsicID;
1550 }
1551
1552 //-------------------------------------------------------------------------
1553 // impSIMDAbs: creates GT_SIMD node to compute Abs value of a given vector.
1554 //
1555 // Arguments:
1556 //    typeHnd     -  type handle of SIMD vector
1557 //    baseType    -  base type of vector
1558 //    size        -  vector size in bytes
1559 //    op1         -  operand of Abs intrinsic
1560 //
1561 GenTree* Compiler::impSIMDAbs(CORINFO_CLASS_HANDLE typeHnd, var_types baseType, unsigned size, GenTree* op1)
1562 {
1563     assert(varTypeIsSIMD(op1));
1564
1565     var_types simdType = op1->TypeGet();
1566     GenTree*  retVal   = nullptr;
1567
1568 #ifdef _TARGET_XARCH_
1569     // When there is no direct support, Abs(v) could be computed
1570     // on integer vectors as follows:
1571     //     BitVector = v < vector.Zero
1572     //     result = ConditionalSelect(BitVector, vector.Zero - v, v)
1573
1574     bool useConditionalSelect = false;
1575     if (getSIMDSupportLevel() == SIMD_SSE2_Supported)
1576     {
1577         // SSE2 doesn't support abs on signed integer type vectors.
1578         if (baseType == TYP_LONG || baseType == TYP_INT || baseType == TYP_SHORT || baseType == TYP_BYTE)
1579         {
1580             useConditionalSelect = true;
1581         }
1582     }
1583     else
1584     {
1585         assert(getSIMDSupportLevel() >= SIMD_SSE4_Supported);
1586         if (baseType == TYP_LONG)
1587         {
1588             // SSE4/AVX2 don't support abs on long type vector.
1589             useConditionalSelect = true;
1590         }
1591     }
1592
1593     if (useConditionalSelect)
1594     {
1595         // This works only on integer vectors not on float/double vectors.
1596         assert(varTypeIsIntegral(baseType));
1597
1598         GenTree* op1Assign;
1599         unsigned op1LclNum;
1600
1601         if (op1->OperGet() == GT_LCL_VAR)
1602         {
1603             op1LclNum = op1->gtLclVarCommon.gtLclNum;
1604             op1Assign = nullptr;
1605         }
1606         else
1607         {
1608             op1LclNum = lvaGrabTemp(true DEBUGARG("SIMD Abs op1"));
1609             lvaSetStruct(op1LclNum, typeHnd, false);
1610             op1Assign = gtNewTempAssign(op1LclNum, op1);
1611             op1       = gtNewLclvNode(op1LclNum, op1->TypeGet());
1612         }
1613
1614         // Assign Vector.Zero to a temp since it is needed more than once
1615         GenTree* vecZero       = gtNewSIMDVectorZero(simdType, baseType, size);
1616         unsigned vecZeroLclNum = lvaGrabTemp(true DEBUGARG("SIMD Abs VecZero"));
1617         lvaSetStruct(vecZeroLclNum, typeHnd, false);
1618         GenTree* vecZeroAssign = gtNewTempAssign(vecZeroLclNum, vecZero);
1619
1620         // Construct BitVector = v < vector.Zero
1621         GenTree*        bitVecOp1     = op1;
1622         GenTree*        bitVecOp2     = gtNewLclvNode(vecZeroLclNum, vecZero->TypeGet());
1623         var_types       relOpBaseType = baseType;
1624         SIMDIntrinsicID relOpIntrinsic =
1625             impSIMDRelOp(SIMDIntrinsicLessThan, typeHnd, size, &relOpBaseType, &bitVecOp1, &bitVecOp2);
1626         GenTree* bitVec       = gtNewSIMDNode(simdType, bitVecOp1, bitVecOp2, relOpIntrinsic, relOpBaseType, size);
1627         unsigned bitVecLclNum = lvaGrabTemp(true DEBUGARG("SIMD Abs bitVec"));
1628         lvaSetStruct(bitVecLclNum, typeHnd, false);
1629         GenTree* bitVecAssign = gtNewTempAssign(bitVecLclNum, bitVec);
1630         bitVec                = gtNewLclvNode(bitVecLclNum, bitVec->TypeGet());
1631
1632         // Construct condSelectOp1 = vector.Zero - v
1633         GenTree* subOp1 = gtNewLclvNode(vecZeroLclNum, vecZero->TypeGet());
1634         GenTree* subOp2 = gtNewLclvNode(op1LclNum, op1->TypeGet());
1635         GenTree* negVec = gtNewSIMDNode(simdType, subOp1, subOp2, SIMDIntrinsicSub, baseType, size);
1636
1637         // Construct ConditionalSelect(bitVec, vector.Zero - v, v)
1638         GenTree* vec = gtNewLclvNode(op1LclNum, op1->TypeGet());
1639         retVal       = impSIMDSelect(typeHnd, baseType, size, bitVec, negVec, vec);
1640
1641         // Prepend bitVec assignment to retVal.
1642         // retVal = (tmp2 = v < tmp1), CondSelect(tmp2, tmp1 - v, v)
1643         retVal = gtNewOperNode(GT_COMMA, simdType, bitVecAssign, retVal);
1644
1645         // Prepend vecZero assignment to retVal.
1646         // retVal =  (tmp1 = vector.Zero), (tmp2 = v < tmp1), CondSelect(tmp2, tmp1 - v, v)
1647         retVal = gtNewOperNode(GT_COMMA, simdType, vecZeroAssign, retVal);
1648
1649         // If op1 was assigned to a temp, prepend that to retVal.
1650         if (op1Assign != nullptr)
1651         {
1652             // retVal = (v=op1), (tmp1 = vector.Zero), (tmp2 = v < tmp1), CondSelect(tmp2, tmp1 - v, v)
1653             retVal = gtNewOperNode(GT_COMMA, simdType, op1Assign, retVal);
1654         }
1655     }
1656     else if (varTypeIsFloating(baseType))
1657     {
1658         // Abs(vf) = vf & new SIMDVector<float>(0x7fffffff);
1659         // Abs(vd) = vf & new SIMDVector<double>(0x7fffffffffffffff);
1660         GenTree* bitMask = nullptr;
1661         if (baseType == TYP_FLOAT)
1662         {
1663             float f;
1664             static_assert_no_msg(sizeof(float) == sizeof(int));
1665             *((int*)&f) = 0x7fffffff;
1666             bitMask     = gtNewDconNode(f);
1667         }
1668         else if (baseType == TYP_DOUBLE)
1669         {
1670             double d;
1671             static_assert_no_msg(sizeof(double) == sizeof(__int64));
1672             *((__int64*)&d) = 0x7fffffffffffffffLL;
1673             bitMask         = gtNewDconNode(d);
1674         }
1675
1676         assert(bitMask != nullptr);
1677         bitMask->gtType        = baseType;
1678         GenTree* bitMaskVector = gtNewSIMDNode(simdType, bitMask, SIMDIntrinsicInit, baseType, size);
1679         retVal                 = gtNewSIMDNode(simdType, op1, bitMaskVector, SIMDIntrinsicBitwiseAnd, baseType, size);
1680     }
1681     else if (baseType == TYP_USHORT || baseType == TYP_UBYTE || baseType == TYP_UINT || baseType == TYP_ULONG)
1682     {
1683         // Abs is a no-op on unsigned integer type vectors
1684         retVal = op1;
1685     }
1686     else
1687     {
1688         assert(getSIMDSupportLevel() >= SIMD_SSE4_Supported);
1689         assert(baseType != TYP_LONG);
1690
1691         retVal = gtNewSIMDNode(simdType, op1, SIMDIntrinsicAbs, baseType, size);
1692     }
1693 #elif defined(_TARGET_ARM64_)
1694     if (varTypeIsUnsigned(baseType))
1695     {
1696         // Abs is a no-op on unsigned integer type vectors
1697         retVal = op1;
1698     }
1699     else
1700     {
1701         retVal = gtNewSIMDNode(simdType, op1, SIMDIntrinsicAbs, baseType, size);
1702     }
1703 #else  // !defined(_TARGET_XARCH)_ && !defined(_TARGET_ARM64_)
1704     assert(!"Abs intrinsic on non-xarch target not implemented");
1705 #endif // !_TARGET_XARCH_
1706
1707     return retVal;
1708 }
1709
1710 // Creates a GT_SIMD tree for Select operation
1711 //
1712 // Argumens:
1713 //    typeHnd          -  type handle of SIMD vector
1714 //    baseType         -  base type of SIMD vector
1715 //    size             -  SIMD vector size
1716 //    op1              -  first operand = Condition vector vc
1717 //    op2              -  second operand = va
1718 //    op3              -  third operand = vb
1719 //
1720 // Return Value:
1721 //    Returns GT_SIMD tree that computes Select(vc, va, vb)
1722 //
1723 GenTree* Compiler::impSIMDSelect(
1724     CORINFO_CLASS_HANDLE typeHnd, var_types baseType, unsigned size, GenTree* op1, GenTree* op2, GenTree* op3)
1725 {
1726     assert(varTypeIsSIMD(op1));
1727     var_types simdType = op1->TypeGet();
1728     assert(op2->TypeGet() == simdType);
1729     assert(op3->TypeGet() == simdType);
1730
1731     // TODO-ARM64-CQ Support generating select instruction for SIMD
1732
1733     // Select(BitVector vc, va, vb) = (va & vc) | (vb & !vc)
1734     // Select(op1, op2, op3)        = (op2 & op1) | (op3 & !op1)
1735     //                              = SIMDIntrinsicBitwiseOr(SIMDIntrinsicBitwiseAnd(op2, op1),
1736     //                                                       SIMDIntrinsicBitwiseAndNot(op3, op1))
1737     //
1738     // If Op1 has side effect, create an assignment to a temp
1739     GenTree* tmp = op1;
1740     GenTree* asg = nullptr;
1741     if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0)
1742     {
1743         unsigned lclNum = lvaGrabTemp(true DEBUGARG("SIMD Select"));
1744         lvaSetStruct(lclNum, typeHnd, false);
1745         tmp = gtNewLclvNode(lclNum, op1->TypeGet());
1746         asg = gtNewTempAssign(lclNum, op1);
1747     }
1748
1749     GenTree* andExpr = gtNewSIMDNode(simdType, op2, tmp, SIMDIntrinsicBitwiseAnd, baseType, size);
1750     GenTree* dupOp1  = gtCloneExpr(tmp);
1751     assert(dupOp1 != nullptr);
1752 #ifdef _TARGET_ARM64_
1753     // ARM64 implements SIMDIntrinsicBitwiseAndNot as Left & ~Right
1754     GenTree* andNotExpr = gtNewSIMDNode(simdType, op3, dupOp1, SIMDIntrinsicBitwiseAndNot, baseType, size);
1755 #else
1756     // XARCH implements SIMDIntrinsicBitwiseAndNot as ~Left & Right
1757     GenTree* andNotExpr = gtNewSIMDNode(simdType, dupOp1, op3, SIMDIntrinsicBitwiseAndNot, baseType, size);
1758 #endif
1759     GenTree* simdTree = gtNewSIMDNode(simdType, andExpr, andNotExpr, SIMDIntrinsicBitwiseOr, baseType, size);
1760
1761     // If asg not null, create a GT_COMMA tree.
1762     if (asg != nullptr)
1763     {
1764         simdTree = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), asg, simdTree);
1765     }
1766
1767     return simdTree;
1768 }
1769
1770 // Creates a GT_SIMD tree for Min/Max operation
1771 //
1772 // Argumens:
1773 //    IntrinsicId      -  SIMD intrinsic Id, either Min or Max
1774 //    typeHnd          -  type handle of SIMD vector
1775 //    baseType         -  base type of SIMD vector
1776 //    size             -  SIMD vector size
1777 //    op1              -  first operand = va
1778 //    op2              -  second operand = vb
1779 //
1780 // Return Value:
1781 //    Returns GT_SIMD tree that computes Max(va, vb)
1782 //
1783 GenTree* Compiler::impSIMDMinMax(SIMDIntrinsicID      intrinsicId,
1784                                  CORINFO_CLASS_HANDLE typeHnd,
1785                                  var_types            baseType,
1786                                  unsigned             size,
1787                                  GenTree*             op1,
1788                                  GenTree*             op2)
1789 {
1790     assert(intrinsicId == SIMDIntrinsicMin || intrinsicId == SIMDIntrinsicMax);
1791     assert(varTypeIsSIMD(op1));
1792     var_types simdType = op1->TypeGet();
1793     assert(op2->TypeGet() == simdType);
1794
1795 #if defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
1796     GenTree* simdTree = nullptr;
1797
1798 #ifdef _TARGET_XARCH_
1799     // SSE2 has direct support for float/double/signed word/unsigned byte.
1800     // SSE4.1 has direct support for int32/uint32/signed byte/unsigned word.
1801     // For other integer types we compute min/max as follows
1802     //
1803     // int32/uint32 (SSE2)
1804     // int64/uint64 (SSE2&SSE4):
1805     //       compResult        = (op1 < op2) in case of Min
1806     //                           (op1 > op2) in case of Max
1807     //       Min/Max(op1, op2) = Select(compResult, op1, op2)
1808     //
1809     // unsigned word (SSE2):
1810     //        op1 = op1 - 2^15  ; to make it fit within a signed word
1811     //        op2 = op2 - 2^15  ; to make it fit within a signed word
1812     //        result = SSE2 signed word Min/Max(op1, op2)
1813     //        result = result + 2^15  ; readjust it back
1814     //
1815     // signed byte (SSE2):
1816     //        op1 = op1 + 2^7  ; to make it unsigned
1817     //        op1 = op1 + 2^7  ; to make it unsigned
1818     //        result = SSE2 unsigned byte Min/Max(op1, op2)
1819     //        result = result - 2^15 ; readjust it back
1820
1821     if (varTypeIsFloating(baseType) || baseType == TYP_SHORT || baseType == TYP_UBYTE ||
1822         (getSIMDSupportLevel() >= SIMD_SSE4_Supported &&
1823          (baseType == TYP_BYTE || baseType == TYP_INT || baseType == TYP_UINT || baseType == TYP_USHORT)))
1824     {
1825         // SSE2 or SSE4.1 has direct support
1826         simdTree = gtNewSIMDNode(simdType, op1, op2, intrinsicId, baseType, size);
1827     }
1828     else if (baseType == TYP_USHORT || baseType == TYP_BYTE)
1829     {
1830         assert(getSIMDSupportLevel() == SIMD_SSE2_Supported);
1831         int             constVal;
1832         SIMDIntrinsicID operIntrinsic;
1833         SIMDIntrinsicID adjustIntrinsic;
1834         var_types       minMaxOperBaseType;
1835         if (baseType == TYP_USHORT)
1836         {
1837             constVal           = 0x80008000;
1838             operIntrinsic      = SIMDIntrinsicSub;
1839             adjustIntrinsic    = SIMDIntrinsicAdd;
1840             minMaxOperBaseType = TYP_SHORT;
1841         }
1842         else
1843         {
1844             assert(baseType == TYP_BYTE);
1845             constVal           = 0x80808080;
1846             operIntrinsic      = SIMDIntrinsicAdd;
1847             adjustIntrinsic    = SIMDIntrinsicSub;
1848             minMaxOperBaseType = TYP_UBYTE;
1849         }
1850
1851         GenTree* initVal     = gtNewIconNode(constVal);
1852         GenTree* constVector = gtNewSIMDNode(simdType, initVal, nullptr, SIMDIntrinsicInit, TYP_INT, size);
1853
1854         // Assign constVector to a temp, since we intend to use it more than once
1855         // TODO-CQ: We have quite a few such constant vectors constructed during
1856         // the importation of SIMD intrinsics.  Make sure that we have a single
1857         // temp per distinct constant per method.
1858         GenTree* tmp = fgInsertCommaFormTemp(&constVector, typeHnd);
1859
1860         // op1 = op1 - constVector
1861         // op2 = op2 - constVector
1862         op1 = gtNewSIMDNode(simdType, op1, constVector, operIntrinsic, baseType, size);
1863         op2 = gtNewSIMDNode(simdType, op2, tmp, operIntrinsic, baseType, size);
1864
1865         // compute min/max of op1 and op2 considering them as if minMaxOperBaseType
1866         simdTree = gtNewSIMDNode(simdType, op1, op2, intrinsicId, minMaxOperBaseType, size);
1867
1868         // re-adjust the value by adding or subtracting constVector
1869         tmp      = gtNewLclvNode(tmp->AsLclVarCommon()->GetLclNum(), tmp->TypeGet());
1870         simdTree = gtNewSIMDNode(simdType, simdTree, tmp, adjustIntrinsic, baseType, size);
1871     }
1872 #elif defined(_TARGET_ARM64_)
1873     // Arm64 has direct support for all types except int64/uint64
1874     // For which we compute min/max as follows
1875     //
1876     // int64/uint64
1877     //       compResult        = (op1 < op2) in case of Min
1878     //                           (op1 > op2) in case of Max
1879     //       Min/Max(op1, op2) = Select(compResult, op1, op2)
1880     if (baseType != TYP_ULONG && baseType != TYP_LONG)
1881     {
1882         simdTree = gtNewSIMDNode(simdType, op1, op2, intrinsicId, baseType, size);
1883     }
1884 #endif
1885     else
1886     {
1887         GenTree* dupOp1    = nullptr;
1888         GenTree* dupOp2    = nullptr;
1889         GenTree* op1Assign = nullptr;
1890         GenTree* op2Assign = nullptr;
1891         unsigned op1LclNum;
1892         unsigned op2LclNum;
1893
1894         if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0)
1895         {
1896             op1LclNum = lvaGrabTemp(true DEBUGARG("SIMD Min/Max"));
1897             dupOp1    = gtNewLclvNode(op1LclNum, op1->TypeGet());
1898             lvaSetStruct(op1LclNum, typeHnd, false);
1899             op1Assign = gtNewTempAssign(op1LclNum, op1);
1900             op1       = gtNewLclvNode(op1LclNum, op1->TypeGet());
1901         }
1902         else
1903         {
1904             dupOp1 = gtCloneExpr(op1);
1905         }
1906
1907         if ((op2->gtFlags & GTF_SIDE_EFFECT) != 0)
1908         {
1909             op2LclNum = lvaGrabTemp(true DEBUGARG("SIMD Min/Max"));
1910             dupOp2    = gtNewLclvNode(op2LclNum, op2->TypeGet());
1911             lvaSetStruct(op2LclNum, typeHnd, false);
1912             op2Assign = gtNewTempAssign(op2LclNum, op2);
1913             op2       = gtNewLclvNode(op2LclNum, op2->TypeGet());
1914         }
1915         else
1916         {
1917             dupOp2 = gtCloneExpr(op2);
1918         }
1919
1920         SIMDIntrinsicID relOpIntrinsic =
1921             (intrinsicId == SIMDIntrinsicMin) ? SIMDIntrinsicLessThan : SIMDIntrinsicGreaterThan;
1922         var_types relOpBaseType = baseType;
1923
1924         // compResult = op1 relOp op2
1925         // simdTree = Select(compResult, op1, op2);
1926         assert(dupOp1 != nullptr);
1927         assert(dupOp2 != nullptr);
1928         relOpIntrinsic            = impSIMDRelOp(relOpIntrinsic, typeHnd, size, &relOpBaseType, &dupOp1, &dupOp2);
1929         GenTree* compResult       = gtNewSIMDNode(simdType, dupOp1, dupOp2, relOpIntrinsic, relOpBaseType, size);
1930         unsigned compResultLclNum = lvaGrabTemp(true DEBUGARG("SIMD Min/Max"));
1931         lvaSetStruct(compResultLclNum, typeHnd, false);
1932         GenTree* compResultAssign = gtNewTempAssign(compResultLclNum, compResult);
1933         compResult                = gtNewLclvNode(compResultLclNum, compResult->TypeGet());
1934         simdTree                  = impSIMDSelect(typeHnd, baseType, size, compResult, op1, op2);
1935         simdTree                  = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), compResultAssign, simdTree);
1936
1937         // Now create comma trees if we have created assignments of op1/op2 to temps
1938         if (op2Assign != nullptr)
1939         {
1940             simdTree = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), op2Assign, simdTree);
1941         }
1942
1943         if (op1Assign != nullptr)
1944         {
1945             simdTree = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), op1Assign, simdTree);
1946         }
1947     }
1948
1949     assert(simdTree != nullptr);
1950     return simdTree;
1951 #else  // !(defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_))
1952     assert(!"impSIMDMinMax() unimplemented on target arch");
1953     unreached();
1954 #endif // !(defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_))
1955 }
1956
1957 //------------------------------------------------------------------------
1958 // getOp1ForConstructor: Get the op1 for a constructor call.
1959 //
1960 // Arguments:
1961 //    opcode     - the opcode being handled (needed to identify the CEE_NEWOBJ case)
1962 //    newobjThis - For CEE_NEWOBJ, this is the temp grabbed for the allocated uninitalized object.
1963 //    clsHnd    - The handle of the class of the method.
1964 //
1965 // Return Value:
1966 //    The tree node representing the object to be initialized with the constructor.
1967 //
1968 // Notes:
1969 //    This method handles the differences between the CEE_NEWOBJ and constructor cases.
1970 //
1971 GenTree* Compiler::getOp1ForConstructor(OPCODE opcode, GenTree* newobjThis, CORINFO_CLASS_HANDLE clsHnd)
1972 {
1973     GenTree* op1;
1974     if (opcode == CEE_NEWOBJ)
1975     {
1976         op1 = newobjThis;
1977         assert(newobjThis->gtOper == GT_ADDR && newobjThis->gtOp.gtOp1->gtOper == GT_LCL_VAR);
1978
1979         // push newobj result on type stack
1980         unsigned tmp = op1->gtOp.gtOp1->gtLclVarCommon.gtLclNum;
1981         impPushOnStack(gtNewLclvNode(tmp, lvaGetRealType(tmp)), verMakeTypeInfo(clsHnd).NormaliseForStack());
1982     }
1983     else
1984     {
1985         op1 = impSIMDPopStack(TYP_BYREF);
1986     }
1987     assert(op1->TypeGet() == TYP_BYREF);
1988     return op1;
1989 }
1990
1991 //-------------------------------------------------------------------
1992 // Set the flag that indicates that the lclVar referenced by this tree
1993 // is used in a SIMD intrinsic.
1994 // Arguments:
1995 //      tree - GenTree*
1996
1997 void Compiler::setLclRelatedToSIMDIntrinsic(GenTree* tree)
1998 {
1999     assert(tree->OperIsLocal());
2000     unsigned   lclNum                = tree->AsLclVarCommon()->GetLclNum();
2001     LclVarDsc* lclVarDsc             = &lvaTable[lclNum];
2002     lclVarDsc->lvUsedInSIMDIntrinsic = true;
2003 }
2004
2005 //-------------------------------------------------------------
2006 // Check if two field nodes reference at the same memory location.
2007 // Notice that this check is just based on pattern matching.
2008 // Arguments:
2009 //      op1 - GenTree*.
2010 //      op2 - GenTree*.
2011 // Return Value:
2012 //    If op1's parents node and op2's parents node are at the same location, return true. Otherwise, return false
2013
2014 bool areFieldsParentsLocatedSame(GenTree* op1, GenTree* op2)
2015 {
2016     assert(op1->OperGet() == GT_FIELD);
2017     assert(op2->OperGet() == GT_FIELD);
2018
2019     GenTree* op1ObjRef = op1->gtField.gtFldObj;
2020     GenTree* op2ObjRef = op2->gtField.gtFldObj;
2021     while (op1ObjRef != nullptr && op2ObjRef != nullptr)
2022     {
2023
2024         if (op1ObjRef->OperGet() != op2ObjRef->OperGet())
2025         {
2026             break;
2027         }
2028         else if (op1ObjRef->OperGet() == GT_ADDR)
2029         {
2030             op1ObjRef = op1ObjRef->gtOp.gtOp1;
2031             op2ObjRef = op2ObjRef->gtOp.gtOp1;
2032         }
2033
2034         if (op1ObjRef->OperIsLocal() && op2ObjRef->OperIsLocal() &&
2035             op1ObjRef->AsLclVarCommon()->GetLclNum() == op2ObjRef->AsLclVarCommon()->GetLclNum())
2036         {
2037             return true;
2038         }
2039         else if (op1ObjRef->OperGet() == GT_FIELD && op2ObjRef->OperGet() == GT_FIELD &&
2040                  op1ObjRef->gtField.gtFldHnd == op2ObjRef->gtField.gtFldHnd)
2041         {
2042             op1ObjRef = op1ObjRef->gtField.gtFldObj;
2043             op2ObjRef = op2ObjRef->gtField.gtFldObj;
2044             continue;
2045         }
2046         else
2047         {
2048             break;
2049         }
2050     }
2051
2052     return false;
2053 }
2054
2055 //----------------------------------------------------------------------
2056 // Check whether two field are contiguous
2057 // Arguments:
2058 //      first - GenTree*. The Type of the node should be TYP_FLOAT
2059 //      second - GenTree*. The Type of the node should be TYP_FLOAT
2060 // Return Value:
2061 //      if the first field is located before second field, and they are located contiguously,
2062 //      then return true. Otherwise, return false.
2063
2064 bool Compiler::areFieldsContiguous(GenTree* first, GenTree* second)
2065 {
2066     assert(first->OperGet() == GT_FIELD);
2067     assert(second->OperGet() == GT_FIELD);
2068     assert(first->gtType == TYP_FLOAT);
2069     assert(second->gtType == TYP_FLOAT);
2070
2071     var_types firstFieldType  = first->gtType;
2072     var_types secondFieldType = second->gtType;
2073
2074     unsigned firstFieldEndOffset = first->gtField.gtFldOffset + genTypeSize(firstFieldType);
2075     unsigned secondFieldOffset   = second->gtField.gtFldOffset;
2076     if (firstFieldEndOffset == secondFieldOffset && firstFieldType == secondFieldType &&
2077         areFieldsParentsLocatedSame(first, second))
2078     {
2079         return true;
2080     }
2081
2082     return false;
2083 }
2084
2085 //-------------------------------------------------------------------------------
2086 // Check whether two array element nodes are located contiguously or not.
2087 // Arguments:
2088 //      op1 - GenTree*.
2089 //      op2 - GenTree*.
2090 // Return Value:
2091 //      if the array element op1 is located before array element op2, and they are contiguous,
2092 //      then return true. Otherwise, return false.
2093 // TODO-CQ:
2094 //      Right this can only check array element with const number as index. In future,
2095 //      we should consider to allow this function to check the index using expression.
2096
2097 bool Compiler::areArrayElementsContiguous(GenTree* op1, GenTree* op2)
2098 {
2099     noway_assert(op1->gtOper == GT_INDEX);
2100     noway_assert(op2->gtOper == GT_INDEX);
2101     GenTreeIndex* op1Index = op1->AsIndex();
2102     GenTreeIndex* op2Index = op2->AsIndex();
2103
2104     GenTree* op1ArrayRef = op1Index->Arr();
2105     GenTree* op2ArrayRef = op2Index->Arr();
2106     assert(op1ArrayRef->TypeGet() == TYP_REF);
2107     assert(op2ArrayRef->TypeGet() == TYP_REF);
2108
2109     GenTree* op1IndexNode = op1Index->Index();
2110     GenTree* op2IndexNode = op2Index->Index();
2111     if ((op1IndexNode->OperGet() == GT_CNS_INT && op2IndexNode->OperGet() == GT_CNS_INT) &&
2112         op1IndexNode->gtIntCon.gtIconVal + 1 == op2IndexNode->gtIntCon.gtIconVal)
2113     {
2114         if (op1ArrayRef->OperGet() == GT_FIELD && op2ArrayRef->OperGet() == GT_FIELD &&
2115             areFieldsParentsLocatedSame(op1ArrayRef, op2ArrayRef))
2116         {
2117             return true;
2118         }
2119         else if (op1ArrayRef->OperIsLocal() && op2ArrayRef->OperIsLocal() &&
2120                  op1ArrayRef->AsLclVarCommon()->GetLclNum() == op2ArrayRef->AsLclVarCommon()->GetLclNum())
2121         {
2122             return true;
2123         }
2124     }
2125     return false;
2126 }
2127
2128 //-------------------------------------------------------------------------------
2129 // Check whether two argument nodes are contiguous or not.
2130 // Arguments:
2131 //      op1 - GenTree*.
2132 //      op2 - GenTree*.
2133 // Return Value:
2134 //      if the argument node op1 is located before argument node op2, and they are located contiguously,
2135 //      then return true. Otherwise, return false.
2136 // TODO-CQ:
2137 //      Right now this can only check field and array. In future we should add more cases.
2138 //
2139
2140 bool Compiler::areArgumentsContiguous(GenTree* op1, GenTree* op2)
2141 {
2142     if (op1->OperGet() == GT_INDEX && op2->OperGet() == GT_INDEX)
2143     {
2144         return areArrayElementsContiguous(op1, op2);
2145     }
2146     else if (op1->OperGet() == GT_FIELD && op2->OperGet() == GT_FIELD)
2147     {
2148         return areFieldsContiguous(op1, op2);
2149     }
2150     return false;
2151 }
2152
2153 //--------------------------------------------------------------------------------------------------------
2154 // createAddressNodeForSIMDInit: Generate the address node(GT_LEA) if we want to intialize vector2, vector3 or vector4
2155 // from first argument's address.
2156 //
2157 // Arguments:
2158 //      tree - GenTree*. This the tree node which is used to get the address for indir.
2159 //      simdsize - unsigned. This the simd vector size.
2160 //      arrayElementsCount - unsigned. This is used for generating the boundary check for array.
2161 //
2162 // Return value:
2163 //      return the address node.
2164 //
2165 // TODO-CQ:
2166 //      1. Currently just support for GT_FIELD and GT_INDEX, because we can only verify the GT_INDEX node or GT_Field
2167 //         are located contiguously or not. In future we should support more cases.
2168 //      2. Though it happens to just work fine front-end phases are not aware of GT_LEA node.  Therefore, convert these
2169 //         to use GT_ADDR.
2170 GenTree* Compiler::createAddressNodeForSIMDInit(GenTree* tree, unsigned simdSize)
2171 {
2172     assert(tree->OperGet() == GT_FIELD || tree->OperGet() == GT_INDEX);
2173     GenTree*  byrefNode  = nullptr;
2174     GenTree*  startIndex = nullptr;
2175     unsigned  offset     = 0;
2176     var_types baseType   = tree->gtType;
2177
2178     if (tree->OperGet() == GT_FIELD)
2179     {
2180         GenTree* objRef = tree->gtField.gtFldObj;
2181         if (objRef != nullptr && objRef->gtOper == GT_ADDR)
2182         {
2183             GenTree* obj = objRef->gtOp.gtOp1;
2184
2185             // If the field is directly from a struct, then in this case,
2186             // we should set this struct's lvUsedInSIMDIntrinsic as true,
2187             // so that this sturct won't be promoted.
2188             // e.g. s.x x is a field, and s is a struct, then we should set the s's lvUsedInSIMDIntrinsic as true.
2189             // so that s won't be promoted.
2190             // Notice that if we have a case like s1.s2.x. s1 s2 are struct, and x is a field, then it is possible that
2191             // s1 can be promoted, so that s2 can be promoted. The reason for that is if we don't allow s1 to be
2192             // promoted, then this will affect the other optimizations which are depend on s1's struct promotion.
2193             // TODO-CQ:
2194             //  In future, we should optimize this case so that if there is a nested field like s1.s2.x and s1.s2.x's
2195             //  address is used for initializing the vector, then s1 can be promoted but s2 can't.
2196             if (varTypeIsSIMD(obj) && obj->OperIsLocal())
2197             {
2198                 setLclRelatedToSIMDIntrinsic(obj);
2199             }
2200         }
2201
2202         byrefNode = gtCloneExpr(tree->gtField.gtFldObj);
2203         assert(byrefNode != nullptr);
2204         offset = tree->gtField.gtFldOffset;
2205     }
2206     else if (tree->OperGet() == GT_INDEX)
2207     {
2208
2209         GenTree* index = tree->AsIndex()->Index();
2210         assert(index->OperGet() == GT_CNS_INT);
2211
2212         GenTree* checkIndexExpr = nullptr;
2213         unsigned indexVal       = (unsigned)(index->gtIntCon.gtIconVal);
2214         offset                  = indexVal * genTypeSize(tree->TypeGet());
2215         GenTree* arrayRef       = tree->AsIndex()->Arr();
2216
2217         // Generate the boundary check exception.
2218         // The length for boundary check should be the maximum index number which should be
2219         // (first argument's index number) + (how many array arguments we have) - 1
2220         // = indexVal + arrayElementsCount - 1
2221         unsigned arrayElementsCount  = simdSize / genTypeSize(baseType);
2222         checkIndexExpr               = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, indexVal + arrayElementsCount - 1);
2223         GenTreeArrLen*    arrLen     = gtNewArrLen(TYP_INT, arrayRef, (int)OFFSETOF__CORINFO_Array__length);
2224         GenTreeBoundsChk* arrBndsChk = new (this, GT_ARR_BOUNDS_CHECK)
2225             GenTreeBoundsChk(GT_ARR_BOUNDS_CHECK, TYP_VOID, checkIndexExpr, arrLen, SCK_RNGCHK_FAIL);
2226
2227         offset += OFFSETOF__CORINFO_Array__data;
2228         byrefNode = gtNewOperNode(GT_COMMA, arrayRef->TypeGet(), arrBndsChk, gtCloneExpr(arrayRef));
2229     }
2230     else
2231     {
2232         unreached();
2233     }
2234     GenTree* address =
2235         new (this, GT_LEA) GenTreeAddrMode(TYP_BYREF, byrefNode, startIndex, genTypeSize(tree->TypeGet()), offset);
2236     return address;
2237 }
2238
2239 //-------------------------------------------------------------------------------
2240 // impMarkContiguousSIMDFieldAssignments: Try to identify if there are contiguous
2241 // assignments from SIMD field to memory. If there are, then mark the related
2242 // lclvar so that it won't be promoted.
2243 //
2244 // Arguments:
2245 //      stmt - GenTree*. Input statement node.
2246
2247 void Compiler::impMarkContiguousSIMDFieldAssignments(GenTree* stmt)
2248 {
2249     if (!featureSIMD || opts.MinOpts())
2250     {
2251         return;
2252     }
2253     GenTree* expr = stmt->gtStmt.gtStmtExpr;
2254     if (expr->OperGet() == GT_ASG && expr->TypeGet() == TYP_FLOAT)
2255     {
2256         GenTree*  curDst            = expr->gtOp.gtOp1;
2257         GenTree*  curSrc            = expr->gtOp.gtOp2;
2258         unsigned  index             = 0;
2259         var_types baseType          = TYP_UNKNOWN;
2260         unsigned  simdSize          = 0;
2261         GenTree*  srcSimdStructNode = getSIMDStructFromField(curSrc, &baseType, &index, &simdSize, true);
2262         if (srcSimdStructNode == nullptr || baseType != TYP_FLOAT)
2263         {
2264             fgPreviousCandidateSIMDFieldAsgStmt = nullptr;
2265         }
2266         else if (index == 0 && isSIMDTypeLocal(srcSimdStructNode))
2267         {
2268             fgPreviousCandidateSIMDFieldAsgStmt = stmt;
2269         }
2270         else if (fgPreviousCandidateSIMDFieldAsgStmt != nullptr)
2271         {
2272             assert(index > 0);
2273             GenTree* prevAsgExpr = fgPreviousCandidateSIMDFieldAsgStmt->gtStmt.gtStmtExpr;
2274             GenTree* prevDst     = prevAsgExpr->gtOp.gtOp1;
2275             GenTree* prevSrc     = prevAsgExpr->gtOp.gtOp2;
2276             if (!areArgumentsContiguous(prevDst, curDst) || !areArgumentsContiguous(prevSrc, curSrc))
2277             {
2278                 fgPreviousCandidateSIMDFieldAsgStmt = nullptr;
2279             }
2280             else
2281             {
2282                 if (index == (simdSize / genTypeSize(baseType) - 1))
2283                 {
2284                     // Successfully found the pattern, mark the lclvar as UsedInSIMDIntrinsic
2285                     if (srcSimdStructNode->OperIsLocal())
2286                     {
2287                         setLclRelatedToSIMDIntrinsic(srcSimdStructNode);
2288                     }
2289
2290                     if (curDst->OperGet() == GT_FIELD)
2291                     {
2292                         GenTree* objRef = curDst->gtField.gtFldObj;
2293                         if (objRef != nullptr && objRef->gtOper == GT_ADDR)
2294                         {
2295                             GenTree* obj = objRef->gtOp.gtOp1;
2296                             if (varTypeIsStruct(obj) && obj->OperIsLocal())
2297                             {
2298                                 setLclRelatedToSIMDIntrinsic(obj);
2299                             }
2300                         }
2301                     }
2302                 }
2303                 else
2304                 {
2305                     fgPreviousCandidateSIMDFieldAsgStmt = stmt;
2306                 }
2307             }
2308         }
2309     }
2310     else
2311     {
2312         fgPreviousCandidateSIMDFieldAsgStmt = nullptr;
2313     }
2314 }
2315
2316 //------------------------------------------------------------------------
2317 // impSIMDIntrinsic: Check method to see if it is a SIMD method
2318 //
2319 // Arguments:
2320 //    opcode     - the opcode being handled (needed to identify the CEE_NEWOBJ case)
2321 //    newobjThis - For CEE_NEWOBJ, this is the temp grabbed for the allocated uninitalized object.
2322 //    clsHnd     - The handle of the class of the method.
2323 //    method     - The handle of the method.
2324 //    sig        - The call signature for the method.
2325 //    memberRef  - The memberRef token for the method reference.
2326 //
2327 // Return Value:
2328 //    If clsHnd is a known SIMD type, and 'method' is one of the methods that are
2329 //    implemented as an intrinsic in the JIT, then return the tree that implements
2330 //    it.
2331 //
2332 GenTree* Compiler::impSIMDIntrinsic(OPCODE                opcode,
2333                                     GenTree*              newobjThis,
2334                                     CORINFO_CLASS_HANDLE  clsHnd,
2335                                     CORINFO_METHOD_HANDLE methodHnd,
2336                                     CORINFO_SIG_INFO*     sig,
2337                                     int                   memberRef)
2338 {
2339     assert(featureSIMD);
2340
2341     if (!isSIMDClass(clsHnd))
2342     {
2343         return nullptr;
2344     }
2345
2346     // Get base type and intrinsic Id
2347     var_types                baseType = TYP_UNKNOWN;
2348     unsigned                 size     = 0;
2349     unsigned                 argCount = 0;
2350     const SIMDIntrinsicInfo* intrinsicInfo =
2351         getSIMDIntrinsicInfo(&clsHnd, methodHnd, sig, (opcode == CEE_NEWOBJ), &argCount, &baseType, &size);
2352     if (intrinsicInfo == nullptr || intrinsicInfo->id == SIMDIntrinsicInvalid)
2353     {
2354         return nullptr;
2355     }
2356
2357     SIMDIntrinsicID simdIntrinsicID = intrinsicInfo->id;
2358     var_types       simdType;
2359     if (baseType != TYP_UNKNOWN)
2360     {
2361         simdType = getSIMDTypeForSize(size);
2362     }
2363     else
2364     {
2365         assert(simdIntrinsicID == SIMDIntrinsicHWAccel);
2366         simdType = TYP_UNKNOWN;
2367     }
2368     bool      instMethod = intrinsicInfo->isInstMethod;
2369     var_types callType   = JITtype2varType(sig->retType);
2370     if (callType == TYP_STRUCT)
2371     {
2372         // Note that here we are assuming that, if the call returns a struct, that it is the same size as the
2373         // struct on which the method is declared. This is currently true for all methods on Vector types,
2374         // but if this ever changes, we will need to determine the callType from the signature.
2375         assert(info.compCompHnd->getClassSize(sig->retTypeClass) == genTypeSize(simdType));
2376         callType = simdType;
2377     }
2378
2379     GenTree* simdTree   = nullptr;
2380     GenTree* op1        = nullptr;
2381     GenTree* op2        = nullptr;
2382     GenTree* op3        = nullptr;
2383     GenTree* retVal     = nullptr;
2384     GenTree* copyBlkDst = nullptr;
2385     bool     doCopyBlk  = false;
2386
2387     switch (simdIntrinsicID)
2388     {
2389         case SIMDIntrinsicGetCount:
2390         {
2391             int            length       = getSIMDVectorLength(clsHnd);
2392             GenTreeIntCon* intConstTree = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, length);
2393             retVal                      = intConstTree;
2394
2395             intConstTree->gtFlags |= GTF_ICON_SIMD_COUNT;
2396         }
2397         break;
2398
2399         case SIMDIntrinsicGetZero:
2400             retVal = gtNewSIMDVectorZero(simdType, baseType, size);
2401             break;
2402
2403         case SIMDIntrinsicGetOne:
2404             retVal = gtNewSIMDVectorOne(simdType, baseType, size);
2405             break;
2406
2407         case SIMDIntrinsicGetAllOnes:
2408         {
2409             // Equivalent to (Vector<T>) new Vector<int>(0xffffffff);
2410             GenTree* initVal = gtNewIconNode(0xffffffff, TYP_INT);
2411             simdTree         = gtNewSIMDNode(simdType, initVal, nullptr, SIMDIntrinsicInit, TYP_INT, size);
2412             if (baseType != TYP_INT)
2413             {
2414                 // cast it to required baseType if different from TYP_INT
2415                 simdTree = gtNewSIMDNode(simdType, simdTree, nullptr, SIMDIntrinsicCast, baseType, size);
2416             }
2417             retVal = simdTree;
2418         }
2419         break;
2420
2421         case SIMDIntrinsicInit:
2422         case SIMDIntrinsicInitN:
2423         {
2424             // SIMDIntrinsicInit:
2425             //    op2 - the initializer value
2426             //    op1 - byref of vector
2427             //
2428             // SIMDIntrinsicInitN
2429             //    op2 - list of initializer values stitched into a list
2430             //    op1 - byref of vector
2431             bool initFromFirstArgIndir = false;
2432             if (simdIntrinsicID == SIMDIntrinsicInit)
2433             {
2434                 op2 = impSIMDPopStack(baseType);
2435             }
2436             else
2437             {
2438                 assert(simdIntrinsicID == SIMDIntrinsicInitN);
2439                 assert(baseType == TYP_FLOAT);
2440
2441                 unsigned initCount    = argCount - 1;
2442                 unsigned elementCount = getSIMDVectorLength(size, baseType);
2443                 noway_assert(initCount == elementCount);
2444                 GenTree* nextArg = op2;
2445
2446                 // Build a GT_LIST with the N values.
2447                 // We must maintain left-to-right order of the args, but we will pop
2448                 // them off in reverse order (the Nth arg was pushed onto the stack last).
2449
2450                 GenTree* list              = nullptr;
2451                 GenTree* firstArg          = nullptr;
2452                 GenTree* prevArg           = nullptr;
2453                 int      offset            = 0;
2454                 bool     areArgsContiguous = true;
2455                 for (unsigned i = 0; i < initCount; i++)
2456                 {
2457                     GenTree* nextArg = impSIMDPopStack(baseType);
2458                     if (areArgsContiguous)
2459                     {
2460                         GenTree* curArg = nextArg;
2461                         firstArg        = curArg;
2462
2463                         if (prevArg != nullptr)
2464                         {
2465                             // Recall that we are popping the args off the stack in reverse order.
2466                             areArgsContiguous = areArgumentsContiguous(curArg, prevArg);
2467                         }
2468                         prevArg = curArg;
2469                     }
2470
2471                     list = new (this, GT_LIST) GenTreeOp(GT_LIST, baseType, nextArg, list);
2472                 }
2473
2474                 if (areArgsContiguous && baseType == TYP_FLOAT)
2475                 {
2476                     // Since Vector2, Vector3 and Vector4's arguments type are only float,
2477                     // we intialize the vector from first argument address, only when
2478                     // the baseType is TYP_FLOAT and the arguments are located contiguously in memory
2479                     initFromFirstArgIndir = true;
2480                     GenTree*  op2Address  = createAddressNodeForSIMDInit(firstArg, size);
2481                     var_types simdType    = getSIMDTypeForSize(size);
2482                     op2                   = gtNewOperNode(GT_IND, simdType, op2Address);
2483                 }
2484                 else
2485                 {
2486                     op2 = list;
2487                 }
2488             }
2489
2490             op1 = getOp1ForConstructor(opcode, newobjThis, clsHnd);
2491
2492             assert(op1->TypeGet() == TYP_BYREF);
2493             assert(genActualType(op2->TypeGet()) == genActualType(baseType) || initFromFirstArgIndir);
2494
2495             // For integral base types of size less than TYP_INT, expand the initializer
2496             // to fill size of TYP_INT bytes.
2497             if (varTypeIsSmallInt(baseType))
2498             {
2499                 // This case should occur only for Init intrinsic.
2500                 assert(simdIntrinsicID == SIMDIntrinsicInit);
2501
2502                 unsigned baseSize = genTypeSize(baseType);
2503                 int      multiplier;
2504                 if (baseSize == 1)
2505                 {
2506                     multiplier = 0x01010101;
2507                 }
2508                 else
2509                 {
2510                     assert(baseSize == 2);
2511                     multiplier = 0x00010001;
2512                 }
2513
2514                 GenTree* t1 = nullptr;
2515                 if (baseType == TYP_BYTE)
2516                 {
2517                     // What we have is a signed byte initializer,
2518                     // which when loaded to a reg will get sign extended to TYP_INT.
2519                     // But what we need is the initializer without sign extended or
2520                     // rather zero extended to 32-bits.
2521                     t1 = gtNewOperNode(GT_AND, TYP_INT, op2, gtNewIconNode(0xff, TYP_INT));
2522                 }
2523                 else if (baseType == TYP_SHORT)
2524                 {
2525                     // What we have is a signed short initializer,
2526                     // which when loaded to a reg will get sign extended to TYP_INT.
2527                     // But what we need is the initializer without sign extended or
2528                     // rather zero extended to 32-bits.
2529                     t1 = gtNewOperNode(GT_AND, TYP_INT, op2, gtNewIconNode(0xffff, TYP_INT));
2530                 }
2531                 else
2532                 {
2533                     assert(baseType == TYP_UBYTE || baseType == TYP_USHORT);
2534                     t1 = gtNewCastNode(TYP_INT, op2, false, TYP_INT);
2535                 }
2536
2537                 assert(t1 != nullptr);
2538                 GenTree* t2 = gtNewIconNode(multiplier, TYP_INT);
2539                 op2         = gtNewOperNode(GT_MUL, TYP_INT, t1, t2);
2540
2541                 // Construct a vector of TYP_INT with the new initializer and cast it back to vector of baseType
2542                 simdTree = gtNewSIMDNode(simdType, op2, nullptr, simdIntrinsicID, TYP_INT, size);
2543                 simdTree = gtNewSIMDNode(simdType, simdTree, nullptr, SIMDIntrinsicCast, baseType, size);
2544             }
2545             else
2546             {
2547
2548                 if (initFromFirstArgIndir)
2549                 {
2550                     simdTree = op2;
2551                     if (op1->gtOp.gtOp1->OperIsLocal())
2552                     {
2553                         // label the dst struct's lclvar is used for SIMD intrinsic,
2554                         // so that this dst struct won't be promoted.
2555                         setLclRelatedToSIMDIntrinsic(op1->gtOp.gtOp1);
2556                     }
2557                 }
2558                 else
2559                 {
2560                     simdTree = gtNewSIMDNode(simdType, op2, nullptr, simdIntrinsicID, baseType, size);
2561                 }
2562             }
2563
2564             copyBlkDst = op1;
2565             doCopyBlk  = true;
2566         }
2567         break;
2568
2569         case SIMDIntrinsicInitArray:
2570         case SIMDIntrinsicInitArrayX:
2571         case SIMDIntrinsicCopyToArray:
2572         case SIMDIntrinsicCopyToArrayX:
2573         {
2574             // op3 - index into array in case of SIMDIntrinsicCopyToArrayX and SIMDIntrinsicInitArrayX
2575             // op2 - array itself
2576             // op1 - byref to vector struct
2577
2578             unsigned int vectorLength = getSIMDVectorLength(size, baseType);
2579             // (This constructor takes only the zero-based arrays.)
2580             // We will add one or two bounds checks:
2581             // 1. If we have an index, we must do a check on that first.
2582             //    We can't combine it with the index + vectorLength check because
2583             //    a. It might be negative, and b. It may need to raise a different exception
2584             //    (captured as SCK_ARG_RNG_EXCPN for CopyTo and SCK_RNGCHK_FAIL for Init).
2585             // 2. We need to generate a check (SCK_ARG_EXCPN for CopyTo and SCK_RNGCHK_FAIL for Init)
2586             //    for the last array element we will access.
2587             //    We'll either check against (vectorLength - 1) or (index + vectorLength - 1).
2588
2589             GenTree* checkIndexExpr = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, vectorLength - 1);
2590
2591             // Get the index into the array.  If it has been provided, it will be on the
2592             // top of the stack.  Otherwise, it is null.
2593             if (argCount == 3)
2594             {
2595                 op3 = impSIMDPopStack(TYP_INT);
2596                 if (op3->IsIntegralConst(0))
2597                 {
2598                     op3 = nullptr;
2599                 }
2600             }
2601             else
2602             {
2603                 // TODO-CQ: Here, or elsewhere, check for the pattern where op2 is a newly constructed array, and
2604                 // change this to the InitN form.
2605                 // op3 = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, 0);
2606                 op3 = nullptr;
2607             }
2608
2609             // Clone the array for use in the bounds check.
2610             op2 = impSIMDPopStack(TYP_REF);
2611             assert(op2->TypeGet() == TYP_REF);
2612             GenTree* arrayRefForArgChk = op2;
2613             GenTree* argRngChk         = nullptr;
2614             GenTree* asg               = nullptr;
2615             if ((arrayRefForArgChk->gtFlags & GTF_SIDE_EFFECT) != 0)
2616             {
2617                 op2 = fgInsertCommaFormTemp(&arrayRefForArgChk);
2618             }
2619             else
2620             {
2621                 op2 = gtCloneExpr(arrayRefForArgChk);
2622             }
2623             assert(op2 != nullptr);
2624
2625             if (op3 != nullptr)
2626             {
2627                 SpecialCodeKind op3CheckKind;
2628                 if (simdIntrinsicID == SIMDIntrinsicInitArrayX)
2629                 {
2630                     op3CheckKind = SCK_RNGCHK_FAIL;
2631                 }
2632                 else
2633                 {
2634                     assert(simdIntrinsicID == SIMDIntrinsicCopyToArrayX);
2635                     op3CheckKind = SCK_ARG_RNG_EXCPN;
2636                 }
2637                 // We need to use the original expression on this, which is the first check.
2638                 GenTree* arrayRefForArgRngChk = arrayRefForArgChk;
2639                 // Then we clone the clone we just made for the next check.
2640                 arrayRefForArgChk = gtCloneExpr(op2);
2641                 // We know we MUST have had a cloneable expression.
2642                 assert(arrayRefForArgChk != nullptr);
2643                 GenTree* index = op3;
2644                 if ((index->gtFlags & GTF_SIDE_EFFECT) != 0)
2645                 {
2646                     op3 = fgInsertCommaFormTemp(&index);
2647                 }
2648                 else
2649                 {
2650                     op3 = gtCloneExpr(index);
2651                 }
2652
2653                 GenTreeArrLen* arrLen =
2654                     gtNewArrLen(TYP_INT, arrayRefForArgRngChk, (int)OFFSETOF__CORINFO_Array__length);
2655                 argRngChk = new (this, GT_ARR_BOUNDS_CHECK)
2656                     GenTreeBoundsChk(GT_ARR_BOUNDS_CHECK, TYP_VOID, index, arrLen, op3CheckKind);
2657                 // Now, clone op3 to create another node for the argChk
2658                 GenTree* index2 = gtCloneExpr(op3);
2659                 assert(index != nullptr);
2660                 checkIndexExpr = gtNewOperNode(GT_ADD, TYP_INT, index2, checkIndexExpr);
2661             }
2662
2663             // Insert a bounds check for index + offset - 1.
2664             // This must be a "normal" array.
2665             SpecialCodeKind op2CheckKind;
2666             if (simdIntrinsicID == SIMDIntrinsicInitArray || simdIntrinsicID == SIMDIntrinsicInitArrayX)
2667             {
2668                 op2CheckKind = SCK_RNGCHK_FAIL;
2669             }
2670             else
2671             {
2672                 op2CheckKind = SCK_ARG_EXCPN;
2673             }
2674             GenTreeArrLen*    arrLen = gtNewArrLen(TYP_INT, arrayRefForArgChk, (int)OFFSETOF__CORINFO_Array__length);
2675             GenTreeBoundsChk* argChk = new (this, GT_ARR_BOUNDS_CHECK)
2676                 GenTreeBoundsChk(GT_ARR_BOUNDS_CHECK, TYP_VOID, checkIndexExpr, arrLen, op2CheckKind);
2677
2678             // Create a GT_COMMA tree for the bounds check(s).
2679             op2 = gtNewOperNode(GT_COMMA, op2->TypeGet(), argChk, op2);
2680             if (argRngChk != nullptr)
2681             {
2682                 op2 = gtNewOperNode(GT_COMMA, op2->TypeGet(), argRngChk, op2);
2683             }
2684
2685             if (simdIntrinsicID == SIMDIntrinsicInitArray || simdIntrinsicID == SIMDIntrinsicInitArrayX)
2686             {
2687                 op1        = getOp1ForConstructor(opcode, newobjThis, clsHnd);
2688                 simdTree   = gtNewSIMDNode(simdType, op2, op3, SIMDIntrinsicInitArray, baseType, size);
2689                 copyBlkDst = op1;
2690                 doCopyBlk  = true;
2691             }
2692             else
2693             {
2694                 assert(simdIntrinsicID == SIMDIntrinsicCopyToArray || simdIntrinsicID == SIMDIntrinsicCopyToArrayX);
2695                 op1 = impSIMDPopStack(simdType, instMethod);
2696                 assert(op1->TypeGet() == simdType);
2697
2698                 // copy vector (op1) to array (op2) starting at index (op3)
2699                 simdTree = op1;
2700
2701                 // TODO-Cleanup: Though it happens to just work fine front-end phases are not aware of GT_LEA node.
2702                 // Therefore, convert these to use GT_ADDR .
2703                 copyBlkDst = new (this, GT_LEA)
2704                     GenTreeAddrMode(TYP_BYREF, op2, op3, genTypeSize(baseType), OFFSETOF__CORINFO_Array__data);
2705                 doCopyBlk = true;
2706             }
2707         }
2708         break;
2709
2710         case SIMDIntrinsicInitFixed:
2711         {
2712             // We are initializing a fixed-length vector VLarge with a smaller fixed-length vector VSmall, plus 1 or 2
2713             // additional floats.
2714             //    op4 (optional) - float value for VLarge.W, if VLarge is Vector4, and VSmall is Vector2
2715             //    op3 - float value for VLarge.Z or VLarge.W
2716             //    op2 - VSmall
2717             //    op1 - byref of VLarge
2718             assert(baseType == TYP_FLOAT);
2719             unsigned elementByteCount = 4;
2720
2721             GenTree* op4 = nullptr;
2722             if (argCount == 4)
2723             {
2724                 op4 = impSIMDPopStack(TYP_FLOAT);
2725                 assert(op4->TypeGet() == TYP_FLOAT);
2726             }
2727             op3 = impSIMDPopStack(TYP_FLOAT);
2728             assert(op3->TypeGet() == TYP_FLOAT);
2729             // The input vector will either be TYP_SIMD8 or TYP_SIMD12.
2730             var_types smallSIMDType = TYP_SIMD8;
2731             if ((op4 == nullptr) && (simdType == TYP_SIMD16))
2732             {
2733                 smallSIMDType = TYP_SIMD12;
2734             }
2735             op2 = impSIMDPopStack(smallSIMDType);
2736             op1 = getOp1ForConstructor(opcode, newobjThis, clsHnd);
2737
2738             // We are going to redefine the operands so that:
2739             // - op3 is the value that's going into the Z position, or null if it's a Vector4 constructor with a single
2740             // operand, and
2741             // - op4 is the W position value, or null if this is a Vector3 constructor.
2742             if (size == 16 && argCount == 3)
2743             {
2744                 op4 = op3;
2745                 op3 = nullptr;
2746             }
2747
2748             simdTree = op2;
2749             if (op3 != nullptr)
2750             {
2751                 simdTree = gtNewSIMDNode(simdType, simdTree, op3, SIMDIntrinsicSetZ, baseType, size);
2752             }
2753             if (op4 != nullptr)
2754             {
2755                 simdTree = gtNewSIMDNode(simdType, simdTree, op4, SIMDIntrinsicSetW, baseType, size);
2756             }
2757
2758             copyBlkDst = op1;
2759             doCopyBlk  = true;
2760         }
2761         break;
2762
2763         case SIMDIntrinsicOpEquality:
2764         case SIMDIntrinsicInstEquals:
2765         {
2766             op2 = impSIMDPopStack(simdType);
2767             op1 = impSIMDPopStack(simdType, instMethod);
2768
2769             assert(op1->TypeGet() == simdType);
2770             assert(op2->TypeGet() == simdType);
2771
2772             simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, SIMDIntrinsicOpEquality, baseType, size);
2773             if (simdType == TYP_SIMD12)
2774             {
2775                 simdTree->gtFlags |= GTF_SIMD12_OP;
2776             }
2777             retVal = simdTree;
2778         }
2779         break;
2780
2781         case SIMDIntrinsicOpInEquality:
2782         {
2783             // op1 is the first operand
2784             // op2 is the second operand
2785             op2      = impSIMDPopStack(simdType);
2786             op1      = impSIMDPopStack(simdType, instMethod);
2787             simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, SIMDIntrinsicOpInEquality, baseType, size);
2788             if (simdType == TYP_SIMD12)
2789             {
2790                 simdTree->gtFlags |= GTF_SIMD12_OP;
2791             }
2792             retVal = simdTree;
2793         }
2794         break;
2795
2796         case SIMDIntrinsicEqual:
2797         case SIMDIntrinsicLessThan:
2798         case SIMDIntrinsicLessThanOrEqual:
2799         case SIMDIntrinsicGreaterThan:
2800         case SIMDIntrinsicGreaterThanOrEqual:
2801         {
2802             op2 = impSIMDPopStack(simdType);
2803             op1 = impSIMDPopStack(simdType, instMethod);
2804
2805             SIMDIntrinsicID intrinsicID = impSIMDRelOp(simdIntrinsicID, clsHnd, size, &baseType, &op1, &op2);
2806             simdTree                    = gtNewSIMDNode(genActualType(callType), op1, op2, intrinsicID, baseType, size);
2807             retVal                      = simdTree;
2808         }
2809         break;
2810
2811         case SIMDIntrinsicAdd:
2812         case SIMDIntrinsicSub:
2813         case SIMDIntrinsicMul:
2814         case SIMDIntrinsicDiv:
2815         case SIMDIntrinsicBitwiseAnd:
2816         case SIMDIntrinsicBitwiseAndNot:
2817         case SIMDIntrinsicBitwiseOr:
2818         case SIMDIntrinsicBitwiseXor:
2819         {
2820 #if defined(DEBUG)
2821             // check for the cases where we don't support intrinsics.
2822             // This check should be done before we make modifications to type stack.
2823             // Note that this is more of a double safety check for robustness since
2824             // we expect getSIMDIntrinsicInfo() to have filtered out intrinsics on
2825             // unsupported base types. If getSIMdIntrinsicInfo() doesn't filter due
2826             // to some bug, assert in chk/dbg will fire.
2827             if (!varTypeIsFloating(baseType))
2828             {
2829                 if (simdIntrinsicID == SIMDIntrinsicMul)
2830                 {
2831 #if defined(_TARGET_XARCH_)
2832                     if ((baseType != TYP_INT) && (baseType != TYP_SHORT))
2833                     {
2834                         // TODO-CQ: implement mul on these integer vectors.
2835                         // Note that SSE2 has no direct support for these vectors.
2836                         assert(!"Mul not supported on long/ulong/uint/small int vectors\n");
2837                         return nullptr;
2838                     }
2839 #endif // _TARGET_XARCH_
2840 #if defined(_TARGET_ARM64_)
2841                     if ((baseType == TYP_ULONG) && (baseType == TYP_LONG))
2842                     {
2843                         // TODO-CQ: implement mul on these integer vectors.
2844                         // Note that ARM64 has no direct support for these vectors.
2845                         assert(!"Mul not supported on long/ulong vectors\n");
2846                         return nullptr;
2847                     }
2848 #endif // _TARGET_ARM64_
2849                 }
2850 #if defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
2851                 // common to all integer type vectors
2852                 if (simdIntrinsicID == SIMDIntrinsicDiv)
2853                 {
2854                     // SSE2 doesn't support div on non-floating point vectors.
2855                     assert(!"Div not supported on integer type vectors\n");
2856                     return nullptr;
2857                 }
2858 #endif // defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
2859             }
2860 #endif // DEBUG
2861
2862             // op1 is the first operand; if instance method, op1 is "this" arg
2863             // op2 is the second operand
2864             op2 = impSIMDPopStack(simdType);
2865             op1 = impSIMDPopStack(simdType, instMethod);
2866
2867             simdTree = gtNewSIMDNode(simdType, op1, op2, simdIntrinsicID, baseType, size);
2868             retVal   = simdTree;
2869         }
2870         break;
2871
2872         case SIMDIntrinsicSelect:
2873         {
2874             // op3 is a SIMD variable that is the second source
2875             // op2 is a SIMD variable that is the first source
2876             // op1 is a SIMD variable which is the bit mask.
2877             op3 = impSIMDPopStack(simdType);
2878             op2 = impSIMDPopStack(simdType);
2879             op1 = impSIMDPopStack(simdType);
2880
2881             retVal = impSIMDSelect(clsHnd, baseType, size, op1, op2, op3);
2882         }
2883         break;
2884
2885         case SIMDIntrinsicMin:
2886         case SIMDIntrinsicMax:
2887         {
2888             // op1 is the first operand; if instance method, op1 is "this" arg
2889             // op2 is the second operand
2890             op2 = impSIMDPopStack(simdType);
2891             op1 = impSIMDPopStack(simdType, instMethod);
2892
2893             retVal = impSIMDMinMax(simdIntrinsicID, clsHnd, baseType, size, op1, op2);
2894         }
2895         break;
2896
2897         case SIMDIntrinsicGetItem:
2898         {
2899             // op1 is a SIMD variable that is "this" arg
2900             // op2 is an index of TYP_INT
2901             op2              = impSIMDPopStack(TYP_INT);
2902             op1              = impSIMDPopStack(simdType, instMethod);
2903             int vectorLength = getSIMDVectorLength(size, baseType);
2904             if (!op2->IsCnsIntOrI() || op2->AsIntCon()->gtIconVal >= vectorLength || op2->AsIntCon()->gtIconVal < 0)
2905             {
2906                 // We need to bounds-check the length of the vector.
2907                 // For that purpose, we need to clone the index expression.
2908                 GenTree* index = op2;
2909                 if ((index->gtFlags & GTF_SIDE_EFFECT) != 0)
2910                 {
2911                     op2 = fgInsertCommaFormTemp(&index);
2912                 }
2913                 else
2914                 {
2915                     op2 = gtCloneExpr(index);
2916                 }
2917
2918                 GenTree*          lengthNode = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, vectorLength);
2919                 GenTreeBoundsChk* simdChk =
2920                     new (this, GT_SIMD_CHK) GenTreeBoundsChk(GT_SIMD_CHK, TYP_VOID, index, lengthNode, SCK_RNGCHK_FAIL);
2921
2922                 // Create a GT_COMMA tree for the bounds check.
2923                 op2 = gtNewOperNode(GT_COMMA, op2->TypeGet(), simdChk, op2);
2924             }
2925
2926             assert(op1->TypeGet() == simdType);
2927             assert(op2->TypeGet() == TYP_INT);
2928
2929             simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, simdIntrinsicID, baseType, size);
2930             retVal   = simdTree;
2931         }
2932         break;
2933
2934         case SIMDIntrinsicDotProduct:
2935         {
2936 #if defined(_TARGET_XARCH_)
2937             // Right now dot product is supported only for float/double vectors and
2938             // int vectors on SSE4/AVX.
2939             if (!varTypeIsFloating(baseType) && !(baseType == TYP_INT && getSIMDSupportLevel() >= SIMD_SSE4_Supported))
2940             {
2941                 return nullptr;
2942             }
2943 #endif // _TARGET_XARCH_
2944
2945             // op1 is a SIMD variable that is the first source and also "this" arg.
2946             // op2 is a SIMD variable which is the second source.
2947             op2 = impSIMDPopStack(simdType);
2948             op1 = impSIMDPopStack(simdType, instMethod);
2949
2950             simdTree = gtNewSIMDNode(baseType, op1, op2, simdIntrinsicID, baseType, size);
2951             if (simdType == TYP_SIMD12)
2952             {
2953                 simdTree->gtFlags |= GTF_SIMD12_OP;
2954             }
2955             retVal = simdTree;
2956         }
2957         break;
2958
2959         case SIMDIntrinsicSqrt:
2960         {
2961 #if (defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)) && defined(DEBUG)
2962             // SSE/AVX/ARM64 doesn't support sqrt on integer type vectors and hence
2963             // should never be seen as an intrinsic here. See SIMDIntrinsicList.h
2964             // for supported base types for this intrinsic.
2965             if (!varTypeIsFloating(baseType))
2966             {
2967                 assert(!"Sqrt not supported on integer vectors\n");
2968                 return nullptr;
2969             }
2970 #endif // (defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)) && defined(DEBUG)
2971
2972             op1 = impSIMDPopStack(simdType);
2973
2974             retVal = gtNewSIMDNode(genActualType(callType), op1, nullptr, simdIntrinsicID, baseType, size);
2975         }
2976         break;
2977
2978         case SIMDIntrinsicAbs:
2979             op1    = impSIMDPopStack(simdType);
2980             retVal = impSIMDAbs(clsHnd, baseType, size, op1);
2981             break;
2982
2983         case SIMDIntrinsicGetW:
2984             retVal = impSIMDGetFixed(simdType, baseType, size, 3);
2985             break;
2986
2987         case SIMDIntrinsicGetZ:
2988             retVal = impSIMDGetFixed(simdType, baseType, size, 2);
2989             break;
2990
2991         case SIMDIntrinsicGetY:
2992             retVal = impSIMDGetFixed(simdType, baseType, size, 1);
2993             break;
2994
2995         case SIMDIntrinsicGetX:
2996             retVal = impSIMDGetFixed(simdType, baseType, size, 0);
2997             break;
2998
2999         case SIMDIntrinsicSetW:
3000         case SIMDIntrinsicSetZ:
3001         case SIMDIntrinsicSetY:
3002         case SIMDIntrinsicSetX:
3003         {
3004             // op2 is the value to be set at indexTemp position
3005             // op1 is SIMD vector that is going to be modified, which is a byref
3006
3007             // If op1 has a side-effect, then don't make it an intrinsic.
3008             // It would be in-efficient to read the entire vector into xmm reg,
3009             // modify it and write back entire xmm reg.
3010             //
3011             // TODO-CQ: revisit this later.
3012             op1 = impStackTop(1).val;
3013             if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0)
3014             {
3015                 return nullptr;
3016             }
3017
3018             op2 = impSIMDPopStack(baseType);
3019             op1 = impSIMDPopStack(simdType, instMethod);
3020
3021             GenTree* src = gtCloneExpr(op1);
3022             assert(src != nullptr);
3023             simdTree = gtNewSIMDNode(simdType, src, op2, simdIntrinsicID, baseType, size);
3024
3025             copyBlkDst = gtNewOperNode(GT_ADDR, TYP_BYREF, op1);
3026             doCopyBlk  = true;
3027         }
3028         break;
3029
3030         // Unary operators that take and return a Vector.
3031         case SIMDIntrinsicCast:
3032         case SIMDIntrinsicConvertToSingle:
3033         case SIMDIntrinsicConvertToDouble:
3034         case SIMDIntrinsicConvertToInt32:
3035         {
3036             op1 = impSIMDPopStack(simdType, instMethod);
3037
3038             simdTree = gtNewSIMDNode(simdType, op1, nullptr, simdIntrinsicID, baseType, size);
3039             retVal   = simdTree;
3040         }
3041         break;
3042
3043         case SIMDIntrinsicConvertToInt64:
3044         {
3045 #ifdef _TARGET_64BIT_
3046             op1 = impSIMDPopStack(simdType, instMethod);
3047
3048             simdTree = gtNewSIMDNode(simdType, op1, nullptr, simdIntrinsicID, baseType, size);
3049             retVal   = simdTree;
3050 #else
3051             JITDUMP("SIMD Conversion to Int64 is not supported on this platform\n");
3052             return nullptr;
3053 #endif
3054         }
3055         break;
3056
3057         case SIMDIntrinsicNarrow:
3058         {
3059             assert(!instMethod);
3060             op2 = impSIMDPopStack(simdType);
3061             op1 = impSIMDPopStack(simdType);
3062             // op1 and op2 are two input Vector<T>.
3063             simdTree = gtNewSIMDNode(simdType, op1, op2, simdIntrinsicID, baseType, size);
3064             retVal   = simdTree;
3065         }
3066         break;
3067
3068         case SIMDIntrinsicWiden:
3069         {
3070             GenTree* dstAddrHi = impSIMDPopStack(TYP_BYREF);
3071             GenTree* dstAddrLo = impSIMDPopStack(TYP_BYREF);
3072             op1                = impSIMDPopStack(simdType);
3073             GenTree* dupOp1    = fgInsertCommaFormTemp(&op1, gtGetStructHandleForSIMD(simdType, baseType));
3074
3075             // Widen the lower half and assign it to dstAddrLo.
3076             simdTree = gtNewSIMDNode(simdType, op1, nullptr, SIMDIntrinsicWidenLo, baseType, size);
3077             GenTree* loDest =
3078                 new (this, GT_BLK) GenTreeBlk(GT_BLK, simdType, dstAddrLo, getSIMDTypeSizeInBytes(clsHnd));
3079             GenTree* loAsg = gtNewBlkOpNode(loDest, simdTree, getSIMDTypeSizeInBytes(clsHnd),
3080                                             false, // not volatile
3081                                             true); // copyBlock
3082             loAsg->gtFlags |= ((simdTree->gtFlags | dstAddrLo->gtFlags) & GTF_ALL_EFFECT);
3083
3084             // Widen the upper half and assign it to dstAddrHi.
3085             simdTree = gtNewSIMDNode(simdType, dupOp1, nullptr, SIMDIntrinsicWidenHi, baseType, size);
3086             GenTree* hiDest =
3087                 new (this, GT_BLK) GenTreeBlk(GT_BLK, simdType, dstAddrHi, getSIMDTypeSizeInBytes(clsHnd));
3088             GenTree* hiAsg = gtNewBlkOpNode(hiDest, simdTree, getSIMDTypeSizeInBytes(clsHnd),
3089                                             false, // not volatile
3090                                             true); // copyBlock
3091             hiAsg->gtFlags |= ((simdTree->gtFlags | dstAddrHi->gtFlags) & GTF_ALL_EFFECT);
3092
3093             retVal = gtNewOperNode(GT_COMMA, simdType, loAsg, hiAsg);
3094         }
3095         break;
3096
3097         case SIMDIntrinsicHWAccel:
3098         {
3099             GenTreeIntCon* intConstTree = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, 1);
3100             retVal                      = intConstTree;
3101         }
3102         break;
3103
3104         default:
3105             assert(!"Unimplemented SIMD Intrinsic");
3106             return nullptr;
3107     }
3108
3109 #if defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
3110     // XArch/Arm64: also indicate that we use floating point registers.
3111     // The need for setting this here is that a method may not have SIMD
3112     // type lclvars, but might be exercising SIMD intrinsics on fields of
3113     // SIMD type.
3114     //
3115     // e.g.  public Vector<float> ComplexVecFloat::sqabs() { return this.r * this.r + this.i * this.i; }
3116     compFloatingPointUsed = true;
3117 #endif // defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
3118
3119     // At this point, we have a tree that we are going to store into a destination.
3120     // TODO-1stClassStructs: This should be a simple store or assignment, and should not require
3121     // GTF_ALL_EFFECT for the dest. This is currently emulating the previous behavior of
3122     // block ops.
3123     if (doCopyBlk)
3124     {
3125         GenTree* dest = new (this, GT_BLK) GenTreeBlk(GT_BLK, simdType, copyBlkDst, getSIMDTypeSizeInBytes(clsHnd));
3126         dest->gtFlags |= GTF_GLOB_REF;
3127         retVal = gtNewBlkOpNode(dest, simdTree, getSIMDTypeSizeInBytes(clsHnd),
3128                                 false, // not volatile
3129                                 true); // copyBlock
3130         retVal->gtFlags |= ((simdTree->gtFlags | copyBlkDst->gtFlags) & GTF_ALL_EFFECT);
3131     }
3132
3133     return retVal;
3134 }
3135
3136 #endif // FEATURE_SIMD