src/jit/hwintrinsicxarch.cpp

   1 // Licensed to the .NET Foundation under one or more agreements.
   2 // The .NET Foundation licenses this file to you under the MIT license.
   3 // See the LICENSE file in the project root for more information.
   4
   5 #include "jitpch.h"
   6
   7 #if FEATURE_HW_INTRINSICS
   8
   9 struct HWIntrinsicInfo
  10 {
  11     NamedIntrinsic      intrinsicID;
  12     const char*         intrinsicName;
  13     InstructionSet      isa;
  14     int                 ival;
  15     unsigned            simdSize;
  16     int                 numArgs;
  17     instruction         ins[10];
  18     HWIntrinsicCategory category;
  19     HWIntrinsicFlag     flags;
  20 };
  21
  22 static const HWIntrinsicInfo hwIntrinsicInfoArray[] = {
  23 #define HARDWARE_INTRINSIC(id, name, isa, ival, size, numarg, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, category, flag) \
  24     {NI_##id, name, InstructionSet_##isa, ival, size, numarg, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, category, flag},
  25 #include "hwintrinsiclistxarch.h"
  26 };
  27
  28 extern const char* getHWIntrinsicName(NamedIntrinsic intrinsic)
  29 {
  30     return hwIntrinsicInfoArray[intrinsic - NI_HW_INTRINSIC_START - 1].intrinsicName;
  31 }
  32
  33 //------------------------------------------------------------------------
  34 // lookupHWIntrinsicISA: map class name to InstructionSet value
  35 //
  36 // Arguments:
  37 //    className -- class name in System.Runtime.Intrinsics.X86
  38 //
  39 // Return Value:
  40 //    Id for the ISA class.
  41 //
  42 InstructionSet Compiler::lookupHWIntrinsicISA(const char* className)
  43 {
  44     if (className != nullptr)
  45     {
  46         if (className[0] == 'A')
  47         {
  48             if (strcmp(className, "Aes") == 0)
  49             {
  50                 return InstructionSet_AES;
  51             }
  52             else if (strcmp(className, "Avx") == 0)
  53             {
  54                 return InstructionSet_AVX;
  55             }
  56             else if (strcmp(className, "Avx2") == 0)
  57             {
  58                 return InstructionSet_AVX2;
  59             }
  60         }
  61         if (className[0] == 'S')
  62         {
  63             if (strcmp(className, "Sse") == 0)
  64             {
  65                 return InstructionSet_SSE;
  66             }
  67             else if (strcmp(className, "Sse2") == 0)
  68             {
  69                 return InstructionSet_SSE2;
  70             }
  71             else if (strcmp(className, "Sse3") == 0)
  72             {
  73                 return InstructionSet_SSE3;
  74             }
  75             else if (strcmp(className, "Ssse3") == 0)
  76             {
  77                 return InstructionSet_SSSE3;
  78             }
  79             else if (strcmp(className, "Sse41") == 0)
  80             {
  81                 return InstructionSet_SSE41;
  82             }
  83             else if (strcmp(className, "Sse42") == 0)
  84             {
  85                 return InstructionSet_SSE42;
  86             }
  87         }
  88
  89         if (strcmp(className, "Bmi1") == 0)
  90         {
  91             return InstructionSet_BMI1;
  92         }
  93         else if (strcmp(className, "Bmi2") == 0)
  94         {
  95             return InstructionSet_BMI2;
  96         }
  97         else if (strcmp(className, "Fma") == 0)
  98         {
  99             return InstructionSet_FMA;
 100         }
 101         else if (strcmp(className, "Lzcnt") == 0)
 102         {
 103             return InstructionSet_LZCNT;
 104         }
 105         else if (strcmp(className, "Pclmulqdq") == 0)
 106         {
 107             return InstructionSet_PCLMULQDQ;
 108         }
 109         else if (strcmp(className, "Popcnt") == 0)
 110         {
 111             return InstructionSet_POPCNT;
 112         }
 113     }
 114
 115     JITDUMP("Unsupported ISA.\n");
 116     return InstructionSet_ILLEGAL;
 117 }
 118
 119 //------------------------------------------------------------------------
 120 // lookupHWIntrinsic: map intrinsic name to named intrinsic value
 121 //
 122 // Arguments:
 123 //    methodName -- name of the intrinsic function.
 124 //    isa        -- instruction set of the intrinsic.
 125 //
 126 // Return Value:
 127 //    Id for the hardware intrinsic.
 128 //
 129 // TODO-Throughput: replace sequential search by binary search
 130 NamedIntrinsic Compiler::lookupHWIntrinsic(const char* methodName, InstructionSet isa)
 131 {
 132     NamedIntrinsic result = NI_Illegal;
 133     if (isa != InstructionSet_ILLEGAL)
 134     {
 135         for (int i = 0; i < NI_HW_INTRINSIC_END - NI_HW_INTRINSIC_START - 1; i++)
 136         {
 137             if (isa == hwIntrinsicInfoArray[i].isa && strcmp(methodName, hwIntrinsicInfoArray[i].intrinsicName) == 0)
 138             {
 139                 result = hwIntrinsicInfoArray[i].intrinsicID;
 140             }
 141         }
 142     }
 143     return result;
 144 }
 145
 146 //------------------------------------------------------------------------
 147 // isaOfHWIntrinsic: map named intrinsic value to its instruction set
 148 //
 149 // Arguments:
 150 //    intrinsic -- id of the intrinsic function.
 151 //
 152 // Return Value:
 153 //    instruction set of the intrinsic.
 154 //
 155 InstructionSet Compiler::isaOfHWIntrinsic(NamedIntrinsic intrinsic)
 156 {
 157     assert(intrinsic != NI_Illegal);
 158     assert(intrinsic > NI_HW_INTRINSIC_START && intrinsic < NI_HW_INTRINSIC_END);
 159     return hwIntrinsicInfoArray[intrinsic - NI_HW_INTRINSIC_START - 1].isa;
 160 }
 161
 162 //------------------------------------------------------------------------
 163 // ivalOfHWIntrinsic: get the imm8 value of this intrinsic from the hwIntrinsicInfoArray table
 164 //
 165 // Arguments:
 166 //    intrinsic -- id of the intrinsic function.
 167 //
 168 // Return Value:
 169 //     The imm8 value that is implicit for this intrinsic, or -1 for intrinsics that do not take an immediate, or for
 170 //     which the immediate is an explicit argument.
 171 //
 172 int Compiler::ivalOfHWIntrinsic(NamedIntrinsic intrinsic)
 173 {
 174     assert(intrinsic != NI_Illegal);
 175     assert(intrinsic > NI_HW_INTRINSIC_START && intrinsic < NI_HW_INTRINSIC_END);
 176     return hwIntrinsicInfoArray[intrinsic - NI_HW_INTRINSIC_START - 1].ival;
 177 }
 178
 179 //------------------------------------------------------------------------
 180 // simdSizeOfHWIntrinsic: get the SIMD size of this intrinsic
 181 //
 182 // Arguments:
 183 //    intrinsic -- id of the intrinsic function.
 184 //
 185 // Return Value:
 186 //     the SIMD size of this intrinsic
 187 //         - from the hwIntrinsicInfoArray table if intrinsic has NO HW_Flag_UnfixedSIMDSize
 188 //         - TODO-XArch-NYI - from the signature if intrinsic has HW_Flag_UnfixedSIMDSize
 189 //
 190 // Note - this function is only used by the importer
 191 //        after importation (i.e., codegen), we can get the SIMD size from GenTreeHWIntrinsic IR
 192 static unsigned simdSizeOfHWIntrinsic(NamedIntrinsic intrinsic, CORINFO_SIG_INFO* sig)
 193 {
 194     assert(intrinsic > NI_HW_INTRINSIC_START && intrinsic < NI_HW_INTRINSIC_END);
 195     assert((hwIntrinsicInfoArray[intrinsic - NI_HW_INTRINSIC_START - 1].flags & HW_Flag_UnfixedSIMDSize) == 0);
 196     return hwIntrinsicInfoArray[intrinsic - NI_HW_INTRINSIC_START - 1].simdSize;
 197 }
 198
 199 //------------------------------------------------------------------------
 200 // numArgsOfHWIntrinsic: get the number of arguments
 201 //
 202 // Arguments:
 203 //    intrinsic -- id of the intrinsic function.
 204 //
 205 // Return Value:
 206 //     number of arguments
 207 //
 208 int Compiler::numArgsOfHWIntrinsic(NamedIntrinsic intrinsic)
 209 {
 210     assert(intrinsic != NI_Illegal);
 211     assert(intrinsic > NI_HW_INTRINSIC_START && intrinsic < NI_HW_INTRINSIC_END);
 212     return hwIntrinsicInfoArray[intrinsic - NI_HW_INTRINSIC_START - 1].numArgs;
 213 }
 214
 215 //------------------------------------------------------------------------
 216 // insOfHWIntrinsic: get the instruction of the given intrinsic
 217 //
 218 // Arguments:
 219 //    intrinsic -- id of the intrinsic function.
 220 //    type      -- vector base type of this intrinsic
 221 //
 222 // Return Value:
 223 //     the instruction of the given intrinsic on the base type
 224 //     return INS_invalid for unsupported base types
 225 //
 226 instruction Compiler::insOfHWIntrinsic(NamedIntrinsic intrinsic, var_types type)
 227 {
 228     assert(intrinsic != NI_Illegal);
 229     assert(intrinsic > NI_HW_INTRINSIC_START && intrinsic < NI_HW_INTRINSIC_END);
 230     assert(type >= TYP_BYTE && type <= TYP_DOUBLE);
 231     return hwIntrinsicInfoArray[intrinsic - NI_HW_INTRINSIC_START - 1].ins[type - TYP_BYTE];
 232 }
 233
 234 //------------------------------------------------------------------------
 235 // categoryOfHWIntrinsic: get the category of the given intrinsic
 236 //
 237 // Arguments:
 238 //    intrinsic -- id of the intrinsic function.
 239 //
 240 // Return Value:
 241 //     the category of the given intrinsic
 242 //
 243 HWIntrinsicCategory Compiler::categoryOfHWIntrinsic(NamedIntrinsic intrinsic)
 244 {
 245     assert(intrinsic != NI_Illegal);
 246     assert(intrinsic > NI_HW_INTRINSIC_START && intrinsic < NI_HW_INTRINSIC_END);
 247     return hwIntrinsicInfoArray[intrinsic - NI_HW_INTRINSIC_START - 1].category;
 248 }
 249
 250 //------------------------------------------------------------------------
 251 // HWIntrinsicFlag: get the flags of the given intrinsic
 252 //
 253 // Arguments:
 254 //    intrinsic -- id of the intrinsic function.
 255 //
 256 // Return Value:
 257 //     the flags of the given intrinsic
 258 //
 259 HWIntrinsicFlag Compiler::flagsOfHWIntrinsic(NamedIntrinsic intrinsic)
 260 {
 261     assert(intrinsic != NI_Illegal);
 262     assert(intrinsic > NI_HW_INTRINSIC_START && intrinsic < NI_HW_INTRINSIC_END);
 263     return hwIntrinsicInfoArray[intrinsic - NI_HW_INTRINSIC_START - 1].flags;
 264 }
 265
 266 //------------------------------------------------------------------------
 267 // getArgForHWIntrinsic: get the argument from the stack and match  the signature
 268 //
 269 // Arguments:
 270 //    argType   -- the required type of argument
 271 //    argClass  -- the class handle of argType
 272 //
 273 // Return Value:
 274 //     get the argument at the given index from the stack and match  the signature
 275 //
 276 GenTree* Compiler::getArgForHWIntrinsic(var_types argType, CORINFO_CLASS_HANDLE argClass)
 277 {
 278     GenTree* arg = nullptr;
 279     if (argType == TYP_STRUCT)
 280     {
 281         unsigned int argSizeBytes;
 282         var_types    base = getBaseTypeAndSizeOfSIMDType(argClass, &argSizeBytes);
 283         argType           = getSIMDTypeForSize(argSizeBytes);
 284         assert(argType == TYP_SIMD32 || argType == TYP_SIMD16);
 285         arg = impSIMDPopStack(argType);
 286         assert(arg->TypeGet() == TYP_SIMD16 || arg->TypeGet() == TYP_SIMD32);
 287     }
 288     else
 289     {
 290         assert(varTypeIsArithmetic(argType));
 291         arg = impPopStack().val;
 292         assert(varTypeIsArithmetic(arg->TypeGet()));
 293         assert(genActualType(arg->gtType) == genActualType(argType));
 294     }
 295     return arg;
 296 }
 297
 298 //------------------------------------------------------------------------
 299 // isFullyImplmentedISAClass: return true if all the hardware intrinsics
 300 //    of this ISA are implemented in RyuJIT.
 301 //
 302 // Arguments:
 303 //    isa - Instruction set
 304 // Return Value:
 305 //    true - all the hardware intrinsics of "isa" are implemented in RyuJIT.
 306 //
 307 bool Compiler::isFullyImplmentedISAClass(InstructionSet isa)
 308 {
 309     switch (isa)
 310     {
 311         case InstructionSet_SSE2:
 312         case InstructionSet_SSE41:
 313         case InstructionSet_SSE42:
 314         case InstructionSet_AVX:
 315         case InstructionSet_AVX2:
 316         case InstructionSet_AES:
 317         case InstructionSet_BMI1:
 318         case InstructionSet_BMI2:
 319         case InstructionSet_FMA:
 320         case InstructionSet_PCLMULQDQ:
 321             return false;
 322
 323         case InstructionSet_SSE:
 324         case InstructionSet_SSE3:
 325         case InstructionSet_SSSE3:
 326         case InstructionSet_LZCNT:
 327         case InstructionSet_POPCNT:
 328             return true;
 329
 330         default:
 331             unreached();
 332     }
 333 }
 334
 335 //------------------------------------------------------------------------
 336 // isScalarISA:
 337 //
 338 // Arguments:
 339 //    isa - Instruction set
 340 // Return Value:
 341 //    true - if "isa" only contains scalar instructions
 342 //
 343 bool Compiler::isScalarISA(InstructionSet isa)
 344 {
 345     switch (isa)
 346     {
 347         case InstructionSet_BMI1:
 348         case InstructionSet_BMI2:
 349         case InstructionSet_LZCNT:
 350         case InstructionSet_POPCNT:
 351             return true;
 352
 353         default:
 354             return false;
 355     }
 356 }
 357
 358 //------------------------------------------------------------------------
 359 // compSupportsHWIntrinsic: compiler support of hardware intrinsics
 360 //
 361 // Arguments:
 362 //    isa - Instruction set
 363 // Return Value:
 364 //    true if
 365 //    - isa is a scalar ISA
 366 //    - isa is a SIMD ISA and featureSIMD=true
 367 //    - isa is fully implemented or EnableIncompleteISAClass=true
 368 bool Compiler::compSupportsHWIntrinsic(InstructionSet isa)
 369 {
 370     return (featureSIMD || isScalarISA(isa)) && (
 371 #ifdef DEBUG
 372                                                     JitConfig.EnableIncompleteISAClass() ||
 373 #endif
 374                                                     isFullyImplmentedISAClass(isa));
 375 }
 376
 377 static bool isTypeSupportedForIntrinsic(var_types type)
 378 {
 379 #ifdef _TARGET_X86_
 380     return !varTypeIsLong(type);
 381 #else
 382     return true;
 383 #endif
 384 }
 385
 386 //------------------------------------------------------------------------
 387 // impIsTableDrivenHWIntrinsic:
 388 //
 389 // Arguments:
 390 //    category - category of a HW intrinsic
 391 //
 392 // Return Value:
 393 //    returns true if this category can be table-driven in the importer
 394 //
 395 static bool impIsTableDrivenHWIntrinsic(HWIntrinsicCategory category, HWIntrinsicFlag flags)
 396 {
 397     // HW_Flag_NoCodeGen implies this intrinsic should be manually morphed in the importer.
 398     return category != HW_Category_Special && category != HW_Category_Scalar && (flags & HW_Flag_NoCodeGen) == 0;
 399 }
 400
 401 //------------------------------------------------------------------------
 402 // impHWIntrinsic: dispatch hardware intrinsics to their own implementation
 403 //
 404 // Arguments:
 405 //    intrinsic -- id of the intrinsic function.
 406 //    method    -- method handle of the intrinsic function.
 407 //    sig       -- signature of the intrinsic call
 408 //
 409 // Return Value:
 410 //    the expanded intrinsic.
 411 //
 412 GenTree* Compiler::impHWIntrinsic(NamedIntrinsic        intrinsic,
 413                                   CORINFO_METHOD_HANDLE method,
 414                                   CORINFO_SIG_INFO*     sig,
 415                                   bool                  mustExpand)
 416 {
 417     InstructionSet      isa      = isaOfHWIntrinsic(intrinsic);
 418     HWIntrinsicCategory category = categoryOfHWIntrinsic(intrinsic);
 419     HWIntrinsicFlag     flags    = flagsOfHWIntrinsic(intrinsic);
 420     int                 numArgs  = sig->numArgs;
 421     var_types           retType  = JITtype2varType(sig->retType);
 422     var_types           baseType = TYP_UNKNOWN;
 423
 424     if (retType == TYP_STRUCT && featureSIMD)
 425     {
 426         unsigned int sizeBytes;
 427         baseType = getBaseTypeAndSizeOfSIMDType(sig->retTypeSigClass, &sizeBytes);
 428         retType  = getSIMDTypeForSize(sizeBytes);
 429         assert(sizeBytes != 0 && baseType != TYP_UNKNOWN);
 430     }
 431
 432     // This intrinsic is supported if
 433     // - the ISA is available on the underlying hardware (compSupports returns true)
 434     // - the compiler supports this hardware intrinsics (compSupportsHWIntrinsic returns true)
 435     // - intrinsics do not require 64-bit registers (r64) on 32-bit platforms (isTypeSupportedForIntrinsic returns
 436     // true)
 437     bool issupported = compSupports(isa) && compSupportsHWIntrinsic(isa) && isTypeSupportedForIntrinsic(retType);
 438
 439     if (category == HW_Category_IsSupportedProperty)
 440     {
 441         return gtNewIconNode(issupported);
 442     }
 443     // - calling to unsupported intrinsics must throw PlatforNotSupportedException
 444     else if (!issupported)
 445     {
 446         return impUnsupportedHWIntrinsic(CORINFO_HELP_THROW_PLATFORM_NOT_SUPPORTED, method, sig, mustExpand);
 447     }
 448     else if (category == HW_Category_IMM)
 449     {
 450         GenTree* lastOp = impStackTop().val;
 451         if (!lastOp->IsCnsIntOrI() && !mustExpand)
 452         {
 453             // When the imm-argument is not a constant and we are not being forced to expand, we need to
 454             // return nullptr so a GT_CALL to the intrinsic method is emitted instead. The
 455             // intrinsic method is recursive and will be forced to expand, at which point
 456             // we emit some less efficient fallback code.
 457             return nullptr;
 458         }
 459     }
 460
 461     if ((flags & (HW_Flag_OneTypeGeneric | HW_Flag_TwoTypeGeneric)) != 0)
 462     {
 463         assert(baseType != TYP_UNKNOWN);
 464         // When the type argument is not a numeric type (and we are not being forced to expand), we need to
 465         // return nullptr so a GT_CALL to the intrinsic method is emitted that will throw NotSupportedException
 466         if (!varTypeIsArithmetic(baseType))
 467         {
 468             assert(!mustExpand);
 469             return nullptr;
 470         }
 471
 472         if ((flags & HW_Flag_TwoTypeGeneric) != 0)
 473         {
 474             // StaticCast<T, U> has two type parameters.
 475             assert(!mustExpand);
 476             assert(numArgs == 1);
 477             var_types srcType = getBaseTypeOfSIMDType(info.compCompHnd->getArgClass(sig, sig->args));
 478             assert(srcType != TYP_UNKNOWN);
 479             if (!varTypeIsArithmetic(srcType))
 480             {
 481                 return nullptr;
 482             }
 483         }
 484     }
 485
 486     if ((flags & HW_Flag_NoFloatingPointUsed) == 0)
 487     {
 488         // Set `compFloatingPointUsed` to cover the scenario where an intrinsic is being on SIMD fields, but
 489         // where no SIMD local vars are in use. This is the same logic as is used for FEATURE_SIMD.
 490         compFloatingPointUsed = true;
 491     }
 492
 493     // table-driven importer of simple intrinsics
 494     if (impIsTableDrivenHWIntrinsic(category, flags))
 495     {
 496         if (!varTypeIsSIMD(retType) || (flags & HW_Flag_BaseTypeFromArg))
 497         {
 498             if (retType != TYP_VOID)
 499             {
 500                 baseType = getBaseTypeOfSIMDType(info.compCompHnd->getArgClass(sig, sig->args));
 501             }
 502             else
 503             {
 504                 assert(category == HW_Category_MemoryStore);
 505                 baseType =
 506                     getBaseTypeOfSIMDType(info.compCompHnd->getArgClass(sig, info.compCompHnd->getArgNext(sig->args)));
 507             }
 508
 509             assert(baseType != TYP_UNKNOWN);
 510         }
 511
 512         unsigned                simdSize = simdSizeOfHWIntrinsic(intrinsic, sig);
 513         CORINFO_ARG_LIST_HANDLE argList  = sig->args;
 514         CORINFO_CLASS_HANDLE    argClass;
 515         var_types               argType = TYP_UNKNOWN;
 516
 517         assert(numArgs >= 0);
 518         assert(insOfHWIntrinsic(intrinsic, baseType) != INS_invalid);
 519         assert(simdSize == 32 || simdSize == 16);
 520
 521         GenTree* retNode = nullptr;
 522         GenTree* op1     = nullptr;
 523         GenTree* op2     = nullptr;
 524
 525         switch (numArgs)
 526         {
 527             case 0:
 528                 retNode = gtNewSimdHWIntrinsicNode(retType, intrinsic, baseType, simdSize);
 529                 break;
 530             case 1:
 531                 argType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, argList, &argClass)));
 532                 op1     = getArgForHWIntrinsic(argType, argClass);
 533                 retNode = gtNewSimdHWIntrinsicNode(retType, op1, intrinsic, baseType, simdSize);
 534                 break;
 535             case 2:
 536                 argType = JITtype2varType(
 537                     strip(info.compCompHnd->getArgType(sig, info.compCompHnd->getArgNext(argList), &argClass)));
 538                 op2 = getArgForHWIntrinsic(argType, argClass);
 539
 540                 argType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, argList, &argClass)));
 541                 op1     = getArgForHWIntrinsic(argType, argClass);
 542
 543                 retNode = gtNewSimdHWIntrinsicNode(retType, op1, op2, intrinsic, baseType, simdSize);
 544                 break;
 545
 546             case 3:
 547             {
 548                 CORINFO_ARG_LIST_HANDLE arg2 = info.compCompHnd->getArgNext(argList);
 549                 CORINFO_ARG_LIST_HANDLE arg3 = info.compCompHnd->getArgNext(arg2);
 550
 551                 argType      = JITtype2varType(strip(info.compCompHnd->getArgType(sig, arg3, &argClass)));
 552                 GenTree* op3 = getArgForHWIntrinsic(argType, argClass);
 553
 554                 argType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, arg2, &argClass)));
 555                 op2     = getArgForHWIntrinsic(argType, argClass);
 556
 557                 argType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, argList, &argClass)));
 558                 op1     = getArgForHWIntrinsic(argType, argClass);
 559
 560                 retNode = gtNewSimdHWIntrinsicNode(retType, op1, op2, op3, intrinsic, baseType, simdSize);
 561                 break;
 562             }
 563             default:
 564                 unreached();
 565         }
 566         return retNode;
 567     }
 568
 569     // other intrinsics need special importation
 570     switch (isa)
 571     {
 572         case InstructionSet_SSE:
 573             return impSSEIntrinsic(intrinsic, method, sig, mustExpand);
 574         case InstructionSet_SSE2:
 575             return impSSE2Intrinsic(intrinsic, method, sig, mustExpand);
 576         case InstructionSet_SSE3:
 577             return impSSE3Intrinsic(intrinsic, method, sig, mustExpand);
 578         case InstructionSet_SSSE3:
 579             return impSSSE3Intrinsic(intrinsic, method, sig, mustExpand);
 580         case InstructionSet_SSE41:
 581             return impSSE41Intrinsic(intrinsic, method, sig, mustExpand);
 582         case InstructionSet_SSE42:
 583             return impSSE42Intrinsic(intrinsic, method, sig, mustExpand);
 584         case InstructionSet_AVX:
 585             return impAVXIntrinsic(intrinsic, method, sig, mustExpand);
 586         case InstructionSet_AVX2:
 587             return impAVX2Intrinsic(intrinsic, method, sig, mustExpand);
 588
 589         case InstructionSet_AES:
 590             return impAESIntrinsic(intrinsic, method, sig, mustExpand);
 591         case InstructionSet_BMI1:
 592             return impBMI1Intrinsic(intrinsic, method, sig, mustExpand);
 593         case InstructionSet_BMI2:
 594             return impBMI2Intrinsic(intrinsic, method, sig, mustExpand);
 595         case InstructionSet_FMA:
 596             return impFMAIntrinsic(intrinsic, method, sig, mustExpand);
 597         case InstructionSet_LZCNT:
 598             return impLZCNTIntrinsic(intrinsic, method, sig, mustExpand);
 599         case InstructionSet_PCLMULQDQ:
 600             return impPCLMULQDQIntrinsic(intrinsic, method, sig, mustExpand);
 601         case InstructionSet_POPCNT:
 602             return impPOPCNTIntrinsic(intrinsic, method, sig, mustExpand);
 603         default:
 604             return nullptr;
 605     }
 606 }
 607
 608 GenTree* Compiler::impSSEIntrinsic(NamedIntrinsic        intrinsic,
 609                                    CORINFO_METHOD_HANDLE method,
 610                                    CORINFO_SIG_INFO*     sig,
 611                                    bool                  mustExpand)
 612 {
 613     GenTree* retNode  = nullptr;
 614     GenTree* op1      = nullptr;
 615     GenTree* op2      = nullptr;
 616     GenTree* op3      = nullptr;
 617     GenTree* op4      = nullptr;
 618     int      simdSize = simdSizeOfHWIntrinsic(intrinsic, sig);
 619
 620     // The Prefetch and StoreFence intrinsics don't take any SIMD operands
 621     // and have a simdSize of 0
 622     assert((simdSize == 16) || (simdSize == 0));
 623
 624     switch (intrinsic)
 625     {
 626         case NI_SSE_SetVector128:
 627         {
 628             assert(sig->numArgs == 4);
 629             assert(getBaseTypeOfSIMDType(sig->retTypeSigClass) == TYP_FLOAT);
 630
 631             op4 = impPopStack().val;
 632             op3 = impPopStack().val;
 633             op2 = impPopStack().val;
 634             op1 = impPopStack().val;
 635
 636             GenTree* left    = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op4, op3, NI_SSE_UnpackLow, TYP_FLOAT, simdSize);
 637             GenTree* right   = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op2, op1, NI_SSE_UnpackLow, TYP_FLOAT, simdSize);
 638             GenTree* control = gtNewIconNode(68, TYP_UBYTE);
 639
 640             retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, left, right, control, NI_SSE_Shuffle, TYP_FLOAT, simdSize);
 641             break;
 642         }
 643
 644         case NI_SSE_ConvertScalarToVector128Single:
 645         {
 646             assert(sig->numArgs == 2);
 647             assert(getBaseTypeOfSIMDType(sig->retTypeSigClass) == TYP_FLOAT);
 648
 649 #ifdef _TARGET_X86_
 650             CORINFO_CLASS_HANDLE argClass;
 651
 652             CORINFO_ARG_LIST_HANDLE argLst = info.compCompHnd->getArgNext(sig->args);
 653             CorInfoType             corType =
 654                 strip(info.compCompHnd->getArgType(sig, argLst, &argClass)); // type of the second argument
 655
 656             if (varTypeIsLong(JITtype2varType(corType)))
 657             {
 658                 return impUnsupportedHWIntrinsic(CORINFO_HELP_THROW_PLATFORM_NOT_SUPPORTED, method, sig, mustExpand);
 659             }
 660 #endif // _TARGET_X86_
 661
 662             op2     = impPopStack().val;
 663             op1     = impSIMDPopStack(TYP_SIMD16);
 664             retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, op2, intrinsic, TYP_FLOAT, simdSize);
 665             break;
 666         }
 667
 668         case NI_SSE_ReciprocalScalar:
 669         case NI_SSE_ReciprocalSqrtScalar:
 670         case NI_SSE_SqrtScalar:
 671         {
 672             assert((sig->numArgs == 1) || (sig->numArgs == 2));
 673             assert(getBaseTypeOfSIMDType(sig->retTypeSigClass) == TYP_FLOAT);
 674
 675             if (sig->numArgs == 2)
 676             {
 677                 op2 = impSIMDPopStack(TYP_SIMD16);
 678             }
 679
 680             op1     = impSIMDPopStack(TYP_SIMD16);
 681             retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, op2, intrinsic, TYP_FLOAT, simdSize);
 682             break;
 683         }
 684
 685         case NI_SSE_MoveMask:
 686             assert(sig->numArgs == 1);
 687             assert(JITtype2varType(sig->retType) == TYP_INT);
 688             assert(getBaseTypeOfSIMDType(info.compCompHnd->getArgClass(sig, sig->args)) == TYP_FLOAT);
 689             op1     = impSIMDPopStack(TYP_SIMD16);
 690             retNode = gtNewSimdHWIntrinsicNode(TYP_INT, op1, intrinsic, TYP_FLOAT, simdSize);
 691             break;
 692
 693         case NI_SSE_Prefetch0:
 694         case NI_SSE_Prefetch1:
 695         case NI_SSE_Prefetch2:
 696         case NI_SSE_PrefetchNonTemporal:
 697         {
 698             assert(sig->numArgs == 1);
 699             assert(JITtype2varType(sig->retType) == TYP_VOID);
 700             op1     = impPopStack().val;
 701             retNode = gtNewSimdHWIntrinsicNode(TYP_VOID, op1, intrinsic, TYP_UBYTE, 0);
 702             break;
 703         }
 704
 705         case NI_SSE_SetAllVector128:
 706             assert(sig->numArgs == 1);
 707             assert(getBaseTypeOfSIMDType(sig->retTypeSigClass) == TYP_FLOAT);
 708             op1     = impPopStack().val;
 709             retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, gtCloneExpr(op1), gtNewIconNode(0), NI_SSE_Shuffle,
 710                                                TYP_FLOAT, simdSize);
 711             break;
 712
 713         case NI_SSE_StoreFence:
 714             assert(sig->numArgs == 0);
 715             assert(JITtype2varType(sig->retType) == TYP_VOID);
 716             retNode = gtNewSimdHWIntrinsicNode(TYP_VOID, intrinsic, TYP_VOID, 0);
 717             break;
 718
 719         default:
 720             JITDUMP("Not implemented hardware intrinsic");
 721             break;
 722     }
 723     return retNode;
 724 }
 725
 726 GenTree* Compiler::impSSE2Intrinsic(NamedIntrinsic        intrinsic,
 727                                     CORINFO_METHOD_HANDLE method,
 728                                     CORINFO_SIG_INFO*     sig,
 729                                     bool                  mustExpand)
 730 {
 731     GenTree*  retNode  = nullptr;
 732     GenTree*  op1      = nullptr;
 733     GenTree*  op2      = nullptr;
 734     int       ival     = -1;
 735     int       simdSize = simdSizeOfHWIntrinsic(intrinsic, sig);
 736     var_types baseType = TYP_UNKNOWN;
 737     var_types retType  = TYP_UNKNOWN;
 738
 739     // The  fencing intrinsics don't take any operands and simdSize is 0
 740     assert((simdSize == 16) || (simdSize == 0));
 741
 742     CORINFO_ARG_LIST_HANDLE argList = sig->args;
 743     CORINFO_CLASS_HANDLE    argClass;
 744     var_types               argType = TYP_UNKNOWN;
 745
 746     switch (intrinsic)
 747     {
 748         case NI_SSE2_CompareLessThan:
 749         {
 750             assert(sig->numArgs == 2);
 751             op2      = impSIMDPopStack(TYP_SIMD16);
 752             op1      = impSIMDPopStack(TYP_SIMD16);
 753             baseType = getBaseTypeOfSIMDType(sig->retTypeSigClass);
 754             if (baseType == TYP_DOUBLE)
 755             {
 756                 retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, op2, intrinsic, baseType, simdSize);
 757             }
 758             else
 759             {
 760                 retNode =
 761                     gtNewSimdHWIntrinsicNode(TYP_SIMD16, op2, op1, NI_SSE2_CompareGreaterThan, baseType, simdSize);
 762             }
 763             break;
 764         }
 765
 766         case NI_SSE2_ConvertScalarToVector128Double:
 767         {
 768             assert(sig->numArgs == 2);
 769             assert(getBaseTypeOfSIMDType(sig->retTypeSigClass) == TYP_DOUBLE);
 770
 771             argList = info.compCompHnd->getArgNext(sig->args);
 772             CorInfoType corType =
 773                 strip(info.compCompHnd->getArgType(sig, argList, &argClass)); // type of the second argument
 774
 775             baseType = JITtype2varType(corType);
 776
 777 #ifdef _TARGET_X86_
 778             if (varTypeIsLong(JITtype2varType(corType)))
 779             {
 780                 return impUnsupportedHWIntrinsic(CORINFO_HELP_THROW_PLATFORM_NOT_SUPPORTED, method, sig, mustExpand);
 781             }
 782 #endif // _TARGET_X86_
 783
 784             if (baseType == TYP_STRUCT)
 785             {
 786                 baseType = TYP_FLOAT; // it is the only type passed as Vector
 787                 op2      = impSIMDPopStack(TYP_SIMD16);
 788             }
 789             else
 790             {
 791                 op2 = impPopStack().val;
 792             }
 793
 794             op1     = impSIMDPopStack(TYP_SIMD16);
 795             retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, op2, intrinsic, baseType, simdSize);
 796
 797             break;
 798         }
 799
 800         case NI_SSE2_ConvertScalarToVector128Int64:
 801         case NI_SSE2_ConvertScalarToVector128UInt64:
 802         {
 803             assert(sig->numArgs == 1);
 804             baseType = getBaseTypeOfSIMDType(sig->retTypeSigClass);
 805             assert(baseType == TYP_LONG || baseType == TYP_ULONG);
 806
 807 #ifdef _TARGET_X86_
 808             return impUnsupportedHWIntrinsic(CORINFO_HELP_THROW_PLATFORM_NOT_SUPPORTED, method, sig, mustExpand);
 809 #endif // _TARGET_X86_
 810
 811             op1     = impPopStack().val;
 812             retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, intrinsic, baseType, simdSize);
 813             break;
 814         }
 815
 816         case NI_SSE2_ConvertScalarToVector128Single:
 817         {
 818             assert(sig->numArgs == 2);
 819             assert(getBaseTypeOfSIMDType(sig->retTypeSigClass) == TYP_FLOAT);
 820
 821             op2     = impSIMDPopStack(TYP_SIMD16);
 822             op1     = impSIMDPopStack(TYP_SIMD16);
 823             retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, op2, intrinsic, TYP_DOUBLE, simdSize);
 824             break;
 825         }
 826
 827         case NI_SSE2_ConvertToInt32:
 828         case NI_SSE2_ConvertToInt64:
 829         {
 830             assert(sig->numArgs == 1);
 831             op1      = impSIMDPopStack(TYP_SIMD16);
 832             retType  = JITtype2varType(sig->retType);
 833             baseType = getBaseTypeOfSIMDType(info.compCompHnd->getArgClass(sig, sig->args));
 834             retNode  = gtNewSimdHWIntrinsicNode(retType, op1, intrinsic, baseType, simdSize);
 835             break;
 836         }
 837
 838         case NI_SSE2_ConvertToUInt32:
 839         case NI_SSE2_ConvertToUInt64:
 840         {
 841             assert(sig->numArgs == 1);
 842             op1      = impSIMDPopStack(TYP_SIMD16);
 843             baseType = JITtype2varType(sig->retType);
 844             retNode  = gtNewSimdHWIntrinsicNode(baseType, op1, intrinsic, baseType, simdSize);
 845             break;
 846         }
 847
 848         case NI_SSE2_LoadFence:
 849         case NI_SSE2_MemoryFence:
 850         {
 851             assert(sig->numArgs == 0);
 852             assert(JITtype2varType(sig->retType) == TYP_VOID);
 853             assert(simdSize == 0);
 854
 855             retNode = gtNewSimdHWIntrinsicNode(TYP_VOID, intrinsic, TYP_VOID, simdSize);
 856             break;
 857         }
 858
 859         case NI_SSE2_MoveMask:
 860         {
 861             assert(sig->numArgs == 1);
 862             retType = JITtype2varType(sig->retType);
 863             assert(retType == TYP_INT);
 864             op1      = impSIMDPopStack(TYP_SIMD16);
 865             baseType = getBaseTypeOfSIMDType(info.compCompHnd->getArgClass(sig, sig->args));
 866             retNode  = gtNewSimdHWIntrinsicNode(retType, op1, intrinsic, baseType, simdSize);
 867             break;
 868         }
 869
 870         default:
 871             JITDUMP("Not implemented hardware intrinsic");
 872             break;
 873     }
 874     return retNode;
 875 }
 876
 877 GenTree* Compiler::impSSE3Intrinsic(NamedIntrinsic        intrinsic,
 878                                     CORINFO_METHOD_HANDLE method,
 879                                     CORINFO_SIG_INFO*     sig,
 880                                     bool                  mustExpand)
 881 {
 882     return nullptr;
 883 }
 884
 885 GenTree* Compiler::impSSSE3Intrinsic(NamedIntrinsic        intrinsic,
 886                                      CORINFO_METHOD_HANDLE method,
 887                                      CORINFO_SIG_INFO*     sig,
 888                                      bool                  mustExpand)
 889 {
 890     return nullptr;
 891 }
 892
 893 GenTree* Compiler::impSSE41Intrinsic(NamedIntrinsic        intrinsic,
 894                                      CORINFO_METHOD_HANDLE method,
 895                                      CORINFO_SIG_INFO*     sig,
 896                                      bool                  mustExpand)
 897 {
 898     return nullptr;
 899 }
 900
 901 GenTree* Compiler::impSSE42Intrinsic(NamedIntrinsic        intrinsic,
 902                                      CORINFO_METHOD_HANDLE method,
 903                                      CORINFO_SIG_INFO*     sig,
 904                                      bool                  mustExpand)
 905 {
 906     GenTree*  retNode  = nullptr;
 907     GenTree*  op1      = nullptr;
 908     GenTree*  op2      = nullptr;
 909     var_types callType = JITtype2varType(sig->retType);
 910
 911     CORINFO_ARG_LIST_HANDLE argList = sig->args;
 912     CORINFO_CLASS_HANDLE    argClass;
 913     CorInfoType             corType;
 914     switch (intrinsic)
 915     {
 916         case NI_SSE42_Crc32:
 917             assert(sig->numArgs == 2);
 918             op2     = impPopStack().val;
 919             op1     = impPopStack().val;
 920             argList = info.compCompHnd->getArgNext(argList);                        // the second argument
 921             corType = strip(info.compCompHnd->getArgType(sig, argList, &argClass)); // type of the second argument
 922
 923             retNode = gtNewScalarHWIntrinsicNode(callType, op1, op2, NI_SSE42_Crc32);
 924
 925             // TODO - currently we use the BaseType to bring the type of the second argument
 926             // to the code generator. May encode the overload info in other way.
 927             retNode->gtHWIntrinsic.gtSIMDBaseType = JITtype2varType(corType);
 928             break;
 929
 930         default:
 931             JITDUMP("Not implemented hardware intrinsic");
 932             break;
 933     }
 934     return retNode;
 935 }
 936
 937 GenTree* Compiler::impAVXIntrinsic(NamedIntrinsic        intrinsic,
 938                                    CORINFO_METHOD_HANDLE method,
 939                                    CORINFO_SIG_INFO*     sig,
 940                                    bool                  mustExpand)
 941 {
 942     GenTree*  retNode  = nullptr;
 943     GenTree*  op1      = nullptr;
 944     GenTree*  op2      = nullptr;
 945     var_types baseType = TYP_UNKNOWN;
 946     switch (intrinsic)
 947     {
 948         default:
 949             JITDUMP("Not implemented hardware intrinsic");
 950             break;
 951     }
 952     return retNode;
 953 }
 954
 955 GenTree* Compiler::impAVX2Intrinsic(NamedIntrinsic        intrinsic,
 956                                     CORINFO_METHOD_HANDLE method,
 957                                     CORINFO_SIG_INFO*     sig,
 958                                     bool                  mustExpand)
 959 {
 960     GenTree*  retNode  = nullptr;
 961     GenTree*  op1      = nullptr;
 962     GenTree*  op2      = nullptr;
 963     var_types baseType = TYP_UNKNOWN;
 964     switch (intrinsic)
 965     {
 966         default:
 967             JITDUMP("Not implemented hardware intrinsic");
 968             break;
 969     }
 970     return retNode;
 971 }
 972
 973 GenTree* Compiler::impAESIntrinsic(NamedIntrinsic        intrinsic,
 974                                    CORINFO_METHOD_HANDLE method,
 975                                    CORINFO_SIG_INFO*     sig,
 976                                    bool                  mustExpand)
 977 {
 978     return nullptr;
 979 }
 980
 981 GenTree* Compiler::impBMI1Intrinsic(NamedIntrinsic        intrinsic,
 982                                     CORINFO_METHOD_HANDLE method,
 983                                     CORINFO_SIG_INFO*     sig,
 984                                     bool                  mustExpand)
 985 {
 986     return nullptr;
 987 }
 988
 989 GenTree* Compiler::impBMI2Intrinsic(NamedIntrinsic        intrinsic,
 990                                     CORINFO_METHOD_HANDLE method,
 991                                     CORINFO_SIG_INFO*     sig,
 992                                     bool                  mustExpand)
 993 {
 994     return nullptr;
 995 }
 996
 997 GenTree* Compiler::impFMAIntrinsic(NamedIntrinsic        intrinsic,
 998                                    CORINFO_METHOD_HANDLE method,
 999                                    CORINFO_SIG_INFO*     sig,
1000                                    bool                  mustExpand)
1001 {
1002     return nullptr;
1003 }
1004
1005 GenTree* Compiler::impLZCNTIntrinsic(NamedIntrinsic        intrinsic,
1006                                      CORINFO_METHOD_HANDLE method,
1007                                      CORINFO_SIG_INFO*     sig,
1008                                      bool                  mustExpand)
1009 {
1010     assert(sig->numArgs == 1);
1011     var_types callType = JITtype2varType(sig->retType);
1012     return gtNewScalarHWIntrinsicNode(callType, impPopStack().val, NI_LZCNT_LeadingZeroCount);
1013 }
1014
1015 GenTree* Compiler::impPCLMULQDQIntrinsic(NamedIntrinsic        intrinsic,
1016                                          CORINFO_METHOD_HANDLE method,
1017                                          CORINFO_SIG_INFO*     sig,
1018                                          bool                  mustExpand)
1019 {
1020     return nullptr;
1021 }
1022
1023 GenTree* Compiler::impPOPCNTIntrinsic(NamedIntrinsic        intrinsic,
1024                                       CORINFO_METHOD_HANDLE method,
1025                                       CORINFO_SIG_INFO*     sig,
1026                                       bool                  mustExpand)
1027 {
1028     assert(sig->numArgs == 1);
1029     var_types callType = JITtype2varType(sig->retType);
1030     return gtNewScalarHWIntrinsicNode(callType, impPopStack().val, NI_POPCNT_PopCount);
1031 }
1032
1033 #endif // FEATURE_HW_INTRINSICS