src/jit/hwintrinsicxarch.cpp

   1 // Licensed to the .NET Foundation under one or more agreements.
   2 // The .NET Foundation licenses this file to you under the MIT license.
   3 // See the LICENSE file in the project root for more information.
   4
   5 #include "jitpch.h"
   6
   7 #if FEATURE_HW_INTRINSICS
   8
   9 struct HWIntrinsicInfo
  10 {
  11     NamedIntrinsic      intrinsicID;
  12     const char*         intrinsicName;
  13     InstructionSet      isa;
  14     int                 ival;
  15     unsigned            simdSize;
  16     int                 numArgs;
  17     instruction         ins[10];
  18     HWIntrinsicCategory category;
  19     HWIntrinsicFlag     flags;
  20 };
  21
  22 static const HWIntrinsicInfo hwIntrinsicInfoArray[] = {
  23 #define HARDWARE_INTRINSIC(id, name, isa, ival, size, numarg, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, category, flag) \
  24     {NI_##id, name, InstructionSet_##isa, ival, size, numarg, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, category, flag},
  25 #include "hwintrinsiclistxarch.h"
  26 };
  27
  28 extern const char* getHWIntrinsicName(NamedIntrinsic intrinsic)
  29 {
  30     return hwIntrinsicInfoArray[intrinsic - NI_HW_INTRINSIC_START - 1].intrinsicName;
  31 }
  32
  33 //------------------------------------------------------------------------
  34 // lookupHWIntrinsicISA: map class name to InstructionSet value
  35 //
  36 // Arguments:
  37 //    className -- class name in System.Runtime.Intrinsics.X86
  38 //
  39 // Return Value:
  40 //    Id for the ISA class.
  41 //
  42 InstructionSet Compiler::lookupHWIntrinsicISA(const char* className)
  43 {
  44     if (className != nullptr)
  45     {
  46         if (className[0] == 'A')
  47         {
  48             if (strcmp(className, "Aes") == 0)
  49             {
  50                 return InstructionSet_AES;
  51             }
  52             else if (strcmp(className, "Avx") == 0)
  53             {
  54                 return InstructionSet_AVX;
  55             }
  56             else if (strcmp(className, "Avx2") == 0)
  57             {
  58                 return InstructionSet_AVX2;
  59             }
  60         }
  61         if (className[0] == 'S')
  62         {
  63             if (strcmp(className, "Sse") == 0)
  64             {
  65                 return InstructionSet_SSE;
  66             }
  67             else if (strcmp(className, "Sse2") == 0)
  68             {
  69                 return InstructionSet_SSE2;
  70             }
  71             else if (strcmp(className, "Sse3") == 0)
  72             {
  73                 return InstructionSet_SSE3;
  74             }
  75             else if (strcmp(className, "Ssse3") == 0)
  76             {
  77                 return InstructionSet_SSSE3;
  78             }
  79             else if (strcmp(className, "Sse41") == 0)
  80             {
  81                 return InstructionSet_SSE41;
  82             }
  83             else if (strcmp(className, "Sse42") == 0)
  84             {
  85                 return InstructionSet_SSE42;
  86             }
  87         }
  88
  89         if (strcmp(className, "Bmi1") == 0)
  90         {
  91             return InstructionSet_BMI1;
  92         }
  93         else if (strcmp(className, "Bmi2") == 0)
  94         {
  95             return InstructionSet_BMI2;
  96         }
  97         else if (strcmp(className, "Fma") == 0)
  98         {
  99             return InstructionSet_FMA;
 100         }
 101         else if (strcmp(className, "Lzcnt") == 0)
 102         {
 103             return InstructionSet_LZCNT;
 104         }
 105         else if (strcmp(className, "Pclmulqdq") == 0)
 106         {
 107             return InstructionSet_PCLMULQDQ;
 108         }
 109         else if (strcmp(className, "Popcnt") == 0)
 110         {
 111             return InstructionSet_POPCNT;
 112         }
 113     }
 114
 115     JITDUMP("Unsupported ISA.\n");
 116     return InstructionSet_ILLEGAL;
 117 }
 118
 119 //------------------------------------------------------------------------
 120 // lookupHWIntrinsic: map intrinsic name to named intrinsic value
 121 //
 122 // Arguments:
 123 //    methodName -- name of the intrinsic function.
 124 //    isa        -- instruction set of the intrinsic.
 125 //
 126 // Return Value:
 127 //    Id for the hardware intrinsic.
 128 //
 129 // TODO-Throughput: replace sequential search by binary search
 130 NamedIntrinsic Compiler::lookupHWIntrinsic(const char* methodName, InstructionSet isa)
 131 {
 132     NamedIntrinsic result = NI_Illegal;
 133     if (isa != InstructionSet_ILLEGAL)
 134     {
 135         for (int i = 0; i < NI_HW_INTRINSIC_END - NI_HW_INTRINSIC_START; i++)
 136         {
 137             if (isa == hwIntrinsicInfoArray[i].isa && strcmp(methodName, hwIntrinsicInfoArray[i].intrinsicName) == 0)
 138             {
 139                 result = hwIntrinsicInfoArray[i].intrinsicID;
 140             }
 141         }
 142     }
 143     return result;
 144 }
 145
 146 //------------------------------------------------------------------------
 147 // isaOfHWIntrinsic: map named intrinsic value to its instruction set
 148 //
 149 // Arguments:
 150 //    intrinsic -- id of the intrinsic function.
 151 //
 152 // Return Value:
 153 //    instruction set of the intrinsic.
 154 //
 155 InstructionSet Compiler::isaOfHWIntrinsic(NamedIntrinsic intrinsic)
 156 {
 157     assert(intrinsic != NI_Illegal);
 158     assert(intrinsic > NI_HW_INTRINSIC_START && intrinsic < NI_HW_INTRINSIC_END);
 159     return hwIntrinsicInfoArray[intrinsic - NI_HW_INTRINSIC_START - 1].isa;
 160 }
 161
 162 //------------------------------------------------------------------------
 163 // ivalOfHWIntrinsic: get the imm8 value of this intrinsic from the hwIntrinsicInfoArray table
 164 //
 165 // Arguments:
 166 //    intrinsic -- id of the intrinsic function.
 167 //
 168 // Return Value:
 169 //     The imm8 value that is implicit for this intrinsic, or -1 for intrinsics that do not take an immediate, or for
 170 //     which the immediate is an explicit argument.
 171 //
 172 int Compiler::ivalOfHWIntrinsic(NamedIntrinsic intrinsic)
 173 {
 174     assert(intrinsic != NI_Illegal);
 175     assert(intrinsic > NI_HW_INTRINSIC_START && intrinsic < NI_HW_INTRINSIC_END);
 176     return hwIntrinsicInfoArray[intrinsic - NI_HW_INTRINSIC_START - 1].ival;
 177 }
 178
 179 //------------------------------------------------------------------------
 180 // simdSizeOfHWIntrinsic: get the SIMD size of this intrinsic
 181 //
 182 // Arguments:
 183 //    intrinsic -- id of the intrinsic function.
 184 //
 185 // Return Value:
 186 //     the SIMD size of this intrinsic
 187 //         - from the hwIntrinsicInfoArray table if intrinsic has NO HW_Flag_UnfixedSIMDSize
 188 //         - TODO-XArch-NYI - from the signature if intrinsic has HW_Flag_UnfixedSIMDSize
 189 //
 190 // Note - this function is only used by the importer
 191 //        after importation (i.e., codegen), we can get the SIMD size from GenTreeHWIntrinsic IR
 192 static unsigned simdSizeOfHWIntrinsic(NamedIntrinsic intrinsic, CORINFO_SIG_INFO* sig)
 193 {
 194     assert(intrinsic > NI_HW_INTRINSIC_START && intrinsic < NI_HW_INTRINSIC_END);
 195     assert((hwIntrinsicInfoArray[intrinsic - NI_HW_INTRINSIC_START - 1].flags & HW_Flag_UnfixedSIMDSize) == 0);
 196     return hwIntrinsicInfoArray[intrinsic - NI_HW_INTRINSIC_START - 1].simdSize;
 197 }
 198
 199 //------------------------------------------------------------------------
 200 // numArgsOfHWIntrinsic: get the number of arguments
 201 //
 202 // Arguments:
 203 //    intrinsic -- id of the intrinsic function.
 204 //
 205 // Return Value:
 206 //     number of arguments
 207 //
 208 int Compiler::numArgsOfHWIntrinsic(NamedIntrinsic intrinsic)
 209 {
 210     assert(intrinsic != NI_Illegal);
 211     assert(intrinsic > NI_HW_INTRINSIC_START && intrinsic < NI_HW_INTRINSIC_END);
 212     return hwIntrinsicInfoArray[intrinsic - NI_HW_INTRINSIC_START - 1].numArgs;
 213 }
 214
 215 //------------------------------------------------------------------------
 216 // insOfHWIntrinsic: get the instruction of the given intrinsic
 217 //
 218 // Arguments:
 219 //    intrinsic -- id of the intrinsic function.
 220 //    type      -- vector base type of this intrinsic
 221 //
 222 // Return Value:
 223 //     the instruction of the given intrinsic on the base type
 224 //     return INS_invalid for unsupported base types
 225 //
 226 instruction Compiler::insOfHWIntrinsic(NamedIntrinsic intrinsic, var_types type)
 227 {
 228     assert(intrinsic != NI_Illegal);
 229     assert(intrinsic > NI_HW_INTRINSIC_START && intrinsic < NI_HW_INTRINSIC_END);
 230     assert(type >= TYP_BYTE && type <= TYP_DOUBLE);
 231     return hwIntrinsicInfoArray[intrinsic - NI_HW_INTRINSIC_START - 1].ins[type - TYP_BYTE];
 232 }
 233
 234 //------------------------------------------------------------------------
 235 // categoryOfHWIntrinsic: get the category of the given intrinsic
 236 //
 237 // Arguments:
 238 //    intrinsic -- id of the intrinsic function.
 239 //
 240 // Return Value:
 241 //     the category of the given intrinsic
 242 //
 243 HWIntrinsicCategory Compiler::categoryOfHWIntrinsic(NamedIntrinsic intrinsic)
 244 {
 245     assert(intrinsic != NI_Illegal);
 246     assert(intrinsic > NI_HW_INTRINSIC_START && intrinsic < NI_HW_INTRINSIC_END);
 247     return hwIntrinsicInfoArray[intrinsic - NI_HW_INTRINSIC_START - 1].category;
 248 }
 249
 250 //------------------------------------------------------------------------
 251 // HWIntrinsicFlag: get the flags of the given intrinsic
 252 //
 253 // Arguments:
 254 //    intrinsic -- id of the intrinsic function.
 255 //
 256 // Return Value:
 257 //     the flags of the given intrinsic
 258 //
 259 HWIntrinsicFlag Compiler::flagsOfHWIntrinsic(NamedIntrinsic intrinsic)
 260 {
 261     assert(intrinsic != NI_Illegal);
 262     assert(intrinsic > NI_HW_INTRINSIC_START && intrinsic < NI_HW_INTRINSIC_END);
 263     return hwIntrinsicInfoArray[intrinsic - NI_HW_INTRINSIC_START - 1].flags;
 264 }
 265
 266 //------------------------------------------------------------------------
 267 // getArgForHWIntrinsic: get the argument from the stack and match  the signature
 268 //
 269 // Arguments:
 270 //    argType   -- the required type of argument
 271 //    argClass  -- the class handle of argType
 272 //
 273 // Return Value:
 274 //     get the argument at the given index from the stack and match  the signature
 275 //
 276 GenTree* Compiler::getArgForHWIntrinsic(var_types argType, CORINFO_CLASS_HANDLE argClass)
 277 {
 278     GenTree* arg = nullptr;
 279     if (argType == TYP_STRUCT)
 280     {
 281         unsigned int argSizeBytes;
 282         var_types    base = getBaseTypeAndSizeOfSIMDType(argClass, &argSizeBytes);
 283         argType           = getSIMDTypeForSize(argSizeBytes);
 284         assert(argType == TYP_SIMD32 || argType == TYP_SIMD16);
 285         arg = impSIMDPopStack(argType);
 286         assert(arg->TypeGet() == TYP_SIMD16 || arg->TypeGet() == TYP_SIMD32);
 287     }
 288     else
 289     {
 290         assert(varTypeIsArithmetic(argType));
 291         arg = impPopStack().val;
 292         assert(varTypeIsArithmetic(arg->TypeGet()));
 293         assert(genActualType(arg->gtType) == genActualType(argType));
 294     }
 295     return arg;
 296 }
 297
 298 //------------------------------------------------------------------------
 299 // isFullyImplmentedISAClass: return true if all the hardware intrinsics
 300 //    of this ISA are implemented in RyuJIT.
 301 //
 302 // Arguments:
 303 //    isa - Instruction set
 304 // Return Value:
 305 //    true - all the hardware intrinsics of "isa" are implemented in RyuJIT.
 306 //
 307 bool Compiler::isFullyImplmentedISAClass(InstructionSet isa)
 308 {
 309     switch (isa)
 310     {
 311         case InstructionSet_SSE:
 312         case InstructionSet_SSE2:
 313         case InstructionSet_SSE3:
 314         case InstructionSet_SSSE3:
 315         case InstructionSet_SSE41:
 316         case InstructionSet_SSE42:
 317         case InstructionSet_AVX:
 318         case InstructionSet_AVX2:
 319         case InstructionSet_AES:
 320         case InstructionSet_BMI1:
 321         case InstructionSet_BMI2:
 322         case InstructionSet_FMA:
 323         case InstructionSet_PCLMULQDQ:
 324             return false;
 325
 326         case InstructionSet_LZCNT:
 327         case InstructionSet_POPCNT:
 328             return true;
 329
 330         default:
 331             unreached();
 332     }
 333 }
 334
 335 //------------------------------------------------------------------------
 336 // isScalarISA:
 337 //
 338 // Arguments:
 339 //    isa - Instruction set
 340 // Return Value:
 341 //    true - if "isa" only contains scalar instructions
 342 //
 343 bool Compiler::isScalarISA(InstructionSet isa)
 344 {
 345     switch (isa)
 346     {
 347         case InstructionSet_BMI1:
 348         case InstructionSet_BMI2:
 349         case InstructionSet_LZCNT:
 350         case InstructionSet_POPCNT:
 351             return true;
 352
 353         default:
 354             return false;
 355     }
 356 }
 357
 358 //------------------------------------------------------------------------
 359 // compSupportsHWIntrinsic: compiler support of hardware intrinsics
 360 //
 361 // Arguments:
 362 //    isa - Instruction set
 363 // Return Value:
 364 //    true if
 365 //    - isa is a scalar ISA
 366 //    - isa is a SIMD ISA and featureSIMD=true
 367 //    - isa is fully implemented or EnableIncompleteISAClass=true
 368 bool Compiler::compSupportsHWIntrinsic(InstructionSet isa)
 369 {
 370     return (featureSIMD || isScalarISA(isa)) && (
 371 #ifdef DEBUG
 372                                                     JitConfig.EnableIncompleteISAClass() ||
 373 #endif
 374                                                     isFullyImplmentedISAClass(isa));
 375 }
 376
 377 static bool isTypeSupportedForIntrinsic(var_types type)
 378 {
 379 #ifdef _TARGET_X86_
 380     return !varTypeIsLong(type);
 381 #else
 382     return true;
 383 #endif
 384 }
 385
 386 //------------------------------------------------------------------------
 387 // impUnsupportedHWIntrinsic: returns a node for an unsupported HWIntrinsic
 388 //
 389 // Arguments:
 390 //    helper     - JIT helper ID for the exception to be thrown
 391 //    method     - method handle of the intrinsic function.
 392 //    sig        - signature of the intrinsic call
 393 //    mustExpand - true if the intrinsic must return a GenTree*; otherwise, false
 394 //
 395 // Return Value:
 396 //    a gtNewMustThrowException if mustExpand is true; otherwise, nullptr
 397 //
 398 GenTree* Compiler::impUnsupportedHWIntrinsic(unsigned              helper,
 399                                              CORINFO_METHOD_HANDLE method,
 400                                              CORINFO_SIG_INFO*     sig,
 401                                              bool                  mustExpand)
 402 {
 403     // We've hit some error case and may need to return a node for the given error.
 404     //
 405     // When `mustExpand=false`, we are attempting to inline the intrinsic directly into another method. In this
 406     // scenario, we need to return `nullptr` so that a GT_CALL to the intrinsic is emitted instead. This is to
 407     // ensure that everything continues to behave correctly when optimizations are enabled (e.g. things like the
 408     // inliner may expect the node we return to have a certain signature, and the `MustThrowException` node won't
 409     // match that).
 410     //
 411     // When `mustExpand=true`, we are in a GT_CALL to the intrinsic and are attempting to JIT it. This will generally
 412     // be in response to an indirect call (e.g. done via reflection) or in response to an earlier attempt returning
 413     // `nullptr` (under `mustExpand=false`). In that scenario, we are safe to return the `MustThrowException` node.
 414
 415     if (mustExpand)
 416     {
 417         for (unsigned i = 0; i < sig->numArgs; i++)
 418         {
 419             impPopStack();
 420         }
 421
 422         return gtNewMustThrowException(helper, JITtype2varType(sig->retType), sig->retTypeClass);
 423     }
 424     else
 425     {
 426         return nullptr;
 427     }
 428 }
 429
 430 //------------------------------------------------------------------------
 431 // impIsTableDrivenHWIntrinsic:
 432 //
 433 // Arguments:
 434 //    category - category of a HW intrinsic
 435 //
 436 // Return Value:
 437 //    returns true if this category can be table-driven in the importer
 438 //
 439 static bool impIsTableDrivenHWIntrinsic(HWIntrinsicCategory category, HWIntrinsicFlag flags)
 440 {
 441     // HW_Flag_NoCodeGen implies this intrinsic should be manually morphed in the importer.
 442     return category != HW_Category_Special && category != HW_Category_Scalar && (flags & HW_Flag_NoCodeGen) == 0;
 443 }
 444
 445 //------------------------------------------------------------------------
 446 // impX86HWIntrinsic: dispatch hardware intrinsics to their own implementation
 447 //
 448 // Arguments:
 449 //    intrinsic -- id of the intrinsic function.
 450 //    method    -- method handle of the intrinsic function.
 451 //    sig       -- signature of the intrinsic call
 452 //
 453 // Return Value:
 454 //    the expanded intrinsic.
 455 //
 456 GenTree* Compiler::impX86HWIntrinsic(NamedIntrinsic        intrinsic,
 457                                      CORINFO_METHOD_HANDLE method,
 458                                      CORINFO_SIG_INFO*     sig,
 459                                      bool                  mustExpand)
 460 {
 461     InstructionSet      isa      = isaOfHWIntrinsic(intrinsic);
 462     HWIntrinsicCategory category = categoryOfHWIntrinsic(intrinsic);
 463     HWIntrinsicFlag     flags    = flagsOfHWIntrinsic(intrinsic);
 464     int                 numArgs  = sig->numArgs;
 465     var_types           retType  = JITtype2varType(sig->retType);
 466     var_types           baseType = TYP_UNKNOWN;
 467     if (retType == TYP_STRUCT && featureSIMD)
 468     {
 469         unsigned int sizeBytes;
 470         baseType = getBaseTypeAndSizeOfSIMDType(sig->retTypeSigClass, &sizeBytes);
 471         retType  = getSIMDTypeForSize(sizeBytes);
 472         assert(sizeBytes != 0 && baseType != TYP_UNKNOWN);
 473     }
 474
 475     // This intrinsic is supported if
 476     // - the ISA is available on the underlying hardware (compSupports returns true)
 477     // - the compiler supports this hardware intrinsics (compSupportsHWIntrinsic returns true)
 478     // - intrinsics do not require 64-bit registers (r64) on 32-bit platforms (isTypeSupportedForIntrinsic returns
 479     // true)
 480     bool issupported = compSupports(isa) && compSupportsHWIntrinsic(isa) && isTypeSupportedForIntrinsic(retType);
 481
 482     if (category == HW_Category_IsSupportedProperty)
 483     {
 484         return gtNewIconNode(issupported);
 485     }
 486     // - calling to unsupported intrinsics must throw PlatforNotSupportedException
 487     else if (!issupported)
 488     {
 489         return impUnsupportedHWIntrinsic(CORINFO_HELP_THROW_PLATFORM_NOT_SUPPORTED, method, sig, mustExpand);
 490     }
 491     else if (category == HW_Category_IMM)
 492     {
 493         GenTree* lastOp = impStackTop().val;
 494         if (!lastOp->IsCnsIntOrI() && !mustExpand)
 495         {
 496             // When the imm-argument is not a constant and we are not being forced to expand, we need to
 497             // return nullptr so a GT_CALL to the intrinsic method is emitted instead. The
 498             // intrinsic method is recursive and will be forced to expand, at which point
 499             // we emit some less efficient fallback code.
 500             return nullptr;
 501         }
 502     }
 503
 504     if ((flags & HW_Flag_Generic) != 0)
 505     {
 506         assert(baseType != TYP_UNKNOWN);
 507         // When the type argument is not a numeric type (and we are not being forced to expand), we need to
 508         // return nullptr so a GT_CALL to the intrinsic method is emitted that will throw NotSupportedException
 509         if (!varTypeIsArithmetic(baseType))
 510         {
 511             assert(!mustExpand);
 512             return nullptr;
 513         }
 514
 515         if ((flags & HW_Flag_TwoTypeGeneric) != 0)
 516         {
 517             // StaticCast<T, U> has two type parameters.
 518             assert(!mustExpand);
 519             assert(numArgs == 1);
 520             var_types srcType = getBaseTypeOfSIMDType(info.compCompHnd->getArgClass(sig, sig->args));
 521             assert(srcType != TYP_UNKNOWN);
 522             if (!varTypeIsArithmetic(srcType))
 523             {
 524                 return nullptr;
 525             }
 526         }
 527     }
 528
 529     // table-driven importer of simple intrinsics
 530     if (impIsTableDrivenHWIntrinsic(category, flags))
 531     {
 532         if (!varTypeIsSIMD(retType))
 533         {
 534             baseType = getBaseTypeOfSIMDType(info.compCompHnd->getArgClass(sig, sig->args));
 535             assert(baseType != TYP_UNKNOWN);
 536         }
 537
 538         unsigned                simdSize = simdSizeOfHWIntrinsic(intrinsic, sig);
 539         CORINFO_ARG_LIST_HANDLE argList  = sig->args;
 540         CORINFO_CLASS_HANDLE    argClass;
 541         var_types               argType = TYP_UNKNOWN;
 542
 543         assert(numArgs >= 0);
 544         assert(insOfHWIntrinsic(intrinsic, baseType) != INS_invalid);
 545         assert(simdSize == 32 || simdSize == 16);
 546
 547         GenTree* retNode = nullptr;
 548         GenTree* op1     = nullptr;
 549         GenTree* op2     = nullptr;
 550
 551         switch (numArgs)
 552         {
 553             case 0:
 554                 retNode = gtNewSimdHWIntrinsicNode(retType, intrinsic, baseType, simdSize);
 555                 break;
 556             case 1:
 557                 argType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, argList, &argClass)));
 558                 op1     = getArgForHWIntrinsic(argType, argClass);
 559                 retNode = gtNewSimdHWIntrinsicNode(retType, op1, intrinsic, baseType, simdSize);
 560                 break;
 561             case 2:
 562                 argType = JITtype2varType(
 563                     strip(info.compCompHnd->getArgType(sig, info.compCompHnd->getArgNext(argList), &argClass)));
 564                 op2 = getArgForHWIntrinsic(argType, argClass);
 565
 566                 argType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, argList, &argClass)));
 567                 op1     = getArgForHWIntrinsic(argType, argClass);
 568
 569                 retNode = gtNewSimdHWIntrinsicNode(retType, op1, op2, intrinsic, baseType, simdSize);
 570                 break;
 571
 572             case 3:
 573             {
 574                 CORINFO_ARG_LIST_HANDLE arg2 = info.compCompHnd->getArgNext(argList);
 575                 CORINFO_ARG_LIST_HANDLE arg3 = info.compCompHnd->getArgNext(arg2);
 576
 577                 argType      = JITtype2varType(strip(info.compCompHnd->getArgType(sig, arg3, &argClass)));
 578                 GenTree* op3 = getArgForHWIntrinsic(argType, argClass);
 579
 580                 argType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, arg2, &argClass)));
 581                 op2     = getArgForHWIntrinsic(argType, argClass);
 582
 583                 argType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, argList, &argClass)));
 584                 op1     = getArgForHWIntrinsic(argType, argClass);
 585
 586                 retNode = gtNewSimdHWIntrinsicNode(retType, op1, op2, op3, intrinsic, baseType, simdSize);
 587                 break;
 588             }
 589             default:
 590                 unreached();
 591         }
 592         return retNode;
 593     }
 594
 595     // other intrinsics need special importation
 596     switch (isa)
 597     {
 598         case InstructionSet_SSE:
 599             return impSSEIntrinsic(intrinsic, method, sig, mustExpand);
 600         case InstructionSet_SSE2:
 601             return impSSE2Intrinsic(intrinsic, method, sig, mustExpand);
 602         case InstructionSet_SSE3:
 603             return impSSE3Intrinsic(intrinsic, method, sig, mustExpand);
 604         case InstructionSet_SSSE3:
 605             return impSSSE3Intrinsic(intrinsic, method, sig, mustExpand);
 606         case InstructionSet_SSE41:
 607             return impSSE41Intrinsic(intrinsic, method, sig, mustExpand);
 608         case InstructionSet_SSE42:
 609             return impSSE42Intrinsic(intrinsic, method, sig, mustExpand);
 610         case InstructionSet_AVX:
 611             return impAVXIntrinsic(intrinsic, method, sig, mustExpand);
 612         case InstructionSet_AVX2:
 613             return impAVX2Intrinsic(intrinsic, method, sig, mustExpand);
 614
 615         case InstructionSet_AES:
 616             return impAESIntrinsic(intrinsic, method, sig, mustExpand);
 617         case InstructionSet_BMI1:
 618             return impBMI1Intrinsic(intrinsic, method, sig, mustExpand);
 619         case InstructionSet_BMI2:
 620             return impBMI2Intrinsic(intrinsic, method, sig, mustExpand);
 621         case InstructionSet_FMA:
 622             return impFMAIntrinsic(intrinsic, method, sig, mustExpand);
 623         case InstructionSet_LZCNT:
 624             return impLZCNTIntrinsic(intrinsic, method, sig, mustExpand);
 625         case InstructionSet_PCLMULQDQ:
 626             return impPCLMULQDQIntrinsic(intrinsic, method, sig, mustExpand);
 627         case InstructionSet_POPCNT:
 628             return impPOPCNTIntrinsic(intrinsic, method, sig, mustExpand);
 629         default:
 630             return nullptr;
 631     }
 632 }
 633
 634 CORINFO_CLASS_HANDLE Compiler::gtGetStructHandleForHWSIMD(var_types simdType, var_types simdBaseType)
 635 {
 636     if (simdType == TYP_SIMD16)
 637     {
 638         switch (simdBaseType)
 639         {
 640             case TYP_FLOAT:
 641                 return Vector128FloatHandle;
 642             case TYP_DOUBLE:
 643                 return Vector128DoubleHandle;
 644             case TYP_INT:
 645                 return Vector128IntHandle;
 646             case TYP_USHORT:
 647                 return Vector128UShortHandle;
 648             case TYP_UBYTE:
 649                 return Vector128UByteHandle;
 650             case TYP_SHORT:
 651                 return Vector128ShortHandle;
 652             case TYP_BYTE:
 653                 return Vector128ByteHandle;
 654             case TYP_LONG:
 655                 return Vector128LongHandle;
 656             case TYP_UINT:
 657                 return Vector128UIntHandle;
 658             case TYP_ULONG:
 659                 return Vector128ULongHandle;
 660             default:
 661                 assert(!"Didn't find a class handle for simdType");
 662         }
 663     }
 664     else if (simdType == TYP_SIMD32)
 665     {
 666         switch (simdBaseType)
 667         {
 668             case TYP_FLOAT:
 669                 return Vector256FloatHandle;
 670             case TYP_DOUBLE:
 671                 return Vector256DoubleHandle;
 672             case TYP_INT:
 673                 return Vector256IntHandle;
 674             case TYP_USHORT:
 675                 return Vector256UShortHandle;
 676             case TYP_UBYTE:
 677                 return Vector256UByteHandle;
 678             case TYP_SHORT:
 679                 return Vector256ShortHandle;
 680             case TYP_BYTE:
 681                 return Vector256ByteHandle;
 682             case TYP_LONG:
 683                 return Vector256LongHandle;
 684             case TYP_UINT:
 685                 return Vector256UIntHandle;
 686             case TYP_ULONG:
 687                 return Vector256ULongHandle;
 688             default:
 689                 assert(!"Didn't find a class handle for simdType");
 690         }
 691     }
 692
 693     return NO_CLASS_HANDLE;
 694 }
 695
 696 GenTree* Compiler::impSSEIntrinsic(NamedIntrinsic        intrinsic,
 697                                    CORINFO_METHOD_HANDLE method,
 698                                    CORINFO_SIG_INFO*     sig,
 699                                    bool                  mustExpand)
 700 {
 701     GenTree* retNode  = nullptr;
 702     GenTree* op1      = nullptr;
 703     GenTree* op2      = nullptr;
 704     GenTree* op3      = nullptr;
 705     GenTree* op4      = nullptr;
 706     int      simdSize = simdSizeOfHWIntrinsic(intrinsic, sig);
 707     assert(simdSize == 16);
 708
 709     switch (intrinsic)
 710     {
 711         case NI_SSE_SetVector128:
 712         {
 713             assert(sig->numArgs == 4);
 714             assert(getBaseTypeOfSIMDType(sig->retTypeSigClass) == TYP_FLOAT);
 715
 716             op4 = impPopStack().val;
 717             op3 = impPopStack().val;
 718             op2 = impPopStack().val;
 719             op1 = impPopStack().val;
 720
 721             GenTree* left    = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op4, op3, NI_SSE_UnpackLow, TYP_FLOAT, simdSize);
 722             GenTree* right   = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op2, op1, NI_SSE_UnpackLow, TYP_FLOAT, simdSize);
 723             GenTree* control = gtNewIconNode(68, TYP_UBYTE);
 724
 725             retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, left, right, control, NI_SSE_Shuffle, TYP_FLOAT, simdSize);
 726             break;
 727         }
 728
 729         case NI_SSE_ConvertToVector128SingleScalar:
 730         {
 731             assert(sig->numArgs == 2);
 732             assert(getBaseTypeOfSIMDType(sig->retTypeSigClass) == TYP_FLOAT);
 733
 734 #ifdef _TARGET_X86_
 735             CORINFO_CLASS_HANDLE argClass;
 736
 737             CORINFO_ARG_LIST_HANDLE argLst = info.compCompHnd->getArgNext(sig->args);
 738             CorInfoType             corType =
 739                 strip(info.compCompHnd->getArgType(sig, argLst, &argClass)); // type of the second argument
 740
 741             if (varTypeIsLong(JITtype2varType(corType)))
 742             {
 743                 return impUnsupportedHWIntrinsic(CORINFO_HELP_THROW_PLATFORM_NOT_SUPPORTED, method, sig, mustExpand);
 744             }
 745 #endif // _TARGET_X86_
 746
 747             op2     = impPopStack().val;
 748             op1     = impSIMDPopStack(TYP_SIMD16);
 749             retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, op2, intrinsic, TYP_FLOAT, simdSize);
 750             break;
 751         }
 752
 753         case NI_SSE_MoveMask:
 754             assert(sig->numArgs == 1);
 755             assert(JITtype2varType(sig->retType) == TYP_INT);
 756             assert(getBaseTypeOfSIMDType(info.compCompHnd->getArgClass(sig, sig->args)) == TYP_FLOAT);
 757             op1     = impSIMDPopStack(TYP_SIMD16);
 758             retNode = gtNewSimdHWIntrinsicNode(TYP_INT, op1, intrinsic, TYP_FLOAT, simdSize);
 759             break;
 760
 761         case NI_SSE_SetAllVector128:
 762             assert(sig->numArgs == 1);
 763             assert(getBaseTypeOfSIMDType(sig->retTypeSigClass) == TYP_FLOAT);
 764             op1     = impPopStack().val;
 765             retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, gtCloneExpr(op1), gtNewIconNode(0), NI_SSE_Shuffle,
 766                                                TYP_FLOAT, simdSize);
 767             break;
 768
 769         default:
 770             JITDUMP("Not implemented hardware intrinsic");
 771             break;
 772     }
 773     return retNode;
 774 }
 775
 776 GenTree* Compiler::impSSE2Intrinsic(NamedIntrinsic        intrinsic,
 777                                     CORINFO_METHOD_HANDLE method,
 778                                     CORINFO_SIG_INFO*     sig,
 779                                     bool                  mustExpand)
 780 {
 781     GenTree*  retNode  = nullptr;
 782     GenTree*  op1      = nullptr;
 783     GenTree*  op2      = nullptr;
 784     var_types baseType = TYP_UNKNOWN;
 785     switch (intrinsic)
 786     {
 787         default:
 788             JITDUMP("Not implemented hardware intrinsic");
 789             break;
 790     }
 791     return retNode;
 792 }
 793
 794 GenTree* Compiler::impSSE3Intrinsic(NamedIntrinsic        intrinsic,
 795                                     CORINFO_METHOD_HANDLE method,
 796                                     CORINFO_SIG_INFO*     sig,
 797                                     bool                  mustExpand)
 798 {
 799     return nullptr;
 800 }
 801
 802 GenTree* Compiler::impSSSE3Intrinsic(NamedIntrinsic        intrinsic,
 803                                      CORINFO_METHOD_HANDLE method,
 804                                      CORINFO_SIG_INFO*     sig,
 805                                      bool                  mustExpand)
 806 {
 807     return nullptr;
 808 }
 809
 810 GenTree* Compiler::impSSE41Intrinsic(NamedIntrinsic        intrinsic,
 811                                      CORINFO_METHOD_HANDLE method,
 812                                      CORINFO_SIG_INFO*     sig,
 813                                      bool                  mustExpand)
 814 {
 815     return nullptr;
 816 }
 817
 818 GenTree* Compiler::impSSE42Intrinsic(NamedIntrinsic        intrinsic,
 819                                      CORINFO_METHOD_HANDLE method,
 820                                      CORINFO_SIG_INFO*     sig,
 821                                      bool                  mustExpand)
 822 {
 823     GenTree*  retNode  = nullptr;
 824     GenTree*  op1      = nullptr;
 825     GenTree*  op2      = nullptr;
 826     var_types callType = JITtype2varType(sig->retType);
 827
 828     CORINFO_ARG_LIST_HANDLE argList = sig->args;
 829     CORINFO_CLASS_HANDLE    argClass;
 830     CorInfoType             corType;
 831     switch (intrinsic)
 832     {
 833         case NI_SSE42_Crc32:
 834             assert(sig->numArgs == 2);
 835             op2     = impPopStack().val;
 836             op1     = impPopStack().val;
 837             argList = info.compCompHnd->getArgNext(argList);                        // the second argument
 838             corType = strip(info.compCompHnd->getArgType(sig, argList, &argClass)); // type of the second argument
 839
 840             retNode = gtNewScalarHWIntrinsicNode(callType, op1, op2, NI_SSE42_Crc32);
 841
 842             // TODO - currently we use the BaseType to bring the type of the second argument
 843             // to the code generator. May encode the overload info in other way.
 844             retNode->gtHWIntrinsic.gtSIMDBaseType = JITtype2varType(corType);
 845             break;
 846
 847         default:
 848             JITDUMP("Not implemented hardware intrinsic");
 849             break;
 850     }
 851     return retNode;
 852 }
 853
 854 GenTree* Compiler::impAVXIntrinsic(NamedIntrinsic        intrinsic,
 855                                    CORINFO_METHOD_HANDLE method,
 856                                    CORINFO_SIG_INFO*     sig,
 857                                    bool                  mustExpand)
 858 {
 859     GenTree*  retNode  = nullptr;
 860     GenTree*  op1      = nullptr;
 861     GenTree*  op2      = nullptr;
 862     var_types baseType = TYP_UNKNOWN;
 863     switch (intrinsic)
 864     {
 865         default:
 866             JITDUMP("Not implemented hardware intrinsic");
 867             break;
 868     }
 869     return retNode;
 870 }
 871
 872 GenTree* Compiler::impAVX2Intrinsic(NamedIntrinsic        intrinsic,
 873                                     CORINFO_METHOD_HANDLE method,
 874                                     CORINFO_SIG_INFO*     sig,
 875                                     bool                  mustExpand)
 876 {
 877     GenTree*  retNode  = nullptr;
 878     GenTree*  op1      = nullptr;
 879     GenTree*  op2      = nullptr;
 880     var_types baseType = TYP_UNKNOWN;
 881     switch (intrinsic)
 882     {
 883         default:
 884             JITDUMP("Not implemented hardware intrinsic");
 885             break;
 886     }
 887     return retNode;
 888 }
 889
 890 GenTree* Compiler::impAESIntrinsic(NamedIntrinsic        intrinsic,
 891                                    CORINFO_METHOD_HANDLE method,
 892                                    CORINFO_SIG_INFO*     sig,
 893                                    bool                  mustExpand)
 894 {
 895     return nullptr;
 896 }
 897
 898 GenTree* Compiler::impBMI1Intrinsic(NamedIntrinsic        intrinsic,
 899                                     CORINFO_METHOD_HANDLE method,
 900                                     CORINFO_SIG_INFO*     sig,
 901                                     bool                  mustExpand)
 902 {
 903     return nullptr;
 904 }
 905
 906 GenTree* Compiler::impBMI2Intrinsic(NamedIntrinsic        intrinsic,
 907                                     CORINFO_METHOD_HANDLE method,
 908                                     CORINFO_SIG_INFO*     sig,
 909                                     bool                  mustExpand)
 910 {
 911     return nullptr;
 912 }
 913
 914 GenTree* Compiler::impFMAIntrinsic(NamedIntrinsic        intrinsic,
 915                                    CORINFO_METHOD_HANDLE method,
 916                                    CORINFO_SIG_INFO*     sig,
 917                                    bool                  mustExpand)
 918 {
 919     return nullptr;
 920 }
 921
 922 GenTree* Compiler::impLZCNTIntrinsic(NamedIntrinsic        intrinsic,
 923                                      CORINFO_METHOD_HANDLE method,
 924                                      CORINFO_SIG_INFO*     sig,
 925                                      bool                  mustExpand)
 926 {
 927     assert(sig->numArgs == 1);
 928     var_types callType = JITtype2varType(sig->retType);
 929     return gtNewScalarHWIntrinsicNode(callType, impPopStack().val, NI_LZCNT_LeadingZeroCount);
 930 }
 931
 932 GenTree* Compiler::impPCLMULQDQIntrinsic(NamedIntrinsic        intrinsic,
 933                                          CORINFO_METHOD_HANDLE method,
 934                                          CORINFO_SIG_INFO*     sig,
 935                                          bool                  mustExpand)
 936 {
 937     return nullptr;
 938 }
 939
 940 GenTree* Compiler::impPOPCNTIntrinsic(NamedIntrinsic        intrinsic,
 941                                       CORINFO_METHOD_HANDLE method,
 942                                       CORINFO_SIG_INFO*     sig,
 943                                       bool                  mustExpand)
 944 {
 945     assert(sig->numArgs == 1);
 946     var_types callType = JITtype2varType(sig->retType);
 947     return gtNewScalarHWIntrinsicNode(callType, impPopStack().val, NI_POPCNT_PopCount);
 948 }
 949
 950 #endif // FEATURE_HW_INTRINSICS