src/jit/hwintrinsicxarch.cpp

   1 // Licensed to the .NET Foundation under one or more agreements.
   2 // The .NET Foundation licenses this file to you under the MIT license.
   3 // See the LICENSE file in the project root for more information.
   4
   5 #include "jitpch.h"
   6
   7 #if FEATURE_HW_INTRINSICS
   8
   9 struct HWIntrinsicInfo
  10 {
  11     NamedIntrinsic      intrinsicID;
  12     const char*         intrinsicName;
  13     InstructionSet      isa;
  14     int                 ival;
  15     unsigned            simdSize;
  16     int                 numArgs;
  17     instruction         ins[10];
  18     HWIntrinsicCategory category;
  19     HWIntrinsicFlag     flags;
  20 };
  21
  22 static const HWIntrinsicInfo hwIntrinsicInfoArray[] = {
  23 #define HARDWARE_INTRINSIC(id, name, isa, ival, size, numarg, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, category, flag) \
  24     {NI_##id, name, InstructionSet_##isa, ival, size, numarg, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, category, flag},
  25 #include "hwintrinsiclistxarch.h"
  26 };
  27
  28 extern const char* getHWIntrinsicName(NamedIntrinsic intrinsic)
  29 {
  30     return hwIntrinsicInfoArray[intrinsic - NI_HW_INTRINSIC_START - 1].intrinsicName;
  31 }
  32
  33 //------------------------------------------------------------------------
  34 // lookupHWIntrinsicISA: map class name to InstructionSet value
  35 //
  36 // Arguments:
  37 //    className -- class name in System.Runtime.Intrinsics.X86
  38 //
  39 // Return Value:
  40 //    Id for the ISA class.
  41 //
  42 InstructionSet Compiler::lookupHWIntrinsicISA(const char* className)
  43 {
  44     if (className != nullptr)
  45     {
  46         if (className[0] == 'A')
  47         {
  48             if (strcmp(className, "Aes") == 0)
  49             {
  50                 return InstructionSet_AES;
  51             }
  52             else if (strcmp(className, "Avx") == 0)
  53             {
  54                 return InstructionSet_AVX;
  55             }
  56             else if (strcmp(className, "Avx2") == 0)
  57             {
  58                 return InstructionSet_AVX2;
  59             }
  60         }
  61         if (className[0] == 'S')
  62         {
  63             if (strcmp(className, "Sse") == 0)
  64             {
  65                 return InstructionSet_SSE;
  66             }
  67             else if (strcmp(className, "Sse2") == 0)
  68             {
  69                 return InstructionSet_SSE2;
  70             }
  71             else if (strcmp(className, "Sse3") == 0)
  72             {
  73                 return InstructionSet_SSE3;
  74             }
  75             else if (strcmp(className, "Ssse3") == 0)
  76             {
  77                 return InstructionSet_SSSE3;
  78             }
  79             else if (strcmp(className, "Sse41") == 0)
  80             {
  81                 return InstructionSet_SSE41;
  82             }
  83             else if (strcmp(className, "Sse42") == 0)
  84             {
  85                 return InstructionSet_SSE42;
  86             }
  87         }
  88
  89         if (strcmp(className, "Bmi1") == 0)
  90         {
  91             return InstructionSet_BMI1;
  92         }
  93         else if (strcmp(className, "Bmi2") == 0)
  94         {
  95             return InstructionSet_BMI2;
  96         }
  97         else if (strcmp(className, "Fma") == 0)
  98         {
  99             return InstructionSet_FMA;
 100         }
 101         else if (strcmp(className, "Lzcnt") == 0)
 102         {
 103             return InstructionSet_LZCNT;
 104         }
 105         else if (strcmp(className, "Pclmulqdq") == 0)
 106         {
 107             return InstructionSet_PCLMULQDQ;
 108         }
 109         else if (strcmp(className, "Popcnt") == 0)
 110         {
 111             return InstructionSet_POPCNT;
 112         }
 113     }
 114
 115     JITDUMP("Unsupported ISA.\n");
 116     return InstructionSet_ILLEGAL;
 117 }
 118
 119 //------------------------------------------------------------------------
 120 // lookupHWIntrinsic: map intrinsic name to named intrinsic value
 121 //
 122 // Arguments:
 123 //    methodName -- name of the intrinsic function.
 124 //    isa        -- instruction set of the intrinsic.
 125 //
 126 // Return Value:
 127 //    Id for the hardware intrinsic.
 128 //
 129 // TODO-Throughput: replace sequential search by binary search
 130 NamedIntrinsic Compiler::lookupHWIntrinsic(const char* methodName, InstructionSet isa)
 131 {
 132     NamedIntrinsic result = NI_Illegal;
 133     if (isa != InstructionSet_ILLEGAL)
 134     {
 135         for (int i = 0; i < NI_HW_INTRINSIC_END - NI_HW_INTRINSIC_START; i++)
 136         {
 137             if (isa == hwIntrinsicInfoArray[i].isa && strcmp(methodName, hwIntrinsicInfoArray[i].intrinsicName) == 0)
 138             {
 139                 result = hwIntrinsicInfoArray[i].intrinsicID;
 140             }
 141         }
 142     }
 143     return result;
 144 }
 145
 146 //------------------------------------------------------------------------
 147 // isaOfHWIntrinsic: map named intrinsic value to its instruction set
 148 //
 149 // Arguments:
 150 //    intrinsic -- id of the intrinsic function.
 151 //
 152 // Return Value:
 153 //    instruction set of the intrinsic.
 154 //
 155 InstructionSet Compiler::isaOfHWIntrinsic(NamedIntrinsic intrinsic)
 156 {
 157     assert(intrinsic != NI_Illegal);
 158     assert(intrinsic > NI_HW_INTRINSIC_START && intrinsic < NI_HW_INTRINSIC_END);
 159     return hwIntrinsicInfoArray[intrinsic - NI_HW_INTRINSIC_START - 1].isa;
 160 }
 161
 162 //------------------------------------------------------------------------
 163 // ivalOfHWIntrinsic: get the imm8 value of this intrinsic from the hwIntrinsicInfoArray table
 164 //
 165 // Arguments:
 166 //    intrinsic -- id of the intrinsic function.
 167 //
 168 // Return Value:
 169 //     The imm8 value that is implicit for this intrinsic, or -1 for intrinsics that do not take an immediate, or for
 170 //     which the immediate is an explicit argument.
 171 //
 172 int Compiler::ivalOfHWIntrinsic(NamedIntrinsic intrinsic)
 173 {
 174     assert(intrinsic != NI_Illegal);
 175     assert(intrinsic > NI_HW_INTRINSIC_START && intrinsic < NI_HW_INTRINSIC_END);
 176     return hwIntrinsicInfoArray[intrinsic - NI_HW_INTRINSIC_START - 1].ival;
 177 }
 178
 179 //------------------------------------------------------------------------
 180 // simdSizeOfHWIntrinsic: get the SIMD size of this intrinsic
 181 //
 182 // Arguments:
 183 //    intrinsic -- id of the intrinsic function.
 184 //
 185 // Return Value:
 186 //     the SIMD size of this intrinsic
 187 //         - from the hwIntrinsicInfoArray table if intrinsic has NO HW_Flag_UnfixedSIMDSize
 188 //         - TODO-XArch-NYI - from the signature if intrinsic has HW_Flag_UnfixedSIMDSize
 189 //
 190 // Note - this function is only used by the importer
 191 //        after importation (i.e., codegen), we can get the SIMD size from GenTreeHWIntrinsic IR
 192 static unsigned simdSizeOfHWIntrinsic(NamedIntrinsic intrinsic, CORINFO_SIG_INFO* sig)
 193 {
 194     assert(intrinsic > NI_HW_INTRINSIC_START && intrinsic < NI_HW_INTRINSIC_END);
 195     assert((hwIntrinsicInfoArray[intrinsic - NI_HW_INTRINSIC_START - 1].flags & HW_Flag_UnfixedSIMDSize) == 0);
 196     return hwIntrinsicInfoArray[intrinsic - NI_HW_INTRINSIC_START - 1].simdSize;
 197 }
 198
 199 //------------------------------------------------------------------------
 200 // numArgsOfHWIntrinsic: get the number of arguments
 201 //
 202 // Arguments:
 203 //    intrinsic -- id of the intrinsic function.
 204 //
 205 // Return Value:
 206 //     number of arguments
 207 //
 208 int Compiler::numArgsOfHWIntrinsic(NamedIntrinsic intrinsic)
 209 {
 210     assert(intrinsic != NI_Illegal);
 211     assert(intrinsic > NI_HW_INTRINSIC_START && intrinsic < NI_HW_INTRINSIC_END);
 212     return hwIntrinsicInfoArray[intrinsic - NI_HW_INTRINSIC_START - 1].numArgs;
 213 }
 214
 215 //------------------------------------------------------------------------
 216 // insOfHWIntrinsic: get the instruction of the given intrinsic
 217 //
 218 // Arguments:
 219 //    intrinsic -- id of the intrinsic function.
 220 //    type      -- vector base type of this intrinsic
 221 //
 222 // Return Value:
 223 //     the instruction of the given intrinsic on the base type
 224 //     return INS_invalid for unsupported base types
 225 //
 226 instruction Compiler::insOfHWIntrinsic(NamedIntrinsic intrinsic, var_types type)
 227 {
 228     assert(intrinsic != NI_Illegal);
 229     assert(intrinsic > NI_HW_INTRINSIC_START && intrinsic < NI_HW_INTRINSIC_END);
 230     assert(type >= TYP_BYTE && type <= TYP_DOUBLE);
 231     return hwIntrinsicInfoArray[intrinsic - NI_HW_INTRINSIC_START - 1].ins[type - TYP_BYTE];
 232 }
 233
 234 //------------------------------------------------------------------------
 235 // categoryOfHWIntrinsic: get the category of the given intrinsic
 236 //
 237 // Arguments:
 238 //    intrinsic -- id of the intrinsic function.
 239 //
 240 // Return Value:
 241 //     the category of the given intrinsic
 242 //
 243 HWIntrinsicCategory Compiler::categoryOfHWIntrinsic(NamedIntrinsic intrinsic)
 244 {
 245     assert(intrinsic != NI_Illegal);
 246     assert(intrinsic > NI_HW_INTRINSIC_START && intrinsic < NI_HW_INTRINSIC_END);
 247     return hwIntrinsicInfoArray[intrinsic - NI_HW_INTRINSIC_START - 1].category;
 248 }
 249
 250 //------------------------------------------------------------------------
 251 // HWIntrinsicFlag: get the flags of the given intrinsic
 252 //
 253 // Arguments:
 254 //    intrinsic -- id of the intrinsic function.
 255 //
 256 // Return Value:
 257 //     the flags of the given intrinsic
 258 //
 259 HWIntrinsicFlag Compiler::flagsOfHWIntrinsic(NamedIntrinsic intrinsic)
 260 {
 261     assert(intrinsic != NI_Illegal);
 262     assert(intrinsic > NI_HW_INTRINSIC_START && intrinsic < NI_HW_INTRINSIC_END);
 263     return hwIntrinsicInfoArray[intrinsic - NI_HW_INTRINSIC_START - 1].flags;
 264 }
 265
 266 //------------------------------------------------------------------------
 267 // getArgForHWIntrinsic: get the argument from the stack and match  the signature
 268 //
 269 // Arguments:
 270 //    argType   -- the required type of argument
 271 //    argClass  -- the class handle of argType
 272 //
 273 // Return Value:
 274 //     get the argument at the given index from the stack and match  the signature
 275 //
 276 GenTree* Compiler::getArgForHWIntrinsic(var_types argType, CORINFO_CLASS_HANDLE argClass)
 277 {
 278     GenTree* arg = nullptr;
 279     if (argType == TYP_STRUCT)
 280     {
 281         unsigned int argSizeBytes;
 282         var_types    base = getBaseTypeAndSizeOfSIMDType(argClass, &argSizeBytes);
 283         argType           = getSIMDTypeForSize(argSizeBytes);
 284         assert(argType == TYP_SIMD32 || argType == TYP_SIMD16);
 285         arg = impSIMDPopStack(argType);
 286         assert(arg->TypeGet() == TYP_SIMD16 || arg->TypeGet() == TYP_SIMD32);
 287     }
 288     else
 289     {
 290         assert(varTypeIsArithmetic(argType));
 291         arg = impPopStack().val;
 292         assert(varTypeIsArithmetic(arg->TypeGet()));
 293         assert(genActualType(arg->gtType) == genActualType(argType));
 294     }
 295     return arg;
 296 }
 297
 298 //------------------------------------------------------------------------
 299 // isFullyImplmentedISAClass: return true if all the hardware intrinsics
 300 //    of this ISA are implemented in RyuJIT.
 301 //
 302 // Arguments:
 303 //    isa - Instruction set
 304 // Return Value:
 305 //    true - all the hardware intrinsics of "isa" are implemented in RyuJIT.
 306 //
 307 bool Compiler::isFullyImplmentedISAClass(InstructionSet isa)
 308 {
 309     switch (isa)
 310     {
 311         case InstructionSet_SSE2:
 312         case InstructionSet_SSE3:
 313         case InstructionSet_SSSE3:
 314         case InstructionSet_SSE41:
 315         case InstructionSet_SSE42:
 316         case InstructionSet_AVX:
 317         case InstructionSet_AVX2:
 318         case InstructionSet_AES:
 319         case InstructionSet_BMI1:
 320         case InstructionSet_BMI2:
 321         case InstructionSet_FMA:
 322         case InstructionSet_PCLMULQDQ:
 323             return false;
 324
 325         case InstructionSet_SSE:
 326         case InstructionSet_LZCNT:
 327         case InstructionSet_POPCNT:
 328             return true;
 329
 330         default:
 331             unreached();
 332     }
 333 }
 334
 335 //------------------------------------------------------------------------
 336 // isScalarISA:
 337 //
 338 // Arguments:
 339 //    isa - Instruction set
 340 // Return Value:
 341 //    true - if "isa" only contains scalar instructions
 342 //
 343 bool Compiler::isScalarISA(InstructionSet isa)
 344 {
 345     switch (isa)
 346     {
 347         case InstructionSet_BMI1:
 348         case InstructionSet_BMI2:
 349         case InstructionSet_LZCNT:
 350         case InstructionSet_POPCNT:
 351             return true;
 352
 353         default:
 354             return false;
 355     }
 356 }
 357
 358 //------------------------------------------------------------------------
 359 // compSupportsHWIntrinsic: compiler support of hardware intrinsics
 360 //
 361 // Arguments:
 362 //    isa - Instruction set
 363 // Return Value:
 364 //    true if
 365 //    - isa is a scalar ISA
 366 //    - isa is a SIMD ISA and featureSIMD=true
 367 //    - isa is fully implemented or EnableIncompleteISAClass=true
 368 bool Compiler::compSupportsHWIntrinsic(InstructionSet isa)
 369 {
 370     return (featureSIMD || isScalarISA(isa)) && (
 371 #ifdef DEBUG
 372                                                     JitConfig.EnableIncompleteISAClass() ||
 373 #endif
 374                                                     isFullyImplmentedISAClass(isa));
 375 }
 376
 377 static bool isTypeSupportedForIntrinsic(var_types type)
 378 {
 379 #ifdef _TARGET_X86_
 380     return !varTypeIsLong(type);
 381 #else
 382     return true;
 383 #endif
 384 }
 385
 386 //------------------------------------------------------------------------
 387 // impUnsupportedHWIntrinsic: returns a node for an unsupported HWIntrinsic
 388 //
 389 // Arguments:
 390 //    helper     - JIT helper ID for the exception to be thrown
 391 //    method     - method handle of the intrinsic function.
 392 //    sig        - signature of the intrinsic call
 393 //    mustExpand - true if the intrinsic must return a GenTree*; otherwise, false
 394 //
 395 // Return Value:
 396 //    a gtNewMustThrowException if mustExpand is true; otherwise, nullptr
 397 //
 398 GenTree* Compiler::impUnsupportedHWIntrinsic(unsigned              helper,
 399                                              CORINFO_METHOD_HANDLE method,
 400                                              CORINFO_SIG_INFO*     sig,
 401                                              bool                  mustExpand)
 402 {
 403     // We've hit some error case and may need to return a node for the given error.
 404     //
 405     // When `mustExpand=false`, we are attempting to inline the intrinsic directly into another method. In this
 406     // scenario, we need to return `nullptr` so that a GT_CALL to the intrinsic is emitted instead. This is to
 407     // ensure that everything continues to behave correctly when optimizations are enabled (e.g. things like the
 408     // inliner may expect the node we return to have a certain signature, and the `MustThrowException` node won't
 409     // match that).
 410     //
 411     // When `mustExpand=true`, we are in a GT_CALL to the intrinsic and are attempting to JIT it. This will generally
 412     // be in response to an indirect call (e.g. done via reflection) or in response to an earlier attempt returning
 413     // `nullptr` (under `mustExpand=false`). In that scenario, we are safe to return the `MustThrowException` node.
 414
 415     if (mustExpand)
 416     {
 417         for (unsigned i = 0; i < sig->numArgs; i++)
 418         {
 419             impPopStack();
 420         }
 421
 422         return gtNewMustThrowException(helper, JITtype2varType(sig->retType), sig->retTypeClass);
 423     }
 424     else
 425     {
 426         return nullptr;
 427     }
 428 }
 429
 430 //------------------------------------------------------------------------
 431 // impIsTableDrivenHWIntrinsic:
 432 //
 433 // Arguments:
 434 //    category - category of a HW intrinsic
 435 //
 436 // Return Value:
 437 //    returns true if this category can be table-driven in the importer
 438 //
 439 static bool impIsTableDrivenHWIntrinsic(HWIntrinsicCategory category, HWIntrinsicFlag flags)
 440 {
 441     // HW_Flag_NoCodeGen implies this intrinsic should be manually morphed in the importer.
 442     return category != HW_Category_Special && category != HW_Category_Scalar && (flags & HW_Flag_NoCodeGen) == 0;
 443 }
 444
 445 //------------------------------------------------------------------------
 446 // impX86HWIntrinsic: dispatch hardware intrinsics to their own implementation
 447 //
 448 // Arguments:
 449 //    intrinsic -- id of the intrinsic function.
 450 //    method    -- method handle of the intrinsic function.
 451 //    sig       -- signature of the intrinsic call
 452 //
 453 // Return Value:
 454 //    the expanded intrinsic.
 455 //
 456 GenTree* Compiler::impX86HWIntrinsic(NamedIntrinsic        intrinsic,
 457                                      CORINFO_METHOD_HANDLE method,
 458                                      CORINFO_SIG_INFO*     sig,
 459                                      bool                  mustExpand)
 460 {
 461     InstructionSet      isa      = isaOfHWIntrinsic(intrinsic);
 462     HWIntrinsicCategory category = categoryOfHWIntrinsic(intrinsic);
 463     HWIntrinsicFlag     flags    = flagsOfHWIntrinsic(intrinsic);
 464     int                 numArgs  = sig->numArgs;
 465     var_types           retType  = JITtype2varType(sig->retType);
 466     var_types           baseType = TYP_UNKNOWN;
 467     if (retType == TYP_STRUCT && featureSIMD)
 468     {
 469         unsigned int sizeBytes;
 470         baseType = getBaseTypeAndSizeOfSIMDType(sig->retTypeSigClass, &sizeBytes);
 471         retType  = getSIMDTypeForSize(sizeBytes);
 472         assert(sizeBytes != 0 && baseType != TYP_UNKNOWN);
 473     }
 474
 475     // This intrinsic is supported if
 476     // - the ISA is available on the underlying hardware (compSupports returns true)
 477     // - the compiler supports this hardware intrinsics (compSupportsHWIntrinsic returns true)
 478     // - intrinsics do not require 64-bit registers (r64) on 32-bit platforms (isTypeSupportedForIntrinsic returns
 479     // true)
 480     bool issupported = compSupports(isa) && compSupportsHWIntrinsic(isa) && isTypeSupportedForIntrinsic(retType);
 481
 482     if (category == HW_Category_IsSupportedProperty)
 483     {
 484         return gtNewIconNode(issupported);
 485     }
 486     // - calling to unsupported intrinsics must throw PlatforNotSupportedException
 487     else if (!issupported)
 488     {
 489         return impUnsupportedHWIntrinsic(CORINFO_HELP_THROW_PLATFORM_NOT_SUPPORTED, method, sig, mustExpand);
 490     }
 491     else if (category == HW_Category_IMM)
 492     {
 493         GenTree* lastOp = impStackTop().val;
 494         if (!lastOp->IsCnsIntOrI() && !mustExpand)
 495         {
 496             // When the imm-argument is not a constant and we are not being forced to expand, we need to
 497             // return nullptr so a GT_CALL to the intrinsic method is emitted instead. The
 498             // intrinsic method is recursive and will be forced to expand, at which point
 499             // we emit some less efficient fallback code.
 500             return nullptr;
 501         }
 502     }
 503
 504     if ((flags & HW_Flag_Generic) != 0)
 505     {
 506         assert(baseType != TYP_UNKNOWN);
 507         // When the type argument is not a numeric type (and we are not being forced to expand), we need to
 508         // return nullptr so a GT_CALL to the intrinsic method is emitted that will throw NotSupportedException
 509         if (!varTypeIsArithmetic(baseType))
 510         {
 511             assert(!mustExpand);
 512             return nullptr;
 513         }
 514
 515         if ((flags & HW_Flag_TwoTypeGeneric) != 0)
 516         {
 517             // StaticCast<T, U> has two type parameters.
 518             assert(!mustExpand);
 519             assert(numArgs == 1);
 520             var_types srcType = getBaseTypeOfSIMDType(info.compCompHnd->getArgClass(sig, sig->args));
 521             assert(srcType != TYP_UNKNOWN);
 522             if (!varTypeIsArithmetic(srcType))
 523             {
 524                 return nullptr;
 525             }
 526         }
 527     }
 528
 529     // table-driven importer of simple intrinsics
 530     if (impIsTableDrivenHWIntrinsic(category, flags))
 531     {
 532         if (!varTypeIsSIMD(retType))
 533         {
 534             if (retType != TYP_VOID)
 535             {
 536                 baseType = getBaseTypeOfSIMDType(info.compCompHnd->getArgClass(sig, sig->args));
 537             }
 538             else
 539             {
 540                 assert(category == HW_Category_MemoryStore);
 541                 baseType =
 542                     getBaseTypeOfSIMDType(info.compCompHnd->getArgClass(sig, info.compCompHnd->getArgNext(sig->args)));
 543             }
 544
 545             assert(baseType != TYP_UNKNOWN);
 546         }
 547
 548         unsigned                simdSize = simdSizeOfHWIntrinsic(intrinsic, sig);
 549         CORINFO_ARG_LIST_HANDLE argList  = sig->args;
 550         CORINFO_CLASS_HANDLE    argClass;
 551         var_types               argType = TYP_UNKNOWN;
 552
 553         assert(numArgs >= 0);
 554         assert(insOfHWIntrinsic(intrinsic, baseType) != INS_invalid);
 555         assert(simdSize == 32 || simdSize == 16);
 556
 557         GenTree* retNode = nullptr;
 558         GenTree* op1     = nullptr;
 559         GenTree* op2     = nullptr;
 560
 561         switch (numArgs)
 562         {
 563             case 0:
 564                 retNode = gtNewSimdHWIntrinsicNode(retType, intrinsic, baseType, simdSize);
 565                 break;
 566             case 1:
 567                 argType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, argList, &argClass)));
 568                 op1     = getArgForHWIntrinsic(argType, argClass);
 569                 retNode = gtNewSimdHWIntrinsicNode(retType, op1, intrinsic, baseType, simdSize);
 570                 break;
 571             case 2:
 572                 argType = JITtype2varType(
 573                     strip(info.compCompHnd->getArgType(sig, info.compCompHnd->getArgNext(argList), &argClass)));
 574                 op2 = getArgForHWIntrinsic(argType, argClass);
 575
 576                 argType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, argList, &argClass)));
 577                 op1     = getArgForHWIntrinsic(argType, argClass);
 578
 579                 retNode = gtNewSimdHWIntrinsicNode(retType, op1, op2, intrinsic, baseType, simdSize);
 580                 break;
 581
 582             case 3:
 583             {
 584                 CORINFO_ARG_LIST_HANDLE arg2 = info.compCompHnd->getArgNext(argList);
 585                 CORINFO_ARG_LIST_HANDLE arg3 = info.compCompHnd->getArgNext(arg2);
 586
 587                 argType      = JITtype2varType(strip(info.compCompHnd->getArgType(sig, arg3, &argClass)));
 588                 GenTree* op3 = getArgForHWIntrinsic(argType, argClass);
 589
 590                 argType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, arg2, &argClass)));
 591                 op2     = getArgForHWIntrinsic(argType, argClass);
 592
 593                 argType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, argList, &argClass)));
 594                 op1     = getArgForHWIntrinsic(argType, argClass);
 595
 596                 retNode = gtNewSimdHWIntrinsicNode(retType, op1, op2, op3, intrinsic, baseType, simdSize);
 597                 break;
 598             }
 599             default:
 600                 unreached();
 601         }
 602         return retNode;
 603     }
 604
 605     // other intrinsics need special importation
 606     switch (isa)
 607     {
 608         case InstructionSet_SSE:
 609             return impSSEIntrinsic(intrinsic, method, sig, mustExpand);
 610         case InstructionSet_SSE2:
 611             return impSSE2Intrinsic(intrinsic, method, sig, mustExpand);
 612         case InstructionSet_SSE3:
 613             return impSSE3Intrinsic(intrinsic, method, sig, mustExpand);
 614         case InstructionSet_SSSE3:
 615             return impSSSE3Intrinsic(intrinsic, method, sig, mustExpand);
 616         case InstructionSet_SSE41:
 617             return impSSE41Intrinsic(intrinsic, method, sig, mustExpand);
 618         case InstructionSet_SSE42:
 619             return impSSE42Intrinsic(intrinsic, method, sig, mustExpand);
 620         case InstructionSet_AVX:
 621             return impAVXIntrinsic(intrinsic, method, sig, mustExpand);
 622         case InstructionSet_AVX2:
 623             return impAVX2Intrinsic(intrinsic, method, sig, mustExpand);
 624
 625         case InstructionSet_AES:
 626             return impAESIntrinsic(intrinsic, method, sig, mustExpand);
 627         case InstructionSet_BMI1:
 628             return impBMI1Intrinsic(intrinsic, method, sig, mustExpand);
 629         case InstructionSet_BMI2:
 630             return impBMI2Intrinsic(intrinsic, method, sig, mustExpand);
 631         case InstructionSet_FMA:
 632             return impFMAIntrinsic(intrinsic, method, sig, mustExpand);
 633         case InstructionSet_LZCNT:
 634             return impLZCNTIntrinsic(intrinsic, method, sig, mustExpand);
 635         case InstructionSet_PCLMULQDQ:
 636             return impPCLMULQDQIntrinsic(intrinsic, method, sig, mustExpand);
 637         case InstructionSet_POPCNT:
 638             return impPOPCNTIntrinsic(intrinsic, method, sig, mustExpand);
 639         default:
 640             return nullptr;
 641     }
 642 }
 643
 644 CORINFO_CLASS_HANDLE Compiler::gtGetStructHandleForHWSIMD(var_types simdType, var_types simdBaseType)
 645 {
 646     if (simdType == TYP_SIMD16)
 647     {
 648         switch (simdBaseType)
 649         {
 650             case TYP_FLOAT:
 651                 return Vector128FloatHandle;
 652             case TYP_DOUBLE:
 653                 return Vector128DoubleHandle;
 654             case TYP_INT:
 655                 return Vector128IntHandle;
 656             case TYP_USHORT:
 657                 return Vector128UShortHandle;
 658             case TYP_UBYTE:
 659                 return Vector128UByteHandle;
 660             case TYP_SHORT:
 661                 return Vector128ShortHandle;
 662             case TYP_BYTE:
 663                 return Vector128ByteHandle;
 664             case TYP_LONG:
 665                 return Vector128LongHandle;
 666             case TYP_UINT:
 667                 return Vector128UIntHandle;
 668             case TYP_ULONG:
 669                 return Vector128ULongHandle;
 670             default:
 671                 assert(!"Didn't find a class handle for simdType");
 672         }
 673     }
 674     else if (simdType == TYP_SIMD32)
 675     {
 676         switch (simdBaseType)
 677         {
 678             case TYP_FLOAT:
 679                 return Vector256FloatHandle;
 680             case TYP_DOUBLE:
 681                 return Vector256DoubleHandle;
 682             case TYP_INT:
 683                 return Vector256IntHandle;
 684             case TYP_USHORT:
 685                 return Vector256UShortHandle;
 686             case TYP_UBYTE:
 687                 return Vector256UByteHandle;
 688             case TYP_SHORT:
 689                 return Vector256ShortHandle;
 690             case TYP_BYTE:
 691                 return Vector256ByteHandle;
 692             case TYP_LONG:
 693                 return Vector256LongHandle;
 694             case TYP_UINT:
 695                 return Vector256UIntHandle;
 696             case TYP_ULONG:
 697                 return Vector256ULongHandle;
 698             default:
 699                 assert(!"Didn't find a class handle for simdType");
 700         }
 701     }
 702
 703     return NO_CLASS_HANDLE;
 704 }
 705
 706 GenTree* Compiler::impSSEIntrinsic(NamedIntrinsic        intrinsic,
 707                                    CORINFO_METHOD_HANDLE method,
 708                                    CORINFO_SIG_INFO*     sig,
 709                                    bool                  mustExpand)
 710 {
 711     GenTree* retNode  = nullptr;
 712     GenTree* op1      = nullptr;
 713     GenTree* op2      = nullptr;
 714     GenTree* op3      = nullptr;
 715     GenTree* op4      = nullptr;
 716     int      simdSize = simdSizeOfHWIntrinsic(intrinsic, sig);
 717     assert(simdSize == 16);
 718
 719     switch (intrinsic)
 720     {
 721         case NI_SSE_SetVector128:
 722         {
 723             assert(sig->numArgs == 4);
 724             assert(getBaseTypeOfSIMDType(sig->retTypeSigClass) == TYP_FLOAT);
 725
 726             op4 = impPopStack().val;
 727             op3 = impPopStack().val;
 728             op2 = impPopStack().val;
 729             op1 = impPopStack().val;
 730
 731             GenTree* left    = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op4, op3, NI_SSE_UnpackLow, TYP_FLOAT, simdSize);
 732             GenTree* right   = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op2, op1, NI_SSE_UnpackLow, TYP_FLOAT, simdSize);
 733             GenTree* control = gtNewIconNode(68, TYP_UBYTE);
 734
 735             retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, left, right, control, NI_SSE_Shuffle, TYP_FLOAT, simdSize);
 736             break;
 737         }
 738
 739         case NI_SSE_ConvertToVector128SingleScalar:
 740         {
 741             assert(sig->numArgs == 2);
 742             assert(getBaseTypeOfSIMDType(sig->retTypeSigClass) == TYP_FLOAT);
 743
 744 #ifdef _TARGET_X86_
 745             CORINFO_CLASS_HANDLE argClass;
 746
 747             CORINFO_ARG_LIST_HANDLE argLst = info.compCompHnd->getArgNext(sig->args);
 748             CorInfoType             corType =
 749                 strip(info.compCompHnd->getArgType(sig, argLst, &argClass)); // type of the second argument
 750
 751             if (varTypeIsLong(JITtype2varType(corType)))
 752             {
 753                 return impUnsupportedHWIntrinsic(CORINFO_HELP_THROW_PLATFORM_NOT_SUPPORTED, method, sig, mustExpand);
 754             }
 755 #endif // _TARGET_X86_
 756
 757             op2     = impPopStack().val;
 758             op1     = impSIMDPopStack(TYP_SIMD16);
 759             retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, op2, intrinsic, TYP_FLOAT, simdSize);
 760             break;
 761         }
 762
 763         case NI_SSE_MoveMask:
 764             assert(sig->numArgs == 1);
 765             assert(JITtype2varType(sig->retType) == TYP_INT);
 766             assert(getBaseTypeOfSIMDType(info.compCompHnd->getArgClass(sig, sig->args)) == TYP_FLOAT);
 767             op1     = impSIMDPopStack(TYP_SIMD16);
 768             retNode = gtNewSimdHWIntrinsicNode(TYP_INT, op1, intrinsic, TYP_FLOAT, simdSize);
 769             break;
 770
 771         case NI_SSE_SetAllVector128:
 772             assert(sig->numArgs == 1);
 773             assert(getBaseTypeOfSIMDType(sig->retTypeSigClass) == TYP_FLOAT);
 774             op1     = impPopStack().val;
 775             retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, gtCloneExpr(op1), gtNewIconNode(0), NI_SSE_Shuffle,
 776                                                TYP_FLOAT, simdSize);
 777             break;
 778
 779         default:
 780             JITDUMP("Not implemented hardware intrinsic");
 781             break;
 782     }
 783     return retNode;
 784 }
 785
 786 GenTree* Compiler::impSSE2Intrinsic(NamedIntrinsic        intrinsic,
 787                                     CORINFO_METHOD_HANDLE method,
 788                                     CORINFO_SIG_INFO*     sig,
 789                                     bool                  mustExpand)
 790 {
 791     GenTree*  retNode  = nullptr;
 792     GenTree*  op1      = nullptr;
 793     GenTree*  op2      = nullptr;
 794     var_types baseType = TYP_UNKNOWN;
 795     switch (intrinsic)
 796     {
 797         default:
 798             JITDUMP("Not implemented hardware intrinsic");
 799             break;
 800     }
 801     return retNode;
 802 }
 803
 804 GenTree* Compiler::impSSE3Intrinsic(NamedIntrinsic        intrinsic,
 805                                     CORINFO_METHOD_HANDLE method,
 806                                     CORINFO_SIG_INFO*     sig,
 807                                     bool                  mustExpand)
 808 {
 809     return nullptr;
 810 }
 811
 812 GenTree* Compiler::impSSSE3Intrinsic(NamedIntrinsic        intrinsic,
 813                                      CORINFO_METHOD_HANDLE method,
 814                                      CORINFO_SIG_INFO*     sig,
 815                                      bool                  mustExpand)
 816 {
 817     return nullptr;
 818 }
 819
 820 GenTree* Compiler::impSSE41Intrinsic(NamedIntrinsic        intrinsic,
 821                                      CORINFO_METHOD_HANDLE method,
 822                                      CORINFO_SIG_INFO*     sig,
 823                                      bool                  mustExpand)
 824 {
 825     return nullptr;
 826 }
 827
 828 GenTree* Compiler::impSSE42Intrinsic(NamedIntrinsic        intrinsic,
 829                                      CORINFO_METHOD_HANDLE method,
 830                                      CORINFO_SIG_INFO*     sig,
 831                                      bool                  mustExpand)
 832 {
 833     GenTree*  retNode  = nullptr;
 834     GenTree*  op1      = nullptr;
 835     GenTree*  op2      = nullptr;
 836     var_types callType = JITtype2varType(sig->retType);
 837
 838     CORINFO_ARG_LIST_HANDLE argList = sig->args;
 839     CORINFO_CLASS_HANDLE    argClass;
 840     CorInfoType             corType;
 841     switch (intrinsic)
 842     {
 843         case NI_SSE42_Crc32:
 844             assert(sig->numArgs == 2);
 845             op2     = impPopStack().val;
 846             op1     = impPopStack().val;
 847             argList = info.compCompHnd->getArgNext(argList);                        // the second argument
 848             corType = strip(info.compCompHnd->getArgType(sig, argList, &argClass)); // type of the second argument
 849
 850             retNode = gtNewScalarHWIntrinsicNode(callType, op1, op2, NI_SSE42_Crc32);
 851
 852             // TODO - currently we use the BaseType to bring the type of the second argument
 853             // to the code generator. May encode the overload info in other way.
 854             retNode->gtHWIntrinsic.gtSIMDBaseType = JITtype2varType(corType);
 855             break;
 856
 857         default:
 858             JITDUMP("Not implemented hardware intrinsic");
 859             break;
 860     }
 861     return retNode;
 862 }
 863
 864 GenTree* Compiler::impAVXIntrinsic(NamedIntrinsic        intrinsic,
 865                                    CORINFO_METHOD_HANDLE method,
 866                                    CORINFO_SIG_INFO*     sig,
 867                                    bool                  mustExpand)
 868 {
 869     GenTree*  retNode  = nullptr;
 870     GenTree*  op1      = nullptr;
 871     GenTree*  op2      = nullptr;
 872     var_types baseType = TYP_UNKNOWN;
 873     switch (intrinsic)
 874     {
 875         default:
 876             JITDUMP("Not implemented hardware intrinsic");
 877             break;
 878     }
 879     return retNode;
 880 }
 881
 882 GenTree* Compiler::impAVX2Intrinsic(NamedIntrinsic        intrinsic,
 883                                     CORINFO_METHOD_HANDLE method,
 884                                     CORINFO_SIG_INFO*     sig,
 885                                     bool                  mustExpand)
 886 {
 887     GenTree*  retNode  = nullptr;
 888     GenTree*  op1      = nullptr;
 889     GenTree*  op2      = nullptr;
 890     var_types baseType = TYP_UNKNOWN;
 891     switch (intrinsic)
 892     {
 893         default:
 894             JITDUMP("Not implemented hardware intrinsic");
 895             break;
 896     }
 897     return retNode;
 898 }
 899
 900 GenTree* Compiler::impAESIntrinsic(NamedIntrinsic        intrinsic,
 901                                    CORINFO_METHOD_HANDLE method,
 902                                    CORINFO_SIG_INFO*     sig,
 903                                    bool                  mustExpand)
 904 {
 905     return nullptr;
 906 }
 907
 908 GenTree* Compiler::impBMI1Intrinsic(NamedIntrinsic        intrinsic,
 909                                     CORINFO_METHOD_HANDLE method,
 910                                     CORINFO_SIG_INFO*     sig,
 911                                     bool                  mustExpand)
 912 {
 913     return nullptr;
 914 }
 915
 916 GenTree* Compiler::impBMI2Intrinsic(NamedIntrinsic        intrinsic,
 917                                     CORINFO_METHOD_HANDLE method,
 918                                     CORINFO_SIG_INFO*     sig,
 919                                     bool                  mustExpand)
 920 {
 921     return nullptr;
 922 }
 923
 924 GenTree* Compiler::impFMAIntrinsic(NamedIntrinsic        intrinsic,
 925                                    CORINFO_METHOD_HANDLE method,
 926                                    CORINFO_SIG_INFO*     sig,
 927                                    bool                  mustExpand)
 928 {
 929     return nullptr;
 930 }
 931
 932 GenTree* Compiler::impLZCNTIntrinsic(NamedIntrinsic        intrinsic,
 933                                      CORINFO_METHOD_HANDLE method,
 934                                      CORINFO_SIG_INFO*     sig,
 935                                      bool                  mustExpand)
 936 {
 937     assert(sig->numArgs == 1);
 938     var_types callType = JITtype2varType(sig->retType);
 939     return gtNewScalarHWIntrinsicNode(callType, impPopStack().val, NI_LZCNT_LeadingZeroCount);
 940 }
 941
 942 GenTree* Compiler::impPCLMULQDQIntrinsic(NamedIntrinsic        intrinsic,
 943                                          CORINFO_METHOD_HANDLE method,
 944                                          CORINFO_SIG_INFO*     sig,
 945                                          bool                  mustExpand)
 946 {
 947     return nullptr;
 948 }
 949
 950 GenTree* Compiler::impPOPCNTIntrinsic(NamedIntrinsic        intrinsic,
 951                                       CORINFO_METHOD_HANDLE method,
 952                                       CORINFO_SIG_INFO*     sig,
 953                                       bool                  mustExpand)
 954 {
 955     assert(sig->numArgs == 1);
 956     var_types callType = JITtype2varType(sig->retType);
 957     return gtNewScalarHWIntrinsicNode(callType, impPopStack().val, NI_POPCNT_PopCount);
 958 }
 959
 960 #endif // FEATURE_HW_INTRINSICS