src/jit/hwintrinsicArm64.cpp

   1 // Licensed to the .NET Foundation under one or more agreements.
   2 // The .NET Foundation licenses this file to you under the MIT license.
   3 // See the LICENSE file in the project root for more information.
   4
   5 #include "jitpch.h"
   6 #include "hwintrinsicArm64.h"
   7
   8 #ifdef FEATURE_HW_INTRINSICS
   9
  10 namespace IsaFlag
  11 {
  12 enum Flag
  13 {
  14 #define HARDWARE_INTRINSIC_CLASS(flag, isa) isa = 1ULL << InstructionSet_##isa,
  15 #include "hwintrinsiclistArm64.h"
  16     None     = 0,
  17     Base     = 1ULL << InstructionSet_Base,
  18     EveryISA = ~0ULL
  19 };
  20
  21 Flag operator|(Flag a, Flag b)
  22 {
  23     return Flag(uint64_t(a) | uint64_t(b));
  24 }
  25
  26 Flag flag(InstructionSet isa)
  27 {
  28     return Flag(1ULL << isa);
  29 }
  30 }
  31
  32 // clang-format off
  33 static const HWIntrinsicInfo hwIntrinsicInfoArray[] = {
  34     // Add lookupHWIntrinsic special cases see lookupHWIntrinsic() below
  35     //     NI_ARM64_IsSupported_True is used to expand get_IsSupported to const true
  36     //     NI_ARM64_IsSupported_False is used to expand get_IsSupported to const false
  37     //     NI_ARM64_PlatformNotSupported to throw PlatformNotSupported exception for every intrinsic not supported on the running platform
  38     {NI_ARM64_IsSupported_True,     "get_IsSupported",                 IsaFlag::EveryISA, HWIntrinsicInfo::IsSupported, HWIntrinsicInfo::None, {}},
  39     {NI_ARM64_IsSupported_False,    "::NI_ARM64_IsSupported_False",    IsaFlag::EveryISA, HWIntrinsicInfo::IsSupported, HWIntrinsicInfo::None, {}},
  40     {NI_ARM64_PlatformNotSupported, "::NI_ARM64_PlatformNotSupported", IsaFlag::EveryISA, HWIntrinsicInfo::Unsupported, HWIntrinsicInfo::None, {}},
  41 #define HARDWARE_INTRINSIC(id, isa, name, form, i0, i1, i2, flags) \
  42     {id,                            #name,                             IsaFlag::isa,      HWIntrinsicInfo::form,        HWIntrinsicInfo::flags, { i0, i1, i2 }},
  43 #include "hwintrinsiclistArm64.h"
  44 };
  45 // clang-format on
  46
  47 extern const char* getHWIntrinsicName(NamedIntrinsic intrinsic)
  48 {
  49     return hwIntrinsicInfoArray[intrinsic - NI_HW_INTRINSIC_START - 1].intrinsicName;
  50 }
  51
  52 const HWIntrinsicInfo& Compiler::getHWIntrinsicInfo(NamedIntrinsic intrinsic)
  53 {
  54     assert(intrinsic > NI_HW_INTRINSIC_START);
  55     assert(intrinsic < NI_HW_INTRINSIC_END);
  56
  57     return hwIntrinsicInfoArray[intrinsic - NI_HW_INTRINSIC_START - 1];
  58 }
  59
  60 //------------------------------------------------------------------------
  61 // lookupHWIntrinsicISA: map class name to InstructionSet value
  62 //
  63 // Arguments:
  64 //    className -- class name in System.Runtime.Intrinsics.Arm.Arm64
  65 //
  66 // Return Value:
  67 //    Id for the ISA class if enabled.
  68 //
  69 InstructionSet Compiler::lookupHWIntrinsicISA(const char* className)
  70 {
  71     if (className != nullptr)
  72     {
  73         if (strcmp(className, "Base") == 0)
  74             return InstructionSet_Base;
  75 #define HARDWARE_INTRINSIC_CLASS(flag, isa)                                                                            \
  76     if (strcmp(className, #isa) == 0)                                                                                  \
  77         return InstructionSet_##isa;
  78 #include "hwintrinsiclistArm64.h"
  79     }
  80
  81     return InstructionSet_NONE;
  82 }
  83
  84 //------------------------------------------------------------------------
  85 // lookupHWIntrinsic: map intrinsic name to named intrinsic value
  86 //
  87 // Arguments:
  88 //    methodName -- name of the intrinsic function.
  89 //    isa        -- instruction set of the intrinsic.
  90 //
  91 // Return Value:
  92 //    Id for the hardware intrinsic.
  93 //
  94 // TODO-Throughput: replace sequential search by hash lookup
  95 NamedIntrinsic Compiler::lookupHWIntrinsic(const char* className, const char* methodName)
  96 {
  97     InstructionSet isa    = lookupHWIntrinsicISA(className);
  98     NamedIntrinsic result = NI_Illegal;
  99     if (isa != InstructionSet_NONE)
 100     {
 101         IsaFlag::Flag isaFlag = IsaFlag::flag(isa);
 102         for (int i = 0; i < NI_HW_INTRINSIC_END - NI_HW_INTRINSIC_START; i++)
 103         {
 104             if ((isaFlag & hwIntrinsicInfoArray[i].isaflags) &&
 105                 strcmp(methodName, hwIntrinsicInfoArray[i].intrinsicName) == 0)
 106             {
 107                 if (compSupports(isa))
 108                 {
 109                     // Intrinsic is supported on platform
 110                     result = hwIntrinsicInfoArray[i].intrinsicID;
 111                 }
 112                 else
 113                 {
 114                     // When the intrinsic class is not supported
 115                     // Return NI_ARM64_PlatformNotSupported for all intrinsics
 116                     // Return NI_ARM64_IsSupported_False for the IsSupported property
 117                     result = (hwIntrinsicInfoArray[i].intrinsicID != NI_ARM64_IsSupported_True)
 118                                  ? NI_ARM64_PlatformNotSupported
 119                                  : NI_ARM64_IsSupported_False;
 120                 }
 121                 break;
 122             }
 123         }
 124     }
 125     return result;
 126 }
 127
 128 //------------------------------------------------------------------------
 129 // impCheckImmediate: check if immediate is const and in range for inlining
 130 //
 131 bool Compiler::impCheckImmediate(GenTree* immediateOp, unsigned int max)
 132 {
 133     return immediateOp->IsCnsIntOrI() && (immediateOp->AsIntConCommon()->IconValue() < max);
 134 }
 135
 136 //------------------------------------------------------------------------
 137 // impHWIntrinsic: dispatch hardware intrinsics to their own implementation
 138 // function
 139 //
 140 // Arguments:
 141 //    intrinsic -- id of the intrinsic function.
 142 //    method    -- method handle of the intrinsic function.
 143 //    sig       -- signature of the intrinsic call
 144 //
 145 // Return Value:
 146 //    the expanded intrinsic.
 147 //
 148 GenTree* Compiler::impHWIntrinsic(NamedIntrinsic        intrinsic,
 149                                   CORINFO_METHOD_HANDLE method,
 150                                   CORINFO_SIG_INFO*     sig,
 151                                   bool                  mustExpand)
 152 {
 153     GenTree*             retNode       = nullptr;
 154     GenTree*             op1           = nullptr;
 155     GenTree*             op2           = nullptr;
 156     GenTree*             op3           = nullptr;
 157     CORINFO_CLASS_HANDLE simdClass     = nullptr;
 158     var_types            simdType      = TYP_UNKNOWN;
 159     var_types            simdBaseType  = TYP_UNKNOWN;
 160     unsigned             simdSizeBytes = 0;
 161
 162     switch (getHWIntrinsicInfo(intrinsic).form)
 163     {
 164         case HWIntrinsicInfo::SimdBinaryOp:
 165         case HWIntrinsicInfo::SimdInsertOp:
 166         case HWIntrinsicInfo::SimdSelectOp:
 167         case HWIntrinsicInfo::SimdSetAllOp:
 168         case HWIntrinsicInfo::SimdUnaryOp:
 169         case HWIntrinsicInfo::SimdBinaryRMWOp:
 170             simdClass = sig->retTypeClass;
 171             break;
 172         case HWIntrinsicInfo::SimdExtractOp:
 173             info.compCompHnd->getArgType(sig, sig->args, &simdClass);
 174             break;
 175         default:
 176             break;
 177     }
 178
 179     // Simd instantiation type check
 180     if (simdClass != nullptr)
 181     {
 182         simdBaseType = getBaseTypeAndSizeOfSIMDType(simdClass, &simdSizeBytes);
 183
 184         if (simdBaseType == TYP_UNKNOWN)
 185         {
 186             return impUnsupportedHWIntrinsic(CORINFO_HELP_THROW_TYPE_NOT_SUPPORTED, method, sig, mustExpand);
 187         }
 188         simdType = getSIMDTypeForSize(simdSizeBytes);
 189     }
 190
 191     switch (getHWIntrinsicInfo(intrinsic).form)
 192     {
 193         case HWIntrinsicInfo::IsSupported:
 194             return gtNewIconNode((intrinsic == NI_ARM64_IsSupported_True) ? 1 : 0);
 195
 196         case HWIntrinsicInfo::Unsupported:
 197             return impUnsupportedHWIntrinsic(CORINFO_HELP_THROW_PLATFORM_NOT_SUPPORTED, method, sig, mustExpand);
 198
 199         case HWIntrinsicInfo::SimdBinaryOp:
 200         case HWIntrinsicInfo::SimdBinaryRMWOp:
 201             // op1 is the first operand
 202             // op2 is the second operand
 203             op2 = impSIMDPopStack(simdType);
 204             op1 = impSIMDPopStack(simdType);
 205
 206             return gtNewSimdHWIntrinsicNode(simdType, op1, op2, intrinsic, simdBaseType, simdSizeBytes);
 207
 208         case HWIntrinsicInfo::SimdSelectOp:
 209             // op1 is the first operand
 210             // op2 is the second operand
 211             // op3 is the third operand
 212             op3 = impSIMDPopStack(simdType);
 213             op2 = impSIMDPopStack(simdType);
 214             op1 = impSIMDPopStack(simdType);
 215
 216             return gtNewSimdHWIntrinsicNode(simdType, op1, op2, op3, intrinsic, simdBaseType, simdSizeBytes);
 217
 218         case HWIntrinsicInfo::SimdSetAllOp:
 219             op1 = impPopStack().val;
 220
 221             return gtNewSimdHWIntrinsicNode(simdType, op1, intrinsic, simdBaseType, simdSizeBytes);
 222
 223         case HWIntrinsicInfo::SimdUnaryOp:
 224             op1 = impSIMDPopStack(simdType);
 225
 226             return gtNewSimdHWIntrinsicNode(simdType, op1, intrinsic, simdBaseType, simdSizeBytes);
 227
 228         case HWIntrinsicInfo::SimdExtractOp:
 229             if (!mustExpand && !impCheckImmediate(impStackTop(0).val, getSIMDVectorLength(simdSizeBytes, simdBaseType)))
 230             {
 231                 // Immediate lane not constant or out of range
 232                 return nullptr;
 233             }
 234             op2 = impPopStack().val;
 235             op1 = impSIMDPopStack(simdType);
 236
 237             return gtNewScalarHWIntrinsicNode(JITtype2varType(sig->retType), op1, op2, intrinsic);
 238
 239         case HWIntrinsicInfo::SimdInsertOp:
 240             if (!mustExpand && !impCheckImmediate(impStackTop(1).val, getSIMDVectorLength(simdSizeBytes, simdBaseType)))
 241             {
 242                 // Immediate lane not constant or out of range
 243                 return nullptr;
 244             }
 245             op3 = impPopStack().val;
 246             op2 = impPopStack().val;
 247             op1 = impSIMDPopStack(simdType);
 248
 249             return gtNewSimdHWIntrinsicNode(simdType, op1, op2, op3, intrinsic, simdBaseType, simdSizeBytes);
 250
 251         default:
 252             JITDUMP("Not implemented hardware intrinsic form");
 253             assert(!"Unimplemented SIMD Intrinsic form");
 254
 255             break;
 256     }
 257     return retNode;
 258 }
 259
 260 #endif // FEATURE_HW_INTRINSICS