1 // Licensed to the .NET Foundation under one or more agreements.
2 // The .NET Foundation licenses this file to you under the MIT license.
3 // See the LICENSE file in the project root for more information.
8 // IMPORTANT NOTES AND CAVEATS:
10 // This implementation is preliminary, and may change dramatically.
12 // New JIT types, TYP_SIMDxx, are introduced, and the SIMD intrinsics are created as GT_SIMD nodes.
13 // Nodes of SIMD types will be typed as TYP_SIMD* (e.g. TYP_SIMD8, TYP_SIMD16, etc.).
15 // Note that currently the "reference implementation" is the same as the runtime dll. As such, it is currently
16 // providing implementations for those methods not currently supported by the JIT as intrinsics.
18 // These are currently recognized using string compares, in order to provide an implementation in the JIT
19 // without taking a dependency on the VM.
20 // Furthermore, in the CTP, in order to limit the impact of doing these string compares
21 // against assembly names, we only look for the SIMDVector assembly if we are compiling a class constructor. This
22 // makes it somewhat more "pay for play" but is a significant usability compromise.
23 // This has been addressed for RTM by doing the assembly recognition in the VM.
24 // --------------------------------------------------------------------------------------
33 #ifndef LEGACY_BACKEND // This file is ONLY used for the RyuJIT backend that uses the linear scan register allocator.
37 // Intrinsic Id to intrinsic info map
38 const SIMDIntrinsicInfo simdIntrinsicInfoArray[] = {
39 #define SIMD_INTRINSIC(mname, inst, id, name, retType, argCount, arg1, arg2, arg3, t1, t2, t3, t4, t5, t6, t7, t8, t9, \
41 {SIMDIntrinsic##id, mname, inst, retType, argCount, arg1, arg2, arg3, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10},
42 #include "simdintrinsiclist.h"
45 //------------------------------------------------------------------------
46 // getSIMDVectorLength: Get the length (number of elements of base type) of
47 // SIMD Vector given its size and base (element) type.
50 // simdSize - size of the SIMD vector
51 // baseType - type of the elements of the SIMD vector
54 int Compiler::getSIMDVectorLength(unsigned simdSize, var_types baseType)
56 return simdSize / genTypeSize(baseType);
59 //------------------------------------------------------------------------
60 // Get the length (number of elements of base type) of SIMD Vector given by typeHnd.
63 // typeHnd - type handle of the SIMD vector
65 int Compiler::getSIMDVectorLength(CORINFO_CLASS_HANDLE typeHnd)
67 unsigned sizeBytes = 0;
68 var_types baseType = getBaseTypeAndSizeOfSIMDType(typeHnd, &sizeBytes);
69 return getSIMDVectorLength(sizeBytes, baseType);
72 //------------------------------------------------------------------------
73 // Get the preferred alignment of SIMD vector type for better performance.
76 // typeHnd - type handle of the SIMD vector
78 int Compiler::getSIMDTypeAlignment(var_types simdType)
81 // Fixed length vectors have the following alignment preference
82 // Vector2 = 8 byte alignment
83 // Vector3/4 = 16-byte alignment
84 unsigned size = genTypeSize(simdType);
86 // preferred alignment for SSE2 128-bit vectors is 16-bytes
93 assert((size == 12) || (size == 16));
101 #elif defined(_TARGET_ARM64_)
104 assert(!"getSIMDTypeAlignment() unimplemented on target arch");
109 //----------------------------------------------------------------------------------
110 // Return the base type and size of SIMD vector type given its type handle.
113 // typeHnd - The handle of the type we're interested in.
114 // sizeBytes - out param
117 // base type of SIMD vector.
118 // sizeBytes if non-null is set to size in bytes.
120 // TODO-Throughput: current implementation parses class name to find base type. Change
121 // this when we implement SIMD intrinsic identification for the final
124 var_types Compiler::getBaseTypeAndSizeOfSIMDType(CORINFO_CLASS_HANDLE typeHnd, unsigned* sizeBytes /*= nullptr */)
127 if (typeHnd == nullptr)
132 // fast path search using cached type handles of important types
133 var_types simdBaseType = TYP_UNKNOWN;
136 // TODO - Optimize SIMD type recognition by IntrinsicAttribute
137 if (isSIMDClass(typeHnd))
139 // The most likely to be used type handles are looked up first followed by
140 // less likely to be used type handles
141 if (typeHnd == SIMDFloatHandle)
143 simdBaseType = TYP_FLOAT;
144 JITDUMP(" Known type SIMD Vector<Float>\n");
146 else if (typeHnd == SIMDIntHandle)
148 simdBaseType = TYP_INT;
149 JITDUMP(" Known type SIMD Vector<Int>\n");
151 else if (typeHnd == SIMDVector2Handle)
153 simdBaseType = TYP_FLOAT;
154 size = 2 * genTypeSize(TYP_FLOAT);
155 assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE));
156 JITDUMP(" Known type Vector2\n");
158 else if (typeHnd == SIMDVector3Handle)
160 simdBaseType = TYP_FLOAT;
161 size = 3 * genTypeSize(TYP_FLOAT);
162 assert(size == info.compCompHnd->getClassSize(typeHnd));
163 JITDUMP(" Known type Vector3\n");
165 else if (typeHnd == SIMDVector4Handle)
167 simdBaseType = TYP_FLOAT;
168 size = 4 * genTypeSize(TYP_FLOAT);
169 assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE));
170 JITDUMP(" Known type Vector4\n");
172 else if (typeHnd == SIMDVectorHandle)
174 JITDUMP(" Known type Vector\n");
176 else if (typeHnd == SIMDUShortHandle)
178 simdBaseType = TYP_USHORT;
179 JITDUMP(" Known type SIMD Vector<ushort>\n");
181 else if (typeHnd == SIMDUByteHandle)
183 simdBaseType = TYP_UBYTE;
184 JITDUMP(" Known type SIMD Vector<ubyte>\n");
186 else if (typeHnd == SIMDDoubleHandle)
188 simdBaseType = TYP_DOUBLE;
189 JITDUMP(" Known type SIMD Vector<Double>\n");
191 else if (typeHnd == SIMDLongHandle)
193 simdBaseType = TYP_LONG;
194 JITDUMP(" Known type SIMD Vector<Long>\n");
196 else if (typeHnd == SIMDShortHandle)
198 simdBaseType = TYP_SHORT;
199 JITDUMP(" Known type SIMD Vector<short>\n");
201 else if (typeHnd == SIMDByteHandle)
203 simdBaseType = TYP_BYTE;
204 JITDUMP(" Known type SIMD Vector<byte>\n");
206 else if (typeHnd == SIMDUIntHandle)
208 simdBaseType = TYP_UINT;
209 JITDUMP(" Known type SIMD Vector<uint>\n");
211 else if (typeHnd == SIMDULongHandle)
213 simdBaseType = TYP_ULONG;
214 JITDUMP(" Known type SIMD Vector<ulong>\n");
218 if (simdBaseType == TYP_UNKNOWN)
220 // Doesn't match with any of the cached type handles.
221 // Obtain base type by parsing fully qualified class name.
223 // TODO-Throughput: implement product shipping solution to query base type.
224 WCHAR className[256] = {0};
225 WCHAR* pbuf = &className[0];
226 int len = _countof(className);
227 info.compCompHnd->appendClassName(&pbuf, &len, typeHnd, TRUE, FALSE, FALSE);
228 noway_assert(pbuf < &className[256]);
229 JITDUMP("SIMD Candidate Type %S\n", className);
231 if (wcsncmp(className, W("System.Numerics."), 16) == 0)
233 if (wcsncmp(&(className[16]), W("Vector`1["), 9) == 0)
235 if (wcsncmp(&(className[25]), W("System.Single"), 13) == 0)
237 SIMDFloatHandle = typeHnd;
238 simdBaseType = TYP_FLOAT;
239 JITDUMP(" Found type SIMD Vector<Float>\n");
241 else if (wcsncmp(&(className[25]), W("System.Int32"), 12) == 0)
243 SIMDIntHandle = typeHnd;
244 simdBaseType = TYP_INT;
245 JITDUMP(" Found type SIMD Vector<Int>\n");
247 else if (wcsncmp(&(className[25]), W("System.UInt16"), 13) == 0)
249 SIMDUShortHandle = typeHnd;
250 simdBaseType = TYP_USHORT;
251 JITDUMP(" Found type SIMD Vector<ushort>\n");
253 else if (wcsncmp(&(className[25]), W("System.Byte"), 11) == 0)
255 SIMDUByteHandle = typeHnd;
256 simdBaseType = TYP_UBYTE;
257 JITDUMP(" Found type SIMD Vector<ubyte>\n");
259 else if (wcsncmp(&(className[25]), W("System.Double"), 13) == 0)
261 SIMDDoubleHandle = typeHnd;
262 simdBaseType = TYP_DOUBLE;
263 JITDUMP(" Found type SIMD Vector<Double>\n");
265 else if (wcsncmp(&(className[25]), W("System.Int64"), 12) == 0)
267 SIMDLongHandle = typeHnd;
268 simdBaseType = TYP_LONG;
269 JITDUMP(" Found type SIMD Vector<Long>\n");
271 else if (wcsncmp(&(className[25]), W("System.Int16"), 12) == 0)
273 SIMDShortHandle = typeHnd;
274 simdBaseType = TYP_SHORT;
275 JITDUMP(" Found type SIMD Vector<short>\n");
277 else if (wcsncmp(&(className[25]), W("System.SByte"), 12) == 0)
279 SIMDByteHandle = typeHnd;
280 simdBaseType = TYP_BYTE;
281 JITDUMP(" Found type SIMD Vector<byte>\n");
283 else if (wcsncmp(&(className[25]), W("System.UInt32"), 13) == 0)
285 SIMDUIntHandle = typeHnd;
286 simdBaseType = TYP_UINT;
287 JITDUMP(" Found type SIMD Vector<uint>\n");
289 else if (wcsncmp(&(className[25]), W("System.UInt64"), 13) == 0)
291 SIMDULongHandle = typeHnd;
292 simdBaseType = TYP_ULONG;
293 JITDUMP(" Found type SIMD Vector<ulong>\n");
297 JITDUMP(" Unknown SIMD Vector<T>\n");
300 else if (wcsncmp(&(className[16]), W("Vector2"), 8) == 0)
302 SIMDVector2Handle = typeHnd;
304 simdBaseType = TYP_FLOAT;
305 size = 2 * genTypeSize(TYP_FLOAT);
306 assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE));
307 JITDUMP(" Found Vector2\n");
309 else if (wcsncmp(&(className[16]), W("Vector3"), 8) == 0)
311 SIMDVector3Handle = typeHnd;
313 simdBaseType = TYP_FLOAT;
314 size = 3 * genTypeSize(TYP_FLOAT);
315 assert(size == info.compCompHnd->getClassSize(typeHnd));
316 JITDUMP(" Found Vector3\n");
318 else if (wcsncmp(&(className[16]), W("Vector4"), 8) == 0)
320 SIMDVector4Handle = typeHnd;
322 simdBaseType = TYP_FLOAT;
323 size = 4 * genTypeSize(TYP_FLOAT);
324 assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE));
325 JITDUMP(" Found Vector4\n");
327 else if (wcsncmp(&(className[16]), W("Vector"), 6) == 0)
329 SIMDVectorHandle = typeHnd;
330 JITDUMP(" Found type Vector\n");
334 JITDUMP(" Unknown SIMD Type\n");
338 if (simdBaseType != TYP_UNKNOWN && sizeBytes != nullptr)
340 // If not a fixed size vector then its size is same as SIMD vector
341 // register length in bytes
344 size = getSIMDVectorRegisterByteLength();
348 setUsesSIMDTypes(true);
351 #ifdef FEATURE_HW_INTRINSICS
352 else if (isIntrinsicType(typeHnd))
354 const size_t Vector64SizeBytes = 64 / 8;
355 const size_t Vector128SizeBytes = 128 / 8;
356 const size_t Vector256SizeBytes = 256 / 8;
358 #if defined(_TARGET_XARCH_)
359 static_assert_no_msg(YMM_REGSIZE_BYTES == Vector256SizeBytes);
360 static_assert_no_msg(XMM_REGSIZE_BYTES == Vector128SizeBytes);
362 if (typeHnd == Vector256FloatHandle)
364 simdBaseType = TYP_FLOAT;
365 size = Vector256SizeBytes;
366 JITDUMP(" Known type Vector256<float>\n");
368 else if (typeHnd == Vector256DoubleHandle)
370 simdBaseType = TYP_DOUBLE;
371 size = Vector256SizeBytes;
372 JITDUMP(" Known type Vector256<double>\n");
374 else if (typeHnd == Vector256IntHandle)
376 simdBaseType = TYP_INT;
377 size = Vector256SizeBytes;
378 JITDUMP(" Known type Vector256<int>\n");
380 else if (typeHnd == Vector256UIntHandle)
382 simdBaseType = TYP_UINT;
383 size = Vector256SizeBytes;
384 JITDUMP(" Known type Vector256<uint>\n");
386 else if (typeHnd == Vector256ShortHandle)
388 simdBaseType = TYP_SHORT;
389 size = Vector256SizeBytes;
390 JITDUMP(" Known type Vector256<short>\n");
392 else if (typeHnd == Vector256UShortHandle)
394 simdBaseType = TYP_USHORT;
395 size = Vector256SizeBytes;
396 JITDUMP(" Known type Vector256<ushort>\n");
398 else if (typeHnd == Vector256ByteHandle)
400 simdBaseType = TYP_BYTE;
401 size = Vector256SizeBytes;
402 JITDUMP(" Known type Vector256<sbyte>\n");
404 else if (typeHnd == Vector256UByteHandle)
406 simdBaseType = TYP_UBYTE;
407 size = Vector256SizeBytes;
408 JITDUMP(" Known type Vector256<byte>\n");
410 else if (typeHnd == Vector256LongHandle)
412 simdBaseType = TYP_LONG;
413 size = Vector256SizeBytes;
414 JITDUMP(" Known type Vector256<long>\n");
416 else if (typeHnd == Vector256ULongHandle)
418 simdBaseType = TYP_ULONG;
419 size = Vector256SizeBytes;
420 JITDUMP(" Known type Vector256<ulong>\n");
422 else if (typeHnd == Vector256FloatHandle)
424 simdBaseType = TYP_FLOAT;
425 size = Vector256SizeBytes;
426 JITDUMP(" Known type Vector256<float>\n");
429 #endif // defined(_TARGET_XARCH)
430 if (typeHnd == Vector128DoubleHandle)
432 simdBaseType = TYP_DOUBLE;
433 size = Vector128SizeBytes;
434 JITDUMP(" Known type Vector128<double>\n");
436 else if (typeHnd == Vector128IntHandle)
438 simdBaseType = TYP_INT;
439 size = Vector128SizeBytes;
440 JITDUMP(" Known type Vector128<int>\n");
442 else if (typeHnd == Vector128UIntHandle)
444 simdBaseType = TYP_UINT;
445 size = Vector128SizeBytes;
446 JITDUMP(" Known type Vector128<uint>\n");
448 else if (typeHnd == Vector128ShortHandle)
450 simdBaseType = TYP_SHORT;
451 size = Vector128SizeBytes;
452 JITDUMP(" Known type Vector128<short>\n");
454 else if (typeHnd == Vector128UShortHandle)
456 simdBaseType = TYP_USHORT;
457 size = Vector128SizeBytes;
458 JITDUMP(" Known type Vector128<ushort>\n");
460 else if (typeHnd == Vector128ByteHandle)
462 simdBaseType = TYP_BYTE;
463 size = Vector128SizeBytes;
464 JITDUMP(" Known type Vector128<sbyte>\n");
466 else if (typeHnd == Vector128UByteHandle)
468 simdBaseType = TYP_UBYTE;
469 size = Vector128SizeBytes;
470 JITDUMP(" Known type Vector128<byte>\n");
472 else if (typeHnd == Vector128LongHandle)
474 simdBaseType = TYP_LONG;
475 size = Vector128SizeBytes;
476 JITDUMP(" Known type Vector128<long>\n");
478 else if (typeHnd == Vector128ULongHandle)
480 simdBaseType = TYP_ULONG;
481 size = Vector128SizeBytes;
482 JITDUMP(" Known type Vector128<ulong>\n");
484 #if defined(_TARGET_ARM64_)
485 else if (typeHnd == Vector64IntHandle)
487 simdBaseType = TYP_INT;
488 size = Vector64SizeBytes;
489 JITDUMP(" Known type Vector64<int>\n");
491 else if (typeHnd == Vector64UIntHandle)
493 simdBaseType = TYP_UINT;
494 size = Vector64SizeBytes;
495 JITDUMP(" Known type Vector64<uint>\n");
497 else if (typeHnd == Vector64ShortHandle)
499 simdBaseType = TYP_SHORT;
500 size = Vector64SizeBytes;
501 JITDUMP(" Known type Vector64<short>\n");
503 else if (typeHnd == Vector64UShortHandle)
505 simdBaseType = TYP_USHORT;
506 size = Vector64SizeBytes;
507 JITDUMP(" Known type Vector64<ushort>\n");
509 else if (typeHnd == Vector64ByteHandle)
511 simdBaseType = TYP_BYTE;
512 size = Vector64SizeBytes;
513 JITDUMP(" Known type Vector64<sbyte>\n");
515 else if (typeHnd == Vector64UByteHandle)
517 simdBaseType = TYP_UBYTE;
518 size = Vector64SizeBytes;
519 JITDUMP(" Known type Vector64<byte>\n");
521 #endif // defined(_TARGET_ARM64_)
524 if (simdBaseType == TYP_UNKNOWN)
526 // Doesn't match with any of the cached type handles.
527 const char* className = getClassNameFromMetadata(typeHnd, nullptr);
528 CORINFO_CLASS_HANDLE baseTypeHnd = getTypeInstantiationArgument(typeHnd, 0);
530 if (baseTypeHnd != nullptr)
532 CorInfoType type = info.compCompHnd->getTypeForPrimitiveNumericClass(baseTypeHnd);
534 JITDUMP("HW Intrinsic SIMD Candidate Type %s with Base Type %s\n", className,
535 getClassNameFromMetadata(baseTypeHnd, nullptr));
537 #if defined(_TARGET_XARCH_)
538 if (strcmp(className, "Vector256`1") == 0)
540 size = Vector256SizeBytes;
543 case CORINFO_TYPE_FLOAT:
544 Vector256FloatHandle = typeHnd;
545 simdBaseType = TYP_FLOAT;
546 JITDUMP(" Found type Hardware Intrinsic SIMD Vector256<float>\n");
548 case CORINFO_TYPE_DOUBLE:
549 Vector256DoubleHandle = typeHnd;
550 simdBaseType = TYP_DOUBLE;
551 JITDUMP(" Found type Hardware Intrinsic SIMD Vector256<double>\n");
553 case CORINFO_TYPE_INT:
554 Vector256IntHandle = typeHnd;
555 simdBaseType = TYP_INT;
556 JITDUMP(" Found type Hardware Intrinsic SIMD Vector256<int>\n");
558 case CORINFO_TYPE_UINT:
559 Vector256UIntHandle = typeHnd;
560 simdBaseType = TYP_UINT;
561 JITDUMP(" Found type Hardware Intrinsic SIMD Vector256<uint>\n");
563 case CORINFO_TYPE_SHORT:
564 Vector256ShortHandle = typeHnd;
565 simdBaseType = TYP_SHORT;
566 JITDUMP(" Found type Hardware Intrinsic SIMD Vector256<short>\n");
568 case CORINFO_TYPE_USHORT:
569 Vector256UShortHandle = typeHnd;
570 simdBaseType = TYP_USHORT;
571 JITDUMP(" Found type Hardware Intrinsic SIMD Vector256<ushort>\n");
573 case CORINFO_TYPE_LONG:
574 Vector256LongHandle = typeHnd;
575 simdBaseType = TYP_LONG;
576 JITDUMP(" Found type Hardware Intrinsic SIMD Vector256<long>\n");
578 case CORINFO_TYPE_ULONG:
579 Vector256ULongHandle = typeHnd;
580 simdBaseType = TYP_ULONG;
581 JITDUMP(" Found type Hardware Intrinsic SIMD Vector256<ulong>\n");
583 case CORINFO_TYPE_UBYTE:
584 Vector256UByteHandle = typeHnd;
585 simdBaseType = TYP_UBYTE;
586 JITDUMP(" Found type Hardware Intrinsic SIMD Vector256<byte>\n");
588 case CORINFO_TYPE_BYTE:
589 Vector256ByteHandle = typeHnd;
590 simdBaseType = TYP_BYTE;
591 JITDUMP(" Found type Hardware Intrinsic SIMD Vector256<sbyte>\n");
595 JITDUMP(" Unknown Hardware Intrinsic SIMD Type Vector256<T>\n");
599 #endif // defined(_TARGET_XARCH_)
600 if (strcmp(className, "Vector128`1") == 0)
602 size = Vector128SizeBytes;
605 case CORINFO_TYPE_FLOAT:
606 Vector128FloatHandle = typeHnd;
607 simdBaseType = TYP_FLOAT;
608 JITDUMP(" Found type Hardware Intrinsic SIMD Vector128<float>\n");
610 case CORINFO_TYPE_DOUBLE:
611 Vector128DoubleHandle = typeHnd;
612 simdBaseType = TYP_DOUBLE;
613 JITDUMP(" Found type Hardware Intrinsic SIMD Vector128<double>\n");
615 case CORINFO_TYPE_INT:
616 Vector128IntHandle = typeHnd;
617 simdBaseType = TYP_INT;
618 JITDUMP(" Found type Hardware Intrinsic SIMD Vector128<int>\n");
620 case CORINFO_TYPE_UINT:
621 Vector128UIntHandle = typeHnd;
622 simdBaseType = TYP_UINT;
623 JITDUMP(" Found type Hardware Intrinsic SIMD Vector128<uint>\n");
625 case CORINFO_TYPE_SHORT:
626 Vector128ShortHandle = typeHnd;
627 simdBaseType = TYP_SHORT;
628 JITDUMP(" Found type Hardware Intrinsic SIMD Vector128<short>\n");
630 case CORINFO_TYPE_USHORT:
631 Vector128UShortHandle = typeHnd;
632 simdBaseType = TYP_USHORT;
633 JITDUMP(" Found type Hardware Intrinsic SIMD Vector128<ushort>\n");
635 case CORINFO_TYPE_LONG:
636 Vector128LongHandle = typeHnd;
637 simdBaseType = TYP_LONG;
638 JITDUMP(" Found type Hardware Intrinsic SIMD Vector128<long>\n");
640 case CORINFO_TYPE_ULONG:
641 Vector128ULongHandle = typeHnd;
642 simdBaseType = TYP_ULONG;
643 JITDUMP(" Found type Hardware Intrinsic SIMD Vector128<ulong>\n");
645 case CORINFO_TYPE_UBYTE:
646 Vector128UByteHandle = typeHnd;
647 simdBaseType = TYP_UBYTE;
648 JITDUMP(" Found type Hardware Intrinsic SIMD Vector128<byte>\n");
650 case CORINFO_TYPE_BYTE:
651 Vector128ByteHandle = typeHnd;
652 simdBaseType = TYP_BYTE;
653 JITDUMP(" Found type Hardware Intrinsic SIMD Vector128<sbyte>\n");
657 JITDUMP(" Unknown Hardware Intrinsic SIMD Type Vector128<T>\n");
660 #if defined(_TARGET_ARM64_)
661 else if (strcmp(className, "Vector64`1") == 0)
663 size = Vector64SizeBytes;
666 case CORINFO_TYPE_FLOAT:
667 Vector64FloatHandle = typeHnd;
668 simdBaseType = TYP_FLOAT;
669 JITDUMP(" Found type Hardware Intrinsic SIMD Vector64<float>\n");
671 case CORINFO_TYPE_INT:
672 Vector64IntHandle = typeHnd;
673 simdBaseType = TYP_INT;
674 JITDUMP(" Found type Hardware Intrinsic SIMD Vector64<int>\n");
676 case CORINFO_TYPE_UINT:
677 Vector64UIntHandle = typeHnd;
678 simdBaseType = TYP_UINT;
679 JITDUMP(" Found type Hardware Intrinsic SIMD Vector64<uint>\n");
681 case CORINFO_TYPE_SHORT:
682 Vector64ShortHandle = typeHnd;
683 simdBaseType = TYP_SHORT;
684 JITDUMP(" Found type Hardware Intrinsic SIMD Vector64<short>\n");
686 case CORINFO_TYPE_USHORT:
687 Vector64UShortHandle = typeHnd;
688 simdBaseType = TYP_USHORT;
689 JITDUMP(" Found type Hardware Intrinsic SIMD Vector64<ushort>\n");
691 case CORINFO_TYPE_UBYTE:
692 Vector64UByteHandle = typeHnd;
693 simdBaseType = TYP_UBYTE;
694 JITDUMP(" Found type Hardware Intrinsic SIMD Vector64<byte>\n");
696 case CORINFO_TYPE_BYTE:
697 Vector64ByteHandle = typeHnd;
698 simdBaseType = TYP_BYTE;
699 JITDUMP(" Found type Hardware Intrinsic SIMD Vector64<sbyte>\n");
703 JITDUMP(" Unknown Hardware Intrinsic SIMD Type Vector64<T>\n");
706 #endif // defined(_TARGET_ARM64_)
710 if (sizeBytes != nullptr)
715 if (simdBaseType != TYP_UNKNOWN)
717 setUsesSIMDTypes(true);
720 #endif // FEATURE_HW_INTRINSICS
725 //--------------------------------------------------------------------------------------
726 // getSIMDIntrinsicInfo: get SIMD intrinsic info given the method handle.
729 // inOutTypeHnd - The handle of the type on which the method is invoked. This is an in-out param.
730 // methodHnd - The handle of the method we're interested in.
731 // sig - method signature info
732 // isNewObj - whether this call represents a newboj constructor call
733 // argCount - argument count - out pram
734 // baseType - base type of the intrinsic - out param
735 // sizeBytes - size of SIMD vector type on which the method is invoked - out param
738 // SIMDIntrinsicInfo struct initialized corresponding to methodHnd.
739 // Sets SIMDIntrinsicInfo.id to SIMDIntrinsicInvalid if methodHnd doesn't correspond
740 // to any SIMD intrinsic. Also, sets the out params inOutTypeHnd, argCount, baseType and
743 // Note that VectorMath class doesn't have a base type and first argument of the method
744 // determines the SIMD vector type on which intrinsic is invoked. In such a case inOutTypeHnd
745 // is modified by this routine.
747 // TODO-Throughput: The current implementation is based on method name string parsing.
748 // Although we now have type identification from the VM, the parsing of intrinsic names
749 // could be made more efficient.
751 const SIMDIntrinsicInfo* Compiler::getSIMDIntrinsicInfo(CORINFO_CLASS_HANDLE* inOutTypeHnd,
752 CORINFO_METHOD_HANDLE methodHnd,
753 CORINFO_SIG_INFO* sig,
760 assert(baseType != nullptr);
761 assert(sizeBytes != nullptr);
763 // get baseType and size of the type
764 CORINFO_CLASS_HANDLE typeHnd = *inOutTypeHnd;
765 *baseType = getBaseTypeAndSizeOfSIMDType(typeHnd, sizeBytes);
767 if (typeHnd == SIMDVectorHandle)
769 // All of the supported intrinsics on this static class take a first argument that's a vector,
770 // which determines the baseType.
771 // The exception is the IsHardwareAccelerated property, which is handled as a special case.
772 assert(*baseType == TYP_UNKNOWN);
773 if (sig->numArgs == 0)
775 const SIMDIntrinsicInfo* hwAccelIntrinsicInfo = &(simdIntrinsicInfoArray[SIMDIntrinsicHWAccel]);
776 if ((strcmp(eeGetMethodName(methodHnd, nullptr), hwAccelIntrinsicInfo->methodName) == 0) &&
777 JITtype2varType(sig->retType) == hwAccelIntrinsicInfo->retType)
780 assert(hwAccelIntrinsicInfo->argCount == 0 && hwAccelIntrinsicInfo->isInstMethod == false);
781 return hwAccelIntrinsicInfo;
787 typeHnd = info.compCompHnd->getArgClass(sig, sig->args);
788 *inOutTypeHnd = typeHnd;
789 *baseType = getBaseTypeAndSizeOfSIMDType(typeHnd, sizeBytes);
793 if (*baseType == TYP_UNKNOWN)
795 JITDUMP("NOT a SIMD Intrinsic: unsupported baseType\n");
799 // account for implicit "this" arg
800 *argCount = sig->numArgs;
806 // Get the Intrinsic Id by parsing method name.
808 // TODO-Throughput: replace sequential search by binary search by arranging entries
809 // sorted by method name.
810 SIMDIntrinsicID intrinsicId = SIMDIntrinsicInvalid;
811 const char* methodName = eeGetMethodName(methodHnd, nullptr);
812 for (int i = SIMDIntrinsicNone + 1; i < SIMDIntrinsicInvalid; ++i)
814 if (strcmp(methodName, simdIntrinsicInfoArray[i].methodName) == 0)
816 // Found an entry for the method; further check whether it is one of
817 // the supported base types.
819 for (int j = 0; j < SIMD_INTRINSIC_MAX_BASETYPE_COUNT; ++j)
821 // Convention: if there are fewer base types supported than MAX_BASETYPE_COUNT,
822 // the end of the list is marked by TYP_UNDEF.
823 if (simdIntrinsicInfoArray[i].supportedBaseTypes[j] == TYP_UNDEF)
828 if (simdIntrinsicInfoArray[i].supportedBaseTypes[j] == *baseType)
840 // Now, check the arguments.
841 unsigned int fixedArgCnt = simdIntrinsicInfoArray[i].argCount;
842 unsigned int expectedArgCnt = fixedArgCnt;
844 // First handle SIMDIntrinsicInitN, where the arg count depends on the type.
845 // The listed arg types include the vector and the first two init values, which is the expected number
846 // for Vector2. For other cases, we'll check their types here.
847 if (*argCount > expectedArgCnt)
849 if (i == SIMDIntrinsicInitN)
851 if (*argCount == 3 && typeHnd == SIMDVector2Handle)
855 else if (*argCount == 4 && typeHnd == SIMDVector3Handle)
859 else if (*argCount == 5 && typeHnd == SIMDVector4Handle)
864 else if (i == SIMDIntrinsicInitFixed)
866 if (*argCount == 4 && typeHnd == SIMDVector4Handle)
872 if (*argCount != expectedArgCnt)
877 // Validate the types of individual args passed are what is expected of.
878 // If any of the types don't match with what is expected, don't consider
879 // as an intrinsic. This will make an older JIT with SIMD capabilities
880 // resilient to breaking changes to SIMD managed API.
882 // Note that from IL type stack, args get popped in right to left order
883 // whereas args get listed in method signatures in left to right order.
885 int stackIndex = (expectedArgCnt - 1);
887 // Track the arguments from the signature - we currently only use this to distinguish
888 // integral and pointer types, both of which will by TYP_I_IMPL on the importer stack.
889 CORINFO_ARG_LIST_HANDLE argLst = sig->args;
891 CORINFO_CLASS_HANDLE argClass;
892 for (unsigned int argIndex = 0; found == true && argIndex < expectedArgCnt; argIndex++)
894 bool isThisPtr = ((argIndex == 0) && sig->hasThis());
896 // In case of "newobj SIMDVector<T>(T val)", thisPtr won't be present on type stack.
897 // We don't check anything in that case.
898 if (!isThisPtr || !isNewObj)
900 GenTree* arg = impStackTop(stackIndex).val;
901 var_types argType = arg->TypeGet();
903 var_types expectedArgType;
904 if (argIndex < fixedArgCnt)
907 // - intrinsicInfo.argType[i] == TYP_UNDEF - intrinsic doesn't have a valid arg at position i
908 // - intrinsicInfo.argType[i] == TYP_UNKNOWN - arg type should be same as basetype
909 // Note that we pop the args off in reverse order.
910 expectedArgType = simdIntrinsicInfoArray[i].argType[argIndex];
911 assert(expectedArgType != TYP_UNDEF);
912 if (expectedArgType == TYP_UNKNOWN)
914 // The type of the argument will be genActualType(*baseType).
915 expectedArgType = genActualType(*baseType);
916 argType = genActualType(argType);
921 expectedArgType = *baseType;
924 if (!isThisPtr && argType == TYP_I_IMPL)
926 // The reference implementation has a constructor that takes a pointer.
927 // We don't want to recognize that one. This requires us to look at the CorInfoType
928 // in order to distinguish a signature with a pointer argument from one with an
929 // integer argument of pointer size, both of which will be TYP_I_IMPL on the stack.
930 // TODO-Review: This seems quite fragile. We should consider beefing up the checking
932 CorInfoType corType = strip(info.compCompHnd->getArgType(sig, argLst, &argClass));
933 if (corType == CORINFO_TYPE_PTR)
939 if (varTypeIsSIMD(argType))
941 argType = TYP_STRUCT;
943 if (argType != expectedArgType)
948 if (argIndex != 0 || !sig->hasThis())
950 argLst = info.compCompHnd->getArgNext(argLst);
955 // Cross check return type and static vs. instance is what we are expecting.
956 // If not, don't consider it as an intrinsic.
957 // Note that ret type of TYP_UNKNOWN means that it is not known apriori and must be same as baseType
960 var_types expectedRetType = simdIntrinsicInfoArray[i].retType;
961 if (expectedRetType == TYP_UNKNOWN)
963 // JIT maps uint/ulong type vars to TYP_INT/TYP_LONG.
965 (*baseType == TYP_UINT || *baseType == TYP_ULONG) ? genActualType(*baseType) : *baseType;
968 if (JITtype2varType(sig->retType) != expectedRetType ||
969 sig->hasThis() != simdIntrinsicInfoArray[i].isInstMethod)
977 intrinsicId = (SIMDIntrinsicID)i;
983 if (intrinsicId != SIMDIntrinsicInvalid)
985 JITDUMP("Method %s maps to SIMD intrinsic %s\n", methodName, simdIntrinsicNames[intrinsicId]);
986 return &simdIntrinsicInfoArray[intrinsicId];
990 JITDUMP("Method %s is NOT a SIMD intrinsic\n", methodName);
996 // Pops and returns GenTree node from importer's type stack.
997 // Normalizes TYP_STRUCT value in case of GT_CALL, GT_RET_EXPR and arg nodes.
1000 // type - the type of value that the caller expects to be popped off the stack.
1001 // expectAddr - if true indicates we are expecting type stack entry to be a TYP_BYREF.
1004 // If the popped value is a struct, and the expected type is a simd type, it will be set
1005 // to that type, otherwise it will assert if the type being popped is not the expected type.
1007 GenTree* Compiler::impSIMDPopStack(var_types type, bool expectAddr)
1009 StackEntry se = impPopStack();
1010 typeInfo ti = se.seTypeInfo;
1011 GenTree* tree = se.val;
1013 // If expectAddr is true implies what we have on stack is address and we need
1014 // SIMD type struct that it points to.
1017 assert(tree->TypeGet() == TYP_BYREF);
1018 if (tree->OperGet() == GT_ADDR)
1020 tree = tree->gtGetOp1();
1024 tree = gtNewOperNode(GT_IND, type, tree);
1028 bool isParam = false;
1030 // If we have a ldobj of a SIMD local we need to transform it.
1031 if (tree->OperGet() == GT_OBJ)
1033 GenTree* addr = tree->gtOp.gtOp1;
1034 if ((addr->OperGet() == GT_ADDR) && isSIMDTypeLocal(addr->gtOp.gtOp1))
1036 tree = addr->gtOp.gtOp1;
1040 if (tree->OperGet() == GT_LCL_VAR)
1042 unsigned lclNum = tree->AsLclVarCommon()->GetLclNum();
1043 LclVarDsc* lclVarDsc = &lvaTable[lclNum];
1044 isParam = lclVarDsc->lvIsParam;
1047 // normalize TYP_STRUCT value
1048 if (varTypeIsStruct(tree) && ((tree->OperGet() == GT_RET_EXPR) || (tree->OperGet() == GT_CALL) || isParam))
1050 assert(ti.IsType(TI_STRUCT));
1051 CORINFO_CLASS_HANDLE structType = ti.GetClassHandleForValueClass();
1052 tree = impNormStructVal(tree, structType, (unsigned)CHECK_SPILL_ALL);
1055 // Now set the type of the tree to the specialized SIMD struct type, if applicable.
1056 if (genActualType(tree->gtType) != genActualType(type))
1058 assert(tree->gtType == TYP_STRUCT);
1059 tree->gtType = type;
1061 else if (tree->gtType == TYP_BYREF)
1063 assert(tree->IsLocal() || (tree->OperGet() == GT_RET_EXPR) || (tree->OperGet() == GT_CALL) ||
1064 ((tree->gtOper == GT_ADDR) && varTypeIsSIMD(tree->gtGetOp1())));
1070 // impSIMDGetFixed: Create a GT_SIMD tree for a Get property of SIMD vector with a fixed index.
1073 // baseType - The base (element) type of the SIMD vector.
1074 // simdSize - The total size in bytes of the SIMD vector.
1075 // index - The index of the field to get.
1078 // Returns a GT_SIMD node with the SIMDIntrinsicGetItem intrinsic id.
1080 GenTreeSIMD* Compiler::impSIMDGetFixed(var_types simdType, var_types baseType, unsigned simdSize, int index)
1082 assert(simdSize >= ((index + 1) * genTypeSize(baseType)));
1084 // op1 is a SIMD source.
1085 GenTree* op1 = impSIMDPopStack(simdType, true);
1087 GenTree* op2 = gtNewIconNode(index);
1088 GenTreeSIMD* simdTree = gtNewSIMDNode(baseType, op1, op2, SIMDIntrinsicGetItem, baseType, simdSize);
1092 #ifdef _TARGET_XARCH_
1093 // impSIMDLongRelOpEqual: transforms operands and returns the SIMD intrinsic to be applied on
1094 // transformed operands to obtain == comparison result.
1097 // typeHnd - type handle of SIMD vector
1098 // size - SIMD vector size
1099 // op1 - in-out parameter; first operand
1100 // op2 - in-out parameter; second operand
1103 // Modifies in-out params op1, op2 and returns intrinsic ID to be applied to modified operands
1105 SIMDIntrinsicID Compiler::impSIMDLongRelOpEqual(CORINFO_CLASS_HANDLE typeHnd,
1110 var_types simdType = (*pOp1)->TypeGet();
1111 assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1113 // There is no direct SSE2 support for comparing TYP_LONG vectors.
1114 // These have to be implemented in terms of TYP_INT vector comparison operations.
1116 // Equality(v1, v2):
1117 // tmp = (v1 == v2) i.e. compare for equality as if v1 and v2 are vector<int>
1118 // result = BitwiseAnd(t, shuffle(t, (2, 3, 0, 1)))
1119 // Shuffle is meant to swap the comparison results of low-32-bits and high 32-bits of respective long elements.
1121 // Compare vector<long> as if they were vector<int> and assign the result to a temp
1122 GenTree* compResult = gtNewSIMDNode(simdType, *pOp1, *pOp2, SIMDIntrinsicEqual, TYP_INT, size);
1123 unsigned lclNum = lvaGrabTemp(true DEBUGARG("SIMD Long =="));
1124 lvaSetStruct(lclNum, typeHnd, false);
1125 GenTree* tmp = gtNewLclvNode(lclNum, simdType);
1126 GenTree* asg = gtNewTempAssign(lclNum, compResult);
1128 // op1 = GT_COMMA(tmp=compResult, tmp)
1129 // op2 = Shuffle(tmp, 0xB1)
1130 // IntrinsicId = BitwiseAnd
1131 *pOp1 = gtNewOperNode(GT_COMMA, simdType, asg, tmp);
1132 *pOp2 = gtNewSIMDNode(simdType, gtNewLclvNode(lclNum, simdType), gtNewIconNode(SHUFFLE_ZWXY, TYP_INT),
1133 SIMDIntrinsicShuffleSSE2, TYP_INT, size);
1134 return SIMDIntrinsicBitwiseAnd;
1137 // impSIMDLongRelOpGreaterThan: transforms operands and returns the SIMD intrinsic to be applied on
1138 // transformed operands to obtain > comparison result.
1141 // typeHnd - type handle of SIMD vector
1142 // size - SIMD vector size
1143 // pOp1 - in-out parameter; first operand
1144 // pOp2 - in-out parameter; second operand
1147 // Modifies in-out params pOp1, pOp2 and returns intrinsic ID to be applied to modified operands
1149 SIMDIntrinsicID Compiler::impSIMDLongRelOpGreaterThan(CORINFO_CLASS_HANDLE typeHnd,
1154 var_types simdType = (*pOp1)->TypeGet();
1155 assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1157 // GreaterThan(v1, v2) where v1 and v2 are vector long.
1158 // Let us consider the case of single long element comparison.
1159 // say L1 = (x1, y1) and L2 = (x2, y2) where x1, y1, x2, and y2 are 32-bit integers that comprise the longs L1 and
1162 // GreaterThan(L1, L2) can be expressed in terms of > relationship between 32-bit integers that comprise L1 and L2
1164 // = (x1, y1) > (x2, y2)
1165 // = (x1 > x2) || [(x1 == x2) && (y1 > y2)] - eq (1)
1167 // t = (v1 > v2) 32-bit signed comparison
1168 // u = (v1 == v2) 32-bit sized element equality
1169 // v = (v1 > v2) 32-bit unsigned comparison
1171 // z = shuffle(t, (3, 3, 1, 1)) - This corresponds to (x1 > x2) in eq(1) above
1172 // t1 = Shuffle(v, (2, 2, 0, 0)) - This corresponds to (y1 > y2) in eq(1) above
1173 // u1 = Shuffle(u, (3, 3, 1, 1)) - This corresponds to (x1 == x2) in eq(1) above
1174 // w = And(t1, u1) - This corresponds to [(x1 == x2) && (y1 > y2)] in eq(1) above
1175 // Result = BitwiseOr(z, w)
1177 // Since op1 and op2 gets used multiple times, make sure side effects are computed.
1178 GenTree* dupOp1 = nullptr;
1179 GenTree* dupOp2 = nullptr;
1180 GenTree* dupDupOp1 = nullptr;
1181 GenTree* dupDupOp2 = nullptr;
1183 if (((*pOp1)->gtFlags & GTF_SIDE_EFFECT) != 0)
1185 dupOp1 = fgInsertCommaFormTemp(pOp1, typeHnd);
1186 dupDupOp1 = gtNewLclvNode(dupOp1->AsLclVarCommon()->GetLclNum(), simdType);
1190 dupOp1 = gtCloneExpr(*pOp1);
1191 dupDupOp1 = gtCloneExpr(*pOp1);
1194 if (((*pOp2)->gtFlags & GTF_SIDE_EFFECT) != 0)
1196 dupOp2 = fgInsertCommaFormTemp(pOp2, typeHnd);
1197 dupDupOp2 = gtNewLclvNode(dupOp2->AsLclVarCommon()->GetLclNum(), simdType);
1201 dupOp2 = gtCloneExpr(*pOp2);
1202 dupDupOp2 = gtCloneExpr(*pOp2);
1205 assert(dupDupOp1 != nullptr && dupDupOp2 != nullptr);
1206 assert(dupOp1 != nullptr && dupOp2 != nullptr);
1207 assert(*pOp1 != nullptr && *pOp2 != nullptr);
1209 // v1GreaterThanv2Signed - signed 32-bit comparison
1210 GenTree* v1GreaterThanv2Signed = gtNewSIMDNode(simdType, *pOp1, *pOp2, SIMDIntrinsicGreaterThan, TYP_INT, size);
1212 // v1Equalsv2 - 32-bit equality
1213 GenTree* v1Equalsv2 = gtNewSIMDNode(simdType, dupOp1, dupOp2, SIMDIntrinsicEqual, TYP_INT, size);
1215 // v1GreaterThanv2Unsigned - unsigned 32-bit comparison
1216 var_types tempBaseType = TYP_UINT;
1217 SIMDIntrinsicID sid = impSIMDRelOp(SIMDIntrinsicGreaterThan, typeHnd, size, &tempBaseType, &dupDupOp1, &dupDupOp2);
1218 GenTree* v1GreaterThanv2Unsigned = gtNewSIMDNode(simdType, dupDupOp1, dupDupOp2, sid, tempBaseType, size);
1220 GenTree* z = gtNewSIMDNode(simdType, v1GreaterThanv2Signed, gtNewIconNode(SHUFFLE_WWYY, TYP_INT),
1221 SIMDIntrinsicShuffleSSE2, TYP_FLOAT, size);
1222 GenTree* t1 = gtNewSIMDNode(simdType, v1GreaterThanv2Unsigned, gtNewIconNode(SHUFFLE_ZZXX, TYP_INT),
1223 SIMDIntrinsicShuffleSSE2, TYP_FLOAT, size);
1224 GenTree* u1 = gtNewSIMDNode(simdType, v1Equalsv2, gtNewIconNode(SHUFFLE_WWYY, TYP_INT), SIMDIntrinsicShuffleSSE2,
1226 GenTree* w = gtNewSIMDNode(simdType, u1, t1, SIMDIntrinsicBitwiseAnd, TYP_INT, size);
1230 return SIMDIntrinsicBitwiseOr;
1233 // impSIMDLongRelOpGreaterThanOrEqual: transforms operands and returns the SIMD intrinsic to be applied on
1234 // transformed operands to obtain >= comparison result.
1237 // typeHnd - type handle of SIMD vector
1238 // size - SIMD vector size
1239 // pOp1 - in-out parameter; first operand
1240 // pOp2 - in-out parameter; second operand
1243 // Modifies in-out params pOp1, pOp2 and returns intrinsic ID to be applied to modified operands
1245 SIMDIntrinsicID Compiler::impSIMDLongRelOpGreaterThanOrEqual(CORINFO_CLASS_HANDLE typeHnd,
1250 var_types simdType = (*pOp1)->TypeGet();
1251 assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1253 // expand this to (a == b) | (a > b)
1254 GenTree* dupOp1 = nullptr;
1255 GenTree* dupOp2 = nullptr;
1257 if (((*pOp1)->gtFlags & GTF_SIDE_EFFECT) != 0)
1259 dupOp1 = fgInsertCommaFormTemp(pOp1, typeHnd);
1263 dupOp1 = gtCloneExpr(*pOp1);
1266 if (((*pOp2)->gtFlags & GTF_SIDE_EFFECT) != 0)
1268 dupOp2 = fgInsertCommaFormTemp(pOp2, typeHnd);
1272 dupOp2 = gtCloneExpr(*pOp2);
1275 assert(dupOp1 != nullptr && dupOp2 != nullptr);
1276 assert(*pOp1 != nullptr && *pOp2 != nullptr);
1279 SIMDIntrinsicID id = impSIMDLongRelOpEqual(typeHnd, size, pOp1, pOp2);
1280 *pOp1 = gtNewSIMDNode(simdType, *pOp1, *pOp2, id, TYP_LONG, size);
1283 id = impSIMDLongRelOpGreaterThan(typeHnd, size, &dupOp1, &dupOp2);
1284 *pOp2 = gtNewSIMDNode(simdType, dupOp1, dupOp2, id, TYP_LONG, size);
1286 return SIMDIntrinsicBitwiseOr;
1289 // impSIMDInt32OrSmallIntRelOpGreaterThanOrEqual: transforms operands and returns the SIMD intrinsic to be applied on
1290 // transformed operands to obtain >= comparison result in case of integer base type vectors
1293 // typeHnd - type handle of SIMD vector
1294 // size - SIMD vector size
1295 // baseType - base type of SIMD vector
1296 // pOp1 - in-out parameter; first operand
1297 // pOp2 - in-out parameter; second operand
1300 // Modifies in-out params pOp1, pOp2 and returns intrinsic ID to be applied to modified operands
1302 SIMDIntrinsicID Compiler::impSIMDIntegralRelOpGreaterThanOrEqual(
1303 CORINFO_CLASS_HANDLE typeHnd, unsigned size, var_types baseType, GenTree** pOp1, GenTree** pOp2)
1305 var_types simdType = (*pOp1)->TypeGet();
1306 assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1308 // This routine should be used only for integer base type vectors
1309 assert(varTypeIsIntegral(baseType));
1310 if ((getSIMDSupportLevel() == SIMD_SSE2_Supported) && ((baseType == TYP_LONG) || baseType == TYP_UBYTE))
1312 return impSIMDLongRelOpGreaterThanOrEqual(typeHnd, size, pOp1, pOp2);
1315 // expand this to (a == b) | (a > b)
1316 GenTree* dupOp1 = nullptr;
1317 GenTree* dupOp2 = nullptr;
1319 if (((*pOp1)->gtFlags & GTF_SIDE_EFFECT) != 0)
1321 dupOp1 = fgInsertCommaFormTemp(pOp1, typeHnd);
1325 dupOp1 = gtCloneExpr(*pOp1);
1328 if (((*pOp2)->gtFlags & GTF_SIDE_EFFECT) != 0)
1330 dupOp2 = fgInsertCommaFormTemp(pOp2, typeHnd);
1334 dupOp2 = gtCloneExpr(*pOp2);
1337 assert(dupOp1 != nullptr && dupOp2 != nullptr);
1338 assert(*pOp1 != nullptr && *pOp2 != nullptr);
1341 *pOp1 = gtNewSIMDNode(simdType, *pOp1, *pOp2, SIMDIntrinsicEqual, baseType, size);
1344 *pOp2 = gtNewSIMDNode(simdType, dupOp1, dupOp2, SIMDIntrinsicGreaterThan, baseType, size);
1346 return SIMDIntrinsicBitwiseOr;
1348 #endif // _TARGET_XARCH_
1350 // Transforms operands and returns the SIMD intrinsic to be applied on
1351 // transformed operands to obtain given relop result.
1354 // relOpIntrinsicId - Relational operator SIMD intrinsic
1355 // typeHnd - type handle of SIMD vector
1356 // size - SIMD vector size
1357 // inOutBaseType - base type of SIMD vector
1358 // pOp1 - in-out parameter; first operand
1359 // pOp2 - in-out parameter; second operand
1362 // Modifies in-out params pOp1, pOp2, inOutBaseType and returns intrinsic ID to be applied to modified operands
1364 SIMDIntrinsicID Compiler::impSIMDRelOp(SIMDIntrinsicID relOpIntrinsicId,
1365 CORINFO_CLASS_HANDLE typeHnd,
1367 var_types* inOutBaseType,
1371 var_types simdType = (*pOp1)->TypeGet();
1372 assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1374 assert(isRelOpSIMDIntrinsic(relOpIntrinsicId));
1376 SIMDIntrinsicID intrinsicID = relOpIntrinsicId;
1377 #ifdef _TARGET_XARCH_
1378 var_types baseType = *inOutBaseType;
1380 if (varTypeIsFloating(baseType))
1382 // SSE2/AVX doesn't support > and >= on vector float/double.
1383 // Therefore, we need to use < and <= with swapped operands
1384 if (relOpIntrinsicId == SIMDIntrinsicGreaterThan || relOpIntrinsicId == SIMDIntrinsicGreaterThanOrEqual)
1386 GenTree* tmp = *pOp1;
1391 (relOpIntrinsicId == SIMDIntrinsicGreaterThan) ? SIMDIntrinsicLessThan : SIMDIntrinsicLessThanOrEqual;
1394 else if (varTypeIsIntegral(baseType))
1396 // SSE/AVX doesn't support < and <= on integer base type vectors.
1397 // Therefore, we need to use > and >= with swapped operands.
1398 if (intrinsicID == SIMDIntrinsicLessThan || intrinsicID == SIMDIntrinsicLessThanOrEqual)
1400 GenTree* tmp = *pOp1;
1404 intrinsicID = (relOpIntrinsicId == SIMDIntrinsicLessThan) ? SIMDIntrinsicGreaterThan
1405 : SIMDIntrinsicGreaterThanOrEqual;
1408 if ((getSIMDSupportLevel() == SIMD_SSE2_Supported) && baseType == TYP_LONG)
1410 // There is no direct SSE2 support for comparing TYP_LONG vectors.
1411 // These have to be implemented interms of TYP_INT vector comparison operations.
1412 if (intrinsicID == SIMDIntrinsicEqual)
1414 intrinsicID = impSIMDLongRelOpEqual(typeHnd, size, pOp1, pOp2);
1416 else if (intrinsicID == SIMDIntrinsicGreaterThan)
1418 intrinsicID = impSIMDLongRelOpGreaterThan(typeHnd, size, pOp1, pOp2);
1420 else if (intrinsicID == SIMDIntrinsicGreaterThanOrEqual)
1422 intrinsicID = impSIMDLongRelOpGreaterThanOrEqual(typeHnd, size, pOp1, pOp2);
1429 // SSE2 and AVX direct support for signed comparison of int32, int16 and int8 types
1430 else if (!varTypeIsUnsigned(baseType))
1432 if (intrinsicID == SIMDIntrinsicGreaterThanOrEqual)
1434 intrinsicID = impSIMDIntegralRelOpGreaterThanOrEqual(typeHnd, size, baseType, pOp1, pOp2);
1439 // Vector<byte>, Vector<ushort>, Vector<uint> and Vector<ulong>:
1440 // SSE2 supports > for signed comparison. Therefore, to use it for
1441 // comparing unsigned numbers, we subtract a constant from both the
1442 // operands such that the result fits within the corresponding signed
1443 // type. The resulting signed numbers are compared using SSE2 signed
1446 // Vector<byte>: constant to be subtracted is 2^7
1447 // Vector<ushort> constant to be subtracted is 2^15
1448 // Vector<uint> constant to be subtracted is 2^31
1449 // Vector<ulong> constant to be subtracted is 2^63
1451 // We need to treat op1 and op2 as signed for comparison purpose after
1452 // the transformation.
1453 __int64 constVal = 0;
1457 constVal = 0x80808080;
1458 *inOutBaseType = TYP_BYTE;
1461 constVal = 0x80008000;
1462 *inOutBaseType = TYP_SHORT;
1465 constVal = 0x80000000;
1466 *inOutBaseType = TYP_INT;
1469 constVal = 0x8000000000000000LL;
1470 *inOutBaseType = TYP_LONG;
1476 assert(constVal != 0);
1478 // This transformation is not required for equality.
1479 if (intrinsicID != SIMDIntrinsicEqual)
1481 // For constructing const vector use either long or int base type.
1482 var_types tempBaseType;
1484 if (baseType == TYP_ULONG)
1486 tempBaseType = TYP_LONG;
1487 initVal = gtNewLconNode(constVal);
1491 tempBaseType = TYP_INT;
1492 initVal = gtNewIconNode((ssize_t)constVal);
1494 initVal->gtType = tempBaseType;
1495 GenTree* constVector = gtNewSIMDNode(simdType, initVal, nullptr, SIMDIntrinsicInit, tempBaseType, size);
1497 // Assign constVector to a temp, since we intend to use it more than once
1498 // TODO-CQ: We have quite a few such constant vectors constructed during
1499 // the importation of SIMD intrinsics. Make sure that we have a single
1500 // temp per distinct constant per method.
1501 GenTree* tmp = fgInsertCommaFormTemp(&constVector, typeHnd);
1503 // op1 = op1 - constVector
1504 // op2 = op2 - constVector
1505 *pOp1 = gtNewSIMDNode(simdType, *pOp1, constVector, SIMDIntrinsicSub, baseType, size);
1506 *pOp2 = gtNewSIMDNode(simdType, *pOp2, tmp, SIMDIntrinsicSub, baseType, size);
1509 return impSIMDRelOp(intrinsicID, typeHnd, size, inOutBaseType, pOp1, pOp2);
1512 #elif defined(_TARGET_ARM64_)
1513 // TODO-ARM64-CQ handle comparisons against zero
1515 // _TARGET_ARM64_ doesn't support < and <= on register register comparisons
1516 // Therefore, we need to use > and >= with swapped operands.
1517 if (intrinsicID == SIMDIntrinsicLessThan || intrinsicID == SIMDIntrinsicLessThanOrEqual)
1519 GenTree* tmp = *pOp1;
1524 (intrinsicID == SIMDIntrinsicLessThan) ? SIMDIntrinsicGreaterThan : SIMDIntrinsicGreaterThanOrEqual;
1526 #else // !_TARGET_XARCH_
1527 assert(!"impSIMDRelOp() unimplemented on target arch");
1529 #endif // !_TARGET_XARCH_
1534 //-------------------------------------------------------------------------
1535 // impSIMDAbs: creates GT_SIMD node to compute Abs value of a given vector.
1538 // typeHnd - type handle of SIMD vector
1539 // baseType - base type of vector
1540 // size - vector size in bytes
1541 // op1 - operand of Abs intrinsic
1543 GenTree* Compiler::impSIMDAbs(CORINFO_CLASS_HANDLE typeHnd, var_types baseType, unsigned size, GenTree* op1)
1545 assert(varTypeIsSIMD(op1));
1547 var_types simdType = op1->TypeGet();
1548 GenTree* retVal = nullptr;
1550 #ifdef _TARGET_XARCH_
1551 // When there is no direct support, Abs(v) could be computed
1552 // on integer vectors as follows:
1553 // BitVector = v < vector.Zero
1554 // result = ConditionalSelect(BitVector, vector.Zero - v, v)
1556 bool useConditionalSelect = false;
1557 if (getSIMDSupportLevel() == SIMD_SSE2_Supported)
1559 // SSE2 doesn't support abs on signed integer type vectors.
1560 if (baseType == TYP_LONG || baseType == TYP_INT || baseType == TYP_SHORT || baseType == TYP_BYTE)
1562 useConditionalSelect = true;
1567 assert(getSIMDSupportLevel() >= SIMD_SSE4_Supported);
1568 if (baseType == TYP_LONG)
1570 // SSE4/AVX2 don't support abs on long type vector.
1571 useConditionalSelect = true;
1575 if (useConditionalSelect)
1577 // This works only on integer vectors not on float/double vectors.
1578 assert(varTypeIsIntegral(baseType));
1583 if (op1->OperGet() == GT_LCL_VAR)
1585 op1LclNum = op1->gtLclVarCommon.gtLclNum;
1586 op1Assign = nullptr;
1590 op1LclNum = lvaGrabTemp(true DEBUGARG("SIMD Abs op1"));
1591 lvaSetStruct(op1LclNum, typeHnd, false);
1592 op1Assign = gtNewTempAssign(op1LclNum, op1);
1593 op1 = gtNewLclvNode(op1LclNum, op1->TypeGet());
1596 // Assign Vector.Zero to a temp since it is needed more than once
1597 GenTree* vecZero = gtNewSIMDVectorZero(simdType, baseType, size);
1598 unsigned vecZeroLclNum = lvaGrabTemp(true DEBUGARG("SIMD Abs VecZero"));
1599 lvaSetStruct(vecZeroLclNum, typeHnd, false);
1600 GenTree* vecZeroAssign = gtNewTempAssign(vecZeroLclNum, vecZero);
1602 // Construct BitVector = v < vector.Zero
1603 GenTree* bitVecOp1 = op1;
1604 GenTree* bitVecOp2 = gtNewLclvNode(vecZeroLclNum, vecZero->TypeGet());
1605 var_types relOpBaseType = baseType;
1606 SIMDIntrinsicID relOpIntrinsic =
1607 impSIMDRelOp(SIMDIntrinsicLessThan, typeHnd, size, &relOpBaseType, &bitVecOp1, &bitVecOp2);
1608 GenTree* bitVec = gtNewSIMDNode(simdType, bitVecOp1, bitVecOp2, relOpIntrinsic, relOpBaseType, size);
1609 unsigned bitVecLclNum = lvaGrabTemp(true DEBUGARG("SIMD Abs bitVec"));
1610 lvaSetStruct(bitVecLclNum, typeHnd, false);
1611 GenTree* bitVecAssign = gtNewTempAssign(bitVecLclNum, bitVec);
1612 bitVec = gtNewLclvNode(bitVecLclNum, bitVec->TypeGet());
1614 // Construct condSelectOp1 = vector.Zero - v
1615 GenTree* subOp1 = gtNewLclvNode(vecZeroLclNum, vecZero->TypeGet());
1616 GenTree* subOp2 = gtNewLclvNode(op1LclNum, op1->TypeGet());
1617 GenTree* negVec = gtNewSIMDNode(simdType, subOp1, subOp2, SIMDIntrinsicSub, baseType, size);
1619 // Construct ConditionalSelect(bitVec, vector.Zero - v, v)
1620 GenTree* vec = gtNewLclvNode(op1LclNum, op1->TypeGet());
1621 retVal = impSIMDSelect(typeHnd, baseType, size, bitVec, negVec, vec);
1623 // Prepend bitVec assignment to retVal.
1624 // retVal = (tmp2 = v < tmp1), CondSelect(tmp2, tmp1 - v, v)
1625 retVal = gtNewOperNode(GT_COMMA, simdType, bitVecAssign, retVal);
1627 // Prepend vecZero assignment to retVal.
1628 // retVal = (tmp1 = vector.Zero), (tmp2 = v < tmp1), CondSelect(tmp2, tmp1 - v, v)
1629 retVal = gtNewOperNode(GT_COMMA, simdType, vecZeroAssign, retVal);
1631 // If op1 was assigned to a temp, prepend that to retVal.
1632 if (op1Assign != nullptr)
1634 // retVal = (v=op1), (tmp1 = vector.Zero), (tmp2 = v < tmp1), CondSelect(tmp2, tmp1 - v, v)
1635 retVal = gtNewOperNode(GT_COMMA, simdType, op1Assign, retVal);
1638 else if (varTypeIsFloating(baseType))
1640 // Abs(vf) = vf & new SIMDVector<float>(0x7fffffff);
1641 // Abs(vd) = vf & new SIMDVector<double>(0x7fffffffffffffff);
1642 GenTree* bitMask = nullptr;
1643 if (baseType == TYP_FLOAT)
1646 static_assert_no_msg(sizeof(float) == sizeof(int));
1647 *((int*)&f) = 0x7fffffff;
1648 bitMask = gtNewDconNode(f);
1650 else if (baseType == TYP_DOUBLE)
1653 static_assert_no_msg(sizeof(double) == sizeof(__int64));
1654 *((__int64*)&d) = 0x7fffffffffffffffLL;
1655 bitMask = gtNewDconNode(d);
1658 assert(bitMask != nullptr);
1659 bitMask->gtType = baseType;
1660 GenTree* bitMaskVector = gtNewSIMDNode(simdType, bitMask, SIMDIntrinsicInit, baseType, size);
1661 retVal = gtNewSIMDNode(simdType, op1, bitMaskVector, SIMDIntrinsicBitwiseAnd, baseType, size);
1663 else if (baseType == TYP_USHORT || baseType == TYP_UBYTE || baseType == TYP_UINT || baseType == TYP_ULONG)
1665 // Abs is a no-op on unsigned integer type vectors
1670 assert(getSIMDSupportLevel() >= SIMD_SSE4_Supported);
1671 assert(baseType != TYP_LONG);
1673 retVal = gtNewSIMDNode(simdType, op1, SIMDIntrinsicAbs, baseType, size);
1675 #elif defined(_TARGET_ARM64_)
1676 if (varTypeIsUnsigned(baseType))
1678 // Abs is a no-op on unsigned integer type vectors
1683 retVal = gtNewSIMDNode(simdType, op1, SIMDIntrinsicAbs, baseType, size);
1685 #else // !defined(_TARGET_XARCH)_ && !defined(_TARGET_ARM64_)
1686 assert(!"Abs intrinsic on non-xarch target not implemented");
1687 #endif // !_TARGET_XARCH_
1692 // Creates a GT_SIMD tree for Select operation
1695 // typeHnd - type handle of SIMD vector
1696 // baseType - base type of SIMD vector
1697 // size - SIMD vector size
1698 // op1 - first operand = Condition vector vc
1699 // op2 - second operand = va
1700 // op3 - third operand = vb
1703 // Returns GT_SIMD tree that computes Select(vc, va, vb)
1705 GenTree* Compiler::impSIMDSelect(
1706 CORINFO_CLASS_HANDLE typeHnd, var_types baseType, unsigned size, GenTree* op1, GenTree* op2, GenTree* op3)
1708 assert(varTypeIsSIMD(op1));
1709 var_types simdType = op1->TypeGet();
1710 assert(op2->TypeGet() == simdType);
1711 assert(op3->TypeGet() == simdType);
1713 // TODO-ARM64-CQ Support generating select instruction for SIMD
1715 // Select(BitVector vc, va, vb) = (va & vc) | (vb & !vc)
1716 // Select(op1, op2, op3) = (op2 & op1) | (op3 & !op1)
1717 // = SIMDIntrinsicBitwiseOr(SIMDIntrinsicBitwiseAnd(op2, op1),
1718 // SIMDIntrinsicBitwiseAndNot(op3, op1))
1720 // If Op1 has side effect, create an assignment to a temp
1722 GenTree* asg = nullptr;
1723 if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0)
1725 unsigned lclNum = lvaGrabTemp(true DEBUGARG("SIMD Select"));
1726 lvaSetStruct(lclNum, typeHnd, false);
1727 tmp = gtNewLclvNode(lclNum, op1->TypeGet());
1728 asg = gtNewTempAssign(lclNum, op1);
1731 GenTree* andExpr = gtNewSIMDNode(simdType, op2, tmp, SIMDIntrinsicBitwiseAnd, baseType, size);
1732 GenTree* dupOp1 = gtCloneExpr(tmp);
1733 assert(dupOp1 != nullptr);
1734 #ifdef _TARGET_ARM64_
1735 // ARM64 implements SIMDIntrinsicBitwiseAndNot as Left & ~Right
1736 GenTree* andNotExpr = gtNewSIMDNode(simdType, op3, dupOp1, SIMDIntrinsicBitwiseAndNot, baseType, size);
1738 // XARCH implements SIMDIntrinsicBitwiseAndNot as ~Left & Right
1739 GenTree* andNotExpr = gtNewSIMDNode(simdType, dupOp1, op3, SIMDIntrinsicBitwiseAndNot, baseType, size);
1741 GenTree* simdTree = gtNewSIMDNode(simdType, andExpr, andNotExpr, SIMDIntrinsicBitwiseOr, baseType, size);
1743 // If asg not null, create a GT_COMMA tree.
1746 simdTree = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), asg, simdTree);
1752 // Creates a GT_SIMD tree for Min/Max operation
1755 // IntrinsicId - SIMD intrinsic Id, either Min or Max
1756 // typeHnd - type handle of SIMD vector
1757 // baseType - base type of SIMD vector
1758 // size - SIMD vector size
1759 // op1 - first operand = va
1760 // op2 - second operand = vb
1763 // Returns GT_SIMD tree that computes Max(va, vb)
1765 GenTree* Compiler::impSIMDMinMax(SIMDIntrinsicID intrinsicId,
1766 CORINFO_CLASS_HANDLE typeHnd,
1772 assert(intrinsicId == SIMDIntrinsicMin || intrinsicId == SIMDIntrinsicMax);
1773 assert(varTypeIsSIMD(op1));
1774 var_types simdType = op1->TypeGet();
1775 assert(op2->TypeGet() == simdType);
1777 #if defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
1778 GenTree* simdTree = nullptr;
1780 #ifdef _TARGET_XARCH_
1781 // SSE2 has direct support for float/double/signed word/unsigned byte.
1782 // SSE4.1 has direct support for int32/uint32/signed byte/unsigned word.
1783 // For other integer types we compute min/max as follows
1785 // int32/uint32 (SSE2)
1786 // int64/uint64 (SSE2&SSE4):
1787 // compResult = (op1 < op2) in case of Min
1788 // (op1 > op2) in case of Max
1789 // Min/Max(op1, op2) = Select(compResult, op1, op2)
1791 // unsigned word (SSE2):
1792 // op1 = op1 - 2^15 ; to make it fit within a signed word
1793 // op2 = op2 - 2^15 ; to make it fit within a signed word
1794 // result = SSE2 signed word Min/Max(op1, op2)
1795 // result = result + 2^15 ; readjust it back
1797 // signed byte (SSE2):
1798 // op1 = op1 + 2^7 ; to make it unsigned
1799 // op1 = op1 + 2^7 ; to make it unsigned
1800 // result = SSE2 unsigned byte Min/Max(op1, op2)
1801 // result = result - 2^15 ; readjust it back
1803 if (varTypeIsFloating(baseType) || baseType == TYP_SHORT || baseType == TYP_UBYTE ||
1804 (getSIMDSupportLevel() >= SIMD_SSE4_Supported &&
1805 (baseType == TYP_BYTE || baseType == TYP_INT || baseType == TYP_UINT || baseType == TYP_USHORT)))
1807 // SSE2 or SSE4.1 has direct support
1808 simdTree = gtNewSIMDNode(simdType, op1, op2, intrinsicId, baseType, size);
1810 else if (baseType == TYP_USHORT || baseType == TYP_BYTE)
1812 assert(getSIMDSupportLevel() == SIMD_SSE2_Supported);
1814 SIMDIntrinsicID operIntrinsic;
1815 SIMDIntrinsicID adjustIntrinsic;
1816 var_types minMaxOperBaseType;
1817 if (baseType == TYP_USHORT)
1819 constVal = 0x80008000;
1820 operIntrinsic = SIMDIntrinsicSub;
1821 adjustIntrinsic = SIMDIntrinsicAdd;
1822 minMaxOperBaseType = TYP_SHORT;
1826 assert(baseType == TYP_BYTE);
1827 constVal = 0x80808080;
1828 operIntrinsic = SIMDIntrinsicAdd;
1829 adjustIntrinsic = SIMDIntrinsicSub;
1830 minMaxOperBaseType = TYP_UBYTE;
1833 GenTree* initVal = gtNewIconNode(constVal);
1834 GenTree* constVector = gtNewSIMDNode(simdType, initVal, nullptr, SIMDIntrinsicInit, TYP_INT, size);
1836 // Assign constVector to a temp, since we intend to use it more than once
1837 // TODO-CQ: We have quite a few such constant vectors constructed during
1838 // the importation of SIMD intrinsics. Make sure that we have a single
1839 // temp per distinct constant per method.
1840 GenTree* tmp = fgInsertCommaFormTemp(&constVector, typeHnd);
1842 // op1 = op1 - constVector
1843 // op2 = op2 - constVector
1844 op1 = gtNewSIMDNode(simdType, op1, constVector, operIntrinsic, baseType, size);
1845 op2 = gtNewSIMDNode(simdType, op2, tmp, operIntrinsic, baseType, size);
1847 // compute min/max of op1 and op2 considering them as if minMaxOperBaseType
1848 simdTree = gtNewSIMDNode(simdType, op1, op2, intrinsicId, minMaxOperBaseType, size);
1850 // re-adjust the value by adding or subtracting constVector
1851 tmp = gtNewLclvNode(tmp->AsLclVarCommon()->GetLclNum(), tmp->TypeGet());
1852 simdTree = gtNewSIMDNode(simdType, simdTree, tmp, adjustIntrinsic, baseType, size);
1854 #elif defined(_TARGET_ARM64_)
1855 // Arm64 has direct support for all types except int64/uint64
1856 // For which we compute min/max as follows
1859 // compResult = (op1 < op2) in case of Min
1860 // (op1 > op2) in case of Max
1861 // Min/Max(op1, op2) = Select(compResult, op1, op2)
1862 if (baseType != TYP_ULONG && baseType != TYP_LONG)
1864 simdTree = gtNewSIMDNode(simdType, op1, op2, intrinsicId, baseType, size);
1869 GenTree* dupOp1 = nullptr;
1870 GenTree* dupOp2 = nullptr;
1871 GenTree* op1Assign = nullptr;
1872 GenTree* op2Assign = nullptr;
1876 if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0)
1878 op1LclNum = lvaGrabTemp(true DEBUGARG("SIMD Min/Max"));
1879 dupOp1 = gtNewLclvNode(op1LclNum, op1->TypeGet());
1880 lvaSetStruct(op1LclNum, typeHnd, false);
1881 op1Assign = gtNewTempAssign(op1LclNum, op1);
1882 op1 = gtNewLclvNode(op1LclNum, op1->TypeGet());
1886 dupOp1 = gtCloneExpr(op1);
1889 if ((op2->gtFlags & GTF_SIDE_EFFECT) != 0)
1891 op2LclNum = lvaGrabTemp(true DEBUGARG("SIMD Min/Max"));
1892 dupOp2 = gtNewLclvNode(op2LclNum, op2->TypeGet());
1893 lvaSetStruct(op2LclNum, typeHnd, false);
1894 op2Assign = gtNewTempAssign(op2LclNum, op2);
1895 op2 = gtNewLclvNode(op2LclNum, op2->TypeGet());
1899 dupOp2 = gtCloneExpr(op2);
1902 SIMDIntrinsicID relOpIntrinsic =
1903 (intrinsicId == SIMDIntrinsicMin) ? SIMDIntrinsicLessThan : SIMDIntrinsicGreaterThan;
1904 var_types relOpBaseType = baseType;
1906 // compResult = op1 relOp op2
1907 // simdTree = Select(compResult, op1, op2);
1908 assert(dupOp1 != nullptr);
1909 assert(dupOp2 != nullptr);
1910 relOpIntrinsic = impSIMDRelOp(relOpIntrinsic, typeHnd, size, &relOpBaseType, &dupOp1, &dupOp2);
1911 GenTree* compResult = gtNewSIMDNode(simdType, dupOp1, dupOp2, relOpIntrinsic, relOpBaseType, size);
1912 unsigned compResultLclNum = lvaGrabTemp(true DEBUGARG("SIMD Min/Max"));
1913 lvaSetStruct(compResultLclNum, typeHnd, false);
1914 GenTree* compResultAssign = gtNewTempAssign(compResultLclNum, compResult);
1915 compResult = gtNewLclvNode(compResultLclNum, compResult->TypeGet());
1916 simdTree = impSIMDSelect(typeHnd, baseType, size, compResult, op1, op2);
1917 simdTree = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), compResultAssign, simdTree);
1919 // Now create comma trees if we have created assignments of op1/op2 to temps
1920 if (op2Assign != nullptr)
1922 simdTree = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), op2Assign, simdTree);
1925 if (op1Assign != nullptr)
1927 simdTree = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), op1Assign, simdTree);
1931 assert(simdTree != nullptr);
1933 #else // !(defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_))
1934 assert(!"impSIMDMinMax() unimplemented on target arch");
1936 #endif // !(defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_))
1939 //------------------------------------------------------------------------
1940 // getOp1ForConstructor: Get the op1 for a constructor call.
1943 // opcode - the opcode being handled (needed to identify the CEE_NEWOBJ case)
1944 // newobjThis - For CEE_NEWOBJ, this is the temp grabbed for the allocated uninitalized object.
1945 // clsHnd - The handle of the class of the method.
1948 // The tree node representing the object to be initialized with the constructor.
1951 // This method handles the differences between the CEE_NEWOBJ and constructor cases.
1953 GenTree* Compiler::getOp1ForConstructor(OPCODE opcode, GenTree* newobjThis, CORINFO_CLASS_HANDLE clsHnd)
1956 if (opcode == CEE_NEWOBJ)
1959 assert(newobjThis->gtOper == GT_ADDR && newobjThis->gtOp.gtOp1->gtOper == GT_LCL_VAR);
1961 // push newobj result on type stack
1962 unsigned tmp = op1->gtOp.gtOp1->gtLclVarCommon.gtLclNum;
1963 impPushOnStack(gtNewLclvNode(tmp, lvaGetRealType(tmp)), verMakeTypeInfo(clsHnd).NormaliseForStack());
1967 op1 = impSIMDPopStack(TYP_BYREF);
1969 assert(op1->TypeGet() == TYP_BYREF);
1973 //-------------------------------------------------------------------
1974 // Set the flag that indicates that the lclVar referenced by this tree
1975 // is used in a SIMD intrinsic.
1979 void Compiler::setLclRelatedToSIMDIntrinsic(GenTree* tree)
1981 assert(tree->OperIsLocal());
1982 unsigned lclNum = tree->AsLclVarCommon()->GetLclNum();
1983 LclVarDsc* lclVarDsc = &lvaTable[lclNum];
1984 lclVarDsc->lvUsedInSIMDIntrinsic = true;
1987 //-------------------------------------------------------------
1988 // Check if two field nodes reference at the same memory location.
1989 // Notice that this check is just based on pattern matching.
1994 // If op1's parents node and op2's parents node are at the same location, return true. Otherwise, return false
1996 bool areFieldsParentsLocatedSame(GenTree* op1, GenTree* op2)
1998 assert(op1->OperGet() == GT_FIELD);
1999 assert(op2->OperGet() == GT_FIELD);
2001 GenTree* op1ObjRef = op1->gtField.gtFldObj;
2002 GenTree* op2ObjRef = op2->gtField.gtFldObj;
2003 while (op1ObjRef != nullptr && op2ObjRef != nullptr)
2006 if (op1ObjRef->OperGet() != op2ObjRef->OperGet())
2010 else if (op1ObjRef->OperGet() == GT_ADDR)
2012 op1ObjRef = op1ObjRef->gtOp.gtOp1;
2013 op2ObjRef = op2ObjRef->gtOp.gtOp1;
2016 if (op1ObjRef->OperIsLocal() && op2ObjRef->OperIsLocal() &&
2017 op1ObjRef->AsLclVarCommon()->GetLclNum() == op2ObjRef->AsLclVarCommon()->GetLclNum())
2021 else if (op1ObjRef->OperGet() == GT_FIELD && op2ObjRef->OperGet() == GT_FIELD &&
2022 op1ObjRef->gtField.gtFldHnd == op2ObjRef->gtField.gtFldHnd)
2024 op1ObjRef = op1ObjRef->gtField.gtFldObj;
2025 op2ObjRef = op2ObjRef->gtField.gtFldObj;
2037 //----------------------------------------------------------------------
2038 // Check whether two field are contiguous
2040 // first - GenTree*. The Type of the node should be TYP_FLOAT
2041 // second - GenTree*. The Type of the node should be TYP_FLOAT
2043 // if the first field is located before second field, and they are located contiguously,
2044 // then return true. Otherwise, return false.
2046 bool Compiler::areFieldsContiguous(GenTree* first, GenTree* second)
2048 assert(first->OperGet() == GT_FIELD);
2049 assert(second->OperGet() == GT_FIELD);
2050 assert(first->gtType == TYP_FLOAT);
2051 assert(second->gtType == TYP_FLOAT);
2053 var_types firstFieldType = first->gtType;
2054 var_types secondFieldType = second->gtType;
2056 unsigned firstFieldEndOffset = first->gtField.gtFldOffset + genTypeSize(firstFieldType);
2057 unsigned secondFieldOffset = second->gtField.gtFldOffset;
2058 if (firstFieldEndOffset == secondFieldOffset && firstFieldType == secondFieldType &&
2059 areFieldsParentsLocatedSame(first, second))
2067 //-------------------------------------------------------------------------------
2068 // Check whether two array element nodes are located contiguously or not.
2073 // if the array element op1 is located before array element op2, and they are contiguous,
2074 // then return true. Otherwise, return false.
2076 // Right this can only check array element with const number as index. In future,
2077 // we should consider to allow this function to check the index using expression.
2079 bool Compiler::areArrayElementsContiguous(GenTree* op1, GenTree* op2)
2081 noway_assert(op1->gtOper == GT_INDEX);
2082 noway_assert(op2->gtOper == GT_INDEX);
2083 GenTreeIndex* op1Index = op1->AsIndex();
2084 GenTreeIndex* op2Index = op2->AsIndex();
2086 GenTree* op1ArrayRef = op1Index->Arr();
2087 GenTree* op2ArrayRef = op2Index->Arr();
2088 assert(op1ArrayRef->TypeGet() == TYP_REF);
2089 assert(op2ArrayRef->TypeGet() == TYP_REF);
2091 GenTree* op1IndexNode = op1Index->Index();
2092 GenTree* op2IndexNode = op2Index->Index();
2093 if ((op1IndexNode->OperGet() == GT_CNS_INT && op2IndexNode->OperGet() == GT_CNS_INT) &&
2094 op1IndexNode->gtIntCon.gtIconVal + 1 == op2IndexNode->gtIntCon.gtIconVal)
2096 if (op1ArrayRef->OperGet() == GT_FIELD && op2ArrayRef->OperGet() == GT_FIELD &&
2097 areFieldsParentsLocatedSame(op1ArrayRef, op2ArrayRef))
2101 else if (op1ArrayRef->OperIsLocal() && op2ArrayRef->OperIsLocal() &&
2102 op1ArrayRef->AsLclVarCommon()->GetLclNum() == op2ArrayRef->AsLclVarCommon()->GetLclNum())
2110 //-------------------------------------------------------------------------------
2111 // Check whether two argument nodes are contiguous or not.
2116 // if the argument node op1 is located before argument node op2, and they are located contiguously,
2117 // then return true. Otherwise, return false.
2119 // Right now this can only check field and array. In future we should add more cases.
2122 bool Compiler::areArgumentsContiguous(GenTree* op1, GenTree* op2)
2124 if (op1->OperGet() == GT_INDEX && op2->OperGet() == GT_INDEX)
2126 return areArrayElementsContiguous(op1, op2);
2128 else if (op1->OperGet() == GT_FIELD && op2->OperGet() == GT_FIELD)
2130 return areFieldsContiguous(op1, op2);
2135 //--------------------------------------------------------------------------------------------------------
2136 // createAddressNodeForSIMDInit: Generate the address node(GT_LEA) if we want to intialize vector2, vector3 or vector4
2137 // from first argument's address.
2140 // tree - GenTree*. This the tree node which is used to get the address for indir.
2141 // simdsize - unsigned. This the simd vector size.
2142 // arrayElementsCount - unsigned. This is used for generating the boundary check for array.
2145 // return the address node.
2148 // 1. Currently just support for GT_FIELD and GT_INDEX, because we can only verify the GT_INDEX node or GT_Field
2149 // are located contiguously or not. In future we should support more cases.
2150 // 2. Though it happens to just work fine front-end phases are not aware of GT_LEA node. Therefore, convert these
2152 GenTree* Compiler::createAddressNodeForSIMDInit(GenTree* tree, unsigned simdSize)
2154 assert(tree->OperGet() == GT_FIELD || tree->OperGet() == GT_INDEX);
2155 GenTree* byrefNode = nullptr;
2156 GenTree* startIndex = nullptr;
2157 unsigned offset = 0;
2158 var_types baseType = tree->gtType;
2160 if (tree->OperGet() == GT_FIELD)
2162 GenTree* objRef = tree->gtField.gtFldObj;
2163 if (objRef != nullptr && objRef->gtOper == GT_ADDR)
2165 GenTree* obj = objRef->gtOp.gtOp1;
2167 // If the field is directly from a struct, then in this case,
2168 // we should set this struct's lvUsedInSIMDIntrinsic as true,
2169 // so that this sturct won't be promoted.
2170 // e.g. s.x x is a field, and s is a struct, then we should set the s's lvUsedInSIMDIntrinsic as true.
2171 // so that s won't be promoted.
2172 // Notice that if we have a case like s1.s2.x. s1 s2 are struct, and x is a field, then it is possible that
2173 // s1 can be promoted, so that s2 can be promoted. The reason for that is if we don't allow s1 to be
2174 // promoted, then this will affect the other optimizations which are depend on s1's struct promotion.
2176 // In future, we should optimize this case so that if there is a nested field like s1.s2.x and s1.s2.x's
2177 // address is used for initializing the vector, then s1 can be promoted but s2 can't.
2178 if (varTypeIsSIMD(obj) && obj->OperIsLocal())
2180 setLclRelatedToSIMDIntrinsic(obj);
2184 byrefNode = gtCloneExpr(tree->gtField.gtFldObj);
2185 assert(byrefNode != nullptr);
2186 offset = tree->gtField.gtFldOffset;
2188 else if (tree->OperGet() == GT_INDEX)
2191 GenTree* index = tree->AsIndex()->Index();
2192 assert(index->OperGet() == GT_CNS_INT);
2194 GenTree* checkIndexExpr = nullptr;
2195 unsigned indexVal = (unsigned)(index->gtIntCon.gtIconVal);
2196 offset = indexVal * genTypeSize(tree->TypeGet());
2197 GenTree* arrayRef = tree->AsIndex()->Arr();
2199 // Generate the boundary check exception.
2200 // The length for boundary check should be the maximum index number which should be
2201 // (first argument's index number) + (how many array arguments we have) - 1
2202 // = indexVal + arrayElementsCount - 1
2203 unsigned arrayElementsCount = simdSize / genTypeSize(baseType);
2204 checkIndexExpr = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, indexVal + arrayElementsCount - 1);
2205 GenTreeArrLen* arrLen = gtNewArrLen(TYP_INT, arrayRef, (int)offsetof(CORINFO_Array, length));
2206 GenTreeBoundsChk* arrBndsChk = new (this, GT_ARR_BOUNDS_CHECK)
2207 GenTreeBoundsChk(GT_ARR_BOUNDS_CHECK, TYP_VOID, checkIndexExpr, arrLen, SCK_RNGCHK_FAIL);
2209 offset += offsetof(CORINFO_Array, u1Elems);
2210 byrefNode = gtNewOperNode(GT_COMMA, arrayRef->TypeGet(), arrBndsChk, gtCloneExpr(arrayRef));
2217 new (this, GT_LEA) GenTreeAddrMode(TYP_BYREF, byrefNode, startIndex, genTypeSize(tree->TypeGet()), offset);
2221 //-------------------------------------------------------------------------------
2222 // impMarkContiguousSIMDFieldAssignments: Try to identify if there are contiguous
2223 // assignments from SIMD field to memory. If there are, then mark the related
2224 // lclvar so that it won't be promoted.
2227 // stmt - GenTree*. Input statement node.
2229 void Compiler::impMarkContiguousSIMDFieldAssignments(GenTree* stmt)
2231 if (!featureSIMD || opts.MinOpts())
2235 GenTree* expr = stmt->gtStmt.gtStmtExpr;
2236 if (expr->OperGet() == GT_ASG && expr->TypeGet() == TYP_FLOAT)
2238 GenTree* curDst = expr->gtOp.gtOp1;
2239 GenTree* curSrc = expr->gtOp.gtOp2;
2241 var_types baseType = TYP_UNKNOWN;
2242 unsigned simdSize = 0;
2243 GenTree* srcSimdStructNode = getSIMDStructFromField(curSrc, &baseType, &index, &simdSize, true);
2244 if (srcSimdStructNode == nullptr || baseType != TYP_FLOAT)
2246 fgPreviousCandidateSIMDFieldAsgStmt = nullptr;
2248 else if (index == 0 && isSIMDTypeLocal(srcSimdStructNode))
2250 fgPreviousCandidateSIMDFieldAsgStmt = stmt;
2252 else if (fgPreviousCandidateSIMDFieldAsgStmt != nullptr)
2255 GenTree* prevAsgExpr = fgPreviousCandidateSIMDFieldAsgStmt->gtStmt.gtStmtExpr;
2256 GenTree* prevDst = prevAsgExpr->gtOp.gtOp1;
2257 GenTree* prevSrc = prevAsgExpr->gtOp.gtOp2;
2258 if (!areArgumentsContiguous(prevDst, curDst) || !areArgumentsContiguous(prevSrc, curSrc))
2260 fgPreviousCandidateSIMDFieldAsgStmt = nullptr;
2264 if (index == (simdSize / genTypeSize(baseType) - 1))
2266 // Successfully found the pattern, mark the lclvar as UsedInSIMDIntrinsic
2267 if (srcSimdStructNode->OperIsLocal())
2269 setLclRelatedToSIMDIntrinsic(srcSimdStructNode);
2272 if (curDst->OperGet() == GT_FIELD)
2274 GenTree* objRef = curDst->gtField.gtFldObj;
2275 if (objRef != nullptr && objRef->gtOper == GT_ADDR)
2277 GenTree* obj = objRef->gtOp.gtOp1;
2278 if (varTypeIsStruct(obj) && obj->OperIsLocal())
2280 setLclRelatedToSIMDIntrinsic(obj);
2287 fgPreviousCandidateSIMDFieldAsgStmt = stmt;
2294 fgPreviousCandidateSIMDFieldAsgStmt = nullptr;
2298 //------------------------------------------------------------------------
2299 // impSIMDIntrinsic: Check method to see if it is a SIMD method
2302 // opcode - the opcode being handled (needed to identify the CEE_NEWOBJ case)
2303 // newobjThis - For CEE_NEWOBJ, this is the temp grabbed for the allocated uninitalized object.
2304 // clsHnd - The handle of the class of the method.
2305 // method - The handle of the method.
2306 // sig - The call signature for the method.
2307 // memberRef - The memberRef token for the method reference.
2310 // If clsHnd is a known SIMD type, and 'method' is one of the methods that are
2311 // implemented as an intrinsic in the JIT, then return the tree that implements
2314 GenTree* Compiler::impSIMDIntrinsic(OPCODE opcode,
2315 GenTree* newobjThis,
2316 CORINFO_CLASS_HANDLE clsHnd,
2317 CORINFO_METHOD_HANDLE methodHnd,
2318 CORINFO_SIG_INFO* sig,
2321 assert(featureSIMD);
2323 if (!isSIMDClass(clsHnd))
2328 // Get base type and intrinsic Id
2329 var_types baseType = TYP_UNKNOWN;
2331 unsigned argCount = 0;
2332 const SIMDIntrinsicInfo* intrinsicInfo =
2333 getSIMDIntrinsicInfo(&clsHnd, methodHnd, sig, (opcode == CEE_NEWOBJ), &argCount, &baseType, &size);
2334 if (intrinsicInfo == nullptr || intrinsicInfo->id == SIMDIntrinsicInvalid)
2339 SIMDIntrinsicID simdIntrinsicID = intrinsicInfo->id;
2341 if (baseType != TYP_UNKNOWN)
2343 simdType = getSIMDTypeForSize(size);
2347 assert(simdIntrinsicID == SIMDIntrinsicHWAccel);
2348 simdType = TYP_UNKNOWN;
2350 bool instMethod = intrinsicInfo->isInstMethod;
2351 var_types callType = JITtype2varType(sig->retType);
2352 if (callType == TYP_STRUCT)
2354 // Note that here we are assuming that, if the call returns a struct, that it is the same size as the
2355 // struct on which the method is declared. This is currently true for all methods on Vector types,
2356 // but if this ever changes, we will need to determine the callType from the signature.
2357 assert(info.compCompHnd->getClassSize(sig->retTypeClass) == genTypeSize(simdType));
2358 callType = simdType;
2361 GenTree* simdTree = nullptr;
2362 GenTree* op1 = nullptr;
2363 GenTree* op2 = nullptr;
2364 GenTree* op3 = nullptr;
2365 GenTree* retVal = nullptr;
2366 GenTree* copyBlkDst = nullptr;
2367 bool doCopyBlk = false;
2369 switch (simdIntrinsicID)
2371 case SIMDIntrinsicGetCount:
2373 int length = getSIMDVectorLength(clsHnd);
2374 GenTreeIntCon* intConstTree = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, length);
2375 retVal = intConstTree;
2377 intConstTree->gtFlags |= GTF_ICON_SIMD_COUNT;
2381 case SIMDIntrinsicGetZero:
2382 retVal = gtNewSIMDVectorZero(simdType, baseType, size);
2385 case SIMDIntrinsicGetOne:
2386 retVal = gtNewSIMDVectorOne(simdType, baseType, size);
2389 case SIMDIntrinsicGetAllOnes:
2391 // Equivalent to (Vector<T>) new Vector<int>(0xffffffff);
2392 GenTree* initVal = gtNewIconNode(0xffffffff, TYP_INT);
2393 simdTree = gtNewSIMDNode(simdType, initVal, nullptr, SIMDIntrinsicInit, TYP_INT, size);
2394 if (baseType != TYP_INT)
2396 // cast it to required baseType if different from TYP_INT
2397 simdTree = gtNewSIMDNode(simdType, simdTree, nullptr, SIMDIntrinsicCast, baseType, size);
2403 case SIMDIntrinsicInit:
2404 case SIMDIntrinsicInitN:
2406 // SIMDIntrinsicInit:
2407 // op2 - the initializer value
2408 // op1 - byref of vector
2410 // SIMDIntrinsicInitN
2411 // op2 - list of initializer values stitched into a list
2412 // op1 - byref of vector
2413 bool initFromFirstArgIndir = false;
2414 if (simdIntrinsicID == SIMDIntrinsicInit)
2416 op2 = impSIMDPopStack(baseType);
2420 assert(simdIntrinsicID == SIMDIntrinsicInitN);
2421 assert(baseType == TYP_FLOAT);
2423 unsigned initCount = argCount - 1;
2424 unsigned elementCount = getSIMDVectorLength(size, baseType);
2425 noway_assert(initCount == elementCount);
2426 GenTree* nextArg = op2;
2428 // Build a GT_LIST with the N values.
2429 // We must maintain left-to-right order of the args, but we will pop
2430 // them off in reverse order (the Nth arg was pushed onto the stack last).
2432 GenTree* list = nullptr;
2433 GenTree* firstArg = nullptr;
2434 GenTree* prevArg = nullptr;
2436 bool areArgsContiguous = true;
2437 for (unsigned i = 0; i < initCount; i++)
2439 GenTree* nextArg = impSIMDPopStack(baseType);
2440 if (areArgsContiguous)
2442 GenTree* curArg = nextArg;
2445 if (prevArg != nullptr)
2447 // Recall that we are popping the args off the stack in reverse order.
2448 areArgsContiguous = areArgumentsContiguous(curArg, prevArg);
2453 list = new (this, GT_LIST) GenTreeOp(GT_LIST, baseType, nextArg, list);
2456 if (areArgsContiguous && baseType == TYP_FLOAT)
2458 // Since Vector2, Vector3 and Vector4's arguments type are only float,
2459 // we intialize the vector from first argument address, only when
2460 // the baseType is TYP_FLOAT and the arguments are located contiguously in memory
2461 initFromFirstArgIndir = true;
2462 GenTree* op2Address = createAddressNodeForSIMDInit(firstArg, size);
2463 var_types simdType = getSIMDTypeForSize(size);
2464 op2 = gtNewOperNode(GT_IND, simdType, op2Address);
2472 op1 = getOp1ForConstructor(opcode, newobjThis, clsHnd);
2474 assert(op1->TypeGet() == TYP_BYREF);
2475 assert(genActualType(op2->TypeGet()) == genActualType(baseType) || initFromFirstArgIndir);
2477 // For integral base types of size less than TYP_INT, expand the initializer
2478 // to fill size of TYP_INT bytes.
2479 if (varTypeIsSmallInt(baseType))
2481 // This case should occur only for Init intrinsic.
2482 assert(simdIntrinsicID == SIMDIntrinsicInit);
2484 unsigned baseSize = genTypeSize(baseType);
2488 multiplier = 0x01010101;
2492 assert(baseSize == 2);
2493 multiplier = 0x00010001;
2496 GenTree* t1 = nullptr;
2497 if (baseType == TYP_BYTE)
2499 // What we have is a signed byte initializer,
2500 // which when loaded to a reg will get sign extended to TYP_INT.
2501 // But what we need is the initializer without sign extended or
2502 // rather zero extended to 32-bits.
2503 t1 = gtNewOperNode(GT_AND, TYP_INT, op2, gtNewIconNode(0xff, TYP_INT));
2505 else if (baseType == TYP_SHORT)
2507 // What we have is a signed short initializer,
2508 // which when loaded to a reg will get sign extended to TYP_INT.
2509 // But what we need is the initializer without sign extended or
2510 // rather zero extended to 32-bits.
2511 t1 = gtNewOperNode(GT_AND, TYP_INT, op2, gtNewIconNode(0xffff, TYP_INT));
2515 assert(baseType == TYP_UBYTE || baseType == TYP_USHORT);
2516 t1 = gtNewCastNode(TYP_INT, op2, false, TYP_INT);
2519 assert(t1 != nullptr);
2520 GenTree* t2 = gtNewIconNode(multiplier, TYP_INT);
2521 op2 = gtNewOperNode(GT_MUL, TYP_INT, t1, t2);
2523 // Construct a vector of TYP_INT with the new initializer and cast it back to vector of baseType
2524 simdTree = gtNewSIMDNode(simdType, op2, nullptr, simdIntrinsicID, TYP_INT, size);
2525 simdTree = gtNewSIMDNode(simdType, simdTree, nullptr, SIMDIntrinsicCast, baseType, size);
2530 if (initFromFirstArgIndir)
2533 if (op1->gtOp.gtOp1->OperIsLocal())
2535 // label the dst struct's lclvar is used for SIMD intrinsic,
2536 // so that this dst struct won't be promoted.
2537 setLclRelatedToSIMDIntrinsic(op1->gtOp.gtOp1);
2542 simdTree = gtNewSIMDNode(simdType, op2, nullptr, simdIntrinsicID, baseType, size);
2551 case SIMDIntrinsicInitArray:
2552 case SIMDIntrinsicInitArrayX:
2553 case SIMDIntrinsicCopyToArray:
2554 case SIMDIntrinsicCopyToArrayX:
2556 // op3 - index into array in case of SIMDIntrinsicCopyToArrayX and SIMDIntrinsicInitArrayX
2557 // op2 - array itself
2558 // op1 - byref to vector struct
2560 unsigned int vectorLength = getSIMDVectorLength(size, baseType);
2561 // (This constructor takes only the zero-based arrays.)
2562 // We will add one or two bounds checks:
2563 // 1. If we have an index, we must do a check on that first.
2564 // We can't combine it with the index + vectorLength check because
2565 // a. It might be negative, and b. It may need to raise a different exception
2566 // (captured as SCK_ARG_RNG_EXCPN for CopyTo and SCK_RNGCHK_FAIL for Init).
2567 // 2. We need to generate a check (SCK_ARG_EXCPN for CopyTo and SCK_RNGCHK_FAIL for Init)
2568 // for the last array element we will access.
2569 // We'll either check against (vectorLength - 1) or (index + vectorLength - 1).
2571 GenTree* checkIndexExpr = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, vectorLength - 1);
2573 // Get the index into the array. If it has been provided, it will be on the
2574 // top of the stack. Otherwise, it is null.
2577 op3 = impSIMDPopStack(TYP_INT);
2578 if (op3->IsIntegralConst(0))
2585 // TODO-CQ: Here, or elsewhere, check for the pattern where op2 is a newly constructed array, and
2586 // change this to the InitN form.
2587 // op3 = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, 0);
2591 // Clone the array for use in the bounds check.
2592 op2 = impSIMDPopStack(TYP_REF);
2593 assert(op2->TypeGet() == TYP_REF);
2594 GenTree* arrayRefForArgChk = op2;
2595 GenTree* argRngChk = nullptr;
2596 GenTree* asg = nullptr;
2597 if ((arrayRefForArgChk->gtFlags & GTF_SIDE_EFFECT) != 0)
2599 op2 = fgInsertCommaFormTemp(&arrayRefForArgChk);
2603 op2 = gtCloneExpr(arrayRefForArgChk);
2605 assert(op2 != nullptr);
2609 SpecialCodeKind op3CheckKind;
2610 if (simdIntrinsicID == SIMDIntrinsicInitArrayX)
2612 op3CheckKind = SCK_RNGCHK_FAIL;
2616 assert(simdIntrinsicID == SIMDIntrinsicCopyToArrayX);
2617 op3CheckKind = SCK_ARG_RNG_EXCPN;
2619 // We need to use the original expression on this, which is the first check.
2620 GenTree* arrayRefForArgRngChk = arrayRefForArgChk;
2621 // Then we clone the clone we just made for the next check.
2622 arrayRefForArgChk = gtCloneExpr(op2);
2623 // We know we MUST have had a cloneable expression.
2624 assert(arrayRefForArgChk != nullptr);
2625 GenTree* index = op3;
2626 if ((index->gtFlags & GTF_SIDE_EFFECT) != 0)
2628 op3 = fgInsertCommaFormTemp(&index);
2632 op3 = gtCloneExpr(index);
2635 GenTreeArrLen* arrLen =
2636 gtNewArrLen(TYP_INT, arrayRefForArgRngChk, (int)offsetof(CORINFO_Array, length));
2637 argRngChk = new (this, GT_ARR_BOUNDS_CHECK)
2638 GenTreeBoundsChk(GT_ARR_BOUNDS_CHECK, TYP_VOID, index, arrLen, op3CheckKind);
2639 // Now, clone op3 to create another node for the argChk
2640 GenTree* index2 = gtCloneExpr(op3);
2641 assert(index != nullptr);
2642 checkIndexExpr = gtNewOperNode(GT_ADD, TYP_INT, index2, checkIndexExpr);
2645 // Insert a bounds check for index + offset - 1.
2646 // This must be a "normal" array.
2647 SpecialCodeKind op2CheckKind;
2648 if (simdIntrinsicID == SIMDIntrinsicInitArray || simdIntrinsicID == SIMDIntrinsicInitArrayX)
2650 op2CheckKind = SCK_RNGCHK_FAIL;
2654 op2CheckKind = SCK_ARG_EXCPN;
2656 GenTreeArrLen* arrLen = gtNewArrLen(TYP_INT, arrayRefForArgChk, (int)offsetof(CORINFO_Array, length));
2657 GenTreeBoundsChk* argChk = new (this, GT_ARR_BOUNDS_CHECK)
2658 GenTreeBoundsChk(GT_ARR_BOUNDS_CHECK, TYP_VOID, checkIndexExpr, arrLen, op2CheckKind);
2660 // Create a GT_COMMA tree for the bounds check(s).
2661 op2 = gtNewOperNode(GT_COMMA, op2->TypeGet(), argChk, op2);
2662 if (argRngChk != nullptr)
2664 op2 = gtNewOperNode(GT_COMMA, op2->TypeGet(), argRngChk, op2);
2667 if (simdIntrinsicID == SIMDIntrinsicInitArray || simdIntrinsicID == SIMDIntrinsicInitArrayX)
2669 op1 = getOp1ForConstructor(opcode, newobjThis, clsHnd);
2670 simdTree = gtNewSIMDNode(simdType, op2, op3, SIMDIntrinsicInitArray, baseType, size);
2676 assert(simdIntrinsicID == SIMDIntrinsicCopyToArray || simdIntrinsicID == SIMDIntrinsicCopyToArrayX);
2677 op1 = impSIMDPopStack(simdType, instMethod);
2678 assert(op1->TypeGet() == simdType);
2680 // copy vector (op1) to array (op2) starting at index (op3)
2683 // TODO-Cleanup: Though it happens to just work fine front-end phases are not aware of GT_LEA node.
2684 // Therefore, convert these to use GT_ADDR .
2685 copyBlkDst = new (this, GT_LEA)
2686 GenTreeAddrMode(TYP_BYREF, op2, op3, genTypeSize(baseType), offsetof(CORINFO_Array, u1Elems));
2692 case SIMDIntrinsicInitFixed:
2694 // We are initializing a fixed-length vector VLarge with a smaller fixed-length vector VSmall, plus 1 or 2
2695 // additional floats.
2696 // op4 (optional) - float value for VLarge.W, if VLarge is Vector4, and VSmall is Vector2
2697 // op3 - float value for VLarge.Z or VLarge.W
2699 // op1 - byref of VLarge
2700 assert(baseType == TYP_FLOAT);
2701 unsigned elementByteCount = 4;
2703 GenTree* op4 = nullptr;
2706 op4 = impSIMDPopStack(TYP_FLOAT);
2707 assert(op4->TypeGet() == TYP_FLOAT);
2709 op3 = impSIMDPopStack(TYP_FLOAT);
2710 assert(op3->TypeGet() == TYP_FLOAT);
2711 // The input vector will either be TYP_SIMD8 or TYP_SIMD12.
2712 var_types smallSIMDType = TYP_SIMD8;
2713 if ((op4 == nullptr) && (simdType == TYP_SIMD16))
2715 smallSIMDType = TYP_SIMD12;
2717 op2 = impSIMDPopStack(smallSIMDType);
2718 op1 = getOp1ForConstructor(opcode, newobjThis, clsHnd);
2720 // We are going to redefine the operands so that:
2721 // - op3 is the value that's going into the Z position, or null if it's a Vector4 constructor with a single
2723 // - op4 is the W position value, or null if this is a Vector3 constructor.
2724 if (size == 16 && argCount == 3)
2733 simdTree = gtNewSIMDNode(simdType, simdTree, op3, SIMDIntrinsicSetZ, baseType, size);
2737 simdTree = gtNewSIMDNode(simdType, simdTree, op4, SIMDIntrinsicSetW, baseType, size);
2745 case SIMDIntrinsicOpEquality:
2746 case SIMDIntrinsicInstEquals:
2748 op2 = impSIMDPopStack(simdType);
2749 op1 = impSIMDPopStack(simdType, instMethod);
2751 assert(op1->TypeGet() == simdType);
2752 assert(op2->TypeGet() == simdType);
2754 simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, SIMDIntrinsicOpEquality, baseType, size);
2755 if (simdType == TYP_SIMD12)
2757 simdTree->gtFlags |= GTF_SIMD12_OP;
2763 case SIMDIntrinsicOpInEquality:
2765 // op1 is the first operand
2766 // op2 is the second operand
2767 op2 = impSIMDPopStack(simdType);
2768 op1 = impSIMDPopStack(simdType, instMethod);
2769 simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, SIMDIntrinsicOpInEquality, baseType, size);
2770 if (simdType == TYP_SIMD12)
2772 simdTree->gtFlags |= GTF_SIMD12_OP;
2778 case SIMDIntrinsicEqual:
2779 case SIMDIntrinsicLessThan:
2780 case SIMDIntrinsicLessThanOrEqual:
2781 case SIMDIntrinsicGreaterThan:
2782 case SIMDIntrinsicGreaterThanOrEqual:
2784 op2 = impSIMDPopStack(simdType);
2785 op1 = impSIMDPopStack(simdType, instMethod);
2787 SIMDIntrinsicID intrinsicID = impSIMDRelOp(simdIntrinsicID, clsHnd, size, &baseType, &op1, &op2);
2788 simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, intrinsicID, baseType, size);
2793 case SIMDIntrinsicAdd:
2794 case SIMDIntrinsicSub:
2795 case SIMDIntrinsicMul:
2796 case SIMDIntrinsicDiv:
2797 case SIMDIntrinsicBitwiseAnd:
2798 case SIMDIntrinsicBitwiseAndNot:
2799 case SIMDIntrinsicBitwiseOr:
2800 case SIMDIntrinsicBitwiseXor:
2803 // check for the cases where we don't support intrinsics.
2804 // This check should be done before we make modifications to type stack.
2805 // Note that this is more of a double safety check for robustness since
2806 // we expect getSIMDIntrinsicInfo() to have filtered out intrinsics on
2807 // unsupported base types. If getSIMdIntrinsicInfo() doesn't filter due
2808 // to some bug, assert in chk/dbg will fire.
2809 if (!varTypeIsFloating(baseType))
2811 if (simdIntrinsicID == SIMDIntrinsicMul)
2813 #if defined(_TARGET_XARCH_)
2814 if ((baseType != TYP_INT) && (baseType != TYP_SHORT))
2816 // TODO-CQ: implement mul on these integer vectors.
2817 // Note that SSE2 has no direct support for these vectors.
2818 assert(!"Mul not supported on long/ulong/uint/small int vectors\n");
2821 #endif // _TARGET_XARCH_
2822 #if defined(_TARGET_ARM64_)
2823 if ((baseType == TYP_ULONG) && (baseType == TYP_LONG))
2825 // TODO-CQ: implement mul on these integer vectors.
2826 // Note that ARM64 has no direct support for these vectors.
2827 assert(!"Mul not supported on long/ulong vectors\n");
2830 #endif // _TARGET_ARM64_
2832 #if defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
2833 // common to all integer type vectors
2834 if (simdIntrinsicID == SIMDIntrinsicDiv)
2836 // SSE2 doesn't support div on non-floating point vectors.
2837 assert(!"Div not supported on integer type vectors\n");
2840 #endif // defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
2844 // op1 is the first operand; if instance method, op1 is "this" arg
2845 // op2 is the second operand
2846 op2 = impSIMDPopStack(simdType);
2847 op1 = impSIMDPopStack(simdType, instMethod);
2849 simdTree = gtNewSIMDNode(simdType, op1, op2, simdIntrinsicID, baseType, size);
2854 case SIMDIntrinsicSelect:
2856 // op3 is a SIMD variable that is the second source
2857 // op2 is a SIMD variable that is the first source
2858 // op1 is a SIMD variable which is the bit mask.
2859 op3 = impSIMDPopStack(simdType);
2860 op2 = impSIMDPopStack(simdType);
2861 op1 = impSIMDPopStack(simdType);
2863 retVal = impSIMDSelect(clsHnd, baseType, size, op1, op2, op3);
2867 case SIMDIntrinsicMin:
2868 case SIMDIntrinsicMax:
2870 // op1 is the first operand; if instance method, op1 is "this" arg
2871 // op2 is the second operand
2872 op2 = impSIMDPopStack(simdType);
2873 op1 = impSIMDPopStack(simdType, instMethod);
2875 retVal = impSIMDMinMax(simdIntrinsicID, clsHnd, baseType, size, op1, op2);
2879 case SIMDIntrinsicGetItem:
2881 // op1 is a SIMD variable that is "this" arg
2882 // op2 is an index of TYP_INT
2883 op2 = impSIMDPopStack(TYP_INT);
2884 op1 = impSIMDPopStack(simdType, instMethod);
2885 int vectorLength = getSIMDVectorLength(size, baseType);
2886 if (!op2->IsCnsIntOrI() || op2->AsIntCon()->gtIconVal >= vectorLength || op2->AsIntCon()->gtIconVal < 0)
2888 // We need to bounds-check the length of the vector.
2889 // For that purpose, we need to clone the index expression.
2890 GenTree* index = op2;
2891 if ((index->gtFlags & GTF_SIDE_EFFECT) != 0)
2893 op2 = fgInsertCommaFormTemp(&index);
2897 op2 = gtCloneExpr(index);
2900 GenTree* lengthNode = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, vectorLength);
2901 GenTreeBoundsChk* simdChk =
2902 new (this, GT_SIMD_CHK) GenTreeBoundsChk(GT_SIMD_CHK, TYP_VOID, index, lengthNode, SCK_RNGCHK_FAIL);
2904 // Create a GT_COMMA tree for the bounds check.
2905 op2 = gtNewOperNode(GT_COMMA, op2->TypeGet(), simdChk, op2);
2908 assert(op1->TypeGet() == simdType);
2909 assert(op2->TypeGet() == TYP_INT);
2911 simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, simdIntrinsicID, baseType, size);
2916 case SIMDIntrinsicDotProduct:
2918 #if defined(_TARGET_XARCH_)
2919 // Right now dot product is supported only for float/double vectors and
2920 // int vectors on SSE4/AVX.
2921 if (!varTypeIsFloating(baseType) && !(baseType == TYP_INT && getSIMDSupportLevel() >= SIMD_SSE4_Supported))
2925 #endif // _TARGET_XARCH_
2927 // op1 is a SIMD variable that is the first source and also "this" arg.
2928 // op2 is a SIMD variable which is the second source.
2929 op2 = impSIMDPopStack(simdType);
2930 op1 = impSIMDPopStack(simdType, instMethod);
2932 simdTree = gtNewSIMDNode(baseType, op1, op2, simdIntrinsicID, baseType, size);
2933 if (simdType == TYP_SIMD12)
2935 simdTree->gtFlags |= GTF_SIMD12_OP;
2941 case SIMDIntrinsicSqrt:
2943 #if (defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)) && defined(DEBUG)
2944 // SSE/AVX/ARM64 doesn't support sqrt on integer type vectors and hence
2945 // should never be seen as an intrinsic here. See SIMDIntrinsicList.h
2946 // for supported base types for this intrinsic.
2947 if (!varTypeIsFloating(baseType))
2949 assert(!"Sqrt not supported on integer vectors\n");
2952 #endif // (defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)) && defined(DEBUG)
2954 op1 = impSIMDPopStack(simdType);
2956 retVal = gtNewSIMDNode(genActualType(callType), op1, nullptr, simdIntrinsicID, baseType, size);
2960 case SIMDIntrinsicAbs:
2961 op1 = impSIMDPopStack(simdType);
2962 retVal = impSIMDAbs(clsHnd, baseType, size, op1);
2965 case SIMDIntrinsicGetW:
2966 retVal = impSIMDGetFixed(simdType, baseType, size, 3);
2969 case SIMDIntrinsicGetZ:
2970 retVal = impSIMDGetFixed(simdType, baseType, size, 2);
2973 case SIMDIntrinsicGetY:
2974 retVal = impSIMDGetFixed(simdType, baseType, size, 1);
2977 case SIMDIntrinsicGetX:
2978 retVal = impSIMDGetFixed(simdType, baseType, size, 0);
2981 case SIMDIntrinsicSetW:
2982 case SIMDIntrinsicSetZ:
2983 case SIMDIntrinsicSetY:
2984 case SIMDIntrinsicSetX:
2986 // op2 is the value to be set at indexTemp position
2987 // op1 is SIMD vector that is going to be modified, which is a byref
2989 // If op1 has a side-effect, then don't make it an intrinsic.
2990 // It would be in-efficient to read the entire vector into xmm reg,
2991 // modify it and write back entire xmm reg.
2993 // TODO-CQ: revisit this later.
2994 op1 = impStackTop(1).val;
2995 if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0)
3000 op2 = impSIMDPopStack(baseType);
3001 op1 = impSIMDPopStack(simdType, instMethod);
3003 GenTree* src = gtCloneExpr(op1);
3004 assert(src != nullptr);
3005 simdTree = gtNewSIMDNode(simdType, src, op2, simdIntrinsicID, baseType, size);
3007 copyBlkDst = gtNewOperNode(GT_ADDR, TYP_BYREF, op1);
3012 // Unary operators that take and return a Vector.
3013 case SIMDIntrinsicCast:
3014 case SIMDIntrinsicConvertToSingle:
3015 case SIMDIntrinsicConvertToDouble:
3016 case SIMDIntrinsicConvertToInt32:
3018 op1 = impSIMDPopStack(simdType, instMethod);
3020 simdTree = gtNewSIMDNode(simdType, op1, nullptr, simdIntrinsicID, baseType, size);
3025 case SIMDIntrinsicConvertToInt64:
3027 #ifdef _TARGET_64BIT_
3028 op1 = impSIMDPopStack(simdType, instMethod);
3030 simdTree = gtNewSIMDNode(simdType, op1, nullptr, simdIntrinsicID, baseType, size);
3033 JITDUMP("SIMD Conversion to Int64 is not supported on this platform\n");
3039 case SIMDIntrinsicNarrow:
3041 assert(!instMethod);
3042 op2 = impSIMDPopStack(simdType);
3043 op1 = impSIMDPopStack(simdType);
3044 // op1 and op2 are two input Vector<T>.
3045 simdTree = gtNewSIMDNode(simdType, op1, op2, simdIntrinsicID, baseType, size);
3050 case SIMDIntrinsicWiden:
3052 GenTree* dstAddrHi = impSIMDPopStack(TYP_BYREF);
3053 GenTree* dstAddrLo = impSIMDPopStack(TYP_BYREF);
3054 op1 = impSIMDPopStack(simdType);
3055 GenTree* dupOp1 = fgInsertCommaFormTemp(&op1, gtGetStructHandleForSIMD(simdType, baseType));
3057 // Widen the lower half and assign it to dstAddrLo.
3058 simdTree = gtNewSIMDNode(simdType, op1, nullptr, SIMDIntrinsicWidenLo, baseType, size);
3060 new (this, GT_BLK) GenTreeBlk(GT_BLK, simdType, dstAddrLo, getSIMDTypeSizeInBytes(clsHnd));
3061 GenTree* loAsg = gtNewBlkOpNode(loDest, simdTree, getSIMDTypeSizeInBytes(clsHnd),
3062 false, // not volatile
3064 loAsg->gtFlags |= ((simdTree->gtFlags | dstAddrLo->gtFlags) & GTF_ALL_EFFECT);
3066 // Widen the upper half and assign it to dstAddrHi.
3067 simdTree = gtNewSIMDNode(simdType, dupOp1, nullptr, SIMDIntrinsicWidenHi, baseType, size);
3069 new (this, GT_BLK) GenTreeBlk(GT_BLK, simdType, dstAddrHi, getSIMDTypeSizeInBytes(clsHnd));
3070 GenTree* hiAsg = gtNewBlkOpNode(hiDest, simdTree, getSIMDTypeSizeInBytes(clsHnd),
3071 false, // not volatile
3073 hiAsg->gtFlags |= ((simdTree->gtFlags | dstAddrHi->gtFlags) & GTF_ALL_EFFECT);
3075 retVal = gtNewOperNode(GT_COMMA, simdType, loAsg, hiAsg);
3079 case SIMDIntrinsicHWAccel:
3081 GenTreeIntCon* intConstTree = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, 1);
3082 retVal = intConstTree;
3087 assert(!"Unimplemented SIMD Intrinsic");
3091 #if defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
3092 // XArch/Arm64: also indicate that we use floating point registers.
3093 // The need for setting this here is that a method may not have SIMD
3094 // type lclvars, but might be exercising SIMD intrinsics on fields of
3097 // e.g. public Vector<float> ComplexVecFloat::sqabs() { return this.r * this.r + this.i * this.i; }
3098 compFloatingPointUsed = true;
3099 #endif // defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
3101 // At this point, we have a tree that we are going to store into a destination.
3102 // TODO-1stClassStructs: This should be a simple store or assignment, and should not require
3103 // GTF_ALL_EFFECT for the dest. This is currently emulating the previous behavior of
3107 GenTree* dest = new (this, GT_BLK) GenTreeBlk(GT_BLK, simdType, copyBlkDst, getSIMDTypeSizeInBytes(clsHnd));
3108 dest->gtFlags |= GTF_GLOB_REF;
3109 retVal = gtNewBlkOpNode(dest, simdTree, getSIMDTypeSizeInBytes(clsHnd),
3110 false, // not volatile
3112 retVal->gtFlags |= ((simdTree->gtFlags | copyBlkDst->gtFlags) & GTF_ALL_EFFECT);
3118 #endif // FEATURE_SIMD
3120 #endif // !LEGACY_BACKEND