1 // Licensed to the .NET Foundation under one or more agreements.
2 // The .NET Foundation licenses this file to you under the MIT license.
3 // See the LICENSE file in the project root for more information.
8 // IMPORTANT NOTES AND CAVEATS:
10 // This implementation is preliminary, and may change dramatically.
12 // New JIT types, TYP_SIMDxx, are introduced, and the SIMD intrinsics are created as GT_SIMD nodes.
13 // Nodes of SIMD types will be typed as TYP_SIMD* (e.g. TYP_SIMD8, TYP_SIMD16, etc.).
15 // Note that currently the "reference implementation" is the same as the runtime dll. As such, it is currently
16 // providing implementations for those methods not currently supported by the JIT as intrinsics.
18 // These are currently recognized using string compares, in order to provide an implementation in the JIT
19 // without taking a dependency on the VM.
20 // Furthermore, in the CTP, in order to limit the impact of doing these string compares
21 // against assembly names, we only look for the SIMDVector assembly if we are compiling a class constructor. This
22 // makes it somewhat more "pay for play" but is a significant usability compromise.
23 // This has been addressed for RTM by doing the assembly recognition in the VM.
24 // --------------------------------------------------------------------------------------
33 #ifndef LEGACY_BACKEND // This file is ONLY used for the RyuJIT backend that uses the linear scan register allocator.
37 // Intrinsic Id to intrinsic info map
38 const SIMDIntrinsicInfo simdIntrinsicInfoArray[] = {
39 #define SIMD_INTRINSIC(mname, inst, id, name, retType, argCount, arg1, arg2, arg3, t1, t2, t3, t4, t5, t6, t7, t8, t9, \
41 {SIMDIntrinsic##id, mname, inst, retType, argCount, arg1, arg2, arg3, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10},
42 #include "simdintrinsiclist.h"
45 //------------------------------------------------------------------------
46 // getSIMDVectorLength: Get the length (number of elements of base type) of
47 // SIMD Vector given its size and base (element) type.
50 // simdSize - size of the SIMD vector
51 // baseType - type of the elements of the SIMD vector
54 int Compiler::getSIMDVectorLength(unsigned simdSize, var_types baseType)
56 return simdSize / genTypeSize(baseType);
59 //------------------------------------------------------------------------
60 // Get the length (number of elements of base type) of SIMD Vector given by typeHnd.
63 // typeHnd - type handle of the SIMD vector
65 int Compiler::getSIMDVectorLength(CORINFO_CLASS_HANDLE typeHnd)
67 unsigned sizeBytes = 0;
68 var_types baseType = getBaseTypeAndSizeOfSIMDType(typeHnd, &sizeBytes);
69 return getSIMDVectorLength(sizeBytes, baseType);
72 //------------------------------------------------------------------------
73 // Get the preferred alignment of SIMD vector type for better performance.
76 // typeHnd - type handle of the SIMD vector
78 int Compiler::getSIMDTypeAlignment(var_types simdType)
81 // Fixed length vectors have the following alignment preference
82 // Vector2 = 8 byte alignment
83 // Vector3/4 = 16-byte alignment
84 unsigned size = genTypeSize(simdType);
86 // preferred alignment for SSE2 128-bit vectors is 16-bytes
93 assert((size == 12) || (size == 16));
101 #elif defined(_TARGET_ARM64_)
104 assert(!"getSIMDTypeAlignment() unimplemented on target arch");
109 //----------------------------------------------------------------------------------
110 // Return the base type and size of SIMD vector type given its type handle.
113 // typeHnd - The handle of the type we're interested in.
114 // sizeBytes - out param
117 // base type of SIMD vector.
118 // sizeBytes if non-null is set to size in bytes.
120 // TODO-Throughput: current implementation parses class name to find base type. Change
121 // this when we implement SIMD intrinsic identification for the final
124 var_types Compiler::getBaseTypeAndSizeOfSIMDType(CORINFO_CLASS_HANDLE typeHnd, unsigned* sizeBytes /*= nullptr */)
127 if (typeHnd == nullptr)
132 // fast path search using cached type handles of important types
133 var_types simdBaseType = TYP_UNKNOWN;
136 // TODO - Optimize SIMD type recognition by IntrinsicAttribute
137 if (isSIMDClass(typeHnd))
139 // The most likely to be used type handles are looked up first followed by
140 // less likely to be used type handles
141 if (typeHnd == SIMDFloatHandle)
143 simdBaseType = TYP_FLOAT;
144 JITDUMP(" Known type SIMD Vector<Float>\n");
146 else if (typeHnd == SIMDIntHandle)
148 simdBaseType = TYP_INT;
149 JITDUMP(" Known type SIMD Vector<Int>\n");
151 else if (typeHnd == SIMDVector2Handle)
153 simdBaseType = TYP_FLOAT;
154 size = 2 * genTypeSize(TYP_FLOAT);
155 assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE));
156 JITDUMP(" Known type Vector2\n");
158 else if (typeHnd == SIMDVector3Handle)
160 simdBaseType = TYP_FLOAT;
161 size = 3 * genTypeSize(TYP_FLOAT);
162 assert(size == info.compCompHnd->getClassSize(typeHnd));
163 JITDUMP(" Known type Vector3\n");
165 else if (typeHnd == SIMDVector4Handle)
167 simdBaseType = TYP_FLOAT;
168 size = 4 * genTypeSize(TYP_FLOAT);
169 assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE));
170 JITDUMP(" Known type Vector4\n");
172 else if (typeHnd == SIMDVectorHandle)
174 JITDUMP(" Known type Vector\n");
176 else if (typeHnd == SIMDUShortHandle)
178 simdBaseType = TYP_USHORT;
179 JITDUMP(" Known type SIMD Vector<ushort>\n");
181 else if (typeHnd == SIMDUByteHandle)
183 simdBaseType = TYP_UBYTE;
184 JITDUMP(" Known type SIMD Vector<ubyte>\n");
186 else if (typeHnd == SIMDDoubleHandle)
188 simdBaseType = TYP_DOUBLE;
189 JITDUMP(" Known type SIMD Vector<Double>\n");
191 else if (typeHnd == SIMDLongHandle)
193 simdBaseType = TYP_LONG;
194 JITDUMP(" Known type SIMD Vector<Long>\n");
196 else if (typeHnd == SIMDShortHandle)
198 simdBaseType = TYP_SHORT;
199 JITDUMP(" Known type SIMD Vector<short>\n");
201 else if (typeHnd == SIMDByteHandle)
203 simdBaseType = TYP_BYTE;
204 JITDUMP(" Known type SIMD Vector<byte>\n");
206 else if (typeHnd == SIMDUIntHandle)
208 simdBaseType = TYP_UINT;
209 JITDUMP(" Known type SIMD Vector<uint>\n");
211 else if (typeHnd == SIMDULongHandle)
213 simdBaseType = TYP_ULONG;
214 JITDUMP(" Known type SIMD Vector<ulong>\n");
218 if (simdBaseType == TYP_UNKNOWN)
220 // Doesn't match with any of the cached type handles.
221 // Obtain base type by parsing fully qualified class name.
223 // TODO-Throughput: implement product shipping solution to query base type.
224 WCHAR className[256] = {0};
225 WCHAR* pbuf = &className[0];
226 int len = _countof(className);
227 info.compCompHnd->appendClassName(&pbuf, &len, typeHnd, TRUE, FALSE, FALSE);
228 noway_assert(pbuf < &className[256]);
229 JITDUMP("SIMD Candidate Type %S\n", className);
231 if (wcsncmp(className, W("System.Numerics."), 16) == 0)
233 if (wcsncmp(&(className[16]), W("Vector`1["), 9) == 0)
235 if (wcsncmp(&(className[25]), W("System.Single"), 13) == 0)
237 SIMDFloatHandle = typeHnd;
238 simdBaseType = TYP_FLOAT;
239 JITDUMP(" Found type SIMD Vector<Float>\n");
241 else if (wcsncmp(&(className[25]), W("System.Int32"), 12) == 0)
243 SIMDIntHandle = typeHnd;
244 simdBaseType = TYP_INT;
245 JITDUMP(" Found type SIMD Vector<Int>\n");
247 else if (wcsncmp(&(className[25]), W("System.UInt16"), 13) == 0)
249 SIMDUShortHandle = typeHnd;
250 simdBaseType = TYP_USHORT;
251 JITDUMP(" Found type SIMD Vector<ushort>\n");
253 else if (wcsncmp(&(className[25]), W("System.Byte"), 11) == 0)
255 SIMDUByteHandle = typeHnd;
256 simdBaseType = TYP_UBYTE;
257 JITDUMP(" Found type SIMD Vector<ubyte>\n");
259 else if (wcsncmp(&(className[25]), W("System.Double"), 13) == 0)
261 SIMDDoubleHandle = typeHnd;
262 simdBaseType = TYP_DOUBLE;
263 JITDUMP(" Found type SIMD Vector<Double>\n");
265 else if (wcsncmp(&(className[25]), W("System.Int64"), 12) == 0)
267 SIMDLongHandle = typeHnd;
268 simdBaseType = TYP_LONG;
269 JITDUMP(" Found type SIMD Vector<Long>\n");
271 else if (wcsncmp(&(className[25]), W("System.Int16"), 12) == 0)
273 SIMDShortHandle = typeHnd;
274 simdBaseType = TYP_SHORT;
275 JITDUMP(" Found type SIMD Vector<short>\n");
277 else if (wcsncmp(&(className[25]), W("System.SByte"), 12) == 0)
279 SIMDByteHandle = typeHnd;
280 simdBaseType = TYP_BYTE;
281 JITDUMP(" Found type SIMD Vector<byte>\n");
283 else if (wcsncmp(&(className[25]), W("System.UInt32"), 13) == 0)
285 SIMDUIntHandle = typeHnd;
286 simdBaseType = TYP_UINT;
287 JITDUMP(" Found type SIMD Vector<uint>\n");
289 else if (wcsncmp(&(className[25]), W("System.UInt64"), 13) == 0)
291 SIMDULongHandle = typeHnd;
292 simdBaseType = TYP_ULONG;
293 JITDUMP(" Found type SIMD Vector<ulong>\n");
297 JITDUMP(" Unknown SIMD Vector<T>\n");
300 else if (wcsncmp(&(className[16]), W("Vector2"), 8) == 0)
302 SIMDVector2Handle = typeHnd;
304 simdBaseType = TYP_FLOAT;
305 size = 2 * genTypeSize(TYP_FLOAT);
306 assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE));
307 JITDUMP(" Found Vector2\n");
309 else if (wcsncmp(&(className[16]), W("Vector3"), 8) == 0)
311 SIMDVector3Handle = typeHnd;
313 simdBaseType = TYP_FLOAT;
314 size = 3 * genTypeSize(TYP_FLOAT);
315 assert(size == info.compCompHnd->getClassSize(typeHnd));
316 JITDUMP(" Found Vector3\n");
318 else if (wcsncmp(&(className[16]), W("Vector4"), 8) == 0)
320 SIMDVector4Handle = typeHnd;
322 simdBaseType = TYP_FLOAT;
323 size = 4 * genTypeSize(TYP_FLOAT);
324 assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE));
325 JITDUMP(" Found Vector4\n");
327 else if (wcsncmp(&(className[16]), W("Vector"), 6) == 0)
329 SIMDVectorHandle = typeHnd;
330 JITDUMP(" Found type Vector\n");
334 JITDUMP(" Unknown SIMD Type\n");
338 if (simdBaseType != TYP_UNKNOWN && sizeBytes != nullptr)
340 // If not a fixed size vector then its size is same as SIMD vector
341 // register length in bytes
344 size = getSIMDVectorRegisterByteLength();
348 setUsesSIMDTypes(true);
351 #if FEATURE_HW_INTRINSICS
352 else if (isIntrinsicType(typeHnd))
354 const size_t Vector64SizeBytes = 64 / 8;
355 const size_t Vector128SizeBytes = 128 / 8;
356 const size_t Vector256SizeBytes = 256 / 8;
358 #if defined(_TARGET_XARCH_)
359 static_assert_no_msg(YMM_REGSIZE_BYTES == Vector256SizeBytes);
360 static_assert_no_msg(XMM_REGSIZE_BYTES == Vector128SizeBytes);
362 if (typeHnd == Vector256FloatHandle)
364 simdBaseType = TYP_FLOAT;
365 size = Vector256SizeBytes;
366 JITDUMP(" Known type Vector256<float>\n");
368 else if (typeHnd == Vector256DoubleHandle)
370 simdBaseType = TYP_DOUBLE;
371 size = Vector256SizeBytes;
372 JITDUMP(" Known type Vector256<double>\n");
374 else if (typeHnd == Vector256IntHandle)
376 simdBaseType = TYP_INT;
377 size = Vector256SizeBytes;
378 JITDUMP(" Known type Vector256<int>\n");
380 else if (typeHnd == Vector256UIntHandle)
382 simdBaseType = TYP_UINT;
383 size = Vector256SizeBytes;
384 JITDUMP(" Known type Vector256<uint>\n");
386 else if (typeHnd == Vector256ShortHandle)
388 simdBaseType = TYP_SHORT;
389 size = Vector256SizeBytes;
390 JITDUMP(" Known type Vector256<short>\n");
392 else if (typeHnd == Vector256UShortHandle)
394 simdBaseType = TYP_USHORT;
395 size = Vector256SizeBytes;
396 JITDUMP(" Known type Vector256<ushort>\n");
398 else if (typeHnd == Vector256ByteHandle)
400 simdBaseType = TYP_BYTE;
401 size = Vector256SizeBytes;
402 JITDUMP(" Known type Vector256<sbyte>\n");
404 else if (typeHnd == Vector256UByteHandle)
406 simdBaseType = TYP_UBYTE;
407 size = Vector256SizeBytes;
408 JITDUMP(" Known type Vector256<byte>\n");
410 else if (typeHnd == Vector256LongHandle)
412 simdBaseType = TYP_LONG;
413 size = Vector256SizeBytes;
414 JITDUMP(" Known type Vector256<long>\n");
416 else if (typeHnd == Vector256ULongHandle)
418 simdBaseType = TYP_ULONG;
419 size = Vector256SizeBytes;
420 JITDUMP(" Known type Vector256<ulong>\n");
422 else if (typeHnd == Vector256FloatHandle)
424 simdBaseType = TYP_FLOAT;
425 size = Vector256SizeBytes;
426 JITDUMP(" Known type Vector256<float>\n");
429 #endif // defined(_TARGET_XARCH)
430 if (typeHnd == Vector128DoubleHandle)
432 simdBaseType = TYP_DOUBLE;
433 size = Vector128SizeBytes;
434 JITDUMP(" Known type Vector128<double>\n");
436 else if (typeHnd == Vector128IntHandle)
438 simdBaseType = TYP_INT;
439 size = Vector128SizeBytes;
440 JITDUMP(" Known type Vector128<int>\n");
442 else if (typeHnd == Vector128UIntHandle)
444 simdBaseType = TYP_UINT;
445 size = Vector128SizeBytes;
446 JITDUMP(" Known type Vector128<uint>\n");
448 else if (typeHnd == Vector128ShortHandle)
450 simdBaseType = TYP_SHORT;
451 size = Vector128SizeBytes;
452 JITDUMP(" Known type Vector128<short>\n");
454 else if (typeHnd == Vector128UShortHandle)
456 simdBaseType = TYP_USHORT;
457 size = Vector128SizeBytes;
458 JITDUMP(" Known type Vector128<ushort>\n");
460 else if (typeHnd == Vector128ByteHandle)
462 simdBaseType = TYP_BYTE;
463 size = Vector128SizeBytes;
464 JITDUMP(" Known type Vector128<sbyte>\n");
466 else if (typeHnd == Vector128UByteHandle)
468 simdBaseType = TYP_UBYTE;
469 size = Vector128SizeBytes;
470 JITDUMP(" Known type Vector128<byte>\n");
472 else if (typeHnd == Vector128LongHandle)
474 simdBaseType = TYP_LONG;
475 size = Vector128SizeBytes;
476 JITDUMP(" Known type Vector128<long>\n");
478 else if (typeHnd == Vector128ULongHandle)
480 simdBaseType = TYP_ULONG;
481 size = Vector128SizeBytes;
482 JITDUMP(" Known type Vector128<ulong>\n");
484 #if defined(_TARGET_ARM64_)
485 else if (typeHnd == Vector64DoubleHandle)
487 simdBaseType = TYP_DOUBLE;
488 size = Vector64SizeBytes;
489 JITDUMP(" Known type Vector64<double>\n");
491 else if (typeHnd == Vector64IntHandle)
493 simdBaseType = TYP_INT;
494 size = Vector64SizeBytes;
495 JITDUMP(" Known type Vector64<int>\n");
497 else if (typeHnd == Vector64UIntHandle)
499 simdBaseType = TYP_UINT;
500 size = Vector64SizeBytes;
501 JITDUMP(" Known type Vector64<uint>\n");
503 else if (typeHnd == Vector64ShortHandle)
505 simdBaseType = TYP_SHORT;
506 size = Vector64SizeBytes;
507 JITDUMP(" Known type Vector64<short>\n");
509 else if (typeHnd == Vector64UShortHandle)
511 simdBaseType = TYP_USHORT;
512 size = Vector64SizeBytes;
513 JITDUMP(" Known type Vector64<ushort>\n");
515 else if (typeHnd == Vector64ByteHandle)
517 simdBaseType = TYP_BYTE;
518 size = Vector64SizeBytes;
519 JITDUMP(" Known type Vector64<sbyte>\n");
521 else if (typeHnd == Vector64UByteHandle)
523 simdBaseType = TYP_UBYTE;
524 size = Vector64SizeBytes;
525 JITDUMP(" Known type Vector64<byte>\n");
527 else if (typeHnd == Vector64LongHandle)
529 simdBaseType = TYP_LONG;
530 size = Vector64SizeBytes;
531 JITDUMP(" Known type Vector64<long>\n");
533 else if (typeHnd == Vector64ULongHandle)
535 simdBaseType = TYP_ULONG;
536 size = Vector64SizeBytes;
537 JITDUMP(" Known type Vector64<ulong>\n");
539 #endif // defined(_TARGET_ARM64_)
542 if (simdBaseType == TYP_UNKNOWN)
544 // Doesn't match with any of the cached type handles.
545 const char* className = getClassNameFromMetadata(typeHnd, nullptr);
546 CORINFO_CLASS_HANDLE baseTypeHnd = getTypeInstantiationArgument(typeHnd, 0);
548 if (baseTypeHnd != nullptr)
550 CorInfoType type = info.compCompHnd->getTypeForPrimitiveNumericClass(baseTypeHnd);
552 JITDUMP("HW Intrinsic SIMD Candidate Type %s with Base Type %s\n", className,
553 getClassNameFromMetadata(baseTypeHnd, nullptr));
555 #if defined(_TARGET_XARCH_)
556 if (strcmp(className, "Vector256`1") == 0)
558 size = Vector256SizeBytes;
561 case CORINFO_TYPE_FLOAT:
562 Vector256FloatHandle = typeHnd;
563 simdBaseType = TYP_FLOAT;
564 JITDUMP(" Found type Hardware Intrinsic SIMD Vector256<float>\n");
566 case CORINFO_TYPE_DOUBLE:
567 Vector256DoubleHandle = typeHnd;
568 simdBaseType = TYP_DOUBLE;
569 JITDUMP(" Found type Hardware Intrinsic SIMD Vector256<double>\n");
571 case CORINFO_TYPE_INT:
572 Vector256IntHandle = typeHnd;
573 simdBaseType = TYP_INT;
574 JITDUMP(" Found type Hardware Intrinsic SIMD Vector256<int>\n");
576 case CORINFO_TYPE_UINT:
577 Vector256UIntHandle = typeHnd;
578 simdBaseType = TYP_UINT;
579 JITDUMP(" Found type Hardware Intrinsic SIMD Vector256<uint>\n");
581 case CORINFO_TYPE_SHORT:
582 Vector256ShortHandle = typeHnd;
583 simdBaseType = TYP_SHORT;
584 JITDUMP(" Found type Hardware Intrinsic SIMD Vector256<short>\n");
586 case CORINFO_TYPE_USHORT:
587 Vector256UShortHandle = typeHnd;
588 simdBaseType = TYP_USHORT;
589 JITDUMP(" Found type Hardware Intrinsic SIMD Vector256<ushort>\n");
591 case CORINFO_TYPE_LONG:
592 Vector256LongHandle = typeHnd;
593 simdBaseType = TYP_LONG;
594 JITDUMP(" Found type Hardware Intrinsic SIMD Vector256<long>\n");
596 case CORINFO_TYPE_ULONG:
597 Vector256ULongHandle = typeHnd;
598 simdBaseType = TYP_ULONG;
599 JITDUMP(" Found type Hardware Intrinsic SIMD Vector256<ulong>\n");
601 case CORINFO_TYPE_UBYTE:
602 Vector256UByteHandle = typeHnd;
603 simdBaseType = TYP_UBYTE;
604 JITDUMP(" Found type Hardware Intrinsic SIMD Vector256<byte>\n");
606 case CORINFO_TYPE_BYTE:
607 Vector256ByteHandle = typeHnd;
608 simdBaseType = TYP_BYTE;
609 JITDUMP(" Found type Hardware Intrinsic SIMD Vector256<sbyte>\n");
613 JITDUMP(" Unknown Hardware Intrinsic SIMD Type Vector256<T>\n");
617 #endif // defined(_TARGET_XARCH_)
618 if (strcmp(className, "Vector128`1") == 0)
620 size = Vector128SizeBytes;
623 case CORINFO_TYPE_FLOAT:
624 Vector128FloatHandle = typeHnd;
625 simdBaseType = TYP_FLOAT;
626 JITDUMP(" Found type Hardware Intrinsic SIMD Vector128<float>\n");
628 case CORINFO_TYPE_DOUBLE:
629 Vector128DoubleHandle = typeHnd;
630 simdBaseType = TYP_DOUBLE;
631 JITDUMP(" Found type Hardware Intrinsic SIMD Vector128<double>\n");
633 case CORINFO_TYPE_INT:
634 Vector128IntHandle = typeHnd;
635 simdBaseType = TYP_INT;
636 JITDUMP(" Found type Hardware Intrinsic SIMD Vector128<int>\n");
638 case CORINFO_TYPE_UINT:
639 Vector128UIntHandle = typeHnd;
640 simdBaseType = TYP_UINT;
641 JITDUMP(" Found type Hardware Intrinsic SIMD Vector128<uint>\n");
643 case CORINFO_TYPE_SHORT:
644 Vector128ShortHandle = typeHnd;
645 simdBaseType = TYP_SHORT;
646 JITDUMP(" Found type Hardware Intrinsic SIMD Vector128<short>\n");
648 case CORINFO_TYPE_USHORT:
649 Vector128UShortHandle = typeHnd;
650 simdBaseType = TYP_USHORT;
651 JITDUMP(" Found type Hardware Intrinsic SIMD Vector128<ushort>\n");
653 case CORINFO_TYPE_LONG:
654 Vector128LongHandle = typeHnd;
655 simdBaseType = TYP_LONG;
656 JITDUMP(" Found type Hardware Intrinsic SIMD Vector128<long>\n");
658 case CORINFO_TYPE_ULONG:
659 Vector128ULongHandle = typeHnd;
660 simdBaseType = TYP_ULONG;
661 JITDUMP(" Found type Hardware Intrinsic SIMD Vector128<ulong>\n");
663 case CORINFO_TYPE_UBYTE:
664 Vector128UByteHandle = typeHnd;
665 simdBaseType = TYP_UBYTE;
666 JITDUMP(" Found type Hardware Intrinsic SIMD Vector128<byte>\n");
668 case CORINFO_TYPE_BYTE:
669 Vector128ByteHandle = typeHnd;
670 simdBaseType = TYP_BYTE;
671 JITDUMP(" Found type Hardware Intrinsic SIMD Vector128<sbyte>\n");
675 JITDUMP(" Unknown Hardware Intrinsic SIMD Type Vector128<T>\n");
678 #if defined(_TARGET_ARM64_)
679 else if (strcmp(className, "Vector64`1") == 0)
681 size = Vector64SizeBytes;
684 case CORINFO_TYPE_FLOAT:
685 Vector64FloatHandle = typeHnd;
686 simdBaseType = TYP_FLOAT;
687 JITDUMP(" Found type Hardware Intrinsic SIMD Vector64<float>\n");
689 case CORINFO_TYPE_DOUBLE:
690 Vector64DoubleHandle = typeHnd;
691 simdBaseType = TYP_DOUBLE;
692 JITDUMP(" Found type Hardware Intrinsic SIMD Vector64<double>\n");
694 case CORINFO_TYPE_INT:
695 Vector64IntHandle = typeHnd;
696 simdBaseType = TYP_INT;
697 JITDUMP(" Found type Hardware Intrinsic SIMD Vector64<int>\n");
699 case CORINFO_TYPE_UINT:
700 Vector64UIntHandle = typeHnd;
701 simdBaseType = TYP_UINT;
702 JITDUMP(" Found type Hardware Intrinsic SIMD Vector64<uint>\n");
704 case CORINFO_TYPE_SHORT:
705 Vector64ShortHandle = typeHnd;
706 simdBaseType = TYP_SHORT;
707 JITDUMP(" Found type Hardware Intrinsic SIMD Vector64<short>\n");
709 case CORINFO_TYPE_USHORT:
710 Vector64UShortHandle = typeHnd;
711 simdBaseType = TYP_USHORT;
712 JITDUMP(" Found type Hardware Intrinsic SIMD Vector64<ushort>\n");
714 case CORINFO_TYPE_LONG:
715 Vector64LongHandle = typeHnd;
716 simdBaseType = TYP_LONG;
717 JITDUMP(" Found type Hardware Intrinsic SIMD Vector64<long>\n");
719 case CORINFO_TYPE_ULONG:
720 Vector64ULongHandle = typeHnd;
721 simdBaseType = TYP_ULONG;
722 JITDUMP(" Found type Hardware Intrinsic SIMD Vector64<ulong>\n");
724 case CORINFO_TYPE_UBYTE:
725 Vector64UByteHandle = typeHnd;
726 simdBaseType = TYP_UBYTE;
727 JITDUMP(" Found type Hardware Intrinsic SIMD Vector64<byte>\n");
729 case CORINFO_TYPE_BYTE:
730 Vector64ByteHandle = typeHnd;
731 simdBaseType = TYP_BYTE;
732 JITDUMP(" Found type Hardware Intrinsic SIMD Vector64<sbyte>\n");
736 JITDUMP(" Unknown Hardware Intrinsic SIMD Type Vector64<T>\n");
739 #endif // defined(_TARGET_ARM64_)
743 if (simdBaseType != TYP_UNKNOWN && sizeBytes != nullptr)
746 setUsesSIMDTypes(true);
749 #endif // FEATURE_HW_INTRINSICS
754 //--------------------------------------------------------------------------------------
755 // getSIMDIntrinsicInfo: get SIMD intrinsic info given the method handle.
758 // inOutTypeHnd - The handle of the type on which the method is invoked. This is an in-out param.
759 // methodHnd - The handle of the method we're interested in.
760 // sig - method signature info
761 // isNewObj - whether this call represents a newboj constructor call
762 // argCount - argument count - out pram
763 // baseType - base type of the intrinsic - out param
764 // sizeBytes - size of SIMD vector type on which the method is invoked - out param
767 // SIMDIntrinsicInfo struct initialized corresponding to methodHnd.
768 // Sets SIMDIntrinsicInfo.id to SIMDIntrinsicInvalid if methodHnd doesn't correspond
769 // to any SIMD intrinsic. Also, sets the out params inOutTypeHnd, argCount, baseType and
772 // Note that VectorMath class doesn't have a base type and first argument of the method
773 // determines the SIMD vector type on which intrinsic is invoked. In such a case inOutTypeHnd
774 // is modified by this routine.
776 // TODO-Throughput: The current implementation is based on method name string parsing.
777 // Although we now have type identification from the VM, the parsing of intrinsic names
778 // could be made more efficient.
780 const SIMDIntrinsicInfo* Compiler::getSIMDIntrinsicInfo(CORINFO_CLASS_HANDLE* inOutTypeHnd,
781 CORINFO_METHOD_HANDLE methodHnd,
782 CORINFO_SIG_INFO* sig,
789 assert(baseType != nullptr);
790 assert(sizeBytes != nullptr);
792 // get baseType and size of the type
793 CORINFO_CLASS_HANDLE typeHnd = *inOutTypeHnd;
794 *baseType = getBaseTypeAndSizeOfSIMDType(typeHnd, sizeBytes);
796 if (typeHnd == SIMDVectorHandle)
798 // All of the supported intrinsics on this static class take a first argument that's a vector,
799 // which determines the baseType.
800 // The exception is the IsHardwareAccelerated property, which is handled as a special case.
801 assert(*baseType == TYP_UNKNOWN);
802 if (sig->numArgs == 0)
804 const SIMDIntrinsicInfo* hwAccelIntrinsicInfo = &(simdIntrinsicInfoArray[SIMDIntrinsicHWAccel]);
805 if ((strcmp(eeGetMethodName(methodHnd, nullptr), hwAccelIntrinsicInfo->methodName) == 0) &&
806 JITtype2varType(sig->retType) == hwAccelIntrinsicInfo->retType)
809 assert(hwAccelIntrinsicInfo->argCount == 0 && hwAccelIntrinsicInfo->isInstMethod == false);
810 return hwAccelIntrinsicInfo;
816 typeHnd = info.compCompHnd->getArgClass(sig, sig->args);
817 *inOutTypeHnd = typeHnd;
818 *baseType = getBaseTypeAndSizeOfSIMDType(typeHnd, sizeBytes);
822 if (*baseType == TYP_UNKNOWN)
824 JITDUMP("NOT a SIMD Intrinsic: unsupported baseType\n");
828 // account for implicit "this" arg
829 *argCount = sig->numArgs;
835 // Get the Intrinsic Id by parsing method name.
837 // TODO-Throughput: replace sequential search by binary search by arranging entries
838 // sorted by method name.
839 SIMDIntrinsicID intrinsicId = SIMDIntrinsicInvalid;
840 const char* methodName = eeGetMethodName(methodHnd, nullptr);
841 for (int i = SIMDIntrinsicNone + 1; i < SIMDIntrinsicInvalid; ++i)
843 if (strcmp(methodName, simdIntrinsicInfoArray[i].methodName) == 0)
845 // Found an entry for the method; further check whether it is one of
846 // the supported base types.
848 for (int j = 0; j < SIMD_INTRINSIC_MAX_BASETYPE_COUNT; ++j)
850 // Convention: if there are fewer base types supported than MAX_BASETYPE_COUNT,
851 // the end of the list is marked by TYP_UNDEF.
852 if (simdIntrinsicInfoArray[i].supportedBaseTypes[j] == TYP_UNDEF)
857 if (simdIntrinsicInfoArray[i].supportedBaseTypes[j] == *baseType)
869 // Now, check the arguments.
870 unsigned int fixedArgCnt = simdIntrinsicInfoArray[i].argCount;
871 unsigned int expectedArgCnt = fixedArgCnt;
873 // First handle SIMDIntrinsicInitN, where the arg count depends on the type.
874 // The listed arg types include the vector and the first two init values, which is the expected number
875 // for Vector2. For other cases, we'll check their types here.
876 if (*argCount > expectedArgCnt)
878 if (i == SIMDIntrinsicInitN)
880 if (*argCount == 3 && typeHnd == SIMDVector2Handle)
884 else if (*argCount == 4 && typeHnd == SIMDVector3Handle)
888 else if (*argCount == 5 && typeHnd == SIMDVector4Handle)
893 else if (i == SIMDIntrinsicInitFixed)
895 if (*argCount == 4 && typeHnd == SIMDVector4Handle)
901 if (*argCount != expectedArgCnt)
906 // Validate the types of individual args passed are what is expected of.
907 // If any of the types don't match with what is expected, don't consider
908 // as an intrinsic. This will make an older JIT with SIMD capabilities
909 // resilient to breaking changes to SIMD managed API.
911 // Note that from IL type stack, args get popped in right to left order
912 // whereas args get listed in method signatures in left to right order.
914 int stackIndex = (expectedArgCnt - 1);
916 // Track the arguments from the signature - we currently only use this to distinguish
917 // integral and pointer types, both of which will by TYP_I_IMPL on the importer stack.
918 CORINFO_ARG_LIST_HANDLE argLst = sig->args;
920 CORINFO_CLASS_HANDLE argClass;
921 for (unsigned int argIndex = 0; found == true && argIndex < expectedArgCnt; argIndex++)
923 bool isThisPtr = ((argIndex == 0) && sig->hasThis());
925 // In case of "newobj SIMDVector<T>(T val)", thisPtr won't be present on type stack.
926 // We don't check anything in that case.
927 if (!isThisPtr || !isNewObj)
929 GenTreePtr arg = impStackTop(stackIndex).val;
930 var_types argType = arg->TypeGet();
932 var_types expectedArgType;
933 if (argIndex < fixedArgCnt)
936 // - intrinsicInfo.argType[i] == TYP_UNDEF - intrinsic doesn't have a valid arg at position i
937 // - intrinsicInfo.argType[i] == TYP_UNKNOWN - arg type should be same as basetype
938 // Note that we pop the args off in reverse order.
939 expectedArgType = simdIntrinsicInfoArray[i].argType[argIndex];
940 assert(expectedArgType != TYP_UNDEF);
941 if (expectedArgType == TYP_UNKNOWN)
943 // The type of the argument will be genActualType(*baseType).
944 expectedArgType = genActualType(*baseType);
945 argType = genActualType(argType);
950 expectedArgType = *baseType;
953 if (!isThisPtr && argType == TYP_I_IMPL)
955 // The reference implementation has a constructor that takes a pointer.
956 // We don't want to recognize that one. This requires us to look at the CorInfoType
957 // in order to distinguish a signature with a pointer argument from one with an
958 // integer argument of pointer size, both of which will be TYP_I_IMPL on the stack.
959 // TODO-Review: This seems quite fragile. We should consider beefing up the checking
961 CorInfoType corType = strip(info.compCompHnd->getArgType(sig, argLst, &argClass));
962 if (corType == CORINFO_TYPE_PTR)
968 if (varTypeIsSIMD(argType))
970 argType = TYP_STRUCT;
972 if (argType != expectedArgType)
977 if (argIndex != 0 || !sig->hasThis())
979 argLst = info.compCompHnd->getArgNext(argLst);
984 // Cross check return type and static vs. instance is what we are expecting.
985 // If not, don't consider it as an intrinsic.
986 // Note that ret type of TYP_UNKNOWN means that it is not known apriori and must be same as baseType
989 var_types expectedRetType = simdIntrinsicInfoArray[i].retType;
990 if (expectedRetType == TYP_UNKNOWN)
992 // JIT maps uint/ulong type vars to TYP_INT/TYP_LONG.
994 (*baseType == TYP_UINT || *baseType == TYP_ULONG) ? genActualType(*baseType) : *baseType;
997 if (JITtype2varType(sig->retType) != expectedRetType ||
998 sig->hasThis() != simdIntrinsicInfoArray[i].isInstMethod)
1006 intrinsicId = (SIMDIntrinsicID)i;
1012 if (intrinsicId != SIMDIntrinsicInvalid)
1014 JITDUMP("Method %s maps to SIMD intrinsic %s\n", methodName, simdIntrinsicNames[intrinsicId]);
1015 return &simdIntrinsicInfoArray[intrinsicId];
1019 JITDUMP("Method %s is NOT a SIMD intrinsic\n", methodName);
1025 // Pops and returns GenTree node from importer's type stack.
1026 // Normalizes TYP_STRUCT value in case of GT_CALL, GT_RET_EXPR and arg nodes.
1029 // type - the type of value that the caller expects to be popped off the stack.
1030 // expectAddr - if true indicates we are expecting type stack entry to be a TYP_BYREF.
1033 // If the popped value is a struct, and the expected type is a simd type, it will be set
1034 // to that type, otherwise it will assert if the type being popped is not the expected type.
1036 GenTreePtr Compiler::impSIMDPopStack(var_types type, bool expectAddr)
1038 StackEntry se = impPopStack();
1039 typeInfo ti = se.seTypeInfo;
1040 GenTreePtr tree = se.val;
1042 // If expectAddr is true implies what we have on stack is address and we need
1043 // SIMD type struct that it points to.
1046 assert(tree->TypeGet() == TYP_BYREF);
1047 if (tree->OperGet() == GT_ADDR)
1049 tree = tree->gtGetOp1();
1053 tree = gtNewOperNode(GT_IND, type, tree);
1057 bool isParam = false;
1059 // If we have a ldobj of a SIMD local we need to transform it.
1060 if (tree->OperGet() == GT_OBJ)
1062 GenTree* addr = tree->gtOp.gtOp1;
1063 if ((addr->OperGet() == GT_ADDR) && isSIMDTypeLocal(addr->gtOp.gtOp1))
1065 tree = addr->gtOp.gtOp1;
1069 if (tree->OperGet() == GT_LCL_VAR)
1071 unsigned lclNum = tree->AsLclVarCommon()->GetLclNum();
1072 LclVarDsc* lclVarDsc = &lvaTable[lclNum];
1073 isParam = lclVarDsc->lvIsParam;
1076 // normalize TYP_STRUCT value
1077 if (varTypeIsStruct(tree) && ((tree->OperGet() == GT_RET_EXPR) || (tree->OperGet() == GT_CALL) || isParam))
1079 assert(ti.IsType(TI_STRUCT));
1080 CORINFO_CLASS_HANDLE structType = ti.GetClassHandleForValueClass();
1081 tree = impNormStructVal(tree, structType, (unsigned)CHECK_SPILL_ALL);
1084 // Now set the type of the tree to the specialized SIMD struct type, if applicable.
1085 if (genActualType(tree->gtType) != genActualType(type))
1087 assert(tree->gtType == TYP_STRUCT);
1088 tree->gtType = type;
1090 else if (tree->gtType == TYP_BYREF)
1092 assert(tree->IsLocal() || (tree->OperGet() == GT_RET_EXPR) || (tree->OperGet() == GT_CALL) ||
1093 ((tree->gtOper == GT_ADDR) && varTypeIsSIMD(tree->gtGetOp1())));
1099 // impSIMDGetFixed: Create a GT_SIMD tree for a Get property of SIMD vector with a fixed index.
1102 // baseType - The base (element) type of the SIMD vector.
1103 // simdSize - The total size in bytes of the SIMD vector.
1104 // index - The index of the field to get.
1107 // Returns a GT_SIMD node with the SIMDIntrinsicGetItem intrinsic id.
1109 GenTreeSIMD* Compiler::impSIMDGetFixed(var_types simdType, var_types baseType, unsigned simdSize, int index)
1111 assert(simdSize >= ((index + 1) * genTypeSize(baseType)));
1113 // op1 is a SIMD source.
1114 GenTree* op1 = impSIMDPopStack(simdType, true);
1116 GenTree* op2 = gtNewIconNode(index);
1117 GenTreeSIMD* simdTree = gtNewSIMDNode(baseType, op1, op2, SIMDIntrinsicGetItem, baseType, simdSize);
1121 #ifdef _TARGET_XARCH_
1122 // impSIMDLongRelOpEqual: transforms operands and returns the SIMD intrinsic to be applied on
1123 // transformed operands to obtain == comparison result.
1126 // typeHnd - type handle of SIMD vector
1127 // size - SIMD vector size
1128 // op1 - in-out parameter; first operand
1129 // op2 - in-out parameter; second operand
1132 // Modifies in-out params op1, op2 and returns intrinsic ID to be applied to modified operands
1134 SIMDIntrinsicID Compiler::impSIMDLongRelOpEqual(CORINFO_CLASS_HANDLE typeHnd,
1139 var_types simdType = (*pOp1)->TypeGet();
1140 assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1142 // There is no direct SSE2 support for comparing TYP_LONG vectors.
1143 // These have to be implemented in terms of TYP_INT vector comparison operations.
1145 // Equality(v1, v2):
1146 // tmp = (v1 == v2) i.e. compare for equality as if v1 and v2 are vector<int>
1147 // result = BitwiseAnd(t, shuffle(t, (2, 3, 0, 1)))
1148 // Shuffle is meant to swap the comparison results of low-32-bits and high 32-bits of respective long elements.
1150 // Compare vector<long> as if they were vector<int> and assign the result to a temp
1151 GenTree* compResult = gtNewSIMDNode(simdType, *pOp1, *pOp2, SIMDIntrinsicEqual, TYP_INT, size);
1152 unsigned lclNum = lvaGrabTemp(true DEBUGARG("SIMD Long =="));
1153 lvaSetStruct(lclNum, typeHnd, false);
1154 GenTree* tmp = gtNewLclvNode(lclNum, simdType);
1155 GenTree* asg = gtNewTempAssign(lclNum, compResult);
1157 // op1 = GT_COMMA(tmp=compResult, tmp)
1158 // op2 = Shuffle(tmp, 0xB1)
1159 // IntrinsicId = BitwiseAnd
1160 *pOp1 = gtNewOperNode(GT_COMMA, simdType, asg, tmp);
1161 *pOp2 = gtNewSIMDNode(simdType, gtNewLclvNode(lclNum, simdType), gtNewIconNode(SHUFFLE_ZWXY, TYP_INT),
1162 SIMDIntrinsicShuffleSSE2, TYP_INT, size);
1163 return SIMDIntrinsicBitwiseAnd;
1166 // impSIMDLongRelOpGreaterThan: transforms operands and returns the SIMD intrinsic to be applied on
1167 // transformed operands to obtain > comparison result.
1170 // typeHnd - type handle of SIMD vector
1171 // size - SIMD vector size
1172 // pOp1 - in-out parameter; first operand
1173 // pOp2 - in-out parameter; second operand
1176 // Modifies in-out params pOp1, pOp2 and returns intrinsic ID to be applied to modified operands
1178 SIMDIntrinsicID Compiler::impSIMDLongRelOpGreaterThan(CORINFO_CLASS_HANDLE typeHnd,
1183 var_types simdType = (*pOp1)->TypeGet();
1184 assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1186 // GreaterThan(v1, v2) where v1 and v2 are vector long.
1187 // Let us consider the case of single long element comparison.
1188 // say L1 = (x1, y1) and L2 = (x2, y2) where x1, y1, x2, and y2 are 32-bit integers that comprise the longs L1 and
1191 // GreaterThan(L1, L2) can be expressed in terms of > relationship between 32-bit integers that comprise L1 and L2
1193 // = (x1, y1) > (x2, y2)
1194 // = (x1 > x2) || [(x1 == x2) && (y1 > y2)] - eq (1)
1196 // t = (v1 > v2) 32-bit signed comparison
1197 // u = (v1 == v2) 32-bit sized element equality
1198 // v = (v1 > v2) 32-bit unsigned comparison
1200 // z = shuffle(t, (3, 3, 1, 1)) - This corresponds to (x1 > x2) in eq(1) above
1201 // t1 = Shuffle(v, (2, 2, 0, 0)) - This corresponds to (y1 > y2) in eq(1) above
1202 // u1 = Shuffle(u, (3, 3, 1, 1)) - This corresponds to (x1 == x2) in eq(1) above
1203 // w = And(t1, u1) - This corresponds to [(x1 == x2) && (y1 > y2)] in eq(1) above
1204 // Result = BitwiseOr(z, w)
1206 // Since op1 and op2 gets used multiple times, make sure side effects are computed.
1207 GenTree* dupOp1 = nullptr;
1208 GenTree* dupOp2 = nullptr;
1209 GenTree* dupDupOp1 = nullptr;
1210 GenTree* dupDupOp2 = nullptr;
1212 if (((*pOp1)->gtFlags & GTF_SIDE_EFFECT) != 0)
1214 dupOp1 = fgInsertCommaFormTemp(pOp1, typeHnd);
1215 dupDupOp1 = gtNewLclvNode(dupOp1->AsLclVarCommon()->GetLclNum(), simdType);
1219 dupOp1 = gtCloneExpr(*pOp1);
1220 dupDupOp1 = gtCloneExpr(*pOp1);
1223 if (((*pOp2)->gtFlags & GTF_SIDE_EFFECT) != 0)
1225 dupOp2 = fgInsertCommaFormTemp(pOp2, typeHnd);
1226 dupDupOp2 = gtNewLclvNode(dupOp2->AsLclVarCommon()->GetLclNum(), simdType);
1230 dupOp2 = gtCloneExpr(*pOp2);
1231 dupDupOp2 = gtCloneExpr(*pOp2);
1234 assert(dupDupOp1 != nullptr && dupDupOp2 != nullptr);
1235 assert(dupOp1 != nullptr && dupOp2 != nullptr);
1236 assert(*pOp1 != nullptr && *pOp2 != nullptr);
1238 // v1GreaterThanv2Signed - signed 32-bit comparison
1239 GenTree* v1GreaterThanv2Signed = gtNewSIMDNode(simdType, *pOp1, *pOp2, SIMDIntrinsicGreaterThan, TYP_INT, size);
1241 // v1Equalsv2 - 32-bit equality
1242 GenTree* v1Equalsv2 = gtNewSIMDNode(simdType, dupOp1, dupOp2, SIMDIntrinsicEqual, TYP_INT, size);
1244 // v1GreaterThanv2Unsigned - unsigned 32-bit comparison
1245 var_types tempBaseType = TYP_UINT;
1246 SIMDIntrinsicID sid = impSIMDRelOp(SIMDIntrinsicGreaterThan, typeHnd, size, &tempBaseType, &dupDupOp1, &dupDupOp2);
1247 GenTree* v1GreaterThanv2Unsigned = gtNewSIMDNode(simdType, dupDupOp1, dupDupOp2, sid, tempBaseType, size);
1249 GenTree* z = gtNewSIMDNode(simdType, v1GreaterThanv2Signed, gtNewIconNode(SHUFFLE_WWYY, TYP_INT),
1250 SIMDIntrinsicShuffleSSE2, TYP_FLOAT, size);
1251 GenTree* t1 = gtNewSIMDNode(simdType, v1GreaterThanv2Unsigned, gtNewIconNode(SHUFFLE_ZZXX, TYP_INT),
1252 SIMDIntrinsicShuffleSSE2, TYP_FLOAT, size);
1253 GenTree* u1 = gtNewSIMDNode(simdType, v1Equalsv2, gtNewIconNode(SHUFFLE_WWYY, TYP_INT), SIMDIntrinsicShuffleSSE2,
1255 GenTree* w = gtNewSIMDNode(simdType, u1, t1, SIMDIntrinsicBitwiseAnd, TYP_INT, size);
1259 return SIMDIntrinsicBitwiseOr;
1262 // impSIMDLongRelOpGreaterThanOrEqual: transforms operands and returns the SIMD intrinsic to be applied on
1263 // transformed operands to obtain >= comparison result.
1266 // typeHnd - type handle of SIMD vector
1267 // size - SIMD vector size
1268 // pOp1 - in-out parameter; first operand
1269 // pOp2 - in-out parameter; second operand
1272 // Modifies in-out params pOp1, pOp2 and returns intrinsic ID to be applied to modified operands
1274 SIMDIntrinsicID Compiler::impSIMDLongRelOpGreaterThanOrEqual(CORINFO_CLASS_HANDLE typeHnd,
1279 var_types simdType = (*pOp1)->TypeGet();
1280 assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1282 // expand this to (a == b) | (a > b)
1283 GenTree* dupOp1 = nullptr;
1284 GenTree* dupOp2 = nullptr;
1286 if (((*pOp1)->gtFlags & GTF_SIDE_EFFECT) != 0)
1288 dupOp1 = fgInsertCommaFormTemp(pOp1, typeHnd);
1292 dupOp1 = gtCloneExpr(*pOp1);
1295 if (((*pOp2)->gtFlags & GTF_SIDE_EFFECT) != 0)
1297 dupOp2 = fgInsertCommaFormTemp(pOp2, typeHnd);
1301 dupOp2 = gtCloneExpr(*pOp2);
1304 assert(dupOp1 != nullptr && dupOp2 != nullptr);
1305 assert(*pOp1 != nullptr && *pOp2 != nullptr);
1308 SIMDIntrinsicID id = impSIMDLongRelOpEqual(typeHnd, size, pOp1, pOp2);
1309 *pOp1 = gtNewSIMDNode(simdType, *pOp1, *pOp2, id, TYP_LONG, size);
1312 id = impSIMDLongRelOpGreaterThan(typeHnd, size, &dupOp1, &dupOp2);
1313 *pOp2 = gtNewSIMDNode(simdType, dupOp1, dupOp2, id, TYP_LONG, size);
1315 return SIMDIntrinsicBitwiseOr;
1318 // impSIMDInt32OrSmallIntRelOpGreaterThanOrEqual: transforms operands and returns the SIMD intrinsic to be applied on
1319 // transformed operands to obtain >= comparison result in case of integer base type vectors
1322 // typeHnd - type handle of SIMD vector
1323 // size - SIMD vector size
1324 // baseType - base type of SIMD vector
1325 // pOp1 - in-out parameter; first operand
1326 // pOp2 - in-out parameter; second operand
1329 // Modifies in-out params pOp1, pOp2 and returns intrinsic ID to be applied to modified operands
1331 SIMDIntrinsicID Compiler::impSIMDIntegralRelOpGreaterThanOrEqual(
1332 CORINFO_CLASS_HANDLE typeHnd, unsigned size, var_types baseType, GenTree** pOp1, GenTree** pOp2)
1334 var_types simdType = (*pOp1)->TypeGet();
1335 assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1337 // This routine should be used only for integer base type vectors
1338 assert(varTypeIsIntegral(baseType));
1339 if ((getSIMDSupportLevel() == SIMD_SSE2_Supported) && ((baseType == TYP_LONG) || baseType == TYP_UBYTE))
1341 return impSIMDLongRelOpGreaterThanOrEqual(typeHnd, size, pOp1, pOp2);
1344 // expand this to (a == b) | (a > b)
1345 GenTree* dupOp1 = nullptr;
1346 GenTree* dupOp2 = nullptr;
1348 if (((*pOp1)->gtFlags & GTF_SIDE_EFFECT) != 0)
1350 dupOp1 = fgInsertCommaFormTemp(pOp1, typeHnd);
1354 dupOp1 = gtCloneExpr(*pOp1);
1357 if (((*pOp2)->gtFlags & GTF_SIDE_EFFECT) != 0)
1359 dupOp2 = fgInsertCommaFormTemp(pOp2, typeHnd);
1363 dupOp2 = gtCloneExpr(*pOp2);
1366 assert(dupOp1 != nullptr && dupOp2 != nullptr);
1367 assert(*pOp1 != nullptr && *pOp2 != nullptr);
1370 *pOp1 = gtNewSIMDNode(simdType, *pOp1, *pOp2, SIMDIntrinsicEqual, baseType, size);
1373 *pOp2 = gtNewSIMDNode(simdType, dupOp1, dupOp2, SIMDIntrinsicGreaterThan, baseType, size);
1375 return SIMDIntrinsicBitwiseOr;
1377 #endif // _TARGET_XARCH_
1379 // Transforms operands and returns the SIMD intrinsic to be applied on
1380 // transformed operands to obtain given relop result.
1383 // relOpIntrinsicId - Relational operator SIMD intrinsic
1384 // typeHnd - type handle of SIMD vector
1385 // size - SIMD vector size
1386 // inOutBaseType - base type of SIMD vector
1387 // pOp1 - in-out parameter; first operand
1388 // pOp2 - in-out parameter; second operand
1391 // Modifies in-out params pOp1, pOp2, inOutBaseType and returns intrinsic ID to be applied to modified operands
1393 SIMDIntrinsicID Compiler::impSIMDRelOp(SIMDIntrinsicID relOpIntrinsicId,
1394 CORINFO_CLASS_HANDLE typeHnd,
1396 var_types* inOutBaseType,
1400 var_types simdType = (*pOp1)->TypeGet();
1401 assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1403 assert(isRelOpSIMDIntrinsic(relOpIntrinsicId));
1405 SIMDIntrinsicID intrinsicID = relOpIntrinsicId;
1406 #ifdef _TARGET_XARCH_
1407 var_types baseType = *inOutBaseType;
1409 if (varTypeIsFloating(baseType))
1411 // SSE2/AVX doesn't support > and >= on vector float/double.
1412 // Therefore, we need to use < and <= with swapped operands
1413 if (relOpIntrinsicId == SIMDIntrinsicGreaterThan || relOpIntrinsicId == SIMDIntrinsicGreaterThanOrEqual)
1415 GenTree* tmp = *pOp1;
1420 (relOpIntrinsicId == SIMDIntrinsicGreaterThan) ? SIMDIntrinsicLessThan : SIMDIntrinsicLessThanOrEqual;
1423 else if (varTypeIsIntegral(baseType))
1425 // SSE/AVX doesn't support < and <= on integer base type vectors.
1426 // Therefore, we need to use > and >= with swapped operands.
1427 if (intrinsicID == SIMDIntrinsicLessThan || intrinsicID == SIMDIntrinsicLessThanOrEqual)
1429 GenTree* tmp = *pOp1;
1433 intrinsicID = (relOpIntrinsicId == SIMDIntrinsicLessThan) ? SIMDIntrinsicGreaterThan
1434 : SIMDIntrinsicGreaterThanOrEqual;
1437 if ((getSIMDSupportLevel() == SIMD_SSE2_Supported) && baseType == TYP_LONG)
1439 // There is no direct SSE2 support for comparing TYP_LONG vectors.
1440 // These have to be implemented interms of TYP_INT vector comparison operations.
1441 if (intrinsicID == SIMDIntrinsicEqual)
1443 intrinsicID = impSIMDLongRelOpEqual(typeHnd, size, pOp1, pOp2);
1445 else if (intrinsicID == SIMDIntrinsicGreaterThan)
1447 intrinsicID = impSIMDLongRelOpGreaterThan(typeHnd, size, pOp1, pOp2);
1449 else if (intrinsicID == SIMDIntrinsicGreaterThanOrEqual)
1451 intrinsicID = impSIMDLongRelOpGreaterThanOrEqual(typeHnd, size, pOp1, pOp2);
1458 // SSE2 and AVX direct support for signed comparison of int32, int16 and int8 types
1459 else if (!varTypeIsUnsigned(baseType))
1461 if (intrinsicID == SIMDIntrinsicGreaterThanOrEqual)
1463 intrinsicID = impSIMDIntegralRelOpGreaterThanOrEqual(typeHnd, size, baseType, pOp1, pOp2);
1468 // Vector<byte>, Vector<ushort>, Vector<uint> and Vector<ulong>:
1469 // SSE2 supports > for signed comparison. Therefore, to use it for
1470 // comparing unsigned numbers, we subtract a constant from both the
1471 // operands such that the result fits within the corresponding signed
1472 // type. The resulting signed numbers are compared using SSE2 signed
1475 // Vector<byte>: constant to be subtracted is 2^7
1476 // Vector<ushort> constant to be subtracted is 2^15
1477 // Vector<uint> constant to be subtracted is 2^31
1478 // Vector<ulong> constant to be subtracted is 2^63
1480 // We need to treat op1 and op2 as signed for comparison purpose after
1481 // the transformation.
1482 __int64 constVal = 0;
1486 constVal = 0x80808080;
1487 *inOutBaseType = TYP_BYTE;
1490 constVal = 0x80008000;
1491 *inOutBaseType = TYP_SHORT;
1494 constVal = 0x80000000;
1495 *inOutBaseType = TYP_INT;
1498 constVal = 0x8000000000000000LL;
1499 *inOutBaseType = TYP_LONG;
1505 assert(constVal != 0);
1507 // This transformation is not required for equality.
1508 if (intrinsicID != SIMDIntrinsicEqual)
1510 // For constructing const vector use either long or int base type.
1511 var_types tempBaseType;
1513 if (baseType == TYP_ULONG)
1515 tempBaseType = TYP_LONG;
1516 initVal = gtNewLconNode(constVal);
1520 tempBaseType = TYP_INT;
1521 initVal = gtNewIconNode((ssize_t)constVal);
1523 initVal->gtType = tempBaseType;
1524 GenTree* constVector = gtNewSIMDNode(simdType, initVal, nullptr, SIMDIntrinsicInit, tempBaseType, size);
1526 // Assign constVector to a temp, since we intend to use it more than once
1527 // TODO-CQ: We have quite a few such constant vectors constructed during
1528 // the importation of SIMD intrinsics. Make sure that we have a single
1529 // temp per distinct constant per method.
1530 GenTree* tmp = fgInsertCommaFormTemp(&constVector, typeHnd);
1532 // op1 = op1 - constVector
1533 // op2 = op2 - constVector
1534 *pOp1 = gtNewSIMDNode(simdType, *pOp1, constVector, SIMDIntrinsicSub, baseType, size);
1535 *pOp2 = gtNewSIMDNode(simdType, *pOp2, tmp, SIMDIntrinsicSub, baseType, size);
1538 return impSIMDRelOp(intrinsicID, typeHnd, size, inOutBaseType, pOp1, pOp2);
1541 #elif defined(_TARGET_ARM64_)
1542 // TODO-ARM64-CQ handle comparisons against zero
1544 // _TARGET_ARM64_ doesn't support < and <= on register register comparisons
1545 // Therefore, we need to use > and >= with swapped operands.
1546 if (intrinsicID == SIMDIntrinsicLessThan || intrinsicID == SIMDIntrinsicLessThanOrEqual)
1548 GenTree* tmp = *pOp1;
1553 (intrinsicID == SIMDIntrinsicLessThan) ? SIMDIntrinsicGreaterThan : SIMDIntrinsicGreaterThanOrEqual;
1555 #else // !_TARGET_XARCH_
1556 assert(!"impSIMDRelOp() unimplemented on target arch");
1558 #endif // !_TARGET_XARCH_
1563 //-------------------------------------------------------------------------
1564 // impSIMDAbs: creates GT_SIMD node to compute Abs value of a given vector.
1567 // typeHnd - type handle of SIMD vector
1568 // baseType - base type of vector
1569 // size - vector size in bytes
1570 // op1 - operand of Abs intrinsic
1572 GenTreePtr Compiler::impSIMDAbs(CORINFO_CLASS_HANDLE typeHnd, var_types baseType, unsigned size, GenTree* op1)
1574 assert(varTypeIsSIMD(op1));
1576 var_types simdType = op1->TypeGet();
1577 GenTreePtr retVal = nullptr;
1579 #ifdef _TARGET_XARCH_
1580 // When there is no direct support, Abs(v) could be computed
1581 // on integer vectors as follows:
1582 // BitVector = v < vector.Zero
1583 // result = ConditionalSelect(BitVector, vector.Zero - v, v)
1585 bool useConditionalSelect = false;
1586 if (getSIMDSupportLevel() == SIMD_SSE2_Supported)
1588 // SSE2 doesn't support abs on signed integer type vectors.
1589 if (baseType == TYP_LONG || baseType == TYP_INT || baseType == TYP_SHORT || baseType == TYP_BYTE)
1591 useConditionalSelect = true;
1596 assert(getSIMDSupportLevel() >= SIMD_SSE4_Supported);
1597 if (baseType == TYP_LONG)
1599 // SSE4/AVX2 don't support abs on long type vector.
1600 useConditionalSelect = true;
1604 if (useConditionalSelect)
1606 // This works only on integer vectors not on float/double vectors.
1607 assert(varTypeIsIntegral(baseType));
1609 GenTreePtr op1Assign;
1612 if (op1->OperGet() == GT_LCL_VAR)
1614 op1LclNum = op1->gtLclVarCommon.gtLclNum;
1615 op1Assign = nullptr;
1619 op1LclNum = lvaGrabTemp(true DEBUGARG("SIMD Abs op1"));
1620 lvaSetStruct(op1LclNum, typeHnd, false);
1621 op1Assign = gtNewTempAssign(op1LclNum, op1);
1622 op1 = gtNewLclvNode(op1LclNum, op1->TypeGet());
1625 // Assign Vector.Zero to a temp since it is needed more than once
1626 GenTreePtr vecZero = gtNewSIMDVectorZero(simdType, baseType, size);
1627 unsigned vecZeroLclNum = lvaGrabTemp(true DEBUGARG("SIMD Abs VecZero"));
1628 lvaSetStruct(vecZeroLclNum, typeHnd, false);
1629 GenTreePtr vecZeroAssign = gtNewTempAssign(vecZeroLclNum, vecZero);
1631 // Construct BitVector = v < vector.Zero
1632 GenTreePtr bitVecOp1 = op1;
1633 GenTreePtr bitVecOp2 = gtNewLclvNode(vecZeroLclNum, vecZero->TypeGet());
1634 var_types relOpBaseType = baseType;
1635 SIMDIntrinsicID relOpIntrinsic =
1636 impSIMDRelOp(SIMDIntrinsicLessThan, typeHnd, size, &relOpBaseType, &bitVecOp1, &bitVecOp2);
1637 GenTreePtr bitVec = gtNewSIMDNode(simdType, bitVecOp1, bitVecOp2, relOpIntrinsic, relOpBaseType, size);
1638 unsigned bitVecLclNum = lvaGrabTemp(true DEBUGARG("SIMD Abs bitVec"));
1639 lvaSetStruct(bitVecLclNum, typeHnd, false);
1640 GenTreePtr bitVecAssign = gtNewTempAssign(bitVecLclNum, bitVec);
1641 bitVec = gtNewLclvNode(bitVecLclNum, bitVec->TypeGet());
1643 // Construct condSelectOp1 = vector.Zero - v
1644 GenTreePtr subOp1 = gtNewLclvNode(vecZeroLclNum, vecZero->TypeGet());
1645 GenTreePtr subOp2 = gtNewLclvNode(op1LclNum, op1->TypeGet());
1646 GenTreePtr negVec = gtNewSIMDNode(simdType, subOp1, subOp2, SIMDIntrinsicSub, baseType, size);
1648 // Construct ConditionalSelect(bitVec, vector.Zero - v, v)
1649 GenTreePtr vec = gtNewLclvNode(op1LclNum, op1->TypeGet());
1650 retVal = impSIMDSelect(typeHnd, baseType, size, bitVec, negVec, vec);
1652 // Prepend bitVec assignment to retVal.
1653 // retVal = (tmp2 = v < tmp1), CondSelect(tmp2, tmp1 - v, v)
1654 retVal = gtNewOperNode(GT_COMMA, simdType, bitVecAssign, retVal);
1656 // Prepend vecZero assignment to retVal.
1657 // retVal = (tmp1 = vector.Zero), (tmp2 = v < tmp1), CondSelect(tmp2, tmp1 - v, v)
1658 retVal = gtNewOperNode(GT_COMMA, simdType, vecZeroAssign, retVal);
1660 // If op1 was assigned to a temp, prepend that to retVal.
1661 if (op1Assign != nullptr)
1663 // retVal = (v=op1), (tmp1 = vector.Zero), (tmp2 = v < tmp1), CondSelect(tmp2, tmp1 - v, v)
1664 retVal = gtNewOperNode(GT_COMMA, simdType, op1Assign, retVal);
1667 else if (varTypeIsFloating(baseType))
1669 // Abs(vf) = vf & new SIMDVector<float>(0x7fffffff);
1670 // Abs(vd) = vf & new SIMDVector<double>(0x7fffffffffffffff);
1671 GenTree* bitMask = nullptr;
1672 if (baseType == TYP_FLOAT)
1675 static_assert_no_msg(sizeof(float) == sizeof(int));
1676 *((int*)&f) = 0x7fffffff;
1677 bitMask = gtNewDconNode(f);
1679 else if (baseType == TYP_DOUBLE)
1682 static_assert_no_msg(sizeof(double) == sizeof(__int64));
1683 *((__int64*)&d) = 0x7fffffffffffffffLL;
1684 bitMask = gtNewDconNode(d);
1687 assert(bitMask != nullptr);
1688 bitMask->gtType = baseType;
1689 GenTree* bitMaskVector = gtNewSIMDNode(simdType, bitMask, SIMDIntrinsicInit, baseType, size);
1690 retVal = gtNewSIMDNode(simdType, op1, bitMaskVector, SIMDIntrinsicBitwiseAnd, baseType, size);
1692 else if (baseType == TYP_USHORT || baseType == TYP_UBYTE || baseType == TYP_UINT || baseType == TYP_ULONG)
1694 // Abs is a no-op on unsigned integer type vectors
1699 assert(getSIMDSupportLevel() >= SIMD_SSE4_Supported);
1700 assert(baseType != TYP_LONG);
1702 retVal = gtNewSIMDNode(simdType, op1, SIMDIntrinsicAbs, baseType, size);
1704 #elif defined(_TARGET_ARM64_)
1705 if (varTypeIsUnsigned(baseType))
1707 // Abs is a no-op on unsigned integer type vectors
1712 retVal = gtNewSIMDNode(simdType, op1, SIMDIntrinsicAbs, baseType, size);
1714 #else // !defined(_TARGET_XARCH)_ && !defined(_TARGET_ARM64_)
1715 assert(!"Abs intrinsic on non-xarch target not implemented");
1716 #endif // !_TARGET_XARCH_
1721 // Creates a GT_SIMD tree for Select operation
1724 // typeHnd - type handle of SIMD vector
1725 // baseType - base type of SIMD vector
1726 // size - SIMD vector size
1727 // op1 - first operand = Condition vector vc
1728 // op2 - second operand = va
1729 // op3 - third operand = vb
1732 // Returns GT_SIMD tree that computes Select(vc, va, vb)
1734 GenTreePtr Compiler::impSIMDSelect(
1735 CORINFO_CLASS_HANDLE typeHnd, var_types baseType, unsigned size, GenTree* op1, GenTree* op2, GenTree* op3)
1737 assert(varTypeIsSIMD(op1));
1738 var_types simdType = op1->TypeGet();
1739 assert(op2->TypeGet() == simdType);
1740 assert(op3->TypeGet() == simdType);
1742 // TODO-ARM64-CQ Support generating select instruction for SIMD
1744 // Select(BitVector vc, va, vb) = (va & vc) | (vb & !vc)
1745 // Select(op1, op2, op3) = (op2 & op1) | (op3 & !op1)
1746 // = SIMDIntrinsicBitwiseOr(SIMDIntrinsicBitwiseAnd(op2, op1),
1747 // SIMDIntrinsicBitwiseAndNot(op3, op1))
1749 // If Op1 has side effect, create an assignment to a temp
1751 GenTree* asg = nullptr;
1752 if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0)
1754 unsigned lclNum = lvaGrabTemp(true DEBUGARG("SIMD Select"));
1755 lvaSetStruct(lclNum, typeHnd, false);
1756 tmp = gtNewLclvNode(lclNum, op1->TypeGet());
1757 asg = gtNewTempAssign(lclNum, op1);
1760 GenTree* andExpr = gtNewSIMDNode(simdType, op2, tmp, SIMDIntrinsicBitwiseAnd, baseType, size);
1761 GenTree* dupOp1 = gtCloneExpr(tmp);
1762 assert(dupOp1 != nullptr);
1763 #ifdef _TARGET_ARM64_
1764 // ARM64 implements SIMDIntrinsicBitwiseAndNot as Left & ~Right
1765 GenTree* andNotExpr = gtNewSIMDNode(simdType, op3, dupOp1, SIMDIntrinsicBitwiseAndNot, baseType, size);
1767 // XARCH implements SIMDIntrinsicBitwiseAndNot as ~Left & Right
1768 GenTree* andNotExpr = gtNewSIMDNode(simdType, dupOp1, op3, SIMDIntrinsicBitwiseAndNot, baseType, size);
1770 GenTree* simdTree = gtNewSIMDNode(simdType, andExpr, andNotExpr, SIMDIntrinsicBitwiseOr, baseType, size);
1772 // If asg not null, create a GT_COMMA tree.
1775 simdTree = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), asg, simdTree);
1781 // Creates a GT_SIMD tree for Min/Max operation
1784 // IntrinsicId - SIMD intrinsic Id, either Min or Max
1785 // typeHnd - type handle of SIMD vector
1786 // baseType - base type of SIMD vector
1787 // size - SIMD vector size
1788 // op1 - first operand = va
1789 // op2 - second operand = vb
1792 // Returns GT_SIMD tree that computes Max(va, vb)
1794 GenTreePtr Compiler::impSIMDMinMax(SIMDIntrinsicID intrinsicId,
1795 CORINFO_CLASS_HANDLE typeHnd,
1801 assert(intrinsicId == SIMDIntrinsicMin || intrinsicId == SIMDIntrinsicMax);
1802 assert(varTypeIsSIMD(op1));
1803 var_types simdType = op1->TypeGet();
1804 assert(op2->TypeGet() == simdType);
1806 #if defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
1807 GenTree* simdTree = nullptr;
1809 #ifdef _TARGET_XARCH_
1810 // SSE2 has direct support for float/double/signed word/unsigned byte.
1811 // SSE4.1 has direct support for int32/uint32/signed byte/unsigned word.
1812 // For other integer types we compute min/max as follows
1814 // int32/uint32 (SSE2)
1815 // int64/uint64 (SSE2&SSE4):
1816 // compResult = (op1 < op2) in case of Min
1817 // (op1 > op2) in case of Max
1818 // Min/Max(op1, op2) = Select(compResult, op1, op2)
1820 // unsigned word (SSE2):
1821 // op1 = op1 - 2^15 ; to make it fit within a signed word
1822 // op2 = op2 - 2^15 ; to make it fit within a signed word
1823 // result = SSE2 signed word Min/Max(op1, op2)
1824 // result = result + 2^15 ; readjust it back
1826 // signed byte (SSE2):
1827 // op1 = op1 + 2^7 ; to make it unsigned
1828 // op1 = op1 + 2^7 ; to make it unsigned
1829 // result = SSE2 unsigned byte Min/Max(op1, op2)
1830 // result = result - 2^15 ; readjust it back
1832 if (varTypeIsFloating(baseType) || baseType == TYP_SHORT || baseType == TYP_UBYTE ||
1833 (getSIMDSupportLevel() >= SIMD_SSE4_Supported &&
1834 (baseType == TYP_BYTE || baseType == TYP_INT || baseType == TYP_UINT || baseType == TYP_USHORT)))
1836 // SSE2 or SSE4.1 has direct support
1837 simdTree = gtNewSIMDNode(simdType, op1, op2, intrinsicId, baseType, size);
1839 else if (baseType == TYP_USHORT || baseType == TYP_BYTE)
1841 assert(getSIMDSupportLevel() == SIMD_SSE2_Supported);
1843 SIMDIntrinsicID operIntrinsic;
1844 SIMDIntrinsicID adjustIntrinsic;
1845 var_types minMaxOperBaseType;
1846 if (baseType == TYP_USHORT)
1848 constVal = 0x80008000;
1849 operIntrinsic = SIMDIntrinsicSub;
1850 adjustIntrinsic = SIMDIntrinsicAdd;
1851 minMaxOperBaseType = TYP_SHORT;
1855 assert(baseType == TYP_BYTE);
1856 constVal = 0x80808080;
1857 operIntrinsic = SIMDIntrinsicAdd;
1858 adjustIntrinsic = SIMDIntrinsicSub;
1859 minMaxOperBaseType = TYP_UBYTE;
1862 GenTree* initVal = gtNewIconNode(constVal);
1863 GenTree* constVector = gtNewSIMDNode(simdType, initVal, nullptr, SIMDIntrinsicInit, TYP_INT, size);
1865 // Assign constVector to a temp, since we intend to use it more than once
1866 // TODO-CQ: We have quite a few such constant vectors constructed during
1867 // the importation of SIMD intrinsics. Make sure that we have a single
1868 // temp per distinct constant per method.
1869 GenTree* tmp = fgInsertCommaFormTemp(&constVector, typeHnd);
1871 // op1 = op1 - constVector
1872 // op2 = op2 - constVector
1873 op1 = gtNewSIMDNode(simdType, op1, constVector, operIntrinsic, baseType, size);
1874 op2 = gtNewSIMDNode(simdType, op2, tmp, operIntrinsic, baseType, size);
1876 // compute min/max of op1 and op2 considering them as if minMaxOperBaseType
1877 simdTree = gtNewSIMDNode(simdType, op1, op2, intrinsicId, minMaxOperBaseType, size);
1879 // re-adjust the value by adding or subtracting constVector
1880 tmp = gtNewLclvNode(tmp->AsLclVarCommon()->GetLclNum(), tmp->TypeGet());
1881 simdTree = gtNewSIMDNode(simdType, simdTree, tmp, adjustIntrinsic, baseType, size);
1883 #elif defined(_TARGET_ARM64_)
1884 // Arm64 has direct support for all types except int64/uint64
1885 // For which we compute min/max as follows
1888 // compResult = (op1 < op2) in case of Min
1889 // (op1 > op2) in case of Max
1890 // Min/Max(op1, op2) = Select(compResult, op1, op2)
1891 if (baseType != TYP_ULONG && baseType != TYP_LONG)
1893 simdTree = gtNewSIMDNode(simdType, op1, op2, intrinsicId, baseType, size);
1898 GenTree* dupOp1 = nullptr;
1899 GenTree* dupOp2 = nullptr;
1900 GenTree* op1Assign = nullptr;
1901 GenTree* op2Assign = nullptr;
1905 if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0)
1907 op1LclNum = lvaGrabTemp(true DEBUGARG("SIMD Min/Max"));
1908 dupOp1 = gtNewLclvNode(op1LclNum, op1->TypeGet());
1909 lvaSetStruct(op1LclNum, typeHnd, false);
1910 op1Assign = gtNewTempAssign(op1LclNum, op1);
1911 op1 = gtNewLclvNode(op1LclNum, op1->TypeGet());
1915 dupOp1 = gtCloneExpr(op1);
1918 if ((op2->gtFlags & GTF_SIDE_EFFECT) != 0)
1920 op2LclNum = lvaGrabTemp(true DEBUGARG("SIMD Min/Max"));
1921 dupOp2 = gtNewLclvNode(op2LclNum, op2->TypeGet());
1922 lvaSetStruct(op2LclNum, typeHnd, false);
1923 op2Assign = gtNewTempAssign(op2LclNum, op2);
1924 op2 = gtNewLclvNode(op2LclNum, op2->TypeGet());
1928 dupOp2 = gtCloneExpr(op2);
1931 SIMDIntrinsicID relOpIntrinsic =
1932 (intrinsicId == SIMDIntrinsicMin) ? SIMDIntrinsicLessThan : SIMDIntrinsicGreaterThan;
1933 var_types relOpBaseType = baseType;
1935 // compResult = op1 relOp op2
1936 // simdTree = Select(compResult, op1, op2);
1937 assert(dupOp1 != nullptr);
1938 assert(dupOp2 != nullptr);
1939 relOpIntrinsic = impSIMDRelOp(relOpIntrinsic, typeHnd, size, &relOpBaseType, &dupOp1, &dupOp2);
1940 GenTree* compResult = gtNewSIMDNode(simdType, dupOp1, dupOp2, relOpIntrinsic, relOpBaseType, size);
1941 unsigned compResultLclNum = lvaGrabTemp(true DEBUGARG("SIMD Min/Max"));
1942 lvaSetStruct(compResultLclNum, typeHnd, false);
1943 GenTree* compResultAssign = gtNewTempAssign(compResultLclNum, compResult);
1944 compResult = gtNewLclvNode(compResultLclNum, compResult->TypeGet());
1945 simdTree = impSIMDSelect(typeHnd, baseType, size, compResult, op1, op2);
1946 simdTree = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), compResultAssign, simdTree);
1948 // Now create comma trees if we have created assignments of op1/op2 to temps
1949 if (op2Assign != nullptr)
1951 simdTree = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), op2Assign, simdTree);
1954 if (op1Assign != nullptr)
1956 simdTree = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), op1Assign, simdTree);
1960 assert(simdTree != nullptr);
1962 #else // !(defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_))
1963 assert(!"impSIMDMinMax() unimplemented on target arch");
1965 #endif // !(defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_))
1968 //------------------------------------------------------------------------
1969 // getOp1ForConstructor: Get the op1 for a constructor call.
1972 // opcode - the opcode being handled (needed to identify the CEE_NEWOBJ case)
1973 // newobjThis - For CEE_NEWOBJ, this is the temp grabbed for the allocated uninitalized object.
1974 // clsHnd - The handle of the class of the method.
1977 // The tree node representing the object to be initialized with the constructor.
1980 // This method handles the differences between the CEE_NEWOBJ and constructor cases.
1982 GenTreePtr Compiler::getOp1ForConstructor(OPCODE opcode, GenTreePtr newobjThis, CORINFO_CLASS_HANDLE clsHnd)
1985 if (opcode == CEE_NEWOBJ)
1988 assert(newobjThis->gtOper == GT_ADDR && newobjThis->gtOp.gtOp1->gtOper == GT_LCL_VAR);
1990 // push newobj result on type stack
1991 unsigned tmp = op1->gtOp.gtOp1->gtLclVarCommon.gtLclNum;
1992 impPushOnStack(gtNewLclvNode(tmp, lvaGetRealType(tmp)), verMakeTypeInfo(clsHnd).NormaliseForStack());
1996 op1 = impSIMDPopStack(TYP_BYREF);
1998 assert(op1->TypeGet() == TYP_BYREF);
2002 //-------------------------------------------------------------------
2003 // Set the flag that indicates that the lclVar referenced by this tree
2004 // is used in a SIMD intrinsic.
2006 // tree - GenTreePtr
2008 void Compiler::setLclRelatedToSIMDIntrinsic(GenTreePtr tree)
2010 assert(tree->OperIsLocal());
2011 unsigned lclNum = tree->AsLclVarCommon()->GetLclNum();
2012 LclVarDsc* lclVarDsc = &lvaTable[lclNum];
2013 lclVarDsc->lvUsedInSIMDIntrinsic = true;
2016 //-------------------------------------------------------------
2017 // Check if two field nodes reference at the same memory location.
2018 // Notice that this check is just based on pattern matching.
2020 // op1 - GenTreePtr.
2021 // op2 - GenTreePtr.
2023 // If op1's parents node and op2's parents node are at the same location, return true. Otherwise, return false
2025 bool areFieldsParentsLocatedSame(GenTreePtr op1, GenTreePtr op2)
2027 assert(op1->OperGet() == GT_FIELD);
2028 assert(op2->OperGet() == GT_FIELD);
2030 GenTreePtr op1ObjRef = op1->gtField.gtFldObj;
2031 GenTreePtr op2ObjRef = op2->gtField.gtFldObj;
2032 while (op1ObjRef != nullptr && op2ObjRef != nullptr)
2035 if (op1ObjRef->OperGet() != op2ObjRef->OperGet())
2039 else if (op1ObjRef->OperGet() == GT_ADDR)
2041 op1ObjRef = op1ObjRef->gtOp.gtOp1;
2042 op2ObjRef = op2ObjRef->gtOp.gtOp1;
2045 if (op1ObjRef->OperIsLocal() && op2ObjRef->OperIsLocal() &&
2046 op1ObjRef->AsLclVarCommon()->GetLclNum() == op2ObjRef->AsLclVarCommon()->GetLclNum())
2050 else if (op1ObjRef->OperGet() == GT_FIELD && op2ObjRef->OperGet() == GT_FIELD &&
2051 op1ObjRef->gtField.gtFldHnd == op2ObjRef->gtField.gtFldHnd)
2053 op1ObjRef = op1ObjRef->gtField.gtFldObj;
2054 op2ObjRef = op2ObjRef->gtField.gtFldObj;
2066 //----------------------------------------------------------------------
2067 // Check whether two field are contiguous
2069 // first - GenTreePtr. The Type of the node should be TYP_FLOAT
2070 // second - GenTreePtr. The Type of the node should be TYP_FLOAT
2072 // if the first field is located before second field, and they are located contiguously,
2073 // then return true. Otherwise, return false.
2075 bool Compiler::areFieldsContiguous(GenTreePtr first, GenTreePtr second)
2077 assert(first->OperGet() == GT_FIELD);
2078 assert(second->OperGet() == GT_FIELD);
2079 assert(first->gtType == TYP_FLOAT);
2080 assert(second->gtType == TYP_FLOAT);
2082 var_types firstFieldType = first->gtType;
2083 var_types secondFieldType = second->gtType;
2085 unsigned firstFieldEndOffset = first->gtField.gtFldOffset + genTypeSize(firstFieldType);
2086 unsigned secondFieldOffset = second->gtField.gtFldOffset;
2087 if (firstFieldEndOffset == secondFieldOffset && firstFieldType == secondFieldType &&
2088 areFieldsParentsLocatedSame(first, second))
2096 //-------------------------------------------------------------------------------
2097 // Check whether two array element nodes are located contiguously or not.
2099 // op1 - GenTreePtr.
2100 // op2 - GenTreePtr.
2102 // if the array element op1 is located before array element op2, and they are contiguous,
2103 // then return true. Otherwise, return false.
2105 // Right this can only check array element with const number as index. In future,
2106 // we should consider to allow this function to check the index using expression.
2108 bool Compiler::areArrayElementsContiguous(GenTreePtr op1, GenTreePtr op2)
2110 noway_assert(op1->gtOper == GT_INDEX);
2111 noway_assert(op2->gtOper == GT_INDEX);
2112 GenTreeIndex* op1Index = op1->AsIndex();
2113 GenTreeIndex* op2Index = op2->AsIndex();
2115 GenTreePtr op1ArrayRef = op1Index->Arr();
2116 GenTreePtr op2ArrayRef = op2Index->Arr();
2117 assert(op1ArrayRef->TypeGet() == TYP_REF);
2118 assert(op2ArrayRef->TypeGet() == TYP_REF);
2120 GenTreePtr op1IndexNode = op1Index->Index();
2121 GenTreePtr op2IndexNode = op2Index->Index();
2122 if ((op1IndexNode->OperGet() == GT_CNS_INT && op2IndexNode->OperGet() == GT_CNS_INT) &&
2123 op1IndexNode->gtIntCon.gtIconVal + 1 == op2IndexNode->gtIntCon.gtIconVal)
2125 if (op1ArrayRef->OperGet() == GT_FIELD && op2ArrayRef->OperGet() == GT_FIELD &&
2126 areFieldsParentsLocatedSame(op1ArrayRef, op2ArrayRef))
2130 else if (op1ArrayRef->OperIsLocal() && op2ArrayRef->OperIsLocal() &&
2131 op1ArrayRef->AsLclVarCommon()->GetLclNum() == op2ArrayRef->AsLclVarCommon()->GetLclNum())
2139 //-------------------------------------------------------------------------------
2140 // Check whether two argument nodes are contiguous or not.
2142 // op1 - GenTreePtr.
2143 // op2 - GenTreePtr.
2145 // if the argument node op1 is located before argument node op2, and they are located contiguously,
2146 // then return true. Otherwise, return false.
2148 // Right now this can only check field and array. In future we should add more cases.
2151 bool Compiler::areArgumentsContiguous(GenTreePtr op1, GenTreePtr op2)
2153 if (op1->OperGet() == GT_INDEX && op2->OperGet() == GT_INDEX)
2155 return areArrayElementsContiguous(op1, op2);
2157 else if (op1->OperGet() == GT_FIELD && op2->OperGet() == GT_FIELD)
2159 return areFieldsContiguous(op1, op2);
2164 //--------------------------------------------------------------------------------------------------------
2165 // createAddressNodeForSIMDInit: Generate the address node(GT_LEA) if we want to intialize vector2, vector3 or vector4
2166 // from first argument's address.
2169 // tree - GenTreePtr. This the tree node which is used to get the address for indir.
2170 // simdsize - unsigned. This the simd vector size.
2171 // arrayElementsCount - unsigned. This is used for generating the boundary check for array.
2174 // return the address node.
2177 // 1. Currently just support for GT_FIELD and GT_INDEX, because we can only verify the GT_INDEX node or GT_Field
2178 // are located contiguously or not. In future we should support more cases.
2179 // 2. Though it happens to just work fine front-end phases are not aware of GT_LEA node. Therefore, convert these
2181 GenTreePtr Compiler::createAddressNodeForSIMDInit(GenTreePtr tree, unsigned simdSize)
2183 assert(tree->OperGet() == GT_FIELD || tree->OperGet() == GT_INDEX);
2184 GenTreePtr byrefNode = nullptr;
2185 GenTreePtr startIndex = nullptr;
2186 unsigned offset = 0;
2187 var_types baseType = tree->gtType;
2189 if (tree->OperGet() == GT_FIELD)
2191 GenTreePtr objRef = tree->gtField.gtFldObj;
2192 if (objRef != nullptr && objRef->gtOper == GT_ADDR)
2194 GenTreePtr obj = objRef->gtOp.gtOp1;
2196 // If the field is directly from a struct, then in this case,
2197 // we should set this struct's lvUsedInSIMDIntrinsic as true,
2198 // so that this sturct won't be promoted.
2199 // e.g. s.x x is a field, and s is a struct, then we should set the s's lvUsedInSIMDIntrinsic as true.
2200 // so that s won't be promoted.
2201 // Notice that if we have a case like s1.s2.x. s1 s2 are struct, and x is a field, then it is possible that
2202 // s1 can be promoted, so that s2 can be promoted. The reason for that is if we don't allow s1 to be
2203 // promoted, then this will affect the other optimizations which are depend on s1's struct promotion.
2205 // In future, we should optimize this case so that if there is a nested field like s1.s2.x and s1.s2.x's
2206 // address is used for initializing the vector, then s1 can be promoted but s2 can't.
2207 if (varTypeIsSIMD(obj) && obj->OperIsLocal())
2209 setLclRelatedToSIMDIntrinsic(obj);
2213 byrefNode = gtCloneExpr(tree->gtField.gtFldObj);
2214 assert(byrefNode != nullptr);
2215 offset = tree->gtField.gtFldOffset;
2217 else if (tree->OperGet() == GT_INDEX)
2220 GenTreePtr index = tree->AsIndex()->Index();
2221 assert(index->OperGet() == GT_CNS_INT);
2223 GenTreePtr checkIndexExpr = nullptr;
2224 unsigned indexVal = (unsigned)(index->gtIntCon.gtIconVal);
2225 offset = indexVal * genTypeSize(tree->TypeGet());
2226 GenTreePtr arrayRef = tree->AsIndex()->Arr();
2228 // Generate the boundary check exception.
2229 // The length for boundary check should be the maximum index number which should be
2230 // (first argument's index number) + (how many array arguments we have) - 1
2231 // = indexVal + arrayElementsCount - 1
2232 unsigned arrayElementsCount = simdSize / genTypeSize(baseType);
2233 checkIndexExpr = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, indexVal + arrayElementsCount - 1);
2234 GenTreeArrLen* arrLen = gtNewArrLen(TYP_INT, arrayRef, (int)offsetof(CORINFO_Array, length));
2235 GenTreeBoundsChk* arrBndsChk = new (this, GT_ARR_BOUNDS_CHECK)
2236 GenTreeBoundsChk(GT_ARR_BOUNDS_CHECK, TYP_VOID, checkIndexExpr, arrLen, SCK_RNGCHK_FAIL);
2238 offset += offsetof(CORINFO_Array, u1Elems);
2239 byrefNode = gtNewOperNode(GT_COMMA, arrayRef->TypeGet(), arrBndsChk, gtCloneExpr(arrayRef));
2245 GenTreePtr address =
2246 new (this, GT_LEA) GenTreeAddrMode(TYP_BYREF, byrefNode, startIndex, genTypeSize(tree->TypeGet()), offset);
2250 //-------------------------------------------------------------------------------
2251 // impMarkContiguousSIMDFieldAssignments: Try to identify if there are contiguous
2252 // assignments from SIMD field to memory. If there are, then mark the related
2253 // lclvar so that it won't be promoted.
2256 // stmt - GenTreePtr. Input statement node.
2258 void Compiler::impMarkContiguousSIMDFieldAssignments(GenTreePtr stmt)
2260 if (!featureSIMD || opts.MinOpts())
2264 GenTreePtr expr = stmt->gtStmt.gtStmtExpr;
2265 if (expr->OperGet() == GT_ASG && expr->TypeGet() == TYP_FLOAT)
2267 GenTreePtr curDst = expr->gtOp.gtOp1;
2268 GenTreePtr curSrc = expr->gtOp.gtOp2;
2270 var_types baseType = TYP_UNKNOWN;
2271 unsigned simdSize = 0;
2272 GenTreePtr srcSimdStructNode = getSIMDStructFromField(curSrc, &baseType, &index, &simdSize, true);
2273 if (srcSimdStructNode == nullptr || baseType != TYP_FLOAT)
2275 fgPreviousCandidateSIMDFieldAsgStmt = nullptr;
2277 else if (index == 0 && isSIMDTypeLocal(srcSimdStructNode))
2279 fgPreviousCandidateSIMDFieldAsgStmt = stmt;
2281 else if (fgPreviousCandidateSIMDFieldAsgStmt != nullptr)
2284 GenTreePtr prevAsgExpr = fgPreviousCandidateSIMDFieldAsgStmt->gtStmt.gtStmtExpr;
2285 GenTreePtr prevDst = prevAsgExpr->gtOp.gtOp1;
2286 GenTreePtr prevSrc = prevAsgExpr->gtOp.gtOp2;
2287 if (!areArgumentsContiguous(prevDst, curDst) || !areArgumentsContiguous(prevSrc, curSrc))
2289 fgPreviousCandidateSIMDFieldAsgStmt = nullptr;
2293 if (index == (simdSize / genTypeSize(baseType) - 1))
2295 // Successfully found the pattern, mark the lclvar as UsedInSIMDIntrinsic
2296 if (srcSimdStructNode->OperIsLocal())
2298 setLclRelatedToSIMDIntrinsic(srcSimdStructNode);
2301 if (curDst->OperGet() == GT_FIELD)
2303 GenTreePtr objRef = curDst->gtField.gtFldObj;
2304 if (objRef != nullptr && objRef->gtOper == GT_ADDR)
2306 GenTreePtr obj = objRef->gtOp.gtOp1;
2307 if (varTypeIsStruct(obj) && obj->OperIsLocal())
2309 setLclRelatedToSIMDIntrinsic(obj);
2316 fgPreviousCandidateSIMDFieldAsgStmt = stmt;
2323 fgPreviousCandidateSIMDFieldAsgStmt = nullptr;
2327 //------------------------------------------------------------------------
2328 // impSIMDIntrinsic: Check method to see if it is a SIMD method
2331 // opcode - the opcode being handled (needed to identify the CEE_NEWOBJ case)
2332 // newobjThis - For CEE_NEWOBJ, this is the temp grabbed for the allocated uninitalized object.
2333 // clsHnd - The handle of the class of the method.
2334 // method - The handle of the method.
2335 // sig - The call signature for the method.
2336 // memberRef - The memberRef token for the method reference.
2339 // If clsHnd is a known SIMD type, and 'method' is one of the methods that are
2340 // implemented as an intrinsic in the JIT, then return the tree that implements
2343 GenTreePtr Compiler::impSIMDIntrinsic(OPCODE opcode,
2344 GenTreePtr newobjThis,
2345 CORINFO_CLASS_HANDLE clsHnd,
2346 CORINFO_METHOD_HANDLE methodHnd,
2347 CORINFO_SIG_INFO* sig,
2350 assert(featureSIMD);
2352 if (!isSIMDClass(clsHnd))
2357 // Get base type and intrinsic Id
2358 var_types baseType = TYP_UNKNOWN;
2360 unsigned argCount = 0;
2361 const SIMDIntrinsicInfo* intrinsicInfo =
2362 getSIMDIntrinsicInfo(&clsHnd, methodHnd, sig, (opcode == CEE_NEWOBJ), &argCount, &baseType, &size);
2363 if (intrinsicInfo == nullptr || intrinsicInfo->id == SIMDIntrinsicInvalid)
2368 SIMDIntrinsicID simdIntrinsicID = intrinsicInfo->id;
2370 if (baseType != TYP_UNKNOWN)
2372 simdType = getSIMDTypeForSize(size);
2376 assert(simdIntrinsicID == SIMDIntrinsicHWAccel);
2377 simdType = TYP_UNKNOWN;
2379 bool instMethod = intrinsicInfo->isInstMethod;
2380 var_types callType = JITtype2varType(sig->retType);
2381 if (callType == TYP_STRUCT)
2383 // Note that here we are assuming that, if the call returns a struct, that it is the same size as the
2384 // struct on which the method is declared. This is currently true for all methods on Vector types,
2385 // but if this ever changes, we will need to determine the callType from the signature.
2386 assert(info.compCompHnd->getClassSize(sig->retTypeClass) == genTypeSize(simdType));
2387 callType = simdType;
2390 GenTree* simdTree = nullptr;
2391 GenTree* op1 = nullptr;
2392 GenTree* op2 = nullptr;
2393 GenTree* op3 = nullptr;
2394 GenTree* retVal = nullptr;
2395 GenTree* copyBlkDst = nullptr;
2396 bool doCopyBlk = false;
2398 switch (simdIntrinsicID)
2400 case SIMDIntrinsicGetCount:
2402 int length = getSIMDVectorLength(clsHnd);
2403 GenTreeIntCon* intConstTree = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, length);
2404 retVal = intConstTree;
2406 intConstTree->gtFlags |= GTF_ICON_SIMD_COUNT;
2410 case SIMDIntrinsicGetZero:
2411 retVal = gtNewSIMDVectorZero(simdType, baseType, size);
2414 case SIMDIntrinsicGetOne:
2415 retVal = gtNewSIMDVectorOne(simdType, baseType, size);
2418 case SIMDIntrinsicGetAllOnes:
2420 // Equivalent to (Vector<T>) new Vector<int>(0xffffffff);
2421 GenTree* initVal = gtNewIconNode(0xffffffff, TYP_INT);
2422 simdTree = gtNewSIMDNode(simdType, initVal, nullptr, SIMDIntrinsicInit, TYP_INT, size);
2423 if (baseType != TYP_INT)
2425 // cast it to required baseType if different from TYP_INT
2426 simdTree = gtNewSIMDNode(simdType, simdTree, nullptr, SIMDIntrinsicCast, baseType, size);
2432 case SIMDIntrinsicInit:
2433 case SIMDIntrinsicInitN:
2435 // SIMDIntrinsicInit:
2436 // op2 - the initializer value
2437 // op1 - byref of vector
2439 // SIMDIntrinsicInitN
2440 // op2 - list of initializer values stitched into a list
2441 // op1 - byref of vector
2442 bool initFromFirstArgIndir = false;
2443 if (simdIntrinsicID == SIMDIntrinsicInit)
2445 op2 = impSIMDPopStack(baseType);
2449 assert(simdIntrinsicID == SIMDIntrinsicInitN);
2450 assert(baseType == TYP_FLOAT);
2452 unsigned initCount = argCount - 1;
2453 unsigned elementCount = getSIMDVectorLength(size, baseType);
2454 noway_assert(initCount == elementCount);
2455 GenTree* nextArg = op2;
2457 // Build a GT_LIST with the N values.
2458 // We must maintain left-to-right order of the args, but we will pop
2459 // them off in reverse order (the Nth arg was pushed onto the stack last).
2461 GenTree* list = nullptr;
2462 GenTreePtr firstArg = nullptr;
2463 GenTreePtr prevArg = nullptr;
2465 bool areArgsContiguous = true;
2466 for (unsigned i = 0; i < initCount; i++)
2468 GenTree* nextArg = impSIMDPopStack(baseType);
2469 if (areArgsContiguous)
2471 GenTreePtr curArg = nextArg;
2474 if (prevArg != nullptr)
2476 // Recall that we are popping the args off the stack in reverse order.
2477 areArgsContiguous = areArgumentsContiguous(curArg, prevArg);
2482 list = new (this, GT_LIST) GenTreeOp(GT_LIST, baseType, nextArg, list);
2485 if (areArgsContiguous && baseType == TYP_FLOAT)
2487 // Since Vector2, Vector3 and Vector4's arguments type are only float,
2488 // we intialize the vector from first argument address, only when
2489 // the baseType is TYP_FLOAT and the arguments are located contiguously in memory
2490 initFromFirstArgIndir = true;
2491 GenTreePtr op2Address = createAddressNodeForSIMDInit(firstArg, size);
2492 var_types simdType = getSIMDTypeForSize(size);
2493 op2 = gtNewOperNode(GT_IND, simdType, op2Address);
2501 op1 = getOp1ForConstructor(opcode, newobjThis, clsHnd);
2503 assert(op1->TypeGet() == TYP_BYREF);
2504 assert(genActualType(op2->TypeGet()) == genActualType(baseType) || initFromFirstArgIndir);
2506 // For integral base types of size less than TYP_INT, expand the initializer
2507 // to fill size of TYP_INT bytes.
2508 if (varTypeIsSmallInt(baseType))
2510 // This case should occur only for Init intrinsic.
2511 assert(simdIntrinsicID == SIMDIntrinsicInit);
2513 unsigned baseSize = genTypeSize(baseType);
2517 multiplier = 0x01010101;
2521 assert(baseSize == 2);
2522 multiplier = 0x00010001;
2525 GenTree* t1 = nullptr;
2526 if (baseType == TYP_BYTE)
2528 // What we have is a signed byte initializer,
2529 // which when loaded to a reg will get sign extended to TYP_INT.
2530 // But what we need is the initializer without sign extended or
2531 // rather zero extended to 32-bits.
2532 t1 = gtNewOperNode(GT_AND, TYP_INT, op2, gtNewIconNode(0xff, TYP_INT));
2534 else if (baseType == TYP_SHORT)
2536 // What we have is a signed short initializer,
2537 // which when loaded to a reg will get sign extended to TYP_INT.
2538 // But what we need is the initializer without sign extended or
2539 // rather zero extended to 32-bits.
2540 t1 = gtNewOperNode(GT_AND, TYP_INT, op2, gtNewIconNode(0xffff, TYP_INT));
2544 assert(baseType == TYP_UBYTE || baseType == TYP_USHORT);
2545 t1 = gtNewCastNode(TYP_INT, op2, TYP_INT);
2548 assert(t1 != nullptr);
2549 GenTree* t2 = gtNewIconNode(multiplier, TYP_INT);
2550 op2 = gtNewOperNode(GT_MUL, TYP_INT, t1, t2);
2552 // Construct a vector of TYP_INT with the new initializer and cast it back to vector of baseType
2553 simdTree = gtNewSIMDNode(simdType, op2, nullptr, simdIntrinsicID, TYP_INT, size);
2554 simdTree = gtNewSIMDNode(simdType, simdTree, nullptr, SIMDIntrinsicCast, baseType, size);
2559 if (initFromFirstArgIndir)
2562 if (op1->gtOp.gtOp1->OperIsLocal())
2564 // label the dst struct's lclvar is used for SIMD intrinsic,
2565 // so that this dst struct won't be promoted.
2566 setLclRelatedToSIMDIntrinsic(op1->gtOp.gtOp1);
2571 simdTree = gtNewSIMDNode(simdType, op2, nullptr, simdIntrinsicID, baseType, size);
2580 case SIMDIntrinsicInitArray:
2581 case SIMDIntrinsicInitArrayX:
2582 case SIMDIntrinsicCopyToArray:
2583 case SIMDIntrinsicCopyToArrayX:
2585 // op3 - index into array in case of SIMDIntrinsicCopyToArrayX and SIMDIntrinsicInitArrayX
2586 // op2 - array itself
2587 // op1 - byref to vector struct
2589 unsigned int vectorLength = getSIMDVectorLength(size, baseType);
2590 // (This constructor takes only the zero-based arrays.)
2591 // We will add one or two bounds checks:
2592 // 1. If we have an index, we must do a check on that first.
2593 // We can't combine it with the index + vectorLength check because
2594 // a. It might be negative, and b. It may need to raise a different exception
2595 // (captured as SCK_ARG_RNG_EXCPN for CopyTo and SCK_RNGCHK_FAIL for Init).
2596 // 2. We need to generate a check (SCK_ARG_EXCPN for CopyTo and SCK_RNGCHK_FAIL for Init)
2597 // for the last array element we will access.
2598 // We'll either check against (vectorLength - 1) or (index + vectorLength - 1).
2600 GenTree* checkIndexExpr = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, vectorLength - 1);
2602 // Get the index into the array. If it has been provided, it will be on the
2603 // top of the stack. Otherwise, it is null.
2606 op3 = impSIMDPopStack(TYP_INT);
2607 if (op3->IsIntegralConst(0))
2614 // TODO-CQ: Here, or elsewhere, check for the pattern where op2 is a newly constructed array, and
2615 // change this to the InitN form.
2616 // op3 = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, 0);
2620 // Clone the array for use in the bounds check.
2621 op2 = impSIMDPopStack(TYP_REF);
2622 assert(op2->TypeGet() == TYP_REF);
2623 GenTree* arrayRefForArgChk = op2;
2624 GenTree* argRngChk = nullptr;
2625 GenTree* asg = nullptr;
2626 if ((arrayRefForArgChk->gtFlags & GTF_SIDE_EFFECT) != 0)
2628 op2 = fgInsertCommaFormTemp(&arrayRefForArgChk);
2632 op2 = gtCloneExpr(arrayRefForArgChk);
2634 assert(op2 != nullptr);
2638 SpecialCodeKind op3CheckKind;
2639 if (simdIntrinsicID == SIMDIntrinsicInitArrayX)
2641 op3CheckKind = SCK_RNGCHK_FAIL;
2645 assert(simdIntrinsicID == SIMDIntrinsicCopyToArrayX);
2646 op3CheckKind = SCK_ARG_RNG_EXCPN;
2648 // We need to use the original expression on this, which is the first check.
2649 GenTree* arrayRefForArgRngChk = arrayRefForArgChk;
2650 // Then we clone the clone we just made for the next check.
2651 arrayRefForArgChk = gtCloneExpr(op2);
2652 // We know we MUST have had a cloneable expression.
2653 assert(arrayRefForArgChk != nullptr);
2654 GenTree* index = op3;
2655 if ((index->gtFlags & GTF_SIDE_EFFECT) != 0)
2657 op3 = fgInsertCommaFormTemp(&index);
2661 op3 = gtCloneExpr(index);
2664 GenTreeArrLen* arrLen =
2665 gtNewArrLen(TYP_INT, arrayRefForArgRngChk, (int)offsetof(CORINFO_Array, length));
2666 argRngChk = new (this, GT_ARR_BOUNDS_CHECK)
2667 GenTreeBoundsChk(GT_ARR_BOUNDS_CHECK, TYP_VOID, index, arrLen, op3CheckKind);
2668 // Now, clone op3 to create another node for the argChk
2669 GenTree* index2 = gtCloneExpr(op3);
2670 assert(index != nullptr);
2671 checkIndexExpr = gtNewOperNode(GT_ADD, TYP_INT, index2, checkIndexExpr);
2674 // Insert a bounds check for index + offset - 1.
2675 // This must be a "normal" array.
2676 SpecialCodeKind op2CheckKind;
2677 if (simdIntrinsicID == SIMDIntrinsicInitArray || simdIntrinsicID == SIMDIntrinsicInitArrayX)
2679 op2CheckKind = SCK_RNGCHK_FAIL;
2683 op2CheckKind = SCK_ARG_EXCPN;
2685 GenTreeArrLen* arrLen = gtNewArrLen(TYP_INT, arrayRefForArgChk, (int)offsetof(CORINFO_Array, length));
2686 GenTreeBoundsChk* argChk = new (this, GT_ARR_BOUNDS_CHECK)
2687 GenTreeBoundsChk(GT_ARR_BOUNDS_CHECK, TYP_VOID, checkIndexExpr, arrLen, op2CheckKind);
2689 // Create a GT_COMMA tree for the bounds check(s).
2690 op2 = gtNewOperNode(GT_COMMA, op2->TypeGet(), argChk, op2);
2691 if (argRngChk != nullptr)
2693 op2 = gtNewOperNode(GT_COMMA, op2->TypeGet(), argRngChk, op2);
2696 if (simdIntrinsicID == SIMDIntrinsicInitArray || simdIntrinsicID == SIMDIntrinsicInitArrayX)
2698 op1 = getOp1ForConstructor(opcode, newobjThis, clsHnd);
2699 simdTree = gtNewSIMDNode(simdType, op2, op3, SIMDIntrinsicInitArray, baseType, size);
2705 assert(simdIntrinsicID == SIMDIntrinsicCopyToArray || simdIntrinsicID == SIMDIntrinsicCopyToArrayX);
2706 op1 = impSIMDPopStack(simdType, instMethod);
2707 assert(op1->TypeGet() == simdType);
2709 // copy vector (op1) to array (op2) starting at index (op3)
2712 // TODO-Cleanup: Though it happens to just work fine front-end phases are not aware of GT_LEA node.
2713 // Therefore, convert these to use GT_ADDR .
2714 copyBlkDst = new (this, GT_LEA)
2715 GenTreeAddrMode(TYP_BYREF, op2, op3, genTypeSize(baseType), offsetof(CORINFO_Array, u1Elems));
2721 case SIMDIntrinsicInitFixed:
2723 // We are initializing a fixed-length vector VLarge with a smaller fixed-length vector VSmall, plus 1 or 2
2724 // additional floats.
2725 // op4 (optional) - float value for VLarge.W, if VLarge is Vector4, and VSmall is Vector2
2726 // op3 - float value for VLarge.Z or VLarge.W
2728 // op1 - byref of VLarge
2729 assert(baseType == TYP_FLOAT);
2730 unsigned elementByteCount = 4;
2732 GenTree* op4 = nullptr;
2735 op4 = impSIMDPopStack(TYP_FLOAT);
2736 assert(op4->TypeGet() == TYP_FLOAT);
2738 op3 = impSIMDPopStack(TYP_FLOAT);
2739 assert(op3->TypeGet() == TYP_FLOAT);
2740 // The input vector will either be TYP_SIMD8 or TYP_SIMD12.
2741 var_types smallSIMDType = TYP_SIMD8;
2742 if ((op4 == nullptr) && (simdType == TYP_SIMD16))
2744 smallSIMDType = TYP_SIMD12;
2746 op2 = impSIMDPopStack(smallSIMDType);
2747 op1 = getOp1ForConstructor(opcode, newobjThis, clsHnd);
2749 // We are going to redefine the operands so that:
2750 // - op3 is the value that's going into the Z position, or null if it's a Vector4 constructor with a single
2752 // - op4 is the W position value, or null if this is a Vector3 constructor.
2753 if (size == 16 && argCount == 3)
2762 simdTree = gtNewSIMDNode(simdType, simdTree, op3, SIMDIntrinsicSetZ, baseType, size);
2766 simdTree = gtNewSIMDNode(simdType, simdTree, op4, SIMDIntrinsicSetW, baseType, size);
2774 case SIMDIntrinsicOpEquality:
2775 case SIMDIntrinsicInstEquals:
2777 op2 = impSIMDPopStack(simdType);
2778 op1 = impSIMDPopStack(simdType, instMethod);
2780 assert(op1->TypeGet() == simdType);
2781 assert(op2->TypeGet() == simdType);
2783 simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, SIMDIntrinsicOpEquality, baseType, size);
2784 if (simdType == TYP_SIMD12)
2786 simdTree->gtFlags |= GTF_SIMD12_OP;
2792 case SIMDIntrinsicOpInEquality:
2794 // op1 is the first operand
2795 // op2 is the second operand
2796 op2 = impSIMDPopStack(simdType);
2797 op1 = impSIMDPopStack(simdType, instMethod);
2798 simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, SIMDIntrinsicOpInEquality, baseType, size);
2799 if (simdType == TYP_SIMD12)
2801 simdTree->gtFlags |= GTF_SIMD12_OP;
2807 case SIMDIntrinsicEqual:
2808 case SIMDIntrinsicLessThan:
2809 case SIMDIntrinsicLessThanOrEqual:
2810 case SIMDIntrinsicGreaterThan:
2811 case SIMDIntrinsicGreaterThanOrEqual:
2813 op2 = impSIMDPopStack(simdType);
2814 op1 = impSIMDPopStack(simdType, instMethod);
2816 SIMDIntrinsicID intrinsicID = impSIMDRelOp(simdIntrinsicID, clsHnd, size, &baseType, &op1, &op2);
2817 simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, intrinsicID, baseType, size);
2822 case SIMDIntrinsicAdd:
2823 case SIMDIntrinsicSub:
2824 case SIMDIntrinsicMul:
2825 case SIMDIntrinsicDiv:
2826 case SIMDIntrinsicBitwiseAnd:
2827 case SIMDIntrinsicBitwiseAndNot:
2828 case SIMDIntrinsicBitwiseOr:
2829 case SIMDIntrinsicBitwiseXor:
2832 // check for the cases where we don't support intrinsics.
2833 // This check should be done before we make modifications to type stack.
2834 // Note that this is more of a double safety check for robustness since
2835 // we expect getSIMDIntrinsicInfo() to have filtered out intrinsics on
2836 // unsupported base types. If getSIMdIntrinsicInfo() doesn't filter due
2837 // to some bug, assert in chk/dbg will fire.
2838 if (!varTypeIsFloating(baseType))
2840 if (simdIntrinsicID == SIMDIntrinsicMul)
2842 #if defined(_TARGET_XARCH_)
2843 if ((baseType != TYP_INT) && (baseType != TYP_SHORT))
2845 // TODO-CQ: implement mul on these integer vectors.
2846 // Note that SSE2 has no direct support for these vectors.
2847 assert(!"Mul not supported on long/ulong/uint/small int vectors\n");
2850 #endif // _TARGET_XARCH_
2851 #if defined(_TARGET_ARM64_)
2852 if ((baseType == TYP_ULONG) && (baseType == TYP_LONG))
2854 // TODO-CQ: implement mul on these integer vectors.
2855 // Note that ARM64 has no direct support for these vectors.
2856 assert(!"Mul not supported on long/ulong vectors\n");
2859 #endif // _TARGET_ARM64_
2861 #if defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
2862 // common to all integer type vectors
2863 if (simdIntrinsicID == SIMDIntrinsicDiv)
2865 // SSE2 doesn't support div on non-floating point vectors.
2866 assert(!"Div not supported on integer type vectors\n");
2869 #endif // defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
2873 // op1 is the first operand; if instance method, op1 is "this" arg
2874 // op2 is the second operand
2875 op2 = impSIMDPopStack(simdType);
2876 op1 = impSIMDPopStack(simdType, instMethod);
2878 simdTree = gtNewSIMDNode(simdType, op1, op2, simdIntrinsicID, baseType, size);
2883 case SIMDIntrinsicSelect:
2885 // op3 is a SIMD variable that is the second source
2886 // op2 is a SIMD variable that is the first source
2887 // op1 is a SIMD variable which is the bit mask.
2888 op3 = impSIMDPopStack(simdType);
2889 op2 = impSIMDPopStack(simdType);
2890 op1 = impSIMDPopStack(simdType);
2892 retVal = impSIMDSelect(clsHnd, baseType, size, op1, op2, op3);
2896 case SIMDIntrinsicMin:
2897 case SIMDIntrinsicMax:
2899 // op1 is the first operand; if instance method, op1 is "this" arg
2900 // op2 is the second operand
2901 op2 = impSIMDPopStack(simdType);
2902 op1 = impSIMDPopStack(simdType, instMethod);
2904 retVal = impSIMDMinMax(simdIntrinsicID, clsHnd, baseType, size, op1, op2);
2908 case SIMDIntrinsicGetItem:
2910 // op1 is a SIMD variable that is "this" arg
2911 // op2 is an index of TYP_INT
2912 op2 = impSIMDPopStack(TYP_INT);
2913 op1 = impSIMDPopStack(simdType, instMethod);
2914 int vectorLength = getSIMDVectorLength(size, baseType);
2915 if (!op2->IsCnsIntOrI() || op2->AsIntCon()->gtIconVal >= vectorLength || op2->AsIntCon()->gtIconVal < 0)
2917 // We need to bounds-check the length of the vector.
2918 // For that purpose, we need to clone the index expression.
2919 GenTree* index = op2;
2920 if ((index->gtFlags & GTF_SIDE_EFFECT) != 0)
2922 op2 = fgInsertCommaFormTemp(&index);
2926 op2 = gtCloneExpr(index);
2929 GenTree* lengthNode = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, vectorLength);
2930 GenTreeBoundsChk* simdChk =
2931 new (this, GT_SIMD_CHK) GenTreeBoundsChk(GT_SIMD_CHK, TYP_VOID, index, lengthNode, SCK_RNGCHK_FAIL);
2933 // Create a GT_COMMA tree for the bounds check.
2934 op2 = gtNewOperNode(GT_COMMA, op2->TypeGet(), simdChk, op2);
2937 assert(op1->TypeGet() == simdType);
2938 assert(op2->TypeGet() == TYP_INT);
2940 simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, simdIntrinsicID, baseType, size);
2945 case SIMDIntrinsicDotProduct:
2947 #if defined(_TARGET_XARCH_)
2948 // Right now dot product is supported only for float/double vectors and
2949 // int vectors on SSE4/AVX.
2950 if (!varTypeIsFloating(baseType) && !(baseType == TYP_INT && getSIMDSupportLevel() >= SIMD_SSE4_Supported))
2954 #endif // _TARGET_XARCH_
2956 // op1 is a SIMD variable that is the first source and also "this" arg.
2957 // op2 is a SIMD variable which is the second source.
2958 op2 = impSIMDPopStack(simdType);
2959 op1 = impSIMDPopStack(simdType, instMethod);
2961 simdTree = gtNewSIMDNode(baseType, op1, op2, simdIntrinsicID, baseType, size);
2962 if (simdType == TYP_SIMD12)
2964 simdTree->gtFlags |= GTF_SIMD12_OP;
2970 case SIMDIntrinsicSqrt:
2972 #if (defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)) && defined(DEBUG)
2973 // SSE/AVX/ARM64 doesn't support sqrt on integer type vectors and hence
2974 // should never be seen as an intrinsic here. See SIMDIntrinsicList.h
2975 // for supported base types for this intrinsic.
2976 if (!varTypeIsFloating(baseType))
2978 assert(!"Sqrt not supported on integer vectors\n");
2981 #endif // (defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)) && defined(DEBUG)
2983 op1 = impSIMDPopStack(simdType);
2985 retVal = gtNewSIMDNode(genActualType(callType), op1, nullptr, simdIntrinsicID, baseType, size);
2989 case SIMDIntrinsicAbs:
2990 op1 = impSIMDPopStack(simdType);
2991 retVal = impSIMDAbs(clsHnd, baseType, size, op1);
2994 case SIMDIntrinsicGetW:
2995 retVal = impSIMDGetFixed(simdType, baseType, size, 3);
2998 case SIMDIntrinsicGetZ:
2999 retVal = impSIMDGetFixed(simdType, baseType, size, 2);
3002 case SIMDIntrinsicGetY:
3003 retVal = impSIMDGetFixed(simdType, baseType, size, 1);
3006 case SIMDIntrinsicGetX:
3007 retVal = impSIMDGetFixed(simdType, baseType, size, 0);
3010 case SIMDIntrinsicSetW:
3011 case SIMDIntrinsicSetZ:
3012 case SIMDIntrinsicSetY:
3013 case SIMDIntrinsicSetX:
3015 // op2 is the value to be set at indexTemp position
3016 // op1 is SIMD vector that is going to be modified, which is a byref
3018 // If op1 has a side-effect, then don't make it an intrinsic.
3019 // It would be in-efficient to read the entire vector into xmm reg,
3020 // modify it and write back entire xmm reg.
3022 // TODO-CQ: revisit this later.
3023 op1 = impStackTop(1).val;
3024 if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0)
3029 op2 = impSIMDPopStack(baseType);
3030 op1 = impSIMDPopStack(simdType, instMethod);
3032 GenTree* src = gtCloneExpr(op1);
3033 assert(src != nullptr);
3034 simdTree = gtNewSIMDNode(simdType, src, op2, simdIntrinsicID, baseType, size);
3036 copyBlkDst = gtNewOperNode(GT_ADDR, TYP_BYREF, op1);
3041 // Unary operators that take and return a Vector.
3042 case SIMDIntrinsicCast:
3043 case SIMDIntrinsicConvertToSingle:
3044 case SIMDIntrinsicConvertToDouble:
3045 case SIMDIntrinsicConvertToInt32:
3046 case SIMDIntrinsicConvertToUInt32:
3048 op1 = impSIMDPopStack(simdType, instMethod);
3050 simdTree = gtNewSIMDNode(simdType, op1, nullptr, simdIntrinsicID, baseType, size);
3055 case SIMDIntrinsicConvertToInt64:
3056 case SIMDIntrinsicConvertToUInt64:
3058 #ifdef _TARGET_64BIT_
3059 op1 = impSIMDPopStack(simdType, instMethod);
3061 simdTree = gtNewSIMDNode(simdType, op1, nullptr, simdIntrinsicID, baseType, size);
3064 JITDUMP("SIMD Conversion to Int64/UInt64 is not supported on this platform\n");
3070 case SIMDIntrinsicNarrow:
3072 assert(!instMethod);
3073 op2 = impSIMDPopStack(simdType);
3074 op1 = impSIMDPopStack(simdType);
3075 // op1 and op2 are two input Vector<T>.
3076 simdTree = gtNewSIMDNode(simdType, op1, op2, simdIntrinsicID, baseType, size);
3081 case SIMDIntrinsicWiden:
3083 GenTree* dstAddrHi = impSIMDPopStack(TYP_BYREF);
3084 GenTree* dstAddrLo = impSIMDPopStack(TYP_BYREF);
3085 op1 = impSIMDPopStack(simdType);
3086 GenTree* dupOp1 = fgInsertCommaFormTemp(&op1, gtGetStructHandleForSIMD(simdType, baseType));
3088 // Widen the lower half and assign it to dstAddrLo.
3089 simdTree = gtNewSIMDNode(simdType, op1, nullptr, SIMDIntrinsicWidenLo, baseType, size);
3091 new (this, GT_BLK) GenTreeBlk(GT_BLK, simdType, dstAddrLo, getSIMDTypeSizeInBytes(clsHnd));
3092 GenTree* loAsg = gtNewBlkOpNode(loDest, simdTree, getSIMDTypeSizeInBytes(clsHnd),
3093 false, // not volatile
3095 loAsg->gtFlags |= ((simdTree->gtFlags | dstAddrLo->gtFlags) & GTF_ALL_EFFECT);
3097 // Widen the upper half and assign it to dstAddrHi.
3098 simdTree = gtNewSIMDNode(simdType, dupOp1, nullptr, SIMDIntrinsicWidenHi, baseType, size);
3100 new (this, GT_BLK) GenTreeBlk(GT_BLK, simdType, dstAddrHi, getSIMDTypeSizeInBytes(clsHnd));
3101 GenTree* hiAsg = gtNewBlkOpNode(hiDest, simdTree, getSIMDTypeSizeInBytes(clsHnd),
3102 false, // not volatile
3104 hiAsg->gtFlags |= ((simdTree->gtFlags | dstAddrHi->gtFlags) & GTF_ALL_EFFECT);
3106 retVal = gtNewOperNode(GT_COMMA, simdType, loAsg, hiAsg);
3110 case SIMDIntrinsicHWAccel:
3112 GenTreeIntCon* intConstTree = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, 1);
3113 retVal = intConstTree;
3118 assert(!"Unimplemented SIMD Intrinsic");
3122 #if defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
3123 // XArch/Arm64: also indicate that we use floating point registers.
3124 // The need for setting this here is that a method may not have SIMD
3125 // type lclvars, but might be exercising SIMD intrinsics on fields of
3128 // e.g. public Vector<float> ComplexVecFloat::sqabs() { return this.r * this.r + this.i * this.i; }
3129 compFloatingPointUsed = true;
3130 #endif // defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
3132 // At this point, we have a tree that we are going to store into a destination.
3133 // TODO-1stClassStructs: This should be a simple store or assignment, and should not require
3134 // GTF_ALL_EFFECT for the dest. This is currently emulating the previous behavior of
3138 GenTree* dest = new (this, GT_BLK) GenTreeBlk(GT_BLK, simdType, copyBlkDst, getSIMDTypeSizeInBytes(clsHnd));
3139 dest->gtFlags |= GTF_GLOB_REF;
3140 retVal = gtNewBlkOpNode(dest, simdTree, getSIMDTypeSizeInBytes(clsHnd),
3141 false, // not volatile
3143 retVal->gtFlags |= ((simdTree->gtFlags | copyBlkDst->gtFlags) & GTF_ALL_EFFECT);
3149 #endif // FEATURE_SIMD
3151 #endif // !LEGACY_BACKEND