1 // Licensed to the .NET Foundation under one or more agreements.
2 // The .NET Foundation licenses this file to you under the MIT license.
3 // See the LICENSE file in the project root for more information.
8 // IMPORTANT NOTES AND CAVEATS:
10 // This implementation is preliminary, and may change dramatically.
12 // New JIT types, TYP_SIMDxx, are introduced, and the SIMD intrinsics are created as GT_SIMD nodes.
13 // Nodes of SIMD types will be typed as TYP_SIMD* (e.g. TYP_SIMD8, TYP_SIMD16, etc.).
15 // Note that currently the "reference implementation" is the same as the runtime dll. As such, it is currently
16 // providing implementations for those methods not currently supported by the JIT as intrinsics.
18 // These are currently recognized using string compares, in order to provide an implementation in the JIT
19 // without taking a dependency on the VM.
20 // Furthermore, in the CTP, in order to limit the impact of doing these string compares
21 // against assembly names, we only look for the SIMDVector assembly if we are compiling a class constructor. This
22 // makes it somewhat more "pay for play" but is a significant usability compromise.
23 // This has been addressed for RTM by doing the assembly recognition in the VM.
24 // --------------------------------------------------------------------------------------
35 // Intrinsic Id to intrinsic info map
36 const SIMDIntrinsicInfo simdIntrinsicInfoArray[] = {
37 #define SIMD_INTRINSIC(mname, inst, id, name, retType, argCount, arg1, arg2, arg3, t1, t2, t3, t4, t5, t6, t7, t8, t9, \
39 {SIMDIntrinsic##id, mname, inst, retType, argCount, arg1, arg2, arg3, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10},
40 #include "simdintrinsiclist.h"
43 //------------------------------------------------------------------------
44 // getSIMDVectorLength: Get the length (number of elements of base type) of
45 // SIMD Vector given its size and base (element) type.
48 // simdSize - size of the SIMD vector
49 // baseType - type of the elements of the SIMD vector
52 int Compiler::getSIMDVectorLength(unsigned simdSize, var_types baseType)
54 return simdSize / genTypeSize(baseType);
57 //------------------------------------------------------------------------
58 // Get the length (number of elements of base type) of SIMD Vector given by typeHnd.
61 // typeHnd - type handle of the SIMD vector
63 int Compiler::getSIMDVectorLength(CORINFO_CLASS_HANDLE typeHnd)
65 unsigned sizeBytes = 0;
66 var_types baseType = getBaseTypeAndSizeOfSIMDType(typeHnd, &sizeBytes);
67 return getSIMDVectorLength(sizeBytes, baseType);
70 //------------------------------------------------------------------------
71 // Get the preferred alignment of SIMD vector type for better performance.
74 // typeHnd - type handle of the SIMD vector
76 int Compiler::getSIMDTypeAlignment(var_types simdType)
79 // Fixed length vectors have the following alignment preference
80 // Vector2 = 8 byte alignment
81 // Vector3/4 = 16-byte alignment
82 unsigned size = genTypeSize(simdType);
84 // preferred alignment for SSE2 128-bit vectors is 16-bytes
91 assert((size == 12) || (size == 16));
99 #elif defined(_TARGET_ARM64_)
102 assert(!"getSIMDTypeAlignment() unimplemented on target arch");
107 //----------------------------------------------------------------------------------
108 // Return the base type and size of SIMD vector type given its type handle.
111 // typeHnd - The handle of the type we're interested in.
112 // sizeBytes - out param
115 // base type of SIMD vector.
116 // sizeBytes if non-null is set to size in bytes.
118 // TODO-Throughput: current implementation parses class name to find base type. Change
119 // this when we implement SIMD intrinsic identification for the final
122 var_types Compiler::getBaseTypeAndSizeOfSIMDType(CORINFO_CLASS_HANDLE typeHnd, unsigned* sizeBytes /*= nullptr */)
126 if (m_simdHandleCache == nullptr)
128 if (impInlineInfo == nullptr)
130 m_simdHandleCache = new (this, CMK_Generic) SIMDHandlesCache();
134 // Steal the inliner compiler's cache (create it if not available).
136 if (impInlineInfo->InlineRoot->m_simdHandleCache == nullptr)
138 impInlineInfo->InlineRoot->m_simdHandleCache = new (this, CMK_Generic) SIMDHandlesCache();
141 m_simdHandleCache = impInlineInfo->InlineRoot->m_simdHandleCache;
145 if (typeHnd == nullptr)
150 // fast path search using cached type handles of important types
151 var_types simdBaseType = TYP_UNKNOWN;
154 // TODO - Optimize SIMD type recognition by IntrinsicAttribute
155 if (isSIMDClass(typeHnd))
157 // The most likely to be used type handles are looked up first followed by
158 // less likely to be used type handles
159 if (typeHnd == m_simdHandleCache->SIMDFloatHandle)
161 simdBaseType = TYP_FLOAT;
162 JITDUMP(" Known type SIMD Vector<Float>\n");
164 else if (typeHnd == m_simdHandleCache->SIMDIntHandle)
166 simdBaseType = TYP_INT;
167 JITDUMP(" Known type SIMD Vector<Int>\n");
169 else if (typeHnd == m_simdHandleCache->SIMDVector2Handle)
171 simdBaseType = TYP_FLOAT;
172 size = 2 * genTypeSize(TYP_FLOAT);
173 assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE));
174 JITDUMP(" Known type Vector2\n");
176 else if (typeHnd == m_simdHandleCache->SIMDVector3Handle)
178 simdBaseType = TYP_FLOAT;
179 size = 3 * genTypeSize(TYP_FLOAT);
180 assert(size == info.compCompHnd->getClassSize(typeHnd));
181 JITDUMP(" Known type Vector3\n");
183 else if (typeHnd == m_simdHandleCache->SIMDVector4Handle)
185 simdBaseType = TYP_FLOAT;
186 size = 4 * genTypeSize(TYP_FLOAT);
187 assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE));
188 JITDUMP(" Known type Vector4\n");
190 else if (typeHnd == m_simdHandleCache->SIMDVectorHandle)
192 JITDUMP(" Known type Vector\n");
194 else if (typeHnd == m_simdHandleCache->SIMDUShortHandle)
196 simdBaseType = TYP_USHORT;
197 JITDUMP(" Known type SIMD Vector<ushort>\n");
199 else if (typeHnd == m_simdHandleCache->SIMDUByteHandle)
201 simdBaseType = TYP_UBYTE;
202 JITDUMP(" Known type SIMD Vector<ubyte>\n");
204 else if (typeHnd == m_simdHandleCache->SIMDDoubleHandle)
206 simdBaseType = TYP_DOUBLE;
207 JITDUMP(" Known type SIMD Vector<Double>\n");
209 else if (typeHnd == m_simdHandleCache->SIMDLongHandle)
211 simdBaseType = TYP_LONG;
212 JITDUMP(" Known type SIMD Vector<Long>\n");
214 else if (typeHnd == m_simdHandleCache->SIMDShortHandle)
216 simdBaseType = TYP_SHORT;
217 JITDUMP(" Known type SIMD Vector<short>\n");
219 else if (typeHnd == m_simdHandleCache->SIMDByteHandle)
221 simdBaseType = TYP_BYTE;
222 JITDUMP(" Known type SIMD Vector<byte>\n");
224 else if (typeHnd == m_simdHandleCache->SIMDUIntHandle)
226 simdBaseType = TYP_UINT;
227 JITDUMP(" Known type SIMD Vector<uint>\n");
229 else if (typeHnd == m_simdHandleCache->SIMDULongHandle)
231 simdBaseType = TYP_ULONG;
232 JITDUMP(" Known type SIMD Vector<ulong>\n");
236 if (simdBaseType == TYP_UNKNOWN)
238 // Doesn't match with any of the cached type handles.
239 // Obtain base type by parsing fully qualified class name.
241 // TODO-Throughput: implement product shipping solution to query base type.
242 WCHAR className[256] = {0};
243 WCHAR* pbuf = &className[0];
244 int len = _countof(className);
245 info.compCompHnd->appendClassName(&pbuf, &len, typeHnd, TRUE, FALSE, FALSE);
246 noway_assert(pbuf < &className[256]);
247 JITDUMP("SIMD Candidate Type %S\n", className);
249 if (wcsncmp(className, W("System.Numerics."), 16) == 0)
251 if (wcsncmp(&(className[16]), W("Vector`1["), 9) == 0)
253 if (wcsncmp(&(className[25]), W("System.Single"), 13) == 0)
255 m_simdHandleCache->SIMDFloatHandle = typeHnd;
256 simdBaseType = TYP_FLOAT;
257 JITDUMP(" Found type SIMD Vector<Float>\n");
259 else if (wcsncmp(&(className[25]), W("System.Int32"), 12) == 0)
261 m_simdHandleCache->SIMDIntHandle = typeHnd;
262 simdBaseType = TYP_INT;
263 JITDUMP(" Found type SIMD Vector<Int>\n");
265 else if (wcsncmp(&(className[25]), W("System.UInt16"), 13) == 0)
267 m_simdHandleCache->SIMDUShortHandle = typeHnd;
268 simdBaseType = TYP_USHORT;
269 JITDUMP(" Found type SIMD Vector<ushort>\n");
271 else if (wcsncmp(&(className[25]), W("System.Byte"), 11) == 0)
273 m_simdHandleCache->SIMDUByteHandle = typeHnd;
274 simdBaseType = TYP_UBYTE;
275 JITDUMP(" Found type SIMD Vector<ubyte>\n");
277 else if (wcsncmp(&(className[25]), W("System.Double"), 13) == 0)
279 m_simdHandleCache->SIMDDoubleHandle = typeHnd;
280 simdBaseType = TYP_DOUBLE;
281 JITDUMP(" Found type SIMD Vector<Double>\n");
283 else if (wcsncmp(&(className[25]), W("System.Int64"), 12) == 0)
285 m_simdHandleCache->SIMDLongHandle = typeHnd;
286 simdBaseType = TYP_LONG;
287 JITDUMP(" Found type SIMD Vector<Long>\n");
289 else if (wcsncmp(&(className[25]), W("System.Int16"), 12) == 0)
291 m_simdHandleCache->SIMDShortHandle = typeHnd;
292 simdBaseType = TYP_SHORT;
293 JITDUMP(" Found type SIMD Vector<short>\n");
295 else if (wcsncmp(&(className[25]), W("System.SByte"), 12) == 0)
297 m_simdHandleCache->SIMDByteHandle = typeHnd;
298 simdBaseType = TYP_BYTE;
299 JITDUMP(" Found type SIMD Vector<byte>\n");
301 else if (wcsncmp(&(className[25]), W("System.UInt32"), 13) == 0)
303 m_simdHandleCache->SIMDUIntHandle = typeHnd;
304 simdBaseType = TYP_UINT;
305 JITDUMP(" Found type SIMD Vector<uint>\n");
307 else if (wcsncmp(&(className[25]), W("System.UInt64"), 13) == 0)
309 m_simdHandleCache->SIMDULongHandle = typeHnd;
310 simdBaseType = TYP_ULONG;
311 JITDUMP(" Found type SIMD Vector<ulong>\n");
315 JITDUMP(" Unknown SIMD Vector<T>\n");
318 else if (wcsncmp(&(className[16]), W("Vector2"), 8) == 0)
320 m_simdHandleCache->SIMDVector2Handle = typeHnd;
322 simdBaseType = TYP_FLOAT;
323 size = 2 * genTypeSize(TYP_FLOAT);
324 assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE));
325 JITDUMP(" Found Vector2\n");
327 else if (wcsncmp(&(className[16]), W("Vector3"), 8) == 0)
329 m_simdHandleCache->SIMDVector3Handle = typeHnd;
331 simdBaseType = TYP_FLOAT;
332 size = 3 * genTypeSize(TYP_FLOAT);
333 assert(size == info.compCompHnd->getClassSize(typeHnd));
334 JITDUMP(" Found Vector3\n");
336 else if (wcsncmp(&(className[16]), W("Vector4"), 8) == 0)
338 m_simdHandleCache->SIMDVector4Handle = typeHnd;
340 simdBaseType = TYP_FLOAT;
341 size = 4 * genTypeSize(TYP_FLOAT);
342 assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE));
343 JITDUMP(" Found Vector4\n");
345 else if (wcsncmp(&(className[16]), W("Vector"), 6) == 0)
347 m_simdHandleCache->SIMDVectorHandle = typeHnd;
348 JITDUMP(" Found type Vector\n");
352 JITDUMP(" Unknown SIMD Type\n");
356 if (simdBaseType != TYP_UNKNOWN && sizeBytes != nullptr)
358 // If not a fixed size vector then its size is same as SIMD vector
359 // register length in bytes
362 size = getSIMDVectorRegisterByteLength();
366 setUsesSIMDTypes(true);
369 #ifdef FEATURE_HW_INTRINSICS
370 else if (isIntrinsicType(typeHnd))
372 const size_t Vector64SizeBytes = 64 / 8;
373 const size_t Vector128SizeBytes = 128 / 8;
374 const size_t Vector256SizeBytes = 256 / 8;
376 #if defined(_TARGET_XARCH_)
377 static_assert_no_msg(YMM_REGSIZE_BYTES == Vector256SizeBytes);
378 static_assert_no_msg(XMM_REGSIZE_BYTES == Vector128SizeBytes);
380 if (typeHnd == m_simdHandleCache->Vector256FloatHandle)
382 simdBaseType = TYP_FLOAT;
383 size = Vector256SizeBytes;
384 JITDUMP(" Known type Vector256<float>\n");
386 else if (typeHnd == m_simdHandleCache->Vector256DoubleHandle)
388 simdBaseType = TYP_DOUBLE;
389 size = Vector256SizeBytes;
390 JITDUMP(" Known type Vector256<double>\n");
392 else if (typeHnd == m_simdHandleCache->Vector256IntHandle)
394 simdBaseType = TYP_INT;
395 size = Vector256SizeBytes;
396 JITDUMP(" Known type Vector256<int>\n");
398 else if (typeHnd == m_simdHandleCache->Vector256UIntHandle)
400 simdBaseType = TYP_UINT;
401 size = Vector256SizeBytes;
402 JITDUMP(" Known type Vector256<uint>\n");
404 else if (typeHnd == m_simdHandleCache->Vector256ShortHandle)
406 simdBaseType = TYP_SHORT;
407 size = Vector256SizeBytes;
408 JITDUMP(" Known type Vector256<short>\n");
410 else if (typeHnd == m_simdHandleCache->Vector256UShortHandle)
412 simdBaseType = TYP_USHORT;
413 size = Vector256SizeBytes;
414 JITDUMP(" Known type Vector256<ushort>\n");
416 else if (typeHnd == m_simdHandleCache->Vector256ByteHandle)
418 simdBaseType = TYP_BYTE;
419 size = Vector256SizeBytes;
420 JITDUMP(" Known type Vector256<sbyte>\n");
422 else if (typeHnd == m_simdHandleCache->Vector256UByteHandle)
424 simdBaseType = TYP_UBYTE;
425 size = Vector256SizeBytes;
426 JITDUMP(" Known type Vector256<byte>\n");
428 else if (typeHnd == m_simdHandleCache->Vector256LongHandle)
430 simdBaseType = TYP_LONG;
431 size = Vector256SizeBytes;
432 JITDUMP(" Known type Vector256<long>\n");
434 else if (typeHnd == m_simdHandleCache->Vector256ULongHandle)
436 simdBaseType = TYP_ULONG;
437 size = Vector256SizeBytes;
438 JITDUMP(" Known type Vector256<ulong>\n");
441 #endif // defined(_TARGET_XARCH)
442 if (typeHnd == m_simdHandleCache->Vector128FloatHandle)
444 simdBaseType = TYP_FLOAT;
445 size = Vector128SizeBytes;
446 JITDUMP(" Known type Vector128<float>\n");
448 else if (typeHnd == m_simdHandleCache->Vector128DoubleHandle)
450 simdBaseType = TYP_DOUBLE;
451 size = Vector128SizeBytes;
452 JITDUMP(" Known type Vector128<double>\n");
454 else if (typeHnd == m_simdHandleCache->Vector128IntHandle)
456 simdBaseType = TYP_INT;
457 size = Vector128SizeBytes;
458 JITDUMP(" Known type Vector128<int>\n");
460 else if (typeHnd == m_simdHandleCache->Vector128UIntHandle)
462 simdBaseType = TYP_UINT;
463 size = Vector128SizeBytes;
464 JITDUMP(" Known type Vector128<uint>\n");
466 else if (typeHnd == m_simdHandleCache->Vector128ShortHandle)
468 simdBaseType = TYP_SHORT;
469 size = Vector128SizeBytes;
470 JITDUMP(" Known type Vector128<short>\n");
472 else if (typeHnd == m_simdHandleCache->Vector128UShortHandle)
474 simdBaseType = TYP_USHORT;
475 size = Vector128SizeBytes;
476 JITDUMP(" Known type Vector128<ushort>\n");
478 else if (typeHnd == m_simdHandleCache->Vector128ByteHandle)
480 simdBaseType = TYP_BYTE;
481 size = Vector128SizeBytes;
482 JITDUMP(" Known type Vector128<sbyte>\n");
484 else if (typeHnd == m_simdHandleCache->Vector128UByteHandle)
486 simdBaseType = TYP_UBYTE;
487 size = Vector128SizeBytes;
488 JITDUMP(" Known type Vector128<byte>\n");
490 else if (typeHnd == m_simdHandleCache->Vector128LongHandle)
492 simdBaseType = TYP_LONG;
493 size = Vector128SizeBytes;
494 JITDUMP(" Known type Vector128<long>\n");
496 else if (typeHnd == m_simdHandleCache->Vector128ULongHandle)
498 simdBaseType = TYP_ULONG;
499 size = Vector128SizeBytes;
500 JITDUMP(" Known type Vector128<ulong>\n");
503 #if defined(_TARGET_ARM64_)
504 if (typeHnd == m_simdHandleCache->Vector64FloatHandle)
506 simdBaseType = TYP_FLOAT;
507 size = Vector64SizeBytes;
508 JITDUMP(" Known type Vector64<float>\n");
510 else if (typeHnd == m_simdHandleCache->Vector64IntHandle)
512 simdBaseType = TYP_INT;
513 size = Vector64SizeBytes;
514 JITDUMP(" Known type Vector64<int>\n");
516 else if (typeHnd == m_simdHandleCache->Vector64UIntHandle)
518 simdBaseType = TYP_UINT;
519 size = Vector64SizeBytes;
520 JITDUMP(" Known type Vector64<uint>\n");
522 else if (typeHnd == m_simdHandleCache->Vector64ShortHandle)
524 simdBaseType = TYP_SHORT;
525 size = Vector64SizeBytes;
526 JITDUMP(" Known type Vector64<short>\n");
528 else if (typeHnd == m_simdHandleCache->Vector64UShortHandle)
530 simdBaseType = TYP_USHORT;
531 size = Vector64SizeBytes;
532 JITDUMP(" Known type Vector64<ushort>\n");
534 else if (typeHnd == m_simdHandleCache->Vector64ByteHandle)
536 simdBaseType = TYP_BYTE;
537 size = Vector64SizeBytes;
538 JITDUMP(" Known type Vector64<sbyte>\n");
540 else if (typeHnd == m_simdHandleCache->Vector64UByteHandle)
542 simdBaseType = TYP_UBYTE;
543 size = Vector64SizeBytes;
544 JITDUMP(" Known type Vector64<byte>\n");
546 #endif // defined(_TARGET_ARM64_)
549 if (simdBaseType == TYP_UNKNOWN)
551 // Doesn't match with any of the cached type handles.
552 const char* className = getClassNameFromMetadata(typeHnd, nullptr);
553 CORINFO_CLASS_HANDLE baseTypeHnd = getTypeInstantiationArgument(typeHnd, 0);
555 if (baseTypeHnd != nullptr)
557 CorInfoType type = info.compCompHnd->getTypeForPrimitiveNumericClass(baseTypeHnd);
559 JITDUMP("HW Intrinsic SIMD Candidate Type %s with Base Type %s\n", className,
560 getClassNameFromMetadata(baseTypeHnd, nullptr));
562 #if defined(_TARGET_XARCH_)
563 if (strcmp(className, "Vector256`1") == 0)
565 size = Vector256SizeBytes;
568 case CORINFO_TYPE_FLOAT:
569 m_simdHandleCache->Vector256FloatHandle = typeHnd;
570 simdBaseType = TYP_FLOAT;
571 JITDUMP(" Found type Hardware Intrinsic SIMD Vector256<float>\n");
573 case CORINFO_TYPE_DOUBLE:
574 m_simdHandleCache->Vector256DoubleHandle = typeHnd;
575 simdBaseType = TYP_DOUBLE;
576 JITDUMP(" Found type Hardware Intrinsic SIMD Vector256<double>\n");
578 case CORINFO_TYPE_INT:
579 m_simdHandleCache->Vector256IntHandle = typeHnd;
580 simdBaseType = TYP_INT;
581 JITDUMP(" Found type Hardware Intrinsic SIMD Vector256<int>\n");
583 case CORINFO_TYPE_UINT:
584 m_simdHandleCache->Vector256UIntHandle = typeHnd;
585 simdBaseType = TYP_UINT;
586 JITDUMP(" Found type Hardware Intrinsic SIMD Vector256<uint>\n");
588 case CORINFO_TYPE_SHORT:
589 m_simdHandleCache->Vector256ShortHandle = typeHnd;
590 simdBaseType = TYP_SHORT;
591 JITDUMP(" Found type Hardware Intrinsic SIMD Vector256<short>\n");
593 case CORINFO_TYPE_USHORT:
594 m_simdHandleCache->Vector256UShortHandle = typeHnd;
595 simdBaseType = TYP_USHORT;
596 JITDUMP(" Found type Hardware Intrinsic SIMD Vector256<ushort>\n");
598 case CORINFO_TYPE_LONG:
599 m_simdHandleCache->Vector256LongHandle = typeHnd;
600 simdBaseType = TYP_LONG;
601 JITDUMP(" Found type Hardware Intrinsic SIMD Vector256<long>\n");
603 case CORINFO_TYPE_ULONG:
604 m_simdHandleCache->Vector256ULongHandle = typeHnd;
605 simdBaseType = TYP_ULONG;
606 JITDUMP(" Found type Hardware Intrinsic SIMD Vector256<ulong>\n");
608 case CORINFO_TYPE_UBYTE:
609 m_simdHandleCache->Vector256UByteHandle = typeHnd;
610 simdBaseType = TYP_UBYTE;
611 JITDUMP(" Found type Hardware Intrinsic SIMD Vector256<byte>\n");
613 case CORINFO_TYPE_BYTE:
614 m_simdHandleCache->Vector256ByteHandle = typeHnd;
615 simdBaseType = TYP_BYTE;
616 JITDUMP(" Found type Hardware Intrinsic SIMD Vector256<sbyte>\n");
620 JITDUMP(" Unknown Hardware Intrinsic SIMD Type Vector256<T>\n");
624 #endif // defined(_TARGET_XARCH_)
625 if (strcmp(className, "Vector128`1") == 0)
627 size = Vector128SizeBytes;
630 case CORINFO_TYPE_FLOAT:
631 m_simdHandleCache->Vector128FloatHandle = typeHnd;
632 simdBaseType = TYP_FLOAT;
633 JITDUMP(" Found type Hardware Intrinsic SIMD Vector128<float>\n");
635 case CORINFO_TYPE_DOUBLE:
636 m_simdHandleCache->Vector128DoubleHandle = typeHnd;
637 simdBaseType = TYP_DOUBLE;
638 JITDUMP(" Found type Hardware Intrinsic SIMD Vector128<double>\n");
640 case CORINFO_TYPE_INT:
641 m_simdHandleCache->Vector128IntHandle = typeHnd;
642 simdBaseType = TYP_INT;
643 JITDUMP(" Found type Hardware Intrinsic SIMD Vector128<int>\n");
645 case CORINFO_TYPE_UINT:
646 m_simdHandleCache->Vector128UIntHandle = typeHnd;
647 simdBaseType = TYP_UINT;
648 JITDUMP(" Found type Hardware Intrinsic SIMD Vector128<uint>\n");
650 case CORINFO_TYPE_SHORT:
651 m_simdHandleCache->Vector128ShortHandle = typeHnd;
652 simdBaseType = TYP_SHORT;
653 JITDUMP(" Found type Hardware Intrinsic SIMD Vector128<short>\n");
655 case CORINFO_TYPE_USHORT:
656 m_simdHandleCache->Vector128UShortHandle = typeHnd;
657 simdBaseType = TYP_USHORT;
658 JITDUMP(" Found type Hardware Intrinsic SIMD Vector128<ushort>\n");
660 case CORINFO_TYPE_LONG:
661 m_simdHandleCache->Vector128LongHandle = typeHnd;
662 simdBaseType = TYP_LONG;
663 JITDUMP(" Found type Hardware Intrinsic SIMD Vector128<long>\n");
665 case CORINFO_TYPE_ULONG:
666 m_simdHandleCache->Vector128ULongHandle = typeHnd;
667 simdBaseType = TYP_ULONG;
668 JITDUMP(" Found type Hardware Intrinsic SIMD Vector128<ulong>\n");
670 case CORINFO_TYPE_UBYTE:
671 m_simdHandleCache->Vector128UByteHandle = typeHnd;
672 simdBaseType = TYP_UBYTE;
673 JITDUMP(" Found type Hardware Intrinsic SIMD Vector128<byte>\n");
675 case CORINFO_TYPE_BYTE:
676 m_simdHandleCache->Vector128ByteHandle = typeHnd;
677 simdBaseType = TYP_BYTE;
678 JITDUMP(" Found type Hardware Intrinsic SIMD Vector128<sbyte>\n");
682 JITDUMP(" Unknown Hardware Intrinsic SIMD Type Vector128<T>\n");
685 #if defined(_TARGET_ARM64_)
686 else if (strcmp(className, "Vector64`1") == 0)
688 size = Vector64SizeBytes;
691 case CORINFO_TYPE_FLOAT:
692 m_simdHandleCache->Vector64FloatHandle = typeHnd;
693 simdBaseType = TYP_FLOAT;
694 JITDUMP(" Found type Hardware Intrinsic SIMD Vector64<float>\n");
696 case CORINFO_TYPE_INT:
697 m_simdHandleCache->Vector64IntHandle = typeHnd;
698 simdBaseType = TYP_INT;
699 JITDUMP(" Found type Hardware Intrinsic SIMD Vector64<int>\n");
701 case CORINFO_TYPE_UINT:
702 m_simdHandleCache->Vector64UIntHandle = typeHnd;
703 simdBaseType = TYP_UINT;
704 JITDUMP(" Found type Hardware Intrinsic SIMD Vector64<uint>\n");
706 case CORINFO_TYPE_SHORT:
707 m_simdHandleCache->Vector64ShortHandle = typeHnd;
708 simdBaseType = TYP_SHORT;
709 JITDUMP(" Found type Hardware Intrinsic SIMD Vector64<short>\n");
711 case CORINFO_TYPE_USHORT:
712 m_simdHandleCache->Vector64UShortHandle = typeHnd;
713 simdBaseType = TYP_USHORT;
714 JITDUMP(" Found type Hardware Intrinsic SIMD Vector64<ushort>\n");
716 case CORINFO_TYPE_UBYTE:
717 m_simdHandleCache->Vector64UByteHandle = typeHnd;
718 simdBaseType = TYP_UBYTE;
719 JITDUMP(" Found type Hardware Intrinsic SIMD Vector64<byte>\n");
721 case CORINFO_TYPE_BYTE:
722 m_simdHandleCache->Vector64ByteHandle = typeHnd;
723 simdBaseType = TYP_BYTE;
724 JITDUMP(" Found type Hardware Intrinsic SIMD Vector64<sbyte>\n");
728 JITDUMP(" Unknown Hardware Intrinsic SIMD Type Vector64<T>\n");
731 #endif // defined(_TARGET_ARM64_)
735 if (sizeBytes != nullptr)
740 if (simdBaseType != TYP_UNKNOWN)
742 setUsesSIMDTypes(true);
745 #endif // FEATURE_HW_INTRINSICS
750 //--------------------------------------------------------------------------------------
751 // getSIMDIntrinsicInfo: get SIMD intrinsic info given the method handle.
754 // inOutTypeHnd - The handle of the type on which the method is invoked. This is an in-out param.
755 // methodHnd - The handle of the method we're interested in.
756 // sig - method signature info
757 // isNewObj - whether this call represents a newboj constructor call
758 // argCount - argument count - out pram
759 // baseType - base type of the intrinsic - out param
760 // sizeBytes - size of SIMD vector type on which the method is invoked - out param
763 // SIMDIntrinsicInfo struct initialized corresponding to methodHnd.
764 // Sets SIMDIntrinsicInfo.id to SIMDIntrinsicInvalid if methodHnd doesn't correspond
765 // to any SIMD intrinsic. Also, sets the out params inOutTypeHnd, argCount, baseType and
768 // Note that VectorMath class doesn't have a base type and first argument of the method
769 // determines the SIMD vector type on which intrinsic is invoked. In such a case inOutTypeHnd
770 // is modified by this routine.
772 // TODO-Throughput: The current implementation is based on method name string parsing.
773 // Although we now have type identification from the VM, the parsing of intrinsic names
774 // could be made more efficient.
776 const SIMDIntrinsicInfo* Compiler::getSIMDIntrinsicInfo(CORINFO_CLASS_HANDLE* inOutTypeHnd,
777 CORINFO_METHOD_HANDLE methodHnd,
778 CORINFO_SIG_INFO* sig,
785 assert(baseType != nullptr);
786 assert(sizeBytes != nullptr);
788 // get baseType and size of the type
789 CORINFO_CLASS_HANDLE typeHnd = *inOutTypeHnd;
790 *baseType = getBaseTypeAndSizeOfSIMDType(typeHnd, sizeBytes);
792 if (typeHnd == m_simdHandleCache->SIMDVectorHandle)
794 // All of the supported intrinsics on this static class take a first argument that's a vector,
795 // which determines the baseType.
796 // The exception is the IsHardwareAccelerated property, which is handled as a special case.
797 assert(*baseType == TYP_UNKNOWN);
798 if (sig->numArgs == 0)
800 const SIMDIntrinsicInfo* hwAccelIntrinsicInfo = &(simdIntrinsicInfoArray[SIMDIntrinsicHWAccel]);
801 if ((strcmp(eeGetMethodName(methodHnd, nullptr), hwAccelIntrinsicInfo->methodName) == 0) &&
802 JITtype2varType(sig->retType) == hwAccelIntrinsicInfo->retType)
805 assert(hwAccelIntrinsicInfo->argCount == 0 && hwAccelIntrinsicInfo->isInstMethod == false);
806 return hwAccelIntrinsicInfo;
812 typeHnd = info.compCompHnd->getArgClass(sig, sig->args);
813 *inOutTypeHnd = typeHnd;
814 *baseType = getBaseTypeAndSizeOfSIMDType(typeHnd, sizeBytes);
818 if (*baseType == TYP_UNKNOWN)
820 JITDUMP("NOT a SIMD Intrinsic: unsupported baseType\n");
824 // account for implicit "this" arg
825 *argCount = sig->numArgs;
831 // Get the Intrinsic Id by parsing method name.
833 // TODO-Throughput: replace sequential search by binary search by arranging entries
834 // sorted by method name.
835 SIMDIntrinsicID intrinsicId = SIMDIntrinsicInvalid;
836 const char* methodName = eeGetMethodName(methodHnd, nullptr);
837 for (int i = SIMDIntrinsicNone + 1; i < SIMDIntrinsicInvalid; ++i)
839 if (strcmp(methodName, simdIntrinsicInfoArray[i].methodName) == 0)
841 // Found an entry for the method; further check whether it is one of
842 // the supported base types.
844 for (int j = 0; j < SIMD_INTRINSIC_MAX_BASETYPE_COUNT; ++j)
846 // Convention: if there are fewer base types supported than MAX_BASETYPE_COUNT,
847 // the end of the list is marked by TYP_UNDEF.
848 if (simdIntrinsicInfoArray[i].supportedBaseTypes[j] == TYP_UNDEF)
853 if (simdIntrinsicInfoArray[i].supportedBaseTypes[j] == *baseType)
865 // Now, check the arguments.
866 unsigned int fixedArgCnt = simdIntrinsicInfoArray[i].argCount;
867 unsigned int expectedArgCnt = fixedArgCnt;
869 // First handle SIMDIntrinsicInitN, where the arg count depends on the type.
870 // The listed arg types include the vector and the first two init values, which is the expected number
871 // for Vector2. For other cases, we'll check their types here.
872 if (*argCount > expectedArgCnt)
874 if (i == SIMDIntrinsicInitN)
876 if (*argCount == 3 && typeHnd == m_simdHandleCache->SIMDVector2Handle)
880 else if (*argCount == 4 && typeHnd == m_simdHandleCache->SIMDVector3Handle)
884 else if (*argCount == 5 && typeHnd == m_simdHandleCache->SIMDVector4Handle)
889 else if (i == SIMDIntrinsicInitFixed)
891 if (*argCount == 4 && typeHnd == m_simdHandleCache->SIMDVector4Handle)
897 if (*argCount != expectedArgCnt)
902 // Validate the types of individual args passed are what is expected of.
903 // If any of the types don't match with what is expected, don't consider
904 // as an intrinsic. This will make an older JIT with SIMD capabilities
905 // resilient to breaking changes to SIMD managed API.
907 // Note that from IL type stack, args get popped in right to left order
908 // whereas args get listed in method signatures in left to right order.
910 int stackIndex = (expectedArgCnt - 1);
912 // Track the arguments from the signature - we currently only use this to distinguish
913 // integral and pointer types, both of which will by TYP_I_IMPL on the importer stack.
914 CORINFO_ARG_LIST_HANDLE argLst = sig->args;
916 CORINFO_CLASS_HANDLE argClass;
917 for (unsigned int argIndex = 0; found == true && argIndex < expectedArgCnt; argIndex++)
919 bool isThisPtr = ((argIndex == 0) && sig->hasThis());
921 // In case of "newobj SIMDVector<T>(T val)", thisPtr won't be present on type stack.
922 // We don't check anything in that case.
923 if (!isThisPtr || !isNewObj)
925 GenTree* arg = impStackTop(stackIndex).val;
926 var_types argType = arg->TypeGet();
928 var_types expectedArgType;
929 if (argIndex < fixedArgCnt)
932 // - intrinsicInfo.argType[i] == TYP_UNDEF - intrinsic doesn't have a valid arg at position i
933 // - intrinsicInfo.argType[i] == TYP_UNKNOWN - arg type should be same as basetype
934 // Note that we pop the args off in reverse order.
935 expectedArgType = simdIntrinsicInfoArray[i].argType[argIndex];
936 assert(expectedArgType != TYP_UNDEF);
937 if (expectedArgType == TYP_UNKNOWN)
939 // The type of the argument will be genActualType(*baseType).
940 expectedArgType = genActualType(*baseType);
941 argType = genActualType(argType);
946 expectedArgType = *baseType;
949 if (!isThisPtr && argType == TYP_I_IMPL)
951 // The reference implementation has a constructor that takes a pointer.
952 // We don't want to recognize that one. This requires us to look at the CorInfoType
953 // in order to distinguish a signature with a pointer argument from one with an
954 // integer argument of pointer size, both of which will be TYP_I_IMPL on the stack.
955 // TODO-Review: This seems quite fragile. We should consider beefing up the checking
957 CorInfoType corType = strip(info.compCompHnd->getArgType(sig, argLst, &argClass));
958 if (corType == CORINFO_TYPE_PTR)
964 if (varTypeIsSIMD(argType))
966 argType = TYP_STRUCT;
968 if (argType != expectedArgType)
973 if (argIndex != 0 || !sig->hasThis())
975 argLst = info.compCompHnd->getArgNext(argLst);
980 // Cross check return type and static vs. instance is what we are expecting.
981 // If not, don't consider it as an intrinsic.
982 // Note that ret type of TYP_UNKNOWN means that it is not known apriori and must be same as baseType
985 var_types expectedRetType = simdIntrinsicInfoArray[i].retType;
986 if (expectedRetType == TYP_UNKNOWN)
988 // JIT maps uint/ulong type vars to TYP_INT/TYP_LONG.
990 (*baseType == TYP_UINT || *baseType == TYP_ULONG) ? genActualType(*baseType) : *baseType;
993 if (JITtype2varType(sig->retType) != expectedRetType ||
994 sig->hasThis() != simdIntrinsicInfoArray[i].isInstMethod)
1002 intrinsicId = (SIMDIntrinsicID)i;
1008 if (intrinsicId != SIMDIntrinsicInvalid)
1010 JITDUMP("Method %s maps to SIMD intrinsic %s\n", methodName, simdIntrinsicNames[intrinsicId]);
1011 return &simdIntrinsicInfoArray[intrinsicId];
1015 JITDUMP("Method %s is NOT a SIMD intrinsic\n", methodName);
1021 // Pops and returns GenTree node from importer's type stack.
1022 // Normalizes TYP_STRUCT value in case of GT_CALL, GT_RET_EXPR and arg nodes.
1025 // type - the type of value that the caller expects to be popped off the stack.
1026 // expectAddr - if true indicates we are expecting type stack entry to be a TYP_BYREF.
1027 // structHandle - the class handle to use when normalizing if it is not the same as the stack entry class handle;
1028 // this can happen for certain scenarios, such as folding away a static cast, where we want the
1029 // value popped to have the type that would have been returned.
1032 // If the popped value is a struct, and the expected type is a simd type, it will be set
1033 // to that type, otherwise it will assert if the type being popped is not the expected type.
1035 GenTree* Compiler::impSIMDPopStack(var_types type, bool expectAddr, CORINFO_CLASS_HANDLE structHandle)
1037 StackEntry se = impPopStack();
1038 typeInfo ti = se.seTypeInfo;
1039 GenTree* tree = se.val;
1041 // If expectAddr is true implies what we have on stack is address and we need
1042 // SIMD type struct that it points to.
1045 assert(tree->TypeGet() == TYP_BYREF);
1046 if (tree->OperGet() == GT_ADDR)
1048 tree = tree->gtGetOp1();
1052 tree = gtNewOperNode(GT_IND, type, tree);
1056 bool isParam = false;
1058 // If we are popping a struct type it must have a matching handle if one is specified.
1059 // - If we have an existing 'OBJ' and 'structHandle' is specified, we will change its
1060 // handle if it doesn't match.
1061 // This can happen when we have a retyping of a vector that doesn't translate to any
1063 // - (If it's not an OBJ and it's used in a parameter context where it is required,
1064 // impNormStructVal will add one).
1066 if (tree->OperGet() == GT_OBJ)
1068 if ((structHandle != NO_CLASS_HANDLE) && (tree->AsObj()->gtClass != structHandle))
1070 // In this case we need to retain the GT_OBJ to retype the value.
1071 tree->AsObj()->gtClass = structHandle;
1075 GenTree* addr = tree->gtOp.gtOp1;
1076 if ((addr->OperGet() == GT_ADDR) && isSIMDTypeLocal(addr->gtOp.gtOp1))
1078 tree = addr->gtOp.gtOp1;
1083 if (tree->OperGet() == GT_LCL_VAR)
1085 unsigned lclNum = tree->AsLclVarCommon()->GetLclNum();
1086 LclVarDsc* lclVarDsc = &lvaTable[lclNum];
1087 isParam = lclVarDsc->lvIsParam;
1090 // normalize TYP_STRUCT value
1091 if (varTypeIsStruct(tree) && ((tree->OperGet() == GT_RET_EXPR) || (tree->OperGet() == GT_CALL) || isParam))
1093 assert(ti.IsType(TI_STRUCT));
1095 if (structHandle == nullptr)
1097 structHandle = ti.GetClassHandleForValueClass();
1100 tree = impNormStructVal(tree, structHandle, (unsigned)CHECK_SPILL_ALL);
1103 // Now set the type of the tree to the specialized SIMD struct type, if applicable.
1104 if (genActualType(tree->gtType) != genActualType(type))
1106 assert(tree->gtType == TYP_STRUCT);
1107 tree->gtType = type;
1109 else if (tree->gtType == TYP_BYREF)
1111 assert(tree->IsLocal() || (tree->OperGet() == GT_RET_EXPR) || (tree->OperGet() == GT_CALL) ||
1112 ((tree->gtOper == GT_ADDR) && varTypeIsSIMD(tree->gtGetOp1())));
1118 // impSIMDGetFixed: Create a GT_SIMD tree for a Get property of SIMD vector with a fixed index.
1121 // baseType - The base (element) type of the SIMD vector.
1122 // simdSize - The total size in bytes of the SIMD vector.
1123 // index - The index of the field to get.
1126 // Returns a GT_SIMD node with the SIMDIntrinsicGetItem intrinsic id.
1128 GenTreeSIMD* Compiler::impSIMDGetFixed(var_types simdType, var_types baseType, unsigned simdSize, int index)
1130 assert(simdSize >= ((index + 1) * genTypeSize(baseType)));
1132 // op1 is a SIMD source.
1133 GenTree* op1 = impSIMDPopStack(simdType, true);
1135 GenTree* op2 = gtNewIconNode(index);
1136 GenTreeSIMD* simdTree = gtNewSIMDNode(baseType, op1, op2, SIMDIntrinsicGetItem, baseType, simdSize);
1140 #ifdef _TARGET_XARCH_
1141 // impSIMDLongRelOpEqual: transforms operands and returns the SIMD intrinsic to be applied on
1142 // transformed operands to obtain == comparison result.
1145 // typeHnd - type handle of SIMD vector
1146 // size - SIMD vector size
1147 // op1 - in-out parameter; first operand
1148 // op2 - in-out parameter; second operand
1151 // Modifies in-out params op1, op2 and returns intrinsic ID to be applied to modified operands
1153 SIMDIntrinsicID Compiler::impSIMDLongRelOpEqual(CORINFO_CLASS_HANDLE typeHnd,
1158 var_types simdType = (*pOp1)->TypeGet();
1159 assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1161 // There is no direct SSE2 support for comparing TYP_LONG vectors.
1162 // These have to be implemented in terms of TYP_INT vector comparison operations.
1164 // Equality(v1, v2):
1165 // tmp = (v1 == v2) i.e. compare for equality as if v1 and v2 are vector<int>
1166 // result = BitwiseAnd(t, shuffle(t, (2, 3, 0, 1)))
1167 // Shuffle is meant to swap the comparison results of low-32-bits and high 32-bits of respective long elements.
1169 // Compare vector<long> as if they were vector<int> and assign the result to a temp
1170 GenTree* compResult = gtNewSIMDNode(simdType, *pOp1, *pOp2, SIMDIntrinsicEqual, TYP_INT, size);
1171 unsigned lclNum = lvaGrabTemp(true DEBUGARG("SIMD Long =="));
1172 lvaSetStruct(lclNum, typeHnd, false);
1173 GenTree* tmp = gtNewLclvNode(lclNum, simdType);
1174 GenTree* asg = gtNewTempAssign(lclNum, compResult);
1176 // op1 = GT_COMMA(tmp=compResult, tmp)
1177 // op2 = Shuffle(tmp, 0xB1)
1178 // IntrinsicId = BitwiseAnd
1179 *pOp1 = gtNewOperNode(GT_COMMA, simdType, asg, tmp);
1180 *pOp2 = gtNewSIMDNode(simdType, gtNewLclvNode(lclNum, simdType), gtNewIconNode(SHUFFLE_ZWXY, TYP_INT),
1181 SIMDIntrinsicShuffleSSE2, TYP_INT, size);
1182 return SIMDIntrinsicBitwiseAnd;
1185 // impSIMDLongRelOpGreaterThan: transforms operands and returns the SIMD intrinsic to be applied on
1186 // transformed operands to obtain > comparison result.
1189 // typeHnd - type handle of SIMD vector
1190 // size - SIMD vector size
1191 // pOp1 - in-out parameter; first operand
1192 // pOp2 - in-out parameter; second operand
1195 // Modifies in-out params pOp1, pOp2 and returns intrinsic ID to be applied to modified operands
1197 SIMDIntrinsicID Compiler::impSIMDLongRelOpGreaterThan(CORINFO_CLASS_HANDLE typeHnd,
1202 var_types simdType = (*pOp1)->TypeGet();
1203 assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1205 // GreaterThan(v1, v2) where v1 and v2 are vector long.
1206 // Let us consider the case of single long element comparison.
1207 // say L1 = (x1, y1) and L2 = (x2, y2) where x1, y1, x2, and y2 are 32-bit integers that comprise the longs L1 and
1210 // GreaterThan(L1, L2) can be expressed in terms of > relationship between 32-bit integers that comprise L1 and L2
1212 // = (x1, y1) > (x2, y2)
1213 // = (x1 > x2) || [(x1 == x2) && (y1 > y2)] - eq (1)
1215 // t = (v1 > v2) 32-bit signed comparison
1216 // u = (v1 == v2) 32-bit sized element equality
1217 // v = (v1 > v2) 32-bit unsigned comparison
1219 // z = shuffle(t, (3, 3, 1, 1)) - This corresponds to (x1 > x2) in eq(1) above
1220 // t1 = Shuffle(v, (2, 2, 0, 0)) - This corresponds to (y1 > y2) in eq(1) above
1221 // u1 = Shuffle(u, (3, 3, 1, 1)) - This corresponds to (x1 == x2) in eq(1) above
1222 // w = And(t1, u1) - This corresponds to [(x1 == x2) && (y1 > y2)] in eq(1) above
1223 // Result = BitwiseOr(z, w)
1225 // Since op1 and op2 gets used multiple times, make sure side effects are computed.
1226 GenTree* dupOp1 = nullptr;
1227 GenTree* dupOp2 = nullptr;
1228 GenTree* dupDupOp1 = nullptr;
1229 GenTree* dupDupOp2 = nullptr;
1231 if (((*pOp1)->gtFlags & GTF_SIDE_EFFECT) != 0)
1233 dupOp1 = fgInsertCommaFormTemp(pOp1, typeHnd);
1234 dupDupOp1 = gtNewLclvNode(dupOp1->AsLclVarCommon()->GetLclNum(), simdType);
1238 dupOp1 = gtCloneExpr(*pOp1);
1239 dupDupOp1 = gtCloneExpr(*pOp1);
1242 if (((*pOp2)->gtFlags & GTF_SIDE_EFFECT) != 0)
1244 dupOp2 = fgInsertCommaFormTemp(pOp2, typeHnd);
1245 dupDupOp2 = gtNewLclvNode(dupOp2->AsLclVarCommon()->GetLclNum(), simdType);
1249 dupOp2 = gtCloneExpr(*pOp2);
1250 dupDupOp2 = gtCloneExpr(*pOp2);
1253 assert(dupDupOp1 != nullptr && dupDupOp2 != nullptr);
1254 assert(dupOp1 != nullptr && dupOp2 != nullptr);
1255 assert(*pOp1 != nullptr && *pOp2 != nullptr);
1257 // v1GreaterThanv2Signed - signed 32-bit comparison
1258 GenTree* v1GreaterThanv2Signed = gtNewSIMDNode(simdType, *pOp1, *pOp2, SIMDIntrinsicGreaterThan, TYP_INT, size);
1260 // v1Equalsv2 - 32-bit equality
1261 GenTree* v1Equalsv2 = gtNewSIMDNode(simdType, dupOp1, dupOp2, SIMDIntrinsicEqual, TYP_INT, size);
1263 // v1GreaterThanv2Unsigned - unsigned 32-bit comparison
1264 var_types tempBaseType = TYP_UINT;
1265 SIMDIntrinsicID sid = impSIMDRelOp(SIMDIntrinsicGreaterThan, typeHnd, size, &tempBaseType, &dupDupOp1, &dupDupOp2);
1266 GenTree* v1GreaterThanv2Unsigned = gtNewSIMDNode(simdType, dupDupOp1, dupDupOp2, sid, tempBaseType, size);
1268 GenTree* z = gtNewSIMDNode(simdType, v1GreaterThanv2Signed, gtNewIconNode(SHUFFLE_WWYY, TYP_INT),
1269 SIMDIntrinsicShuffleSSE2, TYP_FLOAT, size);
1270 GenTree* t1 = gtNewSIMDNode(simdType, v1GreaterThanv2Unsigned, gtNewIconNode(SHUFFLE_ZZXX, TYP_INT),
1271 SIMDIntrinsicShuffleSSE2, TYP_FLOAT, size);
1272 GenTree* u1 = gtNewSIMDNode(simdType, v1Equalsv2, gtNewIconNode(SHUFFLE_WWYY, TYP_INT), SIMDIntrinsicShuffleSSE2,
1274 GenTree* w = gtNewSIMDNode(simdType, u1, t1, SIMDIntrinsicBitwiseAnd, TYP_INT, size);
1278 return SIMDIntrinsicBitwiseOr;
1281 // impSIMDLongRelOpGreaterThanOrEqual: transforms operands and returns the SIMD intrinsic to be applied on
1282 // transformed operands to obtain >= comparison result.
1285 // typeHnd - type handle of SIMD vector
1286 // size - SIMD vector size
1287 // pOp1 - in-out parameter; first operand
1288 // pOp2 - in-out parameter; second operand
1291 // Modifies in-out params pOp1, pOp2 and returns intrinsic ID to be applied to modified operands
1293 SIMDIntrinsicID Compiler::impSIMDLongRelOpGreaterThanOrEqual(CORINFO_CLASS_HANDLE typeHnd,
1298 var_types simdType = (*pOp1)->TypeGet();
1299 assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1301 // expand this to (a == b) | (a > b)
1302 GenTree* dupOp1 = nullptr;
1303 GenTree* dupOp2 = nullptr;
1305 if (((*pOp1)->gtFlags & GTF_SIDE_EFFECT) != 0)
1307 dupOp1 = fgInsertCommaFormTemp(pOp1, typeHnd);
1311 dupOp1 = gtCloneExpr(*pOp1);
1314 if (((*pOp2)->gtFlags & GTF_SIDE_EFFECT) != 0)
1316 dupOp2 = fgInsertCommaFormTemp(pOp2, typeHnd);
1320 dupOp2 = gtCloneExpr(*pOp2);
1323 assert(dupOp1 != nullptr && dupOp2 != nullptr);
1324 assert(*pOp1 != nullptr && *pOp2 != nullptr);
1327 SIMDIntrinsicID id = impSIMDLongRelOpEqual(typeHnd, size, pOp1, pOp2);
1328 *pOp1 = gtNewSIMDNode(simdType, *pOp1, *pOp2, id, TYP_LONG, size);
1331 id = impSIMDLongRelOpGreaterThan(typeHnd, size, &dupOp1, &dupOp2);
1332 *pOp2 = gtNewSIMDNode(simdType, dupOp1, dupOp2, id, TYP_LONG, size);
1334 return SIMDIntrinsicBitwiseOr;
1337 // impSIMDInt32OrSmallIntRelOpGreaterThanOrEqual: transforms operands and returns the SIMD intrinsic to be applied on
1338 // transformed operands to obtain >= comparison result in case of integer base type vectors
1341 // typeHnd - type handle of SIMD vector
1342 // size - SIMD vector size
1343 // baseType - base type of SIMD vector
1344 // pOp1 - in-out parameter; first operand
1345 // pOp2 - in-out parameter; second operand
1348 // Modifies in-out params pOp1, pOp2 and returns intrinsic ID to be applied to modified operands
1350 SIMDIntrinsicID Compiler::impSIMDIntegralRelOpGreaterThanOrEqual(
1351 CORINFO_CLASS_HANDLE typeHnd, unsigned size, var_types baseType, GenTree** pOp1, GenTree** pOp2)
1353 var_types simdType = (*pOp1)->TypeGet();
1354 assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1356 // This routine should be used only for integer base type vectors
1357 assert(varTypeIsIntegral(baseType));
1358 if ((getSIMDSupportLevel() == SIMD_SSE2_Supported) && ((baseType == TYP_LONG) || baseType == TYP_UBYTE))
1360 return impSIMDLongRelOpGreaterThanOrEqual(typeHnd, size, pOp1, pOp2);
1363 // expand this to (a == b) | (a > b)
1364 GenTree* dupOp1 = nullptr;
1365 GenTree* dupOp2 = nullptr;
1367 if (((*pOp1)->gtFlags & GTF_SIDE_EFFECT) != 0)
1369 dupOp1 = fgInsertCommaFormTemp(pOp1, typeHnd);
1373 dupOp1 = gtCloneExpr(*pOp1);
1376 if (((*pOp2)->gtFlags & GTF_SIDE_EFFECT) != 0)
1378 dupOp2 = fgInsertCommaFormTemp(pOp2, typeHnd);
1382 dupOp2 = gtCloneExpr(*pOp2);
1385 assert(dupOp1 != nullptr && dupOp2 != nullptr);
1386 assert(*pOp1 != nullptr && *pOp2 != nullptr);
1389 *pOp1 = gtNewSIMDNode(simdType, *pOp1, *pOp2, SIMDIntrinsicEqual, baseType, size);
1392 *pOp2 = gtNewSIMDNode(simdType, dupOp1, dupOp2, SIMDIntrinsicGreaterThan, baseType, size);
1394 return SIMDIntrinsicBitwiseOr;
1396 #endif // _TARGET_XARCH_
1398 // Transforms operands and returns the SIMD intrinsic to be applied on
1399 // transformed operands to obtain given relop result.
1402 // relOpIntrinsicId - Relational operator SIMD intrinsic
1403 // typeHnd - type handle of SIMD vector
1404 // size - SIMD vector size
1405 // inOutBaseType - base type of SIMD vector
1406 // pOp1 - in-out parameter; first operand
1407 // pOp2 - in-out parameter; second operand
1410 // Modifies in-out params pOp1, pOp2, inOutBaseType and returns intrinsic ID to be applied to modified operands
1412 SIMDIntrinsicID Compiler::impSIMDRelOp(SIMDIntrinsicID relOpIntrinsicId,
1413 CORINFO_CLASS_HANDLE typeHnd,
1415 var_types* inOutBaseType,
1419 var_types simdType = (*pOp1)->TypeGet();
1420 assert(varTypeIsSIMD(simdType) && ((*pOp2)->TypeGet() == simdType));
1422 assert(isRelOpSIMDIntrinsic(relOpIntrinsicId));
1424 SIMDIntrinsicID intrinsicID = relOpIntrinsicId;
1425 #ifdef _TARGET_XARCH_
1426 var_types baseType = *inOutBaseType;
1428 if (varTypeIsFloating(baseType))
1430 // SSE2/AVX doesn't support > and >= on vector float/double.
1431 // Therefore, we need to use < and <= with swapped operands
1432 if (relOpIntrinsicId == SIMDIntrinsicGreaterThan || relOpIntrinsicId == SIMDIntrinsicGreaterThanOrEqual)
1434 GenTree* tmp = *pOp1;
1439 (relOpIntrinsicId == SIMDIntrinsicGreaterThan) ? SIMDIntrinsicLessThan : SIMDIntrinsicLessThanOrEqual;
1442 else if (varTypeIsIntegral(baseType))
1444 // SSE/AVX doesn't support < and <= on integer base type vectors.
1445 // Therefore, we need to use > and >= with swapped operands.
1446 if (intrinsicID == SIMDIntrinsicLessThan || intrinsicID == SIMDIntrinsicLessThanOrEqual)
1448 GenTree* tmp = *pOp1;
1452 intrinsicID = (relOpIntrinsicId == SIMDIntrinsicLessThan) ? SIMDIntrinsicGreaterThan
1453 : SIMDIntrinsicGreaterThanOrEqual;
1456 if ((getSIMDSupportLevel() == SIMD_SSE2_Supported) && baseType == TYP_LONG)
1458 // There is no direct SSE2 support for comparing TYP_LONG vectors.
1459 // These have to be implemented interms of TYP_INT vector comparison operations.
1460 if (intrinsicID == SIMDIntrinsicEqual)
1462 intrinsicID = impSIMDLongRelOpEqual(typeHnd, size, pOp1, pOp2);
1464 else if (intrinsicID == SIMDIntrinsicGreaterThan)
1466 intrinsicID = impSIMDLongRelOpGreaterThan(typeHnd, size, pOp1, pOp2);
1468 else if (intrinsicID == SIMDIntrinsicGreaterThanOrEqual)
1470 intrinsicID = impSIMDLongRelOpGreaterThanOrEqual(typeHnd, size, pOp1, pOp2);
1477 // SSE2 and AVX direct support for signed comparison of int32, int16 and int8 types
1478 else if (!varTypeIsUnsigned(baseType))
1480 if (intrinsicID == SIMDIntrinsicGreaterThanOrEqual)
1482 intrinsicID = impSIMDIntegralRelOpGreaterThanOrEqual(typeHnd, size, baseType, pOp1, pOp2);
1487 // Vector<byte>, Vector<ushort>, Vector<uint> and Vector<ulong>:
1488 // SSE2 supports > for signed comparison. Therefore, to use it for
1489 // comparing unsigned numbers, we subtract a constant from both the
1490 // operands such that the result fits within the corresponding signed
1491 // type. The resulting signed numbers are compared using SSE2 signed
1494 // Vector<byte>: constant to be subtracted is 2^7
1495 // Vector<ushort> constant to be subtracted is 2^15
1496 // Vector<uint> constant to be subtracted is 2^31
1497 // Vector<ulong> constant to be subtracted is 2^63
1499 // We need to treat op1 and op2 as signed for comparison purpose after
1500 // the transformation.
1501 __int64 constVal = 0;
1505 constVal = 0x80808080;
1506 *inOutBaseType = TYP_BYTE;
1509 constVal = 0x80008000;
1510 *inOutBaseType = TYP_SHORT;
1513 constVal = 0x80000000;
1514 *inOutBaseType = TYP_INT;
1517 constVal = 0x8000000000000000LL;
1518 *inOutBaseType = TYP_LONG;
1524 assert(constVal != 0);
1526 // This transformation is not required for equality.
1527 if (intrinsicID != SIMDIntrinsicEqual)
1529 // For constructing const vector use either long or int base type.
1530 var_types tempBaseType;
1532 if (baseType == TYP_ULONG)
1534 tempBaseType = TYP_LONG;
1535 initVal = gtNewLconNode(constVal);
1539 tempBaseType = TYP_INT;
1540 initVal = gtNewIconNode((ssize_t)constVal);
1542 initVal->gtType = tempBaseType;
1543 GenTree* constVector = gtNewSIMDNode(simdType, initVal, nullptr, SIMDIntrinsicInit, tempBaseType, size);
1545 // Assign constVector to a temp, since we intend to use it more than once
1546 // TODO-CQ: We have quite a few such constant vectors constructed during
1547 // the importation of SIMD intrinsics. Make sure that we have a single
1548 // temp per distinct constant per method.
1549 GenTree* tmp = fgInsertCommaFormTemp(&constVector, typeHnd);
1551 // op1 = op1 - constVector
1552 // op2 = op2 - constVector
1553 *pOp1 = gtNewSIMDNode(simdType, *pOp1, constVector, SIMDIntrinsicSub, baseType, size);
1554 *pOp2 = gtNewSIMDNode(simdType, *pOp2, tmp, SIMDIntrinsicSub, baseType, size);
1557 return impSIMDRelOp(intrinsicID, typeHnd, size, inOutBaseType, pOp1, pOp2);
1560 #elif defined(_TARGET_ARM64_)
1561 // TODO-ARM64-CQ handle comparisons against zero
1563 // _TARGET_ARM64_ doesn't support < and <= on register register comparisons
1564 // Therefore, we need to use > and >= with swapped operands.
1565 if (intrinsicID == SIMDIntrinsicLessThan || intrinsicID == SIMDIntrinsicLessThanOrEqual)
1567 GenTree* tmp = *pOp1;
1572 (intrinsicID == SIMDIntrinsicLessThan) ? SIMDIntrinsicGreaterThan : SIMDIntrinsicGreaterThanOrEqual;
1574 #else // !_TARGET_XARCH_
1575 assert(!"impSIMDRelOp() unimplemented on target arch");
1577 #endif // !_TARGET_XARCH_
1582 //-------------------------------------------------------------------------
1583 // impSIMDAbs: creates GT_SIMD node to compute Abs value of a given vector.
1586 // typeHnd - type handle of SIMD vector
1587 // baseType - base type of vector
1588 // size - vector size in bytes
1589 // op1 - operand of Abs intrinsic
1591 GenTree* Compiler::impSIMDAbs(CORINFO_CLASS_HANDLE typeHnd, var_types baseType, unsigned size, GenTree* op1)
1593 assert(varTypeIsSIMD(op1));
1595 var_types simdType = op1->TypeGet();
1596 GenTree* retVal = nullptr;
1598 #ifdef _TARGET_XARCH_
1599 // When there is no direct support, Abs(v) could be computed
1600 // on integer vectors as follows:
1601 // BitVector = v < vector.Zero
1602 // result = ConditionalSelect(BitVector, vector.Zero - v, v)
1604 bool useConditionalSelect = false;
1605 if (getSIMDSupportLevel() == SIMD_SSE2_Supported)
1607 // SSE2 doesn't support abs on signed integer type vectors.
1608 if (baseType == TYP_LONG || baseType == TYP_INT || baseType == TYP_SHORT || baseType == TYP_BYTE)
1610 useConditionalSelect = true;
1615 assert(getSIMDSupportLevel() >= SIMD_SSE4_Supported);
1616 if (baseType == TYP_LONG)
1618 // SSE4/AVX2 don't support abs on long type vector.
1619 useConditionalSelect = true;
1623 if (useConditionalSelect)
1625 // This works only on integer vectors not on float/double vectors.
1626 assert(varTypeIsIntegral(baseType));
1631 if (op1->OperGet() == GT_LCL_VAR)
1633 op1LclNum = op1->gtLclVarCommon.gtLclNum;
1634 op1Assign = nullptr;
1638 op1LclNum = lvaGrabTemp(true DEBUGARG("SIMD Abs op1"));
1639 lvaSetStruct(op1LclNum, typeHnd, false);
1640 op1Assign = gtNewTempAssign(op1LclNum, op1);
1641 op1 = gtNewLclvNode(op1LclNum, op1->TypeGet());
1644 // Assign Vector.Zero to a temp since it is needed more than once
1645 GenTree* vecZero = gtNewSIMDVectorZero(simdType, baseType, size);
1646 unsigned vecZeroLclNum = lvaGrabTemp(true DEBUGARG("SIMD Abs VecZero"));
1647 lvaSetStruct(vecZeroLclNum, typeHnd, false);
1648 GenTree* vecZeroAssign = gtNewTempAssign(vecZeroLclNum, vecZero);
1650 // Construct BitVector = v < vector.Zero
1651 GenTree* bitVecOp1 = op1;
1652 GenTree* bitVecOp2 = gtNewLclvNode(vecZeroLclNum, vecZero->TypeGet());
1653 var_types relOpBaseType = baseType;
1654 SIMDIntrinsicID relOpIntrinsic =
1655 impSIMDRelOp(SIMDIntrinsicLessThan, typeHnd, size, &relOpBaseType, &bitVecOp1, &bitVecOp2);
1656 GenTree* bitVec = gtNewSIMDNode(simdType, bitVecOp1, bitVecOp2, relOpIntrinsic, relOpBaseType, size);
1657 unsigned bitVecLclNum = lvaGrabTemp(true DEBUGARG("SIMD Abs bitVec"));
1658 lvaSetStruct(bitVecLclNum, typeHnd, false);
1659 GenTree* bitVecAssign = gtNewTempAssign(bitVecLclNum, bitVec);
1660 bitVec = gtNewLclvNode(bitVecLclNum, bitVec->TypeGet());
1662 // Construct condSelectOp1 = vector.Zero - v
1663 GenTree* subOp1 = gtNewLclvNode(vecZeroLclNum, vecZero->TypeGet());
1664 GenTree* subOp2 = gtNewLclvNode(op1LclNum, op1->TypeGet());
1665 GenTree* negVec = gtNewSIMDNode(simdType, subOp1, subOp2, SIMDIntrinsicSub, baseType, size);
1667 // Construct ConditionalSelect(bitVec, vector.Zero - v, v)
1668 GenTree* vec = gtNewLclvNode(op1LclNum, op1->TypeGet());
1669 retVal = impSIMDSelect(typeHnd, baseType, size, bitVec, negVec, vec);
1671 // Prepend bitVec assignment to retVal.
1672 // retVal = (tmp2 = v < tmp1), CondSelect(tmp2, tmp1 - v, v)
1673 retVal = gtNewOperNode(GT_COMMA, simdType, bitVecAssign, retVal);
1675 // Prepend vecZero assignment to retVal.
1676 // retVal = (tmp1 = vector.Zero), (tmp2 = v < tmp1), CondSelect(tmp2, tmp1 - v, v)
1677 retVal = gtNewOperNode(GT_COMMA, simdType, vecZeroAssign, retVal);
1679 // If op1 was assigned to a temp, prepend that to retVal.
1680 if (op1Assign != nullptr)
1682 // retVal = (v=op1), (tmp1 = vector.Zero), (tmp2 = v < tmp1), CondSelect(tmp2, tmp1 - v, v)
1683 retVal = gtNewOperNode(GT_COMMA, simdType, op1Assign, retVal);
1686 else if (varTypeIsFloating(baseType))
1688 // Abs(vf) = vf & new SIMDVector<float>(0x7fffffff);
1689 // Abs(vd) = vf & new SIMDVector<double>(0x7fffffffffffffff);
1690 GenTree* bitMask = nullptr;
1691 if (baseType == TYP_FLOAT)
1694 static_assert_no_msg(sizeof(float) == sizeof(int));
1695 *((int*)&f) = 0x7fffffff;
1696 bitMask = gtNewDconNode(f);
1698 else if (baseType == TYP_DOUBLE)
1701 static_assert_no_msg(sizeof(double) == sizeof(__int64));
1702 *((__int64*)&d) = 0x7fffffffffffffffLL;
1703 bitMask = gtNewDconNode(d);
1706 assert(bitMask != nullptr);
1707 bitMask->gtType = baseType;
1708 GenTree* bitMaskVector = gtNewSIMDNode(simdType, bitMask, SIMDIntrinsicInit, baseType, size);
1709 retVal = gtNewSIMDNode(simdType, op1, bitMaskVector, SIMDIntrinsicBitwiseAnd, baseType, size);
1711 else if (baseType == TYP_USHORT || baseType == TYP_UBYTE || baseType == TYP_UINT || baseType == TYP_ULONG)
1713 // Abs is a no-op on unsigned integer type vectors
1718 assert(getSIMDSupportLevel() >= SIMD_SSE4_Supported);
1719 assert(baseType != TYP_LONG);
1721 retVal = gtNewSIMDNode(simdType, op1, SIMDIntrinsicAbs, baseType, size);
1723 #elif defined(_TARGET_ARM64_)
1724 if (varTypeIsUnsigned(baseType))
1726 // Abs is a no-op on unsigned integer type vectors
1731 retVal = gtNewSIMDNode(simdType, op1, SIMDIntrinsicAbs, baseType, size);
1733 #else // !defined(_TARGET_XARCH)_ && !defined(_TARGET_ARM64_)
1734 assert(!"Abs intrinsic on non-xarch target not implemented");
1735 #endif // !_TARGET_XARCH_
1740 // Creates a GT_SIMD tree for Select operation
1743 // typeHnd - type handle of SIMD vector
1744 // baseType - base type of SIMD vector
1745 // size - SIMD vector size
1746 // op1 - first operand = Condition vector vc
1747 // op2 - second operand = va
1748 // op3 - third operand = vb
1751 // Returns GT_SIMD tree that computes Select(vc, va, vb)
1753 GenTree* Compiler::impSIMDSelect(
1754 CORINFO_CLASS_HANDLE typeHnd, var_types baseType, unsigned size, GenTree* op1, GenTree* op2, GenTree* op3)
1756 assert(varTypeIsSIMD(op1));
1757 var_types simdType = op1->TypeGet();
1758 assert(op2->TypeGet() == simdType);
1759 assert(op3->TypeGet() == simdType);
1761 // TODO-ARM64-CQ Support generating select instruction for SIMD
1763 // Select(BitVector vc, va, vb) = (va & vc) | (vb & !vc)
1764 // Select(op1, op2, op3) = (op2 & op1) | (op3 & !op1)
1765 // = SIMDIntrinsicBitwiseOr(SIMDIntrinsicBitwiseAnd(op2, op1),
1766 // SIMDIntrinsicBitwiseAndNot(op3, op1))
1768 // If Op1 has side effect, create an assignment to a temp
1770 GenTree* asg = nullptr;
1771 if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0)
1773 unsigned lclNum = lvaGrabTemp(true DEBUGARG("SIMD Select"));
1774 lvaSetStruct(lclNum, typeHnd, false);
1775 tmp = gtNewLclvNode(lclNum, op1->TypeGet());
1776 asg = gtNewTempAssign(lclNum, op1);
1779 GenTree* andExpr = gtNewSIMDNode(simdType, op2, tmp, SIMDIntrinsicBitwiseAnd, baseType, size);
1780 GenTree* dupOp1 = gtCloneExpr(tmp);
1781 assert(dupOp1 != nullptr);
1782 #ifdef _TARGET_ARM64_
1783 // ARM64 implements SIMDIntrinsicBitwiseAndNot as Left & ~Right
1784 GenTree* andNotExpr = gtNewSIMDNode(simdType, op3, dupOp1, SIMDIntrinsicBitwiseAndNot, baseType, size);
1786 // XARCH implements SIMDIntrinsicBitwiseAndNot as ~Left & Right
1787 GenTree* andNotExpr = gtNewSIMDNode(simdType, dupOp1, op3, SIMDIntrinsicBitwiseAndNot, baseType, size);
1789 GenTree* simdTree = gtNewSIMDNode(simdType, andExpr, andNotExpr, SIMDIntrinsicBitwiseOr, baseType, size);
1791 // If asg not null, create a GT_COMMA tree.
1794 simdTree = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), asg, simdTree);
1800 // Creates a GT_SIMD tree for Min/Max operation
1803 // IntrinsicId - SIMD intrinsic Id, either Min or Max
1804 // typeHnd - type handle of SIMD vector
1805 // baseType - base type of SIMD vector
1806 // size - SIMD vector size
1807 // op1 - first operand = va
1808 // op2 - second operand = vb
1811 // Returns GT_SIMD tree that computes Max(va, vb)
1813 GenTree* Compiler::impSIMDMinMax(SIMDIntrinsicID intrinsicId,
1814 CORINFO_CLASS_HANDLE typeHnd,
1820 assert(intrinsicId == SIMDIntrinsicMin || intrinsicId == SIMDIntrinsicMax);
1821 assert(varTypeIsSIMD(op1));
1822 var_types simdType = op1->TypeGet();
1823 assert(op2->TypeGet() == simdType);
1825 #if defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
1826 GenTree* simdTree = nullptr;
1828 #ifdef _TARGET_XARCH_
1829 // SSE2 has direct support for float/double/signed word/unsigned byte.
1830 // SSE4.1 has direct support for int32/uint32/signed byte/unsigned word.
1831 // For other integer types we compute min/max as follows
1833 // int32/uint32 (SSE2)
1834 // int64/uint64 (SSE2&SSE4):
1835 // compResult = (op1 < op2) in case of Min
1836 // (op1 > op2) in case of Max
1837 // Min/Max(op1, op2) = Select(compResult, op1, op2)
1839 // unsigned word (SSE2):
1840 // op1 = op1 - 2^15 ; to make it fit within a signed word
1841 // op2 = op2 - 2^15 ; to make it fit within a signed word
1842 // result = SSE2 signed word Min/Max(op1, op2)
1843 // result = result + 2^15 ; readjust it back
1845 // signed byte (SSE2):
1846 // op1 = op1 + 2^7 ; to make it unsigned
1847 // op1 = op1 + 2^7 ; to make it unsigned
1848 // result = SSE2 unsigned byte Min/Max(op1, op2)
1849 // result = result - 2^15 ; readjust it back
1851 if (varTypeIsFloating(baseType) || baseType == TYP_SHORT || baseType == TYP_UBYTE ||
1852 (getSIMDSupportLevel() >= SIMD_SSE4_Supported &&
1853 (baseType == TYP_BYTE || baseType == TYP_INT || baseType == TYP_UINT || baseType == TYP_USHORT)))
1855 // SSE2 or SSE4.1 has direct support
1856 simdTree = gtNewSIMDNode(simdType, op1, op2, intrinsicId, baseType, size);
1858 else if (baseType == TYP_USHORT || baseType == TYP_BYTE)
1860 assert(getSIMDSupportLevel() == SIMD_SSE2_Supported);
1862 SIMDIntrinsicID operIntrinsic;
1863 SIMDIntrinsicID adjustIntrinsic;
1864 var_types minMaxOperBaseType;
1865 if (baseType == TYP_USHORT)
1867 constVal = 0x80008000;
1868 operIntrinsic = SIMDIntrinsicSub;
1869 adjustIntrinsic = SIMDIntrinsicAdd;
1870 minMaxOperBaseType = TYP_SHORT;
1874 assert(baseType == TYP_BYTE);
1875 constVal = 0x80808080;
1876 operIntrinsic = SIMDIntrinsicAdd;
1877 adjustIntrinsic = SIMDIntrinsicSub;
1878 minMaxOperBaseType = TYP_UBYTE;
1881 GenTree* initVal = gtNewIconNode(constVal);
1882 GenTree* constVector = gtNewSIMDNode(simdType, initVal, nullptr, SIMDIntrinsicInit, TYP_INT, size);
1884 // Assign constVector to a temp, since we intend to use it more than once
1885 // TODO-CQ: We have quite a few such constant vectors constructed during
1886 // the importation of SIMD intrinsics. Make sure that we have a single
1887 // temp per distinct constant per method.
1888 GenTree* tmp = fgInsertCommaFormTemp(&constVector, typeHnd);
1890 // op1 = op1 - constVector
1891 // op2 = op2 - constVector
1892 op1 = gtNewSIMDNode(simdType, op1, constVector, operIntrinsic, baseType, size);
1893 op2 = gtNewSIMDNode(simdType, op2, tmp, operIntrinsic, baseType, size);
1895 // compute min/max of op1 and op2 considering them as if minMaxOperBaseType
1896 simdTree = gtNewSIMDNode(simdType, op1, op2, intrinsicId, minMaxOperBaseType, size);
1898 // re-adjust the value by adding or subtracting constVector
1899 tmp = gtNewLclvNode(tmp->AsLclVarCommon()->GetLclNum(), tmp->TypeGet());
1900 simdTree = gtNewSIMDNode(simdType, simdTree, tmp, adjustIntrinsic, baseType, size);
1902 #elif defined(_TARGET_ARM64_)
1903 // Arm64 has direct support for all types except int64/uint64
1904 // For which we compute min/max as follows
1907 // compResult = (op1 < op2) in case of Min
1908 // (op1 > op2) in case of Max
1909 // Min/Max(op1, op2) = Select(compResult, op1, op2)
1910 if (baseType != TYP_ULONG && baseType != TYP_LONG)
1912 simdTree = gtNewSIMDNode(simdType, op1, op2, intrinsicId, baseType, size);
1917 GenTree* dupOp1 = nullptr;
1918 GenTree* dupOp2 = nullptr;
1919 GenTree* op1Assign = nullptr;
1920 GenTree* op2Assign = nullptr;
1924 if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0)
1926 op1LclNum = lvaGrabTemp(true DEBUGARG("SIMD Min/Max"));
1927 dupOp1 = gtNewLclvNode(op1LclNum, op1->TypeGet());
1928 lvaSetStruct(op1LclNum, typeHnd, false);
1929 op1Assign = gtNewTempAssign(op1LclNum, op1);
1930 op1 = gtNewLclvNode(op1LclNum, op1->TypeGet());
1934 dupOp1 = gtCloneExpr(op1);
1937 if ((op2->gtFlags & GTF_SIDE_EFFECT) != 0)
1939 op2LclNum = lvaGrabTemp(true DEBUGARG("SIMD Min/Max"));
1940 dupOp2 = gtNewLclvNode(op2LclNum, op2->TypeGet());
1941 lvaSetStruct(op2LclNum, typeHnd, false);
1942 op2Assign = gtNewTempAssign(op2LclNum, op2);
1943 op2 = gtNewLclvNode(op2LclNum, op2->TypeGet());
1947 dupOp2 = gtCloneExpr(op2);
1950 SIMDIntrinsicID relOpIntrinsic =
1951 (intrinsicId == SIMDIntrinsicMin) ? SIMDIntrinsicLessThan : SIMDIntrinsicGreaterThan;
1952 var_types relOpBaseType = baseType;
1954 // compResult = op1 relOp op2
1955 // simdTree = Select(compResult, op1, op2);
1956 assert(dupOp1 != nullptr);
1957 assert(dupOp2 != nullptr);
1958 relOpIntrinsic = impSIMDRelOp(relOpIntrinsic, typeHnd, size, &relOpBaseType, &dupOp1, &dupOp2);
1959 GenTree* compResult = gtNewSIMDNode(simdType, dupOp1, dupOp2, relOpIntrinsic, relOpBaseType, size);
1960 unsigned compResultLclNum = lvaGrabTemp(true DEBUGARG("SIMD Min/Max"));
1961 lvaSetStruct(compResultLclNum, typeHnd, false);
1962 GenTree* compResultAssign = gtNewTempAssign(compResultLclNum, compResult);
1963 compResult = gtNewLclvNode(compResultLclNum, compResult->TypeGet());
1964 simdTree = impSIMDSelect(typeHnd, baseType, size, compResult, op1, op2);
1965 simdTree = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), compResultAssign, simdTree);
1967 // Now create comma trees if we have created assignments of op1/op2 to temps
1968 if (op2Assign != nullptr)
1970 simdTree = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), op2Assign, simdTree);
1973 if (op1Assign != nullptr)
1975 simdTree = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), op1Assign, simdTree);
1979 assert(simdTree != nullptr);
1981 #else // !(defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_))
1982 assert(!"impSIMDMinMax() unimplemented on target arch");
1984 #endif // !(defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_))
1987 //------------------------------------------------------------------------
1988 // getOp1ForConstructor: Get the op1 for a constructor call.
1991 // opcode - the opcode being handled (needed to identify the CEE_NEWOBJ case)
1992 // newobjThis - For CEE_NEWOBJ, this is the temp grabbed for the allocated uninitalized object.
1993 // clsHnd - The handle of the class of the method.
1996 // The tree node representing the object to be initialized with the constructor.
1999 // This method handles the differences between the CEE_NEWOBJ and constructor cases.
2001 GenTree* Compiler::getOp1ForConstructor(OPCODE opcode, GenTree* newobjThis, CORINFO_CLASS_HANDLE clsHnd)
2004 if (opcode == CEE_NEWOBJ)
2007 assert(newobjThis->gtOper == GT_ADDR && newobjThis->gtOp.gtOp1->gtOper == GT_LCL_VAR);
2009 // push newobj result on type stack
2010 unsigned tmp = op1->gtOp.gtOp1->gtLclVarCommon.gtLclNum;
2011 impPushOnStack(gtNewLclvNode(tmp, lvaGetRealType(tmp)), verMakeTypeInfo(clsHnd).NormaliseForStack());
2015 op1 = impSIMDPopStack(TYP_BYREF);
2017 assert(op1->TypeGet() == TYP_BYREF);
2021 //-------------------------------------------------------------------
2022 // Set the flag that indicates that the lclVar referenced by this tree
2023 // is used in a SIMD intrinsic.
2027 void Compiler::setLclRelatedToSIMDIntrinsic(GenTree* tree)
2029 assert(tree->OperIsLocal());
2030 unsigned lclNum = tree->AsLclVarCommon()->GetLclNum();
2031 LclVarDsc* lclVarDsc = &lvaTable[lclNum];
2032 lclVarDsc->lvUsedInSIMDIntrinsic = true;
2035 //-------------------------------------------------------------
2036 // Check if two field nodes reference at the same memory location.
2037 // Notice that this check is just based on pattern matching.
2042 // If op1's parents node and op2's parents node are at the same location, return true. Otherwise, return false
2044 bool areFieldsParentsLocatedSame(GenTree* op1, GenTree* op2)
2046 assert(op1->OperGet() == GT_FIELD);
2047 assert(op2->OperGet() == GT_FIELD);
2049 GenTree* op1ObjRef = op1->gtField.gtFldObj;
2050 GenTree* op2ObjRef = op2->gtField.gtFldObj;
2051 while (op1ObjRef != nullptr && op2ObjRef != nullptr)
2054 if (op1ObjRef->OperGet() != op2ObjRef->OperGet())
2058 else if (op1ObjRef->OperGet() == GT_ADDR)
2060 op1ObjRef = op1ObjRef->gtOp.gtOp1;
2061 op2ObjRef = op2ObjRef->gtOp.gtOp1;
2064 if (op1ObjRef->OperIsLocal() && op2ObjRef->OperIsLocal() &&
2065 op1ObjRef->AsLclVarCommon()->GetLclNum() == op2ObjRef->AsLclVarCommon()->GetLclNum())
2069 else if (op1ObjRef->OperGet() == GT_FIELD && op2ObjRef->OperGet() == GT_FIELD &&
2070 op1ObjRef->gtField.gtFldHnd == op2ObjRef->gtField.gtFldHnd)
2072 op1ObjRef = op1ObjRef->gtField.gtFldObj;
2073 op2ObjRef = op2ObjRef->gtField.gtFldObj;
2085 //----------------------------------------------------------------------
2086 // Check whether two field are contiguous
2088 // first - GenTree*. The Type of the node should be TYP_FLOAT
2089 // second - GenTree*. The Type of the node should be TYP_FLOAT
2091 // if the first field is located before second field, and they are located contiguously,
2092 // then return true. Otherwise, return false.
2094 bool Compiler::areFieldsContiguous(GenTree* first, GenTree* second)
2096 assert(first->OperGet() == GT_FIELD);
2097 assert(second->OperGet() == GT_FIELD);
2098 assert(first->gtType == TYP_FLOAT);
2099 assert(second->gtType == TYP_FLOAT);
2101 var_types firstFieldType = first->gtType;
2102 var_types secondFieldType = second->gtType;
2104 unsigned firstFieldEndOffset = first->gtField.gtFldOffset + genTypeSize(firstFieldType);
2105 unsigned secondFieldOffset = second->gtField.gtFldOffset;
2106 if (firstFieldEndOffset == secondFieldOffset && firstFieldType == secondFieldType &&
2107 areFieldsParentsLocatedSame(first, second))
2115 //-------------------------------------------------------------------------------
2116 // Check whether two array element nodes are located contiguously or not.
2121 // if the array element op1 is located before array element op2, and they are contiguous,
2122 // then return true. Otherwise, return false.
2124 // Right this can only check array element with const number as index. In future,
2125 // we should consider to allow this function to check the index using expression.
2127 bool Compiler::areArrayElementsContiguous(GenTree* op1, GenTree* op2)
2129 noway_assert(op1->gtOper == GT_INDEX);
2130 noway_assert(op2->gtOper == GT_INDEX);
2131 GenTreeIndex* op1Index = op1->AsIndex();
2132 GenTreeIndex* op2Index = op2->AsIndex();
2134 GenTree* op1ArrayRef = op1Index->Arr();
2135 GenTree* op2ArrayRef = op2Index->Arr();
2136 assert(op1ArrayRef->TypeGet() == TYP_REF);
2137 assert(op2ArrayRef->TypeGet() == TYP_REF);
2139 GenTree* op1IndexNode = op1Index->Index();
2140 GenTree* op2IndexNode = op2Index->Index();
2141 if ((op1IndexNode->OperGet() == GT_CNS_INT && op2IndexNode->OperGet() == GT_CNS_INT) &&
2142 op1IndexNode->gtIntCon.gtIconVal + 1 == op2IndexNode->gtIntCon.gtIconVal)
2144 if (op1ArrayRef->OperGet() == GT_FIELD && op2ArrayRef->OperGet() == GT_FIELD &&
2145 areFieldsParentsLocatedSame(op1ArrayRef, op2ArrayRef))
2149 else if (op1ArrayRef->OperIsLocal() && op2ArrayRef->OperIsLocal() &&
2150 op1ArrayRef->AsLclVarCommon()->GetLclNum() == op2ArrayRef->AsLclVarCommon()->GetLclNum())
2158 //-------------------------------------------------------------------------------
2159 // Check whether two argument nodes are contiguous or not.
2164 // if the argument node op1 is located before argument node op2, and they are located contiguously,
2165 // then return true. Otherwise, return false.
2167 // Right now this can only check field and array. In future we should add more cases.
2170 bool Compiler::areArgumentsContiguous(GenTree* op1, GenTree* op2)
2172 if (op1->OperGet() == GT_INDEX && op2->OperGet() == GT_INDEX)
2174 return areArrayElementsContiguous(op1, op2);
2176 else if (op1->OperGet() == GT_FIELD && op2->OperGet() == GT_FIELD)
2178 return areFieldsContiguous(op1, op2);
2183 //--------------------------------------------------------------------------------------------------------
2184 // createAddressNodeForSIMDInit: Generate the address node(GT_LEA) if we want to intialize vector2, vector3 or vector4
2185 // from first argument's address.
2188 // tree - GenTree*. This the tree node which is used to get the address for indir.
2189 // simdsize - unsigned. This the simd vector size.
2190 // arrayElementsCount - unsigned. This is used for generating the boundary check for array.
2193 // return the address node.
2196 // 1. Currently just support for GT_FIELD and GT_INDEX, because we can only verify the GT_INDEX node or GT_Field
2197 // are located contiguously or not. In future we should support more cases.
2198 // 2. Though it happens to just work fine front-end phases are not aware of GT_LEA node. Therefore, convert these
2200 GenTree* Compiler::createAddressNodeForSIMDInit(GenTree* tree, unsigned simdSize)
2202 assert(tree->OperGet() == GT_FIELD || tree->OperGet() == GT_INDEX);
2203 GenTree* byrefNode = nullptr;
2204 GenTree* startIndex = nullptr;
2205 unsigned offset = 0;
2206 var_types baseType = tree->gtType;
2208 if (tree->OperGet() == GT_FIELD)
2210 GenTree* objRef = tree->gtField.gtFldObj;
2211 if (objRef != nullptr && objRef->gtOper == GT_ADDR)
2213 GenTree* obj = objRef->gtOp.gtOp1;
2215 // If the field is directly from a struct, then in this case,
2216 // we should set this struct's lvUsedInSIMDIntrinsic as true,
2217 // so that this sturct won't be promoted.
2218 // e.g. s.x x is a field, and s is a struct, then we should set the s's lvUsedInSIMDIntrinsic as true.
2219 // so that s won't be promoted.
2220 // Notice that if we have a case like s1.s2.x. s1 s2 are struct, and x is a field, then it is possible that
2221 // s1 can be promoted, so that s2 can be promoted. The reason for that is if we don't allow s1 to be
2222 // promoted, then this will affect the other optimizations which are depend on s1's struct promotion.
2224 // In future, we should optimize this case so that if there is a nested field like s1.s2.x and s1.s2.x's
2225 // address is used for initializing the vector, then s1 can be promoted but s2 can't.
2226 if (varTypeIsSIMD(obj) && obj->OperIsLocal())
2228 setLclRelatedToSIMDIntrinsic(obj);
2232 byrefNode = gtCloneExpr(tree->gtField.gtFldObj);
2233 assert(byrefNode != nullptr);
2234 offset = tree->gtField.gtFldOffset;
2236 else if (tree->OperGet() == GT_INDEX)
2239 GenTree* index = tree->AsIndex()->Index();
2240 assert(index->OperGet() == GT_CNS_INT);
2242 GenTree* checkIndexExpr = nullptr;
2243 unsigned indexVal = (unsigned)(index->gtIntCon.gtIconVal);
2244 offset = indexVal * genTypeSize(tree->TypeGet());
2245 GenTree* arrayRef = tree->AsIndex()->Arr();
2247 // Generate the boundary check exception.
2248 // The length for boundary check should be the maximum index number which should be
2249 // (first argument's index number) + (how many array arguments we have) - 1
2250 // = indexVal + arrayElementsCount - 1
2251 unsigned arrayElementsCount = simdSize / genTypeSize(baseType);
2252 checkIndexExpr = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, indexVal + arrayElementsCount - 1);
2253 GenTreeArrLen* arrLen = gtNewArrLen(TYP_INT, arrayRef, (int)OFFSETOF__CORINFO_Array__length);
2254 GenTreeBoundsChk* arrBndsChk = new (this, GT_ARR_BOUNDS_CHECK)
2255 GenTreeBoundsChk(GT_ARR_BOUNDS_CHECK, TYP_VOID, checkIndexExpr, arrLen, SCK_RNGCHK_FAIL);
2257 offset += OFFSETOF__CORINFO_Array__data;
2258 byrefNode = gtNewOperNode(GT_COMMA, arrayRef->TypeGet(), arrBndsChk, gtCloneExpr(arrayRef));
2265 new (this, GT_LEA) GenTreeAddrMode(TYP_BYREF, byrefNode, startIndex, genTypeSize(tree->TypeGet()), offset);
2269 //-------------------------------------------------------------------------------
2270 // impMarkContiguousSIMDFieldAssignments: Try to identify if there are contiguous
2271 // assignments from SIMD field to memory. If there are, then mark the related
2272 // lclvar so that it won't be promoted.
2275 // stmt - GenTree*. Input statement node.
2277 void Compiler::impMarkContiguousSIMDFieldAssignments(GenTree* stmt)
2279 if (!featureSIMD || opts.OptimizationDisabled())
2283 GenTree* expr = stmt->gtStmt.gtStmtExpr;
2284 if (expr->OperGet() == GT_ASG && expr->TypeGet() == TYP_FLOAT)
2286 GenTree* curDst = expr->gtOp.gtOp1;
2287 GenTree* curSrc = expr->gtOp.gtOp2;
2289 var_types baseType = TYP_UNKNOWN;
2290 unsigned simdSize = 0;
2291 GenTree* srcSimdStructNode = getSIMDStructFromField(curSrc, &baseType, &index, &simdSize, true);
2292 if (srcSimdStructNode == nullptr || baseType != TYP_FLOAT)
2294 fgPreviousCandidateSIMDFieldAsgStmt = nullptr;
2296 else if (index == 0 && isSIMDTypeLocal(srcSimdStructNode))
2298 fgPreviousCandidateSIMDFieldAsgStmt = stmt;
2300 else if (fgPreviousCandidateSIMDFieldAsgStmt != nullptr)
2303 GenTree* prevAsgExpr = fgPreviousCandidateSIMDFieldAsgStmt->gtStmt.gtStmtExpr;
2304 GenTree* prevDst = prevAsgExpr->gtOp.gtOp1;
2305 GenTree* prevSrc = prevAsgExpr->gtOp.gtOp2;
2306 if (!areArgumentsContiguous(prevDst, curDst) || !areArgumentsContiguous(prevSrc, curSrc))
2308 fgPreviousCandidateSIMDFieldAsgStmt = nullptr;
2312 if (index == (simdSize / genTypeSize(baseType) - 1))
2314 // Successfully found the pattern, mark the lclvar as UsedInSIMDIntrinsic
2315 if (srcSimdStructNode->OperIsLocal())
2317 setLclRelatedToSIMDIntrinsic(srcSimdStructNode);
2320 if (curDst->OperGet() == GT_FIELD)
2322 GenTree* objRef = curDst->gtField.gtFldObj;
2323 if (objRef != nullptr && objRef->gtOper == GT_ADDR)
2325 GenTree* obj = objRef->gtOp.gtOp1;
2326 if (varTypeIsStruct(obj) && obj->OperIsLocal())
2328 setLclRelatedToSIMDIntrinsic(obj);
2335 fgPreviousCandidateSIMDFieldAsgStmt = stmt;
2342 fgPreviousCandidateSIMDFieldAsgStmt = nullptr;
2346 //------------------------------------------------------------------------
2347 // impSIMDIntrinsic: Check method to see if it is a SIMD method
2350 // opcode - the opcode being handled (needed to identify the CEE_NEWOBJ case)
2351 // newobjThis - For CEE_NEWOBJ, this is the temp grabbed for the allocated uninitalized object.
2352 // clsHnd - The handle of the class of the method.
2353 // method - The handle of the method.
2354 // sig - The call signature for the method.
2355 // memberRef - The memberRef token for the method reference.
2358 // If clsHnd is a known SIMD type, and 'method' is one of the methods that are
2359 // implemented as an intrinsic in the JIT, then return the tree that implements
2362 GenTree* Compiler::impSIMDIntrinsic(OPCODE opcode,
2363 GenTree* newobjThis,
2364 CORINFO_CLASS_HANDLE clsHnd,
2365 CORINFO_METHOD_HANDLE methodHnd,
2366 CORINFO_SIG_INFO* sig,
2367 unsigned methodFlags,
2370 assert(featureSIMD);
2372 // Exit early if we are not in one of the SIMD types.
2373 if (!isSIMDClass(clsHnd))
2378 #ifdef FEATURE_CORECLR
2379 // For coreclr, we also exit early if the method is not a JIT Intrinsic (which requires the [Intrinsic] attribute).
2380 if ((methodFlags & CORINFO_FLG_JIT_INTRINSIC) == 0)
2384 #endif // FEATURE_CORECLR
2386 // Get base type and intrinsic Id
2387 var_types baseType = TYP_UNKNOWN;
2389 unsigned argCount = 0;
2390 const SIMDIntrinsicInfo* intrinsicInfo =
2391 getSIMDIntrinsicInfo(&clsHnd, methodHnd, sig, (opcode == CEE_NEWOBJ), &argCount, &baseType, &size);
2392 if (intrinsicInfo == nullptr || intrinsicInfo->id == SIMDIntrinsicInvalid)
2397 SIMDIntrinsicID simdIntrinsicID = intrinsicInfo->id;
2399 if (baseType != TYP_UNKNOWN)
2401 simdType = getSIMDTypeForSize(size);
2405 assert(simdIntrinsicID == SIMDIntrinsicHWAccel);
2406 simdType = TYP_UNKNOWN;
2408 bool instMethod = intrinsicInfo->isInstMethod;
2409 var_types callType = JITtype2varType(sig->retType);
2410 if (callType == TYP_STRUCT)
2412 // Note that here we are assuming that, if the call returns a struct, that it is the same size as the
2413 // struct on which the method is declared. This is currently true for all methods on Vector types,
2414 // but if this ever changes, we will need to determine the callType from the signature.
2415 assert(info.compCompHnd->getClassSize(sig->retTypeClass) == genTypeSize(simdType));
2416 callType = simdType;
2419 GenTree* simdTree = nullptr;
2420 GenTree* op1 = nullptr;
2421 GenTree* op2 = nullptr;
2422 GenTree* op3 = nullptr;
2423 GenTree* retVal = nullptr;
2424 GenTree* copyBlkDst = nullptr;
2425 bool doCopyBlk = false;
2427 switch (simdIntrinsicID)
2429 case SIMDIntrinsicGetCount:
2431 int length = getSIMDVectorLength(clsHnd);
2432 GenTreeIntCon* intConstTree = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, length);
2433 retVal = intConstTree;
2435 intConstTree->gtFlags |= GTF_ICON_SIMD_COUNT;
2439 case SIMDIntrinsicGetZero:
2440 retVal = gtNewSIMDVectorZero(simdType, baseType, size);
2443 case SIMDIntrinsicGetOne:
2444 retVal = gtNewSIMDVectorOne(simdType, baseType, size);
2447 case SIMDIntrinsicGetAllOnes:
2449 // Equivalent to (Vector<T>) new Vector<int>(0xffffffff);
2450 GenTree* initVal = gtNewIconNode(0xffffffff, TYP_INT);
2451 simdTree = gtNewSIMDNode(simdType, initVal, nullptr, SIMDIntrinsicInit, TYP_INT, size);
2452 if (baseType != TYP_INT)
2454 // cast it to required baseType if different from TYP_INT
2455 simdTree = gtNewSIMDNode(simdType, simdTree, nullptr, SIMDIntrinsicCast, baseType, size);
2461 case SIMDIntrinsicInit:
2462 case SIMDIntrinsicInitN:
2464 // SIMDIntrinsicInit:
2465 // op2 - the initializer value
2466 // op1 - byref of vector
2468 // SIMDIntrinsicInitN
2469 // op2 - list of initializer values stitched into a list
2470 // op1 - byref of vector
2471 bool initFromFirstArgIndir = false;
2472 if (simdIntrinsicID == SIMDIntrinsicInit)
2474 op2 = impSIMDPopStack(baseType);
2478 assert(simdIntrinsicID == SIMDIntrinsicInitN);
2479 assert(baseType == TYP_FLOAT);
2481 unsigned initCount = argCount - 1;
2482 unsigned elementCount = getSIMDVectorLength(size, baseType);
2483 noway_assert(initCount == elementCount);
2485 // Build a GT_LIST with the N values.
2486 // We must maintain left-to-right order of the args, but we will pop
2487 // them off in reverse order (the Nth arg was pushed onto the stack last).
2489 GenTree* list = nullptr;
2490 GenTree* firstArg = nullptr;
2491 GenTree* prevArg = nullptr;
2492 bool areArgsContiguous = true;
2493 for (unsigned i = 0; i < initCount; i++)
2495 GenTree* nextArg = impSIMDPopStack(baseType);
2496 if (areArgsContiguous)
2498 GenTree* curArg = nextArg;
2501 if (prevArg != nullptr)
2503 // Recall that we are popping the args off the stack in reverse order.
2504 areArgsContiguous = areArgumentsContiguous(curArg, prevArg);
2509 list = new (this, GT_LIST) GenTreeOp(GT_LIST, baseType, nextArg, list);
2512 if (areArgsContiguous && baseType == TYP_FLOAT)
2514 // Since Vector2, Vector3 and Vector4's arguments type are only float,
2515 // we intialize the vector from first argument address, only when
2516 // the baseType is TYP_FLOAT and the arguments are located contiguously in memory
2517 initFromFirstArgIndir = true;
2518 GenTree* op2Address = createAddressNodeForSIMDInit(firstArg, size);
2519 var_types simdType = getSIMDTypeForSize(size);
2520 op2 = gtNewOperNode(GT_IND, simdType, op2Address);
2528 op1 = getOp1ForConstructor(opcode, newobjThis, clsHnd);
2530 assert(op1->TypeGet() == TYP_BYREF);
2531 assert(genActualType(op2->TypeGet()) == genActualType(baseType) || initFromFirstArgIndir);
2533 // For integral base types of size less than TYP_INT, expand the initializer
2534 // to fill size of TYP_INT bytes.
2535 if (varTypeIsSmallInt(baseType))
2537 // This case should occur only for Init intrinsic.
2538 assert(simdIntrinsicID == SIMDIntrinsicInit);
2540 unsigned baseSize = genTypeSize(baseType);
2544 multiplier = 0x01010101;
2548 assert(baseSize == 2);
2549 multiplier = 0x00010001;
2552 GenTree* t1 = nullptr;
2553 if (baseType == TYP_BYTE)
2555 // What we have is a signed byte initializer,
2556 // which when loaded to a reg will get sign extended to TYP_INT.
2557 // But what we need is the initializer without sign extended or
2558 // rather zero extended to 32-bits.
2559 t1 = gtNewOperNode(GT_AND, TYP_INT, op2, gtNewIconNode(0xff, TYP_INT));
2561 else if (baseType == TYP_SHORT)
2563 // What we have is a signed short initializer,
2564 // which when loaded to a reg will get sign extended to TYP_INT.
2565 // But what we need is the initializer without sign extended or
2566 // rather zero extended to 32-bits.
2567 t1 = gtNewOperNode(GT_AND, TYP_INT, op2, gtNewIconNode(0xffff, TYP_INT));
2571 assert(baseType == TYP_UBYTE || baseType == TYP_USHORT);
2572 t1 = gtNewCastNode(TYP_INT, op2, false, TYP_INT);
2575 assert(t1 != nullptr);
2576 GenTree* t2 = gtNewIconNode(multiplier, TYP_INT);
2577 op2 = gtNewOperNode(GT_MUL, TYP_INT, t1, t2);
2579 // Construct a vector of TYP_INT with the new initializer and cast it back to vector of baseType
2580 simdTree = gtNewSIMDNode(simdType, op2, nullptr, simdIntrinsicID, TYP_INT, size);
2581 simdTree = gtNewSIMDNode(simdType, simdTree, nullptr, SIMDIntrinsicCast, baseType, size);
2586 if (initFromFirstArgIndir)
2589 if (op1->gtOp.gtOp1->OperIsLocal())
2591 // label the dst struct's lclvar is used for SIMD intrinsic,
2592 // so that this dst struct won't be promoted.
2593 setLclRelatedToSIMDIntrinsic(op1->gtOp.gtOp1);
2598 simdTree = gtNewSIMDNode(simdType, op2, nullptr, simdIntrinsicID, baseType, size);
2607 case SIMDIntrinsicInitArray:
2608 case SIMDIntrinsicInitArrayX:
2609 case SIMDIntrinsicCopyToArray:
2610 case SIMDIntrinsicCopyToArrayX:
2612 // op3 - index into array in case of SIMDIntrinsicCopyToArrayX and SIMDIntrinsicInitArrayX
2613 // op2 - array itself
2614 // op1 - byref to vector struct
2616 unsigned int vectorLength = getSIMDVectorLength(size, baseType);
2617 // (This constructor takes only the zero-based arrays.)
2618 // We will add one or two bounds checks:
2619 // 1. If we have an index, we must do a check on that first.
2620 // We can't combine it with the index + vectorLength check because
2621 // a. It might be negative, and b. It may need to raise a different exception
2622 // (captured as SCK_ARG_RNG_EXCPN for CopyTo and SCK_RNGCHK_FAIL for Init).
2623 // 2. We need to generate a check (SCK_ARG_EXCPN for CopyTo and SCK_RNGCHK_FAIL for Init)
2624 // for the last array element we will access.
2625 // We'll either check against (vectorLength - 1) or (index + vectorLength - 1).
2627 GenTree* checkIndexExpr = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, vectorLength - 1);
2629 // Get the index into the array. If it has been provided, it will be on the
2630 // top of the stack. Otherwise, it is null.
2633 op3 = impSIMDPopStack(TYP_INT);
2634 if (op3->IsIntegralConst(0))
2641 // TODO-CQ: Here, or elsewhere, check for the pattern where op2 is a newly constructed array, and
2642 // change this to the InitN form.
2643 // op3 = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, 0);
2647 // Clone the array for use in the bounds check.
2648 op2 = impSIMDPopStack(TYP_REF);
2649 assert(op2->TypeGet() == TYP_REF);
2650 GenTree* arrayRefForArgChk = op2;
2651 GenTree* argRngChk = nullptr;
2652 if ((arrayRefForArgChk->gtFlags & GTF_SIDE_EFFECT) != 0)
2654 op2 = fgInsertCommaFormTemp(&arrayRefForArgChk);
2658 op2 = gtCloneExpr(arrayRefForArgChk);
2660 assert(op2 != nullptr);
2664 SpecialCodeKind op3CheckKind;
2665 if (simdIntrinsicID == SIMDIntrinsicInitArrayX)
2667 op3CheckKind = SCK_RNGCHK_FAIL;
2671 assert(simdIntrinsicID == SIMDIntrinsicCopyToArrayX);
2672 op3CheckKind = SCK_ARG_RNG_EXCPN;
2674 // We need to use the original expression on this, which is the first check.
2675 GenTree* arrayRefForArgRngChk = arrayRefForArgChk;
2676 // Then we clone the clone we just made for the next check.
2677 arrayRefForArgChk = gtCloneExpr(op2);
2678 // We know we MUST have had a cloneable expression.
2679 assert(arrayRefForArgChk != nullptr);
2680 GenTree* index = op3;
2681 if ((index->gtFlags & GTF_SIDE_EFFECT) != 0)
2683 op3 = fgInsertCommaFormTemp(&index);
2687 op3 = gtCloneExpr(index);
2690 GenTreeArrLen* arrLen =
2691 gtNewArrLen(TYP_INT, arrayRefForArgRngChk, (int)OFFSETOF__CORINFO_Array__length);
2692 argRngChk = new (this, GT_ARR_BOUNDS_CHECK)
2693 GenTreeBoundsChk(GT_ARR_BOUNDS_CHECK, TYP_VOID, index, arrLen, op3CheckKind);
2694 // Now, clone op3 to create another node for the argChk
2695 GenTree* index2 = gtCloneExpr(op3);
2696 assert(index != nullptr);
2697 checkIndexExpr = gtNewOperNode(GT_ADD, TYP_INT, index2, checkIndexExpr);
2700 // Insert a bounds check for index + offset - 1.
2701 // This must be a "normal" array.
2702 SpecialCodeKind op2CheckKind;
2703 if (simdIntrinsicID == SIMDIntrinsicInitArray || simdIntrinsicID == SIMDIntrinsicInitArrayX)
2705 op2CheckKind = SCK_RNGCHK_FAIL;
2709 op2CheckKind = SCK_ARG_EXCPN;
2711 GenTreeArrLen* arrLen = gtNewArrLen(TYP_INT, arrayRefForArgChk, (int)OFFSETOF__CORINFO_Array__length);
2712 GenTreeBoundsChk* argChk = new (this, GT_ARR_BOUNDS_CHECK)
2713 GenTreeBoundsChk(GT_ARR_BOUNDS_CHECK, TYP_VOID, checkIndexExpr, arrLen, op2CheckKind);
2715 // Create a GT_COMMA tree for the bounds check(s).
2716 op2 = gtNewOperNode(GT_COMMA, op2->TypeGet(), argChk, op2);
2717 if (argRngChk != nullptr)
2719 op2 = gtNewOperNode(GT_COMMA, op2->TypeGet(), argRngChk, op2);
2722 if (simdIntrinsicID == SIMDIntrinsicInitArray || simdIntrinsicID == SIMDIntrinsicInitArrayX)
2724 op1 = getOp1ForConstructor(opcode, newobjThis, clsHnd);
2725 simdTree = gtNewSIMDNode(simdType, op2, op3, SIMDIntrinsicInitArray, baseType, size);
2731 assert(simdIntrinsicID == SIMDIntrinsicCopyToArray || simdIntrinsicID == SIMDIntrinsicCopyToArrayX);
2732 op1 = impSIMDPopStack(simdType, instMethod);
2733 assert(op1->TypeGet() == simdType);
2735 // copy vector (op1) to array (op2) starting at index (op3)
2738 // TODO-Cleanup: Though it happens to just work fine front-end phases are not aware of GT_LEA node.
2739 // Therefore, convert these to use GT_ADDR .
2740 copyBlkDst = new (this, GT_LEA)
2741 GenTreeAddrMode(TYP_BYREF, op2, op3, genTypeSize(baseType), OFFSETOF__CORINFO_Array__data);
2747 case SIMDIntrinsicInitFixed:
2749 // We are initializing a fixed-length vector VLarge with a smaller fixed-length vector VSmall, plus 1 or 2
2750 // additional floats.
2751 // op4 (optional) - float value for VLarge.W, if VLarge is Vector4, and VSmall is Vector2
2752 // op3 - float value for VLarge.Z or VLarge.W
2754 // op1 - byref of VLarge
2755 assert(baseType == TYP_FLOAT);
2757 GenTree* op4 = nullptr;
2760 op4 = impSIMDPopStack(TYP_FLOAT);
2761 assert(op4->TypeGet() == TYP_FLOAT);
2763 op3 = impSIMDPopStack(TYP_FLOAT);
2764 assert(op3->TypeGet() == TYP_FLOAT);
2765 // The input vector will either be TYP_SIMD8 or TYP_SIMD12.
2766 var_types smallSIMDType = TYP_SIMD8;
2767 if ((op4 == nullptr) && (simdType == TYP_SIMD16))
2769 smallSIMDType = TYP_SIMD12;
2771 op2 = impSIMDPopStack(smallSIMDType);
2772 op1 = getOp1ForConstructor(opcode, newobjThis, clsHnd);
2774 // We are going to redefine the operands so that:
2775 // - op3 is the value that's going into the Z position, or null if it's a Vector4 constructor with a single
2777 // - op4 is the W position value, or null if this is a Vector3 constructor.
2778 if (size == 16 && argCount == 3)
2787 simdTree = gtNewSIMDNode(simdType, simdTree, op3, SIMDIntrinsicSetZ, baseType, size);
2791 simdTree = gtNewSIMDNode(simdType, simdTree, op4, SIMDIntrinsicSetW, baseType, size);
2799 case SIMDIntrinsicOpEquality:
2800 case SIMDIntrinsicInstEquals:
2802 op2 = impSIMDPopStack(simdType);
2803 op1 = impSIMDPopStack(simdType, instMethod);
2805 assert(op1->TypeGet() == simdType);
2806 assert(op2->TypeGet() == simdType);
2808 simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, SIMDIntrinsicOpEquality, baseType, size);
2809 if (simdType == TYP_SIMD12)
2811 simdTree->gtFlags |= GTF_SIMD12_OP;
2817 case SIMDIntrinsicOpInEquality:
2819 // op1 is the first operand
2820 // op2 is the second operand
2821 op2 = impSIMDPopStack(simdType);
2822 op1 = impSIMDPopStack(simdType, instMethod);
2823 simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, SIMDIntrinsicOpInEquality, baseType, size);
2824 if (simdType == TYP_SIMD12)
2826 simdTree->gtFlags |= GTF_SIMD12_OP;
2832 case SIMDIntrinsicEqual:
2833 case SIMDIntrinsicLessThan:
2834 case SIMDIntrinsicLessThanOrEqual:
2835 case SIMDIntrinsicGreaterThan:
2836 case SIMDIntrinsicGreaterThanOrEqual:
2838 op2 = impSIMDPopStack(simdType);
2839 op1 = impSIMDPopStack(simdType, instMethod);
2841 SIMDIntrinsicID intrinsicID = impSIMDRelOp(simdIntrinsicID, clsHnd, size, &baseType, &op1, &op2);
2842 simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, intrinsicID, baseType, size);
2847 case SIMDIntrinsicAdd:
2848 case SIMDIntrinsicSub:
2849 case SIMDIntrinsicMul:
2850 case SIMDIntrinsicDiv:
2851 case SIMDIntrinsicBitwiseAnd:
2852 case SIMDIntrinsicBitwiseAndNot:
2853 case SIMDIntrinsicBitwiseOr:
2854 case SIMDIntrinsicBitwiseXor:
2857 // check for the cases where we don't support intrinsics.
2858 // This check should be done before we make modifications to type stack.
2859 // Note that this is more of a double safety check for robustness since
2860 // we expect getSIMDIntrinsicInfo() to have filtered out intrinsics on
2861 // unsupported base types. If getSIMdIntrinsicInfo() doesn't filter due
2862 // to some bug, assert in chk/dbg will fire.
2863 if (!varTypeIsFloating(baseType))
2865 if (simdIntrinsicID == SIMDIntrinsicMul)
2867 #if defined(_TARGET_XARCH_)
2868 if ((baseType != TYP_INT) && (baseType != TYP_SHORT))
2870 // TODO-CQ: implement mul on these integer vectors.
2871 // Note that SSE2 has no direct support for these vectors.
2872 assert(!"Mul not supported on long/ulong/uint/small int vectors\n");
2875 #endif // _TARGET_XARCH_
2876 #if defined(_TARGET_ARM64_)
2877 if ((baseType == TYP_ULONG) && (baseType == TYP_LONG))
2879 // TODO-CQ: implement mul on these integer vectors.
2880 // Note that ARM64 has no direct support for these vectors.
2881 assert(!"Mul not supported on long/ulong vectors\n");
2884 #endif // _TARGET_ARM64_
2886 #if defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
2887 // common to all integer type vectors
2888 if (simdIntrinsicID == SIMDIntrinsicDiv)
2890 // SSE2 doesn't support div on non-floating point vectors.
2891 assert(!"Div not supported on integer type vectors\n");
2894 #endif // defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
2898 // op1 is the first operand; if instance method, op1 is "this" arg
2899 // op2 is the second operand
2900 op2 = impSIMDPopStack(simdType);
2901 op1 = impSIMDPopStack(simdType, instMethod);
2903 #ifdef _TARGET_XARCH_
2904 if (simdIntrinsicID == SIMDIntrinsicBitwiseAndNot)
2906 // XARCH implements SIMDIntrinsicBitwiseAndNot as ~op1 & op2, while the
2907 // software implementation does op1 & ~op2, so we need to swap the operands
2913 #endif // _TARGET_XARCH_
2915 simdTree = gtNewSIMDNode(simdType, op1, op2, simdIntrinsicID, baseType, size);
2920 case SIMDIntrinsicSelect:
2922 // op3 is a SIMD variable that is the second source
2923 // op2 is a SIMD variable that is the first source
2924 // op1 is a SIMD variable which is the bit mask.
2925 op3 = impSIMDPopStack(simdType);
2926 op2 = impSIMDPopStack(simdType);
2927 op1 = impSIMDPopStack(simdType);
2929 retVal = impSIMDSelect(clsHnd, baseType, size, op1, op2, op3);
2933 case SIMDIntrinsicMin:
2934 case SIMDIntrinsicMax:
2936 // op1 is the first operand; if instance method, op1 is "this" arg
2937 // op2 is the second operand
2938 op2 = impSIMDPopStack(simdType);
2939 op1 = impSIMDPopStack(simdType, instMethod);
2941 retVal = impSIMDMinMax(simdIntrinsicID, clsHnd, baseType, size, op1, op2);
2945 case SIMDIntrinsicGetItem:
2947 // op1 is a SIMD variable that is "this" arg
2948 // op2 is an index of TYP_INT
2949 op2 = impSIMDPopStack(TYP_INT);
2950 op1 = impSIMDPopStack(simdType, instMethod);
2951 int vectorLength = getSIMDVectorLength(size, baseType);
2952 if (!op2->IsCnsIntOrI() || op2->AsIntCon()->gtIconVal >= vectorLength || op2->AsIntCon()->gtIconVal < 0)
2954 // We need to bounds-check the length of the vector.
2955 // For that purpose, we need to clone the index expression.
2956 GenTree* index = op2;
2957 if ((index->gtFlags & GTF_SIDE_EFFECT) != 0)
2959 op2 = fgInsertCommaFormTemp(&index);
2963 op2 = gtCloneExpr(index);
2966 GenTree* lengthNode = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, vectorLength);
2967 GenTreeBoundsChk* simdChk =
2968 new (this, GT_SIMD_CHK) GenTreeBoundsChk(GT_SIMD_CHK, TYP_VOID, index, lengthNode, SCK_RNGCHK_FAIL);
2970 // Create a GT_COMMA tree for the bounds check.
2971 op2 = gtNewOperNode(GT_COMMA, op2->TypeGet(), simdChk, op2);
2974 assert(op1->TypeGet() == simdType);
2975 assert(op2->TypeGet() == TYP_INT);
2977 simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, simdIntrinsicID, baseType, size);
2982 case SIMDIntrinsicDotProduct:
2984 #if defined(_TARGET_XARCH_)
2985 // Right now dot product is supported only for float/double vectors and
2986 // int vectors on SSE4/AVX.
2987 if (!varTypeIsFloating(baseType) && !(baseType == TYP_INT && getSIMDSupportLevel() >= SIMD_SSE4_Supported))
2991 #endif // _TARGET_XARCH_
2993 // op1 is a SIMD variable that is the first source and also "this" arg.
2994 // op2 is a SIMD variable which is the second source.
2995 op2 = impSIMDPopStack(simdType);
2996 op1 = impSIMDPopStack(simdType, instMethod);
2998 simdTree = gtNewSIMDNode(baseType, op1, op2, simdIntrinsicID, baseType, size);
2999 if (simdType == TYP_SIMD12)
3001 simdTree->gtFlags |= GTF_SIMD12_OP;
3007 case SIMDIntrinsicSqrt:
3009 #if (defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)) && defined(DEBUG)
3010 // SSE/AVX/ARM64 doesn't support sqrt on integer type vectors and hence
3011 // should never be seen as an intrinsic here. See SIMDIntrinsicList.h
3012 // for supported base types for this intrinsic.
3013 if (!varTypeIsFloating(baseType))
3015 assert(!"Sqrt not supported on integer vectors\n");
3018 #endif // (defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)) && defined(DEBUG)
3020 op1 = impSIMDPopStack(simdType);
3022 retVal = gtNewSIMDNode(genActualType(callType), op1, nullptr, simdIntrinsicID, baseType, size);
3026 case SIMDIntrinsicAbs:
3027 op1 = impSIMDPopStack(simdType);
3028 retVal = impSIMDAbs(clsHnd, baseType, size, op1);
3031 case SIMDIntrinsicGetW:
3032 retVal = impSIMDGetFixed(simdType, baseType, size, 3);
3035 case SIMDIntrinsicGetZ:
3036 retVal = impSIMDGetFixed(simdType, baseType, size, 2);
3039 case SIMDIntrinsicGetY:
3040 retVal = impSIMDGetFixed(simdType, baseType, size, 1);
3043 case SIMDIntrinsicGetX:
3044 retVal = impSIMDGetFixed(simdType, baseType, size, 0);
3047 case SIMDIntrinsicSetW:
3048 case SIMDIntrinsicSetZ:
3049 case SIMDIntrinsicSetY:
3050 case SIMDIntrinsicSetX:
3052 // op2 is the value to be set at indexTemp position
3053 // op1 is SIMD vector that is going to be modified, which is a byref
3055 // If op1 has a side-effect, then don't make it an intrinsic.
3056 // It would be in-efficient to read the entire vector into xmm reg,
3057 // modify it and write back entire xmm reg.
3059 // TODO-CQ: revisit this later.
3060 op1 = impStackTop(1).val;
3061 if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0)
3066 op2 = impSIMDPopStack(baseType);
3067 op1 = impSIMDPopStack(simdType, instMethod);
3069 GenTree* src = gtCloneExpr(op1);
3070 assert(src != nullptr);
3071 simdTree = gtNewSIMDNode(simdType, src, op2, simdIntrinsicID, baseType, size);
3073 copyBlkDst = gtNewOperNode(GT_ADDR, TYP_BYREF, op1);
3078 // Unary operators that take and return a Vector.
3079 case SIMDIntrinsicCast:
3080 case SIMDIntrinsicConvertToSingle:
3081 case SIMDIntrinsicConvertToDouble:
3082 case SIMDIntrinsicConvertToInt32:
3084 op1 = impSIMDPopStack(simdType, instMethod);
3086 simdTree = gtNewSIMDNode(simdType, op1, nullptr, simdIntrinsicID, baseType, size);
3091 case SIMDIntrinsicConvertToInt64:
3093 #ifdef _TARGET_64BIT_
3094 op1 = impSIMDPopStack(simdType, instMethod);
3096 simdTree = gtNewSIMDNode(simdType, op1, nullptr, simdIntrinsicID, baseType, size);
3099 JITDUMP("SIMD Conversion to Int64 is not supported on this platform\n");
3105 case SIMDIntrinsicNarrow:
3107 assert(!instMethod);
3108 op2 = impSIMDPopStack(simdType);
3109 op1 = impSIMDPopStack(simdType);
3110 // op1 and op2 are two input Vector<T>.
3111 simdTree = gtNewSIMDNode(simdType, op1, op2, simdIntrinsicID, baseType, size);
3116 case SIMDIntrinsicWiden:
3118 GenTree* dstAddrHi = impSIMDPopStack(TYP_BYREF);
3119 GenTree* dstAddrLo = impSIMDPopStack(TYP_BYREF);
3120 op1 = impSIMDPopStack(simdType);
3121 // op1 must have a valid class handle; the following method will assert it.
3122 CORINFO_CLASS_HANDLE op1Handle = gtGetStructHandle(op1);
3123 GenTree* dupOp1 = fgInsertCommaFormTemp(&op1, op1Handle);
3125 // Widen the lower half and assign it to dstAddrLo.
3126 simdTree = gtNewSIMDNode(simdType, op1, nullptr, SIMDIntrinsicWidenLo, baseType, size);
3128 new (this, GT_BLK) GenTreeBlk(GT_BLK, simdType, dstAddrLo, getSIMDTypeSizeInBytes(clsHnd));
3129 GenTree* loAsg = gtNewBlkOpNode(loDest, simdTree, getSIMDTypeSizeInBytes(clsHnd),
3130 false, // not volatile
3132 loAsg->gtFlags |= ((simdTree->gtFlags | dstAddrLo->gtFlags) & GTF_ALL_EFFECT);
3134 // Widen the upper half and assign it to dstAddrHi.
3135 simdTree = gtNewSIMDNode(simdType, dupOp1, nullptr, SIMDIntrinsicWidenHi, baseType, size);
3137 new (this, GT_BLK) GenTreeBlk(GT_BLK, simdType, dstAddrHi, getSIMDTypeSizeInBytes(clsHnd));
3138 GenTree* hiAsg = gtNewBlkOpNode(hiDest, simdTree, getSIMDTypeSizeInBytes(clsHnd),
3139 false, // not volatile
3141 hiAsg->gtFlags |= ((simdTree->gtFlags | dstAddrHi->gtFlags) & GTF_ALL_EFFECT);
3143 retVal = gtNewOperNode(GT_COMMA, simdType, loAsg, hiAsg);
3147 case SIMDIntrinsicHWAccel:
3149 GenTreeIntCon* intConstTree = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, 1);
3150 retVal = intConstTree;
3155 assert(!"Unimplemented SIMD Intrinsic");
3159 #if defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
3160 // XArch/Arm64: also indicate that we use floating point registers.
3161 // The need for setting this here is that a method may not have SIMD
3162 // type lclvars, but might be exercising SIMD intrinsics on fields of
3165 // e.g. public Vector<float> ComplexVecFloat::sqabs() { return this.r * this.r + this.i * this.i; }
3166 compFloatingPointUsed = true;
3167 #endif // defined(_TARGET_XARCH_) || defined(_TARGET_ARM64_)
3169 // At this point, we have a tree that we are going to store into a destination.
3170 // TODO-1stClassStructs: This should be a simple store or assignment, and should not require
3171 // GTF_ALL_EFFECT for the dest. This is currently emulating the previous behavior of
3175 GenTree* dest = new (this, GT_BLK) GenTreeBlk(GT_BLK, simdType, copyBlkDst, getSIMDTypeSizeInBytes(clsHnd));
3176 dest->gtFlags |= GTF_GLOB_REF;
3177 retVal = gtNewBlkOpNode(dest, simdTree, getSIMDTypeSizeInBytes(clsHnd),
3178 false, // not volatile
3180 retVal->gtFlags |= ((simdTree->gtFlags | copyBlkDst->gtFlags) & GTF_ALL_EFFECT);
3186 #endif // FEATURE_SIMD