From fc56772acb96d52efabec4f58bd09711efbb0fd8 Mon Sep 17 00:00:00 2001 From: Vlad Brezae Date: Mon, 13 Jul 2020 12:04:58 +0300 Subject: [PATCH] [interp] Add intrinsics for some common operations (#38809) * [interp] Add intrinsic for Marvin.Block * [interp] Intrinsify ConvertAllAsciiCharsInUInt32ToUppercase * [interp] Intrinisfy UInt32OrdinalIgnoreCaseAscii * [interp] Intrinsify UInt64OrdinalIgnoreCaseAscii * [interp] Add intrinsic for UInt32ToDecStr * [interp] Add intrinsic opcode for WidenAsciiToUtf16 * [interp] Rename intrinsics to interp-intrins There seem to be some build failures on windows. Probably because of conflict with intrinsics source files from mini. * [interp] Add comments To prevent bcl and interp intrinsics going out of sync. --- .../System.Private.CoreLib/src/System/Marvin.cs | 1 + .../src/System/Number.Formatting.cs | 1 + .../src/System/Text/ASCIIUtility.cs | 1 + .../src/System/Text/Unicode/Utf16Utility.cs | 3 + src/mono/mono/mini/Makefile.am.in | 2 + src/mono/mono/mini/interp/interp-intrins.c | 140 +++++++++++++++++++++ src/mono/mono/mini/interp/interp-intrins.h | 27 ++++ src/mono/mono/mini/interp/interp.c | 37 ++++++ src/mono/mono/mini/interp/mintops.def | 6 + src/mono/mono/mini/interp/transform.c | 34 +++++ src/mono/msvc/libmini-interp.targets | 2 + src/mono/msvc/libmini-interp.targets.filters | 6 + 12 files changed, 260 insertions(+) create mode 100644 src/mono/mono/mini/interp/interp-intrins.c create mode 100644 src/mono/mono/mini/interp/interp-intrins.h diff --git a/src/libraries/System.Private.CoreLib/src/System/Marvin.cs b/src/libraries/System.Private.CoreLib/src/System/Marvin.cs index a621fed..322b788 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Marvin.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Marvin.cs @@ -217,6 +217,7 @@ namespace System [MethodImpl(MethodImplOptions.AggressiveInlining)] private static void Block(ref uint rp0, ref uint rp1) { + // Intrinsified in mono interpreter uint p0 = rp0; uint p1 = rp1; diff --git a/src/libraries/System.Private.CoreLib/src/System/Number.Formatting.cs b/src/libraries/System.Private.CoreLib/src/System/Number.Formatting.cs index 64385f6..299e394 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Number.Formatting.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Number.Formatting.cs @@ -1352,6 +1352,7 @@ namespace System internal static unsafe string UInt32ToDecStr(uint value) { + // Intrinsified in mono interpreter int bufferLength = FormattingHelpers.CountDigits(value); // For single-digit values that are very common, especially 0 and 1, just return cached strings. diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs b/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs index e85c1bb..7628f6f 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs @@ -1460,6 +1460,7 @@ namespace System.Text /// public static unsafe nuint WidenAsciiToUtf16(byte* pAsciiBuffer, char* pUtf16Buffer, nuint elementCount) { + // Intrinsified in mono interpreter nuint currentOffset = 0; // If SSE2 is supported, use those specific intrinsics instead of the generic vectorized diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf16Utility.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf16Utility.cs index 9c45b0a..379de97 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf16Utility.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf16Utility.cs @@ -66,6 +66,7 @@ namespace System.Text.Unicode [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static uint ConvertAllAsciiCharsInUInt32ToUppercase(uint value) { + // Intrinsified in mono interpreter // ASSUMPTION: Caller has validated that input value is ASCII. Debug.Assert(AllCharsInUInt32AreAscii(value)); @@ -144,6 +145,7 @@ namespace System.Text.Unicode [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static bool UInt32OrdinalIgnoreCaseAscii(uint valueA, uint valueB) { + // Intrinsified in mono interpreter // ASSUMPTION: Caller has validated that input values are ASCII. Debug.Assert(AllCharsInUInt32AreAscii(valueA)); Debug.Assert(AllCharsInUInt32AreAscii(valueB)); @@ -185,6 +187,7 @@ namespace System.Text.Unicode [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static bool UInt64OrdinalIgnoreCaseAscii(ulong valueA, ulong valueB) { + // Intrinsified in mono interpreter // ASSUMPTION: Caller has validated that input values are ASCII. Debug.Assert(AllCharsInUInt64AreAscii(valueA)); Debug.Assert(AllCharsInUInt64AreAscii(valueB)); diff --git a/src/mono/mono/mini/Makefile.am.in b/src/mono/mono/mini/Makefile.am.in index a8e9891..867b13b 100755 --- a/src/mono/mono/mini/Makefile.am.in +++ b/src/mono/mono/mini/Makefile.am.in @@ -427,6 +427,8 @@ interp_sources = \ interp/interp.h \ interp/interp-internals.h \ interp/interp.c \ + interp/interp-intrins.h \ + interp/interp-intrins.c \ interp/mintops.h \ interp/mintops.def \ interp/mintops.c \ diff --git a/src/mono/mono/mini/interp/interp-intrins.c b/src/mono/mono/mini/interp/interp-intrins.c new file mode 100644 index 0000000..b409322 --- /dev/null +++ b/src/mono/mono/mini/interp/interp-intrins.c @@ -0,0 +1,140 @@ +/* + * Intrinsics for libraries methods that are heavily used in interpreter relevant + * scenarios and where compiling these methods with the interpreter would have + * heavy performance impact. + */ + +#include "interp-intrins.h" + +#include +#include + +static guint32 +rotate_left (guint32 value, int offset) +{ + return (value << offset) | (value >> (32 - offset)); +} + +void +interp_intrins_marvin_block (guint32 *pp0, guint32 *pp1) +{ + // Marvin.Block + guint32 p0 = *pp0; + guint32 p1 = *pp1; + + p1 ^= p0; + p0 = rotate_left (p0, 20); + + p0 += p1; + p1 = rotate_left (p1, 9); + + p1 ^= p0; + p0 = rotate_left (p0, 27); + + p0 += p1; + p1 = rotate_left (p1, 19); + + *pp0 = p0; + *pp1 = p1; +} + +guint32 +interp_intrins_ascii_chars_to_uppercase (guint32 value) +{ + // Utf16Utility.ConvertAllAsciiCharsInUInt32ToUppercase + guint32 lowerIndicator = value + 0x00800080 - 0x00610061; + guint32 upperIndicator = value + 0x00800080 - 0x007B007B; + guint32 combinedIndicator = (lowerIndicator ^ upperIndicator); + guint32 mask = (combinedIndicator & 0x00800080) >> 2; + + return value ^ mask; +} + +int +interp_intrins_ordinal_ignore_case_ascii (guint32 valueA, guint32 valueB) +{ + // Utf16Utility.UInt32OrdinalIgnoreCaseAscii + guint32 differentBits = valueA ^ valueB; + guint32 lowerIndicator = valueA + 0x01000100 - 0x00410041; + guint32 upperIndicator = (valueA | 0x00200020u) + 0x00800080 - 0x007B007B; + guint32 combinedIndicator = lowerIndicator | upperIndicator; + return (((combinedIndicator >> 2) | ~0x00200020) & differentBits) == 0; +} + +int +interp_intrins_64ordinal_ignore_case_ascii (guint64 valueA, guint64 valueB) +{ + // Utf16Utility.UInt64OrdinalIgnoreCaseAscii + guint64 lowerIndicator = valueA + 0x0080008000800080l - 0x0041004100410041l; + guint64 upperIndicator = (valueA | 0x0020002000200020l) + 0x0100010001000100l - 0x007B007B007B007Bl; + guint64 combinedIndicator = (0x0080008000800080l & lowerIndicator & upperIndicator) >> 2; + return (valueA | combinedIndicator) == (valueB | combinedIndicator); +} + +static int +interp_intrins_count_digits (guint32 value) +{ + int digits = 1; + if (value >= 100000) { + value /= 100000; + digits += 5; + } + if (value < 10) { + // no-op + } else if (value < 100) { + digits++; + } else if (value < 1000) { + digits += 2; + } else if (value < 10000) { + digits += 3; + } else { + digits += 4; + } + return digits; +} + +static guint32 +interp_intrins_math_divrem (guint32 a, guint32 b, guint32 *result) +{ + guint32 div = a / b; + *result = a - (div * b); + return div; +} + +MonoString* +interp_intrins_u32_to_decstr (guint32 value, MonoArray *cache, MonoVTable *vtable) +{ + // Number.UInt32ToDecStr + int bufferLength = interp_intrins_count_digits (value); + + if (bufferLength == 1) + return mono_array_get_fast (cache, MonoString*, value); + + int size = (G_STRUCT_OFFSET (MonoString, chars) + (((size_t)bufferLength + 1) * 2)); + MonoString* result = mono_gc_alloc_string (vtable, size, bufferLength); + mono_unichar2 *buffer = &result->chars [0]; + mono_unichar2 *p = buffer + bufferLength; + do { + guint32 remainder; + value = interp_intrins_math_divrem (value, 10, &remainder); + *(--p) = (mono_unichar2)(remainder + '0'); + } while (value != 0); + return result; +} + +mono_u +interp_intrins_widen_ascii_to_utf16 (guint8 *pAsciiBuffer, mono_unichar2 *pUtf16Buffer, mono_u elementCount) +{ + // ASCIIUtility.WidenAsciiToUtf16 + mono_u currentOffset = 0; + + while (currentOffset < elementCount) { + guint16 asciiData = pAsciiBuffer [currentOffset]; + if ((asciiData & 0x80) != 0) + return currentOffset; + + pUtf16Buffer [currentOffset] = (mono_unichar2)asciiData; + currentOffset++; + } + return currentOffset; +} diff --git a/src/mono/mono/mini/interp/interp-intrins.h b/src/mono/mono/mini/interp/interp-intrins.h new file mode 100644 index 0000000..2be852f --- /dev/null +++ b/src/mono/mono/mini/interp/interp-intrins.h @@ -0,0 +1,27 @@ +#ifndef __MONO_MINI_INTERP_INTRINSICS_H__ +#define __MONO_MINI_INTERP_INTRINSICS_H__ + +#include +#include + +#include "interp-internals.h" + +void +interp_intrins_marvin_block (guint32 *pp0, guint32 *pp1); + +guint32 +interp_intrins_ascii_chars_to_uppercase (guint32 val); + +int +interp_intrins_ordinal_ignore_case_ascii (guint32 valueA, guint32 valueB); + +int +interp_intrins_64ordinal_ignore_case_ascii (guint64 valueA, guint64 valueB); + +MonoString* +interp_intrins_u32_to_decstr (guint32 value, MonoArray *cache, MonoVTable *vtable); + +mono_u +interp_intrins_widen_ascii_to_utf16 (guint8 *pAsciiBuffer, mono_unichar2 *pUtf16Buffer, mono_u elementCount); + +#endif /* __MONO_MINI_INTERP_INTRINSICS_H__ */ diff --git a/src/mono/mono/mini/interp/interp.c b/src/mono/mono/mini/interp/interp.c index c81ad63..6d6aa9d 100644 --- a/src/mono/mono/mini/interp/interp.c +++ b/src/mono/mono/mini/interp/interp.c @@ -63,6 +63,7 @@ #include "interp.h" #include "interp-internals.h" #include "mintops.h" +#include "interp-intrins.h" #include #include @@ -5281,6 +5282,42 @@ call_newobj: ++ip; MINT_IN_BREAK; } + MINT_IN_CASE(MINT_INTRINS_MARVIN_BLOCK) { + sp -= 2; + interp_intrins_marvin_block ((guint32*)sp [0].data.p, (guint32*)sp [1].data.p); + ++ip; + MINT_IN_BREAK; + } + MINT_IN_CASE(MINT_INTRINS_ASCII_CHARS_TO_UPPERCASE) { + sp [-1].data.i = interp_intrins_ascii_chars_to_uppercase ((guint32)sp [-1].data.i); + ++ip; + MINT_IN_BREAK; + } + MINT_IN_CASE(MINT_INTRINS_ORDINAL_IGNORE_CASE_ASCII) { + sp--; + sp [-1].data.i = interp_intrins_ordinal_ignore_case_ascii ((guint32)sp [-1].data.i, (guint32)sp [0].data.i); + ++ip; + MINT_IN_BREAK; + } + MINT_IN_CASE(MINT_INTRINS_64ORDINAL_IGNORE_CASE_ASCII) { + sp--; + sp [-1].data.i = interp_intrins_64ordinal_ignore_case_ascii ((guint64)sp [-1].data.l, (guint64)sp [0].data.l); + ++ip; + MINT_IN_BREAK; + } + MINT_IN_CASE(MINT_INTRINS_U32_TO_DECSTR) { + MonoArray **cache_addr = (MonoArray**)frame->imethod->data_items [ip [1]]; + MonoVTable *string_vtable = (MonoVTable*)frame->imethod->data_items [ip [2]]; + sp [-1].data.o = (MonoObject*)interp_intrins_u32_to_decstr ((guint32)sp [-1].data.i, *cache_addr, string_vtable); + ip += 3; + MINT_IN_BREAK; + } + MINT_IN_CASE(MINT_INTRINS_WIDEN_ASCII_TO_UTF16) { + sp -= 2; + sp [-1].data.nati = interp_intrins_widen_ascii_to_utf16 ((guint8*)sp [-1].data.p, (mono_unichar2*)sp [0].data.p, sp [1].data.nati); + ip++; + MINT_IN_BREAK; + } MINT_IN_CASE(MINT_INTRINS_UNSAFE_BYTE_OFFSET) { sp -= 2; sp [0].data.nati = (guint8*)sp [1].data.p - (guint8*)sp [0].data.p; diff --git a/src/mono/mono/mini/interp/mintops.def b/src/mono/mono/mini/interp/mintops.def index b311688..2367907 100644 --- a/src/mono/mono/mini/interp/mintops.def +++ b/src/mono/mono/mini/interp/mintops.def @@ -792,3 +792,9 @@ OPDEF(MINT_INTRINS_UNSAFE_ADD_BYTE_OFFSET, "intrins_unsafe_add_byte_offset", 1, OPDEF(MINT_INTRINS_UNSAFE_BYTE_OFFSET, "intrins_unsafe_byte_offset", 1, Pop2, Push1, MintOpNoArgs) OPDEF(MINT_INTRINS_RUNTIMEHELPERS_OBJECT_HAS_COMPONENT_SIZE, "intrins_runtimehelpers_object_has_component_size", 1, Pop1, Push1, MintOpNoArgs) OPDEF(MINT_INTRINS_CLEAR_WITH_REFERENCES, "intrin_clear_with_references", 1, Pop2, Push0, MintOpNoArgs) +OPDEF(MINT_INTRINS_MARVIN_BLOCK, "intrins_marvin_block", 1, Pop2, Push0, MintOpNoArgs) +OPDEF(MINT_INTRINS_ASCII_CHARS_TO_UPPERCASE, "intrins_ascii_chars_to_uppercase", 1, Pop1, Push1, MintOpNoArgs) +OPDEF(MINT_INTRINS_ORDINAL_IGNORE_CASE_ASCII, "intrins_ordinal_ignore_case_ascii", 1, Pop2, Push1, MintOpNoArgs) +OPDEF(MINT_INTRINS_64ORDINAL_IGNORE_CASE_ASCII, "intrins_64ordinal_ignore_case_ascii", 1, Pop2, Push1, MintOpNoArgs) +OPDEF(MINT_INTRINS_U32_TO_DECSTR, "intrins_u32_to_decstr", 3, Pop1, Push1, MintOpNoArgs) +OPDEF(MINT_INTRINS_WIDEN_ASCII_TO_UTF16, "intrins_widen_ascii_to_utf16", 1, Pop3, Push1, MintOpNoArgs) diff --git a/src/mono/mono/mini/interp/transform.c b/src/mono/mono/mini/interp/transform.c index 351812f..accc016 100644 --- a/src/mono/mono/mini/interp/transform.c +++ b/src/mono/mono/mini/interp/transform.c @@ -1502,6 +1502,40 @@ interp_handle_intrinsics (TransformData *td, MonoMethod *target_method, MonoClas } else if (in_corlib && !strcmp (klass_name_space, "System") && !strcmp (klass_name, "ByReference`1")) { g_assert (!strcmp (tm, "get_Value")); *op = MINT_INTRINS_BYREFERENCE_GET_VALUE; + } else if (in_corlib && !strcmp (klass_name_space, "System") && !strcmp (klass_name, "Marvin")) { + if (!strcmp (tm, "Block")) + *op = MINT_INTRINS_MARVIN_BLOCK; + } else if (in_corlib && !strcmp (klass_name_space, "System.Text.Unicode") && !strcmp (klass_name, "Utf16Utility")) { + if (!strcmp (tm, "ConvertAllAsciiCharsInUInt32ToUppercase")) + *op = MINT_INTRINS_ASCII_CHARS_TO_UPPERCASE; + else if (!strcmp (tm, "UInt32OrdinalIgnoreCaseAscii")) + *op = MINT_INTRINS_ORDINAL_IGNORE_CASE_ASCII; + else if (!strcmp (tm, "UInt64OrdinalIgnoreCaseAscii")) + *op = MINT_INTRINS_64ORDINAL_IGNORE_CASE_ASCII; + } else if (in_corlib && !strcmp (klass_name_space, "System.Text") && !strcmp (klass_name, "ASCIIUtility")) { + if (!strcmp (tm, "WidenAsciiToUtf16")) + *op = MINT_INTRINS_WIDEN_ASCII_TO_UTF16; + } else if (in_corlib && !strcmp (klass_name_space, "System") && !strcmp (klass_name, "Number")) { + if (!strcmp (tm, "UInt32ToDecStr") && csignature->param_count == 1) { + ERROR_DECL(error); + MonoVTable *vtable = mono_class_vtable_checked (td->rtm->domain, target_method->klass, error); + if (!is_ok (error)) { + mono_error_cleanup (error); + return FALSE; + } + /* Don't use intrinsic if cctor not yet run */ + if (!vtable->initialized) + return FALSE; + /* The cache is the first static field. Update this if bcl code changes */ + MonoClassField *field = m_class_get_fields (target_method->klass); + g_assert (!strcmp (field->name, "s_singleDigitStringCache")); + interp_add_ins (td, MINT_INTRINS_U32_TO_DECSTR); + td->last_ins->data [0] = get_data_item_index (td, (char*)mono_vtable_get_static_field_data (vtable) + field->offset); + td->last_ins->data [1] = get_data_item_index (td, mono_class_vtable_checked (td->rtm->domain, mono_defaults.string_class, error)); + SET_TYPE (td->sp - 1, STACK_TYPE_O, mono_defaults.string_class); + td->ip += 5; + return TRUE; + } } else if (in_corlib && !strcmp (klass_name_space, "System") && (!strcmp (klass_name, "Math") || !strcmp (klass_name, "MathF"))) { gboolean is_float = strcmp (klass_name, "MathF") == 0; diff --git a/src/mono/msvc/libmini-interp.targets b/src/mono/msvc/libmini-interp.targets index 31ef8c0..0bfddcb 100644 --- a/src/mono/msvc/libmini-interp.targets +++ b/src/mono/msvc/libmini-interp.targets @@ -8,6 +8,8 @@ + + diff --git a/src/mono/msvc/libmini-interp.targets.filters b/src/mono/msvc/libmini-interp.targets.filters index 267a6ba..0911468 100644 --- a/src/mono/msvc/libmini-interp.targets.filters +++ b/src/mono/msvc/libmini-interp.targets.filters @@ -13,6 +13,12 @@ Source Files$(MonoMiniFilterSubFolder)\interp + + Header Files$(MonoMiniFilterSubFolder)\interp + + + Source Files$(MonoMiniFilterSubFolder)\interp + Header Files$(MonoMiniFilterSubFolder)\interp -- 2.7.4