1 // Licensed to the .NET Foundation under one or more agreements.
2 // The .NET Foundation licenses this file to you under the MIT license.
3 // See the LICENSE file in the project root for more information.
5 // The worker functions in this file was optimized for performance. If you make changes
6 // you should use care to consider all of the interesting cases.
8 // The code of all worker functions in this file is written twice: Once as as a slow loop, and the
9 // second time as a fast loop. The slow loops handles all special cases, throws exceptions, etc.
10 // The fast loops attempts to blaze through as fast as possible with optimistic range checks,
11 // processing multiple characters at a time, and falling back to the slow loop for all special cases.
13 // This define can be used to turn off the fast loops. Useful for finding whether
14 // the problem is fastloop-specific.
18 using System.Diagnostics;
19 using System.Diagnostics.Contracts;
20 using System.Globalization;
24 // Encodes text into and out of UTF-8. UTF-8 is a way of writing
25 // Unicode characters with variable numbers of bytes per character,
26 // optimized for the lower 127 ASCII characters. It's an efficient way
27 // of encoding US English in an internationalizable way.
29 // Don't override IsAlwaysNormalized because it is just a Unicode Transformation and could be confused.
31 // The UTF-8 byte order mark is simply the Unicode byte order mark
32 // (0xFEFF) written in UTF-8 (0xEF 0xBB 0xBF). The byte order mark is
33 // used mostly to distinguish UTF-8 text from other encodings, and doesn't
34 // switch the byte orderings.
36 public class UTF8Encoding : Encoding
39 bytes bits UTF-8 representation
40 ----- ---- -----------------------------------
42 2 11 110vvvvv 10vvvvvv
43 3 16 1110vvvv 10vvvvvv 10vvvvvv
44 4 21 11110vvv 10vvvvvv 10vvvvvv 10vvvvvv
45 ----- ---- -----------------------------------
48 Real Unicode value = (HighSurrogate - 0xD800) * 0x400 + (LowSurrogate - 0xDC00) + 0x10000
51 private const int UTF8_CODEPAGE = 65001;
53 // Allow for de-virtualization (see https://github.com/dotnet/coreclr/pull/9230)
54 internal sealed class UTF8EncodingSealed : UTF8Encoding
56 public UTF8EncodingSealed(bool encoderShouldEmitUTF8Identifier) : base(encoderShouldEmitUTF8Identifier) { }
59 // Used by Encoding.UTF8 for lazy initialization
60 // The initialization code will not be run until a static member of the class is referenced
61 internal static readonly UTF8EncodingSealed s_default = new UTF8EncodingSealed(encoderShouldEmitUTF8Identifier: true);
63 // Yes, the idea of emitting U+FEFF as a UTF-8 identifier has made it into
65 private bool _emitUTF8Identifier = false;
67 private bool _isThrowException = false;
70 public UTF8Encoding() : this(false)
75 public UTF8Encoding(bool encoderShouldEmitUTF8Identifier) :
76 this(encoderShouldEmitUTF8Identifier, false)
81 public UTF8Encoding(bool encoderShouldEmitUTF8Identifier, bool throwOnInvalidBytes) :
84 _emitUTF8Identifier = encoderShouldEmitUTF8Identifier;
85 _isThrowException = throwOnInvalidBytes;
87 // Encoding's constructor already did this, but it'll be wrong if we're throwing exceptions
88 if (_isThrowException)
89 SetDefaultFallbacks();
92 internal override void SetDefaultFallbacks()
94 // For UTF-X encodings, we use a replacement fallback with an empty string
95 if (_isThrowException)
97 this.encoderFallback = EncoderFallback.ExceptionFallback;
98 this.decoderFallback = DecoderFallback.ExceptionFallback;
102 this.encoderFallback = new EncoderReplacementFallback("\xFFFD");
103 this.decoderFallback = new DecoderReplacementFallback("\xFFFD");
108 // WARNING: GetByteCount(string chars)
109 // WARNING: has different variable names than EncodingNLS.cs, so this can't just be cut & pasted,
110 // WARNING: otherwise it'll break VB's way of declaring these.
112 // The following methods are copied from EncodingNLS.cs.
113 // Unfortunately EncodingNLS.cs is internal and we're public, so we have to re-implement them here.
114 // These should be kept in sync for the following classes:
115 // EncodingNLS, UTF7Encoding, UTF8Encoding, UTF32Encoding, ASCIIEncoding, UnicodeEncoding
117 // Returns the number of bytes required to encode a range of characters in
118 // a character array.
120 // All of our public Encodings that don't use EncodingNLS must have this (including EncodingNLS)
121 // So if you fix this, fix the others. Currently those include:
122 // EncodingNLS, UTF7Encoding, UTF8Encoding, UTF32Encoding, ASCIIEncoding, UnicodeEncoding
123 // parent method is safe
125 public override unsafe int GetByteCount(char[] chars, int index, int count)
127 // Validate input parameters
129 throw new ArgumentNullException("chars", SR.ArgumentNull_Array);
131 if (index < 0 || count < 0)
132 throw new ArgumentOutOfRangeException((index < 0 ? "index" : "count"), SR.ArgumentOutOfRange_NeedNonNegNum);
134 if (chars.Length - index < count)
135 throw new ArgumentOutOfRangeException("chars", SR.ArgumentOutOfRange_IndexCountBuffer);
136 Contract.EndContractBlock();
138 // If no input, return 0, avoid fixed empty array problem
142 // Just call the pointer version
143 fixed (char* pChars = chars)
144 return GetByteCount(pChars + index, count, null);
147 // All of our public Encodings that don't use EncodingNLS must have this (including EncodingNLS)
148 // So if you fix this, fix the others. Currently those include:
149 // EncodingNLS, UTF7Encoding, UTF8Encoding, UTF32Encoding, ASCIIEncoding, UnicodeEncoding
150 // parent method is safe
152 public override unsafe int GetByteCount(String chars)
156 throw new ArgumentNullException("s");
157 Contract.EndContractBlock();
159 fixed (char* pChars = chars)
160 return GetByteCount(pChars, chars.Length, null);
163 // All of our public Encodings that don't use EncodingNLS must have this (including EncodingNLS)
164 // So if you fix this, fix the others. Currently those include:
165 // EncodingNLS, UTF7Encoding, UTF8Encoding, UTF32Encoding, ASCIIEncoding, UnicodeEncoding
167 [CLSCompliant(false)]
168 public override unsafe int GetByteCount(char* chars, int count)
170 // Validate Parameters
172 throw new ArgumentNullException("chars", SR.ArgumentNull_Array);
175 throw new ArgumentOutOfRangeException("count", SR.ArgumentOutOfRange_NeedNonNegNum);
176 Contract.EndContractBlock();
178 // Call it with empty encoder
179 return GetByteCount(chars, count, null);
182 // Parent method is safe.
183 // All of our public Encodings that don't use EncodingNLS must have this (including EncodingNLS)
184 // So if you fix this, fix the others. Currently those include:
185 // EncodingNLS, UTF7Encoding, UTF8Encoding, UTF32Encoding, ASCIIEncoding, UnicodeEncoding
187 public override unsafe int GetBytes(String s, int charIndex, int charCount,
188 byte[] bytes, int byteIndex)
190 if (s == null || bytes == null)
191 throw new ArgumentNullException((s == null ? "s" : "bytes"), SR.ArgumentNull_Array);
193 if (charIndex < 0 || charCount < 0)
194 throw new ArgumentOutOfRangeException((charIndex < 0 ? "charIndex" : "charCount"), SR.ArgumentOutOfRange_NeedNonNegNum);
196 if (s.Length - charIndex < charCount)
197 throw new ArgumentOutOfRangeException("s", SR.ArgumentOutOfRange_IndexCount);
199 if (byteIndex < 0 || byteIndex > bytes.Length)
200 throw new ArgumentOutOfRangeException("byteIndex", SR.ArgumentOutOfRange_Index);
201 Contract.EndContractBlock();
203 int byteCount = bytes.Length - byteIndex;
205 // Fixed doesn't like 0 length arrays.
206 if (bytes.Length == 0)
209 fixed (char* pChars = s) fixed (byte* pBytes = &bytes[0])
210 return GetBytes(pChars + charIndex, charCount, pBytes + byteIndex, byteCount, null);
213 // Encodes a range of characters in a character array into a range of bytes
214 // in a byte array. An exception occurs if the byte array is not large
215 // enough to hold the complete encoding of the characters. The
216 // GetByteCount method can be used to determine the exact number of
217 // bytes that will be produced for a given range of characters.
218 // Alternatively, the GetMaxByteCount method can be used to
219 // determine the maximum number of bytes that will be produced for a given
220 // number of characters, regardless of the actual character values.
222 // All of our public Encodings that don't use EncodingNLS must have this (including EncodingNLS)
223 // So if you fix this, fix the others. Currently those include:
224 // EncodingNLS, UTF7Encoding, UTF8Encoding, UTF32Encoding, ASCIIEncoding, UnicodeEncoding
225 // parent method is safe
227 public override unsafe int GetBytes(char[] chars, int charIndex, int charCount,
228 byte[] bytes, int byteIndex)
230 // Validate parameters
231 if (chars == null || bytes == null)
232 throw new ArgumentNullException((chars == null ? "chars" : "bytes"), SR.ArgumentNull_Array);
234 if (charIndex < 0 || charCount < 0)
235 throw new ArgumentOutOfRangeException((charIndex < 0 ? "charIndex" : "charCount"), SR.ArgumentOutOfRange_NeedNonNegNum);
237 if (chars.Length - charIndex < charCount)
238 throw new ArgumentOutOfRangeException("chars", SR.ArgumentOutOfRange_IndexCountBuffer);
240 if (byteIndex < 0 || byteIndex > bytes.Length)
241 throw new ArgumentOutOfRangeException("byteIndex", SR.ArgumentOutOfRange_Index);
242 Contract.EndContractBlock();
244 // If nothing to encode return 0, avoid fixed problem
248 // Just call pointer version
249 int byteCount = bytes.Length - byteIndex;
251 // Fixed doesn't like 0 length arrays.
252 if (bytes.Length == 0)
255 fixed (char* pChars = chars) fixed (byte* pBytes = &bytes[0])
256 // Remember that byteCount is # to decode, not size of array.
257 return GetBytes(pChars + charIndex, charCount, pBytes + byteIndex, byteCount, null);
260 // All of our public Encodings that don't use EncodingNLS must have this (including EncodingNLS)
261 // So if you fix this, fix the others. Currently those include:
262 // EncodingNLS, UTF7Encoding, UTF8Encoding, UTF32Encoding, ASCIIEncoding, UnicodeEncoding
264 [CLSCompliant(false)]
265 public override unsafe int GetBytes(char* chars, int charCount, byte* bytes, int byteCount)
267 // Validate Parameters
268 if (bytes == null || chars == null)
269 throw new ArgumentNullException(bytes == null ? "bytes" : "chars", SR.ArgumentNull_Array);
271 if (charCount < 0 || byteCount < 0)
272 throw new ArgumentOutOfRangeException((charCount < 0 ? "charCount" : "byteCount"), SR.ArgumentOutOfRange_NeedNonNegNum);
273 Contract.EndContractBlock();
275 return GetBytes(chars, charCount, bytes, byteCount, null);
278 // Returns the number of characters produced by decoding a range of bytes
281 // All of our public Encodings that don't use EncodingNLS must have this (including EncodingNLS)
282 // So if you fix this, fix the others. Currently those include:
283 // EncodingNLS, UTF7Encoding, UTF8Encoding, UTF32Encoding, ASCIIEncoding, UnicodeEncoding
284 // parent method is safe
286 public override unsafe int GetCharCount(byte[] bytes, int index, int count)
288 // Validate Parameters
290 throw new ArgumentNullException("bytes", SR.ArgumentNull_Array);
292 if (index < 0 || count < 0)
293 throw new ArgumentOutOfRangeException((index < 0 ? "index" : "count"), SR.ArgumentOutOfRange_NeedNonNegNum);
295 if (bytes.Length - index < count)
296 throw new ArgumentOutOfRangeException("bytes", SR.ArgumentOutOfRange_IndexCountBuffer);
297 Contract.EndContractBlock();
299 // If no input just return 0, fixed doesn't like 0 length arrays.
303 // Just call pointer version
304 fixed (byte* pBytes = bytes)
305 return GetCharCount(pBytes + index, count, null);
308 // All of our public Encodings that don't use EncodingNLS must have this (including EncodingNLS)
309 // So if you fix this, fix the others. Currently those include:
310 // EncodingNLS, UTF7Encoding, UTF8Encoding, UTF32Encoding, ASCIIEncoding, UnicodeEncoding
312 [CLSCompliant(false)]
313 public override unsafe int GetCharCount(byte* bytes, int count)
315 // Validate Parameters
317 throw new ArgumentNullException("bytes", SR.ArgumentNull_Array);
320 throw new ArgumentOutOfRangeException("count", SR.ArgumentOutOfRange_NeedNonNegNum);
321 Contract.EndContractBlock();
323 return GetCharCount(bytes, count, null);
326 // All of our public Encodings that don't use EncodingNLS must have this (including EncodingNLS)
327 // So if you fix this, fix the others. Currently those include:
328 // EncodingNLS, UTF7Encoding, UTF8Encoding, UTF32Encoding, ASCIIEncoding, UnicodeEncoding
329 // parent method is safe
331 public override unsafe int GetChars(byte[] bytes, int byteIndex, int byteCount,
332 char[] chars, int charIndex)
334 // Validate Parameters
335 if (bytes == null || chars == null)
336 throw new ArgumentNullException(bytes == null ? "bytes" : "chars", SR.ArgumentNull_Array);
338 if (byteIndex < 0 || byteCount < 0)
339 throw new ArgumentOutOfRangeException((byteIndex < 0 ? "byteIndex" : "byteCount"), SR.ArgumentOutOfRange_NeedNonNegNum);
341 if ( bytes.Length - byteIndex < byteCount)
342 throw new ArgumentOutOfRangeException("bytes", SR.ArgumentOutOfRange_IndexCountBuffer);
344 if (charIndex < 0 || charIndex > chars.Length)
345 throw new ArgumentOutOfRangeException("charIndex", SR.ArgumentOutOfRange_Index);
346 Contract.EndContractBlock();
348 // If no input, return 0 & avoid fixed problem
352 // Just call pointer version
353 int charCount = chars.Length - charIndex;
355 // Fixed doesn't like 0 length arrays.
356 if (chars.Length == 0)
359 fixed (byte* pBytes = bytes) fixed (char* pChars = &chars[0])
360 // Remember that charCount is # to decode, not size of array
361 return GetChars(pBytes + byteIndex, byteCount, pChars + charIndex, charCount, null);
364 // All of our public Encodings that don't use EncodingNLS must have this (including EncodingNLS)
365 // So if you fix this, fix the others. Currently those include:
366 // EncodingNLS, UTF7Encoding, UTF8Encoding, UTF32Encoding, ASCIIEncoding, UnicodeEncoding
368 [CLSCompliant(false)]
369 public unsafe override int GetChars(byte* bytes, int byteCount, char* chars, int charCount)
371 // Validate Parameters
372 if (bytes == null || chars == null)
373 throw new ArgumentNullException(bytes == null ? "bytes" : "chars", SR.ArgumentNull_Array);
375 if (charCount < 0 || byteCount < 0)
376 throw new ArgumentOutOfRangeException((charCount < 0 ? "charCount" : "byteCount"), SR.ArgumentOutOfRange_NeedNonNegNum);
377 Contract.EndContractBlock();
379 return GetChars(bytes, byteCount, chars, charCount, null);
382 // Returns a string containing the decoded representation of a range of
383 // bytes in a byte array.
385 // All of our public Encodings that don't use EncodingNLS must have this (including EncodingNLS)
386 // So if you fix this, fix the others. Currently those include:
387 // EncodingNLS, UTF7Encoding, UTF8Encoding, UTF32Encoding, ASCIIEncoding, UnicodeEncoding
388 // parent method is safe
390 public override unsafe String GetString(byte[] bytes, int index, int count)
392 // Validate Parameters
394 throw new ArgumentNullException("bytes", SR.ArgumentNull_Array);
396 if (index < 0 || count < 0)
397 throw new ArgumentOutOfRangeException((index < 0 ? "index" : "count"), SR.ArgumentOutOfRange_NeedNonNegNum);
399 if (bytes.Length - index < count)
400 throw new ArgumentOutOfRangeException("bytes", SR.ArgumentOutOfRange_IndexCountBuffer);
401 Contract.EndContractBlock();
403 // Avoid problems with empty input buffer
404 if (count == 0) return String.Empty;
406 fixed (byte* pBytes = bytes)
407 return String.CreateStringFromEncoding(
408 pBytes + index, count, this);
412 // End of standard methods copied from EncodingNLS.cs
415 // To simplify maintenance, the structure of GetByteCount and GetBytes should be
416 // kept the same as much as possible
417 internal override unsafe int GetByteCount(char* chars, int count, EncoderNLS baseEncoder)
419 // For fallback we may need a fallback buffer.
420 // We wait to initialize it though in case we don't have any broken input unicode
421 EncoderFallbackBuffer fallbackBuffer = null;
422 char* pSrcForFallback;
425 char* pEnd = pSrc + count;
427 // Start by assuming we have as many as count
428 int byteCount = count;
432 if (baseEncoder != null)
434 UTF8Encoder encoder = (UTF8Encoder)baseEncoder;
435 ch = encoder.surrogateChar;
437 // We mustn't have left over fallback data when counting
438 if (encoder.InternalHasFallbackBuffer)
440 fallbackBuffer = encoder.FallbackBuffer;
441 if (fallbackBuffer.Remaining > 0)
442 throw new ArgumentException(SR.Format(SR.Argument_EncoderFallbackNotEmpty, this.EncodingName, encoder.Fallback.GetType()));
444 // Set our internal fallback interesting things.
445 fallbackBuffer.InternalInitialize(chars, pEnd, encoder, false);
451 // SLOWLOOP: does all range checks, handles all special cases, but it is slow
456 // Unroll any fallback that happens at the end
457 ch = fallbackBuffer != null ? fallbackBuffer.InternalGetNextChar() : 0;
466 // Case of surrogates in the fallback.
467 if (fallbackBuffer != null && fallbackBuffer.bFallingBack)
469 Debug.Assert(ch >= 0xD800 && ch <= 0xDBFF,
470 "[UTF8Encoding.GetBytes]expected high surrogate, not 0x" + ((int)ch).ToString("X4", CultureInfo.InvariantCulture));
472 ch = fallbackBuffer.InternalGetNextChar();
475 if (InRange(ch, CharUnicodeInfo.LOW_SURROGATE_START, CharUnicodeInfo.LOW_SURROGATE_END))
487 byteCount--; // ignore last one.
497 if (baseEncoder != null && !baseEncoder.MustFlush)
502 // attempt to encode the partial surrogate (will fallback or ignore it), it'll also subtract 1.
509 Debug.Assert(ch >= 0xD800 && ch <= 0xDBFF,
510 "[UTF8Encoding.GetBytes]expected high surrogate, not 0x" + ((int)ch).ToString("X4", CultureInfo.InvariantCulture));
512 // use separate helper variables for local contexts so that the jit optimizations
513 // won't get confused about the variable lifetimes
516 // count the pending surrogate
519 // In previous byte, we encountered a high surrogate, so we are expecting a low surrogate here.
520 // if (IsLowSurrogate(cha)) {
521 if (InRange(cha, CharUnicodeInfo.LOW_SURROGATE_START, CharUnicodeInfo.LOW_SURROGATE_END))
523 // Don't need a real # because we're just counting, anything > 0x7ff ('cept surrogate) will do.
525 // ch = cha + (ch << 10) +
527 // - CharUnicodeInfo.LOW_SURROGATE_START
528 // - (CharUnicodeInfo.HIGH_SURROGATE_START << 10) );
530 // Use this next char
533 // else ch is still high surrogate and encoding will fail (so don't add count)
535 // attempt to encode the surrogate or partial surrogate
539 // If we've used a fallback, then we have to check for it
540 if (fallbackBuffer != null)
542 ch = fallbackBuffer.InternalGetNextChar();
545 // We have an extra byte we weren't expecting.
551 // read next char. The JIT optimization seems to be getting confused when
552 // compiling "ch = *pSrc++;", so rather use "ch = *pSrc; pSrc++;" instead
557 // if (IsHighSurrogate(ch)) {
558 if (InRange(ch, CharUnicodeInfo.HIGH_SURROGATE_START, CharUnicodeInfo.HIGH_SURROGATE_END))
560 // we will count this surrogate next time around
564 // either good char or partial surrogate
567 // throw exception on partial surrogate if necessary
568 // if (IsLowSurrogate(ch) || IsHighSurrogate(ch))
569 if (InRange(ch, CharUnicodeInfo.HIGH_SURROGATE_START, CharUnicodeInfo.LOW_SURROGATE_END))
571 // Lone surrogates aren't allowed
572 // Have to make a fallback buffer if we don't have one
573 if (fallbackBuffer == null)
575 // wait on fallbacks if we can
576 // For fallback we may need a fallback buffer
577 if (baseEncoder == null)
578 fallbackBuffer = this.encoderFallback.CreateFallbackBuffer();
580 fallbackBuffer = baseEncoder.FallbackBuffer;
582 // Set our internal fallback interesting things.
583 fallbackBuffer.InternalInitialize(chars, chars + count, baseEncoder, false);
586 // Do our fallback. Actually we already know its a mixed up surrogate,
587 // so the ref pSrc isn't gonna do anything.
588 pSrcForFallback = pSrc; // Avoid passing pSrc by reference to allow it to be en-registered
589 fallbackBuffer.InternalFallback(unchecked((char)ch), ref pSrcForFallback);
590 pSrc = pSrcForFallback;
592 // Ignore it if we don't throw (we had preallocated this ch)
603 // the extra surrogate byte was compensated by the second surrogate character
604 // (2 surrogates make 4 bytes. We've already counted 2 bytes, 1 per char)
611 // check for overflow
619 // If still have fallback don't do fast loop
620 if (fallbackBuffer != null && (ch = fallbackBuffer.InternalGetNextChar()) != 0)
622 // We're reserving 1 byte for each char by default
627 int availableChars = PtrDiff(pEnd, pSrc);
629 // don't fall into the fast decoding loop if we don't have enough characters
630 if (availableChars <= 13)
632 // try to get over the remainder of the ascii characters fast though
633 char* pLocalEnd = pEnd; // hint to get pLocalEnd en-registered
634 while (pSrc < pLocalEnd)
647 // make sure that we won't get a silent overflow inside the fast loop
648 // (Fall out to slow loop if we have this many characters)
649 availableChars &= 0x0FFFFFFF;
652 // To compute the upper bound, assume that all characters are ASCII characters at this point,
653 // the boundary will be decreased for every non-ASCII character we encounter
654 // Also, we need 3 + 4 chars reserve for the unrolled ansi decoding loop and for decoding of surrogates
655 char* pStop = pSrc + availableChars - (3 + 4);
662 if (ch > 0x7F) // Not ASCII
664 if (ch > 0x7FF) // Not 2 Byte
666 if ((ch & 0xF800) == 0xD800) // See if its a Surrogate
674 if ((unchecked((int)pSrc) & 0x2) != 0)
678 if (ch > 0x7F) // Not ASCII
680 if (ch > 0x7FF) // Not 2 Byte
682 if ((ch & 0xF800) == 0xD800) // See if its a Surrogate
690 // Run 2 * 4 characters at a time!
694 int chc = *(int*)(pSrc + 2);
695 if (((ch | chc) & unchecked((int)0xFF80FF80)) != 0) // See if not ASCII
697 if (((ch | chc) & unchecked((int)0xF800F800)) != 0) // See if not 2 Byte
699 goto LongCodeWithMask;
703 if ((ch & unchecked((int)0xFF800000)) != 0) // Actually 0x07800780 is all we care about (4 bits)
705 if ((ch & unchecked((int)0xFF80)) != 0)
707 if ((chc & unchecked((int)0xFF800000)) != 0)
709 if ((chc & unchecked((int)0xFF80)) != 0)
715 chc = *(int*)(pSrc + 2);
716 if (((ch | chc) & unchecked((int)0xFF80FF80)) != 0) // See if not ASCII
718 if (((ch | chc) & unchecked((int)0xF800F800)) != 0) // See if not 2 Byte
720 goto LongCodeWithMask;
723 if ((ch & unchecked((int)0xFF800000)) != 0)
725 if ((ch & unchecked((int)0xFF80)) != 0)
727 if ((chc & unchecked((int)0xFF800000)) != 0)
729 if ((chc & unchecked((int)0xFF80)) != 0)
738 // be careful about the sign extension
739 ch = (int)(((uint)ch) >> 16);
751 // use separate helper variables for slow and fast loop so that the jit optimizations
752 // won't get confused about the variable lifetimes
755 // if (IsLowSurrogate(ch) || IsHighSurrogate(ch))
756 if (InRange(ch, CharUnicodeInfo.HIGH_SURROGATE_START, CharUnicodeInfo.LOW_SURROGATE_END))
758 // 4 byte encoding - high surrogate + low surrogate
762 // !IsHighSurrogate(ch) // low without high -> bad
763 ch > CharUnicodeInfo.HIGH_SURROGATE_END ||
764 // !IsLowSurrogate(chd) // high not followed by low -> bad
765 !InRange(chd, CharUnicodeInfo.LOW_SURROGATE_START, CharUnicodeInfo.LOW_SURROGATE_END))
767 // Back up and drop out to slow loop to figure out error
773 // byteCount - this byte is compensated by the second surrogate character
779 // byteCount - the last byte is already included
783 // no pending char at this point
788 // check for overflow
791 throw new ArgumentException(
792 SR.Argument_ConversionOverflow);
796 Debug.Assert(fallbackBuffer == null || fallbackBuffer.Remaining == 0,
797 "[UTF8Encoding.GetByteCount]Expected Empty fallback buffer");
802 // diffs two char pointers using unsigned arithmetic. The unsigned arithmetic
803 // is good enough for us, and it tends to generate better code than the signed
804 // arithmetic generated by default
805 unsafe private static int PtrDiff(char* a, char* b)
807 return (int)(((uint)((byte*)a - (byte*)b)) >> 1);
810 // byte* flavor just for parity
811 unsafe private static int PtrDiff(byte* a, byte* b)
816 private static bool InRange(int ch, int start, int end)
818 return (uint)(ch - start) <= (uint)(end - start);
822 // Note: We ignore mismatched surrogates, unless the exception flag is set in which case we throw
823 internal override unsafe int GetBytes(char* chars, int charCount,
824 byte* bytes, int byteCount, EncoderNLS baseEncoder)
826 Debug.Assert(chars != null, "[UTF8Encoding.GetBytes]chars!=null");
827 Debug.Assert(byteCount >= 0, "[UTF8Encoding.GetBytes]byteCount >=0");
828 Debug.Assert(charCount >= 0, "[UTF8Encoding.GetBytes]charCount >=0");
829 Debug.Assert(bytes != null, "[UTF8Encoding.GetBytes]bytes!=null");
831 UTF8Encoder encoder = null;
833 // For fallback we may need a fallback buffer.
834 // We wait to initialize it though in case we don't have any broken input unicode
835 EncoderFallbackBuffer fallbackBuffer = null;
836 char* pSrcForFallback;
839 byte* pTarget = bytes;
841 char* pEnd = pSrc + charCount;
842 byte* pAllocatedBufferEnd = pTarget + byteCount;
846 // assume that JIT will en-register pSrc, pTarget and ch
848 if (baseEncoder != null)
850 encoder = (UTF8Encoder)baseEncoder;
851 ch = encoder.surrogateChar;
853 // We mustn't have left over fallback data when counting
854 if (encoder.InternalHasFallbackBuffer)
856 // We always need the fallback buffer in get bytes so we can flush any remaining ones if necessary
857 fallbackBuffer = encoder.FallbackBuffer;
858 if (fallbackBuffer.Remaining > 0 && encoder._throwOnOverflow)
859 throw new ArgumentException(SR.Format(SR.Argument_EncoderFallbackNotEmpty, this.EncodingName, encoder.Fallback.GetType()));
861 // Set our internal fallback interesting things.
862 fallbackBuffer.InternalInitialize(chars, pEnd, encoder, true);
868 // SLOWLOOP: does all range checks, handles all special cases, but it is slow
874 // Check if there's anthing left to get out of the fallback buffer
875 ch = fallbackBuffer != null ? fallbackBuffer.InternalGetNextChar() : 0;
883 // Case of leftover surrogates in the fallback buffer
884 if (fallbackBuffer != null && fallbackBuffer.bFallingBack)
886 Debug.Assert(ch >= 0xD800 && ch <= 0xDBFF,
887 "[UTF8Encoding.GetBytes]expected high surrogate, not 0x" + ((int)ch).ToString("X4", CultureInfo.InvariantCulture));
891 ch = fallbackBuffer.InternalGetNextChar();
893 if (InRange(ch, CharUnicodeInfo.LOW_SURROGATE_START, CharUnicodeInfo.LOW_SURROGATE_END))
895 ch = ch + (cha << 10) + (0x10000 - CharUnicodeInfo.LOW_SURROGATE_START - (CharUnicodeInfo.HIGH_SURROGATE_START << 10));
909 // attempt to encode the partial surrogate (will fail or ignore)
910 if (ch > 0 && (encoder == null || encoder.MustFlush))
919 // We have a high surrogate left over from a previous loop.
920 Debug.Assert(ch >= 0xD800 && ch <= 0xDBFF,
921 "[UTF8Encoding.GetBytes]expected high surrogate, not 0x" + ((int)ch).ToString("X4", CultureInfo.InvariantCulture));
923 // use separate helper variables for local contexts so that the jit optimizations
924 // won't get confused about the variable lifetimes
927 // In previous byte, we encountered a high surrogate, so we are expecting a low surrogate here.
928 // if (IsLowSurrogate(cha)) {
929 if (InRange(cha, CharUnicodeInfo.LOW_SURROGATE_START, CharUnicodeInfo.LOW_SURROGATE_END))
931 ch = cha + (ch << 10) +
933 - CharUnicodeInfo.LOW_SURROGATE_START
934 - (CharUnicodeInfo.HIGH_SURROGATE_START << 10));
938 // else ch is still high surrogate and encoding will fail
940 // attempt to encode the surrogate or partial surrogate
944 // If we've used a fallback, then we have to check for it
945 if (fallbackBuffer != null)
947 ch = fallbackBuffer.InternalGetNextChar();
948 if (ch > 0) goto ProcessChar;
951 // read next char. The JIT optimization seems to be getting confused when
952 // compiling "ch = *pSrc++;", so rather use "ch = *pSrc; pSrc++;" instead
957 // if (IsHighSurrogate(ch)) {
958 if (InRange(ch, CharUnicodeInfo.HIGH_SURROGATE_START, CharUnicodeInfo.HIGH_SURROGATE_END))
962 // either good char or partial surrogate
965 // throw exception on partial surrogate if necessary
966 // if (IsLowSurrogate(ch) || IsHighSurrogate(ch))
967 if (InRange(ch, CharUnicodeInfo.HIGH_SURROGATE_START, CharUnicodeInfo.LOW_SURROGATE_END))
969 // Lone surrogates aren't allowed, we have to do fallback for them
970 // Have to make a fallback buffer if we don't have one
971 if (fallbackBuffer == null)
973 // wait on fallbacks if we can
974 // For fallback we may need a fallback buffer
975 if (baseEncoder == null)
976 fallbackBuffer = this.encoderFallback.CreateFallbackBuffer();
978 fallbackBuffer = baseEncoder.FallbackBuffer;
980 // Set our internal fallback interesting things.
981 fallbackBuffer.InternalInitialize(chars, pEnd, baseEncoder, true);
984 // Do our fallback. Actually we already know its a mixed up surrogate,
985 // so the ref pSrc isn't gonna do anything.
986 pSrcForFallback = pSrc; // Avoid passing pSrc by reference to allow it to be en-registered
987 fallbackBuffer.InternalFallback(unchecked((char)ch), ref pSrcForFallback);
988 pSrc = pSrcForFallback;
990 // Ignore it if we don't throw
995 // Count bytes needed
1003 bytesNeeded++; // 4 bytes (surrogate pair)
1005 bytesNeeded++; // 3 bytes (800-FFFF)
1007 bytesNeeded++; // 2 bytes (80-7FF)
1010 if (pTarget > pAllocatedBufferEnd - bytesNeeded)
1012 // Left over surrogate from last time will cause pSrc == chars, so we'll throw
1013 if (fallbackBuffer != null && fallbackBuffer.bFallingBack)
1015 fallbackBuffer.MovePrevious(); // Didn't use this fallback char
1017 fallbackBuffer.MovePrevious(); // Was surrogate, didn't use 2nd part either
1021 pSrc--; // Didn't use this char
1023 pSrc--; // Was surrogate, didn't use 2nd part either
1025 Debug.Assert(pSrc >= chars || pTarget == bytes,
1026 "[UTF8Encoding.GetBytes]Expected pSrc to be within buffer or to throw with insufficient room.");
1027 ThrowBytesOverflow(encoder, pTarget == bytes); // Throw if we must
1028 ch = 0; // Nothing left over (we backed up to start of pair if supplementary)
1034 *pTarget = (byte)ch;
1038 // use separate helper variables for local contexts so that the jit optimizations
1039 // won't get confused about the variable lifetimes
1044 chb = (byte)(unchecked((sbyte)0xC0) | (ch >> 6));
1050 chb = (byte)(unchecked((sbyte)0xE0) | (ch >> 12));
1054 *pTarget = (byte)(unchecked((sbyte)0xF0) | (ch >> 18));
1057 chb = unchecked((sbyte)0x80) | (ch >> 12) & 0x3F;
1059 *pTarget = (byte)chb;
1062 chb = unchecked((sbyte)0x80) | (ch >> 6) & 0x3F;
1064 *pTarget = (byte)chb;
1067 *pTarget = (byte)(unchecked((sbyte)0x80) | ch & 0x3F);
1073 // If still have fallback don't do fast loop
1074 if (fallbackBuffer != null && (ch = fallbackBuffer.InternalGetNextChar()) != 0)
1077 int availableChars = PtrDiff(pEnd, pSrc);
1078 int availableBytes = PtrDiff(pAllocatedBufferEnd, pTarget);
1080 // don't fall into the fast decoding loop if we don't have enough characters
1081 // Note that if we don't have enough bytes, pStop will prevent us from entering the fast loop.
1082 if (availableChars <= 13)
1084 // we are hoping for 1 byte per char
1085 if (availableBytes < availableChars)
1087 // not enough output room. no pending bits at this point
1092 // try to get over the remainder of the ascii characters fast though
1093 char* pLocalEnd = pEnd; // hint to get pLocalEnd en-registered
1094 while (pSrc < pLocalEnd)
1099 // Not ASCII, need more than 1 byte per char
1103 *pTarget = (byte)ch;
1106 // we are done, let ch be 0 to clear encoder
1111 // we need at least 1 byte per character, but Convert might allow us to convert
1112 // only part of the input, so try as much as we can. Reduce charCount if necessary
1113 if (availableBytes < availableChars)
1115 availableChars = availableBytes;
1119 // - optimistic range checks
1120 // - fallbacks to the slow loop for all special cases, exception throwing, etc.
1122 // To compute the upper bound, assume that all characters are ASCII characters at this point,
1123 // the boundary will be decreased for every non-ASCII character we encounter
1124 // Also, we need 5 chars reserve for the unrolled ansi decoding loop and for decoding of surrogates
1125 // If there aren't enough bytes for the output, then pStop will be <= pSrc and will bypass the loop.
1126 char* pStop = pSrc + availableChars - 5;
1128 while (pSrc < pStop)
1137 *pTarget = (byte)ch;
1141 if ((unchecked((int)pSrc) & 0x2) != 0)
1149 *pTarget = (byte)ch;
1153 // Run 4 characters at a time!
1154 while (pSrc < pStop)
1157 int chc = *(int*)(pSrc + 2);
1158 if (((ch | chc) & unchecked((int)0xFF80FF80)) != 0)
1160 goto LongCodeWithMask;
1163 // Unfortunately, this is endianess sensitive
1165 *pTarget = (byte)(ch>>16);
1166 *(pTarget+1) = (byte)ch;
1168 *(pTarget+2) = (byte)(chc>>16);
1169 *(pTarget+3) = (byte)chc;
1172 *pTarget = (byte)ch;
1173 *(pTarget + 1) = (byte)(ch >> 16);
1175 *(pTarget + 2) = (byte)chc;
1176 *(pTarget + 3) = (byte)(chc >> 16);
1184 // be careful about the sign extension
1185 ch = (int)(((uint)ch) >> 16);
1195 *pTarget = (byte)ch;
1200 // use separate helper variables for slow and fast loop so that the jit optimizations
1201 // won't get confused about the variable lifetimes
1206 chd = unchecked((sbyte)0xC0) | (ch >> 6);
1210 // if (!IsLowSurrogate(ch) && !IsHighSurrogate(ch))
1211 if (!InRange(ch, CharUnicodeInfo.HIGH_SURROGATE_START, CharUnicodeInfo.LOW_SURROGATE_END))
1214 chd = unchecked((sbyte)0xE0) | (ch >> 12);
1218 // 4 byte encoding - high surrogate + low surrogate
1219 // if (!IsHighSurrogate(ch))
1220 if (ch > CharUnicodeInfo.HIGH_SURROGATE_END)
1222 // low without high -> bad, try again in slow loop
1230 // if (!IsLowSurrogate(chd)) {
1231 if (!InRange(chd, CharUnicodeInfo.LOW_SURROGATE_START, CharUnicodeInfo.LOW_SURROGATE_END))
1233 // high not followed by low -> bad, try again in slow loop
1238 ch = chd + (ch << 10) +
1240 - CharUnicodeInfo.LOW_SURROGATE_START
1241 - (CharUnicodeInfo.HIGH_SURROGATE_START << 10));
1243 *pTarget = (byte)(unchecked((sbyte)0xF0) | (ch >> 18));
1244 // pStop - this byte is compensated by the second surrogate character
1245 // 2 input chars require 4 output bytes. 2 have been anticipated already
1246 // and 2 more will be accounted for by the 2 pStop-- calls below.
1249 chd = unchecked((sbyte)0x80) | (ch >> 12) & 0x3F;
1251 *pTarget = (byte)chd;
1252 pStop--; // 3 byte sequence for 1 char, so need pStop-- and the one below too.
1255 chd = unchecked((sbyte)0x80) | (ch >> 6) & 0x3F;
1257 *pTarget = (byte)chd;
1258 pStop--; // 2 byte sequence for 1 char so need pStop--.
1261 *pTarget = (byte)(unchecked((sbyte)0x80) | ch & 0x3F);
1262 // pStop - this byte is already included
1266 Debug.Assert(pTarget <= pAllocatedBufferEnd, "[UTF8Encoding.GetBytes]pTarget <= pAllocatedBufferEnd");
1270 // no pending char at this point
1274 // Do we have to set the encoder bytes?
1275 if (encoder != null)
1277 Debug.Assert(!encoder.MustFlush || ch == 0,
1278 "[UTF8Encoding.GetBytes] Expected no mustflush or 0 leftover ch " + ch.ToString("X2", CultureInfo.InvariantCulture));
1280 encoder.surrogateChar = ch;
1281 encoder._charsUsed = (int)(pSrc - chars);
1284 Debug.Assert(fallbackBuffer == null || fallbackBuffer.Remaining == 0 ||
1285 baseEncoder == null || !baseEncoder._throwOnOverflow,
1286 "[UTF8Encoding.GetBytes]Expected empty fallback buffer if not converting");
1288 return (int)(pTarget - bytes);
1292 // These are bitmasks used to maintain the state in the decoder. They occupy the higher bits
1293 // while the actual character is being built in the lower bits. They are shifted together
1294 // with the actual bits of the character.
1296 // bits 30 & 31 are used for pending bits fixup
1297 private const int FinalByte = 1 << 29;
1298 private const int SupplimentarySeq = 1 << 28;
1299 private const int ThreeByteSeq = 1 << 27;
1301 // Note: We throw exceptions on individually encoded surrogates and other non-shortest forms.
1302 // If exceptions aren't turned on, then we drop all non-shortest &individual surrogates.
1304 // To simplify maintenance, the structure of GetCharCount and GetChars should be
1305 // kept the same as much as possible
1306 internal override unsafe int GetCharCount(byte* bytes, int count, DecoderNLS baseDecoder)
1308 Debug.Assert(count >= 0, "[UTF8Encoding.GetCharCount]count >=0");
1309 Debug.Assert(bytes != null, "[UTF8Encoding.GetCharCount]bytes!=null");
1313 byte* pEnd = pSrc + count;
1315 // Start by assuming we have as many as count, charCount always includes the adjustment
1316 // for the character being decoded
1317 int charCount = count;
1319 DecoderFallbackBuffer fallback = null;
1321 if (baseDecoder != null)
1323 UTF8Decoder decoder = (UTF8Decoder)baseDecoder;
1325 charCount -= (ch >> 30); // Adjust char count for # of expected bytes and expected output chars.
1327 // Shouldn't have anything in fallback buffer for GetCharCount
1328 // (don't have to check _throwOnOverflow for count)
1329 Debug.Assert(!decoder.InternalHasFallbackBuffer || decoder.FallbackBuffer.Remaining == 0,
1330 "[UTF8Encoding.GetCharCount]Expected empty fallback buffer at start");
1335 // SLOWLOOP: does all range checks, handles all special cases, but it is slow
1348 // read next byte. The JIT optimization seems to be getting confused when
1349 // compiling "ch = *pSrc++;", so rather use "ch = *pSrc; pSrc++;" instead
1353 // we are expecting to see trailing bytes like 10vvvvvv
1354 if ((cha & unchecked((sbyte)0xC0)) != 0x80)
1356 // This can be a valid starting byte for another UTF8 byte sequence, so let's put
1357 // the current byte back, and try to see if this is a valid byte for another UTF8 byte sequence
1359 charCount += (ch >> 30);
1360 goto InvalidByteSequence;
1363 // fold in the new byte
1364 ch = (ch << 6) | (cha & 0x3F);
1366 if ((ch & FinalByte) == 0)
1368 Debug.Assert((ch & (SupplimentarySeq | ThreeByteSeq)) != 0,
1369 "[UTF8Encoding.GetChars]Invariant volation");
1371 if ((ch & SupplimentarySeq) != 0)
1373 if ((ch & (FinalByte >> 6)) != 0)
1375 // this is 3rd byte (of 4 byte supplementary) - nothing to do
1379 // 2nd byte, check for non-shortest form of supplementary char and the valid
1380 // supplementary characters in range 0x010000 - 0x10FFFF at the same time
1381 if (!InRange(ch & 0x1F0, 0x10, 0x100))
1383 goto InvalidByteSequence;
1388 // Must be 2nd byte of a 3-byte sequence
1389 // check for non-shortest form of 3 byte seq
1390 if ((ch & (0x1F << 5)) == 0 || // non-shortest form
1391 (ch & (0xF800 >> 6)) == (0xD800 >> 6)) // illegal individually encoded surrogate
1393 goto InvalidByteSequence;
1401 // adjust for surrogates in non-shortest form
1402 if ((ch & (SupplimentarySeq | 0x1F0000)) == SupplimentarySeq)
1408 InvalidByteSequence:
1409 // this code fragment should be close to the goto referencing it
1410 // Have to do fallback for invalid bytes
1411 if (fallback == null)
1413 if (baseDecoder == null)
1414 fallback = this.decoderFallback.CreateFallbackBuffer();
1416 fallback = baseDecoder.FallbackBuffer;
1417 fallback.InternalInitialize(bytes, null);
1419 charCount += FallbackInvalidByteSequence(pSrc, ch, fallback);
1431 // If its > 0x7F, its start of a new multi-byte sequence
1433 // Long sequence, so unreserve our char.
1436 // bit 6 has to be non-zero for start of multibyte chars.
1437 if ((ch & 0x40) == 0)
1439 // Unexpected trail byte
1440 goto InvalidByteSequence;
1443 // start a new long code
1444 if ((ch & 0x20) != 0)
1446 if ((ch & 0x10) != 0)
1448 // 4 byte encoding - supplimentary character (2 surrogates)
1452 // check that bit 4 is zero and the valid supplimentary character
1453 // range 0x000000 - 0x10FFFF at the same time
1457 goto InvalidByteSequence;
1460 // Add bit flags so that when we check new characters & rotate we'll be flagged correctly.
1461 // Final byte flag, count fix if we don't make final byte & supplimentary sequence flag.
1462 ch |= (FinalByte >> 3 * 6) | // Final byte is 3 more bytes from now
1463 (1 << 30) | // If it dies on next byte we'll need an extra char
1464 (3 << (30 - 2 * 6)) | // If it dies on last byte we'll need to subtract a char
1465 (SupplimentarySeq) | (SupplimentarySeq >> 6) |
1466 (SupplimentarySeq >> 2 * 6) | (SupplimentarySeq >> 3 * 6);
1468 // Our character count will be 2 characters for these 4 bytes, so subtract another char
1474 // Add bit flags so that when we check new characters & rotate we'll be flagged correctly.
1475 ch = (ch & 0x0F) | ((FinalByte >> 2 * 6) | (1 << 30) |
1476 (ThreeByteSeq) | (ThreeByteSeq >> 6) | (ThreeByteSeq >> 2 * 6));
1478 // We'll expect 1 character for these 3 bytes, so subtract another char.
1488 // check for non-shortest form
1492 goto InvalidByteSequence;
1495 // Add bit flags so we'll be flagged correctly
1496 ch |= (FinalByte >> 6);
1504 int availableBytes = PtrDiff(pEnd, pSrc);
1506 // don't fall into the fast decoding loop if we don't have enough bytes
1507 if (availableBytes <= 13)
1509 // try to get over the remainder of the ascii characters fast though
1510 byte* pLocalEnd = pEnd; // hint to get pLocalEnd en-registered
1511 while (pSrc < pLocalEnd)
1524 // To compute the upper bound, assume that all characters are ASCII characters at this point,
1525 // the boundary will be decreased for every non-ASCII character we encounter
1526 // Also, we need 7 chars reserve for the unrolled ansi decoding loop and for decoding of multibyte sequences
1527 byte* pStop = pSrc + availableBytes - 7;
1529 while (pSrc < pStop)
1539 // get pSrc 2-byte aligned
1540 if ((unchecked((int)pSrc) & 0x1) != 0)
1550 // get pSrc 4-byte aligned
1551 if ((unchecked((int)pSrc) & 0x2) != 0)
1553 ch = *(ushort*)pSrc;
1554 if ((ch & 0x8080) != 0)
1556 goto LongCodeWithMask16;
1561 // Run 8 + 8 characters at a time!
1562 while (pSrc < pStop)
1565 int chb = *(int*)(pSrc + 4);
1566 if (((ch | chb) & unchecked((int)0x80808080)) != 0)
1568 goto LongCodeWithMask32;
1572 // This is a really small loop - unroll it
1577 chb = *(int*)(pSrc + 4);
1578 if (((ch | chb) & unchecked((int)0x80808080)) != 0)
1580 goto LongCodeWithMask32;
1588 // be careful about the sign extension
1589 ch = (int)(((uint)ch) >> 16);
1591 ch = (int)(((uint)ch) >> 8);
1608 // bit 6 has to be zero
1610 // we are expecting to see trailing bytes like 10vvvvvv
1611 (chc & unchecked((sbyte)0xC0)) != 0x80)
1618 // start a new long code
1619 if ((ch & 0x20) != 0)
1621 // fold the first two bytes together
1622 chc |= (ch & 0x0F) << 6;
1624 if ((ch & 0x10) != 0)
1626 // 4 byte encoding - surrogate
1629 // check that bit 4 is zero, the non-shortest form of surrogate
1630 // and the valid surrogate range 0x000000 - 0x10FFFF at the same time
1631 !InRange(chc >> 4, 0x01, 0x10) ||
1632 // we are expecting to see trailing bytes like 10vvvvvv
1633 (ch & unchecked((sbyte)0xC0)) != 0x80)
1638 chc = (chc << 6) | (ch & 0x3F);
1641 // we are expecting to see trailing bytes like 10vvvvvv
1642 if ((ch & unchecked((sbyte)0xC0)) != 0x80)
1656 // check for non-shortest form of 3 byte seq
1657 (chc & (0x1F << 5)) == 0 ||
1658 // Can't have surrogates here.
1659 (chc & (0xF800 >> 6)) == (0xD800 >> 6) ||
1660 // we are expecting to see trailing bytes like 10vvvvvv
1661 (ch & unchecked((sbyte)0xC0)) != 0x80)
1675 // check for non-shortest form
1676 if ((ch & 0x1E) == 0)
1687 // no pending bits at this point
1697 // May have a problem if we have to flush
1700 // We were already adjusting for these, so need to un-adjust
1701 charCount += (ch >> 30);
1702 if (baseDecoder == null || baseDecoder.MustFlush)
1704 // Have to do fallback for invalid bytes
1705 if (fallback == null)
1707 if (baseDecoder == null)
1708 fallback = this.decoderFallback.CreateFallbackBuffer();
1710 fallback = baseDecoder.FallbackBuffer;
1711 fallback.InternalInitialize(bytes, null);
1713 charCount += FallbackInvalidByteSequence(pSrc, ch, fallback);
1717 // Shouldn't have anything in fallback buffer for GetCharCount
1718 // (don't have to check _throwOnOverflow for count)
1719 Debug.Assert(fallback == null || fallback.Remaining == 0,
1720 "[UTF8Encoding.GetCharCount]Expected empty fallback buffer at end");
1725 // WARNING: If we throw an error, then System.Resources.ResourceReader calls this method.
1726 // So if we're really broken, then that could also throw an error... recursively.
1727 // So try to make sure GetChars can at least process all uses by
1728 // System.Resources.ResourceReader!
1730 // Note: We throw exceptions on individually encoded surrogates and other non-shortest forms.
1731 // If exceptions aren't turned on, then we drop all non-shortest &individual surrogates.
1733 // To simplify maintenance, the structure of GetCharCount and GetChars should be
1734 // kept the same as much as possible
1735 internal override unsafe int GetChars(byte* bytes, int byteCount,
1736 char* chars, int charCount, DecoderNLS baseDecoder)
1738 Debug.Assert(chars != null, "[UTF8Encoding.GetChars]chars!=null");
1739 Debug.Assert(byteCount >= 0, "[UTF8Encoding.GetChars]count >=0");
1740 Debug.Assert(charCount >= 0, "[UTF8Encoding.GetChars]charCount >=0");
1741 Debug.Assert(bytes != null, "[UTF8Encoding.GetChars]bytes!=null");
1744 char* pTarget = chars;
1746 byte* pEnd = pSrc + byteCount;
1747 char* pAllocatedBufferEnd = pTarget + charCount;
1751 DecoderFallbackBuffer fallback = null;
1752 byte* pSrcForFallback;
1753 char* pTargetForFallback;
1754 if (baseDecoder != null)
1756 UTF8Decoder decoder = (UTF8Decoder)baseDecoder;
1759 // Shouldn't have anything in fallback buffer for GetChars
1760 // (don't have to check _throwOnOverflow for chars, we always use all or none so always should be empty)
1761 Debug.Assert(!decoder.InternalHasFallbackBuffer || decoder.FallbackBuffer.Remaining == 0,
1762 "[UTF8Encoding.GetChars]Expected empty fallback buffer at start");
1767 // SLOWLOOP: does all range checks, handles all special cases, but it is slow
1780 // read next byte. The JIT optimization seems to be getting confused when
1781 // compiling "ch = *pSrc++;", so rather use "ch = *pSrc; pSrc++;" instead
1785 // we are expecting to see trailing bytes like 10vvvvvv
1786 if ((cha & unchecked((sbyte)0xC0)) != 0x80)
1788 // This can be a valid starting byte for another UTF8 byte sequence, so let's put
1789 // the current byte back, and try to see if this is a valid byte for another UTF8 byte sequence
1791 goto InvalidByteSequence;
1794 // fold in the new byte
1795 ch = (ch << 6) | (cha & 0x3F);
1797 if ((ch & FinalByte) == 0)
1799 // Not at last byte yet
1800 Debug.Assert((ch & (SupplimentarySeq | ThreeByteSeq)) != 0,
1801 "[UTF8Encoding.GetChars]Invariant volation");
1803 if ((ch & SupplimentarySeq) != 0)
1805 // Its a 4-byte supplimentary sequence
1806 if ((ch & (FinalByte >> 6)) != 0)
1808 // this is 3rd byte of 4 byte sequence - nothing to do
1812 // 2nd byte of 4 bytes
1813 // check for non-shortest form of surrogate and the valid surrogate
1814 // range 0x000000 - 0x10FFFF at the same time
1815 if (!InRange(ch & 0x1F0, 0x10, 0x100))
1817 goto InvalidByteSequence;
1822 // Must be 2nd byte of a 3-byte sequence
1823 // check for non-shortest form of 3 byte seq
1824 if ((ch & (0x1F << 5)) == 0 || // non-shortest form
1825 (ch & (0xF800 >> 6)) == (0xD800 >> 6)) // illegal individually encoded surrogate
1827 goto InvalidByteSequence;
1835 // surrogate in shortest form?
1836 // Might be possible to get rid of this? Already did non-shortest check for 4-byte sequence when reading 2nd byte?
1837 if ((ch & (SupplimentarySeq | 0x1F0000)) > SupplimentarySeq)
1839 // let the range check for the second char throw the exception
1840 if (pTarget < pAllocatedBufferEnd)
1842 *pTarget = (char)(((ch >> 10) & 0x7FF) +
1843 unchecked((short)((CharUnicodeInfo.HIGH_SURROGATE_START - (0x10000 >> 10)))));
1847 unchecked((int)(CharUnicodeInfo.LOW_SURROGATE_START));
1853 InvalidByteSequence:
1854 // this code fragment should be close to the gotos referencing it
1855 // Have to do fallback for invalid bytes
1856 if (fallback == null)
1858 if (baseDecoder == null)
1859 fallback = this.decoderFallback.CreateFallbackBuffer();
1861 fallback = baseDecoder.FallbackBuffer;
1862 fallback.InternalInitialize(bytes, pAllocatedBufferEnd);
1864 // That'll back us up the appropriate # of bytes if we didn't get anywhere
1865 pSrcForFallback = pSrc; // Avoid passing pSrc by reference to allow it to be en-registered
1866 pTargetForFallback = pTarget; // Avoid passing pTarget by reference to allow it to be en-registered
1867 bool fallbackResult = FallbackInvalidByteSequence(ref pSrcForFallback, ch, fallback, ref pTargetForFallback);
1868 pSrc = pSrcForFallback;
1869 pTarget = pTargetForFallback;
1871 if (!fallbackResult)
1873 // Ran out of buffer space
1874 // Need to throw an exception?
1875 Debug.Assert(pSrc >= bytes || pTarget == chars,
1876 "[UTF8Encoding.GetChars]Expected to throw or remain in byte buffer after fallback");
1877 fallback.InternalReset();
1878 ThrowCharsOverflow(baseDecoder, pTarget == chars);
1882 Debug.Assert(pSrc >= bytes,
1883 "[UTF8Encoding.GetChars]Expected invalid byte sequence to have remained within the byte array");
1894 // If its > 0x7F, its start of a new multi-byte sequence
1896 // bit 6 has to be non-zero
1897 if ((ch & 0x40) == 0)
1899 goto InvalidByteSequence;
1902 // start a new long code
1903 if ((ch & 0x20) != 0)
1905 if ((ch & 0x10) != 0)
1907 // 4 byte encoding - supplimentary character (2 surrogates)
1911 // check that bit 4 is zero and the valid supplimentary character
1912 // range 0x000000 - 0x10FFFF at the same time
1916 goto InvalidByteSequence;
1919 ch |= (FinalByte >> 3 * 6) | (1 << 30) | (3 << (30 - 2 * 6)) |
1920 (SupplimentarySeq) | (SupplimentarySeq >> 6) |
1921 (SupplimentarySeq >> 2 * 6) | (SupplimentarySeq >> 3 * 6);
1926 ch = (ch & 0x0F) | ((FinalByte >> 2 * 6) | (1 << 30) |
1927 (ThreeByteSeq) | (ThreeByteSeq >> 6) | (ThreeByteSeq >> 2 * 6));
1936 // check for non-shortest form
1940 goto InvalidByteSequence;
1943 ch |= (FinalByte >> 6);
1949 // write the pending character
1950 if (pTarget >= pAllocatedBufferEnd)
1952 // Fix chars so we make sure to throw if we didn't output anything
1958 if (ch >= CharUnicodeInfo.LOW_SURROGATE_START &&
1959 ch <= CharUnicodeInfo.LOW_SURROGATE_END)
1961 pSrc--; // It was 4 bytes
1962 pTarget--; // 1 was stored already, but we can't remember 1/2, so back up
1964 else if (ch > 0xffff)
1966 pSrc--; // It was 4 bytes, nothing was stored
1968 pSrc--; // It was at least 3 bytes
1970 pSrc--; // It was at least 2 bytes
1974 // Throw that we don't have enough room (pSrc could be < chars if we had started to process
1975 // a 4 byte sequence already)
1976 Debug.Assert(pSrc >= bytes || pTarget == chars,
1977 "[UTF8Encoding.GetChars]Expected pSrc to be within input buffer or throw due to no output]");
1978 ThrowCharsOverflow(baseDecoder, pTarget == chars);
1980 // Don't store ch in decoder, we already backed up to its start
1983 // Didn't throw, just use this buffer size.
1986 *pTarget = (char)ch;
1990 int availableChars = PtrDiff(pAllocatedBufferEnd, pTarget);
1991 int availableBytes = PtrDiff(pEnd, pSrc);
1993 // don't fall into the fast decoding loop if we don't have enough bytes
1994 // Test for availableChars is done because pStop would be <= pTarget.
1995 if (availableBytes <= 13)
1997 // we may need as many as 1 character per byte
1998 if (availableChars < availableBytes)
2000 // not enough output room. no pending bits at this point
2005 // try to get over the remainder of the ascii characters fast though
2006 byte* pLocalEnd = pEnd; // hint to get pLocalEnd enregistered
2007 while (pSrc < pLocalEnd)
2015 *pTarget = (char)ch;
2023 // we may need as many as 1 character per byte, so reduce the byte count if necessary.
2024 // If availableChars is too small, pStop will be before pTarget and we won't do fast loop.
2025 if (availableChars < availableBytes)
2027 availableBytes = availableChars;
2030 // To compute the upper bound, assume that all characters are ASCII characters at this point,
2031 // the boundary will be decreased for every non-ASCII character we encounter
2032 // Also, we need 7 chars reserve for the unrolled ansi decoding loop and for decoding of multibyte sequences
2033 char* pStop = pTarget + availableBytes - 7;
2035 while (pTarget < pStop)
2044 *pTarget = (char)ch;
2047 // get pSrc to be 2-byte aligned
2048 if ((unchecked((int)pSrc) & 0x1) != 0)
2056 *pTarget = (char)ch;
2060 // get pSrc to be 4-byte aligned
2061 if ((unchecked((int)pSrc) & 0x2) != 0)
2063 ch = *(ushort*)pSrc;
2064 if ((ch & 0x8080) != 0)
2066 goto LongCodeWithMask16;
2069 // Unfortunately, this is endianess sensitive
2071 *pTarget = (char)((ch >> 8) & 0x7F);
2073 *(pTarget+1) = (char)(ch & 0x7F);
2076 *pTarget = (char)(ch & 0x7F);
2078 *(pTarget + 1) = (char)((ch >> 8) & 0x7F);
2083 // Run 8 characters at a time!
2084 while (pTarget < pStop)
2087 int chb = *(int*)(pSrc + 4);
2088 if (((ch | chb) & unchecked((int)0x80808080)) != 0)
2090 goto LongCodeWithMask32;
2093 // Unfortunately, this is endianess sensitive
2095 *pTarget = (char)((ch >> 24) & 0x7F);
2096 *(pTarget+1) = (char)((ch >> 16) & 0x7F);
2097 *(pTarget+2) = (char)((ch >> 8) & 0x7F);
2098 *(pTarget+3) = (char)(ch & 0x7F);
2100 *(pTarget+4) = (char)((chb >> 24) & 0x7F);
2101 *(pTarget+5) = (char)((chb >> 16) & 0x7F);
2102 *(pTarget+6) = (char)((chb >> 8) & 0x7F);
2103 *(pTarget+7) = (char)(chb & 0x7F);
2106 *pTarget = (char)(ch & 0x7F);
2107 *(pTarget + 1) = (char)((ch >> 8) & 0x7F);
2108 *(pTarget + 2) = (char)((ch >> 16) & 0x7F);
2109 *(pTarget + 3) = (char)((ch >> 24) & 0x7F);
2111 *(pTarget + 4) = (char)(chb & 0x7F);
2112 *(pTarget + 5) = (char)((chb >> 8) & 0x7F);
2113 *(pTarget + 6) = (char)((chb >> 16) & 0x7F);
2114 *(pTarget + 7) = (char)((chb >> 24) & 0x7F);
2122 // be careful about the sign extension
2123 ch = (int)(((uint)ch) >> 16);
2125 ch = (int)(((uint)ch) >> 8);
2134 *pTarget = (char)ch;
2144 // bit 6 has to be zero
2146 // we are expecting to see trailing bytes like 10vvvvvv
2147 (chc & unchecked((sbyte)0xC0)) != 0x80)
2154 // start a new long code
2155 if ((ch & 0x20) != 0)
2157 // fold the first two bytes together
2158 chc |= (ch & 0x0F) << 6;
2160 if ((ch & 0x10) != 0)
2162 // 4 byte encoding - surrogate
2165 // check that bit 4 is zero, the non-shortest form of surrogate
2166 // and the valid surrogate range 0x000000 - 0x10FFFF at the same time
2167 !InRange(chc >> 4, 0x01, 0x10) ||
2168 // we are expecting to see trailing bytes like 10vvvvvv
2169 (ch & unchecked((sbyte)0xC0)) != 0x80)
2174 chc = (chc << 6) | (ch & 0x3F);
2177 // we are expecting to see trailing bytes like 10vvvvvv
2178 if ((ch & unchecked((sbyte)0xC0)) != 0x80)
2184 ch = (chc << 6) | (ch & 0x3F);
2186 *pTarget = (char)(((ch >> 10) & 0x7FF) +
2187 unchecked((short)(CharUnicodeInfo.HIGH_SURROGATE_START - (0x10000 >> 10))));
2191 unchecked((short)(CharUnicodeInfo.LOW_SURROGATE_START));
2193 // extra byte, we're already planning 2 chars for 2 of these bytes,
2194 // but the big loop is testing the target against pStop, so we need
2195 // to subtract 2 more or we risk overrunning the input. Subtract
2196 // one here and one below.
2204 // check for non-shortest form of 3 byte seq
2205 (chc & (0x1F << 5)) == 0 ||
2206 // Can't have surrogates here.
2207 (chc & (0xF800 >> 6)) == (0xD800 >> 6) ||
2208 // we are expecting to see trailing bytes like 10vvvvvv
2209 (ch & unchecked((sbyte)0xC0)) != 0x80)
2215 ch = (chc << 6) | (ch & 0x3F);
2217 // extra byte, we're only expecting 1 char for each of these 3 bytes,
2218 // but the loop is testing the target (not source) against pStop, so
2219 // we need to subtract 2 more or we risk overrunning the input.
2220 // Subtract 1 here and one more below
2230 // check for non-shortest form
2235 ch = (ch << 6) | chc;
2238 *pTarget = (char)ch;
2241 // extra byte, we're only expecting 1 char for each of these 2 bytes,
2242 // but the loop is testing the target (not source) against pStop.
2243 // subtract an extra count from pStop so that we don't overrun the input.
2248 Debug.Assert(pTarget <= pAllocatedBufferEnd, "[UTF8Encoding.GetChars]pTarget <= pAllocatedBufferEnd");
2250 // no pending bits at this point
2260 if (ch != 0 && (baseDecoder == null || baseDecoder.MustFlush))
2262 // Have to do fallback for invalid bytes
2263 if (fallback == null)
2265 if (baseDecoder == null)
2266 fallback = this.decoderFallback.CreateFallbackBuffer();
2268 fallback = baseDecoder.FallbackBuffer;
2269 fallback.InternalInitialize(bytes, pAllocatedBufferEnd);
2272 // That'll back us up the appropriate # of bytes if we didn't get anywhere
2273 pSrcForFallback = pSrc; // Avoid passing pSrc by reference to allow it to be en-registered
2274 pTargetForFallback = pTarget; // Avoid passing pTarget by reference to allow it to be en-registered
2275 bool fallbackResult = FallbackInvalidByteSequence(ref pSrcForFallback, ch, fallback, ref pTargetForFallback);
2276 pSrc = pSrcForFallback;
2277 pTarget = pTargetForFallback;
2279 if (!fallbackResult)
2281 Debug.Assert(pSrc >= bytes || pTarget == chars,
2282 "[UTF8Encoding.GetChars]Expected to throw or remain in byte buffer while flushing");
2284 // Ran out of buffer space
2285 // Need to throw an exception?
2286 fallback.InternalReset();
2287 ThrowCharsOverflow(baseDecoder, pTarget == chars);
2289 Debug.Assert(pSrc >= bytes,
2290 "[UTF8Encoding.GetChars]Expected flushing invalid byte sequence to have remained within the byte array");
2294 if (baseDecoder != null)
2296 UTF8Decoder decoder = (UTF8Decoder)baseDecoder;
2298 // If we're storing flush data we expect all bits to be used or else
2299 // we're stuck in the middle of a conversion
2300 Debug.Assert(!baseDecoder.MustFlush || ch == 0 || !baseDecoder._throwOnOverflow,
2301 "[UTF8Encoding.GetChars]Expected no must flush or no left over bits or no throw on overflow.");
2303 // Remember our leftover bits.
2306 baseDecoder._bytesUsed = (int)(pSrc - bytes);
2309 // Shouldn't have anything in fallback buffer for GetChars
2310 // (don't have to check _throwOnOverflow for chars)
2311 Debug.Assert(fallback == null || fallback.Remaining == 0,
2312 "[UTF8Encoding.GetChars]Expected empty fallback buffer at end");
2314 return PtrDiff(pTarget, chars);
2317 // During GetChars we had an invalid byte sequence
2318 // pSrc is backed up to the start of the bad sequence if we didn't have room to
2319 // fall it back. Otherwise pSrc remains where it is.
2320 private unsafe bool FallbackInvalidByteSequence(
2321 ref byte* pSrc, int ch, DecoderFallbackBuffer fallback, ref char* pTarget)
2324 byte* pStart = pSrc;
2325 byte[] bytesUnknown = GetBytesUnknown(ref pStart, ch);
2327 // Do the actual fallback
2328 if (!fallback.InternalFallback(bytesUnknown, pSrc, ref pTarget))
2330 // Oops, it failed, back up to pStart
2339 // During GetCharCount we had an invalid byte sequence
2340 // pSrc is used to find the index that points to the invalid bytes,
2341 // however the byte[] contains the fallback bytes (in case the index is -1)
2342 private unsafe int FallbackInvalidByteSequence(
2343 byte* pSrc, int ch, DecoderFallbackBuffer fallback)
2346 byte[] bytesUnknown = GetBytesUnknown(ref pSrc, ch);
2348 // Do the actual fallback
2349 int count = fallback.InternalFallback(bytesUnknown, pSrc);
2351 // # of fallback chars expected.
2352 // Note that we only get here for "long" sequences, and have already unreserved
2353 // the count that we prereserved for the input bytes
2357 // Note that some of these bytes may have come from a previous fallback, so we cannot
2358 // just decrement the pointer and use the values we read. In those cases we have
2359 // to regenerate the original values.
2360 private unsafe byte[] GetBytesUnknown(ref byte* pSrc, int ch)
2363 byte[] bytesUnknown = null;
2365 // See if it was a plain char
2366 // (have to check >= 0 because we have all sorts of wierd bit flags)
2367 if (ch < 0x100 && ch >= 0)
2370 bytesUnknown = new byte[] { unchecked((byte)ch) };
2372 // See if its an unfinished 2 byte sequence
2373 else if ((ch & (SupplimentarySeq | ThreeByteSeq)) == 0)
2376 bytesUnknown = new byte[] { unchecked((byte)((ch & 0x1F) | 0xc0)) };
2378 // So now we're either 2nd byte of 3 or 4 byte sequence or
2379 // we hit a non-trail byte or we ran out of space for 3rd byte of 4 byte sequence
2380 // 1st check if its a 4 byte sequence
2381 else if ((ch & SupplimentarySeq) != 0)
2383 // 3rd byte of 4 byte sequence?
2384 if ((ch & (FinalByte >> 6)) != 0)
2386 // 3rd byte of 4 byte sequence
2388 bytesUnknown = new byte[] {
2389 unchecked((byte)(((ch >> 12) & 0x07) | 0xF0)),
2390 unchecked((byte)(((ch >> 6) & 0x3F) | 0x80)),
2391 unchecked((byte)(((ch) & 0x3F) | 0x80)) };
2393 else if ((ch & (FinalByte >> 12)) != 0)
2395 // 2nd byte of a 4 byte sequence
2397 bytesUnknown = new byte[] {
2398 unchecked((byte)(((ch >> 6) & 0x07) | 0xF0)),
2399 unchecked((byte)(((ch) & 0x3F) | 0x80)) };
2403 // 4th byte of a 4 byte sequence
2405 bytesUnknown = new byte[] { unchecked((byte)(((ch) & 0x07) | 0xF0)) };
2410 // 2nd byte of 3 byte sequence?
2411 if ((ch & (FinalByte >> 6)) != 0)
2413 // So its 2nd byte of a 3 byte sequence
2415 bytesUnknown = new byte[] {
2416 unchecked((byte)(((ch >> 6) & 0x0F) | 0xE0)), unchecked ((byte)(((ch) & 0x3F) | 0x80)) };
2420 // 1st byte of a 3 byte sequence
2422 bytesUnknown = new byte[] { unchecked((byte)(((ch) & 0x0F) | 0xE0)) };
2426 return bytesUnknown;
2430 public override Decoder GetDecoder()
2432 return new UTF8Decoder(this);
2436 public override Encoder GetEncoder()
2438 return new UTF8Encoder(this);
2442 public override int GetMaxByteCount(int charCount)
2445 throw new ArgumentOutOfRangeException(nameof(charCount),
2446 SR.ArgumentOutOfRange_NeedNonNegNum);
2447 Contract.EndContractBlock();
2449 // Characters would be # of characters + 1 in case left over high surrogate is ? * max fallback
2450 long byteCount = (long)charCount + 1;
2452 if (EncoderFallback.MaxCharCount > 1)
2453 byteCount *= EncoderFallback.MaxCharCount;
2455 // Max 3 bytes per char. (4 bytes per 2 chars for surrogates)
2458 if (byteCount > 0x7fffffff)
2459 throw new ArgumentOutOfRangeException(nameof(charCount), SR.ArgumentOutOfRange_GetByteCountOverflow);
2461 return (int)byteCount;
2465 public override int GetMaxCharCount(int byteCount)
2468 throw new ArgumentOutOfRangeException(nameof(byteCount),
2469 SR.ArgumentOutOfRange_NeedNonNegNum);
2470 Contract.EndContractBlock();
2472 // Figure out our length, 1 char per input byte + 1 char if 1st byte is last byte of 4 byte surrogate pair
2473 long charCount = ((long)byteCount + 1);
2475 // Non-shortest form would fall back, so get max count from fallback.
2476 // So would 11... followed by 11..., so you could fall back every byte
2477 if (DecoderFallback.MaxCharCount > 1)
2479 charCount *= DecoderFallback.MaxCharCount;
2482 if (charCount > 0x7fffffff)
2483 throw new ArgumentOutOfRangeException(nameof(byteCount), SR.ArgumentOutOfRange_GetCharCountOverflow);
2485 return (int)charCount;
2489 public override byte[] GetPreamble()
2491 if (_emitUTF8Identifier)
2493 // Allocate new array to prevent users from modifying it.
2494 return new byte[3] { 0xEF, 0xBB, 0xBF };
2497 return Array.Empty<byte>();
2501 public override bool Equals(Object value)
2503 UTF8Encoding that = value as UTF8Encoding;
2506 return (_emitUTF8Identifier == that._emitUTF8Identifier) &&
2507 (EncoderFallback.Equals(that.EncoderFallback)) &&
2508 (DecoderFallback.Equals(that.DecoderFallback));
2514 public override int GetHashCode()
2516 //Not great distribution, but this is relatively unlikely to be used as the key in a hashtable.
2517 return this.EncoderFallback.GetHashCode() + this.DecoderFallback.GetHashCode() +
2518 UTF8_CODEPAGE + (_emitUTF8Identifier ? 1 : 0);
2521 private sealed class UTF8Encoder : EncoderNLS
2523 // We must save a high surrogate value until the next call, looking
2524 // for a low surrogate value.
2525 internal int surrogateChar;
2527 public UTF8Encoder(UTF8Encoding encoding) : base(encoding)
2532 public override void Reset()
2535 this.surrogateChar = 0;
2536 if (_fallbackBuffer != null)
2537 _fallbackBuffer.Reset();
2540 // Anything left in our encoder?
2541 internal override bool HasState
2545 return (this.surrogateChar != 0);
2550 private sealed class UTF8Decoder : DecoderNLS
2552 // We'll need to remember the previous information. See the comments around definition
2553 // of FinalByte for details.
2556 public UTF8Decoder(UTF8Encoding encoding) : base(encoding)
2561 public override void Reset()
2564 if (_fallbackBuffer != null)
2565 _fallbackBuffer.Reset();
2568 // Anything left in our decoder?
2569 internal override bool HasState
2573 return (this.bits != 0);