1 // Copyright (C) 2016 and later: Unicode, Inc. and others.
2 // License & terms of use: http://www.unicode.org/copyright.html
4 *******************************************************************************
6 * Copyright (C) 2005-2016, International Business Machines
7 * Corporation and others. All Rights Reserved.
9 *******************************************************************************
10 * file name: utext.cpp
12 * tab size: 8 (not used)
15 * created on: 2005apr12
16 * created by: Markus W. Scherer
19 #include "unicode/utypes.h"
20 #include "unicode/ustring.h"
21 #include "unicode/unistr.h"
22 #include "unicode/chariter.h"
23 #include "unicode/utext.h"
24 #include "unicode/utf.h"
25 #include "unicode/utf8.h"
26 #include "unicode/utf16.h"
35 #define I32_FLAG(bitIndex) ((int32_t)1<<(bitIndex))
39 utext_access(UText *ut, int64_t index, UBool forward) {
40 return ut->pFuncs->access(ut, index, forward);
45 U_CAPI UBool U_EXPORT2
46 utext_moveIndex32(UText *ut, int32_t delta) {
50 if(ut->chunkOffset>=ut->chunkLength && !utext_access(ut, ut->chunkNativeLimit, TRUE)) {
53 c = ut->chunkContents[ut->chunkOffset];
54 if (U16_IS_SURROGATE(c)) {
56 if (c == U_SENTINEL) {
66 if(ut->chunkOffset<=0 && !utext_access(ut, ut->chunkNativeStart, FALSE)) {
69 c = ut->chunkContents[ut->chunkOffset-1];
70 if (U16_IS_SURROGATE(c)) {
71 c = utext_previous32(ut);
72 if (c == U_SENTINEL) {
85 U_CAPI int64_t U_EXPORT2
86 utext_nativeLength(UText *ut) {
87 return ut->pFuncs->nativeLength(ut);
91 U_CAPI UBool U_EXPORT2
92 utext_isLengthExpensive(const UText *ut) {
93 UBool r = (ut->providerProperties & I32_FLAG(UTEXT_PROVIDER_LENGTH_IS_EXPENSIVE)) != 0;
98 U_CAPI int64_t U_EXPORT2
99 utext_getNativeIndex(const UText *ut) {
100 if(ut->chunkOffset <= ut->nativeIndexingLimit) {
101 return ut->chunkNativeStart+ut->chunkOffset;
103 return ut->pFuncs->mapOffsetToNative(ut);
108 U_CAPI void U_EXPORT2
109 utext_setNativeIndex(UText *ut, int64_t index) {
110 if(index<ut->chunkNativeStart || index>=ut->chunkNativeLimit) {
111 // The desired position is outside of the current chunk.
112 // Access the new position. Assume a forward iteration from here,
113 // which will also be optimimum for a single random access.
114 // Reverse iterations may suffer slightly.
115 ut->pFuncs->access(ut, index, TRUE);
116 } else if((int32_t)(index - ut->chunkNativeStart) <= ut->nativeIndexingLimit) {
118 ut->chunkOffset=(int32_t)(index-ut->chunkNativeStart);
120 ut->chunkOffset=ut->pFuncs->mapNativeIndexToUTF16(ut, index);
122 // The convention is that the index must always be on a code point boundary.
123 // Adjust the index position if it is in the middle of a surrogate pair.
124 if (ut->chunkOffset<ut->chunkLength) {
125 UChar c= ut->chunkContents[ut->chunkOffset];
126 if (U16_IS_TRAIL(c)) {
127 if (ut->chunkOffset==0) {
128 ut->pFuncs->access(ut, ut->chunkNativeStart, FALSE);
130 if (ut->chunkOffset>0) {
131 UChar lead = ut->chunkContents[ut->chunkOffset-1];
132 if (U16_IS_LEAD(lead)) {
142 U_CAPI int64_t U_EXPORT2
143 utext_getPreviousNativeIndex(UText *ut) {
145 // Fast-path the common case.
146 // Common means current position is not at the beginning of a chunk
147 // and the preceding character is not supplementary.
149 int32_t i = ut->chunkOffset - 1;
152 UChar c = ut->chunkContents[i];
153 if (U16_IS_TRAIL(c) == FALSE) {
154 if (i <= ut->nativeIndexingLimit) {
155 result = ut->chunkNativeStart + i;
158 result = ut->pFuncs->mapOffsetToNative(ut);
165 // If at the start of text, simply return 0.
166 if (ut->chunkOffset==0 && ut->chunkNativeStart==0) {
170 // Harder, less common cases. We are at a chunk boundary, or on a surrogate.
171 // Keep it simple, use other functions to handle the edges.
173 utext_previous32(ut);
174 result = UTEXT_GETNATIVEINDEX(ut);
181 // utext_current32. Get the UChar32 at the current position.
182 // UText iteration position is always on a code point boundary,
183 // never on the trail half of a surrogate pair.
185 U_CAPI UChar32 U_EXPORT2
186 utext_current32(UText *ut) {
188 if (ut->chunkOffset==ut->chunkLength) {
189 // Current position is just off the end of the chunk.
190 if (ut->pFuncs->access(ut, ut->chunkNativeLimit, TRUE) == FALSE) {
191 // Off the end of the text.
196 c = ut->chunkContents[ut->chunkOffset];
197 if (U16_IS_LEAD(c) == FALSE) {
198 // Normal, non-supplementary case.
203 // Possible supplementary char.
206 UChar32 supplementaryC = c;
207 if ((ut->chunkOffset+1) < ut->chunkLength) {
208 // The trail surrogate is in the same chunk.
209 trail = ut->chunkContents[ut->chunkOffset+1];
211 // The trail surrogate is in a different chunk.
212 // Because we must maintain the iteration position, we need to switch forward
213 // into the new chunk, get the trail surrogate, then revert the chunk back to the
215 // An edge case to be careful of: the entire text may end with an unpaired
216 // leading surrogate. The attempt to access the trail will fail, but
217 // the original position before the unpaired lead still needs to be restored.
218 int64_t nativePosition = ut->chunkNativeLimit;
219 int32_t originalOffset = ut->chunkOffset;
220 if (ut->pFuncs->access(ut, nativePosition, TRUE)) {
221 trail = ut->chunkContents[ut->chunkOffset];
223 UBool r = ut->pFuncs->access(ut, nativePosition, FALSE); // reverse iteration flag loads preceding chunk
225 ut->chunkOffset = originalOffset;
231 if (U16_IS_TRAIL(trail)) {
232 supplementaryC = U16_GET_SUPPLEMENTARY(c, trail);
234 return supplementaryC;
239 U_CAPI UChar32 U_EXPORT2
240 utext_char32At(UText *ut, int64_t nativeIndex) {
241 UChar32 c = U_SENTINEL;
243 // Fast path the common case.
244 if (nativeIndex>=ut->chunkNativeStart && nativeIndex < ut->chunkNativeStart + ut->nativeIndexingLimit) {
245 ut->chunkOffset = (int32_t)(nativeIndex - ut->chunkNativeStart);
246 c = ut->chunkContents[ut->chunkOffset];
247 if (U16_IS_SURROGATE(c) == FALSE) {
253 utext_setNativeIndex(ut, nativeIndex);
254 if (nativeIndex>=ut->chunkNativeStart && ut->chunkOffset<ut->chunkLength) {
255 c = ut->chunkContents[ut->chunkOffset];
256 if (U16_IS_SURROGATE(c)) {
257 // For surrogates, let current32() deal with the complications
258 // of supplementaries that may span chunk boundaries.
259 c = utext_current32(ut);
266 U_CAPI UChar32 U_EXPORT2
267 utext_next32(UText *ut) {
270 if (ut->chunkOffset >= ut->chunkLength) {
271 if (ut->pFuncs->access(ut, ut->chunkNativeLimit, TRUE) == FALSE) {
276 c = ut->chunkContents[ut->chunkOffset++];
277 if (U16_IS_LEAD(c) == FALSE) {
278 // Normal case, not supplementary.
279 // (A trail surrogate seen here is just returned as is, as a surrogate value.
280 // It cannot be part of a pair.)
284 if (ut->chunkOffset >= ut->chunkLength) {
285 if (ut->pFuncs->access(ut, ut->chunkNativeLimit, TRUE) == FALSE) {
286 // c is an unpaired lead surrogate at the end of the text.
287 // return it as it is.
291 UChar32 trail = ut->chunkContents[ut->chunkOffset];
292 if (U16_IS_TRAIL(trail) == FALSE) {
293 // c was an unpaired lead surrogate, not at the end of the text.
294 // return it as it is (unpaired). Iteration position is on the
295 // following character, possibly in the next chunk, where the
296 // trail surrogate would have been if it had existed.
300 UChar32 supplementary = U16_GET_SUPPLEMENTARY(c, trail);
301 ut->chunkOffset++; // move iteration position over the trail surrogate.
302 return supplementary;
306 U_CAPI UChar32 U_EXPORT2
307 utext_previous32(UText *ut) {
310 if (ut->chunkOffset <= 0) {
311 if (ut->pFuncs->access(ut, ut->chunkNativeStart, FALSE) == FALSE) {
316 c = ut->chunkContents[ut->chunkOffset];
317 if (U16_IS_TRAIL(c) == FALSE) {
318 // Normal case, not supplementary.
319 // (A lead surrogate seen here is just returned as is, as a surrogate value.
320 // It cannot be part of a pair.)
324 if (ut->chunkOffset <= 0) {
325 if (ut->pFuncs->access(ut, ut->chunkNativeStart, FALSE) == FALSE) {
326 // c is an unpaired trail surrogate at the start of the text.
327 // return it as it is.
332 UChar32 lead = ut->chunkContents[ut->chunkOffset-1];
333 if (U16_IS_LEAD(lead) == FALSE) {
334 // c was an unpaired trail surrogate, not at the end of the text.
335 // return it as it is (unpaired). Iteration position is at c
339 UChar32 supplementary = U16_GET_SUPPLEMENTARY(lead, c);
340 ut->chunkOffset--; // move iteration position over the lead surrogate.
341 return supplementary;
346 U_CAPI UChar32 U_EXPORT2
347 utext_next32From(UText *ut, int64_t index) {
348 UChar32 c = U_SENTINEL;
350 if(index<ut->chunkNativeStart || index>=ut->chunkNativeLimit) {
351 // Desired position is outside of the current chunk.
352 if(!ut->pFuncs->access(ut, index, TRUE)) {
353 // no chunk available here
356 } else if (index - ut->chunkNativeStart <= (int64_t)ut->nativeIndexingLimit) {
357 // Desired position is in chunk, with direct 1:1 native to UTF16 indexing
358 ut->chunkOffset = (int32_t)(index - ut->chunkNativeStart);
360 // Desired position is in chunk, with non-UTF16 indexing.
361 ut->chunkOffset = ut->pFuncs->mapNativeIndexToUTF16(ut, index);
364 c = ut->chunkContents[ut->chunkOffset++];
365 if (U16_IS_SURROGATE(c)) {
366 // Surrogates. Many edge cases. Use other functions that already
367 // deal with the problems.
368 utext_setNativeIndex(ut, index);
369 c = utext_next32(ut);
375 U_CAPI UChar32 U_EXPORT2
376 utext_previous32From(UText *ut, int64_t index) {
378 // Return the character preceding the specified index.
379 // Leave the iteration position at the start of the character that was returned.
381 UChar32 cPrev; // The character preceding cCurr, which is what we will return.
383 // Address the chunk containg the position preceding the incoming index
384 // A tricky edge case:
385 // We try to test the requested native index against the chunkNativeStart to determine
386 // whether the character preceding the one at the index is in the current chunk.
387 // BUT, this test can fail with UTF-8 (or any other multibyte encoding), when the
388 // requested index is on something other than the first position of the first char.
390 if(index<=ut->chunkNativeStart || index>ut->chunkNativeLimit) {
391 // Requested native index is outside of the current chunk.
392 if(!ut->pFuncs->access(ut, index, FALSE)) {
393 // no chunk available here
396 } else if(index - ut->chunkNativeStart <= (int64_t)ut->nativeIndexingLimit) {
397 // Direct UTF-16 indexing.
398 ut->chunkOffset = (int32_t)(index - ut->chunkNativeStart);
400 ut->chunkOffset=ut->pFuncs->mapNativeIndexToUTF16(ut, index);
401 if (ut->chunkOffset==0 && !ut->pFuncs->access(ut, index, FALSE)) {
402 // no chunk available here
408 // Simple case with no surrogates.
411 cPrev = ut->chunkContents[ut->chunkOffset];
413 if (U16_IS_SURROGATE(cPrev)) {
414 // Possible supplementary. Many edge cases.
415 // Let other functions do the heavy lifting.
416 utext_setNativeIndex(ut, index);
417 cPrev = utext_previous32(ut);
423 U_CAPI int32_t U_EXPORT2
424 utext_extract(UText *ut,
425 int64_t start, int64_t limit,
426 UChar *dest, int32_t destCapacity,
427 UErrorCode *status) {
428 return ut->pFuncs->extract(ut, start, limit, dest, destCapacity, status);
433 U_CAPI UBool U_EXPORT2
434 utext_equals(const UText *a, const UText *b) {
435 if (a==NULL || b==NULL ||
436 a->magic != UTEXT_MAGIC ||
437 b->magic != UTEXT_MAGIC) {
438 // Null or invalid arguments don't compare equal to anything.
442 if (a->pFuncs != b->pFuncs) {
443 // Different types of text providers.
447 if (a->context != b->context) {
448 // Different sources (different strings)
451 if (utext_getNativeIndex(a) != utext_getNativeIndex(b)) {
452 // Different current position in the string.
459 U_CAPI UBool U_EXPORT2
460 utext_isWritable(const UText *ut)
462 UBool b = (ut->providerProperties & I32_FLAG(UTEXT_PROVIDER_WRITABLE)) != 0;
467 U_CAPI void U_EXPORT2
468 utext_freeze(UText *ut) {
469 // Zero out the WRITABLE flag.
470 ut->providerProperties &= ~(I32_FLAG(UTEXT_PROVIDER_WRITABLE));
474 U_CAPI UBool U_EXPORT2
475 utext_hasMetaData(const UText *ut)
477 UBool b = (ut->providerProperties & I32_FLAG(UTEXT_PROVIDER_HAS_META_DATA)) != 0;
483 U_CAPI int32_t U_EXPORT2
484 utext_replace(UText *ut,
485 int64_t nativeStart, int64_t nativeLimit,
486 const UChar *replacementText, int32_t replacementLength,
489 if (U_FAILURE(*status)) {
492 if ((ut->providerProperties & I32_FLAG(UTEXT_PROVIDER_WRITABLE)) == 0) {
493 *status = U_NO_WRITE_PERMISSION;
496 int32_t i = ut->pFuncs->replace(ut, nativeStart, nativeLimit, replacementText, replacementLength, status);
500 U_CAPI void U_EXPORT2
501 utext_copy(UText *ut,
502 int64_t nativeStart, int64_t nativeLimit,
507 if (U_FAILURE(*status)) {
510 if ((ut->providerProperties & I32_FLAG(UTEXT_PROVIDER_WRITABLE)) == 0) {
511 *status = U_NO_WRITE_PERMISSION;
514 ut->pFuncs->copy(ut, nativeStart, nativeLimit, destIndex, move, status);
519 U_CAPI UText * U_EXPORT2
520 utext_clone(UText *dest, const UText *src, UBool deep, UBool readOnly, UErrorCode *status) {
521 if (U_FAILURE(*status)) {
524 UText *result = src->pFuncs->clone(dest, src, deep, status);
525 if (U_FAILURE(*status)) {
528 if (result == NULL) {
529 *status = U_MEMORY_ALLOCATION_ERROR;
533 utext_freeze(result);
540 //------------------------------------------------------------------------------
542 // UText common functions implementation
544 //------------------------------------------------------------------------------
547 // UText.flags bit definitions
550 UTEXT_HEAP_ALLOCATED = 1, // 1 if ICU has allocated this UText struct on the heap.
551 // 0 if caller provided storage for the UText.
553 UTEXT_EXTRA_HEAP_ALLOCATED = 2, // 1 if ICU has allocated extra storage as a separate
555 // 0 if there is no separate allocation. Either no extra
556 // storage was requested, or it is appended to the end
557 // of the main UText storage.
559 UTEXT_OPEN = 4 // 1 if this UText is currently open
560 // 0 if this UText is not open.
565 // Extended form of a UText. The purpose is to aid in computing the total size required
566 // when a provider asks for a UText to be allocated with extra storage.
568 struct ExtendedUText {
570 UAlignedMemory extension;
573 static const UText emptyText = UTEXT_INITIALIZER;
575 U_CAPI UText * U_EXPORT2
576 utext_setup(UText *ut, int32_t extraSpace, UErrorCode *status) {
577 if (U_FAILURE(*status)) {
582 // We need to heap-allocate storage for the new UText
583 int32_t spaceRequired = sizeof(UText);
584 if (extraSpace > 0) {
585 spaceRequired = sizeof(ExtendedUText) + extraSpace - sizeof(UAlignedMemory);
587 ut = (UText *)uprv_malloc(spaceRequired);
589 *status = U_MEMORY_ALLOCATION_ERROR;
593 ut->flags |= UTEXT_HEAP_ALLOCATED;
594 if (spaceRequired>0) {
595 ut->extraSize = extraSpace;
596 ut->pExtra = &((ExtendedUText *)ut)->extension;
600 // We have been supplied with an already existing UText.
601 // Verify that it really appears to be a UText.
602 if (ut->magic != UTEXT_MAGIC) {
603 *status = U_ILLEGAL_ARGUMENT_ERROR;
606 // If the ut is already open and there's a provider supplied close
607 // function, call it.
608 if ((ut->flags & UTEXT_OPEN) && ut->pFuncs->close != NULL) {
609 ut->pFuncs->close(ut);
611 ut->flags &= ~UTEXT_OPEN;
613 // If extra space was requested by our caller, check whether
614 // sufficient already exists, and allocate new if needed.
615 if (extraSpace > ut->extraSize) {
616 // Need more space. If there is existing separately allocated space,
617 // delete it first, then allocate new space.
618 if (ut->flags & UTEXT_EXTRA_HEAP_ALLOCATED) {
619 uprv_free(ut->pExtra);
622 ut->pExtra = uprv_malloc(extraSpace);
623 if (ut->pExtra == NULL) {
624 *status = U_MEMORY_ALLOCATION_ERROR;
626 ut->extraSize = extraSpace;
627 ut->flags |= UTEXT_EXTRA_HEAP_ALLOCATED;
631 if (U_SUCCESS(*status)) {
632 ut->flags |= UTEXT_OPEN;
634 // Initialize all remaining fields of the UText.
637 ut->chunkContents = NULL;
646 ut->chunkNativeStart = 0;
647 ut->chunkNativeLimit = 0;
648 ut->nativeIndexingLimit = 0;
649 ut->providerProperties = 0;
654 if (ut->pExtra!=NULL && ut->extraSize>0)
655 uprv_memset(ut->pExtra, 0, ut->extraSize);
662 U_CAPI UText * U_EXPORT2
663 utext_close(UText *ut) {
665 ut->magic != UTEXT_MAGIC ||
666 (ut->flags & UTEXT_OPEN) == 0)
668 // The supplied ut is not an open UText.
673 // If the provider gave us a close function, call it now.
674 // This will clean up anything allocated specifically by the provider.
675 if (ut->pFuncs->close != NULL) {
676 ut->pFuncs->close(ut);
678 ut->flags &= ~UTEXT_OPEN;
680 // If we (the framework) allocated the UText or subsidiary storage,
682 if (ut->flags & UTEXT_EXTRA_HEAP_ALLOCATED) {
683 uprv_free(ut->pExtra);
685 ut->flags &= ~UTEXT_EXTRA_HEAP_ALLOCATED;
689 // Zero out function table of the closed UText. This is a defensive move,
690 // inteded to cause applications that inadvertantly use a closed
691 // utext to crash with null pointer errors.
694 if (ut->flags & UTEXT_HEAP_ALLOCATED) {
695 // This UText was allocated by UText setup. We need to free it.
696 // Clear magic, so we can detect if the user messes up and immediately
697 // tries to reopen another UText using the deleted storage.
709 // invalidateChunk Reset a chunk to have no contents, so that the next call
710 // to access will cause new data to load.
711 // This is needed when copy/move/replace operate directly on the
712 // backing text, potentially putting it out of sync with the
713 // contents in the chunk.
716 invalidateChunk(UText *ut) {
718 ut->chunkNativeLimit = 0;
719 ut->chunkNativeStart = 0;
721 ut->nativeIndexingLimit = 0;
725 // pinIndex Do range pinning on a native index parameter.
726 // 64 bit pinning is done in place.
727 // 32 bit truncated result is returned as a convenience for
728 // use in providers that don't need 64 bits.
730 pinIndex(int64_t &index, int64_t limit) {
733 } else if (index > limit) {
736 return (int32_t)index;
743 // Pointer relocation function,
744 // a utility used by shallow clone.
745 // Adjust a pointer that refers to something within one UText (the source)
746 // to refer to the same relative offset within a another UText (the target)
748 static void adjustPointer(UText *dest, const void **destPtr, const UText *src) {
749 // convert all pointers to (char *) so that byte address arithmetic will work.
750 char *dptr = (char *)*destPtr;
751 char *dUText = (char *)dest;
752 char *sUText = (char *)src;
754 if (dptr >= (char *)src->pExtra && dptr < ((char*)src->pExtra)+src->extraSize) {
755 // target ptr was to something within the src UText's pExtra storage.
756 // relocate it into the target UText's pExtra region.
757 *destPtr = ((char *)dest->pExtra) + (dptr - (char *)src->pExtra);
758 } else if (dptr>=sUText && dptr < sUText+src->sizeOfStruct) {
759 // target ptr was pointing to somewhere within the source UText itself.
760 // Move it to the same offset within the target UText.
761 *destPtr = dUText + (dptr-sUText);
767 // Clone. This is a generic copy-the-utext-by-value clone function that can be
768 // used as-is with some utext types, and as a helper by other clones.
770 static UText * U_CALLCONV
771 shallowTextClone(UText * dest, const UText * src, UErrorCode * status) {
772 if (U_FAILURE(*status)) {
775 int32_t srcExtraSize = src->extraSize;
778 // Use the generic text_setup to allocate storage if required.
780 dest = utext_setup(dest, srcExtraSize, status);
781 if (U_FAILURE(*status)) {
786 // flags (how the UText was allocated) and the pointer to the
787 // extra storage must retain the values in the cloned utext that
788 // were set up by utext_setup. Save them separately before
789 // copying the whole struct.
791 void *destExtra = dest->pExtra;
792 int32_t flags = dest->flags;
796 // Copy the whole UText struct by value.
797 // Any "Extra" storage is copied also.
799 int sizeToCopy = src->sizeOfStruct;
800 if (sizeToCopy > dest->sizeOfStruct) {
801 sizeToCopy = dest->sizeOfStruct;
803 uprv_memcpy(dest, src, sizeToCopy);
804 dest->pExtra = destExtra;
806 if (srcExtraSize > 0) {
807 uprv_memcpy(dest->pExtra, src->pExtra, srcExtraSize);
811 // Relocate any pointers in the target that refer to the UText itself
812 // to point to the cloned copy rather than the original source.
814 adjustPointer(dest, &dest->context, src);
815 adjustPointer(dest, &dest->p, src);
816 adjustPointer(dest, &dest->q, src);
817 adjustPointer(dest, &dest->r, src);
818 adjustPointer(dest, (const void **)&dest->chunkContents, src);
820 // The newly shallow-cloned UText does _not_ own the underlying storage for the text.
821 // (The source for the clone may or may not have owned the text.)
823 dest->providerProperties &= ~I32_FLAG(UTEXT_PROVIDER_OWNS_TEXT);
833 //------------------------------------------------------------------------------
835 // UText implementation for UTF-8 char * strings (read-only)
836 // Limitation: string length must be <= 0x7fffffff in length.
837 // (length must for in an int32_t variable)
839 // Use of UText data members:
840 // context pointer to UTF-8 string
841 // utext.b is the input string length (bytes).
842 // utext.c Length scanned so far in string
843 // (for optimizing finding length of zero terminated strings.)
844 // utext.p pointer to the current buffer
845 // utext.q pointer to the other buffer.
847 //------------------------------------------------------------------------------
850 // Must be less than 85, because of byte mapping from UChar indexes to native indexes.
851 // Worst case is three native bytes to one UChar. (Supplemenaries are 4 native bytes
854 enum { UTF8_TEXT_CHUNK_SIZE=32 };
857 // UTF8Buf Two of these structs will be set up in the UText's extra allocated space.
858 // Each contains the UChar chunk buffer, the to and from native maps, and
861 // because backwards iteration fills the buffers starting at the end and
862 // working towards the front, the filled part of the buffers may not begin
863 // at the start of the available storage for the buffers.
865 // Buffer size is one bigger than the specified UTF8_TEXT_CHUNK_SIZE to allow for
866 // the last character added being a supplementary, and thus requiring a surrogate
867 // pair. Doing this is simpler than checking for the edge case.
871 int32_t bufNativeStart; // Native index of first char in UChar buf
872 int32_t bufNativeLimit; // Native index following last char in buf.
873 int32_t bufStartIdx; // First filled position in buf.
874 int32_t bufLimitIdx; // Limit of filled range in buf.
875 int32_t bufNILimit; // Limit of native indexing part of buf
876 int32_t toUCharsMapStart; // Native index corresponding to
878 // Set to bufNativeStart when filling forwards.
879 // Set to computed value when filling backwards.
881 UChar buf[UTF8_TEXT_CHUNK_SIZE+4]; // The UChar buffer. Requires one extra position beyond the
882 // the chunk size, to allow for surrogate at the end.
883 // Length must be identical to mapToNative array, below,
884 // because of the way indexing works when the array is
885 // filled backwards during a reverse iteration. Thus,
886 // the additional extra size.
887 uint8_t mapToNative[UTF8_TEXT_CHUNK_SIZE+4]; // map UChar index in buf to
888 // native offset from bufNativeStart.
889 // Requires two extra slots,
890 // one for a supplementary starting in the last normal position,
891 // and one for an entry for the buffer limit position.
892 uint8_t mapToUChars[UTF8_TEXT_CHUNK_SIZE*3+6]; // Map native offset from bufNativeStart to
893 // correspoding offset in filled part of buf.
902 // Get the length of the string. If we don't already know it,
903 // we'll need to scan for the trailing nul.
905 static int64_t U_CALLCONV
906 utf8TextLength(UText *ut) {
908 // Zero terminated string, and we haven't scanned to the end yet.
910 const char *r = (const char *)ut->context + ut->c;
914 if ((r - (const char *)ut->context) < 0x7fffffff) {
915 ut->b = (int32_t)(r - (const char *)ut->context);
917 // Actual string was bigger (more than 2 gig) than we
918 // can handle. Clip it to 2 GB.
921 ut->providerProperties &= ~I32_FLAG(UTEXT_PROVIDER_LENGTH_IS_EXPENSIVE);
931 static UBool U_CALLCONV
932 utf8TextAccess(UText *ut, int64_t index, UBool forward) {
934 // Apologies to those who are allergic to goto statements.
935 // Consider each goto to a labelled block to be the equivalent of
936 // call the named block as if it were a function();
939 const uint8_t *s8=(const uint8_t *)ut->context;
941 int32_t length = ut->b; // Length of original utf-8
942 int32_t ix= (int32_t)index; // Requested index, trimmed to 32 bits.
943 int32_t mapIndex = 0;
946 } else if (index > 0x7fffffff) {
947 // Strings with 64 bit lengths not supported by this UTF-8 provider.
951 // Pin requested index to the string length.
955 } else if (ix>=ut->c) {
956 // Zero terminated string, and requested index is beyond
957 // the region that has already been scanned.
958 // Scan up to either the end of the string or to the
959 // requested position, whichever comes first.
960 while (ut->c<ix && s8[ut->c]!=0) {
963 // TODO: support for null terminated string length > 32 bits.
964 if (s8[ut->c] == 0) {
965 // We just found the actual length of the string.
966 // Trim the requested index back to that.
970 ut->providerProperties &= ~I32_FLAG(UTEXT_PROVIDER_LENGTH_IS_EXPENSIVE);
976 // Dispatch to the appropriate action for a forward iteration request.
979 if (ix==ut->chunkNativeLimit) {
980 // Check for normal sequential iteration cases first.
982 // Just reached end of string
983 // Don't swap buffers, but do set the
984 // current buffer position.
985 ut->chunkOffset = ut->chunkLength;
988 // End of current buffer.
989 // check whether other buffer already has what we need.
990 UTF8Buf *altB = (UTF8Buf *)ut->q;
991 if (ix>=altB->bufNativeStart && ix<altB->bufNativeLimit) {
997 // A random access. Desired index could be in either or niether buf.
998 // For optimizing the order of testing, first check for the index
999 // being in the other buffer. This will be the case for uses that
1000 // move back and forth over a fairly limited range
1002 u8b = (UTF8Buf *)ut->q; // the alternate buffer
1003 if (ix>=u8b->bufNativeStart && ix<u8b->bufNativeLimit) {
1004 // Requested index is in the other buffer.
1008 // Requested index is end-of-string.
1009 // (this is the case of randomly seeking to the end.
1010 // The case of iterating off the end is handled earlier.)
1011 if (ix == ut->chunkNativeLimit) {
1012 // Current buffer extends up to the end of the string.
1013 // Leave it as the current buffer.
1014 ut->chunkOffset = ut->chunkLength;
1017 if (ix == u8b->bufNativeLimit) {
1018 // Alternate buffer extends to the end of string.
1019 // Swap it in as the current buffer.
1020 goto swapBuffersAndFail;
1023 // Neither existing buffer extends to the end of the string.
1024 goto makeStubBuffer;
1027 if (ix<ut->chunkNativeStart || ix>=ut->chunkNativeLimit) {
1028 // Requested index is in neither buffer.
1032 // Requested index is in this buffer.
1033 u8b = (UTF8Buf *)ut->p; // the current buffer
1034 mapIndex = ix - u8b->toUCharsMapStart;
1035 ut->chunkOffset = u8b->mapToUChars[mapIndex] - u8b->bufStartIdx;
1043 // Dispatch to the appropriate action for a
1044 // Backwards Diretion iteration request.
1046 if (ix==ut->chunkNativeStart) {
1047 // Check for normal sequential iteration cases first.
1049 // Just reached the start of string
1050 // Don't swap buffers, but do set the
1051 // current buffer position.
1052 ut->chunkOffset = 0;
1055 // Start of current buffer.
1056 // check whether other buffer already has what we need.
1057 UTF8Buf *altB = (UTF8Buf *)ut->q;
1058 if (ix>altB->bufNativeStart && ix<=altB->bufNativeLimit) {
1064 // A random access. Desired index could be in either or niether buf.
1065 // For optimizing the order of testing,
1066 // Most likely case: in the other buffer.
1067 // Second most likely: in neither buffer.
1068 // Unlikely, but must work: in the current buffer.
1069 u8b = (UTF8Buf *)ut->q; // the alternate buffer
1070 if (ix>u8b->bufNativeStart && ix<=u8b->bufNativeLimit) {
1071 // Requested index is in the other buffer.
1074 // Requested index is start-of-string.
1075 // (this is the case of randomly seeking to the start.
1076 // The case of iterating off the start is handled earlier.)
1078 if (u8b->bufNativeStart==0) {
1079 // Alternate buffer contains the data for the start string.
1080 // Make it be the current buffer.
1081 goto swapBuffersAndFail;
1083 // Request for data before the start of string,
1084 // neither buffer is usable.
1085 // set up a zero-length buffer.
1086 goto makeStubBuffer;
1090 if (ix<=ut->chunkNativeStart || ix>ut->chunkNativeLimit) {
1091 // Requested index is in neither buffer.
1095 // Requested index is in this buffer.
1096 // Set the utf16 buffer index.
1097 u8b = (UTF8Buf *)ut->p;
1098 mapIndex = ix - u8b->toUCharsMapStart;
1099 ut->chunkOffset = u8b->mapToUChars[mapIndex] - u8b->bufStartIdx;
1100 if (ut->chunkOffset==0) {
1101 // This occurs when the first character in the text is
1102 // a multi-byte UTF-8 char, and the requested index is to
1103 // one of the trailing bytes. Because there is no preceding ,
1104 // character, this access fails. We can't pick up on the
1105 // situation sooner because the requested index is not zero.
1114 // The alternate buffer (ut->q) has the string data that was requested.
1115 // Swap the primary and alternate buffers, and set the
1116 // chunk index into the new primary buffer.
1118 u8b = (UTF8Buf *)ut->q;
1121 ut->chunkContents = &u8b->buf[u8b->bufStartIdx];
1122 ut->chunkLength = u8b->bufLimitIdx - u8b->bufStartIdx;
1123 ut->chunkNativeStart = u8b->bufNativeStart;
1124 ut->chunkNativeLimit = u8b->bufNativeLimit;
1125 ut->nativeIndexingLimit = u8b->bufNILimit;
1127 // Index into the (now current) chunk
1128 // Use the map to set the chunk index. It's more trouble than it's worth
1129 // to check whether native indexing can be used.
1130 U_ASSERT(ix>=u8b->bufNativeStart);
1131 U_ASSERT(ix<=u8b->bufNativeLimit);
1132 mapIndex = ix - u8b->toUCharsMapStart;
1133 U_ASSERT(mapIndex>=0);
1134 U_ASSERT(mapIndex<(int32_t)sizeof(u8b->mapToUChars));
1135 ut->chunkOffset = u8b->mapToUChars[mapIndex] - u8b->bufStartIdx;
1142 // We got a request for either the start or end of the string,
1143 // with iteration continuing in the out-of-bounds direction.
1144 // The alternate buffer already contains the data up to the
1146 // Swap the buffers, then return failure, indicating that we couldn't
1147 // make things correct for continuing the iteration in the requested
1148 // direction. The position & buffer are correct should the
1149 // user decide to iterate in the opposite direction.
1150 u8b = (UTF8Buf *)ut->q;
1153 ut->chunkContents = &u8b->buf[u8b->bufStartIdx];
1154 ut->chunkLength = u8b->bufLimitIdx - u8b->bufStartIdx;
1155 ut->chunkNativeStart = u8b->bufNativeStart;
1156 ut->chunkNativeLimit = u8b->bufNativeLimit;
1157 ut->nativeIndexingLimit = u8b->bufNILimit;
1159 // Index into the (now current) chunk
1160 // For this function (swapBuffersAndFail), the requested index
1161 // will always be at either the start or end of the chunk.
1162 if (ix==u8b->bufNativeLimit) {
1163 ut->chunkOffset = ut->chunkLength;
1165 ut->chunkOffset = 0;
1166 U_ASSERT(ix == u8b->bufNativeStart);
1171 // The user has done a seek/access past the start or end
1172 // of the string. Rather than loading data that is likely
1173 // to never be used, just set up a zero-length buffer at
1175 u8b = (UTF8Buf *)ut->q;
1176 u8b->bufNativeStart = ix;
1177 u8b->bufNativeLimit = ix;
1178 u8b->bufStartIdx = 0;
1179 u8b->bufLimitIdx = 0;
1180 u8b->bufNILimit = 0;
1181 u8b->toUCharsMapStart = ix;
1182 u8b->mapToNative[0] = 0;
1183 u8b->mapToUChars[0] = 0;
1184 goto swapBuffersAndFail;
1190 // Move the incoming index to a code point boundary.
1191 U8_SET_CP_START(s8, 0, ix);
1193 // Swap the UText buffers.
1194 // We want to fill what was previously the alternate buffer,
1195 // and make what was the current buffer be the new alternate.
1196 UTF8Buf *u8b = (UTF8Buf *)ut->q;
1200 int32_t strLen = ut->b;
1201 UBool nulTerminated = FALSE;
1203 strLen = 0x7fffffff;
1204 nulTerminated = TRUE;
1207 UChar *buf = u8b->buf;
1208 uint8_t *mapToNative = u8b->mapToNative;
1209 uint8_t *mapToUChars = u8b->mapToUChars;
1212 UBool seenNonAscii = FALSE;
1215 // Fill the chunk buffer and mapping arrays.
1216 while (destIx<UTF8_TEXT_CHUNK_SIZE) {
1218 if (c>0 && c<0x80) {
1219 // Special case ASCII range for speed.
1220 // zero is excluded to simplify bounds checking.
1221 buf[destIx] = (UChar)c;
1222 mapToNative[destIx] = (uint8_t)(srcIx - ix);
1223 mapToUChars[srcIx-ix] = (uint8_t)destIx;
1227 // General case, handle everything.
1228 if (seenNonAscii == FALSE) {
1229 seenNonAscii = TRUE;
1230 u8b->bufNILimit = destIx;
1233 int32_t cIx = srcIx;
1234 int32_t dIx = destIx;
1235 int32_t dIxSaved = destIx;
1236 U8_NEXT_OR_FFFD(s8, srcIx, strLen, c);
1237 if (c==0 && nulTerminated) {
1242 U16_APPEND_UNSAFE(buf, destIx, c);
1244 mapToNative[dIx++] = (uint8_t)(cIx - ix);
1245 } while (dIx < destIx);
1248 mapToUChars[cIx++ - ix] = (uint8_t)dIxSaved;
1249 } while (cIx < srcIx);
1251 if (srcIx>=strLen) {
1257 // store Native <--> Chunk Map entries for the end of the buffer.
1258 // There is no actual character here, but the index position is valid.
1259 mapToNative[destIx] = (uint8_t)(srcIx - ix);
1260 mapToUChars[srcIx - ix] = (uint8_t)destIx;
1262 // fill in Buffer descriptor
1263 u8b->bufNativeStart = ix;
1264 u8b->bufNativeLimit = srcIx;
1265 u8b->bufStartIdx = 0;
1266 u8b->bufLimitIdx = destIx;
1267 if (seenNonAscii == FALSE) {
1268 u8b->bufNILimit = destIx;
1270 u8b->toUCharsMapStart = u8b->bufNativeStart;
1272 // Set UText chunk to refer to this buffer.
1273 ut->chunkContents = buf;
1274 ut->chunkOffset = 0;
1275 ut->chunkLength = u8b->bufLimitIdx;
1276 ut->chunkNativeStart = u8b->bufNativeStart;
1277 ut->chunkNativeLimit = u8b->bufNativeLimit;
1278 ut->nativeIndexingLimit = u8b->bufNILimit;
1280 // For zero terminated strings, keep track of the maximum point
1282 if (nulTerminated && srcIx>ut->c) {
1285 // We scanned to the end.
1286 // Remember the actual length.
1288 ut->providerProperties &= ~I32_FLAG(UTEXT_PROVIDER_LENGTH_IS_EXPENSIVE);
1297 // Move the incoming index to a code point boundary.
1298 // Can only do this if the incoming index is somewhere in the interior of the string.
1299 // If index is at the end, there is no character there to look at.
1301 U8_SET_CP_START(s8, 0, ix);
1304 // Swap the UText buffers.
1305 // We want to fill what was previously the alternate buffer,
1306 // and make what was the current buffer be the new alternate.
1307 UTF8Buf *u8b = (UTF8Buf *)ut->q;
1311 UChar *buf = u8b->buf;
1312 uint8_t *mapToNative = u8b->mapToNative;
1313 uint8_t *mapToUChars = u8b->mapToUChars;
1314 int32_t toUCharsMapStart = ix - (UTF8_TEXT_CHUNK_SIZE*3 + 1);
1315 int32_t destIx = UTF8_TEXT_CHUNK_SIZE+2; // Start in the overflow region
1316 // at end of buffer to leave room
1317 // for a surrogate pair at the
1320 int32_t bufNILimit = destIx;
1323 // Map to/from Native Indexes, fill in for the position at the end of
1326 mapToNative[destIx] = (uint8_t)(srcIx - toUCharsMapStart);
1327 mapToUChars[srcIx - toUCharsMapStart] = (uint8_t)destIx;
1329 // Fill the chunk buffer
1330 // Work backwards, filling from the end of the buffer towards the front.
1332 while (destIx>2 && (srcIx - toUCharsMapStart > 5) && (srcIx > 0)) {
1336 // Get last byte of the UTF-8 character
1339 // Special case ASCII range for speed.
1340 buf[destIx] = (UChar)c;
1341 mapToUChars[srcIx - toUCharsMapStart] = (uint8_t)destIx;
1342 mapToNative[destIx] = (uint8_t)(srcIx - toUCharsMapStart);
1344 // General case, handle everything non-ASCII.
1346 int32_t sIx = srcIx; // ix of last byte of multi-byte u8 char
1348 // Get the full character from the UTF8 string.
1349 // use code derived from tbe macros in utf8.h
1350 // Leaves srcIx pointing at the first byte of the UTF-8 char.
1352 c=utf8_prevCharSafeBody(s8, 0, &srcIx, c, -3);
1353 // leaves srcIx at first byte of the multi-byte char.
1355 // Store the character in UTF-16 buffer.
1357 buf[destIx] = (UChar)c;
1358 mapToNative[destIx] = (uint8_t)(srcIx - toUCharsMapStart);
1360 buf[destIx] = U16_TRAIL(c);
1361 mapToNative[destIx] = (uint8_t)(srcIx - toUCharsMapStart);
1362 buf[--destIx] = U16_LEAD(c);
1363 mapToNative[destIx] = (uint8_t)(srcIx - toUCharsMapStart);
1366 // Fill in the map from native indexes to UChars buf index.
1368 mapToUChars[sIx-- - toUCharsMapStart] = (uint8_t)destIx;
1369 } while (sIx >= srcIx);
1371 // Set native indexing limit to be the current position.
1372 // We are processing a non-ascii, non-native-indexing char now;
1373 // the limit will be here if the rest of the chars to be
1374 // added to this buffer are ascii.
1375 bufNILimit = destIx;
1378 u8b->bufNativeStart = srcIx;
1379 u8b->bufNativeLimit = ix;
1380 u8b->bufStartIdx = destIx;
1381 u8b->bufLimitIdx = UTF8_TEXT_CHUNK_SIZE+2;
1382 u8b->bufNILimit = bufNILimit - u8b->bufStartIdx;
1383 u8b->toUCharsMapStart = toUCharsMapStart;
1385 ut->chunkContents = &buf[u8b->bufStartIdx];
1386 ut->chunkLength = u8b->bufLimitIdx - u8b->bufStartIdx;
1387 ut->chunkOffset = ut->chunkLength;
1388 ut->chunkNativeStart = u8b->bufNativeStart;
1389 ut->chunkNativeLimit = u8b->bufNativeLimit;
1390 ut->nativeIndexingLimit = u8b->bufNILimit;
1399 // This is a slightly modified copy of u_strFromUTF8,
1400 // Inserts a Replacement Char rather than failing on invalid UTF-8
1401 // Removes unnecessary features.
1404 utext_strFromUTF8(UChar *dest,
1405 int32_t destCapacity,
1406 int32_t *pDestLength,
1408 int32_t srcLength, // required. NUL terminated not supported.
1409 UErrorCode *pErrorCode
1413 UChar *pDest = dest;
1414 UChar *pDestLimit = (dest!=NULL)?(dest+destCapacity):NULL;
1417 int32_t reqLength = 0;
1418 uint8_t* pSrc = (uint8_t*) src;
1421 while((index < srcLength)&&(pDest<pDestLimit)){
1426 ch=utf8_nextCharSafeBody(pSrc, &index, srcLength, ch, -3);
1428 *(pDest++)=(UChar)ch;
1430 *(pDest++)=U16_LEAD(ch);
1431 if(pDest<pDestLimit){
1432 *(pDest++)=U16_TRAIL(ch);
1440 /* donot fill the dest buffer just count the UChars needed */
1441 while(index < srcLength){
1446 ch=utf8_nextCharSafeBody(pSrc, &index, srcLength, ch, -3);
1447 reqLength+=U16_LENGTH(ch);
1451 reqLength+=(int32_t)(pDest - dest);
1454 *pDestLength = reqLength;
1457 /* Terminate the buffer */
1458 u_terminateUChars(dest,destCapacity,reqLength,pErrorCode);
1465 static int32_t U_CALLCONV
1466 utf8TextExtract(UText *ut,
1467 int64_t start, int64_t limit,
1468 UChar *dest, int32_t destCapacity,
1469 UErrorCode *pErrorCode) {
1470 if(U_FAILURE(*pErrorCode)) {
1473 if(destCapacity<0 || (dest==NULL && destCapacity>0)) {
1474 *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
1477 int32_t length = ut->b;
1478 int32_t start32 = pinIndex(start, length);
1479 int32_t limit32 = pinIndex(limit, length);
1481 if(start32>limit32) {
1482 *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR;
1487 // adjust the incoming indexes to land on code point boundaries if needed.
1488 // adjust by no more than three, because that is the largest number of trail bytes
1489 // in a well formed UTF8 character.
1490 const uint8_t *buf = (const uint8_t *)ut->context;
1492 if (start32 < ut->chunkNativeLimit) {
1493 for (i=0; i<3; i++) {
1494 if (U8_IS_SINGLE(buf[start32]) || U8_IS_LEAD(buf[start32]) || start32==0) {
1501 if (limit32 < ut->chunkNativeLimit) {
1502 for (i=0; i<3; i++) {
1503 if (U8_IS_SINGLE(buf[limit32]) || U8_IS_LEAD(buf[limit32]) || limit32==0) {
1510 // Do the actual extract.
1511 int32_t destLength=0;
1512 utext_strFromUTF8(dest, destCapacity, &destLength,
1513 (const char *)ut->context+start32, limit32-start32,
1515 utf8TextAccess(ut, limit32, TRUE);
1520 // utf8TextMapOffsetToNative
1522 // Map a chunk (UTF-16) offset to a native index.
1523 static int64_t U_CALLCONV
1524 utf8TextMapOffsetToNative(const UText *ut) {
1526 UTF8Buf *u8b = (UTF8Buf *)ut->p;
1527 U_ASSERT(ut->chunkOffset>ut->nativeIndexingLimit && ut->chunkOffset<=ut->chunkLength);
1528 int32_t nativeOffset = u8b->mapToNative[ut->chunkOffset + u8b->bufStartIdx] + u8b->toUCharsMapStart;
1529 U_ASSERT(nativeOffset >= ut->chunkNativeStart && nativeOffset <= ut->chunkNativeLimit);
1530 return nativeOffset;
1534 // Map a native index to the corrsponding chunk offset
1536 static int32_t U_CALLCONV
1537 utf8TextMapIndexToUTF16(const UText *ut, int64_t index64) {
1538 U_ASSERT(index64 <= 0x7fffffff);
1539 int32_t index = (int32_t)index64;
1540 UTF8Buf *u8b = (UTF8Buf *)ut->p;
1541 U_ASSERT(index>=ut->chunkNativeStart+ut->nativeIndexingLimit);
1542 U_ASSERT(index<=ut->chunkNativeLimit);
1543 int32_t mapIndex = index - u8b->toUCharsMapStart;
1544 int32_t offset = u8b->mapToUChars[mapIndex] - u8b->bufStartIdx;
1545 U_ASSERT(offset>=0 && offset<=ut->chunkLength);
1549 static UText * U_CALLCONV
1550 utf8TextClone(UText *dest, const UText *src, UBool deep, UErrorCode *status)
1552 // First do a generic shallow clone. Does everything needed for the UText struct itself.
1553 dest = shallowTextClone(dest, src, status);
1555 // For deep clones, make a copy of the string.
1556 // The copied storage is owned by the newly created clone.
1558 // TODO: There is an isssue with using utext_nativeLength().
1559 // That function is non-const in cases where the input was NUL terminated
1560 // and the length has not yet been determined.
1561 // This function (clone()) is const.
1562 // There potentially a thread safety issue lurking here.
1564 if (deep && U_SUCCESS(*status)) {
1565 int32_t len = (int32_t)utext_nativeLength((UText *)src);
1566 char *copyStr = (char *)uprv_malloc(len+1);
1567 if (copyStr == NULL) {
1568 *status = U_MEMORY_ALLOCATION_ERROR;
1570 uprv_memcpy(copyStr, src->context, len+1);
1571 dest->context = copyStr;
1572 dest->providerProperties |= I32_FLAG(UTEXT_PROVIDER_OWNS_TEXT);
1579 static void U_CALLCONV
1580 utf8TextClose(UText *ut) {
1581 // Most of the work of close is done by the generic UText framework close.
1582 // All that needs to be done here is to delete the UTF8 string if the UText
1583 // owns it. This occurs if the UText was created by cloning.
1584 if (ut->providerProperties & I32_FLAG(UTEXT_PROVIDER_OWNS_TEXT)) {
1585 char *s = (char *)ut->context;
1594 static const struct UTextFuncs utf8Funcs =
1597 0, 0, 0, // Reserved alignment padding
1604 utf8TextMapOffsetToNative,
1605 utf8TextMapIndexToUTF16,
1613 static const char gEmptyString[] = {0};
1615 U_CAPI UText * U_EXPORT2
1616 utext_openUTF8(UText *ut, const char *s, int64_t length, UErrorCode *status) {
1617 if(U_FAILURE(*status)) {
1620 if(s==NULL && length==0) {
1624 if(s==NULL || length<-1 || length>INT32_MAX) {
1625 *status=U_ILLEGAL_ARGUMENT_ERROR;
1629 ut = utext_setup(ut, sizeof(UTF8Buf) * 2, status);
1630 if (U_FAILURE(*status)) {
1634 ut->pFuncs = &utf8Funcs;
1636 ut->b = (int32_t)length;
1637 ut->c = (int32_t)length;
1640 ut->providerProperties |= I32_FLAG(UTEXT_PROVIDER_LENGTH_IS_EXPENSIVE);
1643 ut->q = (char *)ut->pExtra + sizeof(UTF8Buf);
1655 //------------------------------------------------------------------------------
1657 // UText implementation wrapper for Replaceable (read/write)
1659 // Use of UText data members:
1660 // context pointer to Replaceable.
1661 // p pointer to Replaceable if it is owned by the UText.
1663 //------------------------------------------------------------------------------
1667 // minimum chunk size for this implementation: 3
1668 // to allow for possible trimming for code point boundaries
1669 enum { REP_TEXT_CHUNK_SIZE=10 };
1674 * +1 to simplify filling with surrogate pair at the end.
1676 UChar s[REP_TEXT_CHUNK_SIZE+1];
1682 static UText * U_CALLCONV
1683 repTextClone(UText *dest, const UText *src, UBool deep, UErrorCode *status) {
1684 // First do a generic shallow clone. Does everything needed for the UText struct itself.
1685 dest = shallowTextClone(dest, src, status);
1687 // For deep clones, make a copy of the Replaceable.
1688 // The copied Replaceable storage is owned by the newly created UText clone.
1689 // A non-NULL pointer in UText.p is the signal to the close() function to delete
1692 if (deep && U_SUCCESS(*status)) {
1693 const Replaceable *replSrc = (const Replaceable *)src->context;
1694 dest->context = replSrc->clone();
1695 dest->providerProperties |= I32_FLAG(UTEXT_PROVIDER_OWNS_TEXT);
1697 // with deep clone, the copy is writable, even when the source is not.
1698 dest->providerProperties |= I32_FLAG(UTEXT_PROVIDER_WRITABLE);
1704 static void U_CALLCONV
1705 repTextClose(UText *ut) {
1706 // Most of the work of close is done by the generic UText framework close.
1707 // All that needs to be done here is delete the Replaceable if the UText
1708 // owns it. This occurs if the UText was created by cloning.
1709 if (ut->providerProperties & I32_FLAG(UTEXT_PROVIDER_OWNS_TEXT)) {
1710 Replaceable *rep = (Replaceable *)ut->context;
1717 static int64_t U_CALLCONV
1718 repTextLength(UText *ut) {
1719 const Replaceable *replSrc = (const Replaceable *)ut->context;
1720 int32_t len = replSrc->length();
1725 static UBool U_CALLCONV
1726 repTextAccess(UText *ut, int64_t index, UBool forward) {
1727 const Replaceable *rep=(const Replaceable *)ut->context;
1728 int32_t length=rep->length(); // Full length of the input text (bigger than a chunk)
1730 // clip the requested index to the limits of the text.
1731 int32_t index32 = pinIndex(index, length);
1732 U_ASSERT(index<=INT32_MAX);
1736 * Compute start/limit boundaries around index, for a segment of text
1738 * To allow for the possibility that our user gave an index to the trailing
1739 * half of a surrogate pair, we must request one extra preceding UChar when
1740 * going in the forward direction. This will ensure that the buffer has the
1741 * entire code point at the specified index.
1745 if (index32>=ut->chunkNativeStart && index32<ut->chunkNativeLimit) {
1746 // Buffer already contains the requested position.
1747 ut->chunkOffset = (int32_t)(index - ut->chunkNativeStart);
1750 if (index32>=length && ut->chunkNativeLimit==length) {
1751 // Request for end of string, and buffer already extends up to it.
1752 // Can't get the data, but don't change the buffer.
1753 ut->chunkOffset = length - (int32_t)ut->chunkNativeStart;
1757 ut->chunkNativeLimit = index + REP_TEXT_CHUNK_SIZE - 1;
1758 // Going forward, so we want to have the buffer with stuff at and beyond
1759 // the requested index. The -1 gets us one code point before the
1760 // requested index also, to handle the case of the index being on
1761 // a trail surrogate of a surrogate pair.
1762 if(ut->chunkNativeLimit > length) {
1763 ut->chunkNativeLimit = length;
1765 // unless buffer ran off end, start is index-1.
1766 ut->chunkNativeStart = ut->chunkNativeLimit - REP_TEXT_CHUNK_SIZE;
1767 if(ut->chunkNativeStart < 0) {
1768 ut->chunkNativeStart = 0;
1771 // Reverse iteration. Fill buffer with data preceding the requested index.
1772 if (index32>ut->chunkNativeStart && index32<=ut->chunkNativeLimit) {
1773 // Requested position already in buffer.
1774 ut->chunkOffset = index32 - (int32_t)ut->chunkNativeStart;
1777 if (index32==0 && ut->chunkNativeStart==0) {
1778 // Request for start, buffer already begins at start.
1779 // No data, but keep the buffer as is.
1780 ut->chunkOffset = 0;
1784 // Figure out the bounds of the chunk to extract for reverse iteration.
1785 // Need to worry about chunk not splitting surrogate pairs, and while still
1786 // containing the data we need.
1787 // Fix by requesting a chunk that includes an extra UChar at the end.
1788 // If this turns out to be a lead surrogate, we can lop it off and still have
1789 // the data we wanted.
1790 ut->chunkNativeStart = index32 + 1 - REP_TEXT_CHUNK_SIZE;
1791 if (ut->chunkNativeStart < 0) {
1792 ut->chunkNativeStart = 0;
1795 ut->chunkNativeLimit = index32 + 1;
1796 if (ut->chunkNativeLimit > length) {
1797 ut->chunkNativeLimit = length;
1801 // Extract the new chunk of text from the Replaceable source.
1802 ReplExtra *ex = (ReplExtra *)ut->pExtra;
1803 // UnicodeString with its buffer a writable alias to the chunk buffer
1804 UnicodeString buffer(ex->s, 0 /*buffer length*/, REP_TEXT_CHUNK_SIZE /*buffer capacity*/);
1805 rep->extractBetween((int32_t)ut->chunkNativeStart, (int32_t)ut->chunkNativeLimit, buffer);
1807 ut->chunkContents = ex->s;
1808 ut->chunkLength = (int32_t)(ut->chunkNativeLimit - ut->chunkNativeStart);
1809 ut->chunkOffset = (int32_t)(index32 - ut->chunkNativeStart);
1811 // Surrogate pairs from the input text must not span chunk boundaries.
1812 // If end of chunk could be the start of a surrogate, trim it off.
1813 if (ut->chunkNativeLimit < length &&
1814 U16_IS_LEAD(ex->s[ut->chunkLength-1])) {
1816 ut->chunkNativeLimit--;
1817 if (ut->chunkOffset > ut->chunkLength) {
1818 ut->chunkOffset = ut->chunkLength;
1822 // if the first UChar in the chunk could be the trailing half of a surrogate pair,
1824 if(ut->chunkNativeStart>0 && U16_IS_TRAIL(ex->s[0])) {
1825 ++(ut->chunkContents);
1826 ++(ut->chunkNativeStart);
1827 --(ut->chunkLength);
1828 --(ut->chunkOffset);
1831 // adjust the index/chunkOffset to a code point boundary
1832 U16_SET_CP_START(ut->chunkContents, 0, ut->chunkOffset);
1834 // Use fast indexing for get/setNativeIndex()
1835 ut->nativeIndexingLimit = ut->chunkLength;
1842 static int32_t U_CALLCONV
1843 repTextExtract(UText *ut,
1844 int64_t start, int64_t limit,
1845 UChar *dest, int32_t destCapacity,
1846 UErrorCode *status) {
1847 const Replaceable *rep=(const Replaceable *)ut->context;
1848 int32_t length=rep->length();
1850 if(U_FAILURE(*status)) {
1853 if(destCapacity<0 || (dest==NULL && destCapacity>0)) {
1854 *status=U_ILLEGAL_ARGUMENT_ERROR;
1857 *status=U_INDEX_OUTOFBOUNDS_ERROR;
1861 int32_t start32 = pinIndex(start, length);
1862 int32_t limit32 = pinIndex(limit, length);
1864 // adjust start, limit if they point to trail half of surrogates
1865 if (start32<length && U16_IS_TRAIL(rep->charAt(start32)) &&
1866 U_IS_SUPPLEMENTARY(rep->char32At(start32))){
1869 if (limit32<length && U16_IS_TRAIL(rep->charAt(limit32)) &&
1870 U_IS_SUPPLEMENTARY(rep->char32At(limit32))){
1874 length=limit32-start32;
1875 if(length>destCapacity) {
1876 limit32 = start32 + destCapacity;
1878 UnicodeString buffer(dest, 0, destCapacity); // writable alias
1879 rep->extractBetween(start32, limit32, buffer);
1880 repTextAccess(ut, limit32, TRUE);
1882 return u_terminateUChars(dest, destCapacity, length, status);
1885 static int32_t U_CALLCONV
1886 repTextReplace(UText *ut,
1887 int64_t start, int64_t limit,
1888 const UChar *src, int32_t length,
1889 UErrorCode *status) {
1890 Replaceable *rep=(Replaceable *)ut->context;
1893 if(U_FAILURE(*status)) {
1896 if(src==NULL && length!=0) {
1897 *status=U_ILLEGAL_ARGUMENT_ERROR;
1900 oldLength=rep->length(); // will subtract from new length
1902 *status=U_INDEX_OUTOFBOUNDS_ERROR;
1906 int32_t start32 = pinIndex(start, oldLength);
1907 int32_t limit32 = pinIndex(limit, oldLength);
1909 // Snap start & limit to code point boundaries.
1910 if (start32<oldLength && U16_IS_TRAIL(rep->charAt(start32)) &&
1911 start32>0 && U16_IS_LEAD(rep->charAt(start32-1)))
1915 if (limit32<oldLength && U16_IS_LEAD(rep->charAt(limit32-1)) &&
1916 U16_IS_TRAIL(rep->charAt(limit32)))
1921 // Do the actual replace operation using methods of the Replaceable class
1922 UnicodeString replStr((UBool)(length<0), src, length); // read-only alias
1923 rep->handleReplaceBetween(start32, limit32, replStr);
1924 int32_t newLength = rep->length();
1925 int32_t lengthDelta = newLength - oldLength;
1927 // Is the UText chunk buffer OK?
1928 if (ut->chunkNativeLimit > start32) {
1929 // this replace operation may have impacted the current chunk.
1930 // invalidate it, which will force a reload on the next access.
1931 invalidateChunk(ut);
1934 // set the iteration position to the end of the newly inserted replacement text.
1935 int32_t newIndexPos = limit32 + lengthDelta;
1936 repTextAccess(ut, newIndexPos, TRUE);
1942 static void U_CALLCONV
1943 repTextCopy(UText *ut,
1944 int64_t start, int64_t limit,
1949 Replaceable *rep=(Replaceable *)ut->context;
1950 int32_t length=rep->length();
1952 if(U_FAILURE(*status)) {
1955 if (start>limit || (start<destIndex && destIndex<limit))
1957 *status=U_INDEX_OUTOFBOUNDS_ERROR;
1961 int32_t start32 = pinIndex(start, length);
1962 int32_t limit32 = pinIndex(limit, length);
1963 int32_t destIndex32 = pinIndex(destIndex, length);
1965 // TODO: snap input parameters to code point boundaries.
1968 // move: copy to destIndex, then replace original with nothing
1969 int32_t segLength=limit32-start32;
1970 rep->copy(start32, limit32, destIndex32);
1971 if(destIndex32<start32) {
1975 rep->handleReplaceBetween(start32, limit32, UnicodeString());
1978 rep->copy(start32, limit32, destIndex32);
1981 // If the change to the text touched the region in the chunk buffer,
1982 // invalidate the buffer.
1983 int32_t firstAffectedIndex = destIndex32;
1984 if (move && start32<firstAffectedIndex) {
1985 firstAffectedIndex = start32;
1987 if (firstAffectedIndex < ut->chunkNativeLimit) {
1988 // changes may have affected range covered by the chunk
1989 invalidateChunk(ut);
1992 // Put iteration position at the newly inserted (moved) block,
1993 int32_t nativeIterIndex = destIndex32 + limit32 - start32;
1994 if (move && destIndex32>start32) {
1995 // moved a block of text towards the end of the string.
1996 nativeIterIndex = destIndex32;
1999 // Set position, reload chunk if needed.
2000 repTextAccess(ut, nativeIterIndex, TRUE);
2003 static const struct UTextFuncs repFuncs =
2006 0, 0, 0, // Reserved alignment padding
2013 NULL, // MapOffsetToNative,
2014 NULL, // MapIndexToUTF16,
2022 U_CAPI UText * U_EXPORT2
2023 utext_openReplaceable(UText *ut, Replaceable *rep, UErrorCode *status)
2025 if(U_FAILURE(*status)) {
2029 *status=U_ILLEGAL_ARGUMENT_ERROR;
2032 ut = utext_setup(ut, sizeof(ReplExtra), status);
2033 if(U_FAILURE(*status)) {
2037 ut->providerProperties = I32_FLAG(UTEXT_PROVIDER_WRITABLE);
2038 if(rep->hasMetaData()) {
2039 ut->providerProperties |=I32_FLAG(UTEXT_PROVIDER_HAS_META_DATA);
2042 ut->pFuncs = &repFuncs;
2056 //------------------------------------------------------------------------------
2058 // UText implementation for UnicodeString (read/write) and
2059 // for const UnicodeString (read only)
2060 // (same implementation, only the flags are different)
2062 // Use of UText data members:
2063 // context pointer to UnicodeString
2064 // p pointer to UnicodeString IF this UText owns the string
2065 // and it must be deleted on close(). NULL otherwise.
2067 //------------------------------------------------------------------------------
2072 static UText * U_CALLCONV
2073 unistrTextClone(UText *dest, const UText *src, UBool deep, UErrorCode *status) {
2074 // First do a generic shallow clone. Does everything needed for the UText struct itself.
2075 dest = shallowTextClone(dest, src, status);
2077 // For deep clones, make a copy of the UnicodeSring.
2078 // The copied UnicodeString storage is owned by the newly created UText clone.
2079 // A non-NULL pointer in UText.p is the signal to the close() function to delete
2082 if (deep && U_SUCCESS(*status)) {
2083 const UnicodeString *srcString = (const UnicodeString *)src->context;
2084 dest->context = new UnicodeString(*srcString);
2085 dest->providerProperties |= I32_FLAG(UTEXT_PROVIDER_OWNS_TEXT);
2087 // with deep clone, the copy is writable, even when the source is not.
2088 dest->providerProperties |= I32_FLAG(UTEXT_PROVIDER_WRITABLE);
2093 static void U_CALLCONV
2094 unistrTextClose(UText *ut) {
2095 // Most of the work of close is done by the generic UText framework close.
2096 // All that needs to be done here is delete the UnicodeString if the UText
2097 // owns it. This occurs if the UText was created by cloning.
2098 if (ut->providerProperties & I32_FLAG(UTEXT_PROVIDER_OWNS_TEXT)) {
2099 UnicodeString *str = (UnicodeString *)ut->context;
2106 static int64_t U_CALLCONV
2107 unistrTextLength(UText *t) {
2108 return ((const UnicodeString *)t->context)->length();
2112 static UBool U_CALLCONV
2113 unistrTextAccess(UText *ut, int64_t index, UBool forward) {
2114 int32_t length = ut->chunkLength;
2115 ut->chunkOffset = pinIndex(index, length);
2117 // Check whether request is at the start or end
2118 UBool retVal = (forward && index<length) || (!forward && index>0);
2124 static int32_t U_CALLCONV
2125 unistrTextExtract(UText *t,
2126 int64_t start, int64_t limit,
2127 UChar *dest, int32_t destCapacity,
2128 UErrorCode *pErrorCode) {
2129 const UnicodeString *us=(const UnicodeString *)t->context;
2130 int32_t length=us->length();
2132 if(U_FAILURE(*pErrorCode)) {
2135 if(destCapacity<0 || (dest==NULL && destCapacity>0)) {
2136 *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
2138 if(start<0 || start>limit) {
2139 *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR;
2143 int32_t start32 = start<length ? us->getChar32Start((int32_t)start) : length;
2144 int32_t limit32 = limit<length ? us->getChar32Start((int32_t)limit) : length;
2146 length=limit32-start32;
2147 if (destCapacity>0 && dest!=NULL) {
2148 int32_t trimmedLength = length;
2149 if(trimmedLength>destCapacity) {
2150 trimmedLength=destCapacity;
2152 us->extract(start32, trimmedLength, dest);
2153 t->chunkOffset = start32+trimmedLength;
2155 t->chunkOffset = start32;
2157 u_terminateUChars(dest, destCapacity, length, pErrorCode);
2161 static int32_t U_CALLCONV
2162 unistrTextReplace(UText *ut,
2163 int64_t start, int64_t limit,
2164 const UChar *src, int32_t length,
2165 UErrorCode *pErrorCode) {
2166 UnicodeString *us=(UnicodeString *)ut->context;
2169 if(U_FAILURE(*pErrorCode)) {
2172 if(src==NULL && length!=0) {
2173 *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
2176 *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR;
2179 oldLength=us->length();
2180 int32_t start32 = pinIndex(start, oldLength);
2181 int32_t limit32 = pinIndex(limit, oldLength);
2182 if (start32 < oldLength) {
2183 start32 = us->getChar32Start(start32);
2185 if (limit32 < oldLength) {
2186 limit32 = us->getChar32Start(limit32);
2190 us->replace(start32, limit32-start32, src, length);
2191 int32_t newLength = us->length();
2193 // Update the chunk description.
2194 ut->chunkContents = us->getBuffer();
2195 ut->chunkLength = newLength;
2196 ut->chunkNativeLimit = newLength;
2197 ut->nativeIndexingLimit = newLength;
2199 // Set iteration position to the point just following the newly inserted text.
2200 int32_t lengthDelta = newLength - oldLength;
2201 ut->chunkOffset = limit32 + lengthDelta;
2206 static void U_CALLCONV
2207 unistrTextCopy(UText *ut,
2208 int64_t start, int64_t limit,
2211 UErrorCode *pErrorCode) {
2212 UnicodeString *us=(UnicodeString *)ut->context;
2213 int32_t length=us->length();
2215 if(U_FAILURE(*pErrorCode)) {
2218 int32_t start32 = pinIndex(start, length);
2219 int32_t limit32 = pinIndex(limit, length);
2220 int32_t destIndex32 = pinIndex(destIndex, length);
2222 if( start32>limit32 || (start32<destIndex32 && destIndex32<limit32)) {
2223 *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR;
2228 // move: copy to destIndex, then replace original with nothing
2229 int32_t segLength=limit32-start32;
2230 us->copy(start32, limit32, destIndex32);
2231 if(destIndex32<start32) {
2234 us->replace(start32, segLength, NULL, 0);
2237 us->copy(start32, limit32, destIndex32);
2240 // update chunk description, set iteration position.
2241 ut->chunkContents = us->getBuffer();
2243 // copy operation, string length grows
2244 ut->chunkLength += limit32-start32;
2245 ut->chunkNativeLimit = ut->chunkLength;
2246 ut->nativeIndexingLimit = ut->chunkLength;
2249 // Iteration position to end of the newly inserted text.
2250 ut->chunkOffset = destIndex32+limit32-start32;
2251 if (move && destIndex32>start32) {
2252 ut->chunkOffset = destIndex32;
2257 static const struct UTextFuncs unistrFuncs =
2260 0, 0, 0, // Reserved alignment padding
2267 NULL, // MapOffsetToNative,
2268 NULL, // MapIndexToUTF16,
2280 U_CAPI UText * U_EXPORT2
2281 utext_openUnicodeString(UText *ut, UnicodeString *s, UErrorCode *status) {
2282 ut = utext_openConstUnicodeString(ut, s, status);
2283 if (U_SUCCESS(*status)) {
2284 ut->providerProperties |= I32_FLAG(UTEXT_PROVIDER_WRITABLE);
2291 U_CAPI UText * U_EXPORT2
2292 utext_openConstUnicodeString(UText *ut, const UnicodeString *s, UErrorCode *status) {
2293 if (U_SUCCESS(*status) && s->isBogus()) {
2294 // The UnicodeString is bogus, but we still need to detach the UText
2295 // from whatever it was hooked to before, if anything.
2296 utext_openUChars(ut, NULL, 0, status);
2297 *status = U_ILLEGAL_ARGUMENT_ERROR;
2300 ut = utext_setup(ut, 0, status);
2301 // note: use the standard (writable) function table for UnicodeString.
2302 // The flag settings disable writing, so having the functions in
2303 // the table is harmless.
2304 if (U_SUCCESS(*status)) {
2305 ut->pFuncs = &unistrFuncs;
2307 ut->providerProperties = I32_FLAG(UTEXT_PROVIDER_STABLE_CHUNKS);
2308 ut->chunkContents = s->getBuffer();
2309 ut->chunkLength = s->length();
2310 ut->chunkNativeStart = 0;
2311 ut->chunkNativeLimit = ut->chunkLength;
2312 ut->nativeIndexingLimit = ut->chunkLength;
2317 //------------------------------------------------------------------------------
2319 // UText implementation for const UChar * strings
2321 // Use of UText data members:
2322 // context pointer to UnicodeString
2323 // a length. -1 if not yet known.
2325 // TODO: support 64 bit lengths.
2327 //------------------------------------------------------------------------------
2332 static UText * U_CALLCONV
2333 ucstrTextClone(UText *dest, const UText * src, UBool deep, UErrorCode * status) {
2334 // First do a generic shallow clone.
2335 dest = shallowTextClone(dest, src, status);
2337 // For deep clones, make a copy of the string.
2338 // The copied storage is owned by the newly created clone.
2339 // A non-NULL pointer in UText.p is the signal to the close() function to delete
2342 if (deep && U_SUCCESS(*status)) {
2343 U_ASSERT(utext_nativeLength(dest) < INT32_MAX);
2344 int32_t len = (int32_t)utext_nativeLength(dest);
2346 // The cloned string IS going to be NUL terminated, whether or not the original was.
2347 const UChar *srcStr = (const UChar *)src->context;
2348 UChar *copyStr = (UChar *)uprv_malloc((len+1) * sizeof(UChar));
2349 if (copyStr == NULL) {
2350 *status = U_MEMORY_ALLOCATION_ERROR;
2353 for (i=0; i<len; i++) {
2354 copyStr[i] = srcStr[i];
2357 dest->context = copyStr;
2358 dest->providerProperties |= I32_FLAG(UTEXT_PROVIDER_OWNS_TEXT);
2365 static void U_CALLCONV
2366 ucstrTextClose(UText *ut) {
2367 // Most of the work of close is done by the generic UText framework close.
2368 // All that needs to be done here is delete the string if the UText
2369 // owns it. This occurs if the UText was created by cloning.
2370 if (ut->providerProperties & I32_FLAG(UTEXT_PROVIDER_OWNS_TEXT)) {
2371 UChar *s = (UChar *)ut->context;
2379 static int64_t U_CALLCONV
2380 ucstrTextLength(UText *ut) {
2382 // null terminated, we don't yet know the length. Scan for it.
2383 // Access is not convenient for doing this
2384 // because the current interation postion can't be changed.
2385 const UChar *str = (const UChar *)ut->context;
2387 if (str[ut->chunkNativeLimit] == 0) {
2390 ut->chunkNativeLimit++;
2392 ut->a = ut->chunkNativeLimit;
2393 ut->chunkLength = (int32_t)ut->chunkNativeLimit;
2394 ut->nativeIndexingLimit = ut->chunkLength;
2395 ut->providerProperties &= ~I32_FLAG(UTEXT_PROVIDER_LENGTH_IS_EXPENSIVE);
2401 static UBool U_CALLCONV
2402 ucstrTextAccess(UText *ut, int64_t index, UBool forward) {
2403 const UChar *str = (const UChar *)ut->context;
2405 // pin the requested index to the bounds of the string,
2406 // and set current iteration position.
2409 } else if (index < ut->chunkNativeLimit) {
2410 // The request data is within the chunk as it is known so far.
2411 // Put index on a code point boundary.
2412 U16_SET_CP_START(str, 0, index);
2413 } else if (ut->a >= 0) {
2414 // We know the length of this string, and the user is requesting something
2415 // at or beyond the length. Pin the requested index to the length.
2418 // Null terminated string, length not yet known, and the requested index
2419 // is beyond where we have scanned so far.
2420 // Scan to 32 UChars beyond the requested index. The strategy here is
2421 // to avoid fully scanning a long string when the caller only wants to
2422 // see a few characters at its beginning.
2423 int32_t scanLimit = (int32_t)index + 32;
2424 if ((index + 32)>INT32_MAX || (index + 32)<0 ) { // note: int64 expression
2425 scanLimit = INT32_MAX;
2428 int32_t chunkLimit = (int32_t)ut->chunkNativeLimit;
2429 for (; chunkLimit<scanLimit; chunkLimit++) {
2430 if (str[chunkLimit] == 0) {
2431 // We found the end of the string. Remember it, pin the requested index to it,
2432 // and bail out of here.
2434 ut->chunkLength = chunkLimit;
2435 ut->nativeIndexingLimit = chunkLimit;
2436 if (index >= chunkLimit) {
2439 U16_SET_CP_START(str, 0, index);
2442 ut->chunkNativeLimit = chunkLimit;
2443 ut->providerProperties &= ~I32_FLAG(UTEXT_PROVIDER_LENGTH_IS_EXPENSIVE);
2447 // We scanned through the next batch of UChars without finding the end.
2448 U16_SET_CP_START(str, 0, index);
2449 if (chunkLimit == INT32_MAX) {
2450 // Scanned to the limit of a 32 bit length.
2451 // Forceably trim the overlength string back so length fits in int32
2452 // TODO: add support for 64 bit strings.
2454 ut->chunkLength = chunkLimit;
2455 ut->nativeIndexingLimit = chunkLimit;
2456 if (index > chunkLimit) {
2459 ut->chunkNativeLimit = chunkLimit;
2460 ut->providerProperties &= ~I32_FLAG(UTEXT_PROVIDER_LENGTH_IS_EXPENSIVE);
2462 // The endpoint of a chunk must not be left in the middle of a surrogate pair.
2463 // If the current end is on a lead surrogate, back the end up by one.
2464 // It doesn't matter if the end char happens to be an unpaired surrogate,
2465 // and it's simpler not to worry about it.
2466 if (U16_IS_LEAD(str[chunkLimit-1])) {
2469 // Null-terminated chunk with end still unknown.
2470 // Update the chunk length to reflect what has been scanned thus far.
2471 // That the full length is still unknown is (still) flagged by
2473 ut->chunkNativeLimit = chunkLimit;
2474 ut->nativeIndexingLimit = chunkLimit;
2475 ut->chunkLength = chunkLimit;
2480 U_ASSERT(index<=INT32_MAX);
2481 ut->chunkOffset = (int32_t)index;
2483 // Check whether request is at the start or end
2484 UBool retVal = (forward && index<ut->chunkNativeLimit) || (!forward && index>0);
2490 static int32_t U_CALLCONV
2491 ucstrTextExtract(UText *ut,
2492 int64_t start, int64_t limit,
2493 UChar *dest, int32_t destCapacity,
2494 UErrorCode *pErrorCode)
2496 if(U_FAILURE(*pErrorCode)) {
2499 if(destCapacity<0 || (dest==NULL && destCapacity>0) || start>limit) {
2500 *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
2504 //const UChar *s=(const UChar *)ut->context;
2510 // Access the start. Does two things we need:
2511 // Pins 'start' to the length of the string, if it came in out-of-bounds.
2512 // Snaps 'start' to the beginning of a code point.
2513 ucstrTextAccess(ut, start, TRUE);
2514 const UChar *s=ut->chunkContents;
2515 start32 = ut->chunkOffset;
2517 int32_t strLength=(int32_t)ut->a;
2518 if (strLength >= 0) {
2519 limit32 = pinIndex(limit, strLength);
2521 limit32 = pinIndex(limit, INT32_MAX);
2524 for (si=start32; si<limit32; si++) {
2525 if (strLength<0 && s[si]==0) {
2526 // Just hit the end of a null-terminated string.
2527 ut->a = si; // set string length for this UText
2528 ut->chunkNativeLimit = si;
2529 ut->chunkLength = si;
2530 ut->nativeIndexingLimit = si;
2535 U_ASSERT(di>=0); /* to ensure di never exceeds INT32_MAX, which must not happen logically */
2536 if (di<destCapacity) {
2537 // only store if there is space.
2541 // We have filled the destination buffer, and the string length is known.
2542 // Cut the loop short. There is no need to scan string termination.
2543 di = limit32 - start32;
2551 // If the limit index points to a lead surrogate of a pair,
2552 // add the corresponding trail surrogate to the destination.
2553 if (si>0 && U16_IS_LEAD(s[si-1]) &&
2554 ((si<strLength || strLength<0) && U16_IS_TRAIL(s[si])))
2556 if (di<destCapacity) {
2557 // store only if there is space in the output buffer.
2563 // Put iteration position at the point just following the extracted text
2564 if (si <= ut->chunkNativeLimit) {
2565 ut->chunkOffset = si;
2567 ucstrTextAccess(ut, si, TRUE);
2570 // Add a terminating NUL if space in the buffer permits,
2571 // and set the error status as required.
2572 u_terminateUChars(dest, destCapacity, di, pErrorCode);
2576 static const struct UTextFuncs ucstrFuncs =
2579 0, 0, 0, // Reserved alignment padding
2586 NULL, // MapOffsetToNative,
2587 NULL, // MapIndexToUTF16,
2596 static const UChar gEmptyUString[] = {0};
2598 U_CAPI UText * U_EXPORT2
2599 utext_openUChars(UText *ut, const UChar *s, int64_t length, UErrorCode *status) {
2600 if (U_FAILURE(*status)) {
2603 if(s==NULL && length==0) {
2606 if (s==NULL || length < -1 || length>INT32_MAX) {
2607 *status = U_ILLEGAL_ARGUMENT_ERROR;
2610 ut = utext_setup(ut, 0, status);
2611 if (U_SUCCESS(*status)) {
2612 ut->pFuncs = &ucstrFuncs;
2614 ut->providerProperties = I32_FLAG(UTEXT_PROVIDER_STABLE_CHUNKS);
2616 ut->providerProperties |= I32_FLAG(UTEXT_PROVIDER_LENGTH_IS_EXPENSIVE);
2619 ut->chunkContents = s;
2620 ut->chunkNativeStart = 0;
2621 ut->chunkNativeLimit = length>=0? length : 0;
2622 ut->chunkLength = (int32_t)ut->chunkNativeLimit;
2623 ut->chunkOffset = 0;
2624 ut->nativeIndexingLimit = ut->chunkLength;
2630 //------------------------------------------------------------------------------
2632 // UText implementation for text from ICU CharacterIterators
2634 // Use of UText data members:
2635 // context pointer to the CharacterIterator
2636 // a length of the full text.
2637 // p pointer to buffer 1
2638 // b start index of local buffer 1 contents
2639 // q pointer to buffer 2
2640 // c start index of local buffer 2 contents
2641 // r pointer to the character iterator if the UText owns it.
2644 //------------------------------------------------------------------------------
2645 #define CIBufSize 16
2648 static void U_CALLCONV
2649 charIterTextClose(UText *ut) {
2650 // Most of the work of close is done by the generic UText framework close.
2651 // All that needs to be done here is delete the CharacterIterator if the UText
2652 // owns it. This occurs if the UText was created by cloning.
2653 CharacterIterator *ci = (CharacterIterator *)ut->r;
2658 static int64_t U_CALLCONV
2659 charIterTextLength(UText *ut) {
2660 return (int32_t)ut->a;
2663 static UBool U_CALLCONV
2664 charIterTextAccess(UText *ut, int64_t index, UBool forward) {
2665 CharacterIterator *ci = (CharacterIterator *)ut->context;
2667 int32_t clippedIndex = (int32_t)index;
2668 if (clippedIndex<0) {
2670 } else if (clippedIndex>=ut->a) {
2671 clippedIndex=(int32_t)ut->a;
2673 int32_t neededIndex = clippedIndex;
2674 if (!forward && neededIndex>0) {
2675 // reverse iteration, want the position just before what was asked for.
2677 } else if (forward && neededIndex==ut->a && neededIndex>0) {
2678 // Forward iteration, don't ask for something past the end of the text.
2682 // Find the native index of the start of the buffer containing what we want.
2683 neededIndex -= neededIndex % CIBufSize;
2686 UBool needChunkSetup = TRUE;
2688 if (ut->chunkNativeStart == neededIndex) {
2689 // The buffer we want is already the current chunk.
2690 needChunkSetup = FALSE;
2691 } else if (ut->b == neededIndex) {
2692 // The first buffer (buffer p) has what we need.
2693 buf = (UChar *)ut->p;
2694 } else if (ut->c == neededIndex) {
2695 // The second buffer (buffer q) has what we need.
2696 buf = (UChar *)ut->q;
2698 // Neither buffer already has what we need.
2699 // Load new data from the character iterator.
2700 // Use the buf that is not the current buffer.
2701 buf = (UChar *)ut->p;
2702 if (ut->p == ut->chunkContents) {
2703 buf = (UChar *)ut->q;
2705 ci->setIndex(neededIndex);
2706 for (i=0; i<CIBufSize; i++) {
2707 buf[i] = ci->nextPostInc();
2708 if (i+neededIndex > ut->a) {
2714 // We have a buffer with the data we need.
2715 // Set it up as the current chunk, if it wasn't already.
2716 if (needChunkSetup) {
2717 ut->chunkContents = buf;
2718 ut->chunkLength = CIBufSize;
2719 ut->chunkNativeStart = neededIndex;
2720 ut->chunkNativeLimit = neededIndex + CIBufSize;
2721 if (ut->chunkNativeLimit > ut->a) {
2722 ut->chunkNativeLimit = ut->a;
2723 ut->chunkLength = (int32_t)(ut->chunkNativeLimit)-(int32_t)(ut->chunkNativeStart);
2725 ut->nativeIndexingLimit = ut->chunkLength;
2726 U_ASSERT(ut->chunkOffset>=0 && ut->chunkOffset<=CIBufSize);
2728 ut->chunkOffset = clippedIndex - (int32_t)ut->chunkNativeStart;
2729 UBool success = (forward? ut->chunkOffset<ut->chunkLength : ut->chunkOffset>0);
2733 static UText * U_CALLCONV
2734 charIterTextClone(UText *dest, const UText *src, UBool deep, UErrorCode * status) {
2735 if (U_FAILURE(*status)) {
2740 // There is no CharacterIterator API for cloning the underlying text storage.
2741 *status = U_UNSUPPORTED_ERROR;
2744 CharacterIterator *srcCI =(CharacterIterator *)src->context;
2745 srcCI = srcCI->clone();
2746 dest = utext_openCharacterIterator(dest, srcCI, status);
2747 if (U_FAILURE(*status)) {
2750 // cast off const on getNativeIndex.
2751 // For CharacterIterator based UTexts, this is safe, the operation is const.
2752 int64_t ix = utext_getNativeIndex((UText *)src);
2753 utext_setNativeIndex(dest, ix);
2754 dest->r = srcCI; // flags that this UText owns the CharacterIterator
2759 static int32_t U_CALLCONV
2760 charIterTextExtract(UText *ut,
2761 int64_t start, int64_t limit,
2762 UChar *dest, int32_t destCapacity,
2765 if(U_FAILURE(*status)) {
2768 if(destCapacity<0 || (dest==NULL && destCapacity>0) || start>limit) {
2769 *status=U_ILLEGAL_ARGUMENT_ERROR;
2772 int32_t length = (int32_t)ut->a;
2773 int32_t start32 = pinIndex(start, length);
2774 int32_t limit32 = pinIndex(limit, length);
2779 CharacterIterator *ci = (CharacterIterator *)ut->context;
2780 ci->setIndex32(start32); // Moves ix to lead of surrogate pair, if needed.
2781 srci = ci->getIndex();
2783 while (srci<limit32) {
2784 UChar32 c = ci->next32PostInc();
2785 int32_t len = U16_LENGTH(c);
2786 U_ASSERT(desti+len>0); /* to ensure desti+len never exceeds MAX_INT32, which must not happen logically */
2787 if (desti+len <= destCapacity) {
2788 U16_APPEND_UNSAFE(dest, desti, c);
2789 copyLimit = srci+len;
2792 *status = U_BUFFER_OVERFLOW_ERROR;
2797 charIterTextAccess(ut, copyLimit, TRUE);
2799 u_terminateUChars(dest, destCapacity, desti, status);
2803 static const struct UTextFuncs charIterFuncs =
2806 0, 0, 0, // Reserved alignment padding
2810 charIterTextExtract,
2813 NULL, // MapOffsetToNative,
2814 NULL, // MapIndexToUTF16,
2823 U_CAPI UText * U_EXPORT2
2824 utext_openCharacterIterator(UText *ut, CharacterIterator *ci, UErrorCode *status) {
2825 if (U_FAILURE(*status)) {
2829 if (ci->startIndex() > 0) {
2830 // No support for CharacterIterators that do not start indexing from zero.
2831 *status = U_UNSUPPORTED_ERROR;
2835 // Extra space in UText for 2 buffers of CIBufSize UChars each.
2836 int32_t extraSpace = 2 * CIBufSize * sizeof(UChar);
2837 ut = utext_setup(ut, extraSpace, status);
2838 if (U_SUCCESS(*status)) {
2839 ut->pFuncs = &charIterFuncs;
2841 ut->providerProperties = 0;
2842 ut->a = ci->endIndex(); // Length of text
2843 ut->p = ut->pExtra; // First buffer
2844 ut->b = -1; // Native index of first buffer contents
2845 ut->q = (UChar*)ut->pExtra+CIBufSize; // Second buffer
2846 ut->c = -1; // Native index of second buffer contents
2848 // Initialize current chunk contents to be empty.
2849 // First access will fault something in.
2850 // Note: The initial nativeStart and chunkOffset must sum to zero
2851 // so that getNativeIndex() will correctly compute to zero
2852 // if no call to Access() has ever been made. They can't be both
2853 // zero without Access() thinking that the chunk is valid.
2854 ut->chunkContents = (UChar *)ut->p;
2855 ut->chunkNativeStart = -1;
2856 ut->chunkOffset = 1;
2857 ut->chunkNativeLimit = 0;
2858 ut->chunkLength = 0;
2859 ut->nativeIndexingLimit = ut->chunkOffset; // enables native indexing