2 *******************************************************************************
4 * Copyright (C) 2001-2011, International Business Machines
5 * Corporation and others. All Rights Reserved.
7 *******************************************************************************
8 * file name: ucol_tok.h
10 * tab size: 8 (not used)
14 * created by: Vladimir Weinstein
16 * This module reads a tailoring rule string and produces a list of
17 * tokens that will be turned into collation elements
24 #include "unicode/utypes.h"
25 #include "unicode/uset.h"
27 #if !UCONFIG_NO_COLLATION
31 #include "unicode/parseerr.h"
33 #define UCOL_TOK_UNSET 0xFFFFFFFF
34 #define UCOL_TOK_RESET 0xDEADBEEF
36 #define UCOL_TOK_POLARITY_NEGATIVE 0
37 #define UCOL_TOK_POLARITY_POSITIVE 1
39 #define UCOL_TOK_TOP 0x04
40 #define UCOL_TOK_VARIABLE_TOP 0x08
41 #define UCOL_TOK_BEFORE 0x03
42 #define UCOL_TOK_SUCCESS 0x10
44 /* this is space for the extra strings that need to be unquoted */
45 /* during the parsing of the rules */
46 #define UCOL_TOK_EXTRA_RULE_SPACE_SIZE 4096
47 typedef struct UColToken UColToken;
59 uint32_t previousContCE;
60 int32_t pos[UCOL_STRENGTH_LIMIT];
61 uint32_t gapsLo[3*UCOL_CE_STRENGTH_LIMIT];
62 uint32_t gapsHi[3*UCOL_CE_STRENGTH_LIMIT];
63 uint32_t numStr[UCOL_CE_STRENGTH_LIMIT];
64 UColToken* fStrToken[UCOL_CE_STRENGTH_LIMIT];
65 UColToken* lStrToken[UCOL_CE_STRENGTH_LIMIT];
81 uint32_t polarity; /* 1 for <, <<, <<<, , ; and -1 for >, >>, >>> */
82 UColTokListHeader *listHeader;
85 UChar **rulesToParseHdl;
90 * This is a token that has been parsed
91 * but not yet processed. Used to reduce
92 * the number of arguments in the parser
98 uint32_t extensionOffset;
99 uint32_t extensionLen;
100 uint32_t prefixOffset;
103 uint16_t indirectIndex;
108 UColParsedToken parsedToken;
111 const UChar *current;
112 UChar *sourceCurrent;
115 const InverseUCATableHeader *invUCA;
116 const UCollator *UCA;
117 UHashtable *tailored;
120 uint32_t listCapacity;
121 UColTokListHeader *lh;
125 UBool buildCCTabFlag; /* Tailoring rule requirs building combining class table. */
127 UChar32 previousCp; /* Previous code point. */
128 /* For processing starred lists. */
129 UBool isStarred; /* Are we processing a starred token? */
130 UBool savedIsStarred;
131 uint32_t currentStarredCharIndex; /* Index of the current charrecter in the starred expression. */
132 uint32_t lastStarredCharIndex; /* Index to the last character in the starred expression. */
134 /* For processing ranges. */
135 UBool inRange; /* Are we in a range? */
136 UChar32 currentRangeCp; /* Current code point in the range. */
137 UChar32 lastRangeCp; /* The last code point in the range. */
139 /* reorder codes for collation reordering */
140 int32_t* reorderCodes;
141 int32_t reorderCodesLength;
146 const UChar *subName;
148 UColAttributeValue attrVal;
152 const UChar *optionName;
154 const ucolTokSuboption *subopts;
159 #define ucol_tok_isSpecialChar(ch) \
160 (((((ch) <= 0x002F) && ((ch) >= 0x0020)) || \
161 (((ch) <= 0x003F) && ((ch) >= 0x003A)) || \
162 (((ch) <= 0x0060) && ((ch) >= 0x005B)) || \
163 (((ch) <= 0x007E) && ((ch) >= 0x007D)) || \
168 uint32_t ucol_tok_assembleTokenList(UColTokenParser *src,
169 UParseError *parseError,
173 void ucol_tok_initTokenList(UColTokenParser *src,
175 const uint32_t rulesLength,
176 const UCollator *UCA,
177 GetCollationRulesFunction importFunc,
181 U_CFUNC void ucol_tok_closeTokenList(UColTokenParser *src);
183 U_CAPI const UChar* U_EXPORT2 ucol_tok_parseNextToken(UColTokenParser *src,
185 UParseError *parseError,
189 U_CAPI const UChar * U_EXPORT2
190 ucol_tok_getNextArgument(const UChar *start, const UChar *end,
191 UColAttribute *attrib, UColAttributeValue *value,
193 U_CAPI int32_t U_EXPORT2 ucol_inv_getNextCE(const UColTokenParser *src,
194 uint32_t CE, uint32_t contCE,
195 uint32_t *nextCE, uint32_t *nextContCE,
197 U_CFUNC int32_t U_EXPORT2 ucol_inv_getPrevCE(const UColTokenParser *src,
198 uint32_t CE, uint32_t contCE,
199 uint32_t *prevCE, uint32_t *prevContCE,
202 const UChar* U_CALLCONV ucol_tok_getRulesFromBundle(
209 #endif /* #if !UCONFIG_NO_COLLATION */