2 # Copyright (C) 2002-2013, International Business Machines Corporation
3 # and others. All Rights Reserved.
8 # See Unicode Standard Annex #29.
9 # These rules are based on UAX #29 Revision 22 for Unicode Version 6.3
11 # Note: Updates to word.txt will usually need to be merged into
12 # word_POSIX.txt also.
14 ##############################################################################
16 # Character class definitions from TR 29
18 ##############################################################################
24 # Character Class Definitions.
27 $CR = [\p{Word_Break = CR}];
28 $LF = [\p{Word_Break = LF}];
29 $Newline = [\p{Word_Break = Newline}];
30 $Extend = [\p{Word_Break = Extend}];
31 $Regional_Indicator = [\p{Word_Break = Regional_Indicator}];
32 $Format = [\p{Word_Break = Format}];
33 $Katakana = [\p{Word_Break = Katakana}];
34 $Hebrew_Letter = [\p{Word_Break = Hebrew_Letter}];
35 $ALetter = [\p{Word_Break = ALetter}];
36 $Single_Quote = [\p{Word_Break = Single_Quote}];
37 $Double_Quote = [\p{Word_Break = Double_Quote}];
38 # Remove two full stop characters from $MidNumLet and add them to $MidNum
39 # to break a hostname into its components at the cost of breaking
40 # 'e.g.' and 'i.e.' as well.
41 # $MidNumLet is used in rules 6/7 (rules of our interest) and rules 11/12.
42 # Because it's OR'd with $MidNum in rules 11/12, rules 11/12 are not affected
43 # while rules 6/7 are reverted to the old behavior we want.
44 $MidNumLet = [[\p{Word_Break = MidNumLet}] - [\u002E \uFF0E]];
45 $MidLetter = [\p{Word_Break = MidLetter}];
46 $MidNum = [\p{Word_Break = MidNum}[\u002E \uFF0E]];
47 $Numeric = [\p{Word_Break = Numeric}[\uff10-\uff19]]; #includes fullwidth digits
48 $ExtendNumLet = [\p{Word_Break = ExtendNumLet}];
51 $Hiragana = [:Hiragana:];
54 # Dictionary character set, for triggering language-based break engines. Currently
55 # limited to LineBreak=Complex_Context. Note that this set only works in Unicode
56 # 5.0 or later as the definition of Complex_Context was corrected to include all
57 # characters requiring dictionary break.
59 $Control = [\p{Grapheme_Cluster_Break = Control}];
60 $HangulSyllable = [\uac00-\ud7a3];
61 $ComplexContext = [:LineBreak = Complex_Context:];
62 $KanaKanji = [$Han $Hiragana $Katakana];
63 $dictionaryCJK = [$KanaKanji $HangulSyllable];
64 $dictionary = [$ComplexContext $dictionaryCJK];
66 # leave CJK scripts out of ALetterPlus
67 $ALetterPlus = [$ALetter-$dictionaryCJK [$ComplexContext-$Extend-$Control]];
71 # Rules 4 Ignore Format and Extend characters,
72 # except when they appear at the beginning of a region of text.
74 # TODO: check if handling of katakana in dictionary makes rules incorrect/void
75 $KatakanaEx = $Katakana ($Extend | $Format)*;
76 $Hebrew_LetterEx = $Hebrew_Letter ($Extend | $Format)*;
77 $ALetterEx = $ALetterPlus ($Extend | $Format)*;
78 $Single_QuoteEx = $Single_Quote ($Extend | $Format)*;
79 $Double_QuoteEx = $Double_Quote ($Extend | $Format)*;
80 $MidNumLetEx = $MidNumLet ($Extend | $Format)*;
81 $MidLetterEx = $MidLetter ($Extend | $Format)*;
82 $MidNumEx = $MidNum ($Extend | $Format)*;
83 $NumericEx = $Numeric ($Extend | $Format)*;
84 $ExtendNumLetEx = $ExtendNumLet ($Extend | $Format)*;
85 $Regional_IndicatorEx = $Regional_Indicator ($Extend | $Format)*;
87 $Ideographic = [\p{Ideographic}];
88 $HiraganaEx = $Hiragana ($Extend | $Format)*;
89 $IdeographicEx = $Ideographic ($Extend | $Format)*;
91 ## -------------------------------------------------
100 # Rule 4 - ignore Format and Extend characters, except when they appear at the beginning
101 # of a region of Text. The rule here comes into play when the start of text
102 # begins with a group of Format chars, or with a "word" consisting of a single
103 # char that is not in any of the listed word break categories followed by
104 # format char(s), or is not a CJK dictionary character.
105 [^$CR $LF $Newline]? ($Extend | $Format)+;
109 $HangulSyllable {200};
110 $Hebrew_LetterEx{200};
111 $KatakanaEx {400}; # note: these status values override those from rule 5
112 $HiraganaEx {400}; # by virtue of being numerically larger.
113 $IdeographicEx {400}; #
117 # Do not break between most letters.
119 ($ALetterEx | $Hebrew_LetterEx) ($ALetterEx | $Hebrew_LetterEx) {200};
122 ($ALetterEx | $Hebrew_LetterEx) ($MidLetterEx | $MidNumLetEx | $Single_QuoteEx) ($ALetterEx | $Hebrew_LetterEx) {200};
125 $Hebrew_LetterEx $Single_QuoteEx {200};
128 $Hebrew_LetterEx $Double_QuoteEx $Hebrew_LetterEx {200};
132 $NumericEx $NumericEx {100};
136 ($ALetterEx | $Hebrew_LetterEx) $NumericEx {200};
140 $NumericEx ($ALetterEx | $Hebrew_LetterEx) {200};
144 $NumericEx ($MidNumEx | $MidNumLetEx | $Single_QuoteEx) $NumericEx {100};
147 # to be consistent with $KanaKanji $KanaKanhi, changed
149 # See also TestRuleStatus in intltest/rbbiapts.cpp
150 $KatakanaEx $KatakanaEx {400};
154 $ALetterEx $ExtendNumLetEx {200}; # (13a)
155 $Hebrew_LetterEx $ExtendNumLetEx {200}; # (13a)
156 $NumericEx $ExtendNumLetEx {100}; # (13a)
157 $KatakanaEx $ExtendNumLetEx {400}; # (13a)
158 $ExtendNumLetEx $ExtendNumLetEx {200}; # (13a)
160 $ExtendNumLetEx $ALetterEx {200}; # (13b)
161 $ExtendNumLetEx $Hebrew_Letter {200}; # (13b)
162 $ExtendNumLetEx $NumericEx {100}; # (13b)
163 $ExtendNumLetEx $KatakanaEx {400}; # (13b)
167 $Regional_IndicatorEx $Regional_IndicatorEx;
169 # special handling for CJK characters: chain for later dictionary segmentation
170 $HangulSyllable $HangulSyllable {200};
171 $KanaKanji $KanaKanji {400}; # different rule status if both kana and kanji found
174 ## -------------------------------------------------
178 $BackHebrew_LetterEx = ($Format | $Extend)* $Hebrew_Letter;
179 $BackALetterEx = ($Format | $Extend)* $ALetterPlus;
180 $BackSingle_QuoteEx = ($Format | $Extend)* $Single_Quote;
181 $BackDouble_QuoteEx = ($Format | $Extend)* $Double_Quote;
182 $BackMidNumLetEx = ($Format | $Extend)* $MidNumLet;
183 $BackNumericEx = ($Format | $Extend)* $Numeric;
184 $BackMidNumEx = ($Format | $Extend)* $MidNum;
185 $BackMidLetterEx = ($Format | $Extend)* $MidLetter;
186 $BackKatakanaEx = ($Format | $Extend)* $Katakana;
187 $BackHiraganaEx = ($Format | $Extend)* $Hiragana;
188 $BackExtendNumLetEx = ($Format | $Extend)* $ExtendNumLet;
189 $BackRegional_IndicatorEx = ($Format | $Extend)* $Regional_Indicator;
195 ($Format | $Extend)* [^$CR $LF $Newline]?;
199 ($BackALetterEx | $BackHebrew_LetterEx) ($BackALetterEx | $BackHebrew_LetterEx);
203 ($BackALetterEx | $BackHebrew_LetterEx) ($BackMidLetterEx | $BackMidNumLetEx | $BackSingle_QuoteEx) ($BackALetterEx | $BackHebrew_LetterEx);
206 $BackSingle_QuoteEx $BackHebrew_LetterEx;
209 $BackHebrew_LetterEx $BackDouble_QuoteEx $BackHebrew_LetterEx;
213 $BackNumericEx $BackNumericEx;
217 $BackNumericEx ($BackALetterEx | $BackHebrew_LetterEx);
221 ($BackALetterEx | $BackHebrew_LetterEx) $BackNumericEx;
225 $BackNumericEx ($BackMidNumEx | $BackMidNumLetEx | $BackSingle_QuoteEx) $BackNumericEx;
229 $BackKatakanaEx $BackKatakanaEx;
233 $BackExtendNumLetEx ($BackALetterEx | $BackHebrew_LetterEx | $BackNumericEx | $BackKatakanaEx | $BackExtendNumLetEx);
234 ($BackALetterEx | $BackHebrew_LetterEx | $BackNumericEx | $BackKatakanaEx) $BackExtendNumLetEx;
238 $BackRegional_IndicatorEx $BackRegional_IndicatorEx;
240 # special handling for CJK characters: chain for later dictionary segmentation
241 $HangulSyllable $HangulSyllable;
242 $KanaKanji $KanaKanji; #different rule status if both kanji and kana found
244 ## -------------------------------------------------
249 ($Extend | $Format)+ .?;
252 ($MidLetter | $MidNumLet | $Single_Quote) ($BackALetterEx | $BackHebrew_LetterEx);
255 $Double_Quote $BackHebrew_LetterEx;
259 ($MidNum | $MidNumLet | $Single_Quote) $BackNumericEx;
261 # For dictionary-based break
262 $dictionary $dictionary;
264 ## -------------------------------------------------
269 ($Extend | $Format)+ .?;
272 ($MidLetterEx | $MidNumLetEx | $Single_QuoteEx) ($ALetterEx | $Hebrew_LetterEx);
275 $Double_QuoteEx $Hebrew_LetterEx;
278 ($MidNumEx | $MidNumLetEx | $Single_QuoteEx) $NumericEx;
280 # For dictionary-based break
281 $dictionary $dictionary;