2 * Copyright (c) 2021 Samsung Electronics Co., Ltd.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
19 #include <dali/devel-api/text-abstraction/script.h>
23 namespace TextAbstraction
27 constexpr unsigned int WHITE_SPACE_THRESHOLD = 0x21; ///< All characters below 0x21 are considered white spaces.
28 constexpr unsigned int CHAR_LF = 0x000A; ///< NL Line feed, new line.
29 constexpr unsigned int CHAR_VT = 0x000B; ///< Vertical tab.
30 constexpr unsigned int CHAR_FF = 0x000C; ///< NP Form feed, new page.
31 constexpr unsigned int CHAR_CR = 0x000D; ///< Carriage return, new line.
32 constexpr unsigned int CHAR_NEL = 0x0085; ///< Next line.
33 constexpr unsigned int CHAR_LS = 0x2028; ///< Line separator.
34 constexpr unsigned int CHAR_PS = 0x2029; ///< Paragraph separator
36 constexpr unsigned int CHAR_ZWS = 0x200B; ///< Zero width space.
37 constexpr unsigned int CHAR_ZWNJ = 0x200C; ///< Zero width non joiner.
38 constexpr unsigned int CHAR_ZWJ = 0x200D; ///< Zero width joiner.
39 constexpr unsigned int CHAR_LTRM = 0x200E; ///< Left to Right Mark.
40 constexpr unsigned int CHAR_RTLM = 0x200F; ///< Right to Left Mark.
41 constexpr unsigned int CHAR_TS = 0x2009; ///< Thin Space.
43 // Latin script: It contains punctuation characters and symbols which are not part of the latin script. https://en.wikipedia.org/wiki/Latin_script_in_Unicode
44 // 0x0000 - 0x007f C0 Controls and Basic Latin
46 // ASCII digits (not part of LATIN script):
49 // ASCII punctuation and symbols (not part of LATIN script):
55 // Controls (not part of LATIN script):
58 // 0x0080 - 0x00ff C1 Controls and Latin-1 Supplement
60 // Controls (not part of LATIN script):
63 // Punctuations and symbols (not part of LATIN script):
66 // Mathematical operators (not part of LATIN script):
70 // 0x0100 - 0x017f Latin Extended-A
71 // 0x0180 - 0x024f Latin Extended-B
72 // 0x0250 - 0x02af IPA Extensions
73 // 0x02b0 - 0x02ff Spacing Modifier Letters
75 // Punctuation (not part of LATIN script):
78 // 0x1d00 - 0x1d7f Phonetic Extensions
80 // Uralic Phonetic (not part of LATIN script):
83 // Subscripts and superscripts
88 // 0x1d80 - 0x1dbf Phonetic Extensions Supplement
90 // 0x1DBF (subscript or superscript. Not part of LATIN script )
92 // 0x1e00 - 0x1eff Latin Extended Additional
93 // 0x2070 - 0x209f Superscripts and Subscripts
95 // 0x2070 (not part of LATIN script)
96 // 0x2074 - 0x207E (not part of LATIN script)
98 // 0x2100 - 0x214f Letterlike symbols (not part of LATIN script)
100 // 0x212A - 0x212B (are part of LATIN script)
101 // 0x2132 (are part of LATIN script)
102 // 0x214E (are part of LATIN script)
104 // 0x2150 - 0x2189 Number Forms
106 // 0x2150 - 0x215F Fractions (not part of LATIN script)
107 // 0x2189 Fractions (not part of LATIN script)
109 // 0x2c60 - 0x2c7f Latin Extended-C
110 // 0xa720 - 0xa7ff Latin Extended-D
112 // 0xA720 - 0xA721 Uralic Phonetic (not part of LATIN script)
113 // 0xA788 (not part of LATIN script)
114 // 0xA789 - 0xA78A Budu (not part of LATIN script)
116 // 0xab30 - 0xab6f Latin Extended-E
118 // 0xfb00 - 0xfb06 Latin Alphabetic Presentation Forms
119 // 0xff00 - 0xffef Halfwidth and Fullwidth Forms
121 // 0xFF00 - 0xFF20 HWFW Symbols (not part of LATIN script)
122 // 0xFF3B - 0xFF40 HWFW Symbols (not part of LATIN script)
123 // 0xFF5B - 0xFFEF HWFW Symbols (not part of LATIN script)
126 // 0x0900 - 0x097f Devanagari
127 // 0x0980 - 0x09ff Bengali
128 // 0x0a00 - 0x0a7f Gurmukhi
129 // 0x0a80 - 0x0aff Gujarati
130 // 0x0b00 - 0x0b7f Oriya
131 // 0x0b80 - 0x0bff Tamil
132 // 0x0c00 - 0x0c7f Telugu
133 // 0x0c80 - 0x0cff Kannada
134 // 0x0d00 - 0x0d7f Malayalam
137 // 0x0d80 - 0x0dff Sinhala
140 // 0x0600 - 0x06ff Arabic
141 // 0x0750 - 0x077f Arabic Supplement
142 // 0x08A0 - 0x08ff Arabic Extended-A
143 // 0xfb50 - 0xfdff Arabic Presentation Forms-A
144 // 0xfe70 - 0xfeff Arabic Presentation Forms-B
145 // 0x1ee00 - 0x1eeff Arabic Mathematical Alphabetic Symbols
147 // CJK (Chinese, Japanese and Korean) and Vietnamese script.
148 // 0x2e80 - 0x2eff CJK Radicals Supplement
149 // 0x2f00 - 0x2fdf Kangxi Radicals
150 // 0x3000 - 0x303f CJK Symbols and Punctuation
151 // 0x3200 - 0x32ff Enclosed CJK Letters and Months
152 // 0x3400 - 0x4dbf CJK Unified Ideographs Extension A
153 // 0x4e00 - 0x62ff CJK Unified Ideographs
154 // 0x6300 - 0x77ff CJK Unified Ideographs
155 // 0x7800 - 0x8cff CJK Unified Ideographs
156 // 0x8d00 - 0x9fff CJK Unified Ideographs
157 // 0x20000 - 0x215ff CJK Unified Ideographs Extension B
158 // 0x21600 - 0x230ff CJK Unified Ideographs Extension B
159 // 0x23100 - 0x245ff CJK Unified Ideographs Extension B
160 // 0x24600 - 0x260ff CJK Unified Ideographs Extension B
161 // 0x26100 - 0x275ff CJK Unified Ideographs Extension B
162 // 0x27600 - 0x290ff CJK Unified Ideographs Extension B
163 // 0x29100 - 0x2a6df CJK Unified Ideographs Extension B
164 // 0x2a700 - 0x2b73f CJK Unified Ideographs Extension C
165 // 0x2b740 - 0x2b81f CJK Unified Ideographs Extension D
168 // 0x3040 - 0x309f Hiragana
169 // 0x30a0 - 0x30ff Katakana
172 // 0x1100 - 0x11ff Hangul jamo
173 // 0x3130 - 0x318f Hangul Compatibility Jamo
174 // 0xa960 - 0xa97f Hangul Jamo Extended-A
175 // 0xac00 - 0xd7af Hangul Syllables
176 // 0xd7b0 - 0xd7ff Hangul Jamo Extended-B
179 // 0x3100 - 0x312f Bopomofo
180 // 0x31a0 - 0x31bf Bopomofo Extended
183 // 0x1780 - 0x17ff Khmer
184 // 0x19e0 - 0x19ff Khmer Symbols
187 // 0x0e80 - 0x0eff Lao
190 // 0x0e00 - 0x0e7f Thai
193 // 0x1000 - 0x109f Myanmar
196 // 0x0591 - 0x05f4 Hebrew
197 // 0xfb1d - 0xfb4f Hebrew subset of Alphabetic Presentation Forms
200 // 0x0400 - 0x04ff Cyrillic
201 // 0x0500 - 0x052f Cyrillic suplement
202 // 0x2de0 - 0x2dff Cyrillic Extended-A
203 // 0xa640 - 0xa69f Cyrillic Extended-B
206 // 0x10a0 - 0x10ff Georgian
207 // 0x2d00 - 0x2d2f Georgian suplement
210 // 0x0370 - 0x03ff Greek & Coptic
211 // 0x1f00 - 0x1fff Greek Extended
214 // 0x0530 - 0x058f Armenian
215 // 0xfb13 - 0xfb17 Armenian subset of Alphabetic prefentation forms
218 // 0xa980 - 0xa9fd Javanese
221 // 0x1b80 - 0x1bbf Sundanese
222 // 0x1cc0 - 0x1ccf Sundanese supplement
224 // Ge'ez script (Ethiopic)
225 // 0x1200 - 0x137f Ethiopic
226 // 0x1380 - 0x139f Ethiopic supplement
227 // 0x2d80 - 0x2ddf Ethiopic Extended
228 // 0xab00 - 0xab2f Ethiopic Extended-A
231 // 0x1700 - 0x171f Baybayin
234 // 0x1c50 - 0x1c7f Ol Chiki
237 // 0xabc0 - 0xabff Meetei Mayek
238 // 0xaae0 - 0xaaff Meetei Mayek Extensions
240 // The Emoji which map to standardized Unicode characters
241 // 1. Emoticons ( 1F601 - 1F64F )
242 // 2. Dingbats ( 2700 - 27BF )
243 // 3. Transport and map symbols ( 1F680 - 1F6C0 )
244 // 4. Enclosed characters ( 24C2 - 1F251 )
245 // 5. Uncategorized :-S
246 // 6. Additional Emoticons ( 1F600 - 1F636 )
247 // 6b. Additional transport and map symbols ( 1F680 - 1F6FF ): http://unicode.org/charts/PDF/U1F680.pdf
248 // 6c. Other additional symbols ( 1F30D - 1F567 )
249 // 7. Supplemental Symbols and Pictographs ( 1F900–1F9FF ): http://unicode.org/charts/PDF/U1F900.pdf
251 // Symbols. Work around for these symbols.
264 /// character <= 0x077f
265 inline Script GetScriptTillArabicSupplement(Character character)
267 Script script = UNKNOWN;
269 if((0x0030 <= character) && (character <= 0x0039))
271 script = ASCII_DIGITS;
273 else if(character <= 0x007E)
275 if((0x0020 <= character) && (character <= 0x002F))
279 else if((0x003A <= character) && (character <= 0x0040))
283 else if((0x005B <= character) && (character <= 0x0060))
287 else if((0x007B <= character) && (character <= 0x007E))
296 else if((0x007F <= character) && (character <= 0x009F))
298 // 0x007F is actually part of C0 Controls and Basic Latin. However, is the last and only control character of its block
299 // and the following characters of the next block are consecutive.
300 script = C1_CONTROLS;
302 else if((0x00A0 <= character) && (character <= 0x00BF))
304 if(character == 0x00A9)
306 script = EMOJI; // 5. Uncategorized: copyright sign
308 else if(character == 0x00AE)
310 script = EMOJI; // 5. Uncategorized: registered sign
317 else if(character == 0x00D7)
321 else if(character == 0x00F7)
325 else if((0x00C0 <= character) && (character <= 0x02ff))
327 if((0x02B9 <= character) && (character <= 0x02BF))
336 else if((0x0370 <= character) && (character <= 0x03ff))
340 else if((0x0400 <= character) && (character <= 0x04ff))
344 else if((0x0500 <= character) && (character <= 0x052f))
348 else if((0x0530 <= character) && (character <= 0x058f))
352 else if((0x0591 <= character) && (character <= 0x05f4))
356 else if((0x0600 <= character) && (character <= 0x06ff))
360 else if((0x0750 <= character) && (character <= 0x077f))
368 /// character <= 0x09ff
369 inline Script GetScriptTillBengali(Character character)
371 Script script = UNKNOWN;
373 if(character <= 0x077f)
375 script = GetScriptTillArabicSupplement(character);
379 if((0x08A0 <= character) && (character <= 0x08ff))
383 else if((0x0900 <= character) && (character <= 0x097f))
387 else if((0x0980 <= character) && (character <= 0x09ff))
396 /// 0x09ff < character <= 0x0cff
397 inline Script GetScriptBetweenBengaliAndKannada(Character character)
399 Script script = UNKNOWN;
401 if(character <= 0x0b7f)
403 if((0x0a00 <= character) && (character <= 0x0a7f))
407 else if((0x0a80 <= character) && (character <= 0x0aff))
411 else if((0x0b00 <= character) && (character <= 0x0b7f))
418 if((0x0b80 <= character) && (character <= 0x0bff))
422 else if((0x0c00 <= character) && (character <= 0x0c7f))
426 else if((0x0c80 <= character) && (character <= 0x0cff))
435 /// 0x0cff < character <= 0x1eff
436 inline Script GetScriptBetweenKannadaAndLatinExtendedAdditional(Character character)
438 Script script = UNKNOWN;
440 if((0x0d00 <= character) && (character <= 0x0d7f))
444 else if((0x0d80 <= character) && (character <= 0x0dff))
448 else if((0x0e00 <= character) && (character <= 0x0e7f))
452 else if((0x0e80 <= character) && (character <= 0x0eff))
456 else if((0x1000 <= character) && (character <= 0x109f))
460 else if((0x10a0 <= character) && (character <= 0x10ff))
464 else if((0x1100 <= character) && (character <= 0x11ff))
468 else if((0x1200 <= character) && (character <= 0x137f))
472 else if((0x1380 <= character) && (character <= 0x139f))
476 else if((0x1700 <= character) && (character <= 0x171f))
480 else if((0x1780 <= character) && (character <= 0x17ff))
484 else if((0x19e0 <= character) && (character <= 0x19ff))
488 else if((0x1b80 <= character) && (character <= 0x1bbf))
492 else if((0x1c50 <= character) && (character <= 0x1c7f))
496 else if((0x1cc0 <= character) && (character <= 0x1ccf))
500 else if((0x1d00 <= character) && (character <= 0x1eff))
502 if((0x1D26 <= character) && (character <= 0x1D2B))
506 else if((0x1D5D <= character) && (character <= 0x1D61))
508 script = PHONETIC_SS;
510 else if((0x1D66 <= character) && (character <= 0x1D6A))
512 script = PHONETIC_SS;
514 else if(character == 0x1D78)
516 script = PHONETIC_SS;
518 else if(character == 0x1DBF)
520 script = PHONETIC_SS;
531 /// 0x1eff < character <= 0x2c7f
532 inline Script GetScriptBetweenLatinExtendedAdditionalAndLatinExtendedC(Character character)
534 Script script = UNKNOWN;
536 if((0x1f00 <= character) && (character <= 0x1fff))
540 else if(character == 0x203c)
542 script = EMOJI; // 5. Uncategorized: double exclamation mark
544 else if(character == 0x2049)
546 script = EMOJI; // 5. Uncategorized: exclamation question mark
548 else if((0x2070 <= character) && (character <= 0x209f))
550 if(character == 0x2070)
554 else if((0x2074 <= character) && (character <= 0x207E))
563 else if(character == 0x20e3)
565 script = EMOJI; // 5. Uncategorized: combining enclosing keycap
567 else if(character == 0x2122)
569 script = EMOJI; // 5. Uncategorized: trade mark sign
571 else if(character == 0x2139)
573 script = EMOJI; // 5. Uncategorized: information source
575 else if((0x2100 <= character) && (character <= 0x2189))
577 if((0x2100 <= character) && (character <= 0x214f))
579 if((0x212A <= character) && (character <= 0x212B))
583 else if(character == 0x2132)
587 else if(character == 0x214E)
593 script = LETTER_LIKE;
596 else if((0x2150 <= character) && (character <= 0x215F))
598 script = FRACTIONS_NF;
600 else if(character == 0x2189)
602 script = FRACTIONS_NF;
610 else if((0x25cb == character) ||
611 (0x25cf == character) ||
612 (0x25a1 == character))
616 else if(0x25a0 == character)
620 else if((0x2664 == character) ||
621 (0x2661 == character) ||
622 (0x2662 == character) ||
623 (0x2667 == character))
627 else if((0x2606 == character) ||
628 (0x25aa == character))
632 else if(0x262a == character)
636 // U+2194 5. Uncategorized: left right arrow
637 // U+2B55 5. Uncategorized: heavy large circle
638 else if((0x2194 <= character) && (character <= 0x2B55))
642 else if((0x2c60 <= character) && (character <= 0x2c7f))
650 /// 0x0cff < character <= 0x2c7f
651 inline Script GetScriptBetweenKannadaAndLatinExtendedC(Character character)
653 Script script = UNKNOWN;
655 if(character <= 0x1eff)
657 script = GetScriptBetweenKannadaAndLatinExtendedAdditional(character);
661 script = GetScriptBetweenLatinExtendedAdditionalAndLatinExtendedC(character);
667 /// 0x2c7f < character <= 0xa7ff
668 inline Script GetScriptBetweenLatinExtendedCAndLatinExtendedD(Character character)
670 Script script = UNKNOWN;
672 if((0x2d00 <= character) && (character <= 0x2d2f))
676 else if((0x2d80 <= character) && (character <= 0x2ddf))
680 else if((0x2de0 <= character) && (character <= 0x2dff))
684 else if((0x2e80 <= character) && (character <= 0x2eff))
688 else if((0x2f00 <= character) && (character <= 0x2fdf))
692 else if((0x3000 <= character) && (character <= 0x303f))
696 else if((0x3040 <= character) && (character <= 0x309f))
700 else if((0x30a0 <= character) && (character <= 0x30ff))
704 else if((0x3100 <= character) && (character <= 0x312f))
708 else if((0x3130 <= character) && (character <= 0x318f))
712 else if((0x31a0 <= character) && (character <= 0x31bf))
716 else if((0x3200 <= character) && (character <= 0x32ff))
720 else if((0x3400 <= character) && (character <= 0x4dbf))
724 else if((0x4e00 <= character) && (character <= 0x62ff))
728 else if((0x6300 <= character) && (character <= 0x77ff))
732 else if((0x7800 <= character) && (character <= 0x8cff))
736 else if((0x8d00 <= character) && (character <= 0x9fff))
740 else if((0xa640 <= character) && (character <= 0xa69f))
744 else if((0xa720 <= character) && (character <= 0xa7ff))
746 if(character == 0xA720)
750 else if(character == 0xA721)
754 else if(character == 0xA788)
756 script = NON_LATIN_LED;
758 else if(character == 0xA789)
760 script = NON_LATIN_LED;
762 else if(character == 0xA78A)
764 script = NON_LATIN_LED;
775 /// 0x2c7f < character <= 0xfdff
776 inline Script GetScriptBetweenLatinExtendedCAndArabicPresentationFormsA(Character character)
778 Script script = GetScriptBetweenLatinExtendedCAndLatinExtendedD(character);
780 if((0xa960 <= character) && (character <= 0xa97f))
784 else if((0xa980 <= character) && (character <= 0xa9fd))
788 else if((0xab00 <= character) && (character <= 0xab2f))
792 else if((0xab30 <= character) && (character <= 0xab6f))
796 else if((0xaae0 <= character) && (character <= 0xaaff))
800 else if((0xabc0 <= character) && (character <= 0xabff))
804 else if((0xac00 <= character) && (character <= 0xd7af))
808 else if((0xd7b0 <= character) && (character <= 0xd7ff))
812 else if((0xfb00 <= character) && (character <= 0xfb06))
816 else if((0xfb13 <= character) && (character <= 0xfb17))
820 else if((0xfb1d <= character) && (character <= 0xfb4f))
824 else if((0xfb50 <= character) && (character <= 0xfdff))
832 /// character > 0xfdff
833 inline Script GetScriptAboveArabicPresentationFormsA(Character character)
835 Script script = UNKNOWN;
837 if((0xfe70 <= character) && (character <= 0xfeff))
841 else if((0xff00 <= character) && (character <= 0xffef))
843 if((0xFF00 <= character) && (character <= 0xFF20))
847 else if((0xFF3B <= character) && (character <= 0xFF40))
851 else if((0xFF5B <= character) && (character <= 0xFFEF))
860 else if((0x1ee00 <= character) && (character <= 0x1eeff))
864 // U+1f170 4. Enclosed characters: negative squared latin capital letter A
865 // U+1f6ff 6b. Additional transport and map symbols
866 else if((0x1f170 <= character) && (character <= 0x1f6ff))
870 // 7. Supplemental Symbols and Pictographs
871 else if((0x1f900 <= character) && (character <= 0x1f9ff))
875 else if((0x20000 <= character) && (character <= 0x215ff))
879 else if((0x21600 <= character) && (character <= 0x230ff))
883 else if((0x23100 <= character) && (character <= 0x245ff))
887 else if((0x24600 <= character) && (character <= 0x260ff))
891 else if((0x26100 <= character) && (character <= 0x275ff))
895 else if((0x27600 <= character) && (character <= 0x290ff))
899 else if((0x29100 <= character) && (character <= 0x2a6df))
903 else if((0x2a700 <= character) && (character <= 0x2b73f))
907 else if((0x2b740 <= character) && (character <= 0x2b81f))
915 /// character > 0x2c7f
916 inline Script GetScriptAboveLatinExtendedC(Character character)
918 Script script = UNKNOWN;
920 if(character <= 0xfdff)
922 script = GetScriptBetweenLatinExtendedCAndArabicPresentationFormsA(character);
926 script = GetScriptAboveArabicPresentationFormsA(character);
934 bool IsRightToLeftScript(Script script)
936 return ((ARABIC == script) ||
940 Script GetCharacterScript(Character character)
942 Script script = UNKNOWN;
944 if(IsCommonScript(character))
948 else if(character <= 0x0cff)
950 if(character <= 0x09ff)
952 script = GetScriptTillBengali(character);
956 script = GetScriptBetweenBengaliAndKannada(character);
961 if(character <= 0x2c7f)
963 script = GetScriptBetweenKannadaAndLatinExtendedC(character);
967 script = GetScriptAboveLatinExtendedC(character);
974 bool IsWhiteSpace(Character character)
976 return character < WHITE_SPACE_THRESHOLD;
979 bool IsNewParagraph(Character character)
981 return ((CHAR_LF == character) ||
982 (CHAR_VT == character) ||
983 (CHAR_FF == character) ||
984 (CHAR_CR == character) ||
985 (CHAR_NEL == character) ||
986 (CHAR_LS == character) ||
987 (CHAR_PS == character));
990 bool IsZeroWidthNonJoiner(Character character)
992 return CHAR_ZWNJ == character;
995 bool IsZeroWidthJoiner(Character character)
997 return CHAR_ZWJ == character;
1000 bool IsZeroWidthSpace(Character character)
1002 return CHAR_ZWS == character;
1005 bool IsLeftToRightMark(Character character)
1007 return CHAR_LTRM == character;
1010 bool IsRightToLeftMark(Character character)
1012 return CHAR_RTLM == character;
1015 bool IsThinSpace(Character character)
1017 return CHAR_TS == character;
1020 bool IsCommonScript(Character character)
1022 return (IsWhiteSpace(character) ||
1023 IsZeroWidthNonJoiner(character) ||
1024 IsZeroWidthJoiner(character) ||
1025 IsZeroWidthSpace(character) ||
1026 IsLeftToRightMark(character) ||
1027 IsRightToLeftMark(character) ||
1028 IsThinSpace(character) ||
1029 IsNewParagraph(character));
1032 bool HasLigatureMustBreak(Script script)
1034 return ((LATIN == script) ||
1035 (ARABIC == script));
1038 } // namespace TextAbstraction