4 from __future__ import print_function, division, absolute_import
9 if len (sys.argv) != 5:
10 print ("usage: ./gen-use-table.py IndicSyllabicCategory.txt IndicPositionalCategory.txt UnicodeData.txt Blocks.txt", file=sys.stderr)
13 BLACKLISTED_BLOCKS = ["Thai", "Lao"]
15 files = [io.open (x, encoding='utf-8') for x in sys.argv[1:]]
17 headers = [[f.readline () for i in range (2)] for j,f in enumerate(files) if j != 2]
18 headers.append (["UnicodeData.txt does not have a header."])
20 data = [{} for f in files]
21 values = [{} for f in files]
22 for i, f in enumerate (files):
29 fields = [x.strip () for x in line.split (';')]
33 uu = fields[0].split ('..')
34 start = int (uu[0], 16)
40 t = fields[1 if i != 2 else 2]
42 for u in range (start, end + 1):
44 values[i][t] = values[i].get (t, 0) + end - start + 1
46 defaults = ('Other', 'Not_Applicable', 'Cn', 'No_Block')
48 # TODO Characters that are not in Unicode Indic files, but used in USE
49 data[0][0x034F] = defaults[0]
50 data[0][0x2060] = defaults[0]
51 # TODO https://github.com/roozbehp/unicode-data/issues/9
52 data[0][0x11C44] = 'Consonant_Placeholder'
53 data[0][0x11C45] = 'Consonant_Placeholder'
54 # TODO https://github.com/harfbuzz/harfbuzz/pull/1399
55 data[0][0x111C8] = 'Consonant_Placeholder'
56 for u in range (0xFE00, 0xFE0F + 1):
57 data[0][u] = defaults[0]
59 # Merge data into one dict:
60 for i,v in enumerate (defaults):
61 values[i][v] = values[i].get (v, 0) + 1
63 for i,d in enumerate (data):
64 for u,v in d.items ():
65 if i >= 2 and not u in combined:
68 combined[u] = list (defaults)
70 combined = {k:v for k,v in combined.items() if v[3] not in BLACKLISTED_BLOCKS}
78 'Cc', 'Cf', 'Cn', 'Co', 'Cs', 'Ll', 'Lm', 'Lo', 'Lt', 'Lu', 'Mc',
79 'Me', 'Mn', 'Nd', 'Nl', 'No', 'Pc', 'Pd', 'Pe', 'Pf', 'Pi', 'Po',
80 'Ps', 'Sc', 'Sk', 'Sm', 'So', 'Zl', 'Zp', 'Zs',
81 # Indic_Syllabic_Category
93 'Consonant_Placeholder',
96 'Consonant_With_Stacker',
98 'Consonant_Preceding_Repha',
99 'Consonant_Succeeding_Repha',
100 'Consonant_Subjoined',
103 'Consonant_Head_Letter',
104 'Consonant_Initial_Postfixed',
117 'Brahmi_Joining_Number',
118 # Indic_Positional_Category
129 'Top_And_Left_And_Right',
132 'Top_And_Bottom_And_Right',
141 class PropertyValue(object):
142 def __init__(self, name_):
146 def __eq__(self, other):
147 return self.name == (other if isinstance(other, basestring) else other.name)
148 def __ne__(self, other):
149 return not (self == other)
151 return hash(str(self))
155 for name in property_names:
156 value = PropertyValue(name)
157 assert value not in property_values
158 assert value not in globals()
159 property_values[name] = value
160 globals().update(property_values)
163 def is_BASE(U, UISC, UGC):
164 return (UISC in [Number, Consonant, Consonant_Head_Letter,
165 #SPEC-DRAFT Consonant_Placeholder,
167 Vowel_Independent #SPEC-DRAFT
169 (UGC == Lo and UISC in [Avagraha, Bindu, Consonant_Final, Consonant_Medial,
170 Consonant_Subjoined, Vowel, Vowel_Dependent]))
171 def is_BASE_IND(U, UISC, UGC):
172 #SPEC-DRAFT return (UISC in [Consonant_Dead, Modifying_Letter] or UGC == Po)
173 return (UISC in [Consonant_Dead, Modifying_Letter] or
174 (UGC == Po and not U in [0x104B, 0x104E, 0x2022, 0x111C8, 0x11A3F, 0x11A45, 0x11C44, 0x11C45]) or
175 False # SPEC-DRAFT-OUTDATED! U == 0x002D
177 def is_BASE_NUM(U, UISC, UGC):
178 return UISC == Brahmi_Joining_Number
179 def is_BASE_OTHER(U, UISC, UGC):
180 if UISC == Consonant_Placeholder: return True #SPEC-DRAFT
181 #SPEC-DRAFT return U in [0x00A0, 0x00D7, 0x2015, 0x2022, 0x25CC, 0x25FB, 0x25FC, 0x25FD, 0x25FE]
182 return U in [0x2015, 0x2022, 0x25FB, 0x25FC, 0x25FD, 0x25FE]
183 def is_CGJ(U, UISC, UGC):
185 def is_CONS_FINAL(U, UISC, UGC):
186 # Consonant_Initial_Postfixed is new in Unicode 11; not in the spec.
187 return ((UISC == Consonant_Final and UGC != Lo) or
188 UISC == Consonant_Initial_Postfixed or
189 UISC == Consonant_Succeeding_Repha)
190 def is_CONS_FINAL_MOD(U, UISC, UGC):
191 #SPEC-DRAFT return UISC in [Consonant_Final_Modifier, Syllable_Modifier]
192 return UISC == Syllable_Modifier
193 def is_CONS_MED(U, UISC, UGC):
194 return UISC == Consonant_Medial and UGC != Lo
195 def is_CONS_MOD(U, UISC, UGC):
196 return UISC in [Nukta, Gemination_Mark, Consonant_Killer]
197 def is_CONS_SUB(U, UISC, UGC):
198 #SPEC-DRAFT return UISC == Consonant_Subjoined
199 return UISC == Consonant_Subjoined and UGC != Lo
200 def is_CONS_WITH_STACKER(U, UISC, UGC):
201 return UISC == Consonant_With_Stacker
202 def is_HALANT(U, UISC, UGC):
203 return UISC in [Virama, Invisible_Stacker] and not is_HALANT_OR_VOWEL_MODIFIER(U, UISC, UGC)
204 def is_HALANT_OR_VOWEL_MODIFIER(U, UISC, UGC):
205 # https://github.com/harfbuzz/harfbuzz/issues/1102
206 # https://github.com/harfbuzz/harfbuzz/issues/1379
207 return U in [0x11046, 0x1134D]
208 def is_HALANT_NUM(U, UISC, UGC):
209 return UISC == Number_Joiner
210 def is_ZWNJ(U, UISC, UGC):
211 return UISC == Non_Joiner
212 def is_ZWJ(U, UISC, UGC):
213 return UISC == Joiner
214 def is_Word_Joiner(U, UISC, UGC):
216 def is_OTHER(U, UISC, UGC):
217 #SPEC-OUTDATED return UGC == Zs # or any other SCRIPT_COMMON characters
218 return (UISC == Other
219 and not is_SYM_MOD(U, UISC, UGC)
220 and not is_CGJ(U, UISC, UGC)
221 and not is_Word_Joiner(U, UISC, UGC)
222 and not is_VARIATION_SELECTOR(U, UISC, UGC)
224 def is_Reserved(U, UISC, UGC):
226 def is_REPHA(U, UISC, UGC):
227 return UISC in [Consonant_Preceding_Repha, Consonant_Prefixed]
228 def is_SYM(U, UISC, UGC):
229 if U == 0x25CC: return False #SPEC-DRAFT
230 #SPEC-DRAFT return UGC in [So, Sc] or UISC == Symbol_Letter
231 return UGC in [So, Sc]
232 def is_SYM_MOD(U, UISC, UGC):
233 return U in [0x1B6B, 0x1B6C, 0x1B6D, 0x1B6E, 0x1B6F, 0x1B70, 0x1B71, 0x1B72, 0x1B73]
234 def is_VARIATION_SELECTOR(U, UISC, UGC):
235 return 0xFE00 <= U <= 0xFE0F
236 def is_VOWEL(U, UISC, UGC):
237 # https://github.com/roozbehp/unicode-data/issues/6
238 return (UISC == Pure_Killer or
239 (UGC != Lo and UISC in [Vowel, Vowel_Dependent] and U not in [0xAA29]))
240 def is_VOWEL_MOD(U, UISC, UGC):
241 # https://github.com/roozbehp/unicode-data/issues/6
242 return (UISC in [Tone_Mark, Cantillation_Mark, Register_Shifter, Visarga] or
243 (UGC != Lo and (UISC == Bindu or U in [0xAA29])))
252 'FM': is_CONS_FINAL_MOD,
256 'CS': is_CONS_WITH_STACKER,
258 'HVM': is_HALANT_OR_VOWEL_MODIFIER,
262 'WJ': is_Word_Joiner,
268 'VS': is_VARIATION_SELECTOR,
281 'Blw': [Bottom, Bottom_And_Left],
290 'Abv': [Top, Top_And_Bottom, Top_And_Bottom_And_Right, Top_And_Right],
291 'Blw': [Bottom, Overstruck, Bottom_And_Right],
292 'Pst': [Right, Top_And_Left, Top_And_Left_And_Right, Left_And_Right],
297 'Blw': [Bottom, Overstruck],
312 def map_to_use(data):
314 items = use_mapping.items()
315 for U,(UISC,UIPC,UGC,UBlock) in data.items():
317 # Resolve Indic_Syllabic_Category
319 # TODO: These don't have UISC assigned in Unicode 12.0, but have UIPC
320 if 0x1CE2 <= U <= 0x1CE8: UISC = Cantillation_Mark
323 # TODO: These don't have UISC assigned in Unicode 12.0, but have UIPC
324 if 0x0F18 <= U <= 0x0F19 or 0x0F3E <= U <= 0x0F3F: UISC = Vowel_Dependent
325 if 0x0F86 <= U <= 0x0F87: UISC = Tone_Mark
326 # Overrides to allow NFC order matching syllable
327 # https://github.com/harfbuzz/harfbuzz/issues/1012
328 if UBlock == 'Tibetan' and is_VOWEL (U, UISC, UGC):
332 # TODO: https://github.com/harfbuzz/harfbuzz/pull/982
333 # also https://github.com/harfbuzz/harfbuzz/issues/1012
334 if UBlock == 'Chakma' and is_VOWEL (U, UISC, UGC):
340 # TODO: https://github.com/harfbuzz/harfbuzz/pull/627
341 if 0x1BF2 <= U <= 0x1BF3: UISC = Nukta; UIPC = Bottom
343 # TODO: U+1CED should only be allowed after some of
344 # the nasalization marks, maybe only for U+1CE9..U+1CF1.
345 if U == 0x1CED: UISC = Tone_Mark
347 # TODO: https://github.com/harfbuzz/harfbuzz/issues/525
348 if U == 0x1A7F: UISC = Consonant_Final
350 # TODO: https://github.com/harfbuzz/harfbuzz/issues/1105
351 if U == 0x11134: UISC = Gemination_Mark
353 # TODO: https://github.com/harfbuzz/harfbuzz/pull/1399
354 if U == 0x111C9: UISC = Consonant_Final
356 values = [k for k,v in items if v(U,UISC,UGC)]
357 assert len(values) == 1, "%s %s %s %s" % (hex(U), UISC, UGC, values)
360 # Resolve Indic_Positional_Category
362 # TODO: These should die, but have UIPC in Unicode 12.0
363 if U in [0x953, 0x954]: UIPC = Not_Applicable
365 # TODO: In USE's override list but not in Unicode 12.0
366 if U == 0x103C: UIPC = Left
368 # TODO: These are not in USE's override list that we have, nor are they in Unicode 12.0
369 if 0xA926 <= U <= 0xA92A: UIPC = Top
370 # TODO: https://github.com/harfbuzz/harfbuzz/pull/1037
371 # and https://github.com/harfbuzz/harfbuzz/issues/1631
372 if U in [0x11302, 0x11303, 0x114C1]: UIPC = Top
373 if U == 0x1171E: UIPC = Left
374 if 0x1CF8 <= U <= 0x1CF9: UIPC = Top
376 assert (UIPC in [Not_Applicable, Visual_Order_Left] or
377 USE in use_positions), "%s %s %s %s %s" % (hex(U), UIPC, USE, UISC, UGC)
379 pos_mapping = use_positions.get(USE, None)
381 values = [k for k,v in pos_mapping.items() if v and UIPC in v]
382 assert len(values) == 1, "%s %s %s %s %s %s" % (hex(U), UIPC, USE, UISC, UGC, values)
383 USE = USE + values[0]
385 out[U] = (USE, UBlock)
388 defaults = ('O', 'No_Block')
389 data = map_to_use(data)
391 print ("/* == Start of generated table == */")
393 print (" * The following table is generated by running:")
395 print (" * ./gen-use-table.py IndicSyllabicCategory.txt IndicPositionalCategory.txt UnicodeData.txt Blocks.txt")
397 print (" * on files with these headers:")
401 print (" * %s" % (l.strip()))
404 print ('#include "hb-ot-shape-complex-use.hh"')
410 def print_block (block, start, end, data):
411 global total, used, last_block
412 if block and block != last_block:
415 print (" /* %s */" % block)
417 print (' ' * (20 + (start % 16 * 6)), end='')
419 assert start % 8 == 0
420 assert (end+1) % 8 == 0
421 for u in range (start, end+1):
424 print (" /* %04X */" % u, end='')
427 d = data.get (u, defaults)
428 print ("%6s," % d[0], end='')
430 total += end - start + 1
435 uu = sorted (data.keys ())
442 print ('#pragma GCC diagnostic push')
443 print ('#pragma GCC diagnostic ignored "-Wunused-macros"')
444 for k,v in sorted(use_mapping.items()):
445 if k in use_positions and use_positions[k]: continue
446 print ("#define %s USE_%s /* %s */" % (k, k, v.__name__[3:]))
447 for k,v in sorted(use_positions.items()):
451 print ("#define %s USE_%s" % (tag, tag))
452 print ('#pragma GCC diagnostic pop')
454 print ("static const USE_TABLE_ELEMENT_TYPE use_table[] = {")
462 while end in uu and block == data[end][1]:
464 end = (end-1)//8*8 + 7
466 if start != last + 1:
467 if start - last <= 1+16*3:
468 print_block (None, last+1, start-1, data)
472 ends.append (last + 1)
473 offset += ends[-1] - starts[-1]
476 print ("#define use_offset_0x%04xu %d" % (start, offset))
477 starts.append (start)
479 print_block (block, start, end, data)
481 ends.append (last + 1)
482 offset += ends[-1] - starts[-1]
485 occupancy = used * 100. / total
487 print ("}; /* Table items: %d; occupancy: %d%% */" % (offset, occupancy))
489 print ("USE_TABLE_ELEMENT_TYPE")
490 print ("hb_use_get_category (hb_codepoint_t u)")
492 print (" switch (u >> %d)" % page_bits)
494 pages = set([u>>page_bits for u in starts+ends])
495 for p in sorted(pages):
496 print (" case 0x%0Xu:" % p)
497 for (start,end) in zip (starts, ends):
498 if p not in [start>>page_bits, end>>page_bits]: continue
499 offset = "use_offset_0x%04xu" % start
500 print (" if (hb_in_range<hb_codepoint_t> (u, 0x%04Xu, 0x%04Xu)) return use_table[u - 0x%04Xu + %s];" % (start, end-1, start, offset))
506 print (" return USE_O;")
509 for k in sorted(use_mapping.keys()):
510 if k in use_positions and use_positions[k]: continue
511 print ("#undef %s" % k)
512 for k,v in sorted(use_positions.items()):
516 print ("#undef %s" % tag)
518 print ("/* == End of generated table == */")
520 # Maintain at least 50% occupancy in the table */
522 raise Exception ("Table too sparse, please investigate: ", occupancy)