From af83ed5c4647bda196fc1a7efebbe8019aa83f4a Mon Sep 17 00:00:00 2001 From: Mike FABIAN Date: Fri, 18 Aug 2017 10:12:29 +0200 Subject: [PATCH] Write all ranges of neighbouring characters with the same width using the range notation in charmaps/UTF-8 Writing ranges of neighbouring characters with the same with like this ... 0 in charmaps/UTF-8 is more efficient than writing many single character lines like: 0 0 ... [BZ #21750] * unicode-gen/utf8_gen.py: Write all ranges of neighbouring characters with the same width using the range notation in charmaps/UTF-8. --- ChangeLog | 7 ++++++ localedata/unicode-gen/utf8_gen.py | 51 ++++++++++++++++++++++++++++---------- 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/ChangeLog b/ChangeLog index 5e3bc26..1b1ff8f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,10 @@ +2017-09-06 Mike FABIAN + + [BZ #21750] + * localedata/unicode-gen/utf8_gen.py: Improve the script to + use the range notation for all ranges of neighbouring characters + with the same width. + 2017-09-05 Joseph Myers * sysdeps/generic/math-type-macros-double.h: Include diff --git a/localedata/unicode-gen/utf8_gen.py b/localedata/unicode-gen/utf8_gen.py index 1563aa1..52c79e8 100755 --- a/localedata/unicode-gen/utf8_gen.py +++ b/localedata/unicode-gen/utf8_gen.py @@ -199,7 +199,7 @@ def write_header_charmap(outfile): def write_header_width(outfile): '''Writes the header on top of the WIDTH section to the output file''' - outfile.write('% Character width according to Unicode 7.0.0.\n') + outfile.write('% Character width according to Unicode 10.0.0.\n') outfile.write('% - Default width is 1.\n') outfile.write('% - Double-width characters have width 2; generated from\n') outfile.write('% "grep \'^[^;]*;[WF]\' EastAsianWidth.txt"\n') @@ -229,27 +229,52 @@ def process_width(outfile, ulines, elines): code_points = fields[0].split("..") for key in range(int(code_points[0], 16), int(code_points[1], 16)+1): - width_dict[key] = unicode_utils.ucs_symbol(key) + '\t2' + width_dict[key] = 2 for line in ulines: fields = line.split(";") if fields[4] == "NSM" or fields[2] in ("Cf", "Me", "Mn"): - width_dict[int(fields[0], 16)] = unicode_utils.ucs_symbol( - int(fields[0], 16)) + '\t0' + width_dict[int(fields[0], 16)] = 0 # handle special cases for compatibility - for key in list(range(0x1160, 0x1200)) + list(range(0x3248, 0x3250)) + \ - list(range(0x4DC0, 0x4E00)) + list((0x00AD,)): + for key in list((0x00AD,)): + # https://www.cs.tut.fi/~jkorpela/shy.html if key in width_dict: del width_dict[key] - width_dict[0x1160] = '{:s}...{:s}\t0'.format( - unicode_utils.ucs_symbol(0x1160), unicode_utils.ucs_symbol(0x11FF)) - width_dict[0x3248] = '{:s}...{:s}\t2'.format( - unicode_utils.ucs_symbol(0x3248), unicode_utils.ucs_symbol(0x324F)) - width_dict[0x4DC0] = '{:s}...{:s}\t2'.format( - unicode_utils.ucs_symbol(0x4DC0), unicode_utils.ucs_symbol(0x4DFF)) + for key in list(range(0x1160, 0x1200)): + width_dict[key] = 0 + for key in list(range(0x3248, 0x3250)): + # These are “A” which means we can decide whether to treat them + # as “W” or “N” based on context: + # http://www.unicode.org/mail-arch/unicode-ml/y2017-m08/0023.html + # For us, “W” seems better. + width_dict[key] = 2 + for key in list(range(0x4DC0, 0x4E00)): + width_dict[key] = 2 + same_width_lists = [] + current_width_list = [] for key in sorted(width_dict): - outfile.write(width_dict[key]+'\n') + if not current_width_list: + current_width_list = [key] + elif (key == current_width_list[-1] + 1 + and width_dict[key] == width_dict[current_width_list[0]]): + current_width_list.append(key) + else: + same_width_lists.append(current_width_list) + current_width_list = [key] + if current_width_list: + same_width_lists.append(current_width_list) + + for same_width_list in same_width_lists: + if len(same_width_list) == 1: + outfile.write('{:s}\t{:d}\n'.format( + unicode_utils.ucs_symbol(same_width_list[0]), + width_dict[same_width_list[0]])) + else: + outfile.write('{:s}...{:s}\t{:d}\n'.format( + unicode_utils.ucs_symbol(same_width_list[0]), + unicode_utils.ucs_symbol(same_width_list[-1]), + width_dict[same_width_list[0]])) if __name__ == "__main__": if len(sys.argv) < 3: -- 2.7.4