# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
-# 02111-1307, USA.
+# along with this program; if not, see <http://www.gnu.org/licenses/>.
# Contributer(s):
# Andrew Taylor <andrew.taylor@montage.ca>
# gen-unicode-tables.pl - Generate tables for libunicode from Unicode data.
# See http://www.unicode.org/Public/UNIDATA/UnicodeCharacterDatabase.html
-# Usage: gen-unicode-tables.pl [-decomp | -both] UNICODE-VERSION UnicodeData.txt LineBreak.txt SpecialCasing.txt CaseFolding.txt
# I consider the output of this program to be unrestricted. Use it as
# you will.
# * For decomp table it might make sense to use a shift count other
# than 8. We could easily compute the perfect shift count.
+# we use some perl unicode features
+require 5.006;
+
+use bytes;
+
use vars qw($CODE $NAME $CATEGORY $COMBINING_CLASSES $BIDI_CATEGORY $DECOMPOSITION $DECIMAL_VALUE $DIGIT_VALUE $NUMERIC_VALUE $MIRRORED $OLD_NAME $COMMENT $UPPER $LOWER $TITLE $BREAK_CODE $BREAK_CATEGORY $BREAK_NAME $CASE_CODE $CASE_LOWER $CASE_TITLE $CASE_UPPER $CASE_CONDITION);
+
# Names of fields in Unicode data table.
$CODE = 0;
$NAME = 1;
'Ll' => "G_UNICODE_LOWERCASE_LETTER",
'Lt' => "G_UNICODE_TITLECASE_LETTER",
'Mn' => "G_UNICODE_NON_SPACING_MARK",
- 'Mc' => "G_UNICODE_COMBINING_MARK",
+ 'Mc' => "G_UNICODE_SPACING_MARK",
'Me' => "G_UNICODE_ENCLOSING_MARK",
'Nd' => "G_UNICODE_DECIMAL_NUMBER",
'Nl' => "G_UNICODE_LETTER_NUMBER",
%break_mappings =
(
- 'BK' => "G_UNICODE_BREAK_MANDATORY",
- 'CR' => "G_UNICODE_BREAK_CARRIAGE_RETURN",
- 'LF' => "G_UNICODE_BREAK_LINE_FEED",
- 'CM' => "G_UNICODE_BREAK_COMBINING_MARK",
- 'SG' => "G_UNICODE_BREAK_SURROGATE",
- 'ZW' => "G_UNICODE_BREAK_ZERO_WIDTH_SPACE",
- 'IN' => "G_UNICODE_BREAK_INSEPARABLE",
- 'GL' => "G_UNICODE_BREAK_NON_BREAKING_GLUE",
- 'CB' => "G_UNICODE_BREAK_CONTINGENT",
- 'SP' => "G_UNICODE_BREAK_SPACE",
+ 'AI' => "G_UNICODE_BREAK_AMBIGUOUS",
+ 'AL' => "G_UNICODE_BREAK_ALPHABETIC",
+ 'B2' => "G_UNICODE_BREAK_BEFORE_AND_AFTER",
'BA' => "G_UNICODE_BREAK_AFTER",
'BB' => "G_UNICODE_BREAK_BEFORE",
- 'B2' => "G_UNICODE_BREAK_BEFORE_AND_AFTER",
- 'HY' => "G_UNICODE_BREAK_HYPHEN",
- 'NS' => "G_UNICODE_BREAK_NON_STARTER",
- 'OP' => "G_UNICODE_BREAK_OPEN_PUNCTUATION",
+ 'BK' => "G_UNICODE_BREAK_MANDATORY",
+ 'CB' => "G_UNICODE_BREAK_CONTINGENT",
+ 'CJ' => "G_UNICODE_BREAK_CONDITIONAL_JAPANESE_STARTER",
'CL' => "G_UNICODE_BREAK_CLOSE_PUNCTUATION",
- 'QU' => "G_UNICODE_BREAK_QUOTATION",
+ 'CM' => "G_UNICODE_BREAK_COMBINING_MARK",
+ 'CP' => "G_UNICODE_BREAK_CLOSE_PARANTHESIS",
+ 'CR' => "G_UNICODE_BREAK_CARRIAGE_RETURN",
'EX' => "G_UNICODE_BREAK_EXCLAMATION",
+ 'GL' => "G_UNICODE_BREAK_NON_BREAKING_GLUE",
+ 'H2' => "G_UNICODE_BREAK_HANGUL_LV_SYLLABLE",
+ 'H3' => "G_UNICODE_BREAK_HANGUL_LVT_SYLLABLE",
+ 'HL' => "G_UNICODE_BREAK_HEBREW_LETTER",
+ 'HY' => "G_UNICODE_BREAK_HYPHEN",
'ID' => "G_UNICODE_BREAK_IDEOGRAPHIC",
- 'NU' => "G_UNICODE_BREAK_NUMERIC",
+ 'IN' => "G_UNICODE_BREAK_INSEPARABLE",
'IS' => "G_UNICODE_BREAK_INFIX_SEPARATOR",
- 'SY' => "G_UNICODE_BREAK_SYMBOL",
- 'AL' => "G_UNICODE_BREAK_ALPHABETIC",
- 'PR' => "G_UNICODE_BREAK_PREFIX",
+ 'JL' => "G_UNICODE_BREAK_HANGUL_L_JAMO",
+ 'JT' => "G_UNICODE_BREAK_HANGUL_T_JAMO",
+ 'JV' => "G_UNICODE_BREAK_HANGUL_V_JAMO",
+ 'LF' => "G_UNICODE_BREAK_LINE_FEED",
+ 'NL' => "G_UNICODE_BREAK_NEXT_LINE",
+ 'NS' => "G_UNICODE_BREAK_NON_STARTER",
+ 'NU' => "G_UNICODE_BREAK_NUMERIC",
+ 'OP' => "G_UNICODE_BREAK_OPEN_PUNCTUATION",
'PO' => "G_UNICODE_BREAK_POSTFIX",
+ 'PR' => "G_UNICODE_BREAK_PREFIX",
+ 'QU' => "G_UNICODE_BREAK_QUOTATION",
+ 'RI' => "G_UNICODE_BREAK_REGIONAL_INDICATOR",
'SA' => "G_UNICODE_BREAK_COMPLEX_CONTEXT",
- 'AI' => "G_UNICODE_BREAK_AMBIGUOUS",
- 'XX' => "G_UNICODE_BREAK_UNKNOWN"
+ 'SG' => "G_UNICODE_BREAK_SURROGATE",
+ 'SP' => "G_UNICODE_BREAK_SPACE",
+ 'SY' => "G_UNICODE_BREAK_SYMBOL",
+ 'WJ' => "G_UNICODE_BREAK_WORD_JOINER",
+ 'XX' => "G_UNICODE_BREAK_UNKNOWN",
+ 'ZW' => "G_UNICODE_BREAK_ZERO_WIDTH_SPACE"
);
# Title case mappings.
# Maximum length of special-case strings
-my $special_case_len = 0;
my @special_cases;
+my @special_case_offsets;
+my $special_case_offset = 0;
+
+# Scripts
+
+my @scripts;
+
+# East asian widths
+
+my @eawidths;
$do_decomp = 0;
$do_props = 1;
+$do_scripts = 1;
if (@ARGV && $ARGV[0] eq '-decomp')
{
$do_decomp = 1;
shift @ARGV;
}
-if (@ARGV != 6) {
+if (@ARGV != 2) {
$0 =~ s@.*/@@;
- die "Usage: $0 [-decomp | -both] UNICODE-VERSION UnicodeData.txt LineBreak.txt SpecialCasing.txt CaseFolding.txt CompositionExclusions.txt\n";
+ die "\nUsage: $0 [-decomp | -both] UNICODE-VERSION DIRECTORY\n\n DIRECTORY should contain the following Unicode data files:\n UnicodeData.txt, LineBreak.txt, SpecialCasing.txt, CaseFolding.txt,\n CompositionExclusions.txt Scripts.txt extracted/DerivedEastAsianWidth.txt \n\n";
}
-
+
+my ($unicodedatatxt, $linebreaktxt, $specialcasingtxt, $casefoldingtxt, $compositionexclusionstxt,
+ $scriptstxt, $derivedeastasianwidth);
+
+my $d = $ARGV[1];
+opendir (my $dir, $d) or die "Cannot open Unicode data dir $d: $!\n";
+for my $f (readdir ($dir))
+{
+ $unicodedatatxt = "$d/$f" if ($f =~ /^UnicodeData.*\.txt/);
+ $linebreaktxt = "$d/$f" if ($f =~ /^LineBreak.*\.txt/);
+ $specialcasingtxt = "$d/$f" if ($f =~ /^SpecialCasing.*\.txt/);
+ $casefoldingtxt = "$d/$f" if ($f =~ /^CaseFolding.*\.txt/);
+ $compositionexclusionstxt = "$d/$f" if ($f =~ /^CompositionExclusions.*\.txt/);
+ $scriptstxt = "$d/$f" if ($f =~ /^Scripts.*\.txt/);
+}
+
+my $extd = $ARGV[1] . "/extracted";
+opendir (my $extdir, $extd) or die "Cannot open Unicode/extracted data dir $extd: $!\n";
+for my $f (readdir ($extdir))
+{
+ $derivedeastasianwidthtxt = "$extd/$f" if ($f =~ /^DerivedEastAsianWidth.*\.txt/);
+}
+
+defined $unicodedatatxt or die "Did not find UnicodeData file";
+defined $linebreaktxt or die "Did not find LineBreak file";
+defined $specialcasingtxt or die "Did not find SpecialCasing file";
+defined $casefoldingtxt or die "Did not find CaseFolding file";
+defined $compositionexclusionstxt or die "Did not find CompositionExclusions file";
+defined $scriptstxt or die "Did not find Scripts file";
+defined $derivedeastasianwidthtxt or die "Did not find DerivedEastAsianWidth file";
+
print "Creating decomp table\n" if ($do_decomp);
print "Creating property table\n" if ($do_props);
-print "Composition exlusions from $ARGV[5]\n";
+print "Composition exlusions from $compositionexclusionstxt\n";
-open (INPUT, "< $ARGV[5]") || exit 1;
+open (INPUT, "< $compositionexclusionstxt") || exit 1;
while (<INPUT>) {
close INPUT;
-print "Unicode data from $ARGV[1]\n";
+print "Unicode data from $unicodedatatxt\n";
-open (INPUT, "< $ARGV[1]") || exit 1;
+open (INPUT, "< $unicodedatatxt") || exit 1;
+
+# we save memory by skipping the huge empty area before U+E0000
+my $pages_before_e0000;
$last_code = -1;
while (<INPUT>)
$code = hex ($fields[$CODE]);
- last if ($code > 0xFFFF); # ignore characters out of the basic plane
+ if ($code >= 0xE0000 and $last_code < 0xE0000)
+ {
+ $pages_before_e0000 = ($last_code >> 8) + 1;
+ }
if ($code > $last_code + 1)
{
@gfields = ('', '', 'Cn', '0', '', '', '', '', '', '', '',
'', '', '', '');
-for (++$last_code; $last_code < 0x10000; ++$last_code)
+for (++$last_code; $last_code <= 0x10FFFF; ++$last_code)
{
$gfields{$CODE} = sprintf ("%04x", $last_code);
&process_one ($last_code, @gfields);
}
---$last_code; # Want last to be 0xFFFF.
+--$last_code; # Want last to be 0x10FFFF.
print "Creating line break table\n";
-print "Line break data from $ARGV[2]\n";
+print "Line break data from $linebreaktxt\n";
-open (INPUT, "< $ARGV[2]") || exit 1;
+open (INPUT, "< $linebreaktxt") || exit 1;
$last_code = -1;
while (<INPUT>)
chop;
next if /^#/;
+ next if /^$/;
s/\s*#.*//;
next;
}
- if ($fields[$CODE] =~ /([A-F0-9]{4})..([A-F0-9]{4})/)
+ if ($fields[$CODE] =~ /([A-F0-9]{4,6})\.\.([A-F0-9]{4,6})/)
{
$start_code = hex ($1);
$end_code = hex ($2);
}
- last if ($start_code > 0xFFFF); # FIXME ignore characters out of the basic plane
-
if ($start_code > $last_code + 1)
{
# The gap represents undefined characters. If assigned,
close INPUT;
-for (++$last_code; $last_code < 0x10000; ++$last_code)
+for (++$last_code; $last_code <= 0x10FFFF; ++$last_code)
{
if ($type[$last_code] eq 'Cn')
{
$break_props[$last_code] = 'AL';
}
}
---$last_code; # Want last to be 0xFFFF.
+--$last_code; # Want last to be 0x10FFFF.
-print STDERR "Last code is not 0xFFFF" if ($last_code != 0xFFFF);
+print STDERR "Last code is not 0x10FFFF" if ($last_code != 0x10FFFF);
print "Reading special-casing table for case conversion\n";
-open (INPUT, "< $ARGV[3]") || exit 1;
+open (INPUT, "< $specialcasingtxt") || exit 1;
while (<INPUT>)
{
{
(hex $fields[$CASE_UPPER] == $code) || die "$raw_code is Lu and UCD_Upper($raw_code) != $raw_code";
- &add_special_case ($code, $value[$code],$fields[$CASE_LOWER], $fields[$CASE_TITLE]);
+ &add_special_case ($code, $value[$code], $fields[$CASE_LOWER], $fields[$CASE_TITLE]);
} elsif ($type[$code] eq 'Lt')
{
(hex $fields[$CASE_TITLE] == $code) || die "$raw_code is Lt and UCD_Title($raw_code) != $raw_code";
- &add_special_case ($code, undef,$fields[$CASE_LOWER], $fields[$CASE_UPPER]);
+ &add_special_case ($code, undef, $fields[$CASE_LOWER], $fields[$CASE_UPPER]);
} elsif ($type[$code] eq 'Ll')
{
(hex $fields[$CASE_LOWER] == $code) || die "$raw_code is Ll and UCD_Lower($raw_code) != $raw_code";
- &add_special_case ($code, $value[$code],$fields[$CASE_UPPER], $fields[$CASE_TITLE]);
+ &add_special_case ($code, $value[$code], $fields[$CASE_UPPER], $fields[$CASE_TITLE]);
} else {
printf STDERR "Special case for non-alphabetic code point: $raw_code\n";
next;
close INPUT;
-open (INPUT, "< $ARGV[4]") || exit 1;
+open (INPUT, "< $casefoldingtxt") || exit 1;
my $casefoldlen = 0;
my @casefold;
$raw_code = $fields[$FOLDING_CODE];
$code = hex ($raw_code);
- next if $code > 0xffff; # FIXME!
-
if ($#fields != 3)
{
printf STDERR ("Entry for $raw_code has wrong number of fields (%d)\n", $#fields);
next;
}
- next if ($fields[$FOLDING_STATUS] eq 'S');
+ # we don't use Simple or Turkic rules here
+ next if ($fields[$FOLDING_STATUS] =~ /^[ST]$/);
@values = map { hex ($_) } split /\s+/, $fields[$FOLDING_MAPPING];
# Check simple case
if (@values == 1 &&
- !(defined $value[$code] && $value[$code] >= 0xd800 && $value[$code] < 0xdc00) &&
+ !(defined $value[$code] && $value[$code] >= 0x1000000) &&
defined $type[$code]) {
my $lower;
}
my $string = pack ("U*", @values);
- if (1 + length $string > $casefoldlen) {
- $casefoldlen = 1 + length $string;
+
+ if (1 + &length_in_bytes ($string) > $casefoldlen) {
+ $casefoldlen = 1 + &length_in_bytes ($string);
+ }
+
+ push @casefold, [ $code, &escape ($string) ];
+}
+
+close INPUT;
+
+print "Reading scripts\n";
+
+open (INPUT, "< $scriptstxt") || exit 1;
+
+while (<INPUT>) {
+ s/#.*//;
+ next if /^\s*$/;
+ if (!/^([0-9A-F]+)(?:\.\.([0-9A-F]+))?\s*;\s*([A-Za-z_]+)\s*$/) {
+ die "Cannot parse line: '$_'\n";
+ }
+
+ if (defined $2) {
+ push @scripts, [ hex $1, hex $2, uc $3 ];
+ } else {
+ push @scripts, [ hex $1, hex $1, uc $3 ];
+ }
+}
+
+close INPUT;
+
+print "Reading derived east asian widths\n";
+
+open (INPUT, "< $derivedeastasianwidthtxt") || exit 1;
+
+while (<INPUT>)
+{
+ my ($start_code, $end_code);
+
+ chop;
+
+ s/#.*//;
+ next if /^\s*$/;
+ if (!/^([0-9A-F]+)(?:\.\.([0-9A-F]+))?\s*;\s*([A-Za-z_]+)\s*$/) {
+ die "Cannot parse line: '$_'\n";
}
- push @casefold, [ $code, $string ];
+ if (defined $2) {
+ push @eawidths, [ hex $1, hex $2, $3 ];
+ } else {
+ push @eawidths, [ hex $1, hex $1, $3 ];
+ }
}
close INPUT;
&print_decomp ($last_code);
&output_composition_table;
}
-
&print_line_break ($last_code);
+if ($do_scripts) {
+ &print_scripts
+}
+
exit 0;
+
+# perl "length" returns the length in characters
+sub length_in_bytes
+{
+ my ($string) = @_;
+
+ return length $string;
+}
+
# Process a single character.
sub process_one
{
printf OUT "#define G_UNICODE_LAST_CHAR 0x%04x\n\n", $last;
- printf OUT "#define G_UNICODE_MAX_TABLE_INDEX 1000\n\n";
+ printf OUT "#define G_UNICODE_MAX_TABLE_INDEX 10000\n\n";
+
+ my $last_part1 = ($pages_before_e0000 * 256) - 1;
+ printf OUT "#define G_UNICODE_LAST_CHAR_PART1 0x%04X\n\n", $last_part1;
+ printf OUT "#define G_UNICODE_LAST_PAGE_PART1 %d\n\n", $pages_before_e0000 - 1;
$table_index = 0;
printf OUT "static const char type_data[][256] = {\n";
}
printf OUT "\n};\n\n";
- print OUT "static const short type_table[256] = {\n";
- for ($count = 0; $count <= $last; $count += 256)
+ printf OUT "/* U+0000 through U+%04X */\n", $last_part1;
+ print OUT "static const gint16 type_table_part1[$pages_before_e0000] = {\n";
+ for ($count = 0; $count <= $last_part1; $count += 256)
{
print OUT ",\n" if $count > 0;
print OUT " ", $row[$count / 256];
}
print OUT "\n};\n\n";
+ printf OUT "/* U+E0000 through U+%04X */\n", $last;
+ print OUT "static const gint16 type_table_part2[768] = {\n";
+ for ($count = 0xE0000; $count <= $last; $count += 256)
+ {
+ print OUT ",\n" if $count > 0xE0000;
+ print OUT " ", $row[$count / 256];
+ $bytes_out += 2;
+ }
+ print OUT "\n};\n\n";
+
#
# Now print attribute table.
#
$table_index = 0;
- printf OUT "static const unsigned short attr_data[][256] = {\n";
+ printf OUT "static const gunichar attr_data[][256] = {\n";
for ($count = 0; $count <= $last; $count += 256)
{
- $row[$count / 256] = &print_row ($count, 2, \&fetch_attr);
+ $row[$count / 256] = &print_row ($count, 4, \&fetch_attr);
}
printf OUT "\n};\n\n";
- print OUT "static const short attr_table[256] = {\n";
- for ($count = 0; $count <= $last; $count += 256)
+ printf OUT "/* U+0000 through U+%04X */\n", $last_part1;
+ print OUT "static const gint16 attr_table_part1[$pages_before_e0000] = {\n";
+ for ($count = 0; $count <= $last_part1; $count += 256)
{
print OUT ",\n" if $count > 0;
print OUT " ", $row[$count / 256];
}
print OUT "\n};\n\n";
+ printf OUT "/* U+E0000 through U+%04X */\n", $last;
+ print OUT "static const gint16 attr_table_part2[768] = {\n";
+ for ($count = 0xE0000; $count <= $last; $count += 256)
+ {
+ print OUT ",\n" if $count > 0xE0000;
+ print OUT " ", $row[$count / 256];
+ $bytes_out += 2;
+ }
+ print OUT "\n};\n\n";
+
#
# print title case table
#
- # FIXME: type.
- print OUT "static const unsigned short title_table[][3] = {\n";
+ print OUT "static const gunichar title_table[][3] = {\n";
my ($item);
my ($first) = 1;
foreach $item (sort keys %title_to_lower)
unless $first;
$first = 0;
printf OUT " { 0x%04x, 0x%04x, 0x%04x }", $item, $title_to_upper{$item}, $title_to_lower{$item};
- $bytes_out += 6;
+ $bytes_out += 12;
}
print OUT "\n};\n\n";
&output_special_case_table (\*OUT);
&output_casefold_table (\*OUT);
+ #
+ # And the widths tables
+ #
+ &output_width_tables (\*OUT);
+
print OUT "#endif /* CHARTABLES_H */\n";
close (OUT);
return sprintf "%d /* page %d */", $table_index++, $start / 256;
}
+sub escape
+{
+ my ($string) = @_;
+
+ my $escaped = unpack("H*", $string);
+ $escaped =~ s/(.{2})/\\x$1/g;
+
+ return $escaped;
+}
+
+# Returns the offset of $decomp in the offset string. Updates the
+# referenced variables as appropriate.
+sub handle_decomp ($$$$)
+{
+ my ($decomp, $decomp_offsets_ref, $decomp_string_ref, $decomp_string_offset_ref) = @_;
+ my $offset = "G_UNICODE_NOT_PRESENT_OFFSET";
+
+ if (defined $decomp)
+ {
+ if (defined $decomp_offsets_ref->{$decomp})
+ {
+ $offset = $decomp_offsets_ref->{$decomp};
+ }
+ else
+ {
+ $offset = ${$decomp_string_offset_ref};
+ $decomp_offsets_ref->{$decomp} = $offset;
+ ${$decomp_string_ref} .= "\n \"" . &escape ($decomp) . "\\0\" /* offset ${$decomp_string_offset_ref} */";
+ ${$decomp_string_offset_ref} += &length_in_bytes ($decomp) + 1;
+ }
+ }
+
+ return $offset;
+}
+
# Generate the character decomposition header.
sub print_decomp
{
printf OUT "#define G_UNICODE_LAST_CHAR 0x%04x\n\n", $last;
- printf OUT "#define G_UNICODE_MAX_TABLE_INDEX 1000\n\n";
+ printf OUT "#define G_UNICODE_MAX_TABLE_INDEX (0x110000 / 256)\n\n";
+
+ my $last_part1 = ($pages_before_e0000 * 256) - 1;
+ printf OUT "#define G_UNICODE_LAST_CHAR_PART1 0x%04X\n\n", $last_part1;
+ printf OUT "#define G_UNICODE_LAST_PAGE_PART1 %d\n\n", $pages_before_e0000 - 1;
+
+ $NOT_PRESENT_OFFSET = 65535;
+ print OUT "#define G_UNICODE_NOT_PRESENT_OFFSET $NOT_PRESENT_OFFSET\n\n";
my ($count, @row);
$table_index = 0;
- printf OUT "static const unsigned char cclass_data[][256] = {\n";
+ printf OUT "static const guchar cclass_data[][256] = {\n";
for ($count = 0; $count <= $last; $count += 256)
{
$row[$count / 256] = &print_row ($count, 1, \&fetch_cclass);
}
printf OUT "\n};\n\n";
- print OUT "static const short combining_class_table[256] = {\n";
- for ($count = 0; $count <= $last; $count += 256)
+ print OUT "static const gint16 combining_class_table_part1[$pages_before_e0000] = {\n";
+ for ($count = 0; $count <= $last_part1; $count += 256)
{
print OUT ",\n" if $count > 0;
print OUT " ", $row[$count / 256];
}
print OUT "\n};\n\n";
+ print OUT "static const gint16 combining_class_table_part2[768] = {\n";
+ for ($count = 0xE0000; $count <= $last; $count += 256)
+ {
+ print OUT ",\n" if $count > 0xE0000;
+ print OUT " ", $row[$count / 256];
+ $bytes_out += 2;
+ }
+ print OUT "\n};\n\n";
+
print OUT "typedef struct\n{\n";
- # FIXME: type.
- print OUT " unsigned short ch;\n";
- print OUT " unsigned char canon_offset;\n";
- print OUT " unsigned char compat_offset;\n";
- print OUT " unsigned char *expansion;\n";
+ print OUT " gunichar ch;\n";
+ print OUT " guint16 canon_offset;\n";
+ print OUT " guint16 compat_offset;\n";
print OUT "} decomposition;\n\n";
print OUT "static const decomposition decomp_table[] =\n{\n";
my ($iter);
my ($first) = 1;
+ my ($decomp_string) = "";
+ my ($decomp_string_offset) = 0;
for ($count = 0; $count <= $last; ++$count)
{
if (defined $decompositions[$count])
undef $compat_decomp;
}
- my $string = "";
- my $canon_offset = 0xff;
- my $compat_offset = 0xff;
-
- if (defined $canon_decomp) {
- $canon_offset = 0;
- $string .= $canon_decomp;
- }
- if (defined $compat_decomp) {
- if (defined $canon_decomp) {
- $string .= "\\x00\\x00";
- }
- $compat_offset = (length $string) / 4;
- $string .= $compat_decomp;
- }
+ my $canon_offset = handle_decomp ($canon_decomp, \%decomp_offsets, \$decomp_string, \$decomp_string_offset);
+ my $compat_offset = handle_decomp ($compat_decomp, \%decomp_offsets, \$decomp_string, \$decomp_string_offset);
- $bytes_out += (length $string) / 4; # "\x20"
-
- # Only a single terminator because one is implied in the string.
- printf OUT qq( { 0x%04x, %u, %u, "%s\\0" }),
- $count, $canon_offset, $compat_offset, $string;
-
-
- $bytes_out += 6;
+ die if $decomp_string_offset > $NOT_PRESENT_OFFSET;
+
+ printf OUT qq( { 0x%04x, $canon_offset, $compat_offset }), $count;
+ $bytes_out += 8;
}
}
print OUT "\n};\n\n";
+ $bytes_out += $decomp_string_offset + 1;
+
+ printf OUT "static const gchar decomp_expansion_string[] = %s;\n\n", $decomp_string;
+
+ print OUT "typedef struct\n{\n";
+ print OUT " gunichar ch;\n";
+ print OUT " gunichar a;\n";
+ print OUT " gunichar b;\n";
+ print OUT "} decomposition_step;\n\n";
+
+ # There's lots of room to optimize the following table...
+ print OUT "static const decomposition_step decomp_step_table[] =\n{\n";
+ $first = 1;
+ my @steps = ();
+ for ($count = 0; $count <= $last; ++$count)
+ {
+ if ((defined $decompositions[$count]) && (!$decompose_compat[$count]))
+ {
+ print OUT ",\n"
+ if ! $first;
+ $first = 0;
+ my @list;
+ @list = (split(' ', $decompositions[$count]), "0");
+ printf OUT qq( { 0x%05x, 0x%05x, 0x%05x }), $count, hex($list[0]), hex($list[1]);
+ # don't include 1:1 in the compose table
+ push @steps, [ ($count, hex($list[0]), hex($list[1])) ]
+ if hex($list[1])
+ }
+ }
+ print OUT "\n};\n\n";
print OUT "#endif /* DECOMP_H */\n";
print OUT "#ifndef BREAKTABLES_H\n";
print OUT "#define BREAKTABLES_H\n\n";
+ print OUT "#include <glib/gtypes.h>\n";
+ print OUT "#include <glib/gunicode.h>\n\n";
+
print OUT "#define G_UNICODE_DATA_VERSION \"$ARGV[0]\"\n\n";
- printf OUT "#define G_UNICODE_LAST_CHAR 0x%04x\n\n", $last;
+ printf OUT "#define G_UNICODE_LAST_CHAR 0x%04X\n\n", $last;
- printf OUT "#define G_UNICODE_MAX_TABLE_INDEX 1000\n\n";
+ printf OUT "#define G_UNICODE_MAX_TABLE_INDEX 10000\n\n";
+
+ my $last_part1 = ($pages_before_e0000 * 256) - 1;
+ printf OUT "/* the last code point that should be looked up in break_property_table_part1 */\n";
+ printf OUT "#define G_UNICODE_LAST_CHAR_PART1 0x%04X\n\n", $last_part1;
$table_index = 0;
- printf OUT "static const char break_property_data[][256] = {\n";
+ printf OUT "static const gint8 break_property_data[][256] = {\n";
for ($count = 0; $count <= $last; $count += 256)
{
$row[$count / 256] = &print_row ($count, 1, \&fetch_break_type);
}
printf OUT "\n};\n\n";
- print OUT "static const short break_property_table[256] = {\n";
- for ($count = 0; $count <= $last; $count += 256)
+ printf OUT "/* U+0000 through U+%04X */\n", $last_part1;
+ print OUT "static const gint16 break_property_table_part1[$pages_before_e0000] = {\n";
+ for ($count = 0; $count <= $last_part1; $count += 256)
{
print OUT ",\n" if $count > 0;
print OUT " ", $row[$count / 256];
}
print OUT "\n};\n\n";
+ printf OUT "/* U+E0000 through U+%04X */\n", $last;
+ print OUT "static const gint16 break_property_table_part2[768] = {\n";
+ for ($count = 0xE0000; $count <= $last; $count += 256)
+ {
+ print OUT ",\n" if $count > 0xE0000;
+ print OUT " ", $row[$count / 256];
+ $bytes_out += 2;
+ }
+ print OUT "\n};\n\n";
+
+
print OUT "#endif /* BREAKTABLES_H */\n";
close (OUT);
my $result = "";
foreach $iter (&expand_decomp ($code, $compat))
{
- $result .= sprintf "\\x%02x\\x%02x", $iter / 256, $iter & 0xff;
+ $result .= pack ("U", $iter); # to utf-8
}
$result;
(map { hex ($_) } split /\s+/, $field1),
0,
(map { hex ($_) } split /\s+/, $field2));
- $result = "";
+ $result = "";
for $value (@values) {
- $result .= sprintf ("\\x%02x\\x%02x", $value / 256, $value & 0xff);
+ $result .= pack ("U", $value); # to utf-8
}
-
- $result .= "\\0";
- if (2 * @values + 2 > $special_case_len) {
- $special_case_len = 2 * @values + 2;
- }
+ push @special_case_offsets, $special_case_offset;
- push @special_cases, $result;
+ # We encode special cases up in the 0x1000000 space
+ $value[$code] = 0x1000000 + $special_case_offset;
- #
- # We encode special cases in the surrogate pair space
- #
- $value[$code] = 0xD800 + scalar(@special_cases) - 1;
+ $special_case_offset += 1 + &length_in_bytes ($result);
+
+ push @special_cases, &escape ($result);
}
sub output_special_case_table
* First, the best single character mapping to lowercase if Lu,
* and to uppercase if Ll, followed by the output mapping for the two cases
* other than the case of the codepoint, in the order [Ll],[Lu],[Lt],
- * separated and terminated by a double NUL.
+ * encoded in UTF-8, separated and terminated by a null character.
*/
-static const guchar special_case_table[][$special_case_len] = {
+static const gchar special_case_table[] = {
EOT
+ my $i = 0;
for $case (@special_cases) {
- print $out qq( "$case",\n);
+ print $out qq( "$case\\0" /* offset ${special_case_offsets[$i]} */\n);
+ $i++;
}
print $out <<EOT;
EOT
- print STDERR "Generated ", ($special_case_len * scalar @special_cases), " bytes in special case table\n";
+ print STDERR "Generated " . ($special_case_offset + 1) . " bytes in special case table\n";
}
sub enumerate_ordered
# decompositions. At the same time, record
# the first and second character of each decomposition
- for $code (keys %compositions) {
+ for $code (keys %compositions)
+ {
@values = map { hex ($_) } split /\s+/, $compositions{$code};
+
+ # non-starters
+ if ($cclass[$code]) {
+ delete $compositions{$code};
+ next;
+ }
if ($cclass[$values[0]]) {
delete $compositions{$code};
next;
}
+
+ # single-character decompositions
if (@values == 1) {
delete $compositions{$code};
next;
}
+
if (@values != 2) {
die "$code has more than two elements in its decomposition!\n";
}
}
}
- # Assign integer indicices, removing singletons
+ # Assign integer indices, removing singletons
my $n_first = enumerate_ordered (\%first);
- # Now record the second character if each (non-singleton) decomposition
+ # Now record the second character of each (non-singleton) decomposition
for $code (keys %compositions) {
@values = map { hex ($_) } split /\s+/, $compositions{$code};
$last = $code if $code > $last;
}
+ printf OUT "#define COMPOSE_TABLE_LAST %d\n\n", $last / 256;
+
# Output lookup table
my @row;
$table_index = 0;
- printf OUT "static const gushort compose_data[][256] = {\n";
+ printf OUT "static const guint16 compose_data[][256] = {\n";
for (my $count = 0; $count <= $last; $count += 256)
{
$row[$count / 256] = &print_row ($count, 2, sub { exists $vals{$_[0]} ? $vals{$_[0]} : 0; });
}
printf OUT "\n};\n\n";
- print OUT "static const short compose_table[256] = {\n";
+ print OUT "static const gint16 compose_table[COMPOSE_TABLE_LAST + 1] = {\n";
for (my $count = 0; $count <= $last; $count += 256)
{
print OUT ",\n" if $count > 0;
print OUT " ", $row[$count / 256];
- $bytes_out += 4;
+ $bytes_out += 2;
}
print OUT "\n};\n\n";
# Output first singletons
- print OUT "static const gushort compose_first_single[][2] = {\n";
+ print OUT "static const gunichar compose_first_single[][2] = {\n";
$i = 0;
for $record (@first_singletons) {
print OUT ",\n" if $i++ > 0;
}
print OUT "\n};\n";
- $bytes_out += @first_singletons * 4;
+ $bytes_out += @first_singletons * 4;
# Output second singletons
- print OUT "static const gushort compose_second_single[][2] = {\n";
+ print OUT "static const gunichar compose_second_single[][2] = {\n";
$i = 0;
for $record (@second_singletons) {
print OUT ",\n" if $i++ > 0;
# Output array of composition pairs
print OUT <<EOT;
-static const gushort compose_array[$n_first][$n_second] = {
+static const guint16 compose_array[$n_first][$n_second] = {
EOT
for (my $i = 0; $i < $n_first; $i++) {
for (my $j = 0; $j < $n_second; $j++) {
print OUT ", " if $j;
if (exists $reverse{"$i|$j"}) {
- printf OUT "%#06x", $reverse{"$i|$j"};
+ if ($reverse{"$i|$j"} > 0xFFFF) {
+ die "time to switch compose_array to gunichar" ;
+ }
+ printf OUT "0x%04x", $reverse{"$i|$j"};
} else {
print OUT " 0";
}
@casefold = sort { $a->[0] <=> $b->[0] } @casefold;
- for $case (@casefold) {
+ for $case (@casefold)
+ {
$code = $case->[0];
$string = $case->[1];
- print $out sprintf(qq({ %#04x, "$string" },\n), $code);
+
+ if ($code > 0xFFFF) {
+ die "time to switch casefold_table to gunichar" ;
+ }
+
+ print $out sprintf(qq( { 0x%04x, "$string" },\n), $code);
}
printf "Generated %d bytes for casefold table\n", $recordlen * @casefold;
}
-
+sub output_one_width_table
+{
+ my ($out, $name, $wpe) = @_;
+ my $start;
+ my $end;
+ my $wp;
+ my $rex;
+
+ print $out "static const struct Interval g_unicode_width_table_${name}[] = {\n";
+
+ $rex = qr/$wpe/;
+
+ for (my $i = 0; $i <= $#eawidths; $i++) {
+ $start = $eawidths[$i]->[0];
+ $end = $eawidths[$i]->[1];
+ $wp = $eawidths[$i]->[2];
+
+ next if ($wp !~ $rex);
+
+ while ($i <= $#eawidths - 1 &&
+ $eawidths[$i + 1]->[0] == $end + 1 &&
+ ($eawidths[$i + 1]->[2] =~ $rex)) {
+ $i++;
+ $end = $eawidths[$i]->[1];
+ }
+
+ printf $out "{0x%04X, 0x%04X},\n", $start, $end;
+ }
+
+ printf $out "};\n\n";
+}
+
+sub output_width_tables
+{
+ my $out = shift;
+
+ @eawidths = sort { $a->[0] <=> $b->[0] } @eawidths;
+
+ print $out <<EOT;
+
+struct Interval
+{
+ gunichar start, end;
+};
+
+EOT
+
+ &output_one_width_table ($out,"wide", "[FW]");
+ &output_one_width_table ($out, "ambiguous", "[A]");
+}
+
+sub print_scripts
+{
+ my $start;
+ my $end;
+ my $script;
+ my $easy_range;
+ my $i;
+
+ print STDERR "Writing gscripttable.h\n";
+
+ open OUT, ">gscripttable.h" or die "Cannot open gscripttable.h: $!\n";
+
+ print OUT<<EOT;
+/* This file is automatically generated. DO NOT EDIT!
+ Instead, edit gen-unicode-tables.pl and re-run. */
+
+#ifndef SCRIPTTABLES_H
+#define SCRIPTTABLES_H
+
+EOT
+
+ @scripts = sort { $a->[0] <=> $b->[0] } @scripts;
+
+ $easy_range = 0x2000;
+
+ print OUT<<EOT;
+#define G_EASY_SCRIPTS_RANGE $easy_range
+
+static const guchar g_script_easy_table[$easy_range] = {
+EOT
+
+ $i = 0;
+ $end = -1;
+
+ for (my $c = 0; $c < $easy_range; $c++) {
+
+ if ($c % 3 == 0) {
+ printf OUT "\n ";
+ }
+
+ if ($c > $end) {
+ $start = $scripts[$i]->[0];
+ $end = $scripts[$i]->[1];
+ $script = $scripts[$i]->[2];
+ $i++;
+ }
+
+ if ($c < $start) {
+ printf OUT " G_UNICODE_SCRIPT_UNKNOWN,";
+ } else {
+ printf OUT " G_UNICODE_SCRIPT_%s,", $script;
+ }
+ }
+
+ if ($end >= $easy_range) {
+ $i--;
+ $scripts[$i]->[0] = $easy_range;
+ }
+
+ print OUT<<EOT;
+
+};
+
+static const struct {
+ gunichar start;
+ guint16 chars;
+ guint16 script;
+} g_script_table[] = {
+EOT
+ for (; $i <= $#scripts; $i++) {
+ $start = $scripts[$i]->[0];
+ $end = $scripts[$i]->[1];
+ $script = $scripts[$i]->[2];
+
+ while ($i <= $#scripts - 1 &&
+ $scripts[$i + 1]->[0] == $end + 1 &&
+ $scripts[$i + 1]->[2] eq $script) {
+ $i++;
+ $end = $scripts[$i]->[1];
+ }
+ printf OUT " { %#06x, %5d, G_UNICODE_SCRIPT_%s },\n", $start, $end - $start + 1, $script;
+ }
+
+ printf OUT<<EOT;
+};
+
+#endif /* SCRIPTTABLES_H */
+EOT
+
+ close OUT;
+}
projects / platform / upstream / glib.git / blobdiff