2 /* Copyright (C) 1989-2018 Free Software Foundation, Inc.
3 Written by James Clark (jjc@jclark.com)
5 This file is part of groff.
7 groff is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation, either version 3 of the License, or
10 (at your option) any later version.
12 groff is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with this program. If not, see <http://www.gnu.org/licenses/>. */
23 #define TOKEN_TABLE_SIZE 1009
24 // I believe in Icelandic thorn sorts after z.
25 #define THORN_SORT_KEY "{"
27 struct token_table_entry {
33 token_table_entry token_table[TOKEN_TABLE_SIZE];
36 static void skip_name(const char **ptr, const char *end)
56 int get_token(const char **ptr, const char *end)
61 if (c == '\\' && *ptr < end) {
80 token_info::token_info()
81 : type(TOKEN_OTHER), sort_key(0), other_case(0)
85 void token_info::set(token_type t, const char *sk, const char *oc)
87 assert(oc == 0 || t == TOKEN_UPPER || t == TOKEN_LOWER);
93 void token_info::sortify(const char *start, const char *end, string &result)
98 else if (type == TOKEN_UPPER || type == TOKEN_LOWER) {
99 for (; start < end; start++)
101 result += cmlower(*start);
105 int token_info::sortify_non_empty(const char *start, const char *end) const
108 return *sort_key != '\0';
109 if (type != TOKEN_UPPER && type != TOKEN_LOWER)
111 for (; start < end; start++)
118 void token_info::lower_case(const char *start, const char *end,
119 string &result) const
121 if (type != TOKEN_UPPER) {
126 result += other_case;
129 result += cmlower(*start++);
133 void token_info::upper_case(const char *start, const char *end,
134 string &result) const
136 if (type != TOKEN_LOWER) {
141 result += other_case;
144 result += cmupper(*start++);
148 token_table_entry::token_table_entry()
153 static void store_token(const char *tok, token_type typ,
154 const char *sk = 0, const char *oc = 0)
156 unsigned n = hash_string(tok, strlen(tok)) % TOKEN_TABLE_SIZE;
158 if (token_table[n].tok == 0) {
159 if (++ntokens == TOKEN_TABLE_SIZE)
161 token_table[n].tok = tok;
164 if (strcmp(tok, token_table[n].tok) == 0)
167 n = TOKEN_TABLE_SIZE - 1;
171 token_table[n].ti.set(typ, sk, oc);
175 token_info default_token_info;
177 const token_info *lookup_token(const char *start, const char *end)
179 unsigned n = hash_string(start, end - start) % TOKEN_TABLE_SIZE;
181 if (token_table[n].tok == 0)
183 if (strlen(token_table[n].tok) == size_t(end - start)
184 && memcmp(token_table[n].tok, start, end - start) == 0)
185 return &(token_table[n].ti);
187 n = TOKEN_TABLE_SIZE - 1;
191 return &default_token_info;
194 static void init_ascii()
197 for (p = "abcdefghijklmnopqrstuvwxyz"; *p; p++) {
201 store_token(strsave(buf), TOKEN_LOWER);
202 buf[0] = cmupper(buf[0]);
203 store_token(strsave(buf), TOKEN_UPPER);
205 for (p = "0123456789"; *p; p++) {
209 const char *s = strsave(buf);
210 store_token(s, TOKEN_OTHER, s);
212 for (p = ".,:;?!"; *p; p++) {
216 store_token(strsave(buf), TOKEN_PUNCT);
218 store_token("-", TOKEN_HYPHEN);
221 static void store_letter(const char *lower, const char *upper,
222 const char *sort_key = 0)
224 store_token(lower, TOKEN_LOWER, sort_key, upper);
225 store_token(upper, TOKEN_UPPER, sort_key, lower);
228 static void init_letter(unsigned char uc_code, unsigned char lc_code,
229 const char *sort_key)
237 store_letter(strsave(lbuf), strsave(ubuf), sort_key);
240 static void init_latin1()
242 init_letter(0xc0, 0xe0, "a");
243 init_letter(0xc1, 0xe1, "a");
244 init_letter(0xc2, 0xe2, "a");
245 init_letter(0xc3, 0xe3, "a");
246 init_letter(0xc4, 0xe4, "a");
247 init_letter(0xc5, 0xe5, "a");
248 init_letter(0xc6, 0xe6, "ae");
249 init_letter(0xc7, 0xe7, "c");
250 init_letter(0xc8, 0xe8, "e");
251 init_letter(0xc9, 0xe9, "e");
252 init_letter(0xca, 0xea, "e");
253 init_letter(0xcb, 0xeb, "e");
254 init_letter(0xcc, 0xec, "i");
255 init_letter(0xcd, 0xed, "i");
256 init_letter(0xce, 0xee, "i");
257 init_letter(0xcf, 0xef, "i");
259 init_letter(0xd0, 0xf0, "d");
260 init_letter(0xd1, 0xf1, "n");
261 init_letter(0xd2, 0xf2, "o");
262 init_letter(0xd3, 0xf3, "o");
263 init_letter(0xd4, 0xf4, "o");
264 init_letter(0xd5, 0xf5, "o");
265 init_letter(0xd6, 0xf6, "o");
266 init_letter(0xd8, 0xf8, "o");
267 init_letter(0xd9, 0xf9, "u");
268 init_letter(0xda, 0xfa, "u");
269 init_letter(0xdb, 0xfb, "u");
270 init_letter(0xdc, 0xfc, "u");
271 init_letter(0xdd, 0xfd, "y");
272 init_letter(0xde, 0xfe, THORN_SORT_KEY);
274 store_token("\337", TOKEN_LOWER, "ss", "SS");
275 store_token("\377", TOKEN_LOWER, "y", "Y");
278 static void init_two_char_letter(char l1, char l2, char u1, char u2,
287 const char *p = strsave(buf);
290 store_letter(p, strsave(buf), sk);
297 store_letter(strsave(buf), p, sk);
301 static void init_special_chars()
304 for (p = "':^`~"; *p; p++)
305 for (const char *q = "aeiouy"; *q; q++) {
306 // Use a variable to work around bug in gcc 2.0
307 char c = cmupper(*q);
308 init_two_char_letter(*p, *q, *p, c);
310 for (p = "/l/o~n,coeaeij"; *p; p += 2) {
311 // Use variables to work around bug in gcc 2.0
312 char c0 = cmupper(p[0]);
313 char c1 = cmupper(p[1]);
314 init_two_char_letter(p[0], p[1], c0, c1);
316 init_two_char_letter('v', 's', 'v', 'S', "s");
317 init_two_char_letter('v', 'z', 'v', 'Z', "z");
318 init_two_char_letter('o', 'a', 'o', 'A', "a");
319 init_two_char_letter('T', 'p', 'T', 'P', THORN_SORT_KEY);
320 init_two_char_letter('-', 'd', '-', 'D');
322 store_token("\\(ss", TOKEN_LOWER, 0, "SS");
323 store_token("\\[ss]", TOKEN_LOWER, 0, "SS");
325 store_token("\\(Sd", TOKEN_LOWER, "d", "\\(-D");
326 store_token("\\[Sd]", TOKEN_LOWER, "d", "\\[-D]");
327 store_token("\\(hy", TOKEN_HYPHEN);
328 store_token("\\[hy]", TOKEN_HYPHEN);
329 store_token("\\(en", TOKEN_RANGE_SEP);
330 store_token("\\[en]", TOKEN_RANGE_SEP);
333 static void init_strings()
338 for (const char *p = "'`^^,:~v_o./;"; *p; p++) {
341 store_token(strsave(buf), TOKEN_ACCENT);
346 store_token(strsave(buf), TOKEN_ACCENT);
349 // -ms special letters
350 store_letter("\\*(th", "\\*(Th", THORN_SORT_KEY);
351 store_letter("\\*[th]", "\\*[Th]", THORN_SORT_KEY);
352 store_letter("\\*(d-", "\\*(D-");
353 store_letter("\\*[d-]", "\\*[D-]");
354 store_letter("\\*(ae", "\\*(Ae", "ae");
355 store_letter("\\*[ae]", "\\*[Ae]", "ae");
356 store_letter("\\*(oe", "\\*(Oe", "oe");
357 store_letter("\\*[oe]", "\\*[Oe]", "oe");
359 store_token("\\*3", TOKEN_LOWER, "y", "Y");
360 store_token("\\*8", TOKEN_LOWER, "ss", "SS");
361 store_token("\\*q", TOKEN_LOWER, "o", "O");
364 struct token_initer {
368 static token_initer the_token_initer;
370 token_initer::token_initer()
374 init_special_chars();
376 default_token_info.set(TOKEN_OTHER);