3 * Library to deal with pinyin.
5 * Copyright (C) 2011 Peng Wu <alexepico@gmail.com>
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
23 #include "pinyin_parser2.h"
29 #include "pinyin_phrase2.h"
30 #include "pinyin_custom2.h"
31 #include "chewing_key.h"
32 #include "pinyin_parser_table.h"
33 #include "double_pinyin_table.h"
34 #include "chewing_table.h"
37 using namespace pinyin;
39 static bool check_pinyin_options(pinyin_option_t options, const pinyin_index_item_t * item) {
40 guint32 flags = item->m_flags;
41 assert (flags & IS_PINYIN);
43 /* handle incomplete pinyin. */
44 if (flags & PINYIN_INCOMPLETE) {
45 if (!(options & PINYIN_INCOMPLETE))
49 /* handle correct pinyin, currently only one flag per item. */
50 flags &= PINYIN_CORRECT_ALL;
51 options &= PINYIN_CORRECT_ALL;
54 if ((flags & options) != flags)
61 static bool check_chewing_options(pinyin_option_t options, const chewing_index_item_t * item) {
62 guint32 flags = item->m_flags;
63 assert (flags & IS_CHEWING);
65 /* handle incomplete chewing. */
66 if (flags & CHEWING_INCOMPLETE) {
67 if (!(options & CHEWING_INCOMPLETE))
75 gint ChewingKey::get_table_index() {
76 assert(m_initial < CHEWING_NUMBER_OF_INITIALS);
77 assert(m_middle < CHEWING_NUMBER_OF_MIDDLES);
78 assert(m_final < CHEWING_NUMBER_OF_FINALS);
80 gint index = chewing_key_table[(m_initial * CHEWING_NUMBER_OF_MIDDLES + m_middle) * CHEWING_NUMBER_OF_FINALS + m_final];
81 return index == -1 ? 0 : index;
84 gchar * ChewingKey::get_pinyin_string() {
85 assert(m_tone < CHEWING_NUMBER_OF_TONES);
86 gint index = get_table_index();
87 assert(index < G_N_ELEMENTS(content_table));
88 const content_table_item_t & item = content_table[index];
90 if (CHEWING_ZERO_TONE == m_tone) {
91 return g_strdup(item.m_pinyin_str);
93 return g_strdup_printf("%s%d", item.m_pinyin_str, m_tone);
97 gchar * ChewingKey::get_chewing_string() {
98 assert(m_tone < CHEWING_NUMBER_OF_TONES);
99 gint index = get_table_index();
100 assert(index < G_N_ELEMENTS(content_table));
101 const content_table_item_t & item = content_table[index];
103 if (CHEWING_ZERO_TONE == m_tone) {
104 return g_strdup(item.m_chewing_str);
106 return g_strdup_printf("%s%s", item.m_chewing_str,
107 chewing_tone_table[m_tone]);
114 /* internal information for pinyin parsers. */
115 struct parse_value_t{
117 ChewingKeyRest m_key_rest;
131 const guint16 max_full_pinyin_length = 7; /* include tone. */
133 const guint16 max_double_pinyin_length = 3; /* include tone. */
135 const guint16 max_chewing_length = 4; /* include tone. */
137 static bool compare_pinyin_less_than(const pinyin_index_item_t & lhs,
138 const pinyin_index_item_t & rhs){
139 return 0 > strcmp(lhs.m_pinyin_input, rhs.m_pinyin_input);
142 static inline bool search_pinyin_index(pinyin_option_t options,
145 pinyin_index_item_t item;
146 memset(&item, 0, sizeof(item));
147 item.m_pinyin_input = pinyin;
149 std_lite::pair<const pinyin_index_item_t *,
150 const pinyin_index_item_t *> range;
151 range = std_lite::equal_range
152 (pinyin_index, pinyin_index + G_N_ELEMENTS(pinyin_index),
153 item, compare_pinyin_less_than);
155 guint16 range_len = range.second - range.first;
156 assert(range_len <= 1);
157 if (range_len == 1) {
158 const pinyin_index_item_t * index = range.first;
160 if (!check_pinyin_options(options, index))
163 key = content_table[index->m_table_index].m_chewing_key;
164 assert(key.get_table_index() == index->m_table_index);
171 static bool compare_chewing_less_than(const chewing_index_item_t & lhs,
172 const chewing_index_item_t & rhs){
173 return 0 > strcmp(lhs.m_chewing_input, rhs.m_chewing_input);
176 static inline bool search_chewing_index(pinyin_option_t options,
177 const char * chewing,
179 chewing_index_item_t item;
180 memset(&item, 0, sizeof(item));
181 item.m_chewing_input = chewing;
183 std_lite::pair<const chewing_index_item_t *,
184 const chewing_index_item_t *> range;
185 range = std_lite::equal_range
186 (chewing_index, chewing_index + G_N_ELEMENTS(chewing_index),
187 item, compare_chewing_less_than);
189 guint16 range_len = range.second - range.first;
190 assert (range_len <= 1);
192 if (range_len == 1) {
193 const chewing_index_item_t * index = range.first;
195 if (!check_chewing_options(options, index))
198 key = content_table[index->m_table_index].m_chewing_key;
199 assert(key.get_table_index() == index->m_table_index);
206 /* Full Pinyin Parser */
207 FullPinyinParser2::FullPinyinParser2 (){
208 m_parse_steps = g_array_new(TRUE, FALSE, sizeof(parse_value_t));
212 bool FullPinyinParser2::parse_one_key (pinyin_option_t options,
214 const char * pinyin, int len) const {
215 /* "'" are not accepted in parse_one_key. */
216 gchar * input = g_strndup(pinyin, len);
217 assert(NULL == strchr(input, '\''));
219 guint16 tone = CHEWING_ZERO_TONE; guint16 tone_pos = 0;
220 guint16 parsed_len = len;
223 if (options & USE_TONE) {
224 /* find the tone in the last character. */
225 char chr = input[parsed_len - 1];
226 if ( '0' < chr && chr <= '5' ) {
229 tone_pos = parsed_len;
233 /* parse pinyin core staff here. */
235 /* Note: optimize here? */
236 input[parsed_len] = '\0';
237 if (!search_pinyin_index(options, input, key)) {
242 if (options & USE_TONE) {
243 /* post processing tone. */
244 if ( parsed_len == tone_pos ) {
245 if (tone != CHEWING_ZERO_TONE) {
253 return parsed_len == len;
257 int FullPinyinParser2::parse (pinyin_option_t options, ChewingKeyVector & keys,
258 ChewingKeyRestVector & key_rests,
259 const char *str, int len) const {
262 g_array_set_size(keys, 0);
263 g_array_set_size(key_rests, 0);
265 /* init m_parse_steps, and prepare dynamic programming. */
266 int step_len = len + 1;
267 g_array_set_size(m_parse_steps, 0);
269 for (i = 0; i < step_len; ++i) {
270 g_array_append_val(m_parse_steps, value);
274 gchar * input = g_strndup(str, len);
275 parse_value_t * curstep = NULL, * nextstep = NULL;
277 for (i = 0; i < len; ++i) {
278 if (input[i] == '\'') {
279 curstep = &g_array_index(m_parse_steps, parse_value_t, i);
280 nextstep = &g_array_index(m_parse_steps, parse_value_t, i + 1);
282 /* propagate current step into next step. */
283 nextstep->m_key = ChewingKey();
284 nextstep->m_key_rest = ChewingKeyRest();
285 nextstep->m_num_keys = curstep->m_num_keys;
286 nextstep->m_parsed_len = curstep->m_parsed_len + 1;
287 nextstep->m_last_step = i;
292 /* forward to next "'" */
293 if ( 0 == next_sep ) {
295 for (k = i; k < len; ++k) {
296 if (input[k] == '\'')
304 * do maximum forward match first. */
305 for (size_t pos = i; pos < next_sep; ++pos) {
306 curstep = &g_array_index(m_parse_steps, parse_value_t, pos);
307 size_t try_len = std_lite::min
308 (pos + max_full_pinyin_length, next_sep);
309 for (size_t n = try_len; n > pos; --n) {
310 nextstep = &g_array_index(m_parse_steps, parse_value_t, n);
313 const char * onepinyin = input + pos;
314 gint16 onepinyinlen = n - pos;
315 value = parse_value_t();
317 ChewingKey key; ChewingKeyRest rest;
318 bool parsed = parse_one_key
319 (options, key, onepinyin, onepinyinlen);
320 rest.m_raw_begin = pos; rest.m_raw_end = n;
325 //printf("onepinyin:%s len:%d\n", onepinyin, onepinyinlen);
326 value.m_key = key; value.m_key_rest = rest;
327 value.m_num_keys = curstep->m_num_keys + 1;
328 value.m_parsed_len = curstep->m_parsed_len + onepinyinlen;
329 value.m_last_step = pos;
332 if (-1 == nextstep->m_last_step)
334 if (value.m_parsed_len > nextstep->m_parsed_len)
336 if (value.m_parsed_len == nextstep->m_parsed_len &&
337 value.m_num_keys < nextstep->m_num_keys)
340 /* maximum forward, set pos to n in next iteration. */
347 /* dynamic programming here. */
348 for (size_t m = i; m < next_sep; ++m) {
349 curstep = &g_array_index(m_parse_steps, parse_value_t, m);
350 size_t try_len = std_lite::min
351 (m + max_full_pinyin_length, next_sep);
352 for (size_t n = m + 1; n < try_len + 1; ++n) {
353 nextstep = &g_array_index(m_parse_steps, parse_value_t, n);
356 const char * onepinyin = input + m;
357 gint16 onepinyinlen = n - m;
358 value = parse_value_t();
360 ChewingKey key; ChewingKeyRest rest;
361 bool parsed = parse_one_key
362 (options, key, onepinyin, onepinyinlen);
363 rest.m_raw_begin = m; rest.m_raw_end = n;
367 //printf("onepinyin:%s len:%d\n", onepinyin, onepinyinlen);
369 value.m_key = key; value.m_key_rest = rest;
370 value.m_num_keys = curstep->m_num_keys + 1;
371 value.m_parsed_len = curstep->m_parsed_len + onepinyinlen;
372 value.m_last_step = m;
375 if (-1 == nextstep->m_last_step)
377 if (value.m_parsed_len > nextstep->m_parsed_len)
379 if (value.m_parsed_len == nextstep->m_parsed_len &&
380 value.m_num_keys < nextstep->m_num_keys)
382 if (value.m_parsed_len == nextstep->m_parsed_len &&
383 value.m_num_keys == nextstep->m_num_keys) {
385 /* "kaneiji" -> "ka'nei'ji" */
386 if ((value.m_key.m_initial != CHEWING_ZERO_INITIAL &&
387 !(value.m_key.m_middle == CHEWING_ZERO_MIDDLE &&
388 value.m_key.m_final == CHEWING_ZERO_FINAL)) &&
389 nextstep->m_key.m_initial == CHEWING_ZERO_INITIAL)
392 /* "xierqi" -> "xi'er'qi." */
393 if ((value.m_key.m_initial == CHEWING_ZERO_INITIAL &&
394 value.m_key.m_middle == CHEWING_ZERO_MIDDLE &&
395 value.m_key.m_final == CHEWING_ER) &&
396 (nextstep->m_key.m_initial == CHEWING_R &&
397 nextstep->m_key.m_middle == CHEWING_ZERO_MIDDLE &&
398 nextstep->m_key.m_final == CHEWING_ZERO_FINAL))
401 /* "zheyanga$" -> "zhe'yang'a$" */
402 if (value.m_parsed_len == len &&
403 (nextstep->m_key.m_initial != CHEWING_ZERO_INITIAL &&
404 nextstep->m_key.m_final == CHEWING_A) &&
405 (value.m_key.m_initial == CHEWING_ZERO_INITIAL &&
406 value.m_key.m_middle == CHEWING_ZERO_MIDDLE &&
407 value.m_key.m_final == CHEWING_A))
414 /* final step for back tracing. */
415 gint16 parsed_len = final_step(step_len, keys, key_rests);
417 /* post processing for re-split table. */
418 if (options & USE_RESPLIT_TABLE) {
419 post_process(options, keys, key_rests);
426 int FullPinyinParser2::final_step(size_t step_len, ChewingKeyVector & keys,
427 ChewingKeyRestVector & key_rests) const{
429 gint16 parsed_len = 0;
430 parse_value_t * curstep = NULL;
432 /* find longest match, which starts from the beginning of input. */
433 for (i = step_len - 1; i >= 0; --i) {
434 curstep = &g_array_index(m_parse_steps, parse_value_t, i);
435 if (i == curstep->m_parsed_len)
438 /* prepare saving. */
439 parsed_len = curstep->m_parsed_len;
440 gint16 num_keys = curstep->m_num_keys;
441 g_array_set_size(keys, num_keys);
442 g_array_set_size(key_rests, num_keys);
444 /* save the match. */
445 while (curstep->m_last_step != -1) {
446 gint16 pos = curstep->m_num_keys - 1;
449 if (0 != curstep->m_key.get_table_index()) {
450 ChewingKey * key = &g_array_index(keys, ChewingKey, pos);
451 ChewingKeyRest * rest = &g_array_index
452 (key_rests, ChewingKeyRest, pos);
453 *key = curstep->m_key; *rest = curstep->m_key_rest;
457 curstep = &g_array_index(m_parse_steps, parse_value_t,
458 curstep->m_last_step);
464 bool FullPinyinParser2::post_process(pinyin_option_t options,
465 ChewingKeyVector & keys,
466 ChewingKeyRestVector & key_rests) const {
468 assert(keys->len == key_rests->len);
469 gint16 num_keys = keys->len;
471 ChewingKey * cur_key = NULL, * next_key = NULL;
472 ChewingKeyRest * cur_rest = NULL, * next_rest = NULL;
473 guint16 next_tone = CHEWING_ZERO_TONE;
475 for (i = 0; i < num_keys - 1; ++i) {
476 cur_rest = &g_array_index(key_rests, ChewingKeyRest, i);
477 next_rest = &g_array_index(key_rests, ChewingKeyRest, i + 1);
480 if (cur_rest->m_raw_end != next_rest->m_raw_begin)
483 cur_key = &g_array_index(keys, ChewingKey, i);
484 next_key = &g_array_index(keys, ChewingKey, i + 1);
487 if (CHEWING_ZERO_TONE != cur_key->m_tone)
490 if (options & USE_TONE) {
491 next_tone = next_key->m_tone;
492 next_key->m_tone = CHEWING_ZERO_TONE;
495 /* lookup re-split table */
497 const resplit_table_item_t * item = NULL;
498 for (k = 0; k < G_N_ELEMENTS(resplit_table); ++k) {
499 item = resplit_table + k;
501 if (item->m_orig_freq >= item->m_new_freq)
504 /* use pinyin_exact_compare2 here. */
505 if (0 == pinyin_exact_compare2(item->m_orig_keys,
512 if (k < G_N_ELEMENTS(resplit_table)) {
514 item = resplit_table + k;
515 *cur_key = item->m_new_keys[0];
516 *next_key = item->m_new_keys[1];
517 /* assumes only moved one char in gen_all_resplit script. */
518 cur_rest->m_raw_end ++;
519 next_rest->m_raw_begin ++;
522 /* save back tones */
523 if (options & USE_TONE) {
524 next_key->m_tone = next_tone;
531 #define IS_KEY(x) (('a' <= x && x <= 'z') || x == ';')
533 bool DoublePinyinParser2::parse_one_key(pinyin_option_t options,
535 const char *str, int len) const {
536 options &= ~(PINYIN_CORRECT_ALL|PINYIN_AMB_ALL);
539 if (!(options & PINYIN_INCOMPLETE))
546 int charid = ch == ';' ? 26 : ch - 'a';
547 const char * sheng = m_shengmu_table[charid].m_shengmu;
548 if (NULL == sheng || strcmp(sheng, "'") == 0)
551 if (search_pinyin_index(options, sheng, key)) {
558 ChewingTone tone = CHEWING_ZERO_TONE;
559 options &= ~(PINYIN_INCOMPLETE|CHEWING_INCOMPLETE);
563 if (!(options & USE_TONE))
566 if (!('0' < ch && ch <= '5'))
568 tone = (ChewingTone) (ch - '0');
571 if (2 == len || 3 == len) {
572 /* parse shengmu here. */
577 int charid = ch == ';' ? 26 : ch - 'a';
578 const char * sheng = m_shengmu_table[charid].m_shengmu;
581 if (strcmp(sheng, "'") == 0)
584 /* parse yunmu here. */
589 charid = ch == ';' ? 26 : ch - 'a';
591 const char * yun = m_yunmu_table[charid].m_yunmus[0];
595 gchar * pinyin = g_strdup_printf("%s%s", sheng, yun);
596 if (search_pinyin_index(options, pinyin, key)) {
604 yun = m_yunmu_table[charid].m_yunmus[1];
608 pinyin = g_strdup_printf("%s%s", sheng, yun);
609 if (search_pinyin_index(options, pinyin, key)) {
622 /* only 'a'-'z' and ';' are accepted here. */
623 int DoublePinyinParser2::parse(pinyin_option_t options, ChewingKeyVector & keys,
624 ChewingKeyRestVector & key_rests,
625 const char *str, int len) const {
626 g_array_set_size(keys, 0);
627 g_array_set_size(key_rests, 0);
629 int maximum_len = 0; int i;
630 /* probe the longest possible double pinyin string. */
631 for (i = 0; i < len; ++i) {
632 const char ch = str[i];
633 if (!(IS_KEY(ch) || ('0' < ch && ch <= '5')))
638 /* maximum forward match for double pinyin. */
640 while (parsed_len < maximum_len) {
641 const char * cur_str = str + parsed_len;
642 i = std_lite::min(maximum_len - parsed_len,
643 (int)max_double_pinyin_length);
645 ChewingKey key; ChewingKeyRest key_rest;
647 bool success = parse_one_key(options, key, cur_str, i);
652 if (0 == i) /* no more possible double pinyins. */
655 key_rest.m_raw_begin = parsed_len; key_rest.m_raw_end = parsed_len + i;
658 /* save the pinyin */
659 g_array_append_val(keys, key);
660 g_array_append_val(key_rests, key_rest);
668 bool DoublePinyinParser2::set_scheme(DoublePinyinScheme scheme) {
671 case DOUBLE_PINYIN_ZRM:
672 m_shengmu_table = double_pinyin_zrm_sheng;
673 m_yunmu_table = double_pinyin_zrm_yun;
675 case DOUBLE_PINYIN_MS:
676 m_shengmu_table = double_pinyin_mspy_sheng;
677 m_yunmu_table = double_pinyin_mspy_yun;
679 case DOUBLE_PINYIN_ZIGUANG:
680 m_shengmu_table = double_pinyin_zgpy_sheng;
681 m_yunmu_table = double_pinyin_zgpy_yun;
683 case DOUBLE_PINYIN_ABC:
684 m_shengmu_table = double_pinyin_abc_sheng;
685 m_yunmu_table = double_pinyin_abc_yun;
687 case DOUBLE_PINYIN_PYJJ:
688 m_shengmu_table = double_pinyin_pyjj_sheng;
689 m_yunmu_table = double_pinyin_pyjj_yun;
691 case DOUBLE_PINYIN_XHE:
692 m_shengmu_table = double_pinyin_xhe_sheng;
693 m_yunmu_table = double_pinyin_xhe_yun;
695 case DOUBLE_PINYIN_CUSTOMIZED:
699 return false; /* no such scheme. */
702 /* the chewing string must be freed with g_free. */
703 static bool search_chewing_symbols(const chewing_symbol_item_t * symbol_table,
704 const char key, const char ** chewing) {
706 /* just iterate the table, as we only have < 50 items. */
707 while (symbol_table->m_input != '\0') {
708 if (symbol_table->m_input == key) {
709 *chewing = symbol_table->m_chewing;
717 static bool search_chewing_tones(const chewing_tone_item_t * tone_table,
718 const char key, char * tone) {
719 *tone = CHEWING_ZERO_TONE;
720 /* just iterate the table, as we only have < 10 items. */
721 while (tone_table->m_input != '\0') {
722 if (tone_table->m_input == key) {
723 *tone = tone_table->m_tone;
732 bool ChewingParser2::parse_one_key(pinyin_option_t options,
734 const char *str, int len) const {
735 options &= ~(PINYIN_CORRECT_ALL|PINYIN_AMB_ALL);
736 char tone = CHEWING_ZERO_TONE;
738 int symbols_len = len;
739 /* probe whether the last key is tone key in str. */
740 if (options & USE_TONE) {
741 char ch = str[len - 1];
742 /* remove tone from input */
743 if (search_chewing_tones(m_tone_table, ch, &tone))
748 gchar * chewing = NULL; const char * onechar = NULL;
750 /* probe the possible chewing map in the rest of str. */
751 for (i = 0; i < symbols_len; ++i) {
752 if (!search_chewing_symbols(m_symbol_table, str[i], &onechar)) {
758 chewing = g_strdup(onechar);
760 gchar * tmp = chewing;
761 chewing = g_strconcat(chewing, onechar, NULL);
766 /* search the chewing in the chewing index table. */
767 if (chewing && search_chewing_index(options, chewing, key)) {
768 /* save back tone if available. */
779 /* only characters in chewing keyboard scheme are accepted here. */
780 int ChewingParser2::parse(pinyin_option_t options, ChewingKeyVector & keys,
781 ChewingKeyRestVector & key_rests,
782 const char *str, int len) const {
783 g_array_set_size(keys, 0);
784 g_array_set_size(key_rests, 0);
786 int maximum_len = 0; int i;
787 /* probe the longest possible chewing string. */
788 for (i = 0; i < len; ++i) {
789 if (!in_chewing_scheme(options, str[i], NULL))
794 /* maximum forward match for chewing. */
796 while (parsed_len < maximum_len) {
797 const char * cur_str = str + parsed_len;
798 i = std_lite::min(maximum_len - parsed_len,
799 (int)max_chewing_length);
801 ChewingKey key; ChewingKeyRest key_rest;
803 bool success = parse_one_key(options, key, cur_str, i);
808 if (0 == i) /* no more possible chewings. */
811 key_rest.m_raw_begin = parsed_len; key_rest.m_raw_end = parsed_len + i;
814 /* save the pinyin. */
815 g_array_append_val(keys, key);
816 g_array_append_val(key_rests, key_rest);
823 bool ChewingParser2::set_scheme(ChewingScheme scheme) {
825 case CHEWING_STANDARD:
826 m_symbol_table = chewing_standard_symbols;
827 m_tone_table = chewing_standard_tones;
830 m_symbol_table = chewing_ibm_symbols;
831 m_tone_table = chewing_ibm_tones;
833 case CHEWING_GINYIEH:
834 m_symbol_table = chewing_ginyieh_symbols;
835 m_tone_table = chewing_ginyieh_tones;
838 m_symbol_table = chewing_eten_symbols;
839 m_tone_table = chewing_eten_tones;
847 bool ChewingParser2::in_chewing_scheme(pinyin_option_t options,
848 const char key, const char ** symbol)
850 const gchar * chewing = NULL;
851 char tone = CHEWING_ZERO_TONE;
853 if (search_chewing_symbols(m_symbol_table, key, &chewing)) {
859 if (!(options & USE_TONE))
862 if (search_chewing_tones(m_tone_table, key, &tone)) {
864 *symbol = chewing_tone_table[tone];