From f7bfa0e630aaf59c40cfa2fbed64a986b18505df Mon Sep 17 00:00:00 2001 From: Peng Wu Date: Fri, 19 Oct 2012 12:17:03 +0800 Subject: [PATCH] update eval correction rate --- utils/training/eval_correction_rate.cpp | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/utils/training/eval_correction_rate.cpp b/utils/training/eval_correction_rate.cpp index dd36750..ed37266 100644 --- a/utils/training/eval_correction_rate.cpp +++ b/utils/training/eval_correction_rate.cpp @@ -160,24 +160,13 @@ int main(int argc, char * argv[]){ while( getline(&linebuf, &size, evals_file) ) { if ( feof(evals_file) ) break; - if ( '\n' == linebuf[strlen(linebuf)-1] ) - linebuf[strlen(linebuf)-1] = '\0'; - glong phrase_len = 0; - ucs4_t * phrase = g_utf8_to_ucs4(linebuf, -1, NULL, &phrase_len, NULL); - - token = null_token; - if ( 0 != phrase_len ) { - int result = phrase_table.search(phrase_len, phrase, phrase_tokens); - int num = get_first_token(phrase_tokens, token); - - if ( !(result & SEARCH_OK) ) - token = null_token; - - g_free(phrase); - phrase = NULL; + if ( '\n' == linebuf[strlen(linebuf) - 1] ) { + linebuf[strlen(linebuf) - 1] = '\0'; } + TAGLIB_PARSE_SEGMENTED_LINE(&phrase_index, token, linebuf); + if ( null_token == token ) { if ( tokens->len ) { /* one test. */ if ( do_one_test(&pinyin_lookup, &phrase_index, tokens) ) { -- 2.7.4