fixes read document

diff --git a/utils/training/gen_deleted_ngram.cpp b/utils/training/gen_deleted_ngram.cpp
index 93986d6..b39490f 100644 (file)
--- a/utils/training/gen_deleted_ngram.cpp
+++ b/utils/training/gen_deleted_ngram.cpp
@@ -78,13 +78,14 @@ int main(int argc, char * argv[]){
         glong phrase_len = 0;
         utf16_t * phrase = g_utf8_to_utf16(linebuf, -1, NULL, &phrase_len, NULL);
 
-        if ( phrase_len == 0 )
-            continue;
-
        phrase_token_t token = 0;
-        int result = g_phrases->search( phrase_len, phrase, token);
-       if ( ! (result & SEARCH_OK) )
-           token = 0;
+        if ( 0 != phrase_len ) {
+            int result = g_phrases->search( phrase_len, phrase, token);
+            if ( ! (result & SEARCH_OK) )
+                token = 0;
+            g_free(phrase);
+            phrase = NULL;
+        }
 
        last_token = cur_token;
        cur_token = token;

diff --git a/utils/training/gen_k_mixture_model.cpp b/utils/training/gen_k_mixture_model.cpp
index 13ff04d..bd350b4 100644 (file)
--- a/utils/training/gen_k_mixture_model.cpp
+++ b/utils/training/gen_k_mixture_model.cpp
@@ -60,16 +60,14 @@ bool read_document(PhraseLargeTable * phrases, FILE * document,
         glong phrase_len = 0;
         utf16_t * phrase = g_utf8_to_utf16(linebuf, -1, NULL, &phrase_len, NULL);
 
-        if ( phrase_len == 0 )
-            continue;
-
         phrase_token_t token = 0;
-        int search_result = phrases->search( phrase_len, phrase, token );
-        if ( ! (search_result & SEARCH_OK) )
-            token = 0;
-
-        g_free(phrase);
-        phrase = NULL;
+        if ( 0 != phrase_len ) {
+            int search_result = phrases->search( phrase_len, phrase, token );
+            if ( ! (search_result & SEARCH_OK) )
+                token = 0;
+            g_free(phrase);
+            phrase = NULL;
+        }
 
         last_token = cur_token;
         cur_token = token;

diff --git a/utils/training/gen_ngram.cpp b/utils/training/gen_ngram.cpp
index 9e79f1d..2b0e1dd 100644 (file)
--- a/utils/training/gen_ngram.cpp
+++ b/utils/training/gen_ngram.cpp
@@ -91,13 +91,14 @@ int main(int argc, char * argv[]){
         glong phrase_len = 0;
         utf16_t * phrase = g_utf8_to_utf16(linebuf, -1, NULL, &phrase_len, NULL);
 
-        if ( phrase_len == 0 )
-            continue;
-
        phrase_token_t token = 0;
-        int result = g_phrases->search( phrase_len, phrase, token);
-       if ( ! (result & SEARCH_OK) )
-           token = 0;
+        if ( 0 != phrase_len ) {
+            int result = g_phrases->search( phrase_len, phrase, token);
+            if ( ! (result & SEARCH_OK) )
+                token = 0;
+            g_free(phrase);
+            phrase = NULL;
+        }
 
        last_token = cur_token;
        cur_token = token;