fixes gen_extra_enter

diff --git a/utils/segment/ngseg.cpp b/utils/segment/ngseg.cpp
index a36c554..302eb25 100644 (file)
--- a/utils/segment/ngseg.cpp
+++ b/utils/segment/ngseg.cpp
@@ -159,13 +159,13 @@ int main(int argc, char * argv[]){
         ucs4_t * sentence = g_utf8_to_ucs4(linebuf, -1, NULL, &len, NULL);
         if ( len != num_of_chars ) {
             fprintf(stderr, "non-ucs4 characters encountered:%s.\n", linebuf);
-            fprintf(output, "\n");
+            fprintf(output, "%d \n", null_token);
             continue;
         }
 
         /* only new-line persists. */
         if ( 0  == num_of_chars ) {
-            fprintf(output, "\n");
+            fprintf(output, "%d \n", null_token);
             continue;
         }
 
@@ -214,12 +214,12 @@ int main(int argc, char * argv[]){
 
         /* print extra enter */
         if ( gen_extra_enter )
-            fprintf(output, "\n");
+            fprintf(output, "%d \n", null_token);
     }
     phrase_index.destroy_tokens(tokens);
 
     /* print enter at file tail */
-    fprintf(output, "\n");
+    fprintf(output, "%d \n", null_token);
     g_array_free(current_ucs4, TRUE);
     free(linebuf);
     fclose(input);

diff --git a/utils/segment/spseg.cpp b/utils/segment/spseg.cpp
index f24b7a1..85aa14c 100644 (file)
--- a/utils/segment/spseg.cpp
+++ b/utils/segment/spseg.cpp
@@ -200,7 +200,7 @@ int main(int argc, char * argv[]){
         ucs4_t * sentence = g_utf8_to_ucs4(linebuf, -1, NULL, &len, NULL);
         if ( len != num_of_chars ) {
             fprintf(stderr, "non-ucs4 characters encountered:%s.\n", linebuf);
-            fprintf(output, "\n");
+            fprintf(output, "%d \n", null_token);
             continue;
         }
 
@@ -218,14 +218,14 @@ int main(int argc, char * argv[]){
 
         /* print extra enter */
         if ( gen_extra_enter )
-            fprintf(output, "\n");
+            fprintf(output, "%d \n", null_token);
 
         g_array_free(strings, TRUE);
         g_free(sentence);
     }
 
     /* print enter at file tail */
-    fprintf(output, "\n");
+    fprintf(output, "%d \n", null_token);
     fclose(input);
     fclose(output);
     return 0;