Improve nlp service code

1.make function for each nltk api
2.add Py_DECREF(X) for reduce reference counting

Change-Id: I11bfc3d2f7e3b1f75d970531f0b9063a439622c7

diff --git a/service/inc/service.h b/service/inc/service.h
index a6f723c5a8821acb3ced5d835580afaa226edf36..8c92915a61230704c4a0e3cfd8cf949e44bd3c52 100755 (executable)
--- a/service/inc/service.h
+++ b/service/inc/service.h
@@ -16,9 +16,13 @@ typedef enum {
     NLTK_CMD_UNKNOWN
 }NLTK_CMDS;
 
-// This method is exported from nltk_native_library.so
 void nltk_initialize();
 void nltk_finalize();
+PyObject* nltk_word_tokenize(char* sentence);
+PyObject* nltk_pos_tag(char* sentence);
+PyObject* nltk_ne_chunk(char* sentence);
+PyObject* nltk_lemmatize(char* sentence);
+PyObject* nltk_language_detect(char* sentence);
 PyObject* nltk_get_module(char* name);
 PyObject* nltk_get_function_handle(PyObject* module , char * func_name);
 PyObject* nltk_make_args_from_pyobject(PyObject* pyobj);

diff --git a/service/src/service.c b/service/src/service.c
index 001f5615c43e84a35f5123aeb65697e3a1258efa..39303a41029f8a13941a9af4fcf378618649fe09 100755 (executable)
--- a/service/src/service.c
+++ b/service/src/service.c
@@ -116,18 +116,6 @@ static int __message_send(rpc_port_stub_message_context_h context,
     PyObject* args = NULL;
     PyObject* lists = NULL;
     unsigned static int len = 0;
-    PyObject* postag_module = NULL;
-    PyObject* postag_result = NULL;
-    PyObject* ne_module = NULL;
-    PyObject* ne_result = NULL;
-    PyObject* lv_module = NULL;
-    PyObject* lv_result = NULL;
-    PyObject* lm_module = NULL;
-    PyObject* lm_result = NULL;
-    PyObject* lm_object = NULL;
-    PyObject* ld_module = NULL;
-    PyObject* ld_result = NULL;
-    PyObject* pos_elm_tuple = NULL;
     bundle_get_str(msg, "request_id", &request_id);
     bundle_get_str(msg, "info", &info);
     NLTK_CMDS cmd = NLTK_CMD_NONE;
@@ -161,9 +149,10 @@ static int __message_send(rpc_port_stub_message_context_h context,
     switch (cmd)
     {
         case NLTK_CMD_TOKENIZE:
-            args = nltk_make_args_from_string(info);
-            lists = nltk_call_function_with_args(nltk_get_function_handle(globe_nltk,"word_tokenize"), args);
-            len = nltk_get_size_from_list(lists);
+        {
+            PyObject* wt_lists = NULL;
+            wt_lists = nltk_word_tokenize(info);
+            len = nltk_get_size_from_list(wt_lists);
             char *tokens[BUF_LEN_128] = {NULL,};
             for(int i = 0 ;i < len ;i++)
             {
@@ -171,7 +160,7 @@ static int __message_send(rpc_port_stub_message_context_h context,
                 if(tokens[i]!=NULL)
                 {
                     memset(tokens[i], 0, BUF_LEN_128);
-                    tmp_str = nltk_get_string_from_element(nltk_get_element_from_list_by_index(lists, i));
+                    tmp_str = nltk_get_string_from_element(nltk_get_element_from_list_by_index(wt_lists, i));
                     if(tmp_str!=NULL)
                     {
                         strncpy(tokens[i], tmp_str, BUF_LEN_128-1);
@@ -185,30 +174,31 @@ static int __message_send(rpc_port_stub_message_context_h context,
             }
             bundle_add_str(reply, "command", "word_tokenize");
             bundle_add_str_array(reply, "return_token", tokens, len);
-            PINFO("word_tokenize process done");
             for(int j = 0 ;j < len ;j++)
             {
                 free(tokens[j]);
             }
+            Py_DECREF(wt_lists);
+            PINFO("word_tokenize process done");
             break;
+        }
         case NLTK_CMD_POSTAG:
-            args = nltk_make_args_from_string(info);
-            lists = nltk_call_function_with_args(nltk_get_function_handle(globe_nltk,"word_tokenize"), args);
-            postag_module = nltk_get_function_handle(globe_nltk,"pos_tag");
-            args = nltk_make_args_from_pyobject(lists);
-            postag_result = nltk_call_function_with_args(postag_module, args);
-            len = nltk_get_size_from_list(postag_result);
+        {
+            PyObject* pt_result = NULL;
+            PyObject* pt_elm_tuple = NULL;
+            pt_result = nltk_pos_tag(info);
+            len = nltk_get_size_from_list(pt_result);
             char *tag[BUF_LEN_128] = {NULL,};
             char *token[BUF_LEN_128] = {NULL,};
             for(int i = 0 ;i < len ;i++)
             {
                 token[i] = (char*)malloc(BUF_LEN_128*sizeof(char));
                 tag[i] = (char*)malloc(BUF_LEN_128*sizeof(char));
-                pos_elm_tuple = nltk_get_element_from_list_by_index(postag_result, i);
+                pt_elm_tuple = nltk_get_element_from_list_by_index(pt_result, i);
                 if(tag[i]!=NULL)
                 {
                     memset(tag[i], 0, BUF_LEN_128);
-                    tmp_str = nltk_get_string_from_element(nltk_get_element_from_tuple_by_index(pos_elm_tuple, 0));
+                    tmp_str = nltk_get_string_from_element(nltk_get_element_from_tuple_by_index(pt_elm_tuple, 0));
                     if(tmp_str!=NULL)
                     {
                         strncpy(tag[i], tmp_str, BUF_LEN_128-1);
@@ -222,7 +212,7 @@ static int __message_send(rpc_port_stub_message_context_h context,
                 if(token[i]!=NULL)
                 {
                     memset(token[i], 0, BUF_LEN_128);
-                    tmp_str = nltk_get_string_from_element(nltk_get_element_from_tuple_by_index(pos_elm_tuple, 1));
+                    tmp_str = nltk_get_string_from_element(nltk_get_element_from_tuple_by_index(pt_elm_tuple, 1));
                     if(tmp_str!=NULL)
                     {
                         strncpy(token[i], tmp_str, BUF_LEN_128-1);
@@ -242,31 +232,28 @@ static int __message_send(rpc_port_stub_message_context_h context,
                 free(tag[j]);
                 free(token[j]);
             }
+            Py_DECREF(pt_result);
+            Py_DECREF(pt_elm_tuple);
             PINFO("pos_tag process done");
             break;
+        }
         case NLTK_CMD_NECHUNK:
-            args = nltk_make_args_from_string(info);
-            lists = nltk_call_function_with_args(nltk_get_function_handle(globe_nltk,"word_tokenize"), args);
-            postag_module = nltk_get_function_handle(globe_nltk,"pos_tag");
-            args = nltk_make_args_from_pyobject(lists);
-            postag_result = nltk_call_function_with_args(postag_module, args);
-            args = nltk_make_args_from_pyobject(postag_result);
-            ne_module = nltk_get_function_handle(globe_nltk,"ne_chunk");
-            ne_result = nltk_call_function_with_args(ne_module, args);
-            lv_module = nltk_get_function_handle(ne_result, "leaves");
-            lv_result = nltk_call_function_with_args(lv_module, NULL);
-            len = nltk_get_size_from_list(lv_result);
+        {
+            PyObject* ne_result = NULL;
+            PyObject* ne_elm_tuple = NULL;
+            ne_result = nltk_ne_chunk(info);
+            len = nltk_get_size_from_list(ne_result);
             char *s_tag[BUF_LEN_128] = {NULL,};
             char *s_token[BUF_LEN_128] = {NULL,};
             for(int i = 0 ;i < len ;i++)
             {
                 s_token[i] = (char*)malloc(BUF_LEN_128*sizeof(char));
                 s_tag[i] = (char*)malloc(BUF_LEN_128*sizeof(char));
-                pos_elm_tuple = nltk_get_element_from_list_by_index(lv_result, i);
+                ne_elm_tuple = nltk_get_element_from_list_by_index(ne_result, i);
                 if(s_tag[i]!=NULL)
                 {
                     memset(s_tag[i], 0, BUF_LEN_128);
-                    tmp_str = nltk_get_string_from_element(nltk_get_element_from_tuple_by_index(pos_elm_tuple, 0));
+                    tmp_str = nltk_get_string_from_element(nltk_get_element_from_tuple_by_index(ne_elm_tuple, 0));
                     if(tmp_str!=NULL)
                     {
                         strncpy(s_tag[i], tmp_str, BUF_LEN_128-1);
@@ -280,7 +267,7 @@ static int __message_send(rpc_port_stub_message_context_h context,
                 if(s_token[i]!=NULL)
                 {
                     memset(s_token[i], 0, BUF_LEN_128);
-                    tmp_str = nltk_get_string_from_element(nltk_get_element_from_tuple_by_index(pos_elm_tuple, 1));
+                    tmp_str = nltk_get_string_from_element(nltk_get_element_from_tuple_by_index(ne_elm_tuple, 1));
                     if(tmp_str!=NULL)
                     {
                         strncpy(s_token[i], tmp_str, BUF_LEN_128-1);
@@ -295,19 +282,20 @@ static int __message_send(rpc_port_stub_message_context_h context,
             bundle_add_str(reply, "command", "ne_chunk");
             bundle_add_str_array(reply, "return_tag", s_tag, len);
             bundle_add_str_array(reply, "return_token", s_token, len);
-            PINFO("ne_chunk process done");
             for(int j = 0 ;j < len ;j++)
             {
                 free(s_tag[j]);
                 free(s_token[j]);
             }
+            Py_DECREF(ne_result);
+            Py_DECREF(ne_elm_tuple);
+            PINFO("ne_chunk process done");
             break;
+        }
         case NLTK_CMD_LEMMATIZE:
-            args = nltk_make_args_from_string(info);
-            lm_module = nltk_get_function_handle(globe_lemm,"WordNetLemmatizer");
-            lm_object = nltk_call_function_with_args(lm_module, NULL);
-            lm_module = nltk_get_function_handle(lm_object, "lemmatize");
-            lm_result = nltk_call_function_with_args(lm_module, args);
+        {
+            PyObject* lm_result = NULL;
+            lm_result = nltk_lemmatize(info);
             char *lem_buf[1] = {NULL,};
             lem_buf[0] = (char*)malloc(BUF_LEN_128*sizeof(char));
             if(lem_buf[0]!=NULL)
@@ -321,18 +309,20 @@ static int __message_send(rpc_port_stub_message_context_h context,
                 }
                 bundle_add_str(reply, "command", "lemmatize");
                 bundle_add_str_array(reply, "return_token", lem_buf, 1);
-                PINFO("lemmatize process done");
                 free(lem_buf[0]);
             }
             else
             {
                 PERR("malloc failed");
             }
+            Py_DECREF(lm_result);
+            PINFO("lemmatize process done");
             break;
+        }
         case NLTK_CMD_LANGDETECT:
-            args = nltk_make_args_from_string(info);
-            ld_module = nltk_get_function_handle(globe_lang,"detect");
-            ld_result = nltk_call_function_with_args(ld_module, args);
+        {
+            PyObject* ld_result = NULL;
+            ld_result = nltk_language_detect(info);
             char *lang_buf[1] = {NULL,};
             lang_buf[0] = (char*)malloc(BUF_LEN_128*sizeof(char));
             if(lang_buf[0]!=NULL)
@@ -353,7 +343,9 @@ static int __message_send(rpc_port_stub_message_context_h context,
             {
                 PERR("malloc failed");
             }
+            Py_DECREF(ld_result);
             break;
+        }
         default:
             PWARNING("unknown command, command = [%s]", message);
             bundle_add_str(reply, "command", "Exception happens");
@@ -474,6 +466,89 @@ void nltk_finalize()
     Py_Finalize();
 }
 
+PyObject* nltk_word_tokenize(char* sentence)
+{
+    PyObject* args = NULL;
+    PyObject* func = NULL;
+    PyObject* lists = NULL;
+    args = nltk_make_args_from_string(sentence);
+    func = nltk_get_function_handle(globe_nltk, "word_tokenize");
+    lists = nltk_call_function_with_args(func, args);
+    Py_DECREF(args);
+    Py_DECREF(func);
+    return lists;
+}
+
+PyObject* nltk_pos_tag(char* sentence)
+{
+    PyObject* args = NULL;
+    PyObject* func = NULL;
+    PyObject* wt_result = NULL;
+    PyObject* result = NULL;
+    wt_result = nltk_word_tokenize(sentence);
+    func = nltk_get_function_handle(globe_nltk, "pos_tag");
+    args = nltk_make_args_from_pyobject(wt_result);
+    result = nltk_call_function_with_args(func, args);
+    Py_DECREF(args);
+    Py_DECREF(func);
+    Py_DECREF(wt_result);
+    return result;
+}
+
+PyObject* nltk_ne_chunk(char* sentence)
+{
+    PyObject* args = NULL;
+    PyObject* pt_result = NULL;
+    PyObject* tmp_result = NULL;
+    PyObject* result = NULL;
+    PyObject* func = NULL;
+    PyObject* lv_func = NULL;
+    pt_result = nltk_pos_tag(sentence);
+    args = nltk_make_args_from_pyobject(pt_result);
+    func = nltk_get_function_handle(globe_nltk, "ne_chunk");
+    tmp_result = nltk_call_function_with_args(func, args);
+    lv_func = nltk_get_function_handle(tmp_result, "leaves");
+    result = nltk_call_function_with_args(lv_func, NULL);
+    Py_DECREF(args);
+    Py_DECREF(func);
+    Py_DECREF(pt_result);
+    Py_DECREF(tmp_result);
+    Py_DECREF(lv_func);
+    return result;
+}
+
+PyObject* nltk_lemmatize(char* sentence)
+{
+    PyObject* args = NULL;
+    PyObject* wn_func = NULL;
+    PyObject* func = NULL;
+    PyObject* wn_result = NULL;
+    PyObject* result = NULL;
+    args = nltk_make_args_from_string(sentence);
+    wn_func = nltk_get_function_handle(globe_lemm,"WordNetLemmatizer");
+    wn_result = nltk_call_function_with_args(wn_func, NULL);
+    func = nltk_get_function_handle(wn_result, "lemmatize");
+    result = nltk_call_function_with_args(func, args);
+    Py_DECREF(args);
+    Py_DECREF(wn_func);
+    Py_DECREF(func);
+    Py_DECREF(wn_result);
+    return result;
+}
+
+PyObject* nltk_language_detect(char* sentence)
+{
+    PyObject* args = NULL;
+    PyObject* func = NULL;
+    PyObject* result = NULL;
+    args = nltk_make_args_from_string(sentence);
+    func = nltk_get_function_handle(globe_lang,"detect");
+    result = nltk_call_function_with_args(func, args);
+    Py_DECREF(args);
+    Py_DECREF(func);
+    return result;
+}
+
 PyObject* nltk_get_module(char* name)
 {
     PRET_VM(!name, NULL, "Input parameter [name] is NULL!");