From: zhangzg Date: Sat, 1 Sep 2018 12:10:39 +0000 (+0800) Subject: Improve nlp service code X-Git-Tag: submit/tizen/20180907.090507~2 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=refs%2Fchanges%2F56%2F188156%2F3;p=platform%2Fcore%2Fuifw%2Fnlp.git Improve nlp service code 1.make function for each nltk api 2.add Py_DECREF(X) for reduce reference counting Change-Id: I11bfc3d2f7e3b1f75d970531f0b9063a439622c7 --- diff --git a/service/inc/service.h b/service/inc/service.h index a6f723c..8c92915 100755 --- a/service/inc/service.h +++ b/service/inc/service.h @@ -16,9 +16,13 @@ typedef enum { NLTK_CMD_UNKNOWN }NLTK_CMDS; -// This method is exported from nltk_native_library.so void nltk_initialize(); void nltk_finalize(); +PyObject* nltk_word_tokenize(char* sentence); +PyObject* nltk_pos_tag(char* sentence); +PyObject* nltk_ne_chunk(char* sentence); +PyObject* nltk_lemmatize(char* sentence); +PyObject* nltk_language_detect(char* sentence); PyObject* nltk_get_module(char* name); PyObject* nltk_get_function_handle(PyObject* module , char * func_name); PyObject* nltk_make_args_from_pyobject(PyObject* pyobj); diff --git a/service/src/service.c b/service/src/service.c index 001f561..39303a4 100755 --- a/service/src/service.c +++ b/service/src/service.c @@ -116,18 +116,6 @@ static int __message_send(rpc_port_stub_message_context_h context, PyObject* args = NULL; PyObject* lists = NULL; unsigned static int len = 0; - PyObject* postag_module = NULL; - PyObject* postag_result = NULL; - PyObject* ne_module = NULL; - PyObject* ne_result = NULL; - PyObject* lv_module = NULL; - PyObject* lv_result = NULL; - PyObject* lm_module = NULL; - PyObject* lm_result = NULL; - PyObject* lm_object = NULL; - PyObject* ld_module = NULL; - PyObject* ld_result = NULL; - PyObject* pos_elm_tuple = NULL; bundle_get_str(msg, "request_id", &request_id); bundle_get_str(msg, "info", &info); NLTK_CMDS cmd = NLTK_CMD_NONE; @@ -161,9 +149,10 @@ static int __message_send(rpc_port_stub_message_context_h context, switch (cmd) { case NLTK_CMD_TOKENIZE: - args = nltk_make_args_from_string(info); - lists = nltk_call_function_with_args(nltk_get_function_handle(globe_nltk,"word_tokenize"), args); - len = nltk_get_size_from_list(lists); + { + PyObject* wt_lists = NULL; + wt_lists = nltk_word_tokenize(info); + len = nltk_get_size_from_list(wt_lists); char *tokens[BUF_LEN_128] = {NULL,}; for(int i = 0 ;i < len ;i++) { @@ -171,7 +160,7 @@ static int __message_send(rpc_port_stub_message_context_h context, if(tokens[i]!=NULL) { memset(tokens[i], 0, BUF_LEN_128); - tmp_str = nltk_get_string_from_element(nltk_get_element_from_list_by_index(lists, i)); + tmp_str = nltk_get_string_from_element(nltk_get_element_from_list_by_index(wt_lists, i)); if(tmp_str!=NULL) { strncpy(tokens[i], tmp_str, BUF_LEN_128-1); @@ -185,30 +174,31 @@ static int __message_send(rpc_port_stub_message_context_h context, } bundle_add_str(reply, "command", "word_tokenize"); bundle_add_str_array(reply, "return_token", tokens, len); - PINFO("word_tokenize process done"); for(int j = 0 ;j < len ;j++) { free(tokens[j]); } + Py_DECREF(wt_lists); + PINFO("word_tokenize process done"); break; + } case NLTK_CMD_POSTAG: - args = nltk_make_args_from_string(info); - lists = nltk_call_function_with_args(nltk_get_function_handle(globe_nltk,"word_tokenize"), args); - postag_module = nltk_get_function_handle(globe_nltk,"pos_tag"); - args = nltk_make_args_from_pyobject(lists); - postag_result = nltk_call_function_with_args(postag_module, args); - len = nltk_get_size_from_list(postag_result); + { + PyObject* pt_result = NULL; + PyObject* pt_elm_tuple = NULL; + pt_result = nltk_pos_tag(info); + len = nltk_get_size_from_list(pt_result); char *tag[BUF_LEN_128] = {NULL,}; char *token[BUF_LEN_128] = {NULL,}; for(int i = 0 ;i < len ;i++) { token[i] = (char*)malloc(BUF_LEN_128*sizeof(char)); tag[i] = (char*)malloc(BUF_LEN_128*sizeof(char)); - pos_elm_tuple = nltk_get_element_from_list_by_index(postag_result, i); + pt_elm_tuple = nltk_get_element_from_list_by_index(pt_result, i); if(tag[i]!=NULL) { memset(tag[i], 0, BUF_LEN_128); - tmp_str = nltk_get_string_from_element(nltk_get_element_from_tuple_by_index(pos_elm_tuple, 0)); + tmp_str = nltk_get_string_from_element(nltk_get_element_from_tuple_by_index(pt_elm_tuple, 0)); if(tmp_str!=NULL) { strncpy(tag[i], tmp_str, BUF_LEN_128-1); @@ -222,7 +212,7 @@ static int __message_send(rpc_port_stub_message_context_h context, if(token[i]!=NULL) { memset(token[i], 0, BUF_LEN_128); - tmp_str = nltk_get_string_from_element(nltk_get_element_from_tuple_by_index(pos_elm_tuple, 1)); + tmp_str = nltk_get_string_from_element(nltk_get_element_from_tuple_by_index(pt_elm_tuple, 1)); if(tmp_str!=NULL) { strncpy(token[i], tmp_str, BUF_LEN_128-1); @@ -242,31 +232,28 @@ static int __message_send(rpc_port_stub_message_context_h context, free(tag[j]); free(token[j]); } + Py_DECREF(pt_result); + Py_DECREF(pt_elm_tuple); PINFO("pos_tag process done"); break; + } case NLTK_CMD_NECHUNK: - args = nltk_make_args_from_string(info); - lists = nltk_call_function_with_args(nltk_get_function_handle(globe_nltk,"word_tokenize"), args); - postag_module = nltk_get_function_handle(globe_nltk,"pos_tag"); - args = nltk_make_args_from_pyobject(lists); - postag_result = nltk_call_function_with_args(postag_module, args); - args = nltk_make_args_from_pyobject(postag_result); - ne_module = nltk_get_function_handle(globe_nltk,"ne_chunk"); - ne_result = nltk_call_function_with_args(ne_module, args); - lv_module = nltk_get_function_handle(ne_result, "leaves"); - lv_result = nltk_call_function_with_args(lv_module, NULL); - len = nltk_get_size_from_list(lv_result); + { + PyObject* ne_result = NULL; + PyObject* ne_elm_tuple = NULL; + ne_result = nltk_ne_chunk(info); + len = nltk_get_size_from_list(ne_result); char *s_tag[BUF_LEN_128] = {NULL,}; char *s_token[BUF_LEN_128] = {NULL,}; for(int i = 0 ;i < len ;i++) { s_token[i] = (char*)malloc(BUF_LEN_128*sizeof(char)); s_tag[i] = (char*)malloc(BUF_LEN_128*sizeof(char)); - pos_elm_tuple = nltk_get_element_from_list_by_index(lv_result, i); + ne_elm_tuple = nltk_get_element_from_list_by_index(ne_result, i); if(s_tag[i]!=NULL) { memset(s_tag[i], 0, BUF_LEN_128); - tmp_str = nltk_get_string_from_element(nltk_get_element_from_tuple_by_index(pos_elm_tuple, 0)); + tmp_str = nltk_get_string_from_element(nltk_get_element_from_tuple_by_index(ne_elm_tuple, 0)); if(tmp_str!=NULL) { strncpy(s_tag[i], tmp_str, BUF_LEN_128-1); @@ -280,7 +267,7 @@ static int __message_send(rpc_port_stub_message_context_h context, if(s_token[i]!=NULL) { memset(s_token[i], 0, BUF_LEN_128); - tmp_str = nltk_get_string_from_element(nltk_get_element_from_tuple_by_index(pos_elm_tuple, 1)); + tmp_str = nltk_get_string_from_element(nltk_get_element_from_tuple_by_index(ne_elm_tuple, 1)); if(tmp_str!=NULL) { strncpy(s_token[i], tmp_str, BUF_LEN_128-1); @@ -295,19 +282,20 @@ static int __message_send(rpc_port_stub_message_context_h context, bundle_add_str(reply, "command", "ne_chunk"); bundle_add_str_array(reply, "return_tag", s_tag, len); bundle_add_str_array(reply, "return_token", s_token, len); - PINFO("ne_chunk process done"); for(int j = 0 ;j < len ;j++) { free(s_tag[j]); free(s_token[j]); } + Py_DECREF(ne_result); + Py_DECREF(ne_elm_tuple); + PINFO("ne_chunk process done"); break; + } case NLTK_CMD_LEMMATIZE: - args = nltk_make_args_from_string(info); - lm_module = nltk_get_function_handle(globe_lemm,"WordNetLemmatizer"); - lm_object = nltk_call_function_with_args(lm_module, NULL); - lm_module = nltk_get_function_handle(lm_object, "lemmatize"); - lm_result = nltk_call_function_with_args(lm_module, args); + { + PyObject* lm_result = NULL; + lm_result = nltk_lemmatize(info); char *lem_buf[1] = {NULL,}; lem_buf[0] = (char*)malloc(BUF_LEN_128*sizeof(char)); if(lem_buf[0]!=NULL) @@ -321,18 +309,20 @@ static int __message_send(rpc_port_stub_message_context_h context, } bundle_add_str(reply, "command", "lemmatize"); bundle_add_str_array(reply, "return_token", lem_buf, 1); - PINFO("lemmatize process done"); free(lem_buf[0]); } else { PERR("malloc failed"); } + Py_DECREF(lm_result); + PINFO("lemmatize process done"); break; + } case NLTK_CMD_LANGDETECT: - args = nltk_make_args_from_string(info); - ld_module = nltk_get_function_handle(globe_lang,"detect"); - ld_result = nltk_call_function_with_args(ld_module, args); + { + PyObject* ld_result = NULL; + ld_result = nltk_language_detect(info); char *lang_buf[1] = {NULL,}; lang_buf[0] = (char*)malloc(BUF_LEN_128*sizeof(char)); if(lang_buf[0]!=NULL) @@ -353,7 +343,9 @@ static int __message_send(rpc_port_stub_message_context_h context, { PERR("malloc failed"); } + Py_DECREF(ld_result); break; + } default: PWARNING("unknown command, command = [%s]", message); bundle_add_str(reply, "command", "Exception happens"); @@ -474,6 +466,89 @@ void nltk_finalize() Py_Finalize(); } +PyObject* nltk_word_tokenize(char* sentence) +{ + PyObject* args = NULL; + PyObject* func = NULL; + PyObject* lists = NULL; + args = nltk_make_args_from_string(sentence); + func = nltk_get_function_handle(globe_nltk, "word_tokenize"); + lists = nltk_call_function_with_args(func, args); + Py_DECREF(args); + Py_DECREF(func); + return lists; +} + +PyObject* nltk_pos_tag(char* sentence) +{ + PyObject* args = NULL; + PyObject* func = NULL; + PyObject* wt_result = NULL; + PyObject* result = NULL; + wt_result = nltk_word_tokenize(sentence); + func = nltk_get_function_handle(globe_nltk, "pos_tag"); + args = nltk_make_args_from_pyobject(wt_result); + result = nltk_call_function_with_args(func, args); + Py_DECREF(args); + Py_DECREF(func); + Py_DECREF(wt_result); + return result; +} + +PyObject* nltk_ne_chunk(char* sentence) +{ + PyObject* args = NULL; + PyObject* pt_result = NULL; + PyObject* tmp_result = NULL; + PyObject* result = NULL; + PyObject* func = NULL; + PyObject* lv_func = NULL; + pt_result = nltk_pos_tag(sentence); + args = nltk_make_args_from_pyobject(pt_result); + func = nltk_get_function_handle(globe_nltk, "ne_chunk"); + tmp_result = nltk_call_function_with_args(func, args); + lv_func = nltk_get_function_handle(tmp_result, "leaves"); + result = nltk_call_function_with_args(lv_func, NULL); + Py_DECREF(args); + Py_DECREF(func); + Py_DECREF(pt_result); + Py_DECREF(tmp_result); + Py_DECREF(lv_func); + return result; +} + +PyObject* nltk_lemmatize(char* sentence) +{ + PyObject* args = NULL; + PyObject* wn_func = NULL; + PyObject* func = NULL; + PyObject* wn_result = NULL; + PyObject* result = NULL; + args = nltk_make_args_from_string(sentence); + wn_func = nltk_get_function_handle(globe_lemm,"WordNetLemmatizer"); + wn_result = nltk_call_function_with_args(wn_func, NULL); + func = nltk_get_function_handle(wn_result, "lemmatize"); + result = nltk_call_function_with_args(func, args); + Py_DECREF(args); + Py_DECREF(wn_func); + Py_DECREF(func); + Py_DECREF(wn_result); + return result; +} + +PyObject* nltk_language_detect(char* sentence) +{ + PyObject* args = NULL; + PyObject* func = NULL; + PyObject* result = NULL; + args = nltk_make_args_from_string(sentence); + func = nltk_get_function_handle(globe_lang,"detect"); + result = nltk_call_function_with_args(func, args); + Py_DECREF(args); + Py_DECREF(func); + return result; +} + PyObject* nltk_get_module(char* name) { PRET_VM(!name, NULL, "Input parameter [name] is NULL!");