From 435b8b21df618980b41bbc7b177a4545216563f4 Mon Sep 17 00:00:00 2001 From: Jihoon Lee Date: Fri, 25 Sep 2020 16:41:22 +0900 Subject: [PATCH] [CS] Add inference pipeline **Changes proposed in this PR:** - Add pipeline for the inference **Self evaluation:** 1. Build test: [X]Passed [ ]Failed [ ]Skipped 2. Run test: [X]Passed [ ]Failed [ ]Skipped Signed-off-by: Jihoon Lee --- .../Tizen_native/CustomShortcut/inc/data.h | 17 ++ .../Tizen_native/CustomShortcut/res/model.ini | 7 +- .../Tizen_native/CustomShortcut/src/data.c | 217 +++++++++++++++++---- .../Tizen_native/CustomShortcut/src/main.c | 1 + 4 files changed, 206 insertions(+), 36 deletions(-) diff --git a/Applications/Tizen_native/CustomShortcut/inc/data.h b/Applications/Tizen_native/CustomShortcut/inc/data.h index 7d6cf79..36a4298 100644 --- a/Applications/Tizen_native/CustomShortcut/inc/data.h +++ b/Applications/Tizen_native/CustomShortcut/inc/data.h @@ -115,6 +115,15 @@ int util_parse_route(const char *source, char **route, char **data); int util_get_resource_path(const char *file, char *full_path, bool shared); /** + * @brief save cairo surface to a drawing. + * @todo currently saves to png, later change to other format + * @param cr_surface cairo surface to save + * @param dst destination name, it is save to the data path + * @return int APP_ERROR_NONE if success + */ +int util_save_drawing(cairo_surface_t *cr_surface, const char *dst); + +/** * @brief handle given path_data. If data is invalid, it is essentially noop * * @param ad appdata @@ -171,6 +180,14 @@ void *data_update_train_result(void *ad); */ int data_parse_result_string(const char *src, train_result_s *train_result); +/** + * @brief run inference with nnstreamer + * + * @param ad appdata + * @return int APP_ERROR_NONE if no error + */ +int data_run_inference(appdata_s *ad); + #if !defined(PACKAGE) #define PACKAGE "org.example.nntrainer-example-custom-shortcut" #endif diff --git a/Applications/Tizen_native/CustomShortcut/res/model.ini b/Applications/Tizen_native/CustomShortcut/res/model.ini index d598292..d0e8a74 100644 --- a/Applications/Tizen_native/CustomShortcut/res/model.ini +++ b/Applications/Tizen_native/CustomShortcut/res/model.ini @@ -4,7 +4,7 @@ Type = NeuralNetwork # Model Type : Regression, KNN, NeuralNetwork Learning_rate = 0.0001 # Learning Rate Decay_rate = 0.96 # for the decay_rate for the decayed learning rate Decay_steps = 1000 # decay step for the exponential decayed learning rate -Epochs = 20 # Epoch +Epochs = 10 # Epoch Optimizer = adam # Optimizer : sgd (stochastic gradien decent), # adam (Adamtive Moment Estimation) loss = cross # Cost(loss) function : mse (mean squared error) @@ -24,9 +24,12 @@ LabelData="label.dat" # Layer Section : Name [inputlayer] Type = input -Input_Shape = 1:1:62720 # Input Layer Dimension +Input_Shape = 1280:7:7 # Input Layer Dimension Normalization = true +[flatten] +Type = flatten + [outputlayer] Type = fully_connected Unit = 2 # Output Layer Dimension ( = Weight Width ) diff --git a/Applications/Tizen_native/CustomShortcut/src/data.c b/Applications/Tizen_native/CustomShortcut/src/data.c index c78e576..0827263 100644 --- a/Applications/Tizen_native/CustomShortcut/src/data.c +++ b/Applications/Tizen_native/CustomShortcut/src/data.c @@ -94,8 +94,23 @@ int util_get_data_path(const char *file, char *full_path) { return APP_ERROR_NONE; } -static void on_data_receive_(ml_tensors_data_h data, - const ml_tensors_info_h info, void *user_data) { +int util_save_drawing(cairo_surface_t *cr_surface, const char *dst) { + cairo_status_t cr_stat = CAIRO_STATUS_SUCCESS; + + LOG_D("start writing to png_path: %s ", dst); + /// @todo change to other format + cr_stat = cairo_surface_write_to_png(cr_surface, dst); + + if (cr_stat != CAIRO_STATUS_SUCCESS) { + LOG_E("failed to write cairo surface as a file reason: %d", cr_stat); + return APP_ERROR_INVALID_PARAMETER; + } + + return APP_ERROR_NONE; +} + +static void on_feature_receive_(ml_tensors_data_h data, + const ml_tensors_info_h info, void *user_data) { appdata_s *ad = (appdata_s *)user_data; void *raw_data; @@ -163,21 +178,14 @@ CLEAN: pthread_mutex_unlock(&ad->pipe_lock); } -static int run_nnpipeline_(appdata_s *ad, const char *src) { +static int run_mobilnet_pipeline_(appdata_s *ad, const char *src) { char pipe_description[5000]; - char model_path[PATH_MAX]; int status = ML_ERROR_NONE; util_get_resource_path("mobilenetv2.tflite", model_path, false); - status = pthread_mutex_lock(&ad->pipe_lock); - if (status != 0) { - LOG_E("acquiring lock failed status: %d", status); - return status; - } - LOG_D("pipe ready, starting pipeline"); sprintf(pipe_description, @@ -192,26 +200,29 @@ static int run_nnpipeline_(appdata_s *ad, const char *src) { LOG_D("setting inference \n pipe: %s", pipe_description); status = ml_pipeline_construct(pipe_description, NULL, NULL, &ad->pipeline); - if (status != ML_ERROR_NONE) { LOG_E("something wrong constructing pipeline %d", status); - ml_pipeline_destroy(ad->pipeline); - pthread_mutex_unlock(&ad->pipe_lock); return status; } - status = ml_pipeline_sink_register(ad->pipeline, "sink", on_data_receive_, + status = ml_pipeline_sink_register(ad->pipeline, "sink", on_feature_receive_, (void *)ad, &ad->pipe_sink); if (status != ML_ERROR_NONE) { LOG_E("sink register failed %d", status); - goto CLEAN; + goto PIPE_DESTORY; } LOG_D("starting inference"); status = ml_pipeline_start(ad->pipeline); if (status != ML_ERROR_NONE) { LOG_E("failed to start pipeline %d", status); - goto CLEAN; + goto SINK_UNREGISTER; + } + + status = pthread_mutex_lock(&ad->pipe_lock); + if (status != 0) { + LOG_E("acquiring lock failed status: %d", status); + goto MUTEX_UNLOCK; } pthread_cond_wait(&ad->pipe_cond, &ad->pipe_lock); @@ -220,21 +231,20 @@ static int run_nnpipeline_(appdata_s *ad, const char *src) { status = ml_pipeline_stop(ad->pipeline); if (status != ML_ERROR_NONE) { LOG_E("stopping pipeline failed"); - goto CLEAN; + goto MUTEX_UNLOCK; } - LOG_D("unregister pipeline"); - if (status != ML_ERROR_NONE) { - LOG_E("unregistering sink failed"); - } +MUTEX_UNLOCK: + pthread_mutex_unlock(&ad->pipe_lock); -CLEAN: - LOG_D("destroying pipeline"); +SINK_UNREGISTER: ml_pipeline_sink_unregister(ad->pipe_sink); - ml_pipeline_destroy(ad->pipeline); ad->pipe_sink = NULL; + +PIPE_DESTORY: + ml_pipeline_destroy(ad->pipeline); ad->pipeline = NULL; - pthread_mutex_unlock(&ad->pipe_lock); + return status; } @@ -265,22 +275,24 @@ int data_extract_feature(appdata_s *ad) { char png_path[PATH_MAX]; const char *dst = ad->tries < MAX_TRAIN_TRIES ? TRAIN_SET_PATH : VALIDATION_SET_PATH; - cairo_status_t cr_stat = CAIRO_STATUS_SUCCESS; int status = APP_ERROR_NONE; - util_get_data_path("temp.png", png_path); - LOG_D("start writing to png_path: %s ", png_path); - cr_stat = cairo_surface_write_to_png(ad->cr_surface, png_path); + status = util_get_data_path("test.png", png_path); + if (status != APP_ERROR_NONE) { + LOG_E("getting data path failed"); + return status; + } - if (cr_stat != CAIRO_STATUS_SUCCESS) { - LOG_E("failed to write cairo surface as a file reason: %d", cr_stat); - return APP_ERROR_INVALID_PARAMETER; + status = util_save_drawing(ad->cr_surface, png_path); + if (status != APP_ERROR_NONE) { + LOG_E("failed to save drawing to a file"); + return status; } util_get_data_path(dst, ad->pipe_dst); LOG_I("start inference to dataset: %s ", ad->pipe_dst); - status = run_nnpipeline_(ad, png_path); + status = run_mobilnet_pipeline_(ad, png_path); return status; } @@ -388,7 +400,6 @@ void *data_update_train_result(void *data) { LOG_E("pipe write error"); return NULL; }; - usleep(150); } LOG_D("training finished"); @@ -486,3 +497,141 @@ CLEAN: return status; } + +static void on_inference_end_(ml_tensors_data_h data, + const ml_tensors_info_h info, void *user_data) { + appdata_s *ad = (appdata_s *)user_data; + + float *raw_data; + size_t data_size; + + int status = pthread_mutex_lock(&ad->pipe_lock); + if (status != 0) { + LOG_E("acquiring lock failed %d", status); + pthread_cond_signal(&ad->pipe_cond); + return; + } + + status = + ml_tensors_data_get_tensor_data(data, 0, (void **)&raw_data, &data_size); + if (status != ML_ERROR_NONE) { + LOG_E("get tensor data failed: reason %s", strerror(status)); + goto RESUME; + } + + if (data_size != sizeof(float) * NUM_CLASS) { + LOG_E("output tensor size mismatch, %d", (int)data_size); + goto RESUME; + } + + /// SMILE: 0 1 + /// FROWN: 1 0 + LOG_D("label: %lf %lf", raw_data[0], raw_data[1]); + +RESUME: + status = pthread_cond_signal(&ad->pipe_cond); + if (status != 0) { + LOG_E("cond signal failed %d", status); + } + pthread_mutex_unlock(&ad->pipe_lock); +} + +int run_inference_pipeline_(appdata_s *ad, const char *filesrc) { + char pipe_description[9000]; + char tf_model_path[PATH_MAX]; + char trainer_model_path[PATH_MAX]; + + int status = APP_ERROR_NONE; + + status = util_get_resource_path("mobilenetv2.tflite", tf_model_path, false); + if (status != APP_ERROR_NONE) { + LOG_E("error getting resource path, reason: %d", status); + return status; + } + + status = util_get_resource_path("model.ini", trainer_model_path, false); + if (status != APP_ERROR_NONE) { + LOG_E("error getting data path, reason: %d", status); + return status; + } + + sprintf(pipe_description, + "filesrc location=%s ! pngdec ! videoconvert ! " + "videoscale ! video/x-raw,width=224,height=224,format=RGB ! " + "tensor_converter ! " + "tensor_transform mode=arithmetic option=%s ! " + "tensor_filter framework=tensorflow-lite model=%s ! " + "tensor_filter framework=nntrainer model=%s input=1280:7:7:1 " + "inputtype=float32 output=1:%d:1:1 outputtype=float32 ! " + "tensor_sink name=sink", + filesrc, "typecast:float32,add:-127.5,div:127.5", tf_model_path, + trainer_model_path, NUM_CLASS); + + LOG_D("pipe description: %s", pipe_description); + status = ml_pipeline_construct(pipe_description, NULL, NULL, &ad->pipeline); + if (status != ML_ERROR_NONE) { + LOG_E("constructing pipeline failed, reason: %d", status); + goto PIPE_UNLOCK; + } + + status = ml_pipeline_sink_register(ad->pipeline, "sink", on_inference_end_, + (void *)ad, &ad->pipe_sink); + if (status != ML_ERROR_NONE) { + LOG_E("sink register failed, reason: %d", status); + goto DESTORY_PIPE; + } + + status = ml_pipeline_start(ad->pipeline); + if (status != ML_ERROR_NONE) { + LOG_E("failed to start pipeline %d", status); + goto UNREGISTER_SINK; + } + + status = pthread_mutex_lock(&ad->pipe_lock); + if (status != 0) { + LOG_E("acquiring lock failed status: %d", status); + goto UNREGISTER_SINK; + } + + pthread_cond_wait(&ad->pipe_cond, &ad->pipe_lock); + + status = ml_pipeline_stop(ad->pipeline); + if (status != ML_ERROR_NONE) { + LOG_E("stopping pipeline failed"); + } + +PIPE_UNLOCK: + pthread_mutex_unlock(&ad->pipe_lock); + +UNREGISTER_SINK: + ml_pipeline_sink_unregister(ad->pipe_sink); + ad->pipe_sink = NULL; + +DESTORY_PIPE: + ml_pipeline_destroy(ad->pipeline); + ad->pipeline = NULL; + + return status; +} + +int data_run_inference(appdata_s *ad) { + char png_path[PATH_MAX]; + + int status = APP_ERROR_NONE; + + status = util_get_data_path("test.png", png_path); + if (status != APP_ERROR_NONE) { + LOG_E("getting data path failed"); + return status; + } + + status = util_save_drawing(ad->cr_surface, png_path); + if (status != APP_ERROR_NONE) { + LOG_E("saving the cairo drawing failed"); + return status; + } + + status = run_inference_pipeline_(ad, png_path); + + return status; +} diff --git a/Applications/Tizen_native/CustomShortcut/src/main.c b/Applications/Tizen_native/CustomShortcut/src/main.c index 92c2cbe..250f7a4 100644 --- a/Applications/Tizen_native/CustomShortcut/src/main.c +++ b/Applications/Tizen_native/CustomShortcut/src/main.c @@ -175,6 +175,7 @@ void presenter_on_canvas_submit_inference(void *data, Evas_Object *obj, const char *source) { appdata_s *ad = (appdata_s *)data; /** appdata handling NYI */ + data_run_inference(ad); ad->tries = 0; elm_naviframe_item_pop(ad->naviframe); -- 2.7.4