For example, user can draw `^^` few times to train a model to label the drawing as 😊.
-Built and tested at `tizen studio 3.7` with `wearable-5.5 profile`
+Built and tested at `tizen studio 3.7` with `wearable-6.0 profile`
+
+tested on `SM-R800` (it should run in other wearable but it is not guaranteed)
## How to Build and Run
+// SPDX-License-Identifier: Apache-2.0-only
/**
+ * Copyright (C) 2020 Jihoon Lee <jhoon.it.lee@samsung.com>
+ *
* @file data.h
* @date 15 May 2020
* @brief TIZEN Native Example App dataentry with NNTrainer/CAPI.
#include <cairo/cairo-evas-gl.h>
#include <dlog.h>
#include <efl_extension.h>
+#include <nnstreamer.h>
#include <nntrainer.h>
+#include <pthread.h>
#include <tizen.h>
#ifdef LOG_TAG
Elm_Object_Item *home;
Evas_Object *layout;
+ char edj_path[PATH_MAX];
+
/**< drawing related */
Evas_Object *canvas; /**< image object that cairo surface flushes to */
unsigned char *pixels; /**< actual pixel data */
DRAW_MODE mode; /**< drawing mode of current canvas */
int tries; /**< tells how many data has been labeled */
- char edj_path[PATH_MAX];
+ /**< ML related */
+ ml_pipeline_h pipeline; /**< handle of feature extractor */
+ ml_pipeline_sink_h pipe_sink; /**< sink of pipeline */
+ char pipe_dst[PATH_MAX]; /**< destination path where to save */
+ pthread_mutex_t pipe_lock; /**< ensures that only one pipe runs at a time */
+ pthread_cond_t pipe_cond; /**< pipe condition to block at a point */
} appdata_s;
/**
* @param[out] length of data len
* @retval 0 if no error
*/
-int parse_route(const char *source, char **route, char **data);
+int data_parse_route(const char *source, char **route, char **data);
+
+/**
+ * @brief get full resource path for given file.
+ * @param[in] file relative file path from resource path
+ * @param[out] full_path path of the output file
+ * @param[in] shared true if resource is in shared/res
+ * @retval APP_ERROR_NONE if no error
+ */
+int data_get_resource_path(const char *file, char *full_path, bool shared);
+
+/**
+ * @brief extract data feature from given model.
+ * @param[in] ad appdata
+ * @param[in] dst state the name of the data set
+ * @param[in] append decide whether to append to the exisiting file
+ *
+ * This function runs a mobilnetv2 last layer detached and saves an output
+ * vector. input for this model is png file drawn to the canvas(stored in
+ * appdata) output can be pased to nntrainer and used.
+ */
+int data_extract_feature(appdata_s *ad, const char *dst, bool append);
/**
* @brief nntrainer sanity test. contructing model and destroy. This will be
#endif
#if !defined(_D)
-#define _D(fmt, arg...) \
+#define LOG_D(fmt, arg...) \
dlog_print(DLOG_DEBUG, LOG_TAG, "[%s:%d] " fmt "\n", __func__, __LINE__, \
##arg)
#endif
#if !defined(_I)
-#define _I(fmt, arg...) \
+#define LOG_I(fmt, arg...) \
dlog_print(DLOG_INFO, LOG_TAG, "[%s:%d] " fmt "\n", __func__, __LINE__, ##arg)
#endif
#if !defined(_W)
-#define _W(fmt, arg...) \
+#define LOG_W(fmt, arg...) \
dlog_print(DLOG_WARN, LOG_TAG, "[%s:%d] " fmt "\n", __func__, __LINE__, ##arg)
#endif
#if !defined(_E)
-#define _E(fmt, arg...) \
+#define LOG_E(fmt, arg...) \
dlog_print(DLOG_DEBUG, LOG_TAG, "[%s:%d] " fmt "\n", __func__, __LINE__, \
##arg)
#endif
rel2.relative: 1.0 0.8;
}
}
- PART_BUTTON("draw/reset", "↩️", 0.2, 0.8, 0.45, 0.9)
- PART_BUTTON("draw/proceed", "▶️", 0.55, 0.8, 0.8, 0.9)
+ PART_BUTTON("draw/reset", "↩️", 0.2, 0.83, 0.45, 0.9)
+ PART_BUTTON("draw/proceed", "▶️", 0.55, 0.83, 0.81, 0.9)
}
programs {
PROGRAM_BUTTON("draw/reset", "draw/reset", "")
// SPDX-License-Identifier: Apache-2.0-only
/**
- * Copyright (C) 2020 Jijoong Moon <jijoong.moon@samsung.com>
+ * Copyright (C) 2020 Jihoon Lee <jhoon.it.lee@samsung.com>
*
* @file data.c
* @date 21 Jul 2020
*
*/
#include "data.h"
-
+#include <stdio.h>
#include <string.h>
-int parse_route(const char *source, char **route, char **data) {
+int data_parse_route(const char *source, char **route, char **data) {
char *dst = strdup(source);
const char sep = ':';
char *i;
return APP_ERROR_NONE;
}
+int data_get_resource_path(const char *file, char *full_path, bool shared) {
+ char *root_path;
+ if (shared) {
+ root_path = app_get_shared_resource_path();
+ } else {
+ root_path = app_get_resource_path();
+ }
+
+ if (root_path == NULL) {
+ LOG_E("failed to get resource path");
+ return APP_ERROR_INVALID_PARAMETER;
+ }
+
+ if (full_path == NULL) {
+ LOG_E("full_path is null");
+ free(root_path);
+ return APP_ERROR_INVALID_PARAMETER;
+ }
+
+ snprintf(full_path, PATH_MAX, "%s%s", root_path, file);
+ LOG_D("resource path: %s", full_path);
+ free(root_path);
+
+ return APP_ERROR_NONE;
+}
+
+int data_get_data_path(const char *file, char *full_path) {
+ char *root_path;
+
+ root_path = app_get_data_path();
+
+ if (root_path == NULL) {
+ LOG_E("failed to get data path");
+ return APP_ERROR_INVALID_PARAMETER;
+ }
+
+ if (full_path == NULL) {
+ LOG_E("full_path is null");
+ free(root_path);
+ return APP_ERROR_INVALID_PARAMETER;
+ }
+
+ snprintf(full_path, PATH_MAX, "%s%s", root_path, file);
+ LOG_D("data path: %s", full_path);
+ free(root_path);
+
+ return APP_ERROR_NONE;
+}
+
+static void _on_data_receive(ml_tensors_data_h data,
+ const ml_tensors_info_h info, void *user_data) {
+ appdata_s *ad = (appdata_s *)user_data;
+
+ void *raw_data;
+ size_t data_size, write_result;
+ FILE *file;
+
+ pthread_mutex_lock(&ad->pipe_lock);
+
+ /// @note data_size written here will be overriden and it is intended.
+ int status = ml_tensors_data_get_tensor_data(data, 0, &raw_data, &data_size);
+ if (status != ML_ERROR_NONE) {
+ LOG_E("get tensor data failed %d", status);
+ status = ml_tensors_info_get_tensor_size(info, -1, &data_size);
+ goto CLEAN;
+ }
+
+ file = fopen(ad->pipe_dst, ad->tries == 1 ? "wb" : "wb+");
+
+ if (file == NULL) {
+ LOG_E("cannot open file");
+ goto CLEAN;
+ }
+
+ if (write_result = fwrite(raw_data, data_size, 1, file) < 0) {
+ LOG_E("write error happend");
+ }
+
+ if (write_result < data_size) {
+ LOG_E("data was not fully written to file");
+ }
+
+ if (fclose(file) < 0) {
+ LOG_E("there was error closing");
+ goto CLEAN;
+ }
+
+ LOG_D("using pipeline finished, destroying pipeline");
+
+CLEAN:
+ pthread_cond_signal(&ad->pipe_cond);
+ pthread_mutex_unlock(&ad->pipe_lock);
+}
+
+static int _run_nnpipeline(appdata_s *ad, const char *src, bool append) {
+ char pipe_description[5000];
+
+ char model_path[PATH_MAX];
+
+ int status = ML_ERROR_NONE;
+
+ data_get_resource_path("mobilenetv2.tflite", model_path, false);
+
+ status = pthread_mutex_lock(&ad->pipe_lock);
+ if (status != 0) {
+ LOG_E("acquiring lock failed status: %d", status);
+ return status;
+ }
+
+ LOG_D("pipe ready, starting pipeline");
+
+ sprintf(pipe_description,
+ "filesrc location=%s ! pngdec ! "
+ "videoconvert ! videoscale ! "
+ "video/x-raw,width=224,height=224,format=RGB ! "
+ "tensor_converter ! "
+ "tensor_transform mode=arithmetic option=%s ! "
+ "tensor_filter framework=tensorflow-lite model=%s ! "
+ "tensor_sink name=sink",
+ src, "typecast:float32,add:-127.5,div:127.5", model_path);
+
+ LOG_D("setting inference \n pipe: %s", pipe_description);
+ status = ml_pipeline_construct(pipe_description, NULL, NULL, &ad->pipeline);
+
+ if (status != ML_ERROR_NONE) {
+ LOG_E("something wrong constructing pipeline %d", status);
+ ml_pipeline_destroy(ad->pipeline);
+ pthread_mutex_unlock(&ad->pipe_lock);
+ return status;
+ }
+
+ status = ml_pipeline_sink_register(ad->pipeline, "sink", _on_data_receive,
+ (void *)ad, &ad->pipe_sink);
+ if (status != ML_ERROR_NONE) {
+ LOG_E("sink register failed %d", status);
+ goto CLEAN;
+ }
+
+ LOG_D("starting inference");
+ status = ml_pipeline_start(ad->pipeline);
+ if (status != ML_ERROR_NONE) {
+ LOG_E("failed to start pipeline %d", status);
+ goto CLEAN;
+ }
+
+ pthread_cond_wait(&ad->pipe_cond, &ad->pipe_lock);
+
+ LOG_D("stopping pipeline");
+ status = ml_pipeline_stop(ad->pipeline);
+ if (status != ML_ERROR_NONE) {
+ LOG_E("stopping pipeline failed");
+ goto CLEAN;
+ }
+
+ LOG_D("unregister pipeline");
+ if (status != ML_ERROR_NONE) {
+ LOG_E("unregistering sink failed");
+ }
+
+CLEAN:
+ LOG_D("destroying pipeline");
+ ml_pipeline_sink_unregister(ad->pipe_sink);
+ ml_pipeline_destroy(ad->pipeline);
+ ad->pipe_sink = NULL;
+ ad->pipeline = NULL;
+ pthread_mutex_unlock(&ad->pipe_lock);
+ return status;
+}
+
+int data_extract_feature(appdata_s *ad, const char *dst, bool append) {
+ char png_path[PATH_MAX];
+ cairo_status_t cr_stat = CAIRO_STATUS_SUCCESS;
+ int status = APP_ERROR_NONE;
+
+ data_get_data_path("temp.png", png_path);
+ LOG_D("start writing to png_path: %s ", png_path);
+ cr_stat = cairo_surface_write_to_png(ad->cr_surface, png_path);
+
+ if (cr_stat != CAIRO_STATUS_SUCCESS) {
+ LOG_E("failed to write cairo surface as a file reason: %d", cr_stat);
+ return APP_ERROR_INVALID_PARAMETER;
+ }
+
+ data_get_data_path(dst, ad->pipe_dst);
+
+ LOG_I("start inference to dataset: %s ", ad->pipe_dst);
+ status = _run_nnpipeline(ad, png_path, append);
+
+ return status;
+}
+
void nntrainer_test() {
ml_train_model_h model = NULL;
+// SPDX-License-Identifier: Apache-2.0-only
/**
* @file main.c
* @date 14 May 2020
#include "main.h"
#include "data.h"
#include "view.h"
+#include <pthread.h>
#include <tizen.h>
static bool app_create(void *data) {
If this function returns false, the application is terminated */
appdata_s *ad = data;
- char *res_path = app_get_resource_path();
- if (res_path == NULL) {
- dlog_print(DLOG_ERROR, LOG_TAG, "failed to get resource.");
- return false;
- }
+ pthread_mutex_init(&ad->pipe_lock, NULL);
+ pthread_cond_init(&ad->pipe_cond, NULL);
- snprintf(ad->edj_path, sizeof(ad->edj_path), "%s%s", res_path, EDJ_PATH);
- free(res_path);
+ data_get_resource_path(EDJ_PATH, ad->edj_path, false);
view_init(ad);
}
static void app_terminate(void *data) { /* Release all resources. */
+ appdata_s *ad = data;
+
+ pthread_mutex_destroy(&ad->pipe_lock);
+ pthread_cond_destroy(&ad->pipe_cond);
}
static void ui_app_lang_changed(app_event_info_h event_info, void *user_data) {
ret = app_event_get_language(event_info, &language);
if (ret != APP_ERROR_NONE) {
- dlog_print(DLOG_ERROR, LOG_TAG,
- "app_event_get_language() failed. Err = %d.", ret);
+ LOG_E("app_event_get_language() failed. Err = %d.", ret);
return;
}
appdata_s *ad = data;
Elm_Widget_Item *nf_it = elm_naviframe_top_item_get(obj);
+ ad->tries = 0;
+
if (!nf_it) {
/* app should not reach hear */
- dlog_print(DLOG_ERROR, LOG_TAG, "naviframe is empty.");
+ LOG_E("naviframe is null");
+ dlog_print(DLOG_ERROR, LOG_TAG, "naviframe is e.");
ui_app_exit();
return;
}
if (nf_it == ad->home) {
- dlog_print(DLOG_DEBUG, LOG_TAG, "navi empty");
+ LOG_D("naviframe is empty");
elm_win_lower(ad->win);
return;
}
win = elm_win_util_standard_add(PACKAGE, PACKAGE);
if (win == NULL) {
- dlog_print(DLOG_ERROR, LOG_TAG, "failed to get create window err = %d",
- status);
+ LOG_E("failed to create window err = %d", status);
return status;
}
elm_win_conformant_set(win, EINA_TRUE);
// Adding conformant
conform = elm_conformant_add(win);
if (conform == NULL) {
- dlog_print(DLOG_ERROR, LOG_TAG, "failed to create conformant object");
+ LOG_E("failed to create conformant object");
evas_object_del(win);
return APP_ERROR_INVALID_CONTEXT;
}
// Adding naviframe
nf = elm_naviframe_add(conform);
if (nf == NULL) {
- dlog_print(DLOG_ERROR, LOG_TAG, "failed to create naviframe object");
+ LOG_E("failed to create naviframe object");
evas_object_del(win);
return APP_ERROR_INVALID_CONTEXT;
}
char *path, *path_data;
int status;
- status = parse_route(group_name, &path, &path_data);
+ status = data_parse_route(group_name, &path, &path_data);
if (status) {
- _E("something wrong with parsing %s", group_name);
+ LOG_E("something wrong with parsing %s", group_name);
return status;
}
- _D("%s %s", path, path_data);
+ LOG_D("%s %s", path, path_data);
ad->layout = _create_layout(ad->naviframe, ad->edj_path, path, NULL, NULL);
if (ad->layout == NULL) {
- _E("failed to create layout");
+ LOG_E("failed to create layout");
status = APP_ERROR_INVALID_CONTEXT;
evas_object_del(ad->win);
goto CLEAN_UP;
ad->layout, "empty");
if (ad->nf_it == NULL) {
+ LOG_E("naviframe_item_push failed");
status = APP_ERROR_INVALID_PARAMETER;
goto CLEAN_UP;
}
Evas_Object *layout = NULL;
if (parent == NULL) {
- dlog_print(DLOG_ERROR, LOG_TAG, "parent cannot be NULL");
+ LOG_E("parent cannot be NULL");
return NULL;
}
elm_layout_file_set(layout, edj_path, group_name);
if (layout == NULL) {
- dlog_print(DLOG_ERROR, LOG_TAG, "There was error making layout");
+ LOG_E("There was error making layout");
evas_object_del(layout);
return NULL;
}
void *event_info) {
appdata_s *ad = (appdata_s *)data;
Evas_Event_Mouse_Down *eemd = (Evas_Event_Mouse_Down *)event_info;
- _D("x: %d, y: %d", eemd->canvas.x, eemd->canvas.y);
+ LOG_D("x: %d, y: %d", eemd->canvas.x, eemd->canvas.y);
cairo_set_source_rgba(ad->cr, 1, 1, 1, 1);
cairo_move_to(ad->cr, eemd->canvas.x - ad->x_offset,
appdata_s *ad = (appdata_s *)data;
Evas_Event_Mouse_Move *eemm = (Evas_Event_Mouse_Move *)event_info;
- _D("x: %d, y: %d", eemm->cur.canvas.x, eemm->cur.canvas.y);
+ LOG_D("x: %d, y: %d", eemm->cur.canvas.x, eemm->cur.canvas.y);
cairo_line_to(ad->cr, eemm->cur.canvas.x - ad->x_offset,
eemm->cur.canvas.y - ad->y_offset);
}
static void _on_draw_end(void *data, Evas *e, Evas_Object *obj,
void *event_info) {
appdata_s *ad = (appdata_s *)data;
- _D("draw end");
+ LOG_D("draw end");
cairo_stroke(ad->cr);
cairo_surface_flush(ad->cr_surface);
evas_object_image_data_update_add(ad->canvas, 0, 0, ad->width, ad->height);
}
-static Eina_Bool _on_canvas_exit(void *data, Elm_Object_Item *it) {
- _D("exiting canvas");
+static void _on_canvas_exit(void *data, Evas *e, Evas_Object *obj,
+ void *event_info) {
+ LOG_D("deleting canvas");
appdata_s *ad = (appdata_s *)data;
- /// @todo save file to loadable data format
evas_object_del(ad->canvas);
- cairo_surface_destroy(ad->cr_surface);
cairo_destroy(ad->cr);
- return EINA_TRUE;
+ if (cairo_status(ad->cr) != CAIRO_STATUS_SUCCESS) {
+ LOG_E("delete cairo failed");
+ }
+ cairo_surface_destroy(ad->cr_surface);
+ if (cairo_surface_status(ad->cr_surface) != CAIRO_STATUS_SUCCESS) {
+ LOG_E("delete cr_surface failed");
+ }
+}
+
+static void _canvas_erase_all(appdata_s *ad) {
+ cairo_set_source_rgba(ad->cr, 0.5, 0.5, 0.5, 0.5);
+ cairo_set_operator(ad->cr, CAIRO_OPERATOR_SOURCE);
+ cairo_paint(ad->cr);
+ cairo_surface_flush(ad->cr_surface);
+ evas_object_image_data_update_add(ad->canvas, 0, 0, ad->width, ad->height);
+}
+
+static void _on_draw_reset(void *data, Evas_Object *obj, const char *emission,
+ const char *source) {
+ appdata_s *ad = (appdata_s *)data;
+ LOG_D("draw reset");
+ _canvas_erase_all(ad);
+}
+
+static void _on_draw_proceed(void *data, Evas_Object *obj, const char *emission,
+ const char *source) {
+ appdata_s *ad = (appdata_s *)data;
+ int status = APP_ERROR_NONE;
+ LOG_D("draw proceed");
+
+ char buf[256];
+ ad->tries++;
+ if (ad->tries >= MAX_TRIES) {
+ ad->tries = 0;
+ elm_naviframe_item_pop(ad->naviframe);
+ view_routes_to(ad, "train_result");
+ return;
+ }
+
+ sprintf(buf, "draw your symbol [%d/%d]", ad->tries, MAX_TRIES);
+ elm_object_part_text_set(obj, "draw/title", buf);
+ LOG_D("starting extraction");
+ status = data_extract_feature(ad, "trainingSet.dat", true);
+
+ if (status != APP_ERROR_NONE) {
+ LOG_E("feature extraction failed");
+ }
+
+ _canvas_erase_all(ad);
}
static int _create_canvas(appdata_s *ad, const char *draw_mode) {
- _D("init canvas");
+ LOG_D("init canvas");
Eina_Bool status;
Evas_Object *frame = elm_layout_add(ad->layout);
status = elm_layout_content_set(ad->layout, "draw/canvas", frame);
if (status == EINA_FALSE) {
- _E("failed to get canvas object");
+ LOG_E("failed to get canvas object");
return APP_ERROR_INVALID_PARAMETER;
}
- evas_object_move(frame, 72, 72);
- evas_object_resize(frame, 216, 216);
+ evas_object_move(frame, 70, 70);
+ evas_object_resize(frame, 224, 224);
evas_object_show(frame);
Evas_Coord width, height, x, y;
evas_object_geometry_get(frame, &x, &y, &width, &height);
- _D("frame info, %d %d width: %d height: %d", x, y, width, height);
+ LOG_D("frame info, %d %d width: %d height: %d", x, y, width, height);
Evas_Object *canvas =
evas_object_image_filled_add(evas_object_evas_get(frame));
if (canvas == NULL) {
- _E("failed to initiate canvas");
+ LOG_E("failed to initiate canvas");
return APP_ERROR_INVALID_PARAMETER;
}
ad->pixels = (unsigned char *)evas_object_image_data_get(canvas, 1);
if (ad->pixels == NULL) {
- _E("cannot fetch pixels from image");
+ LOG_E("cannot fetch pixels from image");
return APP_ERROR_INVALID_PARAMETER;
}
cairo_surface_t *cairo_surface = cairo_image_surface_create_for_data(
ad->pixels, CAIRO_FORMAT_ARGB32, width, height, width * 4);
if (cairo_surface_status(cairo_surface) != CAIRO_STATUS_SUCCESS) {
- _E("cannot make cairo surface");
+ LOG_E("cannot make cairo surface");
evas_object_del(canvas);
cairo_surface_destroy(cairo_surface);
return APP_ERROR_INVALID_PARAMETER;
cairo_t *cr = cairo_create(cairo_surface);
if (cairo_status(cr) != CAIRO_STATUS_SUCCESS) {
- _E("Cannot initiate cairo surface");
+ LOG_E("Cannot initiate cairo surface");
evas_object_del(canvas);
cairo_surface_destroy(cairo_surface);
cairo_destroy(cr);
evas_object_event_callback_add(canvas, EVAS_CALLBACK_MOUSE_MOVE,
_on_draw_move, (void *)ad);
- elm_naviframe_item_pop_cb_set(ad->nf_it, _on_canvas_exit, ad);
+ evas_object_event_callback_add(ad->layout, EVAS_CALLBACK_DEL, _on_canvas_exit,
+ (void *)ad);
+
+ elm_layout_signal_callback_add(ad->layout, "draw/reset", "", _on_draw_reset,
+ ad);
+
+ elm_layout_signal_callback_add(ad->layout, "draw/proceed", "",
+ _on_draw_proceed, ad);
+ ad->tries = 0;
ad->canvas = canvas;
ad->cr_surface = cairo_surface;
ad->cr = cr;
projects / platform / core / ml / nntrainer.git / commitdiff