modality_vision : vision.c - add logic about classification from video

Change-Id: I455cf885027f6b9d5f62f824e23856cbf3fed23a

diff --git a/src/modules/modality_vision/vision.c b/src/modules/modality_vision/vision.c
index b75c3a3..d1fc3f4 100644 (file)
--- a/src/modules/modality_vision/vision.c
+++ b/src/modules/modality_vision/vision.c
@@ -80,6 +80,11 @@ int max_confidence_idx = 0;
 mv_engine_config_h _engine_cfg = NULL;
 mv_inference_h _infer = NULL;
 
+typedef struct _video_info{
+       pthread_mutex_t* mutex;
+       mv_video_reader_h reader;
+} video_info;
+
 void _send_vision_event_cb(void *data)
 {
        mmi_provider_event_vision *ev = (mmi_provider_event_vision *)data;
@@ -224,6 +229,227 @@ int engine_config_user_hosted_tflite_cpu(mv_engine_config_h handle,
        return MEDIA_VISION_ERROR_NONE;
 }
 
+void _vision_finish_cb(void *user_data)
+{
+       LOGD("callback vision finish");
+       if (NULL == user_data) {
+               LOGE("ERROR: eos callback can't stop tracking process.");
+               return;
+       }
+
+       video_info *video_infos = (video_info *)user_data;
+       if(!video_infos)
+       {
+               LOGE("video_info malloc fail\n");
+               return;
+       }
+
+       if(video_infos->mutex)
+       {
+               pthread_mutex_unlock(video_infos->mutex);
+               pthread_mutex_destroy(video_infos->mutex);
+               LOGD("destroy mutex well");
+       }
+
+       if(video_infos->reader)
+       {
+               mv_destroy_video_reader(video_infos->reader);
+               LOGD("destroy reader well");
+       }
+
+       LOGD("end of destroy");
+}
+
+void eos_frame_cb(mv_video_reader_h reader,
+               void *user_data)
+{
+       if (NULL == user_data) {
+               LOGE("ERROR: eos callback can't stop tracking process.");
+               return;
+       }
+       video_info *video_infos = calloc(1, sizeof(video_info));
+       if(!video_infos)
+       {
+               LOGE("video_info malloc fail\n");
+               return;
+       }
+
+       video_infos->reader = reader;
+       video_infos->mutex = (pthread_mutex_t*)user_data;
+
+       ecore_main_loop_thread_safe_call_async(_vision_finish_cb, video_infos);
+}
+
+int infer_task_with_frame(mv_source_h frame, int task_id, void *user_data)
+{
+       struct timespec s_tspec;
+       struct timespec e_tspec;
+       int err;
+       clock_gettime(CLOCK_MONOTONIC, &s_tspec);
+
+       switch (task_id) {
+       case TASK_IC:
+               err = mv_inference_image_classify(frame, _infer, NULL,
+                                                                                 _image_classified_cb, user_data);
+               break;
+       default:
+               err = MEDIA_VISION_ERROR_INVALID_PARAMETER;
+               break;
+       }
+       if (err != MEDIA_VISION_ERROR_NONE)
+               LOGE("Fail to infer task [err:%i]\n", err);
+
+       clock_gettime(CLOCK_MONOTONIC, &e_tspec);
+       struct timespec diffspec = diff(s_tspec, e_tspec);
+       unsigned long timeDiff = gettotalmillisec(diffspec);
+       LOGD("elapsed time : %lu(ms)\n", timeDiff);
+       return err;
+}
+
+void new_frame_cb(
+               char *buffer,
+               unsigned int buffer_size,
+               image_data_s image_data,
+               void *user_data)
+{
+       mv_source_h frame = NULL;
+
+#define release_resources() \
+       if (frame) { \
+               const int err2 = mv_destroy_source(frame); \
+               if (MEDIA_VISION_ERROR_NONE != err2) { \
+                       LOGE( \
+                                       "\nERROR: Errors were occurred during source destroying; " \
+                                       "code %i\n", \
+                                       err2); \
+               } \
+       }
+
+       int err = mv_create_source(&frame);
+       if (MEDIA_VISION_ERROR_NONE != err) {
+               LOGE(
+                               "\nERROR: Errors were occurred during source creating; "
+                               "code %i\n",
+                               err);
+               release_resources();
+               return;
+       }
+
+       err = mv_source_fill_by_buffer(
+                       frame,
+                       (unsigned char*)buffer,
+                       buffer_size,
+                       image_data.image_width,
+                       image_data.image_height,
+                       image_data.image_colorspace);
+       if (MEDIA_VISION_ERROR_NONE != err) {
+               LOGE("ERROR: mv_source_h for frame is not filled; code %i\n", err);
+               release_resources();
+               return;
+       }
+
+       err = infer_task_with_frame(frame,TASK_IC, user_data);
+               if (MEDIA_VISION_ERROR_NONE != err) {
+               LOGE("ERROR: mv_source_h for frame is can't classification; code %i\n", err);
+               release_resources();
+               return;
+       }
+
+       release_resources();
+
+#undef release_resources
+}
+
+int infer_task_with_video(const char *path_to_video, mv_inference_h infer, int task_id)
+{
+       mv_video_reader_h reader = NULL;
+       image_data_s image_data = {0};
+       unsigned int fps = 0;
+       int err;
+
+#define release_resources() \
+       int err2 = MEDIA_VISION_ERROR_NONE; \
+       if (reader) { \
+               err2 = mv_destroy_video_reader(reader); \
+               if (MEDIA_VISION_ERROR_NONE != err2) { \
+                       LOGE("ERROR: Errors were occurred during video reader destroying; code %i\n", err2); \
+               } \
+       }
+
+       err = mv_create_video_reader(&reader);
+
+       if (MEDIA_VISION_ERROR_NONE != err) {
+               LOGE("ERROR: Errors were occurred during video reader creating; code %i\n", err);
+               release_resources();
+               return err;
+       }
+
+       err = mv_video_reader_load(
+               reader,
+               path_to_video,
+               &image_data,
+               &fps);
+       if (MEDIA_VISION_ERROR_NONE != err) {
+               LOGE("ERROR: Errors were occurred during video loading; code %i\n", err);
+               release_resources();
+               return err;
+       }
+
+       //Temporary we accept only RGB888
+       image_data.image_colorspace = MEDIA_VISION_COLORSPACE_RGB888;
+
+       LOGI("Receive frame metadata: wxh - %ux%u, fps - %u, format - %d\n",
+               image_data.image_width, image_data.image_height, fps, image_data.image_colorspace);
+
+       pthread_mutex_t block_during_tracking_mutex;
+       pthread_mutex_init(&block_during_tracking_mutex, NULL);
+       err = mv_video_reader_set_new_sample_cb(reader, new_frame_cb, &block_during_tracking_mutex);
+       if (MEDIA_VISION_ERROR_NONE != err) {
+               LOGE("ERROR: Errors were occurred during set new frame callback; code %i\n", err);
+               release_resources();
+               return err;
+       }
+
+       pthread_mutex_lock(&block_during_tracking_mutex);
+       err = mv_video_reader_set_eos_cb(reader, eos_frame_cb, &block_during_tracking_mutex);
+       if (MEDIA_VISION_ERROR_NONE != err) {
+               LOGE("ERROR: Errors were occurred during set new frame callback; code %i\n", err);
+                       release_resources();
+                       pthread_mutex_unlock(&block_during_tracking_mutex);
+                       pthread_mutex_destroy(&block_during_tracking_mutex);
+                       return err;
+       }
+       err = mv_video_reader_start(reader);
+       if (MEDIA_VISION_ERROR_NONE != err) {
+               LOGE("ERROR: Errors were occurred during video reading starts; code %i\n", err);
+               release_resources();
+               pthread_mutex_unlock(&block_during_tracking_mutex);
+               pthread_mutex_destroy(&block_during_tracking_mutex);
+               return err;
+       }
+
+#undef release_resources
+       return err;
+}
+
+int classification_from_video(const char *path_to_video)
+{
+       int err = MEDIA_VISION_ERROR_NONE;
+
+       if(_infer)
+       {
+               err = infer_task_with_video(path_to_video, _infer, TASK_IC);
+               if (err != MEDIA_VISION_ERROR_NONE) {
+                       LOGE("classification_from_video failed");
+               }
+       }
+       else
+       {
+               LOGE("Fail to find infer handler\n");
+       }
+       return err;
+}
+
 int infer_task_with_image(const char *img_file_name, mv_inference_h infer, int task_id)
 {
        mv_source_h mvSource = NULL;