add_subdirectory(${PROJECT_SOURCE_DIR}/surveillance)
add_subdirectory(${PROJECT_SOURCE_DIR}/inference)
add_subdirectory(${PROJECT_SOURCE_DIR}/stream_infer)
+add_subdirectory(${PROJECT_SOURCE_DIR}/stream_handpose)
+add_subdirectory(${PROJECT_SOURCE_DIR}/stream_handpose_browser)
--- /dev/null
+project(mv_handpose_test_suite)
+cmake_minimum_required(VERSION 2.6)
+
+set_property(DIRECTORY APPEND PROPERTY COMPILE_DEFINITIONS_DEBUG _DEBUG)
+
+if(NOT SKIP_WARNINGS)
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Werror")
+endif()
+
+set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/${LIB_INSTALL_DIR})
+set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/${LIB_INSTALL_DIR})
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
+
+include_directories(${PROJECT_SOURCE_DIR})
+include_directories(${MV_CAPI_MEDIA_VISION_INC_DIR})
+include_directories(${INC_IMAGE_HELPER})
+include_directories(${INC_VIDEO_HELPER})
+include_directories(${INC_TS_COMMON})
+
+file(GLOB MV_STREAMINFER_TEST_SUITE_INC_LIST "${PROJECT_SOURCE_DIR}/*.h")
+file(GLOB MV_STREAMINFER_TEST_SUITE_SRC_LIST "${PROJECT_SOURCE_DIR}/*.c")
+
+find_package(PkgConfig REQUIRED)
+pkg_check_modules(GLIB_PKG glib-2.0)
+
+if (NOT GLIB_PKG_FOUND)
+ message(SEND_ERROR "Failed to find glib")
+ return()
+else()
+ include_directories(${GLIB_PKG_INCLUDE_DIRS})
+endif()
+
+
+SET(dependents "gstreamer-1.0 gstreamer-app-1.0 gstreamer-video-1.0 cairo elementary ecore-wl2 appcore-efl capi-ui-efl-util")
+
+INCLUDE(FindPkgConfig)
+pkg_check_modules(${PROJECT_NAME} REQUIRED ${dependents})
+FOREACH(flag ${${PROJECT_NAME}_CFLAGS})
+ SET(EXTRA_CFLAGS "${EXTRA_CFLAGS} ${flag}")
+ENDFOREACH(flag)
+
+SET(CMAKE_C_FLAGS "-I./include -I./include/headers ${CMAKE_C_FLAGS} ${EXTRA_CFLAGS} -fPIC -Wall -DEFL_BETA_API_SUPPORT=1")
+SET(CMAKE_C_FLAGS_DEBUG "-O0 -g")
+
+add_executable(${PROJECT_NAME}
+ ${MV_STREAMINFER_TEST_SUITE_INC_LIST}
+ ${MV_STREAMINFER_TEST_SUITE_SRC_LIST}
+ ${MV_CAPI_MEDIA_VISION_INC_LIST})
+
+target_link_libraries(${PROJECT_NAME} ${MV_INFERENCE_LIB_NAME}
+ gstreamer-1.0
+ glib-2.0
+ capi-system-info
+ dlog
+ mv_image_helper
+ mv_video_helper
+ mv_testsuite_common
+ cairo
+ m
+ ${${PROJECT_NAME}_LIBRARIES}
+ )
+
+install(TARGETS ${PROJECT_NAME} DESTINATION ${testbin_dir})
--- /dev/null
+/**
+ * Copyright (c) 2015 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define _USE_MATH_DEFINES
+#include <fcntl.h>
+#include <mv_common.h>
+#include <mv_inference.h>
+
+#include <mv_testsuite_common.h>
+
+#include <image_helper.h>
+#include <mv_video_helper.h>
+
+#include <mv_log_cfg.h>
+
+#include <math.h>
+#include <ctype.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <dirent.h>
+#include <string.h>
+#include <limits.h>
+#include <time.h>
+#define MAX(a, b) \
+({ __typeof__ (a) _a = (a); \
+__typeof__ (b) _b = (b); \
+_a > _b ? _a : _b; })
+
+#define MIN(a,b) \
+({ __typeof__ (a) _a = (a); \
+__typeof__ (b) _b = (b); \
+_a < _b ? _a : _b; })
+
+#include <glib-2.0/glib.h>
+#include <gst/gst.h>
+#include <gst/video/video.h>
+#include <cairo.h>
+#include <cairo-gobject.h>
+
+#include <Elementary.h>
+#include <appcore-efl.h>
+#include <Ecore.h>
+#include <Ecore_Evas.h>
+#include <Ecore_Wl2.h>
+#include <tizen-extension-client-protocol.h>
+#include <efl_util.h>
+
+#include <gst/gst.h>
+#include <gst/video/videooverlay.h>
+#include <unistd.h>
+#include <time.h>
+
+#define WIDTH (640)
+#define HEIGHT (480)
+
+#ifdef PACKAGE
+#undef PACKAGE
+#endif
+#define PACKAGE "test"
+
+static char* gHandGestureLabel[] = { "None", "One", "Two", "Three", "Four", "Five",
+ "Six", "Eight", "SmallHeart", "OK", "Like", "DisLike",
+ "Fist", "Rock", "Love"};
+static int st = 0;
+static Evas_Object *g_eo = NULL;
+static Evas_Object *icon = NULL;
+
+/* for video display */
+static Evas_Object *g_win_id;
+static Evas_Object *selected_win_id;
+
+typedef enum {
+ MODEL_TYPE_POSE_CPM = 0,
+ MODEL_TYPE_POSE_HAND_AICLite, // posenet lite 224
+ MODEL_TYPE_POSE_HAND_AICLite2, // posenet 0709 f
+ MODEL_TYPE_POSE_HAND_AICLite2Q // posenet 0709 q
+};
+
+typedef struct {
+ gchar *filename;
+ gchar *filename2;
+ int numbuffers;
+ int modelType;
+ Evas_Object *win;
+ Evas_Object *layout_main; /* layout widget based on EDJ */
+ /* add more variables here */
+
+} appdata;
+
+static mv_rectangle_s poseRoi;
+
+static appdata ad;
+static GstBus *bus;
+static guint bus_watch_id;
+
+#define FILE_PATH_SIZE 1024
+
+// pose estimation
+#define PE_TFLITE_CPM_WEIGHT_PATH "/usr/share/capi-media-vision/models/PLD/tflite/pld-tflite-001.tflite"
+#define PE_POSE_LABEL_PATH "/usr/share/capi-media-vision/models/PLD/tflite/pose-label.txt"
+
+
+#define PE_TFLITE_AIC_1_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE/tflite/posenet1_lite_224.tflite"
+#define PE_TFLITE_AIC_2_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE/tflite/posenet2_lite_224.tflite"
+
+
+#define PE_TFLITE_AICLite_1_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE/tflite/posenet1_0709_f.tflite"
+#define PE_TFLITE_AICLite_2_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE/tflite/posenet2_0709_f.tflite"
+
+#define PE_TFLITE_AICLiteQ_1_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE/tflite/posenet1_0709_dq.tflite"
+#define PE_TFLITE_AICLiteQ_2_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE/tflite/posenet2_0709_f.tflite"
+
+static float thValNeck = 0.3f; // 15%
+static float thValArm = 0.1f; // 10 %
+static float thValLeg = 0.2f; // 5%
+
+typedef struct _rect {
+ int left;
+ int top;
+ int right;
+ int bottom;
+ int type;
+ bool updated;
+ bool cr_updated;
+} rect;
+
+
+typedef struct _humanSkeleton {
+ mv_point_s pose[21/*14*/];
+ mv_point_s prevPose[21/*14*/];
+ mv_rectangle_s loc;
+ mv_rectangle_s prevLoc;
+ mv_rectangle_s handRoi;
+ float scores[21/*14*/];
+ bool isPrevPose;
+ bool updated; // detection is find and pose is also good. then update drawing
+ bool locUpdated; // track previous detection
+ bool IsDetected; // detection
+ int label;
+} HumanSkeleton;
+
+typedef struct
+{
+ gboolean valid;
+ GstVideoInfo vinfo;
+} CairoOverlayState;
+
+typedef struct
+{
+ GstBuffer *buffer;
+ gpointer user_data;
+} HandOffData;
+
+//gchar *gesturenames;
+
+static HandOffData hdata_p;
+
+static GMutex pose_mutex;
+static guint old_timeout = 0;
+static guint nFrames = 0;
+
+// Gstreamer
+GstElement *pipeline, *source, *filter, *toverlay, *sink, *sink2;
+GstElement *tee, *vscale, *vsfilter, *vconv, *vcfilter;
+GstElement *vrate, *vrfilter, *fsink, *vrsink;
+GstElement *queue1, *queue2, *queue3;
+GstElement *oconv, *coverlay;
+GstElement *vcrop, *vcrscale, *vcrsfilter, *vcrsconv, *vcrscfilter, *vcrssink;
+CairoOverlayState *overlay_state;
+
+GstElement *dbin, *dscale, *dconv;
+GstElement *dsfilter, *dcfilter;
+
+GstElement *tee2, *enc, *muxmp4, *fsink2, *queue4, *queue5, *encconv;
+
+//static HandSkeleton handSkeleton;
+static HumanSkeleton humanSkeleton;
+gulong handler_p;
+GList *line_list = NULL;
+
+mv_source_h mv_src_p;
+mv_source_h mv_src_p2;
+
+// Human pose
+mv_engine_config_h hp_mv_engine_cfg;
+mv_inference_h hp_mv_infer;
+
+mv_engine_config_h hp_mv_engine_cfg2;
+mv_inference_h hp_mv_infer2;
+
+static void * outputTensorData;
+
+FILE *fp;
+
+static bool IsGestureMode;
+int gesture_x;
+int gesture_y;
+
+static int poseCropSize = 0;
+
+#define IMAGE_SIZE_WIDTH 640
+#define IMAGE_SIZE_HEIGHT 480
+
+#define NANO_PER_SEC ((__clock_t) 1000000000)
+#define NANO_PER_MILLI ((__clock_t) 1000000)
+#define MILLI_PER_SEC ((__clock_t) 1000)
+
+#define VOLUME_INTERVAL 10
+
+struct timespec diff(struct timespec start, struct timespec end)
+{
+ struct timespec temp;
+ if ((end.tv_nsec - start.tv_nsec) < 0) {
+ temp.tv_sec = end.tv_sec - start.tv_sec - 1;
+ temp.tv_nsec = NANO_PER_SEC + end.tv_nsec - start.tv_nsec;
+ }
+ else {
+ temp.tv_sec = end.tv_sec - start.tv_sec;
+ temp.tv_nsec = end.tv_nsec - start.tv_nsec;
+ }
+ return temp;
+}
+
+unsigned long gettotalmillisec(const struct timespec time)
+{
+ return time.tv_sec * MILLI_PER_SEC + time.tv_nsec / NANO_PER_MILLI;
+}
+
+
+void int_handler(int sig)
+{
+ char c;
+
+ signal(sig, SIG_IGN);
+ while ((getchar()) != '\n');
+
+ printf(TEXT_YELLOW "Do you want to quit? [y/n]\n" TEXT_RESET);
+ c = getchar();
+ if (c == 'y' || c == "Y") {
+
+ g_signal_handler_disconnect(vrsink, handler_p);
+#if 0
+ g_signal_handler_disconnect(vcrssink, handler_gp);
+#endif
+
+ gst_element_send_event(pipeline, gst_event_new_eos());
+
+ sleep(4);
+
+
+ if (mv_src_p)
+ mv_destroy_source(mv_src_p);
+
+ if (hp_mv_infer)
+ mv_inference_destroy(hp_mv_infer);
+
+ if (hp_mv_engine_cfg)
+ mv_destroy_engine_config(hp_mv_engine_cfg);
+
+ if (mv_src_p2)
+ mv_destroy_source(mv_src_p2);
+
+ if (hp_mv_infer2)
+ mv_inference_destroy(hp_mv_infer2);
+
+ if (hp_mv_engine_cfg)
+ mv_destroy_engine_config(hp_mv_engine_cfg2);
+
+ if (outputTensorData) {
+ free(outputTensorData);
+ outputTensorData = NULL;
+ }
+ printf(TEXT_YELLOW "exit..\n" TEXT_RESET);
+ signal(SIGINT, SIG_DFL);
+ exit(0);
+ } else {
+ printf("no");
+ signal(SIGINT, int_handler);
+ }
+
+ getchar(); // Get new line character
+}
+
+void _hand_pose_cb (
+ mv_source_h source,
+ int number_of_pose_estimation,
+ mv_inference_pose_s *locations,
+ int label,
+ void *user_data)
+{
+ //printf("%d landmarks\n", number_of_pose_estimation);
+ int width, height;
+ mv_source_get_height(source, &width);
+ mv_source_get_height(source, &height);
+ for (int n = 0; n < number_of_pose_estimation; ++n) {
+ humanSkeleton.pose[n].x = (int)(640.f*(float)locations->landmarks[n].point.x/(float)width);
+ humanSkeleton.pose[n].y = (int)(480.f*(float)locations->landmarks[n].point.y/(float)height);
+ humanSkeleton.scores[n] = 1.0f; /* confidences[n];*/
+
+ //printf("(%d,%d): %f\n", humanSkeleton.pose[n].x, humanSkeleton.pose[n].y, confidences[n]);
+ //printf("(%d,%d)\n", humanSkeleton.pose[n].x, humanSkeleton.pose[n].y);
+ }
+ humanSkeleton.label = label;
+ humanSkeleton.IsDetected = true;
+}
+
+static void _hand_detected_cb (
+ mv_source_h source,
+ const int number_of_hands,
+ const float *confidences,
+ const mv_rectangle_s *locations,
+ void *user_data) //user_data can be mv_source?
+{
+
+#if 0
+ if (0 /*confidences[1] < thValNeck*/) {
+ printf("lost pose\n");
+ humanSkeleton.IsDetected = false;
+ humanSkeleton.isPrevPose = false;
+ return;
+ }
+
+ printf("%d landmarks, %d crop\n", number_of_landmarks, poseCropSize);
+ for (int n = 0; n < number_of_landmarks; ++n) {
+
+ humanSkeleton.pose[n].x = (int)((float)(locations[n].x + poseRoi.point.x) / (float)poseCropSize * 640.f);
+ humanSkeleton.pose[n].y = (int)((float)(locations[n].y + poseRoi.point.y) / (float)poseCropSize * 480.f);
+ humanSkeleton.scores[n] = 1.0f; /* confidences[n];*/
+
+ //printf("(%d,%d): %f\n", humanSkeleton.pose[n].x, humanSkeleton.pose[n].y, confidences[n]);
+ printf("(%d,%d)\n", humanSkeleton.pose[n].x, humanSkeleton.pose[n].y);
+ }
+ humanSkeleton.IsDetected = true;
+#else
+
+ if (number_of_hands <= 0) {
+ humanSkeleton.IsDetected = false;
+ humanSkeleton.label = -1;
+ return;
+ }
+
+ //seg fault?
+ struct timespec s_tspec;
+ struct timespec e_tspec;
+
+ clock_gettime(CLOCK_MONOTONIC, &s_tspec);
+
+ mv_source_clear(mv_src_p2);
+ mv_source_fill_by_tensor_buffer(mv_src_p2, user_data,
+ MV_INFERENCE_DATA_FLOAT32,
+ 56 * 56 * 21 * sizeof(float),
+ 56, 56, 21, 3);
+
+ clock_gettime(CLOCK_MONOTONIC, &e_tspec);
+
+ struct timespec diffspec = diff(s_tspec, e_tspec);
+ unsigned long timeDiff = gettotalmillisec(diffspec);
+ //printf("memcpy time: %lu(ms)\n", timeDiff);
+
+ clock_gettime(CLOCK_MONOTONIC, &s_tspec);
+
+ mv_inference_pose_landmark_detect(mv_src_p2, hp_mv_infer2, NULL, _hand_pose_cb, NULL);
+
+ clock_gettime(CLOCK_MONOTONIC, &e_tspec);
+
+ diffspec = diff(s_tspec, e_tspec);
+ timeDiff = gettotalmillisec(diffspec);
+ printf("pose_estimation time: %lu(ms)\n", timeDiff);
+ //humanSkeleton.IsDetected = true;
+#endif
+ return;
+}
+
+
+void _human_pose_cb (
+ mv_source_h source,
+ int number_of_pose_estimation,
+ mv_inference_pose_s *locations,
+ int label,
+ void *user_data)
+{
+ //printf("%d landmarks\n", number_of_pose_estimation);
+
+ float score = 0.0f;
+ int cnt = 0;
+ for (int n = 0; n < number_of_pose_estimation; ++n) {
+ humanSkeleton.pose[n].x = locations->landmarks[n].point.x;
+ humanSkeleton.pose[n].y = locations->landmarks[n].point.y;
+ humanSkeleton.scores[n] = locations->landmarks[n].score;
+ if (score >= 0.0 ) {
+ score += humanSkeleton.scores[n];
+ cnt++;
+ }
+
+ //printf("(%d,%d): %f\n", humanSkeleton.pose[n].x, humanSkeleton.pose[n].y, confidences[n]);
+ //printf("(%d,%d)\n", humanSkeleton.pose[n].x, humanSkeleton.pose[n].y);
+ }
+ if(cnt == 0){
+ printf("count is 0 !!! \n");
+ }else{
+ score /= (float)cnt;
+ }
+
+ if (score >= 0.5) {
+ humanSkeleton.IsDetected = true;
+ } else {
+ humanSkeleton.IsDetected = false;
+ humanSkeleton.label = -1;
+ }
+}
+
+static gboolean
+run_pose (void *user_data)
+{
+ HandOffData *udata = (HandOffData *)user_data;
+ if (!GST_IS_BUFFER(udata->buffer))
+ return FALSE;
+
+ GstMapInfo map;
+
+ gst_buffer_map(udata->buffer, &map, GST_MAP_READ);
+
+ mv_source_clear(mv_src_p);
+
+ mv_source_fill_by_buffer(mv_src_p, map.data, 192*192*3, 192, 192, MEDIA_VISION_COLORSPACE_RGB888);
+
+ gst_buffer_unmap(udata->buffer, &map);
+
+ struct timespec s_tspec;
+ struct timespec e_tspec;
+
+ clock_gettime(CLOCK_MONOTONIC, &s_tspec);
+
+ mv_inference_pose_landmark_detect(mv_src_p, hp_mv_infer, NULL, _human_pose_cb, NULL);
+
+ clock_gettime(CLOCK_MONOTONIC, &e_tspec);
+ struct timespec diffspec = diff(s_tspec, e_tspec);
+ unsigned long timeDiff = gettotalmillisec(diffspec);
+ printf("detect + pose time: %lu(ms)\n", timeDiff);
+
+ return FALSE;
+
+}
+
+static gboolean
+run_hand (void *user_data)
+{
+ HandOffData *udata = (HandOffData *)user_data;
+ if (!GST_IS_BUFFER(udata->buffer))
+ return FALSE;
+
+ GstMapInfo map;
+
+ gst_buffer_map(udata->buffer, &map, GST_MAP_READ);
+
+ mv_source_clear(mv_src_p);
+
+ mv_source_fill_by_buffer(mv_src_p, map.data, 224*224*3, 224, 224, MEDIA_VISION_COLORSPACE_RGB888);
+
+ gst_buffer_unmap(udata->buffer, &map);
+
+
+ struct timespec s_tspec;
+ struct timespec e_tspec;
+
+ void * outputTensorBuffer = (void*)udata->user_data;
+
+ clock_gettime(CLOCK_MONOTONIC, &s_tspec);
+
+ // invoke tflite -> _hand_detected_cb -> memcpy output -> invoke tflite -> _pose_cb
+ mv_inference_hand_detect(mv_src_p, hp_mv_infer, _hand_detected_cb, outputTensorBuffer);
+
+ clock_gettime(CLOCK_MONOTONIC, &e_tspec);
+ struct timespec diffspec = diff(s_tspec, e_tspec);
+ unsigned long timeDiff = gettotalmillisec(diffspec);
+ printf("detect + pose time: %lu(ms)\n", timeDiff);
+
+ return FALSE;
+
+}
+
+static void
+_pose_est_handoff(GstElement *object, GstBuffer *buffer, GstPad *pad, gpointer user_data)
+{
+
+ nFrames++;
+ hdata_p.buffer = buffer;
+ hdata_p.user_data = user_data;
+
+#if 0
+ if (nFrames % 15 == 0) {
+ g_mutex_lock(&pose_mutex);
+ g_idle_add (run_pose, &hdata_p);
+ g_mutex_unlock(&pose_mutex);
+ }
+#else
+ g_mutex_lock(&pose_mutex);
+ g_idle_add (run_pose, &hdata_p);
+ g_mutex_unlock(&pose_mutex);
+#endif
+}
+
+static void
+_hand_est_handoff(GstElement *object, GstBuffer *buffer, GstPad *pad, gpointer user_data)
+{
+
+ nFrames++;
+ hdata_p.buffer = buffer;
+ hdata_p.user_data = user_data;
+
+#if 0
+ if (nFrames % 15 == 0) {
+ g_mutex_lock(&pose_mutex);
+ g_idle_add (run_hand, &hdata_p);
+ g_mutex_unlock(&pose_mutex);
+ }
+#else
+ g_mutex_lock(&pose_mutex);
+ g_idle_add (run_hand, &hdata_p);
+ g_mutex_unlock(&pose_mutex);
+#endif
+}
+
+
+static void
+prepare_overlay (GstElement * overlay, GstCaps * caps, gpointer user_data)
+{
+ CairoOverlayState *state = (CairoOverlayState *) user_data;
+
+ state->valid = gst_video_info_from_caps (&state->vinfo, caps);
+}
+
+static void
+draw_overlay_pose (GstElement * overlay, cairo_t * cr, guint64 timestamp,
+ guint64 duration, gpointer user_data)
+{
+ CairoOverlayState *s = (CairoOverlayState *) user_data;
+
+ if (!s->valid) {
+ printf("not ready draw_overlay");
+ return;
+ }
+ //printf("draw_overlay_pose");
+ cairo_set_source_rgba(cr, 0.1, 0.9, 0.0, 0.7);
+ cairo_set_line_width(cr, 2.0);
+
+
+ if (!humanSkeleton.IsDetected)
+ return;
+
+ //
+ if (humanSkeleton.isPrevPose == false) {
+ humanSkeleton.prevPose[1] = humanSkeleton.pose[1];
+ // head - neck
+ if (humanSkeleton.scores[0] >= thValNeck ) {
+ humanSkeleton.prevPose[0] = humanSkeleton.pose[0];
+ }
+
+ // right arm
+ if (humanSkeleton.scores[2] >= thValArm) {
+ // neck - right shoulder
+ humanSkeleton.prevPose[2] = humanSkeleton.pose[2];
+ if (humanSkeleton.scores[3] >= thValArm) {
+ // right shoulder - right elbow
+ humanSkeleton.prevPose[3] = humanSkeleton.pose[3];
+ if (humanSkeleton.scores[4] >= thValArm) {
+ // right elbow - right wrist
+ humanSkeleton.prevPose[4] = humanSkeleton.pose[4];
+ }
+ }
+ }
+
+ // left arm
+ if (humanSkeleton.scores[5] >= thValArm) {
+ // neck - right shoulder
+ humanSkeleton.prevPose[5] = humanSkeleton.pose[5];
+ if (humanSkeleton.scores[6] >= thValArm) {
+ // right shoulder - right elbow
+ humanSkeleton.prevPose[6] = humanSkeleton.pose[6];
+ if (humanSkeleton.scores[7] >= thValArm) {
+ // right elbow - right wrist
+ humanSkeleton.prevPose[7] = humanSkeleton.pose[7];
+ }
+ }
+ }
+
+ // right leg
+ if (humanSkeleton.scores[8] >= thValLeg) {
+ // neck - right shoulder
+ humanSkeleton.prevPose[8] = humanSkeleton.pose[8];
+ if (humanSkeleton.scores[9] >= thValLeg) {
+ // right shoulder - right elbow
+ humanSkeleton.prevPose[9] = humanSkeleton.pose[9];
+ if (humanSkeleton.scores[10] >= thValLeg) {
+ // right elbow - right wrist
+ humanSkeleton.prevPose[10] = humanSkeleton.pose[10];
+ }
+ }
+ }
+
+ // left leg
+ if (humanSkeleton.scores[11] >= thValLeg) {
+ // neck - right shoulder
+ humanSkeleton.prevPose[11] = humanSkeleton.pose[11];
+ if (humanSkeleton.scores[12] >= thValLeg) {
+ // right shoulder - right elbow
+ humanSkeleton.prevPose[12] = humanSkeleton.pose[12];
+ if (humanSkeleton.scores[13] >= thValLeg) {
+ // right elbow - right wrist
+ humanSkeleton.prevPose[13] = humanSkeleton.pose[13];
+ }
+ }
+ }
+ humanSkeleton.isPrevPose = true;
+ } else {
+ // weighted sum of pose and prevPose
+ // method1: fixed weights (pose : prevPose = 0.7 : 0.3)
+ float poseWeight = 0.7f;
+ float prevPoseWeight = 0.3f;
+ humanSkeleton.prevPose[1].x = (poseWeight * humanSkeleton.pose[1].x +
+ prevPoseWeight * humanSkeleton.prevPose[1].x);
+ humanSkeleton.prevPose[1].y = (poseWeight * humanSkeleton.pose[1].y +
+ prevPoseWeight * humanSkeleton.prevPose[1].y);
+ // head - neck
+ if (humanSkeleton.scores[0] >= thValNeck ) {
+ humanSkeleton.prevPose[0].x = (poseWeight * humanSkeleton.pose[0].x +
+ prevPoseWeight * humanSkeleton.prevPose[0].x);
+ humanSkeleton.prevPose[0].y = (poseWeight * humanSkeleton.pose[0].y +
+ prevPoseWeight * humanSkeleton.prevPose[0].y);
+ }
+
+ // right arm
+ if (humanSkeleton.scores[2] >= thValArm) {
+ // neck - right shoulder
+ humanSkeleton.prevPose[2].x = (poseWeight * humanSkeleton.pose[2].x +
+ prevPoseWeight * humanSkeleton.prevPose[2].x);
+ humanSkeleton.prevPose[2].y = (poseWeight * humanSkeleton.pose[2].y +
+ prevPoseWeight * humanSkeleton.prevPose[2].y);
+ if (humanSkeleton.scores[3] >= thValArm) {
+ // right shoulder - right elbow
+ humanSkeleton.prevPose[3].x = (poseWeight * humanSkeleton.pose[3].x +
+ prevPoseWeight * humanSkeleton.prevPose[3].x);
+ humanSkeleton.prevPose[3].y = (poseWeight * humanSkeleton.pose[3].y +
+ prevPoseWeight * humanSkeleton.prevPose[3].y);
+ if (humanSkeleton.scores[4] >= thValArm) {
+ // right elbow - right wrist
+ humanSkeleton.prevPose[4].x = (poseWeight * humanSkeleton.pose[4].x +
+ prevPoseWeight * humanSkeleton.prevPose[4].x);
+ humanSkeleton.prevPose[4].y = (poseWeight * humanSkeleton.pose[4].y +
+ prevPoseWeight * humanSkeleton.prevPose[4].y);
+ }
+ }
+ }
+
+ // left arm
+ if (humanSkeleton.scores[5] >= thValArm) {
+ // neck - right shoulder
+ humanSkeleton.prevPose[5].x = (poseWeight * humanSkeleton.pose[5].x +
+ prevPoseWeight * humanSkeleton.prevPose[5].x);
+ humanSkeleton.prevPose[5].y = (poseWeight * humanSkeleton.pose[5].y +
+ prevPoseWeight * humanSkeleton.prevPose[5].y);
+ if (humanSkeleton.scores[6] >= thValArm) {
+ // right shoulder - right elbow
+ humanSkeleton.prevPose[6].x = (poseWeight * humanSkeleton.pose[6].x +
+ prevPoseWeight * humanSkeleton.prevPose[6].x);
+ humanSkeleton.prevPose[6].y = (poseWeight * humanSkeleton.pose[6].y +
+ prevPoseWeight * humanSkeleton.prevPose[6].y);
+ if (humanSkeleton.scores[7] >= thValArm) {
+ // right elbow - right wrist
+ humanSkeleton.prevPose[7].x = (poseWeight * humanSkeleton.pose[7].x +
+ prevPoseWeight * humanSkeleton.prevPose[7].x);
+ humanSkeleton.prevPose[7].y = (poseWeight * humanSkeleton.pose[7].y +
+ prevPoseWeight * humanSkeleton.prevPose[7].y);
+ }
+ }
+ }
+
+ // right leg
+ if (humanSkeleton.scores[8] >= thValLeg) {
+ // neck - right shoulder
+ humanSkeleton.prevPose[8].x = (poseWeight * humanSkeleton.pose[8].x +
+ prevPoseWeight * humanSkeleton.prevPose[8].x);
+ humanSkeleton.prevPose[8].y = (poseWeight * humanSkeleton.pose[8].y +
+ prevPoseWeight * humanSkeleton.prevPose[8].y);
+ if (humanSkeleton.scores[9] >= thValLeg) {
+ // right shoulder - right elbow
+ humanSkeleton.prevPose[9].x = (poseWeight * humanSkeleton.pose[9].x +
+ prevPoseWeight * humanSkeleton.prevPose[9].x);
+ humanSkeleton.prevPose[9].y = (poseWeight * humanSkeleton.pose[9].y +
+ prevPoseWeight * humanSkeleton.prevPose[9].y);
+ if (humanSkeleton.scores[10] >= thValLeg) {
+ // right elbow - right wrist
+ humanSkeleton.prevPose[10].x = (poseWeight * humanSkeleton.pose[10].x +
+ prevPoseWeight * humanSkeleton.prevPose[10].x);
+ humanSkeleton.prevPose[10].y = (poseWeight * humanSkeleton.pose[10].y +
+ prevPoseWeight * humanSkeleton.prevPose[10].y);
+ }
+ }
+ }
+
+ // left leg
+ if (humanSkeleton.scores[11] >= thValLeg) {
+ // neck - right shoulder
+ humanSkeleton.prevPose[11].x = (poseWeight * humanSkeleton.pose[11].x +
+ prevPoseWeight * humanSkeleton.prevPose[11].x);
+ humanSkeleton.prevPose[11].y = (poseWeight * humanSkeleton.pose[11].y +
+ prevPoseWeight * humanSkeleton.prevPose[11].y);
+ if (humanSkeleton.scores[12] >= thValLeg) {
+ // right shoulder - right elbow
+ humanSkeleton.prevPose[12].x = (poseWeight * humanSkeleton.pose[12].x +
+ prevPoseWeight * humanSkeleton.prevPose[12].x);
+ humanSkeleton.prevPose[12].y = (poseWeight * humanSkeleton.pose[12].y +
+ prevPoseWeight * humanSkeleton.prevPose[12].y);
+ if (humanSkeleton.scores[13] >= thValLeg) {
+ // right elbow - right wrist
+ humanSkeleton.prevPose[13].x = (poseWeight * humanSkeleton.pose[13].x +
+ prevPoseWeight * humanSkeleton.prevPose[13].x);
+ humanSkeleton.prevPose[13].y = (poseWeight * humanSkeleton.pose[13].y +
+ prevPoseWeight * humanSkeleton.prevPose[13].y);
+ }
+ }
+ }
+ }
+
+ //
+ //draw..
+ // head - neck
+ if (humanSkeleton.scores[0] >= thValNeck ) {
+ cairo_move_to(cr, humanSkeleton.prevPose[0].x, humanSkeleton.prevPose[0].y);
+ cairo_line_to(cr, humanSkeleton.prevPose[1].x, humanSkeleton.prevPose[1].y);
+ }
+
+ // right arm
+ cairo_move_to(cr, humanSkeleton.prevPose[1].x, humanSkeleton.prevPose[1].y);
+ if (humanSkeleton.scores[2] >= thValArm) {
+ // neck - right shoulder
+ cairo_line_to(cr, humanSkeleton.prevPose[2].x, humanSkeleton.prevPose[2].y);
+ if (humanSkeleton.scores[3] >= thValArm) {
+ // right shoulder - right elbow
+ cairo_line_to(cr, humanSkeleton.prevPose[3].x, humanSkeleton.prevPose[3].y);
+ if (humanSkeleton.scores[4] >= thValArm) {
+ // right elbow - right wrist
+ cairo_line_to(cr, humanSkeleton.prevPose[4].x, humanSkeleton.prevPose[4].y);
+ }
+ }
+ }
+ cairo_stroke(cr);
+
+ // left arm
+ cairo_move_to(cr, humanSkeleton.prevPose[1].x, humanSkeleton.prevPose[1].y);
+ if (humanSkeleton.scores[5] >= thValArm) {
+ // neck - right shoulder
+ cairo_line_to(cr, humanSkeleton.prevPose[5].x, humanSkeleton.prevPose[5].y);
+ if (humanSkeleton.scores[6] >= thValArm) {
+ // right shoulder - right elbow
+ cairo_line_to(cr, humanSkeleton.prevPose[6].x, humanSkeleton.prevPose[6].y);
+ if (humanSkeleton.scores[7] >= thValArm) {
+ // right elbow - right wrist
+ cairo_line_to(cr, humanSkeleton.prevPose[7].x, humanSkeleton.prevPose[7].y);
+ }
+ }
+ }
+ cairo_stroke(cr);
+
+
+ // right leg
+ cairo_move_to(cr, humanSkeleton.prevPose[1].x, humanSkeleton.prevPose[1].y);
+ if (humanSkeleton.scores[8] >= thValLeg) {
+ // neck - right shoulder
+ cairo_line_to(cr, humanSkeleton.prevPose[8].x, humanSkeleton.prevPose[8].y);
+ if (humanSkeleton.scores[9] >= thValLeg) {
+ // right shoulder - right elbow
+ cairo_line_to(cr, humanSkeleton.prevPose[9].x, humanSkeleton.prevPose[9].y);
+ if (humanSkeleton.scores[10] >= thValLeg) {
+ // right elbow - right wrist
+ cairo_line_to(cr, humanSkeleton.prevPose[10].x, humanSkeleton.prevPose[10].y);
+ }
+ }
+ }
+ cairo_stroke(cr);
+
+ // left leg
+ cairo_move_to(cr, humanSkeleton.prevPose[1].x, humanSkeleton.prevPose[1].y);
+ if (humanSkeleton.scores[11] >= thValLeg) {
+ // neck - right shoulder
+ cairo_line_to(cr, humanSkeleton.prevPose[11].x, humanSkeleton.prevPose[11].y);
+ if (humanSkeleton.scores[12] >= thValLeg) {
+ // right shoulder - right elbow
+ cairo_line_to(cr, humanSkeleton.prevPose[12].x, humanSkeleton.prevPose[12].y);
+ if (humanSkeleton.scores[13] >= thValLeg) {
+ // right elbow - right wrist
+ cairo_line_to(cr, humanSkeleton.prevPose[13].x, humanSkeleton.prevPose[13].y);
+ }
+ }
+ }
+ cairo_stroke(cr);
+}
+
+static void
+draw_overlay_hand (GstElement * overlay, cairo_t * cr, guint64 timestamp,
+ guint64 duration, gpointer user_data)
+{
+ CairoOverlayState *s = (CairoOverlayState *) user_data;
+
+ if (!s->valid) {
+ printf("not ready draw_overlay");
+ return;
+ }
+
+ cairo_set_source_rgba(cr, 0.0, 0.0, 1.0, 0.7);
+ cairo_set_line_width(cr, 2.0);
+
+
+ if (!humanSkeleton.IsDetected)
+ return;
+
+ if (humanSkeleton.label >= 0 && humanSkeleton.label <= 14) {
+ cairo_select_font_face(cr, "sans-serif", CAIRO_FONT_SLANT_NORMAL, CAIRO_FONT_WEIGHT_BOLD);
+ cairo_set_font_size(cr,15);
+ cairo_move_to(cr, 320.0, 48.0);
+ cairo_show_text(cr, gHandGestureLabel[humanSkeleton.label]);
+ }
+
+ cairo_set_source_rgba(cr, 0.1, 0.9, 0.0, 0.7);
+ cairo_set_line_width(cr, 2.0);
+
+ //printf("gHandGestureLabel[humanSkeleton.label] : [%d], x: %d y: %d",gHandGestureLabel[humanSkeleton.label], humanSkeleton.pose[0].x, humanSkeleton.pose[0].y);
+
+
+ // thumb - red
+ cairo_set_source_rgba (cr, 0.9, 0.1, 0.0, 0.7);
+ cairo_move_to(cr, humanSkeleton.pose[0].x, humanSkeleton.pose[0].y);
+ for (int k = 1 ; k < 5; ++k) {
+ cairo_line_to(cr, humanSkeleton.pose[k].x, humanSkeleton.pose[k].y);
+ }
+ cairo_stroke(cr);
+
+ // fore - red
+ cairo_set_source_rgba (cr, 0.9, 0.1, 0.0, 0.7);
+ cairo_move_to(cr, humanSkeleton.pose[0].x, humanSkeleton.pose[0].y);
+ for (int k = 5 ; k < 9; ++k) {
+ cairo_line_to(cr, humanSkeleton.pose[k].x, humanSkeleton.pose[k].y);
+ }
+ cairo_stroke(cr);
+
+ // middle - grean
+ cairo_set_source_rgba (cr, 0.1, 0.9, 0.0, 0.7);
+ cairo_move_to(cr, humanSkeleton.pose[0].x, humanSkeleton.pose[0].y);
+ for (int k = 9 ; k < 13; ++k) {
+ cairo_line_to(cr, humanSkeleton.pose[k].x, humanSkeleton.pose[k].y);
+ }
+ cairo_stroke(cr);
+
+ // ring - blue
+ cairo_set_source_rgba (cr, 0.1, 0.0, 0.9, 0.7);
+ cairo_move_to(cr, humanSkeleton.pose[0].x, humanSkeleton.pose[0].y);
+ for (int k = 13 ; k < 17; ++k) {
+ cairo_line_to(cr, humanSkeleton.pose[k].x, humanSkeleton.pose[k].y);
+ }
+ cairo_stroke(cr);
+
+ // little - purple
+ cairo_set_source_rgba (cr, 0.5, 0.0, 0.5, 0.7);
+ cairo_move_to(cr, humanSkeleton.pose[0].x, humanSkeleton.pose[0].y);
+ for (int k = 17 ; k < 21; ++k) {
+ cairo_line_to(cr, humanSkeleton.pose[k].x, humanSkeleton.pose[k].y);
+ }
+ cairo_stroke(cr);
+
+
+}
+
+static gboolean bus_call (GstBus *bus, GstMessage *msg, gpointer data)
+{
+
+ switch (GST_MESSAGE_TYPE (msg)) {
+
+ case GST_MESSAGE_EOS:
+ printf ("End of stream\n");
+ break;
+
+ case GST_MESSAGE_ERROR: {
+ gchar *debug;
+ GError *error;
+
+ gst_message_parse_error (msg, &error, &debug);
+ g_free (debug);
+
+ printf ("Error: %s\n", error->message);
+ g_error_free (error);
+
+ break;
+ }
+ default:
+ break;
+ }
+
+ return TRUE;
+}
+
+int perform_armnn_human_pose_cpm_configure(mv_engine_config_h mv_engine_cfg)
+{
+ if (mv_engine_cfg == NULL) {
+ printf("mv_engine_cfg is null\n");
+ return MEDIA_VISION_ERROR_INVALID_PARAMETER;
+ }
+
+ char *inputNodeName = "image";
+ char *outputNodeName[1] = {"Convolutional_Pose_Machine/stage_5_out"};
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_WEIGHT_FILE_PATH,
+ PE_TFLITE_CPM_WEIGHT_PATH);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_DATA_TYPE,
+ MV_INFERENCE_DATA_FLOAT32);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_MEAN_VALUE,
+ 0.0);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_STD_VALUE,
+ 1.0);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_BACKEND_TYPE,
+ MV_INFERENCE_BACKEND_ARMNN);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_TARGET_TYPE,
+ MV_INFERENCE_TARGET_GPU);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_WIDTH,
+ 192);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_HEIGHT,
+ 192);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_CHANNELS,
+ 3);
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_NODE_NAME,
+ inputNodeName);
+
+ mv_engine_config_set_array_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_OUTPUT_NODE_NAMES,
+ outputNodeName,
+ 1);
+
+ return MEDIA_VISION_ERROR_NONE;
+}
+
+int perform_tflite_human_pose_cpm_configure(mv_engine_config_h mv_engine_cfg)
+{
+ if (mv_engine_cfg == NULL) {
+ printf("mv_engine_cfg is null\n");
+ return MEDIA_VISION_ERROR_INVALID_PARAMETER;
+ }
+
+ char *inputNodeName = "image";
+ char *outputNodeName[1] = {"Convolutional_Pose_Machine/stage_5_out"};
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_WEIGHT_FILE_PATH,
+ PE_TFLITE_CPM_WEIGHT_PATH);
+
+ mv_engine_config_set_string_attribute(
+ mv_engine_cfg,
+ MV_INFERENCE_MODEL_USER_FILE_PATH,
+ PE_POSE_LABEL_PATH);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_DATA_TYPE,
+ MV_INFERENCE_DATA_FLOAT32);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_MEAN_VALUE,
+ 0.0);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_STD_VALUE,
+ 1.0);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_BACKEND_TYPE,
+ MV_INFERENCE_BACKEND_TFLITE);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_TARGET_TYPE,
+ MV_INFERENCE_TARGET_CPU);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_WIDTH,
+ 192);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_HEIGHT,
+ 192);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_CHANNELS,
+ 3);
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_NODE_NAME,
+ inputNodeName);
+
+ mv_engine_config_set_array_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_OUTPUT_NODE_NAMES,
+ outputNodeName,
+ 1);
+
+ return MEDIA_VISION_ERROR_NONE;
+}
+
+int perform_tflite_hand_detection_AIC(mv_engine_config_h mv_engine_cfg)
+{
+ if (mv_engine_cfg == NULL) {
+ printf("mv_engine_cfg is null\n");
+ return MEDIA_VISION_ERROR_INVALID_PARAMETER;
+ }
+
+ char *inputNodeName = "input";
+ char *outputNodeNames[2] = {"mobilenetv2/boundingbox2", "mobilenetv2/heatmap"};
+
+ //outputTensorData = (void*)calloc(56*56*21, sizeof(float));
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_WEIGHT_FILE_PATH,
+ PE_TFLITE_AIC_1_WEIGHT_PATH);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_DATA_TYPE,
+ MV_INFERENCE_DATA_FLOAT32);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_MEAN_VALUE,
+ 0.0);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_STD_VALUE,
+ 1.0);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_BACKEND_TYPE,
+ MV_INFERENCE_BACKEND_TFLITE);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_TARGET_DEVICE_TYPE,
+ MV_INFERENCE_TARGET_DEVICE_CPU);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_WIDTH,
+ 224);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_HEIGHT,
+ 224);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_CHANNELS,
+ 3);
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_NODE_NAME,
+ inputNodeName);
+
+ mv_engine_config_set_array_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_OUTPUT_NODE_NAMES,
+ outputNodeNames,
+ 2);
+
+ return MEDIA_VISION_ERROR_NONE;
+}
+
+int perform_tflite_hand_detection_AIC2(mv_engine_config_h mv_engine_cfg)
+{
+ char *inputNodeName = "input";
+ char *outputNodeNames[2] = {"mobilenetv2/coord_refine", "mobilenetv2/gesture"};
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_WEIGHT_FILE_PATH,
+ PE_TFLITE_AIC_2_WEIGHT_PATH);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_DATA_TYPE,
+ MV_INFERENCE_DATA_FLOAT32);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_MEAN_VALUE,
+ 0.0);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_STD_VALUE,
+ 1.0);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_BACKEND_TYPE,
+ MV_INFERENCE_BACKEND_TFLITE);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_TARGET_DEVICE_TYPE,
+ MV_INFERENCE_TARGET_DEVICE_CPU);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_WIDTH,
+ 56);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_HEIGHT,
+ 56);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_CHANNELS,
+ 21);
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_NODE_NAME,
+ inputNodeName);
+
+ mv_engine_config_set_array_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_OUTPUT_NODE_NAMES,
+ outputNodeNames,
+ 2);
+
+ return MEDIA_VISION_ERROR_NONE;
+}
+
+int perform_tflite_hand_detection_AICLite(mv_engine_config_h mv_engine_cfg)
+{
+ if (mv_engine_cfg == NULL) {
+ printf("mv_engine_cfg is null\n");
+ return MEDIA_VISION_ERROR_INVALID_PARAMETER;
+ }
+
+ char *inputNodeName = "input";
+ char *outputNodeNames[2] = {"mobilenetv2/boundingbox", "mobilenetv2/heatmap"};
+
+ //outputTensorData = (void*)calloc(56*56*21, sizeof(float));
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_WEIGHT_FILE_PATH,
+ PE_TFLITE_AICLite_1_WEIGHT_PATH);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_DATA_TYPE,
+ MV_INFERENCE_DATA_FLOAT32);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_MEAN_VALUE,
+ 0.0);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_STD_VALUE,
+ 1.0);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_BACKEND_TYPE,
+ MV_INFERENCE_BACKEND_TFLITE);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_TARGET_DEVICE_TYPE,
+ MV_INFERENCE_TARGET_DEVICE_CPU);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_WIDTH,
+ 224);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_HEIGHT,
+ 224);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_CHANNELS,
+ 3);
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_NODE_NAME,
+ inputNodeName);
+
+ mv_engine_config_set_array_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_OUTPUT_NODE_NAMES,
+ outputNodeNames,
+ 2);
+
+ return MEDIA_VISION_ERROR_NONE;
+}
+
+int perform_tflite_hand_detection_AICLite2(mv_engine_config_h mv_engine_cfg)
+{
+ char *inputNodeName = "input";
+ char *outputNodeNames[2] = {"mobilenetv2/coord_refine", "mobilenetv2/gesture"};
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_WEIGHT_FILE_PATH,
+ PE_TFLITE_AICLite_2_WEIGHT_PATH);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_DATA_TYPE,
+ MV_INFERENCE_DATA_FLOAT32);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_MEAN_VALUE,
+ 0.0);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_STD_VALUE,
+ 1.0);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_BACKEND_TYPE,
+ MV_INFERENCE_BACKEND_TFLITE);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_TARGET_DEVICE_TYPE,
+ MV_INFERENCE_TARGET_DEVICE_CPU);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_WIDTH,
+ 56);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_HEIGHT,
+ 56);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_CHANNELS,
+ 21);
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_NODE_NAME,
+ inputNodeName);
+
+ mv_engine_config_set_array_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_OUTPUT_NODE_NAMES,
+ outputNodeNames,
+ 2);
+
+ return MEDIA_VISION_ERROR_NONE;
+}
+
+int perform_tflite_hand_detection_AICLiteQ(mv_engine_config_h mv_engine_cfg)
+{
+ if (mv_engine_cfg == NULL) {
+ printf("mv_engine_cfg is null\n");
+ return MEDIA_VISION_ERROR_INVALID_PARAMETER;
+ }
+
+ char *inputNodeName = "input";
+ char *outputNodeNames[2] = {"mobilenetv2/boundingbox", "mobilenetv2/heatmap"};
+
+ //outputTensorData = (void*)calloc(56*56*21, sizeof(char));
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_WEIGHT_FILE_PATH,
+ PE_TFLITE_AICLiteQ_1_WEIGHT_PATH);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_DATA_TYPE,
+ MV_INFERENCE_DATA_FLOAT32);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_MEAN_VALUE,
+ 0.0);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_STD_VALUE,
+ 1.0);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_BACKEND_TYPE,
+ MV_INFERENCE_BACKEND_TFLITE);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_TARGET_DEVICE_TYPE,
+ MV_INFERENCE_TARGET_DEVICE_CPU);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_WIDTH,
+ 224);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_HEIGHT,
+ 224);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_CHANNELS,
+ 3);
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_NODE_NAME,
+ inputNodeName);
+
+ mv_engine_config_set_array_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_OUTPUT_NODE_NAMES,
+ outputNodeNames,
+ 2);
+
+ return MEDIA_VISION_ERROR_NONE;
+}
+
+int perform_tflite_hand_detection_AICLiteQ2(mv_engine_config_h mv_engine_cfg)
+{
+ char *inputNodeName = "input";
+ char *outputNodeNames[2] = {"mobilenetv2/coord_refine", "mobilenetv2/gesture"};
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_WEIGHT_FILE_PATH,
+ PE_TFLITE_AICLiteQ_2_WEIGHT_PATH);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_DATA_TYPE,
+ MV_INFERENCE_DATA_FLOAT32);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_MEAN_VALUE,
+ 0.0);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_STD_VALUE,
+ 1.0);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_BACKEND_TYPE,
+ MV_INFERENCE_BACKEND_TFLITE);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_TARGET_DEVICE_TYPE,
+ MV_INFERENCE_TARGET_DEVICE_CPU);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_WIDTH,
+ 56);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_HEIGHT,
+ 56);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_CHANNELS,
+ 21);
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_NODE_NAME,
+ inputNodeName);
+
+ mv_engine_config_set_array_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_OUTPUT_NODE_NAMES,
+ outputNodeNames,
+ 2);
+
+ return MEDIA_VISION_ERROR_NONE;
+}
+
+static void __global(void *data, struct wl_registry *registry,
+ uint32_t name, const char *interface, uint32_t version)
+{
+ struct tizen_surface **tz_surface = NULL;
+
+ if (!data) {
+ printf("NULL data\n");
+ return;
+ }
+
+ tz_surface = (struct tizen_surface **)data;
+
+ if (!interface) {
+ printf("NULL interface\n");
+ return;
+ }
+
+ if (strcmp(interface, "tizen_surface") == 0) {
+ printf("binding tizen surface for wayland\n");
+
+ *tz_surface = wl_registry_bind(registry, name, &tizen_surface_interface, 1);
+ if (*tz_surface == NULL)
+ printf("failed to bind\n");
+
+ printf("done\n");
+ }
+
+ return;
+}
+
+static void __global_remove(void *data, struct wl_registry *wl_registry, uint32_t name)
+{
+ printf("enter\n");
+ return;
+}
+
+static const struct wl_registry_listener _wl_registry_listener = {
+ __global,
+ __global_remove
+};
+
+void __parent_id_getter(void *data, struct tizen_resource *tizen_resource, uint32_t id)
+{
+ if (!data) {
+ printf("NULL data\n");
+ return;
+ }
+
+ *((unsigned int *)data) = id;
+
+ printf("[CLIENT] got parent_id [%u] from server\n", id);
+
+ return;
+}
+
+static const struct tizen_resource_listener _tz_resource_listener = {
+ __parent_id_getter
+};
+
+static void set_overlay(Ecore_Evas *ee)
+{
+ Ecore_Wl2_Window *window = NULL;
+ Ecore_Wl2_Display *e_wl2_display = NULL;
+
+ struct wl_display *display = NULL;
+ struct wl_display *display_wrapper = NULL;
+ struct wl_surface *surface = NULL;
+ struct wl_registry *registry = NULL;
+ struct wl_event_queue *queue = NULL;
+ struct tizen_surface *tz_surface = NULL;
+ struct tizen_resource *tz_resource = NULL;
+
+ window = ecore_evas_wayland2_window_get(ee);
+ if (!window) {
+ printf("failed to get wayland window\n");
+ goto _DONE;
+ }
+
+ /* set video_has flag to a video application window */
+ ecore_wl2_window_video_has(window, EINA_TRUE);
+
+ surface = (struct wl_surface *)ecore_wl2_window_surface_get(window);
+ if (!surface) {
+ printf("failed to get wayland surface\n");
+ goto _DONE;
+ }
+
+ e_wl2_display = ecore_wl2_connected_display_get(NULL);
+ if (!e_wl2_display) {
+ printf("failed to get ecore wl2 display\n");
+ goto _DONE;
+ }
+
+ display = (struct wl_display *)ecore_wl2_display_get(e_wl2_display);
+ if (!display) {
+ printf("failed to get wayland display\n");
+ goto _DONE;
+ }
+
+ display_wrapper = wl_proxy_create_wrapper(display);
+ if (!display_wrapper) {
+ printf("failed to create wl display wrapper\n");
+ }
+
+ queue = wl_display_create_queue(display);
+ if (!queue) {
+ printf("failed to create wl display queue\n");
+ goto _DONE;
+ }
+
+ wl_proxy_set_queue((struct wl_proxy *)display_wrapper, queue);
+
+ registry = wl_display_get_registry(display_wrapper);
+ if (!registry) {
+ printf("failed to get wayland registry\n");
+ goto _DONE;
+ }
+
+ wl_registry_add_listener(registry, &_wl_registry_listener, &tz_surface);
+
+ wl_display_dispatch_queue(display, queue);
+ wl_display_roundtrip_queue(display, queue);
+
+ if (!tz_surface) {
+ printf("failed to get tizen surface\n");
+ goto _DONE;
+ }
+
+ /* Get parent_id which is unique in a entire systemw. */
+ tz_resource = tizen_surface_get_tizen_resource(tz_surface, surface);
+ if (!tz_resource) {
+ printf("failed to get tizen resurce\n");
+ goto _DONE;
+ }
+
+ int parent_id = 0;
+
+ tizen_resource_add_listener(tz_resource, &_tz_resource_listener, &parent_id);
+
+ wl_display_roundtrip_queue(display, queue);
+
+ if (parent_id > 0) {
+ printf("parent id : %u\n", parent_id);
+
+ gst_video_overlay_set_wl_window_wl_surface_id(GST_VIDEO_OVERLAY(sink2), parent_id);
+ gst_element_set_state (pipeline, GST_STATE_PLAYING);
+ } else {
+ printf("failed to get parent id\n");
+ }
+
+_DONE:
+ if (tz_surface) {
+ tizen_surface_destroy(tz_surface);
+ tz_surface = NULL;
+ }
+
+ if (tz_resource) {
+ tizen_resource_destroy(tz_resource);
+ tz_resource = NULL;
+ }
+
+ if (registry) {
+ wl_registry_destroy(registry);
+ registry = NULL;
+ }
+
+ if (queue) {
+ wl_event_queue_destroy(queue);
+ queue = NULL;
+ }
+
+ if (display_wrapper) {
+ wl_proxy_wrapper_destroy(display_wrapper);
+ display_wrapper = NULL;
+ }
+}
+
+static void win_resize_cb (void *data, Evas *e, Evas_Object *obj, void *event_info)
+{
+ int w, h;
+ int wid;
+
+ printf("_canvas_resize_cb\n");
+
+ Evas * _e = evas_object_evas_get(obj);
+ Ecore_Evas *ee = ecore_evas_ecore_evas_get(_e);
+ ecore_evas_geometry_get(ee, NULL, NULL, &w, &h);
+
+ if (!st) {
+ set_overlay(ee);
+ int x, y;
+ elm_win_screen_position_get(obj, &x, &y);
+ printf("x = %d, y = %d\n", x, y);
+ } else
+ st++;
+}
+
+static void cb_new_pad (GstElement *element, GstPad *pad, gpointer data)
+{
+ gchar *name;
+ GstElement *other = data;
+
+ name = gst_pad_get_name (pad);
+ printf ("A new pad %s was created for %s\n", name, gst_element_get_name(element));
+ g_free (name);
+
+ printf ("element %s will be linked to %s\n",
+ gst_element_get_name(element),
+ gst_element_get_name(dscale));
+ gst_element_link(element, dscale);
+}
+
+static int app_create(void *data)
+{
+ appdata *ad = data;
+ Evas_Object *win = NULL;
+
+ // GST
+ g_mutex_init(&pose_mutex);
+
+ signal(SIGINT, int_handler);
+
+ /* initialization */
+ gst_init(NULL, NULL);
+
+ /* mediavision configuration*/
+ IsGestureMode = false;
+ gesture_x = 0;
+ gesture_y = 0;
+ if (hp_mv_engine_cfg == NULL) {
+ mv_create_engine_config(&hp_mv_engine_cfg);
+ }
+
+ if (hp_mv_engine_cfg2 == NULL) {
+ mv_create_engine_config(&hp_mv_engine_cfg2);
+ }
+
+ int err = MEDIA_VISION_ERROR_NONE;
+
+
+ if (ad->modelType == MODEL_TYPE_POSE_CPM) {
+ //err = perform_armnn_human_pose_cpm_configure(hp_mv_engine_cfg);
+ err = perform_tflite_human_pose_cpm_configure(hp_mv_engine_cfg);
+ } else if (ad->modelType == MODEL_TYPE_POSE_HAND_AICLite) {
+ outputTensorData = (void*)calloc(56*56*21, sizeof(float));
+ err = perform_tflite_hand_detection_AIC(hp_mv_engine_cfg);
+
+ err = perform_tflite_hand_detection_AIC2(hp_mv_engine_cfg2);
+ } else if (ad->modelType == MODEL_TYPE_POSE_HAND_AICLite2) {
+ outputTensorData = (void*)calloc(56*56*21, sizeof(float));
+ err = perform_tflite_hand_detection_AICLite(hp_mv_engine_cfg);
+
+ err = perform_tflite_hand_detection_AICLite2(hp_mv_engine_cfg2);
+ } else {
+ outputTensorData = (void*)calloc(56*56*21, sizeof(float));
+ err = perform_tflite_hand_detection_AICLiteQ(hp_mv_engine_cfg);
+
+ err = perform_tflite_hand_detection_AICLiteQ2(hp_mv_engine_cfg2);
+ }
+
+ if (err != MEDIA_VISION_ERROR_NONE) {
+ printf("Error on perform_armnn_human_pose_configure");
+ }
+
+ printf("configuration done\n");
+
+ printf("loading pose model: ");
+ mv_inference_create(&hp_mv_infer);
+
+ mv_inference_configure(hp_mv_infer, hp_mv_engine_cfg);
+
+ clock_t start = clock();
+ mv_inference_prepare(hp_mv_infer);
+ clock_t end = clock();
+ printf("time: %2.3f\n", (double)(end - start)/CLOCKS_PER_SEC);
+
+ /* mediavision source */
+ mv_create_source(&mv_src_p);
+
+ if (ad->modelType >= MODEL_TYPE_POSE_HAND_AICLite) {
+ mv_inference_create(&hp_mv_infer2);
+
+ mv_inference_configure(hp_mv_infer2, hp_mv_engine_cfg2);
+
+ start = clock();
+ mv_inference_prepare(hp_mv_infer2);
+ end = clock();
+ printf("time: %2.3f\n", (double)(end - start)/CLOCKS_PER_SEC);
+
+ mv_create_source(&mv_src_p2);
+ }
+
+ pipeline = gst_pipeline_new("app");
+
+ overlay_state = g_new0 (CairoOverlayState, 1);
+
+ /* create gstreamer elements */
+ if (!ad->filename) {
+ source = gst_element_factory_make("v4l2src", "src");
+ filter = gst_element_factory_make("capsfilter", "filter");
+ } else {
+ source = gst_element_factory_make("filesrc", "src");
+
+ dbin = gst_element_factory_make("decodebin", "dbin");
+ dscale = gst_element_factory_make("videoscale", "dscale");
+ dconv = gst_element_factory_make("videoconvert", "dconv");
+ dsfilter = gst_element_factory_make("capsfilter", "dsfilter");
+ dcfilter = gst_element_factory_make("capsfilter", "dcfilter");
+ }
+
+ if (ad->filename2) {
+ tee2 = gst_element_factory_make("tee", "tee2");
+ enc = gst_element_factory_make("avenc_mpeg4", "enc");
+ muxmp4 = gst_element_factory_make("mp4mux", "muxmp4");
+ fsink2 = gst_element_factory_make("filesink", "fsink2");
+ queue4 = gst_element_factory_make("queue", "queue4");
+ queue5 = gst_element_factory_make("queue", "queue5");
+ encconv = gst_element_factory_make("videoconvert", "encconv");
+ }
+
+
+ tee = gst_element_factory_make("tee", "tee");
+ queue1 = gst_element_factory_make("queue", "queue1");
+ queue2 = gst_element_factory_make("queue", "queue2");
+
+ if (0 /*ad->modelType == MODEL_TYPE_POSE_HAND_AIC*/) {
+ queue3 = gst_element_factory_make("queue", "queue3");
+ }
+
+ // queue1 - videoscale - capsfilter -viedoeconvert - capsfilter - videorate - capsfilter -fakesink
+ vscale = gst_element_factory_make("videoscale", "scale");
+ vsfilter = gst_element_factory_make("capsfilter", "vsfilter");
+ vconv = gst_element_factory_make("videoconvert", "convert");
+ vcfilter = gst_element_factory_make("capsfilter", "vcfilter");
+ vrate = gst_element_factory_make("videorate", "rate");
+ vrfilter = gst_element_factory_make("capsfilter", "vrfilter");
+ vrsink = gst_element_factory_make("fakesink", "vrsink");
+
+ // queue2 - videoconvert - cairooveray - tizenwlsink
+ oconv = gst_element_factory_make("videoconvert", "oconv");
+ coverlay = gst_element_factory_make("cairooverlay", "coverlay");
+ sink = gst_element_factory_make("fpsdisplaysink", "vsink");
+ sink2 = gst_element_factory_make("tizenwlsink", "vsink2");
+
+ // after detection, crop using video crop
+ // queue3 - videocrop - videoscale -capsfilter - videoconvert - capsfilter -fakesink
+
+ if (0/*ad->modelType == MODEL_TYPE_POSE_HAND_AIC*/) {
+ vcrop = gst_element_factory_make("videocrop", "crop");
+ vcrscale = gst_element_factory_make("videoscale", "crscale");
+ vcrsfilter = gst_element_factory_make("capsfilter", "vcrsfilter");
+ vcrsconv = gst_element_factory_make("videoconvert", "vcrsconvert");
+ vcrscfilter = gst_element_factory_make("capsfilter", "vcrscfilter");
+ vcrssink = gst_element_factory_make("fakesink", "vcrssink");
+ }
+
+ if (!pipeline || !source ||
+ !tee || !queue1 || !vscale || !vsfilter || !vconv || !vcfilter ||
+ !vrate || !vrfilter || !vrsink ||
+ !queue2 || !oconv || !coverlay || !sink || !sink2) {
+ printf(TEXT_RED "One element(queue1 or queue2) might be not created. Exiting.\n" TEXT_RESET);
+ return -1;
+ }
+
+ if (0 /*ad->modelType == MODEL_TYPE_POSE_HAND_AIC && (!pipeline || !queue3
+ || !vcrop || !vcrscale || !vcrsfilter || !vcrsconv || !vcrscfilter || !vcrssink*/) {
+ printf(TEXT_RED "One element(queue3) might be not created. Exiting.\n" TEXT_RESET);
+ return -1;
+ }
+
+ if (!ad->filename) {
+ if (!filter) {
+ printf(TEXT_RED "One element might be not created. Existing.\n" TEXT_RESET);
+ return -1;
+ }
+ g_object_set(G_OBJECT(filter), "caps", gst_caps_from_string("video/x-raw, format=YUY2, width=640, height=480"), NULL);
+ } else {
+ if (!dbin || !dscale || !dconv) {
+ printf(TEXT_RED "One element might be not created. Exiting.\n" TEXT_RESET);
+ return -1;
+ }
+ g_object_set(G_OBJECT(dsfilter), "caps", gst_caps_from_string("video/x-raw, width=640, height=480"), NULL);
+ g_object_set(G_OBJECT(dcfilter), "caps", gst_caps_from_string("video/x-raw, format=YUY2, width=640, height=480"), NULL);
+ g_signal_connect (dbin, "pad-added", G_CALLBACK (cb_new_pad), NULL);
+ }
+
+ if (ad->filename2) {
+ if (!tee2 || !enc || !muxmp4 || !queue4 || !queue5 || !fsink2 || !encconv) {
+ printf(TEXT_RED "One element might be not created. Exiting.\n" TEXT_RESET);
+ printf("%p, %p, %p, %p, %p, %p, %p\n", tee2, enc, muxmp4, queue4, queue5, fsink2, encconv);
+ return -1;
+ }
+ g_object_set(G_OBJECT(fsink2), "location", ad->filename2, NULL);
+
+ }
+
+ /* set up the pipeline */
+ //g_signal_connect (coverlay, "draw", G_CALLBACK (draw_overlay), overlay_state);
+ if (ad->modelType == MODEL_TYPE_POSE_CPM) {
+ g_signal_connect (coverlay, "draw", G_CALLBACK (draw_overlay_pose), overlay_state);
+ } else {
+ g_signal_connect (coverlay, "draw", G_CALLBACK (draw_overlay_hand), overlay_state);
+ }
+
+ g_signal_connect (coverlay, "caps-changed", G_CALLBACK (prepare_overlay), overlay_state);
+
+ if (!ad->filename) {
+ g_object_set(G_OBJECT(source), "device", "/dev/video252", NULL); // 252
+ } else {
+ g_object_set(G_OBJECT(source), "location", ad->filename, NULL);
+ //g_object_set(G_OBJECT(source), "num-buffers", ad->numbuffers, NULL);
+ }
+
+ //g_object_set(G_OBJECT(sink), "text-overlay", FALSE, NULL);
+ g_object_set(G_OBJECT(sink2), "use-tbm", FALSE, NULL);
+ g_object_set(G_OBJECT(sink2), "sync", FALSE, NULL);
+ g_object_set(G_OBJECT(sink), "video-sink", sink2, NULL);
+ g_object_set(G_OBJECT(sink), "sync", FALSE, NULL);
+
+ if (ad->modelType == MODEL_TYPE_POSE_HAND_AICLite ||
+ ad->modelType == MODEL_TYPE_POSE_HAND_AICLite2 ||
+ ad->modelType == MODEL_TYPE_POSE_HAND_AICLite2Q) {
+ g_object_set(G_OBJECT(vsfilter), "caps", gst_caps_from_string("video/x-raw, width=224, height=224"), NULL);
+ poseCropSize = 224;
+ } else {
+ g_object_set(G_OBJECT(vsfilter), "caps", gst_caps_from_string("video/x-raw, width=192, height=192"), NULL);
+ poseCropSize = 192;
+ }
+
+ g_object_set(G_OBJECT(vcfilter), "caps", gst_caps_from_string("video/x-raw, format=RGB"), NULL);
+ g_object_set(G_OBJECT(vrfilter), "caps", gst_caps_from_string("video/x-raw, framerate=15/1"), NULL);
+ g_object_set(G_OBJECT(queue2), "max-size-buffers", 1, NULL);
+ //g_object_set(G_OBJECT(vrate), "drop-only", TRUE, NULL);
+
+ //g_object_set(G_OBJECT(queue2), "leaky", 2, NULL);
+#if 0
+ g_object_set(G_OBJECT(queue3), "max-size-buffers", 0, NULL);
+ g_object_set(G_OBJECT(queue3), "max-size-time", 0, NULL);
+ g_object_set(G_OBJECT(queue3), "max-size-bytes", 0, NULL);
+#endif
+ g_object_set(G_OBJECT(queue3), "max-size-buffers", 1, NULL);
+ //g_object_set(G_OBJECT(queue3), "leaky", 2, NULL);
+
+ // here to be continue
+ printf("vrsink signal-handoffs\n");
+ g_object_set(G_OBJECT(vrsink), "signal-handoffs", TRUE, NULL);
+
+ if (ad->modelType == MODEL_TYPE_POSE_CPM) {
+ handler_p = g_signal_connect (vrsink, "handoff", G_CALLBACK(_pose_est_handoff), outputTensorData);
+ } else {
+ handler_p = g_signal_connect (vrsink, "handoff", G_CALLBACK(_hand_est_handoff), outputTensorData);
+ }
+
+ g_object_set(G_OBJECT(vrsink), "sync", FALSE, NULL);
+
+
+ g_object_set(G_OBJECT(vcrssink), "sync", FALSE, NULL);
+
+
+ /* add a message handler */
+ bus = gst_pipeline_get_bus (GST_PIPELINE(pipeline));
+ bus_watch_id = gst_bus_add_watch(bus, bus_call, NULL);
+ gst_object_unref(bus);
+
+ /* add elements into the pipeline */
+ gst_bin_add_many(GST_BIN(pipeline),
+ source,
+ tee, queue1, vscale, vsfilter, vconv, vcfilter,
+ vrate, vrfilter, vrsink,
+ queue2, oconv, coverlay, sink,
+ NULL);
+
+ if (0 /*ad->modelType == MODEL_TYPE_POSE_HAND_AIC*/) {
+ gst_bin_add_many(GST_BIN(pipeline),
+ queue3, vcrop, vcrscale, vcrsfilter, vcrsconv, vcrscfilter, vcrssink,
+ NULL);
+ gst_element_link_many (tee, queue3, vcrop, vcrscale, vcrsfilter, vcrsconv, vcrscfilter, vcrssink, NULL);
+ }
+
+ /* link elements */
+ if (!ad->filename) {
+ gst_bin_add(GST_BIN(pipeline), filter);
+ gst_element_link_many(source, filter, tee, NULL);
+ } else {
+ gst_bin_add_many(GST_BIN(pipeline), dbin, dscale, dconv, dsfilter, dcfilter, NULL);
+ gst_element_link_many(source, dbin, NULL);
+ gst_element_link_many(dscale, dsfilter, dconv, dcfilter, tee, NULL);
+ }
+
+ if (!ad->filename2) {
+ // display
+ gst_element_link_many (tee, queue2, oconv, coverlay, /*toverlay,*/ sink, NULL);
+ // pose
+ gst_element_link_many (tee, queue1, vrate, vrfilter, vconv, vcfilter, vscale, vsfilter, vrsink, NULL);
+ } else {
+
+ gst_bin_add_many(GST_BIN(pipeline), tee2, enc, muxmp4, fsink2, queue4, queue5, encconv);
+
+ // display
+ gst_element_link_many (tee, queue2, oconv, coverlay, tee2, NULL);
+ gst_element_link_many (tee2, queue4, sink, NULL);
+ gst_element_link_many (tee2, queue5, encconv, enc, muxmp4, fsink2, NULL);
+
+ // pose
+ gst_element_link_many (tee, queue1, vrate, vrfilter, vconv, vcfilter, vscale, vsfilter, vrsink, NULL);
+ }
+
+ /* set the pipeline state to "playing" state */
+ //gst_element_set_state(pipeline, GST_STATE_PLAYING);
+
+ /* loop */
+ humanSkeleton.IsDetected = false;
+ humanSkeleton.isPrevPose = false;
+ humanSkeleton.label = -1;
+ printf(TEXT_GREEN "Running.....\n" TEXT_RESET);
+ // GST_END
+#if 1
+ /* use gl backend */
+ elm_config_accel_preference_set("opengl");
+
+ /* create window */
+ win = elm_win_add(NULL, PACKAGE, ELM_WIN_SPLASH );
+ //win = elm_win_add(NULL, PACKAGE, ELM_WIN_BASIC);
+ if (win) {
+ elm_win_title_set(win, PACKAGE);
+ elm_win_borderless_set(win, EINA_TRUE);
+ elm_win_autodel_set(win, EINA_TRUE);
+ elm_win_alpha_set(win, EINA_FALSE);
+ evas_object_show(win);
+ }
+ elm_win_layer_set(win, 9);
+ elm_win_prop_focus_skip_set(win, EINA_TRUE);
+
+ ad->win = win;
+ g_win_id = win;
+ selected_win_id = g_win_id;
+
+ Evas_Object *bg = elm_bg_add(win);
+ elm_win_resize_object_add(win, bg);
+ evas_object_size_hint_min_set(bg, WIDTH, HEIGHT);
+ evas_object_size_hint_max_set(bg, WIDTH, HEIGHT);
+ evas_object_show(bg);
+
+ elm_win_activate(win);
+
+
+ evas_object_event_callback_add(win, EVAS_CALLBACK_RESIZE, win_resize_cb, NULL);
+#else
+ gst_element_set_state (pipeline, GST_STATE_PLAYING);
+#endif
+ return 0;
+}
+
+
+static int app_terminate(void *data)
+{
+ appdata *ad = data;
+ int i = 0;
+
+ /* out of loop */
+ printf(TEXT_GREEN "Stopping.....\n" TEXT_RESET);
+ gst_element_set_state(pipeline, GST_STATE_NULL);
+
+ printf(TEXT_GREEN "pipe unref.....\n" TEXT_RESET);
+ gst_object_unref(GST_OBJECT(pipeline));
+
+ printf(TEXT_GREEN "remove bus_watch id.....\n" TEXT_RESET);
+ g_source_remove(bus_watch_id);
+
+ if (overlay_state != NULL) {
+ printf(TEXT_GREEN "g_free overlay.....\n" TEXT_RESET);
+ g_free(overlay_state);
+ }
+
+ g_mutex_clear(&pose_mutex);
+ printf(TEXT_GREEN "done.....\n" TEXT_RESET);
+
+ if (g_win_id) {
+ evas_object_del(g_win_id);
+ g_win_id = NULL;
+ }
+ ad->win = NULL;
+ selected_win_id = NULL;
+
+ return 0;
+}
+struct appcore_ops ops = {
+ .create = app_create,
+ .terminate = app_terminate,
+};
+
+int main (int argc, char *argv[])
+{
+ memset(&ad, 0x0, sizeof(appdata));
+ ops.data = &ad;
+
+ if (argc >= 6) {
+ ad.filename = g_strdup(argv[5]);
+ printf("launch with file source (%s)\n", ad.filename);
+ if (argc > 6) {
+ ad.filename2 = g_strdup(argv[6]);
+ ad.numbuffers = -1;
+ printf("records output(%s)\n", ad.filename2);
+ }
+ if (argc > 7) {
+ ad.numbuffers = atoi(argv[7]);
+ }
+ } else {
+ printf("launch with camera source\n");
+ }
+
+ if (argc < 2) {
+ printf("usage: mv_stream_infer model [NeckThresVal, ArmThresVal, LegThresVal, [filename]]");
+ printf("model: 0(CPM), 1(AIC Hand), 2(AIC Lite Hand), 3(AIC Lite Q Hand)\n");
+ return -1;
+ }
+
+ ad.modelType = atoi(argv[1]);
+ if (ad.modelType < 0 || ad.modelType > 3) {
+ printf("not supported model type [%d]\n", ad.modelType);
+ return -1;
+ }
+
+ if (ad.modelType != MODEL_TYPE_POSE_HAND_AICLite &&
+ ad.modelType != MODEL_TYPE_POSE_HAND_AICLite2 &&
+ ad.modelType != MODEL_TYPE_POSE_HAND_AICLite2Q) {
+ thValNeck = (float)atoi(argv[2])/100.f;
+ thValArm = (float)atoi(argv[3])/100.f;
+ thValLeg = (float)atoi(argv[4])/100.f;
+
+ poseRoi.point.x = 0;
+ poseRoi.point.y = 0;
+ poseRoi.width = 0;
+ poseRoi.height = 0;
+ } else {
+ if (argc > 2) {
+ ad.filename2 = g_strdup(argv[2]);
+ }
+ poseRoi.point.x = 0;
+ poseRoi.point.y = 0;
+ poseRoi.width = 0;
+ poseRoi.height = 0;
+ }
+
+ return appcore_efl_main(PACKAGE, &argc, &argv, &ops);
+}
--- /dev/null
+cmake_minimum_required(VERSION 2.6)
+
+set_property(DIRECTORY APPEND PROPERTY COMPILE_DEFINITIONS_DEBUG _DEBUG)
+
+if(NOT SKIP_WARNINGS)
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Werror")
+endif()
+
+set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/${LIB_INSTALL_DIR})
+set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/${LIB_INSTALL_DIR})
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
+
+set(TFLITE_TEST mv_handpose_browser_test_suite)
+set(ARMNN_TEST mv_handpose_browser_armnn_test_suite)
+
+include_directories(${PROJECT_SOURCE_DIR})
+include_directories(${MV_CAPI_MEDIA_VISION_INC_DIR})
+include_directories(${INC_IMAGE_HELPER})
+include_directories(${INC_VIDEO_HELPER})
+include_directories(${INC_TS_COMMON})
+
+file(GLOB MV_STREAMINFER_TEST_SUITE_INC_LIST "${PROJECT_SOURCE_DIR}/*.h")
+file(GLOB MV_STREAMINFER_TEST_SUITE_SRC_LIST "${PROJECT_SOURCE_DIR}/*.c")
+
+find_package(PkgConfig REQUIRED)
+pkg_check_modules(GLIB_PKG glib-2.0)
+
+if (NOT GLIB_PKG_FOUND)
+ message(SEND_ERROR "Failed to find glib")
+ return()
+else()
+ include_directories(${GLIB_PKG_INCLUDE_DIRS})
+endif()
+
+
+SET(dependents "gstreamer-1.0 gstreamer-app-1.0 gstreamer-video-1.0 cairo elementary ecore-wl2 appcore-efl capi-ui-efl-util")
+
+INCLUDE(FindPkgConfig)
+pkg_check_modules(${TFLITE_TEST} REQUIRED ${dependents})
+pkg_check_modules(${ARMNN_TEST} REQUIRED ${dependents})
+FOREACH(flag ${${TFLITE_TEST}_CFLAGS})
+ SET(EXTRA_CFLAGS "${EXTRA_CFLAGS} ${flag}")
+ENDFOREACH(flag)
+FOREACH(flag ${${ARMNN_TEST}_CFLAGS})
+ SET(EXTRA_CFLAGS "${EXTRA_CFLAGS} ${flag}")
+ENDFOREACH(flag)
+
+SET(CMAKE_C_FLAGS "-I./include -I./include/headers ${CMAKE_C_FLAGS} ${EXTRA_CFLAGS} -fPIC -Wall -DEFL_BETA_API_SUPPORT=1")
+SET(CMAKE_C_FLAGS_DEBUG "-O0 -g")
+
+add_executable(${TFLITE_TEST}
+ ${PROJECT_SOURCE_DIR}/stream_handpose_browser/stream_handpose_browser.c
+ ${MV_STREAMINFER_TEST_SUITE_INC_LIST}
+ ${MV_CAPI_MEDIA_VISION_INC_LIST})
+
+target_link_libraries(${TFLITE_TEST} ${MV_INFERENCE_LIB_NAME}
+ gstreamer-1.0
+ glib-2.0
+ capi-system-info
+ dlog
+ mv_image_helper
+ mv_video_helper
+ mv_testsuite_common
+ cairo
+ m
+ ${${TFLITE_TEST}_LIBRARIES}
+ )
+
+install(TARGETS ${TFLITE_TEST} DESTINATION ${testbin_dir})
+
+add_executable(${ARMNN_TEST}
+ ${PROJECT_SOURCE_DIR}/stream_handpose_browser/stream_handpose_browser_armnn.c
+ ${MV_STREAMINFER_TEST_SUITE_INC_LIST}
+ ${MV_CAPI_MEDIA_VISION_INC_LIST})
+
+target_link_libraries(${ARMNN_TEST} ${MV_INFERENCE_LIB_NAME}
+ gstreamer-1.0
+ glib-2.0
+ capi-system-info
+ dlog
+ mv_image_helper
+ mv_video_helper
+ mv_testsuite_common
+ cairo
+ m
+ ${${ARMNN_TEST}_LIBRARIES}
+ )
+
+install(TARGETS ${ARMNN_TEST} DESTINATION ${testbin_dir})
--- /dev/null
+/**
+ * Copyright (c) 2015 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define _USE_MATH_DEFINES
+
+#include <fcntl.h>
+#include <mv_common.h>
+#include <mv_inference.h>
+
+#include <mv_testsuite_common.h>
+
+#include <image_helper.h>
+#include <mv_video_helper.h>
+
+#include <mv_log_cfg.h>
+
+#include <math.h>
+#include <ctype.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <dirent.h>
+#include <string.h>
+#include <limits.h>
+#include <time.h>
+#define MAX(a, b) \
+({ __typeof__ (a) _a = (a); \
+__typeof__ (b) _b = (b); \
+_a > _b ? _a : _b; })
+
+#define MIN(a,b) \
+({ __typeof__ (a) _a = (a); \
+__typeof__ (b) _b = (b); \
+_a < _b ? _a : _b; })
+
+#include <glib-2.0/glib.h>
+#include <gst/gst.h>
+#include <gst/video/video.h>
+#include <cairo.h>
+#include <cairo-gobject.h>
+
+#include <Elementary.h>
+#include <appcore-efl.h>
+#include <Ecore.h>
+#include <Ecore_Evas.h>
+#include <Ecore_Wl2.h>
+#include <tizen-extension-client-protocol.h>
+#include <efl_util.h>
+
+#include <gst/gst.h>
+#include <gst/video/videooverlay.h>
+#include <unistd.h>
+#include <time.h>
+
+#define WIDTH (640)
+#define HEIGHT (480)
+
+#ifdef PACKAGE
+#undef PACKAGE
+#endif
+#define PACKAGE "test"
+
+static char* gHandGestureLabel[] = { "None", "One", "Two", "Three", "Four", "Five",
+ "Six", "Eight", "SmallHeart", "OK", "Like", "DisLike",
+ "Fist", "Rock", "Love"};
+static int st = 0;
+static Evas_Object *g_eo = NULL;
+static Evas_Object *icon = NULL;
+
+/* for video display */
+static Evas_Object *g_win_id;
+static Evas_Object *selected_win_id;
+
+int recording_fd;
+int fd, ret;
+FILE *fp;
+
+typedef enum {
+ MODEL_TYPE_POSE_CPM = 0,
+ MODEL_TYPE_POSE_HAND_AICLite, // posenet lite 224
+ MODEL_TYPE_POSE_HAND_AICLite2, // posenet 0709 f
+ MODEL_TYPE_POSE_HAND_AICLite2Q // posenet 0709 q
+};
+
+typedef struct {
+ gchar *filename;
+ gchar *filename2;
+ int numbuffers;
+ int modelType;
+ Evas_Object *win;
+ Evas_Object *layout_main; /* layout widget based on EDJ */
+ /* add more variables here */
+
+} appdata;
+
+static mv_rectangle_s poseRoi;
+
+static appdata ad;
+static GstBus *bus;
+static guint bus_watch_id;
+int sending = FALSE;
+static efl_util_inputgen_h my_cursor = NULL;
+
+#define FILE_PATH_SIZE 1024
+
+// pose estimation
+#define PE_TFLITE_CPM_WEIGHT_PATH "/usr/share/capi-media-vision/models/PLD/tflite/pld-tflite-001.tflite"
+#define PE_POSE_LABEL_PATH "/usr/share/capi-media-vision/models/PLD/tflite/pose-label.txt"
+
+
+#define PE_TFLITE_AIC_1_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE/tflite/posenet1_lite_224.tflite"
+#define PE_TFLITE_AIC_2_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE/tflite/posenet2_lite_224.tflite"
+
+
+#define PE_TFLITE_AICLite_1_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE/tflite/posenet1_0709_f.tflite"
+#define PE_TFLITE_AICLite_2_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE/tflite/posenet2_0709_f.tflite"
+
+#define PE_TFLITE_AICLiteQ_1_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE/tflite/posenet1_0709_dq.tflite"
+#define PE_TFLITE_AICLiteQ_2_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE/tflite/posenet2_0709_f.tflite"
+
+static float thValNeck = 0.3f; // 15%
+static float thValArm = 0.1f; // 10 %
+static float thValLeg = 0.2f; // 5%
+
+typedef struct _rect {
+ int left;
+ int top;
+ int right;
+ int bottom;
+ int type;
+ bool updated;
+ bool cr_updated;
+} rect;
+
+
+typedef struct _humanSkeleton {
+ mv_point_s pose[21/*14*/];
+ mv_point_s prevPose[21/*14*/];
+ mv_rectangle_s loc;
+ mv_rectangle_s prevLoc;
+ mv_rectangle_s handRoi;
+ float scores[21/*14*/];
+ bool isPrevPose;
+ bool updated; // detection is find and pose is also good. then update drawing
+ bool locUpdated; // track previous detection
+ bool IsDetected; // detection
+ int label;
+} HumanSkeleton;
+
+typedef struct
+{
+ gboolean valid;
+ GstVideoInfo vinfo;
+} CairoOverlayState;
+
+typedef struct
+{
+ GstBuffer *buffer;
+ gpointer user_data;
+} HandOffData;
+
+//gchar *gesturenames;
+
+static HandOffData hdata_p;
+
+static GMutex pose_mutex;
+static guint old_timeout = 0;
+static guint nFrames = 0;
+
+// Gstreamer
+GstElement *pipeline, *source, *filter, *toverlay, *sink, *sink2;
+GstElement *tee, *vscale, *vsfilter, *vconv, *vcfilter;
+GstElement *vrate, *vrfilter, *fsink, *vrsink;
+GstElement *queue1, *queue2, *queue3;
+GstElement *oconv, *coverlay;
+GstElement *vcrop, *vcrscale, *vcrsfilter, *vcrsconv, *vcrscfilter, *vcrssink;
+CairoOverlayState *overlay_state;
+
+GstElement *dbin, *dscale, *dconv;
+GstElement *dsfilter, *dcfilter;
+
+GstElement *tee2, *enc, *muxmp4, *fsink2, *queue4, *queue5, *encconv;
+
+//static HandSkeleton handSkeleton;
+static HumanSkeleton humanSkeleton;
+gulong handler_p;
+GList *line_list = NULL;
+
+mv_source_h mv_src_p;
+mv_source_h mv_src_p2;
+
+// Human pose
+mv_engine_config_h hp_mv_engine_cfg;
+mv_inference_h hp_mv_infer;
+
+mv_engine_config_h hp_mv_engine_cfg2;
+mv_inference_h hp_mv_infer2;
+
+static void * outputTensorData;
+
+FILE *fp;
+
+static bool IsGestureMode;
+
+static int poseCropSize = 0;
+
+#define IMAGE_SIZE_WIDTH 640
+#define IMAGE_SIZE_HEIGHT 480
+
+#define NANO_PER_SEC ((__clock_t) 1000000000)
+#define NANO_PER_MILLI ((__clock_t) 1000000)
+#define MILLI_PER_SEC ((__clock_t) 1000)
+
+#define VOLUME_INTERVAL 10
+
+void send_mouse_event_efl(int x, int y){
+ efl_util_input_generate_pointer(my_cursor, 1, EFL_UTIL_INPUT_POINTER_MOVE, x, y);
+}
+struct timespec diff(struct timespec start, struct timespec end)
+{
+ struct timespec temp;
+ if ((end.tv_nsec - start.tv_nsec) < 0) {
+ temp.tv_sec = end.tv_sec - start.tv_sec - 1;
+ temp.tv_nsec = NANO_PER_SEC + end.tv_nsec - start.tv_nsec;
+ }
+ else {
+ temp.tv_sec = end.tv_sec - start.tv_sec;
+ temp.tv_nsec = end.tv_nsec - start.tv_nsec;
+ }
+ return temp;
+}
+
+unsigned long gettotalmillisec(const struct timespec time)
+{
+ return time.tv_sec * MILLI_PER_SEC + time.tv_nsec / NANO_PER_MILLI;
+}
+
+
+void int_handler(int sig)
+{
+ char c;
+
+ signal(sig, SIG_IGN);
+ while ((getchar()) != '\n');
+
+ printf(TEXT_YELLOW "Do you want to quit? [y/n]\n" TEXT_RESET);
+ c = getchar();
+ if (c == 'y' || c == "Y") {
+
+ g_signal_handler_disconnect(vrsink, handler_p);
+#if 0
+ g_signal_handler_disconnect(vcrssink, handler_gp);
+#endif
+
+ gst_element_send_event(pipeline, gst_event_new_eos());
+
+ sleep(4);
+
+
+ if (mv_src_p)
+ mv_destroy_source(mv_src_p);
+
+ if (hp_mv_infer)
+ mv_inference_destroy(hp_mv_infer);
+
+ if (hp_mv_engine_cfg)
+ mv_destroy_engine_config(hp_mv_engine_cfg);
+
+ if (mv_src_p2)
+ mv_destroy_source(mv_src_p2);
+
+ if (hp_mv_infer2)
+ mv_inference_destroy(hp_mv_infer2);
+
+ if (hp_mv_engine_cfg)
+ mv_destroy_engine_config(hp_mv_engine_cfg2);
+
+ if (outputTensorData) {
+ free(outputTensorData);
+ outputTensorData = NULL;
+ }
+ printf(TEXT_YELLOW "exit..\n" TEXT_RESET);
+ signal(SIGINT, SIG_DFL);
+ exit(0);
+ } else {
+ printf("no");
+ signal(SIGINT, int_handler);
+ }
+
+ getchar(); // Get new line character
+}
+
+void _hand_pose_cb (
+ mv_source_h source,
+ int number_of_pose_estimation,
+ mv_inference_pose_s *locations,
+ int label,
+ void *user_data)
+{
+ //printf("%d landmarks\n", number_of_pose_estimation);
+ int width, height;
+ mv_source_get_height(source, &width);
+ mv_source_get_height(source, &height);
+ for (int n = 0; n < number_of_pose_estimation; ++n) {
+ humanSkeleton.pose[n].x = (int)(640.f*(float)locations->landmarks[n].point.x/(float)width);
+ humanSkeleton.pose[n].y = (int)(480.f*(float)locations->landmarks[n].point.y/(float)height);
+ humanSkeleton.scores[n] = 1.0f; /* confidences[n];*/
+
+ //printf("(%d,%d): %f\n", humanSkeleton.pose[n].x, humanSkeleton.pose[n].y, confidences[n]);
+ //printf("(%d,%d)\n", humanSkeleton.pose[n].x, humanSkeleton.pose[n].y);
+ }
+ humanSkeleton.label = label;
+ humanSkeleton.IsDetected = true;
+}
+
+static void _hand_detected_cb (
+ mv_source_h source,
+ const int number_of_hands,
+ const float *confidences,
+ const mv_rectangle_s *locations,
+ void *user_data) //user_data can be mv_source?
+{
+
+#if 0
+ if (0 /*confidences[1] < thValNeck*/) {
+ printf("lost pose\n");
+ humanSkeleton.IsDetected = false;
+ humanSkeleton.isPrevPose = false;
+ return;
+ }
+
+ printf("%d landmarks, %d crop\n", number_of_landmarks, poseCropSize);
+ for (int n = 0; n < number_of_landmarks; ++n) {
+
+ humanSkeleton.pose[n].x = (int)((float)(locations[n].x + poseRoi.point.x) / (float)poseCropSize * 640.f);
+ humanSkeleton.pose[n].y = (int)((float)(locations[n].y + poseRoi.point.y) / (float)poseCropSize * 480.f);
+ humanSkeleton.scores[n] = 1.0f; /* confidences[n];*/
+
+ //printf("(%d,%d): %f\n", humanSkeleton.pose[n].x, humanSkeleton.pose[n].y, confidences[n]);
+ printf("(%d,%d)\n", humanSkeleton.pose[n].x, humanSkeleton.pose[n].y);
+ }
+ humanSkeleton.IsDetected = true;
+#else
+
+ if (number_of_hands <= 0) {
+ humanSkeleton.IsDetected = false;
+ humanSkeleton.label = -1;
+ return;
+ }
+
+ //seg fault?
+ struct timespec s_tspec;
+ struct timespec e_tspec;
+
+ clock_gettime(CLOCK_MONOTONIC, &s_tspec);
+
+ mv_source_clear(mv_src_p2);
+ mv_source_fill_by_tensor_buffer(mv_src_p2, user_data,
+ MV_INFERENCE_DATA_FLOAT32,
+ 56 * 56 * 21 * sizeof(float),
+ 56, 56, 21, 3);
+
+ clock_gettime(CLOCK_MONOTONIC, &e_tspec);
+
+ struct timespec diffspec = diff(s_tspec, e_tspec);
+ unsigned long timeDiff = gettotalmillisec(diffspec);
+ //printf("memcpy time: %lu(ms)\n", timeDiff);
+
+ clock_gettime(CLOCK_MONOTONIC, &s_tspec);
+
+ mv_inference_pose_landmark_detect(mv_src_p2, hp_mv_infer2, NULL, _hand_pose_cb, NULL);
+
+ clock_gettime(CLOCK_MONOTONIC, &e_tspec);
+
+ diffspec = diff(s_tspec, e_tspec);
+ timeDiff = gettotalmillisec(diffspec);
+ //printf("pose_estimation time: %lu(ms)\n", timeDiff);
+ //humanSkeleton.IsDetected = true;
+#endif
+ return;
+}
+
+
+void _human_pose_cb (
+ mv_source_h source,
+ int number_of_pose_estimation,
+ mv_inference_pose_s *locations,
+ int label,
+ void *user_data)
+{
+ //printf("%d landmarks\n", number_of_pose_estimation);
+
+ float score = 0.0f;
+ int cnt = 0;
+ for (int n = 0; n < number_of_pose_estimation; ++n) {
+ humanSkeleton.pose[n].x = locations->landmarks[n].point.x;
+ humanSkeleton.pose[n].y = locations->landmarks[n].point.y;
+ humanSkeleton.scores[n] = locations->landmarks[n].score;
+ if (score >= 0.0 ) {
+ score += humanSkeleton.scores[n];
+ cnt++;
+ }
+
+ //printf("(%d,%d): %f\n", humanSkeleton.pose[n].x, humanSkeleton.pose[n].y, confidences[n]);
+ //printf("(%d,%d)\n", humanSkeleton.pose[n].x, humanSkeleton.pose[n].y);
+ }
+ if(cnt == 0){
+ printf("count is 0 !!! \n");
+ }else{
+ score /= (float)cnt;
+ }
+
+ if (score >= 0.5) {
+ humanSkeleton.IsDetected = true;
+ } else {
+ humanSkeleton.IsDetected = false;
+ humanSkeleton.label = -1;
+ }
+}
+
+static gboolean
+run_pose (void *user_data)
+{
+ HandOffData *udata = (HandOffData *)user_data;
+ if (!GST_IS_BUFFER(udata->buffer))
+ return FALSE;
+
+ GstMapInfo map;
+
+ gst_buffer_map(udata->buffer, &map, GST_MAP_READ);
+
+ mv_source_clear(mv_src_p);
+
+ mv_source_fill_by_buffer(mv_src_p, map.data, 192*192*3, 192, 192, MEDIA_VISION_COLORSPACE_RGB888);
+
+ gst_buffer_unmap(udata->buffer, &map);
+
+ struct timespec s_tspec;
+ struct timespec e_tspec;
+
+ clock_gettime(CLOCK_MONOTONIC, &s_tspec);
+
+ mv_inference_pose_landmark_detect(mv_src_p, hp_mv_infer, NULL, _human_pose_cb, NULL);
+
+ clock_gettime(CLOCK_MONOTONIC, &e_tspec);
+ struct timespec diffspec = diff(s_tspec, e_tspec);
+ unsigned long timeDiff = gettotalmillisec(diffspec);
+ //printf("detect + pose time: %lu(ms)\n", timeDiff);
+
+ return FALSE;
+
+}
+
+static gboolean
+run_hand (void *user_data)
+{
+ HandOffData *udata = (HandOffData *)user_data;
+ if (!GST_IS_BUFFER(udata->buffer))
+ return FALSE;
+
+ GstMapInfo map;
+
+ gst_buffer_map(udata->buffer, &map, GST_MAP_READ);
+
+ mv_source_clear(mv_src_p);
+
+ mv_source_fill_by_buffer(mv_src_p, map.data, 224*224*3, 224, 224, MEDIA_VISION_COLORSPACE_RGB888);
+
+ gst_buffer_unmap(udata->buffer, &map);
+
+
+ struct timespec s_tspec;
+ struct timespec e_tspec;
+
+ void * outputTensorBuffer = (void*)udata->user_data;
+
+ clock_gettime(CLOCK_MONOTONIC, &s_tspec);
+
+ // invoke tflite -> _hand_detected_cb -> memcpy output -> invoke tflite -> _pose_cb
+ mv_inference_hand_detect(mv_src_p, hp_mv_infer, _hand_detected_cb, outputTensorBuffer);
+
+ clock_gettime(CLOCK_MONOTONIC, &e_tspec);
+ struct timespec diffspec = diff(s_tspec, e_tspec);
+ unsigned long timeDiff = gettotalmillisec(diffspec);
+ //printf("detect + pose time: %lu(ms)\n", timeDiff);
+
+ return FALSE;
+
+}
+
+static void
+_pose_est_handoff(GstElement *object, GstBuffer *buffer, GstPad *pad, gpointer user_data)
+{
+
+ nFrames++;
+ hdata_p.buffer = buffer;
+ hdata_p.user_data = user_data;
+
+#if 0
+ if (nFrames % 15 == 0) {
+ g_mutex_lock(&pose_mutex);
+ g_idle_add (run_pose, &hdata_p);
+ g_mutex_unlock(&pose_mutex);
+ }
+#else
+ g_mutex_lock(&pose_mutex);
+ g_idle_add (run_pose, &hdata_p);
+ g_mutex_unlock(&pose_mutex);
+#endif
+}
+
+static void
+_hand_est_handoff(GstElement *object, GstBuffer *buffer, GstPad *pad, gpointer user_data)
+{
+
+ nFrames++;
+ hdata_p.buffer = buffer;
+ hdata_p.user_data = user_data;
+
+#if 0
+ if (nFrames % 15 == 0) {
+ g_mutex_lock(&pose_mutex);
+ g_idle_add (run_hand, &hdata_p);
+ g_mutex_unlock(&pose_mutex);
+ }
+#else
+ g_mutex_lock(&pose_mutex);
+ g_idle_add (run_hand, &hdata_p);
+ g_mutex_unlock(&pose_mutex);
+#endif
+}
+
+
+static void
+prepare_overlay (GstElement * overlay, GstCaps * caps, gpointer user_data)
+{
+ CairoOverlayState *state = (CairoOverlayState *) user_data;
+
+ state->valid = gst_video_info_from_caps (&state->vinfo, caps);
+}
+
+static void
+draw_overlay_pose (GstElement * overlay, cairo_t * cr, guint64 timestamp,
+ guint64 duration, gpointer user_data)
+{
+ CairoOverlayState *s = (CairoOverlayState *) user_data;
+
+ if (!s->valid) {
+ printf("not ready draw_overlay");
+ return;
+ }
+ //printf("draw_overlay_pose");
+ cairo_set_source_rgba(cr, 0.1, 0.9, 0.0, 0.7);
+ cairo_set_line_width(cr, 2.0);
+
+
+ if (!humanSkeleton.IsDetected)
+ return;
+
+ //
+ if (humanSkeleton.isPrevPose == false) {
+ humanSkeleton.prevPose[1] = humanSkeleton.pose[1];
+ // head - neck
+ if (humanSkeleton.scores[0] >= thValNeck ) {
+ humanSkeleton.prevPose[0] = humanSkeleton.pose[0];
+ }
+
+ // right arm
+ if (humanSkeleton.scores[2] >= thValArm) {
+ // neck - right shoulder
+ humanSkeleton.prevPose[2] = humanSkeleton.pose[2];
+ if (humanSkeleton.scores[3] >= thValArm) {
+ // right shoulder - right elbow
+ humanSkeleton.prevPose[3] = humanSkeleton.pose[3];
+ if (humanSkeleton.scores[4] >= thValArm) {
+ // right elbow - right wrist
+ humanSkeleton.prevPose[4] = humanSkeleton.pose[4];
+ }
+ }
+ }
+
+ // left arm
+ if (humanSkeleton.scores[5] >= thValArm) {
+ // neck - right shoulder
+ humanSkeleton.prevPose[5] = humanSkeleton.pose[5];
+ if (humanSkeleton.scores[6] >= thValArm) {
+ // right shoulder - right elbow
+ humanSkeleton.prevPose[6] = humanSkeleton.pose[6];
+ if (humanSkeleton.scores[7] >= thValArm) {
+ // right elbow - right wrist
+ humanSkeleton.prevPose[7] = humanSkeleton.pose[7];
+ }
+ }
+ }
+
+ // right leg
+ if (humanSkeleton.scores[8] >= thValLeg) {
+ // neck - right shoulder
+ humanSkeleton.prevPose[8] = humanSkeleton.pose[8];
+ if (humanSkeleton.scores[9] >= thValLeg) {
+ // right shoulder - right elbow
+ humanSkeleton.prevPose[9] = humanSkeleton.pose[9];
+ if (humanSkeleton.scores[10] >= thValLeg) {
+ // right elbow - right wrist
+ humanSkeleton.prevPose[10] = humanSkeleton.pose[10];
+ }
+ }
+ }
+
+ // left leg
+ if (humanSkeleton.scores[11] >= thValLeg) {
+ // neck - right shoulder
+ humanSkeleton.prevPose[11] = humanSkeleton.pose[11];
+ if (humanSkeleton.scores[12] >= thValLeg) {
+ // right shoulder - right elbow
+ humanSkeleton.prevPose[12] = humanSkeleton.pose[12];
+ if (humanSkeleton.scores[13] >= thValLeg) {
+ // right elbow - right wrist
+ humanSkeleton.prevPose[13] = humanSkeleton.pose[13];
+ }
+ }
+ }
+ humanSkeleton.isPrevPose = true;
+ } else {
+ // weighted sum of pose and prevPose
+ // method1: fixed weights (pose : prevPose = 0.7 : 0.3)
+ float poseWeight = 0.7f;
+ float prevPoseWeight = 0.3f;
+ humanSkeleton.prevPose[1].x = (poseWeight * humanSkeleton.pose[1].x +
+ prevPoseWeight * humanSkeleton.prevPose[1].x);
+ humanSkeleton.prevPose[1].y = (poseWeight * humanSkeleton.pose[1].y +
+ prevPoseWeight * humanSkeleton.prevPose[1].y);
+ // head - neck
+ if (humanSkeleton.scores[0] >= thValNeck ) {
+ humanSkeleton.prevPose[0].x = (poseWeight * humanSkeleton.pose[0].x +
+ prevPoseWeight * humanSkeleton.prevPose[0].x);
+ humanSkeleton.prevPose[0].y = (poseWeight * humanSkeleton.pose[0].y +
+ prevPoseWeight * humanSkeleton.prevPose[0].y);
+ }
+
+ // right arm
+ if (humanSkeleton.scores[2] >= thValArm) {
+ // neck - right shoulder
+ humanSkeleton.prevPose[2].x = (poseWeight * humanSkeleton.pose[2].x +
+ prevPoseWeight * humanSkeleton.prevPose[2].x);
+ humanSkeleton.prevPose[2].y = (poseWeight * humanSkeleton.pose[2].y +
+ prevPoseWeight * humanSkeleton.prevPose[2].y);
+ if (humanSkeleton.scores[3] >= thValArm) {
+ // right shoulder - right elbow
+ humanSkeleton.prevPose[3].x = (poseWeight * humanSkeleton.pose[3].x +
+ prevPoseWeight * humanSkeleton.prevPose[3].x);
+ humanSkeleton.prevPose[3].y = (poseWeight * humanSkeleton.pose[3].y +
+ prevPoseWeight * humanSkeleton.prevPose[3].y);
+ if (humanSkeleton.scores[4] >= thValArm) {
+ // right elbow - right wrist
+ humanSkeleton.prevPose[4].x = (poseWeight * humanSkeleton.pose[4].x +
+ prevPoseWeight * humanSkeleton.prevPose[4].x);
+ humanSkeleton.prevPose[4].y = (poseWeight * humanSkeleton.pose[4].y +
+ prevPoseWeight * humanSkeleton.prevPose[4].y);
+ }
+ }
+ }
+
+ // left arm
+ if (humanSkeleton.scores[5] >= thValArm) {
+ // neck - right shoulder
+ humanSkeleton.prevPose[5].x = (poseWeight * humanSkeleton.pose[5].x +
+ prevPoseWeight * humanSkeleton.prevPose[5].x);
+ humanSkeleton.prevPose[5].y = (poseWeight * humanSkeleton.pose[5].y +
+ prevPoseWeight * humanSkeleton.prevPose[5].y);
+ if (humanSkeleton.scores[6] >= thValArm) {
+ // right shoulder - right elbow
+ humanSkeleton.prevPose[6].x = (poseWeight * humanSkeleton.pose[6].x +
+ prevPoseWeight * humanSkeleton.prevPose[6].x);
+ humanSkeleton.prevPose[6].y = (poseWeight * humanSkeleton.pose[6].y +
+ prevPoseWeight * humanSkeleton.prevPose[6].y);
+ if (humanSkeleton.scores[7] >= thValArm) {
+ // right elbow - right wrist
+ humanSkeleton.prevPose[7].x = (poseWeight * humanSkeleton.pose[7].x +
+ prevPoseWeight * humanSkeleton.prevPose[7].x);
+ humanSkeleton.prevPose[7].y = (poseWeight * humanSkeleton.pose[7].y +
+ prevPoseWeight * humanSkeleton.prevPose[7].y);
+ }
+ }
+ }
+
+ // right leg
+ if (humanSkeleton.scores[8] >= thValLeg) {
+ // neck - right shoulder
+ humanSkeleton.prevPose[8].x = (poseWeight * humanSkeleton.pose[8].x +
+ prevPoseWeight * humanSkeleton.prevPose[8].x);
+ humanSkeleton.prevPose[8].y = (poseWeight * humanSkeleton.pose[8].y +
+ prevPoseWeight * humanSkeleton.prevPose[8].y);
+ if (humanSkeleton.scores[9] >= thValLeg) {
+ // right shoulder - right elbow
+ humanSkeleton.prevPose[9].x = (poseWeight * humanSkeleton.pose[9].x +
+ prevPoseWeight * humanSkeleton.prevPose[9].x);
+ humanSkeleton.prevPose[9].y = (poseWeight * humanSkeleton.pose[9].y +
+ prevPoseWeight * humanSkeleton.prevPose[9].y);
+ if (humanSkeleton.scores[10] >= thValLeg) {
+ // right elbow - right wrist
+ humanSkeleton.prevPose[10].x = (poseWeight * humanSkeleton.pose[10].x +
+ prevPoseWeight * humanSkeleton.prevPose[10].x);
+ humanSkeleton.prevPose[10].y = (poseWeight * humanSkeleton.pose[10].y +
+ prevPoseWeight * humanSkeleton.prevPose[10].y);
+ }
+ }
+ }
+
+ // left leg
+ if (humanSkeleton.scores[11] >= thValLeg) {
+ // neck - right shoulder
+ humanSkeleton.prevPose[11].x = (poseWeight * humanSkeleton.pose[11].x +
+ prevPoseWeight * humanSkeleton.prevPose[11].x);
+ humanSkeleton.prevPose[11].y = (poseWeight * humanSkeleton.pose[11].y +
+ prevPoseWeight * humanSkeleton.prevPose[11].y);
+ if (humanSkeleton.scores[12] >= thValLeg) {
+ // right shoulder - right elbow
+ humanSkeleton.prevPose[12].x = (poseWeight * humanSkeleton.pose[12].x +
+ prevPoseWeight * humanSkeleton.prevPose[12].x);
+ humanSkeleton.prevPose[12].y = (poseWeight * humanSkeleton.pose[12].y +
+ prevPoseWeight * humanSkeleton.prevPose[12].y);
+ if (humanSkeleton.scores[13] >= thValLeg) {
+ // right elbow - right wrist
+ humanSkeleton.prevPose[13].x = (poseWeight * humanSkeleton.pose[13].x +
+ prevPoseWeight * humanSkeleton.prevPose[13].x);
+ humanSkeleton.prevPose[13].y = (poseWeight * humanSkeleton.pose[13].y +
+ prevPoseWeight * humanSkeleton.prevPose[13].y);
+ }
+ }
+ }
+ }
+
+ //
+ //draw..
+ // head - neck
+ if (humanSkeleton.scores[0] >= thValNeck ) {
+ cairo_move_to(cr, humanSkeleton.prevPose[0].x, humanSkeleton.prevPose[0].y);
+ cairo_line_to(cr, humanSkeleton.prevPose[1].x, humanSkeleton.prevPose[1].y);
+ }
+
+ // right arm
+ cairo_move_to(cr, humanSkeleton.prevPose[1].x, humanSkeleton.prevPose[1].y);
+ if (humanSkeleton.scores[2] >= thValArm) {
+ // neck - right shoulder
+ cairo_line_to(cr, humanSkeleton.prevPose[2].x, humanSkeleton.prevPose[2].y);
+ if (humanSkeleton.scores[3] >= thValArm) {
+ // right shoulder - right elbow
+ cairo_line_to(cr, humanSkeleton.prevPose[3].x, humanSkeleton.prevPose[3].y);
+ if (humanSkeleton.scores[4] >= thValArm) {
+ // right elbow - right wrist
+ cairo_line_to(cr, humanSkeleton.prevPose[4].x, humanSkeleton.prevPose[4].y);
+ }
+ }
+ }
+ cairo_stroke(cr);
+
+ // left arm
+ cairo_move_to(cr, humanSkeleton.prevPose[1].x, humanSkeleton.prevPose[1].y);
+ if (humanSkeleton.scores[5] >= thValArm) {
+ // neck - right shoulder
+ cairo_line_to(cr, humanSkeleton.prevPose[5].x, humanSkeleton.prevPose[5].y);
+ if (humanSkeleton.scores[6] >= thValArm) {
+ // right shoulder - right elbow
+ cairo_line_to(cr, humanSkeleton.prevPose[6].x, humanSkeleton.prevPose[6].y);
+ if (humanSkeleton.scores[7] >= thValArm) {
+ // right elbow - right wrist
+ cairo_line_to(cr, humanSkeleton.prevPose[7].x, humanSkeleton.prevPose[7].y);
+ }
+ }
+ }
+ cairo_stroke(cr);
+
+
+ // right leg
+ cairo_move_to(cr, humanSkeleton.prevPose[1].x, humanSkeleton.prevPose[1].y);
+ if (humanSkeleton.scores[8] >= thValLeg) {
+ // neck - right shoulder
+ cairo_line_to(cr, humanSkeleton.prevPose[8].x, humanSkeleton.prevPose[8].y);
+ if (humanSkeleton.scores[9] >= thValLeg) {
+ // right shoulder - right elbow
+ cairo_line_to(cr, humanSkeleton.prevPose[9].x, humanSkeleton.prevPose[9].y);
+ if (humanSkeleton.scores[10] >= thValLeg) {
+ // right elbow - right wrist
+ cairo_line_to(cr, humanSkeleton.prevPose[10].x, humanSkeleton.prevPose[10].y);
+ }
+ }
+ }
+ cairo_stroke(cr);
+
+ // left leg
+ cairo_move_to(cr, humanSkeleton.prevPose[1].x, humanSkeleton.prevPose[1].y);
+ if (humanSkeleton.scores[11] >= thValLeg) {
+ // neck - right shoulder
+ cairo_line_to(cr, humanSkeleton.prevPose[11].x, humanSkeleton.prevPose[11].y);
+ if (humanSkeleton.scores[12] >= thValLeg) {
+ // right shoulder - right elbow
+ cairo_line_to(cr, humanSkeleton.prevPose[12].x, humanSkeleton.prevPose[12].y);
+ if (humanSkeleton.scores[13] >= thValLeg) {
+ // right elbow - right wrist
+ cairo_line_to(cr, humanSkeleton.prevPose[13].x, humanSkeleton.prevPose[13].y);
+ }
+ }
+ }
+ cairo_stroke(cr);
+}
+
+static void
+draw_overlay_hand (GstElement * overlay, cairo_t * cr, guint64 timestamp,
+ guint64 duration, gpointer user_data)
+{
+ CairoOverlayState *s = (CairoOverlayState *) user_data;
+
+ if (!s->valid) {
+ printf("not ready draw_overlay");
+ return;
+ }
+
+ //printf("draw_overlay_hand");
+ cairo_set_source_rgba(cr, 0.0, 0.0, 1.0, 0.7);
+ cairo_set_line_width(cr, 2.0);
+
+
+ if (!humanSkeleton.IsDetected)
+ return;
+
+ /*printf("humanSkeleton.label [%d] \n", humanSkeleton.label);
+ if (humanSkeleton.label >= 0 && humanSkeleton.label <= 14) {
+ cairo_select_font_face(cr, "sans-serif", CAIRO_FONT_SLANT_NORMAL, CAIRO_FONT_WEIGHT_BOLD);
+ cairo_set_font_size(cr,15);
+ cairo_move_to(cr, 320.0, 48.0);
+ cairo_show_text(cr, gHandGestureLabel[humanSkeleton.label]);
+ }*/
+#define MOUSE_CONTROLLER
+#ifdef MOUSE_CONTROLLER
+ static int move_x = 0;
+ static int move_y = 0;
+ move_x = (WIDTH - humanSkeleton.pose[9].x)*3;
+ move_y = humanSkeleton.pose[9].y*2;
+
+ send_mouse_event_efl(move_x, move_y);
+
+#elif REMOTE_CONTROLLER
+ int fd = -1;
+ //int signal = 8;
+
+ static int skip = 0;
+
+ if (skip++%10!=0)
+ return;
+
+ fprintf(stderr, ">>> send start \n");
+ if (!sending && humanSkeleton.label == 2) {
+ sending = TRUE;
+ //send_signal_play(&signal);
+ //const char *path = "/dev/input/event2 --type 1 --code 105 --value 10";
+ send_event(0x004, 0x004, 0x101);
+ send_event(0x001, 0x069, 0x001);
+ send_event(0x000, 0x000, 0x000);
+ send_event(0x004, 0x004, 0x101);
+ send_event(0x000, 0x000, 0x000);
+ for (int i=0; i<5; i++) {
+ send_event(0x001, 0x069, 0x002);
+ send_event(0x000, 0x000, 0x001);
+ send_event(0x004, 0x004, 0x101);
+ send_event(0x001, 0x069, 0x001);
+ }
+ send_event(0x001, 0x069, 0x000);
+ send_event(0x000, 0x000, 0x000);
+ } else if (!sending && humanSkeleton.label == 3) {
+ sending = TRUE;
+ send_event(0x004, 0x004, 0x098);
+ send_event(0x001, 0x06a, 0x001);
+ send_event(0x000, 0x000, 0x000);
+ send_event(0x004, 0x004, 0x098);
+ send_event(0x000, 0x000, 0x000);
+ for (int i=0; i<5; i++) {
+ send_event(0x001, 0x06a, 0x002);
+ send_event(0x000, 0x000, 0x001);
+ send_event(0x004, 0x004, 0x098);
+ send_event(0x001, 0x06a, 0x001);
+ }
+ send_event(0x001, 0x06a, 0x000);
+ send_event(0x000, 0x000, 0x000);
+ } else if (!sending && humanSkeleton.label == 4) {
+ sending = TRUE;
+ send_event(0x004, 0x004, 0x096);
+ send_event(0x001, 0x067, 0x001);
+ send_event(0x000, 0x000, 0x000);
+ send_event(0x004, 0x004, 0x096);
+ send_event(0x000, 0x000, 0x000);
+ for (int i=0; i<5; i++) {
+ send_event(0x001, 0x067, 0x002);
+ send_event(0x000, 0x000, 0x001);
+ send_event(0x004, 0x004, 0x096);
+ send_event(0x001, 0x067, 0x001);
+ }
+ send_event(0x001, 0x067, 0x000);
+ send_event(0x000, 0x000, 0x000);
+
+ } else if (!sending && humanSkeleton.label == 5) {
+ sending = TRUE;
+ send_event(0x004, 0x004, 0x097);
+ send_event(0x001, 0x06c, 0x001);
+ send_event(0x000, 0x000, 0x000);
+ send_event(0x004, 0x004, 0x097);
+ send_event(0x000, 0x000, 0x000);
+ for (int i=0; i<5; i++) {
+ send_event(0x001, 0x06c, 0x002);
+ send_event(0x000, 0x000, 0x001);
+ send_event(0x004, 0x004, 0x097);
+ send_event(0x001, 0x06c, 0x001);
+ }
+ send_event(0x001, 0x06c, 0x000);
+ send_event(0x000, 0x000, 0x000);
+ }
+ fprintf(stderr, ">>> send end \n");
+ sending = FALSE;
+
+ cairo_set_source_rgba(cr, 0.1, 0.9, 0.0, 0.7);
+ cairo_set_line_width(cr, 2.0);
+
+ //printf("gHandGestureLabel[humanSkeleton.label] : [%d], x: %d y: %d",gHandGestureLabel[humanSkeleton.label], humanSkeleton.pose[0].x, humanSkeleton.pose[0].y);
+
+
+ // thumb - red
+ cairo_set_source_rgba (cr, 0.9, 0.1, 0.0, 0.7);
+ cairo_move_to(cr, humanSkeleton.pose[0].x, humanSkeleton.pose[0].y);
+ for (int k = 1 ; k < 5; ++k) {
+ cairo_line_to(cr, humanSkeleton.pose[k].x, humanSkeleton.pose[k].y);
+ }
+ cairo_stroke(cr);
+
+ // fore - red
+ cairo_set_source_rgba (cr, 0.9, 0.1, 0.0, 0.7);
+ cairo_move_to(cr, humanSkeleton.pose[0].x, humanSkeleton.pose[0].y);
+ for (int k = 5 ; k < 9; ++k) {
+ cairo_line_to(cr, humanSkeleton.pose[k].x, humanSkeleton.pose[k].y);
+ }
+ cairo_stroke(cr);
+
+ // middle - grean
+ cairo_set_source_rgba (cr, 0.1, 0.9, 0.0, 0.7);
+ cairo_move_to(cr, humanSkeleton.pose[0].x, humanSkeleton.pose[0].y);
+ for (int k = 9 ; k < 13; ++k) {
+ cairo_line_to(cr, humanSkeleton.pose[k].x, humanSkeleton.pose[k].y);
+ }
+ cairo_stroke(cr);
+
+ // ring - blue
+ cairo_set_source_rgba (cr, 0.1, 0.0, 0.9, 0.7);
+ cairo_move_to(cr, humanSkeleton.pose[0].x, humanSkeleton.pose[0].y);
+ for (int k = 13 ; k < 17; ++k) {
+ cairo_line_to(cr, humanSkeleton.pose[k].x, humanSkeleton.pose[k].y);
+ }
+ cairo_stroke(cr);
+
+ // little - purple
+ cairo_set_source_rgba (cr, 0.5, 0.0, 0.5, 0.7);
+ cairo_move_to(cr, humanSkeleton.pose[0].x, humanSkeleton.pose[0].y);
+ for (int k = 17 ; k < 21; ++k) {
+ cairo_line_to(cr, humanSkeleton.pose[k].x, humanSkeleton.pose[k].y);
+ }
+ cairo_stroke(cr);
+#endif
+
+}
+
+static gboolean bus_call (GstBus *bus, GstMessage *msg, gpointer data)
+{
+
+ switch (GST_MESSAGE_TYPE (msg)) {
+
+ case GST_MESSAGE_EOS:
+ printf ("End of stream\n");
+ break;
+
+ case GST_MESSAGE_ERROR: {
+ gchar *debug;
+ GError *error;
+
+ gst_message_parse_error (msg, &error, &debug);
+ g_free (debug);
+
+ printf ("Error: %s\n", error->message);
+ g_error_free (error);
+
+ break;
+ }
+ default:
+ break;
+ }
+
+ return TRUE;
+}
+
+int perform_armnn_human_pose_cpm_configure(mv_engine_config_h mv_engine_cfg)
+{
+ if (mv_engine_cfg == NULL) {
+ printf("mv_engine_cfg is null\n");
+ return MEDIA_VISION_ERROR_INVALID_PARAMETER;
+ }
+
+ char *inputNodeName = "image";
+ char *outputNodeName[1] = {"Convolutional_Pose_Machine/stage_5_out"};
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_WEIGHT_FILE_PATH,
+ PE_TFLITE_CPM_WEIGHT_PATH);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_DATA_TYPE,
+ MV_INFERENCE_DATA_FLOAT32);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_MEAN_VALUE,
+ 0.0);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_STD_VALUE,
+ 1.0);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_BACKEND_TYPE,
+ MV_INFERENCE_BACKEND_ARMNN);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_TARGET_TYPE,
+ MV_INFERENCE_TARGET_GPU);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_WIDTH,
+ 192);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_HEIGHT,
+ 192);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_CHANNELS,
+ 3);
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_NODE_NAME,
+ inputNodeName);
+
+ mv_engine_config_set_array_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_OUTPUT_NODE_NAMES,
+ outputNodeName,
+ 1);
+
+ return MEDIA_VISION_ERROR_NONE;
+}
+
+int perform_tflite_human_pose_cpm_configure(mv_engine_config_h mv_engine_cfg)
+{
+ if (mv_engine_cfg == NULL) {
+ printf("mv_engine_cfg is null\n");
+ return MEDIA_VISION_ERROR_INVALID_PARAMETER;
+ }
+
+ char *inputNodeName = "image";
+ char *outputNodeName[1] = {"Convolutional_Pose_Machine/stage_5_out"};
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_WEIGHT_FILE_PATH,
+ PE_TFLITE_CPM_WEIGHT_PATH);
+
+ mv_engine_config_set_string_attribute(
+ mv_engine_cfg,
+ MV_INFERENCE_MODEL_USER_FILE_PATH,
+ PE_POSE_LABEL_PATH);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_DATA_TYPE,
+ MV_INFERENCE_DATA_FLOAT32);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_MEAN_VALUE,
+ 0.0);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_STD_VALUE,
+ 1.0);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_BACKEND_TYPE,
+ MV_INFERENCE_BACKEND_TFLITE);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_TARGET_TYPE,
+ MV_INFERENCE_TARGET_CPU);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_WIDTH,
+ 192);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_HEIGHT,
+ 192);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_CHANNELS,
+ 3);
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_NODE_NAME,
+ inputNodeName);
+
+ mv_engine_config_set_array_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_OUTPUT_NODE_NAMES,
+ outputNodeName,
+ 1);
+
+ return MEDIA_VISION_ERROR_NONE;
+}
+
+int perform_tflite_hand_detection_AIC(mv_engine_config_h mv_engine_cfg)
+{
+ if (mv_engine_cfg == NULL) {
+ printf("mv_engine_cfg is null\n");
+ return MEDIA_VISION_ERROR_INVALID_PARAMETER;
+ }
+
+ char *inputNodeName = "input";
+ char *outputNodeNames[2] = {"mobilenetv2/boundingbox2", "mobilenetv2/heatmap"};
+
+ //outputTensorData = (void*)calloc(56*56*21, sizeof(float));
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_WEIGHT_FILE_PATH,
+ PE_TFLITE_AIC_1_WEIGHT_PATH);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_DATA_TYPE,
+ MV_INFERENCE_DATA_FLOAT32);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_MEAN_VALUE,
+ 0.0);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_STD_VALUE,
+ 1.0);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_BACKEND_TYPE,
+ MV_INFERENCE_BACKEND_TFLITE);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_TARGET_DEVICE_TYPE,
+ MV_INFERENCE_TARGET_DEVICE_CPU);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_WIDTH,
+ 224);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_HEIGHT,
+ 224);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_CHANNELS,
+ 3);
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_NODE_NAME,
+ inputNodeName);
+
+ mv_engine_config_set_array_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_OUTPUT_NODE_NAMES,
+ outputNodeNames,
+ 2);
+
+ return MEDIA_VISION_ERROR_NONE;
+}
+
+int perform_tflite_hand_detection_AIC2(mv_engine_config_h mv_engine_cfg)
+{
+ char *inputNodeName = "input";
+ char *outputNodeNames[2] = {"mobilenetv2/coord_refine", "mobilenetv2/gesture"};
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_WEIGHT_FILE_PATH,
+ PE_TFLITE_AIC_2_WEIGHT_PATH);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_DATA_TYPE,
+ MV_INFERENCE_DATA_FLOAT32);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_MEAN_VALUE,
+ 0.0);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_STD_VALUE,
+ 1.0);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_BACKEND_TYPE,
+ MV_INFERENCE_BACKEND_TFLITE);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_TARGET_DEVICE_TYPE,
+ MV_INFERENCE_TARGET_DEVICE_CPU);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_WIDTH,
+ 56);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_HEIGHT,
+ 56);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_CHANNELS,
+ 21);
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_NODE_NAME,
+ inputNodeName);
+
+ mv_engine_config_set_array_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_OUTPUT_NODE_NAMES,
+ outputNodeNames,
+ 2);
+
+ return MEDIA_VISION_ERROR_NONE;
+}
+
+int perform_tflite_hand_detection_AICLite(mv_engine_config_h mv_engine_cfg)
+{
+ if (mv_engine_cfg == NULL) {
+ printf("mv_engine_cfg is null\n");
+ return MEDIA_VISION_ERROR_INVALID_PARAMETER;
+ }
+
+ char *inputNodeName = "input";
+ char *outputNodeNames[2] = {"mobilenetv2/boundingbox", "mobilenetv2/heatmap"};
+
+ //outputTensorData = (void*)calloc(56*56*21, sizeof(float));
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_WEIGHT_FILE_PATH,
+ PE_TFLITE_AICLite_1_WEIGHT_PATH);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_DATA_TYPE,
+ MV_INFERENCE_DATA_FLOAT32);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_MEAN_VALUE,
+ 0.0);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_STD_VALUE,
+ 1.0);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_BACKEND_TYPE,
+ MV_INFERENCE_BACKEND_TFLITE);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_TARGET_DEVICE_TYPE,
+ MV_INFERENCE_TARGET_DEVICE_CPU);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_WIDTH,
+ 224);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_HEIGHT,
+ 224);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_CHANNELS,
+ 3);
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_NODE_NAME,
+ inputNodeName);
+
+ mv_engine_config_set_array_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_OUTPUT_NODE_NAMES,
+ outputNodeNames,
+ 2);
+
+ return MEDIA_VISION_ERROR_NONE;
+}
+
+int perform_tflite_hand_detection_AICLite2(mv_engine_config_h mv_engine_cfg)
+{
+ char *inputNodeName = "input";
+ char *outputNodeNames[2] = {"mobilenetv2/coord_refine", "mobilenetv2/gesture"};
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_WEIGHT_FILE_PATH,
+ PE_TFLITE_AICLite_2_WEIGHT_PATH);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_DATA_TYPE,
+ MV_INFERENCE_DATA_FLOAT32);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_MEAN_VALUE,
+ 0.0);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_STD_VALUE,
+ 1.0);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_BACKEND_TYPE,
+ MV_INFERENCE_BACKEND_TFLITE);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_TARGET_DEVICE_TYPE,
+ MV_INFERENCE_TARGET_DEVICE_CPU);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_WIDTH,
+ 56);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_HEIGHT,
+ 56);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_CHANNELS,
+ 21);
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_NODE_NAME,
+ inputNodeName);
+
+ mv_engine_config_set_array_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_OUTPUT_NODE_NAMES,
+ outputNodeNames,
+ 2);
+
+ return MEDIA_VISION_ERROR_NONE;
+}
+
+int perform_tflite_hand_detection_AICLiteQ(mv_engine_config_h mv_engine_cfg)
+{
+ if (mv_engine_cfg == NULL) {
+ printf("mv_engine_cfg is null\n");
+ return MEDIA_VISION_ERROR_INVALID_PARAMETER;
+ }
+
+ char *inputNodeName = "input";
+ char *outputNodeNames[2] = {"mobilenetv2/boundingbox", "mobilenetv2/heatmap"};
+
+ //outputTensorData = (void*)calloc(56*56*21, sizeof(char));
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_WEIGHT_FILE_PATH,
+ PE_TFLITE_AICLiteQ_1_WEIGHT_PATH);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_DATA_TYPE,
+ MV_INFERENCE_DATA_FLOAT32);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_MEAN_VALUE,
+ 0.0);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_STD_VALUE,
+ 1.0);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_BACKEND_TYPE,
+ MV_INFERENCE_BACKEND_TFLITE);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_TARGET_DEVICE_TYPE,
+ MV_INFERENCE_TARGET_DEVICE_CPU);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_WIDTH,
+ 224);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_HEIGHT,
+ 224);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_CHANNELS,
+ 3);
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_NODE_NAME,
+ inputNodeName);
+
+ mv_engine_config_set_array_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_OUTPUT_NODE_NAMES,
+ outputNodeNames,
+ 2);
+
+ return MEDIA_VISION_ERROR_NONE;
+}
+
+int perform_tflite_hand_detection_AICLiteQ2(mv_engine_config_h mv_engine_cfg)
+{
+ char *inputNodeName = "input";
+ char *outputNodeNames[2] = {"mobilenetv2/coord_refine", "mobilenetv2/gesture"};
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_WEIGHT_FILE_PATH,
+ PE_TFLITE_AICLiteQ_2_WEIGHT_PATH);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_DATA_TYPE,
+ MV_INFERENCE_DATA_FLOAT32);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_MEAN_VALUE,
+ 0.0);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_STD_VALUE,
+ 1.0);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_BACKEND_TYPE,
+ MV_INFERENCE_BACKEND_TFLITE);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_TARGET_DEVICE_TYPE,
+ MV_INFERENCE_TARGET_DEVICE_CPU);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_WIDTH,
+ 56);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_HEIGHT,
+ 56);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_CHANNELS,
+ 21);
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_NODE_NAME,
+ inputNodeName);
+
+ mv_engine_config_set_array_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_OUTPUT_NODE_NAMES,
+ outputNodeNames,
+ 2);
+
+ return MEDIA_VISION_ERROR_NONE;
+}
+
+static void __global(void *data, struct wl_registry *registry,
+ uint32_t name, const char *interface, uint32_t version)
+{
+ struct tizen_surface **tz_surface = NULL;
+
+ if (!data) {
+ printf("NULL data\n");
+ return;
+ }
+
+ tz_surface = (struct tizen_surface **)data;
+
+ if (!interface) {
+ printf("NULL interface\n");
+ return;
+ }
+
+ if (strcmp(interface, "tizen_surface") == 0) {
+ printf("binding tizen surface for wayland\n");
+
+ *tz_surface = wl_registry_bind(registry, name, &tizen_surface_interface, 1);
+ if (*tz_surface == NULL)
+ printf("failed to bind\n");
+
+ printf("done\n");
+ }
+
+ return;
+}
+
+static void __global_remove(void *data, struct wl_registry *wl_registry, uint32_t name)
+{
+ printf("enter\n");
+ return;
+}
+
+static const struct wl_registry_listener _wl_registry_listener = {
+ __global,
+ __global_remove
+};
+
+void __parent_id_getter(void *data, struct tizen_resource *tizen_resource, uint32_t id)
+{
+ if (!data) {
+ printf("NULL data\n");
+ return;
+ }
+
+ *((unsigned int *)data) = id;
+
+ printf("[CLIENT] got parent_id [%u] from server\n", id);
+
+ return;
+}
+
+static const struct tizen_resource_listener _tz_resource_listener = {
+ __parent_id_getter
+};
+
+static void set_overlay(Ecore_Evas *ee)
+{
+ Ecore_Wl2_Window *window = NULL;
+ Ecore_Wl2_Display *e_wl2_display = NULL;
+
+ struct wl_display *display = NULL;
+ struct wl_display *display_wrapper = NULL;
+ struct wl_surface *surface = NULL;
+ struct wl_registry *registry = NULL;
+ struct wl_event_queue *queue = NULL;
+ struct tizen_surface *tz_surface = NULL;
+ struct tizen_resource *tz_resource = NULL;
+
+ window = ecore_evas_wayland2_window_get(ee);
+ if (!window) {
+ printf("failed to get wayland window\n");
+ goto _DONE;
+ }
+
+ /* set video_has flag to a video application window */
+ ecore_wl2_window_video_has(window, EINA_TRUE);
+
+ surface = (struct wl_surface *)ecore_wl2_window_surface_get(window);
+ if (!surface) {
+ printf("failed to get wayland surface\n");
+ goto _DONE;
+ }
+
+ e_wl2_display = ecore_wl2_connected_display_get(NULL);
+ if (!e_wl2_display) {
+ printf("failed to get ecore wl2 display\n");
+ goto _DONE;
+ }
+
+ display = (struct wl_display *)ecore_wl2_display_get(e_wl2_display);
+ if (!display) {
+ printf("failed to get wayland display\n");
+ goto _DONE;
+ }
+
+ display_wrapper = wl_proxy_create_wrapper(display);
+ if (!display_wrapper) {
+ printf("failed to create wl display wrapper\n");
+ }
+
+ queue = wl_display_create_queue(display);
+ if (!queue) {
+ printf("failed to create wl display queue\n");
+ goto _DONE;
+ }
+
+ wl_proxy_set_queue((struct wl_proxy *)display_wrapper, queue);
+
+ registry = wl_display_get_registry(display_wrapper);
+ if (!registry) {
+ printf("failed to get wayland registry\n");
+ goto _DONE;
+ }
+
+ wl_registry_add_listener(registry, &_wl_registry_listener, &tz_surface);
+
+ wl_display_dispatch_queue(display, queue);
+ wl_display_roundtrip_queue(display, queue);
+
+ if (!tz_surface) {
+ printf("failed to get tizen surface\n");
+ goto _DONE;
+ }
+
+ /* Get parent_id which is unique in a entire systemw. */
+ tz_resource = tizen_surface_get_tizen_resource(tz_surface, surface);
+ if (!tz_resource) {
+ printf("failed to get tizen resurce\n");
+ goto _DONE;
+ }
+
+ int parent_id = 0;
+
+ tizen_resource_add_listener(tz_resource, &_tz_resource_listener, &parent_id);
+
+ wl_display_roundtrip_queue(display, queue);
+
+ if (parent_id > 0) {
+ printf("parent id : %u\n", parent_id);
+
+ gst_video_overlay_set_wl_window_wl_surface_id(GST_VIDEO_OVERLAY(sink2), parent_id);
+ gst_element_set_state (pipeline, GST_STATE_PLAYING);
+ } else {
+ printf("failed to get parent id\n");
+ }
+
+_DONE:
+ if (tz_surface) {
+ tizen_surface_destroy(tz_surface);
+ tz_surface = NULL;
+ }
+
+ if (tz_resource) {
+ tizen_resource_destroy(tz_resource);
+ tz_resource = NULL;
+ }
+
+ if (registry) {
+ wl_registry_destroy(registry);
+ registry = NULL;
+ }
+
+ if (queue) {
+ wl_event_queue_destroy(queue);
+ queue = NULL;
+ }
+
+ if (display_wrapper) {
+ wl_proxy_wrapper_destroy(display_wrapper);
+ display_wrapper = NULL;
+ }
+}
+
+static void win_resize_cb (void *data, Evas *e, Evas_Object *obj, void *event_info)
+{
+ int w, h;
+ int wid;
+
+ printf("_canvas_resize_cb\n");
+
+ Evas * _e = evas_object_evas_get(obj);
+ Ecore_Evas *ee = ecore_evas_ecore_evas_get(_e);
+ ecore_evas_geometry_get(ee, NULL, NULL, &w, &h);
+
+ if (!st) {
+ set_overlay(ee);
+ int x, y;
+ elm_win_screen_position_get(obj, &x, &y);
+ printf("x = %d, y = %d\n", x, y);
+ } else
+ st++;
+}
+
+static void cb_new_pad (GstElement *element, GstPad *pad, gpointer data)
+{
+ gchar *name;
+ GstElement *other = data;
+
+ name = gst_pad_get_name (pad);
+ printf ("A new pad %s was created for %s\n", name, gst_element_get_name(element));
+ g_free (name);
+
+ printf ("element %s will be linked to %s\n",
+ gst_element_get_name(element),
+ gst_element_get_name(dscale));
+ gst_element_link(element, dscale);
+}
+
+static int app_create(void *data)
+{
+ appdata *ad = data;
+ Evas_Object *win = NULL;
+
+ // GST
+ g_mutex_init(&pose_mutex);
+
+ signal(SIGINT, int_handler);
+
+ /* initialization */
+ gst_init(NULL, NULL);
+
+ /* mediavision configuration*/
+ IsGestureMode = false;
+ if (hp_mv_engine_cfg == NULL) {
+ mv_create_engine_config(&hp_mv_engine_cfg);
+ }
+
+ if (hp_mv_engine_cfg2 == NULL) {
+ mv_create_engine_config(&hp_mv_engine_cfg2);
+ }
+
+ int err = MEDIA_VISION_ERROR_NONE;
+
+
+ if (ad->modelType == MODEL_TYPE_POSE_CPM) {
+ //err = perform_armnn_human_pose_cpm_configure(hp_mv_engine_cfg);
+ err = perform_tflite_human_pose_cpm_configure(hp_mv_engine_cfg);
+ } else if (ad->modelType == MODEL_TYPE_POSE_HAND_AICLite) {
+ outputTensorData = (void*)calloc(56*56*21, sizeof(float));
+ err = perform_tflite_hand_detection_AIC(hp_mv_engine_cfg);
+
+ err = perform_tflite_hand_detection_AIC2(hp_mv_engine_cfg2);
+ } else if (ad->modelType == MODEL_TYPE_POSE_HAND_AICLite2) {
+ outputTensorData = (void*)calloc(56*56*21, sizeof(float));
+ err = perform_tflite_hand_detection_AICLite(hp_mv_engine_cfg);
+
+ err = perform_tflite_hand_detection_AICLite2(hp_mv_engine_cfg2);
+ } else {
+ outputTensorData = (void*)calloc(56*56*21, sizeof(float));
+ err = perform_tflite_hand_detection_AICLiteQ(hp_mv_engine_cfg);
+
+ err = perform_tflite_hand_detection_AICLiteQ2(hp_mv_engine_cfg2);
+ }
+
+ if (err != MEDIA_VISION_ERROR_NONE) {
+ printf("Error on perform_armnn_human_pose_configure");
+ }
+
+ printf("configuration done\n");
+
+ printf("loading pose model: ");
+ mv_inference_create(&hp_mv_infer);
+
+ mv_inference_configure(hp_mv_infer, hp_mv_engine_cfg);
+
+ clock_t start = clock();
+ mv_inference_prepare(hp_mv_infer);
+ clock_t end = clock();
+ printf("time: %2.3f\n", (double)(end - start)/CLOCKS_PER_SEC);
+
+ /* mediavision source */
+ mv_create_source(&mv_src_p);
+
+ if (ad->modelType >= MODEL_TYPE_POSE_HAND_AICLite) {
+ mv_inference_create(&hp_mv_infer2);
+
+ mv_inference_configure(hp_mv_infer2, hp_mv_engine_cfg2);
+
+ start = clock();
+ mv_inference_prepare(hp_mv_infer2);
+ end = clock();
+ printf("time: %2.3f\n", (double)(end - start)/CLOCKS_PER_SEC);
+
+ mv_create_source(&mv_src_p2);
+ }
+
+ pipeline = gst_pipeline_new("app");
+
+ overlay_state = g_new0 (CairoOverlayState, 1);
+
+ /* create gstreamer elements */
+ if (!ad->filename) {
+ source = gst_element_factory_make("v4l2src", "src");
+ filter = gst_element_factory_make("capsfilter", "filter");
+ } else {
+ source = gst_element_factory_make("filesrc", "src");
+
+ dbin = gst_element_factory_make("decodebin", "dbin");
+ dscale = gst_element_factory_make("videoscale", "dscale");
+ dconv = gst_element_factory_make("videoconvert", "dconv");
+ dsfilter = gst_element_factory_make("capsfilter", "dsfilter");
+ dcfilter = gst_element_factory_make("capsfilter", "dcfilter");
+ }
+
+ if (ad->filename2) {
+ tee2 = gst_element_factory_make("tee", "tee2");
+ enc = gst_element_factory_make("avenc_mpeg4", "enc");
+ muxmp4 = gst_element_factory_make("mp4mux", "muxmp4");
+ fsink2 = gst_element_factory_make("filesink", "fsink2");
+ queue4 = gst_element_factory_make("queue", "queue4");
+ queue5 = gst_element_factory_make("queue", "queue5");
+ encconv = gst_element_factory_make("videoconvert", "encconv");
+ }
+
+
+ tee = gst_element_factory_make("tee", "tee");
+ queue1 = gst_element_factory_make("queue", "queue1");
+ queue2 = gst_element_factory_make("queue", "queue2");
+
+ if (0 /*ad->modelType == MODEL_TYPE_POSE_HAND_AIC*/) {
+ queue3 = gst_element_factory_make("queue", "queue3");
+ }
+
+ // queue1 - videoscale - capsfilter -viedoeconvert - capsfilter - videorate - capsfilter -fakesink
+ vscale = gst_element_factory_make("videoscale", "scale");
+ vsfilter = gst_element_factory_make("capsfilter", "vsfilter");
+ vconv = gst_element_factory_make("videoconvert", "convert");
+ vcfilter = gst_element_factory_make("capsfilter", "vcfilter");
+ vrate = gst_element_factory_make("videorate", "rate");
+ vrfilter = gst_element_factory_make("capsfilter", "vrfilter");
+ vrsink = gst_element_factory_make("fakesink", "vrsink");
+
+ // queue2 - videoconvert - cairooveray - tizenwlsink
+ oconv = gst_element_factory_make("videoconvert", "oconv");
+ coverlay = gst_element_factory_make("cairooverlay", "coverlay");
+ sink = gst_element_factory_make("fpsdisplaysink", "vsink");
+ sink2 = gst_element_factory_make("fakesink", "vsink2");
+
+ // after detection, crop using video crop
+ // queue3 - videocrop - videoscale -capsfilter - videoconvert - capsfilter -fakesink
+
+ if (0/*ad->modelType == MODEL_TYPE_POSE_HAND_AIC*/) {
+ vcrop = gst_element_factory_make("videocrop", "crop");
+ vcrscale = gst_element_factory_make("videoscale", "crscale");
+ vcrsfilter = gst_element_factory_make("capsfilter", "vcrsfilter");
+ vcrsconv = gst_element_factory_make("videoconvert", "vcrsconvert");
+ vcrscfilter = gst_element_factory_make("capsfilter", "vcrscfilter");
+ vcrssink = gst_element_factory_make("fakesink", "vcrssink");
+ }
+
+ if (!pipeline || !source ||
+ !tee || !queue1 || !vscale || !vsfilter || !vconv || !vcfilter ||
+ !vrate || !vrfilter || !vrsink ||
+ !queue2 || !oconv || !coverlay || !sink || !sink2) {
+ printf(TEXT_RED "One element(queue1 or queue2) might be not created. Exiting.\n" TEXT_RESET);
+ return -1;
+ }
+
+ if (0 /*ad->modelType == MODEL_TYPE_POSE_HAND_AIC && (!pipeline || !queue3
+ || !vcrop || !vcrscale || !vcrsfilter || !vcrsconv || !vcrscfilter || !vcrssink*/) {
+ printf(TEXT_RED "One element(queue3) might be not created. Exiting.\n" TEXT_RESET);
+ return -1;
+ }
+
+ if (!ad->filename) {
+ if (!filter) {
+ printf(TEXT_RED "One element might be not created. Existing.\n" TEXT_RESET);
+ return -1;
+ }
+ g_object_set(G_OBJECT(filter), "caps", gst_caps_from_string("video/x-raw, format=YUY2, width=640, height=480"), NULL);
+ } else {
+ if (!dbin || !dscale || !dconv) {
+ printf(TEXT_RED "One element might be not created. Exiting.\n" TEXT_RESET);
+ return -1;
+ }
+ g_object_set(G_OBJECT(dsfilter), "caps", gst_caps_from_string("video/x-raw, width=640, height=480"), NULL);
+ g_object_set(G_OBJECT(dcfilter), "caps", gst_caps_from_string("video/x-raw, format=YUY2, width=640, height=480"), NULL);
+ g_signal_connect (dbin, "pad-added", G_CALLBACK (cb_new_pad), NULL);
+ }
+
+ if (ad->filename2) {
+ if (!tee2 || !enc || !muxmp4 || !queue4 || !queue5 || !fsink2 || !encconv) {
+ printf(TEXT_RED "One element might be not created. Exiting.\n" TEXT_RESET);
+ printf("%p, %p, %p, %p, %p, %p, %p\n", tee2, enc, muxmp4, queue4, queue5, fsink2, encconv);
+ return -1;
+ }
+ g_object_set(G_OBJECT(fsink2), "location", ad->filename2, NULL);
+
+ }
+
+ /* set up the pipeline */
+ //g_signal_connect (coverlay, "draw", G_CALLBACK (draw_overlay), overlay_state);
+ if (ad->modelType == MODEL_TYPE_POSE_CPM) {
+ g_signal_connect (coverlay, "draw", G_CALLBACK (draw_overlay_pose), overlay_state);
+ } else {
+ g_signal_connect (coverlay, "draw", G_CALLBACK (draw_overlay_hand), overlay_state);
+ }
+
+ g_signal_connect (coverlay, "caps-changed", G_CALLBACK (prepare_overlay), overlay_state);
+
+ if (!ad->filename) {
+ g_object_set(G_OBJECT(source), "device", "/dev/video252", NULL); // 252
+ } else {
+ g_object_set(G_OBJECT(source), "location", ad->filename, NULL);
+ //g_object_set(G_OBJECT(source), "num-buffers", ad->numbuffers, NULL);
+ }
+
+ //g_object_set(G_OBJECT(sink), "text-overlay", FALSE, NULL);
+ //g_object_set(G_OBJECT(sink2), "use-tbm", FALSE, NULL);
+ g_object_set(G_OBJECT(sink2), "sync", FALSE, NULL);
+ g_object_set(G_OBJECT(sink), "video-sink", sink2, NULL);
+ g_object_set(G_OBJECT(sink), "sync", FALSE, NULL);
+
+ if (ad->modelType == MODEL_TYPE_POSE_HAND_AICLite ||
+ ad->modelType == MODEL_TYPE_POSE_HAND_AICLite2 ||
+ ad->modelType == MODEL_TYPE_POSE_HAND_AICLite2Q) {
+ g_object_set(G_OBJECT(vsfilter), "caps", gst_caps_from_string("video/x-raw, width=224, height=224"), NULL);
+ poseCropSize = 224;
+ } else {
+ g_object_set(G_OBJECT(vsfilter), "caps", gst_caps_from_string("video/x-raw, width=192, height=192"), NULL);
+ poseCropSize = 192;
+ }
+
+ g_object_set(G_OBJECT(vcfilter), "caps", gst_caps_from_string("video/x-raw, format=RGB"), NULL);
+ g_object_set(G_OBJECT(vrfilter), "caps", gst_caps_from_string("video/x-raw, framerate=15/1"), NULL);
+ g_object_set(G_OBJECT(queue2), "max-size-buffers", 1, NULL);
+ //g_object_set(G_OBJECT(vrate), "drop-only", TRUE, NULL);
+
+ //g_object_set(G_OBJECT(queue2), "leaky", 2, NULL);
+#if 0
+ g_object_set(G_OBJECT(queue3), "max-size-buffers", 0, NULL);
+ g_object_set(G_OBJECT(queue3), "max-size-time", 0, NULL);
+ g_object_set(G_OBJECT(queue3), "max-size-bytes", 0, NULL);
+#endif
+ g_object_set(G_OBJECT(queue3), "max-size-buffers", 1, NULL);
+ //g_object_set(G_OBJECT(queue3), "leaky", 2, NULL);
+
+ // here to be continue
+ printf("vrsink signal-handoffs\n");
+ g_object_set(G_OBJECT(vrsink), "signal-handoffs", TRUE, NULL);
+
+ if (ad->modelType == MODEL_TYPE_POSE_CPM) {
+ handler_p = g_signal_connect (vrsink, "handoff", G_CALLBACK(_pose_est_handoff), outputTensorData);
+ } else {
+ handler_p = g_signal_connect (vrsink, "handoff", G_CALLBACK(_hand_est_handoff), outputTensorData);
+ }
+
+ g_object_set(G_OBJECT(vrsink), "sync", FALSE, NULL);
+
+
+ g_object_set(G_OBJECT(vcrssink), "sync", FALSE, NULL);
+
+
+ /* add a message handler */
+ bus = gst_pipeline_get_bus (GST_PIPELINE(pipeline));
+ bus_watch_id = gst_bus_add_watch(bus, bus_call, NULL);
+ gst_object_unref(bus);
+
+ /* add elements into the pipeline */
+ gst_bin_add_many(GST_BIN(pipeline),
+ source,
+ tee, queue1, vscale, vsfilter, vconv, vcfilter,
+ vrate, vrfilter, vrsink,
+ queue2, oconv, coverlay, sink,
+ NULL);
+
+ if (0 /*ad->modelType == MODEL_TYPE_POSE_HAND_AIC*/) {
+ gst_bin_add_many(GST_BIN(pipeline),
+ queue3, vcrop, vcrscale, vcrsfilter, vcrsconv, vcrscfilter, vcrssink,
+ NULL);
+ gst_element_link_many (tee, queue3, vcrop, vcrscale, vcrsfilter, vcrsconv, vcrscfilter, vcrssink, NULL);
+ }
+
+ /* link elements */
+ if (!ad->filename) {
+ gst_bin_add(GST_BIN(pipeline), filter);
+ gst_element_link_many(source, filter, tee, NULL);
+ } else {
+ gst_bin_add_many(GST_BIN(pipeline), dbin, dscale, dconv, dsfilter, dcfilter, NULL);
+ gst_element_link_many(source, dbin, NULL);
+ gst_element_link_many(dscale, dsfilter, dconv, dcfilter, tee, NULL);
+ }
+
+ if (!ad->filename2) {
+ // display
+ gst_element_link_many (tee, queue2, oconv, coverlay, /*toverlay,*/ sink, NULL);
+ // pose
+ gst_element_link_many (tee, queue1, vrate, vrfilter, vconv, vcfilter, vscale, vsfilter, vrsink, NULL);
+ } else {
+
+ gst_bin_add_many(GST_BIN(pipeline), tee2, enc, muxmp4, fsink2, queue4, queue5, encconv);
+
+ // display
+ gst_element_link_many (tee, queue2, oconv, coverlay, tee2, NULL);
+ gst_element_link_many (tee2, queue4, sink, NULL);
+ gst_element_link_many (tee2, queue5, encconv, enc, muxmp4, fsink2, NULL);
+
+ // pose
+ gst_element_link_many (tee, queue1, vrate, vrfilter, vconv, vcfilter, vscale, vsfilter, vrsink, NULL);
+ }
+
+ /* set the pipeline state to "playing" state */
+ //gst_element_set_state(pipeline, GST_STATE_PLAYING);
+
+ /* loop */
+ humanSkeleton.IsDetected = false;
+ humanSkeleton.isPrevPose = false;
+ humanSkeleton.label = -1;
+ printf(TEXT_GREEN "Running.....\n" TEXT_RESET);
+ // GST_END
+#if 0
+ /* use gl backend */
+ elm_config_accel_preference_set("opengl");
+
+ /* create window */
+ win = elm_win_add(NULL, PACKAGE, ELM_WIN_SPLASH );
+ //win = elm_win_add(NULL, PACKAGE, ELM_WIN_BASIC);
+ if (win) {
+ elm_win_title_set(win, PACKAGE);
+ elm_win_borderless_set(win, EINA_TRUE);
+ elm_win_autodel_set(win, EINA_TRUE);
+ elm_win_alpha_set(win, EINA_FALSE);
+ evas_object_show(win);
+ }
+ elm_win_layer_set(win, 9);
+ elm_win_prop_focus_skip_set(win, EINA_TRUE);
+
+ ad->win = win;
+ g_win_id = win;
+ selected_win_id = g_win_id;
+
+ Evas_Object *bg = elm_bg_add(win);
+ elm_win_resize_object_add(win, bg);
+ evas_object_size_hint_min_set(bg, WIDTH, HEIGHT);
+ evas_object_size_hint_max_set(bg, WIDTH, HEIGHT);
+ evas_object_show(bg);
+
+ elm_win_activate(win);
+
+
+ evas_object_event_callback_add(win, EVAS_CALLBACK_RESIZE, win_resize_cb, NULL);
+#else
+ gst_element_set_state (pipeline, GST_STATE_PLAYING);
+#endif
+ return 0;
+}
+
+
+static int app_terminate(void *data)
+{
+ appdata *ad = data;
+ int i = 0;
+
+ /* out of loop */
+ printf(TEXT_GREEN "Stopping.....\n" TEXT_RESET);
+ gst_element_set_state(pipeline, GST_STATE_NULL);
+
+ printf(TEXT_GREEN "pipe unref.....\n" TEXT_RESET);
+ gst_object_unref(GST_OBJECT(pipeline));
+
+ printf(TEXT_GREEN "remove bus_watch id.....\n" TEXT_RESET);
+ g_source_remove(bus_watch_id);
+
+ if (overlay_state != NULL) {
+ printf(TEXT_GREEN "g_free overlay.....\n" TEXT_RESET);
+ g_free(overlay_state);
+ }
+
+ g_mutex_clear(&pose_mutex);
+ printf(TEXT_GREEN "done.....\n" TEXT_RESET);
+
+ if (g_win_id) {
+ evas_object_del(g_win_id);
+ g_win_id = NULL;
+ }
+ ad->win = NULL;
+ selected_win_id = NULL;
+ free(my_cursor);
+ return 0;
+}
+struct appcore_ops ops = {
+ .create = app_create,
+ .terminate = app_terminate,
+};
+
+int main (int argc, char *argv[])
+{
+ memset(&ad, 0x0, sizeof(appdata));
+ ops.data = &ad;
+
+ if (argc >= 6) {
+ ad.filename = g_strdup(argv[5]);
+ printf("launch with file source (%s)\n", ad.filename);
+ if (argc > 6) {
+ ad.filename2 = g_strdup(argv[6]);
+ ad.numbuffers = -1;
+ printf("records output(%s)\n", ad.filename2);
+ }
+ if (argc > 7) {
+ ad.numbuffers = atoi(argv[7]);
+ }
+ } else {
+ printf("launch with camera source\n");
+ }
+
+ if (argc < 2) {
+ printf("usage: mv_stream_infer model [NeckThresVal, ArmThresVal, LegThresVal, [filename]]");
+ printf("model: 0(CPM), 1(AIC Hand), 2(AIC Lite Hand), 3(AIC Lite Q Hand)\n");
+ return -1;
+ }
+
+ ad.modelType = atoi(argv[1]);
+ if (ad.modelType < 0 || ad.modelType > 3) {
+ printf("not supported model type [%d]\n", ad.modelType);
+ return -1;
+ }
+
+ if (ad.modelType != MODEL_TYPE_POSE_HAND_AICLite &&
+ ad.modelType != MODEL_TYPE_POSE_HAND_AICLite2 &&
+ ad.modelType != MODEL_TYPE_POSE_HAND_AICLite2Q) {
+ thValNeck = (float)atoi(argv[2])/100.f;
+ thValArm = (float)atoi(argv[3])/100.f;
+ thValLeg = (float)atoi(argv[4])/100.f;
+
+ poseRoi.point.x = 0;
+ poseRoi.point.y = 0;
+ poseRoi.width = 0;
+ poseRoi.height = 0;
+ } else {
+ if (argc > 2) {
+ ad.filename2 = g_strdup(argv[2]);
+ }
+ poseRoi.point.x = 0;
+ poseRoi.point.y = 0;
+ poseRoi.width = 0;
+ poseRoi.height = 0;
+ }
+ my_cursor = efl_util_input_initialize_generator(EFL_UTIL_INPUT_DEVTYPE_POINTER);
+
+ return appcore_efl_main(PACKAGE, &argc, &argv, &ops);
+}
+
--- /dev/null
+/**
+ * Copyright (c) 2015 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define _USE_MATH_DEFINES
+#include <fcntl.h>
+#include <mv_common.h>
+#include <mv_inference.h>
+
+#include <mv_testsuite_common.h>
+
+#include <image_helper.h>
+#include <mv_video_helper.h>
+
+#include <mv_log_cfg.h>
+
+#include <math.h>
+#include <ctype.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <dirent.h>
+#include <string.h>
+#include <limits.h>
+#include <time.h>
+#define MAX(a, b) \
+({ __typeof__ (a) _a = (a); \
+__typeof__ (b) _b = (b); \
+_a > _b ? _a : _b; })
+
+#define MIN(a,b) \
+({ __typeof__ (a) _a = (a); \
+__typeof__ (b) _b = (b); \
+_a < _b ? _a : _b; })
+
+#include <glib-2.0/glib.h>
+#include <gst/gst.h>
+#include <gst/video/video.h>
+#include <cairo.h>
+#include <cairo-gobject.h>
+
+#include <Elementary.h>
+#include <appcore-efl.h>
+#include <Ecore.h>
+#include <Ecore_Evas.h>
+#include <Ecore_Wl2.h>
+#include <tizen-extension-client-protocol.h>
+#include <efl_util.h>
+
+#include <gst/gst.h>
+#include <gst/video/videooverlay.h>
+#include <unistd.h>
+#include <time.h>
+
+#define WIDTH (640)
+#define HEIGHT (480)
+
+#ifdef PACKAGE
+#undef PACKAGE
+#endif
+#define PACKAGE "test"
+
+static char* gHandGestureLabel[] = { "None", "One", "Two", "Three", "Four", "Five",
+ "Six", "Eight", "SmallHeart", "OK", "Like", "DisLike",
+ "Fist", "Rock", "Love"};
+static int st = 0;
+static Evas_Object *g_eo = NULL;
+static Evas_Object *icon = NULL;
+
+/* for video display */
+static Evas_Object *g_win_id;
+static Evas_Object *selected_win_id;
+
+int recording_fd;
+int fd, ret;
+FILE *fp;
+
+typedef enum {
+ MODEL_TYPE_POSE_CPM = 0,
+ MODEL_TYPE_POSE_HAND_AICLite, // posenet lite 224
+ MODEL_TYPE_POSE_HAND_AICLite2, // posenet 0709 f
+ MODEL_TYPE_POSE_HAND_AICLite2Q // posenet 0709 q
+};
+
+typedef struct {
+ gchar *filename;
+ gchar *filename2;
+ int numbuffers;
+ int modelType;
+ Evas_Object *win;
+ Evas_Object *layout_main; /* layout widget based on EDJ */
+ /* add more variables here */
+
+} appdata;
+
+static mv_rectangle_s poseRoi;
+
+static appdata ad;
+static GstBus *bus;
+static guint bus_watch_id;
+int sending = FALSE;
+static efl_util_inputgen_h my_cursor = NULL;
+
+#define FILE_PATH_SIZE 1024
+
+// pose estimation
+#define PE_TFLITE_CPM_WEIGHT_PATH "/usr/share/capi-media-vision/models/PLD/tflite/pld-tflite-001.tflite"
+#define PE_POSE_LABEL_PATH "/usr/share/capi-media-vision/models/PLD/tflite/pose-label.txt"
+
+
+#define PE_TFLITE_AIC_1_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE/tflite/posenet1_lite_224.tflite"
+#define PE_TFLITE_AIC_2_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE/tflite/posenet2_lite_224.tflite"
+
+
+#define PE_TFLITE_AICLite_1_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE/tflite/posenet1_0709_f.tflite"
+#define PE_TFLITE_AICLite_2_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE/tflite/posenet2_0709_f.tflite"
+
+#define PE_TFLITE_AICLiteQ_1_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE/tflite/posenet1_0709_dq.tflite"
+#define PE_TFLITE_AICLiteQ_2_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE/tflite/posenet2_0709_f.tflite"
+
+static float thValNeck = 0.3f; // 15%
+static float thValArm = 0.1f; // 10 %
+static float thValLeg = 0.2f; // 5%
+
+typedef struct _rect {
+ int left;
+ int top;
+ int right;
+ int bottom;
+ int type;
+ bool updated;
+ bool cr_updated;
+} rect;
+
+
+typedef struct _humanSkeleton {
+ mv_point_s pose[21/*14*/];
+ mv_point_s prevPose[21/*14*/];
+ mv_rectangle_s loc;
+ mv_rectangle_s prevLoc;
+ mv_rectangle_s handRoi;
+ float scores[21/*14*/];
+ bool isPrevPose;
+ bool updated; // detection is find and pose is also good. then update drawing
+ bool locUpdated; // track previous detection
+ bool IsDetected; // detection
+ int label;
+} HumanSkeleton;
+
+typedef struct
+{
+ gboolean valid;
+ GstVideoInfo vinfo;
+} CairoOverlayState;
+
+typedef struct
+{
+ GstBuffer *buffer;
+ gpointer user_data;
+} HandOffData;
+
+//gchar *gesturenames;
+
+static HandOffData hdata_p;
+
+static GMutex pose_mutex;
+static guint old_timeout = 0;
+static guint nFrames = 0;
+
+// Gstreamer
+GstElement *pipeline, *source, *filter, *toverlay, *sink, *sink2;
+GstElement *tee, *vscale, *vsfilter, *vconv, *vcfilter;
+GstElement *vrate, *vrfilter, *fsink, *vrsink;
+GstElement *queue1, *queue2, *queue3;
+GstElement *oconv, *coverlay;
+GstElement *vcrop, *vcrscale, *vcrsfilter, *vcrsconv, *vcrscfilter, *vcrssink;
+CairoOverlayState *overlay_state;
+
+GstElement *dbin, *dscale, *dconv;
+GstElement *dsfilter, *dcfilter;
+
+GstElement *tee2, *enc, *muxmp4, *fsink2, *queue4, *queue5, *encconv;
+
+//static HandSkeleton handSkeleton;
+static HumanSkeleton humanSkeleton;
+gulong handler_p;
+GList *line_list = NULL;
+
+mv_source_h mv_src_p;
+mv_source_h mv_src_p2;
+
+// Human pose
+mv_engine_config_h hp_mv_engine_cfg;
+mv_inference_h hp_mv_infer;
+
+mv_engine_config_h hp_mv_engine_cfg2;
+mv_inference_h hp_mv_infer2;
+
+static void * outputTensorData;
+
+FILE *fp;
+
+static bool IsGestureMode;
+
+static int poseCropSize = 0;
+
+#define IMAGE_SIZE_WIDTH 640
+#define IMAGE_SIZE_HEIGHT 480
+
+#define NANO_PER_SEC ((__clock_t) 1000000000)
+#define NANO_PER_MILLI ((__clock_t) 1000000)
+#define MILLI_PER_SEC ((__clock_t) 1000)
+
+#define VOLUME_INTERVAL 10
+
+void send_mouse_event_efl(int x, int y){
+ efl_util_input_generate_pointer(my_cursor, 1, EFL_UTIL_INPUT_POINTER_MOVE, x, y);
+}
+
+struct timespec diff(struct timespec start, struct timespec end)
+{
+ struct timespec temp;
+ if ((end.tv_nsec - start.tv_nsec) < 0) {
+ temp.tv_sec = end.tv_sec - start.tv_sec - 1;
+ temp.tv_nsec = NANO_PER_SEC + end.tv_nsec - start.tv_nsec;
+ }
+ else {
+ temp.tv_sec = end.tv_sec - start.tv_sec;
+ temp.tv_nsec = end.tv_nsec - start.tv_nsec;
+ }
+ return temp;
+}
+
+unsigned long gettotalmillisec(const struct timespec time)
+{
+ return time.tv_sec * MILLI_PER_SEC + time.tv_nsec / NANO_PER_MILLI;
+}
+
+
+void int_handler(int sig)
+{
+ char c;
+
+ signal(sig, SIG_IGN);
+ while ((getchar()) != '\n');
+
+ printf(TEXT_YELLOW "Do you want to quit? [y/n]\n" TEXT_RESET);
+ c = getchar();
+ if (c == 'y' || c == "Y") {
+
+ g_signal_handler_disconnect(vrsink, handler_p);
+#if 0
+ g_signal_handler_disconnect(vcrssink, handler_gp);
+#endif
+
+ gst_element_send_event(pipeline, gst_event_new_eos());
+
+ sleep(4);
+
+
+ if (mv_src_p)
+ mv_destroy_source(mv_src_p);
+
+ if (hp_mv_infer)
+ mv_inference_destroy(hp_mv_infer);
+
+ if (hp_mv_engine_cfg)
+ mv_destroy_engine_config(hp_mv_engine_cfg);
+
+ if (mv_src_p2)
+ mv_destroy_source(mv_src_p2);
+
+ if (hp_mv_infer2)
+ mv_inference_destroy(hp_mv_infer2);
+
+ if (hp_mv_engine_cfg)
+ mv_destroy_engine_config(hp_mv_engine_cfg2);
+
+ if (outputTensorData) {
+ free(outputTensorData);
+ outputTensorData = NULL;
+ }
+ printf(TEXT_YELLOW "exit..\n" TEXT_RESET);
+ signal(SIGINT, SIG_DFL);
+ exit(0);
+ } else {
+ printf("no");
+ signal(SIGINT, int_handler);
+ }
+
+ getchar(); // Get new line character
+}
+
+void _hand_pose_cb (
+ mv_source_h source,
+ int number_of_pose_estimation,
+ mv_inference_pose_s *locations,
+ int label,
+ void *user_data)
+{
+ //printf("%d landmarks\n", number_of_pose_estimation);
+ int width, height;
+ mv_source_get_height(source, &width);
+ mv_source_get_height(source, &height);
+ for (int n = 0; n < number_of_pose_estimation; ++n) {
+ humanSkeleton.pose[n].x = (int)(640.f*(float)locations->landmarks[n].point.x/(float)width);
+ humanSkeleton.pose[n].y = (int)(480.f*(float)locations->landmarks[n].point.y/(float)height);
+ humanSkeleton.scores[n] = 1.0f; /* confidences[n];*/
+
+ //printf("(%d,%d): %f\n", humanSkeleton.pose[n].x, humanSkeleton.pose[n].y, confidences[n]);
+ //printf("(%d,%d)\n", humanSkeleton.pose[n].x, humanSkeleton.pose[n].y);
+ }
+ humanSkeleton.label = label;
+ humanSkeleton.IsDetected = true;
+}
+
+static void _hand_detected_cb (
+ mv_source_h source,
+ const int number_of_hands,
+ const float *confidences,
+ const mv_rectangle_s *locations,
+ void *user_data) //user_data can be mv_source?
+{
+
+#if 0
+ if (0 /*confidences[1] < thValNeck*/) {
+ printf("lost pose\n");
+ humanSkeleton.IsDetected = false;
+ humanSkeleton.isPrevPose = false;
+ return;
+ }
+
+ printf("%d landmarks, %d crop\n", number_of_landmarks, poseCropSize);
+ for (int n = 0; n < number_of_landmarks; ++n) {
+
+ humanSkeleton.pose[n].x = (int)((float)(locations[n].x + poseRoi.point.x) / (float)poseCropSize * 640.f);
+ humanSkeleton.pose[n].y = (int)((float)(locations[n].y + poseRoi.point.y) / (float)poseCropSize * 480.f);
+ humanSkeleton.scores[n] = 1.0f; /* confidences[n];*/
+
+ //printf("(%d,%d): %f\n", humanSkeleton.pose[n].x, humanSkeleton.pose[n].y, confidences[n]);
+ printf("(%d,%d)\n", humanSkeleton.pose[n].x, humanSkeleton.pose[n].y);
+ }
+ humanSkeleton.IsDetected = true;
+#else
+
+ if (number_of_hands <= 0) {
+ humanSkeleton.IsDetected = false;
+ humanSkeleton.label = -1;
+ return;
+ }
+
+ //seg fault?
+ struct timespec s_tspec;
+ struct timespec e_tspec;
+
+ clock_gettime(CLOCK_MONOTONIC, &s_tspec);
+
+ mv_source_clear(mv_src_p2);
+ mv_source_fill_by_tensor_buffer(mv_src_p2, user_data,
+ MV_INFERENCE_DATA_FLOAT32,
+ 56 * 56 * 21 * sizeof(float),
+ 56, 56, 21, 3);
+
+ clock_gettime(CLOCK_MONOTONIC, &e_tspec);
+
+ struct timespec diffspec = diff(s_tspec, e_tspec);
+ unsigned long timeDiff = gettotalmillisec(diffspec);
+ //printf("memcpy time: %lu(ms)\n", timeDiff);
+
+ clock_gettime(CLOCK_MONOTONIC, &s_tspec);
+
+ mv_inference_pose_landmark_detect(mv_src_p2, hp_mv_infer2, NULL, _hand_pose_cb, NULL);
+
+ clock_gettime(CLOCK_MONOTONIC, &e_tspec);
+
+ diffspec = diff(s_tspec, e_tspec);
+ timeDiff = gettotalmillisec(diffspec);
+ //printf("pose_estimation time: %lu(ms)\n", timeDiff);
+ //humanSkeleton.IsDetected = true;
+#endif
+ return;
+}
+
+
+void _human_pose_cb (
+ mv_source_h source,
+ int number_of_pose_estimation,
+ mv_inference_pose_s *locations,
+ int label,
+ void *user_data)
+{
+ //printf("%d landmarks\n", number_of_pose_estimation);
+
+ float score = 0.0f;
+ int cnt = 0;
+ for (int n = 0; n < number_of_pose_estimation; ++n) {
+ humanSkeleton.pose[n].x = locations->landmarks[n].point.x;
+ humanSkeleton.pose[n].y = locations->landmarks[n].point.y;
+ humanSkeleton.scores[n] = locations->landmarks[n].score;
+ if (score >= 0.0 ) {
+ score += humanSkeleton.scores[n];
+ cnt++;
+ }
+
+ //printf("(%d,%d): %f\n", humanSkeleton.pose[n].x, humanSkeleton.pose[n].y, confidences[n]);
+ //printf("(%d,%d)\n", humanSkeleton.pose[n].x, humanSkeleton.pose[n].y);
+ }
+ if(cnt == 0){
+ printf("count is 0 !!! \n");
+ }else{
+ score /= (float)cnt;
+ }
+
+ if (score >= 0.5) {
+ humanSkeleton.IsDetected = true;
+ } else {
+ humanSkeleton.IsDetected = false;
+ humanSkeleton.label = -1;
+ }
+}
+
+static gboolean
+run_pose (void *user_data)
+{
+ HandOffData *udata = (HandOffData *)user_data;
+ if (!GST_IS_BUFFER(udata->buffer))
+ return FALSE;
+
+ GstMapInfo map;
+
+ gst_buffer_map(udata->buffer, &map, GST_MAP_READ);
+
+ mv_source_clear(mv_src_p);
+
+ mv_source_fill_by_buffer(mv_src_p, map.data, 192*192*3, 192, 192, MEDIA_VISION_COLORSPACE_RGB888);
+
+ gst_buffer_unmap(udata->buffer, &map);
+
+ struct timespec s_tspec;
+ struct timespec e_tspec;
+
+ clock_gettime(CLOCK_MONOTONIC, &s_tspec);
+
+ mv_inference_pose_landmark_detect(mv_src_p, hp_mv_infer, NULL, _human_pose_cb, NULL);
+
+ clock_gettime(CLOCK_MONOTONIC, &e_tspec);
+ struct timespec diffspec = diff(s_tspec, e_tspec);
+ unsigned long timeDiff = gettotalmillisec(diffspec);
+ //printf("detect + pose time: %lu(ms)\n", timeDiff);
+
+ return FALSE;
+
+}
+
+static gboolean
+run_hand (void *user_data)
+{
+ HandOffData *udata = (HandOffData *)user_data;
+ if (!GST_IS_BUFFER(udata->buffer))
+ return FALSE;
+
+ GstMapInfo map;
+
+ gst_buffer_map(udata->buffer, &map, GST_MAP_READ);
+
+ mv_source_clear(mv_src_p);
+
+ mv_source_fill_by_buffer(mv_src_p, map.data, 224*224*3, 224, 224, MEDIA_VISION_COLORSPACE_RGB888);
+
+ gst_buffer_unmap(udata->buffer, &map);
+
+
+ struct timespec s_tspec;
+ struct timespec e_tspec;
+
+ void * outputTensorBuffer = (void*)udata->user_data;
+
+ clock_gettime(CLOCK_MONOTONIC, &s_tspec);
+
+ // invoke tflite -> _hand_detected_cb -> memcpy output -> invoke tflite -> _pose_cb
+ mv_inference_hand_detect(mv_src_p, hp_mv_infer, _hand_detected_cb, outputTensorBuffer);
+
+ clock_gettime(CLOCK_MONOTONIC, &e_tspec);
+ struct timespec diffspec = diff(s_tspec, e_tspec);
+ unsigned long timeDiff = gettotalmillisec(diffspec);
+ //printf("detect + pose time: %lu(ms)\n", timeDiff);
+
+ return FALSE;
+
+}
+
+static void
+_pose_est_handoff(GstElement *object, GstBuffer *buffer, GstPad *pad, gpointer user_data)
+{
+
+ nFrames++;
+ hdata_p.buffer = buffer;
+ hdata_p.user_data = user_data;
+
+#if 0
+ if (nFrames % 15 == 0) {
+ g_mutex_lock(&pose_mutex);
+ g_idle_add (run_pose, &hdata_p);
+ g_mutex_unlock(&pose_mutex);
+ }
+#else
+ g_mutex_lock(&pose_mutex);
+ g_idle_add (run_pose, &hdata_p);
+ g_mutex_unlock(&pose_mutex);
+#endif
+}
+
+static void
+_hand_est_handoff(GstElement *object, GstBuffer *buffer, GstPad *pad, gpointer user_data)
+{
+
+ nFrames++;
+ hdata_p.buffer = buffer;
+ hdata_p.user_data = user_data;
+
+#if 0
+ if (nFrames % 15 == 0) {
+ g_mutex_lock(&pose_mutex);
+ g_idle_add (run_hand, &hdata_p);
+ g_mutex_unlock(&pose_mutex);
+ }
+#else
+ g_mutex_lock(&pose_mutex);
+ g_idle_add (run_hand, &hdata_p);
+ g_mutex_unlock(&pose_mutex);
+#endif
+}
+
+
+static void
+prepare_overlay (GstElement * overlay, GstCaps * caps, gpointer user_data)
+{
+ CairoOverlayState *state = (CairoOverlayState *) user_data;
+
+ state->valid = gst_video_info_from_caps (&state->vinfo, caps);
+}
+
+static void
+draw_overlay_pose (GstElement * overlay, cairo_t * cr, guint64 timestamp,
+ guint64 duration, gpointer user_data)
+{
+ CairoOverlayState *s = (CairoOverlayState *) user_data;
+
+ if (!s->valid) {
+ printf("not ready draw_overlay");
+ return;
+ }
+ //printf("draw_overlay_pose");
+ cairo_set_source_rgba(cr, 0.1, 0.9, 0.0, 0.7);
+ cairo_set_line_width(cr, 2.0);
+
+
+ if (!humanSkeleton.IsDetected)
+ return;
+
+ //
+ if (humanSkeleton.isPrevPose == false) {
+ humanSkeleton.prevPose[1] = humanSkeleton.pose[1];
+ // head - neck
+ if (humanSkeleton.scores[0] >= thValNeck ) {
+ humanSkeleton.prevPose[0] = humanSkeleton.pose[0];
+ }
+
+ // right arm
+ if (humanSkeleton.scores[2] >= thValArm) {
+ // neck - right shoulder
+ humanSkeleton.prevPose[2] = humanSkeleton.pose[2];
+ if (humanSkeleton.scores[3] >= thValArm) {
+ // right shoulder - right elbow
+ humanSkeleton.prevPose[3] = humanSkeleton.pose[3];
+ if (humanSkeleton.scores[4] >= thValArm) {
+ // right elbow - right wrist
+ humanSkeleton.prevPose[4] = humanSkeleton.pose[4];
+ }
+ }
+ }
+
+ // left arm
+ if (humanSkeleton.scores[5] >= thValArm) {
+ // neck - right shoulder
+ humanSkeleton.prevPose[5] = humanSkeleton.pose[5];
+ if (humanSkeleton.scores[6] >= thValArm) {
+ // right shoulder - right elbow
+ humanSkeleton.prevPose[6] = humanSkeleton.pose[6];
+ if (humanSkeleton.scores[7] >= thValArm) {
+ // right elbow - right wrist
+ humanSkeleton.prevPose[7] = humanSkeleton.pose[7];
+ }
+ }
+ }
+
+ // right leg
+ if (humanSkeleton.scores[8] >= thValLeg) {
+ // neck - right shoulder
+ humanSkeleton.prevPose[8] = humanSkeleton.pose[8];
+ if (humanSkeleton.scores[9] >= thValLeg) {
+ // right shoulder - right elbow
+ humanSkeleton.prevPose[9] = humanSkeleton.pose[9];
+ if (humanSkeleton.scores[10] >= thValLeg) {
+ // right elbow - right wrist
+ humanSkeleton.prevPose[10] = humanSkeleton.pose[10];
+ }
+ }
+ }
+
+ // left leg
+ if (humanSkeleton.scores[11] >= thValLeg) {
+ // neck - right shoulder
+ humanSkeleton.prevPose[11] = humanSkeleton.pose[11];
+ if (humanSkeleton.scores[12] >= thValLeg) {
+ // right shoulder - right elbow
+ humanSkeleton.prevPose[12] = humanSkeleton.pose[12];
+ if (humanSkeleton.scores[13] >= thValLeg) {
+ // right elbow - right wrist
+ humanSkeleton.prevPose[13] = humanSkeleton.pose[13];
+ }
+ }
+ }
+ humanSkeleton.isPrevPose = true;
+ } else {
+ // weighted sum of pose and prevPose
+ // method1: fixed weights (pose : prevPose = 0.7 : 0.3)
+ float poseWeight = 0.7f;
+ float prevPoseWeight = 0.3f;
+ humanSkeleton.prevPose[1].x = (poseWeight * humanSkeleton.pose[1].x +
+ prevPoseWeight * humanSkeleton.prevPose[1].x);
+ humanSkeleton.prevPose[1].y = (poseWeight * humanSkeleton.pose[1].y +
+ prevPoseWeight * humanSkeleton.prevPose[1].y);
+ // head - neck
+ if (humanSkeleton.scores[0] >= thValNeck ) {
+ humanSkeleton.prevPose[0].x = (poseWeight * humanSkeleton.pose[0].x +
+ prevPoseWeight * humanSkeleton.prevPose[0].x);
+ humanSkeleton.prevPose[0].y = (poseWeight * humanSkeleton.pose[0].y +
+ prevPoseWeight * humanSkeleton.prevPose[0].y);
+ }
+
+ // right arm
+ if (humanSkeleton.scores[2] >= thValArm) {
+ // neck - right shoulder
+ humanSkeleton.prevPose[2].x = (poseWeight * humanSkeleton.pose[2].x +
+ prevPoseWeight * humanSkeleton.prevPose[2].x);
+ humanSkeleton.prevPose[2].y = (poseWeight * humanSkeleton.pose[2].y +
+ prevPoseWeight * humanSkeleton.prevPose[2].y);
+ if (humanSkeleton.scores[3] >= thValArm) {
+ // right shoulder - right elbow
+ humanSkeleton.prevPose[3].x = (poseWeight * humanSkeleton.pose[3].x +
+ prevPoseWeight * humanSkeleton.prevPose[3].x);
+ humanSkeleton.prevPose[3].y = (poseWeight * humanSkeleton.pose[3].y +
+ prevPoseWeight * humanSkeleton.prevPose[3].y);
+ if (humanSkeleton.scores[4] >= thValArm) {
+ // right elbow - right wrist
+ humanSkeleton.prevPose[4].x = (poseWeight * humanSkeleton.pose[4].x +
+ prevPoseWeight * humanSkeleton.prevPose[4].x);
+ humanSkeleton.prevPose[4].y = (poseWeight * humanSkeleton.pose[4].y +
+ prevPoseWeight * humanSkeleton.prevPose[4].y);
+ }
+ }
+ }
+
+ // left arm
+ if (humanSkeleton.scores[5] >= thValArm) {
+ // neck - right shoulder
+ humanSkeleton.prevPose[5].x = (poseWeight * humanSkeleton.pose[5].x +
+ prevPoseWeight * humanSkeleton.prevPose[5].x);
+ humanSkeleton.prevPose[5].y = (poseWeight * humanSkeleton.pose[5].y +
+ prevPoseWeight * humanSkeleton.prevPose[5].y);
+ if (humanSkeleton.scores[6] >= thValArm) {
+ // right shoulder - right elbow
+ humanSkeleton.prevPose[6].x = (poseWeight * humanSkeleton.pose[6].x +
+ prevPoseWeight * humanSkeleton.prevPose[6].x);
+ humanSkeleton.prevPose[6].y = (poseWeight * humanSkeleton.pose[6].y +
+ prevPoseWeight * humanSkeleton.prevPose[6].y);
+ if (humanSkeleton.scores[7] >= thValArm) {
+ // right elbow - right wrist
+ humanSkeleton.prevPose[7].x = (poseWeight * humanSkeleton.pose[7].x +
+ prevPoseWeight * humanSkeleton.prevPose[7].x);
+ humanSkeleton.prevPose[7].y = (poseWeight * humanSkeleton.pose[7].y +
+ prevPoseWeight * humanSkeleton.prevPose[7].y);
+ }
+ }
+ }
+
+ // right leg
+ if (humanSkeleton.scores[8] >= thValLeg) {
+ // neck - right shoulder
+ humanSkeleton.prevPose[8].x = (poseWeight * humanSkeleton.pose[8].x +
+ prevPoseWeight * humanSkeleton.prevPose[8].x);
+ humanSkeleton.prevPose[8].y = (poseWeight * humanSkeleton.pose[8].y +
+ prevPoseWeight * humanSkeleton.prevPose[8].y);
+ if (humanSkeleton.scores[9] >= thValLeg) {
+ // right shoulder - right elbow
+ humanSkeleton.prevPose[9].x = (poseWeight * humanSkeleton.pose[9].x +
+ prevPoseWeight * humanSkeleton.prevPose[9].x);
+ humanSkeleton.prevPose[9].y = (poseWeight * humanSkeleton.pose[9].y +
+ prevPoseWeight * humanSkeleton.prevPose[9].y);
+ if (humanSkeleton.scores[10] >= thValLeg) {
+ // right elbow - right wrist
+ humanSkeleton.prevPose[10].x = (poseWeight * humanSkeleton.pose[10].x +
+ prevPoseWeight * humanSkeleton.prevPose[10].x);
+ humanSkeleton.prevPose[10].y = (poseWeight * humanSkeleton.pose[10].y +
+ prevPoseWeight * humanSkeleton.prevPose[10].y);
+ }
+ }
+ }
+
+ // left leg
+ if (humanSkeleton.scores[11] >= thValLeg) {
+ // neck - right shoulder
+ humanSkeleton.prevPose[11].x = (poseWeight * humanSkeleton.pose[11].x +
+ prevPoseWeight * humanSkeleton.prevPose[11].x);
+ humanSkeleton.prevPose[11].y = (poseWeight * humanSkeleton.pose[11].y +
+ prevPoseWeight * humanSkeleton.prevPose[11].y);
+ if (humanSkeleton.scores[12] >= thValLeg) {
+ // right shoulder - right elbow
+ humanSkeleton.prevPose[12].x = (poseWeight * humanSkeleton.pose[12].x +
+ prevPoseWeight * humanSkeleton.prevPose[12].x);
+ humanSkeleton.prevPose[12].y = (poseWeight * humanSkeleton.pose[12].y +
+ prevPoseWeight * humanSkeleton.prevPose[12].y);
+ if (humanSkeleton.scores[13] >= thValLeg) {
+ // right elbow - right wrist
+ humanSkeleton.prevPose[13].x = (poseWeight * humanSkeleton.pose[13].x +
+ prevPoseWeight * humanSkeleton.prevPose[13].x);
+ humanSkeleton.prevPose[13].y = (poseWeight * humanSkeleton.pose[13].y +
+ prevPoseWeight * humanSkeleton.prevPose[13].y);
+ }
+ }
+ }
+ }
+
+ //
+ //draw..
+ // head - neck
+ if (humanSkeleton.scores[0] >= thValNeck ) {
+ cairo_move_to(cr, humanSkeleton.prevPose[0].x, humanSkeleton.prevPose[0].y);
+ cairo_line_to(cr, humanSkeleton.prevPose[1].x, humanSkeleton.prevPose[1].y);
+ }
+
+ // right arm
+ cairo_move_to(cr, humanSkeleton.prevPose[1].x, humanSkeleton.prevPose[1].y);
+ if (humanSkeleton.scores[2] >= thValArm) {
+ // neck - right shoulder
+ cairo_line_to(cr, humanSkeleton.prevPose[2].x, humanSkeleton.prevPose[2].y);
+ if (humanSkeleton.scores[3] >= thValArm) {
+ // right shoulder - right elbow
+ cairo_line_to(cr, humanSkeleton.prevPose[3].x, humanSkeleton.prevPose[3].y);
+ if (humanSkeleton.scores[4] >= thValArm) {
+ // right elbow - right wrist
+ cairo_line_to(cr, humanSkeleton.prevPose[4].x, humanSkeleton.prevPose[4].y);
+ }
+ }
+ }
+ cairo_stroke(cr);
+
+ // left arm
+ cairo_move_to(cr, humanSkeleton.prevPose[1].x, humanSkeleton.prevPose[1].y);
+ if (humanSkeleton.scores[5] >= thValArm) {
+ // neck - right shoulder
+ cairo_line_to(cr, humanSkeleton.prevPose[5].x, humanSkeleton.prevPose[5].y);
+ if (humanSkeleton.scores[6] >= thValArm) {
+ // right shoulder - right elbow
+ cairo_line_to(cr, humanSkeleton.prevPose[6].x, humanSkeleton.prevPose[6].y);
+ if (humanSkeleton.scores[7] >= thValArm) {
+ // right elbow - right wrist
+ cairo_line_to(cr, humanSkeleton.prevPose[7].x, humanSkeleton.prevPose[7].y);
+ }
+ }
+ }
+ cairo_stroke(cr);
+
+
+ // right leg
+ cairo_move_to(cr, humanSkeleton.prevPose[1].x, humanSkeleton.prevPose[1].y);
+ if (humanSkeleton.scores[8] >= thValLeg) {
+ // neck - right shoulder
+ cairo_line_to(cr, humanSkeleton.prevPose[8].x, humanSkeleton.prevPose[8].y);
+ if (humanSkeleton.scores[9] >= thValLeg) {
+ // right shoulder - right elbow
+ cairo_line_to(cr, humanSkeleton.prevPose[9].x, humanSkeleton.prevPose[9].y);
+ if (humanSkeleton.scores[10] >= thValLeg) {
+ // right elbow - right wrist
+ cairo_line_to(cr, humanSkeleton.prevPose[10].x, humanSkeleton.prevPose[10].y);
+ }
+ }
+ }
+ cairo_stroke(cr);
+
+ // left leg
+ cairo_move_to(cr, humanSkeleton.prevPose[1].x, humanSkeleton.prevPose[1].y);
+ if (humanSkeleton.scores[11] >= thValLeg) {
+ // neck - right shoulder
+ cairo_line_to(cr, humanSkeleton.prevPose[11].x, humanSkeleton.prevPose[11].y);
+ if (humanSkeleton.scores[12] >= thValLeg) {
+ // right shoulder - right elbow
+ cairo_line_to(cr, humanSkeleton.prevPose[12].x, humanSkeleton.prevPose[12].y);
+ if (humanSkeleton.scores[13] >= thValLeg) {
+ // right elbow - right wrist
+ cairo_line_to(cr, humanSkeleton.prevPose[13].x, humanSkeleton.prevPose[13].y);
+ }
+ }
+ }
+ cairo_stroke(cr);
+}
+
+static void
+draw_overlay_hand (GstElement * overlay, cairo_t * cr, guint64 timestamp,
+ guint64 duration, gpointer user_data)
+{
+ CairoOverlayState *s = (CairoOverlayState *) user_data;
+
+ if (!s->valid) {
+ printf("not ready draw_overlay");
+ return;
+ }
+
+ //printf("draw_overlay_hand");
+ cairo_set_source_rgba(cr, 0.0, 0.0, 1.0, 0.7);
+ cairo_set_line_width(cr, 2.0);
+
+
+ if (!humanSkeleton.IsDetected)
+ return;
+
+ /*printf("humanSkeleton.label [%d] \n", humanSkeleton.label);
+ if (humanSkeleton.label >= 0 && humanSkeleton.label <= 14) {
+ cairo_select_font_face(cr, "sans-serif", CAIRO_FONT_SLANT_NORMAL, CAIRO_FONT_WEIGHT_BOLD);
+ cairo_set_font_size(cr,15);
+ cairo_move_to(cr, 320.0, 48.0);
+ cairo_show_text(cr, gHandGestureLabel[humanSkeleton.label]);
+ }*/
+#define MOUSE_CONTROLLER
+#ifdef MOUSE_CONTROLLER
+ static int move_x = 0;
+ static int move_y = 0;
+ move_x = (WIDTH - humanSkeleton.pose[9].x) * 3;
+ move_y = humanSkeleton.pose[9].y * 2;
+
+ send_mouse_event_efl(move_x, move_y);
+ //printf("move_x [%d] move_y [%d] \n", move_x, move_y);
+#elif REMOTE_CONTROLLER
+ int fd = -1;
+ //int signal = 8;
+
+ static int skip = 0;
+
+ if (skip++%10!=0)
+ return;
+
+ fprintf(stderr, ">>> send start \n");
+ if (!sending && humanSkeleton.label == 2) {
+ sending = TRUE;
+ //send_signal_play(&signal);
+ //const char *path = "/dev/input/event2 --type 1 --code 105 --value 10";
+ send_event(0x004, 0x004, 0x101);
+ send_event(0x001, 0x069, 0x001);
+ send_event(0x000, 0x000, 0x000);
+ send_event(0x004, 0x004, 0x101);
+ send_event(0x000, 0x000, 0x000);
+ for (int i=0; i<5; i++) {
+ send_event(0x001, 0x069, 0x002);
+ send_event(0x000, 0x000, 0x001);
+ send_event(0x004, 0x004, 0x101);
+ send_event(0x001, 0x069, 0x001);
+ }
+ send_event(0x001, 0x069, 0x000);
+ send_event(0x000, 0x000, 0x000);
+ } else if (!sending && humanSkeleton.label == 3) {
+ sending = TRUE;
+ send_event(0x004, 0x004, 0x098);
+ send_event(0x001, 0x06a, 0x001);
+ send_event(0x000, 0x000, 0x000);
+ send_event(0x004, 0x004, 0x098);
+ send_event(0x000, 0x000, 0x000);
+ for (int i=0; i<5; i++) {
+ send_event(0x001, 0x06a, 0x002);
+ send_event(0x000, 0x000, 0x001);
+ send_event(0x004, 0x004, 0x098);
+ send_event(0x001, 0x06a, 0x001);
+ }
+ send_event(0x001, 0x06a, 0x000);
+ send_event(0x000, 0x000, 0x000);
+ } else if (!sending && humanSkeleton.label == 4) {
+ sending = TRUE;
+ send_event(0x004, 0x004, 0x096);
+ send_event(0x001, 0x067, 0x001);
+ send_event(0x000, 0x000, 0x000);
+ send_event(0x004, 0x004, 0x096);
+ send_event(0x000, 0x000, 0x000);
+ for (int i=0; i<5; i++) {
+ send_event(0x001, 0x067, 0x002);
+ send_event(0x000, 0x000, 0x001);
+ send_event(0x004, 0x004, 0x096);
+ send_event(0x001, 0x067, 0x001);
+ }
+ send_event(0x001, 0x067, 0x000);
+ send_event(0x000, 0x000, 0x000);
+
+ } else if (!sending && humanSkeleton.label == 5) {
+ sending = TRUE;
+ send_event(0x004, 0x004, 0x097);
+ send_event(0x001, 0x06c, 0x001);
+ send_event(0x000, 0x000, 0x000);
+ send_event(0x004, 0x004, 0x097);
+ send_event(0x000, 0x000, 0x000);
+ for (int i=0; i<5; i++) {
+ send_event(0x001, 0x06c, 0x002);
+ send_event(0x000, 0x000, 0x001);
+ send_event(0x004, 0x004, 0x097);
+ send_event(0x001, 0x06c, 0x001);
+ }
+ send_event(0x001, 0x06c, 0x000);
+ send_event(0x000, 0x000, 0x000);
+ }
+ fprintf(stderr, ">>> send end \n");
+ sending = FALSE;
+
+ cairo_set_source_rgba(cr, 0.1, 0.9, 0.0, 0.7);
+ cairo_set_line_width(cr, 2.0);
+
+ //printf("gHandGestureLabel[humanSkeleton.label] : [%d], x: %d y: %d",gHandGestureLabel[humanSkeleton.label], humanSkeleton.pose[0].x, humanSkeleton.pose[0].y);
+
+
+ // thumb - red
+ cairo_set_source_rgba (cr, 0.9, 0.1, 0.0, 0.7);
+ cairo_move_to(cr, humanSkeleton.pose[0].x, humanSkeleton.pose[0].y);
+ for (int k = 1 ; k < 5; ++k) {
+ cairo_line_to(cr, humanSkeleton.pose[k].x, humanSkeleton.pose[k].y);
+ }
+ cairo_stroke(cr);
+
+ // fore - red
+ cairo_set_source_rgba (cr, 0.9, 0.1, 0.0, 0.7);
+ cairo_move_to(cr, humanSkeleton.pose[0].x, humanSkeleton.pose[0].y);
+ for (int k = 5 ; k < 9; ++k) {
+ cairo_line_to(cr, humanSkeleton.pose[k].x, humanSkeleton.pose[k].y);
+ }
+ cairo_stroke(cr);
+
+ // middle - grean
+ cairo_set_source_rgba (cr, 0.1, 0.9, 0.0, 0.7);
+ cairo_move_to(cr, humanSkeleton.pose[0].x, humanSkeleton.pose[0].y);
+ for (int k = 9 ; k < 13; ++k) {
+ cairo_line_to(cr, humanSkeleton.pose[k].x, humanSkeleton.pose[k].y);
+ }
+ cairo_stroke(cr);
+
+ // ring - blue
+ cairo_set_source_rgba (cr, 0.1, 0.0, 0.9, 0.7);
+ cairo_move_to(cr, humanSkeleton.pose[0].x, humanSkeleton.pose[0].y);
+ for (int k = 13 ; k < 17; ++k) {
+ cairo_line_to(cr, humanSkeleton.pose[k].x, humanSkeleton.pose[k].y);
+ }
+ cairo_stroke(cr);
+
+ // little - purple
+ cairo_set_source_rgba (cr, 0.5, 0.0, 0.5, 0.7);
+ cairo_move_to(cr, humanSkeleton.pose[0].x, humanSkeleton.pose[0].y);
+ for (int k = 17 ; k < 21; ++k) {
+ cairo_line_to(cr, humanSkeleton.pose[k].x, humanSkeleton.pose[k].y);
+ }
+ cairo_stroke(cr);
+#endif
+
+}
+
+static gboolean bus_call (GstBus *bus, GstMessage *msg, gpointer data)
+{
+
+ switch (GST_MESSAGE_TYPE (msg)) {
+
+ case GST_MESSAGE_EOS:
+ printf ("End of stream\n");
+ break;
+
+ case GST_MESSAGE_ERROR: {
+ gchar *debug;
+ GError *error;
+
+ gst_message_parse_error (msg, &error, &debug);
+ g_free (debug);
+
+ printf ("Error: %s\n", error->message);
+ g_error_free (error);
+
+ break;
+ }
+ default:
+ break;
+ }
+
+ return TRUE;
+}
+
+int perform_armnn_human_pose_cpm_configure(mv_engine_config_h mv_engine_cfg)
+{
+ if (mv_engine_cfg == NULL) {
+ printf("mv_engine_cfg is null\n");
+ return MEDIA_VISION_ERROR_INVALID_PARAMETER;
+ }
+
+ char *inputNodeName = "image";
+ char *outputNodeName[1] = {"Convolutional_Pose_Machine/stage_5_out"};
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_WEIGHT_FILE_PATH,
+ PE_TFLITE_CPM_WEIGHT_PATH);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_DATA_TYPE,
+ MV_INFERENCE_DATA_FLOAT32);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_MEAN_VALUE,
+ 0.0);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_STD_VALUE,
+ 1.0);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_BACKEND_TYPE,
+ MV_INFERENCE_BACKEND_ARMNN);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_TARGET_DEVICE_TYPE,
+ MV_INFERENCE_TARGET_DEVICE_GPU);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_WIDTH,
+ 192);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_HEIGHT,
+ 192);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_CHANNELS,
+ 3);
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_NODE_NAME,
+ inputNodeName);
+
+ mv_engine_config_set_array_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_OUTPUT_NODE_NAMES,
+ outputNodeName,
+ 1);
+
+ return MEDIA_VISION_ERROR_NONE;
+}
+
+int perform_tflite_human_pose_cpm_configure(mv_engine_config_h mv_engine_cfg)
+{
+ if (mv_engine_cfg == NULL) {
+ printf("mv_engine_cfg is null\n");
+ return MEDIA_VISION_ERROR_INVALID_PARAMETER;
+ }
+
+ char *inputNodeName = "image";
+ char *outputNodeName[1] = {"Convolutional_Pose_Machine/stage_5_out"};
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_WEIGHT_FILE_PATH,
+ PE_TFLITE_CPM_WEIGHT_PATH);
+
+ mv_engine_config_set_string_attribute(
+ mv_engine_cfg,
+ MV_INFERENCE_MODEL_USER_FILE_PATH,
+ PE_POSE_LABEL_PATH);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_DATA_TYPE,
+ MV_INFERENCE_DATA_FLOAT32);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_MEAN_VALUE,
+ 0.0);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_STD_VALUE,
+ 1.0);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_BACKEND_TYPE,
+ MV_INFERENCE_BACKEND_ARMNN);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_TARGET_DEVICE_TYPE,
+ MV_INFERENCE_TARGET_DEVICE_CPU);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_WIDTH,
+ 192);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_HEIGHT,
+ 192);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_CHANNELS,
+ 3);
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_NODE_NAME,
+ inputNodeName);
+
+ mv_engine_config_set_array_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_OUTPUT_NODE_NAMES,
+ outputNodeName,
+ 1);
+
+ return MEDIA_VISION_ERROR_NONE;
+}
+
+int perform_tflite_hand_detection_AIC(mv_engine_config_h mv_engine_cfg)
+{
+ if (mv_engine_cfg == NULL) {
+ printf("mv_engine_cfg is null\n");
+ return MEDIA_VISION_ERROR_INVALID_PARAMETER;
+ }
+
+ char *inputNodeName = "input";
+ char *outputNodeNames[2] = {"mobilenetv2/boundingbox", "mobilenetv2/heatmap"};
+
+ //outputTensorData = (void*)calloc(56*56*21, sizeof(float));
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_WEIGHT_FILE_PATH,
+ PE_TFLITE_AIC_1_WEIGHT_PATH);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_DATA_TYPE,
+ MV_INFERENCE_DATA_FLOAT32);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_MEAN_VALUE,
+ 0.0);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_STD_VALUE,
+ 1.0);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_BACKEND_TYPE,
+ MV_INFERENCE_BACKEND_ARMNN);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_TARGET_DEVICE_TYPE,
+ MV_INFERENCE_TARGET_DEVICE_GPU);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_WIDTH,
+ 224);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_HEIGHT,
+ 224);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_CHANNELS,
+ 3);
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_NODE_NAME,
+ inputNodeName);
+
+ mv_engine_config_set_array_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_OUTPUT_NODE_NAMES,
+ outputNodeNames,
+ 2);
+
+ return MEDIA_VISION_ERROR_NONE;
+}
+
+int perform_tflite_hand_detection_AIC2(mv_engine_config_h mv_engine_cfg)
+{
+ char *inputNodeName = "input";
+ char *outputNodeNames[2] = {"mobilenetv2/coord_refine", "mobilenetv2/gesture"};
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_WEIGHT_FILE_PATH,
+ PE_TFLITE_AIC_2_WEIGHT_PATH);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_DATA_TYPE,
+ MV_INFERENCE_DATA_FLOAT32);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_MEAN_VALUE,
+ 0.0);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_STD_VALUE,
+ 1.0);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_BACKEND_TYPE,
+ MV_INFERENCE_BACKEND_ARMNN);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_TARGET_DEVICE_TYPE,
+ MV_INFERENCE_TARGET_DEVICE_GPU);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_WIDTH,
+ 56);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_HEIGHT,
+ 56);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_CHANNELS,
+ 21);
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_NODE_NAME,
+ inputNodeName);
+
+ mv_engine_config_set_array_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_OUTPUT_NODE_NAMES,
+ outputNodeNames,
+ 2);
+
+ return MEDIA_VISION_ERROR_NONE;
+}
+
+int perform_tflite_hand_detection_AICLite(mv_engine_config_h mv_engine_cfg)
+{
+ if (mv_engine_cfg == NULL) {
+ printf("mv_engine_cfg is null\n");
+ return MEDIA_VISION_ERROR_INVALID_PARAMETER;
+ }
+
+ char *inputNodeName = "input";
+ char *outputNodeNames[2] = {"mobilenetv2/boundingbox", "mobilenetv2/heatmap"};
+
+ //outputTensorData = (void*)calloc(56*56*21, sizeof(float));
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_WEIGHT_FILE_PATH,
+ PE_TFLITE_AICLite_1_WEIGHT_PATH);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_DATA_TYPE,
+ MV_INFERENCE_DATA_FLOAT32);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_MEAN_VALUE,
+ 0.0);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_STD_VALUE,
+ 1.0);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_BACKEND_TYPE,
+ MV_INFERENCE_BACKEND_ARMNN);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_TARGET_DEVICE_TYPE,
+ MV_INFERENCE_TARGET_DEVICE_GPU);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_WIDTH,
+ 224);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_HEIGHT,
+ 224);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_CHANNELS,
+ 3);
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_NODE_NAME,
+ inputNodeName);
+
+ mv_engine_config_set_array_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_OUTPUT_NODE_NAMES,
+ outputNodeNames,
+ 2);
+
+ return MEDIA_VISION_ERROR_NONE;
+}
+
+int perform_tflite_hand_detection_AICLite2(mv_engine_config_h mv_engine_cfg)
+{
+ char *inputNodeName = "input";
+ char *outputNodeNames[2] = {"mobilenetv2/coord_refine", "mobilenetv2/gesture"};
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_WEIGHT_FILE_PATH,
+ PE_TFLITE_AICLite_2_WEIGHT_PATH);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_DATA_TYPE,
+ MV_INFERENCE_DATA_FLOAT32);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_MEAN_VALUE,
+ 0.0);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_STD_VALUE,
+ 1.0);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_BACKEND_TYPE,
+ MV_INFERENCE_BACKEND_ARMNN);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_TARGET_DEVICE_TYPE,
+ MV_INFERENCE_TARGET_DEVICE_GPU);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_WIDTH,
+ 56);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_HEIGHT,
+ 56);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_CHANNELS,
+ 21);
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_NODE_NAME,
+ inputNodeName);
+
+ mv_engine_config_set_array_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_OUTPUT_NODE_NAMES,
+ outputNodeNames,
+ 2);
+
+ return MEDIA_VISION_ERROR_NONE;
+}
+
+int perform_tflite_hand_detection_AICLiteQ(mv_engine_config_h mv_engine_cfg)
+{
+ if (mv_engine_cfg == NULL) {
+ printf("mv_engine_cfg is null\n");
+ return MEDIA_VISION_ERROR_INVALID_PARAMETER;
+ }
+
+ char *inputNodeName = "input";
+ char *outputNodeNames[2] = {"mobilenetv2/boundingbox", "mobilenetv2/heatmap"};
+
+ //outputTensorData = (void*)calloc(56*56*21, sizeof(char));
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_WEIGHT_FILE_PATH,
+ PE_TFLITE_AICLiteQ_1_WEIGHT_PATH);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_DATA_TYPE,
+ MV_INFERENCE_DATA_FLOAT32);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_MEAN_VALUE,
+ 0.0);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_STD_VALUE,
+ 1.0);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_BACKEND_TYPE,
+ MV_INFERENCE_BACKEND_ARMNN);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_TARGET_DEVICE_TYPE,
+ MV_INFERENCE_TARGET_DEVICE_GPU);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_WIDTH,
+ 224);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_HEIGHT,
+ 224);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_CHANNELS,
+ 3);
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_NODE_NAME,
+ inputNodeName);
+
+ mv_engine_config_set_array_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_OUTPUT_NODE_NAMES,
+ outputNodeNames,
+ 2);
+
+ return MEDIA_VISION_ERROR_NONE;
+}
+
+int perform_tflite_hand_detection_AICLiteQ2(mv_engine_config_h mv_engine_cfg)
+{
+ char *inputNodeName = "input";
+ char *outputNodeNames[2] = {"mobilenetv2/coord_refine", "mobilenetv2/gesture"};
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_WEIGHT_FILE_PATH,
+ PE_TFLITE_AICLiteQ_2_WEIGHT_PATH);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_DATA_TYPE,
+ MV_INFERENCE_DATA_FLOAT32);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_MEAN_VALUE,
+ 0.0);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_STD_VALUE,
+ 1.0);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_BACKEND_TYPE,
+ MV_INFERENCE_BACKEND_ARMNN);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_TARGET_DEVICE_TYPE,
+ MV_INFERENCE_TARGET_DEVICE_GPU);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_WIDTH,
+ 56);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_HEIGHT,
+ 56);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_CHANNELS,
+ 21);
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_NODE_NAME,
+ inputNodeName);
+
+ mv_engine_config_set_array_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_OUTPUT_NODE_NAMES,
+ outputNodeNames,
+ 2);
+
+ return MEDIA_VISION_ERROR_NONE;
+}
+
+static void __global(void *data, struct wl_registry *registry,
+ uint32_t name, const char *interface, uint32_t version)
+{
+ struct tizen_surface **tz_surface = NULL;
+
+ if (!data) {
+ printf("NULL data\n");
+ return;
+ }
+
+ tz_surface = (struct tizen_surface **)data;
+
+ if (!interface) {
+ printf("NULL interface\n");
+ return;
+ }
+
+ if (strcmp(interface, "tizen_surface") == 0) {
+ printf("binding tizen surface for wayland\n");
+
+ *tz_surface = wl_registry_bind(registry, name, &tizen_surface_interface, 1);
+ if (*tz_surface == NULL)
+ printf("failed to bind\n");
+
+ printf("done\n");
+ }
+
+ return;
+}
+
+static void __global_remove(void *data, struct wl_registry *wl_registry, uint32_t name)
+{
+ printf("enter\n");
+ return;
+}
+
+static const struct wl_registry_listener _wl_registry_listener = {
+ __global,
+ __global_remove
+};
+
+void __parent_id_getter(void *data, struct tizen_resource *tizen_resource, uint32_t id)
+{
+ if (!data) {
+ printf("NULL data\n");
+ return;
+ }
+
+ *((unsigned int *)data) = id;
+
+ printf("[CLIENT] got parent_id [%u] from server\n", id);
+
+ return;
+}
+
+static const struct tizen_resource_listener _tz_resource_listener = {
+ __parent_id_getter
+};
+
+static void set_overlay(Ecore_Evas *ee)
+{
+ Ecore_Wl2_Window *window = NULL;
+ Ecore_Wl2_Display *e_wl2_display = NULL;
+
+ struct wl_display *display = NULL;
+ struct wl_display *display_wrapper = NULL;
+ struct wl_surface *surface = NULL;
+ struct wl_registry *registry = NULL;
+ struct wl_event_queue *queue = NULL;
+ struct tizen_surface *tz_surface = NULL;
+ struct tizen_resource *tz_resource = NULL;
+
+ window = ecore_evas_wayland2_window_get(ee);
+ if (!window) {
+ printf("failed to get wayland window\n");
+ goto _DONE;
+ }
+
+ /* set video_has flag to a video application window */
+ ecore_wl2_window_video_has(window, EINA_TRUE);
+
+ surface = (struct wl_surface *)ecore_wl2_window_surface_get(window);
+ if (!surface) {
+ printf("failed to get wayland surface\n");
+ goto _DONE;
+ }
+
+ e_wl2_display = ecore_wl2_connected_display_get(NULL);
+ if (!e_wl2_display) {
+ printf("failed to get ecore wl2 display\n");
+ goto _DONE;
+ }
+
+ display = (struct wl_display *)ecore_wl2_display_get(e_wl2_display);
+ if (!display) {
+ printf("failed to get wayland display\n");
+ goto _DONE;
+ }
+
+ display_wrapper = wl_proxy_create_wrapper(display);
+ if (!display_wrapper) {
+ printf("failed to create wl display wrapper\n");
+ }
+
+ queue = wl_display_create_queue(display);
+ if (!queue) {
+ printf("failed to create wl display queue\n");
+ goto _DONE;
+ }
+
+ wl_proxy_set_queue((struct wl_proxy *)display_wrapper, queue);
+
+ registry = wl_display_get_registry(display_wrapper);
+ if (!registry) {
+ printf("failed to get wayland registry\n");
+ goto _DONE;
+ }
+
+ wl_registry_add_listener(registry, &_wl_registry_listener, &tz_surface);
+
+ wl_display_dispatch_queue(display, queue);
+ wl_display_roundtrip_queue(display, queue);
+
+ if (!tz_surface) {
+ printf("failed to get tizen surface\n");
+ goto _DONE;
+ }
+
+ /* Get parent_id which is unique in a entire systemw. */
+ tz_resource = tizen_surface_get_tizen_resource(tz_surface, surface);
+ if (!tz_resource) {
+ printf("failed to get tizen resurce\n");
+ goto _DONE;
+ }
+
+ int parent_id = 0;
+
+ tizen_resource_add_listener(tz_resource, &_tz_resource_listener, &parent_id);
+
+ wl_display_roundtrip_queue(display, queue);
+
+ if (parent_id > 0) {
+ printf("parent id : %u\n", parent_id);
+
+ gst_video_overlay_set_wl_window_wl_surface_id(GST_VIDEO_OVERLAY(sink2), parent_id);
+ gst_element_set_state (pipeline, GST_STATE_PLAYING);
+ } else {
+ printf("failed to get parent id\n");
+ }
+
+_DONE:
+ if (tz_surface) {
+ tizen_surface_destroy(tz_surface);
+ tz_surface = NULL;
+ }
+
+ if (tz_resource) {
+ tizen_resource_destroy(tz_resource);
+ tz_resource = NULL;
+ }
+
+ if (registry) {
+ wl_registry_destroy(registry);
+ registry = NULL;
+ }
+
+ if (queue) {
+ wl_event_queue_destroy(queue);
+ queue = NULL;
+ }
+
+ if (display_wrapper) {
+ wl_proxy_wrapper_destroy(display_wrapper);
+ display_wrapper = NULL;
+ }
+}
+
+static void win_resize_cb (void *data, Evas *e, Evas_Object *obj, void *event_info)
+{
+ int w, h;
+ int wid;
+
+ printf("_canvas_resize_cb\n");
+
+ Evas * _e = evas_object_evas_get(obj);
+ Ecore_Evas *ee = ecore_evas_ecore_evas_get(_e);
+ ecore_evas_geometry_get(ee, NULL, NULL, &w, &h);
+
+ if (!st) {
+ set_overlay(ee);
+ int x, y;
+ elm_win_screen_position_get(obj, &x, &y);
+ printf("x = %d, y = %d\n", x, y);
+ } else
+ st++;
+}
+
+static void cb_new_pad (GstElement *element, GstPad *pad, gpointer data)
+{
+ gchar *name;
+ GstElement *other = data;
+
+ name = gst_pad_get_name (pad);
+ printf ("A new pad %s was created for %s\n", name, gst_element_get_name(element));
+ g_free (name);
+
+ printf ("element %s will be linked to %s\n",
+ gst_element_get_name(element),
+ gst_element_get_name(dscale));
+ gst_element_link(element, dscale);
+}
+
+static int app_create(void *data)
+{
+ appdata *ad = data;
+ Evas_Object *win = NULL;
+
+ // GST
+ g_mutex_init(&pose_mutex);
+
+ signal(SIGINT, int_handler);
+
+ /* initialization */
+ gst_init(NULL, NULL);
+
+ /* mediavision configuration*/
+ IsGestureMode = false;
+ if (hp_mv_engine_cfg == NULL) {
+ mv_create_engine_config(&hp_mv_engine_cfg);
+ }
+
+ if (hp_mv_engine_cfg2 == NULL) {
+ mv_create_engine_config(&hp_mv_engine_cfg2);
+ }
+
+ int err = MEDIA_VISION_ERROR_NONE;
+
+
+ if (ad->modelType == MODEL_TYPE_POSE_CPM) {
+ //err = perform_armnn_human_pose_cpm_configure(hp_mv_engine_cfg);
+ err = perform_tflite_human_pose_cpm_configure(hp_mv_engine_cfg);
+ } else if (ad->modelType == MODEL_TYPE_POSE_HAND_AICLite) {
+ outputTensorData = (void*)calloc(56*56*21, sizeof(float));
+ err = perform_tflite_hand_detection_AIC(hp_mv_engine_cfg);
+
+ err = perform_tflite_hand_detection_AIC2(hp_mv_engine_cfg2);
+ } else if (ad->modelType == MODEL_TYPE_POSE_HAND_AICLite2) {
+ outputTensorData = (void*)calloc(56*56*21, sizeof(float));
+ err = perform_tflite_hand_detection_AICLite(hp_mv_engine_cfg);
+
+ err = perform_tflite_hand_detection_AICLite2(hp_mv_engine_cfg2);
+ } else {
+ outputTensorData = (void*)calloc(56*56*21, sizeof(float));
+ err = perform_tflite_hand_detection_AICLiteQ(hp_mv_engine_cfg);
+
+ err = perform_tflite_hand_detection_AICLiteQ2(hp_mv_engine_cfg2);
+ }
+
+ if (err != MEDIA_VISION_ERROR_NONE) {
+ printf("Error on perform_armnn_human_pose_configure");
+ }
+
+ printf("configuration done\n");
+
+ printf("loading pose model: ");
+ mv_inference_create(&hp_mv_infer);
+
+ mv_inference_configure(hp_mv_infer, hp_mv_engine_cfg);
+
+ clock_t start = clock();
+ mv_inference_prepare(hp_mv_infer);
+ clock_t end = clock();
+ printf("time: %2.3f\n", (double)(end - start)/CLOCKS_PER_SEC);
+
+ /* mediavision source */
+ mv_create_source(&mv_src_p);
+
+ if (ad->modelType >= MODEL_TYPE_POSE_HAND_AICLite) {
+ mv_inference_create(&hp_mv_infer2);
+
+ mv_inference_configure(hp_mv_infer2, hp_mv_engine_cfg2);
+
+ start = clock();
+ mv_inference_prepare(hp_mv_infer2);
+ end = clock();
+ printf("time: %2.3f\n", (double)(end - start)/CLOCKS_PER_SEC);
+
+ mv_create_source(&mv_src_p2);
+ }
+
+ pipeline = gst_pipeline_new("app");
+
+ overlay_state = g_new0 (CairoOverlayState, 1);
+
+ /* create gstreamer elements */
+ if (!ad->filename) {
+ source = gst_element_factory_make("v4l2src", "src");
+ filter = gst_element_factory_make("capsfilter", "filter");
+ } else {
+ source = gst_element_factory_make("filesrc", "src");
+
+ dbin = gst_element_factory_make("decodebin", "dbin");
+ dscale = gst_element_factory_make("videoscale", "dscale");
+ dconv = gst_element_factory_make("videoconvert", "dconv");
+ dsfilter = gst_element_factory_make("capsfilter", "dsfilter");
+ dcfilter = gst_element_factory_make("capsfilter", "dcfilter");
+ }
+
+ if (ad->filename2) {
+ tee2 = gst_element_factory_make("tee", "tee2");
+ enc = gst_element_factory_make("avenc_mpeg4", "enc");
+ muxmp4 = gst_element_factory_make("mp4mux", "muxmp4");
+ fsink2 = gst_element_factory_make("filesink", "fsink2");
+ queue4 = gst_element_factory_make("queue", "queue4");
+ queue5 = gst_element_factory_make("queue", "queue5");
+ encconv = gst_element_factory_make("videoconvert", "encconv");
+ }
+
+
+ tee = gst_element_factory_make("tee", "tee");
+ queue1 = gst_element_factory_make("queue", "queue1");
+ queue2 = gst_element_factory_make("queue", "queue2");
+
+ if (0 /*ad->modelType == MODEL_TYPE_POSE_HAND_AIC*/) {
+ queue3 = gst_element_factory_make("queue", "queue3");
+ }
+
+ // queue1 - videoscale - capsfilter -viedoeconvert - capsfilter - videorate - capsfilter -fakesink
+ vscale = gst_element_factory_make("videoscale", "scale");
+ vsfilter = gst_element_factory_make("capsfilter", "vsfilter");
+ vconv = gst_element_factory_make("videoconvert", "convert");
+ vcfilter = gst_element_factory_make("capsfilter", "vcfilter");
+ vrate = gst_element_factory_make("videorate", "rate");
+ vrfilter = gst_element_factory_make("capsfilter", "vrfilter");
+ vrsink = gst_element_factory_make("fakesink", "vrsink");
+
+ // queue2 - videoconvert - cairooveray - tizenwlsink
+ oconv = gst_element_factory_make("videoconvert", "oconv");
+ coverlay = gst_element_factory_make("cairooverlay", "coverlay");
+ sink = gst_element_factory_make("fpsdisplaysink", "vsink");
+ sink2 = gst_element_factory_make("fakesink", "vsink2");
+
+ // after detection, crop using video crop
+ // queue3 - videocrop - videoscale -capsfilter - videoconvert - capsfilter -fakesink
+
+ if (0/*ad->modelType == MODEL_TYPE_POSE_HAND_AIC*/) {
+ vcrop = gst_element_factory_make("videocrop", "crop");
+ vcrscale = gst_element_factory_make("videoscale", "crscale");
+ vcrsfilter = gst_element_factory_make("capsfilter", "vcrsfilter");
+ vcrsconv = gst_element_factory_make("videoconvert", "vcrsconvert");
+ vcrscfilter = gst_element_factory_make("capsfilter", "vcrscfilter");
+ vcrssink = gst_element_factory_make("fakesink", "vcrssink");
+ }
+
+ if (!pipeline || !source ||
+ !tee || !queue1 || !vscale || !vsfilter || !vconv || !vcfilter ||
+ !vrate || !vrfilter || !vrsink ||
+ !queue2 || !oconv || !coverlay || !sink || !sink2) {
+ printf(TEXT_RED "One element(queue1 or queue2) might be not created. Exiting.\n" TEXT_RESET);
+ return -1;
+ }
+
+ if (0 /*ad->modelType == MODEL_TYPE_POSE_HAND_AIC && (!pipeline || !queue3
+ || !vcrop || !vcrscale || !vcrsfilter || !vcrsconv || !vcrscfilter || !vcrssink*/) {
+ printf(TEXT_RED "One element(queue3) might be not created. Exiting.\n" TEXT_RESET);
+ return -1;
+ }
+
+ if (!ad->filename) {
+ if (!filter) {
+ printf(TEXT_RED "One element might be not created. Existing.\n" TEXT_RESET);
+ return -1;
+ }
+ g_object_set(G_OBJECT(filter), "caps", gst_caps_from_string("video/x-raw, format=YUY2, width=640, height=480"), NULL);
+ } else {
+ if (!dbin || !dscale || !dconv) {
+ printf(TEXT_RED "One element might be not created. Exiting.\n" TEXT_RESET);
+ return -1;
+ }
+ g_object_set(G_OBJECT(dsfilter), "caps", gst_caps_from_string("video/x-raw, width=640, height=480"), NULL);
+ g_object_set(G_OBJECT(dcfilter), "caps", gst_caps_from_string("video/x-raw, format=YUY2, width=640, height=480"), NULL);
+ g_signal_connect (dbin, "pad-added", G_CALLBACK (cb_new_pad), NULL);
+ }
+
+ if (ad->filename2) {
+ if (!tee2 || !enc || !muxmp4 || !queue4 || !queue5 || !fsink2 || !encconv) {
+ printf(TEXT_RED "One element might be not created. Exiting.\n" TEXT_RESET);
+ printf("%p, %p, %p, %p, %p, %p, %p\n", tee2, enc, muxmp4, queue4, queue5, fsink2, encconv);
+ return -1;
+ }
+ g_object_set(G_OBJECT(fsink2), "location", ad->filename2, NULL);
+
+ }
+
+ /* set up the pipeline */
+ //g_signal_connect (coverlay, "draw", G_CALLBACK (draw_overlay), overlay_state);
+ if (ad->modelType == MODEL_TYPE_POSE_CPM) {
+ g_signal_connect (coverlay, "draw", G_CALLBACK (draw_overlay_pose), overlay_state);
+ } else {
+ g_signal_connect (coverlay, "draw", G_CALLBACK (draw_overlay_hand), overlay_state);
+ }
+
+ g_signal_connect (coverlay, "caps-changed", G_CALLBACK (prepare_overlay), overlay_state);
+
+ if (!ad->filename) {
+ g_object_set(G_OBJECT(source), "device", "/dev/video252", NULL); // 252
+ } else {
+ g_object_set(G_OBJECT(source), "location", ad->filename, NULL);
+ //g_object_set(G_OBJECT(source), "num-buffers", ad->numbuffers, NULL);
+ }
+
+ //g_object_set(G_OBJECT(sink), "text-overlay", FALSE, NULL);
+ //g_object_set(G_OBJECT(sink2), "use-tbm", FALSE, NULL);
+ g_object_set(G_OBJECT(sink2), "sync", FALSE, NULL);
+ g_object_set(G_OBJECT(sink), "video-sink", sink2, NULL);
+ g_object_set(G_OBJECT(sink), "sync", FALSE, NULL);
+
+ if (ad->modelType == MODEL_TYPE_POSE_HAND_AICLite ||
+ ad->modelType == MODEL_TYPE_POSE_HAND_AICLite2 ||
+ ad->modelType == MODEL_TYPE_POSE_HAND_AICLite2Q) {
+ g_object_set(G_OBJECT(vsfilter), "caps", gst_caps_from_string("video/x-raw, width=224, height=224"), NULL);
+ poseCropSize = 224;
+ } else {
+ g_object_set(G_OBJECT(vsfilter), "caps", gst_caps_from_string("video/x-raw, width=192, height=192"), NULL);
+ poseCropSize = 192;
+ }
+
+ g_object_set(G_OBJECT(vcfilter), "caps", gst_caps_from_string("video/x-raw, format=RGB"), NULL);
+ g_object_set(G_OBJECT(vrfilter), "caps", gst_caps_from_string("video/x-raw, framerate=15/1"), NULL);
+ g_object_set(G_OBJECT(queue2), "max-size-buffers", 1, NULL);
+ //g_object_set(G_OBJECT(vrate), "drop-only", TRUE, NULL);
+
+ //g_object_set(G_OBJECT(queue2), "leaky", 2, NULL);
+#if 0
+ g_object_set(G_OBJECT(queue3), "max-size-buffers", 0, NULL);
+ g_object_set(G_OBJECT(queue3), "max-size-time", 0, NULL);
+ g_object_set(G_OBJECT(queue3), "max-size-bytes", 0, NULL);
+#endif
+ g_object_set(G_OBJECT(queue3), "max-size-buffers", 1, NULL);
+ //g_object_set(G_OBJECT(queue3), "leaky", 2, NULL);
+
+ // here to be continue
+ printf("vrsink signal-handoffs\n");
+ g_object_set(G_OBJECT(vrsink), "signal-handoffs", TRUE, NULL);
+
+ if (ad->modelType == MODEL_TYPE_POSE_CPM) {
+ handler_p = g_signal_connect (vrsink, "handoff", G_CALLBACK(_pose_est_handoff), outputTensorData);
+ } else {
+ handler_p = g_signal_connect (vrsink, "handoff", G_CALLBACK(_hand_est_handoff), outputTensorData);
+ }
+
+ g_object_set(G_OBJECT(vrsink), "sync", FALSE, NULL);
+
+
+ g_object_set(G_OBJECT(vcrssink), "sync", FALSE, NULL);
+
+
+ /* add a message handler */
+ bus = gst_pipeline_get_bus (GST_PIPELINE(pipeline));
+ bus_watch_id = gst_bus_add_watch(bus, bus_call, NULL);
+ gst_object_unref(bus);
+
+ /* add elements into the pipeline */
+ gst_bin_add_many(GST_BIN(pipeline),
+ source,
+ tee, queue1, vscale, vsfilter, vconv, vcfilter,
+ vrate, vrfilter, vrsink,
+ queue2, oconv, coverlay, sink,
+ NULL);
+
+ if (0 /*ad->modelType == MODEL_TYPE_POSE_HAND_AIC*/) {
+ gst_bin_add_many(GST_BIN(pipeline),
+ queue3, vcrop, vcrscale, vcrsfilter, vcrsconv, vcrscfilter, vcrssink,
+ NULL);
+ gst_element_link_many (tee, queue3, vcrop, vcrscale, vcrsfilter, vcrsconv, vcrscfilter, vcrssink, NULL);
+ }
+
+ /* link elements */
+ if (!ad->filename) {
+ gst_bin_add(GST_BIN(pipeline), filter);
+ gst_element_link_many(source, filter, tee, NULL);
+ } else {
+ gst_bin_add_many(GST_BIN(pipeline), dbin, dscale, dconv, dsfilter, dcfilter, NULL);
+ gst_element_link_many(source, dbin, NULL);
+ gst_element_link_many(dscale, dsfilter, dconv, dcfilter, tee, NULL);
+ }
+
+ if (!ad->filename2) {
+ // display
+ gst_element_link_many (tee, queue2, oconv, coverlay, /*toverlay,*/ sink, NULL);
+ // pose
+ gst_element_link_many (tee, queue1, vrate, vrfilter, vconv, vcfilter, vscale, vsfilter, vrsink, NULL);
+ } else {
+
+ gst_bin_add_many(GST_BIN(pipeline), tee2, enc, muxmp4, fsink2, queue4, queue5, encconv);
+
+ // display
+ gst_element_link_many (tee, queue2, oconv, coverlay, tee2, NULL);
+ gst_element_link_many (tee2, queue4, sink, NULL);
+ gst_element_link_many (tee2, queue5, encconv, enc, muxmp4, fsink2, NULL);
+
+ // pose
+ gst_element_link_many (tee, queue1, vrate, vrfilter, vconv, vcfilter, vscale, vsfilter, vrsink, NULL);
+ }
+
+ /* set the pipeline state to "playing" state */
+ //gst_element_set_state(pipeline, GST_STATE_PLAYING);
+
+ /* loop */
+ humanSkeleton.IsDetected = false;
+ humanSkeleton.isPrevPose = false;
+ humanSkeleton.label = -1;
+ printf(TEXT_GREEN "Running.....\n" TEXT_RESET);
+ // GST_END
+#if 0
+ /* use gl backend */
+ elm_config_accel_preference_set("opengl");
+
+ /* create window */
+ win = elm_win_add(NULL, PACKAGE, ELM_WIN_SPLASH );
+ //win = elm_win_add(NULL, PACKAGE, ELM_WIN_BASIC);
+ if (win) {
+ elm_win_title_set(win, PACKAGE);
+ elm_win_borderless_set(win, EINA_TRUE);
+ elm_win_autodel_set(win, EINA_TRUE);
+ elm_win_alpha_set(win, EINA_FALSE);
+ evas_object_show(win);
+ }
+ elm_win_layer_set(win, 9);
+ elm_win_prop_focus_skip_set(win, EINA_TRUE);
+
+ ad->win = win;
+ g_win_id = win;
+ selected_win_id = g_win_id;
+
+ Evas_Object *bg = elm_bg_add(win);
+ elm_win_resize_object_add(win, bg);
+ evas_object_size_hint_min_set(bg, WIDTH, HEIGHT);
+ evas_object_size_hint_max_set(bg, WIDTH, HEIGHT);
+ evas_object_show(bg);
+
+ elm_win_activate(win);
+
+
+ evas_object_event_callback_add(win, EVAS_CALLBACK_RESIZE, win_resize_cb, NULL);
+#else
+ gst_element_set_state (pipeline, GST_STATE_PLAYING);
+#endif
+ return 0;
+}
+
+
+static int app_terminate(void *data)
+{
+ appdata *ad = data;
+ int i = 0;
+
+ /* out of loop */
+ printf(TEXT_GREEN "Stopping.....\n" TEXT_RESET);
+ gst_element_set_state(pipeline, GST_STATE_NULL);
+
+ printf(TEXT_GREEN "pipe unref.....\n" TEXT_RESET);
+ gst_object_unref(GST_OBJECT(pipeline));
+
+ printf(TEXT_GREEN "remove bus_watch id.....\n" TEXT_RESET);
+ g_source_remove(bus_watch_id);
+
+ if (overlay_state != NULL) {
+ printf(TEXT_GREEN "g_free overlay.....\n" TEXT_RESET);
+ g_free(overlay_state);
+ }
+
+ g_mutex_clear(&pose_mutex);
+ printf(TEXT_GREEN "done.....\n" TEXT_RESET);
+
+ if (g_win_id) {
+ evas_object_del(g_win_id);
+ g_win_id = NULL;
+ }
+ ad->win = NULL;
+ selected_win_id = NULL;
+ free(my_cursor);
+ return 0;
+}
+struct appcore_ops ops = {
+ .create = app_create,
+ .terminate = app_terminate,
+};
+
+int main (int argc, char *argv[])
+{
+ memset(&ad, 0x0, sizeof(appdata));
+ ops.data = &ad;
+
+ if (argc >= 6) {
+ ad.filename = g_strdup(argv[5]);
+ printf("launch with file source (%s)\n", ad.filename);
+ if (argc > 6) {
+ ad.filename2 = g_strdup(argv[6]);
+ ad.numbuffers = -1;
+ printf("records output(%s)\n", ad.filename2);
+ }
+ if (argc > 7) {
+ ad.numbuffers = atoi(argv[7]);
+ }
+ } else {
+ printf("launch with camera source\n");
+ }
+
+ if (argc < 2) {
+ printf("usage: mv_stream_infer model [NeckThresVal, ArmThresVal, LegThresVal, [filename]]");
+ printf("model: 0(CPM), 1(AIC Hand), 2(AIC Lite Hand), 3(AIC Lite Q Hand)\n");
+ return -1;
+ }
+
+ ad.modelType = atoi(argv[1]);
+ if (ad.modelType < 0 || ad.modelType > 3) {
+ printf("not supported model type [%d]\n", ad.modelType);
+ return -1;
+ }
+
+ if (ad.modelType != MODEL_TYPE_POSE_HAND_AICLite &&
+ ad.modelType != MODEL_TYPE_POSE_HAND_AICLite2 &&
+ ad.modelType != MODEL_TYPE_POSE_HAND_AICLite2Q) {
+ thValNeck = (float)atoi(argv[2])/100.f;
+ thValArm = (float)atoi(argv[3])/100.f;
+ thValLeg = (float)atoi(argv[4])/100.f;
+
+ poseRoi.point.x = 0;
+ poseRoi.point.y = 0;
+ poseRoi.width = 0;
+ poseRoi.height = 0;
+ } else {
+ if (argc > 2) {
+ ad.filename2 = g_strdup(argv[2]);
+ }
+ poseRoi.point.x = 0;
+ poseRoi.point.y = 0;
+ poseRoi.width = 0;
+ poseRoi.height = 0;
+ }
+ my_cursor = efl_util_input_initialize_generator(EFL_UTIL_INPUT_DEVTYPE_POINTER);
+
+ return appcore_efl_main(PACKAGE, &argc, &argv, &ops);
+}
+
*/
#define _USE_MATH_DEFINES
-#if 0
-#include <evemu.h>
-#endif
#include <fcntl.h>
#include <mv_common.h>
#include <mv_inference.h>
static guint nFrames = 0;
// Gstreamer
-GstElement *pipeline, *source, *filter, *toverlay, *sink, *sink2;
+GstElement *pipeline, *source, *flip, *filter, *toverlay, *sink, *sink2;
GstElement *tee, *vscale, *vsfilter, *vconv, *vcfilter;
GstElement *vrate, *vrfilter, *fsink, *vrsink;
GstElement *queue1, *queue2, *queue3;
g_object_set(G_OBJECT(enc), "bitrate", 800000, NULL);
}
-
+ flip = gst_element_factory_make("videoflip", "vflip" );
tee = gst_element_factory_make("tee", "tee");
queue1 = gst_element_factory_make("queue", "queue1");
queue2 = gst_element_factory_make("queue", "queue2");
} else {
g_object_set(G_OBJECT(source), "location", ad->filename, NULL);
}
-
+ g_object_set(G_OBJECT(flip), "method", 4, NULL);
g_object_set(G_OBJECT(sink2), "use-tbm", FALSE, NULL);
g_object_set(G_OBJECT(sink2), "sync", FALSE, NULL);
g_object_set(G_OBJECT(sink), "video-sink", sink2, NULL);
/* add elements into the pipeline */
gst_bin_add_many(GST_BIN(pipeline),
source,
- tee, queue1, vscale, vsfilter, vconv, vcfilter,
+ flip, tee, queue1, vscale, vsfilter, vconv, vcfilter,
vrate, vrfilter, vrsink,
queue2, oconv, coverlay, sink,
NULL);
/* link elements */
if (!ad->filename) {
gst_bin_add(GST_BIN(pipeline), filter);
- gst_element_link_many(source, filter, tee, NULL);
+ gst_element_link_many(source, flip, filter, tee, NULL);
} else {
gst_bin_add_many(GST_BIN(pipeline), dbin, dscale, dconv, dsfilter, dcfilter, NULL);
gst_element_link_many(source, dbin, NULL);
}
return appcore_efl_main(PACKAGE, &argc, &argv, &ops);
-}
\ No newline at end of file
+}
*/
#define _USE_MATH_DEFINES
-#if 0
-#include <evemu.h>
-#endif
#include <fcntl.h>
#include <mv_common.h>
#include <mv_inference.h>
*/
#define _USE_MATH_DEFINES
-#if 0
-#include <evemu.h>
-#endif
#include <fcntl.h>
#include <mv_common.h>
#include <mv_inference.h>
#endif
#define PACKAGE "test"
+static char* gHandGestureLabel[] = { "None", "One", "Two", "Three", "Four", "Five",
+ "Six", "Seven", "SmallHeart", "OK", "DisLike", "Like",
+ "Fist", "Rock", "Love"};
static int st = 0;
static Evas_Object *g_eo = NULL;
static Evas_Object *icon = NULL;
typedef enum {
MODEL_TYPE_POSE_CPM = 0,
- MODEL_TYPE_POSE_HOURGLASS,
- MODEL_TYPE_POSE_HAND_AIC,
- MODEL_TYPE_POSE_HAND_AICLite
+ MODEL_TYPE_POSE_HAND_AICLite, // posenet lite 224
+ MODEL_TYPE_POSE_HAND_AICLite2, // posenet 0709 f
+ MODEL_TYPE_POSE_HAND_AICLite2Q // posenet 0709 q
};
typedef struct {
#define FILE_PATH_SIZE 1024
// pose estimation
-#define PE_TFLITE_CPM_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE/tflite/ped_tflite_model_cpm.tflite"
-#define PE_TFLITE_HOURGLASS_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE/tflite/ped_tflite_model_hourglass.tflite"
-
-// On-device Neural Engine needs pre-trained model file and its metadata which describes version and file name of the model.
-// So for using ONE enigne, a model file and its metafile should be located like below,
-// - /path/to/model/xxx.tflite
-// - /path/to/model/metadata/MANIFEST
-// MANIFEST file should contain below contents,
-// {
-// "major-version" : "1",
-// "minor-version" : "0",
-// "patch-version" : "0",
-// "models" : [ "model_file_name.tflite" ],
-// "model-types" : [ "tflite" ]
-// }
-#define PE_TFLITE_AIC_1_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE_1/tflite/posenet1.tflite"
-#define PE_TFLITE_AIC_2_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE_2/tflite/posenet2_b_1.tflite"
-#define PE_TFLITE_AICLite_1_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE_1/tflite/posenet1_lite_224.tflite"
-#define PE_TFLITE_AICLite_2_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE_2/tflite/posenet2_lite_224.tflite"
-
-static float thValNeck = 0.3f; // 15%
-static float thValArm = 0.1f; // 10 %
-static float thValLeg = 0.2f; // 5%
+#define PE_TFLITE_CPM_WEIGHT_PATH "/usr/share/capi-media-vision/models/PLD/tflite/pld-tflite-001.tflite"
+#define PE_POSE_LABEL_PATH "/usr/share/capi-media-vision/models/PLD/tflite/pose-label.txt"
+
+
+#define PE_TFLITE_AIC_1_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE_1/tflite/posenet1_lite_224.tflite"
+#define PE_TFLITE_AIC_2_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE_2/tflite/posenet2_lite_224.tflite"
+
+
+#define PE_TFLITE_AICLite_1_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE/tflite/posenet1_0709_f.tflite"
+#define PE_TFLITE_AICLite_2_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE/tflite/posenet2_0709_f.tflite"
+
+#define PE_TFLITE_AICLiteQ_1_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE/tflite/posenet1_0709_dq.tflite"
+#define PE_TFLITE_AICLiteQ_2_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE/tflite/posenet2_0709_f.tflite"
+
+#define PLD_MOTION_CAPTURE_FILE_PATH "/usr/share/capi-media-vision/models/PLD/mocap/example.bvh"
+#define PLD_MOTION_CAPTURE_MAPPING_FILE_PATH "/usr/share/capi-media-vision/models/PLD/mocap/example-mocap-mapping.txt"
+
+#define POSE_RESET_COUNT 5
+
+static float thPoseScore = 0.3f;
+static int thResetCount = 5;
+static float thCustom = 0.0f;
typedef struct _rect {
int left;
bool updated; // detection is find and pose is also good. then update drawing
bool locUpdated; // track previous detection
bool IsDetected; // detection
+ int label;
} HumanSkeleton;
typedef struct
static guint nFrames = 0;
// Gstreamer
-GstElement *pipeline, *source, *filter, *toverlay, *sink, *sink2;
+GstElement *pipeline, *source, *flip, *filter, *toverlay, *sink, *sink2;
GstElement *tee, *vscale, *vsfilter, *vconv, *vcfilter;
GstElement *vrate, *vrfilter, *fsink, *vrsink;
GstElement *queue1, *queue2, *queue3;
GstElement *vcrop, *vcrscale, *vcrsfilter, *vcrsconv, *vcrscfilter, *vcrssink;
CairoOverlayState *overlay_state;
-GstElement *decodebin, *dscale, *dconv;
+GstElement *dbin, *dscale, *dconv;
GstElement *dsfilter, *dcfilter;
GstElement *tee2, *enc, *muxmp4, *fsink2, *queue4, *queue5, *encconv;
mv_engine_config_h hp_mv_engine_cfg2;
mv_inference_h hp_mv_infer2;
+// pose comparison
+mv_pose_h hpPoser;
+float hpPoseScore;
+int hpPoseHoldTime;
+int hpPoseCount;
+
static void * outputTensorData;
FILE *fp;
free(outputTensorData);
outputTensorData = NULL;
}
+
+ if (hpPoser)
+ mv_pose_destroy(hpPoser);
+
printf(TEXT_YELLOW "exit..\n" TEXT_RESET);
signal(SIGINT, SIG_DFL);
exit(0);
//printf("(%d,%d): %f\n", humanSkeleton.pose[n].x, humanSkeleton.pose[n].y, confidences[n]);
//printf("(%d,%d)\n", humanSkeleton.pose[n].x, humanSkeleton.pose[n].y);
}
+ humanSkeleton.label = label;
humanSkeleton.IsDetected = true;
-
}
static void _hand_detected_cb (
if (number_of_hands <= 0) {
humanSkeleton.IsDetected = false;
+ humanSkeleton.label = -1;
return;
}
clock_gettime(CLOCK_MONOTONIC, &s_tspec);
- mv_inference_pose_landmark_detect(mv_src_p2, hp_mv_infer2, NULL, _hand_pose_cb, NULL);
+ mv_inference_pose_landmark_detect(mv_src_p2, hp_mv_infer2, NULL, _hand_pose_cb, NULL);
clock_gettime(CLOCK_MONOTONIC, &e_tspec);
}
+void _human_pose_cb (
+ mv_source_h source,
+ int number_of_pose_estimation,
+ mv_inference_pose_s *locations,
+ int label,
+ void *user_data)
+{
+ printf("%d landmarks\n", number_of_pose_estimation);
+
+ mv_pose_compare(hpPoser, locations,
+ (MV_INFERENCE_HUMAN_BODY_PART_LEG_LEFT | MV_INFERENCE_HUMAN_BODY_PART_LEG_RIGHT),
+ &hpPoseScore);
+
+
+ if (hpPoseScore >= thPoseScore) {
+ hpPoseHoldTime++;
+ } else {
+ if (hpPoseHoldTime > thResetCount) {
+ hpPoseCount++;
+ }
+ hpPoseHoldTime = 0;
+ }
+
+ printf("Score: %.4f, HoldTime: %d, Count: %d\n", hpPoseScore, hpPoseHoldTime, hpPoseCount);
+ for (int n = 0; n < number_of_pose_estimation; ++n) {
+
+ humanSkeleton.pose[n].x = (int)(640.f*(float)(locations->landmarks[n].point.x)/192.f);
+ humanSkeleton.pose[n].y = (int)(480.f*(float)(locations->landmarks[n].point.y)/192.f);
+ humanSkeleton.scores[n] = locations->landmarks[n].score;
+
+ }
+ humanSkeleton.IsDetected = true;
+}
+
static gboolean
run_pose (void *user_data)
{
GstMapInfo map;
- /*
gst_buffer_map(udata->buffer, &map, GST_MAP_READ);
mv_source_clear(mv_src_p);
gst_buffer_unmap(udata->buffer, &map);
- clock_t start = clock();
- mv_inference_pose_estimation_detect(mv_src_p, hp_mv_infer, &poseRoi, 1.f, 1.f, _human_pose_cb, NULL);
- clock_t end = clock();
- */
+ struct timespec s_tspec;
+ struct timespec e_tspec;
+
+ clock_gettime(CLOCK_MONOTONIC, &s_tspec);
+
+ mv_inference_pose_landmark_detect(mv_src_p, hp_mv_infer, &poseRoi, _human_pose_cb, NULL);
+
+ clock_gettime(CLOCK_MONOTONIC, &e_tspec);
+ struct timespec diffspec = diff(s_tspec, e_tspec);
+ unsigned long timeDiff = gettotalmillisec(diffspec);
+ printf("detect + pose time: %lu(ms)\n", timeDiff);
+
+ return FALSE;
+
+}
+
+static gboolean
+run_hand (void *user_data)
+{
+ HandOffData *udata = (HandOffData *)user_data;
+ if (!GST_IS_BUFFER(udata->buffer))
+ return FALSE;
+
+ GstMapInfo map;
gst_buffer_map(udata->buffer, &map, GST_MAP_READ);
#endif
}
+static void
+_hand_est_handoff(GstElement *object, GstBuffer *buffer, GstPad *pad, gpointer user_data)
+{
+
+ nFrames++;
+ hdata_p.buffer = buffer;
+ hdata_p.user_data = user_data;
+
+#if 0
+ if (nFrames % 15 == 0) {
+ g_mutex_lock(&pose_mutex);
+ g_idle_add (run_hand, &hdata_p);
+ g_mutex_unlock(&pose_mutex);
+ }
+#else
+ g_mutex_lock(&pose_mutex);
+ g_idle_add (run_hand, &hdata_p);
+ g_mutex_unlock(&pose_mutex);
+#endif
+}
+
+
static void
prepare_overlay (GstElement * overlay, GstCaps * caps, gpointer user_data)
{
state->valid = gst_video_info_from_caps (&state->vinfo, caps);
}
-/*
static void
-draw_overlay (GstElement * overlay, cairo_t * cr, guint64 timestamp,
+draw_overlay_pose (GstElement * overlay, cairo_t * cr, guint64 timestamp,
guint64 duration, gpointer user_data)
{
CairoOverlayState *s = (CairoOverlayState *) user_data;
cairo_set_source_rgba(cr, 0.1, 0.9, 0.0, 0.7);
cairo_set_line_width(cr, 2.0);
-
+ float poseWeight = 0.7f;
+ float prevPoseWeight = 0.3f;
if (!humanSkeleton.IsDetected)
return;
+ for (int k = 0; k < 16; ++k) {
+ if (humanSkeleton.isPrevPose == false) {
+ humanSkeleton.prevPose[k] = humanSkeleton.pose[k];
+ } else {
- //
- if (humanSkeleton.isPrevPose == false) {
- humanSkeleton.prevPose[1] = humanSkeleton.pose[1];
- // head - neck
- if (humanSkeleton.scores[0] >= thValNeck ) {
- humanSkeleton.prevPose[0] = humanSkeleton.pose[0];
- }
-
- // right arm
- if (humanSkeleton.scores[2] >= thValArm) {
- // neck - right shoulder
- humanSkeleton.prevPose[2] = humanSkeleton.pose[2];
- if (humanSkeleton.scores[3] >= thValArm) {
- // right shoulder - right elbow
- humanSkeleton.prevPose[3] = humanSkeleton.pose[3];
- if (humanSkeleton.scores[4] >= thValArm) {
- // right elbow - right wrist
- humanSkeleton.prevPose[4] = humanSkeleton.pose[4];
- }
- }
+ humanSkeleton.prevPose[k].x = (poseWeight * humanSkeleton.pose[k].x +
+ prevPoseWeight * humanSkeleton.prevPose[k].x);
+ humanSkeleton.prevPose[k].y = (poseWeight * humanSkeleton.pose[k].y +
+ prevPoseWeight * humanSkeleton.prevPose[k].y);
}
+ }
- // left arm
- if (humanSkeleton.scores[5] >= thValArm) {
- // neck - right shoulder
- humanSkeleton.prevPose[5] = humanSkeleton.pose[5];
- if (humanSkeleton.scores[6] >= thValArm) {
- // right shoulder - right elbow
- humanSkeleton.prevPose[6] = humanSkeleton.pose[6];
- if (humanSkeleton.scores[7] >= thValArm) {
- // right elbow - right wrist
- humanSkeleton.prevPose[7] = humanSkeleton.pose[7];
- }
- }
- }
+ for (int k = 0; k < 16; ++k) {
+ if (humanSkeleton.scores[k] > 0.0f) {
+ if (k > 9)
+ cairo_set_source_rgba(cr, 0.9, 0.1, 0.0, 0.7);
- // right leg
- if (humanSkeleton.scores[8] >= thValLeg) {
- // neck - right shoulder
- humanSkeleton.prevPose[8] = humanSkeleton.pose[8];
- if (humanSkeleton.scores[9] >= thValLeg) {
- // right shoulder - right elbow
- humanSkeleton.prevPose[9] = humanSkeleton.pose[9];
- if (humanSkeleton.scores[10] >= thValLeg) {
- // right elbow - right wrist
- humanSkeleton.prevPose[10] = humanSkeleton.pose[10];
- }
- }
+ cairo_arc(cr, humanSkeleton.prevPose[k].x, humanSkeleton.prevPose[k].y, 3,0, 2*M_PI);
+ cairo_stroke(cr);
}
+ }
- // left leg
- if (humanSkeleton.scores[11] >= thValLeg) {
- // neck - right shoulder
- humanSkeleton.prevPose[11] = humanSkeleton.pose[11];
- if (humanSkeleton.scores[12] >= thValLeg) {
- // right shoulder - right elbow
- humanSkeleton.prevPose[12] = humanSkeleton.pose[12];
- if (humanSkeleton.scores[13] >= thValLeg) {
- // right elbow - right wrist
- humanSkeleton.prevPose[13] = humanSkeleton.pose[13];
- }
- }
- }
- humanSkeleton.isPrevPose = true;
- } else {
- // weighted sum of pose and prevPose
- // method1: fixed weights (pose : prevPose = 0.7 : 0.3)
- float poseWeight = 0.7f;
- float prevPoseWeight = 0.3f;
- humanSkeleton.prevPose[1].x = (poseWeight * humanSkeleton.pose[1].x +
- prevPoseWeight * humanSkeleton.prevPose[1].x);
- humanSkeleton.prevPose[1].y = (poseWeight * humanSkeleton.pose[1].y +
- prevPoseWeight * humanSkeleton.prevPose[1].y);
- // head - neck
- if (humanSkeleton.scores[0] >= thValNeck ) {
- humanSkeleton.prevPose[0].x = (poseWeight * humanSkeleton.pose[0].x +
- prevPoseWeight * humanSkeleton.prevPose[0].x);
- humanSkeleton.prevPose[0].y = (poseWeight * humanSkeleton.pose[0].y +
- prevPoseWeight * humanSkeleton.prevPose[0].y);
- }
- // right arm
- if (humanSkeleton.scores[2] >= thValArm) {
- // neck - right shoulder
- humanSkeleton.prevPose[2].x = (poseWeight * humanSkeleton.pose[2].x +
- prevPoseWeight * humanSkeleton.prevPose[2].x);
- humanSkeleton.prevPose[2].y = (poseWeight * humanSkeleton.pose[2].y +
- prevPoseWeight * humanSkeleton.prevPose[2].y);
- if (humanSkeleton.scores[3] >= thValArm) {
- // right shoulder - right elbow
- humanSkeleton.prevPose[3].x = (poseWeight * humanSkeleton.pose[3].x +
- prevPoseWeight * humanSkeleton.prevPose[3].x);
- humanSkeleton.prevPose[3].y = (poseWeight * humanSkeleton.pose[3].y +
- prevPoseWeight * humanSkeleton.prevPose[3].y);
- if (humanSkeleton.scores[4] >= thValArm) {
- // right elbow - right wrist
- humanSkeleton.prevPose[4].x = (poseWeight * humanSkeleton.pose[4].x +
- prevPoseWeight * humanSkeleton.prevPose[4].x);
- humanSkeleton.prevPose[4].y = (poseWeight * humanSkeleton.pose[4].y +
- prevPoseWeight * humanSkeleton.prevPose[4].y);
- }
- }
- }
+ cairo_select_font_face(cr, "sans-serif", CAIRO_FONT_SLANT_NORMAL, CAIRO_FONT_WEIGHT_BOLD);
+ cairo_set_font_size(cr,15);
- // left arm
- if (humanSkeleton.scores[5] >= thValArm) {
- // neck - right shoulder
- humanSkeleton.prevPose[5].x = (poseWeight * humanSkeleton.pose[5].x +
- prevPoseWeight * humanSkeleton.prevPose[5].x);
- humanSkeleton.prevPose[5].y = (poseWeight * humanSkeleton.pose[5].y +
- prevPoseWeight * humanSkeleton.prevPose[5].y);
- if (humanSkeleton.scores[6] >= thValArm) {
- // right shoulder - right elbow
- humanSkeleton.prevPose[6].x = (poseWeight * humanSkeleton.pose[6].x +
- prevPoseWeight * humanSkeleton.prevPose[6].x);
- humanSkeleton.prevPose[6].y = (poseWeight * humanSkeleton.pose[6].y +
- prevPoseWeight * humanSkeleton.prevPose[6].y);
- if (humanSkeleton.scores[7] >= thValArm) {
- // right elbow - right wrist
- humanSkeleton.prevPose[7].x = (poseWeight * humanSkeleton.pose[7].x +
- prevPoseWeight * humanSkeleton.prevPose[7].x);
- humanSkeleton.prevPose[7].y = (poseWeight * humanSkeleton.pose[7].y +
- prevPoseWeight * humanSkeleton.prevPose[7].y);
- }
- }
- }
+ char howToText[2][1024];
+ char tmpText[3][1024];
+ snprintf(howToText[0], 1024, "Until %d hold time over Score: %.2f", thResetCount, thPoseScore);
+ snprintf(howToText[1], 1024, "Green-Hold, Blue-Release");
+ snprintf(tmpText[0], 1024, "Score: %.4f", hpPoseScore);
+ snprintf(tmpText[1], 1024, "Hold time: %d", hpPoseHoldTime > thResetCount ? thResetCount : hpPoseHoldTime);
+ snprintf(tmpText[2], 1024, "Count: %2d", hpPoseCount);
- // right leg
- if (humanSkeleton.scores[8] >= thValLeg) {
- // neck - right shoulder
- humanSkeleton.prevPose[8].x = (poseWeight * humanSkeleton.pose[8].x +
- prevPoseWeight * humanSkeleton.prevPose[8].x);
- humanSkeleton.prevPose[8].y = (poseWeight * humanSkeleton.pose[8].y +
- prevPoseWeight * humanSkeleton.prevPose[8].y);
- if (humanSkeleton.scores[9] >= thValLeg) {
- // right shoulder - right elbow
- humanSkeleton.prevPose[9].x = (poseWeight * humanSkeleton.pose[9].x +
- prevPoseWeight * humanSkeleton.prevPose[9].x);
- humanSkeleton.prevPose[9].y = (poseWeight * humanSkeleton.pose[9].y +
- prevPoseWeight * humanSkeleton.prevPose[9].y);
- if (humanSkeleton.scores[10] >= thValLeg) {
- // right elbow - right wrist
- humanSkeleton.prevPose[10].x = (poseWeight * humanSkeleton.pose[10].x +
- prevPoseWeight * humanSkeleton.prevPose[10].x);
- humanSkeleton.prevPose[10].y = (poseWeight * humanSkeleton.pose[10].y +
- prevPoseWeight * humanSkeleton.prevPose[10].y);
- }
- }
- }
- // left leg
- if (humanSkeleton.scores[11] >= thValLeg) {
- // neck - right shoulder
- humanSkeleton.prevPose[11].x = (poseWeight * humanSkeleton.pose[11].x +
- prevPoseWeight * humanSkeleton.prevPose[11].x);
- humanSkeleton.prevPose[11].y = (poseWeight * humanSkeleton.pose[11].y +
- prevPoseWeight * humanSkeleton.prevPose[11].y);
- if (humanSkeleton.scores[12] >= thValLeg) {
- // right shoulder - right elbow
- humanSkeleton.prevPose[12].x = (poseWeight * humanSkeleton.pose[12].x +
- prevPoseWeight * humanSkeleton.prevPose[12].x);
- humanSkeleton.prevPose[12].y = (poseWeight * humanSkeleton.pose[12].y +
- prevPoseWeight * humanSkeleton.prevPose[12].y);
- if (humanSkeleton.scores[13] >= thValLeg) {
- // right elbow - right wrist
- humanSkeleton.prevPose[13].x = (poseWeight * humanSkeleton.pose[13].x +
- prevPoseWeight * humanSkeleton.prevPose[13].x);
- humanSkeleton.prevPose[13].y = (poseWeight * humanSkeleton.pose[13].y +
- prevPoseWeight * humanSkeleton.prevPose[13].y);
- }
- }
- }
- }
+ cairo_set_source_rgba(cr, 1.0, 1.0, 1.0, 0.7);
+ cairo_move_to(cr, 400.0, 30.0);
+ cairo_show_text(cr, howToText[0]);
+ cairo_move_to(cr, 420.0, 45.0);
+ cairo_show_text(cr, howToText[1]);
+ if (hpPoseScore >= thPoseScore) {
+ if (hpPoseHoldTime > thResetCount)
+ cairo_set_source_rgba(cr, 0.0, 0.1, 0.9, 0.7);
+ else
+ cairo_set_source_rgba(cr, 0.1, 0.9, 0.0, 0.7);
- //
- //draw..
- // head - neck
- if (humanSkeleton.scores[0] >= thValNeck ) {
- cairo_move_to(cr, humanSkeleton.prevPose[0].x, humanSkeleton.prevPose[0].y);
- cairo_line_to(cr, humanSkeleton.prevPose[1].x, humanSkeleton.prevPose[1].y);
- }
-
- // right arm
- cairo_move_to(cr, humanSkeleton.prevPose[1].x, humanSkeleton.prevPose[1].y);
- if (humanSkeleton.scores[2] >= thValArm) {
- // neck - right shoulder
- cairo_line_to(cr, humanSkeleton.prevPose[2].x, humanSkeleton.prevPose[2].y);
- if (humanSkeleton.scores[3] >= thValArm) {
- // right shoulder - right elbow
- cairo_line_to(cr, humanSkeleton.prevPose[3].x, humanSkeleton.prevPose[3].y);
- if (humanSkeleton.scores[4] >= thValArm) {
- // right elbow - right wrist
- cairo_line_to(cr, humanSkeleton.prevPose[4].x, humanSkeleton.prevPose[4].y);
- }
- }
- }
- cairo_stroke(cr);
+ cairo_move_to(cr, 420.0, 60.0);
+ cairo_show_text(cr, tmpText[0]);
+ cairo_move_to(cr, 420.0, 75.0);
+ cairo_show_text(cr, tmpText[1]);
+ cairo_move_to(cr, 420.0, 90.0);
+ cairo_show_text(cr, tmpText[2]);
- // left arm
- cairo_move_to(cr, humanSkeleton.prevPose[1].x, humanSkeleton.prevPose[1].y);
- if (humanSkeleton.scores[5] >= thValArm) {
- // neck - right shoulder
- cairo_line_to(cr, humanSkeleton.prevPose[5].x, humanSkeleton.prevPose[5].y);
- if (humanSkeleton.scores[6] >= thValArm) {
- // right shoulder - right elbow
- cairo_line_to(cr, humanSkeleton.prevPose[6].x, humanSkeleton.prevPose[6].y);
- if (humanSkeleton.scores[7] >= thValArm) {
- // right elbow - right wrist
- cairo_line_to(cr, humanSkeleton.prevPose[7].x, humanSkeleton.prevPose[7].y);
- }
- }
- }
- cairo_stroke(cr);
-
-
- // right leg
- cairo_move_to(cr, humanSkeleton.prevPose[1].x, humanSkeleton.prevPose[1].y);
- if (humanSkeleton.scores[8] >= thValLeg) {
- // neck - right shoulder
- cairo_line_to(cr, humanSkeleton.prevPose[8].x, humanSkeleton.prevPose[8].y);
- if (humanSkeleton.scores[9] >= thValLeg) {
- // right shoulder - right elbow
- cairo_line_to(cr, humanSkeleton.prevPose[9].x, humanSkeleton.prevPose[9].y);
- if (humanSkeleton.scores[10] >= thValLeg) {
- // right elbow - right wrist
- cairo_line_to(cr, humanSkeleton.prevPose[10].x, humanSkeleton.prevPose[10].y);
- }
- }
- }
- cairo_stroke(cr);
-
- // left leg
- cairo_move_to(cr, humanSkeleton.prevPose[1].x, humanSkeleton.prevPose[1].y);
- if (humanSkeleton.scores[11] >= thValLeg) {
- // neck - right shoulder
- cairo_line_to(cr, humanSkeleton.prevPose[11].x, humanSkeleton.prevPose[11].y);
- if (humanSkeleton.scores[12] >= thValLeg) {
- // right shoulder - right elbow
- cairo_line_to(cr, humanSkeleton.prevPose[12].x, humanSkeleton.prevPose[12].y);
- if (humanSkeleton.scores[13] >= thValLeg) {
- // right elbow - right wrist
- cairo_line_to(cr, humanSkeleton.prevPose[13].x, humanSkeleton.prevPose[13].y);
- }
- }
+ } else {
+ cairo_set_source_rgba(cr, 0.9, 0.1, 0.0, 0.7);
+ cairo_move_to(cr, 420.0, 60.0);
+ cairo_show_text(cr, tmpText[0]);
+ cairo_move_to(cr, 420.0, 75.0);
+ cairo_show_text(cr, tmpText[1]);
+ cairo_move_to(cr, 420.0, 90.0);
+ cairo_show_text(cr, tmpText[2]);
}
- cairo_stroke(cr);
}
-*/
+
static void
draw_overlay_hand (GstElement * overlay, cairo_t * cr, guint64 timestamp,
guint64 duration, gpointer user_data)
return;
}
- cairo_set_source_rgba(cr, 0.1, 0.9, 0.0, 0.7);
- cairo_set_line_width(cr, 2.0);
+ cairo_set_source_rgba(cr, 0.0, 0.0, 1.0, 0.7);
+ cairo_set_line_width(cr, 2.0);
if (!humanSkeleton.IsDetected)
return;
+ cairo_select_font_face(cr, "sans-serif", CAIRO_FONT_SLANT_NORMAL, CAIRO_FONT_WEIGHT_BOLD);
+ cairo_set_font_size(cr,15);
+ cairo_move_to(cr, 320.0, 48.0);
+ cairo_show_text(cr, gHandGestureLabel[humanSkeleton.label]);
+
+ cairo_set_source_rgba(cr, 0.1, 0.9, 0.0, 0.7);
+ cairo_set_line_width(cr, 2.0);
// thumb - red
cairo_set_source_rgba (cr, 0.9, 0.1, 0.0, 0.7);
cairo_line_to(cr, humanSkeleton.pose[k].x, humanSkeleton.pose[k].y);
}
cairo_stroke(cr);
+
+
}
static gboolean bus_call (GstBus *bus, GstMessage *msg, gpointer data)
MV_INFERENCE_MODEL_WEIGHT_FILE_PATH,
PE_TFLITE_CPM_WEIGHT_PATH);
+ mv_engine_config_set_string_attribute(
+ mv_engine_cfg,
+ MV_INFERENCE_MODEL_USER_FILE_PATH,
+ PE_POSE_LABEL_PATH);
+
mv_engine_config_set_int_attribute(mv_engine_cfg,
MV_INFERENCE_INPUT_DATA_TYPE,
MV_INFERENCE_DATA_FLOAT32);
mv_engine_config_set_int_attribute(mv_engine_cfg,
MV_INFERENCE_BACKEND_TYPE,
- MV_INFERENCE_BACKEND_ARMNN);
+ MV_INFERENCE_BACKEND_ONE);
mv_engine_config_set_int_attribute(mv_engine_cfg,
MV_INFERENCE_TARGET_TYPE,
- MV_INFERENCE_TARGET_CPU);
+ MV_INFERENCE_TARGET_GPU);
mv_engine_config_set_int_attribute(mv_engine_cfg,
MV_INFERENCE_INPUT_TENSOR_WIDTH,
return MEDIA_VISION_ERROR_NONE;
}
-int perform_armnn_human_pose_hourglass_configure(mv_engine_config_h mv_engine_cfg)
+int perform_tflite_human_pose_cpm_configure(mv_engine_config_h mv_engine_cfg)
{
if (mv_engine_cfg == NULL) {
printf("mv_engine_cfg is null\n");
}
char *inputNodeName = "image";
- char *outputNodeName[1] = {"hourglass_out_3"};
+ char *outputNodeName[1] = {"Convolutional_Pose_Machine/stage_5_out"};
mv_engine_config_set_string_attribute(mv_engine_cfg,
MV_INFERENCE_MODEL_WEIGHT_FILE_PATH,
- PE_TFLITE_HOURGLASS_WEIGHT_PATH);
+ PE_TFLITE_CPM_WEIGHT_PATH);
+
+ mv_engine_config_set_string_attribute(
+ mv_engine_cfg,
+ MV_INFERENCE_MODEL_USER_FILE_PATH,
+ PE_POSE_LABEL_PATH);
mv_engine_config_set_int_attribute(mv_engine_cfg,
MV_INFERENCE_INPUT_DATA_TYPE,
mv_engine_config_set_int_attribute(mv_engine_cfg,
MV_INFERENCE_BACKEND_TYPE,
- MV_INFERENCE_BACKEND_ARMNN);
+ MV_INFERENCE_BACKEND_ONE);
mv_engine_config_set_int_attribute(mv_engine_cfg,
MV_INFERENCE_TARGET_TYPE,
}
char *inputNodeName = "input";
- char *outputNodeNames[2] = {"mobilenetv2/boundingbox2", "mobilenetv2/heatmap"};
+ char *outputNodeNames[2] = {"mobilenetv2/boundingbox", "mobilenetv2/heatmap"};
//outputTensorData = (void*)calloc(56*56*21, sizeof(float));
mv_engine_config_set_string_attribute(mv_engine_cfg,
mv_engine_config_set_int_attribute(mv_engine_cfg,
MV_INFERENCE_TARGET_DEVICE_TYPE,
- MV_INFERENCE_TARGET_DEVICE_CPU);
+ MV_INFERENCE_TARGET_DEVICE_GPU);
mv_engine_config_set_int_attribute(mv_engine_cfg,
MV_INFERENCE_INPUT_TENSOR_WIDTH,
mv_engine_config_set_int_attribute(mv_engine_cfg,
MV_INFERENCE_TARGET_DEVICE_TYPE,
- MV_INFERENCE_TARGET_DEVICE_CPU);
+ MV_INFERENCE_TARGET_DEVICE_GPU);
mv_engine_config_set_int_attribute(mv_engine_cfg,
MV_INFERENCE_INPUT_TENSOR_WIDTH,
mv_engine_config_set_int_attribute(mv_engine_cfg,
MV_INFERENCE_TARGET_DEVICE_TYPE,
- MV_INFERENCE_TARGET_DEVICE_CPU);
+ MV_INFERENCE_TARGET_DEVICE_GPU);
mv_engine_config_set_int_attribute(mv_engine_cfg,
MV_INFERENCE_INPUT_TENSOR_WIDTH,
mv_engine_config_set_int_attribute(mv_engine_cfg,
MV_INFERENCE_TARGET_DEVICE_TYPE,
- MV_INFERENCE_TARGET_DEVICE_CPU);
+ MV_INFERENCE_TARGET_DEVICE_GPU);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_WIDTH,
+ 56);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_HEIGHT,
+ 56);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_CHANNELS,
+ 21);
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_NODE_NAME,
+ inputNodeName);
+
+ mv_engine_config_set_array_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_OUTPUT_NODE_NAMES,
+ outputNodeNames,
+ 2);
+
+ return MEDIA_VISION_ERROR_NONE;
+}
+
+int perform_tflite_hand_detection_AICLiteQ(mv_engine_config_h mv_engine_cfg)
+{
+ if (mv_engine_cfg == NULL) {
+ printf("mv_engine_cfg is null\n");
+ return MEDIA_VISION_ERROR_INVALID_PARAMETER;
+ }
+
+ char *inputNodeName = "input";
+ char *outputNodeNames[2] = {"mobilenetv2/boundingbox", "mobilenetv2/heatmap"};
+
+ //outputTensorData = (void*)calloc(56*56*21, sizeof(char));
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_WEIGHT_FILE_PATH,
+ PE_TFLITE_AICLiteQ_1_WEIGHT_PATH);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_DATA_TYPE,
+ MV_INFERENCE_DATA_FLOAT32);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_MEAN_VALUE,
+ 0.0);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_STD_VALUE,
+ 1.0);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_BACKEND_TYPE,
+ MV_INFERENCE_BACKEND_ONE);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_TARGET_DEVICE_TYPE,
+ MV_INFERENCE_TARGET_DEVICE_GPU);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_WIDTH,
+ 224);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_HEIGHT,
+ 224);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_TENSOR_CHANNELS,
+ 3);
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_NODE_NAME,
+ inputNodeName);
+
+ mv_engine_config_set_array_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_OUTPUT_NODE_NAMES,
+ outputNodeNames,
+ 2);
+
+ return MEDIA_VISION_ERROR_NONE;
+}
+
+int perform_tflite_hand_detection_AICLiteQ2(mv_engine_config_h mv_engine_cfg)
+{
+ char *inputNodeName = "input";
+ char *outputNodeNames[2] = {"mobilenetv2/coord_refine", "mobilenetv2/gesture"};
+
+ mv_engine_config_set_string_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_WEIGHT_FILE_PATH,
+ PE_TFLITE_AICLiteQ_2_WEIGHT_PATH);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_INPUT_DATA_TYPE,
+ MV_INFERENCE_DATA_FLOAT32);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_MEAN_VALUE,
+ 0.0);
+
+ mv_engine_config_set_double_attribute(mv_engine_cfg,
+ MV_INFERENCE_MODEL_STD_VALUE,
+ 1.0);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_BACKEND_TYPE,
+ MV_INFERENCE_BACKEND_ONE);
+
+ mv_engine_config_set_int_attribute(mv_engine_cfg,
+ MV_INFERENCE_TARGET_DEVICE_TYPE,
+ MV_INFERENCE_TARGET_DEVICE_GPU);
mv_engine_config_set_int_attribute(mv_engine_cfg,
MV_INFERENCE_INPUT_TENSOR_WIDTH,
if (ad->modelType == MODEL_TYPE_POSE_CPM) {
err = perform_armnn_human_pose_cpm_configure(hp_mv_engine_cfg);
- } else if (ad->modelType == MODEL_TYPE_POSE_HOURGLASS) {
- err = perform_armnn_human_pose_hourglass_configure(hp_mv_engine_cfg);
- } else if (ad->modelType == MODEL_TYPE_POSE_HAND_AIC) {
+ //err = perform_tflite_human_pose_cpm_configure(hp_mv_engine_cfg);
+
+ mv_pose_create(&hpPoser);
+ mv_pose_set_from_file(hpPoser,
+ PLD_MOTION_CAPTURE_FILE_PATH,
+ PLD_MOTION_CAPTURE_MAPPING_FILE_PATH);
+
+ } else if (ad->modelType == MODEL_TYPE_POSE_HAND_AICLite) {
outputTensorData = (void*)calloc(56*56*21, sizeof(float));
err = perform_tflite_hand_detection_AIC(hp_mv_engine_cfg);
err = perform_tflite_hand_detection_AIC2(hp_mv_engine_cfg2);
- } else {
+ } else if (ad->modelType == MODEL_TYPE_POSE_HAND_AICLite2) {
outputTensorData = (void*)calloc(56*56*21, sizeof(float));
err = perform_tflite_hand_detection_AICLite(hp_mv_engine_cfg);
err = perform_tflite_hand_detection_AICLite2(hp_mv_engine_cfg2);
+ } else {
+ outputTensorData = (void*)calloc(56*56*21, sizeof(float));
+ err = perform_tflite_hand_detection_AICLiteQ(hp_mv_engine_cfg);
+
+ err = perform_tflite_hand_detection_AICLiteQ2(hp_mv_engine_cfg2);
}
if (err != MEDIA_VISION_ERROR_NONE) {
clock_t end = clock();
printf("time: %2.3f\n", (double)(end - start)/CLOCKS_PER_SEC);
+ /* mediavision source */
+ mv_create_source(&mv_src_p);
- mv_inference_create(&hp_mv_infer2);
+ if (ad->modelType >= MODEL_TYPE_POSE_HAND_AICLite) {
+ mv_inference_create(&hp_mv_infer2);
- mv_inference_configure(hp_mv_infer2, hp_mv_engine_cfg2);
+ mv_inference_configure(hp_mv_infer2, hp_mv_engine_cfg2);
- start = clock();
- mv_inference_prepare(hp_mv_infer2);
- end = clock();
- printf("time: %2.3f\n", (double)(end - start)/CLOCKS_PER_SEC);
+ start = clock();
+ mv_inference_prepare(hp_mv_infer2);
+ end = clock();
+ printf("time: %2.3f\n", (double)(end - start)/CLOCKS_PER_SEC);
- /* mediavision source */
- mv_create_source(&mv_src_p);
- mv_create_source(&mv_src_p2);
+ mv_create_source(&mv_src_p2);
+ }
pipeline = gst_pipeline_new("app");
} else {
source = gst_element_factory_make("filesrc", "src");
- decodebin = gst_element_factory_make("decodebin", "dbin");
+ dbin = gst_element_factory_make("decodebin", "dbin");
dscale = gst_element_factory_make("videoscale", "dscale");
dconv = gst_element_factory_make("videoconvert", "dconv");
dsfilter = gst_element_factory_make("capsfilter", "dsfilter");
queue4 = gst_element_factory_make("queue", "queue4");
queue5 = gst_element_factory_make("queue", "queue5");
encconv = gst_element_factory_make("videoconvert", "encconv");
+ g_object_set(G_OBJECT(enc), "bitrate", 800000, NULL);
}
-
+ flip = gst_element_factory_make("videoflip", "vflip" );
tee = gst_element_factory_make("tee", "tee");
queue1 = gst_element_factory_make("queue", "queue1");
queue2 = gst_element_factory_make("queue", "queue2");
}
g_object_set(G_OBJECT(filter), "caps", gst_caps_from_string("video/x-raw, format=YUY2, width=640, height=480"), NULL);
} else {
- if (!decodebin || !dscale || !dconv) {
+ if (!dbin || !dscale || !dconv) {
printf(TEXT_RED "One element might be not created. Exiting.\n" TEXT_RESET);
return -1;
}
g_object_set(G_OBJECT(dsfilter), "caps", gst_caps_from_string("video/x-raw, width=640, height=480"), NULL);
g_object_set(G_OBJECT(dcfilter), "caps", gst_caps_from_string("video/x-raw, format=YUY2, width=640, height=480"), NULL);
- g_signal_connect (decodebin, "pad-added", G_CALLBACK (cb_new_pad), NULL);
+ g_signal_connect (dbin, "pad-added", G_CALLBACK (cb_new_pad), NULL);
}
if (ad->filename2) {
/* set up the pipeline */
//g_signal_connect (coverlay, "draw", G_CALLBACK (draw_overlay), overlay_state);
- g_signal_connect (coverlay, "draw", G_CALLBACK (draw_overlay_hand), overlay_state);
+ if (ad->modelType == MODEL_TYPE_POSE_CPM) {
+ g_signal_connect (coverlay, "draw", G_CALLBACK (draw_overlay_pose), overlay_state);
+ } else {
+ g_signal_connect (coverlay, "draw", G_CALLBACK (draw_overlay_hand), overlay_state);
+ }
g_signal_connect (coverlay, "caps-changed", G_CALLBACK (prepare_overlay), overlay_state);
if (!ad->filename) {
- g_object_set(G_OBJECT(source), "device", "/dev/video8", NULL); // 252
+ g_object_set(G_OBJECT(source), "device", "/dev/video252", NULL); // 252
+
} else {
g_object_set(G_OBJECT(source), "location", ad->filename, NULL);
- g_object_set(G_OBJECT(source), "num-buffers", ad->numbuffers, NULL);
}
-
+ g_object_set(G_OBJECT(flip), "method", 4, NULL);
g_object_set(G_OBJECT(sink2), "use-tbm", FALSE, NULL);
g_object_set(G_OBJECT(sink2), "sync", FALSE, NULL);
g_object_set(G_OBJECT(sink), "video-sink", sink2, NULL);
g_object_set(G_OBJECT(sink), "sync", FALSE, NULL);
- if (ad->modelType == MODEL_TYPE_POSE_HAND_AIC ||
- ad->modelType == MODEL_TYPE_POSE_HAND_AICLite) {
+ if (ad->modelType == MODEL_TYPE_POSE_HAND_AICLite ||
+ ad->modelType == MODEL_TYPE_POSE_HAND_AICLite2 ||
+ ad->modelType == MODEL_TYPE_POSE_HAND_AICLite2Q) {
g_object_set(G_OBJECT(vsfilter), "caps", gst_caps_from_string("video/x-raw, width=224, height=224"), NULL);
poseCropSize = 224;
} else {
// here to be continue
printf("vrsink signal-handoffs\n");
g_object_set(G_OBJECT(vrsink), "signal-handoffs", TRUE, NULL);
- handler_p = g_signal_connect (vrsink, "handoff", G_CALLBACK(_pose_est_handoff), outputTensorData);
+
+ if (ad->modelType == MODEL_TYPE_POSE_CPM) {
+ handler_p = g_signal_connect (vrsink, "handoff", G_CALLBACK(_pose_est_handoff), NULL);
+ } else {
+ handler_p = g_signal_connect (vrsink, "handoff", G_CALLBACK(_hand_est_handoff), outputTensorData);
+ }
+
g_object_set(G_OBJECT(vrsink), "sync", FALSE, NULL);
bus_watch_id = gst_bus_add_watch(bus, bus_call, NULL);
gst_object_unref(bus);
+ printf("eleeent would be added.\n");
/* add elements into the pipeline */
gst_bin_add_many(GST_BIN(pipeline),
source,
- tee, queue1, vscale, vsfilter, vconv, vcfilter,
+ flip, tee, queue1, vscale, vsfilter, vconv, vcfilter,
vrate, vrfilter, vrsink,
queue2, oconv, coverlay, sink,
NULL);
/* link elements */
if (!ad->filename) {
+ printf("take !ad->filename 1\n");
gst_bin_add(GST_BIN(pipeline), filter);
- gst_element_link_many(source, filter, tee, NULL);
+ gst_element_link_many(source, flip, filter, tee, NULL);
} else {
- gst_bin_add_many(GST_BIN(pipeline), decodebin, dscale, dconv, dsfilter, dcfilter, NULL);
- gst_element_link_many(source, decodebin, NULL);
+ gst_bin_add_many(GST_BIN(pipeline), dbin, dscale, dconv, dsfilter, dcfilter, NULL);
+ gst_element_link_many(source, dbin, NULL);
gst_element_link_many(dscale, dsfilter, dconv, dcfilter, tee, NULL);
}
-
if (!ad->filename2) {
// display
gst_element_link_many (tee, queue2, oconv, coverlay, /*toverlay,*/ sink, NULL);
/* loop */
humanSkeleton.IsDetected = false;
humanSkeleton.isPrevPose = false;
+ humanSkeleton.label = -1;
printf(TEXT_GREEN "Running.....\n" TEXT_RESET);
// GST_END
#if 0
}
if (argc < 2) {
- printf("usage: mv_stream_infer model [NeckThresVal, ArmThresVal, LegThresVal, [filename]]");
- printf("model: 0(CPM), 1(HOURGLASS), 2(AIC Hand), 3(AIC Lite Hand\n");
+ printf("usage: mv_stream_infer model [thPoseScore, thResetCount, thCustom, [filename]]");
+ printf("model: 0(CPM), 1(AIC Hand), 2(AIC Lite Hand), 3(AIC Lite Q Hand)\n");
return -1;
}
ad.modelType = atoi(argv[1]);
if (ad.modelType < 0 || ad.modelType > 3) {
- printf("not supported model type\n");
+ printf("not supported model type [%d]\n", ad.modelType);
return -1;
}
- if (ad.modelType != MODEL_TYPE_POSE_HAND_AIC &&
- ad.modelType != MODEL_TYPE_POSE_HAND_AICLite) {
- thValNeck = (float)atoi(argv[2])/100.f;
- thValArm = (float)atoi(argv[3])/100.f;
- thValLeg = (float)atoi(argv[4])/100.f;
+ if (ad.modelType != MODEL_TYPE_POSE_HAND_AICLite &&
+ ad.modelType != MODEL_TYPE_POSE_HAND_AICLite2 &&
+ ad.modelType != MODEL_TYPE_POSE_HAND_AICLite2Q) {
+ thPoseScore = (float)atoi(argv[2])/100.f;
+ thResetCount = atoi(argv[3]);
+ thCustom = (float)atoi(argv[4])/100.f;
poseRoi.point.x = 50;
poseRoi.point.y = 0;
projects / platform / core / api / mediavision.git / commitdiff