#endif
#define PACKAGE "test"
+static char* gHandGestureLabel[] = { "None", "One", "Two", "Three", "Four", "Five",
+ "Six", "Seven", "SmallHeart", "OK", "DisLike", "Like",
+ "Fist", "Rock", "Love"};
static int st = 0;
static Evas_Object *g_eo = NULL;
static Evas_Object *icon = NULL;
typedef enum {
MODEL_TYPE_POSE_CPM = 0,
- MODEL_TYPE_POSE_HOURGLASS,
- MODEL_TYPE_POSE_HAND_AIC,
- MODEL_TYPE_POSE_HAND_AICLite,
- MODEL_TYPE_POSE_HAND_AICLite3
+ MODEL_TYPE_POSE_HAND_AICLite, // posenet lite 224
+ MODEL_TYPE_POSE_HAND_AICLite2, // posenet 0709 f
+ MODEL_TYPE_POSE_HAND_AICLite2Q // posenet 0709 q
};
typedef struct {
#define FILE_PATH_SIZE 1024
// pose estimation
-#define PE_TFLITE_CPM_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE/tflite/ped_tflite_model_cpm.tflite"
-#define PE_TFLITE_HOURGLASS_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE/tflite/ped_tflite_model_hourglass.tflite"
-
-// On-device Neural Engine needs pre-trained model file and its metadata which describes version and file name of the model.
-// So for using ONE enigne, a model file and its metafile should be located like below,
-// - /path/to/model/xxx.tflite
-// - /path/to/model/metadata/MANIFEST
-// MANIFEST file should contain below contents,
-// {
-// "major-version" : "1",
-// "minor-version" : "0",
-// "patch-version" : "0",
-// "models" : [ "model_file_name.tflite" ],
-// "model-types" : [ "tflite" ]
-// }
-#define PE_TFLITE_AIC_1_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE_1/tflite/posenet1.tflite"
-#define PE_TFLITE_AIC_2_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE_2/tflite/posenet2_b_1.tflite"
-#define PE_TFLITE_AICLite_1_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE_1/tflite/posenet1_lite_224.tflite"
-#define PE_TFLITE_AICLite_2_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE_2/tflite/posenet2_lite_224.tflite"
-#define PE_TFLITE_AICLite3_1_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE/tflite/0831_posenet1_q.tflite"
-#define PE_TFLITE_AICLite3_2_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE/tflite/0831_posenet2_q.tflite"
-
-static float thValNeck = 0.3f; // 15%
-static float thValArm = 0.1f; // 10 %
-static float thValLeg = 0.2f; // 5%
+#define PE_TFLITE_CPM_WEIGHT_PATH "/usr/share/capi-media-vision/models/PLD/tflite/pld-tflite-001.tflite"
+#define PE_POSE_LABEL_PATH "/usr/share/capi-media-vision/models/PLD/tflite/pose-label.txt"
+
+
+#define PE_TFLITE_AIC_1_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE_1/tflite/posenet1_lite_224.tflite"
+#define PE_TFLITE_AIC_2_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE_2/tflite/posenet2_lite_224.tflite"
+
+
+#define PE_TFLITE_AICLite_1_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE/tflite/posenet1_0709_f.tflite"
+#define PE_TFLITE_AICLite_2_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE/tflite/posenet2_0709_f.tflite"
+
+#define PE_TFLITE_AICLiteQ_1_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE/tflite/posenet1_0709_dq.tflite"
+#define PE_TFLITE_AICLiteQ_2_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE/tflite/posenet2_0709_f.tflite"
+
+#define PLD_MOTION_CAPTURE_FILE_PATH "/usr/share/capi-media-vision/models/PLD/mocap/example.bvh"
+#define PLD_MOTION_CAPTURE_MAPPING_FILE_PATH "/usr/share/capi-media-vision/models/PLD/mocap/example-mocap-mapping.txt"
+
+#define POSE_RESET_COUNT 5
+
+static float thPoseScore = 0.3f;
+static int thResetCount = 5;
+static float thCustom = 0.0f;
typedef struct _rect {
int left;
bool updated; // detection is find and pose is also good. then update drawing
bool locUpdated; // track previous detection
bool IsDetected; // detection
+ int label;
} HumanSkeleton;
typedef struct
static guint nFrames = 0;
// Gstreamer
-GstElement *pipeline, *source, *filter, *toverlay, *sink, *sink2;
+GstElement *pipeline, *source, *flip, *filter, *toverlay, *sink, *sink2;
GstElement *tee, *vscale, *vsfilter, *vconv, *vcfilter;
GstElement *vrate, *vrfilter, *fsink, *vrsink;
GstElement *queue1, *queue2, *queue3;
GstElement *vcrop, *vcrscale, *vcrsfilter, *vcrsconv, *vcrscfilter, *vcrssink;
CairoOverlayState *overlay_state;
-GstElement *decodebin, *dscale, *dconv;
+GstElement *dbin, *dscale, *dconv;
GstElement *dsfilter, *dcfilter;
GstElement *tee2, *enc, *muxmp4, *fsink2, *queue4, *queue5, *encconv;
mv_engine_config_h hp_mv_engine_cfg2;
mv_inference_h hp_mv_infer2;
+// pose comparison
+mv_pose_h hpPoser;
+float hpPoseScore;
+int hpPoseHoldTime;
+int hpPoseCount;
+
static void * outputTensorData;
FILE *fp;
free(outputTensorData);
outputTensorData = NULL;
}
+
+ if (hpPoser)
+ mv_pose_destroy(hpPoser);
+
printf(TEXT_YELLOW "exit..\n" TEXT_RESET);
signal(SIGINT, SIG_DFL);
exit(0);
int number_of_pose_estimation,
mv_inference_pose_s *locations,
int label,
- void *user_data)
+ void *user_data)
{
printf("%d landmarks\n", number_of_pose_estimation);
int width, height;
//printf("(%d,%d): %f\n", humanSkeleton.pose[n].x, humanSkeleton.pose[n].y, confidences[n]);
//printf("(%d,%d)\n", humanSkeleton.pose[n].x, humanSkeleton.pose[n].y);
}
+ humanSkeleton.label = label;
humanSkeleton.IsDetected = true;
-
}
static void _hand_detected_cb (
if (number_of_hands <= 0) {
humanSkeleton.IsDetected = false;
+ humanSkeleton.label = -1;
return;
}
clock_gettime(CLOCK_MONOTONIC, &s_tspec);
- mv_inference_pose_landmark_detect(mv_src_p2, hp_mv_infer2, NULL, _hand_pose_cb, NULL);
+ mv_inference_pose_landmark_detect(mv_src_p2, hp_mv_infer2, NULL, _hand_pose_cb, NULL);
clock_gettime(CLOCK_MONOTONIC, &e_tspec);
}
+void _human_pose_cb (
+ mv_source_h source,
+ int number_of_pose_estimation,
+ mv_inference_pose_s *locations,
+ int label,
+ void *user_data)
+{
+ printf("%d landmarks\n", number_of_pose_estimation);
+
+ mv_pose_compare(hpPoser, locations,
+ (MV_INFERENCE_HUMAN_BODY_PART_LEG_LEFT | MV_INFERENCE_HUMAN_BODY_PART_LEG_RIGHT),
+ &hpPoseScore);
+
+
+ if (hpPoseScore >= thPoseScore) {
+ hpPoseHoldTime++;
+ } else {
+ if (hpPoseHoldTime > thResetCount) {
+ hpPoseCount++;
+ }
+ hpPoseHoldTime = 0;
+ }
+
+ printf("Score: %.4f, HoldTime: %d, Count: %d\n", hpPoseScore, hpPoseHoldTime, hpPoseCount);
+ for (int n = 0; n < number_of_pose_estimation; ++n) {
+
+ humanSkeleton.pose[n].x = (int)(640.f*(float)(locations->landmarks[n].point.x)/192.f);
+ humanSkeleton.pose[n].y = (int)(480.f*(float)(locations->landmarks[n].point.y)/192.f);
+ humanSkeleton.scores[n] = locations->landmarks[n].score;
+
+ }
+ humanSkeleton.IsDetected = true;
+}
+
static gboolean
run_pose (void *user_data)
{
GstMapInfo map;
- /*
gst_buffer_map(udata->buffer, &map, GST_MAP_READ);
mv_source_clear(mv_src_p);
gst_buffer_unmap(udata->buffer, &map);
- clock_t start = clock();
- mv_inference_pose_estimation_detect(mv_src_p, hp_mv_infer, &poseRoi, 1.f, 1.f, _human_pose_cb, NULL);
- clock_t end = clock();
- */
+ struct timespec s_tspec;
+ struct timespec e_tspec;
+
+ clock_gettime(CLOCK_MONOTONIC, &s_tspec);
+
+ mv_inference_pose_landmark_detect(mv_src_p, hp_mv_infer, &poseRoi, _human_pose_cb, NULL);
+
+ clock_gettime(CLOCK_MONOTONIC, &e_tspec);
+ struct timespec diffspec = diff(s_tspec, e_tspec);
+ unsigned long timeDiff = gettotalmillisec(diffspec);
+ printf("detect + pose time: %lu(ms)\n", timeDiff);
+
+ return FALSE;
+
+}
+
+static gboolean
+run_hand (void *user_data)
+{
+ HandOffData *udata = (HandOffData *)user_data;
+ if (!GST_IS_BUFFER(udata->buffer))
+ return FALSE;
+
+ GstMapInfo map;
gst_buffer_map(udata->buffer, &map, GST_MAP_READ);
#endif
}
+static void
+_hand_est_handoff(GstElement *object, GstBuffer *buffer, GstPad *pad, gpointer user_data)
+{
+
+ nFrames++;
+ hdata_p.buffer = buffer;
+ hdata_p.user_data = user_data;
+
+#if 0
+ if (nFrames % 15 == 0) {
+ g_mutex_lock(&pose_mutex);
+ g_idle_add (run_hand, &hdata_p);
+ g_mutex_unlock(&pose_mutex);
+ }
+#else
+ g_mutex_lock(&pose_mutex);
+ g_idle_add (run_hand, &hdata_p);
+ g_mutex_unlock(&pose_mutex);
+#endif
+}
+
+
static void
prepare_overlay (GstElement * overlay, GstCaps * caps, gpointer user_data)
{
state->valid = gst_video_info_from_caps (&state->vinfo, caps);
}
-/*
static void
-draw_overlay (GstElement * overlay, cairo_t * cr, guint64 timestamp,
+draw_overlay_pose (GstElement * overlay, cairo_t * cr, guint64 timestamp,
guint64 duration, gpointer user_data)
{
CairoOverlayState *s = (CairoOverlayState *) user_data;
cairo_set_source_rgba(cr, 0.1, 0.9, 0.0, 0.7);
cairo_set_line_width(cr, 2.0);
-
+ float poseWeight = 0.7f;
+ float prevPoseWeight = 0.3f;
if (!humanSkeleton.IsDetected)
return;
+ for (int k = 0; k < 16; ++k) {
+ if (humanSkeleton.isPrevPose == false) {
+ humanSkeleton.prevPose[k] = humanSkeleton.pose[k];
+ } else {
- //
- if (humanSkeleton.isPrevPose == false) {
- humanSkeleton.prevPose[1] = humanSkeleton.pose[1];
- // head - neck
- if (humanSkeleton.scores[0] >= thValNeck ) {
- humanSkeleton.prevPose[0] = humanSkeleton.pose[0];
- }
-
- // right arm
- if (humanSkeleton.scores[2] >= thValArm) {
- // neck - right shoulder
- humanSkeleton.prevPose[2] = humanSkeleton.pose[2];
- if (humanSkeleton.scores[3] >= thValArm) {
- // right shoulder - right elbow
- humanSkeleton.prevPose[3] = humanSkeleton.pose[3];
- if (humanSkeleton.scores[4] >= thValArm) {
- // right elbow - right wrist
- humanSkeleton.prevPose[4] = humanSkeleton.pose[4];
- }
- }
- }
-
- // left arm
- if (humanSkeleton.scores[5] >= thValArm) {
- // neck - right shoulder
- humanSkeleton.prevPose[5] = humanSkeleton.pose[5];
- if (humanSkeleton.scores[6] >= thValArm) {
- // right shoulder - right elbow
- humanSkeleton.prevPose[6] = humanSkeleton.pose[6];
- if (humanSkeleton.scores[7] >= thValArm) {
- // right elbow - right wrist
- humanSkeleton.prevPose[7] = humanSkeleton.pose[7];
- }
- }
- }
-
- // right leg
- if (humanSkeleton.scores[8] >= thValLeg) {
- // neck - right shoulder
- humanSkeleton.prevPose[8] = humanSkeleton.pose[8];
- if (humanSkeleton.scores[9] >= thValLeg) {
- // right shoulder - right elbow
- humanSkeleton.prevPose[9] = humanSkeleton.pose[9];
- if (humanSkeleton.scores[10] >= thValLeg) {
- // right elbow - right wrist
- humanSkeleton.prevPose[10] = humanSkeleton.pose[10];
- }
- }
- }
-
- // left leg
- if (humanSkeleton.scores[11] >= thValLeg) {
- // neck - right shoulder
- humanSkeleton.prevPose[11] = humanSkeleton.pose[11];
- if (humanSkeleton.scores[12] >= thValLeg) {
- // right shoulder - right elbow
- humanSkeleton.prevPose[12] = humanSkeleton.pose[12];
- if (humanSkeleton.scores[13] >= thValLeg) {
- // right elbow - right wrist
- humanSkeleton.prevPose[13] = humanSkeleton.pose[13];
- }
- }
- }
- humanSkeleton.isPrevPose = true;
- } else {
- // weighted sum of pose and prevPose
- // method1: fixed weights (pose : prevPose = 0.7 : 0.3)
- float poseWeight = 0.7f;
- float prevPoseWeight = 0.3f;
- humanSkeleton.prevPose[1].x = (poseWeight * humanSkeleton.pose[1].x +
- prevPoseWeight * humanSkeleton.prevPose[1].x);
- humanSkeleton.prevPose[1].y = (poseWeight * humanSkeleton.pose[1].y +
- prevPoseWeight * humanSkeleton.prevPose[1].y);
- // head - neck
- if (humanSkeleton.scores[0] >= thValNeck ) {
- humanSkeleton.prevPose[0].x = (poseWeight * humanSkeleton.pose[0].x +
- prevPoseWeight * humanSkeleton.prevPose[0].x);
- humanSkeleton.prevPose[0].y = (poseWeight * humanSkeleton.pose[0].y +
- prevPoseWeight * humanSkeleton.prevPose[0].y);
+ humanSkeleton.prevPose[k].x = (poseWeight * humanSkeleton.pose[k].x +
+ prevPoseWeight * humanSkeleton.prevPose[k].x);
+ humanSkeleton.prevPose[k].y = (poseWeight * humanSkeleton.pose[k].y +
+ prevPoseWeight * humanSkeleton.prevPose[k].y);
}
+ }
- // right arm
- if (humanSkeleton.scores[2] >= thValArm) {
- // neck - right shoulder
- humanSkeleton.prevPose[2].x = (poseWeight * humanSkeleton.pose[2].x +
- prevPoseWeight * humanSkeleton.prevPose[2].x);
- humanSkeleton.prevPose[2].y = (poseWeight * humanSkeleton.pose[2].y +
- prevPoseWeight * humanSkeleton.prevPose[2].y);
- if (humanSkeleton.scores[3] >= thValArm) {
- // right shoulder - right elbow
- humanSkeleton.prevPose[3].x = (poseWeight * humanSkeleton.pose[3].x +
- prevPoseWeight * humanSkeleton.prevPose[3].x);
- humanSkeleton.prevPose[3].y = (poseWeight * humanSkeleton.pose[3].y +
- prevPoseWeight * humanSkeleton.prevPose[3].y);
- if (humanSkeleton.scores[4] >= thValArm) {
- // right elbow - right wrist
- humanSkeleton.prevPose[4].x = (poseWeight * humanSkeleton.pose[4].x +
- prevPoseWeight * humanSkeleton.prevPose[4].x);
- humanSkeleton.prevPose[4].y = (poseWeight * humanSkeleton.pose[4].y +
- prevPoseWeight * humanSkeleton.prevPose[4].y);
- }
- }
- }
+ for (int k = 0; k < 16; ++k) {
+ if (humanSkeleton.scores[k] > 0.0f) {
+ if (k > 9)
+ cairo_set_source_rgba(cr, 0.9, 0.1, 0.0, 0.7);
- // left arm
- if (humanSkeleton.scores[5] >= thValArm) {
- // neck - right shoulder
- humanSkeleton.prevPose[5].x = (poseWeight * humanSkeleton.pose[5].x +
- prevPoseWeight * humanSkeleton.prevPose[5].x);
- humanSkeleton.prevPose[5].y = (poseWeight * humanSkeleton.pose[5].y +
- prevPoseWeight * humanSkeleton.prevPose[5].y);
- if (humanSkeleton.scores[6] >= thValArm) {
- // right shoulder - right elbow
- humanSkeleton.prevPose[6].x = (poseWeight * humanSkeleton.pose[6].x +
- prevPoseWeight * humanSkeleton.prevPose[6].x);
- humanSkeleton.prevPose[6].y = (poseWeight * humanSkeleton.pose[6].y +
- prevPoseWeight * humanSkeleton.prevPose[6].y);
- if (humanSkeleton.scores[7] >= thValArm) {
- // right elbow - right wrist
- humanSkeleton.prevPose[7].x = (poseWeight * humanSkeleton.pose[7].x +
- prevPoseWeight * humanSkeleton.prevPose[7].x);
- humanSkeleton.prevPose[7].y = (poseWeight * humanSkeleton.pose[7].y +
- prevPoseWeight * humanSkeleton.prevPose[7].y);
- }
- }
+ cairo_arc(cr, humanSkeleton.prevPose[k].x, humanSkeleton.prevPose[k].y, 3,0, 2*M_PI);
+ cairo_stroke(cr);
}
+ }
- // right leg
- if (humanSkeleton.scores[8] >= thValLeg) {
- // neck - right shoulder
- humanSkeleton.prevPose[8].x = (poseWeight * humanSkeleton.pose[8].x +
- prevPoseWeight * humanSkeleton.prevPose[8].x);
- humanSkeleton.prevPose[8].y = (poseWeight * humanSkeleton.pose[8].y +
- prevPoseWeight * humanSkeleton.prevPose[8].y);
- if (humanSkeleton.scores[9] >= thValLeg) {
- // right shoulder - right elbow
- humanSkeleton.prevPose[9].x = (poseWeight * humanSkeleton.pose[9].x +
- prevPoseWeight * humanSkeleton.prevPose[9].x);
- humanSkeleton.prevPose[9].y = (poseWeight * humanSkeleton.pose[9].y +
- prevPoseWeight * humanSkeleton.prevPose[9].y);
- if (humanSkeleton.scores[10] >= thValLeg) {
- // right elbow - right wrist
- humanSkeleton.prevPose[10].x = (poseWeight * humanSkeleton.pose[10].x +
- prevPoseWeight * humanSkeleton.prevPose[10].x);
- humanSkeleton.prevPose[10].y = (poseWeight * humanSkeleton.pose[10].y +
- prevPoseWeight * humanSkeleton.prevPose[10].y);
- }
- }
- }
- // left leg
- if (humanSkeleton.scores[11] >= thValLeg) {
- // neck - right shoulder
- humanSkeleton.prevPose[11].x = (poseWeight * humanSkeleton.pose[11].x +
- prevPoseWeight * humanSkeleton.prevPose[11].x);
- humanSkeleton.prevPose[11].y = (poseWeight * humanSkeleton.pose[11].y +
- prevPoseWeight * humanSkeleton.prevPose[11].y);
- if (humanSkeleton.scores[12] >= thValLeg) {
- // right shoulder - right elbow
- humanSkeleton.prevPose[12].x = (poseWeight * humanSkeleton.pose[12].x +
- prevPoseWeight * humanSkeleton.prevPose[12].x);
- humanSkeleton.prevPose[12].y = (poseWeight * humanSkeleton.pose[12].y +
- prevPoseWeight * humanSkeleton.prevPose[12].y);
- if (humanSkeleton.scores[13] >= thValLeg) {
- // right elbow - right wrist
- humanSkeleton.prevPose[13].x = (poseWeight * humanSkeleton.pose[13].x +
- prevPoseWeight * humanSkeleton.prevPose[13].x);
- humanSkeleton.prevPose[13].y = (poseWeight * humanSkeleton.pose[13].y +
- prevPoseWeight * humanSkeleton.prevPose[13].y);
- }
- }
- }
- }
+ cairo_select_font_face(cr, "sans-serif", CAIRO_FONT_SLANT_NORMAL, CAIRO_FONT_WEIGHT_BOLD);
+ cairo_set_font_size(cr,15);
- //
- //draw..
- // head - neck
- if (humanSkeleton.scores[0] >= thValNeck ) {
- cairo_move_to(cr, humanSkeleton.prevPose[0].x, humanSkeleton.prevPose[0].y);
- cairo_line_to(cr, humanSkeleton.prevPose[1].x, humanSkeleton.prevPose[1].y);
- }
-
- // right arm
- cairo_move_to(cr, humanSkeleton.prevPose[1].x, humanSkeleton.prevPose[1].y);
- if (humanSkeleton.scores[2] >= thValArm) {
- // neck - right shoulder
- cairo_line_to(cr, humanSkeleton.prevPose[2].x, humanSkeleton.prevPose[2].y);
- if (humanSkeleton.scores[3] >= thValArm) {
- // right shoulder - right elbow
- cairo_line_to(cr, humanSkeleton.prevPose[3].x, humanSkeleton.prevPose[3].y);
- if (humanSkeleton.scores[4] >= thValArm) {
- // right elbow - right wrist
- cairo_line_to(cr, humanSkeleton.prevPose[4].x, humanSkeleton.prevPose[4].y);
- }
- }
- }
- cairo_stroke(cr);
+ char howToText[2][1024];
+ char tmpText[3][1024];
+ snprintf(howToText[0], 1024, "Until %d hold time over Score: %.2f", thResetCount, thPoseScore);
+ snprintf(howToText[1], 1024, "Green-Hold, Blue-Release");
+ snprintf(tmpText[0], 1024, "Score: %.4f", hpPoseScore);
+ snprintf(tmpText[1], 1024, "Hold time: %d", hpPoseHoldTime > thResetCount ? thResetCount : hpPoseHoldTime);
+ snprintf(tmpText[2], 1024, "Count: %2d", hpPoseCount);
- // left arm
- cairo_move_to(cr, humanSkeleton.prevPose[1].x, humanSkeleton.prevPose[1].y);
- if (humanSkeleton.scores[5] >= thValArm) {
- // neck - right shoulder
- cairo_line_to(cr, humanSkeleton.prevPose[5].x, humanSkeleton.prevPose[5].y);
- if (humanSkeleton.scores[6] >= thValArm) {
- // right shoulder - right elbow
- cairo_line_to(cr, humanSkeleton.prevPose[6].x, humanSkeleton.prevPose[6].y);
- if (humanSkeleton.scores[7] >= thValArm) {
- // right elbow - right wrist
- cairo_line_to(cr, humanSkeleton.prevPose[7].x, humanSkeleton.prevPose[7].y);
- }
- }
- }
- cairo_stroke(cr);
+ cairo_set_source_rgba(cr, 1.0, 1.0, 1.0, 0.7);
+ cairo_move_to(cr, 400.0, 30.0);
+ cairo_show_text(cr, howToText[0]);
+ cairo_move_to(cr, 420.0, 45.0);
+ cairo_show_text(cr, howToText[1]);
+ if (hpPoseScore >= thPoseScore) {
+ if (hpPoseHoldTime > thResetCount)
+ cairo_set_source_rgba(cr, 0.0, 0.1, 0.9, 0.7);
+ else
+ cairo_set_source_rgba(cr, 0.1, 0.9, 0.0, 0.7);
- // right leg
- cairo_move_to(cr, humanSkeleton.prevPose[1].x, humanSkeleton.prevPose[1].y);
- if (humanSkeleton.scores[8] >= thValLeg) {
- // neck - right shoulder
- cairo_line_to(cr, humanSkeleton.prevPose[8].x, humanSkeleton.prevPose[8].y);
- if (humanSkeleton.scores[9] >= thValLeg) {
- // right shoulder - right elbow
- cairo_line_to(cr, humanSkeleton.prevPose[9].x, humanSkeleton.prevPose[9].y);
- if (humanSkeleton.scores[10] >= thValLeg) {
- // right elbow - right wrist
- cairo_line_to(cr, humanSkeleton.prevPose[10].x, humanSkeleton.prevPose[10].y);
- }
- }
- }
- cairo_stroke(cr);
+ cairo_move_to(cr, 420.0, 60.0);
+ cairo_show_text(cr, tmpText[0]);
+ cairo_move_to(cr, 420.0, 75.0);
+ cairo_show_text(cr, tmpText[1]);
+ cairo_move_to(cr, 420.0, 90.0);
+ cairo_show_text(cr, tmpText[2]);
- // left leg
- cairo_move_to(cr, humanSkeleton.prevPose[1].x, humanSkeleton.prevPose[1].y);
- if (humanSkeleton.scores[11] >= thValLeg) {
- // neck - right shoulder
- cairo_line_to(cr, humanSkeleton.prevPose[11].x, humanSkeleton.prevPose[11].y);
- if (humanSkeleton.scores[12] >= thValLeg) {
- // right shoulder - right elbow
- cairo_line_to(cr, humanSkeleton.prevPose[12].x, humanSkeleton.prevPose[12].y);
- if (humanSkeleton.scores[13] >= thValLeg) {
- // right elbow - right wrist
- cairo_line_to(cr, humanSkeleton.prevPose[13].x, humanSkeleton.prevPose[13].y);
- }
- }
+ } else {
+ cairo_set_source_rgba(cr, 0.9, 0.1, 0.0, 0.7);
+ cairo_move_to(cr, 420.0, 60.0);
+ cairo_show_text(cr, tmpText[0]);
+ cairo_move_to(cr, 420.0, 75.0);
+ cairo_show_text(cr, tmpText[1]);
+ cairo_move_to(cr, 420.0, 90.0);
+ cairo_show_text(cr, tmpText[2]);
}
- cairo_stroke(cr);
}
-*/
+
static void
draw_overlay_hand (GstElement * overlay, cairo_t * cr, guint64 timestamp,
guint64 duration, gpointer user_data)
return;
}
- cairo_set_source_rgba(cr, 0.1, 0.9, 0.0, 0.7);
- cairo_set_line_width(cr, 2.0);
+ cairo_set_source_rgba(cr, 0.0, 0.0, 1.0, 0.7);
+ cairo_set_line_width(cr, 2.0);
if (!humanSkeleton.IsDetected)
return;
+ cairo_select_font_face(cr, "sans-serif", CAIRO_FONT_SLANT_NORMAL, CAIRO_FONT_WEIGHT_BOLD);
+ cairo_set_font_size(cr,15);
+ cairo_move_to(cr, 320.0, 48.0);
+ cairo_show_text(cr, gHandGestureLabel[humanSkeleton.label]);
+
+ cairo_set_source_rgba(cr, 0.1, 0.9, 0.0, 0.7);
+ cairo_set_line_width(cr, 2.0);
// thumb - red
cairo_set_source_rgba (cr, 0.9, 0.1, 0.0, 0.7);
cairo_line_to(cr, humanSkeleton.pose[k].x, humanSkeleton.pose[k].y);
}
cairo_stroke(cr);
+
+
}
static gboolean bus_call (GstBus *bus, GstMessage *msg, gpointer data)
MV_INFERENCE_MODEL_WEIGHT_FILE_PATH,
PE_TFLITE_CPM_WEIGHT_PATH);
+ mv_engine_config_set_string_attribute(
+ mv_engine_cfg,
+ MV_INFERENCE_MODEL_USER_FILE_PATH,
+ PE_POSE_LABEL_PATH);
+
mv_engine_config_set_int_attribute(mv_engine_cfg,
MV_INFERENCE_INPUT_DATA_TYPE,
MV_INFERENCE_DATA_FLOAT32);
mv_engine_config_set_int_attribute(mv_engine_cfg,
MV_INFERENCE_TARGET_TYPE,
- MV_INFERENCE_TARGET_CPU);
+ MV_INFERENCE_TARGET_GPU);
mv_engine_config_set_int_attribute(mv_engine_cfg,
MV_INFERENCE_INPUT_TENSOR_WIDTH,
return MEDIA_VISION_ERROR_NONE;
}
-int perform_armnn_human_pose_hourglass_configure(mv_engine_config_h mv_engine_cfg)
+int perform_tflite_human_pose_cpm_configure(mv_engine_config_h mv_engine_cfg)
{
if (mv_engine_cfg == NULL) {
printf("mv_engine_cfg is null\n");
}
char *inputNodeName = "image";
- char *outputNodeName[1] = {"hourglass_out_3"};
+ char *outputNodeName[1] = {"Convolutional_Pose_Machine/stage_5_out"};
mv_engine_config_set_string_attribute(mv_engine_cfg,
MV_INFERENCE_MODEL_WEIGHT_FILE_PATH,
- PE_TFLITE_HOURGLASS_WEIGHT_PATH);
+ PE_TFLITE_CPM_WEIGHT_PATH);
+
+ mv_engine_config_set_string_attribute(
+ mv_engine_cfg,
+ MV_INFERENCE_MODEL_USER_FILE_PATH,
+ PE_POSE_LABEL_PATH);
mv_engine_config_set_int_attribute(mv_engine_cfg,
MV_INFERENCE_INPUT_DATA_TYPE,
}
char *inputNodeName = "input";
- char *outputNodeNames[2] = {"mobilenetv2/boundingbox2", "mobilenetv2/heatmap"};
+ char *outputNodeNames[2] = {"mobilenetv2/boundingbox", "mobilenetv2/heatmap"};
//outputTensorData = (void*)calloc(56*56*21, sizeof(float));
mv_engine_config_set_string_attribute(mv_engine_cfg,
mv_engine_config_set_int_attribute(mv_engine_cfg,
MV_INFERENCE_TARGET_DEVICE_TYPE,
- MV_INFERENCE_TARGET_DEVICE_CPU);
+ MV_INFERENCE_TARGET_DEVICE_GPU);
mv_engine_config_set_int_attribute(mv_engine_cfg,
MV_INFERENCE_INPUT_TENSOR_WIDTH,
mv_engine_config_set_int_attribute(mv_engine_cfg,
MV_INFERENCE_TARGET_DEVICE_TYPE,
- MV_INFERENCE_TARGET_DEVICE_CPU);
+ MV_INFERENCE_TARGET_DEVICE_GPU);
mv_engine_config_set_int_attribute(mv_engine_cfg,
MV_INFERENCE_INPUT_TENSOR_WIDTH,
mv_engine_config_set_int_attribute(mv_engine_cfg,
MV_INFERENCE_TARGET_DEVICE_TYPE,
- MV_INFERENCE_TARGET_DEVICE_CPU);
+ MV_INFERENCE_TARGET_DEVICE_GPU);
mv_engine_config_set_int_attribute(mv_engine_cfg,
MV_INFERENCE_INPUT_TENSOR_WIDTH,
mv_engine_config_set_int_attribute(mv_engine_cfg,
MV_INFERENCE_TARGET_DEVICE_TYPE,
- MV_INFERENCE_TARGET_DEVICE_CPU);
+ MV_INFERENCE_TARGET_DEVICE_GPU);
mv_engine_config_set_int_attribute(mv_engine_cfg,
MV_INFERENCE_INPUT_TENSOR_WIDTH,
return MEDIA_VISION_ERROR_NONE;
}
-int perform_tflite_hand_detection_AICLite3_1(mv_engine_config_h mv_engine_cfg)
+int perform_tflite_hand_detection_AICLiteQ(mv_engine_config_h mv_engine_cfg)
{
if (mv_engine_cfg == NULL) {
printf("mv_engine_cfg is null\n");
}
char *inputNodeName = "input";
- char *outputNodeNames[2] = {"ban_1_6/boundingbox", "ban_1_6/heatmap"};
+ char *outputNodeNames[2] = {"mobilenetv2/boundingbox", "mobilenetv2/heatmap"};
- //outputTensorData = (void*)calloc(56*56*21, sizeof(float));
+ //outputTensorData = (void*)calloc(56*56*21, sizeof(char));
mv_engine_config_set_string_attribute(mv_engine_cfg,
MV_INFERENCE_MODEL_WEIGHT_FILE_PATH,
- PE_TFLITE_AICLite3_1_WEIGHT_PATH);
+ PE_TFLITE_AICLiteQ_1_WEIGHT_PATH);
mv_engine_config_set_int_attribute(mv_engine_cfg,
MV_INFERENCE_INPUT_DATA_TYPE,
return MEDIA_VISION_ERROR_NONE;
}
-int perform_tflite_hand_detection_AICLite3_2(mv_engine_config_h mv_engine_cfg)
+int perform_tflite_hand_detection_AICLiteQ2(mv_engine_config_h mv_engine_cfg)
{
char *inputNodeName = "input";
- char *outputNodeNames[2] = {"ban_1_6/coords", "ban_1_6/gesture"};
+ char *outputNodeNames[2] = {"mobilenetv2/coord_refine", "mobilenetv2/gesture"};
mv_engine_config_set_string_attribute(mv_engine_cfg,
MV_INFERENCE_MODEL_WEIGHT_FILE_PATH,
- PE_TFLITE_AICLite3_2_WEIGHT_PATH);
+ PE_TFLITE_AICLiteQ_2_WEIGHT_PATH);
mv_engine_config_set_int_attribute(mv_engine_cfg,
MV_INFERENCE_INPUT_DATA_TYPE,
if (ad->modelType == MODEL_TYPE_POSE_CPM) {
err = perform_armnn_human_pose_cpm_configure(hp_mv_engine_cfg);
- } else if (ad->modelType == MODEL_TYPE_POSE_HOURGLASS) {
- err = perform_armnn_human_pose_hourglass_configure(hp_mv_engine_cfg);
- } else if (ad->modelType == MODEL_TYPE_POSE_HAND_AIC) {
+ //err = perform_tflite_human_pose_cpm_configure(hp_mv_engine_cfg);
+
+ mv_pose_create(&hpPoser);
+ mv_pose_set_from_file(hpPoser,
+ PLD_MOTION_CAPTURE_FILE_PATH,
+ PLD_MOTION_CAPTURE_MAPPING_FILE_PATH);
+
+ } else if (ad->modelType == MODEL_TYPE_POSE_HAND_AICLite) {
outputTensorData = (void*)calloc(56*56*21, sizeof(float));
err = perform_tflite_hand_detection_AIC(hp_mv_engine_cfg);
err = perform_tflite_hand_detection_AIC2(hp_mv_engine_cfg2);
- } else if (ad->modelType == MODEL_TYPE_POSE_HAND_AICLite3) {
+ } else if (ad->modelType == MODEL_TYPE_POSE_HAND_AICLite2) {
outputTensorData = (void*)calloc(56*56*21, sizeof(float));
- err = perform_tflite_hand_detection_AICLite3_1(hp_mv_engine_cfg);
+ err = perform_tflite_hand_detection_AICLite(hp_mv_engine_cfg);
- err = perform_tflite_hand_detection_AICLite3_2(hp_mv_engine_cfg2);
+ err = perform_tflite_hand_detection_AICLite2(hp_mv_engine_cfg2);
} else {
outputTensorData = (void*)calloc(56*56*21, sizeof(float));
- err = perform_tflite_hand_detection_AICLite(hp_mv_engine_cfg);
+ err = perform_tflite_hand_detection_AICLiteQ(hp_mv_engine_cfg);
- err = perform_tflite_hand_detection_AICLite2(hp_mv_engine_cfg2);
+ err = perform_tflite_hand_detection_AICLiteQ2(hp_mv_engine_cfg2);
}
if (err != MEDIA_VISION_ERROR_NONE) {
clock_t end = clock();
printf("time: %2.3f\n", (double)(end - start)/CLOCKS_PER_SEC);
+ /* mediavision source */
+ mv_create_source(&mv_src_p);
- mv_inference_create(&hp_mv_infer2);
+ if (ad->modelType >= MODEL_TYPE_POSE_HAND_AICLite) {
+ mv_inference_create(&hp_mv_infer2);
- mv_inference_configure(hp_mv_infer2, hp_mv_engine_cfg2);
+ mv_inference_configure(hp_mv_infer2, hp_mv_engine_cfg2);
- start = clock();
- mv_inference_prepare(hp_mv_infer2);
- end = clock();
- printf("time: %2.3f\n", (double)(end - start)/CLOCKS_PER_SEC);
+ start = clock();
+ mv_inference_prepare(hp_mv_infer2);
+ end = clock();
+ printf("time: %2.3f\n", (double)(end - start)/CLOCKS_PER_SEC);
- /* mediavision source */
- mv_create_source(&mv_src_p);
- mv_create_source(&mv_src_p2);
+ mv_create_source(&mv_src_p2);
+ }
pipeline = gst_pipeline_new("app");
} else {
source = gst_element_factory_make("filesrc", "src");
- decodebin = gst_element_factory_make("decodebin", "dbin");
+ dbin = gst_element_factory_make("decodebin", "dbin");
dscale = gst_element_factory_make("videoscale", "dscale");
dconv = gst_element_factory_make("videoconvert", "dconv");
dsfilter = gst_element_factory_make("capsfilter", "dsfilter");
queue4 = gst_element_factory_make("queue", "queue4");
queue5 = gst_element_factory_make("queue", "queue5");
encconv = gst_element_factory_make("videoconvert", "encconv");
+ g_object_set(G_OBJECT(enc), "bitrate", 800000, NULL);
}
-
+ flip = gst_element_factory_make("videoflip", "vflip" );
tee = gst_element_factory_make("tee", "tee");
queue1 = gst_element_factory_make("queue", "queue1");
queue2 = gst_element_factory_make("queue", "queue2");
}
g_object_set(G_OBJECT(filter), "caps", gst_caps_from_string("video/x-raw, format=YUY2, width=640, height=480"), NULL);
} else {
- if (!decodebin || !dscale || !dconv) {
+ if (!dbin || !dscale || !dconv) {
printf(TEXT_RED "One element might be not created. Exiting.\n" TEXT_RESET);
return -1;
}
g_object_set(G_OBJECT(dsfilter), "caps", gst_caps_from_string("video/x-raw, width=640, height=480"), NULL);
g_object_set(G_OBJECT(dcfilter), "caps", gst_caps_from_string("video/x-raw, format=YUY2, width=640, height=480"), NULL);
- g_signal_connect (decodebin, "pad-added", G_CALLBACK (cb_new_pad), NULL);
+ g_signal_connect (dbin, "pad-added", G_CALLBACK (cb_new_pad), NULL);
}
if (ad->filename2) {
/* set up the pipeline */
//g_signal_connect (coverlay, "draw", G_CALLBACK (draw_overlay), overlay_state);
- g_signal_connect (coverlay, "draw", G_CALLBACK (draw_overlay_hand), overlay_state);
+ if (ad->modelType == MODEL_TYPE_POSE_CPM) {
+ g_signal_connect (coverlay, "draw", G_CALLBACK (draw_overlay_pose), overlay_state);
+ } else {
+ g_signal_connect (coverlay, "draw", G_CALLBACK (draw_overlay_hand), overlay_state);
+ }
g_signal_connect (coverlay, "caps-changed", G_CALLBACK (prepare_overlay), overlay_state);
if (!ad->filename) {
- g_object_set(G_OBJECT(source), "device", "/dev/video8", NULL); // 252
+ g_object_set(G_OBJECT(source), "device", "/dev/video252", NULL); // 252
+
} else {
g_object_set(G_OBJECT(source), "location", ad->filename, NULL);
- g_object_set(G_OBJECT(source), "num-buffers", ad->numbuffers, NULL);
}
-
+ g_object_set(G_OBJECT(flip), "method", 4, NULL);
g_object_set(G_OBJECT(sink2), "use-tbm", FALSE, NULL);
g_object_set(G_OBJECT(sink2), "sync", FALSE, NULL);
g_object_set(G_OBJECT(sink), "video-sink", sink2, NULL);
g_object_set(G_OBJECT(sink), "sync", FALSE, NULL);
- if (ad->modelType == MODEL_TYPE_POSE_HAND_AIC ||
- ad->modelType == MODEL_TYPE_POSE_HAND_AICLite ||
- ad->modelType == MODEL_TYPE_POSE_HAND_AICLite3) {
+ if (ad->modelType == MODEL_TYPE_POSE_HAND_AICLite ||
+ ad->modelType == MODEL_TYPE_POSE_HAND_AICLite2 ||
+ ad->modelType == MODEL_TYPE_POSE_HAND_AICLite2Q) {
g_object_set(G_OBJECT(vsfilter), "caps", gst_caps_from_string("video/x-raw, width=224, height=224"), NULL);
poseCropSize = 224;
} else {
// here to be continue
printf("vrsink signal-handoffs\n");
g_object_set(G_OBJECT(vrsink), "signal-handoffs", TRUE, NULL);
- handler_p = g_signal_connect (vrsink, "handoff", G_CALLBACK(_pose_est_handoff), outputTensorData);
+
+ if (ad->modelType == MODEL_TYPE_POSE_CPM) {
+ handler_p = g_signal_connect (vrsink, "handoff", G_CALLBACK(_pose_est_handoff), NULL);
+ } else {
+ handler_p = g_signal_connect (vrsink, "handoff", G_CALLBACK(_hand_est_handoff), outputTensorData);
+ }
+
g_object_set(G_OBJECT(vrsink), "sync", FALSE, NULL);
bus_watch_id = gst_bus_add_watch(bus, bus_call, NULL);
gst_object_unref(bus);
+ printf("eleeent would be added.\n");
/* add elements into the pipeline */
gst_bin_add_many(GST_BIN(pipeline),
source,
- tee, queue1, vscale, vsfilter, vconv, vcfilter,
+ flip, tee, queue1, vscale, vsfilter, vconv, vcfilter,
vrate, vrfilter, vrsink,
queue2, oconv, coverlay, sink,
NULL);
/* link elements */
if (!ad->filename) {
+ printf("take !ad->filename 1\n");
gst_bin_add(GST_BIN(pipeline), filter);
- gst_element_link_many(source, filter, tee, NULL);
+ gst_element_link_many(source, flip, filter, tee, NULL);
} else {
- gst_bin_add_many(GST_BIN(pipeline), decodebin, dscale, dconv, dsfilter, dcfilter, NULL);
- gst_element_link_many(source, decodebin, NULL);
+ gst_bin_add_many(GST_BIN(pipeline), dbin, dscale, dconv, dsfilter, dcfilter, NULL);
+ gst_element_link_many(source, dbin, NULL);
gst_element_link_many(dscale, dsfilter, dconv, dcfilter, tee, NULL);
}
-
if (!ad->filename2) {
// display
gst_element_link_many (tee, queue2, oconv, coverlay, /*toverlay,*/ sink, NULL);
/* loop */
humanSkeleton.IsDetected = false;
humanSkeleton.isPrevPose = false;
+ humanSkeleton.label = -1;
printf(TEXT_GREEN "Running.....\n" TEXT_RESET);
// GST_END
#if 0
}
if (argc < 2) {
- printf("usage: mv_stream_infer model [NeckThresVal, ArmThresVal, LegThresVal, [filename]]");
- printf("model: 0(CPM), 1(HOURGLASS), 2(AIC Hand), 3(AIC Lite Hand\n");
+ printf("usage: mv_stream_infer model [thPoseScore, thResetCount, thCustom, [filename]]");
+ printf("model: 0(CPM), 1(AIC Hand), 2(AIC Lite Hand), 3(AIC Lite Q Hand)\n");
return -1;
}
ad.modelType = atoi(argv[1]);
- if (ad.modelType < 0 || ad.modelType > 4) {
- printf("not supported model type\n");
+ if (ad.modelType < 0 || ad.modelType > 3) {
+ printf("not supported model type [%d]\n", ad.modelType);
return -1;
}
- if (ad.modelType != MODEL_TYPE_POSE_HAND_AIC &&
- ad.modelType != MODEL_TYPE_POSE_HAND_AICLite &&
- ad.modelType != MODEL_TYPE_POSE_HAND_AICLite3) {
- thValNeck = (float)atoi(argv[2])/100.f;
- thValArm = (float)atoi(argv[3])/100.f;
- thValLeg = (float)atoi(argv[4])/100.f;
+ if (ad.modelType != MODEL_TYPE_POSE_HAND_AICLite &&
+ ad.modelType != MODEL_TYPE_POSE_HAND_AICLite2 &&
+ ad.modelType != MODEL_TYPE_POSE_HAND_AICLite2Q) {
+ thPoseScore = (float)atoi(argv[2])/100.f;
+ thResetCount = atoi(argv[3]);
+ thCustom = (float)atoi(argv[4])/100.f;
poseRoi.point.x = 50;
poseRoi.point.y = 0;
projects / platform / core / api / mediavision.git / commitdiff