From 65f821fa12033d35e13a80850a94e220fb86830e Mon Sep 17 00:00:00 2001 From: HyoungjooAhn Date: Fri, 19 Oct 2018 19:52:35 +0900 Subject: [PATCH] [Filter/Example] update multi objects detection Multi-Output Tensor Example: detect the multiple objects through ssd model. Signed-off-by: HyoungjooAhn --- debian/control | 2 +- nnstreamer_example/CMakeLists.txt | 1 + .../example_object_detection/CMakeLists.txt | 42 ++ .../nnstreamer_example_object_detection.cc | 739 +++++++++++++++++++++ packaging/nnstreamer.spec | 3 + 5 files changed, 786 insertions(+), 1 deletion(-) create mode 100644 nnstreamer_example/example_object_detection/CMakeLists.txt create mode 100644 nnstreamer_example/example_object_detection/nnstreamer_example_object_detection.cc diff --git a/debian/control b/debian/control index 8c6e87e..985c1f5 100644 --- a/debian/control +++ b/debian/control @@ -6,7 +6,7 @@ Build-Depends: gcc, cmake, libgstreamer1.0-dev, libgstreamer-plugins-base1.0-dev libgtest-dev, debhelper (>=9), gstreamer1.0-tools, gstreamer1.0-plugins-base, gstreamer1.0-plugins-good, - libpng-dev, tensorflow-lite-dev, tensorflow-dev + libpng-dev, tensorflow-lite-dev, tensorflow-dev, libcairo2-dev Standards-Version: 3.9.6 Homepage: https://github.com/nnsuite/nnstreamer diff --git a/nnstreamer_example/CMakeLists.txt b/nnstreamer_example/CMakeLists.txt index 1e20392..1358d6f 100644 --- a/nnstreamer_example/CMakeLists.txt +++ b/nnstreamer_example/CMakeLists.txt @@ -15,5 +15,6 @@ ADD_SUBDIRECTORY(custom_example_average) ADD_SUBDIRECTORY(example_cam) ADD_SUBDIRECTORY(example_sink) ADD_SUBDIRECTORY(example_filter) +ADD_SUBDIRECTORY(example_object_detection) ADD_SUBDIRECTORY(example_decoder_image_labelling) ADD_SUBDIRECTORY(example_filter_performance_profile) diff --git a/nnstreamer_example/example_object_detection/CMakeLists.txt b/nnstreamer_example/example_object_detection/CMakeLists.txt new file mode 100644 index 0000000..fd542bb --- /dev/null +++ b/nnstreamer_example/example_object_detection/CMakeLists.txt @@ -0,0 +1,42 @@ +# demo app +pkg_check_modules(cairo_pkg REQUIRED cairo) + +if(NOT EXISTS "${CMAKE_BINARY_DIR}/tflite_model_ssd/ssd_mobilenet_v2_coco.tflite") + file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/tflite_model_ssd) + + message("-- [START TO DOWNLOAD REQUIRED RESOURCES: Model]") + file(DOWNLOAD + "https://github.com/nnsuite/testcases/raw/master/DeepLearningModels/tensorflow-lite/ssd_mobilenet_v2_coco/ssd_mobilenet_v2_coco.tflite" + "${CMAKE_BINARY_DIR}/tflite_model_ssd/ssd_mobilenet_v2_coco.tflite" + SHOW_PROGRESS + ) +endif() + +if(NOT EXISTS "${CMAKE_BINARY_DIR}/tflite_model_ssd/coco_labels_list.txt") + message("-- [START TO DOWNLOAD REQUIRED RESOURCES: Label]") + file(DOWNLOAD + "https://github.com/nnsuite/testcases/raw/master/DeepLearningModels/tensorflow-lite/ssd_mobilenet_v2_coco/coco_labels_list.txt" + "${CMAKE_BINARY_DIR}/tflite_model_ssd/coco_labels_list.txt" + SHOW_PROGRESS + ) +endif() + + +if(NOT EXISTS "${CMAKE_BINARY_DIR}/tflite_model_ssd/box_priors.txt") + message("-- [START TO DOWNLOAD REQUIRED RESOURCES: Box_prior]") + file(DOWNLOAD + "https://github.com/nnsuite/testcases/raw/master/DeepLearningModels/tensorflow-lite/ssd_mobilenet_v2_coco/box_priors.txt" + "${CMAKE_BINARY_DIR}/tflite_model_ssd/box_priors.txt" + SHOW_PROGRESS + ) +endif() + +ADD_EXECUTABLE(nnstreamer_example_object_detection nnstreamer_example_object_detection.cc) + +TARGET_LINK_LIBRARIES(nnstreamer_example_object_detection ${apppkgs_LIBRARIES} ${cairo_pkg_LIBRARIES}) +TARGET_INCLUDE_DIRECTORIES(nnstreamer_example_object_detection PUBLIC ${apppkgs_INCLUDE_DIRS} ${cairo_pkg_INCLUDE_DIRS}) +TARGET_COMPILE_OPTIONS(nnstreamer_example_object_detection PUBLIC ${apppkgs_CFLAGS_OTHER}) + +IF (INSTALL_EXAMPLE_APP) + INSTALL(TARGETS nnstreamer_example_object_detection RUNTIME DESTINATION ${EXEC_PREFIX}) +ENDIF (INSTALL_EXAMPLE_APP) diff --git a/nnstreamer_example/example_object_detection/nnstreamer_example_object_detection.cc b/nnstreamer_example/example_object_detection/nnstreamer_example_object_detection.cc new file mode 100644 index 0000000..6e4ce43 --- /dev/null +++ b/nnstreamer_example/example_object_detection/nnstreamer_example_object_detection.cc @@ -0,0 +1,739 @@ +/** + * @file nnstreamer_example_object_detection.cc + * @date 22 October 2018 + * @brief Tensor stream example with TF-Lite model for object detection + * @author HyoungJoo Ahn + * @bug No known bugs. + * + * Run example : + * Before running this example, GST_PLUGIN_PATH should be updated for nnstreamer plug-in. + * $ export GST_PLUGIN_PATH=$GST_PLUGIN_PATH: + * $ ./nnstreamer_example_object_detection +* + * Required model and resources are stored at below link + * https://github.com/nnsuite/testcases/tree/master/DeepLearningModels/tensorflow-lite/ssd_mobilenet_v2_coco + */ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include + +/** + * @brief Macro for debug mode. + */ +#ifndef DBG +#define DBG FALSE +#endif + +/** + * @brief Macro for debug message. + */ +#define _print_log(...) if (DBG) g_message (__VA_ARGS__) + +/** + * @brief Macro to check error case. + */ +#define _check_cond_err(cond) \ + if (!(cond)) { \ + _print_log ("app failed! [line : %d]", __LINE__); \ + goto error; \ + } + +#define Y_SCALE 10.0f +#define X_SCALE 10.0f +#define H_SCALE 5.0f +#define W_SCALE 5.0f + +#define VIDEO_WIDTH 640 +#define VIDEO_HEIGHT 480 +#define MODEL_WIDTH 300 +#define MODEL_HEIGHT 300 + +#define BOX_SIZE 4 +#define LABEL_SIZE 91 +#define DETECTION_MAX 1917 + +/** + * @brief Max objects in display. + */ +#define MAX_OBJECT_DETECTION 5 + +typedef struct +{ + gint x; + gint y; + gint width; + gint height; + gint class_id; + gfloat prob; +} DetectedObject; + +typedef struct +{ + gboolean valid; + GstVideoInfo vinfo; +} CairoOverlayState; + +/** + * @brief Data structure for tflite model info. + */ +typedef struct +{ + gchar *model_path; /**< tflite model file path */ + gchar *label_path; /**< label file path */ + gchar *box_prior_path; /**< box prior file path */ + gfloat box_priors[BOX_SIZE][DETECTION_MAX]; /**< box prior */ + GList *labels; /**< list of loaded labels */ +} TFLiteModelInfo; + +/** + * @brief Data structure for app. + */ +typedef struct +{ + GMainLoop *loop; /**< main event loop */ + GstElement *pipeline; /**< gst pipeline for data stream */ + GstBus *bus; /**< gst bus for data pipeline */ + gboolean running; /**< true when app is running */ + GMutex mutex; /**< mutex for processing */ + TFLiteModelInfo tflite_info; /**< tflite model info */ + CairoOverlayState overlay_state; + std::vector < DetectedObject > detected_objects; +} AppData; + +/** + * @brief Data for pipeline and result. + */ +static AppData g_app; + +/** + * @brief Read strings from file. + */ +static gboolean +read_lines (const gchar * file_name, GList ** lines) +{ + std::ifstream file (file_name); + if (!file) { + _print_log ("Failed to open file %s", file_name); + return FALSE; + } + + std::string str; + while (std::getline (file, str)) { + *lines = g_list_append (*lines, g_strdup (str.c_str ())); + } + + return TRUE; +} + +/** + * @brief Load box priors. + */ +static gboolean +tflite_load_box_priors (TFLiteModelInfo * tflite_info) +{ + GList *box_priors = NULL; + gchar *box_row; + + g_return_val_if_fail (tflite_info != NULL, FALSE); + g_return_val_if_fail (read_lines (tflite_info->box_prior_path, &box_priors), + FALSE); + + for (int row = 0; row < BOX_SIZE; row++) { + int column = 0; + int i = 0, j = 0; + char buff[11]; + + memset (buff, 0, 11); + box_row = (gchar *) g_list_nth_data (box_priors, row); + + while ((box_row[i] != '\n') && (box_row[i] != '\0')) { + if (box_row[i] != ' ') { + buff[j] = box_row[i]; + j++; + } else { + if (j != 0) { + tflite_info->box_priors[row][column++] = atof (buff); + memset (buff, 0, 11); + } + j = 0; + } + i++; + } + + tflite_info->box_priors[row][column++] = atof (buff); + } + + g_list_free (box_priors); + return TRUE; +} + +/** + * @brief Load labels. + */ +static gboolean +tflite_load_labels (TFLiteModelInfo * tflite_info) +{ + g_return_val_if_fail (tflite_info != NULL, FALSE); + + return read_lines (tflite_info->label_path, &tflite_info->labels); +} + +/** + * @brief Check tflite model and load labels. + */ +static gboolean +tflite_init_info (TFLiteModelInfo * tflite_info, const gchar * path) +{ + const gchar tflite_model[] = "ssd_mobilenet_v2_coco.tflite"; + const gchar tflite_label[] = "coco_labels_list.txt"; + const gchar tflite_box_priors[] = "box_priors.txt"; + + g_return_val_if_fail (tflite_info != NULL, FALSE); + + tflite_info->model_path = g_strdup_printf ("%s/%s", path, tflite_model); + tflite_info->label_path = g_strdup_printf ("%s/%s", path, tflite_label); + tflite_info->box_prior_path = + g_strdup_printf ("%s/%s", path, tflite_box_priors); + + tflite_info->labels = NULL; + + g_return_val_if_fail (tflite_load_box_priors (tflite_info), FALSE); + g_return_val_if_fail (tflite_load_labels (tflite_info), FALSE); + + return TRUE; +} + +/** + * @brief Free data in tflite info structure. + */ +static void +tflite_free_info (TFLiteModelInfo * tflite_info) +{ + g_return_if_fail (tflite_info != NULL); + + if (tflite_info->model_path) { + g_free (tflite_info->model_path); + tflite_info->model_path = NULL; + } + + if (tflite_info->label_path) { + g_free (tflite_info->label_path); + tflite_info->label_path = NULL; + } + + if (tflite_info->box_prior_path) { + g_free (tflite_info->box_prior_path); + tflite_info->box_prior_path = NULL; + } + + if (tflite_info->labels) { + g_list_free (tflite_info->labels); + tflite_info->labels = NULL; + } +} + +/** + * @brief Free resources in app data. + */ +static void +free_app_data (void) +{ + if (g_app.loop) { + g_main_loop_unref (g_app.loop); + g_app.loop = NULL; + } + + if (g_app.bus) { + gst_bus_remove_signal_watch (g_app.bus); + gst_object_unref (g_app.bus); + g_app.bus = NULL; + } + + if (g_app.pipeline) { + gst_object_unref (g_app.pipeline); + g_app.pipeline = NULL; + } + + g_app.detected_objects.clear (); + + tflite_free_info (&g_app.tflite_info); + g_mutex_clear (&g_app.mutex); +} + +/** + * @brief Function to print error message. + */ +static void +parse_err_message (GstMessage * message) +{ + gchar *debug; + GError *error; + + g_return_if_fail (message != NULL); + + switch (GST_MESSAGE_TYPE (message)) { + case GST_MESSAGE_ERROR: + gst_message_parse_error (message, &error, &debug); + break; + + case GST_MESSAGE_WARNING: + gst_message_parse_warning (message, &error, &debug); + break; + + default: + return; + } + + gst_object_default_error (GST_MESSAGE_SRC (message), error, debug); + g_error_free (error); + g_free (debug); +} + +/** + * @brief Function to print qos message. + */ +static void +parse_qos_message (GstMessage * message) +{ + GstFormat format; + guint64 processed; + guint64 dropped; + + gst_message_parse_qos_stats (message, &format, &processed, &dropped); + _print_log ("format[%d] processed[%" G_GUINT64_FORMAT "] dropped[%" + G_GUINT64_FORMAT "]", format, processed, dropped); +} + +/** + * @brief Callback for message. + */ +static void +bus_message_cb (GstBus * bus, GstMessage * message, gpointer user_data) +{ + switch (GST_MESSAGE_TYPE (message)) { + case GST_MESSAGE_EOS: + _print_log ("received eos message"); + g_main_loop_quit (g_app.loop); + break; + + case GST_MESSAGE_ERROR: + _print_log ("received error message"); + parse_err_message (message); + g_main_loop_quit (g_app.loop); + break; + + case GST_MESSAGE_WARNING: + _print_log ("received warning message"); + parse_err_message (message); + break; + + case GST_MESSAGE_STREAM_START: + _print_log ("received start message"); + break; + + case GST_MESSAGE_QOS: + parse_qos_message (message); + break; + + default: + break; + } +} + +/** + * @brief Compare score of detected objects. + */ +static bool +compare_objs (DetectedObject & a, DetectedObject & b) +{ + return a.prob > b.prob; +} + +/** + * @brief Intersection of union + */ +static gfloat +iou (DetectedObject & A, DetectedObject & B) +{ + int x1 = std::max (A.x, B.x); + int y1 = std::max (A.y, B.y); + int x2 = std::min (A.x + A.width, B.x + B.width); + int y2 = std::min (A.y + A.height, B.y + B.height); + int w = std::max (0, (x2 - x1 + 1)); + int h = std::max (0, (y2 - y1 + 1)); + float inter = w * h; + float areaA = A.width * A.height; + float areaB = B.width * B.height; + float o = inter / (areaA + areaB - inter); + return (o >= 0) ? o : 0; +} + +/** + * @brief NMS (non-maximum suppression) + */ +static void +nms (std::vector < DetectedObject > &detected) +{ + const float threshold_iou = .5f; + guint boxes_size; + guint i, j; + + std::sort (detected.begin (), detected.end (), compare_objs); + boxes_size = detected.size (); + + std::vector < bool > del (boxes_size, false); + for (i = 0; i < boxes_size; i++) { + if (!del[i]) { + for (j = i + 1; j < boxes_size; j++) { + if (iou (detected.at (i), detected.at (j)) > threshold_iou) { + del[j] = true; + } + } + } + } + + /* update result */ + g_mutex_lock (&g_app.mutex); + + g_app.detected_objects.clear (); + for (i = 0; i < boxes_size; i++) { + if (!del[i]) { + g_app.detected_objects.push_back (detected[i]); + + if (DBG) { + _print_log ("=============================="); + _print_log ("Label : %s", + (gchar *) g_list_nth_data (g_app.tflite_info.labels, + detected[i].class_id)); + _print_log ("x : %d", detected[i].x); + _print_log ("y : %d", detected[i].y); + _print_log ("width : %d", detected[i].width); + _print_log ("height : %d", detected[i].height); + _print_log ("Confidence Score: %f", detected[i].prob); + } + } + } + + g_mutex_unlock (&g_app.mutex); +} + +#define _expit(x) \ + (1.f / (1.f + expf (-x))) + +/** + * @brief Get detected objects. + */ +static void +get_detected_objects (gfloat * detections, gfloat * boxes) +{ + const float threshold_score = .5f; + std::vector < DetectedObject > detected; + + for (int d = 0; d < DETECTION_MAX; d++) { + float ycenter = + boxes[0] / Y_SCALE * g_app.tflite_info.box_priors[2][d] + + g_app.tflite_info.box_priors[0][d]; + float xcenter = + boxes[1] / X_SCALE * g_app.tflite_info.box_priors[3][d] + + g_app.tflite_info.box_priors[1][d]; + float h = + (float) expf (boxes[2] / H_SCALE) * g_app.tflite_info.box_priors[2][d]; + float w = + (float) expf (boxes[3] / W_SCALE) * g_app.tflite_info.box_priors[3][d]; + + float ymin = ycenter - h / 2.f; + float xmin = xcenter - w / 2.f; + float ymax = ycenter + h / 2.f; + float xmax = xcenter + w / 2.f; + + int x = xmin * MODEL_WIDTH; + int y = ymin * MODEL_HEIGHT; + int width = (xmax - xmin) * MODEL_WIDTH; + int height = (ymax - ymin) * MODEL_HEIGHT; + + for (int c = 1; c < LABEL_SIZE; c++) { + gfloat score = _expit (detections[c]); + /** + * This score cutoff is taken from Tensorflow's demo app. + * There are quite a lot of nodes to be run to convert it to the useful possibility + * scores. As a result of that, this cutoff will cause it to lose good detections in + * some scenarios and generate too much noise in other scenario. + */ + if (score < threshold_score) + continue; + + DetectedObject object; + + object.class_id = c; + object.x = x; + object.y = y; + object.width = width; + object.height = height; + object.prob = score; + + detected.push_back (object); + } + + detections += LABEL_SIZE; + boxes += BOX_SIZE; + } + + nms (detected); +} + +/** + * @brief Callback for tensor sink signal. + */ +static void +new_data_cb (GstElement * element, GstBuffer * buffer, gpointer user_data) +{ + GstMemory *mem_boxes, *mem_detections; + GstMapInfo info_boxes, info_detections; + gfloat *boxes, *detections; + + g_return_if_fail (g_app.running); + + /** + * tensor type is float32. + * [0] dim of boxes > BOX_SIZE : 1 : DETECTION_MAX : 1 + * [1] dim of labels > LABEL_SIZE : DETECTION_MAX : 1 : 1 + */ + g_assert (gst_buffer_n_memory (buffer) == 2); + + /* boxes */ + mem_boxes = gst_buffer_get_memory (buffer, 0); + g_assert (gst_memory_map (mem_boxes, &info_boxes, GST_MAP_READ)); + g_assert (info_boxes.size == BOX_SIZE * DETECTION_MAX * 4); + boxes = (gfloat *) info_boxes.data; + + /* detections */ + mem_detections = gst_buffer_get_memory (buffer, 1); + g_assert (gst_memory_map (mem_detections, &info_detections, GST_MAP_READ)); + g_assert (info_detections.size == LABEL_SIZE * DETECTION_MAX * 4); + detections = (gfloat *) info_detections.data; + + get_detected_objects (detections, boxes); + + gst_memory_unmap (mem_boxes, &info_boxes); + gst_memory_unmap (mem_detections, &info_detections); + + gst_memory_unref (mem_boxes); + gst_memory_unref (mem_detections); +} + +/** + * @brief Set window title. + * @param name GstXImageSink element name + * @param title window title + */ +static void +set_window_title (const gchar * name, const gchar * title) +{ + GstTagList *tags; + GstPad *sink_pad; + GstElement *element; + + element = gst_bin_get_by_name (GST_BIN (g_app.pipeline), name); + + g_return_if_fail (element != NULL); + + sink_pad = gst_element_get_static_pad (element, "sink"); + + if (sink_pad) { + tags = gst_tag_list_new (GST_TAG_TITLE, title, NULL); + gst_pad_send_event (sink_pad, gst_event_new_tag (tags)); + gst_object_unref (sink_pad); + } + + gst_object_unref (element); +} + +/** + * @brief Store the information from the caps that we are interested in. + */ +static void +prepare_overlay_cb (GstElement * overlay, GstCaps * caps, gpointer user_data) +{ + CairoOverlayState *state = &g_app.overlay_state; + + state->valid = gst_video_info_from_caps (&state->vinfo, caps); +} + +/** + * @brief Callback to draw the overlay. + */ +static void +draw_overlay_cb (GstElement * overlay, cairo_t * cr, guint64 timestamp, + guint64 duration, gpointer user_data) +{ + CairoOverlayState *state = &g_app.overlay_state; + gfloat x, y, width, height; + gchar *label; + guint drawed = 0; + + g_return_if_fail (state->valid); + g_return_if_fail (g_app.running); + + std::vector < DetectedObject > detected; + std::vector < DetectedObject >::iterator iter; + + g_mutex_lock (&g_app.mutex); + detected = g_app.detected_objects; + g_mutex_unlock (&g_app.mutex); + + /* set font props */ + cairo_select_font_face (cr, "Sans", CAIRO_FONT_SLANT_NORMAL, + CAIRO_FONT_WEIGHT_BOLD); + cairo_set_font_size (cr, 20.0); + + for (iter = detected.begin (); iter != detected.end (); ++iter) { + label = + (gchar *) g_list_nth_data (g_app.tflite_info.labels, iter->class_id); + + x = iter->x * VIDEO_WIDTH / MODEL_WIDTH; + y = iter->y * VIDEO_HEIGHT / MODEL_HEIGHT; + width = iter->width * VIDEO_WIDTH / MODEL_WIDTH; + height = iter->height * VIDEO_WIDTH / MODEL_HEIGHT; + + /* draw rectangle */ + cairo_rectangle (cr, x, y, width, height); + cairo_set_source_rgb (cr, 1, 0, 0); + cairo_set_line_width (cr, 1.5); + cairo_stroke (cr); + cairo_fill_preserve (cr); + + /* draw title */ + cairo_move_to (cr, x + 5, y + 25); + cairo_text_path (cr, label); + cairo_set_source_rgb (cr, 1, 0, 0); + cairo_fill_preserve (cr); + cairo_set_source_rgb (cr, 1, 1, 1); + cairo_set_line_width (cr, .3); + cairo_stroke (cr); + cairo_fill_preserve (cr); + + if (++drawed >= MAX_OBJECT_DETECTION) { + /* max objects drawed */ + break; + } + } +} + +/** + * @brief Main function. + */ +int +main (int argc, char ** argv) +{ + const gchar tflite_model_path[] = "./tflite_model_ssd"; + + gchar *str_pipeline; + GstElement *element; + + _print_log ("start app.."); + + /* init app variable */ + g_app.running = FALSE; + g_app.loop = NULL; + g_app.bus = NULL; + g_app.pipeline = NULL; + g_app.detected_objects.clear (); + g_mutex_init (&g_app.mutex); + + _check_cond_err (tflite_init_info (&g_app.tflite_info, tflite_model_path)); + + /* init gstreamer */ + gst_init (&argc, &argv); + + /* main loop */ + g_app.loop = g_main_loop_new (NULL, FALSE); + _check_cond_err (g_app.loop != NULL); + + /* init pipeline */ + str_pipeline = + g_strdup_printf + ("v4l2src name=src ! videoscale ! " + "video/x-raw,width=%d,height=%d,format=RGB ! tee name=t_raw " + "t_raw. ! queue ! videoconvert ! cairooverlay name=tensor_res ! ximagesink name=img_tensor " + "t_raw. ! queue ! videoscale ! video/x-raw,width=%d,height=%d ! tensor_converter ! " + "tensor_transform mode=typecast option=float32 ! " + "tensor_transform mode=arithmetic option=add:-127 ! " + "tensor_transform mode=arithmetic option=mul:0.007843 ! " + "tensor_filter framework=tensorflow-lite model=%s ! " + "tensor_sink name=tensor_sink", + VIDEO_WIDTH, VIDEO_HEIGHT, MODEL_WIDTH, MODEL_HEIGHT, + g_app.tflite_info.model_path); + g_app.pipeline = gst_parse_launch (str_pipeline, NULL); + g_free (str_pipeline); + _check_cond_err (g_app.pipeline != NULL); + + /* bus and message callback */ + g_app.bus = gst_element_get_bus (g_app.pipeline); + _check_cond_err (g_app.bus != NULL); + + gst_bus_add_signal_watch (g_app.bus); + g_signal_connect (g_app.bus, "message", G_CALLBACK (bus_message_cb), NULL); + + /* tensor sink signal : new data callback */ + element = gst_bin_get_by_name (GST_BIN (g_app.pipeline), "tensor_sink"); + g_signal_connect (element, "new-data", G_CALLBACK (new_data_cb), NULL); + gst_object_unref (element); + + /* cairo overlay */ + element = gst_bin_get_by_name (GST_BIN (g_app.pipeline), "tensor_res"); + g_signal_connect (element, "draw", G_CALLBACK (draw_overlay_cb), NULL); + g_signal_connect (element, "caps-changed", G_CALLBACK (prepare_overlay_cb), + NULL); + gst_object_unref (element); + + /* start pipeline */ + gst_element_set_state (g_app.pipeline, GST_STATE_PLAYING); + g_app.running = TRUE; + + /* set window title */ + set_window_title ("img_tensor", "NNStreamer Example"); + + /* run main loop */ + g_main_loop_run (g_app.loop); + + /* quit when received eos or error message */ + g_app.running = FALSE; + + /* cam source element */ + element = gst_bin_get_by_name (GST_BIN (g_app.pipeline), "src"); + + gst_element_set_state (element, GST_STATE_READY); + gst_element_set_state (g_app.pipeline, GST_STATE_READY); + + g_usleep (200 * 1000); + + gst_element_set_state (element, GST_STATE_NULL); + gst_element_set_state (g_app.pipeline, GST_STATE_NULL); + + g_usleep (200 * 1000); + gst_object_unref (element); + +error: + _print_log ("close app.."); + + free_app_data (); + return 0; +} diff --git a/packaging/nnstreamer.spec b/packaging/nnstreamer.spec index 9a38f63..a123ba2 100644 --- a/packaging/nnstreamer.spec +++ b/packaging/nnstreamer.spec @@ -35,6 +35,9 @@ BuildRequires: python-numpy BuildRequires: pkgconfig(libpng) # for tensorflow-lite BuildRequires: tensorflow-lite-devel +# for cairo (nnstreamer_example_object_detection) +BuildRequires: coregl-devel +BuildRequires: cairo-devel %if 0%{?testcoverage} BuildRequires: lcov -- 2.7.4