--- /dev/null
+/**\r
+ * GStreamer / NNStreamer tensor_decoder subplugin, "bounding boxes"\r
+ * Copyright (C) 2018 Samsung Electronics Co. Ltd.\r
+ * Copyright (C) 2018 MyungJoo Ham <myungjoo.ham@samsung.com>\r
+ * Copyright 2021 NXP\r
+ *\r
+ * This library is free software; you can redistribute it and/or\r
+ * modify it under the terms of the GNU Library General Public\r
+ * License as published by the Free Software Foundation;\r
+ * version 2.1 of the License.\r
+ *\r
+ * This library is distributed in the hope that it will be useful,\r
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of\r
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU\r
+ * Library General Public License for more details.\r
+ *\r
+ */\r
+/**\r
+ * @file tensordec-tensor_region.c\r
+ * @date 15th June, 2023\r
+ * @brief NNStreamer tensor-decoder subplugin, "tensor region",\r
+ * which converts tensors to cropping info for tensor _crop element\r
+ * This code is NYI/WIP and not compilable.\r
+ *\r
+ * @see https://github.com/nnstreamer/nnstreamer\r
+ * @author Harsh Jain <hjain24in@gmail.com>\r
+ * @bug No known bugs except for NYI items\r
+ *\r
+ * option1: number of cropping regions required (default is 1)\r
+ * option2: Location of label file\r
+ * This is independent from option1\r
+ * option3: \r
+ * for mobilenet-ssd mode:\r
+ * The option3 definition scheme is, in order, the following:\r
+ * - box priors location file (mandatory)\r
+ * - Detection threshold (optional, default set to 0.5)\r
+ * - Y box scale (optional, default set to 10.0)\r
+ * - X box scale (optional, default set to 10.0)\r
+ * - h box scale (optional, default set to 5.0)\r
+ * - w box scale (optional, default set to 5.0)\r
+ * - IOU box valid threshold (optional, default set to 0.5)\r
+ * The default parameters value could be set in the following ways:\r
+ * option3=box-priors.txt:0.5:10.0:10.0:5.0:5.0:0.5\r
+ * option3=box-priors.txt\r
+ * option3=box-priors.txt::::::\r
+ *\r
+ * It's possible to set only few values, using the default values for\r
+ * those not specified through the command line.\r
+ * You could specify respectively the detection and IOU thresholds to 0.65\r
+ * and 0.6 with the option3 parameter as follow:\r
+ * option3=box-priors.txt:0.65:::::0.6\r
+ * option4: Video input Dimension (WIDTH:HEIGHT) (default 300:300)\r
+ * This is independent from option1\r
+ *\r
+ * @todo Remove duplicate codes*\r
+ * @todo give support for other models*\r
+ */\r
+#ifndef _GNU_SOURCE\r
+#define _GNU_SOURCE\r
+#endif\r
+#include <glib.h>\r
+#include <gst/gst.h>\r
+#include <math.h> /** expf */\r
+#include <nnstreamer_log.h>\r
+#include <nnstreamer_plugin_api.h>\r
+#include <nnstreamer_plugin_api_decoder.h>\r
+#include <nnstreamer_util.h>\r
+#include <stdint.h>\r
+#include <stdio.h>\r
+#include <stdlib.h>\r
+#include <string.h>\r
+#include "tensordecutil.h"\r
+\r
+#define _tensor_region_size_default_ 1\r
+void init_tr (void) __attribute__ ((constructor));\r
+void fini_tr (void) __attribute__ ((destructor));\r
+\r
+#define BOX_SIZE (4)\r
+#define MOBILENET_SSD_DETECTION_MAX (2034) /**add ssd_mobilenet v3 */\r
+#define MOBILENET_SSD_MAX_TENSORS (2U)\r
+#define INPUT_VIDEO_WIDTH_DEFAULT (300)\r
+#define INPUT_VIDEO_HEIGHT_DEFAULT (300)\r
+/**\r
+ * @brief There can be different schemes for input tensor.\r
+ */\r
+typedef enum\r
+{\r
+ MOBILENET_SSD_BOUNDING_BOX = 0,\r
+ BOUNDING_BOX_UNKNOWN,\r
+} tensor_region_modes;\r
+\r
+/** @brief Internal data structure for identifying cropping region */\r
+typedef struct {\r
+ guint x;\r
+ guint y;\r
+ guint w;\r
+ guint h;\r
+} crop_region;\r
+\r
+/**\r
+ * @brief Data structure for SSD tensor_region info for mobilenet ssd model.\r
+ */\r
+typedef struct {\r
+ /**From option3, box prior data */\r
+ char *box_prior_path; /**< Box Prior file path */\r
+ gfloat box_priors[BOX_SIZE][MOBILENET_SSD_DETECTION_MAX + 1]; /** loaded box prior */\r
+#define MOBILENET_SSD_PARAMS_THRESHOLD_IDX 0\r
+#define MOBILENET_SSD_PARAMS_Y_SCALE_IDX 1\r
+#define MOBILENET_SSD_PARAMS_X_SCALE_IDX 2\r
+#define MOBILENET_SSD_PARAMS_H_SCALE_IDX 3\r
+#define MOBILENET_SSD_PARAMS_W_SCALE_IDX 4\r
+#define MOBILENET_SSD_PARAMS_IOU_THRESHOLD_IDX 5\r
+#define MOBILENET_SSD_PARAMS_MAX 6\r
+\r
+ gfloat params[MOBILENET_SSD_PARAMS_MAX]; /** Post Processing parameters */\r
+ gfloat sigmoid_threshold; /** Inverse value of valid detection threshold in sigmoid domain */\r
+} properties_MOBILENET_SSD;\r
+\r
+/** @brief Internal data structure for tensor region */\r
+typedef struct {\r
+ tensor_region_modes mode;\r
+ union {\r
+ properties_MOBILENET_SSD mobilenet_ssd; /**< Properties for mobilenet_ssd */\r
+ };\r
+ imglabel_t labeldata;\r
+ char *label_path;\r
+\r
+ /**From option4 */\r
+ guint i_width; /**< Input Video Width */\r
+ guint i_height; /**< Input Video Height */\r
+\r
+ /**From option1 */\r
+ guint num; /**number of cropped regions required */\r
+\r
+ guint max_detections;\r
+ GArray *regions;\r
+ gboolean flag_use_label;\r
+\r
+} tensor_region;\r
+\r
+\r
+/** @brief Mathematic inverse of sigmoid function, aka logit */\r
+static float\r
+logit (float x)\r
+{\r
+ if (x <= 0.0f)\r
+ return -INFINITY;\r
+\r
+ if (x >= 1.0f)\r
+ return INFINITY;\r
+\r
+ return log (x / (1.0 - x));\r
+}\r
+\r
+/**\r
+ * @brief Load box-prior data from a file\r
+ * @param[in/out] trData The internal data.\r
+ * @return TRUE if loaded and configured. FALSE if failed to do so.\r
+ */\r
+static int\r
+_mobilenet_ssd_loadBoxPrior (tensor_region *trData)\r
+{\r
+ properties_MOBILENET_SSD *mobilenet_ssd = &trData->mobilenet_ssd;\r
+ gboolean failed = FALSE;\r
+ GError *err = NULL;\r
+ gchar **priors;\r
+ gchar *line = NULL;\r
+ gchar *contents = NULL;\r
+ guint row;\r
+ gint prev_reg = -1;\r
+\r
+ /**Read file contents */\r
+ if (!g_file_get_contents (mobilenet_ssd->box_prior_path, &contents, NULL, &err)) {\r
+ GST_ERROR ("Decoder/Tensor-Region/SSD's box prior file %s cannot be read: %s",\r
+ mobilenet_ssd->box_prior_path, err->message);\r
+ g_clear_error (&err);\r
+ return FALSE;\r
+ }\r
+\r
+ priors = g_strsplit (contents, "\n", -1);\r
+ /**If given prior file is inappropriate, report back to tensor-decoder */\r
+ if (g_strv_length (priors) < BOX_SIZE) {\r
+ ml_loge ("The given prior file, %s, should have at least %d lines.\n",\r
+ mobilenet_ssd->box_prior_path, BOX_SIZE);\r
+ failed = TRUE;\r
+ goto error;\r
+ }\r
+\r
+ for (row = 0; row < BOX_SIZE; row++) {\r
+ gint column = 0, registered = 0;\r
+\r
+ line = priors[row];\r
+ if (line) {\r
+ gchar **list = g_strsplit_set (line, " \t,", -1);\r
+ gchar *word;\r
+\r
+ while ((word = list[column]) != NULL) {\r
+ column++;\r
+\r
+ if (word && *word) {\r
+ if (registered > MOBILENET_SSD_DETECTION_MAX) {\r
+ GST_WARNING ("Decoder/Tensor-region/SSD's box prior data file has too many priors. %d >= %d",\r
+ registered, MOBILENET_SSD_DETECTION_MAX);\r
+ break;\r
+ }\r
+ mobilenet_ssd->box_priors[row][registered]\r
+ = (gfloat) g_ascii_strtod (word, NULL);\r
+ registered++;\r
+ }\r
+ }\r
+\r
+ g_strfreev (list);\r
+ }\r
+\r
+ if (prev_reg != -1 && prev_reg != registered) {\r
+ GST_ERROR ("Decoder/Tensor-Region/SSD's box prior data file is not consistent.");\r
+ failed = TRUE;\r
+ break;\r
+ }\r
+ prev_reg = registered;\r
+ }\r
+\r
+error:\r
+ g_strfreev (priors);\r
+ g_free (contents);\r
+ return !failed;\r
+}\r
+\r
+\r
+/** @brief Initialize tensor_region per mode */\r
+static int\r
+_init_modes (tensor_region * trData){\r
+ if(trData->mode == MOBILENET_SSD_BOUNDING_BOX){\r
+ properties_MOBILENET_SSD *data = &trData->mobilenet_ssd;\r
+#define DETECTION_THRESHOLD (.5f)\r
+#define THRESHOLD_IOU (.5f)\r
+#define Y_SCALE (10.0f)\r
+#define X_SCALE (10.0f)\r
+#define H_SCALE (5.0f)\r
+#define W_SCALE (5.0f)\r
+\r
+ data->params[MOBILENET_SSD_PARAMS_THRESHOLD_IDX] = DETECTION_THRESHOLD;\r
+ data->params[MOBILENET_SSD_PARAMS_Y_SCALE_IDX] = Y_SCALE;\r
+ data->params[MOBILENET_SSD_PARAMS_X_SCALE_IDX] = X_SCALE;\r
+ data->params[MOBILENET_SSD_PARAMS_H_SCALE_IDX] = H_SCALE;\r
+ data->params[MOBILENET_SSD_PARAMS_W_SCALE_IDX] = W_SCALE;\r
+ data->params[MOBILENET_SSD_PARAMS_IOU_THRESHOLD_IDX] = THRESHOLD_IOU;\r
+ data->sigmoid_threshold = logit (DETECTION_THRESHOLD);\r
+ return TRUE;\r
+ }\r
+ return TRUE;\r
+}\r
+\r
+\r
+/** @brief tensordec-plugin's GstTensorDecoderDef callback */\r
+static int\r
+tr_init (void **pdata)\r
+{\r
+ tensor_region *trData;\r
+ trData = *pdata = g_new0 (tensor_region, 1);\r
+ if (*pdata == NULL) {\r
+ GST_ERROR ("Failed to allocate memory for decoder subplugin.");\r
+ return FALSE;\r
+ }\r
+ trData->mode = MOBILENET_SSD_BOUNDING_BOX;\r
+ trData->num = 1;\r
+ trData->max_detections = 0;\r
+ trData->regions = NULL;\r
+ trData->flag_use_label = FALSE;\r
+ trData->i_width = INPUT_VIDEO_WIDTH_DEFAULT;\r
+ trData->i_height = INPUT_VIDEO_HEIGHT_DEFAULT;\r
+ return _init_modes(trData);\r
+}\r
+\r
+/** @brief tensordec-plugin's GstTensorDecoderDef callback */\r
+static void\r
+tr_exit (void **pdata)\r
+{\r
+ tensor_region *trData = *pdata;\r
+ g_array_free (trData->regions, TRUE);\r
+ _free_labels (&trData->labeldata);\r
+\r
+ if (trData->label_path)\r
+ g_free (trData->label_path);\r
+ g_free (*pdata);\r
+ *pdata = NULL;\r
+}\r
+\r
+/** @brief configure per-mode option (option1) */\r
+static int\r
+_setOption_mode (tensor_region *trData, const char *param)\r
+{\r
+ if(trData->mode == MOBILENET_SSD_BOUNDING_BOX){\r
+ properties_MOBILENET_SSD *mobilenet_ssd = &trData->mobilenet_ssd;\r
+ gchar **options;\r
+ int noptions, idx;\r
+ int ret = 1;\r
+\r
+ options = g_strsplit (param, ":", -1);\r
+ noptions = g_strv_length (options);\r
+\r
+ if (mobilenet_ssd->box_prior_path)\r
+ g_free (mobilenet_ssd->box_prior_path);\r
+\r
+ mobilenet_ssd->box_prior_path = g_strdup (options[0]);\r
+\r
+ if (NULL != mobilenet_ssd->box_prior_path) {\r
+ ret = _mobilenet_ssd_loadBoxPrior (trData);\r
+ if (ret == 0)\r
+ goto exit_mobilenet_ssd;\r
+ }\r
+\r
+ for (idx = 1; idx < noptions; idx++) {\r
+ if (strlen (options[idx]) == 0)\r
+ continue;\r
+ mobilenet_ssd->params[idx - 1] = strtod (options[idx], NULL);\r
+ }\r
+\r
+ mobilenet_ssd->sigmoid_threshold\r
+ = logit (mobilenet_ssd->params[MOBILENET_SSD_PARAMS_THRESHOLD_IDX]);\r
+ exit_mobilenet_ssd:\r
+ g_strfreev (options);\r
+ return ret;\r
+ }\r
+ return TRUE;\r
+}\r
+\r
+/** @brief tensordec-plugin's GstTensorDecoderDef callback */\r
+static int\r
+tr_setOption (void **pdata, int opNum, const char *param)\r
+{\r
+ tensor_region *trData = *pdata;\r
+ if (opNum == 0) {\r
+ /**option1 number of crop regions required */\r
+ trData->num = atoi (param);\r
+ return TRUE;\r
+ } else if (opNum == 1) {\r
+ /**option2 label path for mobilenet_ssd model */\r
+ if (NULL != trData->label_path)\r
+ g_free (trData->label_path);\r
+ trData->label_path = g_strdup (param);\r
+\r
+ if (NULL != trData->label_path)\r
+ loadImageLabels (trData->label_path, &trData->labeldata);\r
+\r
+ if (trData->labeldata.total_labels > 0)\r
+ return TRUE;\r
+ else\r
+ return FALSE;\r
+ } else if (opNum == 2){\r
+ /**option3 setting box prior path for mobilenet ssd model */\r
+ return _setOption_mode (trData, param);\r
+ }\r
+ else if (opNum == 3) {\r
+ /**option4 = input model size (width:height) */\r
+ tensor_dim dim;\r
+ int rank = gst_tensor_parse_dimension (param, dim);\r
+\r
+ trData->i_width = 0;\r
+ trData->i_height = 0;\r
+ if (param == NULL || *param == '\0')\r
+ return TRUE;\r
+\r
+ if (rank < 2) {\r
+ GST_ERROR ("mode-option-4 of tensor region is input video dimension (WIDTH:HEIGHT). The given parameter, \"%s\", is not acceptable.",\r
+ param);\r
+ return TRUE; /**Ignore this param */\r
+ }\r
+ if (rank > 2) {\r
+ GST_WARNING ("mode-option-4 of tensor region is input video dimension (WIDTH:HEIGHT). The third and later elements of the given parameter, \"%s\", are ignored.",\r
+ param);\r
+ }\r
+ trData->i_width = dim[0];\r
+ trData->i_height = dim[1];\r
+ return TRUE;\r
+ }\r
+\r
+ GST_INFO ("Property mode-option-%d is ignored", opNum + 1);\r
+ return TRUE;\r
+}\r
+\r
+typedef struct {\r
+ int valid;\r
+ int class_id;\r
+ gfloat score;\r
+ int x;\r
+ int y;\r
+ int height;\r
+ int width;\r
+} detected_object;\r
+\r
+/**\r
+ * @brief transfer crop region info with the given results to the output buffer\r
+ * @param[out] out_info The output buffer \r
+ * @param[in] trData The Tensor_region internal data.\r
+ * @param[in] results The final results to be transfered.\r
+ */\r
+static void\r
+finalize (GstMapInfo *out_info, tensor_region *data, GArray *results)\r
+{\r
+\r
+ guint i;\r
+ guint size = sizeof (crop_region); /**Assuming crop_region is a structure with four integer fields */\r
+ guint8 *out_data = out_info->data;\r
+ crop_region region;\r
+ guint maxx = MIN (results->len, data->num);\r
+\r
+ for (i = 0; i < maxx; i++) {\r
+ detected_object *temp = &g_array_index (results, detected_object, i);\r
+ region.x = temp->x;\r
+ region.y = temp->y;\r
+ region.w = temp->width;\r
+ region.h = temp->height;\r
+ memcpy (out_data, ®ion, size);\r
+ out_data += size;\r
+ }\r
+}\r
+\r
+#define _expit(x) (1.f / (1.f + expf (-((float) x))))\r
+\r
+\r
+/**\r
+ * @brief C++-Template-like box location calculation for box-priors\r
+ * @bug This is not macro-argument safe. Use paranthesis!\r
+ * @param[in] bb The configuration, "tensor region"\r
+ * @param[in] index The index (3rd dimension of BOX_SIZE:1:MOBILENET_SSD_DETECTION_MAX:1)\r
+ * @param[in] total_labels The count of total labels. We can get this from input tensor info. (1st dimension of LABEL_SIZE:MOBILENET_SSD_DETECTION_MAX:1:1)\r
+ * @param[in] boxprior The box prior data from the box file of SSD.\r
+ * @param[in] boxinputptr Cursor pointer of input + byte-per-index * index (box)\r
+ * @param[in] detinputptr Cursor pointer of input + byte-per-index * index (detection)\r
+ * @param[in] result The object returned. (pointer to object)\r
+ */\r
+#define _get_object_i_mobilenet_ssd( \\r
+ bb, index, total_labels, boxprior, boxinputptr, detinputptr, result) \\r
+ do { \\r
+ unsigned int c; \\r
+ properties_MOBILENET_SSD *data = &bb->mobilenet_ssd; \\r
+ float sigmoid_threshold = data->sigmoid_threshold; \\r
+ float y_scale = data->params[MOBILENET_SSD_PARAMS_Y_SCALE_IDX]; \\r
+ float x_scale = data->params[MOBILENET_SSD_PARAMS_X_SCALE_IDX]; \\r
+ float h_scale = data->params[MOBILENET_SSD_PARAMS_H_SCALE_IDX]; \\r
+ float w_scale = data->params[MOBILENET_SSD_PARAMS_W_SCALE_IDX]; \\r
+ result->valid = FALSE; \\r
+ for (c = 1; c < total_labels; c++) { \\r
+ if (detinputptr[c] >= sigmoid_threshold) { \\r
+ gfloat score = _expit (detinputptr[c]); \\r
+ float ycenter \\r
+ = boxinputptr[0] / y_scale * boxprior[2][index] + boxprior[0][index]; \\r
+ float xcenter \\r
+ = boxinputptr[1] / x_scale * boxprior[3][index] + boxprior[1][index]; \\r
+ float h = (float) expf (boxinputptr[2] / h_scale) * boxprior[2][index]; \\r
+ float w = (float) expf (boxinputptr[3] / w_scale) * boxprior[3][index]; \\r
+ float ymin = ycenter - h / 2.f; \\r
+ float xmin = xcenter - w / 2.f; \\r
+ int x = xmin * bb->i_width; \\r
+ int y = ymin * bb->i_height; \\r
+ int width = w * bb->i_width; \\r
+ int height = h * bb->i_height; \\r
+ result->class_id = c; \\r
+ result->x = MAX (0, x); \\r
+ result->y = MAX (0, y); \\r
+ result->width = width; \\r
+ result->height = height; \\r
+ result->score = score; \\r
+ result->valid = TRUE; \\r
+ break; \\r
+ } \\r
+ } \\r
+ } while (0);\r
+\r
+/**\r
+ * @brief C++-Template-like box location calculation for box-priors for Mobilenet SSD Model\r
+ * @param[in] bb The configuration, "tensor region"\r
+ * @param[in] type The tensor type of inputptr\r
+ * @param[in] typename nnstreamer enum corresponding to the type\r
+ * @param[in] boxprior The box prior data from the box file of MOBILENET_SSD.\r
+ * @param[in] boxinput Input Tensor Data (Boxes)\r
+ * @param[in] detinput Input Tensor Data (Detection). Null if not available. (numtensor ==1)\r
+ * @param[in] config Tensor configs of the input tensors\r
+ * @param[out] results The object returned. (GArray with detectedObject)\r
+ */\r
+#define _get_objects_mobilenet_ssd( \\r
+ bb, _type, typename, boxprior, boxinput, detinput, config, results) \\r
+ case typename: \\r
+ { \\r
+ int d; \\r
+ _type *boxinput_ = (_type *) boxinput; \\r
+ size_t boxbpi = config->info.info[0].dimension[0]; \\r
+ _type *detinput_ = (_type *) detinput; \\r
+ size_t detbpi = config->info.info[1].dimension[0]; \\r
+ int num = (MOBILENET_SSD_DETECTION_MAX > bb->max_detections) ? \\r
+ bb->max_detections : \\r
+ MOBILENET_SSD_DETECTION_MAX; \\r
+ detected_object object = { \\r
+ .valid = FALSE, .class_id = 0, .x = 0, .y = 0, .width = 0, .height = 0, .score = .0 \\r
+ }; \\r
+ for (d = 0; d < num; d++) { \\r
+ _get_object_i_mobilenet_ssd (bb, d, detbpi, boxprior, \\r
+ (boxinput_ + (d * boxbpi)), (detinput_ + (d * detbpi)), (&object)); \\r
+ if (object.valid == TRUE) { \\r
+ g_array_append_val (results, object); \\r
+ } \\r
+ } \\r
+ } \\r
+ break\r
+\r
+/** @brief Macro to simplify calling _get_objects_mobilenet_ssd */\r
+#define _get_objects_mobilenet_ssd_(type, typename) \\r
+ _get_objects_mobilenet_ssd (trData, type, typename, (trData->mobilenet_ssd.box_priors), \\r
+ (boxes->data), (detections->data), config, results)\r
+\r
+/**\r
+ * @brief Compare Function for g_array_sort with detectedObject.\r
+ */\r
+static gint\r
+compare_detection (gconstpointer _a, gconstpointer _b)\r
+{\r
+ const detected_object *a = _a;\r
+ const detected_object *b = _b;\r
+\r
+ /**Larger comes first */\r
+ return (a->score > b->score) ? -1 : ((a->score == b->score) ? 0 : 1);\r
+}\r
+\r
+/**\r
+ * @brief Calculate the intersected surface\r
+ */\r
+static gfloat\r
+iou (detected_object *a, detected_object *b)\r
+{\r
+ int x1 = MAX (a->x, b->x);\r
+ int y1 = MAX (a->y, b->y);\r
+ int x2 = MIN (a->x + a->width, b->x + b->width);\r
+ int y2 = MIN (a->y + a->height, b->y + b->height);\r
+ int w = MAX (0, (x2 - x1 + 1));\r
+ int h = MAX (0, (y2 - y1 + 1));\r
+ float inter = w * h;\r
+ float areaA = a->width * a->height;\r
+ float areaB = b->width * b->height;\r
+ float o = inter / (areaA + areaB - inter);\r
+ return (o >= 0) ? o : 0;\r
+}\r
+\r
+/**\r
+ * @brief Apply NMS to the given results (objects[])\r
+ * @param[in/out] results The results to be filtered with nms\r
+ */\r
+static void\r
+nms (GArray *results, gfloat threshold)\r
+{\r
+ guint boxes_size;\r
+ guint i, j;\r
+ \r
+ g_array_sort (results, compare_detection);\r
+ boxes_size = results->len;\r
+ \r
+ for (i = 0; i < boxes_size; i++) {\r
+ detected_object *a = &g_array_index (results, detected_object, i);\r
+ if (a->valid == TRUE) {\r
+ for (j = i + 1; j < boxes_size; j++) {\r
+ detected_object *b = &g_array_index (results, detected_object, j);\r
+ if (b->valid == TRUE) {\r
+ if (iou (a, b) > threshold) {\r
+ b->valid = FALSE;\r
+ }\r
+ }\r
+ }\r
+ }\r
+ }\r
+\r
+ i = 0;\r
+ do {\r
+ detected_object *a = &g_array_index (results, detected_object, i);\r
+ if (a->valid == FALSE)\r
+ g_array_remove_index (results, i);\r
+ else\r
+ i++;\r
+ } while (i < results->len);\r
+}\r
+\r
+/** @brief tensordec-plugin's GstTensorDecoderDef callback */\r
+static GstFlowReturn\r
+tr_decode (void **pdata, const GstTensorsConfig *config,\r
+ const GstTensorMemory *input, GstBuffer *outbuf)\r
+{\r
+ tensor_region *trData = *pdata;\r
+ GstMapInfo out_info;\r
+ GstMemory *out_mem;\r
+ GArray *results = NULL;\r
+ const guint num_tensors = config->info.num_tensors;\r
+ gboolean need_output_alloc = gst_buffer_get_size (outbuf) == 0;\r
+ const size_t size = (size_t) 4 * trData->num; /**4 field per block */\r
+\r
+ g_assert (outbuf);\r
+ /**Ensure we have outbuf properly allocated */\r
+ if (need_output_alloc) {\r
+ out_mem = gst_allocator_alloc (NULL, size, NULL);\r
+ } else {\r
+ if (gst_buffer_get_size (outbuf) < size) {\r
+ gst_buffer_set_size (outbuf, size);\r
+ }\r
+ out_mem = gst_buffer_get_all_memory (outbuf);\r
+ }\r
+ if (!gst_memory_map (out_mem, &out_info, GST_MAP_WRITE)) {\r
+ ml_loge ("Cannot map output memory / tensordec-tensor_region.\n");\r
+ goto error_free;\r
+ }\r
+\r
+ /** reset the buffer with 0 */\r
+ memset (out_info.data, 0, size);\r
+ if(trData->mode == MOBILENET_SSD_BOUNDING_BOX){\r
+ const GstTensorMemory *boxes, *detections = NULL;\r
+ properties_MOBILENET_SSD *data = &trData->mobilenet_ssd;\r
+\r
+ g_assert (num_tensors >= MOBILENET_SSD_MAX_TENSORS);\r
+ results = g_array_sized_new (FALSE, TRUE, sizeof (detected_object), 100);\r
+\r
+ boxes = &input[0];\r
+ if (num_tensors >= MOBILENET_SSD_MAX_TENSORS) /**lgtm[cpp/constant-comparison] */\r
+ detections = &input[1];\r
+ \r
+ switch (config->info.info[0].type) {\r
+ _get_objects_mobilenet_ssd_ (uint8_t, _NNS_UINT8);\r
+ _get_objects_mobilenet_ssd_ (int8_t, _NNS_INT8);\r
+ _get_objects_mobilenet_ssd_ (uint16_t, _NNS_UINT16);\r
+ _get_objects_mobilenet_ssd_ (int16_t, _NNS_INT16);\r
+ _get_objects_mobilenet_ssd_ (uint32_t, _NNS_UINT32);\r
+ _get_objects_mobilenet_ssd_ (int32_t, _NNS_INT32);\r
+ _get_objects_mobilenet_ssd_ (uint64_t, _NNS_UINT64);\r
+ _get_objects_mobilenet_ssd_ (int64_t, _NNS_INT64);\r
+ _get_objects_mobilenet_ssd_ (float, _NNS_FLOAT32);\r
+ _get_objects_mobilenet_ssd_ (double, _NNS_FLOAT64);\r
+ default:\r
+ g_assert (0);\r
+ }\r
+\r
+ nms (results, data->params[MOBILENET_SSD_PARAMS_IOU_THRESHOLD_IDX]);\r
+ } else {\r
+ GST_ERROR ("Failed to get output buffer, unknown mode %d.", trData->mode);\r
+ goto error_unmap;\r
+ }\r
+ finalize (&out_info, trData, results);\r
+\r
+ g_array_free (results, TRUE);\r
+\r
+ gst_memory_unmap (out_mem, &out_info);\r
+\r
+ if (need_output_alloc)\r
+ gst_buffer_append_memory (outbuf, out_mem);\r
+ else\r
+ gst_memory_unref (out_mem);\r
+\r
+ return GST_FLOW_OK;\r
+error_unmap:\r
+ gst_memory_unmap (out_mem, &out_info);\r
+error_free:\r
+ gst_memory_unref (out_mem);\r
+\r
+ return GST_FLOW_ERROR;\r
+}\r
+\r
+/**\r
+ * @brief set the max_detection\r
+ */\r
+static int\r
+_set_max_detection (tensor_region *data, const guint max_detection, const unsigned int limit)\r
+{\r
+ /**Check consistency with max_detection */\r
+ if (data->max_detections == 0)\r
+ data->max_detections = max_detection;\r
+ else\r
+ g_return_val_if_fail (max_detection == data->max_detections, FALSE);\r
+\r
+ if (data->max_detections > limit) {\r
+ GST_ERROR ("Incoming tensor has too large detection-max : %u", max_detection);\r
+ return FALSE;\r
+ }\r
+ return TRUE;\r
+}\r
+\r
+\r
+/**\r
+ * @brief tensordec-plugin's GstTensorDecoderDef callback\r
+ *\r
+ * [Mobilenet SSD Model]\r
+ * The first tensor is boxes. BOX_SIZE : 1 : #MaxDetection, ANY-TYPE\r
+ * The second tensor is labels. #MaxLabel : #MaxDetection, ANY-TYPE\r
+ * Both tensors are MANDATORY!\r
+ * If there are third or more tensors, such tensors will be ignored.\r
+ */\r
+\r
+static GstCaps *\r
+tr_getOutCaps (void **pdata, const GstTensorsConfig *config)\r
+{\r
+ tensor_region *data = *pdata;\r
+ GstCaps *caps;\r
+ char *str;\r
+ guint max_detection, max_label;\r
+ const uint32_t *dim1, *dim2;\r
+ int i;\r
+\r
+ /**Check if the first tensor is compatible */\r
+ dim1 = config->info.info[0].dimension;\r
+ g_return_val_if_fail (dim1[0] == BOX_SIZE, NULL);\r
+ g_return_val_if_fail (dim1[1] == 1, NULL);\r
+ max_detection = dim1[2];\r
+ g_return_val_if_fail (max_detection > 0, NULL);\r
+ for (i = 3; i < NNS_TENSOR_RANK_LIMIT; i++)\r
+ g_return_val_if_fail (dim1[i] == 1, NULL);\r
+\r
+ /**Check if the second tensor is compatible */\r
+ dim2 = config->info.info[1].dimension;\r
+ max_label = dim2[0];\r
+ g_return_val_if_fail (max_label <= data->labeldata.total_labels, NULL);\r
+ if (max_label < data->labeldata.total_labels)\r
+ GST_WARNING ("The given tensor (2nd) has max_label (first dimension: %u) smaller than the number of labels in labels file (%s: %u).",\r
+ max_label, data->label_path, data->labeldata.total_labels);\r
+ g_return_val_if_fail (max_detection == dim2[1], NULL);\r
+ for (i = 2; i < NNS_TENSOR_RANK_LIMIT; i++)\r
+ g_return_val_if_fail (dim2[i] == 1, NULL);\r
+\r
+ /**Check consistency with max_detection */\r
+ if (!_set_max_detection (data, max_detection, MOBILENET_SSD_DETECTION_MAX)) {\r
+ return NULL;\r
+ }\r
+ str = g_strdup_printf ("testing");\r
+ caps = gst_caps_from_string (str);\r
+ setFramerateFromConfig (caps, config);\r
+ g_free (str);\r
+ (void) *pdata;\r
+ return caps;\r
+}\r
+\r
+\r
+static gchar decoder_subplugin_tensor_region[] = "tensor_region";\r
+\r
+/** @brief Tensor Region tensordec-plugin GstTensorDecoderDef instance */\r
+static GstTensorDecoderDef tensorRegion = { .modename = decoder_subplugin_tensor_region,\r
+ .init = tr_init,\r
+ .exit = tr_exit,\r
+ .setOption = tr_setOption,\r
+ .getOutCaps = tr_getOutCaps,\r
+ /** .getTransformSize = tr_getTransformsize, */\r
+ .decode = tr_decode };\r
+\r
+/** @brief Initialize this object for tensordec-plugin */\r
+void\r
+init_tr (void)\r
+{\r
+ nnstreamer_decoder_probe (&tensorRegion);\r
+}\r
+\r
+/** @brief Destruct this object for tensordec-plugin */\r
+void\r
+fini_tr (void)\r
+{\r
+ nnstreamer_decoder_exit (tensorRegion.modename);\r
+}\r
projects / platform / upstream / nnstreamer.git / commitdiff