Allow to add external converters in run-time as subplugins.
Policies to be determined:
- Build & integration policies for external converters.
- Would we allow a few "basic" external converters available by default? (without dlopen)
- Tizen feature keys and privileges along with "NOT SUPPORTED" handling.
- Which profiles would use which of external converters?
TODO:
- Implement multiple external subplugins for unit tests
Changes in V2:
- Typo fixed (comments)
Signed-off-by: MyungJoo Ham <myungjoo.ham@samsung.com>
[NNSCONF_PATH_DECODERS] = NNSTREAMER_PREFIX_DECODER,
[NNSCONF_PATH_CUSTOM_FILTERS] = NNSTREAMER_PREFIX_CUSTOMFILTERS,
[NNSCONF_PATH_EASY_CUSTOM_FILTERS] = NNSTREAMER_PREFIX_CUSTOMFILTERS, /**< Same as Custom Filters */
+ [NNSCONF_PATH_CONVERTERS] = NNSTREAMER_PREFIX_CONVERTER,
[NNSCONF_PATH_END] = NULL
};
gchar *pathFILTERS[CONF_SOURCES]; /**< directory paths for FILTERS */
gchar *pathDECODERS[CONF_SOURCES]; /**< directory paths for DECODERS */
gchar *pathCUSTOM_FILTERS[CONF_SOURCES]; /**< directory paths for CUSTOM FILTERS */
+ gchar *pathCONVERTERS[CONF_SOURCES]; /**< directory paths for CONVERTERS */
/*************************************************
* Processed Values *
gchar **filesFILTERS; /**< Null terminated list of full filepaths for FILTERS */
gchar **filesDECODERS;; /**< Null terminated list of full filepaths for DECODERS */
gchar **filesCUSTOM_FILTERS; /**< Null terminated list of full filepaths for CUSTOM FILTERS */
+ gchar **filesCONVERTERS; /**< Null terminated list of full filepaths for CONVERTERS */
gchar **basenameFILTERS; /**< Null terminated list of basenames for FILTERS */
gchar **basenameDECODERS;; /**< Null terminated list of basenames for DECODERS */
gchar **basenameCUSTOM_FILTERS; /**< Null terminated list of basenames for CUSTOM FILTERS */
+ gchar **basenameCONVERTERS; /**< Null terminated list of basenames for CONVERTERS */
} confdata;
static confdata conf = { 0 };
vstr = conf.basenameCUSTOM_FILTERS;
vstrFull = conf.filesCUSTOM_FILTERS;
break;
+ case NNSCONF_PATH_CONVERTERS:
+ vstr = conf.basenameCONVERTERS;
+ vstrFull = conf.filesCONVERTERS;
+ break;
default:
/* unknown type */
g_critical ("Failed to get sub-plugins, unknown sub-plugin type.");
g_free (conf.pathFILTERS[i]);
g_free (conf.pathDECODERS[i]);
g_free (conf.pathCUSTOM_FILTERS[i]);
+ g_free (conf.pathCONVERTERS[i]);
}
g_strfreev (conf.filesFILTERS);
g_strfreev (conf.filesDECODERS);
g_strfreev (conf.filesCUSTOM_FILTERS);
+ g_strfreev (conf.filesCONVERTERS);
g_strfreev (conf.basenameFILTERS);
g_strfreev (conf.basenameDECODERS);
g_strfreev (conf.basenameCUSTOM_FILTERS);
+ g_strfreev (conf.basenameCONVERTERS);
/* init with 0 */
memset (&conf, 0, sizeof (confdata));
g_key_file_get_string (key_file, "decoder", "decoders", NULL);
conf.pathCUSTOM_FILTERS[1] =
g_key_file_get_string (key_file, "filter", "customfilters", NULL);
+ conf.pathCONVERTERS[1] =
+ g_key_file_get_string (key_file, "converter", "converters", NULL);
}
/* Read from env variables. */
conf.pathDECODERS[0] = _strdup_getenv (NNSTREAMER_ENVVAR_DECODERS);
conf.pathCUSTOM_FILTERS[0] =
_strdup_getenv (NNSTREAMER_ENVVAR_CUSTOMFILTERS);
+ conf.pathCONVERTERS[0] = _strdup_getenv (NNSTREAMER_ENVVAR_CONVERTERS);
}
/* Strdup the hardcoded */
conf.pathFILTERS[2] = g_strdup (NNSTREAMER_FILTERS);
conf.pathDECODERS[2] = g_strdup (NNSTREAMER_DECODERS);
conf.pathCUSTOM_FILTERS[2] = g_strdup (NNSTREAMER_CUSTOM_FILTERS);
+ conf.pathCONVERTERS[2] = g_strdup (NNSTREAMER_CONVERTERS);
/* Fill in conf.files* */
_fill_in_vstr (&conf.filesFILTERS, &conf.basenameFILTERS, conf.pathFILTERS,
NNSCONF_PATH_DECODERS);
_fill_in_vstr (&conf.filesCUSTOM_FILTERS, &conf.basenameCUSTOM_FILTERS,
conf.pathCUSTOM_FILTERS, NNSCONF_PATH_CUSTOM_FILTERS);
+ _fill_in_vstr (&conf.filesCONVERTERS, &conf.basenameCONVERTERS,
+ conf.pathCONVERTERS, NNSCONF_PATH_CONVERTERS);
conf.loaded = TRUE;
return TRUE;
#define NNSTREAMER_ENVVAR_FILTERS "NNSTREAMER_FILTERS"
#define NNSTREAMER_ENVVAR_DECODERS "NNSTREAMER_DECODERS"
#define NNSTREAMER_ENVVAR_CUSTOMFILTERS "NNSTREAMER_CUSTOMFILTERS"
+#define NNSTREAMER_ENVVAR_CONVERTERS "NNSTREAMER_CONVERTERS"
/* Internal Hardcoded Values */
#define NNSTREAMER_DEFAULT_CONF_FILE "/etc/nnstreamer.ini"
#define NNSTREAMER_FILTERS "/usr/lib/nnstreamer/filters/"
#define NNSTREAMER_DECODERS "/usr/lib/nnstreamer/decoders/"
#define NNSTREAMER_CUSTOM_FILTERS "/usr/lib/nnstreamer/customfilters/"
+#define NNSTREAMER_CONVERTERS "/usr/lib/nnstreamer/converters/"
/**
* Note that users still can place their custom filters anywhere if they
* designate them with the full path.
#define NNSTREAMER_PREFIX_DECODER "libnnstreamer_decoder_"
#define NNSTREAMER_PREFIX_FILTER "libnnstreamer_filter_"
#define NNSTREAMER_PREFIX_CUSTOMFILTERS ""
+#define NNSTREAMER_PREFIX_CONVERTER ""
/* Custom filter does not have prefix */
/* struct for sub-plugins info (name and full path) */
NNSCONF_PATH_DECODERS,
NNSCONF_PATH_CUSTOM_FILTERS,
NNSCONF_PATH_EASY_CUSTOM_FILTERS,
+ NNSCONF_PATH_CONVERTERS,
NNSCONF_PATH_END,
} nnsconf_type_path;
/**
* @brief Get media type from structure
* @param structure structure to be interpreted
- * @return corresponding media type (returns _NNS_MEDIA_END for unsupported type)
+ * @return corresponding media type (returns _NNS_MEDIA_INVALID for unsupported type)
*/
extern media_type
gst_tensor_media_type_from_structure (const GstStructure * structure);
--- /dev/null
+/**
+ * NNStreamer API for Tensor_Decoder Sub-Plugins
+ * Copyright (C) 2019 MyungJoo Ham <myungjoo.ham@samsung.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Library General Public License for more details.
+ *
+ */
+/**
+ * @file nnstreamer_plugin_api_converter.h
+ * @date 09 Dec 2019
+ * @brief Mandatory APIs for NNStreamer Converter sub-plugins (Need Gst Devel)
+ * @see https://github.com/nnsuite/nnstreamer
+ * @author MyungJoo Ham <myungjoo.ham@samsung.com>
+ * @bug No known bugs except for NYI items
+ */
+#ifndef __NNS_PLUGIN_API_CONVERTER_H__
+#define __NNS_PLUGIN_API_CONVERTER_H__
+
+#include "tensor_typedef.h"
+#include <gst/gst.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/***********************************************
+* External Converters *
+************************************************/
+
+/**
+ * @brief Converter's subplugin implementation.
+ */
+struct _NNStreamerExternalConverter {
+ const char *media_type_name;
+
+ /** 1. chain func, data handling. */
+ GstBuffer * (*convert) (GstTensorConverter * self,
+ const GstBuffer * buf, gsize * frame_size, guint * frames_in);
+ /**< Convert the given input stream to tensor/tensors stream.
+ * @param[in/out] self A pointer designating "this".
+ * @param[in] buf The input stream buffer
+ * @param[out] frame_size The size of each frame (output buffer)
+ * @param[out] frames_in The number of frames in the given input buffer.
+ * @retval Return inbuf if the data is to be kept untouched.
+ * @retval Retrun a new GstBuf if the data is to be modified.
+ */
+
+ /** 2. parse_caps (type conv, input(media) to output(tensor)) */
+ gboolean (*get_caps) (GstTensorConverter * self, const GstStructure * st,
+ GstTensorConfig * config);
+ /**< Set the tensor config structure from the given stream frame
+ * @param[in/out] self A pointer designating "this".
+ * @param[in] st The input (original/media data) stream's metadata
+ * @param[out] config The output (tensor/tensors) emtadata
+ */
+ /** 3. query_cap (tpye conf, output(tensor) to input(media)) */
+ gboolean (*query_caps) (GstTensorConverter * self,
+ const GstTensorConfig *config, GstStructure *st);
+ /**< Filters (narrows down) the GstCap (st) with the given config.
+ * @param[in/out] self A pointer designating "this".
+ * @param[in] config The config of output tensor/tensors
+ * @param[in/out] st The gstcap of input to be filtered with config.
+ */
+};
+
+
+/**
+ * @brief Converter's sub-plugin should call this function to register itself.
+ * @param[in] ex Converter sub-plugin to be registered.
+ * @return TRUE if registered. FALSE is failed or duplicated.
+ */
+extern int
+registerExternalConverter (NNStreamerExternalConverter *ex);
+
+/**
+ * @brief Converter's sub-plugin may call this to unregister itself.
+ * @param[in] prefix The name of converter sub-plugin.
+ */
+extern void
+unregisterExternalConverter (const char *prefix);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* __NNS_PLUGIN_API_CONVERTER_H__ */
g_free (data);
}
+typedef enum
+{
+ NNS_SEARCH_FILENAME,
+ NNS_SEARCH_GETALL,
+ NNS_SEARCH_NO_OP,
+} subpluginSearchLogic;
+
+static subpluginSearchLogic searchAlgorithm[] = {
+ [NNS_SUBPLUGIN_FILTER] = NNS_SEARCH_FILENAME,
+ [NNS_SUBPLUGIN_DECODER] = NNS_SEARCH_FILENAME,
+ [NNS_EASY_CUSTOM_FILTER] = NNS_SEARCH_FILENAME,
+ [NNS_SUBPLUGIN_CONVERTER] = NNS_SEARCH_GETALL,
+ [NNS_SUBPLUGIN_END] = NNS_SEARCH_NO_OP,
+};
+
/** @brief Public function defined in the header */
const void *
get_subplugin (subpluginType type, const char *name)
_spdata_destroy);
table = subplugins[type];
- data = g_hash_table_lookup (table, name);
- if (data == NULL) {
+ if (searchAlgorithm[type] == NNS_SEARCH_GETALL) {
+ nnsconf_type_path conf_type = (nnsconf_type_path) type;
+ subplugin_info_s info;
+ guint i;
+ guint ret = nnsconf_get_subplugin_info (conf_type, &info);
+
+ for (i = 0; i < ret; i++) {
+ const gchar *fullpath = info.paths[i];
+
+ dlerror ();
+ G_UNLOCK (splock);
+ handle = dlopen (fullpath, RTLD_NOW);
+ /* If this is a correct subplugin, it will register itself */
+ G_LOCK (splock);
+ if (NULL == handle) {
+ g_critical ("Cannot dlopen %s with error %s.", fullpath, dlerror ());
+ continue;
+ }
+ }
+
+ searchAlgorithm[type] = NNS_SEARCH_NO_OP;
+ }
+
+ data = g_hash_table_lookup (table, name);
+ if (data == NULL && searchAlgorithm[type] == NNS_SEARCH_FILENAME) {
/** Search and register if found with the conf */
nnsconf_type_path conf_type = (nnsconf_type_path) type;
const gchar *fullpath = nnsconf_get_fullpath (name, conf_type);
case NNS_SUBPLUGIN_FILTER:
case NNS_SUBPLUGIN_DECODER:
case NNS_EASY_CUSTOM_FILTER:
+ case NNS_SUBPLUGIN_CONVERTER:
break;
default:
/* unknown sub-plugin type */
NNS_SUBPLUGIN_FILTER = NNSCONF_PATH_FILTERS,
NNS_SUBPLUGIN_DECODER = NNSCONF_PATH_DECODERS,
NNS_EASY_CUSTOM_FILTER = NNSCONF_PATH_EASY_CUSTOM_FILTERS,
+ NNS_SUBPLUGIN_CONVERTER = NNSCONF_PATH_CONVERTERS,
NNS_SUBPLUGIN_END,
} subpluginType;
/**
* @brief Get media type from structure
* @param structure structure to be interpreted
- * @return corresponding media type (returns _NNS_MEDIA_END for unsupported type)
+ * @return corresponding media type (returns _NNS_MEDIA_INVALID for unsupported type)
*/
media_type
gst_tensor_media_type_from_structure (const GstStructure * structure)
name = gst_structure_get_name (structure);
- g_return_val_if_fail (name != NULL, _NNS_MEDIA_END);
+ g_return_val_if_fail (name != NULL, _NNS_MEDIA_INVALID);
if (g_str_has_prefix (name, "video/")) {
return _NNS_VIDEO;
}
/* unknown or unsupported type */
- return _NNS_MEDIA_END;
+ return _NNS_MEDIA_INVALID;
}
/**
* @see https://github.com/nnsuite/nnstreamer
* @author MyungJoo Ham <myungjoo.ham@samsung.com>
* @bug No known bugs except for NYI items
+ * @todo For flatbuffers, support other/tensors with properties
+ * @todo Subplugins are not tested, yet.
*/
/**
#include <string.h>
#include "tensor_converter.h"
#include "converter-media-info.h"
+#include <nnstreamer_subplugin.h>
+#include <nnstreamer_plugin_api_converter.h>
/**
* @brief Macro for debug mode.
/**
* @brief tensor_converter properties
+ * @todo For flatbuffers, support other/tensors.
*/
enum
{
static gboolean gst_tensor_converter_parse_caps (GstTensorConverter * self,
const GstCaps * caps);
+static const NNStreamerExternalConverter *findExternalConverter (const char
+ *media_type_name);
+
/**
* @brief Initialize the tensor_converter's class.
*/
self->silent = DEFAULT_SILENT;
self->set_timestamp = DEFAULT_SET_TIMESTAMP;
self->frames_per_tensor = DEFAULT_FRAMES_PER_TENSOR;
- self->in_media_type = _NNS_MEDIA_END;
+ self->in_media_type = _NNS_MEDIA_INVALID;
self->frame_size = 0;
self->remove_padding = FALSE;
gst_tensor_info_init (&self->tensor_info);
g_assert ((buf_size % frame_size) == 0);
frames_in = buf_size / frame_size;
break;
+ case _NNS_MEDIA_PLUGINS:
+ {
+ if (self->externalConverter == NULL ||
+ self->externalConverter->convert == NULL)
+ return GST_FLOW_NOT_SUPPORTED;
+ inbuf = self->externalConverter->convert (self, buf, &frame_size,
+ &frames_in);
+
+ g_assert (inbuf != NULL);
+ g_assert (frame_size > 0);
+ g_assert ((buf_size % frame_size) == 0);
+ break;
+ }
default:
GST_ERROR_OBJECT (self, "Unsupported type %d\n", self->in_media_type);
g_assert (0);
}
}
break;
+ case _NNS_MEDIA_INVALID: /* this could be MEDIA_PLUGIN */
+ {
+ const gchar *name = gst_structure_get_name (st);
+ const NNStreamerExternalConverter *ex;
+ gboolean ret;
+
+ if (name == NULL)
+ break;
+ ex = findExternalConverter (name);
+ if (ex == NULL)
+ break;
+
+ /** @todo What if this is inconsistent with self->ex? */
+
+ g_assert (ex->query_caps);
+ ret = ex->query_caps (self, &config, st);
+ if (ret == FALSE) {
+ GST_ERROR_OBJECT (self,
+ "Failed to filter GstCap structure with the given config");
+ }
+
+ break;
+ }
default:
/* do nothing for text and octet stream */
break;
structure = gst_caps_get_structure (caps, 0);
in_type = gst_tensor_media_type_from_structure (structure);
+ self->externalConverter = NULL;
switch (in_type) {
case _NNS_VIDEO:
break;
}
default:
+ {
+ /* if found, configure in_mdeia_type = _NNS_MEDIA_PLUGINS */
+ const gchar *name = gst_structure_get_name (structure);
+ if (name != NULL) {
+ const NNStreamerExternalConverter *ex = findExternalConverter (name);
+ if (ex != NULL) {
+ in_type = _NNS_MEDIA_PLUGINS;
+ self->externalConverter = ex;
+
+ if (NULL == ex->get_caps || !ex->get_caps (self, structure, &config)) {
+ GST_ERROR_OBJECT (self,
+ "Failed to get tensor info from %s. Check the given options.",
+ name);
+ g_critical ("Please set the options property correctly.\n");
+ self->externalConverter = NULL;
+ return FALSE;
+ }
+ break;
+ }
+ }
GST_ERROR_OBJECT (self, "Unsupported type %d\n", in_type);
return FALSE;
+ }
}
/** set the number of frames in dimension */
self->tensor_config = config;
return TRUE;
}
+
+/**
+ * @brief Converter's external subplugins should call this at init.
+ */
+int
+registerExternalConverter (NNStreamerExternalConverter * ex)
+{
+ return register_subplugin (NNS_SUBPLUGIN_CONVERTER, ex->media_type_name, ex);
+}
+
+/**
+ * @brief Converter's external subplugins should call this at exit.
+ */
+void
+unregisterExternalConverter (const char *media_type_name)
+{
+ unregister_subplugin (NNS_SUBPLUGIN_CONVERTER, media_type_name);
+}
+
+/**
+ * @brief Internal static function to find registered subplugins.
+ */
+static const NNStreamerExternalConverter *
+findExternalConverter (const char *media_type_name)
+{
+ return get_subplugin (NNS_SUBPLUGIN_CONVERTER, media_type_name);
+}
typedef struct _GstTensorConverter GstTensorConverter;
typedef struct _GstTensorConverterClass GstTensorConverterClass;
+typedef struct _NNStreamerExternalConverter NNStreamerExternalConverter;
/**
* @brief Internal data structure for tensor_converter instances.
GstAdapter *adapter; /**< adapt incoming media stream */
media_type in_media_type; /**< incoming media type */
+ const NNStreamerExternalConverter *externalConverter;
+ /**< used if in_media_type == _NNS_MEDIA_PLUGINS */
+
gsize frame_size; /**< size of one frame */
gboolean remove_padding; /**< If true, zero-padding must be removed */
gboolean tensor_configured; /**< True if already successfully configured tensor metadata */
*/
GType gst_tensor_converter_get_type (void);
+
G_END_DECLS
#endif /** __GST_TENSOR_CONVERTER_H__ */
*
* This is realted with media input stream to other/tensor.
* There is no restrictions for the outputs.
+ *
+ * In order to prevent enum-mix issues between device profiles,
+ * we explicitly define numbers for each enum type.
*/
typedef enum _nns_media_type
{
+ _NNS_MEDIA_INVALID = -1, /**< Uninitialized */
_NNS_VIDEO = 0, /**< supposedly video/x-raw */
- _NNS_AUDIO, /**< supposedly audio/x-raw */
- _NNS_TEXT, /**< supposedly text/x-raw */
- _NNS_OCTET, /**< supposedly application/octet-stream */
-
- _NNS_MEDIA_END, /**< End Marker */
+ _NNS_AUDIO = 1, /**< supposedly audio/x-raw */
+ _NNS_TEXT = 2, /**< supposedly text/x-raw */
+ _NNS_OCTET = 3, /**< supposedly application/octet-stream */
+ _NNS_MEDIA_PLUGINS = 0x1000, /**< external converters */
} media_type;
/**
[decoder]
decoders=@SUBPLUGIN_INSTALL_PREFIX@/decoders/
+[converter]
+converters=@SUBPLUGIN_INSTALL_PREFIX@/converters/
+
# Set 1 or True if you want to use NNAPI with tensorflow-lite, which enables to use NNAPI backend, which may use GPU or NPU/TPU.
[tensorflowlite]
enable_nnapi=False
projects / platform / upstream / nnstreamer.git / commitdiff