- Allow to configure input/output dimension in run-time.
- After pad-cap negotiation, if gstremaer somehow fixes input-dimenson,
a filter subplugin can configure output-dimension accordingly.
- An example custom filter, "passthrough_variable", supports any
arbitrary tensor for passthrough operations.
- Filter/Main and Filter/Custom has been amended during the development
of the example custom plugin.
FOUND BUG (mentioned as "known bug" in the code):
- If GST-BaseTransform tries to fixate pad-caps without getting
any hints on the input dimension, fixate vmethod fixes the input
dimension to 1:1:1:1. We need to implement non-fixed pad-cap
declarations in fixated vmethod.
- The disabled test case #4 in tests/nnstreamer_filter_custom
is the corresponding regression detector.
- However, this bug is to be fixed with another commit.
Signed-off-by: MyungJoo Ham <myungjoo.ham@samsung.com>
* @param dim The tensor dimension
*/
size_t
-get_tensor_element_count (uint32_t dim[NNS_TENSOR_RANK_LIMIT])
+get_tensor_element_count (const uint32_t dim[NNS_TENSOR_RANK_LIMIT])
{
size_t count = 1;
int i;
dimension = filter->prop.outputDimension;
}
+ /* @TODO KNOWN BUG: when prop.i/o-dim is not configured, this is going to screw all */
+ /* This known bug breaks case 4 of nnstreamer_filter_custom */
+
/* 2. configure caps based on type & dimension */
rank = gst_tensor_filter_get_rank (dimension);
tmp = gst_caps_new_simple ("other/tensor", "rank", G_TYPE_INT, rank, "type", G_TYPE_STRING, tensor_element_typename[*type], "dim1", G_TYPE_INT, dimension[0], "dim2", G_TYPE_INT, dimension[1], "dim3", G_TYPE_INT, dimension[2], "dim4", G_TYPE_INT, dimension[3], "framerate", GST_TYPE_FRACTION, 0, 1, /* @TODO: support other framerates! */
tensor_type type;
int i;
- g_assert (!(filter->prop.fw->getInputDimension && filter->prop.fw->getOutputDimension) != !filter->prop.fw->setInputDimension); /* This is "XOR" */
+ /* Ensure the subplugin is contacted first before checking the XOR assert */
+ if (!prop->fwOpened && fw->open)
+ fw->open (filter, &filter->privateData);
+ prop->fwOpened = TRUE;
+
+ g_assert (!(fw->getInputDimension && fw->getOutputDimension) != !fw->setInputDimension); /* This is "XOR" */
if (fw->getInputDimension != NULL) {
g_assert (fw->getOutputDimension != NULL);
ret =
fw->setInputDimension (filter, &filter->privateData,
prop->inputDimension, prop->inputType, dim, &type);
- if (prop->outputConfigured & _TFC_TYPE)
+ if (prop->outputConfigured & _TFC_TYPE) {
g_assert (prop->outputType == type);
+ }
if (prop->outputConfigured & _TFC_DIMENSION)
for (i = 0; i < NNS_TENSOR_RANK_LIMIT; i++)
g_assert (prop->outputDimension[i] == dim[i]);
* However, one of the two must be NULL
* And, if getOutputDimension != NULL, getInputDimension != NULL.
*/
- int (*setInputDimension)(const GstTensor_Filter *filter, void **private_data, const tensor_dim inputDimension, tensor_type inputType, tensor_dim outputDimension, tensor_type *outputType);
+ int (*setInputDimension)(const GstTensor_Filter *filter, void **private_data, const tensor_dim inputDimension, const tensor_type inputType, tensor_dim outputDimension, tensor_type *outputType);
/**< Optional. Set Null if not supported. Tensor_filter::main will
* configure input dimension from pad-cap in run-time for the sub-plugin.
* Then, the sub-plugin is required to return corresponding output dimension
* If this is NULL, both getInput/OutputDimension must be non-NULL.
* If this is non-NULL, both getInput/OutputDimension must be NULL.
+ *
+ * When you use this, do NOT allocate or fix internal data structure based on it
+ * until invoke is called. Gstreamer may try different dimensions before
+ * settling down.
*/
void (*open)(const GstTensor_Filter *filter, void **private_data); /**< Optional. tensor_filter.c will call this before any of other callbacks and will call once before calling close */
g_assert (ptr->methods->initfunc);
ptr->customFW_private_data = ptr->methods->initfunc (&(filter->prop));
+
+ /* After init func, (getInput XOR setInput) && (getOutput XOR setInput) must hold! */
+ g_assert (!ptr->methods->getInputDim != !ptr->methods->setInputDim &&
+ !ptr->methods->getOutputDim != !ptr->methods->setInputDim);
+
+ /**
+ * Depending on which callbacks the custom filter supplies, remove
+ * unnecessary callbacks from thisself.
+ */
+ if (ptr->methods->getInputDim && ptr->methods->getOutputDim &&
+ !ptr->methods->setInputDim) {
+ NNS_support_custom.setInputDimension = NULL;
+ } else if (!ptr->methods->getInputDim && !ptr->methods->getOutputDim &&
+ ptr->methods->setInputDim) {
+ NNS_support_custom.getInputDimension = NULL;
+ NNS_support_custom.getOutputDimension = NULL;
+ } else {
+ g_assert (TRUE); /* Cannot reach here! */
+ }
+
return 0;
}
/**
+ * @brief The open callback for GstTensor_Filter_Framework. Called before anything else
+ */
+static void
+custom_open (const GstTensor_Filter * filter, void **private_data)
+{
+ int retval = custom_loadlib (filter, private_data);
+
+ g_assert (retval == 0); /* This must be called only once */
+}
+
+/**
* @brief The mandatory callback for GstTensor_Filter_Framework
* @param filter The parent object
* @param[in] inptr The input tensor
internal_data *ptr;
/* Actually, tensor_filter must have called getInput/OotputDim first. */
- g_assert (retval != 0);
+ g_assert (retval == 1);
if (retval < 0)
return retval;
int retval = custom_loadlib (filter, private_data);
internal_data *ptr;
- if (retval < 0)
- return retval;
+ g_assert (retval == 1); /* open must be called before */
g_assert (filter->privateData && *private_data == filter->privateData);
ptr = *private_data;
int retval = custom_loadlib (filter, private_data);
internal_data *ptr;
- if (retval < 0)
- return retval;
+ g_assert (retval == 1); /* open must be called before */
g_assert (filter->privateData && *private_data == filter->privateData);
ptr = *private_data;
}
/**
+ * @brief The set-input-dim callback for GstTensor_Filter_Framework
+ */
+static int
+custom_setInputDim (const GstTensor_Filter * filter, void **private_data,
+ const tensor_dim iDimension, const tensor_type iType,
+ tensor_dim oDimension, tensor_type * oType)
+{
+ int retval = custom_loadlib (filter, private_data);
+ internal_data *ptr;
+
+ g_assert (retval == 1); /* open must be called before */
+
+ g_assert (filter->privateData && *private_data == filter->privateData);
+ ptr = *private_data;
+
+ return ptr->methods->setInputDim (ptr->customFW_private_data,
+ &(filter->prop), iDimension, iType, oDimension, oType);
+}
+
+/**
* @brief Free privateData and move on.
*/
static void
.name = "custom",
.allow_in_place = FALSE, /* custom cannot support in-place (outptr == inptr). */
.invoke_NN = custom_invoke,
+
+ /* We need to disable getI/O-dim or setI-dim with the first call */
.getInputDimension = custom_getInputDim,
.getOutputDimension = custom_getOutputDim,
+ .setInputDimension = custom_setInputDim,
+ .open = custom_open,
.close = custom_close,
};
* @return The number of elements. 0 if error.
* @param dim The tensor dimension
*/
-extern size_t get_tensor_element_count(tensor_dim dim);
+extern size_t get_tensor_element_count(const tensor_dim dim);
/**
* @brief Read pad-cap, return corresponding tensor-dim/type.
/**
* @brief Get input tensor type.
- * @param[in] private_data The pointer returned by NNStreamer_custom_exit.
+ * @param[in] private_data The pointer returned by NNStreamer_custom_init.
* @param[in] prop Tensor_Filter's property values. Do not change its values.
* @param[out] inputDimension uint32_t[NNS_TENSOR_RANK_LIMIT] (tensor_dim)
* @param[out] type Type of each element in the input tensor
/**
* @brief Get output tensor type.
- * @param[in] private_data The pointer returned by NNStreamer_custom_exit.
+ * @param[in] private_data The pointer returned by NNStreamer_custom_init.
* @param[in] prop Tensor_Filter's property values. Do not change its values.
* @param[out] outputDimension uint32_t[NNS_TENSOR_RANK_LIMIT] (tensor_dim)
* @param[out] type Type of each element in the output tensor
tensor_dim outputDimension, tensor_type *type);
/**
+ * @brief Set input dim by framework. Let custom plutin set output dim accordingly.
+ * @param[in] private_data The pointer returned by NNStreamer_custom_init
+ * @param[in] prop Tensor_Filter's property values. Do not change its values.
+ * @param[in] inputDimension Input dimension designated by the gstreamer framework. Note that this is not a fixed value and gstreamer may try different values during pad-cap negotiations.
+ * @param[in] inputType Input element type designated by the gstreamer framework.
+ * @param[out] outputDimension Output dimension according to the inputDimension/type.
+ * @param[out] outputType Output element type according to the inputDimension/type.
+ *
+ * @caution Do not fix internal values based on this call. Gstreamer may call
+ * this function repeatedly with different values during pad-cap negotiations.
+ * Fix values when invoke is finally called.
+ */
+typedef int (*NNS_custom_set_input_dimension)(void *private_data, const GstTensor_Filter_Properties *prop,
+ const tensor_dim inputDimension, const tensor_type inputType,
+ tensor_dim outputDimension, tensor_type *outputType);
+
+/**
* @brief Invoke the "main function".
- * @param[in] private_data The pointer returned by NNStreamer_custom_exit.
+ * @param[in] private_data The pointer returned by NNStreamer_custom_init.
* @param[in] prop Tensor_Filter's property values. Do not change its values.
* @param[in] inputPtr pointer to input tensor, size = dim1 x dim2 x dim3 x dim4 x typesize, allocated by caller
* @param[in] inputPtr pointer to output tensor, size = dim1 x dim2 x dim3 x dim4 x typesize, allocated by caller
struct _NNStreamer_custom_class {
NNS_custom_init_func initfunc; /**< called before any other callbacks from tensor_filter_custom.c */
NNS_custom_exit_func exitfunc; /**< will not call other callbacks after this call */
- NNS_custom_get_input_dimension getInputDim; /**< a custom filter is required to provide input tensor dimension */
- NNS_custom_get_output_dimension getOutputDim; /**< a custom filter is require dto provide output tensor dimension */
+ NNS_custom_get_input_dimension getInputDim; /**< a custom filter is required to provide input tensor dimension unless setInputdim is defined. */
+ NNS_custom_get_output_dimension getOutputDim; /**< a custom filter is require dto provide output tensor dimension unless setInputDim is defined. */
+ NNS_custom_set_input_dimension setInputDim; /**< without getI/O-Dim, this allows framework to set input dimension and get output dimension from the custom filter according to the input dimension */
NNS_custom_invoke invoke; /**< the main function, "invoke", that transforms input to output */
};
typedef struct _NNStreamer_custom_class NNStreamer_custom_class;
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
ADD_LIBRARY(nnstreamer_customfilter_passthrough SHARED nnstreamer_customfilter_example_passthrough.c)
+ADD_LIBRARY(nnstreamer_customfilter_passthrough_variable SHARED nnstreamer_customfilter_example_passthrough_variable.c)
TARGET_LINK_LIBRARIES(nnstreamer_customfilter_passthrough ${pkgs_LIBRARIES})
TARGET_INCLUDE_DIRECTORIES(nnstreamer_customfilter_passthrough PUBLIC ${pkgs_INCLUDE_DIRS})
TARGET_COMPILE_OPTIONS(nnstreamer_customfilter_passthrough PUBLIC ${pkgs_CFLAGS_OTHER})
-INSTALL(TARGETS nnstreamer_customfilter_passthrough
+TARGET_LINK_LIBRARIES(nnstreamer_customfilter_passthrough_variable ${pkgs_LIBRARIES})
+TARGET_INCLUDE_DIRECTORIES(nnstreamer_customfilter_passthrough_variable PUBLIC ${pkgs_INCLUDE_DIRS})
+TARGET_COMPILE_OPTIONS(nnstreamer_customfilter_passthrough_variable PUBLIC ${pkgs_CFLAGS_OTHER})
+
+INSTALL(TARGETS nnstreamer_customfilter_passthrough nnstreamer_customfilter_passthrough_variable
RUNTIME DESTINATION ${EXEC_PREFIX}
LIBRARY DESTINATION ${LIB_INSTALL_DIR}
ARCHIVE DESTINATION ${LIB_INSTALL_DIR}
* NNStreamer Custom Filter Example 1. Pass-Through
* Copyright (C) 2018 MyungJoo Ham <myungjoo.ham@samsung.com>
*
- * LICENSE: Apache-2.0
+ * LICENSE: LGPL-2.1
*
* @file nnstreamer_customfilter_example_passthrough.c
* @date 11 Jun 2018
* @brief Custom NNStreamer Filter Example 1. "Pass-Through"
* @author MyungJoo Ham <myungjoo.ham@samsung.com>
*
- * As a test, before #71 is resolved, this will support "3x280x40" uint8 tensors.
+ * this will supports "3x280x40" uint8 tensors (hardcoded dimensions)
*/
#include <stdlib.h>
--- /dev/null
+/**
+ * NNStreamer Custom Filter Example 2. Pass-Through with Variable Dimensions
+ * Copyright (C) 2018 MyungJoo Ham <myungjoo.ham@samsung.com>
+ *
+ * LICENSE: LGPL-2.1
+ *
+ * @file nnstreamer_customfilter_example_passthrough_variable.c
+ * @date 22 Jun 2018
+ * @brief Custom NNStreamer Filter Example 2. "Pass-Through with Variable Dimensions"
+ * @author MyungJoo Ham <myungjoo.ham@samsung.com>
+ */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <tensor_filter_custom.h>
+#include <tensor_common.h>
+
+typedef struct _pt_data
+{
+ uint32_t id; /***< Just for testing */
+} pt_data;
+
+static void *
+pt_init (const GstTensor_Filter_Properties * prop)
+{
+ pt_data *data = (pt_data *) malloc (sizeof (pt_data));
+
+ data->id = 0;
+ return data;
+}
+
+static void
+pt_exit (void *private_data, const GstTensor_Filter_Properties * prop)
+{
+ pt_data *data = private_data;
+ g_assert (data);
+ free (data);
+}
+
+static int
+set_inputDim (void *private_data, const GstTensor_Filter_Properties * prop,
+ const tensor_dim iDim, const tensor_type iType,
+ tensor_dim oDim, tensor_type * oType)
+{
+ int i;
+
+ for (i = 0; i < NNS_TENSOR_RANK_LIMIT; i++)
+ oDim[i] = iDim[i];
+ *oType = iType;
+
+ return 0;
+}
+
+static int
+pt_invoke (void *private_data, const GstTensor_Filter_Properties * prop,
+ const uint8_t * inptr, uint8_t * outptr)
+{
+ pt_data *data = private_data;
+ size_t size;
+
+ g_assert (data);
+ g_assert (inptr);
+ g_assert (outptr);
+
+ size = get_tensor_element_count (prop->outputDimension) *
+ tensor_element_size[prop->outputType];
+
+ g_assert (inptr != outptr);
+ memcpy (outptr, inptr, size);
+
+ return 0;
+}
+
+static NNStreamer_custom_class NNStreamer_custom_body = {
+ .initfunc = pt_init,
+ .exitfunc = pt_exit,
+ .setInputDim = set_inputDim,
+ .invoke = pt_invoke,
+};
+
+/* The dyn-loaded object */
+NNStreamer_custom_class *NNStreamer_custom = &NNStreamer_custom_body;
gstTest "videotestsrc num-buffers=1 ! video/x-raw,format=RGB,width=280,height=40,framerate=0/1 ! videoconvert ! video/x-raw, format=RGB ! filesink location=\"testcase02.apitest.log\" sync=true" 0
PATH_TO_MODEL="../../build/nnstreamer_example/custom_example_passthrough/libnnstreamer_customfilter_passthrough.so"
+PATH_TO_MODEL_V="../../build/nnstreamer_example/custom_example_passthrough/libnnstreamer_customfilter_passthrough_variable.so"
gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} videotestsrc num-buffers=1 ! video/x-raw,format=RGB,width=280,height=40,framerate=0/1 ! videoconvert ! video/x-raw, format=RGB ! tensor_converter ! tee name=t ! queue ! tensor_filter framework=\"custom\" model=\"${PATH_TO_MODEL}\" input=\"3:280:40\" inputtype=\"uint8\" output=\"3:280:40\" outputtype=\"uint8\" ! filesink location=\"testcase01.passthrough.log\" sync=true t. ! queue ! filesink location=\"testcase01.direct.log\" sync=true" 1
compareAll testcase02.direct.log testcase02.passthrough.log 2
+
+gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} videotestsrc num-buffers=1 ! video/x-raw,format=RGB,width=640,height=480,framerate=0/1 ! videoconvert ! video/x-raw, format=RGB ! tensor_converter ! tee name=t ! queue ! tensor_filter framework=\"custom\" model=\"${PATH_TO_MODEL_V}\" input=\"3:640:480\" inputtype=\"uint8\" output=\"3:640:480\" outputtype=\"uint8\" ! filesink location=\"testcase03.passthrough.log\" sync=true t. ! queue ! filesink location=\"testcase03.direct.log\" sync=true" 3
+
+compareAll testcase03.direct.log testcase03.passthrough.log 3
+
+## @TODO there is a known bug that breaks case 4.
+#gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} videotestsrc num-buffers=1 ! video/x-raw,format=RGB,width=640,height=480,framerate=0/1 ! videoconvert ! video/x-raw, format=RGB ! tensor_converter ! tee name=t ! queue ! tensor_filter framework=\"custom\" model=\"${PATH_TO_MODEL_V}\" ! filesink location=\"testcase04.passthrough.log\" sync=true t. ! queue ! filesink location=\"testcase04.direct.log\" sync=true" 4
+
+#compareAll testcase04.direct.log testcase04.passthrough.log 4
+
report
projects / platform / upstream / nnstreamer.git / commitdiff