From: MyungJoo Ham Date: Fri, 22 Jun 2018 06:26:44 +0000 (+0900) Subject: [Filter/Custom] Support flexible dimension X-Git-Tag: v0.0.1~118 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=4d7ce401cda845b1760ae68a391202a0efbf566c;p=platform%2Fupstream%2Fnnstreamer.git [Filter/Custom] Support flexible dimension - Allow to configure input/output dimension in run-time. - After pad-cap negotiation, if gstremaer somehow fixes input-dimenson, a filter subplugin can configure output-dimension accordingly. - An example custom filter, "passthrough_variable", supports any arbitrary tensor for passthrough operations. - Filter/Main and Filter/Custom has been amended during the development of the example custom plugin. FOUND BUG (mentioned as "known bug" in the code): - If GST-BaseTransform tries to fixate pad-caps without getting any hints on the input dimension, fixate vmethod fixes the input dimension to 1:1:1:1. We need to implement non-fixed pad-cap declarations in fixated vmethod. - The disabled test case #4 in tests/nnstreamer_filter_custom is the corresponding regression detector. - However, this bug is to be fixed with another commit. Signed-off-by: MyungJoo Ham --- diff --git a/common/tensor_common.c b/common/tensor_common.c index 2d0f42d..f25162c 100644 --- a/common/tensor_common.c +++ b/common/tensor_common.c @@ -174,7 +174,7 @@ get_tensor_dimension (const gchar * param, uint32_t dim[NNS_TENSOR_RANK_LIMIT]) * @param dim The tensor dimension */ size_t -get_tensor_element_count (uint32_t dim[NNS_TENSOR_RANK_LIMIT]) +get_tensor_element_count (const uint32_t dim[NNS_TENSOR_RANK_LIMIT]) { size_t count = 1; int i; diff --git a/gst/tensor_filter/tensor_filter.c b/gst/tensor_filter/tensor_filter.c index 88d0325..e30afc0 100644 --- a/gst/tensor_filter/tensor_filter.c +++ b/gst/tensor_filter/tensor_filter.c @@ -350,6 +350,9 @@ gst_tensor_filter_fix_caps (GstTensor_Filter * filter, gboolean isInput, dimension = filter->prop.outputDimension; } + /* @TODO KNOWN BUG: when prop.i/o-dim is not configured, this is going to screw all */ + /* This known bug breaks case 4 of nnstreamer_filter_custom */ + /* 2. configure caps based on type & dimension */ rank = gst_tensor_filter_get_rank (dimension); tmp = gst_caps_new_simple ("other/tensor", "rank", G_TYPE_INT, rank, "type", G_TYPE_STRING, tensor_element_typename[*type], "dim1", G_TYPE_INT, dimension[0], "dim2", G_TYPE_INT, dimension[1], "dim3", G_TYPE_INT, dimension[2], "dim4", G_TYPE_INT, dimension[3], "framerate", GST_TYPE_FRACTION, 0, 1, /* @TODO: support other framerates! */ @@ -789,7 +792,12 @@ gst_tensor_filter_property_process (GstTensor_Filter * filter) tensor_type type; int i; - g_assert (!(filter->prop.fw->getInputDimension && filter->prop.fw->getOutputDimension) != !filter->prop.fw->setInputDimension); /* This is "XOR" */ + /* Ensure the subplugin is contacted first before checking the XOR assert */ + if (!prop->fwOpened && fw->open) + fw->open (filter, &filter->privateData); + prop->fwOpened = TRUE; + + g_assert (!(fw->getInputDimension && fw->getOutputDimension) != !fw->setInputDimension); /* This is "XOR" */ if (fw->getInputDimension != NULL) { g_assert (fw->getOutputDimension != NULL); @@ -838,8 +846,9 @@ gst_tensor_filter_property_process (GstTensor_Filter * filter) ret = fw->setInputDimension (filter, &filter->privateData, prop->inputDimension, prop->inputType, dim, &type); - if (prop->outputConfigured & _TFC_TYPE) + if (prop->outputConfigured & _TFC_TYPE) { g_assert (prop->outputType == type); + } if (prop->outputConfigured & _TFC_DIMENSION) for (i = 0; i < NNS_TENSOR_RANK_LIMIT; i++) g_assert (prop->outputDimension[i] == dim[i]); diff --git a/gst/tensor_filter/tensor_filter.h b/gst/tensor_filter/tensor_filter.h index 27ed0b7..e3b0829 100644 --- a/gst/tensor_filter/tensor_filter.h +++ b/gst/tensor_filter/tensor_filter.h @@ -161,12 +161,16 @@ struct _GstTensor_Filter_Framework * However, one of the two must be NULL * And, if getOutputDimension != NULL, getInputDimension != NULL. */ - int (*setInputDimension)(const GstTensor_Filter *filter, void **private_data, const tensor_dim inputDimension, tensor_type inputType, tensor_dim outputDimension, tensor_type *outputType); + int (*setInputDimension)(const GstTensor_Filter *filter, void **private_data, const tensor_dim inputDimension, const tensor_type inputType, tensor_dim outputDimension, tensor_type *outputType); /**< Optional. Set Null if not supported. Tensor_filter::main will * configure input dimension from pad-cap in run-time for the sub-plugin. * Then, the sub-plugin is required to return corresponding output dimension * If this is NULL, both getInput/OutputDimension must be non-NULL. * If this is non-NULL, both getInput/OutputDimension must be NULL. + * + * When you use this, do NOT allocate or fix internal data structure based on it + * until invoke is called. Gstreamer may try different dimensions before + * settling down. */ void (*open)(const GstTensor_Filter *filter, void **private_data); /**< Optional. tensor_filter.c will call this before any of other callbacks and will call once before calling close */ diff --git a/gst/tensor_filter/tensor_filter_custom.c b/gst/tensor_filter/tensor_filter_custom.c index 4a5a2e1..a170b55 100644 --- a/gst/tensor_filter/tensor_filter_custom.c +++ b/gst/tensor_filter/tensor_filter_custom.c @@ -110,10 +110,41 @@ custom_loadlib (const GstTensor_Filter * filter, void **private_data) g_assert (ptr->methods->initfunc); ptr->customFW_private_data = ptr->methods->initfunc (&(filter->prop)); + + /* After init func, (getInput XOR setInput) && (getOutput XOR setInput) must hold! */ + g_assert (!ptr->methods->getInputDim != !ptr->methods->setInputDim && + !ptr->methods->getOutputDim != !ptr->methods->setInputDim); + + /** + * Depending on which callbacks the custom filter supplies, remove + * unnecessary callbacks from thisself. + */ + if (ptr->methods->getInputDim && ptr->methods->getOutputDim && + !ptr->methods->setInputDim) { + NNS_support_custom.setInputDimension = NULL; + } else if (!ptr->methods->getInputDim && !ptr->methods->getOutputDim && + ptr->methods->setInputDim) { + NNS_support_custom.getInputDimension = NULL; + NNS_support_custom.getOutputDimension = NULL; + } else { + g_assert (TRUE); /* Cannot reach here! */ + } + return 0; } /** + * @brief The open callback for GstTensor_Filter_Framework. Called before anything else + */ +static void +custom_open (const GstTensor_Filter * filter, void **private_data) +{ + int retval = custom_loadlib (filter, private_data); + + g_assert (retval == 0); /* This must be called only once */ +} + +/** * @brief The mandatory callback for GstTensor_Filter_Framework * @param filter The parent object * @param[in] inptr The input tensor @@ -127,7 +158,7 @@ custom_invoke (const GstTensor_Filter * filter, void **private_data, internal_data *ptr; /* Actually, tensor_filter must have called getInput/OotputDim first. */ - g_assert (retval != 0); + g_assert (retval == 1); if (retval < 0) return retval; @@ -149,8 +180,7 @@ custom_getInputDim (const GstTensor_Filter * filter, void **private_data, int retval = custom_loadlib (filter, private_data); internal_data *ptr; - if (retval < 0) - return retval; + g_assert (retval == 1); /* open must be called before */ g_assert (filter->privateData && *private_data == filter->privateData); ptr = *private_data; @@ -169,8 +199,7 @@ custom_getOutputDim (const GstTensor_Filter * filter, void **private_data, int retval = custom_loadlib (filter, private_data); internal_data *ptr; - if (retval < 0) - return retval; + g_assert (retval == 1); /* open must be called before */ g_assert (filter->privateData && *private_data == filter->privateData); ptr = *private_data; @@ -180,6 +209,26 @@ custom_getOutputDim (const GstTensor_Filter * filter, void **private_data, } /** + * @brief The set-input-dim callback for GstTensor_Filter_Framework + */ +static int +custom_setInputDim (const GstTensor_Filter * filter, void **private_data, + const tensor_dim iDimension, const tensor_type iType, + tensor_dim oDimension, tensor_type * oType) +{ + int retval = custom_loadlib (filter, private_data); + internal_data *ptr; + + g_assert (retval == 1); /* open must be called before */ + + g_assert (filter->privateData && *private_data == filter->privateData); + ptr = *private_data; + + return ptr->methods->setInputDim (ptr->customFW_private_data, + &(filter->prop), iDimension, iType, oDimension, oType); +} + +/** * @brief Free privateData and move on. */ static void @@ -197,7 +246,11 @@ GstTensor_Filter_Framework NNS_support_custom = { .name = "custom", .allow_in_place = FALSE, /* custom cannot support in-place (outptr == inptr). */ .invoke_NN = custom_invoke, + + /* We need to disable getI/O-dim or setI-dim with the first call */ .getInputDimension = custom_getInputDim, .getOutputDimension = custom_getOutputDim, + .setInputDimension = custom_setInputDim, + .open = custom_open, .close = custom_close, }; diff --git a/include/tensor_common.h b/include/tensor_common.h index 87057a6..597a166 100644 --- a/include/tensor_common.h +++ b/include/tensor_common.h @@ -130,7 +130,7 @@ extern int get_tensor_dimension(const gchar* param, tensor_dim dim); * @return The number of elements. 0 if error. * @param dim The tensor dimension */ -extern size_t get_tensor_element_count(tensor_dim dim); +extern size_t get_tensor_element_count(const tensor_dim dim); /** * @brief Read pad-cap, return corresponding tensor-dim/type. diff --git a/include/tensor_filter_custom.h b/include/tensor_filter_custom.h index 9a978f4..60ff4a6 100644 --- a/include/tensor_filter_custom.h +++ b/include/tensor_filter_custom.h @@ -80,7 +80,7 @@ typedef void (*NNS_custom_exit_func)(void *private_data, const GstTensor_Filter_ /** * @brief Get input tensor type. - * @param[in] private_data The pointer returned by NNStreamer_custom_exit. + * @param[in] private_data The pointer returned by NNStreamer_custom_init. * @param[in] prop Tensor_Filter's property values. Do not change its values. * @param[out] inputDimension uint32_t[NNS_TENSOR_RANK_LIMIT] (tensor_dim) * @param[out] type Type of each element in the input tensor @@ -90,7 +90,7 @@ typedef int (*NNS_custom_get_input_dimension)(void *private_data, const GstTenso /** * @brief Get output tensor type. - * @param[in] private_data The pointer returned by NNStreamer_custom_exit. + * @param[in] private_data The pointer returned by NNStreamer_custom_init. * @param[in] prop Tensor_Filter's property values. Do not change its values. * @param[out] outputDimension uint32_t[NNS_TENSOR_RANK_LIMIT] (tensor_dim) * @param[out] type Type of each element in the output tensor @@ -99,8 +99,25 @@ typedef int (*NNS_custom_get_output_dimension)(void *private_data, const GstTens tensor_dim outputDimension, tensor_type *type); /** + * @brief Set input dim by framework. Let custom plutin set output dim accordingly. + * @param[in] private_data The pointer returned by NNStreamer_custom_init + * @param[in] prop Tensor_Filter's property values. Do not change its values. + * @param[in] inputDimension Input dimension designated by the gstreamer framework. Note that this is not a fixed value and gstreamer may try different values during pad-cap negotiations. + * @param[in] inputType Input element type designated by the gstreamer framework. + * @param[out] outputDimension Output dimension according to the inputDimension/type. + * @param[out] outputType Output element type according to the inputDimension/type. + * + * @caution Do not fix internal values based on this call. Gstreamer may call + * this function repeatedly with different values during pad-cap negotiations. + * Fix values when invoke is finally called. + */ +typedef int (*NNS_custom_set_input_dimension)(void *private_data, const GstTensor_Filter_Properties *prop, + const tensor_dim inputDimension, const tensor_type inputType, + tensor_dim outputDimension, tensor_type *outputType); + +/** * @brief Invoke the "main function". - * @param[in] private_data The pointer returned by NNStreamer_custom_exit. + * @param[in] private_data The pointer returned by NNStreamer_custom_init. * @param[in] prop Tensor_Filter's property values. Do not change its values. * @param[in] inputPtr pointer to input tensor, size = dim1 x dim2 x dim3 x dim4 x typesize, allocated by caller * @param[in] inputPtr pointer to output tensor, size = dim1 x dim2 x dim3 x dim4 x typesize, allocated by caller @@ -116,8 +133,9 @@ typedef int (*NNS_custom_invoke)(void *private_data, const GstTensor_Filter_Prop struct _NNStreamer_custom_class { NNS_custom_init_func initfunc; /**< called before any other callbacks from tensor_filter_custom.c */ NNS_custom_exit_func exitfunc; /**< will not call other callbacks after this call */ - NNS_custom_get_input_dimension getInputDim; /**< a custom filter is required to provide input tensor dimension */ - NNS_custom_get_output_dimension getOutputDim; /**< a custom filter is require dto provide output tensor dimension */ + NNS_custom_get_input_dimension getInputDim; /**< a custom filter is required to provide input tensor dimension unless setInputdim is defined. */ + NNS_custom_get_output_dimension getOutputDim; /**< a custom filter is require dto provide output tensor dimension unless setInputDim is defined. */ + NNS_custom_set_input_dimension setInputDim; /**< without getI/O-Dim, this allows framework to set input dimension and get output dimension from the custom filter according to the input dimension */ NNS_custom_invoke invoke; /**< the main function, "invoke", that transforms input to output */ }; typedef struct _NNStreamer_custom_class NNStreamer_custom_class; diff --git a/nnstreamer_example/custom_example_passthrough/CMakeLists.txt b/nnstreamer_example/custom_example_passthrough/CMakeLists.txt index 0f3c917..4467547 100644 --- a/nnstreamer_example/custom_example_passthrough/CMakeLists.txt +++ b/nnstreamer_example/custom_example_passthrough/CMakeLists.txt @@ -1,12 +1,17 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) ADD_LIBRARY(nnstreamer_customfilter_passthrough SHARED nnstreamer_customfilter_example_passthrough.c) +ADD_LIBRARY(nnstreamer_customfilter_passthrough_variable SHARED nnstreamer_customfilter_example_passthrough_variable.c) TARGET_LINK_LIBRARIES(nnstreamer_customfilter_passthrough ${pkgs_LIBRARIES}) TARGET_INCLUDE_DIRECTORIES(nnstreamer_customfilter_passthrough PUBLIC ${pkgs_INCLUDE_DIRS}) TARGET_COMPILE_OPTIONS(nnstreamer_customfilter_passthrough PUBLIC ${pkgs_CFLAGS_OTHER}) -INSTALL(TARGETS nnstreamer_customfilter_passthrough +TARGET_LINK_LIBRARIES(nnstreamer_customfilter_passthrough_variable ${pkgs_LIBRARIES}) +TARGET_INCLUDE_DIRECTORIES(nnstreamer_customfilter_passthrough_variable PUBLIC ${pkgs_INCLUDE_DIRS}) +TARGET_COMPILE_OPTIONS(nnstreamer_customfilter_passthrough_variable PUBLIC ${pkgs_CFLAGS_OTHER}) + +INSTALL(TARGETS nnstreamer_customfilter_passthrough nnstreamer_customfilter_passthrough_variable RUNTIME DESTINATION ${EXEC_PREFIX} LIBRARY DESTINATION ${LIB_INSTALL_DIR} ARCHIVE DESTINATION ${LIB_INSTALL_DIR} diff --git a/nnstreamer_example/custom_example_passthrough/nnstreamer_customfilter_example_passthrough.c b/nnstreamer_example/custom_example_passthrough/nnstreamer_customfilter_example_passthrough.c index af175a7..6a2e651 100644 --- a/nnstreamer_example/custom_example_passthrough/nnstreamer_customfilter_example_passthrough.c +++ b/nnstreamer_example/custom_example_passthrough/nnstreamer_customfilter_example_passthrough.c @@ -2,14 +2,14 @@ * NNStreamer Custom Filter Example 1. Pass-Through * Copyright (C) 2018 MyungJoo Ham * - * LICENSE: Apache-2.0 + * LICENSE: LGPL-2.1 * * @file nnstreamer_customfilter_example_passthrough.c * @date 11 Jun 2018 * @brief Custom NNStreamer Filter Example 1. "Pass-Through" * @author MyungJoo Ham * - * As a test, before #71 is resolved, this will support "3x280x40" uint8 tensors. + * this will supports "3x280x40" uint8 tensors (hardcoded dimensions) */ #include diff --git a/nnstreamer_example/custom_example_passthrough/nnstreamer_customfilter_example_passthrough_variable.c b/nnstreamer_example/custom_example_passthrough/nnstreamer_customfilter_example_passthrough_variable.c new file mode 100644 index 0000000..1f931e6 --- /dev/null +++ b/nnstreamer_example/custom_example_passthrough/nnstreamer_customfilter_example_passthrough_variable.c @@ -0,0 +1,83 @@ +/** + * NNStreamer Custom Filter Example 2. Pass-Through with Variable Dimensions + * Copyright (C) 2018 MyungJoo Ham + * + * LICENSE: LGPL-2.1 + * + * @file nnstreamer_customfilter_example_passthrough_variable.c + * @date 22 Jun 2018 + * @brief Custom NNStreamer Filter Example 2. "Pass-Through with Variable Dimensions" + * @author MyungJoo Ham + */ + +#include +#include +#include +#include +#include + +typedef struct _pt_data +{ + uint32_t id; /***< Just for testing */ +} pt_data; + +static void * +pt_init (const GstTensor_Filter_Properties * prop) +{ + pt_data *data = (pt_data *) malloc (sizeof (pt_data)); + + data->id = 0; + return data; +} + +static void +pt_exit (void *private_data, const GstTensor_Filter_Properties * prop) +{ + pt_data *data = private_data; + g_assert (data); + free (data); +} + +static int +set_inputDim (void *private_data, const GstTensor_Filter_Properties * prop, + const tensor_dim iDim, const tensor_type iType, + tensor_dim oDim, tensor_type * oType) +{ + int i; + + for (i = 0; i < NNS_TENSOR_RANK_LIMIT; i++) + oDim[i] = iDim[i]; + *oType = iType; + + return 0; +} + +static int +pt_invoke (void *private_data, const GstTensor_Filter_Properties * prop, + const uint8_t * inptr, uint8_t * outptr) +{ + pt_data *data = private_data; + size_t size; + + g_assert (data); + g_assert (inptr); + g_assert (outptr); + + size = get_tensor_element_count (prop->outputDimension) * + tensor_element_size[prop->outputType]; + + g_assert (inptr != outptr); + memcpy (outptr, inptr, size); + + return 0; +} + +static NNStreamer_custom_class NNStreamer_custom_body = { + .initfunc = pt_init, + .exitfunc = pt_exit, + .setInputDim = set_inputDim, + .invoke = pt_invoke, +}; + +/* The dyn-loaded object */ +NNStreamer_custom_class *NNStreamer_custom = &NNStreamer_custom_body; diff --git a/tests/nnstreamer_filter_custom/runTest.sh b/tests/nnstreamer_filter_custom/runTest.sh index 0d52ef6..70669e0 100755 --- a/tests/nnstreamer_filter_custom/runTest.sh +++ b/tests/nnstreamer_filter_custom/runTest.sh @@ -4,6 +4,7 @@ source ../testAPI.sh gstTest "videotestsrc num-buffers=1 ! video/x-raw,format=RGB,width=280,height=40,framerate=0/1 ! videoconvert ! video/x-raw, format=RGB ! filesink location=\"testcase02.apitest.log\" sync=true" 0 PATH_TO_MODEL="../../build/nnstreamer_example/custom_example_passthrough/libnnstreamer_customfilter_passthrough.so" +PATH_TO_MODEL_V="../../build/nnstreamer_example/custom_example_passthrough/libnnstreamer_customfilter_passthrough_variable.so" gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} videotestsrc num-buffers=1 ! video/x-raw,format=RGB,width=280,height=40,framerate=0/1 ! videoconvert ! video/x-raw, format=RGB ! tensor_converter ! tee name=t ! queue ! tensor_filter framework=\"custom\" model=\"${PATH_TO_MODEL}\" input=\"3:280:40\" inputtype=\"uint8\" output=\"3:280:40\" outputtype=\"uint8\" ! filesink location=\"testcase01.passthrough.log\" sync=true t. ! queue ! filesink location=\"testcase01.direct.log\" sync=true" 1 @@ -13,4 +14,14 @@ gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} videotestsrc num-buffers=1 ! video/ compareAll testcase02.direct.log testcase02.passthrough.log 2 + +gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} videotestsrc num-buffers=1 ! video/x-raw,format=RGB,width=640,height=480,framerate=0/1 ! videoconvert ! video/x-raw, format=RGB ! tensor_converter ! tee name=t ! queue ! tensor_filter framework=\"custom\" model=\"${PATH_TO_MODEL_V}\" input=\"3:640:480\" inputtype=\"uint8\" output=\"3:640:480\" outputtype=\"uint8\" ! filesink location=\"testcase03.passthrough.log\" sync=true t. ! queue ! filesink location=\"testcase03.direct.log\" sync=true" 3 + +compareAll testcase03.direct.log testcase03.passthrough.log 3 + +## @TODO there is a known bug that breaks case 4. +#gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} videotestsrc num-buffers=1 ! video/x-raw,format=RGB,width=640,height=480,framerate=0/1 ! videoconvert ! video/x-raw, format=RGB ! tensor_converter ! tee name=t ! queue ! tensor_filter framework=\"custom\" model=\"${PATH_TO_MODEL_V}\" ! filesink location=\"testcase04.passthrough.log\" sync=true t. ! queue ! filesink location=\"testcase04.direct.log\" sync=true" 4 + +#compareAll testcase04.direct.log testcase04.passthrough.log 4 + report