[Filter] Flexible input/output dimension, prep data structure

In order to allow flexible input/output dimension for subplugins,
prepare proper data structure and virtual method, "setInputDimension

Additional mod: constify Gst_Tensor_Filter for subplugins

CC: @jinhyuck83-park @hello-ahn : I need your opinion on this vmethod update.
Preparing #71

Signed-off-by: MyungJoo Ham <myungjoo.ham@samsung.com>

diff --git a/gst/tensor_filter/tensor_filter.c b/gst/tensor_filter/tensor_filter.c
index 38ba13f..7c3c2bd 100644 (file)
--- a/gst/tensor_filter/tensor_filter.c
+++ b/gst/tensor_filter/tensor_filter.c
@@ -412,6 +412,10 @@ gst_tensor_filter_set_property (GObject * object, guint prop_id,
       g_assert (nnfw_support_status[filter->prop.nnfw] == TRUE);
       filter->prop.fw = tensor_filter_supported[filter->prop.nnfw];
       g_assert (filter->prop.fw != NULL);
+
+      /* See if mandatory methods are filled in */
+      g_assert (filter->prop.fw->invoke_NN);
+      g_assert (!(filter->prop.fw->getInputDimension && filter->prop.fw->getOutputDimension) != !filter->prop.fw->setInputDimension);   /* This is "XOR" */
       break;
     case PROP_MODEL:
       g_assert (filter->prop.modelFilename == NULL && value);
@@ -611,18 +615,25 @@ gst_tensor_filter_transform (GstBaseTransform * trans,
       filter->prop.fw->name, filter->prop.modelFilename);
 
   if (filter->prop.fw->getInputDimension) {
-    ret = filter->prop.fw->getInputDimension (filter, inputDimChk, &inputType);
+    ret =
+        filter->prop.fw->getInputDimension (filter, &filter->privateData,
+        inputDimChk, &inputType);
     /* @TODO check inputDimChk / inputType with filter internal info */
   } else {
-    /* @TODO printout debug msg */
+    /* Input dimension is determined by the pad-cap negotiation */
+
+    g_assert (FALSE);           /* @TODO NYI */
   }
 
   if (filter->prop.fw->getOutputDimension) {
     ret =
-        filter->prop.fw->getOutputDimension (filter, outputDimChk, &outputType);
+        filter->prop.fw->getOutputDimension (filter, &filter->privateData,
+        outputDimChk, &outputType);
     /* @TODO check outputDimChk / outputType with filter internal info */
   } else {
-    /* @TODO printout debug msg */
+    /* Output dimension is determined by the invokation or input dimension */
+
+    g_assert (FALSE);           /* @TODO NYI */
   }
 
   /* 1. Allocate outbuf */
@@ -640,7 +651,8 @@ gst_tensor_filter_transform (GstBaseTransform * trans,
   inptr = inInfo.data;
   outptr = outInfo.data;
 
-  ret = filter->prop.fw->invoke_NN (filter, inptr, outptr);
+  ret =
+      filter->prop.fw->invoke_NN (filter, &filter->privateData, inptr, outptr);
 
   gst_buffer_unmap (inbuf, &inInfo);
   gst_buffer_unmap (outbuf, &outInfo);

diff --git a/gst/tensor_filter/tensor_filter.h b/gst/tensor_filter/tensor_filter.h
index 10f9561..0b3ed0a 100644 (file)
--- a/gst/tensor_filter/tensor_filter.h
+++ b/gst/tensor_filter/tensor_filter.h
@@ -110,7 +110,15 @@ struct _GstTensor_Filter
   GstTensor_Filter_Properties prop;
 };
 
-/*
+/** @brief Location of GstTensor_Filter from privateData
+ *  @param p the "privateData" pointer of GstTensor_Filter
+ *  @return the pointer to GstTensor_Filter containing p as privateData
+ */
+#define GstTensor_Filter_of_privateData(p) ({ \
+    const typeof( ((GstTensor_Filter *)0)->privateData ) *__mptr = (p); \
+    (GstTensor_Filter *)( (char *)__mptr - offsetof(GstTensor_Filter, privateData) );})
+
+/**
  * @brief GstTensor_FilterClass inherits GstBaseTransformClass.
  *
  * Referring another child (sibiling), GstVideoFilter (abstract class) and
@@ -122,19 +130,46 @@ struct _GstTensor_FilterClass
   GstBaseTransformClass parent_class;  /**< Inherits GstBaseTransformClass */
 };
 
-/*
+/**
  * @brief Get Type function required for gst elements
  */
 GType gst_tensor_filter_get_type (void);
 
+/**
+ * @brief Subplugin definition
+ *
+ * Common callback parameters:
+ * filter Filter properties. Read Only
+ * private_data Subplugin's private data. Set this (*private_data = XXX) if you want to change filter->private_data
+ */
 struct _GstTensor_Filter_Framework
 {
   gchar *name; /**< Name of the neural network framework, searchable by FRAMEWORK property */
   gboolean allow_in_place; /**< TRUE if InPlace transfer of input-to-output is allowed. Not supported in main, yet */
-  int (*invoke_NN)(GstTensor_Filter *filter, const uint8_t *inputptr, uint8_t *outputptr); /**< Mandatory callback. Invoke the given network model. */
-  int (*getInputDimension)(GstTensor_Filter *filter, tensor_dim inputDimension, tensor_type *type); /**< Optional. Set NULL if not supported. Get dimension of input tensor */
-  int (*getOutputDimension)(GstTensor_Filter *filter, tensor_dim outputDimension, tensor_type *type); /**< Optional. Set NULL if not supported. Get dimension of output tensor */
-  void (*close)(GstTensor_Filter *filter); /**< Optional. Close this instance! */
+  int (*invoke_NN)(const GstTensor_Filter *filter, void **private_data, const uint8_t *inputptr, uint8_t *outputptr);
+      /**< Mandatory callback. Invoke the given network model. */
+
+  int (*getInputDimension)(const GstTensor_Filter *filter, void **private_data, tensor_dim inputDimension, tensor_type *type);
+      /**< Optional. Set NULL if not supported. Get dimension of input tensor
+       * If getInputDimension is NULL, setInputDimension must be defined.
+       * However, one of the two must be NULL.
+       * And, if getInputDimension != NULL, getOutputDimension != NULL.
+       */
+  int (*getOutputDimension)(const GstTensor_Filter *filter, void **private_data, tensor_dim outputDimension, tensor_type *type);
+      /**< Optional. Set NULL if not supported. Get dimension of output tensor
+       * If getOutputDimension is NULL, setInputDimension must be defined.
+       * However, one of the two must be NULL
+       * And, if getOutputDimension != NULL, getInputDimension != NULL.
+       */
+  int (*setInputDimension)(const GstTensor_Filter *filter, void **private_data, const tensor_dim inputDimension, tensor_type inputType, tensor_dim outputDimension, tensor_type *outputType);
+      /**< Optional. Set Null if not supported. Tensor_filter::main will
+       * configure input dimension from pad-cap in run-time for the sub-plugin.
+       * Then, the sub-plugin is required to return corresponding output dimension
+       * If this is NULL, both getInput/OutputDimension must be non-NULL.
+       * If this is non-NULL, both getInput/OutputDimension must be NULL.
+       */
+
+  void (*close)(const GstTensor_Filter *filter, void **private_data); /**< Optional. Close this instance! Free-ing private_data is this function's responsibility. Set NULL after that. */
 };
 
 extern GstTensor_Filter_Framework NNS_support_tensorflow_lite;

diff --git a/gst/tensor_filter/tensor_filter_custom.c b/gst/tensor_filter/tensor_filter_custom.c
index ed9a771..4a5a2e1 100644 (file)
--- a/gst/tensor_filter/tensor_filter_custom.c
+++ b/gst/tensor_filter/tensor_filter_custom.c
@@ -73,7 +73,7 @@ typedef struct _internal_data internal_data;
  * @return 0 if successfully loaded. 1 if skipped (already loaded). -1 if error
  */
 static int
-custom_loadlib (GstTensor_Filter * filter)
+custom_loadlib (const GstTensor_Filter * filter, void **private_data)
 {
   internal_data *ptr;
   char *dlsym_error;
@@ -84,14 +84,15 @@ custom_loadlib (GstTensor_Filter * filter)
   }
 
   ptr = g_new0 (internal_data, 1);      /* Fill Zero! */
-  filter->privateData = ptr;
-  ptr->parent = filter;
+  *private_data = ptr;
+  g_assert (filter->privateData == ptr);
+  ptr->parent = GstTensor_Filter_of_privateData (private_data);
 
   /* Load .so if this is the first time for this instance. */
   ptr->handle = dlopen (filter->prop.modelFilename, RTLD_NOW);
   if (!ptr->handle) {
     g_free (ptr);
-    filter->privateData = NULL;
+    *private_data = NULL;
     return -1;
   }
 
@@ -103,7 +104,7 @@ custom_loadlib (GstTensor_Filter * filter)
     err_print ("tensor_filter_custom:loadlib error: %s\n", dlsym_error);
     dlclose (ptr->handle);
     g_free (ptr);
-    filter->privateData = NULL;
+    *private_data = NULL;
     return -1;
   }
 
@@ -119,10 +120,10 @@ custom_loadlib (GstTensor_Filter * filter)
  * @param[out] outptr The output tensor
  */
 static int
-custom_invoke (GstTensor_Filter * filter, const uint8_t * inptr,
-    uint8_t * outptr)
+custom_invoke (const GstTensor_Filter * filter, void **private_data,
+    const uint8_t * inptr, uint8_t * outptr)
 {
-  int retval = custom_loadlib (filter);
+  int retval = custom_loadlib (filter, private_data);
   internal_data *ptr;
 
   /* Actually, tensor_filter must have called getInput/OotputDim first. */
@@ -131,8 +132,8 @@ custom_invoke (GstTensor_Filter * filter, const uint8_t * inptr,
   if (retval < 0)
     return retval;
 
-  g_assert (filter->privateData);
-  ptr = filter->privateData;
+  g_assert (filter->privateData && *private_data == filter->privateData);
+  ptr = *private_data;
 
   return ptr->methods->invoke (ptr->customFW_private_data, &(filter->prop),
       inptr, outptr);
@@ -142,17 +143,17 @@ custom_invoke (GstTensor_Filter * filter, const uint8_t * inptr,
  * @brief The optional callback for GstTensor_Filter_Framework
  */
 static int
-custom_getInputDim (GstTensor_Filter * filter, tensor_dim inputDimension,
-    tensor_type * type)
+custom_getInputDim (const GstTensor_Filter * filter, void **private_data,
+    tensor_dim inputDimension, tensor_type * type)
 {
-  int retval = custom_loadlib (filter);
+  int retval = custom_loadlib (filter, private_data);
   internal_data *ptr;
 
   if (retval < 0)
     return retval;
 
-  g_assert (filter->privateData);
-  ptr = filter->privateData;
+  g_assert (filter->privateData && *private_data == filter->privateData);
+  ptr = *private_data;
 
   return ptr->methods->getInputDim (ptr->customFW_private_data, &(filter->prop),
       inputDimension, type);
@@ -162,17 +163,17 @@ custom_getInputDim (GstTensor_Filter * filter, tensor_dim inputDimension,
  * @brief The optional callback for GstTensor_Filter_Framework
  */
 static int
-custom_getOutputDim (GstTensor_Filter * filter, tensor_dim outputDimension,
-    tensor_type * type)
+custom_getOutputDim (const GstTensor_Filter * filter, void **private_data,
+    tensor_dim outputDimension, tensor_type * type)
 {
-  int retval = custom_loadlib (filter);
+  int retval = custom_loadlib (filter, private_data);
   internal_data *ptr;
 
   if (retval < 0)
     return retval;
 
-  g_assert (filter->privateData);
-  ptr = filter->privateData;
+  g_assert (filter->privateData && *private_data == filter->privateData);
+  ptr = *private_data;
 
   return ptr->methods->getOutputDim (ptr->customFW_private_data,
       &(filter->prop), outputDimension, type);
@@ -182,13 +183,14 @@ custom_getOutputDim (GstTensor_Filter * filter, tensor_dim outputDimension,
  * @brief Free privateData and move on.
  */
 static void
-custom_close (GstTensor_Filter * filter)
+custom_close (const GstTensor_Filter * filter, void **private_data)
 {
-  internal_data *ptr = filter->privateData;
+  internal_data *ptr = *private_data;
 
   ptr->methods->exitfunc (ptr->customFW_private_data, &(filter->prop));
   g_free (ptr);
-  filter->privateData = NULL;
+  *private_data = NULL;
+  g_assert (filter->privateData == NULL);
 }
 
 GstTensor_Filter_Framework NNS_support_custom = {

diff --git a/gst/tensor_filter/tensor_filter_tensorflow_lite.c b/gst/tensor_filter/tensor_filter_tensorflow_lite.c
index 75f07f3..9a03003 100644 (file)
--- a/gst/tensor_filter/tensor_filter_tensorflow_lite.c
+++ b/gst/tensor_filter/tensor_filter_tensorflow_lite.c
@@ -58,8 +58,8 @@
  * @brief The mandatory callback for GstTensor_Filter_Framework
  */
 static int
-tflite_invoke (GstTensor_Filter * filter, const uint8_t * inptr,
-    uint8_t * outptr)
+tflite_invoke (const GstTensor_Filter * filter, void **private_data,
+    const uint8_t * inptr, uint8_t * outptr)
 {
   return 0;                     /* NYI */
 }
@@ -68,8 +68,8 @@ tflite_invoke (GstTensor_Filter * filter, const uint8_t * inptr,
  * @brief The optional callback for GstTensor_Filter_Framework
  */
 static int
-tflite_getInputDim (GstTensor_Filter * filter, tensor_dim inputDimension,
-    tensor_type * type)
+tflite_getInputDim (const GstTensor_Filter * filter, void **private_data,
+    tensor_dim inputDimension, tensor_type * type)
 {
   /* @TODO fill in *inputDimension (uint32_t[MAX_RANK]), *type */
   return 0;                     // NYI
@@ -79,8 +79,8 @@ tflite_getInputDim (GstTensor_Filter * filter, tensor_dim inputDimension,
  * @brief The optional callback for GstTensor_Filter_Framework
  */
 static int
-tflite_getOutputDim (GstTensor_Filter * filter, tensor_dim outputDimension,
-    tensor_type * type)
+tflite_getOutputDim (const GstTensor_Filter * filter, void **private_data,
+    tensor_dim outputDimension, tensor_type * type)
 {
   /* @TODO fill in *outputDimension (uint32_t[MAX_RANK]), *type */
   return 0;                     /* NYI */