[Repo] Add checking caps compatibility between repo_src & repo_sink

It is difficult to check caps compatibility during negotiation because
they are not pipelined each other. That's why reposrc needs caps
property by the way. However, we have to check compatibility somehow
during runtime. In order to do that, GstMetaRepo is introduced.
In this PR, functions and defines are included to manipulate the meta
for repo. During generating buffer in reposink, insert GstMetaRepo
data with GstCaps and push to repo. After that, reposrc extracts the
GstMetaRepo when pulling the gstbuffer from repo
and compare it with caps defined by user. If they are not compatible,
it shows error and return GST_FLOW_EOS.

**Changes proposed in this PR:**
- Add GstMetaRepo type and manipulation functions
- Add insert GstMetaRepo into buffer
- Add comparison routine during get gstbuffer from repo
- Add Error handling if they are not compatible

Resolves: Issue #1026

**Self evaluation:**
1. Build test:	 [X]Passed [ ]Failed [ ]Skipped
2. Run test:	 [X]Passed [ ]Failed [ ]Skipped

Signed-off-by: jijoong.moon <jijoong.moon@samsung.com>

diff --git a/gst/nnstreamer/tensor_repo.c b/gst/nnstreamer/tensor_repo.c
index 8e80288..98a6ffa 100644 (file)
--- a/gst/nnstreamer/tensor_repo.c
+++ b/gst/nnstreamer/tensor_repo.c
@@ -36,6 +36,95 @@
 GstTensorRepo _repo = {.num_data = 0,.initialized = FALSE };
 
 /**
+ * @brief Define tensor_repo meta data type to register
+ */
+GType
+gst_meta_repo_api_get_type (void)
+{
+  static volatile GType type;
+  static const gchar *tags[] = { "tensor", "tensors", NULL };
+  if (g_once_init_enter (&type)) {
+    GType _type;
+    const GstMetaInfo *meta_info = gst_meta_get_info ("GstMetaRepo");
+    if (meta_info) {
+      _type = meta_info->api;
+    } else {
+      _type = gst_meta_api_type_register ("GstMetaRepoAPI", tags);
+    }
+    g_once_init_leave (&type, _type);
+  }
+  return type;
+}
+
+/**
+ * @brief tensor_repo meta data init
+ */
+static gboolean
+gst_meta_repo_init (GstMeta * meta, gpointer params, GstBuffer * buffer)
+{
+  GstMetaRepo *emeta = (GstMetaRepo *) meta;
+  emeta->caps = NULL;
+  return TRUE;
+}
+
+/**
+ * @brief tensor_repo meta data transform (source to dest)
+ */
+static gboolean
+gst_meta_repo_transform (GstBuffer * transbuf, GstMeta * meta,
+    GstBuffer * buffer, GQuark type, gpointer data)
+{
+  GstMetaRepo *dest_meta = GST_META_REPO_ADD (transbuf);
+  GstMetaRepo *src_meta = (GstMetaRepo *) meta;
+  dest_meta->caps = src_meta->caps;
+  return TRUE;
+}
+
+/**
+ * @brief tensor_repo meta data free
+ */
+static void
+gst_meta_repo_free (GstMeta * meta, GstBuffer * buffer)
+{
+  GstMetaRepo *emeta = (GstMetaRepo *) meta;
+  emeta->caps = NULL;
+}
+
+/**
+ * @brief tensor_repo meta data info
+ * @return GstMetaInfo
+ */
+const GstMetaInfo *
+gst_meta_repo_get_info (void)
+{
+  static const GstMetaInfo *meta_info = NULL;
+  if (g_once_init_enter (&meta_info)) {
+    const GstMetaInfo *mi = gst_meta_register (GST_META_REPO_API_TYPE,
+        "GstMetaRepo",
+        sizeof (GstMetaRepo),
+        (GstMetaInitFunction) gst_meta_repo_init,
+        (GstMetaFreeFunction) gst_meta_repo_free,
+        (GstMetaTransformFunction) gst_meta_repo_transform);
+    g_once_init_leave (&meta_info, mi);
+  }
+  return meta_info;
+}
+
+/**
+ * @brief Add tensor_repo meta format in gstbuffer
+ * @return GstMetaRepo *
+ */
+GstMetaRepo *
+gst_buffer_add_meta_repo (GstBuffer * buffer)
+{
+  GstMetaRepo *meta;
+  g_return_val_if_fail (GST_IS_BUFFER (buffer), NULL);
+  meta = (GstMetaRepo *) gst_buffer_add_meta (buffer, GST_META_REPO_INFO, NULL);
+  return meta;
+}
+
+
+/**
  * @brief getter to get nth GstTensorRepoData
  */
 GstTensorRepoData *
@@ -86,17 +175,27 @@ gst_tensor_repo_add_repodata (guint nth)
  * @brief push GstBuffer into repo
  */
 gboolean
-gst_tensor_repo_set_buffer (guint nth, GstBuffer * buffer)
+gst_tensor_repo_set_buffer (guint nth, GstBuffer * buffer, GstCaps * caps)
 {
   GST_TENSOR_REPO_LOCK (nth);
 
   GstTensorRepoData *data = gst_tensor_repo_get_repodata (nth);
 
+  if (data->eos) {
+    GST_TENSOR_REPO_UNLOCK (nth);
+    return FALSE;
+  }
+
   while (data->buffer != NULL) {
     GST_TENSOR_REPO_WAIT_PULL (nth);
   }
 
-  data->buffer = buffer;
+  data->buffer = gst_buffer_copy (buffer);
+
+  GstMetaRepo *meta = GST_META_REPO_ADD (data->buffer);
+
+  gst_caps_replace (&meta->caps, caps);
+
   if (DBG) {
     unsigned long size = gst_buffer_get_size (data->buffer);
     GST_DEBUG ("Pushed [%d] (size : %lu)\n", nth, size);

diff --git a/gst/nnstreamer/tensor_repo.h b/gst/nnstreamer/tensor_repo.h
index 3496c79..5d2740d 100644 (file)
--- a/gst/nnstreamer/tensor_repo.h
+++ b/gst/nnstreamer/tensor_repo.h
@@ -32,6 +32,44 @@
 #include <gst/gst.h>
 
 G_BEGIN_DECLS
+/**
+ * @brief GstTensorRepo meta structure.
+ */
+typedef struct
+{
+  GstMeta meta;
+  GstCaps *caps;
+} GstMetaRepo;
+
+/**
+ * @brief Define tensor_repo meta data type to register & macro
+ */
+GType gst_meta_repo_api_get_type (void);
+#define GST_META_REPO_API_TYPE (gst_meta_repo_api_get_type())
+
+/**
+ * @brief get tensor_repo meta data info & macro
+ */
+const GstMetaInfo *gst_meta_repo_get_info (void);
+#define GST_META_REPO_INFO ((GstMetaInfo*) gst_meta_repo_get_info())
+
+/**
+ * @brief macro of get_tensor_repo meta data
+ */
+#define gst_buffer_get_meta_repo(b) \
+  ((GstMetaRepo*) gst_buffer_get_meta((b), GST_META_REPO_API_TYPE))
+
+/**
+ * @brief add get_tensor_repo meta data in buffer
+ */
+GstMetaRepo *gst_buffer_add_meta_repo (GstBuffer * buffer);
+
+/**
+ * @brief Macro of get & add meta
+ */
+#define GST_META_REPO_GET(buf) ((GstMetaRepo*) gst_buffer_get_meta_repo(buf))
+#define GST_META_REPO_ADD(buf) ((GstMetaRepo*) gst_buffer_add_meta_repo(buf))
+
 
 /**
  * @brief GstTensorRepo internal data structure.
@@ -78,7 +116,7 @@ gst_tensor_repo_add_repodata (guint myid);
  * @brief push GstBuffer into repo
  */
 gboolean
-gst_tensor_repo_set_buffer (guint nth, GstBuffer * buffer);
+gst_tensor_repo_set_buffer (guint nth, GstBuffer * buffer, GstCaps * caps);
 
 /**
  * @brief get EOS

diff --git a/gst/nnstreamer/tensor_reposink/tensor_reposink.c b/gst/nnstreamer/tensor_reposink/tensor_reposink.c
index e1e8f37..03a158e 100644 (file)
--- a/gst/nnstreamer/tensor_reposink/tensor_reposink.c
+++ b/gst/nnstreamer/tensor_reposink/tensor_reposink.c
@@ -334,7 +334,7 @@ gst_tensor_reposink_render_buffer (GstTensorRepoSink * self, GstBuffer * buffer)
   if (notify) {
     gboolean ret = FALSE;
     self->last_render_time = now;
-    ret = gst_tensor_repo_set_buffer (self->myid, buffer);
+    ret = gst_tensor_repo_set_buffer (self->myid, buffer, self->in_caps);
     if (!ret)
       GST_ELEMENT_ERROR (self, RESOURCE, WRITE,
           ("Cannot Set buffer into repo [key: %d]", self->myid), NULL);

diff --git a/gst/nnstreamer/tensor_reposrc/tensor_reposrc.c b/gst/nnstreamer/tensor_reposrc/tensor_reposrc.c
index 11255ee..3c9c275 100644 (file)
--- a/gst/nnstreamer/tensor_reposrc/tensor_reposrc.c
+++ b/gst/nnstreamer/tensor_reposrc/tensor_reposrc.c
@@ -128,6 +128,7 @@ gst_tensor_reposrc_init (GstTensorRepoSrc * self)
 {
   self->silent = TRUE;
   self->ini = FALSE;
+  self->negotiation = FALSE;
   gst_tensors_config_init (&self->config);
   self->caps = NULL;
 }
@@ -210,6 +211,7 @@ gst_tensor_reposrc_set_property (GObject * object, guint prop_id,
       break;
     case PROP_SLOT_ID:
       self->myid = g_value_get_uint (value);
+      self->negotiation = FALSE;
       break;
     case PROP_CAPS:
     {
@@ -235,6 +237,7 @@ gst_tensor_reposrc_set_property (GObject * object, guint prop_id,
         self->fps_n = -1;
         self->fps_d = -1;
       }
+      self->negotiation = FALSE;
     }
       break;
     default:
@@ -298,8 +301,26 @@ gst_tensor_reposrc_create (GstPushSrc * src, GstBuffer ** buffer)
     self->ini = TRUE;
   } else {
     buf = gst_tensor_repo_get_buffer (self->myid);
+
     if (buf == NULL)
       return GST_FLOW_EOS;
+
+    GstMetaRepo *meta = GST_META_REPO_GET (buf);
+
+    if (!self->negotiation) {
+      if (!gst_caps_can_intersect (self->caps, meta->caps)) {
+        GST_ELEMENT_ERROR (GST_ELEMENT (self), CORE, NEGOTIATION,
+            ("Negotiation Failed! : repo_sink & repos_src"), (NULL));
+        gst_buffer_remove_meta (buf, (GstMeta *) meta);
+        gst_buffer_unref (buf);
+        gst_tensor_repo_set_eos (self->myid);
+        return GST_FLOW_EOS;
+      }
+
+      self->negotiation = TRUE;
+    }
+
+    gst_buffer_remove_meta (buf, (GstMeta *) meta);
   }
 
   *buffer = buf;

diff --git a/gst/nnstreamer/tensor_reposrc/tensor_reposrc.h b/gst/nnstreamer/tensor_reposrc/tensor_reposrc.h
index 5bdf6fa..748bdde 100644 (file)
--- a/gst/nnstreamer/tensor_reposrc/tensor_reposrc.h
+++ b/gst/nnstreamer/tensor_reposrc/tensor_reposrc.h
@@ -59,6 +59,7 @@ struct _GstTensorRepoSrc
   GstCaps *caps;
   gboolean ini;
   gint fps_n, fps_d;
+  gboolean negotiation;
 };
 
 /**

diff --git a/tests/nnstreamer_repo_lstm/runTest.sh b/tests/nnstreamer_repo_lstm/runTest.sh
index 76c24f7..d6bf8c2 100755 (executable)
--- a/tests/nnstreamer_repo_lstm/runTest.sh
+++ b/tests/nnstreamer_repo_lstm/runTest.sh
@@ -38,7 +38,7 @@ testInit $1 # You may replace this with Test Group Name
 # Generate video_4x4xBGRx.xraw & golden
 python generateTestCase.py
 
-gstTest "--gst-plugin-path=../../build tensor_mux name=mux ! tensor_filter framework=custom model=../../build/nnstreamer_example/custom_example_LSTM/libdummyLSTM.so ! tensor_demux name=demux ! queue ! tensor_reposink slot-index=0 silent=false demux.src_1 ! queue ! tee name=t ! queue ! tensor_reposink slot-index=1 silent=false tensor_reposrc slot-index=0 silent=false caps=\"other/tensor, dim1=4, dim2=4, dim3=4, dim4=1, type=float32, framerate=30/1\" ! mux.sink_0 tensor_reposrc slot-index=1 silent=false caps=\"other/tensor, dim1=4, dim2=4, dim3=4, dim4=1, type=float32, framerate=30/1\" ! mux.sink_1 filesrc location=\"video_4x4xBGRx.xraw\" ! application/octet-stream ! tensor_converter input-dim=4:4:4:1 input-type=float32 ! mux.sink_2 t. ! queue ! multifilesink location=\"out_%1d.log\"" 1 0 0 $PERFORMANCE
+gstTest "--gst-plugin-path=../../build tensor_mux name=mux ! tensor_filter framework=custom model=../../build/nnstreamer_example/custom_example_LSTM/libdummyLSTM.so ! tensor_demux name=demux ! queue ! tensor_reposink slot-index=0 silent=false demux.src_1 ! queue ! tee name=t ! queue ! tensor_reposink slot-index=1 silent=false tensor_reposrc slot-index=0 silent=false caps=\"other/tensor, dim1=4, dim2=4, dim3=4, dim4=1, type=float32\" ! mux.sink_0 tensor_reposrc slot-index=1 silent=false caps=\"other/tensor, dim1=4, dim2=4, dim3=4, dim4=1, type=float32\" ! mux.sink_1 filesrc location=\"video_4x4xBGRx.xraw\" ! application/octet-stream ! tensor_converter input-dim=4:4:4:1 input-type=float32 ! mux.sink_2 t. ! queue ! multifilesink location=\"out_%1d.log\"" 1 0 0 $PERFORMANCE
 
 callCompareTest lstm.golden out_9.log 1-1 "Compare 1-1" 1 0
 

diff --git a/tests/nnstreamer_repo_rnn/runTest.sh b/tests/nnstreamer_repo_rnn/runTest.sh
index 3a8c875..299421c 100755 (executable)
--- a/tests/nnstreamer_repo_rnn/runTest.sh
+++ b/tests/nnstreamer_repo_rnn/runTest.sh
@@ -20,7 +20,7 @@ testInit $1 # You may replace this with Test Group Name
 # Generate video_4x4xBGRx.xraw
 python generateTestCase.py
 
-gstTest "--gst-plugin-path=../../build tensor_mux name=mux ! tensor_filter framework=custom model=../../build/nnstreamer_example/custom_example_RNN/libdummyRNN.so ! tee name=t ! queue ! tensor_reposink slot-index=0 silent=false filesrc location=\"video_4x4xBGRx.xraw\" ! application/octet-stream ! tensor_converter input-dim=4:4:4:1 input-type=uint8 ! mux.sink_0 tensor_reposrc slot-index=0 silent=false caps=\"other/tensor, dim1=4, dim2=4, dim3=4, dim4=1, type=uint8, framerate=30/1\" ! mux.sink_1 t. ! queue ! multifilesink location=\"out_%1d.log\"" 1 0 0 $PERFORMANCE
+gstTest "--gst-plugin-path=../../build tensor_mux name=mux ! tensor_filter framework=custom model=../../build/nnstreamer_example/custom_example_RNN/libdummyRNN.so ! tee name=t ! queue ! tensor_reposink slot-index=0 silent=false filesrc location=\"video_4x4xBGRx.xraw\" ! application/octet-stream ! tensor_converter input-dim=4:4:4:1 input-type=uint8 ! mux.sink_0 tensor_reposrc slot-index=0 silent=false caps=\"other/tensor, dim1=4, dim2=4, dim3=4, dim4=1, type=uint8\" ! mux.sink_1 t. ! queue ! multifilesink location=\"out_%1d.log\"" 1 0 0 $PERFORMANCE
 
 callCompareTest rnn.golden out_9.log 1-1 "Compare 1-1" 1 0