[Tensor-IF] An anchor for further development.

Anyone can jump in and start the development!

Signed-off-by: MyungJoo Ham <myungjoo.ham@samsung.com>

diff --git a/gst/nnstreamer/tensor_if/gsttensorif.c b/gst/nnstreamer/tensor_if/gsttensorif.c
new file mode 100644 (file)
index 0000000..4fa9683
--- /dev/null
+++ b/gst/nnstreamer/tensor_if/gsttensorif.c
@@ -0,0 +1,132 @@
+/* SPDX-License-Identifier: LGPL-2.1-only */
+/**
+ * GStreamer/NNStreamer Tensor-IF
+ * Copyright (C) 2020 MyungJoo Ham <myungjoo.ham@samsung.com>
+ */
+/**
+ * @file       gsttensorif.c
+ * @date       08 April 2020
+ * @brief      GStreamer plugin to control flow based on tensor values
+ * @see                https://github.com/nnstreamer/nnstreamer
+ * @author     MyungJoo Ham <myungjoo.ham@samsung.com>
+ * @bug                No known bugs except for NYI items
+ */
+
+/**
+ * SECTION:element-tensor_if
+ *
+ * A filter that controls its src-pad based on the values (other/tensor(s))
+ * of its sink-pad.
+ * For example, you may skip frames if there is no object detected with
+ * high confidence.
+ *
+ * The format of statement with tensor-if is:
+ * if (Compared_Value OPERATOR Supplied_Value(s))) then THEN else ELSE
+ * Compared_Value and Supplied_Value are the operands.
+ * Compared_Value is a value from input tensor(s).
+ * SUpplied_Value is a value from tensor-if properties.
+ *
+ * If the given if-condition is simple enough (e.g., if a specific element
+ * is between a given range in a tensor frame), it can be expressed as:
+ * <refsect2>
+ * <title>Example launch line with simple if condition</title>
+ * gst-launch ... (some tensor stream) !
+ *      tensor_if compared_value=A_VALUE compared_value_option=3:4:2:5,0
+ *      operator=RANGE_INCLUSIVE
+ *      supplied_values=10,100
+ *      then=PASSTHROUGH
+ *      else=FILL_WITH_FILE else_option=${path_to_file}
+ *    ! (tensor stream) ...
+ * </refsect2>
+ *
+ * However, if the if-condition is complex and cannot be expressed with
+ * tensor-if expressions, you may create a corresponding custom filter
+ * with tensor-filter, whose output is other/tensors with an additional tensor
+ * that is "1:1:1:1, uint8", which is 1 (true) or 0 (false) as the
+ * first tensor of other/tensors and the input tensor/tensors.
+ * Then, you can create a pipeline as follows:
+ * <refsect2>
+ * <title>Example launch line with complex if condition</title>
+ * gst-launch ... (some tensor stream)
+ *      ! tensor_filter framework=custom name=your_code.so
+ *      ! tensor_if compared_value=A_VALUE
+ *          compared_value_option=0:0:0:0,0 # 1st tensor's [0][0][0][0].
+ *          operator=EQ
+ *          supplied_values=1
+ *          then=PASSTHROUGH # or whatsoever you want
+ *          else=SKIP # or whatsoever you want
+ *      ! tensor_demux name=d
+ *        d.src_0 ! queue ! fakesink # throw away the 1/0 value.
+ *        d.src_1 ! queue ! do whatever you want here...
+ *        ...
+ * </refsect2>
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <nnstreamer_log.h>
+
+#include "gsttensorif.h"
+
+/**
+ * @brief tensor_if properties
+ */
+enum
+{
+  PROP_0,
+  PROP_SILENT,
+  PROP_CV, /**< Compared Value, operand 1 (from input tensor(s)) */
+  PROP_CV_OPTION, /**< Compared Value Option */
+  PROP_OP, /**< Operator */
+  PROP_SV, /**< Supplied Value, operand 2 (from the properties) */
+  PROP_SV_OPTION, /**< Supplied Value Option */
+  PROP_THEN, /**< Action if it is TRUE */
+  PROP_THEN_OPTION, /**< Option for TRUE Action */
+  PROP_ELSE, /**< Action if it is FALSE */
+  PROP_ELSE_OPTION, /**< Option for FALSE Action */
+};
+
+#define CAPS_STRING GST_TENSOR_CAP_DEFAULT "; " GST_TENSORS_CAP_DEFAULT
+/**
+ * @brief The capabilities of the inputs
+ */
+static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink",
+    GST_PAD_SINK,
+    GST_PAD_ALWAYS,
+    GST_STATIC_CAPS (CAPS_STRING));
+
+/**
+ * @brief The capabilities of the outputs
+ */
+static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src",
+    GST_PAD_SRC,
+    GST_PAD_ALWAYS,
+    GST_STATIC_CAPS (CAPS_STRING));
+
+#define gst_tensor_if_parent_class parent_class
+G_DEFINE_TYPE (GstTensorIf, gst_tensor_if, GST_TYPE_BASE_TRANSFORM);
+
+/* GObject vmethod implementations */
+static void gst_tensor_if_set_property (GObject * object, guint prop_id,
+    const GValue * value, GParamSpec * pspec);
+static void gst_tensor_if_get_property (GObject * object, guint prop_id,
+    GValue * value, GParamSpec * pspec);
+static void gst_tensor_if_finalize (GObject * object);
+
+/* GstBaseTransform vmethod implementations */
+static GstFlowReturn gst_tensor_if_transform (GstBaseTransform * trans,
+    GstBuffer * inbuf, GstBuffer * outbuf);
+static GstCaps *gst_tensor_if_transform_caps (GstBaseTransform * trans,
+    GstPadDirection direction, GstCaps * caps, GstCaps * _if);
+static GstCaps *gst_tensor_if_fixate_caps (GstBaseTransform * trans,
+    GstPadDirection direction, GstCaps * caps, GstCaps * othercaps);
+static gboolean gst_tensor_if_set_caps (GstBaseTransform * trans,
+    GstCaps * incaps, GstCaps * outcaps);
+static gboolean gst_tensor_if_transform_size (GstBaseTransform * trans,
+    GstPadDirection direction, GstCaps * caps, gsize size,
+    GstCaps * othercaps, gsize * othersize);
+static gboolean gst_tensor_if_start (GstBaseTransform * trans);
+static gboolean gst_tensor_if_stop (GstBaseTransform * trans);
+static gboolean gst_tensor_if_sink_event (GstBaseTransform * trans,
+    GstEvent * event);

diff --git a/gst/nnstreamer/tensor_if/gsttensorif.h b/gst/nnstreamer/tensor_if/gsttensorif.h
new file mode 100644 (file)
index 0000000..a78240d
--- /dev/null
+++ b/gst/nnstreamer/tensor_if/gsttensorif.h
@@ -0,0 +1,115 @@
+/* SPDX-License-Identifier: LGPL-2.1-only */
+/**
+ * GStreamer/NNStreamer Tensor-IF
+ * Copyright (C) 2020 MyungJoo Ham <myungjoo.ham@samsung.com>
+ */
+/**
+ * @file       gsttensorif.h
+ * @date       08 April 2020
+ * @brief      GStreamer plugin to control flow based on tensor values
+ * @see                https://github.com/nnstreamer/nnstreamer
+ * @author     MyungJoo Ham <myungjoo.ham@samsung.com>
+ * @bug                No known bugs except for NYI items
+ *
+ * @todo Add "event/signal" to reload FILL_WITH_FILE* file??? (TBD)
+ *
+ * @details
+ *     The output dimension is SAME with input dimension.
+ */
+#ifndef __GST_TENSOR_IF_H__
+#define __GST_TENSOR_IF_H__
+
+#include <gst/gst.h>
+#include <tensor_common.h>
+
+G_BEGIN_DECLS
+
+#define GST_TYPE_TENSOR_IF (gst_tensor_if_get_type ())
+#define GST_TENSOR_IF(obj) (G_TYPE_CHECK_INSTANCE_CAST ((obj), GST_TYPE_TENSOR_IF, GstTensorIf))
+#define GST_TENSOR_IF_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST ((klass), GST_TYPE_TENSOR_IF, GstTensorIfClass))
+#define GST_TENSOR_IF_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS ((obj), GST_TYPE_TENSOR_IF, GstTensorIfClass))
+#define GST_IS_TENSOR_IF(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_TENSOR_IF))
+#define GST_IS_TENSOR_IF_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_TENSOR_IF))
+#define GST_TENSOR_IF_CAST(obj)((GstTensorIf*)(obj))
+typedef struct _GstTensorIf GstTensorIf;
+typedef struct _GstTensorIfClass GstTensorIfClass;
+
+/**
+ * @brief Compared_Value
+ */
+typedef enum {
+  TIFCV_A_VALUE = 0,   /**< Decide based on a single scalar value
+                            of tensors */
+  TIFCV_TENSOR_TOTAL_VALUE = 1,        /**< Decide based on a total (sum) value of
+                                    a specific tensor */
+  TIFCV_ALL_TENSORS_TOTAL_VALUE = 2,   /**< Decide based on a total (sum) value of
+                                            of tensors or a specific tensor */
+  TIFCV_TENSOR_AVERAGE_VALUE = 3,      /**< Decide based on a average value of
+                                    a specific tensor */
+  TIFCV_ALL_TENSORS_AVERAGE_VALUE = 4, /**< Decide based on a average value of
+                                            tensors or a specific tensor */
+  TIFCV_END,
+} tensor_if_compared_value;
+
+/**
+ * @brief OPERAND
+ */
+typedef enum {
+  TIFOP_EQ = 0,        /**< == */
+  TIFOP_NE,    /**< != */
+  TIFOP_GT,    /**< > */
+  TIFOP_GE,    /**< >= */
+  TIFOP_LT,    /**< < */
+  TIFOP_LE,    /**< <= */
+  TIFOP_RANGE_INCLUSIVE,       /**< in [min, max] */
+  TIFOP_RANGE_EXCLUSIVE,       /**< in (min, max) */
+  TIFOP_NOT_IN_RANGE_INCLUSIVE,        /**< not in [min, max] */
+  TIFOP_NOT_IN_RANGE_EXCLUSIVE, /**< not in (min, max) */
+  TIFOP_END,
+} tensor_if_operator;
+
+/**
+ * @brief Behaviors that may fit in THEN and ELSE
+ * @details FILL_WITH_FILE, FILL_WITH_FILE_RPT, and REPEAT_PREVIOUS_FRAME caches an output frame
+ *          and thus, may consume additional memory and incur an additional memcpy.
+ */
+typedef enum {
+  TIFB_PASSTHROUGH = 0,        /**< The input frame becomes the output frame */
+  TIFB_SKIP,   /**< Do not generate output frame (frame skip) */
+  TIFB_FILL_ZERO,      /**< Fill output frame with zeros */
+  TIFB_FILL_VALUES,    /**< Fill output frame with a user given value */
+  TIFB_FILL_WITH_FILE, /**< Fill output frame with a user given file (a raw data of tensor/tensors)
+                            If the filesize is smaller, the reset is filled with 0 */
+  TIFB_FILL_WITH_FILE_RPT,     /**< Fill output frame with a user given file (a raw data of tensor/tensors)
+                                    If the filesize is smally, the file is repeatedly used */
+  TIFB_REPEAT_PREVIOUS_FRAME,  /**< Resend the previous output frame. If this is the first, send ZERO values. */
+  TIFB_END,
+} tensor_if_behavior;
+
+
+/**
+ * @brief Tensor If data structure
+ */
+struct _GstTensorIf
+{
+  GstBaseTransform element;     /**< This is the parent object */
+
+  gboolean silent;
+};
+
+/**
+ * @brief GstTensorIfClass inherits GstElementClass
+ */
+struct _GstTensorIfClass
+{
+  GstBaseTransformClass parent_class;   /**< Inherits GstBaseTransformClass */
+};
+
+/**
+ * @brief Get Type function required for gst elements
+ */
+GType gst_tensor_if_get_type (void);
+
+G_END_DECLS
+
+#endif /* __GST_TENSOR_IF_H__ */