static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink",
GST_PAD_SINK,
GST_PAD_ALWAYS,
- GST_STATIC_CAPS ("video/x-raw, format = (string)RGB")
+ GST_STATIC_CAPS ("video/x-raw, format = (string)RGB, views = (int)1, interlace-mode = (string)progressive")
);
/* the capabilities of the outputs
tensor_type type;
gint framerate_numerator;
gint framerate_denominator;
+ gsize tensorFrameSize;
gboolean ret;
GstCaps *outcaps;
int i;
type = _C2T_UINT8; /* Assume color depth per component is 8 bit */
dimension[2] = 3; /* R G B */
dimension[3] = 1; /* This is 3-D Tensor */
+ tensorFrameSize = GstConvert2TensorDataSize[type] * dimension[0] * dimension[1] * dimension[2] * dimension[3];
/* Refer: https://gstreamer.freedesktop.org/documentation/design/mediatype-video-raw.html */
if (filter->tensorConfigured == TRUE) {
if (rank == filter->rank &&
type == filter->type &&
framerate_numerator == filter->framerate_numerator &&
+ tensorFrameSize == filter->tensorFrameSize &&
framerate_denominator == filter->framerate_denominator) {
for (i = 0; i < GST_CONVERT2TENSOR_TENSOR_RANK_LIMIT; i++)
if (dimension[i] != filter->dimension[i])
filter->type = type;
filter->framerate_numerator = framerate_numerator;
filter->framerate_denominator = framerate_denominator;
+ filter->tensorFrameSize = tensorFrameSize;
filter->tensorConfigured = TRUE;
return TRUE;
"http://gstreamer.net/"
)
+static GstFlowReturn gst_c2t_transformer_videoframe(GstConvert2Tensor *filter,
+ GstVideoFrame *inframe, GstBuffer *outbuf)
+{
+ /* @TODO RGB are interlaced! FIXME!
+ @TODO Width has stride-4. FIXME!
+ */
+ return gst_buffer_copy_into(outbuf, inframe->buffer,
+ GST_BUFFER_COPY_FLAGS | GST_BUFFER_COPY_TIMESTAMPS, 0,
+ GST_VIDEO_FRAME_SIZE(inframe));
+}
static GstFlowReturn gst_convert2tensor_transform(GstBaseTransform *trans,
GstBuffer *inbuf,
GstBuffer *outbuf)
{
+ GstVideoFrame in_frame;
+ GstFlowReturn res;
+ GstConvert2Tensor *filter = GST_CONVERT2TENSOR_CAST(trans);
+
+ if (G_UNLIKELY(!filter->negotiated))
+ goto unknown_format;
+ if (G_UNLIKELY(!filter->tensorConfigured))
+ goto unknown_tensor;
+
+ switch(filter->input_media_type) {
+ case _C2T_VIDEO:
+ // CAUTION! in_info.video must be already configured!
+ if (!gst_video_frame_map(&in_frame, &filter->in_info.video, inbuf,
+ GST_MAP_READ | GST_VIDEO_FRAME_MAP_FLAG_NO_REF))
+ goto invalid_buffer;
+
+ if (gst_c2t_transformer_videoframe(filter, &in_frame, outbuf))
+ res = GST_FLOW_OK;
+ else
+ res = GST_FLOW_ERROR;
+ gst_video_frame_unmap(&in_frame);
+ break;
+ /* NOT SUPPORTED */
+ case _C2T_AUDIO:
+ case _C2T_STRING:
+ default:
+ g_printerr(" Unsupported Media Type (%d)\n", filter->input_media_type);
+ goto unknown_type;
+ }
+
+ return res;
+
+unknown_format:
+ GST_ELEMENT_ERROR(filter, CORE, NOT_IMPLEMENTED, (NULL), ("unknown format"));
+ return GST_FLOW_NOT_NEGOTIATED;
+unknown_tensor:
+ GST_ELEMENT_ERROR(filter, CORE, NOT_IMPLEMENTED, (NULL), ("unknown format for tensor"));
+ return GST_FLOW_NOT_NEGOTIATED;
+unknown_type:
+ GST_ELEMENT_ERROR(filter, CORE, NOT_IMPLEMENTED, (NULL), ("not implemented type of media"));
+ return GST_FLOW_NOT_SUPPORTED;
+invalid_buffer:
+ GST_ELEMENT_ERROR(filter, CORE, NOT_IMPLEMENTED, (NULL), ("invalid video buffer received from input"));
+ return GST_FLOW_ERROR;
}
static GstFlowReturn gst_convert2tensor_transform_ip(GstBaseTransform *trans,
GstBuffer *buf)
{
+ /* DO NOTHING. THIS WORKS AS A PASSTHROUGH. We just remove metadata from video */
+ return GST_FLOW_OK;
}
static GstCaps* gst_convert2tensor_transform_caps(GstBaseTransform *trans,
* @file convert2tensor.c
* @date 26 Mar 2018
* @brief GStreamer plugin to convert media types to tensors (as a filter for other general neural network filters)
+ *
+ * Be careful: this filter assumes that the user has attached
+ * rawvideoparser as a preprocessor for this filter so that
+ * the incoming buffer is nicely aligned in the array of
+ * uint8[RGB][height][width].
+ *
* @see http://github.com/TO-BE-DETERMINED-SOON
* @author MyungJoo Ham <myungjoo.ham@samsung.com>
*
#include <gst/gst.h>
#include <gst/base/gstbasetransform.h>
+#include <gst/video/video-info.h>
+#include <gst/video/video-frame.h>
G_BEGIN_DECLS
#define GST_CONVERT2TENSOR_TENSOR_RANK_LIMIT (4)
typedef enum _tensor_type {
- _C2T_INT32 = 0,
- _C2T_UINT32,
- _C2T_INT16,
- _C2T_UINT16,
- _C2T_INT8,
- _C2T_UINT8,
- _C2T_FLOAT64,
- _C2T_FLOAT32,
+ _C2T_INT32 = 0,
+ _C2T_UINT32,
+ _C2T_INT16,
+ _C2T_UINT16,
+ _C2T_INT8,
+ _C2T_UINT8,
+ _C2T_FLOAT64,
+ _C2T_FLOAT32,
- _C2T_END,
+ _C2T_END,
} tensor_type;
+typedef enum _media_type {
+ _C2T_VIDEO = 0,
+ _C2T_AUDIO, /* Not Supported Yet */
+ _C2T_STRING, /* Not Supported Yet */
+
+ _C2T_MEDIA_END,
+} media_type;
struct _GstConvert2Tensor
{
GstBaseTransform element; /**< This is the parent object */
+ /* For transformer */
+ gboolean negotiated; /* When this is %TRUE, tensor metadata must be set */
+ media_type input_media_type;
+ union {
+ GstVideoInfo video;
+ /* @TODO: Add other media types */
+ } in_info;
+
+ /* For Tensor */
gboolean silent; /**< True if logging is minimized */
gboolean tensorConfigured; /**< True if already successfully configured tensor metadata */
gint rank; /**< Tensor Rank (# dimensions) */
- gint dimension[GST_CONVERT2TENSOR_TENSOR_RANK_LIMIT]; /**< Dimensions. We support up to 4th ranks */
+ gint dimension[GST_CONVERT2TENSOR_TENSOR_RANK_LIMIT];
+ /**< Dimensions. We support up to 4th ranks.
+ * @caution The first dimension is always 4 x N.
+ **/
tensor_type type; /**< Type of each element in the tensor. User must designate this. Otherwise, this is UINT8 for video/x-raw byte stream */
gint framerate_numerator; /**< framerate is in fraction, which is numerator/denominator */
gint framerate_denominator; /**< framerate is in fraction, which is numerator/denominator */
+ gsize tensorFrameSize;
};
const unsigned int GstConvert2TensorDataSize[] = {
[_C2T_INT32] = 4,
[_C2T_FLOAT32] = "float32",
};
-
+/*
+ * GstConvert2TensorClass inherits GstBaseTransformClass.
+ *
+ * Referring another child (sibiling), GstVideoFilter (abstract class) and
+ * its child (concrete class) GstVideoConverter.
+ * Note that GstConvert2TensorClass is a concrete class; thus we need to look at both.
+ */
struct _GstConvert2TensorClass
{
GstBaseTransformClass parent_class;