v4l2codec: Pass system frame number as timestamp and use it to retrieve back frames...

System frame numbers are supposed to be unique and correct drivers are
passing through timestamps without modification from the output/sink to the
capture/src side.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/-/merge_requests/778>

diff --git a/sys/v4l2/gstv4l2bufferpool.c b/sys/v4l2/gstv4l2bufferpool.c
index 18f019a..52c9916 100644 (file)
--- a/sys/v4l2/gstv4l2bufferpool.c
+++ b/sys/v4l2/gstv4l2bufferpool.c
@@ -1139,10 +1139,9 @@ no_buffers:
 
 static GstFlowReturn
 gst_v4l2_buffer_pool_qbuf (GstV4l2BufferPool * pool, GstBuffer * buf,
-    GstV4l2MemoryGroup * group)
+    GstV4l2MemoryGroup * group, guint32 * frame_number)
 {
   const GstV4l2Object *obj = pool->obj;
-  GstClockTime timestamp;
   gint index;
 
   index = group->buffer.index;
@@ -1164,9 +1163,17 @@ gst_v4l2_buffer_pool_qbuf (GstV4l2BufferPool * pool, GstBuffer * buf,
     group->buffer.field = field;
   }
 
-  if (GST_BUFFER_TIMESTAMP_IS_VALID (buf)) {
-    timestamp = GST_BUFFER_TIMESTAMP (buf);
-    GST_TIME_TO_TIMEVAL (timestamp, group->buffer.timestamp);
+  if (frame_number) {
+    group->buffer.timestamp.tv_sec = *frame_number;
+    group->buffer.timestamp.tv_usec = 0;
+  } else {
+    if (GST_BUFFER_TIMESTAMP_IS_VALID (buf)) {
+      GstClockTime timestamp = GST_BUFFER_TIMESTAMP (buf);
+      GST_TIME_TO_TIMEVAL (timestamp, group->buffer.timestamp);
+    } else {
+      group->buffer.timestamp.tv_sec = -1;
+      group->buffer.timestamp.tv_usec = -1;
+    }
   }
 
   GST_OBJECT_LOCK (pool);
@@ -1506,7 +1513,8 @@ gst_v4l2_buffer_pool_release_buffer (GstBufferPool * bpool, GstBuffer * buffer)
             if (pool->other_pool)
               ret = gst_v4l2_buffer_pool_prepare_buffer (pool, buffer, NULL);
             if (ret != GST_FLOW_OK ||
-                gst_v4l2_buffer_pool_qbuf (pool, buffer, group) != GST_FLOW_OK)
+                gst_v4l2_buffer_pool_qbuf (pool, buffer, group,
+                    NULL) != GST_FLOW_OK)
               pclass->release_buffer (bpool, buffer);
           } else {
             /* Simply release invalid/modified buffer, the allocator will
@@ -1795,15 +1803,20 @@ cleanup:
  * gst_v4l2_buffer_pool_process:
  * @bpool: a #GstBufferPool
  * @buf: a #GstBuffer, maybe be replaced
+ * @frame_number: 32 bit frame number or %NULL
  *
  * Process @buf in @bpool. For capture devices, this functions fills @buf with
  * data from the device. For output devices, this functions send the contents of
  * @buf to the device for playback.
  *
+ * If non-%NULL and an output device, @frame_number is stored inside the timestamp for output devices and read
+ * back from the timestamp for capture devices.
+ *
  * Returns: %GST_FLOW_OK on success.
  */
 GstFlowReturn
-gst_v4l2_buffer_pool_process (GstV4l2BufferPool * pool, GstBuffer ** buf)
+gst_v4l2_buffer_pool_process (GstV4l2BufferPool * pool, GstBuffer ** buf,
+    guint32 * frame_number)
 {
   GstFlowReturn ret = GST_FLOW_OK;
   GstBufferPool *bpool = GST_BUFFER_POOL_CAST (pool);
@@ -2006,7 +2019,9 @@ gst_v4l2_buffer_pool_process (GstV4l2BufferPool * pool, GstBuffer ** buf)
             gst_v4l2_is_buffer_valid (to_queue, &group);
           }
 
-          if ((ret = gst_v4l2_buffer_pool_qbuf (pool, to_queue, group))
+          if ((ret =
+                  gst_v4l2_buffer_pool_qbuf (pool, to_queue, group,
+                      frame_number))
               != GST_FLOW_OK)
             goto queue_failed;
 

diff --git a/sys/v4l2/gstv4l2bufferpool.h b/sys/v4l2/gstv4l2bufferpool.h
index 62bcb0f..f8972b6 100644 (file)
--- a/sys/v4l2/gstv4l2bufferpool.h
+++ b/sys/v4l2/gstv4l2bufferpool.h
@@ -103,7 +103,7 @@ GType gst_v4l2_buffer_pool_get_type (void);
 
 GstBufferPool *     gst_v4l2_buffer_pool_new     (GstV4l2Object *obj, GstCaps *caps);
 
-GstFlowReturn       gst_v4l2_buffer_pool_process (GstV4l2BufferPool * bpool, GstBuffer ** buf);
+GstFlowReturn       gst_v4l2_buffer_pool_process (GstV4l2BufferPool * bpool, GstBuffer ** buf, guint32 * frame_number);
 
 void                gst_v4l2_buffer_pool_set_other_pool (GstV4l2BufferPool * pool,
                                                          GstBufferPool * other_pool);

diff --git a/sys/v4l2/gstv4l2sink.c b/sys/v4l2/gstv4l2sink.c
index a971ef4..e4728ac 100644 (file)
--- a/sys/v4l2/gstv4l2sink.c
+++ b/sys/v4l2/gstv4l2sink.c
@@ -610,7 +610,7 @@ gst_v4l2sink_show_frame (GstVideoSink * vsink, GstBuffer * buf)
   gst_buffer_ref (buf);
 again:
   ret = gst_v4l2_buffer_pool_process (GST_V4L2_BUFFER_POOL_CAST (obj->pool),
-      &buf);
+      &buf, NULL);
   if (ret == GST_FLOW_FLUSHING) {
     ret = gst_base_sink_wait_preroll (GST_BASE_SINK (vsink));
     if (ret == GST_FLOW_OK)

diff --git a/sys/v4l2/gstv4l2src.c b/sys/v4l2/gstv4l2src.c
index a518aca..2c30a51 100644 (file)
--- a/sys/v4l2/gstv4l2src.c
+++ b/sys/v4l2/gstv4l2src.c
@@ -844,7 +844,7 @@ gst_v4l2src_create (GstPushSrc * src, GstBuffer ** buf)
     if (G_UNLIKELY (ret != GST_FLOW_OK))
       goto alloc_failed;
 
-    ret = gst_v4l2_buffer_pool_process (pool, buf);
+    ret = gst_v4l2_buffer_pool_process (pool, buf, NULL);
 
   } while (ret == GST_V4L2_FLOW_CORRUPTED_BUFFER);
 

diff --git a/sys/v4l2/gstv4l2transform.c b/sys/v4l2/gstv4l2transform.c
index cc5564d..edb3203 100644 (file)
--- a/sys/v4l2/gstv4l2transform.c
+++ b/sys/v4l2/gstv4l2transform.c
@@ -927,7 +927,8 @@ gst_v4l2_transform_prepare_output_buffer (GstBaseTransform * trans,
   }
 
   GST_DEBUG_OBJECT (self, "Queue input buffer");
-  ret = gst_v4l2_buffer_pool_process (GST_V4L2_BUFFER_POOL (pool), &inbuf);
+  ret =
+      gst_v4l2_buffer_pool_process (GST_V4L2_BUFFER_POOL (pool), &inbuf, NULL);
   if (G_UNLIKELY (ret != GST_FLOW_OK))
     goto beach;
 
@@ -945,7 +946,9 @@ gst_v4l2_transform_prepare_output_buffer (GstBaseTransform * trans,
       goto alloc_failed;
 
     pool = self->v4l2capture->pool;
-    ret = gst_v4l2_buffer_pool_process (GST_V4L2_BUFFER_POOL (pool), outbuf);
+    ret =
+        gst_v4l2_buffer_pool_process (GST_V4L2_BUFFER_POOL (pool), outbuf,
+        NULL);
 
   } while (ret == GST_V4L2_FLOW_CORRUPTED_BUFFER);
 

diff --git a/sys/v4l2/gstv4l2videodec.c b/sys/v4l2/gstv4l2videodec.c
index 8c06c90..4f0ff30 100644 (file)
--- a/sys/v4l2/gstv4l2videodec.c
+++ b/sys/v4l2/gstv4l2videodec.c
@@ -438,7 +438,7 @@ gst_v4l2_video_dec_finish (GstVideoDecoder * decoder)
       buffer = gst_buffer_new ();
       ret =
           gst_v4l2_buffer_pool_process (GST_V4L2_BUFFER_POOL (self->
-              v4l2output->pool), &buffer);
+              v4l2output->pool), &buffer, NULL);
       gst_buffer_unref (buffer);
     }
   }
@@ -472,36 +472,6 @@ gst_v4l2_video_dec_drain (GstVideoDecoder * decoder)
   return GST_FLOW_OK;
 }
 
-static GstVideoCodecFrame *
-gst_v4l2_video_dec_get_oldest_frame (GstVideoDecoder * decoder)
-{
-  GstVideoCodecFrame *frame = NULL;
-  GList *frames, *l;
-  gint count = 0;
-
-  frames = gst_video_decoder_get_frames (decoder);
-
-  for (l = frames; l != NULL; l = l->next) {
-    GstVideoCodecFrame *f = l->data;
-
-    if (!frame || frame->pts > f->pts)
-      frame = f;
-
-    count++;
-  }
-
-  if (frame) {
-    GST_LOG_OBJECT (decoder,
-        "Oldest frame is %d %" GST_TIME_FORMAT " and %d frames left",
-        frame->system_frame_number, GST_TIME_ARGS (frame->pts), count - 1);
-    gst_video_codec_frame_ref (frame);
-  }
-
-  g_list_free_full (frames, (GDestroyNotify) gst_video_codec_frame_unref);
-
-  return frame;
-}
-
 static void
 gst_v4l2_video_dec_loop (GstVideoDecoder * decoder)
 {
@@ -535,15 +505,22 @@ gst_v4l2_video_dec_loop (GstVideoDecoder * decoder)
       goto beach;
 
     GST_LOG_OBJECT (decoder, "Process output buffer");
-    ret = gst_v4l2_buffer_pool_process (v4l2_pool, &buffer);
-
+    ret = gst_v4l2_buffer_pool_process (v4l2_pool, &buffer, NULL);
   } while (ret == GST_V4L2_FLOW_CORRUPTED_BUFFER);
 
   if (ret != GST_FLOW_OK)
     goto beach;
 
-  frame = gst_v4l2_video_dec_get_oldest_frame (decoder);
+  if (GST_BUFFER_TIMESTAMP (buffer) % GST_SECOND != 0)
+    GST_ERROR_OBJECT (decoder,
+        "Driver bug detected - check driver with v4l2-compliance from http://git.linuxtv.org/v4l-utils.git");
+  GST_LOG_OBJECT (decoder, "Got buffer for frame number %u",
+      (guint32) (GST_BUFFER_TIMESTAMP (buffer) / GST_SECOND));
 
+  /* FIXME: Add garbage collection for the frames */
+  frame =
+      gst_video_decoder_get_frame (decoder,
+      GST_BUFFER_TIMESTAMP (buffer) / GST_SECOND);
   if (frame) {
     frame->output_buffer = buffer;
     buffer = NULL;
@@ -633,6 +610,7 @@ gst_v4l2_video_dec_handle_frame (GstVideoDecoder * decoder,
     GstBuffer *codec_data;
     GstCaps *acquired_caps, *available_caps, *caps, *filter;
     GstStructure *st;
+    guint32 dummy_frame_number = 0;
 
     GST_DEBUG_OBJECT (self, "Sending header");
 
@@ -668,9 +646,12 @@ gst_v4l2_video_dec_handle_frame (GstVideoDecoder * decoder,
     }
 
     GST_VIDEO_DECODER_STREAM_UNLOCK (decoder);
+    GST_LOG_OBJECT (decoder, "Passing buffer with system frame number %u",
+        processed ? frame->system_frame_number : 0);
     ret =
         gst_v4l2_buffer_pool_process (GST_V4L2_BUFFER_POOL (self->
-            v4l2output->pool), &codec_data);
+            v4l2output->pool), &codec_data,
+        processed ? &frame->system_frame_number : &dummy_frame_number);
     GST_VIDEO_DECODER_STREAM_LOCK (decoder);
 
     gst_buffer_unref (codec_data);
@@ -765,9 +746,11 @@ gst_v4l2_video_dec_handle_frame (GstVideoDecoder * decoder,
 
   if (!processed) {
     GST_VIDEO_DECODER_STREAM_UNLOCK (decoder);
+    GST_LOG_OBJECT (decoder, "Passing buffer with system frame number %u",
+        frame->system_frame_number);
     ret =
         gst_v4l2_buffer_pool_process (GST_V4L2_BUFFER_POOL (self->v4l2output->
-            pool), &frame->input_buffer);
+            pool), &frame->input_buffer, &frame->system_frame_number);
     GST_VIDEO_DECODER_STREAM_LOCK (decoder);
 
     if (ret == GST_FLOW_FLUSHING) {

diff --git a/sys/v4l2/gstv4l2videoenc.c b/sys/v4l2/gstv4l2videoenc.c
index a6c44cb..193d1f7 100644 (file)
--- a/sys/v4l2/gstv4l2videoenc.c
+++ b/sys/v4l2/gstv4l2videoenc.c
@@ -612,37 +612,6 @@ not_negotiated:
   return FALSE;
 }
 
-static GstVideoCodecFrame *
-gst_v4l2_video_enc_get_oldest_frame (GstVideoEncoder * encoder)
-{
-  GstVideoCodecFrame *frame = NULL;
-  GList *frames, *l;
-  gint count = 0;
-
-  frames = gst_video_encoder_get_frames (encoder);
-
-  for (l = frames; l != NULL; l = l->next) {
-    GstVideoCodecFrame *f = l->data;
-
-    if (!frame || frame->pts > f->pts)
-      frame = f;
-
-    count++;
-  }
-
-  if (frame) {
-    GST_LOG_OBJECT (encoder,
-        "Oldest frame is %d %" GST_TIME_FORMAT
-        " and %d frames left",
-        frame->system_frame_number, GST_TIME_ARGS (frame->pts), count - 1);
-    gst_video_codec_frame_ref (frame);
-  }
-
-  g_list_free_full (frames, (GDestroyNotify) gst_video_codec_frame_unref);
-
-  return frame;
-}
-
 static void
 gst_v4l2_video_enc_loop (GstVideoEncoder * encoder)
 {
@@ -661,18 +630,24 @@ gst_v4l2_video_enc_loop (GstVideoEncoder * encoder)
     goto beach;
   }
 
-
   /* FIXME Check if buffer isn't the last one here */
 
   GST_LOG_OBJECT (encoder, "Process output buffer");
   ret =
       gst_v4l2_buffer_pool_process (GST_V4L2_BUFFER_POOL
-      (self->v4l2capture->pool), &buffer);
+      (self->v4l2capture->pool), &buffer, NULL);
 
   if (ret != GST_FLOW_OK)
     goto beach;
 
-  frame = gst_v4l2_video_enc_get_oldest_frame (encoder);
+  if (GST_BUFFER_TIMESTAMP (buffer) % GST_SECOND != 0)
+    GST_ERROR_OBJECT (encoder,
+        "Driver bug detected - check driver with v4l2-compliance from http://git.linuxtv.org/v4l-utils.git");
+  GST_LOG_OBJECT (encoder, "Got buffer for frame number %u",
+      (guint32) (GST_BUFFER_PTS (buffer) / GST_SECOND));
+  frame =
+      gst_video_encoder_get_frame (encoder,
+      GST_BUFFER_TIMESTAMP (buffer) / GST_SECOND);
 
   if (frame) {
     /* At this point, the delta unit buffer flag is already correctly set by
@@ -781,9 +756,12 @@ gst_v4l2_video_enc_handle_frame (GstVideoEncoder * encoder,
 
   if (frame->input_buffer) {
     GST_VIDEO_ENCODER_STREAM_UNLOCK (encoder);
+    GST_LOG_OBJECT (encoder, "Passing buffer with frame number %u",
+        frame->system_frame_number);
     ret =
-        gst_v4l2_buffer_pool_process (GST_V4L2_BUFFER_POOL
-        (self->v4l2output->pool), &frame->input_buffer);
+        gst_v4l2_buffer_pool_process (GST_V4L2_BUFFER_POOL (self->
+            v4l2output->pool), &frame->input_buffer,
+        &frame->system_frame_number);
     GST_VIDEO_ENCODER_STREAM_LOCK (encoder);
 
     if (ret == GST_FLOW_FLUSHING) {