deinterlace: Closed caption pass-through

Pass through closed caption data when deinterlacing. When two
deinterlaced frames are created for the same interlaced frame (e.g.
fields=all), the second of the two frames will have no closed caption
data.

Also fixed memory leaks related to timecode meta pass-through.

https://bugzilla.gnome.org/show_bug.cgi?id=796876

diff --git a/gst/deinterlace/gstdeinterlace.c b/gst/deinterlace/gstdeinterlace.c
index ed19a5c..51e9133 100644 (file)
--- a/gst/deinterlace/gstdeinterlace.c
+++ b/gst/deinterlace/gstdeinterlace.c
@@ -760,6 +760,77 @@ gst_video_frame_unmap_and_free (GstVideoFrame * frame)
   g_free (frame);
 }
 
+static GstVideoFrame *
+gst_deinterlace_pop_history (GstDeinterlace * self)
+{
+  GstVideoFrame *frame;
+
+  g_return_val_if_fail (self->history_count > 0, NULL);
+
+  GST_DEBUG_OBJECT (self, "Pop last history frame -- current history size %d",
+      self->history_count);
+
+  frame = self->field_history[self->history_count - 1].frame;
+
+  self->history_count--;
+  if (self->locking != GST_DEINTERLACE_LOCKING_NONE && (!self->history_count
+          || GST_VIDEO_FRAME_PLANE_DATA (frame, 0) !=
+          GST_VIDEO_FRAME_PLANE_DATA (self->field_history[self->history_count -
+                  1].frame, 0))) {
+    if (!self->low_latency)
+      self->state_count--;
+    if (self->pattern_lock) {
+      self->pattern_count++;
+      if (self->pattern != -1
+          && self->pattern_count >= telecine_patterns[self->pattern].length) {
+        self->pattern_count = 0;
+        self->output_count = 0;
+      }
+    }
+  }
+
+  GST_DEBUG_OBJECT (self, "Returning frame: %p %" GST_TIME_FORMAT
+      " with duration %" GST_TIME_FORMAT " and size %" G_GSIZE_FORMAT, frame,
+      GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (frame->buffer)),
+      GST_TIME_ARGS (GST_BUFFER_DURATION (frame->buffer)),
+      GST_VIDEO_FRAME_SIZE (frame));
+
+  return frame;
+}
+
+static void
+gst_deinterlace_delete_meta_at (GstDeinterlace * self, gint idx)
+{
+  if (self->field_history[idx].frame) {
+    if (self->field_history[idx].tc) {
+      gst_video_time_code_free (self->field_history[idx].tc);
+      self->field_history[idx].tc = NULL;
+    }
+    if (self->field_history[idx].caption) {
+      g_free (self->field_history[idx].caption->data);
+      g_free (self->field_history[idx].caption);
+      self->field_history[idx].caption = NULL;
+    }
+  }
+}
+
+static void
+gst_deinterlace_pop_and_clear (GstDeinterlace * self)
+{
+  gint idx;
+
+  if (self->history_count <= 0)
+    return;
+
+  idx = self->history_count - 1;
+  gst_deinterlace_delete_meta_at (self, idx);
+
+  /* FIXME: pop_history should return a structure with the frame and its meta.
+   * Currently we're just doing guesswork with the indices. Maybe just
+   * refactor the history functionality to make something clearer */
+  gst_video_frame_unmap_and_free (gst_deinterlace_pop_history (self));
+}
+
 static void
 gst_deinterlace_reset_history (GstDeinterlace * self, gboolean drop_all)
 {
@@ -783,9 +854,7 @@ gst_deinterlace_reset_history (GstDeinterlace * self, gboolean drop_all)
       if (self->field_history[i].frame) {
         gst_video_frame_unmap_and_free (self->field_history[i].frame);
         self->field_history[i].frame = NULL;
-        if (self->field_history[i].tc) {
-          gst_video_time_code_free (self->field_history[i].tc);
-        }
+        gst_deinterlace_delete_meta_at (self, i);
       }
     }
   }
@@ -985,44 +1054,6 @@ gst_deinterlace_update_pattern_timestamps (GstDeinterlace * self)
       GST_TIME_ARGS (self->pattern_buf_dur));
 }
 
-static GstVideoFrame *
-gst_deinterlace_pop_history (GstDeinterlace * self)
-{
-  GstVideoFrame *frame;
-
-  g_return_val_if_fail (self->history_count > 0, NULL);
-
-  GST_DEBUG_OBJECT (self, "Pop last history frame -- current history size %d",
-      self->history_count);
-
-  frame = self->field_history[self->history_count - 1].frame;
-
-  self->history_count--;
-  if (self->locking != GST_DEINTERLACE_LOCKING_NONE && (!self->history_count
-          || GST_VIDEO_FRAME_PLANE_DATA (frame, 0) !=
-          GST_VIDEO_FRAME_PLANE_DATA (self->field_history[self->history_count -
-                  1].frame, 0))) {
-    if (!self->low_latency)
-      self->state_count--;
-    if (self->pattern_lock) {
-      self->pattern_count++;
-      if (self->pattern != -1
-          && self->pattern_count >= telecine_patterns[self->pattern].length) {
-        self->pattern_count = 0;
-        self->output_count = 0;
-      }
-    }
-  }
-
-  GST_DEBUG_OBJECT (self, "Returning frame: %p %" GST_TIME_FORMAT
-      " with duration %" GST_TIME_FORMAT " and size %" G_GSIZE_FORMAT, frame,
-      GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (frame->buffer)),
-      GST_TIME_ARGS (GST_BUFFER_DURATION (frame->buffer)),
-      GST_VIDEO_FRAME_SIZE (frame));
-
-  return frame;
-}
-
 static void
 gst_deinterlace_get_buffer_state (GstDeinterlace * self, GstVideoFrame * frame,
     guint8 * state, GstVideoInterlaceMode * i_mode)
@@ -1129,14 +1160,9 @@ gst_deinterlace_push_history (GstDeinterlace * self, GstBuffer * buffer)
         self->field_history[i - fields_to_push].frame;
     self->field_history[i].flags =
         self->field_history[i - fields_to_push].flags;
-    if (self->field_history[i].tc)
-      gst_video_time_code_free (self->field_history[i].tc);
-    if (self->field_history[i - fields_to_push].tc) {
-      self->field_history[i].tc =
-          gst_video_time_code_copy (self->field_history[i - fields_to_push].tc);
-    } else {
-      self->field_history[i].tc = NULL;
-    }
+    self->field_history[i].tc = self->field_history[i - fields_to_push].tc;
+    self->field_history[i].caption =
+        self->field_history[i - fields_to_push].caption;
   }
 
   if (field_layout == GST_DEINTERLACE_LAYOUT_AUTO) {
@@ -1171,6 +1197,7 @@ gst_deinterlace_push_history (GstDeinterlace * self, GstBuffer * buffer)
 
   if (!onefield) {
     GstVideoTimeCodeMeta *meta = gst_buffer_get_video_time_code_meta (buffer);
+    GstVideoCaptionMeta *cc_meta = gst_buffer_get_video_caption_meta (buffer);
 
     GST_DEBUG_OBJECT (self, "Two fields");
     self->field_history[1].frame = field1;
@@ -1187,8 +1214,24 @@ gst_deinterlace_push_history (GstDeinterlace * self, GstBuffer * buffer)
       self->field_history[1].tc->config.flags &=
           ~GST_VIDEO_TIME_CODE_FLAGS_INTERLACED;
     }
+
+    if (cc_meta) {
+      self->field_history[0].caption = g_new (GstVideoCaptionMeta, 1);
+      self->field_history[0].caption->data = g_malloc (cc_meta->size);
+      self->field_history[0].caption->caption_type = cc_meta->caption_type;
+      self->field_history[0].caption->size = cc_meta->size;
+      memcpy (self->field_history[0].caption->data, cc_meta->data,
+          cc_meta->size);
+      self->field_history[1].caption = g_new (GstVideoCaptionMeta, 1);
+      self->field_history[1].caption->data = g_malloc (cc_meta->size);
+      self->field_history[1].caption->caption_type = cc_meta->caption_type;
+      self->field_history[1].caption->size = cc_meta->size;
+      memcpy (self->field_history[1].caption->data, cc_meta->data,
+          cc_meta->size);
+    }
   } else {                      /* onefield */
     GstVideoTimeCodeMeta *meta = gst_buffer_get_video_time_code_meta (buffer);
+    GstVideoCaptionMeta *cc_meta = gst_buffer_get_video_caption_meta (buffer);
 
     GST_DEBUG_OBJECT (self, "One field");
     self->field_history[0].frame = field1;
@@ -1198,6 +1241,15 @@ gst_deinterlace_push_history (GstDeinterlace * self, GstBuffer * buffer)
       self->field_history[0].tc->config.flags &=
           ~GST_VIDEO_TIME_CODE_FLAGS_INTERLACED;
     }
+
+    if (cc_meta) {
+      self->field_history[0].caption = g_new (GstVideoCaptionMeta, 1);
+      self->field_history[0].caption->data = g_malloc (cc_meta->size);
+      self->field_history[0].caption->caption_type = cc_meta->caption_type;
+      self->field_history[0].caption->size = cc_meta->size;
+      memcpy (self->field_history[0].caption->data, cc_meta->data,
+          cc_meta->size);
+    }
     gst_video_frame_unmap_and_free (field2);
   }
 
@@ -1525,6 +1577,7 @@ gst_deinterlace_output_frame (GstDeinterlace * self, gboolean flushing)
   TelecinePattern pattern;
   guint8 phase, count;
   const GstDeinterlaceLocking locking = self->locking;
+  gboolean cc_added = FALSE;
 
   memset (&pattern, 0, sizeof (pattern));
 
@@ -1604,7 +1657,7 @@ restart:
       if (flush_one && self->drop_orphans) {
         GST_DEBUG_OBJECT (self, "Dropping orphan first field");
         self->cur_field_idx--;
-        gst_video_frame_unmap_and_free (gst_deinterlace_pop_history (self));
+        gst_deinterlace_pop_and_clear (self);
         goto restart;
       }
     }
@@ -1664,15 +1717,17 @@ restart:
     GST_DEBUG_OBJECT (self,
         "Frame type: Progressive; pushing buffer as a frame");
     /* pop and push */
+    gst_deinterlace_delete_meta_at (self, self->history_count - 1);
     self->cur_field_idx--;
     field1_frame = gst_deinterlace_pop_history (self);
     field1_buffer = field1_frame->buffer;
     gst_buffer_ref (field1_buffer);
     gst_video_frame_unmap_and_free (field1_frame);
+
     /* field2 is the same buffer as field1, but we need to remove it from the
      * history anyway */
     self->cur_field_idx--;
-    gst_video_frame_unmap_and_free (gst_deinterlace_pop_history (self));
+    gst_deinterlace_pop_and_clear (self);
     GST_DEBUG_OBJECT (self,
         "[OUT] ts %" GST_TIME_FORMAT ", dur %" GST_TIME_FORMAT ", end %"
         GST_TIME_FORMAT,
@@ -1786,6 +1841,15 @@ restart:
       gst_buffer_add_video_time_code_meta (outbuf,
           self->field_history[index].tc);
     }
+    if (self->field_history[index].caption) {
+      g_assert (self->field_history[index].caption->data != NULL);
+      g_assert (!cc_added);
+      gst_buffer_add_video_caption_meta (outbuf,
+          self->field_history[index].caption->caption_type,
+          self->field_history[index].caption->data,
+          self->field_history[index].caption->size);
+      cc_added = TRUE;
+    }
     if (IS_TELECINE (interlacing_mode) && !self->telecine_tc_warned) {
       self->telecine_tc_warned = TRUE;
       GST_FIXME_OBJECT (self,
@@ -1828,14 +1892,14 @@ restart:
     /* Check if we need to drop the frame because of QoS */
     if (!gst_deinterlace_do_qos (self, buf)) {
       self->cur_field_idx--;
-      gst_video_frame_unmap_and_free (gst_deinterlace_pop_history (self));
+      gst_deinterlace_pop_and_clear (self);
       gst_buffer_unref (outbuf);
       outbuf = NULL;
       ret = GST_FLOW_OK;
     } else {
       if (self->cur_field_idx < 0 && flushing) {
         if (self->history_count == 1) {
-          gst_video_frame_unmap_and_free (gst_deinterlace_pop_history (self));
+          gst_deinterlace_pop_and_clear (self);
           goto need_more;
         }
         self->cur_field_idx++;
@@ -1866,7 +1930,7 @@ restart:
           || self->cur_field_idx + 1 +
           gst_deinterlace_method_get_latency (self->method) <
           self->history_count || flushing) {
-        gst_video_frame_unmap_and_free (gst_deinterlace_pop_history (self));
+        gst_deinterlace_pop_and_clear (self);
       }
 
       if (gst_deinterlace_clip_buffer (self, outbuf)) {
@@ -1895,7 +1959,7 @@ restart:
         GST_DEBUG_OBJECT (self, "Removing unused field (count: %d)",
             self->history_count);
         self->cur_field_idx--;
-        gst_video_frame_unmap_and_free (gst_deinterlace_pop_history (self));
+        gst_deinterlace_pop_and_clear (self);
         interlacing_mode = GST_VIDEO_INTERLACE_MODE_INTERLEAVED;
         return ret;
       }
@@ -1912,7 +1976,7 @@ restart:
           && !IS_TELECINE (interlacing_mode))) {
     GST_DEBUG_OBJECT (self, "Removing unused top field");
     self->cur_field_idx--;
-    gst_video_frame_unmap_and_free (gst_deinterlace_pop_history (self));
+    gst_deinterlace_pop_and_clear (self);
 
     if (flush_one && !self->drop_orphans) {
       GST_DEBUG_OBJECT (self, "Orphan field deinterlaced - reconfiguring");
@@ -1953,6 +2017,14 @@ restart:
       gst_buffer_add_video_time_code_meta (outbuf,
           self->field_history[index].tc);
     }
+    if (self->field_history[index].caption && !cc_added) {
+      g_assert (self->field_history[index].caption->data != NULL);
+      gst_buffer_add_video_caption_meta (outbuf,
+          self->field_history[index].caption->caption_type,
+          self->field_history[index].caption->data,
+          self->field_history[index].caption->size);
+      cc_added = TRUE;
+    }
     if (IS_TELECINE (interlacing_mode) && !self->telecine_tc_warned) {
       self->telecine_tc_warned = TRUE;
       GST_FIXME_OBJECT (self,
@@ -1995,7 +2067,7 @@ restart:
     /* Check if we need to drop the frame because of QoS */
     if (!gst_deinterlace_do_qos (self, buf)) {
       self->cur_field_idx--;
-      gst_video_frame_unmap_and_free (gst_deinterlace_pop_history (self));
+      gst_deinterlace_pop_and_clear (self);
       gst_buffer_unref (outbuf);
       outbuf = NULL;
       ret = GST_FLOW_OK;
@@ -2019,7 +2091,7 @@ restart:
           || self->cur_field_idx + 1 +
           gst_deinterlace_method_get_latency (self->method) <
           self->history_count) {
-        gst_video_frame_unmap_and_free (gst_deinterlace_pop_history (self));
+        gst_deinterlace_pop_and_clear (self);
       }
 
       if (gst_deinterlace_clip_buffer (self, outbuf)) {
@@ -2044,7 +2116,7 @@ restart:
         GST_DEBUG_OBJECT (self, "Removing unused field (count: %d)",
             self->history_count);
         self->cur_field_idx--;
-        gst_video_frame_unmap_and_free (gst_deinterlace_pop_history (self));
+        gst_deinterlace_pop_and_clear (self);
         interlacing_mode = GST_VIDEO_INTERLACE_MODE_INTERLEAVED;
         return ret;
       }
@@ -2061,7 +2133,7 @@ restart:
           && !IS_TELECINE (interlacing_mode))) {
     GST_DEBUG_OBJECT (self, "Removing unused bottom field");
     self->cur_field_idx--;
-    gst_video_frame_unmap_and_free (gst_deinterlace_pop_history (self));
+    gst_deinterlace_pop_and_clear (self);
 
     if (flush_one && !self->drop_orphans) {
       GST_DEBUG_OBJECT (self, "Orphan field deinterlaced - reconfiguring");

diff --git a/gst/deinterlace/gstdeinterlacemethod.h b/gst/deinterlace/gstdeinterlacemethod.h
index c4ab201..e967f2c 100644 (file)
--- a/gst/deinterlace/gstdeinterlacemethod.h
+++ b/gst/deinterlace/gstdeinterlacemethod.h
@@ -55,6 +55,7 @@ typedef struct
   /* see PICTURE_ flags in *.c */
   guint flags;
   GstVideoTimeCode *tc;
+  GstVideoCaptionMeta *caption;
 } GstDeinterlaceField;
 
 /*