h264parse: extract CEA-708 closed captions

Expose SEI data in the H.264 bitstream parser API and
extract closed captions and other things that are not
specified in the H.264 spec itself in the videoparser.

Based on patch by: Mathieu Duponchelle <mathieu@centricular.com>

https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/940

diff --git a/gst-libs/gst/codecparsers/gsth264parser.c b/gst-libs/gst/codecparsers/gsth264parser.c
index 30d6aea49552fb4a5d60ef105484adb2db48599e..acc6fd91ac4605df9ae22ba61feb46bb41fe75de 100644 (file)
--- a/gst-libs/gst/codecparsers/gsth264parser.c
+++ b/gst-libs/gst/codecparsers/gsth264parser.c
@@ -1008,6 +1008,51 @@ error:
   return GST_H264_PARSER_ERROR;
 }
 
+static GstH264ParserResult
+gst_h264_parser_parse_registered_user_data (GstH264NalParser * nalparser,
+    GstH264RegisteredUserData * rud, NalReader * nr, guint payload_size)
+{
+  guint8 *data = NULL;
+  guint i;
+
+  rud->data = NULL;
+  rud->size = 0;
+
+  if (payload_size < 2)
+    return GST_H264_PARSER_ERROR;
+
+  READ_UINT8 (nr, rud->country_code, 8);
+  --payload_size;
+
+  if (rud->country_code == 0xFF) {
+    READ_UINT8 (nr, rud->country_code_extension, 8);
+    --payload_size;
+  } else {
+    rud->country_code_extension = 0;
+  }
+
+  if (payload_size < 8)
+    return GST_H264_PARSER_ERROR;
+
+  data = g_malloc (payload_size);
+  for (i = 0; i < payload_size / 8; ++i) {
+    READ_UINT8 (nr, data[i], 8);
+  }
+
+  GST_MEMDUMP ("SEI user data", data, payload_size / 8);
+
+  rud->data = data;
+  rud->size = payload_size;
+  return GST_H264_PARSER_OK;
+
+error:
+  {
+    GST_WARNING ("error parsing \"Registered User Data\"");
+    g_free (data);
+    return GST_H264_PARSER_ERROR;
+  }
+}
+
 static GstH264ParserResult
 gst_h264_parser_parse_recovery_point (GstH264NalParser * nalparser,
     GstH264RecoveryPoint * rp, NalReader * nr)
@@ -1157,6 +1202,10 @@ gst_h264_parser_parse_sei_message (GstH264NalParser * nalparser,
       res = gst_h264_parser_parse_pic_timing (nalparser,
           &sei->payload.pic_timing, nr);
       break;
+    case GST_H264_SEI_REGISTERED_USER_DATA:
+      res = gst_h264_parser_parse_registered_user_data (nalparser,
+          &sei->payload.registered_user_data, nr, payload_size);
+      break;
     case GST_H264_SEI_RECOVERY_POINT:
       res = gst_h264_parser_parse_recovery_point (nalparser,
           &sei->payload.recovery_point, nr);
@@ -2276,6 +2325,22 @@ gst_h264_sps_clear (GstH264SPS * sps)
   }
 }
 
+static void
+h264_sei_message_clear (GstH264SEIMessage * sei_msg)
+{
+  switch (sei_msg->payloadType) {
+    case GST_H264_SEI_REGISTERED_USER_DATA:{
+      GstH264RegisteredUserData *rud = &sei_msg->payload.registered_user_data;
+
+      g_free ((guint8 *) rud->data);
+      rud->data = NULL;
+      break;
+    }
+    default:
+      break;
+  }
+}
+
 /**
  * gst_h264_parser_parse_sei:
  * @nalparser: a #GstH264NalParser
@@ -2299,6 +2364,7 @@ gst_h264_parser_parse_sei (GstH264NalParser * nalparser, GstH264NalUnit * nalu,
   nal_reader_init (&nr, nalu->data + nalu->offset + nalu->header_bytes,
       nalu->size - nalu->header_bytes);
   *messages = g_array_new (FALSE, FALSE, sizeof (GstH264SEIMessage));
+  g_array_set_clear_func (*messages, (GDestroyNotify) h264_sei_message_clear);
 
   do {
     res = gst_h264_parser_parse_sei_message (nalparser, &nr, &sei);

diff --git a/gst-libs/gst/codecparsers/gsth264parser.h b/gst-libs/gst/codecparsers/gsth264parser.h
index 671b5e5fd29ecd97369338085b2af14a9fe7f2cc..4e063d59601ec75946a9468bb57127966ea4e8df 100644 (file)
--- a/gst-libs/gst/codecparsers/gsth264parser.h
+++ b/gst-libs/gst/codecparsers/gsth264parser.h
@@ -240,6 +240,7 @@ typedef enum
  * GstH264SEIPayloadType:
  * @GST_H264_SEI_BUF_PERIOD: Buffering Period SEI Message
  * @GST_H264_SEI_PIC_TIMING: Picture Timing SEI Message
+ * @GST_H264_SEI_REGISTERED_USER_DATA: Registered user data (D.2.5)
  * @GST_H264_SEI_RECOVERY_POINT: Recovery Point SEI Message (D.2.7)
  * @GST_H264_SEI_STEREO_VIDEO_INFO: stereo video info SEI message (Since: 1.6)
  * @GST_H264_SEI_FRAME_PACKING: Frame Packing Arrangement (FPA) message that
@@ -252,6 +253,7 @@ typedef enum
 {
   GST_H264_SEI_BUF_PERIOD = 0,
   GST_H264_SEI_PIC_TIMING = 1,
+  GST_H264_SEI_REGISTERED_USER_DATA = 4,
   GST_H264_SEI_RECOVERY_POINT = 6,
   GST_H264_SEI_STEREO_VIDEO_INFO = 21,
   GST_H264_SEI_FRAME_PACKING = 45
@@ -345,6 +347,7 @@ typedef struct _GstH264SliceHdr               GstH264SliceHdr;
 
 typedef struct _GstH264ClockTimestamp         GstH264ClockTimestamp;
 typedef struct _GstH264PicTiming              GstH264PicTiming;
+typedef struct _GstH264RegisteredUserData     GstH264RegisteredUserData;
 typedef struct _GstH264BufferingPeriod        GstH264BufferingPeriod;
 typedef struct _GstH264RecoveryPoint          GstH264RecoveryPoint;
 typedef struct _GstH264StereoVideoInfo        GstH264StereoVideoInfo;
@@ -994,6 +997,14 @@ struct _GstH264PicTiming
   GstH264ClockTimestamp clock_timestamp[3];
 };
 
+struct _GstH264RegisteredUserData
+{
+  guint8 country_code;
+  guint8 country_code_extension;
+  const guint8 *data;
+  guint size;
+};
+
 struct _GstH264BufferingPeriod
 {
   GstH264SPS *sps;
@@ -1022,6 +1033,7 @@ struct _GstH264SEIMessage
   union {
     GstH264BufferingPeriod buffering_period;
     GstH264PicTiming pic_timing;
+    GstH264RegisteredUserData registered_user_data;
     GstH264RecoveryPoint recovery_point;
     GstH264StereoVideoInfo stereo_video_info;
     GstH264FramePacking frame_packing;

diff --git a/gst/videoparsers/gsth264parse.c b/gst/videoparsers/gsth264parse.c
index c2729102f287217af8c08ee94e46f929557a7e92..c2b3d81e40e8ba9d0496d9eb10f595f3a51180a2 100644 (file)
--- a/gst/videoparsers/gsth264parse.c
+++ b/gst/videoparsers/gsth264parse.c
@@ -519,6 +519,95 @@ _nal_name (GstH264NalUnitType nal_type)
 }
 #endif
 
+static void
+gst_h264_parse_process_sei_user_data (GstH264Parse * h264parse,
+    GstH264RegisteredUserData * rud)
+{
+  guint16 provider_code;
+  guint32 atsc_user_id;
+  GstByteReader br;
+
+  if (rud->country_code != 0xB5)
+    return;
+
+  if (rud->data == NULL || rud->size < 2)
+    return;
+
+  gst_byte_reader_init (&br, rud->data, rud->size);
+
+  provider_code = gst_byte_reader_get_uint16_be_unchecked (&br);
+
+  /* There is also 0x29 / 47 for DirecTV, but we don't handle that for now.
+   * https://en.wikipedia.org/wiki/CEA-708#Picture_User_Data */
+  if (provider_code != 0x0031)
+    return;
+
+  /* ANSI/SCTE 128-2010a section 8.1.2 */
+  if (!gst_byte_reader_get_uint32_be (&br, &atsc_user_id))
+    return;
+
+  atsc_user_id = GUINT32_FROM_BE (atsc_user_id);
+  switch (atsc_user_id) {
+    case GST_MAKE_FOURCC ('G', 'A', '9', '4'):{
+      guint8 user_data_type_code, m;
+
+      /* 8.2.1 ATSC1_data() Semantics, Table 15 */
+      if (!gst_byte_reader_get_uint8 (&br, &user_data_type_code))
+        return;
+
+      switch (user_data_type_code) {
+        case 0x03:{
+          guint8 process_cc_data_flag;
+          guint8 cc_count, em_data, b;
+          guint i;
+
+          if (!gst_byte_reader_get_uint8 (&br, &b) || (b & 0x20) != 0)
+            break;
+
+          if (!gst_byte_reader_get_uint8 (&br, &em_data) || em_data != 0xff)
+            break;
+
+          process_cc_data_flag = (b & 0x40) != 0;
+          cc_count = b & 0x1f;
+
+          if (cc_count * 3 > gst_byte_reader_get_remaining (&br))
+            break;
+
+          if (!process_cc_data_flag || cc_count == 0) {
+            gst_byte_reader_skip_unchecked (&br, cc_count * 3);
+            break;
+          }
+
+          /* Shouldn't really happen so let's not go out of our way to handle it */
+          if (h264parse->closedcaptions_size > 0) {
+            GST_WARNING_OBJECT (h264parse, "unused pending closed captions!");
+            GST_MEMDUMP_OBJECT (h264parse, "unused captions being dropped",
+                h264parse->closedcaptions, h264parse->closedcaptions_size);
+          }
+
+          for (i = 0; i < cc_count * 3; i++) {
+            h264parse->closedcaptions[i] =
+                gst_byte_reader_get_uint8_unchecked (&br);
+          }
+          h264parse->closedcaptions_size = cc_count * 3;
+          h264parse->closedcaptions_type = GST_VIDEO_CAPTION_TYPE_CEA708_RAW;
+
+          GST_LOG_OBJECT (h264parse, "Extracted closed captions");
+          break;
+        }
+        default:
+          GST_LOG ("Unhandled atsc1 user data type %d", atsc_user_id);
+          break;
+      }
+      if (!gst_byte_reader_get_uint8 (&br, &m) || m != 0xff)
+        GST_WARNING_OBJECT (h264parse, "Marker bits mismatch after ATSC1_data");
+      break;
+    }
+    default:
+      break;
+  }
+}
+
 static void
 gst_h264_parse_process_sei (GstH264Parse * h264parse, GstH264NalUnit * nalu)
 {
@@ -563,6 +652,10 @@ gst_h264_parse_process_sei (GstH264Parse * h264parse, GstH264NalUnit * nalu)
         GST_LOG_OBJECT (h264parse, "pic timing updated");
         break;
       }
+      case GST_H264_SEI_REGISTERED_USER_DATA:
+        gst_h264_parse_process_sei_user_data (h264parse,
+            &sei.payload.registered_user_data);
+        break;
       case GST_H264_SEI_BUF_PERIOD:
         if (h264parse->ts_trn_nb == GST_CLOCK_TIME_NONE ||
             h264parse->dts == GST_CLOCK_TIME_NONE)
@@ -2486,6 +2579,15 @@ gst_h264_parse_pre_push_frame (GstBaseParse * parse, GstBaseParseFrame * frame)
     buffer = frame->buffer;
   }
 
+  if (h264parse->closedcaptions_size > 0) {
+    gst_buffer_add_video_caption_meta (buffer,
+        h264parse->closedcaptions_type, h264parse->closedcaptions,
+        h264parse->closedcaptions_size);
+
+    h264parse->closedcaptions_type = GST_VIDEO_CAPTION_TYPE_UNKNOWN;
+    h264parse->closedcaptions_size = 0;
+  }
+
   if ((event = check_pending_key_unit_event (h264parse->force_key_unit_event,
               &parse->segment, GST_BUFFER_TIMESTAMP (buffer),
               GST_BUFFER_FLAGS (buffer), h264parse->pending_key_unit_ts))) {

diff --git a/gst/videoparsers/gsth264parse.h b/gst/videoparsers/gsth264parse.h
index 3d4d8e28d1f084009721bef9c61fd30e7c76c0b9..a8e0120f297198838a10cebdd3bd59855a9102ab 100644 (file)
--- a/gst/videoparsers/gsth264parse.h
+++ b/gst/videoparsers/gsth264parse.h
@@ -141,6 +141,11 @@ struct _GstH264Parse
   /* For insertion of AU Delimiter */
   gboolean aud_needed;
   gboolean aud_insert;
+
+  /* pending closed captions */
+  guint8 closedcaptions[96];
+  guint closedcaptions_size;
+  GstVideoCaptionType closedcaptions_type;
 };
 
 struct _GstH264ParseClass