1 /* GStreamer H.264 Parser
2 * Copyright (C) <2010> Collabora ltd
3 * Copyright (C) <2010> Nokia Corporation
4 * Copyright (C) <2011> Intel Corporation
6 * Copyright (C) <2010> Mark Nauwelaerts <mark.nauwelaerts@collabora.co.uk>
7 * Copyright (C) <2011> Thibault Saunier <thibault.saunier@collabora.com>
9 * This library is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Library General Public
11 * License as published by the Free Software Foundation; either
12 * version 2 of the License, or (at your option) any later version.
14 * This library is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Library General Public License for more details.
19 * You should have received a copy of the GNU Library General Public
20 * License along with this library; if not, write to the
21 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
22 * Boston, MA 02110-1301, USA.
29 #include <gst/base/base.h>
30 #include <gst/pbutils/pbutils.h>
31 #include <gst/video/video.h>
32 #include "gsth264parse.h"
36 GST_DEBUG_CATEGORY (h264_parse_debug);
37 #define GST_CAT_DEFAULT h264_parse_debug
39 #define DEFAULT_CONFIG_INTERVAL (0)
49 GST_H264_PARSE_FORMAT_NONE,
50 GST_H264_PARSE_FORMAT_AVC,
51 GST_H264_PARSE_FORMAT_BYTE,
52 GST_H264_PARSE_FORMAT_AVC3
57 GST_H264_PARSE_ALIGN_NONE = 0,
58 GST_H264_PARSE_ALIGN_NAL,
59 GST_H264_PARSE_ALIGN_AU
64 GST_H264_PARSE_STATE_GOT_SPS = 1 << 0,
65 GST_H264_PARSE_STATE_GOT_PPS = 1 << 1,
66 GST_H264_PARSE_STATE_GOT_SLICE = 1 << 2,
68 GST_H264_PARSE_STATE_VALID_PICTURE_HEADERS = (GST_H264_PARSE_STATE_GOT_SPS |
69 GST_H264_PARSE_STATE_GOT_PPS),
70 GST_H264_PARSE_STATE_VALID_PICTURE =
71 (GST_H264_PARSE_STATE_VALID_PICTURE_HEADERS |
72 GST_H264_PARSE_STATE_GOT_SLICE)
75 #define GST_H264_PARSE_STATE_VALID(parse, expected_state) \
76 (((parse)->state & (expected_state)) == (expected_state))
78 static GstStaticPadTemplate sinktemplate = GST_STATIC_PAD_TEMPLATE ("sink",
81 GST_STATIC_CAPS ("video/x-h264"));
83 static GstStaticPadTemplate srctemplate = GST_STATIC_PAD_TEMPLATE ("src",
86 GST_STATIC_CAPS ("video/x-h264, parsed = (boolean) true, "
87 "stream-format=(string) { avc, avc3, byte-stream }, "
88 "alignment=(string) { au, nal }"));
90 #define parent_class gst_h264_parse_parent_class
91 G_DEFINE_TYPE (GstH264Parse, gst_h264_parse, GST_TYPE_BASE_PARSE);
93 static void gst_h264_parse_finalize (GObject * object);
95 static gboolean gst_h264_parse_start (GstBaseParse * parse);
96 static gboolean gst_h264_parse_stop (GstBaseParse * parse);
97 static GstFlowReturn gst_h264_parse_handle_frame (GstBaseParse * parse,
98 GstBaseParseFrame * frame, gint * skipsize);
99 static GstFlowReturn gst_h264_parse_parse_frame (GstBaseParse * parse,
100 GstBaseParseFrame * frame);
101 static GstFlowReturn gst_h264_parse_pre_push_frame (GstBaseParse * parse,
102 GstBaseParseFrame * frame);
104 static void gst_h264_parse_set_property (GObject * object, guint prop_id,
105 const GValue * value, GParamSpec * pspec);
106 static void gst_h264_parse_get_property (GObject * object, guint prop_id,
107 GValue * value, GParamSpec * pspec);
109 static gboolean gst_h264_parse_set_caps (GstBaseParse * parse, GstCaps * caps);
110 static GstCaps *gst_h264_parse_get_caps (GstBaseParse * parse,
112 static gboolean gst_h264_parse_event (GstBaseParse * parse, GstEvent * event);
113 static gboolean gst_h264_parse_src_event (GstBaseParse * parse,
115 static void gst_h264_parse_update_src_caps (GstH264Parse * h264parse,
119 gst_h264_parse_class_init (GstH264ParseClass * klass)
121 GObjectClass *gobject_class = (GObjectClass *) klass;
122 GstBaseParseClass *parse_class = GST_BASE_PARSE_CLASS (klass);
123 GstElementClass *gstelement_class = GST_ELEMENT_CLASS (klass);
125 GST_DEBUG_CATEGORY_INIT (h264_parse_debug, "h264parse", 0, "h264 parser");
127 gobject_class->finalize = gst_h264_parse_finalize;
128 gobject_class->set_property = gst_h264_parse_set_property;
129 gobject_class->get_property = gst_h264_parse_get_property;
131 g_object_class_install_property (gobject_class, PROP_CONFIG_INTERVAL,
132 g_param_spec_int ("config-interval",
133 "SPS PPS Send Interval",
134 "Send SPS and PPS Insertion Interval in seconds (sprop parameter sets "
135 "will be multiplexed in the data stream when detected.) "
136 "(0 = disabled, -1 = send with every IDR frame)",
137 -1, 3600, DEFAULT_CONFIG_INTERVAL,
138 G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS));
140 /* Override BaseParse vfuncs */
141 parse_class->start = GST_DEBUG_FUNCPTR (gst_h264_parse_start);
142 parse_class->stop = GST_DEBUG_FUNCPTR (gst_h264_parse_stop);
143 parse_class->handle_frame = GST_DEBUG_FUNCPTR (gst_h264_parse_handle_frame);
144 parse_class->pre_push_frame =
145 GST_DEBUG_FUNCPTR (gst_h264_parse_pre_push_frame);
146 parse_class->set_sink_caps = GST_DEBUG_FUNCPTR (gst_h264_parse_set_caps);
147 parse_class->get_sink_caps = GST_DEBUG_FUNCPTR (gst_h264_parse_get_caps);
148 parse_class->sink_event = GST_DEBUG_FUNCPTR (gst_h264_parse_event);
149 parse_class->src_event = GST_DEBUG_FUNCPTR (gst_h264_parse_src_event);
151 gst_element_class_add_static_pad_template (gstelement_class, &srctemplate);
152 gst_element_class_add_static_pad_template (gstelement_class, &sinktemplate);
154 gst_element_class_set_static_metadata (gstelement_class, "H.264 parser",
155 "Codec/Parser/Converter/Video",
156 "Parses H.264 streams",
157 "Mark Nauwelaerts <mark.nauwelaerts@collabora.co.uk>");
161 gst_h264_parse_init (GstH264Parse * h264parse)
163 h264parse->frame_out = gst_adapter_new ();
164 gst_base_parse_set_pts_interpolation (GST_BASE_PARSE (h264parse), FALSE);
165 GST_PAD_SET_ACCEPT_INTERSECT (GST_BASE_PARSE_SINK_PAD (h264parse));
166 GST_PAD_SET_ACCEPT_TEMPLATE (GST_BASE_PARSE_SINK_PAD (h264parse));
168 h264parse->aud_needed = TRUE;
169 h264parse->aud_insert = TRUE;
174 gst_h264_parse_finalize (GObject * object)
176 GstH264Parse *h264parse = GST_H264_PARSE (object);
178 g_object_unref (h264parse->frame_out);
180 G_OBJECT_CLASS (parent_class)->finalize (object);
184 gst_h264_parse_reset_frame (GstH264Parse * h264parse)
186 GST_DEBUG_OBJECT (h264parse, "reset frame");
188 /* done parsing; reset state */
189 h264parse->current_off = -1;
191 h264parse->picture_start = FALSE;
192 h264parse->update_caps = FALSE;
193 h264parse->idr_pos = -1;
194 h264parse->sei_pos = -1;
195 h264parse->keyframe = FALSE;
196 h264parse->predicted = FALSE;
197 h264parse->bidirectional = FALSE;
198 h264parse->header = FALSE;
199 h264parse->frame_start = FALSE;
200 h264parse->aud_insert = TRUE;
201 h264parse->have_sps_in_frame = FALSE;
202 h264parse->have_pps_in_frame = FALSE;
203 gst_adapter_clear (h264parse->frame_out);
207 gst_h264_parse_reset_stream_info (GstH264Parse * h264parse)
211 h264parse->width = 0;
212 h264parse->height = 0;
213 h264parse->fps_num = 0;
214 h264parse->fps_den = 0;
215 h264parse->upstream_par_n = -1;
216 h264parse->upstream_par_d = -1;
217 h264parse->parsed_par_n = 0;
218 h264parse->parsed_par_d = 0;
219 h264parse->parsed_colorimetry.range = GST_VIDEO_COLOR_RANGE_UNKNOWN;
220 h264parse->parsed_colorimetry.matrix = GST_VIDEO_COLOR_MATRIX_UNKNOWN;
221 h264parse->parsed_colorimetry.transfer = GST_VIDEO_TRANSFER_UNKNOWN;
222 h264parse->parsed_colorimetry.primaries = GST_VIDEO_COLOR_PRIMARIES_UNKNOWN;
224 h264parse->have_pps = FALSE;
225 h264parse->have_sps = FALSE;
227 h264parse->multiview_mode = GST_VIDEO_MULTIVIEW_MODE_NONE;
228 h264parse->multiview_flags = GST_VIDEO_MULTIVIEW_FLAGS_NONE;
229 h264parse->first_in_bundle = TRUE;
231 h264parse->align = GST_H264_PARSE_ALIGN_NONE;
232 h264parse->format = GST_H264_PARSE_FORMAT_NONE;
234 h264parse->transform = FALSE;
235 h264parse->nal_length_size = 4;
236 h264parse->packetized = FALSE;
237 h264parse->push_codec = FALSE;
239 gst_buffer_replace (&h264parse->codec_data, NULL);
240 gst_buffer_replace (&h264parse->codec_data_in, NULL);
242 gst_h264_parse_reset_frame (h264parse);
244 for (i = 0; i < GST_H264_MAX_SPS_COUNT; i++)
245 gst_buffer_replace (&h264parse->sps_nals[i], NULL);
246 for (i = 0; i < GST_H264_MAX_PPS_COUNT; i++)
247 gst_buffer_replace (&h264parse->pps_nals[i], NULL);
251 gst_h264_parse_reset (GstH264Parse * h264parse)
253 h264parse->last_report = GST_CLOCK_TIME_NONE;
255 h264parse->dts = GST_CLOCK_TIME_NONE;
256 h264parse->ts_trn_nb = GST_CLOCK_TIME_NONE;
257 h264parse->do_ts = TRUE;
259 h264parse->sent_codec_tag = FALSE;
261 h264parse->pending_key_unit_ts = GST_CLOCK_TIME_NONE;
262 gst_event_replace (&h264parse->force_key_unit_event, NULL);
264 h264parse->discont = FALSE;
265 h264parse->discard_bidirectional = FALSE;
267 gst_h264_parse_reset_stream_info (h264parse);
271 gst_h264_parse_start (GstBaseParse * parse)
273 GstH264Parse *h264parse = GST_H264_PARSE (parse);
275 GST_DEBUG_OBJECT (parse, "start");
276 gst_h264_parse_reset (h264parse);
278 h264parse->nalparser = gst_h264_nal_parser_new ();
280 h264parse->state = 0;
281 h264parse->dts = GST_CLOCK_TIME_NONE;
282 h264parse->ts_trn_nb = GST_CLOCK_TIME_NONE;
283 h264parse->sei_pic_struct_pres_flag = FALSE;
284 h264parse->sei_pic_struct = 0;
285 h264parse->field_pic_flag = 0;
287 gst_base_parse_set_min_frame_size (parse, 4);
293 gst_h264_parse_stop (GstBaseParse * parse)
295 GstH264Parse *h264parse = GST_H264_PARSE (parse);
297 GST_DEBUG_OBJECT (parse, "stop");
298 gst_h264_parse_reset (h264parse);
300 gst_h264_nal_parser_free (h264parse->nalparser);
306 gst_h264_parse_get_string (GstH264Parse * parse, gboolean format, gint code)
310 case GST_H264_PARSE_FORMAT_AVC:
312 case GST_H264_PARSE_FORMAT_BYTE:
313 return "byte-stream";
314 case GST_H264_PARSE_FORMAT_AVC3:
321 case GST_H264_PARSE_ALIGN_NAL:
323 case GST_H264_PARSE_ALIGN_AU:
332 gst_h264_parse_format_from_caps (GstCaps * caps, guint * format, guint * align)
335 *format = GST_H264_PARSE_FORMAT_NONE;
338 *align = GST_H264_PARSE_ALIGN_NONE;
340 g_return_if_fail (gst_caps_is_fixed (caps));
342 GST_DEBUG ("parsing caps: %" GST_PTR_FORMAT, caps);
344 if (caps && gst_caps_get_size (caps) > 0) {
345 GstStructure *s = gst_caps_get_structure (caps, 0);
346 const gchar *str = NULL;
349 if ((str = gst_structure_get_string (s, "stream-format"))) {
350 if (strcmp (str, "avc") == 0)
351 *format = GST_H264_PARSE_FORMAT_AVC;
352 else if (strcmp (str, "byte-stream") == 0)
353 *format = GST_H264_PARSE_FORMAT_BYTE;
354 else if (strcmp (str, "avc3") == 0)
355 *format = GST_H264_PARSE_FORMAT_AVC3;
360 if ((str = gst_structure_get_string (s, "alignment"))) {
361 if (strcmp (str, "au") == 0)
362 *align = GST_H264_PARSE_ALIGN_AU;
363 else if (strcmp (str, "nal") == 0)
364 *align = GST_H264_PARSE_ALIGN_NAL;
370 /* check downstream caps to configure format and alignment */
372 gst_h264_parse_negotiate (GstH264Parse * h264parse, gint in_format,
376 guint format = h264parse->format;
377 guint align = h264parse->align;
379 g_return_if_fail ((in_caps == NULL) || gst_caps_is_fixed (in_caps));
381 caps = gst_pad_get_allowed_caps (GST_BASE_PARSE_SRC_PAD (h264parse));
382 GST_DEBUG_OBJECT (h264parse, "allowed caps: %" GST_PTR_FORMAT, caps);
384 /* concentrate on leading structure, since decodebin parser
385 * capsfilter always includes parser template caps */
387 caps = gst_caps_truncate (caps);
388 GST_DEBUG_OBJECT (h264parse, "negotiating with caps: %" GST_PTR_FORMAT,
392 h264parse->can_passthrough = FALSE;
394 if (in_caps && caps) {
395 if (gst_caps_can_intersect (in_caps, caps)) {
396 GST_DEBUG_OBJECT (h264parse, "downstream accepts upstream caps");
397 gst_h264_parse_format_from_caps (in_caps, &format, &align);
398 gst_caps_unref (caps);
400 h264parse->can_passthrough = TRUE;
404 /* FIXME We could fail the negotiation immediatly if caps are empty */
405 if (caps && !gst_caps_is_empty (caps)) {
406 /* fixate to avoid ambiguity with lists when parsing */
407 caps = gst_caps_fixate (caps);
408 gst_h264_parse_format_from_caps (caps, &format, &align);
413 format = GST_H264_PARSE_FORMAT_BYTE;
415 align = GST_H264_PARSE_ALIGN_AU;
417 GST_DEBUG_OBJECT (h264parse, "selected format %s, alignment %s",
418 gst_h264_parse_get_string (h264parse, TRUE, format),
419 gst_h264_parse_get_string (h264parse, FALSE, align));
421 h264parse->format = format;
422 h264parse->align = align;
424 h264parse->transform = in_format != h264parse->format ||
425 align == GST_H264_PARSE_ALIGN_AU;
428 gst_caps_unref (caps);
432 gst_h264_parse_wrap_nal (GstH264Parse * h264parse, guint format, guint8 * data,
436 guint nl = h264parse->nal_length_size;
439 GST_DEBUG_OBJECT (h264parse, "nal length %d", size);
441 buf = gst_buffer_new_allocate (NULL, 4 + size, NULL);
442 if (format == GST_H264_PARSE_FORMAT_AVC
443 || format == GST_H264_PARSE_FORMAT_AVC3) {
444 tmp = GUINT32_TO_BE (size << (32 - 8 * nl));
446 /* HACK: nl should always be 4 here, otherwise this won't work.
447 * There are legit cases where nl in avc stream is 2, but byte-stream
448 * SC is still always 4 bytes. */
450 tmp = GUINT32_TO_BE (1);
453 gst_buffer_fill (buf, 0, &tmp, sizeof (guint32));
454 gst_buffer_fill (buf, nl, data, size);
455 gst_buffer_set_size (buf, size + nl);
461 gst_h264_parser_store_nal (GstH264Parse * h264parse, guint id,
462 GstH264NalUnitType naltype, GstH264NalUnit * nalu)
464 GstBuffer *buf, **store;
465 guint size = nalu->size, store_size;
467 if (naltype == GST_H264_NAL_SPS || naltype == GST_H264_NAL_SUBSET_SPS) {
468 store_size = GST_H264_MAX_SPS_COUNT;
469 store = h264parse->sps_nals;
470 GST_DEBUG_OBJECT (h264parse, "storing sps %u", id);
471 } else if (naltype == GST_H264_NAL_PPS) {
472 store_size = GST_H264_MAX_PPS_COUNT;
473 store = h264parse->pps_nals;
474 GST_DEBUG_OBJECT (h264parse, "storing pps %u", id);
478 if (id >= store_size) {
479 GST_DEBUG_OBJECT (h264parse, "unable to store nal, id out-of-range %d", id);
483 buf = gst_buffer_new_allocate (NULL, size, NULL);
484 gst_buffer_fill (buf, 0, nalu->data + nalu->offset, size);
486 /* Indicate that buffer contain a header needed for decoding */
487 if (naltype == GST_H264_NAL_SPS || naltype == GST_H264_NAL_PPS)
488 GST_BUFFER_FLAG_SET (buf, GST_BUFFER_FLAG_HEADER);
491 gst_buffer_unref (store[id]);
496 #ifndef GST_DISABLE_GST_DEBUG
497 static const gchar *nal_names[] = {
514 "Depth Parameter Set",
515 "Reserved", "Reserved",
516 "Slice Aux Unpartitioned",
518 "Slice Depth/3D-AVC Extension"
522 _nal_name (GstH264NalUnitType nal_type)
524 if (nal_type <= GST_H264_NAL_SLICE_DEPTH)
525 return nal_names[nal_type];
531 gst_h264_parse_process_sei_user_data (GstH264Parse * h264parse,
532 GstH264RegisteredUserData * rud)
534 guint16 provider_code;
536 GstVideoParseUtilsField field = GST_VIDEO_PARSE_UTILS_FIELD_1;
538 /* only US country code is currently supported */
539 switch (rud->country_code) {
540 case ITU_T_T35_COUNTRY_CODE_US:
543 GST_LOG_OBJECT (h264parse, "Unsupported country code %d",
548 if (rud->data == NULL || rud->size < 2)
551 gst_byte_reader_init (&br, rud->data, rud->size);
553 provider_code = gst_byte_reader_get_uint16_be_unchecked (&br);
555 if (h264parse->sei_pic_struct ==
556 (guint8) GST_H264_SEI_PIC_STRUCT_BOTTOM_FIELD)
557 field = GST_VIDEO_PARSE_UTILS_FIELD_2;
558 gst_video_parse_user_data ((GstElement *) h264parse, &h264parse->user_data,
559 &br, field, provider_code);
564 gst_h264_parse_process_sei (GstH264Parse * h264parse, GstH264NalUnit * nalu)
566 GstH264SEIMessage sei;
567 GstH264NalParser *nalparser = h264parse->nalparser;
568 GstH264ParserResult pres;
572 pres = gst_h264_parser_parse_sei (nalparser, nalu, &messages);
573 if (pres != GST_H264_PARSER_OK)
574 GST_WARNING_OBJECT (h264parse, "failed to parse one or more SEI message");
576 /* Even if pres != GST_H264_PARSER_OK, some message could have been parsed and
577 * stored in messages.
579 for (i = 0; i < messages->len; i++) {
580 sei = g_array_index (messages, GstH264SEIMessage, i);
581 switch (sei.payloadType) {
582 case GST_H264_SEI_PIC_TIMING:
585 h264parse->sei_pic_struct_pres_flag =
586 sei.payload.pic_timing.pic_struct_present_flag;
587 h264parse->sei_cpb_removal_delay =
588 sei.payload.pic_timing.cpb_removal_delay;
589 if (h264parse->sei_pic_struct_pres_flag) {
590 h264parse->sei_pic_struct = sei.payload.pic_timing.pic_struct;
593 h264parse->num_clock_timestamp = 0;
595 for (j = 0; j < 3; j++) {
596 if (sei.payload.pic_timing.clock_timestamp_flag[j]) {
597 memcpy (&h264parse->clock_timestamp[h264parse->
598 num_clock_timestamp++],
599 &sei.payload.pic_timing.clock_timestamp[j],
600 sizeof (GstH264ClockTimestamp));
604 GST_LOG_OBJECT (h264parse, "pic timing updated");
607 case GST_H264_SEI_REGISTERED_USER_DATA:
608 gst_h264_parse_process_sei_user_data (h264parse,
609 &sei.payload.registered_user_data);
611 case GST_H264_SEI_BUF_PERIOD:
612 if (h264parse->ts_trn_nb == GST_CLOCK_TIME_NONE ||
613 h264parse->dts == GST_CLOCK_TIME_NONE)
614 h264parse->ts_trn_nb = 0;
616 h264parse->ts_trn_nb = h264parse->dts;
618 GST_LOG_OBJECT (h264parse,
619 "new buffering period; ts_trn_nb updated: %" GST_TIME_FORMAT,
620 GST_TIME_ARGS (h264parse->ts_trn_nb));
623 /* Additional messages that are not innerly useful to the
624 * element but for debugging purposes */
625 case GST_H264_SEI_RECOVERY_POINT:
626 GST_LOG_OBJECT (h264parse, "recovery point found: %u %u %u %u",
627 sei.payload.recovery_point.recovery_frame_cnt,
628 sei.payload.recovery_point.exact_match_flag,
629 sei.payload.recovery_point.broken_link_flag,
630 sei.payload.recovery_point.changing_slice_group_idc);
631 h264parse->keyframe = TRUE;
634 /* Additional messages that are not innerly useful to the
635 * element but for debugging purposes */
636 case GST_H264_SEI_STEREO_VIDEO_INFO:{
637 GstVideoMultiviewMode mview_mode = GST_VIDEO_MULTIVIEW_MODE_NONE;
638 GstVideoMultiviewFlags mview_flags = GST_VIDEO_MULTIVIEW_FLAGS_NONE;
640 GST_LOG_OBJECT (h264parse, "Stereo video information %u %u %u %u %u %u",
641 sei.payload.stereo_video_info.field_views_flag,
642 sei.payload.stereo_video_info.top_field_is_left_view_flag,
643 sei.payload.stereo_video_info.current_frame_is_left_view_flag,
644 sei.payload.stereo_video_info.next_frame_is_second_view_flag,
645 sei.payload.stereo_video_info.left_view_self_contained_flag,
646 sei.payload.stereo_video_info.right_view_self_contained_flag);
648 if (sei.payload.stereo_video_info.field_views_flag) {
649 mview_mode = GST_VIDEO_MULTIVIEW_MODE_ROW_INTERLEAVED;
650 if (!sei.payload.stereo_video_info.top_field_is_left_view_flag)
651 mview_flags |= GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST;
653 mview_mode = GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME;
654 if (sei.payload.stereo_video_info.next_frame_is_second_view_flag) {
655 /* Mark current frame as first in bundle */
656 h264parse->first_in_bundle = TRUE;
657 if (!sei.payload.stereo_video_info.current_frame_is_left_view_flag)
658 mview_flags |= GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST;
661 if (mview_mode != h264parse->multiview_mode ||
662 mview_flags != h264parse->multiview_flags) {
663 h264parse->multiview_mode = mview_mode;
664 h264parse->multiview_flags = mview_flags;
665 /* output caps need to be changed */
666 gst_h264_parse_update_src_caps (h264parse, NULL);
670 case GST_H264_SEI_FRAME_PACKING:{
671 GstVideoMultiviewMode mview_mode = GST_VIDEO_MULTIVIEW_MODE_NONE;
672 GstVideoMultiviewFlags mview_flags = GST_VIDEO_MULTIVIEW_FLAGS_NONE;
674 GST_LOG_OBJECT (h264parse,
675 "frame packing arrangement message: id %u cancelled %u "
676 "type %u quincunx %u content_interpretation %d flip %u "
677 "right_first %u field_views %u is_frame0 %u",
678 sei.payload.frame_packing.frame_packing_id,
679 sei.payload.frame_packing.frame_packing_cancel_flag,
680 sei.payload.frame_packing.frame_packing_type,
681 sei.payload.frame_packing.quincunx_sampling_flag,
682 sei.payload.frame_packing.content_interpretation_type,
683 sei.payload.frame_packing.spatial_flipping_flag,
684 sei.payload.frame_packing.frame0_flipped_flag,
685 sei.payload.frame_packing.field_views_flag,
686 sei.payload.frame_packing.current_frame_is_frame0_flag);
688 /* Only IDs from 0->255 and 512->2^31-1 are valid. Ignore others */
689 if ((sei.payload.frame_packing.frame_packing_id >= 256 &&
690 sei.payload.frame_packing.frame_packing_id < 512) ||
691 (sei.payload.frame_packing.frame_packing_id >= (1U << 31)))
694 if (!sei.payload.frame_packing.frame_packing_cancel_flag) {
695 /* Cancel flag sets things back to no-info */
697 if (sei.payload.frame_packing.content_interpretation_type == 2)
698 mview_flags |= GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST;
700 switch (sei.payload.frame_packing.frame_packing_type) {
702 mview_mode = GST_VIDEO_MULTIVIEW_MODE_CHECKERBOARD;
705 mview_mode = GST_VIDEO_MULTIVIEW_MODE_COLUMN_INTERLEAVED;
708 mview_mode = GST_VIDEO_MULTIVIEW_MODE_ROW_INTERLEAVED;
711 if (sei.payload.frame_packing.quincunx_sampling_flag)
712 mview_mode = GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE_QUINCUNX;
714 mview_mode = GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE;
715 if (sei.payload.frame_packing.spatial_flipping_flag) {
716 /* One of the views is flopped. */
717 if (sei.payload.frame_packing.frame0_flipped_flag !=
719 GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST))
720 /* the left view is flopped */
721 mview_flags |= GST_VIDEO_MULTIVIEW_FLAGS_LEFT_FLOPPED;
723 mview_flags |= GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_FLOPPED;
727 mview_mode = GST_VIDEO_MULTIVIEW_MODE_TOP_BOTTOM;
728 if (sei.payload.frame_packing.spatial_flipping_flag) {
729 /* One of the views is flipped, */
730 if (sei.payload.frame_packing.frame0_flipped_flag !=
732 GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST))
733 /* the left view is flipped */
734 mview_flags |= GST_VIDEO_MULTIVIEW_FLAGS_LEFT_FLIPPED;
736 mview_flags |= GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_FLIPPED;
740 if (sei.payload.frame_packing.content_interpretation_type == 0)
741 mview_mode = GST_VIDEO_MULTIVIEW_MODE_MULTIVIEW_FRAME_BY_FRAME;
743 mview_mode = GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME;
746 GST_DEBUG_OBJECT (h264parse, "Invalid frame packing type %u",
747 sei.payload.frame_packing.frame_packing_type);
752 if (mview_mode != h264parse->multiview_mode ||
753 mview_flags != h264parse->multiview_flags) {
754 h264parse->multiview_mode = mview_mode;
755 h264parse->multiview_flags = mview_flags;
756 /* output caps need to be changed */
757 gst_h264_parse_update_src_caps (h264parse, NULL);
763 g_array_free (messages, TRUE);
766 /* caller guarantees 2 bytes of nal payload */
768 gst_h264_parse_process_nal (GstH264Parse * h264parse, GstH264NalUnit * nalu)
771 GstH264PPS pps = { 0, };
772 GstH264SPS sps = { 0, };
773 GstH264NalParser *nalparser = h264parse->nalparser;
774 GstH264ParserResult pres;
776 /* nothing to do for broken input */
777 if (G_UNLIKELY (nalu->size < 2)) {
778 GST_DEBUG_OBJECT (h264parse, "not processing nal size %u", nalu->size);
782 /* we have a peek as well */
783 nal_type = nalu->type;
785 GST_DEBUG_OBJECT (h264parse, "processing nal of type %u %s, size %u",
786 nal_type, _nal_name (nal_type), nalu->size);
789 case GST_H264_NAL_SUBSET_SPS:
790 if (!GST_H264_PARSE_STATE_VALID (h264parse, GST_H264_PARSE_STATE_GOT_SPS))
792 pres = gst_h264_parser_parse_subset_sps (nalparser, nalu, &sps);
795 case GST_H264_NAL_SPS:
796 /* reset state, everything else is obsolete */
797 h264parse->state = 0;
798 pres = gst_h264_parser_parse_sps (nalparser, nalu, &sps);
801 /* arranged for a fallback sps.id, so use that one and only warn */
802 if (pres != GST_H264_PARSER_OK) {
803 GST_WARNING_OBJECT (h264parse, "failed to parse SPS:");
804 h264parse->state |= GST_H264_PARSE_STATE_GOT_SPS;
805 h264parse->header = TRUE;
809 GST_DEBUG_OBJECT (h264parse, "triggering src caps check");
810 h264parse->update_caps = TRUE;
811 h264parse->have_sps = TRUE;
812 h264parse->have_sps_in_frame = TRUE;
813 if (h264parse->push_codec && h264parse->have_pps) {
814 /* SPS and PPS found in stream before the first pre_push_frame, no need
815 * to forcibly push at start */
816 GST_INFO_OBJECT (h264parse, "have SPS/PPS in stream");
817 h264parse->push_codec = FALSE;
818 h264parse->have_sps = FALSE;
819 h264parse->have_pps = FALSE;
822 gst_h264_parser_store_nal (h264parse, sps.id, nal_type, nalu);
823 gst_h264_sps_clear (&sps);
824 h264parse->state |= GST_H264_PARSE_STATE_GOT_SPS;
825 h264parse->header = TRUE;
827 case GST_H264_NAL_PPS:
828 /* expected state: got-sps */
829 h264parse->state &= GST_H264_PARSE_STATE_GOT_SPS;
830 if (!GST_H264_PARSE_STATE_VALID (h264parse, GST_H264_PARSE_STATE_GOT_SPS))
833 pres = gst_h264_parser_parse_pps (nalparser, nalu, &pps);
834 /* arranged for a fallback pps.id, so use that one and only warn */
835 if (pres != GST_H264_PARSER_OK) {
836 GST_WARNING_OBJECT (h264parse, "failed to parse PPS:");
837 if (pres != GST_H264_PARSER_BROKEN_LINK)
841 /* parameters might have changed, force caps check */
842 if (!h264parse->have_pps) {
843 GST_DEBUG_OBJECT (h264parse, "triggering src caps check");
844 h264parse->update_caps = TRUE;
846 h264parse->have_pps = TRUE;
847 h264parse->have_pps_in_frame = TRUE;
848 if (h264parse->push_codec && h264parse->have_sps) {
849 /* SPS and PPS found in stream before the first pre_push_frame, no need
850 * to forcibly push at start */
851 GST_INFO_OBJECT (h264parse, "have SPS/PPS in stream");
852 h264parse->push_codec = FALSE;
853 h264parse->have_sps = FALSE;
854 h264parse->have_pps = FALSE;
857 gst_h264_parser_store_nal (h264parse, pps.id, nal_type, nalu);
858 gst_h264_pps_clear (&pps);
859 h264parse->state |= GST_H264_PARSE_STATE_GOT_PPS;
860 h264parse->header = TRUE;
862 case GST_H264_NAL_SEI:
863 /* expected state: got-sps */
864 if (!GST_H264_PARSE_STATE_VALID (h264parse, GST_H264_PARSE_STATE_GOT_SPS))
867 h264parse->header = TRUE;
868 gst_h264_parse_process_sei (h264parse, nalu);
870 if (h264parse->sei_pos == -1) {
871 if (h264parse->transform)
872 h264parse->sei_pos = gst_adapter_available (h264parse->frame_out);
874 h264parse->sei_pos = nalu->sc_offset;
875 GST_DEBUG_OBJECT (h264parse, "marking SEI in frame at offset %d",
880 case GST_H264_NAL_SLICE:
881 case GST_H264_NAL_SLICE_DPA:
882 case GST_H264_NAL_SLICE_DPB:
883 case GST_H264_NAL_SLICE_DPC:
884 case GST_H264_NAL_SLICE_IDR:
885 case GST_H264_NAL_SLICE_EXT:
886 /* expected state: got-sps|got-pps (valid picture headers) */
887 h264parse->state &= GST_H264_PARSE_STATE_VALID_PICTURE_HEADERS;
888 if (!GST_H264_PARSE_STATE_VALID (h264parse,
889 GST_H264_PARSE_STATE_VALID_PICTURE_HEADERS))
892 /* don't need to parse the whole slice (header) here */
893 if (*(nalu->data + nalu->offset + nalu->header_bytes) & 0x80) {
894 /* means first_mb_in_slice == 0 */
895 /* real frame data */
896 GST_DEBUG_OBJECT (h264parse, "first_mb_in_slice = 0");
897 h264parse->frame_start = TRUE;
899 GST_DEBUG_OBJECT (h264parse, "frame start: %i", h264parse->frame_start);
900 if (nal_type == GST_H264_NAL_SLICE_EXT && !GST_H264_IS_MVC_NALU (nalu))
903 GstH264SliceHdr slice;
905 pres = gst_h264_parser_parse_slice_hdr (nalparser, nalu, &slice,
907 GST_DEBUG_OBJECT (h264parse,
908 "parse result %d, first MB: %u, slice type: %u",
909 pres, slice.first_mb_in_slice, slice.type);
910 if (pres == GST_H264_PARSER_OK) {
911 if (GST_H264_IS_I_SLICE (&slice) || GST_H264_IS_SI_SLICE (&slice))
912 h264parse->keyframe = TRUE;
913 else if (GST_H264_IS_P_SLICE (&slice)
914 || GST_H264_IS_SP_SLICE (&slice))
915 h264parse->predicted = TRUE;
916 else if (GST_H264_IS_B_SLICE (&slice))
917 h264parse->bidirectional = TRUE;
919 h264parse->state |= GST_H264_PARSE_STATE_GOT_SLICE;
920 h264parse->field_pic_flag = slice.field_pic_flag;
923 if (G_LIKELY (nal_type != GST_H264_NAL_SLICE_IDR &&
924 !h264parse->push_codec))
926 /* if we need to sneak codec NALs into the stream,
927 * this is a good place, so fake it as IDR
928 * (which should be at start anyway) */
929 /* mark where config needs to go if interval expired */
930 /* mind replacement buffer if applicable */
931 if (h264parse->idr_pos == -1) {
932 if (h264parse->transform)
933 h264parse->idr_pos = gst_adapter_available (h264parse->frame_out);
935 h264parse->idr_pos = nalu->sc_offset;
936 GST_DEBUG_OBJECT (h264parse, "marking IDR in frame at offset %d",
939 /* if SEI preceeds (faked) IDR, then we have to insert config there */
940 if (h264parse->sei_pos >= 0 && h264parse->idr_pos > h264parse->sei_pos) {
941 h264parse->idr_pos = h264parse->sei_pos;
942 GST_DEBUG_OBJECT (h264parse, "moved IDR mark to SEI position %d",
946 case GST_H264_NAL_AU_DELIMITER:
947 /* Just accumulate AU Delimiter, whether it's before SPS or not */
948 pres = gst_h264_parser_parse_nal (nalparser, nalu);
949 if (pres != GST_H264_PARSER_OK)
951 h264parse->aud_insert = FALSE;
954 /* drop anything before the initial SPS */
955 if (!GST_H264_PARSE_STATE_VALID (h264parse, GST_H264_PARSE_STATE_GOT_SPS))
958 pres = gst_h264_parser_parse_nal (nalparser, nalu);
959 if (pres != GST_H264_PARSER_OK)
964 /* if AVC output needed, collect properly prefixed nal in adapter,
965 * and use that to replace outgoing buffer data later on */
966 if (h264parse->transform) {
969 GST_LOG_OBJECT (h264parse, "collecting NAL in AVC frame");
970 buf = gst_h264_parse_wrap_nal (h264parse, h264parse->format,
971 nalu->data + nalu->offset, nalu->size);
972 gst_adapter_push (h264parse->frame_out, buf);
977 /* caller guarantees at least 2 bytes of nal payload for each nal
978 * returns TRUE if next_nal indicates that nal terminates an AU */
979 static inline gboolean
980 gst_h264_parse_collect_nal (GstH264Parse * h264parse, const guint8 * data,
981 guint size, GstH264NalUnit * nalu)
984 GstH264ParserResult parse_res;
985 GstH264NalUnitType nal_type = nalu->type;
986 GstH264NalUnit nnalu;
988 GST_DEBUG_OBJECT (h264parse, "parsing collected nal");
989 parse_res = gst_h264_parser_identify_nalu_unchecked (h264parse->nalparser,
990 data, nalu->offset + nalu->size, size, &nnalu);
992 if (parse_res != GST_H264_PARSER_OK)
995 /* determine if AU complete */
996 GST_LOG_OBJECT (h264parse, "nal type: %d %s", nal_type, _nal_name (nal_type));
997 /* coded slice NAL starts a picture,
998 * i.e. other types become aggregated in front of it */
999 h264parse->picture_start |= (nal_type == GST_H264_NAL_SLICE ||
1000 nal_type == GST_H264_NAL_SLICE_DPA || nal_type == GST_H264_NAL_SLICE_IDR);
1002 /* consider a coded slices (IDR or not) to start a picture,
1003 * (so ending the previous one) if first_mb_in_slice == 0
1004 * (non-0 is part of previous one) */
1005 /* NOTE this is not entirely according to Access Unit specs in 7.4.1.2.4,
1006 * but in practice it works in sane cases, needs not much parsing,
1007 * and also works with broken frame_num in NAL
1008 * (where spec-wise would fail) */
1009 nal_type = nnalu.type;
1010 complete = h264parse->picture_start && ((nal_type >= GST_H264_NAL_SEI &&
1011 nal_type <= GST_H264_NAL_AU_DELIMITER) ||
1012 (nal_type >= 14 && nal_type <= 18));
1014 GST_LOG_OBJECT (h264parse, "next nal type: %d %s", nal_type,
1015 _nal_name (nal_type));
1016 /* first_mb_in_slice == 0 considered start of frame */
1017 if (nnalu.size > nnalu.header_bytes)
1018 complete |= h264parse->picture_start && (nal_type == GST_H264_NAL_SLICE
1019 || nal_type == GST_H264_NAL_SLICE_DPA
1020 || nal_type == GST_H264_NAL_SLICE_IDR) &&
1021 (nnalu.data[nnalu.offset + nnalu.header_bytes] & 0x80);
1023 GST_LOG_OBJECT (h264parse, "au complete: %d", complete);
1028 static guint8 au_delim[6] = {
1029 0x00, 0x00, 0x00, 0x01, /* nal prefix */
1030 0x09, /* nal unit type = access unit delimiter */
1031 0xf0 /* allow any slice type */
1034 static GstFlowReturn
1035 gst_h264_parse_handle_frame_packetized (GstBaseParse * parse,
1036 GstBaseParseFrame * frame)
1038 GstH264Parse *h264parse = GST_H264_PARSE (parse);
1039 GstBuffer *buffer = frame->buffer;
1040 GstFlowReturn ret = GST_FLOW_OK;
1041 GstH264ParserResult parse_res;
1042 GstH264NalUnit nalu;
1043 const guint nl = h264parse->nal_length_size;
1047 if (nl < 1 || nl > 4) {
1048 GST_DEBUG_OBJECT (h264parse, "insufficient data to split input");
1049 return GST_FLOW_NOT_NEGOTIATED;
1052 /* need to save buffer from invalidation upon _finish_frame */
1053 if (h264parse->split_packetized)
1054 buffer = gst_buffer_copy (frame->buffer);
1056 gst_buffer_map (buffer, &map, GST_MAP_READ);
1060 GST_LOG_OBJECT (h264parse,
1061 "processing packet buffer of size %" G_GSIZE_FORMAT, map.size);
1063 parse_res = gst_h264_parser_identify_nalu_avc (h264parse->nalparser,
1064 map.data, 0, map.size, nl, &nalu);
1066 while (parse_res == GST_H264_PARSER_OK) {
1067 GST_DEBUG_OBJECT (h264parse, "AVC nal offset %d", nalu.offset + nalu.size);
1069 /* either way, have a look at it */
1070 gst_h264_parse_process_nal (h264parse, &nalu);
1072 /* dispatch per NALU if needed */
1073 if (h264parse->split_packetized) {
1074 GstBaseParseFrame tmp_frame;
1076 gst_base_parse_frame_init (&tmp_frame);
1077 tmp_frame.flags |= frame->flags;
1078 tmp_frame.offset = frame->offset;
1079 tmp_frame.overhead = frame->overhead;
1080 tmp_frame.buffer = gst_buffer_copy_region (buffer, GST_BUFFER_COPY_ALL,
1081 nalu.offset, nalu.size);
1083 /* note we don't need to come up with a sub-buffer, since
1084 * subsequent code only considers input buffer's metadata.
1085 * Real data is either taken from input by baseclass or
1086 * a replacement output buffer is provided anyway. */
1087 gst_h264_parse_parse_frame (parse, &tmp_frame);
1088 ret = gst_base_parse_finish_frame (parse, &tmp_frame, nl + nalu.size);
1089 left -= nl + nalu.size;
1092 parse_res = gst_h264_parser_identify_nalu_avc (h264parse->nalparser,
1093 map.data, nalu.offset + nalu.size, map.size, nl, &nalu);
1096 gst_buffer_unmap (buffer, &map);
1098 if (!h264parse->split_packetized) {
1099 gst_h264_parse_parse_frame (parse, frame);
1100 ret = gst_base_parse_finish_frame (parse, frame, map.size);
1102 gst_buffer_unref (buffer);
1103 if (G_UNLIKELY (left)) {
1104 /* should not be happening for nice AVC */
1105 GST_WARNING_OBJECT (parse, "skipping leftover AVC data %d", left);
1106 frame->flags |= GST_BASE_PARSE_FRAME_FLAG_DROP;
1107 ret = gst_base_parse_finish_frame (parse, frame, map.size);
1111 if (parse_res == GST_H264_PARSER_NO_NAL_END ||
1112 parse_res == GST_H264_PARSER_BROKEN_DATA) {
1114 if (h264parse->split_packetized) {
1115 GST_ELEMENT_ERROR (h264parse, STREAM, FAILED, (NULL),
1116 ("invalid AVC input data"));
1118 return GST_FLOW_ERROR;
1120 /* do not meddle to much in this case */
1121 GST_DEBUG_OBJECT (h264parse, "parsing packet failed");
1128 static GstFlowReturn
1129 gst_h264_parse_handle_frame (GstBaseParse * parse,
1130 GstBaseParseFrame * frame, gint * skipsize)
1132 GstH264Parse *h264parse = GST_H264_PARSE (parse);
1133 GstBuffer *buffer = frame->buffer;
1137 gint current_off = 0;
1138 gboolean drain, nonext;
1139 GstH264NalParser *nalparser = h264parse->nalparser;
1140 GstH264NalUnit nalu;
1141 GstH264ParserResult pres;
1144 gboolean au_complete;
1146 if (G_UNLIKELY (GST_BUFFER_FLAG_IS_SET (frame->buffer,
1147 GST_BUFFER_FLAG_DISCONT))) {
1148 h264parse->discont = TRUE;
1151 /* delegate in packetized case, no skipping should be needed */
1152 if (h264parse->packetized)
1153 return gst_h264_parse_handle_frame_packetized (parse, frame);
1155 gst_buffer_map (buffer, &map, GST_MAP_READ);
1159 /* expect at least 3 bytes start_code, and 1 bytes NALU header.
1160 * the length of the NALU payload can be zero.
1161 * (e.g. EOS/EOB placed at the end of an AU.) */
1162 if (G_UNLIKELY (size < 4)) {
1163 gst_buffer_unmap (buffer, &map);
1168 /* need to configure aggregation */
1169 if (G_UNLIKELY (h264parse->format == GST_H264_PARSE_FORMAT_NONE))
1170 gst_h264_parse_negotiate (h264parse, GST_H264_PARSE_FORMAT_BYTE, NULL);
1172 /* avoid stale cached parsing state */
1173 if (frame->flags & GST_BASE_PARSE_FRAME_FLAG_NEW_FRAME) {
1174 GST_LOG_OBJECT (h264parse, "parsing new frame");
1175 gst_h264_parse_reset_frame (h264parse);
1177 GST_LOG_OBJECT (h264parse, "resuming frame parsing");
1180 /* Always consume the entire input buffer when in_align == ALIGN_AU */
1181 drain = GST_BASE_PARSE_DRAINING (parse)
1182 || h264parse->in_align == GST_H264_PARSE_ALIGN_AU;
1185 current_off = h264parse->current_off;
1186 if (current_off < 0)
1188 g_assert (current_off < size);
1189 GST_DEBUG_OBJECT (h264parse, "last parse position %d", current_off);
1191 /* check for initial skip */
1192 if (h264parse->current_off == -1) {
1194 gst_h264_parser_identify_nalu_unchecked (nalparser, data, current_off,
1197 case GST_H264_PARSER_OK:
1198 if (nalu.sc_offset > 0) {
1200 gboolean is_filler_data = TRUE;
1201 /* Handle filler data */
1202 for (i = 0; i < nalu.sc_offset; i++) {
1203 if (data[i] != 0x00) {
1204 is_filler_data = FALSE;
1208 if (is_filler_data) {
1209 GST_DEBUG_OBJECT (parse, "Dropping filler data %d", nalu.sc_offset);
1210 frame->flags |= GST_BASE_PARSE_FRAME_FLAG_DROP;
1211 gst_buffer_unmap (buffer, &map);
1212 ret = gst_base_parse_finish_frame (parse, frame, nalu.sc_offset);
1215 *skipsize = nalu.sc_offset;
1219 case GST_H264_PARSER_NO_NAL:
1220 *skipsize = size - 3;
1224 /* should not really occur either */
1225 GST_ELEMENT_ERROR (h264parse, STREAM, FORMAT,
1226 ("Error parsing H.264 stream"), ("Invalid H.264 stream"));
1227 goto invalid_stream;
1233 gst_h264_parser_identify_nalu (nalparser, data, current_off, size,
1237 case GST_H264_PARSER_OK:
1238 GST_DEBUG_OBJECT (h264parse, "complete nal (offset, size): (%u, %u) ",
1239 nalu.offset, nalu.size);
1241 case GST_H264_PARSER_NO_NAL_END:
1242 GST_DEBUG_OBJECT (h264parse, "not a complete nal found at offset %u",
1244 /* if draining, accept it as complete nal */
1247 nalu.size = size - nalu.offset;
1248 GST_DEBUG_OBJECT (h264parse, "draining, accepting with size %u",
1250 /* if it's not too short at least */
1255 /* otherwise need more */
1257 case GST_H264_PARSER_BROKEN_LINK:
1258 GST_ELEMENT_ERROR (h264parse, STREAM, FORMAT,
1259 ("Error parsing H.264 stream"),
1260 ("The link to structure needed for the parsing couldn't be found"));
1261 goto invalid_stream;
1262 case GST_H264_PARSER_ERROR:
1263 /* should not really occur either */
1264 GST_ELEMENT_ERROR (h264parse, STREAM, FORMAT,
1265 ("Error parsing H.264 stream"), ("Invalid H.264 stream"));
1266 goto invalid_stream;
1267 case GST_H264_PARSER_NO_NAL:
1268 GST_ELEMENT_ERROR (h264parse, STREAM, FORMAT,
1269 ("Error parsing H.264 stream"), ("No H.264 NAL unit found"));
1270 goto invalid_stream;
1271 case GST_H264_PARSER_BROKEN_DATA:
1272 GST_WARNING_OBJECT (h264parse, "input stream is corrupt; "
1273 "it contains a NAL unit of length %u", nalu.size);
1275 /* broken nal at start -> arrange to skip it,
1276 * otherwise have it terminate current au
1277 * (and so it will be skipped on next frame round) */
1278 GST_ELEMENT_WARNING (h264parse, STREAM, DECODE,
1279 (NULL), ("Broken bit stream"));
1280 if (current_off == 0) {
1281 GST_DEBUG_OBJECT (h264parse, "skipping broken nal");
1282 *skipsize = nalu.offset;
1283 h264parse->aud_needed = TRUE;
1286 GST_DEBUG_OBJECT (h264parse, "terminating au");
1288 nalu.offset = nalu.sc_offset;
1293 g_assert_not_reached ();
1297 GST_DEBUG_OBJECT (h264parse, "%p complete nal found. Off: %u, Size: %u",
1298 data, nalu.offset, nalu.size);
1301 /* expect at least 3 bytes start_code, and 1 bytes NALU header.
1302 * the length of the NALU payload can be zero.
1303 * (e.g. EOS/EOB placed at the end of an AU.) */
1304 if (nalu.offset + nalu.size + 3 + 1 > size) {
1305 GST_DEBUG_OBJECT (h264parse, "not enough data for next NALU");
1307 GST_DEBUG_OBJECT (h264parse, "but draining anyway");
1315 if (!gst_h264_parse_process_nal (h264parse, &nalu)) {
1316 GST_WARNING_OBJECT (h264parse,
1317 "broken/invalid nal Type: %d %s, Size: %u will be dropped",
1318 nalu.type, _nal_name (nalu.type), nalu.size);
1319 *skipsize = nalu.size;
1320 h264parse->aud_needed = TRUE;
1324 /* Judge whether or not to insert AU Delimiter in case of byte-stream
1325 * If we're in the middle of au, we don't need to insert aud.
1326 * Otherwise, we honor the result in gst_h264_parse_process_nal.
1327 * Note that this should be done until draining if it's happening.
1329 if (h264parse->align == GST_H264_PARSE_ALIGN_NAL && !h264parse->aud_needed)
1330 h264parse->aud_insert = FALSE;
1335 /* if no next nal, we know it's complete here */
1336 au_complete = gst_h264_parse_collect_nal (h264parse, data, size, &nalu);
1338 if (h264parse->align == GST_H264_PARSE_ALIGN_NAL) {
1339 h264parse->aud_needed = au_complete;
1346 GST_DEBUG_OBJECT (h264parse, "Looking for more");
1347 current_off = nalu.offset + nalu.size;
1351 framesize = nalu.offset + nalu.size;
1353 gst_buffer_unmap (buffer, &map);
1355 gst_h264_parse_parse_frame (parse, frame);
1357 return gst_base_parse_finish_frame (parse, frame, framesize);
1362 /* Restart parsing from here next time */
1363 if (current_off > 0)
1364 h264parse->current_off = current_off;
1368 gst_buffer_unmap (buffer, &map);
1372 GST_DEBUG_OBJECT (h264parse, "Dropped data");
1376 GST_DEBUG_OBJECT (h264parse, "skipping %d", *skipsize);
1377 /* If we are collecting access units, we need to preserve the initial
1378 * config headers (SPS, PPS et al.) and only reset the frame if another
1379 * slice NAL was received. This means that broken pictures are discarded */
1380 if (h264parse->align != GST_H264_PARSE_ALIGN_AU ||
1381 !(h264parse->state & GST_H264_PARSE_STATE_VALID_PICTURE_HEADERS) ||
1382 (h264parse->state & GST_H264_PARSE_STATE_GOT_SLICE))
1383 gst_h264_parse_reset_frame (h264parse);
1387 gst_buffer_unmap (buffer, &map);
1388 return GST_FLOW_ERROR;
1391 /* byte together avc codec data based on collected pps and sps so far */
1393 gst_h264_parse_make_codec_data (GstH264Parse * h264parse)
1395 GstBuffer *buf, *nal;
1396 gint i, sps_size = 0, pps_size = 0, num_sps = 0, num_pps = 0;
1397 guint8 profile_idc = 0, profile_comp = 0, level_idc = 0;
1398 gboolean found = FALSE;
1403 /* only nal payload in stored nals */
1405 for (i = 0; i < GST_H264_MAX_SPS_COUNT; i++) {
1406 if ((nal = h264parse->sps_nals[i])) {
1407 gsize size = gst_buffer_get_size (nal);
1409 /* size bytes also count */
1410 sps_size += size + 2;
1414 gst_buffer_extract (nal, 1, tmp, 3);
1415 profile_idc = tmp[0];
1416 profile_comp = tmp[1];
1421 for (i = 0; i < GST_H264_MAX_PPS_COUNT; i++) {
1422 if ((nal = h264parse->pps_nals[i])) {
1424 /* size bytes also count */
1425 pps_size += gst_buffer_get_size (nal) + 2;
1429 /* AVC3 has SPS/PPS inside the stream, not in the codec_data */
1430 if (h264parse->format == GST_H264_PARSE_FORMAT_AVC3) {
1431 num_sps = sps_size = 0;
1432 num_pps = pps_size = 0;
1435 GST_DEBUG_OBJECT (h264parse,
1436 "constructing codec_data: num_sps=%d, num_pps=%d", num_sps, num_pps);
1438 if (!found || (0 == num_pps
1439 && GST_H264_PARSE_FORMAT_AVC3 != h264parse->format))
1442 buf = gst_buffer_new_allocate (NULL, 5 + 1 + sps_size + 1 + pps_size, NULL);
1443 gst_buffer_map (buf, &map, GST_MAP_WRITE);
1445 nl = h264parse->nal_length_size;
1447 data[0] = 1; /* AVC Decoder Configuration Record ver. 1 */
1448 data[1] = profile_idc; /* profile_idc */
1449 data[2] = profile_comp; /* profile_compability */
1450 data[3] = level_idc; /* level_idc */
1451 data[4] = 0xfc | (nl - 1); /* nal_length_size_minus1 */
1452 data[5] = 0xe0 | num_sps; /* number of SPSs */
1455 if (h264parse->format != GST_H264_PARSE_FORMAT_AVC3) {
1456 for (i = 0; i < GST_H264_MAX_SPS_COUNT; i++) {
1457 if ((nal = h264parse->sps_nals[i])) {
1458 gsize nal_size = gst_buffer_get_size (nal);
1459 GST_WRITE_UINT16_BE (data, nal_size);
1460 gst_buffer_extract (nal, 0, data + 2, nal_size);
1461 data += 2 + nal_size;
1468 if (h264parse->format != GST_H264_PARSE_FORMAT_AVC3) {
1469 for (i = 0; i < GST_H264_MAX_PPS_COUNT; i++) {
1470 if ((nal = h264parse->pps_nals[i])) {
1471 gsize nal_size = gst_buffer_get_size (nal);
1472 GST_WRITE_UINT16_BE (data, nal_size);
1473 gst_buffer_extract (nal, 0, data + 2, nal_size);
1474 data += 2 + nal_size;
1479 gst_buffer_unmap (buf, &map);
1485 gst_h264_parse_get_par (GstH264Parse * h264parse, gint * num, gint * den)
1487 if (h264parse->upstream_par_n != -1 && h264parse->upstream_par_d != -1) {
1488 *num = h264parse->upstream_par_n;
1489 *den = h264parse->upstream_par_d;
1491 *num = h264parse->parsed_par_n;
1492 *den = h264parse->parsed_par_d;
1497 get_compatible_profile_caps (GstH264SPS * sps)
1499 GstCaps *caps = NULL;
1500 const gchar **profiles = NULL;
1502 GValue compat_profiles = G_VALUE_INIT;
1503 g_value_init (&compat_profiles, GST_TYPE_LIST);
1505 switch (sps->profile_idc) {
1506 case GST_H264_PROFILE_EXTENDED:
1507 if (sps->constraint_set0_flag) { /* A.2.1 */
1508 if (sps->constraint_set1_flag) {
1509 static const gchar *profile_array[] =
1510 { "constrained-baseline", "baseline", "main", "high",
1511 "high-10", "high-4:2:2", "high-4:4:4", NULL
1513 profiles = profile_array;
1515 static const gchar *profile_array[] = { "baseline", NULL };
1516 profiles = profile_array;
1518 } else if (sps->constraint_set1_flag) { /* A.2.2 */
1519 static const gchar *profile_array[] =
1520 { "main", "high", "high-10", "high-4:2:2", "high-4:4:4", NULL };
1521 profiles = profile_array;
1524 case GST_H264_PROFILE_BASELINE:
1525 if (sps->constraint_set1_flag) { /* A.2.1 */
1526 static const gchar *profile_array[] =
1527 { "baseline", "main", "high", "high-10", "high-4:2:2",
1530 profiles = profile_array;
1532 static const gchar *profile_array[] = { "extended", NULL };
1533 profiles = profile_array;
1536 case GST_H264_PROFILE_MAIN:
1538 static const gchar *profile_array[] =
1539 { "high", "high-10", "high-4:2:2", "high-4:4:4", NULL };
1540 profiles = profile_array;
1543 case GST_H264_PROFILE_HIGH:
1544 if (sps->constraint_set1_flag) {
1545 static const gchar *profile_array[] =
1546 { "main", "high-10", "high-4:2:2", "high-4:4:4", NULL };
1547 profiles = profile_array;
1549 static const gchar *profile_array[] =
1550 { "high-10", "high-4:2:2", "high-4:4:4", NULL };
1551 profiles = profile_array;
1554 case GST_H264_PROFILE_HIGH10:
1555 if (sps->constraint_set1_flag) {
1556 static const gchar *profile_array[] =
1557 { "main", "high", "high-4:2:2", "high-4:4:4", NULL };
1558 profiles = profile_array;
1560 if (sps->constraint_set3_flag) { /* A.2.8 */
1561 static const gchar *profile_array[] =
1562 { "high-10", "high-4:2:2", "high-4:4:4", "high-4:2:2-intra",
1563 "high-4:4:4-intra", NULL
1565 profiles = profile_array;
1567 static const gchar *profile_array[] =
1568 { "high-4:2:2", "high-4:4:4", NULL };
1569 profiles = profile_array;
1573 case GST_H264_PROFILE_HIGH_422:
1574 if (sps->constraint_set1_flag) {
1575 static const gchar *profile_array[] =
1576 { "main", "high", "high-10", "high-4:4:4", NULL };
1577 profiles = profile_array;
1579 if (sps->constraint_set3_flag) { /* A.2.9 */
1580 static const gchar *profile_array[] =
1581 { "high-4:2:2", "high-4:4:4", "high-4:4:4-intra", NULL };
1582 profiles = profile_array;
1584 static const gchar *profile_array[] = { "high-4:4:4", NULL };
1585 profiles = profile_array;
1589 case GST_H264_PROFILE_HIGH_444:
1590 if (sps->constraint_set1_flag) {
1591 static const gchar *profile_array[] =
1592 { "main", "high", "high-10", "high-4:2:2", NULL };
1593 profiles = profile_array;
1594 } else if (sps->constraint_set3_flag) { /* A.2.10 */
1595 static const gchar *profile_array[] = { "high-4:4:4", NULL };
1596 profiles = profile_array;
1599 case GST_H264_PROFILE_MULTIVIEW_HIGH:
1600 if (sps->extension_type == GST_H264_NAL_EXTENSION_MVC
1601 && sps->extension.mvc.num_views_minus1 == 1) {
1602 static const gchar *profile_array[] =
1603 { "stereo-high", "multiview-high", NULL };
1604 profiles = profile_array;
1606 static const gchar *profile_array[] = { "multiview-high", NULL };
1607 profiles = profile_array;
1615 GValue value = G_VALUE_INIT;
1616 caps = gst_caps_new_empty_simple ("video/x-h264");
1617 for (i = 0; profiles[i]; i++) {
1618 g_value_init (&value, G_TYPE_STRING);
1619 g_value_set_string (&value, profiles[i]);
1620 gst_value_list_append_value (&compat_profiles, &value);
1621 g_value_unset (&value);
1623 gst_caps_set_value (caps, "profile", &compat_profiles);
1624 g_value_unset (&compat_profiles);
1630 /* if downstream didn't support the exact profile indicated in sps header,
1631 * check for the compatible profiles also */
1633 ensure_caps_profile (GstH264Parse * h264parse, GstCaps * caps, GstH264SPS * sps)
1635 GstCaps *peer_caps, *compat_caps;
1637 peer_caps = gst_pad_get_current_caps (GST_BASE_PARSE_SRC_PAD (h264parse));
1638 if (!peer_caps || !gst_caps_can_intersect (caps, peer_caps)) {
1639 GstCaps *filter_caps = gst_caps_new_empty_simple ("video/x-h264");
1642 gst_caps_unref (peer_caps);
1644 gst_pad_peer_query_caps (GST_BASE_PARSE_SRC_PAD (h264parse),
1647 gst_caps_unref (filter_caps);
1650 if (peer_caps && !gst_caps_can_intersect (caps, peer_caps)) {
1651 GstStructure *structure;
1653 compat_caps = get_compatible_profile_caps (sps);
1654 if (compat_caps != NULL) {
1655 GstCaps *res_caps = NULL;
1657 res_caps = gst_caps_intersect (peer_caps, compat_caps);
1659 if (res_caps && !gst_caps_is_empty (res_caps)) {
1660 const gchar *profile_str = NULL;
1662 res_caps = gst_caps_fixate (res_caps);
1663 structure = gst_caps_get_structure (res_caps, 0);
1664 profile_str = gst_structure_get_string (structure, "profile");
1666 gst_caps_set_simple (caps, "profile", G_TYPE_STRING, profile_str,
1668 GST_DEBUG_OBJECT (h264parse,
1669 "Setting compatible profile %s to the caps", profile_str);
1673 gst_caps_unref (res_caps);
1674 gst_caps_unref (compat_caps);
1678 gst_caps_unref (peer_caps);
1681 static const gchar *
1682 digit_to_string (guint digit)
1684 static const char itoa[][2] = {
1685 "0", "1", "2", "3", "4", "5", "6", "7", "8", "9"
1688 if (G_LIKELY (digit < 10))
1694 static const gchar *
1695 get_profile_string (GstH264SPS * sps)
1697 const gchar *profile = NULL;
1699 switch (sps->profile_idc) {
1701 if (sps->constraint_set1_flag)
1702 profile = "constrained-baseline";
1704 profile = "baseline";
1710 profile = "extended";
1713 if (sps->constraint_set4_flag) {
1714 if (sps->constraint_set5_flag)
1715 profile = "constrained-high";
1717 profile = "progressive-high";
1722 if (sps->constraint_set3_flag)
1723 profile = "high-10-intra";
1724 else if (sps->constraint_set4_flag)
1725 profile = "progressive-high-10";
1727 profile = "high-10";
1730 if (sps->constraint_set3_flag)
1731 profile = "high-4:2:2-intra";
1733 profile = "high-4:2:2";
1736 if (sps->constraint_set3_flag)
1737 profile = "high-4:4:4-intra";
1739 profile = "high-4:4:4";
1742 profile = "cavlc-4:4:4-intra";
1745 profile = "multiview-high";
1748 profile = "stereo-high";
1751 if (sps->constraint_set5_flag)
1752 profile = "scalable-constrained-baseline";
1754 profile = "scalable-baseline";
1757 if (sps->constraint_set3_flag)
1758 profile = "scalable-high-intra";
1759 else if (sps->constraint_set5_flag)
1760 profile = "scalable-constrained-high";
1762 profile = "scalable-high";
1771 static const gchar *
1772 get_level_string (GstH264SPS * sps)
1774 if (sps->level_idc == 0)
1776 else if ((sps->level_idc == 11 && sps->constraint_set3_flag)
1777 || sps->level_idc == 9)
1779 else if (sps->level_idc % 10 == 0)
1780 return digit_to_string (sps->level_idc / 10);
1782 switch (sps->level_idc) {
1812 gst_h264_parse_update_src_caps (GstH264Parse * h264parse, GstCaps * caps)
1815 GstCaps *sink_caps, *src_caps;
1816 gboolean modified = FALSE;
1817 GstBuffer *buf = NULL;
1818 GstStructure *s = NULL;
1820 if (G_UNLIKELY (!gst_pad_has_current_caps (GST_BASE_PARSE_SRC_PAD
1823 else if (G_UNLIKELY (!h264parse->update_caps))
1826 /* if this is being called from the first _setcaps call, caps on the sinkpad
1827 * aren't set yet and so they need to be passed as an argument */
1829 sink_caps = gst_caps_ref (caps);
1831 sink_caps = gst_pad_get_current_caps (GST_BASE_PARSE_SINK_PAD (h264parse));
1833 /* carry over input caps as much as possible; override with our own stuff */
1835 sink_caps = gst_caps_new_empty_simple ("video/x-h264");
1837 s = gst_caps_get_structure (sink_caps, 0);
1839 sps = h264parse->nalparser->last_sps;
1840 GST_DEBUG_OBJECT (h264parse, "sps: %p", sps);
1842 /* only codec-data for nice-and-clean au aligned packetized avc format */
1843 if ((h264parse->format == GST_H264_PARSE_FORMAT_AVC
1844 || h264parse->format == GST_H264_PARSE_FORMAT_AVC3)
1845 && h264parse->align == GST_H264_PARSE_ALIGN_AU) {
1846 buf = gst_h264_parse_make_codec_data (h264parse);
1847 if (buf && h264parse->codec_data) {
1850 gst_buffer_map (buf, &map, GST_MAP_READ);
1851 if (map.size != gst_buffer_get_size (h264parse->codec_data) ||
1852 gst_buffer_memcmp (h264parse->codec_data, 0, map.data, map.size))
1855 gst_buffer_unmap (buf, &map);
1857 if (!buf && h264parse->codec_data_in)
1858 buf = gst_buffer_ref (h264parse->codec_data_in);
1864 if (G_UNLIKELY (!sps)) {
1865 caps = gst_caps_copy (sink_caps);
1867 gint crop_width, crop_height;
1868 gint fps_num, fps_den;
1870 GstH264VUIParams *vui = &sps->vui_parameters;
1871 gchar *colorimetry = NULL;
1873 if (sps->frame_cropping_flag) {
1874 crop_width = sps->crop_rect_width;
1875 crop_height = sps->crop_rect_height;
1877 crop_width = sps->width;
1878 crop_height = sps->height;
1881 if (G_UNLIKELY (h264parse->width != crop_width ||
1882 h264parse->height != crop_height)) {
1883 GST_INFO_OBJECT (h264parse, "resolution changed %dx%d",
1884 crop_width, crop_height);
1885 h264parse->width = crop_width;
1886 h264parse->height = crop_height;
1890 /* 0/1 is set as the default in the codec parser, we will set
1891 * it in case we have no info */
1892 gst_h264_video_calculate_framerate (sps, h264parse->field_pic_flag,
1893 h264parse->sei_pic_struct, &fps_num, &fps_den);
1894 if (G_UNLIKELY (h264parse->fps_num != fps_num
1895 || h264parse->fps_den != fps_den)) {
1896 GST_DEBUG_OBJECT (h264parse, "framerate changed %d/%d", fps_num, fps_den);
1897 h264parse->fps_num = fps_num;
1898 h264parse->fps_den = fps_den;
1902 if (vui->aspect_ratio_info_present_flag) {
1903 if (G_UNLIKELY ((h264parse->parsed_par_n != vui->par_n)
1904 || (h264parse->parsed_par_d != vui->par_d))) {
1905 h264parse->parsed_par_n = vui->par_n;
1906 h264parse->parsed_par_d = vui->par_d;
1907 GST_INFO_OBJECT (h264parse, "pixel aspect ratio has been changed %d/%d",
1908 h264parse->parsed_par_n, h264parse->parsed_par_d);
1913 if (vui->video_signal_type_present_flag &&
1914 vui->colour_description_present_flag) {
1915 GstVideoColorimetry ci = { 0, };
1916 gchar *old_colorimetry = NULL;
1918 if (vui->video_full_range_flag)
1919 ci.range = GST_VIDEO_COLOR_RANGE_0_255;
1921 ci.range = GST_VIDEO_COLOR_RANGE_16_235;
1923 ci.matrix = gst_video_color_matrix_from_iso (vui->matrix_coefficients);
1925 gst_video_color_transfer_from_iso (vui->transfer_characteristics);
1926 ci.primaries = gst_video_color_primaries_from_iso (vui->colour_primaries);
1929 gst_video_colorimetry_to_string (&h264parse->parsed_colorimetry);
1930 colorimetry = gst_video_colorimetry_to_string (&ci);
1932 if (colorimetry && g_strcmp0 (old_colorimetry, colorimetry)) {
1933 GST_INFO_OBJECT (h264parse,
1934 "colorimetry has been changed from %s to %s",
1935 GST_STR_NULL (old_colorimetry), colorimetry);
1936 h264parse->parsed_colorimetry = ci;
1940 g_free (old_colorimetry);
1943 if (G_UNLIKELY (modified || h264parse->update_caps)) {
1944 GstVideoInterlaceMode imode = GST_VIDEO_INTERLACE_MODE_PROGRESSIVE;
1946 GstClockTime latency;
1948 const gchar *caps_mview_mode = NULL;
1949 GstVideoMultiviewMode mview_mode = h264parse->multiview_mode;
1950 GstVideoMultiviewFlags mview_flags = h264parse->multiview_flags;
1951 const gchar *chroma_format = NULL;
1952 guint bit_depth_chroma;
1954 fps_num = h264parse->fps_num;
1955 fps_den = h264parse->fps_den;
1957 caps = gst_caps_copy (sink_caps);
1959 /* sps should give this but upstream overrides */
1960 if (s && gst_structure_has_field (s, "width"))
1961 gst_structure_get_int (s, "width", &width);
1963 width = h264parse->width;
1965 if (s && gst_structure_has_field (s, "height"))
1966 gst_structure_get_int (s, "height", &height);
1968 height = h264parse->height;
1971 !gst_structure_get_fraction (s, "pixel-aspect-ratio", &par_n,
1973 gst_h264_parse_get_par (h264parse, &par_n, &par_d);
1974 if (par_n != 0 && par_d != 0) {
1975 GST_INFO_OBJECT (h264parse, "PAR %d/%d", par_n, par_d);
1976 gst_caps_set_simple (caps, "pixel-aspect-ratio", GST_TYPE_FRACTION,
1977 par_n, par_d, NULL);
1979 /* Assume par_n/par_d of 1/1 for calcs below, but don't set into caps */
1984 /* Pass through or set output stereo/multiview config */
1985 if (s && gst_structure_has_field (s, "multiview-mode")) {
1986 caps_mview_mode = gst_structure_get_string (s, "multiview-mode");
1987 gst_structure_get_flagset (s, "multiview-flags",
1988 (guint *) & mview_flags, NULL);
1989 } else if (mview_mode != GST_VIDEO_MULTIVIEW_MODE_NONE) {
1990 if (gst_video_multiview_guess_half_aspect (mview_mode,
1991 width, height, par_n, par_d)) {
1992 mview_flags |= GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT;
1995 caps_mview_mode = gst_video_multiview_mode_to_caps_string (mview_mode);
1996 gst_caps_set_simple (caps, "multiview-mode", G_TYPE_STRING,
1997 caps_mview_mode, "multiview-flags",
1998 GST_TYPE_VIDEO_MULTIVIEW_FLAGSET, mview_flags,
1999 GST_FLAG_SET_MASK_EXACT, NULL);
2002 gst_caps_set_simple (caps, "width", G_TYPE_INT, width,
2003 "height", G_TYPE_INT, height, NULL);
2005 /* upstream overrides */
2006 if (s && gst_structure_has_field (s, "framerate")) {
2007 gst_structure_get_fraction (s, "framerate", &fps_num, &fps_den);
2010 /* but not necessarily or reliably this */
2013 gst_caps_set_simple (caps, "framerate",
2014 GST_TYPE_FRACTION, fps_num, fps_den, NULL);
2015 s2 = gst_caps_get_structure (caps, 0);
2016 gst_structure_get_fraction (s2, "framerate", &h264parse->parsed_fps_n,
2017 &h264parse->parsed_fps_d);
2018 gst_base_parse_set_frame_rate (GST_BASE_PARSE (h264parse), fps_num,
2021 latency = gst_util_uint64_scale (GST_SECOND, fps_den, fps_num);
2022 gst_base_parse_set_latency (GST_BASE_PARSE (h264parse), latency,
2028 /* upstream overrides or uses sps info */
2029 if (s && gst_structure_has_field (s, "interlace-mode"))
2031 gst_video_interlace_mode_from_string (gst_structure_get_string (s,
2033 else if (sps->frame_mbs_only_flag == 0)
2034 imode = GST_VIDEO_INTERLACE_MODE_MIXED;
2036 gst_caps_set_simple (caps, "interlace-mode", G_TYPE_STRING,
2037 gst_video_interlace_mode_to_string (imode), NULL);
2039 bit_depth_chroma = sps->bit_depth_chroma_minus8 + 8;
2041 switch (sps->chroma_format_idc) {
2043 chroma_format = "4:0:0";
2044 bit_depth_chroma = 0;
2047 chroma_format = "4:2:0";
2050 chroma_format = "4:2:2";
2053 chroma_format = "4:4:4";
2060 gst_caps_set_simple (caps,
2061 "chroma-format", G_TYPE_STRING, chroma_format,
2062 "bit-depth-luma", G_TYPE_UINT, sps->bit_depth_luma_minus8 + 8,
2063 "bit-depth-chroma", G_TYPE_UINT, bit_depth_chroma, NULL);
2065 if (colorimetry && (!s || !gst_structure_has_field (s, "colorimetry"))) {
2066 gst_caps_set_simple (caps, "colorimetry", G_TYPE_STRING, colorimetry,
2071 g_free (colorimetry);
2075 gst_caps_set_simple (caps, "parsed", G_TYPE_BOOLEAN, TRUE,
2076 "stream-format", G_TYPE_STRING,
2077 gst_h264_parse_get_string (h264parse, TRUE, h264parse->format),
2078 "alignment", G_TYPE_STRING,
2079 gst_h264_parse_get_string (h264parse, FALSE, h264parse->align), NULL);
2081 /* set profile and level in caps */
2083 const gchar *profile, *level;
2085 profile = get_profile_string (sps);
2086 if (profile != NULL)
2087 gst_caps_set_simple (caps, "profile", G_TYPE_STRING, profile, NULL);
2089 level = get_level_string (sps);
2091 gst_caps_set_simple (caps, "level", G_TYPE_STRING, level, NULL);
2093 /* relax the profile constraint to find a suitable decoder */
2094 ensure_caps_profile (h264parse, caps, sps);
2097 src_caps = gst_pad_get_current_caps (GST_BASE_PARSE_SRC_PAD (h264parse));
2100 /* use codec data from old caps for comparison; we don't want to resend caps
2101 if everything is same except codec data; */
2102 if (gst_structure_has_field (gst_caps_get_structure (src_caps, 0),
2104 gst_caps_set_value (caps, "codec_data",
2105 gst_structure_get_value (gst_caps_get_structure (src_caps, 0),
2109 /* remove any left-over codec-data hanging around */
2110 s = gst_caps_get_structure (caps, 0);
2111 gst_structure_remove_field (s, "codec_data");
2115 if (!(src_caps && gst_caps_is_strictly_equal (src_caps, caps))) {
2116 /* update codec data to new value */
2118 gst_caps_set_simple (caps, "codec_data", GST_TYPE_BUFFER, buf, NULL);
2119 gst_buffer_replace (&h264parse->codec_data, buf);
2120 gst_buffer_unref (buf);
2124 /* remove any left-over codec-data hanging around */
2125 s = gst_caps_get_structure (caps, 0);
2126 gst_structure_remove_field (s, "codec_data");
2127 gst_buffer_replace (&h264parse->codec_data, NULL);
2130 gst_pad_set_caps (GST_BASE_PARSE_SRC_PAD (h264parse), caps);
2134 gst_caps_unref (src_caps);
2135 gst_caps_unref (caps);
2138 gst_caps_unref (sink_caps);
2140 gst_buffer_unref (buf);
2144 gst_h264_parse_get_timestamp (GstH264Parse * h264parse,
2145 GstClockTime * out_ts, GstClockTime * out_dur, gboolean frame)
2147 GstH264SPS *sps = h264parse->nalparser->last_sps;
2148 GstClockTime upstream;
2151 g_return_if_fail (out_dur != NULL);
2152 g_return_if_fail (out_ts != NULL);
2155 GST_LOG_OBJECT (h264parse, "Upstream ts %" GST_TIME_FORMAT,
2156 GST_TIME_ARGS (upstream));
2159 GST_LOG_OBJECT (h264parse, "no frame data -> 0 duration");
2167 GST_DEBUG_OBJECT (h264parse, "referred SPS invalid");
2169 } else if (!sps->vui_parameters_present_flag) {
2170 GST_DEBUG_OBJECT (h264parse,
2171 "unable to compute timestamp: VUI not present");
2173 } else if (!sps->vui_parameters.timing_info_present_flag) {
2174 GST_DEBUG_OBJECT (h264parse,
2175 "unable to compute timestamp: timing info not present");
2177 } else if (sps->vui_parameters.time_scale == 0) {
2178 GST_DEBUG_OBJECT (h264parse,
2179 "unable to compute timestamp: time_scale = 0 "
2180 "(this is forbidden in spec; bitstream probably contains error)");
2184 if (h264parse->sei_pic_struct_pres_flag &&
2185 h264parse->sei_pic_struct != (guint8) - 1) {
2186 /* Note that when h264parse->sei_pic_struct == -1 (unspecified), there
2187 * are ways to infer its value. This is related to computing the
2188 * TopFieldOrderCnt and BottomFieldOrderCnt, which looks
2189 * complicated and thus not implemented for the time being. Yet
2190 * the value we have here is correct for many applications
2192 switch (h264parse->sei_pic_struct) {
2193 case GST_H264_SEI_PIC_STRUCT_TOP_FIELD:
2194 case GST_H264_SEI_PIC_STRUCT_BOTTOM_FIELD:
2197 case GST_H264_SEI_PIC_STRUCT_FRAME:
2198 case GST_H264_SEI_PIC_STRUCT_TOP_BOTTOM:
2199 case GST_H264_SEI_PIC_STRUCT_BOTTOM_TOP:
2202 case GST_H264_SEI_PIC_STRUCT_TOP_BOTTOM_TOP:
2203 case GST_H264_SEI_PIC_STRUCT_BOTTOM_TOP_BOTTOM:
2206 case GST_H264_SEI_PIC_STRUCT_FRAME_DOUBLING:
2209 case GST_H264_SEI_PIC_STRUCT_FRAME_TRIPLING:
2213 GST_DEBUG_OBJECT (h264parse,
2214 "h264parse->sei_pic_struct of unknown value %d. Not parsed",
2215 h264parse->sei_pic_struct);
2219 duration = h264parse->field_pic_flag ? 1 : 2;
2222 GST_LOG_OBJECT (h264parse, "frame tick duration %d", duration);
2225 * h264parse.264 C.1.2 Timing of coded picture removal (equivalent to DTS):
2226 * Tr,n(0) = initial_cpb_removal_delay[ SchedSelIdx ] / 90000
2227 * Tr,n(n) = Tr,n(nb) + Tc * cpb_removal_delay(n)
2229 * Tc = num_units_in_tick / time_scale
2232 if (h264parse->ts_trn_nb != GST_CLOCK_TIME_NONE) {
2233 GST_LOG_OBJECT (h264parse, "buffering based ts");
2234 /* buffering period is present */
2235 if (upstream != GST_CLOCK_TIME_NONE) {
2236 /* If upstream timestamp is valid, we respect it and adjust current
2237 * reference point */
2238 h264parse->ts_trn_nb = upstream -
2239 (GstClockTime) gst_util_uint64_scale
2240 (h264parse->sei_cpb_removal_delay * GST_SECOND,
2241 sps->vui_parameters.num_units_in_tick,
2242 sps->vui_parameters.time_scale);
2244 /* If no upstream timestamp is given, we write in new timestamp */
2245 upstream = h264parse->dts = h264parse->ts_trn_nb +
2246 (GstClockTime) gst_util_uint64_scale
2247 (h264parse->sei_cpb_removal_delay * GST_SECOND,
2248 sps->vui_parameters.num_units_in_tick,
2249 sps->vui_parameters.time_scale);
2254 GST_LOG_OBJECT (h264parse, "duration based ts");
2255 /* naive method: no removal delay specified
2256 * track upstream timestamp and provide best guess frame duration */
2257 dur = gst_util_uint64_scale (duration * GST_SECOND,
2258 sps->vui_parameters.num_units_in_tick, sps->vui_parameters.time_scale);
2260 if (dur < GST_MSECOND) {
2261 GST_DEBUG_OBJECT (h264parse, "discarding dur %" GST_TIME_FORMAT,
2262 GST_TIME_ARGS (dur));
2269 if (GST_CLOCK_TIME_IS_VALID (upstream))
2270 *out_ts = h264parse->dts = upstream;
2272 if (GST_CLOCK_TIME_IS_VALID (*out_dur) &&
2273 GST_CLOCK_TIME_IS_VALID (h264parse->dts))
2274 h264parse->dts += *out_dur;
2277 static GstFlowReturn
2278 gst_h264_parse_parse_frame (GstBaseParse * parse, GstBaseParseFrame * frame)
2280 GstH264Parse *h264parse;
2284 h264parse = GST_H264_PARSE (parse);
2285 buffer = frame->buffer;
2287 gst_h264_parse_update_src_caps (h264parse, NULL);
2289 /* don't mess with timestamps if provided by upstream,
2290 * particularly since our ts not that good they handle seeking etc */
2291 if (h264parse->do_ts)
2292 gst_h264_parse_get_timestamp (h264parse,
2293 &GST_BUFFER_TIMESTAMP (buffer), &GST_BUFFER_DURATION (buffer),
2294 h264parse->frame_start);
2296 if (h264parse->keyframe)
2297 GST_BUFFER_FLAG_UNSET (buffer, GST_BUFFER_FLAG_DELTA_UNIT);
2299 GST_BUFFER_FLAG_SET (buffer, GST_BUFFER_FLAG_DELTA_UNIT);
2301 if (h264parse->discard_bidirectional && h264parse->bidirectional)
2304 if (h264parse->header)
2305 GST_BUFFER_FLAG_SET (buffer, GST_BUFFER_FLAG_HEADER);
2307 GST_BUFFER_FLAG_UNSET (buffer, GST_BUFFER_FLAG_HEADER);
2309 if (h264parse->discont) {
2310 GST_BUFFER_FLAG_SET (buffer, GST_BUFFER_FLAG_DISCONT);
2311 h264parse->discont = FALSE;
2314 /* replace with transformed AVC output if applicable */
2315 av = gst_adapter_available (h264parse->frame_out);
2319 buf = gst_adapter_take_buffer (h264parse->frame_out, av);
2320 gst_buffer_copy_into (buf, buffer, GST_BUFFER_COPY_METADATA, 0, -1);
2321 gst_buffer_replace (&frame->out_buffer, buf);
2322 gst_buffer_unref (buf);
2329 GST_DEBUG_OBJECT (h264parse, "Discarding bidirectional frame");
2330 frame->flags |= GST_BASE_PARSE_FRAME_FLAG_DROP;
2331 gst_h264_parse_reset_frame (h264parse);
2335 /* sends a codec NAL downstream, decorating and transforming as needed.
2336 * No ownership is taken of @nal */
2337 static GstFlowReturn
2338 gst_h264_parse_push_codec_buffer (GstH264Parse * h264parse,
2339 GstBuffer * nal, GstClockTime ts)
2343 gst_buffer_map (nal, &map, GST_MAP_READ);
2344 nal = gst_h264_parse_wrap_nal (h264parse, h264parse->format,
2345 map.data, map.size);
2346 gst_buffer_unmap (nal, &map);
2348 GST_BUFFER_TIMESTAMP (nal) = ts;
2349 GST_BUFFER_DURATION (nal) = 0;
2351 return gst_pad_push (GST_BASE_PARSE_SRC_PAD (h264parse), nal);
2355 check_pending_key_unit_event (GstEvent * pending_event,
2356 GstSegment * segment, GstClockTime timestamp, guint flags,
2357 GstClockTime pending_key_unit_ts)
2359 GstClockTime running_time, stream_time;
2360 gboolean all_headers;
2362 GstEvent *event = NULL;
2364 g_return_val_if_fail (segment != NULL, NULL);
2366 if (pending_event == NULL)
2369 if (GST_CLOCK_TIME_IS_VALID (pending_key_unit_ts) &&
2370 timestamp == GST_CLOCK_TIME_NONE)
2373 running_time = gst_segment_to_running_time (segment,
2374 GST_FORMAT_TIME, timestamp);
2376 GST_INFO ("now %" GST_TIME_FORMAT " wanted %" GST_TIME_FORMAT,
2377 GST_TIME_ARGS (running_time), GST_TIME_ARGS (pending_key_unit_ts));
2378 if (GST_CLOCK_TIME_IS_VALID (pending_key_unit_ts) &&
2379 running_time < pending_key_unit_ts)
2382 if (flags & GST_BUFFER_FLAG_DELTA_UNIT) {
2383 GST_DEBUG ("pending force key unit, waiting for keyframe");
2387 stream_time = gst_segment_to_stream_time (segment,
2388 GST_FORMAT_TIME, timestamp);
2390 if (!gst_video_event_parse_upstream_force_key_unit (pending_event,
2391 NULL, &all_headers, &count)) {
2392 gst_video_event_parse_downstream_force_key_unit (pending_event, NULL,
2393 NULL, NULL, &all_headers, &count);
2397 gst_video_event_new_downstream_force_key_unit (timestamp, stream_time,
2398 running_time, all_headers, count);
2399 gst_event_set_seqnum (event, gst_event_get_seqnum (pending_event));
2406 gst_h264_parse_prepare_key_unit (GstH264Parse * parse, GstEvent * event)
2408 GstClockTime running_time;
2410 #ifndef GST_DISABLE_GST_DEBUG
2411 gboolean have_sps, have_pps;
2415 parse->pending_key_unit_ts = GST_CLOCK_TIME_NONE;
2416 gst_event_replace (&parse->force_key_unit_event, NULL);
2418 gst_video_event_parse_downstream_force_key_unit (event,
2419 NULL, NULL, &running_time, NULL, &count);
2421 GST_INFO_OBJECT (parse, "pushing downstream force-key-unit event %d "
2422 "%" GST_TIME_FORMAT " count %d", gst_event_get_seqnum (event),
2423 GST_TIME_ARGS (running_time), count);
2424 gst_pad_push_event (GST_BASE_PARSE_SRC_PAD (parse), event);
2426 #ifndef GST_DISABLE_GST_DEBUG
2427 have_sps = have_pps = FALSE;
2428 for (i = 0; i < GST_H264_MAX_SPS_COUNT; i++) {
2429 if (parse->sps_nals[i] != NULL) {
2434 for (i = 0; i < GST_H264_MAX_PPS_COUNT; i++) {
2435 if (parse->pps_nals[i] != NULL) {
2441 GST_INFO_OBJECT (parse, "preparing key unit, have sps %d have pps %d",
2442 have_sps, have_pps);
2445 /* set push_codec to TRUE so that pre_push_frame sends SPS/PPS again */
2446 parse->push_codec = TRUE;
2450 gst_h264_parse_handle_sps_pps_nals (GstH264Parse * h264parse,
2451 GstBuffer * buffer, GstBaseParseFrame * frame)
2453 GstBuffer *codec_nal;
2455 gboolean send_done = FALSE;
2456 GstClockTime timestamp = GST_BUFFER_TIMESTAMP (buffer);
2458 if (h264parse->have_sps_in_frame && h264parse->have_pps_in_frame) {
2459 GST_DEBUG_OBJECT (h264parse, "SPS/PPS exist in frame, will not insert");
2463 if (h264parse->align == GST_H264_PARSE_ALIGN_NAL) {
2464 /* send separate config NAL buffers */
2465 GST_DEBUG_OBJECT (h264parse, "- sending SPS/PPS");
2466 for (i = 0; i < GST_H264_MAX_SPS_COUNT; i++) {
2467 if ((codec_nal = h264parse->sps_nals[i])) {
2468 GST_DEBUG_OBJECT (h264parse, "sending SPS nal");
2469 gst_h264_parse_push_codec_buffer (h264parse, codec_nal, timestamp);
2473 for (i = 0; i < GST_H264_MAX_PPS_COUNT; i++) {
2474 if ((codec_nal = h264parse->pps_nals[i])) {
2475 GST_DEBUG_OBJECT (h264parse, "sending PPS nal");
2476 gst_h264_parse_push_codec_buffer (h264parse, codec_nal, timestamp);
2481 /* insert config NALs into AU */
2484 const gboolean bs = h264parse->format == GST_H264_PARSE_FORMAT_BYTE;
2485 const gint nls = 4 - h264parse->nal_length_size;
2488 gst_byte_writer_init_with_size (&bw, gst_buffer_get_size (buffer), FALSE);
2489 ok = gst_byte_writer_put_buffer (&bw, buffer, 0, h264parse->idr_pos);
2490 GST_DEBUG_OBJECT (h264parse, "- inserting SPS/PPS");
2491 for (i = 0; i < GST_H264_MAX_SPS_COUNT; i++) {
2492 if ((codec_nal = h264parse->sps_nals[i])) {
2493 gsize nal_size = gst_buffer_get_size (codec_nal);
2494 GST_DEBUG_OBJECT (h264parse, "inserting SPS nal");
2496 ok &= gst_byte_writer_put_uint32_be (&bw, 1);
2498 ok &= gst_byte_writer_put_uint32_be (&bw, (nal_size << (nls * 8)));
2499 ok &= gst_byte_writer_set_pos (&bw,
2500 gst_byte_writer_get_pos (&bw) - nls);
2503 ok &= gst_byte_writer_put_buffer (&bw, codec_nal, 0, nal_size);
2507 for (i = 0; i < GST_H264_MAX_PPS_COUNT; i++) {
2508 if ((codec_nal = h264parse->pps_nals[i])) {
2509 gsize nal_size = gst_buffer_get_size (codec_nal);
2510 GST_DEBUG_OBJECT (h264parse, "inserting PPS nal");
2512 ok &= gst_byte_writer_put_uint32_be (&bw, 1);
2514 ok &= gst_byte_writer_put_uint32_be (&bw, (nal_size << (nls * 8)));
2515 ok &= gst_byte_writer_set_pos (&bw,
2516 gst_byte_writer_get_pos (&bw) - nls);
2518 ok &= gst_byte_writer_put_buffer (&bw, codec_nal, 0, nal_size);
2522 ok &= gst_byte_writer_put_buffer (&bw, buffer, h264parse->idr_pos, -1);
2523 /* collect result and push */
2524 new_buf = gst_byte_writer_reset_and_get_buffer (&bw);
2525 gst_buffer_copy_into (new_buf, buffer, GST_BUFFER_COPY_METADATA, 0, -1);
2526 /* should already be keyframe/IDR, but it may not have been,
2527 * so mark it as such to avoid being discarded by picky decoder */
2528 GST_BUFFER_FLAG_UNSET (new_buf, GST_BUFFER_FLAG_DELTA_UNIT);
2529 gst_buffer_replace (&frame->out_buffer, new_buf);
2530 gst_buffer_unref (new_buf);
2531 /* some result checking seems to make some compilers happy */
2532 if (G_UNLIKELY (!ok)) {
2533 GST_ERROR_OBJECT (h264parse, "failed to insert SPS/PPS");
2540 static GstFlowReturn
2541 gst_h264_parse_pre_push_frame (GstBaseParse * parse, GstBaseParseFrame * frame)
2543 GstH264Parse *h264parse;
2546 GstBuffer *parse_buffer = NULL;
2547 gboolean is_interlaced = FALSE;
2549 h264parse = GST_H264_PARSE (parse);
2551 if (!h264parse->sent_codec_tag) {
2552 GstTagList *taglist;
2556 caps = gst_pad_get_current_caps (GST_BASE_PARSE_SRC_PAD (parse));
2558 if (GST_PAD_IS_FLUSHING (GST_BASE_PARSE_SRC_PAD (h264parse))) {
2559 GST_INFO_OBJECT (h264parse, "Src pad is flushing");
2560 return GST_FLOW_FLUSHING;
2562 GST_INFO_OBJECT (h264parse, "Src pad is not negotiated!");
2563 return GST_FLOW_NOT_NEGOTIATED;
2567 taglist = gst_tag_list_new_empty ();
2568 gst_pb_utils_add_codec_description_to_tag_list (taglist,
2569 GST_TAG_VIDEO_CODEC, caps);
2570 gst_caps_unref (caps);
2572 gst_base_parse_merge_tags (parse, taglist, GST_TAG_MERGE_REPLACE);
2573 gst_tag_list_unref (taglist);
2575 /* also signals the end of first-frame processing */
2576 h264parse->sent_codec_tag = TRUE;
2579 /* In case of byte-stream, insert au delimeter by default
2580 * if it doesn't exist */
2581 if (h264parse->aud_insert && h264parse->format == GST_H264_PARSE_FORMAT_BYTE) {
2582 if (h264parse->align == GST_H264_PARSE_ALIGN_AU) {
2584 gst_memory_new_wrapped (GST_MEMORY_FLAG_READONLY, (guint8 *) au_delim,
2585 sizeof (au_delim), 0, sizeof (au_delim), NULL, NULL);
2587 frame->out_buffer = gst_buffer_copy (frame->buffer);
2588 gst_buffer_prepend_memory (frame->out_buffer, mem);
2589 if (h264parse->idr_pos >= 0)
2590 h264parse->idr_pos += sizeof (au_delim);
2592 buffer = frame->out_buffer;
2594 GstBuffer *aud_buffer = gst_buffer_new_allocate (NULL, 2, NULL);
2595 gst_buffer_fill (aud_buffer, 0, (guint8 *) (au_delim + 4), 2);
2597 buffer = frame->buffer;
2598 gst_h264_parse_push_codec_buffer (h264parse, aud_buffer,
2599 GST_BUFFER_TIMESTAMP (buffer));
2600 gst_buffer_unref (aud_buffer);
2603 buffer = frame->buffer;
2606 if ((event = check_pending_key_unit_event (h264parse->force_key_unit_event,
2607 &parse->segment, GST_BUFFER_TIMESTAMP (buffer),
2608 GST_BUFFER_FLAGS (buffer), h264parse->pending_key_unit_ts))) {
2609 gst_h264_parse_prepare_key_unit (h264parse, event);
2612 /* periodic SPS/PPS sending */
2613 if (h264parse->interval > 0 || h264parse->push_codec) {
2614 GstClockTime timestamp = GST_BUFFER_TIMESTAMP (buffer);
2616 gboolean initial_frame = FALSE;
2619 if (!GST_CLOCK_TIME_IS_VALID (h264parse->last_report)) {
2620 h264parse->last_report = timestamp;
2621 initial_frame = TRUE;
2624 if (h264parse->idr_pos >= 0) {
2625 GST_LOG_OBJECT (h264parse, "IDR nal at offset %d", h264parse->idr_pos);
2627 if (timestamp > h264parse->last_report)
2628 diff = timestamp - h264parse->last_report;
2632 GST_LOG_OBJECT (h264parse,
2633 "now %" GST_TIME_FORMAT ", last SPS/PPS %" GST_TIME_FORMAT,
2634 GST_TIME_ARGS (timestamp), GST_TIME_ARGS (h264parse->last_report));
2636 GST_DEBUG_OBJECT (h264parse,
2637 "interval since last SPS/PPS %" GST_TIME_FORMAT,
2638 GST_TIME_ARGS (diff));
2640 if (GST_TIME_AS_SECONDS (diff) >= h264parse->interval ||
2641 initial_frame || h264parse->push_codec) {
2642 GstClockTime new_ts;
2644 /* avoid overwriting a perfectly fine timestamp */
2645 new_ts = GST_CLOCK_TIME_IS_VALID (timestamp) ? timestamp :
2646 h264parse->last_report;
2648 if (gst_h264_parse_handle_sps_pps_nals (h264parse, buffer, frame)) {
2649 h264parse->last_report = new_ts;
2652 /* we pushed whatever we had */
2653 h264parse->push_codec = FALSE;
2654 h264parse->have_sps = FALSE;
2655 h264parse->have_pps = FALSE;
2656 h264parse->state &= GST_H264_PARSE_STATE_VALID_PICTURE_HEADERS;
2658 } else if (h264parse->interval == -1) {
2659 if (h264parse->idr_pos >= 0) {
2660 GST_LOG_OBJECT (h264parse, "IDR nal at offset %d", h264parse->idr_pos);
2662 gst_h264_parse_handle_sps_pps_nals (h264parse, buffer, frame);
2664 /* we pushed whatever we had */
2665 h264parse->push_codec = FALSE;
2666 h264parse->have_sps = FALSE;
2667 h264parse->have_pps = FALSE;
2668 h264parse->state &= GST_H264_PARSE_STATE_VALID_PICTURE_HEADERS;
2672 /* Fixme: setting passthrough mode casuing multiple issues:
2673 * For nal aligned multiresoluton streams, passthrough mode make h264parse
2674 * unable to advertise the new resoultions. Also causing issues while
2675 * parsing MVC streams when it has two layers.
2676 * Disabing passthourgh mode for now */
2678 /* If SPS/PPS and a keyframe have been parsed, and we're not converting,
2679 * we might switch to passthrough mode now on the basis that we've seen
2680 * the SEI packets and know optional caps params (such as multiview).
2681 * This is an efficiency optimisation that relies on stream properties
2682 * remaining uniform in practice. */
2683 if (h264parse->can_passthrough) {
2684 if (h264parse->keyframe && h264parse->have_sps && h264parse->have_pps) {
2685 GST_LOG_OBJECT (parse, "Switching to passthrough mode");
2686 gst_base_parse_set_passthrough (parse, TRUE);
2694 for (i = 0; i < h264parse->num_clock_timestamp; i++) {
2695 GstH264ClockTimestamp *tim = &h264parse->clock_timestamp[i];
2696 gint field_count = -1;
2698 GstVideoTimeCodeFlags flags = 0;
2701 switch (h264parse->sei_pic_struct) {
2702 case GST_H264_SEI_PIC_STRUCT_FRAME:
2703 case GST_H264_SEI_PIC_STRUCT_TOP_FIELD:
2704 case GST_H264_SEI_PIC_STRUCT_BOTTOM_FIELD:
2705 field_count = h264parse->sei_pic_struct;
2707 case GST_H264_SEI_PIC_STRUCT_TOP_BOTTOM:
2708 field_count = i + 1;
2710 case GST_H264_SEI_PIC_STRUCT_BOTTOM_TOP:
2711 field_count = 2 - i;
2713 case GST_H264_SEI_PIC_STRUCT_TOP_BOTTOM_TOP:
2714 field_count = i % 2 ? 2 : 1;
2716 case GST_H264_SEI_PIC_STRUCT_BOTTOM_TOP_BOTTOM:
2717 field_count = i % 2 ? 1 : 2;
2719 case GST_H264_SEI_PIC_STRUCT_FRAME_DOUBLING:
2720 case GST_H264_SEI_PIC_STRUCT_FRAME_TRIPLING:
2725 if (field_count == -1) {
2726 GST_WARNING_OBJECT (parse,
2727 "failed to determine field count for timecode");
2731 /* dropping of the two lowest (value 0 and 1) n_frames
2732 * counts when seconds_value is equal to 0 and
2733 * minutes_value is not an integer multiple of 10 */
2734 if (tim->counting_type == 4)
2735 flags |= GST_VIDEO_TIME_CODE_FLAGS_DROP_FRAME;
2737 if (tim->ct_type == GST_H264_CT_TYPE_INTERLACED) {
2738 flags |= GST_VIDEO_TIME_CODE_FLAGS_INTERLACED;
2739 is_interlaced = TRUE;
2743 gst_util_uint64_scale_int (tim->n_frames, 1,
2744 2 - tim->nuit_field_based_flag);
2746 gst_buffer_add_video_time_code_meta_full (buffer,
2747 h264parse->parsed_fps_n,
2748 h264parse->parsed_fps_d,
2751 tim->hours_flag ? tim->hours_value : 0,
2752 tim->minutes_flag ? tim->minutes_value : 0,
2753 tim->seconds_flag ? tim->seconds_value : 0, n_frames, field_count);
2756 h264parse->num_clock_timestamp = 0;
2759 if (frame->out_buffer) {
2760 parse_buffer = frame->out_buffer =
2761 gst_buffer_make_writable (frame->out_buffer);
2763 parse_buffer = frame->buffer = gst_buffer_make_writable (frame->buffer);
2766 if (is_interlaced) {
2767 GST_BUFFER_FLAG_SET (parse_buffer, GST_VIDEO_BUFFER_FLAG_INTERLACED);
2768 if (h264parse->sei_pic_struct == GST_H264_SEI_PIC_STRUCT_TOP_FIELD)
2769 GST_BUFFER_FLAG_SET (parse_buffer, GST_VIDEO_BUFFER_FLAG_TFF);
2772 gst_video_push_user_data ((GstElement *) h264parse, &h264parse->user_data,
2775 gst_h264_parse_reset_frame (h264parse);
2781 gst_h264_parse_set_caps (GstBaseParse * parse, GstCaps * caps)
2783 GstH264Parse *h264parse;
2785 const GValue *codec_data_value;
2786 GstBuffer *codec_data = NULL;
2788 guint format, align, off;
2789 GstH264NalUnit nalu;
2790 GstH264ParserResult parseres;
2793 h264parse = GST_H264_PARSE (parse);
2796 h264parse->push_codec = FALSE;
2798 old_caps = gst_pad_get_current_caps (GST_BASE_PARSE_SINK_PAD (parse));
2800 if (!gst_caps_is_equal (old_caps, caps))
2801 gst_h264_parse_reset_stream_info (h264parse);
2802 gst_caps_unref (old_caps);
2805 str = gst_caps_get_structure (caps, 0);
2807 /* accept upstream info if provided */
2808 gst_structure_get_int (str, "width", &h264parse->width);
2809 gst_structure_get_int (str, "height", &h264parse->height);
2810 gst_structure_get_fraction (str, "framerate", &h264parse->fps_num,
2811 &h264parse->fps_den);
2812 gst_structure_get_fraction (str, "pixel-aspect-ratio",
2813 &h264parse->upstream_par_n, &h264parse->upstream_par_d);
2815 /* get upstream format and align from caps */
2816 gst_h264_parse_format_from_caps (caps, &format, &align);
2818 codec_data_value = gst_structure_get_value (str, "codec_data");
2820 /* fix up caps without stream-format for max. backwards compatibility */
2821 if (format == GST_H264_PARSE_FORMAT_NONE) {
2822 /* codec_data implies avc */
2823 if (codec_data_value != NULL) {
2824 GST_ERROR ("video/x-h264 caps with codec_data but no stream-format=avc");
2825 format = GST_H264_PARSE_FORMAT_AVC;
2827 /* otherwise assume bytestream input */
2828 GST_ERROR ("video/x-h264 caps without codec_data or stream-format");
2829 format = GST_H264_PARSE_FORMAT_BYTE;
2833 /* avc caps sanity checks */
2834 if (format == GST_H264_PARSE_FORMAT_AVC) {
2835 /* AVC requires codec_data, AVC3 might have one and/or SPS/PPS inline */
2836 if (codec_data_value == NULL)
2837 goto avc_caps_codec_data_missing;
2839 /* AVC implies alignment=au, everything else is not allowed */
2840 if (align == GST_H264_PARSE_ALIGN_NONE)
2841 align = GST_H264_PARSE_ALIGN_AU;
2842 else if (align != GST_H264_PARSE_ALIGN_AU)
2843 goto avc_caps_wrong_alignment;
2846 /* bytestream caps sanity checks */
2847 if (format == GST_H264_PARSE_FORMAT_BYTE) {
2848 /* should have SPS/PSS in-band (and/or oob in streamheader field) */
2849 if (codec_data_value != NULL)
2850 goto bytestream_caps_with_codec_data;
2853 /* packetized video has codec_data (required for AVC, optional for AVC3) */
2854 if (codec_data_value != NULL) {
2857 guint num_sps, num_pps;
2858 #ifndef GST_DISABLE_GST_DEBUG
2863 GST_DEBUG_OBJECT (h264parse, "have packetized h264");
2864 /* make note for optional split processing */
2865 h264parse->packetized = TRUE;
2867 /* codec_data field should hold a buffer */
2868 if (!GST_VALUE_HOLDS_BUFFER (codec_data_value))
2869 goto avc_caps_codec_data_wrong_type;
2871 codec_data = gst_value_get_buffer (codec_data_value);
2873 goto avc_caps_codec_data_missing;
2874 gst_buffer_map (codec_data, &map, GST_MAP_READ);
2878 /* parse the avcC data */
2879 if (size < 7) { /* when numSPS==0 and numPPS==0, length is 7 bytes */
2880 gst_buffer_unmap (codec_data, &map);
2881 goto avcc_too_small;
2883 /* parse the version, this must be 1 */
2885 gst_buffer_unmap (codec_data, &map);
2888 #ifndef GST_DISABLE_GST_DEBUG
2889 /* AVCProfileIndication */
2890 /* profile_compat */
2891 /* AVCLevelIndication */
2892 profile = (data[1] << 16) | (data[2] << 8) | data[3];
2893 GST_DEBUG_OBJECT (h264parse, "profile %06x", profile);
2896 /* 6 bits reserved | 2 bits lengthSizeMinusOne */
2897 /* this is the number of bytes in front of the NAL units to mark their
2899 h264parse->nal_length_size = (data[4] & 0x03) + 1;
2900 GST_DEBUG_OBJECT (h264parse, "nal length size %u",
2901 h264parse->nal_length_size);
2903 num_sps = data[5] & 0x1f;
2905 for (i = 0; i < num_sps; i++) {
2906 parseres = gst_h264_parser_identify_nalu_avc (h264parse->nalparser,
2907 data, off, size, 2, &nalu);
2908 if (parseres != GST_H264_PARSER_OK) {
2909 gst_buffer_unmap (codec_data, &map);
2910 goto avcc_too_small;
2913 gst_h264_parse_process_nal (h264parse, &nalu);
2914 off = nalu.offset + nalu.size;
2918 gst_buffer_unmap (codec_data, &map);
2919 goto avcc_too_small;
2921 num_pps = data[off];
2924 for (i = 0; i < num_pps; i++) {
2925 parseres = gst_h264_parser_identify_nalu_avc (h264parse->nalparser,
2926 data, off, size, 2, &nalu);
2927 if (parseres != GST_H264_PARSER_OK) {
2928 gst_buffer_unmap (codec_data, &map);
2929 goto avcc_too_small;
2932 gst_h264_parse_process_nal (h264parse, &nalu);
2933 off = nalu.offset + nalu.size;
2936 gst_buffer_unmap (codec_data, &map);
2938 gst_buffer_replace (&h264parse->codec_data_in, codec_data);
2940 /* don't confuse codec_data with inband sps/pps */
2941 h264parse->have_sps_in_frame = FALSE;
2942 h264parse->have_pps_in_frame = FALSE;
2943 } else if (format == GST_H264_PARSE_FORMAT_BYTE) {
2944 GST_DEBUG_OBJECT (h264parse, "have bytestream h264");
2945 /* nothing to pre-process */
2946 h264parse->packetized = FALSE;
2947 /* we have 4 sync bytes */
2948 h264parse->nal_length_size = 4;
2950 /* probably AVC3 without codec_data field, anything to do here? */
2956 /* prefer input type determined above */
2957 in_caps = gst_caps_new_simple ("video/x-h264",
2958 "parsed", G_TYPE_BOOLEAN, TRUE,
2959 "stream-format", G_TYPE_STRING,
2960 gst_h264_parse_get_string (h264parse, TRUE, format),
2961 "alignment", G_TYPE_STRING,
2962 gst_h264_parse_get_string (h264parse, FALSE, align), NULL);
2963 /* negotiate with downstream, sets ->format and ->align */
2964 gst_h264_parse_negotiate (h264parse, format, in_caps);
2965 gst_caps_unref (in_caps);
2968 if (format == h264parse->format && align == h264parse->align) {
2969 /* we did parse codec-data and might supplement src caps */
2970 gst_h264_parse_update_src_caps (h264parse, caps);
2971 } else if (format == GST_H264_PARSE_FORMAT_AVC
2972 || format == GST_H264_PARSE_FORMAT_AVC3) {
2973 /* if input != output, and input is avc, must split before anything else */
2974 /* arrange to insert codec-data in-stream if needed.
2975 * src caps are only arranged for later on */
2976 h264parse->push_codec = TRUE;
2977 h264parse->have_sps = FALSE;
2978 h264parse->have_pps = FALSE;
2979 if (h264parse->align == GST_H264_PARSE_ALIGN_NAL)
2980 h264parse->split_packetized = TRUE;
2981 h264parse->packetized = TRUE;
2984 h264parse->in_align = align;
2989 avc_caps_codec_data_wrong_type:
2991 GST_WARNING_OBJECT (parse, "H.264 AVC caps, codec_data field not a buffer");
2994 avc_caps_codec_data_missing:
2996 GST_WARNING_OBJECT (parse, "H.264 AVC caps, but no codec_data");
2999 avc_caps_wrong_alignment:
3001 GST_WARNING_OBJECT (parse, "H.264 AVC caps with NAL alignment, must be AU");
3004 bytestream_caps_with_codec_data:
3006 GST_WARNING_OBJECT (parse, "H.264 bytestream caps with codec_data is not "
3007 "expected, send SPS/PPS in-band with data or in streamheader field");
3012 GST_DEBUG_OBJECT (h264parse, "avcC size %" G_GSIZE_FORMAT " < 8", size);
3017 GST_DEBUG_OBJECT (h264parse, "wrong avcC version");
3022 GST_WARNING_OBJECT (h264parse, "refused caps %" GST_PTR_FORMAT, caps);
3028 remove_fields (GstCaps * caps, gboolean all)
3032 n = gst_caps_get_size (caps);
3033 for (i = 0; i < n; i++) {
3034 GstStructure *s = gst_caps_get_structure (caps, i);
3037 gst_structure_remove_field (s, "alignment");
3038 gst_structure_remove_field (s, "stream-format");
3040 gst_structure_remove_field (s, "parsed");
3045 gst_h264_parse_get_caps (GstBaseParse * parse, GstCaps * filter)
3047 GstCaps *peercaps, *templ;
3048 GstCaps *res, *tmp, *pcopy;
3050 templ = gst_pad_get_pad_template_caps (GST_BASE_PARSE_SINK_PAD (parse));
3052 GstCaps *fcopy = gst_caps_copy (filter);
3053 /* Remove the fields we convert */
3054 remove_fields (fcopy, TRUE);
3055 peercaps = gst_pad_peer_query_caps (GST_BASE_PARSE_SRC_PAD (parse), fcopy);
3056 gst_caps_unref (fcopy);
3058 peercaps = gst_pad_peer_query_caps (GST_BASE_PARSE_SRC_PAD (parse), NULL);
3060 pcopy = gst_caps_copy (peercaps);
3061 remove_fields (pcopy, TRUE);
3063 res = gst_caps_intersect_full (pcopy, templ, GST_CAPS_INTERSECT_FIRST);
3064 gst_caps_unref (pcopy);
3065 gst_caps_unref (templ);
3068 GstCaps *tmp = gst_caps_intersect_full (res, filter,
3069 GST_CAPS_INTERSECT_FIRST);
3070 gst_caps_unref (res);
3074 /* Try if we can put the downstream caps first */
3075 pcopy = gst_caps_copy (peercaps);
3076 remove_fields (pcopy, FALSE);
3077 tmp = gst_caps_intersect_full (pcopy, res, GST_CAPS_INTERSECT_FIRST);
3078 gst_caps_unref (pcopy);
3079 if (!gst_caps_is_empty (tmp))
3080 res = gst_caps_merge (tmp, res);
3082 gst_caps_unref (tmp);
3084 gst_caps_unref (peercaps);
3089 gst_h264_parse_event (GstBaseParse * parse, GstEvent * event)
3092 GstH264Parse *h264parse = GST_H264_PARSE (parse);
3094 switch (GST_EVENT_TYPE (event)) {
3095 case GST_EVENT_CUSTOM_DOWNSTREAM:
3097 GstClockTime timestamp, stream_time, running_time;
3098 gboolean all_headers;
3101 if (gst_video_event_is_force_key_unit (event)) {
3102 gst_video_event_parse_downstream_force_key_unit (event,
3103 ×tamp, &stream_time, &running_time, &all_headers, &count);
3105 GST_INFO_OBJECT (h264parse,
3106 "received downstream force key unit event, "
3107 "seqnum %d running_time %" GST_TIME_FORMAT
3108 " all_headers %d count %d", gst_event_get_seqnum (event),
3109 GST_TIME_ARGS (running_time), all_headers, count);
3110 if (h264parse->force_key_unit_event) {
3111 GST_INFO_OBJECT (h264parse, "ignoring force key unit event "
3112 "as one is already queued");
3114 h264parse->pending_key_unit_ts = running_time;
3115 gst_event_replace (&h264parse->force_key_unit_event, event);
3117 gst_event_unref (event);
3120 res = GST_BASE_PARSE_CLASS (parent_class)->sink_event (parse, event);
3125 case GST_EVENT_FLUSH_STOP:
3126 h264parse->dts = GST_CLOCK_TIME_NONE;
3127 h264parse->ts_trn_nb = GST_CLOCK_TIME_NONE;
3128 h264parse->push_codec = TRUE;
3130 res = GST_BASE_PARSE_CLASS (parent_class)->sink_event (parse, event);
3132 case GST_EVENT_SEGMENT:
3134 const GstSegment *segment;
3136 gst_event_parse_segment (event, &segment);
3137 /* don't try to mess with more subtle cases (e.g. seek) */
3138 if (segment->format == GST_FORMAT_TIME &&
3139 (segment->start != 0 || segment->rate != 1.0
3140 || segment->applied_rate != 1.0))
3141 h264parse->do_ts = FALSE;
3143 if (segment->flags & GST_SEEK_FLAG_TRICKMODE_FORWARD_PREDICTED) {
3144 GST_DEBUG_OBJECT (h264parse, "Will discard bidirectional frames");
3145 h264parse->discard_bidirectional = TRUE;
3149 h264parse->last_report = GST_CLOCK_TIME_NONE;
3151 res = GST_BASE_PARSE_CLASS (parent_class)->sink_event (parse, event);
3155 res = GST_BASE_PARSE_CLASS (parent_class)->sink_event (parse, event);
3162 gst_h264_parse_src_event (GstBaseParse * parse, GstEvent * event)
3165 GstH264Parse *h264parse = GST_H264_PARSE (parse);
3167 switch (GST_EVENT_TYPE (event)) {
3168 case GST_EVENT_CUSTOM_UPSTREAM:
3170 GstClockTime running_time;
3171 gboolean all_headers;
3174 if (gst_video_event_is_force_key_unit (event)) {
3175 gst_video_event_parse_upstream_force_key_unit (event,
3176 &running_time, &all_headers, &count);
3178 GST_INFO_OBJECT (h264parse, "received upstream force-key-unit event, "
3179 "seqnum %d running_time %" GST_TIME_FORMAT
3180 " all_headers %d count %d", gst_event_get_seqnum (event),
3181 GST_TIME_ARGS (running_time), all_headers, count);
3184 h264parse->pending_key_unit_ts = running_time;
3185 gst_event_replace (&h264parse->force_key_unit_event, event);
3188 res = GST_BASE_PARSE_CLASS (parent_class)->src_event (parse, event);
3192 res = GST_BASE_PARSE_CLASS (parent_class)->src_event (parse, event);
3200 gst_h264_parse_set_property (GObject * object, guint prop_id,
3201 const GValue * value, GParamSpec * pspec)
3203 GstH264Parse *parse;
3205 parse = GST_H264_PARSE (object);
3208 case PROP_CONFIG_INTERVAL:
3209 parse->interval = g_value_get_int (value);
3212 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
3218 gst_h264_parse_get_property (GObject * object, guint prop_id,
3219 GValue * value, GParamSpec * pspec)
3221 GstH264Parse *parse;
3223 parse = GST_H264_PARSE (object);
3226 case PROP_CONFIG_INTERVAL:
3227 g_value_set_int (value, parse->interval);
3230 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);