2 * Copyright (C) 2020 He Junyan <junyan.he@intel.com>
4 * This library is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Library General Public
6 * License as published by the Free Software Foundation; either
7 * version 2 of the License, or (at your option) any later version.
9 * This library is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Library General Public License for more details.
14 * You should have received a copy of the GNU Library General Public
15 * License along with this library; if not, write to the
16 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
17 * Boston, MA 02110-1301, USA.
21 * SECTION:element-av1parse
23 * @short_description: An AV1 stream parse.
25 * The minimal unit should be the BYTE.
26 * There are four types of AV1 alignment in the AV1 stream.
28 * alignment: byte, obu, frame, tu
30 * 1. Aligned to byte. The basic and default one for input.
31 * 2. Aligned to obu(Open Bitstream Units).
32 * 3. Aligned to frame. The default one for output. This ensures that
33 * each buffer contains only one frame or frame header with the
34 * show_existing flag for the base or sub layer. It is useful for
36 * 4. Aligned to tu(Temporal Unit). A temporal unit consists of all the
37 * OBUs that are associated with a specific, distinct time instant.
38 * When scalability is disabled, it contains just exact one showing
39 * frame(may contain several unshowing frames). When scalability is
40 * enabled, it contains frames depending on the layer number. It should
41 * begin with a temporal delimiter obu. It may be useful for mux/demux
42 * to index the data of some timestamp.
44 * The annex B define a special format for the temporal unit. The size of
45 * each temporal unit is extract out to the header of the buffer, and no
46 * size field inside the each obu. There are two stream formats:
48 * stream-format: obu-stream, annexb
50 * 1. obu-stream. The basic and default one.
51 * 2. annexb. A special stream of temporal unit. It also implies that the
52 * alignment should be TU.
54 * This AV1 parse implements the conversion between the alignments and the
55 * stream-formats. If the input and output have the same alignment and the
56 * same stream-format, it will check and bypass the data.
58 * ## Example launch line to generate annex B format AV1 stream:
60 * gst-launch-1.0 filesrc location=sample.av1 ! ivfparse ! av1parse ! \
61 * video/x-av1,alignment=\(string\)tu,stream-format=\(string\)annexb ! \
62 * filesink location=matroskamux ! filesink location=trans.mkv
72 #include <gst/base/gstbitreader.h>
73 #include <gst/base/gstbitwriter.h>
74 #include <gst/codecparsers/gstav1parser.h>
75 #include <gst/video/video.h>
76 #include "gstvideoparserselements.h"
77 #include "gstav1parse.h"
81 #define GST_AV1_MAX_LEB_128_SIZE 8
83 GST_DEBUG_CATEGORY (av1_parse_debug);
84 #define GST_CAT_DEFAULT av1_parse_debug
86 /* We combine the stream format and the alignment
87 together. When stream format is annexb, the
88 alignment must be TU. */
91 GST_AV1_PARSE_ALIGN_ERROR = -1,
92 GST_AV1_PARSE_ALIGN_NONE = 0,
93 GST_AV1_PARSE_ALIGN_BYTE,
94 GST_AV1_PARSE_ALIGN_OBU,
95 GST_AV1_PARSE_ALIGN_FRAME,
96 GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT,
97 GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT_ANNEX_B,
98 } GstAV1ParseAligment;
108 gboolean mono_chrome;
111 GstAV1Profile profile;
113 GstAV1ParseAligment in_align;
114 gboolean detect_annex_b;
115 GstAV1ParseAligment align;
117 GstAV1Parser *parser;
118 GstAdapter *cache_out;
119 guint last_parsed_offset;
120 GstAdapter *frame_cache;
121 guint highest_spatial_id;
122 gint last_shown_frame_temporal_id;
123 gint last_shown_frame_spatial_id;
124 gboolean within_one_frame;
125 gboolean update_caps;
132 static GstStaticPadTemplate sinktemplate = GST_STATIC_PAD_TEMPLATE ("sink",
135 GST_STATIC_CAPS ("video/x-av1"));
137 static GstStaticPadTemplate srctemplate = GST_STATIC_PAD_TEMPLATE ("src",
140 GST_STATIC_CAPS ("video/x-av1, parsed = (boolean) true, "
141 "stream-format=(string) { obu-stream, annexb }, "
142 "alignment=(string) { obu, tu, frame }"));
144 #define parent_class gst_av1_parse_parent_class
145 G_DEFINE_TYPE (GstAV1Parse, gst_av1_parse, GST_TYPE_BASE_PARSE);
146 GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (av1parse, "av1parse", GST_RANK_SECONDARY,
147 GST_TYPE_AV1_PARSE, videoparsers_element_init (plugin));
150 remove_fields (GstCaps * caps, gboolean all)
154 n = gst_caps_get_size (caps);
155 for (i = 0; i < n; i++) {
156 GstStructure *s = gst_caps_get_structure (caps, i);
159 gst_structure_remove_field (s, "alignment");
160 gst_structure_remove_field (s, "stream-format");
162 gst_structure_remove_field (s, "parsed");
167 _obu_name (GstAV1OBUType type)
170 case GST_AV1_OBU_SEQUENCE_HEADER:
171 return "sequence header";
172 case GST_AV1_OBU_TEMPORAL_DELIMITER:
173 return "temporal delimiter";
174 case GST_AV1_OBU_FRAME_HEADER:
175 return "frame header";
176 case GST_AV1_OBU_TILE_GROUP:
178 case GST_AV1_OBU_METADATA:
180 case GST_AV1_OBU_FRAME:
182 case GST_AV1_OBU_REDUNDANT_FRAME_HEADER:
183 return "redundant frame header";
184 case GST_AV1_OBU_TILE_LIST:
186 case GST_AV1_OBU_PADDING:
196 _read_leb128 (guint8 * data, GstAV1ParserResult * retval, guint32 * comsumed)
198 guint8 leb128_byte = 0;
205 gst_bit_reader_init (&br, data, 8);
207 cur_pos = gst_bit_reader_get_pos (&br);
208 for (i = 0; i < 8; i++) {
210 result = gst_bit_reader_get_bits_uint8 (&br, &leb128_byte, 8);
211 if (result == FALSE) {
212 *retval = GST_AV1_PARSER_BITSTREAM_ERROR;
216 value |= (((gint) leb128_byte & 0x7f) << (i * 7));
217 if (!(leb128_byte & 0x80))
221 *comsumed = (gst_bit_reader_get_pos (&br) - cur_pos) / 8;
222 /* check for bitstream conformance see chapter4.10.5 */
223 if (value < G_MAXUINT32) {
224 *retval = GST_AV1_PARSER_OK;
225 return (guint32) value;
227 GST_WARNING ("invalid leb128");
228 *retval = GST_AV1_PARSER_BITSTREAM_ERROR;
234 _leb_size_in_bytes (guint64 value)
239 } while ((value >>= 7) != 0);
245 _write_leb128 (guint8 * data, guint * len, guint64 value)
247 guint leb_size = _leb_size_in_bytes (value);
250 if (value > G_MAXUINT32 || leb_size > GST_AV1_MAX_LEB_128_SIZE)
253 for (i = 0; i < leb_size; ++i) {
254 guint8 byte = value & 0x7f;
257 /* Signal that more bytes follow. */
268 static gboolean gst_av1_parse_start (GstBaseParse * parse);
269 static gboolean gst_av1_parse_stop (GstBaseParse * parse);
270 static GstFlowReturn gst_av1_parse_handle_frame (GstBaseParse * parse,
271 GstBaseParseFrame * frame, gint * skipsize);
272 static gboolean gst_av1_parse_set_sink_caps (GstBaseParse * parse,
274 static GstCaps *gst_av1_parse_get_sink_caps (GstBaseParse * parse,
277 /* Clear the parse state related to data kind OBUs. */
279 gst_av1_parse_reset_obu_data_state (GstAV1Parse * self)
281 self->last_shown_frame_temporal_id = -1;
282 self->last_shown_frame_spatial_id = -1;
283 self->within_one_frame = FALSE;
287 gst_av1_parse_reset (GstAV1Parse * self)
291 self->subsampling_x = -1;
292 self->subsampling_y = -1;
293 self->mono_chrome = FALSE;
294 self->profile = GST_AV1_PROFILE_UNDEFINED;
296 self->align = GST_AV1_PARSE_ALIGN_NONE;
297 self->in_align = GST_AV1_PARSE_ALIGN_NONE;
298 self->detect_annex_b = FALSE;
299 self->discont = TRUE;
300 self->header = FALSE;
301 self->keyframe = FALSE;
302 self->show_frame = FALSE;
303 self->last_parsed_offset = 0;
304 self->highest_spatial_id = 0;
305 gst_av1_parse_reset_obu_data_state (self);
306 g_clear_pointer (&self->colorimetry, g_free);
307 g_clear_pointer (&self->parser, gst_av1_parser_free);
308 gst_adapter_clear (self->cache_out);
309 gst_adapter_clear (self->frame_cache);
313 gst_av1_parse_init (GstAV1Parse * self)
315 gst_base_parse_set_pts_interpolation (GST_BASE_PARSE (self), FALSE);
316 gst_base_parse_set_infer_ts (GST_BASE_PARSE (self), FALSE);
318 GST_PAD_SET_ACCEPT_INTERSECT (GST_BASE_PARSE_SINK_PAD (self));
319 GST_PAD_SET_ACCEPT_TEMPLATE (GST_BASE_PARSE_SINK_PAD (self));
321 self->cache_out = gst_adapter_new ();
322 self->frame_cache = gst_adapter_new ();
326 gst_av1_parse_finalize (GObject * object)
328 GstAV1Parse *self = GST_AV1_PARSE (object);
330 gst_av1_parse_reset (self);
331 g_object_unref (self->cache_out);
332 g_object_unref (self->frame_cache);
334 G_OBJECT_CLASS (parent_class)->finalize (object);
338 gst_av1_parse_class_init (GstAV1ParseClass * klass)
340 GObjectClass *gobject_class = (GObjectClass *) klass;
341 GstBaseParseClass *parse_class = GST_BASE_PARSE_CLASS (klass);
342 GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
344 gobject_class->finalize = gst_av1_parse_finalize;
345 parse_class->start = GST_DEBUG_FUNCPTR (gst_av1_parse_start);
346 parse_class->stop = GST_DEBUG_FUNCPTR (gst_av1_parse_stop);
347 parse_class->handle_frame = GST_DEBUG_FUNCPTR (gst_av1_parse_handle_frame);
348 parse_class->set_sink_caps = GST_DEBUG_FUNCPTR (gst_av1_parse_set_sink_caps);
349 parse_class->get_sink_caps = GST_DEBUG_FUNCPTR (gst_av1_parse_get_sink_caps);
351 gst_element_class_add_static_pad_template (element_class, &srctemplate);
352 gst_element_class_add_static_pad_template (element_class, &sinktemplate);
354 gst_element_class_set_static_metadata (element_class, "AV1 parser",
355 "Codec/Parser/Converter/Video",
356 "Parses AV1 streams", "He Junyan <junyan.he@intel.com>");
358 GST_DEBUG_CATEGORY_INIT (av1_parse_debug, "av1parse", 0, "av1 parser");
362 gst_av1_parse_start (GstBaseParse * parse)
364 GstAV1Parse *self = GST_AV1_PARSE (parse);
366 GST_DEBUG_OBJECT (self, "start");
368 gst_av1_parse_reset (self);
369 self->parser = gst_av1_parser_new ();
371 /* At least the OBU header. */
372 gst_base_parse_set_min_frame_size (parse, 1);
378 gst_av1_parse_stop (GstBaseParse * parse)
380 GstAV1Parse *self = GST_AV1_PARSE (parse);
382 GST_DEBUG_OBJECT (self, "stop");
383 g_clear_pointer (&self->parser, gst_av1_parser_free);
389 gst_av1_parse_profile_to_string (GstAV1Profile profile)
392 case GST_AV1_PROFILE_0:
394 case GST_AV1_PROFILE_1:
396 case GST_AV1_PROFILE_2:
397 return "professional";
406 gst_av1_parse_profile_from_string (const gchar * profile)
409 return GST_AV1_PROFILE_UNDEFINED;
411 if (g_strcmp0 (profile, "main") == 0)
412 return GST_AV1_PROFILE_0;
413 else if (g_strcmp0 (profile, "high") == 0)
414 return GST_AV1_PROFILE_1;
415 else if (g_strcmp0 (profile, "professional") == 0)
416 return GST_AV1_PROFILE_2;
418 return GST_AV1_PROFILE_UNDEFINED;
422 gst_av1_parse_alignment_to_steam_format_string (GstAV1ParseAligment align)
425 case GST_AV1_PARSE_ALIGN_BYTE:
427 case GST_AV1_PARSE_ALIGN_OBU:
428 case GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT:
429 case GST_AV1_PARSE_ALIGN_FRAME:
431 case GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT_ANNEX_B:
434 GST_WARNING ("Unrecognized steam format");
442 gst_av1_parse_alignment_to_string (GstAV1ParseAligment align)
445 case GST_AV1_PARSE_ALIGN_BYTE:
447 case GST_AV1_PARSE_ALIGN_OBU:
449 case GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT:
450 case GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT_ANNEX_B:
452 case GST_AV1_PARSE_ALIGN_FRAME:
455 GST_WARNING ("Unrecognized alignment");
462 static GstAV1ParseAligment
463 gst_av1_parse_alignment_from_string (const gchar * align,
464 const gchar * stream_format)
466 if (!align && !stream_format)
467 return GST_AV1_PARSE_ALIGN_NONE;
470 if (g_strcmp0 (stream_format, "annexb") == 0) {
471 if (align && g_strcmp0 (align, "tu") != 0) {
472 /* annex b stream must align to TU. */
473 return GST_AV1_PARSE_ALIGN_ERROR;
475 return GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT_ANNEX_B;
477 } else if (g_strcmp0 (stream_format, "obu-stream") != 0) {
479 return GST_AV1_PARSE_ALIGN_NONE;
482 /* stream-format is obu-stream, depends on align */
486 if (g_strcmp0 (align, "byte") == 0) {
487 return GST_AV1_PARSE_ALIGN_BYTE;
488 } else if (g_strcmp0 (align, "obu") == 0) {
489 return GST_AV1_PARSE_ALIGN_OBU;
490 } else if (g_strcmp0 (align, "tu") == 0) {
491 return GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT;
492 } else if (g_strcmp0 (align, "frame") == 0) {
493 return GST_AV1_PARSE_ALIGN_FRAME;
496 return GST_AV1_PARSE_ALIGN_NONE;
500 return GST_AV1_PARSE_ALIGN_NONE;
504 gst_av1_parse_caps_has_alignment (GstCaps * caps, GstAV1ParseAligment alignment)
506 guint i, j, caps_size;
507 const gchar *cmp_align_str = NULL;
508 const gchar *cmp_stream_str = NULL;
510 GST_DEBUG ("Try to find alignment %d in caps: %" GST_PTR_FORMAT,
513 caps_size = gst_caps_get_size (caps);
518 case GST_AV1_PARSE_ALIGN_BYTE:
519 cmp_align_str = "byte";
520 cmp_stream_str = "obu-stream";
522 case GST_AV1_PARSE_ALIGN_OBU:
523 cmp_align_str = "obu";
524 cmp_stream_str = "obu-stream";
526 case GST_AV1_PARSE_ALIGN_FRAME:
527 cmp_align_str = "frame";
528 cmp_stream_str = "obu-stream";
530 case GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT:
531 cmp_align_str = "tu";
532 cmp_stream_str = "obu-stream";
534 case GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT_ANNEX_B:
535 cmp_align_str = "tu";
536 cmp_stream_str = "annexb";
542 for (i = 0; i < caps_size; i++) {
543 GstStructure *s = gst_caps_get_structure (caps, i);
544 const GValue *alignment_value = gst_structure_get_value (s, "alignment");
545 const GValue *stream_value = gst_structure_get_value (s, "stream-format");
547 if (!alignment_value || !stream_value)
550 if (G_VALUE_HOLDS_STRING (alignment_value)) {
551 const gchar *align_str = g_value_get_string (alignment_value);
553 if (g_strcmp0 (align_str, cmp_align_str) != 0)
555 } else if (GST_VALUE_HOLDS_LIST (alignment_value)) {
556 guint num_values = gst_value_list_get_size (alignment_value);
558 for (j = 0; j < num_values; j++) {
559 const GValue *v = gst_value_list_get_value (alignment_value, j);
560 const gchar *align_str = g_value_get_string (v);
562 if (g_strcmp0 (align_str, cmp_align_str) == 0)
570 if (G_VALUE_HOLDS_STRING (stream_value)) {
571 const gchar *stream_str = g_value_get_string (stream_value);
573 if (g_strcmp0 (stream_str, cmp_stream_str) != 0)
575 } else if (GST_VALUE_HOLDS_LIST (stream_value)) {
576 guint num_values = gst_value_list_get_size (stream_value);
578 for (j = 0; j < num_values; j++) {
579 const GValue *v = gst_value_list_get_value (stream_value, j);
580 const gchar *stream_str = g_value_get_string (v);
582 if (g_strcmp0 (stream_str, cmp_stream_str) == 0)
596 static GstAV1ParseAligment
597 gst_av1_parse_alignment_from_caps (GstCaps * caps)
599 GstAV1ParseAligment align;
601 align = GST_AV1_PARSE_ALIGN_NONE;
603 GST_DEBUG ("parsing caps: %" GST_PTR_FORMAT, caps);
605 if (caps && gst_caps_get_size (caps) > 0) {
606 GstStructure *s = gst_caps_get_structure (caps, 0);
607 const gchar *str_align = NULL;
608 const gchar *str_stream = NULL;
610 str_align = gst_structure_get_string (s, "alignment");
611 str_stream = gst_structure_get_string (s, "stream-format");
613 align = gst_av1_parse_alignment_from_string (str_align, str_stream);
620 gst_av1_parse_update_src_caps (GstAV1Parse * self, GstCaps * caps)
622 GstCaps *sink_caps, *src_caps;
623 GstCaps *final_caps = NULL;
624 GstStructure *s = NULL;
626 gint par_n = 0, par_d = 0;
627 gint fps_n = 0, fps_d = 0;
628 const gchar *profile = NULL;
630 if (G_UNLIKELY (!gst_pad_has_current_caps (GST_BASE_PARSE_SRC_PAD (self))))
631 self->update_caps = TRUE;
633 if (!self->update_caps)
636 /* if this is being called from the first _setcaps call, caps on the sinkpad
637 * aren't set yet and so they need to be passed as an argument */
639 sink_caps = gst_caps_ref (caps);
641 sink_caps = gst_pad_get_current_caps (GST_BASE_PARSE_SINK_PAD (self));
643 /* carry over input caps as much as possible; override with our own stuff */
645 sink_caps = gst_caps_new_empty_simple ("video/x-av1");
647 s = gst_caps_get_structure (sink_caps, 0);
649 final_caps = gst_caps_copy (sink_caps);
651 if (s && gst_structure_has_field (s, "width") &&
652 gst_structure_has_field (s, "height")) {
653 gst_structure_get_int (s, "width", &width);
654 gst_structure_get_int (s, "height", &height);
657 height = self->height;
660 if (width > 0 && height > 0)
661 gst_caps_set_simple (final_caps, "width", G_TYPE_INT, width,
662 "height", G_TYPE_INT, height, NULL);
664 if (s && gst_structure_get_fraction (s, "pixel-aspect-ratio", &par_n, &par_d)) {
665 if (par_n != 0 && par_d != 0) {
666 gst_caps_set_simple (final_caps, "pixel-aspect-ratio",
667 GST_TYPE_FRACTION, par_n, par_d, NULL);
671 if (s && gst_structure_has_field (s, "framerate")) {
672 gst_structure_get_fraction (s, "framerate", &fps_n, &fps_d);
675 if (fps_n > 0 && fps_d > 0) {
676 gst_caps_set_simple (final_caps, "framerate",
677 GST_TYPE_FRACTION, fps_n, fps_d, NULL);
678 gst_base_parse_set_frame_rate (GST_BASE_PARSE (self), fps_n, fps_d, 0, 0);
681 /* When not RGB, the chroma format is needed. */
682 if (self->colorimetry == NULL ||
683 (g_strcmp0 (self->colorimetry, GST_VIDEO_COLORIMETRY_SRGB) != 0)) {
684 const gchar *chroma_format = NULL;
686 if (self->subsampling_x == 1 && self->subsampling_y == 1) {
687 if (!self->mono_chrome) {
688 chroma_format = "4:2:0";
690 chroma_format = "4:0:0";
692 } else if (self->subsampling_x == 1 && self->subsampling_y == 0) {
693 chroma_format = "4:2:2";
694 } else if (self->subsampling_x == 0 && self->subsampling_y == 0) {
695 chroma_format = "4:4:4";
699 gst_caps_set_simple (final_caps,
700 "chroma-format", G_TYPE_STRING, chroma_format, NULL);
704 gst_caps_set_simple (final_caps,
705 "bit-depth-luma", G_TYPE_UINT, self->bit_depth,
706 "bit-depth-chroma", G_TYPE_UINT, self->bit_depth, NULL);
708 if (self->colorimetry && (!s || !gst_structure_has_field (s, "colorimetry")))
709 gst_caps_set_simple (final_caps,
710 "colorimetry", G_TYPE_STRING, self->colorimetry, NULL);
712 g_assert (self->align > GST_AV1_PARSE_ALIGN_NONE);
713 gst_caps_set_simple (final_caps, "parsed", G_TYPE_BOOLEAN, TRUE,
714 "stream-format", G_TYPE_STRING,
715 gst_av1_parse_alignment_to_steam_format_string (self->align),
716 "alignment", G_TYPE_STRING,
717 gst_av1_parse_alignment_to_string (self->align), NULL);
719 profile = gst_av1_parse_profile_to_string (self->profile);
721 gst_caps_set_simple (final_caps, "profile", G_TYPE_STRING, profile, NULL);
723 src_caps = gst_pad_get_current_caps (GST_BASE_PARSE_SRC_PAD (self));
725 if (!(src_caps && gst_caps_is_strictly_equal (src_caps, final_caps))) {
726 GST_DEBUG_OBJECT (self, "Update src caps %" GST_PTR_FORMAT, final_caps);
727 gst_pad_set_caps (GST_BASE_PARSE_SRC_PAD (self), final_caps);
730 gst_clear_caps (&src_caps);
731 gst_caps_unref (final_caps);
732 gst_caps_unref (sink_caps);
734 self->update_caps = FALSE;
737 /* check downstream caps to configure format and alignment */
739 gst_av1_parse_negotiate (GstAV1Parse * self, GstCaps * in_caps)
742 GstAV1ParseAligment align;
744 caps = gst_pad_get_allowed_caps (GST_BASE_PARSE_SRC_PAD (self));
745 GST_DEBUG_OBJECT (self, "allowed caps: %" GST_PTR_FORMAT, caps);
747 /* concentrate on leading structure, since decodebin parser
748 * capsfilter always includes parser template caps */
750 caps = gst_caps_truncate (caps);
751 GST_DEBUG_OBJECT (self, "negotiating with caps: %" GST_PTR_FORMAT, caps);
754 /* prefer TU as default */
755 if (gst_av1_parse_caps_has_alignment (caps,
756 GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT)) {
757 self->align = GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT;
761 /* Both upsteam and downstream support, best */
762 if (in_caps && caps) {
763 if (gst_caps_can_intersect (in_caps, caps)) {
764 GstCaps *common_caps = NULL;
766 common_caps = gst_caps_intersect (in_caps, caps);
767 align = gst_av1_parse_alignment_from_caps (common_caps);
768 gst_clear_caps (&common_caps);
770 if (align != GST_AV1_PARSE_ALIGN_NONE
771 && align != GST_AV1_PARSE_ALIGN_ERROR) {
778 /* Select first one of downstream support */
779 if (caps && !gst_caps_is_empty (caps)) {
780 /* fixate to avoid ambiguity with lists when parsing */
781 caps = gst_caps_fixate (caps);
782 align = gst_av1_parse_alignment_from_caps (caps);
784 if (align != GST_AV1_PARSE_ALIGN_NONE && align != GST_AV1_PARSE_ALIGN_ERROR) {
791 self->align = GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT;
794 GST_INFO_OBJECT (self, "selected alignment %s",
795 gst_av1_parse_alignment_to_string (self->align));
797 gst_clear_caps (&caps);
801 gst_av1_parse_get_sink_caps (GstBaseParse * parse, GstCaps * filter)
803 GstCaps *peercaps, *templ;
804 GstCaps *res, *tmp, *pcopy;
806 templ = gst_pad_get_pad_template_caps (GST_BASE_PARSE_SINK_PAD (parse));
808 GstCaps *fcopy = gst_caps_copy (filter);
809 /* Remove the fields we convert */
810 remove_fields (fcopy, TRUE);
811 peercaps = gst_pad_peer_query_caps (GST_BASE_PARSE_SRC_PAD (parse), fcopy);
812 gst_caps_unref (fcopy);
814 peercaps = gst_pad_peer_query_caps (GST_BASE_PARSE_SRC_PAD (parse), NULL);
817 pcopy = gst_caps_copy (peercaps);
818 remove_fields (pcopy, TRUE);
820 res = gst_caps_intersect_full (pcopy, templ, GST_CAPS_INTERSECT_FIRST);
821 gst_caps_unref (pcopy);
822 gst_caps_unref (templ);
825 GstCaps *tmp = gst_caps_intersect_full (res, filter,
826 GST_CAPS_INTERSECT_FIRST);
827 gst_caps_unref (res);
831 /* Try if we can put the downstream caps first */
832 pcopy = gst_caps_copy (peercaps);
833 remove_fields (pcopy, FALSE);
834 tmp = gst_caps_intersect_full (pcopy, res, GST_CAPS_INTERSECT_FIRST);
835 gst_caps_unref (pcopy);
836 if (!gst_caps_is_empty (tmp))
837 res = gst_caps_merge (tmp, res);
839 gst_caps_unref (tmp);
841 gst_caps_unref (peercaps);
847 gst_av1_parse_set_sink_caps (GstBaseParse * parse, GstCaps * caps)
849 GstAV1Parse *self = GST_AV1_PARSE (parse);
851 GstAV1ParseAligment align;
852 GstCaps *in_caps = NULL;
853 const gchar *profile;
855 str = gst_caps_get_structure (caps, 0);
857 /* accept upstream info if provided */
858 gst_structure_get_int (str, "width", &self->width);
859 gst_structure_get_int (str, "height", &self->height);
860 profile = gst_structure_get_string (str, "profile");
862 self->profile = gst_av1_parse_profile_from_string (profile);
864 /* get upstream align from caps */
865 align = gst_av1_parse_alignment_from_caps (caps);
866 if (align == GST_AV1_PARSE_ALIGN_ERROR) {
867 GST_ERROR_OBJECT (self, "Sink caps %" GST_PTR_FORMAT " set stream-format"
868 " and alignment conflict.", caps);
872 in_caps = gst_caps_copy (caps);
874 if (align == GST_AV1_PARSE_ALIGN_NONE) {
875 align = GST_AV1_PARSE_ALIGN_BYTE;
876 gst_caps_set_simple (in_caps, "alignment", G_TYPE_STRING,
877 gst_av1_parse_alignment_to_string (align),
878 "stream-format", G_TYPE_STRING, "obu-stream", NULL);
881 /* negotiate with downstream, set output align */
882 gst_av1_parse_negotiate (self, in_caps);
884 self->update_caps = TRUE;
886 /* if all of decoder's capability related values are provided
887 * by upstream, update src caps now */
888 if (self->width > 0 && self->height > 0 && profile)
889 gst_av1_parse_update_src_caps (self, in_caps);
891 gst_caps_unref (in_caps);
893 self->in_align = align;
895 if (self->in_align == GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT)
896 self->detect_annex_b = TRUE;
898 if (self->in_align == GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT_ANNEX_B) {
899 gst_av1_parser_reset (self->parser, TRUE);
901 gst_av1_parser_reset (self->parser, FALSE);
908 gst_av1_parse_push_data (GstAV1Parse * self, GstBaseParseFrame * frame,
909 guint32 finish_sz, gboolean frame_finished)
912 GstBuffer *buf, *header_buf;
913 GstBuffer *buffer = frame->buffer;
914 GstFlowReturn ret = GST_FLOW_OK;
916 /* Need to generate the final TU annex-b format */
917 if (self->align == GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT_ANNEX_B) {
918 guint8 size_data[GST_AV1_MAX_LEB_128_SIZE];
922 /* When push a TU, it must also be a frame end. */
923 g_assert (frame_finished);
925 /* Still some left in the frame cache */
926 len = gst_adapter_available (self->frame_cache);
928 buf = gst_adapter_take_buffer (self->frame_cache, len);
930 /* frame_unit_size */
931 _write_leb128 (size_data, &size_len, len);
932 header_buf = gst_buffer_new_memdup (size_data, size_len);
933 GST_BUFFER_PTS (header_buf) = GST_BUFFER_PTS (buf);
934 GST_BUFFER_DTS (header_buf) = GST_BUFFER_DTS (buf);
935 GST_BUFFER_DURATION (header_buf) = GST_BUFFER_DURATION (buf);
937 gst_adapter_push (self->cache_out, header_buf);
938 gst_adapter_push (self->cache_out, buf);
941 len = gst_adapter_available (self->cache_out);
943 buf = gst_adapter_take_buffer (self->cache_out, len);
945 /* temporal_unit_size */
946 _write_leb128 (size_data, &size_len, len);
947 header_buf = gst_buffer_new_memdup (size_data, size_len);
948 GST_BUFFER_PTS (header_buf) = GST_BUFFER_PTS (buf);
949 GST_BUFFER_DTS (header_buf) = GST_BUFFER_DTS (buf);
950 GST_BUFFER_DURATION (header_buf) = GST_BUFFER_DURATION (buf);
952 gst_adapter_push (self->cache_out, header_buf);
953 gst_adapter_push (self->cache_out, buf);
957 sz = gst_adapter_available (self->cache_out);
959 buf = gst_adapter_take_buffer (self->cache_out, sz);
960 gst_buffer_copy_into (buf, buffer, GST_BUFFER_COPY_METADATA, 0, -1);
962 GST_BUFFER_FLAG_SET (buf, GST_BUFFER_FLAG_DISCONT);
963 self->discont = FALSE;
965 GST_BUFFER_FLAG_UNSET (buf, GST_BUFFER_FLAG_DISCONT);
969 GST_BUFFER_FLAG_SET (buf, GST_BUFFER_FLAG_HEADER);
970 self->header = FALSE;
972 GST_BUFFER_FLAG_UNSET (buf, GST_BUFFER_FLAG_HEADER);
975 if (self->keyframe) {
976 GST_BUFFER_FLAG_UNSET (buf, GST_BUFFER_FLAG_DELTA_UNIT);
977 self->keyframe = FALSE;
979 GST_BUFFER_FLAG_SET (buf, GST_BUFFER_FLAG_DELTA_UNIT);
982 if (frame_finished) {
983 GST_BUFFER_FLAG_SET (buf, GST_BUFFER_FLAG_MARKER);
985 GST_BUFFER_FLAG_UNSET (buf, GST_BUFFER_FLAG_MARKER);
988 if (self->align == GST_AV1_PARSE_ALIGN_FRAME) {
989 if (!self->show_frame) {
990 GST_BUFFER_FLAG_SET (buf, GST_BUFFER_FLAG_DECODE_ONLY);
992 GST_BUFFER_FLAG_UNSET (buf, GST_BUFFER_FLAG_DECODE_ONLY);
995 GST_BUFFER_FLAG_UNSET (buf, GST_BUFFER_FLAG_DECODE_ONLY);
998 gst_buffer_replace (&frame->out_buffer, buf);
999 gst_buffer_unref (buf);
1001 gst_av1_parse_update_src_caps (self, NULL);
1002 GST_LOG_OBJECT (self, "comsumed %d, output one buffer with size %"
1003 G_GSSIZE_FORMAT, finish_sz, sz);
1004 ret = gst_base_parse_finish_frame (GST_BASE_PARSE (self), frame, finish_sz);
1011 gst_av1_parse_convert_to_annexb (GstAV1Parse * self, GstBuffer * buffer,
1012 GstAV1OBU * obu, gboolean frame_complete)
1014 guint8 size_data[GST_AV1_MAX_LEB_128_SIZE];
1017 GstBuffer *buf, *buf2;
1019 guint len, len2, offset;
1022 _write_leb128 (size_data, &size_len,
1023 obu->obu_size + 1 + obu->header.obu_extention_flag);
1025 gst_bit_writer_init_with_size (&bs, 128, FALSE);
1026 /* obu_forbidden_bit */
1027 gst_bit_writer_put_bits_uint8 (&bs, 0, 1);
1029 gst_bit_writer_put_bits_uint8 (&bs, obu->obu_type, 4);
1030 /* obu_extension_flag */
1031 gst_bit_writer_put_bits_uint8 (&bs, obu->header.obu_extention_flag, 1);
1032 /* obu_has_size_field */
1033 gst_bit_writer_put_bits_uint8 (&bs, 0, 1);
1034 /* obu_reserved_1bit */
1035 gst_bit_writer_put_bits_uint8 (&bs, 0, 1);
1036 if (obu->header.obu_extention_flag) {
1038 gst_bit_writer_put_bits_uint8 (&bs, obu->header.obu_temporal_id, 3);
1040 gst_bit_writer_put_bits_uint8 (&bs, obu->header.obu_spatial_id, 2);
1041 /* extension_header_reserved_3bits */
1042 gst_bit_writer_put_bits_uint8 (&bs, 0, 3);
1044 g_assert (GST_BIT_WRITER_BIT_SIZE (&bs) % 8 == 0);
1047 len += GST_BIT_WRITER_BIT_SIZE (&bs) / 8;
1048 len += obu->obu_size;
1050 data = g_malloc (len);
1053 memcpy (data + offset, size_data, size_len);
1056 memcpy (data + offset, GST_BIT_WRITER_DATA (&bs),
1057 GST_BIT_WRITER_BIT_SIZE (&bs) / 8);
1058 offset += GST_BIT_WRITER_BIT_SIZE (&bs) / 8;
1060 memcpy (data + offset, obu->data, obu->obu_size);
1062 /* The buf of this OBU */
1063 buf = gst_buffer_new_wrapped (data, len);
1064 GST_BUFFER_PTS (buf) = GST_BUFFER_PTS (buffer);
1065 GST_BUFFER_DTS (buf) = GST_BUFFER_DTS (buffer);
1066 GST_BUFFER_DURATION (buf) = GST_BUFFER_DURATION (buffer);
1068 gst_adapter_push (self->frame_cache, buf);
1070 if (frame_complete) {
1071 len2 = gst_adapter_available (self->frame_cache);
1072 buf2 = gst_adapter_take_buffer (self->frame_cache, len2);
1074 /* frame_unit_size */
1075 _write_leb128 (size_data, &size_len, len2);
1076 buf = gst_buffer_new_memdup (size_data, size_len);
1077 GST_BUFFER_PTS (buf) = GST_BUFFER_PTS (buf2);
1078 GST_BUFFER_DTS (buf) = GST_BUFFER_DTS (buf2);
1079 GST_BUFFER_DURATION (buf) = GST_BUFFER_DURATION (buf2);
1081 gst_adapter_push (self->cache_out, buf);
1082 gst_adapter_push (self->cache_out, buf2);
1085 gst_bit_writer_reset (&bs);
1089 gst_av1_parse_convert_from_annexb (GstAV1Parse * self, GstBuffer * buffer,
1092 guint8 size_data[GST_AV1_MAX_LEB_128_SIZE];
1099 _write_leb128 (size_data, &size_len, obu->obu_size);
1103 if (obu->header.obu_extention_flag)
1106 len += obu->obu_size;
1108 gst_bit_writer_init_with_size (&bs, 128, FALSE);
1109 /* obu_forbidden_bit */
1110 gst_bit_writer_put_bits_uint8 (&bs, 0, 1);
1112 gst_bit_writer_put_bits_uint8 (&bs, obu->obu_type, 4);
1113 /* obu_extension_flag */
1114 gst_bit_writer_put_bits_uint8 (&bs, obu->header.obu_extention_flag, 1);
1115 /* obu_has_size_field */
1116 gst_bit_writer_put_bits_uint8 (&bs, 1, 1);
1117 /* obu_reserved_1bit */
1118 gst_bit_writer_put_bits_uint8 (&bs, 0, 1);
1119 if (obu->header.obu_extention_flag) {
1121 gst_bit_writer_put_bits_uint8 (&bs, obu->header.obu_temporal_id, 3);
1123 gst_bit_writer_put_bits_uint8 (&bs, obu->header.obu_spatial_id, 2);
1124 /* extension_header_reserved_3bits */
1125 gst_bit_writer_put_bits_uint8 (&bs, 0, 3);
1127 g_assert (GST_BIT_WRITER_BIT_SIZE (&bs) % 8 == 0);
1129 data = g_malloc (len);
1131 memcpy (data + offset, GST_BIT_WRITER_DATA (&bs),
1132 GST_BIT_WRITER_BIT_SIZE (&bs) / 8);
1133 offset += GST_BIT_WRITER_BIT_SIZE (&bs) / 8;
1135 memcpy (data + offset, size_data, size_len);
1138 memcpy (data + offset, obu->data, obu->obu_size);
1140 buf = gst_buffer_new_wrapped (data, len);
1141 GST_BUFFER_PTS (buf) = GST_BUFFER_PTS (buffer);
1142 GST_BUFFER_DTS (buf) = GST_BUFFER_DTS (buffer);
1143 GST_BUFFER_DURATION (buf) = GST_BUFFER_DURATION (buffer);
1145 gst_adapter_push (self->cache_out, buf);
1147 gst_bit_writer_reset (&bs);
1151 gst_av1_parse_cache_one_obu (GstAV1Parse * self, GstBuffer * buffer,
1152 GstAV1OBU * obu, guint8 * data, guint32 size, gboolean frame_complete)
1154 gboolean need_convert = FALSE;
1157 if (self->in_align != self->align
1158 && (self->in_align == GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT_ANNEX_B
1159 || self->align == GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT_ANNEX_B))
1160 need_convert = TRUE;
1163 if (self->in_align == GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT_ANNEX_B) {
1164 gst_av1_parse_convert_from_annexb (self, buffer, obu);
1166 gst_av1_parse_convert_to_annexb (self, buffer, obu, frame_complete);
1168 } else if (self->align == GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT_ANNEX_B) {
1169 g_assert (self->in_align == GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT_ANNEX_B);
1170 gst_av1_parse_convert_to_annexb (self, buffer, obu, frame_complete);
1172 buf = gst_buffer_new_memdup (data, size);
1173 GST_BUFFER_PTS (buf) = GST_BUFFER_PTS (buffer);
1174 GST_BUFFER_DTS (buf) = GST_BUFFER_DTS (buffer);
1175 GST_BUFFER_DURATION (buf) = GST_BUFFER_DURATION (buffer);
1177 gst_adapter_push (self->cache_out, buf);
1181 static GstAV1ParserResult
1182 gst_av1_parse_handle_sequence_obu (GstAV1Parse * self, GstAV1OBU * obu)
1184 GstAV1SequenceHeaderOBU seq_header;
1185 GstAV1ParserResult res;
1189 res = gst_av1_parser_parse_sequence_header_obu (self->parser,
1191 if (res != GST_AV1_PARSER_OK)
1194 if (self->width != seq_header.max_frame_width_minus_1 + 1) {
1195 self->width = seq_header.max_frame_width_minus_1 + 1;
1196 self->update_caps = TRUE;
1198 if (self->height != seq_header.max_frame_height_minus_1 + 1) {
1199 self->height = seq_header.max_frame_height_minus_1 + 1;
1200 self->update_caps = TRUE;
1203 if (seq_header.color_config.color_description_present_flag) {
1204 GstVideoColorimetry cinfo;
1205 gchar *colorimetry = NULL;
1207 if (seq_header.color_config.color_range)
1208 cinfo.range = GST_VIDEO_COLOR_RANGE_0_255;
1210 cinfo.range = GST_VIDEO_COLOR_RANGE_16_235;
1212 cinfo.matrix = gst_video_color_matrix_from_iso
1213 (seq_header.color_config.matrix_coefficients);
1214 cinfo.transfer = gst_video_transfer_function_from_iso
1215 (seq_header.color_config.transfer_characteristics);
1216 cinfo.primaries = gst_video_color_primaries_from_iso
1217 (seq_header.color_config.color_primaries);
1219 colorimetry = gst_video_colorimetry_to_string (&cinfo);
1221 if (g_strcmp0 (colorimetry, self->colorimetry) != 0) {
1222 g_free (self->colorimetry);
1223 self->colorimetry = colorimetry;
1225 self->update_caps = TRUE;
1228 g_clear_pointer (&colorimetry, g_free);
1231 if (self->subsampling_x != seq_header.color_config.subsampling_x) {
1232 self->subsampling_x = seq_header.color_config.subsampling_x;
1233 self->update_caps = TRUE;
1236 if (self->subsampling_y != seq_header.color_config.subsampling_y) {
1237 self->subsampling_y = seq_header.color_config.subsampling_y;
1238 self->update_caps = TRUE;
1241 if (self->mono_chrome != seq_header.color_config.mono_chrome) {
1242 self->mono_chrome = seq_header.color_config.mono_chrome;
1243 self->update_caps = TRUE;
1246 if (self->bit_depth != seq_header.bit_depth) {
1247 self->bit_depth = seq_header.bit_depth;
1248 self->update_caps = TRUE;
1251 if (self->profile != seq_header.seq_profile) {
1252 self->profile = seq_header.seq_profile;
1253 self->update_caps = TRUE;
1256 val = (self->parser->state.operating_point_idc >> 8) & 0x0f;
1257 for (i = 0; i < (1 << GST_AV1_MAX_SPATIAL_LAYERS); i++) {
1259 self->highest_spatial_id = i;
1262 return GST_AV1_PARSER_OK;
1265 /* Check whether the frame start a new TU.
1266 The obu here should be a shown frame/frame header. */
1268 gst_av1_parse_frame_start_new_temporal_unit (GstAV1Parse * self,
1271 gboolean ret = FALSE;
1273 g_assert (obu->obu_type == GST_AV1_OBU_FRAME_HEADER
1274 || obu->obu_type == GST_AV1_OBU_FRAME);
1276 /* 7.5.Ordering of OBUs: The value of temporal_id must be the same in all
1277 OBU extension headers that are contained in the same temporal unit. */
1278 if (self->last_shown_frame_temporal_id >= 0 &&
1279 obu->header.obu_temporal_id != self->last_shown_frame_temporal_id) {
1284 /* If scalability is not being used, only one shown frame for each
1285 temporal unit. So the new frame belongs to a new temporal unit. */
1286 if (!self->within_one_frame && self->last_shown_frame_temporal_id >= 0 &&
1287 self->parser->state.operating_point_idc == 0) {
1292 /* The new frame has the same layer IDs with the last shown frame,
1293 it should belong to a new temporal unit. */
1294 if (!self->within_one_frame &&
1295 obu->header.obu_temporal_id == self->last_shown_frame_temporal_id &&
1296 obu->header.obu_spatial_id == self->last_shown_frame_spatial_id) {
1303 if (self->within_one_frame)
1304 GST_WARNING_OBJECT (self,
1305 "Start a new temporal unit with incompleted frame.");
1307 gst_av1_parse_reset_obu_data_state (self);
1313 /* frame_complete will be set true if it is the frame edge. */
1314 static GstAV1ParserResult
1315 gst_av1_parse_handle_one_obu (GstAV1Parse * self, GstAV1OBU * obu,
1316 gboolean * frame_complete, gboolean * check_new_tu)
1318 GstAV1ParserResult res = GST_AV1_PARSER_OK;
1319 GstAV1MetadataOBU metadata;
1320 GstAV1FrameHeaderOBU frame_header;
1321 GstAV1TileListOBU tile_list;
1322 GstAV1TileGroupOBU tile_group;
1323 GstAV1FrameOBU frame;
1325 *frame_complete = FALSE;
1327 switch (obu->obu_type) {
1328 case GST_AV1_OBU_TEMPORAL_DELIMITER:
1329 res = gst_av1_parser_parse_temporal_delimiter_obu (self->parser, obu);
1331 case GST_AV1_OBU_SEQUENCE_HEADER:
1332 res = gst_av1_parse_handle_sequence_obu (self, obu);
1334 case GST_AV1_OBU_REDUNDANT_FRAME_HEADER:
1335 res = gst_av1_parser_parse_frame_header_obu (self->parser, obu,
1338 case GST_AV1_OBU_FRAME_HEADER:
1339 res = gst_av1_parser_parse_frame_header_obu (self->parser, obu,
1342 case GST_AV1_OBU_FRAME:
1343 res = gst_av1_parser_parse_frame_obu (self->parser, obu, &frame);
1345 case GST_AV1_OBU_METADATA:
1346 res = gst_av1_parser_parse_metadata_obu (self->parser, obu, &metadata);
1348 case GST_AV1_OBU_TILE_GROUP:
1350 gst_av1_parser_parse_tile_group_obu (self->parser, obu, &tile_group);
1352 case GST_AV1_OBU_TILE_LIST:
1353 res = gst_av1_parser_parse_tile_list_obu (self->parser, obu, &tile_list);
1355 case GST_AV1_OBU_PADDING:
1358 GST_WARNING_OBJECT (self, "an unrecognized obu type %d", obu->obu_type);
1359 res = GST_AV1_PARSER_BITSTREAM_ERROR;
1363 GST_LOG_OBJECT (self, "parsing the obu %s, result is %d",
1364 _obu_name (obu->obu_type), res);
1365 if (res != GST_AV1_PARSER_OK)
1369 All OBU extension headers that are contained in the same temporal
1370 unit and have the same spatial_id value must have the same temporal_id
1373 OBUs with spatial level IDs (spatial_id) greater than 0 must
1374 appear within a temporal unit in increasing order of the spatial
1376 if (obu->header.obu_spatial_id > self->highest_spatial_id) {
1377 GST_WARNING_OBJECT (self,
1378 "spatial_id %d is bigger than highest_spatial_id %d",
1379 obu->header.obu_spatial_id, self->highest_spatial_id);
1380 res = GST_AV1_PARSER_BITSTREAM_ERROR;
1384 /* If to check a new temporal starts, return early.
1385 In 7.5.Ordering of OBUs: Sequence header OBUs may appear in any order
1386 within a coded video sequence. So it is allowed to repeat the sequence
1387 header within one temporal unit, and sequence header does not definitely
1388 start a TU. We only check TD here. */
1389 if (obu->obu_type == GST_AV1_OBU_TEMPORAL_DELIMITER) {
1390 gst_av1_parse_reset_obu_data_state (self);
1393 *check_new_tu = TRUE;
1394 res = GST_AV1_PARSER_OK;
1399 if (obu->obu_type == GST_AV1_OBU_SEQUENCE_HEADER)
1400 self->header = TRUE;
1402 if (obu->obu_type == GST_AV1_OBU_FRAME_HEADER
1403 || obu->obu_type == GST_AV1_OBU_FRAME
1404 || obu->obu_type == GST_AV1_OBU_REDUNDANT_FRAME_HEADER) {
1405 GstAV1FrameHeaderOBU *fh = &frame_header;
1407 if (obu->obu_type == GST_AV1_OBU_FRAME)
1408 fh = &frame.frame_header;
1410 self->show_frame = fh->show_frame || fh->show_existing_frame;
1411 if (self->show_frame) {
1412 /* Check whether a new temporal starts, and return early. */
1413 if (check_new_tu && obu->obu_type != GST_AV1_OBU_REDUNDANT_FRAME_HEADER
1414 && gst_av1_parse_frame_start_new_temporal_unit (self, obu)) {
1415 *check_new_tu = TRUE;
1416 res = GST_AV1_PARSER_OK;
1420 self->last_shown_frame_temporal_id = obu->header.obu_temporal_id;
1421 self->last_shown_frame_spatial_id = obu->header.obu_spatial_id;
1424 self->within_one_frame = TRUE;
1426 /* if a show_existing_frame case, only update key frame.
1427 otherwise, update all type of frame. */
1428 if (!fh->show_existing_frame || fh->frame_type == GST_AV1_KEY_FRAME)
1429 res = gst_av1_parser_reference_frame_update (self->parser, fh);
1431 if (res != GST_AV1_PARSER_OK)
1432 GST_WARNING_OBJECT (self, "update frame get result %d", res);
1434 if (fh->show_existing_frame) {
1435 *frame_complete = TRUE;
1436 self->within_one_frame = FALSE;
1439 if (fh->frame_type == GST_AV1_KEY_FRAME)
1440 self->keyframe = TRUE;
1443 if (obu->obu_type == GST_AV1_OBU_TILE_GROUP
1444 || obu->obu_type == GST_AV1_OBU_FRAME) {
1445 GstAV1TileGroupOBU *tg = &tile_group;
1447 self->within_one_frame = TRUE;
1449 if (obu->obu_type == GST_AV1_OBU_FRAME)
1450 tg = &frame.tile_group;
1452 if (tg->tg_end == tg->num_tiles - 1) {
1453 *frame_complete = TRUE;
1454 self->within_one_frame = FALSE;
1459 if (res != GST_AV1_PARSER_OK) {
1460 /* Some verbose OBU can be skip */
1461 if (obu->obu_type == GST_AV1_OBU_REDUNDANT_FRAME_HEADER) {
1462 GST_WARNING_OBJECT (self, "Ignore a verbose %s OBU parsing error",
1463 _obu_name (obu->obu_type));
1464 gst_av1_parse_reset_obu_data_state (self);
1465 res = GST_AV1_PARSER_OK;
1472 static GstFlowReturn
1473 gst_av1_parse_handle_obu_to_obu (GstBaseParse * parse,
1474 GstBaseParseFrame * frame, gint * skipsize)
1476 GstAV1Parse *self = GST_AV1_PARSE (parse);
1477 GstMapInfo map_info;
1479 GstFlowReturn ret = GST_FLOW_OK;
1480 GstAV1ParserResult res;
1481 GstBuffer *buffer = gst_buffer_ref (frame->buffer);
1483 gboolean frame_complete;
1485 if (!gst_buffer_map (buffer, &map_info, GST_MAP_READ)) {
1487 GST_ERROR_OBJECT (parse, "Couldn't map incoming buffer");
1488 return GST_FLOW_ERROR;
1492 frame_complete = FALSE;
1493 res = gst_av1_parser_identify_one_obu (self->parser, map_info.data,
1494 map_info.size, &obu, &consumed);
1495 if (res == GST_AV1_PARSER_OK)
1496 res = gst_av1_parse_handle_one_obu (self, &obu, &frame_complete, NULL);
1498 g_assert (consumed <= map_info.size);
1500 if (res == GST_AV1_PARSER_BITSTREAM_ERROR ||
1501 res == GST_AV1_PARSER_MISSING_OBU_REFERENCE) {
1503 *skipsize = consumed;
1505 *skipsize = map_info.size;
1507 GST_WARNING_OBJECT (parse, "Parse obu error, discard %d.", *skipsize);
1508 gst_av1_parse_reset_obu_data_state (self);
1511 } else if (res == GST_AV1_PARSER_NO_MORE_DATA) {
1514 if (self->in_align == GST_AV1_PARSE_ALIGN_OBU) {
1515 /* The buffer is already aligned to OBU, should not happen. */
1517 *skipsize = consumed;
1519 *skipsize = map_info.size;
1521 GST_WARNING_OBJECT (parse, "Parse obu need more data, discard %d.",
1523 gst_av1_parse_reset_obu_data_state (self);
1527 } else if (res == GST_AV1_PARSER_DROP) {
1528 GST_DEBUG_OBJECT (parse, "Drop %d data", consumed);
1529 *skipsize = consumed;
1530 gst_av1_parse_reset_obu_data_state (self);
1533 } else if (res != GST_AV1_PARSER_OK) {
1534 GST_ERROR_OBJECT (parse, "Parse obu get unexpect error %d", res);
1536 ret = GST_FLOW_ERROR;
1540 g_assert (consumed);
1542 gst_av1_parse_update_src_caps (self, NULL);
1543 if (self->discont) {
1544 GST_BUFFER_FLAG_SET (buffer, GST_BUFFER_FLAG_DISCONT);
1545 self->discont = FALSE;
1548 GST_BUFFER_FLAG_SET (buffer, GST_BUFFER_FLAG_HEADER);
1549 self->header = FALSE;
1551 /* happen to be a frame boundary */
1553 GST_BUFFER_FLAG_SET (buffer, GST_BUFFER_FLAG_MARKER);
1555 GST_LOG_OBJECT (self, "Output one buffer with size %d", consumed);
1556 ret = gst_base_parse_finish_frame (parse, frame, consumed);
1560 gst_buffer_unmap (buffer, &map_info);
1561 gst_buffer_unref (buffer);
1566 gst_av1_parse_create_subframe (GstBaseParseFrame * frame,
1567 GstBaseParseFrame * subframe, GstBuffer * buffer)
1569 gst_base_parse_frame_init (subframe);
1570 subframe->flags |= frame->flags;
1571 subframe->offset = frame->offset;
1572 subframe->overhead = frame->overhead;
1573 /* Just ref the input buffer. The base parse will check that
1574 pointer, and it will be replaced by its out_buffer later. */
1575 subframe->buffer = gst_buffer_ref (buffer);
1578 static GstFlowReturn
1579 gst_av1_parse_handle_to_small_and_equal_align (GstBaseParse * parse,
1580 GstBaseParseFrame * frame, gint * skipsize)
1582 GstAV1Parse *self = GST_AV1_PARSE (parse);
1583 GstMapInfo map_info;
1585 GstFlowReturn ret = GST_FLOW_OK;
1586 GstAV1ParserResult res = GST_AV1_PARSER_INVALID_OPERATION;
1587 GstBuffer *buffer = gst_buffer_ref (frame->buffer);
1588 guint32 offset, consumed_before_push, consumed;
1589 gboolean frame_complete;
1590 GstBaseParseFrame subframe;
1592 if (!gst_buffer_map (buffer, &map_info, GST_MAP_READ)) {
1593 GST_ERROR_OBJECT (parse, "Couldn't map incoming buffer");
1594 return GST_FLOW_ERROR;
1597 consumed_before_push = 0;
1599 frame_complete = FALSE;
1601 while (offset < map_info.size) {
1602 res = gst_av1_parser_identify_one_obu (self->parser,
1603 map_info.data + offset, map_info.size - offset, &obu, &consumed);
1604 if (res == GST_AV1_PARSER_OK)
1605 res = gst_av1_parse_handle_one_obu (self, &obu, &frame_complete, NULL);
1606 if (res != GST_AV1_PARSER_OK)
1609 if (obu.obu_type == GST_AV1_OBU_TEMPORAL_DELIMITER
1610 && consumed_before_push > 0) {
1611 GST_DEBUG_OBJECT (self, "Encounter TD inside one %s aligned"
1612 " buffer, should not happen normally.",
1613 gst_av1_parse_alignment_to_string (self->in_align));
1615 if (self->in_align == GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT_ANNEX_B)
1616 gst_av1_parser_reset_annex_b (self->parser);
1618 /* Not include this TD obu, it should belong to the next TU or frame,
1619 we push all the data we already got. */
1620 gst_av1_parse_create_subframe (frame, &subframe, buffer);
1621 ret = gst_av1_parse_push_data (self, &subframe,
1622 consumed_before_push, TRUE);
1623 if (ret != GST_FLOW_OK)
1626 /* Begin to find the next. */
1627 frame_complete = FALSE;
1628 consumed_before_push = 0;
1632 gst_av1_parse_cache_one_obu (self, buffer, &obu,
1633 map_info.data + offset, consumed, frame_complete);
1636 consumed_before_push += consumed;
1638 if ((self->align == GST_AV1_PARSE_ALIGN_OBU) ||
1639 (self->align == GST_AV1_PARSE_ALIGN_FRAME && frame_complete)) {
1640 gst_av1_parse_create_subframe (frame, &subframe, buffer);
1641 ret = gst_av1_parse_push_data (self, &subframe,
1642 consumed_before_push, frame_complete);
1643 if (ret != GST_FLOW_OK)
1646 /* Begin to find the next. */
1647 frame_complete = FALSE;
1648 consumed_before_push = 0;
1653 if (res == GST_AV1_PARSER_BITSTREAM_ERROR ||
1654 res == GST_AV1_PARSER_MISSING_OBU_REFERENCE) {
1655 /* Discard the whole frame */
1656 *skipsize = map_info.size;
1657 GST_WARNING_OBJECT (parse, "Parse obu error, discard %d", *skipsize);
1658 if (self->in_align == GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT_ANNEX_B)
1659 gst_av1_parser_reset_annex_b (self->parser);
1660 gst_av1_parse_reset_obu_data_state (self);
1663 } else if (res == GST_AV1_PARSER_NO_MORE_DATA) {
1664 /* Discard the whole buffer */
1665 *skipsize = map_info.size;
1666 GST_WARNING_OBJECT (parse, "Parse obu need more data, discard %d.",
1668 if (self->in_align == GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT_ANNEX_B)
1669 gst_av1_parser_reset_annex_b (self->parser);
1671 gst_av1_parse_reset_obu_data_state (self);
1674 } else if (res == GST_AV1_PARSER_DROP) {
1675 GST_DEBUG_OBJECT (parse, "Drop %d data", consumed);
1677 gst_av1_parse_reset_obu_data_state (self);
1678 res = GST_AV1_PARSER_OK;
1680 } else if (res != GST_AV1_PARSER_OK) {
1681 GST_ERROR_OBJECT (parse, "Parse obu get unexpect error %d", res);
1683 ret = GST_FLOW_ERROR;
1687 /* If the total buffer exhausted but frame is not complete, we just
1688 push the left data and consider it as a frame. */
1689 if (consumed_before_push > 0 && !frame_complete
1690 && self->align == GST_AV1_PARSE_ALIGN_FRAME) {
1691 g_assert (offset >= map_info.size);
1692 /* Warning and still consider the frame is complete */
1693 GST_WARNING_OBJECT (self, "Exhaust the buffer but still incomplete frame,"
1694 " should not happend in %s alignment",
1695 gst_av1_parse_alignment_to_string (self->in_align));
1698 ret = gst_av1_parse_push_data (self, frame, consumed_before_push, TRUE);
1701 gst_buffer_unmap (buffer, &map_info);
1702 gst_buffer_unref (buffer);
1706 static GstFlowReturn
1707 gst_av1_parse_handle_to_big_align (GstBaseParse * parse,
1708 GstBaseParseFrame * frame, gint * skipsize)
1710 GstAV1Parse *self = GST_AV1_PARSE (parse);
1711 GstMapInfo map_info;
1713 GstFlowReturn ret = GST_FLOW_OK;
1714 GstAV1ParserResult res = GST_AV1_PARSER_OK;
1715 GstBuffer *buffer = gst_buffer_ref (frame->buffer);
1717 gboolean frame_complete;
1718 gboolean check_new_tu;
1721 g_assert (self->in_align <= GST_AV1_PARSE_ALIGN_FRAME);
1723 if (!gst_buffer_map (buffer, &map_info, GST_MAP_READ)) {
1725 GST_ERROR_OBJECT (parse, "Couldn't map incoming buffer");
1726 return GST_FLOW_ERROR;
1731 while (self->last_parsed_offset < map_info.size) {
1732 res = gst_av1_parser_identify_one_obu (self->parser,
1733 map_info.data + self->last_parsed_offset,
1734 map_info.size - self->last_parsed_offset, &obu, &consumed);
1735 if (res != GST_AV1_PARSER_OK)
1738 check_new_tu = FALSE;
1739 res = gst_av1_parse_handle_one_obu (self, &obu, &frame_complete,
1741 if (res != GST_AV1_PARSER_OK)
1744 if (check_new_tu && (gst_adapter_available (self->cache_out) ||
1745 gst_adapter_available (self->frame_cache))) {
1750 if (self->align == GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT ||
1751 self->align == GST_AV1_PARSE_ALIGN_FRAME) {
1752 GstBuffer *buf = gst_buffer_copy_region (buffer, GST_BUFFER_COPY_ALL,
1753 self->last_parsed_offset, consumed);
1754 gst_adapter_push (self->cache_out, buf);
1755 } else if (self->align == GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT_ANNEX_B) {
1756 gst_av1_parse_convert_to_annexb (self, buffer, &obu, frame_complete);
1758 g_assert_not_reached ();
1760 self->last_parsed_offset += consumed;
1762 if (self->align == GST_AV1_PARSE_ALIGN_FRAME && frame_complete)
1769 /* Finish a complete frame anyway */
1770 if (complete || GST_BASE_PARSE_DRAINING (parse)) {
1773 /* push the left anyway if no error */
1774 if (res == GST_AV1_PARSER_OK)
1775 ret = gst_av1_parse_push_data (self, frame,
1776 self->last_parsed_offset, TRUE);
1778 self->last_parsed_offset = 0;
1783 if (res == GST_AV1_PARSER_BITSTREAM_ERROR ||
1784 res == GST_AV1_PARSER_MISSING_OBU_REFERENCE) {
1785 *skipsize = map_info.size;
1786 GST_WARNING_OBJECT (parse, "Parse obu error, discard whole buffer %d.",
1788 /* The adapter will be cleared in next loop because of
1789 GST_BASE_PARSE_FRAME_FLAG_NEW_FRAME flag */
1790 gst_av1_parse_reset_obu_data_state (self);
1792 } else if (res == GST_AV1_PARSER_NO_MORE_DATA) {
1795 if (self->in_align >= GST_AV1_PARSE_ALIGN_OBU) {
1796 /* The buffer is already aligned to OBU, should not happen.
1797 The adapter will be cleared in next loop because of
1798 GST_BASE_PARSE_FRAME_FLAG_NEW_FRAME flag */
1799 *skipsize = map_info.size;
1800 gst_av1_parse_reset_obu_data_state (self);
1801 GST_WARNING_OBJECT (parse,
1802 "Parse obu need more data, discard whole buffer %d.", *skipsize);
1805 } else if (res == GST_AV1_PARSER_DROP) {
1806 GST_DEBUG_OBJECT (parse, "Drop %d data", consumed);
1807 self->last_parsed_offset += consumed;
1808 gst_av1_parse_reset_obu_data_state (self);
1809 res = GST_AV1_PARSER_OK;
1811 } else if (res == GST_AV1_PARSER_OK) {
1812 /* Everything is correct but still not get a frame or tu,
1814 GST_DEBUG_OBJECT (parse, "Need more data");
1818 GST_ERROR_OBJECT (parse, "Parse obu get unexpect error %d", res);
1820 ret = GST_FLOW_ERROR;
1824 gst_buffer_unmap (buffer, &map_info);
1825 gst_buffer_unref (buffer);
1829 /* Try to recognize whether the input is annex-b format.
1830 return TRUE if we decide, FALSE if we can not decide or
1831 encounter some error. */
1833 gst_av1_parse_detect_stream_format (GstBaseParse * parse,
1834 GstBaseParseFrame * frame)
1836 GstAV1Parse *self = GST_AV1_PARSE (parse);
1837 GstMapInfo map_info;
1839 GstAV1ParserResult res = GST_AV1_PARSER_INVALID_OPERATION;
1840 GstBuffer *buffer = gst_buffer_ref (frame->buffer);
1841 gboolean got_seq, got_frame;
1842 gboolean frame_complete;
1844 guint32 total_consumed;
1846 gboolean ret = FALSE;
1848 g_assert (self->in_align == GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT);
1849 g_assert (self->detect_annex_b == TRUE);
1851 if (!gst_buffer_map (buffer, &map_info, GST_MAP_READ)) {
1852 GST_ERROR_OBJECT (parse, "Couldn't map incoming buffer");
1856 gst_av1_parser_reset (self->parser, FALSE);
1863 while (total_consumed < map_info.size) {
1864 res = gst_av1_parser_identify_one_obu (self->parser,
1865 map_info.data + total_consumed, map_info.size - total_consumed,
1867 if (res == GST_AV1_PARSER_OK) {
1868 total_consumed += consumed;
1869 res = gst_av1_parse_handle_one_obu (self, &obu, &frame_complete, NULL);
1872 if (res != GST_AV1_PARSER_OK)
1875 if (obu.obu_type == GST_AV1_OBU_SEQUENCE_HEADER)
1878 if (obu.obu_type == GST_AV1_OBU_REDUNDANT_FRAME_HEADER ||
1879 obu.obu_type == GST_AV1_OBU_FRAME ||
1880 obu.obu_type == GST_AV1_OBU_FRAME_HEADER)
1883 if (got_seq || got_frame)
1887 gst_av1_parser_reset (self->parser, FALSE);
1889 /* If succeed recognize seq or frame, it's done.
1890 otherwise, just need to get more data. */
1891 if (got_seq || got_frame) {
1893 self->detect_annex_b = FALSE;
1897 if (res == GST_AV1_PARSER_DROP) {
1898 total_consumed += consumed;
1899 res = GST_AV1_PARSER_OK;
1900 gst_av1_parse_reset_obu_data_state (self);
1904 /* Try the annex b format. The buffer should contain the whole TU,
1905 and the buffer start with the TU size in leb128() format. */
1906 if (map_info.size < 8) {
1911 tu_sz = _read_leb128 (map_info.data, &res, &consumed);
1912 if (tu_sz == 0 || res != GST_AV1_PARSER_OK) {
1913 /* error to get the TU size, should not be annex b. */
1917 if (tu_sz + consumed != map_info.size) {
1918 GST_DEBUG_OBJECT (self, "Buffer size %" G_GSSIZE_FORMAT ", TU size %d,"
1919 " do not match.", map_info.size, tu_sz);
1923 GST_INFO_OBJECT (self, "Detect the annex-b format");
1924 self->in_align = GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT_ANNEX_B;
1925 self->detect_annex_b = FALSE;
1926 gst_av1_parser_reset (self->parser, TRUE);
1930 gst_av1_parse_reset_obu_data_state (self);
1931 gst_buffer_unmap (buffer, &map_info);
1932 gst_buffer_unref (buffer);
1936 static GstFlowReturn
1937 gst_av1_parse_handle_frame (GstBaseParse * parse,
1938 GstBaseParseFrame * frame, gint * skipsize)
1940 GstAV1Parse *self = GST_AV1_PARSE (parse);
1941 GstFlowReturn ret = GST_FLOW_OK;
1942 guint in_level, out_level;
1944 if (GST_BUFFER_FLAG_IS_SET (frame->buffer, GST_BUFFER_FLAG_DISCONT)) {
1945 self->discont = TRUE;
1947 if (frame->flags & GST_BASE_PARSE_FRAME_FLAG_NEW_FRAME)
1948 gst_av1_parse_reset_obu_data_state (self);
1950 self->discont = FALSE;
1953 GST_LOG_OBJECT (self, "Input frame size %" G_GSSIZE_FORMAT,
1954 gst_buffer_get_size (frame->buffer));
1956 /* avoid stale cached parsing state */
1957 if (frame->flags & GST_BASE_PARSE_FRAME_FLAG_NEW_FRAME) {
1958 GST_LOG_OBJECT (self, "parsing new frame");
1959 gst_adapter_clear (self->cache_out);
1960 gst_adapter_clear (self->frame_cache);
1961 self->last_parsed_offset = 0;
1962 self->header = FALSE;
1963 self->keyframe = FALSE;
1964 self->show_frame = FALSE;
1966 GST_LOG_OBJECT (self, "resuming frame parsing");
1969 /* When in pull mode, the sink pad has no caps, we may get the
1970 caps by query the upstream element */
1971 if (self->in_align == GST_AV1_PARSE_ALIGN_NONE) {
1972 GstCaps *upstream_caps;
1975 gst_pad_peer_query_caps (GST_BASE_PARSE_SINK_PAD (self), NULL);
1976 if (upstream_caps) {
1977 if (!gst_caps_is_empty (upstream_caps)
1978 && !gst_caps_is_any (upstream_caps)) {
1979 GstAV1ParseAligment align;
1981 GST_LOG_OBJECT (self, "upstream caps: %" GST_PTR_FORMAT, upstream_caps);
1983 /* fixate to avoid ambiguity with lists when parsing */
1984 upstream_caps = gst_caps_fixate (upstream_caps);
1985 align = gst_av1_parse_alignment_from_caps (upstream_caps);
1986 if (align == GST_AV1_PARSE_ALIGN_ERROR) {
1987 GST_ERROR_OBJECT (self, "upstream caps %" GST_PTR_FORMAT
1988 " set stream-format and alignment conflict.", upstream_caps);
1990 gst_caps_unref (upstream_caps);
1991 return GST_FLOW_ERROR;
1994 self->in_align = align;
1997 gst_caps_unref (upstream_caps);
1999 gst_av1_parser_reset (self->parser,
2000 self->in_align == GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT_ANNEX_B);
2003 if (self->in_align != GST_AV1_PARSE_ALIGN_NONE) {
2004 GST_LOG_OBJECT (self, "Query the upstream get the alignment %s",
2005 gst_av1_parse_alignment_to_string (self->in_align));
2007 self->in_align = GST_AV1_PARSE_ALIGN_BYTE;
2008 GST_DEBUG_OBJECT (self, "alignment set to default %s",
2009 gst_av1_parse_alignment_to_string (GST_AV1_PARSE_ALIGN_BYTE));
2013 if (self->in_align == GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT
2014 && self->detect_annex_b) {
2015 /* Only happend at the first time of handle_frame, try to
2016 recognize the annex b stream format. */
2017 if (gst_av1_parse_detect_stream_format (parse, frame)) {
2018 GST_INFO_OBJECT (self, "Input alignment %s",
2019 gst_av1_parse_alignment_to_string (self->in_align));
2021 /* Because the input is already TU aligned, we should skip
2022 the whole problematic TU and check the next one. */
2023 *skipsize = gst_buffer_get_size (frame->buffer);
2024 GST_WARNING_OBJECT (self, "Fail to detect the stream format for TU,"
2025 " skip the whole TU %d", *skipsize);
2030 /* We may in pull mode and no caps is set */
2031 if (self->align == GST_AV1_PARSE_ALIGN_NONE)
2032 gst_av1_parse_negotiate (self, NULL);
2034 in_level = self->in_align;
2035 if (self->in_align == GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT_ANNEX_B)
2036 in_level = GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT;
2037 out_level = self->align;
2038 if (self->align == GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT_ANNEX_B)
2039 out_level = GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT;
2041 if (self->in_align <= GST_AV1_PARSE_ALIGN_OBU
2042 && self->align == GST_AV1_PARSE_ALIGN_OBU) {
2043 ret = gst_av1_parse_handle_obu_to_obu (parse, frame, skipsize);
2044 } else if (in_level < out_level) {
2045 ret = gst_av1_parse_handle_to_big_align (parse, frame, skipsize);
2047 ret = gst_av1_parse_handle_to_small_and_equal_align (parse,