2 * Copyright (C) <1999> Erik Walthinsen <omega@cse.ogi.edu>
4 * This library is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Library General Public
6 * License as published by the Free Software Foundation; either
7 * version 2 of the License, or (at your option) any later version.
9 * This library is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Library General Public License for more details.
14 * You should have received a copy of the GNU Library General Public
15 * License along with this library; if not, write to the
16 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
17 * Boston, MA 02110-1301, USA.
27 #include <libavcodec/avcodec.h>
28 #include <libavutil/stereo3d.h>
29 #include <libavutil/mastering_display_metadata.h>
32 #include "gstavcodecmap.h"
33 #include "gstavutils.h"
34 #include "gstavviddec.h"
36 GST_DEBUG_CATEGORY_STATIC (GST_CAT_PERFORMANCE);
38 #define MAX_TS_MASK 0xff
40 #define DEFAULT_LOWRES 0
41 #define DEFAULT_SKIPFRAME 0
42 #define DEFAULT_DIRECT_RENDERING TRUE
43 #define DEFAULT_DEBUG_MV FALSE
44 #define DEFAULT_MAX_THREADS 0
45 #define DEFAULT_OUTPUT_CORRUPT TRUE
46 #define REQUIRED_POOL_MAX_BUFFERS 32
47 #define DEFAULT_STRIDE_ALIGN 31
48 #define DEFAULT_ALLOC_PARAM { 0, DEFAULT_STRIDE_ALIGN, 0, 0, }
49 #define DEFAULT_THREAD_TYPE 0
56 PROP_DIRECT_RENDERING,
64 /* A number of function prototypes are given so we can refer to them later. */
65 static void gst_ffmpegviddec_base_init (GstFFMpegVidDecClass * klass);
66 static void gst_ffmpegviddec_class_init (GstFFMpegVidDecClass * klass);
67 static void gst_ffmpegviddec_init (GstFFMpegVidDec * ffmpegdec);
68 static void gst_ffmpegviddec_finalize (GObject * object);
70 static gboolean gst_ffmpegviddec_set_format (GstVideoDecoder * decoder,
71 GstVideoCodecState * state);
72 static GstFlowReturn gst_ffmpegviddec_handle_frame (GstVideoDecoder * decoder,
73 GstVideoCodecFrame * frame);
74 static gboolean gst_ffmpegviddec_start (GstVideoDecoder * decoder);
75 static gboolean gst_ffmpegviddec_stop (GstVideoDecoder * decoder);
76 static gboolean gst_ffmpegviddec_flush (GstVideoDecoder * decoder);
77 static gboolean gst_ffmpegviddec_decide_allocation (GstVideoDecoder * decoder,
79 static gboolean gst_ffmpegviddec_propose_allocation (GstVideoDecoder * decoder,
82 static void gst_ffmpegviddec_set_property (GObject * object,
83 guint prop_id, const GValue * value, GParamSpec * pspec);
84 static void gst_ffmpegviddec_get_property (GObject * object,
85 guint prop_id, GValue * value, GParamSpec * pspec);
87 static gboolean gst_ffmpegviddec_negotiate (GstFFMpegVidDec * ffmpegdec,
88 AVCodecContext * context, AVFrame * picture);
90 /* some sort of bufferpool handling, but different */
91 static int gst_ffmpegviddec_get_buffer2 (AVCodecContext * context,
92 AVFrame * picture, int flags);
94 static GstFlowReturn gst_ffmpegviddec_finish (GstVideoDecoder * decoder);
95 static GstFlowReturn gst_ffmpegviddec_drain (GstVideoDecoder * decoder);
97 static gboolean picture_changed (GstFFMpegVidDec * ffmpegdec,
99 static gboolean context_changed (GstFFMpegVidDec * ffmpegdec,
100 AVCodecContext * context);
102 #define GST_FFDEC_PARAMS_QDATA g_quark_from_static_string("avdec-params")
104 static GstElementClass *parent_class = NULL;
106 #define GST_FFMPEGVIDDEC_TYPE_LOWRES (gst_ffmpegviddec_lowres_get_type())
108 gst_ffmpegviddec_lowres_get_type (void)
110 static GType ffmpegdec_lowres_type = 0;
112 if (!ffmpegdec_lowres_type) {
113 static const GEnumValue ffmpegdec_lowres[] = {
115 {1, "1", "1/2-size"},
116 {2, "2", "1/4-size"},
120 ffmpegdec_lowres_type =
121 g_enum_register_static ("GstLibAVVidDecLowres", ffmpegdec_lowres);
124 return ffmpegdec_lowres_type;
127 #define GST_FFMPEGVIDDEC_TYPE_SKIPFRAME (gst_ffmpegviddec_skipframe_get_type())
129 gst_ffmpegviddec_skipframe_get_type (void)
131 static GType ffmpegdec_skipframe_type = 0;
133 if (!ffmpegdec_skipframe_type) {
134 static const GEnumValue ffmpegdec_skipframe[] = {
135 {0, "0", "Skip nothing"},
136 {1, "1", "Skip B-frames"},
137 {2, "2", "Skip IDCT/Dequantization"},
138 {5, "5", "Skip everything"},
142 ffmpegdec_skipframe_type =
143 g_enum_register_static ("GstLibAVVidDecSkipFrame", ffmpegdec_skipframe);
146 return ffmpegdec_skipframe_type;
149 static const GFlagsValue ffmpegdec_thread_types[] = {
150 {0x0, "Auto", "auto"},
151 {0x1, "Frame", "frame"},
152 {0x2, "Slice", "slice"},
156 #define GST_FFMPEGVIDDEC_TYPE_THREAD_TYPE (gst_ffmpegviddec_thread_type_get_type())
158 gst_ffmpegviddec_thread_type_get_type (void)
160 static GType ffmpegdec_thread_type_type = 0;
162 if (!ffmpegdec_thread_type_type) {
163 ffmpegdec_thread_type_type =
164 g_flags_register_static ("GstLibAVVidDecThreadType",
165 ffmpegdec_thread_types);
167 return ffmpegdec_thread_type_type;
171 gst_ffmpegviddec_base_init (GstFFMpegVidDecClass * klass)
173 GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
174 GstPadTemplate *sinktempl, *srctempl;
175 GstCaps *sinkcaps, *srccaps;
177 gchar *longname, *description;
180 (AVCodec *) g_type_get_qdata (G_OBJECT_CLASS_TYPE (klass),
181 GST_FFDEC_PARAMS_QDATA);
182 g_assert (in_plugin != NULL);
184 /* construct the element details struct */
185 longname = g_strdup_printf ("libav %s decoder", in_plugin->long_name);
186 description = g_strdup_printf ("libav %s decoder", in_plugin->name);
187 gst_element_class_set_metadata (element_class, longname,
188 "Codec/Decoder/Video", description,
189 "Wim Taymans <wim.taymans@gmail.com>, "
190 "Ronald Bultje <rbultje@ronald.bitfreak.net>, "
191 "Edward Hervey <bilboed@bilboed.com>");
193 g_free (description);
196 sinkcaps = gst_ffmpeg_codecid_to_caps (in_plugin->id, NULL, FALSE);
198 GST_DEBUG ("Couldn't get sink caps for decoder '%s'", in_plugin->name);
199 sinkcaps = gst_caps_new_empty_simple ("unknown/unknown");
201 srccaps = gst_ffmpeg_codectype_to_video_caps (NULL,
202 in_plugin->id, FALSE, in_plugin);
204 GST_DEBUG ("Couldn't get source caps for decoder '%s'", in_plugin->name);
205 srccaps = gst_caps_from_string ("video/x-raw");
209 sinktempl = gst_pad_template_new ("sink", GST_PAD_SINK,
210 GST_PAD_ALWAYS, sinkcaps);
211 srctempl = gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS, srccaps);
213 gst_element_class_add_pad_template (element_class, srctempl);
214 gst_element_class_add_pad_template (element_class, sinktempl);
216 gst_caps_unref (sinkcaps);
217 gst_caps_unref (srccaps);
219 klass->in_plugin = in_plugin;
223 gst_ffmpegviddec_class_init (GstFFMpegVidDecClass * klass)
225 GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
226 GstVideoDecoderClass *viddec_class = GST_VIDEO_DECODER_CLASS (klass);
229 parent_class = g_type_class_peek_parent (klass);
231 gobject_class->finalize = gst_ffmpegviddec_finalize;
233 gobject_class->set_property = gst_ffmpegviddec_set_property;
234 gobject_class->get_property = gst_ffmpegviddec_get_property;
236 g_object_class_install_property (gobject_class, PROP_SKIPFRAME,
237 g_param_spec_enum ("skip-frame", "Skip frames",
238 "Which types of frames to skip during decoding",
239 GST_FFMPEGVIDDEC_TYPE_SKIPFRAME, 0,
240 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
241 g_object_class_install_property (gobject_class, PROP_LOWRES,
242 g_param_spec_enum ("lowres", "Low resolution",
243 "At which resolution to decode images",
244 GST_FFMPEGVIDDEC_TYPE_LOWRES, 0,
245 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
246 g_object_class_install_property (gobject_class, PROP_DIRECT_RENDERING,
247 g_param_spec_boolean ("direct-rendering", "Direct Rendering",
248 "Enable direct rendering", DEFAULT_DIRECT_RENDERING,
249 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
250 g_object_class_install_property (gobject_class, PROP_DEBUG_MV,
251 g_param_spec_boolean ("debug-mv", "Debug motion vectors",
252 "Whether libav should print motion vectors on top of the image",
253 DEFAULT_DEBUG_MV, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
254 g_object_class_install_property (gobject_class, PROP_OUTPUT_CORRUPT,
255 g_param_spec_boolean ("output-corrupt", "Output corrupt buffers",
256 "Whether libav should output frames even if corrupted",
257 DEFAULT_OUTPUT_CORRUPT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
259 caps = klass->in_plugin->capabilities;
260 if (caps & (AV_CODEC_CAP_FRAME_THREADS | AV_CODEC_CAP_SLICE_THREADS)) {
261 g_object_class_install_property (G_OBJECT_CLASS (klass), PROP_MAX_THREADS,
262 g_param_spec_int ("max-threads", "Maximum decode threads",
263 "Maximum number of worker threads to spawn. (0 = auto)",
264 0, G_MAXINT, DEFAULT_MAX_THREADS,
265 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
266 g_object_class_install_property (G_OBJECT_CLASS (klass), PROP_THREAD_TYPE,
267 g_param_spec_flags ("thread-type", "Thread type",
268 "Multithreading methods to use",
269 GST_FFMPEGVIDDEC_TYPE_THREAD_TYPE,
270 DEFAULT_THREAD_TYPE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
273 viddec_class->set_format = gst_ffmpegviddec_set_format;
274 viddec_class->handle_frame = gst_ffmpegviddec_handle_frame;
275 viddec_class->start = gst_ffmpegviddec_start;
276 viddec_class->stop = gst_ffmpegviddec_stop;
277 viddec_class->flush = gst_ffmpegviddec_flush;
278 viddec_class->finish = gst_ffmpegviddec_finish;
279 viddec_class->drain = gst_ffmpegviddec_drain;
280 viddec_class->decide_allocation = gst_ffmpegviddec_decide_allocation;
281 viddec_class->propose_allocation = gst_ffmpegviddec_propose_allocation;
283 GST_DEBUG_CATEGORY_GET (GST_CAT_PERFORMANCE, "GST_PERFORMANCE");
287 gst_ffmpegviddec_init (GstFFMpegVidDec * ffmpegdec)
289 GstFFMpegVidDecClass *klass =
290 (GstFFMpegVidDecClass *) G_OBJECT_GET_CLASS (ffmpegdec);
292 /* some ffmpeg data */
293 ffmpegdec->context = avcodec_alloc_context3 (klass->in_plugin);
294 ffmpegdec->context->opaque = ffmpegdec;
295 ffmpegdec->picture = av_frame_alloc ();
296 ffmpegdec->opened = FALSE;
297 ffmpegdec->skip_frame = ffmpegdec->lowres = 0;
298 ffmpegdec->direct_rendering = DEFAULT_DIRECT_RENDERING;
299 ffmpegdec->debug_mv = DEFAULT_DEBUG_MV;
300 ffmpegdec->max_threads = DEFAULT_MAX_THREADS;
301 ffmpegdec->output_corrupt = DEFAULT_OUTPUT_CORRUPT;
302 ffmpegdec->thread_type = DEFAULT_THREAD_TYPE;
304 GST_PAD_SET_ACCEPT_TEMPLATE (GST_VIDEO_DECODER_SINK_PAD (ffmpegdec));
305 gst_video_decoder_set_use_default_pad_acceptcaps (GST_VIDEO_DECODER_CAST
308 gst_video_decoder_set_needs_format (GST_VIDEO_DECODER (ffmpegdec), TRUE);
312 gst_ffmpegviddec_finalize (GObject * object)
314 GstFFMpegVidDec *ffmpegdec = (GstFFMpegVidDec *) object;
316 av_frame_free (&ffmpegdec->picture);
318 if (ffmpegdec->context != NULL) {
319 gst_ffmpeg_avcodec_close (ffmpegdec->context);
320 av_free (ffmpegdec->context);
321 ffmpegdec->context = NULL;
324 G_OBJECT_CLASS (parent_class)->finalize (object);
328 gst_ffmpegviddec_context_set_flags (AVCodecContext * context, guint flags,
331 g_return_if_fail (context != NULL);
334 context->flags |= flags;
336 context->flags &= ~flags;
341 gst_ffmpegviddec_close (GstFFMpegVidDec * ffmpegdec, gboolean reset)
343 GstFFMpegVidDecClass *oclass;
346 oclass = (GstFFMpegVidDecClass *) (G_OBJECT_GET_CLASS (ffmpegdec));
348 GST_LOG_OBJECT (ffmpegdec, "closing ffmpeg codec");
350 gst_caps_replace (&ffmpegdec->last_caps, NULL);
352 gst_ffmpeg_avcodec_close (ffmpegdec->context);
353 ffmpegdec->opened = FALSE;
355 for (i = 0; i < G_N_ELEMENTS (ffmpegdec->stride); i++)
356 ffmpegdec->stride[i] = -1;
358 gst_buffer_replace (&ffmpegdec->palette, NULL);
360 if (ffmpegdec->context->extradata) {
361 av_free (ffmpegdec->context->extradata);
362 ffmpegdec->context->extradata = NULL;
365 if (avcodec_get_context_defaults3 (ffmpegdec->context,
366 oclass->in_plugin) < 0) {
367 GST_DEBUG_OBJECT (ffmpegdec, "Failed to set context defaults");
370 ffmpegdec->context->opaque = ffmpegdec;
377 gst_ffmpegviddec_open (GstFFMpegVidDec * ffmpegdec)
379 GstFFMpegVidDecClass *oclass;
382 oclass = (GstFFMpegVidDecClass *) (G_OBJECT_GET_CLASS (ffmpegdec));
384 if (gst_ffmpeg_avcodec_open (ffmpegdec->context, oclass->in_plugin) < 0)
387 for (i = 0; i < G_N_ELEMENTS (ffmpegdec->stride); i++)
388 ffmpegdec->stride[i] = -1;
390 ffmpegdec->opened = TRUE;
392 GST_LOG_OBJECT (ffmpegdec, "Opened libav codec %s, id %d",
393 oclass->in_plugin->name, oclass->in_plugin->id);
395 gst_ffmpegviddec_context_set_flags (ffmpegdec->context,
396 AV_CODEC_FLAG_OUTPUT_CORRUPT, ffmpegdec->output_corrupt);
403 gst_ffmpegviddec_close (ffmpegdec, TRUE);
404 GST_DEBUG_OBJECT (ffmpegdec, "avdec_%s: Failed to open libav codec",
405 oclass->in_plugin->name);
411 gst_ffmpegviddec_get_palette (GstFFMpegVidDec * ffmpegdec,
412 GstVideoCodecState * state)
414 GstStructure *str = gst_caps_get_structure (state->caps, 0);
415 const GValue *palette_v;
418 /* do we have a palette? */
419 if ((palette_v = gst_structure_get_value (str, "palette_data"))) {
420 palette = gst_value_get_buffer (palette_v);
421 GST_DEBUG ("got palette data %p", palette);
422 if (gst_buffer_get_size (palette) >= AVPALETTE_SIZE) {
423 gst_buffer_replace (&ffmpegdec->palette, palette);
430 gst_ffmpegviddec_set_format (GstVideoDecoder * decoder,
431 GstVideoCodecState * state)
433 GstFFMpegVidDec *ffmpegdec;
434 GstFFMpegVidDecClass *oclass;
435 GstClockTime latency = GST_CLOCK_TIME_NONE;
436 gboolean ret = FALSE;
438 ffmpegdec = (GstFFMpegVidDec *) decoder;
439 oclass = (GstFFMpegVidDecClass *) (G_OBJECT_GET_CLASS (ffmpegdec));
441 if (ffmpegdec->last_caps != NULL &&
442 gst_caps_is_equal (ffmpegdec->last_caps, state->caps)) {
446 GST_DEBUG_OBJECT (ffmpegdec, "setcaps called");
448 GST_OBJECT_LOCK (ffmpegdec);
449 /* stupid check for VC1 */
450 if ((oclass->in_plugin->id == AV_CODEC_ID_WMV3) ||
451 (oclass->in_plugin->id == AV_CODEC_ID_VC1))
452 oclass->in_plugin->id = gst_ffmpeg_caps_to_codecid (state->caps, NULL);
454 /* close old session */
455 if (ffmpegdec->opened) {
456 GST_OBJECT_UNLOCK (ffmpegdec);
457 gst_ffmpegviddec_finish (decoder);
458 GST_OBJECT_LOCK (ffmpegdec);
459 if (!gst_ffmpegviddec_close (ffmpegdec, TRUE)) {
460 GST_OBJECT_UNLOCK (ffmpegdec);
463 ffmpegdec->pic_pix_fmt = 0;
464 ffmpegdec->pic_width = 0;
465 ffmpegdec->pic_height = 0;
466 ffmpegdec->pic_par_n = 0;
467 ffmpegdec->pic_par_d = 0;
468 ffmpegdec->pic_interlaced = 0;
469 ffmpegdec->pic_field_order = 0;
470 ffmpegdec->pic_field_order_changed = FALSE;
471 ffmpegdec->ctx_ticks = 0;
472 ffmpegdec->ctx_time_n = 0;
473 ffmpegdec->ctx_time_d = 0;
474 ffmpegdec->cur_multiview_mode = GST_VIDEO_MULTIVIEW_MODE_NONE;
475 ffmpegdec->cur_multiview_flags = GST_VIDEO_MULTIVIEW_FLAGS_NONE;
478 gst_caps_replace (&ffmpegdec->last_caps, state->caps);
480 /* set buffer functions */
481 ffmpegdec->context->get_buffer2 = gst_ffmpegviddec_get_buffer2;
482 ffmpegdec->context->draw_horiz_band = NULL;
484 /* reset coded_width/_height to prevent it being reused from last time when
485 * the codec is opened again, causing a mismatch and possible
486 * segfault/corruption. (Common scenario when renegotiating caps) */
487 ffmpegdec->context->coded_width = 0;
488 ffmpegdec->context->coded_height = 0;
490 GST_LOG_OBJECT (ffmpegdec, "size %dx%d", ffmpegdec->context->width,
491 ffmpegdec->context->height);
493 /* FIXME : Create a method that takes GstVideoCodecState instead */
494 /* get size and so */
495 gst_ffmpeg_caps_with_codecid (oclass->in_plugin->id,
496 oclass->in_plugin->type, state->caps, ffmpegdec->context);
498 GST_LOG_OBJECT (ffmpegdec, "size after %dx%d", ffmpegdec->context->width,
499 ffmpegdec->context->height);
501 gst_ffmpegviddec_get_palette (ffmpegdec, state);
503 if (!ffmpegdec->context->time_base.den || !ffmpegdec->context->time_base.num) {
504 GST_DEBUG_OBJECT (ffmpegdec, "forcing 25/1 framerate");
505 ffmpegdec->context->time_base.num = 1;
506 ffmpegdec->context->time_base.den = 25;
509 /* workaround encoder bugs */
510 ffmpegdec->context->workaround_bugs |= FF_BUG_AUTODETECT;
511 ffmpegdec->context->err_recognition = 1;
514 ffmpegdec->context->lowres = ffmpegdec->lowres;
515 ffmpegdec->context->skip_frame = ffmpegdec->skip_frame;
517 /* ffmpeg can draw motion vectors on top of the image (not every decoder
519 ffmpegdec->context->debug_mv = ffmpegdec->debug_mv;
521 if (ffmpegdec->thread_type) {
522 GST_DEBUG_OBJECT (ffmpegdec, "Use requested thread type 0x%x",
523 ffmpegdec->thread_type);
524 ffmpegdec->context->thread_type = ffmpegdec->thread_type;
529 query = gst_query_new_latency ();
531 /* Check if upstream is live. If it isn't we can enable frame based
532 * threading, which is adding latency */
533 if (gst_pad_peer_query (GST_VIDEO_DECODER_SINK_PAD (ffmpegdec), query)) {
534 gst_query_parse_latency (query, &is_live, NULL, NULL);
536 gst_query_unref (query);
539 ffmpegdec->context->thread_type = FF_THREAD_SLICE;
541 ffmpegdec->context->thread_type = FF_THREAD_SLICE | FF_THREAD_FRAME;
544 if (ffmpegdec->max_threads == 0) {
545 /* When thread type is FF_THREAD_FRAME, extra latency is introduced equal
546 * to one frame per thread. We thus need to calculate the thread count ourselves */
547 if ((!(oclass->in_plugin->capabilities & AV_CODEC_CAP_AUTO_THREADS)) ||
548 (ffmpegdec->context->thread_type & FF_THREAD_FRAME))
549 ffmpegdec->context->thread_count = gst_ffmpeg_auto_max_threads ();
551 ffmpegdec->context->thread_count = 0;
553 ffmpegdec->context->thread_count = ffmpegdec->max_threads;
555 /* open codec - we don't select an output pix_fmt yet,
556 * simply because we don't know! We only get it
557 * during playback... */
558 if (!gst_ffmpegviddec_open (ffmpegdec))
561 if (ffmpegdec->input_state)
562 gst_video_codec_state_unref (ffmpegdec->input_state);
563 ffmpegdec->input_state = gst_video_codec_state_ref (state);
565 if (ffmpegdec->input_state->info.fps_n) {
566 GstVideoInfo *info = &ffmpegdec->input_state->info;
567 latency = gst_util_uint64_scale_ceil (
568 (ffmpegdec->context->has_b_frames) * GST_SECOND, info->fps_d,
571 if (ffmpegdec->context->thread_type & FF_THREAD_FRAME) {
573 gst_util_uint64_scale_ceil (ffmpegdec->context->thread_count *
574 GST_SECOND, info->fps_d, info->fps_n);
581 GST_OBJECT_UNLOCK (ffmpegdec);
583 if (GST_CLOCK_TIME_IS_VALID (latency))
584 gst_video_decoder_set_latency (decoder, latency, latency);
591 GST_DEBUG_OBJECT (ffmpegdec, "Failed to open");
598 GstFFMpegVidDec *ffmpegdec;
599 GstVideoCodecFrame *frame;
601 GstVideoFrame vframe;
603 AVBufferRef *avbuffer;
604 } GstFFMpegVidDecVideoFrame;
606 static GstFFMpegVidDecVideoFrame *
607 gst_ffmpegviddec_video_frame_new (GstFFMpegVidDec * ffmpegdec,
608 GstVideoCodecFrame * frame)
610 GstFFMpegVidDecVideoFrame *dframe;
612 dframe = g_slice_new0 (GstFFMpegVidDecVideoFrame);
613 dframe->ffmpegdec = ffmpegdec;
614 dframe->frame = frame;
616 GST_DEBUG_OBJECT (ffmpegdec, "new video frame %p", dframe);
622 gst_ffmpegviddec_video_frame_free (GstFFMpegVidDec * ffmpegdec,
623 GstFFMpegVidDecVideoFrame * frame)
625 GST_DEBUG_OBJECT (ffmpegdec, "free video frame %p", frame);
628 gst_video_frame_unmap (&frame->vframe);
629 gst_video_decoder_release_frame (GST_VIDEO_DECODER (ffmpegdec), frame->frame);
630 gst_buffer_replace (&frame->buffer, NULL);
631 if (frame->avbuffer) {
632 av_buffer_unref (&frame->avbuffer);
634 g_slice_free (GstFFMpegVidDecVideoFrame, frame);
638 dummy_free_buffer (void *opaque, uint8_t * data)
640 GstFFMpegVidDecVideoFrame *frame = opaque;
642 gst_ffmpegviddec_video_frame_free (frame->ffmpegdec, frame);
645 /* This function prepares the pool configuration for direct rendering. To use
646 * this method, the codec should support direct rendering and the pool should
647 * support video meta and video alignment */
649 gst_ffmpegvideodec_prepare_dr_pool (GstFFMpegVidDec * ffmpegdec,
650 GstBufferPool * pool, GstVideoInfo * info, GstStructure * config)
652 GstAllocationParams params;
653 GstVideoAlignment align;
654 GstAllocator *allocator = NULL;
656 gint linesize_align[4];
660 width = GST_VIDEO_INFO_WIDTH (info);
661 height = GST_VIDEO_INFO_HEIGHT (info);
663 /* let ffmpeg find the alignment and padding */
664 avcodec_align_dimensions2 (ffmpegdec->context, &width, &height,
667 align.padding_top = 0;
668 align.padding_left = 0;
669 align.padding_right = width - GST_VIDEO_INFO_WIDTH (info);
670 align.padding_bottom = height - GST_VIDEO_INFO_HEIGHT (info);
672 /* add extra padding to match libav buffer allocation sizes */
673 align.padding_bottom++;
675 gst_buffer_pool_config_get_allocator (config, &allocator, ¶ms);
677 max_align = DEFAULT_STRIDE_ALIGN;
678 max_align |= params.align;
680 for (i = 0; i < GST_VIDEO_MAX_PLANES; i++) {
681 if (linesize_align[i] > 0)
682 max_align |= linesize_align[i] - 1;
685 for (i = 0; i < GST_VIDEO_MAX_PLANES; i++)
686 align.stride_align[i] = max_align;
688 params.align = max_align;
690 gst_buffer_pool_config_set_allocator (config, allocator, ¶ms);
692 GST_DEBUG_OBJECT (ffmpegdec, "aligned dimension %dx%d -> %dx%d "
693 "padding t:%u l:%u r:%u b:%u, stride_align %d:%d:%d:%d",
694 GST_VIDEO_INFO_WIDTH (info),
695 GST_VIDEO_INFO_HEIGHT (info), width, height, align.padding_top,
696 align.padding_left, align.padding_right, align.padding_bottom,
697 align.stride_align[0], align.stride_align[1], align.stride_align[2],
698 align.stride_align[3]);
700 gst_buffer_pool_config_add_option (config,
701 GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT);
702 gst_buffer_pool_config_set_video_alignment (config, &align);
706 gst_ffmpegviddec_ensure_internal_pool (GstFFMpegVidDec * ffmpegdec,
709 GstAllocationParams params = DEFAULT_ALLOC_PARAM;
711 GstVideoFormat format;
713 GstStructure *config;
716 if (ffmpegdec->internal_pool != NULL &&
717 ffmpegdec->pool_width == picture->width &&
718 ffmpegdec->pool_height == picture->height &&
719 ffmpegdec->pool_format == picture->format)
722 GST_DEBUG_OBJECT (ffmpegdec, "Updating internal pool (%i, %i)",
723 picture->width, picture->height);
725 format = gst_ffmpeg_pixfmt_to_videoformat (picture->format);
726 gst_video_info_set_format (&info, format, picture->width, picture->height);
728 /* If we have not yet been negotiated, a NONE format here would
729 * result in invalid initial dimension alignments, and potential
730 * out of bounds writes.
732 ffmpegdec->context->pix_fmt = picture->format;
734 for (i = 0; i < G_N_ELEMENTS (ffmpegdec->stride); i++)
735 ffmpegdec->stride[i] = -1;
737 if (ffmpegdec->internal_pool)
738 gst_object_unref (ffmpegdec->internal_pool);
740 ffmpegdec->internal_pool = gst_video_buffer_pool_new ();
741 config = gst_buffer_pool_get_config (ffmpegdec->internal_pool);
743 caps = gst_video_info_to_caps (&info);
744 gst_buffer_pool_config_set_params (config, caps, info.size, 2, 0);
745 gst_buffer_pool_config_set_allocator (config, NULL, ¶ms);
746 gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META);
748 gst_ffmpegvideodec_prepare_dr_pool (ffmpegdec,
749 ffmpegdec->internal_pool, &info, config);
750 /* generic video pool never fails */
751 gst_buffer_pool_set_config (ffmpegdec->internal_pool, config);
752 gst_caps_unref (caps);
754 gst_buffer_pool_set_active (ffmpegdec->internal_pool, TRUE);
756 /* Remember pool size so we can detect changes */
757 ffmpegdec->pool_width = picture->width;
758 ffmpegdec->pool_height = picture->height;
759 ffmpegdec->pool_format = picture->format;
760 ffmpegdec->pool_info = info;
764 gst_ffmpegviddec_can_direct_render (GstFFMpegVidDec * ffmpegdec)
766 GstFFMpegVidDecClass *oclass;
768 if (!ffmpegdec->direct_rendering)
771 oclass = (GstFFMpegVidDecClass *) (G_OBJECT_GET_CLASS (ffmpegdec));
772 return ((oclass->in_plugin->capabilities & AV_CODEC_CAP_DR1) ==
776 /* called when ffmpeg wants us to allocate a buffer to write the decoded frame
777 * into. We try to give it memory from our pool */
779 gst_ffmpegviddec_get_buffer2 (AVCodecContext * context, AVFrame * picture,
782 GstVideoCodecFrame *frame;
783 GstFFMpegVidDecVideoFrame *dframe;
784 GstFFMpegVidDec *ffmpegdec;
788 ffmpegdec = (GstFFMpegVidDec *) context->opaque;
790 GST_DEBUG_OBJECT (ffmpegdec, "getting buffer picture %p", picture);
792 /* apply the last info we have seen to this picture, when we get the
793 * picture back from ffmpeg we can use this to correctly timestamp the output
795 GST_DEBUG_OBJECT (ffmpegdec, "opaque value SN %d",
796 (gint32) picture->reordered_opaque);
799 gst_video_decoder_get_frame (GST_VIDEO_DECODER (ffmpegdec),
800 picture->reordered_opaque);
801 if (G_UNLIKELY (frame == NULL))
804 /* now it has a buffer allocated, so it is real and will also
806 GST_VIDEO_CODEC_FRAME_FLAG_UNSET (frame,
807 GST_VIDEO_CODEC_FRAME_FLAG_DECODE_ONLY);
809 if (G_UNLIKELY (frame->output_buffer != NULL))
810 goto duplicate_frame;
812 /* GstFFMpegVidDecVideoFrame receives the frame ref */
813 if (picture->opaque) {
814 dframe = picture->opaque;
815 dframe->frame = frame;
817 picture->opaque = dframe =
818 gst_ffmpegviddec_video_frame_new (ffmpegdec, frame);
821 GST_DEBUG_OBJECT (ffmpegdec, "storing opaque %p", dframe);
823 if (!gst_ffmpegviddec_can_direct_render (ffmpegdec))
826 gst_ffmpegviddec_ensure_internal_pool (ffmpegdec, picture);
828 ret = gst_buffer_pool_acquire_buffer (ffmpegdec->internal_pool,
829 &frame->output_buffer, NULL);
830 if (ret != GST_FLOW_OK)
833 /* piggy-backed alloc'ed on the frame,
834 * and there was much rejoicing and we are grateful.
835 * Now take away buffer from frame, we will give it back later when decoded.
836 * This allows multiple request for a buffer per frame; unusual but possible. */
837 gst_buffer_replace (&dframe->buffer, frame->output_buffer);
838 gst_buffer_replace (&frame->output_buffer, NULL);
841 if (!gst_video_frame_map (&dframe->vframe, &ffmpegdec->pool_info,
842 dframe->buffer, GST_MAP_READWRITE))
844 dframe->mapped = TRUE;
846 for (c = 0; c < AV_NUM_DATA_POINTERS; c++) {
847 if (c < GST_VIDEO_INFO_N_PLANES (&ffmpegdec->pool_info)) {
848 picture->data[c] = GST_VIDEO_FRAME_PLANE_DATA (&dframe->vframe, c);
849 picture->linesize[c] = GST_VIDEO_FRAME_PLANE_STRIDE (&dframe->vframe, c);
851 if (ffmpegdec->stride[c] == -1)
852 ffmpegdec->stride[c] = picture->linesize[c];
854 /* libav does not allow stride changes, decide allocation should check
855 * before replacing the internal pool with a downstream pool.
856 * https://bugzilla.gnome.org/show_bug.cgi?id=704769
857 * https://bugzilla.libav.org/show_bug.cgi?id=556
859 g_assert (picture->linesize[c] == ffmpegdec->stride[c]);
861 picture->data[c] = NULL;
862 picture->linesize[c] = 0;
864 GST_LOG_OBJECT (ffmpegdec, "linesize %d, data %p", picture->linesize[c],
868 picture->buf[0] = av_buffer_create (NULL, 0, dummy_free_buffer, dframe, 0);
870 GST_LOG_OBJECT (ffmpegdec, "returned frame %p", dframe->buffer);
877 int ret = avcodec_default_get_buffer2 (context, picture, flags);
879 GST_LOG_OBJECT (ffmpegdec, "direct rendering disabled, fallback alloc");
881 for (c = 0; c < AV_NUM_DATA_POINTERS; c++) {
882 ffmpegdec->stride[c] = picture->linesize[c];
884 /* Wrap our buffer around the default one to be able to have a callback
885 * when our data can be freed. Just putting our data into the first free
886 * buffer might not work if there are too many allocated already
888 if (picture->buf[0]) {
889 dframe->avbuffer = picture->buf[0];
891 av_buffer_create (picture->buf[0]->data, picture->buf[0]->size,
892 dummy_free_buffer, dframe, 0);
895 av_buffer_create (NULL, 0, dummy_free_buffer, dframe, 0);
902 GST_ELEMENT_ERROR (ffmpegdec, RESOURCE, FAILED,
903 ("Unable to allocate memory"),
904 ("The downstream pool failed to allocated buffer."));
909 GST_ELEMENT_ERROR (ffmpegdec, RESOURCE, OPEN_READ_WRITE,
910 ("Cannot access memory for read and write operation."),
911 ("The video memory allocated from downstream pool could not mapped for"
917 GST_WARNING_OBJECT (ffmpegdec, "already alloc'ed output buffer for frame");
918 gst_video_codec_frame_unref (frame);
923 GST_WARNING_OBJECT (ffmpegdec, "Couldn't get codec frame !");
929 picture_changed (GstFFMpegVidDec * ffmpegdec, AVFrame * picture)
931 gint pic_field_order = 0;
933 if (picture->interlaced_frame) {
934 if (picture->repeat_pict)
935 pic_field_order |= GST_VIDEO_BUFFER_FLAG_RFF;
936 if (picture->top_field_first)
937 pic_field_order |= GST_VIDEO_BUFFER_FLAG_TFF;
940 return !(ffmpegdec->pic_width == picture->width
941 && ffmpegdec->pic_height == picture->height
942 && ffmpegdec->pic_pix_fmt == picture->format
943 && ffmpegdec->pic_par_n == picture->sample_aspect_ratio.num
944 && ffmpegdec->pic_par_d == picture->sample_aspect_ratio.den
945 && ffmpegdec->pic_interlaced == picture->interlaced_frame
946 && ffmpegdec->pic_field_order == pic_field_order
947 && ffmpegdec->cur_multiview_mode == ffmpegdec->picture_multiview_mode
948 && ffmpegdec->cur_multiview_flags == ffmpegdec->picture_multiview_flags);
952 context_changed (GstFFMpegVidDec * ffmpegdec, AVCodecContext * context)
954 return !(ffmpegdec->ctx_ticks == context->ticks_per_frame
955 && ffmpegdec->ctx_time_n == context->time_base.num
956 && ffmpegdec->ctx_time_d == context->time_base.den);
960 update_video_context (GstFFMpegVidDec * ffmpegdec, AVCodecContext * context,
963 gint pic_field_order = 0;
965 if (picture->interlaced_frame) {
966 if (picture->repeat_pict)
967 pic_field_order |= GST_VIDEO_BUFFER_FLAG_RFF;
968 if (picture->top_field_first)
969 pic_field_order |= GST_VIDEO_BUFFER_FLAG_TFF;
972 if (!picture_changed (ffmpegdec, picture)
973 && !context_changed (ffmpegdec, context))
976 GST_DEBUG_OBJECT (ffmpegdec,
977 "Renegotiating video from %dx%d@ %d:%d PAR %d/%d fps pixfmt %d to %dx%d@ %d:%d PAR %d/%d fps pixfmt %d",
978 ffmpegdec->pic_width, ffmpegdec->pic_height,
979 ffmpegdec->pic_par_n, ffmpegdec->pic_par_d,
980 ffmpegdec->ctx_time_n, ffmpegdec->ctx_time_d,
981 ffmpegdec->pic_pix_fmt,
982 picture->width, picture->height,
983 picture->sample_aspect_ratio.num,
984 picture->sample_aspect_ratio.den,
985 context->time_base.num, context->time_base.den, picture->format);
987 ffmpegdec->pic_pix_fmt = picture->format;
988 ffmpegdec->pic_width = picture->width;
989 ffmpegdec->pic_height = picture->height;
990 ffmpegdec->pic_par_n = picture->sample_aspect_ratio.num;
991 ffmpegdec->pic_par_d = picture->sample_aspect_ratio.den;
992 ffmpegdec->cur_multiview_mode = ffmpegdec->picture_multiview_mode;
993 ffmpegdec->cur_multiview_flags = ffmpegdec->picture_multiview_flags;
995 /* Remember if we have interlaced content and the field order changed
996 * at least once. If that happens, we must be interlace-mode=mixed
998 if (ffmpegdec->pic_field_order_changed ||
999 (ffmpegdec->pic_field_order != pic_field_order &&
1000 ffmpegdec->pic_interlaced))
1001 ffmpegdec->pic_field_order_changed = TRUE;
1003 ffmpegdec->pic_field_order = pic_field_order;
1004 ffmpegdec->pic_interlaced = picture->interlaced_frame;
1006 if (!ffmpegdec->pic_interlaced)
1007 ffmpegdec->pic_field_order_changed = FALSE;
1009 ffmpegdec->ctx_ticks = context->ticks_per_frame;
1010 ffmpegdec->ctx_time_n = context->time_base.num;
1011 ffmpegdec->ctx_time_d = context->time_base.den;
1017 gst_ffmpegviddec_update_par (GstFFMpegVidDec * ffmpegdec,
1018 GstVideoInfo * in_info, GstVideoInfo * out_info)
1020 gboolean demuxer_par_set = FALSE;
1021 gboolean decoder_par_set = FALSE;
1022 gint demuxer_num = 1, demuxer_denom = 1;
1023 gint decoder_num = 1, decoder_denom = 1;
1025 if (in_info->par_n && in_info->par_d) {
1026 demuxer_num = in_info->par_n;
1027 demuxer_denom = in_info->par_d;
1028 demuxer_par_set = TRUE;
1029 GST_DEBUG_OBJECT (ffmpegdec, "Demuxer PAR: %d:%d", demuxer_num,
1033 if (ffmpegdec->pic_par_n && ffmpegdec->pic_par_d) {
1034 decoder_num = ffmpegdec->pic_par_n;
1035 decoder_denom = ffmpegdec->pic_par_d;
1036 decoder_par_set = TRUE;
1037 GST_DEBUG_OBJECT (ffmpegdec, "Decoder PAR: %d:%d", decoder_num,
1041 if (!demuxer_par_set && !decoder_par_set)
1044 if (demuxer_par_set && !decoder_par_set)
1045 goto use_demuxer_par;
1047 if (decoder_par_set && !demuxer_par_set)
1048 goto use_decoder_par;
1050 /* Both the demuxer and the decoder provide a PAR. If one of
1051 * the two PARs is 1:1 and the other one is not, use the one
1052 * that is not 1:1. */
1053 if (demuxer_num == demuxer_denom && decoder_num != decoder_denom)
1054 goto use_decoder_par;
1056 if (decoder_num == decoder_denom && demuxer_num != demuxer_denom)
1057 goto use_demuxer_par;
1059 /* Both PARs are non-1:1, so use the PAR provided by the demuxer */
1060 goto use_demuxer_par;
1064 GST_DEBUG_OBJECT (ffmpegdec,
1065 "Setting decoder provided pixel-aspect-ratio of %u:%u", decoder_num,
1067 out_info->par_n = decoder_num;
1068 out_info->par_d = decoder_denom;
1073 GST_DEBUG_OBJECT (ffmpegdec,
1074 "Setting demuxer provided pixel-aspect-ratio of %u:%u", demuxer_num,
1076 out_info->par_n = demuxer_num;
1077 out_info->par_d = demuxer_denom;
1082 GST_DEBUG_OBJECT (ffmpegdec,
1083 "Neither demuxer nor codec provide a pixel-aspect-ratio");
1084 out_info->par_n = 1;
1085 out_info->par_d = 1;
1090 static GstVideoMultiviewMode
1091 stereo_av_to_gst (enum AVStereo3DType type)
1094 case AV_STEREO3D_SIDEBYSIDE:
1095 return GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE;
1096 case AV_STEREO3D_TOPBOTTOM:
1097 return GST_VIDEO_MULTIVIEW_MODE_TOP_BOTTOM;
1098 case AV_STEREO3D_FRAMESEQUENCE:
1099 return GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME;
1100 case AV_STEREO3D_CHECKERBOARD:
1101 return GST_VIDEO_MULTIVIEW_MODE_CHECKERBOARD;
1102 case AV_STEREO3D_SIDEBYSIDE_QUINCUNX:
1103 return GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE_QUINCUNX;
1104 case AV_STEREO3D_LINES:
1105 return GST_VIDEO_MULTIVIEW_MODE_ROW_INTERLEAVED;
1106 case AV_STEREO3D_COLUMNS:
1107 return GST_VIDEO_MULTIVIEW_MODE_COLUMN_INTERLEAVED;
1112 return GST_VIDEO_MULTIVIEW_MODE_NONE;
1116 mastering_display_metadata_av_to_gst (AVMasteringDisplayMetadata * av,
1117 GstVideoMasteringDisplayInfo * gst)
1119 /* Use only complete mastering meta */
1120 if (!av->has_primaries || !av->has_luminance)
1123 gst->Wx_n = av->white_point[0].num;
1124 gst->Wx_d = av->white_point[0].den;
1126 gst->Wy_n = av->white_point[1].num;
1127 gst->Wy_d = av->white_point[1].den;
1129 gst->Rx_n = av->display_primaries[0][0].num;
1130 gst->Rx_d = av->display_primaries[0][0].den;
1132 gst->Ry_n = av->display_primaries[0][1].num;
1133 gst->Ry_d = av->display_primaries[0][1].den;
1135 gst->Gx_n = av->display_primaries[1][0].num;
1136 gst->Gx_d = av->display_primaries[1][0].den;
1138 gst->Gy_n = av->display_primaries[1][1].num;
1139 gst->Gy_d = av->display_primaries[1][1].den;
1141 gst->Bx_n = av->display_primaries[2][0].num;
1142 gst->Bx_d = av->display_primaries[2][0].den;
1144 gst->By_n = av->display_primaries[2][1].num;
1145 gst->By_d = av->display_primaries[2][1].den;
1147 gst->max_luma_n = av->max_luminance.num;
1148 gst->max_luma_d = av->max_luminance.den;
1150 gst->min_luma_n = av->min_luminance.num;
1151 gst->min_luma_d = av->min_luminance.den;
1157 content_light_metadata_av_to_gst (AVContentLightMetadata * av,
1158 GstVideoContentLightLevel * gst)
1160 gst->maxCLL_n = av->MaxCLL;
1163 gst->maxFALL_n = av->MaxFALL;
1170 gst_ffmpegviddec_negotiate (GstFFMpegVidDec * ffmpegdec,
1171 AVCodecContext * context, AVFrame * picture)
1174 GstVideoInfo *in_info, *out_info;
1175 GstVideoCodecState *output_state;
1177 GstClockTime latency;
1180 if (!update_video_context (ffmpegdec, context, picture))
1183 fmt = gst_ffmpeg_pixfmt_to_videoformat (ffmpegdec->pic_pix_fmt);
1184 if (G_UNLIKELY (fmt == GST_VIDEO_FORMAT_UNKNOWN))
1185 goto unknown_format;
1188 gst_video_decoder_set_output_state (GST_VIDEO_DECODER (ffmpegdec), fmt,
1189 ffmpegdec->pic_width, ffmpegdec->pic_height, ffmpegdec->input_state);
1190 if (ffmpegdec->output_state)
1191 gst_video_codec_state_unref (ffmpegdec->output_state);
1192 ffmpegdec->output_state = output_state;
1194 in_info = &ffmpegdec->input_state->info;
1195 out_info = &ffmpegdec->output_state->info;
1197 /* set the interlaced flag */
1198 in_s = gst_caps_get_structure (ffmpegdec->input_state->caps, 0);
1200 if (!gst_structure_has_field (in_s, "interlace-mode")) {
1201 if (ffmpegdec->pic_interlaced) {
1202 if (ffmpegdec->pic_field_order_changed ||
1203 (ffmpegdec->pic_field_order & GST_VIDEO_BUFFER_FLAG_RFF)) {
1204 out_info->interlace_mode = GST_VIDEO_INTERLACE_MODE_MIXED;
1206 out_info->interlace_mode = GST_VIDEO_INTERLACE_MODE_INTERLEAVED;
1207 if ((ffmpegdec->pic_field_order & GST_VIDEO_BUFFER_FLAG_TFF))
1208 GST_VIDEO_INFO_FIELD_ORDER (out_info) =
1209 GST_VIDEO_FIELD_ORDER_TOP_FIELD_FIRST;
1211 GST_VIDEO_INFO_FIELD_ORDER (out_info) =
1212 GST_VIDEO_FIELD_ORDER_BOTTOM_FIELD_FIRST;
1215 out_info->interlace_mode = GST_VIDEO_INTERLACE_MODE_PROGRESSIVE;
1219 if (!gst_structure_has_field (in_s, "chroma-site")) {
1220 switch (context->chroma_sample_location) {
1221 case AVCHROMA_LOC_LEFT:
1222 out_info->chroma_site = GST_VIDEO_CHROMA_SITE_MPEG2;
1224 case AVCHROMA_LOC_CENTER:
1225 out_info->chroma_site = GST_VIDEO_CHROMA_SITE_JPEG;
1227 case AVCHROMA_LOC_TOPLEFT:
1228 out_info->chroma_site = GST_VIDEO_CHROMA_SITE_DV;
1230 case AVCHROMA_LOC_TOP:
1231 out_info->chroma_site = GST_VIDEO_CHROMA_SITE_V_COSITED;
1238 if (!gst_structure_has_field (in_s, "colorimetry")
1239 || in_info->colorimetry.primaries == GST_VIDEO_COLOR_PRIMARIES_UNKNOWN) {
1240 switch (context->color_primaries) {
1241 case AVCOL_PRI_BT709:
1242 out_info->colorimetry.primaries = GST_VIDEO_COLOR_PRIMARIES_BT709;
1244 case AVCOL_PRI_BT470M:
1245 out_info->colorimetry.primaries = GST_VIDEO_COLOR_PRIMARIES_BT470M;
1247 case AVCOL_PRI_BT470BG:
1248 out_info->colorimetry.primaries = GST_VIDEO_COLOR_PRIMARIES_BT470BG;
1250 case AVCOL_PRI_SMPTE170M:
1251 out_info->colorimetry.primaries = GST_VIDEO_COLOR_PRIMARIES_SMPTE170M;
1253 case AVCOL_PRI_SMPTE240M:
1254 out_info->colorimetry.primaries = GST_VIDEO_COLOR_PRIMARIES_SMPTE240M;
1256 case AVCOL_PRI_FILM:
1257 out_info->colorimetry.primaries = GST_VIDEO_COLOR_PRIMARIES_FILM;
1259 case AVCOL_PRI_BT2020:
1260 out_info->colorimetry.primaries = GST_VIDEO_COLOR_PRIMARIES_BT2020;
1267 if (!gst_structure_has_field (in_s, "colorimetry")
1268 || in_info->colorimetry.transfer == GST_VIDEO_TRANSFER_UNKNOWN) {
1269 switch (context->color_trc) {
1270 case AVCOL_TRC_BT709:
1271 case AVCOL_TRC_SMPTE170M:
1272 out_info->colorimetry.transfer = GST_VIDEO_TRANSFER_BT709;
1274 case AVCOL_TRC_GAMMA22:
1275 out_info->colorimetry.transfer = GST_VIDEO_TRANSFER_GAMMA22;
1277 case AVCOL_TRC_GAMMA28:
1278 out_info->colorimetry.transfer = GST_VIDEO_TRANSFER_GAMMA28;
1280 case AVCOL_TRC_SMPTE240M:
1281 out_info->colorimetry.transfer = GST_VIDEO_TRANSFER_SMPTE240M;
1283 case AVCOL_TRC_LINEAR:
1284 out_info->colorimetry.transfer = GST_VIDEO_TRANSFER_GAMMA10;
1287 out_info->colorimetry.transfer = GST_VIDEO_TRANSFER_LOG100;
1289 case AVCOL_TRC_LOG_SQRT:
1290 out_info->colorimetry.transfer = GST_VIDEO_TRANSFER_LOG316;
1292 case AVCOL_TRC_BT2020_10:
1293 out_info->colorimetry.transfer = GST_VIDEO_TRANSFER_BT2020_10;
1295 case AVCOL_TRC_BT2020_12:
1296 out_info->colorimetry.transfer = GST_VIDEO_TRANSFER_BT2020_12;
1298 case AVCOL_TRC_SMPTE2084:
1299 out_info->colorimetry.transfer = GST_VIDEO_TRANSFER_SMPTE2084;
1301 case AVCOL_TRC_ARIB_STD_B67:
1302 out_info->colorimetry.transfer = GST_VIDEO_TRANSFER_ARIB_STD_B67;
1309 if (!gst_structure_has_field (in_s, "colorimetry")
1310 || in_info->colorimetry.matrix == GST_VIDEO_COLOR_MATRIX_UNKNOWN) {
1311 switch (context->colorspace) {
1313 out_info->colorimetry.matrix = GST_VIDEO_COLOR_MATRIX_RGB;
1315 case AVCOL_SPC_BT709:
1316 out_info->colorimetry.matrix = GST_VIDEO_COLOR_MATRIX_BT709;
1319 out_info->colorimetry.matrix = GST_VIDEO_COLOR_MATRIX_FCC;
1321 case AVCOL_SPC_BT470BG:
1322 case AVCOL_SPC_SMPTE170M:
1323 out_info->colorimetry.matrix = GST_VIDEO_COLOR_MATRIX_BT601;
1325 case AVCOL_SPC_SMPTE240M:
1326 out_info->colorimetry.matrix = GST_VIDEO_COLOR_MATRIX_SMPTE240M;
1328 case AVCOL_SPC_BT2020_NCL:
1329 out_info->colorimetry.matrix = GST_VIDEO_COLOR_MATRIX_BT2020;
1336 if (!gst_structure_has_field (in_s, "colorimetry")
1337 || in_info->colorimetry.range == GST_VIDEO_COLOR_RANGE_UNKNOWN) {
1338 if (context->color_range == AVCOL_RANGE_JPEG) {
1339 out_info->colorimetry.range = GST_VIDEO_COLOR_RANGE_0_255;
1341 out_info->colorimetry.range = GST_VIDEO_COLOR_RANGE_16_235;
1345 /* try to find a good framerate */
1346 if ((in_info->fps_d && in_info->fps_n) ||
1347 GST_VIDEO_INFO_FLAG_IS_SET (in_info, GST_VIDEO_FLAG_VARIABLE_FPS)) {
1348 /* take framerate from input when it was specified (#313970) */
1349 fps_n = in_info->fps_n;
1350 fps_d = in_info->fps_d;
1352 fps_n = ffmpegdec->ctx_time_d / ffmpegdec->ctx_ticks;
1353 fps_d = ffmpegdec->ctx_time_n;
1356 GST_LOG_OBJECT (ffmpegdec, "invalid framerate: %d/0, -> %d/1", fps_n,
1360 if (gst_util_fraction_compare (fps_n, fps_d, 1000, 1) > 0) {
1361 GST_LOG_OBJECT (ffmpegdec, "excessive framerate: %d/%d, -> 0/1", fps_n,
1368 GST_LOG_OBJECT (ffmpegdec, "setting framerate: %d/%d", fps_n, fps_d);
1369 out_info->fps_n = fps_n;
1370 out_info->fps_d = fps_d;
1372 /* calculate and update par now */
1373 gst_ffmpegviddec_update_par (ffmpegdec, in_info, out_info);
1375 GST_VIDEO_INFO_MULTIVIEW_MODE (out_info) = ffmpegdec->cur_multiview_mode;
1376 GST_VIDEO_INFO_MULTIVIEW_FLAGS (out_info) = ffmpegdec->cur_multiview_flags;
1378 /* To passing HDR information to caps directly */
1379 if (output_state->caps == NULL) {
1380 output_state->caps = gst_video_info_to_caps (out_info);
1382 output_state->caps = gst_caps_make_writable (output_state->caps);
1385 if (!gst_structure_has_field (in_s, "mastering-display-info")) {
1386 AVFrameSideData *sd = av_frame_get_side_data (picture,
1387 AV_FRAME_DATA_MASTERING_DISPLAY_METADATA);
1388 GstVideoMasteringDisplayInfo mastering;
1391 && mastering_display_metadata_av_to_gst ((AVMasteringDisplayMetadata *)
1392 sd->data, &mastering)) {
1393 GST_LOG_OBJECT (ffmpegdec, "update mastering display info");
1394 GST_LOG_OBJECT (ffmpegdec, "\tRed (%u/%u, %u/%u)", mastering.Rx_n,
1395 mastering.Rx_d, mastering.Ry_n, mastering.Ry_d);
1396 GST_LOG_OBJECT (ffmpegdec, "\tGreen(%u/%u, %u/%u)", mastering.Gx_n,
1397 mastering.Gx_d, mastering.Gy_n, mastering.Gy_d);
1398 GST_LOG_OBJECT (ffmpegdec, "\tBlue (%u/%u, %u/%u)", mastering.Bx_n,
1399 mastering.Bx_d, mastering.By_n, mastering.By_d);
1400 GST_LOG_OBJECT (ffmpegdec, "\tWhite(%u/%u, %u/%u)", mastering.Wx_n,
1401 mastering.Wx_d, mastering.Wy_n, mastering.Wy_d);
1402 GST_LOG_OBJECT (ffmpegdec,
1403 "\tmax_luminance:(%u/%u), min_luminance:(%u/%u)",
1404 mastering.max_luma_n, mastering.max_luma_d, mastering.min_luma_n,
1405 mastering.min_luma_d);
1407 if (!gst_video_mastering_display_info_add_to_caps (&mastering,
1408 output_state->caps)) {
1409 GST_WARNING_OBJECT (ffmpegdec,
1410 "Couldn't set mastering display info to caps");
1415 if (!gst_structure_has_field (in_s, "content-light-level")) {
1416 AVFrameSideData *sd = av_frame_get_side_data (picture,
1417 AV_FRAME_DATA_CONTENT_LIGHT_LEVEL);
1418 GstVideoContentLightLevel level;
1420 if (sd && content_light_metadata_av_to_gst ((AVContentLightMetadata *)
1421 sd->data, &level)) {
1422 GST_LOG_OBJECT (ffmpegdec, "update content light level");
1423 GST_LOG_OBJECT (ffmpegdec,
1424 "\tmaxCLL:(%u/%u), maxFALL:(%u/%u)", level.maxCLL_n, level.maxCLL_d,
1425 level.maxFALL_n, level.maxFALL_d);
1427 if (!gst_video_content_light_level_add_to_caps (&level,
1428 output_state->caps)) {
1429 GST_WARNING_OBJECT (ffmpegdec,
1430 "Couldn't set content light level to caps");
1435 if (!gst_video_decoder_negotiate (GST_VIDEO_DECODER (ffmpegdec)))
1436 goto negotiate_failed;
1438 /* The decoder is configured, we now know the true latency */
1441 gst_util_uint64_scale_ceil (ffmpegdec->context->has_b_frames *
1442 GST_SECOND, fps_d, fps_n);
1443 if (ffmpegdec->context->thread_type & FF_THREAD_FRAME) {
1445 gst_util_uint64_scale_ceil (ffmpegdec->context->thread_count *
1446 GST_SECOND, fps_d, fps_n);
1448 gst_video_decoder_set_latency (GST_VIDEO_DECODER (ffmpegdec), latency,
1457 GST_ERROR_OBJECT (ffmpegdec,
1458 "decoder requires a video format unsupported by GStreamer");
1463 /* Reset so we try again next time even if force==FALSE */
1464 ffmpegdec->pic_pix_fmt = 0;
1465 ffmpegdec->pic_width = 0;
1466 ffmpegdec->pic_height = 0;
1467 ffmpegdec->pic_par_n = 0;
1468 ffmpegdec->pic_par_d = 0;
1469 ffmpegdec->pic_interlaced = 0;
1470 ffmpegdec->pic_field_order = 0;
1471 ffmpegdec->pic_field_order_changed = FALSE;
1472 ffmpegdec->ctx_ticks = 0;
1473 ffmpegdec->ctx_time_n = 0;
1474 ffmpegdec->ctx_time_d = 0;
1476 GST_ERROR_OBJECT (ffmpegdec, "negotiation failed");
1481 /* perform qos calculations before decoding the next frame.
1483 * Sets the skip_frame flag and if things are really bad, skips to the next
1488 gst_ffmpegviddec_do_qos (GstFFMpegVidDec * ffmpegdec,
1489 GstVideoCodecFrame * frame, gboolean * mode_switch)
1491 GstClockTimeDiff diff;
1492 GstSegmentFlags skip_flags =
1493 GST_VIDEO_DECODER_INPUT_SEGMENT (ffmpegdec).flags;
1495 *mode_switch = FALSE;
1500 if (skip_flags & GST_SEGMENT_FLAG_TRICKMODE_KEY_UNITS) {
1501 ffmpegdec->context->skip_frame = AVDISCARD_NONKEY;
1502 *mode_switch = TRUE;
1503 } else if (skip_flags & GST_SEGMENT_FLAG_TRICKMODE) {
1504 ffmpegdec->context->skip_frame = AVDISCARD_NONREF;
1505 *mode_switch = TRUE;
1508 if (*mode_switch == TRUE) {
1509 /* We've already switched mode, we can return straight away
1510 * without any further calculation */
1515 gst_video_decoder_get_max_decode_time (GST_VIDEO_DECODER (ffmpegdec),
1518 /* if we don't have timing info, then we don't do QoS */
1519 if (G_UNLIKELY (diff == G_MAXINT64)) {
1520 /* Ensure the skipping strategy is the default one */
1521 ffmpegdec->context->skip_frame = ffmpegdec->skip_frame;
1525 GST_DEBUG_OBJECT (ffmpegdec, "decoding time %" G_GINT64_FORMAT, diff);
1527 if (diff > 0 && ffmpegdec->context->skip_frame != AVDISCARD_DEFAULT) {
1528 ffmpegdec->context->skip_frame = AVDISCARD_DEFAULT;
1529 *mode_switch = TRUE;
1530 GST_DEBUG_OBJECT (ffmpegdec, "QOS: normal mode");
1533 else if (diff <= 0 && ffmpegdec->context->skip_frame != AVDISCARD_NONREF) {
1534 ffmpegdec->context->skip_frame = AVDISCARD_NONREF;
1535 *mode_switch = TRUE;
1536 GST_DEBUG_OBJECT (ffmpegdec,
1537 "QOS: hurry up, diff %" G_GINT64_FORMAT " >= 0", diff);
1541 /* get an outbuf buffer with the current picture */
1542 static GstFlowReturn
1543 get_output_buffer (GstFFMpegVidDec * ffmpegdec, GstVideoCodecFrame * frame)
1545 GstFlowReturn ret = GST_FLOW_OK;
1546 AVFrame pic, *outpic;
1547 GstVideoFrame vframe;
1551 GST_LOG_OBJECT (ffmpegdec, "get output buffer");
1553 if (!ffmpegdec->output_state)
1554 goto not_negotiated;
1557 gst_video_decoder_allocate_output_frame (GST_VIDEO_DECODER (ffmpegdec),
1559 if (G_UNLIKELY (ret != GST_FLOW_OK))
1562 /* original ffmpeg code does not handle odd sizes correctly.
1563 * This patched up version does */
1564 /* Fill avpicture */
1565 info = &ffmpegdec->output_state->info;
1566 if (!gst_video_frame_map (&vframe, info, frame->output_buffer,
1567 GST_MAP_READ | GST_MAP_WRITE))
1570 memset (&pic, 0, sizeof (pic));
1571 pic.format = ffmpegdec->pic_pix_fmt;
1572 pic.width = GST_VIDEO_FRAME_WIDTH (&vframe);
1573 pic.height = GST_VIDEO_FRAME_HEIGHT (&vframe);
1574 for (c = 0; c < AV_NUM_DATA_POINTERS; c++) {
1575 if (c < GST_VIDEO_INFO_N_PLANES (info)) {
1576 pic.data[c] = GST_VIDEO_FRAME_PLANE_DATA (&vframe, c);
1577 pic.linesize[c] = GST_VIDEO_FRAME_PLANE_STRIDE (&vframe, c);
1578 GST_LOG_OBJECT (ffmpegdec, "[%i] linesize %d, data %p", c,
1579 pic.linesize[c], pic.data[c]);
1582 pic.linesize[c] = 0;
1586 outpic = ffmpegdec->picture;
1588 if (av_frame_copy (&pic, outpic) != 0) {
1589 GST_ERROR_OBJECT (ffmpegdec, "Failed to copy output frame");
1590 ret = GST_FLOW_ERROR;
1593 gst_video_frame_unmap (&vframe);
1595 ffmpegdec->picture->reordered_opaque = -1;
1602 GST_ELEMENT_ERROR (ffmpegdec, RESOURCE, FAILED,
1603 ("Unable to allocate memory"),
1604 ("The downstream pool failed to allocated buffer."));
1609 GST_ELEMENT_ERROR (ffmpegdec, RESOURCE, OPEN_READ_WRITE,
1610 ("Cannot access memory for read and write operation."),
1611 ("The video memory allocated from downstream pool could not mapped for"
1612 "read and write."));
1617 GST_DEBUG_OBJECT (ffmpegdec, "not negotiated");
1618 return GST_FLOW_NOT_NEGOTIATED;
1623 gst_avpacket_init (AVPacket * packet, guint8 * data, guint size)
1625 memset (packet, 0, sizeof (AVPacket));
1626 packet->data = data;
1627 packet->size = size;
1631 * Returns: whether a frame was decoded
1634 gst_ffmpegviddec_video_frame (GstFFMpegVidDec * ffmpegdec,
1635 GstVideoCodecFrame * frame, GstFlowReturn * ret)
1638 gboolean got_frame = FALSE;
1639 gboolean mode_switch;
1640 GstVideoCodecFrame *out_frame;
1641 GstFFMpegVidDecVideoFrame *out_dframe;
1642 GstBufferPool *pool;
1646 /* in case we skip frames */
1647 ffmpegdec->picture->pict_type = -1;
1649 /* run QoS code, we don't stop decoding the frame when we are late because
1650 * else we might skip a reference frame */
1651 gst_ffmpegviddec_do_qos (ffmpegdec, frame, &mode_switch);
1653 res = avcodec_receive_frame (ffmpegdec->context, ffmpegdec->picture);
1655 /* No frames available at this time */
1656 if (res == AVERROR (EAGAIN))
1658 else if (res == AVERROR_EOF) {
1659 *ret = GST_FLOW_EOS;
1660 GST_DEBUG_OBJECT (ffmpegdec, "Context was entirely flushed");
1662 } else if (res < 0) {
1664 GST_WARNING_OBJECT (ffmpegdec, "Legitimate decoding error");
1670 /* get the output picture timing info again */
1671 out_dframe = ffmpegdec->picture->opaque;
1672 out_frame = gst_video_codec_frame_ref (out_dframe->frame);
1674 /* also give back a buffer allocated by the frame, if any */
1675 gst_buffer_replace (&out_frame->output_buffer, out_dframe->buffer);
1676 gst_buffer_replace (&out_dframe->buffer, NULL);
1678 /* Extract auxilliary info not stored in the main AVframe */
1680 GstVideoInfo *in_info = &ffmpegdec->input_state->info;
1681 /* Take multiview mode from upstream if present */
1682 ffmpegdec->picture_multiview_mode = GST_VIDEO_INFO_MULTIVIEW_MODE (in_info);
1683 ffmpegdec->picture_multiview_flags =
1684 GST_VIDEO_INFO_MULTIVIEW_FLAGS (in_info);
1686 /* Otherwise, see if there's info in the frame */
1687 if (ffmpegdec->picture_multiview_mode == GST_VIDEO_MULTIVIEW_MODE_NONE) {
1688 AVFrameSideData *side_data =
1689 av_frame_get_side_data (ffmpegdec->picture, AV_FRAME_DATA_STEREO3D);
1691 AVStereo3D *stereo = (AVStereo3D *) side_data->data;
1692 ffmpegdec->picture_multiview_mode = stereo_av_to_gst (stereo->type);
1693 if (stereo->flags & AV_STEREO3D_FLAG_INVERT) {
1694 ffmpegdec->picture_multiview_flags =
1695 GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST;
1697 ffmpegdec->picture_multiview_flags = GST_VIDEO_MULTIVIEW_FLAGS_NONE;
1703 GST_DEBUG_OBJECT (ffmpegdec,
1704 "pts %" G_GUINT64_FORMAT " duration %" G_GUINT64_FORMAT,
1705 out_frame->pts, out_frame->duration);
1706 GST_DEBUG_OBJECT (ffmpegdec, "picture: pts %" G_GUINT64_FORMAT,
1707 (guint64) ffmpegdec->picture->pts);
1708 GST_DEBUG_OBJECT (ffmpegdec, "picture: num %d",
1709 ffmpegdec->picture->coded_picture_number);
1710 GST_DEBUG_OBJECT (ffmpegdec, "picture: display %d",
1711 ffmpegdec->picture->display_picture_number);
1712 GST_DEBUG_OBJECT (ffmpegdec, "picture: opaque %p",
1713 ffmpegdec->picture->opaque);
1714 GST_DEBUG_OBJECT (ffmpegdec, "picture: reordered opaque %" G_GUINT64_FORMAT,
1715 (guint64) ffmpegdec->picture->reordered_opaque);
1716 GST_DEBUG_OBJECT (ffmpegdec, "repeat_pict:%d",
1717 ffmpegdec->picture->repeat_pict);
1718 GST_DEBUG_OBJECT (ffmpegdec, "corrupted frame: %d",
1719 ! !(ffmpegdec->picture->flags & AV_FRAME_FLAG_CORRUPT));
1721 if (!gst_ffmpegviddec_negotiate (ffmpegdec, ffmpegdec->context,
1722 ffmpegdec->picture))
1723 goto negotiation_error;
1725 pool = gst_video_decoder_get_buffer_pool (GST_VIDEO_DECODER (ffmpegdec));
1726 if (G_UNLIKELY (out_frame->output_buffer == NULL)) {
1727 *ret = get_output_buffer (ffmpegdec, out_frame);
1728 } else if (G_UNLIKELY (out_frame->output_buffer->pool != pool)) {
1729 GstBuffer *tmp = out_frame->output_buffer;
1730 out_frame->output_buffer = NULL;
1731 *ret = get_output_buffer (ffmpegdec, out_frame);
1732 gst_buffer_unref (tmp);
1734 #ifndef G_DISABLE_ASSERT
1736 GstVideoMeta *vmeta = gst_buffer_get_video_meta (out_frame->output_buffer);
1738 GstVideoInfo *info = &ffmpegdec->output_state->info;
1739 g_assert ((gint) vmeta->width == GST_VIDEO_INFO_WIDTH (info));
1740 g_assert ((gint) vmeta->height == GST_VIDEO_INFO_HEIGHT (info));
1744 gst_object_unref (pool);
1746 if (G_UNLIKELY (*ret != GST_FLOW_OK))
1749 /* Mark corrupted frames as corrupted */
1750 if (ffmpegdec->picture->flags & AV_FRAME_FLAG_CORRUPT)
1751 GST_BUFFER_FLAG_SET (out_frame->output_buffer, GST_BUFFER_FLAG_CORRUPTED);
1753 if (ffmpegdec->pic_interlaced) {
1754 /* set interlaced flags */
1755 if (ffmpegdec->picture->repeat_pict)
1756 GST_BUFFER_FLAG_SET (out_frame->output_buffer, GST_VIDEO_BUFFER_FLAG_RFF);
1757 if (ffmpegdec->picture->top_field_first)
1758 GST_BUFFER_FLAG_SET (out_frame->output_buffer, GST_VIDEO_BUFFER_FLAG_TFF);
1759 if (ffmpegdec->picture->interlaced_frame)
1760 GST_BUFFER_FLAG_SET (out_frame->output_buffer,
1761 GST_VIDEO_BUFFER_FLAG_INTERLACED);
1765 AVFrameSideData *side_data =
1766 av_frame_get_side_data (ffmpegdec->picture, AV_FRAME_DATA_A53_CC);
1768 GstVideoCaptionMeta *cc_meta = NULL;
1769 gpointer iter = NULL;
1770 gboolean found_708_raw_meta = FALSE;
1772 GST_LOG_OBJECT (ffmpegdec,
1773 "Found CC side data of type AV_FRAME_DATA_A53_CC, size %d",
1775 GST_MEMDUMP ("A53 CC", side_data->data, side_data->size);
1777 while ((cc_meta = (GstVideoCaptionMeta *)
1778 gst_buffer_iterate_meta_filtered (out_frame->input_buffer, &iter,
1779 GST_VIDEO_CAPTION_META_API_TYPE))) {
1780 if (cc_meta->caption_type != GST_VIDEO_CAPTION_TYPE_CEA708_RAW)
1782 found_708_raw_meta = TRUE;
1786 /* do not add CEA 708 caption meta if it already exists */
1787 if (!found_708_raw_meta) {
1788 out_frame->output_buffer =
1789 gst_buffer_make_writable (out_frame->output_buffer);
1790 gst_buffer_add_video_caption_meta (out_frame->output_buffer,
1791 GST_VIDEO_CAPTION_TYPE_CEA708_RAW, side_data->data,
1794 GST_LOG_OBJECT (ffmpegdec,
1795 "CEA 708 caption meta already exists: will not add new caption meta");
1801 /* so we decoded this frame, frames preceding it in decoding order
1802 * that still do not have a buffer allocated seem rather useless,
1803 * and can be discarded, due to e.g. misparsed bogus frame
1804 * or non-keyframe in skipped decoding, ...
1805 * In any case, not likely to be seen again, so discard those,
1806 * before they pile up and/or mess with timestamping */
1809 GstVideoDecoder *dec = GST_VIDEO_DECODER (ffmpegdec);
1810 gboolean old = TRUE;
1812 ol = l = gst_video_decoder_get_frames (dec);
1814 GstVideoCodecFrame *tmp = l->data;
1819 if (old && GST_VIDEO_CODEC_FRAME_IS_DECODE_ONLY (tmp)) {
1820 GST_LOG_OBJECT (dec,
1821 "discarding ghost frame %p (#%d) PTS:%" GST_TIME_FORMAT " DTS:%"
1822 GST_TIME_FORMAT, tmp, tmp->system_frame_number,
1823 GST_TIME_ARGS (tmp->pts), GST_TIME_ARGS (tmp->dts));
1824 /* drop extra ref and remove from frame list */
1825 gst_video_decoder_release_frame (dec, tmp);
1827 /* drop extra ref we got */
1828 gst_video_codec_frame_unref (tmp);
1835 av_frame_unref (ffmpegdec->picture);
1837 /* FIXME: Ideally we would remap the buffer read-only now before pushing but
1838 * libav might still have a reference to it!
1841 gst_video_decoder_finish_frame (GST_VIDEO_DECODER (ffmpegdec), out_frame);
1844 GST_DEBUG_OBJECT (ffmpegdec, "return flow %s, got frame: %d",
1845 gst_flow_get_name (*ret), got_frame);
1851 GST_DEBUG_OBJECT (ffmpegdec, "no output buffer");
1852 gst_video_decoder_drop_frame (GST_VIDEO_DECODER (ffmpegdec), out_frame);
1858 if (GST_PAD_IS_FLUSHING (GST_VIDEO_DECODER_SRC_PAD (ffmpegdec))) {
1859 *ret = GST_FLOW_FLUSHING;
1862 GST_WARNING_OBJECT (ffmpegdec, "Error negotiating format");
1863 *ret = GST_FLOW_NOT_NEGOTIATED;
1869 /* Returns: Whether a frame was decoded */
1871 gst_ffmpegviddec_frame (GstFFMpegVidDec * ffmpegdec, GstVideoCodecFrame * frame,
1872 GstFlowReturn * ret)
1874 gboolean got_frame = FALSE;
1876 if (G_UNLIKELY (ffmpegdec->context->codec == NULL))
1880 ffmpegdec->context->frame_number++;
1882 got_frame = gst_ffmpegviddec_video_frame (ffmpegdec, frame, ret);
1889 GST_ERROR_OBJECT (ffmpegdec, "no codec context");
1890 *ret = GST_FLOW_NOT_NEGOTIATED;
1895 static GstFlowReturn
1896 gst_ffmpegviddec_drain (GstVideoDecoder * decoder)
1898 GstFFMpegVidDec *ffmpegdec = (GstFFMpegVidDec *) decoder;
1900 gboolean got_frame = FALSE;
1902 if (!ffmpegdec->opened)
1905 if (avcodec_send_packet (ffmpegdec->context, NULL))
1906 goto send_packet_failed;
1909 got_frame = gst_ffmpegviddec_frame (ffmpegdec, NULL, &ret);
1910 } while (got_frame && ret == GST_FLOW_OK);
1911 avcodec_flush_buffers (ffmpegdec->context);
1917 GST_WARNING_OBJECT (ffmpegdec, "send packet failed, could not drain decoder");
1921 static GstFlowReturn
1922 gst_ffmpegviddec_handle_frame (GstVideoDecoder * decoder,
1923 GstVideoCodecFrame * frame)
1925 GstFFMpegVidDec *ffmpegdec = (GstFFMpegVidDec *) decoder;
1930 GstFlowReturn ret = GST_FLOW_OK;
1933 GST_LOG_OBJECT (ffmpegdec,
1934 "Received new data of size %" G_GSIZE_FORMAT ", dts %" GST_TIME_FORMAT
1935 ", pts:%" GST_TIME_FORMAT ", dur:%" GST_TIME_FORMAT,
1936 gst_buffer_get_size (frame->input_buffer), GST_TIME_ARGS (frame->dts),
1937 GST_TIME_ARGS (frame->pts), GST_TIME_ARGS (frame->duration));
1939 if (!gst_buffer_map (frame->input_buffer, &minfo, GST_MAP_READ)) {
1940 GST_ELEMENT_ERROR (ffmpegdec, STREAM, DECODE, ("Decoding problem"),
1941 ("Failed to map buffer for reading"));
1942 return GST_FLOW_ERROR;
1945 /* treat frame as void until a buffer is requested for it */
1946 GST_VIDEO_CODEC_FRAME_FLAG_SET (frame,
1947 GST_VIDEO_CODEC_FRAME_FLAG_DECODE_ONLY);
1952 if (size > 0 && (!GST_MEMORY_IS_ZERO_PADDED (minfo.memory)
1953 || (minfo.maxsize - minfo.size) < AV_INPUT_BUFFER_PADDING_SIZE)) {
1955 if (ffmpegdec->padded_size < size + AV_INPUT_BUFFER_PADDING_SIZE) {
1956 ffmpegdec->padded_size = size + AV_INPUT_BUFFER_PADDING_SIZE;
1957 ffmpegdec->padded = g_realloc (ffmpegdec->padded, ffmpegdec->padded_size);
1958 GST_LOG_OBJECT (ffmpegdec, "resized padding buffer to %d",
1959 ffmpegdec->padded_size);
1961 GST_CAT_TRACE_OBJECT (GST_CAT_PERFORMANCE, ffmpegdec,
1962 "Copy input to add padding");
1963 memcpy (ffmpegdec->padded, data, size);
1964 memset (ffmpegdec->padded + size, 0, AV_INPUT_BUFFER_PADDING_SIZE);
1966 data = ffmpegdec->padded;
1969 /* now decode the frame */
1970 gst_avpacket_init (&packet, data, size);
1972 if (ffmpegdec->palette) {
1975 pal = av_packet_new_side_data (&packet, AV_PKT_DATA_PALETTE,
1977 gst_buffer_extract (ffmpegdec->palette, 0, pal, AVPALETTE_SIZE);
1978 GST_DEBUG_OBJECT (ffmpegdec, "copy pal %p %p", &packet, pal);
1984 /* save reference to the timing info */
1985 ffmpegdec->context->reordered_opaque = (gint64) frame->system_frame_number;
1986 ffmpegdec->picture->reordered_opaque = (gint64) frame->system_frame_number;
1988 GST_DEBUG_OBJECT (ffmpegdec, "stored opaque values idx %d",
1989 frame->system_frame_number);
1991 /* This might call into get_buffer() from another thread,
1992 * which would cause a deadlock. Release the lock here
1993 * and taking it again later seems safe
1994 * See https://bugzilla.gnome.org/show_bug.cgi?id=726020
1996 GST_VIDEO_DECODER_STREAM_UNLOCK (ffmpegdec);
1997 if (avcodec_send_packet (ffmpegdec->context, &packet) < 0) {
1998 GST_VIDEO_DECODER_STREAM_LOCK (ffmpegdec);
1999 goto send_packet_failed;
2001 GST_VIDEO_DECODER_STREAM_LOCK (ffmpegdec);
2004 /* decode a frame of audio/video now */
2005 got_frame = gst_ffmpegviddec_frame (ffmpegdec, frame, &ret);
2007 if (ret != GST_FLOW_OK) {
2008 GST_LOG_OBJECT (ffmpegdec, "breaking because of flow ret %s",
2009 gst_flow_get_name (ret));
2012 } while (got_frame);
2015 gst_buffer_unmap (frame->input_buffer, &minfo);
2016 gst_video_codec_frame_unref (frame);
2022 GST_WARNING_OBJECT (ffmpegdec, "Failed to send data for decoding");
2028 gst_ffmpegviddec_start (GstVideoDecoder * decoder)
2030 GstFFMpegVidDec *ffmpegdec = (GstFFMpegVidDec *) decoder;
2031 GstFFMpegVidDecClass *oclass;
2033 oclass = (GstFFMpegVidDecClass *) (G_OBJECT_GET_CLASS (ffmpegdec));
2035 GST_OBJECT_LOCK (ffmpegdec);
2036 gst_ffmpeg_avcodec_close (ffmpegdec->context);
2037 if (avcodec_get_context_defaults3 (ffmpegdec->context, oclass->in_plugin) < 0) {
2038 GST_DEBUG_OBJECT (ffmpegdec, "Failed to set context defaults");
2039 GST_OBJECT_UNLOCK (ffmpegdec);
2042 ffmpegdec->context->opaque = ffmpegdec;
2043 GST_OBJECT_UNLOCK (ffmpegdec);
2049 gst_ffmpegviddec_stop (GstVideoDecoder * decoder)
2051 GstFFMpegVidDec *ffmpegdec = (GstFFMpegVidDec *) decoder;
2053 GST_OBJECT_LOCK (ffmpegdec);
2054 gst_ffmpegviddec_close (ffmpegdec, FALSE);
2055 GST_OBJECT_UNLOCK (ffmpegdec);
2056 g_free (ffmpegdec->padded);
2057 ffmpegdec->padded = NULL;
2058 ffmpegdec->padded_size = 0;
2059 if (ffmpegdec->input_state)
2060 gst_video_codec_state_unref (ffmpegdec->input_state);
2061 ffmpegdec->input_state = NULL;
2062 if (ffmpegdec->output_state)
2063 gst_video_codec_state_unref (ffmpegdec->output_state);
2064 ffmpegdec->output_state = NULL;
2066 if (ffmpegdec->internal_pool)
2067 gst_object_unref (ffmpegdec->internal_pool);
2068 ffmpegdec->internal_pool = NULL;
2070 ffmpegdec->pic_pix_fmt = 0;
2071 ffmpegdec->pic_width = 0;
2072 ffmpegdec->pic_height = 0;
2073 ffmpegdec->pic_par_n = 0;
2074 ffmpegdec->pic_par_d = 0;
2075 ffmpegdec->pic_interlaced = 0;
2076 ffmpegdec->pic_field_order = 0;
2077 ffmpegdec->pic_field_order_changed = FALSE;
2078 ffmpegdec->ctx_ticks = 0;
2079 ffmpegdec->ctx_time_n = 0;
2080 ffmpegdec->ctx_time_d = 0;
2082 ffmpegdec->pool_width = 0;
2083 ffmpegdec->pool_height = 0;
2084 ffmpegdec->pool_format = 0;
2089 static GstFlowReturn
2090 gst_ffmpegviddec_finish (GstVideoDecoder * decoder)
2092 gst_ffmpegviddec_drain (decoder);
2093 /* note that finish can and should clean up more drastically,
2094 * but drain is also invoked on e.g. packet loss in GAP handling */
2095 gst_ffmpegviddec_flush (decoder);
2101 gst_ffmpegviddec_flush (GstVideoDecoder * decoder)
2103 GstFFMpegVidDec *ffmpegdec = (GstFFMpegVidDec *) decoder;
2105 if (ffmpegdec->opened) {
2106 GST_LOG_OBJECT (decoder, "flushing buffers");
2107 avcodec_flush_buffers (ffmpegdec->context);
2114 gst_ffmpegviddec_decide_allocation (GstVideoDecoder * decoder, GstQuery * query)
2116 GstFFMpegVidDec *ffmpegdec = (GstFFMpegVidDec *) decoder;
2117 GstVideoCodecState *state;
2118 GstBufferPool *pool;
2119 guint size, min, max;
2120 GstStructure *config;
2121 gboolean have_pool, have_videometa, have_alignment, update_pool = FALSE;
2122 GstAllocator *allocator = NULL;
2123 GstAllocationParams params = DEFAULT_ALLOC_PARAM;
2125 have_pool = (gst_query_get_n_allocation_pools (query) != 0);
2127 if (!GST_VIDEO_DECODER_CLASS (parent_class)->decide_allocation (decoder,
2131 state = gst_video_decoder_get_output_state (decoder);
2133 if (gst_query_get_n_allocation_params (query) > 0) {
2134 gst_query_parse_nth_allocation_param (query, 0, &allocator, ¶ms);
2135 params.align = MAX (params.align, DEFAULT_STRIDE_ALIGN);
2137 gst_query_add_allocation_param (query, allocator, ¶ms);
2140 gst_query_parse_nth_allocation_pool (query, 0, &pool, &size, &min, &max);
2142 /* Don't use pool that can't grow, as we don't know how many buffer we'll
2143 * need, otherwise we may stall */
2144 if (max != 0 && max < REQUIRED_POOL_MAX_BUFFERS) {
2145 gst_object_unref (pool);
2146 pool = gst_video_buffer_pool_new ();
2151 /* if there is an allocator, also drop it, as it might be the reason we
2152 * have this limit. Default will be used */
2154 gst_object_unref (allocator);
2159 config = gst_buffer_pool_get_config (pool);
2160 gst_buffer_pool_config_set_params (config, state->caps, size, min, max);
2161 gst_buffer_pool_config_set_allocator (config, allocator, ¶ms);
2164 gst_query_find_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL);
2167 gst_buffer_pool_config_add_option (config,
2168 GST_BUFFER_POOL_OPTION_VIDEO_META);
2171 gst_buffer_pool_has_option (pool, GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT);
2173 /* If we have videometa, we never have to copy */
2174 if (have_videometa && have_pool && have_alignment &&
2175 gst_ffmpegviddec_can_direct_render (ffmpegdec)) {
2176 GstStructure *config_copy = gst_structure_copy (config);
2178 gst_ffmpegvideodec_prepare_dr_pool (ffmpegdec, pool, &state->info,
2181 /* FIXME validate and retry */
2182 if (gst_buffer_pool_set_config (pool, config_copy)) {
2186 gst_buffer_pool_set_active (pool, TRUE);
2187 ret = gst_buffer_pool_acquire_buffer (pool, &tmp, NULL);
2188 if (ret == GST_FLOW_OK) {
2189 GstVideoMeta *vmeta = gst_buffer_get_video_meta (tmp);
2190 gboolean same_stride = TRUE;
2193 for (i = 0; i < vmeta->n_planes; i++) {
2194 if (vmeta->stride[i] != ffmpegdec->stride[i]) {
2195 same_stride = FALSE;
2200 gst_buffer_unref (tmp);
2203 if (ffmpegdec->internal_pool)
2204 gst_object_unref (ffmpegdec->internal_pool);
2205 ffmpegdec->internal_pool = gst_object_ref (pool);
2206 ffmpegdec->pool_info = state->info;
2207 gst_structure_free (config);
2214 if (have_videometa && ffmpegdec->internal_pool
2215 && ffmpegdec->pool_width == state->info.width
2216 && ffmpegdec->pool_height == state->info.height) {
2218 gst_object_unref (pool);
2219 pool = gst_object_ref (ffmpegdec->internal_pool);
2220 gst_structure_free (config);
2225 if (!gst_buffer_pool_set_config (pool, config)) {
2226 gboolean working_pool = FALSE;
2227 config = gst_buffer_pool_get_config (pool);
2229 if (gst_buffer_pool_config_validate_params (config, state->caps, size, min,
2231 working_pool = gst_buffer_pool_set_config (pool, config);
2233 gst_structure_free (config);
2236 if (!working_pool) {
2237 gst_object_unref (pool);
2238 pool = gst_video_buffer_pool_new ();
2239 config = gst_buffer_pool_get_config (pool);
2240 gst_buffer_pool_config_set_params (config, state->caps, size, min, max);
2241 gst_buffer_pool_config_set_allocator (config, NULL, ¶ms);
2242 gst_buffer_pool_set_config (pool, config);
2250 gst_query_set_nth_allocation_pool (query, 0, pool, size, min, max);
2252 gst_object_unref (pool);
2254 gst_object_unref (allocator);
2255 gst_video_codec_state_unref (state);
2261 gst_ffmpegviddec_propose_allocation (GstVideoDecoder * decoder,
2264 GstAllocationParams params;
2266 gst_allocation_params_init (¶ms);
2267 params.flags = GST_MEMORY_FLAG_ZERO_PADDED;
2268 params.align = DEFAULT_STRIDE_ALIGN;
2269 params.padding = AV_INPUT_BUFFER_PADDING_SIZE;
2270 /* we would like to have some padding so that we don't have to
2271 * memcpy. We don't suggest an allocator. */
2272 gst_query_add_allocation_param (query, NULL, ¶ms);
2274 return GST_VIDEO_DECODER_CLASS (parent_class)->propose_allocation (decoder,
2279 gst_ffmpegviddec_set_property (GObject * object,
2280 guint prop_id, const GValue * value, GParamSpec * pspec)
2282 GstFFMpegVidDec *ffmpegdec = (GstFFMpegVidDec *) object;
2286 ffmpegdec->lowres = ffmpegdec->context->lowres = g_value_get_enum (value);
2288 case PROP_SKIPFRAME:
2289 ffmpegdec->skip_frame = ffmpegdec->context->skip_frame =
2290 g_value_get_enum (value);
2292 case PROP_DIRECT_RENDERING:
2293 ffmpegdec->direct_rendering = g_value_get_boolean (value);
2296 ffmpegdec->debug_mv = ffmpegdec->context->debug_mv =
2297 g_value_get_boolean (value);
2299 case PROP_MAX_THREADS:
2300 ffmpegdec->max_threads = g_value_get_int (value);
2302 case PROP_OUTPUT_CORRUPT:
2303 ffmpegdec->output_corrupt = g_value_get_boolean (value);
2305 case PROP_THREAD_TYPE:
2306 ffmpegdec->thread_type = g_value_get_flags (value);
2309 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
2315 gst_ffmpegviddec_get_property (GObject * object,
2316 guint prop_id, GValue * value, GParamSpec * pspec)
2318 GstFFMpegVidDec *ffmpegdec = (GstFFMpegVidDec *) object;
2322 g_value_set_enum (value, ffmpegdec->context->lowres);
2324 case PROP_SKIPFRAME:
2325 g_value_set_enum (value, ffmpegdec->context->skip_frame);
2327 case PROP_DIRECT_RENDERING:
2328 g_value_set_boolean (value, ffmpegdec->direct_rendering);
2331 g_value_set_boolean (value, ffmpegdec->context->debug_mv);
2333 case PROP_MAX_THREADS:
2334 g_value_set_int (value, ffmpegdec->max_threads);
2336 case PROP_OUTPUT_CORRUPT:
2337 g_value_set_boolean (value, ffmpegdec->output_corrupt);
2339 case PROP_THREAD_TYPE:
2340 g_value_set_flags (value, ffmpegdec->thread_type);
2343 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
2349 gst_ffmpegviddec_register (GstPlugin * plugin)
2351 GTypeInfo typeinfo = {
2352 sizeof (GstFFMpegVidDecClass),
2353 (GBaseInitFunc) gst_ffmpegviddec_base_init,
2355 (GClassInitFunc) gst_ffmpegviddec_class_init,
2358 sizeof (GstFFMpegVidDec),
2360 (GInstanceInitFunc) gst_ffmpegviddec_init,
2367 GST_LOG ("Registering decoders");
2369 while ((in_plugin = (AVCodec *) av_codec_iterate (&i))) {
2373 /* only video decoders */
2374 if (!av_codec_is_decoder (in_plugin)
2375 || in_plugin->type != AVMEDIA_TYPE_VIDEO)
2378 /* no quasi codecs, please */
2379 if (in_plugin->id == AV_CODEC_ID_RAWVIDEO ||
2380 in_plugin->id == AV_CODEC_ID_V210 ||
2381 in_plugin->id == AV_CODEC_ID_V210X ||
2382 in_plugin->id == AV_CODEC_ID_V308 ||
2383 in_plugin->id == AV_CODEC_ID_V408 ||
2384 in_plugin->id == AV_CODEC_ID_V410 ||
2385 in_plugin->id == AV_CODEC_ID_R210
2386 || in_plugin->id == AV_CODEC_ID_AYUV
2387 || in_plugin->id == AV_CODEC_ID_Y41P
2388 || in_plugin->id == AV_CODEC_ID_012V
2389 || in_plugin->id == AV_CODEC_ID_YUV4
2390 #if AV_VERSION_INT (LIBAVCODEC_VERSION_MAJOR, LIBAVCODEC_VERSION_MINOR, LIBAVCODEC_VERSION_MICRO) >= \
2391 AV_VERSION_INT (57,4,0)
2392 || in_plugin->id == AV_CODEC_ID_WRAPPED_AVFRAME
2394 || in_plugin->id == AV_CODEC_ID_ZLIB) {
2398 /* No decoders depending on external libraries (we don't build them, but
2399 * people who build against an external ffmpeg might have them.
2400 * We have native gstreamer plugins for all of those libraries anyway. */
2401 if (!strncmp (in_plugin->name, "lib", 3)) {
2403 ("Not using external library decoder %s. Use the gstreamer-native ones instead.",
2408 /* No vdpau plugins until we can figure out how to properly use them
2409 * outside of ffmpeg. */
2410 if (g_str_has_suffix (in_plugin->name, "_vdpau")) {
2412 ("Ignoring VDPAU decoder %s. We can't handle this outside of ffmpeg",
2417 if (g_str_has_suffix (in_plugin->name, "_xvmc")) {
2419 ("Ignoring XVMC decoder %s. We can't handle this outside of ffmpeg",
2424 if (strstr (in_plugin->name, "vaapi")) {
2426 ("Ignoring VAAPI decoder %s. We can't handle this outside of ffmpeg",
2431 if (g_str_has_suffix (in_plugin->name, "_qsv")) {
2433 ("Ignoring qsv decoder %s. We can't handle this outside of ffmpeg",
2438 if (g_str_has_suffix (in_plugin->name, "_cuvid")) {
2440 ("Ignoring CUVID decoder %s. We can't handle this outside of ffmpeg",
2445 if (g_str_has_suffix (in_plugin->name, "_v4l2m2m")) {
2447 ("Ignoring V4L2 mem-to-mem decoder %s. We can't handle this outside of ffmpeg",
2452 GST_DEBUG ("Trying plugin %s [%s]", in_plugin->name, in_plugin->long_name);
2454 /* no codecs for which we're GUARANTEED to have better alternatives */
2455 /* MPEG1VIDEO : the mpeg2video decoder is preferred */
2456 /* MP1 : Use MP3 for decoding */
2457 /* MP2 : Use MP3 for decoding */
2458 /* Theora: Use libtheora based theoradec */
2459 /* CDG: use cdgdec */
2460 if (!strcmp (in_plugin->name, "theora") ||
2461 !strcmp (in_plugin->name, "mpeg1video") ||
2462 strstr (in_plugin->name, "crystalhd") != NULL ||
2463 !strcmp (in_plugin->name, "ass") ||
2464 !strcmp (in_plugin->name, "srt") ||
2465 !strcmp (in_plugin->name, "pgssub") ||
2466 !strcmp (in_plugin->name, "dvdsub") ||
2467 !strcmp (in_plugin->name, "dvbsub") ||
2468 !strcmp (in_plugin->name, "cdgraphics")) {
2469 GST_LOG ("Ignoring decoder %s", in_plugin->name);
2473 /* construct the type */
2474 if (!strcmp (in_plugin->name, "hevc")) {
2475 plugin_name = g_strdup ("h265");
2477 plugin_name = g_strdup ((gchar *) in_plugin->name);
2479 g_strdelimit (plugin_name, NULL, '_');
2480 type_name = g_strdup_printf ("avdec_%s", plugin_name);
2481 g_free (plugin_name);
2483 type = g_type_from_name (type_name);
2486 /* create the gtype now */
2488 g_type_register_static (GST_TYPE_VIDEO_DECODER, type_name, &typeinfo,
2490 g_type_set_qdata (type, GST_FFDEC_PARAMS_QDATA, (gpointer) in_plugin);
2493 /* (Ronald) MPEG-4 gets a higher priority because it has been well-
2494 * tested and by far outperforms divxdec/xviddec - so we prefer it.
2495 * msmpeg4v3 same, as it outperforms divxdec for divx3 playback.
2496 * VC1/WMV3 are not working and thus unpreferred for now. */
2497 switch (in_plugin->id) {
2498 case AV_CODEC_ID_MPEG1VIDEO:
2499 case AV_CODEC_ID_MPEG2VIDEO:
2500 case AV_CODEC_ID_MPEG4:
2501 case AV_CODEC_ID_MSMPEG4V3:
2502 case AV_CODEC_ID_H264:
2503 case AV_CODEC_ID_HEVC:
2504 case AV_CODEC_ID_RV10:
2505 case AV_CODEC_ID_RV20:
2506 case AV_CODEC_ID_RV30:
2507 case AV_CODEC_ID_RV40:
2508 rank = GST_RANK_PRIMARY;
2510 /* DVVIDEO: we have a good dv decoder, fast on both ppc as well as x86.
2511 * They say libdv's quality is better though. leave as secondary.
2512 * note: if you change this, see the code in gstdv.c in good/ext/dv.
2514 case AV_CODEC_ID_DVVIDEO:
2515 rank = GST_RANK_SECONDARY;
2518 rank = GST_RANK_MARGINAL;
2521 if (!gst_element_register (plugin, type_name, rank, type)) {
2522 g_warning ("Failed to register %s", type_name);
2530 GST_LOG ("Finished Registering decoders");