2 * gstvaapidecoder_mpeg4.c - MPEG-4 decoder
4 * Copyright (C) 2011-2013 Intel Corporation
5 * Author: Halley Zhao <halley.zhao@intel.com>
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public License
9 * as published by the Free Software Foundation; either version 2.1
10 * of the License, or (at your option) any later version.
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, write to the Free
19 * Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
20 * Boston, MA 02110-1301 USA
24 * SECTION:gstvaapidecoder_mpeg4
25 * @short_description: MPEG-4 decoder, include h263/divx/xvid support
30 #include <gst/base/gstbitreader.h>
31 #include <gst/codecparsers/gstmpeg4parser.h>
32 #include "gstvaapidecoder_mpeg4.h"
33 #include "gstvaapidecoder_objects.h"
34 #include "gstvaapidecoder_priv.h"
35 #include "gstvaapidisplay_priv.h"
36 #include "gstvaapiobject_priv.h"
39 #include "gstvaapidebug.h"
41 #define GST_VAAPI_DECODER_MPEG4_CAST(decoder) \
42 ((GstVaapiDecoderMpeg4 *)(decoder))
44 typedef struct _GstVaapiDecoderMpeg4Private GstVaapiDecoderMpeg4Private;
45 typedef struct _GstVaapiDecoderMpeg4Class GstVaapiDecoderMpeg4Class;
47 struct _GstVaapiDecoderMpeg4Private {
48 GstVaapiProfile profile;
55 GstMpeg4VisualObjectSequence vos_hdr;
56 GstMpeg4VisualObject vo_hdr;
57 GstMpeg4VideoSignalType signal_type;
58 GstMpeg4VideoObjectLayer vol_hdr;
59 GstMpeg4VideoObjectPlane vop_hdr;
60 GstMpeg4VideoPlaneShortHdr svh_hdr;
61 GstMpeg4VideoPacketHdr packet_hdr;
62 GstMpeg4SpriteTrajectory sprite_trajectory;
63 VAIQMatrixBufferMPEG4 iq_matrix;
64 GstVaapiPicture *curr_picture;
65 // forward reference pic
66 GstVaapiPicture *next_picture;
67 // backward reference pic
68 GstVaapiPicture *prev_picture;
71 GstClockTime pts_diff;
73 // anchor sync time base for any picture type,
74 // it is time base of backward reference frame
75 GstClockTime last_sync_time;
76 // time base for recent I/P/S frame,
77 // it is time base of forward reference frame for B frame
78 GstClockTime sync_time;
80 /* last non-b-frame time by resolution */
81 GstClockTime last_non_b_scale_time;
82 GstClockTime non_b_scale_time;
85 // temporal_reference of previous frame of svh
88 guint is_first_field : 1;
89 guint size_changed : 1;
90 guint profile_changed : 1;
91 guint progressive_sequence : 1;
93 guint broken_link : 1;
94 guint calculate_pts_diff : 1;
99 * GstVaapiDecoderMpeg4:
101 * A decoder based on Mpeg4.
103 struct _GstVaapiDecoderMpeg4 {
105 GstVaapiDecoder parent_instance;
106 GstVaapiDecoderMpeg4Private priv;
110 * GstVaapiDecoderMpeg4Class:
112 * A decoder class based on Mpeg4.
114 struct _GstVaapiDecoderMpeg4Class {
116 GstVaapiDecoderClass parent_class;
120 gst_vaapi_decoder_mpeg4_close(GstVaapiDecoderMpeg4 *decoder)
122 GstVaapiDecoderMpeg4Private * const priv = &decoder->priv;
124 gst_vaapi_picture_replace(&priv->curr_picture, NULL);
125 gst_vaapi_picture_replace(&priv->next_picture, NULL);
126 gst_vaapi_picture_replace(&priv->prev_picture, NULL);
130 gst_vaapi_decoder_mpeg4_open(GstVaapiDecoderMpeg4 *decoder)
132 GstVaapiDecoder *const base_decoder = GST_VAAPI_DECODER(decoder);
133 GstVaapiDecoderMpeg4Private * const priv = &decoder->priv;
134 GstCaps *caps = NULL;
135 GstStructure *structure = NULL;
137 gst_vaapi_decoder_mpeg4_close(decoder);
140 caps = gst_vaapi_decoder_get_caps(base_decoder);
142 structure = gst_caps_get_structure(caps, 0);
144 if (gst_structure_has_name(structure, "video/x-h263")) {
146 priv->profile = GST_VAAPI_PROFILE_MPEG4_SIMPLE;
147 priv->prev_t_ref = -1;
155 gst_vaapi_decoder_mpeg4_destroy(GstVaapiDecoder *base_decoder)
157 GstVaapiDecoderMpeg4 * const decoder =
158 GST_VAAPI_DECODER_MPEG4_CAST(base_decoder);
160 gst_vaapi_decoder_mpeg4_close(decoder);
164 gst_vaapi_decoder_mpeg4_create(GstVaapiDecoder *base_decoder)
166 GstVaapiDecoderMpeg4 * const decoder =
167 GST_VAAPI_DECODER_MPEG4_CAST(base_decoder);
168 GstVaapiDecoderMpeg4Private * const priv = &decoder->priv;
170 priv->profile = GST_VAAPI_PROFILE_MPEG4_SIMPLE;
171 priv->seq_pts = GST_CLOCK_TIME_NONE;
172 priv->gop_pts = GST_CLOCK_TIME_NONE;
173 priv->max_pts = GST_CLOCK_TIME_NONE;
174 priv->calculate_pts_diff = TRUE;
175 priv->size_changed = TRUE;
176 priv->profile_changed = TRUE;
181 copy_quant_matrix(guint8 dst[64], const guint8 src[64])
183 memcpy(dst, src, 64);
186 static GstVaapiDecoderStatus
187 ensure_context(GstVaapiDecoderMpeg4 *decoder)
189 GstVaapiDecoderMpeg4Private * const priv = &decoder->priv;
190 GstVaapiProfile profiles[2];
191 GstVaapiEntrypoint entrypoint = GST_VAAPI_ENTRYPOINT_VLD;
192 guint i, n_profiles = 0;
193 gboolean reset_context = FALSE;
195 if (priv->profile_changed) {
196 GST_DEBUG("profile changed");
197 priv->profile_changed = FALSE;
198 reset_context = TRUE;
200 profiles[n_profiles++] = priv->profile;
201 if (priv->profile == GST_VAAPI_PROFILE_MPEG4_SIMPLE)
202 profiles[n_profiles++] = GST_VAAPI_PROFILE_MPEG4_ADVANCED_SIMPLE;
204 for (i = 0; i < n_profiles; i++) {
205 if (gst_vaapi_display_has_decoder(GST_VAAPI_DECODER_DISPLAY(decoder),
206 profiles[i], entrypoint))
210 return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_PROFILE;
211 priv->profile = profiles[i];
214 if (priv->size_changed) {
215 GST_DEBUG("size changed");
216 priv->size_changed = FALSE;
217 reset_context = TRUE;
221 GstVaapiContextInfo info;
223 info.profile = priv->profile;
224 info.entrypoint = entrypoint;
225 info.chroma_type = GST_VAAPI_CHROMA_TYPE_YUV420;
226 info.width = priv->width;
227 info.height = priv->height;
229 reset_context = gst_vaapi_decoder_ensure_context(
230 GST_VAAPI_DECODER(decoder),
234 return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
236 return GST_VAAPI_DECODER_STATUS_SUCCESS;
239 static GstVaapiDecoderStatus
240 ensure_quant_matrix(GstVaapiDecoderMpeg4 *decoder, GstVaapiPicture *picture)
242 GstVaapiDecoderMpeg4Private * const priv = &decoder->priv;
243 VAIQMatrixBufferMPEG4 *iq_matrix;
245 if (!priv->vol_hdr.load_intra_quant_mat && !priv->vol_hdr.load_non_intra_quant_mat) {
246 return GST_VAAPI_DECODER_STATUS_SUCCESS;
249 picture->iq_matrix = GST_VAAPI_IQ_MATRIX_NEW(MPEG4, decoder);
250 if (!picture->iq_matrix) {
251 GST_DEBUG("failed to allocate IQ matrix");
252 return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
254 iq_matrix = picture->iq_matrix->param;
256 if (priv->vol_hdr.load_intra_quant_mat) {
257 iq_matrix->load_intra_quant_mat = 1;
258 copy_quant_matrix(iq_matrix->intra_quant_mat,
259 priv->vol_hdr.intra_quant_mat);
262 iq_matrix->load_intra_quant_mat = 0;
264 if (priv->vol_hdr.load_non_intra_quant_mat) {
265 iq_matrix->load_non_intra_quant_mat = 1;
266 copy_quant_matrix(iq_matrix->non_intra_quant_mat,
267 priv->vol_hdr.non_intra_quant_mat);
270 iq_matrix->load_non_intra_quant_mat = 0;
273 return GST_VAAPI_DECODER_STATUS_SUCCESS;
276 static inline GstVaapiDecoderStatus
277 render_picture(GstVaapiDecoderMpeg4 *decoder, GstVaapiPicture *picture)
279 if (!gst_vaapi_picture_output(picture))
280 return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
281 return GST_VAAPI_DECODER_STATUS_SUCCESS;
284 /* decode_picture() start to decode a frame/picture
285 * decode_current_picture() finishe decoding a frame/picture
286 * (commit buffer to driver for decoding)
288 static GstVaapiDecoderStatus
289 decode_current_picture(GstVaapiDecoderMpeg4 *decoder)
291 GstVaapiDecoderMpeg4Private * const priv = &decoder->priv;
292 GstVaapiPicture * const picture = priv->curr_picture;
293 GstVaapiDecoderStatus status = GST_VAAPI_DECODER_STATUS_SUCCESS;
296 if (!gst_vaapi_picture_decode(picture))
297 status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
298 if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture)) {
299 if ((priv->prev_picture && priv->next_picture) ||
300 (priv->closed_gop && priv->next_picture))
301 status = render_picture(decoder, picture);
303 gst_vaapi_picture_replace(&priv->curr_picture, NULL);
308 static GstVaapiDecoderStatus
309 decode_sequence(GstVaapiDecoderMpeg4 *decoder, const guint8 *buf, guint buf_size)
311 GstVaapiDecoderMpeg4Private * const priv = &decoder->priv;
312 GstMpeg4VisualObjectSequence * const vos_hdr = &priv->vos_hdr;
313 GstVaapiProfile profile;
315 if (gst_mpeg4_parse_visual_object_sequence(vos_hdr, buf, buf_size) != GST_MPEG4_PARSER_OK) {
316 GST_DEBUG("failed to parse sequence header");
317 return GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
320 priv->level = vos_hdr->level;
321 switch (vos_hdr->profile) {
322 case GST_MPEG4_PROFILE_SIMPLE:
323 profile = GST_VAAPI_PROFILE_MPEG4_SIMPLE;
325 case GST_MPEG4_PROFILE_ADVANCED_SIMPLE:
326 case GST_MPEG4_PROFILE_SIMPLE_SCALABLE: /* shared profile with ADVANCED_SIMPLE */
327 profile = GST_VAAPI_PROFILE_MPEG4_ADVANCED_SIMPLE;
330 GST_DEBUG("unsupported profile %d", vos_hdr->profile);
331 return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_PROFILE;
333 if (priv->profile != profile) {
334 priv->profile = profile;
335 priv->profile_changed = TRUE;
337 priv->seq_pts = GST_VAAPI_DECODER_CODEC_FRAME(decoder)->pts;
338 priv->size_changed = TRUE;
340 return GST_VAAPI_DECODER_STATUS_SUCCESS;
343 static GstVaapiDecoderStatus
344 decode_sequence_end(GstVaapiDecoderMpeg4 *decoder)
346 GstVaapiDecoderMpeg4Private * const priv = &decoder->priv;
347 GstVaapiDecoderStatus status;
349 if (priv->curr_picture) {
350 status = decode_current_picture(decoder);
351 if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
353 status = render_picture(decoder, priv->curr_picture);
354 if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
358 if (priv->next_picture) {
359 status = render_picture(decoder, priv->next_picture);
360 if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
363 return GST_VAAPI_DECODER_STATUS_END_OF_STREAM;
366 static GstVaapiDecoderStatus
367 decode_visual_object(GstVaapiDecoderMpeg4 *decoder, const guint8 *buf, guint buf_size)
369 GstVaapiDecoderMpeg4Private * const priv = &decoder->priv;
370 GstMpeg4VisualObject * vo_hdr = &priv->vo_hdr;
371 GstMpeg4VideoSignalType * signal_type = &priv->signal_type;
373 if (gst_mpeg4_parse_visual_object (vo_hdr, signal_type, buf, buf_size) != GST_MPEG4_PARSER_OK) {
374 GST_DEBUG("failed to parse visual object");
375 return GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
378 /* XXX: video_signal_type isn't used for decoding */
379 return GST_VAAPI_DECODER_STATUS_SUCCESS;
382 static GstVaapiDecoderStatus
383 decode_video_object_layer(GstVaapiDecoderMpeg4 *decoder, const guint8 *buf, guint buf_size)
385 GstVaapiDecoder * const base_decoder = GST_VAAPI_DECODER(decoder);
386 GstVaapiDecoderMpeg4Private * const priv = &decoder->priv;
387 GstMpeg4VisualObject * vo_hdr = &priv->vo_hdr;
388 GstMpeg4VideoObjectLayer * vol_hdr = &priv->vol_hdr;
390 if (gst_mpeg4_parse_video_object_layer (vol_hdr, vo_hdr, buf, buf_size) != GST_MPEG4_PARSER_OK) {
391 GST_DEBUG("failed to parse video object layer");
392 return GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
395 priv->width = vol_hdr->width;
396 priv->height = vol_hdr->height;
398 priv->progressive_sequence = !vol_hdr->interlaced;
400 if (vol_hdr->fixed_vop_rate) {
401 priv->fps_n = vol_hdr->vop_time_increment_resolution;
402 priv->fps_d = vol_hdr->fixed_vop_time_increment;
403 gst_vaapi_decoder_set_framerate(base_decoder, priv->fps_n, priv->fps_d);
406 gst_vaapi_decoder_set_pixel_aspect_ratio(base_decoder, priv->vol_hdr.par_width, priv->vol_hdr.par_height);
407 gst_vaapi_decoder_set_picture_size(base_decoder, priv->width, priv->height);
409 return GST_VAAPI_DECODER_STATUS_SUCCESS;
412 static GstVaapiDecoderStatus
413 decode_gop(GstVaapiDecoderMpeg4 *decoder, const guint8 *buf, guint buf_size)
415 GstVaapiDecoderMpeg4Private * const priv = &decoder->priv;
416 GstMpeg4GroupOfVOP gop;
417 GstClockTime gop_time;
420 if (gst_mpeg4_parse_group_of_vop(&gop, buf, buf_size) != GST_MPEG4_PARSER_OK) {
421 GST_DEBUG("failed to parse GOP");
422 return GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
433 priv->closed_gop = gop.closed;
434 priv->broken_link = gop.broken_link;
436 GST_DEBUG("GOP %02u:%02u:%02u (closed_gop %d, broken_link %d)",
437 gop.hours, gop.minutes, gop.seconds,
438 priv->closed_gop, priv->broken_link);
440 gop_time = gop.hours * 3600 + gop.minutes * 60 + gop.seconds;
441 priv->last_sync_time = gop_time;
442 priv->sync_time = gop_time;
444 if (priv->gop_pts != GST_CLOCK_TIME_NONE)
445 priv->pts_diff += gop_time * GST_SECOND - priv->gop_pts;
446 priv->gop_pts = gop_time * GST_SECOND;
447 priv->calculate_pts_diff = TRUE;
448 priv->is_first_field = TRUE;
450 return GST_VAAPI_DECODER_STATUS_SUCCESS;
454 calculate_pts_diff(GstVaapiDecoderMpeg4 *decoder,
455 GstMpeg4VideoObjectLayer *vol_hdr,
456 GstMpeg4VideoObjectPlane *vop_hdr)
458 GstVaapiDecoderMpeg4Private * const priv = &decoder->priv;
459 GstClockTime frame_timestamp;
461 frame_timestamp = GST_VAAPI_DECODER_CODEC_FRAME(decoder)->pts;
462 if (frame_timestamp && frame_timestamp != GST_CLOCK_TIME_NONE) {
463 /* Buffer with timestamp */
464 if (priv->max_pts != GST_CLOCK_TIME_NONE &&
465 frame_timestamp < priv->max_pts) {
466 frame_timestamp = priv->max_pts +
467 gst_util_uint64_scale((vol_hdr->fixed_vop_rate ?
468 vol_hdr->fixed_vop_time_increment : 1),
470 vol_hdr->vop_time_increment_resolution);
473 /* Buffer without timestamp set */
474 if (priv->max_pts == GST_CLOCK_TIME_NONE) /* first buffer */
477 GstClockTime tmp_pts;
478 tmp_pts = priv->pts_diff + priv->gop_pts +
479 vop_hdr->modulo_time_base * GST_SECOND +
480 gst_util_uint64_scale(vop_hdr->time_increment,
482 vol_hdr->vop_time_increment_resolution);
483 if (tmp_pts > priv->max_pts)
484 frame_timestamp = tmp_pts;
486 frame_timestamp = priv->max_pts +
487 gst_util_uint64_scale((vol_hdr->fixed_vop_rate ?
488 vol_hdr->fixed_vop_time_increment : 1),
490 vol_hdr->vop_time_increment_resolution);
494 priv->pts_diff = frame_timestamp -
495 (priv->gop_pts + vop_hdr->modulo_time_base * GST_SECOND +
496 gst_util_uint64_scale(vop_hdr->time_increment, GST_SECOND,
497 vol_hdr->vop_time_increment_resolution));
500 static GstVaapiDecoderStatus
501 decode_picture(GstVaapiDecoderMpeg4 *decoder, const guint8 *buf, guint buf_size)
503 GstMpeg4ParseResult parser_result = GST_MPEG4_PARSER_OK;
504 GstVaapiDecoderMpeg4Private * const priv = &decoder->priv;
505 GstMpeg4VideoObjectPlane * const vop_hdr = &priv->vop_hdr;
506 GstMpeg4VideoObjectLayer * const vol_hdr = &priv->vol_hdr;
507 GstMpeg4SpriteTrajectory * const sprite_trajectory = &priv->sprite_trajectory;
508 GstVaapiPicture *picture;
509 GstVaapiDecoderStatus status;
512 // context depends on priv->width and priv->height, so we move parse_vop a little earlier
514 parser_result = gst_mpeg4_parse_video_plane_short_header(&priv->svh_hdr, buf, buf_size);
518 parser_result = gst_mpeg4_parse_video_object_plane(vop_hdr, sprite_trajectory, vol_hdr, buf, buf_size);
519 /* Need to skip this frame if VOP was not coded */
520 if (GST_MPEG4_PARSER_OK == parser_result && !vop_hdr->coded)
521 return GST_VAAPI_DECODER_STATUS_DROP_FRAME;
524 if (parser_result != GST_MPEG4_PARSER_OK) {
525 GST_DEBUG("failed to parse picture header");
526 return GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
530 priv->width = priv->svh_hdr.vop_width;
531 priv->height = priv->svh_hdr.vop_height;
534 if (!vop_hdr->width && !vop_hdr->height) {
535 vop_hdr->width = vol_hdr->width;
536 vop_hdr->height = vol_hdr->height;
538 priv->width = vop_hdr->width;
539 priv->height = vop_hdr->height;
542 status = ensure_context(decoder);
543 if (status != GST_VAAPI_DECODER_STATUS_SUCCESS) {
544 GST_DEBUG("failed to reset context");
548 if (priv->curr_picture) {
549 status = decode_current_picture(decoder);
550 if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
554 priv->curr_picture = GST_VAAPI_PICTURE_NEW(MPEG4, decoder);
555 if (!priv->curr_picture) {
556 GST_DEBUG("failed to allocate picture");
557 return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
559 picture = priv->curr_picture;
561 status = ensure_quant_matrix(decoder, picture);
562 if (status != GST_VAAPI_DECODER_STATUS_SUCCESS) {
563 GST_DEBUG("failed to reset quantizer matrix");
567 /* 7.6.7 Temporal prediction structure
568 * forward reference frame B B B B B B backward reference frame
570 * nearest I/P/S in the past with vop_coded ==1 |
571 * nearest I/P/S in the future with any vop_coded
572 * fixme, it said that B frame shouldn't use backward reference frame
573 * when backward reference frame coded is 0
576 priv->coding_type = priv->svh_hdr.picture_coding_type;
579 priv->coding_type = priv->vop_hdr.coding_type;
581 switch (priv->coding_type) {
582 case GST_MPEG4_I_VOP:
583 picture->type = GST_VAAPI_PICTURE_TYPE_I;
584 if (priv->is_svh || vop_hdr->coded)
585 GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_REFERENCE);
587 case GST_MPEG4_P_VOP:
588 picture->type = GST_VAAPI_PICTURE_TYPE_P;
589 if (priv->is_svh || vop_hdr->coded)
590 GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_REFERENCE);
592 case GST_MPEG4_B_VOP:
593 picture->type = GST_VAAPI_PICTURE_TYPE_B;
595 case GST_MPEG4_S_VOP:
596 picture->type = GST_VAAPI_PICTURE_TYPE_S;
597 // see 3.175 reference VOP
599 GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_REFERENCE);
602 GST_DEBUG("unsupported picture type %d", priv->coding_type);
603 return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
606 if (!priv->is_svh && !vop_hdr->coded) {
607 status = render_picture(decoder, priv->prev_picture);
612 guint temp_ref = priv->svh_hdr.temporal_reference;
613 if (temp_ref < priv->prev_t_ref) {
616 guint delta_ref = temp_ref - priv->prev_t_ref;
618 pts = priv->sync_time;
619 // see temporal_reference definition in spec, 30000/1001Hz
620 pts += gst_util_uint64_scale(delta_ref, GST_SECOND*1001, 30000);
621 priv->sync_time = pts;
622 priv->prev_t_ref = priv->svh_hdr.temporal_reference;
625 /* Update priv->pts_diff */
626 if (priv->calculate_pts_diff) {
627 calculate_pts_diff(decoder, vol_hdr, vop_hdr);
628 priv->calculate_pts_diff = FALSE;
631 /* Update presentation time, 6.3.5 */
632 if(vop_hdr->coding_type != GST_MPEG4_B_VOP) {
633 // increment basing on decoding order
634 priv->last_sync_time = priv->sync_time;
635 priv->sync_time = priv->last_sync_time + vop_hdr->modulo_time_base;
636 pts = priv->sync_time * GST_SECOND;
637 pts += gst_util_uint64_scale(vop_hdr->time_increment, GST_SECOND, vol_hdr->vop_time_increment_resolution);
638 priv->last_non_b_scale_time = priv->non_b_scale_time;
639 priv->non_b_scale_time = priv->sync_time * vol_hdr->vop_time_increment_resolution + vop_hdr->time_increment;
640 priv->trd = priv->non_b_scale_time - priv->last_non_b_scale_time;
643 // increment basing on display oder
644 pts = (priv->last_sync_time + vop_hdr->modulo_time_base) * GST_SECOND;
645 pts += gst_util_uint64_scale(vop_hdr->time_increment, GST_SECOND, vol_hdr->vop_time_increment_resolution);
646 priv->trb = (priv->last_sync_time + vop_hdr->modulo_time_base) * vol_hdr->vop_time_increment_resolution +
647 vop_hdr->time_increment - priv->last_non_b_scale_time;
650 picture->pts = pts + priv->pts_diff;
651 if (priv->max_pts == GST_CLOCK_TIME_NONE || priv->max_pts < picture->pts)
652 priv->max_pts = picture->pts;
654 /* Update reference pictures */
655 /* XXX: consider priv->vol_hdr.low_delay, consider packed video frames for DivX/XviD */
656 if (GST_VAAPI_PICTURE_IS_REFERENCE(picture)) {
657 if (priv->next_picture)
658 status = render_picture(decoder, priv->next_picture);
659 gst_vaapi_picture_replace(&priv->prev_picture, priv->next_picture);
660 gst_vaapi_picture_replace(&priv->next_picture, picture);
666 get_vop_coding_type(GstVaapiPicture *picture)
668 return picture->type - GST_VAAPI_PICTURE_TYPE_I;
672 fill_picture(GstVaapiDecoderMpeg4 *decoder, GstVaapiPicture *picture)
674 GstVaapiDecoderMpeg4Private * const priv = &decoder->priv;
675 VAPictureParameterBufferMPEG4 * const pic_param = picture->param;
676 GstMpeg4VideoObjectPlane * const vop_hdr = &priv->vop_hdr;
678 /* Fill in VAPictureParameterBufferMPEG4 */
679 pic_param->forward_reference_picture = VA_INVALID_ID;
680 pic_param->backward_reference_picture = VA_INVALID_ID;
682 pic_param->vol_fields.value = 0;
683 pic_param->vop_fields.value = 0;
685 // vol_hdr Parameters
686 pic_param->vol_fields.bits.short_video_header = 1;
687 // does the following vol_hdr parameters matter for short video header?
688 pic_param->vol_fields.bits.chroma_format = 1; // I420, see table 6-15.
689 pic_param->vol_fields.bits.interlaced = 0;
690 pic_param->vol_fields.bits.obmc_disable = 1;
691 pic_param->vol_fields.bits.sprite_enable = 0;
692 pic_param->vol_fields.bits.sprite_warping_accuracy = 0;
693 pic_param->vol_fields.bits.quant_type = 0; //method 1; $7.4.4
694 pic_param->vol_fields.bits.quarter_sample = 0;
695 pic_param->vol_fields.bits.data_partitioned = 0;
696 pic_param->vol_fields.bits.reversible_vlc = 0;
697 pic_param->vol_fields.bits.resync_marker_disable = 1;
698 pic_param->no_of_sprite_warping_points = 0;
699 pic_param->quant_precision = 5;
701 pic_param->vop_width = priv->svh_hdr.vop_width;
702 pic_param->vop_height = priv->svh_hdr.vop_height;
703 pic_param->vop_fields.bits.vop_coding_type = priv->svh_hdr.picture_coding_type;
704 pic_param->vop_time_increment_resolution = priv->vol_hdr.vop_time_increment_resolution;
706 pic_param->num_gobs_in_vop = priv->svh_hdr.num_gobs_in_vop;
707 pic_param->num_macroblocks_in_gob = priv->svh_hdr.num_macroblocks_in_gob;
711 pic_param->vol_fields.bits.short_video_header = 0;
712 pic_param->vol_fields.bits.chroma_format = priv->vol_hdr.chroma_format;
713 pic_param->vol_fields.bits.interlaced = priv->vol_hdr.interlaced;
714 pic_param->vol_fields.bits.obmc_disable = priv->vol_hdr.obmc_disable;
715 pic_param->vol_fields.bits.sprite_enable = priv->vol_hdr.sprite_enable;
716 pic_param->vol_fields.bits.sprite_warping_accuracy = priv->vol_hdr.sprite_warping_accuracy;
717 pic_param->vol_fields.bits.quant_type = priv->vol_hdr.quant_type;
718 pic_param->vol_fields.bits.quarter_sample = priv->vol_hdr.quarter_sample;
719 pic_param->vol_fields.bits.data_partitioned = priv->vol_hdr.data_partitioned;
720 pic_param->vol_fields.bits.reversible_vlc = priv->vol_hdr.reversible_vlc;
721 pic_param->vol_fields.bits.resync_marker_disable = priv->vol_hdr.resync_marker_disable;
722 pic_param->no_of_sprite_warping_points = priv->vol_hdr.no_of_sprite_warping_points;
724 for (i=0; i<3 && i<priv->vol_hdr.no_of_sprite_warping_points ; i++) {
725 pic_param->sprite_trajectory_du[i] = priv->sprite_trajectory.vop_ref_points[i];
726 pic_param->sprite_trajectory_dv[i] = priv->sprite_trajectory.sprite_ref_points[i];
728 pic_param->quant_precision = priv->vol_hdr.quant_precision;
731 pic_param->vop_width = vop_hdr->width;
732 pic_param->vop_height = vop_hdr->height;
733 pic_param->vop_fields.bits.vop_coding_type = vop_hdr->coding_type;
734 pic_param->vop_fields.bits.vop_rounding_type = vop_hdr->rounding_type;
735 pic_param->vop_fields.bits.intra_dc_vlc_thr = vop_hdr->intra_dc_vlc_thr;
736 pic_param->vop_fields.bits.top_field_first = vop_hdr->top_field_first;
737 pic_param->vop_fields.bits.alternate_vertical_scan_flag = vop_hdr->alternate_vertical_scan_flag;
739 pic_param->vop_fcode_forward = vop_hdr->fcode_forward;
740 pic_param->vop_fcode_backward = vop_hdr->fcode_backward;
741 pic_param->vop_time_increment_resolution = priv->vol_hdr.vop_time_increment_resolution;
746 switch (priv->coding_type) {
747 case GST_MPEG4_B_VOP:
748 pic_param->TRB = priv->trb;
749 pic_param->backward_reference_picture = priv->next_picture->surface_id;
750 pic_param->vop_fields.bits.backward_reference_vop_coding_type = get_vop_coding_type(priv->next_picture);
752 case GST_MPEG4_P_VOP:
753 pic_param->TRD = priv->trd;
754 if (priv->prev_picture)
755 pic_param->forward_reference_picture = priv->prev_picture->surface_id;
759 if (priv->vol_hdr.interlaced) {
760 priv->is_first_field ^= 1;
765 static GstVaapiDecoderStatus
767 GstVaapiDecoderMpeg4 *decoder,
770 gboolean has_packet_header
773 GstVaapiDecoderMpeg4Private * const priv = &decoder->priv;
774 GstVaapiPicture * const picture = priv->curr_picture;
775 GstVaapiSlice *slice;
776 VASliceParameterBufferMPEG4 *slice_param;
778 GST_DEBUG("decoder silce: %p, %u bytes)", buf, buf_size);
780 // has_packet_header is ture for the 2+ slice
781 if (!has_packet_header && !fill_picture(decoder, picture))
782 return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
784 slice = GST_VAAPI_SLICE_NEW(MPEG4, decoder, buf, buf_size);
786 GST_DEBUG("failed to allocate slice");
787 return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
789 gst_vaapi_picture_add_slice(picture, slice);
791 /* Fill in VASliceParameterBufferMPEG4 */
792 slice_param = slice->param;
794 slice_param->macroblock_offset = (priv->svh_hdr.size)%8;
795 slice_param->macroblock_number = 0;
796 // the header of first gob_layer is empty (gob_header_empty=1), use vop_quant
797 slice_param->quant_scale = priv->svh_hdr.vop_quant;
800 if (has_packet_header) {
801 slice_param->macroblock_offset = priv->packet_hdr.size % 8;
802 slice_param->macroblock_number = priv->packet_hdr.macroblock_number;
803 slice_param->quant_scale = priv->packet_hdr.quant_scale;
806 slice_param->macroblock_offset = priv->vop_hdr.size % 8;
807 slice_param->macroblock_number = 0;
808 slice_param->quant_scale = priv->vop_hdr.quant;
811 return GST_VAAPI_DECODER_STATUS_SUCCESS;
814 static GstVaapiDecoderStatus
815 decode_packet(GstVaapiDecoderMpeg4 *decoder, GstMpeg4Packet packet)
817 GstVaapiDecoderMpeg4Private * const priv = &decoder->priv;
818 GstMpeg4Packet *tos = &packet;
819 GstVaapiDecoderStatus status;
822 return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
824 // packet.size is the size from current marker to the next.
825 if (tos->type == GST_MPEG4_VISUAL_OBJ_SEQ_START) {
826 status = decode_sequence(decoder, packet.data + packet.offset, packet.size);
828 else if (tos->type == GST_MPEG4_VISUAL_OBJ_SEQ_END) {
829 status = decode_sequence_end(decoder);
831 else if (tos->type == GST_MPEG4_VISUAL_OBJ) {
832 status = decode_visual_object(decoder, packet.data + packet.offset, packet.size);
834 else if (tos->type >= GST_MPEG4_VIDEO_OBJ_FIRST && tos->type <= GST_MPEG4_VIDEO_OBJ_LAST) {
835 GST_WARNING("unexpected marker: (GST_MPEG4_VIDEO_OBJ_FIRST, GST_MPEG4_VIDEO_OBJ_LAST)");
836 status = GST_VAAPI_DECODER_STATUS_SUCCESS;
838 else if (tos->type >= GST_MPEG4_VIDEO_LAYER_FIRST && tos->type <= GST_MPEG4_VIDEO_LAYER_LAST) {
839 status = decode_video_object_layer(decoder, packet.data + packet.offset, packet.size);
841 else if (tos->type == GST_MPEG4_GROUP_OF_VOP) {
842 status = decode_gop(decoder, packet.data + packet.offset, packet.size);
844 else if (tos->type == GST_MPEG4_VIDEO_OBJ_PLANE) {
845 status = decode_picture(decoder, packet.data + packet.offset, packet.size);
846 if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
850 * A resync marker shall only be located immediately before a macroblock
851 * (or video packet header if exists) and aligned with a byte
852 * either start_code or resync_marker are scaned/measured by byte,
853 * while the header itself are parsed/measured in bit
854 * it means: resync_marker(video_packet_header) start from byte boundary,
855 * while MB doesn't start from byte boundary -- it is what 'macroblock_offset'
858 const guint8 *_data = packet.data + packet.offset + priv->vop_hdr.size/8;
859 gint _data_size = packet.size - (priv->vop_hdr.size/8);
860 GstMpeg4Packet video_packet;
862 if (priv->vol_hdr.resync_marker_disable) {
863 status = decode_slice(decoder, _data, _data_size, FALSE);
864 if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
868 // next start_code is required to determine the end of last slice
870 GstMpeg4ParseResult ret = GST_MPEG4_PARSER_OK;
872 gboolean first_slice = TRUE;
873 while (_data_size > 0) {
874 // we can skip user data here
875 ret = gst_mpeg4_parse(&video_packet, TRUE, &priv->vop_hdr, _data, 0, _data_size);
876 if(ret != GST_MPEG4_PARSER_OK) {
881 status = decode_slice(decoder, _data, video_packet.size, FALSE);
885 _data += video_packet.offset;
886 _data_size -= video_packet.offset;
888 ret = gst_mpeg4_parse_video_packet_header (&priv->packet_hdr, &priv->vol_hdr, &priv->vop_hdr, &priv->sprite_trajectory, _data, _data_size);
889 status = decode_slice(decoder,_data + priv->packet_hdr.size/8, video_packet.size - priv->packet_hdr.size/8, TRUE);
892 _data += video_packet.size;
893 _data_size -= video_packet.size;
896 status = decode_current_picture(decoder);
898 else if (tos->type == GST_MPEG4_USER_DATA
899 || tos->type == GST_MPEG4_VIDEO_SESSION_ERR
900 || tos->type == GST_MPEG4_FBA
901 || tos->type == GST_MPEG4_FBA_PLAN
902 || tos->type == GST_MPEG4_MESH
903 || tos->type == GST_MPEG4_MESH_PLAN
904 || tos->type == GST_MPEG4_STILL_TEXTURE_OBJ
905 || tos->type == GST_MPEG4_TEXTURE_SPATIAL
906 || tos->type == GST_MPEG4_TEXTURE_SNR_LAYER
907 || tos->type == GST_MPEG4_TEXTURE_TILE
908 || tos->type == GST_MPEG4_SHAPE_LAYER
909 || tos->type == GST_MPEG4_STUFFING
910 || tos->type == GST_MPEG4_SYSTEM_FIRST
911 || tos->type == GST_MPEG4_SYSTEM_LAST) {
912 GST_WARNING("Ignore marker: %x\n", tos->type);
913 status = GST_VAAPI_DECODER_STATUS_SUCCESS;
916 GST_ERROR("unsupported start code %x\n", tos->type);
917 status = GST_VAAPI_DECODER_STATUS_SUCCESS;
923 static GstVaapiDecoderStatus
924 decode_buffer(GstVaapiDecoderMpeg4 *decoder, const guchar *buf, guint buf_size)
926 GstVaapiDecoderMpeg4Private * const priv = &decoder->priv;
927 GstVaapiDecoderStatus status;
928 GstMpeg4Packet packet;
932 status = decode_picture(decoder, buf, buf_size);
933 if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
936 ofs = priv->svh_hdr.size / 8;
937 status = decode_slice(decoder, buf + ofs, buf_size - ofs, FALSE);
938 if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
944 packet.size = buf_size;
945 packet.type = (GstMpeg4StartCode)packet.data[0];
947 status = decode_packet(decoder, packet);
948 if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
951 return GST_VAAPI_DECODER_STATUS_SUCCESS;
954 static GstVaapiDecoderStatus
955 gst_vaapi_decoder_mpeg4_decode_codec_data(GstVaapiDecoder *base_decoder,
956 const guchar *_buf, guint _buf_size)
958 GstVaapiDecoderMpeg4 * const decoder =
959 GST_VAAPI_DECODER_MPEG4_CAST(base_decoder);
960 GstVaapiDecoderStatus status;
964 // add additional 0x000001b2 to enclose the last header
965 buf_size = _buf_size + 4;
966 buf = malloc(buf_size);
967 memcpy(buf, _buf, buf_size);
971 buf[buf_size-1] = 0xb2;
974 GstMpeg4Packet packet;
975 GstMpeg4ParseResult result = GST_MPEG4_PARSER_OK;
977 while (result == GST_MPEG4_PARSER_OK && pos < buf_size) {
978 result = gst_mpeg4_parse(&packet, FALSE, NULL, buf, pos, buf_size);
979 if (result != GST_MPEG4_PARSER_OK) {
982 status = decode_packet(decoder, packet);
983 if (GST_VAAPI_DECODER_STATUS_SUCCESS == status) {
984 pos = packet.offset + packet.size;
987 GST_WARNING("decode mp4 packet failed when decoding codec data\n");
995 static GstVaapiDecoderStatus
996 ensure_decoder(GstVaapiDecoderMpeg4 *decoder)
998 GstVaapiDecoderMpeg4Private * const priv = &decoder->priv;
999 GstVaapiDecoderStatus status;
1001 if (!priv->is_opened) {
1002 priv->is_opened = gst_vaapi_decoder_mpeg4_open(decoder);
1003 if (!priv->is_opened)
1004 return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CODEC;
1006 status = gst_vaapi_decoder_decode_codec_data(
1007 GST_VAAPI_DECODER_CAST(decoder));
1008 if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
1011 return GST_VAAPI_DECODER_STATUS_SUCCESS;
1014 static GstVaapiDecoderStatus
1015 gst_vaapi_decoder_mpeg4_parse(GstVaapiDecoder *base_decoder,
1016 GstAdapter *adapter, gboolean at_eos, GstVaapiDecoderUnit *unit)
1018 GstVaapiDecoderMpeg4 * const decoder =
1019 GST_VAAPI_DECODER_MPEG4_CAST(base_decoder);
1020 GstVaapiDecoderMpeg4Private * const priv = &decoder->priv;
1021 GstVaapiDecoderStatus status;
1022 GstMpeg4Packet packet;
1023 GstMpeg4ParseResult result;
1025 guint size, buf_size, flags = 0;
1027 status = ensure_decoder(decoder);
1028 if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
1031 size = gst_adapter_available(adapter);
1032 buf = gst_adapter_map(adapter, size);
1034 return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
1037 result = gst_h263_parse(&packet, buf, 0, size);
1039 result = gst_mpeg4_parse(&packet, FALSE, NULL, buf, 0, size);
1040 if (result == GST_MPEG4_PARSER_NO_PACKET_END && at_eos)
1041 packet.size = size - packet.offset;
1042 else if (result == GST_MPEG4_PARSER_ERROR)
1043 return GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
1044 else if (result != GST_MPEG4_PARSER_OK)
1045 return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
1047 buf_size = packet.size;
1048 gst_adapter_flush(adapter, packet.offset);
1049 unit->size = buf_size;
1051 /* Check for start of new picture */
1052 switch (packet.type) {
1053 case GST_MPEG4_VIDEO_SESSION_ERR:
1055 case GST_MPEG4_FBA_PLAN:
1056 case GST_MPEG4_MESH:
1057 case GST_MPEG4_MESH_PLAN:
1058 case GST_MPEG4_STILL_TEXTURE_OBJ:
1059 case GST_MPEG4_TEXTURE_SPATIAL:
1060 case GST_MPEG4_TEXTURE_SNR_LAYER:
1061 case GST_MPEG4_TEXTURE_TILE:
1062 case GST_MPEG4_SHAPE_LAYER:
1063 case GST_MPEG4_STUFFING:
1064 gst_adapter_flush(adapter, packet.size);
1065 return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
1066 case GST_MPEG4_USER_DATA:
1067 flags |= GST_VAAPI_DECODER_UNIT_FLAG_SKIP;
1069 case GST_MPEG4_VISUAL_OBJ_SEQ_END:
1070 flags |= GST_VAAPI_DECODER_UNIT_FLAG_FRAME_END;
1071 flags |= GST_VAAPI_DECODER_UNIT_FLAG_STREAM_END;
1073 case GST_MPEG4_VIDEO_OBJ_PLANE:
1074 flags |= GST_VAAPI_DECODER_UNIT_FLAG_SLICE;
1075 flags |= GST_VAAPI_DECODER_UNIT_FLAG_FRAME_END;
1077 case GST_MPEG4_VISUAL_OBJ_SEQ_START:
1078 case GST_MPEG4_VISUAL_OBJ:
1079 case GST_MPEG4_GROUP_OF_VOP:
1080 flags |= GST_VAAPI_DECODER_UNIT_FLAG_FRAME_START;
1083 if (packet.type >= GST_MPEG4_VIDEO_OBJ_FIRST &&
1084 packet.type <= GST_MPEG4_VIDEO_OBJ_LAST) {
1085 gst_adapter_flush(adapter, packet.size);
1086 return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
1088 if (packet.type >= GST_MPEG4_VIDEO_LAYER_FIRST &&
1089 packet.type <= GST_MPEG4_VIDEO_LAYER_LAST) {
1092 if (packet.type >= GST_MPEG4_SYSTEM_FIRST &&
1093 packet.type <= GST_MPEG4_SYSTEM_LAST) {
1094 flags |= GST_VAAPI_DECODER_UNIT_FLAG_SKIP;
1097 GST_WARNING("unsupported start code (0x%02x)", packet.type);
1098 return GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
1100 GST_VAAPI_DECODER_UNIT_FLAG_SET(unit, flags);
1101 return GST_VAAPI_DECODER_STATUS_SUCCESS;
1104 static GstVaapiDecoderStatus
1105 gst_vaapi_decoder_mpeg4_decode(GstVaapiDecoder *base_decoder,
1106 GstVaapiDecoderUnit *unit)
1108 GstVaapiDecoderMpeg4 * const decoder =
1109 GST_VAAPI_DECODER_MPEG4_CAST(base_decoder);
1110 GstVaapiDecoderStatus status;
1111 GstBuffer * const buffer =
1112 GST_VAAPI_DECODER_CODEC_FRAME(decoder)->input_buffer;
1113 GstMapInfo map_info;
1115 status = ensure_decoder(decoder);
1116 if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
1119 if (!gst_buffer_map(buffer, &map_info, GST_MAP_READ)) {
1120 GST_ERROR("failed to map buffer");
1121 return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
1124 status = decode_buffer(decoder, map_info.data + unit->offset, unit->size);
1125 gst_buffer_unmap(buffer, &map_info);
1126 if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
1128 return GST_VAAPI_DECODER_STATUS_SUCCESS;
1132 gst_vaapi_decoder_mpeg4_class_init(GstVaapiDecoderMpeg4Class *klass)
1134 GstVaapiMiniObjectClass * const object_class =
1135 GST_VAAPI_MINI_OBJECT_CLASS(klass);
1136 GstVaapiDecoderClass * const decoder_class = GST_VAAPI_DECODER_CLASS(klass);
1138 object_class->size = sizeof(GstVaapiDecoderMpeg4);
1139 object_class->finalize = (GDestroyNotify)gst_vaapi_decoder_finalize;
1141 decoder_class->create = gst_vaapi_decoder_mpeg4_create;
1142 decoder_class->destroy = gst_vaapi_decoder_mpeg4_destroy;
1143 decoder_class->parse = gst_vaapi_decoder_mpeg4_parse;
1144 decoder_class->decode = gst_vaapi_decoder_mpeg4_decode;
1146 decoder_class->decode_codec_data =
1147 gst_vaapi_decoder_mpeg4_decode_codec_data;
1150 static inline const GstVaapiDecoderClass *
1151 gst_vaapi_decoder_mpeg4_class(void)
1153 static GstVaapiDecoderMpeg4Class g_class;
1154 static gsize g_class_init = FALSE;
1156 if (g_once_init_enter(&g_class_init)) {
1157 gst_vaapi_decoder_mpeg4_class_init(&g_class);
1158 g_once_init_leave(&g_class_init, TRUE);
1160 return GST_VAAPI_DECODER_CLASS(&g_class);
1164 * gst_vaapi_decoder_mpeg4_new:
1165 * @display: a #GstVaapiDisplay
1166 * @caps: a #GstCaps holding codec information
1168 * Creates a new #GstVaapiDecoder for MPEG-2 decoding. The @caps can
1169 * hold extra information like codec-data and pictured coded size.
1171 * Return value: the newly allocated #GstVaapiDecoder object
1174 gst_vaapi_decoder_mpeg4_new(GstVaapiDisplay *display, GstCaps *caps)
1176 return gst_vaapi_decoder_new(gst_vaapi_decoder_mpeg4_class(),