2 * gstvaapidecoder_mpeg4.c - MPEG-4 decoder
4 * Copyright (C) 2011 Intel Corporation
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public License
8 * as published by the Free Software Foundation; either version 2.1
9 * of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free
18 * Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
19 * Boston, MA 02110-1301 USA
23 * SECTION:gstvaapidecoder_mpeg4
24 * @short_description: MPEG-4 decoder, include h263/divx/xvid support
30 #include <gst/base/gstbitreader.h>
31 #include <gst/codecparsers/gstmpeg4parser.h>
32 #include "gstvaapidecoder_mpeg4.h"
33 #include "gstvaapidecoder_objects.h"
34 #include "gstvaapidecoder_priv.h"
35 #include "gstvaapidisplay_priv.h"
36 #include "gstvaapiobject_priv.h"
39 #include "gstvaapidebug.h"
41 G_DEFINE_TYPE(GstVaapiDecoderMpeg4,
42 gst_vaapi_decoder_mpeg4,
43 GST_VAAPI_TYPE_DECODER);
45 #define GST_VAAPI_DECODER_MPEG4_GET_PRIVATE(obj) \
46 (G_TYPE_INSTANCE_GET_PRIVATE((obj), \
47 GST_VAAPI_TYPE_DECODER_MPEG4, \
48 GstVaapiDecoderMpeg4Private))
50 struct _GstVaapiDecoderMpeg4Private {
51 GstVaapiProfile profile;
58 GstMpeg4VisualObjectSequence vos_hdr;
59 GstMpeg4VisualObject vo_hdr;
60 GstMpeg4VideoSignalType signal_type;
61 GstMpeg4VideoObjectLayer vol_hdr;
62 GstMpeg4VideoObjectPlane vop_hdr;
63 GstMpeg4VideoPlaneShortHdr svh_hdr;
64 GstMpeg4VideoPacketHdr packet_hdr;
65 GstMpeg4SpriteTrajectory sprite_trajectory;
66 VAIQMatrixBufferMPEG4 iq_matrix;
67 GstVaapiPicture *curr_picture;
68 // forward reference pic
69 GstVaapiPicture *next_picture;
70 // backward reference pic
71 GstVaapiPicture *prev_picture;
73 GstBuffer *sub_buffer;
76 GstClockTime pts_diff;
78 // anchor sync time base for any picture type,
79 // it is time base of backward reference frame
80 GstClockTime last_sync_time;
81 // time base for recent I/P/S frame,
82 // it is time base of forward reference frame for B frame
83 GstClockTime sync_time;
85 /* last non-b-frame time by resolution */
86 GstClockTime last_non_b_scale_time;
87 GstClockTime non_b_scale_time;
90 // temporal_reference of previous frame of svh
92 guint is_constructed : 1;
94 guint is_first_field : 1;
95 guint size_changed : 1;
96 guint profile_changed : 1;
97 guint progressive_sequence : 1;
99 guint broken_link : 1;
100 guint calculate_pts_diff : 1;
105 gst_vaapi_decoder_mpeg4_clear_buffer(GstVaapiDecoder *base)
107 GstVaapiDecoderMpeg4* const decoder = GST_VAAPI_DECODER_MPEG4(base);
108 GstVaapiDecoderMpeg4Private * const priv = decoder->priv;
110 priv->seq_pts = GST_CLOCK_TIME_NONE;
111 priv->gop_pts = GST_CLOCK_TIME_NONE;
112 priv->max_pts = GST_CLOCK_TIME_NONE;
114 priv->prev_t_ref = -1;
115 priv->calculate_pts_diff = TRUE;
116 priv->is_first_field = FALSE;
117 priv->closed_gop = FALSE;
118 priv->broken_link = FALSE;
119 priv->last_non_b_scale_time = 0;
120 priv->non_b_scale_time = 0;
124 gst_vaapi_picture_replace(&priv->curr_picture, NULL);
125 gst_vaapi_picture_replace(&priv->next_picture, NULL);
126 gst_vaapi_picture_replace(&priv->prev_picture, NULL);
128 if (priv->sub_buffer) {
129 gst_buffer_unref(priv->sub_buffer);
130 priv->sub_buffer = NULL;
134 gst_adapter_clear(priv->adapter);
139 gst_vaapi_decoder_mpeg4_close(GstVaapiDecoderMpeg4 *decoder)
141 GstVaapiDecoderMpeg4Private * const priv = decoder->priv;
143 gst_vaapi_decoder_mpeg4_clear_buffer(GST_VAAPI_DECODER_CAST(decoder));
146 g_object_unref(priv->adapter);
147 priv->adapter = NULL;
152 gst_vaapi_decoder_mpeg4_open(GstVaapiDecoderMpeg4 *decoder, GstBuffer *buffer)
154 GstVaapiDecoder *const base_decoder = GST_VAAPI_DECODER(decoder);
155 GstVaapiDecoderMpeg4Private * const priv = decoder->priv;
156 GstCaps *caps = NULL;
157 GstStructure *structure = NULL;
159 gst_vaapi_decoder_mpeg4_close(decoder);
161 priv->adapter = gst_adapter_new();
166 caps = gst_vaapi_decoder_get_caps(base_decoder);
168 structure = gst_caps_get_structure(caps, 0);
170 if (gst_structure_has_name(structure, "video/x-h263")) {
172 priv->profile = GST_VAAPI_PROFILE_MPEG4_SIMPLE;
173 priv->prev_t_ref = -1;
181 gst_vaapi_decoder_mpeg4_destroy(GstVaapiDecoderMpeg4 *decoder)
183 gst_vaapi_decoder_mpeg4_close(decoder);
187 gst_vaapi_decoder_mpeg4_create(GstVaapiDecoderMpeg4 *decoder)
189 if (!GST_VAAPI_DECODER_CODEC(decoder))
195 copy_quant_matrix(guint8 dst[64], const guint8 src[64])
197 memcpy(dst, src, 64);
200 static GstVaapiDecoderStatus
201 ensure_context(GstVaapiDecoderMpeg4 *decoder)
203 GstVaapiDecoderMpeg4Private * const priv = decoder->priv;
204 GstVaapiProfile profiles[2];
205 GstVaapiEntrypoint entrypoint = GST_VAAPI_ENTRYPOINT_VLD;
206 guint i, n_profiles = 0;
207 gboolean reset_context = FALSE;
209 if (priv->profile_changed) {
210 GST_DEBUG("profile changed");
211 priv->profile_changed = FALSE;
212 reset_context = TRUE;
214 profiles[n_profiles++] = priv->profile;
215 if (priv->profile == GST_VAAPI_PROFILE_MPEG4_SIMPLE)
216 profiles[n_profiles++] = GST_VAAPI_PROFILE_MPEG4_ADVANCED_SIMPLE;
218 for (i = 0; i < n_profiles; i++) {
219 if (gst_vaapi_display_has_decoder(GST_VAAPI_DECODER_DISPLAY(decoder),
220 profiles[i], entrypoint))
224 return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_PROFILE;
225 priv->profile = profiles[i];
228 if (priv->size_changed) {
229 GST_DEBUG("size changed");
230 priv->size_changed = FALSE;
231 reset_context = TRUE;
235 reset_context = gst_vaapi_decoder_ensure_context(
236 GST_VAAPI_DECODER(decoder),
241 GST_DECODER_DEFAULT_SURFACES_COUNT
244 return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
246 return GST_VAAPI_DECODER_STATUS_SUCCESS;
249 static GstVaapiDecoderStatus
250 ensure_quant_matrix(GstVaapiDecoderMpeg4 *decoder, GstVaapiPicture *picture)
252 GstVaapiDecoderMpeg4Private * const priv = decoder->priv;
253 VAIQMatrixBufferMPEG4 *iq_matrix;
255 if (!priv->vol_hdr.load_intra_quant_mat && !priv->vol_hdr.load_non_intra_quant_mat) {
256 return GST_VAAPI_DECODER_STATUS_SUCCESS;
259 picture->iq_matrix = GST_VAAPI_IQ_MATRIX_NEW(MPEG4, decoder);
260 if (!picture->iq_matrix) {
261 GST_DEBUG("failed to allocate IQ matrix");
262 return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
264 iq_matrix = picture->iq_matrix->param;
266 if (priv->vol_hdr.load_intra_quant_mat) {
267 iq_matrix->load_intra_quant_mat = 1;
268 copy_quant_matrix(iq_matrix->intra_quant_mat,
269 priv->vol_hdr.intra_quant_mat);
272 iq_matrix->load_intra_quant_mat = 0;
274 if (priv->vol_hdr.load_non_intra_quant_mat) {
275 iq_matrix->load_non_intra_quant_mat = 1;
276 copy_quant_matrix(iq_matrix->non_intra_quant_mat,
277 priv->vol_hdr.non_intra_quant_mat);
280 iq_matrix->load_non_intra_quant_mat = 0;
283 return GST_VAAPI_DECODER_STATUS_SUCCESS;
286 static inline GstVaapiDecoderStatus
287 render_picture(GstVaapiDecoderMpeg4 *decoder, GstVaapiPicture *picture)
289 if (!gst_vaapi_picture_output(picture))
290 return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
291 return GST_VAAPI_DECODER_STATUS_SUCCESS;
294 /* decode_picture() start to decode a frame/picture
295 * decode_current_picture() finishe decoding a frame/picture
296 * (commit buffer to driver for decoding)
298 static GstVaapiDecoderStatus
299 decode_current_picture(GstVaapiDecoderMpeg4 *decoder)
301 GstVaapiDecoderMpeg4Private * const priv = decoder->priv;
302 GstVaapiPicture * const picture = priv->curr_picture;
303 GstVaapiDecoderStatus status = GST_VAAPI_DECODER_STATUS_SUCCESS;
306 if (!gst_vaapi_picture_decode(picture))
307 status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
308 if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture)) {
309 if ((priv->prev_picture && priv->next_picture) ||
310 (priv->closed_gop && priv->next_picture))
311 status = render_picture(decoder, picture);
313 gst_vaapi_picture_replace(&priv->curr_picture, NULL);
318 static GstVaapiDecoderStatus
319 decode_sequence(GstVaapiDecoderMpeg4 *decoder, const guint8 *buf, guint buf_size)
321 GstVaapiDecoderMpeg4Private * const priv = decoder->priv;
322 GstMpeg4VisualObjectSequence * const vos_hdr = &priv->vos_hdr;
323 GstVaapiProfile profile;
325 if (gst_mpeg4_parse_visual_object_sequence(vos_hdr, buf, buf_size) != GST_MPEG4_PARSER_OK) {
326 GST_DEBUG("failed to parse sequence header");
327 return GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
330 priv->level = vos_hdr->level;
331 switch (vos_hdr->profile) {
332 case GST_MPEG4_PROFILE_SIMPLE:
333 profile = GST_VAAPI_PROFILE_MPEG4_SIMPLE;
335 case GST_MPEG4_PROFILE_ADVANCED_SIMPLE:
336 case GST_MPEG4_PROFILE_SIMPLE_SCALABLE: /* shared profile with ADVANCED_SIMPLE */
337 profile = GST_VAAPI_PROFILE_MPEG4_ADVANCED_SIMPLE;
340 GST_DEBUG("unsupported profile %d", profile);
341 return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_PROFILE;
343 if (priv->profile != profile) {
344 priv->profile = profile;
345 priv->profile_changed = TRUE;
347 priv->seq_pts = gst_adapter_prev_timestamp(priv->adapter, NULL);
348 priv->size_changed = TRUE;
350 return GST_VAAPI_DECODER_STATUS_SUCCESS;
353 static GstVaapiDecoderStatus
354 decode_sequence_end(GstVaapiDecoderMpeg4 *decoder)
356 GstVaapiDecoderMpeg4Private * const priv = decoder->priv;
357 GstVaapiDecoderStatus status;
359 if (priv->curr_picture) {
360 status = decode_current_picture(decoder);
361 if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
363 status = render_picture(decoder, priv->curr_picture);
364 if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
368 if (priv->next_picture) {
369 status = render_picture(decoder, priv->next_picture);
370 if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
373 return GST_VAAPI_DECODER_STATUS_END_OF_STREAM;
376 static GstVaapiDecoderStatus
377 decode_visual_object(GstVaapiDecoderMpeg4 *decoder, const guint8 *buf, guint buf_size)
379 GstVaapiDecoderMpeg4Private * const priv = decoder->priv;
380 GstMpeg4VisualObject * vo_hdr = &priv->vo_hdr;
381 GstMpeg4VideoSignalType * signal_type = &priv->signal_type;
383 if (gst_mpeg4_parse_visual_object (vo_hdr, signal_type, buf, buf_size) != GST_MPEG4_PARSER_OK) {
384 GST_DEBUG("failed to parse visual object");
385 return GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
388 /* XXX: video_signal_type isn't used for decoding */
389 return GST_VAAPI_DECODER_STATUS_SUCCESS;
392 static GstVaapiDecoderStatus
393 decode_video_object_layer(GstVaapiDecoderMpeg4 *decoder, const guint8 *buf, guint buf_size)
395 GstVaapiDecoder * const base_decoder = GST_VAAPI_DECODER(decoder);
396 GstVaapiDecoderMpeg4Private * const priv = decoder->priv;
397 GstMpeg4VisualObject * vo_hdr = &priv->vo_hdr;
398 GstMpeg4VideoObjectLayer * vol_hdr = &priv->vol_hdr;
400 if (gst_mpeg4_parse_video_object_layer (vol_hdr, vo_hdr, buf, buf_size) != GST_MPEG4_PARSER_OK) {
401 GST_DEBUG("failed to parse video object layer");
402 return GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
405 priv->width = vol_hdr->width;
406 priv->height = vol_hdr->height;
408 priv->progressive_sequence = !vol_hdr->interlaced;
410 if (vol_hdr->fixed_vop_rate) {
411 priv->fps_n = vol_hdr->vop_time_increment_resolution;
412 priv->fps_d = vol_hdr->fixed_vop_time_increment;
413 gst_vaapi_decoder_set_framerate(base_decoder, priv->fps_n, priv->fps_d);
416 gst_vaapi_decoder_set_pixel_aspect_ratio(base_decoder, priv->vol_hdr.par_width, priv->vol_hdr.par_height);
417 gst_vaapi_decoder_set_picture_size(base_decoder, priv->width, priv->height);
419 return GST_VAAPI_DECODER_STATUS_SUCCESS;
422 static GstVaapiDecoderStatus
423 decode_gop(GstVaapiDecoderMpeg4 *decoder, const guint8 *buf, guint buf_size)
425 GstVaapiDecoderMpeg4Private * const priv = decoder->priv;
426 GstMpeg4GroupOfVOP gop;
427 GstClockTime gop_time;
430 if (gst_mpeg4_parse_group_of_vop(&gop, buf, buf_size) != GST_MPEG4_PARSER_OK) {
431 GST_DEBUG("failed to parse GOP");
432 return GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
443 priv->closed_gop = gop.closed;
444 priv->broken_link = gop.broken_link;
446 GST_DEBUG("GOP %02u:%02u:%02u (closed_gop %d, broken_link %d)",
447 gop.hours, gop.minutes, gop.seconds,
448 priv->closed_gop, priv->broken_link);
450 gop_time = gop.hours * 3600 + gop.minutes * 60 + gop.seconds;
451 priv->last_sync_time = gop_time;
452 priv->sync_time = gop_time;
454 if (priv->gop_pts != GST_CLOCK_TIME_NONE)
455 priv->pts_diff += gop_time * GST_SECOND - priv->gop_pts;
456 priv->gop_pts = gop_time * GST_SECOND;
457 priv->calculate_pts_diff = TRUE;
458 priv->is_first_field = TRUE;
460 return GST_VAAPI_DECODER_STATUS_SUCCESS;
464 calculate_pts_diff(GstVaapiDecoderMpeg4 *decoder,
465 GstMpeg4VideoObjectLayer *vol_hdr,
466 GstMpeg4VideoObjectPlane *vop_hdr)
468 GstVaapiDecoderMpeg4Private * const priv = decoder->priv;
469 GstClockTime frame_timestamp;
471 frame_timestamp = gst_adapter_prev_timestamp(priv->adapter, NULL);
472 if (frame_timestamp && frame_timestamp != GST_CLOCK_TIME_NONE) {
473 /* Buffer with timestamp */
474 if (priv->max_pts != GST_CLOCK_TIME_NONE &&
475 frame_timestamp < priv->max_pts) {
476 frame_timestamp = priv->max_pts +
477 gst_util_uint64_scale((vol_hdr->fixed_vop_rate ?
478 vol_hdr->fixed_vop_time_increment : 1),
480 vol_hdr->vop_time_increment_resolution);
483 /* Buffer without timestamp set */
484 if (priv->max_pts == GST_CLOCK_TIME_NONE) /* first buffer */
487 GstClockTime tmp_pts;
488 tmp_pts = priv->pts_diff + priv->gop_pts +
489 vop_hdr->modulo_time_base * GST_SECOND +
490 gst_util_uint64_scale(vop_hdr->time_increment,
492 vol_hdr->vop_time_increment_resolution);
493 if (tmp_pts > priv->max_pts)
494 frame_timestamp = tmp_pts;
496 frame_timestamp = priv->max_pts +
497 gst_util_uint64_scale((vol_hdr->fixed_vop_rate ?
498 vol_hdr->fixed_vop_time_increment : 1),
500 vol_hdr->vop_time_increment_resolution);
504 priv->pts_diff = frame_timestamp -
505 (priv->gop_pts + vop_hdr->modulo_time_base * GST_SECOND +
506 gst_util_uint64_scale(vop_hdr->time_increment, GST_SECOND,
507 vol_hdr->vop_time_increment_resolution));
510 static GstVaapiDecoderStatus
511 decode_picture(GstVaapiDecoderMpeg4 *decoder, const guint8 *buf, guint buf_size)
513 GstMpeg4ParseResult parser_result = GST_MPEG4_PARSER_OK;
514 GstVaapiDecoderMpeg4Private * const priv = decoder->priv;
515 GstMpeg4VideoObjectPlane * const vop_hdr = &priv->vop_hdr;
516 GstMpeg4VideoObjectLayer * const vol_hdr = &priv->vol_hdr;
517 GstMpeg4SpriteTrajectory * const sprite_trajectory = &priv->sprite_trajectory;
518 GstVaapiPicture *picture;
519 GstVaapiDecoderStatus status;
522 // context depends on priv->width and priv->height, so we move parse_vop a little earlier
524 parser_result = gst_mpeg4_parse_video_plane_short_header(&priv->svh_hdr, buf, buf_size);
528 parser_result = gst_mpeg4_parse_video_object_plane(vop_hdr, sprite_trajectory, vol_hdr, buf, buf_size);
529 /* Need to skip this frame if VOP was not coded */
530 if (GST_MPEG4_PARSER_OK == parser_result && !vop_hdr->coded)
531 return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
534 if (parser_result != GST_MPEG4_PARSER_OK) {
535 GST_DEBUG("failed to parse picture header");
536 return GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
540 priv->width = priv->svh_hdr.vop_width;
541 priv->height = priv->svh_hdr.vop_height;
544 if (!vop_hdr->width && !vop_hdr->height) {
545 vop_hdr->width = vol_hdr->width;
546 vop_hdr->height = vol_hdr->height;
548 priv->width = vop_hdr->width;
549 priv->height = vop_hdr->height;
552 status = ensure_context(decoder);
553 if (status != GST_VAAPI_DECODER_STATUS_SUCCESS) {
554 GST_DEBUG("failed to reset context");
558 if (priv->curr_picture) {
559 status = decode_current_picture(decoder);
560 if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
564 priv->curr_picture = GST_VAAPI_PICTURE_NEW(MPEG4, decoder);
565 if (!priv->curr_picture) {
566 GST_DEBUG("failed to allocate picture");
567 return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
569 picture = priv->curr_picture;
571 status = ensure_quant_matrix(decoder, picture);
572 if (status != GST_VAAPI_DECODER_STATUS_SUCCESS) {
573 GST_DEBUG("failed to reset quantizer matrix");
577 /* 7.6.7 Temporal prediction structure
578 * forward reference frame B B B B B B backward reference frame
580 * nearest I/P/S in the past with vop_coded ==1 |
581 * nearest I/P/S in the future with any vop_coded
582 * fixme, it said that B frame shouldn't use backward reference frame
583 * when backward reference frame coded is 0
586 priv->coding_type = priv->svh_hdr.picture_coding_type;
589 priv->coding_type = priv->vop_hdr.coding_type;
591 switch (priv->coding_type) {
592 case GST_MPEG4_I_VOP:
593 picture->type = GST_VAAPI_PICTURE_TYPE_I;
594 if (priv->is_svh || vop_hdr->coded)
595 GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_REFERENCE);
597 case GST_MPEG4_P_VOP:
598 picture->type = GST_VAAPI_PICTURE_TYPE_P;
599 if (priv->is_svh || vop_hdr->coded)
600 GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_REFERENCE);
602 case GST_MPEG4_B_VOP:
603 picture->type = GST_VAAPI_PICTURE_TYPE_B;
605 case GST_MPEG4_S_VOP:
606 picture->type = GST_VAAPI_PICTURE_TYPE_S;
607 // see 3.175 reference VOP
609 GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_REFERENCE);
612 GST_DEBUG("unsupported picture type %d", priv->coding_type);
613 return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
616 if (!priv->is_svh && !vop_hdr->coded) {
617 status = render_picture(decoder, priv->prev_picture);
622 if (priv->calculate_pts_diff) {
623 priv->prev_t_ref = priv->svh_hdr.temporal_reference;
624 priv->sync_time = gst_adapter_prev_timestamp(priv->adapter, NULL);
625 if (priv->sync_time == GST_CLOCK_TIME_NONE)
627 priv->calculate_pts_diff = FALSE;
630 pts = gst_adapter_prev_timestamp(priv->adapter, NULL);
631 if (pts == GST_CLOCK_TIME_NONE) { /* Buffer without timestamp set*/
632 guint temp_ref = priv->svh_hdr.temporal_reference;
633 if (temp_ref < priv->prev_t_ref) {
636 guint delta_ref = temp_ref - priv->prev_t_ref;
638 pts = priv->sync_time;
639 // see temporal_reference definition in spec, 30000/1001Hz
640 pts += gst_util_uint64_scale(delta_ref, GST_SECOND*1001, 30000);
642 priv->sync_time = pts;
643 priv->prev_t_ref = priv->svh_hdr.temporal_reference;
646 /* Update priv->pts_diff */
647 if (priv->calculate_pts_diff) {
648 calculate_pts_diff(decoder, vol_hdr, vop_hdr);
649 priv->calculate_pts_diff = FALSE;
652 /* Update presentation time, 6.3.5 */
653 if(vop_hdr->coding_type != GST_MPEG4_B_VOP) {
654 // increment basing on decoding order
655 priv->last_sync_time = priv->sync_time;
656 priv->sync_time = priv->last_sync_time + vop_hdr->modulo_time_base;
657 pts = priv->sync_time * GST_SECOND;
658 pts += gst_util_uint64_scale(vop_hdr->time_increment, GST_SECOND, vol_hdr->vop_time_increment_resolution);
659 priv->last_non_b_scale_time = priv->non_b_scale_time;
660 priv->non_b_scale_time = priv->sync_time * vol_hdr->vop_time_increment_resolution + vop_hdr->time_increment;
661 priv->trd = priv->non_b_scale_time - priv->last_non_b_scale_time;
664 // increment basing on display oder
665 pts = (priv->last_sync_time + vop_hdr->modulo_time_base) * GST_SECOND;
666 pts += gst_util_uint64_scale(vop_hdr->time_increment, GST_SECOND, vol_hdr->vop_time_increment_resolution);
667 priv->trb = (priv->last_sync_time + vop_hdr->modulo_time_base) * vol_hdr->vop_time_increment_resolution +
668 vop_hdr->time_increment - priv->last_non_b_scale_time;
671 picture->pts = pts + priv->pts_diff;
672 if (priv->max_pts == GST_CLOCK_TIME_NONE || priv->max_pts < picture->pts)
673 priv->max_pts = picture->pts;
675 /* Update reference pictures */
676 /* XXX: consider priv->vol_hdr.low_delay, consider packed video frames for DivX/XviD */
677 if (GST_VAAPI_PICTURE_IS_REFERENCE(picture)) {
678 if (priv->next_picture)
679 status = render_picture(decoder, priv->next_picture);
680 gst_vaapi_picture_replace(&priv->prev_picture, priv->next_picture);
681 gst_vaapi_picture_replace(&priv->next_picture, picture);
687 get_vop_coding_type(GstVaapiPicture *picture)
689 return picture->type - GST_VAAPI_PICTURE_TYPE_I;
693 fill_picture(GstVaapiDecoderMpeg4 *decoder, GstVaapiPicture *picture)
695 GstVaapiDecoderMpeg4Private * const priv = decoder->priv;
696 VAPictureParameterBufferMPEG4 * const pic_param = picture->param;
697 GstMpeg4VideoObjectPlane * const vop_hdr = &priv->vop_hdr;
699 /* Fill in VAPictureParameterBufferMPEG4 */
700 pic_param->forward_reference_picture = VA_INVALID_ID;
701 pic_param->backward_reference_picture = VA_INVALID_ID;
703 pic_param->vol_fields.value = 0;
704 pic_param->vop_fields.value = 0;
706 // vol_hdr Parameters
707 pic_param->vol_fields.bits.short_video_header = 1;
708 // does the following vol_hdr parameters matter for short video header?
709 pic_param->vol_fields.bits.chroma_format = 1; // I420, see table 6-15.
710 pic_param->vol_fields.bits.interlaced = 0;
711 pic_param->vol_fields.bits.obmc_disable = 1;
712 pic_param->vol_fields.bits.sprite_enable = 0;
713 pic_param->vol_fields.bits.sprite_warping_accuracy = 0;
714 pic_param->vol_fields.bits.quant_type = 0; //method 1; $7.4.4
715 pic_param->vol_fields.bits.quarter_sample = 0;
716 pic_param->vol_fields.bits.data_partitioned = 0;
717 pic_param->vol_fields.bits.reversible_vlc = 0;
718 pic_param->vol_fields.bits.resync_marker_disable = 1;
719 pic_param->no_of_sprite_warping_points = 0;
720 pic_param->quant_precision = 5;
722 pic_param->vop_width = priv->svh_hdr.vop_width;
723 pic_param->vop_height = priv->svh_hdr.vop_height;
724 pic_param->vop_fields.bits.vop_coding_type = priv->svh_hdr.picture_coding_type;
725 pic_param->vop_time_increment_resolution = priv->vol_hdr.vop_time_increment_resolution;
727 pic_param->num_gobs_in_vop = priv->svh_hdr.num_gobs_in_vop;
728 pic_param->num_macroblocks_in_gob = priv->svh_hdr.num_macroblocks_in_gob;
732 pic_param->vol_fields.bits.short_video_header = 0;
733 pic_param->vol_fields.bits.chroma_format = priv->vol_hdr.chroma_format;
734 pic_param->vol_fields.bits.interlaced = priv->vol_hdr.interlaced;
735 pic_param->vol_fields.bits.obmc_disable = priv->vol_hdr.obmc_disable;
736 pic_param->vol_fields.bits.sprite_enable = priv->vol_hdr.sprite_enable;
737 pic_param->vol_fields.bits.sprite_warping_accuracy = priv->vol_hdr.sprite_warping_accuracy;
738 pic_param->vol_fields.bits.quant_type = priv->vol_hdr.quant_type;
739 pic_param->vol_fields.bits.quarter_sample = priv->vol_hdr.quarter_sample;
740 pic_param->vol_fields.bits.data_partitioned = priv->vol_hdr.data_partitioned;
741 pic_param->vol_fields.bits.reversible_vlc = priv->vol_hdr.reversible_vlc;
742 pic_param->vol_fields.bits.resync_marker_disable = priv->vol_hdr.resync_marker_disable;
743 pic_param->no_of_sprite_warping_points = priv->vol_hdr.no_of_sprite_warping_points;
745 for (i=0; i<3 && i<priv->vol_hdr.no_of_sprite_warping_points ; i++) {
746 pic_param->sprite_trajectory_du[i] = priv->sprite_trajectory.vop_ref_points[i];
747 pic_param->sprite_trajectory_dv[i] = priv->sprite_trajectory.sprite_ref_points[i];
749 pic_param->quant_precision = priv->vol_hdr.quant_precision;
752 pic_param->vop_width = vop_hdr->width;
753 pic_param->vop_height = vop_hdr->height;
754 pic_param->vop_fields.bits.vop_coding_type = vop_hdr->coding_type;
755 pic_param->vop_fields.bits.vop_rounding_type = vop_hdr->rounding_type;
756 pic_param->vop_fields.bits.intra_dc_vlc_thr = vop_hdr->intra_dc_vlc_thr;
757 pic_param->vop_fields.bits.top_field_first = vop_hdr->top_field_first;
758 pic_param->vop_fields.bits.alternate_vertical_scan_flag = vop_hdr->alternate_vertical_scan_flag;
760 pic_param->vop_fcode_forward = vop_hdr->fcode_forward;
761 pic_param->vop_fcode_backward = vop_hdr->fcode_backward;
762 pic_param->vop_time_increment_resolution = priv->vol_hdr.vop_time_increment_resolution;
767 switch (priv->coding_type) {
768 case GST_MPEG4_B_VOP:
769 pic_param->TRB = priv->trb;
770 pic_param->backward_reference_picture = priv->next_picture->surface_id;
771 pic_param->vop_fields.bits.backward_reference_vop_coding_type = get_vop_coding_type(priv->next_picture);
773 case GST_MPEG4_P_VOP:
774 pic_param->TRD = priv->trd;
775 if (priv->prev_picture)
776 pic_param->forward_reference_picture = priv->prev_picture->surface_id;
780 if (priv->vol_hdr.interlaced) {
781 priv->is_first_field ^= 1;
786 static GstVaapiDecoderStatus
788 GstVaapiDecoderMpeg4 *decoder,
791 gboolean has_packet_header
794 GstVaapiDecoderMpeg4Private * const priv = decoder->priv;
795 GstVaapiPicture * const picture = priv->curr_picture;
796 GstVaapiSlice *slice;
797 VASliceParameterBufferMPEG4 *slice_param;
799 GST_DEBUG("decoder silce: %p, %u bytes)", buf, buf_size);
801 // has_packet_header is ture for the 2+ slice
802 if (!has_packet_header && !fill_picture(decoder, picture))
803 return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
805 slice = GST_VAAPI_SLICE_NEW(MPEG4, decoder, buf, buf_size);
807 GST_DEBUG("failed to allocate slice");
808 return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
810 gst_vaapi_picture_add_slice(picture, slice);
812 /* Fill in VASliceParameterBufferMPEG4 */
813 slice_param = slice->param;
815 slice_param->macroblock_offset = (priv->svh_hdr.size)%8;
816 slice_param->macroblock_number = 0;
817 // the header of first gob_layer is empty (gob_header_empty=1), use vop_quant
818 slice_param->quant_scale = priv->svh_hdr.vop_quant;
821 if (has_packet_header) {
822 slice_param->macroblock_offset = priv->packet_hdr.size % 8;
823 slice_param->macroblock_number = priv->packet_hdr.macroblock_number;
824 slice_param->quant_scale = priv->packet_hdr.quant_scale;
827 slice_param->macroblock_offset = priv->vop_hdr.size % 8;
828 slice_param->macroblock_number = 0;
829 slice_param->quant_scale = priv->vop_hdr.quant;
832 return GST_VAAPI_DECODER_STATUS_SUCCESS;
835 static GstVaapiDecoderStatus
836 decode_packet(GstVaapiDecoderMpeg4 *decoder, GstMpeg4Packet packet)
838 GstVaapiDecoderMpeg4Private * const priv = decoder->priv;
839 GstMpeg4Packet *tos = &packet;
840 GstVaapiDecoderStatus status;
843 return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
845 status = gst_vaapi_decoder_check_status(GST_VAAPI_DECODER(decoder));
846 if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
849 // packet.size is the size from current marker to the next.
850 if (tos->type == GST_MPEG4_VISUAL_OBJ_SEQ_START) {
851 status = decode_sequence(decoder, packet.data + packet.offset, packet.size);
853 else if (tos->type == GST_MPEG4_VISUAL_OBJ_SEQ_END) {
854 status = decode_sequence_end(decoder);
856 else if (tos->type == GST_MPEG4_VISUAL_OBJ) {
857 status = decode_visual_object(decoder, packet.data + packet.offset, packet.size);
859 else if (tos->type >= GST_MPEG4_VIDEO_OBJ_FIRST && tos->type <= GST_MPEG4_VIDEO_OBJ_LAST) {
860 GST_WARNING("unexpected marker: (GST_MPEG4_VIDEO_OBJ_FIRST, GST_MPEG4_VIDEO_OBJ_LAST)");
861 status = GST_VAAPI_DECODER_STATUS_SUCCESS;
863 else if (tos->type >= GST_MPEG4_VIDEO_LAYER_FIRST && tos->type <= GST_MPEG4_VIDEO_LAYER_LAST) {
864 status = decode_video_object_layer(decoder, packet.data + packet.offset, packet.size);
866 else if (tos->type == GST_MPEG4_GROUP_OF_VOP) {
867 status = decode_gop(decoder, packet.data + packet.offset, packet.size);
869 else if (tos->type == GST_MPEG4_VIDEO_OBJ_PLANE) {
870 status = decode_picture(decoder, packet.data + packet.offset, packet.size);
871 if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
875 * A resync marker shall only be located immediately before a macroblock
876 * (or video packet header if exists) and aligned with a byte
877 * either start_code or resync_marker are scaned/measured by byte,
878 * while the header itself are parsed/measured in bit
879 * it means: resync_marker(video_packet_header) start from byte boundary,
880 * while MB doesn't start from byte boundary -- it is what 'macroblock_offset'
883 const guint8 *_data = packet.data + packet.offset + priv->vop_hdr.size/8;
884 gint _data_size = packet.size - (priv->vop_hdr.size/8);
885 GstMpeg4Packet video_packet;
887 if (priv->vol_hdr.resync_marker_disable) {
888 status = decode_slice(decoder, _data, _data_size, FALSE);
889 if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
893 // next start_code is required to determine the end of last slice
895 GstMpeg4ParseResult ret = GST_MPEG4_PARSER_OK;
897 gboolean first_slice = TRUE;
898 while (_data_size > 0) {
899 // we can skip user data here
900 ret = gst_mpeg4_parse(&video_packet, TRUE, &priv->vop_hdr, _data, 0, _data_size);
901 if(ret != GST_MPEG4_PARSER_OK) {
906 status = decode_slice(decoder, _data, video_packet.size, FALSE);
910 _data += video_packet.offset;
911 _data_size -= video_packet.offset;
913 ret = gst_mpeg4_parse_video_packet_header (&priv->packet_hdr, &priv->vol_hdr, &priv->vop_hdr, &priv->sprite_trajectory, _data, _data_size);
914 status = decode_slice(decoder,_data + priv->packet_hdr.size/8, video_packet.size - priv->packet_hdr.size/8, TRUE);
917 _data += video_packet.size;
918 _data_size -= video_packet.size;
921 status = decode_current_picture(decoder);
923 else if (tos->type == GST_MPEG4_USER_DATA
924 || tos->type == GST_MPEG4_VIDEO_SESSION_ERR
925 || tos->type == GST_MPEG4_FBA
926 || tos->type == GST_MPEG4_FBA_PLAN
927 || tos->type == GST_MPEG4_MESH
928 || tos->type == GST_MPEG4_MESH_PLAN
929 || tos->type == GST_MPEG4_STILL_TEXTURE_OBJ
930 || tos->type == GST_MPEG4_TEXTURE_SPATIAL
931 || tos->type == GST_MPEG4_TEXTURE_SNR_LAYER
932 || tos->type == GST_MPEG4_TEXTURE_TILE
933 || tos->type == GST_MPEG4_SHAPE_LAYER
934 || tos->type == GST_MPEG4_STUFFING
935 || tos->type == GST_MPEG4_SYSTEM_FIRST
936 || tos->type == GST_MPEG4_SYSTEM_LAST) {
937 GST_WARNING("Ignore marker: %x\n", tos->type);
938 status = GST_VAAPI_DECODER_STATUS_SUCCESS;
944 static GstVaapiDecoderStatus
945 decode_buffer(GstVaapiDecoderMpeg4 *decoder, GstBuffer *buffer)
947 GstVaapiDecoderMpeg4Private * const priv = decoder->priv;
948 GstVaapiDecoderStatus status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
952 buf = GST_BUFFER_DATA(buffer);
953 buf_size = GST_BUFFER_SIZE(buffer);
955 // visual object sequence end
956 if (!buf && buf_size == 0)
957 return decode_sequence_end(decoder);
959 gst_buffer_ref(buffer);
960 gst_adapter_push(priv->adapter, buffer);
962 if (priv->sub_buffer) {
963 buffer = gst_buffer_merge(priv->sub_buffer, buffer);
965 return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
966 gst_buffer_unref(priv->sub_buffer);
967 priv->sub_buffer = NULL;
970 buf = GST_BUFFER_DATA(buffer);
971 buf_size = GST_BUFFER_SIZE(buffer);
974 GstMpeg4Packet packet;
975 GstMpeg4ParseResult result = GST_MPEG4_PARSER_OK;
976 guint consumed_size = 0;
979 while (result == GST_MPEG4_PARSER_OK && pos < buf_size) {
980 result = gst_h263_parse (&packet,buf, pos, buf_size);
981 if (result != GST_MPEG4_PARSER_OK) {
984 status = decode_picture(decoder, packet.data+packet.offset, packet.size);
985 if (GST_VAAPI_DECODER_STATUS_SUCCESS == status) {
986 // MBs are not byte aligned, so we set the start address with byte aligned
987 // and mb offset with (priv->svh_hdr.size)%8
988 status = decode_slice(decoder, packet.data+packet.offset+(priv->svh_hdr.size)/8,
989 packet.size - (priv->svh_hdr.size)/8, FALSE);
990 status = decode_current_picture(decoder);
992 consumed_size = packet.offset + packet.size;
993 pos += consumed_size;
994 if (gst_adapter_available(priv->adapter) >= consumed_size)
995 gst_adapter_flush(priv->adapter, consumed_size);
998 GST_WARNING("decode h263 packet failed\n");
1004 while (pos < buf_size) {
1005 // don't skip user data, we need the size to pop tsb buffer
1006 result = gst_mpeg4_parse(&packet, FALSE, NULL, buf, pos, buf_size);
1007 if (result != GST_MPEG4_PARSER_OK) {
1010 status = decode_packet(decoder, packet);
1011 if (GST_VAAPI_DECODER_STATUS_SUCCESS == status ||
1012 GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA == status) {
1013 consumed_size = packet.offset + packet.size - pos;
1014 pos = packet.offset + packet.size;
1015 if (gst_adapter_available(priv->adapter) >= consumed_size)
1016 gst_adapter_flush(priv->adapter, consumed_size);
1019 GST_WARNING("decode mp4 packet failed\n");
1025 if ((result == GST_MPEG4_PARSER_NO_PACKET ||
1026 result == GST_MPEG4_PARSER_NO_PACKET_END ||
1027 status == GST_VAAPI_DECODER_STATUS_ERROR_NO_SURFACE) &&
1029 priv->sub_buffer = gst_buffer_create_sub(buffer, pos, buf_size-pos);
1030 status = GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
1035 static GstVaapiDecoderStatus
1036 decode_codec_data(GstVaapiDecoderMpeg4 *decoder, GstBuffer *buffer)
1038 GstVaapiDecoderStatus status;
1040 guint pos, buf_size, _buf_size;
1042 _buf = GST_BUFFER_DATA(buffer);
1043 _buf_size = GST_BUFFER_SIZE(buffer);
1044 // add additional 0x000001b2 to enclose the last header
1045 buf_size = _buf_size + 4;
1046 buf = malloc(buf_size);
1047 memcpy(buf, _buf, buf_size);
1048 buf[buf_size-4] = 0;
1049 buf[buf_size-3] = 0;
1050 buf[buf_size-2] = 1;
1051 buf[buf_size-1] = 0xb2;
1054 GstMpeg4Packet packet;
1055 GstMpeg4ParseResult result = GST_MPEG4_PARSER_OK;
1057 while (result == GST_MPEG4_PARSER_OK && pos < buf_size) {
1058 result = gst_mpeg4_parse(&packet, FALSE, NULL, buf, pos, buf_size);
1059 if (result != GST_MPEG4_PARSER_OK) {
1062 status = decode_packet(decoder, packet);
1063 if (GST_VAAPI_DECODER_STATUS_SUCCESS == status) {
1064 pos = packet.offset + packet.size;
1067 GST_WARNING("decode mp4 packet failed when decoding codec data\n");
1075 GstVaapiDecoderStatus
1076 gst_vaapi_decoder_mpeg4_decode(GstVaapiDecoder *base, GstBuffer *buffer)
1078 GstVaapiDecoderMpeg4 * const decoder = GST_VAAPI_DECODER_MPEG4(base);
1079 GstVaapiDecoderMpeg4Private * const priv = decoder->priv;
1080 GstBuffer *codec_data = NULL;
1081 GstVaapiDecoderStatus status;
1083 g_return_val_if_fail(priv->is_constructed,
1084 GST_VAAPI_DECODER_STATUS_ERROR_INIT_FAILED);
1086 if (!priv->is_opened) {
1087 priv->is_opened = gst_vaapi_decoder_mpeg4_open(decoder, buffer);
1088 if (!priv->is_opened)
1089 return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CODEC;
1091 codec_data = GST_VAAPI_DECODER_CODEC_DATA(decoder);
1093 status = decode_codec_data(decoder, codec_data);
1094 if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
1098 return decode_buffer(decoder, buffer);
1102 gst_vaapi_decoder_mpeg4_finalize(GObject *object)
1104 GstVaapiDecoderMpeg4 * const decoder = GST_VAAPI_DECODER_MPEG4(object);
1106 gst_vaapi_decoder_mpeg4_destroy(decoder);
1108 G_OBJECT_CLASS(gst_vaapi_decoder_mpeg4_parent_class)->finalize(object);
1112 gst_vaapi_decoder_mpeg4_constructed(GObject *object)
1114 GstVaapiDecoderMpeg4 * const decoder = GST_VAAPI_DECODER_MPEG4(object);
1115 GstVaapiDecoderMpeg4Private * const priv = decoder->priv;
1116 GObjectClass *parent_class;
1118 parent_class = G_OBJECT_CLASS(gst_vaapi_decoder_mpeg4_parent_class);
1119 if (parent_class->constructed)
1120 parent_class->constructed(object);
1122 priv->is_constructed = gst_vaapi_decoder_mpeg4_create(decoder);
1126 gst_vaapi_decoder_mpeg4_class_init(GstVaapiDecoderMpeg4Class *klass)
1128 GObjectClass * const object_class = G_OBJECT_CLASS(klass);
1129 GstVaapiDecoderClass * const decoder_class = GST_VAAPI_DECODER_CLASS(klass);
1131 g_type_class_add_private(klass, sizeof(GstVaapiDecoderMpeg4Private));
1133 object_class->finalize = gst_vaapi_decoder_mpeg4_finalize;
1134 object_class->constructed = gst_vaapi_decoder_mpeg4_constructed;
1136 decoder_class->decode = gst_vaapi_decoder_mpeg4_decode;
1137 decoder_class->clear_buffer = gst_vaapi_decoder_mpeg4_clear_buffer;
1141 gst_vaapi_decoder_mpeg4_init(GstVaapiDecoderMpeg4 *decoder)
1143 GstVaapiDecoderMpeg4Private *priv;
1145 priv = GST_VAAPI_DECODER_MPEG4_GET_PRIVATE(decoder);
1146 decoder->priv = priv;
1151 priv->profile = GST_VAAPI_PROFILE_MPEG4_SIMPLE;
1152 priv->curr_picture = NULL;
1153 priv->next_picture = NULL;
1154 priv->prev_picture = NULL;
1155 priv->adapter = NULL;
1156 priv->sub_buffer = NULL;
1157 priv->seq_pts = GST_CLOCK_TIME_NONE;
1158 priv->gop_pts = GST_CLOCK_TIME_NONE;
1159 priv->max_pts = GST_CLOCK_TIME_NONE;
1161 priv->calculate_pts_diff = TRUE;
1162 priv->is_constructed = FALSE;
1163 priv->is_opened = FALSE;
1164 priv->is_first_field = FALSE;
1165 priv->size_changed = TRUE;
1166 priv->profile_changed = TRUE;
1167 priv->progressive_sequence = FALSE;
1168 priv->closed_gop = FALSE;
1169 priv->broken_link = FALSE;
1170 priv->last_non_b_scale_time = 0;
1171 priv->non_b_scale_time = 0;
1177 * gst_vaapi_decoder_mpeg4_new:
1178 * @display: a #GstVaapiDisplay
1179 * @caps: a #GstCaps holding codec information
1181 * Creates a new #GstVaapiDecoder for MPEG-2 decoding. The @caps can
1182 * hold extra information like codec-data and pictured coded size.
1184 * Return value: the newly allocated #GstVaapiDecoder object
1187 gst_vaapi_decoder_mpeg4_new(GstVaapiDisplay *display, GstCaps *caps)
1189 GstVaapiDecoderMpeg4 *decoder;
1191 g_return_val_if_fail(GST_VAAPI_IS_DISPLAY(display), NULL);
1192 g_return_val_if_fail(GST_IS_CAPS(caps), NULL);
1194 decoder = g_object_new(
1195 GST_VAAPI_TYPE_DECODER_MPEG4,
1200 if (!decoder->priv->is_constructed) {
1201 g_object_unref(decoder);
1204 return GST_VAAPI_DECODER_CAST(decoder);