gst-libs/gst/vaapi/gstvaapidecoder_h264.c

   1 /*
   2  *  gstvaapidecoder_h264.c - H.264 decoder
   3  *
   4  *  Copyright (C) 2011-2012 Intel Corporation
   5  *
   6  *  This library is free software; you can redistribute it and/or
   7  *  modify it under the terms of the GNU Lesser General Public License
   8  *  as published by the Free Software Foundation; either version 2.1
   9  *  of the License, or (at your option) any later version.
  10  *
  11  *  This library is distributed in the hope that it will be useful,
  12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  *  Lesser General Public License for more details.
  15  *
  16  *  You should have received a copy of the GNU Lesser General Public
  17  *  License along with this library; if not, write to the Free
  18  *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  19  *  Boston, MA 02110-1301 USA
  20  */
  21
  22 /**
  23  * SECTION:gstvaapidecoder_h264
  24  * @short_description: H.264 decoder
  25  */
  26
  27 #include "sysdeps.h"
  28 #include <string.h>
  29 #include <stdlib.h>
  30 #include <gst/base/gstadapter.h>
  31 #include <gst/codecparsers/gsth264parser.h>
  32 #include "gstvaapidecoder_h264.h"
  33 #include "gstvaapidecoder_objects.h"
  34 #include "gstvaapidecoder_priv.h"
  35 #include "gstvaapidisplay_priv.h"
  36 #include "gstvaapiobject_priv.h"
  37
  38 #define DEBUG 1
  39 #include "gstvaapidebug.h"
  40
  41 typedef struct _GstVaapiPictureH264             GstVaapiPictureH264;
  42 typedef struct _GstVaapiPictureH264Class        GstVaapiPictureH264Class;
  43 typedef struct _GstVaapiSliceH264               GstVaapiSliceH264;
  44 typedef struct _GstVaapiSliceH264Class          GstVaapiSliceH264Class;
  45
  46 /* ------------------------------------------------------------------------- */
  47 /* --- H.264 Pictures                                                    --- */
  48 /* ------------------------------------------------------------------------- */
  49
  50 #define GST_VAAPI_TYPE_PICTURE_H264 \
  51     (gst_vaapi_picture_h264_get_type())
  52
  53 #define GST_VAAPI_PICTURE_H264_CAST(obj) \
  54     ((GstVaapiPictureH264 *)(obj))
  55
  56 #define GST_VAAPI_PICTURE_H264(obj)                             \
  57     (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
  58                                 GST_VAAPI_TYPE_PICTURE_H264,    \
  59                                 GstVaapiPictureH264))
  60
  61 #define GST_VAAPI_PICTURE_H264_CLASS(klass)                     \
  62     (G_TYPE_CHECK_CLASS_CAST((klass),                           \
  63                              GST_VAAPI_TYPE_PICTURE_H264,       \
  64                              GstVaapiPictureH264Class))
  65
  66 #define GST_VAAPI_IS_PICTURE_H264(obj) \
  67     (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_VAAPI_TYPE_PICTURE_H264))
  68
  69 #define GST_VAAPI_IS_PICTURE_H264_CLASS(klass) \
  70     (G_TYPE_CHECK_CLASS_TYPE((klass), GST_VAAPI_TYPE_PICTURE_H264))
  71
  72 #define GST_VAAPI_PICTURE_H264_GET_CLASS(obj)                   \
  73     (G_TYPE_INSTANCE_GET_CLASS((obj),                           \
  74                                GST_VAAPI_TYPE_PICTURE_H264,     \
  75                                GstVaapiPictureH264Class))
  76
  77 struct _GstVaapiPictureH264 {
  78     GstVaapiPicture             base;
  79     VAPictureH264               info;
  80     gint32                      poc;
  81     gint32                      frame_num;              // Original frame_num from slice_header()
  82     gint32                      frame_num_wrap;         // Temporary for ref pic marking: FrameNumWrap
  83     gint32                      pic_num;                // Temporary for ref pic marking: PicNum
  84     gint32                      long_term_pic_num;      // Temporary for ref pic marking: LongTermPicNum
  85     guint                       is_idr                  : 1;
  86     guint                       is_long_term            : 1;
  87     guint                       field_pic_flag          : 1;
  88     guint                       bottom_field_flag       : 1;
  89     guint                       has_mmco_5              : 1;
  90     guint                       output_flag             : 1;
  91     guint                       output_needed           : 1;
  92 };
  93
  94 struct _GstVaapiPictureH264Class {
  95     /*< private >*/
  96     GstVaapiPictureClass        parent_class;
  97 };
  98
  99 GST_VAAPI_CODEC_DEFINE_TYPE(GstVaapiPictureH264,
 100                             gst_vaapi_picture_h264,
 101                             GST_VAAPI_TYPE_PICTURE)
 102
 103 static void
 104 gst_vaapi_picture_h264_destroy(GstVaapiPictureH264 *decoder)
 105 {
 106 }
 107
 108 static gboolean
 109 gst_vaapi_picture_h264_create(
 110     GstVaapiPictureH264                      *picture,
 111     const GstVaapiCodecObjectConstructorArgs *args
 112 )
 113 {
 114     return TRUE;
 115 }
 116
 117 static void
 118 gst_vaapi_picture_h264_init(GstVaapiPictureH264 *picture)
 119 {
 120     VAPictureH264 *va_pic;
 121
 122     va_pic                      = &picture->info;
 123     va_pic->flags               = 0;
 124     va_pic->TopFieldOrderCnt    = 0;
 125     va_pic->BottomFieldOrderCnt = 0;
 126
 127     picture->poc                = 0;
 128     picture->is_long_term       = FALSE;
 129     picture->is_idr             = FALSE;
 130     picture->has_mmco_5         = FALSE;
 131     picture->output_needed      = FALSE;
 132 }
 133
 134 static inline GstVaapiPictureH264 *
 135 gst_vaapi_picture_h264_new(GstVaapiDecoderH264 *decoder)
 136 {
 137     GstVaapiCodecObject *object;
 138
 139     g_return_val_if_fail(GST_VAAPI_IS_DECODER(decoder), NULL);
 140
 141     object = gst_vaapi_codec_object_new(
 142         GST_VAAPI_TYPE_PICTURE_H264,
 143         GST_VAAPI_CODEC_BASE(decoder),
 144         NULL, sizeof(VAPictureParameterBufferH264),
 145         NULL, 0
 146     );
 147     if (!object)
 148         return NULL;
 149     return GST_VAAPI_PICTURE_H264_CAST(object);
 150 }
 151
 152 /* ------------------------------------------------------------------------- */
 153 /* --- Slices                                                            --- */
 154 /* ------------------------------------------------------------------------- */
 155
 156 #define GST_VAAPI_TYPE_SLICE_H264 \
 157     (gst_vaapi_slice_h264_get_type())
 158
 159 #define GST_VAAPI_SLICE_H264_CAST(obj) \
 160     ((GstVaapiSliceH264 *)(obj))
 161
 162 #define GST_VAAPI_SLICE_H264(obj)                               \
 163     (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
 164                                 GST_VAAPI_TYPE_SLICE_H264,      \
 165                                 GstVaapiSliceH264))
 166
 167 #define GST_VAAPI_SLICE_H264_CLASS(klass)                       \
 168     (G_TYPE_CHECK_CLASS_CAST((klass),                           \
 169                              GST_VAAPI_TYPE_SLICE_H264,         \
 170                              GstVaapiSliceH264Class))
 171
 172 #define GST_VAAPI_IS_SLICE_H264(obj) \
 173     (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_VAAPI_TYPE_SLICE_H264))
 174
 175 #define GST_VAAPI_IS_SLICE_H264_CLASS(klass) \
 176     (G_TYPE_CHECK_CLASS_TYPE((klass), GST_VAAPI_TYPE_SLICE_H264))
 177
 178 #define GST_VAAPI_SLICE_H264_GET_CLASS(obj)                     \
 179     (G_TYPE_INSTANCE_GET_CLASS((obj),                           \
 180                                GST_VAAPI_TYPE_SLICE_H264,       \
 181                                GstVaapiSliceH264Class))
 182
 183 struct _GstVaapiSliceH264 {
 184     GstVaapiSlice               base;
 185     GstH264SliceHdr             slice_hdr;              // parsed slice_header()
 186 };
 187
 188 struct _GstVaapiSliceH264Class {
 189     /*< private >*/
 190     GstVaapiSliceClass          parent_class;
 191 };
 192
 193 GST_VAAPI_CODEC_DEFINE_TYPE(GstVaapiSliceH264,
 194                             gst_vaapi_slice_h264,
 195                             GST_VAAPI_TYPE_SLICE)
 196
 197 static void
 198 gst_vaapi_slice_h264_destroy(GstVaapiSliceH264 *slice)
 199 {
 200 }
 201
 202 static gboolean
 203 gst_vaapi_slice_h264_create(
 204     GstVaapiSliceH264                        *slice,
 205     const GstVaapiCodecObjectConstructorArgs *args
 206 )
 207 {
 208     return TRUE;
 209 }
 210
 211 static void
 212 gst_vaapi_slice_h264_init(GstVaapiSliceH264 *slice)
 213 {
 214 }
 215
 216 static inline GstVaapiSliceH264 *
 217 gst_vaapi_slice_h264_new(
 218     GstVaapiDecoderH264 *decoder,
 219     const guint8        *data,
 220     guint                data_size
 221 )
 222 {
 223     GstVaapiCodecObject *object;
 224
 225     g_return_val_if_fail(GST_VAAPI_IS_DECODER(decoder), NULL);
 226
 227     object = gst_vaapi_codec_object_new(
 228         GST_VAAPI_TYPE_SLICE_H264,
 229         GST_VAAPI_CODEC_BASE(decoder),
 230         NULL, sizeof(VASliceParameterBufferH264),
 231         data, data_size
 232     );
 233     if (!object)
 234         return NULL;
 235     return GST_VAAPI_SLICE_H264_CAST(object);
 236 }
 237
 238 /* ------------------------------------------------------------------------- */
 239 /* --- H.264 Decoder                                                     --- */
 240 /* ------------------------------------------------------------------------- */
 241
 242 G_DEFINE_TYPE(GstVaapiDecoderH264,
 243               gst_vaapi_decoder_h264,
 244               GST_VAAPI_TYPE_DECODER);
 245
 246 #define GST_VAAPI_DECODER_H264_GET_PRIVATE(obj)                 \
 247     (G_TYPE_INSTANCE_GET_PRIVATE((obj),                         \
 248                                  GST_VAAPI_TYPE_DECODER_H264,   \
 249                                  GstVaapiDecoderH264Private))
 250
 251 // Used for field_poc[]
 252 #define TOP_FIELD       0
 253 #define BOTTOM_FIELD    1
 254
 255 struct _GstVaapiDecoderH264Private {
 256     GstAdapter                 *adapter;
 257     GstBuffer                  *sub_buffer;
 258     GstH264NalParser           *parser;
 259     GstH264SPS                 *sps;
 260     GstH264SPS                  last_sps;
 261     GstH264PPS                 *pps;
 262     GstH264PPS                  last_pps;
 263     GstVaapiPictureH264        *current_picture;
 264     GstVaapiPictureH264        *dpb[16];
 265     guint                       dpb_count;
 266     guint                       dpb_size;
 267     GstVaapiProfile             profile;
 268     GstVaapiPictureH264        *short_ref[32];
 269     guint                       short_ref_count;
 270     GstVaapiPictureH264        *long_ref[32];
 271     guint                       long_ref_count;
 272     GstVaapiPictureH264        *RefPicList0[32];
 273     guint                       RefPicList0_count;
 274     GstVaapiPictureH264        *RefPicList1[32];
 275     guint                       RefPicList1_count;
 276     guint                       nal_length_size;
 277     guint                       width;
 278     guint                       height;
 279     guint                       mb_x;
 280     guint                       mb_y;
 281     guint                       mb_width;
 282     guint                       mb_height;
 283     guint8                      scaling_list_4x4[6][16];
 284     guint8                      scaling_list_8x8[6][64];
 285     gint32                      field_poc[2];           // 0:TopFieldOrderCnt / 1:BottomFieldOrderCnt
 286     gint32                      poc_msb;                // PicOrderCntMsb
 287     gint32                      poc_lsb;                // pic_order_cnt_lsb (from slice_header())
 288     gint32                      prev_poc_msb;           // prevPicOrderCntMsb
 289     gint32                      prev_poc_lsb;           // prevPicOrderCntLsb
 290     gint32                      frame_num_offset;       // FrameNumOffset
 291     gint32                      prev_frame_num_offset;  // prevFrameNumOffset
 292     gint32                      frame_num;              // frame_num (from slice_header())
 293     gint32                      prev_frame_num;         // prevFrameNum
 294     guint                       is_constructed          : 1;
 295     guint                       is_opened               : 1;
 296     guint                       is_avc                  : 1;
 297     guint                       has_context             : 1;
 298 };
 299
 300 static gboolean
 301 decode_picture_end(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture);
 302
 303 static void
 304 clear_references(
 305     GstVaapiDecoderH264  *decoder,
 306     GstVaapiPictureH264 **pictures,
 307     guint                *picture_count
 308 );
 309
 310 static void
 311 dpb_remove_index(GstVaapiDecoderH264 *decoder, guint index)
 312 {
 313     GstVaapiDecoderH264Private * const priv = decoder->priv;
 314     guint num_pictures = --priv->dpb_count;
 315
 316     if (index != num_pictures)
 317         gst_vaapi_picture_replace(&priv->dpb[index], priv->dpb[num_pictures]);
 318     gst_vaapi_picture_replace(&priv->dpb[num_pictures], NULL);
 319 }
 320
 321 static inline gboolean
 322 dpb_output(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
 323 {
 324     /* XXX: update cropping rectangle */
 325     picture->output_needed = FALSE;
 326     return gst_vaapi_picture_output(GST_VAAPI_PICTURE_CAST(picture));
 327 }
 328
 329 static gboolean
 330 dpb_bump(GstVaapiDecoderH264 *decoder)
 331 {
 332     GstVaapiDecoderH264Private * const priv = decoder->priv;
 333     guint i, lowest_poc_index;
 334     gboolean success;
 335
 336     for (i = 0; i < priv->dpb_count; i++) {
 337         if (priv->dpb[i]->output_needed)
 338             break;
 339     }
 340     if (i == priv->dpb_count)
 341         return FALSE;
 342
 343     lowest_poc_index = i++;
 344     for (; i < priv->dpb_count; i++) {
 345         GstVaapiPictureH264 * const picture = priv->dpb[i];
 346         if (picture->output_needed && picture->poc < priv->dpb[lowest_poc_index]->poc)
 347             lowest_poc_index = i;
 348     }
 349
 350     success = dpb_output(decoder, priv->dpb[lowest_poc_index]);
 351     if (!GST_VAAPI_PICTURE_IS_REFERENCE(priv->dpb[lowest_poc_index]))
 352         dpb_remove_index(decoder, lowest_poc_index);
 353     return success;
 354 }
 355
 356 static void
 357 dpb_flush(GstVaapiDecoderH264 *decoder)
 358 {
 359     GstVaapiDecoderH264Private * const priv = decoder->priv;
 360
 361     while (dpb_bump(decoder))
 362         ;
 363     clear_references(decoder, priv->dpb, &priv->dpb_count);
 364 }
 365
 366 static gboolean
 367 dpb_add(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
 368 {
 369     GstVaapiDecoderH264Private * const priv = decoder->priv;
 370     guint i;
 371
 372     // Remove all unused pictures
 373     if (picture->is_idr)
 374         dpb_flush(decoder);
 375     else {
 376         i = 0;
 377         while (i < priv->dpb_count) {
 378             GstVaapiPictureH264 * const picture = priv->dpb[i];
 379             if (!picture->output_needed &&
 380                 !GST_VAAPI_PICTURE_IS_REFERENCE(picture))
 381                 dpb_remove_index(decoder, i);
 382             else
 383                 i++;
 384         }
 385     }
 386
 387     // C.4.5.1 - Storage and marking of a reference decoded picture into the DPB
 388     if (GST_VAAPI_PICTURE_IS_REFERENCE(picture)) {
 389         while (priv->dpb_count == priv->dpb_size) {
 390             if (!dpb_bump(decoder))
 391                 return FALSE;
 392         }
 393         gst_vaapi_picture_replace(&priv->dpb[priv->dpb_count++], picture);
 394         if (picture->output_flag)
 395             picture->output_needed = TRUE;
 396     }
 397
 398     // C.4.5.2 - Storage and marking of a non-reference decoded picture into the DPB
 399     else {
 400         if (!picture->output_flag)
 401             return TRUE;
 402         while (priv->dpb_count == priv->dpb_size) {
 403             for (i = 0; i < priv->dpb_count; i++) {
 404                 if (priv->dpb[i]->output_needed &&
 405                     priv->dpb[i]->poc < picture->poc)
 406                     break;
 407             }
 408             if (i == priv->dpb_count)
 409                 return dpb_output(decoder, picture);
 410             if (!dpb_bump(decoder))
 411                 return FALSE;
 412         }
 413         gst_vaapi_picture_replace(&priv->dpb[priv->dpb_count++], picture);
 414         picture->output_needed = TRUE;
 415     }
 416     return TRUE;
 417 }
 418
 419 static void
 420 dpb_reset(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 421 {
 422     GstVaapiDecoderH264Private * const priv = decoder->priv;
 423     guint max_dec_frame_buffering, MaxDpbMbs, PicSizeMbs;
 424
 425     /* Table A-1 - Level limits */
 426     switch (sps->level_idc) {
 427     case 10: MaxDpbMbs = 396;    break;
 428     case 11: MaxDpbMbs = 900;    break;
 429     case 12: MaxDpbMbs = 2376;   break;
 430     case 13: MaxDpbMbs = 2376;   break;
 431     case 20: MaxDpbMbs = 2376;   break;
 432     case 21: MaxDpbMbs = 4752;   break;
 433     case 22: MaxDpbMbs = 8100;   break;
 434     case 30: MaxDpbMbs = 8100;   break;
 435     case 31: MaxDpbMbs = 18000;  break;
 436     case 32: MaxDpbMbs = 20480;  break;
 437     case 40: MaxDpbMbs = 32768;  break;
 438     case 41: MaxDpbMbs = 32768;  break;
 439     case 42: MaxDpbMbs = 34816;  break;
 440     case 50: MaxDpbMbs = 110400; break;
 441     case 51: MaxDpbMbs = 184320; break;
 442     default:
 443         g_assert(0 && "unhandled level");
 444         break;
 445     }
 446
 447     PicSizeMbs = ((sps->pic_width_in_mbs_minus1 + 1) *
 448                   (sps->pic_height_in_map_units_minus1 + 1) *
 449                   (sps->frame_mbs_only_flag ? 1 : 2));
 450     max_dec_frame_buffering = MaxDpbMbs / PicSizeMbs;
 451     if (max_dec_frame_buffering > 8)
 452         max_dec_frame_buffering = 8;
 453
 454     /* VUI parameters */
 455     if (sps->vui_parameters_present_flag) {
 456         GstH264VUIParams * const vui_params = &sps->vui_parameters;
 457         if (vui_params->bitstream_restriction_flag)
 458             max_dec_frame_buffering = vui_params->max_dec_frame_buffering;
 459         else {
 460             switch (sps->profile_idc) {
 461             case 44:  // CAVLC 4:4:4 Intra profile
 462             case 86:  // Scalable High profile
 463             case 100: // High profile
 464             case 110: // High 10 profile
 465             case 122: // High 4:2:2 profile
 466             case 244: // High 4:4:4 Predictive profile
 467                 if (sps->constraint_set3_flag)
 468                     max_dec_frame_buffering = 0;
 469                 break;
 470             }
 471         }
 472     }
 473
 474     if (max_dec_frame_buffering > 16)
 475         max_dec_frame_buffering = 16;
 476     else if (max_dec_frame_buffering < sps->num_ref_frames)
 477         max_dec_frame_buffering = sps->num_ref_frames;
 478     priv->dpb_size = MAX(1, max_dec_frame_buffering);
 479     GST_DEBUG("DPB size %u", priv->dpb_size);
 480 }
 481
 482 static GstVaapiDecoderStatus
 483 get_status(GstH264ParserResult result)
 484 {
 485     GstVaapiDecoderStatus status;
 486
 487     switch (result) {
 488     case GST_H264_PARSER_OK:
 489         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
 490         break;
 491     case GST_H264_PARSER_NO_NAL_END:
 492         status = GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
 493         break;
 494     case GST_H264_PARSER_ERROR:
 495         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
 496         break;
 497     default:
 498         status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 499         break;
 500     }
 501     return status;
 502 }
 503
 504 static inline GstH264DecRefPicMarking *
 505 get_dec_ref_pic_marking(GstVaapiPictureH264 *picture_h264)
 506 {
 507     GstVaapiPicture * const picture = GST_VAAPI_PICTURE_CAST(picture_h264);
 508     GstVaapiSliceH264 *slice;
 509
 510     slice = g_ptr_array_index(picture->slices, picture->slices->len - 1);
 511     return &slice->slice_hdr.dec_ref_pic_marking;
 512 }
 513
 514 static void
 515 gst_vaapi_decoder_h264_clear_buffer(GstVaapiDecoder *base)
 516 {
 517     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264(base);
 518     GstVaapiDecoderH264Private * const priv = decoder->priv;
 519
 520     gst_vaapi_picture_replace(&priv->current_picture, NULL);
 521     clear_references(decoder, priv->short_ref, &priv->short_ref_count);
 522     clear_references(decoder, priv->long_ref,  &priv->long_ref_count );
 523     clear_references(decoder, priv->dpb,       &priv->dpb_count      );
 524
 525     if (priv->sub_buffer) {
 526         gst_buffer_unref(priv->sub_buffer);
 527         priv->sub_buffer = NULL;
 528     }
 529
 530     if (priv->adapter) {
 531         gst_adapter_clear(priv->adapter);
 532     }
 533 }
 534
 535 static void
 536 gst_vaapi_decoder_h264_close(GstVaapiDecoderH264 *decoder)
 537 {
 538     GstVaapiDecoderH264Private * const priv = decoder->priv;
 539
 540     gst_vaapi_decoder_h264_clear_buffer(GST_VAAPI_DECODER_CAST(decoder));
 541
 542     if (priv->parser) {
 543         gst_h264_nal_parser_free(priv->parser);
 544         priv->parser = NULL;
 545     }
 546
 547     if (priv->adapter) {
 548         g_object_unref(priv->adapter);
 549         priv->adapter = NULL;
 550     }
 551 }
 552
 553 static gboolean
 554 gst_vaapi_decoder_h264_open(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
 555 {
 556     GstVaapiDecoderH264Private * const priv = decoder->priv;
 557
 558     gst_vaapi_decoder_h264_close(decoder);
 559
 560     priv->adapter = gst_adapter_new();
 561     if (!priv->adapter)
 562         return FALSE;
 563
 564     priv->parser = gst_h264_nal_parser_new();
 565     if (!priv->parser)
 566         return FALSE;
 567     return TRUE;
 568 }
 569
 570 static void
 571 gst_vaapi_decoder_h264_destroy(GstVaapiDecoderH264 *decoder)
 572 {
 573     gst_vaapi_decoder_h264_close(decoder);
 574 }
 575
 576 static gboolean
 577 gst_vaapi_decoder_h264_create(GstVaapiDecoderH264 *decoder)
 578 {
 579     if (!GST_VAAPI_DECODER_CODEC(decoder))
 580         return FALSE;
 581     return TRUE;
 582 }
 583
 584 static inline guint
 585 min_surfaces_count(guint dbp_size)
 586 {
 587     guint count = dbp_size + 4;
 588
 589     if (count < GST_DECODER_DEFAULT_SURFACES_COUNT)
 590         count = GST_DECODER_DEFAULT_SURFACES_COUNT;
 591     return count;
 592 }
 593
 594 static GstVaapiDecoderStatus
 595 ensure_context(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 596 {
 597     GstVaapiDecoderH264Private * const priv = decoder->priv;
 598     GstVaapiProfile profiles[2];
 599     GstVaapiEntrypoint entrypoint = GST_VAAPI_ENTRYPOINT_VLD;
 600     guint i, n_profiles = 0;
 601     gboolean success, reset_context = FALSE;
 602
 603     if (!priv->has_context || priv->sps->profile_idc != sps->profile_idc) {
 604         GST_DEBUG("profile changed");
 605         reset_context = TRUE;
 606
 607         switch (sps->profile_idc) {
 608         case 66:
 609             profiles[n_profiles++] = GST_VAAPI_PROFILE_H264_BASELINE;
 610             break;
 611         case 77:
 612             profiles[n_profiles++] = GST_VAAPI_PROFILE_H264_MAIN;
 613             // fall-through
 614         case 100:
 615             profiles[n_profiles++] = GST_VAAPI_PROFILE_H264_HIGH;
 616             break;
 617         default:
 618             GST_DEBUG("unsupported profile %d", sps->profile_idc);
 619             return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_PROFILE;
 620         }
 621
 622         for (i = 0; i < n_profiles; i++) {
 623             success = gst_vaapi_display_has_decoder(
 624                 GST_VAAPI_DECODER_DISPLAY(decoder),
 625                 profiles[i],
 626                 entrypoint
 627             );
 628             if (success)
 629                 break;
 630         }
 631         if (i == n_profiles)
 632             return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_PROFILE;
 633         priv->profile = profiles[i];
 634     }
 635
 636     if (!priv->has_context ||
 637         priv->sps->chroma_format_idc != sps->chroma_format_idc) {
 638         GST_DEBUG("chroma format changed");
 639         reset_context = TRUE;
 640
 641         /* XXX: theoritically, we could handle 4:2:2 format */
 642         if (sps->chroma_format_idc != 1)
 643             return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;
 644     }
 645
 646     if (!priv->has_context              ||
 647         priv->sps->width  != sps->width ||
 648         priv->sps->height != sps->height) {
 649         GST_DEBUG("size changed");
 650         reset_context      = TRUE;
 651
 652         priv->width      = sps->width;
 653         priv->height     = sps->height;
 654         priv->mb_width   = sps->pic_width_in_mbs_minus1 + 1;
 655         priv->mb_height  = sps->pic_height_in_map_units_minus1 + 1;
 656         priv->mb_height *= 2 - sps->frame_mbs_only_flag;
 657     }
 658
 659     if (reset_context) {
 660         /* Reset DPB first*/
 661         dpb_reset(decoder, sps);
 662
 663         success = gst_vaapi_decoder_ensure_context(
 664             GST_VAAPI_DECODER(decoder),
 665             priv->profile,
 666             entrypoint,
 667             priv->width,
 668             priv->height,
 669             min_surfaces_count(priv->dpb_size)
 670         );
 671         if (!success)
 672             return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 673         priv->has_context = TRUE;
 674
 675     }
 676     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 677 }
 678
 679 static GstVaapiDecoderStatus
 680 ensure_quant_matrix(GstVaapiDecoderH264 *decoder, GstH264PPS *pps)
 681 {
 682     GstVaapiDecoderH264Private * const priv = decoder->priv;
 683
 684     if (priv->pps != pps) {
 685         memcpy(priv->scaling_list_4x4, pps->scaling_lists_4x4,
 686                sizeof(priv->scaling_list_4x4));
 687         memcpy(priv->scaling_list_8x8, pps->scaling_lists_8x8,
 688                sizeof(priv->scaling_list_8x8));
 689     }
 690     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 691 }
 692
 693 static gboolean
 694 decode_current_picture(GstVaapiDecoderH264 *decoder)
 695 {
 696     GstVaapiDecoderH264Private * const priv = decoder->priv;
 697     GstVaapiPictureH264 * const picture = priv->current_picture;
 698     gboolean success = FALSE;
 699
 700     if (!picture)
 701         return TRUE;
 702
 703     if (!decode_picture_end(decoder, picture))
 704         goto end;
 705     if (!gst_vaapi_picture_decode(GST_VAAPI_PICTURE_CAST(picture)))
 706         goto end;
 707     success = TRUE;
 708 end:
 709     gst_vaapi_picture_replace(&priv->current_picture, NULL);
 710     return success;
 711 }
 712
 713 static GstVaapiDecoderStatus
 714 decode_sps(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
 715 {
 716     GstVaapiDecoderH264Private * const priv = decoder->priv;
 717     GstH264SPS * const sps = &priv->last_sps;
 718     GstH264ParserResult result;
 719
 720     GST_DEBUG("decode SPS");
 721
 722     if (priv->current_picture && !decode_current_picture(decoder))
 723         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 724
 725     memset(sps, 0, sizeof(*sps));
 726     result = gst_h264_parser_parse_sps(priv->parser, nalu, sps, TRUE);
 727     if (result != GST_H264_PARSER_OK)
 728         return get_status(result);
 729
 730     return ensure_context(decoder, sps);
 731 }
 732
 733 static GstVaapiDecoderStatus
 734 decode_pps(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
 735 {
 736     GstVaapiDecoderH264Private * const priv = decoder->priv;
 737     GstH264PPS * const pps = &priv->last_pps;
 738     GstH264ParserResult result;
 739
 740     GST_DEBUG("decode PPS");
 741
 742     if (priv->current_picture && !decode_current_picture(decoder))
 743         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 744
 745     memset(pps, 0, sizeof(*pps));
 746     result = gst_h264_parser_parse_pps(priv->parser, nalu, pps);
 747     if (result != GST_H264_PARSER_OK)
 748         return get_status(result);
 749
 750     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 751 }
 752
 753 static GstVaapiDecoderStatus
 754 decode_sei(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
 755 {
 756     GstVaapiDecoderH264Private * const priv = decoder->priv;
 757     GstH264SEIMessage sei;
 758     GstH264ParserResult result;
 759
 760     GST_DEBUG("decode SEI");
 761
 762     memset(&sei, 0, sizeof(sei));
 763     result = gst_h264_parser_parse_sei(priv->parser, nalu, &sei);
 764     if (result != GST_H264_PARSER_OK) {
 765         GST_WARNING("failed to decode SEI, payload type:%d", sei.payloadType);
 766         return get_status(result);
 767     }
 768
 769     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 770 }
 771
 772 static GstVaapiDecoderStatus
 773 decode_sequence_end(GstVaapiDecoderH264 *decoder)
 774 {
 775     GstVaapiDecoderH264Private * const priv = decoder->priv;
 776
 777     GST_DEBUG("decode sequence-end");
 778
 779     if (priv->current_picture && !decode_current_picture(decoder))
 780         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 781     dpb_flush(decoder);
 782     return GST_VAAPI_DECODER_STATUS_END_OF_STREAM;
 783 }
 784
 785 /* 8.2.1.1 - Decoding process for picture order count type 0 */
 786 static void
 787 init_picture_poc_0(
 788     GstVaapiDecoderH264 *decoder,
 789     GstVaapiPictureH264 *picture,
 790     GstH264SliceHdr     *slice_hdr
 791 )
 792 {
 793     GstVaapiDecoderH264Private * const priv = decoder->priv;
 794     GstH264PPS * const pps = slice_hdr->pps;
 795     GstH264SPS * const sps = pps->sequence;
 796     const gint32 MaxPicOrderCntLsb = 1 << (sps->log2_max_pic_order_cnt_lsb_minus4 + 4);
 797
 798     GST_DEBUG("decode picture order count type 0");
 799
 800     // (8-3)
 801     priv->poc_lsb = slice_hdr->pic_order_cnt_lsb;
 802     if (priv->poc_lsb < priv->prev_poc_lsb &&
 803         (priv->prev_poc_lsb - priv->poc_lsb) >= (MaxPicOrderCntLsb / 2))
 804         priv->poc_msb = priv->prev_poc_msb + MaxPicOrderCntLsb;
 805     else if (priv->poc_lsb > priv->prev_poc_lsb &&
 806              (priv->poc_lsb - priv->prev_poc_lsb) > (MaxPicOrderCntLsb / 2))
 807         priv->poc_msb = priv->prev_poc_msb - MaxPicOrderCntLsb;
 808     else
 809         priv->poc_msb = priv->prev_poc_msb;
 810
 811     // (8-4)
 812     if (!slice_hdr->field_pic_flag || !slice_hdr->bottom_field_flag)
 813         priv->field_poc[TOP_FIELD] = priv->poc_msb + priv->poc_lsb;
 814
 815     // (8-5)
 816     if (!slice_hdr->field_pic_flag)
 817         priv->field_poc[BOTTOM_FIELD] = priv->field_poc[TOP_FIELD] +
 818             slice_hdr->delta_pic_order_cnt_bottom;
 819     else if (slice_hdr->bottom_field_flag)
 820         priv->field_poc[BOTTOM_FIELD] = priv->poc_msb + priv->poc_lsb;
 821 }
 822
 823 /* 8.2.1.2 - Decoding process for picture order count type 1 */
 824 static void
 825 init_picture_poc_1(
 826     GstVaapiDecoderH264 *decoder,
 827     GstVaapiPictureH264 *picture,
 828     GstH264SliceHdr     *slice_hdr
 829 )
 830 {
 831     GstVaapiDecoderH264Private * const priv = decoder->priv;
 832     GstH264PPS * const pps = slice_hdr->pps;
 833     GstH264SPS * const sps = pps->sequence;
 834     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
 835     gint32 abs_frame_num, expected_poc;
 836     guint i;
 837
 838     GST_DEBUG("decode picture order count type 1");
 839
 840     // (8-6)
 841     if (picture->is_idr)
 842         priv->frame_num_offset = 0;
 843     else if (priv->prev_frame_num > priv->frame_num)
 844         priv->frame_num_offset = priv->prev_frame_num_offset + MaxFrameNum;
 845     else
 846         priv->frame_num_offset = priv->prev_frame_num_offset;
 847
 848     // (8-7)
 849     if (sps->num_ref_frames_in_pic_order_cnt_cycle != 0)
 850         abs_frame_num = priv->frame_num_offset + priv->frame_num;
 851     else
 852         abs_frame_num = 0;
 853     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture) && abs_frame_num > 0)
 854         abs_frame_num = abs_frame_num - 1;
 855
 856     if (abs_frame_num > 0) {
 857         gint32 expected_delta_per_poc_cycle;
 858         gint32 poc_cycle_cnt, frame_num_in_poc_cycle;
 859
 860         expected_delta_per_poc_cycle = 0;
 861         for (i = 0; i < sps->num_ref_frames_in_pic_order_cnt_cycle; i++)
 862             expected_delta_per_poc_cycle += sps->offset_for_ref_frame[i];
 863
 864         // (8-8)
 865         poc_cycle_cnt = (abs_frame_num - 1) /
 866             sps->num_ref_frames_in_pic_order_cnt_cycle;
 867         frame_num_in_poc_cycle = (abs_frame_num - 1) %
 868             sps->num_ref_frames_in_pic_order_cnt_cycle;
 869
 870         // (8-9)
 871         expected_poc = poc_cycle_cnt * expected_delta_per_poc_cycle;
 872         for (i = 0; i <= frame_num_in_poc_cycle; i++)
 873             expected_poc += sps->offset_for_ref_frame[i];
 874     }
 875     else
 876         expected_poc = 0;
 877     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
 878         expected_poc += sps->offset_for_non_ref_pic;
 879
 880     // (8-10)
 881     if (!slice_hdr->field_pic_flag) {
 882         priv->field_poc[TOP_FIELD] = expected_poc +
 883             slice_hdr->delta_pic_order_cnt[0];
 884         priv->field_poc[BOTTOM_FIELD] = priv->field_poc[TOP_FIELD] +
 885             sps->offset_for_top_to_bottom_field +
 886             slice_hdr->delta_pic_order_cnt[1];
 887     }
 888     else if (!slice_hdr->bottom_field_flag)
 889         priv->field_poc[TOP_FIELD] = expected_poc +
 890             slice_hdr->delta_pic_order_cnt[0];
 891     else
 892         priv->field_poc[BOTTOM_FIELD] = expected_poc +
 893             sps->offset_for_top_to_bottom_field + slice_hdr->delta_pic_order_cnt[0];
 894 }
 895
 896 /* 8.2.1.3 - Decoding process for picture order count type 2 */
 897 static void
 898 init_picture_poc_2(
 899     GstVaapiDecoderH264 *decoder,
 900     GstVaapiPictureH264 *picture,
 901     GstH264SliceHdr     *slice_hdr
 902 )
 903 {
 904     GstVaapiDecoderH264Private * const priv = decoder->priv;
 905     GstH264PPS * const pps = slice_hdr->pps;
 906     GstH264SPS * const sps = pps->sequence;
 907     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
 908     guint temp_poc;
 909
 910     GST_DEBUG("decode picture order count type 2");
 911
 912     // (8-11)
 913     if (picture->is_idr)
 914         priv->frame_num_offset = 0;
 915     else if (priv->prev_frame_num > priv->frame_num)
 916         priv->frame_num_offset = priv->prev_frame_num_offset + MaxFrameNum;
 917     else
 918         priv->frame_num_offset = priv->prev_frame_num_offset;
 919
 920     // (8-12)
 921     if (picture->is_idr)
 922         temp_poc = 0;
 923     else if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
 924         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num) - 1;
 925     else
 926         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num);
 927
 928     // (8-13)
 929     if (!slice_hdr->field_pic_flag) {
 930         priv->field_poc[TOP_FIELD] = temp_poc;
 931         priv->field_poc[BOTTOM_FIELD] = temp_poc;
 932     }
 933     else if (slice_hdr->bottom_field_flag)
 934         priv->field_poc[BOTTOM_FIELD] = temp_poc;
 935     else
 936         priv->field_poc[TOP_FIELD] = temp_poc;
 937 }
 938
 939 /* 8.2.1 - Decoding process for picture order count */
 940 static void
 941 init_picture_poc(
 942     GstVaapiDecoderH264 *decoder,
 943     GstVaapiPictureH264 *picture,
 944     GstH264SliceHdr     *slice_hdr
 945 )
 946 {
 947     GstVaapiDecoderH264Private * const priv = decoder->priv;
 948     VAPictureH264 * const pic = &picture->info;
 949     GstH264PPS * const pps = slice_hdr->pps;
 950     GstH264SPS * const sps = pps->sequence;
 951
 952     switch (sps->pic_order_cnt_type) {
 953     case 0:
 954         init_picture_poc_0(decoder, picture, slice_hdr);
 955         break;
 956     case 1:
 957         init_picture_poc_1(decoder, picture, slice_hdr);
 958         break;
 959     case 2:
 960         init_picture_poc_2(decoder, picture, slice_hdr);
 961         break;
 962     }
 963
 964     if (!(pic->flags & VA_PICTURE_H264_BOTTOM_FIELD))
 965         pic->TopFieldOrderCnt = priv->field_poc[TOP_FIELD];
 966     if (!(pic->flags & VA_PICTURE_H264_TOP_FIELD))
 967         pic->BottomFieldOrderCnt = priv->field_poc[BOTTOM_FIELD];
 968     picture->poc = MIN(pic->TopFieldOrderCnt, pic->BottomFieldOrderCnt);
 969 }
 970
 971 static int
 972 compare_picture_pic_num_dec(const void *a, const void *b)
 973 {
 974     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
 975     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
 976
 977     return picB->pic_num - picA->pic_num;
 978 }
 979
 980 static int
 981 compare_picture_long_term_pic_num_inc(const void *a, const void *b)
 982 {
 983     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
 984     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
 985
 986     return picA->long_term_pic_num - picB->long_term_pic_num;
 987 }
 988
 989 static int
 990 compare_picture_poc_dec(const void *a, const void *b)
 991 {
 992     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
 993     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
 994
 995     return picB->poc - picA->poc;
 996 }
 997
 998 static int
 999 compare_picture_poc_inc(const void *a, const void *b)
1000 {
1001     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1002     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1003
1004     return picA->poc - picB->poc;
1005 }
1006
1007 static int
1008 compare_picture_frame_num_wrap_dec(const void *a, const void *b)
1009 {
1010     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1011     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1012
1013     return picB->frame_num_wrap - picA->frame_num_wrap;
1014 }
1015
1016 static int
1017 compare_picture_long_term_frame_idx_inc(const void *a, const void *b)
1018 {
1019     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1020     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1021
1022     return picA->info.frame_idx - picB->info.frame_idx;
1023 }
1024
1025 /* 8.2.4.1 - Decoding process for picture numbers */
1026 static void
1027 init_picture_refs_pic_num(
1028     GstVaapiDecoderH264 *decoder,
1029     GstVaapiPictureH264 *picture,
1030     GstH264SliceHdr     *slice_hdr
1031 )
1032 {
1033     GstVaapiDecoderH264Private * const priv = decoder->priv;
1034     GstH264PPS * const pps = slice_hdr->pps;
1035     GstH264SPS * const sps = pps->sequence;
1036     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1037     const guint field_flags = VA_PICTURE_H264_TOP_FIELD | VA_PICTURE_H264_BOTTOM_FIELD;
1038     guint i;
1039
1040     GST_DEBUG("decode picture numbers");
1041
1042     for (i = 0; i < priv->short_ref_count; i++) {
1043         GstVaapiPictureH264 * const pic = priv->short_ref[i];
1044
1045         // (8-27)
1046         if (pic->frame_num > priv->frame_num)
1047             pic->frame_num_wrap = pic->frame_num - MaxFrameNum;
1048         else
1049             pic->frame_num_wrap = pic->frame_num;
1050
1051         // (8-28, 8-30, 8-31)
1052         if (!pic->field_pic_flag)
1053             pic->pic_num = pic->frame_num_wrap;
1054         else {
1055             if (((picture->info.flags ^ pic->info.flags) & field_flags) == 0)
1056                 pic->pic_num = 2 * pic->frame_num_wrap + 1;
1057             else
1058                 pic->pic_num = 2 * pic->frame_num_wrap;
1059         }
1060     }
1061
1062     for (i = 0; i < priv->long_ref_count; i++) {
1063         GstVaapiPictureH264 * const pic = priv->long_ref[i];
1064
1065         // (8-29, 8-32, 8-33)
1066         if (!pic->field_pic_flag)
1067             pic->long_term_pic_num = pic->info.frame_idx;
1068         else {
1069             if (((picture->info.flags ^ pic->info.flags) & field_flags) == 0)
1070                 pic->long_term_pic_num = 2 * pic->info.frame_idx + 1;
1071             else
1072                 pic->long_term_pic_num = 2 * pic->info.frame_idx;
1073         }
1074     }
1075 }
1076
1077 #define SORT_REF_LIST(list, n, compare_func) \
1078     qsort(list, n, sizeof(*(list)), compare_picture_##compare_func)
1079
1080 static void
1081 init_picture_refs_p_slice(
1082     GstVaapiDecoderH264 *decoder,
1083     GstVaapiPictureH264 *picture,
1084     GstH264SliceHdr     *slice_hdr
1085 )
1086 {
1087     GstVaapiDecoderH264Private * const priv = decoder->priv;
1088     GstVaapiPictureH264 **ref_list;
1089     guint i;
1090
1091     GST_DEBUG("decode reference picture list for P and SP slices");
1092
1093     if (!picture->field_pic_flag) {
1094         /* 8.2.4.2.1 - P and SP slices in frames */
1095         if (priv->short_ref_count > 0) {
1096             ref_list = priv->RefPicList0;
1097             for (i = 0; i < priv->short_ref_count; i++)
1098                 ref_list[i] = priv->short_ref[i];
1099             SORT_REF_LIST(ref_list, i, pic_num_dec);
1100             priv->RefPicList0_count += i;
1101         }
1102
1103         if (priv->long_ref_count > 0) {
1104             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1105             for (i = 0; i < priv->long_ref_count; i++)
1106                 ref_list[i] = priv->long_ref[i];
1107             SORT_REF_LIST(ref_list, i, long_term_pic_num_inc);
1108             priv->RefPicList0_count += i;
1109         }
1110     }
1111     else {
1112         /* 8.2.4.2.2 - P and SP slices in fields */
1113         GstVaapiPictureH264 *short_ref[32];
1114         guint short_ref_count = 0;
1115         GstVaapiPictureH264 *long_ref[32];
1116         guint long_ref_count = 0;
1117
1118         // XXX: handle second field if current field is marked as
1119         // "used for short-term reference"
1120         if (priv->short_ref_count > 0) {
1121             for (i = 0; i < priv->short_ref_count; i++)
1122                 short_ref[i] = priv->short_ref[i];
1123             SORT_REF_LIST(short_ref, i, frame_num_wrap_dec);
1124             short_ref_count = i;
1125         }
1126
1127         // XXX: handle second field if current field is marked as
1128         // "used for long-term reference"
1129         if (priv->long_ref_count > 0) {
1130             for (i = 0; i < priv->long_ref_count; i++)
1131                 long_ref[i] = priv->long_ref[i];
1132             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1133             long_ref_count = i;
1134         }
1135
1136         // XXX: handle 8.2.4.2.5
1137     }
1138 }
1139
1140 static void
1141 init_picture_refs_b_slice(
1142     GstVaapiDecoderH264 *decoder,
1143     GstVaapiPictureH264 *picture,
1144     GstH264SliceHdr     *slice_hdr
1145 )
1146 {
1147     GstVaapiDecoderH264Private * const priv = decoder->priv;
1148     GstVaapiPictureH264 **ref_list;
1149     guint i, n;
1150
1151     GST_DEBUG("decode reference picture list for B slices");
1152
1153     if (!picture->field_pic_flag) {
1154         /* 8.2.4.2.3 - B slices in frames */
1155
1156         /* RefPicList0 */
1157         if (priv->short_ref_count > 0) {
1158             // 1. Short-term references
1159             ref_list = priv->RefPicList0;
1160             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1161                 if (priv->short_ref[i]->poc < picture->poc)
1162                     ref_list[n++] = priv->short_ref[i];
1163             }
1164             SORT_REF_LIST(ref_list, n, poc_dec);
1165             priv->RefPicList0_count += n;
1166
1167             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1168             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1169                 if (priv->short_ref[i]->poc >= picture->poc)
1170                     ref_list[n++] = priv->short_ref[i];
1171             }
1172             SORT_REF_LIST(ref_list, n, poc_inc);
1173             priv->RefPicList0_count += n;
1174         }
1175
1176         if (priv->long_ref_count > 0) {
1177             // 2. Long-term references
1178             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1179             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1180                 ref_list[n++] = priv->long_ref[i];
1181             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1182             priv->RefPicList0_count += n;
1183         }
1184
1185         /* RefPicList1 */
1186         if (priv->short_ref_count > 0) {
1187             // 1. Short-term references
1188             ref_list = priv->RefPicList1;
1189             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1190                 if (priv->short_ref[i]->poc > picture->poc)
1191                     ref_list[n++] = priv->short_ref[i];
1192             }
1193             SORT_REF_LIST(ref_list, n, poc_inc);
1194             priv->RefPicList1_count += n;
1195
1196             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1197             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1198                 if (priv->short_ref[i]->poc <= picture->poc)
1199                     ref_list[n++] = priv->short_ref[i];
1200             }
1201             SORT_REF_LIST(ref_list, n, poc_dec);
1202             priv->RefPicList1_count += n;
1203         }
1204
1205         if (priv->long_ref_count > 0) {
1206             // 2. Long-term references
1207             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1208             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1209                 ref_list[n++] = priv->long_ref[i];
1210             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1211             priv->RefPicList1_count += n;
1212         }
1213     }
1214     else {
1215         /* 8.2.4.2.4 - B slices in fields */
1216         GstVaapiPictureH264 *short_ref0[32];
1217         guint short_ref0_count = 0;
1218         GstVaapiPictureH264 *short_ref1[32];
1219         guint short_ref1_count = 0;
1220         GstVaapiPictureH264 *long_ref[32];
1221         guint long_ref_count = 0;
1222
1223         /* refFrameList0ShortTerm */
1224         if (priv->short_ref_count > 0) {
1225             ref_list = short_ref0;
1226             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1227                 if (priv->short_ref[i]->poc <= picture->poc)
1228                     ref_list[n++] = priv->short_ref[i];
1229             }
1230             SORT_REF_LIST(ref_list, n, poc_dec);
1231             short_ref0_count += n;
1232
1233             ref_list = &short_ref0[short_ref0_count];
1234             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1235                 if (priv->short_ref[i]->poc > picture->poc)
1236                     ref_list[n++] = priv->short_ref[i];
1237             }
1238             SORT_REF_LIST(ref_list, n, poc_inc);
1239             short_ref0_count += n;
1240         }
1241
1242         /* refFrameList1ShortTerm */
1243         if (priv->short_ref_count > 0) {
1244             ref_list = short_ref1;
1245             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1246                 if (priv->short_ref[i]->poc > picture->poc)
1247                     ref_list[n++] = priv->short_ref[i];
1248             }
1249             SORT_REF_LIST(ref_list, n, poc_inc);
1250             short_ref1_count += n;
1251
1252             ref_list = &short_ref1[short_ref1_count];
1253             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1254                 if (priv->short_ref[i]->poc <= picture->poc)
1255                     ref_list[n++] = priv->short_ref[i];
1256             }
1257             SORT_REF_LIST(ref_list, n, poc_dec);
1258             short_ref1_count += n;
1259         }
1260
1261         /* refFrameListLongTerm */
1262         if (priv->long_ref_count > 0) {
1263             for (i = 0; i < priv->long_ref_count; i++)
1264                 long_ref[i] = priv->long_ref[i];
1265             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1266             long_ref_count = i;
1267         }
1268
1269         // XXX: handle 8.2.4.2.5
1270     }
1271
1272     /* Check whether RefPicList1 is identical to RefPicList0, then
1273        swap if necessary */
1274     if (priv->RefPicList1_count > 1 &&
1275         priv->RefPicList1_count == priv->RefPicList0_count &&
1276         memcmp(priv->RefPicList0, priv->RefPicList1,
1277                priv->RefPicList0_count * sizeof(priv->RefPicList0[0])) == 0) {
1278         GstVaapiPictureH264 * const tmp = priv->RefPicList1[0];
1279         priv->RefPicList1[0] = priv->RefPicList1[1];
1280         priv->RefPicList1[1] = tmp;
1281     }
1282 }
1283
1284 #undef SORT_REF_LIST
1285
1286 static void
1287 clear_references(
1288     GstVaapiDecoderH264  *decoder,
1289     GstVaapiPictureH264 **pictures,
1290     guint                *picture_count
1291 )
1292 {
1293     const guint num_pictures = *picture_count;
1294     guint i;
1295
1296     for (i = 0; i < num_pictures; i++)
1297         gst_vaapi_picture_replace(&pictures[i], NULL);
1298     *picture_count = 0;
1299 }
1300
1301 static gboolean
1302 remove_reference_at(
1303     GstVaapiDecoderH264  *decoder,
1304     GstVaapiPictureH264 **pictures,
1305     guint                *picture_count,
1306     guint                 index
1307 )
1308 {
1309     guint num_pictures = *picture_count;
1310
1311     g_return_val_if_fail(index < num_pictures, FALSE);
1312
1313     GST_VAAPI_PICTURE_FLAG_UNSET(pictures[index], GST_VAAPI_PICTURE_FLAG_REFERENCE);
1314     if (index != --num_pictures)
1315         gst_vaapi_picture_replace(&pictures[index], pictures[num_pictures]);
1316     gst_vaapi_picture_replace(&pictures[num_pictures], NULL);
1317     *picture_count = num_pictures;
1318     return TRUE;
1319 }
1320
1321 static gint
1322 find_short_term_reference(GstVaapiDecoderH264 *decoder, gint32 pic_num)
1323 {
1324     GstVaapiDecoderH264Private * const priv = decoder->priv;
1325     guint i;
1326
1327     for (i = 0; i < priv->short_ref_count; i++) {
1328         if (priv->short_ref[i]->pic_num == pic_num)
1329             return i;
1330     }
1331     GST_ERROR("found no short-term reference picture with PicNum = %d",
1332               pic_num);
1333     return -1;
1334 }
1335
1336 static gint
1337 find_long_term_reference(GstVaapiDecoderH264 *decoder, gint32 long_term_pic_num)
1338 {
1339     GstVaapiDecoderH264Private * const priv = decoder->priv;
1340     guint i;
1341
1342     for (i = 0; i < priv->long_ref_count; i++) {
1343         if (priv->long_ref[i]->long_term_pic_num == long_term_pic_num)
1344             return i;
1345     }
1346     GST_ERROR("found no long-term reference picture with LongTermPicNum = %d",
1347               long_term_pic_num);
1348     return -1;
1349 }
1350
1351 static void
1352 exec_picture_refs_modification_1(
1353     GstVaapiDecoderH264           *decoder,
1354     GstVaapiPictureH264           *picture,
1355     GstH264SliceHdr               *slice_hdr,
1356     guint                          list
1357 )
1358 {
1359     GstVaapiDecoderH264Private * const priv = decoder->priv;
1360     GstH264PPS * const pps = slice_hdr->pps;
1361     GstH264SPS * const sps = pps->sequence;
1362     GstH264RefPicListModification *ref_pic_list_modification;
1363     guint num_ref_pic_list_modifications;
1364     GstVaapiPictureH264 **ref_list;
1365     guint *ref_list_count_ptr, ref_list_count, ref_list_idx = 0;
1366     guint i, j, n, num_refs;
1367     gint found_ref_idx;
1368     gint32 MaxPicNum, CurrPicNum, picNumPred;
1369
1370     GST_DEBUG("modification process of reference picture list %u", list);
1371
1372     if (list == 0) {
1373         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l0;
1374         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l0;
1375         ref_list                       = priv->RefPicList0;
1376         ref_list_count_ptr             = &priv->RefPicList0_count;
1377         num_refs                       = slice_hdr->num_ref_idx_l0_active_minus1 + 1;
1378     }
1379     else {
1380         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l1;
1381         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l1;
1382         ref_list                       = priv->RefPicList1;
1383         ref_list_count_ptr             = &priv->RefPicList1_count;
1384         num_refs                       = slice_hdr->num_ref_idx_l1_active_minus1 + 1;
1385     }
1386     ref_list_count = *ref_list_count_ptr;
1387
1388     if (picture->field_pic_flag) {
1389         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 5); // 2 * MaxFrameNum
1390         CurrPicNum = 2 * slice_hdr->frame_num + 1;              // 2 * frame_num + 1
1391     }
1392     else {
1393         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 4); // MaxFrameNum
1394         CurrPicNum = slice_hdr->frame_num;                      // frame_num
1395     }
1396
1397     picNumPred = CurrPicNum;
1398
1399     for (i = 0; i < num_ref_pic_list_modifications; i++) {
1400         GstH264RefPicListModification * const l = &ref_pic_list_modification[i];
1401         if (l->modification_of_pic_nums_idc == 3)
1402             break;
1403
1404         /* 8.2.4.3.1 - Short-term reference pictures */
1405         if (l->modification_of_pic_nums_idc == 0 || l->modification_of_pic_nums_idc == 1) {
1406             gint32 abs_diff_pic_num = l->value.abs_diff_pic_num_minus1 + 1;
1407             gint32 picNum, picNumNoWrap;
1408
1409             // (8-34)
1410             if (l->modification_of_pic_nums_idc == 0) {
1411                 picNumNoWrap = picNumPred - abs_diff_pic_num;
1412                 if (picNumNoWrap < 0)
1413                     picNumNoWrap += MaxPicNum;
1414             }
1415
1416             // (8-35)
1417             else {
1418                 picNumNoWrap = picNumPred + abs_diff_pic_num;
1419                 if (picNumNoWrap >= MaxPicNum)
1420                     picNumNoWrap -= MaxPicNum;
1421             }
1422             picNumPred = picNumNoWrap;
1423
1424             // (8-36)
1425             picNum = picNumNoWrap;
1426             if (picNum > CurrPicNum)
1427                 picNum -= MaxPicNum;
1428
1429             // (8-37)
1430             for (j = num_refs; j > ref_list_idx; j--)
1431                 ref_list[j] = ref_list[j - 1];
1432             found_ref_idx = find_short_term_reference(decoder, picNum);
1433             ref_list[ref_list_idx++] =
1434                 found_ref_idx >= 0 ? priv->short_ref[found_ref_idx] : NULL;
1435             n = ref_list_idx;
1436             for (j = ref_list_idx; j <= num_refs; j++) {
1437                 gint32 PicNumF;
1438                 if (!ref_list[j])
1439                     continue;
1440                 PicNumF = ref_list[j]->is_long_term ?
1441                     MaxPicNum : ref_list[j]->pic_num;
1442                 if (PicNumF != picNum)
1443                     ref_list[n++] = ref_list[j];
1444             }
1445         }
1446
1447         /* 8.2.4.3.2 - Long-term reference pictures */
1448         else {
1449
1450             for (j = num_refs; j > ref_list_idx; j--)
1451                 ref_list[j] = ref_list[j - 1];
1452             found_ref_idx =
1453                 find_long_term_reference(decoder, l->value.long_term_pic_num);
1454             ref_list[ref_list_idx++] =
1455                 found_ref_idx >= 0 ? priv->long_ref[found_ref_idx] : NULL;
1456             n = ref_list_idx;
1457             for (j = ref_list_idx; j <= num_refs; j++) {
1458                 gint32 LongTermPicNumF;
1459                 if (!ref_list[j])
1460                     continue;
1461                 LongTermPicNumF = ref_list[j]->is_long_term ?
1462                     ref_list[j]->long_term_pic_num : INT_MAX;
1463                 if (LongTermPicNumF != l->value.long_term_pic_num)
1464                     ref_list[n++] = ref_list[j];
1465             }
1466         }
1467     }
1468
1469 #if DEBUG
1470     for (i = 0; i < num_refs; i++)
1471         if (!ref_list[i])
1472             GST_ERROR("list %u entry %u is empty", list, i);
1473 #endif
1474     *ref_list_count_ptr = num_refs;
1475 }
1476
1477 /* 8.2.4.3 - Modification process for reference picture lists */
1478 static void
1479 exec_picture_refs_modification(
1480     GstVaapiDecoderH264 *decoder,
1481     GstVaapiPictureH264 *picture,
1482     GstH264SliceHdr     *slice_hdr
1483 )
1484 {
1485     GST_DEBUG("execute ref_pic_list_modification()");
1486
1487     /* RefPicList0 */
1488     if (!GST_H264_IS_I_SLICE(slice_hdr) && !GST_H264_IS_SI_SLICE(slice_hdr) &&
1489         slice_hdr->ref_pic_list_modification_flag_l0)
1490         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 0);
1491
1492     /* RefPicList1 */
1493     if (GST_H264_IS_B_SLICE(slice_hdr) &&
1494         slice_hdr->ref_pic_list_modification_flag_l1)
1495         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 1);
1496 }
1497
1498 static gboolean
1499 init_picture_refs(
1500     GstVaapiDecoderH264 *decoder,
1501     GstVaapiPictureH264 *picture,
1502     GstH264SliceHdr     *slice_hdr
1503 )
1504 {
1505     GstVaapiDecoderH264Private * const priv = decoder->priv;
1506     GstVaapiPicture * const base_picture = &picture->base;
1507     guint i, num_refs;
1508
1509     init_picture_refs_pic_num(decoder, picture, slice_hdr);
1510
1511     priv->RefPicList0_count = 0;
1512     priv->RefPicList1_count = 0;
1513
1514     switch (base_picture->type) {
1515     case GST_VAAPI_PICTURE_TYPE_P:
1516     case GST_VAAPI_PICTURE_TYPE_SP:
1517         init_picture_refs_p_slice(decoder, picture, slice_hdr);
1518         break;
1519     case GST_VAAPI_PICTURE_TYPE_B:
1520         init_picture_refs_b_slice(decoder, picture, slice_hdr);
1521         break;
1522     default:
1523         break;
1524     }
1525
1526     exec_picture_refs_modification(decoder, picture, slice_hdr);
1527
1528     switch (base_picture->type) {
1529     case GST_VAAPI_PICTURE_TYPE_B:
1530         num_refs = 1 + slice_hdr->num_ref_idx_l1_active_minus1;
1531         for (i = priv->RefPicList1_count; i < num_refs; i++)
1532             priv->RefPicList1[i] = NULL;
1533         priv->RefPicList1_count = num_refs;
1534
1535         // fall-through
1536     case GST_VAAPI_PICTURE_TYPE_P:
1537     case GST_VAAPI_PICTURE_TYPE_SP:
1538         num_refs = 1 + slice_hdr->num_ref_idx_l0_active_minus1;
1539         for (i = priv->RefPicList0_count; i < num_refs; i++)
1540             priv->RefPicList0[i] = NULL;
1541         priv->RefPicList0_count = num_refs;
1542         break;
1543     default:
1544         break;
1545     }
1546     return TRUE;
1547 }
1548
1549 static gboolean
1550 init_picture(
1551     GstVaapiDecoderH264 *decoder,
1552     GstVaapiPictureH264 *picture,
1553     GstH264SliceHdr     *slice_hdr,
1554     GstH264NalUnit      *nalu
1555 )
1556 {
1557     GstVaapiDecoderH264Private * const priv = decoder->priv;
1558     GstVaapiPicture * const base_picture = &picture->base;
1559     VAPictureH264 *pic;
1560     guint i;
1561
1562     priv->frame_num             = slice_hdr->frame_num;
1563     picture->frame_num          = priv->frame_num;
1564     picture->frame_num_wrap     = priv->frame_num;
1565     picture->is_idr             = nalu->type == GST_H264_NAL_SLICE_IDR;
1566     picture->field_pic_flag     = slice_hdr->field_pic_flag;
1567     picture->bottom_field_flag  = slice_hdr->bottom_field_flag;
1568     picture->output_flag        = TRUE; /* XXX: conformant to Annex A only */
1569     base_picture->pts           = gst_adapter_prev_timestamp(priv->adapter, NULL);
1570
1571     /* Reset decoder state for IDR pictures */
1572     if (picture->is_idr) {
1573         GST_DEBUG("<IDR>");
1574         clear_references(decoder, priv->short_ref, &priv->short_ref_count);
1575         clear_references(decoder, priv->long_ref,  &priv->long_ref_count );
1576         priv->prev_poc_msb = 0;
1577         priv->prev_poc_lsb = 0;
1578     }
1579
1580     /* Initialize VA picture info */
1581     pic = &picture->info;
1582     pic->picture_id = picture->base.surface_id;
1583     pic->frame_idx  = priv->frame_num;
1584     if (picture->field_pic_flag) {
1585         if (picture->bottom_field_flag)
1586             pic->flags |= VA_PICTURE_H264_BOTTOM_FIELD;
1587         else
1588             pic->flags |= VA_PICTURE_H264_TOP_FIELD;
1589     }
1590
1591     /* Initialize base picture */
1592     switch (slice_hdr->type % 5) {
1593     case GST_H264_P_SLICE:
1594         base_picture->type = GST_VAAPI_PICTURE_TYPE_P;
1595         break;
1596     case GST_H264_B_SLICE:
1597         base_picture->type = GST_VAAPI_PICTURE_TYPE_B;
1598         break;
1599     case GST_H264_I_SLICE:
1600         base_picture->type = GST_VAAPI_PICTURE_TYPE_I;
1601         break;
1602     case GST_H264_SP_SLICE:
1603         base_picture->type = GST_VAAPI_PICTURE_TYPE_SP;
1604         break;
1605     case GST_H264_SI_SLICE:
1606         base_picture->type = GST_VAAPI_PICTURE_TYPE_SI;
1607         break;
1608     }
1609
1610     if (nalu->ref_idc) {
1611         GstH264DecRefPicMarking * const dec_ref_pic_marking =
1612             &slice_hdr->dec_ref_pic_marking;
1613         GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_REFERENCE);
1614         if (picture->is_idr) {
1615             if (dec_ref_pic_marking->long_term_reference_flag)
1616                 picture->is_long_term = TRUE;
1617         }
1618         else if (dec_ref_pic_marking->adaptive_ref_pic_marking_mode_flag) {
1619             for (i = 0; i < dec_ref_pic_marking->n_ref_pic_marking; i++) {
1620                 GstH264RefPicMarking * const ref_pic_marking =
1621                     &dec_ref_pic_marking->ref_pic_marking[i];
1622                 switch (ref_pic_marking->memory_management_control_operation) {
1623                 case 3:
1624                 case 6:
1625                     picture->is_long_term = TRUE;
1626                     pic->frame_idx = ref_pic_marking->long_term_frame_idx;
1627                     break;
1628                 case 5:
1629                     picture->has_mmco_5 = TRUE;
1630                     break;
1631                 }
1632             }
1633         }
1634         if (picture->is_long_term)
1635             pic->flags |= VA_PICTURE_H264_LONG_TERM_REFERENCE;
1636         else
1637             pic->flags |= VA_PICTURE_H264_SHORT_TERM_REFERENCE;
1638     }
1639
1640     init_picture_poc(decoder, picture, slice_hdr);
1641     if (!init_picture_refs(decoder, picture, slice_hdr)) {
1642         GST_DEBUG("failed to initialize references");
1643         return FALSE;
1644     }
1645     return TRUE;
1646 }
1647
1648 /* 8.2.5.3 - Sliding window decoded reference picture marking process */
1649 static gboolean
1650 exec_ref_pic_marking_sliding_window(GstVaapiDecoderH264 *decoder)
1651 {
1652     GstVaapiDecoderH264Private * const priv = decoder->priv;
1653     GstH264SPS * const sps = priv->sps;
1654     guint i, max_num_ref_frames, lowest_frame_num_index;
1655     gint32 lowest_frame_num;
1656
1657     GST_DEBUG("reference picture marking process (sliding window)");
1658
1659     max_num_ref_frames = sps->num_ref_frames;
1660     if (max_num_ref_frames == 0)
1661         max_num_ref_frames = 1;
1662
1663     if (priv->short_ref_count + priv->long_ref_count < max_num_ref_frames)
1664         return TRUE;
1665     if (priv->short_ref_count < 1)
1666         return FALSE;
1667
1668     lowest_frame_num = priv->short_ref[0]->frame_num_wrap;
1669     lowest_frame_num_index = 0;
1670     for (i = 1; i < priv->short_ref_count; i++) {
1671         if (priv->short_ref[i]->frame_num_wrap < lowest_frame_num) {
1672             lowest_frame_num = priv->short_ref[i]->frame_num_wrap;
1673             lowest_frame_num_index = i;
1674         }
1675     }
1676
1677     remove_reference_at(
1678         decoder,
1679         priv->short_ref, &priv->short_ref_count,
1680         lowest_frame_num_index
1681     );
1682     return TRUE;
1683 }
1684
1685 /* 8.2.5.4. Adaptive memory control decoded reference picture marking process */
1686 static gboolean
1687 exec_ref_pic_marking_adaptive(
1688     GstVaapiDecoderH264     *decoder,
1689     GstVaapiPictureH264     *picture,
1690     GstH264DecRefPicMarking *dec_ref_pic_marking
1691 )
1692 {
1693     GstVaapiDecoderH264Private * const priv = decoder->priv;
1694     gint32 pic_num, ref_idx;
1695     guint i;
1696
1697     GST_DEBUG("reference picture marking process (adaptive memory control)");
1698
1699     for (i = 0; i < dec_ref_pic_marking->n_ref_pic_marking; i++) {
1700         GstH264RefPicMarking * const ref_pic_marking =
1701             &dec_ref_pic_marking->ref_pic_marking[i];
1702
1703         switch (ref_pic_marking->memory_management_control_operation) {
1704         case 1:
1705             // Mark short-term reference picture as "unused for reference"
1706             if (!picture->field_pic_flag)
1707                 pic_num = picture->frame_num_wrap;
1708             else
1709                 pic_num = 2 * picture->frame_num_wrap + 1;
1710             pic_num -= ref_pic_marking->difference_of_pic_nums_minus1 + 1;
1711             ref_idx = find_short_term_reference(decoder, pic_num);
1712             if (ref_idx < 0)
1713                 break;
1714             remove_reference_at(
1715                 decoder,
1716                 priv->short_ref, &priv->short_ref_count,
1717                 ref_idx
1718             );
1719             break;
1720         case 2:
1721             // Mark long-term reference picture as "unused for reference"
1722             pic_num = picture->long_term_pic_num;
1723             ref_idx = find_long_term_reference(decoder, pic_num);
1724             if (ref_idx < 0)
1725                 break;
1726             remove_reference_at(
1727                 decoder,
1728                 priv->long_ref, &priv->long_ref_count,
1729                 ref_idx
1730             );
1731             break;
1732         case 3:
1733             // Assign LongTermFrameIdx to a short-term reference picture
1734             if (!picture->field_pic_flag)
1735                 pic_num = picture->frame_num_wrap;
1736             else
1737                 pic_num = 2 * picture->frame_num_wrap + 1;
1738             pic_num -= ref_pic_marking->difference_of_pic_nums_minus1 + 1;
1739             ref_idx = find_short_term_reference(decoder, pic_num);
1740             if (ref_idx < 0)
1741                 break;
1742             break;
1743         case 5:
1744             // Mark all reference pictures as "unused for reference"
1745             clear_references(decoder, priv->short_ref, &priv->short_ref_count);
1746             clear_references(decoder, priv->long_ref,  &priv->long_ref_count );
1747             break;
1748         default:
1749             g_assert(0 && "unhandled MMCO");
1750             break;
1751         }
1752     }
1753     return TRUE;
1754 }
1755
1756 /* 8.2.5 - Execute reference picture marking process */
1757 static gboolean
1758 exec_ref_pic_marking(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
1759 {
1760     GstVaapiDecoderH264Private * const priv = decoder->priv;
1761     GstVaapiPictureH264 **picture_ptr;
1762
1763     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
1764         return TRUE;
1765
1766     if (!picture->is_idr) {
1767         GstH264DecRefPicMarking * const dec_ref_pic_marking =
1768             get_dec_ref_pic_marking(picture);
1769         if (dec_ref_pic_marking->adaptive_ref_pic_marking_mode_flag) {
1770             if (!exec_ref_pic_marking_adaptive(decoder, picture, dec_ref_pic_marking))
1771                 return FALSE;
1772         }
1773         else {
1774             if (!exec_ref_pic_marking_sliding_window(decoder))
1775                 return FALSE;
1776         }
1777     }
1778
1779     if (picture->is_long_term)
1780         picture_ptr = &priv->long_ref[priv->long_ref_count++];
1781     else
1782         picture_ptr = &priv->short_ref[priv->short_ref_count++];
1783     gst_vaapi_picture_replace(picture_ptr, picture);
1784     return TRUE;
1785 }
1786
1787 /* Update picture order count */
1788 static void
1789 exit_picture_poc(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
1790 {
1791     GstVaapiDecoderH264Private * const priv = decoder->priv;
1792     GstH264SPS * const sps = priv->sps;
1793
1794     switch (sps->pic_order_cnt_type) {
1795     case 0:
1796         if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
1797             break;
1798         if (picture->has_mmco_5) {
1799             priv->prev_poc_msb = 0;
1800             if (!picture->field_pic_flag || !picture->bottom_field_flag)
1801                 priv->prev_poc_lsb = picture->info.TopFieldOrderCnt;
1802             else
1803                 priv->prev_poc_lsb = 0;
1804         }
1805         else {
1806             priv->prev_poc_msb = priv->poc_msb;
1807             priv->prev_poc_lsb = priv->poc_lsb;
1808         }
1809         break;
1810     case 1:
1811     case 2:
1812         priv->prev_frame_num = priv->frame_num;
1813         if (picture->has_mmco_5)
1814             priv->prev_frame_num_offset = 0;
1815         else
1816             priv->prev_frame_num_offset = priv->frame_num_offset;
1817         break;
1818     }
1819 }
1820
1821 static inline gboolean
1822 exit_picture(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
1823 {
1824     /* Update picture order count */
1825     exit_picture_poc(decoder, picture);
1826
1827     /* Decoded reference picture marking process */
1828     if (!exec_ref_pic_marking(decoder, picture))
1829         return FALSE;
1830     return TRUE;
1831 }
1832
1833 static void
1834 vaapi_init_picture(VAPictureH264 *pic)
1835 {
1836     pic->picture_id           = VA_INVALID_ID;
1837     pic->frame_idx            = 0;
1838     pic->flags                = VA_PICTURE_H264_INVALID;
1839     pic->TopFieldOrderCnt     = 0;
1840     pic->BottomFieldOrderCnt  = 0;
1841 }
1842
1843 static gboolean
1844 fill_picture(
1845     GstVaapiDecoderH264 *decoder,
1846     GstVaapiPictureH264 *picture,
1847     GstH264SliceHdr     *slice_hdr,
1848     GstH264NalUnit      *nalu
1849 )
1850 {
1851     GstVaapiDecoderH264Private * const priv = decoder->priv;
1852     GstVaapiPicture * const base_picture = &picture->base;
1853     GstH264SPS * const sps = priv->sps;
1854     GstH264PPS * const pps = priv->pps;
1855     VAPictureParameterBufferH264 * const pic_param = base_picture->param;
1856     guint i, n;
1857
1858     /* Fill in VAPictureParameterBufferH264 */
1859     pic_param->CurrPic = picture->info;
1860     for (i = 0, n = 0; i < priv->short_ref_count; i++)
1861         pic_param->ReferenceFrames[n++] = priv->short_ref[i]->info;
1862     for (i = 0; i < priv->long_ref_count; i++)
1863         pic_param->ReferenceFrames[n++] = priv->long_ref[i]->info;
1864     for (; n < G_N_ELEMENTS(pic_param->ReferenceFrames); n++)
1865         vaapi_init_picture(&pic_param->ReferenceFrames[n]);
1866
1867 #define COPY_FIELD(s, f) \
1868     pic_param->f = (s)->f
1869
1870 #define COPY_BFM(a, s, f) \
1871     pic_param->a.bits.f = (s)->f
1872
1873     pic_param->picture_width_in_mbs_minus1      = priv->mb_width  - 1;
1874     pic_param->picture_height_in_mbs_minus1     = priv->mb_height - 1;
1875     pic_param->frame_num                        = priv->frame_num;
1876
1877     COPY_FIELD(sps, bit_depth_luma_minus8);
1878     COPY_FIELD(sps, bit_depth_chroma_minus8);
1879     COPY_FIELD(sps, num_ref_frames);
1880     COPY_FIELD(pps, num_slice_groups_minus1);
1881     COPY_FIELD(pps, slice_group_map_type);
1882     COPY_FIELD(pps, slice_group_change_rate_minus1);
1883     COPY_FIELD(pps, pic_init_qp_minus26);
1884     COPY_FIELD(pps, pic_init_qs_minus26);
1885     COPY_FIELD(pps, chroma_qp_index_offset);
1886     COPY_FIELD(pps, second_chroma_qp_index_offset);
1887
1888     pic_param->seq_fields.value                                         = 0; /* reset all bits */
1889     pic_param->seq_fields.bits.residual_colour_transform_flag           = sps->separate_colour_plane_flag;
1890     pic_param->seq_fields.bits.MinLumaBiPredSize8x8                     = sps->level_idc >= 31; /* A.3.3.2 */
1891
1892     COPY_BFM(seq_fields, sps, chroma_format_idc);
1893     COPY_BFM(seq_fields, sps, gaps_in_frame_num_value_allowed_flag);
1894     COPY_BFM(seq_fields, sps, frame_mbs_only_flag);
1895     COPY_BFM(seq_fields, sps, mb_adaptive_frame_field_flag);
1896     COPY_BFM(seq_fields, sps, direct_8x8_inference_flag);
1897     COPY_BFM(seq_fields, sps, log2_max_frame_num_minus4);
1898     COPY_BFM(seq_fields, sps, pic_order_cnt_type);
1899     COPY_BFM(seq_fields, sps, log2_max_pic_order_cnt_lsb_minus4);
1900     COPY_BFM(seq_fields, sps, delta_pic_order_always_zero_flag);
1901
1902     pic_param->pic_fields.value                                         = 0; /* reset all bits */
1903     pic_param->pic_fields.bits.field_pic_flag                           = slice_hdr->field_pic_flag;
1904     pic_param->pic_fields.bits.reference_pic_flag                       = GST_VAAPI_PICTURE_IS_REFERENCE(picture);
1905
1906     COPY_BFM(pic_fields, pps, entropy_coding_mode_flag);
1907     COPY_BFM(pic_fields, pps, weighted_pred_flag);
1908     COPY_BFM(pic_fields, pps, weighted_bipred_idc);
1909     COPY_BFM(pic_fields, pps, transform_8x8_mode_flag);
1910     COPY_BFM(pic_fields, pps, constrained_intra_pred_flag);
1911     COPY_BFM(pic_fields, pps, pic_order_present_flag);
1912     COPY_BFM(pic_fields, pps, deblocking_filter_control_present_flag);
1913     COPY_BFM(pic_fields, pps, redundant_pic_cnt_present_flag);
1914     return TRUE;
1915 }
1916
1917 static gboolean
1918 fill_quant_matrix(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
1919 {
1920     GstVaapiDecoderH264Private * const priv = decoder->priv;
1921     VAIQMatrixBufferH264 * const iq_matrix = picture->base.iq_matrix->param;
1922
1923     /* XXX: we can only support 4:2:0 or 4:2:2 since ScalingLists8x8[]
1924        is not large enough to hold lists for 4:4:4 */
1925     if (priv->sps->chroma_format_idc == 3 &&
1926         sizeof(iq_matrix->ScalingList8x8) != sizeof(priv->scaling_list_8x8))
1927         return FALSE;
1928
1929     /* Fill in VAIQMatrixBufferH264 */
1930     memcpy(iq_matrix->ScalingList4x4, priv->scaling_list_4x4,
1931            sizeof(iq_matrix->ScalingList4x4));
1932     memcpy(iq_matrix->ScalingList8x8, priv->scaling_list_8x8,
1933            sizeof(iq_matrix->ScalingList8x8));
1934     return TRUE;
1935 }
1936
1937 static GstVaapiDecoderStatus
1938 decode_picture(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu, GstH264SliceHdr *slice_hdr)
1939 {
1940     GstVaapiDecoderH264Private * const priv = decoder->priv;
1941     GstVaapiPictureH264 *picture;
1942     GstVaapiDecoderStatus status;
1943     GstH264PPS * const pps = slice_hdr->pps;
1944     GstH264SPS * const sps = pps->sequence;
1945
1946     status = ensure_context(decoder, sps);
1947     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS) {
1948         GST_DEBUG("failed to reset context");
1949         return status;
1950     }
1951
1952     if (priv->current_picture && !decode_current_picture(decoder))
1953         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
1954
1955     picture = gst_vaapi_picture_h264_new(decoder);
1956     if (!picture) {
1957         GST_DEBUG("failed to allocate picture");
1958         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
1959     }
1960     priv->current_picture = picture;
1961
1962     picture->base.iq_matrix = GST_VAAPI_IQ_MATRIX_NEW(H264, decoder);
1963     if (!picture->base.iq_matrix) {
1964         GST_DEBUG("failed to allocate IQ matrix");
1965         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
1966     }
1967
1968     status = ensure_quant_matrix(decoder, pps);
1969     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS) {
1970         GST_DEBUG("failed to reset quantizer matrix");
1971         return status;
1972     }
1973
1974     priv->sps = sps;
1975     priv->pps = pps;
1976
1977     if (!init_picture(decoder, picture, slice_hdr, nalu))
1978         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
1979     if (!fill_picture(decoder, picture, slice_hdr, nalu))
1980         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
1981     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1982 }
1983
1984 static gboolean
1985 decode_picture_end(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
1986 {
1987     if (!fill_quant_matrix(decoder, picture))
1988         return FALSE;
1989     if (!exit_picture(decoder, picture))
1990         return FALSE;
1991     if (!dpb_add(decoder, picture))
1992         return FALSE;
1993     return TRUE;
1994 }
1995
1996 #ifndef HAVE_GST_H264_SLICE_HDR_EPB_COUNT
1997 static guint
1998 get_epb_count(const guint8 *buf, guint buf_size, guint header_size)
1999 {
2000     guint i, n = 0;
2001
2002     if (buf_size > header_size)
2003         buf_size = header_size;
2004
2005     for (i = 2; i < buf_size; i++) {
2006         if (!buf[i - 2] && !buf[i - 1] && buf[i] == 0x03)
2007             i += 2, n++;
2008     }
2009     return n;
2010 }
2011 #endif
2012
2013 static inline guint
2014 get_slice_data_bit_offset(GstH264SliceHdr *slice_hdr, GstH264NalUnit *nalu)
2015 {
2016     guint epb_count;
2017
2018 #ifdef HAVE_GST_H264_SLICE_HDR_EPB_COUNT
2019     epb_count = slice_hdr->n_emulation_prevention_bytes;
2020 #else
2021     epb_count = get_epb_count(
2022         nalu->data + nalu->offset,
2023         nalu->size,
2024         slice_hdr->header_size / 8
2025     );
2026 #endif
2027     return 8 /* nal_unit_type */ + slice_hdr->header_size - epb_count * 8;
2028 }
2029
2030 static gboolean
2031 fill_pred_weight_table(GstVaapiDecoderH264 *decoder, GstVaapiSliceH264 *slice)
2032 {
2033     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2034     GstH264PPS * const pps = slice_hdr->pps;
2035     GstH264SPS * const sps = pps->sequence;
2036     GstH264PredWeightTable * const w = &slice_hdr->pred_weight_table;
2037     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2038     guint num_weight_tables = 0;
2039     gint i, j;
2040
2041     if (pps->weighted_pred_flag &&
2042         (GST_H264_IS_P_SLICE(slice_hdr) || GST_H264_IS_SP_SLICE(slice_hdr)))
2043         num_weight_tables = 1;
2044     else if (pps->weighted_bipred_idc == 1 && GST_H264_IS_B_SLICE(slice_hdr))
2045         num_weight_tables = 2;
2046     else
2047         num_weight_tables = 0;
2048
2049     slice_param->luma_log2_weight_denom   = w->luma_log2_weight_denom;
2050     slice_param->chroma_log2_weight_denom = w->chroma_log2_weight_denom;
2051     slice_param->luma_weight_l0_flag      = 0;
2052     slice_param->chroma_weight_l0_flag    = 0;
2053     slice_param->luma_weight_l1_flag      = 0;
2054     slice_param->chroma_weight_l1_flag    = 0;
2055
2056     if (num_weight_tables < 1)
2057         return TRUE;
2058
2059     slice_param->luma_weight_l0_flag = 1;
2060     for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2061         slice_param->luma_weight_l0[i] = w->luma_weight_l0[i];
2062         slice_param->luma_offset_l0[i] = w->luma_offset_l0[i];
2063     }
2064
2065     slice_param->chroma_weight_l0_flag = sps->chroma_array_type != 0;
2066     if (slice_param->chroma_weight_l0_flag) {
2067         for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2068             for (j = 0; j < 2; j++) {
2069                 slice_param->chroma_weight_l0[i][j] = w->chroma_weight_l0[i][j];
2070                 slice_param->chroma_offset_l0[i][j] = w->chroma_offset_l0[i][j];
2071             }
2072         }
2073     }
2074
2075     if (num_weight_tables < 2)
2076         return TRUE;
2077
2078     slice_param->luma_weight_l1_flag = 1;
2079     for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2080         slice_param->luma_weight_l1[i] = w->luma_weight_l1[i];
2081         slice_param->luma_offset_l1[i] = w->luma_offset_l1[i];
2082     }
2083
2084     slice_param->chroma_weight_l1_flag = sps->chroma_array_type != 0;
2085     if (slice_param->chroma_weight_l1_flag) {
2086         for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2087             for (j = 0; j < 2; j++) {
2088                 slice_param->chroma_weight_l1[i][j] = w->chroma_weight_l1[i][j];
2089                 slice_param->chroma_offset_l1[i][j] = w->chroma_offset_l1[i][j];
2090             }
2091         }
2092     }
2093     return TRUE;
2094 }
2095
2096 static gboolean
2097 fill_RefPicList(GstVaapiDecoderH264 *decoder, GstVaapiSliceH264 *slice)
2098 {
2099     GstVaapiDecoderH264Private * const priv = decoder->priv;
2100     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2101     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2102     guint i, num_ref_lists = 0;
2103
2104     slice_param->num_ref_idx_l0_active_minus1 = 0;
2105     slice_param->num_ref_idx_l1_active_minus1 = 0;
2106
2107     if (GST_H264_IS_B_SLICE(slice_hdr))
2108         num_ref_lists = 2;
2109     else if (GST_H264_IS_I_SLICE(slice_hdr))
2110         num_ref_lists = 0;
2111     else
2112         num_ref_lists = 1;
2113
2114     if (num_ref_lists < 1)
2115         return TRUE;
2116
2117     slice_param->num_ref_idx_l0_active_minus1 =
2118         slice_hdr->num_ref_idx_l0_active_minus1;
2119
2120     for (i = 0; i < priv->RefPicList0_count && priv->RefPicList0[i]; i++)
2121         slice_param->RefPicList0[i] = priv->RefPicList0[i]->info;
2122     for (; i <= slice_param->num_ref_idx_l0_active_minus1; i++)
2123         vaapi_init_picture(&slice_param->RefPicList0[i]);
2124
2125     if (num_ref_lists < 2)
2126         return TRUE;
2127
2128     slice_param->num_ref_idx_l1_active_minus1 =
2129         slice_hdr->num_ref_idx_l1_active_minus1;
2130
2131     for (i = 0; i < priv->RefPicList1_count && priv->RefPicList1[i]; i++)
2132         slice_param->RefPicList1[i] = priv->RefPicList1[i]->info;
2133     for (; i <= slice_param->num_ref_idx_l1_active_minus1; i++)
2134         vaapi_init_picture(&slice_param->RefPicList1[i]);
2135     return TRUE;
2136 }
2137
2138 static gboolean
2139 fill_slice(
2140     GstVaapiDecoderH264 *decoder,
2141     GstVaapiSliceH264   *slice,
2142     GstH264NalUnit      *nalu
2143 )
2144 {
2145     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2146     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2147
2148     /* Fill in VASliceParameterBufferH264 */
2149     slice_param->slice_data_bit_offset          = get_slice_data_bit_offset(slice_hdr, nalu);
2150     slice_param->first_mb_in_slice              = slice_hdr->first_mb_in_slice;
2151     slice_param->slice_type                     = slice_hdr->type % 5;
2152     slice_param->direct_spatial_mv_pred_flag    = slice_hdr->direct_spatial_mv_pred_flag;
2153     slice_param->cabac_init_idc                 = slice_hdr->cabac_init_idc;
2154     slice_param->slice_qp_delta                 = slice_hdr->slice_qp_delta;
2155     slice_param->disable_deblocking_filter_idc  = slice_hdr->disable_deblocking_filter_idc;
2156     slice_param->slice_alpha_c0_offset_div2     = slice_hdr->slice_alpha_c0_offset_div2;
2157     slice_param->slice_beta_offset_div2         = slice_hdr->slice_beta_offset_div2;
2158
2159     if (!fill_RefPicList(decoder, slice))
2160         return FALSE;
2161     if (!fill_pred_weight_table(decoder, slice))
2162         return FALSE;
2163     return TRUE;
2164 }
2165
2166 static GstVaapiDecoderStatus
2167 decode_slice(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
2168 {
2169     GstVaapiDecoderH264Private * const priv = decoder->priv;
2170     GstVaapiDecoderStatus status;
2171     GstVaapiPictureH264 *picture;
2172     GstVaapiSliceH264 *slice = NULL;
2173     GstH264SliceHdr *slice_hdr;
2174     GstH264ParserResult result;
2175
2176     GST_DEBUG("slice (%u bytes)", nalu->size);
2177
2178     slice = gst_vaapi_slice_h264_new(
2179         decoder,
2180         nalu->data + nalu->offset,
2181         nalu->size
2182     );
2183     if (!slice) {
2184         GST_DEBUG("failed to allocate slice");
2185         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2186     }
2187
2188     slice_hdr = &slice->slice_hdr;
2189     memset(slice_hdr, 0, sizeof(*slice_hdr));
2190     result = gst_h264_parser_parse_slice_hdr(priv->parser, nalu, slice_hdr, TRUE, TRUE);
2191     if (result != GST_H264_PARSER_OK) {
2192         status = get_status(result);
2193         goto error;
2194     }
2195
2196     if (slice_hdr->first_mb_in_slice == 0) {
2197         status = decode_picture(decoder, nalu, slice_hdr);
2198         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2199             goto error;
2200     }
2201     picture = priv->current_picture;
2202
2203     priv->mb_x = slice_hdr->first_mb_in_slice % priv->mb_width;
2204     priv->mb_y = slice_hdr->first_mb_in_slice / priv->mb_width; // FIXME: MBAFF or field
2205
2206     if (!fill_slice(decoder, slice, nalu)) {
2207         status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2208         goto error;
2209     }
2210     gst_vaapi_picture_add_slice(
2211         GST_VAAPI_PICTURE_CAST(picture),
2212         GST_VAAPI_SLICE_CAST(slice)
2213     );
2214
2215     /* Commit picture for decoding if we reached the last slice */
2216     if (++priv->mb_y >= priv->mb_height) {
2217         if (!decode_current_picture(decoder)) {
2218             status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2219             goto error;
2220         }
2221         GST_DEBUG("done");
2222     }
2223     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2224
2225 error:
2226     if (slice)
2227         gst_mini_object_unref(GST_MINI_OBJECT(slice));
2228     return status;
2229 }
2230
2231 static GstVaapiDecoderStatus
2232 decode_buffer(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
2233 {
2234     GstVaapiDecoderH264Private * const priv = decoder->priv;
2235     GstVaapiDecoderStatus status;
2236     GstH264ParserResult result;
2237     GstH264NalUnit nalu;
2238     guchar *buf;
2239     guint buf_size, ofs;
2240
2241     buf      = GST_BUFFER_DATA(buffer);
2242     buf_size = GST_BUFFER_SIZE(buffer);
2243     if (!buf && buf_size == 0)
2244         return decode_sequence_end(decoder);
2245
2246     gst_buffer_ref(buffer);
2247     gst_adapter_push(priv->adapter, buffer);
2248
2249     if (priv->sub_buffer) {
2250         buffer = gst_buffer_merge(priv->sub_buffer, buffer);
2251         if (!buffer)
2252             return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2253         gst_buffer_unref(priv->sub_buffer);
2254         priv->sub_buffer = NULL;
2255     }
2256
2257     buf      = GST_BUFFER_DATA(buffer);
2258     buf_size = GST_BUFFER_SIZE(buffer);
2259     ofs      = 0;
2260     do {
2261         if (priv->is_avc) {
2262             result = gst_h264_parser_identify_nalu_avc(
2263                 priv->parser,
2264                 buf, ofs, buf_size, priv->nal_length_size,
2265                 &nalu
2266             );
2267         }
2268         else {
2269             result = gst_h264_parser_identify_nalu(
2270                 priv->parser,
2271                 buf, ofs, buf_size,
2272                 &nalu
2273             );
2274         }
2275         status = get_status(result);
2276
2277         if (status == GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA) {
2278             priv->sub_buffer = gst_buffer_create_sub(buffer, ofs, buf_size - ofs);
2279             break;
2280         }
2281         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2282             break;
2283
2284         ofs = nalu.offset - ofs;
2285         if (gst_adapter_available(priv->adapter) >= ofs)
2286             gst_adapter_flush(priv->adapter, ofs);
2287
2288         switch (nalu.type) {
2289         case GST_H264_NAL_SLICE_IDR:
2290             /* fall-through. IDR specifics are handled in init_picture() */
2291         case GST_H264_NAL_SLICE:
2292             status = decode_slice(decoder, &nalu);
2293             break;
2294         case GST_H264_NAL_SPS:
2295             status = decode_sps(decoder, &nalu);
2296             break;
2297         case GST_H264_NAL_PPS:
2298             status = decode_pps(decoder, &nalu);
2299             break;
2300         case GST_H264_NAL_SEI:
2301             status = decode_sei(decoder, &nalu);
2302             break;
2303         case GST_H264_NAL_SEQ_END:
2304             status = decode_sequence_end(decoder);
2305             break;
2306         case GST_H264_NAL_AU_DELIMITER:
2307             /* skip all Access Unit NALs */
2308             status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2309             break;
2310         case GST_H264_NAL_FILLER_DATA:
2311             /* skip all Filler Data NALs */
2312             status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2313             break;
2314         default:
2315             GST_DEBUG("unsupported NAL unit type %d", nalu.type);
2316             status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
2317             break;
2318         }
2319
2320         if (gst_adapter_available(priv->adapter) >= nalu.size)
2321             gst_adapter_flush(priv->adapter, nalu.size);
2322         ofs = nalu.offset + nalu.size;
2323     } while (status == GST_VAAPI_DECODER_STATUS_SUCCESS && ofs < buf_size);
2324     return status;
2325 }
2326
2327 static GstVaapiDecoderStatus
2328 decode_codec_data(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
2329 {
2330     GstVaapiDecoderH264Private * const priv = decoder->priv;
2331     GstVaapiDecoderStatus status;
2332     GstH264NalUnit nalu;
2333     GstH264ParserResult result;
2334     guchar *buf;
2335     guint buf_size;
2336     guint i, ofs, num_sps, num_pps;
2337
2338     buf      = GST_BUFFER_DATA(buffer);
2339     buf_size = GST_BUFFER_SIZE(buffer);
2340     if (!buf || buf_size == 0)
2341         return GST_VAAPI_DECODER_STATUS_SUCCESS;
2342
2343     if (buf_size < 8)
2344         return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2345
2346     if (buf[0] != 1) {
2347         GST_DEBUG("failed to decode codec-data, not in avcC format");
2348         return GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
2349     }
2350
2351     priv->nal_length_size = (buf[4] & 0x03) + 1;
2352
2353     num_sps = buf[5] & 0x1f;
2354     ofs = 6;
2355
2356     for (i = 0; i < num_sps; i++) {
2357         result = gst_h264_parser_identify_nalu_avc(
2358             priv->parser,
2359             buf, ofs, buf_size, 2,
2360             &nalu
2361         );
2362         if (result != GST_H264_PARSER_OK)
2363             return get_status(result);
2364
2365         status = decode_sps(decoder, &nalu);
2366         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2367             return status;
2368         ofs = nalu.offset + nalu.size;
2369     }
2370
2371     num_pps = buf[ofs];
2372     ofs++;
2373
2374     for (i = 0; i < num_pps; i++) {
2375         result = gst_h264_parser_identify_nalu_avc(
2376             priv->parser,
2377             buf, ofs, buf_size, 2,
2378             &nalu
2379         );
2380         if (result != GST_H264_PARSER_OK)
2381             return get_status(result);
2382
2383         status = decode_pps(decoder, &nalu);
2384         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2385             return status;
2386         ofs = nalu.offset + nalu.size;
2387     }
2388
2389     priv->is_avc = TRUE;
2390     return status;
2391 }
2392
2393 GstVaapiDecoderStatus
2394 gst_vaapi_decoder_h264_decode(GstVaapiDecoder *base, GstBuffer *buffer)
2395 {
2396     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264(base);
2397     GstVaapiDecoderH264Private * const priv = decoder->priv;
2398     GstVaapiDecoderStatus status;
2399     GstBuffer *codec_data;
2400
2401     g_return_val_if_fail(priv->is_constructed,
2402                          GST_VAAPI_DECODER_STATUS_ERROR_INIT_FAILED);
2403
2404     if (!priv->is_opened) {
2405         priv->is_opened = gst_vaapi_decoder_h264_open(decoder, buffer);
2406         if (!priv->is_opened)
2407             return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CODEC;
2408
2409         codec_data = GST_VAAPI_DECODER_CODEC_DATA(decoder);
2410         if (codec_data) {
2411             status = decode_codec_data(decoder, codec_data);
2412             if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2413                 return status;
2414         }
2415      }
2416      return decode_buffer(decoder, buffer);
2417 }
2418
2419 static void
2420 gst_vaapi_decoder_h264_finalize(GObject *object)
2421 {
2422     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264(object);
2423
2424     gst_vaapi_decoder_h264_destroy(decoder);
2425
2426     G_OBJECT_CLASS(gst_vaapi_decoder_h264_parent_class)->finalize(object);
2427 }
2428
2429 static void
2430 gst_vaapi_decoder_h264_constructed(GObject *object)
2431 {
2432     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264(object);
2433     GstVaapiDecoderH264Private * const priv = decoder->priv;
2434     GObjectClass *parent_class;
2435
2436     parent_class = G_OBJECT_CLASS(gst_vaapi_decoder_h264_parent_class);
2437     if (parent_class->constructed)
2438         parent_class->constructed(object);
2439
2440     priv->is_constructed = gst_vaapi_decoder_h264_create(decoder);
2441 }
2442
2443 static void
2444 gst_vaapi_decoder_h264_class_init(GstVaapiDecoderH264Class *klass)
2445 {
2446     GObjectClass * const object_class = G_OBJECT_CLASS(klass);
2447     GstVaapiDecoderClass * const decoder_class = GST_VAAPI_DECODER_CLASS(klass);
2448
2449     g_type_class_add_private(klass, sizeof(GstVaapiDecoderH264Private));
2450
2451     object_class->finalize      = gst_vaapi_decoder_h264_finalize;
2452     object_class->constructed   = gst_vaapi_decoder_h264_constructed;
2453
2454     decoder_class->decode       = gst_vaapi_decoder_h264_decode;
2455     decoder_class->clear_buffer = gst_vaapi_decoder_h264_clear_buffer;
2456 }
2457
2458 static void
2459 gst_vaapi_decoder_h264_init(GstVaapiDecoderH264 *decoder)
2460 {
2461     GstVaapiDecoderH264Private *priv;
2462
2463     priv                        = GST_VAAPI_DECODER_H264_GET_PRIVATE(decoder);
2464     decoder->priv               = priv;
2465     priv->parser                = NULL;
2466     priv->sps                   = &priv->last_sps;
2467     priv->pps                   = &priv->last_pps;
2468     priv->current_picture       = NULL;
2469     priv->dpb_count             = 0;
2470     priv->dpb_size              = 0;
2471     priv->profile               = GST_VAAPI_PROFILE_H264_HIGH;
2472     priv->short_ref_count       = 0;
2473     priv->long_ref_count        = 0;
2474     priv->RefPicList0_count     = 0;
2475     priv->RefPicList1_count     = 0;
2476     priv->nal_length_size       = 0;
2477     priv->width                 = 0;
2478     priv->height                = 0;
2479     priv->mb_x                  = 0;
2480     priv->mb_y                  = 0;
2481     priv->mb_width              = 0;
2482     priv->mb_height             = 0;
2483     priv->adapter               = NULL;
2484     priv->sub_buffer            = NULL;
2485     priv->field_poc[0]          = 0;
2486     priv->field_poc[1]          = 0;
2487     priv->poc_msb               = 0;
2488     priv->poc_lsb               = 0;
2489     priv->prev_poc_msb          = 0;
2490     priv->prev_poc_lsb          = 0;
2491     priv->frame_num_offset      = 0;
2492     priv->prev_frame_num_offset = 0;
2493     priv->frame_num             = 0;
2494     priv->prev_frame_num        = 0;
2495     priv->is_constructed        = FALSE;
2496     priv->is_opened             = FALSE;
2497     priv->is_avc                = FALSE;
2498     priv->has_context           = FALSE;
2499
2500     memset(priv->dpb, 0, sizeof(priv->dpb));
2501     memset(priv->short_ref, 0, sizeof(priv->short_ref));
2502     memset(priv->long_ref, 0, sizeof(priv->long_ref));
2503     memset(priv->RefPicList0, 0, sizeof(priv->RefPicList0));
2504     memset(priv->RefPicList1, 0, sizeof(priv->RefPicList1));
2505 }
2506
2507 /**
2508  * gst_vaapi_decoder_h264_new:
2509  * @display: a #GstVaapiDisplay
2510  * @caps: a #GstCaps holding codec information
2511  *
2512  * Creates a new #GstVaapiDecoder for MPEG-2 decoding.  The @caps can
2513  * hold extra information like codec-data and pictured coded size.
2514  *
2515  * Return value: the newly allocated #GstVaapiDecoder object
2516  */
2517 GstVaapiDecoder *
2518 gst_vaapi_decoder_h264_new(GstVaapiDisplay *display, GstCaps *caps)
2519 {
2520     GstVaapiDecoderH264 *decoder;
2521
2522     g_return_val_if_fail(GST_VAAPI_IS_DISPLAY(display), NULL);
2523     g_return_val_if_fail(GST_IS_CAPS(caps), NULL);
2524
2525     decoder = g_object_new(
2526         GST_VAAPI_TYPE_DECODER_H264,
2527         "display",      display,
2528         "caps",         caps,
2529         NULL
2530     );
2531     if (!decoder->priv->is_constructed) {
2532         g_object_unref(decoder);
2533         return NULL;
2534     }
2535     return GST_VAAPI_DECODER_CAST(decoder);
2536 }