gst-libs/gst/vaapi/gstvaapidecoder_h264.c

   1 /*
   2  *  gstvaapidecoder_h264.c - H.264 decoder
   3  *
   4  *  Copyright (C) 2011-2012 Intel Corporation
   5  *
   6  *  This library is free software; you can redistribute it and/or
   7  *  modify it under the terms of the GNU Lesser General Public License
   8  *  as published by the Free Software Foundation; either version 2.1
   9  *  of the License, or (at your option) any later version.
  10  *
  11  *  This library is distributed in the hope that it will be useful,
  12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  *  Lesser General Public License for more details.
  15  *
  16  *  You should have received a copy of the GNU Lesser General Public
  17  *  License along with this library; if not, write to the Free
  18  *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  19  *  Boston, MA 02110-1301 USA
  20  */
  21
  22 /**
  23  * SECTION:gstvaapidecoder_h264
  24  * @short_description: H.264 decoder
  25  */
  26
  27 #include "sysdeps.h"
  28 #include <string.h>
  29 #include <gst/base/gstadapter.h>
  30 #include <gst/codecparsers/gsth264parser.h>
  31 #include "gstvaapidecoder_h264.h"
  32 #include "gstvaapidecoder_objects.h"
  33 #include "gstvaapidecoder_priv.h"
  34 #include "gstvaapidisplay_priv.h"
  35 #include "gstvaapiobject_priv.h"
  36
  37 #define DEBUG 1
  38 #include "gstvaapidebug.h"
  39
  40 /* Defined to 1 if strict ordering of DPB is needed. Only useful for debug */
  41 #define USE_STRICT_DPB_ORDERING 0
  42
  43 typedef struct _GstVaapiPictureH264             GstVaapiPictureH264;
  44 typedef struct _GstVaapiPictureH264Class        GstVaapiPictureH264Class;
  45 typedef struct _GstVaapiSliceH264               GstVaapiSliceH264;
  46 typedef struct _GstVaapiSliceH264Class          GstVaapiSliceH264Class;
  47
  48 /* ------------------------------------------------------------------------- */
  49 /* --- H.264 Pictures                                                    --- */
  50 /* ------------------------------------------------------------------------- */
  51
  52 #define GST_VAAPI_TYPE_PICTURE_H264 \
  53     (gst_vaapi_picture_h264_get_type())
  54
  55 #define GST_VAAPI_PICTURE_H264_CAST(obj) \
  56     ((GstVaapiPictureH264 *)(obj))
  57
  58 #define GST_VAAPI_PICTURE_H264(obj)                             \
  59     (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
  60                                 GST_VAAPI_TYPE_PICTURE_H264,    \
  61                                 GstVaapiPictureH264))
  62
  63 #define GST_VAAPI_PICTURE_H264_CLASS(klass)                     \
  64     (G_TYPE_CHECK_CLASS_CAST((klass),                           \
  65                              GST_VAAPI_TYPE_PICTURE_H264,       \
  66                              GstVaapiPictureH264Class))
  67
  68 #define GST_VAAPI_IS_PICTURE_H264(obj) \
  69     (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_VAAPI_TYPE_PICTURE_H264))
  70
  71 #define GST_VAAPI_IS_PICTURE_H264_CLASS(klass) \
  72     (G_TYPE_CHECK_CLASS_TYPE((klass), GST_VAAPI_TYPE_PICTURE_H264))
  73
  74 #define GST_VAAPI_PICTURE_H264_GET_CLASS(obj)                   \
  75     (G_TYPE_INSTANCE_GET_CLASS((obj),                           \
  76                                GST_VAAPI_TYPE_PICTURE_H264,     \
  77                                GstVaapiPictureH264Class))
  78
  79 /*
  80  * Extended picture flags:
  81  *
  82  * @GST_VAAPI_PICTURE_FLAG_IDR: flag that specifies an IDR picture
  83  */
  84 enum {
  85     GST_VAAPI_PICTURE_FLAG_IDR = (GST_VAAPI_PICTURE_FLAG_LAST << 0),
  86 };
  87
  88 #define GST_VAAPI_PICTURE_IS_IDR(picture) \
  89     (GST_VAAPI_PICTURE_FLAG_IS_SET(picture, GST_VAAPI_PICTURE_FLAG_IDR))
  90
  91 struct _GstVaapiPictureH264 {
  92     GstVaapiPicture             base;
  93     VAPictureH264               info;
  94     GstH264PPS                 *pps;
  95     gint32                      poc;
  96     gint32                      frame_num;              // Original frame_num from slice_header()
  97     gint32                      frame_num_wrap;         // Temporary for ref pic marking: FrameNumWrap
  98     gint32                      pic_num;                // Temporary for ref pic marking: PicNum
  99     gint32                      long_term_pic_num;      // Temporary for ref pic marking: LongTermPicNum
 100     guint                       is_long_term            : 1;
 101     guint                       output_flag             : 1;
 102     guint                       output_needed           : 1;
 103 };
 104
 105 struct _GstVaapiPictureH264Class {
 106     /*< private >*/
 107     GstVaapiPictureClass        parent_class;
 108 };
 109
 110 GST_VAAPI_CODEC_DEFINE_TYPE(GstVaapiPictureH264,
 111                             gst_vaapi_picture_h264,
 112                             GST_VAAPI_TYPE_PICTURE)
 113
 114 static void
 115 gst_vaapi_picture_h264_destroy(GstVaapiPictureH264 *decoder)
 116 {
 117 }
 118
 119 static gboolean
 120 gst_vaapi_picture_h264_create(
 121     GstVaapiPictureH264                      *picture,
 122     const GstVaapiCodecObjectConstructorArgs *args
 123 )
 124 {
 125     return TRUE;
 126 }
 127
 128 static void
 129 gst_vaapi_picture_h264_init(GstVaapiPictureH264 *picture)
 130 {
 131     VAPictureH264 *va_pic;
 132
 133     va_pic                      = &picture->info;
 134     va_pic->flags               = 0;
 135     va_pic->TopFieldOrderCnt    = 0;
 136     va_pic->BottomFieldOrderCnt = 0;
 137
 138     picture->poc                = 0;
 139     picture->is_long_term       = FALSE;
 140     picture->output_needed      = FALSE;
 141 }
 142
 143 static inline GstVaapiPictureH264 *
 144 gst_vaapi_picture_h264_new(GstVaapiDecoderH264 *decoder)
 145 {
 146     GstVaapiCodecObject *object;
 147
 148     g_return_val_if_fail(GST_VAAPI_IS_DECODER(decoder), NULL);
 149
 150     object = gst_vaapi_codec_object_new(
 151         GST_VAAPI_TYPE_PICTURE_H264,
 152         GST_VAAPI_CODEC_BASE(decoder),
 153         NULL, sizeof(VAPictureParameterBufferH264),
 154         NULL, 0
 155     );
 156     if (!object)
 157         return NULL;
 158     return GST_VAAPI_PICTURE_H264_CAST(object);
 159 }
 160
 161 static inline GstVaapiSliceH264 *
 162 gst_vaapi_picture_h264_get_last_slice(GstVaapiPictureH264 *picture)
 163 {
 164     g_return_val_if_fail(GST_VAAPI_IS_PICTURE_H264(picture), NULL);
 165
 166     if (G_UNLIKELY(picture->base.slices->len < 1))
 167         return NULL;
 168     return g_ptr_array_index(picture->base.slices,
 169         picture->base.slices->len - 1);
 170 }
 171
 172 /* ------------------------------------------------------------------------- */
 173 /* --- Slices                                                            --- */
 174 /* ------------------------------------------------------------------------- */
 175
 176 #define GST_VAAPI_TYPE_SLICE_H264 \
 177     (gst_vaapi_slice_h264_get_type())
 178
 179 #define GST_VAAPI_SLICE_H264_CAST(obj) \
 180     ((GstVaapiSliceH264 *)(obj))
 181
 182 #define GST_VAAPI_SLICE_H264(obj)                               \
 183     (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
 184                                 GST_VAAPI_TYPE_SLICE_H264,      \
 185                                 GstVaapiSliceH264))
 186
 187 #define GST_VAAPI_SLICE_H264_CLASS(klass)                       \
 188     (G_TYPE_CHECK_CLASS_CAST((klass),                           \
 189                              GST_VAAPI_TYPE_SLICE_H264,         \
 190                              GstVaapiSliceH264Class))
 191
 192 #define GST_VAAPI_IS_SLICE_H264(obj) \
 193     (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_VAAPI_TYPE_SLICE_H264))
 194
 195 #define GST_VAAPI_IS_SLICE_H264_CLASS(klass) \
 196     (G_TYPE_CHECK_CLASS_TYPE((klass), GST_VAAPI_TYPE_SLICE_H264))
 197
 198 #define GST_VAAPI_SLICE_H264_GET_CLASS(obj)                     \
 199     (G_TYPE_INSTANCE_GET_CLASS((obj),                           \
 200                                GST_VAAPI_TYPE_SLICE_H264,       \
 201                                GstVaapiSliceH264Class))
 202
 203 struct _GstVaapiSliceH264 {
 204     GstVaapiSlice               base;
 205     GstH264SliceHdr             slice_hdr;              // parsed slice_header()
 206 };
 207
 208 struct _GstVaapiSliceH264Class {
 209     /*< private >*/
 210     GstVaapiSliceClass          parent_class;
 211 };
 212
 213 GST_VAAPI_CODEC_DEFINE_TYPE(GstVaapiSliceH264,
 214                             gst_vaapi_slice_h264,
 215                             GST_VAAPI_TYPE_SLICE)
 216
 217 static void
 218 gst_vaapi_slice_h264_destroy(GstVaapiSliceH264 *slice)
 219 {
 220 }
 221
 222 static gboolean
 223 gst_vaapi_slice_h264_create(
 224     GstVaapiSliceH264                        *slice,
 225     const GstVaapiCodecObjectConstructorArgs *args
 226 )
 227 {
 228     return TRUE;
 229 }
 230
 231 static void
 232 gst_vaapi_slice_h264_init(GstVaapiSliceH264 *slice)
 233 {
 234 }
 235
 236 static inline GstVaapiSliceH264 *
 237 gst_vaapi_slice_h264_new(
 238     GstVaapiDecoderH264 *decoder,
 239     const guint8        *data,
 240     guint                data_size
 241 )
 242 {
 243     GstVaapiCodecObject *object;
 244
 245     g_return_val_if_fail(GST_VAAPI_IS_DECODER(decoder), NULL);
 246
 247     object = gst_vaapi_codec_object_new(
 248         GST_VAAPI_TYPE_SLICE_H264,
 249         GST_VAAPI_CODEC_BASE(decoder),
 250         NULL, sizeof(VASliceParameterBufferH264),
 251         data, data_size
 252     );
 253     if (!object)
 254         return NULL;
 255     return GST_VAAPI_SLICE_H264_CAST(object);
 256 }
 257
 258 /* ------------------------------------------------------------------------- */
 259 /* --- H.264 Decoder                                                     --- */
 260 /* ------------------------------------------------------------------------- */
 261
 262 G_DEFINE_TYPE(GstVaapiDecoderH264,
 263               gst_vaapi_decoder_h264,
 264               GST_VAAPI_TYPE_DECODER)
 265
 266 #define GST_VAAPI_DECODER_H264_GET_PRIVATE(obj)                 \
 267     (G_TYPE_INSTANCE_GET_PRIVATE((obj),                         \
 268                                  GST_VAAPI_TYPE_DECODER_H264,   \
 269                                  GstVaapiDecoderH264Private))
 270
 271 // Used for field_poc[]
 272 #define TOP_FIELD       0
 273 #define BOTTOM_FIELD    1
 274
 275 struct _GstVaapiDecoderH264Private {
 276     GstAdapter                 *adapter;
 277     GstH264NalParser           *parser;
 278     /* Last decoded SPS. May not be the last activated one. Just here because
 279        it may not fit stack memory allocation in decode_sps() */
 280     GstH264SPS                  last_sps;
 281     /* Last decoded PPS. May not be the last activated one. Just here because
 282        it may not fit stack memory allocation in decode_pps() */
 283     GstH264PPS                  last_pps;
 284     GstVaapiPictureH264        *current_picture;
 285     GstVaapiPictureH264        *dpb[16];
 286     guint                       dpb_count;
 287     guint                       dpb_size;
 288     GstVaapiProfile             profile;
 289     GstVaapiEntrypoint          entrypoint;
 290     GstVaapiChromaType          chroma_type;
 291     GstVaapiPictureH264        *short_ref[32];
 292     guint                       short_ref_count;
 293     GstVaapiPictureH264        *long_ref[32];
 294     guint                       long_ref_count;
 295     GstVaapiPictureH264        *RefPicList0[32];
 296     guint                       RefPicList0_count;
 297     GstVaapiPictureH264        *RefPicList1[32];
 298     guint                       RefPicList1_count;
 299     guint                       nal_length_size;
 300     guint                       width;
 301     guint                       height;
 302     gint32                      field_poc[2];           // 0:TopFieldOrderCnt / 1:BottomFieldOrderCnt
 303     gint32                      poc_msb;                // PicOrderCntMsb
 304     gint32                      poc_lsb;                // pic_order_cnt_lsb (from slice_header())
 305     gint32                      prev_poc_msb;           // prevPicOrderCntMsb
 306     gint32                      prev_poc_lsb;           // prevPicOrderCntLsb
 307     gint32                      frame_num_offset;       // FrameNumOffset
 308     gint32                      frame_num;              // frame_num (from slice_header())
 309     gint32                      prev_frame_num;         // prevFrameNum
 310     gboolean                    prev_pic_has_mmco5;     // prevMmco5Pic
 311     gboolean                    prev_pic_structure;     // previous picture structure
 312     guint                       is_constructed          : 1;
 313     guint                       is_opened               : 1;
 314     guint                       is_avc                  : 1;
 315     guint                       has_context             : 1;
 316 };
 317
 318 static gboolean
 319 decode_picture_end(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture);
 320
 321 static void
 322 clear_references(
 323     GstVaapiDecoderH264  *decoder,
 324     GstVaapiPictureH264 **pictures,
 325     guint                *picture_count
 326 );
 327
 328 /* Get number of reference frames to use */
 329 static guint
 330 get_max_dec_frame_buffering(GstH264SPS *sps)
 331 {
 332     guint max_dec_frame_buffering, MaxDpbMbs, PicSizeMbs;
 333
 334     /* Table A-1 - Level limits */
 335     switch (sps->level_idc) {
 336     case 10: MaxDpbMbs = 396;    break;
 337     case 11: MaxDpbMbs = 900;    break;
 338     case 12: MaxDpbMbs = 2376;   break;
 339     case 13: MaxDpbMbs = 2376;   break;
 340     case 20: MaxDpbMbs = 2376;   break;
 341     case 21: MaxDpbMbs = 4752;   break;
 342     case 22: MaxDpbMbs = 8100;   break;
 343     case 30: MaxDpbMbs = 8100;   break;
 344     case 31: MaxDpbMbs = 18000;  break;
 345     case 32: MaxDpbMbs = 20480;  break;
 346     case 40: MaxDpbMbs = 32768;  break;
 347     case 41: MaxDpbMbs = 32768;  break;
 348     case 42: MaxDpbMbs = 34816;  break;
 349     case 50: MaxDpbMbs = 110400; break;
 350     case 51: MaxDpbMbs = 184320; break;
 351     default:
 352         g_assert(0 && "unhandled level");
 353         break;
 354     }
 355
 356     PicSizeMbs = ((sps->pic_width_in_mbs_minus1 + 1) *
 357                   (sps->pic_height_in_map_units_minus1 + 1) *
 358                   (sps->frame_mbs_only_flag ? 1 : 2));
 359     max_dec_frame_buffering = MaxDpbMbs / PicSizeMbs;
 360
 361     /* VUI parameters */
 362     if (sps->vui_parameters_present_flag) {
 363         GstH264VUIParams * const vui_params = &sps->vui_parameters;
 364         if (vui_params->bitstream_restriction_flag)
 365             max_dec_frame_buffering = vui_params->max_dec_frame_buffering;
 366         else {
 367             switch (sps->profile_idc) {
 368             case 44:  // CAVLC 4:4:4 Intra profile
 369             case 86:  // Scalable High profile
 370             case 100: // High profile
 371             case 110: // High 10 profile
 372             case 122: // High 4:2:2 profile
 373             case 244: // High 4:4:4 Predictive profile
 374                 if (sps->constraint_set3_flag)
 375                     max_dec_frame_buffering = 0;
 376                 break;
 377             }
 378         }
 379     }
 380
 381     if (max_dec_frame_buffering > 16)
 382         max_dec_frame_buffering = 16;
 383     else if (max_dec_frame_buffering < sps->num_ref_frames)
 384         max_dec_frame_buffering = sps->num_ref_frames;
 385     return MAX(1, max_dec_frame_buffering);
 386 }
 387
 388 static void
 389 dpb_remove_index(GstVaapiDecoderH264 *decoder, guint index)
 390 {
 391     GstVaapiDecoderH264Private * const priv = decoder->priv;
 392     guint i, num_pictures = --priv->dpb_count;
 393
 394     if (USE_STRICT_DPB_ORDERING) {
 395         for (i = index; i < num_pictures; i++)
 396             gst_vaapi_picture_replace(&priv->dpb[i], priv->dpb[i + 1]);
 397     }
 398     else if (index != num_pictures)
 399         gst_vaapi_picture_replace(&priv->dpb[index], priv->dpb[num_pictures]);
 400     gst_vaapi_picture_replace(&priv->dpb[num_pictures], NULL);
 401 }
 402
 403 static inline gboolean
 404 dpb_output(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
 405 {
 406     /* XXX: update cropping rectangle */
 407     picture->output_needed = FALSE;
 408     return gst_vaapi_picture_output(GST_VAAPI_PICTURE_CAST(picture));
 409 }
 410
 411 static gboolean
 412 dpb_bump(GstVaapiDecoderH264 *decoder)
 413 {
 414     GstVaapiDecoderH264Private * const priv = decoder->priv;
 415     guint i, lowest_poc_index;
 416     gboolean success;
 417
 418     for (i = 0; i < priv->dpb_count; i++) {
 419         if (priv->dpb[i]->output_needed)
 420             break;
 421     }
 422     if (i == priv->dpb_count)
 423         return FALSE;
 424
 425     lowest_poc_index = i++;
 426     for (; i < priv->dpb_count; i++) {
 427         GstVaapiPictureH264 * const picture = priv->dpb[i];
 428         if (picture->output_needed && picture->poc < priv->dpb[lowest_poc_index]->poc)
 429             lowest_poc_index = i;
 430     }
 431
 432     success = dpb_output(decoder, priv->dpb[lowest_poc_index]);
 433     if (!GST_VAAPI_PICTURE_IS_REFERENCE(priv->dpb[lowest_poc_index]))
 434         dpb_remove_index(decoder, lowest_poc_index);
 435     return success;
 436 }
 437
 438 static void
 439 dpb_flush(GstVaapiDecoderH264 *decoder)
 440 {
 441     GstVaapiDecoderH264Private * const priv = decoder->priv;
 442
 443     while (dpb_bump(decoder))
 444         ;
 445     clear_references(decoder, priv->dpb, &priv->dpb_count);
 446 }
 447
 448 static gboolean
 449 dpb_add(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
 450 {
 451     GstVaapiDecoderH264Private * const priv = decoder->priv;
 452     guint i;
 453
 454     // Remove all unused pictures
 455     if (GST_VAAPI_PICTURE_IS_IDR(picture))
 456         dpb_flush(decoder);
 457     else {
 458         i = 0;
 459         while (i < priv->dpb_count) {
 460             GstVaapiPictureH264 * const picture = priv->dpb[i];
 461             if (!picture->output_needed &&
 462                 !GST_VAAPI_PICTURE_IS_REFERENCE(picture))
 463                 dpb_remove_index(decoder, i);
 464             else
 465                 i++;
 466         }
 467     }
 468
 469     // C.4.5.1 - Storage and marking of a reference decoded picture into the DPB
 470     if (GST_VAAPI_PICTURE_IS_REFERENCE(picture)) {
 471         while (priv->dpb_count == priv->dpb_size) {
 472             if (!dpb_bump(decoder))
 473                 return FALSE;
 474         }
 475         gst_vaapi_picture_replace(&priv->dpb[priv->dpb_count++], picture);
 476         if (picture->output_flag)
 477             picture->output_needed = TRUE;
 478     }
 479
 480     // C.4.5.2 - Storage and marking of a non-reference decoded picture into the DPB
 481     else {
 482         if (!picture->output_flag)
 483             return TRUE;
 484         while (priv->dpb_count == priv->dpb_size) {
 485             for (i = 0; i < priv->dpb_count; i++) {
 486                 if (priv->dpb[i]->output_needed &&
 487                     priv->dpb[i]->poc < picture->poc)
 488                     break;
 489             }
 490             if (i == priv->dpb_count)
 491                 return dpb_output(decoder, picture);
 492             if (!dpb_bump(decoder))
 493                 return FALSE;
 494         }
 495         gst_vaapi_picture_replace(&priv->dpb[priv->dpb_count++], picture);
 496         picture->output_needed = TRUE;
 497     }
 498     return TRUE;
 499 }
 500
 501 static inline void
 502 dpb_reset(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 503 {
 504     GstVaapiDecoderH264Private * const priv = decoder->priv;
 505
 506     priv->dpb_size = get_max_dec_frame_buffering(sps);
 507     GST_DEBUG("DPB size %u", priv->dpb_size);
 508 }
 509
 510 static GstVaapiDecoderStatus
 511 get_status(GstH264ParserResult result)
 512 {
 513     GstVaapiDecoderStatus status;
 514
 515     switch (result) {
 516     case GST_H264_PARSER_OK:
 517         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
 518         break;
 519     case GST_H264_PARSER_NO_NAL_END:
 520         status = GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
 521         break;
 522     case GST_H264_PARSER_ERROR:
 523         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
 524         break;
 525     default:
 526         status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 527         break;
 528     }
 529     return status;
 530 }
 531
 532 static void
 533 gst_vaapi_decoder_h264_close(GstVaapiDecoderH264 *decoder)
 534 {
 535     GstVaapiDecoderH264Private * const priv = decoder->priv;
 536
 537     gst_vaapi_picture_replace(&priv->current_picture, NULL);
 538     clear_references(decoder, priv->short_ref, &priv->short_ref_count);
 539     clear_references(decoder, priv->long_ref,  &priv->long_ref_count );
 540     clear_references(decoder, priv->dpb,       &priv->dpb_count      );
 541
 542     if (priv->parser) {
 543         gst_h264_nal_parser_free(priv->parser);
 544         priv->parser = NULL;
 545     }
 546
 547     if (priv->adapter) {
 548         gst_adapter_clear(priv->adapter);
 549         g_object_unref(priv->adapter);
 550         priv->adapter = NULL;
 551     }
 552 }
 553
 554 static gboolean
 555 gst_vaapi_decoder_h264_open(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
 556 {
 557     GstVaapiDecoderH264Private * const priv = decoder->priv;
 558
 559     gst_vaapi_decoder_h264_close(decoder);
 560
 561     priv->adapter = gst_adapter_new();
 562     if (!priv->adapter)
 563         return FALSE;
 564
 565     priv->parser = gst_h264_nal_parser_new();
 566     if (!priv->parser)
 567         return FALSE;
 568     return TRUE;
 569 }
 570
 571 static void
 572 gst_vaapi_decoder_h264_destroy(GstVaapiDecoderH264 *decoder)
 573 {
 574     gst_vaapi_decoder_h264_close(decoder);
 575 }
 576
 577 static gboolean
 578 gst_vaapi_decoder_h264_create(GstVaapiDecoderH264 *decoder)
 579 {
 580     if (!GST_VAAPI_DECODER_CODEC(decoder))
 581         return FALSE;
 582     return TRUE;
 583 }
 584
 585 static guint
 586 h264_get_profile(GstH264SPS *sps)
 587 {
 588     guint profile = 0;
 589
 590     switch (sps->profile_idc) {
 591     case 66:
 592         profile = GST_VAAPI_PROFILE_H264_BASELINE;
 593         break;
 594     case 77:
 595         profile = GST_VAAPI_PROFILE_H264_MAIN;
 596         break;
 597     case 100:
 598         profile = GST_VAAPI_PROFILE_H264_HIGH;
 599         break;
 600     }
 601     return profile;
 602 }
 603
 604 static guint
 605 h264_get_chroma_type(GstH264SPS *sps)
 606 {
 607     guint chroma_type = 0;
 608
 609     switch (sps->chroma_format_idc) {
 610     case 1:
 611         chroma_type = GST_VAAPI_CHROMA_TYPE_YUV420;
 612         break;
 613     case 2:
 614         chroma_type = GST_VAAPI_CHROMA_TYPE_YUV422;
 615         break;
 616     case 3:
 617         if (!sps->separate_colour_plane_flag)
 618             chroma_type = GST_VAAPI_CHROMA_TYPE_YUV444;
 619         break;
 620     }
 621     return chroma_type;
 622 }
 623
 624 static GstVaapiProfile
 625 get_profile(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 626 {
 627     GstVaapiDecoderH264Private * const priv = decoder->priv;
 628     GstVaapiDisplay * const display = GST_VAAPI_DECODER_DISPLAY(decoder);
 629     GstVaapiProfile profile, profiles[2];
 630     guint i, n_profiles = 0;
 631
 632     profile = h264_get_profile(sps);
 633     if (!profile)
 634         return GST_VAAPI_PROFILE_UNKNOWN;
 635
 636     profiles[n_profiles++] = profile;
 637     switch (profile) {
 638     case GST_VAAPI_PROFILE_H264_MAIN:
 639         profiles[n_profiles++] = GST_VAAPI_PROFILE_H264_HIGH;
 640         break;
 641     default:
 642         break;
 643     }
 644
 645     /* If the preferred profile (profiles[0]) matches one that we already
 646        found, then just return it now instead of searching for it again */
 647     if (profiles[0] == priv->profile)
 648         return priv->profile;
 649
 650     for (i = 0; i < n_profiles; i++) {
 651         if (gst_vaapi_display_has_decoder(display, profiles[i], priv->entrypoint))
 652             return profiles[i];
 653     }
 654     return GST_VAAPI_PROFILE_UNKNOWN;
 655 }
 656
 657 static GstVaapiDecoderStatus
 658 ensure_context(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 659 {
 660     GstVaapiDecoder * const base_decoder = GST_VAAPI_DECODER_CAST(decoder);
 661     GstVaapiDecoderH264Private * const priv = decoder->priv;
 662     GstVaapiContextInfo info;
 663     GstVaapiProfile profile;
 664     GstVaapiChromaType chroma_type;
 665     gboolean reset_context = FALSE;
 666
 667     profile = get_profile(decoder, sps);
 668     if (!profile) {
 669         GST_ERROR("unsupported profile_idc %u", sps->profile_idc);
 670         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_PROFILE;
 671     }
 672
 673     if (priv->profile != profile) {
 674         GST_DEBUG("profile changed");
 675         reset_context = TRUE;
 676         priv->profile = profile;
 677     }
 678
 679     chroma_type = h264_get_chroma_type(sps);
 680     if (!chroma_type || chroma_type != GST_VAAPI_CHROMA_TYPE_YUV420) {
 681         GST_ERROR("unsupported chroma_format_idc %u", sps->chroma_format_idc);
 682         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;
 683     }
 684
 685     if (priv->chroma_type != chroma_type) {
 686         GST_DEBUG("chroma format changed");
 687         reset_context     = TRUE;
 688         priv->chroma_type = chroma_type;
 689     }
 690
 691     if (priv->width != sps->width || priv->height != sps->height) {
 692         GST_DEBUG("size changed");
 693         reset_context = TRUE;
 694         priv->width   = sps->width;
 695         priv->height  = sps->height;
 696     }
 697
 698     gst_vaapi_decoder_set_pixel_aspect_ratio(
 699         base_decoder,
 700         sps->vui_parameters.par_n,
 701         sps->vui_parameters.par_d
 702     );
 703
 704     if (!reset_context && priv->has_context)
 705         return GST_VAAPI_DECODER_STATUS_SUCCESS;
 706
 707     info.profile    = priv->profile;
 708     info.entrypoint = priv->entrypoint;
 709     info.width      = priv->width;
 710     info.height     = priv->height;
 711     info.ref_frames = get_max_dec_frame_buffering(sps);
 712
 713     if (!gst_vaapi_decoder_ensure_context(GST_VAAPI_DECODER(decoder), &info))
 714         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 715     priv->has_context = TRUE;
 716
 717     /* Reset DPB */
 718     dpb_reset(decoder, sps);
 719     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 720 }
 721
 722 static void
 723 fill_iq_matrix_4x4(VAIQMatrixBufferH264 *iq_matrix, const GstH264PPS *pps)
 724 {
 725     const guint8 (* const ScalingList4x4)[6][16] = &pps->scaling_lists_4x4;
 726     guint i, j;
 727
 728     /* There are always 6 4x4 scaling lists */
 729     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList4x4) == 6);
 730     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList4x4[0]) == 16);
 731
 732     if (sizeof(iq_matrix->ScalingList4x4[0][0]) == 1)
 733         memcpy(iq_matrix->ScalingList4x4, *ScalingList4x4,
 734                sizeof(iq_matrix->ScalingList4x4));
 735     else {
 736         for (i = 0; i < G_N_ELEMENTS(iq_matrix->ScalingList4x4); i++) {
 737             for (j = 0; j < G_N_ELEMENTS(iq_matrix->ScalingList4x4[i]); j++)
 738                 iq_matrix->ScalingList4x4[i][j] = (*ScalingList4x4)[i][j];
 739         }
 740     }
 741 }
 742
 743 static void
 744 fill_iq_matrix_8x8(VAIQMatrixBufferH264 *iq_matrix, const GstH264PPS *pps)
 745 {
 746     const guint8 (* const ScalingList8x8)[6][64] = &pps->scaling_lists_8x8;
 747     const GstH264SPS * const sps = pps->sequence;
 748     guint i, j, n;
 749
 750     /* If chroma_format_idc != 3, there are up to 2 8x8 scaling lists */
 751     if (!pps->transform_8x8_mode_flag)
 752         return;
 753
 754     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList8x8) >= 2);
 755     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList8x8[0]) == 64);
 756
 757     if (sizeof(iq_matrix->ScalingList8x8[0][0]) == 1)
 758         memcpy(iq_matrix->ScalingList8x8, *ScalingList8x8,
 759                sizeof(iq_matrix->ScalingList8x8));
 760     else {
 761         n = (sps->chroma_format_idc != 3) ? 2 : 6;
 762         for (i = 0; i < n; i++) {
 763             for (j = 0; j < G_N_ELEMENTS(iq_matrix->ScalingList8x8[i]); j++)
 764                 iq_matrix->ScalingList8x8[i][j] = (*ScalingList8x8)[i][j];
 765         }
 766     }
 767 }
 768
 769 static GstVaapiDecoderStatus
 770 ensure_quant_matrix(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
 771 {
 772     GstVaapiPicture * const base_picture = &picture->base;
 773     GstH264PPS * const pps = picture->pps;
 774     GstH264SPS * const sps = pps->sequence;
 775     VAIQMatrixBufferH264 *iq_matrix;
 776
 777     base_picture->iq_matrix = GST_VAAPI_IQ_MATRIX_NEW(H264, decoder);
 778     if (!base_picture->iq_matrix) {
 779         GST_ERROR("failed to allocate IQ matrix");
 780         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
 781     }
 782     iq_matrix = base_picture->iq_matrix->param;
 783
 784     /* XXX: we can only support 4:2:0 or 4:2:2 since ScalingLists8x8[]
 785        is not large enough to hold lists for 4:4:4 */
 786     if (sps->chroma_format_idc == 3)
 787         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;
 788
 789     fill_iq_matrix_4x4(iq_matrix, pps);
 790     fill_iq_matrix_8x8(iq_matrix, pps);
 791
 792     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 793 }
 794
 795 static GstVaapiDecoderStatus
 796 decode_current_picture(GstVaapiDecoderH264 *decoder)
 797 {
 798     GstVaapiDecoderH264Private * const priv = decoder->priv;
 799     GstVaapiPictureH264 * const picture = priv->current_picture;
 800     GstVaapiDecoderStatus status;
 801
 802     if (!picture)
 803         return GST_VAAPI_DECODER_STATUS_SUCCESS;
 804
 805     status = ensure_context(decoder, picture->pps->sequence);
 806     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
 807         return status;
 808
 809     if (!decode_picture_end(decoder, picture))
 810         goto error;
 811     if (!gst_vaapi_picture_decode(GST_VAAPI_PICTURE_CAST(picture)))
 812         goto error;
 813     gst_vaapi_picture_replace(&priv->current_picture, NULL);
 814     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 815
 816 error:
 817     gst_vaapi_picture_replace(&priv->current_picture, NULL);
 818     return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 819 }
 820
 821 static GstVaapiDecoderStatus
 822 decode_sps(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
 823 {
 824     GstVaapiDecoderH264Private * const priv = decoder->priv;
 825     GstH264SPS * const sps = &priv->last_sps;
 826     GstH264ParserResult result;
 827
 828     GST_DEBUG("decode SPS");
 829
 830     memset(sps, 0, sizeof(*sps));
 831     result = gst_h264_parser_parse_sps(priv->parser, nalu, sps, TRUE);
 832     if (result != GST_H264_PARSER_OK)
 833         return get_status(result);
 834
 835     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 836 }
 837
 838 static GstVaapiDecoderStatus
 839 decode_pps(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
 840 {
 841     GstVaapiDecoderH264Private * const priv = decoder->priv;
 842     GstH264PPS * const pps = &priv->last_pps;
 843     GstH264ParserResult result;
 844
 845     GST_DEBUG("decode PPS");
 846
 847     memset(pps, 0, sizeof(*pps));
 848     result = gst_h264_parser_parse_pps(priv->parser, nalu, pps);
 849     if (result != GST_H264_PARSER_OK)
 850         return get_status(result);
 851
 852     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 853 }
 854
 855 static GstVaapiDecoderStatus
 856 decode_sei(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
 857 {
 858     GstVaapiDecoderH264Private * const priv = decoder->priv;
 859     GstH264SEIMessage sei;
 860     GstH264ParserResult result;
 861
 862     GST_DEBUG("decode SEI");
 863
 864     memset(&sei, 0, sizeof(sei));
 865     result = gst_h264_parser_parse_sei(priv->parser, nalu, &sei);
 866     if (result != GST_H264_PARSER_OK) {
 867         GST_WARNING("failed to decode SEI, payload type:%d", sei.payloadType);
 868         return get_status(result);
 869     }
 870
 871     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 872 }
 873
 874 static GstVaapiDecoderStatus
 875 decode_sequence_end(GstVaapiDecoderH264 *decoder)
 876 {
 877     GstVaapiDecoderStatus status;
 878
 879     GST_DEBUG("decode sequence-end");
 880
 881     status = decode_current_picture(decoder);
 882     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
 883         return status;
 884
 885     dpb_flush(decoder);
 886     return GST_VAAPI_DECODER_STATUS_END_OF_STREAM;
 887 }
 888
 889 /* 8.2.1.1 - Decoding process for picture order count type 0 */
 890 static void
 891 init_picture_poc_0(
 892     GstVaapiDecoderH264 *decoder,
 893     GstVaapiPictureH264 *picture,
 894     GstH264SliceHdr     *slice_hdr
 895 )
 896 {
 897     GstVaapiDecoderH264Private * const priv = decoder->priv;
 898     GstH264PPS * const pps = slice_hdr->pps;
 899     GstH264SPS * const sps = pps->sequence;
 900     const gint32 MaxPicOrderCntLsb = 1 << (sps->log2_max_pic_order_cnt_lsb_minus4 + 4);
 901     gint32 temp_poc;
 902
 903     GST_DEBUG("decode picture order count type 0");
 904
 905     if (GST_VAAPI_PICTURE_IS_IDR(picture)) {
 906         priv->prev_poc_msb = 0;
 907         priv->prev_poc_lsb = 0;
 908     }
 909     else if (priv->prev_pic_has_mmco5) {
 910         priv->prev_poc_msb = 0;
 911         priv->prev_poc_lsb =
 912             (priv->prev_pic_structure == GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD ?
 913              0 : priv->field_poc[TOP_FIELD]);
 914     }
 915     else {
 916         priv->prev_poc_msb = priv->poc_msb;
 917         priv->prev_poc_lsb = priv->poc_lsb;
 918     }
 919
 920     // (8-3)
 921     priv->poc_lsb = slice_hdr->pic_order_cnt_lsb;
 922     if (priv->poc_lsb < priv->prev_poc_lsb &&
 923         (priv->prev_poc_lsb - priv->poc_lsb) >= (MaxPicOrderCntLsb / 2))
 924         priv->poc_msb = priv->prev_poc_msb + MaxPicOrderCntLsb;
 925     else if (priv->poc_lsb > priv->prev_poc_lsb &&
 926              (priv->poc_lsb - priv->prev_poc_lsb) > (MaxPicOrderCntLsb / 2))
 927         priv->poc_msb = priv->prev_poc_msb - MaxPicOrderCntLsb;
 928     else
 929         priv->poc_msb = priv->prev_poc_msb;
 930
 931     temp_poc = priv->poc_msb + priv->poc_lsb;
 932     switch (picture->base.structure) {
 933     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
 934         // (8-4, 8-5)
 935         priv->field_poc[TOP_FIELD] = temp_poc;
 936         priv->field_poc[BOTTOM_FIELD] = temp_poc +
 937             slice_hdr->delta_pic_order_cnt_bottom;
 938         break;
 939     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
 940         // (8-4)
 941         priv->field_poc[TOP_FIELD] = temp_poc;
 942         break;
 943     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
 944         // (8-5)
 945         priv->field_poc[BOTTOM_FIELD] = temp_poc;
 946         break;
 947     }
 948 }
 949
 950 /* 8.2.1.2 - Decoding process for picture order count type 1 */
 951 static void
 952 init_picture_poc_1(
 953     GstVaapiDecoderH264 *decoder,
 954     GstVaapiPictureH264 *picture,
 955     GstH264SliceHdr     *slice_hdr
 956 )
 957 {
 958     GstVaapiDecoderH264Private * const priv = decoder->priv;
 959     GstH264PPS * const pps = slice_hdr->pps;
 960     GstH264SPS * const sps = pps->sequence;
 961     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
 962     gint32 prev_frame_num_offset, abs_frame_num, expected_poc;
 963     guint i;
 964
 965     GST_DEBUG("decode picture order count type 1");
 966
 967     if (priv->prev_pic_has_mmco5)
 968         prev_frame_num_offset = 0;
 969     else
 970         prev_frame_num_offset = priv->frame_num_offset;
 971
 972     // (8-6)
 973     if (GST_VAAPI_PICTURE_IS_IDR(picture))
 974         priv->frame_num_offset = 0;
 975     else if (priv->prev_frame_num > priv->frame_num)
 976         priv->frame_num_offset = prev_frame_num_offset + MaxFrameNum;
 977     else
 978         priv->frame_num_offset = prev_frame_num_offset;
 979
 980     // (8-7)
 981     if (sps->num_ref_frames_in_pic_order_cnt_cycle != 0)
 982         abs_frame_num = priv->frame_num_offset + priv->frame_num;
 983     else
 984         abs_frame_num = 0;
 985     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture) && abs_frame_num > 0)
 986         abs_frame_num = abs_frame_num - 1;
 987
 988     if (abs_frame_num > 0) {
 989         gint32 expected_delta_per_poc_cycle;
 990         gint32 poc_cycle_cnt, frame_num_in_poc_cycle;
 991
 992         expected_delta_per_poc_cycle = 0;
 993         for (i = 0; i < sps->num_ref_frames_in_pic_order_cnt_cycle; i++)
 994             expected_delta_per_poc_cycle += sps->offset_for_ref_frame[i];
 995
 996         // (8-8)
 997         poc_cycle_cnt = (abs_frame_num - 1) /
 998             sps->num_ref_frames_in_pic_order_cnt_cycle;
 999         frame_num_in_poc_cycle = (abs_frame_num - 1) %
1000             sps->num_ref_frames_in_pic_order_cnt_cycle;
1001
1002         // (8-9)
1003         expected_poc = poc_cycle_cnt * expected_delta_per_poc_cycle;
1004         for (i = 0; i <= frame_num_in_poc_cycle; i++)
1005             expected_poc += sps->offset_for_ref_frame[i];
1006     }
1007     else
1008         expected_poc = 0;
1009     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
1010         expected_poc += sps->offset_for_non_ref_pic;
1011
1012     // (8-10)
1013     switch (picture->base.structure) {
1014     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
1015         priv->field_poc[TOP_FIELD] = expected_poc +
1016             slice_hdr->delta_pic_order_cnt[0];
1017         priv->field_poc[BOTTOM_FIELD] = priv->field_poc[TOP_FIELD] +
1018             sps->offset_for_top_to_bottom_field +
1019             slice_hdr->delta_pic_order_cnt[1];
1020         break;
1021     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
1022         priv->field_poc[TOP_FIELD] = expected_poc +
1023             slice_hdr->delta_pic_order_cnt[0];
1024         break;
1025     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
1026         priv->field_poc[BOTTOM_FIELD] = expected_poc +
1027             sps->offset_for_top_to_bottom_field +
1028             slice_hdr->delta_pic_order_cnt[0];
1029         break;
1030     }
1031 }
1032
1033 /* 8.2.1.3 - Decoding process for picture order count type 2 */
1034 static void
1035 init_picture_poc_2(
1036     GstVaapiDecoderH264 *decoder,
1037     GstVaapiPictureH264 *picture,
1038     GstH264SliceHdr     *slice_hdr
1039 )
1040 {
1041     GstVaapiDecoderH264Private * const priv = decoder->priv;
1042     GstH264PPS * const pps = slice_hdr->pps;
1043     GstH264SPS * const sps = pps->sequence;
1044     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1045     gint32 prev_frame_num_offset, temp_poc;
1046
1047     GST_DEBUG("decode picture order count type 2");
1048
1049     if (priv->prev_pic_has_mmco5)
1050         prev_frame_num_offset = 0;
1051     else
1052         prev_frame_num_offset = priv->frame_num_offset;
1053
1054     // (8-11)
1055     if (GST_VAAPI_PICTURE_IS_IDR(picture))
1056         priv->frame_num_offset = 0;
1057     else if (priv->prev_frame_num > priv->frame_num)
1058         priv->frame_num_offset = prev_frame_num_offset + MaxFrameNum;
1059     else
1060         priv->frame_num_offset = prev_frame_num_offset;
1061
1062     // (8-12)
1063     if (GST_VAAPI_PICTURE_IS_IDR(picture))
1064         temp_poc = 0;
1065     else if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
1066         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num) - 1;
1067     else
1068         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num);
1069
1070     // (8-13)
1071     if (picture->base.structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
1072         priv->field_poc[TOP_FIELD] = temp_poc;
1073     if (picture->base.structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
1074         priv->field_poc[BOTTOM_FIELD] = temp_poc;
1075 }
1076
1077 /* 8.2.1 - Decoding process for picture order count */
1078 static void
1079 init_picture_poc(
1080     GstVaapiDecoderH264 *decoder,
1081     GstVaapiPictureH264 *picture,
1082     GstH264SliceHdr     *slice_hdr
1083 )
1084 {
1085     GstVaapiDecoderH264Private * const priv = decoder->priv;
1086     VAPictureH264 * const pic = &picture->info;
1087     GstH264PPS * const pps = slice_hdr->pps;
1088     GstH264SPS * const sps = pps->sequence;
1089
1090     switch (sps->pic_order_cnt_type) {
1091     case 0:
1092         init_picture_poc_0(decoder, picture, slice_hdr);
1093         break;
1094     case 1:
1095         init_picture_poc_1(decoder, picture, slice_hdr);
1096         break;
1097     case 2:
1098         init_picture_poc_2(decoder, picture, slice_hdr);
1099         break;
1100     }
1101
1102     if (picture->base.structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
1103         pic->TopFieldOrderCnt = priv->field_poc[TOP_FIELD];
1104     if (picture->base.structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
1105         pic->BottomFieldOrderCnt = priv->field_poc[BOTTOM_FIELD];
1106     picture->poc = MIN(pic->TopFieldOrderCnt, pic->BottomFieldOrderCnt);
1107 }
1108
1109 static int
1110 compare_picture_pic_num_dec(const void *a, const void *b)
1111 {
1112     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1113     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1114
1115     return picB->pic_num - picA->pic_num;
1116 }
1117
1118 static int
1119 compare_picture_long_term_pic_num_inc(const void *a, const void *b)
1120 {
1121     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1122     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1123
1124     return picA->long_term_pic_num - picB->long_term_pic_num;
1125 }
1126
1127 static int
1128 compare_picture_poc_dec(const void *a, const void *b)
1129 {
1130     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1131     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1132
1133     return picB->poc - picA->poc;
1134 }
1135
1136 static int
1137 compare_picture_poc_inc(const void *a, const void *b)
1138 {
1139     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1140     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1141
1142     return picA->poc - picB->poc;
1143 }
1144
1145 static int
1146 compare_picture_frame_num_wrap_dec(const void *a, const void *b)
1147 {
1148     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1149     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1150
1151     return picB->frame_num_wrap - picA->frame_num_wrap;
1152 }
1153
1154 static int
1155 compare_picture_long_term_frame_idx_inc(const void *a, const void *b)
1156 {
1157     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1158     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1159
1160     return picA->info.frame_idx - picB->info.frame_idx;
1161 }
1162
1163 /* 8.2.4.1 - Decoding process for picture numbers */
1164 static void
1165 init_picture_refs_pic_num(
1166     GstVaapiDecoderH264 *decoder,
1167     GstVaapiPictureH264 *picture,
1168     GstH264SliceHdr     *slice_hdr
1169 )
1170 {
1171     GstVaapiDecoderH264Private * const priv = decoder->priv;
1172     GstH264PPS * const pps = slice_hdr->pps;
1173     GstH264SPS * const sps = pps->sequence;
1174     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1175     guint i;
1176
1177     GST_DEBUG("decode picture numbers");
1178
1179     for (i = 0; i < priv->short_ref_count; i++) {
1180         GstVaapiPictureH264 * const pic = priv->short_ref[i];
1181
1182         // (8-27)
1183         if (pic->frame_num > priv->frame_num)
1184             pic->frame_num_wrap = pic->frame_num - MaxFrameNum;
1185         else
1186             pic->frame_num_wrap = pic->frame_num;
1187
1188         // (8-28, 8-30, 8-31)
1189         if (GST_VAAPI_PICTURE_IS_FRAME(picture))
1190             pic->pic_num = pic->frame_num_wrap;
1191         else {
1192             if (pic->base.structure == picture->base.structure)
1193                 pic->pic_num = 2 * pic->frame_num_wrap + 1;
1194             else
1195                 pic->pic_num = 2 * pic->frame_num_wrap;
1196         }
1197     }
1198
1199     for (i = 0; i < priv->long_ref_count; i++) {
1200         GstVaapiPictureH264 * const pic = priv->long_ref[i];
1201
1202         // (8-29, 8-32, 8-33)
1203         if (GST_VAAPI_PICTURE_IS_FRAME(picture))
1204             pic->long_term_pic_num = pic->info.frame_idx;
1205         else {
1206             if (pic->base.structure == picture->base.structure)
1207                 pic->long_term_pic_num = 2 * pic->info.frame_idx + 1;
1208             else
1209                 pic->long_term_pic_num = 2 * pic->info.frame_idx;
1210         }
1211     }
1212 }
1213
1214 #define SORT_REF_LIST(list, n, compare_func) \
1215     qsort(list, n, sizeof(*(list)), compare_picture_##compare_func)
1216
1217 static void
1218 init_picture_refs_p_slice(
1219     GstVaapiDecoderH264 *decoder,
1220     GstVaapiPictureH264 *picture,
1221     GstH264SliceHdr     *slice_hdr
1222 )
1223 {
1224     GstVaapiDecoderH264Private * const priv = decoder->priv;
1225     GstVaapiPictureH264 **ref_list;
1226     guint i;
1227
1228     GST_DEBUG("decode reference picture list for P and SP slices");
1229
1230     if (GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1231         /* 8.2.4.2.1 - P and SP slices in frames */
1232         if (priv->short_ref_count > 0) {
1233             ref_list = priv->RefPicList0;
1234             for (i = 0; i < priv->short_ref_count; i++)
1235                 ref_list[i] = priv->short_ref[i];
1236             SORT_REF_LIST(ref_list, i, pic_num_dec);
1237             priv->RefPicList0_count += i;
1238         }
1239
1240         if (priv->long_ref_count > 0) {
1241             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1242             for (i = 0; i < priv->long_ref_count; i++)
1243                 ref_list[i] = priv->long_ref[i];
1244             SORT_REF_LIST(ref_list, i, long_term_pic_num_inc);
1245             priv->RefPicList0_count += i;
1246         }
1247     }
1248     else {
1249         /* 8.2.4.2.2 - P and SP slices in fields */
1250         GstVaapiPictureH264 *short_ref[32];
1251         guint short_ref_count = 0;
1252         GstVaapiPictureH264 *long_ref[32];
1253         guint long_ref_count = 0;
1254
1255         // XXX: handle second field if current field is marked as
1256         // "used for short-term reference"
1257         if (priv->short_ref_count > 0) {
1258             for (i = 0; i < priv->short_ref_count; i++)
1259                 short_ref[i] = priv->short_ref[i];
1260             SORT_REF_LIST(short_ref, i, frame_num_wrap_dec);
1261             short_ref_count = i;
1262         }
1263
1264         // XXX: handle second field if current field is marked as
1265         // "used for long-term reference"
1266         if (priv->long_ref_count > 0) {
1267             for (i = 0; i < priv->long_ref_count; i++)
1268                 long_ref[i] = priv->long_ref[i];
1269             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1270             long_ref_count = i;
1271         }
1272
1273         // XXX: handle 8.2.4.2.5
1274     }
1275 }
1276
1277 static void
1278 init_picture_refs_b_slice(
1279     GstVaapiDecoderH264 *decoder,
1280     GstVaapiPictureH264 *picture,
1281     GstH264SliceHdr     *slice_hdr
1282 )
1283 {
1284     GstVaapiDecoderH264Private * const priv = decoder->priv;
1285     GstVaapiPictureH264 **ref_list;
1286     guint i, n;
1287
1288     GST_DEBUG("decode reference picture list for B slices");
1289
1290     if (GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1291         /* 8.2.4.2.3 - B slices in frames */
1292
1293         /* RefPicList0 */
1294         if (priv->short_ref_count > 0) {
1295             // 1. Short-term references
1296             ref_list = priv->RefPicList0;
1297             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1298                 if (priv->short_ref[i]->poc < picture->poc)
1299                     ref_list[n++] = priv->short_ref[i];
1300             }
1301             SORT_REF_LIST(ref_list, n, poc_dec);
1302             priv->RefPicList0_count += n;
1303
1304             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1305             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1306                 if (priv->short_ref[i]->poc >= picture->poc)
1307                     ref_list[n++] = priv->short_ref[i];
1308             }
1309             SORT_REF_LIST(ref_list, n, poc_inc);
1310             priv->RefPicList0_count += n;
1311         }
1312
1313         if (priv->long_ref_count > 0) {
1314             // 2. Long-term references
1315             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1316             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1317                 ref_list[n++] = priv->long_ref[i];
1318             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1319             priv->RefPicList0_count += n;
1320         }
1321
1322         /* RefPicList1 */
1323         if (priv->short_ref_count > 0) {
1324             // 1. Short-term references
1325             ref_list = priv->RefPicList1;
1326             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1327                 if (priv->short_ref[i]->poc > picture->poc)
1328                     ref_list[n++] = priv->short_ref[i];
1329             }
1330             SORT_REF_LIST(ref_list, n, poc_inc);
1331             priv->RefPicList1_count += n;
1332
1333             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1334             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1335                 if (priv->short_ref[i]->poc <= picture->poc)
1336                     ref_list[n++] = priv->short_ref[i];
1337             }
1338             SORT_REF_LIST(ref_list, n, poc_dec);
1339             priv->RefPicList1_count += n;
1340         }
1341
1342         if (priv->long_ref_count > 0) {
1343             // 2. Long-term references
1344             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1345             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1346                 ref_list[n++] = priv->long_ref[i];
1347             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1348             priv->RefPicList1_count += n;
1349         }
1350     }
1351     else {
1352         /* 8.2.4.2.4 - B slices in fields */
1353         GstVaapiPictureH264 *short_ref0[32];
1354         guint short_ref0_count = 0;
1355         GstVaapiPictureH264 *short_ref1[32];
1356         guint short_ref1_count = 0;
1357         GstVaapiPictureH264 *long_ref[32];
1358         guint long_ref_count = 0;
1359
1360         /* refFrameList0ShortTerm */
1361         if (priv->short_ref_count > 0) {
1362             ref_list = short_ref0;
1363             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1364                 if (priv->short_ref[i]->poc <= picture->poc)
1365                     ref_list[n++] = priv->short_ref[i];
1366             }
1367             SORT_REF_LIST(ref_list, n, poc_dec);
1368             short_ref0_count += n;
1369
1370             ref_list = &short_ref0[short_ref0_count];
1371             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1372                 if (priv->short_ref[i]->poc > picture->poc)
1373                     ref_list[n++] = priv->short_ref[i];
1374             }
1375             SORT_REF_LIST(ref_list, n, poc_inc);
1376             short_ref0_count += n;
1377         }
1378
1379         /* refFrameList1ShortTerm */
1380         if (priv->short_ref_count > 0) {
1381             ref_list = short_ref1;
1382             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1383                 if (priv->short_ref[i]->poc > picture->poc)
1384                     ref_list[n++] = priv->short_ref[i];
1385             }
1386             SORT_REF_LIST(ref_list, n, poc_inc);
1387             short_ref1_count += n;
1388
1389             ref_list = &short_ref1[short_ref1_count];
1390             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1391                 if (priv->short_ref[i]->poc <= picture->poc)
1392                     ref_list[n++] = priv->short_ref[i];
1393             }
1394             SORT_REF_LIST(ref_list, n, poc_dec);
1395             short_ref1_count += n;
1396         }
1397
1398         /* refFrameListLongTerm */
1399         if (priv->long_ref_count > 0) {
1400             for (i = 0; i < priv->long_ref_count; i++)
1401                 long_ref[i] = priv->long_ref[i];
1402             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1403             long_ref_count = i;
1404         }
1405
1406         // XXX: handle 8.2.4.2.5
1407     }
1408
1409     /* Check whether RefPicList1 is identical to RefPicList0, then
1410        swap if necessary */
1411     if (priv->RefPicList1_count > 1 &&
1412         priv->RefPicList1_count == priv->RefPicList0_count &&
1413         memcmp(priv->RefPicList0, priv->RefPicList1,
1414                priv->RefPicList0_count * sizeof(priv->RefPicList0[0])) == 0) {
1415         GstVaapiPictureH264 * const tmp = priv->RefPicList1[0];
1416         priv->RefPicList1[0] = priv->RefPicList1[1];
1417         priv->RefPicList1[1] = tmp;
1418     }
1419 }
1420
1421 #undef SORT_REF_LIST
1422
1423 static void
1424 clear_references(
1425     GstVaapiDecoderH264  *decoder,
1426     GstVaapiPictureH264 **pictures,
1427     guint                *picture_count
1428 )
1429 {
1430     const guint num_pictures = *picture_count;
1431     guint i;
1432
1433     for (i = 0; i < num_pictures; i++)
1434         gst_vaapi_picture_replace(&pictures[i], NULL);
1435     *picture_count = 0;
1436 }
1437
1438 static gboolean
1439 remove_reference_at(
1440     GstVaapiDecoderH264  *decoder,
1441     GstVaapiPictureH264 **pictures,
1442     guint                *picture_count,
1443     guint                 index
1444 )
1445 {
1446     guint num_pictures = *picture_count;
1447     GstVaapiPictureH264 *picture;
1448
1449     g_return_val_if_fail(index < num_pictures, FALSE);
1450
1451     picture = pictures[index];
1452     GST_VAAPI_PICTURE_FLAG_UNSET(picture, GST_VAAPI_PICTURE_FLAG_REFERENCE);
1453     picture->is_long_term = FALSE;
1454     picture->info.flags &= ~(VA_PICTURE_H264_SHORT_TERM_REFERENCE |
1455                              VA_PICTURE_H264_LONG_TERM_REFERENCE);
1456
1457     if (index != --num_pictures)
1458         gst_vaapi_picture_replace(&pictures[index], pictures[num_pictures]);
1459     gst_vaapi_picture_replace(&pictures[num_pictures], NULL);
1460     *picture_count = num_pictures;
1461     return TRUE;
1462 }
1463
1464 static gint
1465 find_short_term_reference(GstVaapiDecoderH264 *decoder, gint32 pic_num)
1466 {
1467     GstVaapiDecoderH264Private * const priv = decoder->priv;
1468     guint i;
1469
1470     for (i = 0; i < priv->short_ref_count; i++) {
1471         if (priv->short_ref[i]->pic_num == pic_num)
1472             return i;
1473     }
1474     GST_ERROR("found no short-term reference picture with PicNum = %d",
1475               pic_num);
1476     return -1;
1477 }
1478
1479 static gint
1480 find_long_term_reference(GstVaapiDecoderH264 *decoder, gint32 long_term_pic_num)
1481 {
1482     GstVaapiDecoderH264Private * const priv = decoder->priv;
1483     guint i;
1484
1485     for (i = 0; i < priv->long_ref_count; i++) {
1486         if (priv->long_ref[i]->long_term_pic_num == long_term_pic_num)
1487             return i;
1488     }
1489     GST_ERROR("found no long-term reference picture with LongTermPicNum = %d",
1490               long_term_pic_num);
1491     return -1;
1492 }
1493
1494 static void
1495 exec_picture_refs_modification_1(
1496     GstVaapiDecoderH264           *decoder,
1497     GstVaapiPictureH264           *picture,
1498     GstH264SliceHdr               *slice_hdr,
1499     guint                          list
1500 )
1501 {
1502     GstVaapiDecoderH264Private * const priv = decoder->priv;
1503     GstH264PPS * const pps = slice_hdr->pps;
1504     GstH264SPS * const sps = pps->sequence;
1505     GstH264RefPicListModification *ref_pic_list_modification;
1506     guint num_ref_pic_list_modifications;
1507     GstVaapiPictureH264 **ref_list;
1508     guint *ref_list_count_ptr, ref_list_count, ref_list_idx = 0;
1509     guint i, j, n, num_refs;
1510     gint found_ref_idx;
1511     gint32 MaxPicNum, CurrPicNum, picNumPred;
1512
1513     GST_DEBUG("modification process of reference picture list %u", list);
1514
1515     if (list == 0) {
1516         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l0;
1517         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l0;
1518         ref_list                       = priv->RefPicList0;
1519         ref_list_count_ptr             = &priv->RefPicList0_count;
1520         num_refs                       = slice_hdr->num_ref_idx_l0_active_minus1 + 1;
1521     }
1522     else {
1523         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l1;
1524         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l1;
1525         ref_list                       = priv->RefPicList1;
1526         ref_list_count_ptr             = &priv->RefPicList1_count;
1527         num_refs                       = slice_hdr->num_ref_idx_l1_active_minus1 + 1;
1528     }
1529     ref_list_count = *ref_list_count_ptr;
1530
1531     if (!GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1532         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 5); // 2 * MaxFrameNum
1533         CurrPicNum = 2 * slice_hdr->frame_num + 1;              // 2 * frame_num + 1
1534     }
1535     else {
1536         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 4); // MaxFrameNum
1537         CurrPicNum = slice_hdr->frame_num;                      // frame_num
1538     }
1539
1540     picNumPred = CurrPicNum;
1541
1542     for (i = 0; i < num_ref_pic_list_modifications; i++) {
1543         GstH264RefPicListModification * const l = &ref_pic_list_modification[i];
1544         if (l->modification_of_pic_nums_idc == 3)
1545             break;
1546
1547         /* 8.2.4.3.1 - Short-term reference pictures */
1548         if (l->modification_of_pic_nums_idc == 0 || l->modification_of_pic_nums_idc == 1) {
1549             gint32 abs_diff_pic_num = l->value.abs_diff_pic_num_minus1 + 1;
1550             gint32 picNum, picNumNoWrap;
1551
1552             // (8-34)
1553             if (l->modification_of_pic_nums_idc == 0) {
1554                 picNumNoWrap = picNumPred - abs_diff_pic_num;
1555                 if (picNumNoWrap < 0)
1556                     picNumNoWrap += MaxPicNum;
1557             }
1558
1559             // (8-35)
1560             else {
1561                 picNumNoWrap = picNumPred + abs_diff_pic_num;
1562                 if (picNumNoWrap >= MaxPicNum)
1563                     picNumNoWrap -= MaxPicNum;
1564             }
1565             picNumPred = picNumNoWrap;
1566
1567             // (8-36)
1568             picNum = picNumNoWrap;
1569             if (picNum > CurrPicNum)
1570                 picNum -= MaxPicNum;
1571
1572             // (8-37)
1573             for (j = num_refs; j > ref_list_idx; j--)
1574                 ref_list[j] = ref_list[j - 1];
1575             found_ref_idx = find_short_term_reference(decoder, picNum);
1576             ref_list[ref_list_idx++] =
1577                 found_ref_idx >= 0 ? priv->short_ref[found_ref_idx] : NULL;
1578             n = ref_list_idx;
1579             for (j = ref_list_idx; j <= num_refs; j++) {
1580                 gint32 PicNumF;
1581                 if (!ref_list[j])
1582                     continue;
1583                 PicNumF = ref_list[j]->is_long_term ?
1584                     MaxPicNum : ref_list[j]->pic_num;
1585                 if (PicNumF != picNum)
1586                     ref_list[n++] = ref_list[j];
1587             }
1588         }
1589
1590         /* 8.2.4.3.2 - Long-term reference pictures */
1591         else {
1592
1593             for (j = num_refs; j > ref_list_idx; j--)
1594                 ref_list[j] = ref_list[j - 1];
1595             found_ref_idx =
1596                 find_long_term_reference(decoder, l->value.long_term_pic_num);
1597             ref_list[ref_list_idx++] =
1598                 found_ref_idx >= 0 ? priv->long_ref[found_ref_idx] : NULL;
1599             n = ref_list_idx;
1600             for (j = ref_list_idx; j <= num_refs; j++) {
1601                 gint32 LongTermPicNumF;
1602                 if (!ref_list[j])
1603                     continue;
1604                 LongTermPicNumF = ref_list[j]->is_long_term ?
1605                     ref_list[j]->long_term_pic_num : INT_MAX;
1606                 if (LongTermPicNumF != l->value.long_term_pic_num)
1607                     ref_list[n++] = ref_list[j];
1608             }
1609         }
1610     }
1611
1612 #if DEBUG
1613     for (i = 0; i < num_refs; i++)
1614         if (!ref_list[i])
1615             GST_ERROR("list %u entry %u is empty", list, i);
1616 #endif
1617     *ref_list_count_ptr = num_refs;
1618 }
1619
1620 /* 8.2.4.3 - Modification process for reference picture lists */
1621 static void
1622 exec_picture_refs_modification(
1623     GstVaapiDecoderH264 *decoder,
1624     GstVaapiPictureH264 *picture,
1625     GstH264SliceHdr     *slice_hdr
1626 )
1627 {
1628     GST_DEBUG("execute ref_pic_list_modification()");
1629
1630     /* RefPicList0 */
1631     if (!GST_H264_IS_I_SLICE(slice_hdr) && !GST_H264_IS_SI_SLICE(slice_hdr) &&
1632         slice_hdr->ref_pic_list_modification_flag_l0)
1633         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 0);
1634
1635     /* RefPicList1 */
1636     if (GST_H264_IS_B_SLICE(slice_hdr) &&
1637         slice_hdr->ref_pic_list_modification_flag_l1)
1638         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 1);
1639 }
1640
1641 static gboolean
1642 init_picture_refs(
1643     GstVaapiDecoderH264 *decoder,
1644     GstVaapiPictureH264 *picture,
1645     GstH264SliceHdr     *slice_hdr
1646 )
1647 {
1648     GstVaapiDecoderH264Private * const priv = decoder->priv;
1649     GstVaapiPicture * const base_picture = &picture->base;
1650     guint i, num_refs;
1651
1652     init_picture_refs_pic_num(decoder, picture, slice_hdr);
1653
1654     priv->RefPicList0_count = 0;
1655     priv->RefPicList1_count = 0;
1656
1657     switch (base_picture->type) {
1658     case GST_VAAPI_PICTURE_TYPE_P:
1659     case GST_VAAPI_PICTURE_TYPE_SP:
1660         init_picture_refs_p_slice(decoder, picture, slice_hdr);
1661         break;
1662     case GST_VAAPI_PICTURE_TYPE_B:
1663         init_picture_refs_b_slice(decoder, picture, slice_hdr);
1664         break;
1665     default:
1666         break;
1667     }
1668
1669     exec_picture_refs_modification(decoder, picture, slice_hdr);
1670
1671     switch (base_picture->type) {
1672     case GST_VAAPI_PICTURE_TYPE_B:
1673         num_refs = 1 + slice_hdr->num_ref_idx_l1_active_minus1;
1674         for (i = priv->RefPicList1_count; i < num_refs; i++)
1675             priv->RefPicList1[i] = NULL;
1676         priv->RefPicList1_count = num_refs;
1677
1678         // fall-through
1679     case GST_VAAPI_PICTURE_TYPE_P:
1680     case GST_VAAPI_PICTURE_TYPE_SP:
1681         num_refs = 1 + slice_hdr->num_ref_idx_l0_active_minus1;
1682         for (i = priv->RefPicList0_count; i < num_refs; i++)
1683             priv->RefPicList0[i] = NULL;
1684         priv->RefPicList0_count = num_refs;
1685         break;
1686     default:
1687         break;
1688     }
1689     return TRUE;
1690 }
1691
1692 static gboolean
1693 init_picture(
1694     GstVaapiDecoderH264 *decoder,
1695     GstVaapiPictureH264 *picture,
1696     GstH264SliceHdr     *slice_hdr,
1697     GstH264NalUnit      *nalu
1698 )
1699 {
1700     GstVaapiDecoderH264Private * const priv = decoder->priv;
1701     GstVaapiPicture * const base_picture = &picture->base;
1702     VAPictureH264 *pic;
1703
1704     priv->prev_frame_num        = priv->frame_num;
1705     priv->frame_num             = slice_hdr->frame_num;
1706     picture->frame_num          = priv->frame_num;
1707     picture->frame_num_wrap     = priv->frame_num;
1708     picture->output_flag        = TRUE; /* XXX: conformant to Annex A only */
1709     base_picture->pts           = gst_adapter_prev_timestamp(priv->adapter, NULL);
1710
1711     /* Reset decoder state for IDR pictures */
1712     if (nalu->type == GST_H264_NAL_SLICE_IDR) {
1713         GST_DEBUG("<IDR>");
1714         GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_IDR);
1715         clear_references(decoder, priv->short_ref, &priv->short_ref_count);
1716         clear_references(decoder, priv->long_ref,  &priv->long_ref_count );
1717     }
1718
1719     /* Initialize VA picture info */
1720     pic = &picture->info;
1721     pic->picture_id = picture->base.surface_id;
1722     pic->frame_idx  = priv->frame_num;
1723
1724     /* Initialize slice type */
1725     switch (slice_hdr->type % 5) {
1726     case GST_H264_P_SLICE:
1727         base_picture->type = GST_VAAPI_PICTURE_TYPE_P;
1728         break;
1729     case GST_H264_B_SLICE:
1730         base_picture->type = GST_VAAPI_PICTURE_TYPE_B;
1731         break;
1732     case GST_H264_I_SLICE:
1733         base_picture->type = GST_VAAPI_PICTURE_TYPE_I;
1734         break;
1735     case GST_H264_SP_SLICE:
1736         base_picture->type = GST_VAAPI_PICTURE_TYPE_SP;
1737         break;
1738     case GST_H264_SI_SLICE:
1739         base_picture->type = GST_VAAPI_PICTURE_TYPE_SI;
1740         break;
1741     }
1742
1743     /* Initialize picture structure */
1744     if (!slice_hdr->field_pic_flag)
1745         base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
1746     else if (!slice_hdr->bottom_field_flag)
1747         base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD;
1748     else
1749         base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD;
1750
1751     if (nalu->ref_idc) {
1752         GstH264DecRefPicMarking * const dec_ref_pic_marking =
1753             &slice_hdr->dec_ref_pic_marking;
1754         GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_REFERENCE);
1755         if (GST_VAAPI_PICTURE_IS_IDR(picture)) {
1756             if (dec_ref_pic_marking->long_term_reference_flag)
1757                 picture->is_long_term = TRUE;
1758         }
1759     }
1760
1761     init_picture_poc(decoder, picture, slice_hdr);
1762     if (!init_picture_refs(decoder, picture, slice_hdr)) {
1763         GST_ERROR("failed to initialize references");
1764         return FALSE;
1765     }
1766     return TRUE;
1767 }
1768
1769 /* 8.2.5.3 - Sliding window decoded reference picture marking process */
1770 static gboolean
1771 exec_ref_pic_marking_sliding_window(GstVaapiDecoderH264 *decoder)
1772 {
1773     GstVaapiDecoderH264Private * const priv = decoder->priv;
1774     GstH264PPS * const pps = priv->current_picture->pps;
1775     GstH264SPS * const sps = pps->sequence;
1776     guint i, max_num_ref_frames, lowest_frame_num_index;
1777     gint32 lowest_frame_num;
1778
1779     GST_DEBUG("reference picture marking process (sliding window)");
1780
1781     max_num_ref_frames = sps->num_ref_frames;
1782     if (max_num_ref_frames == 0)
1783         max_num_ref_frames = 1;
1784
1785     if (priv->short_ref_count + priv->long_ref_count < max_num_ref_frames)
1786         return TRUE;
1787     if (priv->short_ref_count < 1)
1788         return FALSE;
1789
1790     lowest_frame_num = priv->short_ref[0]->frame_num_wrap;
1791     lowest_frame_num_index = 0;
1792     for (i = 1; i < priv->short_ref_count; i++) {
1793         if (priv->short_ref[i]->frame_num_wrap < lowest_frame_num) {
1794             lowest_frame_num = priv->short_ref[i]->frame_num_wrap;
1795             lowest_frame_num_index = i;
1796         }
1797     }
1798
1799     remove_reference_at(
1800         decoder,
1801         priv->short_ref, &priv->short_ref_count,
1802         lowest_frame_num_index
1803     );
1804     return TRUE;
1805 }
1806
1807 static inline gint32
1808 get_picNumX(GstVaapiPictureH264 *picture, GstH264RefPicMarking *ref_pic_marking)
1809 {
1810     gint32 pic_num;
1811
1812     if (GST_VAAPI_PICTURE_IS_FRAME(picture))
1813         pic_num = picture->frame_num_wrap;
1814     else
1815         pic_num = 2 * picture->frame_num_wrap + 1;
1816     pic_num -= ref_pic_marking->difference_of_pic_nums_minus1 + 1;
1817     return pic_num;
1818 }
1819
1820 /* 8.2.5.4.1. Mark-term reference picture as "unused for reference" */
1821 static void
1822 exec_ref_pic_marking_adaptive_mmco_1(
1823     GstVaapiDecoderH264  *decoder,
1824     GstVaapiPictureH264  *picture,
1825     GstH264RefPicMarking *ref_pic_marking
1826 )
1827 {
1828     GstVaapiDecoderH264Private * const priv = decoder->priv;
1829     gint32 i, picNumX;
1830
1831     picNumX = get_picNumX(picture, ref_pic_marking);
1832     i = find_short_term_reference(decoder, picNumX);
1833     if (i < 0)
1834         return;
1835     remove_reference_at(decoder, priv->short_ref, &priv->short_ref_count, i);
1836 }
1837
1838 /* 8.2.5.4.2. Mark long-term reference picture as "unused for reference" */
1839 static void
1840 exec_ref_pic_marking_adaptive_mmco_2(
1841     GstVaapiDecoderH264  *decoder,
1842     GstVaapiPictureH264  *picture,
1843     GstH264RefPicMarking *ref_pic_marking
1844 )
1845 {
1846     GstVaapiDecoderH264Private * const priv = decoder->priv;
1847     gint32 i;
1848
1849     i = find_long_term_reference(decoder, ref_pic_marking->long_term_pic_num);
1850     if (i < 0)
1851         return;
1852     remove_reference_at(decoder, priv->long_ref, &priv->long_ref_count, i);
1853 }
1854
1855 /* 8.2.5.4.3. Assign LongTermFrameIdx to a short-term reference picture */
1856 static void
1857 exec_ref_pic_marking_adaptive_mmco_3(
1858     GstVaapiDecoderH264  *decoder,
1859     GstVaapiPictureH264  *picture,
1860     GstH264RefPicMarking *ref_pic_marking
1861 )
1862 {
1863     GstVaapiDecoderH264Private * const priv = decoder->priv;
1864     VAPictureH264 *pic;
1865     gint32 i, picNumX;
1866
1867     for (i = 0; i < priv->long_ref_count; i++) {
1868         if (priv->long_ref[i]->info.frame_idx == ref_pic_marking->long_term_frame_idx)
1869             break;
1870     }
1871     if (i != priv->long_ref_count)
1872         remove_reference_at(decoder, priv->long_ref, &priv->long_ref_count, i);
1873
1874     picNumX = get_picNumX(picture, ref_pic_marking);
1875     i = find_short_term_reference(decoder, picNumX);
1876     if (i < 0)
1877         return;
1878
1879     picture = gst_vaapi_picture_ref(priv->short_ref[i]);
1880     remove_reference_at(decoder, priv->short_ref, &priv->short_ref_count, i);
1881     gst_vaapi_picture_replace(&priv->long_ref[priv->long_ref_count++], picture);
1882     gst_vaapi_picture_unref(picture);
1883
1884     picture->is_long_term = TRUE;
1885     pic = &picture->info;
1886     pic->frame_idx = ref_pic_marking->long_term_frame_idx;
1887     pic->flags &= ~VA_PICTURE_H264_SHORT_TERM_REFERENCE;
1888     pic->flags |= VA_PICTURE_H264_LONG_TERM_REFERENCE;
1889     GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_REFERENCE);
1890 }
1891
1892 /* 8.2.5.4.4. Mark pictures with LongTermFramIdx > max_long_term_frame_idx
1893  * as "unused for reference" */
1894 static void
1895 exec_ref_pic_marking_adaptive_mmco_4(
1896     GstVaapiDecoderH264  *decoder,
1897     GstVaapiPictureH264  *picture,
1898     GstH264RefPicMarking *ref_pic_marking
1899 )
1900 {
1901     GstVaapiDecoderH264Private * const priv = decoder->priv;
1902     gint32 i, long_term_frame_idx;
1903
1904     long_term_frame_idx = ref_pic_marking->max_long_term_frame_idx_plus1 - 1;
1905
1906     for (i = 0; i < priv->long_ref_count; i++) {
1907         if ((gint32)priv->long_ref[i]->info.frame_idx <= long_term_frame_idx)
1908             continue;
1909         remove_reference_at(decoder, priv->long_ref, &priv->long_ref_count, i);
1910         i--;
1911     }
1912 }
1913
1914 /* 8.2.5.4.5. Mark all reference pictures as "unused for reference" */
1915 static void
1916 exec_ref_pic_marking_adaptive_mmco_5(
1917     GstVaapiDecoderH264  *decoder,
1918     GstVaapiPictureH264  *picture,
1919     GstH264RefPicMarking *ref_pic_marking
1920 )
1921 {
1922     GstVaapiDecoderH264Private * const priv = decoder->priv;
1923     VAPictureH264 * const pic = &picture->info;
1924
1925     clear_references(decoder, priv->short_ref, &priv->short_ref_count);
1926     clear_references(decoder, priv->long_ref,  &priv->long_ref_count );
1927     dpb_flush(decoder);
1928
1929     priv->prev_pic_has_mmco5 = TRUE;
1930
1931     /* The picture shall be inferred to have had frame_num equal to 0 (7.4.3) */
1932     priv->frame_num = 0;
1933     priv->frame_num_offset = 0;
1934     picture->frame_num = 0;
1935
1936     /* Update TopFieldOrderCnt and BottomFieldOrderCnt (8.2.1) */
1937     if (picture->base.structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
1938         pic->TopFieldOrderCnt -= picture->poc;
1939     if (picture->base.structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
1940         pic->BottomFieldOrderCnt -= picture->poc;
1941     picture->poc = 0;
1942 }
1943
1944 /* 8.2.5.4.6. Assign a long-term frame index to the current picture */
1945 static void
1946 exec_ref_pic_marking_adaptive_mmco_6(
1947     GstVaapiDecoderH264  *decoder,
1948     GstVaapiPictureH264  *picture,
1949     GstH264RefPicMarking *ref_pic_marking
1950 )
1951 {
1952     picture->is_long_term = TRUE;
1953     picture->info.frame_idx = ref_pic_marking->long_term_frame_idx;
1954 }
1955
1956 /* 8.2.5.4. Adaptive memory control decoded reference picture marking process */
1957 static gboolean
1958 exec_ref_pic_marking_adaptive(
1959     GstVaapiDecoderH264     *decoder,
1960     GstVaapiPictureH264     *picture,
1961     GstH264DecRefPicMarking *dec_ref_pic_marking
1962 )
1963 {
1964     guint i;
1965
1966     GST_DEBUG("reference picture marking process (adaptive memory control)");
1967
1968     typedef void (*exec_ref_pic_marking_adaptive_mmco_func)(
1969         GstVaapiDecoderH264  *decoder,
1970         GstVaapiPictureH264  *picture,
1971         GstH264RefPicMarking *ref_pic_marking
1972     );
1973
1974     static const exec_ref_pic_marking_adaptive_mmco_func mmco_funcs[] = {
1975         NULL,
1976         exec_ref_pic_marking_adaptive_mmco_1,
1977         exec_ref_pic_marking_adaptive_mmco_2,
1978         exec_ref_pic_marking_adaptive_mmco_3,
1979         exec_ref_pic_marking_adaptive_mmco_4,
1980         exec_ref_pic_marking_adaptive_mmco_5,
1981         exec_ref_pic_marking_adaptive_mmco_6,
1982     };
1983
1984     for (i = 0; i < dec_ref_pic_marking->n_ref_pic_marking; i++) {
1985         GstH264RefPicMarking * const ref_pic_marking =
1986             &dec_ref_pic_marking->ref_pic_marking[i];
1987
1988         const guint mmco = ref_pic_marking->memory_management_control_operation;
1989         if (mmco < G_N_ELEMENTS(mmco_funcs) && mmco_funcs[mmco])
1990             mmco_funcs[mmco](decoder, picture, ref_pic_marking);
1991         else {
1992             GST_ERROR("unhandled MMCO %u", mmco);
1993             return FALSE;
1994         }
1995     }
1996     return TRUE;
1997 }
1998
1999 /* 8.2.5 - Execute reference picture marking process */
2000 static gboolean
2001 exec_ref_pic_marking(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
2002 {
2003     GstVaapiDecoderH264Private * const priv = decoder->priv;
2004     GstVaapiPictureH264 **picture_ptr;
2005
2006     priv->prev_pic_has_mmco5 = FALSE;
2007     priv->prev_pic_structure = picture->base.structure;
2008
2009     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
2010         return TRUE;
2011
2012     if (!GST_VAAPI_PICTURE_IS_IDR(picture)) {
2013         GstVaapiSliceH264 * const slice =
2014             gst_vaapi_picture_h264_get_last_slice(picture);
2015         GstH264DecRefPicMarking * const dec_ref_pic_marking =
2016             &slice->slice_hdr.dec_ref_pic_marking;
2017         if (dec_ref_pic_marking->adaptive_ref_pic_marking_mode_flag) {
2018             if (!exec_ref_pic_marking_adaptive(decoder, picture, dec_ref_pic_marking))
2019                 return FALSE;
2020         }
2021         else {
2022             if (!exec_ref_pic_marking_sliding_window(decoder))
2023                 return FALSE;
2024         }
2025     }
2026
2027     if (picture->is_long_term) {
2028         picture_ptr = &priv->long_ref[priv->long_ref_count++];
2029         picture->info.flags |= VA_PICTURE_H264_LONG_TERM_REFERENCE;
2030     }
2031     else {
2032         picture_ptr = &priv->short_ref[priv->short_ref_count++];
2033         picture->info.flags |= VA_PICTURE_H264_SHORT_TERM_REFERENCE;
2034     }
2035     gst_vaapi_picture_replace(picture_ptr, picture);
2036     return TRUE;
2037 }
2038
2039 static void
2040 vaapi_init_picture(VAPictureH264 *pic)
2041 {
2042     pic->picture_id           = VA_INVALID_ID;
2043     pic->frame_idx            = 0;
2044     pic->flags                = VA_PICTURE_H264_INVALID;
2045     pic->TopFieldOrderCnt     = 0;
2046     pic->BottomFieldOrderCnt  = 0;
2047 }
2048
2049 static void
2050 vaapi_fill_picture(VAPictureH264 *pic, GstVaapiPictureH264 *picture)
2051 {
2052     pic->picture_id = picture->base.surface_id;
2053     pic->flags = 0;
2054
2055     if (picture->info.flags & VA_PICTURE_H264_LONG_TERM_REFERENCE) {
2056         pic->flags |= VA_PICTURE_H264_LONG_TERM_REFERENCE;
2057         pic->frame_idx = picture->info.frame_idx;
2058     }
2059     else {
2060         if (picture->info.flags & VA_PICTURE_H264_SHORT_TERM_REFERENCE)
2061             pic->flags |= VA_PICTURE_H264_SHORT_TERM_REFERENCE;
2062         pic->frame_idx = picture->frame_num;
2063     }
2064
2065     switch (picture->base.structure) {
2066     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
2067         pic->TopFieldOrderCnt = picture->info.TopFieldOrderCnt;
2068         pic->BottomFieldOrderCnt = picture->info.BottomFieldOrderCnt;
2069         break;
2070     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
2071         pic->flags |= VA_PICTURE_H264_TOP_FIELD;
2072         pic->TopFieldOrderCnt = picture->info.BottomFieldOrderCnt;
2073         pic->BottomFieldOrderCnt = 0;
2074         break;
2075     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
2076         pic->flags |= VA_PICTURE_H264_BOTTOM_FIELD;
2077         pic->BottomFieldOrderCnt = picture->info.BottomFieldOrderCnt;
2078         pic->TopFieldOrderCnt = 0;
2079         break;
2080     }
2081 }
2082
2083 static gboolean
2084 fill_picture(
2085     GstVaapiDecoderH264 *decoder,
2086     GstVaapiPictureH264 *picture,
2087     GstH264SliceHdr     *slice_hdr,
2088     GstH264NalUnit      *nalu
2089 )
2090 {
2091     GstVaapiDecoderH264Private * const priv = decoder->priv;
2092     GstVaapiPicture * const base_picture = &picture->base;
2093     GstH264PPS * const pps = picture->pps;
2094     GstH264SPS * const sps = pps->sequence;
2095     VAPictureParameterBufferH264 * const pic_param = base_picture->param;
2096     guint i, n;
2097
2098     /* Fill in VAPictureParameterBufferH264 */
2099     vaapi_fill_picture(&pic_param->CurrPic, picture);
2100     for (i = 0, n = 0; i < priv->short_ref_count; i++, n++)
2101         vaapi_fill_picture(&pic_param->ReferenceFrames[n], priv->short_ref[i]);
2102     for (i = 0; i < priv->long_ref_count; i++, n++)
2103         vaapi_fill_picture(&pic_param->ReferenceFrames[n], priv->long_ref[i]);
2104     for (; n < G_N_ELEMENTS(pic_param->ReferenceFrames); n++)
2105         vaapi_init_picture(&pic_param->ReferenceFrames[n]);
2106
2107 #define COPY_FIELD(s, f) \
2108     pic_param->f = (s)->f
2109
2110 #define COPY_BFM(a, s, f) \
2111     pic_param->a.bits.f = (s)->f
2112
2113     pic_param->picture_width_in_mbs_minus1  = ((priv->width + 15) >> 4)  - 1;
2114     pic_param->picture_height_in_mbs_minus1 = ((priv->height + 15) >> 4) - 1;
2115     pic_param->frame_num                    = priv->frame_num;
2116
2117     COPY_FIELD(sps, bit_depth_luma_minus8);
2118     COPY_FIELD(sps, bit_depth_chroma_minus8);
2119     COPY_FIELD(sps, num_ref_frames);
2120     COPY_FIELD(pps, num_slice_groups_minus1);
2121     COPY_FIELD(pps, slice_group_map_type);
2122     COPY_FIELD(pps, slice_group_change_rate_minus1);
2123     COPY_FIELD(pps, pic_init_qp_minus26);
2124     COPY_FIELD(pps, pic_init_qs_minus26);
2125     COPY_FIELD(pps, chroma_qp_index_offset);
2126     COPY_FIELD(pps, second_chroma_qp_index_offset);
2127
2128     pic_param->seq_fields.value                                         = 0; /* reset all bits */
2129     pic_param->seq_fields.bits.residual_colour_transform_flag           = sps->separate_colour_plane_flag;
2130     pic_param->seq_fields.bits.MinLumaBiPredSize8x8                     = sps->level_idc >= 31; /* A.3.3.2 */
2131
2132     COPY_BFM(seq_fields, sps, chroma_format_idc);
2133     COPY_BFM(seq_fields, sps, gaps_in_frame_num_value_allowed_flag);
2134     COPY_BFM(seq_fields, sps, frame_mbs_only_flag);
2135     COPY_BFM(seq_fields, sps, mb_adaptive_frame_field_flag);
2136     COPY_BFM(seq_fields, sps, direct_8x8_inference_flag);
2137     COPY_BFM(seq_fields, sps, log2_max_frame_num_minus4);
2138     COPY_BFM(seq_fields, sps, pic_order_cnt_type);
2139     COPY_BFM(seq_fields, sps, log2_max_pic_order_cnt_lsb_minus4);
2140     COPY_BFM(seq_fields, sps, delta_pic_order_always_zero_flag);
2141
2142     pic_param->pic_fields.value                                         = 0; /* reset all bits */
2143     pic_param->pic_fields.bits.field_pic_flag                           = slice_hdr->field_pic_flag;
2144     pic_param->pic_fields.bits.reference_pic_flag                       = GST_VAAPI_PICTURE_IS_REFERENCE(picture);
2145
2146     COPY_BFM(pic_fields, pps, entropy_coding_mode_flag);
2147     COPY_BFM(pic_fields, pps, weighted_pred_flag);
2148     COPY_BFM(pic_fields, pps, weighted_bipred_idc);
2149     COPY_BFM(pic_fields, pps, transform_8x8_mode_flag);
2150     COPY_BFM(pic_fields, pps, constrained_intra_pred_flag);
2151     COPY_BFM(pic_fields, pps, pic_order_present_flag);
2152     COPY_BFM(pic_fields, pps, deblocking_filter_control_present_flag);
2153     COPY_BFM(pic_fields, pps, redundant_pic_cnt_present_flag);
2154     return TRUE;
2155 }
2156
2157 /* Detection of the first VCL NAL unit of a primary coded picture (7.4.1.2.4) */
2158 static gboolean
2159 is_new_picture(
2160     GstVaapiDecoderH264 *decoder,
2161     GstH264NalUnit      *nalu,
2162     GstH264SliceHdr     *slice_hdr
2163 )
2164 {
2165     GstVaapiDecoderH264Private * const priv = decoder->priv;
2166     GstH264PPS * const pps = slice_hdr->pps;
2167     GstH264SPS * const sps = pps->sequence;
2168     GstVaapiSliceH264 *slice;
2169     GstH264SliceHdr *prev_slice_hdr;
2170
2171     if (!priv->current_picture)
2172         return TRUE;
2173
2174     slice = gst_vaapi_picture_h264_get_last_slice(priv->current_picture);
2175     if (!slice)
2176         return FALSE;
2177     prev_slice_hdr = &slice->slice_hdr;
2178
2179 #define CHECK_EXPR(expr, field_name) do {              \
2180         if (!(expr)) {                                 \
2181             GST_DEBUG(field_name " differs in value"); \
2182             return TRUE;                               \
2183         }                                              \
2184     } while (0)
2185
2186 #define CHECK_VALUE(new_slice_hdr, old_slice_hdr, field) \
2187     CHECK_EXPR(((new_slice_hdr)->field == (old_slice_hdr)->field), #field)
2188
2189     /* frame_num differs in value, regardless of inferred values to 0 */
2190     CHECK_VALUE(slice_hdr, prev_slice_hdr, frame_num);
2191
2192     /* pic_parameter_set_id differs in value */
2193     CHECK_VALUE(slice_hdr, prev_slice_hdr, pps);
2194
2195     /* field_pic_flag differs in value */
2196     CHECK_VALUE(slice_hdr, prev_slice_hdr, field_pic_flag);
2197
2198     /* bottom_field_flag is present in both and differs in value */
2199     if (slice_hdr->field_pic_flag && prev_slice_hdr->field_pic_flag)
2200         CHECK_VALUE(slice_hdr, prev_slice_hdr, bottom_field_flag);
2201
2202     /* nal_ref_idc differs in value with one of the nal_ref_idc values is 0 */
2203     CHECK_EXPR(((GST_VAAPI_PICTURE_IS_REFERENCE(priv->current_picture) ^
2204                  (nalu->ref_idc != 0)) == 0), "nal_ref_idc");
2205
2206     /* POC type is 0 for both and either pic_order_cnt_lsb differs in
2207        value or delta_pic_order_cnt_bottom differs in value */
2208     if (sps->pic_order_cnt_type == 0) {
2209         CHECK_VALUE(slice_hdr, prev_slice_hdr, pic_order_cnt_lsb);
2210         if (pps->pic_order_present_flag && !slice_hdr->field_pic_flag)
2211             CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt_bottom);
2212     }
2213
2214     /* POC type is 1 for both and either delta_pic_order_cnt[0]
2215        differs in value or delta_pic_order_cnt[1] differs in value */
2216     else if (sps->pic_order_cnt_type == 1) {
2217         CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt[0]);
2218         CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt[1]);
2219     }
2220
2221     /* IdrPicFlag differs in value */
2222     CHECK_EXPR(((GST_VAAPI_PICTURE_IS_IDR(priv->current_picture) ^
2223                  (nalu->type == GST_H264_NAL_SLICE_IDR)) == 0), "IdrPicFlag");
2224
2225     /* IdrPicFlag is equal to 1 for both and idr_pic_id differs in value */
2226     if (GST_VAAPI_PICTURE_IS_IDR(priv->current_picture))
2227         CHECK_VALUE(slice_hdr, prev_slice_hdr, idr_pic_id);
2228
2229 #undef CHECK_EXPR
2230 #undef CHECK_VALUE
2231     return FALSE;
2232 }
2233
2234 static GstVaapiDecoderStatus
2235 decode_picture(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu, GstH264SliceHdr *slice_hdr)
2236 {
2237     GstVaapiDecoderH264Private * const priv = decoder->priv;
2238     GstVaapiPictureH264 *picture;
2239     GstVaapiDecoderStatus status;
2240     GstH264PPS * const pps = slice_hdr->pps;
2241     GstH264SPS * const sps = pps->sequence;
2242
2243     status = decode_current_picture(decoder);
2244     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2245         return status;
2246
2247     status = ensure_context(decoder, sps);
2248     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2249         return status;
2250
2251     picture = gst_vaapi_picture_h264_new(decoder);
2252     if (!picture) {
2253         GST_ERROR("failed to allocate picture");
2254         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2255     }
2256     priv->current_picture = picture;
2257
2258     picture->pps = pps;
2259
2260     status = ensure_quant_matrix(decoder, picture);
2261     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS) {
2262         GST_ERROR("failed to reset quantizer matrix");
2263         return status;
2264     }
2265
2266     if (!init_picture(decoder, picture, slice_hdr, nalu))
2267         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2268     if (!fill_picture(decoder, picture, slice_hdr, nalu))
2269         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2270     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2271 }
2272
2273 static gboolean
2274 decode_picture_end(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
2275 {
2276     if (!exec_ref_pic_marking(decoder, picture))
2277         return FALSE;
2278     if (!dpb_add(decoder, picture))
2279         return FALSE;
2280     return TRUE;
2281 }
2282
2283 static inline guint
2284 get_slice_data_bit_offset(GstH264SliceHdr *slice_hdr, GstH264NalUnit *nalu)
2285 {
2286     guint epb_count;
2287
2288     epb_count = slice_hdr->n_emulation_prevention_bytes;
2289     return 8 /* nal_unit_type */ + slice_hdr->header_size - epb_count * 8;
2290 }
2291
2292 static gboolean
2293 fill_pred_weight_table(GstVaapiDecoderH264 *decoder, GstVaapiSliceH264 *slice)
2294 {
2295     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2296     GstH264PPS * const pps = slice_hdr->pps;
2297     GstH264SPS * const sps = pps->sequence;
2298     GstH264PredWeightTable * const w = &slice_hdr->pred_weight_table;
2299     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2300     guint num_weight_tables = 0;
2301     gint i, j;
2302
2303     if (pps->weighted_pred_flag &&
2304         (GST_H264_IS_P_SLICE(slice_hdr) || GST_H264_IS_SP_SLICE(slice_hdr)))
2305         num_weight_tables = 1;
2306     else if (pps->weighted_bipred_idc == 1 && GST_H264_IS_B_SLICE(slice_hdr))
2307         num_weight_tables = 2;
2308     else
2309         num_weight_tables = 0;
2310
2311     slice_param->luma_log2_weight_denom   = w->luma_log2_weight_denom;
2312     slice_param->chroma_log2_weight_denom = w->chroma_log2_weight_denom;
2313     slice_param->luma_weight_l0_flag      = 0;
2314     slice_param->chroma_weight_l0_flag    = 0;
2315     slice_param->luma_weight_l1_flag      = 0;
2316     slice_param->chroma_weight_l1_flag    = 0;
2317
2318     if (num_weight_tables < 1)
2319         return TRUE;
2320
2321     slice_param->luma_weight_l0_flag = 1;
2322     for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2323         slice_param->luma_weight_l0[i] = w->luma_weight_l0[i];
2324         slice_param->luma_offset_l0[i] = w->luma_offset_l0[i];
2325     }
2326
2327     slice_param->chroma_weight_l0_flag = sps->chroma_array_type != 0;
2328     if (slice_param->chroma_weight_l0_flag) {
2329         for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2330             for (j = 0; j < 2; j++) {
2331                 slice_param->chroma_weight_l0[i][j] = w->chroma_weight_l0[i][j];
2332                 slice_param->chroma_offset_l0[i][j] = w->chroma_offset_l0[i][j];
2333             }
2334         }
2335     }
2336
2337     if (num_weight_tables < 2)
2338         return TRUE;
2339
2340     slice_param->luma_weight_l1_flag = 1;
2341     for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2342         slice_param->luma_weight_l1[i] = w->luma_weight_l1[i];
2343         slice_param->luma_offset_l1[i] = w->luma_offset_l1[i];
2344     }
2345
2346     slice_param->chroma_weight_l1_flag = sps->chroma_array_type != 0;
2347     if (slice_param->chroma_weight_l1_flag) {
2348         for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2349             for (j = 0; j < 2; j++) {
2350                 slice_param->chroma_weight_l1[i][j] = w->chroma_weight_l1[i][j];
2351                 slice_param->chroma_offset_l1[i][j] = w->chroma_offset_l1[i][j];
2352             }
2353         }
2354     }
2355     return TRUE;
2356 }
2357
2358 static gboolean
2359 fill_RefPicList(GstVaapiDecoderH264 *decoder, GstVaapiSliceH264 *slice)
2360 {
2361     GstVaapiDecoderH264Private * const priv = decoder->priv;
2362     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2363     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2364     guint i, num_ref_lists = 0;
2365
2366     slice_param->num_ref_idx_l0_active_minus1 = 0;
2367     slice_param->num_ref_idx_l1_active_minus1 = 0;
2368
2369     if (GST_H264_IS_B_SLICE(slice_hdr))
2370         num_ref_lists = 2;
2371     else if (GST_H264_IS_I_SLICE(slice_hdr))
2372         num_ref_lists = 0;
2373     else
2374         num_ref_lists = 1;
2375
2376     if (num_ref_lists < 1)
2377         return TRUE;
2378
2379     slice_param->num_ref_idx_l0_active_minus1 =
2380         slice_hdr->num_ref_idx_l0_active_minus1;
2381
2382     for (i = 0; i < priv->RefPicList0_count && priv->RefPicList0[i]; i++)
2383         vaapi_fill_picture(&slice_param->RefPicList0[i], priv->RefPicList0[i]);
2384     for (; i <= slice_param->num_ref_idx_l0_active_minus1; i++)
2385         vaapi_init_picture(&slice_param->RefPicList0[i]);
2386
2387     if (num_ref_lists < 2)
2388         return TRUE;
2389
2390     slice_param->num_ref_idx_l1_active_minus1 =
2391         slice_hdr->num_ref_idx_l1_active_minus1;
2392
2393     for (i = 0; i < priv->RefPicList1_count && priv->RefPicList1[i]; i++)
2394         vaapi_fill_picture(&slice_param->RefPicList1[i], priv->RefPicList1[i]);
2395     for (; i <= slice_param->num_ref_idx_l1_active_minus1; i++)
2396         vaapi_init_picture(&slice_param->RefPicList1[i]);
2397     return TRUE;
2398 }
2399
2400 static gboolean
2401 fill_slice(
2402     GstVaapiDecoderH264 *decoder,
2403     GstVaapiSliceH264   *slice,
2404     GstH264NalUnit      *nalu
2405 )
2406 {
2407     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2408     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2409
2410     /* Fill in VASliceParameterBufferH264 */
2411     slice_param->slice_data_bit_offset          = get_slice_data_bit_offset(slice_hdr, nalu);
2412     slice_param->first_mb_in_slice              = slice_hdr->first_mb_in_slice;
2413     slice_param->slice_type                     = slice_hdr->type % 5;
2414     slice_param->direct_spatial_mv_pred_flag    = slice_hdr->direct_spatial_mv_pred_flag;
2415     slice_param->cabac_init_idc                 = slice_hdr->cabac_init_idc;
2416     slice_param->slice_qp_delta                 = slice_hdr->slice_qp_delta;
2417     slice_param->disable_deblocking_filter_idc  = slice_hdr->disable_deblocking_filter_idc;
2418     slice_param->slice_alpha_c0_offset_div2     = slice_hdr->slice_alpha_c0_offset_div2;
2419     slice_param->slice_beta_offset_div2         = slice_hdr->slice_beta_offset_div2;
2420
2421     if (!fill_RefPicList(decoder, slice))
2422         return FALSE;
2423     if (!fill_pred_weight_table(decoder, slice))
2424         return FALSE;
2425     return TRUE;
2426 }
2427
2428 static GstVaapiDecoderStatus
2429 decode_slice(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
2430 {
2431     GstVaapiDecoderH264Private * const priv = decoder->priv;
2432     GstVaapiDecoderStatus status;
2433     GstVaapiPictureH264 *picture;
2434     GstVaapiSliceH264 *slice = NULL;
2435     GstH264SliceHdr *slice_hdr;
2436     GstH264ParserResult result;
2437
2438     GST_DEBUG("slice (%u bytes)", nalu->size);
2439
2440     slice = gst_vaapi_slice_h264_new(
2441         decoder,
2442         nalu->data + nalu->offset,
2443         nalu->size
2444     );
2445     if (!slice) {
2446         GST_ERROR("failed to allocate slice");
2447         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2448     }
2449
2450     slice_hdr = &slice->slice_hdr;
2451     memset(slice_hdr, 0, sizeof(*slice_hdr));
2452     result = gst_h264_parser_parse_slice_hdr(priv->parser, nalu, slice_hdr, TRUE, TRUE);
2453     if (result != GST_H264_PARSER_OK) {
2454         status = get_status(result);
2455         goto error;
2456     }
2457
2458     if (is_new_picture(decoder, nalu, slice_hdr)) {
2459         status = decode_picture(decoder, nalu, slice_hdr);
2460         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2461             goto error;
2462     }
2463     picture = priv->current_picture;
2464
2465     if (!fill_slice(decoder, slice, nalu)) {
2466         status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2467         goto error;
2468     }
2469     gst_vaapi_picture_add_slice(
2470         GST_VAAPI_PICTURE_CAST(picture),
2471         GST_VAAPI_SLICE_CAST(slice)
2472     );
2473     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2474
2475 error:
2476     if (slice)
2477         gst_mini_object_unref(GST_MINI_OBJECT(slice));
2478     return status;
2479 }
2480
2481 static inline gint
2482 scan_for_start_code(GstAdapter *adapter, guint ofs, guint size, guint32 *scp)
2483 {
2484     return (gint)gst_adapter_masked_scan_uint32_peek(adapter,
2485                                                      0xffffff00, 0x00000100,
2486                                                      ofs, size,
2487                                                      scp);
2488 }
2489
2490 static GstVaapiDecoderStatus
2491 decode_nalu(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
2492 {
2493     GstVaapiDecoderStatus status;
2494
2495     switch (nalu->type) {
2496     case GST_H264_NAL_SLICE_IDR:
2497         /* fall-through. IDR specifics are handled in init_picture() */
2498     case GST_H264_NAL_SLICE:
2499         status = decode_slice(decoder, nalu);
2500         break;
2501     case GST_H264_NAL_SPS:
2502         status = decode_sps(decoder, nalu);
2503         break;
2504     case GST_H264_NAL_PPS:
2505         status = decode_pps(decoder, nalu);
2506         break;
2507     case GST_H264_NAL_SEI:
2508         status = decode_sei(decoder, nalu);
2509         break;
2510     case GST_H264_NAL_SEQ_END:
2511         status = decode_sequence_end(decoder);
2512         break;
2513     case GST_H264_NAL_AU_DELIMITER:
2514         /* skip all Access Unit NALs */
2515         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2516         break;
2517     case GST_H264_NAL_FILLER_DATA:
2518         /* skip all Filler Data NALs */
2519         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2520         break;
2521     default:
2522         GST_WARNING("unsupported NAL unit type %d", nalu->type);
2523         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
2524         break;
2525     }
2526     return status;
2527 }
2528
2529 static GstVaapiDecoderStatus
2530 decode_buffer(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
2531 {
2532     GstVaapiDecoderH264Private * const priv = decoder->priv;
2533     GstVaapiDecoderStatus status;
2534     GstH264ParserResult result;
2535     GstH264NalUnit nalu;
2536     gboolean is_eos;
2537     const guchar *buf;
2538     guint i, buf_size, nalu_size, size;
2539     guint32 start_code;
2540     gint ofs;
2541
2542     buf      = GST_BUFFER_DATA(buffer);
2543     buf_size = GST_BUFFER_SIZE(buffer);
2544     is_eos   = GST_BUFFER_IS_EOS(buffer);
2545     if (buf && buf_size > 0)
2546         gst_adapter_push(priv->adapter, gst_buffer_ref(buffer));
2547
2548     size = gst_adapter_available(priv->adapter);
2549     do {
2550         if (size == 0) {
2551             status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2552             break;
2553         }
2554
2555         status = gst_vaapi_decoder_check_status(GST_VAAPI_DECODER(decoder));
2556         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2557             break;
2558
2559         status = GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2560         if (priv->is_avc) {
2561             if (size < priv->nal_length_size)
2562                 break;
2563             buf = gst_adapter_peek(priv->adapter, priv->nal_length_size);
2564
2565             nalu_size = 0;
2566             for (i = 0; i < priv->nal_length_size; i++)
2567                 nalu_size = (nalu_size << 8) | buf[i];
2568
2569             buf_size = priv->nal_length_size + nalu_size;
2570             if (size < buf_size)
2571                 break;
2572             buffer = gst_adapter_take_buffer(priv->adapter, buf_size);
2573             size -= buf_size;
2574
2575             buf      = GST_BUFFER_DATA(buffer);
2576             buf_size = GST_BUFFER_SIZE(buffer);
2577
2578             result = gst_h264_parser_identify_nalu_avc(
2579                 priv->parser,
2580                 buf, 0, buf_size, priv->nal_length_size,
2581                 &nalu
2582             );
2583         }
2584         else {
2585             if (size < 4)
2586                 break;
2587             ofs = scan_for_start_code(priv->adapter, 0, size, &start_code);
2588             if (ofs < 0)
2589                 break;
2590             gst_adapter_flush(priv->adapter, ofs);
2591             size -= ofs;
2592
2593             ofs = G_UNLIKELY(size < 8) ? -1 :
2594                 scan_for_start_code(priv->adapter, 4, size - 4, NULL);
2595             if (ofs < 0) {
2596                 // Assume the whole NAL unit is present if end-of-stream
2597                 if (!is_eos)
2598                     break;
2599                 ofs = size;
2600             }
2601             buffer = gst_adapter_take_buffer(priv->adapter, ofs);
2602             size -= ofs;
2603
2604             buf      = GST_BUFFER_DATA(buffer);
2605             buf_size = GST_BUFFER_SIZE(buffer);
2606
2607             result = gst_h264_parser_identify_nalu_unchecked(
2608                 priv->parser,
2609                 buf, 0, buf_size,
2610                 &nalu
2611             );
2612         }
2613         status = get_status(result);
2614         if (status == GST_VAAPI_DECODER_STATUS_SUCCESS)
2615             status = decode_nalu(decoder, &nalu);
2616         gst_buffer_unref(buffer);
2617     } while (status == GST_VAAPI_DECODER_STATUS_SUCCESS);
2618
2619     if (is_eos && (status == GST_VAAPI_DECODER_STATUS_SUCCESS ||
2620                    status == GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA))
2621         status = decode_sequence_end(decoder);
2622     return status;
2623 }
2624
2625 static GstVaapiDecoderStatus
2626 decode_codec_data(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
2627 {
2628     GstVaapiDecoderH264Private * const priv = decoder->priv;
2629     GstVaapiDecoderStatus status;
2630     GstH264NalUnit nalu;
2631     GstH264ParserResult result;
2632     guchar *buf;
2633     guint buf_size;
2634     guint i, ofs, num_sps, num_pps;
2635
2636     buf      = GST_BUFFER_DATA(buffer);
2637     buf_size = GST_BUFFER_SIZE(buffer);
2638     if (!buf || buf_size == 0)
2639         return GST_VAAPI_DECODER_STATUS_SUCCESS;
2640
2641     if (buf_size < 8)
2642         return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2643
2644     if (buf[0] != 1) {
2645         GST_ERROR("failed to decode codec-data, not in avcC format");
2646         return GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
2647     }
2648
2649     priv->nal_length_size = (buf[4] & 0x03) + 1;
2650
2651     num_sps = buf[5] & 0x1f;
2652     ofs = 6;
2653
2654     for (i = 0; i < num_sps; i++) {
2655         result = gst_h264_parser_identify_nalu_avc(
2656             priv->parser,
2657             buf, ofs, buf_size, 2,
2658             &nalu
2659         );
2660         if (result != GST_H264_PARSER_OK)
2661             return get_status(result);
2662
2663         status = decode_sps(decoder, &nalu);
2664         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2665             return status;
2666         ofs = nalu.offset + nalu.size;
2667     }
2668
2669     num_pps = buf[ofs];
2670     ofs++;
2671
2672     for (i = 0; i < num_pps; i++) {
2673         result = gst_h264_parser_identify_nalu_avc(
2674             priv->parser,
2675             buf, ofs, buf_size, 2,
2676             &nalu
2677         );
2678         if (result != GST_H264_PARSER_OK)
2679             return get_status(result);
2680
2681         status = decode_pps(decoder, &nalu);
2682         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2683             return status;
2684         ofs = nalu.offset + nalu.size;
2685     }
2686
2687     priv->is_avc = TRUE;
2688     return status;
2689 }
2690
2691 GstVaapiDecoderStatus
2692 gst_vaapi_decoder_h264_decode(GstVaapiDecoder *base, GstBuffer *buffer)
2693 {
2694     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264(base);
2695     GstVaapiDecoderH264Private * const priv = decoder->priv;
2696     GstVaapiDecoderStatus status;
2697     GstBuffer *codec_data;
2698
2699     g_return_val_if_fail(priv->is_constructed,
2700                          GST_VAAPI_DECODER_STATUS_ERROR_INIT_FAILED);
2701
2702     if (!priv->is_opened) {
2703         priv->is_opened = gst_vaapi_decoder_h264_open(decoder, buffer);
2704         if (!priv->is_opened)
2705             return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CODEC;
2706
2707         codec_data = GST_VAAPI_DECODER_CODEC_DATA(decoder);
2708         if (codec_data) {
2709             status = decode_codec_data(decoder, codec_data);
2710             if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2711                 return status;
2712         }
2713      }
2714      return decode_buffer(decoder, buffer);
2715 }
2716
2717 static void
2718 gst_vaapi_decoder_h264_finalize(GObject *object)
2719 {
2720     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264(object);
2721
2722     gst_vaapi_decoder_h264_destroy(decoder);
2723
2724     G_OBJECT_CLASS(gst_vaapi_decoder_h264_parent_class)->finalize(object);
2725 }
2726
2727 static void
2728 gst_vaapi_decoder_h264_constructed(GObject *object)
2729 {
2730     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264(object);
2731     GstVaapiDecoderH264Private * const priv = decoder->priv;
2732     GObjectClass *parent_class;
2733
2734     parent_class = G_OBJECT_CLASS(gst_vaapi_decoder_h264_parent_class);
2735     if (parent_class->constructed)
2736         parent_class->constructed(object);
2737
2738     priv->is_constructed = gst_vaapi_decoder_h264_create(decoder);
2739 }
2740
2741 static void
2742 gst_vaapi_decoder_h264_class_init(GstVaapiDecoderH264Class *klass)
2743 {
2744     GObjectClass * const object_class = G_OBJECT_CLASS(klass);
2745     GstVaapiDecoderClass * const decoder_class = GST_VAAPI_DECODER_CLASS(klass);
2746
2747     g_type_class_add_private(klass, sizeof(GstVaapiDecoderH264Private));
2748
2749     object_class->finalize      = gst_vaapi_decoder_h264_finalize;
2750     object_class->constructed   = gst_vaapi_decoder_h264_constructed;
2751
2752     decoder_class->decode       = gst_vaapi_decoder_h264_decode;
2753 }
2754
2755 static void
2756 gst_vaapi_decoder_h264_init(GstVaapiDecoderH264 *decoder)
2757 {
2758     GstVaapiDecoderH264Private *priv;
2759
2760     priv                        = GST_VAAPI_DECODER_H264_GET_PRIVATE(decoder);
2761     decoder->priv               = priv;
2762     priv->parser                = NULL;
2763     priv->current_picture       = NULL;
2764     priv->dpb_count             = 0;
2765     priv->dpb_size              = 0;
2766     priv->profile               = GST_VAAPI_PROFILE_UNKNOWN;
2767     priv->entrypoint            = GST_VAAPI_ENTRYPOINT_VLD;
2768     priv->chroma_type           = GST_VAAPI_CHROMA_TYPE_YUV420;
2769     priv->short_ref_count       = 0;
2770     priv->long_ref_count        = 0;
2771     priv->RefPicList0_count     = 0;
2772     priv->RefPicList1_count     = 0;
2773     priv->nal_length_size       = 0;
2774     priv->width                 = 0;
2775     priv->height                = 0;
2776     priv->adapter               = NULL;
2777     priv->field_poc[0]          = 0;
2778     priv->field_poc[1]          = 0;
2779     priv->poc_msb               = 0;
2780     priv->poc_lsb               = 0;
2781     priv->prev_poc_msb          = 0;
2782     priv->prev_poc_lsb          = 0;
2783     priv->frame_num_offset      = 0;
2784     priv->frame_num             = 0;
2785     priv->prev_frame_num        = 0;
2786     priv->prev_pic_has_mmco5    = FALSE;
2787     priv->prev_pic_structure    = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
2788     priv->is_constructed        = FALSE;
2789     priv->is_opened             = FALSE;
2790     priv->is_avc                = FALSE;
2791     priv->has_context           = FALSE;
2792
2793     memset(priv->dpb, 0, sizeof(priv->dpb));
2794     memset(priv->short_ref, 0, sizeof(priv->short_ref));
2795     memset(priv->long_ref, 0, sizeof(priv->long_ref));
2796     memset(priv->RefPicList0, 0, sizeof(priv->RefPicList0));
2797     memset(priv->RefPicList1, 0, sizeof(priv->RefPicList1));
2798 }
2799
2800 /**
2801  * gst_vaapi_decoder_h264_new:
2802  * @display: a #GstVaapiDisplay
2803  * @caps: a #GstCaps holding codec information
2804  *
2805  * Creates a new #GstVaapiDecoder for MPEG-2 decoding.  The @caps can
2806  * hold extra information like codec-data and pictured coded size.
2807  *
2808  * Return value: the newly allocated #GstVaapiDecoder object
2809  */
2810 GstVaapiDecoder *
2811 gst_vaapi_decoder_h264_new(GstVaapiDisplay *display, GstCaps *caps)
2812 {
2813     GstVaapiDecoderH264 *decoder;
2814
2815     g_return_val_if_fail(GST_VAAPI_IS_DISPLAY(display), NULL);
2816     g_return_val_if_fail(GST_IS_CAPS(caps), NULL);
2817
2818     decoder = g_object_new(
2819         GST_VAAPI_TYPE_DECODER_H264,
2820         "display",      display,
2821         "caps",         caps,
2822         NULL
2823     );
2824     if (!decoder->priv->is_constructed) {
2825         g_object_unref(decoder);
2826         return NULL;
2827     }
2828     return GST_VAAPI_DECODER_CAST(decoder);
2829 }