gst-libs/gst/vaapi/gstvaapidecoder_h264.c

   1 /*
   2  *  gstvaapidecoder_h264.c - H.264 decoder
   3  *
   4  *  Copyright (C) 2011-2012 Intel Corporation
   5  *
   6  *  This library is free software; you can redistribute it and/or
   7  *  modify it under the terms of the GNU Lesser General Public License
   8  *  as published by the Free Software Foundation; either version 2.1
   9  *  of the License, or (at your option) any later version.
  10  *
  11  *  This library is distributed in the hope that it will be useful,
  12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  *  Lesser General Public License for more details.
  15  *
  16  *  You should have received a copy of the GNU Lesser General Public
  17  *  License along with this library; if not, write to the Free
  18  *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  19  *  Boston, MA 02110-1301 USA
  20  */
  21
  22 /**
  23  * SECTION:gstvaapidecoder_h264
  24  * @short_description: H.264 decoder
  25  */
  26
  27 #include "sysdeps.h"
  28 #include <string.h>
  29 #include <gst/base/gstadapter.h>
  30 #include <gst/codecparsers/gsth264parser.h>
  31 #include "gstvaapidecoder_h264.h"
  32 #include "gstvaapidecoder_objects.h"
  33 #include "gstvaapidecoder_priv.h"
  34 #include "gstvaapidisplay_priv.h"
  35 #include "gstvaapiobject_priv.h"
  36
  37 #define DEBUG 1
  38 #include "gstvaapidebug.h"
  39
  40 /* Defined to 1 if strict ordering of DPB is needed. Only useful for debug */
  41 #define USE_STRICT_DPB_ORDERING 0
  42
  43 typedef struct _GstVaapiPictureH264             GstVaapiPictureH264;
  44 typedef struct _GstVaapiPictureH264Class        GstVaapiPictureH264Class;
  45 typedef struct _GstVaapiSliceH264               GstVaapiSliceH264;
  46 typedef struct _GstVaapiSliceH264Class          GstVaapiSliceH264Class;
  47
  48 /* ------------------------------------------------------------------------- */
  49 /* --- H.264 Pictures                                                    --- */
  50 /* ------------------------------------------------------------------------- */
  51
  52 #define GST_VAAPI_TYPE_PICTURE_H264 \
  53     (gst_vaapi_picture_h264_get_type())
  54
  55 #define GST_VAAPI_PICTURE_H264_CAST(obj) \
  56     ((GstVaapiPictureH264 *)(obj))
  57
  58 #define GST_VAAPI_PICTURE_H264(obj)                             \
  59     (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
  60                                 GST_VAAPI_TYPE_PICTURE_H264,    \
  61                                 GstVaapiPictureH264))
  62
  63 #define GST_VAAPI_PICTURE_H264_CLASS(klass)                     \
  64     (G_TYPE_CHECK_CLASS_CAST((klass),                           \
  65                              GST_VAAPI_TYPE_PICTURE_H264,       \
  66                              GstVaapiPictureH264Class))
  67
  68 #define GST_VAAPI_IS_PICTURE_H264(obj) \
  69     (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_VAAPI_TYPE_PICTURE_H264))
  70
  71 #define GST_VAAPI_IS_PICTURE_H264_CLASS(klass) \
  72     (G_TYPE_CHECK_CLASS_TYPE((klass), GST_VAAPI_TYPE_PICTURE_H264))
  73
  74 #define GST_VAAPI_PICTURE_H264_GET_CLASS(obj)                   \
  75     (G_TYPE_INSTANCE_GET_CLASS((obj),                           \
  76                                GST_VAAPI_TYPE_PICTURE_H264,     \
  77                                GstVaapiPictureH264Class))
  78
  79 /*
  80  * Extended picture flags:
  81  *
  82  * @GST_VAAPI_PICTURE_FLAG_IDR: flag that specifies an IDR picture
  83  * @GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE: flag that specifies
  84  *     "used for short-term reference"
  85  * @GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE: flag that specifies
  86  *     "used for long-term reference"
  87  * @GST_VAAPI_PICTURE_FLAGS_REFERENCE: mask covering any kind of
  88  *     reference picture (short-term reference or long-term reference)
  89  */
  90 enum {
  91     GST_VAAPI_PICTURE_FLAG_IDR = (GST_VAAPI_PICTURE_FLAG_LAST << 0),
  92
  93     GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE = (
  94         GST_VAAPI_PICTURE_FLAG_REFERENCE),
  95     GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE = (
  96         GST_VAAPI_PICTURE_FLAG_REFERENCE | (GST_VAAPI_PICTURE_FLAG_LAST << 1)),
  97     GST_VAAPI_PICTURE_FLAGS_REFERENCE = (
  98         GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE |
  99         GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE),
 100 };
 101
 102 #define GST_VAAPI_PICTURE_IS_IDR(picture) \
 103     (GST_VAAPI_PICTURE_FLAG_IS_SET(picture, GST_VAAPI_PICTURE_FLAG_IDR))
 104
 105 #define GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture)      \
 106     ((GST_VAAPI_PICTURE_FLAGS(picture) &                        \
 107       GST_VAAPI_PICTURE_FLAGS_REFERENCE) ==                     \
 108      GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE)
 109
 110 #define GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture)       \
 111     ((GST_VAAPI_PICTURE_FLAGS(picture) &                        \
 112       GST_VAAPI_PICTURE_FLAGS_REFERENCE) ==                     \
 113      GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE)
 114
 115 struct _GstVaapiPictureH264 {
 116     GstVaapiPicture             base;
 117     VAPictureH264               info;
 118     GstH264PPS                 *pps;
 119     gint32                      poc;
 120     gint32                      frame_num;              // Original frame_num from slice_header()
 121     gint32                      frame_num_wrap;         // Temporary for ref pic marking: FrameNumWrap
 122     gint32                      long_term_frame_idx;    // Temporary for ref pic marking: LongTermFrameIdx
 123     gint32                      pic_num;                // Temporary for ref pic marking: PicNum
 124     gint32                      long_term_pic_num;      // Temporary for ref pic marking: LongTermPicNum
 125     guint                       output_flag             : 1;
 126     guint                       output_needed           : 1;
 127 };
 128
 129 struct _GstVaapiPictureH264Class {
 130     /*< private >*/
 131     GstVaapiPictureClass        parent_class;
 132 };
 133
 134 GST_VAAPI_CODEC_DEFINE_TYPE(GstVaapiPictureH264,
 135                             gst_vaapi_picture_h264,
 136                             GST_VAAPI_TYPE_PICTURE)
 137
 138 static void
 139 gst_vaapi_picture_h264_destroy(GstVaapiPictureH264 *decoder)
 140 {
 141 }
 142
 143 static gboolean
 144 gst_vaapi_picture_h264_create(
 145     GstVaapiPictureH264                      *picture,
 146     const GstVaapiCodecObjectConstructorArgs *args
 147 )
 148 {
 149     return TRUE;
 150 }
 151
 152 static void
 153 gst_vaapi_picture_h264_init(GstVaapiPictureH264 *picture)
 154 {
 155     VAPictureH264 *va_pic;
 156
 157     va_pic                      = &picture->info;
 158     va_pic->flags               = 0;
 159     va_pic->TopFieldOrderCnt    = 0;
 160     va_pic->BottomFieldOrderCnt = 0;
 161
 162     picture->poc                = 0;
 163     picture->output_needed      = FALSE;
 164 }
 165
 166 static inline GstVaapiPictureH264 *
 167 gst_vaapi_picture_h264_new(GstVaapiDecoderH264 *decoder)
 168 {
 169     GstVaapiCodecObject *object;
 170
 171     g_return_val_if_fail(GST_VAAPI_IS_DECODER(decoder), NULL);
 172
 173     object = gst_vaapi_codec_object_new(
 174         GST_VAAPI_TYPE_PICTURE_H264,
 175         GST_VAAPI_CODEC_BASE(decoder),
 176         NULL, sizeof(VAPictureParameterBufferH264),
 177         NULL, 0
 178     );
 179     if (!object)
 180         return NULL;
 181     return GST_VAAPI_PICTURE_H264_CAST(object);
 182 }
 183
 184 static inline GstVaapiSliceH264 *
 185 gst_vaapi_picture_h264_get_last_slice(GstVaapiPictureH264 *picture)
 186 {
 187     g_return_val_if_fail(GST_VAAPI_IS_PICTURE_H264(picture), NULL);
 188
 189     if (G_UNLIKELY(picture->base.slices->len < 1))
 190         return NULL;
 191     return g_ptr_array_index(picture->base.slices,
 192         picture->base.slices->len - 1);
 193 }
 194
 195 /* ------------------------------------------------------------------------- */
 196 /* --- Slices                                                            --- */
 197 /* ------------------------------------------------------------------------- */
 198
 199 #define GST_VAAPI_TYPE_SLICE_H264 \
 200     (gst_vaapi_slice_h264_get_type())
 201
 202 #define GST_VAAPI_SLICE_H264_CAST(obj) \
 203     ((GstVaapiSliceH264 *)(obj))
 204
 205 #define GST_VAAPI_SLICE_H264(obj)                               \
 206     (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
 207                                 GST_VAAPI_TYPE_SLICE_H264,      \
 208                                 GstVaapiSliceH264))
 209
 210 #define GST_VAAPI_SLICE_H264_CLASS(klass)                       \
 211     (G_TYPE_CHECK_CLASS_CAST((klass),                           \
 212                              GST_VAAPI_TYPE_SLICE_H264,         \
 213                              GstVaapiSliceH264Class))
 214
 215 #define GST_VAAPI_IS_SLICE_H264(obj) \
 216     (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_VAAPI_TYPE_SLICE_H264))
 217
 218 #define GST_VAAPI_IS_SLICE_H264_CLASS(klass) \
 219     (G_TYPE_CHECK_CLASS_TYPE((klass), GST_VAAPI_TYPE_SLICE_H264))
 220
 221 #define GST_VAAPI_SLICE_H264_GET_CLASS(obj)                     \
 222     (G_TYPE_INSTANCE_GET_CLASS((obj),                           \
 223                                GST_VAAPI_TYPE_SLICE_H264,       \
 224                                GstVaapiSliceH264Class))
 225
 226 struct _GstVaapiSliceH264 {
 227     GstVaapiSlice               base;
 228     GstH264SliceHdr             slice_hdr;              // parsed slice_header()
 229 };
 230
 231 struct _GstVaapiSliceH264Class {
 232     /*< private >*/
 233     GstVaapiSliceClass          parent_class;
 234 };
 235
 236 GST_VAAPI_CODEC_DEFINE_TYPE(GstVaapiSliceH264,
 237                             gst_vaapi_slice_h264,
 238                             GST_VAAPI_TYPE_SLICE)
 239
 240 static void
 241 gst_vaapi_slice_h264_destroy(GstVaapiSliceH264 *slice)
 242 {
 243 }
 244
 245 static gboolean
 246 gst_vaapi_slice_h264_create(
 247     GstVaapiSliceH264                        *slice,
 248     const GstVaapiCodecObjectConstructorArgs *args
 249 )
 250 {
 251     return TRUE;
 252 }
 253
 254 static void
 255 gst_vaapi_slice_h264_init(GstVaapiSliceH264 *slice)
 256 {
 257 }
 258
 259 static inline GstVaapiSliceH264 *
 260 gst_vaapi_slice_h264_new(
 261     GstVaapiDecoderH264 *decoder,
 262     const guint8        *data,
 263     guint                data_size
 264 )
 265 {
 266     GstVaapiCodecObject *object;
 267
 268     g_return_val_if_fail(GST_VAAPI_IS_DECODER(decoder), NULL);
 269
 270     object = gst_vaapi_codec_object_new(
 271         GST_VAAPI_TYPE_SLICE_H264,
 272         GST_VAAPI_CODEC_BASE(decoder),
 273         NULL, sizeof(VASliceParameterBufferH264),
 274         data, data_size
 275     );
 276     if (!object)
 277         return NULL;
 278     return GST_VAAPI_SLICE_H264_CAST(object);
 279 }
 280
 281 /* ------------------------------------------------------------------------- */
 282 /* --- H.264 Decoder                                                     --- */
 283 /* ------------------------------------------------------------------------- */
 284
 285 G_DEFINE_TYPE(GstVaapiDecoderH264,
 286               gst_vaapi_decoder_h264,
 287               GST_VAAPI_TYPE_DECODER)
 288
 289 #define GST_VAAPI_DECODER_H264_GET_PRIVATE(obj)                 \
 290     (G_TYPE_INSTANCE_GET_PRIVATE((obj),                         \
 291                                  GST_VAAPI_TYPE_DECODER_H264,   \
 292                                  GstVaapiDecoderH264Private))
 293
 294 // Used for field_poc[]
 295 #define TOP_FIELD       0
 296 #define BOTTOM_FIELD    1
 297
 298 struct _GstVaapiDecoderH264Private {
 299     GstAdapter                 *adapter;
 300     GstH264NalParser           *parser;
 301     /* Last decoded SPS. May not be the last activated one. Just here because
 302        it may not fit stack memory allocation in decode_sps() */
 303     GstH264SPS                  last_sps;
 304     /* Last decoded PPS. May not be the last activated one. Just here because
 305        it may not fit stack memory allocation in decode_pps() */
 306     GstH264PPS                  last_pps;
 307     GstVaapiPictureH264        *current_picture;
 308     GstVaapiPictureH264        *dpb[16];
 309     guint                       dpb_count;
 310     guint                       dpb_size;
 311     GstVaapiProfile             profile;
 312     GstVaapiEntrypoint          entrypoint;
 313     GstVaapiChromaType          chroma_type;
 314     GstVaapiPictureH264        *short_ref[32];
 315     guint                       short_ref_count;
 316     GstVaapiPictureH264        *long_ref[32];
 317     guint                       long_ref_count;
 318     GstVaapiPictureH264        *RefPicList0[32];
 319     guint                       RefPicList0_count;
 320     GstVaapiPictureH264        *RefPicList1[32];
 321     guint                       RefPicList1_count;
 322     guint                       nal_length_size;
 323     guint                       width;
 324     guint                       height;
 325     gint32                      field_poc[2];           // 0:TopFieldOrderCnt / 1:BottomFieldOrderCnt
 326     gint32                      poc_msb;                // PicOrderCntMsb
 327     gint32                      poc_lsb;                // pic_order_cnt_lsb (from slice_header())
 328     gint32                      prev_poc_msb;           // prevPicOrderCntMsb
 329     gint32                      prev_poc_lsb;           // prevPicOrderCntLsb
 330     gint32                      frame_num_offset;       // FrameNumOffset
 331     gint32                      frame_num;              // frame_num (from slice_header())
 332     gint32                      prev_frame_num;         // prevFrameNum
 333     gboolean                    prev_pic_has_mmco5;     // prevMmco5Pic
 334     gboolean                    prev_pic_structure;     // previous picture structure
 335     guint                       is_constructed          : 1;
 336     guint                       is_opened               : 1;
 337     guint                       is_avc                  : 1;
 338     guint                       has_context             : 1;
 339 };
 340
 341 static gboolean
 342 decode_picture_end(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture);
 343
 344 static void
 345 clear_references(
 346     GstVaapiDecoderH264  *decoder,
 347     GstVaapiPictureH264 **pictures,
 348     guint                *picture_count
 349 );
 350
 351 /* Get number of reference frames to use */
 352 static guint
 353 get_max_dec_frame_buffering(GstH264SPS *sps)
 354 {
 355     guint max_dec_frame_buffering, MaxDpbMbs, PicSizeMbs;
 356
 357     /* Table A-1 - Level limits */
 358     switch (sps->level_idc) {
 359     case 10: MaxDpbMbs = 396;    break;
 360     case 11: MaxDpbMbs = 900;    break;
 361     case 12: MaxDpbMbs = 2376;   break;
 362     case 13: MaxDpbMbs = 2376;   break;
 363     case 20: MaxDpbMbs = 2376;   break;
 364     case 21: MaxDpbMbs = 4752;   break;
 365     case 22: MaxDpbMbs = 8100;   break;
 366     case 30: MaxDpbMbs = 8100;   break;
 367     case 31: MaxDpbMbs = 18000;  break;
 368     case 32: MaxDpbMbs = 20480;  break;
 369     case 40: MaxDpbMbs = 32768;  break;
 370     case 41: MaxDpbMbs = 32768;  break;
 371     case 42: MaxDpbMbs = 34816;  break;
 372     case 50: MaxDpbMbs = 110400; break;
 373     case 51: MaxDpbMbs = 184320; break;
 374     default:
 375         g_assert(0 && "unhandled level");
 376         break;
 377     }
 378
 379     PicSizeMbs = ((sps->pic_width_in_mbs_minus1 + 1) *
 380                   (sps->pic_height_in_map_units_minus1 + 1) *
 381                   (sps->frame_mbs_only_flag ? 1 : 2));
 382     max_dec_frame_buffering = MaxDpbMbs / PicSizeMbs;
 383
 384     /* VUI parameters */
 385     if (sps->vui_parameters_present_flag) {
 386         GstH264VUIParams * const vui_params = &sps->vui_parameters;
 387         if (vui_params->bitstream_restriction_flag)
 388             max_dec_frame_buffering = vui_params->max_dec_frame_buffering;
 389         else {
 390             switch (sps->profile_idc) {
 391             case 44:  // CAVLC 4:4:4 Intra profile
 392             case 86:  // Scalable High profile
 393             case 100: // High profile
 394             case 110: // High 10 profile
 395             case 122: // High 4:2:2 profile
 396             case 244: // High 4:4:4 Predictive profile
 397                 if (sps->constraint_set3_flag)
 398                     max_dec_frame_buffering = 0;
 399                 break;
 400             }
 401         }
 402     }
 403
 404     if (max_dec_frame_buffering > 16)
 405         max_dec_frame_buffering = 16;
 406     else if (max_dec_frame_buffering < sps->num_ref_frames)
 407         max_dec_frame_buffering = sps->num_ref_frames;
 408     return MAX(1, max_dec_frame_buffering);
 409 }
 410
 411 static void
 412 dpb_remove_index(GstVaapiDecoderH264 *decoder, guint index)
 413 {
 414     GstVaapiDecoderH264Private * const priv = decoder->priv;
 415     guint i, num_pictures = --priv->dpb_count;
 416
 417     if (USE_STRICT_DPB_ORDERING) {
 418         for (i = index; i < num_pictures; i++)
 419             gst_vaapi_picture_replace(&priv->dpb[i], priv->dpb[i + 1]);
 420     }
 421     else if (index != num_pictures)
 422         gst_vaapi_picture_replace(&priv->dpb[index], priv->dpb[num_pictures]);
 423     gst_vaapi_picture_replace(&priv->dpb[num_pictures], NULL);
 424 }
 425
 426 static inline gboolean
 427 dpb_output(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
 428 {
 429     /* XXX: update cropping rectangle */
 430     picture->output_needed = FALSE;
 431     return gst_vaapi_picture_output(GST_VAAPI_PICTURE_CAST(picture));
 432 }
 433
 434 static gboolean
 435 dpb_bump(GstVaapiDecoderH264 *decoder)
 436 {
 437     GstVaapiDecoderH264Private * const priv = decoder->priv;
 438     guint i, lowest_poc_index;
 439     gboolean success;
 440
 441     for (i = 0; i < priv->dpb_count; i++) {
 442         if (priv->dpb[i]->output_needed)
 443             break;
 444     }
 445     if (i == priv->dpb_count)
 446         return FALSE;
 447
 448     lowest_poc_index = i++;
 449     for (; i < priv->dpb_count; i++) {
 450         GstVaapiPictureH264 * const picture = priv->dpb[i];
 451         if (picture->output_needed && picture->poc < priv->dpb[lowest_poc_index]->poc)
 452             lowest_poc_index = i;
 453     }
 454
 455     success = dpb_output(decoder, priv->dpb[lowest_poc_index]);
 456     if (!GST_VAAPI_PICTURE_IS_REFERENCE(priv->dpb[lowest_poc_index]))
 457         dpb_remove_index(decoder, lowest_poc_index);
 458     return success;
 459 }
 460
 461 static void
 462 dpb_flush(GstVaapiDecoderH264 *decoder)
 463 {
 464     GstVaapiDecoderH264Private * const priv = decoder->priv;
 465
 466     while (dpb_bump(decoder))
 467         ;
 468     clear_references(decoder, priv->dpb, &priv->dpb_count);
 469 }
 470
 471 static gboolean
 472 dpb_add(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
 473 {
 474     GstVaapiDecoderH264Private * const priv = decoder->priv;
 475     guint i;
 476
 477     // Remove all unused pictures
 478     if (GST_VAAPI_PICTURE_IS_IDR(picture))
 479         dpb_flush(decoder);
 480     else {
 481         i = 0;
 482         while (i < priv->dpb_count) {
 483             GstVaapiPictureH264 * const picture = priv->dpb[i];
 484             if (!picture->output_needed &&
 485                 !GST_VAAPI_PICTURE_IS_REFERENCE(picture))
 486                 dpb_remove_index(decoder, i);
 487             else
 488                 i++;
 489         }
 490     }
 491
 492     // C.4.5.1 - Storage and marking of a reference decoded picture into the DPB
 493     if (GST_VAAPI_PICTURE_IS_REFERENCE(picture)) {
 494         while (priv->dpb_count == priv->dpb_size) {
 495             if (!dpb_bump(decoder))
 496                 return FALSE;
 497         }
 498         gst_vaapi_picture_replace(&priv->dpb[priv->dpb_count++], picture);
 499         if (picture->output_flag)
 500             picture->output_needed = TRUE;
 501     }
 502
 503     // C.4.5.2 - Storage and marking of a non-reference decoded picture into the DPB
 504     else {
 505         if (!picture->output_flag)
 506             return TRUE;
 507         while (priv->dpb_count == priv->dpb_size) {
 508             for (i = 0; i < priv->dpb_count; i++) {
 509                 if (priv->dpb[i]->output_needed &&
 510                     priv->dpb[i]->poc < picture->poc)
 511                     break;
 512             }
 513             if (i == priv->dpb_count)
 514                 return dpb_output(decoder, picture);
 515             if (!dpb_bump(decoder))
 516                 return FALSE;
 517         }
 518         gst_vaapi_picture_replace(&priv->dpb[priv->dpb_count++], picture);
 519         picture->output_needed = TRUE;
 520     }
 521     return TRUE;
 522 }
 523
 524 static inline void
 525 dpb_reset(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 526 {
 527     GstVaapiDecoderH264Private * const priv = decoder->priv;
 528
 529     priv->dpb_size = get_max_dec_frame_buffering(sps);
 530     GST_DEBUG("DPB size %u", priv->dpb_size);
 531 }
 532
 533 static GstVaapiDecoderStatus
 534 get_status(GstH264ParserResult result)
 535 {
 536     GstVaapiDecoderStatus status;
 537
 538     switch (result) {
 539     case GST_H264_PARSER_OK:
 540         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
 541         break;
 542     case GST_H264_PARSER_NO_NAL_END:
 543         status = GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
 544         break;
 545     case GST_H264_PARSER_ERROR:
 546         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
 547         break;
 548     default:
 549         status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 550         break;
 551     }
 552     return status;
 553 }
 554
 555 static void
 556 gst_vaapi_decoder_h264_close(GstVaapiDecoderH264 *decoder)
 557 {
 558     GstVaapiDecoderH264Private * const priv = decoder->priv;
 559
 560     gst_vaapi_picture_replace(&priv->current_picture, NULL);
 561     clear_references(decoder, priv->short_ref, &priv->short_ref_count);
 562     clear_references(decoder, priv->long_ref,  &priv->long_ref_count );
 563     clear_references(decoder, priv->dpb,       &priv->dpb_count      );
 564
 565     if (priv->parser) {
 566         gst_h264_nal_parser_free(priv->parser);
 567         priv->parser = NULL;
 568     }
 569
 570     if (priv->adapter) {
 571         gst_adapter_clear(priv->adapter);
 572         g_object_unref(priv->adapter);
 573         priv->adapter = NULL;
 574     }
 575 }
 576
 577 static gboolean
 578 gst_vaapi_decoder_h264_open(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
 579 {
 580     GstVaapiDecoderH264Private * const priv = decoder->priv;
 581
 582     gst_vaapi_decoder_h264_close(decoder);
 583
 584     priv->adapter = gst_adapter_new();
 585     if (!priv->adapter)
 586         return FALSE;
 587
 588     priv->parser = gst_h264_nal_parser_new();
 589     if (!priv->parser)
 590         return FALSE;
 591     return TRUE;
 592 }
 593
 594 static void
 595 gst_vaapi_decoder_h264_destroy(GstVaapiDecoderH264 *decoder)
 596 {
 597     gst_vaapi_decoder_h264_close(decoder);
 598 }
 599
 600 static gboolean
 601 gst_vaapi_decoder_h264_create(GstVaapiDecoderH264 *decoder)
 602 {
 603     if (!GST_VAAPI_DECODER_CODEC(decoder))
 604         return FALSE;
 605     return TRUE;
 606 }
 607
 608 static guint
 609 h264_get_profile(GstH264SPS *sps)
 610 {
 611     guint profile = 0;
 612
 613     switch (sps->profile_idc) {
 614     case 66:
 615         profile = GST_VAAPI_PROFILE_H264_BASELINE;
 616         break;
 617     case 77:
 618         profile = GST_VAAPI_PROFILE_H264_MAIN;
 619         break;
 620     case 100:
 621         profile = GST_VAAPI_PROFILE_H264_HIGH;
 622         break;
 623     }
 624     return profile;
 625 }
 626
 627 static guint
 628 h264_get_chroma_type(GstH264SPS *sps)
 629 {
 630     guint chroma_type = 0;
 631
 632     switch (sps->chroma_format_idc) {
 633     case 1:
 634         chroma_type = GST_VAAPI_CHROMA_TYPE_YUV420;
 635         break;
 636     case 2:
 637         chroma_type = GST_VAAPI_CHROMA_TYPE_YUV422;
 638         break;
 639     case 3:
 640         if (!sps->separate_colour_plane_flag)
 641             chroma_type = GST_VAAPI_CHROMA_TYPE_YUV444;
 642         break;
 643     }
 644     return chroma_type;
 645 }
 646
 647 static GstVaapiProfile
 648 get_profile(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 649 {
 650     GstVaapiDecoderH264Private * const priv = decoder->priv;
 651     GstVaapiDisplay * const display = GST_VAAPI_DECODER_DISPLAY(decoder);
 652     GstVaapiProfile profile, profiles[2];
 653     guint i, n_profiles = 0;
 654
 655     profile = h264_get_profile(sps);
 656     if (!profile)
 657         return GST_VAAPI_PROFILE_UNKNOWN;
 658
 659     profiles[n_profiles++] = profile;
 660     switch (profile) {
 661     case GST_VAAPI_PROFILE_H264_MAIN:
 662         profiles[n_profiles++] = GST_VAAPI_PROFILE_H264_HIGH;
 663         break;
 664     default:
 665         break;
 666     }
 667
 668     /* If the preferred profile (profiles[0]) matches one that we already
 669        found, then just return it now instead of searching for it again */
 670     if (profiles[0] == priv->profile)
 671         return priv->profile;
 672
 673     for (i = 0; i < n_profiles; i++) {
 674         if (gst_vaapi_display_has_decoder(display, profiles[i], priv->entrypoint))
 675             return profiles[i];
 676     }
 677     return GST_VAAPI_PROFILE_UNKNOWN;
 678 }
 679
 680 static GstVaapiDecoderStatus
 681 ensure_context(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 682 {
 683     GstVaapiDecoder * const base_decoder = GST_VAAPI_DECODER_CAST(decoder);
 684     GstVaapiDecoderH264Private * const priv = decoder->priv;
 685     GstVaapiContextInfo info;
 686     GstVaapiProfile profile;
 687     GstVaapiChromaType chroma_type;
 688     gboolean reset_context = FALSE;
 689
 690     profile = get_profile(decoder, sps);
 691     if (!profile) {
 692         GST_ERROR("unsupported profile_idc %u", sps->profile_idc);
 693         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_PROFILE;
 694     }
 695
 696     if (priv->profile != profile) {
 697         GST_DEBUG("profile changed");
 698         reset_context = TRUE;
 699         priv->profile = profile;
 700     }
 701
 702     chroma_type = h264_get_chroma_type(sps);
 703     if (!chroma_type || chroma_type != GST_VAAPI_CHROMA_TYPE_YUV420) {
 704         GST_ERROR("unsupported chroma_format_idc %u", sps->chroma_format_idc);
 705         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;
 706     }
 707
 708     if (priv->chroma_type != chroma_type) {
 709         GST_DEBUG("chroma format changed");
 710         reset_context     = TRUE;
 711         priv->chroma_type = chroma_type;
 712     }
 713
 714     if (priv->width != sps->width || priv->height != sps->height) {
 715         GST_DEBUG("size changed");
 716         reset_context = TRUE;
 717         priv->width   = sps->width;
 718         priv->height  = sps->height;
 719     }
 720
 721     gst_vaapi_decoder_set_pixel_aspect_ratio(
 722         base_decoder,
 723         sps->vui_parameters.par_n,
 724         sps->vui_parameters.par_d
 725     );
 726
 727     if (!reset_context && priv->has_context)
 728         return GST_VAAPI_DECODER_STATUS_SUCCESS;
 729
 730     info.profile    = priv->profile;
 731     info.entrypoint = priv->entrypoint;
 732     info.width      = priv->width;
 733     info.height     = priv->height;
 734     info.ref_frames = get_max_dec_frame_buffering(sps);
 735
 736     if (!gst_vaapi_decoder_ensure_context(GST_VAAPI_DECODER(decoder), &info))
 737         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 738     priv->has_context = TRUE;
 739
 740     /* Reset DPB */
 741     dpb_reset(decoder, sps);
 742     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 743 }
 744
 745 static void
 746 fill_iq_matrix_4x4(VAIQMatrixBufferH264 *iq_matrix, const GstH264PPS *pps)
 747 {
 748     const guint8 (* const ScalingList4x4)[6][16] = &pps->scaling_lists_4x4;
 749     guint i, j;
 750
 751     /* There are always 6 4x4 scaling lists */
 752     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList4x4) == 6);
 753     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList4x4[0]) == 16);
 754
 755     if (sizeof(iq_matrix->ScalingList4x4[0][0]) == 1)
 756         memcpy(iq_matrix->ScalingList4x4, *ScalingList4x4,
 757                sizeof(iq_matrix->ScalingList4x4));
 758     else {
 759         for (i = 0; i < G_N_ELEMENTS(iq_matrix->ScalingList4x4); i++) {
 760             for (j = 0; j < G_N_ELEMENTS(iq_matrix->ScalingList4x4[i]); j++)
 761                 iq_matrix->ScalingList4x4[i][j] = (*ScalingList4x4)[i][j];
 762         }
 763     }
 764 }
 765
 766 static void
 767 fill_iq_matrix_8x8(VAIQMatrixBufferH264 *iq_matrix, const GstH264PPS *pps)
 768 {
 769     const guint8 (* const ScalingList8x8)[6][64] = &pps->scaling_lists_8x8;
 770     const GstH264SPS * const sps = pps->sequence;
 771     guint i, j, n;
 772
 773     /* If chroma_format_idc != 3, there are up to 2 8x8 scaling lists */
 774     if (!pps->transform_8x8_mode_flag)
 775         return;
 776
 777     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList8x8) >= 2);
 778     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList8x8[0]) == 64);
 779
 780     if (sizeof(iq_matrix->ScalingList8x8[0][0]) == 1)
 781         memcpy(iq_matrix->ScalingList8x8, *ScalingList8x8,
 782                sizeof(iq_matrix->ScalingList8x8));
 783     else {
 784         n = (sps->chroma_format_idc != 3) ? 2 : 6;
 785         for (i = 0; i < n; i++) {
 786             for (j = 0; j < G_N_ELEMENTS(iq_matrix->ScalingList8x8[i]); j++)
 787                 iq_matrix->ScalingList8x8[i][j] = (*ScalingList8x8)[i][j];
 788         }
 789     }
 790 }
 791
 792 static GstVaapiDecoderStatus
 793 ensure_quant_matrix(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
 794 {
 795     GstVaapiPicture * const base_picture = &picture->base;
 796     GstH264PPS * const pps = picture->pps;
 797     GstH264SPS * const sps = pps->sequence;
 798     VAIQMatrixBufferH264 *iq_matrix;
 799
 800     base_picture->iq_matrix = GST_VAAPI_IQ_MATRIX_NEW(H264, decoder);
 801     if (!base_picture->iq_matrix) {
 802         GST_ERROR("failed to allocate IQ matrix");
 803         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
 804     }
 805     iq_matrix = base_picture->iq_matrix->param;
 806
 807     /* XXX: we can only support 4:2:0 or 4:2:2 since ScalingLists8x8[]
 808        is not large enough to hold lists for 4:4:4 */
 809     if (sps->chroma_format_idc == 3)
 810         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;
 811
 812     fill_iq_matrix_4x4(iq_matrix, pps);
 813     fill_iq_matrix_8x8(iq_matrix, pps);
 814
 815     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 816 }
 817
 818 static GstVaapiDecoderStatus
 819 decode_current_picture(GstVaapiDecoderH264 *decoder)
 820 {
 821     GstVaapiDecoderH264Private * const priv = decoder->priv;
 822     GstVaapiPictureH264 * const picture = priv->current_picture;
 823     GstVaapiDecoderStatus status;
 824
 825     if (!picture)
 826         return GST_VAAPI_DECODER_STATUS_SUCCESS;
 827
 828     status = ensure_context(decoder, picture->pps->sequence);
 829     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
 830         return status;
 831
 832     if (!decode_picture_end(decoder, picture))
 833         goto error;
 834     if (!gst_vaapi_picture_decode(GST_VAAPI_PICTURE_CAST(picture)))
 835         goto error;
 836     gst_vaapi_picture_replace(&priv->current_picture, NULL);
 837     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 838
 839 error:
 840     gst_vaapi_picture_replace(&priv->current_picture, NULL);
 841     return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 842 }
 843
 844 static GstVaapiDecoderStatus
 845 decode_sps(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
 846 {
 847     GstVaapiDecoderH264Private * const priv = decoder->priv;
 848     GstH264SPS * const sps = &priv->last_sps;
 849     GstH264ParserResult result;
 850
 851     GST_DEBUG("decode SPS");
 852
 853     memset(sps, 0, sizeof(*sps));
 854     result = gst_h264_parser_parse_sps(priv->parser, nalu, sps, TRUE);
 855     if (result != GST_H264_PARSER_OK)
 856         return get_status(result);
 857
 858     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 859 }
 860
 861 static GstVaapiDecoderStatus
 862 decode_pps(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
 863 {
 864     GstVaapiDecoderH264Private * const priv = decoder->priv;
 865     GstH264PPS * const pps = &priv->last_pps;
 866     GstH264ParserResult result;
 867
 868     GST_DEBUG("decode PPS");
 869
 870     memset(pps, 0, sizeof(*pps));
 871     result = gst_h264_parser_parse_pps(priv->parser, nalu, pps);
 872     if (result != GST_H264_PARSER_OK)
 873         return get_status(result);
 874
 875     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 876 }
 877
 878 static GstVaapiDecoderStatus
 879 decode_sei(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
 880 {
 881     GstVaapiDecoderH264Private * const priv = decoder->priv;
 882     GstH264SEIMessage sei;
 883     GstH264ParserResult result;
 884
 885     GST_DEBUG("decode SEI");
 886
 887     memset(&sei, 0, sizeof(sei));
 888     result = gst_h264_parser_parse_sei(priv->parser, nalu, &sei);
 889     if (result != GST_H264_PARSER_OK) {
 890         GST_WARNING("failed to decode SEI, payload type:%d", sei.payloadType);
 891         return get_status(result);
 892     }
 893
 894     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 895 }
 896
 897 static GstVaapiDecoderStatus
 898 decode_sequence_end(GstVaapiDecoderH264 *decoder)
 899 {
 900     GstVaapiDecoderStatus status;
 901
 902     GST_DEBUG("decode sequence-end");
 903
 904     status = decode_current_picture(decoder);
 905     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
 906         return status;
 907
 908     dpb_flush(decoder);
 909     return GST_VAAPI_DECODER_STATUS_END_OF_STREAM;
 910 }
 911
 912 /* 8.2.1.1 - Decoding process for picture order count type 0 */
 913 static void
 914 init_picture_poc_0(
 915     GstVaapiDecoderH264 *decoder,
 916     GstVaapiPictureH264 *picture,
 917     GstH264SliceHdr     *slice_hdr
 918 )
 919 {
 920     GstVaapiDecoderH264Private * const priv = decoder->priv;
 921     GstH264PPS * const pps = slice_hdr->pps;
 922     GstH264SPS * const sps = pps->sequence;
 923     const gint32 MaxPicOrderCntLsb = 1 << (sps->log2_max_pic_order_cnt_lsb_minus4 + 4);
 924     gint32 temp_poc;
 925
 926     GST_DEBUG("decode picture order count type 0");
 927
 928     if (GST_VAAPI_PICTURE_IS_IDR(picture)) {
 929         priv->prev_poc_msb = 0;
 930         priv->prev_poc_lsb = 0;
 931     }
 932     else if (priv->prev_pic_has_mmco5) {
 933         priv->prev_poc_msb = 0;
 934         priv->prev_poc_lsb =
 935             (priv->prev_pic_structure == GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD ?
 936              0 : priv->field_poc[TOP_FIELD]);
 937     }
 938     else {
 939         priv->prev_poc_msb = priv->poc_msb;
 940         priv->prev_poc_lsb = priv->poc_lsb;
 941     }
 942
 943     // (8-3)
 944     priv->poc_lsb = slice_hdr->pic_order_cnt_lsb;
 945     if (priv->poc_lsb < priv->prev_poc_lsb &&
 946         (priv->prev_poc_lsb - priv->poc_lsb) >= (MaxPicOrderCntLsb / 2))
 947         priv->poc_msb = priv->prev_poc_msb + MaxPicOrderCntLsb;
 948     else if (priv->poc_lsb > priv->prev_poc_lsb &&
 949              (priv->poc_lsb - priv->prev_poc_lsb) > (MaxPicOrderCntLsb / 2))
 950         priv->poc_msb = priv->prev_poc_msb - MaxPicOrderCntLsb;
 951     else
 952         priv->poc_msb = priv->prev_poc_msb;
 953
 954     temp_poc = priv->poc_msb + priv->poc_lsb;
 955     switch (picture->base.structure) {
 956     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
 957         // (8-4, 8-5)
 958         priv->field_poc[TOP_FIELD] = temp_poc;
 959         priv->field_poc[BOTTOM_FIELD] = temp_poc +
 960             slice_hdr->delta_pic_order_cnt_bottom;
 961         break;
 962     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
 963         // (8-4)
 964         priv->field_poc[TOP_FIELD] = temp_poc;
 965         break;
 966     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
 967         // (8-5)
 968         priv->field_poc[BOTTOM_FIELD] = temp_poc;
 969         break;
 970     }
 971 }
 972
 973 /* 8.2.1.2 - Decoding process for picture order count type 1 */
 974 static void
 975 init_picture_poc_1(
 976     GstVaapiDecoderH264 *decoder,
 977     GstVaapiPictureH264 *picture,
 978     GstH264SliceHdr     *slice_hdr
 979 )
 980 {
 981     GstVaapiDecoderH264Private * const priv = decoder->priv;
 982     GstH264PPS * const pps = slice_hdr->pps;
 983     GstH264SPS * const sps = pps->sequence;
 984     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
 985     gint32 prev_frame_num_offset, abs_frame_num, expected_poc;
 986     guint i;
 987
 988     GST_DEBUG("decode picture order count type 1");
 989
 990     if (priv->prev_pic_has_mmco5)
 991         prev_frame_num_offset = 0;
 992     else
 993         prev_frame_num_offset = priv->frame_num_offset;
 994
 995     // (8-6)
 996     if (GST_VAAPI_PICTURE_IS_IDR(picture))
 997         priv->frame_num_offset = 0;
 998     else if (priv->prev_frame_num > priv->frame_num)
 999         priv->frame_num_offset = prev_frame_num_offset + MaxFrameNum;
1000     else
1001         priv->frame_num_offset = prev_frame_num_offset;
1002
1003     // (8-7)
1004     if (sps->num_ref_frames_in_pic_order_cnt_cycle != 0)
1005         abs_frame_num = priv->frame_num_offset + priv->frame_num;
1006     else
1007         abs_frame_num = 0;
1008     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture) && abs_frame_num > 0)
1009         abs_frame_num = abs_frame_num - 1;
1010
1011     if (abs_frame_num > 0) {
1012         gint32 expected_delta_per_poc_cycle;
1013         gint32 poc_cycle_cnt, frame_num_in_poc_cycle;
1014
1015         expected_delta_per_poc_cycle = 0;
1016         for (i = 0; i < sps->num_ref_frames_in_pic_order_cnt_cycle; i++)
1017             expected_delta_per_poc_cycle += sps->offset_for_ref_frame[i];
1018
1019         // (8-8)
1020         poc_cycle_cnt = (abs_frame_num - 1) /
1021             sps->num_ref_frames_in_pic_order_cnt_cycle;
1022         frame_num_in_poc_cycle = (abs_frame_num - 1) %
1023             sps->num_ref_frames_in_pic_order_cnt_cycle;
1024
1025         // (8-9)
1026         expected_poc = poc_cycle_cnt * expected_delta_per_poc_cycle;
1027         for (i = 0; i <= frame_num_in_poc_cycle; i++)
1028             expected_poc += sps->offset_for_ref_frame[i];
1029     }
1030     else
1031         expected_poc = 0;
1032     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
1033         expected_poc += sps->offset_for_non_ref_pic;
1034
1035     // (8-10)
1036     switch (picture->base.structure) {
1037     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
1038         priv->field_poc[TOP_FIELD] = expected_poc +
1039             slice_hdr->delta_pic_order_cnt[0];
1040         priv->field_poc[BOTTOM_FIELD] = priv->field_poc[TOP_FIELD] +
1041             sps->offset_for_top_to_bottom_field +
1042             slice_hdr->delta_pic_order_cnt[1];
1043         break;
1044     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
1045         priv->field_poc[TOP_FIELD] = expected_poc +
1046             slice_hdr->delta_pic_order_cnt[0];
1047         break;
1048     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
1049         priv->field_poc[BOTTOM_FIELD] = expected_poc +
1050             sps->offset_for_top_to_bottom_field +
1051             slice_hdr->delta_pic_order_cnt[0];
1052         break;
1053     }
1054 }
1055
1056 /* 8.2.1.3 - Decoding process for picture order count type 2 */
1057 static void
1058 init_picture_poc_2(
1059     GstVaapiDecoderH264 *decoder,
1060     GstVaapiPictureH264 *picture,
1061     GstH264SliceHdr     *slice_hdr
1062 )
1063 {
1064     GstVaapiDecoderH264Private * const priv = decoder->priv;
1065     GstH264PPS * const pps = slice_hdr->pps;
1066     GstH264SPS * const sps = pps->sequence;
1067     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1068     gint32 prev_frame_num_offset, temp_poc;
1069
1070     GST_DEBUG("decode picture order count type 2");
1071
1072     if (priv->prev_pic_has_mmco5)
1073         prev_frame_num_offset = 0;
1074     else
1075         prev_frame_num_offset = priv->frame_num_offset;
1076
1077     // (8-11)
1078     if (GST_VAAPI_PICTURE_IS_IDR(picture))
1079         priv->frame_num_offset = 0;
1080     else if (priv->prev_frame_num > priv->frame_num)
1081         priv->frame_num_offset = prev_frame_num_offset + MaxFrameNum;
1082     else
1083         priv->frame_num_offset = prev_frame_num_offset;
1084
1085     // (8-12)
1086     if (GST_VAAPI_PICTURE_IS_IDR(picture))
1087         temp_poc = 0;
1088     else if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
1089         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num) - 1;
1090     else
1091         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num);
1092
1093     // (8-13)
1094     if (picture->base.structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
1095         priv->field_poc[TOP_FIELD] = temp_poc;
1096     if (picture->base.structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
1097         priv->field_poc[BOTTOM_FIELD] = temp_poc;
1098 }
1099
1100 /* 8.2.1 - Decoding process for picture order count */
1101 static void
1102 init_picture_poc(
1103     GstVaapiDecoderH264 *decoder,
1104     GstVaapiPictureH264 *picture,
1105     GstH264SliceHdr     *slice_hdr
1106 )
1107 {
1108     GstVaapiDecoderH264Private * const priv = decoder->priv;
1109     VAPictureH264 * const pic = &picture->info;
1110     GstH264PPS * const pps = slice_hdr->pps;
1111     GstH264SPS * const sps = pps->sequence;
1112
1113     switch (sps->pic_order_cnt_type) {
1114     case 0:
1115         init_picture_poc_0(decoder, picture, slice_hdr);
1116         break;
1117     case 1:
1118         init_picture_poc_1(decoder, picture, slice_hdr);
1119         break;
1120     case 2:
1121         init_picture_poc_2(decoder, picture, slice_hdr);
1122         break;
1123     }
1124
1125     if (picture->base.structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
1126         pic->TopFieldOrderCnt = priv->field_poc[TOP_FIELD];
1127     if (picture->base.structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
1128         pic->BottomFieldOrderCnt = priv->field_poc[BOTTOM_FIELD];
1129     picture->poc = MIN(pic->TopFieldOrderCnt, pic->BottomFieldOrderCnt);
1130 }
1131
1132 static int
1133 compare_picture_pic_num_dec(const void *a, const void *b)
1134 {
1135     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1136     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1137
1138     return picB->pic_num - picA->pic_num;
1139 }
1140
1141 static int
1142 compare_picture_long_term_pic_num_inc(const void *a, const void *b)
1143 {
1144     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1145     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1146
1147     return picA->long_term_pic_num - picB->long_term_pic_num;
1148 }
1149
1150 static int
1151 compare_picture_poc_dec(const void *a, const void *b)
1152 {
1153     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1154     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1155
1156     return picB->poc - picA->poc;
1157 }
1158
1159 static int
1160 compare_picture_poc_inc(const void *a, const void *b)
1161 {
1162     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1163     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1164
1165     return picA->poc - picB->poc;
1166 }
1167
1168 static int
1169 compare_picture_frame_num_wrap_dec(const void *a, const void *b)
1170 {
1171     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1172     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1173
1174     return picB->frame_num_wrap - picA->frame_num_wrap;
1175 }
1176
1177 static int
1178 compare_picture_long_term_frame_idx_inc(const void *a, const void *b)
1179 {
1180     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1181     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1182
1183     return picA->long_term_frame_idx - picB->long_term_frame_idx;
1184 }
1185
1186 /* 8.2.4.1 - Decoding process for picture numbers */
1187 static void
1188 init_picture_refs_pic_num(
1189     GstVaapiDecoderH264 *decoder,
1190     GstVaapiPictureH264 *picture,
1191     GstH264SliceHdr     *slice_hdr
1192 )
1193 {
1194     GstVaapiDecoderH264Private * const priv = decoder->priv;
1195     GstH264PPS * const pps = slice_hdr->pps;
1196     GstH264SPS * const sps = pps->sequence;
1197     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1198     guint i;
1199
1200     GST_DEBUG("decode picture numbers");
1201
1202     for (i = 0; i < priv->short_ref_count; i++) {
1203         GstVaapiPictureH264 * const pic = priv->short_ref[i];
1204
1205         // (8-27)
1206         if (pic->frame_num > priv->frame_num)
1207             pic->frame_num_wrap = pic->frame_num - MaxFrameNum;
1208         else
1209             pic->frame_num_wrap = pic->frame_num;
1210
1211         // (8-28, 8-30, 8-31)
1212         if (GST_VAAPI_PICTURE_IS_FRAME(picture))
1213             pic->pic_num = pic->frame_num_wrap;
1214         else {
1215             if (pic->base.structure == picture->base.structure)
1216                 pic->pic_num = 2 * pic->frame_num_wrap + 1;
1217             else
1218                 pic->pic_num = 2 * pic->frame_num_wrap;
1219         }
1220     }
1221
1222     for (i = 0; i < priv->long_ref_count; i++) {
1223         GstVaapiPictureH264 * const pic = priv->long_ref[i];
1224
1225         // (8-29, 8-32, 8-33)
1226         if (GST_VAAPI_PICTURE_IS_FRAME(picture))
1227             pic->long_term_pic_num = pic->long_term_frame_idx;
1228         else {
1229             if (pic->base.structure == picture->base.structure)
1230                 pic->long_term_pic_num = 2 * pic->long_term_frame_idx + 1;
1231             else
1232                 pic->long_term_pic_num = 2 * pic->long_term_frame_idx;
1233         }
1234     }
1235 }
1236
1237 #define SORT_REF_LIST(list, n, compare_func) \
1238     qsort(list, n, sizeof(*(list)), compare_picture_##compare_func)
1239
1240 static void
1241 init_picture_refs_p_slice(
1242     GstVaapiDecoderH264 *decoder,
1243     GstVaapiPictureH264 *picture,
1244     GstH264SliceHdr     *slice_hdr
1245 )
1246 {
1247     GstVaapiDecoderH264Private * const priv = decoder->priv;
1248     GstVaapiPictureH264 **ref_list;
1249     guint i;
1250
1251     GST_DEBUG("decode reference picture list for P and SP slices");
1252
1253     if (GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1254         /* 8.2.4.2.1 - P and SP slices in frames */
1255         if (priv->short_ref_count > 0) {
1256             ref_list = priv->RefPicList0;
1257             for (i = 0; i < priv->short_ref_count; i++)
1258                 ref_list[i] = priv->short_ref[i];
1259             SORT_REF_LIST(ref_list, i, pic_num_dec);
1260             priv->RefPicList0_count += i;
1261         }
1262
1263         if (priv->long_ref_count > 0) {
1264             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1265             for (i = 0; i < priv->long_ref_count; i++)
1266                 ref_list[i] = priv->long_ref[i];
1267             SORT_REF_LIST(ref_list, i, long_term_pic_num_inc);
1268             priv->RefPicList0_count += i;
1269         }
1270     }
1271     else {
1272         /* 8.2.4.2.2 - P and SP slices in fields */
1273         GstVaapiPictureH264 *short_ref[32];
1274         guint short_ref_count = 0;
1275         GstVaapiPictureH264 *long_ref[32];
1276         guint long_ref_count = 0;
1277
1278         // XXX: handle second field if current field is marked as
1279         // "used for short-term reference"
1280         if (priv->short_ref_count > 0) {
1281             for (i = 0; i < priv->short_ref_count; i++)
1282                 short_ref[i] = priv->short_ref[i];
1283             SORT_REF_LIST(short_ref, i, frame_num_wrap_dec);
1284             short_ref_count = i;
1285         }
1286
1287         // XXX: handle second field if current field is marked as
1288         // "used for long-term reference"
1289         if (priv->long_ref_count > 0) {
1290             for (i = 0; i < priv->long_ref_count; i++)
1291                 long_ref[i] = priv->long_ref[i];
1292             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1293             long_ref_count = i;
1294         }
1295
1296         // XXX: handle 8.2.4.2.5
1297     }
1298 }
1299
1300 static void
1301 init_picture_refs_b_slice(
1302     GstVaapiDecoderH264 *decoder,
1303     GstVaapiPictureH264 *picture,
1304     GstH264SliceHdr     *slice_hdr
1305 )
1306 {
1307     GstVaapiDecoderH264Private * const priv = decoder->priv;
1308     GstVaapiPictureH264 **ref_list;
1309     guint i, n;
1310
1311     GST_DEBUG("decode reference picture list for B slices");
1312
1313     if (GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1314         /* 8.2.4.2.3 - B slices in frames */
1315
1316         /* RefPicList0 */
1317         if (priv->short_ref_count > 0) {
1318             // 1. Short-term references
1319             ref_list = priv->RefPicList0;
1320             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1321                 if (priv->short_ref[i]->poc < picture->poc)
1322                     ref_list[n++] = priv->short_ref[i];
1323             }
1324             SORT_REF_LIST(ref_list, n, poc_dec);
1325             priv->RefPicList0_count += n;
1326
1327             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1328             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1329                 if (priv->short_ref[i]->poc >= picture->poc)
1330                     ref_list[n++] = priv->short_ref[i];
1331             }
1332             SORT_REF_LIST(ref_list, n, poc_inc);
1333             priv->RefPicList0_count += n;
1334         }
1335
1336         if (priv->long_ref_count > 0) {
1337             // 2. Long-term references
1338             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1339             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1340                 ref_list[n++] = priv->long_ref[i];
1341             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1342             priv->RefPicList0_count += n;
1343         }
1344
1345         /* RefPicList1 */
1346         if (priv->short_ref_count > 0) {
1347             // 1. Short-term references
1348             ref_list = priv->RefPicList1;
1349             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1350                 if (priv->short_ref[i]->poc > picture->poc)
1351                     ref_list[n++] = priv->short_ref[i];
1352             }
1353             SORT_REF_LIST(ref_list, n, poc_inc);
1354             priv->RefPicList1_count += n;
1355
1356             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1357             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1358                 if (priv->short_ref[i]->poc <= picture->poc)
1359                     ref_list[n++] = priv->short_ref[i];
1360             }
1361             SORT_REF_LIST(ref_list, n, poc_dec);
1362             priv->RefPicList1_count += n;
1363         }
1364
1365         if (priv->long_ref_count > 0) {
1366             // 2. Long-term references
1367             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1368             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1369                 ref_list[n++] = priv->long_ref[i];
1370             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1371             priv->RefPicList1_count += n;
1372         }
1373     }
1374     else {
1375         /* 8.2.4.2.4 - B slices in fields */
1376         GstVaapiPictureH264 *short_ref0[32];
1377         guint short_ref0_count = 0;
1378         GstVaapiPictureH264 *short_ref1[32];
1379         guint short_ref1_count = 0;
1380         GstVaapiPictureH264 *long_ref[32];
1381         guint long_ref_count = 0;
1382
1383         /* refFrameList0ShortTerm */
1384         if (priv->short_ref_count > 0) {
1385             ref_list = short_ref0;
1386             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1387                 if (priv->short_ref[i]->poc <= picture->poc)
1388                     ref_list[n++] = priv->short_ref[i];
1389             }
1390             SORT_REF_LIST(ref_list, n, poc_dec);
1391             short_ref0_count += n;
1392
1393             ref_list = &short_ref0[short_ref0_count];
1394             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1395                 if (priv->short_ref[i]->poc > picture->poc)
1396                     ref_list[n++] = priv->short_ref[i];
1397             }
1398             SORT_REF_LIST(ref_list, n, poc_inc);
1399             short_ref0_count += n;
1400         }
1401
1402         /* refFrameList1ShortTerm */
1403         if (priv->short_ref_count > 0) {
1404             ref_list = short_ref1;
1405             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1406                 if (priv->short_ref[i]->poc > picture->poc)
1407                     ref_list[n++] = priv->short_ref[i];
1408             }
1409             SORT_REF_LIST(ref_list, n, poc_inc);
1410             short_ref1_count += n;
1411
1412             ref_list = &short_ref1[short_ref1_count];
1413             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1414                 if (priv->short_ref[i]->poc <= picture->poc)
1415                     ref_list[n++] = priv->short_ref[i];
1416             }
1417             SORT_REF_LIST(ref_list, n, poc_dec);
1418             short_ref1_count += n;
1419         }
1420
1421         /* refFrameListLongTerm */
1422         if (priv->long_ref_count > 0) {
1423             for (i = 0; i < priv->long_ref_count; i++)
1424                 long_ref[i] = priv->long_ref[i];
1425             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1426             long_ref_count = i;
1427         }
1428
1429         // XXX: handle 8.2.4.2.5
1430     }
1431
1432     /* Check whether RefPicList1 is identical to RefPicList0, then
1433        swap if necessary */
1434     if (priv->RefPicList1_count > 1 &&
1435         priv->RefPicList1_count == priv->RefPicList0_count &&
1436         memcmp(priv->RefPicList0, priv->RefPicList1,
1437                priv->RefPicList0_count * sizeof(priv->RefPicList0[0])) == 0) {
1438         GstVaapiPictureH264 * const tmp = priv->RefPicList1[0];
1439         priv->RefPicList1[0] = priv->RefPicList1[1];
1440         priv->RefPicList1[1] = tmp;
1441     }
1442 }
1443
1444 #undef SORT_REF_LIST
1445
1446 static void
1447 clear_references(
1448     GstVaapiDecoderH264  *decoder,
1449     GstVaapiPictureH264 **pictures,
1450     guint                *picture_count
1451 )
1452 {
1453     const guint num_pictures = *picture_count;
1454     guint i;
1455
1456     for (i = 0; i < num_pictures; i++)
1457         gst_vaapi_picture_replace(&pictures[i], NULL);
1458     *picture_count = 0;
1459 }
1460
1461 static gboolean
1462 remove_reference_at(
1463     GstVaapiDecoderH264  *decoder,
1464     GstVaapiPictureH264 **pictures,
1465     guint                *picture_count,
1466     guint                 index
1467 )
1468 {
1469     guint num_pictures = *picture_count;
1470     GstVaapiPictureH264 *picture;
1471
1472     g_return_val_if_fail(index < num_pictures, FALSE);
1473
1474     picture = pictures[index];
1475     GST_VAAPI_PICTURE_FLAG_UNSET(picture, GST_VAAPI_PICTURE_FLAGS_REFERENCE);
1476
1477     if (index != --num_pictures)
1478         gst_vaapi_picture_replace(&pictures[index], pictures[num_pictures]);
1479     gst_vaapi_picture_replace(&pictures[num_pictures], NULL);
1480     *picture_count = num_pictures;
1481     return TRUE;
1482 }
1483
1484 static gint
1485 find_short_term_reference(GstVaapiDecoderH264 *decoder, gint32 pic_num)
1486 {
1487     GstVaapiDecoderH264Private * const priv = decoder->priv;
1488     guint i;
1489
1490     for (i = 0; i < priv->short_ref_count; i++) {
1491         if (priv->short_ref[i]->pic_num == pic_num)
1492             return i;
1493     }
1494     GST_ERROR("found no short-term reference picture with PicNum = %d",
1495               pic_num);
1496     return -1;
1497 }
1498
1499 static gint
1500 find_long_term_reference(GstVaapiDecoderH264 *decoder, gint32 long_term_pic_num)
1501 {
1502     GstVaapiDecoderH264Private * const priv = decoder->priv;
1503     guint i;
1504
1505     for (i = 0; i < priv->long_ref_count; i++) {
1506         if (priv->long_ref[i]->long_term_pic_num == long_term_pic_num)
1507             return i;
1508     }
1509     GST_ERROR("found no long-term reference picture with LongTermPicNum = %d",
1510               long_term_pic_num);
1511     return -1;
1512 }
1513
1514 static void
1515 exec_picture_refs_modification_1(
1516     GstVaapiDecoderH264           *decoder,
1517     GstVaapiPictureH264           *picture,
1518     GstH264SliceHdr               *slice_hdr,
1519     guint                          list
1520 )
1521 {
1522     GstVaapiDecoderH264Private * const priv = decoder->priv;
1523     GstH264PPS * const pps = slice_hdr->pps;
1524     GstH264SPS * const sps = pps->sequence;
1525     GstH264RefPicListModification *ref_pic_list_modification;
1526     guint num_ref_pic_list_modifications;
1527     GstVaapiPictureH264 **ref_list;
1528     guint *ref_list_count_ptr, ref_list_count, ref_list_idx = 0;
1529     guint i, j, n, num_refs;
1530     gint found_ref_idx;
1531     gint32 MaxPicNum, CurrPicNum, picNumPred;
1532
1533     GST_DEBUG("modification process of reference picture list %u", list);
1534
1535     if (list == 0) {
1536         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l0;
1537         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l0;
1538         ref_list                       = priv->RefPicList0;
1539         ref_list_count_ptr             = &priv->RefPicList0_count;
1540         num_refs                       = slice_hdr->num_ref_idx_l0_active_minus1 + 1;
1541     }
1542     else {
1543         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l1;
1544         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l1;
1545         ref_list                       = priv->RefPicList1;
1546         ref_list_count_ptr             = &priv->RefPicList1_count;
1547         num_refs                       = slice_hdr->num_ref_idx_l1_active_minus1 + 1;
1548     }
1549     ref_list_count = *ref_list_count_ptr;
1550
1551     if (!GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1552         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 5); // 2 * MaxFrameNum
1553         CurrPicNum = 2 * slice_hdr->frame_num + 1;              // 2 * frame_num + 1
1554     }
1555     else {
1556         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 4); // MaxFrameNum
1557         CurrPicNum = slice_hdr->frame_num;                      // frame_num
1558     }
1559
1560     picNumPred = CurrPicNum;
1561
1562     for (i = 0; i < num_ref_pic_list_modifications; i++) {
1563         GstH264RefPicListModification * const l = &ref_pic_list_modification[i];
1564         if (l->modification_of_pic_nums_idc == 3)
1565             break;
1566
1567         /* 8.2.4.3.1 - Short-term reference pictures */
1568         if (l->modification_of_pic_nums_idc == 0 || l->modification_of_pic_nums_idc == 1) {
1569             gint32 abs_diff_pic_num = l->value.abs_diff_pic_num_minus1 + 1;
1570             gint32 picNum, picNumNoWrap;
1571
1572             // (8-34)
1573             if (l->modification_of_pic_nums_idc == 0) {
1574                 picNumNoWrap = picNumPred - abs_diff_pic_num;
1575                 if (picNumNoWrap < 0)
1576                     picNumNoWrap += MaxPicNum;
1577             }
1578
1579             // (8-35)
1580             else {
1581                 picNumNoWrap = picNumPred + abs_diff_pic_num;
1582                 if (picNumNoWrap >= MaxPicNum)
1583                     picNumNoWrap -= MaxPicNum;
1584             }
1585             picNumPred = picNumNoWrap;
1586
1587             // (8-36)
1588             picNum = picNumNoWrap;
1589             if (picNum > CurrPicNum)
1590                 picNum -= MaxPicNum;
1591
1592             // (8-37)
1593             for (j = num_refs; j > ref_list_idx; j--)
1594                 ref_list[j] = ref_list[j - 1];
1595             found_ref_idx = find_short_term_reference(decoder, picNum);
1596             ref_list[ref_list_idx++] =
1597                 found_ref_idx >= 0 ? priv->short_ref[found_ref_idx] : NULL;
1598             n = ref_list_idx;
1599             for (j = ref_list_idx; j <= num_refs; j++) {
1600                 gint32 PicNumF;
1601                 if (!ref_list[j])
1602                     continue;
1603                 PicNumF =
1604                     GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(ref_list[j]) ?
1605                     ref_list[j]->pic_num : MaxPicNum;
1606                 if (PicNumF != picNum)
1607                     ref_list[n++] = ref_list[j];
1608             }
1609         }
1610
1611         /* 8.2.4.3.2 - Long-term reference pictures */
1612         else {
1613
1614             for (j = num_refs; j > ref_list_idx; j--)
1615                 ref_list[j] = ref_list[j - 1];
1616             found_ref_idx =
1617                 find_long_term_reference(decoder, l->value.long_term_pic_num);
1618             ref_list[ref_list_idx++] =
1619                 found_ref_idx >= 0 ? priv->long_ref[found_ref_idx] : NULL;
1620             n = ref_list_idx;
1621             for (j = ref_list_idx; j <= num_refs; j++) {
1622                 gint32 LongTermPicNumF;
1623                 if (!ref_list[j])
1624                     continue;
1625                 LongTermPicNumF =
1626                     GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(ref_list[j]) ?
1627                     ref_list[j]->long_term_pic_num : INT_MAX;
1628                 if (LongTermPicNumF != l->value.long_term_pic_num)
1629                     ref_list[n++] = ref_list[j];
1630             }
1631         }
1632     }
1633
1634 #if DEBUG
1635     for (i = 0; i < num_refs; i++)
1636         if (!ref_list[i])
1637             GST_ERROR("list %u entry %u is empty", list, i);
1638 #endif
1639     *ref_list_count_ptr = num_refs;
1640 }
1641
1642 /* 8.2.4.3 - Modification process for reference picture lists */
1643 static void
1644 exec_picture_refs_modification(
1645     GstVaapiDecoderH264 *decoder,
1646     GstVaapiPictureH264 *picture,
1647     GstH264SliceHdr     *slice_hdr
1648 )
1649 {
1650     GST_DEBUG("execute ref_pic_list_modification()");
1651
1652     /* RefPicList0 */
1653     if (!GST_H264_IS_I_SLICE(slice_hdr) && !GST_H264_IS_SI_SLICE(slice_hdr) &&
1654         slice_hdr->ref_pic_list_modification_flag_l0)
1655         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 0);
1656
1657     /* RefPicList1 */
1658     if (GST_H264_IS_B_SLICE(slice_hdr) &&
1659         slice_hdr->ref_pic_list_modification_flag_l1)
1660         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 1);
1661 }
1662
1663 static gboolean
1664 init_picture_refs(
1665     GstVaapiDecoderH264 *decoder,
1666     GstVaapiPictureH264 *picture,
1667     GstH264SliceHdr     *slice_hdr
1668 )
1669 {
1670     GstVaapiDecoderH264Private * const priv = decoder->priv;
1671     GstVaapiPicture * const base_picture = &picture->base;
1672     guint i, num_refs;
1673
1674     init_picture_refs_pic_num(decoder, picture, slice_hdr);
1675
1676     priv->RefPicList0_count = 0;
1677     priv->RefPicList1_count = 0;
1678
1679     switch (base_picture->type) {
1680     case GST_VAAPI_PICTURE_TYPE_P:
1681     case GST_VAAPI_PICTURE_TYPE_SP:
1682         init_picture_refs_p_slice(decoder, picture, slice_hdr);
1683         break;
1684     case GST_VAAPI_PICTURE_TYPE_B:
1685         init_picture_refs_b_slice(decoder, picture, slice_hdr);
1686         break;
1687     default:
1688         break;
1689     }
1690
1691     exec_picture_refs_modification(decoder, picture, slice_hdr);
1692
1693     switch (base_picture->type) {
1694     case GST_VAAPI_PICTURE_TYPE_B:
1695         num_refs = 1 + slice_hdr->num_ref_idx_l1_active_minus1;
1696         for (i = priv->RefPicList1_count; i < num_refs; i++)
1697             priv->RefPicList1[i] = NULL;
1698         priv->RefPicList1_count = num_refs;
1699
1700         // fall-through
1701     case GST_VAAPI_PICTURE_TYPE_P:
1702     case GST_VAAPI_PICTURE_TYPE_SP:
1703         num_refs = 1 + slice_hdr->num_ref_idx_l0_active_minus1;
1704         for (i = priv->RefPicList0_count; i < num_refs; i++)
1705             priv->RefPicList0[i] = NULL;
1706         priv->RefPicList0_count = num_refs;
1707         break;
1708     default:
1709         break;
1710     }
1711     return TRUE;
1712 }
1713
1714 static gboolean
1715 init_picture(
1716     GstVaapiDecoderH264 *decoder,
1717     GstVaapiPictureH264 *picture,
1718     GstH264SliceHdr     *slice_hdr,
1719     GstH264NalUnit      *nalu
1720 )
1721 {
1722     GstVaapiDecoderH264Private * const priv = decoder->priv;
1723     GstVaapiPicture * const base_picture = &picture->base;
1724
1725     priv->prev_frame_num        = priv->frame_num;
1726     priv->frame_num             = slice_hdr->frame_num;
1727     picture->frame_num          = priv->frame_num;
1728     picture->frame_num_wrap     = priv->frame_num;
1729     picture->output_flag        = TRUE; /* XXX: conformant to Annex A only */
1730     base_picture->pts           = gst_adapter_prev_timestamp(priv->adapter, NULL);
1731
1732     /* Reset decoder state for IDR pictures */
1733     if (nalu->type == GST_H264_NAL_SLICE_IDR) {
1734         GST_DEBUG("<IDR>");
1735         GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_IDR);
1736         clear_references(decoder, priv->short_ref, &priv->short_ref_count);
1737         clear_references(decoder, priv->long_ref,  &priv->long_ref_count );
1738     }
1739
1740     /* Initialize slice type */
1741     switch (slice_hdr->type % 5) {
1742     case GST_H264_P_SLICE:
1743         base_picture->type = GST_VAAPI_PICTURE_TYPE_P;
1744         break;
1745     case GST_H264_B_SLICE:
1746         base_picture->type = GST_VAAPI_PICTURE_TYPE_B;
1747         break;
1748     case GST_H264_I_SLICE:
1749         base_picture->type = GST_VAAPI_PICTURE_TYPE_I;
1750         break;
1751     case GST_H264_SP_SLICE:
1752         base_picture->type = GST_VAAPI_PICTURE_TYPE_SP;
1753         break;
1754     case GST_H264_SI_SLICE:
1755         base_picture->type = GST_VAAPI_PICTURE_TYPE_SI;
1756         break;
1757     }
1758
1759     /* Initialize picture structure */
1760     if (!slice_hdr->field_pic_flag)
1761         base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
1762     else if (!slice_hdr->bottom_field_flag)
1763         base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD;
1764     else
1765         base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD;
1766
1767     if (nalu->ref_idc) {
1768         GstH264DecRefPicMarking * const dec_ref_pic_marking =
1769             &slice_hdr->dec_ref_pic_marking;
1770
1771         if (GST_VAAPI_PICTURE_IS_IDR(picture) &&
1772             dec_ref_pic_marking->long_term_reference_flag)
1773             GST_VAAPI_PICTURE_FLAG_SET(picture,
1774                 GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE);
1775         else
1776             GST_VAAPI_PICTURE_FLAG_SET(picture,
1777                 GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE);
1778     }
1779
1780     init_picture_poc(decoder, picture, slice_hdr);
1781     if (!init_picture_refs(decoder, picture, slice_hdr)) {
1782         GST_ERROR("failed to initialize references");
1783         return FALSE;
1784     }
1785     return TRUE;
1786 }
1787
1788 /* 8.2.5.3 - Sliding window decoded reference picture marking process */
1789 static gboolean
1790 exec_ref_pic_marking_sliding_window(GstVaapiDecoderH264 *decoder)
1791 {
1792     GstVaapiDecoderH264Private * const priv = decoder->priv;
1793     GstH264PPS * const pps = priv->current_picture->pps;
1794     GstH264SPS * const sps = pps->sequence;
1795     guint i, max_num_ref_frames, lowest_frame_num_index;
1796     gint32 lowest_frame_num;
1797
1798     GST_DEBUG("reference picture marking process (sliding window)");
1799
1800     max_num_ref_frames = sps->num_ref_frames;
1801     if (max_num_ref_frames == 0)
1802         max_num_ref_frames = 1;
1803
1804     if (priv->short_ref_count + priv->long_ref_count < max_num_ref_frames)
1805         return TRUE;
1806     if (priv->short_ref_count < 1)
1807         return FALSE;
1808
1809     lowest_frame_num = priv->short_ref[0]->frame_num_wrap;
1810     lowest_frame_num_index = 0;
1811     for (i = 1; i < priv->short_ref_count; i++) {
1812         if (priv->short_ref[i]->frame_num_wrap < lowest_frame_num) {
1813             lowest_frame_num = priv->short_ref[i]->frame_num_wrap;
1814             lowest_frame_num_index = i;
1815         }
1816     }
1817
1818     remove_reference_at(
1819         decoder,
1820         priv->short_ref, &priv->short_ref_count,
1821         lowest_frame_num_index
1822     );
1823     return TRUE;
1824 }
1825
1826 static inline gint32
1827 get_picNumX(GstVaapiPictureH264 *picture, GstH264RefPicMarking *ref_pic_marking)
1828 {
1829     gint32 pic_num;
1830
1831     if (GST_VAAPI_PICTURE_IS_FRAME(picture))
1832         pic_num = picture->frame_num_wrap;
1833     else
1834         pic_num = 2 * picture->frame_num_wrap + 1;
1835     pic_num -= ref_pic_marking->difference_of_pic_nums_minus1 + 1;
1836     return pic_num;
1837 }
1838
1839 /* 8.2.5.4.1. Mark-term reference picture as "unused for reference" */
1840 static void
1841 exec_ref_pic_marking_adaptive_mmco_1(
1842     GstVaapiDecoderH264  *decoder,
1843     GstVaapiPictureH264  *picture,
1844     GstH264RefPicMarking *ref_pic_marking
1845 )
1846 {
1847     GstVaapiDecoderH264Private * const priv = decoder->priv;
1848     gint32 i, picNumX;
1849
1850     picNumX = get_picNumX(picture, ref_pic_marking);
1851     i = find_short_term_reference(decoder, picNumX);
1852     if (i < 0)
1853         return;
1854     remove_reference_at(decoder, priv->short_ref, &priv->short_ref_count, i);
1855 }
1856
1857 /* 8.2.5.4.2. Mark long-term reference picture as "unused for reference" */
1858 static void
1859 exec_ref_pic_marking_adaptive_mmco_2(
1860     GstVaapiDecoderH264  *decoder,
1861     GstVaapiPictureH264  *picture,
1862     GstH264RefPicMarking *ref_pic_marking
1863 )
1864 {
1865     GstVaapiDecoderH264Private * const priv = decoder->priv;
1866     gint32 i;
1867
1868     i = find_long_term_reference(decoder, ref_pic_marking->long_term_pic_num);
1869     if (i < 0)
1870         return;
1871     remove_reference_at(decoder, priv->long_ref, &priv->long_ref_count, i);
1872 }
1873
1874 /* 8.2.5.4.3. Assign LongTermFrameIdx to a short-term reference picture */
1875 static void
1876 exec_ref_pic_marking_adaptive_mmco_3(
1877     GstVaapiDecoderH264  *decoder,
1878     GstVaapiPictureH264  *picture,
1879     GstH264RefPicMarking *ref_pic_marking
1880 )
1881 {
1882     GstVaapiDecoderH264Private * const priv = decoder->priv;
1883     gint32 i, picNumX;
1884
1885     for (i = 0; i < priv->long_ref_count; i++) {
1886         if (priv->long_ref[i]->long_term_frame_idx == ref_pic_marking->long_term_frame_idx)
1887             break;
1888     }
1889     if (i != priv->long_ref_count)
1890         remove_reference_at(decoder, priv->long_ref, &priv->long_ref_count, i);
1891
1892     picNumX = get_picNumX(picture, ref_pic_marking);
1893     i = find_short_term_reference(decoder, picNumX);
1894     if (i < 0)
1895         return;
1896
1897     picture = gst_vaapi_picture_ref(priv->short_ref[i]);
1898     remove_reference_at(decoder, priv->short_ref, &priv->short_ref_count, i);
1899     gst_vaapi_picture_replace(&priv->long_ref[priv->long_ref_count++], picture);
1900     gst_vaapi_picture_unref(picture);
1901
1902     picture->long_term_frame_idx = ref_pic_marking->long_term_frame_idx;
1903     GST_VAAPI_PICTURE_FLAG_SET(picture,
1904         GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE);
1905 }
1906
1907 /* 8.2.5.4.4. Mark pictures with LongTermFramIdx > max_long_term_frame_idx
1908  * as "unused for reference" */
1909 static void
1910 exec_ref_pic_marking_adaptive_mmco_4(
1911     GstVaapiDecoderH264  *decoder,
1912     GstVaapiPictureH264  *picture,
1913     GstH264RefPicMarking *ref_pic_marking
1914 )
1915 {
1916     GstVaapiDecoderH264Private * const priv = decoder->priv;
1917     gint32 i, long_term_frame_idx;
1918
1919     long_term_frame_idx = ref_pic_marking->max_long_term_frame_idx_plus1 - 1;
1920
1921     for (i = 0; i < priv->long_ref_count; i++) {
1922         if (priv->long_ref[i]->long_term_frame_idx <= long_term_frame_idx)
1923             continue;
1924         remove_reference_at(decoder, priv->long_ref, &priv->long_ref_count, i);
1925         i--;
1926     }
1927 }
1928
1929 /* 8.2.5.4.5. Mark all reference pictures as "unused for reference" */
1930 static void
1931 exec_ref_pic_marking_adaptive_mmco_5(
1932     GstVaapiDecoderH264  *decoder,
1933     GstVaapiPictureH264  *picture,
1934     GstH264RefPicMarking *ref_pic_marking
1935 )
1936 {
1937     GstVaapiDecoderH264Private * const priv = decoder->priv;
1938     VAPictureH264 * const pic = &picture->info;
1939
1940     clear_references(decoder, priv->short_ref, &priv->short_ref_count);
1941     clear_references(decoder, priv->long_ref,  &priv->long_ref_count );
1942     dpb_flush(decoder);
1943
1944     priv->prev_pic_has_mmco5 = TRUE;
1945
1946     /* The picture shall be inferred to have had frame_num equal to 0 (7.4.3) */
1947     priv->frame_num = 0;
1948     priv->frame_num_offset = 0;
1949     picture->frame_num = 0;
1950
1951     /* Update TopFieldOrderCnt and BottomFieldOrderCnt (8.2.1) */
1952     if (picture->base.structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
1953         pic->TopFieldOrderCnt -= picture->poc;
1954     if (picture->base.structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
1955         pic->BottomFieldOrderCnt -= picture->poc;
1956     picture->poc = 0;
1957 }
1958
1959 /* 8.2.5.4.6. Assign a long-term frame index to the current picture */
1960 static void
1961 exec_ref_pic_marking_adaptive_mmco_6(
1962     GstVaapiDecoderH264  *decoder,
1963     GstVaapiPictureH264  *picture,
1964     GstH264RefPicMarking *ref_pic_marking
1965 )
1966 {
1967     picture->long_term_frame_idx = ref_pic_marking->long_term_frame_idx;
1968     GST_VAAPI_PICTURE_FLAG_SET(picture,
1969         GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE);
1970 }
1971
1972 /* 8.2.5.4. Adaptive memory control decoded reference picture marking process */
1973 static gboolean
1974 exec_ref_pic_marking_adaptive(
1975     GstVaapiDecoderH264     *decoder,
1976     GstVaapiPictureH264     *picture,
1977     GstH264DecRefPicMarking *dec_ref_pic_marking
1978 )
1979 {
1980     guint i;
1981
1982     GST_DEBUG("reference picture marking process (adaptive memory control)");
1983
1984     typedef void (*exec_ref_pic_marking_adaptive_mmco_func)(
1985         GstVaapiDecoderH264  *decoder,
1986         GstVaapiPictureH264  *picture,
1987         GstH264RefPicMarking *ref_pic_marking
1988     );
1989
1990     static const exec_ref_pic_marking_adaptive_mmco_func mmco_funcs[] = {
1991         NULL,
1992         exec_ref_pic_marking_adaptive_mmco_1,
1993         exec_ref_pic_marking_adaptive_mmco_2,
1994         exec_ref_pic_marking_adaptive_mmco_3,
1995         exec_ref_pic_marking_adaptive_mmco_4,
1996         exec_ref_pic_marking_adaptive_mmco_5,
1997         exec_ref_pic_marking_adaptive_mmco_6,
1998     };
1999
2000     for (i = 0; i < dec_ref_pic_marking->n_ref_pic_marking; i++) {
2001         GstH264RefPicMarking * const ref_pic_marking =
2002             &dec_ref_pic_marking->ref_pic_marking[i];
2003
2004         const guint mmco = ref_pic_marking->memory_management_control_operation;
2005         if (mmco < G_N_ELEMENTS(mmco_funcs) && mmco_funcs[mmco])
2006             mmco_funcs[mmco](decoder, picture, ref_pic_marking);
2007         else {
2008             GST_ERROR("unhandled MMCO %u", mmco);
2009             return FALSE;
2010         }
2011     }
2012     return TRUE;
2013 }
2014
2015 /* 8.2.5 - Execute reference picture marking process */
2016 static gboolean
2017 exec_ref_pic_marking(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
2018 {
2019     GstVaapiDecoderH264Private * const priv = decoder->priv;
2020     GstVaapiPictureH264 **picture_ptr;
2021
2022     priv->prev_pic_has_mmco5 = FALSE;
2023     priv->prev_pic_structure = picture->base.structure;
2024
2025     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
2026         return TRUE;
2027
2028     if (!GST_VAAPI_PICTURE_IS_IDR(picture)) {
2029         GstVaapiSliceH264 * const slice =
2030             gst_vaapi_picture_h264_get_last_slice(picture);
2031         GstH264DecRefPicMarking * const dec_ref_pic_marking =
2032             &slice->slice_hdr.dec_ref_pic_marking;
2033         if (dec_ref_pic_marking->adaptive_ref_pic_marking_mode_flag) {
2034             if (!exec_ref_pic_marking_adaptive(decoder, picture, dec_ref_pic_marking))
2035                 return FALSE;
2036         }
2037         else {
2038             if (!exec_ref_pic_marking_sliding_window(decoder))
2039                 return FALSE;
2040         }
2041     }
2042
2043     if (GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture))
2044         picture_ptr = &priv->long_ref[priv->long_ref_count++];
2045     else
2046         picture_ptr = &priv->short_ref[priv->short_ref_count++];
2047     gst_vaapi_picture_replace(picture_ptr, picture);
2048     return TRUE;
2049 }
2050
2051 static void
2052 vaapi_init_picture(VAPictureH264 *pic)
2053 {
2054     pic->picture_id           = VA_INVALID_ID;
2055     pic->frame_idx            = 0;
2056     pic->flags                = VA_PICTURE_H264_INVALID;
2057     pic->TopFieldOrderCnt     = 0;
2058     pic->BottomFieldOrderCnt  = 0;
2059 }
2060
2061 static void
2062 vaapi_fill_picture(VAPictureH264 *pic, GstVaapiPictureH264 *picture)
2063 {
2064     pic->picture_id = picture->base.surface_id;
2065     pic->flags = 0;
2066
2067     if (GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture)) {
2068         pic->flags |= VA_PICTURE_H264_LONG_TERM_REFERENCE;
2069         pic->frame_idx = picture->long_term_frame_idx;
2070     }
2071     else {
2072         if (GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture))
2073             pic->flags |= VA_PICTURE_H264_SHORT_TERM_REFERENCE;
2074         pic->frame_idx = picture->frame_num;
2075     }
2076
2077     switch (picture->base.structure) {
2078     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
2079         pic->TopFieldOrderCnt = picture->info.TopFieldOrderCnt;
2080         pic->BottomFieldOrderCnt = picture->info.BottomFieldOrderCnt;
2081         break;
2082     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
2083         pic->flags |= VA_PICTURE_H264_TOP_FIELD;
2084         pic->TopFieldOrderCnt = picture->info.BottomFieldOrderCnt;
2085         pic->BottomFieldOrderCnt = 0;
2086         break;
2087     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
2088         pic->flags |= VA_PICTURE_H264_BOTTOM_FIELD;
2089         pic->BottomFieldOrderCnt = picture->info.BottomFieldOrderCnt;
2090         pic->TopFieldOrderCnt = 0;
2091         break;
2092     }
2093 }
2094
2095 static gboolean
2096 fill_picture(
2097     GstVaapiDecoderH264 *decoder,
2098     GstVaapiPictureH264 *picture,
2099     GstH264SliceHdr     *slice_hdr,
2100     GstH264NalUnit      *nalu
2101 )
2102 {
2103     GstVaapiDecoderH264Private * const priv = decoder->priv;
2104     GstVaapiPicture * const base_picture = &picture->base;
2105     GstH264PPS * const pps = picture->pps;
2106     GstH264SPS * const sps = pps->sequence;
2107     VAPictureParameterBufferH264 * const pic_param = base_picture->param;
2108     guint i, n;
2109
2110     /* Fill in VAPictureParameterBufferH264 */
2111     vaapi_fill_picture(&pic_param->CurrPic, picture);
2112     for (i = 0, n = 0; i < priv->short_ref_count; i++, n++)
2113         vaapi_fill_picture(&pic_param->ReferenceFrames[n], priv->short_ref[i]);
2114     for (i = 0; i < priv->long_ref_count; i++, n++)
2115         vaapi_fill_picture(&pic_param->ReferenceFrames[n], priv->long_ref[i]);
2116     for (; n < G_N_ELEMENTS(pic_param->ReferenceFrames); n++)
2117         vaapi_init_picture(&pic_param->ReferenceFrames[n]);
2118
2119 #define COPY_FIELD(s, f) \
2120     pic_param->f = (s)->f
2121
2122 #define COPY_BFM(a, s, f) \
2123     pic_param->a.bits.f = (s)->f
2124
2125     pic_param->picture_width_in_mbs_minus1  = ((priv->width + 15) >> 4)  - 1;
2126     pic_param->picture_height_in_mbs_minus1 = ((priv->height + 15) >> 4) - 1;
2127     pic_param->frame_num                    = priv->frame_num;
2128
2129     COPY_FIELD(sps, bit_depth_luma_minus8);
2130     COPY_FIELD(sps, bit_depth_chroma_minus8);
2131     COPY_FIELD(sps, num_ref_frames);
2132     COPY_FIELD(pps, num_slice_groups_minus1);
2133     COPY_FIELD(pps, slice_group_map_type);
2134     COPY_FIELD(pps, slice_group_change_rate_minus1);
2135     COPY_FIELD(pps, pic_init_qp_minus26);
2136     COPY_FIELD(pps, pic_init_qs_minus26);
2137     COPY_FIELD(pps, chroma_qp_index_offset);
2138     COPY_FIELD(pps, second_chroma_qp_index_offset);
2139
2140     pic_param->seq_fields.value                                         = 0; /* reset all bits */
2141     pic_param->seq_fields.bits.residual_colour_transform_flag           = sps->separate_colour_plane_flag;
2142     pic_param->seq_fields.bits.MinLumaBiPredSize8x8                     = sps->level_idc >= 31; /* A.3.3.2 */
2143
2144     COPY_BFM(seq_fields, sps, chroma_format_idc);
2145     COPY_BFM(seq_fields, sps, gaps_in_frame_num_value_allowed_flag);
2146     COPY_BFM(seq_fields, sps, frame_mbs_only_flag);
2147     COPY_BFM(seq_fields, sps, mb_adaptive_frame_field_flag);
2148     COPY_BFM(seq_fields, sps, direct_8x8_inference_flag);
2149     COPY_BFM(seq_fields, sps, log2_max_frame_num_minus4);
2150     COPY_BFM(seq_fields, sps, pic_order_cnt_type);
2151     COPY_BFM(seq_fields, sps, log2_max_pic_order_cnt_lsb_minus4);
2152     COPY_BFM(seq_fields, sps, delta_pic_order_always_zero_flag);
2153
2154     pic_param->pic_fields.value                                         = 0; /* reset all bits */
2155     pic_param->pic_fields.bits.field_pic_flag                           = slice_hdr->field_pic_flag;
2156     pic_param->pic_fields.bits.reference_pic_flag                       = GST_VAAPI_PICTURE_IS_REFERENCE(picture);
2157
2158     COPY_BFM(pic_fields, pps, entropy_coding_mode_flag);
2159     COPY_BFM(pic_fields, pps, weighted_pred_flag);
2160     COPY_BFM(pic_fields, pps, weighted_bipred_idc);
2161     COPY_BFM(pic_fields, pps, transform_8x8_mode_flag);
2162     COPY_BFM(pic_fields, pps, constrained_intra_pred_flag);
2163     COPY_BFM(pic_fields, pps, pic_order_present_flag);
2164     COPY_BFM(pic_fields, pps, deblocking_filter_control_present_flag);
2165     COPY_BFM(pic_fields, pps, redundant_pic_cnt_present_flag);
2166     return TRUE;
2167 }
2168
2169 /* Detection of the first VCL NAL unit of a primary coded picture (7.4.1.2.4) */
2170 static gboolean
2171 is_new_picture(
2172     GstVaapiDecoderH264 *decoder,
2173     GstH264NalUnit      *nalu,
2174     GstH264SliceHdr     *slice_hdr
2175 )
2176 {
2177     GstVaapiDecoderH264Private * const priv = decoder->priv;
2178     GstH264PPS * const pps = slice_hdr->pps;
2179     GstH264SPS * const sps = pps->sequence;
2180     GstVaapiSliceH264 *slice;
2181     GstH264SliceHdr *prev_slice_hdr;
2182
2183     if (!priv->current_picture)
2184         return TRUE;
2185
2186     slice = gst_vaapi_picture_h264_get_last_slice(priv->current_picture);
2187     if (!slice)
2188         return FALSE;
2189     prev_slice_hdr = &slice->slice_hdr;
2190
2191 #define CHECK_EXPR(expr, field_name) do {              \
2192         if (!(expr)) {                                 \
2193             GST_DEBUG(field_name " differs in value"); \
2194             return TRUE;                               \
2195         }                                              \
2196     } while (0)
2197
2198 #define CHECK_VALUE(new_slice_hdr, old_slice_hdr, field) \
2199     CHECK_EXPR(((new_slice_hdr)->field == (old_slice_hdr)->field), #field)
2200
2201     /* frame_num differs in value, regardless of inferred values to 0 */
2202     CHECK_VALUE(slice_hdr, prev_slice_hdr, frame_num);
2203
2204     /* pic_parameter_set_id differs in value */
2205     CHECK_VALUE(slice_hdr, prev_slice_hdr, pps);
2206
2207     /* field_pic_flag differs in value */
2208     CHECK_VALUE(slice_hdr, prev_slice_hdr, field_pic_flag);
2209
2210     /* bottom_field_flag is present in both and differs in value */
2211     if (slice_hdr->field_pic_flag && prev_slice_hdr->field_pic_flag)
2212         CHECK_VALUE(slice_hdr, prev_slice_hdr, bottom_field_flag);
2213
2214     /* nal_ref_idc differs in value with one of the nal_ref_idc values is 0 */
2215     CHECK_EXPR(((GST_VAAPI_PICTURE_IS_REFERENCE(priv->current_picture) ^
2216                  (nalu->ref_idc != 0)) == 0), "nal_ref_idc");
2217
2218     /* POC type is 0 for both and either pic_order_cnt_lsb differs in
2219        value or delta_pic_order_cnt_bottom differs in value */
2220     if (sps->pic_order_cnt_type == 0) {
2221         CHECK_VALUE(slice_hdr, prev_slice_hdr, pic_order_cnt_lsb);
2222         if (pps->pic_order_present_flag && !slice_hdr->field_pic_flag)
2223             CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt_bottom);
2224     }
2225
2226     /* POC type is 1 for both and either delta_pic_order_cnt[0]
2227        differs in value or delta_pic_order_cnt[1] differs in value */
2228     else if (sps->pic_order_cnt_type == 1) {
2229         CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt[0]);
2230         CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt[1]);
2231     }
2232
2233     /* IdrPicFlag differs in value */
2234     CHECK_EXPR(((GST_VAAPI_PICTURE_IS_IDR(priv->current_picture) ^
2235                  (nalu->type == GST_H264_NAL_SLICE_IDR)) == 0), "IdrPicFlag");
2236
2237     /* IdrPicFlag is equal to 1 for both and idr_pic_id differs in value */
2238     if (GST_VAAPI_PICTURE_IS_IDR(priv->current_picture))
2239         CHECK_VALUE(slice_hdr, prev_slice_hdr, idr_pic_id);
2240
2241 #undef CHECK_EXPR
2242 #undef CHECK_VALUE
2243     return FALSE;
2244 }
2245
2246 static GstVaapiDecoderStatus
2247 decode_picture(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu, GstH264SliceHdr *slice_hdr)
2248 {
2249     GstVaapiDecoderH264Private * const priv = decoder->priv;
2250     GstVaapiPictureH264 *picture;
2251     GstVaapiDecoderStatus status;
2252     GstH264PPS * const pps = slice_hdr->pps;
2253     GstH264SPS * const sps = pps->sequence;
2254
2255     status = decode_current_picture(decoder);
2256     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2257         return status;
2258
2259     status = ensure_context(decoder, sps);
2260     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2261         return status;
2262
2263     picture = gst_vaapi_picture_h264_new(decoder);
2264     if (!picture) {
2265         GST_ERROR("failed to allocate picture");
2266         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2267     }
2268     priv->current_picture = picture;
2269
2270     picture->pps = pps;
2271
2272     status = ensure_quant_matrix(decoder, picture);
2273     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS) {
2274         GST_ERROR("failed to reset quantizer matrix");
2275         return status;
2276     }
2277
2278     if (!init_picture(decoder, picture, slice_hdr, nalu))
2279         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2280     if (!fill_picture(decoder, picture, slice_hdr, nalu))
2281         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2282     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2283 }
2284
2285 static gboolean
2286 decode_picture_end(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
2287 {
2288     if (!exec_ref_pic_marking(decoder, picture))
2289         return FALSE;
2290     if (!dpb_add(decoder, picture))
2291         return FALSE;
2292     return TRUE;
2293 }
2294
2295 static inline guint
2296 get_slice_data_bit_offset(GstH264SliceHdr *slice_hdr, GstH264NalUnit *nalu)
2297 {
2298     guint epb_count;
2299
2300     epb_count = slice_hdr->n_emulation_prevention_bytes;
2301     return 8 /* nal_unit_type */ + slice_hdr->header_size - epb_count * 8;
2302 }
2303
2304 static gboolean
2305 fill_pred_weight_table(GstVaapiDecoderH264 *decoder, GstVaapiSliceH264 *slice)
2306 {
2307     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2308     GstH264PPS * const pps = slice_hdr->pps;
2309     GstH264SPS * const sps = pps->sequence;
2310     GstH264PredWeightTable * const w = &slice_hdr->pred_weight_table;
2311     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2312     guint num_weight_tables = 0;
2313     gint i, j;
2314
2315     if (pps->weighted_pred_flag &&
2316         (GST_H264_IS_P_SLICE(slice_hdr) || GST_H264_IS_SP_SLICE(slice_hdr)))
2317         num_weight_tables = 1;
2318     else if (pps->weighted_bipred_idc == 1 && GST_H264_IS_B_SLICE(slice_hdr))
2319         num_weight_tables = 2;
2320     else
2321         num_weight_tables = 0;
2322
2323     slice_param->luma_log2_weight_denom   = w->luma_log2_weight_denom;
2324     slice_param->chroma_log2_weight_denom = w->chroma_log2_weight_denom;
2325     slice_param->luma_weight_l0_flag      = 0;
2326     slice_param->chroma_weight_l0_flag    = 0;
2327     slice_param->luma_weight_l1_flag      = 0;
2328     slice_param->chroma_weight_l1_flag    = 0;
2329
2330     if (num_weight_tables < 1)
2331         return TRUE;
2332
2333     slice_param->luma_weight_l0_flag = 1;
2334     for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2335         slice_param->luma_weight_l0[i] = w->luma_weight_l0[i];
2336         slice_param->luma_offset_l0[i] = w->luma_offset_l0[i];
2337     }
2338
2339     slice_param->chroma_weight_l0_flag = sps->chroma_array_type != 0;
2340     if (slice_param->chroma_weight_l0_flag) {
2341         for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2342             for (j = 0; j < 2; j++) {
2343                 slice_param->chroma_weight_l0[i][j] = w->chroma_weight_l0[i][j];
2344                 slice_param->chroma_offset_l0[i][j] = w->chroma_offset_l0[i][j];
2345             }
2346         }
2347     }
2348
2349     if (num_weight_tables < 2)
2350         return TRUE;
2351
2352     slice_param->luma_weight_l1_flag = 1;
2353     for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2354         slice_param->luma_weight_l1[i] = w->luma_weight_l1[i];
2355         slice_param->luma_offset_l1[i] = w->luma_offset_l1[i];
2356     }
2357
2358     slice_param->chroma_weight_l1_flag = sps->chroma_array_type != 0;
2359     if (slice_param->chroma_weight_l1_flag) {
2360         for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2361             for (j = 0; j < 2; j++) {
2362                 slice_param->chroma_weight_l1[i][j] = w->chroma_weight_l1[i][j];
2363                 slice_param->chroma_offset_l1[i][j] = w->chroma_offset_l1[i][j];
2364             }
2365         }
2366     }
2367     return TRUE;
2368 }
2369
2370 static gboolean
2371 fill_RefPicList(GstVaapiDecoderH264 *decoder, GstVaapiSliceH264 *slice)
2372 {
2373     GstVaapiDecoderH264Private * const priv = decoder->priv;
2374     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2375     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2376     guint i, num_ref_lists = 0;
2377
2378     slice_param->num_ref_idx_l0_active_minus1 = 0;
2379     slice_param->num_ref_idx_l1_active_minus1 = 0;
2380
2381     if (GST_H264_IS_B_SLICE(slice_hdr))
2382         num_ref_lists = 2;
2383     else if (GST_H264_IS_I_SLICE(slice_hdr))
2384         num_ref_lists = 0;
2385     else
2386         num_ref_lists = 1;
2387
2388     if (num_ref_lists < 1)
2389         return TRUE;
2390
2391     slice_param->num_ref_idx_l0_active_minus1 =
2392         slice_hdr->num_ref_idx_l0_active_minus1;
2393
2394     for (i = 0; i < priv->RefPicList0_count && priv->RefPicList0[i]; i++)
2395         vaapi_fill_picture(&slice_param->RefPicList0[i], priv->RefPicList0[i]);
2396     for (; i <= slice_param->num_ref_idx_l0_active_minus1; i++)
2397         vaapi_init_picture(&slice_param->RefPicList0[i]);
2398
2399     if (num_ref_lists < 2)
2400         return TRUE;
2401
2402     slice_param->num_ref_idx_l1_active_minus1 =
2403         slice_hdr->num_ref_idx_l1_active_minus1;
2404
2405     for (i = 0; i < priv->RefPicList1_count && priv->RefPicList1[i]; i++)
2406         vaapi_fill_picture(&slice_param->RefPicList1[i], priv->RefPicList1[i]);
2407     for (; i <= slice_param->num_ref_idx_l1_active_minus1; i++)
2408         vaapi_init_picture(&slice_param->RefPicList1[i]);
2409     return TRUE;
2410 }
2411
2412 static gboolean
2413 fill_slice(
2414     GstVaapiDecoderH264 *decoder,
2415     GstVaapiSliceH264   *slice,
2416     GstH264NalUnit      *nalu
2417 )
2418 {
2419     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2420     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2421
2422     /* Fill in VASliceParameterBufferH264 */
2423     slice_param->slice_data_bit_offset          = get_slice_data_bit_offset(slice_hdr, nalu);
2424     slice_param->first_mb_in_slice              = slice_hdr->first_mb_in_slice;
2425     slice_param->slice_type                     = slice_hdr->type % 5;
2426     slice_param->direct_spatial_mv_pred_flag    = slice_hdr->direct_spatial_mv_pred_flag;
2427     slice_param->cabac_init_idc                 = slice_hdr->cabac_init_idc;
2428     slice_param->slice_qp_delta                 = slice_hdr->slice_qp_delta;
2429     slice_param->disable_deblocking_filter_idc  = slice_hdr->disable_deblocking_filter_idc;
2430     slice_param->slice_alpha_c0_offset_div2     = slice_hdr->slice_alpha_c0_offset_div2;
2431     slice_param->slice_beta_offset_div2         = slice_hdr->slice_beta_offset_div2;
2432
2433     if (!fill_RefPicList(decoder, slice))
2434         return FALSE;
2435     if (!fill_pred_weight_table(decoder, slice))
2436         return FALSE;
2437     return TRUE;
2438 }
2439
2440 static GstVaapiDecoderStatus
2441 decode_slice(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
2442 {
2443     GstVaapiDecoderH264Private * const priv = decoder->priv;
2444     GstVaapiDecoderStatus status;
2445     GstVaapiPictureH264 *picture;
2446     GstVaapiSliceH264 *slice = NULL;
2447     GstH264SliceHdr *slice_hdr;
2448     GstH264ParserResult result;
2449
2450     GST_DEBUG("slice (%u bytes)", nalu->size);
2451
2452     slice = gst_vaapi_slice_h264_new(
2453         decoder,
2454         nalu->data + nalu->offset,
2455         nalu->size
2456     );
2457     if (!slice) {
2458         GST_ERROR("failed to allocate slice");
2459         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2460     }
2461
2462     slice_hdr = &slice->slice_hdr;
2463     memset(slice_hdr, 0, sizeof(*slice_hdr));
2464     result = gst_h264_parser_parse_slice_hdr(priv->parser, nalu, slice_hdr, TRUE, TRUE);
2465     if (result != GST_H264_PARSER_OK) {
2466         status = get_status(result);
2467         goto error;
2468     }
2469
2470     if (is_new_picture(decoder, nalu, slice_hdr)) {
2471         status = decode_picture(decoder, nalu, slice_hdr);
2472         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2473             goto error;
2474     }
2475     picture = priv->current_picture;
2476
2477     if (!fill_slice(decoder, slice, nalu)) {
2478         status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2479         goto error;
2480     }
2481     gst_vaapi_picture_add_slice(
2482         GST_VAAPI_PICTURE_CAST(picture),
2483         GST_VAAPI_SLICE_CAST(slice)
2484     );
2485     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2486
2487 error:
2488     if (slice)
2489         gst_mini_object_unref(GST_MINI_OBJECT(slice));
2490     return status;
2491 }
2492
2493 static inline gint
2494 scan_for_start_code(GstAdapter *adapter, guint ofs, guint size, guint32 *scp)
2495 {
2496     return (gint)gst_adapter_masked_scan_uint32_peek(adapter,
2497                                                      0xffffff00, 0x00000100,
2498                                                      ofs, size,
2499                                                      scp);
2500 }
2501
2502 static GstVaapiDecoderStatus
2503 decode_nalu(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
2504 {
2505     GstVaapiDecoderStatus status;
2506
2507     switch (nalu->type) {
2508     case GST_H264_NAL_SLICE_IDR:
2509         /* fall-through. IDR specifics are handled in init_picture() */
2510     case GST_H264_NAL_SLICE:
2511         status = decode_slice(decoder, nalu);
2512         break;
2513     case GST_H264_NAL_SPS:
2514         status = decode_sps(decoder, nalu);
2515         break;
2516     case GST_H264_NAL_PPS:
2517         status = decode_pps(decoder, nalu);
2518         break;
2519     case GST_H264_NAL_SEI:
2520         status = decode_sei(decoder, nalu);
2521         break;
2522     case GST_H264_NAL_SEQ_END:
2523         status = decode_sequence_end(decoder);
2524         break;
2525     case GST_H264_NAL_AU_DELIMITER:
2526         /* skip all Access Unit NALs */
2527         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2528         break;
2529     case GST_H264_NAL_FILLER_DATA:
2530         /* skip all Filler Data NALs */
2531         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2532         break;
2533     default:
2534         GST_WARNING("unsupported NAL unit type %d", nalu->type);
2535         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
2536         break;
2537     }
2538     return status;
2539 }
2540
2541 static GstVaapiDecoderStatus
2542 decode_buffer(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
2543 {
2544     GstVaapiDecoderH264Private * const priv = decoder->priv;
2545     GstVaapiDecoderStatus status;
2546     GstH264ParserResult result;
2547     GstH264NalUnit nalu;
2548     gboolean is_eos;
2549     const guchar *buf;
2550     guint i, buf_size, nalu_size, size;
2551     guint32 start_code;
2552     gint ofs;
2553
2554     buf      = GST_BUFFER_DATA(buffer);
2555     buf_size = GST_BUFFER_SIZE(buffer);
2556     is_eos   = GST_BUFFER_IS_EOS(buffer);
2557     if (buf && buf_size > 0)
2558         gst_adapter_push(priv->adapter, gst_buffer_ref(buffer));
2559
2560     size = gst_adapter_available(priv->adapter);
2561     do {
2562         if (size == 0) {
2563             status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2564             break;
2565         }
2566
2567         status = gst_vaapi_decoder_check_status(GST_VAAPI_DECODER(decoder));
2568         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2569             break;
2570
2571         status = GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2572         if (priv->is_avc) {
2573             if (size < priv->nal_length_size)
2574                 break;
2575             buf = gst_adapter_peek(priv->adapter, priv->nal_length_size);
2576
2577             nalu_size = 0;
2578             for (i = 0; i < priv->nal_length_size; i++)
2579                 nalu_size = (nalu_size << 8) | buf[i];
2580
2581             buf_size = priv->nal_length_size + nalu_size;
2582             if (size < buf_size)
2583                 break;
2584             buffer = gst_adapter_take_buffer(priv->adapter, buf_size);
2585             size -= buf_size;
2586
2587             buf      = GST_BUFFER_DATA(buffer);
2588             buf_size = GST_BUFFER_SIZE(buffer);
2589
2590             result = gst_h264_parser_identify_nalu_avc(
2591                 priv->parser,
2592                 buf, 0, buf_size, priv->nal_length_size,
2593                 &nalu
2594             );
2595         }
2596         else {
2597             if (size < 4)
2598                 break;
2599             ofs = scan_for_start_code(priv->adapter, 0, size, &start_code);
2600             if (ofs < 0)
2601                 break;
2602             gst_adapter_flush(priv->adapter, ofs);
2603             size -= ofs;
2604
2605             ofs = G_UNLIKELY(size < 8) ? -1 :
2606                 scan_for_start_code(priv->adapter, 4, size - 4, NULL);
2607             if (ofs < 0) {
2608                 // Assume the whole NAL unit is present if end-of-stream
2609                 if (!is_eos)
2610                     break;
2611                 ofs = size;
2612             }
2613             buffer = gst_adapter_take_buffer(priv->adapter, ofs);
2614             size -= ofs;
2615
2616             buf      = GST_BUFFER_DATA(buffer);
2617             buf_size = GST_BUFFER_SIZE(buffer);
2618
2619             result = gst_h264_parser_identify_nalu_unchecked(
2620                 priv->parser,
2621                 buf, 0, buf_size,
2622                 &nalu
2623             );
2624         }
2625         status = get_status(result);
2626         if (status == GST_VAAPI_DECODER_STATUS_SUCCESS)
2627             status = decode_nalu(decoder, &nalu);
2628         gst_buffer_unref(buffer);
2629     } while (status == GST_VAAPI_DECODER_STATUS_SUCCESS);
2630
2631     if (is_eos && (status == GST_VAAPI_DECODER_STATUS_SUCCESS ||
2632                    status == GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA))
2633         status = decode_sequence_end(decoder);
2634     return status;
2635 }
2636
2637 static GstVaapiDecoderStatus
2638 decode_codec_data(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
2639 {
2640     GstVaapiDecoderH264Private * const priv = decoder->priv;
2641     GstVaapiDecoderStatus status;
2642     GstH264NalUnit nalu;
2643     GstH264ParserResult result;
2644     guchar *buf;
2645     guint buf_size;
2646     guint i, ofs, num_sps, num_pps;
2647
2648     buf      = GST_BUFFER_DATA(buffer);
2649     buf_size = GST_BUFFER_SIZE(buffer);
2650     if (!buf || buf_size == 0)
2651         return GST_VAAPI_DECODER_STATUS_SUCCESS;
2652
2653     if (buf_size < 8)
2654         return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2655
2656     if (buf[0] != 1) {
2657         GST_ERROR("failed to decode codec-data, not in avcC format");
2658         return GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
2659     }
2660
2661     priv->nal_length_size = (buf[4] & 0x03) + 1;
2662
2663     num_sps = buf[5] & 0x1f;
2664     ofs = 6;
2665
2666     for (i = 0; i < num_sps; i++) {
2667         result = gst_h264_parser_identify_nalu_avc(
2668             priv->parser,
2669             buf, ofs, buf_size, 2,
2670             &nalu
2671         );
2672         if (result != GST_H264_PARSER_OK)
2673             return get_status(result);
2674
2675         status = decode_sps(decoder, &nalu);
2676         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2677             return status;
2678         ofs = nalu.offset + nalu.size;
2679     }
2680
2681     num_pps = buf[ofs];
2682     ofs++;
2683
2684     for (i = 0; i < num_pps; i++) {
2685         result = gst_h264_parser_identify_nalu_avc(
2686             priv->parser,
2687             buf, ofs, buf_size, 2,
2688             &nalu
2689         );
2690         if (result != GST_H264_PARSER_OK)
2691             return get_status(result);
2692
2693         status = decode_pps(decoder, &nalu);
2694         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2695             return status;
2696         ofs = nalu.offset + nalu.size;
2697     }
2698
2699     priv->is_avc = TRUE;
2700     return status;
2701 }
2702
2703 GstVaapiDecoderStatus
2704 gst_vaapi_decoder_h264_decode(GstVaapiDecoder *base, GstBuffer *buffer)
2705 {
2706     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264(base);
2707     GstVaapiDecoderH264Private * const priv = decoder->priv;
2708     GstVaapiDecoderStatus status;
2709     GstBuffer *codec_data;
2710
2711     g_return_val_if_fail(priv->is_constructed,
2712                          GST_VAAPI_DECODER_STATUS_ERROR_INIT_FAILED);
2713
2714     if (!priv->is_opened) {
2715         priv->is_opened = gst_vaapi_decoder_h264_open(decoder, buffer);
2716         if (!priv->is_opened)
2717             return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CODEC;
2718
2719         codec_data = GST_VAAPI_DECODER_CODEC_DATA(decoder);
2720         if (codec_data) {
2721             status = decode_codec_data(decoder, codec_data);
2722             if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2723                 return status;
2724         }
2725      }
2726      return decode_buffer(decoder, buffer);
2727 }
2728
2729 static void
2730 gst_vaapi_decoder_h264_finalize(GObject *object)
2731 {
2732     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264(object);
2733
2734     gst_vaapi_decoder_h264_destroy(decoder);
2735
2736     G_OBJECT_CLASS(gst_vaapi_decoder_h264_parent_class)->finalize(object);
2737 }
2738
2739 static void
2740 gst_vaapi_decoder_h264_constructed(GObject *object)
2741 {
2742     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264(object);
2743     GstVaapiDecoderH264Private * const priv = decoder->priv;
2744     GObjectClass *parent_class;
2745
2746     parent_class = G_OBJECT_CLASS(gst_vaapi_decoder_h264_parent_class);
2747     if (parent_class->constructed)
2748         parent_class->constructed(object);
2749
2750     priv->is_constructed = gst_vaapi_decoder_h264_create(decoder);
2751 }
2752
2753 static void
2754 gst_vaapi_decoder_h264_class_init(GstVaapiDecoderH264Class *klass)
2755 {
2756     GObjectClass * const object_class = G_OBJECT_CLASS(klass);
2757     GstVaapiDecoderClass * const decoder_class = GST_VAAPI_DECODER_CLASS(klass);
2758
2759     g_type_class_add_private(klass, sizeof(GstVaapiDecoderH264Private));
2760
2761     object_class->finalize      = gst_vaapi_decoder_h264_finalize;
2762     object_class->constructed   = gst_vaapi_decoder_h264_constructed;
2763
2764     decoder_class->decode       = gst_vaapi_decoder_h264_decode;
2765 }
2766
2767 static void
2768 gst_vaapi_decoder_h264_init(GstVaapiDecoderH264 *decoder)
2769 {
2770     GstVaapiDecoderH264Private *priv;
2771
2772     priv                        = GST_VAAPI_DECODER_H264_GET_PRIVATE(decoder);
2773     decoder->priv               = priv;
2774     priv->parser                = NULL;
2775     priv->current_picture       = NULL;
2776     priv->dpb_count             = 0;
2777     priv->dpb_size              = 0;
2778     priv->profile               = GST_VAAPI_PROFILE_UNKNOWN;
2779     priv->entrypoint            = GST_VAAPI_ENTRYPOINT_VLD;
2780     priv->chroma_type           = GST_VAAPI_CHROMA_TYPE_YUV420;
2781     priv->short_ref_count       = 0;
2782     priv->long_ref_count        = 0;
2783     priv->RefPicList0_count     = 0;
2784     priv->RefPicList1_count     = 0;
2785     priv->nal_length_size       = 0;
2786     priv->width                 = 0;
2787     priv->height                = 0;
2788     priv->adapter               = NULL;
2789     priv->field_poc[0]          = 0;
2790     priv->field_poc[1]          = 0;
2791     priv->poc_msb               = 0;
2792     priv->poc_lsb               = 0;
2793     priv->prev_poc_msb          = 0;
2794     priv->prev_poc_lsb          = 0;
2795     priv->frame_num_offset      = 0;
2796     priv->frame_num             = 0;
2797     priv->prev_frame_num        = 0;
2798     priv->prev_pic_has_mmco5    = FALSE;
2799     priv->prev_pic_structure    = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
2800     priv->is_constructed        = FALSE;
2801     priv->is_opened             = FALSE;
2802     priv->is_avc                = FALSE;
2803     priv->has_context           = FALSE;
2804
2805     memset(priv->dpb, 0, sizeof(priv->dpb));
2806     memset(priv->short_ref, 0, sizeof(priv->short_ref));
2807     memset(priv->long_ref, 0, sizeof(priv->long_ref));
2808     memset(priv->RefPicList0, 0, sizeof(priv->RefPicList0));
2809     memset(priv->RefPicList1, 0, sizeof(priv->RefPicList1));
2810 }
2811
2812 /**
2813  * gst_vaapi_decoder_h264_new:
2814  * @display: a #GstVaapiDisplay
2815  * @caps: a #GstCaps holding codec information
2816  *
2817  * Creates a new #GstVaapiDecoder for MPEG-2 decoding.  The @caps can
2818  * hold extra information like codec-data and pictured coded size.
2819  *
2820  * Return value: the newly allocated #GstVaapiDecoder object
2821  */
2822 GstVaapiDecoder *
2823 gst_vaapi_decoder_h264_new(GstVaapiDisplay *display, GstCaps *caps)
2824 {
2825     GstVaapiDecoderH264 *decoder;
2826
2827     g_return_val_if_fail(GST_VAAPI_IS_DISPLAY(display), NULL);
2828     g_return_val_if_fail(GST_IS_CAPS(caps), NULL);
2829
2830     decoder = g_object_new(
2831         GST_VAAPI_TYPE_DECODER_H264,
2832         "display",      display,
2833         "caps",         caps,
2834         NULL
2835     );
2836     if (!decoder->priv->is_constructed) {
2837         g_object_unref(decoder);
2838         return NULL;
2839     }
2840     return GST_VAAPI_DECODER_CAST(decoder);
2841 }