gst-libs/gst/vaapi/gstvaapidecoder_h264.c

   1 /*
   2  *  gstvaapidecoder_h264.c - H.264 decoder
   3  *
   4  *  Copyright (C) 2011-2013 Intel Corporation
   5  *    Author: Gwenole Beauchesne <gwenole.beauchesne@intel.com>
   6  *
   7  *  This library is free software; you can redistribute it and/or
   8  *  modify it under the terms of the GNU Lesser General Public License
   9  *  as published by the Free Software Foundation; either version 2.1
  10  *  of the License, or (at your option) any later version.
  11  *
  12  *  This library is distributed in the hope that it will be useful,
  13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  *  Lesser General Public License for more details.
  16  *
  17  *  You should have received a copy of the GNU Lesser General Public
  18  *  License along with this library; if not, write to the Free
  19  *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  20  *  Boston, MA 02110-1301 USA
  21  */
  22
  23 /**
  24  * SECTION:gstvaapidecoder_h264
  25  * @short_description: H.264 decoder
  26  */
  27
  28 #include "sysdeps.h"
  29 #include <string.h>
  30 #include <gst/base/gstadapter.h>
  31 #include <gst/codecparsers/gsth264parser.h>
  32 #include "gstvaapidecoder_h264.h"
  33 #include "gstvaapidecoder_objects.h"
  34 #include "gstvaapidecoder_priv.h"
  35 #include "gstvaapidisplay_priv.h"
  36 #include "gstvaapiobject_priv.h"
  37 #include "gstvaapiutils_h264.h"
  38
  39 #define DEBUG 1
  40 #include "gstvaapidebug.h"
  41
  42 /* Defined to 1 if strict ordering of DPB is needed. Only useful for debug */
  43 #define USE_STRICT_DPB_ORDERING 0
  44
  45 typedef struct _GstVaapiDecoderH264Private      GstVaapiDecoderH264Private;
  46 typedef struct _GstVaapiDecoderH264Class        GstVaapiDecoderH264Class;
  47 typedef struct _GstVaapiFrameStore              GstVaapiFrameStore;
  48 typedef struct _GstVaapiFrameStoreClass         GstVaapiFrameStoreClass;
  49 typedef struct _GstVaapiParserInfoH264          GstVaapiParserInfoH264;
  50 typedef struct _GstVaapiPictureH264             GstVaapiPictureH264;
  51
  52 // Used for field_poc[]
  53 #define TOP_FIELD       0
  54 #define BOTTOM_FIELD    1
  55
  56 /* ------------------------------------------------------------------------- */
  57 /* --- H.264 Parser Info                                                 --- */
  58 /* ------------------------------------------------------------------------- */
  59
  60 #define GST_VAAPI_PARSER_INFO_H264(obj) \
  61     ((GstVaapiParserInfoH264 *)(obj))
  62
  63 struct _GstVaapiParserInfoH264 {
  64     GstVaapiMiniObject  parent_instance;
  65     GstH264NalUnit      nalu;
  66     union {
  67         GstH264SPS      sps;
  68         GstH264PPS      pps;
  69         GstH264SliceHdr slice_hdr;
  70     }                   data;
  71 };
  72
  73 static inline const GstVaapiMiniObjectClass *
  74 gst_vaapi_parser_info_h264_class(void)
  75 {
  76     static const GstVaapiMiniObjectClass GstVaapiParserInfoH264Class = {
  77         sizeof(GstVaapiParserInfoH264),
  78         NULL
  79     };
  80     return &GstVaapiParserInfoH264Class;
  81 }
  82
  83 static inline GstVaapiParserInfoH264 *
  84 gst_vaapi_parser_info_h264_new(void)
  85 {
  86     return (GstVaapiParserInfoH264 *)
  87         gst_vaapi_mini_object_new(gst_vaapi_parser_info_h264_class());
  88 }
  89
  90 #define gst_vaapi_parser_info_h264_ref(pi) \
  91     gst_vaapi_mini_object_ref(GST_VAAPI_MINI_OBJECT(pi))
  92
  93 #define gst_vaapi_parser_info_h264_unref(pi) \
  94     gst_vaapi_mini_object_unref(GST_VAAPI_MINI_OBJECT(pi))
  95
  96 #define gst_vaapi_parser_info_h264_replace(old_pi_ptr, new_pi)          \
  97     gst_vaapi_mini_object_replace((GstVaapiMiniObject **)(old_pi_ptr),  \
  98         (GstVaapiMiniObject *)(new_pi))
  99
 100 /* ------------------------------------------------------------------------- */
 101 /* --- H.264 Pictures                                                    --- */
 102 /* ------------------------------------------------------------------------- */
 103
 104 /*
 105  * Extended picture flags:
 106  *
 107  * @GST_VAAPI_PICTURE_FLAG_IDR: flag that specifies an IDR picture
 108  * @GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE: flag that specifies
 109  *     "used for short-term reference"
 110  * @GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE: flag that specifies
 111  *     "used for long-term reference"
 112  * @GST_VAAPI_PICTURE_FLAGS_REFERENCE: mask covering any kind of
 113  *     reference picture (short-term reference or long-term reference)
 114  */
 115 enum {
 116     GST_VAAPI_PICTURE_FLAG_IDR = (GST_VAAPI_PICTURE_FLAG_LAST << 0),
 117
 118     GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE = (
 119         GST_VAAPI_PICTURE_FLAG_REFERENCE),
 120     GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE = (
 121         GST_VAAPI_PICTURE_FLAG_REFERENCE | (GST_VAAPI_PICTURE_FLAG_LAST << 1)),
 122     GST_VAAPI_PICTURE_FLAGS_REFERENCE = (
 123         GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE |
 124         GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE),
 125 };
 126
 127 #define GST_VAAPI_PICTURE_IS_IDR(picture) \
 128     (GST_VAAPI_PICTURE_FLAG_IS_SET(picture, GST_VAAPI_PICTURE_FLAG_IDR))
 129
 130 #define GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture)      \
 131     ((GST_VAAPI_PICTURE_FLAGS(picture) &                        \
 132       GST_VAAPI_PICTURE_FLAGS_REFERENCE) ==                     \
 133      GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE)
 134
 135 #define GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture)       \
 136     ((GST_VAAPI_PICTURE_FLAGS(picture) &                        \
 137       GST_VAAPI_PICTURE_FLAGS_REFERENCE) ==                     \
 138      GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE)
 139
 140 struct _GstVaapiPictureH264 {
 141     GstVaapiPicture             base;
 142     GstH264PPS                 *pps;
 143     GstH264SliceHdr            *last_slice_hdr;
 144     guint                       structure;
 145     gint32                      field_poc[2];
 146     gint32                      frame_num;              // Original frame_num from slice_header()
 147     gint32                      frame_num_wrap;         // Temporary for ref pic marking: FrameNumWrap
 148     gint32                      long_term_frame_idx;    // Temporary for ref pic marking: LongTermFrameIdx
 149     gint32                      pic_num;                // Temporary for ref pic marking: PicNum
 150     gint32                      long_term_pic_num;      // Temporary for ref pic marking: LongTermPicNum
 151     GstVaapiPictureH264        *other_field;            // Temporary for ref pic marking: other field in the same frame store
 152     guint                       output_flag             : 1;
 153     guint                       output_needed           : 1;
 154 };
 155
 156 GST_VAAPI_CODEC_DEFINE_TYPE(GstVaapiPictureH264, gst_vaapi_picture_h264);
 157
 158 void
 159 gst_vaapi_picture_h264_destroy(GstVaapiPictureH264 *picture)
 160 {
 161     gst_vaapi_picture_destroy(GST_VAAPI_PICTURE(picture));
 162 }
 163
 164 gboolean
 165 gst_vaapi_picture_h264_create(
 166     GstVaapiPictureH264                      *picture,
 167     const GstVaapiCodecObjectConstructorArgs *args
 168 )
 169 {
 170     if (!gst_vaapi_picture_create(GST_VAAPI_PICTURE(picture), args))
 171         return FALSE;
 172
 173     picture->field_poc[0]       = G_MAXINT32;
 174     picture->field_poc[1]       = G_MAXINT32;
 175     picture->output_needed      = FALSE;
 176     return TRUE;
 177 }
 178
 179 static inline GstVaapiPictureH264 *
 180 gst_vaapi_picture_h264_new(GstVaapiDecoderH264 *decoder)
 181 {
 182     return (GstVaapiPictureH264 *)gst_vaapi_codec_object_new(
 183         &GstVaapiPictureH264Class,
 184         GST_VAAPI_CODEC_BASE(decoder),
 185         NULL, sizeof(VAPictureParameterBufferH264),
 186         NULL, 0,
 187         0);
 188 }
 189
 190 static inline void
 191 gst_vaapi_picture_h264_set_reference(
 192     GstVaapiPictureH264 *picture,
 193     guint                reference_flags,
 194     gboolean             other_field
 195 )
 196 {
 197     if (!picture)
 198         return;
 199     GST_VAAPI_PICTURE_FLAG_UNSET(picture, GST_VAAPI_PICTURE_FLAGS_REFERENCE);
 200     GST_VAAPI_PICTURE_FLAG_SET(picture, reference_flags);
 201
 202     if (!other_field || !(picture = picture->other_field))
 203         return;
 204     GST_VAAPI_PICTURE_FLAG_UNSET(picture, GST_VAAPI_PICTURE_FLAGS_REFERENCE);
 205     GST_VAAPI_PICTURE_FLAG_SET(picture, reference_flags);
 206 }
 207
 208 static inline GstVaapiPictureH264 *
 209 gst_vaapi_picture_h264_new_field(GstVaapiPictureH264 *picture)
 210 {
 211     g_return_val_if_fail(picture, NULL);
 212
 213     return (GstVaapiPictureH264 *)gst_vaapi_picture_new_field(&picture->base);
 214 }
 215
 216 /* ------------------------------------------------------------------------- */
 217 /* --- Frame Buffers (DPB)                                               --- */
 218 /* ------------------------------------------------------------------------- */
 219
 220 struct _GstVaapiFrameStore {
 221     /*< private >*/
 222     GstVaapiMiniObject          parent_instance;
 223
 224     guint                       structure;
 225     GstVaapiPictureH264        *buffers[2];
 226     guint                       num_buffers;
 227     guint                       output_needed;
 228 };
 229
 230 static void
 231 gst_vaapi_frame_store_finalize(gpointer object)
 232 {
 233     GstVaapiFrameStore * const fs = object;
 234     guint i;
 235
 236     for (i = 0; i < fs->num_buffers; i++)
 237         gst_vaapi_picture_replace(&fs->buffers[i], NULL);
 238 }
 239
 240 static GstVaapiFrameStore *
 241 gst_vaapi_frame_store_new(GstVaapiPictureH264 *picture)
 242 {
 243     GstVaapiFrameStore *fs;
 244
 245     static const GstVaapiMiniObjectClass GstVaapiFrameStoreClass = {
 246         sizeof(GstVaapiFrameStore),
 247         gst_vaapi_frame_store_finalize
 248     };
 249
 250     fs = (GstVaapiFrameStore *)
 251         gst_vaapi_mini_object_new(&GstVaapiFrameStoreClass);
 252     if (!fs)
 253         return NULL;
 254
 255     fs->structure       = picture->structure;
 256     fs->buffers[0]      = gst_vaapi_picture_ref(picture);
 257     fs->buffers[1]      = NULL;
 258     fs->num_buffers     = 1;
 259     fs->output_needed   = picture->output_needed;
 260     return fs;
 261 }
 262
 263 static gboolean
 264 gst_vaapi_frame_store_add(GstVaapiFrameStore *fs, GstVaapiPictureH264 *picture)
 265 {
 266     guint field;
 267
 268     g_return_val_if_fail(fs->num_buffers == 1, FALSE);
 269     g_return_val_if_fail(!GST_VAAPI_PICTURE_IS_FRAME(picture), FALSE);
 270     g_return_val_if_fail(!GST_VAAPI_PICTURE_IS_FIRST_FIELD(picture), FALSE);
 271
 272     gst_vaapi_picture_replace(&fs->buffers[fs->num_buffers++], picture);
 273     if (picture->output_flag) {
 274         picture->output_needed = TRUE;
 275         fs->output_needed++;
 276     }
 277
 278     fs->structure = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
 279
 280     field = picture->structure == GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD ?
 281         TOP_FIELD : BOTTOM_FIELD;
 282     g_return_val_if_fail(fs->buffers[0]->field_poc[field] == G_MAXINT32, FALSE);
 283     fs->buffers[0]->field_poc[field] = picture->field_poc[field];
 284     g_return_val_if_fail(picture->field_poc[!field] == G_MAXINT32, FALSE);
 285     picture->field_poc[!field] = fs->buffers[0]->field_poc[!field];
 286     return TRUE;
 287 }
 288
 289 static gboolean
 290 gst_vaapi_frame_store_split_fields(GstVaapiFrameStore *fs)
 291 {
 292     GstVaapiPictureH264 * const first_field = fs->buffers[0];
 293     GstVaapiPictureH264 *second_field;
 294
 295     g_return_val_if_fail(fs->num_buffers == 1, FALSE);
 296
 297     first_field->base.structure = GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD;
 298     GST_VAAPI_PICTURE_FLAG_SET(first_field, GST_VAAPI_PICTURE_FLAG_INTERLACED);
 299
 300     second_field = gst_vaapi_picture_h264_new_field(first_field);
 301     if (!second_field)
 302         return FALSE;
 303     gst_vaapi_picture_replace(&fs->buffers[fs->num_buffers++], second_field);
 304     gst_vaapi_picture_unref(second_field);
 305
 306     second_field->frame_num    = first_field->frame_num;
 307     second_field->field_poc[0] = first_field->field_poc[0];
 308     second_field->field_poc[1] = first_field->field_poc[1];
 309     second_field->output_flag  = first_field->output_flag;
 310     if (second_field->output_flag) {
 311         second_field->output_needed = TRUE;
 312         fs->output_needed++;
 313     }
 314     return TRUE;
 315 }
 316
 317 static inline gboolean
 318 gst_vaapi_frame_store_has_frame(GstVaapiFrameStore *fs)
 319 {
 320     return fs->structure == GST_VAAPI_PICTURE_STRUCTURE_FRAME;
 321 }
 322
 323 static inline gboolean
 324 gst_vaapi_frame_store_has_reference(GstVaapiFrameStore *fs)
 325 {
 326     guint i;
 327
 328     for (i = 0; i < fs->num_buffers; i++) {
 329         if (GST_VAAPI_PICTURE_IS_REFERENCE(fs->buffers[i]))
 330             return TRUE;
 331     }
 332     return FALSE;
 333 }
 334
 335 #define gst_vaapi_frame_store_ref(fs) \
 336     gst_vaapi_mini_object_ref(GST_VAAPI_MINI_OBJECT(fs))
 337
 338 #define gst_vaapi_frame_store_unref(fs) \
 339     gst_vaapi_mini_object_unref(GST_VAAPI_MINI_OBJECT(fs))
 340
 341 #define gst_vaapi_frame_store_replace(old_fs_p, new_fs)                 \
 342     gst_vaapi_mini_object_replace((GstVaapiMiniObject **)(old_fs_p),    \
 343         (GstVaapiMiniObject *)(new_fs))
 344
 345 /* ------------------------------------------------------------------------- */
 346 /* --- H.264 Decoder                                                     --- */
 347 /* ------------------------------------------------------------------------- */
 348
 349 #define GST_VAAPI_DECODER_H264_CAST(decoder) \
 350     ((GstVaapiDecoderH264 *)(decoder))
 351
 352 struct _GstVaapiDecoderH264Private {
 353     GstH264NalParser           *parser;
 354     GstVaapiPictureH264        *current_picture;
 355     GstVaapiParserInfoH264     *prev_slice_pi;
 356     GstVaapiFrameStore         *prev_frame;
 357     GstVaapiFrameStore         *dpb[16];
 358     guint                       dpb_count;
 359     guint                       dpb_size;
 360     GstVaapiProfile             profile;
 361     GstVaapiEntrypoint          entrypoint;
 362     GstVaapiChromaType          chroma_type;
 363     GstVaapiPictureH264        *short_ref[32];
 364     guint                       short_ref_count;
 365     GstVaapiPictureH264        *long_ref[32];
 366     guint                       long_ref_count;
 367     GstVaapiPictureH264        *RefPicList0[32];
 368     guint                       RefPicList0_count;
 369     GstVaapiPictureH264        *RefPicList1[32];
 370     guint                       RefPicList1_count;
 371     guint                       nal_length_size;
 372     guint                       mb_width;
 373     guint                       mb_height;
 374     gint32                      field_poc[2];           // 0:TopFieldOrderCnt / 1:BottomFieldOrderCnt
 375     gint32                      poc_msb;                // PicOrderCntMsb
 376     gint32                      poc_lsb;                // pic_order_cnt_lsb (from slice_header())
 377     gint32                      prev_poc_msb;           // prevPicOrderCntMsb
 378     gint32                      prev_poc_lsb;           // prevPicOrderCntLsb
 379     gint32                      frame_num_offset;       // FrameNumOffset
 380     gint32                      frame_num;              // frame_num (from slice_header())
 381     gint32                      prev_frame_num;         // prevFrameNum
 382     gboolean                    prev_pic_has_mmco5;     // prevMmco5Pic
 383     gboolean                    prev_pic_structure;     // previous picture structure
 384     guint                       is_opened               : 1;
 385     guint                       is_avcC                 : 1;
 386     guint                       got_sps                 : 1;
 387     guint                       got_pps                 : 1;
 388     guint                       has_context             : 1;
 389     guint                       progressive_sequence    : 1;
 390 };
 391
 392 /**
 393  * GstVaapiDecoderH264:
 394  *
 395  * A decoder based on H264.
 396  */
 397 struct _GstVaapiDecoderH264 {
 398     /*< private >*/
 399     GstVaapiDecoder             parent_instance;
 400     GstVaapiDecoderH264Private  priv;
 401 };
 402
 403 /**
 404  * GstVaapiDecoderH264Class:
 405  *
 406  * A decoder class based on H264.
 407  */
 408 struct _GstVaapiDecoderH264Class {
 409     /*< private >*/
 410     GstVaapiDecoderClass parent_class;
 411 };
 412
 413 static gboolean
 414 exec_ref_pic_marking(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture);
 415
 416 /* Get number of reference frames to use */
 417 static guint
 418 get_max_dec_frame_buffering(GstH264SPS *sps)
 419 {
 420     guint max_dec_frame_buffering, PicSizeMbs;
 421     GstVaapiLevelH264 level;
 422     const GstVaapiH264LevelLimits *level_limits;
 423
 424     /* Table A-1 - Level limits */
 425     if (G_UNLIKELY(sps->level_idc == 11 && sps->constraint_set3_flag))
 426         level = GST_VAAPI_LEVEL_H264_L1b;
 427     else
 428         level = gst_vaapi_utils_h264_get_level(sps->level_idc);
 429     level_limits = gst_vaapi_utils_h264_get_level_limits(level);
 430     if (!level_limits)
 431         return 16;
 432
 433     PicSizeMbs = ((sps->pic_width_in_mbs_minus1 + 1) *
 434                   (sps->pic_height_in_map_units_minus1 + 1) *
 435                   (sps->frame_mbs_only_flag ? 1 : 2));
 436     max_dec_frame_buffering = level_limits->MaxDpbMbs / PicSizeMbs;
 437
 438     /* VUI parameters */
 439     if (sps->vui_parameters_present_flag) {
 440         GstH264VUIParams * const vui_params = &sps->vui_parameters;
 441         if (vui_params->bitstream_restriction_flag)
 442             max_dec_frame_buffering = vui_params->max_dec_frame_buffering;
 443         else {
 444             switch (sps->profile_idc) {
 445             case 44:  // CAVLC 4:4:4 Intra profile
 446             case GST_H264_PROFILE_SCALABLE_HIGH:
 447             case GST_H264_PROFILE_HIGH:
 448             case GST_H264_PROFILE_HIGH10:
 449             case GST_H264_PROFILE_HIGH_422:
 450             case GST_H264_PROFILE_HIGH_444:
 451                 if (sps->constraint_set3_flag)
 452                     max_dec_frame_buffering = 0;
 453                 break;
 454             }
 455         }
 456     }
 457
 458     if (max_dec_frame_buffering > 16)
 459         max_dec_frame_buffering = 16;
 460     else if (max_dec_frame_buffering < sps->num_ref_frames)
 461         max_dec_frame_buffering = sps->num_ref_frames;
 462     return MAX(1, max_dec_frame_buffering);
 463 }
 464
 465 static void
 466 array_remove_index_fast(void *array, guint *array_length_ptr, guint index)
 467 {
 468     gpointer * const entries = array;
 469     guint num_entries = *array_length_ptr;
 470
 471     g_return_if_fail(index < num_entries);
 472
 473     if (index != --num_entries)
 474         entries[index] = entries[num_entries];
 475     entries[num_entries] = NULL;
 476     *array_length_ptr = num_entries;
 477 }
 478
 479 #if 1
 480 static inline void
 481 array_remove_index(void *array, guint *array_length_ptr, guint index)
 482 {
 483     array_remove_index_fast(array, array_length_ptr, index);
 484 }
 485 #else
 486 static void
 487 array_remove_index(void *array, guint *array_length_ptr, guint index)
 488 {
 489     gpointer * const entries = array;
 490     const guint num_entries = *array_length_ptr - 1;
 491     guint i;
 492
 493     g_return_if_fail(index <= num_entries);
 494
 495     for (i = index; i < num_entries; i++)
 496         entries[i] = entries[i + 1];
 497     entries[num_entries] = NULL;
 498     *array_length_ptr = num_entries;
 499 }
 500 #endif
 501
 502 #define ARRAY_REMOVE_INDEX(array, index) \
 503     array_remove_index(array, &array##_count, index)
 504
 505 static void
 506 dpb_remove_index(GstVaapiDecoderH264 *decoder, guint index)
 507 {
 508     GstVaapiDecoderH264Private * const priv = &decoder->priv;
 509     guint i, num_frames = --priv->dpb_count;
 510
 511     if (USE_STRICT_DPB_ORDERING) {
 512         for (i = index; i < num_frames; i++)
 513             gst_vaapi_frame_store_replace(&priv->dpb[i], priv->dpb[i + 1]);
 514     }
 515     else if (index != num_frames)
 516         gst_vaapi_frame_store_replace(&priv->dpb[index], priv->dpb[num_frames]);
 517     gst_vaapi_frame_store_replace(&priv->dpb[num_frames], NULL);
 518 }
 519
 520 static gboolean
 521 dpb_output(
 522     GstVaapiDecoderH264 *decoder,
 523     GstVaapiFrameStore  *fs,
 524     GstVaapiPictureH264 *picture
 525 )
 526 {
 527     picture->output_needed = FALSE;
 528
 529     if (fs) {
 530         if (--fs->output_needed > 0)
 531             return TRUE;
 532         picture = fs->buffers[0];
 533     }
 534     return gst_vaapi_picture_output(GST_VAAPI_PICTURE_CAST(picture));
 535 }
 536
 537 static inline void
 538 dpb_evict(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture, guint i)
 539 {
 540     GstVaapiDecoderH264Private * const priv = &decoder->priv;
 541     GstVaapiFrameStore * const fs = priv->dpb[i];
 542
 543     if (!fs->output_needed && !gst_vaapi_frame_store_has_reference(fs))
 544         dpb_remove_index(decoder, i);
 545 }
 546
 547 static gboolean
 548 dpb_bump(GstVaapiDecoderH264 *decoder)
 549 {
 550     GstVaapiDecoderH264Private * const priv = &decoder->priv;
 551     GstVaapiPictureH264 *found_picture = NULL;
 552     guint i, j, found_index;
 553     gboolean success;
 554
 555     for (i = 0; i < priv->dpb_count; i++) {
 556         GstVaapiFrameStore * const fs = priv->dpb[i];
 557         if (!fs->output_needed)
 558             continue;
 559         for (j = 0; j < fs->num_buffers; j++) {
 560             GstVaapiPictureH264 * const picture = fs->buffers[j];
 561             if (!picture->output_needed)
 562                 continue;
 563             if (!found_picture || found_picture->base.poc > picture->base.poc)
 564                 found_picture = picture, found_index = i;
 565         }
 566     }
 567     if (!found_picture)
 568         return FALSE;
 569
 570     success = dpb_output(decoder, priv->dpb[found_index], found_picture);
 571     dpb_evict(decoder, found_picture, found_index);
 572     return success;
 573 }
 574
 575 static void
 576 dpb_clear(GstVaapiDecoderH264 *decoder)
 577 {
 578     GstVaapiDecoderH264Private * const priv = &decoder->priv;
 579     guint i;
 580
 581     for (i = 0; i < priv->dpb_count; i++)
 582         gst_vaapi_frame_store_replace(&priv->dpb[i], NULL);
 583     priv->dpb_count = 0;
 584
 585     gst_vaapi_frame_store_replace(&priv->prev_frame, NULL);
 586 }
 587
 588 static void
 589 dpb_flush(GstVaapiDecoderH264 *decoder)
 590 {
 591     while (dpb_bump(decoder))
 592         ;
 593     dpb_clear(decoder);
 594 }
 595
 596 static gboolean
 597 dpb_add(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
 598 {
 599     GstVaapiDecoderH264Private * const priv = &decoder->priv;
 600     GstVaapiFrameStore *fs;
 601     guint i, j;
 602
 603     // Remove all unused pictures
 604     if (!GST_VAAPI_PICTURE_IS_IDR(picture)) {
 605         i = 0;
 606         while (i < priv->dpb_count) {
 607             GstVaapiFrameStore * const fs = priv->dpb[i];
 608             if (!fs->output_needed && !gst_vaapi_frame_store_has_reference(fs))
 609                 dpb_remove_index(decoder, i);
 610             else
 611                 i++;
 612         }
 613     }
 614
 615     // Check if picture is the second field and the first field is still in DPB
 616     fs = priv->prev_frame;
 617     if (fs && !gst_vaapi_frame_store_has_frame(fs))
 618         return gst_vaapi_frame_store_add(fs, picture);
 619
 620     // Create new frame store, and split fields if necessary
 621     fs = gst_vaapi_frame_store_new(picture);
 622     if (!fs)
 623         return FALSE;
 624     gst_vaapi_frame_store_replace(&priv->prev_frame, fs);
 625     gst_vaapi_frame_store_unref(fs);
 626
 627     if (!priv->progressive_sequence && gst_vaapi_frame_store_has_frame(fs)) {
 628         if (!gst_vaapi_frame_store_split_fields(fs))
 629             return FALSE;
 630     }
 631
 632     // C.4.5.1 - Storage and marking of a reference decoded picture into the DPB
 633     if (GST_VAAPI_PICTURE_IS_REFERENCE(picture)) {
 634         while (priv->dpb_count == priv->dpb_size) {
 635             if (!dpb_bump(decoder))
 636                 return FALSE;
 637         }
 638         gst_vaapi_frame_store_replace(&priv->dpb[priv->dpb_count++], fs);
 639         if (picture->output_flag) {
 640             picture->output_needed = TRUE;
 641             fs->output_needed++;
 642         }
 643     }
 644
 645     // C.4.5.2 - Storage and marking of a non-reference decoded picture into the DPB
 646     else {
 647         if (!picture->output_flag)
 648             return TRUE;
 649         while (priv->dpb_count == priv->dpb_size) {
 650             gboolean found_picture = FALSE;
 651             for (i = 0; !found_picture && i < priv->dpb_count; i++) {
 652                 GstVaapiFrameStore * const fs = priv->dpb[i];
 653                 if (!fs->output_needed)
 654                     continue;
 655                 for (j = 0; !found_picture && j < fs->num_buffers; j++)
 656                     found_picture = fs->buffers[j]->output_needed &&
 657                         fs->buffers[j]->base.poc < picture->base.poc;
 658             }
 659             if (!found_picture)
 660                 return dpb_output(decoder, NULL, picture);
 661             if (!dpb_bump(decoder))
 662                 return FALSE;
 663         }
 664         gst_vaapi_frame_store_replace(&priv->dpb[priv->dpb_count++], fs);
 665         picture->output_needed = TRUE;
 666         fs->output_needed++;
 667     }
 668     return TRUE;
 669 }
 670
 671 static inline void
 672 dpb_reset(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 673 {
 674     GstVaapiDecoderH264Private * const priv = &decoder->priv;
 675
 676     priv->dpb_size = get_max_dec_frame_buffering(sps);
 677     GST_DEBUG("DPB size %u", priv->dpb_size);
 678 }
 679
 680 static GstVaapiDecoderStatus
 681 get_status(GstH264ParserResult result)
 682 {
 683     GstVaapiDecoderStatus status;
 684
 685     switch (result) {
 686     case GST_H264_PARSER_OK:
 687         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
 688         break;
 689     case GST_H264_PARSER_NO_NAL_END:
 690         status = GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
 691         break;
 692     case GST_H264_PARSER_ERROR:
 693         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
 694         break;
 695     default:
 696         status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 697         break;
 698     }
 699     return status;
 700 }
 701
 702 static void
 703 gst_vaapi_decoder_h264_close(GstVaapiDecoderH264 *decoder)
 704 {
 705     GstVaapiDecoderH264Private * const priv = &decoder->priv;
 706
 707     gst_vaapi_picture_replace(&priv->current_picture, NULL);
 708     gst_vaapi_parser_info_h264_replace(&priv->prev_slice_pi, NULL);
 709
 710     dpb_clear(decoder);
 711
 712     if (priv->parser) {
 713         gst_h264_nal_parser_free(priv->parser);
 714         priv->parser = NULL;
 715     }
 716 }
 717
 718 static gboolean
 719 gst_vaapi_decoder_h264_open(GstVaapiDecoderH264 *decoder)
 720 {
 721     GstVaapiDecoderH264Private * const priv = &decoder->priv;
 722
 723     gst_vaapi_decoder_h264_close(decoder);
 724
 725     priv->parser = gst_h264_nal_parser_new();
 726     if (!priv->parser)
 727         return FALSE;
 728     return TRUE;
 729 }
 730
 731 static void
 732 gst_vaapi_decoder_h264_destroy(GstVaapiDecoder *base_decoder)
 733 {
 734     GstVaapiDecoderH264 * const decoder =
 735         GST_VAAPI_DECODER_H264_CAST(base_decoder);
 736
 737     gst_vaapi_decoder_h264_close(decoder);
 738 }
 739
 740 static gboolean
 741 gst_vaapi_decoder_h264_create(GstVaapiDecoder *base_decoder)
 742 {
 743     GstVaapiDecoderH264 * const decoder =
 744         GST_VAAPI_DECODER_H264_CAST(base_decoder);
 745     GstVaapiDecoderH264Private * const priv = &decoder->priv;
 746
 747     priv->profile               = GST_VAAPI_PROFILE_UNKNOWN;
 748     priv->entrypoint            = GST_VAAPI_ENTRYPOINT_VLD;
 749     priv->chroma_type           = GST_VAAPI_CHROMA_TYPE_YUV420;
 750     priv->prev_pic_structure    = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
 751     priv->progressive_sequence  = TRUE;
 752     return TRUE;
 753 }
 754
 755 static GstVaapiProfile
 756 get_profile(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 757 {
 758     GstVaapiDecoderH264Private * const priv = &decoder->priv;
 759     GstVaapiDisplay * const display = GST_VAAPI_DECODER_DISPLAY(decoder);
 760     GstVaapiProfile profile, profiles[2];
 761     guint i, n_profiles = 0;
 762
 763     profile = gst_vaapi_utils_h264_get_profile(sps->profile_idc);
 764     if (!profile)
 765         return GST_VAAPI_PROFILE_UNKNOWN;
 766
 767     profiles[n_profiles++] = profile;
 768     switch (profile) {
 769     case GST_VAAPI_PROFILE_H264_MAIN:
 770         profiles[n_profiles++] = GST_VAAPI_PROFILE_H264_HIGH;
 771         break;
 772     default:
 773         break;
 774     }
 775
 776     /* If the preferred profile (profiles[0]) matches one that we already
 777        found, then just return it now instead of searching for it again */
 778     if (profiles[0] == priv->profile)
 779         return priv->profile;
 780
 781     for (i = 0; i < n_profiles; i++) {
 782         if (gst_vaapi_display_has_decoder(display, profiles[i], priv->entrypoint))
 783             return profiles[i];
 784     }
 785     return GST_VAAPI_PROFILE_UNKNOWN;
 786 }
 787
 788 static GstVaapiDecoderStatus
 789 ensure_context(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 790 {
 791     GstVaapiDecoder * const base_decoder = GST_VAAPI_DECODER_CAST(decoder);
 792     GstVaapiDecoderH264Private * const priv = &decoder->priv;
 793     GstVaapiContextInfo info;
 794     GstVaapiProfile profile;
 795     GstVaapiChromaType chroma_type;
 796     gboolean reset_context = FALSE;
 797     guint mb_width, mb_height;
 798
 799     profile = get_profile(decoder, sps);
 800     if (!profile) {
 801         GST_ERROR("unsupported profile_idc %u", sps->profile_idc);
 802         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_PROFILE;
 803     }
 804
 805     if (priv->profile != profile) {
 806         GST_DEBUG("profile changed");
 807         reset_context = TRUE;
 808         priv->profile = profile;
 809     }
 810
 811     chroma_type = gst_vaapi_utils_h264_get_chroma_type(sps->chroma_format_idc);
 812     if (!chroma_type || chroma_type != GST_VAAPI_CHROMA_TYPE_YUV420) {
 813         GST_ERROR("unsupported chroma_format_idc %u", sps->chroma_format_idc);
 814         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;
 815     }
 816
 817     if (priv->chroma_type != chroma_type) {
 818         GST_DEBUG("chroma format changed");
 819         reset_context     = TRUE;
 820         priv->chroma_type = chroma_type;
 821     }
 822
 823     mb_width  = sps->pic_width_in_mbs_minus1 + 1;
 824     mb_height = (sps->pic_height_in_map_units_minus1 + 1) <<
 825         !sps->frame_mbs_only_flag;
 826     if (priv->mb_width != mb_width || priv->mb_height != mb_height) {
 827         GST_DEBUG("size changed");
 828         reset_context   = TRUE;
 829         priv->mb_width  = mb_width;
 830         priv->mb_height = mb_height;
 831     }
 832
 833     priv->progressive_sequence = sps->frame_mbs_only_flag;
 834 #if 0
 835     /* XXX: we only output complete frames for now */
 836     gst_vaapi_decoder_set_interlaced(base_decoder, !priv->progressive_sequence);
 837 #endif
 838
 839     gst_vaapi_decoder_set_pixel_aspect_ratio(
 840         base_decoder,
 841         sps->vui_parameters.par_n,
 842         sps->vui_parameters.par_d
 843     );
 844
 845     if (!reset_context && priv->has_context)
 846         return GST_VAAPI_DECODER_STATUS_SUCCESS;
 847
 848     /* XXX: fix surface size when cropping is implemented */
 849     info.profile    = priv->profile;
 850     info.entrypoint = priv->entrypoint;
 851     info.width      = sps->width;
 852     info.height     = sps->height;
 853     info.ref_frames = get_max_dec_frame_buffering(sps);
 854
 855     if (!gst_vaapi_decoder_ensure_context(GST_VAAPI_DECODER(decoder), &info))
 856         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 857     priv->has_context = TRUE;
 858
 859     /* Reset DPB */
 860     dpb_reset(decoder, sps);
 861     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 862 }
 863
 864 static void
 865 fill_iq_matrix_4x4(VAIQMatrixBufferH264 *iq_matrix, const GstH264PPS *pps)
 866 {
 867     const guint8 (* const ScalingList4x4)[6][16] = &pps->scaling_lists_4x4;
 868     guint i, j;
 869
 870     /* There are always 6 4x4 scaling lists */
 871     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList4x4) == 6);
 872     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList4x4[0]) == 16);
 873
 874     if (sizeof(iq_matrix->ScalingList4x4[0][0]) == 1)
 875         memcpy(iq_matrix->ScalingList4x4, *ScalingList4x4,
 876                sizeof(iq_matrix->ScalingList4x4));
 877     else {
 878         for (i = 0; i < G_N_ELEMENTS(iq_matrix->ScalingList4x4); i++) {
 879             for (j = 0; j < G_N_ELEMENTS(iq_matrix->ScalingList4x4[i]); j++)
 880                 iq_matrix->ScalingList4x4[i][j] = (*ScalingList4x4)[i][j];
 881         }
 882     }
 883 }
 884
 885 static void
 886 fill_iq_matrix_8x8(VAIQMatrixBufferH264 *iq_matrix, const GstH264PPS *pps)
 887 {
 888     const guint8 (* const ScalingList8x8)[6][64] = &pps->scaling_lists_8x8;
 889     const GstH264SPS * const sps = pps->sequence;
 890     guint i, j, n;
 891
 892     /* If chroma_format_idc != 3, there are up to 2 8x8 scaling lists */
 893     if (!pps->transform_8x8_mode_flag)
 894         return;
 895
 896     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList8x8) >= 2);
 897     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList8x8[0]) == 64);
 898
 899     if (sizeof(iq_matrix->ScalingList8x8[0][0]) == 1)
 900         memcpy(iq_matrix->ScalingList8x8, *ScalingList8x8,
 901                sizeof(iq_matrix->ScalingList8x8));
 902     else {
 903         n = (sps->chroma_format_idc != 3) ? 2 : 6;
 904         for (i = 0; i < n; i++) {
 905             for (j = 0; j < G_N_ELEMENTS(iq_matrix->ScalingList8x8[i]); j++)
 906                 iq_matrix->ScalingList8x8[i][j] = (*ScalingList8x8)[i][j];
 907         }
 908     }
 909 }
 910
 911 static GstVaapiDecoderStatus
 912 ensure_quant_matrix(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
 913 {
 914     GstVaapiPicture * const base_picture = &picture->base;
 915     GstH264PPS * const pps = picture->pps;
 916     GstH264SPS * const sps = pps->sequence;
 917     VAIQMatrixBufferH264 *iq_matrix;
 918
 919     base_picture->iq_matrix = GST_VAAPI_IQ_MATRIX_NEW(H264, decoder);
 920     if (!base_picture->iq_matrix) {
 921         GST_ERROR("failed to allocate IQ matrix");
 922         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
 923     }
 924     iq_matrix = base_picture->iq_matrix->param;
 925
 926     /* XXX: we can only support 4:2:0 or 4:2:2 since ScalingLists8x8[]
 927        is not large enough to hold lists for 4:4:4 */
 928     if (sps->chroma_format_idc == 3)
 929         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;
 930
 931     fill_iq_matrix_4x4(iq_matrix, pps);
 932     fill_iq_matrix_8x8(iq_matrix, pps);
 933
 934     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 935 }
 936
 937 static GstVaapiDecoderStatus
 938 decode_current_picture(GstVaapiDecoderH264 *decoder)
 939 {
 940     GstVaapiDecoderH264Private * const priv = &decoder->priv;
 941     GstVaapiPictureH264 * const picture = priv->current_picture;
 942
 943     if (!picture)
 944         return GST_VAAPI_DECODER_STATUS_SUCCESS;
 945
 946     if (!exec_ref_pic_marking(decoder, picture))
 947         goto error;
 948     if (!dpb_add(decoder, picture))
 949         goto error;
 950     if (!gst_vaapi_picture_decode(GST_VAAPI_PICTURE_CAST(picture)))
 951         goto error;
 952     if (priv->prev_frame && gst_vaapi_frame_store_has_frame(priv->prev_frame))
 953         gst_vaapi_picture_replace(&priv->current_picture, NULL);
 954     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 955
 956 error:
 957     /* XXX: fix for cases where first field failed to be decoded */
 958     gst_vaapi_picture_replace(&priv->current_picture, NULL);
 959     return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 960 }
 961
 962 static GstVaapiDecoderStatus
 963 parse_sps(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnit *unit)
 964 {
 965     GstVaapiDecoderH264Private * const priv = &decoder->priv;
 966     GstVaapiParserInfoH264 * const pi = unit->parsed_info;
 967     GstH264SPS * const sps = &pi->data.sps;
 968     GstH264ParserResult result;
 969
 970     GST_DEBUG("parse SPS");
 971
 972     /* Variables that don't have inferred values per the H.264
 973        standard but that should get a default value anyway */
 974     sps->log2_max_pic_order_cnt_lsb_minus4 = 0;
 975
 976     result = gst_h264_parser_parse_sps(priv->parser, &pi->nalu, sps, TRUE);
 977     if (result != GST_H264_PARSER_OK)
 978         return get_status(result);
 979
 980     priv->got_sps = TRUE;
 981     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 982 }
 983
 984 static GstVaapiDecoderStatus
 985 parse_pps(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnit *unit)
 986 {
 987     GstVaapiDecoderH264Private * const priv = &decoder->priv;
 988     GstVaapiParserInfoH264 * const pi = unit->parsed_info;
 989     GstH264PPS * const pps = &pi->data.pps;
 990     GstH264ParserResult result;
 991
 992     GST_DEBUG("parse PPS");
 993
 994     /* Variables that don't have inferred values per the H.264
 995        standard but that should get a default value anyway */
 996     pps->slice_group_map_type = 0;
 997     pps->slice_group_change_rate_minus1 = 0;
 998
 999     result = gst_h264_parser_parse_pps(priv->parser, &pi->nalu, pps);
1000     if (result != GST_H264_PARSER_OK)
1001         return get_status(result);
1002
1003     priv->got_pps = TRUE;
1004     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1005 }
1006
1007 static GstVaapiDecoderStatus
1008 parse_sei(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnit *unit)
1009 {
1010     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1011     GstVaapiParserInfoH264 * const pi = unit->parsed_info;
1012     GstH264SEIMessage sei;
1013     GstH264ParserResult result;
1014
1015     GST_DEBUG("parse SEI");
1016
1017     memset(&sei, 0, sizeof(sei));
1018     result = gst_h264_parser_parse_sei(priv->parser, &pi->nalu, &sei);
1019     if (result != GST_H264_PARSER_OK) {
1020         GST_WARNING("failed to parse SEI, payload type:%d", sei.payloadType);
1021         return get_status(result);
1022     }
1023
1024     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1025 }
1026
1027 static GstVaapiDecoderStatus
1028 parse_slice(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnit *unit)
1029 {
1030     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1031     GstVaapiParserInfoH264 * const pi = unit->parsed_info;
1032     GstH264SliceHdr * const slice_hdr = &pi->data.slice_hdr;
1033     GstH264ParserResult result;
1034
1035     GST_DEBUG("parse slice");
1036
1037     /* Variables that don't have inferred values per the H.264
1038        standard but that should get a default value anyway */
1039     slice_hdr->cabac_init_idc = 0;
1040     slice_hdr->direct_spatial_mv_pred_flag = 0;
1041
1042     result = gst_h264_parser_parse_slice_hdr(priv->parser, &pi->nalu,
1043         slice_hdr, TRUE, TRUE);
1044     if (result != GST_H264_PARSER_OK)
1045         return get_status(result);
1046
1047     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1048 }
1049
1050 static GstVaapiDecoderStatus
1051 decode_sequence_end(GstVaapiDecoderH264 *decoder)
1052 {
1053     GstVaapiDecoderStatus status;
1054
1055     GST_DEBUG("decode sequence-end");
1056
1057     status = decode_current_picture(decoder);
1058     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
1059         return status;
1060
1061     dpb_flush(decoder);
1062     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1063 }
1064
1065 /* 8.2.1.1 - Decoding process for picture order count type 0 */
1066 static void
1067 init_picture_poc_0(
1068     GstVaapiDecoderH264 *decoder,
1069     GstVaapiPictureH264 *picture,
1070     GstH264SliceHdr     *slice_hdr
1071 )
1072 {
1073     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1074     GstH264PPS * const pps = slice_hdr->pps;
1075     GstH264SPS * const sps = pps->sequence;
1076     const gint32 MaxPicOrderCntLsb = 1 << (sps->log2_max_pic_order_cnt_lsb_minus4 + 4);
1077     gint32 temp_poc;
1078
1079     GST_DEBUG("decode picture order count type 0");
1080
1081     if (GST_VAAPI_PICTURE_IS_IDR(picture)) {
1082         priv->prev_poc_msb = 0;
1083         priv->prev_poc_lsb = 0;
1084     }
1085     else if (priv->prev_pic_has_mmco5) {
1086         priv->prev_poc_msb = 0;
1087         priv->prev_poc_lsb =
1088             (priv->prev_pic_structure == GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD ?
1089              0 : priv->field_poc[TOP_FIELD]);
1090     }
1091     else {
1092         priv->prev_poc_msb = priv->poc_msb;
1093         priv->prev_poc_lsb = priv->poc_lsb;
1094     }
1095
1096     // (8-3)
1097     priv->poc_lsb = slice_hdr->pic_order_cnt_lsb;
1098     if (priv->poc_lsb < priv->prev_poc_lsb &&
1099         (priv->prev_poc_lsb - priv->poc_lsb) >= (MaxPicOrderCntLsb / 2))
1100         priv->poc_msb = priv->prev_poc_msb + MaxPicOrderCntLsb;
1101     else if (priv->poc_lsb > priv->prev_poc_lsb &&
1102              (priv->poc_lsb - priv->prev_poc_lsb) > (MaxPicOrderCntLsb / 2))
1103         priv->poc_msb = priv->prev_poc_msb - MaxPicOrderCntLsb;
1104     else
1105         priv->poc_msb = priv->prev_poc_msb;
1106
1107     temp_poc = priv->poc_msb + priv->poc_lsb;
1108     switch (picture->structure) {
1109     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
1110         // (8-4, 8-5)
1111         priv->field_poc[TOP_FIELD] = temp_poc;
1112         priv->field_poc[BOTTOM_FIELD] = temp_poc +
1113             slice_hdr->delta_pic_order_cnt_bottom;
1114         break;
1115     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
1116         // (8-4)
1117         priv->field_poc[TOP_FIELD] = temp_poc;
1118         break;
1119     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
1120         // (8-5)
1121         priv->field_poc[BOTTOM_FIELD] = temp_poc;
1122         break;
1123     }
1124 }
1125
1126 /* 8.2.1.2 - Decoding process for picture order count type 1 */
1127 static void
1128 init_picture_poc_1(
1129     GstVaapiDecoderH264 *decoder,
1130     GstVaapiPictureH264 *picture,
1131     GstH264SliceHdr     *slice_hdr
1132 )
1133 {
1134     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1135     GstH264PPS * const pps = slice_hdr->pps;
1136     GstH264SPS * const sps = pps->sequence;
1137     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1138     gint32 prev_frame_num_offset, abs_frame_num, expected_poc;
1139     guint i;
1140
1141     GST_DEBUG("decode picture order count type 1");
1142
1143     if (priv->prev_pic_has_mmco5)
1144         prev_frame_num_offset = 0;
1145     else
1146         prev_frame_num_offset = priv->frame_num_offset;
1147
1148     // (8-6)
1149     if (GST_VAAPI_PICTURE_IS_IDR(picture))
1150         priv->frame_num_offset = 0;
1151     else if (priv->prev_frame_num > priv->frame_num)
1152         priv->frame_num_offset = prev_frame_num_offset + MaxFrameNum;
1153     else
1154         priv->frame_num_offset = prev_frame_num_offset;
1155
1156     // (8-7)
1157     if (sps->num_ref_frames_in_pic_order_cnt_cycle != 0)
1158         abs_frame_num = priv->frame_num_offset + priv->frame_num;
1159     else
1160         abs_frame_num = 0;
1161     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture) && abs_frame_num > 0)
1162         abs_frame_num = abs_frame_num - 1;
1163
1164     if (abs_frame_num > 0) {
1165         gint32 expected_delta_per_poc_cycle;
1166         gint32 poc_cycle_cnt, frame_num_in_poc_cycle;
1167
1168         expected_delta_per_poc_cycle = 0;
1169         for (i = 0; i < sps->num_ref_frames_in_pic_order_cnt_cycle; i++)
1170             expected_delta_per_poc_cycle += sps->offset_for_ref_frame[i];
1171
1172         // (8-8)
1173         poc_cycle_cnt = (abs_frame_num - 1) /
1174             sps->num_ref_frames_in_pic_order_cnt_cycle;
1175         frame_num_in_poc_cycle = (abs_frame_num - 1) %
1176             sps->num_ref_frames_in_pic_order_cnt_cycle;
1177
1178         // (8-9)
1179         expected_poc = poc_cycle_cnt * expected_delta_per_poc_cycle;
1180         for (i = 0; i <= frame_num_in_poc_cycle; i++)
1181             expected_poc += sps->offset_for_ref_frame[i];
1182     }
1183     else
1184         expected_poc = 0;
1185     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
1186         expected_poc += sps->offset_for_non_ref_pic;
1187
1188     // (8-10)
1189     switch (picture->structure) {
1190     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
1191         priv->field_poc[TOP_FIELD] = expected_poc +
1192             slice_hdr->delta_pic_order_cnt[0];
1193         priv->field_poc[BOTTOM_FIELD] = priv->field_poc[TOP_FIELD] +
1194             sps->offset_for_top_to_bottom_field +
1195             slice_hdr->delta_pic_order_cnt[1];
1196         break;
1197     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
1198         priv->field_poc[TOP_FIELD] = expected_poc +
1199             slice_hdr->delta_pic_order_cnt[0];
1200         break;
1201     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
1202         priv->field_poc[BOTTOM_FIELD] = expected_poc +
1203             sps->offset_for_top_to_bottom_field +
1204             slice_hdr->delta_pic_order_cnt[0];
1205         break;
1206     }
1207 }
1208
1209 /* 8.2.1.3 - Decoding process for picture order count type 2 */
1210 static void
1211 init_picture_poc_2(
1212     GstVaapiDecoderH264 *decoder,
1213     GstVaapiPictureH264 *picture,
1214     GstH264SliceHdr     *slice_hdr
1215 )
1216 {
1217     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1218     GstH264PPS * const pps = slice_hdr->pps;
1219     GstH264SPS * const sps = pps->sequence;
1220     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1221     gint32 prev_frame_num_offset, temp_poc;
1222
1223     GST_DEBUG("decode picture order count type 2");
1224
1225     if (priv->prev_pic_has_mmco5)
1226         prev_frame_num_offset = 0;
1227     else
1228         prev_frame_num_offset = priv->frame_num_offset;
1229
1230     // (8-11)
1231     if (GST_VAAPI_PICTURE_IS_IDR(picture))
1232         priv->frame_num_offset = 0;
1233     else if (priv->prev_frame_num > priv->frame_num)
1234         priv->frame_num_offset = prev_frame_num_offset + MaxFrameNum;
1235     else
1236         priv->frame_num_offset = prev_frame_num_offset;
1237
1238     // (8-12)
1239     if (GST_VAAPI_PICTURE_IS_IDR(picture))
1240         temp_poc = 0;
1241     else if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
1242         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num) - 1;
1243     else
1244         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num);
1245
1246     // (8-13)
1247     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
1248         priv->field_poc[TOP_FIELD] = temp_poc;
1249     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
1250         priv->field_poc[BOTTOM_FIELD] = temp_poc;
1251 }
1252
1253 /* 8.2.1 - Decoding process for picture order count */
1254 static void
1255 init_picture_poc(
1256     GstVaapiDecoderH264 *decoder,
1257     GstVaapiPictureH264 *picture,
1258     GstH264SliceHdr     *slice_hdr
1259 )
1260 {
1261     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1262     GstH264PPS * const pps = slice_hdr->pps;
1263     GstH264SPS * const sps = pps->sequence;
1264
1265     switch (sps->pic_order_cnt_type) {
1266     case 0:
1267         init_picture_poc_0(decoder, picture, slice_hdr);
1268         break;
1269     case 1:
1270         init_picture_poc_1(decoder, picture, slice_hdr);
1271         break;
1272     case 2:
1273         init_picture_poc_2(decoder, picture, slice_hdr);
1274         break;
1275     }
1276
1277     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
1278         picture->field_poc[TOP_FIELD] = priv->field_poc[TOP_FIELD];
1279     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
1280         picture->field_poc[BOTTOM_FIELD] = priv->field_poc[BOTTOM_FIELD];
1281     picture->base.poc = MIN(picture->field_poc[0], picture->field_poc[1]);
1282 }
1283
1284 static int
1285 compare_picture_pic_num_dec(const void *a, const void *b)
1286 {
1287     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1288     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1289
1290     return picB->pic_num - picA->pic_num;
1291 }
1292
1293 static int
1294 compare_picture_long_term_pic_num_inc(const void *a, const void *b)
1295 {
1296     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1297     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1298
1299     return picA->long_term_pic_num - picB->long_term_pic_num;
1300 }
1301
1302 static int
1303 compare_picture_poc_dec(const void *a, const void *b)
1304 {
1305     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1306     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1307
1308     return picB->base.poc - picA->base.poc;
1309 }
1310
1311 static int
1312 compare_picture_poc_inc(const void *a, const void *b)
1313 {
1314     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1315     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1316
1317     return picA->base.poc - picB->base.poc;
1318 }
1319
1320 static int
1321 compare_picture_frame_num_wrap_dec(const void *a, const void *b)
1322 {
1323     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1324     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1325
1326     return picB->frame_num_wrap - picA->frame_num_wrap;
1327 }
1328
1329 static int
1330 compare_picture_long_term_frame_idx_inc(const void *a, const void *b)
1331 {
1332     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1333     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1334
1335     return picA->long_term_frame_idx - picB->long_term_frame_idx;
1336 }
1337
1338 /* 8.2.4.1 - Decoding process for picture numbers */
1339 static void
1340 init_picture_refs_pic_num(
1341     GstVaapiDecoderH264 *decoder,
1342     GstVaapiPictureH264 *picture,
1343     GstH264SliceHdr     *slice_hdr
1344 )
1345 {
1346     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1347     GstH264PPS * const pps = slice_hdr->pps;
1348     GstH264SPS * const sps = pps->sequence;
1349     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1350     guint i;
1351
1352     GST_DEBUG("decode picture numbers");
1353
1354     for (i = 0; i < priv->short_ref_count; i++) {
1355         GstVaapiPictureH264 * const pic = priv->short_ref[i];
1356
1357         // (8-27)
1358         if (pic->frame_num > priv->frame_num)
1359             pic->frame_num_wrap = pic->frame_num - MaxFrameNum;
1360         else
1361             pic->frame_num_wrap = pic->frame_num;
1362
1363         // (8-28, 8-30, 8-31)
1364         if (GST_VAAPI_PICTURE_IS_FRAME(picture))
1365             pic->pic_num = pic->frame_num_wrap;
1366         else {
1367             if (pic->structure == picture->structure)
1368                 pic->pic_num = 2 * pic->frame_num_wrap + 1;
1369             else
1370                 pic->pic_num = 2 * pic->frame_num_wrap;
1371         }
1372     }
1373
1374     for (i = 0; i < priv->long_ref_count; i++) {
1375         GstVaapiPictureH264 * const pic = priv->long_ref[i];
1376
1377         // (8-29, 8-32, 8-33)
1378         if (GST_VAAPI_PICTURE_IS_FRAME(picture))
1379             pic->long_term_pic_num = pic->long_term_frame_idx;
1380         else {
1381             if (pic->structure == picture->structure)
1382                 pic->long_term_pic_num = 2 * pic->long_term_frame_idx + 1;
1383             else
1384                 pic->long_term_pic_num = 2 * pic->long_term_frame_idx;
1385         }
1386     }
1387 }
1388
1389 #define SORT_REF_LIST(list, n, compare_func) \
1390     qsort(list, n, sizeof(*(list)), compare_picture_##compare_func)
1391
1392 static void
1393 init_picture_refs_fields_1(
1394     guint                picture_structure,
1395     GstVaapiPictureH264 *RefPicList[32],
1396     guint               *RefPicList_count,
1397     GstVaapiPictureH264 *ref_list[32],
1398     guint                ref_list_count
1399 )
1400 {
1401     guint i, j, n;
1402
1403     i = 0;
1404     j = 0;
1405     n = *RefPicList_count;
1406     do {
1407         g_assert(n < 32);
1408         for (; i < ref_list_count; i++) {
1409             if (ref_list[i]->structure == picture_structure) {
1410                 RefPicList[n++] = ref_list[i++];
1411                 break;
1412             }
1413         }
1414         for (; j < ref_list_count; j++) {
1415             if (ref_list[j]->structure != picture_structure) {
1416                 RefPicList[n++] = ref_list[j++];
1417                 break;
1418             }
1419         }
1420     } while (i < ref_list_count || j < ref_list_count);
1421     *RefPicList_count = n;
1422 }
1423
1424 static inline void
1425 init_picture_refs_fields(
1426     GstVaapiPictureH264 *picture,
1427     GstVaapiPictureH264 *RefPicList[32],
1428     guint               *RefPicList_count,
1429     GstVaapiPictureH264 *short_ref[32],
1430     guint                short_ref_count,
1431     GstVaapiPictureH264 *long_ref[32],
1432     guint                long_ref_count
1433 )
1434 {
1435     guint n = 0;
1436
1437     /* 8.2.4.2.5 - reference picture lists in fields */
1438     init_picture_refs_fields_1(picture->structure, RefPicList, &n,
1439         short_ref, short_ref_count);
1440     init_picture_refs_fields_1(picture->structure, RefPicList, &n,
1441         long_ref, long_ref_count);
1442     *RefPicList_count = n;
1443 }
1444
1445 static void
1446 init_picture_refs_p_slice(
1447     GstVaapiDecoderH264 *decoder,
1448     GstVaapiPictureH264 *picture,
1449     GstH264SliceHdr     *slice_hdr
1450 )
1451 {
1452     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1453     GstVaapiPictureH264 **ref_list;
1454     guint i;
1455
1456     GST_DEBUG("decode reference picture list for P and SP slices");
1457
1458     if (GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1459         /* 8.2.4.2.1 - P and SP slices in frames */
1460         if (priv->short_ref_count > 0) {
1461             ref_list = priv->RefPicList0;
1462             for (i = 0; i < priv->short_ref_count; i++)
1463                 ref_list[i] = priv->short_ref[i];
1464             SORT_REF_LIST(ref_list, i, pic_num_dec);
1465             priv->RefPicList0_count += i;
1466         }
1467
1468         if (priv->long_ref_count > 0) {
1469             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1470             for (i = 0; i < priv->long_ref_count; i++)
1471                 ref_list[i] = priv->long_ref[i];
1472             SORT_REF_LIST(ref_list, i, long_term_pic_num_inc);
1473             priv->RefPicList0_count += i;
1474         }
1475     }
1476     else {
1477         /* 8.2.4.2.2 - P and SP slices in fields */
1478         GstVaapiPictureH264 *short_ref[32];
1479         guint short_ref_count = 0;
1480         GstVaapiPictureH264 *long_ref[32];
1481         guint long_ref_count = 0;
1482
1483         if (priv->short_ref_count > 0) {
1484             for (i = 0; i < priv->short_ref_count; i++)
1485                 short_ref[i] = priv->short_ref[i];
1486             SORT_REF_LIST(short_ref, i, frame_num_wrap_dec);
1487             short_ref_count = i;
1488         }
1489
1490         if (priv->long_ref_count > 0) {
1491             for (i = 0; i < priv->long_ref_count; i++)
1492                 long_ref[i] = priv->long_ref[i];
1493             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1494             long_ref_count = i;
1495         }
1496
1497         init_picture_refs_fields(
1498             picture,
1499             priv->RefPicList0, &priv->RefPicList0_count,
1500             short_ref,          short_ref_count,
1501             long_ref,           long_ref_count
1502         );
1503     }
1504 }
1505
1506 static void
1507 init_picture_refs_b_slice(
1508     GstVaapiDecoderH264 *decoder,
1509     GstVaapiPictureH264 *picture,
1510     GstH264SliceHdr     *slice_hdr
1511 )
1512 {
1513     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1514     GstVaapiPictureH264 **ref_list;
1515     guint i, n;
1516
1517     GST_DEBUG("decode reference picture list for B slices");
1518
1519     if (GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1520         /* 8.2.4.2.3 - B slices in frames */
1521
1522         /* RefPicList0 */
1523         if (priv->short_ref_count > 0) {
1524             // 1. Short-term references
1525             ref_list = priv->RefPicList0;
1526             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1527                 if (priv->short_ref[i]->base.poc < picture->base.poc)
1528                     ref_list[n++] = priv->short_ref[i];
1529             }
1530             SORT_REF_LIST(ref_list, n, poc_dec);
1531             priv->RefPicList0_count += n;
1532
1533             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1534             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1535                 if (priv->short_ref[i]->base.poc >= picture->base.poc)
1536                     ref_list[n++] = priv->short_ref[i];
1537             }
1538             SORT_REF_LIST(ref_list, n, poc_inc);
1539             priv->RefPicList0_count += n;
1540         }
1541
1542         if (priv->long_ref_count > 0) {
1543             // 2. Long-term references
1544             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1545             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1546                 ref_list[n++] = priv->long_ref[i];
1547             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1548             priv->RefPicList0_count += n;
1549         }
1550
1551         /* RefPicList1 */
1552         if (priv->short_ref_count > 0) {
1553             // 1. Short-term references
1554             ref_list = priv->RefPicList1;
1555             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1556                 if (priv->short_ref[i]->base.poc > picture->base.poc)
1557                     ref_list[n++] = priv->short_ref[i];
1558             }
1559             SORT_REF_LIST(ref_list, n, poc_inc);
1560             priv->RefPicList1_count += n;
1561
1562             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1563             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1564                 if (priv->short_ref[i]->base.poc <= picture->base.poc)
1565                     ref_list[n++] = priv->short_ref[i];
1566             }
1567             SORT_REF_LIST(ref_list, n, poc_dec);
1568             priv->RefPicList1_count += n;
1569         }
1570
1571         if (priv->long_ref_count > 0) {
1572             // 2. Long-term references
1573             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1574             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1575                 ref_list[n++] = priv->long_ref[i];
1576             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1577             priv->RefPicList1_count += n;
1578         }
1579     }
1580     else {
1581         /* 8.2.4.2.4 - B slices in fields */
1582         GstVaapiPictureH264 *short_ref0[32];
1583         guint short_ref0_count = 0;
1584         GstVaapiPictureH264 *short_ref1[32];
1585         guint short_ref1_count = 0;
1586         GstVaapiPictureH264 *long_ref[32];
1587         guint long_ref_count = 0;
1588
1589         /* refFrameList0ShortTerm */
1590         if (priv->short_ref_count > 0) {
1591             ref_list = short_ref0;
1592             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1593                 if (priv->short_ref[i]->base.poc <= picture->base.poc)
1594                     ref_list[n++] = priv->short_ref[i];
1595             }
1596             SORT_REF_LIST(ref_list, n, poc_dec);
1597             short_ref0_count += n;
1598
1599             ref_list = &short_ref0[short_ref0_count];
1600             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1601                 if (priv->short_ref[i]->base.poc > picture->base.poc)
1602                     ref_list[n++] = priv->short_ref[i];
1603             }
1604             SORT_REF_LIST(ref_list, n, poc_inc);
1605             short_ref0_count += n;
1606         }
1607
1608         /* refFrameList1ShortTerm */
1609         if (priv->short_ref_count > 0) {
1610             ref_list = short_ref1;
1611             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1612                 if (priv->short_ref[i]->base.poc > picture->base.poc)
1613                     ref_list[n++] = priv->short_ref[i];
1614             }
1615             SORT_REF_LIST(ref_list, n, poc_inc);
1616             short_ref1_count += n;
1617
1618             ref_list = &short_ref1[short_ref1_count];
1619             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1620                 if (priv->short_ref[i]->base.poc <= picture->base.poc)
1621                     ref_list[n++] = priv->short_ref[i];
1622             }
1623             SORT_REF_LIST(ref_list, n, poc_dec);
1624             short_ref1_count += n;
1625         }
1626
1627         /* refFrameListLongTerm */
1628         if (priv->long_ref_count > 0) {
1629             for (i = 0; i < priv->long_ref_count; i++)
1630                 long_ref[i] = priv->long_ref[i];
1631             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1632             long_ref_count = i;
1633         }
1634
1635         init_picture_refs_fields(
1636             picture,
1637             priv->RefPicList0, &priv->RefPicList0_count,
1638             short_ref0,         short_ref0_count,
1639             long_ref,           long_ref_count
1640         );
1641
1642         init_picture_refs_fields(
1643             picture,
1644             priv->RefPicList1, &priv->RefPicList1_count,
1645             short_ref1,         short_ref1_count,
1646             long_ref,           long_ref_count
1647         );
1648    }
1649
1650     /* Check whether RefPicList1 is identical to RefPicList0, then
1651        swap if necessary */
1652     if (priv->RefPicList1_count > 1 &&
1653         priv->RefPicList1_count == priv->RefPicList0_count &&
1654         memcmp(priv->RefPicList0, priv->RefPicList1,
1655                priv->RefPicList0_count * sizeof(priv->RefPicList0[0])) == 0) {
1656         GstVaapiPictureH264 * const tmp = priv->RefPicList1[0];
1657         priv->RefPicList1[0] = priv->RefPicList1[1];
1658         priv->RefPicList1[1] = tmp;
1659     }
1660 }
1661
1662 #undef SORT_REF_LIST
1663
1664 static gint
1665 find_short_term_reference(GstVaapiDecoderH264 *decoder, gint32 pic_num)
1666 {
1667     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1668     guint i;
1669
1670     for (i = 0; i < priv->short_ref_count; i++) {
1671         if (priv->short_ref[i]->pic_num == pic_num)
1672             return i;
1673     }
1674     GST_ERROR("found no short-term reference picture with PicNum = %d",
1675               pic_num);
1676     return -1;
1677 }
1678
1679 static gint
1680 find_long_term_reference(GstVaapiDecoderH264 *decoder, gint32 long_term_pic_num)
1681 {
1682     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1683     guint i;
1684
1685     for (i = 0; i < priv->long_ref_count; i++) {
1686         if (priv->long_ref[i]->long_term_pic_num == long_term_pic_num)
1687             return i;
1688     }
1689     GST_ERROR("found no long-term reference picture with LongTermPicNum = %d",
1690               long_term_pic_num);
1691     return -1;
1692 }
1693
1694 static void
1695 exec_picture_refs_modification_1(
1696     GstVaapiDecoderH264           *decoder,
1697     GstVaapiPictureH264           *picture,
1698     GstH264SliceHdr               *slice_hdr,
1699     guint                          list
1700 )
1701 {
1702     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1703     GstH264PPS * const pps = slice_hdr->pps;
1704     GstH264SPS * const sps = pps->sequence;
1705     GstH264RefPicListModification *ref_pic_list_modification;
1706     guint num_ref_pic_list_modifications;
1707     GstVaapiPictureH264 **ref_list;
1708     guint *ref_list_count_ptr, ref_list_count, ref_list_idx = 0;
1709     guint i, j, n, num_refs;
1710     gint found_ref_idx;
1711     gint32 MaxPicNum, CurrPicNum, picNumPred;
1712
1713     GST_DEBUG("modification process of reference picture list %u", list);
1714
1715     if (list == 0) {
1716         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l0;
1717         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l0;
1718         ref_list                       = priv->RefPicList0;
1719         ref_list_count_ptr             = &priv->RefPicList0_count;
1720         num_refs                       = slice_hdr->num_ref_idx_l0_active_minus1 + 1;
1721     }
1722     else {
1723         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l1;
1724         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l1;
1725         ref_list                       = priv->RefPicList1;
1726         ref_list_count_ptr             = &priv->RefPicList1_count;
1727         num_refs                       = slice_hdr->num_ref_idx_l1_active_minus1 + 1;
1728     }
1729     ref_list_count = *ref_list_count_ptr;
1730
1731     if (!GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1732         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 5); // 2 * MaxFrameNum
1733         CurrPicNum = 2 * slice_hdr->frame_num + 1;              // 2 * frame_num + 1
1734     }
1735     else {
1736         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 4); // MaxFrameNum
1737         CurrPicNum = slice_hdr->frame_num;                      // frame_num
1738     }
1739
1740     picNumPred = CurrPicNum;
1741
1742     for (i = 0; i < num_ref_pic_list_modifications; i++) {
1743         GstH264RefPicListModification * const l = &ref_pic_list_modification[i];
1744         if (l->modification_of_pic_nums_idc == 3)
1745             break;
1746
1747         /* 8.2.4.3.1 - Short-term reference pictures */
1748         if (l->modification_of_pic_nums_idc == 0 || l->modification_of_pic_nums_idc == 1) {
1749             gint32 abs_diff_pic_num = l->value.abs_diff_pic_num_minus1 + 1;
1750             gint32 picNum, picNumNoWrap;
1751
1752             // (8-34)
1753             if (l->modification_of_pic_nums_idc == 0) {
1754                 picNumNoWrap = picNumPred - abs_diff_pic_num;
1755                 if (picNumNoWrap < 0)
1756                     picNumNoWrap += MaxPicNum;
1757             }
1758
1759             // (8-35)
1760             else {
1761                 picNumNoWrap = picNumPred + abs_diff_pic_num;
1762                 if (picNumNoWrap >= MaxPicNum)
1763                     picNumNoWrap -= MaxPicNum;
1764             }
1765             picNumPred = picNumNoWrap;
1766
1767             // (8-36)
1768             picNum = picNumNoWrap;
1769             if (picNum > CurrPicNum)
1770                 picNum -= MaxPicNum;
1771
1772             // (8-37)
1773             for (j = num_refs; j > ref_list_idx; j--)
1774                 ref_list[j] = ref_list[j - 1];
1775             found_ref_idx = find_short_term_reference(decoder, picNum);
1776             ref_list[ref_list_idx++] =
1777                 found_ref_idx >= 0 ? priv->short_ref[found_ref_idx] : NULL;
1778             n = ref_list_idx;
1779             for (j = ref_list_idx; j <= num_refs; j++) {
1780                 gint32 PicNumF;
1781                 if (!ref_list[j])
1782                     continue;
1783                 PicNumF =
1784                     GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(ref_list[j]) ?
1785                     ref_list[j]->pic_num : MaxPicNum;
1786                 if (PicNumF != picNum)
1787                     ref_list[n++] = ref_list[j];
1788             }
1789         }
1790
1791         /* 8.2.4.3.2 - Long-term reference pictures */
1792         else {
1793
1794             for (j = num_refs; j > ref_list_idx; j--)
1795                 ref_list[j] = ref_list[j - 1];
1796             found_ref_idx =
1797                 find_long_term_reference(decoder, l->value.long_term_pic_num);
1798             ref_list[ref_list_idx++] =
1799                 found_ref_idx >= 0 ? priv->long_ref[found_ref_idx] : NULL;
1800             n = ref_list_idx;
1801             for (j = ref_list_idx; j <= num_refs; j++) {
1802                 gint32 LongTermPicNumF;
1803                 if (!ref_list[j])
1804                     continue;
1805                 LongTermPicNumF =
1806                     GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(ref_list[j]) ?
1807                     ref_list[j]->long_term_pic_num : INT_MAX;
1808                 if (LongTermPicNumF != l->value.long_term_pic_num)
1809                     ref_list[n++] = ref_list[j];
1810             }
1811         }
1812     }
1813
1814 #if DEBUG
1815     for (i = 0; i < num_refs; i++)
1816         if (!ref_list[i])
1817             GST_ERROR("list %u entry %u is empty", list, i);
1818 #endif
1819     *ref_list_count_ptr = num_refs;
1820 }
1821
1822 /* 8.2.4.3 - Modification process for reference picture lists */
1823 static void
1824 exec_picture_refs_modification(
1825     GstVaapiDecoderH264 *decoder,
1826     GstVaapiPictureH264 *picture,
1827     GstH264SliceHdr     *slice_hdr
1828 )
1829 {
1830     GST_DEBUG("execute ref_pic_list_modification()");
1831
1832     /* RefPicList0 */
1833     if (!GST_H264_IS_I_SLICE(slice_hdr) && !GST_H264_IS_SI_SLICE(slice_hdr) &&
1834         slice_hdr->ref_pic_list_modification_flag_l0)
1835         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 0);
1836
1837     /* RefPicList1 */
1838     if (GST_H264_IS_B_SLICE(slice_hdr) &&
1839         slice_hdr->ref_pic_list_modification_flag_l1)
1840         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 1);
1841 }
1842
1843 static void
1844 init_picture_ref_lists(GstVaapiDecoderH264 *decoder)
1845 {
1846     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1847     guint i, j, short_ref_count, long_ref_count;
1848
1849     short_ref_count = 0;
1850     long_ref_count  = 0;
1851     if (GST_VAAPI_PICTURE_IS_FRAME(priv->current_picture)) {
1852         for (i = 0; i < priv->dpb_count; i++) {
1853             GstVaapiFrameStore * const fs = priv->dpb[i];
1854             GstVaapiPictureH264 *picture;
1855             if (!gst_vaapi_frame_store_has_frame(fs))
1856                 continue;
1857             picture = fs->buffers[0];
1858             if (GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture))
1859                 priv->short_ref[short_ref_count++] = picture;
1860             else if (GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture))
1861                 priv->long_ref[long_ref_count++] = picture;
1862             picture->structure = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
1863             picture->other_field = fs->buffers[1];
1864         }
1865     }
1866     else {
1867         for (i = 0; i < priv->dpb_count; i++) {
1868             GstVaapiFrameStore * const fs = priv->dpb[i];
1869             for (j = 0; j < fs->num_buffers; j++) {
1870                 GstVaapiPictureH264 * const picture = fs->buffers[j];
1871                 if (GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture))
1872                     priv->short_ref[short_ref_count++] = picture;
1873                 else if (GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture))
1874                     priv->long_ref[long_ref_count++] = picture;
1875                 picture->structure = picture->base.structure;
1876                 picture->other_field = fs->buffers[j ^ 1];
1877             }
1878         }
1879     }
1880
1881     for (i = short_ref_count; i < priv->short_ref_count; i++)
1882         priv->short_ref[i] = NULL;
1883     priv->short_ref_count = short_ref_count;
1884
1885     for (i = long_ref_count; i < priv->long_ref_count; i++)
1886         priv->long_ref[i] = NULL;
1887     priv->long_ref_count = long_ref_count;
1888 }
1889
1890 static void
1891 init_picture_refs(
1892     GstVaapiDecoderH264 *decoder,
1893     GstVaapiPictureH264 *picture,
1894     GstH264SliceHdr     *slice_hdr
1895 )
1896 {
1897     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1898     GstVaapiPicture * const base_picture = &picture->base;
1899     guint i, num_refs;
1900
1901     init_picture_ref_lists(decoder);
1902     init_picture_refs_pic_num(decoder, picture, slice_hdr);
1903
1904     priv->RefPicList0_count = 0;
1905     priv->RefPicList1_count = 0;
1906
1907     switch (base_picture->type) {
1908     case GST_VAAPI_PICTURE_TYPE_P:
1909     case GST_VAAPI_PICTURE_TYPE_SP:
1910         init_picture_refs_p_slice(decoder, picture, slice_hdr);
1911         break;
1912     case GST_VAAPI_PICTURE_TYPE_B:
1913         init_picture_refs_b_slice(decoder, picture, slice_hdr);
1914         break;
1915     default:
1916         break;
1917     }
1918
1919     exec_picture_refs_modification(decoder, picture, slice_hdr);
1920
1921     switch (base_picture->type) {
1922     case GST_VAAPI_PICTURE_TYPE_B:
1923         num_refs = 1 + slice_hdr->num_ref_idx_l1_active_minus1;
1924         for (i = priv->RefPicList1_count; i < num_refs; i++)
1925             priv->RefPicList1[i] = NULL;
1926         priv->RefPicList1_count = num_refs;
1927
1928         // fall-through
1929     case GST_VAAPI_PICTURE_TYPE_P:
1930     case GST_VAAPI_PICTURE_TYPE_SP:
1931         num_refs = 1 + slice_hdr->num_ref_idx_l0_active_minus1;
1932         for (i = priv->RefPicList0_count; i < num_refs; i++)
1933             priv->RefPicList0[i] = NULL;
1934         priv->RefPicList0_count = num_refs;
1935         break;
1936     default:
1937         break;
1938     }
1939 }
1940
1941 static gboolean
1942 init_picture(
1943     GstVaapiDecoderH264 *decoder,
1944     GstVaapiPictureH264 *picture, GstVaapiParserInfoH264 *pi)
1945 {
1946     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1947     GstVaapiPicture * const base_picture = &picture->base;
1948     GstH264SliceHdr * const slice_hdr = &pi->data.slice_hdr;
1949
1950     priv->prev_frame_num        = priv->frame_num;
1951     priv->frame_num             = slice_hdr->frame_num;
1952     picture->frame_num          = priv->frame_num;
1953     picture->frame_num_wrap     = priv->frame_num;
1954     picture->output_flag        = TRUE; /* XXX: conformant to Annex A only */
1955     base_picture->pts           = GST_VAAPI_DECODER_CODEC_FRAME(decoder)->pts;
1956
1957     /* Reset decoder state for IDR pictures */
1958     if (pi->nalu.type == GST_H264_NAL_SLICE_IDR) {
1959         GST_DEBUG("<IDR>");
1960         GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_IDR);
1961         dpb_flush(decoder);
1962     }
1963
1964     /* Initialize slice type */
1965     switch (slice_hdr->type % 5) {
1966     case GST_H264_P_SLICE:
1967         base_picture->type = GST_VAAPI_PICTURE_TYPE_P;
1968         break;
1969     case GST_H264_B_SLICE:
1970         base_picture->type = GST_VAAPI_PICTURE_TYPE_B;
1971         break;
1972     case GST_H264_I_SLICE:
1973         base_picture->type = GST_VAAPI_PICTURE_TYPE_I;
1974         break;
1975     case GST_H264_SP_SLICE:
1976         base_picture->type = GST_VAAPI_PICTURE_TYPE_SP;
1977         break;
1978     case GST_H264_SI_SLICE:
1979         base_picture->type = GST_VAAPI_PICTURE_TYPE_SI;
1980         break;
1981     }
1982
1983     /* Initialize picture structure */
1984     if (!slice_hdr->field_pic_flag)
1985         base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
1986     else {
1987         GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_INTERLACED);
1988         if (!slice_hdr->bottom_field_flag)
1989             base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD;
1990         else
1991             base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD;
1992     }
1993     picture->structure = base_picture->structure;
1994
1995     /* Initialize reference flags */
1996     if (pi->nalu.ref_idc) {
1997         GstH264DecRefPicMarking * const dec_ref_pic_marking =
1998             &slice_hdr->dec_ref_pic_marking;
1999
2000         if (GST_VAAPI_PICTURE_IS_IDR(picture) &&
2001             dec_ref_pic_marking->long_term_reference_flag)
2002             GST_VAAPI_PICTURE_FLAG_SET(picture,
2003                 GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE);
2004         else
2005             GST_VAAPI_PICTURE_FLAG_SET(picture,
2006                 GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE);
2007     }
2008
2009     init_picture_poc(decoder, picture, slice_hdr);
2010     init_picture_refs(decoder, picture, slice_hdr);
2011     return TRUE;
2012 }
2013
2014 /* 8.2.5.3 - Sliding window decoded reference picture marking process */
2015 static gboolean
2016 exec_ref_pic_marking_sliding_window(GstVaapiDecoderH264 *decoder)
2017 {
2018     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2019     GstH264PPS * const pps = priv->current_picture->pps;
2020     GstH264SPS * const sps = pps->sequence;
2021     GstVaapiPictureH264 *ref_picture;
2022     guint i, m, max_num_ref_frames;
2023
2024     GST_DEBUG("reference picture marking process (sliding window)");
2025
2026     if (!GST_VAAPI_PICTURE_IS_FIRST_FIELD(priv->current_picture))
2027         return TRUE;
2028
2029     max_num_ref_frames = sps->num_ref_frames;
2030     if (max_num_ref_frames == 0)
2031         max_num_ref_frames = 1;
2032     if (!GST_VAAPI_PICTURE_IS_FRAME(priv->current_picture))
2033         max_num_ref_frames <<= 1;
2034
2035     if (priv->short_ref_count + priv->long_ref_count < max_num_ref_frames)
2036         return TRUE;
2037     if (priv->short_ref_count < 1)
2038         return FALSE;
2039
2040     for (m = 0, i = 1; i < priv->short_ref_count; i++) {
2041         GstVaapiPictureH264 * const picture = priv->short_ref[i];
2042         if (picture->frame_num_wrap < priv->short_ref[m]->frame_num_wrap)
2043             m = i;
2044     }
2045
2046     ref_picture = priv->short_ref[m];
2047     gst_vaapi_picture_h264_set_reference(ref_picture, 0, TRUE);
2048     ARRAY_REMOVE_INDEX(priv->short_ref, m);
2049
2050     /* Both fields need to be marked as "unused for reference", so
2051        remove the other field from the short_ref[] list as well */
2052     if (!GST_VAAPI_PICTURE_IS_FRAME(priv->current_picture) && ref_picture->other_field) {
2053         for (i = 0; i < priv->short_ref_count; i++) {
2054             if (priv->short_ref[i] == ref_picture->other_field) {
2055                 ARRAY_REMOVE_INDEX(priv->short_ref, i);
2056                 break;
2057             }
2058         }
2059     }
2060     return TRUE;
2061 }
2062
2063 static inline gint32
2064 get_picNumX(GstVaapiPictureH264 *picture, GstH264RefPicMarking *ref_pic_marking)
2065 {
2066     gint32 pic_num;
2067
2068     if (GST_VAAPI_PICTURE_IS_FRAME(picture))
2069         pic_num = picture->frame_num_wrap;
2070     else
2071         pic_num = 2 * picture->frame_num_wrap + 1;
2072     pic_num -= ref_pic_marking->difference_of_pic_nums_minus1 + 1;
2073     return pic_num;
2074 }
2075
2076 /* 8.2.5.4.1. Mark short-term reference picture as "unused for reference" */
2077 static void
2078 exec_ref_pic_marking_adaptive_mmco_1(
2079     GstVaapiDecoderH264  *decoder,
2080     GstVaapiPictureH264  *picture,
2081     GstH264RefPicMarking *ref_pic_marking
2082 )
2083 {
2084     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2085     gint32 i, picNumX;
2086
2087     picNumX = get_picNumX(picture, ref_pic_marking);
2088     i = find_short_term_reference(decoder, picNumX);
2089     if (i < 0)
2090         return;
2091
2092     gst_vaapi_picture_h264_set_reference(priv->short_ref[i], 0,
2093         GST_VAAPI_PICTURE_IS_FRAME(picture));
2094     ARRAY_REMOVE_INDEX(priv->short_ref, i);
2095 }
2096
2097 /* 8.2.5.4.2. Mark long-term reference picture as "unused for reference" */
2098 static void
2099 exec_ref_pic_marking_adaptive_mmco_2(
2100     GstVaapiDecoderH264  *decoder,
2101     GstVaapiPictureH264  *picture,
2102     GstH264RefPicMarking *ref_pic_marking
2103 )
2104 {
2105     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2106     gint32 i;
2107
2108     i = find_long_term_reference(decoder, ref_pic_marking->long_term_pic_num);
2109     if (i < 0)
2110         return;
2111
2112     gst_vaapi_picture_h264_set_reference(priv->long_ref[i], 0,
2113         GST_VAAPI_PICTURE_IS_FRAME(picture));
2114     ARRAY_REMOVE_INDEX(priv->long_ref, i);
2115 }
2116
2117 /* 8.2.5.4.3. Assign LongTermFrameIdx to a short-term reference picture */
2118 static void
2119 exec_ref_pic_marking_adaptive_mmco_3(
2120     GstVaapiDecoderH264  *decoder,
2121     GstVaapiPictureH264  *picture,
2122     GstH264RefPicMarking *ref_pic_marking
2123 )
2124 {
2125     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2126     GstVaapiPictureH264 *ref_picture;
2127     gint32 i, picNumX;
2128
2129     for (i = 0; i < priv->long_ref_count; i++) {
2130         if (priv->long_ref[i]->long_term_frame_idx == ref_pic_marking->long_term_frame_idx)
2131             break;
2132     }
2133     if (i != priv->long_ref_count) {
2134         gst_vaapi_picture_h264_set_reference(priv->long_ref[i], 0, TRUE);
2135         ARRAY_REMOVE_INDEX(priv->long_ref, i);
2136     }
2137
2138     picNumX = get_picNumX(picture, ref_pic_marking);
2139     i = find_short_term_reference(decoder, picNumX);
2140     if (i < 0)
2141         return;
2142
2143     ref_picture = priv->short_ref[i];
2144     ARRAY_REMOVE_INDEX(priv->short_ref, i);
2145     priv->long_ref[priv->long_ref_count++] = ref_picture;
2146
2147     ref_picture->long_term_frame_idx = ref_pic_marking->long_term_frame_idx;
2148     gst_vaapi_picture_h264_set_reference(ref_picture,
2149         GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE,
2150         GST_VAAPI_PICTURE_IS_FRAME(picture));
2151 }
2152
2153 /* 8.2.5.4.4. Mark pictures with LongTermFramIdx > max_long_term_frame_idx
2154  * as "unused for reference" */
2155 static void
2156 exec_ref_pic_marking_adaptive_mmco_4(
2157     GstVaapiDecoderH264  *decoder,
2158     GstVaapiPictureH264  *picture,
2159     GstH264RefPicMarking *ref_pic_marking
2160 )
2161 {
2162     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2163     gint32 i, long_term_frame_idx;
2164
2165     long_term_frame_idx = ref_pic_marking->max_long_term_frame_idx_plus1 - 1;
2166
2167     for (i = 0; i < priv->long_ref_count; i++) {
2168         if (priv->long_ref[i]->long_term_frame_idx <= long_term_frame_idx)
2169             continue;
2170         gst_vaapi_picture_h264_set_reference(priv->long_ref[i], 0, FALSE);
2171         ARRAY_REMOVE_INDEX(priv->long_ref, i);
2172         i--;
2173     }
2174 }
2175
2176 /* 8.2.5.4.5. Mark all reference pictures as "unused for reference" */
2177 static void
2178 exec_ref_pic_marking_adaptive_mmco_5(
2179     GstVaapiDecoderH264  *decoder,
2180     GstVaapiPictureH264  *picture,
2181     GstH264RefPicMarking *ref_pic_marking
2182 )
2183 {
2184     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2185
2186     dpb_flush(decoder);
2187
2188     priv->prev_pic_has_mmco5 = TRUE;
2189
2190     /* The picture shall be inferred to have had frame_num equal to 0 (7.4.3) */
2191     priv->frame_num = 0;
2192     priv->frame_num_offset = 0;
2193     picture->frame_num = 0;
2194
2195     /* Update TopFieldOrderCnt and BottomFieldOrderCnt (8.2.1) */
2196     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
2197         picture->field_poc[TOP_FIELD] -= picture->base.poc;
2198     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
2199         picture->field_poc[BOTTOM_FIELD] -= picture->base.poc;
2200     picture->base.poc = 0;
2201 }
2202
2203 /* 8.2.5.4.6. Assign a long-term frame index to the current picture */
2204 static void
2205 exec_ref_pic_marking_adaptive_mmco_6(
2206     GstVaapiDecoderH264  *decoder,
2207     GstVaapiPictureH264  *picture,
2208     GstH264RefPicMarking *ref_pic_marking
2209 )
2210 {
2211     picture->long_term_frame_idx = ref_pic_marking->long_term_frame_idx;
2212     gst_vaapi_picture_h264_set_reference(picture,
2213         GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE, FALSE);
2214 }
2215
2216 /* 8.2.5.4. Adaptive memory control decoded reference picture marking process */
2217 static gboolean
2218 exec_ref_pic_marking_adaptive(
2219     GstVaapiDecoderH264     *decoder,
2220     GstVaapiPictureH264     *picture,
2221     GstH264DecRefPicMarking *dec_ref_pic_marking
2222 )
2223 {
2224     guint i;
2225
2226     GST_DEBUG("reference picture marking process (adaptive memory control)");
2227
2228     typedef void (*exec_ref_pic_marking_adaptive_mmco_func)(
2229         GstVaapiDecoderH264  *decoder,
2230         GstVaapiPictureH264  *picture,
2231         GstH264RefPicMarking *ref_pic_marking
2232     );
2233
2234     static const exec_ref_pic_marking_adaptive_mmco_func mmco_funcs[] = {
2235         NULL,
2236         exec_ref_pic_marking_adaptive_mmco_1,
2237         exec_ref_pic_marking_adaptive_mmco_2,
2238         exec_ref_pic_marking_adaptive_mmco_3,
2239         exec_ref_pic_marking_adaptive_mmco_4,
2240         exec_ref_pic_marking_adaptive_mmco_5,
2241         exec_ref_pic_marking_adaptive_mmco_6,
2242     };
2243
2244     for (i = 0; i < dec_ref_pic_marking->n_ref_pic_marking; i++) {
2245         GstH264RefPicMarking * const ref_pic_marking =
2246             &dec_ref_pic_marking->ref_pic_marking[i];
2247
2248         const guint mmco = ref_pic_marking->memory_management_control_operation;
2249         if (mmco < G_N_ELEMENTS(mmco_funcs) && mmco_funcs[mmco])
2250             mmco_funcs[mmco](decoder, picture, ref_pic_marking);
2251         else {
2252             GST_ERROR("unhandled MMCO %u", mmco);
2253             return FALSE;
2254         }
2255     }
2256     return TRUE;
2257 }
2258
2259 /* 8.2.5 - Execute reference picture marking process */
2260 static gboolean
2261 exec_ref_pic_marking(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
2262 {
2263     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2264
2265     priv->prev_pic_has_mmco5 = FALSE;
2266     priv->prev_pic_structure = picture->structure;
2267
2268     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
2269         return TRUE;
2270
2271     if (!GST_VAAPI_PICTURE_IS_IDR(picture)) {
2272         GstH264DecRefPicMarking * const dec_ref_pic_marking =
2273             &picture->last_slice_hdr->dec_ref_pic_marking;
2274         if (dec_ref_pic_marking->adaptive_ref_pic_marking_mode_flag) {
2275             if (!exec_ref_pic_marking_adaptive(decoder, picture, dec_ref_pic_marking))
2276                 return FALSE;
2277         }
2278         else {
2279             if (!exec_ref_pic_marking_sliding_window(decoder))
2280                 return FALSE;
2281         }
2282     }
2283     return TRUE;
2284 }
2285
2286 static void
2287 vaapi_init_picture(VAPictureH264 *pic)
2288 {
2289     pic->picture_id           = VA_INVALID_ID;
2290     pic->frame_idx            = 0;
2291     pic->flags                = VA_PICTURE_H264_INVALID;
2292     pic->TopFieldOrderCnt     = 0;
2293     pic->BottomFieldOrderCnt  = 0;
2294 }
2295
2296 static void
2297 vaapi_fill_picture(VAPictureH264 *pic, GstVaapiPictureH264 *picture,
2298     guint picture_structure)
2299 {
2300     if (!picture_structure)
2301         picture_structure = picture->structure;
2302
2303     pic->picture_id = picture->base.surface_id;
2304     pic->flags = 0;
2305
2306     if (GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture)) {
2307         pic->flags |= VA_PICTURE_H264_LONG_TERM_REFERENCE;
2308         pic->frame_idx = picture->long_term_frame_idx;
2309     }
2310     else {
2311         if (GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture))
2312             pic->flags |= VA_PICTURE_H264_SHORT_TERM_REFERENCE;
2313         pic->frame_idx = picture->frame_num;
2314     }
2315
2316     switch (picture_structure) {
2317     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
2318         pic->TopFieldOrderCnt = picture->field_poc[TOP_FIELD];
2319         pic->BottomFieldOrderCnt = picture->field_poc[BOTTOM_FIELD];
2320         break;
2321     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
2322         pic->flags |= VA_PICTURE_H264_TOP_FIELD;
2323         pic->TopFieldOrderCnt = picture->field_poc[TOP_FIELD];
2324         pic->BottomFieldOrderCnt = 0;
2325         break;
2326     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
2327         pic->flags |= VA_PICTURE_H264_BOTTOM_FIELD;
2328         pic->BottomFieldOrderCnt = picture->field_poc[BOTTOM_FIELD];
2329         pic->TopFieldOrderCnt = 0;
2330         break;
2331     }
2332 }
2333
2334 static gboolean
2335 fill_picture(GstVaapiDecoderH264 *decoder,
2336     GstVaapiPictureH264 *picture, GstVaapiParserInfoH264 *pi)
2337 {
2338     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2339     GstVaapiPicture * const base_picture = &picture->base;
2340     GstH264SliceHdr * const slice_hdr = &pi->data.slice_hdr;
2341     GstH264PPS * const pps = picture->pps;
2342     GstH264SPS * const sps = pps->sequence;
2343     VAPictureParameterBufferH264 * const pic_param = base_picture->param;
2344     guint i, n;
2345
2346     /* Fill in VAPictureParameterBufferH264 */
2347     vaapi_fill_picture(&pic_param->CurrPic, picture, 0);
2348
2349     for (i = 0, n = 0; i < priv->dpb_count; i++) {
2350         GstVaapiFrameStore * const fs = priv->dpb[i];
2351         if (gst_vaapi_frame_store_has_reference(fs))
2352             vaapi_fill_picture(&pic_param->ReferenceFrames[n++],
2353                 fs->buffers[0], fs->structure);
2354     }
2355     for (; n < G_N_ELEMENTS(pic_param->ReferenceFrames); n++)
2356         vaapi_init_picture(&pic_param->ReferenceFrames[n]);
2357
2358 #define COPY_FIELD(s, f) \
2359     pic_param->f = (s)->f
2360
2361 #define COPY_BFM(a, s, f) \
2362     pic_param->a.bits.f = (s)->f
2363
2364     pic_param->picture_width_in_mbs_minus1  = priv->mb_width - 1;
2365     pic_param->picture_height_in_mbs_minus1 = priv->mb_height - 1;
2366     pic_param->frame_num                    = priv->frame_num;
2367
2368     COPY_FIELD(sps, bit_depth_luma_minus8);
2369     COPY_FIELD(sps, bit_depth_chroma_minus8);
2370     COPY_FIELD(sps, num_ref_frames);
2371     COPY_FIELD(pps, num_slice_groups_minus1);
2372     COPY_FIELD(pps, slice_group_map_type);
2373     COPY_FIELD(pps, slice_group_change_rate_minus1);
2374     COPY_FIELD(pps, pic_init_qp_minus26);
2375     COPY_FIELD(pps, pic_init_qs_minus26);
2376     COPY_FIELD(pps, chroma_qp_index_offset);
2377     COPY_FIELD(pps, second_chroma_qp_index_offset);
2378
2379     pic_param->seq_fields.value                                         = 0; /* reset all bits */
2380     pic_param->seq_fields.bits.residual_colour_transform_flag           = sps->separate_colour_plane_flag;
2381     pic_param->seq_fields.bits.MinLumaBiPredSize8x8                     = sps->level_idc >= 31; /* A.3.3.2 */
2382
2383     COPY_BFM(seq_fields, sps, chroma_format_idc);
2384     COPY_BFM(seq_fields, sps, gaps_in_frame_num_value_allowed_flag);
2385     COPY_BFM(seq_fields, sps, frame_mbs_only_flag);
2386     COPY_BFM(seq_fields, sps, mb_adaptive_frame_field_flag);
2387     COPY_BFM(seq_fields, sps, direct_8x8_inference_flag);
2388     COPY_BFM(seq_fields, sps, log2_max_frame_num_minus4);
2389     COPY_BFM(seq_fields, sps, pic_order_cnt_type);
2390     COPY_BFM(seq_fields, sps, log2_max_pic_order_cnt_lsb_minus4);
2391     COPY_BFM(seq_fields, sps, delta_pic_order_always_zero_flag);
2392
2393     pic_param->pic_fields.value                                         = 0; /* reset all bits */
2394     pic_param->pic_fields.bits.field_pic_flag                           = slice_hdr->field_pic_flag;
2395     pic_param->pic_fields.bits.reference_pic_flag                       = GST_VAAPI_PICTURE_IS_REFERENCE(picture);
2396
2397     COPY_BFM(pic_fields, pps, entropy_coding_mode_flag);
2398     COPY_BFM(pic_fields, pps, weighted_pred_flag);
2399     COPY_BFM(pic_fields, pps, weighted_bipred_idc);
2400     COPY_BFM(pic_fields, pps, transform_8x8_mode_flag);
2401     COPY_BFM(pic_fields, pps, constrained_intra_pred_flag);
2402     COPY_BFM(pic_fields, pps, pic_order_present_flag);
2403     COPY_BFM(pic_fields, pps, deblocking_filter_control_present_flag);
2404     COPY_BFM(pic_fields, pps, redundant_pic_cnt_present_flag);
2405     return TRUE;
2406 }
2407
2408 /* Detection of the first VCL NAL unit of a primary coded picture (7.4.1.2.4) */
2409 static gboolean
2410 is_new_picture(GstVaapiParserInfoH264 *pi, GstVaapiParserInfoH264 *prev_pi)
2411 {
2412     GstH264SliceHdr * const slice_hdr = &pi->data.slice_hdr;
2413     GstH264PPS * const pps = slice_hdr->pps;
2414     GstH264SPS * const sps = pps->sequence;
2415     GstH264SliceHdr *prev_slice_hdr;
2416
2417     if (!prev_pi)
2418         return TRUE;
2419     prev_slice_hdr = &prev_pi->data.slice_hdr;
2420
2421 #define CHECK_EXPR(expr, field_name) do {              \
2422         if (!(expr)) {                                 \
2423             GST_DEBUG(field_name " differs in value"); \
2424             return TRUE;                               \
2425         }                                              \
2426     } while (0)
2427
2428 #define CHECK_VALUE(new_slice_hdr, old_slice_hdr, field) \
2429     CHECK_EXPR(((new_slice_hdr)->field == (old_slice_hdr)->field), #field)
2430
2431     /* frame_num differs in value, regardless of inferred values to 0 */
2432     CHECK_VALUE(slice_hdr, prev_slice_hdr, frame_num);
2433
2434     /* pic_parameter_set_id differs in value */
2435     CHECK_VALUE(slice_hdr, prev_slice_hdr, pps);
2436
2437     /* field_pic_flag differs in value */
2438     CHECK_VALUE(slice_hdr, prev_slice_hdr, field_pic_flag);
2439
2440     /* bottom_field_flag is present in both and differs in value */
2441     if (slice_hdr->field_pic_flag && prev_slice_hdr->field_pic_flag)
2442         CHECK_VALUE(slice_hdr, prev_slice_hdr, bottom_field_flag);
2443
2444     /* nal_ref_idc differs in value with one of the nal_ref_idc values is 0 */
2445     CHECK_EXPR((pi->nalu.ref_idc != 0) ==
2446                (prev_pi->nalu.ref_idc != 0), "nal_ref_idc");
2447
2448     /* POC type is 0 for both and either pic_order_cnt_lsb differs in
2449        value or delta_pic_order_cnt_bottom differs in value */
2450     if (sps->pic_order_cnt_type == 0) {
2451         CHECK_VALUE(slice_hdr, prev_slice_hdr, pic_order_cnt_lsb);
2452         if (pps->pic_order_present_flag && !slice_hdr->field_pic_flag)
2453             CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt_bottom);
2454     }
2455
2456     /* POC type is 1 for both and either delta_pic_order_cnt[0]
2457        differs in value or delta_pic_order_cnt[1] differs in value */
2458     else if (sps->pic_order_cnt_type == 1) {
2459         CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt[0]);
2460         CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt[1]);
2461     }
2462
2463     /* IdrPicFlag differs in value */
2464     CHECK_VALUE(&pi->nalu, &prev_pi->nalu, idr_pic_flag);
2465
2466     /* IdrPicFlag is equal to 1 for both and idr_pic_id differs in value */
2467     if (pi->nalu.idr_pic_flag)
2468         CHECK_VALUE(slice_hdr, prev_slice_hdr, idr_pic_id);
2469
2470 #undef CHECK_EXPR
2471 #undef CHECK_VALUE
2472     return FALSE;
2473 }
2474
2475 static GstVaapiDecoderStatus
2476 decode_picture(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnit *unit)
2477 {
2478     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2479     GstVaapiParserInfoH264 * const pi = unit->parsed_info;
2480     GstH264SliceHdr * const slice_hdr = &pi->data.slice_hdr;
2481     GstH264PPS * const pps = slice_hdr->pps;
2482     GstH264SPS * const sps = pps->sequence;
2483     GstVaapiPictureH264 *picture;
2484     GstVaapiDecoderStatus status;
2485
2486     status = ensure_context(decoder, sps);
2487     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2488         return status;
2489
2490     if (priv->current_picture) {
2491         /* Re-use current picture where the first field was decoded */
2492         picture = gst_vaapi_picture_h264_new_field(priv->current_picture);
2493         if (!picture) {
2494             GST_ERROR("failed to allocate field picture");
2495             return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2496         }
2497     }
2498     else {
2499         /* Create new picture */
2500         picture = gst_vaapi_picture_h264_new(decoder);
2501         if (!picture) {
2502             GST_ERROR("failed to allocate picture");
2503             return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2504         }
2505     }
2506     gst_vaapi_picture_replace(&priv->current_picture, picture);
2507     gst_vaapi_picture_unref(picture);
2508
2509     /* Update cropping rectangle */
2510     if (sps->frame_cropping_flag) {
2511         GstVaapiRectangle crop_rect;
2512         crop_rect.x = sps->crop_rect_x;
2513         crop_rect.y = sps->crop_rect_y;
2514         crop_rect.width = sps->crop_rect_width;
2515         crop_rect.height = sps->crop_rect_height;
2516         gst_vaapi_picture_set_crop_rect(&picture->base, &crop_rect);
2517     }
2518
2519     picture->pps = pps;
2520
2521     status = ensure_quant_matrix(decoder, picture);
2522     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS) {
2523         GST_ERROR("failed to reset quantizer matrix");
2524         return status;
2525     }
2526
2527     if (!init_picture(decoder, picture, pi))
2528         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2529     if (!fill_picture(decoder, picture, pi))
2530         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2531     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2532 }
2533
2534 static inline guint
2535 get_slice_data_bit_offset(GstH264SliceHdr *slice_hdr)
2536 {
2537     guint epb_count;
2538
2539     epb_count = slice_hdr->n_emulation_prevention_bytes;
2540     return 8 /* nal_unit_type */ + slice_hdr->header_size - epb_count * 8;
2541 }
2542
2543 static gboolean
2544 fill_pred_weight_table(GstVaapiDecoderH264 *decoder,
2545     GstVaapiSlice *slice, GstH264SliceHdr *slice_hdr)
2546 {
2547     VASliceParameterBufferH264 * const slice_param = slice->param;
2548     GstH264PPS * const pps = slice_hdr->pps;
2549     GstH264SPS * const sps = pps->sequence;
2550     GstH264PredWeightTable * const w = &slice_hdr->pred_weight_table;
2551     guint num_weight_tables = 0;
2552     gint i, j;
2553
2554     if (pps->weighted_pred_flag &&
2555         (GST_H264_IS_P_SLICE(slice_hdr) || GST_H264_IS_SP_SLICE(slice_hdr)))
2556         num_weight_tables = 1;
2557     else if (pps->weighted_bipred_idc == 1 && GST_H264_IS_B_SLICE(slice_hdr))
2558         num_weight_tables = 2;
2559     else
2560         num_weight_tables = 0;
2561
2562     slice_param->luma_log2_weight_denom   = 0;
2563     slice_param->chroma_log2_weight_denom = 0;
2564     slice_param->luma_weight_l0_flag      = 0;
2565     slice_param->chroma_weight_l0_flag    = 0;
2566     slice_param->luma_weight_l1_flag      = 0;
2567     slice_param->chroma_weight_l1_flag    = 0;
2568
2569     if (num_weight_tables < 1)
2570         return TRUE;
2571
2572     slice_param->luma_log2_weight_denom   = w->luma_log2_weight_denom;
2573     slice_param->chroma_log2_weight_denom = w->chroma_log2_weight_denom;
2574
2575     slice_param->luma_weight_l0_flag = 1;
2576     for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2577         slice_param->luma_weight_l0[i] = w->luma_weight_l0[i];
2578         slice_param->luma_offset_l0[i] = w->luma_offset_l0[i];
2579     }
2580
2581     slice_param->chroma_weight_l0_flag = sps->chroma_array_type != 0;
2582     if (slice_param->chroma_weight_l0_flag) {
2583         for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2584             for (j = 0; j < 2; j++) {
2585                 slice_param->chroma_weight_l0[i][j] = w->chroma_weight_l0[i][j];
2586                 slice_param->chroma_offset_l0[i][j] = w->chroma_offset_l0[i][j];
2587             }
2588         }
2589     }
2590
2591     if (num_weight_tables < 2)
2592         return TRUE;
2593
2594     slice_param->luma_weight_l1_flag = 1;
2595     for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2596         slice_param->luma_weight_l1[i] = w->luma_weight_l1[i];
2597         slice_param->luma_offset_l1[i] = w->luma_offset_l1[i];
2598     }
2599
2600     slice_param->chroma_weight_l1_flag = sps->chroma_array_type != 0;
2601     if (slice_param->chroma_weight_l1_flag) {
2602         for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2603             for (j = 0; j < 2; j++) {
2604                 slice_param->chroma_weight_l1[i][j] = w->chroma_weight_l1[i][j];
2605                 slice_param->chroma_offset_l1[i][j] = w->chroma_offset_l1[i][j];
2606             }
2607         }
2608     }
2609     return TRUE;
2610 }
2611
2612 static gboolean
2613 fill_RefPicList(GstVaapiDecoderH264 *decoder,
2614     GstVaapiSlice *slice, GstH264SliceHdr *slice_hdr)
2615 {
2616     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2617     VASliceParameterBufferH264 * const slice_param = slice->param;
2618     guint i, num_ref_lists = 0;
2619
2620     slice_param->num_ref_idx_l0_active_minus1 = 0;
2621     slice_param->num_ref_idx_l1_active_minus1 = 0;
2622
2623     if (GST_H264_IS_B_SLICE(slice_hdr))
2624         num_ref_lists = 2;
2625     else if (GST_H264_IS_I_SLICE(slice_hdr))
2626         num_ref_lists = 0;
2627     else
2628         num_ref_lists = 1;
2629
2630     if (num_ref_lists < 1)
2631         return TRUE;
2632
2633     slice_param->num_ref_idx_l0_active_minus1 =
2634         slice_hdr->num_ref_idx_l0_active_minus1;
2635
2636     for (i = 0; i < priv->RefPicList0_count && priv->RefPicList0[i]; i++)
2637         vaapi_fill_picture(&slice_param->RefPicList0[i], priv->RefPicList0[i], 0);
2638     for (; i <= slice_param->num_ref_idx_l0_active_minus1; i++)
2639         vaapi_init_picture(&slice_param->RefPicList0[i]);
2640
2641     if (num_ref_lists < 2)
2642         return TRUE;
2643
2644     slice_param->num_ref_idx_l1_active_minus1 =
2645         slice_hdr->num_ref_idx_l1_active_minus1;
2646
2647     for (i = 0; i < priv->RefPicList1_count && priv->RefPicList1[i]; i++)
2648         vaapi_fill_picture(&slice_param->RefPicList1[i], priv->RefPicList1[i], 0);
2649     for (; i <= slice_param->num_ref_idx_l1_active_minus1; i++)
2650         vaapi_init_picture(&slice_param->RefPicList1[i]);
2651     return TRUE;
2652 }
2653
2654 static gboolean
2655 fill_slice(GstVaapiDecoderH264 *decoder,
2656     GstVaapiSlice *slice, GstH264SliceHdr *slice_hdr)
2657 {
2658     VASliceParameterBufferH264 * const slice_param = slice->param;
2659
2660     /* Fill in VASliceParameterBufferH264 */
2661     slice_param->slice_data_bit_offset          = get_slice_data_bit_offset(slice_hdr);
2662     slice_param->first_mb_in_slice              = slice_hdr->first_mb_in_slice;
2663     slice_param->slice_type                     = slice_hdr->type % 5;
2664     slice_param->direct_spatial_mv_pred_flag    = slice_hdr->direct_spatial_mv_pred_flag;
2665     slice_param->cabac_init_idc                 = slice_hdr->cabac_init_idc;
2666     slice_param->slice_qp_delta                 = slice_hdr->slice_qp_delta;
2667     slice_param->disable_deblocking_filter_idc  = slice_hdr->disable_deblocking_filter_idc;
2668     slice_param->slice_alpha_c0_offset_div2     = slice_hdr->slice_alpha_c0_offset_div2;
2669     slice_param->slice_beta_offset_div2         = slice_hdr->slice_beta_offset_div2;
2670
2671     if (!fill_RefPicList(decoder, slice, slice_hdr))
2672         return FALSE;
2673     if (!fill_pred_weight_table(decoder, slice, slice_hdr))
2674         return FALSE;
2675     return TRUE;
2676 }
2677
2678 static GstVaapiDecoderStatus
2679 decode_slice(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnit *unit)
2680 {
2681     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2682     GstVaapiParserInfoH264 * const pi = unit->parsed_info;
2683     GstVaapiPictureH264 * const picture = priv->current_picture;
2684     GstH264SliceHdr * const slice_hdr = &pi->data.slice_hdr;
2685     GstVaapiSlice *slice;
2686     GstBuffer * const buffer =
2687         GST_VAAPI_DECODER_CODEC_FRAME(decoder)->input_buffer;
2688     GstMapInfo map_info;
2689
2690     GST_DEBUG("slice (%u bytes)", pi->nalu.size);
2691
2692     if (!priv->got_sps || !priv->got_pps) {
2693         GST_ERROR("not initialized yet");
2694         return GST_VAAPI_DECODER_STATUS_SUCCESS;
2695     }
2696
2697     if (!gst_buffer_map(buffer, &map_info, GST_MAP_READ)) {
2698         GST_ERROR("failed to map buffer");
2699         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2700     }
2701
2702     slice = GST_VAAPI_SLICE_NEW(H264, decoder,
2703         (map_info.data + unit->offset + pi->nalu.offset), pi->nalu.size);
2704     gst_buffer_unmap(buffer, &map_info);
2705     if (!slice) {
2706         GST_ERROR("failed to allocate slice");
2707         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2708     }
2709
2710     if (!fill_slice(decoder, slice, slice_hdr)) {
2711         gst_vaapi_mini_object_unref(GST_VAAPI_MINI_OBJECT(slice));
2712         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2713     }
2714
2715     gst_vaapi_picture_add_slice(GST_VAAPI_PICTURE_CAST(picture), slice);
2716     picture->last_slice_hdr = slice_hdr;
2717     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2718 }
2719
2720 static inline gint
2721 scan_for_start_code(GstAdapter *adapter, guint ofs, guint size, guint32 *scp)
2722 {
2723     return (gint)gst_adapter_masked_scan_uint32_peek(adapter,
2724                                                      0xffffff00, 0x00000100,
2725                                                      ofs, size,
2726                                                      scp);
2727 }
2728
2729 static GstVaapiDecoderStatus
2730 decode_unit(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnit *unit)
2731 {
2732     GstVaapiParserInfoH264 * const pi = unit->parsed_info;
2733     GstVaapiDecoderStatus status;
2734
2735     switch (pi->nalu.type) {
2736     case GST_H264_NAL_SLICE_IDR:
2737         /* fall-through. IDR specifics are handled in init_picture() */
2738     case GST_H264_NAL_SLICE:
2739         status = decode_slice(decoder, unit);
2740         break;
2741     case GST_H264_NAL_SEQ_END:
2742     case GST_H264_NAL_STREAM_END:
2743         status = decode_sequence_end(decoder);
2744         break;
2745     case GST_H264_NAL_SEI:
2746         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2747         break;
2748     default:
2749         GST_WARNING("unsupported NAL unit type %d", pi->nalu.type);
2750         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
2751         break;
2752     }
2753     return status;
2754 }
2755
2756 static GstVaapiDecoderStatus
2757 gst_vaapi_decoder_h264_decode_codec_data(GstVaapiDecoder *base_decoder,
2758     const guchar *buf, guint buf_size)
2759 {
2760     GstVaapiDecoderH264 * const decoder =
2761         GST_VAAPI_DECODER_H264_CAST(base_decoder);
2762     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2763     GstVaapiDecoderStatus status;
2764     GstVaapiDecoderUnit unit;
2765     GstVaapiParserInfoH264 pi;
2766     GstH264ParserResult result;
2767     guint i, ofs, num_sps, num_pps;
2768
2769     unit.parsed_info = &pi;
2770
2771     if (buf_size < 8)
2772         return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2773
2774     if (buf[0] != 1) {
2775         GST_ERROR("failed to decode codec-data, not in avcC format");
2776         return GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
2777     }
2778
2779     priv->nal_length_size = (buf[4] & 0x03) + 1;
2780
2781     num_sps = buf[5] & 0x1f;
2782     ofs = 6;
2783
2784     for (i = 0; i < num_sps; i++) {
2785         result = gst_h264_parser_identify_nalu_avc(
2786             priv->parser,
2787             buf, ofs, buf_size, 2,
2788             &pi.nalu
2789         );
2790         if (result != GST_H264_PARSER_OK)
2791             return get_status(result);
2792
2793         status = parse_sps(decoder, &unit);
2794         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2795             return status;
2796         ofs = pi.nalu.offset + pi.nalu.size;
2797     }
2798
2799     num_pps = buf[ofs];
2800     ofs++;
2801
2802     for (i = 0; i < num_pps; i++) {
2803         result = gst_h264_parser_identify_nalu_avc(
2804             priv->parser,
2805             buf, ofs, buf_size, 2,
2806             &pi.nalu
2807         );
2808         if (result != GST_H264_PARSER_OK)
2809             return get_status(result);
2810
2811         status = parse_pps(decoder, &unit);
2812         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2813             return status;
2814         ofs = pi.nalu.offset + pi.nalu.size;
2815     }
2816
2817     priv->is_avcC = TRUE;
2818     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2819 }
2820
2821 static GstVaapiDecoderStatus
2822 ensure_decoder(GstVaapiDecoderH264 *decoder)
2823 {
2824     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2825     GstVaapiDecoderStatus status;
2826
2827     if (!priv->is_opened) {
2828         priv->is_opened = gst_vaapi_decoder_h264_open(decoder);
2829         if (!priv->is_opened)
2830             return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CODEC;
2831
2832         status = gst_vaapi_decoder_decode_codec_data(
2833             GST_VAAPI_DECODER_CAST(decoder));
2834         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2835             return status;
2836     }
2837     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2838 }
2839
2840 static GstVaapiDecoderStatus
2841 gst_vaapi_decoder_h264_parse(GstVaapiDecoder *base_decoder,
2842     GstAdapter *adapter, gboolean at_eos, GstVaapiDecoderUnit *unit)
2843 {
2844     GstVaapiDecoderH264 * const decoder =
2845         GST_VAAPI_DECODER_H264_CAST(base_decoder);
2846     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2847     GstVaapiParserState * const ps = GST_VAAPI_PARSER_STATE(base_decoder);
2848     GstVaapiParserInfoH264 *pi;
2849     GstVaapiDecoderStatus status;
2850     GstH264ParserResult result;
2851     guchar *buf;
2852     guint i, size, buf_size, nalu_size, flags;
2853     guint32 start_code;
2854     gint ofs, ofs2;
2855
2856     status = ensure_decoder(decoder);
2857     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2858         return status;
2859
2860     size = gst_adapter_available(adapter);
2861
2862     if (priv->is_avcC) {
2863         if (size < priv->nal_length_size)
2864             return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2865
2866         buf = (guchar *)&start_code;
2867         g_assert(priv->nal_length_size <= sizeof(start_code));
2868         gst_adapter_copy(adapter, buf, 0, priv->nal_length_size);
2869
2870         nalu_size = 0;
2871         for (i = 0; i < priv->nal_length_size; i++)
2872             nalu_size = (nalu_size << 8) | buf[i];
2873
2874         buf_size = priv->nal_length_size + nalu_size;
2875         if (size < buf_size)
2876             return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2877     }
2878     else {
2879         if (size < 4)
2880             return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2881
2882         ofs = scan_for_start_code(adapter, 0, size, NULL);
2883         if (ofs < 0)
2884             return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2885
2886         if (ofs > 0) {
2887             gst_adapter_flush(adapter, ofs);
2888             size -= ofs;
2889         }
2890
2891         ofs2 = ps->input_offset2 - ofs - 4;
2892         if (ofs2 < 4)
2893             ofs2 = 4;
2894
2895         ofs = G_UNLIKELY(size < ofs2 + 4) ? -1 :
2896             scan_for_start_code(adapter, ofs2, size - ofs2, NULL);
2897         if (ofs < 0) {
2898             // Assume the whole NAL unit is present if end-of-stream
2899             if (!at_eos) {
2900                 ps->input_offset2 = size;
2901                 return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2902             }
2903             ofs = size;
2904         }
2905         buf_size = ofs;
2906     }
2907     ps->input_offset2 = 0;
2908
2909     buf = (guchar *)gst_adapter_map(adapter, buf_size);
2910     if (!buf)
2911         return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2912
2913     unit->size = buf_size;
2914
2915     pi = gst_vaapi_parser_info_h264_new();
2916     if (!pi)
2917         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2918
2919     gst_vaapi_decoder_unit_set_parsed_info(unit,
2920         pi, (GDestroyNotify)gst_vaapi_mini_object_unref);
2921
2922     if (priv->is_avcC)
2923         result = gst_h264_parser_identify_nalu_avc(priv->parser,
2924             buf, 0, buf_size, priv->nal_length_size, &pi->nalu);
2925     else
2926         result = gst_h264_parser_identify_nalu_unchecked(priv->parser,
2927             buf, 0, buf_size, &pi->nalu);
2928     status = get_status(result);
2929     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2930         return status;
2931
2932     switch (pi->nalu.type) {
2933     case GST_H264_NAL_SPS:
2934         status = parse_sps(decoder, unit);
2935         break;
2936     case GST_H264_NAL_PPS:
2937         status = parse_pps(decoder, unit);
2938         break;
2939     case GST_H264_NAL_SEI:
2940         status = parse_sei(decoder, unit);
2941         break;
2942     case GST_H264_NAL_SLICE_IDR:
2943     case GST_H264_NAL_SLICE:
2944         status = parse_slice(decoder, unit);
2945         break;
2946     default:
2947         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2948         break;
2949     }
2950     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2951         return status;
2952
2953     flags = 0;
2954     switch (pi->nalu.type) {
2955     case GST_H264_NAL_AU_DELIMITER:
2956         flags |= GST_VAAPI_DECODER_UNIT_FLAG_FRAME_START;
2957         /* fall-through */
2958     case GST_H264_NAL_FILLER_DATA:
2959         flags |= GST_VAAPI_DECODER_UNIT_FLAG_SKIP;
2960         break;
2961     case GST_H264_NAL_STREAM_END:
2962         flags |= GST_VAAPI_DECODER_UNIT_FLAG_STREAM_END;
2963         /* fall-through */
2964     case GST_H264_NAL_SEQ_END:
2965         flags |= GST_VAAPI_DECODER_UNIT_FLAG_FRAME_END;
2966         break;
2967     case GST_H264_NAL_SPS:
2968     case GST_H264_NAL_PPS:
2969         flags |= GST_VAAPI_DECODER_UNIT_FLAG_SKIP;
2970         /* fall-through */
2971     case GST_H264_NAL_SEI:
2972         flags |= GST_VAAPI_DECODER_UNIT_FLAG_FRAME_START;
2973         break;
2974     case GST_H264_NAL_SLICE_IDR:
2975     case GST_H264_NAL_SLICE:
2976         flags |= GST_VAAPI_DECODER_UNIT_FLAG_SLICE;
2977         if (is_new_picture(pi, priv->prev_slice_pi))
2978             flags |= GST_VAAPI_DECODER_UNIT_FLAG_FRAME_START;
2979         gst_vaapi_parser_info_h264_replace(&priv->prev_slice_pi, pi);
2980         break;
2981     default:
2982         if (pi->nalu.type >= 14 && pi->nalu.type <= 18)
2983             flags |= GST_VAAPI_DECODER_UNIT_FLAG_FRAME_START;
2984         break;
2985     }
2986     GST_VAAPI_DECODER_UNIT_FLAG_SET(unit, flags);
2987
2988     pi->nalu.data = NULL;
2989     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2990 }
2991
2992 static GstVaapiDecoderStatus
2993 gst_vaapi_decoder_h264_decode(GstVaapiDecoder *base_decoder,
2994     GstVaapiDecoderUnit *unit)
2995 {
2996     GstVaapiDecoderH264 * const decoder =
2997         GST_VAAPI_DECODER_H264_CAST(base_decoder);
2998     GstVaapiDecoderStatus status;
2999
3000     status = ensure_decoder(decoder);
3001     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
3002         return status;
3003     return decode_unit(decoder, unit);
3004 }
3005
3006 static GstVaapiDecoderStatus
3007 gst_vaapi_decoder_h264_start_frame(GstVaapiDecoder *base_decoder,
3008     GstVaapiDecoderUnit *unit)
3009 {
3010     GstVaapiDecoderH264 * const decoder =
3011         GST_VAAPI_DECODER_H264_CAST(base_decoder);
3012
3013     return decode_picture(decoder, unit);
3014 }
3015
3016 static GstVaapiDecoderStatus
3017 gst_vaapi_decoder_h264_end_frame(GstVaapiDecoder *base_decoder)
3018 {
3019     GstVaapiDecoderH264 * const decoder =
3020         GST_VAAPI_DECODER_H264_CAST(base_decoder);
3021
3022     return decode_current_picture(decoder);
3023 }
3024
3025 static GstVaapiDecoderStatus
3026 gst_vaapi_decoder_h264_flush(GstVaapiDecoder *base_decoder)
3027 {
3028     GstVaapiDecoderH264 * const decoder =
3029         GST_VAAPI_DECODER_H264_CAST(base_decoder);
3030
3031     dpb_flush(decoder);
3032     return GST_VAAPI_DECODER_STATUS_SUCCESS;
3033 }
3034
3035 static void
3036 gst_vaapi_decoder_h264_class_init(GstVaapiDecoderH264Class *klass)
3037 {
3038     GstVaapiMiniObjectClass * const object_class =
3039         GST_VAAPI_MINI_OBJECT_CLASS(klass);
3040     GstVaapiDecoderClass * const decoder_class = GST_VAAPI_DECODER_CLASS(klass);
3041
3042     object_class->size          = sizeof(GstVaapiDecoderH264);
3043     object_class->finalize      = (GDestroyNotify)gst_vaapi_decoder_finalize;
3044
3045     decoder_class->create       = gst_vaapi_decoder_h264_create;
3046     decoder_class->destroy      = gst_vaapi_decoder_h264_destroy;
3047     decoder_class->parse        = gst_vaapi_decoder_h264_parse;
3048     decoder_class->decode       = gst_vaapi_decoder_h264_decode;
3049     decoder_class->start_frame  = gst_vaapi_decoder_h264_start_frame;
3050     decoder_class->end_frame    = gst_vaapi_decoder_h264_end_frame;
3051     decoder_class->flush        = gst_vaapi_decoder_h264_flush;
3052
3053     decoder_class->decode_codec_data =
3054         gst_vaapi_decoder_h264_decode_codec_data;
3055 }
3056
3057 static inline const GstVaapiDecoderClass *
3058 gst_vaapi_decoder_h264_class(void)
3059 {
3060     static GstVaapiDecoderH264Class g_class;
3061     static gsize g_class_init = FALSE;
3062
3063     if (g_once_init_enter(&g_class_init)) {
3064         gst_vaapi_decoder_h264_class_init(&g_class);
3065         g_once_init_leave(&g_class_init, TRUE);
3066     }
3067     return GST_VAAPI_DECODER_CLASS(&g_class);
3068 }
3069
3070 /**
3071  * gst_vaapi_decoder_h264_new:
3072  * @display: a #GstVaapiDisplay
3073  * @caps: a #GstCaps holding codec information
3074  *
3075  * Creates a new #GstVaapiDecoder for MPEG-2 decoding.  The @caps can
3076  * hold extra information like codec-data and pictured coded size.
3077  *
3078  * Return value: the newly allocated #GstVaapiDecoder object
3079  */
3080 GstVaapiDecoder *
3081 gst_vaapi_decoder_h264_new(GstVaapiDisplay *display, GstCaps *caps)
3082 {
3083     return gst_vaapi_decoder_new(gst_vaapi_decoder_h264_class(), display, caps);
3084 }