gst-libs/gst/vaapi/gstvaapidecoder_h264.c

   1 /*
   2  *  gstvaapidecoder_h264.c - H.264 decoder
   3  *
   4  *  Copyright (C) 2011-2014 Intel Corporation
   5  *    Author: Gwenole Beauchesne <gwenole.beauchesne@intel.com>
   6  *
   7  *  This library is free software; you can redistribute it and/or
   8  *  modify it under the terms of the GNU Lesser General Public License
   9  *  as published by the Free Software Foundation; either version 2.1
  10  *  of the License, or (at your option) any later version.
  11  *
  12  *  This library is distributed in the hope that it will be useful,
  13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  *  Lesser General Public License for more details.
  16  *
  17  *  You should have received a copy of the GNU Lesser General Public
  18  *  License along with this library; if not, write to the Free
  19  *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  20  *  Boston, MA 02110-1301 USA
  21  */
  22
  23 /**
  24  * SECTION:gstvaapidecoder_h264
  25  * @short_description: H.264 decoder
  26  */
  27
  28 #include "sysdeps.h"
  29 #include <string.h>
  30 #include <gst/base/gstadapter.h>
  31 #include <gst/codecparsers/gsth264parser.h>
  32 #include "gstvaapidecoder_h264.h"
  33 #include "gstvaapidecoder_objects.h"
  34 #include "gstvaapidecoder_priv.h"
  35 #include "gstvaapidisplay_priv.h"
  36 #include "gstvaapiobject_priv.h"
  37 #include "gstvaapiutils_h264_priv.h"
  38
  39 #define DEBUG 1
  40 #include "gstvaapidebug.h"
  41
  42 /* Defined to 1 if strict ordering of DPB is needed. Only useful for debug */
  43 #define USE_STRICT_DPB_ORDERING 0
  44
  45 typedef struct _GstVaapiDecoderH264Private      GstVaapiDecoderH264Private;
  46 typedef struct _GstVaapiDecoderH264Class        GstVaapiDecoderH264Class;
  47 typedef struct _GstVaapiFrameStore              GstVaapiFrameStore;
  48 typedef struct _GstVaapiFrameStoreClass         GstVaapiFrameStoreClass;
  49 typedef struct _GstVaapiParserInfoH264          GstVaapiParserInfoH264;
  50 typedef struct _GstVaapiPictureH264             GstVaapiPictureH264;
  51
  52 // Used for field_poc[]
  53 #define TOP_FIELD       0
  54 #define BOTTOM_FIELD    1
  55
  56 /* ------------------------------------------------------------------------- */
  57 /* --- H.264 Parser Info                                                 --- */
  58 /* ------------------------------------------------------------------------- */
  59
  60 #define GST_VAAPI_PARSER_INFO_H264(obj) \
  61     ((GstVaapiParserInfoH264 *)(obj))
  62
  63 struct _GstVaapiParserInfoH264 {
  64     GstVaapiMiniObject  parent_instance;
  65     GstH264NalUnit      nalu;
  66     union {
  67         GstH264SPS      sps;
  68         GstH264PPS      pps;
  69         GstH264SliceHdr slice_hdr;
  70     }                   data;
  71     guint               state;
  72 };
  73
  74 static inline const GstVaapiMiniObjectClass *
  75 gst_vaapi_parser_info_h264_class(void)
  76 {
  77     static const GstVaapiMiniObjectClass GstVaapiParserInfoH264Class = {
  78         sizeof(GstVaapiParserInfoH264),
  79         NULL
  80     };
  81     return &GstVaapiParserInfoH264Class;
  82 }
  83
  84 static inline GstVaapiParserInfoH264 *
  85 gst_vaapi_parser_info_h264_new(void)
  86 {
  87     return (GstVaapiParserInfoH264 *)
  88         gst_vaapi_mini_object_new(gst_vaapi_parser_info_h264_class());
  89 }
  90
  91 #define gst_vaapi_parser_info_h264_ref(pi) \
  92     gst_vaapi_mini_object_ref(GST_VAAPI_MINI_OBJECT(pi))
  93
  94 #define gst_vaapi_parser_info_h264_unref(pi) \
  95     gst_vaapi_mini_object_unref(GST_VAAPI_MINI_OBJECT(pi))
  96
  97 #define gst_vaapi_parser_info_h264_replace(old_pi_ptr, new_pi)          \
  98     gst_vaapi_mini_object_replace((GstVaapiMiniObject **)(old_pi_ptr),  \
  99         (GstVaapiMiniObject *)(new_pi))
 100
 101 /* ------------------------------------------------------------------------- */
 102 /* --- H.264 Pictures                                                    --- */
 103 /* ------------------------------------------------------------------------- */
 104
 105 /*
 106  * Extended picture flags:
 107  *
 108  * @GST_VAAPI_PICTURE_FLAG_IDR: flag that specifies an IDR picture
 109  * @GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE: flag that specifies
 110  *     "used for short-term reference"
 111  * @GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE: flag that specifies
 112  *     "used for long-term reference"
 113  * @GST_VAAPI_PICTURE_FLAGS_REFERENCE: mask covering any kind of
 114  *     reference picture (short-term reference or long-term reference)
 115  */
 116 enum {
 117     GST_VAAPI_PICTURE_FLAG_IDR = (GST_VAAPI_PICTURE_FLAG_LAST << 0),
 118
 119     GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE = (
 120         GST_VAAPI_PICTURE_FLAG_REFERENCE),
 121     GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE = (
 122         GST_VAAPI_PICTURE_FLAG_REFERENCE | (GST_VAAPI_PICTURE_FLAG_LAST << 1)),
 123     GST_VAAPI_PICTURE_FLAGS_REFERENCE = (
 124         GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE |
 125         GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE),
 126 };
 127
 128 #define GST_VAAPI_PICTURE_IS_IDR(picture) \
 129     (GST_VAAPI_PICTURE_FLAG_IS_SET(picture, GST_VAAPI_PICTURE_FLAG_IDR))
 130
 131 #define GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture)      \
 132     ((GST_VAAPI_PICTURE_FLAGS(picture) &                        \
 133       GST_VAAPI_PICTURE_FLAGS_REFERENCE) ==                     \
 134      GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE)
 135
 136 #define GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture)       \
 137     ((GST_VAAPI_PICTURE_FLAGS(picture) &                        \
 138       GST_VAAPI_PICTURE_FLAGS_REFERENCE) ==                     \
 139      GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE)
 140
 141 struct _GstVaapiPictureH264 {
 142     GstVaapiPicture             base;
 143     GstH264PPS                 *pps;
 144     GstH264SliceHdr            *last_slice_hdr;
 145     guint                       structure;
 146     gint32                      field_poc[2];
 147     gint32                      frame_num;              // Original frame_num from slice_header()
 148     gint32                      frame_num_wrap;         // Temporary for ref pic marking: FrameNumWrap
 149     gint32                      long_term_frame_idx;    // Temporary for ref pic marking: LongTermFrameIdx
 150     gint32                      pic_num;                // Temporary for ref pic marking: PicNum
 151     gint32                      long_term_pic_num;      // Temporary for ref pic marking: LongTermPicNum
 152     GstVaapiPictureH264        *other_field;            // Temporary for ref pic marking: other field in the same frame store
 153     guint                       output_flag             : 1;
 154     guint                       output_needed           : 1;
 155 };
 156
 157 GST_VAAPI_CODEC_DEFINE_TYPE(GstVaapiPictureH264, gst_vaapi_picture_h264);
 158
 159 void
 160 gst_vaapi_picture_h264_destroy(GstVaapiPictureH264 *picture)
 161 {
 162     gst_vaapi_picture_destroy(GST_VAAPI_PICTURE(picture));
 163 }
 164
 165 gboolean
 166 gst_vaapi_picture_h264_create(
 167     GstVaapiPictureH264                      *picture,
 168     const GstVaapiCodecObjectConstructorArgs *args
 169 )
 170 {
 171     if (!gst_vaapi_picture_create(GST_VAAPI_PICTURE(picture), args))
 172         return FALSE;
 173
 174     picture->field_poc[0]       = G_MAXINT32;
 175     picture->field_poc[1]       = G_MAXINT32;
 176     picture->output_needed      = FALSE;
 177     return TRUE;
 178 }
 179
 180 static inline GstVaapiPictureH264 *
 181 gst_vaapi_picture_h264_new(GstVaapiDecoderH264 *decoder)
 182 {
 183     return (GstVaapiPictureH264 *)gst_vaapi_codec_object_new(
 184         &GstVaapiPictureH264Class,
 185         GST_VAAPI_CODEC_BASE(decoder),
 186         NULL, sizeof(VAPictureParameterBufferH264),
 187         NULL, 0,
 188         0);
 189 }
 190
 191 static inline void
 192 gst_vaapi_picture_h264_set_reference(
 193     GstVaapiPictureH264 *picture,
 194     guint                reference_flags,
 195     gboolean             other_field
 196 )
 197 {
 198     if (!picture)
 199         return;
 200     GST_VAAPI_PICTURE_FLAG_UNSET(picture, GST_VAAPI_PICTURE_FLAGS_REFERENCE);
 201     GST_VAAPI_PICTURE_FLAG_SET(picture, reference_flags);
 202
 203     if (!other_field || !(picture = picture->other_field))
 204         return;
 205     GST_VAAPI_PICTURE_FLAG_UNSET(picture, GST_VAAPI_PICTURE_FLAGS_REFERENCE);
 206     GST_VAAPI_PICTURE_FLAG_SET(picture, reference_flags);
 207 }
 208
 209 static inline GstVaapiPictureH264 *
 210 gst_vaapi_picture_h264_new_field(GstVaapiPictureH264 *picture)
 211 {
 212     g_return_val_if_fail(picture, NULL);
 213
 214     return (GstVaapiPictureH264 *)gst_vaapi_picture_new_field(&picture->base);
 215 }
 216
 217 /* ------------------------------------------------------------------------- */
 218 /* --- Frame Buffers (DPB)                                               --- */
 219 /* ------------------------------------------------------------------------- */
 220
 221 struct _GstVaapiFrameStore {
 222     /*< private >*/
 223     GstVaapiMiniObject          parent_instance;
 224
 225     guint                       structure;
 226     GstVaapiPictureH264        *buffers[2];
 227     guint                       num_buffers;
 228     guint                       output_needed;
 229 };
 230
 231 static void
 232 gst_vaapi_frame_store_finalize(gpointer object)
 233 {
 234     GstVaapiFrameStore * const fs = object;
 235     guint i;
 236
 237     for (i = 0; i < fs->num_buffers; i++)
 238         gst_vaapi_picture_replace(&fs->buffers[i], NULL);
 239 }
 240
 241 static GstVaapiFrameStore *
 242 gst_vaapi_frame_store_new(GstVaapiPictureH264 *picture)
 243 {
 244     GstVaapiFrameStore *fs;
 245
 246     static const GstVaapiMiniObjectClass GstVaapiFrameStoreClass = {
 247         sizeof(GstVaapiFrameStore),
 248         gst_vaapi_frame_store_finalize
 249     };
 250
 251     fs = (GstVaapiFrameStore *)
 252         gst_vaapi_mini_object_new(&GstVaapiFrameStoreClass);
 253     if (!fs)
 254         return NULL;
 255
 256     fs->structure       = picture->structure;
 257     fs->buffers[0]      = gst_vaapi_picture_ref(picture);
 258     fs->buffers[1]      = NULL;
 259     fs->num_buffers     = 1;
 260     fs->output_needed   = picture->output_needed;
 261     return fs;
 262 }
 263
 264 static gboolean
 265 gst_vaapi_frame_store_add(GstVaapiFrameStore *fs, GstVaapiPictureH264 *picture)
 266 {
 267     guint field;
 268
 269     g_return_val_if_fail(fs->num_buffers == 1, FALSE);
 270     g_return_val_if_fail(!GST_VAAPI_PICTURE_IS_FRAME(picture), FALSE);
 271     g_return_val_if_fail(!GST_VAAPI_PICTURE_IS_FIRST_FIELD(picture), FALSE);
 272
 273     gst_vaapi_picture_replace(&fs->buffers[fs->num_buffers++], picture);
 274     if (picture->output_flag) {
 275         picture->output_needed = TRUE;
 276         fs->output_needed++;
 277     }
 278
 279     fs->structure = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
 280
 281     field = picture->structure == GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD ?
 282         TOP_FIELD : BOTTOM_FIELD;
 283     g_return_val_if_fail(fs->buffers[0]->field_poc[field] == G_MAXINT32, FALSE);
 284     fs->buffers[0]->field_poc[field] = picture->field_poc[field];
 285     g_return_val_if_fail(picture->field_poc[!field] == G_MAXINT32, FALSE);
 286     picture->field_poc[!field] = fs->buffers[0]->field_poc[!field];
 287     return TRUE;
 288 }
 289
 290 static gboolean
 291 gst_vaapi_frame_store_split_fields(GstVaapiFrameStore *fs)
 292 {
 293     GstVaapiPictureH264 * const first_field = fs->buffers[0];
 294     GstVaapiPictureH264 *second_field;
 295
 296     g_return_val_if_fail(fs->num_buffers == 1, FALSE);
 297
 298     first_field->base.structure = GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD;
 299     GST_VAAPI_PICTURE_FLAG_SET(first_field, GST_VAAPI_PICTURE_FLAG_INTERLACED);
 300
 301     second_field = gst_vaapi_picture_h264_new_field(first_field);
 302     if (!second_field)
 303         return FALSE;
 304     gst_vaapi_picture_replace(&fs->buffers[fs->num_buffers++], second_field);
 305     gst_vaapi_picture_unref(second_field);
 306
 307     second_field->frame_num    = first_field->frame_num;
 308     second_field->field_poc[0] = first_field->field_poc[0];
 309     second_field->field_poc[1] = first_field->field_poc[1];
 310     second_field->output_flag  = first_field->output_flag;
 311     if (second_field->output_flag) {
 312         second_field->output_needed = TRUE;
 313         fs->output_needed++;
 314     }
 315     return TRUE;
 316 }
 317
 318 static inline gboolean
 319 gst_vaapi_frame_store_has_frame(GstVaapiFrameStore *fs)
 320 {
 321     return fs->structure == GST_VAAPI_PICTURE_STRUCTURE_FRAME;
 322 }
 323
 324 static inline gboolean
 325 gst_vaapi_frame_store_has_reference(GstVaapiFrameStore *fs)
 326 {
 327     guint i;
 328
 329     for (i = 0; i < fs->num_buffers; i++) {
 330         if (GST_VAAPI_PICTURE_IS_REFERENCE(fs->buffers[i]))
 331             return TRUE;
 332     }
 333     return FALSE;
 334 }
 335
 336 #define gst_vaapi_frame_store_ref(fs) \
 337     gst_vaapi_mini_object_ref(GST_VAAPI_MINI_OBJECT(fs))
 338
 339 #define gst_vaapi_frame_store_unref(fs) \
 340     gst_vaapi_mini_object_unref(GST_VAAPI_MINI_OBJECT(fs))
 341
 342 #define gst_vaapi_frame_store_replace(old_fs_p, new_fs)                 \
 343     gst_vaapi_mini_object_replace((GstVaapiMiniObject **)(old_fs_p),    \
 344         (GstVaapiMiniObject *)(new_fs))
 345
 346 /* ------------------------------------------------------------------------- */
 347 /* --- H.264 Decoder                                                     --- */
 348 /* ------------------------------------------------------------------------- */
 349
 350 #define GST_VAAPI_DECODER_H264_CAST(decoder) \
 351     ((GstVaapiDecoderH264 *)(decoder))
 352
 353 typedef enum {
 354     GST_H264_VIDEO_STATE_GOT_SPS        = 1 << 0,
 355     GST_H264_VIDEO_STATE_GOT_PPS        = 1 << 1,
 356     GST_H264_VIDEO_STATE_GOT_SLICE      = 1 << 2,
 357
 358     GST_H264_VIDEO_STATE_VALID_PICTURE_HEADERS = (
 359         GST_H264_VIDEO_STATE_GOT_SPS |
 360         GST_H264_VIDEO_STATE_GOT_PPS),
 361     GST_H264_VIDEO_STATE_VALID_PICTURE = (
 362         GST_H264_VIDEO_STATE_VALID_PICTURE_HEADERS |
 363         GST_H264_VIDEO_STATE_GOT_SLICE)
 364 } GstH264VideoState;
 365
 366 struct _GstVaapiDecoderH264Private {
 367     GstH264NalParser           *parser;
 368     guint                       parser_state;
 369     guint                       decoder_state;
 370     GstVaapiPictureH264        *current_picture;
 371     GstVaapiParserInfoH264     *prev_slice_pi;
 372     GstVaapiFrameStore         *prev_frame;
 373     GstVaapiFrameStore         *dpb[16];
 374     guint                       dpb_count;
 375     guint                       dpb_size;
 376     GstVaapiProfile             profile;
 377     GstVaapiEntrypoint          entrypoint;
 378     GstVaapiChromaType          chroma_type;
 379     GstVaapiPictureH264        *short_ref[32];
 380     guint                       short_ref_count;
 381     GstVaapiPictureH264        *long_ref[32];
 382     guint                       long_ref_count;
 383     GstVaapiPictureH264        *RefPicList0[32];
 384     guint                       RefPicList0_count;
 385     GstVaapiPictureH264        *RefPicList1[32];
 386     guint                       RefPicList1_count;
 387     guint                       nal_length_size;
 388     guint                       mb_width;
 389     guint                       mb_height;
 390     gint32                      field_poc[2];           // 0:TopFieldOrderCnt / 1:BottomFieldOrderCnt
 391     gint32                      poc_msb;                // PicOrderCntMsb
 392     gint32                      poc_lsb;                // pic_order_cnt_lsb (from slice_header())
 393     gint32                      prev_poc_msb;           // prevPicOrderCntMsb
 394     gint32                      prev_poc_lsb;           // prevPicOrderCntLsb
 395     gint32                      frame_num_offset;       // FrameNumOffset
 396     gint32                      frame_num;              // frame_num (from slice_header())
 397     gint32                      prev_frame_num;         // prevFrameNum
 398     gboolean                    prev_pic_has_mmco5;     // prevMmco5Pic
 399     gboolean                    prev_pic_structure;     // previous picture structure
 400     guint                       is_opened               : 1;
 401     guint                       is_avcC                 : 1;
 402     guint                       has_context             : 1;
 403     guint                       progressive_sequence    : 1;
 404 };
 405
 406 /**
 407  * GstVaapiDecoderH264:
 408  *
 409  * A decoder based on H264.
 410  */
 411 struct _GstVaapiDecoderH264 {
 412     /*< private >*/
 413     GstVaapiDecoder             parent_instance;
 414     GstVaapiDecoderH264Private  priv;
 415 };
 416
 417 /**
 418  * GstVaapiDecoderH264Class:
 419  *
 420  * A decoder class based on H264.
 421  */
 422 struct _GstVaapiDecoderH264Class {
 423     /*< private >*/
 424     GstVaapiDecoderClass parent_class;
 425 };
 426
 427 static gboolean
 428 exec_ref_pic_marking(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture);
 429
 430 /* Get number of reference frames to use */
 431 static guint
 432 get_max_dec_frame_buffering(GstH264SPS *sps)
 433 {
 434     guint max_dec_frame_buffering, PicSizeMbs;
 435     GstVaapiLevelH264 level;
 436     const GstVaapiH264LevelLimits *level_limits;
 437
 438     /* Table A-1 - Level limits */
 439     if (G_UNLIKELY(sps->level_idc == 11 && sps->constraint_set3_flag))
 440         level = GST_VAAPI_LEVEL_H264_L1b;
 441     else
 442         level = gst_vaapi_utils_h264_get_level(sps->level_idc);
 443     level_limits = gst_vaapi_utils_h264_get_level_limits(level);
 444     if (!level_limits)
 445         return 16;
 446
 447     PicSizeMbs = ((sps->pic_width_in_mbs_minus1 + 1) *
 448                   (sps->pic_height_in_map_units_minus1 + 1) *
 449                   (sps->frame_mbs_only_flag ? 1 : 2));
 450     max_dec_frame_buffering = level_limits->MaxDpbMbs / PicSizeMbs;
 451
 452     /* VUI parameters */
 453     if (sps->vui_parameters_present_flag) {
 454         GstH264VUIParams * const vui_params = &sps->vui_parameters;
 455         if (vui_params->bitstream_restriction_flag)
 456             max_dec_frame_buffering = vui_params->max_dec_frame_buffering;
 457         else {
 458             switch (sps->profile_idc) {
 459             case 44:  // CAVLC 4:4:4 Intra profile
 460             case GST_H264_PROFILE_SCALABLE_HIGH:
 461             case GST_H264_PROFILE_HIGH:
 462             case GST_H264_PROFILE_HIGH10:
 463             case GST_H264_PROFILE_HIGH_422:
 464             case GST_H264_PROFILE_HIGH_444:
 465                 if (sps->constraint_set3_flag)
 466                     max_dec_frame_buffering = 0;
 467                 break;
 468             }
 469         }
 470     }
 471
 472     if (max_dec_frame_buffering > 16)
 473         max_dec_frame_buffering = 16;
 474     else if (max_dec_frame_buffering < sps->num_ref_frames)
 475         max_dec_frame_buffering = sps->num_ref_frames;
 476     return MAX(1, max_dec_frame_buffering);
 477 }
 478
 479 static void
 480 array_remove_index_fast(void *array, guint *array_length_ptr, guint index)
 481 {
 482     gpointer * const entries = array;
 483     guint num_entries = *array_length_ptr;
 484
 485     g_return_if_fail(index < num_entries);
 486
 487     if (index != --num_entries)
 488         entries[index] = entries[num_entries];
 489     entries[num_entries] = NULL;
 490     *array_length_ptr = num_entries;
 491 }
 492
 493 #if 1
 494 static inline void
 495 array_remove_index(void *array, guint *array_length_ptr, guint index)
 496 {
 497     array_remove_index_fast(array, array_length_ptr, index);
 498 }
 499 #else
 500 static void
 501 array_remove_index(void *array, guint *array_length_ptr, guint index)
 502 {
 503     gpointer * const entries = array;
 504     const guint num_entries = *array_length_ptr - 1;
 505     guint i;
 506
 507     g_return_if_fail(index <= num_entries);
 508
 509     for (i = index; i < num_entries; i++)
 510         entries[i] = entries[i + 1];
 511     entries[num_entries] = NULL;
 512     *array_length_ptr = num_entries;
 513 }
 514 #endif
 515
 516 #define ARRAY_REMOVE_INDEX(array, index) \
 517     array_remove_index(array, &array##_count, index)
 518
 519 static void
 520 dpb_remove_index(GstVaapiDecoderH264 *decoder, guint index)
 521 {
 522     GstVaapiDecoderH264Private * const priv = &decoder->priv;
 523     guint i, num_frames = --priv->dpb_count;
 524
 525     if (USE_STRICT_DPB_ORDERING) {
 526         for (i = index; i < num_frames; i++)
 527             gst_vaapi_frame_store_replace(&priv->dpb[i], priv->dpb[i + 1]);
 528     }
 529     else if (index != num_frames)
 530         gst_vaapi_frame_store_replace(&priv->dpb[index], priv->dpb[num_frames]);
 531     gst_vaapi_frame_store_replace(&priv->dpb[num_frames], NULL);
 532 }
 533
 534 static gboolean
 535 dpb_output(
 536     GstVaapiDecoderH264 *decoder,
 537     GstVaapiFrameStore  *fs,
 538     GstVaapiPictureH264 *picture
 539 )
 540 {
 541     picture->output_needed = FALSE;
 542
 543     if (fs) {
 544         if (--fs->output_needed > 0)
 545             return TRUE;
 546         picture = fs->buffers[0];
 547     }
 548     return gst_vaapi_picture_output(GST_VAAPI_PICTURE_CAST(picture));
 549 }
 550
 551 static inline void
 552 dpb_evict(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture, guint i)
 553 {
 554     GstVaapiDecoderH264Private * const priv = &decoder->priv;
 555     GstVaapiFrameStore * const fs = priv->dpb[i];
 556
 557     if (!fs->output_needed && !gst_vaapi_frame_store_has_reference(fs))
 558         dpb_remove_index(decoder, i);
 559 }
 560
 561 static gboolean
 562 dpb_bump(GstVaapiDecoderH264 *decoder)
 563 {
 564     GstVaapiDecoderH264Private * const priv = &decoder->priv;
 565     GstVaapiPictureH264 *found_picture = NULL;
 566     guint i, j, found_index;
 567     gboolean success;
 568
 569     for (i = 0; i < priv->dpb_count; i++) {
 570         GstVaapiFrameStore * const fs = priv->dpb[i];
 571         if (!fs->output_needed)
 572             continue;
 573         for (j = 0; j < fs->num_buffers; j++) {
 574             GstVaapiPictureH264 * const picture = fs->buffers[j];
 575             if (!picture->output_needed)
 576                 continue;
 577             if (!found_picture || found_picture->base.poc > picture->base.poc)
 578                 found_picture = picture, found_index = i;
 579         }
 580     }
 581     if (!found_picture)
 582         return FALSE;
 583
 584     success = dpb_output(decoder, priv->dpb[found_index], found_picture);
 585     dpb_evict(decoder, found_picture, found_index);
 586     return success;
 587 }
 588
 589 static void
 590 dpb_clear(GstVaapiDecoderH264 *decoder)
 591 {
 592     GstVaapiDecoderH264Private * const priv = &decoder->priv;
 593     guint i;
 594
 595     for (i = 0; i < priv->dpb_count; i++)
 596         gst_vaapi_frame_store_replace(&priv->dpb[i], NULL);
 597     priv->dpb_count = 0;
 598
 599     gst_vaapi_frame_store_replace(&priv->prev_frame, NULL);
 600 }
 601
 602 static void
 603 dpb_flush(GstVaapiDecoderH264 *decoder)
 604 {
 605     while (dpb_bump(decoder))
 606         ;
 607     dpb_clear(decoder);
 608 }
 609
 610 static gboolean
 611 dpb_add(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
 612 {
 613     GstVaapiDecoderH264Private * const priv = &decoder->priv;
 614     GstVaapiFrameStore *fs;
 615     guint i, j;
 616
 617     // Remove all unused pictures
 618     if (!GST_VAAPI_PICTURE_IS_IDR(picture)) {
 619         i = 0;
 620         while (i < priv->dpb_count) {
 621             GstVaapiFrameStore * const fs = priv->dpb[i];
 622             if (!fs->output_needed && !gst_vaapi_frame_store_has_reference(fs))
 623                 dpb_remove_index(decoder, i);
 624             else
 625                 i++;
 626         }
 627     }
 628
 629     // Check if picture is the second field and the first field is still in DPB
 630     fs = priv->prev_frame;
 631     if (fs && !gst_vaapi_frame_store_has_frame(fs))
 632         return gst_vaapi_frame_store_add(fs, picture);
 633
 634     // Create new frame store, and split fields if necessary
 635     fs = gst_vaapi_frame_store_new(picture);
 636     if (!fs)
 637         return FALSE;
 638     gst_vaapi_frame_store_replace(&priv->prev_frame, fs);
 639     gst_vaapi_frame_store_unref(fs);
 640
 641     if (!priv->progressive_sequence && gst_vaapi_frame_store_has_frame(fs)) {
 642         if (!gst_vaapi_frame_store_split_fields(fs))
 643             return FALSE;
 644     }
 645
 646     // C.4.5.1 - Storage and marking of a reference decoded picture into the DPB
 647     if (GST_VAAPI_PICTURE_IS_REFERENCE(picture)) {
 648         while (priv->dpb_count == priv->dpb_size) {
 649             if (!dpb_bump(decoder))
 650                 return FALSE;
 651         }
 652         gst_vaapi_frame_store_replace(&priv->dpb[priv->dpb_count++], fs);
 653         if (picture->output_flag) {
 654             picture->output_needed = TRUE;
 655             fs->output_needed++;
 656         }
 657     }
 658
 659     // C.4.5.2 - Storage and marking of a non-reference decoded picture into the DPB
 660     else {
 661         if (!picture->output_flag)
 662             return TRUE;
 663         while (priv->dpb_count == priv->dpb_size) {
 664             gboolean found_picture = FALSE;
 665             for (i = 0; !found_picture && i < priv->dpb_count; i++) {
 666                 GstVaapiFrameStore * const fs = priv->dpb[i];
 667                 if (!fs->output_needed)
 668                     continue;
 669                 for (j = 0; !found_picture && j < fs->num_buffers; j++)
 670                     found_picture = fs->buffers[j]->output_needed &&
 671                         fs->buffers[j]->base.poc < picture->base.poc;
 672             }
 673             if (!found_picture)
 674                 return dpb_output(decoder, NULL, picture);
 675             if (!dpb_bump(decoder))
 676                 return FALSE;
 677         }
 678         gst_vaapi_frame_store_replace(&priv->dpb[priv->dpb_count++], fs);
 679         picture->output_needed = TRUE;
 680         fs->output_needed++;
 681     }
 682     return TRUE;
 683 }
 684
 685 static inline void
 686 dpb_reset(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 687 {
 688     GstVaapiDecoderH264Private * const priv = &decoder->priv;
 689
 690     priv->dpb_size = get_max_dec_frame_buffering(sps);
 691     GST_DEBUG("DPB size %u", priv->dpb_size);
 692 }
 693
 694 static GstVaapiDecoderStatus
 695 get_status(GstH264ParserResult result)
 696 {
 697     GstVaapiDecoderStatus status;
 698
 699     switch (result) {
 700     case GST_H264_PARSER_OK:
 701         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
 702         break;
 703     case GST_H264_PARSER_NO_NAL_END:
 704         status = GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
 705         break;
 706     case GST_H264_PARSER_ERROR:
 707         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
 708         break;
 709     default:
 710         status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 711         break;
 712     }
 713     return status;
 714 }
 715
 716 static void
 717 gst_vaapi_decoder_h264_close(GstVaapiDecoderH264 *decoder)
 718 {
 719     GstVaapiDecoderH264Private * const priv = &decoder->priv;
 720
 721     gst_vaapi_picture_replace(&priv->current_picture, NULL);
 722     gst_vaapi_parser_info_h264_replace(&priv->prev_slice_pi, NULL);
 723
 724     dpb_clear(decoder);
 725
 726     if (priv->parser) {
 727         gst_h264_nal_parser_free(priv->parser);
 728         priv->parser = NULL;
 729     }
 730 }
 731
 732 static gboolean
 733 gst_vaapi_decoder_h264_open(GstVaapiDecoderH264 *decoder)
 734 {
 735     GstVaapiDecoderH264Private * const priv = &decoder->priv;
 736
 737     gst_vaapi_decoder_h264_close(decoder);
 738
 739     priv->parser = gst_h264_nal_parser_new();
 740     if (!priv->parser)
 741         return FALSE;
 742     return TRUE;
 743 }
 744
 745 static void
 746 gst_vaapi_decoder_h264_destroy(GstVaapiDecoder *base_decoder)
 747 {
 748     GstVaapiDecoderH264 * const decoder =
 749         GST_VAAPI_DECODER_H264_CAST(base_decoder);
 750
 751     gst_vaapi_decoder_h264_close(decoder);
 752 }
 753
 754 static gboolean
 755 gst_vaapi_decoder_h264_create(GstVaapiDecoder *base_decoder)
 756 {
 757     GstVaapiDecoderH264 * const decoder =
 758         GST_VAAPI_DECODER_H264_CAST(base_decoder);
 759     GstVaapiDecoderH264Private * const priv = &decoder->priv;
 760
 761     priv->profile               = GST_VAAPI_PROFILE_UNKNOWN;
 762     priv->entrypoint            = GST_VAAPI_ENTRYPOINT_VLD;
 763     priv->chroma_type           = GST_VAAPI_CHROMA_TYPE_YUV420;
 764     priv->prev_pic_structure    = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
 765     priv->progressive_sequence  = TRUE;
 766     return TRUE;
 767 }
 768
 769 static void
 770 fill_profiles(GstVaapiProfile profiles[16], guint *n_profiles_ptr,
 771     GstVaapiProfile profile)
 772 {
 773     guint n_profiles = *n_profiles_ptr;
 774
 775     profiles[n_profiles++] = profile;
 776     switch (profile) {
 777     case GST_VAAPI_PROFILE_H264_MAIN:
 778         profiles[n_profiles++] = GST_VAAPI_PROFILE_H264_HIGH;
 779         break;
 780     default:
 781         break;
 782     }
 783     *n_profiles_ptr = n_profiles;
 784 }
 785
 786 static GstVaapiProfile
 787 get_profile(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 788 {
 789     GstVaapiDecoderH264Private * const priv = &decoder->priv;
 790     GstVaapiDisplay * const display = GST_VAAPI_DECODER_DISPLAY(decoder);
 791     GstVaapiProfile profile, profiles[4];
 792     guint i, n_profiles = 0;
 793
 794     profile = gst_vaapi_utils_h264_get_profile(sps->profile_idc);
 795     if (!profile)
 796         return GST_VAAPI_PROFILE_UNKNOWN;
 797
 798     fill_profiles(profiles, &n_profiles, profile);
 799     switch (profile) {
 800     case GST_VAAPI_PROFILE_H264_BASELINE:
 801         if (sps->constraint_set1_flag) { // A.2.2 (main profile)
 802             fill_profiles(profiles, &n_profiles,
 803                 GST_VAAPI_PROFILE_H264_CONSTRAINED_BASELINE);
 804             fill_profiles(profiles, &n_profiles,
 805                 GST_VAAPI_PROFILE_H264_MAIN);
 806         }
 807         break;
 808     case GST_VAAPI_PROFILE_H264_EXTENDED:
 809         if (sps->constraint_set1_flag) { // A.2.2 (main profile)
 810             fill_profiles(profiles, &n_profiles,
 811                 GST_VAAPI_PROFILE_H264_MAIN);
 812         }
 813         break;
 814     default:
 815         break;
 816     }
 817
 818     /* If the preferred profile (profiles[0]) matches one that we already
 819        found, then just return it now instead of searching for it again */
 820     if (profiles[0] == priv->profile)
 821         return priv->profile;
 822
 823     for (i = 0; i < n_profiles; i++) {
 824         if (gst_vaapi_display_has_decoder(display, profiles[i], priv->entrypoint))
 825             return profiles[i];
 826     }
 827     return GST_VAAPI_PROFILE_UNKNOWN;
 828 }
 829
 830 static GstVaapiDecoderStatus
 831 ensure_context(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 832 {
 833     GstVaapiDecoder * const base_decoder = GST_VAAPI_DECODER_CAST(decoder);
 834     GstVaapiDecoderH264Private * const priv = &decoder->priv;
 835     GstVaapiContextInfo info;
 836     GstVaapiProfile profile;
 837     GstVaapiChromaType chroma_type;
 838     gboolean reset_context = FALSE;
 839     guint mb_width, mb_height;
 840
 841     profile = get_profile(decoder, sps);
 842     if (!profile) {
 843         GST_ERROR("unsupported profile_idc %u", sps->profile_idc);
 844         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_PROFILE;
 845     }
 846
 847     if (priv->profile != profile) {
 848         GST_DEBUG("profile changed");
 849         reset_context = TRUE;
 850         priv->profile = profile;
 851     }
 852
 853     chroma_type = gst_vaapi_utils_h264_get_chroma_type(sps->chroma_format_idc);
 854     if (!chroma_type) {
 855         GST_ERROR("unsupported chroma_format_idc %u", sps->chroma_format_idc);
 856         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;
 857     }
 858
 859     if (priv->chroma_type != chroma_type) {
 860         GST_DEBUG("chroma format changed");
 861         reset_context     = TRUE;
 862         priv->chroma_type = chroma_type;
 863     }
 864
 865     mb_width  = sps->pic_width_in_mbs_minus1 + 1;
 866     mb_height = (sps->pic_height_in_map_units_minus1 + 1) <<
 867         !sps->frame_mbs_only_flag;
 868     if (priv->mb_width != mb_width || priv->mb_height != mb_height) {
 869         GST_DEBUG("size changed");
 870         reset_context   = TRUE;
 871         priv->mb_width  = mb_width;
 872         priv->mb_height = mb_height;
 873     }
 874
 875     priv->progressive_sequence = sps->frame_mbs_only_flag;
 876 #if 0
 877     /* XXX: we only output complete frames for now */
 878     gst_vaapi_decoder_set_interlaced(base_decoder, !priv->progressive_sequence);
 879 #endif
 880
 881     gst_vaapi_decoder_set_pixel_aspect_ratio(
 882         base_decoder,
 883         sps->vui_parameters.par_n,
 884         sps->vui_parameters.par_d
 885     );
 886
 887     if (!reset_context && priv->has_context)
 888         return GST_VAAPI_DECODER_STATUS_SUCCESS;
 889
 890     /* XXX: fix surface size when cropping is implemented */
 891     info.profile    = priv->profile;
 892     info.entrypoint = priv->entrypoint;
 893     info.chroma_type = GST_VAAPI_CHROMA_TYPE_YUV420;
 894     info.width      = sps->width;
 895     info.height     = sps->height;
 896     info.ref_frames = get_max_dec_frame_buffering(sps);
 897
 898     if (!gst_vaapi_decoder_ensure_context(GST_VAAPI_DECODER(decoder), &info))
 899         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 900     priv->has_context = TRUE;
 901
 902     /* Reset DPB */
 903     dpb_reset(decoder, sps);
 904     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 905 }
 906
 907 static void
 908 fill_iq_matrix_4x4(VAIQMatrixBufferH264 *iq_matrix, const GstH264PPS *pps)
 909 {
 910     guint i;
 911
 912     /* There are always 6 4x4 scaling lists */
 913     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList4x4) == 6);
 914     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList4x4[0]) == 16);
 915
 916     for (i = 0; i < G_N_ELEMENTS(iq_matrix->ScalingList4x4); i++)
 917         gst_h264_video_quant_matrix_4x4_get_raster_from_zigzag(
 918             iq_matrix->ScalingList4x4[i], pps->scaling_lists_4x4[i]);
 919 }
 920
 921 static void
 922 fill_iq_matrix_8x8(VAIQMatrixBufferH264 *iq_matrix, const GstH264PPS *pps)
 923 {
 924     const GstH264SPS * const sps = pps->sequence;
 925     guint i, n;
 926
 927     /* If chroma_format_idc != 3, there are up to 2 8x8 scaling lists */
 928     if (!pps->transform_8x8_mode_flag)
 929         return;
 930
 931     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList8x8) >= 2);
 932     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList8x8[0]) == 64);
 933
 934     n = (sps->chroma_format_idc != 3) ? 2 : 6;
 935     for (i = 0; i < n; i++) {
 936         gst_h264_video_quant_matrix_8x8_get_raster_from_zigzag(
 937             iq_matrix->ScalingList8x8[i], pps->scaling_lists_8x8[i]);
 938     }
 939 }
 940
 941 static GstVaapiDecoderStatus
 942 ensure_quant_matrix(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
 943 {
 944     GstVaapiPicture * const base_picture = &picture->base;
 945     GstH264PPS * const pps = picture->pps;
 946     GstH264SPS * const sps = pps->sequence;
 947     VAIQMatrixBufferH264 *iq_matrix;
 948
 949     base_picture->iq_matrix = GST_VAAPI_IQ_MATRIX_NEW(H264, decoder);
 950     if (!base_picture->iq_matrix) {
 951         GST_ERROR("failed to allocate IQ matrix");
 952         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
 953     }
 954     iq_matrix = base_picture->iq_matrix->param;
 955
 956     /* XXX: we can only support 4:2:0 or 4:2:2 since ScalingLists8x8[]
 957        is not large enough to hold lists for 4:4:4 */
 958     if (sps->chroma_format_idc == 3)
 959         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;
 960
 961     fill_iq_matrix_4x4(iq_matrix, pps);
 962     fill_iq_matrix_8x8(iq_matrix, pps);
 963
 964     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 965 }
 966
 967 static inline gboolean
 968 is_valid_state(guint state, guint ref_state)
 969 {
 970     return (state & ref_state) == ref_state;
 971 }
 972
 973 static GstVaapiDecoderStatus
 974 decode_current_picture(GstVaapiDecoderH264 *decoder)
 975 {
 976     GstVaapiDecoderH264Private * const priv = &decoder->priv;
 977     GstVaapiPictureH264 * const picture = priv->current_picture;
 978
 979     if (!is_valid_state(priv->decoder_state, GST_H264_VIDEO_STATE_VALID_PICTURE))
 980         goto drop_frame;
 981     priv->decoder_state = 0;
 982
 983     if (!picture)
 984         return GST_VAAPI_DECODER_STATUS_SUCCESS;
 985
 986     if (!exec_ref_pic_marking(decoder, picture))
 987         goto error;
 988     if (!dpb_add(decoder, picture))
 989         goto error;
 990     if (!gst_vaapi_picture_decode(GST_VAAPI_PICTURE_CAST(picture)))
 991         goto error;
 992     if (priv->prev_frame && gst_vaapi_frame_store_has_frame(priv->prev_frame))
 993         gst_vaapi_picture_replace(&priv->current_picture, NULL);
 994     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 995
 996 error:
 997     /* XXX: fix for cases where first field failed to be decoded */
 998     gst_vaapi_picture_replace(&priv->current_picture, NULL);
 999     return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
1000
1001 drop_frame:
1002     priv->decoder_state = 0;
1003     return GST_VAAPI_DECODER_STATUS_DROP_FRAME;
1004 }
1005
1006 static GstVaapiDecoderStatus
1007 parse_sps(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnit *unit)
1008 {
1009     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1010     GstVaapiParserInfoH264 * const pi = unit->parsed_info;
1011     GstH264SPS * const sps = &pi->data.sps;
1012     GstH264ParserResult result;
1013
1014     GST_DEBUG("parse SPS");
1015
1016     priv->parser_state = 0;
1017
1018     /* Variables that don't have inferred values per the H.264
1019        standard but that should get a default value anyway */
1020     sps->log2_max_pic_order_cnt_lsb_minus4 = 0;
1021
1022     result = gst_h264_parser_parse_sps(priv->parser, &pi->nalu, sps, TRUE);
1023     if (result != GST_H264_PARSER_OK)
1024         return get_status(result);
1025
1026     priv->parser_state |= GST_H264_VIDEO_STATE_GOT_SPS;
1027     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1028 }
1029
1030 static GstVaapiDecoderStatus
1031 parse_pps(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnit *unit)
1032 {
1033     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1034     GstVaapiParserInfoH264 * const pi = unit->parsed_info;
1035     GstH264PPS * const pps = &pi->data.pps;
1036     GstH264ParserResult result;
1037
1038     GST_DEBUG("parse PPS");
1039
1040     priv->parser_state &= GST_H264_VIDEO_STATE_GOT_SPS;
1041
1042     /* Variables that don't have inferred values per the H.264
1043        standard but that should get a default value anyway */
1044     pps->slice_group_map_type = 0;
1045     pps->slice_group_change_rate_minus1 = 0;
1046
1047     result = gst_h264_parser_parse_pps(priv->parser, &pi->nalu, pps);
1048     if (result != GST_H264_PARSER_OK)
1049         return get_status(result);
1050
1051     priv->parser_state |= GST_H264_VIDEO_STATE_GOT_PPS;
1052     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1053 }
1054
1055 static GstVaapiDecoderStatus
1056 parse_sei(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnit *unit)
1057 {
1058     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1059     GstVaapiParserInfoH264 * const pi = unit->parsed_info;
1060     GArray *sei_messages = NULL;
1061     GstH264ParserResult result;
1062
1063     GST_DEBUG("parse SEI");
1064
1065     result = gst_h264_parser_parse_sei(priv->parser, &pi->nalu, &sei_messages);
1066     if (result != GST_H264_PARSER_OK) {
1067         GST_WARNING("failed to parse SEI messages");
1068         g_array_unref(sei_messages);
1069         return get_status(result);
1070     }
1071
1072     g_array_unref(sei_messages);
1073     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1074 }
1075
1076 static GstVaapiDecoderStatus
1077 parse_slice(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnit *unit)
1078 {
1079     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1080     GstVaapiParserInfoH264 * const pi = unit->parsed_info;
1081     GstH264SliceHdr * const slice_hdr = &pi->data.slice_hdr;
1082     GstH264ParserResult result;
1083
1084     GST_DEBUG("parse slice");
1085
1086     priv->parser_state &= (GST_H264_VIDEO_STATE_GOT_SPS|
1087                            GST_H264_VIDEO_STATE_GOT_PPS);
1088
1089     /* Variables that don't have inferred values per the H.264
1090        standard but that should get a default value anyway */
1091     slice_hdr->cabac_init_idc = 0;
1092     slice_hdr->direct_spatial_mv_pred_flag = 0;
1093
1094     result = gst_h264_parser_parse_slice_hdr(priv->parser, &pi->nalu,
1095         slice_hdr, TRUE, TRUE);
1096     if (result != GST_H264_PARSER_OK)
1097         return get_status(result);
1098
1099     priv->parser_state |= GST_H264_VIDEO_STATE_GOT_SLICE;
1100     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1101 }
1102
1103 static GstVaapiDecoderStatus
1104 decode_sequence_end(GstVaapiDecoderH264 *decoder)
1105 {
1106     GstVaapiDecoderStatus status;
1107
1108     GST_DEBUG("decode sequence-end");
1109
1110     status = decode_current_picture(decoder);
1111     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
1112         return status;
1113
1114     dpb_flush(decoder);
1115     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1116 }
1117
1118 /* 8.2.1.1 - Decoding process for picture order count type 0 */
1119 static void
1120 init_picture_poc_0(
1121     GstVaapiDecoderH264 *decoder,
1122     GstVaapiPictureH264 *picture,
1123     GstH264SliceHdr     *slice_hdr
1124 )
1125 {
1126     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1127     GstH264PPS * const pps = slice_hdr->pps;
1128     GstH264SPS * const sps = pps->sequence;
1129     const gint32 MaxPicOrderCntLsb = 1 << (sps->log2_max_pic_order_cnt_lsb_minus4 + 4);
1130     gint32 temp_poc;
1131
1132     GST_DEBUG("decode picture order count type 0");
1133
1134     if (GST_VAAPI_PICTURE_IS_IDR(picture)) {
1135         priv->prev_poc_msb = 0;
1136         priv->prev_poc_lsb = 0;
1137     }
1138     else if (priv->prev_pic_has_mmco5) {
1139         priv->prev_poc_msb = 0;
1140         priv->prev_poc_lsb =
1141             (priv->prev_pic_structure == GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD ?
1142              0 : priv->field_poc[TOP_FIELD]);
1143     }
1144     else {
1145         priv->prev_poc_msb = priv->poc_msb;
1146         priv->prev_poc_lsb = priv->poc_lsb;
1147     }
1148
1149     // (8-3)
1150     priv->poc_lsb = slice_hdr->pic_order_cnt_lsb;
1151     if (priv->poc_lsb < priv->prev_poc_lsb &&
1152         (priv->prev_poc_lsb - priv->poc_lsb) >= (MaxPicOrderCntLsb / 2))
1153         priv->poc_msb = priv->prev_poc_msb + MaxPicOrderCntLsb;
1154     else if (priv->poc_lsb > priv->prev_poc_lsb &&
1155              (priv->poc_lsb - priv->prev_poc_lsb) > (MaxPicOrderCntLsb / 2))
1156         priv->poc_msb = priv->prev_poc_msb - MaxPicOrderCntLsb;
1157     else
1158         priv->poc_msb = priv->prev_poc_msb;
1159
1160     temp_poc = priv->poc_msb + priv->poc_lsb;
1161     switch (picture->structure) {
1162     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
1163         // (8-4, 8-5)
1164         priv->field_poc[TOP_FIELD] = temp_poc;
1165         priv->field_poc[BOTTOM_FIELD] = temp_poc +
1166             slice_hdr->delta_pic_order_cnt_bottom;
1167         break;
1168     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
1169         // (8-4)
1170         priv->field_poc[TOP_FIELD] = temp_poc;
1171         break;
1172     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
1173         // (8-5)
1174         priv->field_poc[BOTTOM_FIELD] = temp_poc;
1175         break;
1176     }
1177 }
1178
1179 /* 8.2.1.2 - Decoding process for picture order count type 1 */
1180 static void
1181 init_picture_poc_1(
1182     GstVaapiDecoderH264 *decoder,
1183     GstVaapiPictureH264 *picture,
1184     GstH264SliceHdr     *slice_hdr
1185 )
1186 {
1187     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1188     GstH264PPS * const pps = slice_hdr->pps;
1189     GstH264SPS * const sps = pps->sequence;
1190     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1191     gint32 prev_frame_num_offset, abs_frame_num, expected_poc;
1192     guint i;
1193
1194     GST_DEBUG("decode picture order count type 1");
1195
1196     if (priv->prev_pic_has_mmco5)
1197         prev_frame_num_offset = 0;
1198     else
1199         prev_frame_num_offset = priv->frame_num_offset;
1200
1201     // (8-6)
1202     if (GST_VAAPI_PICTURE_IS_IDR(picture))
1203         priv->frame_num_offset = 0;
1204     else if (priv->prev_frame_num > priv->frame_num)
1205         priv->frame_num_offset = prev_frame_num_offset + MaxFrameNum;
1206     else
1207         priv->frame_num_offset = prev_frame_num_offset;
1208
1209     // (8-7)
1210     if (sps->num_ref_frames_in_pic_order_cnt_cycle != 0)
1211         abs_frame_num = priv->frame_num_offset + priv->frame_num;
1212     else
1213         abs_frame_num = 0;
1214     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture) && abs_frame_num > 0)
1215         abs_frame_num = abs_frame_num - 1;
1216
1217     if (abs_frame_num > 0) {
1218         gint32 expected_delta_per_poc_cycle;
1219         gint32 poc_cycle_cnt, frame_num_in_poc_cycle;
1220
1221         expected_delta_per_poc_cycle = 0;
1222         for (i = 0; i < sps->num_ref_frames_in_pic_order_cnt_cycle; i++)
1223             expected_delta_per_poc_cycle += sps->offset_for_ref_frame[i];
1224
1225         // (8-8)
1226         poc_cycle_cnt = (abs_frame_num - 1) /
1227             sps->num_ref_frames_in_pic_order_cnt_cycle;
1228         frame_num_in_poc_cycle = (abs_frame_num - 1) %
1229             sps->num_ref_frames_in_pic_order_cnt_cycle;
1230
1231         // (8-9)
1232         expected_poc = poc_cycle_cnt * expected_delta_per_poc_cycle;
1233         for (i = 0; i <= frame_num_in_poc_cycle; i++)
1234             expected_poc += sps->offset_for_ref_frame[i];
1235     }
1236     else
1237         expected_poc = 0;
1238     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
1239         expected_poc += sps->offset_for_non_ref_pic;
1240
1241     // (8-10)
1242     switch (picture->structure) {
1243     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
1244         priv->field_poc[TOP_FIELD] = expected_poc +
1245             slice_hdr->delta_pic_order_cnt[0];
1246         priv->field_poc[BOTTOM_FIELD] = priv->field_poc[TOP_FIELD] +
1247             sps->offset_for_top_to_bottom_field +
1248             slice_hdr->delta_pic_order_cnt[1];
1249         break;
1250     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
1251         priv->field_poc[TOP_FIELD] = expected_poc +
1252             slice_hdr->delta_pic_order_cnt[0];
1253         break;
1254     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
1255         priv->field_poc[BOTTOM_FIELD] = expected_poc +
1256             sps->offset_for_top_to_bottom_field +
1257             slice_hdr->delta_pic_order_cnt[0];
1258         break;
1259     }
1260 }
1261
1262 /* 8.2.1.3 - Decoding process for picture order count type 2 */
1263 static void
1264 init_picture_poc_2(
1265     GstVaapiDecoderH264 *decoder,
1266     GstVaapiPictureH264 *picture,
1267     GstH264SliceHdr     *slice_hdr
1268 )
1269 {
1270     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1271     GstH264PPS * const pps = slice_hdr->pps;
1272     GstH264SPS * const sps = pps->sequence;
1273     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1274     gint32 prev_frame_num_offset, temp_poc;
1275
1276     GST_DEBUG("decode picture order count type 2");
1277
1278     if (priv->prev_pic_has_mmco5)
1279         prev_frame_num_offset = 0;
1280     else
1281         prev_frame_num_offset = priv->frame_num_offset;
1282
1283     // (8-11)
1284     if (GST_VAAPI_PICTURE_IS_IDR(picture))
1285         priv->frame_num_offset = 0;
1286     else if (priv->prev_frame_num > priv->frame_num)
1287         priv->frame_num_offset = prev_frame_num_offset + MaxFrameNum;
1288     else
1289         priv->frame_num_offset = prev_frame_num_offset;
1290
1291     // (8-12)
1292     if (GST_VAAPI_PICTURE_IS_IDR(picture))
1293         temp_poc = 0;
1294     else if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
1295         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num) - 1;
1296     else
1297         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num);
1298
1299     // (8-13)
1300     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
1301         priv->field_poc[TOP_FIELD] = temp_poc;
1302     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
1303         priv->field_poc[BOTTOM_FIELD] = temp_poc;
1304 }
1305
1306 /* 8.2.1 - Decoding process for picture order count */
1307 static void
1308 init_picture_poc(
1309     GstVaapiDecoderH264 *decoder,
1310     GstVaapiPictureH264 *picture,
1311     GstH264SliceHdr     *slice_hdr
1312 )
1313 {
1314     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1315     GstH264PPS * const pps = slice_hdr->pps;
1316     GstH264SPS * const sps = pps->sequence;
1317
1318     switch (sps->pic_order_cnt_type) {
1319     case 0:
1320         init_picture_poc_0(decoder, picture, slice_hdr);
1321         break;
1322     case 1:
1323         init_picture_poc_1(decoder, picture, slice_hdr);
1324         break;
1325     case 2:
1326         init_picture_poc_2(decoder, picture, slice_hdr);
1327         break;
1328     }
1329
1330     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
1331         picture->field_poc[TOP_FIELD] = priv->field_poc[TOP_FIELD];
1332     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
1333         picture->field_poc[BOTTOM_FIELD] = priv->field_poc[BOTTOM_FIELD];
1334     picture->base.poc = MIN(picture->field_poc[0], picture->field_poc[1]);
1335 }
1336
1337 static int
1338 compare_picture_pic_num_dec(const void *a, const void *b)
1339 {
1340     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1341     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1342
1343     return picB->pic_num - picA->pic_num;
1344 }
1345
1346 static int
1347 compare_picture_long_term_pic_num_inc(const void *a, const void *b)
1348 {
1349     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1350     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1351
1352     return picA->long_term_pic_num - picB->long_term_pic_num;
1353 }
1354
1355 static int
1356 compare_picture_poc_dec(const void *a, const void *b)
1357 {
1358     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1359     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1360
1361     return picB->base.poc - picA->base.poc;
1362 }
1363
1364 static int
1365 compare_picture_poc_inc(const void *a, const void *b)
1366 {
1367     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1368     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1369
1370     return picA->base.poc - picB->base.poc;
1371 }
1372
1373 static int
1374 compare_picture_frame_num_wrap_dec(const void *a, const void *b)
1375 {
1376     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1377     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1378
1379     return picB->frame_num_wrap - picA->frame_num_wrap;
1380 }
1381
1382 static int
1383 compare_picture_long_term_frame_idx_inc(const void *a, const void *b)
1384 {
1385     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1386     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1387
1388     return picA->long_term_frame_idx - picB->long_term_frame_idx;
1389 }
1390
1391 /* 8.2.4.1 - Decoding process for picture numbers */
1392 static void
1393 init_picture_refs_pic_num(
1394     GstVaapiDecoderH264 *decoder,
1395     GstVaapiPictureH264 *picture,
1396     GstH264SliceHdr     *slice_hdr
1397 )
1398 {
1399     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1400     GstH264PPS * const pps = slice_hdr->pps;
1401     GstH264SPS * const sps = pps->sequence;
1402     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1403     guint i;
1404
1405     GST_DEBUG("decode picture numbers");
1406
1407     for (i = 0; i < priv->short_ref_count; i++) {
1408         GstVaapiPictureH264 * const pic = priv->short_ref[i];
1409
1410         // (8-27)
1411         if (pic->frame_num > priv->frame_num)
1412             pic->frame_num_wrap = pic->frame_num - MaxFrameNum;
1413         else
1414             pic->frame_num_wrap = pic->frame_num;
1415
1416         // (8-28, 8-30, 8-31)
1417         if (GST_VAAPI_PICTURE_IS_FRAME(picture))
1418             pic->pic_num = pic->frame_num_wrap;
1419         else {
1420             if (pic->structure == picture->structure)
1421                 pic->pic_num = 2 * pic->frame_num_wrap + 1;
1422             else
1423                 pic->pic_num = 2 * pic->frame_num_wrap;
1424         }
1425     }
1426
1427     for (i = 0; i < priv->long_ref_count; i++) {
1428         GstVaapiPictureH264 * const pic = priv->long_ref[i];
1429
1430         // (8-29, 8-32, 8-33)
1431         if (GST_VAAPI_PICTURE_IS_FRAME(picture))
1432             pic->long_term_pic_num = pic->long_term_frame_idx;
1433         else {
1434             if (pic->structure == picture->structure)
1435                 pic->long_term_pic_num = 2 * pic->long_term_frame_idx + 1;
1436             else
1437                 pic->long_term_pic_num = 2 * pic->long_term_frame_idx;
1438         }
1439     }
1440 }
1441
1442 #define SORT_REF_LIST(list, n, compare_func) \
1443     qsort(list, n, sizeof(*(list)), compare_picture_##compare_func)
1444
1445 static void
1446 init_picture_refs_fields_1(
1447     guint                picture_structure,
1448     GstVaapiPictureH264 *RefPicList[32],
1449     guint               *RefPicList_count,
1450     GstVaapiPictureH264 *ref_list[32],
1451     guint                ref_list_count
1452 )
1453 {
1454     guint i, j, n;
1455
1456     i = 0;
1457     j = 0;
1458     n = *RefPicList_count;
1459     do {
1460         g_assert(n < 32);
1461         for (; i < ref_list_count; i++) {
1462             if (ref_list[i]->structure == picture_structure) {
1463                 RefPicList[n++] = ref_list[i++];
1464                 break;
1465             }
1466         }
1467         for (; j < ref_list_count; j++) {
1468             if (ref_list[j]->structure != picture_structure) {
1469                 RefPicList[n++] = ref_list[j++];
1470                 break;
1471             }
1472         }
1473     } while (i < ref_list_count || j < ref_list_count);
1474     *RefPicList_count = n;
1475 }
1476
1477 static inline void
1478 init_picture_refs_fields(
1479     GstVaapiPictureH264 *picture,
1480     GstVaapiPictureH264 *RefPicList[32],
1481     guint               *RefPicList_count,
1482     GstVaapiPictureH264 *short_ref[32],
1483     guint                short_ref_count,
1484     GstVaapiPictureH264 *long_ref[32],
1485     guint                long_ref_count
1486 )
1487 {
1488     guint n = 0;
1489
1490     /* 8.2.4.2.5 - reference picture lists in fields */
1491     init_picture_refs_fields_1(picture->structure, RefPicList, &n,
1492         short_ref, short_ref_count);
1493     init_picture_refs_fields_1(picture->structure, RefPicList, &n,
1494         long_ref, long_ref_count);
1495     *RefPicList_count = n;
1496 }
1497
1498 static void
1499 init_picture_refs_p_slice(
1500     GstVaapiDecoderH264 *decoder,
1501     GstVaapiPictureH264 *picture,
1502     GstH264SliceHdr     *slice_hdr
1503 )
1504 {
1505     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1506     GstVaapiPictureH264 **ref_list;
1507     guint i;
1508
1509     GST_DEBUG("decode reference picture list for P and SP slices");
1510
1511     if (GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1512         /* 8.2.4.2.1 - P and SP slices in frames */
1513         if (priv->short_ref_count > 0) {
1514             ref_list = priv->RefPicList0;
1515             for (i = 0; i < priv->short_ref_count; i++)
1516                 ref_list[i] = priv->short_ref[i];
1517             SORT_REF_LIST(ref_list, i, pic_num_dec);
1518             priv->RefPicList0_count += i;
1519         }
1520
1521         if (priv->long_ref_count > 0) {
1522             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1523             for (i = 0; i < priv->long_ref_count; i++)
1524                 ref_list[i] = priv->long_ref[i];
1525             SORT_REF_LIST(ref_list, i, long_term_pic_num_inc);
1526             priv->RefPicList0_count += i;
1527         }
1528     }
1529     else {
1530         /* 8.2.4.2.2 - P and SP slices in fields */
1531         GstVaapiPictureH264 *short_ref[32];
1532         guint short_ref_count = 0;
1533         GstVaapiPictureH264 *long_ref[32];
1534         guint long_ref_count = 0;
1535
1536         if (priv->short_ref_count > 0) {
1537             for (i = 0; i < priv->short_ref_count; i++)
1538                 short_ref[i] = priv->short_ref[i];
1539             SORT_REF_LIST(short_ref, i, frame_num_wrap_dec);
1540             short_ref_count = i;
1541         }
1542
1543         if (priv->long_ref_count > 0) {
1544             for (i = 0; i < priv->long_ref_count; i++)
1545                 long_ref[i] = priv->long_ref[i];
1546             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1547             long_ref_count = i;
1548         }
1549
1550         init_picture_refs_fields(
1551             picture,
1552             priv->RefPicList0, &priv->RefPicList0_count,
1553             short_ref,          short_ref_count,
1554             long_ref,           long_ref_count
1555         );
1556     }
1557 }
1558
1559 static void
1560 init_picture_refs_b_slice(
1561     GstVaapiDecoderH264 *decoder,
1562     GstVaapiPictureH264 *picture,
1563     GstH264SliceHdr     *slice_hdr
1564 )
1565 {
1566     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1567     GstVaapiPictureH264 **ref_list;
1568     guint i, n;
1569
1570     GST_DEBUG("decode reference picture list for B slices");
1571
1572     if (GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1573         /* 8.2.4.2.3 - B slices in frames */
1574
1575         /* RefPicList0 */
1576         if (priv->short_ref_count > 0) {
1577             // 1. Short-term references
1578             ref_list = priv->RefPicList0;
1579             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1580                 if (priv->short_ref[i]->base.poc < picture->base.poc)
1581                     ref_list[n++] = priv->short_ref[i];
1582             }
1583             SORT_REF_LIST(ref_list, n, poc_dec);
1584             priv->RefPicList0_count += n;
1585
1586             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1587             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1588                 if (priv->short_ref[i]->base.poc >= picture->base.poc)
1589                     ref_list[n++] = priv->short_ref[i];
1590             }
1591             SORT_REF_LIST(ref_list, n, poc_inc);
1592             priv->RefPicList0_count += n;
1593         }
1594
1595         if (priv->long_ref_count > 0) {
1596             // 2. Long-term references
1597             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1598             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1599                 ref_list[n++] = priv->long_ref[i];
1600             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1601             priv->RefPicList0_count += n;
1602         }
1603
1604         /* RefPicList1 */
1605         if (priv->short_ref_count > 0) {
1606             // 1. Short-term references
1607             ref_list = priv->RefPicList1;
1608             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1609                 if (priv->short_ref[i]->base.poc > picture->base.poc)
1610                     ref_list[n++] = priv->short_ref[i];
1611             }
1612             SORT_REF_LIST(ref_list, n, poc_inc);
1613             priv->RefPicList1_count += n;
1614
1615             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1616             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1617                 if (priv->short_ref[i]->base.poc <= picture->base.poc)
1618                     ref_list[n++] = priv->short_ref[i];
1619             }
1620             SORT_REF_LIST(ref_list, n, poc_dec);
1621             priv->RefPicList1_count += n;
1622         }
1623
1624         if (priv->long_ref_count > 0) {
1625             // 2. Long-term references
1626             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1627             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1628                 ref_list[n++] = priv->long_ref[i];
1629             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1630             priv->RefPicList1_count += n;
1631         }
1632     }
1633     else {
1634         /* 8.2.4.2.4 - B slices in fields */
1635         GstVaapiPictureH264 *short_ref0[32];
1636         guint short_ref0_count = 0;
1637         GstVaapiPictureH264 *short_ref1[32];
1638         guint short_ref1_count = 0;
1639         GstVaapiPictureH264 *long_ref[32];
1640         guint long_ref_count = 0;
1641
1642         /* refFrameList0ShortTerm */
1643         if (priv->short_ref_count > 0) {
1644             ref_list = short_ref0;
1645             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1646                 if (priv->short_ref[i]->base.poc <= picture->base.poc)
1647                     ref_list[n++] = priv->short_ref[i];
1648             }
1649             SORT_REF_LIST(ref_list, n, poc_dec);
1650             short_ref0_count += n;
1651
1652             ref_list = &short_ref0[short_ref0_count];
1653             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1654                 if (priv->short_ref[i]->base.poc > picture->base.poc)
1655                     ref_list[n++] = priv->short_ref[i];
1656             }
1657             SORT_REF_LIST(ref_list, n, poc_inc);
1658             short_ref0_count += n;
1659         }
1660
1661         /* refFrameList1ShortTerm */
1662         if (priv->short_ref_count > 0) {
1663             ref_list = short_ref1;
1664             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1665                 if (priv->short_ref[i]->base.poc > picture->base.poc)
1666                     ref_list[n++] = priv->short_ref[i];
1667             }
1668             SORT_REF_LIST(ref_list, n, poc_inc);
1669             short_ref1_count += n;
1670
1671             ref_list = &short_ref1[short_ref1_count];
1672             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1673                 if (priv->short_ref[i]->base.poc <= picture->base.poc)
1674                     ref_list[n++] = priv->short_ref[i];
1675             }
1676             SORT_REF_LIST(ref_list, n, poc_dec);
1677             short_ref1_count += n;
1678         }
1679
1680         /* refFrameListLongTerm */
1681         if (priv->long_ref_count > 0) {
1682             for (i = 0; i < priv->long_ref_count; i++)
1683                 long_ref[i] = priv->long_ref[i];
1684             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1685             long_ref_count = i;
1686         }
1687
1688         init_picture_refs_fields(
1689             picture,
1690             priv->RefPicList0, &priv->RefPicList0_count,
1691             short_ref0,         short_ref0_count,
1692             long_ref,           long_ref_count
1693         );
1694
1695         init_picture_refs_fields(
1696             picture,
1697             priv->RefPicList1, &priv->RefPicList1_count,
1698             short_ref1,         short_ref1_count,
1699             long_ref,           long_ref_count
1700         );
1701    }
1702
1703     /* Check whether RefPicList1 is identical to RefPicList0, then
1704        swap if necessary */
1705     if (priv->RefPicList1_count > 1 &&
1706         priv->RefPicList1_count == priv->RefPicList0_count &&
1707         memcmp(priv->RefPicList0, priv->RefPicList1,
1708                priv->RefPicList0_count * sizeof(priv->RefPicList0[0])) == 0) {
1709         GstVaapiPictureH264 * const tmp = priv->RefPicList1[0];
1710         priv->RefPicList1[0] = priv->RefPicList1[1];
1711         priv->RefPicList1[1] = tmp;
1712     }
1713 }
1714
1715 #undef SORT_REF_LIST
1716
1717 static gint
1718 find_short_term_reference(GstVaapiDecoderH264 *decoder, gint32 pic_num)
1719 {
1720     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1721     guint i;
1722
1723     for (i = 0; i < priv->short_ref_count; i++) {
1724         if (priv->short_ref[i]->pic_num == pic_num)
1725             return i;
1726     }
1727     GST_ERROR("found no short-term reference picture with PicNum = %d",
1728               pic_num);
1729     return -1;
1730 }
1731
1732 static gint
1733 find_long_term_reference(GstVaapiDecoderH264 *decoder, gint32 long_term_pic_num)
1734 {
1735     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1736     guint i;
1737
1738     for (i = 0; i < priv->long_ref_count; i++) {
1739         if (priv->long_ref[i]->long_term_pic_num == long_term_pic_num)
1740             return i;
1741     }
1742     GST_ERROR("found no long-term reference picture with LongTermPicNum = %d",
1743               long_term_pic_num);
1744     return -1;
1745 }
1746
1747 static void
1748 exec_picture_refs_modification_1(
1749     GstVaapiDecoderH264           *decoder,
1750     GstVaapiPictureH264           *picture,
1751     GstH264SliceHdr               *slice_hdr,
1752     guint                          list
1753 )
1754 {
1755     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1756     GstH264PPS * const pps = slice_hdr->pps;
1757     GstH264SPS * const sps = pps->sequence;
1758     GstH264RefPicListModification *ref_pic_list_modification;
1759     guint num_ref_pic_list_modifications;
1760     GstVaapiPictureH264 **ref_list;
1761     guint *ref_list_count_ptr, ref_list_count, ref_list_idx = 0;
1762     guint i, j, n, num_refs;
1763     gint found_ref_idx;
1764     gint32 MaxPicNum, CurrPicNum, picNumPred;
1765
1766     GST_DEBUG("modification process of reference picture list %u", list);
1767
1768     if (list == 0) {
1769         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l0;
1770         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l0;
1771         ref_list                       = priv->RefPicList0;
1772         ref_list_count_ptr             = &priv->RefPicList0_count;
1773         num_refs                       = slice_hdr->num_ref_idx_l0_active_minus1 + 1;
1774     }
1775     else {
1776         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l1;
1777         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l1;
1778         ref_list                       = priv->RefPicList1;
1779         ref_list_count_ptr             = &priv->RefPicList1_count;
1780         num_refs                       = slice_hdr->num_ref_idx_l1_active_minus1 + 1;
1781     }
1782     ref_list_count = *ref_list_count_ptr;
1783
1784     if (!GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1785         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 5); // 2 * MaxFrameNum
1786         CurrPicNum = 2 * slice_hdr->frame_num + 1;              // 2 * frame_num + 1
1787     }
1788     else {
1789         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 4); // MaxFrameNum
1790         CurrPicNum = slice_hdr->frame_num;                      // frame_num
1791     }
1792
1793     picNumPred = CurrPicNum;
1794
1795     for (i = 0; i < num_ref_pic_list_modifications; i++) {
1796         GstH264RefPicListModification * const l = &ref_pic_list_modification[i];
1797         if (l->modification_of_pic_nums_idc == 3)
1798             break;
1799
1800         /* 8.2.4.3.1 - Short-term reference pictures */
1801         if (l->modification_of_pic_nums_idc == 0 || l->modification_of_pic_nums_idc == 1) {
1802             gint32 abs_diff_pic_num = l->value.abs_diff_pic_num_minus1 + 1;
1803             gint32 picNum, picNumNoWrap;
1804
1805             // (8-34)
1806             if (l->modification_of_pic_nums_idc == 0) {
1807                 picNumNoWrap = picNumPred - abs_diff_pic_num;
1808                 if (picNumNoWrap < 0)
1809                     picNumNoWrap += MaxPicNum;
1810             }
1811
1812             // (8-35)
1813             else {
1814                 picNumNoWrap = picNumPred + abs_diff_pic_num;
1815                 if (picNumNoWrap >= MaxPicNum)
1816                     picNumNoWrap -= MaxPicNum;
1817             }
1818             picNumPred = picNumNoWrap;
1819
1820             // (8-36)
1821             picNum = picNumNoWrap;
1822             if (picNum > CurrPicNum)
1823                 picNum -= MaxPicNum;
1824
1825             // (8-37)
1826             for (j = num_refs; j > ref_list_idx; j--)
1827                 ref_list[j] = ref_list[j - 1];
1828             found_ref_idx = find_short_term_reference(decoder, picNum);
1829             ref_list[ref_list_idx++] =
1830                 found_ref_idx >= 0 ? priv->short_ref[found_ref_idx] : NULL;
1831             n = ref_list_idx;
1832             for (j = ref_list_idx; j <= num_refs; j++) {
1833                 gint32 PicNumF;
1834                 if (!ref_list[j])
1835                     continue;
1836                 PicNumF =
1837                     GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(ref_list[j]) ?
1838                     ref_list[j]->pic_num : MaxPicNum;
1839                 if (PicNumF != picNum)
1840                     ref_list[n++] = ref_list[j];
1841             }
1842         }
1843
1844         /* 8.2.4.3.2 - Long-term reference pictures */
1845         else {
1846
1847             for (j = num_refs; j > ref_list_idx; j--)
1848                 ref_list[j] = ref_list[j - 1];
1849             found_ref_idx =
1850                 find_long_term_reference(decoder, l->value.long_term_pic_num);
1851             ref_list[ref_list_idx++] =
1852                 found_ref_idx >= 0 ? priv->long_ref[found_ref_idx] : NULL;
1853             n = ref_list_idx;
1854             for (j = ref_list_idx; j <= num_refs; j++) {
1855                 gint32 LongTermPicNumF;
1856                 if (!ref_list[j])
1857                     continue;
1858                 LongTermPicNumF =
1859                     GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(ref_list[j]) ?
1860                     ref_list[j]->long_term_pic_num : INT_MAX;
1861                 if (LongTermPicNumF != l->value.long_term_pic_num)
1862                     ref_list[n++] = ref_list[j];
1863             }
1864         }
1865     }
1866
1867 #if DEBUG
1868     for (i = 0; i < num_refs; i++)
1869         if (!ref_list[i])
1870             GST_ERROR("list %u entry %u is empty", list, i);
1871 #endif
1872     *ref_list_count_ptr = num_refs;
1873 }
1874
1875 /* 8.2.4.3 - Modification process for reference picture lists */
1876 static void
1877 exec_picture_refs_modification(
1878     GstVaapiDecoderH264 *decoder,
1879     GstVaapiPictureH264 *picture,
1880     GstH264SliceHdr     *slice_hdr
1881 )
1882 {
1883     GST_DEBUG("execute ref_pic_list_modification()");
1884
1885     /* RefPicList0 */
1886     if (!GST_H264_IS_I_SLICE(slice_hdr) && !GST_H264_IS_SI_SLICE(slice_hdr) &&
1887         slice_hdr->ref_pic_list_modification_flag_l0)
1888         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 0);
1889
1890     /* RefPicList1 */
1891     if (GST_H264_IS_B_SLICE(slice_hdr) &&
1892         slice_hdr->ref_pic_list_modification_flag_l1)
1893         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 1);
1894 }
1895
1896 static void
1897 init_picture_ref_lists(GstVaapiDecoderH264 *decoder)
1898 {
1899     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1900     guint i, j, short_ref_count, long_ref_count;
1901
1902     short_ref_count = 0;
1903     long_ref_count  = 0;
1904     if (GST_VAAPI_PICTURE_IS_FRAME(priv->current_picture)) {
1905         for (i = 0; i < priv->dpb_count; i++) {
1906             GstVaapiFrameStore * const fs = priv->dpb[i];
1907             GstVaapiPictureH264 *picture;
1908             if (!gst_vaapi_frame_store_has_frame(fs))
1909                 continue;
1910             picture = fs->buffers[0];
1911             if (GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture))
1912                 priv->short_ref[short_ref_count++] = picture;
1913             else if (GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture))
1914                 priv->long_ref[long_ref_count++] = picture;
1915             picture->structure = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
1916             picture->other_field = fs->buffers[1];
1917         }
1918     }
1919     else {
1920         for (i = 0; i < priv->dpb_count; i++) {
1921             GstVaapiFrameStore * const fs = priv->dpb[i];
1922             for (j = 0; j < fs->num_buffers; j++) {
1923                 GstVaapiPictureH264 * const picture = fs->buffers[j];
1924                 if (GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture))
1925                     priv->short_ref[short_ref_count++] = picture;
1926                 else if (GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture))
1927                     priv->long_ref[long_ref_count++] = picture;
1928                 picture->structure = picture->base.structure;
1929                 picture->other_field = fs->buffers[j ^ 1];
1930             }
1931         }
1932     }
1933
1934     for (i = short_ref_count; i < priv->short_ref_count; i++)
1935         priv->short_ref[i] = NULL;
1936     priv->short_ref_count = short_ref_count;
1937
1938     for (i = long_ref_count; i < priv->long_ref_count; i++)
1939         priv->long_ref[i] = NULL;
1940     priv->long_ref_count = long_ref_count;
1941 }
1942
1943 static void
1944 init_picture_refs(
1945     GstVaapiDecoderH264 *decoder,
1946     GstVaapiPictureH264 *picture,
1947     GstH264SliceHdr     *slice_hdr
1948 )
1949 {
1950     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1951     GstVaapiPicture * const base_picture = &picture->base;
1952     guint i, num_refs;
1953
1954     init_picture_ref_lists(decoder);
1955     init_picture_refs_pic_num(decoder, picture, slice_hdr);
1956
1957     priv->RefPicList0_count = 0;
1958     priv->RefPicList1_count = 0;
1959
1960     switch (base_picture->type) {
1961     case GST_VAAPI_PICTURE_TYPE_P:
1962     case GST_VAAPI_PICTURE_TYPE_SP:
1963         init_picture_refs_p_slice(decoder, picture, slice_hdr);
1964         break;
1965     case GST_VAAPI_PICTURE_TYPE_B:
1966         init_picture_refs_b_slice(decoder, picture, slice_hdr);
1967         break;
1968     default:
1969         break;
1970     }
1971
1972     exec_picture_refs_modification(decoder, picture, slice_hdr);
1973
1974     switch (base_picture->type) {
1975     case GST_VAAPI_PICTURE_TYPE_B:
1976         num_refs = 1 + slice_hdr->num_ref_idx_l1_active_minus1;
1977         for (i = priv->RefPicList1_count; i < num_refs; i++)
1978             priv->RefPicList1[i] = NULL;
1979         priv->RefPicList1_count = num_refs;
1980
1981         // fall-through
1982     case GST_VAAPI_PICTURE_TYPE_P:
1983     case GST_VAAPI_PICTURE_TYPE_SP:
1984         num_refs = 1 + slice_hdr->num_ref_idx_l0_active_minus1;
1985         for (i = priv->RefPicList0_count; i < num_refs; i++)
1986             priv->RefPicList0[i] = NULL;
1987         priv->RefPicList0_count = num_refs;
1988         break;
1989     default:
1990         break;
1991     }
1992 }
1993
1994 static gboolean
1995 init_picture(
1996     GstVaapiDecoderH264 *decoder,
1997     GstVaapiPictureH264 *picture, GstVaapiParserInfoH264 *pi)
1998 {
1999     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2000     GstVaapiPicture * const base_picture = &picture->base;
2001     GstH264SliceHdr * const slice_hdr = &pi->data.slice_hdr;
2002
2003     priv->prev_frame_num        = priv->frame_num;
2004     priv->frame_num             = slice_hdr->frame_num;
2005     picture->frame_num          = priv->frame_num;
2006     picture->frame_num_wrap     = priv->frame_num;
2007     picture->output_flag        = TRUE; /* XXX: conformant to Annex A only */
2008     base_picture->pts           = GST_VAAPI_DECODER_CODEC_FRAME(decoder)->pts;
2009
2010     /* Reset decoder state for IDR pictures */
2011     if (pi->nalu.type == GST_H264_NAL_SLICE_IDR) {
2012         GST_DEBUG("<IDR>");
2013         GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_IDR);
2014         dpb_flush(decoder);
2015     }
2016
2017     /* Initialize slice type */
2018     switch (slice_hdr->type % 5) {
2019     case GST_H264_P_SLICE:
2020         base_picture->type = GST_VAAPI_PICTURE_TYPE_P;
2021         break;
2022     case GST_H264_B_SLICE:
2023         base_picture->type = GST_VAAPI_PICTURE_TYPE_B;
2024         break;
2025     case GST_H264_I_SLICE:
2026         base_picture->type = GST_VAAPI_PICTURE_TYPE_I;
2027         break;
2028     case GST_H264_SP_SLICE:
2029         base_picture->type = GST_VAAPI_PICTURE_TYPE_SP;
2030         break;
2031     case GST_H264_SI_SLICE:
2032         base_picture->type = GST_VAAPI_PICTURE_TYPE_SI;
2033         break;
2034     }
2035
2036     /* Initialize picture structure */
2037     if (!slice_hdr->field_pic_flag)
2038         base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
2039     else {
2040         GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_INTERLACED);
2041         if (!slice_hdr->bottom_field_flag)
2042             base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD;
2043         else
2044             base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD;
2045     }
2046     picture->structure = base_picture->structure;
2047
2048     /* Initialize reference flags */
2049     if (pi->nalu.ref_idc) {
2050         GstH264DecRefPicMarking * const dec_ref_pic_marking =
2051             &slice_hdr->dec_ref_pic_marking;
2052
2053         if (GST_VAAPI_PICTURE_IS_IDR(picture) &&
2054             dec_ref_pic_marking->long_term_reference_flag)
2055             GST_VAAPI_PICTURE_FLAG_SET(picture,
2056                 GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE);
2057         else
2058             GST_VAAPI_PICTURE_FLAG_SET(picture,
2059                 GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE);
2060     }
2061
2062     init_picture_poc(decoder, picture, slice_hdr);
2063     init_picture_refs(decoder, picture, slice_hdr);
2064     return TRUE;
2065 }
2066
2067 /* 8.2.5.3 - Sliding window decoded reference picture marking process */
2068 static gboolean
2069 exec_ref_pic_marking_sliding_window(GstVaapiDecoderH264 *decoder)
2070 {
2071     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2072     GstH264PPS * const pps = priv->current_picture->pps;
2073     GstH264SPS * const sps = pps->sequence;
2074     GstVaapiPictureH264 *ref_picture;
2075     guint i, m, max_num_ref_frames;
2076
2077     GST_DEBUG("reference picture marking process (sliding window)");
2078
2079     if (!GST_VAAPI_PICTURE_IS_FIRST_FIELD(priv->current_picture))
2080         return TRUE;
2081
2082     max_num_ref_frames = sps->num_ref_frames;
2083     if (max_num_ref_frames == 0)
2084         max_num_ref_frames = 1;
2085     if (!GST_VAAPI_PICTURE_IS_FRAME(priv->current_picture))
2086         max_num_ref_frames <<= 1;
2087
2088     if (priv->short_ref_count + priv->long_ref_count < max_num_ref_frames)
2089         return TRUE;
2090     if (priv->short_ref_count < 1)
2091         return FALSE;
2092
2093     for (m = 0, i = 1; i < priv->short_ref_count; i++) {
2094         GstVaapiPictureH264 * const picture = priv->short_ref[i];
2095         if (picture->frame_num_wrap < priv->short_ref[m]->frame_num_wrap)
2096             m = i;
2097     }
2098
2099     ref_picture = priv->short_ref[m];
2100     gst_vaapi_picture_h264_set_reference(ref_picture, 0, TRUE);
2101     ARRAY_REMOVE_INDEX(priv->short_ref, m);
2102
2103     /* Both fields need to be marked as "unused for reference", so
2104        remove the other field from the short_ref[] list as well */
2105     if (!GST_VAAPI_PICTURE_IS_FRAME(priv->current_picture) && ref_picture->other_field) {
2106         for (i = 0; i < priv->short_ref_count; i++) {
2107             if (priv->short_ref[i] == ref_picture->other_field) {
2108                 ARRAY_REMOVE_INDEX(priv->short_ref, i);
2109                 break;
2110             }
2111         }
2112     }
2113     return TRUE;
2114 }
2115
2116 static inline gint32
2117 get_picNumX(GstVaapiPictureH264 *picture, GstH264RefPicMarking *ref_pic_marking)
2118 {
2119     gint32 pic_num;
2120
2121     if (GST_VAAPI_PICTURE_IS_FRAME(picture))
2122         pic_num = picture->frame_num_wrap;
2123     else
2124         pic_num = 2 * picture->frame_num_wrap + 1;
2125     pic_num -= ref_pic_marking->difference_of_pic_nums_minus1 + 1;
2126     return pic_num;
2127 }
2128
2129 /* 8.2.5.4.1. Mark short-term reference picture as "unused for reference" */
2130 static void
2131 exec_ref_pic_marking_adaptive_mmco_1(
2132     GstVaapiDecoderH264  *decoder,
2133     GstVaapiPictureH264  *picture,
2134     GstH264RefPicMarking *ref_pic_marking
2135 )
2136 {
2137     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2138     gint32 i, picNumX;
2139
2140     picNumX = get_picNumX(picture, ref_pic_marking);
2141     i = find_short_term_reference(decoder, picNumX);
2142     if (i < 0)
2143         return;
2144
2145     gst_vaapi_picture_h264_set_reference(priv->short_ref[i], 0,
2146         GST_VAAPI_PICTURE_IS_FRAME(picture));
2147     ARRAY_REMOVE_INDEX(priv->short_ref, i);
2148 }
2149
2150 /* 8.2.5.4.2. Mark long-term reference picture as "unused for reference" */
2151 static void
2152 exec_ref_pic_marking_adaptive_mmco_2(
2153     GstVaapiDecoderH264  *decoder,
2154     GstVaapiPictureH264  *picture,
2155     GstH264RefPicMarking *ref_pic_marking
2156 )
2157 {
2158     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2159     gint32 i;
2160
2161     i = find_long_term_reference(decoder, ref_pic_marking->long_term_pic_num);
2162     if (i < 0)
2163         return;
2164
2165     gst_vaapi_picture_h264_set_reference(priv->long_ref[i], 0,
2166         GST_VAAPI_PICTURE_IS_FRAME(picture));
2167     ARRAY_REMOVE_INDEX(priv->long_ref, i);
2168 }
2169
2170 /* 8.2.5.4.3. Assign LongTermFrameIdx to a short-term reference picture */
2171 static void
2172 exec_ref_pic_marking_adaptive_mmco_3(
2173     GstVaapiDecoderH264  *decoder,
2174     GstVaapiPictureH264  *picture,
2175     GstH264RefPicMarking *ref_pic_marking
2176 )
2177 {
2178     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2179     GstVaapiPictureH264 *ref_picture;
2180     gint32 i, picNumX;
2181
2182     for (i = 0; i < priv->long_ref_count; i++) {
2183         if (priv->long_ref[i]->long_term_frame_idx == ref_pic_marking->long_term_frame_idx)
2184             break;
2185     }
2186     if (i != priv->long_ref_count) {
2187         gst_vaapi_picture_h264_set_reference(priv->long_ref[i], 0, TRUE);
2188         ARRAY_REMOVE_INDEX(priv->long_ref, i);
2189     }
2190
2191     picNumX = get_picNumX(picture, ref_pic_marking);
2192     i = find_short_term_reference(decoder, picNumX);
2193     if (i < 0)
2194         return;
2195
2196     ref_picture = priv->short_ref[i];
2197     ARRAY_REMOVE_INDEX(priv->short_ref, i);
2198     priv->long_ref[priv->long_ref_count++] = ref_picture;
2199
2200     ref_picture->long_term_frame_idx = ref_pic_marking->long_term_frame_idx;
2201     gst_vaapi_picture_h264_set_reference(ref_picture,
2202         GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE,
2203         GST_VAAPI_PICTURE_IS_FRAME(picture));
2204 }
2205
2206 /* 8.2.5.4.4. Mark pictures with LongTermFramIdx > max_long_term_frame_idx
2207  * as "unused for reference" */
2208 static void
2209 exec_ref_pic_marking_adaptive_mmco_4(
2210     GstVaapiDecoderH264  *decoder,
2211     GstVaapiPictureH264  *picture,
2212     GstH264RefPicMarking *ref_pic_marking
2213 )
2214 {
2215     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2216     gint32 i, long_term_frame_idx;
2217
2218     long_term_frame_idx = ref_pic_marking->max_long_term_frame_idx_plus1 - 1;
2219
2220     for (i = 0; i < priv->long_ref_count; i++) {
2221         if (priv->long_ref[i]->long_term_frame_idx <= long_term_frame_idx)
2222             continue;
2223         gst_vaapi_picture_h264_set_reference(priv->long_ref[i], 0, FALSE);
2224         ARRAY_REMOVE_INDEX(priv->long_ref, i);
2225         i--;
2226     }
2227 }
2228
2229 /* 8.2.5.4.5. Mark all reference pictures as "unused for reference" */
2230 static void
2231 exec_ref_pic_marking_adaptive_mmco_5(
2232     GstVaapiDecoderH264  *decoder,
2233     GstVaapiPictureH264  *picture,
2234     GstH264RefPicMarking *ref_pic_marking
2235 )
2236 {
2237     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2238
2239     dpb_flush(decoder);
2240
2241     priv->prev_pic_has_mmco5 = TRUE;
2242
2243     /* The picture shall be inferred to have had frame_num equal to 0 (7.4.3) */
2244     priv->frame_num = 0;
2245     priv->frame_num_offset = 0;
2246     picture->frame_num = 0;
2247
2248     /* Update TopFieldOrderCnt and BottomFieldOrderCnt (8.2.1) */
2249     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
2250         picture->field_poc[TOP_FIELD] -= picture->base.poc;
2251     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
2252         picture->field_poc[BOTTOM_FIELD] -= picture->base.poc;
2253     picture->base.poc = 0;
2254 }
2255
2256 /* 8.2.5.4.6. Assign a long-term frame index to the current picture */
2257 static void
2258 exec_ref_pic_marking_adaptive_mmco_6(
2259     GstVaapiDecoderH264  *decoder,
2260     GstVaapiPictureH264  *picture,
2261     GstH264RefPicMarking *ref_pic_marking
2262 )
2263 {
2264     picture->long_term_frame_idx = ref_pic_marking->long_term_frame_idx;
2265     gst_vaapi_picture_h264_set_reference(picture,
2266         GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE, FALSE);
2267 }
2268
2269 /* 8.2.5.4. Adaptive memory control decoded reference picture marking process */
2270 static gboolean
2271 exec_ref_pic_marking_adaptive(
2272     GstVaapiDecoderH264     *decoder,
2273     GstVaapiPictureH264     *picture,
2274     GstH264DecRefPicMarking *dec_ref_pic_marking
2275 )
2276 {
2277     guint i;
2278
2279     GST_DEBUG("reference picture marking process (adaptive memory control)");
2280
2281     typedef void (*exec_ref_pic_marking_adaptive_mmco_func)(
2282         GstVaapiDecoderH264  *decoder,
2283         GstVaapiPictureH264  *picture,
2284         GstH264RefPicMarking *ref_pic_marking
2285     );
2286
2287     static const exec_ref_pic_marking_adaptive_mmco_func mmco_funcs[] = {
2288         NULL,
2289         exec_ref_pic_marking_adaptive_mmco_1,
2290         exec_ref_pic_marking_adaptive_mmco_2,
2291         exec_ref_pic_marking_adaptive_mmco_3,
2292         exec_ref_pic_marking_adaptive_mmco_4,
2293         exec_ref_pic_marking_adaptive_mmco_5,
2294         exec_ref_pic_marking_adaptive_mmco_6,
2295     };
2296
2297     for (i = 0; i < dec_ref_pic_marking->n_ref_pic_marking; i++) {
2298         GstH264RefPicMarking * const ref_pic_marking =
2299             &dec_ref_pic_marking->ref_pic_marking[i];
2300
2301         const guint mmco = ref_pic_marking->memory_management_control_operation;
2302         if (mmco < G_N_ELEMENTS(mmco_funcs) && mmco_funcs[mmco])
2303             mmco_funcs[mmco](decoder, picture, ref_pic_marking);
2304         else {
2305             GST_ERROR("unhandled MMCO %u", mmco);
2306             return FALSE;
2307         }
2308     }
2309     return TRUE;
2310 }
2311
2312 /* 8.2.5 - Execute reference picture marking process */
2313 static gboolean
2314 exec_ref_pic_marking(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
2315 {
2316     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2317
2318     priv->prev_pic_has_mmco5 = FALSE;
2319     priv->prev_pic_structure = picture->structure;
2320
2321     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
2322         return TRUE;
2323
2324     if (!GST_VAAPI_PICTURE_IS_IDR(picture)) {
2325         GstH264DecRefPicMarking * const dec_ref_pic_marking =
2326             &picture->last_slice_hdr->dec_ref_pic_marking;
2327         if (dec_ref_pic_marking->adaptive_ref_pic_marking_mode_flag) {
2328             if (!exec_ref_pic_marking_adaptive(decoder, picture, dec_ref_pic_marking))
2329                 return FALSE;
2330         }
2331         else {
2332             if (!exec_ref_pic_marking_sliding_window(decoder))
2333                 return FALSE;
2334         }
2335     }
2336     return TRUE;
2337 }
2338
2339 static void
2340 vaapi_init_picture(VAPictureH264 *pic)
2341 {
2342     pic->picture_id           = VA_INVALID_ID;
2343     pic->frame_idx            = 0;
2344     pic->flags                = VA_PICTURE_H264_INVALID;
2345     pic->TopFieldOrderCnt     = 0;
2346     pic->BottomFieldOrderCnt  = 0;
2347 }
2348
2349 static void
2350 vaapi_fill_picture(VAPictureH264 *pic, GstVaapiPictureH264 *picture,
2351     guint picture_structure)
2352 {
2353     if (!picture_structure)
2354         picture_structure = picture->structure;
2355
2356     pic->picture_id = picture->base.surface_id;
2357     pic->flags = 0;
2358
2359     if (GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture)) {
2360         pic->flags |= VA_PICTURE_H264_LONG_TERM_REFERENCE;
2361         pic->frame_idx = picture->long_term_frame_idx;
2362     }
2363     else {
2364         if (GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture))
2365             pic->flags |= VA_PICTURE_H264_SHORT_TERM_REFERENCE;
2366         pic->frame_idx = picture->frame_num;
2367     }
2368
2369     switch (picture_structure) {
2370     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
2371         pic->TopFieldOrderCnt = picture->field_poc[TOP_FIELD];
2372         pic->BottomFieldOrderCnt = picture->field_poc[BOTTOM_FIELD];
2373         break;
2374     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
2375         pic->flags |= VA_PICTURE_H264_TOP_FIELD;
2376         pic->TopFieldOrderCnt = picture->field_poc[TOP_FIELD];
2377         pic->BottomFieldOrderCnt = 0;
2378         break;
2379     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
2380         pic->flags |= VA_PICTURE_H264_BOTTOM_FIELD;
2381         pic->BottomFieldOrderCnt = picture->field_poc[BOTTOM_FIELD];
2382         pic->TopFieldOrderCnt = 0;
2383         break;
2384     }
2385 }
2386
2387 static gboolean
2388 fill_picture(GstVaapiDecoderH264 *decoder,
2389     GstVaapiPictureH264 *picture, GstVaapiParserInfoH264 *pi)
2390 {
2391     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2392     GstVaapiPicture * const base_picture = &picture->base;
2393     GstH264SliceHdr * const slice_hdr = &pi->data.slice_hdr;
2394     GstH264PPS * const pps = picture->pps;
2395     GstH264SPS * const sps = pps->sequence;
2396     VAPictureParameterBufferH264 * const pic_param = base_picture->param;
2397     guint i, n;
2398
2399     /* Fill in VAPictureParameterBufferH264 */
2400     vaapi_fill_picture(&pic_param->CurrPic, picture, 0);
2401
2402     for (i = 0, n = 0; i < priv->dpb_count; i++) {
2403         GstVaapiFrameStore * const fs = priv->dpb[i];
2404         if (gst_vaapi_frame_store_has_reference(fs))
2405             vaapi_fill_picture(&pic_param->ReferenceFrames[n++],
2406                 fs->buffers[0], fs->structure);
2407     }
2408     for (; n < G_N_ELEMENTS(pic_param->ReferenceFrames); n++)
2409         vaapi_init_picture(&pic_param->ReferenceFrames[n]);
2410
2411 #define COPY_FIELD(s, f) \
2412     pic_param->f = (s)->f
2413
2414 #define COPY_BFM(a, s, f) \
2415     pic_param->a.bits.f = (s)->f
2416
2417     pic_param->picture_width_in_mbs_minus1  = priv->mb_width - 1;
2418     pic_param->picture_height_in_mbs_minus1 = priv->mb_height - 1;
2419     pic_param->frame_num                    = priv->frame_num;
2420
2421     COPY_FIELD(sps, bit_depth_luma_minus8);
2422     COPY_FIELD(sps, bit_depth_chroma_minus8);
2423     COPY_FIELD(sps, num_ref_frames);
2424     COPY_FIELD(pps, num_slice_groups_minus1);
2425     COPY_FIELD(pps, slice_group_map_type);
2426     COPY_FIELD(pps, slice_group_change_rate_minus1);
2427     COPY_FIELD(pps, pic_init_qp_minus26);
2428     COPY_FIELD(pps, pic_init_qs_minus26);
2429     COPY_FIELD(pps, chroma_qp_index_offset);
2430     COPY_FIELD(pps, second_chroma_qp_index_offset);
2431
2432     pic_param->seq_fields.value                                         = 0; /* reset all bits */
2433     pic_param->seq_fields.bits.residual_colour_transform_flag           = sps->separate_colour_plane_flag;
2434     pic_param->seq_fields.bits.MinLumaBiPredSize8x8                     = sps->level_idc >= 31; /* A.3.3.2 */
2435
2436     COPY_BFM(seq_fields, sps, chroma_format_idc);
2437     COPY_BFM(seq_fields, sps, gaps_in_frame_num_value_allowed_flag);
2438     COPY_BFM(seq_fields, sps, frame_mbs_only_flag);
2439     COPY_BFM(seq_fields, sps, mb_adaptive_frame_field_flag);
2440     COPY_BFM(seq_fields, sps, direct_8x8_inference_flag);
2441     COPY_BFM(seq_fields, sps, log2_max_frame_num_minus4);
2442     COPY_BFM(seq_fields, sps, pic_order_cnt_type);
2443     COPY_BFM(seq_fields, sps, log2_max_pic_order_cnt_lsb_minus4);
2444     COPY_BFM(seq_fields, sps, delta_pic_order_always_zero_flag);
2445
2446     pic_param->pic_fields.value                                         = 0; /* reset all bits */
2447     pic_param->pic_fields.bits.field_pic_flag                           = slice_hdr->field_pic_flag;
2448     pic_param->pic_fields.bits.reference_pic_flag                       = GST_VAAPI_PICTURE_IS_REFERENCE(picture);
2449
2450     COPY_BFM(pic_fields, pps, entropy_coding_mode_flag);
2451     COPY_BFM(pic_fields, pps, weighted_pred_flag);
2452     COPY_BFM(pic_fields, pps, weighted_bipred_idc);
2453     COPY_BFM(pic_fields, pps, transform_8x8_mode_flag);
2454     COPY_BFM(pic_fields, pps, constrained_intra_pred_flag);
2455     COPY_BFM(pic_fields, pps, pic_order_present_flag);
2456     COPY_BFM(pic_fields, pps, deblocking_filter_control_present_flag);
2457     COPY_BFM(pic_fields, pps, redundant_pic_cnt_present_flag);
2458     return TRUE;
2459 }
2460
2461 /* Detection of the first VCL NAL unit of a primary coded picture (7.4.1.2.4) */
2462 static gboolean
2463 is_new_picture(GstVaapiParserInfoH264 *pi, GstVaapiParserInfoH264 *prev_pi)
2464 {
2465     GstH264SliceHdr * const slice_hdr = &pi->data.slice_hdr;
2466     GstH264PPS * const pps = slice_hdr->pps;
2467     GstH264SPS * const sps = pps->sequence;
2468     GstH264SliceHdr *prev_slice_hdr;
2469
2470     if (!prev_pi)
2471         return TRUE;
2472     prev_slice_hdr = &prev_pi->data.slice_hdr;
2473
2474 #define CHECK_EXPR(expr, field_name) do {              \
2475         if (!(expr)) {                                 \
2476             GST_DEBUG(field_name " differs in value"); \
2477             return TRUE;                               \
2478         }                                              \
2479     } while (0)
2480
2481 #define CHECK_VALUE(new_slice_hdr, old_slice_hdr, field) \
2482     CHECK_EXPR(((new_slice_hdr)->field == (old_slice_hdr)->field), #field)
2483
2484     /* frame_num differs in value, regardless of inferred values to 0 */
2485     CHECK_VALUE(slice_hdr, prev_slice_hdr, frame_num);
2486
2487     /* pic_parameter_set_id differs in value */
2488     CHECK_VALUE(slice_hdr, prev_slice_hdr, pps);
2489
2490     /* field_pic_flag differs in value */
2491     CHECK_VALUE(slice_hdr, prev_slice_hdr, field_pic_flag);
2492
2493     /* bottom_field_flag is present in both and differs in value */
2494     if (slice_hdr->field_pic_flag && prev_slice_hdr->field_pic_flag)
2495         CHECK_VALUE(slice_hdr, prev_slice_hdr, bottom_field_flag);
2496
2497     /* nal_ref_idc differs in value with one of the nal_ref_idc values is 0 */
2498     CHECK_EXPR((pi->nalu.ref_idc != 0) ==
2499                (prev_pi->nalu.ref_idc != 0), "nal_ref_idc");
2500
2501     /* POC type is 0 for both and either pic_order_cnt_lsb differs in
2502        value or delta_pic_order_cnt_bottom differs in value */
2503     if (sps->pic_order_cnt_type == 0) {
2504         CHECK_VALUE(slice_hdr, prev_slice_hdr, pic_order_cnt_lsb);
2505         if (pps->pic_order_present_flag && !slice_hdr->field_pic_flag)
2506             CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt_bottom);
2507     }
2508
2509     /* POC type is 1 for both and either delta_pic_order_cnt[0]
2510        differs in value or delta_pic_order_cnt[1] differs in value */
2511     else if (sps->pic_order_cnt_type == 1) {
2512         CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt[0]);
2513         CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt[1]);
2514     }
2515
2516     /* IdrPicFlag differs in value */
2517     CHECK_VALUE(&pi->nalu, &prev_pi->nalu, idr_pic_flag);
2518
2519     /* IdrPicFlag is equal to 1 for both and idr_pic_id differs in value */
2520     if (pi->nalu.idr_pic_flag)
2521         CHECK_VALUE(slice_hdr, prev_slice_hdr, idr_pic_id);
2522
2523 #undef CHECK_EXPR
2524 #undef CHECK_VALUE
2525     return FALSE;
2526 }
2527
2528 static GstVaapiDecoderStatus
2529 decode_picture(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnit *unit)
2530 {
2531     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2532     GstVaapiParserInfoH264 * const pi = unit->parsed_info;
2533     GstH264SliceHdr * const slice_hdr = &pi->data.slice_hdr;
2534     GstH264PPS * const pps = slice_hdr->pps;
2535     GstH264SPS * const sps = pps->sequence;
2536     GstVaapiPictureH264 *picture;
2537     GstVaapiDecoderStatus status;
2538
2539     status = ensure_context(decoder, sps);
2540     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2541         return status;
2542
2543     priv->decoder_state = 0;
2544
2545     if (priv->current_picture) {
2546         /* Re-use current picture where the first field was decoded */
2547         picture = gst_vaapi_picture_h264_new_field(priv->current_picture);
2548         if (!picture) {
2549             GST_ERROR("failed to allocate field picture");
2550             return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2551         }
2552     }
2553     else {
2554         /* Create new picture */
2555         picture = gst_vaapi_picture_h264_new(decoder);
2556         if (!picture) {
2557             GST_ERROR("failed to allocate picture");
2558             return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2559         }
2560     }
2561     gst_vaapi_picture_replace(&priv->current_picture, picture);
2562     gst_vaapi_picture_unref(picture);
2563
2564     /* Update cropping rectangle */
2565     if (sps->frame_cropping_flag) {
2566         GstVaapiRectangle crop_rect;
2567         crop_rect.x = sps->crop_rect_x;
2568         crop_rect.y = sps->crop_rect_y;
2569         crop_rect.width = sps->crop_rect_width;
2570         crop_rect.height = sps->crop_rect_height;
2571         gst_vaapi_picture_set_crop_rect(&picture->base, &crop_rect);
2572     }
2573
2574     picture->pps = pps;
2575
2576     status = ensure_quant_matrix(decoder, picture);
2577     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS) {
2578         GST_ERROR("failed to reset quantizer matrix");
2579         return status;
2580     }
2581
2582     if (!init_picture(decoder, picture, pi))
2583         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2584     if (!fill_picture(decoder, picture, pi))
2585         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2586
2587     priv->decoder_state = pi->state;
2588     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2589 }
2590
2591 static inline guint
2592 get_slice_data_bit_offset(GstH264SliceHdr *slice_hdr)
2593 {
2594     guint epb_count;
2595
2596     epb_count = slice_hdr->n_emulation_prevention_bytes;
2597     return 8 /* nal_unit_type */ + slice_hdr->header_size - epb_count * 8;
2598 }
2599
2600 static gboolean
2601 fill_pred_weight_table(GstVaapiDecoderH264 *decoder,
2602     GstVaapiSlice *slice, GstH264SliceHdr *slice_hdr)
2603 {
2604     VASliceParameterBufferH264 * const slice_param = slice->param;
2605     GstH264PPS * const pps = slice_hdr->pps;
2606     GstH264SPS * const sps = pps->sequence;
2607     GstH264PredWeightTable * const w = &slice_hdr->pred_weight_table;
2608     guint num_weight_tables = 0;
2609     gint i, j;
2610
2611     if (pps->weighted_pred_flag &&
2612         (GST_H264_IS_P_SLICE(slice_hdr) || GST_H264_IS_SP_SLICE(slice_hdr)))
2613         num_weight_tables = 1;
2614     else if (pps->weighted_bipred_idc == 1 && GST_H264_IS_B_SLICE(slice_hdr))
2615         num_weight_tables = 2;
2616     else
2617         num_weight_tables = 0;
2618
2619     slice_param->luma_log2_weight_denom   = 0;
2620     slice_param->chroma_log2_weight_denom = 0;
2621     slice_param->luma_weight_l0_flag      = 0;
2622     slice_param->chroma_weight_l0_flag    = 0;
2623     slice_param->luma_weight_l1_flag      = 0;
2624     slice_param->chroma_weight_l1_flag    = 0;
2625
2626     if (num_weight_tables < 1)
2627         return TRUE;
2628
2629     slice_param->luma_log2_weight_denom   = w->luma_log2_weight_denom;
2630     slice_param->chroma_log2_weight_denom = w->chroma_log2_weight_denom;
2631
2632     slice_param->luma_weight_l0_flag = 1;
2633     for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2634         slice_param->luma_weight_l0[i] = w->luma_weight_l0[i];
2635         slice_param->luma_offset_l0[i] = w->luma_offset_l0[i];
2636     }
2637
2638     slice_param->chroma_weight_l0_flag = sps->chroma_array_type != 0;
2639     if (slice_param->chroma_weight_l0_flag) {
2640         for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2641             for (j = 0; j < 2; j++) {
2642                 slice_param->chroma_weight_l0[i][j] = w->chroma_weight_l0[i][j];
2643                 slice_param->chroma_offset_l0[i][j] = w->chroma_offset_l0[i][j];
2644             }
2645         }
2646     }
2647
2648     if (num_weight_tables < 2)
2649         return TRUE;
2650
2651     slice_param->luma_weight_l1_flag = 1;
2652     for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2653         slice_param->luma_weight_l1[i] = w->luma_weight_l1[i];
2654         slice_param->luma_offset_l1[i] = w->luma_offset_l1[i];
2655     }
2656
2657     slice_param->chroma_weight_l1_flag = sps->chroma_array_type != 0;
2658     if (slice_param->chroma_weight_l1_flag) {
2659         for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2660             for (j = 0; j < 2; j++) {
2661                 slice_param->chroma_weight_l1[i][j] = w->chroma_weight_l1[i][j];
2662                 slice_param->chroma_offset_l1[i][j] = w->chroma_offset_l1[i][j];
2663             }
2664         }
2665     }
2666     return TRUE;
2667 }
2668
2669 static gboolean
2670 fill_RefPicList(GstVaapiDecoderH264 *decoder,
2671     GstVaapiSlice *slice, GstH264SliceHdr *slice_hdr)
2672 {
2673     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2674     VASliceParameterBufferH264 * const slice_param = slice->param;
2675     guint i, num_ref_lists = 0;
2676
2677     slice_param->num_ref_idx_l0_active_minus1 = 0;
2678     slice_param->num_ref_idx_l1_active_minus1 = 0;
2679
2680     if (GST_H264_IS_B_SLICE(slice_hdr))
2681         num_ref_lists = 2;
2682     else if (GST_H264_IS_I_SLICE(slice_hdr))
2683         num_ref_lists = 0;
2684     else
2685         num_ref_lists = 1;
2686
2687     if (num_ref_lists < 1)
2688         return TRUE;
2689
2690     slice_param->num_ref_idx_l0_active_minus1 =
2691         slice_hdr->num_ref_idx_l0_active_minus1;
2692
2693     for (i = 0; i < priv->RefPicList0_count && priv->RefPicList0[i]; i++)
2694         vaapi_fill_picture(&slice_param->RefPicList0[i], priv->RefPicList0[i], 0);
2695     for (; i <= slice_param->num_ref_idx_l0_active_minus1; i++)
2696         vaapi_init_picture(&slice_param->RefPicList0[i]);
2697
2698     if (num_ref_lists < 2)
2699         return TRUE;
2700
2701     slice_param->num_ref_idx_l1_active_minus1 =
2702         slice_hdr->num_ref_idx_l1_active_minus1;
2703
2704     for (i = 0; i < priv->RefPicList1_count && priv->RefPicList1[i]; i++)
2705         vaapi_fill_picture(&slice_param->RefPicList1[i], priv->RefPicList1[i], 0);
2706     for (; i <= slice_param->num_ref_idx_l1_active_minus1; i++)
2707         vaapi_init_picture(&slice_param->RefPicList1[i]);
2708     return TRUE;
2709 }
2710
2711 static gboolean
2712 fill_slice(GstVaapiDecoderH264 *decoder,
2713     GstVaapiSlice *slice, GstH264SliceHdr *slice_hdr)
2714 {
2715     VASliceParameterBufferH264 * const slice_param = slice->param;
2716
2717     /* Fill in VASliceParameterBufferH264 */
2718     slice_param->slice_data_bit_offset          = get_slice_data_bit_offset(slice_hdr);
2719     slice_param->first_mb_in_slice              = slice_hdr->first_mb_in_slice;
2720     slice_param->slice_type                     = slice_hdr->type % 5;
2721     slice_param->direct_spatial_mv_pred_flag    = slice_hdr->direct_spatial_mv_pred_flag;
2722     slice_param->cabac_init_idc                 = slice_hdr->cabac_init_idc;
2723     slice_param->slice_qp_delta                 = slice_hdr->slice_qp_delta;
2724     slice_param->disable_deblocking_filter_idc  = slice_hdr->disable_deblocking_filter_idc;
2725     slice_param->slice_alpha_c0_offset_div2     = slice_hdr->slice_alpha_c0_offset_div2;
2726     slice_param->slice_beta_offset_div2         = slice_hdr->slice_beta_offset_div2;
2727
2728     if (!fill_RefPicList(decoder, slice, slice_hdr))
2729         return FALSE;
2730     if (!fill_pred_weight_table(decoder, slice, slice_hdr))
2731         return FALSE;
2732     return TRUE;
2733 }
2734
2735 static GstVaapiDecoderStatus
2736 decode_slice(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnit *unit)
2737 {
2738     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2739     GstVaapiParserInfoH264 * const pi = unit->parsed_info;
2740     GstVaapiPictureH264 * const picture = priv->current_picture;
2741     GstH264SliceHdr * const slice_hdr = &pi->data.slice_hdr;
2742     GstVaapiSlice *slice;
2743     GstBuffer * const buffer =
2744         GST_VAAPI_DECODER_CODEC_FRAME(decoder)->input_buffer;
2745     GstMapInfo map_info;
2746
2747     GST_DEBUG("slice (%u bytes)", pi->nalu.size);
2748
2749     if (!is_valid_state(pi->state,
2750             GST_H264_VIDEO_STATE_VALID_PICTURE_HEADERS)) {
2751         GST_WARNING("failed to receive enough headers to decode slice");
2752         return GST_VAAPI_DECODER_STATUS_SUCCESS;
2753     }
2754
2755     if (!gst_buffer_map(buffer, &map_info, GST_MAP_READ)) {
2756         GST_ERROR("failed to map buffer");
2757         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2758     }
2759
2760     slice = GST_VAAPI_SLICE_NEW(H264, decoder,
2761         (map_info.data + unit->offset + pi->nalu.offset), pi->nalu.size);
2762     gst_buffer_unmap(buffer, &map_info);
2763     if (!slice) {
2764         GST_ERROR("failed to allocate slice");
2765         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2766     }
2767
2768     if (!fill_slice(decoder, slice, slice_hdr)) {
2769         gst_vaapi_mini_object_unref(GST_VAAPI_MINI_OBJECT(slice));
2770         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2771     }
2772
2773     gst_vaapi_picture_add_slice(GST_VAAPI_PICTURE_CAST(picture), slice);
2774     picture->last_slice_hdr = slice_hdr;
2775     priv->decoder_state |= GST_H264_VIDEO_STATE_GOT_SLICE;
2776     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2777 }
2778
2779 static inline gint
2780 scan_for_start_code(GstAdapter *adapter, guint ofs, guint size, guint32 *scp)
2781 {
2782     return (gint)gst_adapter_masked_scan_uint32_peek(adapter,
2783                                                      0xffffff00, 0x00000100,
2784                                                      ofs, size,
2785                                                      scp);
2786 }
2787
2788 static GstVaapiDecoderStatus
2789 decode_unit(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnit *unit)
2790 {
2791     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2792     GstVaapiParserInfoH264 * const pi = unit->parsed_info;
2793     GstVaapiDecoderStatus status;
2794
2795     priv->decoder_state |= pi->state;
2796     switch (pi->nalu.type) {
2797     case GST_H264_NAL_SLICE_IDR:
2798         /* fall-through. IDR specifics are handled in init_picture() */
2799     case GST_H264_NAL_SLICE:
2800         status = decode_slice(decoder, unit);
2801         break;
2802     case GST_H264_NAL_SEQ_END:
2803     case GST_H264_NAL_STREAM_END:
2804         status = decode_sequence_end(decoder);
2805         break;
2806     case GST_H264_NAL_SEI:
2807         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2808         break;
2809     default:
2810         GST_WARNING("unsupported NAL unit type %d", pi->nalu.type);
2811         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
2812         break;
2813     }
2814     return status;
2815 }
2816
2817 static GstVaapiDecoderStatus
2818 gst_vaapi_decoder_h264_decode_codec_data(GstVaapiDecoder *base_decoder,
2819     const guchar *buf, guint buf_size)
2820 {
2821     GstVaapiDecoderH264 * const decoder =
2822         GST_VAAPI_DECODER_H264_CAST(base_decoder);
2823     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2824     GstVaapiDecoderStatus status;
2825     GstVaapiDecoderUnit unit;
2826     GstVaapiParserInfoH264 pi;
2827     GstH264ParserResult result;
2828     guint i, ofs, num_sps, num_pps;
2829
2830     unit.parsed_info = &pi;
2831
2832     if (buf_size < 8)
2833         return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2834
2835     if (buf[0] != 1) {
2836         GST_ERROR("failed to decode codec-data, not in avcC format");
2837         return GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
2838     }
2839
2840     priv->nal_length_size = (buf[4] & 0x03) + 1;
2841
2842     num_sps = buf[5] & 0x1f;
2843     ofs = 6;
2844
2845     for (i = 0; i < num_sps; i++) {
2846         result = gst_h264_parser_identify_nalu_avc(
2847             priv->parser,
2848             buf, ofs, buf_size, 2,
2849             &pi.nalu
2850         );
2851         if (result != GST_H264_PARSER_OK)
2852             return get_status(result);
2853
2854         status = parse_sps(decoder, &unit);
2855         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2856             return status;
2857         ofs = pi.nalu.offset + pi.nalu.size;
2858     }
2859
2860     num_pps = buf[ofs];
2861     ofs++;
2862
2863     for (i = 0; i < num_pps; i++) {
2864         result = gst_h264_parser_identify_nalu_avc(
2865             priv->parser,
2866             buf, ofs, buf_size, 2,
2867             &pi.nalu
2868         );
2869         if (result != GST_H264_PARSER_OK)
2870             return get_status(result);
2871
2872         status = parse_pps(decoder, &unit);
2873         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2874             return status;
2875         ofs = pi.nalu.offset + pi.nalu.size;
2876     }
2877
2878     priv->is_avcC = TRUE;
2879     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2880 }
2881
2882 static GstVaapiDecoderStatus
2883 ensure_decoder(GstVaapiDecoderH264 *decoder)
2884 {
2885     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2886     GstVaapiDecoderStatus status;
2887
2888     if (!priv->is_opened) {
2889         priv->is_opened = gst_vaapi_decoder_h264_open(decoder);
2890         if (!priv->is_opened)
2891             return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CODEC;
2892
2893         status = gst_vaapi_decoder_decode_codec_data(
2894             GST_VAAPI_DECODER_CAST(decoder));
2895         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2896             return status;
2897     }
2898     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2899 }
2900
2901 static GstVaapiDecoderStatus
2902 gst_vaapi_decoder_h264_parse(GstVaapiDecoder *base_decoder,
2903     GstAdapter *adapter, gboolean at_eos, GstVaapiDecoderUnit *unit)
2904 {
2905     GstVaapiDecoderH264 * const decoder =
2906         GST_VAAPI_DECODER_H264_CAST(base_decoder);
2907     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2908     GstVaapiParserState * const ps = GST_VAAPI_PARSER_STATE(base_decoder);
2909     GstVaapiParserInfoH264 *pi;
2910     GstVaapiDecoderStatus status;
2911     GstH264ParserResult result;
2912     guchar *buf;
2913     guint i, size, buf_size, nalu_size, flags;
2914     guint32 start_code;
2915     gint ofs, ofs2;
2916
2917     status = ensure_decoder(decoder);
2918     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2919         return status;
2920
2921     size = gst_adapter_available(adapter);
2922
2923     if (priv->is_avcC) {
2924         if (size < priv->nal_length_size)
2925             return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2926
2927         buf = (guchar *)&start_code;
2928         g_assert(priv->nal_length_size <= sizeof(start_code));
2929         gst_adapter_copy(adapter, buf, 0, priv->nal_length_size);
2930
2931         nalu_size = 0;
2932         for (i = 0; i < priv->nal_length_size; i++)
2933             nalu_size = (nalu_size << 8) | buf[i];
2934
2935         buf_size = priv->nal_length_size + nalu_size;
2936         if (size < buf_size)
2937             return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2938     }
2939     else {
2940         if (size < 4)
2941             return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2942
2943         ofs = scan_for_start_code(adapter, 0, size, NULL);
2944         if (ofs < 0)
2945             return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2946
2947         if (ofs > 0) {
2948             gst_adapter_flush(adapter, ofs);
2949             size -= ofs;
2950         }
2951
2952         ofs2 = ps->input_offset2 - ofs - 4;
2953         if (ofs2 < 4)
2954             ofs2 = 4;
2955
2956         ofs = G_UNLIKELY(size < ofs2 + 4) ? -1 :
2957             scan_for_start_code(adapter, ofs2, size - ofs2, NULL);
2958         if (ofs < 0) {
2959             // Assume the whole NAL unit is present if end-of-stream
2960             if (!at_eos) {
2961                 ps->input_offset2 = size;
2962                 return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2963             }
2964             ofs = size;
2965         }
2966         buf_size = ofs;
2967     }
2968     ps->input_offset2 = 0;
2969
2970     buf = (guchar *)gst_adapter_map(adapter, buf_size);
2971     if (!buf)
2972         return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2973
2974     unit->size = buf_size;
2975
2976     pi = gst_vaapi_parser_info_h264_new();
2977     if (!pi)
2978         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2979
2980     gst_vaapi_decoder_unit_set_parsed_info(unit,
2981         pi, (GDestroyNotify)gst_vaapi_mini_object_unref);
2982
2983     if (priv->is_avcC)
2984         result = gst_h264_parser_identify_nalu_avc(priv->parser,
2985             buf, 0, buf_size, priv->nal_length_size, &pi->nalu);
2986     else
2987         result = gst_h264_parser_identify_nalu_unchecked(priv->parser,
2988             buf, 0, buf_size, &pi->nalu);
2989     status = get_status(result);
2990     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2991         return status;
2992
2993     switch (pi->nalu.type) {
2994     case GST_H264_NAL_SPS:
2995         status = parse_sps(decoder, unit);
2996         break;
2997     case GST_H264_NAL_PPS:
2998         status = parse_pps(decoder, unit);
2999         break;
3000     case GST_H264_NAL_SEI:
3001         status = parse_sei(decoder, unit);
3002         break;
3003     case GST_H264_NAL_SLICE_IDR:
3004     case GST_H264_NAL_SLICE:
3005         status = parse_slice(decoder, unit);
3006         break;
3007     default:
3008         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
3009         break;
3010     }
3011     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
3012         return status;
3013
3014     flags = 0;
3015     switch (pi->nalu.type) {
3016     case GST_H264_NAL_AU_DELIMITER:
3017         flags |= GST_VAAPI_DECODER_UNIT_FLAG_FRAME_START;
3018         /* fall-through */
3019     case GST_H264_NAL_FILLER_DATA:
3020         flags |= GST_VAAPI_DECODER_UNIT_FLAG_SKIP;
3021         break;
3022     case GST_H264_NAL_STREAM_END:
3023         flags |= GST_VAAPI_DECODER_UNIT_FLAG_STREAM_END;
3024         /* fall-through */
3025     case GST_H264_NAL_SEQ_END:
3026         flags |= GST_VAAPI_DECODER_UNIT_FLAG_FRAME_END;
3027         break;
3028     case GST_H264_NAL_SPS:
3029     case GST_H264_NAL_PPS:
3030         flags |= GST_VAAPI_DECODER_UNIT_FLAG_SKIP;
3031         /* fall-through */
3032     case GST_H264_NAL_SEI:
3033         flags |= GST_VAAPI_DECODER_UNIT_FLAG_FRAME_START;
3034         break;
3035     case GST_H264_NAL_SLICE_IDR:
3036     case GST_H264_NAL_SLICE:
3037         flags |= GST_VAAPI_DECODER_UNIT_FLAG_SLICE;
3038         if (is_new_picture(pi, priv->prev_slice_pi))
3039             flags |= GST_VAAPI_DECODER_UNIT_FLAG_FRAME_START;
3040         gst_vaapi_parser_info_h264_replace(&priv->prev_slice_pi, pi);
3041         break;
3042     default:
3043         if (pi->nalu.type >= 14 && pi->nalu.type <= 18)
3044             flags |= GST_VAAPI_DECODER_UNIT_FLAG_FRAME_START;
3045         break;
3046     }
3047     GST_VAAPI_DECODER_UNIT_FLAG_SET(unit, flags);
3048
3049     pi->nalu.data = NULL;
3050     pi->state = priv->parser_state;
3051     return GST_VAAPI_DECODER_STATUS_SUCCESS;
3052 }
3053
3054 static GstVaapiDecoderStatus
3055 gst_vaapi_decoder_h264_decode(GstVaapiDecoder *base_decoder,
3056     GstVaapiDecoderUnit *unit)
3057 {
3058     GstVaapiDecoderH264 * const decoder =
3059         GST_VAAPI_DECODER_H264_CAST(base_decoder);
3060     GstVaapiDecoderStatus status;
3061
3062     status = ensure_decoder(decoder);
3063     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
3064         return status;
3065     return decode_unit(decoder, unit);
3066 }
3067
3068 static GstVaapiDecoderStatus
3069 gst_vaapi_decoder_h264_start_frame(GstVaapiDecoder *base_decoder,
3070     GstVaapiDecoderUnit *unit)
3071 {
3072     GstVaapiDecoderH264 * const decoder =
3073         GST_VAAPI_DECODER_H264_CAST(base_decoder);
3074
3075     return decode_picture(decoder, unit);
3076 }
3077
3078 static GstVaapiDecoderStatus
3079 gst_vaapi_decoder_h264_end_frame(GstVaapiDecoder *base_decoder)
3080 {
3081     GstVaapiDecoderH264 * const decoder =
3082         GST_VAAPI_DECODER_H264_CAST(base_decoder);
3083
3084     return decode_current_picture(decoder);
3085 }
3086
3087 static GstVaapiDecoderStatus
3088 gst_vaapi_decoder_h264_flush(GstVaapiDecoder *base_decoder)
3089 {
3090     GstVaapiDecoderH264 * const decoder =
3091         GST_VAAPI_DECODER_H264_CAST(base_decoder);
3092
3093     dpb_flush(decoder);
3094     return GST_VAAPI_DECODER_STATUS_SUCCESS;
3095 }
3096
3097 static void
3098 gst_vaapi_decoder_h264_class_init(GstVaapiDecoderH264Class *klass)
3099 {
3100     GstVaapiMiniObjectClass * const object_class =
3101         GST_VAAPI_MINI_OBJECT_CLASS(klass);
3102     GstVaapiDecoderClass * const decoder_class = GST_VAAPI_DECODER_CLASS(klass);
3103
3104     object_class->size          = sizeof(GstVaapiDecoderH264);
3105     object_class->finalize      = (GDestroyNotify)gst_vaapi_decoder_finalize;
3106
3107     decoder_class->create       = gst_vaapi_decoder_h264_create;
3108     decoder_class->destroy      = gst_vaapi_decoder_h264_destroy;
3109     decoder_class->parse        = gst_vaapi_decoder_h264_parse;
3110     decoder_class->decode       = gst_vaapi_decoder_h264_decode;
3111     decoder_class->start_frame  = gst_vaapi_decoder_h264_start_frame;
3112     decoder_class->end_frame    = gst_vaapi_decoder_h264_end_frame;
3113     decoder_class->flush        = gst_vaapi_decoder_h264_flush;
3114
3115     decoder_class->decode_codec_data =
3116         gst_vaapi_decoder_h264_decode_codec_data;
3117 }
3118
3119 static inline const GstVaapiDecoderClass *
3120 gst_vaapi_decoder_h264_class(void)
3121 {
3122     static GstVaapiDecoderH264Class g_class;
3123     static gsize g_class_init = FALSE;
3124
3125     if (g_once_init_enter(&g_class_init)) {
3126         gst_vaapi_decoder_h264_class_init(&g_class);
3127         g_once_init_leave(&g_class_init, TRUE);
3128     }
3129     return GST_VAAPI_DECODER_CLASS(&g_class);
3130 }
3131
3132 /**
3133  * gst_vaapi_decoder_h264_new:
3134  * @display: a #GstVaapiDisplay
3135  * @caps: a #GstCaps holding codec information
3136  *
3137  * Creates a new #GstVaapiDecoder for MPEG-2 decoding.  The @caps can
3138  * hold extra information like codec-data and pictured coded size.
3139  *
3140  * Return value: the newly allocated #GstVaapiDecoder object
3141  */
3142 GstVaapiDecoder *
3143 gst_vaapi_decoder_h264_new(GstVaapiDisplay *display, GstCaps *caps)
3144 {
3145     return gst_vaapi_decoder_new(gst_vaapi_decoder_h264_class(), display, caps);
3146 }