gst-libs/gst/vaapi/gstvaapidecoder_h264.c

   1 /*
   2  *  gstvaapidecoder_h264.c - H.264 decoder
   3  *
   4  *  Copyright (C) 2011-2013 Intel Corporation
   5  *
   6  *  This library is free software; you can redistribute it and/or
   7  *  modify it under the terms of the GNU Lesser General Public License
   8  *  as published by the Free Software Foundation; either version 2.1
   9  *  of the License, or (at your option) any later version.
  10  *
  11  *  This library is distributed in the hope that it will be useful,
  12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  *  Lesser General Public License for more details.
  15  *
  16  *  You should have received a copy of the GNU Lesser General Public
  17  *  License along with this library; if not, write to the Free
  18  *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  19  *  Boston, MA 02110-1301 USA
  20  */
  21
  22 /**
  23  * SECTION:gstvaapidecoder_h264
  24  * @short_description: H.264 decoder
  25  */
  26
  27 #include "sysdeps.h"
  28 #include <string.h>
  29 #include <gst/base/gstadapter.h>
  30 #include <gst/codecparsers/gsth264parser.h>
  31 #include "gstvaapidecoder_h264.h"
  32 #include "gstvaapidecoder_objects.h"
  33 #include "gstvaapidecoder_priv.h"
  34 #include "gstvaapidisplay_priv.h"
  35 #include "gstvaapiobject_priv.h"
  36
  37 #define DEBUG 1
  38 #include "gstvaapidebug.h"
  39
  40 /* Defined to 1 if strict ordering of DPB is needed. Only useful for debug */
  41 #define USE_STRICT_DPB_ORDERING 0
  42
  43 typedef struct _GstVaapiDecoderH264Private      GstVaapiDecoderH264Private;
  44 typedef struct _GstVaapiDecoderH264Class        GstVaapiDecoderH264Class;
  45 typedef struct _GstVaapiFrameStore              GstVaapiFrameStore;
  46 typedef struct _GstVaapiFrameStoreClass         GstVaapiFrameStoreClass;
  47 typedef struct _GstVaapiParserInfoH264          GstVaapiParserInfoH264;
  48 typedef struct _GstVaapiPictureH264             GstVaapiPictureH264;
  49
  50 // Used for field_poc[]
  51 #define TOP_FIELD       0
  52 #define BOTTOM_FIELD    1
  53
  54 /* ------------------------------------------------------------------------- */
  55 /* --- H.264 Parser Info                                                 --- */
  56 /* ------------------------------------------------------------------------- */
  57
  58 #define GST_VAAPI_PARSER_INFO_H264(obj) \
  59     ((GstVaapiParserInfoH264 *)(obj))
  60
  61 struct _GstVaapiParserInfoH264 {
  62     GstVaapiMiniObject  parent_instance;
  63     GstH264NalUnit      nalu;
  64     union {
  65         GstH264SPS      sps;
  66         GstH264PPS      pps;
  67         GstH264SliceHdr slice_hdr;
  68     }                   data;
  69 };
  70
  71 static inline const GstVaapiMiniObjectClass *
  72 gst_vaapi_parser_info_h264_class(void)
  73 {
  74     static const GstVaapiMiniObjectClass GstVaapiParserInfoH264Class = {
  75         sizeof(GstVaapiParserInfoH264),
  76         NULL
  77     };
  78     return &GstVaapiParserInfoH264Class;
  79 }
  80
  81 static inline GstVaapiParserInfoH264 *
  82 gst_vaapi_parser_info_h264_new(void)
  83 {
  84     return (GstVaapiParserInfoH264 *)
  85         gst_vaapi_mini_object_new(gst_vaapi_parser_info_h264_class());
  86 }
  87
  88 #define gst_vaapi_parser_info_h264_ref(pi) \
  89     gst_vaapi_mini_object_ref(GST_VAAPI_MINI_OBJECT(pi))
  90
  91 #define gst_vaapi_parser_info_h264_unref(pi) \
  92     gst_vaapi_mini_object_unref(GST_VAAPI_MINI_OBJECT(pi))
  93
  94 #define gst_vaapi_parser_info_h264_replace(old_pi_ptr, new_pi)          \
  95     gst_vaapi_mini_object_replace((GstVaapiMiniObject **)(old_pi_ptr),  \
  96         (GstVaapiMiniObject *)(new_pi))
  97
  98 /* ------------------------------------------------------------------------- */
  99 /* --- H.264 Pictures                                                    --- */
 100 /* ------------------------------------------------------------------------- */
 101
 102 /*
 103  * Extended picture flags:
 104  *
 105  * @GST_VAAPI_PICTURE_FLAG_IDR: flag that specifies an IDR picture
 106  * @GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE: flag that specifies
 107  *     "used for short-term reference"
 108  * @GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE: flag that specifies
 109  *     "used for long-term reference"
 110  * @GST_VAAPI_PICTURE_FLAGS_REFERENCE: mask covering any kind of
 111  *     reference picture (short-term reference or long-term reference)
 112  */
 113 enum {
 114     GST_VAAPI_PICTURE_FLAG_IDR = (GST_VAAPI_PICTURE_FLAG_LAST << 0),
 115
 116     GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE = (
 117         GST_VAAPI_PICTURE_FLAG_REFERENCE),
 118     GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE = (
 119         GST_VAAPI_PICTURE_FLAG_REFERENCE | (GST_VAAPI_PICTURE_FLAG_LAST << 1)),
 120     GST_VAAPI_PICTURE_FLAGS_REFERENCE = (
 121         GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE |
 122         GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE),
 123 };
 124
 125 #define GST_VAAPI_PICTURE_IS_IDR(picture) \
 126     (GST_VAAPI_PICTURE_FLAG_IS_SET(picture, GST_VAAPI_PICTURE_FLAG_IDR))
 127
 128 #define GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture)      \
 129     ((GST_VAAPI_PICTURE_FLAGS(picture) &                        \
 130       GST_VAAPI_PICTURE_FLAGS_REFERENCE) ==                     \
 131      GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE)
 132
 133 #define GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture)       \
 134     ((GST_VAAPI_PICTURE_FLAGS(picture) &                        \
 135       GST_VAAPI_PICTURE_FLAGS_REFERENCE) ==                     \
 136      GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE)
 137
 138 struct _GstVaapiPictureH264 {
 139     GstVaapiPicture             base;
 140     GstH264PPS                 *pps;
 141     GstH264SliceHdr            *last_slice_hdr;
 142     guint                       structure;
 143     gint32                      field_poc[2];
 144     gint32                      frame_num;              // Original frame_num from slice_header()
 145     gint32                      frame_num_wrap;         // Temporary for ref pic marking: FrameNumWrap
 146     gint32                      long_term_frame_idx;    // Temporary for ref pic marking: LongTermFrameIdx
 147     gint32                      pic_num;                // Temporary for ref pic marking: PicNum
 148     gint32                      long_term_pic_num;      // Temporary for ref pic marking: LongTermPicNum
 149     GstVaapiPictureH264        *other_field;            // Temporary for ref pic marking: other field in the same frame store
 150     guint                       output_flag             : 1;
 151     guint                       output_needed           : 1;
 152 };
 153
 154 GST_VAAPI_CODEC_DEFINE_TYPE(GstVaapiPictureH264, gst_vaapi_picture_h264);
 155
 156 void
 157 gst_vaapi_picture_h264_destroy(GstVaapiPictureH264 *picture)
 158 {
 159     gst_vaapi_picture_destroy(GST_VAAPI_PICTURE(picture));
 160 }
 161
 162 gboolean
 163 gst_vaapi_picture_h264_create(
 164     GstVaapiPictureH264                      *picture,
 165     const GstVaapiCodecObjectConstructorArgs *args
 166 )
 167 {
 168     if (!gst_vaapi_picture_create(GST_VAAPI_PICTURE(picture), args))
 169         return FALSE;
 170
 171     picture->field_poc[0]       = G_MAXINT32;
 172     picture->field_poc[1]       = G_MAXINT32;
 173     picture->output_needed      = FALSE;
 174     return TRUE;
 175 }
 176
 177 static inline GstVaapiPictureH264 *
 178 gst_vaapi_picture_h264_new(GstVaapiDecoderH264 *decoder)
 179 {
 180     return (GstVaapiPictureH264 *)gst_vaapi_codec_object_new(
 181         &GstVaapiPictureH264Class,
 182         GST_VAAPI_CODEC_BASE(decoder),
 183         NULL, sizeof(VAPictureParameterBufferH264),
 184         NULL, 0,
 185         0);
 186 }
 187
 188 static inline void
 189 gst_vaapi_picture_h264_set_reference(
 190     GstVaapiPictureH264 *picture,
 191     guint                reference_flags,
 192     gboolean             other_field
 193 )
 194 {
 195     if (!picture)
 196         return;
 197     GST_VAAPI_PICTURE_FLAG_UNSET(picture, GST_VAAPI_PICTURE_FLAGS_REFERENCE);
 198     GST_VAAPI_PICTURE_FLAG_SET(picture, reference_flags);
 199
 200     if (!other_field || !(picture = picture->other_field))
 201         return;
 202     GST_VAAPI_PICTURE_FLAG_UNSET(picture, GST_VAAPI_PICTURE_FLAGS_REFERENCE);
 203     GST_VAAPI_PICTURE_FLAG_SET(picture, reference_flags);
 204 }
 205
 206 static inline GstVaapiPictureH264 *
 207 gst_vaapi_picture_h264_new_field(GstVaapiPictureH264 *picture)
 208 {
 209     g_return_val_if_fail(picture, NULL);
 210
 211     return (GstVaapiPictureH264 *)gst_vaapi_picture_new_field(&picture->base);
 212 }
 213
 214 /* ------------------------------------------------------------------------- */
 215 /* --- Frame Buffers (DPB)                                               --- */
 216 /* ------------------------------------------------------------------------- */
 217
 218 struct _GstVaapiFrameStore {
 219     /*< private >*/
 220     GstVaapiMiniObject          parent_instance;
 221
 222     guint                       structure;
 223     GstVaapiPictureH264        *buffers[2];
 224     guint                       num_buffers;
 225     guint                       output_needed;
 226 };
 227
 228 static void
 229 gst_vaapi_frame_store_finalize(gpointer object)
 230 {
 231     GstVaapiFrameStore * const fs = object;
 232     guint i;
 233
 234     for (i = 0; i < fs->num_buffers; i++)
 235         gst_vaapi_picture_replace(&fs->buffers[i], NULL);
 236 }
 237
 238 static GstVaapiFrameStore *
 239 gst_vaapi_frame_store_new(GstVaapiPictureH264 *picture)
 240 {
 241     GstVaapiFrameStore *fs;
 242
 243     static const GstVaapiMiniObjectClass GstVaapiFrameStoreClass = {
 244         sizeof(GstVaapiFrameStore),
 245         gst_vaapi_frame_store_finalize
 246     };
 247
 248     fs = (GstVaapiFrameStore *)
 249         gst_vaapi_mini_object_new(&GstVaapiFrameStoreClass);
 250     if (!fs)
 251         return NULL;
 252
 253     fs->structure       = picture->structure;
 254     fs->buffers[0]      = gst_vaapi_picture_ref(picture);
 255     fs->buffers[1]      = NULL;
 256     fs->num_buffers     = 1;
 257     fs->output_needed   = picture->output_needed;
 258     return fs;
 259 }
 260
 261 static gboolean
 262 gst_vaapi_frame_store_add(GstVaapiFrameStore *fs, GstVaapiPictureH264 *picture)
 263 {
 264     guint field;
 265
 266     g_return_val_if_fail(fs->num_buffers == 1, FALSE);
 267     g_return_val_if_fail(!GST_VAAPI_PICTURE_IS_FRAME(picture), FALSE);
 268     g_return_val_if_fail(!GST_VAAPI_PICTURE_IS_FIRST_FIELD(picture), FALSE);
 269
 270     gst_vaapi_picture_replace(&fs->buffers[fs->num_buffers++], picture);
 271     if (picture->output_flag) {
 272         picture->output_needed = TRUE;
 273         fs->output_needed++;
 274     }
 275
 276     fs->structure = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
 277
 278     field = picture->structure == GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD ?
 279         TOP_FIELD : BOTTOM_FIELD;
 280     g_return_val_if_fail(fs->buffers[0]->field_poc[field] == G_MAXINT32, FALSE);
 281     fs->buffers[0]->field_poc[field] = picture->field_poc[field];
 282     g_return_val_if_fail(picture->field_poc[!field] == G_MAXINT32, FALSE);
 283     picture->field_poc[!field] = fs->buffers[0]->field_poc[!field];
 284     return TRUE;
 285 }
 286
 287 static gboolean
 288 gst_vaapi_frame_store_split_fields(GstVaapiFrameStore *fs)
 289 {
 290     GstVaapiPictureH264 * const first_field = fs->buffers[0];
 291     GstVaapiPictureH264 *second_field;
 292
 293     g_return_val_if_fail(fs->num_buffers == 1, FALSE);
 294
 295     first_field->base.structure = GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD;
 296     GST_VAAPI_PICTURE_FLAG_SET(first_field, GST_VAAPI_PICTURE_FLAG_INTERLACED);
 297
 298     second_field = gst_vaapi_picture_h264_new_field(first_field);
 299     if (!second_field)
 300         return FALSE;
 301     gst_vaapi_picture_replace(&fs->buffers[fs->num_buffers++], second_field);
 302     gst_vaapi_picture_unref(second_field);
 303
 304     second_field->frame_num    = first_field->frame_num;
 305     second_field->field_poc[0] = first_field->field_poc[0];
 306     second_field->field_poc[1] = first_field->field_poc[1];
 307     second_field->output_flag  = first_field->output_flag;
 308     if (second_field->output_flag) {
 309         second_field->output_needed = TRUE;
 310         fs->output_needed++;
 311     }
 312     return TRUE;
 313 }
 314
 315 static inline gboolean
 316 gst_vaapi_frame_store_has_frame(GstVaapiFrameStore *fs)
 317 {
 318     return fs->structure == GST_VAAPI_PICTURE_STRUCTURE_FRAME;
 319 }
 320
 321 static inline gboolean
 322 gst_vaapi_frame_store_has_reference(GstVaapiFrameStore *fs)
 323 {
 324     guint i;
 325
 326     for (i = 0; i < fs->num_buffers; i++) {
 327         if (GST_VAAPI_PICTURE_IS_REFERENCE(fs->buffers[i]))
 328             return TRUE;
 329     }
 330     return FALSE;
 331 }
 332
 333 #define gst_vaapi_frame_store_ref(fs) \
 334     gst_vaapi_mini_object_ref(GST_VAAPI_MINI_OBJECT(fs))
 335
 336 #define gst_vaapi_frame_store_unref(fs) \
 337     gst_vaapi_mini_object_unref(GST_VAAPI_MINI_OBJECT(fs))
 338
 339 #define gst_vaapi_frame_store_replace(old_fs_p, new_fs)                 \
 340     gst_vaapi_mini_object_replace((GstVaapiMiniObject **)(old_fs_p),    \
 341         (GstVaapiMiniObject *)(new_fs))
 342
 343 /* ------------------------------------------------------------------------- */
 344 /* --- H.264 Decoder                                                     --- */
 345 /* ------------------------------------------------------------------------- */
 346
 347 #define GST_VAAPI_DECODER_H264_CAST(decoder) \
 348     ((GstVaapiDecoderH264 *)(decoder))
 349
 350 struct _GstVaapiDecoderH264Private {
 351     GstH264NalParser           *parser;
 352     GstVaapiPictureH264        *current_picture;
 353     GstVaapiParserInfoH264     *prev_slice_pi;
 354     GstVaapiFrameStore         *prev_frame;
 355     GstVaapiFrameStore         *dpb[16];
 356     guint                       dpb_count;
 357     guint                       dpb_size;
 358     GstVaapiProfile             profile;
 359     GstVaapiEntrypoint          entrypoint;
 360     GstVaapiChromaType          chroma_type;
 361     GstVaapiPictureH264        *short_ref[32];
 362     guint                       short_ref_count;
 363     GstVaapiPictureH264        *long_ref[32];
 364     guint                       long_ref_count;
 365     GstVaapiPictureH264        *RefPicList0[32];
 366     guint                       RefPicList0_count;
 367     GstVaapiPictureH264        *RefPicList1[32];
 368     guint                       RefPicList1_count;
 369     guint                       nal_length_size;
 370     guint                       mb_width;
 371     guint                       mb_height;
 372     gint32                      field_poc[2];           // 0:TopFieldOrderCnt / 1:BottomFieldOrderCnt
 373     gint32                      poc_msb;                // PicOrderCntMsb
 374     gint32                      poc_lsb;                // pic_order_cnt_lsb (from slice_header())
 375     gint32                      prev_poc_msb;           // prevPicOrderCntMsb
 376     gint32                      prev_poc_lsb;           // prevPicOrderCntLsb
 377     gint32                      frame_num_offset;       // FrameNumOffset
 378     gint32                      frame_num;              // frame_num (from slice_header())
 379     gint32                      prev_frame_num;         // prevFrameNum
 380     gboolean                    prev_pic_has_mmco5;     // prevMmco5Pic
 381     gboolean                    prev_pic_structure;     // previous picture structure
 382     guint                       is_opened               : 1;
 383     guint                       is_avcC                 : 1;
 384     guint                       got_sps                 : 1;
 385     guint                       got_pps                 : 1;
 386     guint                       has_context             : 1;
 387     guint                       progressive_sequence    : 1;
 388 };
 389
 390 /**
 391  * GstVaapiDecoderH264:
 392  *
 393  * A decoder based on H264.
 394  */
 395 struct _GstVaapiDecoderH264 {
 396     /*< private >*/
 397     GstVaapiDecoder             parent_instance;
 398     GstVaapiDecoderH264Private  priv;
 399 };
 400
 401 /**
 402  * GstVaapiDecoderH264Class:
 403  *
 404  * A decoder class based on H264.
 405  */
 406 struct _GstVaapiDecoderH264Class {
 407     /*< private >*/
 408     GstVaapiDecoderClass parent_class;
 409 };
 410
 411 static gboolean
 412 exec_ref_pic_marking(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture);
 413
 414 /* Get number of reference frames to use */
 415 static guint
 416 get_max_dec_frame_buffering(GstH264SPS *sps)
 417 {
 418     guint max_dec_frame_buffering, MaxDpbMbs, PicSizeMbs;
 419
 420     /* Table A-1 - Level limits */
 421     switch (sps->level_idc) {
 422     case 10: MaxDpbMbs = 396;    break;
 423     case 11: MaxDpbMbs = 900;    break;
 424     case 12: MaxDpbMbs = 2376;   break;
 425     case 13: MaxDpbMbs = 2376;   break;
 426     case 20: MaxDpbMbs = 2376;   break;
 427     case 21: MaxDpbMbs = 4752;   break;
 428     case 22: MaxDpbMbs = 8100;   break;
 429     case 30: MaxDpbMbs = 8100;   break;
 430     case 31: MaxDpbMbs = 18000;  break;
 431     case 32: MaxDpbMbs = 20480;  break;
 432     case 40: MaxDpbMbs = 32768;  break;
 433     case 41: MaxDpbMbs = 32768;  break;
 434     case 42: MaxDpbMbs = 34816;  break;
 435     case 50: MaxDpbMbs = 110400; break;
 436     case 51: MaxDpbMbs = 184320; break;
 437     default:
 438         g_assert(0 && "unhandled level");
 439         break;
 440     }
 441
 442     PicSizeMbs = ((sps->pic_width_in_mbs_minus1 + 1) *
 443                   (sps->pic_height_in_map_units_minus1 + 1) *
 444                   (sps->frame_mbs_only_flag ? 1 : 2));
 445     max_dec_frame_buffering = MaxDpbMbs / PicSizeMbs;
 446
 447     /* VUI parameters */
 448     if (sps->vui_parameters_present_flag) {
 449         GstH264VUIParams * const vui_params = &sps->vui_parameters;
 450         if (vui_params->bitstream_restriction_flag)
 451             max_dec_frame_buffering = vui_params->max_dec_frame_buffering;
 452         else {
 453             switch (sps->profile_idc) {
 454             case 44:  // CAVLC 4:4:4 Intra profile
 455             case GST_H264_PROFILE_SCALABLE_HIGH:
 456             case GST_H264_PROFILE_HIGH:
 457             case GST_H264_PROFILE_HIGH10:
 458             case GST_H264_PROFILE_HIGH_422:
 459             case GST_H264_PROFILE_HIGH_444:
 460                 if (sps->constraint_set3_flag)
 461                     max_dec_frame_buffering = 0;
 462                 break;
 463             }
 464         }
 465     }
 466
 467     if (max_dec_frame_buffering > 16)
 468         max_dec_frame_buffering = 16;
 469     else if (max_dec_frame_buffering < sps->num_ref_frames)
 470         max_dec_frame_buffering = sps->num_ref_frames;
 471     return MAX(1, max_dec_frame_buffering);
 472 }
 473
 474 static void
 475 array_remove_index_fast(void *array, guint *array_length_ptr, guint index)
 476 {
 477     gpointer * const entries = array;
 478     guint num_entries = *array_length_ptr;
 479
 480     g_return_if_fail(index < num_entries);
 481
 482     if (index != --num_entries)
 483         entries[index] = entries[num_entries];
 484     entries[num_entries] = NULL;
 485     *array_length_ptr = num_entries;
 486 }
 487
 488 #if 1
 489 static inline void
 490 array_remove_index(void *array, guint *array_length_ptr, guint index)
 491 {
 492     array_remove_index_fast(array, array_length_ptr, index);
 493 }
 494 #else
 495 static void
 496 array_remove_index(void *array, guint *array_length_ptr, guint index)
 497 {
 498     gpointer * const entries = array;
 499     const guint num_entries = *array_length_ptr - 1;
 500     guint i;
 501
 502     g_return_if_fail(index <= num_entries);
 503
 504     for (i = index; i < num_entries; i++)
 505         entries[i] = entries[i + 1];
 506     entries[num_entries] = NULL;
 507     *array_length_ptr = num_entries;
 508 }
 509 #endif
 510
 511 #define ARRAY_REMOVE_INDEX(array, index) \
 512     array_remove_index(array, &array##_count, index)
 513
 514 static void
 515 dpb_remove_index(GstVaapiDecoderH264 *decoder, guint index)
 516 {
 517     GstVaapiDecoderH264Private * const priv = &decoder->priv;
 518     guint i, num_frames = --priv->dpb_count;
 519
 520     if (USE_STRICT_DPB_ORDERING) {
 521         for (i = index; i < num_frames; i++)
 522             gst_vaapi_frame_store_replace(&priv->dpb[i], priv->dpb[i + 1]);
 523     }
 524     else if (index != num_frames)
 525         gst_vaapi_frame_store_replace(&priv->dpb[index], priv->dpb[num_frames]);
 526     gst_vaapi_frame_store_replace(&priv->dpb[num_frames], NULL);
 527 }
 528
 529 static gboolean
 530 dpb_output(
 531     GstVaapiDecoderH264 *decoder,
 532     GstVaapiFrameStore  *fs,
 533     GstVaapiPictureH264 *picture
 534 )
 535 {
 536     picture->output_needed = FALSE;
 537
 538     if (fs) {
 539         if (--fs->output_needed > 0)
 540             return TRUE;
 541         picture = fs->buffers[0];
 542     }
 543     return gst_vaapi_picture_output(GST_VAAPI_PICTURE_CAST(picture));
 544 }
 545
 546 static inline void
 547 dpb_evict(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture, guint i)
 548 {
 549     GstVaapiDecoderH264Private * const priv = &decoder->priv;
 550     GstVaapiFrameStore * const fs = priv->dpb[i];
 551
 552     if (!fs->output_needed && !gst_vaapi_frame_store_has_reference(fs))
 553         dpb_remove_index(decoder, i);
 554 }
 555
 556 static gboolean
 557 dpb_bump(GstVaapiDecoderH264 *decoder)
 558 {
 559     GstVaapiDecoderH264Private * const priv = &decoder->priv;
 560     GstVaapiPictureH264 *found_picture = NULL;
 561     guint i, j, found_index;
 562     gboolean success;
 563
 564     for (i = 0; i < priv->dpb_count; i++) {
 565         GstVaapiFrameStore * const fs = priv->dpb[i];
 566         if (!fs->output_needed)
 567             continue;
 568         for (j = 0; j < fs->num_buffers; j++) {
 569             GstVaapiPictureH264 * const picture = fs->buffers[j];
 570             if (!picture->output_needed)
 571                 continue;
 572             if (!found_picture || found_picture->base.poc > picture->base.poc)
 573                 found_picture = picture, found_index = i;
 574         }
 575     }
 576     if (!found_picture)
 577         return FALSE;
 578
 579     success = dpb_output(decoder, priv->dpb[found_index], found_picture);
 580     dpb_evict(decoder, found_picture, found_index);
 581     return success;
 582 }
 583
 584 static void
 585 dpb_clear(GstVaapiDecoderH264 *decoder)
 586 {
 587     GstVaapiDecoderH264Private * const priv = &decoder->priv;
 588     guint i;
 589
 590     for (i = 0; i < priv->dpb_count; i++)
 591         gst_vaapi_frame_store_replace(&priv->dpb[i], NULL);
 592     priv->dpb_count = 0;
 593
 594     gst_vaapi_frame_store_replace(&priv->prev_frame, NULL);
 595 }
 596
 597 static void
 598 dpb_flush(GstVaapiDecoderH264 *decoder)
 599 {
 600     while (dpb_bump(decoder))
 601         ;
 602     dpb_clear(decoder);
 603 }
 604
 605 static gboolean
 606 dpb_add(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
 607 {
 608     GstVaapiDecoderH264Private * const priv = &decoder->priv;
 609     GstVaapiFrameStore *fs;
 610     guint i, j;
 611
 612     // Remove all unused pictures
 613     if (!GST_VAAPI_PICTURE_IS_IDR(picture)) {
 614         i = 0;
 615         while (i < priv->dpb_count) {
 616             GstVaapiFrameStore * const fs = priv->dpb[i];
 617             if (!fs->output_needed && !gst_vaapi_frame_store_has_reference(fs))
 618                 dpb_remove_index(decoder, i);
 619             else
 620                 i++;
 621         }
 622     }
 623
 624     // Check if picture is the second field and the first field is still in DPB
 625     fs = priv->prev_frame;
 626     if (fs && !gst_vaapi_frame_store_has_frame(fs))
 627         return gst_vaapi_frame_store_add(fs, picture);
 628
 629     // Create new frame store, and split fields if necessary
 630     fs = gst_vaapi_frame_store_new(picture);
 631     if (!fs)
 632         return FALSE;
 633     gst_vaapi_frame_store_replace(&priv->prev_frame, fs);
 634     gst_vaapi_frame_store_unref(fs);
 635
 636     if (!priv->progressive_sequence && gst_vaapi_frame_store_has_frame(fs)) {
 637         if (!gst_vaapi_frame_store_split_fields(fs))
 638             return FALSE;
 639     }
 640
 641     // C.4.5.1 - Storage and marking of a reference decoded picture into the DPB
 642     if (GST_VAAPI_PICTURE_IS_REFERENCE(picture)) {
 643         while (priv->dpb_count == priv->dpb_size) {
 644             if (!dpb_bump(decoder))
 645                 return FALSE;
 646         }
 647         gst_vaapi_frame_store_replace(&priv->dpb[priv->dpb_count++], fs);
 648         if (picture->output_flag) {
 649             picture->output_needed = TRUE;
 650             fs->output_needed++;
 651         }
 652     }
 653
 654     // C.4.5.2 - Storage and marking of a non-reference decoded picture into the DPB
 655     else {
 656         if (!picture->output_flag)
 657             return TRUE;
 658         while (priv->dpb_count == priv->dpb_size) {
 659             gboolean found_picture = FALSE;
 660             for (i = 0; !found_picture && i < priv->dpb_count; i++) {
 661                 GstVaapiFrameStore * const fs = priv->dpb[i];
 662                 if (!fs->output_needed)
 663                     continue;
 664                 for (j = 0; !found_picture && j < fs->num_buffers; j++)
 665                     found_picture = fs->buffers[j]->output_needed &&
 666                         fs->buffers[j]->base.poc < picture->base.poc;
 667             }
 668             if (!found_picture)
 669                 return dpb_output(decoder, NULL, picture);
 670             if (!dpb_bump(decoder))
 671                 return FALSE;
 672         }
 673         gst_vaapi_frame_store_replace(&priv->dpb[priv->dpb_count++], fs);
 674         picture->output_needed = TRUE;
 675         fs->output_needed++;
 676     }
 677     return TRUE;
 678 }
 679
 680 static inline void
 681 dpb_reset(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 682 {
 683     GstVaapiDecoderH264Private * const priv = &decoder->priv;
 684
 685     priv->dpb_size = get_max_dec_frame_buffering(sps);
 686     GST_DEBUG("DPB size %u", priv->dpb_size);
 687 }
 688
 689 static GstVaapiDecoderStatus
 690 get_status(GstH264ParserResult result)
 691 {
 692     GstVaapiDecoderStatus status;
 693
 694     switch (result) {
 695     case GST_H264_PARSER_OK:
 696         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
 697         break;
 698     case GST_H264_PARSER_NO_NAL_END:
 699         status = GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
 700         break;
 701     case GST_H264_PARSER_ERROR:
 702         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
 703         break;
 704     default:
 705         status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 706         break;
 707     }
 708     return status;
 709 }
 710
 711 static void
 712 gst_vaapi_decoder_h264_close(GstVaapiDecoderH264 *decoder)
 713 {
 714     GstVaapiDecoderH264Private * const priv = &decoder->priv;
 715
 716     gst_vaapi_picture_replace(&priv->current_picture, NULL);
 717     gst_vaapi_parser_info_h264_replace(&priv->prev_slice_pi, NULL);
 718
 719     dpb_clear(decoder);
 720
 721     if (priv->parser) {
 722         gst_h264_nal_parser_free(priv->parser);
 723         priv->parser = NULL;
 724     }
 725 }
 726
 727 static gboolean
 728 gst_vaapi_decoder_h264_open(GstVaapiDecoderH264 *decoder)
 729 {
 730     GstVaapiDecoderH264Private * const priv = &decoder->priv;
 731
 732     gst_vaapi_decoder_h264_close(decoder);
 733
 734     priv->parser = gst_h264_nal_parser_new();
 735     if (!priv->parser)
 736         return FALSE;
 737     return TRUE;
 738 }
 739
 740 static void
 741 gst_vaapi_decoder_h264_destroy(GstVaapiDecoder *base_decoder)
 742 {
 743     GstVaapiDecoderH264 * const decoder =
 744         GST_VAAPI_DECODER_H264_CAST(base_decoder);
 745
 746     gst_vaapi_decoder_h264_close(decoder);
 747 }
 748
 749 static gboolean
 750 gst_vaapi_decoder_h264_create(GstVaapiDecoder *base_decoder)
 751 {
 752     GstVaapiDecoderH264 * const decoder =
 753         GST_VAAPI_DECODER_H264_CAST(base_decoder);
 754     GstVaapiDecoderH264Private * const priv = &decoder->priv;
 755
 756     priv->profile               = GST_VAAPI_PROFILE_UNKNOWN;
 757     priv->entrypoint            = GST_VAAPI_ENTRYPOINT_VLD;
 758     priv->chroma_type           = GST_VAAPI_CHROMA_TYPE_YUV420;
 759     priv->prev_pic_structure    = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
 760     priv->progressive_sequence  = TRUE;
 761     return TRUE;
 762 }
 763
 764 static guint
 765 h264_get_profile(GstH264SPS *sps)
 766 {
 767     guint profile = 0;
 768
 769     switch (sps->profile_idc) {
 770     case GST_H264_PROFILE_BASELINE:
 771         profile = GST_VAAPI_PROFILE_H264_BASELINE;
 772         break;
 773     case GST_H264_PROFILE_MAIN:
 774         profile = GST_VAAPI_PROFILE_H264_MAIN;
 775         break;
 776     case GST_H264_PROFILE_HIGH:
 777         profile = GST_VAAPI_PROFILE_H264_HIGH;
 778         break;
 779     }
 780     return profile;
 781 }
 782
 783 static guint
 784 h264_get_chroma_type(GstH264SPS *sps)
 785 {
 786     guint chroma_type = 0;
 787
 788     switch (sps->chroma_format_idc) {
 789     case 1:
 790         chroma_type = GST_VAAPI_CHROMA_TYPE_YUV420;
 791         break;
 792     case 2:
 793         chroma_type = GST_VAAPI_CHROMA_TYPE_YUV422;
 794         break;
 795     case 3:
 796         if (!sps->separate_colour_plane_flag)
 797             chroma_type = GST_VAAPI_CHROMA_TYPE_YUV444;
 798         break;
 799     }
 800     return chroma_type;
 801 }
 802
 803 static GstVaapiProfile
 804 get_profile(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 805 {
 806     GstVaapiDecoderH264Private * const priv = &decoder->priv;
 807     GstVaapiDisplay * const display = GST_VAAPI_DECODER_DISPLAY(decoder);
 808     GstVaapiProfile profile, profiles[2];
 809     guint i, n_profiles = 0;
 810
 811     profile = h264_get_profile(sps);
 812     if (!profile)
 813         return GST_VAAPI_PROFILE_UNKNOWN;
 814
 815     profiles[n_profiles++] = profile;
 816     switch (profile) {
 817     case GST_VAAPI_PROFILE_H264_MAIN:
 818         profiles[n_profiles++] = GST_VAAPI_PROFILE_H264_HIGH;
 819         break;
 820     default:
 821         break;
 822     }
 823
 824     /* If the preferred profile (profiles[0]) matches one that we already
 825        found, then just return it now instead of searching for it again */
 826     if (profiles[0] == priv->profile)
 827         return priv->profile;
 828
 829     for (i = 0; i < n_profiles; i++) {
 830         if (gst_vaapi_display_has_decoder(display, profiles[i], priv->entrypoint))
 831             return profiles[i];
 832     }
 833     return GST_VAAPI_PROFILE_UNKNOWN;
 834 }
 835
 836 static GstVaapiDecoderStatus
 837 ensure_context(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 838 {
 839     GstVaapiDecoder * const base_decoder = GST_VAAPI_DECODER_CAST(decoder);
 840     GstVaapiDecoderH264Private * const priv = &decoder->priv;
 841     GstVaapiContextInfo info;
 842     GstVaapiProfile profile;
 843     GstVaapiChromaType chroma_type;
 844     gboolean reset_context = FALSE;
 845     guint mb_width, mb_height;
 846
 847     profile = get_profile(decoder, sps);
 848     if (!profile) {
 849         GST_ERROR("unsupported profile_idc %u", sps->profile_idc);
 850         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_PROFILE;
 851     }
 852
 853     if (priv->profile != profile) {
 854         GST_DEBUG("profile changed");
 855         reset_context = TRUE;
 856         priv->profile = profile;
 857     }
 858
 859     chroma_type = h264_get_chroma_type(sps);
 860     if (!chroma_type || chroma_type != GST_VAAPI_CHROMA_TYPE_YUV420) {
 861         GST_ERROR("unsupported chroma_format_idc %u", sps->chroma_format_idc);
 862         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;
 863     }
 864
 865     if (priv->chroma_type != chroma_type) {
 866         GST_DEBUG("chroma format changed");
 867         reset_context     = TRUE;
 868         priv->chroma_type = chroma_type;
 869     }
 870
 871     mb_width  = sps->pic_width_in_mbs_minus1 + 1;
 872     mb_height = (sps->pic_height_in_map_units_minus1 + 1) <<
 873         !sps->frame_mbs_only_flag;
 874     if (priv->mb_width != mb_width || priv->mb_height != mb_height) {
 875         GST_DEBUG("size changed");
 876         reset_context   = TRUE;
 877         priv->mb_width  = mb_width;
 878         priv->mb_height = mb_height;
 879     }
 880
 881     priv->progressive_sequence = sps->frame_mbs_only_flag;
 882 #if 0
 883     /* XXX: we only output complete frames for now */
 884     gst_vaapi_decoder_set_interlaced(base_decoder, !priv->progressive_sequence);
 885 #endif
 886
 887     gst_vaapi_decoder_set_pixel_aspect_ratio(
 888         base_decoder,
 889         sps->vui_parameters.par_n,
 890         sps->vui_parameters.par_d
 891     );
 892
 893     if (!reset_context && priv->has_context)
 894         return GST_VAAPI_DECODER_STATUS_SUCCESS;
 895
 896     /* XXX: fix surface size when cropping is implemented */
 897     info.profile    = priv->profile;
 898     info.entrypoint = priv->entrypoint;
 899     info.width      = sps->width;
 900     info.height     = sps->height;
 901     info.ref_frames = get_max_dec_frame_buffering(sps);
 902
 903     if (!gst_vaapi_decoder_ensure_context(GST_VAAPI_DECODER(decoder), &info))
 904         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 905     priv->has_context = TRUE;
 906
 907     /* Reset DPB */
 908     dpb_reset(decoder, sps);
 909     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 910 }
 911
 912 static void
 913 fill_iq_matrix_4x4(VAIQMatrixBufferH264 *iq_matrix, const GstH264PPS *pps)
 914 {
 915     const guint8 (* const ScalingList4x4)[6][16] = &pps->scaling_lists_4x4;
 916     guint i, j;
 917
 918     /* There are always 6 4x4 scaling lists */
 919     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList4x4) == 6);
 920     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList4x4[0]) == 16);
 921
 922     if (sizeof(iq_matrix->ScalingList4x4[0][0]) == 1)
 923         memcpy(iq_matrix->ScalingList4x4, *ScalingList4x4,
 924                sizeof(iq_matrix->ScalingList4x4));
 925     else {
 926         for (i = 0; i < G_N_ELEMENTS(iq_matrix->ScalingList4x4); i++) {
 927             for (j = 0; j < G_N_ELEMENTS(iq_matrix->ScalingList4x4[i]); j++)
 928                 iq_matrix->ScalingList4x4[i][j] = (*ScalingList4x4)[i][j];
 929         }
 930     }
 931 }
 932
 933 static void
 934 fill_iq_matrix_8x8(VAIQMatrixBufferH264 *iq_matrix, const GstH264PPS *pps)
 935 {
 936     const guint8 (* const ScalingList8x8)[6][64] = &pps->scaling_lists_8x8;
 937     const GstH264SPS * const sps = pps->sequence;
 938     guint i, j, n;
 939
 940     /* If chroma_format_idc != 3, there are up to 2 8x8 scaling lists */
 941     if (!pps->transform_8x8_mode_flag)
 942         return;
 943
 944     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList8x8) >= 2);
 945     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList8x8[0]) == 64);
 946
 947     if (sizeof(iq_matrix->ScalingList8x8[0][0]) == 1)
 948         memcpy(iq_matrix->ScalingList8x8, *ScalingList8x8,
 949                sizeof(iq_matrix->ScalingList8x8));
 950     else {
 951         n = (sps->chroma_format_idc != 3) ? 2 : 6;
 952         for (i = 0; i < n; i++) {
 953             for (j = 0; j < G_N_ELEMENTS(iq_matrix->ScalingList8x8[i]); j++)
 954                 iq_matrix->ScalingList8x8[i][j] = (*ScalingList8x8)[i][j];
 955         }
 956     }
 957 }
 958
 959 static GstVaapiDecoderStatus
 960 ensure_quant_matrix(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
 961 {
 962     GstVaapiPicture * const base_picture = &picture->base;
 963     GstH264PPS * const pps = picture->pps;
 964     GstH264SPS * const sps = pps->sequence;
 965     VAIQMatrixBufferH264 *iq_matrix;
 966
 967     base_picture->iq_matrix = GST_VAAPI_IQ_MATRIX_NEW(H264, decoder);
 968     if (!base_picture->iq_matrix) {
 969         GST_ERROR("failed to allocate IQ matrix");
 970         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
 971     }
 972     iq_matrix = base_picture->iq_matrix->param;
 973
 974     /* XXX: we can only support 4:2:0 or 4:2:2 since ScalingLists8x8[]
 975        is not large enough to hold lists for 4:4:4 */
 976     if (sps->chroma_format_idc == 3)
 977         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;
 978
 979     fill_iq_matrix_4x4(iq_matrix, pps);
 980     fill_iq_matrix_8x8(iq_matrix, pps);
 981
 982     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 983 }
 984
 985 static GstVaapiDecoderStatus
 986 decode_current_picture(GstVaapiDecoderH264 *decoder)
 987 {
 988     GstVaapiDecoderH264Private * const priv = &decoder->priv;
 989     GstVaapiPictureH264 * const picture = priv->current_picture;
 990
 991     if (!picture)
 992         return GST_VAAPI_DECODER_STATUS_SUCCESS;
 993
 994     if (!exec_ref_pic_marking(decoder, picture))
 995         goto error;
 996     if (!dpb_add(decoder, picture))
 997         goto error;
 998     if (!gst_vaapi_picture_decode(GST_VAAPI_PICTURE_CAST(picture)))
 999         goto error;
1000     if (priv->prev_frame && gst_vaapi_frame_store_has_frame(priv->prev_frame))
1001         gst_vaapi_picture_replace(&priv->current_picture, NULL);
1002     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1003
1004 error:
1005     /* XXX: fix for cases where first field failed to be decoded */
1006     gst_vaapi_picture_replace(&priv->current_picture, NULL);
1007     return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
1008 }
1009
1010 static GstVaapiDecoderStatus
1011 parse_sps(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnit *unit)
1012 {
1013     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1014     GstVaapiParserInfoH264 * const pi = unit->parsed_info;
1015     GstH264SPS * const sps = &pi->data.sps;
1016     GstH264ParserResult result;
1017
1018     GST_DEBUG("parse SPS");
1019
1020     /* Variables that don't have inferred values per the H.264
1021        standard but that should get a default value anyway */
1022     sps->log2_max_pic_order_cnt_lsb_minus4 = 0;
1023
1024     result = gst_h264_parser_parse_sps(priv->parser, &pi->nalu, sps, TRUE);
1025     if (result != GST_H264_PARSER_OK)
1026         return get_status(result);
1027
1028     priv->got_sps = TRUE;
1029     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1030 }
1031
1032 static GstVaapiDecoderStatus
1033 parse_pps(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnit *unit)
1034 {
1035     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1036     GstVaapiParserInfoH264 * const pi = unit->parsed_info;
1037     GstH264PPS * const pps = &pi->data.pps;
1038     GstH264ParserResult result;
1039
1040     GST_DEBUG("parse PPS");
1041
1042     /* Variables that don't have inferred values per the H.264
1043        standard but that should get a default value anyway */
1044     pps->slice_group_map_type = 0;
1045     pps->slice_group_change_rate_minus1 = 0;
1046
1047     result = gst_h264_parser_parse_pps(priv->parser, &pi->nalu, pps);
1048     if (result != GST_H264_PARSER_OK)
1049         return get_status(result);
1050
1051     priv->got_pps = TRUE;
1052     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1053 }
1054
1055 static GstVaapiDecoderStatus
1056 parse_sei(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnit *unit)
1057 {
1058     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1059     GstVaapiParserInfoH264 * const pi = unit->parsed_info;
1060     GstH264SEIMessage sei;
1061     GstH264ParserResult result;
1062
1063     GST_DEBUG("parse SEI");
1064
1065     memset(&sei, 0, sizeof(sei));
1066     result = gst_h264_parser_parse_sei(priv->parser, &pi->nalu, &sei);
1067     if (result != GST_H264_PARSER_OK) {
1068         GST_WARNING("failed to parse SEI, payload type:%d", sei.payloadType);
1069         return get_status(result);
1070     }
1071
1072     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1073 }
1074
1075 static GstVaapiDecoderStatus
1076 parse_slice(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnit *unit)
1077 {
1078     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1079     GstVaapiParserInfoH264 * const pi = unit->parsed_info;
1080     GstH264SliceHdr * const slice_hdr = &pi->data.slice_hdr;
1081     GstH264ParserResult result;
1082
1083     GST_DEBUG("parse slice");
1084
1085     /* Variables that don't have inferred values per the H.264
1086        standard but that should get a default value anyway */
1087     slice_hdr->cabac_init_idc = 0;
1088     slice_hdr->direct_spatial_mv_pred_flag = 0;
1089
1090     result = gst_h264_parser_parse_slice_hdr(priv->parser, &pi->nalu,
1091         slice_hdr, TRUE, TRUE);
1092     if (result != GST_H264_PARSER_OK)
1093         return get_status(result);
1094
1095     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1096 }
1097
1098 static GstVaapiDecoderStatus
1099 decode_sequence_end(GstVaapiDecoderH264 *decoder)
1100 {
1101     GstVaapiDecoderStatus status;
1102
1103     GST_DEBUG("decode sequence-end");
1104
1105     status = decode_current_picture(decoder);
1106     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
1107         return status;
1108
1109     dpb_flush(decoder);
1110     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1111 }
1112
1113 /* 8.2.1.1 - Decoding process for picture order count type 0 */
1114 static void
1115 init_picture_poc_0(
1116     GstVaapiDecoderH264 *decoder,
1117     GstVaapiPictureH264 *picture,
1118     GstH264SliceHdr     *slice_hdr
1119 )
1120 {
1121     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1122     GstH264PPS * const pps = slice_hdr->pps;
1123     GstH264SPS * const sps = pps->sequence;
1124     const gint32 MaxPicOrderCntLsb = 1 << (sps->log2_max_pic_order_cnt_lsb_minus4 + 4);
1125     gint32 temp_poc;
1126
1127     GST_DEBUG("decode picture order count type 0");
1128
1129     if (GST_VAAPI_PICTURE_IS_IDR(picture)) {
1130         priv->prev_poc_msb = 0;
1131         priv->prev_poc_lsb = 0;
1132     }
1133     else if (priv->prev_pic_has_mmco5) {
1134         priv->prev_poc_msb = 0;
1135         priv->prev_poc_lsb =
1136             (priv->prev_pic_structure == GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD ?
1137              0 : priv->field_poc[TOP_FIELD]);
1138     }
1139     else {
1140         priv->prev_poc_msb = priv->poc_msb;
1141         priv->prev_poc_lsb = priv->poc_lsb;
1142     }
1143
1144     // (8-3)
1145     priv->poc_lsb = slice_hdr->pic_order_cnt_lsb;
1146     if (priv->poc_lsb < priv->prev_poc_lsb &&
1147         (priv->prev_poc_lsb - priv->poc_lsb) >= (MaxPicOrderCntLsb / 2))
1148         priv->poc_msb = priv->prev_poc_msb + MaxPicOrderCntLsb;
1149     else if (priv->poc_lsb > priv->prev_poc_lsb &&
1150              (priv->poc_lsb - priv->prev_poc_lsb) > (MaxPicOrderCntLsb / 2))
1151         priv->poc_msb = priv->prev_poc_msb - MaxPicOrderCntLsb;
1152     else
1153         priv->poc_msb = priv->prev_poc_msb;
1154
1155     temp_poc = priv->poc_msb + priv->poc_lsb;
1156     switch (picture->structure) {
1157     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
1158         // (8-4, 8-5)
1159         priv->field_poc[TOP_FIELD] = temp_poc;
1160         priv->field_poc[BOTTOM_FIELD] = temp_poc +
1161             slice_hdr->delta_pic_order_cnt_bottom;
1162         break;
1163     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
1164         // (8-4)
1165         priv->field_poc[TOP_FIELD] = temp_poc;
1166         break;
1167     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
1168         // (8-5)
1169         priv->field_poc[BOTTOM_FIELD] = temp_poc;
1170         break;
1171     }
1172 }
1173
1174 /* 8.2.1.2 - Decoding process for picture order count type 1 */
1175 static void
1176 init_picture_poc_1(
1177     GstVaapiDecoderH264 *decoder,
1178     GstVaapiPictureH264 *picture,
1179     GstH264SliceHdr     *slice_hdr
1180 )
1181 {
1182     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1183     GstH264PPS * const pps = slice_hdr->pps;
1184     GstH264SPS * const sps = pps->sequence;
1185     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1186     gint32 prev_frame_num_offset, abs_frame_num, expected_poc;
1187     guint i;
1188
1189     GST_DEBUG("decode picture order count type 1");
1190
1191     if (priv->prev_pic_has_mmco5)
1192         prev_frame_num_offset = 0;
1193     else
1194         prev_frame_num_offset = priv->frame_num_offset;
1195
1196     // (8-6)
1197     if (GST_VAAPI_PICTURE_IS_IDR(picture))
1198         priv->frame_num_offset = 0;
1199     else if (priv->prev_frame_num > priv->frame_num)
1200         priv->frame_num_offset = prev_frame_num_offset + MaxFrameNum;
1201     else
1202         priv->frame_num_offset = prev_frame_num_offset;
1203
1204     // (8-7)
1205     if (sps->num_ref_frames_in_pic_order_cnt_cycle != 0)
1206         abs_frame_num = priv->frame_num_offset + priv->frame_num;
1207     else
1208         abs_frame_num = 0;
1209     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture) && abs_frame_num > 0)
1210         abs_frame_num = abs_frame_num - 1;
1211
1212     if (abs_frame_num > 0) {
1213         gint32 expected_delta_per_poc_cycle;
1214         gint32 poc_cycle_cnt, frame_num_in_poc_cycle;
1215
1216         expected_delta_per_poc_cycle = 0;
1217         for (i = 0; i < sps->num_ref_frames_in_pic_order_cnt_cycle; i++)
1218             expected_delta_per_poc_cycle += sps->offset_for_ref_frame[i];
1219
1220         // (8-8)
1221         poc_cycle_cnt = (abs_frame_num - 1) /
1222             sps->num_ref_frames_in_pic_order_cnt_cycle;
1223         frame_num_in_poc_cycle = (abs_frame_num - 1) %
1224             sps->num_ref_frames_in_pic_order_cnt_cycle;
1225
1226         // (8-9)
1227         expected_poc = poc_cycle_cnt * expected_delta_per_poc_cycle;
1228         for (i = 0; i <= frame_num_in_poc_cycle; i++)
1229             expected_poc += sps->offset_for_ref_frame[i];
1230     }
1231     else
1232         expected_poc = 0;
1233     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
1234         expected_poc += sps->offset_for_non_ref_pic;
1235
1236     // (8-10)
1237     switch (picture->structure) {
1238     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
1239         priv->field_poc[TOP_FIELD] = expected_poc +
1240             slice_hdr->delta_pic_order_cnt[0];
1241         priv->field_poc[BOTTOM_FIELD] = priv->field_poc[TOP_FIELD] +
1242             sps->offset_for_top_to_bottom_field +
1243             slice_hdr->delta_pic_order_cnt[1];
1244         break;
1245     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
1246         priv->field_poc[TOP_FIELD] = expected_poc +
1247             slice_hdr->delta_pic_order_cnt[0];
1248         break;
1249     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
1250         priv->field_poc[BOTTOM_FIELD] = expected_poc +
1251             sps->offset_for_top_to_bottom_field +
1252             slice_hdr->delta_pic_order_cnt[0];
1253         break;
1254     }
1255 }
1256
1257 /* 8.2.1.3 - Decoding process for picture order count type 2 */
1258 static void
1259 init_picture_poc_2(
1260     GstVaapiDecoderH264 *decoder,
1261     GstVaapiPictureH264 *picture,
1262     GstH264SliceHdr     *slice_hdr
1263 )
1264 {
1265     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1266     GstH264PPS * const pps = slice_hdr->pps;
1267     GstH264SPS * const sps = pps->sequence;
1268     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1269     gint32 prev_frame_num_offset, temp_poc;
1270
1271     GST_DEBUG("decode picture order count type 2");
1272
1273     if (priv->prev_pic_has_mmco5)
1274         prev_frame_num_offset = 0;
1275     else
1276         prev_frame_num_offset = priv->frame_num_offset;
1277
1278     // (8-11)
1279     if (GST_VAAPI_PICTURE_IS_IDR(picture))
1280         priv->frame_num_offset = 0;
1281     else if (priv->prev_frame_num > priv->frame_num)
1282         priv->frame_num_offset = prev_frame_num_offset + MaxFrameNum;
1283     else
1284         priv->frame_num_offset = prev_frame_num_offset;
1285
1286     // (8-12)
1287     if (GST_VAAPI_PICTURE_IS_IDR(picture))
1288         temp_poc = 0;
1289     else if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
1290         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num) - 1;
1291     else
1292         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num);
1293
1294     // (8-13)
1295     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
1296         priv->field_poc[TOP_FIELD] = temp_poc;
1297     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
1298         priv->field_poc[BOTTOM_FIELD] = temp_poc;
1299 }
1300
1301 /* 8.2.1 - Decoding process for picture order count */
1302 static void
1303 init_picture_poc(
1304     GstVaapiDecoderH264 *decoder,
1305     GstVaapiPictureH264 *picture,
1306     GstH264SliceHdr     *slice_hdr
1307 )
1308 {
1309     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1310     GstH264PPS * const pps = slice_hdr->pps;
1311     GstH264SPS * const sps = pps->sequence;
1312
1313     switch (sps->pic_order_cnt_type) {
1314     case 0:
1315         init_picture_poc_0(decoder, picture, slice_hdr);
1316         break;
1317     case 1:
1318         init_picture_poc_1(decoder, picture, slice_hdr);
1319         break;
1320     case 2:
1321         init_picture_poc_2(decoder, picture, slice_hdr);
1322         break;
1323     }
1324
1325     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
1326         picture->field_poc[TOP_FIELD] = priv->field_poc[TOP_FIELD];
1327     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
1328         picture->field_poc[BOTTOM_FIELD] = priv->field_poc[BOTTOM_FIELD];
1329     picture->base.poc = MIN(picture->field_poc[0], picture->field_poc[1]);
1330 }
1331
1332 static int
1333 compare_picture_pic_num_dec(const void *a, const void *b)
1334 {
1335     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1336     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1337
1338     return picB->pic_num - picA->pic_num;
1339 }
1340
1341 static int
1342 compare_picture_long_term_pic_num_inc(const void *a, const void *b)
1343 {
1344     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1345     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1346
1347     return picA->long_term_pic_num - picB->long_term_pic_num;
1348 }
1349
1350 static int
1351 compare_picture_poc_dec(const void *a, const void *b)
1352 {
1353     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1354     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1355
1356     return picB->base.poc - picA->base.poc;
1357 }
1358
1359 static int
1360 compare_picture_poc_inc(const void *a, const void *b)
1361 {
1362     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1363     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1364
1365     return picA->base.poc - picB->base.poc;
1366 }
1367
1368 static int
1369 compare_picture_frame_num_wrap_dec(const void *a, const void *b)
1370 {
1371     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1372     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1373
1374     return picB->frame_num_wrap - picA->frame_num_wrap;
1375 }
1376
1377 static int
1378 compare_picture_long_term_frame_idx_inc(const void *a, const void *b)
1379 {
1380     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1381     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1382
1383     return picA->long_term_frame_idx - picB->long_term_frame_idx;
1384 }
1385
1386 /* 8.2.4.1 - Decoding process for picture numbers */
1387 static void
1388 init_picture_refs_pic_num(
1389     GstVaapiDecoderH264 *decoder,
1390     GstVaapiPictureH264 *picture,
1391     GstH264SliceHdr     *slice_hdr
1392 )
1393 {
1394     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1395     GstH264PPS * const pps = slice_hdr->pps;
1396     GstH264SPS * const sps = pps->sequence;
1397     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1398     guint i;
1399
1400     GST_DEBUG("decode picture numbers");
1401
1402     for (i = 0; i < priv->short_ref_count; i++) {
1403         GstVaapiPictureH264 * const pic = priv->short_ref[i];
1404
1405         // (8-27)
1406         if (pic->frame_num > priv->frame_num)
1407             pic->frame_num_wrap = pic->frame_num - MaxFrameNum;
1408         else
1409             pic->frame_num_wrap = pic->frame_num;
1410
1411         // (8-28, 8-30, 8-31)
1412         if (GST_VAAPI_PICTURE_IS_FRAME(picture))
1413             pic->pic_num = pic->frame_num_wrap;
1414         else {
1415             if (pic->structure == picture->structure)
1416                 pic->pic_num = 2 * pic->frame_num_wrap + 1;
1417             else
1418                 pic->pic_num = 2 * pic->frame_num_wrap;
1419         }
1420     }
1421
1422     for (i = 0; i < priv->long_ref_count; i++) {
1423         GstVaapiPictureH264 * const pic = priv->long_ref[i];
1424
1425         // (8-29, 8-32, 8-33)
1426         if (GST_VAAPI_PICTURE_IS_FRAME(picture))
1427             pic->long_term_pic_num = pic->long_term_frame_idx;
1428         else {
1429             if (pic->structure == picture->structure)
1430                 pic->long_term_pic_num = 2 * pic->long_term_frame_idx + 1;
1431             else
1432                 pic->long_term_pic_num = 2 * pic->long_term_frame_idx;
1433         }
1434     }
1435 }
1436
1437 #define SORT_REF_LIST(list, n, compare_func) \
1438     qsort(list, n, sizeof(*(list)), compare_picture_##compare_func)
1439
1440 static void
1441 init_picture_refs_fields_1(
1442     guint                picture_structure,
1443     GstVaapiPictureH264 *RefPicList[32],
1444     guint               *RefPicList_count,
1445     GstVaapiPictureH264 *ref_list[32],
1446     guint                ref_list_count
1447 )
1448 {
1449     guint i, j, n;
1450
1451     i = 0;
1452     j = 0;
1453     n = *RefPicList_count;
1454     do {
1455         g_assert(n < 32);
1456         for (; i < ref_list_count; i++) {
1457             if (ref_list[i]->structure == picture_structure) {
1458                 RefPicList[n++] = ref_list[i++];
1459                 break;
1460             }
1461         }
1462         for (; j < ref_list_count; j++) {
1463             if (ref_list[j]->structure != picture_structure) {
1464                 RefPicList[n++] = ref_list[j++];
1465                 break;
1466             }
1467         }
1468     } while (i < ref_list_count || j < ref_list_count);
1469     *RefPicList_count = n;
1470 }
1471
1472 static inline void
1473 init_picture_refs_fields(
1474     GstVaapiPictureH264 *picture,
1475     GstVaapiPictureH264 *RefPicList[32],
1476     guint               *RefPicList_count,
1477     GstVaapiPictureH264 *short_ref[32],
1478     guint                short_ref_count,
1479     GstVaapiPictureH264 *long_ref[32],
1480     guint                long_ref_count
1481 )
1482 {
1483     guint n = 0;
1484
1485     /* 8.2.4.2.5 - reference picture lists in fields */
1486     init_picture_refs_fields_1(picture->structure, RefPicList, &n,
1487         short_ref, short_ref_count);
1488     init_picture_refs_fields_1(picture->structure, RefPicList, &n,
1489         long_ref, long_ref_count);
1490     *RefPicList_count = n;
1491 }
1492
1493 static void
1494 init_picture_refs_p_slice(
1495     GstVaapiDecoderH264 *decoder,
1496     GstVaapiPictureH264 *picture,
1497     GstH264SliceHdr     *slice_hdr
1498 )
1499 {
1500     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1501     GstVaapiPictureH264 **ref_list;
1502     guint i;
1503
1504     GST_DEBUG("decode reference picture list for P and SP slices");
1505
1506     if (GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1507         /* 8.2.4.2.1 - P and SP slices in frames */
1508         if (priv->short_ref_count > 0) {
1509             ref_list = priv->RefPicList0;
1510             for (i = 0; i < priv->short_ref_count; i++)
1511                 ref_list[i] = priv->short_ref[i];
1512             SORT_REF_LIST(ref_list, i, pic_num_dec);
1513             priv->RefPicList0_count += i;
1514         }
1515
1516         if (priv->long_ref_count > 0) {
1517             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1518             for (i = 0; i < priv->long_ref_count; i++)
1519                 ref_list[i] = priv->long_ref[i];
1520             SORT_REF_LIST(ref_list, i, long_term_pic_num_inc);
1521             priv->RefPicList0_count += i;
1522         }
1523     }
1524     else {
1525         /* 8.2.4.2.2 - P and SP slices in fields */
1526         GstVaapiPictureH264 *short_ref[32];
1527         guint short_ref_count = 0;
1528         GstVaapiPictureH264 *long_ref[32];
1529         guint long_ref_count = 0;
1530
1531         if (priv->short_ref_count > 0) {
1532             for (i = 0; i < priv->short_ref_count; i++)
1533                 short_ref[i] = priv->short_ref[i];
1534             SORT_REF_LIST(short_ref, i, frame_num_wrap_dec);
1535             short_ref_count = i;
1536         }
1537
1538         if (priv->long_ref_count > 0) {
1539             for (i = 0; i < priv->long_ref_count; i++)
1540                 long_ref[i] = priv->long_ref[i];
1541             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1542             long_ref_count = i;
1543         }
1544
1545         init_picture_refs_fields(
1546             picture,
1547             priv->RefPicList0, &priv->RefPicList0_count,
1548             short_ref,          short_ref_count,
1549             long_ref,           long_ref_count
1550         );
1551     }
1552 }
1553
1554 static void
1555 init_picture_refs_b_slice(
1556     GstVaapiDecoderH264 *decoder,
1557     GstVaapiPictureH264 *picture,
1558     GstH264SliceHdr     *slice_hdr
1559 )
1560 {
1561     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1562     GstVaapiPictureH264 **ref_list;
1563     guint i, n;
1564
1565     GST_DEBUG("decode reference picture list for B slices");
1566
1567     if (GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1568         /* 8.2.4.2.3 - B slices in frames */
1569
1570         /* RefPicList0 */
1571         if (priv->short_ref_count > 0) {
1572             // 1. Short-term references
1573             ref_list = priv->RefPicList0;
1574             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1575                 if (priv->short_ref[i]->base.poc < picture->base.poc)
1576                     ref_list[n++] = priv->short_ref[i];
1577             }
1578             SORT_REF_LIST(ref_list, n, poc_dec);
1579             priv->RefPicList0_count += n;
1580
1581             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1582             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1583                 if (priv->short_ref[i]->base.poc >= picture->base.poc)
1584                     ref_list[n++] = priv->short_ref[i];
1585             }
1586             SORT_REF_LIST(ref_list, n, poc_inc);
1587             priv->RefPicList0_count += n;
1588         }
1589
1590         if (priv->long_ref_count > 0) {
1591             // 2. Long-term references
1592             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1593             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1594                 ref_list[n++] = priv->long_ref[i];
1595             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1596             priv->RefPicList0_count += n;
1597         }
1598
1599         /* RefPicList1 */
1600         if (priv->short_ref_count > 0) {
1601             // 1. Short-term references
1602             ref_list = priv->RefPicList1;
1603             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1604                 if (priv->short_ref[i]->base.poc > picture->base.poc)
1605                     ref_list[n++] = priv->short_ref[i];
1606             }
1607             SORT_REF_LIST(ref_list, n, poc_inc);
1608             priv->RefPicList1_count += n;
1609
1610             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1611             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1612                 if (priv->short_ref[i]->base.poc <= picture->base.poc)
1613                     ref_list[n++] = priv->short_ref[i];
1614             }
1615             SORT_REF_LIST(ref_list, n, poc_dec);
1616             priv->RefPicList1_count += n;
1617         }
1618
1619         if (priv->long_ref_count > 0) {
1620             // 2. Long-term references
1621             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1622             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1623                 ref_list[n++] = priv->long_ref[i];
1624             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1625             priv->RefPicList1_count += n;
1626         }
1627     }
1628     else {
1629         /* 8.2.4.2.4 - B slices in fields */
1630         GstVaapiPictureH264 *short_ref0[32];
1631         guint short_ref0_count = 0;
1632         GstVaapiPictureH264 *short_ref1[32];
1633         guint short_ref1_count = 0;
1634         GstVaapiPictureH264 *long_ref[32];
1635         guint long_ref_count = 0;
1636
1637         /* refFrameList0ShortTerm */
1638         if (priv->short_ref_count > 0) {
1639             ref_list = short_ref0;
1640             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1641                 if (priv->short_ref[i]->base.poc <= picture->base.poc)
1642                     ref_list[n++] = priv->short_ref[i];
1643             }
1644             SORT_REF_LIST(ref_list, n, poc_dec);
1645             short_ref0_count += n;
1646
1647             ref_list = &short_ref0[short_ref0_count];
1648             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1649                 if (priv->short_ref[i]->base.poc > picture->base.poc)
1650                     ref_list[n++] = priv->short_ref[i];
1651             }
1652             SORT_REF_LIST(ref_list, n, poc_inc);
1653             short_ref0_count += n;
1654         }
1655
1656         /* refFrameList1ShortTerm */
1657         if (priv->short_ref_count > 0) {
1658             ref_list = short_ref1;
1659             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1660                 if (priv->short_ref[i]->base.poc > picture->base.poc)
1661                     ref_list[n++] = priv->short_ref[i];
1662             }
1663             SORT_REF_LIST(ref_list, n, poc_inc);
1664             short_ref1_count += n;
1665
1666             ref_list = &short_ref1[short_ref1_count];
1667             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1668                 if (priv->short_ref[i]->base.poc <= picture->base.poc)
1669                     ref_list[n++] = priv->short_ref[i];
1670             }
1671             SORT_REF_LIST(ref_list, n, poc_dec);
1672             short_ref1_count += n;
1673         }
1674
1675         /* refFrameListLongTerm */
1676         if (priv->long_ref_count > 0) {
1677             for (i = 0; i < priv->long_ref_count; i++)
1678                 long_ref[i] = priv->long_ref[i];
1679             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1680             long_ref_count = i;
1681         }
1682
1683         init_picture_refs_fields(
1684             picture,
1685             priv->RefPicList0, &priv->RefPicList0_count,
1686             short_ref0,         short_ref0_count,
1687             long_ref,           long_ref_count
1688         );
1689
1690         init_picture_refs_fields(
1691             picture,
1692             priv->RefPicList1, &priv->RefPicList1_count,
1693             short_ref1,         short_ref1_count,
1694             long_ref,           long_ref_count
1695         );
1696    }
1697
1698     /* Check whether RefPicList1 is identical to RefPicList0, then
1699        swap if necessary */
1700     if (priv->RefPicList1_count > 1 &&
1701         priv->RefPicList1_count == priv->RefPicList0_count &&
1702         memcmp(priv->RefPicList0, priv->RefPicList1,
1703                priv->RefPicList0_count * sizeof(priv->RefPicList0[0])) == 0) {
1704         GstVaapiPictureH264 * const tmp = priv->RefPicList1[0];
1705         priv->RefPicList1[0] = priv->RefPicList1[1];
1706         priv->RefPicList1[1] = tmp;
1707     }
1708 }
1709
1710 #undef SORT_REF_LIST
1711
1712 static gint
1713 find_short_term_reference(GstVaapiDecoderH264 *decoder, gint32 pic_num)
1714 {
1715     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1716     guint i;
1717
1718     for (i = 0; i < priv->short_ref_count; i++) {
1719         if (priv->short_ref[i]->pic_num == pic_num)
1720             return i;
1721     }
1722     GST_ERROR("found no short-term reference picture with PicNum = %d",
1723               pic_num);
1724     return -1;
1725 }
1726
1727 static gint
1728 find_long_term_reference(GstVaapiDecoderH264 *decoder, gint32 long_term_pic_num)
1729 {
1730     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1731     guint i;
1732
1733     for (i = 0; i < priv->long_ref_count; i++) {
1734         if (priv->long_ref[i]->long_term_pic_num == long_term_pic_num)
1735             return i;
1736     }
1737     GST_ERROR("found no long-term reference picture with LongTermPicNum = %d",
1738               long_term_pic_num);
1739     return -1;
1740 }
1741
1742 static void
1743 exec_picture_refs_modification_1(
1744     GstVaapiDecoderH264           *decoder,
1745     GstVaapiPictureH264           *picture,
1746     GstH264SliceHdr               *slice_hdr,
1747     guint                          list
1748 )
1749 {
1750     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1751     GstH264PPS * const pps = slice_hdr->pps;
1752     GstH264SPS * const sps = pps->sequence;
1753     GstH264RefPicListModification *ref_pic_list_modification;
1754     guint num_ref_pic_list_modifications;
1755     GstVaapiPictureH264 **ref_list;
1756     guint *ref_list_count_ptr, ref_list_count, ref_list_idx = 0;
1757     guint i, j, n, num_refs;
1758     gint found_ref_idx;
1759     gint32 MaxPicNum, CurrPicNum, picNumPred;
1760
1761     GST_DEBUG("modification process of reference picture list %u", list);
1762
1763     if (list == 0) {
1764         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l0;
1765         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l0;
1766         ref_list                       = priv->RefPicList0;
1767         ref_list_count_ptr             = &priv->RefPicList0_count;
1768         num_refs                       = slice_hdr->num_ref_idx_l0_active_minus1 + 1;
1769     }
1770     else {
1771         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l1;
1772         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l1;
1773         ref_list                       = priv->RefPicList1;
1774         ref_list_count_ptr             = &priv->RefPicList1_count;
1775         num_refs                       = slice_hdr->num_ref_idx_l1_active_minus1 + 1;
1776     }
1777     ref_list_count = *ref_list_count_ptr;
1778
1779     if (!GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1780         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 5); // 2 * MaxFrameNum
1781         CurrPicNum = 2 * slice_hdr->frame_num + 1;              // 2 * frame_num + 1
1782     }
1783     else {
1784         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 4); // MaxFrameNum
1785         CurrPicNum = slice_hdr->frame_num;                      // frame_num
1786     }
1787
1788     picNumPred = CurrPicNum;
1789
1790     for (i = 0; i < num_ref_pic_list_modifications; i++) {
1791         GstH264RefPicListModification * const l = &ref_pic_list_modification[i];
1792         if (l->modification_of_pic_nums_idc == 3)
1793             break;
1794
1795         /* 8.2.4.3.1 - Short-term reference pictures */
1796         if (l->modification_of_pic_nums_idc == 0 || l->modification_of_pic_nums_idc == 1) {
1797             gint32 abs_diff_pic_num = l->value.abs_diff_pic_num_minus1 + 1;
1798             gint32 picNum, picNumNoWrap;
1799
1800             // (8-34)
1801             if (l->modification_of_pic_nums_idc == 0) {
1802                 picNumNoWrap = picNumPred - abs_diff_pic_num;
1803                 if (picNumNoWrap < 0)
1804                     picNumNoWrap += MaxPicNum;
1805             }
1806
1807             // (8-35)
1808             else {
1809                 picNumNoWrap = picNumPred + abs_diff_pic_num;
1810                 if (picNumNoWrap >= MaxPicNum)
1811                     picNumNoWrap -= MaxPicNum;
1812             }
1813             picNumPred = picNumNoWrap;
1814
1815             // (8-36)
1816             picNum = picNumNoWrap;
1817             if (picNum > CurrPicNum)
1818                 picNum -= MaxPicNum;
1819
1820             // (8-37)
1821             for (j = num_refs; j > ref_list_idx; j--)
1822                 ref_list[j] = ref_list[j - 1];
1823             found_ref_idx = find_short_term_reference(decoder, picNum);
1824             ref_list[ref_list_idx++] =
1825                 found_ref_idx >= 0 ? priv->short_ref[found_ref_idx] : NULL;
1826             n = ref_list_idx;
1827             for (j = ref_list_idx; j <= num_refs; j++) {
1828                 gint32 PicNumF;
1829                 if (!ref_list[j])
1830                     continue;
1831                 PicNumF =
1832                     GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(ref_list[j]) ?
1833                     ref_list[j]->pic_num : MaxPicNum;
1834                 if (PicNumF != picNum)
1835                     ref_list[n++] = ref_list[j];
1836             }
1837         }
1838
1839         /* 8.2.4.3.2 - Long-term reference pictures */
1840         else {
1841
1842             for (j = num_refs; j > ref_list_idx; j--)
1843                 ref_list[j] = ref_list[j - 1];
1844             found_ref_idx =
1845                 find_long_term_reference(decoder, l->value.long_term_pic_num);
1846             ref_list[ref_list_idx++] =
1847                 found_ref_idx >= 0 ? priv->long_ref[found_ref_idx] : NULL;
1848             n = ref_list_idx;
1849             for (j = ref_list_idx; j <= num_refs; j++) {
1850                 gint32 LongTermPicNumF;
1851                 if (!ref_list[j])
1852                     continue;
1853                 LongTermPicNumF =
1854                     GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(ref_list[j]) ?
1855                     ref_list[j]->long_term_pic_num : INT_MAX;
1856                 if (LongTermPicNumF != l->value.long_term_pic_num)
1857                     ref_list[n++] = ref_list[j];
1858             }
1859         }
1860     }
1861
1862 #if DEBUG
1863     for (i = 0; i < num_refs; i++)
1864         if (!ref_list[i])
1865             GST_ERROR("list %u entry %u is empty", list, i);
1866 #endif
1867     *ref_list_count_ptr = num_refs;
1868 }
1869
1870 /* 8.2.4.3 - Modification process for reference picture lists */
1871 static void
1872 exec_picture_refs_modification(
1873     GstVaapiDecoderH264 *decoder,
1874     GstVaapiPictureH264 *picture,
1875     GstH264SliceHdr     *slice_hdr
1876 )
1877 {
1878     GST_DEBUG("execute ref_pic_list_modification()");
1879
1880     /* RefPicList0 */
1881     if (!GST_H264_IS_I_SLICE(slice_hdr) && !GST_H264_IS_SI_SLICE(slice_hdr) &&
1882         slice_hdr->ref_pic_list_modification_flag_l0)
1883         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 0);
1884
1885     /* RefPicList1 */
1886     if (GST_H264_IS_B_SLICE(slice_hdr) &&
1887         slice_hdr->ref_pic_list_modification_flag_l1)
1888         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 1);
1889 }
1890
1891 static void
1892 init_picture_ref_lists(GstVaapiDecoderH264 *decoder)
1893 {
1894     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1895     guint i, j, short_ref_count, long_ref_count;
1896
1897     short_ref_count = 0;
1898     long_ref_count  = 0;
1899     if (GST_VAAPI_PICTURE_IS_FRAME(priv->current_picture)) {
1900         for (i = 0; i < priv->dpb_count; i++) {
1901             GstVaapiFrameStore * const fs = priv->dpb[i];
1902             GstVaapiPictureH264 *picture;
1903             if (!gst_vaapi_frame_store_has_frame(fs))
1904                 continue;
1905             picture = fs->buffers[0];
1906             if (GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture))
1907                 priv->short_ref[short_ref_count++] = picture;
1908             else if (GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture))
1909                 priv->long_ref[long_ref_count++] = picture;
1910             picture->structure = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
1911             picture->other_field = fs->buffers[1];
1912         }
1913     }
1914     else {
1915         for (i = 0; i < priv->dpb_count; i++) {
1916             GstVaapiFrameStore * const fs = priv->dpb[i];
1917             for (j = 0; j < fs->num_buffers; j++) {
1918                 GstVaapiPictureH264 * const picture = fs->buffers[j];
1919                 if (GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture))
1920                     priv->short_ref[short_ref_count++] = picture;
1921                 else if (GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture))
1922                     priv->long_ref[long_ref_count++] = picture;
1923                 picture->structure = picture->base.structure;
1924                 picture->other_field = fs->buffers[j ^ 1];
1925             }
1926         }
1927     }
1928
1929     for (i = short_ref_count; i < priv->short_ref_count; i++)
1930         priv->short_ref[i] = NULL;
1931     priv->short_ref_count = short_ref_count;
1932
1933     for (i = long_ref_count; i < priv->long_ref_count; i++)
1934         priv->long_ref[i] = NULL;
1935     priv->long_ref_count = long_ref_count;
1936 }
1937
1938 static void
1939 init_picture_refs(
1940     GstVaapiDecoderH264 *decoder,
1941     GstVaapiPictureH264 *picture,
1942     GstH264SliceHdr     *slice_hdr
1943 )
1944 {
1945     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1946     GstVaapiPicture * const base_picture = &picture->base;
1947     guint i, num_refs;
1948
1949     init_picture_ref_lists(decoder);
1950     init_picture_refs_pic_num(decoder, picture, slice_hdr);
1951
1952     priv->RefPicList0_count = 0;
1953     priv->RefPicList1_count = 0;
1954
1955     switch (base_picture->type) {
1956     case GST_VAAPI_PICTURE_TYPE_P:
1957     case GST_VAAPI_PICTURE_TYPE_SP:
1958         init_picture_refs_p_slice(decoder, picture, slice_hdr);
1959         break;
1960     case GST_VAAPI_PICTURE_TYPE_B:
1961         init_picture_refs_b_slice(decoder, picture, slice_hdr);
1962         break;
1963     default:
1964         break;
1965     }
1966
1967     exec_picture_refs_modification(decoder, picture, slice_hdr);
1968
1969     switch (base_picture->type) {
1970     case GST_VAAPI_PICTURE_TYPE_B:
1971         num_refs = 1 + slice_hdr->num_ref_idx_l1_active_minus1;
1972         for (i = priv->RefPicList1_count; i < num_refs; i++)
1973             priv->RefPicList1[i] = NULL;
1974         priv->RefPicList1_count = num_refs;
1975
1976         // fall-through
1977     case GST_VAAPI_PICTURE_TYPE_P:
1978     case GST_VAAPI_PICTURE_TYPE_SP:
1979         num_refs = 1 + slice_hdr->num_ref_idx_l0_active_minus1;
1980         for (i = priv->RefPicList0_count; i < num_refs; i++)
1981             priv->RefPicList0[i] = NULL;
1982         priv->RefPicList0_count = num_refs;
1983         break;
1984     default:
1985         break;
1986     }
1987 }
1988
1989 static gboolean
1990 init_picture(
1991     GstVaapiDecoderH264 *decoder,
1992     GstVaapiPictureH264 *picture, GstVaapiParserInfoH264 *pi)
1993 {
1994     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1995     GstVaapiPicture * const base_picture = &picture->base;
1996     GstH264SliceHdr * const slice_hdr = &pi->data.slice_hdr;
1997
1998     priv->prev_frame_num        = priv->frame_num;
1999     priv->frame_num             = slice_hdr->frame_num;
2000     picture->frame_num          = priv->frame_num;
2001     picture->frame_num_wrap     = priv->frame_num;
2002     picture->output_flag        = TRUE; /* XXX: conformant to Annex A only */
2003     base_picture->pts           = GST_VAAPI_DECODER_CODEC_FRAME(decoder)->pts;
2004
2005     /* Reset decoder state for IDR pictures */
2006     if (pi->nalu.type == GST_H264_NAL_SLICE_IDR) {
2007         GST_DEBUG("<IDR>");
2008         GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_IDR);
2009         dpb_flush(decoder);
2010     }
2011
2012     /* Initialize slice type */
2013     switch (slice_hdr->type % 5) {
2014     case GST_H264_P_SLICE:
2015         base_picture->type = GST_VAAPI_PICTURE_TYPE_P;
2016         break;
2017     case GST_H264_B_SLICE:
2018         base_picture->type = GST_VAAPI_PICTURE_TYPE_B;
2019         break;
2020     case GST_H264_I_SLICE:
2021         base_picture->type = GST_VAAPI_PICTURE_TYPE_I;
2022         break;
2023     case GST_H264_SP_SLICE:
2024         base_picture->type = GST_VAAPI_PICTURE_TYPE_SP;
2025         break;
2026     case GST_H264_SI_SLICE:
2027         base_picture->type = GST_VAAPI_PICTURE_TYPE_SI;
2028         break;
2029     }
2030
2031     /* Initialize picture structure */
2032     if (!slice_hdr->field_pic_flag)
2033         base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
2034     else {
2035         GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_INTERLACED);
2036         if (!slice_hdr->bottom_field_flag)
2037             base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD;
2038         else
2039             base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD;
2040     }
2041     picture->structure = base_picture->structure;
2042
2043     /* Initialize reference flags */
2044     if (pi->nalu.ref_idc) {
2045         GstH264DecRefPicMarking * const dec_ref_pic_marking =
2046             &slice_hdr->dec_ref_pic_marking;
2047
2048         if (GST_VAAPI_PICTURE_IS_IDR(picture) &&
2049             dec_ref_pic_marking->long_term_reference_flag)
2050             GST_VAAPI_PICTURE_FLAG_SET(picture,
2051                 GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE);
2052         else
2053             GST_VAAPI_PICTURE_FLAG_SET(picture,
2054                 GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE);
2055     }
2056
2057     init_picture_poc(decoder, picture, slice_hdr);
2058     init_picture_refs(decoder, picture, slice_hdr);
2059     return TRUE;
2060 }
2061
2062 /* 8.2.5.3 - Sliding window decoded reference picture marking process */
2063 static gboolean
2064 exec_ref_pic_marking_sliding_window(GstVaapiDecoderH264 *decoder)
2065 {
2066     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2067     GstH264PPS * const pps = priv->current_picture->pps;
2068     GstH264SPS * const sps = pps->sequence;
2069     GstVaapiPictureH264 *ref_picture;
2070     guint i, m, max_num_ref_frames;
2071
2072     GST_DEBUG("reference picture marking process (sliding window)");
2073
2074     if (!GST_VAAPI_PICTURE_IS_FIRST_FIELD(priv->current_picture))
2075         return TRUE;
2076
2077     max_num_ref_frames = sps->num_ref_frames;
2078     if (max_num_ref_frames == 0)
2079         max_num_ref_frames = 1;
2080     if (!GST_VAAPI_PICTURE_IS_FRAME(priv->current_picture))
2081         max_num_ref_frames <<= 1;
2082
2083     if (priv->short_ref_count + priv->long_ref_count < max_num_ref_frames)
2084         return TRUE;
2085     if (priv->short_ref_count < 1)
2086         return FALSE;
2087
2088     for (m = 0, i = 1; i < priv->short_ref_count; i++) {
2089         GstVaapiPictureH264 * const picture = priv->short_ref[i];
2090         if (picture->frame_num_wrap < priv->short_ref[m]->frame_num_wrap)
2091             m = i;
2092     }
2093
2094     ref_picture = priv->short_ref[m];
2095     gst_vaapi_picture_h264_set_reference(ref_picture, 0, TRUE);
2096     ARRAY_REMOVE_INDEX(priv->short_ref, m);
2097
2098     /* Both fields need to be marked as "unused for reference", so
2099        remove the other field from the short_ref[] list as well */
2100     if (!GST_VAAPI_PICTURE_IS_FRAME(priv->current_picture) && ref_picture->other_field) {
2101         for (i = 0; i < priv->short_ref_count; i++) {
2102             if (priv->short_ref[i] == ref_picture->other_field) {
2103                 ARRAY_REMOVE_INDEX(priv->short_ref, i);
2104                 break;
2105             }
2106         }
2107     }
2108     return TRUE;
2109 }
2110
2111 static inline gint32
2112 get_picNumX(GstVaapiPictureH264 *picture, GstH264RefPicMarking *ref_pic_marking)
2113 {
2114     gint32 pic_num;
2115
2116     if (GST_VAAPI_PICTURE_IS_FRAME(picture))
2117         pic_num = picture->frame_num_wrap;
2118     else
2119         pic_num = 2 * picture->frame_num_wrap + 1;
2120     pic_num -= ref_pic_marking->difference_of_pic_nums_minus1 + 1;
2121     return pic_num;
2122 }
2123
2124 /* 8.2.5.4.1. Mark short-term reference picture as "unused for reference" */
2125 static void
2126 exec_ref_pic_marking_adaptive_mmco_1(
2127     GstVaapiDecoderH264  *decoder,
2128     GstVaapiPictureH264  *picture,
2129     GstH264RefPicMarking *ref_pic_marking
2130 )
2131 {
2132     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2133     gint32 i, picNumX;
2134
2135     picNumX = get_picNumX(picture, ref_pic_marking);
2136     i = find_short_term_reference(decoder, picNumX);
2137     if (i < 0)
2138         return;
2139
2140     gst_vaapi_picture_h264_set_reference(priv->short_ref[i], 0,
2141         GST_VAAPI_PICTURE_IS_FRAME(picture));
2142     ARRAY_REMOVE_INDEX(priv->short_ref, i);
2143 }
2144
2145 /* 8.2.5.4.2. Mark long-term reference picture as "unused for reference" */
2146 static void
2147 exec_ref_pic_marking_adaptive_mmco_2(
2148     GstVaapiDecoderH264  *decoder,
2149     GstVaapiPictureH264  *picture,
2150     GstH264RefPicMarking *ref_pic_marking
2151 )
2152 {
2153     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2154     gint32 i;
2155
2156     i = find_long_term_reference(decoder, ref_pic_marking->long_term_pic_num);
2157     if (i < 0)
2158         return;
2159
2160     gst_vaapi_picture_h264_set_reference(priv->long_ref[i], 0,
2161         GST_VAAPI_PICTURE_IS_FRAME(picture));
2162     ARRAY_REMOVE_INDEX(priv->long_ref, i);
2163 }
2164
2165 /* 8.2.5.4.3. Assign LongTermFrameIdx to a short-term reference picture */
2166 static void
2167 exec_ref_pic_marking_adaptive_mmco_3(
2168     GstVaapiDecoderH264  *decoder,
2169     GstVaapiPictureH264  *picture,
2170     GstH264RefPicMarking *ref_pic_marking
2171 )
2172 {
2173     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2174     GstVaapiPictureH264 *ref_picture;
2175     gint32 i, picNumX;
2176
2177     for (i = 0; i < priv->long_ref_count; i++) {
2178         if (priv->long_ref[i]->long_term_frame_idx == ref_pic_marking->long_term_frame_idx)
2179             break;
2180     }
2181     if (i != priv->long_ref_count) {
2182         gst_vaapi_picture_h264_set_reference(priv->long_ref[i], 0, TRUE);
2183         ARRAY_REMOVE_INDEX(priv->long_ref, i);
2184     }
2185
2186     picNumX = get_picNumX(picture, ref_pic_marking);
2187     i = find_short_term_reference(decoder, picNumX);
2188     if (i < 0)
2189         return;
2190
2191     ref_picture = priv->short_ref[i];
2192     ARRAY_REMOVE_INDEX(priv->short_ref, i);
2193     priv->long_ref[priv->long_ref_count++] = ref_picture;
2194
2195     ref_picture->long_term_frame_idx = ref_pic_marking->long_term_frame_idx;
2196     gst_vaapi_picture_h264_set_reference(ref_picture,
2197         GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE,
2198         GST_VAAPI_PICTURE_IS_FRAME(picture));
2199 }
2200
2201 /* 8.2.5.4.4. Mark pictures with LongTermFramIdx > max_long_term_frame_idx
2202  * as "unused for reference" */
2203 static void
2204 exec_ref_pic_marking_adaptive_mmco_4(
2205     GstVaapiDecoderH264  *decoder,
2206     GstVaapiPictureH264  *picture,
2207     GstH264RefPicMarking *ref_pic_marking
2208 )
2209 {
2210     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2211     gint32 i, long_term_frame_idx;
2212
2213     long_term_frame_idx = ref_pic_marking->max_long_term_frame_idx_plus1 - 1;
2214
2215     for (i = 0; i < priv->long_ref_count; i++) {
2216         if (priv->long_ref[i]->long_term_frame_idx <= long_term_frame_idx)
2217             continue;
2218         gst_vaapi_picture_h264_set_reference(priv->long_ref[i], 0, FALSE);
2219         ARRAY_REMOVE_INDEX(priv->long_ref, i);
2220         i--;
2221     }
2222 }
2223
2224 /* 8.2.5.4.5. Mark all reference pictures as "unused for reference" */
2225 static void
2226 exec_ref_pic_marking_adaptive_mmco_5(
2227     GstVaapiDecoderH264  *decoder,
2228     GstVaapiPictureH264  *picture,
2229     GstH264RefPicMarking *ref_pic_marking
2230 )
2231 {
2232     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2233
2234     dpb_flush(decoder);
2235
2236     priv->prev_pic_has_mmco5 = TRUE;
2237
2238     /* The picture shall be inferred to have had frame_num equal to 0 (7.4.3) */
2239     priv->frame_num = 0;
2240     priv->frame_num_offset = 0;
2241     picture->frame_num = 0;
2242
2243     /* Update TopFieldOrderCnt and BottomFieldOrderCnt (8.2.1) */
2244     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
2245         picture->field_poc[TOP_FIELD] -= picture->base.poc;
2246     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
2247         picture->field_poc[BOTTOM_FIELD] -= picture->base.poc;
2248     picture->base.poc = 0;
2249 }
2250
2251 /* 8.2.5.4.6. Assign a long-term frame index to the current picture */
2252 static void
2253 exec_ref_pic_marking_adaptive_mmco_6(
2254     GstVaapiDecoderH264  *decoder,
2255     GstVaapiPictureH264  *picture,
2256     GstH264RefPicMarking *ref_pic_marking
2257 )
2258 {
2259     picture->long_term_frame_idx = ref_pic_marking->long_term_frame_idx;
2260     gst_vaapi_picture_h264_set_reference(picture,
2261         GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE, FALSE);
2262 }
2263
2264 /* 8.2.5.4. Adaptive memory control decoded reference picture marking process */
2265 static gboolean
2266 exec_ref_pic_marking_adaptive(
2267     GstVaapiDecoderH264     *decoder,
2268     GstVaapiPictureH264     *picture,
2269     GstH264DecRefPicMarking *dec_ref_pic_marking
2270 )
2271 {
2272     guint i;
2273
2274     GST_DEBUG("reference picture marking process (adaptive memory control)");
2275
2276     typedef void (*exec_ref_pic_marking_adaptive_mmco_func)(
2277         GstVaapiDecoderH264  *decoder,
2278         GstVaapiPictureH264  *picture,
2279         GstH264RefPicMarking *ref_pic_marking
2280     );
2281
2282     static const exec_ref_pic_marking_adaptive_mmco_func mmco_funcs[] = {
2283         NULL,
2284         exec_ref_pic_marking_adaptive_mmco_1,
2285         exec_ref_pic_marking_adaptive_mmco_2,
2286         exec_ref_pic_marking_adaptive_mmco_3,
2287         exec_ref_pic_marking_adaptive_mmco_4,
2288         exec_ref_pic_marking_adaptive_mmco_5,
2289         exec_ref_pic_marking_adaptive_mmco_6,
2290     };
2291
2292     for (i = 0; i < dec_ref_pic_marking->n_ref_pic_marking; i++) {
2293         GstH264RefPicMarking * const ref_pic_marking =
2294             &dec_ref_pic_marking->ref_pic_marking[i];
2295
2296         const guint mmco = ref_pic_marking->memory_management_control_operation;
2297         if (mmco < G_N_ELEMENTS(mmco_funcs) && mmco_funcs[mmco])
2298             mmco_funcs[mmco](decoder, picture, ref_pic_marking);
2299         else {
2300             GST_ERROR("unhandled MMCO %u", mmco);
2301             return FALSE;
2302         }
2303     }
2304     return TRUE;
2305 }
2306
2307 /* 8.2.5 - Execute reference picture marking process */
2308 static gboolean
2309 exec_ref_pic_marking(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
2310 {
2311     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2312
2313     priv->prev_pic_has_mmco5 = FALSE;
2314     priv->prev_pic_structure = picture->structure;
2315
2316     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
2317         return TRUE;
2318
2319     if (!GST_VAAPI_PICTURE_IS_IDR(picture)) {
2320         GstH264DecRefPicMarking * const dec_ref_pic_marking =
2321             &picture->last_slice_hdr->dec_ref_pic_marking;
2322         if (dec_ref_pic_marking->adaptive_ref_pic_marking_mode_flag) {
2323             if (!exec_ref_pic_marking_adaptive(decoder, picture, dec_ref_pic_marking))
2324                 return FALSE;
2325         }
2326         else {
2327             if (!exec_ref_pic_marking_sliding_window(decoder))
2328                 return FALSE;
2329         }
2330     }
2331     return TRUE;
2332 }
2333
2334 static void
2335 vaapi_init_picture(VAPictureH264 *pic)
2336 {
2337     pic->picture_id           = VA_INVALID_ID;
2338     pic->frame_idx            = 0;
2339     pic->flags                = VA_PICTURE_H264_INVALID;
2340     pic->TopFieldOrderCnt     = 0;
2341     pic->BottomFieldOrderCnt  = 0;
2342 }
2343
2344 static void
2345 vaapi_fill_picture(VAPictureH264 *pic, GstVaapiPictureH264 *picture,
2346     guint picture_structure)
2347 {
2348     if (!picture_structure)
2349         picture_structure = picture->structure;
2350
2351     pic->picture_id = picture->base.surface_id;
2352     pic->flags = 0;
2353
2354     if (GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture)) {
2355         pic->flags |= VA_PICTURE_H264_LONG_TERM_REFERENCE;
2356         pic->frame_idx = picture->long_term_frame_idx;
2357     }
2358     else {
2359         if (GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture))
2360             pic->flags |= VA_PICTURE_H264_SHORT_TERM_REFERENCE;
2361         pic->frame_idx = picture->frame_num;
2362     }
2363
2364     switch (picture_structure) {
2365     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
2366         pic->TopFieldOrderCnt = picture->field_poc[TOP_FIELD];
2367         pic->BottomFieldOrderCnt = picture->field_poc[BOTTOM_FIELD];
2368         break;
2369     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
2370         pic->flags |= VA_PICTURE_H264_TOP_FIELD;
2371         pic->TopFieldOrderCnt = picture->field_poc[TOP_FIELD];
2372         pic->BottomFieldOrderCnt = 0;
2373         break;
2374     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
2375         pic->flags |= VA_PICTURE_H264_BOTTOM_FIELD;
2376         pic->BottomFieldOrderCnt = picture->field_poc[BOTTOM_FIELD];
2377         pic->TopFieldOrderCnt = 0;
2378         break;
2379     }
2380 }
2381
2382 static gboolean
2383 fill_picture(GstVaapiDecoderH264 *decoder,
2384     GstVaapiPictureH264 *picture, GstVaapiParserInfoH264 *pi)
2385 {
2386     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2387     GstVaapiPicture * const base_picture = &picture->base;
2388     GstH264SliceHdr * const slice_hdr = &pi->data.slice_hdr;
2389     GstH264PPS * const pps = picture->pps;
2390     GstH264SPS * const sps = pps->sequence;
2391     VAPictureParameterBufferH264 * const pic_param = base_picture->param;
2392     guint i, n;
2393
2394     /* Fill in VAPictureParameterBufferH264 */
2395     vaapi_fill_picture(&pic_param->CurrPic, picture, 0);
2396
2397     for (i = 0, n = 0; i < priv->dpb_count; i++) {
2398         GstVaapiFrameStore * const fs = priv->dpb[i];
2399         if (gst_vaapi_frame_store_has_reference(fs))
2400             vaapi_fill_picture(&pic_param->ReferenceFrames[n++],
2401                 fs->buffers[0], fs->structure);
2402     }
2403     for (; n < G_N_ELEMENTS(pic_param->ReferenceFrames); n++)
2404         vaapi_init_picture(&pic_param->ReferenceFrames[n]);
2405
2406 #define COPY_FIELD(s, f) \
2407     pic_param->f = (s)->f
2408
2409 #define COPY_BFM(a, s, f) \
2410     pic_param->a.bits.f = (s)->f
2411
2412     pic_param->picture_width_in_mbs_minus1  = priv->mb_width - 1;
2413     pic_param->picture_height_in_mbs_minus1 = priv->mb_height - 1;
2414     pic_param->frame_num                    = priv->frame_num;
2415
2416     COPY_FIELD(sps, bit_depth_luma_minus8);
2417     COPY_FIELD(sps, bit_depth_chroma_minus8);
2418     COPY_FIELD(sps, num_ref_frames);
2419     COPY_FIELD(pps, num_slice_groups_minus1);
2420     COPY_FIELD(pps, slice_group_map_type);
2421     COPY_FIELD(pps, slice_group_change_rate_minus1);
2422     COPY_FIELD(pps, pic_init_qp_minus26);
2423     COPY_FIELD(pps, pic_init_qs_minus26);
2424     COPY_FIELD(pps, chroma_qp_index_offset);
2425     COPY_FIELD(pps, second_chroma_qp_index_offset);
2426
2427     pic_param->seq_fields.value                                         = 0; /* reset all bits */
2428     pic_param->seq_fields.bits.residual_colour_transform_flag           = sps->separate_colour_plane_flag;
2429     pic_param->seq_fields.bits.MinLumaBiPredSize8x8                     = sps->level_idc >= 31; /* A.3.3.2 */
2430
2431     COPY_BFM(seq_fields, sps, chroma_format_idc);
2432     COPY_BFM(seq_fields, sps, gaps_in_frame_num_value_allowed_flag);
2433     COPY_BFM(seq_fields, sps, frame_mbs_only_flag);
2434     COPY_BFM(seq_fields, sps, mb_adaptive_frame_field_flag);
2435     COPY_BFM(seq_fields, sps, direct_8x8_inference_flag);
2436     COPY_BFM(seq_fields, sps, log2_max_frame_num_minus4);
2437     COPY_BFM(seq_fields, sps, pic_order_cnt_type);
2438     COPY_BFM(seq_fields, sps, log2_max_pic_order_cnt_lsb_minus4);
2439     COPY_BFM(seq_fields, sps, delta_pic_order_always_zero_flag);
2440
2441     pic_param->pic_fields.value                                         = 0; /* reset all bits */
2442     pic_param->pic_fields.bits.field_pic_flag                           = slice_hdr->field_pic_flag;
2443     pic_param->pic_fields.bits.reference_pic_flag                       = GST_VAAPI_PICTURE_IS_REFERENCE(picture);
2444
2445     COPY_BFM(pic_fields, pps, entropy_coding_mode_flag);
2446     COPY_BFM(pic_fields, pps, weighted_pred_flag);
2447     COPY_BFM(pic_fields, pps, weighted_bipred_idc);
2448     COPY_BFM(pic_fields, pps, transform_8x8_mode_flag);
2449     COPY_BFM(pic_fields, pps, constrained_intra_pred_flag);
2450     COPY_BFM(pic_fields, pps, pic_order_present_flag);
2451     COPY_BFM(pic_fields, pps, deblocking_filter_control_present_flag);
2452     COPY_BFM(pic_fields, pps, redundant_pic_cnt_present_flag);
2453     return TRUE;
2454 }
2455
2456 /* Detection of the first VCL NAL unit of a primary coded picture (7.4.1.2.4) */
2457 static gboolean
2458 is_new_picture(GstVaapiParserInfoH264 *pi, GstVaapiParserInfoH264 *prev_pi)
2459 {
2460     GstH264SliceHdr * const slice_hdr = &pi->data.slice_hdr;
2461     GstH264PPS * const pps = slice_hdr->pps;
2462     GstH264SPS * const sps = pps->sequence;
2463     GstH264SliceHdr *prev_slice_hdr;
2464
2465     if (!prev_pi)
2466         return TRUE;
2467     prev_slice_hdr = &prev_pi->data.slice_hdr;
2468
2469 #define CHECK_EXPR(expr, field_name) do {              \
2470         if (!(expr)) {                                 \
2471             GST_DEBUG(field_name " differs in value"); \
2472             return TRUE;                               \
2473         }                                              \
2474     } while (0)
2475
2476 #define CHECK_VALUE(new_slice_hdr, old_slice_hdr, field) \
2477     CHECK_EXPR(((new_slice_hdr)->field == (old_slice_hdr)->field), #field)
2478
2479     /* frame_num differs in value, regardless of inferred values to 0 */
2480     CHECK_VALUE(slice_hdr, prev_slice_hdr, frame_num);
2481
2482     /* pic_parameter_set_id differs in value */
2483     CHECK_VALUE(slice_hdr, prev_slice_hdr, pps);
2484
2485     /* field_pic_flag differs in value */
2486     CHECK_VALUE(slice_hdr, prev_slice_hdr, field_pic_flag);
2487
2488     /* bottom_field_flag is present in both and differs in value */
2489     if (slice_hdr->field_pic_flag && prev_slice_hdr->field_pic_flag)
2490         CHECK_VALUE(slice_hdr, prev_slice_hdr, bottom_field_flag);
2491
2492     /* nal_ref_idc differs in value with one of the nal_ref_idc values is 0 */
2493     CHECK_EXPR((pi->nalu.ref_idc != 0) ==
2494                (prev_pi->nalu.ref_idc != 0), "nal_ref_idc");
2495
2496     /* POC type is 0 for both and either pic_order_cnt_lsb differs in
2497        value or delta_pic_order_cnt_bottom differs in value */
2498     if (sps->pic_order_cnt_type == 0) {
2499         CHECK_VALUE(slice_hdr, prev_slice_hdr, pic_order_cnt_lsb);
2500         if (pps->pic_order_present_flag && !slice_hdr->field_pic_flag)
2501             CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt_bottom);
2502     }
2503
2504     /* POC type is 1 for both and either delta_pic_order_cnt[0]
2505        differs in value or delta_pic_order_cnt[1] differs in value */
2506     else if (sps->pic_order_cnt_type == 1) {
2507         CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt[0]);
2508         CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt[1]);
2509     }
2510
2511     /* IdrPicFlag differs in value */
2512     CHECK_VALUE(&pi->nalu, &prev_pi->nalu, idr_pic_flag);
2513
2514     /* IdrPicFlag is equal to 1 for both and idr_pic_id differs in value */
2515     if (pi->nalu.idr_pic_flag)
2516         CHECK_VALUE(slice_hdr, prev_slice_hdr, idr_pic_id);
2517
2518 #undef CHECK_EXPR
2519 #undef CHECK_VALUE
2520     return FALSE;
2521 }
2522
2523 static GstVaapiDecoderStatus
2524 decode_picture(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnit *unit)
2525 {
2526     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2527     GstVaapiParserInfoH264 * const pi = unit->parsed_info;
2528     GstH264SliceHdr * const slice_hdr = &pi->data.slice_hdr;
2529     GstH264PPS * const pps = slice_hdr->pps;
2530     GstH264SPS * const sps = pps->sequence;
2531     GstVaapiPictureH264 *picture;
2532     GstVaapiDecoderStatus status;
2533
2534     status = ensure_context(decoder, sps);
2535     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2536         return status;
2537
2538     if (priv->current_picture) {
2539         /* Re-use current picture where the first field was decoded */
2540         picture = gst_vaapi_picture_h264_new_field(priv->current_picture);
2541         if (!picture) {
2542             GST_ERROR("failed to allocate field picture");
2543             return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2544         }
2545     }
2546     else {
2547         /* Create new picture */
2548         picture = gst_vaapi_picture_h264_new(decoder);
2549         if (!picture) {
2550             GST_ERROR("failed to allocate picture");
2551             return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2552         }
2553     }
2554     gst_vaapi_picture_replace(&priv->current_picture, picture);
2555     gst_vaapi_picture_unref(picture);
2556
2557     /* Update cropping rectangle */
2558     if (sps->frame_cropping_flag) {
2559         GstVaapiRectangle crop_rect;
2560         crop_rect.x = sps->crop_rect_x;
2561         crop_rect.y = sps->crop_rect_y;
2562         crop_rect.width = sps->crop_rect_width;
2563         crop_rect.height = sps->crop_rect_height;
2564         gst_vaapi_picture_set_crop_rect(&picture->base, &crop_rect);
2565     }
2566
2567     picture->pps = pps;
2568
2569     status = ensure_quant_matrix(decoder, picture);
2570     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS) {
2571         GST_ERROR("failed to reset quantizer matrix");
2572         return status;
2573     }
2574
2575     if (!init_picture(decoder, picture, pi))
2576         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2577     if (!fill_picture(decoder, picture, pi))
2578         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2579     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2580 }
2581
2582 static inline guint
2583 get_slice_data_bit_offset(GstH264SliceHdr *slice_hdr)
2584 {
2585     guint epb_count;
2586
2587     epb_count = slice_hdr->n_emulation_prevention_bytes;
2588     return 8 /* nal_unit_type */ + slice_hdr->header_size - epb_count * 8;
2589 }
2590
2591 static gboolean
2592 fill_pred_weight_table(GstVaapiDecoderH264 *decoder,
2593     GstVaapiSlice *slice, GstH264SliceHdr *slice_hdr)
2594 {
2595     VASliceParameterBufferH264 * const slice_param = slice->param;
2596     GstH264PPS * const pps = slice_hdr->pps;
2597     GstH264SPS * const sps = pps->sequence;
2598     GstH264PredWeightTable * const w = &slice_hdr->pred_weight_table;
2599     guint num_weight_tables = 0;
2600     gint i, j;
2601
2602     if (pps->weighted_pred_flag &&
2603         (GST_H264_IS_P_SLICE(slice_hdr) || GST_H264_IS_SP_SLICE(slice_hdr)))
2604         num_weight_tables = 1;
2605     else if (pps->weighted_bipred_idc == 1 && GST_H264_IS_B_SLICE(slice_hdr))
2606         num_weight_tables = 2;
2607     else
2608         num_weight_tables = 0;
2609
2610     slice_param->luma_log2_weight_denom   = 0;
2611     slice_param->chroma_log2_weight_denom = 0;
2612     slice_param->luma_weight_l0_flag      = 0;
2613     slice_param->chroma_weight_l0_flag    = 0;
2614     slice_param->luma_weight_l1_flag      = 0;
2615     slice_param->chroma_weight_l1_flag    = 0;
2616
2617     if (num_weight_tables < 1)
2618         return TRUE;
2619
2620     slice_param->luma_log2_weight_denom   = w->luma_log2_weight_denom;
2621     slice_param->chroma_log2_weight_denom = w->chroma_log2_weight_denom;
2622
2623     slice_param->luma_weight_l0_flag = 1;
2624     for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2625         slice_param->luma_weight_l0[i] = w->luma_weight_l0[i];
2626         slice_param->luma_offset_l0[i] = w->luma_offset_l0[i];
2627     }
2628
2629     slice_param->chroma_weight_l0_flag = sps->chroma_array_type != 0;
2630     if (slice_param->chroma_weight_l0_flag) {
2631         for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2632             for (j = 0; j < 2; j++) {
2633                 slice_param->chroma_weight_l0[i][j] = w->chroma_weight_l0[i][j];
2634                 slice_param->chroma_offset_l0[i][j] = w->chroma_offset_l0[i][j];
2635             }
2636         }
2637     }
2638
2639     if (num_weight_tables < 2)
2640         return TRUE;
2641
2642     slice_param->luma_weight_l1_flag = 1;
2643     for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2644         slice_param->luma_weight_l1[i] = w->luma_weight_l1[i];
2645         slice_param->luma_offset_l1[i] = w->luma_offset_l1[i];
2646     }
2647
2648     slice_param->chroma_weight_l1_flag = sps->chroma_array_type != 0;
2649     if (slice_param->chroma_weight_l1_flag) {
2650         for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2651             for (j = 0; j < 2; j++) {
2652                 slice_param->chroma_weight_l1[i][j] = w->chroma_weight_l1[i][j];
2653                 slice_param->chroma_offset_l1[i][j] = w->chroma_offset_l1[i][j];
2654             }
2655         }
2656     }
2657     return TRUE;
2658 }
2659
2660 static gboolean
2661 fill_RefPicList(GstVaapiDecoderH264 *decoder,
2662     GstVaapiSlice *slice, GstH264SliceHdr *slice_hdr)
2663 {
2664     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2665     VASliceParameterBufferH264 * const slice_param = slice->param;
2666     guint i, num_ref_lists = 0;
2667
2668     slice_param->num_ref_idx_l0_active_minus1 = 0;
2669     slice_param->num_ref_idx_l1_active_minus1 = 0;
2670
2671     if (GST_H264_IS_B_SLICE(slice_hdr))
2672         num_ref_lists = 2;
2673     else if (GST_H264_IS_I_SLICE(slice_hdr))
2674         num_ref_lists = 0;
2675     else
2676         num_ref_lists = 1;
2677
2678     if (num_ref_lists < 1)
2679         return TRUE;
2680
2681     slice_param->num_ref_idx_l0_active_minus1 =
2682         slice_hdr->num_ref_idx_l0_active_minus1;
2683
2684     for (i = 0; i < priv->RefPicList0_count && priv->RefPicList0[i]; i++)
2685         vaapi_fill_picture(&slice_param->RefPicList0[i], priv->RefPicList0[i], 0);
2686     for (; i <= slice_param->num_ref_idx_l0_active_minus1; i++)
2687         vaapi_init_picture(&slice_param->RefPicList0[i]);
2688
2689     if (num_ref_lists < 2)
2690         return TRUE;
2691
2692     slice_param->num_ref_idx_l1_active_minus1 =
2693         slice_hdr->num_ref_idx_l1_active_minus1;
2694
2695     for (i = 0; i < priv->RefPicList1_count && priv->RefPicList1[i]; i++)
2696         vaapi_fill_picture(&slice_param->RefPicList1[i], priv->RefPicList1[i], 0);
2697     for (; i <= slice_param->num_ref_idx_l1_active_minus1; i++)
2698         vaapi_init_picture(&slice_param->RefPicList1[i]);
2699     return TRUE;
2700 }
2701
2702 static gboolean
2703 fill_slice(GstVaapiDecoderH264 *decoder,
2704     GstVaapiSlice *slice, GstH264SliceHdr *slice_hdr)
2705 {
2706     VASliceParameterBufferH264 * const slice_param = slice->param;
2707
2708     /* Fill in VASliceParameterBufferH264 */
2709     slice_param->slice_data_bit_offset          = get_slice_data_bit_offset(slice_hdr);
2710     slice_param->first_mb_in_slice              = slice_hdr->first_mb_in_slice;
2711     slice_param->slice_type                     = slice_hdr->type % 5;
2712     slice_param->direct_spatial_mv_pred_flag    = slice_hdr->direct_spatial_mv_pred_flag;
2713     slice_param->cabac_init_idc                 = slice_hdr->cabac_init_idc;
2714     slice_param->slice_qp_delta                 = slice_hdr->slice_qp_delta;
2715     slice_param->disable_deblocking_filter_idc  = slice_hdr->disable_deblocking_filter_idc;
2716     slice_param->slice_alpha_c0_offset_div2     = slice_hdr->slice_alpha_c0_offset_div2;
2717     slice_param->slice_beta_offset_div2         = slice_hdr->slice_beta_offset_div2;
2718
2719     if (!fill_RefPicList(decoder, slice, slice_hdr))
2720         return FALSE;
2721     if (!fill_pred_weight_table(decoder, slice, slice_hdr))
2722         return FALSE;
2723     return TRUE;
2724 }
2725
2726 static GstVaapiDecoderStatus
2727 decode_slice(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnit *unit)
2728 {
2729     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2730     GstVaapiParserInfoH264 * const pi = unit->parsed_info;
2731     GstVaapiPictureH264 * const picture = priv->current_picture;
2732     GstH264SliceHdr * const slice_hdr = &pi->data.slice_hdr;
2733     GstVaapiSlice *slice;
2734     GstBuffer * const buffer =
2735         GST_VAAPI_DECODER_CODEC_FRAME(decoder)->input_buffer;
2736     GstMapInfo map_info;
2737
2738     GST_DEBUG("slice (%u bytes)", pi->nalu.size);
2739
2740     if (!priv->got_sps || !priv->got_pps) {
2741         GST_ERROR("not initialized yet");
2742         return GST_VAAPI_DECODER_STATUS_SUCCESS;
2743     }
2744
2745     if (!gst_buffer_map(buffer, &map_info, GST_MAP_READ)) {
2746         GST_ERROR("failed to map buffer");
2747         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2748     }
2749
2750     slice = GST_VAAPI_SLICE_NEW(H264, decoder,
2751         (map_info.data + unit->offset + pi->nalu.offset), pi->nalu.size);
2752     gst_buffer_unmap(buffer, &map_info);
2753     if (!slice) {
2754         GST_ERROR("failed to allocate slice");
2755         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2756     }
2757
2758     if (!fill_slice(decoder, slice, slice_hdr)) {
2759         gst_vaapi_mini_object_unref(GST_VAAPI_MINI_OBJECT(slice));
2760         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2761     }
2762
2763     gst_vaapi_picture_add_slice(GST_VAAPI_PICTURE_CAST(picture), slice);
2764     picture->last_slice_hdr = slice_hdr;
2765     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2766 }
2767
2768 static inline gint
2769 scan_for_start_code(GstAdapter *adapter, guint ofs, guint size, guint32 *scp)
2770 {
2771     return (gint)gst_adapter_masked_scan_uint32_peek(adapter,
2772                                                      0xffffff00, 0x00000100,
2773                                                      ofs, size,
2774                                                      scp);
2775 }
2776
2777 static GstVaapiDecoderStatus
2778 decode_unit(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnit *unit)
2779 {
2780     GstVaapiParserInfoH264 * const pi = unit->parsed_info;
2781     GstVaapiDecoderStatus status;
2782
2783     switch (pi->nalu.type) {
2784     case GST_H264_NAL_SLICE_IDR:
2785         /* fall-through. IDR specifics are handled in init_picture() */
2786     case GST_H264_NAL_SLICE:
2787         status = decode_slice(decoder, unit);
2788         break;
2789     case GST_H264_NAL_SEQ_END:
2790     case GST_H264_NAL_STREAM_END:
2791         status = decode_sequence_end(decoder);
2792         break;
2793     case GST_H264_NAL_SEI:
2794         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2795         break;
2796     default:
2797         GST_WARNING("unsupported NAL unit type %d", pi->nalu.type);
2798         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
2799         break;
2800     }
2801     return status;
2802 }
2803
2804 static GstVaapiDecoderStatus
2805 gst_vaapi_decoder_h264_decode_codec_data(GstVaapiDecoder *base_decoder,
2806     const guchar *buf, guint buf_size)
2807 {
2808     GstVaapiDecoderH264 * const decoder =
2809         GST_VAAPI_DECODER_H264_CAST(base_decoder);
2810     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2811     GstVaapiDecoderStatus status;
2812     GstVaapiDecoderUnit unit;
2813     GstVaapiParserInfoH264 pi;
2814     GstH264ParserResult result;
2815     guint i, ofs, num_sps, num_pps;
2816
2817     unit.parsed_info = &pi;
2818
2819     if (buf_size < 8)
2820         return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2821
2822     if (buf[0] != 1) {
2823         GST_ERROR("failed to decode codec-data, not in avcC format");
2824         return GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
2825     }
2826
2827     priv->nal_length_size = (buf[4] & 0x03) + 1;
2828
2829     num_sps = buf[5] & 0x1f;
2830     ofs = 6;
2831
2832     for (i = 0; i < num_sps; i++) {
2833         result = gst_h264_parser_identify_nalu_avc(
2834             priv->parser,
2835             buf, ofs, buf_size, 2,
2836             &pi.nalu
2837         );
2838         if (result != GST_H264_PARSER_OK)
2839             return get_status(result);
2840
2841         status = parse_sps(decoder, &unit);
2842         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2843             return status;
2844         ofs = pi.nalu.offset + pi.nalu.size;
2845     }
2846
2847     num_pps = buf[ofs];
2848     ofs++;
2849
2850     for (i = 0; i < num_pps; i++) {
2851         result = gst_h264_parser_identify_nalu_avc(
2852             priv->parser,
2853             buf, ofs, buf_size, 2,
2854             &pi.nalu
2855         );
2856         if (result != GST_H264_PARSER_OK)
2857             return get_status(result);
2858
2859         status = parse_pps(decoder, &unit);
2860         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2861             return status;
2862         ofs = pi.nalu.offset + pi.nalu.size;
2863     }
2864
2865     priv->is_avcC = TRUE;
2866     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2867 }
2868
2869 static GstVaapiDecoderStatus
2870 ensure_decoder(GstVaapiDecoderH264 *decoder)
2871 {
2872     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2873     GstVaapiDecoderStatus status;
2874
2875     if (!priv->is_opened) {
2876         priv->is_opened = gst_vaapi_decoder_h264_open(decoder);
2877         if (!priv->is_opened)
2878             return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CODEC;
2879
2880         status = gst_vaapi_decoder_decode_codec_data(
2881             GST_VAAPI_DECODER_CAST(decoder));
2882         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2883             return status;
2884     }
2885     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2886 }
2887
2888 static GstVaapiDecoderStatus
2889 gst_vaapi_decoder_h264_parse(GstVaapiDecoder *base_decoder,
2890     GstAdapter *adapter, gboolean at_eos, GstVaapiDecoderUnit *unit)
2891 {
2892     GstVaapiDecoderH264 * const decoder =
2893         GST_VAAPI_DECODER_H264_CAST(base_decoder);
2894     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2895     GstVaapiParserState * const ps = GST_VAAPI_PARSER_STATE(base_decoder);
2896     GstVaapiParserInfoH264 *pi;
2897     GstVaapiDecoderStatus status;
2898     GstH264ParserResult result;
2899     guchar *buf;
2900     guint i, size, buf_size, nalu_size, flags;
2901     guint32 start_code;
2902     gint ofs, ofs2;
2903
2904     status = ensure_decoder(decoder);
2905     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2906         return status;
2907
2908     size = gst_adapter_available(adapter);
2909
2910     if (priv->is_avcC) {
2911         if (size < priv->nal_length_size)
2912             return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2913
2914         buf = (guchar *)&start_code;
2915         g_assert(priv->nal_length_size <= sizeof(start_code));
2916         gst_adapter_copy(adapter, buf, 0, priv->nal_length_size);
2917
2918         nalu_size = 0;
2919         for (i = 0; i < priv->nal_length_size; i++)
2920             nalu_size = (nalu_size << 8) | buf[i];
2921
2922         buf_size = priv->nal_length_size + nalu_size;
2923         if (size < buf_size)
2924             return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2925     }
2926     else {
2927         if (size < 4)
2928             return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2929
2930         ofs = scan_for_start_code(adapter, 0, size, NULL);
2931         if (ofs < 0)
2932             return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2933
2934         if (ofs > 0) {
2935             gst_adapter_flush(adapter, ofs);
2936             size -= ofs;
2937         }
2938
2939         ofs2 = ps->input_offset2 - ofs - 4;
2940         if (ofs2 < 4)
2941             ofs2 = 4;
2942
2943         ofs = G_UNLIKELY(size < ofs2 + 4) ? -1 :
2944             scan_for_start_code(adapter, ofs2, size - ofs2, NULL);
2945         if (ofs < 0) {
2946             // Assume the whole NAL unit is present if end-of-stream
2947             if (!at_eos) {
2948                 ps->input_offset2 = size;
2949                 return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2950             }
2951             ofs = size;
2952         }
2953         buf_size = ofs;
2954     }
2955     ps->input_offset2 = 0;
2956
2957     buf = (guchar *)gst_adapter_map(adapter, buf_size);
2958     if (!buf)
2959         return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2960
2961     unit->size = buf_size;
2962
2963     pi = gst_vaapi_parser_info_h264_new();
2964     if (!pi)
2965         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2966
2967     gst_vaapi_decoder_unit_set_parsed_info(unit,
2968         pi, (GDestroyNotify)gst_vaapi_mini_object_unref);
2969
2970     if (priv->is_avcC)
2971         result = gst_h264_parser_identify_nalu_avc(priv->parser,
2972             buf, 0, buf_size, priv->nal_length_size, &pi->nalu);
2973     else
2974         result = gst_h264_parser_identify_nalu_unchecked(priv->parser,
2975             buf, 0, buf_size, &pi->nalu);
2976     status = get_status(result);
2977     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2978         return status;
2979
2980     switch (pi->nalu.type) {
2981     case GST_H264_NAL_SPS:
2982         status = parse_sps(decoder, unit);
2983         break;
2984     case GST_H264_NAL_PPS:
2985         status = parse_pps(decoder, unit);
2986         break;
2987     case GST_H264_NAL_SEI:
2988         status = parse_sei(decoder, unit);
2989         break;
2990     case GST_H264_NAL_SLICE_IDR:
2991     case GST_H264_NAL_SLICE:
2992         status = parse_slice(decoder, unit);
2993         break;
2994     default:
2995         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2996         break;
2997     }
2998     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2999         return status;
3000
3001     flags = 0;
3002     switch (pi->nalu.type) {
3003     case GST_H264_NAL_AU_DELIMITER:
3004         flags |= GST_VAAPI_DECODER_UNIT_FLAG_FRAME_START;
3005         /* fall-through */
3006     case GST_H264_NAL_FILLER_DATA:
3007         flags |= GST_VAAPI_DECODER_UNIT_FLAG_SKIP;
3008         break;
3009     case GST_H264_NAL_STREAM_END:
3010         flags |= GST_VAAPI_DECODER_UNIT_FLAG_STREAM_END;
3011         /* fall-through */
3012     case GST_H264_NAL_SEQ_END:
3013         flags |= GST_VAAPI_DECODER_UNIT_FLAG_FRAME_END;
3014         break;
3015     case GST_H264_NAL_SPS:
3016     case GST_H264_NAL_PPS:
3017         flags |= GST_VAAPI_DECODER_UNIT_FLAG_SKIP;
3018         /* fall-through */
3019     case GST_H264_NAL_SEI:
3020         flags |= GST_VAAPI_DECODER_UNIT_FLAG_FRAME_START;
3021         break;
3022     case GST_H264_NAL_SLICE_IDR:
3023     case GST_H264_NAL_SLICE:
3024         flags |= GST_VAAPI_DECODER_UNIT_FLAG_SLICE;
3025         if (is_new_picture(pi, priv->prev_slice_pi))
3026             flags |= GST_VAAPI_DECODER_UNIT_FLAG_FRAME_START;
3027         gst_vaapi_parser_info_h264_replace(&priv->prev_slice_pi, pi);
3028         break;
3029     default:
3030         if (pi->nalu.type >= 14 && pi->nalu.type <= 18)
3031             flags |= GST_VAAPI_DECODER_UNIT_FLAG_FRAME_START;
3032         break;
3033     }
3034     GST_VAAPI_DECODER_UNIT_FLAG_SET(unit, flags);
3035
3036     pi->nalu.data = NULL;
3037     return GST_VAAPI_DECODER_STATUS_SUCCESS;
3038 }
3039
3040 static GstVaapiDecoderStatus
3041 gst_vaapi_decoder_h264_decode(GstVaapiDecoder *base_decoder,
3042     GstVaapiDecoderUnit *unit)
3043 {
3044     GstVaapiDecoderH264 * const decoder =
3045         GST_VAAPI_DECODER_H264_CAST(base_decoder);
3046     GstVaapiDecoderStatus status;
3047
3048     status = ensure_decoder(decoder);
3049     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
3050         return status;
3051     return decode_unit(decoder, unit);
3052 }
3053
3054 static GstVaapiDecoderStatus
3055 gst_vaapi_decoder_h264_start_frame(GstVaapiDecoder *base_decoder,
3056     GstVaapiDecoderUnit *unit)
3057 {
3058     GstVaapiDecoderH264 * const decoder =
3059         GST_VAAPI_DECODER_H264_CAST(base_decoder);
3060
3061     return decode_picture(decoder, unit);
3062 }
3063
3064 static GstVaapiDecoderStatus
3065 gst_vaapi_decoder_h264_end_frame(GstVaapiDecoder *base_decoder)
3066 {
3067     GstVaapiDecoderH264 * const decoder =
3068         GST_VAAPI_DECODER_H264_CAST(base_decoder);
3069
3070     return decode_current_picture(decoder);
3071 }
3072
3073 static GstVaapiDecoderStatus
3074 gst_vaapi_decoder_h264_flush(GstVaapiDecoder *base_decoder)
3075 {
3076     GstVaapiDecoderH264 * const decoder =
3077         GST_VAAPI_DECODER_H264_CAST(base_decoder);
3078
3079     dpb_flush(decoder);
3080     return GST_VAAPI_DECODER_STATUS_SUCCESS;
3081 }
3082
3083 static void
3084 gst_vaapi_decoder_h264_class_init(GstVaapiDecoderH264Class *klass)
3085 {
3086     GstVaapiMiniObjectClass * const object_class =
3087         GST_VAAPI_MINI_OBJECT_CLASS(klass);
3088     GstVaapiDecoderClass * const decoder_class = GST_VAAPI_DECODER_CLASS(klass);
3089
3090     object_class->size          = sizeof(GstVaapiDecoderH264);
3091     object_class->finalize      = (GDestroyNotify)gst_vaapi_decoder_finalize;
3092
3093     decoder_class->create       = gst_vaapi_decoder_h264_create;
3094     decoder_class->destroy      = gst_vaapi_decoder_h264_destroy;
3095     decoder_class->parse        = gst_vaapi_decoder_h264_parse;
3096     decoder_class->decode       = gst_vaapi_decoder_h264_decode;
3097     decoder_class->start_frame  = gst_vaapi_decoder_h264_start_frame;
3098     decoder_class->end_frame    = gst_vaapi_decoder_h264_end_frame;
3099     decoder_class->flush        = gst_vaapi_decoder_h264_flush;
3100
3101     decoder_class->decode_codec_data =
3102         gst_vaapi_decoder_h264_decode_codec_data;
3103 }
3104
3105 static inline const GstVaapiDecoderClass *
3106 gst_vaapi_decoder_h264_class(void)
3107 {
3108     static GstVaapiDecoderH264Class g_class;
3109     static gsize g_class_init = FALSE;
3110
3111     if (g_once_init_enter(&g_class_init)) {
3112         gst_vaapi_decoder_h264_class_init(&g_class);
3113         g_once_init_leave(&g_class_init, TRUE);
3114     }
3115     return GST_VAAPI_DECODER_CLASS(&g_class);
3116 }
3117
3118 /**
3119  * gst_vaapi_decoder_h264_new:
3120  * @display: a #GstVaapiDisplay
3121  * @caps: a #GstCaps holding codec information
3122  *
3123  * Creates a new #GstVaapiDecoder for MPEG-2 decoding.  The @caps can
3124  * hold extra information like codec-data and pictured coded size.
3125  *
3126  * Return value: the newly allocated #GstVaapiDecoder object
3127  */
3128 GstVaapiDecoder *
3129 gst_vaapi_decoder_h264_new(GstVaapiDisplay *display, GstCaps *caps)
3130 {
3131     return gst_vaapi_decoder_new(gst_vaapi_decoder_h264_class(), display, caps);
3132 }