gst-libs/gst/vaapi/gstvaapidecoder_h264.c

   1 /*
   2  *  gstvaapidecoder_h264.c - H.264 decoder
   3  *
   4  *  Copyright (C) 2011-2013 Intel Corporation
   5  *
   6  *  This library is free software; you can redistribute it and/or
   7  *  modify it under the terms of the GNU Lesser General Public License
   8  *  as published by the Free Software Foundation; either version 2.1
   9  *  of the License, or (at your option) any later version.
  10  *
  11  *  This library is distributed in the hope that it will be useful,
  12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  *  Lesser General Public License for more details.
  15  *
  16  *  You should have received a copy of the GNU Lesser General Public
  17  *  License along with this library; if not, write to the Free
  18  *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  19  *  Boston, MA 02110-1301 USA
  20  */
  21
  22 /**
  23  * SECTION:gstvaapidecoder_h264
  24  * @short_description: H.264 decoder
  25  */
  26
  27 #include "sysdeps.h"
  28 #include <string.h>
  29 #include <gst/base/gstadapter.h>
  30 #include <gst/codecparsers/gsth264parser.h>
  31 #include "gstvaapidecoder_h264.h"
  32 #include "gstvaapidecoder_objects.h"
  33 #include "gstvaapidecoder_priv.h"
  34 #include "gstvaapidisplay_priv.h"
  35 #include "gstvaapiobject_priv.h"
  36
  37 #define DEBUG 1
  38 #include "gstvaapidebug.h"
  39
  40 /* Defined to 1 if strict ordering of DPB is needed. Only useful for debug */
  41 #define USE_STRICT_DPB_ORDERING 0
  42
  43 typedef struct _GstVaapiFrameStore              GstVaapiFrameStore;
  44 typedef struct _GstVaapiFrameStoreClass         GstVaapiFrameStoreClass;
  45 typedef struct _GstVaapiParserInfoH264          GstVaapiParserInfoH264;
  46 typedef struct _GstVaapiPictureH264             GstVaapiPictureH264;
  47
  48 // Used for field_poc[]
  49 #define TOP_FIELD       0
  50 #define BOTTOM_FIELD    1
  51
  52 /* ------------------------------------------------------------------------- */
  53 /* --- H.264 Parser Info                                                 --- */
  54 /* ------------------------------------------------------------------------- */
  55
  56 #define GST_VAAPI_PARSER_INFO_H264(obj) \
  57     ((GstVaapiParserInfoH264 *)(obj))
  58
  59 struct _GstVaapiParserInfoH264 {
  60     GstH264NalUnit      nalu;
  61     union {
  62         GstH264SPS      sps;
  63         GstH264PPS      pps;
  64         GstH264SliceHdr slice_hdr;
  65     }                   data;
  66 };
  67
  68 static inline const GstVaapiMiniObjectClass *
  69 gst_vaapi_parser_info_h264_class(void)
  70 {
  71     static const GstVaapiMiniObjectClass GstVaapiParserInfoH264Class = {
  72         sizeof(GstVaapiParserInfoH264),
  73         NULL
  74     };
  75     return &GstVaapiParserInfoH264Class;
  76 }
  77
  78 static inline GstVaapiParserInfoH264 *
  79 gst_vaapi_parser_info_h264_new(void)
  80 {
  81     return (GstVaapiParserInfoH264 *)
  82         gst_vaapi_mini_object_new(gst_vaapi_parser_info_h264_class());
  83 }
  84
  85 #define gst_vaapi_parser_info_h264_ref(pi) \
  86     gst_vaapi_mini_object_ref(GST_VAAPI_MINI_OBJECT(pi))
  87
  88 #define gst_vaapi_parser_info_h264_unref(pi) \
  89     gst_vaapi_mini_object_unref(GST_VAAPI_MINI_OBJECT(pi))
  90
  91 #define gst_vaapi_parser_info_h264_replace(old_pi_ptr, new_pi)          \
  92     gst_vaapi_mini_object_replace((GstVaapiMiniObject **)(old_pi_ptr),  \
  93         (GstVaapiMiniObject *)(new_pi))
  94
  95 /* ------------------------------------------------------------------------- */
  96 /* --- H.264 Pictures                                                    --- */
  97 /* ------------------------------------------------------------------------- */
  98
  99 /*
 100  * Extended picture flags:
 101  *
 102  * @GST_VAAPI_PICTURE_FLAG_IDR: flag that specifies an IDR picture
 103  * @GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE: flag that specifies
 104  *     "used for short-term reference"
 105  * @GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE: flag that specifies
 106  *     "used for long-term reference"
 107  * @GST_VAAPI_PICTURE_FLAGS_REFERENCE: mask covering any kind of
 108  *     reference picture (short-term reference or long-term reference)
 109  */
 110 enum {
 111     GST_VAAPI_PICTURE_FLAG_IDR = (GST_VAAPI_PICTURE_FLAG_LAST << 0),
 112
 113     GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE = (
 114         GST_VAAPI_PICTURE_FLAG_REFERENCE),
 115     GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE = (
 116         GST_VAAPI_PICTURE_FLAG_REFERENCE | (GST_VAAPI_PICTURE_FLAG_LAST << 1)),
 117     GST_VAAPI_PICTURE_FLAGS_REFERENCE = (
 118         GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE |
 119         GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE),
 120 };
 121
 122 #define GST_VAAPI_PICTURE_IS_IDR(picture) \
 123     (GST_VAAPI_PICTURE_FLAG_IS_SET(picture, GST_VAAPI_PICTURE_FLAG_IDR))
 124
 125 #define GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture)      \
 126     ((GST_VAAPI_PICTURE_FLAGS(picture) &                        \
 127       GST_VAAPI_PICTURE_FLAGS_REFERENCE) ==                     \
 128      GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE)
 129
 130 #define GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture)       \
 131     ((GST_VAAPI_PICTURE_FLAGS(picture) &                        \
 132       GST_VAAPI_PICTURE_FLAGS_REFERENCE) ==                     \
 133      GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE)
 134
 135 struct _GstVaapiPictureH264 {
 136     GstVaapiPicture             base;
 137     GstH264PPS                 *pps;
 138     GstH264SliceHdr            *last_slice_hdr;
 139     guint                       structure;
 140     gint32                      field_poc[2];
 141     gint32                      frame_num;              // Original frame_num from slice_header()
 142     gint32                      frame_num_wrap;         // Temporary for ref pic marking: FrameNumWrap
 143     gint32                      long_term_frame_idx;    // Temporary for ref pic marking: LongTermFrameIdx
 144     gint32                      pic_num;                // Temporary for ref pic marking: PicNum
 145     gint32                      long_term_pic_num;      // Temporary for ref pic marking: LongTermPicNum
 146     GstVaapiPictureH264        *other_field;            // Temporary for ref pic marking: other field in the same frame store
 147     guint                       output_flag             : 1;
 148     guint                       output_needed           : 1;
 149 };
 150
 151 GST_VAAPI_CODEC_DEFINE_TYPE(GstVaapiPictureH264, gst_vaapi_picture_h264);
 152
 153 void
 154 gst_vaapi_picture_h264_destroy(GstVaapiPictureH264 *picture)
 155 {
 156     gst_vaapi_picture_destroy(GST_VAAPI_PICTURE(picture));
 157 }
 158
 159 gboolean
 160 gst_vaapi_picture_h264_create(
 161     GstVaapiPictureH264                      *picture,
 162     const GstVaapiCodecObjectConstructorArgs *args
 163 )
 164 {
 165     if (!gst_vaapi_picture_create(GST_VAAPI_PICTURE(picture), args))
 166         return FALSE;
 167
 168     picture->field_poc[0]       = G_MAXINT32;
 169     picture->field_poc[1]       = G_MAXINT32;
 170     picture->output_needed      = FALSE;
 171     return TRUE;
 172 }
 173
 174 static inline GstVaapiPictureH264 *
 175 gst_vaapi_picture_h264_new(GstVaapiDecoderH264 *decoder)
 176 {
 177     return (GstVaapiPictureH264 *)gst_vaapi_codec_object_new(
 178         &GstVaapiPictureH264Class,
 179         GST_VAAPI_CODEC_BASE(decoder),
 180         NULL, sizeof(VAPictureParameterBufferH264),
 181         NULL, 0,
 182         0);
 183 }
 184
 185 static inline void
 186 gst_vaapi_picture_h264_set_reference(
 187     GstVaapiPictureH264 *picture,
 188     guint                reference_flags,
 189     gboolean             other_field
 190 )
 191 {
 192     if (!picture)
 193         return;
 194     GST_VAAPI_PICTURE_FLAG_UNSET(picture, GST_VAAPI_PICTURE_FLAGS_REFERENCE);
 195     GST_VAAPI_PICTURE_FLAG_SET(picture, reference_flags);
 196
 197     if (!other_field || !(picture = picture->other_field))
 198         return;
 199     GST_VAAPI_PICTURE_FLAG_UNSET(picture, GST_VAAPI_PICTURE_FLAGS_REFERENCE);
 200     GST_VAAPI_PICTURE_FLAG_SET(picture, reference_flags);
 201 }
 202
 203 static inline GstVaapiPictureH264 *
 204 gst_vaapi_picture_h264_new_field(GstVaapiPictureH264 *picture)
 205 {
 206     g_return_val_if_fail(picture, NULL);
 207
 208     return (GstVaapiPictureH264 *)gst_vaapi_picture_new_field(&picture->base);
 209 }
 210
 211 /* ------------------------------------------------------------------------- */
 212 /* --- Frame Buffers (DPB)                                               --- */
 213 /* ------------------------------------------------------------------------- */
 214
 215 struct _GstVaapiFrameStore {
 216     /*< private >*/
 217     GstVaapiMiniObject          parent_instance;
 218
 219     guint                       structure;
 220     GstVaapiPictureH264        *buffers[2];
 221     guint                       num_buffers;
 222     guint                       output_needed;
 223 };
 224
 225 static void
 226 gst_vaapi_frame_store_finalize(gpointer object)
 227 {
 228     GstVaapiFrameStore * const fs = object;
 229     guint i;
 230
 231     for (i = 0; i < fs->num_buffers; i++)
 232         gst_vaapi_picture_replace(&fs->buffers[i], NULL);
 233 }
 234
 235 static GstVaapiFrameStore *
 236 gst_vaapi_frame_store_new(GstVaapiPictureH264 *picture)
 237 {
 238     GstVaapiFrameStore *fs;
 239
 240     static const GstVaapiMiniObjectClass GstVaapiFrameStoreClass = {
 241         sizeof(GstVaapiFrameStore),
 242         gst_vaapi_frame_store_finalize
 243     };
 244
 245     fs = (GstVaapiFrameStore *)
 246         gst_vaapi_mini_object_new(&GstVaapiFrameStoreClass);
 247     if (!fs)
 248         return NULL;
 249
 250     fs->structure       = picture->structure;
 251     fs->buffers[0]      = gst_vaapi_picture_ref(picture);
 252     fs->buffers[1]      = NULL;
 253     fs->num_buffers     = 1;
 254     fs->output_needed   = picture->output_needed;
 255     return fs;
 256 }
 257
 258 static gboolean
 259 gst_vaapi_frame_store_add(GstVaapiFrameStore *fs, GstVaapiPictureH264 *picture)
 260 {
 261     guint field;
 262
 263     g_return_val_if_fail(fs->num_buffers == 1, FALSE);
 264     g_return_val_if_fail(!GST_VAAPI_PICTURE_IS_FRAME(picture), FALSE);
 265     g_return_val_if_fail(!GST_VAAPI_PICTURE_IS_FIRST_FIELD(picture), FALSE);
 266
 267     gst_vaapi_picture_replace(&fs->buffers[fs->num_buffers++], picture);
 268     if (picture->output_flag) {
 269         picture->output_needed = TRUE;
 270         fs->output_needed++;
 271     }
 272
 273     fs->structure = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
 274
 275     field = picture->structure == GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD ?
 276         TOP_FIELD : BOTTOM_FIELD;
 277     g_return_val_if_fail(fs->buffers[0]->field_poc[field] == G_MAXINT32, FALSE);
 278     fs->buffers[0]->field_poc[field] = picture->field_poc[field];
 279     g_return_val_if_fail(picture->field_poc[!field] == G_MAXINT32, FALSE);
 280     picture->field_poc[!field] = fs->buffers[0]->field_poc[!field];
 281     return TRUE;
 282 }
 283
 284 static gboolean
 285 gst_vaapi_frame_store_split_fields(GstVaapiFrameStore *fs)
 286 {
 287     GstVaapiPictureH264 * const first_field = fs->buffers[0];
 288     GstVaapiPictureH264 *second_field;
 289
 290     g_return_val_if_fail(fs->num_buffers == 1, FALSE);
 291
 292     first_field->base.structure = GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD;
 293     GST_VAAPI_PICTURE_FLAG_SET(first_field, GST_VAAPI_PICTURE_FLAG_INTERLACED);
 294
 295     second_field = gst_vaapi_picture_h264_new_field(first_field);
 296     if (!second_field)
 297         return FALSE;
 298     gst_vaapi_picture_replace(&fs->buffers[fs->num_buffers++], second_field);
 299     gst_vaapi_picture_unref(second_field);
 300
 301     second_field->frame_num    = first_field->frame_num;
 302     second_field->field_poc[0] = first_field->field_poc[0];
 303     second_field->field_poc[1] = first_field->field_poc[1];
 304     second_field->output_flag  = first_field->output_flag;
 305     if (second_field->output_flag) {
 306         second_field->output_needed = TRUE;
 307         fs->output_needed++;
 308     }
 309     return TRUE;
 310 }
 311
 312 static inline gboolean
 313 gst_vaapi_frame_store_has_frame(GstVaapiFrameStore *fs)
 314 {
 315     return fs->structure == GST_VAAPI_PICTURE_STRUCTURE_FRAME;
 316 }
 317
 318 static inline gboolean
 319 gst_vaapi_frame_store_has_reference(GstVaapiFrameStore *fs)
 320 {
 321     guint i;
 322
 323     for (i = 0; i < fs->num_buffers; i++) {
 324         if (GST_VAAPI_PICTURE_IS_REFERENCE(fs->buffers[i]))
 325             return TRUE;
 326     }
 327     return FALSE;
 328 }
 329
 330 #define gst_vaapi_frame_store_ref(fs) \
 331     gst_vaapi_mini_object_ref(GST_VAAPI_MINI_OBJECT(fs))
 332
 333 #define gst_vaapi_frame_store_unref(fs) \
 334     gst_vaapi_mini_object_unref(GST_VAAPI_MINI_OBJECT(fs))
 335
 336 #define gst_vaapi_frame_store_replace(old_fs_p, new_fs)                 \
 337     gst_vaapi_mini_object_replace((GstVaapiMiniObject **)(old_fs_p),    \
 338         (GstVaapiMiniObject *)(new_fs))
 339
 340 /* ------------------------------------------------------------------------- */
 341 /* --- H.264 Decoder                                                     --- */
 342 /* ------------------------------------------------------------------------- */
 343
 344 G_DEFINE_TYPE(GstVaapiDecoderH264,
 345               gst_vaapi_decoder_h264,
 346               GST_VAAPI_TYPE_DECODER)
 347
 348 #define GST_VAAPI_DECODER_H264_CAST(decoder) \
 349     ((GstVaapiDecoderH264 *)(decoder))
 350
 351 #define GST_VAAPI_DECODER_H264_GET_PRIVATE(obj)                 \
 352     (G_TYPE_INSTANCE_GET_PRIVATE((obj),                         \
 353                                  GST_VAAPI_TYPE_DECODER_H264,   \
 354                                  GstVaapiDecoderH264Private))
 355
 356 struct _GstVaapiDecoderH264Private {
 357     GstH264NalParser           *parser;
 358     GstVaapiPictureH264        *current_picture;
 359     GstVaapiParserInfoH264     *prev_slice_pi;
 360     GstVaapiFrameStore         *prev_frame;
 361     GstVaapiFrameStore         *dpb[16];
 362     guint                       dpb_count;
 363     guint                       dpb_size;
 364     GstVaapiProfile             profile;
 365     GstVaapiEntrypoint          entrypoint;
 366     GstVaapiChromaType          chroma_type;
 367     GstVaapiPictureH264        *short_ref[32];
 368     guint                       short_ref_count;
 369     GstVaapiPictureH264        *long_ref[32];
 370     guint                       long_ref_count;
 371     GstVaapiPictureH264        *RefPicList0[32];
 372     guint                       RefPicList0_count;
 373     GstVaapiPictureH264        *RefPicList1[32];
 374     guint                       RefPicList1_count;
 375     guint                       nal_length_size;
 376     guint                       mb_width;
 377     guint                       mb_height;
 378     gint32                      field_poc[2];           // 0:TopFieldOrderCnt / 1:BottomFieldOrderCnt
 379     gint32                      poc_msb;                // PicOrderCntMsb
 380     gint32                      poc_lsb;                // pic_order_cnt_lsb (from slice_header())
 381     gint32                      prev_poc_msb;           // prevPicOrderCntMsb
 382     gint32                      prev_poc_lsb;           // prevPicOrderCntLsb
 383     gint32                      frame_num_offset;       // FrameNumOffset
 384     gint32                      frame_num;              // frame_num (from slice_header())
 385     gint32                      prev_frame_num;         // prevFrameNum
 386     gboolean                    prev_pic_has_mmco5;     // prevMmco5Pic
 387     gboolean                    prev_pic_structure;     // previous picture structure
 388     guint                       is_constructed          : 1;
 389     guint                       is_opened               : 1;
 390     guint                       is_avcC                 : 1;
 391     guint                       got_sps                 : 1;
 392     guint                       got_pps                 : 1;
 393     guint                       has_context             : 1;
 394     guint                       progressive_sequence    : 1;
 395 };
 396
 397 static gboolean
 398 exec_ref_pic_marking(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture);
 399
 400 /* Get number of reference frames to use */
 401 static guint
 402 get_max_dec_frame_buffering(GstH264SPS *sps)
 403 {
 404     guint max_dec_frame_buffering, MaxDpbMbs, PicSizeMbs;
 405
 406     /* Table A-1 - Level limits */
 407     switch (sps->level_idc) {
 408     case 10: MaxDpbMbs = 396;    break;
 409     case 11: MaxDpbMbs = 900;    break;
 410     case 12: MaxDpbMbs = 2376;   break;
 411     case 13: MaxDpbMbs = 2376;   break;
 412     case 20: MaxDpbMbs = 2376;   break;
 413     case 21: MaxDpbMbs = 4752;   break;
 414     case 22: MaxDpbMbs = 8100;   break;
 415     case 30: MaxDpbMbs = 8100;   break;
 416     case 31: MaxDpbMbs = 18000;  break;
 417     case 32: MaxDpbMbs = 20480;  break;
 418     case 40: MaxDpbMbs = 32768;  break;
 419     case 41: MaxDpbMbs = 32768;  break;
 420     case 42: MaxDpbMbs = 34816;  break;
 421     case 50: MaxDpbMbs = 110400; break;
 422     case 51: MaxDpbMbs = 184320; break;
 423     default:
 424         g_assert(0 && "unhandled level");
 425         break;
 426     }
 427
 428     PicSizeMbs = ((sps->pic_width_in_mbs_minus1 + 1) *
 429                   (sps->pic_height_in_map_units_minus1 + 1) *
 430                   (sps->frame_mbs_only_flag ? 1 : 2));
 431     max_dec_frame_buffering = MaxDpbMbs / PicSizeMbs;
 432
 433     /* VUI parameters */
 434     if (sps->vui_parameters_present_flag) {
 435         GstH264VUIParams * const vui_params = &sps->vui_parameters;
 436         if (vui_params->bitstream_restriction_flag)
 437             max_dec_frame_buffering = vui_params->max_dec_frame_buffering;
 438         else {
 439             switch (sps->profile_idc) {
 440             case 44:  // CAVLC 4:4:4 Intra profile
 441             case GST_H264_PROFILE_SCALABLE_HIGH:
 442             case GST_H264_PROFILE_HIGH:
 443             case GST_H264_PROFILE_HIGH10:
 444             case GST_H264_PROFILE_HIGH_422:
 445             case GST_H264_PROFILE_HIGH_444:
 446                 if (sps->constraint_set3_flag)
 447                     max_dec_frame_buffering = 0;
 448                 break;
 449             }
 450         }
 451     }
 452
 453     if (max_dec_frame_buffering > 16)
 454         max_dec_frame_buffering = 16;
 455     else if (max_dec_frame_buffering < sps->num_ref_frames)
 456         max_dec_frame_buffering = sps->num_ref_frames;
 457     return MAX(1, max_dec_frame_buffering);
 458 }
 459
 460 static void
 461 array_remove_index_fast(void *array, guint *array_length_ptr, guint index)
 462 {
 463     gpointer * const entries = array;
 464     guint num_entries = *array_length_ptr;
 465
 466     g_return_if_fail(index < num_entries);
 467
 468     if (index != --num_entries)
 469         entries[index] = entries[num_entries];
 470     entries[num_entries] = NULL;
 471     *array_length_ptr = num_entries;
 472 }
 473
 474 #if 1
 475 static inline void
 476 array_remove_index(void *array, guint *array_length_ptr, guint index)
 477 {
 478     array_remove_index_fast(array, array_length_ptr, index);
 479 }
 480 #else
 481 static void
 482 array_remove_index(void *array, guint *array_length_ptr, guint index)
 483 {
 484     gpointer * const entries = array;
 485     const guint num_entries = *array_length_ptr - 1;
 486     guint i;
 487
 488     g_return_if_fail(index <= num_entries);
 489
 490     for (i = index; i < num_entries; i++)
 491         entries[i] = entries[i + 1];
 492     entries[num_entries] = NULL;
 493     *array_length_ptr = num_entries;
 494 }
 495 #endif
 496
 497 #define ARRAY_REMOVE_INDEX(array, index) \
 498     array_remove_index(array, &array##_count, index)
 499
 500 static void
 501 dpb_remove_index(GstVaapiDecoderH264 *decoder, guint index)
 502 {
 503     GstVaapiDecoderH264Private * const priv = decoder->priv;
 504     guint i, num_frames = --priv->dpb_count;
 505
 506     if (USE_STRICT_DPB_ORDERING) {
 507         for (i = index; i < num_frames; i++)
 508             gst_vaapi_frame_store_replace(&priv->dpb[i], priv->dpb[i + 1]);
 509     }
 510     else if (index != num_frames)
 511         gst_vaapi_frame_store_replace(&priv->dpb[index], priv->dpb[num_frames]);
 512     gst_vaapi_frame_store_replace(&priv->dpb[num_frames], NULL);
 513 }
 514
 515 static gboolean
 516 dpb_output(
 517     GstVaapiDecoderH264 *decoder,
 518     GstVaapiFrameStore  *fs,
 519     GstVaapiPictureH264 *picture
 520 )
 521 {
 522     picture->output_needed = FALSE;
 523
 524     if (fs) {
 525         if (--fs->output_needed > 0)
 526             return TRUE;
 527         picture = fs->buffers[0];
 528     }
 529
 530     /* XXX: update cropping rectangle */
 531     return gst_vaapi_picture_output(GST_VAAPI_PICTURE_CAST(picture));
 532 }
 533
 534 static inline void
 535 dpb_evict(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture, guint i)
 536 {
 537     GstVaapiFrameStore * const fs = decoder->priv->dpb[i];
 538
 539     if (!fs->output_needed && !gst_vaapi_frame_store_has_reference(fs))
 540         dpb_remove_index(decoder, i);
 541 }
 542
 543 static gboolean
 544 dpb_bump(GstVaapiDecoderH264 *decoder)
 545 {
 546     GstVaapiDecoderH264Private * const priv = decoder->priv;
 547     GstVaapiPictureH264 *found_picture = NULL;
 548     guint i, j, found_index;
 549     gboolean success;
 550
 551     for (i = 0; i < priv->dpb_count; i++) {
 552         GstVaapiFrameStore * const fs = priv->dpb[i];
 553         if (!fs->output_needed)
 554             continue;
 555         for (j = 0; j < fs->num_buffers; j++) {
 556             GstVaapiPictureH264 * const picture = fs->buffers[j];
 557             if (!picture->output_needed)
 558                 continue;
 559             if (!found_picture || found_picture->base.poc > picture->base.poc)
 560                 found_picture = picture, found_index = i;
 561         }
 562     }
 563     if (!found_picture)
 564         return FALSE;
 565
 566     success = dpb_output(decoder, priv->dpb[found_index], found_picture);
 567     dpb_evict(decoder, found_picture, found_index);
 568     return success;
 569 }
 570
 571 static void
 572 dpb_clear(GstVaapiDecoderH264 *decoder)
 573 {
 574     GstVaapiDecoderH264Private * const priv = decoder->priv;
 575     guint i;
 576
 577     for (i = 0; i < priv->dpb_count; i++)
 578         gst_vaapi_frame_store_replace(&priv->dpb[i], NULL);
 579     priv->dpb_count = 0;
 580
 581     gst_vaapi_frame_store_replace(&priv->prev_frame, NULL);
 582 }
 583
 584 static void
 585 dpb_flush(GstVaapiDecoderH264 *decoder)
 586 {
 587     while (dpb_bump(decoder))
 588         ;
 589     dpb_clear(decoder);
 590 }
 591
 592 static gboolean
 593 dpb_add(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
 594 {
 595     GstVaapiDecoderH264Private * const priv = decoder->priv;
 596     GstVaapiFrameStore *fs;
 597     guint i, j;
 598
 599     // Remove all unused pictures
 600     if (!GST_VAAPI_PICTURE_IS_IDR(picture)) {
 601         i = 0;
 602         while (i < priv->dpb_count) {
 603             GstVaapiFrameStore * const fs = priv->dpb[i];
 604             if (!fs->output_needed && !gst_vaapi_frame_store_has_reference(fs))
 605                 dpb_remove_index(decoder, i);
 606             else
 607                 i++;
 608         }
 609     }
 610
 611     // Check if picture is the second field and the first field is still in DPB
 612     fs = priv->prev_frame;
 613     if (fs && !gst_vaapi_frame_store_has_frame(fs))
 614         return gst_vaapi_frame_store_add(fs, picture);
 615
 616     // Create new frame store, and split fields if necessary
 617     fs = gst_vaapi_frame_store_new(picture);
 618     if (!fs)
 619         return FALSE;
 620     gst_vaapi_frame_store_replace(&priv->prev_frame, fs);
 621     gst_vaapi_frame_store_unref(fs);
 622
 623     if (!priv->progressive_sequence && gst_vaapi_frame_store_has_frame(fs)) {
 624         if (!gst_vaapi_frame_store_split_fields(fs))
 625             return FALSE;
 626     }
 627
 628     // C.4.5.1 - Storage and marking of a reference decoded picture into the DPB
 629     if (GST_VAAPI_PICTURE_IS_REFERENCE(picture)) {
 630         while (priv->dpb_count == priv->dpb_size) {
 631             if (!dpb_bump(decoder))
 632                 return FALSE;
 633         }
 634         gst_vaapi_frame_store_replace(&priv->dpb[priv->dpb_count++], fs);
 635         if (picture->output_flag) {
 636             picture->output_needed = TRUE;
 637             fs->output_needed++;
 638         }
 639     }
 640
 641     // C.4.5.2 - Storage and marking of a non-reference decoded picture into the DPB
 642     else {
 643         if (!picture->output_flag)
 644             return TRUE;
 645         while (priv->dpb_count == priv->dpb_size) {
 646             gboolean found_picture = FALSE;
 647             for (i = 0; !found_picture && i < priv->dpb_count; i++) {
 648                 GstVaapiFrameStore * const fs = priv->dpb[i];
 649                 if (!fs->output_needed)
 650                     continue;
 651                 for (j = 0; !found_picture && j < fs->num_buffers; j++)
 652                     found_picture = fs->buffers[j]->output_needed &&
 653                         fs->buffers[j]->base.poc < picture->base.poc;
 654             }
 655             if (!found_picture)
 656                 return dpb_output(decoder, NULL, picture);
 657             if (!dpb_bump(decoder))
 658                 return FALSE;
 659         }
 660         gst_vaapi_frame_store_replace(&priv->dpb[priv->dpb_count++], fs);
 661         picture->output_needed = TRUE;
 662         fs->output_needed++;
 663     }
 664     return TRUE;
 665 }
 666
 667 static inline void
 668 dpb_reset(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 669 {
 670     GstVaapiDecoderH264Private * const priv = decoder->priv;
 671
 672     priv->dpb_size = get_max_dec_frame_buffering(sps);
 673     GST_DEBUG("DPB size %u", priv->dpb_size);
 674 }
 675
 676 static GstVaapiDecoderStatus
 677 get_status(GstH264ParserResult result)
 678 {
 679     GstVaapiDecoderStatus status;
 680
 681     switch (result) {
 682     case GST_H264_PARSER_OK:
 683         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
 684         break;
 685     case GST_H264_PARSER_NO_NAL_END:
 686         status = GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
 687         break;
 688     case GST_H264_PARSER_ERROR:
 689         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
 690         break;
 691     default:
 692         status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 693         break;
 694     }
 695     return status;
 696 }
 697
 698 static void
 699 gst_vaapi_decoder_h264_close(GstVaapiDecoderH264 *decoder)
 700 {
 701     GstVaapiDecoderH264Private * const priv = decoder->priv;
 702
 703     gst_vaapi_picture_replace(&priv->current_picture, NULL);
 704     gst_vaapi_parser_info_h264_replace(&priv->prev_slice_pi, NULL);
 705
 706     dpb_clear(decoder);
 707
 708     if (priv->parser) {
 709         gst_h264_nal_parser_free(priv->parser);
 710         priv->parser = NULL;
 711     }
 712 }
 713
 714 static gboolean
 715 gst_vaapi_decoder_h264_open(GstVaapiDecoderH264 *decoder)
 716 {
 717     GstVaapiDecoderH264Private * const priv = decoder->priv;
 718
 719     gst_vaapi_decoder_h264_close(decoder);
 720
 721     priv->parser = gst_h264_nal_parser_new();
 722     if (!priv->parser)
 723         return FALSE;
 724     return TRUE;
 725 }
 726
 727 static void
 728 gst_vaapi_decoder_h264_destroy(GstVaapiDecoderH264 *decoder)
 729 {
 730     gst_vaapi_decoder_h264_close(decoder);
 731 }
 732
 733 static gboolean
 734 gst_vaapi_decoder_h264_create(GstVaapiDecoderH264 *decoder)
 735 {
 736     if (!GST_VAAPI_DECODER_CODEC(decoder))
 737         return FALSE;
 738     return TRUE;
 739 }
 740
 741 static guint
 742 h264_get_profile(GstH264SPS *sps)
 743 {
 744     guint profile = 0;
 745
 746     switch (sps->profile_idc) {
 747     case GST_H264_PROFILE_BASELINE:
 748         profile = GST_VAAPI_PROFILE_H264_BASELINE;
 749         break;
 750     case GST_H264_PROFILE_MAIN:
 751         profile = GST_VAAPI_PROFILE_H264_MAIN;
 752         break;
 753     case GST_H264_PROFILE_HIGH:
 754         profile = GST_VAAPI_PROFILE_H264_HIGH;
 755         break;
 756     }
 757     return profile;
 758 }
 759
 760 static guint
 761 h264_get_chroma_type(GstH264SPS *sps)
 762 {
 763     guint chroma_type = 0;
 764
 765     switch (sps->chroma_format_idc) {
 766     case 1:
 767         chroma_type = GST_VAAPI_CHROMA_TYPE_YUV420;
 768         break;
 769     case 2:
 770         chroma_type = GST_VAAPI_CHROMA_TYPE_YUV422;
 771         break;
 772     case 3:
 773         if (!sps->separate_colour_plane_flag)
 774             chroma_type = GST_VAAPI_CHROMA_TYPE_YUV444;
 775         break;
 776     }
 777     return chroma_type;
 778 }
 779
 780 static GstVaapiProfile
 781 get_profile(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 782 {
 783     GstVaapiDecoderH264Private * const priv = decoder->priv;
 784     GstVaapiDisplay * const display = GST_VAAPI_DECODER_DISPLAY(decoder);
 785     GstVaapiProfile profile, profiles[2];
 786     guint i, n_profiles = 0;
 787
 788     profile = h264_get_profile(sps);
 789     if (!profile)
 790         return GST_VAAPI_PROFILE_UNKNOWN;
 791
 792     profiles[n_profiles++] = profile;
 793     switch (profile) {
 794     case GST_VAAPI_PROFILE_H264_MAIN:
 795         profiles[n_profiles++] = GST_VAAPI_PROFILE_H264_HIGH;
 796         break;
 797     default:
 798         break;
 799     }
 800
 801     /* If the preferred profile (profiles[0]) matches one that we already
 802        found, then just return it now instead of searching for it again */
 803     if (profiles[0] == priv->profile)
 804         return priv->profile;
 805
 806     for (i = 0; i < n_profiles; i++) {
 807         if (gst_vaapi_display_has_decoder(display, profiles[i], priv->entrypoint))
 808             return profiles[i];
 809     }
 810     return GST_VAAPI_PROFILE_UNKNOWN;
 811 }
 812
 813 static GstVaapiDecoderStatus
 814 ensure_context(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 815 {
 816     GstVaapiDecoder * const base_decoder = GST_VAAPI_DECODER_CAST(decoder);
 817     GstVaapiDecoderH264Private * const priv = decoder->priv;
 818     GstVaapiContextInfo info;
 819     GstVaapiProfile profile;
 820     GstVaapiChromaType chroma_type;
 821     gboolean reset_context = FALSE;
 822     guint mb_width, mb_height;
 823
 824     profile = get_profile(decoder, sps);
 825     if (!profile) {
 826         GST_ERROR("unsupported profile_idc %u", sps->profile_idc);
 827         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_PROFILE;
 828     }
 829
 830     if (priv->profile != profile) {
 831         GST_DEBUG("profile changed");
 832         reset_context = TRUE;
 833         priv->profile = profile;
 834     }
 835
 836     chroma_type = h264_get_chroma_type(sps);
 837     if (!chroma_type || chroma_type != GST_VAAPI_CHROMA_TYPE_YUV420) {
 838         GST_ERROR("unsupported chroma_format_idc %u", sps->chroma_format_idc);
 839         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;
 840     }
 841
 842     if (priv->chroma_type != chroma_type) {
 843         GST_DEBUG("chroma format changed");
 844         reset_context     = TRUE;
 845         priv->chroma_type = chroma_type;
 846     }
 847
 848     mb_width  = sps->pic_width_in_mbs_minus1 + 1;
 849     mb_height = (sps->pic_height_in_map_units_minus1 + 1) <<
 850         !sps->frame_mbs_only_flag;
 851     if (priv->mb_width != mb_width || priv->mb_height != mb_height) {
 852         GST_DEBUG("size changed");
 853         reset_context   = TRUE;
 854         priv->mb_width  = mb_width;
 855         priv->mb_height = mb_height;
 856     }
 857
 858     priv->progressive_sequence = sps->frame_mbs_only_flag;
 859 #if 0
 860     /* XXX: we only output complete frames for now */
 861     gst_vaapi_decoder_set_interlaced(base_decoder, !priv->progressive_sequence);
 862 #endif
 863
 864     gst_vaapi_decoder_set_pixel_aspect_ratio(
 865         base_decoder,
 866         sps->vui_parameters.par_n,
 867         sps->vui_parameters.par_d
 868     );
 869
 870     if (!reset_context && priv->has_context)
 871         return GST_VAAPI_DECODER_STATUS_SUCCESS;
 872
 873     /* XXX: fix surface size when cropping is implemented */
 874     info.profile    = priv->profile;
 875     info.entrypoint = priv->entrypoint;
 876     info.width      = sps->width;
 877     info.height     = sps->height;
 878     info.ref_frames = get_max_dec_frame_buffering(sps);
 879
 880     if (!gst_vaapi_decoder_ensure_context(GST_VAAPI_DECODER(decoder), &info))
 881         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 882     priv->has_context = TRUE;
 883
 884     /* Reset DPB */
 885     dpb_reset(decoder, sps);
 886     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 887 }
 888
 889 static void
 890 fill_iq_matrix_4x4(VAIQMatrixBufferH264 *iq_matrix, const GstH264PPS *pps)
 891 {
 892     const guint8 (* const ScalingList4x4)[6][16] = &pps->scaling_lists_4x4;
 893     guint i, j;
 894
 895     /* There are always 6 4x4 scaling lists */
 896     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList4x4) == 6);
 897     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList4x4[0]) == 16);
 898
 899     if (sizeof(iq_matrix->ScalingList4x4[0][0]) == 1)
 900         memcpy(iq_matrix->ScalingList4x4, *ScalingList4x4,
 901                sizeof(iq_matrix->ScalingList4x4));
 902     else {
 903         for (i = 0; i < G_N_ELEMENTS(iq_matrix->ScalingList4x4); i++) {
 904             for (j = 0; j < G_N_ELEMENTS(iq_matrix->ScalingList4x4[i]); j++)
 905                 iq_matrix->ScalingList4x4[i][j] = (*ScalingList4x4)[i][j];
 906         }
 907     }
 908 }
 909
 910 static void
 911 fill_iq_matrix_8x8(VAIQMatrixBufferH264 *iq_matrix, const GstH264PPS *pps)
 912 {
 913     const guint8 (* const ScalingList8x8)[6][64] = &pps->scaling_lists_8x8;
 914     const GstH264SPS * const sps = pps->sequence;
 915     guint i, j, n;
 916
 917     /* If chroma_format_idc != 3, there are up to 2 8x8 scaling lists */
 918     if (!pps->transform_8x8_mode_flag)
 919         return;
 920
 921     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList8x8) >= 2);
 922     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList8x8[0]) == 64);
 923
 924     if (sizeof(iq_matrix->ScalingList8x8[0][0]) == 1)
 925         memcpy(iq_matrix->ScalingList8x8, *ScalingList8x8,
 926                sizeof(iq_matrix->ScalingList8x8));
 927     else {
 928         n = (sps->chroma_format_idc != 3) ? 2 : 6;
 929         for (i = 0; i < n; i++) {
 930             for (j = 0; j < G_N_ELEMENTS(iq_matrix->ScalingList8x8[i]); j++)
 931                 iq_matrix->ScalingList8x8[i][j] = (*ScalingList8x8)[i][j];
 932         }
 933     }
 934 }
 935
 936 static GstVaapiDecoderStatus
 937 ensure_quant_matrix(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
 938 {
 939     GstVaapiPicture * const base_picture = &picture->base;
 940     GstH264PPS * const pps = picture->pps;
 941     GstH264SPS * const sps = pps->sequence;
 942     VAIQMatrixBufferH264 *iq_matrix;
 943
 944     base_picture->iq_matrix = GST_VAAPI_IQ_MATRIX_NEW(H264, decoder);
 945     if (!base_picture->iq_matrix) {
 946         GST_ERROR("failed to allocate IQ matrix");
 947         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
 948     }
 949     iq_matrix = base_picture->iq_matrix->param;
 950
 951     /* XXX: we can only support 4:2:0 or 4:2:2 since ScalingLists8x8[]
 952        is not large enough to hold lists for 4:4:4 */
 953     if (sps->chroma_format_idc == 3)
 954         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;
 955
 956     fill_iq_matrix_4x4(iq_matrix, pps);
 957     fill_iq_matrix_8x8(iq_matrix, pps);
 958
 959     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 960 }
 961
 962 static GstVaapiDecoderStatus
 963 decode_current_picture(GstVaapiDecoderH264 *decoder)
 964 {
 965     GstVaapiDecoderH264Private * const priv = decoder->priv;
 966     GstVaapiPictureH264 * const picture = priv->current_picture;
 967
 968     if (!picture)
 969         return GST_VAAPI_DECODER_STATUS_SUCCESS;
 970
 971     if (!exec_ref_pic_marking(decoder, picture))
 972         goto error;
 973     if (!dpb_add(decoder, picture))
 974         goto error;
 975     if (!gst_vaapi_picture_decode(GST_VAAPI_PICTURE_CAST(picture)))
 976         goto error;
 977     if (priv->prev_frame && gst_vaapi_frame_store_has_frame(priv->prev_frame))
 978         gst_vaapi_picture_replace(&priv->current_picture, NULL);
 979     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 980
 981 error:
 982     /* XXX: fix for cases where first field failed to be decoded */
 983     gst_vaapi_picture_replace(&priv->current_picture, NULL);
 984     return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 985 }
 986
 987 static GstVaapiDecoderStatus
 988 parse_sps(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnit *unit)
 989 {
 990     GstVaapiDecoderH264Private * const priv = decoder->priv;
 991     GstVaapiParserInfoH264 * const pi = unit->parsed_info;
 992     GstH264SPS * const sps = &pi->data.sps;
 993     GstH264ParserResult result;
 994
 995     GST_DEBUG("parse SPS");
 996
 997     /* Variables that don't have inferred values per the H.264
 998        standard but that should get a default value anyway */
 999     sps->log2_max_pic_order_cnt_lsb_minus4 = 0;
1000
1001     result = gst_h264_parser_parse_sps(priv->parser, &pi->nalu, sps, TRUE);
1002     if (result != GST_H264_PARSER_OK)
1003         return get_status(result);
1004
1005     priv->got_sps = TRUE;
1006     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1007 }
1008
1009 static GstVaapiDecoderStatus
1010 parse_pps(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnit *unit)
1011 {
1012     GstVaapiDecoderH264Private * const priv = decoder->priv;
1013     GstVaapiParserInfoH264 * const pi = unit->parsed_info;
1014     GstH264PPS * const pps = &pi->data.pps;
1015     GstH264ParserResult result;
1016
1017     GST_DEBUG("parse PPS");
1018
1019     /* Variables that don't have inferred values per the H.264
1020        standard but that should get a default value anyway */
1021     pps->slice_group_map_type = 0;
1022     pps->slice_group_change_rate_minus1 = 0;
1023
1024     result = gst_h264_parser_parse_pps(priv->parser, &pi->nalu, pps);
1025     if (result != GST_H264_PARSER_OK)
1026         return get_status(result);
1027
1028     priv->got_pps = TRUE;
1029     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1030 }
1031
1032 static GstVaapiDecoderStatus
1033 parse_sei(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnit *unit)
1034 {
1035     GstVaapiDecoderH264Private * const priv = decoder->priv;
1036     GstVaapiParserInfoH264 * const pi = unit->parsed_info;
1037     GstH264SEIMessage sei;
1038     GstH264ParserResult result;
1039
1040     GST_DEBUG("parse SEI");
1041
1042     memset(&sei, 0, sizeof(sei));
1043     result = gst_h264_parser_parse_sei(priv->parser, &pi->nalu, &sei);
1044     if (result != GST_H264_PARSER_OK) {
1045         GST_WARNING("failed to parse SEI, payload type:%d", sei.payloadType);
1046         return get_status(result);
1047     }
1048
1049     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1050 }
1051
1052 static GstVaapiDecoderStatus
1053 parse_slice(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnit *unit)
1054 {
1055     GstVaapiDecoderH264Private * const priv = decoder->priv;
1056     GstVaapiParserInfoH264 * const pi = unit->parsed_info;
1057     GstH264SliceHdr * const slice_hdr = &pi->data.slice_hdr;
1058     GstH264ParserResult result;
1059
1060     GST_DEBUG("parse slice");
1061
1062     /* Variables that don't have inferred values per the H.264
1063        standard but that should get a default value anyway */
1064     slice_hdr->cabac_init_idc = 0;
1065     slice_hdr->direct_spatial_mv_pred_flag = 0;
1066
1067     result = gst_h264_parser_parse_slice_hdr(priv->parser, &pi->nalu,
1068         slice_hdr, TRUE, TRUE);
1069     if (result != GST_H264_PARSER_OK)
1070         return get_status(result);
1071
1072     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1073 }
1074
1075 static GstVaapiDecoderStatus
1076 decode_sequence_end(GstVaapiDecoderH264 *decoder)
1077 {
1078     GstVaapiDecoderStatus status;
1079
1080     GST_DEBUG("decode sequence-end");
1081
1082     status = decode_current_picture(decoder);
1083     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
1084         return status;
1085
1086     dpb_flush(decoder);
1087     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1088 }
1089
1090 /* 8.2.1.1 - Decoding process for picture order count type 0 */
1091 static void
1092 init_picture_poc_0(
1093     GstVaapiDecoderH264 *decoder,
1094     GstVaapiPictureH264 *picture,
1095     GstH264SliceHdr     *slice_hdr
1096 )
1097 {
1098     GstVaapiDecoderH264Private * const priv = decoder->priv;
1099     GstH264PPS * const pps = slice_hdr->pps;
1100     GstH264SPS * const sps = pps->sequence;
1101     const gint32 MaxPicOrderCntLsb = 1 << (sps->log2_max_pic_order_cnt_lsb_minus4 + 4);
1102     gint32 temp_poc;
1103
1104     GST_DEBUG("decode picture order count type 0");
1105
1106     if (GST_VAAPI_PICTURE_IS_IDR(picture)) {
1107         priv->prev_poc_msb = 0;
1108         priv->prev_poc_lsb = 0;
1109     }
1110     else if (priv->prev_pic_has_mmco5) {
1111         priv->prev_poc_msb = 0;
1112         priv->prev_poc_lsb =
1113             (priv->prev_pic_structure == GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD ?
1114              0 : priv->field_poc[TOP_FIELD]);
1115     }
1116     else {
1117         priv->prev_poc_msb = priv->poc_msb;
1118         priv->prev_poc_lsb = priv->poc_lsb;
1119     }
1120
1121     // (8-3)
1122     priv->poc_lsb = slice_hdr->pic_order_cnt_lsb;
1123     if (priv->poc_lsb < priv->prev_poc_lsb &&
1124         (priv->prev_poc_lsb - priv->poc_lsb) >= (MaxPicOrderCntLsb / 2))
1125         priv->poc_msb = priv->prev_poc_msb + MaxPicOrderCntLsb;
1126     else if (priv->poc_lsb > priv->prev_poc_lsb &&
1127              (priv->poc_lsb - priv->prev_poc_lsb) > (MaxPicOrderCntLsb / 2))
1128         priv->poc_msb = priv->prev_poc_msb - MaxPicOrderCntLsb;
1129     else
1130         priv->poc_msb = priv->prev_poc_msb;
1131
1132     temp_poc = priv->poc_msb + priv->poc_lsb;
1133     switch (picture->structure) {
1134     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
1135         // (8-4, 8-5)
1136         priv->field_poc[TOP_FIELD] = temp_poc;
1137         priv->field_poc[BOTTOM_FIELD] = temp_poc +
1138             slice_hdr->delta_pic_order_cnt_bottom;
1139         break;
1140     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
1141         // (8-4)
1142         priv->field_poc[TOP_FIELD] = temp_poc;
1143         break;
1144     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
1145         // (8-5)
1146         priv->field_poc[BOTTOM_FIELD] = temp_poc;
1147         break;
1148     }
1149 }
1150
1151 /* 8.2.1.2 - Decoding process for picture order count type 1 */
1152 static void
1153 init_picture_poc_1(
1154     GstVaapiDecoderH264 *decoder,
1155     GstVaapiPictureH264 *picture,
1156     GstH264SliceHdr     *slice_hdr
1157 )
1158 {
1159     GstVaapiDecoderH264Private * const priv = decoder->priv;
1160     GstH264PPS * const pps = slice_hdr->pps;
1161     GstH264SPS * const sps = pps->sequence;
1162     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1163     gint32 prev_frame_num_offset, abs_frame_num, expected_poc;
1164     guint i;
1165
1166     GST_DEBUG("decode picture order count type 1");
1167
1168     if (priv->prev_pic_has_mmco5)
1169         prev_frame_num_offset = 0;
1170     else
1171         prev_frame_num_offset = priv->frame_num_offset;
1172
1173     // (8-6)
1174     if (GST_VAAPI_PICTURE_IS_IDR(picture))
1175         priv->frame_num_offset = 0;
1176     else if (priv->prev_frame_num > priv->frame_num)
1177         priv->frame_num_offset = prev_frame_num_offset + MaxFrameNum;
1178     else
1179         priv->frame_num_offset = prev_frame_num_offset;
1180
1181     // (8-7)
1182     if (sps->num_ref_frames_in_pic_order_cnt_cycle != 0)
1183         abs_frame_num = priv->frame_num_offset + priv->frame_num;
1184     else
1185         abs_frame_num = 0;
1186     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture) && abs_frame_num > 0)
1187         abs_frame_num = abs_frame_num - 1;
1188
1189     if (abs_frame_num > 0) {
1190         gint32 expected_delta_per_poc_cycle;
1191         gint32 poc_cycle_cnt, frame_num_in_poc_cycle;
1192
1193         expected_delta_per_poc_cycle = 0;
1194         for (i = 0; i < sps->num_ref_frames_in_pic_order_cnt_cycle; i++)
1195             expected_delta_per_poc_cycle += sps->offset_for_ref_frame[i];
1196
1197         // (8-8)
1198         poc_cycle_cnt = (abs_frame_num - 1) /
1199             sps->num_ref_frames_in_pic_order_cnt_cycle;
1200         frame_num_in_poc_cycle = (abs_frame_num - 1) %
1201             sps->num_ref_frames_in_pic_order_cnt_cycle;
1202
1203         // (8-9)
1204         expected_poc = poc_cycle_cnt * expected_delta_per_poc_cycle;
1205         for (i = 0; i <= frame_num_in_poc_cycle; i++)
1206             expected_poc += sps->offset_for_ref_frame[i];
1207     }
1208     else
1209         expected_poc = 0;
1210     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
1211         expected_poc += sps->offset_for_non_ref_pic;
1212
1213     // (8-10)
1214     switch (picture->structure) {
1215     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
1216         priv->field_poc[TOP_FIELD] = expected_poc +
1217             slice_hdr->delta_pic_order_cnt[0];
1218         priv->field_poc[BOTTOM_FIELD] = priv->field_poc[TOP_FIELD] +
1219             sps->offset_for_top_to_bottom_field +
1220             slice_hdr->delta_pic_order_cnt[1];
1221         break;
1222     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
1223         priv->field_poc[TOP_FIELD] = expected_poc +
1224             slice_hdr->delta_pic_order_cnt[0];
1225         break;
1226     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
1227         priv->field_poc[BOTTOM_FIELD] = expected_poc +
1228             sps->offset_for_top_to_bottom_field +
1229             slice_hdr->delta_pic_order_cnt[0];
1230         break;
1231     }
1232 }
1233
1234 /* 8.2.1.3 - Decoding process for picture order count type 2 */
1235 static void
1236 init_picture_poc_2(
1237     GstVaapiDecoderH264 *decoder,
1238     GstVaapiPictureH264 *picture,
1239     GstH264SliceHdr     *slice_hdr
1240 )
1241 {
1242     GstVaapiDecoderH264Private * const priv = decoder->priv;
1243     GstH264PPS * const pps = slice_hdr->pps;
1244     GstH264SPS * const sps = pps->sequence;
1245     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1246     gint32 prev_frame_num_offset, temp_poc;
1247
1248     GST_DEBUG("decode picture order count type 2");
1249
1250     if (priv->prev_pic_has_mmco5)
1251         prev_frame_num_offset = 0;
1252     else
1253         prev_frame_num_offset = priv->frame_num_offset;
1254
1255     // (8-11)
1256     if (GST_VAAPI_PICTURE_IS_IDR(picture))
1257         priv->frame_num_offset = 0;
1258     else if (priv->prev_frame_num > priv->frame_num)
1259         priv->frame_num_offset = prev_frame_num_offset + MaxFrameNum;
1260     else
1261         priv->frame_num_offset = prev_frame_num_offset;
1262
1263     // (8-12)
1264     if (GST_VAAPI_PICTURE_IS_IDR(picture))
1265         temp_poc = 0;
1266     else if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
1267         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num) - 1;
1268     else
1269         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num);
1270
1271     // (8-13)
1272     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
1273         priv->field_poc[TOP_FIELD] = temp_poc;
1274     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
1275         priv->field_poc[BOTTOM_FIELD] = temp_poc;
1276 }
1277
1278 /* 8.2.1 - Decoding process for picture order count */
1279 static void
1280 init_picture_poc(
1281     GstVaapiDecoderH264 *decoder,
1282     GstVaapiPictureH264 *picture,
1283     GstH264SliceHdr     *slice_hdr
1284 )
1285 {
1286     GstVaapiDecoderH264Private * const priv = decoder->priv;
1287     GstH264PPS * const pps = slice_hdr->pps;
1288     GstH264SPS * const sps = pps->sequence;
1289
1290     switch (sps->pic_order_cnt_type) {
1291     case 0:
1292         init_picture_poc_0(decoder, picture, slice_hdr);
1293         break;
1294     case 1:
1295         init_picture_poc_1(decoder, picture, slice_hdr);
1296         break;
1297     case 2:
1298         init_picture_poc_2(decoder, picture, slice_hdr);
1299         break;
1300     }
1301
1302     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
1303         picture->field_poc[TOP_FIELD] = priv->field_poc[TOP_FIELD];
1304     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
1305         picture->field_poc[BOTTOM_FIELD] = priv->field_poc[BOTTOM_FIELD];
1306     picture->base.poc = MIN(picture->field_poc[0], picture->field_poc[1]);
1307 }
1308
1309 static int
1310 compare_picture_pic_num_dec(const void *a, const void *b)
1311 {
1312     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1313     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1314
1315     return picB->pic_num - picA->pic_num;
1316 }
1317
1318 static int
1319 compare_picture_long_term_pic_num_inc(const void *a, const void *b)
1320 {
1321     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1322     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1323
1324     return picA->long_term_pic_num - picB->long_term_pic_num;
1325 }
1326
1327 static int
1328 compare_picture_poc_dec(const void *a, const void *b)
1329 {
1330     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1331     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1332
1333     return picB->base.poc - picA->base.poc;
1334 }
1335
1336 static int
1337 compare_picture_poc_inc(const void *a, const void *b)
1338 {
1339     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1340     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1341
1342     return picA->base.poc - picB->base.poc;
1343 }
1344
1345 static int
1346 compare_picture_frame_num_wrap_dec(const void *a, const void *b)
1347 {
1348     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1349     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1350
1351     return picB->frame_num_wrap - picA->frame_num_wrap;
1352 }
1353
1354 static int
1355 compare_picture_long_term_frame_idx_inc(const void *a, const void *b)
1356 {
1357     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1358     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1359
1360     return picA->long_term_frame_idx - picB->long_term_frame_idx;
1361 }
1362
1363 /* 8.2.4.1 - Decoding process for picture numbers */
1364 static void
1365 init_picture_refs_pic_num(
1366     GstVaapiDecoderH264 *decoder,
1367     GstVaapiPictureH264 *picture,
1368     GstH264SliceHdr     *slice_hdr
1369 )
1370 {
1371     GstVaapiDecoderH264Private * const priv = decoder->priv;
1372     GstH264PPS * const pps = slice_hdr->pps;
1373     GstH264SPS * const sps = pps->sequence;
1374     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1375     guint i;
1376
1377     GST_DEBUG("decode picture numbers");
1378
1379     for (i = 0; i < priv->short_ref_count; i++) {
1380         GstVaapiPictureH264 * const pic = priv->short_ref[i];
1381
1382         // (8-27)
1383         if (pic->frame_num > priv->frame_num)
1384             pic->frame_num_wrap = pic->frame_num - MaxFrameNum;
1385         else
1386             pic->frame_num_wrap = pic->frame_num;
1387
1388         // (8-28, 8-30, 8-31)
1389         if (GST_VAAPI_PICTURE_IS_FRAME(picture))
1390             pic->pic_num = pic->frame_num_wrap;
1391         else {
1392             if (pic->structure == picture->structure)
1393                 pic->pic_num = 2 * pic->frame_num_wrap + 1;
1394             else
1395                 pic->pic_num = 2 * pic->frame_num_wrap;
1396         }
1397     }
1398
1399     for (i = 0; i < priv->long_ref_count; i++) {
1400         GstVaapiPictureH264 * const pic = priv->long_ref[i];
1401
1402         // (8-29, 8-32, 8-33)
1403         if (GST_VAAPI_PICTURE_IS_FRAME(picture))
1404             pic->long_term_pic_num = pic->long_term_frame_idx;
1405         else {
1406             if (pic->structure == picture->structure)
1407                 pic->long_term_pic_num = 2 * pic->long_term_frame_idx + 1;
1408             else
1409                 pic->long_term_pic_num = 2 * pic->long_term_frame_idx;
1410         }
1411     }
1412 }
1413
1414 #define SORT_REF_LIST(list, n, compare_func) \
1415     qsort(list, n, sizeof(*(list)), compare_picture_##compare_func)
1416
1417 static void
1418 init_picture_refs_fields_1(
1419     guint                picture_structure,
1420     GstVaapiPictureH264 *RefPicList[32],
1421     guint               *RefPicList_count,
1422     GstVaapiPictureH264 *ref_list[32],
1423     guint                ref_list_count
1424 )
1425 {
1426     guint i, j, n;
1427
1428     i = 0;
1429     j = 0;
1430     n = *RefPicList_count;
1431     do {
1432         g_assert(n < 32);
1433         for (; i < ref_list_count; i++) {
1434             if (ref_list[i]->structure == picture_structure) {
1435                 RefPicList[n++] = ref_list[i++];
1436                 break;
1437             }
1438         }
1439         for (; j < ref_list_count; j++) {
1440             if (ref_list[j]->structure != picture_structure) {
1441                 RefPicList[n++] = ref_list[j++];
1442                 break;
1443             }
1444         }
1445     } while (i < ref_list_count || j < ref_list_count);
1446     *RefPicList_count = n;
1447 }
1448
1449 static inline void
1450 init_picture_refs_fields(
1451     GstVaapiPictureH264 *picture,
1452     GstVaapiPictureH264 *RefPicList[32],
1453     guint               *RefPicList_count,
1454     GstVaapiPictureH264 *short_ref[32],
1455     guint                short_ref_count,
1456     GstVaapiPictureH264 *long_ref[32],
1457     guint                long_ref_count
1458 )
1459 {
1460     guint n = 0;
1461
1462     /* 8.2.4.2.5 - reference picture lists in fields */
1463     init_picture_refs_fields_1(picture->structure, RefPicList, &n,
1464         short_ref, short_ref_count);
1465     init_picture_refs_fields_1(picture->structure, RefPicList, &n,
1466         long_ref, long_ref_count);
1467     *RefPicList_count = n;
1468 }
1469
1470 static void
1471 init_picture_refs_p_slice(
1472     GstVaapiDecoderH264 *decoder,
1473     GstVaapiPictureH264 *picture,
1474     GstH264SliceHdr     *slice_hdr
1475 )
1476 {
1477     GstVaapiDecoderH264Private * const priv = decoder->priv;
1478     GstVaapiPictureH264 **ref_list;
1479     guint i;
1480
1481     GST_DEBUG("decode reference picture list for P and SP slices");
1482
1483     if (GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1484         /* 8.2.4.2.1 - P and SP slices in frames */
1485         if (priv->short_ref_count > 0) {
1486             ref_list = priv->RefPicList0;
1487             for (i = 0; i < priv->short_ref_count; i++)
1488                 ref_list[i] = priv->short_ref[i];
1489             SORT_REF_LIST(ref_list, i, pic_num_dec);
1490             priv->RefPicList0_count += i;
1491         }
1492
1493         if (priv->long_ref_count > 0) {
1494             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1495             for (i = 0; i < priv->long_ref_count; i++)
1496                 ref_list[i] = priv->long_ref[i];
1497             SORT_REF_LIST(ref_list, i, long_term_pic_num_inc);
1498             priv->RefPicList0_count += i;
1499         }
1500     }
1501     else {
1502         /* 8.2.4.2.2 - P and SP slices in fields */
1503         GstVaapiPictureH264 *short_ref[32];
1504         guint short_ref_count = 0;
1505         GstVaapiPictureH264 *long_ref[32];
1506         guint long_ref_count = 0;
1507
1508         if (priv->short_ref_count > 0) {
1509             for (i = 0; i < priv->short_ref_count; i++)
1510                 short_ref[i] = priv->short_ref[i];
1511             SORT_REF_LIST(short_ref, i, frame_num_wrap_dec);
1512             short_ref_count = i;
1513         }
1514
1515         if (priv->long_ref_count > 0) {
1516             for (i = 0; i < priv->long_ref_count; i++)
1517                 long_ref[i] = priv->long_ref[i];
1518             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1519             long_ref_count = i;
1520         }
1521
1522         init_picture_refs_fields(
1523             picture,
1524             priv->RefPicList0, &priv->RefPicList0_count,
1525             short_ref,          short_ref_count,
1526             long_ref,           long_ref_count
1527         );
1528     }
1529 }
1530
1531 static void
1532 init_picture_refs_b_slice(
1533     GstVaapiDecoderH264 *decoder,
1534     GstVaapiPictureH264 *picture,
1535     GstH264SliceHdr     *slice_hdr
1536 )
1537 {
1538     GstVaapiDecoderH264Private * const priv = decoder->priv;
1539     GstVaapiPictureH264 **ref_list;
1540     guint i, n;
1541
1542     GST_DEBUG("decode reference picture list for B slices");
1543
1544     if (GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1545         /* 8.2.4.2.3 - B slices in frames */
1546
1547         /* RefPicList0 */
1548         if (priv->short_ref_count > 0) {
1549             // 1. Short-term references
1550             ref_list = priv->RefPicList0;
1551             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1552                 if (priv->short_ref[i]->base.poc < picture->base.poc)
1553                     ref_list[n++] = priv->short_ref[i];
1554             }
1555             SORT_REF_LIST(ref_list, n, poc_dec);
1556             priv->RefPicList0_count += n;
1557
1558             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1559             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1560                 if (priv->short_ref[i]->base.poc >= picture->base.poc)
1561                     ref_list[n++] = priv->short_ref[i];
1562             }
1563             SORT_REF_LIST(ref_list, n, poc_inc);
1564             priv->RefPicList0_count += n;
1565         }
1566
1567         if (priv->long_ref_count > 0) {
1568             // 2. Long-term references
1569             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1570             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1571                 ref_list[n++] = priv->long_ref[i];
1572             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1573             priv->RefPicList0_count += n;
1574         }
1575
1576         /* RefPicList1 */
1577         if (priv->short_ref_count > 0) {
1578             // 1. Short-term references
1579             ref_list = priv->RefPicList1;
1580             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1581                 if (priv->short_ref[i]->base.poc > picture->base.poc)
1582                     ref_list[n++] = priv->short_ref[i];
1583             }
1584             SORT_REF_LIST(ref_list, n, poc_inc);
1585             priv->RefPicList1_count += n;
1586
1587             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1588             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1589                 if (priv->short_ref[i]->base.poc <= picture->base.poc)
1590                     ref_list[n++] = priv->short_ref[i];
1591             }
1592             SORT_REF_LIST(ref_list, n, poc_dec);
1593             priv->RefPicList1_count += n;
1594         }
1595
1596         if (priv->long_ref_count > 0) {
1597             // 2. Long-term references
1598             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1599             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1600                 ref_list[n++] = priv->long_ref[i];
1601             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1602             priv->RefPicList1_count += n;
1603         }
1604     }
1605     else {
1606         /* 8.2.4.2.4 - B slices in fields */
1607         GstVaapiPictureH264 *short_ref0[32];
1608         guint short_ref0_count = 0;
1609         GstVaapiPictureH264 *short_ref1[32];
1610         guint short_ref1_count = 0;
1611         GstVaapiPictureH264 *long_ref[32];
1612         guint long_ref_count = 0;
1613
1614         /* refFrameList0ShortTerm */
1615         if (priv->short_ref_count > 0) {
1616             ref_list = short_ref0;
1617             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1618                 if (priv->short_ref[i]->base.poc <= picture->base.poc)
1619                     ref_list[n++] = priv->short_ref[i];
1620             }
1621             SORT_REF_LIST(ref_list, n, poc_dec);
1622             short_ref0_count += n;
1623
1624             ref_list = &short_ref0[short_ref0_count];
1625             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1626                 if (priv->short_ref[i]->base.poc > picture->base.poc)
1627                     ref_list[n++] = priv->short_ref[i];
1628             }
1629             SORT_REF_LIST(ref_list, n, poc_inc);
1630             short_ref0_count += n;
1631         }
1632
1633         /* refFrameList1ShortTerm */
1634         if (priv->short_ref_count > 0) {
1635             ref_list = short_ref1;
1636             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1637                 if (priv->short_ref[i]->base.poc > picture->base.poc)
1638                     ref_list[n++] = priv->short_ref[i];
1639             }
1640             SORT_REF_LIST(ref_list, n, poc_inc);
1641             short_ref1_count += n;
1642
1643             ref_list = &short_ref1[short_ref1_count];
1644             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1645                 if (priv->short_ref[i]->base.poc <= picture->base.poc)
1646                     ref_list[n++] = priv->short_ref[i];
1647             }
1648             SORT_REF_LIST(ref_list, n, poc_dec);
1649             short_ref1_count += n;
1650         }
1651
1652         /* refFrameListLongTerm */
1653         if (priv->long_ref_count > 0) {
1654             for (i = 0; i < priv->long_ref_count; i++)
1655                 long_ref[i] = priv->long_ref[i];
1656             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1657             long_ref_count = i;
1658         }
1659
1660         init_picture_refs_fields(
1661             picture,
1662             priv->RefPicList0, &priv->RefPicList0_count,
1663             short_ref0,         short_ref0_count,
1664             long_ref,           long_ref_count
1665         );
1666
1667         init_picture_refs_fields(
1668             picture,
1669             priv->RefPicList1, &priv->RefPicList1_count,
1670             short_ref1,         short_ref1_count,
1671             long_ref,           long_ref_count
1672         );
1673    }
1674
1675     /* Check whether RefPicList1 is identical to RefPicList0, then
1676        swap if necessary */
1677     if (priv->RefPicList1_count > 1 &&
1678         priv->RefPicList1_count == priv->RefPicList0_count &&
1679         memcmp(priv->RefPicList0, priv->RefPicList1,
1680                priv->RefPicList0_count * sizeof(priv->RefPicList0[0])) == 0) {
1681         GstVaapiPictureH264 * const tmp = priv->RefPicList1[0];
1682         priv->RefPicList1[0] = priv->RefPicList1[1];
1683         priv->RefPicList1[1] = tmp;
1684     }
1685 }
1686
1687 #undef SORT_REF_LIST
1688
1689 static gint
1690 find_short_term_reference(GstVaapiDecoderH264 *decoder, gint32 pic_num)
1691 {
1692     GstVaapiDecoderH264Private * const priv = decoder->priv;
1693     guint i;
1694
1695     for (i = 0; i < priv->short_ref_count; i++) {
1696         if (priv->short_ref[i]->pic_num == pic_num)
1697             return i;
1698     }
1699     GST_ERROR("found no short-term reference picture with PicNum = %d",
1700               pic_num);
1701     return -1;
1702 }
1703
1704 static gint
1705 find_long_term_reference(GstVaapiDecoderH264 *decoder, gint32 long_term_pic_num)
1706 {
1707     GstVaapiDecoderH264Private * const priv = decoder->priv;
1708     guint i;
1709
1710     for (i = 0; i < priv->long_ref_count; i++) {
1711         if (priv->long_ref[i]->long_term_pic_num == long_term_pic_num)
1712             return i;
1713     }
1714     GST_ERROR("found no long-term reference picture with LongTermPicNum = %d",
1715               long_term_pic_num);
1716     return -1;
1717 }
1718
1719 static void
1720 exec_picture_refs_modification_1(
1721     GstVaapiDecoderH264           *decoder,
1722     GstVaapiPictureH264           *picture,
1723     GstH264SliceHdr               *slice_hdr,
1724     guint                          list
1725 )
1726 {
1727     GstVaapiDecoderH264Private * const priv = decoder->priv;
1728     GstH264PPS * const pps = slice_hdr->pps;
1729     GstH264SPS * const sps = pps->sequence;
1730     GstH264RefPicListModification *ref_pic_list_modification;
1731     guint num_ref_pic_list_modifications;
1732     GstVaapiPictureH264 **ref_list;
1733     guint *ref_list_count_ptr, ref_list_count, ref_list_idx = 0;
1734     guint i, j, n, num_refs;
1735     gint found_ref_idx;
1736     gint32 MaxPicNum, CurrPicNum, picNumPred;
1737
1738     GST_DEBUG("modification process of reference picture list %u", list);
1739
1740     if (list == 0) {
1741         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l0;
1742         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l0;
1743         ref_list                       = priv->RefPicList0;
1744         ref_list_count_ptr             = &priv->RefPicList0_count;
1745         num_refs                       = slice_hdr->num_ref_idx_l0_active_minus1 + 1;
1746     }
1747     else {
1748         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l1;
1749         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l1;
1750         ref_list                       = priv->RefPicList1;
1751         ref_list_count_ptr             = &priv->RefPicList1_count;
1752         num_refs                       = slice_hdr->num_ref_idx_l1_active_minus1 + 1;
1753     }
1754     ref_list_count = *ref_list_count_ptr;
1755
1756     if (!GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1757         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 5); // 2 * MaxFrameNum
1758         CurrPicNum = 2 * slice_hdr->frame_num + 1;              // 2 * frame_num + 1
1759     }
1760     else {
1761         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 4); // MaxFrameNum
1762         CurrPicNum = slice_hdr->frame_num;                      // frame_num
1763     }
1764
1765     picNumPred = CurrPicNum;
1766
1767     for (i = 0; i < num_ref_pic_list_modifications; i++) {
1768         GstH264RefPicListModification * const l = &ref_pic_list_modification[i];
1769         if (l->modification_of_pic_nums_idc == 3)
1770             break;
1771
1772         /* 8.2.4.3.1 - Short-term reference pictures */
1773         if (l->modification_of_pic_nums_idc == 0 || l->modification_of_pic_nums_idc == 1) {
1774             gint32 abs_diff_pic_num = l->value.abs_diff_pic_num_minus1 + 1;
1775             gint32 picNum, picNumNoWrap;
1776
1777             // (8-34)
1778             if (l->modification_of_pic_nums_idc == 0) {
1779                 picNumNoWrap = picNumPred - abs_diff_pic_num;
1780                 if (picNumNoWrap < 0)
1781                     picNumNoWrap += MaxPicNum;
1782             }
1783
1784             // (8-35)
1785             else {
1786                 picNumNoWrap = picNumPred + abs_diff_pic_num;
1787                 if (picNumNoWrap >= MaxPicNum)
1788                     picNumNoWrap -= MaxPicNum;
1789             }
1790             picNumPred = picNumNoWrap;
1791
1792             // (8-36)
1793             picNum = picNumNoWrap;
1794             if (picNum > CurrPicNum)
1795                 picNum -= MaxPicNum;
1796
1797             // (8-37)
1798             for (j = num_refs; j > ref_list_idx; j--)
1799                 ref_list[j] = ref_list[j - 1];
1800             found_ref_idx = find_short_term_reference(decoder, picNum);
1801             ref_list[ref_list_idx++] =
1802                 found_ref_idx >= 0 ? priv->short_ref[found_ref_idx] : NULL;
1803             n = ref_list_idx;
1804             for (j = ref_list_idx; j <= num_refs; j++) {
1805                 gint32 PicNumF;
1806                 if (!ref_list[j])
1807                     continue;
1808                 PicNumF =
1809                     GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(ref_list[j]) ?
1810                     ref_list[j]->pic_num : MaxPicNum;
1811                 if (PicNumF != picNum)
1812                     ref_list[n++] = ref_list[j];
1813             }
1814         }
1815
1816         /* 8.2.4.3.2 - Long-term reference pictures */
1817         else {
1818
1819             for (j = num_refs; j > ref_list_idx; j--)
1820                 ref_list[j] = ref_list[j - 1];
1821             found_ref_idx =
1822                 find_long_term_reference(decoder, l->value.long_term_pic_num);
1823             ref_list[ref_list_idx++] =
1824                 found_ref_idx >= 0 ? priv->long_ref[found_ref_idx] : NULL;
1825             n = ref_list_idx;
1826             for (j = ref_list_idx; j <= num_refs; j++) {
1827                 gint32 LongTermPicNumF;
1828                 if (!ref_list[j])
1829                     continue;
1830                 LongTermPicNumF =
1831                     GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(ref_list[j]) ?
1832                     ref_list[j]->long_term_pic_num : INT_MAX;
1833                 if (LongTermPicNumF != l->value.long_term_pic_num)
1834                     ref_list[n++] = ref_list[j];
1835             }
1836         }
1837     }
1838
1839 #if DEBUG
1840     for (i = 0; i < num_refs; i++)
1841         if (!ref_list[i])
1842             GST_ERROR("list %u entry %u is empty", list, i);
1843 #endif
1844     *ref_list_count_ptr = num_refs;
1845 }
1846
1847 /* 8.2.4.3 - Modification process for reference picture lists */
1848 static void
1849 exec_picture_refs_modification(
1850     GstVaapiDecoderH264 *decoder,
1851     GstVaapiPictureH264 *picture,
1852     GstH264SliceHdr     *slice_hdr
1853 )
1854 {
1855     GST_DEBUG("execute ref_pic_list_modification()");
1856
1857     /* RefPicList0 */
1858     if (!GST_H264_IS_I_SLICE(slice_hdr) && !GST_H264_IS_SI_SLICE(slice_hdr) &&
1859         slice_hdr->ref_pic_list_modification_flag_l0)
1860         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 0);
1861
1862     /* RefPicList1 */
1863     if (GST_H264_IS_B_SLICE(slice_hdr) &&
1864         slice_hdr->ref_pic_list_modification_flag_l1)
1865         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 1);
1866 }
1867
1868 static void
1869 init_picture_ref_lists(GstVaapiDecoderH264 *decoder)
1870 {
1871     GstVaapiDecoderH264Private * const priv = decoder->priv;
1872     guint i, j, short_ref_count, long_ref_count;
1873
1874     short_ref_count = 0;
1875     long_ref_count  = 0;
1876     if (GST_VAAPI_PICTURE_IS_FRAME(priv->current_picture)) {
1877         for (i = 0; i < priv->dpb_count; i++) {
1878             GstVaapiFrameStore * const fs = priv->dpb[i];
1879             GstVaapiPictureH264 *picture;
1880             if (!gst_vaapi_frame_store_has_frame(fs))
1881                 continue;
1882             picture = fs->buffers[0];
1883             if (GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture))
1884                 priv->short_ref[short_ref_count++] = picture;
1885             else if (GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture))
1886                 priv->long_ref[long_ref_count++] = picture;
1887             picture->structure = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
1888             picture->other_field = fs->buffers[1];
1889         }
1890     }
1891     else {
1892         for (i = 0; i < priv->dpb_count; i++) {
1893             GstVaapiFrameStore * const fs = priv->dpb[i];
1894             for (j = 0; j < fs->num_buffers; j++) {
1895                 GstVaapiPictureH264 * const picture = fs->buffers[j];
1896                 if (GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture))
1897                     priv->short_ref[short_ref_count++] = picture;
1898                 else if (GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture))
1899                     priv->long_ref[long_ref_count++] = picture;
1900                 picture->structure = picture->base.structure;
1901                 picture->other_field = fs->buffers[j ^ 1];
1902             }
1903         }
1904     }
1905
1906     for (i = short_ref_count; i < priv->short_ref_count; i++)
1907         priv->short_ref[i] = NULL;
1908     priv->short_ref_count = short_ref_count;
1909
1910     for (i = long_ref_count; i < priv->long_ref_count; i++)
1911         priv->long_ref[i] = NULL;
1912     priv->long_ref_count = long_ref_count;
1913 }
1914
1915 static void
1916 init_picture_refs(
1917     GstVaapiDecoderH264 *decoder,
1918     GstVaapiPictureH264 *picture,
1919     GstH264SliceHdr     *slice_hdr
1920 )
1921 {
1922     GstVaapiDecoderH264Private * const priv = decoder->priv;
1923     GstVaapiPicture * const base_picture = &picture->base;
1924     guint i, num_refs;
1925
1926     init_picture_ref_lists(decoder);
1927     init_picture_refs_pic_num(decoder, picture, slice_hdr);
1928
1929     priv->RefPicList0_count = 0;
1930     priv->RefPicList1_count = 0;
1931
1932     switch (base_picture->type) {
1933     case GST_VAAPI_PICTURE_TYPE_P:
1934     case GST_VAAPI_PICTURE_TYPE_SP:
1935         init_picture_refs_p_slice(decoder, picture, slice_hdr);
1936         break;
1937     case GST_VAAPI_PICTURE_TYPE_B:
1938         init_picture_refs_b_slice(decoder, picture, slice_hdr);
1939         break;
1940     default:
1941         break;
1942     }
1943
1944     exec_picture_refs_modification(decoder, picture, slice_hdr);
1945
1946     switch (base_picture->type) {
1947     case GST_VAAPI_PICTURE_TYPE_B:
1948         num_refs = 1 + slice_hdr->num_ref_idx_l1_active_minus1;
1949         for (i = priv->RefPicList1_count; i < num_refs; i++)
1950             priv->RefPicList1[i] = NULL;
1951         priv->RefPicList1_count = num_refs;
1952
1953         // fall-through
1954     case GST_VAAPI_PICTURE_TYPE_P:
1955     case GST_VAAPI_PICTURE_TYPE_SP:
1956         num_refs = 1 + slice_hdr->num_ref_idx_l0_active_minus1;
1957         for (i = priv->RefPicList0_count; i < num_refs; i++)
1958             priv->RefPicList0[i] = NULL;
1959         priv->RefPicList0_count = num_refs;
1960         break;
1961     default:
1962         break;
1963     }
1964 }
1965
1966 static gboolean
1967 init_picture(
1968     GstVaapiDecoderH264 *decoder,
1969     GstVaapiPictureH264 *picture, GstVaapiParserInfoH264 *pi)
1970 {
1971     GstVaapiDecoderH264Private * const priv = decoder->priv;
1972     GstVaapiPicture * const base_picture = &picture->base;
1973     GstH264SliceHdr * const slice_hdr = &pi->data.slice_hdr;
1974
1975     priv->prev_frame_num        = priv->frame_num;
1976     priv->frame_num             = slice_hdr->frame_num;
1977     picture->frame_num          = priv->frame_num;
1978     picture->frame_num_wrap     = priv->frame_num;
1979     picture->output_flag        = TRUE; /* XXX: conformant to Annex A only */
1980     base_picture->pts           = GST_VAAPI_DECODER_CODEC_FRAME(decoder)->pts;
1981
1982     /* Reset decoder state for IDR pictures */
1983     if (pi->nalu.type == GST_H264_NAL_SLICE_IDR) {
1984         GST_DEBUG("<IDR>");
1985         GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_IDR);
1986         dpb_flush(decoder);
1987     }
1988
1989     /* Initialize slice type */
1990     switch (slice_hdr->type % 5) {
1991     case GST_H264_P_SLICE:
1992         base_picture->type = GST_VAAPI_PICTURE_TYPE_P;
1993         break;
1994     case GST_H264_B_SLICE:
1995         base_picture->type = GST_VAAPI_PICTURE_TYPE_B;
1996         break;
1997     case GST_H264_I_SLICE:
1998         base_picture->type = GST_VAAPI_PICTURE_TYPE_I;
1999         break;
2000     case GST_H264_SP_SLICE:
2001         base_picture->type = GST_VAAPI_PICTURE_TYPE_SP;
2002         break;
2003     case GST_H264_SI_SLICE:
2004         base_picture->type = GST_VAAPI_PICTURE_TYPE_SI;
2005         break;
2006     }
2007
2008     /* Initialize picture structure */
2009     if (!slice_hdr->field_pic_flag)
2010         base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
2011     else {
2012         GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_INTERLACED);
2013         if (!slice_hdr->bottom_field_flag)
2014             base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD;
2015         else
2016             base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD;
2017     }
2018     picture->structure = base_picture->structure;
2019
2020     /* Initialize reference flags */
2021     if (pi->nalu.ref_idc) {
2022         GstH264DecRefPicMarking * const dec_ref_pic_marking =
2023             &slice_hdr->dec_ref_pic_marking;
2024
2025         if (GST_VAAPI_PICTURE_IS_IDR(picture) &&
2026             dec_ref_pic_marking->long_term_reference_flag)
2027             GST_VAAPI_PICTURE_FLAG_SET(picture,
2028                 GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE);
2029         else
2030             GST_VAAPI_PICTURE_FLAG_SET(picture,
2031                 GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE);
2032     }
2033
2034     init_picture_poc(decoder, picture, slice_hdr);
2035     init_picture_refs(decoder, picture, slice_hdr);
2036     return TRUE;
2037 }
2038
2039 /* 8.2.5.3 - Sliding window decoded reference picture marking process */
2040 static gboolean
2041 exec_ref_pic_marking_sliding_window(GstVaapiDecoderH264 *decoder)
2042 {
2043     GstVaapiDecoderH264Private * const priv = decoder->priv;
2044     GstH264PPS * const pps = priv->current_picture->pps;
2045     GstH264SPS * const sps = pps->sequence;
2046     GstVaapiPictureH264 *ref_picture;
2047     guint i, m, max_num_ref_frames;
2048
2049     GST_DEBUG("reference picture marking process (sliding window)");
2050
2051     if (!GST_VAAPI_PICTURE_IS_FIRST_FIELD(priv->current_picture))
2052         return TRUE;
2053
2054     max_num_ref_frames = sps->num_ref_frames;
2055     if (max_num_ref_frames == 0)
2056         max_num_ref_frames = 1;
2057     if (!GST_VAAPI_PICTURE_IS_FRAME(priv->current_picture))
2058         max_num_ref_frames <<= 1;
2059
2060     if (priv->short_ref_count + priv->long_ref_count < max_num_ref_frames)
2061         return TRUE;
2062     if (priv->short_ref_count < 1)
2063         return FALSE;
2064
2065     for (m = 0, i = 1; i < priv->short_ref_count; i++) {
2066         GstVaapiPictureH264 * const picture = priv->short_ref[i];
2067         if (picture->frame_num_wrap < priv->short_ref[m]->frame_num_wrap)
2068             m = i;
2069     }
2070
2071     ref_picture = priv->short_ref[m];
2072     gst_vaapi_picture_h264_set_reference(ref_picture, 0, TRUE);
2073     ARRAY_REMOVE_INDEX(priv->short_ref, m);
2074
2075     /* Both fields need to be marked as "unused for reference", so
2076        remove the other field from the short_ref[] list as well */
2077     if (!GST_VAAPI_PICTURE_IS_FRAME(priv->current_picture) && ref_picture->other_field) {
2078         for (i = 0; i < priv->short_ref_count; i++) {
2079             if (priv->short_ref[i] == ref_picture->other_field) {
2080                 ARRAY_REMOVE_INDEX(priv->short_ref, i);
2081                 break;
2082             }
2083         }
2084     }
2085     return TRUE;
2086 }
2087
2088 static inline gint32
2089 get_picNumX(GstVaapiPictureH264 *picture, GstH264RefPicMarking *ref_pic_marking)
2090 {
2091     gint32 pic_num;
2092
2093     if (GST_VAAPI_PICTURE_IS_FRAME(picture))
2094         pic_num = picture->frame_num_wrap;
2095     else
2096         pic_num = 2 * picture->frame_num_wrap + 1;
2097     pic_num -= ref_pic_marking->difference_of_pic_nums_minus1 + 1;
2098     return pic_num;
2099 }
2100
2101 /* 8.2.5.4.1. Mark short-term reference picture as "unused for reference" */
2102 static void
2103 exec_ref_pic_marking_adaptive_mmco_1(
2104     GstVaapiDecoderH264  *decoder,
2105     GstVaapiPictureH264  *picture,
2106     GstH264RefPicMarking *ref_pic_marking
2107 )
2108 {
2109     GstVaapiDecoderH264Private * const priv = decoder->priv;
2110     gint32 i, picNumX;
2111
2112     picNumX = get_picNumX(picture, ref_pic_marking);
2113     i = find_short_term_reference(decoder, picNumX);
2114     if (i < 0)
2115         return;
2116
2117     gst_vaapi_picture_h264_set_reference(priv->short_ref[i], 0,
2118         GST_VAAPI_PICTURE_IS_FRAME(picture));
2119     ARRAY_REMOVE_INDEX(priv->short_ref, i);
2120 }
2121
2122 /* 8.2.5.4.2. Mark long-term reference picture as "unused for reference" */
2123 static void
2124 exec_ref_pic_marking_adaptive_mmco_2(
2125     GstVaapiDecoderH264  *decoder,
2126     GstVaapiPictureH264  *picture,
2127     GstH264RefPicMarking *ref_pic_marking
2128 )
2129 {
2130     GstVaapiDecoderH264Private * const priv = decoder->priv;
2131     gint32 i;
2132
2133     i = find_long_term_reference(decoder, ref_pic_marking->long_term_pic_num);
2134     if (i < 0)
2135         return;
2136
2137     gst_vaapi_picture_h264_set_reference(priv->long_ref[i], 0,
2138         GST_VAAPI_PICTURE_IS_FRAME(picture));
2139     ARRAY_REMOVE_INDEX(priv->long_ref, i);
2140 }
2141
2142 /* 8.2.5.4.3. Assign LongTermFrameIdx to a short-term reference picture */
2143 static void
2144 exec_ref_pic_marking_adaptive_mmco_3(
2145     GstVaapiDecoderH264  *decoder,
2146     GstVaapiPictureH264  *picture,
2147     GstH264RefPicMarking *ref_pic_marking
2148 )
2149 {
2150     GstVaapiDecoderH264Private * const priv = decoder->priv;
2151     GstVaapiPictureH264 *ref_picture;
2152     gint32 i, picNumX;
2153
2154     for (i = 0; i < priv->long_ref_count; i++) {
2155         if (priv->long_ref[i]->long_term_frame_idx == ref_pic_marking->long_term_frame_idx)
2156             break;
2157     }
2158     if (i != priv->long_ref_count) {
2159         gst_vaapi_picture_h264_set_reference(priv->long_ref[i], 0, TRUE);
2160         ARRAY_REMOVE_INDEX(priv->long_ref, i);
2161     }
2162
2163     picNumX = get_picNumX(picture, ref_pic_marking);
2164     i = find_short_term_reference(decoder, picNumX);
2165     if (i < 0)
2166         return;
2167
2168     ref_picture = priv->short_ref[i];
2169     ARRAY_REMOVE_INDEX(priv->short_ref, i);
2170     priv->long_ref[priv->long_ref_count++] = ref_picture;
2171
2172     ref_picture->long_term_frame_idx = ref_pic_marking->long_term_frame_idx;
2173     gst_vaapi_picture_h264_set_reference(ref_picture,
2174         GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE,
2175         GST_VAAPI_PICTURE_IS_FRAME(picture));
2176 }
2177
2178 /* 8.2.5.4.4. Mark pictures with LongTermFramIdx > max_long_term_frame_idx
2179  * as "unused for reference" */
2180 static void
2181 exec_ref_pic_marking_adaptive_mmco_4(
2182     GstVaapiDecoderH264  *decoder,
2183     GstVaapiPictureH264  *picture,
2184     GstH264RefPicMarking *ref_pic_marking
2185 )
2186 {
2187     GstVaapiDecoderH264Private * const priv = decoder->priv;
2188     gint32 i, long_term_frame_idx;
2189
2190     long_term_frame_idx = ref_pic_marking->max_long_term_frame_idx_plus1 - 1;
2191
2192     for (i = 0; i < priv->long_ref_count; i++) {
2193         if (priv->long_ref[i]->long_term_frame_idx <= long_term_frame_idx)
2194             continue;
2195         gst_vaapi_picture_h264_set_reference(priv->long_ref[i], 0, FALSE);
2196         ARRAY_REMOVE_INDEX(priv->long_ref, i);
2197         i--;
2198     }
2199 }
2200
2201 /* 8.2.5.4.5. Mark all reference pictures as "unused for reference" */
2202 static void
2203 exec_ref_pic_marking_adaptive_mmco_5(
2204     GstVaapiDecoderH264  *decoder,
2205     GstVaapiPictureH264  *picture,
2206     GstH264RefPicMarking *ref_pic_marking
2207 )
2208 {
2209     GstVaapiDecoderH264Private * const priv = decoder->priv;
2210
2211     dpb_flush(decoder);
2212
2213     priv->prev_pic_has_mmco5 = TRUE;
2214
2215     /* The picture shall be inferred to have had frame_num equal to 0 (7.4.3) */
2216     priv->frame_num = 0;
2217     priv->frame_num_offset = 0;
2218     picture->frame_num = 0;
2219
2220     /* Update TopFieldOrderCnt and BottomFieldOrderCnt (8.2.1) */
2221     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
2222         picture->field_poc[TOP_FIELD] -= picture->base.poc;
2223     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
2224         picture->field_poc[BOTTOM_FIELD] -= picture->base.poc;
2225     picture->base.poc = 0;
2226 }
2227
2228 /* 8.2.5.4.6. Assign a long-term frame index to the current picture */
2229 static void
2230 exec_ref_pic_marking_adaptive_mmco_6(
2231     GstVaapiDecoderH264  *decoder,
2232     GstVaapiPictureH264  *picture,
2233     GstH264RefPicMarking *ref_pic_marking
2234 )
2235 {
2236     picture->long_term_frame_idx = ref_pic_marking->long_term_frame_idx;
2237     gst_vaapi_picture_h264_set_reference(picture,
2238         GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE, FALSE);
2239 }
2240
2241 /* 8.2.5.4. Adaptive memory control decoded reference picture marking process */
2242 static gboolean
2243 exec_ref_pic_marking_adaptive(
2244     GstVaapiDecoderH264     *decoder,
2245     GstVaapiPictureH264     *picture,
2246     GstH264DecRefPicMarking *dec_ref_pic_marking
2247 )
2248 {
2249     guint i;
2250
2251     GST_DEBUG("reference picture marking process (adaptive memory control)");
2252
2253     typedef void (*exec_ref_pic_marking_adaptive_mmco_func)(
2254         GstVaapiDecoderH264  *decoder,
2255         GstVaapiPictureH264  *picture,
2256         GstH264RefPicMarking *ref_pic_marking
2257     );
2258
2259     static const exec_ref_pic_marking_adaptive_mmco_func mmco_funcs[] = {
2260         NULL,
2261         exec_ref_pic_marking_adaptive_mmco_1,
2262         exec_ref_pic_marking_adaptive_mmco_2,
2263         exec_ref_pic_marking_adaptive_mmco_3,
2264         exec_ref_pic_marking_adaptive_mmco_4,
2265         exec_ref_pic_marking_adaptive_mmco_5,
2266         exec_ref_pic_marking_adaptive_mmco_6,
2267     };
2268
2269     for (i = 0; i < dec_ref_pic_marking->n_ref_pic_marking; i++) {
2270         GstH264RefPicMarking * const ref_pic_marking =
2271             &dec_ref_pic_marking->ref_pic_marking[i];
2272
2273         const guint mmco = ref_pic_marking->memory_management_control_operation;
2274         if (mmco < G_N_ELEMENTS(mmco_funcs) && mmco_funcs[mmco])
2275             mmco_funcs[mmco](decoder, picture, ref_pic_marking);
2276         else {
2277             GST_ERROR("unhandled MMCO %u", mmco);
2278             return FALSE;
2279         }
2280     }
2281     return TRUE;
2282 }
2283
2284 /* 8.2.5 - Execute reference picture marking process */
2285 static gboolean
2286 exec_ref_pic_marking(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
2287 {
2288     GstVaapiDecoderH264Private * const priv = decoder->priv;
2289
2290     priv->prev_pic_has_mmco5 = FALSE;
2291     priv->prev_pic_structure = picture->structure;
2292
2293     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
2294         return TRUE;
2295
2296     if (!GST_VAAPI_PICTURE_IS_IDR(picture)) {
2297         GstH264DecRefPicMarking * const dec_ref_pic_marking =
2298             &picture->last_slice_hdr->dec_ref_pic_marking;
2299         if (dec_ref_pic_marking->adaptive_ref_pic_marking_mode_flag) {
2300             if (!exec_ref_pic_marking_adaptive(decoder, picture, dec_ref_pic_marking))
2301                 return FALSE;
2302         }
2303         else {
2304             if (!exec_ref_pic_marking_sliding_window(decoder))
2305                 return FALSE;
2306         }
2307     }
2308     return TRUE;
2309 }
2310
2311 static void
2312 vaapi_init_picture(VAPictureH264 *pic)
2313 {
2314     pic->picture_id           = VA_INVALID_ID;
2315     pic->frame_idx            = 0;
2316     pic->flags                = VA_PICTURE_H264_INVALID;
2317     pic->TopFieldOrderCnt     = 0;
2318     pic->BottomFieldOrderCnt  = 0;
2319 }
2320
2321 static void
2322 vaapi_fill_picture(VAPictureH264 *pic, GstVaapiPictureH264 *picture,
2323     guint picture_structure)
2324 {
2325     if (!picture_structure)
2326         picture_structure = picture->structure;
2327
2328     pic->picture_id = picture->base.surface_id;
2329     pic->flags = 0;
2330
2331     if (GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture)) {
2332         pic->flags |= VA_PICTURE_H264_LONG_TERM_REFERENCE;
2333         pic->frame_idx = picture->long_term_frame_idx;
2334     }
2335     else {
2336         if (GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture))
2337             pic->flags |= VA_PICTURE_H264_SHORT_TERM_REFERENCE;
2338         pic->frame_idx = picture->frame_num;
2339     }
2340
2341     switch (picture_structure) {
2342     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
2343         pic->TopFieldOrderCnt = picture->field_poc[TOP_FIELD];
2344         pic->BottomFieldOrderCnt = picture->field_poc[BOTTOM_FIELD];
2345         break;
2346     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
2347         pic->flags |= VA_PICTURE_H264_TOP_FIELD;
2348         pic->TopFieldOrderCnt = picture->field_poc[TOP_FIELD];
2349         pic->BottomFieldOrderCnt = 0;
2350         break;
2351     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
2352         pic->flags |= VA_PICTURE_H264_BOTTOM_FIELD;
2353         pic->BottomFieldOrderCnt = picture->field_poc[BOTTOM_FIELD];
2354         pic->TopFieldOrderCnt = 0;
2355         break;
2356     }
2357 }
2358
2359 static gboolean
2360 fill_picture(GstVaapiDecoderH264 *decoder,
2361     GstVaapiPictureH264 *picture, GstVaapiParserInfoH264 *pi)
2362 {
2363     GstVaapiDecoderH264Private * const priv = decoder->priv;
2364     GstVaapiPicture * const base_picture = &picture->base;
2365     GstH264SliceHdr * const slice_hdr = &pi->data.slice_hdr;
2366     GstH264PPS * const pps = picture->pps;
2367     GstH264SPS * const sps = pps->sequence;
2368     VAPictureParameterBufferH264 * const pic_param = base_picture->param;
2369     guint i, n;
2370
2371     /* Fill in VAPictureParameterBufferH264 */
2372     vaapi_fill_picture(&pic_param->CurrPic, picture, 0);
2373
2374     for (i = 0, n = 0; i < priv->dpb_count; i++) {
2375         GstVaapiFrameStore * const fs = priv->dpb[i];
2376         if (gst_vaapi_frame_store_has_reference(fs))
2377             vaapi_fill_picture(&pic_param->ReferenceFrames[n++],
2378                 fs->buffers[0], fs->structure);
2379     }
2380     for (; n < G_N_ELEMENTS(pic_param->ReferenceFrames); n++)
2381         vaapi_init_picture(&pic_param->ReferenceFrames[n]);
2382
2383 #define COPY_FIELD(s, f) \
2384     pic_param->f = (s)->f
2385
2386 #define COPY_BFM(a, s, f) \
2387     pic_param->a.bits.f = (s)->f
2388
2389     pic_param->picture_width_in_mbs_minus1  = priv->mb_width - 1;
2390     pic_param->picture_height_in_mbs_minus1 = priv->mb_height - 1;
2391     pic_param->frame_num                    = priv->frame_num;
2392
2393     COPY_FIELD(sps, bit_depth_luma_minus8);
2394     COPY_FIELD(sps, bit_depth_chroma_minus8);
2395     COPY_FIELD(sps, num_ref_frames);
2396     COPY_FIELD(pps, num_slice_groups_minus1);
2397     COPY_FIELD(pps, slice_group_map_type);
2398     COPY_FIELD(pps, slice_group_change_rate_minus1);
2399     COPY_FIELD(pps, pic_init_qp_minus26);
2400     COPY_FIELD(pps, pic_init_qs_minus26);
2401     COPY_FIELD(pps, chroma_qp_index_offset);
2402     COPY_FIELD(pps, second_chroma_qp_index_offset);
2403
2404     pic_param->seq_fields.value                                         = 0; /* reset all bits */
2405     pic_param->seq_fields.bits.residual_colour_transform_flag           = sps->separate_colour_plane_flag;
2406     pic_param->seq_fields.bits.MinLumaBiPredSize8x8                     = sps->level_idc >= 31; /* A.3.3.2 */
2407
2408     COPY_BFM(seq_fields, sps, chroma_format_idc);
2409     COPY_BFM(seq_fields, sps, gaps_in_frame_num_value_allowed_flag);
2410     COPY_BFM(seq_fields, sps, frame_mbs_only_flag);
2411     COPY_BFM(seq_fields, sps, mb_adaptive_frame_field_flag);
2412     COPY_BFM(seq_fields, sps, direct_8x8_inference_flag);
2413     COPY_BFM(seq_fields, sps, log2_max_frame_num_minus4);
2414     COPY_BFM(seq_fields, sps, pic_order_cnt_type);
2415     COPY_BFM(seq_fields, sps, log2_max_pic_order_cnt_lsb_minus4);
2416     COPY_BFM(seq_fields, sps, delta_pic_order_always_zero_flag);
2417
2418     pic_param->pic_fields.value                                         = 0; /* reset all bits */
2419     pic_param->pic_fields.bits.field_pic_flag                           = slice_hdr->field_pic_flag;
2420     pic_param->pic_fields.bits.reference_pic_flag                       = GST_VAAPI_PICTURE_IS_REFERENCE(picture);
2421
2422     COPY_BFM(pic_fields, pps, entropy_coding_mode_flag);
2423     COPY_BFM(pic_fields, pps, weighted_pred_flag);
2424     COPY_BFM(pic_fields, pps, weighted_bipred_idc);
2425     COPY_BFM(pic_fields, pps, transform_8x8_mode_flag);
2426     COPY_BFM(pic_fields, pps, constrained_intra_pred_flag);
2427     COPY_BFM(pic_fields, pps, pic_order_present_flag);
2428     COPY_BFM(pic_fields, pps, deblocking_filter_control_present_flag);
2429     COPY_BFM(pic_fields, pps, redundant_pic_cnt_present_flag);
2430     return TRUE;
2431 }
2432
2433 /* Detection of the first VCL NAL unit of a primary coded picture (7.4.1.2.4) */
2434 static gboolean
2435 is_new_picture(GstVaapiParserInfoH264 *pi, GstVaapiParserInfoH264 *prev_pi)
2436 {
2437     GstH264SliceHdr * const slice_hdr = &pi->data.slice_hdr;
2438     GstH264PPS * const pps = slice_hdr->pps;
2439     GstH264SPS * const sps = pps->sequence;
2440     GstH264SliceHdr *prev_slice_hdr;
2441
2442     if (!prev_pi)
2443         return TRUE;
2444     prev_slice_hdr = &prev_pi->data.slice_hdr;
2445
2446 #define CHECK_EXPR(expr, field_name) do {              \
2447         if (!(expr)) {                                 \
2448             GST_DEBUG(field_name " differs in value"); \
2449             return TRUE;                               \
2450         }                                              \
2451     } while (0)
2452
2453 #define CHECK_VALUE(new_slice_hdr, old_slice_hdr, field) \
2454     CHECK_EXPR(((new_slice_hdr)->field == (old_slice_hdr)->field), #field)
2455
2456     /* frame_num differs in value, regardless of inferred values to 0 */
2457     CHECK_VALUE(slice_hdr, prev_slice_hdr, frame_num);
2458
2459     /* pic_parameter_set_id differs in value */
2460     CHECK_VALUE(slice_hdr, prev_slice_hdr, pps);
2461
2462     /* field_pic_flag differs in value */
2463     CHECK_VALUE(slice_hdr, prev_slice_hdr, field_pic_flag);
2464
2465     /* bottom_field_flag is present in both and differs in value */
2466     if (slice_hdr->field_pic_flag && prev_slice_hdr->field_pic_flag)
2467         CHECK_VALUE(slice_hdr, prev_slice_hdr, bottom_field_flag);
2468
2469     /* nal_ref_idc differs in value with one of the nal_ref_idc values is 0 */
2470     CHECK_EXPR((pi->nalu.ref_idc != 0) ==
2471                (prev_pi->nalu.ref_idc != 0), "nal_ref_idc");
2472
2473     /* POC type is 0 for both and either pic_order_cnt_lsb differs in
2474        value or delta_pic_order_cnt_bottom differs in value */
2475     if (sps->pic_order_cnt_type == 0) {
2476         CHECK_VALUE(slice_hdr, prev_slice_hdr, pic_order_cnt_lsb);
2477         if (pps->pic_order_present_flag && !slice_hdr->field_pic_flag)
2478             CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt_bottom);
2479     }
2480
2481     /* POC type is 1 for both and either delta_pic_order_cnt[0]
2482        differs in value or delta_pic_order_cnt[1] differs in value */
2483     else if (sps->pic_order_cnt_type == 1) {
2484         CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt[0]);
2485         CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt[1]);
2486     }
2487
2488     /* IdrPicFlag differs in value */
2489     CHECK_VALUE(&pi->nalu, &prev_pi->nalu, idr_pic_flag);
2490
2491     /* IdrPicFlag is equal to 1 for both and idr_pic_id differs in value */
2492     if (pi->nalu.idr_pic_flag)
2493         CHECK_VALUE(slice_hdr, prev_slice_hdr, idr_pic_id);
2494
2495 #undef CHECK_EXPR
2496 #undef CHECK_VALUE
2497     return FALSE;
2498 }
2499
2500 static GstVaapiDecoderStatus
2501 decode_picture(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnit *unit)
2502 {
2503     GstVaapiDecoderH264Private * const priv = decoder->priv;
2504     GstVaapiParserInfoH264 * const pi = unit->parsed_info;
2505     GstH264SliceHdr * const slice_hdr = &pi->data.slice_hdr;
2506     GstH264PPS * const pps = slice_hdr->pps;
2507     GstH264SPS * const sps = pps->sequence;
2508     GstVaapiPictureH264 *picture;
2509     GstVaapiDecoderStatus status;
2510
2511     status = ensure_context(decoder, sps);
2512     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2513         return status;
2514
2515     if (priv->current_picture) {
2516         /* Re-use current picture where the first field was decoded */
2517         picture = gst_vaapi_picture_h264_new_field(priv->current_picture);
2518         if (!picture) {
2519             GST_ERROR("failed to allocate field picture");
2520             return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2521         }
2522     }
2523     else {
2524         /* Create new picture */
2525         picture = gst_vaapi_picture_h264_new(decoder);
2526         if (!picture) {
2527             GST_ERROR("failed to allocate picture");
2528             return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2529         }
2530     }
2531     gst_vaapi_picture_replace(&priv->current_picture, picture);
2532     gst_vaapi_picture_unref(picture);
2533
2534     picture->pps = pps;
2535
2536     status = ensure_quant_matrix(decoder, picture);
2537     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS) {
2538         GST_ERROR("failed to reset quantizer matrix");
2539         return status;
2540     }
2541
2542     if (!init_picture(decoder, picture, pi))
2543         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2544     if (!fill_picture(decoder, picture, pi))
2545         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2546     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2547 }
2548
2549 static inline guint
2550 get_slice_data_bit_offset(GstH264SliceHdr *slice_hdr)
2551 {
2552     guint epb_count;
2553
2554     epb_count = slice_hdr->n_emulation_prevention_bytes;
2555     return 8 /* nal_unit_type */ + slice_hdr->header_size - epb_count * 8;
2556 }
2557
2558 static gboolean
2559 fill_pred_weight_table(GstVaapiDecoderH264 *decoder,
2560     GstVaapiSlice *slice, GstH264SliceHdr *slice_hdr)
2561 {
2562     VASliceParameterBufferH264 * const slice_param = slice->param;
2563     GstH264PPS * const pps = slice_hdr->pps;
2564     GstH264SPS * const sps = pps->sequence;
2565     GstH264PredWeightTable * const w = &slice_hdr->pred_weight_table;
2566     guint num_weight_tables = 0;
2567     gint i, j;
2568
2569     if (pps->weighted_pred_flag &&
2570         (GST_H264_IS_P_SLICE(slice_hdr) || GST_H264_IS_SP_SLICE(slice_hdr)))
2571         num_weight_tables = 1;
2572     else if (pps->weighted_bipred_idc == 1 && GST_H264_IS_B_SLICE(slice_hdr))
2573         num_weight_tables = 2;
2574     else
2575         num_weight_tables = 0;
2576
2577     slice_param->luma_log2_weight_denom   = 0;
2578     slice_param->chroma_log2_weight_denom = 0;
2579     slice_param->luma_weight_l0_flag      = 0;
2580     slice_param->chroma_weight_l0_flag    = 0;
2581     slice_param->luma_weight_l1_flag      = 0;
2582     slice_param->chroma_weight_l1_flag    = 0;
2583
2584     if (num_weight_tables < 1)
2585         return TRUE;
2586
2587     slice_param->luma_log2_weight_denom   = w->luma_log2_weight_denom;
2588     slice_param->chroma_log2_weight_denom = w->chroma_log2_weight_denom;
2589
2590     slice_param->luma_weight_l0_flag = 1;
2591     for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2592         slice_param->luma_weight_l0[i] = w->luma_weight_l0[i];
2593         slice_param->luma_offset_l0[i] = w->luma_offset_l0[i];
2594     }
2595
2596     slice_param->chroma_weight_l0_flag = sps->chroma_array_type != 0;
2597     if (slice_param->chroma_weight_l0_flag) {
2598         for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2599             for (j = 0; j < 2; j++) {
2600                 slice_param->chroma_weight_l0[i][j] = w->chroma_weight_l0[i][j];
2601                 slice_param->chroma_offset_l0[i][j] = w->chroma_offset_l0[i][j];
2602             }
2603         }
2604     }
2605
2606     if (num_weight_tables < 2)
2607         return TRUE;
2608
2609     slice_param->luma_weight_l1_flag = 1;
2610     for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2611         slice_param->luma_weight_l1[i] = w->luma_weight_l1[i];
2612         slice_param->luma_offset_l1[i] = w->luma_offset_l1[i];
2613     }
2614
2615     slice_param->chroma_weight_l1_flag = sps->chroma_array_type != 0;
2616     if (slice_param->chroma_weight_l1_flag) {
2617         for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2618             for (j = 0; j < 2; j++) {
2619                 slice_param->chroma_weight_l1[i][j] = w->chroma_weight_l1[i][j];
2620                 slice_param->chroma_offset_l1[i][j] = w->chroma_offset_l1[i][j];
2621             }
2622         }
2623     }
2624     return TRUE;
2625 }
2626
2627 static gboolean
2628 fill_RefPicList(GstVaapiDecoderH264 *decoder,
2629     GstVaapiSlice *slice, GstH264SliceHdr *slice_hdr)
2630 {
2631     GstVaapiDecoderH264Private * const priv = decoder->priv;
2632     VASliceParameterBufferH264 * const slice_param = slice->param;
2633     guint i, num_ref_lists = 0;
2634
2635     slice_param->num_ref_idx_l0_active_minus1 = 0;
2636     slice_param->num_ref_idx_l1_active_minus1 = 0;
2637
2638     if (GST_H264_IS_B_SLICE(slice_hdr))
2639         num_ref_lists = 2;
2640     else if (GST_H264_IS_I_SLICE(slice_hdr))
2641         num_ref_lists = 0;
2642     else
2643         num_ref_lists = 1;
2644
2645     if (num_ref_lists < 1)
2646         return TRUE;
2647
2648     slice_param->num_ref_idx_l0_active_minus1 =
2649         slice_hdr->num_ref_idx_l0_active_minus1;
2650
2651     for (i = 0; i < priv->RefPicList0_count && priv->RefPicList0[i]; i++)
2652         vaapi_fill_picture(&slice_param->RefPicList0[i], priv->RefPicList0[i], 0);
2653     for (; i <= slice_param->num_ref_idx_l0_active_minus1; i++)
2654         vaapi_init_picture(&slice_param->RefPicList0[i]);
2655
2656     if (num_ref_lists < 2)
2657         return TRUE;
2658
2659     slice_param->num_ref_idx_l1_active_minus1 =
2660         slice_hdr->num_ref_idx_l1_active_minus1;
2661
2662     for (i = 0; i < priv->RefPicList1_count && priv->RefPicList1[i]; i++)
2663         vaapi_fill_picture(&slice_param->RefPicList1[i], priv->RefPicList1[i], 0);
2664     for (; i <= slice_param->num_ref_idx_l1_active_minus1; i++)
2665         vaapi_init_picture(&slice_param->RefPicList1[i]);
2666     return TRUE;
2667 }
2668
2669 static gboolean
2670 fill_slice(GstVaapiDecoderH264 *decoder,
2671     GstVaapiSlice *slice, GstH264SliceHdr *slice_hdr)
2672 {
2673     VASliceParameterBufferH264 * const slice_param = slice->param;
2674
2675     /* Fill in VASliceParameterBufferH264 */
2676     slice_param->slice_data_bit_offset          = get_slice_data_bit_offset(slice_hdr);
2677     slice_param->first_mb_in_slice              = slice_hdr->first_mb_in_slice;
2678     slice_param->slice_type                     = slice_hdr->type % 5;
2679     slice_param->direct_spatial_mv_pred_flag    = slice_hdr->direct_spatial_mv_pred_flag;
2680     slice_param->cabac_init_idc                 = slice_hdr->cabac_init_idc;
2681     slice_param->slice_qp_delta                 = slice_hdr->slice_qp_delta;
2682     slice_param->disable_deblocking_filter_idc  = slice_hdr->disable_deblocking_filter_idc;
2683     slice_param->slice_alpha_c0_offset_div2     = slice_hdr->slice_alpha_c0_offset_div2;
2684     slice_param->slice_beta_offset_div2         = slice_hdr->slice_beta_offset_div2;
2685
2686     if (!fill_RefPicList(decoder, slice, slice_hdr))
2687         return FALSE;
2688     if (!fill_pred_weight_table(decoder, slice, slice_hdr))
2689         return FALSE;
2690     return TRUE;
2691 }
2692
2693 static GstVaapiDecoderStatus
2694 decode_slice(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnit *unit)
2695 {
2696     GstVaapiDecoderH264Private * const priv = decoder->priv;
2697     GstVaapiParserInfoH264 * const pi = unit->parsed_info;
2698     GstVaapiPictureH264 * const picture = priv->current_picture;
2699     GstH264SliceHdr * const slice_hdr = &pi->data.slice_hdr;
2700     GstVaapiSlice *slice;
2701     GstBuffer * const buffer =
2702         GST_VAAPI_DECODER_CODEC_FRAME(decoder)->input_buffer;
2703
2704     GST_DEBUG("slice (%u bytes)", pi->nalu.size);
2705
2706     if (!priv->got_sps || !priv->got_pps) {
2707         GST_ERROR("not initialized yet");
2708         return GST_VAAPI_DECODER_STATUS_SUCCESS;
2709     }
2710
2711     slice = GST_VAAPI_SLICE_NEW(H264, decoder,
2712         (GST_BUFFER_DATA(buffer) + unit->offset + pi->nalu.offset),
2713         pi->nalu.size);
2714     if (!slice) {
2715         GST_ERROR("failed to allocate slice");
2716         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2717     }
2718
2719     if (!fill_slice(decoder, slice, slice_hdr)) {
2720         gst_vaapi_mini_object_unref(GST_VAAPI_MINI_OBJECT(slice));
2721         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2722     }
2723
2724     gst_vaapi_picture_add_slice(GST_VAAPI_PICTURE_CAST(picture), slice);
2725     picture->last_slice_hdr = slice_hdr;
2726     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2727 }
2728
2729 static inline gint
2730 scan_for_start_code(GstAdapter *adapter, guint ofs, guint size, guint32 *scp)
2731 {
2732     return (gint)gst_adapter_masked_scan_uint32_peek(adapter,
2733                                                      0xffffff00, 0x00000100,
2734                                                      ofs, size,
2735                                                      scp);
2736 }
2737
2738 static GstVaapiDecoderStatus
2739 decode_unit(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnit *unit)
2740 {
2741     GstVaapiParserInfoH264 * const pi = unit->parsed_info;
2742     GstVaapiDecoderStatus status;
2743
2744     switch (pi->nalu.type) {
2745     case GST_H264_NAL_SLICE_IDR:
2746         /* fall-through. IDR specifics are handled in init_picture() */
2747     case GST_H264_NAL_SLICE:
2748         status = decode_slice(decoder, unit);
2749         break;
2750     case GST_H264_NAL_SEQ_END:
2751     case GST_H264_NAL_STREAM_END:
2752         status = decode_sequence_end(decoder);
2753         break;
2754     case GST_H264_NAL_SEI:
2755         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2756         break;
2757     default:
2758         GST_WARNING("unsupported NAL unit type %d", pi->nalu.type);
2759         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
2760         break;
2761     }
2762     return status;
2763 }
2764
2765 static GstVaapiDecoderStatus
2766 gst_vaapi_decoder_h264_decode_codec_data(GstVaapiDecoder *base_decoder,
2767     const guchar *buf, guint buf_size)
2768 {
2769     GstVaapiDecoderH264 * const decoder =
2770         GST_VAAPI_DECODER_H264_CAST(base_decoder);
2771     GstVaapiDecoderH264Private * const priv = decoder->priv;
2772     GstVaapiDecoderStatus status;
2773     GstVaapiDecoderUnit unit;
2774     GstVaapiParserInfoH264 pi;
2775     GstH264ParserResult result;
2776     guint i, ofs, num_sps, num_pps;
2777
2778     unit.parsed_info = &pi;
2779
2780     if (buf_size < 8)
2781         return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2782
2783     if (buf[0] != 1) {
2784         GST_ERROR("failed to decode codec-data, not in avcC format");
2785         return GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
2786     }
2787
2788     priv->nal_length_size = (buf[4] & 0x03) + 1;
2789
2790     num_sps = buf[5] & 0x1f;
2791     ofs = 6;
2792
2793     for (i = 0; i < num_sps; i++) {
2794         result = gst_h264_parser_identify_nalu_avc(
2795             priv->parser,
2796             buf, ofs, buf_size, 2,
2797             &pi.nalu
2798         );
2799         if (result != GST_H264_PARSER_OK)
2800             return get_status(result);
2801
2802         status = parse_sps(decoder, &unit);
2803         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2804             return status;
2805         ofs = pi.nalu.offset + pi.nalu.size;
2806     }
2807
2808     num_pps = buf[ofs];
2809     ofs++;
2810
2811     for (i = 0; i < num_pps; i++) {
2812         result = gst_h264_parser_identify_nalu_avc(
2813             priv->parser,
2814             buf, ofs, buf_size, 2,
2815             &pi.nalu
2816         );
2817         if (result != GST_H264_PARSER_OK)
2818             return get_status(result);
2819
2820         status = parse_pps(decoder, &unit);
2821         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2822             return status;
2823         ofs = pi.nalu.offset + pi.nalu.size;
2824     }
2825
2826     priv->is_avcC = TRUE;
2827     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2828 }
2829
2830 static GstVaapiDecoderStatus
2831 ensure_decoder(GstVaapiDecoderH264 *decoder)
2832 {
2833     GstVaapiDecoderH264Private * const priv = decoder->priv;
2834     GstVaapiDecoderStatus status;
2835
2836     g_return_val_if_fail(priv->is_constructed,
2837                          GST_VAAPI_DECODER_STATUS_ERROR_INIT_FAILED);
2838
2839     if (!priv->is_opened) {
2840         priv->is_opened = gst_vaapi_decoder_h264_open(decoder);
2841         if (!priv->is_opened)
2842             return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CODEC;
2843
2844         status = gst_vaapi_decoder_decode_codec_data(
2845             GST_VAAPI_DECODER_CAST(decoder));
2846         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2847             return status;
2848     }
2849     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2850 }
2851
2852 static GstVaapiDecoderStatus
2853 gst_vaapi_decoder_h264_parse(GstVaapiDecoder *base_decoder,
2854     GstAdapter *adapter, gboolean at_eos, GstVaapiDecoderUnit *unit)
2855 {
2856     GstVaapiDecoderH264 * const decoder =
2857         GST_VAAPI_DECODER_H264_CAST(base_decoder);
2858     GstVaapiDecoderH264Private * const priv = decoder->priv;
2859     GstVaapiParserState * const ps = GST_VAAPI_PARSER_STATE(base_decoder);
2860     GstVaapiParserInfoH264 *pi;
2861     GstVaapiDecoderStatus status;
2862     GstH264ParserResult result;
2863     guchar *buf;
2864     guint i, size, buf_size, nalu_size, flags;
2865     guint32 start_code;
2866     gint ofs, ofs2;
2867
2868     status = ensure_decoder(decoder);
2869     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2870         return status;
2871
2872     size = gst_adapter_available(adapter);
2873
2874     if (priv->is_avcC) {
2875         if (size < priv->nal_length_size)
2876             return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2877
2878         buf = (guchar *)&start_code;
2879         g_assert(priv->nal_length_size <= sizeof(start_code));
2880         gst_adapter_copy(adapter, buf, 0, priv->nal_length_size);
2881
2882         nalu_size = 0;
2883         for (i = 0; i < priv->nal_length_size; i++)
2884             nalu_size = (nalu_size << 8) | buf[i];
2885
2886         buf_size = priv->nal_length_size + nalu_size;
2887         if (size < buf_size)
2888             return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2889     }
2890     else {
2891         if (size < 4)
2892             return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2893
2894         ofs = scan_for_start_code(adapter, 0, size, NULL);
2895         if (ofs < 0)
2896             return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2897
2898         if (ofs > 0) {
2899             gst_adapter_flush(adapter, ofs);
2900             size -= ofs;
2901         }
2902
2903         ofs2 = ps->input_offset2 - ofs - 4;
2904         if (ofs2 < 4)
2905             ofs2 = 4;
2906
2907         ofs = G_UNLIKELY(size < ofs2 + 4) ? -1 :
2908             scan_for_start_code(adapter, ofs2, size - ofs2, NULL);
2909         if (ofs < 0) {
2910             // Assume the whole NAL unit is present if end-of-stream
2911             if (!at_eos) {
2912                 ps->input_offset2 = size;
2913                 return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2914             }
2915             ofs = size;
2916         }
2917         buf_size = ofs;
2918     }
2919     ps->input_offset2 = 0;
2920
2921     buf = (guchar *)gst_adapter_peek(adapter, buf_size);
2922     if (!buf)
2923         return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2924
2925     unit->size = buf_size;
2926
2927     pi = gst_vaapi_parser_info_h264_new();
2928     if (!pi)
2929         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2930
2931     gst_vaapi_decoder_unit_set_parsed_info(unit,
2932         pi, (GDestroyNotify)gst_vaapi_mini_object_unref);
2933
2934     if (priv->is_avcC)
2935         result = gst_h264_parser_identify_nalu_avc(priv->parser,
2936             buf, 0, buf_size, priv->nal_length_size, &pi->nalu);
2937     else
2938         result = gst_h264_parser_identify_nalu_unchecked(priv->parser,
2939             buf, 0, buf_size, &pi->nalu);
2940     status = get_status(result);
2941     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2942         return status;
2943
2944     switch (pi->nalu.type) {
2945     case GST_H264_NAL_SPS:
2946         status = parse_sps(decoder, unit);
2947         break;
2948     case GST_H264_NAL_PPS:
2949         status = parse_pps(decoder, unit);
2950         break;
2951     case GST_H264_NAL_SEI:
2952         status = parse_sei(decoder, unit);
2953         break;
2954     case GST_H264_NAL_SLICE_IDR:
2955     case GST_H264_NAL_SLICE:
2956         status = parse_slice(decoder, unit);
2957         break;
2958     default:
2959         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2960         break;
2961     }
2962     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2963         return status;
2964
2965     flags = 0;
2966     switch (pi->nalu.type) {
2967     case GST_H264_NAL_AU_DELIMITER:
2968         flags |= GST_VAAPI_DECODER_UNIT_FLAG_FRAME_START;
2969         /* fall-through */
2970     case GST_H264_NAL_FILLER_DATA:
2971         flags |= GST_VAAPI_DECODER_UNIT_FLAG_SKIP;
2972         break;
2973     case GST_H264_NAL_STREAM_END:
2974         flags |= GST_VAAPI_DECODER_UNIT_FLAG_STREAM_END;
2975         /* fall-through */
2976     case GST_H264_NAL_SEQ_END:
2977         flags |= GST_VAAPI_DECODER_UNIT_FLAG_FRAME_END;
2978         break;
2979     case GST_H264_NAL_SPS:
2980     case GST_H264_NAL_PPS:
2981         flags |= GST_VAAPI_DECODER_UNIT_FLAG_SKIP;
2982         /* fall-through */
2983     case GST_H264_NAL_SEI:
2984         flags |= GST_VAAPI_DECODER_UNIT_FLAG_FRAME_START;
2985         break;
2986     case GST_H264_NAL_SLICE_IDR:
2987     case GST_H264_NAL_SLICE:
2988         flags |= GST_VAAPI_DECODER_UNIT_FLAG_SLICE;
2989         if (is_new_picture(pi, priv->prev_slice_pi))
2990             flags |= GST_VAAPI_DECODER_UNIT_FLAG_FRAME_START;
2991         gst_vaapi_parser_info_h264_replace(&priv->prev_slice_pi, pi);
2992         break;
2993     default:
2994         if (pi->nalu.type >= 14 && pi->nalu.type <= 18)
2995             flags |= GST_VAAPI_DECODER_UNIT_FLAG_FRAME_START;
2996         break;
2997     }
2998     GST_VAAPI_DECODER_UNIT_FLAG_SET(unit, flags);
2999
3000     pi->nalu.data = NULL;
3001     return GST_VAAPI_DECODER_STATUS_SUCCESS;
3002 }
3003
3004 static GstVaapiDecoderStatus
3005 gst_vaapi_decoder_h264_decode(GstVaapiDecoder *base_decoder,
3006     GstVaapiDecoderUnit *unit)
3007 {
3008     GstVaapiDecoderH264 * const decoder =
3009         GST_VAAPI_DECODER_H264_CAST(base_decoder);
3010     GstVaapiDecoderStatus status;
3011
3012     status = ensure_decoder(decoder);
3013     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
3014         return status;
3015     return decode_unit(decoder, unit);
3016 }
3017
3018 static GstVaapiDecoderStatus
3019 gst_vaapi_decoder_h264_start_frame(GstVaapiDecoder *base_decoder,
3020     GstVaapiDecoderUnit *unit)
3021 {
3022     GstVaapiDecoderH264 * const decoder =
3023         GST_VAAPI_DECODER_H264_CAST(base_decoder);
3024
3025     return decode_picture(decoder, unit);
3026 }
3027
3028 static GstVaapiDecoderStatus
3029 gst_vaapi_decoder_h264_end_frame(GstVaapiDecoder *base_decoder)
3030 {
3031     GstVaapiDecoderH264 * const decoder =
3032         GST_VAAPI_DECODER_H264_CAST(base_decoder);
3033
3034     return decode_current_picture(decoder);
3035 }
3036
3037 static GstVaapiDecoderStatus
3038 gst_vaapi_decoder_h264_flush(GstVaapiDecoder *base_decoder)
3039 {
3040     GstVaapiDecoderH264 * const decoder =
3041         GST_VAAPI_DECODER_H264_CAST(base_decoder);
3042
3043     dpb_flush(decoder);
3044     return GST_VAAPI_DECODER_STATUS_SUCCESS;
3045 }
3046
3047 static void
3048 gst_vaapi_decoder_h264_finalize(GObject *object)
3049 {
3050     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264_CAST(object);
3051
3052     gst_vaapi_decoder_h264_destroy(decoder);
3053
3054     G_OBJECT_CLASS(gst_vaapi_decoder_h264_parent_class)->finalize(object);
3055 }
3056
3057 static void
3058 gst_vaapi_decoder_h264_constructed(GObject *object)
3059 {
3060     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264_CAST(object);
3061     GstVaapiDecoderH264Private * const priv = decoder->priv;
3062     GObjectClass *parent_class;
3063
3064     parent_class = G_OBJECT_CLASS(gst_vaapi_decoder_h264_parent_class);
3065     if (parent_class->constructed)
3066         parent_class->constructed(object);
3067
3068     priv->is_constructed = gst_vaapi_decoder_h264_create(decoder);
3069 }
3070
3071 static void
3072 gst_vaapi_decoder_h264_class_init(GstVaapiDecoderH264Class *klass)
3073 {
3074     GObjectClass * const object_class = G_OBJECT_CLASS(klass);
3075     GstVaapiDecoderClass * const decoder_class = GST_VAAPI_DECODER_CLASS(klass);
3076
3077     g_type_class_add_private(klass, sizeof(GstVaapiDecoderH264Private));
3078
3079     object_class->finalize      = gst_vaapi_decoder_h264_finalize;
3080     object_class->constructed   = gst_vaapi_decoder_h264_constructed;
3081
3082     decoder_class->parse        = gst_vaapi_decoder_h264_parse;
3083     decoder_class->decode       = gst_vaapi_decoder_h264_decode;
3084     decoder_class->start_frame  = gst_vaapi_decoder_h264_start_frame;
3085     decoder_class->end_frame    = gst_vaapi_decoder_h264_end_frame;
3086     decoder_class->flush        = gst_vaapi_decoder_h264_flush;
3087
3088     decoder_class->decode_codec_data =
3089         gst_vaapi_decoder_h264_decode_codec_data;
3090 }
3091
3092 static void
3093 gst_vaapi_decoder_h264_init(GstVaapiDecoderH264 *decoder)
3094 {
3095     GstVaapiDecoderH264Private *priv;
3096
3097     priv                        = GST_VAAPI_DECODER_H264_GET_PRIVATE(decoder);
3098     decoder->priv               = priv;
3099     priv->profile               = GST_VAAPI_PROFILE_UNKNOWN;
3100     priv->entrypoint            = GST_VAAPI_ENTRYPOINT_VLD;
3101     priv->chroma_type           = GST_VAAPI_CHROMA_TYPE_YUV420;
3102     priv->prev_pic_structure    = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
3103     priv->progressive_sequence  = TRUE;
3104 }
3105
3106 /**
3107  * gst_vaapi_decoder_h264_new:
3108  * @display: a #GstVaapiDisplay
3109  * @caps: a #GstCaps holding codec information
3110  *
3111  * Creates a new #GstVaapiDecoder for MPEG-2 decoding.  The @caps can
3112  * hold extra information like codec-data and pictured coded size.
3113  *
3114  * Return value: the newly allocated #GstVaapiDecoder object
3115  */
3116 GstVaapiDecoder *
3117 gst_vaapi_decoder_h264_new(GstVaapiDisplay *display, GstCaps *caps)
3118 {
3119     GstVaapiDecoderH264 *decoder;
3120
3121     g_return_val_if_fail(GST_VAAPI_IS_DISPLAY(display), NULL);
3122     g_return_val_if_fail(GST_IS_CAPS(caps), NULL);
3123
3124     decoder = g_object_new(
3125         GST_VAAPI_TYPE_DECODER_H264,
3126         "display",      display,
3127         "caps",         caps,
3128         NULL
3129     );
3130     if (!decoder->priv->is_constructed) {
3131         g_object_unref(decoder);
3132         return NULL;
3133     }
3134     return GST_VAAPI_DECODER_CAST(decoder);
3135 }