gst-libs/gst/vaapi/gstvaapidecoder_h264.c

   1 /*
   2  *  gstvaapidecoder_h264.c - H.264 decoder
   3  *
   4  *  Copyright (C) 2011-2013 Intel Corporation
   5  *
   6  *  This library is free software; you can redistribute it and/or
   7  *  modify it under the terms of the GNU Lesser General Public License
   8  *  as published by the Free Software Foundation; either version 2.1
   9  *  of the License, or (at your option) any later version.
  10  *
  11  *  This library is distributed in the hope that it will be useful,
  12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  *  Lesser General Public License for more details.
  15  *
  16  *  You should have received a copy of the GNU Lesser General Public
  17  *  License along with this library; if not, write to the Free
  18  *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  19  *  Boston, MA 02110-1301 USA
  20  */
  21
  22 /**
  23  * SECTION:gstvaapidecoder_h264
  24  * @short_description: H.264 decoder
  25  */
  26
  27 #include "sysdeps.h"
  28 #include <string.h>
  29 #include <gst/base/gstadapter.h>
  30 #include <gst/codecparsers/gsth264parser.h>
  31 #include "gstvaapidecoder_h264.h"
  32 #include "gstvaapidecoder_objects.h"
  33 #include "gstvaapidecoder_priv.h"
  34 #include "gstvaapidisplay_priv.h"
  35 #include "gstvaapiobject_priv.h"
  36
  37 #define DEBUG 1
  38 #include "gstvaapidebug.h"
  39
  40 /* Defined to 1 if strict ordering of DPB is needed. Only useful for debug */
  41 #define USE_STRICT_DPB_ORDERING 0
  42
  43 typedef struct _GstVaapiFrameStore              GstVaapiFrameStore;
  44 typedef struct _GstVaapiFrameStoreClass         GstVaapiFrameStoreClass;
  45 typedef struct _GstVaapiParserInfoH264          GstVaapiParserInfoH264;
  46 typedef struct _GstVaapiPictureH264             GstVaapiPictureH264;
  47
  48 // Used for field_poc[]
  49 #define TOP_FIELD       0
  50 #define BOTTOM_FIELD    1
  51
  52 /* ------------------------------------------------------------------------- */
  53 /* --- H.264 Parser Info                                                 --- */
  54 /* ------------------------------------------------------------------------- */
  55
  56 #define GST_VAAPI_PARSER_INFO_H264(obj) \
  57     ((GstVaapiParserInfoH264 *)(obj))
  58
  59 struct _GstVaapiParserInfoH264 {
  60     GstVaapiMiniObject  parent_instance;
  61     GstH264NalUnit      nalu;
  62     union {
  63         GstH264SPS      sps;
  64         GstH264PPS      pps;
  65         GstH264SliceHdr slice_hdr;
  66     }                   data;
  67 };
  68
  69 static inline const GstVaapiMiniObjectClass *
  70 gst_vaapi_parser_info_h264_class(void)
  71 {
  72     static const GstVaapiMiniObjectClass GstVaapiParserInfoH264Class = {
  73         sizeof(GstVaapiParserInfoH264),
  74         NULL
  75     };
  76     return &GstVaapiParserInfoH264Class;
  77 }
  78
  79 static inline GstVaapiParserInfoH264 *
  80 gst_vaapi_parser_info_h264_new(void)
  81 {
  82     return (GstVaapiParserInfoH264 *)
  83         gst_vaapi_mini_object_new(gst_vaapi_parser_info_h264_class());
  84 }
  85
  86 #define gst_vaapi_parser_info_h264_ref(pi) \
  87     gst_vaapi_mini_object_ref(GST_VAAPI_MINI_OBJECT(pi))
  88
  89 #define gst_vaapi_parser_info_h264_unref(pi) \
  90     gst_vaapi_mini_object_unref(GST_VAAPI_MINI_OBJECT(pi))
  91
  92 #define gst_vaapi_parser_info_h264_replace(old_pi_ptr, new_pi)          \
  93     gst_vaapi_mini_object_replace((GstVaapiMiniObject **)(old_pi_ptr),  \
  94         (GstVaapiMiniObject *)(new_pi))
  95
  96 /* ------------------------------------------------------------------------- */
  97 /* --- H.264 Pictures                                                    --- */
  98 /* ------------------------------------------------------------------------- */
  99
 100 /*
 101  * Extended picture flags:
 102  *
 103  * @GST_VAAPI_PICTURE_FLAG_IDR: flag that specifies an IDR picture
 104  * @GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE: flag that specifies
 105  *     "used for short-term reference"
 106  * @GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE: flag that specifies
 107  *     "used for long-term reference"
 108  * @GST_VAAPI_PICTURE_FLAGS_REFERENCE: mask covering any kind of
 109  *     reference picture (short-term reference or long-term reference)
 110  */
 111 enum {
 112     GST_VAAPI_PICTURE_FLAG_IDR = (GST_VAAPI_PICTURE_FLAG_LAST << 0),
 113
 114     GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE = (
 115         GST_VAAPI_PICTURE_FLAG_REFERENCE),
 116     GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE = (
 117         GST_VAAPI_PICTURE_FLAG_REFERENCE | (GST_VAAPI_PICTURE_FLAG_LAST << 1)),
 118     GST_VAAPI_PICTURE_FLAGS_REFERENCE = (
 119         GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE |
 120         GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE),
 121 };
 122
 123 #define GST_VAAPI_PICTURE_IS_IDR(picture) \
 124     (GST_VAAPI_PICTURE_FLAG_IS_SET(picture, GST_VAAPI_PICTURE_FLAG_IDR))
 125
 126 #define GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture)      \
 127     ((GST_VAAPI_PICTURE_FLAGS(picture) &                        \
 128       GST_VAAPI_PICTURE_FLAGS_REFERENCE) ==                     \
 129      GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE)
 130
 131 #define GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture)       \
 132     ((GST_VAAPI_PICTURE_FLAGS(picture) &                        \
 133       GST_VAAPI_PICTURE_FLAGS_REFERENCE) ==                     \
 134      GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE)
 135
 136 struct _GstVaapiPictureH264 {
 137     GstVaapiPicture             base;
 138     GstH264PPS                 *pps;
 139     GstH264SliceHdr            *last_slice_hdr;
 140     guint                       structure;
 141     gint32                      field_poc[2];
 142     gint32                      frame_num;              // Original frame_num from slice_header()
 143     gint32                      frame_num_wrap;         // Temporary for ref pic marking: FrameNumWrap
 144     gint32                      long_term_frame_idx;    // Temporary for ref pic marking: LongTermFrameIdx
 145     gint32                      pic_num;                // Temporary for ref pic marking: PicNum
 146     gint32                      long_term_pic_num;      // Temporary for ref pic marking: LongTermPicNum
 147     GstVaapiPictureH264        *other_field;            // Temporary for ref pic marking: other field in the same frame store
 148     guint                       output_flag             : 1;
 149     guint                       output_needed           : 1;
 150 };
 151
 152 GST_VAAPI_CODEC_DEFINE_TYPE(GstVaapiPictureH264, gst_vaapi_picture_h264);
 153
 154 void
 155 gst_vaapi_picture_h264_destroy(GstVaapiPictureH264 *picture)
 156 {
 157     gst_vaapi_picture_destroy(GST_VAAPI_PICTURE(picture));
 158 }
 159
 160 gboolean
 161 gst_vaapi_picture_h264_create(
 162     GstVaapiPictureH264                      *picture,
 163     const GstVaapiCodecObjectConstructorArgs *args
 164 )
 165 {
 166     if (!gst_vaapi_picture_create(GST_VAAPI_PICTURE(picture), args))
 167         return FALSE;
 168
 169     picture->field_poc[0]       = G_MAXINT32;
 170     picture->field_poc[1]       = G_MAXINT32;
 171     picture->output_needed      = FALSE;
 172     return TRUE;
 173 }
 174
 175 static inline GstVaapiPictureH264 *
 176 gst_vaapi_picture_h264_new(GstVaapiDecoderH264 *decoder)
 177 {
 178     return (GstVaapiPictureH264 *)gst_vaapi_codec_object_new(
 179         &GstVaapiPictureH264Class,
 180         GST_VAAPI_CODEC_BASE(decoder),
 181         NULL, sizeof(VAPictureParameterBufferH264),
 182         NULL, 0,
 183         0);
 184 }
 185
 186 static inline void
 187 gst_vaapi_picture_h264_set_reference(
 188     GstVaapiPictureH264 *picture,
 189     guint                reference_flags,
 190     gboolean             other_field
 191 )
 192 {
 193     if (!picture)
 194         return;
 195     GST_VAAPI_PICTURE_FLAG_UNSET(picture, GST_VAAPI_PICTURE_FLAGS_REFERENCE);
 196     GST_VAAPI_PICTURE_FLAG_SET(picture, reference_flags);
 197
 198     if (!other_field || !(picture = picture->other_field))
 199         return;
 200     GST_VAAPI_PICTURE_FLAG_UNSET(picture, GST_VAAPI_PICTURE_FLAGS_REFERENCE);
 201     GST_VAAPI_PICTURE_FLAG_SET(picture, reference_flags);
 202 }
 203
 204 static inline GstVaapiPictureH264 *
 205 gst_vaapi_picture_h264_new_field(GstVaapiPictureH264 *picture)
 206 {
 207     g_return_val_if_fail(picture, NULL);
 208
 209     return (GstVaapiPictureH264 *)gst_vaapi_picture_new_field(&picture->base);
 210 }
 211
 212 /* ------------------------------------------------------------------------- */
 213 /* --- Frame Buffers (DPB)                                               --- */
 214 /* ------------------------------------------------------------------------- */
 215
 216 struct _GstVaapiFrameStore {
 217     /*< private >*/
 218     GstVaapiMiniObject          parent_instance;
 219
 220     guint                       structure;
 221     GstVaapiPictureH264        *buffers[2];
 222     guint                       num_buffers;
 223     guint                       output_needed;
 224 };
 225
 226 static void
 227 gst_vaapi_frame_store_finalize(gpointer object)
 228 {
 229     GstVaapiFrameStore * const fs = object;
 230     guint i;
 231
 232     for (i = 0; i < fs->num_buffers; i++)
 233         gst_vaapi_picture_replace(&fs->buffers[i], NULL);
 234 }
 235
 236 static GstVaapiFrameStore *
 237 gst_vaapi_frame_store_new(GstVaapiPictureH264 *picture)
 238 {
 239     GstVaapiFrameStore *fs;
 240
 241     static const GstVaapiMiniObjectClass GstVaapiFrameStoreClass = {
 242         sizeof(GstVaapiFrameStore),
 243         gst_vaapi_frame_store_finalize
 244     };
 245
 246     fs = (GstVaapiFrameStore *)
 247         gst_vaapi_mini_object_new(&GstVaapiFrameStoreClass);
 248     if (!fs)
 249         return NULL;
 250
 251     fs->structure       = picture->structure;
 252     fs->buffers[0]      = gst_vaapi_picture_ref(picture);
 253     fs->buffers[1]      = NULL;
 254     fs->num_buffers     = 1;
 255     fs->output_needed   = picture->output_needed;
 256     return fs;
 257 }
 258
 259 static gboolean
 260 gst_vaapi_frame_store_add(GstVaapiFrameStore *fs, GstVaapiPictureH264 *picture)
 261 {
 262     guint field;
 263
 264     g_return_val_if_fail(fs->num_buffers == 1, FALSE);
 265     g_return_val_if_fail(!GST_VAAPI_PICTURE_IS_FRAME(picture), FALSE);
 266     g_return_val_if_fail(!GST_VAAPI_PICTURE_IS_FIRST_FIELD(picture), FALSE);
 267
 268     gst_vaapi_picture_replace(&fs->buffers[fs->num_buffers++], picture);
 269     if (picture->output_flag) {
 270         picture->output_needed = TRUE;
 271         fs->output_needed++;
 272     }
 273
 274     fs->structure = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
 275
 276     field = picture->structure == GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD ?
 277         TOP_FIELD : BOTTOM_FIELD;
 278     g_return_val_if_fail(fs->buffers[0]->field_poc[field] == G_MAXINT32, FALSE);
 279     fs->buffers[0]->field_poc[field] = picture->field_poc[field];
 280     g_return_val_if_fail(picture->field_poc[!field] == G_MAXINT32, FALSE);
 281     picture->field_poc[!field] = fs->buffers[0]->field_poc[!field];
 282     return TRUE;
 283 }
 284
 285 static gboolean
 286 gst_vaapi_frame_store_split_fields(GstVaapiFrameStore *fs)
 287 {
 288     GstVaapiPictureH264 * const first_field = fs->buffers[0];
 289     GstVaapiPictureH264 *second_field;
 290
 291     g_return_val_if_fail(fs->num_buffers == 1, FALSE);
 292
 293     first_field->base.structure = GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD;
 294     GST_VAAPI_PICTURE_FLAG_SET(first_field, GST_VAAPI_PICTURE_FLAG_INTERLACED);
 295
 296     second_field = gst_vaapi_picture_h264_new_field(first_field);
 297     if (!second_field)
 298         return FALSE;
 299     gst_vaapi_picture_replace(&fs->buffers[fs->num_buffers++], second_field);
 300     gst_vaapi_picture_unref(second_field);
 301
 302     second_field->frame_num    = first_field->frame_num;
 303     second_field->field_poc[0] = first_field->field_poc[0];
 304     second_field->field_poc[1] = first_field->field_poc[1];
 305     second_field->output_flag  = first_field->output_flag;
 306     if (second_field->output_flag) {
 307         second_field->output_needed = TRUE;
 308         fs->output_needed++;
 309     }
 310     return TRUE;
 311 }
 312
 313 static inline gboolean
 314 gst_vaapi_frame_store_has_frame(GstVaapiFrameStore *fs)
 315 {
 316     return fs->structure == GST_VAAPI_PICTURE_STRUCTURE_FRAME;
 317 }
 318
 319 static inline gboolean
 320 gst_vaapi_frame_store_has_reference(GstVaapiFrameStore *fs)
 321 {
 322     guint i;
 323
 324     for (i = 0; i < fs->num_buffers; i++) {
 325         if (GST_VAAPI_PICTURE_IS_REFERENCE(fs->buffers[i]))
 326             return TRUE;
 327     }
 328     return FALSE;
 329 }
 330
 331 #define gst_vaapi_frame_store_ref(fs) \
 332     gst_vaapi_mini_object_ref(GST_VAAPI_MINI_OBJECT(fs))
 333
 334 #define gst_vaapi_frame_store_unref(fs) \
 335     gst_vaapi_mini_object_unref(GST_VAAPI_MINI_OBJECT(fs))
 336
 337 #define gst_vaapi_frame_store_replace(old_fs_p, new_fs)                 \
 338     gst_vaapi_mini_object_replace((GstVaapiMiniObject **)(old_fs_p),    \
 339         (GstVaapiMiniObject *)(new_fs))
 340
 341 /* ------------------------------------------------------------------------- */
 342 /* --- H.264 Decoder                                                     --- */
 343 /* ------------------------------------------------------------------------- */
 344
 345 G_DEFINE_TYPE(GstVaapiDecoderH264,
 346               gst_vaapi_decoder_h264,
 347               GST_VAAPI_TYPE_DECODER)
 348
 349 #define GST_VAAPI_DECODER_H264_CAST(decoder) \
 350     ((GstVaapiDecoderH264 *)(decoder))
 351
 352 #define GST_VAAPI_DECODER_H264_GET_PRIVATE(obj)                 \
 353     (G_TYPE_INSTANCE_GET_PRIVATE((obj),                         \
 354                                  GST_VAAPI_TYPE_DECODER_H264,   \
 355                                  GstVaapiDecoderH264Private))
 356
 357 struct _GstVaapiDecoderH264Private {
 358     GstH264NalParser           *parser;
 359     GstVaapiPictureH264        *current_picture;
 360     GstVaapiParserInfoH264     *prev_slice_pi;
 361     GstVaapiFrameStore         *prev_frame;
 362     GstVaapiFrameStore         *dpb[16];
 363     guint                       dpb_count;
 364     guint                       dpb_size;
 365     GstVaapiProfile             profile;
 366     GstVaapiEntrypoint          entrypoint;
 367     GstVaapiChromaType          chroma_type;
 368     GstVaapiPictureH264        *short_ref[32];
 369     guint                       short_ref_count;
 370     GstVaapiPictureH264        *long_ref[32];
 371     guint                       long_ref_count;
 372     GstVaapiPictureH264        *RefPicList0[32];
 373     guint                       RefPicList0_count;
 374     GstVaapiPictureH264        *RefPicList1[32];
 375     guint                       RefPicList1_count;
 376     guint                       nal_length_size;
 377     guint                       mb_width;
 378     guint                       mb_height;
 379     gint32                      field_poc[2];           // 0:TopFieldOrderCnt / 1:BottomFieldOrderCnt
 380     gint32                      poc_msb;                // PicOrderCntMsb
 381     gint32                      poc_lsb;                // pic_order_cnt_lsb (from slice_header())
 382     gint32                      prev_poc_msb;           // prevPicOrderCntMsb
 383     gint32                      prev_poc_lsb;           // prevPicOrderCntLsb
 384     gint32                      frame_num_offset;       // FrameNumOffset
 385     gint32                      frame_num;              // frame_num (from slice_header())
 386     gint32                      prev_frame_num;         // prevFrameNum
 387     gboolean                    prev_pic_has_mmco5;     // prevMmco5Pic
 388     gboolean                    prev_pic_structure;     // previous picture structure
 389     guint                       is_constructed          : 1;
 390     guint                       is_opened               : 1;
 391     guint                       is_avcC                 : 1;
 392     guint                       got_sps                 : 1;
 393     guint                       got_pps                 : 1;
 394     guint                       has_context             : 1;
 395     guint                       progressive_sequence    : 1;
 396 };
 397
 398 static gboolean
 399 exec_ref_pic_marking(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture);
 400
 401 /* Get number of reference frames to use */
 402 static guint
 403 get_max_dec_frame_buffering(GstH264SPS *sps)
 404 {
 405     guint max_dec_frame_buffering, MaxDpbMbs, PicSizeMbs;
 406
 407     /* Table A-1 - Level limits */
 408     switch (sps->level_idc) {
 409     case 10: MaxDpbMbs = 396;    break;
 410     case 11: MaxDpbMbs = 900;    break;
 411     case 12: MaxDpbMbs = 2376;   break;
 412     case 13: MaxDpbMbs = 2376;   break;
 413     case 20: MaxDpbMbs = 2376;   break;
 414     case 21: MaxDpbMbs = 4752;   break;
 415     case 22: MaxDpbMbs = 8100;   break;
 416     case 30: MaxDpbMbs = 8100;   break;
 417     case 31: MaxDpbMbs = 18000;  break;
 418     case 32: MaxDpbMbs = 20480;  break;
 419     case 40: MaxDpbMbs = 32768;  break;
 420     case 41: MaxDpbMbs = 32768;  break;
 421     case 42: MaxDpbMbs = 34816;  break;
 422     case 50: MaxDpbMbs = 110400; break;
 423     case 51: MaxDpbMbs = 184320; break;
 424     default:
 425         g_assert(0 && "unhandled level");
 426         break;
 427     }
 428
 429     PicSizeMbs = ((sps->pic_width_in_mbs_minus1 + 1) *
 430                   (sps->pic_height_in_map_units_minus1 + 1) *
 431                   (sps->frame_mbs_only_flag ? 1 : 2));
 432     max_dec_frame_buffering = MaxDpbMbs / PicSizeMbs;
 433
 434     /* VUI parameters */
 435     if (sps->vui_parameters_present_flag) {
 436         GstH264VUIParams * const vui_params = &sps->vui_parameters;
 437         if (vui_params->bitstream_restriction_flag)
 438             max_dec_frame_buffering = vui_params->max_dec_frame_buffering;
 439         else {
 440             switch (sps->profile_idc) {
 441             case 44:  // CAVLC 4:4:4 Intra profile
 442             case GST_H264_PROFILE_SCALABLE_HIGH:
 443             case GST_H264_PROFILE_HIGH:
 444             case GST_H264_PROFILE_HIGH10:
 445             case GST_H264_PROFILE_HIGH_422:
 446             case GST_H264_PROFILE_HIGH_444:
 447                 if (sps->constraint_set3_flag)
 448                     max_dec_frame_buffering = 0;
 449                 break;
 450             }
 451         }
 452     }
 453
 454     if (max_dec_frame_buffering > 16)
 455         max_dec_frame_buffering = 16;
 456     else if (max_dec_frame_buffering < sps->num_ref_frames)
 457         max_dec_frame_buffering = sps->num_ref_frames;
 458     return MAX(1, max_dec_frame_buffering);
 459 }
 460
 461 static void
 462 array_remove_index_fast(void *array, guint *array_length_ptr, guint index)
 463 {
 464     gpointer * const entries = array;
 465     guint num_entries = *array_length_ptr;
 466
 467     g_return_if_fail(index < num_entries);
 468
 469     if (index != --num_entries)
 470         entries[index] = entries[num_entries];
 471     entries[num_entries] = NULL;
 472     *array_length_ptr = num_entries;
 473 }
 474
 475 #if 1
 476 static inline void
 477 array_remove_index(void *array, guint *array_length_ptr, guint index)
 478 {
 479     array_remove_index_fast(array, array_length_ptr, index);
 480 }
 481 #else
 482 static void
 483 array_remove_index(void *array, guint *array_length_ptr, guint index)
 484 {
 485     gpointer * const entries = array;
 486     const guint num_entries = *array_length_ptr - 1;
 487     guint i;
 488
 489     g_return_if_fail(index <= num_entries);
 490
 491     for (i = index; i < num_entries; i++)
 492         entries[i] = entries[i + 1];
 493     entries[num_entries] = NULL;
 494     *array_length_ptr = num_entries;
 495 }
 496 #endif
 497
 498 #define ARRAY_REMOVE_INDEX(array, index) \
 499     array_remove_index(array, &array##_count, index)
 500
 501 static void
 502 dpb_remove_index(GstVaapiDecoderH264 *decoder, guint index)
 503 {
 504     GstVaapiDecoderH264Private * const priv = decoder->priv;
 505     guint i, num_frames = --priv->dpb_count;
 506
 507     if (USE_STRICT_DPB_ORDERING) {
 508         for (i = index; i < num_frames; i++)
 509             gst_vaapi_frame_store_replace(&priv->dpb[i], priv->dpb[i + 1]);
 510     }
 511     else if (index != num_frames)
 512         gst_vaapi_frame_store_replace(&priv->dpb[index], priv->dpb[num_frames]);
 513     gst_vaapi_frame_store_replace(&priv->dpb[num_frames], NULL);
 514 }
 515
 516 static gboolean
 517 dpb_output(
 518     GstVaapiDecoderH264 *decoder,
 519     GstVaapiFrameStore  *fs,
 520     GstVaapiPictureH264 *picture
 521 )
 522 {
 523     picture->output_needed = FALSE;
 524
 525     if (fs) {
 526         if (--fs->output_needed > 0)
 527             return TRUE;
 528         picture = fs->buffers[0];
 529     }
 530
 531     /* XXX: update cropping rectangle */
 532     return gst_vaapi_picture_output(GST_VAAPI_PICTURE_CAST(picture));
 533 }
 534
 535 static inline void
 536 dpb_evict(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture, guint i)
 537 {
 538     GstVaapiFrameStore * const fs = decoder->priv->dpb[i];
 539
 540     if (!fs->output_needed && !gst_vaapi_frame_store_has_reference(fs))
 541         dpb_remove_index(decoder, i);
 542 }
 543
 544 static gboolean
 545 dpb_bump(GstVaapiDecoderH264 *decoder)
 546 {
 547     GstVaapiDecoderH264Private * const priv = decoder->priv;
 548     GstVaapiPictureH264 *found_picture = NULL;
 549     guint i, j, found_index;
 550     gboolean success;
 551
 552     for (i = 0; i < priv->dpb_count; i++) {
 553         GstVaapiFrameStore * const fs = priv->dpb[i];
 554         if (!fs->output_needed)
 555             continue;
 556         for (j = 0; j < fs->num_buffers; j++) {
 557             GstVaapiPictureH264 * const picture = fs->buffers[j];
 558             if (!picture->output_needed)
 559                 continue;
 560             if (!found_picture || found_picture->base.poc > picture->base.poc)
 561                 found_picture = picture, found_index = i;
 562         }
 563     }
 564     if (!found_picture)
 565         return FALSE;
 566
 567     success = dpb_output(decoder, priv->dpb[found_index], found_picture);
 568     dpb_evict(decoder, found_picture, found_index);
 569     return success;
 570 }
 571
 572 static void
 573 dpb_clear(GstVaapiDecoderH264 *decoder)
 574 {
 575     GstVaapiDecoderH264Private * const priv = decoder->priv;
 576     guint i;
 577
 578     for (i = 0; i < priv->dpb_count; i++)
 579         gst_vaapi_frame_store_replace(&priv->dpb[i], NULL);
 580     priv->dpb_count = 0;
 581
 582     gst_vaapi_frame_store_replace(&priv->prev_frame, NULL);
 583 }
 584
 585 static void
 586 dpb_flush(GstVaapiDecoderH264 *decoder)
 587 {
 588     while (dpb_bump(decoder))
 589         ;
 590     dpb_clear(decoder);
 591 }
 592
 593 static gboolean
 594 dpb_add(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
 595 {
 596     GstVaapiDecoderH264Private * const priv = decoder->priv;
 597     GstVaapiFrameStore *fs;
 598     guint i, j;
 599
 600     // Remove all unused pictures
 601     if (!GST_VAAPI_PICTURE_IS_IDR(picture)) {
 602         i = 0;
 603         while (i < priv->dpb_count) {
 604             GstVaapiFrameStore * const fs = priv->dpb[i];
 605             if (!fs->output_needed && !gst_vaapi_frame_store_has_reference(fs))
 606                 dpb_remove_index(decoder, i);
 607             else
 608                 i++;
 609         }
 610     }
 611
 612     // Check if picture is the second field and the first field is still in DPB
 613     fs = priv->prev_frame;
 614     if (fs && !gst_vaapi_frame_store_has_frame(fs))
 615         return gst_vaapi_frame_store_add(fs, picture);
 616
 617     // Create new frame store, and split fields if necessary
 618     fs = gst_vaapi_frame_store_new(picture);
 619     if (!fs)
 620         return FALSE;
 621     gst_vaapi_frame_store_replace(&priv->prev_frame, fs);
 622     gst_vaapi_frame_store_unref(fs);
 623
 624     if (!priv->progressive_sequence && gst_vaapi_frame_store_has_frame(fs)) {
 625         if (!gst_vaapi_frame_store_split_fields(fs))
 626             return FALSE;
 627     }
 628
 629     // C.4.5.1 - Storage and marking of a reference decoded picture into the DPB
 630     if (GST_VAAPI_PICTURE_IS_REFERENCE(picture)) {
 631         while (priv->dpb_count == priv->dpb_size) {
 632             if (!dpb_bump(decoder))
 633                 return FALSE;
 634         }
 635         gst_vaapi_frame_store_replace(&priv->dpb[priv->dpb_count++], fs);
 636         if (picture->output_flag) {
 637             picture->output_needed = TRUE;
 638             fs->output_needed++;
 639         }
 640     }
 641
 642     // C.4.5.2 - Storage and marking of a non-reference decoded picture into the DPB
 643     else {
 644         if (!picture->output_flag)
 645             return TRUE;
 646         while (priv->dpb_count == priv->dpb_size) {
 647             gboolean found_picture = FALSE;
 648             for (i = 0; !found_picture && i < priv->dpb_count; i++) {
 649                 GstVaapiFrameStore * const fs = priv->dpb[i];
 650                 if (!fs->output_needed)
 651                     continue;
 652                 for (j = 0; !found_picture && j < fs->num_buffers; j++)
 653                     found_picture = fs->buffers[j]->output_needed &&
 654                         fs->buffers[j]->base.poc < picture->base.poc;
 655             }
 656             if (!found_picture)
 657                 return dpb_output(decoder, NULL, picture);
 658             if (!dpb_bump(decoder))
 659                 return FALSE;
 660         }
 661         gst_vaapi_frame_store_replace(&priv->dpb[priv->dpb_count++], fs);
 662         picture->output_needed = TRUE;
 663         fs->output_needed++;
 664     }
 665     return TRUE;
 666 }
 667
 668 static inline void
 669 dpb_reset(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 670 {
 671     GstVaapiDecoderH264Private * const priv = decoder->priv;
 672
 673     priv->dpb_size = get_max_dec_frame_buffering(sps);
 674     GST_DEBUG("DPB size %u", priv->dpb_size);
 675 }
 676
 677 static GstVaapiDecoderStatus
 678 get_status(GstH264ParserResult result)
 679 {
 680     GstVaapiDecoderStatus status;
 681
 682     switch (result) {
 683     case GST_H264_PARSER_OK:
 684         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
 685         break;
 686     case GST_H264_PARSER_NO_NAL_END:
 687         status = GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
 688         break;
 689     case GST_H264_PARSER_ERROR:
 690         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
 691         break;
 692     default:
 693         status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 694         break;
 695     }
 696     return status;
 697 }
 698
 699 static void
 700 gst_vaapi_decoder_h264_close(GstVaapiDecoderH264 *decoder)
 701 {
 702     GstVaapiDecoderH264Private * const priv = decoder->priv;
 703
 704     gst_vaapi_picture_replace(&priv->current_picture, NULL);
 705     gst_vaapi_parser_info_h264_replace(&priv->prev_slice_pi, NULL);
 706
 707     dpb_clear(decoder);
 708
 709     if (priv->parser) {
 710         gst_h264_nal_parser_free(priv->parser);
 711         priv->parser = NULL;
 712     }
 713 }
 714
 715 static gboolean
 716 gst_vaapi_decoder_h264_open(GstVaapiDecoderH264 *decoder)
 717 {
 718     GstVaapiDecoderH264Private * const priv = decoder->priv;
 719
 720     gst_vaapi_decoder_h264_close(decoder);
 721
 722     priv->parser = gst_h264_nal_parser_new();
 723     if (!priv->parser)
 724         return FALSE;
 725     return TRUE;
 726 }
 727
 728 static void
 729 gst_vaapi_decoder_h264_destroy(GstVaapiDecoderH264 *decoder)
 730 {
 731     gst_vaapi_decoder_h264_close(decoder);
 732 }
 733
 734 static gboolean
 735 gst_vaapi_decoder_h264_create(GstVaapiDecoderH264 *decoder)
 736 {
 737     if (!GST_VAAPI_DECODER_CODEC(decoder))
 738         return FALSE;
 739     return TRUE;
 740 }
 741
 742 static guint
 743 h264_get_profile(GstH264SPS *sps)
 744 {
 745     guint profile = 0;
 746
 747     switch (sps->profile_idc) {
 748     case GST_H264_PROFILE_BASELINE:
 749         profile = GST_VAAPI_PROFILE_H264_BASELINE;
 750         break;
 751     case GST_H264_PROFILE_MAIN:
 752         profile = GST_VAAPI_PROFILE_H264_MAIN;
 753         break;
 754     case GST_H264_PROFILE_HIGH:
 755         profile = GST_VAAPI_PROFILE_H264_HIGH;
 756         break;
 757     }
 758     return profile;
 759 }
 760
 761 static guint
 762 h264_get_chroma_type(GstH264SPS *sps)
 763 {
 764     guint chroma_type = 0;
 765
 766     switch (sps->chroma_format_idc) {
 767     case 1:
 768         chroma_type = GST_VAAPI_CHROMA_TYPE_YUV420;
 769         break;
 770     case 2:
 771         chroma_type = GST_VAAPI_CHROMA_TYPE_YUV422;
 772         break;
 773     case 3:
 774         if (!sps->separate_colour_plane_flag)
 775             chroma_type = GST_VAAPI_CHROMA_TYPE_YUV444;
 776         break;
 777     }
 778     return chroma_type;
 779 }
 780
 781 static GstVaapiProfile
 782 get_profile(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 783 {
 784     GstVaapiDecoderH264Private * const priv = decoder->priv;
 785     GstVaapiDisplay * const display = GST_VAAPI_DECODER_DISPLAY(decoder);
 786     GstVaapiProfile profile, profiles[2];
 787     guint i, n_profiles = 0;
 788
 789     profile = h264_get_profile(sps);
 790     if (!profile)
 791         return GST_VAAPI_PROFILE_UNKNOWN;
 792
 793     profiles[n_profiles++] = profile;
 794     switch (profile) {
 795     case GST_VAAPI_PROFILE_H264_MAIN:
 796         profiles[n_profiles++] = GST_VAAPI_PROFILE_H264_HIGH;
 797         break;
 798     default:
 799         break;
 800     }
 801
 802     /* If the preferred profile (profiles[0]) matches one that we already
 803        found, then just return it now instead of searching for it again */
 804     if (profiles[0] == priv->profile)
 805         return priv->profile;
 806
 807     for (i = 0; i < n_profiles; i++) {
 808         if (gst_vaapi_display_has_decoder(display, profiles[i], priv->entrypoint))
 809             return profiles[i];
 810     }
 811     return GST_VAAPI_PROFILE_UNKNOWN;
 812 }
 813
 814 static GstVaapiDecoderStatus
 815 ensure_context(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 816 {
 817     GstVaapiDecoder * const base_decoder = GST_VAAPI_DECODER_CAST(decoder);
 818     GstVaapiDecoderH264Private * const priv = decoder->priv;
 819     GstVaapiContextInfo info;
 820     GstVaapiProfile profile;
 821     GstVaapiChromaType chroma_type;
 822     gboolean reset_context = FALSE;
 823     guint mb_width, mb_height;
 824
 825     profile = get_profile(decoder, sps);
 826     if (!profile) {
 827         GST_ERROR("unsupported profile_idc %u", sps->profile_idc);
 828         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_PROFILE;
 829     }
 830
 831     if (priv->profile != profile) {
 832         GST_DEBUG("profile changed");
 833         reset_context = TRUE;
 834         priv->profile = profile;
 835     }
 836
 837     chroma_type = h264_get_chroma_type(sps);
 838     if (!chroma_type || chroma_type != GST_VAAPI_CHROMA_TYPE_YUV420) {
 839         GST_ERROR("unsupported chroma_format_idc %u", sps->chroma_format_idc);
 840         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;
 841     }
 842
 843     if (priv->chroma_type != chroma_type) {
 844         GST_DEBUG("chroma format changed");
 845         reset_context     = TRUE;
 846         priv->chroma_type = chroma_type;
 847     }
 848
 849     mb_width  = sps->pic_width_in_mbs_minus1 + 1;
 850     mb_height = (sps->pic_height_in_map_units_minus1 + 1) <<
 851         !sps->frame_mbs_only_flag;
 852     if (priv->mb_width != mb_width || priv->mb_height != mb_height) {
 853         GST_DEBUG("size changed");
 854         reset_context   = TRUE;
 855         priv->mb_width  = mb_width;
 856         priv->mb_height = mb_height;
 857     }
 858
 859     priv->progressive_sequence = sps->frame_mbs_only_flag;
 860 #if 0
 861     /* XXX: we only output complete frames for now */
 862     gst_vaapi_decoder_set_interlaced(base_decoder, !priv->progressive_sequence);
 863 #endif
 864
 865     gst_vaapi_decoder_set_pixel_aspect_ratio(
 866         base_decoder,
 867         sps->vui_parameters.par_n,
 868         sps->vui_parameters.par_d
 869     );
 870
 871     if (!reset_context && priv->has_context)
 872         return GST_VAAPI_DECODER_STATUS_SUCCESS;
 873
 874     /* XXX: fix surface size when cropping is implemented */
 875     info.profile    = priv->profile;
 876     info.entrypoint = priv->entrypoint;
 877     info.width      = sps->width;
 878     info.height     = sps->height;
 879     info.ref_frames = get_max_dec_frame_buffering(sps);
 880
 881     if (!gst_vaapi_decoder_ensure_context(GST_VAAPI_DECODER(decoder), &info))
 882         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 883     priv->has_context = TRUE;
 884
 885     /* Reset DPB */
 886     dpb_reset(decoder, sps);
 887     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 888 }
 889
 890 static void
 891 fill_iq_matrix_4x4(VAIQMatrixBufferH264 *iq_matrix, const GstH264PPS *pps)
 892 {
 893     const guint8 (* const ScalingList4x4)[6][16] = &pps->scaling_lists_4x4;
 894     guint i, j;
 895
 896     /* There are always 6 4x4 scaling lists */
 897     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList4x4) == 6);
 898     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList4x4[0]) == 16);
 899
 900     if (sizeof(iq_matrix->ScalingList4x4[0][0]) == 1)
 901         memcpy(iq_matrix->ScalingList4x4, *ScalingList4x4,
 902                sizeof(iq_matrix->ScalingList4x4));
 903     else {
 904         for (i = 0; i < G_N_ELEMENTS(iq_matrix->ScalingList4x4); i++) {
 905             for (j = 0; j < G_N_ELEMENTS(iq_matrix->ScalingList4x4[i]); j++)
 906                 iq_matrix->ScalingList4x4[i][j] = (*ScalingList4x4)[i][j];
 907         }
 908     }
 909 }
 910
 911 static void
 912 fill_iq_matrix_8x8(VAIQMatrixBufferH264 *iq_matrix, const GstH264PPS *pps)
 913 {
 914     const guint8 (* const ScalingList8x8)[6][64] = &pps->scaling_lists_8x8;
 915     const GstH264SPS * const sps = pps->sequence;
 916     guint i, j, n;
 917
 918     /* If chroma_format_idc != 3, there are up to 2 8x8 scaling lists */
 919     if (!pps->transform_8x8_mode_flag)
 920         return;
 921
 922     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList8x8) >= 2);
 923     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList8x8[0]) == 64);
 924
 925     if (sizeof(iq_matrix->ScalingList8x8[0][0]) == 1)
 926         memcpy(iq_matrix->ScalingList8x8, *ScalingList8x8,
 927                sizeof(iq_matrix->ScalingList8x8));
 928     else {
 929         n = (sps->chroma_format_idc != 3) ? 2 : 6;
 930         for (i = 0; i < n; i++) {
 931             for (j = 0; j < G_N_ELEMENTS(iq_matrix->ScalingList8x8[i]); j++)
 932                 iq_matrix->ScalingList8x8[i][j] = (*ScalingList8x8)[i][j];
 933         }
 934     }
 935 }
 936
 937 static GstVaapiDecoderStatus
 938 ensure_quant_matrix(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
 939 {
 940     GstVaapiPicture * const base_picture = &picture->base;
 941     GstH264PPS * const pps = picture->pps;
 942     GstH264SPS * const sps = pps->sequence;
 943     VAIQMatrixBufferH264 *iq_matrix;
 944
 945     base_picture->iq_matrix = GST_VAAPI_IQ_MATRIX_NEW(H264, decoder);
 946     if (!base_picture->iq_matrix) {
 947         GST_ERROR("failed to allocate IQ matrix");
 948         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
 949     }
 950     iq_matrix = base_picture->iq_matrix->param;
 951
 952     /* XXX: we can only support 4:2:0 or 4:2:2 since ScalingLists8x8[]
 953        is not large enough to hold lists for 4:4:4 */
 954     if (sps->chroma_format_idc == 3)
 955         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;
 956
 957     fill_iq_matrix_4x4(iq_matrix, pps);
 958     fill_iq_matrix_8x8(iq_matrix, pps);
 959
 960     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 961 }
 962
 963 static GstVaapiDecoderStatus
 964 decode_current_picture(GstVaapiDecoderH264 *decoder)
 965 {
 966     GstVaapiDecoderH264Private * const priv = decoder->priv;
 967     GstVaapiPictureH264 * const picture = priv->current_picture;
 968
 969     if (!picture)
 970         return GST_VAAPI_DECODER_STATUS_SUCCESS;
 971
 972     if (!exec_ref_pic_marking(decoder, picture))
 973         goto error;
 974     if (!dpb_add(decoder, picture))
 975         goto error;
 976     if (!gst_vaapi_picture_decode(GST_VAAPI_PICTURE_CAST(picture)))
 977         goto error;
 978     if (priv->prev_frame && gst_vaapi_frame_store_has_frame(priv->prev_frame))
 979         gst_vaapi_picture_replace(&priv->current_picture, NULL);
 980     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 981
 982 error:
 983     /* XXX: fix for cases where first field failed to be decoded */
 984     gst_vaapi_picture_replace(&priv->current_picture, NULL);
 985     return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 986 }
 987
 988 static GstVaapiDecoderStatus
 989 parse_sps(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnit *unit)
 990 {
 991     GstVaapiDecoderH264Private * const priv = decoder->priv;
 992     GstVaapiParserInfoH264 * const pi = unit->parsed_info;
 993     GstH264SPS * const sps = &pi->data.sps;
 994     GstH264ParserResult result;
 995
 996     GST_DEBUG("parse SPS");
 997
 998     /* Variables that don't have inferred values per the H.264
 999        standard but that should get a default value anyway */
1000     sps->log2_max_pic_order_cnt_lsb_minus4 = 0;
1001
1002     result = gst_h264_parser_parse_sps(priv->parser, &pi->nalu, sps, TRUE);
1003     if (result != GST_H264_PARSER_OK)
1004         return get_status(result);
1005
1006     priv->got_sps = TRUE;
1007     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1008 }
1009
1010 static GstVaapiDecoderStatus
1011 parse_pps(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnit *unit)
1012 {
1013     GstVaapiDecoderH264Private * const priv = decoder->priv;
1014     GstVaapiParserInfoH264 * const pi = unit->parsed_info;
1015     GstH264PPS * const pps = &pi->data.pps;
1016     GstH264ParserResult result;
1017
1018     GST_DEBUG("parse PPS");
1019
1020     /* Variables that don't have inferred values per the H.264
1021        standard but that should get a default value anyway */
1022     pps->slice_group_map_type = 0;
1023     pps->slice_group_change_rate_minus1 = 0;
1024
1025     result = gst_h264_parser_parse_pps(priv->parser, &pi->nalu, pps);
1026     if (result != GST_H264_PARSER_OK)
1027         return get_status(result);
1028
1029     priv->got_pps = TRUE;
1030     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1031 }
1032
1033 static GstVaapiDecoderStatus
1034 parse_sei(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnit *unit)
1035 {
1036     GstVaapiDecoderH264Private * const priv = decoder->priv;
1037     GstVaapiParserInfoH264 * const pi = unit->parsed_info;
1038     GstH264SEIMessage sei;
1039     GstH264ParserResult result;
1040
1041     GST_DEBUG("parse SEI");
1042
1043     memset(&sei, 0, sizeof(sei));
1044     result = gst_h264_parser_parse_sei(priv->parser, &pi->nalu, &sei);
1045     if (result != GST_H264_PARSER_OK) {
1046         GST_WARNING("failed to parse SEI, payload type:%d", sei.payloadType);
1047         return get_status(result);
1048     }
1049
1050     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1051 }
1052
1053 static GstVaapiDecoderStatus
1054 parse_slice(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnit *unit)
1055 {
1056     GstVaapiDecoderH264Private * const priv = decoder->priv;
1057     GstVaapiParserInfoH264 * const pi = unit->parsed_info;
1058     GstH264SliceHdr * const slice_hdr = &pi->data.slice_hdr;
1059     GstH264ParserResult result;
1060
1061     GST_DEBUG("parse slice");
1062
1063     /* Variables that don't have inferred values per the H.264
1064        standard but that should get a default value anyway */
1065     slice_hdr->cabac_init_idc = 0;
1066     slice_hdr->direct_spatial_mv_pred_flag = 0;
1067
1068     result = gst_h264_parser_parse_slice_hdr(priv->parser, &pi->nalu,
1069         slice_hdr, TRUE, TRUE);
1070     if (result != GST_H264_PARSER_OK)
1071         return get_status(result);
1072
1073     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1074 }
1075
1076 static GstVaapiDecoderStatus
1077 decode_sequence_end(GstVaapiDecoderH264 *decoder)
1078 {
1079     GstVaapiDecoderStatus status;
1080
1081     GST_DEBUG("decode sequence-end");
1082
1083     status = decode_current_picture(decoder);
1084     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
1085         return status;
1086
1087     dpb_flush(decoder);
1088     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1089 }
1090
1091 /* 8.2.1.1 - Decoding process for picture order count type 0 */
1092 static void
1093 init_picture_poc_0(
1094     GstVaapiDecoderH264 *decoder,
1095     GstVaapiPictureH264 *picture,
1096     GstH264SliceHdr     *slice_hdr
1097 )
1098 {
1099     GstVaapiDecoderH264Private * const priv = decoder->priv;
1100     GstH264PPS * const pps = slice_hdr->pps;
1101     GstH264SPS * const sps = pps->sequence;
1102     const gint32 MaxPicOrderCntLsb = 1 << (sps->log2_max_pic_order_cnt_lsb_minus4 + 4);
1103     gint32 temp_poc;
1104
1105     GST_DEBUG("decode picture order count type 0");
1106
1107     if (GST_VAAPI_PICTURE_IS_IDR(picture)) {
1108         priv->prev_poc_msb = 0;
1109         priv->prev_poc_lsb = 0;
1110     }
1111     else if (priv->prev_pic_has_mmco5) {
1112         priv->prev_poc_msb = 0;
1113         priv->prev_poc_lsb =
1114             (priv->prev_pic_structure == GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD ?
1115              0 : priv->field_poc[TOP_FIELD]);
1116     }
1117     else {
1118         priv->prev_poc_msb = priv->poc_msb;
1119         priv->prev_poc_lsb = priv->poc_lsb;
1120     }
1121
1122     // (8-3)
1123     priv->poc_lsb = slice_hdr->pic_order_cnt_lsb;
1124     if (priv->poc_lsb < priv->prev_poc_lsb &&
1125         (priv->prev_poc_lsb - priv->poc_lsb) >= (MaxPicOrderCntLsb / 2))
1126         priv->poc_msb = priv->prev_poc_msb + MaxPicOrderCntLsb;
1127     else if (priv->poc_lsb > priv->prev_poc_lsb &&
1128              (priv->poc_lsb - priv->prev_poc_lsb) > (MaxPicOrderCntLsb / 2))
1129         priv->poc_msb = priv->prev_poc_msb - MaxPicOrderCntLsb;
1130     else
1131         priv->poc_msb = priv->prev_poc_msb;
1132
1133     temp_poc = priv->poc_msb + priv->poc_lsb;
1134     switch (picture->structure) {
1135     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
1136         // (8-4, 8-5)
1137         priv->field_poc[TOP_FIELD] = temp_poc;
1138         priv->field_poc[BOTTOM_FIELD] = temp_poc +
1139             slice_hdr->delta_pic_order_cnt_bottom;
1140         break;
1141     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
1142         // (8-4)
1143         priv->field_poc[TOP_FIELD] = temp_poc;
1144         break;
1145     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
1146         // (8-5)
1147         priv->field_poc[BOTTOM_FIELD] = temp_poc;
1148         break;
1149     }
1150 }
1151
1152 /* 8.2.1.2 - Decoding process for picture order count type 1 */
1153 static void
1154 init_picture_poc_1(
1155     GstVaapiDecoderH264 *decoder,
1156     GstVaapiPictureH264 *picture,
1157     GstH264SliceHdr     *slice_hdr
1158 )
1159 {
1160     GstVaapiDecoderH264Private * const priv = decoder->priv;
1161     GstH264PPS * const pps = slice_hdr->pps;
1162     GstH264SPS * const sps = pps->sequence;
1163     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1164     gint32 prev_frame_num_offset, abs_frame_num, expected_poc;
1165     guint i;
1166
1167     GST_DEBUG("decode picture order count type 1");
1168
1169     if (priv->prev_pic_has_mmco5)
1170         prev_frame_num_offset = 0;
1171     else
1172         prev_frame_num_offset = priv->frame_num_offset;
1173
1174     // (8-6)
1175     if (GST_VAAPI_PICTURE_IS_IDR(picture))
1176         priv->frame_num_offset = 0;
1177     else if (priv->prev_frame_num > priv->frame_num)
1178         priv->frame_num_offset = prev_frame_num_offset + MaxFrameNum;
1179     else
1180         priv->frame_num_offset = prev_frame_num_offset;
1181
1182     // (8-7)
1183     if (sps->num_ref_frames_in_pic_order_cnt_cycle != 0)
1184         abs_frame_num = priv->frame_num_offset + priv->frame_num;
1185     else
1186         abs_frame_num = 0;
1187     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture) && abs_frame_num > 0)
1188         abs_frame_num = abs_frame_num - 1;
1189
1190     if (abs_frame_num > 0) {
1191         gint32 expected_delta_per_poc_cycle;
1192         gint32 poc_cycle_cnt, frame_num_in_poc_cycle;
1193
1194         expected_delta_per_poc_cycle = 0;
1195         for (i = 0; i < sps->num_ref_frames_in_pic_order_cnt_cycle; i++)
1196             expected_delta_per_poc_cycle += sps->offset_for_ref_frame[i];
1197
1198         // (8-8)
1199         poc_cycle_cnt = (abs_frame_num - 1) /
1200             sps->num_ref_frames_in_pic_order_cnt_cycle;
1201         frame_num_in_poc_cycle = (abs_frame_num - 1) %
1202             sps->num_ref_frames_in_pic_order_cnt_cycle;
1203
1204         // (8-9)
1205         expected_poc = poc_cycle_cnt * expected_delta_per_poc_cycle;
1206         for (i = 0; i <= frame_num_in_poc_cycle; i++)
1207             expected_poc += sps->offset_for_ref_frame[i];
1208     }
1209     else
1210         expected_poc = 0;
1211     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
1212         expected_poc += sps->offset_for_non_ref_pic;
1213
1214     // (8-10)
1215     switch (picture->structure) {
1216     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
1217         priv->field_poc[TOP_FIELD] = expected_poc +
1218             slice_hdr->delta_pic_order_cnt[0];
1219         priv->field_poc[BOTTOM_FIELD] = priv->field_poc[TOP_FIELD] +
1220             sps->offset_for_top_to_bottom_field +
1221             slice_hdr->delta_pic_order_cnt[1];
1222         break;
1223     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
1224         priv->field_poc[TOP_FIELD] = expected_poc +
1225             slice_hdr->delta_pic_order_cnt[0];
1226         break;
1227     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
1228         priv->field_poc[BOTTOM_FIELD] = expected_poc +
1229             sps->offset_for_top_to_bottom_field +
1230             slice_hdr->delta_pic_order_cnt[0];
1231         break;
1232     }
1233 }
1234
1235 /* 8.2.1.3 - Decoding process for picture order count type 2 */
1236 static void
1237 init_picture_poc_2(
1238     GstVaapiDecoderH264 *decoder,
1239     GstVaapiPictureH264 *picture,
1240     GstH264SliceHdr     *slice_hdr
1241 )
1242 {
1243     GstVaapiDecoderH264Private * const priv = decoder->priv;
1244     GstH264PPS * const pps = slice_hdr->pps;
1245     GstH264SPS * const sps = pps->sequence;
1246     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1247     gint32 prev_frame_num_offset, temp_poc;
1248
1249     GST_DEBUG("decode picture order count type 2");
1250
1251     if (priv->prev_pic_has_mmco5)
1252         prev_frame_num_offset = 0;
1253     else
1254         prev_frame_num_offset = priv->frame_num_offset;
1255
1256     // (8-11)
1257     if (GST_VAAPI_PICTURE_IS_IDR(picture))
1258         priv->frame_num_offset = 0;
1259     else if (priv->prev_frame_num > priv->frame_num)
1260         priv->frame_num_offset = prev_frame_num_offset + MaxFrameNum;
1261     else
1262         priv->frame_num_offset = prev_frame_num_offset;
1263
1264     // (8-12)
1265     if (GST_VAAPI_PICTURE_IS_IDR(picture))
1266         temp_poc = 0;
1267     else if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
1268         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num) - 1;
1269     else
1270         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num);
1271
1272     // (8-13)
1273     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
1274         priv->field_poc[TOP_FIELD] = temp_poc;
1275     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
1276         priv->field_poc[BOTTOM_FIELD] = temp_poc;
1277 }
1278
1279 /* 8.2.1 - Decoding process for picture order count */
1280 static void
1281 init_picture_poc(
1282     GstVaapiDecoderH264 *decoder,
1283     GstVaapiPictureH264 *picture,
1284     GstH264SliceHdr     *slice_hdr
1285 )
1286 {
1287     GstVaapiDecoderH264Private * const priv = decoder->priv;
1288     GstH264PPS * const pps = slice_hdr->pps;
1289     GstH264SPS * const sps = pps->sequence;
1290
1291     switch (sps->pic_order_cnt_type) {
1292     case 0:
1293         init_picture_poc_0(decoder, picture, slice_hdr);
1294         break;
1295     case 1:
1296         init_picture_poc_1(decoder, picture, slice_hdr);
1297         break;
1298     case 2:
1299         init_picture_poc_2(decoder, picture, slice_hdr);
1300         break;
1301     }
1302
1303     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
1304         picture->field_poc[TOP_FIELD] = priv->field_poc[TOP_FIELD];
1305     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
1306         picture->field_poc[BOTTOM_FIELD] = priv->field_poc[BOTTOM_FIELD];
1307     picture->base.poc = MIN(picture->field_poc[0], picture->field_poc[1]);
1308 }
1309
1310 static int
1311 compare_picture_pic_num_dec(const void *a, const void *b)
1312 {
1313     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1314     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1315
1316     return picB->pic_num - picA->pic_num;
1317 }
1318
1319 static int
1320 compare_picture_long_term_pic_num_inc(const void *a, const void *b)
1321 {
1322     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1323     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1324
1325     return picA->long_term_pic_num - picB->long_term_pic_num;
1326 }
1327
1328 static int
1329 compare_picture_poc_dec(const void *a, const void *b)
1330 {
1331     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1332     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1333
1334     return picB->base.poc - picA->base.poc;
1335 }
1336
1337 static int
1338 compare_picture_poc_inc(const void *a, const void *b)
1339 {
1340     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1341     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1342
1343     return picA->base.poc - picB->base.poc;
1344 }
1345
1346 static int
1347 compare_picture_frame_num_wrap_dec(const void *a, const void *b)
1348 {
1349     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1350     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1351
1352     return picB->frame_num_wrap - picA->frame_num_wrap;
1353 }
1354
1355 static int
1356 compare_picture_long_term_frame_idx_inc(const void *a, const void *b)
1357 {
1358     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1359     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1360
1361     return picA->long_term_frame_idx - picB->long_term_frame_idx;
1362 }
1363
1364 /* 8.2.4.1 - Decoding process for picture numbers */
1365 static void
1366 init_picture_refs_pic_num(
1367     GstVaapiDecoderH264 *decoder,
1368     GstVaapiPictureH264 *picture,
1369     GstH264SliceHdr     *slice_hdr
1370 )
1371 {
1372     GstVaapiDecoderH264Private * const priv = decoder->priv;
1373     GstH264PPS * const pps = slice_hdr->pps;
1374     GstH264SPS * const sps = pps->sequence;
1375     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1376     guint i;
1377
1378     GST_DEBUG("decode picture numbers");
1379
1380     for (i = 0; i < priv->short_ref_count; i++) {
1381         GstVaapiPictureH264 * const pic = priv->short_ref[i];
1382
1383         // (8-27)
1384         if (pic->frame_num > priv->frame_num)
1385             pic->frame_num_wrap = pic->frame_num - MaxFrameNum;
1386         else
1387             pic->frame_num_wrap = pic->frame_num;
1388
1389         // (8-28, 8-30, 8-31)
1390         if (GST_VAAPI_PICTURE_IS_FRAME(picture))
1391             pic->pic_num = pic->frame_num_wrap;
1392         else {
1393             if (pic->structure == picture->structure)
1394                 pic->pic_num = 2 * pic->frame_num_wrap + 1;
1395             else
1396                 pic->pic_num = 2 * pic->frame_num_wrap;
1397         }
1398     }
1399
1400     for (i = 0; i < priv->long_ref_count; i++) {
1401         GstVaapiPictureH264 * const pic = priv->long_ref[i];
1402
1403         // (8-29, 8-32, 8-33)
1404         if (GST_VAAPI_PICTURE_IS_FRAME(picture))
1405             pic->long_term_pic_num = pic->long_term_frame_idx;
1406         else {
1407             if (pic->structure == picture->structure)
1408                 pic->long_term_pic_num = 2 * pic->long_term_frame_idx + 1;
1409             else
1410                 pic->long_term_pic_num = 2 * pic->long_term_frame_idx;
1411         }
1412     }
1413 }
1414
1415 #define SORT_REF_LIST(list, n, compare_func) \
1416     qsort(list, n, sizeof(*(list)), compare_picture_##compare_func)
1417
1418 static void
1419 init_picture_refs_fields_1(
1420     guint                picture_structure,
1421     GstVaapiPictureH264 *RefPicList[32],
1422     guint               *RefPicList_count,
1423     GstVaapiPictureH264 *ref_list[32],
1424     guint                ref_list_count
1425 )
1426 {
1427     guint i, j, n;
1428
1429     i = 0;
1430     j = 0;
1431     n = *RefPicList_count;
1432     do {
1433         g_assert(n < 32);
1434         for (; i < ref_list_count; i++) {
1435             if (ref_list[i]->structure == picture_structure) {
1436                 RefPicList[n++] = ref_list[i++];
1437                 break;
1438             }
1439         }
1440         for (; j < ref_list_count; j++) {
1441             if (ref_list[j]->structure != picture_structure) {
1442                 RefPicList[n++] = ref_list[j++];
1443                 break;
1444             }
1445         }
1446     } while (i < ref_list_count || j < ref_list_count);
1447     *RefPicList_count = n;
1448 }
1449
1450 static inline void
1451 init_picture_refs_fields(
1452     GstVaapiPictureH264 *picture,
1453     GstVaapiPictureH264 *RefPicList[32],
1454     guint               *RefPicList_count,
1455     GstVaapiPictureH264 *short_ref[32],
1456     guint                short_ref_count,
1457     GstVaapiPictureH264 *long_ref[32],
1458     guint                long_ref_count
1459 )
1460 {
1461     guint n = 0;
1462
1463     /* 8.2.4.2.5 - reference picture lists in fields */
1464     init_picture_refs_fields_1(picture->structure, RefPicList, &n,
1465         short_ref, short_ref_count);
1466     init_picture_refs_fields_1(picture->structure, RefPicList, &n,
1467         long_ref, long_ref_count);
1468     *RefPicList_count = n;
1469 }
1470
1471 static void
1472 init_picture_refs_p_slice(
1473     GstVaapiDecoderH264 *decoder,
1474     GstVaapiPictureH264 *picture,
1475     GstH264SliceHdr     *slice_hdr
1476 )
1477 {
1478     GstVaapiDecoderH264Private * const priv = decoder->priv;
1479     GstVaapiPictureH264 **ref_list;
1480     guint i;
1481
1482     GST_DEBUG("decode reference picture list for P and SP slices");
1483
1484     if (GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1485         /* 8.2.4.2.1 - P and SP slices in frames */
1486         if (priv->short_ref_count > 0) {
1487             ref_list = priv->RefPicList0;
1488             for (i = 0; i < priv->short_ref_count; i++)
1489                 ref_list[i] = priv->short_ref[i];
1490             SORT_REF_LIST(ref_list, i, pic_num_dec);
1491             priv->RefPicList0_count += i;
1492         }
1493
1494         if (priv->long_ref_count > 0) {
1495             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1496             for (i = 0; i < priv->long_ref_count; i++)
1497                 ref_list[i] = priv->long_ref[i];
1498             SORT_REF_LIST(ref_list, i, long_term_pic_num_inc);
1499             priv->RefPicList0_count += i;
1500         }
1501     }
1502     else {
1503         /* 8.2.4.2.2 - P and SP slices in fields */
1504         GstVaapiPictureH264 *short_ref[32];
1505         guint short_ref_count = 0;
1506         GstVaapiPictureH264 *long_ref[32];
1507         guint long_ref_count = 0;
1508
1509         if (priv->short_ref_count > 0) {
1510             for (i = 0; i < priv->short_ref_count; i++)
1511                 short_ref[i] = priv->short_ref[i];
1512             SORT_REF_LIST(short_ref, i, frame_num_wrap_dec);
1513             short_ref_count = i;
1514         }
1515
1516         if (priv->long_ref_count > 0) {
1517             for (i = 0; i < priv->long_ref_count; i++)
1518                 long_ref[i] = priv->long_ref[i];
1519             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1520             long_ref_count = i;
1521         }
1522
1523         init_picture_refs_fields(
1524             picture,
1525             priv->RefPicList0, &priv->RefPicList0_count,
1526             short_ref,          short_ref_count,
1527             long_ref,           long_ref_count
1528         );
1529     }
1530 }
1531
1532 static void
1533 init_picture_refs_b_slice(
1534     GstVaapiDecoderH264 *decoder,
1535     GstVaapiPictureH264 *picture,
1536     GstH264SliceHdr     *slice_hdr
1537 )
1538 {
1539     GstVaapiDecoderH264Private * const priv = decoder->priv;
1540     GstVaapiPictureH264 **ref_list;
1541     guint i, n;
1542
1543     GST_DEBUG("decode reference picture list for B slices");
1544
1545     if (GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1546         /* 8.2.4.2.3 - B slices in frames */
1547
1548         /* RefPicList0 */
1549         if (priv->short_ref_count > 0) {
1550             // 1. Short-term references
1551             ref_list = priv->RefPicList0;
1552             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1553                 if (priv->short_ref[i]->base.poc < picture->base.poc)
1554                     ref_list[n++] = priv->short_ref[i];
1555             }
1556             SORT_REF_LIST(ref_list, n, poc_dec);
1557             priv->RefPicList0_count += n;
1558
1559             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1560             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1561                 if (priv->short_ref[i]->base.poc >= picture->base.poc)
1562                     ref_list[n++] = priv->short_ref[i];
1563             }
1564             SORT_REF_LIST(ref_list, n, poc_inc);
1565             priv->RefPicList0_count += n;
1566         }
1567
1568         if (priv->long_ref_count > 0) {
1569             // 2. Long-term references
1570             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1571             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1572                 ref_list[n++] = priv->long_ref[i];
1573             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1574             priv->RefPicList0_count += n;
1575         }
1576
1577         /* RefPicList1 */
1578         if (priv->short_ref_count > 0) {
1579             // 1. Short-term references
1580             ref_list = priv->RefPicList1;
1581             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1582                 if (priv->short_ref[i]->base.poc > picture->base.poc)
1583                     ref_list[n++] = priv->short_ref[i];
1584             }
1585             SORT_REF_LIST(ref_list, n, poc_inc);
1586             priv->RefPicList1_count += n;
1587
1588             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1589             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1590                 if (priv->short_ref[i]->base.poc <= picture->base.poc)
1591                     ref_list[n++] = priv->short_ref[i];
1592             }
1593             SORT_REF_LIST(ref_list, n, poc_dec);
1594             priv->RefPicList1_count += n;
1595         }
1596
1597         if (priv->long_ref_count > 0) {
1598             // 2. Long-term references
1599             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1600             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1601                 ref_list[n++] = priv->long_ref[i];
1602             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1603             priv->RefPicList1_count += n;
1604         }
1605     }
1606     else {
1607         /* 8.2.4.2.4 - B slices in fields */
1608         GstVaapiPictureH264 *short_ref0[32];
1609         guint short_ref0_count = 0;
1610         GstVaapiPictureH264 *short_ref1[32];
1611         guint short_ref1_count = 0;
1612         GstVaapiPictureH264 *long_ref[32];
1613         guint long_ref_count = 0;
1614
1615         /* refFrameList0ShortTerm */
1616         if (priv->short_ref_count > 0) {
1617             ref_list = short_ref0;
1618             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1619                 if (priv->short_ref[i]->base.poc <= picture->base.poc)
1620                     ref_list[n++] = priv->short_ref[i];
1621             }
1622             SORT_REF_LIST(ref_list, n, poc_dec);
1623             short_ref0_count += n;
1624
1625             ref_list = &short_ref0[short_ref0_count];
1626             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1627                 if (priv->short_ref[i]->base.poc > picture->base.poc)
1628                     ref_list[n++] = priv->short_ref[i];
1629             }
1630             SORT_REF_LIST(ref_list, n, poc_inc);
1631             short_ref0_count += n;
1632         }
1633
1634         /* refFrameList1ShortTerm */
1635         if (priv->short_ref_count > 0) {
1636             ref_list = short_ref1;
1637             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1638                 if (priv->short_ref[i]->base.poc > picture->base.poc)
1639                     ref_list[n++] = priv->short_ref[i];
1640             }
1641             SORT_REF_LIST(ref_list, n, poc_inc);
1642             short_ref1_count += n;
1643
1644             ref_list = &short_ref1[short_ref1_count];
1645             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1646                 if (priv->short_ref[i]->base.poc <= picture->base.poc)
1647                     ref_list[n++] = priv->short_ref[i];
1648             }
1649             SORT_REF_LIST(ref_list, n, poc_dec);
1650             short_ref1_count += n;
1651         }
1652
1653         /* refFrameListLongTerm */
1654         if (priv->long_ref_count > 0) {
1655             for (i = 0; i < priv->long_ref_count; i++)
1656                 long_ref[i] = priv->long_ref[i];
1657             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1658             long_ref_count = i;
1659         }
1660
1661         init_picture_refs_fields(
1662             picture,
1663             priv->RefPicList0, &priv->RefPicList0_count,
1664             short_ref0,         short_ref0_count,
1665             long_ref,           long_ref_count
1666         );
1667
1668         init_picture_refs_fields(
1669             picture,
1670             priv->RefPicList1, &priv->RefPicList1_count,
1671             short_ref1,         short_ref1_count,
1672             long_ref,           long_ref_count
1673         );
1674    }
1675
1676     /* Check whether RefPicList1 is identical to RefPicList0, then
1677        swap if necessary */
1678     if (priv->RefPicList1_count > 1 &&
1679         priv->RefPicList1_count == priv->RefPicList0_count &&
1680         memcmp(priv->RefPicList0, priv->RefPicList1,
1681                priv->RefPicList0_count * sizeof(priv->RefPicList0[0])) == 0) {
1682         GstVaapiPictureH264 * const tmp = priv->RefPicList1[0];
1683         priv->RefPicList1[0] = priv->RefPicList1[1];
1684         priv->RefPicList1[1] = tmp;
1685     }
1686 }
1687
1688 #undef SORT_REF_LIST
1689
1690 static gint
1691 find_short_term_reference(GstVaapiDecoderH264 *decoder, gint32 pic_num)
1692 {
1693     GstVaapiDecoderH264Private * const priv = decoder->priv;
1694     guint i;
1695
1696     for (i = 0; i < priv->short_ref_count; i++) {
1697         if (priv->short_ref[i]->pic_num == pic_num)
1698             return i;
1699     }
1700     GST_ERROR("found no short-term reference picture with PicNum = %d",
1701               pic_num);
1702     return -1;
1703 }
1704
1705 static gint
1706 find_long_term_reference(GstVaapiDecoderH264 *decoder, gint32 long_term_pic_num)
1707 {
1708     GstVaapiDecoderH264Private * const priv = decoder->priv;
1709     guint i;
1710
1711     for (i = 0; i < priv->long_ref_count; i++) {
1712         if (priv->long_ref[i]->long_term_pic_num == long_term_pic_num)
1713             return i;
1714     }
1715     GST_ERROR("found no long-term reference picture with LongTermPicNum = %d",
1716               long_term_pic_num);
1717     return -1;
1718 }
1719
1720 static void
1721 exec_picture_refs_modification_1(
1722     GstVaapiDecoderH264           *decoder,
1723     GstVaapiPictureH264           *picture,
1724     GstH264SliceHdr               *slice_hdr,
1725     guint                          list
1726 )
1727 {
1728     GstVaapiDecoderH264Private * const priv = decoder->priv;
1729     GstH264PPS * const pps = slice_hdr->pps;
1730     GstH264SPS * const sps = pps->sequence;
1731     GstH264RefPicListModification *ref_pic_list_modification;
1732     guint num_ref_pic_list_modifications;
1733     GstVaapiPictureH264 **ref_list;
1734     guint *ref_list_count_ptr, ref_list_count, ref_list_idx = 0;
1735     guint i, j, n, num_refs;
1736     gint found_ref_idx;
1737     gint32 MaxPicNum, CurrPicNum, picNumPred;
1738
1739     GST_DEBUG("modification process of reference picture list %u", list);
1740
1741     if (list == 0) {
1742         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l0;
1743         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l0;
1744         ref_list                       = priv->RefPicList0;
1745         ref_list_count_ptr             = &priv->RefPicList0_count;
1746         num_refs                       = slice_hdr->num_ref_idx_l0_active_minus1 + 1;
1747     }
1748     else {
1749         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l1;
1750         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l1;
1751         ref_list                       = priv->RefPicList1;
1752         ref_list_count_ptr             = &priv->RefPicList1_count;
1753         num_refs                       = slice_hdr->num_ref_idx_l1_active_minus1 + 1;
1754     }
1755     ref_list_count = *ref_list_count_ptr;
1756
1757     if (!GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1758         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 5); // 2 * MaxFrameNum
1759         CurrPicNum = 2 * slice_hdr->frame_num + 1;              // 2 * frame_num + 1
1760     }
1761     else {
1762         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 4); // MaxFrameNum
1763         CurrPicNum = slice_hdr->frame_num;                      // frame_num
1764     }
1765
1766     picNumPred = CurrPicNum;
1767
1768     for (i = 0; i < num_ref_pic_list_modifications; i++) {
1769         GstH264RefPicListModification * const l = &ref_pic_list_modification[i];
1770         if (l->modification_of_pic_nums_idc == 3)
1771             break;
1772
1773         /* 8.2.4.3.1 - Short-term reference pictures */
1774         if (l->modification_of_pic_nums_idc == 0 || l->modification_of_pic_nums_idc == 1) {
1775             gint32 abs_diff_pic_num = l->value.abs_diff_pic_num_minus1 + 1;
1776             gint32 picNum, picNumNoWrap;
1777
1778             // (8-34)
1779             if (l->modification_of_pic_nums_idc == 0) {
1780                 picNumNoWrap = picNumPred - abs_diff_pic_num;
1781                 if (picNumNoWrap < 0)
1782                     picNumNoWrap += MaxPicNum;
1783             }
1784
1785             // (8-35)
1786             else {
1787                 picNumNoWrap = picNumPred + abs_diff_pic_num;
1788                 if (picNumNoWrap >= MaxPicNum)
1789                     picNumNoWrap -= MaxPicNum;
1790             }
1791             picNumPred = picNumNoWrap;
1792
1793             // (8-36)
1794             picNum = picNumNoWrap;
1795             if (picNum > CurrPicNum)
1796                 picNum -= MaxPicNum;
1797
1798             // (8-37)
1799             for (j = num_refs; j > ref_list_idx; j--)
1800                 ref_list[j] = ref_list[j - 1];
1801             found_ref_idx = find_short_term_reference(decoder, picNum);
1802             ref_list[ref_list_idx++] =
1803                 found_ref_idx >= 0 ? priv->short_ref[found_ref_idx] : NULL;
1804             n = ref_list_idx;
1805             for (j = ref_list_idx; j <= num_refs; j++) {
1806                 gint32 PicNumF;
1807                 if (!ref_list[j])
1808                     continue;
1809                 PicNumF =
1810                     GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(ref_list[j]) ?
1811                     ref_list[j]->pic_num : MaxPicNum;
1812                 if (PicNumF != picNum)
1813                     ref_list[n++] = ref_list[j];
1814             }
1815         }
1816
1817         /* 8.2.4.3.2 - Long-term reference pictures */
1818         else {
1819
1820             for (j = num_refs; j > ref_list_idx; j--)
1821                 ref_list[j] = ref_list[j - 1];
1822             found_ref_idx =
1823                 find_long_term_reference(decoder, l->value.long_term_pic_num);
1824             ref_list[ref_list_idx++] =
1825                 found_ref_idx >= 0 ? priv->long_ref[found_ref_idx] : NULL;
1826             n = ref_list_idx;
1827             for (j = ref_list_idx; j <= num_refs; j++) {
1828                 gint32 LongTermPicNumF;
1829                 if (!ref_list[j])
1830                     continue;
1831                 LongTermPicNumF =
1832                     GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(ref_list[j]) ?
1833                     ref_list[j]->long_term_pic_num : INT_MAX;
1834                 if (LongTermPicNumF != l->value.long_term_pic_num)
1835                     ref_list[n++] = ref_list[j];
1836             }
1837         }
1838     }
1839
1840 #if DEBUG
1841     for (i = 0; i < num_refs; i++)
1842         if (!ref_list[i])
1843             GST_ERROR("list %u entry %u is empty", list, i);
1844 #endif
1845     *ref_list_count_ptr = num_refs;
1846 }
1847
1848 /* 8.2.4.3 - Modification process for reference picture lists */
1849 static void
1850 exec_picture_refs_modification(
1851     GstVaapiDecoderH264 *decoder,
1852     GstVaapiPictureH264 *picture,
1853     GstH264SliceHdr     *slice_hdr
1854 )
1855 {
1856     GST_DEBUG("execute ref_pic_list_modification()");
1857
1858     /* RefPicList0 */
1859     if (!GST_H264_IS_I_SLICE(slice_hdr) && !GST_H264_IS_SI_SLICE(slice_hdr) &&
1860         slice_hdr->ref_pic_list_modification_flag_l0)
1861         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 0);
1862
1863     /* RefPicList1 */
1864     if (GST_H264_IS_B_SLICE(slice_hdr) &&
1865         slice_hdr->ref_pic_list_modification_flag_l1)
1866         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 1);
1867 }
1868
1869 static void
1870 init_picture_ref_lists(GstVaapiDecoderH264 *decoder)
1871 {
1872     GstVaapiDecoderH264Private * const priv = decoder->priv;
1873     guint i, j, short_ref_count, long_ref_count;
1874
1875     short_ref_count = 0;
1876     long_ref_count  = 0;
1877     if (GST_VAAPI_PICTURE_IS_FRAME(priv->current_picture)) {
1878         for (i = 0; i < priv->dpb_count; i++) {
1879             GstVaapiFrameStore * const fs = priv->dpb[i];
1880             GstVaapiPictureH264 *picture;
1881             if (!gst_vaapi_frame_store_has_frame(fs))
1882                 continue;
1883             picture = fs->buffers[0];
1884             if (GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture))
1885                 priv->short_ref[short_ref_count++] = picture;
1886             else if (GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture))
1887                 priv->long_ref[long_ref_count++] = picture;
1888             picture->structure = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
1889             picture->other_field = fs->buffers[1];
1890         }
1891     }
1892     else {
1893         for (i = 0; i < priv->dpb_count; i++) {
1894             GstVaapiFrameStore * const fs = priv->dpb[i];
1895             for (j = 0; j < fs->num_buffers; j++) {
1896                 GstVaapiPictureH264 * const picture = fs->buffers[j];
1897                 if (GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture))
1898                     priv->short_ref[short_ref_count++] = picture;
1899                 else if (GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture))
1900                     priv->long_ref[long_ref_count++] = picture;
1901                 picture->structure = picture->base.structure;
1902                 picture->other_field = fs->buffers[j ^ 1];
1903             }
1904         }
1905     }
1906
1907     for (i = short_ref_count; i < priv->short_ref_count; i++)
1908         priv->short_ref[i] = NULL;
1909     priv->short_ref_count = short_ref_count;
1910
1911     for (i = long_ref_count; i < priv->long_ref_count; i++)
1912         priv->long_ref[i] = NULL;
1913     priv->long_ref_count = long_ref_count;
1914 }
1915
1916 static void
1917 init_picture_refs(
1918     GstVaapiDecoderH264 *decoder,
1919     GstVaapiPictureH264 *picture,
1920     GstH264SliceHdr     *slice_hdr
1921 )
1922 {
1923     GstVaapiDecoderH264Private * const priv = decoder->priv;
1924     GstVaapiPicture * const base_picture = &picture->base;
1925     guint i, num_refs;
1926
1927     init_picture_ref_lists(decoder);
1928     init_picture_refs_pic_num(decoder, picture, slice_hdr);
1929
1930     priv->RefPicList0_count = 0;
1931     priv->RefPicList1_count = 0;
1932
1933     switch (base_picture->type) {
1934     case GST_VAAPI_PICTURE_TYPE_P:
1935     case GST_VAAPI_PICTURE_TYPE_SP:
1936         init_picture_refs_p_slice(decoder, picture, slice_hdr);
1937         break;
1938     case GST_VAAPI_PICTURE_TYPE_B:
1939         init_picture_refs_b_slice(decoder, picture, slice_hdr);
1940         break;
1941     default:
1942         break;
1943     }
1944
1945     exec_picture_refs_modification(decoder, picture, slice_hdr);
1946
1947     switch (base_picture->type) {
1948     case GST_VAAPI_PICTURE_TYPE_B:
1949         num_refs = 1 + slice_hdr->num_ref_idx_l1_active_minus1;
1950         for (i = priv->RefPicList1_count; i < num_refs; i++)
1951             priv->RefPicList1[i] = NULL;
1952         priv->RefPicList1_count = num_refs;
1953
1954         // fall-through
1955     case GST_VAAPI_PICTURE_TYPE_P:
1956     case GST_VAAPI_PICTURE_TYPE_SP:
1957         num_refs = 1 + slice_hdr->num_ref_idx_l0_active_minus1;
1958         for (i = priv->RefPicList0_count; i < num_refs; i++)
1959             priv->RefPicList0[i] = NULL;
1960         priv->RefPicList0_count = num_refs;
1961         break;
1962     default:
1963         break;
1964     }
1965 }
1966
1967 static gboolean
1968 init_picture(
1969     GstVaapiDecoderH264 *decoder,
1970     GstVaapiPictureH264 *picture, GstVaapiParserInfoH264 *pi)
1971 {
1972     GstVaapiDecoderH264Private * const priv = decoder->priv;
1973     GstVaapiPicture * const base_picture = &picture->base;
1974     GstH264SliceHdr * const slice_hdr = &pi->data.slice_hdr;
1975
1976     priv->prev_frame_num        = priv->frame_num;
1977     priv->frame_num             = slice_hdr->frame_num;
1978     picture->frame_num          = priv->frame_num;
1979     picture->frame_num_wrap     = priv->frame_num;
1980     picture->output_flag        = TRUE; /* XXX: conformant to Annex A only */
1981     base_picture->pts           = GST_VAAPI_DECODER_CODEC_FRAME(decoder)->pts;
1982
1983     /* Reset decoder state for IDR pictures */
1984     if (pi->nalu.type == GST_H264_NAL_SLICE_IDR) {
1985         GST_DEBUG("<IDR>");
1986         GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_IDR);
1987         dpb_flush(decoder);
1988     }
1989
1990     /* Initialize slice type */
1991     switch (slice_hdr->type % 5) {
1992     case GST_H264_P_SLICE:
1993         base_picture->type = GST_VAAPI_PICTURE_TYPE_P;
1994         break;
1995     case GST_H264_B_SLICE:
1996         base_picture->type = GST_VAAPI_PICTURE_TYPE_B;
1997         break;
1998     case GST_H264_I_SLICE:
1999         base_picture->type = GST_VAAPI_PICTURE_TYPE_I;
2000         break;
2001     case GST_H264_SP_SLICE:
2002         base_picture->type = GST_VAAPI_PICTURE_TYPE_SP;
2003         break;
2004     case GST_H264_SI_SLICE:
2005         base_picture->type = GST_VAAPI_PICTURE_TYPE_SI;
2006         break;
2007     }
2008
2009     /* Initialize picture structure */
2010     if (!slice_hdr->field_pic_flag)
2011         base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
2012     else {
2013         GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_INTERLACED);
2014         if (!slice_hdr->bottom_field_flag)
2015             base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD;
2016         else
2017             base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD;
2018     }
2019     picture->structure = base_picture->structure;
2020
2021     /* Initialize reference flags */
2022     if (pi->nalu.ref_idc) {
2023         GstH264DecRefPicMarking * const dec_ref_pic_marking =
2024             &slice_hdr->dec_ref_pic_marking;
2025
2026         if (GST_VAAPI_PICTURE_IS_IDR(picture) &&
2027             dec_ref_pic_marking->long_term_reference_flag)
2028             GST_VAAPI_PICTURE_FLAG_SET(picture,
2029                 GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE);
2030         else
2031             GST_VAAPI_PICTURE_FLAG_SET(picture,
2032                 GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE);
2033     }
2034
2035     init_picture_poc(decoder, picture, slice_hdr);
2036     init_picture_refs(decoder, picture, slice_hdr);
2037     return TRUE;
2038 }
2039
2040 /* 8.2.5.3 - Sliding window decoded reference picture marking process */
2041 static gboolean
2042 exec_ref_pic_marking_sliding_window(GstVaapiDecoderH264 *decoder)
2043 {
2044     GstVaapiDecoderH264Private * const priv = decoder->priv;
2045     GstH264PPS * const pps = priv->current_picture->pps;
2046     GstH264SPS * const sps = pps->sequence;
2047     GstVaapiPictureH264 *ref_picture;
2048     guint i, m, max_num_ref_frames;
2049
2050     GST_DEBUG("reference picture marking process (sliding window)");
2051
2052     if (!GST_VAAPI_PICTURE_IS_FIRST_FIELD(priv->current_picture))
2053         return TRUE;
2054
2055     max_num_ref_frames = sps->num_ref_frames;
2056     if (max_num_ref_frames == 0)
2057         max_num_ref_frames = 1;
2058     if (!GST_VAAPI_PICTURE_IS_FRAME(priv->current_picture))
2059         max_num_ref_frames <<= 1;
2060
2061     if (priv->short_ref_count + priv->long_ref_count < max_num_ref_frames)
2062         return TRUE;
2063     if (priv->short_ref_count < 1)
2064         return FALSE;
2065
2066     for (m = 0, i = 1; i < priv->short_ref_count; i++) {
2067         GstVaapiPictureH264 * const picture = priv->short_ref[i];
2068         if (picture->frame_num_wrap < priv->short_ref[m]->frame_num_wrap)
2069             m = i;
2070     }
2071
2072     ref_picture = priv->short_ref[m];
2073     gst_vaapi_picture_h264_set_reference(ref_picture, 0, TRUE);
2074     ARRAY_REMOVE_INDEX(priv->short_ref, m);
2075
2076     /* Both fields need to be marked as "unused for reference", so
2077        remove the other field from the short_ref[] list as well */
2078     if (!GST_VAAPI_PICTURE_IS_FRAME(priv->current_picture) && ref_picture->other_field) {
2079         for (i = 0; i < priv->short_ref_count; i++) {
2080             if (priv->short_ref[i] == ref_picture->other_field) {
2081                 ARRAY_REMOVE_INDEX(priv->short_ref, i);
2082                 break;
2083             }
2084         }
2085     }
2086     return TRUE;
2087 }
2088
2089 static inline gint32
2090 get_picNumX(GstVaapiPictureH264 *picture, GstH264RefPicMarking *ref_pic_marking)
2091 {
2092     gint32 pic_num;
2093
2094     if (GST_VAAPI_PICTURE_IS_FRAME(picture))
2095         pic_num = picture->frame_num_wrap;
2096     else
2097         pic_num = 2 * picture->frame_num_wrap + 1;
2098     pic_num -= ref_pic_marking->difference_of_pic_nums_minus1 + 1;
2099     return pic_num;
2100 }
2101
2102 /* 8.2.5.4.1. Mark short-term reference picture as "unused for reference" */
2103 static void
2104 exec_ref_pic_marking_adaptive_mmco_1(
2105     GstVaapiDecoderH264  *decoder,
2106     GstVaapiPictureH264  *picture,
2107     GstH264RefPicMarking *ref_pic_marking
2108 )
2109 {
2110     GstVaapiDecoderH264Private * const priv = decoder->priv;
2111     gint32 i, picNumX;
2112
2113     picNumX = get_picNumX(picture, ref_pic_marking);
2114     i = find_short_term_reference(decoder, picNumX);
2115     if (i < 0)
2116         return;
2117
2118     gst_vaapi_picture_h264_set_reference(priv->short_ref[i], 0,
2119         GST_VAAPI_PICTURE_IS_FRAME(picture));
2120     ARRAY_REMOVE_INDEX(priv->short_ref, i);
2121 }
2122
2123 /* 8.2.5.4.2. Mark long-term reference picture as "unused for reference" */
2124 static void
2125 exec_ref_pic_marking_adaptive_mmco_2(
2126     GstVaapiDecoderH264  *decoder,
2127     GstVaapiPictureH264  *picture,
2128     GstH264RefPicMarking *ref_pic_marking
2129 )
2130 {
2131     GstVaapiDecoderH264Private * const priv = decoder->priv;
2132     gint32 i;
2133
2134     i = find_long_term_reference(decoder, ref_pic_marking->long_term_pic_num);
2135     if (i < 0)
2136         return;
2137
2138     gst_vaapi_picture_h264_set_reference(priv->long_ref[i], 0,
2139         GST_VAAPI_PICTURE_IS_FRAME(picture));
2140     ARRAY_REMOVE_INDEX(priv->long_ref, i);
2141 }
2142
2143 /* 8.2.5.4.3. Assign LongTermFrameIdx to a short-term reference picture */
2144 static void
2145 exec_ref_pic_marking_adaptive_mmco_3(
2146     GstVaapiDecoderH264  *decoder,
2147     GstVaapiPictureH264  *picture,
2148     GstH264RefPicMarking *ref_pic_marking
2149 )
2150 {
2151     GstVaapiDecoderH264Private * const priv = decoder->priv;
2152     GstVaapiPictureH264 *ref_picture;
2153     gint32 i, picNumX;
2154
2155     for (i = 0; i < priv->long_ref_count; i++) {
2156         if (priv->long_ref[i]->long_term_frame_idx == ref_pic_marking->long_term_frame_idx)
2157             break;
2158     }
2159     if (i != priv->long_ref_count) {
2160         gst_vaapi_picture_h264_set_reference(priv->long_ref[i], 0, TRUE);
2161         ARRAY_REMOVE_INDEX(priv->long_ref, i);
2162     }
2163
2164     picNumX = get_picNumX(picture, ref_pic_marking);
2165     i = find_short_term_reference(decoder, picNumX);
2166     if (i < 0)
2167         return;
2168
2169     ref_picture = priv->short_ref[i];
2170     ARRAY_REMOVE_INDEX(priv->short_ref, i);
2171     priv->long_ref[priv->long_ref_count++] = ref_picture;
2172
2173     ref_picture->long_term_frame_idx = ref_pic_marking->long_term_frame_idx;
2174     gst_vaapi_picture_h264_set_reference(ref_picture,
2175         GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE,
2176         GST_VAAPI_PICTURE_IS_FRAME(picture));
2177 }
2178
2179 /* 8.2.5.4.4. Mark pictures with LongTermFramIdx > max_long_term_frame_idx
2180  * as "unused for reference" */
2181 static void
2182 exec_ref_pic_marking_adaptive_mmco_4(
2183     GstVaapiDecoderH264  *decoder,
2184     GstVaapiPictureH264  *picture,
2185     GstH264RefPicMarking *ref_pic_marking
2186 )
2187 {
2188     GstVaapiDecoderH264Private * const priv = decoder->priv;
2189     gint32 i, long_term_frame_idx;
2190
2191     long_term_frame_idx = ref_pic_marking->max_long_term_frame_idx_plus1 - 1;
2192
2193     for (i = 0; i < priv->long_ref_count; i++) {
2194         if (priv->long_ref[i]->long_term_frame_idx <= long_term_frame_idx)
2195             continue;
2196         gst_vaapi_picture_h264_set_reference(priv->long_ref[i], 0, FALSE);
2197         ARRAY_REMOVE_INDEX(priv->long_ref, i);
2198         i--;
2199     }
2200 }
2201
2202 /* 8.2.5.4.5. Mark all reference pictures as "unused for reference" */
2203 static void
2204 exec_ref_pic_marking_adaptive_mmco_5(
2205     GstVaapiDecoderH264  *decoder,
2206     GstVaapiPictureH264  *picture,
2207     GstH264RefPicMarking *ref_pic_marking
2208 )
2209 {
2210     GstVaapiDecoderH264Private * const priv = decoder->priv;
2211
2212     dpb_flush(decoder);
2213
2214     priv->prev_pic_has_mmco5 = TRUE;
2215
2216     /* The picture shall be inferred to have had frame_num equal to 0 (7.4.3) */
2217     priv->frame_num = 0;
2218     priv->frame_num_offset = 0;
2219     picture->frame_num = 0;
2220
2221     /* Update TopFieldOrderCnt and BottomFieldOrderCnt (8.2.1) */
2222     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
2223         picture->field_poc[TOP_FIELD] -= picture->base.poc;
2224     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
2225         picture->field_poc[BOTTOM_FIELD] -= picture->base.poc;
2226     picture->base.poc = 0;
2227 }
2228
2229 /* 8.2.5.4.6. Assign a long-term frame index to the current picture */
2230 static void
2231 exec_ref_pic_marking_adaptive_mmco_6(
2232     GstVaapiDecoderH264  *decoder,
2233     GstVaapiPictureH264  *picture,
2234     GstH264RefPicMarking *ref_pic_marking
2235 )
2236 {
2237     picture->long_term_frame_idx = ref_pic_marking->long_term_frame_idx;
2238     gst_vaapi_picture_h264_set_reference(picture,
2239         GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE, FALSE);
2240 }
2241
2242 /* 8.2.5.4. Adaptive memory control decoded reference picture marking process */
2243 static gboolean
2244 exec_ref_pic_marking_adaptive(
2245     GstVaapiDecoderH264     *decoder,
2246     GstVaapiPictureH264     *picture,
2247     GstH264DecRefPicMarking *dec_ref_pic_marking
2248 )
2249 {
2250     guint i;
2251
2252     GST_DEBUG("reference picture marking process (adaptive memory control)");
2253
2254     typedef void (*exec_ref_pic_marking_adaptive_mmco_func)(
2255         GstVaapiDecoderH264  *decoder,
2256         GstVaapiPictureH264  *picture,
2257         GstH264RefPicMarking *ref_pic_marking
2258     );
2259
2260     static const exec_ref_pic_marking_adaptive_mmco_func mmco_funcs[] = {
2261         NULL,
2262         exec_ref_pic_marking_adaptive_mmco_1,
2263         exec_ref_pic_marking_adaptive_mmco_2,
2264         exec_ref_pic_marking_adaptive_mmco_3,
2265         exec_ref_pic_marking_adaptive_mmco_4,
2266         exec_ref_pic_marking_adaptive_mmco_5,
2267         exec_ref_pic_marking_adaptive_mmco_6,
2268     };
2269
2270     for (i = 0; i < dec_ref_pic_marking->n_ref_pic_marking; i++) {
2271         GstH264RefPicMarking * const ref_pic_marking =
2272             &dec_ref_pic_marking->ref_pic_marking[i];
2273
2274         const guint mmco = ref_pic_marking->memory_management_control_operation;
2275         if (mmco < G_N_ELEMENTS(mmco_funcs) && mmco_funcs[mmco])
2276             mmco_funcs[mmco](decoder, picture, ref_pic_marking);
2277         else {
2278             GST_ERROR("unhandled MMCO %u", mmco);
2279             return FALSE;
2280         }
2281     }
2282     return TRUE;
2283 }
2284
2285 /* 8.2.5 - Execute reference picture marking process */
2286 static gboolean
2287 exec_ref_pic_marking(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
2288 {
2289     GstVaapiDecoderH264Private * const priv = decoder->priv;
2290
2291     priv->prev_pic_has_mmco5 = FALSE;
2292     priv->prev_pic_structure = picture->structure;
2293
2294     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
2295         return TRUE;
2296
2297     if (!GST_VAAPI_PICTURE_IS_IDR(picture)) {
2298         GstH264DecRefPicMarking * const dec_ref_pic_marking =
2299             &picture->last_slice_hdr->dec_ref_pic_marking;
2300         if (dec_ref_pic_marking->adaptive_ref_pic_marking_mode_flag) {
2301             if (!exec_ref_pic_marking_adaptive(decoder, picture, dec_ref_pic_marking))
2302                 return FALSE;
2303         }
2304         else {
2305             if (!exec_ref_pic_marking_sliding_window(decoder))
2306                 return FALSE;
2307         }
2308     }
2309     return TRUE;
2310 }
2311
2312 static void
2313 vaapi_init_picture(VAPictureH264 *pic)
2314 {
2315     pic->picture_id           = VA_INVALID_ID;
2316     pic->frame_idx            = 0;
2317     pic->flags                = VA_PICTURE_H264_INVALID;
2318     pic->TopFieldOrderCnt     = 0;
2319     pic->BottomFieldOrderCnt  = 0;
2320 }
2321
2322 static void
2323 vaapi_fill_picture(VAPictureH264 *pic, GstVaapiPictureH264 *picture,
2324     guint picture_structure)
2325 {
2326     if (!picture_structure)
2327         picture_structure = picture->structure;
2328
2329     pic->picture_id = picture->base.surface_id;
2330     pic->flags = 0;
2331
2332     if (GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture)) {
2333         pic->flags |= VA_PICTURE_H264_LONG_TERM_REFERENCE;
2334         pic->frame_idx = picture->long_term_frame_idx;
2335     }
2336     else {
2337         if (GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture))
2338             pic->flags |= VA_PICTURE_H264_SHORT_TERM_REFERENCE;
2339         pic->frame_idx = picture->frame_num;
2340     }
2341
2342     switch (picture_structure) {
2343     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
2344         pic->TopFieldOrderCnt = picture->field_poc[TOP_FIELD];
2345         pic->BottomFieldOrderCnt = picture->field_poc[BOTTOM_FIELD];
2346         break;
2347     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
2348         pic->flags |= VA_PICTURE_H264_TOP_FIELD;
2349         pic->TopFieldOrderCnt = picture->field_poc[TOP_FIELD];
2350         pic->BottomFieldOrderCnt = 0;
2351         break;
2352     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
2353         pic->flags |= VA_PICTURE_H264_BOTTOM_FIELD;
2354         pic->BottomFieldOrderCnt = picture->field_poc[BOTTOM_FIELD];
2355         pic->TopFieldOrderCnt = 0;
2356         break;
2357     }
2358 }
2359
2360 static gboolean
2361 fill_picture(GstVaapiDecoderH264 *decoder,
2362     GstVaapiPictureH264 *picture, GstVaapiParserInfoH264 *pi)
2363 {
2364     GstVaapiDecoderH264Private * const priv = decoder->priv;
2365     GstVaapiPicture * const base_picture = &picture->base;
2366     GstH264SliceHdr * const slice_hdr = &pi->data.slice_hdr;
2367     GstH264PPS * const pps = picture->pps;
2368     GstH264SPS * const sps = pps->sequence;
2369     VAPictureParameterBufferH264 * const pic_param = base_picture->param;
2370     guint i, n;
2371
2372     /* Fill in VAPictureParameterBufferH264 */
2373     vaapi_fill_picture(&pic_param->CurrPic, picture, 0);
2374
2375     for (i = 0, n = 0; i < priv->dpb_count; i++) {
2376         GstVaapiFrameStore * const fs = priv->dpb[i];
2377         if (gst_vaapi_frame_store_has_reference(fs))
2378             vaapi_fill_picture(&pic_param->ReferenceFrames[n++],
2379                 fs->buffers[0], fs->structure);
2380     }
2381     for (; n < G_N_ELEMENTS(pic_param->ReferenceFrames); n++)
2382         vaapi_init_picture(&pic_param->ReferenceFrames[n]);
2383
2384 #define COPY_FIELD(s, f) \
2385     pic_param->f = (s)->f
2386
2387 #define COPY_BFM(a, s, f) \
2388     pic_param->a.bits.f = (s)->f
2389
2390     pic_param->picture_width_in_mbs_minus1  = priv->mb_width - 1;
2391     pic_param->picture_height_in_mbs_minus1 = priv->mb_height - 1;
2392     pic_param->frame_num                    = priv->frame_num;
2393
2394     COPY_FIELD(sps, bit_depth_luma_minus8);
2395     COPY_FIELD(sps, bit_depth_chroma_minus8);
2396     COPY_FIELD(sps, num_ref_frames);
2397     COPY_FIELD(pps, num_slice_groups_minus1);
2398     COPY_FIELD(pps, slice_group_map_type);
2399     COPY_FIELD(pps, slice_group_change_rate_minus1);
2400     COPY_FIELD(pps, pic_init_qp_minus26);
2401     COPY_FIELD(pps, pic_init_qs_minus26);
2402     COPY_FIELD(pps, chroma_qp_index_offset);
2403     COPY_FIELD(pps, second_chroma_qp_index_offset);
2404
2405     pic_param->seq_fields.value                                         = 0; /* reset all bits */
2406     pic_param->seq_fields.bits.residual_colour_transform_flag           = sps->separate_colour_plane_flag;
2407     pic_param->seq_fields.bits.MinLumaBiPredSize8x8                     = sps->level_idc >= 31; /* A.3.3.2 */
2408
2409     COPY_BFM(seq_fields, sps, chroma_format_idc);
2410     COPY_BFM(seq_fields, sps, gaps_in_frame_num_value_allowed_flag);
2411     COPY_BFM(seq_fields, sps, frame_mbs_only_flag);
2412     COPY_BFM(seq_fields, sps, mb_adaptive_frame_field_flag);
2413     COPY_BFM(seq_fields, sps, direct_8x8_inference_flag);
2414     COPY_BFM(seq_fields, sps, log2_max_frame_num_minus4);
2415     COPY_BFM(seq_fields, sps, pic_order_cnt_type);
2416     COPY_BFM(seq_fields, sps, log2_max_pic_order_cnt_lsb_minus4);
2417     COPY_BFM(seq_fields, sps, delta_pic_order_always_zero_flag);
2418
2419     pic_param->pic_fields.value                                         = 0; /* reset all bits */
2420     pic_param->pic_fields.bits.field_pic_flag                           = slice_hdr->field_pic_flag;
2421     pic_param->pic_fields.bits.reference_pic_flag                       = GST_VAAPI_PICTURE_IS_REFERENCE(picture);
2422
2423     COPY_BFM(pic_fields, pps, entropy_coding_mode_flag);
2424     COPY_BFM(pic_fields, pps, weighted_pred_flag);
2425     COPY_BFM(pic_fields, pps, weighted_bipred_idc);
2426     COPY_BFM(pic_fields, pps, transform_8x8_mode_flag);
2427     COPY_BFM(pic_fields, pps, constrained_intra_pred_flag);
2428     COPY_BFM(pic_fields, pps, pic_order_present_flag);
2429     COPY_BFM(pic_fields, pps, deblocking_filter_control_present_flag);
2430     COPY_BFM(pic_fields, pps, redundant_pic_cnt_present_flag);
2431     return TRUE;
2432 }
2433
2434 /* Detection of the first VCL NAL unit of a primary coded picture (7.4.1.2.4) */
2435 static gboolean
2436 is_new_picture(GstVaapiParserInfoH264 *pi, GstVaapiParserInfoH264 *prev_pi)
2437 {
2438     GstH264SliceHdr * const slice_hdr = &pi->data.slice_hdr;
2439     GstH264PPS * const pps = slice_hdr->pps;
2440     GstH264SPS * const sps = pps->sequence;
2441     GstH264SliceHdr *prev_slice_hdr;
2442
2443     if (!prev_pi)
2444         return TRUE;
2445     prev_slice_hdr = &prev_pi->data.slice_hdr;
2446
2447 #define CHECK_EXPR(expr, field_name) do {              \
2448         if (!(expr)) {                                 \
2449             GST_DEBUG(field_name " differs in value"); \
2450             return TRUE;                               \
2451         }                                              \
2452     } while (0)
2453
2454 #define CHECK_VALUE(new_slice_hdr, old_slice_hdr, field) \
2455     CHECK_EXPR(((new_slice_hdr)->field == (old_slice_hdr)->field), #field)
2456
2457     /* frame_num differs in value, regardless of inferred values to 0 */
2458     CHECK_VALUE(slice_hdr, prev_slice_hdr, frame_num);
2459
2460     /* pic_parameter_set_id differs in value */
2461     CHECK_VALUE(slice_hdr, prev_slice_hdr, pps);
2462
2463     /* field_pic_flag differs in value */
2464     CHECK_VALUE(slice_hdr, prev_slice_hdr, field_pic_flag);
2465
2466     /* bottom_field_flag is present in both and differs in value */
2467     if (slice_hdr->field_pic_flag && prev_slice_hdr->field_pic_flag)
2468         CHECK_VALUE(slice_hdr, prev_slice_hdr, bottom_field_flag);
2469
2470     /* nal_ref_idc differs in value with one of the nal_ref_idc values is 0 */
2471     CHECK_EXPR((pi->nalu.ref_idc != 0) ==
2472                (prev_pi->nalu.ref_idc != 0), "nal_ref_idc");
2473
2474     /* POC type is 0 for both and either pic_order_cnt_lsb differs in
2475        value or delta_pic_order_cnt_bottom differs in value */
2476     if (sps->pic_order_cnt_type == 0) {
2477         CHECK_VALUE(slice_hdr, prev_slice_hdr, pic_order_cnt_lsb);
2478         if (pps->pic_order_present_flag && !slice_hdr->field_pic_flag)
2479             CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt_bottom);
2480     }
2481
2482     /* POC type is 1 for both and either delta_pic_order_cnt[0]
2483        differs in value or delta_pic_order_cnt[1] differs in value */
2484     else if (sps->pic_order_cnt_type == 1) {
2485         CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt[0]);
2486         CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt[1]);
2487     }
2488
2489     /* IdrPicFlag differs in value */
2490     CHECK_VALUE(&pi->nalu, &prev_pi->nalu, idr_pic_flag);
2491
2492     /* IdrPicFlag is equal to 1 for both and idr_pic_id differs in value */
2493     if (pi->nalu.idr_pic_flag)
2494         CHECK_VALUE(slice_hdr, prev_slice_hdr, idr_pic_id);
2495
2496 #undef CHECK_EXPR
2497 #undef CHECK_VALUE
2498     return FALSE;
2499 }
2500
2501 static GstVaapiDecoderStatus
2502 decode_picture(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnit *unit)
2503 {
2504     GstVaapiDecoderH264Private * const priv = decoder->priv;
2505     GstVaapiParserInfoH264 * const pi = unit->parsed_info;
2506     GstH264SliceHdr * const slice_hdr = &pi->data.slice_hdr;
2507     GstH264PPS * const pps = slice_hdr->pps;
2508     GstH264SPS * const sps = pps->sequence;
2509     GstVaapiPictureH264 *picture;
2510     GstVaapiDecoderStatus status;
2511
2512     status = ensure_context(decoder, sps);
2513     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2514         return status;
2515
2516     if (priv->current_picture) {
2517         /* Re-use current picture where the first field was decoded */
2518         picture = gst_vaapi_picture_h264_new_field(priv->current_picture);
2519         if (!picture) {
2520             GST_ERROR("failed to allocate field picture");
2521             return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2522         }
2523     }
2524     else {
2525         /* Create new picture */
2526         picture = gst_vaapi_picture_h264_new(decoder);
2527         if (!picture) {
2528             GST_ERROR("failed to allocate picture");
2529             return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2530         }
2531     }
2532     gst_vaapi_picture_replace(&priv->current_picture, picture);
2533     gst_vaapi_picture_unref(picture);
2534
2535     picture->pps = pps;
2536
2537     status = ensure_quant_matrix(decoder, picture);
2538     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS) {
2539         GST_ERROR("failed to reset quantizer matrix");
2540         return status;
2541     }
2542
2543     if (!init_picture(decoder, picture, pi))
2544         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2545     if (!fill_picture(decoder, picture, pi))
2546         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2547     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2548 }
2549
2550 static inline guint
2551 get_slice_data_bit_offset(GstH264SliceHdr *slice_hdr)
2552 {
2553     guint epb_count;
2554
2555     epb_count = slice_hdr->n_emulation_prevention_bytes;
2556     return 8 /* nal_unit_type */ + slice_hdr->header_size - epb_count * 8;
2557 }
2558
2559 static gboolean
2560 fill_pred_weight_table(GstVaapiDecoderH264 *decoder,
2561     GstVaapiSlice *slice, GstH264SliceHdr *slice_hdr)
2562 {
2563     VASliceParameterBufferH264 * const slice_param = slice->param;
2564     GstH264PPS * const pps = slice_hdr->pps;
2565     GstH264SPS * const sps = pps->sequence;
2566     GstH264PredWeightTable * const w = &slice_hdr->pred_weight_table;
2567     guint num_weight_tables = 0;
2568     gint i, j;
2569
2570     if (pps->weighted_pred_flag &&
2571         (GST_H264_IS_P_SLICE(slice_hdr) || GST_H264_IS_SP_SLICE(slice_hdr)))
2572         num_weight_tables = 1;
2573     else if (pps->weighted_bipred_idc == 1 && GST_H264_IS_B_SLICE(slice_hdr))
2574         num_weight_tables = 2;
2575     else
2576         num_weight_tables = 0;
2577
2578     slice_param->luma_log2_weight_denom   = 0;
2579     slice_param->chroma_log2_weight_denom = 0;
2580     slice_param->luma_weight_l0_flag      = 0;
2581     slice_param->chroma_weight_l0_flag    = 0;
2582     slice_param->luma_weight_l1_flag      = 0;
2583     slice_param->chroma_weight_l1_flag    = 0;
2584
2585     if (num_weight_tables < 1)
2586         return TRUE;
2587
2588     slice_param->luma_log2_weight_denom   = w->luma_log2_weight_denom;
2589     slice_param->chroma_log2_weight_denom = w->chroma_log2_weight_denom;
2590
2591     slice_param->luma_weight_l0_flag = 1;
2592     for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2593         slice_param->luma_weight_l0[i] = w->luma_weight_l0[i];
2594         slice_param->luma_offset_l0[i] = w->luma_offset_l0[i];
2595     }
2596
2597     slice_param->chroma_weight_l0_flag = sps->chroma_array_type != 0;
2598     if (slice_param->chroma_weight_l0_flag) {
2599         for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2600             for (j = 0; j < 2; j++) {
2601                 slice_param->chroma_weight_l0[i][j] = w->chroma_weight_l0[i][j];
2602                 slice_param->chroma_offset_l0[i][j] = w->chroma_offset_l0[i][j];
2603             }
2604         }
2605     }
2606
2607     if (num_weight_tables < 2)
2608         return TRUE;
2609
2610     slice_param->luma_weight_l1_flag = 1;
2611     for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2612         slice_param->luma_weight_l1[i] = w->luma_weight_l1[i];
2613         slice_param->luma_offset_l1[i] = w->luma_offset_l1[i];
2614     }
2615
2616     slice_param->chroma_weight_l1_flag = sps->chroma_array_type != 0;
2617     if (slice_param->chroma_weight_l1_flag) {
2618         for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2619             for (j = 0; j < 2; j++) {
2620                 slice_param->chroma_weight_l1[i][j] = w->chroma_weight_l1[i][j];
2621                 slice_param->chroma_offset_l1[i][j] = w->chroma_offset_l1[i][j];
2622             }
2623         }
2624     }
2625     return TRUE;
2626 }
2627
2628 static gboolean
2629 fill_RefPicList(GstVaapiDecoderH264 *decoder,
2630     GstVaapiSlice *slice, GstH264SliceHdr *slice_hdr)
2631 {
2632     GstVaapiDecoderH264Private * const priv = decoder->priv;
2633     VASliceParameterBufferH264 * const slice_param = slice->param;
2634     guint i, num_ref_lists = 0;
2635
2636     slice_param->num_ref_idx_l0_active_minus1 = 0;
2637     slice_param->num_ref_idx_l1_active_minus1 = 0;
2638
2639     if (GST_H264_IS_B_SLICE(slice_hdr))
2640         num_ref_lists = 2;
2641     else if (GST_H264_IS_I_SLICE(slice_hdr))
2642         num_ref_lists = 0;
2643     else
2644         num_ref_lists = 1;
2645
2646     if (num_ref_lists < 1)
2647         return TRUE;
2648
2649     slice_param->num_ref_idx_l0_active_minus1 =
2650         slice_hdr->num_ref_idx_l0_active_minus1;
2651
2652     for (i = 0; i < priv->RefPicList0_count && priv->RefPicList0[i]; i++)
2653         vaapi_fill_picture(&slice_param->RefPicList0[i], priv->RefPicList0[i], 0);
2654     for (; i <= slice_param->num_ref_idx_l0_active_minus1; i++)
2655         vaapi_init_picture(&slice_param->RefPicList0[i]);
2656
2657     if (num_ref_lists < 2)
2658         return TRUE;
2659
2660     slice_param->num_ref_idx_l1_active_minus1 =
2661         slice_hdr->num_ref_idx_l1_active_minus1;
2662
2663     for (i = 0; i < priv->RefPicList1_count && priv->RefPicList1[i]; i++)
2664         vaapi_fill_picture(&slice_param->RefPicList1[i], priv->RefPicList1[i], 0);
2665     for (; i <= slice_param->num_ref_idx_l1_active_minus1; i++)
2666         vaapi_init_picture(&slice_param->RefPicList1[i]);
2667     return TRUE;
2668 }
2669
2670 static gboolean
2671 fill_slice(GstVaapiDecoderH264 *decoder,
2672     GstVaapiSlice *slice, GstH264SliceHdr *slice_hdr)
2673 {
2674     VASliceParameterBufferH264 * const slice_param = slice->param;
2675
2676     /* Fill in VASliceParameterBufferH264 */
2677     slice_param->slice_data_bit_offset          = get_slice_data_bit_offset(slice_hdr);
2678     slice_param->first_mb_in_slice              = slice_hdr->first_mb_in_slice;
2679     slice_param->slice_type                     = slice_hdr->type % 5;
2680     slice_param->direct_spatial_mv_pred_flag    = slice_hdr->direct_spatial_mv_pred_flag;
2681     slice_param->cabac_init_idc                 = slice_hdr->cabac_init_idc;
2682     slice_param->slice_qp_delta                 = slice_hdr->slice_qp_delta;
2683     slice_param->disable_deblocking_filter_idc  = slice_hdr->disable_deblocking_filter_idc;
2684     slice_param->slice_alpha_c0_offset_div2     = slice_hdr->slice_alpha_c0_offset_div2;
2685     slice_param->slice_beta_offset_div2         = slice_hdr->slice_beta_offset_div2;
2686
2687     if (!fill_RefPicList(decoder, slice, slice_hdr))
2688         return FALSE;
2689     if (!fill_pred_weight_table(decoder, slice, slice_hdr))
2690         return FALSE;
2691     return TRUE;
2692 }
2693
2694 static GstVaapiDecoderStatus
2695 decode_slice(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnit *unit)
2696 {
2697     GstVaapiDecoderH264Private * const priv = decoder->priv;
2698     GstVaapiParserInfoH264 * const pi = unit->parsed_info;
2699     GstVaapiPictureH264 * const picture = priv->current_picture;
2700     GstH264SliceHdr * const slice_hdr = &pi->data.slice_hdr;
2701     GstVaapiSlice *slice;
2702     GstBuffer * const buffer =
2703         GST_VAAPI_DECODER_CODEC_FRAME(decoder)->input_buffer;
2704     GstMapInfo map_info;
2705
2706     GST_DEBUG("slice (%u bytes)", pi->nalu.size);
2707
2708     if (!priv->got_sps || !priv->got_pps) {
2709         GST_ERROR("not initialized yet");
2710         return GST_VAAPI_DECODER_STATUS_SUCCESS;
2711     }
2712
2713     if (!gst_buffer_map(buffer, &map_info, GST_MAP_READ)) {
2714         GST_ERROR("failed to map buffer");
2715         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2716     }
2717
2718     slice = GST_VAAPI_SLICE_NEW(H264, decoder,
2719         (map_info.data + unit->offset + pi->nalu.offset), pi->nalu.size);
2720     gst_buffer_unmap(buffer, &map_info);
2721     if (!slice) {
2722         GST_ERROR("failed to allocate slice");
2723         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2724     }
2725
2726     if (!fill_slice(decoder, slice, slice_hdr)) {
2727         gst_vaapi_mini_object_unref(GST_VAAPI_MINI_OBJECT(slice));
2728         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2729     }
2730
2731     gst_vaapi_picture_add_slice(GST_VAAPI_PICTURE_CAST(picture), slice);
2732     picture->last_slice_hdr = slice_hdr;
2733     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2734 }
2735
2736 static inline gint
2737 scan_for_start_code(GstAdapter *adapter, guint ofs, guint size, guint32 *scp)
2738 {
2739     return (gint)gst_adapter_masked_scan_uint32_peek(adapter,
2740                                                      0xffffff00, 0x00000100,
2741                                                      ofs, size,
2742                                                      scp);
2743 }
2744
2745 static GstVaapiDecoderStatus
2746 decode_unit(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnit *unit)
2747 {
2748     GstVaapiParserInfoH264 * const pi = unit->parsed_info;
2749     GstVaapiDecoderStatus status;
2750
2751     switch (pi->nalu.type) {
2752     case GST_H264_NAL_SLICE_IDR:
2753         /* fall-through. IDR specifics are handled in init_picture() */
2754     case GST_H264_NAL_SLICE:
2755         status = decode_slice(decoder, unit);
2756         break;
2757     case GST_H264_NAL_SEQ_END:
2758     case GST_H264_NAL_STREAM_END:
2759         status = decode_sequence_end(decoder);
2760         break;
2761     case GST_H264_NAL_SEI:
2762         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2763         break;
2764     default:
2765         GST_WARNING("unsupported NAL unit type %d", pi->nalu.type);
2766         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
2767         break;
2768     }
2769     return status;
2770 }
2771
2772 static GstVaapiDecoderStatus
2773 gst_vaapi_decoder_h264_decode_codec_data(GstVaapiDecoder *base_decoder,
2774     const guchar *buf, guint buf_size)
2775 {
2776     GstVaapiDecoderH264 * const decoder =
2777         GST_VAAPI_DECODER_H264_CAST(base_decoder);
2778     GstVaapiDecoderH264Private * const priv = decoder->priv;
2779     GstVaapiDecoderStatus status;
2780     GstVaapiDecoderUnit unit;
2781     GstVaapiParserInfoH264 pi;
2782     GstH264ParserResult result;
2783     guint i, ofs, num_sps, num_pps;
2784
2785     unit.parsed_info = &pi;
2786
2787     if (buf_size < 8)
2788         return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2789
2790     if (buf[0] != 1) {
2791         GST_ERROR("failed to decode codec-data, not in avcC format");
2792         return GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
2793     }
2794
2795     priv->nal_length_size = (buf[4] & 0x03) + 1;
2796
2797     num_sps = buf[5] & 0x1f;
2798     ofs = 6;
2799
2800     for (i = 0; i < num_sps; i++) {
2801         result = gst_h264_parser_identify_nalu_avc(
2802             priv->parser,
2803             buf, ofs, buf_size, 2,
2804             &pi.nalu
2805         );
2806         if (result != GST_H264_PARSER_OK)
2807             return get_status(result);
2808
2809         status = parse_sps(decoder, &unit);
2810         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2811             return status;
2812         ofs = pi.nalu.offset + pi.nalu.size;
2813     }
2814
2815     num_pps = buf[ofs];
2816     ofs++;
2817
2818     for (i = 0; i < num_pps; i++) {
2819         result = gst_h264_parser_identify_nalu_avc(
2820             priv->parser,
2821             buf, ofs, buf_size, 2,
2822             &pi.nalu
2823         );
2824         if (result != GST_H264_PARSER_OK)
2825             return get_status(result);
2826
2827         status = parse_pps(decoder, &unit);
2828         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2829             return status;
2830         ofs = pi.nalu.offset + pi.nalu.size;
2831     }
2832
2833     priv->is_avcC = TRUE;
2834     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2835 }
2836
2837 static GstVaapiDecoderStatus
2838 ensure_decoder(GstVaapiDecoderH264 *decoder)
2839 {
2840     GstVaapiDecoderH264Private * const priv = decoder->priv;
2841     GstVaapiDecoderStatus status;
2842
2843     g_return_val_if_fail(priv->is_constructed,
2844                          GST_VAAPI_DECODER_STATUS_ERROR_INIT_FAILED);
2845
2846     if (!priv->is_opened) {
2847         priv->is_opened = gst_vaapi_decoder_h264_open(decoder);
2848         if (!priv->is_opened)
2849             return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CODEC;
2850
2851         status = gst_vaapi_decoder_decode_codec_data(
2852             GST_VAAPI_DECODER_CAST(decoder));
2853         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2854             return status;
2855     }
2856     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2857 }
2858
2859 static GstVaapiDecoderStatus
2860 gst_vaapi_decoder_h264_parse(GstVaapiDecoder *base_decoder,
2861     GstAdapter *adapter, gboolean at_eos, GstVaapiDecoderUnit *unit)
2862 {
2863     GstVaapiDecoderH264 * const decoder =
2864         GST_VAAPI_DECODER_H264_CAST(base_decoder);
2865     GstVaapiDecoderH264Private * const priv = decoder->priv;
2866     GstVaapiParserState * const ps = GST_VAAPI_PARSER_STATE(base_decoder);
2867     GstVaapiParserInfoH264 *pi;
2868     GstVaapiDecoderStatus status;
2869     GstH264ParserResult result;
2870     guchar *buf;
2871     guint i, size, buf_size, nalu_size, flags;
2872     guint32 start_code;
2873     gint ofs, ofs2;
2874
2875     status = ensure_decoder(decoder);
2876     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2877         return status;
2878
2879     size = gst_adapter_available(adapter);
2880
2881     if (priv->is_avcC) {
2882         if (size < priv->nal_length_size)
2883             return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2884
2885         buf = (guchar *)&start_code;
2886         g_assert(priv->nal_length_size <= sizeof(start_code));
2887         gst_adapter_copy(adapter, buf, 0, priv->nal_length_size);
2888
2889         nalu_size = 0;
2890         for (i = 0; i < priv->nal_length_size; i++)
2891             nalu_size = (nalu_size << 8) | buf[i];
2892
2893         buf_size = priv->nal_length_size + nalu_size;
2894         if (size < buf_size)
2895             return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2896     }
2897     else {
2898         if (size < 4)
2899             return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2900
2901         ofs = scan_for_start_code(adapter, 0, size, NULL);
2902         if (ofs < 0)
2903             return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2904
2905         if (ofs > 0) {
2906             gst_adapter_flush(adapter, ofs);
2907             size -= ofs;
2908         }
2909
2910         ofs2 = ps->input_offset2 - ofs - 4;
2911         if (ofs2 < 4)
2912             ofs2 = 4;
2913
2914         ofs = G_UNLIKELY(size < ofs2 + 4) ? -1 :
2915             scan_for_start_code(adapter, ofs2, size - ofs2, NULL);
2916         if (ofs < 0) {
2917             // Assume the whole NAL unit is present if end-of-stream
2918             if (!at_eos) {
2919                 ps->input_offset2 = size;
2920                 return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2921             }
2922             ofs = size;
2923         }
2924         buf_size = ofs;
2925     }
2926     ps->input_offset2 = 0;
2927
2928     buf = (guchar *)gst_adapter_map(adapter, buf_size);
2929     if (!buf)
2930         return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2931
2932     unit->size = buf_size;
2933
2934     pi = gst_vaapi_parser_info_h264_new();
2935     if (!pi)
2936         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2937
2938     gst_vaapi_decoder_unit_set_parsed_info(unit,
2939         pi, (GDestroyNotify)gst_vaapi_mini_object_unref);
2940
2941     if (priv->is_avcC)
2942         result = gst_h264_parser_identify_nalu_avc(priv->parser,
2943             buf, 0, buf_size, priv->nal_length_size, &pi->nalu);
2944     else
2945         result = gst_h264_parser_identify_nalu_unchecked(priv->parser,
2946             buf, 0, buf_size, &pi->nalu);
2947     status = get_status(result);
2948     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2949         return status;
2950
2951     switch (pi->nalu.type) {
2952     case GST_H264_NAL_SPS:
2953         status = parse_sps(decoder, unit);
2954         break;
2955     case GST_H264_NAL_PPS:
2956         status = parse_pps(decoder, unit);
2957         break;
2958     case GST_H264_NAL_SEI:
2959         status = parse_sei(decoder, unit);
2960         break;
2961     case GST_H264_NAL_SLICE_IDR:
2962     case GST_H264_NAL_SLICE:
2963         status = parse_slice(decoder, unit);
2964         break;
2965     default:
2966         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2967         break;
2968     }
2969     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2970         return status;
2971
2972     flags = 0;
2973     switch (pi->nalu.type) {
2974     case GST_H264_NAL_AU_DELIMITER:
2975         flags |= GST_VAAPI_DECODER_UNIT_FLAG_FRAME_START;
2976         /* fall-through */
2977     case GST_H264_NAL_FILLER_DATA:
2978         flags |= GST_VAAPI_DECODER_UNIT_FLAG_SKIP;
2979         break;
2980     case GST_H264_NAL_STREAM_END:
2981         flags |= GST_VAAPI_DECODER_UNIT_FLAG_STREAM_END;
2982         /* fall-through */
2983     case GST_H264_NAL_SEQ_END:
2984         flags |= GST_VAAPI_DECODER_UNIT_FLAG_FRAME_END;
2985         break;
2986     case GST_H264_NAL_SPS:
2987     case GST_H264_NAL_PPS:
2988         flags |= GST_VAAPI_DECODER_UNIT_FLAG_SKIP;
2989         /* fall-through */
2990     case GST_H264_NAL_SEI:
2991         flags |= GST_VAAPI_DECODER_UNIT_FLAG_FRAME_START;
2992         break;
2993     case GST_H264_NAL_SLICE_IDR:
2994     case GST_H264_NAL_SLICE:
2995         flags |= GST_VAAPI_DECODER_UNIT_FLAG_SLICE;
2996         if (is_new_picture(pi, priv->prev_slice_pi))
2997             flags |= GST_VAAPI_DECODER_UNIT_FLAG_FRAME_START;
2998         gst_vaapi_parser_info_h264_replace(&priv->prev_slice_pi, pi);
2999         break;
3000     default:
3001         if (pi->nalu.type >= 14 && pi->nalu.type <= 18)
3002             flags |= GST_VAAPI_DECODER_UNIT_FLAG_FRAME_START;
3003         break;
3004     }
3005     GST_VAAPI_DECODER_UNIT_FLAG_SET(unit, flags);
3006
3007     pi->nalu.data = NULL;
3008     return GST_VAAPI_DECODER_STATUS_SUCCESS;
3009 }
3010
3011 static GstVaapiDecoderStatus
3012 gst_vaapi_decoder_h264_decode(GstVaapiDecoder *base_decoder,
3013     GstVaapiDecoderUnit *unit)
3014 {
3015     GstVaapiDecoderH264 * const decoder =
3016         GST_VAAPI_DECODER_H264_CAST(base_decoder);
3017     GstVaapiDecoderStatus status;
3018
3019     status = ensure_decoder(decoder);
3020     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
3021         return status;
3022     return decode_unit(decoder, unit);
3023 }
3024
3025 static GstVaapiDecoderStatus
3026 gst_vaapi_decoder_h264_start_frame(GstVaapiDecoder *base_decoder,
3027     GstVaapiDecoderUnit *unit)
3028 {
3029     GstVaapiDecoderH264 * const decoder =
3030         GST_VAAPI_DECODER_H264_CAST(base_decoder);
3031
3032     return decode_picture(decoder, unit);
3033 }
3034
3035 static GstVaapiDecoderStatus
3036 gst_vaapi_decoder_h264_end_frame(GstVaapiDecoder *base_decoder)
3037 {
3038     GstVaapiDecoderH264 * const decoder =
3039         GST_VAAPI_DECODER_H264_CAST(base_decoder);
3040
3041     return decode_current_picture(decoder);
3042 }
3043
3044 static GstVaapiDecoderStatus
3045 gst_vaapi_decoder_h264_flush(GstVaapiDecoder *base_decoder)
3046 {
3047     GstVaapiDecoderH264 * const decoder =
3048         GST_VAAPI_DECODER_H264_CAST(base_decoder);
3049
3050     dpb_flush(decoder);
3051     return GST_VAAPI_DECODER_STATUS_SUCCESS;
3052 }
3053
3054 static void
3055 gst_vaapi_decoder_h264_finalize(GObject *object)
3056 {
3057     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264_CAST(object);
3058
3059     gst_vaapi_decoder_h264_destroy(decoder);
3060
3061     G_OBJECT_CLASS(gst_vaapi_decoder_h264_parent_class)->finalize(object);
3062 }
3063
3064 static void
3065 gst_vaapi_decoder_h264_constructed(GObject *object)
3066 {
3067     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264_CAST(object);
3068     GstVaapiDecoderH264Private * const priv = decoder->priv;
3069     GObjectClass *parent_class;
3070
3071     parent_class = G_OBJECT_CLASS(gst_vaapi_decoder_h264_parent_class);
3072     if (parent_class->constructed)
3073         parent_class->constructed(object);
3074
3075     priv->is_constructed = gst_vaapi_decoder_h264_create(decoder);
3076 }
3077
3078 static void
3079 gst_vaapi_decoder_h264_class_init(GstVaapiDecoderH264Class *klass)
3080 {
3081     GObjectClass * const object_class = G_OBJECT_CLASS(klass);
3082     GstVaapiDecoderClass * const decoder_class = GST_VAAPI_DECODER_CLASS(klass);
3083
3084     g_type_class_add_private(klass, sizeof(GstVaapiDecoderH264Private));
3085
3086     object_class->finalize      = gst_vaapi_decoder_h264_finalize;
3087     object_class->constructed   = gst_vaapi_decoder_h264_constructed;
3088
3089     decoder_class->parse        = gst_vaapi_decoder_h264_parse;
3090     decoder_class->decode       = gst_vaapi_decoder_h264_decode;
3091     decoder_class->start_frame  = gst_vaapi_decoder_h264_start_frame;
3092     decoder_class->end_frame    = gst_vaapi_decoder_h264_end_frame;
3093     decoder_class->flush        = gst_vaapi_decoder_h264_flush;
3094
3095     decoder_class->decode_codec_data =
3096         gst_vaapi_decoder_h264_decode_codec_data;
3097 }
3098
3099 static void
3100 gst_vaapi_decoder_h264_init(GstVaapiDecoderH264 *decoder)
3101 {
3102     GstVaapiDecoderH264Private *priv;
3103
3104     priv                        = GST_VAAPI_DECODER_H264_GET_PRIVATE(decoder);
3105     decoder->priv               = priv;
3106     priv->profile               = GST_VAAPI_PROFILE_UNKNOWN;
3107     priv->entrypoint            = GST_VAAPI_ENTRYPOINT_VLD;
3108     priv->chroma_type           = GST_VAAPI_CHROMA_TYPE_YUV420;
3109     priv->prev_pic_structure    = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
3110     priv->progressive_sequence  = TRUE;
3111 }
3112
3113 /**
3114  * gst_vaapi_decoder_h264_new:
3115  * @display: a #GstVaapiDisplay
3116  * @caps: a #GstCaps holding codec information
3117  *
3118  * Creates a new #GstVaapiDecoder for MPEG-2 decoding.  The @caps can
3119  * hold extra information like codec-data and pictured coded size.
3120  *
3121  * Return value: the newly allocated #GstVaapiDecoder object
3122  */
3123 GstVaapiDecoder *
3124 gst_vaapi_decoder_h264_new(GstVaapiDisplay *display, GstCaps *caps)
3125 {
3126     GstVaapiDecoderH264 *decoder;
3127
3128     g_return_val_if_fail(GST_VAAPI_IS_DISPLAY(display), NULL);
3129     g_return_val_if_fail(GST_IS_CAPS(caps), NULL);
3130
3131     decoder = g_object_new(
3132         GST_VAAPI_TYPE_DECODER_H264,
3133         "display",      display,
3134         "caps",         caps,
3135         NULL
3136     );
3137     if (!decoder->priv->is_constructed) {
3138         g_object_unref(decoder);
3139         return NULL;
3140     }
3141     return GST_VAAPI_DECODER_CAST(decoder);
3142 }