gst-libs/gst/vaapi/gstvaapidecoder_h264.c

   1 /*
   2  *  gstvaapidecoder_h264.c - H.264 decoder
   3  *
   4  *  Copyright (C) 2011-2012 Intel Corporation
   5  *
   6  *  This library is free software; you can redistribute it and/or
   7  *  modify it under the terms of the GNU Lesser General Public License
   8  *  as published by the Free Software Foundation; either version 2.1
   9  *  of the License, or (at your option) any later version.
  10  *
  11  *  This library is distributed in the hope that it will be useful,
  12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  *  Lesser General Public License for more details.
  15  *
  16  *  You should have received a copy of the GNU Lesser General Public
  17  *  License along with this library; if not, write to the Free
  18  *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  19  *  Boston, MA 02110-1301 USA
  20  */
  21
  22 /**
  23  * SECTION:gstvaapidecoder_h264
  24  * @short_description: H.264 decoder
  25  */
  26
  27 #include "sysdeps.h"
  28 #include <string.h>
  29 #include <gst/base/gstadapter.h>
  30 #include <gst/codecparsers/gsth264parser.h>
  31 #include "gstvaapidecoder_h264.h"
  32 #include "gstvaapidecoder_objects.h"
  33 #include "gstvaapidecoder_priv.h"
  34 #include "gstvaapidisplay_priv.h"
  35 #include "gstvaapiobject_priv.h"
  36
  37 #define DEBUG 1
  38 #include "gstvaapidebug.h"
  39
  40 /* Defined to 1 if strict ordering of DPB is needed. Only useful for debug */
  41 #define USE_STRICT_DPB_ORDERING 0
  42
  43 typedef struct _GstVaapiFrameStore              GstVaapiFrameStore;
  44 typedef struct _GstVaapiFrameStoreClass         GstVaapiFrameStoreClass;
  45 typedef struct _GstVaapiDecoderUnitH264         GstVaapiDecoderUnitH264;
  46 typedef struct _GstVaapiPictureH264             GstVaapiPictureH264;
  47
  48 // Used for field_poc[]
  49 #define TOP_FIELD       0
  50 #define BOTTOM_FIELD    1
  51
  52 /* ------------------------------------------------------------------------- */
  53 /* --- H.264 Decoder Units                                               --- */
  54 /* ------------------------------------------------------------------------- */
  55
  56 struct _GstVaapiDecoderUnitH264 {
  57     GstVaapiDecoderUnit base;
  58     GstH264NalUnit      nalu;
  59     union {
  60         GstH264SPS      sps;
  61         GstH264PPS      pps;
  62         GstH264SliceHdr slice_hdr;
  63     }                   data;
  64 };
  65
  66 static GstVaapiDecoderUnitH264 *
  67 gst_vaapi_decoder_unit_h264_new(guint size)
  68 {
  69     GstVaapiDecoderUnitH264 *unit;
  70
  71     static const GstVaapiMiniObjectClass GstVaapiDecoderUnitH264Class = {
  72         sizeof(GstVaapiDecoderUnitH264),
  73         (GDestroyNotify)gst_vaapi_decoder_unit_finalize
  74     };
  75
  76     unit = (GstVaapiDecoderUnitH264 *)
  77         gst_vaapi_mini_object_new0(&GstVaapiDecoderUnitH264Class);
  78     if (!unit)
  79         return NULL;
  80
  81     unit->base.size = size;
  82     return unit;
  83 }
  84
  85 /* ------------------------------------------------------------------------- */
  86 /* --- H.264 Pictures                                                    --- */
  87 /* ------------------------------------------------------------------------- */
  88
  89 /*
  90  * Extended picture flags:
  91  *
  92  * @GST_VAAPI_PICTURE_FLAG_IDR: flag that specifies an IDR picture
  93  * @GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE: flag that specifies
  94  *     "used for short-term reference"
  95  * @GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE: flag that specifies
  96  *     "used for long-term reference"
  97  * @GST_VAAPI_PICTURE_FLAGS_REFERENCE: mask covering any kind of
  98  *     reference picture (short-term reference or long-term reference)
  99  */
 100 enum {
 101     GST_VAAPI_PICTURE_FLAG_IDR = (GST_VAAPI_PICTURE_FLAG_LAST << 0),
 102
 103     GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE = (
 104         GST_VAAPI_PICTURE_FLAG_REFERENCE),
 105     GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE = (
 106         GST_VAAPI_PICTURE_FLAG_REFERENCE | (GST_VAAPI_PICTURE_FLAG_LAST << 1)),
 107     GST_VAAPI_PICTURE_FLAGS_REFERENCE = (
 108         GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE |
 109         GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE),
 110 };
 111
 112 #define GST_VAAPI_PICTURE_IS_IDR(picture) \
 113     (GST_VAAPI_PICTURE_FLAG_IS_SET(picture, GST_VAAPI_PICTURE_FLAG_IDR))
 114
 115 #define GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture)      \
 116     ((GST_VAAPI_PICTURE_FLAGS(picture) &                        \
 117       GST_VAAPI_PICTURE_FLAGS_REFERENCE) ==                     \
 118      GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE)
 119
 120 #define GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture)       \
 121     ((GST_VAAPI_PICTURE_FLAGS(picture) &                        \
 122       GST_VAAPI_PICTURE_FLAGS_REFERENCE) ==                     \
 123      GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE)
 124
 125 struct _GstVaapiPictureH264 {
 126     GstVaapiPicture             base;
 127     GstH264PPS                 *pps;
 128     GstH264SliceHdr            *last_slice_hdr;
 129     guint                       structure;
 130     gint32                      field_poc[2];
 131     gint32                      frame_num;              // Original frame_num from slice_header()
 132     gint32                      frame_num_wrap;         // Temporary for ref pic marking: FrameNumWrap
 133     gint32                      long_term_frame_idx;    // Temporary for ref pic marking: LongTermFrameIdx
 134     gint32                      pic_num;                // Temporary for ref pic marking: PicNum
 135     gint32                      long_term_pic_num;      // Temporary for ref pic marking: LongTermPicNum
 136     GstVaapiPictureH264        *other_field;            // Temporary for ref pic marking: other field in the same frame store
 137     guint                       output_flag             : 1;
 138     guint                       output_needed           : 1;
 139 };
 140
 141 GST_VAAPI_CODEC_DEFINE_TYPE(GstVaapiPictureH264, gst_vaapi_picture_h264);
 142
 143 void
 144 gst_vaapi_picture_h264_destroy(GstVaapiPictureH264 *picture)
 145 {
 146     gst_vaapi_picture_destroy(GST_VAAPI_PICTURE(picture));
 147 }
 148
 149 gboolean
 150 gst_vaapi_picture_h264_create(
 151     GstVaapiPictureH264                      *picture,
 152     const GstVaapiCodecObjectConstructorArgs *args
 153 )
 154 {
 155     if (!gst_vaapi_picture_create(GST_VAAPI_PICTURE(picture), args))
 156         return FALSE;
 157
 158     picture->field_poc[0]       = G_MAXINT32;
 159     picture->field_poc[1]       = G_MAXINT32;
 160     picture->output_needed      = FALSE;
 161     return TRUE;
 162 }
 163
 164 static inline GstVaapiPictureH264 *
 165 gst_vaapi_picture_h264_new(GstVaapiDecoderH264 *decoder)
 166 {
 167     return (GstVaapiPictureH264 *)gst_vaapi_codec_object_new(
 168         &GstVaapiPictureH264Class,
 169         GST_VAAPI_CODEC_BASE(decoder),
 170         NULL, sizeof(VAPictureParameterBufferH264),
 171         NULL, 0,
 172         0);
 173 }
 174
 175 static inline void
 176 gst_vaapi_picture_h264_set_reference(
 177     GstVaapiPictureH264 *picture,
 178     guint                reference_flags,
 179     gboolean             other_field
 180 )
 181 {
 182     if (!picture)
 183         return;
 184     GST_VAAPI_PICTURE_FLAG_UNSET(picture, GST_VAAPI_PICTURE_FLAGS_REFERENCE);
 185     GST_VAAPI_PICTURE_FLAG_SET(picture, reference_flags);
 186
 187     if (!other_field || !(picture = picture->other_field))
 188         return;
 189     GST_VAAPI_PICTURE_FLAG_UNSET(picture, GST_VAAPI_PICTURE_FLAGS_REFERENCE);
 190     GST_VAAPI_PICTURE_FLAG_SET(picture, reference_flags);
 191 }
 192
 193 static inline GstVaapiPictureH264 *
 194 gst_vaapi_picture_h264_new_field(GstVaapiPictureH264 *picture)
 195 {
 196     g_return_val_if_fail(picture, NULL);
 197
 198     return (GstVaapiPictureH264 *)gst_vaapi_picture_new_field(&picture->base);
 199 }
 200
 201 /* ------------------------------------------------------------------------- */
 202 /* --- Frame Buffers (DPB)                                               --- */
 203 /* ------------------------------------------------------------------------- */
 204
 205 struct _GstVaapiFrameStore {
 206     /*< private >*/
 207     GstVaapiMiniObject          parent_instance;
 208
 209     guint                       structure;
 210     GstVaapiPictureH264        *buffers[2];
 211     guint                       num_buffers;
 212     guint                       output_needed;
 213 };
 214
 215 static void
 216 gst_vaapi_frame_store_finalize(gpointer object)
 217 {
 218     GstVaapiFrameStore * const fs = object;
 219     guint i;
 220
 221     for (i = 0; i < fs->num_buffers; i++)
 222         gst_vaapi_picture_replace(&fs->buffers[i], NULL);
 223 }
 224
 225 static GstVaapiFrameStore *
 226 gst_vaapi_frame_store_new(GstVaapiPictureH264 *picture)
 227 {
 228     GstVaapiFrameStore *fs;
 229
 230     static const GstVaapiMiniObjectClass GstVaapiFrameStoreClass = {
 231         sizeof(GstVaapiFrameStore),
 232         gst_vaapi_frame_store_finalize
 233     };
 234
 235     fs = (GstVaapiFrameStore *)
 236         gst_vaapi_mini_object_new(&GstVaapiFrameStoreClass);
 237     if (!fs)
 238         return NULL;
 239
 240     fs->structure       = picture->structure;
 241     fs->buffers[0]      = gst_vaapi_picture_ref(picture);
 242     fs->buffers[1]      = NULL;
 243     fs->num_buffers     = 1;
 244     fs->output_needed   = picture->output_needed;
 245     return fs;
 246 }
 247
 248 static gboolean
 249 gst_vaapi_frame_store_add(GstVaapiFrameStore *fs, GstVaapiPictureH264 *picture)
 250 {
 251     guint field;
 252
 253     g_return_val_if_fail(fs->num_buffers == 1, FALSE);
 254     g_return_val_if_fail(!GST_VAAPI_PICTURE_IS_FRAME(picture), FALSE);
 255     g_return_val_if_fail(!GST_VAAPI_PICTURE_IS_FIRST_FIELD(picture), FALSE);
 256
 257     gst_vaapi_picture_replace(&fs->buffers[fs->num_buffers++], picture);
 258     if (picture->output_flag) {
 259         picture->output_needed = TRUE;
 260         fs->output_needed++;
 261     }
 262
 263     fs->structure = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
 264
 265     field = picture->structure == GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD ?
 266         TOP_FIELD : BOTTOM_FIELD;
 267     g_return_val_if_fail(fs->buffers[0]->field_poc[field] == G_MAXINT32, FALSE);
 268     fs->buffers[0]->field_poc[field] = picture->field_poc[field];
 269     g_return_val_if_fail(picture->field_poc[!field] == G_MAXINT32, FALSE);
 270     picture->field_poc[!field] = fs->buffers[0]->field_poc[!field];
 271     return TRUE;
 272 }
 273
 274 static gboolean
 275 gst_vaapi_frame_store_split_fields(GstVaapiFrameStore *fs)
 276 {
 277     GstVaapiPictureH264 * const first_field = fs->buffers[0];
 278     GstVaapiPictureH264 *second_field;
 279
 280     g_return_val_if_fail(fs->num_buffers == 1, FALSE);
 281
 282     first_field->base.structure = GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD;
 283     GST_VAAPI_PICTURE_FLAG_SET(first_field, GST_VAAPI_PICTURE_FLAG_INTERLACED);
 284
 285     second_field = gst_vaapi_picture_h264_new_field(first_field);
 286     if (!second_field)
 287         return FALSE;
 288     gst_vaapi_picture_replace(&fs->buffers[fs->num_buffers++], second_field);
 289     gst_vaapi_picture_unref(second_field);
 290
 291     second_field->frame_num    = first_field->frame_num;
 292     second_field->field_poc[0] = first_field->field_poc[0];
 293     second_field->field_poc[1] = first_field->field_poc[1];
 294     second_field->output_flag  = first_field->output_flag;
 295     if (second_field->output_flag) {
 296         second_field->output_needed = TRUE;
 297         fs->output_needed++;
 298     }
 299     return TRUE;
 300 }
 301
 302 static inline gboolean
 303 gst_vaapi_frame_store_has_frame(GstVaapiFrameStore *fs)
 304 {
 305     return fs->structure == GST_VAAPI_PICTURE_STRUCTURE_FRAME;
 306 }
 307
 308 static inline gboolean
 309 gst_vaapi_frame_store_has_reference(GstVaapiFrameStore *fs)
 310 {
 311     guint i;
 312
 313     for (i = 0; i < fs->num_buffers; i++) {
 314         if (GST_VAAPI_PICTURE_IS_REFERENCE(fs->buffers[i]))
 315             return TRUE;
 316     }
 317     return FALSE;
 318 }
 319
 320 #define gst_vaapi_frame_store_ref(fs) \
 321     gst_vaapi_mini_object_ref(GST_VAAPI_MINI_OBJECT(fs))
 322
 323 #define gst_vaapi_frame_store_unref(fs) \
 324     gst_vaapi_mini_object_unref(GST_VAAPI_MINI_OBJECT(fs))
 325
 326 #define gst_vaapi_frame_store_replace(old_fs_p, new_fs)                 \
 327     gst_vaapi_mini_object_replace((GstVaapiMiniObject **)(old_fs_p),    \
 328         (GstVaapiMiniObject *)(new_fs))
 329
 330 /* ------------------------------------------------------------------------- */
 331 /* --- H.264 Decoder                                                     --- */
 332 /* ------------------------------------------------------------------------- */
 333
 334 G_DEFINE_TYPE(GstVaapiDecoderH264,
 335               gst_vaapi_decoder_h264,
 336               GST_VAAPI_TYPE_DECODER)
 337
 338 #define GST_VAAPI_DECODER_H264_CAST(decoder) \
 339     ((GstVaapiDecoderH264 *)(decoder))
 340
 341 #define GST_VAAPI_DECODER_H264_GET_PRIVATE(obj)                 \
 342     (G_TYPE_INSTANCE_GET_PRIVATE((obj),                         \
 343                                  GST_VAAPI_TYPE_DECODER_H264,   \
 344                                  GstVaapiDecoderH264Private))
 345
 346 struct _GstVaapiDecoderH264Private {
 347     GstH264NalParser           *parser;
 348     GstVaapiPictureH264        *current_picture;
 349     GstVaapiDecoderUnitH264    *prev_slice_unit;
 350     GstVaapiFrameStore         *prev_frame;
 351     GstVaapiFrameStore         *dpb[16];
 352     guint                       dpb_count;
 353     guint                       dpb_size;
 354     GstVaapiProfile             profile;
 355     GstVaapiEntrypoint          entrypoint;
 356     GstVaapiChromaType          chroma_type;
 357     GstVaapiPictureH264        *short_ref[32];
 358     guint                       short_ref_count;
 359     GstVaapiPictureH264        *long_ref[32];
 360     guint                       long_ref_count;
 361     GstVaapiPictureH264        *RefPicList0[32];
 362     guint                       RefPicList0_count;
 363     GstVaapiPictureH264        *RefPicList1[32];
 364     guint                       RefPicList1_count;
 365     guint                       nal_length_size;
 366     guint                       mb_width;
 367     guint                       mb_height;
 368     gint32                      field_poc[2];           // 0:TopFieldOrderCnt / 1:BottomFieldOrderCnt
 369     gint32                      poc_msb;                // PicOrderCntMsb
 370     gint32                      poc_lsb;                // pic_order_cnt_lsb (from slice_header())
 371     gint32                      prev_poc_msb;           // prevPicOrderCntMsb
 372     gint32                      prev_poc_lsb;           // prevPicOrderCntLsb
 373     gint32                      frame_num_offset;       // FrameNumOffset
 374     gint32                      frame_num;              // frame_num (from slice_header())
 375     gint32                      prev_frame_num;         // prevFrameNum
 376     gboolean                    prev_pic_has_mmco5;     // prevMmco5Pic
 377     gboolean                    prev_pic_structure;     // previous picture structure
 378     guint                       is_constructed          : 1;
 379     guint                       is_opened               : 1;
 380     guint                       is_avcC                 : 1;
 381     guint                       got_sps                 : 1;
 382     guint                       got_pps                 : 1;
 383     guint                       has_context             : 1;
 384     guint                       progressive_sequence    : 1;
 385 };
 386
 387 static gboolean
 388 exec_ref_pic_marking(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture);
 389
 390 /* Get number of reference frames to use */
 391 static guint
 392 get_max_dec_frame_buffering(GstH264SPS *sps)
 393 {
 394     guint max_dec_frame_buffering, MaxDpbMbs, PicSizeMbs;
 395
 396     /* Table A-1 - Level limits */
 397     switch (sps->level_idc) {
 398     case 10: MaxDpbMbs = 396;    break;
 399     case 11: MaxDpbMbs = 900;    break;
 400     case 12: MaxDpbMbs = 2376;   break;
 401     case 13: MaxDpbMbs = 2376;   break;
 402     case 20: MaxDpbMbs = 2376;   break;
 403     case 21: MaxDpbMbs = 4752;   break;
 404     case 22: MaxDpbMbs = 8100;   break;
 405     case 30: MaxDpbMbs = 8100;   break;
 406     case 31: MaxDpbMbs = 18000;  break;
 407     case 32: MaxDpbMbs = 20480;  break;
 408     case 40: MaxDpbMbs = 32768;  break;
 409     case 41: MaxDpbMbs = 32768;  break;
 410     case 42: MaxDpbMbs = 34816;  break;
 411     case 50: MaxDpbMbs = 110400; break;
 412     case 51: MaxDpbMbs = 184320; break;
 413     default:
 414         g_assert(0 && "unhandled level");
 415         break;
 416     }
 417
 418     PicSizeMbs = ((sps->pic_width_in_mbs_minus1 + 1) *
 419                   (sps->pic_height_in_map_units_minus1 + 1) *
 420                   (sps->frame_mbs_only_flag ? 1 : 2));
 421     max_dec_frame_buffering = MaxDpbMbs / PicSizeMbs;
 422
 423     /* VUI parameters */
 424     if (sps->vui_parameters_present_flag) {
 425         GstH264VUIParams * const vui_params = &sps->vui_parameters;
 426         if (vui_params->bitstream_restriction_flag)
 427             max_dec_frame_buffering = vui_params->max_dec_frame_buffering;
 428         else {
 429             switch (sps->profile_idc) {
 430             case 44:  // CAVLC 4:4:4 Intra profile
 431             case 86:  // Scalable High profile
 432             case 100: // High profile
 433             case 110: // High 10 profile
 434             case 122: // High 4:2:2 profile
 435             case 244: // High 4:4:4 Predictive profile
 436                 if (sps->constraint_set3_flag)
 437                     max_dec_frame_buffering = 0;
 438                 break;
 439             }
 440         }
 441     }
 442
 443     if (max_dec_frame_buffering > 16)
 444         max_dec_frame_buffering = 16;
 445     else if (max_dec_frame_buffering < sps->num_ref_frames)
 446         max_dec_frame_buffering = sps->num_ref_frames;
 447     return MAX(1, max_dec_frame_buffering);
 448 }
 449
 450 static void
 451 array_remove_index_fast(void *array, guint *array_length_ptr, guint index)
 452 {
 453     gpointer * const entries = array;
 454     guint num_entries = *array_length_ptr;
 455
 456     g_return_if_fail(index < num_entries);
 457
 458     if (index != --num_entries)
 459         entries[index] = entries[num_entries];
 460     entries[num_entries] = NULL;
 461     *array_length_ptr = num_entries;
 462 }
 463
 464 #if 1
 465 static inline void
 466 array_remove_index(void *array, guint *array_length_ptr, guint index)
 467 {
 468     array_remove_index_fast(array, array_length_ptr, index);
 469 }
 470 #else
 471 static void
 472 array_remove_index(void *array, guint *array_length_ptr, guint index)
 473 {
 474     gpointer * const entries = array;
 475     const guint num_entries = *array_length_ptr - 1;
 476     guint i;
 477
 478     g_return_if_fail(index <= num_entries);
 479
 480     for (i = index; i < num_entries; i++)
 481         entries[i] = entries[i + 1];
 482     entries[num_entries] = NULL;
 483     *array_length_ptr = num_entries;
 484 }
 485 #endif
 486
 487 #define ARRAY_REMOVE_INDEX(array, index) \
 488     array_remove_index(array, &array##_count, index)
 489
 490 static void
 491 dpb_remove_index(GstVaapiDecoderH264 *decoder, guint index)
 492 {
 493     GstVaapiDecoderH264Private * const priv = decoder->priv;
 494     guint i, num_frames = --priv->dpb_count;
 495
 496     if (USE_STRICT_DPB_ORDERING) {
 497         for (i = index; i < num_frames; i++)
 498             gst_vaapi_frame_store_replace(&priv->dpb[i], priv->dpb[i + 1]);
 499     }
 500     else if (index != num_frames)
 501         gst_vaapi_frame_store_replace(&priv->dpb[index], priv->dpb[num_frames]);
 502     gst_vaapi_frame_store_replace(&priv->dpb[num_frames], NULL);
 503 }
 504
 505 static gboolean
 506 dpb_output(
 507     GstVaapiDecoderH264 *decoder,
 508     GstVaapiFrameStore  *fs,
 509     GstVaapiPictureH264 *picture
 510 )
 511 {
 512     picture->output_needed = FALSE;
 513
 514     if (fs) {
 515         if (--fs->output_needed > 0)
 516             return TRUE;
 517         picture = fs->buffers[0];
 518     }
 519
 520     /* XXX: update cropping rectangle */
 521     return gst_vaapi_picture_output(GST_VAAPI_PICTURE_CAST(picture));
 522 }
 523
 524 static inline void
 525 dpb_evict(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture, guint i)
 526 {
 527     GstVaapiFrameStore * const fs = decoder->priv->dpb[i];
 528
 529     if (!fs->output_needed && !gst_vaapi_frame_store_has_reference(fs))
 530         dpb_remove_index(decoder, i);
 531 }
 532
 533 static gboolean
 534 dpb_bump(GstVaapiDecoderH264 *decoder)
 535 {
 536     GstVaapiDecoderH264Private * const priv = decoder->priv;
 537     GstVaapiPictureH264 *found_picture = NULL;
 538     guint i, j, found_index;
 539     gboolean success;
 540
 541     for (i = 0; i < priv->dpb_count; i++) {
 542         GstVaapiFrameStore * const fs = priv->dpb[i];
 543         if (!fs->output_needed)
 544             continue;
 545         for (j = 0; j < fs->num_buffers; j++) {
 546             GstVaapiPictureH264 * const picture = fs->buffers[j];
 547             if (!picture->output_needed)
 548                 continue;
 549             if (!found_picture || found_picture->base.poc > picture->base.poc)
 550                 found_picture = picture, found_index = i;
 551         }
 552     }
 553     if (!found_picture)
 554         return FALSE;
 555
 556     success = dpb_output(decoder, priv->dpb[found_index], found_picture);
 557     dpb_evict(decoder, found_picture, found_index);
 558     return success;
 559 }
 560
 561 static void
 562 dpb_clear(GstVaapiDecoderH264 *decoder)
 563 {
 564     GstVaapiDecoderH264Private * const priv = decoder->priv;
 565     guint i;
 566
 567     for (i = 0; i < priv->dpb_count; i++)
 568         gst_vaapi_frame_store_replace(&priv->dpb[i], NULL);
 569     priv->dpb_count = 0;
 570
 571     gst_vaapi_frame_store_replace(&priv->prev_frame, NULL);
 572 }
 573
 574 static void
 575 dpb_flush(GstVaapiDecoderH264 *decoder)
 576 {
 577     while (dpb_bump(decoder))
 578         ;
 579     dpb_clear(decoder);
 580 }
 581
 582 static gboolean
 583 dpb_add(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
 584 {
 585     GstVaapiDecoderH264Private * const priv = decoder->priv;
 586     GstVaapiFrameStore *fs;
 587     guint i, j;
 588
 589     // Remove all unused pictures
 590     if (!GST_VAAPI_PICTURE_IS_IDR(picture)) {
 591         i = 0;
 592         while (i < priv->dpb_count) {
 593             GstVaapiFrameStore * const fs = priv->dpb[i];
 594             if (!fs->output_needed && !gst_vaapi_frame_store_has_reference(fs))
 595                 dpb_remove_index(decoder, i);
 596             else
 597                 i++;
 598         }
 599     }
 600
 601     // Check if picture is the second field and the first field is still in DPB
 602     fs = priv->prev_frame;
 603     if (fs && !gst_vaapi_frame_store_has_frame(fs))
 604         return gst_vaapi_frame_store_add(fs, picture);
 605
 606     // Create new frame store, and split fields if necessary
 607     fs = gst_vaapi_frame_store_new(picture);
 608     if (!fs)
 609         return FALSE;
 610     gst_vaapi_frame_store_replace(&priv->prev_frame, fs);
 611     gst_vaapi_frame_store_unref(fs);
 612
 613     if (!priv->progressive_sequence && gst_vaapi_frame_store_has_frame(fs)) {
 614         if (!gst_vaapi_frame_store_split_fields(fs))
 615             return FALSE;
 616     }
 617
 618     // C.4.5.1 - Storage and marking of a reference decoded picture into the DPB
 619     if (GST_VAAPI_PICTURE_IS_REFERENCE(picture)) {
 620         while (priv->dpb_count == priv->dpb_size) {
 621             if (!dpb_bump(decoder))
 622                 return FALSE;
 623         }
 624         gst_vaapi_frame_store_replace(&priv->dpb[priv->dpb_count++], fs);
 625         if (picture->output_flag) {
 626             picture->output_needed = TRUE;
 627             fs->output_needed++;
 628         }
 629     }
 630
 631     // C.4.5.2 - Storage and marking of a non-reference decoded picture into the DPB
 632     else {
 633         if (!picture->output_flag)
 634             return TRUE;
 635         while (priv->dpb_count == priv->dpb_size) {
 636             gboolean found_picture = FALSE;
 637             for (i = 0; !found_picture && i < priv->dpb_count; i++) {
 638                 GstVaapiFrameStore * const fs = priv->dpb[i];
 639                 if (!fs->output_needed)
 640                     continue;
 641                 for (j = 0; !found_picture && j < fs->num_buffers; j++)
 642                     found_picture = fs->buffers[j]->output_needed &&
 643                         fs->buffers[j]->base.poc < picture->base.poc;
 644             }
 645             if (!found_picture)
 646                 return dpb_output(decoder, NULL, picture);
 647             if (!dpb_bump(decoder))
 648                 return FALSE;
 649         }
 650         gst_vaapi_frame_store_replace(&priv->dpb[priv->dpb_count++], fs);
 651         picture->output_needed = TRUE;
 652         fs->output_needed++;
 653     }
 654     return TRUE;
 655 }
 656
 657 static inline void
 658 dpb_reset(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 659 {
 660     GstVaapiDecoderH264Private * const priv = decoder->priv;
 661
 662     priv->dpb_size = get_max_dec_frame_buffering(sps);
 663     GST_DEBUG("DPB size %u", priv->dpb_size);
 664 }
 665
 666 static GstVaapiDecoderStatus
 667 get_status(GstH264ParserResult result)
 668 {
 669     GstVaapiDecoderStatus status;
 670
 671     switch (result) {
 672     case GST_H264_PARSER_OK:
 673         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
 674         break;
 675     case GST_H264_PARSER_NO_NAL_END:
 676         status = GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
 677         break;
 678     case GST_H264_PARSER_ERROR:
 679         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
 680         break;
 681     default:
 682         status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 683         break;
 684     }
 685     return status;
 686 }
 687
 688 static void
 689 gst_vaapi_decoder_h264_close(GstVaapiDecoderH264 *decoder)
 690 {
 691     GstVaapiDecoderH264Private * const priv = decoder->priv;
 692
 693     gst_vaapi_picture_replace(&priv->current_picture, NULL);
 694     gst_vaapi_decoder_unit_replace(&priv->prev_slice_unit, NULL);
 695
 696     dpb_clear(decoder);
 697
 698     if (priv->parser) {
 699         gst_h264_nal_parser_free(priv->parser);
 700         priv->parser = NULL;
 701     }
 702 }
 703
 704 static gboolean
 705 gst_vaapi_decoder_h264_open(GstVaapiDecoderH264 *decoder)
 706 {
 707     GstVaapiDecoderH264Private * const priv = decoder->priv;
 708
 709     gst_vaapi_decoder_h264_close(decoder);
 710
 711     priv->parser = gst_h264_nal_parser_new();
 712     if (!priv->parser)
 713         return FALSE;
 714     return TRUE;
 715 }
 716
 717 static void
 718 gst_vaapi_decoder_h264_destroy(GstVaapiDecoderH264 *decoder)
 719 {
 720     gst_vaapi_decoder_h264_close(decoder);
 721 }
 722
 723 static gboolean
 724 gst_vaapi_decoder_h264_create(GstVaapiDecoderH264 *decoder)
 725 {
 726     if (!GST_VAAPI_DECODER_CODEC(decoder))
 727         return FALSE;
 728     return TRUE;
 729 }
 730
 731 static guint
 732 h264_get_profile(GstH264SPS *sps)
 733 {
 734     guint profile = 0;
 735
 736     switch (sps->profile_idc) {
 737     case 66:
 738         profile = GST_VAAPI_PROFILE_H264_BASELINE;
 739         break;
 740     case 77:
 741         profile = GST_VAAPI_PROFILE_H264_MAIN;
 742         break;
 743     case 100:
 744         profile = GST_VAAPI_PROFILE_H264_HIGH;
 745         break;
 746     }
 747     return profile;
 748 }
 749
 750 static guint
 751 h264_get_chroma_type(GstH264SPS *sps)
 752 {
 753     guint chroma_type = 0;
 754
 755     switch (sps->chroma_format_idc) {
 756     case 1:
 757         chroma_type = GST_VAAPI_CHROMA_TYPE_YUV420;
 758         break;
 759     case 2:
 760         chroma_type = GST_VAAPI_CHROMA_TYPE_YUV422;
 761         break;
 762     case 3:
 763         if (!sps->separate_colour_plane_flag)
 764             chroma_type = GST_VAAPI_CHROMA_TYPE_YUV444;
 765         break;
 766     }
 767     return chroma_type;
 768 }
 769
 770 static GstVaapiProfile
 771 get_profile(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 772 {
 773     GstVaapiDecoderH264Private * const priv = decoder->priv;
 774     GstVaapiDisplay * const display = GST_VAAPI_DECODER_DISPLAY(decoder);
 775     GstVaapiProfile profile, profiles[2];
 776     guint i, n_profiles = 0;
 777
 778     profile = h264_get_profile(sps);
 779     if (!profile)
 780         return GST_VAAPI_PROFILE_UNKNOWN;
 781
 782     profiles[n_profiles++] = profile;
 783     switch (profile) {
 784     case GST_VAAPI_PROFILE_H264_MAIN:
 785         profiles[n_profiles++] = GST_VAAPI_PROFILE_H264_HIGH;
 786         break;
 787     default:
 788         break;
 789     }
 790
 791     /* If the preferred profile (profiles[0]) matches one that we already
 792        found, then just return it now instead of searching for it again */
 793     if (profiles[0] == priv->profile)
 794         return priv->profile;
 795
 796     for (i = 0; i < n_profiles; i++) {
 797         if (gst_vaapi_display_has_decoder(display, profiles[i], priv->entrypoint))
 798             return profiles[i];
 799     }
 800     return GST_VAAPI_PROFILE_UNKNOWN;
 801 }
 802
 803 static GstVaapiDecoderStatus
 804 ensure_context(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 805 {
 806     GstVaapiDecoder * const base_decoder = GST_VAAPI_DECODER_CAST(decoder);
 807     GstVaapiDecoderH264Private * const priv = decoder->priv;
 808     GstVaapiContextInfo info;
 809     GstVaapiProfile profile;
 810     GstVaapiChromaType chroma_type;
 811     gboolean reset_context = FALSE;
 812     guint mb_width, mb_height;
 813
 814     profile = get_profile(decoder, sps);
 815     if (!profile) {
 816         GST_ERROR("unsupported profile_idc %u", sps->profile_idc);
 817         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_PROFILE;
 818     }
 819
 820     if (priv->profile != profile) {
 821         GST_DEBUG("profile changed");
 822         reset_context = TRUE;
 823         priv->profile = profile;
 824     }
 825
 826     chroma_type = h264_get_chroma_type(sps);
 827     if (!chroma_type || chroma_type != GST_VAAPI_CHROMA_TYPE_YUV420) {
 828         GST_ERROR("unsupported chroma_format_idc %u", sps->chroma_format_idc);
 829         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;
 830     }
 831
 832     if (priv->chroma_type != chroma_type) {
 833         GST_DEBUG("chroma format changed");
 834         reset_context     = TRUE;
 835         priv->chroma_type = chroma_type;
 836     }
 837
 838     mb_width  = sps->pic_width_in_mbs_minus1 + 1;
 839     mb_height = (sps->pic_height_in_map_units_minus1 + 1) <<
 840         !sps->frame_mbs_only_flag;
 841     if (priv->mb_width != mb_width || priv->mb_height != mb_height) {
 842         GST_DEBUG("size changed");
 843         reset_context   = TRUE;
 844         priv->mb_width  = mb_width;
 845         priv->mb_height = mb_height;
 846     }
 847
 848     priv->progressive_sequence = sps->frame_mbs_only_flag;
 849 #if 0
 850     /* XXX: we only output complete frames for now */
 851     gst_vaapi_decoder_set_interlaced(base_decoder, !priv->progressive_sequence);
 852 #endif
 853
 854     gst_vaapi_decoder_set_pixel_aspect_ratio(
 855         base_decoder,
 856         sps->vui_parameters.par_n,
 857         sps->vui_parameters.par_d
 858     );
 859
 860     if (!reset_context && priv->has_context)
 861         return GST_VAAPI_DECODER_STATUS_SUCCESS;
 862
 863     /* XXX: fix surface size when cropping is implemented */
 864     info.profile    = priv->profile;
 865     info.entrypoint = priv->entrypoint;
 866     info.width      = sps->width;
 867     info.height     = sps->height;
 868     info.ref_frames = get_max_dec_frame_buffering(sps);
 869
 870     if (!gst_vaapi_decoder_ensure_context(GST_VAAPI_DECODER(decoder), &info))
 871         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 872     priv->has_context = TRUE;
 873
 874     /* Reset DPB */
 875     dpb_reset(decoder, sps);
 876     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 877 }
 878
 879 static void
 880 fill_iq_matrix_4x4(VAIQMatrixBufferH264 *iq_matrix, const GstH264PPS *pps)
 881 {
 882     const guint8 (* const ScalingList4x4)[6][16] = &pps->scaling_lists_4x4;
 883     guint i, j;
 884
 885     /* There are always 6 4x4 scaling lists */
 886     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList4x4) == 6);
 887     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList4x4[0]) == 16);
 888
 889     if (sizeof(iq_matrix->ScalingList4x4[0][0]) == 1)
 890         memcpy(iq_matrix->ScalingList4x4, *ScalingList4x4,
 891                sizeof(iq_matrix->ScalingList4x4));
 892     else {
 893         for (i = 0; i < G_N_ELEMENTS(iq_matrix->ScalingList4x4); i++) {
 894             for (j = 0; j < G_N_ELEMENTS(iq_matrix->ScalingList4x4[i]); j++)
 895                 iq_matrix->ScalingList4x4[i][j] = (*ScalingList4x4)[i][j];
 896         }
 897     }
 898 }
 899
 900 static void
 901 fill_iq_matrix_8x8(VAIQMatrixBufferH264 *iq_matrix, const GstH264PPS *pps)
 902 {
 903     const guint8 (* const ScalingList8x8)[6][64] = &pps->scaling_lists_8x8;
 904     const GstH264SPS * const sps = pps->sequence;
 905     guint i, j, n;
 906
 907     /* If chroma_format_idc != 3, there are up to 2 8x8 scaling lists */
 908     if (!pps->transform_8x8_mode_flag)
 909         return;
 910
 911     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList8x8) >= 2);
 912     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList8x8[0]) == 64);
 913
 914     if (sizeof(iq_matrix->ScalingList8x8[0][0]) == 1)
 915         memcpy(iq_matrix->ScalingList8x8, *ScalingList8x8,
 916                sizeof(iq_matrix->ScalingList8x8));
 917     else {
 918         n = (sps->chroma_format_idc != 3) ? 2 : 6;
 919         for (i = 0; i < n; i++) {
 920             for (j = 0; j < G_N_ELEMENTS(iq_matrix->ScalingList8x8[i]); j++)
 921                 iq_matrix->ScalingList8x8[i][j] = (*ScalingList8x8)[i][j];
 922         }
 923     }
 924 }
 925
 926 static GstVaapiDecoderStatus
 927 ensure_quant_matrix(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
 928 {
 929     GstVaapiPicture * const base_picture = &picture->base;
 930     GstH264PPS * const pps = picture->pps;
 931     GstH264SPS * const sps = pps->sequence;
 932     VAIQMatrixBufferH264 *iq_matrix;
 933
 934     base_picture->iq_matrix = GST_VAAPI_IQ_MATRIX_NEW(H264, decoder);
 935     if (!base_picture->iq_matrix) {
 936         GST_ERROR("failed to allocate IQ matrix");
 937         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
 938     }
 939     iq_matrix = base_picture->iq_matrix->param;
 940
 941     /* XXX: we can only support 4:2:0 or 4:2:2 since ScalingLists8x8[]
 942        is not large enough to hold lists for 4:4:4 */
 943     if (sps->chroma_format_idc == 3)
 944         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;
 945
 946     fill_iq_matrix_4x4(iq_matrix, pps);
 947     fill_iq_matrix_8x8(iq_matrix, pps);
 948
 949     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 950 }
 951
 952 static GstVaapiDecoderStatus
 953 decode_current_picture(GstVaapiDecoderH264 *decoder)
 954 {
 955     GstVaapiDecoderH264Private * const priv = decoder->priv;
 956     GstVaapiPictureH264 * const picture = priv->current_picture;
 957
 958     if (!picture)
 959         return GST_VAAPI_DECODER_STATUS_SUCCESS;
 960
 961     if (!exec_ref_pic_marking(decoder, picture))
 962         goto error;
 963     if (!dpb_add(decoder, picture))
 964         goto error;
 965     if (!gst_vaapi_picture_decode(GST_VAAPI_PICTURE_CAST(picture)))
 966         goto error;
 967     if (priv->prev_frame && gst_vaapi_frame_store_has_frame(priv->prev_frame))
 968         gst_vaapi_picture_replace(&priv->current_picture, NULL);
 969     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 970
 971 error:
 972     /* XXX: fix for cases where first field failed to be decoded */
 973     gst_vaapi_picture_replace(&priv->current_picture, NULL);
 974     return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 975 }
 976
 977 static GstVaapiDecoderStatus
 978 parse_sps(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnitH264 *unit)
 979 {
 980     GstVaapiDecoderH264Private * const priv = decoder->priv;
 981     GstH264SPS * const sps = &unit->data.sps;
 982     GstH264ParserResult result;
 983
 984     GST_DEBUG("parse SPS");
 985
 986     result = gst_h264_parser_parse_sps(priv->parser, &unit->nalu, sps, TRUE);
 987     if (result != GST_H264_PARSER_OK)
 988         return get_status(result);
 989
 990     priv->got_sps = TRUE;
 991     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 992 }
 993
 994 static GstVaapiDecoderStatus
 995 parse_pps(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnitH264 *unit)
 996 {
 997     GstVaapiDecoderH264Private * const priv = decoder->priv;
 998     GstH264PPS * const pps = &unit->data.pps;
 999     GstH264ParserResult result;
1000
1001     GST_DEBUG("parse PPS");
1002
1003     result = gst_h264_parser_parse_pps(priv->parser, &unit->nalu, pps);
1004     if (result != GST_H264_PARSER_OK)
1005         return get_status(result);
1006
1007     priv->got_pps = TRUE;
1008     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1009 }
1010
1011 static GstVaapiDecoderStatus
1012 parse_sei(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnitH264 *unit)
1013 {
1014     GstVaapiDecoderH264Private * const priv = decoder->priv;
1015     GstH264SEIMessage sei;
1016     GstH264ParserResult result;
1017
1018     GST_DEBUG("parse SEI");
1019
1020     memset(&sei, 0, sizeof(sei));
1021     result = gst_h264_parser_parse_sei(priv->parser, &unit->nalu, &sei);
1022     if (result != GST_H264_PARSER_OK) {
1023         GST_WARNING("failed to parse SEI, payload type:%d", sei.payloadType);
1024         return get_status(result);
1025     }
1026
1027     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1028 }
1029
1030 static GstVaapiDecoderStatus
1031 parse_slice(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnitH264 *unit)
1032 {
1033     GstVaapiDecoderH264Private * const priv = decoder->priv;
1034     GstH264SliceHdr * const slice_hdr = &unit->data.slice_hdr;
1035     GstH264ParserResult result;
1036
1037     GST_DEBUG("parse slice");
1038
1039     result = gst_h264_parser_parse_slice_hdr(priv->parser, &unit->nalu,
1040         slice_hdr, TRUE, TRUE);
1041     if (result != GST_H264_PARSER_OK)
1042         return get_status(result);
1043
1044     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1045 }
1046
1047 static GstVaapiDecoderStatus
1048 decode_sequence_end(GstVaapiDecoderH264 *decoder)
1049 {
1050     GstVaapiDecoderStatus status;
1051
1052     GST_DEBUG("decode sequence-end");
1053
1054     status = decode_current_picture(decoder);
1055     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
1056         return status;
1057
1058     dpb_flush(decoder);
1059     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1060 }
1061
1062 /* 8.2.1.1 - Decoding process for picture order count type 0 */
1063 static void
1064 init_picture_poc_0(
1065     GstVaapiDecoderH264 *decoder,
1066     GstVaapiPictureH264 *picture,
1067     GstH264SliceHdr     *slice_hdr
1068 )
1069 {
1070     GstVaapiDecoderH264Private * const priv = decoder->priv;
1071     GstH264PPS * const pps = slice_hdr->pps;
1072     GstH264SPS * const sps = pps->sequence;
1073     const gint32 MaxPicOrderCntLsb = 1 << (sps->log2_max_pic_order_cnt_lsb_minus4 + 4);
1074     gint32 temp_poc;
1075
1076     GST_DEBUG("decode picture order count type 0");
1077
1078     if (GST_VAAPI_PICTURE_IS_IDR(picture)) {
1079         priv->prev_poc_msb = 0;
1080         priv->prev_poc_lsb = 0;
1081     }
1082     else if (priv->prev_pic_has_mmco5) {
1083         priv->prev_poc_msb = 0;
1084         priv->prev_poc_lsb =
1085             (priv->prev_pic_structure == GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD ?
1086              0 : priv->field_poc[TOP_FIELD]);
1087     }
1088     else {
1089         priv->prev_poc_msb = priv->poc_msb;
1090         priv->prev_poc_lsb = priv->poc_lsb;
1091     }
1092
1093     // (8-3)
1094     priv->poc_lsb = slice_hdr->pic_order_cnt_lsb;
1095     if (priv->poc_lsb < priv->prev_poc_lsb &&
1096         (priv->prev_poc_lsb - priv->poc_lsb) >= (MaxPicOrderCntLsb / 2))
1097         priv->poc_msb = priv->prev_poc_msb + MaxPicOrderCntLsb;
1098     else if (priv->poc_lsb > priv->prev_poc_lsb &&
1099              (priv->poc_lsb - priv->prev_poc_lsb) > (MaxPicOrderCntLsb / 2))
1100         priv->poc_msb = priv->prev_poc_msb - MaxPicOrderCntLsb;
1101     else
1102         priv->poc_msb = priv->prev_poc_msb;
1103
1104     temp_poc = priv->poc_msb + priv->poc_lsb;
1105     switch (picture->structure) {
1106     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
1107         // (8-4, 8-5)
1108         priv->field_poc[TOP_FIELD] = temp_poc;
1109         priv->field_poc[BOTTOM_FIELD] = temp_poc +
1110             slice_hdr->delta_pic_order_cnt_bottom;
1111         break;
1112     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
1113         // (8-4)
1114         priv->field_poc[TOP_FIELD] = temp_poc;
1115         break;
1116     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
1117         // (8-5)
1118         priv->field_poc[BOTTOM_FIELD] = temp_poc;
1119         break;
1120     }
1121 }
1122
1123 /* 8.2.1.2 - Decoding process for picture order count type 1 */
1124 static void
1125 init_picture_poc_1(
1126     GstVaapiDecoderH264 *decoder,
1127     GstVaapiPictureH264 *picture,
1128     GstH264SliceHdr     *slice_hdr
1129 )
1130 {
1131     GstVaapiDecoderH264Private * const priv = decoder->priv;
1132     GstH264PPS * const pps = slice_hdr->pps;
1133     GstH264SPS * const sps = pps->sequence;
1134     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1135     gint32 prev_frame_num_offset, abs_frame_num, expected_poc;
1136     guint i;
1137
1138     GST_DEBUG("decode picture order count type 1");
1139
1140     if (priv->prev_pic_has_mmco5)
1141         prev_frame_num_offset = 0;
1142     else
1143         prev_frame_num_offset = priv->frame_num_offset;
1144
1145     // (8-6)
1146     if (GST_VAAPI_PICTURE_IS_IDR(picture))
1147         priv->frame_num_offset = 0;
1148     else if (priv->prev_frame_num > priv->frame_num)
1149         priv->frame_num_offset = prev_frame_num_offset + MaxFrameNum;
1150     else
1151         priv->frame_num_offset = prev_frame_num_offset;
1152
1153     // (8-7)
1154     if (sps->num_ref_frames_in_pic_order_cnt_cycle != 0)
1155         abs_frame_num = priv->frame_num_offset + priv->frame_num;
1156     else
1157         abs_frame_num = 0;
1158     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture) && abs_frame_num > 0)
1159         abs_frame_num = abs_frame_num - 1;
1160
1161     if (abs_frame_num > 0) {
1162         gint32 expected_delta_per_poc_cycle;
1163         gint32 poc_cycle_cnt, frame_num_in_poc_cycle;
1164
1165         expected_delta_per_poc_cycle = 0;
1166         for (i = 0; i < sps->num_ref_frames_in_pic_order_cnt_cycle; i++)
1167             expected_delta_per_poc_cycle += sps->offset_for_ref_frame[i];
1168
1169         // (8-8)
1170         poc_cycle_cnt = (abs_frame_num - 1) /
1171             sps->num_ref_frames_in_pic_order_cnt_cycle;
1172         frame_num_in_poc_cycle = (abs_frame_num - 1) %
1173             sps->num_ref_frames_in_pic_order_cnt_cycle;
1174
1175         // (8-9)
1176         expected_poc = poc_cycle_cnt * expected_delta_per_poc_cycle;
1177         for (i = 0; i <= frame_num_in_poc_cycle; i++)
1178             expected_poc += sps->offset_for_ref_frame[i];
1179     }
1180     else
1181         expected_poc = 0;
1182     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
1183         expected_poc += sps->offset_for_non_ref_pic;
1184
1185     // (8-10)
1186     switch (picture->structure) {
1187     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
1188         priv->field_poc[TOP_FIELD] = expected_poc +
1189             slice_hdr->delta_pic_order_cnt[0];
1190         priv->field_poc[BOTTOM_FIELD] = priv->field_poc[TOP_FIELD] +
1191             sps->offset_for_top_to_bottom_field +
1192             slice_hdr->delta_pic_order_cnt[1];
1193         break;
1194     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
1195         priv->field_poc[TOP_FIELD] = expected_poc +
1196             slice_hdr->delta_pic_order_cnt[0];
1197         break;
1198     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
1199         priv->field_poc[BOTTOM_FIELD] = expected_poc +
1200             sps->offset_for_top_to_bottom_field +
1201             slice_hdr->delta_pic_order_cnt[0];
1202         break;
1203     }
1204 }
1205
1206 /* 8.2.1.3 - Decoding process for picture order count type 2 */
1207 static void
1208 init_picture_poc_2(
1209     GstVaapiDecoderH264 *decoder,
1210     GstVaapiPictureH264 *picture,
1211     GstH264SliceHdr     *slice_hdr
1212 )
1213 {
1214     GstVaapiDecoderH264Private * const priv = decoder->priv;
1215     GstH264PPS * const pps = slice_hdr->pps;
1216     GstH264SPS * const sps = pps->sequence;
1217     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1218     gint32 prev_frame_num_offset, temp_poc;
1219
1220     GST_DEBUG("decode picture order count type 2");
1221
1222     if (priv->prev_pic_has_mmco5)
1223         prev_frame_num_offset = 0;
1224     else
1225         prev_frame_num_offset = priv->frame_num_offset;
1226
1227     // (8-11)
1228     if (GST_VAAPI_PICTURE_IS_IDR(picture))
1229         priv->frame_num_offset = 0;
1230     else if (priv->prev_frame_num > priv->frame_num)
1231         priv->frame_num_offset = prev_frame_num_offset + MaxFrameNum;
1232     else
1233         priv->frame_num_offset = prev_frame_num_offset;
1234
1235     // (8-12)
1236     if (GST_VAAPI_PICTURE_IS_IDR(picture))
1237         temp_poc = 0;
1238     else if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
1239         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num) - 1;
1240     else
1241         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num);
1242
1243     // (8-13)
1244     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
1245         priv->field_poc[TOP_FIELD] = temp_poc;
1246     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
1247         priv->field_poc[BOTTOM_FIELD] = temp_poc;
1248 }
1249
1250 /* 8.2.1 - Decoding process for picture order count */
1251 static void
1252 init_picture_poc(
1253     GstVaapiDecoderH264 *decoder,
1254     GstVaapiPictureH264 *picture,
1255     GstH264SliceHdr     *slice_hdr
1256 )
1257 {
1258     GstVaapiDecoderH264Private * const priv = decoder->priv;
1259     GstH264PPS * const pps = slice_hdr->pps;
1260     GstH264SPS * const sps = pps->sequence;
1261
1262     switch (sps->pic_order_cnt_type) {
1263     case 0:
1264         init_picture_poc_0(decoder, picture, slice_hdr);
1265         break;
1266     case 1:
1267         init_picture_poc_1(decoder, picture, slice_hdr);
1268         break;
1269     case 2:
1270         init_picture_poc_2(decoder, picture, slice_hdr);
1271         break;
1272     }
1273
1274     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
1275         picture->field_poc[TOP_FIELD] = priv->field_poc[TOP_FIELD];
1276     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
1277         picture->field_poc[BOTTOM_FIELD] = priv->field_poc[BOTTOM_FIELD];
1278     picture->base.poc = MIN(picture->field_poc[0], picture->field_poc[1]);
1279 }
1280
1281 static int
1282 compare_picture_pic_num_dec(const void *a, const void *b)
1283 {
1284     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1285     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1286
1287     return picB->pic_num - picA->pic_num;
1288 }
1289
1290 static int
1291 compare_picture_long_term_pic_num_inc(const void *a, const void *b)
1292 {
1293     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1294     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1295
1296     return picA->long_term_pic_num - picB->long_term_pic_num;
1297 }
1298
1299 static int
1300 compare_picture_poc_dec(const void *a, const void *b)
1301 {
1302     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1303     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1304
1305     return picB->base.poc - picA->base.poc;
1306 }
1307
1308 static int
1309 compare_picture_poc_inc(const void *a, const void *b)
1310 {
1311     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1312     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1313
1314     return picA->base.poc - picB->base.poc;
1315 }
1316
1317 static int
1318 compare_picture_frame_num_wrap_dec(const void *a, const void *b)
1319 {
1320     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1321     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1322
1323     return picB->frame_num_wrap - picA->frame_num_wrap;
1324 }
1325
1326 static int
1327 compare_picture_long_term_frame_idx_inc(const void *a, const void *b)
1328 {
1329     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1330     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1331
1332     return picA->long_term_frame_idx - picB->long_term_frame_idx;
1333 }
1334
1335 /* 8.2.4.1 - Decoding process for picture numbers */
1336 static void
1337 init_picture_refs_pic_num(
1338     GstVaapiDecoderH264 *decoder,
1339     GstVaapiPictureH264 *picture,
1340     GstH264SliceHdr     *slice_hdr
1341 )
1342 {
1343     GstVaapiDecoderH264Private * const priv = decoder->priv;
1344     GstH264PPS * const pps = slice_hdr->pps;
1345     GstH264SPS * const sps = pps->sequence;
1346     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1347     guint i;
1348
1349     GST_DEBUG("decode picture numbers");
1350
1351     for (i = 0; i < priv->short_ref_count; i++) {
1352         GstVaapiPictureH264 * const pic = priv->short_ref[i];
1353
1354         // (8-27)
1355         if (pic->frame_num > priv->frame_num)
1356             pic->frame_num_wrap = pic->frame_num - MaxFrameNum;
1357         else
1358             pic->frame_num_wrap = pic->frame_num;
1359
1360         // (8-28, 8-30, 8-31)
1361         if (GST_VAAPI_PICTURE_IS_FRAME(picture))
1362             pic->pic_num = pic->frame_num_wrap;
1363         else {
1364             if (pic->structure == picture->structure)
1365                 pic->pic_num = 2 * pic->frame_num_wrap + 1;
1366             else
1367                 pic->pic_num = 2 * pic->frame_num_wrap;
1368         }
1369     }
1370
1371     for (i = 0; i < priv->long_ref_count; i++) {
1372         GstVaapiPictureH264 * const pic = priv->long_ref[i];
1373
1374         // (8-29, 8-32, 8-33)
1375         if (GST_VAAPI_PICTURE_IS_FRAME(picture))
1376             pic->long_term_pic_num = pic->long_term_frame_idx;
1377         else {
1378             if (pic->structure == picture->structure)
1379                 pic->long_term_pic_num = 2 * pic->long_term_frame_idx + 1;
1380             else
1381                 pic->long_term_pic_num = 2 * pic->long_term_frame_idx;
1382         }
1383     }
1384 }
1385
1386 #define SORT_REF_LIST(list, n, compare_func) \
1387     qsort(list, n, sizeof(*(list)), compare_picture_##compare_func)
1388
1389 static void
1390 init_picture_refs_fields_1(
1391     guint                picture_structure,
1392     GstVaapiPictureH264 *RefPicList[32],
1393     guint               *RefPicList_count,
1394     GstVaapiPictureH264 *ref_list[32],
1395     guint                ref_list_count
1396 )
1397 {
1398     guint i, j, n;
1399
1400     i = 0;
1401     j = 0;
1402     n = *RefPicList_count;
1403     do {
1404         g_assert(n < 32);
1405         for (; i < ref_list_count; i++) {
1406             if (ref_list[i]->structure == picture_structure) {
1407                 RefPicList[n++] = ref_list[i++];
1408                 break;
1409             }
1410         }
1411         for (; j < ref_list_count; j++) {
1412             if (ref_list[j]->structure != picture_structure) {
1413                 RefPicList[n++] = ref_list[j++];
1414                 break;
1415             }
1416         }
1417     } while (i < ref_list_count || j < ref_list_count);
1418     *RefPicList_count = n;
1419 }
1420
1421 static inline void
1422 init_picture_refs_fields(
1423     GstVaapiPictureH264 *picture,
1424     GstVaapiPictureH264 *RefPicList[32],
1425     guint               *RefPicList_count,
1426     GstVaapiPictureH264 *short_ref[32],
1427     guint                short_ref_count,
1428     GstVaapiPictureH264 *long_ref[32],
1429     guint                long_ref_count
1430 )
1431 {
1432     guint n = 0;
1433
1434     /* 8.2.4.2.5 - reference picture lists in fields */
1435     init_picture_refs_fields_1(picture->structure, RefPicList, &n,
1436         short_ref, short_ref_count);
1437     init_picture_refs_fields_1(picture->structure, RefPicList, &n,
1438         long_ref, long_ref_count);
1439     *RefPicList_count = n;
1440 }
1441
1442 static void
1443 init_picture_refs_p_slice(
1444     GstVaapiDecoderH264 *decoder,
1445     GstVaapiPictureH264 *picture,
1446     GstH264SliceHdr     *slice_hdr
1447 )
1448 {
1449     GstVaapiDecoderH264Private * const priv = decoder->priv;
1450     GstVaapiPictureH264 **ref_list;
1451     guint i;
1452
1453     GST_DEBUG("decode reference picture list for P and SP slices");
1454
1455     if (GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1456         /* 8.2.4.2.1 - P and SP slices in frames */
1457         if (priv->short_ref_count > 0) {
1458             ref_list = priv->RefPicList0;
1459             for (i = 0; i < priv->short_ref_count; i++)
1460                 ref_list[i] = priv->short_ref[i];
1461             SORT_REF_LIST(ref_list, i, pic_num_dec);
1462             priv->RefPicList0_count += i;
1463         }
1464
1465         if (priv->long_ref_count > 0) {
1466             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1467             for (i = 0; i < priv->long_ref_count; i++)
1468                 ref_list[i] = priv->long_ref[i];
1469             SORT_REF_LIST(ref_list, i, long_term_pic_num_inc);
1470             priv->RefPicList0_count += i;
1471         }
1472     }
1473     else {
1474         /* 8.2.4.2.2 - P and SP slices in fields */
1475         GstVaapiPictureH264 *short_ref[32];
1476         guint short_ref_count = 0;
1477         GstVaapiPictureH264 *long_ref[32];
1478         guint long_ref_count = 0;
1479
1480         if (priv->short_ref_count > 0) {
1481             for (i = 0; i < priv->short_ref_count; i++)
1482                 short_ref[i] = priv->short_ref[i];
1483             SORT_REF_LIST(short_ref, i, frame_num_wrap_dec);
1484             short_ref_count = i;
1485         }
1486
1487         if (priv->long_ref_count > 0) {
1488             for (i = 0; i < priv->long_ref_count; i++)
1489                 long_ref[i] = priv->long_ref[i];
1490             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1491             long_ref_count = i;
1492         }
1493
1494         init_picture_refs_fields(
1495             picture,
1496             priv->RefPicList0, &priv->RefPicList0_count,
1497             short_ref,          short_ref_count,
1498             long_ref,           long_ref_count
1499         );
1500     }
1501 }
1502
1503 static void
1504 init_picture_refs_b_slice(
1505     GstVaapiDecoderH264 *decoder,
1506     GstVaapiPictureH264 *picture,
1507     GstH264SliceHdr     *slice_hdr
1508 )
1509 {
1510     GstVaapiDecoderH264Private * const priv = decoder->priv;
1511     GstVaapiPictureH264 **ref_list;
1512     guint i, n;
1513
1514     GST_DEBUG("decode reference picture list for B slices");
1515
1516     if (GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1517         /* 8.2.4.2.3 - B slices in frames */
1518
1519         /* RefPicList0 */
1520         if (priv->short_ref_count > 0) {
1521             // 1. Short-term references
1522             ref_list = priv->RefPicList0;
1523             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1524                 if (priv->short_ref[i]->base.poc < picture->base.poc)
1525                     ref_list[n++] = priv->short_ref[i];
1526             }
1527             SORT_REF_LIST(ref_list, n, poc_dec);
1528             priv->RefPicList0_count += n;
1529
1530             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1531             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1532                 if (priv->short_ref[i]->base.poc >= picture->base.poc)
1533                     ref_list[n++] = priv->short_ref[i];
1534             }
1535             SORT_REF_LIST(ref_list, n, poc_inc);
1536             priv->RefPicList0_count += n;
1537         }
1538
1539         if (priv->long_ref_count > 0) {
1540             // 2. Long-term references
1541             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1542             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1543                 ref_list[n++] = priv->long_ref[i];
1544             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1545             priv->RefPicList0_count += n;
1546         }
1547
1548         /* RefPicList1 */
1549         if (priv->short_ref_count > 0) {
1550             // 1. Short-term references
1551             ref_list = priv->RefPicList1;
1552             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1553                 if (priv->short_ref[i]->base.poc > picture->base.poc)
1554                     ref_list[n++] = priv->short_ref[i];
1555             }
1556             SORT_REF_LIST(ref_list, n, poc_inc);
1557             priv->RefPicList1_count += n;
1558
1559             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1560             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1561                 if (priv->short_ref[i]->base.poc <= picture->base.poc)
1562                     ref_list[n++] = priv->short_ref[i];
1563             }
1564             SORT_REF_LIST(ref_list, n, poc_dec);
1565             priv->RefPicList1_count += n;
1566         }
1567
1568         if (priv->long_ref_count > 0) {
1569             // 2. Long-term references
1570             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1571             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1572                 ref_list[n++] = priv->long_ref[i];
1573             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1574             priv->RefPicList1_count += n;
1575         }
1576     }
1577     else {
1578         /* 8.2.4.2.4 - B slices in fields */
1579         GstVaapiPictureH264 *short_ref0[32];
1580         guint short_ref0_count = 0;
1581         GstVaapiPictureH264 *short_ref1[32];
1582         guint short_ref1_count = 0;
1583         GstVaapiPictureH264 *long_ref[32];
1584         guint long_ref_count = 0;
1585
1586         /* refFrameList0ShortTerm */
1587         if (priv->short_ref_count > 0) {
1588             ref_list = short_ref0;
1589             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1590                 if (priv->short_ref[i]->base.poc <= picture->base.poc)
1591                     ref_list[n++] = priv->short_ref[i];
1592             }
1593             SORT_REF_LIST(ref_list, n, poc_dec);
1594             short_ref0_count += n;
1595
1596             ref_list = &short_ref0[short_ref0_count];
1597             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1598                 if (priv->short_ref[i]->base.poc > picture->base.poc)
1599                     ref_list[n++] = priv->short_ref[i];
1600             }
1601             SORT_REF_LIST(ref_list, n, poc_inc);
1602             short_ref0_count += n;
1603         }
1604
1605         /* refFrameList1ShortTerm */
1606         if (priv->short_ref_count > 0) {
1607             ref_list = short_ref1;
1608             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1609                 if (priv->short_ref[i]->base.poc > picture->base.poc)
1610                     ref_list[n++] = priv->short_ref[i];
1611             }
1612             SORT_REF_LIST(ref_list, n, poc_inc);
1613             short_ref1_count += n;
1614
1615             ref_list = &short_ref1[short_ref1_count];
1616             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1617                 if (priv->short_ref[i]->base.poc <= picture->base.poc)
1618                     ref_list[n++] = priv->short_ref[i];
1619             }
1620             SORT_REF_LIST(ref_list, n, poc_dec);
1621             short_ref1_count += n;
1622         }
1623
1624         /* refFrameListLongTerm */
1625         if (priv->long_ref_count > 0) {
1626             for (i = 0; i < priv->long_ref_count; i++)
1627                 long_ref[i] = priv->long_ref[i];
1628             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1629             long_ref_count = i;
1630         }
1631
1632         init_picture_refs_fields(
1633             picture,
1634             priv->RefPicList0, &priv->RefPicList0_count,
1635             short_ref0,         short_ref0_count,
1636             long_ref,           long_ref_count
1637         );
1638
1639         init_picture_refs_fields(
1640             picture,
1641             priv->RefPicList1, &priv->RefPicList1_count,
1642             short_ref1,         short_ref1_count,
1643             long_ref,           long_ref_count
1644         );
1645    }
1646
1647     /* Check whether RefPicList1 is identical to RefPicList0, then
1648        swap if necessary */
1649     if (priv->RefPicList1_count > 1 &&
1650         priv->RefPicList1_count == priv->RefPicList0_count &&
1651         memcmp(priv->RefPicList0, priv->RefPicList1,
1652                priv->RefPicList0_count * sizeof(priv->RefPicList0[0])) == 0) {
1653         GstVaapiPictureH264 * const tmp = priv->RefPicList1[0];
1654         priv->RefPicList1[0] = priv->RefPicList1[1];
1655         priv->RefPicList1[1] = tmp;
1656     }
1657 }
1658
1659 #undef SORT_REF_LIST
1660
1661 static gint
1662 find_short_term_reference(GstVaapiDecoderH264 *decoder, gint32 pic_num)
1663 {
1664     GstVaapiDecoderH264Private * const priv = decoder->priv;
1665     guint i;
1666
1667     for (i = 0; i < priv->short_ref_count; i++) {
1668         if (priv->short_ref[i]->pic_num == pic_num)
1669             return i;
1670     }
1671     GST_ERROR("found no short-term reference picture with PicNum = %d",
1672               pic_num);
1673     return -1;
1674 }
1675
1676 static gint
1677 find_long_term_reference(GstVaapiDecoderH264 *decoder, gint32 long_term_pic_num)
1678 {
1679     GstVaapiDecoderH264Private * const priv = decoder->priv;
1680     guint i;
1681
1682     for (i = 0; i < priv->long_ref_count; i++) {
1683         if (priv->long_ref[i]->long_term_pic_num == long_term_pic_num)
1684             return i;
1685     }
1686     GST_ERROR("found no long-term reference picture with LongTermPicNum = %d",
1687               long_term_pic_num);
1688     return -1;
1689 }
1690
1691 static void
1692 exec_picture_refs_modification_1(
1693     GstVaapiDecoderH264           *decoder,
1694     GstVaapiPictureH264           *picture,
1695     GstH264SliceHdr               *slice_hdr,
1696     guint                          list
1697 )
1698 {
1699     GstVaapiDecoderH264Private * const priv = decoder->priv;
1700     GstH264PPS * const pps = slice_hdr->pps;
1701     GstH264SPS * const sps = pps->sequence;
1702     GstH264RefPicListModification *ref_pic_list_modification;
1703     guint num_ref_pic_list_modifications;
1704     GstVaapiPictureH264 **ref_list;
1705     guint *ref_list_count_ptr, ref_list_count, ref_list_idx = 0;
1706     guint i, j, n, num_refs;
1707     gint found_ref_idx;
1708     gint32 MaxPicNum, CurrPicNum, picNumPred;
1709
1710     GST_DEBUG("modification process of reference picture list %u", list);
1711
1712     if (list == 0) {
1713         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l0;
1714         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l0;
1715         ref_list                       = priv->RefPicList0;
1716         ref_list_count_ptr             = &priv->RefPicList0_count;
1717         num_refs                       = slice_hdr->num_ref_idx_l0_active_minus1 + 1;
1718     }
1719     else {
1720         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l1;
1721         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l1;
1722         ref_list                       = priv->RefPicList1;
1723         ref_list_count_ptr             = &priv->RefPicList1_count;
1724         num_refs                       = slice_hdr->num_ref_idx_l1_active_minus1 + 1;
1725     }
1726     ref_list_count = *ref_list_count_ptr;
1727
1728     if (!GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1729         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 5); // 2 * MaxFrameNum
1730         CurrPicNum = 2 * slice_hdr->frame_num + 1;              // 2 * frame_num + 1
1731     }
1732     else {
1733         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 4); // MaxFrameNum
1734         CurrPicNum = slice_hdr->frame_num;                      // frame_num
1735     }
1736
1737     picNumPred = CurrPicNum;
1738
1739     for (i = 0; i < num_ref_pic_list_modifications; i++) {
1740         GstH264RefPicListModification * const l = &ref_pic_list_modification[i];
1741         if (l->modification_of_pic_nums_idc == 3)
1742             break;
1743
1744         /* 8.2.4.3.1 - Short-term reference pictures */
1745         if (l->modification_of_pic_nums_idc == 0 || l->modification_of_pic_nums_idc == 1) {
1746             gint32 abs_diff_pic_num = l->value.abs_diff_pic_num_minus1 + 1;
1747             gint32 picNum, picNumNoWrap;
1748
1749             // (8-34)
1750             if (l->modification_of_pic_nums_idc == 0) {
1751                 picNumNoWrap = picNumPred - abs_diff_pic_num;
1752                 if (picNumNoWrap < 0)
1753                     picNumNoWrap += MaxPicNum;
1754             }
1755
1756             // (8-35)
1757             else {
1758                 picNumNoWrap = picNumPred + abs_diff_pic_num;
1759                 if (picNumNoWrap >= MaxPicNum)
1760                     picNumNoWrap -= MaxPicNum;
1761             }
1762             picNumPred = picNumNoWrap;
1763
1764             // (8-36)
1765             picNum = picNumNoWrap;
1766             if (picNum > CurrPicNum)
1767                 picNum -= MaxPicNum;
1768
1769             // (8-37)
1770             for (j = num_refs; j > ref_list_idx; j--)
1771                 ref_list[j] = ref_list[j - 1];
1772             found_ref_idx = find_short_term_reference(decoder, picNum);
1773             ref_list[ref_list_idx++] =
1774                 found_ref_idx >= 0 ? priv->short_ref[found_ref_idx] : NULL;
1775             n = ref_list_idx;
1776             for (j = ref_list_idx; j <= num_refs; j++) {
1777                 gint32 PicNumF;
1778                 if (!ref_list[j])
1779                     continue;
1780                 PicNumF =
1781                     GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(ref_list[j]) ?
1782                     ref_list[j]->pic_num : MaxPicNum;
1783                 if (PicNumF != picNum)
1784                     ref_list[n++] = ref_list[j];
1785             }
1786         }
1787
1788         /* 8.2.4.3.2 - Long-term reference pictures */
1789         else {
1790
1791             for (j = num_refs; j > ref_list_idx; j--)
1792                 ref_list[j] = ref_list[j - 1];
1793             found_ref_idx =
1794                 find_long_term_reference(decoder, l->value.long_term_pic_num);
1795             ref_list[ref_list_idx++] =
1796                 found_ref_idx >= 0 ? priv->long_ref[found_ref_idx] : NULL;
1797             n = ref_list_idx;
1798             for (j = ref_list_idx; j <= num_refs; j++) {
1799                 gint32 LongTermPicNumF;
1800                 if (!ref_list[j])
1801                     continue;
1802                 LongTermPicNumF =
1803                     GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(ref_list[j]) ?
1804                     ref_list[j]->long_term_pic_num : INT_MAX;
1805                 if (LongTermPicNumF != l->value.long_term_pic_num)
1806                     ref_list[n++] = ref_list[j];
1807             }
1808         }
1809     }
1810
1811 #if DEBUG
1812     for (i = 0; i < num_refs; i++)
1813         if (!ref_list[i])
1814             GST_ERROR("list %u entry %u is empty", list, i);
1815 #endif
1816     *ref_list_count_ptr = num_refs;
1817 }
1818
1819 /* 8.2.4.3 - Modification process for reference picture lists */
1820 static void
1821 exec_picture_refs_modification(
1822     GstVaapiDecoderH264 *decoder,
1823     GstVaapiPictureH264 *picture,
1824     GstH264SliceHdr     *slice_hdr
1825 )
1826 {
1827     GST_DEBUG("execute ref_pic_list_modification()");
1828
1829     /* RefPicList0 */
1830     if (!GST_H264_IS_I_SLICE(slice_hdr) && !GST_H264_IS_SI_SLICE(slice_hdr) &&
1831         slice_hdr->ref_pic_list_modification_flag_l0)
1832         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 0);
1833
1834     /* RefPicList1 */
1835     if (GST_H264_IS_B_SLICE(slice_hdr) &&
1836         slice_hdr->ref_pic_list_modification_flag_l1)
1837         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 1);
1838 }
1839
1840 static void
1841 init_picture_ref_lists(GstVaapiDecoderH264 *decoder)
1842 {
1843     GstVaapiDecoderH264Private * const priv = decoder->priv;
1844     guint i, j, short_ref_count, long_ref_count;
1845
1846     short_ref_count = 0;
1847     long_ref_count  = 0;
1848     if (GST_VAAPI_PICTURE_IS_FRAME(priv->current_picture)) {
1849         for (i = 0; i < priv->dpb_count; i++) {
1850             GstVaapiFrameStore * const fs = priv->dpb[i];
1851             GstVaapiPictureH264 *picture;
1852             if (!gst_vaapi_frame_store_has_frame(fs))
1853                 continue;
1854             picture = fs->buffers[0];
1855             if (GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture))
1856                 priv->short_ref[short_ref_count++] = picture;
1857             else if (GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture))
1858                 priv->long_ref[long_ref_count++] = picture;
1859             picture->structure = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
1860             picture->other_field = fs->buffers[1];
1861         }
1862     }
1863     else {
1864         for (i = 0; i < priv->dpb_count; i++) {
1865             GstVaapiFrameStore * const fs = priv->dpb[i];
1866             for (j = 0; j < fs->num_buffers; j++) {
1867                 GstVaapiPictureH264 * const picture = fs->buffers[j];
1868                 if (GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture))
1869                     priv->short_ref[short_ref_count++] = picture;
1870                 else if (GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture))
1871                     priv->long_ref[long_ref_count++] = picture;
1872                 picture->structure = picture->base.structure;
1873                 picture->other_field = fs->buffers[j ^ 1];
1874             }
1875         }
1876     }
1877
1878     for (i = short_ref_count; i < priv->short_ref_count; i++)
1879         priv->short_ref[i] = NULL;
1880     priv->short_ref_count = short_ref_count;
1881
1882     for (i = long_ref_count; i < priv->long_ref_count; i++)
1883         priv->long_ref[i] = NULL;
1884     priv->long_ref_count = long_ref_count;
1885 }
1886
1887 static void
1888 init_picture_refs(
1889     GstVaapiDecoderH264 *decoder,
1890     GstVaapiPictureH264 *picture,
1891     GstH264SliceHdr     *slice_hdr
1892 )
1893 {
1894     GstVaapiDecoderH264Private * const priv = decoder->priv;
1895     GstVaapiPicture * const base_picture = &picture->base;
1896     guint i, num_refs;
1897
1898     init_picture_ref_lists(decoder);
1899     init_picture_refs_pic_num(decoder, picture, slice_hdr);
1900
1901     priv->RefPicList0_count = 0;
1902     priv->RefPicList1_count = 0;
1903
1904     switch (base_picture->type) {
1905     case GST_VAAPI_PICTURE_TYPE_P:
1906     case GST_VAAPI_PICTURE_TYPE_SP:
1907         init_picture_refs_p_slice(decoder, picture, slice_hdr);
1908         break;
1909     case GST_VAAPI_PICTURE_TYPE_B:
1910         init_picture_refs_b_slice(decoder, picture, slice_hdr);
1911         break;
1912     default:
1913         break;
1914     }
1915
1916     exec_picture_refs_modification(decoder, picture, slice_hdr);
1917
1918     switch (base_picture->type) {
1919     case GST_VAAPI_PICTURE_TYPE_B:
1920         num_refs = 1 + slice_hdr->num_ref_idx_l1_active_minus1;
1921         for (i = priv->RefPicList1_count; i < num_refs; i++)
1922             priv->RefPicList1[i] = NULL;
1923         priv->RefPicList1_count = num_refs;
1924
1925         // fall-through
1926     case GST_VAAPI_PICTURE_TYPE_P:
1927     case GST_VAAPI_PICTURE_TYPE_SP:
1928         num_refs = 1 + slice_hdr->num_ref_idx_l0_active_minus1;
1929         for (i = priv->RefPicList0_count; i < num_refs; i++)
1930             priv->RefPicList0[i] = NULL;
1931         priv->RefPicList0_count = num_refs;
1932         break;
1933     default:
1934         break;
1935     }
1936 }
1937
1938 static gboolean
1939 init_picture(
1940     GstVaapiDecoderH264 *decoder,
1941     GstVaapiPictureH264 *picture, GstVaapiDecoderUnitH264 *unit)
1942 {
1943     GstVaapiDecoderH264Private * const priv = decoder->priv;
1944     GstVaapiPicture * const base_picture = &picture->base;
1945     GstH264SliceHdr * const slice_hdr = &unit->data.slice_hdr;
1946
1947     priv->prev_frame_num        = priv->frame_num;
1948     priv->frame_num             = slice_hdr->frame_num;
1949     picture->frame_num          = priv->frame_num;
1950     picture->frame_num_wrap     = priv->frame_num;
1951     picture->output_flag        = TRUE; /* XXX: conformant to Annex A only */
1952     base_picture->pts           = GST_VAAPI_DECODER_CODEC_FRAME(decoder)->pts;
1953
1954     /* Reset decoder state for IDR pictures */
1955     if (unit->nalu.type == GST_H264_NAL_SLICE_IDR) {
1956         GST_DEBUG("<IDR>");
1957         GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_IDR);
1958         dpb_flush(decoder);
1959     }
1960
1961     /* Initialize slice type */
1962     switch (slice_hdr->type % 5) {
1963     case GST_H264_P_SLICE:
1964         base_picture->type = GST_VAAPI_PICTURE_TYPE_P;
1965         break;
1966     case GST_H264_B_SLICE:
1967         base_picture->type = GST_VAAPI_PICTURE_TYPE_B;
1968         break;
1969     case GST_H264_I_SLICE:
1970         base_picture->type = GST_VAAPI_PICTURE_TYPE_I;
1971         break;
1972     case GST_H264_SP_SLICE:
1973         base_picture->type = GST_VAAPI_PICTURE_TYPE_SP;
1974         break;
1975     case GST_H264_SI_SLICE:
1976         base_picture->type = GST_VAAPI_PICTURE_TYPE_SI;
1977         break;
1978     }
1979
1980     /* Initialize picture structure */
1981     if (!slice_hdr->field_pic_flag)
1982         base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
1983     else {
1984         GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_INTERLACED);
1985         if (!slice_hdr->bottom_field_flag)
1986             base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD;
1987         else
1988             base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD;
1989     }
1990     picture->structure = base_picture->structure;
1991
1992     /* Initialize reference flags */
1993     if (unit->nalu.ref_idc) {
1994         GstH264DecRefPicMarking * const dec_ref_pic_marking =
1995             &slice_hdr->dec_ref_pic_marking;
1996
1997         if (GST_VAAPI_PICTURE_IS_IDR(picture) &&
1998             dec_ref_pic_marking->long_term_reference_flag)
1999             GST_VAAPI_PICTURE_FLAG_SET(picture,
2000                 GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE);
2001         else
2002             GST_VAAPI_PICTURE_FLAG_SET(picture,
2003                 GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE);
2004     }
2005
2006     init_picture_poc(decoder, picture, slice_hdr);
2007     init_picture_refs(decoder, picture, slice_hdr);
2008     return TRUE;
2009 }
2010
2011 /* 8.2.5.3 - Sliding window decoded reference picture marking process */
2012 static gboolean
2013 exec_ref_pic_marking_sliding_window(GstVaapiDecoderH264 *decoder)
2014 {
2015     GstVaapiDecoderH264Private * const priv = decoder->priv;
2016     GstH264PPS * const pps = priv->current_picture->pps;
2017     GstH264SPS * const sps = pps->sequence;
2018     GstVaapiPictureH264 *ref_picture;
2019     guint i, m, max_num_ref_frames;
2020
2021     GST_DEBUG("reference picture marking process (sliding window)");
2022
2023     if (!GST_VAAPI_PICTURE_IS_FIRST_FIELD(priv->current_picture))
2024         return TRUE;
2025
2026     max_num_ref_frames = sps->num_ref_frames;
2027     if (max_num_ref_frames == 0)
2028         max_num_ref_frames = 1;
2029     if (!GST_VAAPI_PICTURE_IS_FRAME(priv->current_picture))
2030         max_num_ref_frames <<= 1;
2031
2032     if (priv->short_ref_count + priv->long_ref_count < max_num_ref_frames)
2033         return TRUE;
2034     if (priv->short_ref_count < 1)
2035         return FALSE;
2036
2037     for (m = 0, i = 1; i < priv->short_ref_count; i++) {
2038         GstVaapiPictureH264 * const picture = priv->short_ref[i];
2039         if (picture->frame_num_wrap < priv->short_ref[m]->frame_num_wrap)
2040             m = i;
2041     }
2042
2043     ref_picture = priv->short_ref[m];
2044     gst_vaapi_picture_h264_set_reference(ref_picture, 0, TRUE);
2045     ARRAY_REMOVE_INDEX(priv->short_ref, m);
2046
2047     /* Both fields need to be marked as "unused for reference", so
2048        remove the other field from the short_ref[] list as well */
2049     if (!GST_VAAPI_PICTURE_IS_FRAME(priv->current_picture) && ref_picture->other_field) {
2050         for (i = 0; i < priv->short_ref_count; i++) {
2051             if (priv->short_ref[i] == ref_picture->other_field) {
2052                 ARRAY_REMOVE_INDEX(priv->short_ref, i);
2053                 break;
2054             }
2055         }
2056     }
2057     return TRUE;
2058 }
2059
2060 static inline gint32
2061 get_picNumX(GstVaapiPictureH264 *picture, GstH264RefPicMarking *ref_pic_marking)
2062 {
2063     gint32 pic_num;
2064
2065     if (GST_VAAPI_PICTURE_IS_FRAME(picture))
2066         pic_num = picture->frame_num_wrap;
2067     else
2068         pic_num = 2 * picture->frame_num_wrap + 1;
2069     pic_num -= ref_pic_marking->difference_of_pic_nums_minus1 + 1;
2070     return pic_num;
2071 }
2072
2073 /* 8.2.5.4.1. Mark short-term reference picture as "unused for reference" */
2074 static void
2075 exec_ref_pic_marking_adaptive_mmco_1(
2076     GstVaapiDecoderH264  *decoder,
2077     GstVaapiPictureH264  *picture,
2078     GstH264RefPicMarking *ref_pic_marking
2079 )
2080 {
2081     GstVaapiDecoderH264Private * const priv = decoder->priv;
2082     gint32 i, picNumX;
2083
2084     picNumX = get_picNumX(picture, ref_pic_marking);
2085     i = find_short_term_reference(decoder, picNumX);
2086     if (i < 0)
2087         return;
2088
2089     gst_vaapi_picture_h264_set_reference(priv->short_ref[i], 0,
2090         GST_VAAPI_PICTURE_IS_FRAME(picture));
2091     ARRAY_REMOVE_INDEX(priv->short_ref, i);
2092 }
2093
2094 /* 8.2.5.4.2. Mark long-term reference picture as "unused for reference" */
2095 static void
2096 exec_ref_pic_marking_adaptive_mmco_2(
2097     GstVaapiDecoderH264  *decoder,
2098     GstVaapiPictureH264  *picture,
2099     GstH264RefPicMarking *ref_pic_marking
2100 )
2101 {
2102     GstVaapiDecoderH264Private * const priv = decoder->priv;
2103     gint32 i;
2104
2105     i = find_long_term_reference(decoder, ref_pic_marking->long_term_pic_num);
2106     if (i < 0)
2107         return;
2108
2109     gst_vaapi_picture_h264_set_reference(priv->long_ref[i], 0,
2110         GST_VAAPI_PICTURE_IS_FRAME(picture));
2111     ARRAY_REMOVE_INDEX(priv->long_ref, i);
2112 }
2113
2114 /* 8.2.5.4.3. Assign LongTermFrameIdx to a short-term reference picture */
2115 static void
2116 exec_ref_pic_marking_adaptive_mmco_3(
2117     GstVaapiDecoderH264  *decoder,
2118     GstVaapiPictureH264  *picture,
2119     GstH264RefPicMarking *ref_pic_marking
2120 )
2121 {
2122     GstVaapiDecoderH264Private * const priv = decoder->priv;
2123     GstVaapiPictureH264 *ref_picture;
2124     gint32 i, picNumX;
2125
2126     for (i = 0; i < priv->long_ref_count; i++) {
2127         if (priv->long_ref[i]->long_term_frame_idx == ref_pic_marking->long_term_frame_idx)
2128             break;
2129     }
2130     if (i != priv->long_ref_count) {
2131         gst_vaapi_picture_h264_set_reference(priv->long_ref[i], 0, TRUE);
2132         ARRAY_REMOVE_INDEX(priv->long_ref, i);
2133     }
2134
2135     picNumX = get_picNumX(picture, ref_pic_marking);
2136     i = find_short_term_reference(decoder, picNumX);
2137     if (i < 0)
2138         return;
2139
2140     ref_picture = priv->short_ref[i];
2141     ARRAY_REMOVE_INDEX(priv->short_ref, i);
2142     priv->long_ref[priv->long_ref_count++] = ref_picture;
2143
2144     ref_picture->long_term_frame_idx = ref_pic_marking->long_term_frame_idx;
2145     gst_vaapi_picture_h264_set_reference(ref_picture,
2146         GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE,
2147         GST_VAAPI_PICTURE_IS_FRAME(picture));
2148 }
2149
2150 /* 8.2.5.4.4. Mark pictures with LongTermFramIdx > max_long_term_frame_idx
2151  * as "unused for reference" */
2152 static void
2153 exec_ref_pic_marking_adaptive_mmco_4(
2154     GstVaapiDecoderH264  *decoder,
2155     GstVaapiPictureH264  *picture,
2156     GstH264RefPicMarking *ref_pic_marking
2157 )
2158 {
2159     GstVaapiDecoderH264Private * const priv = decoder->priv;
2160     gint32 i, long_term_frame_idx;
2161
2162     long_term_frame_idx = ref_pic_marking->max_long_term_frame_idx_plus1 - 1;
2163
2164     for (i = 0; i < priv->long_ref_count; i++) {
2165         if (priv->long_ref[i]->long_term_frame_idx <= long_term_frame_idx)
2166             continue;
2167         gst_vaapi_picture_h264_set_reference(priv->long_ref[i], 0, FALSE);
2168         ARRAY_REMOVE_INDEX(priv->long_ref, i);
2169         i--;
2170     }
2171 }
2172
2173 /* 8.2.5.4.5. Mark all reference pictures as "unused for reference" */
2174 static void
2175 exec_ref_pic_marking_adaptive_mmco_5(
2176     GstVaapiDecoderH264  *decoder,
2177     GstVaapiPictureH264  *picture,
2178     GstH264RefPicMarking *ref_pic_marking
2179 )
2180 {
2181     GstVaapiDecoderH264Private * const priv = decoder->priv;
2182
2183     dpb_flush(decoder);
2184
2185     priv->prev_pic_has_mmco5 = TRUE;
2186
2187     /* The picture shall be inferred to have had frame_num equal to 0 (7.4.3) */
2188     priv->frame_num = 0;
2189     priv->frame_num_offset = 0;
2190     picture->frame_num = 0;
2191
2192     /* Update TopFieldOrderCnt and BottomFieldOrderCnt (8.2.1) */
2193     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
2194         picture->field_poc[TOP_FIELD] -= picture->base.poc;
2195     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
2196         picture->field_poc[BOTTOM_FIELD] -= picture->base.poc;
2197     picture->base.poc = 0;
2198 }
2199
2200 /* 8.2.5.4.6. Assign a long-term frame index to the current picture */
2201 static void
2202 exec_ref_pic_marking_adaptive_mmco_6(
2203     GstVaapiDecoderH264  *decoder,
2204     GstVaapiPictureH264  *picture,
2205     GstH264RefPicMarking *ref_pic_marking
2206 )
2207 {
2208     picture->long_term_frame_idx = ref_pic_marking->long_term_frame_idx;
2209     gst_vaapi_picture_h264_set_reference(picture,
2210         GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE, FALSE);
2211 }
2212
2213 /* 8.2.5.4. Adaptive memory control decoded reference picture marking process */
2214 static gboolean
2215 exec_ref_pic_marking_adaptive(
2216     GstVaapiDecoderH264     *decoder,
2217     GstVaapiPictureH264     *picture,
2218     GstH264DecRefPicMarking *dec_ref_pic_marking
2219 )
2220 {
2221     guint i;
2222
2223     GST_DEBUG("reference picture marking process (adaptive memory control)");
2224
2225     typedef void (*exec_ref_pic_marking_adaptive_mmco_func)(
2226         GstVaapiDecoderH264  *decoder,
2227         GstVaapiPictureH264  *picture,
2228         GstH264RefPicMarking *ref_pic_marking
2229     );
2230
2231     static const exec_ref_pic_marking_adaptive_mmco_func mmco_funcs[] = {
2232         NULL,
2233         exec_ref_pic_marking_adaptive_mmco_1,
2234         exec_ref_pic_marking_adaptive_mmco_2,
2235         exec_ref_pic_marking_adaptive_mmco_3,
2236         exec_ref_pic_marking_adaptive_mmco_4,
2237         exec_ref_pic_marking_adaptive_mmco_5,
2238         exec_ref_pic_marking_adaptive_mmco_6,
2239     };
2240
2241     for (i = 0; i < dec_ref_pic_marking->n_ref_pic_marking; i++) {
2242         GstH264RefPicMarking * const ref_pic_marking =
2243             &dec_ref_pic_marking->ref_pic_marking[i];
2244
2245         const guint mmco = ref_pic_marking->memory_management_control_operation;
2246         if (mmco < G_N_ELEMENTS(mmco_funcs) && mmco_funcs[mmco])
2247             mmco_funcs[mmco](decoder, picture, ref_pic_marking);
2248         else {
2249             GST_ERROR("unhandled MMCO %u", mmco);
2250             return FALSE;
2251         }
2252     }
2253     return TRUE;
2254 }
2255
2256 /* 8.2.5 - Execute reference picture marking process */
2257 static gboolean
2258 exec_ref_pic_marking(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
2259 {
2260     GstVaapiDecoderH264Private * const priv = decoder->priv;
2261
2262     priv->prev_pic_has_mmco5 = FALSE;
2263     priv->prev_pic_structure = picture->structure;
2264
2265     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
2266         return TRUE;
2267
2268     if (!GST_VAAPI_PICTURE_IS_IDR(picture)) {
2269         GstH264DecRefPicMarking * const dec_ref_pic_marking =
2270             &picture->last_slice_hdr->dec_ref_pic_marking;
2271         if (dec_ref_pic_marking->adaptive_ref_pic_marking_mode_flag) {
2272             if (!exec_ref_pic_marking_adaptive(decoder, picture, dec_ref_pic_marking))
2273                 return FALSE;
2274         }
2275         else {
2276             if (!exec_ref_pic_marking_sliding_window(decoder))
2277                 return FALSE;
2278         }
2279     }
2280     return TRUE;
2281 }
2282
2283 static void
2284 vaapi_init_picture(VAPictureH264 *pic)
2285 {
2286     pic->picture_id           = VA_INVALID_ID;
2287     pic->frame_idx            = 0;
2288     pic->flags                = VA_PICTURE_H264_INVALID;
2289     pic->TopFieldOrderCnt     = 0;
2290     pic->BottomFieldOrderCnt  = 0;
2291 }
2292
2293 static void
2294 vaapi_fill_picture(VAPictureH264 *pic, GstVaapiPictureH264 *picture,
2295     guint picture_structure)
2296 {
2297     if (!picture_structure)
2298         picture_structure = picture->structure;
2299
2300     pic->picture_id = picture->base.surface_id;
2301     pic->flags = 0;
2302
2303     if (GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture)) {
2304         pic->flags |= VA_PICTURE_H264_LONG_TERM_REFERENCE;
2305         pic->frame_idx = picture->long_term_frame_idx;
2306     }
2307     else {
2308         if (GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture))
2309             pic->flags |= VA_PICTURE_H264_SHORT_TERM_REFERENCE;
2310         pic->frame_idx = picture->frame_num;
2311     }
2312
2313     switch (picture_structure) {
2314     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
2315         pic->TopFieldOrderCnt = picture->field_poc[TOP_FIELD];
2316         pic->BottomFieldOrderCnt = picture->field_poc[BOTTOM_FIELD];
2317         break;
2318     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
2319         pic->flags |= VA_PICTURE_H264_TOP_FIELD;
2320         pic->TopFieldOrderCnt = picture->field_poc[TOP_FIELD];
2321         pic->BottomFieldOrderCnt = 0;
2322         break;
2323     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
2324         pic->flags |= VA_PICTURE_H264_BOTTOM_FIELD;
2325         pic->BottomFieldOrderCnt = picture->field_poc[BOTTOM_FIELD];
2326         pic->TopFieldOrderCnt = 0;
2327         break;
2328     }
2329 }
2330
2331 static gboolean
2332 fill_picture(GstVaapiDecoderH264 *decoder,
2333     GstVaapiPictureH264 *picture, GstVaapiDecoderUnitH264 *unit)
2334 {
2335     GstVaapiDecoderH264Private * const priv = decoder->priv;
2336     GstVaapiPicture * const base_picture = &picture->base;
2337     GstH264SliceHdr * const slice_hdr = &unit->data.slice_hdr;
2338     GstH264PPS * const pps = picture->pps;
2339     GstH264SPS * const sps = pps->sequence;
2340     VAPictureParameterBufferH264 * const pic_param = base_picture->param;
2341     guint i, n;
2342
2343     /* Fill in VAPictureParameterBufferH264 */
2344     vaapi_fill_picture(&pic_param->CurrPic, picture, 0);
2345
2346     for (i = 0, n = 0; i < priv->dpb_count; i++) {
2347         GstVaapiFrameStore * const fs = priv->dpb[i];
2348         if (gst_vaapi_frame_store_has_reference(fs))
2349             vaapi_fill_picture(&pic_param->ReferenceFrames[n++],
2350                 fs->buffers[0], fs->structure);
2351     }
2352     for (; n < G_N_ELEMENTS(pic_param->ReferenceFrames); n++)
2353         vaapi_init_picture(&pic_param->ReferenceFrames[n]);
2354
2355 #define COPY_FIELD(s, f) \
2356     pic_param->f = (s)->f
2357
2358 #define COPY_BFM(a, s, f) \
2359     pic_param->a.bits.f = (s)->f
2360
2361     pic_param->picture_width_in_mbs_minus1  = priv->mb_width - 1;
2362     pic_param->picture_height_in_mbs_minus1 = priv->mb_height - 1;
2363     pic_param->frame_num                    = priv->frame_num;
2364
2365     COPY_FIELD(sps, bit_depth_luma_minus8);
2366     COPY_FIELD(sps, bit_depth_chroma_minus8);
2367     COPY_FIELD(sps, num_ref_frames);
2368     COPY_FIELD(pps, num_slice_groups_minus1);
2369     COPY_FIELD(pps, slice_group_map_type);
2370     COPY_FIELD(pps, slice_group_change_rate_minus1);
2371     COPY_FIELD(pps, pic_init_qp_minus26);
2372     COPY_FIELD(pps, pic_init_qs_minus26);
2373     COPY_FIELD(pps, chroma_qp_index_offset);
2374     COPY_FIELD(pps, second_chroma_qp_index_offset);
2375
2376     pic_param->seq_fields.value                                         = 0; /* reset all bits */
2377     pic_param->seq_fields.bits.residual_colour_transform_flag           = sps->separate_colour_plane_flag;
2378     pic_param->seq_fields.bits.MinLumaBiPredSize8x8                     = sps->level_idc >= 31; /* A.3.3.2 */
2379
2380     COPY_BFM(seq_fields, sps, chroma_format_idc);
2381     COPY_BFM(seq_fields, sps, gaps_in_frame_num_value_allowed_flag);
2382     COPY_BFM(seq_fields, sps, frame_mbs_only_flag);
2383     COPY_BFM(seq_fields, sps, mb_adaptive_frame_field_flag);
2384     COPY_BFM(seq_fields, sps, direct_8x8_inference_flag);
2385     COPY_BFM(seq_fields, sps, log2_max_frame_num_minus4);
2386     COPY_BFM(seq_fields, sps, pic_order_cnt_type);
2387     COPY_BFM(seq_fields, sps, log2_max_pic_order_cnt_lsb_minus4);
2388     COPY_BFM(seq_fields, sps, delta_pic_order_always_zero_flag);
2389
2390     pic_param->pic_fields.value                                         = 0; /* reset all bits */
2391     pic_param->pic_fields.bits.field_pic_flag                           = slice_hdr->field_pic_flag;
2392     pic_param->pic_fields.bits.reference_pic_flag                       = GST_VAAPI_PICTURE_IS_REFERENCE(picture);
2393
2394     COPY_BFM(pic_fields, pps, entropy_coding_mode_flag);
2395     COPY_BFM(pic_fields, pps, weighted_pred_flag);
2396     COPY_BFM(pic_fields, pps, weighted_bipred_idc);
2397     COPY_BFM(pic_fields, pps, transform_8x8_mode_flag);
2398     COPY_BFM(pic_fields, pps, constrained_intra_pred_flag);
2399     COPY_BFM(pic_fields, pps, pic_order_present_flag);
2400     COPY_BFM(pic_fields, pps, deblocking_filter_control_present_flag);
2401     COPY_BFM(pic_fields, pps, redundant_pic_cnt_present_flag);
2402     return TRUE;
2403 }
2404
2405 /* Detection of the first VCL NAL unit of a primary coded picture (7.4.1.2.4) */
2406 static gboolean
2407 is_new_picture(GstVaapiDecoderUnitH264 *unit,
2408     GstVaapiDecoderUnitH264 *prev_unit)
2409 {
2410     GstH264SliceHdr * const slice_hdr = &unit->data.slice_hdr;
2411     GstH264PPS * const pps = slice_hdr->pps;
2412     GstH264SPS * const sps = pps->sequence;
2413     GstH264SliceHdr *prev_slice_hdr;
2414
2415     if (!prev_unit)
2416         return TRUE;
2417     prev_slice_hdr = &prev_unit->data.slice_hdr;
2418
2419 #define CHECK_EXPR(expr, field_name) do {              \
2420         if (!(expr)) {                                 \
2421             GST_DEBUG(field_name " differs in value"); \
2422             return TRUE;                               \
2423         }                                              \
2424     } while (0)
2425
2426 #define CHECK_VALUE(new_slice_hdr, old_slice_hdr, field) \
2427     CHECK_EXPR(((new_slice_hdr)->field == (old_slice_hdr)->field), #field)
2428
2429     /* frame_num differs in value, regardless of inferred values to 0 */
2430     CHECK_VALUE(slice_hdr, prev_slice_hdr, frame_num);
2431
2432     /* pic_parameter_set_id differs in value */
2433     CHECK_VALUE(slice_hdr, prev_slice_hdr, pps);
2434
2435     /* field_pic_flag differs in value */
2436     CHECK_VALUE(slice_hdr, prev_slice_hdr, field_pic_flag);
2437
2438     /* bottom_field_flag is present in both and differs in value */
2439     if (slice_hdr->field_pic_flag && prev_slice_hdr->field_pic_flag)
2440         CHECK_VALUE(slice_hdr, prev_slice_hdr, bottom_field_flag);
2441
2442     /* nal_ref_idc differs in value with one of the nal_ref_idc values is 0 */
2443     CHECK_EXPR((unit->nalu.ref_idc != 0) ==
2444                (prev_unit->nalu.ref_idc != 0), "nal_ref_idc");
2445
2446     /* POC type is 0 for both and either pic_order_cnt_lsb differs in
2447        value or delta_pic_order_cnt_bottom differs in value */
2448     if (sps->pic_order_cnt_type == 0) {
2449         CHECK_VALUE(slice_hdr, prev_slice_hdr, pic_order_cnt_lsb);
2450         if (pps->pic_order_present_flag && !slice_hdr->field_pic_flag)
2451             CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt_bottom);
2452     }
2453
2454     /* POC type is 1 for both and either delta_pic_order_cnt[0]
2455        differs in value or delta_pic_order_cnt[1] differs in value */
2456     else if (sps->pic_order_cnt_type == 1) {
2457         CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt[0]);
2458         CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt[1]);
2459     }
2460
2461     /* IdrPicFlag differs in value */
2462     CHECK_VALUE(&unit->nalu, &prev_unit->nalu, idr_pic_flag);
2463
2464     /* IdrPicFlag is equal to 1 for both and idr_pic_id differs in value */
2465     if (unit->nalu.idr_pic_flag)
2466         CHECK_VALUE(slice_hdr, prev_slice_hdr, idr_pic_id);
2467
2468 #undef CHECK_EXPR
2469 #undef CHECK_VALUE
2470     return FALSE;
2471 }
2472
2473 static GstVaapiDecoderStatus
2474 decode_picture(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnitH264 *unit)
2475 {
2476     GstVaapiDecoderH264Private * const priv = decoder->priv;
2477     GstH264SliceHdr * const slice_hdr = &unit->data.slice_hdr;
2478     GstH264PPS * const pps = slice_hdr->pps;
2479     GstH264SPS * const sps = pps->sequence;
2480     GstVaapiPictureH264 *picture;
2481     GstVaapiDecoderStatus status;
2482
2483     status = ensure_context(decoder, sps);
2484     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2485         return status;
2486
2487     if (priv->current_picture) {
2488         /* Re-use current picture where the first field was decoded */
2489         picture = gst_vaapi_picture_h264_new_field(priv->current_picture);
2490         if (!picture) {
2491             GST_ERROR("failed to allocate field picture");
2492             return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2493         }
2494     }
2495     else {
2496         /* Create new picture */
2497         picture = gst_vaapi_picture_h264_new(decoder);
2498         if (!picture) {
2499             GST_ERROR("failed to allocate picture");
2500             return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2501         }
2502     }
2503     gst_vaapi_picture_replace(&priv->current_picture, picture);
2504     gst_vaapi_picture_unref(picture);
2505
2506     picture->pps = pps;
2507
2508     status = ensure_quant_matrix(decoder, picture);
2509     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS) {
2510         GST_ERROR("failed to reset quantizer matrix");
2511         return status;
2512     }
2513
2514     if (!init_picture(decoder, picture, unit))
2515         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2516     if (!fill_picture(decoder, picture, unit))
2517         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2518     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2519 }
2520
2521 static inline guint
2522 get_slice_data_bit_offset(GstH264SliceHdr *slice_hdr)
2523 {
2524     guint epb_count;
2525
2526     epb_count = slice_hdr->n_emulation_prevention_bytes;
2527     return 8 /* nal_unit_type */ + slice_hdr->header_size - epb_count * 8;
2528 }
2529
2530 static gboolean
2531 fill_pred_weight_table(GstVaapiDecoderH264 *decoder,
2532     GstVaapiSlice *slice, GstH264SliceHdr *slice_hdr)
2533 {
2534     VASliceParameterBufferH264 * const slice_param = slice->param;
2535     GstH264PPS * const pps = slice_hdr->pps;
2536     GstH264SPS * const sps = pps->sequence;
2537     GstH264PredWeightTable * const w = &slice_hdr->pred_weight_table;
2538     guint num_weight_tables = 0;
2539     gint i, j;
2540
2541     if (pps->weighted_pred_flag &&
2542         (GST_H264_IS_P_SLICE(slice_hdr) || GST_H264_IS_SP_SLICE(slice_hdr)))
2543         num_weight_tables = 1;
2544     else if (pps->weighted_bipred_idc == 1 && GST_H264_IS_B_SLICE(slice_hdr))
2545         num_weight_tables = 2;
2546     else
2547         num_weight_tables = 0;
2548
2549     slice_param->luma_log2_weight_denom   = w->luma_log2_weight_denom;
2550     slice_param->chroma_log2_weight_denom = w->chroma_log2_weight_denom;
2551     slice_param->luma_weight_l0_flag      = 0;
2552     slice_param->chroma_weight_l0_flag    = 0;
2553     slice_param->luma_weight_l1_flag      = 0;
2554     slice_param->chroma_weight_l1_flag    = 0;
2555
2556     if (num_weight_tables < 1)
2557         return TRUE;
2558
2559     slice_param->luma_weight_l0_flag = 1;
2560     for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2561         slice_param->luma_weight_l0[i] = w->luma_weight_l0[i];
2562         slice_param->luma_offset_l0[i] = w->luma_offset_l0[i];
2563     }
2564
2565     slice_param->chroma_weight_l0_flag = sps->chroma_array_type != 0;
2566     if (slice_param->chroma_weight_l0_flag) {
2567         for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2568             for (j = 0; j < 2; j++) {
2569                 slice_param->chroma_weight_l0[i][j] = w->chroma_weight_l0[i][j];
2570                 slice_param->chroma_offset_l0[i][j] = w->chroma_offset_l0[i][j];
2571             }
2572         }
2573     }
2574
2575     if (num_weight_tables < 2)
2576         return TRUE;
2577
2578     slice_param->luma_weight_l1_flag = 1;
2579     for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2580         slice_param->luma_weight_l1[i] = w->luma_weight_l1[i];
2581         slice_param->luma_offset_l1[i] = w->luma_offset_l1[i];
2582     }
2583
2584     slice_param->chroma_weight_l1_flag = sps->chroma_array_type != 0;
2585     if (slice_param->chroma_weight_l1_flag) {
2586         for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2587             for (j = 0; j < 2; j++) {
2588                 slice_param->chroma_weight_l1[i][j] = w->chroma_weight_l1[i][j];
2589                 slice_param->chroma_offset_l1[i][j] = w->chroma_offset_l1[i][j];
2590             }
2591         }
2592     }
2593     return TRUE;
2594 }
2595
2596 static gboolean
2597 fill_RefPicList(GstVaapiDecoderH264 *decoder,
2598     GstVaapiSlice *slice, GstH264SliceHdr *slice_hdr)
2599 {
2600     GstVaapiDecoderH264Private * const priv = decoder->priv;
2601     VASliceParameterBufferH264 * const slice_param = slice->param;
2602     guint i, num_ref_lists = 0;
2603
2604     slice_param->num_ref_idx_l0_active_minus1 = 0;
2605     slice_param->num_ref_idx_l1_active_minus1 = 0;
2606
2607     if (GST_H264_IS_B_SLICE(slice_hdr))
2608         num_ref_lists = 2;
2609     else if (GST_H264_IS_I_SLICE(slice_hdr))
2610         num_ref_lists = 0;
2611     else
2612         num_ref_lists = 1;
2613
2614     if (num_ref_lists < 1)
2615         return TRUE;
2616
2617     slice_param->num_ref_idx_l0_active_minus1 =
2618         slice_hdr->num_ref_idx_l0_active_minus1;
2619
2620     for (i = 0; i < priv->RefPicList0_count && priv->RefPicList0[i]; i++)
2621         vaapi_fill_picture(&slice_param->RefPicList0[i], priv->RefPicList0[i], 0);
2622     for (; i <= slice_param->num_ref_idx_l0_active_minus1; i++)
2623         vaapi_init_picture(&slice_param->RefPicList0[i]);
2624
2625     if (num_ref_lists < 2)
2626         return TRUE;
2627
2628     slice_param->num_ref_idx_l1_active_minus1 =
2629         slice_hdr->num_ref_idx_l1_active_minus1;
2630
2631     for (i = 0; i < priv->RefPicList1_count && priv->RefPicList1[i]; i++)
2632         vaapi_fill_picture(&slice_param->RefPicList1[i], priv->RefPicList1[i], 0);
2633     for (; i <= slice_param->num_ref_idx_l1_active_minus1; i++)
2634         vaapi_init_picture(&slice_param->RefPicList1[i]);
2635     return TRUE;
2636 }
2637
2638 static gboolean
2639 fill_slice(GstVaapiDecoderH264 *decoder,
2640     GstVaapiSlice *slice, GstH264SliceHdr *slice_hdr)
2641 {
2642     VASliceParameterBufferH264 * const slice_param = slice->param;
2643
2644     /* Fill in VASliceParameterBufferH264 */
2645     slice_param->slice_data_bit_offset          = get_slice_data_bit_offset(slice_hdr);
2646     slice_param->first_mb_in_slice              = slice_hdr->first_mb_in_slice;
2647     slice_param->slice_type                     = slice_hdr->type % 5;
2648     slice_param->direct_spatial_mv_pred_flag    = slice_hdr->direct_spatial_mv_pred_flag;
2649     slice_param->cabac_init_idc                 = slice_hdr->cabac_init_idc;
2650     slice_param->slice_qp_delta                 = slice_hdr->slice_qp_delta;
2651     slice_param->disable_deblocking_filter_idc  = slice_hdr->disable_deblocking_filter_idc;
2652     slice_param->slice_alpha_c0_offset_div2     = slice_hdr->slice_alpha_c0_offset_div2;
2653     slice_param->slice_beta_offset_div2         = slice_hdr->slice_beta_offset_div2;
2654
2655     if (!fill_RefPicList(decoder, slice, slice_hdr))
2656         return FALSE;
2657     if (!fill_pred_weight_table(decoder, slice, slice_hdr))
2658         return FALSE;
2659     return TRUE;
2660 }
2661
2662 static GstVaapiDecoderStatus
2663 decode_slice(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnitH264 *unit)
2664 {
2665     GstVaapiDecoderH264Private * const priv = decoder->priv;
2666     GstVaapiPictureH264 * const picture = priv->current_picture;
2667     GstH264SliceHdr * const slice_hdr = &unit->data.slice_hdr;
2668     GstH264NalUnit * const nalu = &unit->nalu;
2669     GstVaapiSlice *slice;
2670
2671     GST_DEBUG("slice (%u bytes)", nalu->size);
2672
2673     if (!priv->got_sps || !priv->got_pps) {
2674         GST_ERROR("not initialized yet");
2675         return GST_VAAPI_DECODER_STATUS_SUCCESS;
2676     }
2677
2678     unit->base.buffer = gst_buffer_create_sub(
2679         GST_VAAPI_DECODER_CODEC_FRAME(decoder)->input_buffer,
2680         unit->base.offset, unit->base.size);
2681     if (!unit->base.buffer) {
2682         GST_ERROR("failed to allocate slice data");
2683         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2684     }
2685
2686     slice = GST_VAAPI_SLICE_NEW(H264, decoder,
2687         GST_BUFFER_DATA(unit->base.buffer) + nalu->offset, nalu->size);
2688     if (!slice) {
2689         GST_ERROR("failed to allocate slice");
2690         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2691     }
2692
2693     if (!fill_slice(decoder, slice, slice_hdr)) {
2694         gst_vaapi_mini_object_unref(GST_VAAPI_MINI_OBJECT(slice));
2695         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2696     }
2697
2698     gst_vaapi_picture_add_slice(GST_VAAPI_PICTURE_CAST(picture), slice);
2699     picture->last_slice_hdr = slice_hdr;
2700     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2701 }
2702
2703 static inline gint
2704 scan_for_start_code(GstAdapter *adapter, guint ofs, guint size, guint32 *scp)
2705 {
2706     return (gint)gst_adapter_masked_scan_uint32_peek(adapter,
2707                                                      0xffffff00, 0x00000100,
2708                                                      ofs, size,
2709                                                      scp);
2710 }
2711
2712 static GstVaapiDecoderStatus
2713 decode_unit(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnitH264 *unit)
2714 {
2715     GstVaapiDecoderStatus status;
2716
2717     switch (unit->nalu.type) {
2718     case GST_H264_NAL_SLICE_IDR:
2719         /* fall-through. IDR specifics are handled in init_picture() */
2720     case GST_H264_NAL_SLICE:
2721         status = decode_slice(decoder, unit);
2722         break;
2723     case GST_H264_NAL_SEQ_END:
2724         status = decode_sequence_end(decoder);
2725         break;
2726     case GST_H264_NAL_SEI:
2727         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2728         break;
2729     default:
2730         GST_WARNING("unsupported NAL unit type %d", unit->nalu.type);
2731         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
2732         break;
2733     }
2734     return status;
2735 }
2736
2737 static GstVaapiDecoderStatus
2738 decode_codec_data(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
2739 {
2740     GstVaapiDecoderH264Private * const priv = decoder->priv;
2741     GstVaapiDecoderStatus status;
2742     GstVaapiDecoderUnitH264 unit;
2743     GstH264ParserResult result;
2744     guchar *buf;
2745     guint buf_size;
2746     guint i, ofs, num_sps, num_pps;
2747
2748     buf      = GST_BUFFER_DATA(buffer);
2749     buf_size = GST_BUFFER_SIZE(buffer);
2750     if (!buf || buf_size == 0)
2751         return GST_VAAPI_DECODER_STATUS_SUCCESS;
2752
2753     if (buf_size < 8)
2754         return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2755
2756     if (buf[0] != 1) {
2757         GST_ERROR("failed to decode codec-data, not in avcC format");
2758         return GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
2759     }
2760
2761     priv->nal_length_size = (buf[4] & 0x03) + 1;
2762
2763     num_sps = buf[5] & 0x1f;
2764     ofs = 6;
2765
2766     for (i = 0; i < num_sps; i++) {
2767         result = gst_h264_parser_identify_nalu_avc(
2768             priv->parser,
2769             buf, ofs, buf_size, 2,
2770             &unit.nalu
2771         );
2772         if (result != GST_H264_PARSER_OK)
2773             return get_status(result);
2774
2775         status = parse_sps(decoder, &unit);
2776         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2777             return status;
2778         ofs = unit.nalu.offset + unit.nalu.size;
2779     }
2780
2781     num_pps = buf[ofs];
2782     ofs++;
2783
2784     for (i = 0; i < num_pps; i++) {
2785         result = gst_h264_parser_identify_nalu_avc(
2786             priv->parser,
2787             buf, ofs, buf_size, 2,
2788             &unit.nalu
2789         );
2790         if (result != GST_H264_PARSER_OK)
2791             return get_status(result);
2792
2793         status = parse_pps(decoder, &unit);
2794         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2795             return status;
2796         ofs = unit.nalu.offset + unit.nalu.size;
2797     }
2798
2799     priv->is_avcC = TRUE;
2800     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2801 }
2802
2803 static GstVaapiDecoderStatus
2804 ensure_decoder(GstVaapiDecoderH264 *decoder)
2805 {
2806     GstVaapiDecoderH264Private * const priv = decoder->priv;
2807     GstVaapiDecoderStatus status;
2808     GstBuffer *codec_data;
2809
2810     g_return_val_if_fail(priv->is_constructed,
2811                          GST_VAAPI_DECODER_STATUS_ERROR_INIT_FAILED);
2812
2813     if (!priv->is_opened) {
2814         priv->is_opened = gst_vaapi_decoder_h264_open(decoder);
2815         if (!priv->is_opened)
2816             return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CODEC;
2817
2818         codec_data = GST_VAAPI_DECODER_CODEC_DATA(decoder);
2819         if (codec_data) {
2820             status = decode_codec_data(decoder, codec_data);
2821             if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2822                 return status;
2823         }
2824     }
2825     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2826 }
2827
2828 static GstVaapiDecoderStatus
2829 gst_vaapi_decoder_h264_parse(GstVaapiDecoder *base_decoder,
2830     GstAdapter *adapter, gboolean at_eos, GstVaapiDecoderUnit **unit_ptr)
2831 {
2832     GstVaapiDecoderH264 * const decoder =
2833         GST_VAAPI_DECODER_H264_CAST(base_decoder);
2834     GstVaapiDecoderH264Private * const priv = decoder->priv;
2835     GstVaapiParserState * const ps = GST_VAAPI_PARSER_STATE(base_decoder);
2836     GstVaapiDecoderUnitH264 *unit;
2837     GstVaapiDecoderStatus status;
2838     GstH264ParserResult result;
2839     guchar *buf;
2840     guint i, size, buf_size, nalu_size, flags;
2841     guint32 start_code;
2842     gint ofs, ofs2;
2843
2844     status = ensure_decoder(decoder);
2845     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2846         return status;
2847
2848     size = gst_adapter_available(adapter);
2849
2850     if (priv->is_avcC) {
2851         if (size < priv->nal_length_size)
2852             return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2853
2854         buf = (guchar *)&start_code;
2855         g_assert(priv->nal_length_size <= sizeof(start_code));
2856         gst_adapter_copy(adapter, buf, 0, priv->nal_length_size);
2857
2858         nalu_size = 0;
2859         for (i = 0; i < priv->nal_length_size; i++)
2860             nalu_size = (nalu_size << 8) | buf[i];
2861
2862         buf_size = priv->nal_length_size + nalu_size;
2863         if (size < buf_size)
2864             return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2865     }
2866     else {
2867         if (size < 4)
2868             return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2869
2870         ofs = scan_for_start_code(adapter, 0, size, NULL);
2871         if (ofs < 0)
2872             return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2873
2874         if (ofs > 0) {
2875             gst_adapter_flush(adapter, ofs);
2876             size -= ofs;
2877         }
2878
2879         ofs2 = ps->input_offset2 - ofs - 4;
2880         if (ofs2 < 4)
2881             ofs2 = 4;
2882
2883         ofs = G_UNLIKELY(size < ofs2 + 4) ? -1 :
2884             scan_for_start_code(adapter, ofs2, size - ofs2, NULL);
2885         if (ofs < 0) {
2886             // Assume the whole NAL unit is present if end-of-stream
2887             if (!at_eos) {
2888                 ps->input_offset2 = size;
2889                 return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2890             }
2891             ofs = size;
2892         }
2893         buf_size = ofs;
2894     }
2895     ps->input_offset2 = 0;
2896
2897     buf = (guchar *)gst_adapter_peek(adapter, buf_size);
2898     if (!buf)
2899         return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2900
2901     unit = gst_vaapi_decoder_unit_h264_new(buf_size);
2902     if (!unit)
2903         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2904
2905     if (priv->is_avcC)
2906         result = gst_h264_parser_identify_nalu_avc(priv->parser,
2907             buf, 0, buf_size, priv->nal_length_size, &unit->nalu);
2908     else
2909         result = gst_h264_parser_identify_nalu_unchecked(priv->parser,
2910             buf, 0, buf_size, &unit->nalu);
2911     status = get_status(result);
2912     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS) {
2913         gst_vaapi_decoder_unit_unref(unit);
2914         return status;
2915     }
2916
2917     switch (unit->nalu.type) {
2918     case GST_H264_NAL_SPS:
2919         status = parse_sps(decoder, unit);
2920         break;
2921     case GST_H264_NAL_PPS:
2922         status = parse_pps(decoder, unit);
2923         break;
2924     case GST_H264_NAL_SEI:
2925         status = parse_sei(decoder, unit);
2926         break;
2927     case GST_H264_NAL_SLICE_IDR:
2928     case GST_H264_NAL_SLICE:
2929         status = parse_slice(decoder, unit);
2930         break;
2931     default:
2932         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2933         break;
2934     }
2935     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2936         return status;
2937
2938     flags = 0;
2939     switch (unit->nalu.type) {
2940     case GST_H264_NAL_AU_DELIMITER:
2941         flags |= GST_VAAPI_DECODER_UNIT_FLAG_FRAME_START;
2942         /* fall-through */
2943     case GST_H264_NAL_FILLER_DATA:
2944         flags |= GST_VAAPI_DECODER_UNIT_FLAG_SKIP;
2945         break;
2946     case GST_H264_NAL_STREAM_END:
2947         flags |= GST_VAAPI_DECODER_UNIT_FLAG_STREAM_END;
2948         /* fall-through */
2949     case GST_H264_NAL_SEQ_END:
2950         flags |= GST_VAAPI_DECODER_UNIT_FLAG_FRAME_END;
2951         break;
2952     case GST_H264_NAL_SPS:
2953     case GST_H264_NAL_PPS:
2954         flags |= GST_VAAPI_DECODER_UNIT_FLAG_SKIP;
2955         /* fall-through */
2956     case GST_H264_NAL_SEI:
2957         flags |= GST_VAAPI_DECODER_UNIT_FLAG_FRAME_START;
2958         break;
2959     case GST_H264_NAL_SLICE_IDR:
2960     case GST_H264_NAL_SLICE:
2961         flags |= GST_VAAPI_DECODER_UNIT_FLAG_SLICE;
2962         if (is_new_picture(unit, priv->prev_slice_unit))
2963             flags |= GST_VAAPI_DECODER_UNIT_FLAG_FRAME_START;
2964         gst_vaapi_decoder_unit_replace(&priv->prev_slice_unit, unit);
2965         break;
2966     default:
2967         if (unit->nalu.type >= 14 && unit->nalu.type <= 18)
2968             flags |= GST_VAAPI_DECODER_UNIT_FLAG_FRAME_START;
2969         break;
2970     }
2971     GST_VAAPI_DECODER_UNIT_FLAG_SET(unit, flags);
2972
2973     unit->nalu.data = NULL;
2974     *unit_ptr = &unit->base;
2975     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2976 }
2977
2978 static GstVaapiDecoderStatus
2979 gst_vaapi_decoder_h264_decode(GstVaapiDecoder *base_decoder,
2980     GstVaapiDecoderUnit *unit)
2981 {
2982     GstVaapiDecoderH264 * const decoder =
2983         GST_VAAPI_DECODER_H264_CAST(base_decoder);
2984     GstVaapiDecoderStatus status;
2985
2986     status = ensure_decoder(decoder);
2987     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2988         return status;
2989     return decode_unit(decoder, (GstVaapiDecoderUnitH264 *)unit);
2990 }
2991
2992 static GstVaapiDecoderStatus
2993 gst_vaapi_decoder_h264_start_frame(GstVaapiDecoder *base_decoder,
2994     GstVaapiDecoderUnit *base_unit)
2995 {
2996     GstVaapiDecoderH264 * const decoder =
2997         GST_VAAPI_DECODER_H264_CAST(base_decoder);
2998     GstVaapiDecoderUnitH264 * const unit = (GstVaapiDecoderUnitH264 *)base_unit;
2999
3000     return decode_picture(decoder, unit);
3001 }
3002
3003 static GstVaapiDecoderStatus
3004 gst_vaapi_decoder_h264_end_frame(GstVaapiDecoder *base_decoder)
3005 {
3006     GstVaapiDecoderH264 * const decoder =
3007         GST_VAAPI_DECODER_H264_CAST(base_decoder);
3008
3009     return decode_current_picture(decoder);
3010 }
3011
3012 static void
3013 gst_vaapi_decoder_h264_finalize(GObject *object)
3014 {
3015     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264_CAST(object);
3016
3017     gst_vaapi_decoder_h264_destroy(decoder);
3018
3019     G_OBJECT_CLASS(gst_vaapi_decoder_h264_parent_class)->finalize(object);
3020 }
3021
3022 static void
3023 gst_vaapi_decoder_h264_constructed(GObject *object)
3024 {
3025     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264_CAST(object);
3026     GstVaapiDecoderH264Private * const priv = decoder->priv;
3027     GObjectClass *parent_class;
3028
3029     parent_class = G_OBJECT_CLASS(gst_vaapi_decoder_h264_parent_class);
3030     if (parent_class->constructed)
3031         parent_class->constructed(object);
3032
3033     priv->is_constructed = gst_vaapi_decoder_h264_create(decoder);
3034 }
3035
3036 static void
3037 gst_vaapi_decoder_h264_class_init(GstVaapiDecoderH264Class *klass)
3038 {
3039     GObjectClass * const object_class = G_OBJECT_CLASS(klass);
3040     GstVaapiDecoderClass * const decoder_class = GST_VAAPI_DECODER_CLASS(klass);
3041
3042     g_type_class_add_private(klass, sizeof(GstVaapiDecoderH264Private));
3043
3044     object_class->finalize      = gst_vaapi_decoder_h264_finalize;
3045     object_class->constructed   = gst_vaapi_decoder_h264_constructed;
3046
3047     decoder_class->parse        = gst_vaapi_decoder_h264_parse;
3048     decoder_class->decode       = gst_vaapi_decoder_h264_decode;
3049     decoder_class->start_frame  = gst_vaapi_decoder_h264_start_frame;
3050     decoder_class->end_frame    = gst_vaapi_decoder_h264_end_frame;
3051 }
3052
3053 static void
3054 gst_vaapi_decoder_h264_init(GstVaapiDecoderH264 *decoder)
3055 {
3056     GstVaapiDecoderH264Private *priv;
3057
3058     priv                        = GST_VAAPI_DECODER_H264_GET_PRIVATE(decoder);
3059     decoder->priv               = priv;
3060     priv->profile               = GST_VAAPI_PROFILE_UNKNOWN;
3061     priv->entrypoint            = GST_VAAPI_ENTRYPOINT_VLD;
3062     priv->chroma_type           = GST_VAAPI_CHROMA_TYPE_YUV420;
3063     priv->prev_pic_structure    = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
3064     priv->progressive_sequence  = TRUE;
3065 }
3066
3067 /**
3068  * gst_vaapi_decoder_h264_new:
3069  * @display: a #GstVaapiDisplay
3070  * @caps: a #GstCaps holding codec information
3071  *
3072  * Creates a new #GstVaapiDecoder for MPEG-2 decoding.  The @caps can
3073  * hold extra information like codec-data and pictured coded size.
3074  *
3075  * Return value: the newly allocated #GstVaapiDecoder object
3076  */
3077 GstVaapiDecoder *
3078 gst_vaapi_decoder_h264_new(GstVaapiDisplay *display, GstCaps *caps)
3079 {
3080     GstVaapiDecoderH264 *decoder;
3081
3082     g_return_val_if_fail(GST_VAAPI_IS_DISPLAY(display), NULL);
3083     g_return_val_if_fail(GST_IS_CAPS(caps), NULL);
3084
3085     decoder = g_object_new(
3086         GST_VAAPI_TYPE_DECODER_H264,
3087         "display",      display,
3088         "caps",         caps,
3089         NULL
3090     );
3091     if (!decoder->priv->is_constructed) {
3092         g_object_unref(decoder);
3093         return NULL;
3094     }
3095     return GST_VAAPI_DECODER_CAST(decoder);
3096 }