gst-libs/gst/vaapi/gstvaapidecoder_h264.c

   1 /*
   2  *  gstvaapidecoder_h264.c - H.264 decoder
   3  *
   4  *  Copyright (C) 2011-2012 Intel Corporation
   5  *
   6  *  This library is free software; you can redistribute it and/or
   7  *  modify it under the terms of the GNU Lesser General Public License
   8  *  as published by the Free Software Foundation; either version 2.1
   9  *  of the License, or (at your option) any later version.
  10  *
  11  *  This library is distributed in the hope that it will be useful,
  12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  *  Lesser General Public License for more details.
  15  *
  16  *  You should have received a copy of the GNU Lesser General Public
  17  *  License along with this library; if not, write to the Free
  18  *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  19  *  Boston, MA 02110-1301 USA
  20  */
  21
  22 /**
  23  * SECTION:gstvaapidecoder_h264
  24  * @short_description: H.264 decoder
  25  */
  26
  27 #include "sysdeps.h"
  28 #include <string.h>
  29 #include <gst/base/gstadapter.h>
  30 #include <gst/codecparsers/gsth264parser.h>
  31 #include "gstvaapidecoder_h264.h"
  32 #include "gstvaapidecoder_objects.h"
  33 #include "gstvaapidecoder_priv.h"
  34 #include "gstvaapidisplay_priv.h"
  35 #include "gstvaapiobject_priv.h"
  36
  37 #define DEBUG 1
  38 #include "gstvaapidebug.h"
  39
  40 /* Defined to 1 if strict ordering of DPB is needed. Only useful for debug */
  41 #define USE_STRICT_DPB_ORDERING 0
  42
  43 typedef struct _GstVaapiFrameStore              GstVaapiFrameStore;
  44 typedef struct _GstVaapiFrameStoreClass         GstVaapiFrameStoreClass;
  45 typedef struct _GstVaapiPictureH264             GstVaapiPictureH264;
  46 typedef struct _GstVaapiPictureH264Class        GstVaapiPictureH264Class;
  47 typedef struct _GstVaapiSliceH264               GstVaapiSliceH264;
  48 typedef struct _GstVaapiSliceH264Class          GstVaapiSliceH264Class;
  49
  50 // Used for field_poc[]
  51 #define TOP_FIELD       0
  52 #define BOTTOM_FIELD    1
  53
  54 /* ------------------------------------------------------------------------- */
  55 /* --- H.264 Pictures                                                    --- */
  56 /* ------------------------------------------------------------------------- */
  57
  58 #define GST_VAAPI_TYPE_PICTURE_H264 \
  59     (gst_vaapi_picture_h264_get_type())
  60
  61 #define GST_VAAPI_PICTURE_H264_CAST(obj) \
  62     ((GstVaapiPictureH264 *)(obj))
  63
  64 #define GST_VAAPI_PICTURE_H264(obj)                             \
  65     (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
  66                                 GST_VAAPI_TYPE_PICTURE_H264,    \
  67                                 GstVaapiPictureH264))
  68
  69 #define GST_VAAPI_PICTURE_H264_CLASS(klass)                     \
  70     (G_TYPE_CHECK_CLASS_CAST((klass),                           \
  71                              GST_VAAPI_TYPE_PICTURE_H264,       \
  72                              GstVaapiPictureH264Class))
  73
  74 #define GST_VAAPI_IS_PICTURE_H264(obj) \
  75     (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_VAAPI_TYPE_PICTURE_H264))
  76
  77 #define GST_VAAPI_IS_PICTURE_H264_CLASS(klass) \
  78     (G_TYPE_CHECK_CLASS_TYPE((klass), GST_VAAPI_TYPE_PICTURE_H264))
  79
  80 #define GST_VAAPI_PICTURE_H264_GET_CLASS(obj)                   \
  81     (G_TYPE_INSTANCE_GET_CLASS((obj),                           \
  82                                GST_VAAPI_TYPE_PICTURE_H264,     \
  83                                GstVaapiPictureH264Class))
  84
  85 /*
  86  * Extended picture flags:
  87  *
  88  * @GST_VAAPI_PICTURE_FLAG_IDR: flag that specifies an IDR picture
  89  * @GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE: flag that specifies
  90  *     "used for short-term reference"
  91  * @GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE: flag that specifies
  92  *     "used for long-term reference"
  93  * @GST_VAAPI_PICTURE_FLAGS_REFERENCE: mask covering any kind of
  94  *     reference picture (short-term reference or long-term reference)
  95  */
  96 enum {
  97     GST_VAAPI_PICTURE_FLAG_IDR = (GST_VAAPI_PICTURE_FLAG_LAST << 0),
  98
  99     GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE = (
 100         GST_VAAPI_PICTURE_FLAG_REFERENCE),
 101     GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE = (
 102         GST_VAAPI_PICTURE_FLAG_REFERENCE | (GST_VAAPI_PICTURE_FLAG_LAST << 1)),
 103     GST_VAAPI_PICTURE_FLAGS_REFERENCE = (
 104         GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE |
 105         GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE),
 106 };
 107
 108 #define GST_VAAPI_PICTURE_IS_IDR(picture) \
 109     (GST_VAAPI_PICTURE_FLAG_IS_SET(picture, GST_VAAPI_PICTURE_FLAG_IDR))
 110
 111 #define GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture)      \
 112     ((GST_VAAPI_PICTURE_FLAGS(picture) &                        \
 113       GST_VAAPI_PICTURE_FLAGS_REFERENCE) ==                     \
 114      GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE)
 115
 116 #define GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture)       \
 117     ((GST_VAAPI_PICTURE_FLAGS(picture) &                        \
 118       GST_VAAPI_PICTURE_FLAGS_REFERENCE) ==                     \
 119      GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE)
 120
 121 struct _GstVaapiPictureH264 {
 122     GstVaapiPicture             base;
 123     GstH264PPS                 *pps;
 124     guint                       structure;
 125     gint32                      field_poc[2];
 126     gint32                      frame_num;              // Original frame_num from slice_header()
 127     gint32                      frame_num_wrap;         // Temporary for ref pic marking: FrameNumWrap
 128     gint32                      long_term_frame_idx;    // Temporary for ref pic marking: LongTermFrameIdx
 129     gint32                      pic_num;                // Temporary for ref pic marking: PicNum
 130     gint32                      long_term_pic_num;      // Temporary for ref pic marking: LongTermPicNum
 131     GstVaapiPictureH264        *other_field;            // Temporary for ref pic marking: other field in the same frame store
 132     guint                       output_flag             : 1;
 133     guint                       output_needed           : 1;
 134 };
 135
 136 struct _GstVaapiPictureH264Class {
 137     /*< private >*/
 138     GstVaapiPictureClass        parent_class;
 139 };
 140
 141 GST_VAAPI_CODEC_DEFINE_TYPE(GstVaapiPictureH264,
 142                             gst_vaapi_picture_h264,
 143                             GST_VAAPI_TYPE_PICTURE)
 144
 145 static void
 146 gst_vaapi_picture_h264_destroy(GstVaapiPictureH264 *decoder)
 147 {
 148 }
 149
 150 static gboolean
 151 gst_vaapi_picture_h264_create(
 152     GstVaapiPictureH264                      *picture,
 153     const GstVaapiCodecObjectConstructorArgs *args
 154 )
 155 {
 156     return TRUE;
 157 }
 158
 159 static void
 160 gst_vaapi_picture_h264_init(GstVaapiPictureH264 *picture)
 161 {
 162     picture->field_poc[0]       = G_MAXINT32;
 163     picture->field_poc[1]       = G_MAXINT32;
 164     picture->output_needed      = FALSE;
 165 }
 166
 167 static inline GstVaapiPictureH264 *
 168 gst_vaapi_picture_h264_new(GstVaapiDecoderH264 *decoder)
 169 {
 170     GstVaapiCodecObject *object;
 171
 172     g_return_val_if_fail(GST_VAAPI_IS_DECODER(decoder), NULL);
 173
 174     object = gst_vaapi_codec_object_new(
 175         GST_VAAPI_TYPE_PICTURE_H264,
 176         GST_VAAPI_CODEC_BASE(decoder),
 177         NULL, sizeof(VAPictureParameterBufferH264),
 178         NULL, 0
 179     );
 180     if (!object)
 181         return NULL;
 182     return GST_VAAPI_PICTURE_H264_CAST(object);
 183 }
 184
 185 static inline void
 186 gst_vaapi_picture_h264_set_reference(
 187     GstVaapiPictureH264 *picture,
 188     guint                reference_flags,
 189     gboolean             other_field
 190 )
 191 {
 192     g_return_if_fail(GST_VAAPI_IS_PICTURE_H264(picture));
 193
 194     GST_VAAPI_PICTURE_FLAG_UNSET(picture, GST_VAAPI_PICTURE_FLAGS_REFERENCE);
 195     GST_VAAPI_PICTURE_FLAG_SET(picture, reference_flags);
 196
 197     if (!other_field || !(picture = picture->other_field))
 198         return;
 199     GST_VAAPI_PICTURE_FLAG_UNSET(picture, GST_VAAPI_PICTURE_FLAGS_REFERENCE);
 200     GST_VAAPI_PICTURE_FLAG_SET(picture, reference_flags);
 201 }
 202
 203 static inline GstVaapiPictureH264 *
 204 gst_vaapi_picture_h264_new_field(GstVaapiPictureH264 *picture)
 205 {
 206     GstVaapiPicture *base_picture;
 207
 208     g_return_val_if_fail(GST_VAAPI_IS_PICTURE_H264(picture), NULL);
 209
 210     base_picture = gst_vaapi_picture_new_field(&picture->base);
 211     if (!base_picture)
 212         return NULL;
 213     return GST_VAAPI_PICTURE_H264_CAST(base_picture);
 214 }
 215
 216 static inline GstVaapiSliceH264 *
 217 gst_vaapi_picture_h264_get_last_slice(GstVaapiPictureH264 *picture)
 218 {
 219     g_return_val_if_fail(GST_VAAPI_IS_PICTURE_H264(picture), NULL);
 220
 221     if (G_UNLIKELY(picture->base.slices->len < 1))
 222         return NULL;
 223     return g_ptr_array_index(picture->base.slices,
 224         picture->base.slices->len - 1);
 225 }
 226
 227 /* ------------------------------------------------------------------------- */
 228 /* --- Slices                                                            --- */
 229 /* ------------------------------------------------------------------------- */
 230
 231 #define GST_VAAPI_TYPE_SLICE_H264 \
 232     (gst_vaapi_slice_h264_get_type())
 233
 234 #define GST_VAAPI_SLICE_H264_CAST(obj) \
 235     ((GstVaapiSliceH264 *)(obj))
 236
 237 #define GST_VAAPI_SLICE_H264(obj)                               \
 238     (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
 239                                 GST_VAAPI_TYPE_SLICE_H264,      \
 240                                 GstVaapiSliceH264))
 241
 242 #define GST_VAAPI_SLICE_H264_CLASS(klass)                       \
 243     (G_TYPE_CHECK_CLASS_CAST((klass),                           \
 244                              GST_VAAPI_TYPE_SLICE_H264,         \
 245                              GstVaapiSliceH264Class))
 246
 247 #define GST_VAAPI_IS_SLICE_H264(obj) \
 248     (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_VAAPI_TYPE_SLICE_H264))
 249
 250 #define GST_VAAPI_IS_SLICE_H264_CLASS(klass) \
 251     (G_TYPE_CHECK_CLASS_TYPE((klass), GST_VAAPI_TYPE_SLICE_H264))
 252
 253 #define GST_VAAPI_SLICE_H264_GET_CLASS(obj)                     \
 254     (G_TYPE_INSTANCE_GET_CLASS((obj),                           \
 255                                GST_VAAPI_TYPE_SLICE_H264,       \
 256                                GstVaapiSliceH264Class))
 257
 258 struct _GstVaapiSliceH264 {
 259     GstVaapiSlice               base;
 260     GstH264SliceHdr             slice_hdr;              // parsed slice_header()
 261 };
 262
 263 struct _GstVaapiSliceH264Class {
 264     /*< private >*/
 265     GstVaapiSliceClass          parent_class;
 266 };
 267
 268 GST_VAAPI_CODEC_DEFINE_TYPE(GstVaapiSliceH264,
 269                             gst_vaapi_slice_h264,
 270                             GST_VAAPI_TYPE_SLICE)
 271
 272 static void
 273 gst_vaapi_slice_h264_destroy(GstVaapiSliceH264 *slice)
 274 {
 275 }
 276
 277 static gboolean
 278 gst_vaapi_slice_h264_create(
 279     GstVaapiSliceH264                        *slice,
 280     const GstVaapiCodecObjectConstructorArgs *args
 281 )
 282 {
 283     return TRUE;
 284 }
 285
 286 static void
 287 gst_vaapi_slice_h264_init(GstVaapiSliceH264 *slice)
 288 {
 289 }
 290
 291 static inline GstVaapiSliceH264 *
 292 gst_vaapi_slice_h264_new(
 293     GstVaapiDecoderH264 *decoder,
 294     const guint8        *data,
 295     guint                data_size
 296 )
 297 {
 298     GstVaapiCodecObject *object;
 299
 300     g_return_val_if_fail(GST_VAAPI_IS_DECODER(decoder), NULL);
 301
 302     object = gst_vaapi_codec_object_new(
 303         GST_VAAPI_TYPE_SLICE_H264,
 304         GST_VAAPI_CODEC_BASE(decoder),
 305         NULL, sizeof(VASliceParameterBufferH264),
 306         data, data_size
 307     );
 308     if (!object)
 309         return NULL;
 310     return GST_VAAPI_SLICE_H264_CAST(object);
 311 }
 312
 313 /* ------------------------------------------------------------------------- */
 314 /* --- Frame Buffers (DPB)                                               --- */
 315 /* ------------------------------------------------------------------------- */
 316
 317 #define GST_VAAPI_TYPE_FRAME_STORE \
 318     (gst_vaapi_frame_store_get_type())
 319
 320 #define GST_VAAPI_FRAME_STORE_CAST(obj) \
 321     ((GstVaapiFrameStore *)(obj))
 322
 323 #define GST_VAAPI_FRAME_STORE(obj)                              \
 324     (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
 325                                 GST_VAAPI_TYPE_FRAME_STORE,     \
 326                                 GstVaapiFrameStore))
 327
 328 #define GST_VAAPI_FRAME_STORE_CLASS(klass)                      \
 329     (G_TYPE_CHECK_CLASS_CAST((klass),                           \
 330                              GST_VAAPI_TYPE_FRAME_STORE,        \
 331                              GstVaapiFrameStoreClass))
 332
 333 #define GST_VAAPI_IS_FRAME_STORE(obj) \
 334     (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_VAAPI_TYPE_FRAME_STORE))
 335
 336 #define GST_VAAPI_IS_FRAME_STORE_CLASS(klass) \
 337     (G_TYPE_CHECK_CLASS_TYPE((klass), GST_VAAPI_TYPE_FRAME_STORE))
 338
 339 #define GST_VAAPI_FRAME_STORE_GET_CLASS(obj)                   \
 340     (G_TYPE_INSTANCE_GET_CLASS((obj),                          \
 341                                GST_VAAPI_TYPE_FRAME_STORE,     \
 342                                GstVaapiFrameStoreClass))
 343
 344 struct _GstVaapiFrameStore {
 345     /*< private >*/
 346     GstMiniObject               parent_instance;
 347
 348     guint                       structure;
 349     GstVaapiPictureH264        *buffers[2];
 350     guint                       num_buffers;
 351     guint                       output_needed;
 352 };
 353
 354 struct _GstVaapiFrameStoreClass {
 355     /*< private >*/
 356     GstMiniObjectClass          parent_class;
 357 };
 358
 359 G_DEFINE_TYPE(GstVaapiFrameStore, gst_vaapi_frame_store, GST_TYPE_MINI_OBJECT)
 360
 361 static void
 362 gst_vaapi_frame_store_finalize(GstMiniObject *object)
 363 {
 364     GstVaapiFrameStore * const fs = GST_VAAPI_FRAME_STORE_CAST(object);
 365     GstMiniObjectClass *parent_class;
 366     guint i;
 367
 368     for (i = 0; i < fs->num_buffers; i++)
 369         gst_vaapi_picture_replace(&fs->buffers[i], NULL);
 370
 371     parent_class = GST_MINI_OBJECT_CLASS(gst_vaapi_frame_store_parent_class);
 372     if (parent_class->finalize)
 373         parent_class->finalize(object);
 374 }
 375
 376 static void
 377 gst_vaapi_frame_store_init(GstVaapiFrameStore *fs)
 378 {
 379 }
 380
 381 static void
 382 gst_vaapi_frame_store_class_init(GstVaapiFrameStoreClass *klass)
 383 {
 384     GstMiniObjectClass * const object_class = GST_MINI_OBJECT_CLASS(klass);
 385
 386     object_class->finalize = gst_vaapi_frame_store_finalize;
 387 }
 388
 389 static inline gpointer
 390 _gst_vaapi_frame_store_new(void)
 391 {
 392     return gst_mini_object_new(GST_VAAPI_TYPE_FRAME_STORE);
 393 }
 394
 395 static GstVaapiFrameStore *
 396 gst_vaapi_frame_store_new(GstVaapiPictureH264 *picture)
 397 {
 398     GstVaapiFrameStore *fs;
 399
 400     g_return_val_if_fail(GST_VAAPI_IS_PICTURE_H264(picture), NULL);
 401
 402     fs = _gst_vaapi_frame_store_new();
 403     if (!fs)
 404         return NULL;
 405
 406     fs->structure       = picture->structure;
 407     fs->buffers[0]      = gst_vaapi_picture_ref(picture);
 408     fs->num_buffers     = 1;
 409     fs->output_needed   = picture->output_needed;
 410     return fs;
 411 }
 412
 413 static gboolean
 414 gst_vaapi_frame_store_add(GstVaapiFrameStore *fs, GstVaapiPictureH264 *picture)
 415 {
 416     guint field;
 417
 418     g_return_val_if_fail(GST_VAAPI_IS_FRAME_STORE(fs), FALSE);
 419     g_return_val_if_fail(fs->num_buffers == 1, FALSE);
 420     g_return_val_if_fail(GST_VAAPI_IS_PICTURE_H264(picture), FALSE);
 421     g_return_val_if_fail(!GST_VAAPI_PICTURE_IS_FRAME(picture), FALSE);
 422
 423     gst_vaapi_picture_replace(&fs->buffers[fs->num_buffers++], picture);
 424     if (picture->output_flag) {
 425         picture->output_needed = TRUE;
 426         fs->output_needed++;
 427     }
 428
 429     fs->structure = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
 430
 431     field = picture->structure == GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD ?
 432         TOP_FIELD : BOTTOM_FIELD;
 433     g_return_val_if_fail(fs->buffers[0]->field_poc[field] == G_MAXINT32, FALSE);
 434     fs->buffers[0]->field_poc[field] = picture->field_poc[field];
 435     g_return_val_if_fail(picture->field_poc[!field] == G_MAXINT32, FALSE);
 436     picture->field_poc[!field] = fs->buffers[0]->field_poc[!field];
 437     return TRUE;
 438 }
 439
 440 static gboolean
 441 gst_vaapi_frame_store_split_fields(GstVaapiFrameStore *fs)
 442 {
 443     GstVaapiPictureH264 * const first_field = fs->buffers[0];
 444     GstVaapiPictureH264 *second_field;
 445
 446     g_return_val_if_fail(fs->num_buffers == 1, FALSE);
 447
 448     first_field->base.structure = GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD;
 449     GST_VAAPI_PICTURE_FLAG_SET(first_field, GST_VAAPI_PICTURE_FLAG_INTERLACED);
 450
 451     second_field = gst_vaapi_picture_h264_new_field(first_field);
 452     if (!second_field)
 453         return FALSE;
 454     gst_vaapi_picture_replace(&fs->buffers[fs->num_buffers++], second_field);
 455     gst_vaapi_picture_unref(second_field);
 456
 457     second_field->frame_num    = first_field->frame_num;
 458     second_field->field_poc[0] = first_field->field_poc[0];
 459     second_field->field_poc[1] = first_field->field_poc[1];
 460     second_field->output_flag  = first_field->output_flag;
 461     if (second_field->output_flag) {
 462         second_field->output_needed = TRUE;
 463         fs->output_needed++;
 464     }
 465     return TRUE;
 466 }
 467
 468 static inline gboolean
 469 gst_vaapi_frame_store_has_frame(GstVaapiFrameStore *fs)
 470 {
 471     return fs->structure == GST_VAAPI_PICTURE_STRUCTURE_FRAME;
 472 }
 473
 474 static inline gboolean
 475 gst_vaapi_frame_store_has_reference(GstVaapiFrameStore *fs)
 476 {
 477     guint i;
 478
 479     for (i = 0; i < fs->num_buffers; i++) {
 480         if (GST_VAAPI_PICTURE_IS_REFERENCE(fs->buffers[i]))
 481             return TRUE;
 482     }
 483     return FALSE;
 484 }
 485
 486 #define gst_vaapi_frame_store_ref(fs) \
 487     gst_mini_object_ref(GST_MINI_OBJECT(fs))
 488
 489 #define gst_vaapi_frame_store_unref(fs) \
 490     gst_mini_object_unref(GST_MINI_OBJECT(fs))
 491
 492 #define gst_vaapi_frame_store_replace(old_fs_p, new_fs)         \
 493     gst_mini_object_replace((GstMiniObject **)(old_fs_p),       \
 494                             (GstMiniObject *)(new_fs))
 495
 496 /* ------------------------------------------------------------------------- */
 497 /* --- H.264 Decoder                                                     --- */
 498 /* ------------------------------------------------------------------------- */
 499
 500 G_DEFINE_TYPE(GstVaapiDecoderH264,
 501               gst_vaapi_decoder_h264,
 502               GST_VAAPI_TYPE_DECODER)
 503
 504 #define GST_VAAPI_DECODER_H264_GET_PRIVATE(obj)                 \
 505     (G_TYPE_INSTANCE_GET_PRIVATE((obj),                         \
 506                                  GST_VAAPI_TYPE_DECODER_H264,   \
 507                                  GstVaapiDecoderH264Private))
 508
 509 struct _GstVaapiDecoderH264Private {
 510     GstAdapter                 *adapter;
 511     GstH264NalParser           *parser;
 512     /* Last decoded SPS. May not be the last activated one. Just here because
 513        it may not fit stack memory allocation in decode_sps() */
 514     GstH264SPS                  last_sps;
 515     /* Last decoded PPS. May not be the last activated one. Just here because
 516        it may not fit stack memory allocation in decode_pps() */
 517     GstH264PPS                  last_pps;
 518     GstVaapiPictureH264        *current_picture;
 519     GstVaapiFrameStore         *prev_frame;
 520     GstVaapiFrameStore         *dpb[16];
 521     guint                       dpb_count;
 522     guint                       dpb_size;
 523     GstVaapiProfile             profile;
 524     GstVaapiEntrypoint          entrypoint;
 525     GstVaapiChromaType          chroma_type;
 526     GstVaapiPictureH264        *short_ref[32];
 527     guint                       short_ref_count;
 528     GstVaapiPictureH264        *long_ref[32];
 529     guint                       long_ref_count;
 530     GstVaapiPictureH264        *RefPicList0[32];
 531     guint                       RefPicList0_count;
 532     GstVaapiPictureH264        *RefPicList1[32];
 533     guint                       RefPicList1_count;
 534     guint                       nal_length_size;
 535     guint                       mb_width;
 536     guint                       mb_height;
 537     gint32                      field_poc[2];           // 0:TopFieldOrderCnt / 1:BottomFieldOrderCnt
 538     gint32                      poc_msb;                // PicOrderCntMsb
 539     gint32                      poc_lsb;                // pic_order_cnt_lsb (from slice_header())
 540     gint32                      prev_poc_msb;           // prevPicOrderCntMsb
 541     gint32                      prev_poc_lsb;           // prevPicOrderCntLsb
 542     gint32                      frame_num_offset;       // FrameNumOffset
 543     gint32                      frame_num;              // frame_num (from slice_header())
 544     gint32                      prev_frame_num;         // prevFrameNum
 545     gboolean                    prev_pic_has_mmco5;     // prevMmco5Pic
 546     gboolean                    prev_pic_structure;     // previous picture structure
 547     guint                       is_constructed          : 1;
 548     guint                       is_opened               : 1;
 549     guint                       is_avc                  : 1;
 550     guint                       got_sps                 : 1;
 551     guint                       got_pps                 : 1;
 552     guint                       has_context             : 1;
 553     guint                       progressive_sequence    : 1;
 554 };
 555
 556 static gboolean
 557 exec_ref_pic_marking(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture);
 558
 559 /* Get number of reference frames to use */
 560 static guint
 561 get_max_dec_frame_buffering(GstH264SPS *sps)
 562 {
 563     guint max_dec_frame_buffering, MaxDpbMbs, PicSizeMbs;
 564
 565     /* Table A-1 - Level limits */
 566     switch (sps->level_idc) {
 567     case 10: MaxDpbMbs = 396;    break;
 568     case 11: MaxDpbMbs = 900;    break;
 569     case 12: MaxDpbMbs = 2376;   break;
 570     case 13: MaxDpbMbs = 2376;   break;
 571     case 20: MaxDpbMbs = 2376;   break;
 572     case 21: MaxDpbMbs = 4752;   break;
 573     case 22: MaxDpbMbs = 8100;   break;
 574     case 30: MaxDpbMbs = 8100;   break;
 575     case 31: MaxDpbMbs = 18000;  break;
 576     case 32: MaxDpbMbs = 20480;  break;
 577     case 40: MaxDpbMbs = 32768;  break;
 578     case 41: MaxDpbMbs = 32768;  break;
 579     case 42: MaxDpbMbs = 34816;  break;
 580     case 50: MaxDpbMbs = 110400; break;
 581     case 51: MaxDpbMbs = 184320; break;
 582     default:
 583         g_assert(0 && "unhandled level");
 584         break;
 585     }
 586
 587     PicSizeMbs = ((sps->pic_width_in_mbs_minus1 + 1) *
 588                   (sps->pic_height_in_map_units_minus1 + 1) *
 589                   (sps->frame_mbs_only_flag ? 1 : 2));
 590     max_dec_frame_buffering = MaxDpbMbs / PicSizeMbs;
 591
 592     /* VUI parameters */
 593     if (sps->vui_parameters_present_flag) {
 594         GstH264VUIParams * const vui_params = &sps->vui_parameters;
 595         if (vui_params->bitstream_restriction_flag)
 596             max_dec_frame_buffering = vui_params->max_dec_frame_buffering;
 597         else {
 598             switch (sps->profile_idc) {
 599             case 44:  // CAVLC 4:4:4 Intra profile
 600             case 86:  // Scalable High profile
 601             case 100: // High profile
 602             case 110: // High 10 profile
 603             case 122: // High 4:2:2 profile
 604             case 244: // High 4:4:4 Predictive profile
 605                 if (sps->constraint_set3_flag)
 606                     max_dec_frame_buffering = 0;
 607                 break;
 608             }
 609         }
 610     }
 611
 612     if (max_dec_frame_buffering > 16)
 613         max_dec_frame_buffering = 16;
 614     else if (max_dec_frame_buffering < sps->num_ref_frames)
 615         max_dec_frame_buffering = sps->num_ref_frames;
 616     return MAX(1, max_dec_frame_buffering);
 617 }
 618
 619 static void
 620 array_remove_index_fast(void *array, guint *array_length_ptr, guint index)
 621 {
 622     gpointer * const entries = array;
 623     guint num_entries = *array_length_ptr;
 624
 625     g_return_if_fail(index < num_entries);
 626
 627     if (index != --num_entries)
 628         entries[index] = entries[num_entries];
 629     entries[num_entries] = NULL;
 630     *array_length_ptr = num_entries;
 631 }
 632
 633 #if 1
 634 static inline void
 635 array_remove_index(void *array, guint *array_length_ptr, guint index)
 636 {
 637     array_remove_index_fast(array, array_length_ptr, index);
 638 }
 639 #else
 640 static void
 641 array_remove_index(void *array, guint *array_length_ptr, guint index)
 642 {
 643     gpointer * const entries = array;
 644     const guint num_entries = *array_length_ptr - 1;
 645     guint i;
 646
 647     g_return_if_fail(index <= num_entries);
 648
 649     for (i = index; i < num_entries; i++)
 650         entries[i] = entries[i + 1];
 651     entries[num_entries] = NULL;
 652     *array_length_ptr = num_entries;
 653 }
 654 #endif
 655
 656 #define ARRAY_REMOVE_INDEX(array, index) \
 657     array_remove_index(array, &array##_count, index)
 658
 659 static void
 660 dpb_remove_index(GstVaapiDecoderH264 *decoder, guint index)
 661 {
 662     GstVaapiDecoderH264Private * const priv = decoder->priv;
 663     guint i, num_frames = --priv->dpb_count;
 664
 665     if (USE_STRICT_DPB_ORDERING) {
 666         for (i = index; i < num_frames; i++)
 667             gst_vaapi_frame_store_replace(&priv->dpb[i], priv->dpb[i + 1]);
 668     }
 669     else if (index != num_frames)
 670         gst_vaapi_frame_store_replace(&priv->dpb[index], priv->dpb[num_frames]);
 671     gst_vaapi_frame_store_replace(&priv->dpb[num_frames], NULL);
 672 }
 673
 674 static gboolean
 675 dpb_output(
 676     GstVaapiDecoderH264 *decoder,
 677     GstVaapiFrameStore  *fs,
 678     GstVaapiPictureH264 *picture
 679 )
 680 {
 681     picture->output_needed = FALSE;
 682
 683     if (fs) {
 684         if (--fs->output_needed > 0)
 685             return TRUE;
 686         picture = fs->buffers[0];
 687     }
 688
 689     /* XXX: update cropping rectangle */
 690     return gst_vaapi_picture_output(GST_VAAPI_PICTURE_CAST(picture));
 691 }
 692
 693 static inline void
 694 dpb_evict(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture, guint i)
 695 {
 696     GstVaapiFrameStore * const fs = decoder->priv->dpb[i];
 697
 698     if (!fs->output_needed && !gst_vaapi_frame_store_has_reference(fs))
 699         dpb_remove_index(decoder, i);
 700 }
 701
 702 static gboolean
 703 dpb_bump(GstVaapiDecoderH264 *decoder)
 704 {
 705     GstVaapiDecoderH264Private * const priv = decoder->priv;
 706     GstVaapiPictureH264 *found_picture = NULL;
 707     guint i, j, found_index;
 708     gboolean success;
 709
 710     for (i = 0; i < priv->dpb_count; i++) {
 711         GstVaapiFrameStore * const fs = priv->dpb[i];
 712         if (!fs->output_needed)
 713             continue;
 714         for (j = 0; j < fs->num_buffers; j++) {
 715             GstVaapiPictureH264 * const picture = fs->buffers[j];
 716             if (!picture->output_needed)
 717                 continue;
 718             if (!found_picture || found_picture->base.poc > picture->base.poc)
 719                 found_picture = picture, found_index = i;
 720         }
 721     }
 722     if (!found_picture)
 723         return FALSE;
 724
 725     success = dpb_output(decoder, priv->dpb[found_index], found_picture);
 726     dpb_evict(decoder, found_picture, found_index);
 727     return success;
 728 }
 729
 730 static void
 731 dpb_clear(GstVaapiDecoderH264 *decoder)
 732 {
 733     GstVaapiDecoderH264Private * const priv = decoder->priv;
 734     guint i;
 735
 736     for (i = 0; i < priv->dpb_count; i++)
 737         gst_vaapi_frame_store_replace(&priv->dpb[i], NULL);
 738     priv->dpb_count = 0;
 739
 740     gst_vaapi_frame_store_replace(&priv->prev_frame, NULL);
 741 }
 742
 743 static void
 744 dpb_flush(GstVaapiDecoderH264 *decoder)
 745 {
 746     while (dpb_bump(decoder))
 747         ;
 748     dpb_clear(decoder);
 749 }
 750
 751 static gboolean
 752 dpb_add(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
 753 {
 754     GstVaapiDecoderH264Private * const priv = decoder->priv;
 755     GstVaapiFrameStore *fs;
 756     guint i, j;
 757
 758     // Remove all unused pictures
 759     if (!GST_VAAPI_PICTURE_IS_IDR(picture)) {
 760         i = 0;
 761         while (i < priv->dpb_count) {
 762             GstVaapiFrameStore * const fs = priv->dpb[i];
 763             if (!fs->output_needed && !gst_vaapi_frame_store_has_reference(fs))
 764                 dpb_remove_index(decoder, i);
 765             else
 766                 i++;
 767         }
 768     }
 769
 770     // Check if picture is the second field and the first field is still in DPB
 771     fs = priv->prev_frame;
 772     if (fs && !gst_vaapi_frame_store_has_frame(fs)) {
 773         g_return_val_if_fail(fs->num_buffers == 1, FALSE);
 774         g_return_val_if_fail(!GST_VAAPI_PICTURE_IS_FRAME(picture), FALSE);
 775         g_return_val_if_fail(!GST_VAAPI_PICTURE_IS_FIRST_FIELD(picture), FALSE);
 776         return gst_vaapi_frame_store_add(fs, picture);
 777     }
 778
 779     // Create new frame store, and split fields if necessary
 780     fs = gst_vaapi_frame_store_new(picture);
 781     if (!fs)
 782         return FALSE;
 783     gst_vaapi_frame_store_replace(&priv->prev_frame, fs);
 784     gst_vaapi_frame_store_unref(fs);
 785
 786     if (!priv->progressive_sequence && gst_vaapi_frame_store_has_frame(fs)) {
 787         if (!gst_vaapi_frame_store_split_fields(fs))
 788             return FALSE;
 789     }
 790
 791     // C.4.5.1 - Storage and marking of a reference decoded picture into the DPB
 792     if (GST_VAAPI_PICTURE_IS_REFERENCE(picture)) {
 793         while (priv->dpb_count == priv->dpb_size) {
 794             if (!dpb_bump(decoder))
 795                 return FALSE;
 796         }
 797         gst_vaapi_frame_store_replace(&priv->dpb[priv->dpb_count++], fs);
 798         if (picture->output_flag) {
 799             picture->output_needed = TRUE;
 800             fs->output_needed++;
 801         }
 802     }
 803
 804     // C.4.5.2 - Storage and marking of a non-reference decoded picture into the DPB
 805     else {
 806         if (!picture->output_flag)
 807             return TRUE;
 808         while (priv->dpb_count == priv->dpb_size) {
 809             gboolean found_picture = FALSE;
 810             for (i = 0; !found_picture && i < priv->dpb_count; i++) {
 811                 GstVaapiFrameStore * const fs = priv->dpb[i];
 812                 if (!fs->output_needed)
 813                     continue;
 814                 for (j = 0; !found_picture && j < fs->num_buffers; j++)
 815                     found_picture = fs->buffers[j]->output_needed &&
 816                         fs->buffers[j]->base.poc < picture->base.poc;
 817             }
 818             if (!found_picture)
 819                 return dpb_output(decoder, NULL, picture);
 820             if (!dpb_bump(decoder))
 821                 return FALSE;
 822         }
 823         gst_vaapi_frame_store_replace(&priv->dpb[priv->dpb_count++], fs);
 824         picture->output_needed = TRUE;
 825         fs->output_needed++;
 826     }
 827     return TRUE;
 828 }
 829
 830 static inline void
 831 dpb_reset(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 832 {
 833     GstVaapiDecoderH264Private * const priv = decoder->priv;
 834
 835     priv->dpb_size = get_max_dec_frame_buffering(sps);
 836     GST_DEBUG("DPB size %u", priv->dpb_size);
 837 }
 838
 839 static GstVaapiDecoderStatus
 840 get_status(GstH264ParserResult result)
 841 {
 842     GstVaapiDecoderStatus status;
 843
 844     switch (result) {
 845     case GST_H264_PARSER_OK:
 846         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
 847         break;
 848     case GST_H264_PARSER_NO_NAL_END:
 849         status = GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
 850         break;
 851     case GST_H264_PARSER_ERROR:
 852         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
 853         break;
 854     default:
 855         status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 856         break;
 857     }
 858     return status;
 859 }
 860
 861 static void
 862 gst_vaapi_decoder_h264_close(GstVaapiDecoderH264 *decoder)
 863 {
 864     GstVaapiDecoderH264Private * const priv = decoder->priv;
 865
 866     gst_vaapi_picture_replace(&priv->current_picture, NULL);
 867
 868     dpb_clear(decoder);
 869
 870     if (priv->parser) {
 871         gst_h264_nal_parser_free(priv->parser);
 872         priv->parser = NULL;
 873     }
 874
 875     if (priv->adapter) {
 876         gst_adapter_clear(priv->adapter);
 877         g_object_unref(priv->adapter);
 878         priv->adapter = NULL;
 879     }
 880 }
 881
 882 static gboolean
 883 gst_vaapi_decoder_h264_open(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
 884 {
 885     GstVaapiDecoderH264Private * const priv = decoder->priv;
 886
 887     gst_vaapi_decoder_h264_close(decoder);
 888
 889     priv->adapter = gst_adapter_new();
 890     if (!priv->adapter)
 891         return FALSE;
 892
 893     priv->parser = gst_h264_nal_parser_new();
 894     if (!priv->parser)
 895         return FALSE;
 896     return TRUE;
 897 }
 898
 899 static void
 900 gst_vaapi_decoder_h264_destroy(GstVaapiDecoderH264 *decoder)
 901 {
 902     gst_vaapi_decoder_h264_close(decoder);
 903 }
 904
 905 static gboolean
 906 gst_vaapi_decoder_h264_create(GstVaapiDecoderH264 *decoder)
 907 {
 908     if (!GST_VAAPI_DECODER_CODEC(decoder))
 909         return FALSE;
 910     return TRUE;
 911 }
 912
 913 static guint
 914 h264_get_profile(GstH264SPS *sps)
 915 {
 916     guint profile = 0;
 917
 918     switch (sps->profile_idc) {
 919     case 66:
 920         profile = GST_VAAPI_PROFILE_H264_BASELINE;
 921         break;
 922     case 77:
 923         profile = GST_VAAPI_PROFILE_H264_MAIN;
 924         break;
 925     case 100:
 926         profile = GST_VAAPI_PROFILE_H264_HIGH;
 927         break;
 928     }
 929     return profile;
 930 }
 931
 932 static guint
 933 h264_get_chroma_type(GstH264SPS *sps)
 934 {
 935     guint chroma_type = 0;
 936
 937     switch (sps->chroma_format_idc) {
 938     case 1:
 939         chroma_type = GST_VAAPI_CHROMA_TYPE_YUV420;
 940         break;
 941     case 2:
 942         chroma_type = GST_VAAPI_CHROMA_TYPE_YUV422;
 943         break;
 944     case 3:
 945         if (!sps->separate_colour_plane_flag)
 946             chroma_type = GST_VAAPI_CHROMA_TYPE_YUV444;
 947         break;
 948     }
 949     return chroma_type;
 950 }
 951
 952 static GstVaapiProfile
 953 get_profile(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 954 {
 955     GstVaapiDecoderH264Private * const priv = decoder->priv;
 956     GstVaapiDisplay * const display = GST_VAAPI_DECODER_DISPLAY(decoder);
 957     GstVaapiProfile profile, profiles[2];
 958     guint i, n_profiles = 0;
 959
 960     profile = h264_get_profile(sps);
 961     if (!profile)
 962         return GST_VAAPI_PROFILE_UNKNOWN;
 963
 964     profiles[n_profiles++] = profile;
 965     switch (profile) {
 966     case GST_VAAPI_PROFILE_H264_MAIN:
 967         profiles[n_profiles++] = GST_VAAPI_PROFILE_H264_HIGH;
 968         break;
 969     default:
 970         break;
 971     }
 972
 973     /* If the preferred profile (profiles[0]) matches one that we already
 974        found, then just return it now instead of searching for it again */
 975     if (profiles[0] == priv->profile)
 976         return priv->profile;
 977
 978     for (i = 0; i < n_profiles; i++) {
 979         if (gst_vaapi_display_has_decoder(display, profiles[i], priv->entrypoint))
 980             return profiles[i];
 981     }
 982     return GST_VAAPI_PROFILE_UNKNOWN;
 983 }
 984
 985 static GstVaapiDecoderStatus
 986 ensure_context(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 987 {
 988     GstVaapiDecoder * const base_decoder = GST_VAAPI_DECODER_CAST(decoder);
 989     GstVaapiDecoderH264Private * const priv = decoder->priv;
 990     GstVaapiContextInfo info;
 991     GstVaapiProfile profile;
 992     GstVaapiChromaType chroma_type;
 993     gboolean reset_context = FALSE;
 994     guint mb_width, mb_height;
 995
 996     profile = get_profile(decoder, sps);
 997     if (!profile) {
 998         GST_ERROR("unsupported profile_idc %u", sps->profile_idc);
 999         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_PROFILE;
1000     }
1001
1002     if (priv->profile != profile) {
1003         GST_DEBUG("profile changed");
1004         reset_context = TRUE;
1005         priv->profile = profile;
1006     }
1007
1008     chroma_type = h264_get_chroma_type(sps);
1009     if (!chroma_type || chroma_type != GST_VAAPI_CHROMA_TYPE_YUV420) {
1010         GST_ERROR("unsupported chroma_format_idc %u", sps->chroma_format_idc);
1011         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;
1012     }
1013
1014     if (priv->chroma_type != chroma_type) {
1015         GST_DEBUG("chroma format changed");
1016         reset_context     = TRUE;
1017         priv->chroma_type = chroma_type;
1018     }
1019
1020     mb_width  = sps->pic_width_in_mbs_minus1 + 1;
1021     mb_height = (sps->pic_height_in_map_units_minus1 + 1) <<
1022         !sps->frame_mbs_only_flag;
1023     if (priv->mb_width != mb_width || priv->mb_height != mb_height) {
1024         GST_DEBUG("size changed");
1025         reset_context   = TRUE;
1026         priv->mb_width  = mb_width;
1027         priv->mb_height = mb_height;
1028     }
1029
1030     priv->progressive_sequence = sps->frame_mbs_only_flag;
1031 #if 0
1032     /* XXX: we only output complete frames for now */
1033     gst_vaapi_decoder_set_interlaced(base_decoder, !priv->progressive_sequence);
1034 #endif
1035
1036     gst_vaapi_decoder_set_pixel_aspect_ratio(
1037         base_decoder,
1038         sps->vui_parameters.par_n,
1039         sps->vui_parameters.par_d
1040     );
1041
1042     if (!reset_context && priv->has_context)
1043         return GST_VAAPI_DECODER_STATUS_SUCCESS;
1044
1045     /* XXX: fix surface size when cropping is implemented */
1046     info.profile    = priv->profile;
1047     info.entrypoint = priv->entrypoint;
1048     info.width      = sps->width;
1049     info.height     = sps->height;
1050     info.ref_frames = get_max_dec_frame_buffering(sps);
1051
1052     if (!gst_vaapi_decoder_ensure_context(GST_VAAPI_DECODER(decoder), &info))
1053         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
1054     priv->has_context = TRUE;
1055
1056     /* Reset DPB */
1057     dpb_reset(decoder, sps);
1058     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1059 }
1060
1061 static void
1062 fill_iq_matrix_4x4(VAIQMatrixBufferH264 *iq_matrix, const GstH264PPS *pps)
1063 {
1064     const guint8 (* const ScalingList4x4)[6][16] = &pps->scaling_lists_4x4;
1065     guint i, j;
1066
1067     /* There are always 6 4x4 scaling lists */
1068     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList4x4) == 6);
1069     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList4x4[0]) == 16);
1070
1071     if (sizeof(iq_matrix->ScalingList4x4[0][0]) == 1)
1072         memcpy(iq_matrix->ScalingList4x4, *ScalingList4x4,
1073                sizeof(iq_matrix->ScalingList4x4));
1074     else {
1075         for (i = 0; i < G_N_ELEMENTS(iq_matrix->ScalingList4x4); i++) {
1076             for (j = 0; j < G_N_ELEMENTS(iq_matrix->ScalingList4x4[i]); j++)
1077                 iq_matrix->ScalingList4x4[i][j] = (*ScalingList4x4)[i][j];
1078         }
1079     }
1080 }
1081
1082 static void
1083 fill_iq_matrix_8x8(VAIQMatrixBufferH264 *iq_matrix, const GstH264PPS *pps)
1084 {
1085     const guint8 (* const ScalingList8x8)[6][64] = &pps->scaling_lists_8x8;
1086     const GstH264SPS * const sps = pps->sequence;
1087     guint i, j, n;
1088
1089     /* If chroma_format_idc != 3, there are up to 2 8x8 scaling lists */
1090     if (!pps->transform_8x8_mode_flag)
1091         return;
1092
1093     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList8x8) >= 2);
1094     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList8x8[0]) == 64);
1095
1096     if (sizeof(iq_matrix->ScalingList8x8[0][0]) == 1)
1097         memcpy(iq_matrix->ScalingList8x8, *ScalingList8x8,
1098                sizeof(iq_matrix->ScalingList8x8));
1099     else {
1100         n = (sps->chroma_format_idc != 3) ? 2 : 6;
1101         for (i = 0; i < n; i++) {
1102             for (j = 0; j < G_N_ELEMENTS(iq_matrix->ScalingList8x8[i]); j++)
1103                 iq_matrix->ScalingList8x8[i][j] = (*ScalingList8x8)[i][j];
1104         }
1105     }
1106 }
1107
1108 static GstVaapiDecoderStatus
1109 ensure_quant_matrix(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
1110 {
1111     GstVaapiPicture * const base_picture = &picture->base;
1112     GstH264PPS * const pps = picture->pps;
1113     GstH264SPS * const sps = pps->sequence;
1114     VAIQMatrixBufferH264 *iq_matrix;
1115
1116     base_picture->iq_matrix = GST_VAAPI_IQ_MATRIX_NEW(H264, decoder);
1117     if (!base_picture->iq_matrix) {
1118         GST_ERROR("failed to allocate IQ matrix");
1119         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
1120     }
1121     iq_matrix = base_picture->iq_matrix->param;
1122
1123     /* XXX: we can only support 4:2:0 or 4:2:2 since ScalingLists8x8[]
1124        is not large enough to hold lists for 4:4:4 */
1125     if (sps->chroma_format_idc == 3)
1126         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;
1127
1128     fill_iq_matrix_4x4(iq_matrix, pps);
1129     fill_iq_matrix_8x8(iq_matrix, pps);
1130
1131     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1132 }
1133
1134 static GstVaapiDecoderStatus
1135 decode_current_picture(GstVaapiDecoderH264 *decoder)
1136 {
1137     GstVaapiDecoderH264Private * const priv = decoder->priv;
1138     GstVaapiPictureH264 * const picture = priv->current_picture;
1139     GstVaapiDecoderStatus status;
1140
1141     if (!picture)
1142         return GST_VAAPI_DECODER_STATUS_SUCCESS;
1143
1144     status = ensure_context(decoder, picture->pps->sequence);
1145     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
1146         return status;
1147
1148     if (!exec_ref_pic_marking(decoder, picture))
1149         goto error;
1150     if (!dpb_add(decoder, picture))
1151         goto error;
1152     if (!gst_vaapi_picture_decode(GST_VAAPI_PICTURE_CAST(picture)))
1153         goto error;
1154     if (priv->prev_frame && gst_vaapi_frame_store_has_frame(priv->prev_frame))
1155         gst_vaapi_picture_replace(&priv->current_picture, NULL);
1156     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1157
1158 error:
1159     /* XXX: fix for cases where first field failed to be decoded */
1160     gst_vaapi_picture_replace(&priv->current_picture, NULL);
1161     return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
1162 }
1163
1164 static GstVaapiDecoderStatus
1165 decode_sps(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
1166 {
1167     GstVaapiDecoderH264Private * const priv = decoder->priv;
1168     GstH264SPS * const sps = &priv->last_sps;
1169     GstH264ParserResult result;
1170
1171     GST_DEBUG("decode SPS");
1172
1173     memset(sps, 0, sizeof(*sps));
1174     result = gst_h264_parser_parse_sps(priv->parser, nalu, sps, TRUE);
1175     if (result != GST_H264_PARSER_OK)
1176         return get_status(result);
1177
1178     priv->got_sps = TRUE;
1179     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1180 }
1181
1182 static GstVaapiDecoderStatus
1183 decode_pps(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
1184 {
1185     GstVaapiDecoderH264Private * const priv = decoder->priv;
1186     GstH264PPS * const pps = &priv->last_pps;
1187     GstH264ParserResult result;
1188
1189     GST_DEBUG("decode PPS");
1190
1191     memset(pps, 0, sizeof(*pps));
1192     result = gst_h264_parser_parse_pps(priv->parser, nalu, pps);
1193     if (result != GST_H264_PARSER_OK)
1194         return get_status(result);
1195
1196     priv->got_pps = TRUE;
1197     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1198 }
1199
1200 static GstVaapiDecoderStatus
1201 decode_sei(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
1202 {
1203     GstVaapiDecoderH264Private * const priv = decoder->priv;
1204     GstH264SEIMessage sei;
1205     GstH264ParserResult result;
1206
1207     GST_DEBUG("decode SEI");
1208
1209     memset(&sei, 0, sizeof(sei));
1210     result = gst_h264_parser_parse_sei(priv->parser, nalu, &sei);
1211     if (result != GST_H264_PARSER_OK) {
1212         GST_WARNING("failed to decode SEI, payload type:%d", sei.payloadType);
1213         return get_status(result);
1214     }
1215
1216     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1217 }
1218
1219 static GstVaapiDecoderStatus
1220 decode_sequence_end(GstVaapiDecoderH264 *decoder)
1221 {
1222     GstVaapiDecoderStatus status;
1223
1224     GST_DEBUG("decode sequence-end");
1225
1226     status = decode_current_picture(decoder);
1227     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
1228         return status;
1229
1230     dpb_flush(decoder);
1231     return GST_VAAPI_DECODER_STATUS_END_OF_STREAM;
1232 }
1233
1234 /* 8.2.1.1 - Decoding process for picture order count type 0 */
1235 static void
1236 init_picture_poc_0(
1237     GstVaapiDecoderH264 *decoder,
1238     GstVaapiPictureH264 *picture,
1239     GstH264SliceHdr     *slice_hdr
1240 )
1241 {
1242     GstVaapiDecoderH264Private * const priv = decoder->priv;
1243     GstH264PPS * const pps = slice_hdr->pps;
1244     GstH264SPS * const sps = pps->sequence;
1245     const gint32 MaxPicOrderCntLsb = 1 << (sps->log2_max_pic_order_cnt_lsb_minus4 + 4);
1246     gint32 temp_poc;
1247
1248     GST_DEBUG("decode picture order count type 0");
1249
1250     if (GST_VAAPI_PICTURE_IS_IDR(picture)) {
1251         priv->prev_poc_msb = 0;
1252         priv->prev_poc_lsb = 0;
1253     }
1254     else if (priv->prev_pic_has_mmco5) {
1255         priv->prev_poc_msb = 0;
1256         priv->prev_poc_lsb =
1257             (priv->prev_pic_structure == GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD ?
1258              0 : priv->field_poc[TOP_FIELD]);
1259     }
1260     else {
1261         priv->prev_poc_msb = priv->poc_msb;
1262         priv->prev_poc_lsb = priv->poc_lsb;
1263     }
1264
1265     // (8-3)
1266     priv->poc_lsb = slice_hdr->pic_order_cnt_lsb;
1267     if (priv->poc_lsb < priv->prev_poc_lsb &&
1268         (priv->prev_poc_lsb - priv->poc_lsb) >= (MaxPicOrderCntLsb / 2))
1269         priv->poc_msb = priv->prev_poc_msb + MaxPicOrderCntLsb;
1270     else if (priv->poc_lsb > priv->prev_poc_lsb &&
1271              (priv->poc_lsb - priv->prev_poc_lsb) > (MaxPicOrderCntLsb / 2))
1272         priv->poc_msb = priv->prev_poc_msb - MaxPicOrderCntLsb;
1273     else
1274         priv->poc_msb = priv->prev_poc_msb;
1275
1276     temp_poc = priv->poc_msb + priv->poc_lsb;
1277     switch (picture->structure) {
1278     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
1279         // (8-4, 8-5)
1280         priv->field_poc[TOP_FIELD] = temp_poc;
1281         priv->field_poc[BOTTOM_FIELD] = temp_poc +
1282             slice_hdr->delta_pic_order_cnt_bottom;
1283         break;
1284     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
1285         // (8-4)
1286         priv->field_poc[TOP_FIELD] = temp_poc;
1287         break;
1288     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
1289         // (8-5)
1290         priv->field_poc[BOTTOM_FIELD] = temp_poc;
1291         break;
1292     }
1293 }
1294
1295 /* 8.2.1.2 - Decoding process for picture order count type 1 */
1296 static void
1297 init_picture_poc_1(
1298     GstVaapiDecoderH264 *decoder,
1299     GstVaapiPictureH264 *picture,
1300     GstH264SliceHdr     *slice_hdr
1301 )
1302 {
1303     GstVaapiDecoderH264Private * const priv = decoder->priv;
1304     GstH264PPS * const pps = slice_hdr->pps;
1305     GstH264SPS * const sps = pps->sequence;
1306     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1307     gint32 prev_frame_num_offset, abs_frame_num, expected_poc;
1308     guint i;
1309
1310     GST_DEBUG("decode picture order count type 1");
1311
1312     if (priv->prev_pic_has_mmco5)
1313         prev_frame_num_offset = 0;
1314     else
1315         prev_frame_num_offset = priv->frame_num_offset;
1316
1317     // (8-6)
1318     if (GST_VAAPI_PICTURE_IS_IDR(picture))
1319         priv->frame_num_offset = 0;
1320     else if (priv->prev_frame_num > priv->frame_num)
1321         priv->frame_num_offset = prev_frame_num_offset + MaxFrameNum;
1322     else
1323         priv->frame_num_offset = prev_frame_num_offset;
1324
1325     // (8-7)
1326     if (sps->num_ref_frames_in_pic_order_cnt_cycle != 0)
1327         abs_frame_num = priv->frame_num_offset + priv->frame_num;
1328     else
1329         abs_frame_num = 0;
1330     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture) && abs_frame_num > 0)
1331         abs_frame_num = abs_frame_num - 1;
1332
1333     if (abs_frame_num > 0) {
1334         gint32 expected_delta_per_poc_cycle;
1335         gint32 poc_cycle_cnt, frame_num_in_poc_cycle;
1336
1337         expected_delta_per_poc_cycle = 0;
1338         for (i = 0; i < sps->num_ref_frames_in_pic_order_cnt_cycle; i++)
1339             expected_delta_per_poc_cycle += sps->offset_for_ref_frame[i];
1340
1341         // (8-8)
1342         poc_cycle_cnt = (abs_frame_num - 1) /
1343             sps->num_ref_frames_in_pic_order_cnt_cycle;
1344         frame_num_in_poc_cycle = (abs_frame_num - 1) %
1345             sps->num_ref_frames_in_pic_order_cnt_cycle;
1346
1347         // (8-9)
1348         expected_poc = poc_cycle_cnt * expected_delta_per_poc_cycle;
1349         for (i = 0; i <= frame_num_in_poc_cycle; i++)
1350             expected_poc += sps->offset_for_ref_frame[i];
1351     }
1352     else
1353         expected_poc = 0;
1354     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
1355         expected_poc += sps->offset_for_non_ref_pic;
1356
1357     // (8-10)
1358     switch (picture->structure) {
1359     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
1360         priv->field_poc[TOP_FIELD] = expected_poc +
1361             slice_hdr->delta_pic_order_cnt[0];
1362         priv->field_poc[BOTTOM_FIELD] = priv->field_poc[TOP_FIELD] +
1363             sps->offset_for_top_to_bottom_field +
1364             slice_hdr->delta_pic_order_cnt[1];
1365         break;
1366     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
1367         priv->field_poc[TOP_FIELD] = expected_poc +
1368             slice_hdr->delta_pic_order_cnt[0];
1369         break;
1370     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
1371         priv->field_poc[BOTTOM_FIELD] = expected_poc +
1372             sps->offset_for_top_to_bottom_field +
1373             slice_hdr->delta_pic_order_cnt[0];
1374         break;
1375     }
1376 }
1377
1378 /* 8.2.1.3 - Decoding process for picture order count type 2 */
1379 static void
1380 init_picture_poc_2(
1381     GstVaapiDecoderH264 *decoder,
1382     GstVaapiPictureH264 *picture,
1383     GstH264SliceHdr     *slice_hdr
1384 )
1385 {
1386     GstVaapiDecoderH264Private * const priv = decoder->priv;
1387     GstH264PPS * const pps = slice_hdr->pps;
1388     GstH264SPS * const sps = pps->sequence;
1389     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1390     gint32 prev_frame_num_offset, temp_poc;
1391
1392     GST_DEBUG("decode picture order count type 2");
1393
1394     if (priv->prev_pic_has_mmco5)
1395         prev_frame_num_offset = 0;
1396     else
1397         prev_frame_num_offset = priv->frame_num_offset;
1398
1399     // (8-11)
1400     if (GST_VAAPI_PICTURE_IS_IDR(picture))
1401         priv->frame_num_offset = 0;
1402     else if (priv->prev_frame_num > priv->frame_num)
1403         priv->frame_num_offset = prev_frame_num_offset + MaxFrameNum;
1404     else
1405         priv->frame_num_offset = prev_frame_num_offset;
1406
1407     // (8-12)
1408     if (GST_VAAPI_PICTURE_IS_IDR(picture))
1409         temp_poc = 0;
1410     else if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
1411         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num) - 1;
1412     else
1413         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num);
1414
1415     // (8-13)
1416     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
1417         priv->field_poc[TOP_FIELD] = temp_poc;
1418     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
1419         priv->field_poc[BOTTOM_FIELD] = temp_poc;
1420 }
1421
1422 /* 8.2.1 - Decoding process for picture order count */
1423 static void
1424 init_picture_poc(
1425     GstVaapiDecoderH264 *decoder,
1426     GstVaapiPictureH264 *picture,
1427     GstH264SliceHdr     *slice_hdr
1428 )
1429 {
1430     GstVaapiDecoderH264Private * const priv = decoder->priv;
1431     GstH264PPS * const pps = slice_hdr->pps;
1432     GstH264SPS * const sps = pps->sequence;
1433
1434     switch (sps->pic_order_cnt_type) {
1435     case 0:
1436         init_picture_poc_0(decoder, picture, slice_hdr);
1437         break;
1438     case 1:
1439         init_picture_poc_1(decoder, picture, slice_hdr);
1440         break;
1441     case 2:
1442         init_picture_poc_2(decoder, picture, slice_hdr);
1443         break;
1444     }
1445
1446     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
1447         picture->field_poc[TOP_FIELD] = priv->field_poc[TOP_FIELD];
1448     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
1449         picture->field_poc[BOTTOM_FIELD] = priv->field_poc[BOTTOM_FIELD];
1450     picture->base.poc = MIN(picture->field_poc[0], picture->field_poc[1]);
1451 }
1452
1453 static int
1454 compare_picture_pic_num_dec(const void *a, const void *b)
1455 {
1456     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1457     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1458
1459     return picB->pic_num - picA->pic_num;
1460 }
1461
1462 static int
1463 compare_picture_long_term_pic_num_inc(const void *a, const void *b)
1464 {
1465     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1466     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1467
1468     return picA->long_term_pic_num - picB->long_term_pic_num;
1469 }
1470
1471 static int
1472 compare_picture_poc_dec(const void *a, const void *b)
1473 {
1474     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1475     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1476
1477     return picB->base.poc - picA->base.poc;
1478 }
1479
1480 static int
1481 compare_picture_poc_inc(const void *a, const void *b)
1482 {
1483     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1484     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1485
1486     return picA->base.poc - picB->base.poc;
1487 }
1488
1489 static int
1490 compare_picture_frame_num_wrap_dec(const void *a, const void *b)
1491 {
1492     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1493     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1494
1495     return picB->frame_num_wrap - picA->frame_num_wrap;
1496 }
1497
1498 static int
1499 compare_picture_long_term_frame_idx_inc(const void *a, const void *b)
1500 {
1501     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1502     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1503
1504     return picA->long_term_frame_idx - picB->long_term_frame_idx;
1505 }
1506
1507 /* 8.2.4.1 - Decoding process for picture numbers */
1508 static void
1509 init_picture_refs_pic_num(
1510     GstVaapiDecoderH264 *decoder,
1511     GstVaapiPictureH264 *picture,
1512     GstH264SliceHdr     *slice_hdr
1513 )
1514 {
1515     GstVaapiDecoderH264Private * const priv = decoder->priv;
1516     GstH264PPS * const pps = slice_hdr->pps;
1517     GstH264SPS * const sps = pps->sequence;
1518     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1519     guint i;
1520
1521     GST_DEBUG("decode picture numbers");
1522
1523     for (i = 0; i < priv->short_ref_count; i++) {
1524         GstVaapiPictureH264 * const pic = priv->short_ref[i];
1525
1526         // (8-27)
1527         if (pic->frame_num > priv->frame_num)
1528             pic->frame_num_wrap = pic->frame_num - MaxFrameNum;
1529         else
1530             pic->frame_num_wrap = pic->frame_num;
1531
1532         // (8-28, 8-30, 8-31)
1533         if (GST_VAAPI_PICTURE_IS_FRAME(picture))
1534             pic->pic_num = pic->frame_num_wrap;
1535         else {
1536             if (pic->structure == picture->structure)
1537                 pic->pic_num = 2 * pic->frame_num_wrap + 1;
1538             else
1539                 pic->pic_num = 2 * pic->frame_num_wrap;
1540         }
1541     }
1542
1543     for (i = 0; i < priv->long_ref_count; i++) {
1544         GstVaapiPictureH264 * const pic = priv->long_ref[i];
1545
1546         // (8-29, 8-32, 8-33)
1547         if (GST_VAAPI_PICTURE_IS_FRAME(picture))
1548             pic->long_term_pic_num = pic->long_term_frame_idx;
1549         else {
1550             if (pic->structure == picture->structure)
1551                 pic->long_term_pic_num = 2 * pic->long_term_frame_idx + 1;
1552             else
1553                 pic->long_term_pic_num = 2 * pic->long_term_frame_idx;
1554         }
1555     }
1556 }
1557
1558 #define SORT_REF_LIST(list, n, compare_func) \
1559     qsort(list, n, sizeof(*(list)), compare_picture_##compare_func)
1560
1561 static void
1562 init_picture_refs_fields_1(
1563     guint                picture_structure,
1564     GstVaapiPictureH264 *RefPicList[32],
1565     guint               *RefPicList_count,
1566     GstVaapiPictureH264 *ref_list[32],
1567     guint                ref_list_count
1568 )
1569 {
1570     guint i, j, n;
1571
1572     i = 0;
1573     j = 0;
1574     n = *RefPicList_count;
1575     do {
1576         g_assert(n < 32);
1577         for (; i < ref_list_count; i++) {
1578             if (ref_list[i]->structure == picture_structure) {
1579                 RefPicList[n++] = ref_list[i++];
1580                 break;
1581             }
1582         }
1583         for (; j < ref_list_count; j++) {
1584             if (ref_list[j]->structure != picture_structure) {
1585                 RefPicList[n++] = ref_list[j++];
1586                 break;
1587             }
1588         }
1589     } while (i < ref_list_count || j < ref_list_count);
1590     *RefPicList_count = n;
1591 }
1592
1593 static inline void
1594 init_picture_refs_fields(
1595     GstVaapiPictureH264 *picture,
1596     GstVaapiPictureH264 *RefPicList[32],
1597     guint               *RefPicList_count,
1598     GstVaapiPictureH264 *short_ref[32],
1599     guint                short_ref_count,
1600     GstVaapiPictureH264 *long_ref[32],
1601     guint                long_ref_count
1602 )
1603 {
1604     guint n = 0;
1605
1606     /* 8.2.4.2.5 - reference picture lists in fields */
1607     init_picture_refs_fields_1(picture->structure, RefPicList, &n,
1608         short_ref, short_ref_count);
1609     init_picture_refs_fields_1(picture->structure, RefPicList, &n,
1610         long_ref, long_ref_count);
1611     *RefPicList_count = n;
1612 }
1613
1614 static void
1615 init_picture_refs_p_slice(
1616     GstVaapiDecoderH264 *decoder,
1617     GstVaapiPictureH264 *picture,
1618     GstH264SliceHdr     *slice_hdr
1619 )
1620 {
1621     GstVaapiDecoderH264Private * const priv = decoder->priv;
1622     GstVaapiPictureH264 **ref_list;
1623     guint i;
1624
1625     GST_DEBUG("decode reference picture list for P and SP slices");
1626
1627     if (GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1628         /* 8.2.4.2.1 - P and SP slices in frames */
1629         if (priv->short_ref_count > 0) {
1630             ref_list = priv->RefPicList0;
1631             for (i = 0; i < priv->short_ref_count; i++)
1632                 ref_list[i] = priv->short_ref[i];
1633             SORT_REF_LIST(ref_list, i, pic_num_dec);
1634             priv->RefPicList0_count += i;
1635         }
1636
1637         if (priv->long_ref_count > 0) {
1638             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1639             for (i = 0; i < priv->long_ref_count; i++)
1640                 ref_list[i] = priv->long_ref[i];
1641             SORT_REF_LIST(ref_list, i, long_term_pic_num_inc);
1642             priv->RefPicList0_count += i;
1643         }
1644     }
1645     else {
1646         /* 8.2.4.2.2 - P and SP slices in fields */
1647         GstVaapiPictureH264 *short_ref[32];
1648         guint short_ref_count = 0;
1649         GstVaapiPictureH264 *long_ref[32];
1650         guint long_ref_count = 0;
1651
1652         if (priv->short_ref_count > 0) {
1653             for (i = 0; i < priv->short_ref_count; i++)
1654                 short_ref[i] = priv->short_ref[i];
1655             SORT_REF_LIST(short_ref, i, frame_num_wrap_dec);
1656             short_ref_count = i;
1657         }
1658
1659         if (priv->long_ref_count > 0) {
1660             for (i = 0; i < priv->long_ref_count; i++)
1661                 long_ref[i] = priv->long_ref[i];
1662             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1663             long_ref_count = i;
1664         }
1665
1666         init_picture_refs_fields(
1667             picture,
1668             priv->RefPicList0, &priv->RefPicList0_count,
1669             short_ref,          short_ref_count,
1670             long_ref,           long_ref_count
1671         );
1672     }
1673 }
1674
1675 static void
1676 init_picture_refs_b_slice(
1677     GstVaapiDecoderH264 *decoder,
1678     GstVaapiPictureH264 *picture,
1679     GstH264SliceHdr     *slice_hdr
1680 )
1681 {
1682     GstVaapiDecoderH264Private * const priv = decoder->priv;
1683     GstVaapiPictureH264 **ref_list;
1684     guint i, n;
1685
1686     GST_DEBUG("decode reference picture list for B slices");
1687
1688     if (GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1689         /* 8.2.4.2.3 - B slices in frames */
1690
1691         /* RefPicList0 */
1692         if (priv->short_ref_count > 0) {
1693             // 1. Short-term references
1694             ref_list = priv->RefPicList0;
1695             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1696                 if (priv->short_ref[i]->base.poc < picture->base.poc)
1697                     ref_list[n++] = priv->short_ref[i];
1698             }
1699             SORT_REF_LIST(ref_list, n, poc_dec);
1700             priv->RefPicList0_count += n;
1701
1702             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1703             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1704                 if (priv->short_ref[i]->base.poc >= picture->base.poc)
1705                     ref_list[n++] = priv->short_ref[i];
1706             }
1707             SORT_REF_LIST(ref_list, n, poc_inc);
1708             priv->RefPicList0_count += n;
1709         }
1710
1711         if (priv->long_ref_count > 0) {
1712             // 2. Long-term references
1713             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1714             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1715                 ref_list[n++] = priv->long_ref[i];
1716             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1717             priv->RefPicList0_count += n;
1718         }
1719
1720         /* RefPicList1 */
1721         if (priv->short_ref_count > 0) {
1722             // 1. Short-term references
1723             ref_list = priv->RefPicList1;
1724             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1725                 if (priv->short_ref[i]->base.poc > picture->base.poc)
1726                     ref_list[n++] = priv->short_ref[i];
1727             }
1728             SORT_REF_LIST(ref_list, n, poc_inc);
1729             priv->RefPicList1_count += n;
1730
1731             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1732             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1733                 if (priv->short_ref[i]->base.poc <= picture->base.poc)
1734                     ref_list[n++] = priv->short_ref[i];
1735             }
1736             SORT_REF_LIST(ref_list, n, poc_dec);
1737             priv->RefPicList1_count += n;
1738         }
1739
1740         if (priv->long_ref_count > 0) {
1741             // 2. Long-term references
1742             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1743             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1744                 ref_list[n++] = priv->long_ref[i];
1745             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1746             priv->RefPicList1_count += n;
1747         }
1748     }
1749     else {
1750         /* 8.2.4.2.4 - B slices in fields */
1751         GstVaapiPictureH264 *short_ref0[32];
1752         guint short_ref0_count = 0;
1753         GstVaapiPictureH264 *short_ref1[32];
1754         guint short_ref1_count = 0;
1755         GstVaapiPictureH264 *long_ref[32];
1756         guint long_ref_count = 0;
1757
1758         /* refFrameList0ShortTerm */
1759         if (priv->short_ref_count > 0) {
1760             ref_list = short_ref0;
1761             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1762                 if (priv->short_ref[i]->base.poc <= picture->base.poc)
1763                     ref_list[n++] = priv->short_ref[i];
1764             }
1765             SORT_REF_LIST(ref_list, n, poc_dec);
1766             short_ref0_count += n;
1767
1768             ref_list = &short_ref0[short_ref0_count];
1769             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1770                 if (priv->short_ref[i]->base.poc > picture->base.poc)
1771                     ref_list[n++] = priv->short_ref[i];
1772             }
1773             SORT_REF_LIST(ref_list, n, poc_inc);
1774             short_ref0_count += n;
1775         }
1776
1777         /* refFrameList1ShortTerm */
1778         if (priv->short_ref_count > 0) {
1779             ref_list = short_ref1;
1780             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1781                 if (priv->short_ref[i]->base.poc > picture->base.poc)
1782                     ref_list[n++] = priv->short_ref[i];
1783             }
1784             SORT_REF_LIST(ref_list, n, poc_inc);
1785             short_ref1_count += n;
1786
1787             ref_list = &short_ref1[short_ref1_count];
1788             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1789                 if (priv->short_ref[i]->base.poc <= picture->base.poc)
1790                     ref_list[n++] = priv->short_ref[i];
1791             }
1792             SORT_REF_LIST(ref_list, n, poc_dec);
1793             short_ref1_count += n;
1794         }
1795
1796         /* refFrameListLongTerm */
1797         if (priv->long_ref_count > 0) {
1798             for (i = 0; i < priv->long_ref_count; i++)
1799                 long_ref[i] = priv->long_ref[i];
1800             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1801             long_ref_count = i;
1802         }
1803
1804         init_picture_refs_fields(
1805             picture,
1806             priv->RefPicList0, &priv->RefPicList0_count,
1807             short_ref0,         short_ref0_count,
1808             long_ref,           long_ref_count
1809         );
1810
1811         init_picture_refs_fields(
1812             picture,
1813             priv->RefPicList1, &priv->RefPicList1_count,
1814             short_ref1,         short_ref1_count,
1815             long_ref,           long_ref_count
1816         );
1817    }
1818
1819     /* Check whether RefPicList1 is identical to RefPicList0, then
1820        swap if necessary */
1821     if (priv->RefPicList1_count > 1 &&
1822         priv->RefPicList1_count == priv->RefPicList0_count &&
1823         memcmp(priv->RefPicList0, priv->RefPicList1,
1824                priv->RefPicList0_count * sizeof(priv->RefPicList0[0])) == 0) {
1825         GstVaapiPictureH264 * const tmp = priv->RefPicList1[0];
1826         priv->RefPicList1[0] = priv->RefPicList1[1];
1827         priv->RefPicList1[1] = tmp;
1828     }
1829 }
1830
1831 #undef SORT_REF_LIST
1832
1833 static gint
1834 find_short_term_reference(GstVaapiDecoderH264 *decoder, gint32 pic_num)
1835 {
1836     GstVaapiDecoderH264Private * const priv = decoder->priv;
1837     guint i;
1838
1839     for (i = 0; i < priv->short_ref_count; i++) {
1840         if (priv->short_ref[i]->pic_num == pic_num)
1841             return i;
1842     }
1843     GST_ERROR("found no short-term reference picture with PicNum = %d",
1844               pic_num);
1845     return -1;
1846 }
1847
1848 static gint
1849 find_long_term_reference(GstVaapiDecoderH264 *decoder, gint32 long_term_pic_num)
1850 {
1851     GstVaapiDecoderH264Private * const priv = decoder->priv;
1852     guint i;
1853
1854     for (i = 0; i < priv->long_ref_count; i++) {
1855         if (priv->long_ref[i]->long_term_pic_num == long_term_pic_num)
1856             return i;
1857     }
1858     GST_ERROR("found no long-term reference picture with LongTermPicNum = %d",
1859               long_term_pic_num);
1860     return -1;
1861 }
1862
1863 static void
1864 exec_picture_refs_modification_1(
1865     GstVaapiDecoderH264           *decoder,
1866     GstVaapiPictureH264           *picture,
1867     GstH264SliceHdr               *slice_hdr,
1868     guint                          list
1869 )
1870 {
1871     GstVaapiDecoderH264Private * const priv = decoder->priv;
1872     GstH264PPS * const pps = slice_hdr->pps;
1873     GstH264SPS * const sps = pps->sequence;
1874     GstH264RefPicListModification *ref_pic_list_modification;
1875     guint num_ref_pic_list_modifications;
1876     GstVaapiPictureH264 **ref_list;
1877     guint *ref_list_count_ptr, ref_list_count, ref_list_idx = 0;
1878     guint i, j, n, num_refs;
1879     gint found_ref_idx;
1880     gint32 MaxPicNum, CurrPicNum, picNumPred;
1881
1882     GST_DEBUG("modification process of reference picture list %u", list);
1883
1884     if (list == 0) {
1885         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l0;
1886         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l0;
1887         ref_list                       = priv->RefPicList0;
1888         ref_list_count_ptr             = &priv->RefPicList0_count;
1889         num_refs                       = slice_hdr->num_ref_idx_l0_active_minus1 + 1;
1890     }
1891     else {
1892         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l1;
1893         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l1;
1894         ref_list                       = priv->RefPicList1;
1895         ref_list_count_ptr             = &priv->RefPicList1_count;
1896         num_refs                       = slice_hdr->num_ref_idx_l1_active_minus1 + 1;
1897     }
1898     ref_list_count = *ref_list_count_ptr;
1899
1900     if (!GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1901         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 5); // 2 * MaxFrameNum
1902         CurrPicNum = 2 * slice_hdr->frame_num + 1;              // 2 * frame_num + 1
1903     }
1904     else {
1905         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 4); // MaxFrameNum
1906         CurrPicNum = slice_hdr->frame_num;                      // frame_num
1907     }
1908
1909     picNumPred = CurrPicNum;
1910
1911     for (i = 0; i < num_ref_pic_list_modifications; i++) {
1912         GstH264RefPicListModification * const l = &ref_pic_list_modification[i];
1913         if (l->modification_of_pic_nums_idc == 3)
1914             break;
1915
1916         /* 8.2.4.3.1 - Short-term reference pictures */
1917         if (l->modification_of_pic_nums_idc == 0 || l->modification_of_pic_nums_idc == 1) {
1918             gint32 abs_diff_pic_num = l->value.abs_diff_pic_num_minus1 + 1;
1919             gint32 picNum, picNumNoWrap;
1920
1921             // (8-34)
1922             if (l->modification_of_pic_nums_idc == 0) {
1923                 picNumNoWrap = picNumPred - abs_diff_pic_num;
1924                 if (picNumNoWrap < 0)
1925                     picNumNoWrap += MaxPicNum;
1926             }
1927
1928             // (8-35)
1929             else {
1930                 picNumNoWrap = picNumPred + abs_diff_pic_num;
1931                 if (picNumNoWrap >= MaxPicNum)
1932                     picNumNoWrap -= MaxPicNum;
1933             }
1934             picNumPred = picNumNoWrap;
1935
1936             // (8-36)
1937             picNum = picNumNoWrap;
1938             if (picNum > CurrPicNum)
1939                 picNum -= MaxPicNum;
1940
1941             // (8-37)
1942             for (j = num_refs; j > ref_list_idx; j--)
1943                 ref_list[j] = ref_list[j - 1];
1944             found_ref_idx = find_short_term_reference(decoder, picNum);
1945             ref_list[ref_list_idx++] =
1946                 found_ref_idx >= 0 ? priv->short_ref[found_ref_idx] : NULL;
1947             n = ref_list_idx;
1948             for (j = ref_list_idx; j <= num_refs; j++) {
1949                 gint32 PicNumF;
1950                 if (!ref_list[j])
1951                     continue;
1952                 PicNumF =
1953                     GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(ref_list[j]) ?
1954                     ref_list[j]->pic_num : MaxPicNum;
1955                 if (PicNumF != picNum)
1956                     ref_list[n++] = ref_list[j];
1957             }
1958         }
1959
1960         /* 8.2.4.3.2 - Long-term reference pictures */
1961         else {
1962
1963             for (j = num_refs; j > ref_list_idx; j--)
1964                 ref_list[j] = ref_list[j - 1];
1965             found_ref_idx =
1966                 find_long_term_reference(decoder, l->value.long_term_pic_num);
1967             ref_list[ref_list_idx++] =
1968                 found_ref_idx >= 0 ? priv->long_ref[found_ref_idx] : NULL;
1969             n = ref_list_idx;
1970             for (j = ref_list_idx; j <= num_refs; j++) {
1971                 gint32 LongTermPicNumF;
1972                 if (!ref_list[j])
1973                     continue;
1974                 LongTermPicNumF =
1975                     GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(ref_list[j]) ?
1976                     ref_list[j]->long_term_pic_num : INT_MAX;
1977                 if (LongTermPicNumF != l->value.long_term_pic_num)
1978                     ref_list[n++] = ref_list[j];
1979             }
1980         }
1981     }
1982
1983 #if DEBUG
1984     for (i = 0; i < num_refs; i++)
1985         if (!ref_list[i])
1986             GST_ERROR("list %u entry %u is empty", list, i);
1987 #endif
1988     *ref_list_count_ptr = num_refs;
1989 }
1990
1991 /* 8.2.4.3 - Modification process for reference picture lists */
1992 static void
1993 exec_picture_refs_modification(
1994     GstVaapiDecoderH264 *decoder,
1995     GstVaapiPictureH264 *picture,
1996     GstH264SliceHdr     *slice_hdr
1997 )
1998 {
1999     GST_DEBUG("execute ref_pic_list_modification()");
2000
2001     /* RefPicList0 */
2002     if (!GST_H264_IS_I_SLICE(slice_hdr) && !GST_H264_IS_SI_SLICE(slice_hdr) &&
2003         slice_hdr->ref_pic_list_modification_flag_l0)
2004         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 0);
2005
2006     /* RefPicList1 */
2007     if (GST_H264_IS_B_SLICE(slice_hdr) &&
2008         slice_hdr->ref_pic_list_modification_flag_l1)
2009         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 1);
2010 }
2011
2012 static void
2013 init_picture_ref_lists(GstVaapiDecoderH264 *decoder)
2014 {
2015     GstVaapiDecoderH264Private * const priv = decoder->priv;
2016     guint i, j, short_ref_count, long_ref_count;
2017
2018     short_ref_count = 0;
2019     long_ref_count  = 0;
2020     if (GST_VAAPI_PICTURE_IS_FRAME(priv->current_picture)) {
2021         for (i = 0; i < priv->dpb_count; i++) {
2022             GstVaapiFrameStore * const fs = priv->dpb[i];
2023             GstVaapiPictureH264 *picture;
2024             if (!gst_vaapi_frame_store_has_frame(fs))
2025                 continue;
2026             picture = fs->buffers[0];
2027             if (GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture))
2028                 priv->short_ref[short_ref_count++] = picture;
2029             else if (GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture))
2030                 priv->long_ref[long_ref_count++] = picture;
2031             picture->structure = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
2032             picture->other_field = fs->buffers[1];
2033         }
2034     }
2035     else {
2036         for (i = 0; i < priv->dpb_count; i++) {
2037             GstVaapiFrameStore * const fs = priv->dpb[i];
2038             for (j = 0; j < fs->num_buffers; j++) {
2039                 GstVaapiPictureH264 * const picture = fs->buffers[j];
2040                 if (GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture))
2041                     priv->short_ref[short_ref_count++] = picture;
2042                 else if (GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture))
2043                     priv->long_ref[long_ref_count++] = picture;
2044                 picture->structure = picture->base.structure;
2045                 picture->other_field = fs->buffers[j ^ 1];
2046             }
2047         }
2048     }
2049
2050     for (i = short_ref_count; i < priv->short_ref_count; i++)
2051         priv->short_ref[i] = NULL;
2052     priv->short_ref_count = short_ref_count;
2053
2054     for (i = long_ref_count; i < priv->long_ref_count; i++)
2055         priv->long_ref[i] = NULL;
2056     priv->long_ref_count = long_ref_count;
2057 }
2058
2059 static void
2060 init_picture_refs(
2061     GstVaapiDecoderH264 *decoder,
2062     GstVaapiPictureH264 *picture,
2063     GstH264SliceHdr     *slice_hdr
2064 )
2065 {
2066     GstVaapiDecoderH264Private * const priv = decoder->priv;
2067     GstVaapiPicture * const base_picture = &picture->base;
2068     guint i, num_refs;
2069
2070     init_picture_ref_lists(decoder);
2071     init_picture_refs_pic_num(decoder, picture, slice_hdr);
2072
2073     priv->RefPicList0_count = 0;
2074     priv->RefPicList1_count = 0;
2075
2076     switch (base_picture->type) {
2077     case GST_VAAPI_PICTURE_TYPE_P:
2078     case GST_VAAPI_PICTURE_TYPE_SP:
2079         init_picture_refs_p_slice(decoder, picture, slice_hdr);
2080         break;
2081     case GST_VAAPI_PICTURE_TYPE_B:
2082         init_picture_refs_b_slice(decoder, picture, slice_hdr);
2083         break;
2084     default:
2085         break;
2086     }
2087
2088     exec_picture_refs_modification(decoder, picture, slice_hdr);
2089
2090     switch (base_picture->type) {
2091     case GST_VAAPI_PICTURE_TYPE_B:
2092         num_refs = 1 + slice_hdr->num_ref_idx_l1_active_minus1;
2093         for (i = priv->RefPicList1_count; i < num_refs; i++)
2094             priv->RefPicList1[i] = NULL;
2095         priv->RefPicList1_count = num_refs;
2096
2097         // fall-through
2098     case GST_VAAPI_PICTURE_TYPE_P:
2099     case GST_VAAPI_PICTURE_TYPE_SP:
2100         num_refs = 1 + slice_hdr->num_ref_idx_l0_active_minus1;
2101         for (i = priv->RefPicList0_count; i < num_refs; i++)
2102             priv->RefPicList0[i] = NULL;
2103         priv->RefPicList0_count = num_refs;
2104         break;
2105     default:
2106         break;
2107     }
2108 }
2109
2110 static gboolean
2111 init_picture(
2112     GstVaapiDecoderH264 *decoder,
2113     GstVaapiPictureH264 *picture,
2114     GstH264SliceHdr     *slice_hdr,
2115     GstH264NalUnit      *nalu
2116 )
2117 {
2118     GstVaapiDecoderH264Private * const priv = decoder->priv;
2119     GstVaapiPicture * const base_picture = &picture->base;
2120
2121     priv->prev_frame_num        = priv->frame_num;
2122     priv->frame_num             = slice_hdr->frame_num;
2123     picture->frame_num          = priv->frame_num;
2124     picture->frame_num_wrap     = priv->frame_num;
2125     picture->output_flag        = TRUE; /* XXX: conformant to Annex A only */
2126     base_picture->pts           = gst_adapter_prev_timestamp(priv->adapter, NULL);
2127
2128     /* Reset decoder state for IDR pictures */
2129     if (nalu->type == GST_H264_NAL_SLICE_IDR) {
2130         GST_DEBUG("<IDR>");
2131         GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_IDR);
2132         dpb_flush(decoder);
2133     }
2134
2135     /* Initialize slice type */
2136     switch (slice_hdr->type % 5) {
2137     case GST_H264_P_SLICE:
2138         base_picture->type = GST_VAAPI_PICTURE_TYPE_P;
2139         break;
2140     case GST_H264_B_SLICE:
2141         base_picture->type = GST_VAAPI_PICTURE_TYPE_B;
2142         break;
2143     case GST_H264_I_SLICE:
2144         base_picture->type = GST_VAAPI_PICTURE_TYPE_I;
2145         break;
2146     case GST_H264_SP_SLICE:
2147         base_picture->type = GST_VAAPI_PICTURE_TYPE_SP;
2148         break;
2149     case GST_H264_SI_SLICE:
2150         base_picture->type = GST_VAAPI_PICTURE_TYPE_SI;
2151         break;
2152     }
2153
2154     /* Initialize picture structure */
2155     if (!slice_hdr->field_pic_flag)
2156         base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
2157     else {
2158         GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_INTERLACED);
2159         if (!slice_hdr->bottom_field_flag)
2160             base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD;
2161         else
2162             base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD;
2163     }
2164     picture->structure = base_picture->structure;
2165
2166     /* Initialize reference flags */
2167     if (nalu->ref_idc) {
2168         GstH264DecRefPicMarking * const dec_ref_pic_marking =
2169             &slice_hdr->dec_ref_pic_marking;
2170
2171         if (GST_VAAPI_PICTURE_IS_IDR(picture) &&
2172             dec_ref_pic_marking->long_term_reference_flag)
2173             GST_VAAPI_PICTURE_FLAG_SET(picture,
2174                 GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE);
2175         else
2176             GST_VAAPI_PICTURE_FLAG_SET(picture,
2177                 GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE);
2178     }
2179
2180     init_picture_poc(decoder, picture, slice_hdr);
2181     init_picture_refs(decoder, picture, slice_hdr);
2182     return TRUE;
2183 }
2184
2185 /* 8.2.5.3 - Sliding window decoded reference picture marking process */
2186 static gboolean
2187 exec_ref_pic_marking_sliding_window(GstVaapiDecoderH264 *decoder)
2188 {
2189     GstVaapiDecoderH264Private * const priv = decoder->priv;
2190     GstH264PPS * const pps = priv->current_picture->pps;
2191     GstH264SPS * const sps = pps->sequence;
2192     GstVaapiPictureH264 *ref_picture;
2193     guint i, m, max_num_ref_frames;
2194
2195     GST_DEBUG("reference picture marking process (sliding window)");
2196
2197     if (!GST_VAAPI_PICTURE_IS_FIRST_FIELD(priv->current_picture))
2198         return TRUE;
2199
2200     max_num_ref_frames = sps->num_ref_frames;
2201     if (max_num_ref_frames == 0)
2202         max_num_ref_frames = 1;
2203     if (!GST_VAAPI_PICTURE_IS_FRAME(priv->current_picture))
2204         max_num_ref_frames <<= 1;
2205
2206     if (priv->short_ref_count + priv->long_ref_count < max_num_ref_frames)
2207         return TRUE;
2208     if (priv->short_ref_count < 1)
2209         return FALSE;
2210
2211     for (m = 0, i = 1; i < priv->short_ref_count; i++) {
2212         GstVaapiPictureH264 * const picture = priv->short_ref[i];
2213         if (picture->frame_num_wrap < priv->short_ref[m]->frame_num_wrap)
2214             m = i;
2215     }
2216
2217     ref_picture = priv->short_ref[m];
2218     gst_vaapi_picture_h264_set_reference(ref_picture, 0, TRUE);
2219     ARRAY_REMOVE_INDEX(priv->short_ref, m);
2220
2221     /* Both fields need to be marked as "unused for reference", so
2222        remove the other field from the short_ref[] list as well */
2223     if (!GST_VAAPI_PICTURE_IS_FRAME(priv->current_picture) && ref_picture->other_field) {
2224         for (i = 0; i < priv->short_ref_count; i++) {
2225             if (priv->short_ref[i] == ref_picture->other_field) {
2226                 ARRAY_REMOVE_INDEX(priv->short_ref, i);
2227                 break;
2228             }
2229         }
2230     }
2231     return TRUE;
2232 }
2233
2234 static inline gint32
2235 get_picNumX(GstVaapiPictureH264 *picture, GstH264RefPicMarking *ref_pic_marking)
2236 {
2237     gint32 pic_num;
2238
2239     if (GST_VAAPI_PICTURE_IS_FRAME(picture))
2240         pic_num = picture->frame_num_wrap;
2241     else
2242         pic_num = 2 * picture->frame_num_wrap + 1;
2243     pic_num -= ref_pic_marking->difference_of_pic_nums_minus1 + 1;
2244     return pic_num;
2245 }
2246
2247 /* 8.2.5.4.1. Mark short-term reference picture as "unused for reference" */
2248 static void
2249 exec_ref_pic_marking_adaptive_mmco_1(
2250     GstVaapiDecoderH264  *decoder,
2251     GstVaapiPictureH264  *picture,
2252     GstH264RefPicMarking *ref_pic_marking
2253 )
2254 {
2255     GstVaapiDecoderH264Private * const priv = decoder->priv;
2256     gint32 i, picNumX;
2257
2258     picNumX = get_picNumX(picture, ref_pic_marking);
2259     i = find_short_term_reference(decoder, picNumX);
2260     if (i < 0)
2261         return;
2262
2263     gst_vaapi_picture_h264_set_reference(priv->short_ref[i], 0,
2264         GST_VAAPI_PICTURE_IS_FRAME(picture));
2265     ARRAY_REMOVE_INDEX(priv->short_ref, i);
2266 }
2267
2268 /* 8.2.5.4.2. Mark long-term reference picture as "unused for reference" */
2269 static void
2270 exec_ref_pic_marking_adaptive_mmco_2(
2271     GstVaapiDecoderH264  *decoder,
2272     GstVaapiPictureH264  *picture,
2273     GstH264RefPicMarking *ref_pic_marking
2274 )
2275 {
2276     GstVaapiDecoderH264Private * const priv = decoder->priv;
2277     gint32 i;
2278
2279     i = find_long_term_reference(decoder, ref_pic_marking->long_term_pic_num);
2280     if (i < 0)
2281         return;
2282
2283     gst_vaapi_picture_h264_set_reference(priv->long_ref[i], 0,
2284         GST_VAAPI_PICTURE_IS_FRAME(picture));
2285     ARRAY_REMOVE_INDEX(priv->long_ref, i);
2286 }
2287
2288 /* 8.2.5.4.3. Assign LongTermFrameIdx to a short-term reference picture */
2289 static void
2290 exec_ref_pic_marking_adaptive_mmco_3(
2291     GstVaapiDecoderH264  *decoder,
2292     GstVaapiPictureH264  *picture,
2293     GstH264RefPicMarking *ref_pic_marking
2294 )
2295 {
2296     GstVaapiDecoderH264Private * const priv = decoder->priv;
2297     GstVaapiPictureH264 *ref_picture;
2298     gint32 i, picNumX;
2299
2300     for (i = 0; i < priv->long_ref_count; i++) {
2301         if (priv->long_ref[i]->long_term_frame_idx == ref_pic_marking->long_term_frame_idx)
2302             break;
2303     }
2304     if (i != priv->long_ref_count) {
2305         gst_vaapi_picture_h264_set_reference(priv->long_ref[i], 0, TRUE);
2306         ARRAY_REMOVE_INDEX(priv->long_ref, i);
2307     }
2308
2309     picNumX = get_picNumX(picture, ref_pic_marking);
2310     i = find_short_term_reference(decoder, picNumX);
2311     if (i < 0)
2312         return;
2313
2314     ref_picture = priv->short_ref[i];
2315     ARRAY_REMOVE_INDEX(priv->short_ref, i);
2316     priv->long_ref[priv->long_ref_count++] = ref_picture;
2317
2318     ref_picture->long_term_frame_idx = ref_pic_marking->long_term_frame_idx;
2319     gst_vaapi_picture_h264_set_reference(ref_picture,
2320         GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE,
2321         GST_VAAPI_PICTURE_IS_FRAME(picture));
2322 }
2323
2324 /* 8.2.5.4.4. Mark pictures with LongTermFramIdx > max_long_term_frame_idx
2325  * as "unused for reference" */
2326 static void
2327 exec_ref_pic_marking_adaptive_mmco_4(
2328     GstVaapiDecoderH264  *decoder,
2329     GstVaapiPictureH264  *picture,
2330     GstH264RefPicMarking *ref_pic_marking
2331 )
2332 {
2333     GstVaapiDecoderH264Private * const priv = decoder->priv;
2334     gint32 i, long_term_frame_idx;
2335
2336     long_term_frame_idx = ref_pic_marking->max_long_term_frame_idx_plus1 - 1;
2337
2338     for (i = 0; i < priv->long_ref_count; i++) {
2339         if (priv->long_ref[i]->long_term_frame_idx <= long_term_frame_idx)
2340             continue;
2341         gst_vaapi_picture_h264_set_reference(priv->long_ref[i], 0, FALSE);
2342         ARRAY_REMOVE_INDEX(priv->long_ref, i);
2343         i--;
2344     }
2345 }
2346
2347 /* 8.2.5.4.5. Mark all reference pictures as "unused for reference" */
2348 static void
2349 exec_ref_pic_marking_adaptive_mmco_5(
2350     GstVaapiDecoderH264  *decoder,
2351     GstVaapiPictureH264  *picture,
2352     GstH264RefPicMarking *ref_pic_marking
2353 )
2354 {
2355     GstVaapiDecoderH264Private * const priv = decoder->priv;
2356
2357     dpb_flush(decoder);
2358
2359     priv->prev_pic_has_mmco5 = TRUE;
2360
2361     /* The picture shall be inferred to have had frame_num equal to 0 (7.4.3) */
2362     priv->frame_num = 0;
2363     priv->frame_num_offset = 0;
2364     picture->frame_num = 0;
2365
2366     /* Update TopFieldOrderCnt and BottomFieldOrderCnt (8.2.1) */
2367     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
2368         picture->field_poc[TOP_FIELD] -= picture->base.poc;
2369     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
2370         picture->field_poc[BOTTOM_FIELD] -= picture->base.poc;
2371     picture->base.poc = 0;
2372 }
2373
2374 /* 8.2.5.4.6. Assign a long-term frame index to the current picture */
2375 static void
2376 exec_ref_pic_marking_adaptive_mmco_6(
2377     GstVaapiDecoderH264  *decoder,
2378     GstVaapiPictureH264  *picture,
2379     GstH264RefPicMarking *ref_pic_marking
2380 )
2381 {
2382     picture->long_term_frame_idx = ref_pic_marking->long_term_frame_idx;
2383     gst_vaapi_picture_h264_set_reference(picture,
2384         GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE, FALSE);
2385 }
2386
2387 /* 8.2.5.4. Adaptive memory control decoded reference picture marking process */
2388 static gboolean
2389 exec_ref_pic_marking_adaptive(
2390     GstVaapiDecoderH264     *decoder,
2391     GstVaapiPictureH264     *picture,
2392     GstH264DecRefPicMarking *dec_ref_pic_marking
2393 )
2394 {
2395     guint i;
2396
2397     GST_DEBUG("reference picture marking process (adaptive memory control)");
2398
2399     typedef void (*exec_ref_pic_marking_adaptive_mmco_func)(
2400         GstVaapiDecoderH264  *decoder,
2401         GstVaapiPictureH264  *picture,
2402         GstH264RefPicMarking *ref_pic_marking
2403     );
2404
2405     static const exec_ref_pic_marking_adaptive_mmco_func mmco_funcs[] = {
2406         NULL,
2407         exec_ref_pic_marking_adaptive_mmco_1,
2408         exec_ref_pic_marking_adaptive_mmco_2,
2409         exec_ref_pic_marking_adaptive_mmco_3,
2410         exec_ref_pic_marking_adaptive_mmco_4,
2411         exec_ref_pic_marking_adaptive_mmco_5,
2412         exec_ref_pic_marking_adaptive_mmco_6,
2413     };
2414
2415     for (i = 0; i < dec_ref_pic_marking->n_ref_pic_marking; i++) {
2416         GstH264RefPicMarking * const ref_pic_marking =
2417             &dec_ref_pic_marking->ref_pic_marking[i];
2418
2419         const guint mmco = ref_pic_marking->memory_management_control_operation;
2420         if (mmco < G_N_ELEMENTS(mmco_funcs) && mmco_funcs[mmco])
2421             mmco_funcs[mmco](decoder, picture, ref_pic_marking);
2422         else {
2423             GST_ERROR("unhandled MMCO %u", mmco);
2424             return FALSE;
2425         }
2426     }
2427     return TRUE;
2428 }
2429
2430 /* 8.2.5 - Execute reference picture marking process */
2431 static gboolean
2432 exec_ref_pic_marking(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
2433 {
2434     GstVaapiDecoderH264Private * const priv = decoder->priv;
2435
2436     priv->prev_pic_has_mmco5 = FALSE;
2437     priv->prev_pic_structure = picture->structure;
2438
2439     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
2440         return TRUE;
2441
2442     if (!GST_VAAPI_PICTURE_IS_IDR(picture)) {
2443         GstVaapiSliceH264 * const slice =
2444             gst_vaapi_picture_h264_get_last_slice(picture);
2445         GstH264DecRefPicMarking * const dec_ref_pic_marking =
2446             &slice->slice_hdr.dec_ref_pic_marking;
2447         if (dec_ref_pic_marking->adaptive_ref_pic_marking_mode_flag) {
2448             if (!exec_ref_pic_marking_adaptive(decoder, picture, dec_ref_pic_marking))
2449                 return FALSE;
2450         }
2451         else {
2452             if (!exec_ref_pic_marking_sliding_window(decoder))
2453                 return FALSE;
2454         }
2455     }
2456     return TRUE;
2457 }
2458
2459 static void
2460 vaapi_init_picture(VAPictureH264 *pic)
2461 {
2462     pic->picture_id           = VA_INVALID_ID;
2463     pic->frame_idx            = 0;
2464     pic->flags                = VA_PICTURE_H264_INVALID;
2465     pic->TopFieldOrderCnt     = 0;
2466     pic->BottomFieldOrderCnt  = 0;
2467 }
2468
2469 static void
2470 vaapi_fill_picture(VAPictureH264 *pic, GstVaapiPictureH264 *picture,
2471     guint picture_structure)
2472 {
2473     if (!picture_structure)
2474         picture_structure = picture->structure;
2475
2476     pic->picture_id = picture->base.surface_id;
2477     pic->flags = 0;
2478
2479     if (GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture)) {
2480         pic->flags |= VA_PICTURE_H264_LONG_TERM_REFERENCE;
2481         pic->frame_idx = picture->long_term_frame_idx;
2482     }
2483     else {
2484         if (GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture))
2485             pic->flags |= VA_PICTURE_H264_SHORT_TERM_REFERENCE;
2486         pic->frame_idx = picture->frame_num;
2487     }
2488
2489     switch (picture_structure) {
2490     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
2491         pic->TopFieldOrderCnt = picture->field_poc[TOP_FIELD];
2492         pic->BottomFieldOrderCnt = picture->field_poc[BOTTOM_FIELD];
2493         break;
2494     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
2495         pic->flags |= VA_PICTURE_H264_TOP_FIELD;
2496         pic->TopFieldOrderCnt = picture->field_poc[TOP_FIELD];
2497         pic->BottomFieldOrderCnt = 0;
2498         break;
2499     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
2500         pic->flags |= VA_PICTURE_H264_BOTTOM_FIELD;
2501         pic->BottomFieldOrderCnt = picture->field_poc[BOTTOM_FIELD];
2502         pic->TopFieldOrderCnt = 0;
2503         break;
2504     }
2505 }
2506
2507 static gboolean
2508 fill_picture(
2509     GstVaapiDecoderH264 *decoder,
2510     GstVaapiPictureH264 *picture,
2511     GstH264SliceHdr     *slice_hdr,
2512     GstH264NalUnit      *nalu
2513 )
2514 {
2515     GstVaapiDecoderH264Private * const priv = decoder->priv;
2516     GstVaapiPicture * const base_picture = &picture->base;
2517     GstH264PPS * const pps = picture->pps;
2518     GstH264SPS * const sps = pps->sequence;
2519     VAPictureParameterBufferH264 * const pic_param = base_picture->param;
2520     guint i, n;
2521
2522     /* Fill in VAPictureParameterBufferH264 */
2523     vaapi_fill_picture(&pic_param->CurrPic, picture, 0);
2524
2525     for (i = 0, n = 0; i < priv->dpb_count; i++) {
2526         GstVaapiFrameStore * const fs = priv->dpb[i];
2527         if (gst_vaapi_frame_store_has_reference(fs))
2528             vaapi_fill_picture(&pic_param->ReferenceFrames[n++],
2529                 fs->buffers[0], fs->structure);
2530     }
2531     for (; n < G_N_ELEMENTS(pic_param->ReferenceFrames); n++)
2532         vaapi_init_picture(&pic_param->ReferenceFrames[n]);
2533
2534 #define COPY_FIELD(s, f) \
2535     pic_param->f = (s)->f
2536
2537 #define COPY_BFM(a, s, f) \
2538     pic_param->a.bits.f = (s)->f
2539
2540     pic_param->picture_width_in_mbs_minus1  = priv->mb_width - 1;
2541     pic_param->picture_height_in_mbs_minus1 = priv->mb_height - 1;
2542     pic_param->frame_num                    = priv->frame_num;
2543
2544     COPY_FIELD(sps, bit_depth_luma_minus8);
2545     COPY_FIELD(sps, bit_depth_chroma_minus8);
2546     COPY_FIELD(sps, num_ref_frames);
2547     COPY_FIELD(pps, num_slice_groups_minus1);
2548     COPY_FIELD(pps, slice_group_map_type);
2549     COPY_FIELD(pps, slice_group_change_rate_minus1);
2550     COPY_FIELD(pps, pic_init_qp_minus26);
2551     COPY_FIELD(pps, pic_init_qs_minus26);
2552     COPY_FIELD(pps, chroma_qp_index_offset);
2553     COPY_FIELD(pps, second_chroma_qp_index_offset);
2554
2555     pic_param->seq_fields.value                                         = 0; /* reset all bits */
2556     pic_param->seq_fields.bits.residual_colour_transform_flag           = sps->separate_colour_plane_flag;
2557     pic_param->seq_fields.bits.MinLumaBiPredSize8x8                     = sps->level_idc >= 31; /* A.3.3.2 */
2558
2559     COPY_BFM(seq_fields, sps, chroma_format_idc);
2560     COPY_BFM(seq_fields, sps, gaps_in_frame_num_value_allowed_flag);
2561     COPY_BFM(seq_fields, sps, frame_mbs_only_flag);
2562     COPY_BFM(seq_fields, sps, mb_adaptive_frame_field_flag);
2563     COPY_BFM(seq_fields, sps, direct_8x8_inference_flag);
2564     COPY_BFM(seq_fields, sps, log2_max_frame_num_minus4);
2565     COPY_BFM(seq_fields, sps, pic_order_cnt_type);
2566     COPY_BFM(seq_fields, sps, log2_max_pic_order_cnt_lsb_minus4);
2567     COPY_BFM(seq_fields, sps, delta_pic_order_always_zero_flag);
2568
2569     pic_param->pic_fields.value                                         = 0; /* reset all bits */
2570     pic_param->pic_fields.bits.field_pic_flag                           = slice_hdr->field_pic_flag;
2571     pic_param->pic_fields.bits.reference_pic_flag                       = GST_VAAPI_PICTURE_IS_REFERENCE(picture);
2572
2573     COPY_BFM(pic_fields, pps, entropy_coding_mode_flag);
2574     COPY_BFM(pic_fields, pps, weighted_pred_flag);
2575     COPY_BFM(pic_fields, pps, weighted_bipred_idc);
2576     COPY_BFM(pic_fields, pps, transform_8x8_mode_flag);
2577     COPY_BFM(pic_fields, pps, constrained_intra_pred_flag);
2578     COPY_BFM(pic_fields, pps, pic_order_present_flag);
2579     COPY_BFM(pic_fields, pps, deblocking_filter_control_present_flag);
2580     COPY_BFM(pic_fields, pps, redundant_pic_cnt_present_flag);
2581     return TRUE;
2582 }
2583
2584 /* Detection of the first VCL NAL unit of a primary coded picture (7.4.1.2.4) */
2585 static gboolean
2586 is_new_picture(
2587     GstVaapiDecoderH264 *decoder,
2588     GstH264NalUnit      *nalu,
2589     GstH264SliceHdr     *slice_hdr
2590 )
2591 {
2592     GstVaapiDecoderH264Private * const priv = decoder->priv;
2593     GstH264PPS * const pps = slice_hdr->pps;
2594     GstH264SPS * const sps = pps->sequence;
2595     GstVaapiSliceH264 *slice;
2596     GstH264SliceHdr *prev_slice_hdr;
2597
2598     if (!priv->current_picture)
2599         return TRUE;
2600
2601     slice = gst_vaapi_picture_h264_get_last_slice(priv->current_picture);
2602     if (!slice)
2603         return FALSE;
2604     prev_slice_hdr = &slice->slice_hdr;
2605
2606 #define CHECK_EXPR(expr, field_name) do {              \
2607         if (!(expr)) {                                 \
2608             GST_DEBUG(field_name " differs in value"); \
2609             return TRUE;                               \
2610         }                                              \
2611     } while (0)
2612
2613 #define CHECK_VALUE(new_slice_hdr, old_slice_hdr, field) \
2614     CHECK_EXPR(((new_slice_hdr)->field == (old_slice_hdr)->field), #field)
2615
2616     /* frame_num differs in value, regardless of inferred values to 0 */
2617     CHECK_VALUE(slice_hdr, prev_slice_hdr, frame_num);
2618
2619     /* pic_parameter_set_id differs in value */
2620     CHECK_VALUE(slice_hdr, prev_slice_hdr, pps);
2621
2622     /* field_pic_flag differs in value */
2623     CHECK_VALUE(slice_hdr, prev_slice_hdr, field_pic_flag);
2624
2625     /* bottom_field_flag is present in both and differs in value */
2626     if (slice_hdr->field_pic_flag && prev_slice_hdr->field_pic_flag)
2627         CHECK_VALUE(slice_hdr, prev_slice_hdr, bottom_field_flag);
2628
2629     /* nal_ref_idc differs in value with one of the nal_ref_idc values is 0 */
2630     CHECK_EXPR(((GST_VAAPI_PICTURE_IS_REFERENCE(priv->current_picture) ^
2631                  (nalu->ref_idc != 0)) == 0), "nal_ref_idc");
2632
2633     /* POC type is 0 for both and either pic_order_cnt_lsb differs in
2634        value or delta_pic_order_cnt_bottom differs in value */
2635     if (sps->pic_order_cnt_type == 0) {
2636         CHECK_VALUE(slice_hdr, prev_slice_hdr, pic_order_cnt_lsb);
2637         if (pps->pic_order_present_flag && !slice_hdr->field_pic_flag)
2638             CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt_bottom);
2639     }
2640
2641     /* POC type is 1 for both and either delta_pic_order_cnt[0]
2642        differs in value or delta_pic_order_cnt[1] differs in value */
2643     else if (sps->pic_order_cnt_type == 1) {
2644         CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt[0]);
2645         CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt[1]);
2646     }
2647
2648     /* IdrPicFlag differs in value */
2649     CHECK_EXPR(((GST_VAAPI_PICTURE_IS_IDR(priv->current_picture) ^
2650                  (nalu->type == GST_H264_NAL_SLICE_IDR)) == 0), "IdrPicFlag");
2651
2652     /* IdrPicFlag is equal to 1 for both and idr_pic_id differs in value */
2653     if (GST_VAAPI_PICTURE_IS_IDR(priv->current_picture))
2654         CHECK_VALUE(slice_hdr, prev_slice_hdr, idr_pic_id);
2655
2656 #undef CHECK_EXPR
2657 #undef CHECK_VALUE
2658     return FALSE;
2659 }
2660
2661 static GstVaapiDecoderStatus
2662 decode_picture(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu, GstH264SliceHdr *slice_hdr)
2663 {
2664     GstVaapiDecoderH264Private * const priv = decoder->priv;
2665     GstVaapiPictureH264 *picture;
2666     GstVaapiDecoderStatus status;
2667     GstH264PPS * const pps = slice_hdr->pps;
2668     GstH264SPS * const sps = pps->sequence;
2669
2670     status = decode_current_picture(decoder);
2671     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2672         return status;
2673
2674     status = ensure_context(decoder, sps);
2675     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2676         return status;
2677
2678     if (priv->current_picture) {
2679         /* Re-use current picture where the first field was decoded */
2680         picture = gst_vaapi_picture_h264_new_field(priv->current_picture);
2681         if (!picture) {
2682             GST_ERROR("failed to allocate field picture");
2683             return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2684         }
2685     }
2686     else {
2687         /* Create new picture */
2688         picture = gst_vaapi_picture_h264_new(decoder);
2689         if (!picture) {
2690             GST_ERROR("failed to allocate picture");
2691             return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2692         }
2693     }
2694     gst_vaapi_picture_replace(&priv->current_picture, picture);
2695     gst_vaapi_picture_unref(picture);
2696
2697     picture->pps = pps;
2698
2699     status = ensure_quant_matrix(decoder, picture);
2700     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS) {
2701         GST_ERROR("failed to reset quantizer matrix");
2702         return status;
2703     }
2704
2705     if (!init_picture(decoder, picture, slice_hdr, nalu))
2706         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2707     if (!fill_picture(decoder, picture, slice_hdr, nalu))
2708         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2709     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2710 }
2711
2712 static inline guint
2713 get_slice_data_bit_offset(GstH264SliceHdr *slice_hdr, GstH264NalUnit *nalu)
2714 {
2715     guint epb_count;
2716
2717     epb_count = slice_hdr->n_emulation_prevention_bytes;
2718     return 8 /* nal_unit_type */ + slice_hdr->header_size - epb_count * 8;
2719 }
2720
2721 static gboolean
2722 fill_pred_weight_table(GstVaapiDecoderH264 *decoder, GstVaapiSliceH264 *slice)
2723 {
2724     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2725     GstH264PPS * const pps = slice_hdr->pps;
2726     GstH264SPS * const sps = pps->sequence;
2727     GstH264PredWeightTable * const w = &slice_hdr->pred_weight_table;
2728     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2729     guint num_weight_tables = 0;
2730     gint i, j;
2731
2732     if (pps->weighted_pred_flag &&
2733         (GST_H264_IS_P_SLICE(slice_hdr) || GST_H264_IS_SP_SLICE(slice_hdr)))
2734         num_weight_tables = 1;
2735     else if (pps->weighted_bipred_idc == 1 && GST_H264_IS_B_SLICE(slice_hdr))
2736         num_weight_tables = 2;
2737     else
2738         num_weight_tables = 0;
2739
2740     slice_param->luma_log2_weight_denom   = w->luma_log2_weight_denom;
2741     slice_param->chroma_log2_weight_denom = w->chroma_log2_weight_denom;
2742     slice_param->luma_weight_l0_flag      = 0;
2743     slice_param->chroma_weight_l0_flag    = 0;
2744     slice_param->luma_weight_l1_flag      = 0;
2745     slice_param->chroma_weight_l1_flag    = 0;
2746
2747     if (num_weight_tables < 1)
2748         return TRUE;
2749
2750     slice_param->luma_weight_l0_flag = 1;
2751     for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2752         slice_param->luma_weight_l0[i] = w->luma_weight_l0[i];
2753         slice_param->luma_offset_l0[i] = w->luma_offset_l0[i];
2754     }
2755
2756     slice_param->chroma_weight_l0_flag = sps->chroma_array_type != 0;
2757     if (slice_param->chroma_weight_l0_flag) {
2758         for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2759             for (j = 0; j < 2; j++) {
2760                 slice_param->chroma_weight_l0[i][j] = w->chroma_weight_l0[i][j];
2761                 slice_param->chroma_offset_l0[i][j] = w->chroma_offset_l0[i][j];
2762             }
2763         }
2764     }
2765
2766     if (num_weight_tables < 2)
2767         return TRUE;
2768
2769     slice_param->luma_weight_l1_flag = 1;
2770     for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2771         slice_param->luma_weight_l1[i] = w->luma_weight_l1[i];
2772         slice_param->luma_offset_l1[i] = w->luma_offset_l1[i];
2773     }
2774
2775     slice_param->chroma_weight_l1_flag = sps->chroma_array_type != 0;
2776     if (slice_param->chroma_weight_l1_flag) {
2777         for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2778             for (j = 0; j < 2; j++) {
2779                 slice_param->chroma_weight_l1[i][j] = w->chroma_weight_l1[i][j];
2780                 slice_param->chroma_offset_l1[i][j] = w->chroma_offset_l1[i][j];
2781             }
2782         }
2783     }
2784     return TRUE;
2785 }
2786
2787 static gboolean
2788 fill_RefPicList(GstVaapiDecoderH264 *decoder, GstVaapiSliceH264 *slice)
2789 {
2790     GstVaapiDecoderH264Private * const priv = decoder->priv;
2791     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2792     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2793     guint i, num_ref_lists = 0;
2794
2795     slice_param->num_ref_idx_l0_active_minus1 = 0;
2796     slice_param->num_ref_idx_l1_active_minus1 = 0;
2797
2798     if (GST_H264_IS_B_SLICE(slice_hdr))
2799         num_ref_lists = 2;
2800     else if (GST_H264_IS_I_SLICE(slice_hdr))
2801         num_ref_lists = 0;
2802     else
2803         num_ref_lists = 1;
2804
2805     if (num_ref_lists < 1)
2806         return TRUE;
2807
2808     slice_param->num_ref_idx_l0_active_minus1 =
2809         slice_hdr->num_ref_idx_l0_active_minus1;
2810
2811     for (i = 0; i < priv->RefPicList0_count && priv->RefPicList0[i]; i++)
2812         vaapi_fill_picture(&slice_param->RefPicList0[i], priv->RefPicList0[i], 0);
2813     for (; i <= slice_param->num_ref_idx_l0_active_minus1; i++)
2814         vaapi_init_picture(&slice_param->RefPicList0[i]);
2815
2816     if (num_ref_lists < 2)
2817         return TRUE;
2818
2819     slice_param->num_ref_idx_l1_active_minus1 =
2820         slice_hdr->num_ref_idx_l1_active_minus1;
2821
2822     for (i = 0; i < priv->RefPicList1_count && priv->RefPicList1[i]; i++)
2823         vaapi_fill_picture(&slice_param->RefPicList1[i], priv->RefPicList1[i], 0);
2824     for (; i <= slice_param->num_ref_idx_l1_active_minus1; i++)
2825         vaapi_init_picture(&slice_param->RefPicList1[i]);
2826     return TRUE;
2827 }
2828
2829 static gboolean
2830 fill_slice(
2831     GstVaapiDecoderH264 *decoder,
2832     GstVaapiSliceH264   *slice,
2833     GstH264NalUnit      *nalu
2834 )
2835 {
2836     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2837     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2838
2839     /* Fill in VASliceParameterBufferH264 */
2840     slice_param->slice_data_bit_offset          = get_slice_data_bit_offset(slice_hdr, nalu);
2841     slice_param->first_mb_in_slice              = slice_hdr->first_mb_in_slice;
2842     slice_param->slice_type                     = slice_hdr->type % 5;
2843     slice_param->direct_spatial_mv_pred_flag    = slice_hdr->direct_spatial_mv_pred_flag;
2844     slice_param->cabac_init_idc                 = slice_hdr->cabac_init_idc;
2845     slice_param->slice_qp_delta                 = slice_hdr->slice_qp_delta;
2846     slice_param->disable_deblocking_filter_idc  = slice_hdr->disable_deblocking_filter_idc;
2847     slice_param->slice_alpha_c0_offset_div2     = slice_hdr->slice_alpha_c0_offset_div2;
2848     slice_param->slice_beta_offset_div2         = slice_hdr->slice_beta_offset_div2;
2849
2850     if (!fill_RefPicList(decoder, slice))
2851         return FALSE;
2852     if (!fill_pred_weight_table(decoder, slice))
2853         return FALSE;
2854     return TRUE;
2855 }
2856
2857 static GstVaapiDecoderStatus
2858 decode_slice(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
2859 {
2860     GstVaapiDecoderH264Private * const priv = decoder->priv;
2861     GstVaapiDecoderStatus status;
2862     GstVaapiPictureH264 *picture;
2863     GstVaapiSliceH264 *slice = NULL;
2864     GstH264SliceHdr *slice_hdr;
2865     GstH264ParserResult result;
2866
2867     GST_DEBUG("slice (%u bytes)", nalu->size);
2868
2869     slice = gst_vaapi_slice_h264_new(
2870         decoder,
2871         nalu->data + nalu->offset,
2872         nalu->size
2873     );
2874     if (!slice) {
2875         GST_ERROR("failed to allocate slice");
2876         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2877     }
2878
2879     slice_hdr = &slice->slice_hdr;
2880     memset(slice_hdr, 0, sizeof(*slice_hdr));
2881     result = gst_h264_parser_parse_slice_hdr(priv->parser, nalu, slice_hdr, TRUE, TRUE);
2882     if (result != GST_H264_PARSER_OK) {
2883         status = get_status(result);
2884         goto error;
2885     }
2886
2887     if (is_new_picture(decoder, nalu, slice_hdr)) {
2888         status = decode_picture(decoder, nalu, slice_hdr);
2889         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2890             goto error;
2891     }
2892     picture = priv->current_picture;
2893
2894     if (!fill_slice(decoder, slice, nalu)) {
2895         status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2896         goto error;
2897     }
2898     gst_vaapi_picture_add_slice(
2899         GST_VAAPI_PICTURE_CAST(picture),
2900         GST_VAAPI_SLICE_CAST(slice)
2901     );
2902     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2903
2904 error:
2905     if (slice)
2906         gst_mini_object_unref(GST_MINI_OBJECT(slice));
2907     return status;
2908 }
2909
2910 static inline gint
2911 scan_for_start_code(GstAdapter *adapter, guint ofs, guint size, guint32 *scp)
2912 {
2913     return (gint)gst_adapter_masked_scan_uint32_peek(adapter,
2914                                                      0xffffff00, 0x00000100,
2915                                                      ofs, size,
2916                                                      scp);
2917 }
2918
2919 static GstVaapiDecoderStatus
2920 decode_nalu(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
2921 {
2922     GstVaapiDecoderH264Private * const priv = decoder->priv;
2923     GstVaapiDecoderStatus status;
2924
2925     switch (nalu->type) {
2926     case GST_H264_NAL_SLICE_IDR:
2927         /* fall-through. IDR specifics are handled in init_picture() */
2928     case GST_H264_NAL_SLICE:
2929         if (!priv->got_sps || !priv->got_pps)
2930             return GST_VAAPI_DECODER_STATUS_SUCCESS;
2931         status = decode_slice(decoder, nalu);
2932         break;
2933     case GST_H264_NAL_SPS:
2934         status = decode_sps(decoder, nalu);
2935         break;
2936     case GST_H264_NAL_PPS:
2937         status = decode_pps(decoder, nalu);
2938         break;
2939     case GST_H264_NAL_SEI:
2940         status = decode_sei(decoder, nalu);
2941         break;
2942     case GST_H264_NAL_SEQ_END:
2943         status = decode_sequence_end(decoder);
2944         break;
2945     case GST_H264_NAL_AU_DELIMITER:
2946         /* skip all Access Unit NALs */
2947         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2948         break;
2949     case GST_H264_NAL_FILLER_DATA:
2950         /* skip all Filler Data NALs */
2951         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2952         break;
2953     default:
2954         GST_WARNING("unsupported NAL unit type %d", nalu->type);
2955         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
2956         break;
2957     }
2958     return status;
2959 }
2960
2961 static GstVaapiDecoderStatus
2962 decode_buffer(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
2963 {
2964     GstVaapiDecoderH264Private * const priv = decoder->priv;
2965     GstVaapiDecoderStatus status;
2966     GstH264ParserResult result;
2967     GstH264NalUnit nalu;
2968     gboolean is_eos;
2969     const guchar *buf;
2970     guint i, buf_size, nalu_size, size;
2971     guint32 start_code;
2972     gint ofs;
2973
2974     buf      = GST_BUFFER_DATA(buffer);
2975     buf_size = GST_BUFFER_SIZE(buffer);
2976     is_eos   = GST_BUFFER_IS_EOS(buffer);
2977     if (buf && buf_size > 0)
2978         gst_adapter_push(priv->adapter, gst_buffer_ref(buffer));
2979
2980     size = gst_adapter_available(priv->adapter);
2981     do {
2982         if (size == 0) {
2983             status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2984             break;
2985         }
2986
2987         status = gst_vaapi_decoder_check_status(GST_VAAPI_DECODER(decoder));
2988         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2989             break;
2990
2991         status = GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2992         if (priv->is_avc) {
2993             if (size < priv->nal_length_size)
2994                 break;
2995             buf = gst_adapter_peek(priv->adapter, priv->nal_length_size);
2996
2997             nalu_size = 0;
2998             for (i = 0; i < priv->nal_length_size; i++)
2999                 nalu_size = (nalu_size << 8) | buf[i];
3000
3001             buf_size = priv->nal_length_size + nalu_size;
3002             if (size < buf_size)
3003                 break;
3004             buffer = gst_adapter_take_buffer(priv->adapter, buf_size);
3005             size -= buf_size;
3006
3007             buf      = GST_BUFFER_DATA(buffer);
3008             buf_size = GST_BUFFER_SIZE(buffer);
3009
3010             result = gst_h264_parser_identify_nalu_avc(
3011                 priv->parser,
3012                 buf, 0, buf_size, priv->nal_length_size,
3013                 &nalu
3014             );
3015         }
3016         else {
3017             if (size < 4)
3018                 break;
3019             ofs = scan_for_start_code(priv->adapter, 0, size, &start_code);
3020             if (ofs < 0)
3021                 break;
3022             gst_adapter_flush(priv->adapter, ofs);
3023             size -= ofs;
3024
3025             ofs = G_UNLIKELY(size < 8) ? -1 :
3026                 scan_for_start_code(priv->adapter, 4, size - 4, NULL);
3027             if (ofs < 0) {
3028                 // Assume the whole NAL unit is present if end-of-stream
3029                 if (!is_eos)
3030                     break;
3031                 ofs = size;
3032             }
3033             buffer = gst_adapter_take_buffer(priv->adapter, ofs);
3034             size -= ofs;
3035
3036             buf      = GST_BUFFER_DATA(buffer);
3037             buf_size = GST_BUFFER_SIZE(buffer);
3038
3039             result = gst_h264_parser_identify_nalu_unchecked(
3040                 priv->parser,
3041                 buf, 0, buf_size,
3042                 &nalu
3043             );
3044         }
3045         status = get_status(result);
3046         if (status == GST_VAAPI_DECODER_STATUS_SUCCESS)
3047             status = decode_nalu(decoder, &nalu);
3048         gst_buffer_unref(buffer);
3049     } while (status == GST_VAAPI_DECODER_STATUS_SUCCESS);
3050
3051     if (is_eos && (status == GST_VAAPI_DECODER_STATUS_SUCCESS ||
3052                    status == GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA))
3053         status = decode_sequence_end(decoder);
3054     return status;
3055 }
3056
3057 static GstVaapiDecoderStatus
3058 decode_codec_data(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
3059 {
3060     GstVaapiDecoderH264Private * const priv = decoder->priv;
3061     GstVaapiDecoderStatus status;
3062     GstH264NalUnit nalu;
3063     GstH264ParserResult result;
3064     guchar *buf;
3065     guint buf_size;
3066     guint i, ofs, num_sps, num_pps;
3067
3068     buf      = GST_BUFFER_DATA(buffer);
3069     buf_size = GST_BUFFER_SIZE(buffer);
3070     if (!buf || buf_size == 0)
3071         return GST_VAAPI_DECODER_STATUS_SUCCESS;
3072
3073     if (buf_size < 8)
3074         return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
3075
3076     if (buf[0] != 1) {
3077         GST_ERROR("failed to decode codec-data, not in avcC format");
3078         return GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
3079     }
3080
3081     priv->nal_length_size = (buf[4] & 0x03) + 1;
3082
3083     num_sps = buf[5] & 0x1f;
3084     ofs = 6;
3085
3086     for (i = 0; i < num_sps; i++) {
3087         result = gst_h264_parser_identify_nalu_avc(
3088             priv->parser,
3089             buf, ofs, buf_size, 2,
3090             &nalu
3091         );
3092         if (result != GST_H264_PARSER_OK)
3093             return get_status(result);
3094
3095         status = decode_sps(decoder, &nalu);
3096         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
3097             return status;
3098         ofs = nalu.offset + nalu.size;
3099     }
3100
3101     num_pps = buf[ofs];
3102     ofs++;
3103
3104     for (i = 0; i < num_pps; i++) {
3105         result = gst_h264_parser_identify_nalu_avc(
3106             priv->parser,
3107             buf, ofs, buf_size, 2,
3108             &nalu
3109         );
3110         if (result != GST_H264_PARSER_OK)
3111             return get_status(result);
3112
3113         status = decode_pps(decoder, &nalu);
3114         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
3115             return status;
3116         ofs = nalu.offset + nalu.size;
3117     }
3118
3119     priv->is_avc = TRUE;
3120     return status;
3121 }
3122
3123 GstVaapiDecoderStatus
3124 gst_vaapi_decoder_h264_decode(GstVaapiDecoder *base, GstBuffer *buffer)
3125 {
3126     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264(base);
3127     GstVaapiDecoderH264Private * const priv = decoder->priv;
3128     GstVaapiDecoderStatus status;
3129     GstBuffer *codec_data;
3130
3131     g_return_val_if_fail(priv->is_constructed,
3132                          GST_VAAPI_DECODER_STATUS_ERROR_INIT_FAILED);
3133
3134     if (!priv->is_opened) {
3135         priv->is_opened = gst_vaapi_decoder_h264_open(decoder, buffer);
3136         if (!priv->is_opened)
3137             return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CODEC;
3138
3139         codec_data = GST_VAAPI_DECODER_CODEC_DATA(decoder);
3140         if (codec_data) {
3141             status = decode_codec_data(decoder, codec_data);
3142             if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
3143                 return status;
3144         }
3145      }
3146      return decode_buffer(decoder, buffer);
3147 }
3148
3149 static void
3150 gst_vaapi_decoder_h264_finalize(GObject *object)
3151 {
3152     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264(object);
3153
3154     gst_vaapi_decoder_h264_destroy(decoder);
3155
3156     G_OBJECT_CLASS(gst_vaapi_decoder_h264_parent_class)->finalize(object);
3157 }
3158
3159 static void
3160 gst_vaapi_decoder_h264_constructed(GObject *object)
3161 {
3162     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264(object);
3163     GstVaapiDecoderH264Private * const priv = decoder->priv;
3164     GObjectClass *parent_class;
3165
3166     parent_class = G_OBJECT_CLASS(gst_vaapi_decoder_h264_parent_class);
3167     if (parent_class->constructed)
3168         parent_class->constructed(object);
3169
3170     priv->is_constructed = gst_vaapi_decoder_h264_create(decoder);
3171 }
3172
3173 static void
3174 gst_vaapi_decoder_h264_class_init(GstVaapiDecoderH264Class *klass)
3175 {
3176     GObjectClass * const object_class = G_OBJECT_CLASS(klass);
3177     GstVaapiDecoderClass * const decoder_class = GST_VAAPI_DECODER_CLASS(klass);
3178
3179     g_type_class_add_private(klass, sizeof(GstVaapiDecoderH264Private));
3180
3181     object_class->finalize      = gst_vaapi_decoder_h264_finalize;
3182     object_class->constructed   = gst_vaapi_decoder_h264_constructed;
3183
3184     decoder_class->decode       = gst_vaapi_decoder_h264_decode;
3185 }
3186
3187 static void
3188 gst_vaapi_decoder_h264_init(GstVaapiDecoderH264 *decoder)
3189 {
3190     GstVaapiDecoderH264Private *priv;
3191
3192     priv                        = GST_VAAPI_DECODER_H264_GET_PRIVATE(decoder);
3193     decoder->priv               = priv;
3194     priv->parser                = NULL;
3195     priv->current_picture       = NULL;
3196     priv->dpb_count             = 0;
3197     priv->dpb_size              = 0;
3198     priv->profile               = GST_VAAPI_PROFILE_UNKNOWN;
3199     priv->entrypoint            = GST_VAAPI_ENTRYPOINT_VLD;
3200     priv->chroma_type           = GST_VAAPI_CHROMA_TYPE_YUV420;
3201     priv->short_ref_count       = 0;
3202     priv->long_ref_count        = 0;
3203     priv->RefPicList0_count     = 0;
3204     priv->RefPicList1_count     = 0;
3205     priv->nal_length_size       = 0;
3206     priv->adapter               = NULL;
3207     priv->field_poc[0]          = 0;
3208     priv->field_poc[1]          = 0;
3209     priv->poc_msb               = 0;
3210     priv->poc_lsb               = 0;
3211     priv->prev_poc_msb          = 0;
3212     priv->prev_poc_lsb          = 0;
3213     priv->frame_num_offset      = 0;
3214     priv->frame_num             = 0;
3215     priv->prev_frame_num        = 0;
3216     priv->prev_pic_has_mmco5    = FALSE;
3217     priv->prev_pic_structure    = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
3218     priv->is_constructed        = FALSE;
3219     priv->is_opened             = FALSE;
3220     priv->is_avc                = FALSE;
3221     priv->has_context           = FALSE;
3222     priv->progressive_sequence  = TRUE;
3223
3224     memset(priv->dpb, 0, sizeof(priv->dpb));
3225     memset(priv->short_ref, 0, sizeof(priv->short_ref));
3226     memset(priv->long_ref, 0, sizeof(priv->long_ref));
3227     memset(priv->RefPicList0, 0, sizeof(priv->RefPicList0));
3228     memset(priv->RefPicList1, 0, sizeof(priv->RefPicList1));
3229 }
3230
3231 /**
3232  * gst_vaapi_decoder_h264_new:
3233  * @display: a #GstVaapiDisplay
3234  * @caps: a #GstCaps holding codec information
3235  *
3236  * Creates a new #GstVaapiDecoder for MPEG-2 decoding.  The @caps can
3237  * hold extra information like codec-data and pictured coded size.
3238  *
3239  * Return value: the newly allocated #GstVaapiDecoder object
3240  */
3241 GstVaapiDecoder *
3242 gst_vaapi_decoder_h264_new(GstVaapiDisplay *display, GstCaps *caps)
3243 {
3244     GstVaapiDecoderH264 *decoder;
3245
3246     g_return_val_if_fail(GST_VAAPI_IS_DISPLAY(display), NULL);
3247     g_return_val_if_fail(GST_IS_CAPS(caps), NULL);
3248
3249     decoder = g_object_new(
3250         GST_VAAPI_TYPE_DECODER_H264,
3251         "display",      display,
3252         "caps",         caps,
3253         NULL
3254     );
3255     if (!decoder->priv->is_constructed) {
3256         g_object_unref(decoder);
3257         return NULL;
3258     }
3259     return GST_VAAPI_DECODER_CAST(decoder);
3260 }