gst-libs/gst/vaapi/gstvaapidecoder_h264.c

   1 /*
   2  *  gstvaapidecoder_h264.c - H.264 decoder
   3  *
   4  *  Copyright (C) 2011-2012 Intel Corporation
   5  *
   6  *  This library is free software; you can redistribute it and/or
   7  *  modify it under the terms of the GNU Lesser General Public License
   8  *  as published by the Free Software Foundation; either version 2.1
   9  *  of the License, or (at your option) any later version.
  10  *
  11  *  This library is distributed in the hope that it will be useful,
  12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  *  Lesser General Public License for more details.
  15  *
  16  *  You should have received a copy of the GNU Lesser General Public
  17  *  License along with this library; if not, write to the Free
  18  *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  19  *  Boston, MA 02110-1301 USA
  20  */
  21
  22 /**
  23  * SECTION:gstvaapidecoder_h264
  24  * @short_description: H.264 decoder
  25  */
  26
  27 #include "sysdeps.h"
  28 #include <string.h>
  29 #include <stdlib.h>
  30 #include <gst/base/gstadapter.h>
  31 #include <gst/codecparsers/gsth264parser.h>
  32 #include "gstvaapidecoder_h264.h"
  33 #include "gstvaapidecoder_objects.h"
  34 #include "gstvaapidecoder_priv.h"
  35 #include "gstvaapidisplay_priv.h"
  36 #include "gstvaapiobject_priv.h"
  37
  38 #define DEBUG 1
  39 #include "gstvaapidebug.h"
  40
  41 /* Defined to 1 if strict ordering of DPB is needed. Only useful for debug */
  42 #define USE_STRICT_DPB_ORDERING 0
  43
  44 typedef struct _GstVaapiFrameStore              GstVaapiFrameStore;
  45 typedef struct _GstVaapiFrameStoreClass         GstVaapiFrameStoreClass;
  46 typedef struct _GstVaapiPictureH264             GstVaapiPictureH264;
  47 typedef struct _GstVaapiPictureH264Class        GstVaapiPictureH264Class;
  48 typedef struct _GstVaapiSliceH264               GstVaapiSliceH264;
  49 typedef struct _GstVaapiSliceH264Class          GstVaapiSliceH264Class;
  50
  51 // Used for field_poc[]
  52 #define TOP_FIELD       0
  53 #define BOTTOM_FIELD    1
  54
  55 /* ------------------------------------------------------------------------- */
  56 /* --- H.264 Pictures                                                    --- */
  57 /* ------------------------------------------------------------------------- */
  58
  59 #define GST_VAAPI_TYPE_PICTURE_H264 \
  60     (gst_vaapi_picture_h264_get_type())
  61
  62 #define GST_VAAPI_PICTURE_H264_CAST(obj) \
  63     ((GstVaapiPictureH264 *)(obj))
  64
  65 #define GST_VAAPI_PICTURE_H264(obj)                             \
  66     (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
  67                                 GST_VAAPI_TYPE_PICTURE_H264,    \
  68                                 GstVaapiPictureH264))
  69
  70 #define GST_VAAPI_PICTURE_H264_CLASS(klass)                     \
  71     (G_TYPE_CHECK_CLASS_CAST((klass),                           \
  72                              GST_VAAPI_TYPE_PICTURE_H264,       \
  73                              GstVaapiPictureH264Class))
  74
  75 #define GST_VAAPI_IS_PICTURE_H264(obj) \
  76     (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_VAAPI_TYPE_PICTURE_H264))
  77
  78 #define GST_VAAPI_IS_PICTURE_H264_CLASS(klass) \
  79     (G_TYPE_CHECK_CLASS_TYPE((klass), GST_VAAPI_TYPE_PICTURE_H264))
  80
  81 #define GST_VAAPI_PICTURE_H264_GET_CLASS(obj)                   \
  82     (G_TYPE_INSTANCE_GET_CLASS((obj),                           \
  83                                GST_VAAPI_TYPE_PICTURE_H264,     \
  84                                GstVaapiPictureH264Class))
  85
  86 /*
  87  * Extended picture flags:
  88  *
  89  * @GST_VAAPI_PICTURE_FLAG_IDR: flag that specifies an IDR picture
  90  * @GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE: flag that specifies
  91  *     "used for short-term reference"
  92  * @GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE: flag that specifies
  93  *     "used for long-term reference"
  94  * @GST_VAAPI_PICTURE_FLAGS_REFERENCE: mask covering any kind of
  95  *     reference picture (short-term reference or long-term reference)
  96  */
  97 enum {
  98     GST_VAAPI_PICTURE_FLAG_IDR = (GST_VAAPI_PICTURE_FLAG_LAST << 0),
  99
 100     GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE = (
 101         GST_VAAPI_PICTURE_FLAG_REFERENCE),
 102     GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE = (
 103         GST_VAAPI_PICTURE_FLAG_REFERENCE | (GST_VAAPI_PICTURE_FLAG_LAST << 1)),
 104     GST_VAAPI_PICTURE_FLAGS_REFERENCE = (
 105         GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE |
 106         GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE),
 107 };
 108
 109 #define GST_VAAPI_PICTURE_IS_IDR(picture) \
 110     (GST_VAAPI_PICTURE_FLAG_IS_SET(picture, GST_VAAPI_PICTURE_FLAG_IDR))
 111
 112 #define GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture)      \
 113     ((GST_VAAPI_PICTURE_FLAGS(picture) &                        \
 114       GST_VAAPI_PICTURE_FLAGS_REFERENCE) ==                     \
 115      GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE)
 116
 117 #define GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture)       \
 118     ((GST_VAAPI_PICTURE_FLAGS(picture) &                        \
 119       GST_VAAPI_PICTURE_FLAGS_REFERENCE) ==                     \
 120      GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE)
 121
 122 struct _GstVaapiPictureH264 {
 123     GstVaapiPicture             base;
 124     GstH264PPS                 *pps;
 125     guint                       structure;
 126     gint32                      field_poc[2];
 127     gint32                      frame_num;              // Original frame_num from slice_header()
 128     gint32                      frame_num_wrap;         // Temporary for ref pic marking: FrameNumWrap
 129     gint32                      long_term_frame_idx;    // Temporary for ref pic marking: LongTermFrameIdx
 130     gint32                      pic_num;                // Temporary for ref pic marking: PicNum
 131     gint32                      long_term_pic_num;      // Temporary for ref pic marking: LongTermPicNum
 132     GstVaapiPictureH264        *other_field;            // Temporary for ref pic marking: other field in the same frame store
 133     guint                       output_flag             : 1;
 134     guint                       output_needed           : 1;
 135 };
 136
 137 struct _GstVaapiPictureH264Class {
 138     /*< private >*/
 139     GstVaapiPictureClass        parent_class;
 140 };
 141
 142 GST_VAAPI_CODEC_DEFINE_TYPE(GstVaapiPictureH264,
 143                             gst_vaapi_picture_h264,
 144                             GST_VAAPI_TYPE_PICTURE)
 145
 146 static void
 147 gst_vaapi_picture_h264_destroy(GstVaapiPictureH264 *decoder)
 148 {
 149 }
 150
 151 static gboolean
 152 gst_vaapi_picture_h264_create(
 153     GstVaapiPictureH264                      *picture,
 154     const GstVaapiCodecObjectConstructorArgs *args
 155 )
 156 {
 157     return TRUE;
 158 }
 159
 160 static void
 161 gst_vaapi_picture_h264_init(GstVaapiPictureH264 *picture)
 162 {
 163     picture->field_poc[0]       = G_MAXINT32;
 164     picture->field_poc[1]       = G_MAXINT32;
 165     picture->output_needed      = FALSE;
 166 }
 167
 168 static inline GstVaapiPictureH264 *
 169 gst_vaapi_picture_h264_new(GstVaapiDecoderH264 *decoder)
 170 {
 171     GstVaapiCodecObject *object;
 172
 173     g_return_val_if_fail(GST_VAAPI_IS_DECODER(decoder), NULL);
 174
 175     object = gst_vaapi_codec_object_new(
 176         GST_VAAPI_TYPE_PICTURE_H264,
 177         GST_VAAPI_CODEC_BASE(decoder),
 178         NULL, sizeof(VAPictureParameterBufferH264),
 179         NULL, 0
 180     );
 181     if (!object)
 182         return NULL;
 183     return GST_VAAPI_PICTURE_H264_CAST(object);
 184 }
 185
 186 static inline void
 187 gst_vaapi_picture_h264_set_reference(
 188     GstVaapiPictureH264 *picture,
 189     guint                reference_flags,
 190     gboolean             other_field
 191 )
 192 {
 193     g_return_if_fail(GST_VAAPI_IS_PICTURE_H264(picture));
 194
 195     GST_VAAPI_PICTURE_FLAG_UNSET(picture, GST_VAAPI_PICTURE_FLAGS_REFERENCE);
 196     GST_VAAPI_PICTURE_FLAG_SET(picture, reference_flags);
 197
 198     if (!other_field || !(picture = picture->other_field))
 199         return;
 200     GST_VAAPI_PICTURE_FLAG_UNSET(picture, GST_VAAPI_PICTURE_FLAGS_REFERENCE);
 201     GST_VAAPI_PICTURE_FLAG_SET(picture, reference_flags);
 202 }
 203
 204 static inline GstVaapiPictureH264 *
 205 gst_vaapi_picture_h264_new_field(GstVaapiPictureH264 *picture)
 206 {
 207     GstVaapiPicture *base_picture;
 208
 209     g_return_val_if_fail(GST_VAAPI_IS_PICTURE_H264(picture), NULL);
 210
 211     base_picture = gst_vaapi_picture_new_field(&picture->base);
 212     if (!base_picture)
 213         return NULL;
 214     return GST_VAAPI_PICTURE_H264_CAST(base_picture);
 215 }
 216
 217 static inline GstVaapiSliceH264 *
 218 gst_vaapi_picture_h264_get_last_slice(GstVaapiPictureH264 *picture)
 219 {
 220     g_return_val_if_fail(GST_VAAPI_IS_PICTURE_H264(picture), NULL);
 221
 222     if (G_UNLIKELY(picture->base.slices->len < 1))
 223         return NULL;
 224     return g_ptr_array_index(picture->base.slices,
 225         picture->base.slices->len - 1);
 226 }
 227
 228 /* ------------------------------------------------------------------------- */
 229 /* --- Slices                                                            --- */
 230 /* ------------------------------------------------------------------------- */
 231
 232 #define GST_VAAPI_TYPE_SLICE_H264 \
 233     (gst_vaapi_slice_h264_get_type())
 234
 235 #define GST_VAAPI_SLICE_H264_CAST(obj) \
 236     ((GstVaapiSliceH264 *)(obj))
 237
 238 #define GST_VAAPI_SLICE_H264(obj)                               \
 239     (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
 240                                 GST_VAAPI_TYPE_SLICE_H264,      \
 241                                 GstVaapiSliceH264))
 242
 243 #define GST_VAAPI_SLICE_H264_CLASS(klass)                       \
 244     (G_TYPE_CHECK_CLASS_CAST((klass),                           \
 245                              GST_VAAPI_TYPE_SLICE_H264,         \
 246                              GstVaapiSliceH264Class))
 247
 248 #define GST_VAAPI_IS_SLICE_H264(obj) \
 249     (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_VAAPI_TYPE_SLICE_H264))
 250
 251 #define GST_VAAPI_IS_SLICE_H264_CLASS(klass) \
 252     (G_TYPE_CHECK_CLASS_TYPE((klass), GST_VAAPI_TYPE_SLICE_H264))
 253
 254 #define GST_VAAPI_SLICE_H264_GET_CLASS(obj)                     \
 255     (G_TYPE_INSTANCE_GET_CLASS((obj),                           \
 256                                GST_VAAPI_TYPE_SLICE_H264,       \
 257                                GstVaapiSliceH264Class))
 258
 259 struct _GstVaapiSliceH264 {
 260     GstVaapiSlice               base;
 261     GstH264SliceHdr             slice_hdr;              // parsed slice_header()
 262 };
 263
 264 struct _GstVaapiSliceH264Class {
 265     /*< private >*/
 266     GstVaapiSliceClass          parent_class;
 267 };
 268
 269 GST_VAAPI_CODEC_DEFINE_TYPE(GstVaapiSliceH264,
 270                             gst_vaapi_slice_h264,
 271                             GST_VAAPI_TYPE_SLICE)
 272
 273 static void
 274 gst_vaapi_slice_h264_destroy(GstVaapiSliceH264 *slice)
 275 {
 276 }
 277
 278 static gboolean
 279 gst_vaapi_slice_h264_create(
 280     GstVaapiSliceH264                        *slice,
 281     const GstVaapiCodecObjectConstructorArgs *args
 282 )
 283 {
 284     return TRUE;
 285 }
 286
 287 static void
 288 gst_vaapi_slice_h264_init(GstVaapiSliceH264 *slice)
 289 {
 290 }
 291
 292 static inline GstVaapiSliceH264 *
 293 gst_vaapi_slice_h264_new(
 294     GstVaapiDecoderH264 *decoder,
 295     const guint8        *data,
 296     guint                data_size
 297 )
 298 {
 299     GstVaapiCodecObject *object;
 300
 301     g_return_val_if_fail(GST_VAAPI_IS_DECODER(decoder), NULL);
 302
 303     object = gst_vaapi_codec_object_new(
 304         GST_VAAPI_TYPE_SLICE_H264,
 305         GST_VAAPI_CODEC_BASE(decoder),
 306         NULL, sizeof(VASliceParameterBufferH264),
 307         data, data_size
 308     );
 309     if (!object)
 310         return NULL;
 311     return GST_VAAPI_SLICE_H264_CAST(object);
 312 }
 313
 314 /* ------------------------------------------------------------------------- */
 315 /* --- Frame Buffers (DPB)                                               --- */
 316 /* ------------------------------------------------------------------------- */
 317
 318 #define GST_VAAPI_TYPE_FRAME_STORE \
 319     (gst_vaapi_frame_store_get_type())
 320
 321 #define GST_VAAPI_FRAME_STORE_CAST(obj) \
 322     ((GstVaapiFrameStore *)(obj))
 323
 324 #define GST_VAAPI_FRAME_STORE(obj)                              \
 325     (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
 326                                 GST_VAAPI_TYPE_FRAME_STORE,     \
 327                                 GstVaapiFrameStore))
 328
 329 #define GST_VAAPI_FRAME_STORE_CLASS(klass)                      \
 330     (G_TYPE_CHECK_CLASS_CAST((klass),                           \
 331                              GST_VAAPI_TYPE_FRAME_STORE,        \
 332                              GstVaapiFrameStoreClass))
 333
 334 #define GST_VAAPI_IS_FRAME_STORE(obj) \
 335     (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_VAAPI_TYPE_FRAME_STORE))
 336
 337 #define GST_VAAPI_IS_FRAME_STORE_CLASS(klass) \
 338     (G_TYPE_CHECK_CLASS_TYPE((klass), GST_VAAPI_TYPE_FRAME_STORE))
 339
 340 #define GST_VAAPI_FRAME_STORE_GET_CLASS(obj)                   \
 341     (G_TYPE_INSTANCE_GET_CLASS((obj),                          \
 342                                GST_VAAPI_TYPE_FRAME_STORE,     \
 343                                GstVaapiFrameStoreClass))
 344
 345 struct _GstVaapiFrameStore {
 346     /*< private >*/
 347     GstMiniObject               parent_instance;
 348
 349     guint                       structure;
 350     GstVaapiPictureH264        *buffers[2];
 351     guint                       num_buffers;
 352     guint                       output_needed;
 353 };
 354
 355 struct _GstVaapiFrameStoreClass {
 356     /*< private >*/
 357     GstMiniObjectClass          parent_class;
 358 };
 359
 360 G_DEFINE_TYPE(GstVaapiFrameStore, gst_vaapi_frame_store, GST_TYPE_MINI_OBJECT)
 361
 362 static void
 363 gst_vaapi_frame_store_finalize(GstMiniObject *object)
 364 {
 365     GstVaapiFrameStore * const fs = GST_VAAPI_FRAME_STORE_CAST(object);
 366     GstMiniObjectClass *parent_class;
 367     guint i;
 368
 369     for (i = 0; i < fs->num_buffers; i++)
 370         gst_vaapi_picture_replace(&fs->buffers[i], NULL);
 371
 372     parent_class = GST_MINI_OBJECT_CLASS(gst_vaapi_frame_store_parent_class);
 373     if (parent_class->finalize)
 374         parent_class->finalize(object);
 375 }
 376
 377 static void
 378 gst_vaapi_frame_store_init(GstVaapiFrameStore *fs)
 379 {
 380 }
 381
 382 static void
 383 gst_vaapi_frame_store_class_init(GstVaapiFrameStoreClass *klass)
 384 {
 385     GstMiniObjectClass * const object_class = GST_MINI_OBJECT_CLASS(klass);
 386
 387     object_class->finalize = gst_vaapi_frame_store_finalize;
 388 }
 389
 390 static inline gpointer
 391 _gst_vaapi_frame_store_new(void)
 392 {
 393     return gst_mini_object_new(GST_VAAPI_TYPE_FRAME_STORE);
 394 }
 395
 396 static GstVaapiFrameStore *
 397 gst_vaapi_frame_store_new(GstVaapiPictureH264 *picture)
 398 {
 399     GstVaapiFrameStore *fs;
 400
 401     g_return_val_if_fail(GST_VAAPI_IS_PICTURE_H264(picture), NULL);
 402
 403     fs = _gst_vaapi_frame_store_new();
 404     if (!fs)
 405         return NULL;
 406
 407     fs->structure       = picture->structure;
 408     fs->buffers[0]      = gst_vaapi_picture_ref(picture);
 409     fs->num_buffers     = 1;
 410     fs->output_needed   = picture->output_needed;
 411     return fs;
 412 }
 413
 414 static gboolean
 415 gst_vaapi_frame_store_add(GstVaapiFrameStore *fs, GstVaapiPictureH264 *picture)
 416 {
 417     guint field;
 418
 419     g_return_val_if_fail(GST_VAAPI_IS_FRAME_STORE(fs), FALSE);
 420     g_return_val_if_fail(fs->num_buffers == 1, FALSE);
 421     g_return_val_if_fail(GST_VAAPI_IS_PICTURE_H264(picture), FALSE);
 422     g_return_val_if_fail(!GST_VAAPI_PICTURE_IS_FRAME(picture), FALSE);
 423
 424     gst_vaapi_picture_replace(&fs->buffers[fs->num_buffers++], picture);
 425     if (picture->output_flag) {
 426         picture->output_needed = TRUE;
 427         fs->output_needed++;
 428     }
 429
 430     fs->structure = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
 431
 432     field = picture->structure == GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD ?
 433         TOP_FIELD : BOTTOM_FIELD;
 434     g_return_val_if_fail(fs->buffers[0]->field_poc[field] == G_MAXINT32, FALSE);
 435     fs->buffers[0]->field_poc[field] = picture->field_poc[field];
 436     g_return_val_if_fail(picture->field_poc[!field] == G_MAXINT32, FALSE);
 437     picture->field_poc[!field] = fs->buffers[0]->field_poc[!field];
 438     return TRUE;
 439 }
 440
 441 static gboolean
 442 gst_vaapi_frame_store_split_fields(GstVaapiFrameStore *fs)
 443 {
 444     GstVaapiPictureH264 * const first_field = fs->buffers[0];
 445     GstVaapiPictureH264 *second_field;
 446
 447     g_return_val_if_fail(fs->num_buffers == 1, FALSE);
 448
 449     first_field->base.structure = GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD;
 450     GST_VAAPI_PICTURE_FLAG_SET(first_field, GST_VAAPI_PICTURE_FLAG_INTERLACED);
 451
 452     second_field = gst_vaapi_picture_h264_new_field(first_field);
 453     if (!second_field)
 454         return FALSE;
 455     gst_vaapi_picture_replace(&fs->buffers[fs->num_buffers++], second_field);
 456     gst_vaapi_picture_unref(second_field);
 457
 458     second_field->frame_num    = first_field->frame_num;
 459     second_field->field_poc[0] = first_field->field_poc[0];
 460     second_field->field_poc[1] = first_field->field_poc[1];
 461     second_field->output_flag  = first_field->output_flag;
 462     if (second_field->output_flag) {
 463         second_field->output_needed = TRUE;
 464         fs->output_needed++;
 465     }
 466     return TRUE;
 467 }
 468
 469 static inline gboolean
 470 gst_vaapi_frame_store_has_frame(GstVaapiFrameStore *fs)
 471 {
 472     return fs->structure == GST_VAAPI_PICTURE_STRUCTURE_FRAME;
 473 }
 474
 475 static inline gboolean
 476 gst_vaapi_frame_store_has_reference(GstVaapiFrameStore *fs)
 477 {
 478     guint i;
 479
 480     for (i = 0; i < fs->num_buffers; i++) {
 481         if (GST_VAAPI_PICTURE_IS_REFERENCE(fs->buffers[i]))
 482             return TRUE;
 483     }
 484     return FALSE;
 485 }
 486
 487 #define gst_vaapi_frame_store_ref(fs) \
 488     gst_mini_object_ref(GST_MINI_OBJECT(fs))
 489
 490 #define gst_vaapi_frame_store_unref(fs) \
 491     gst_mini_object_unref(GST_MINI_OBJECT(fs))
 492
 493 #define gst_vaapi_frame_store_replace(old_fs_p, new_fs)         \
 494     gst_mini_object_replace((GstMiniObject **)(old_fs_p),       \
 495                             (GstMiniObject *)(new_fs))
 496
 497 /* ------------------------------------------------------------------------- */
 498 /* --- H.264 Decoder                                                     --- */
 499 /* ------------------------------------------------------------------------- */
 500
 501 G_DEFINE_TYPE(GstVaapiDecoderH264,
 502               gst_vaapi_decoder_h264,
 503               GST_VAAPI_TYPE_DECODER)
 504
 505 #define GST_VAAPI_DECODER_H264_GET_PRIVATE(obj)                 \
 506     (G_TYPE_INSTANCE_GET_PRIVATE((obj),                         \
 507                                  GST_VAAPI_TYPE_DECODER_H264,   \
 508                                  GstVaapiDecoderH264Private))
 509
 510 struct _GstVaapiDecoderH264Private {
 511     GstAdapter                 *adapter;
 512     GstH264NalParser           *parser;
 513     /* Last decoded SPS. May not be the last activated one. Just here because
 514        it may not fit stack memory allocation in decode_sps() */
 515     GstH264SPS                  last_sps;
 516     /* Last decoded PPS. May not be the last activated one. Just here because
 517        it may not fit stack memory allocation in decode_pps() */
 518     GstH264PPS                  last_pps;
 519     /* Temporary slice header. Just here because it may not fit stack
 520        memory allocation in decode_slice() */
 521     GstH264SliceHdr             temp_slice_hdr;
 522     GstVaapiPictureH264        *current_picture;
 523     GstVaapiFrameStore         *prev_frame;
 524     GstVaapiFrameStore         *dpb[16];
 525     guint                       dpb_count;
 526     guint                       dpb_size;
 527     GstVaapiProfile             profile;
 528     GstVaapiEntrypoint          entrypoint;
 529     GstVaapiChromaType          chroma_type;
 530     GstVaapiPictureH264        *short_ref[32];
 531     guint                       short_ref_count;
 532     GstVaapiPictureH264        *long_ref[32];
 533     guint                       long_ref_count;
 534     GstVaapiPictureH264        *RefPicList0[32];
 535     guint                       RefPicList0_count;
 536     GstVaapiPictureH264        *RefPicList1[32];
 537     guint                       RefPicList1_count;
 538     guint                       nal_length_size;
 539     guint                       mb_width;
 540     guint                       mb_height;
 541     gint32                      field_poc[2];           // 0:TopFieldOrderCnt / 1:BottomFieldOrderCnt
 542     gint32                      poc_msb;                // PicOrderCntMsb
 543     gint32                      poc_lsb;                // pic_order_cnt_lsb (from slice_header())
 544     gint32                      prev_poc_msb;           // prevPicOrderCntMsb
 545     gint32                      prev_poc_lsb;           // prevPicOrderCntLsb
 546     gint32                      frame_num_offset;       // FrameNumOffset
 547     gint32                      frame_num;              // frame_num (from slice_header())
 548     gint32                      prev_frame_num;         // prevFrameNum
 549     gboolean                    prev_pic_has_mmco5;     // prevMmco5Pic
 550     gboolean                    prev_pic_structure;     // previous picture structure
 551     guint                       is_constructed          : 1;
 552     guint                       is_opened               : 1;
 553     guint                       is_avc                  : 1;
 554     guint                       got_sps                 : 1;
 555     guint                       got_pps                 : 1;
 556     guint                       has_context             : 1;
 557     guint                       progressive_sequence    : 1;
 558 };
 559
 560 static gboolean
 561 exec_ref_pic_marking(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture);
 562
 563 /* Get number of reference frames to use */
 564 static guint
 565 get_max_dec_frame_buffering(GstH264SPS *sps)
 566 {
 567     guint max_dec_frame_buffering, MaxDpbMbs, PicSizeMbs;
 568
 569     /* Table A-1 - Level limits */
 570     switch (sps->level_idc) {
 571     case 10: MaxDpbMbs = 396;    break;
 572     case 11: MaxDpbMbs = 900;    break;
 573     case 12: MaxDpbMbs = 2376;   break;
 574     case 13: MaxDpbMbs = 2376;   break;
 575     case 20: MaxDpbMbs = 2376;   break;
 576     case 21: MaxDpbMbs = 4752;   break;
 577     case 22: MaxDpbMbs = 8100;   break;
 578     case 30: MaxDpbMbs = 8100;   break;
 579     case 31: MaxDpbMbs = 18000;  break;
 580     case 32: MaxDpbMbs = 20480;  break;
 581     case 40: MaxDpbMbs = 32768;  break;
 582     case 41: MaxDpbMbs = 32768;  break;
 583     case 42: MaxDpbMbs = 34816;  break;
 584     case 50: MaxDpbMbs = 110400; break;
 585     case 51: MaxDpbMbs = 184320; break;
 586     default:
 587         g_assert(0 && "unhandled level");
 588         break;
 589     }
 590
 591     PicSizeMbs = ((sps->pic_width_in_mbs_minus1 + 1) *
 592                   (sps->pic_height_in_map_units_minus1 + 1) *
 593                   (sps->frame_mbs_only_flag ? 1 : 2));
 594     max_dec_frame_buffering = MaxDpbMbs / PicSizeMbs;
 595
 596     /* VUI parameters */
 597     if (sps->vui_parameters_present_flag) {
 598         GstH264VUIParams * const vui_params = &sps->vui_parameters;
 599         if (vui_params->bitstream_restriction_flag)
 600             max_dec_frame_buffering = vui_params->max_dec_frame_buffering;
 601         else {
 602             switch (sps->profile_idc) {
 603             case 44:  // CAVLC 4:4:4 Intra profile
 604             case 86:  // Scalable High profile
 605             case 100: // High profile
 606             case 110: // High 10 profile
 607             case 122: // High 4:2:2 profile
 608             case 244: // High 4:4:4 Predictive profile
 609                 if (sps->constraint_set3_flag)
 610                     max_dec_frame_buffering = 0;
 611                 break;
 612             }
 613         }
 614     }
 615
 616     if (max_dec_frame_buffering > 16)
 617         max_dec_frame_buffering = 16;
 618     else if (max_dec_frame_buffering < sps->num_ref_frames)
 619         max_dec_frame_buffering = sps->num_ref_frames;
 620     return MAX(1, max_dec_frame_buffering);
 621 }
 622
 623 static void
 624 array_remove_index_fast(void *array, guint *array_length_ptr, guint index)
 625 {
 626     gpointer * const entries = array;
 627     guint num_entries = *array_length_ptr;
 628
 629     g_return_if_fail(index < num_entries);
 630
 631     if (index != --num_entries)
 632         entries[index] = entries[num_entries];
 633     entries[num_entries] = NULL;
 634     *array_length_ptr = num_entries;
 635 }
 636
 637 #if 1
 638 static inline void
 639 array_remove_index(void *array, guint *array_length_ptr, guint index)
 640 {
 641     array_remove_index_fast(array, array_length_ptr, index);
 642 }
 643 #else
 644 static void
 645 array_remove_index(void *array, guint *array_length_ptr, guint index)
 646 {
 647     gpointer * const entries = array;
 648     const guint num_entries = *array_length_ptr - 1;
 649     guint i;
 650
 651     g_return_if_fail(index <= num_entries);
 652
 653     for (i = index; i < num_entries; i++)
 654         entries[i] = entries[i + 1];
 655     entries[num_entries] = NULL;
 656     *array_length_ptr = num_entries;
 657 }
 658 #endif
 659
 660 #define ARRAY_REMOVE_INDEX(array, index) \
 661     array_remove_index(array, &array##_count, index)
 662
 663 static void
 664 dpb_remove_index(GstVaapiDecoderH264 *decoder, guint index)
 665 {
 666     GstVaapiDecoderH264Private * const priv = decoder->priv;
 667     guint i, num_frames = --priv->dpb_count;
 668
 669     if (USE_STRICT_DPB_ORDERING) {
 670         for (i = index; i < num_frames; i++)
 671             gst_vaapi_frame_store_replace(&priv->dpb[i], priv->dpb[i + 1]);
 672     }
 673     else if (index != num_frames)
 674         gst_vaapi_frame_store_replace(&priv->dpb[index], priv->dpb[num_frames]);
 675     gst_vaapi_frame_store_replace(&priv->dpb[num_frames], NULL);
 676 }
 677
 678 static gboolean
 679 dpb_output(
 680     GstVaapiDecoderH264 *decoder,
 681     GstVaapiFrameStore  *fs,
 682     GstVaapiPictureH264 *picture
 683 )
 684 {
 685     picture->output_needed = FALSE;
 686
 687     if (fs) {
 688         if (--fs->output_needed > 0)
 689             return TRUE;
 690         picture = fs->buffers[0];
 691     }
 692
 693     /* XXX: update cropping rectangle */
 694     return gst_vaapi_picture_output(GST_VAAPI_PICTURE_CAST(picture));
 695 }
 696
 697 static inline void
 698 dpb_evict(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture, guint i)
 699 {
 700     GstVaapiFrameStore * const fs = decoder->priv->dpb[i];
 701
 702     if (!fs->output_needed && !gst_vaapi_frame_store_has_reference(fs))
 703         dpb_remove_index(decoder, i);
 704 }
 705
 706 static gboolean
 707 dpb_bump(GstVaapiDecoderH264 *decoder)
 708 {
 709     GstVaapiDecoderH264Private * const priv = decoder->priv;
 710     GstVaapiPictureH264 *found_picture = NULL;
 711     guint i, j, found_index;
 712     gboolean success;
 713
 714     for (i = 0; i < priv->dpb_count; i++) {
 715         GstVaapiFrameStore * const fs = priv->dpb[i];
 716         if (!fs->output_needed)
 717             continue;
 718         for (j = 0; j < fs->num_buffers; j++) {
 719             GstVaapiPictureH264 * const picture = fs->buffers[j];
 720             if (!picture->output_needed)
 721                 continue;
 722             if (!found_picture || found_picture->base.poc > picture->base.poc)
 723                 found_picture = picture, found_index = i;
 724         }
 725     }
 726     if (!found_picture)
 727         return FALSE;
 728
 729     success = dpb_output(decoder, priv->dpb[found_index], found_picture);
 730     dpb_evict(decoder, found_picture, found_index);
 731     return success;
 732 }
 733
 734 static void
 735 dpb_clear(GstVaapiDecoderH264 *decoder)
 736 {
 737     GstVaapiDecoderH264Private * const priv = decoder->priv;
 738     guint i;
 739
 740     for (i = 0; i < priv->dpb_count; i++)
 741         gst_vaapi_frame_store_replace(&priv->dpb[i], NULL);
 742     priv->dpb_count = 0;
 743
 744     gst_vaapi_frame_store_replace(&priv->prev_frame, NULL);
 745 }
 746
 747 static void
 748 dpb_flush(GstVaapiDecoderH264 *decoder)
 749 {
 750     while (dpb_bump(decoder))
 751         ;
 752     dpb_clear(decoder);
 753 }
 754
 755 static gboolean
 756 dpb_add(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
 757 {
 758     GstVaapiDecoderH264Private * const priv = decoder->priv;
 759     GstVaapiFrameStore *fs;
 760     guint i, j;
 761
 762     // Remove all unused pictures
 763     if (!GST_VAAPI_PICTURE_IS_IDR(picture)) {
 764         i = 0;
 765         while (i < priv->dpb_count) {
 766             GstVaapiFrameStore * const fs = priv->dpb[i];
 767             if (!fs->output_needed && !gst_vaapi_frame_store_has_reference(fs))
 768                 dpb_remove_index(decoder, i);
 769             else
 770                 i++;
 771         }
 772     }
 773
 774     // Check if picture is the second field and the first field is still in DPB
 775     fs = priv->prev_frame;
 776     if (fs && !gst_vaapi_frame_store_has_frame(fs)) {
 777         g_return_val_if_fail(fs->num_buffers == 1, FALSE);
 778         g_return_val_if_fail(!GST_VAAPI_PICTURE_IS_FRAME(picture), FALSE);
 779         g_return_val_if_fail(!GST_VAAPI_PICTURE_IS_FIRST_FIELD(picture), FALSE);
 780         return gst_vaapi_frame_store_add(fs, picture);
 781     }
 782
 783     // Create new frame store, and split fields if necessary
 784     fs = gst_vaapi_frame_store_new(picture);
 785     if (!fs)
 786         return FALSE;
 787     gst_vaapi_frame_store_replace(&priv->prev_frame, fs);
 788     gst_vaapi_frame_store_unref(fs);
 789
 790     if (!priv->progressive_sequence && gst_vaapi_frame_store_has_frame(fs)) {
 791         if (!gst_vaapi_frame_store_split_fields(fs))
 792             return FALSE;
 793     }
 794
 795     // C.4.5.1 - Storage and marking of a reference decoded picture into the DPB
 796     if (GST_VAAPI_PICTURE_IS_REFERENCE(picture)) {
 797         while (priv->dpb_count == priv->dpb_size) {
 798             if (!dpb_bump(decoder))
 799                 return FALSE;
 800         }
 801         gst_vaapi_frame_store_replace(&priv->dpb[priv->dpb_count++], fs);
 802         if (picture->output_flag) {
 803             picture->output_needed = TRUE;
 804             fs->output_needed++;
 805         }
 806     }
 807
 808     // C.4.5.2 - Storage and marking of a non-reference decoded picture into the DPB
 809     else {
 810         if (!picture->output_flag)
 811             return TRUE;
 812         while (priv->dpb_count == priv->dpb_size) {
 813             gboolean found_picture = FALSE;
 814             for (i = 0; !found_picture && i < priv->dpb_count; i++) {
 815                 GstVaapiFrameStore * const fs = priv->dpb[i];
 816                 if (!fs->output_needed)
 817                     continue;
 818                 for (j = 0; !found_picture && j < fs->num_buffers; j++)
 819                     found_picture = fs->buffers[j]->output_needed &&
 820                         fs->buffers[j]->base.poc < picture->base.poc;
 821             }
 822             if (!found_picture)
 823                 return dpb_output(decoder, NULL, picture);
 824             if (!dpb_bump(decoder))
 825                 return FALSE;
 826         }
 827         gst_vaapi_frame_store_replace(&priv->dpb[priv->dpb_count++], fs);
 828         picture->output_needed = TRUE;
 829         fs->output_needed++;
 830     }
 831     return TRUE;
 832 }
 833
 834 static inline void
 835 dpb_reset(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 836 {
 837     GstVaapiDecoderH264Private * const priv = decoder->priv;
 838
 839     priv->dpb_size = get_max_dec_frame_buffering(sps);
 840     GST_DEBUG("DPB size %u", priv->dpb_size);
 841 }
 842
 843 static GstVaapiDecoderStatus
 844 get_status(GstH264ParserResult result)
 845 {
 846     GstVaapiDecoderStatus status;
 847
 848     switch (result) {
 849     case GST_H264_PARSER_OK:
 850         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
 851         break;
 852     case GST_H264_PARSER_NO_NAL_END:
 853         status = GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
 854         break;
 855     case GST_H264_PARSER_ERROR:
 856         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
 857         break;
 858     default:
 859         status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 860         break;
 861     }
 862     return status;
 863 }
 864
 865 static void
 866 gst_vaapi_decoder_h264_clear_buffer(GstVaapiDecoder *base)
 867 {
 868     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264(base);
 869     GstVaapiDecoderH264Private * const priv = decoder->priv;
 870
 871     gst_vaapi_picture_replace(&priv->current_picture, NULL);
 872
 873     dpb_clear(decoder);
 874
 875     if (priv->adapter) {
 876         gst_adapter_clear(priv->adapter);
 877     }
 878 }
 879
 880 static void
 881 gst_vaapi_decoder_h264_close(GstVaapiDecoderH264 *decoder)
 882 {
 883     GstVaapiDecoderH264Private * const priv = decoder->priv;
 884
 885     gst_vaapi_decoder_h264_clear_buffer(GST_VAAPI_DECODER_CAST(decoder));
 886
 887     if (priv->parser) {
 888         gst_h264_nal_parser_free(priv->parser);
 889         priv->parser = NULL;
 890     }
 891
 892     if (priv->adapter) {
 893         g_object_unref(priv->adapter);
 894         priv->adapter = NULL;
 895     }
 896 }
 897
 898 static gboolean
 899 gst_vaapi_decoder_h264_open(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
 900 {
 901     GstVaapiDecoderH264Private * const priv = decoder->priv;
 902
 903     gst_vaapi_decoder_h264_close(decoder);
 904
 905     priv->adapter = gst_adapter_new();
 906     if (!priv->adapter)
 907         return FALSE;
 908
 909     priv->parser = gst_h264_nal_parser_new();
 910     if (!priv->parser)
 911         return FALSE;
 912     return TRUE;
 913 }
 914
 915 static void
 916 gst_vaapi_decoder_h264_destroy(GstVaapiDecoderH264 *decoder)
 917 {
 918     gst_vaapi_decoder_h264_close(decoder);
 919 }
 920
 921 static gboolean
 922 gst_vaapi_decoder_h264_create(GstVaapiDecoderH264 *decoder)
 923 {
 924     if (!GST_VAAPI_DECODER_CODEC(decoder))
 925         return FALSE;
 926     return TRUE;
 927 }
 928
 929 static guint
 930 h264_get_profile(GstH264SPS *sps)
 931 {
 932     guint profile = 0;
 933
 934     switch (sps->profile_idc) {
 935     case 66:
 936         profile = GST_VAAPI_PROFILE_H264_BASELINE;
 937         break;
 938     case 77:
 939         profile = GST_VAAPI_PROFILE_H264_MAIN;
 940         break;
 941     case 100:
 942         profile = GST_VAAPI_PROFILE_H264_HIGH;
 943         break;
 944     }
 945     return profile;
 946 }
 947
 948 static guint
 949 h264_get_chroma_type(GstH264SPS *sps)
 950 {
 951     guint chroma_type = 0;
 952
 953     switch (sps->chroma_format_idc) {
 954     case 1:
 955         chroma_type = GST_VAAPI_CHROMA_TYPE_YUV420;
 956         break;
 957     case 2:
 958         chroma_type = GST_VAAPI_CHROMA_TYPE_YUV422;
 959         break;
 960     case 3:
 961         if (!sps->separate_colour_plane_flag)
 962             chroma_type = GST_VAAPI_CHROMA_TYPE_YUV444;
 963         break;
 964     }
 965     return chroma_type;
 966 }
 967
 968 static GstVaapiProfile
 969 get_profile(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 970 {
 971     GstVaapiDecoderH264Private * const priv = decoder->priv;
 972     GstVaapiDisplay * const display = GST_VAAPI_DECODER_DISPLAY(decoder);
 973     GstVaapiProfile profile, profiles[2];
 974     guint i, n_profiles = 0;
 975
 976     profile = h264_get_profile(sps);
 977     if (!profile)
 978         return GST_VAAPI_PROFILE_UNKNOWN;
 979
 980     profiles[n_profiles++] = profile;
 981     switch (profile) {
 982     case GST_VAAPI_PROFILE_H264_MAIN:
 983         profiles[n_profiles++] = GST_VAAPI_PROFILE_H264_HIGH;
 984         break;
 985     default:
 986         break;
 987     }
 988
 989     /* If the preferred profile (profiles[0]) matches one that we already
 990        found, then just return it now instead of searching for it again */
 991     if (profiles[0] == priv->profile)
 992         return priv->profile;
 993
 994     for (i = 0; i < n_profiles; i++) {
 995         if (gst_vaapi_display_has_decoder(display, profiles[i], priv->entrypoint))
 996             return profiles[i];
 997     }
 998     return GST_VAAPI_PROFILE_UNKNOWN;
 999 }
1000
1001 static GstVaapiDecoderStatus
1002 ensure_context(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
1003 {
1004     GstVaapiDecoder * const base_decoder = GST_VAAPI_DECODER_CAST(decoder);
1005     GstVaapiDecoderH264Private * const priv = decoder->priv;
1006     GstVaapiContextInfo info;
1007     GstVaapiProfile profile;
1008     GstVaapiChromaType chroma_type;
1009     gboolean reset_context = FALSE;
1010     guint mb_width, mb_height;
1011
1012     profile = get_profile(decoder, sps);
1013     if (!profile) {
1014         GST_ERROR("unsupported profile_idc %u", sps->profile_idc);
1015         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_PROFILE;
1016     }
1017
1018     if (priv->profile != profile) {
1019         GST_DEBUG("profile changed");
1020         reset_context = TRUE;
1021         priv->profile = profile;
1022     }
1023
1024     chroma_type = h264_get_chroma_type(sps);
1025     if (!chroma_type || chroma_type != GST_VAAPI_CHROMA_TYPE_YUV420) {
1026         GST_ERROR("unsupported chroma_format_idc %u", sps->chroma_format_idc);
1027         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;
1028     }
1029
1030     if (priv->chroma_type != chroma_type) {
1031         GST_DEBUG("chroma format changed");
1032         reset_context     = TRUE;
1033         priv->chroma_type = chroma_type;
1034     }
1035
1036     mb_width  = sps->pic_width_in_mbs_minus1 + 1;
1037     mb_height = (sps->pic_height_in_map_units_minus1 + 1) <<
1038         !sps->frame_mbs_only_flag;
1039     if (priv->mb_width != mb_width || priv->mb_height != mb_height) {
1040         GST_DEBUG("size changed");
1041         reset_context   = TRUE;
1042         priv->mb_width  = mb_width;
1043         priv->mb_height = mb_height;
1044     }
1045
1046     priv->progressive_sequence = sps->frame_mbs_only_flag;
1047 #if 0
1048     /* XXX: we only output complete frames for now */
1049     gst_vaapi_decoder_set_interlaced(base_decoder, !priv->progressive_sequence);
1050 #endif
1051
1052     gst_vaapi_decoder_set_pixel_aspect_ratio(
1053         base_decoder,
1054         sps->vui_parameters.par_n,
1055         sps->vui_parameters.par_d
1056     );
1057
1058     if (!reset_context && priv->has_context)
1059         return GST_VAAPI_DECODER_STATUS_SUCCESS;
1060
1061     /* XXX: fix surface size when cropping is implemented */
1062     info.profile    = priv->profile;
1063     info.entrypoint = priv->entrypoint;
1064     info.width      = sps->width;
1065     info.height     = sps->height;
1066     info.ref_frames = get_max_dec_frame_buffering(sps);
1067
1068     if (!gst_vaapi_decoder_ensure_context(GST_VAAPI_DECODER(decoder), &info))
1069         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
1070     priv->has_context = TRUE;
1071
1072     /* Reset DPB */
1073     dpb_reset(decoder, sps);
1074     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1075 }
1076
1077 static void
1078 fill_iq_matrix_4x4(VAIQMatrixBufferH264 *iq_matrix, const GstH264PPS *pps)
1079 {
1080     const guint8 (* const ScalingList4x4)[6][16] = &pps->scaling_lists_4x4;
1081     guint i, j;
1082
1083     /* There are always 6 4x4 scaling lists */
1084     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList4x4) == 6);
1085     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList4x4[0]) == 16);
1086
1087     if (sizeof(iq_matrix->ScalingList4x4[0][0]) == 1)
1088         memcpy(iq_matrix->ScalingList4x4, *ScalingList4x4,
1089                sizeof(iq_matrix->ScalingList4x4));
1090     else {
1091         for (i = 0; i < G_N_ELEMENTS(iq_matrix->ScalingList4x4); i++) {
1092             for (j = 0; j < G_N_ELEMENTS(iq_matrix->ScalingList4x4[i]); j++)
1093                 iq_matrix->ScalingList4x4[i][j] = (*ScalingList4x4)[i][j];
1094         }
1095     }
1096 }
1097
1098 static void
1099 fill_iq_matrix_8x8(VAIQMatrixBufferH264 *iq_matrix, const GstH264PPS *pps)
1100 {
1101     const guint8 (* const ScalingList8x8)[6][64] = &pps->scaling_lists_8x8;
1102     const GstH264SPS * const sps = pps->sequence;
1103     guint i, j, n;
1104
1105     /* If chroma_format_idc != 3, there are up to 2 8x8 scaling lists */
1106     if (!pps->transform_8x8_mode_flag)
1107         return;
1108
1109     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList8x8) >= 2);
1110     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList8x8[0]) == 64);
1111
1112     if (sizeof(iq_matrix->ScalingList8x8[0][0]) == 1)
1113         memcpy(iq_matrix->ScalingList8x8, *ScalingList8x8,
1114                sizeof(iq_matrix->ScalingList8x8));
1115     else {
1116         n = (sps->chroma_format_idc != 3) ? 2 : 6;
1117         for (i = 0; i < n; i++) {
1118             for (j = 0; j < G_N_ELEMENTS(iq_matrix->ScalingList8x8[i]); j++)
1119                 iq_matrix->ScalingList8x8[i][j] = (*ScalingList8x8)[i][j];
1120         }
1121     }
1122 }
1123
1124 static GstVaapiDecoderStatus
1125 ensure_quant_matrix(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
1126 {
1127     GstVaapiPicture * const base_picture = &picture->base;
1128     GstH264PPS * const pps = picture->pps;
1129     GstH264SPS * const sps = pps->sequence;
1130     VAIQMatrixBufferH264 *iq_matrix;
1131
1132     base_picture->iq_matrix = GST_VAAPI_IQ_MATRIX_NEW(H264, decoder);
1133     if (!base_picture->iq_matrix) {
1134         GST_ERROR("failed to allocate IQ matrix");
1135         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
1136     }
1137     iq_matrix = base_picture->iq_matrix->param;
1138
1139     /* XXX: we can only support 4:2:0 or 4:2:2 since ScalingLists8x8[]
1140        is not large enough to hold lists for 4:4:4 */
1141     if (sps->chroma_format_idc == 3)
1142         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;
1143
1144     fill_iq_matrix_4x4(iq_matrix, pps);
1145     fill_iq_matrix_8x8(iq_matrix, pps);
1146
1147     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1148 }
1149
1150 static GstVaapiDecoderStatus
1151 decode_current_picture(GstVaapiDecoderH264 *decoder)
1152 {
1153     GstVaapiDecoderH264Private * const priv = decoder->priv;
1154     GstVaapiPictureH264 * const picture = priv->current_picture;
1155     GstVaapiDecoderStatus status;
1156
1157     if (!picture)
1158         return GST_VAAPI_DECODER_STATUS_SUCCESS;
1159
1160     status = ensure_context(decoder, picture->pps->sequence);
1161     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
1162         return status;
1163
1164     if (!exec_ref_pic_marking(decoder, picture))
1165         goto error;
1166     if (!dpb_add(decoder, picture))
1167         goto error;
1168     if (!gst_vaapi_picture_decode(GST_VAAPI_PICTURE_CAST(picture)))
1169         goto error;
1170     if (priv->prev_frame && gst_vaapi_frame_store_has_frame(priv->prev_frame))
1171         gst_vaapi_picture_replace(&priv->current_picture, NULL);
1172     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1173
1174 error:
1175     /* XXX: fix for cases where first field failed to be decoded */
1176     gst_vaapi_picture_replace(&priv->current_picture, NULL);
1177     return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
1178 }
1179
1180 static GstVaapiDecoderStatus
1181 decode_sps(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
1182 {
1183     GstVaapiDecoderH264Private * const priv = decoder->priv;
1184     GstH264SPS * const sps = &priv->last_sps;
1185     GstH264ParserResult result;
1186
1187     GST_DEBUG("decode SPS");
1188
1189     memset(sps, 0, sizeof(*sps));
1190     result = gst_h264_parser_parse_sps(priv->parser, nalu, sps, TRUE);
1191     if (result != GST_H264_PARSER_OK) {
1192         priv->got_sps = FALSE;
1193         return get_status(result);
1194     }
1195
1196     priv->got_sps = TRUE;
1197     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1198 }
1199
1200 static GstVaapiDecoderStatus
1201 decode_pps(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
1202 {
1203     GstVaapiDecoderH264Private * const priv = decoder->priv;
1204     GstH264PPS * const pps = &priv->last_pps;
1205     GstH264ParserResult result;
1206
1207     GST_DEBUG("decode PPS");
1208
1209     memset(pps, 0, sizeof(*pps));
1210     result = gst_h264_parser_parse_pps(priv->parser, nalu, pps);
1211     if (result != GST_H264_PARSER_OK) {
1212         priv->got_pps = FALSE;
1213         return get_status(result);
1214     }
1215
1216     priv->got_pps = TRUE;
1217     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1218 }
1219
1220 static GstVaapiDecoderStatus
1221 decode_sei(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
1222 {
1223     GstVaapiDecoderH264Private * const priv = decoder->priv;
1224     GstH264SEIMessage sei;
1225     GstH264ParserResult result;
1226
1227     GST_DEBUG("decode SEI");
1228
1229     memset(&sei, 0, sizeof(sei));
1230     result = gst_h264_parser_parse_sei(priv->parser, nalu, &sei);
1231     if (result != GST_H264_PARSER_OK) {
1232         GST_WARNING("failed to decode SEI, payload type:%d", sei.payloadType);
1233         return get_status(result);
1234     }
1235
1236     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1237 }
1238
1239 static GstVaapiDecoderStatus
1240 decode_sequence_end(GstVaapiDecoderH264 *decoder)
1241 {
1242     GstVaapiDecoderStatus status;
1243
1244     GST_DEBUG("decode sequence-end");
1245
1246     status = decode_current_picture(decoder);
1247     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
1248         return status;
1249
1250     dpb_flush(decoder);
1251     return GST_VAAPI_DECODER_STATUS_END_OF_STREAM;
1252 }
1253
1254 /* 8.2.1.1 - Decoding process for picture order count type 0 */
1255 static void
1256 init_picture_poc_0(
1257     GstVaapiDecoderH264 *decoder,
1258     GstVaapiPictureH264 *picture,
1259     GstH264SliceHdr     *slice_hdr
1260 )
1261 {
1262     GstVaapiDecoderH264Private * const priv = decoder->priv;
1263     GstH264PPS * const pps = slice_hdr->pps;
1264     GstH264SPS * const sps = pps->sequence;
1265     const gint32 MaxPicOrderCntLsb = 1 << (sps->log2_max_pic_order_cnt_lsb_minus4 + 4);
1266     gint32 temp_poc;
1267
1268     GST_DEBUG("decode picture order count type 0");
1269
1270     if (GST_VAAPI_PICTURE_IS_IDR(picture)) {
1271         priv->prev_poc_msb = 0;
1272         priv->prev_poc_lsb = 0;
1273     }
1274     else if (priv->prev_pic_has_mmco5) {
1275         priv->prev_poc_msb = 0;
1276         priv->prev_poc_lsb =
1277             (priv->prev_pic_structure == GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD ?
1278              0 : priv->field_poc[TOP_FIELD]);
1279     }
1280     else {
1281         priv->prev_poc_msb = priv->poc_msb;
1282         priv->prev_poc_lsb = priv->poc_lsb;
1283     }
1284
1285     // (8-3)
1286     priv->poc_lsb = slice_hdr->pic_order_cnt_lsb;
1287     if (priv->poc_lsb < priv->prev_poc_lsb &&
1288         (priv->prev_poc_lsb - priv->poc_lsb) >= (MaxPicOrderCntLsb / 2))
1289         priv->poc_msb = priv->prev_poc_msb + MaxPicOrderCntLsb;
1290     else if (priv->poc_lsb > priv->prev_poc_lsb &&
1291              (priv->poc_lsb - priv->prev_poc_lsb) > (MaxPicOrderCntLsb / 2))
1292         priv->poc_msb = priv->prev_poc_msb - MaxPicOrderCntLsb;
1293     else
1294         priv->poc_msb = priv->prev_poc_msb;
1295
1296     temp_poc = priv->poc_msb + priv->poc_lsb;
1297     switch (picture->structure) {
1298     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
1299         // (8-4, 8-5)
1300         priv->field_poc[TOP_FIELD] = temp_poc;
1301         priv->field_poc[BOTTOM_FIELD] = temp_poc +
1302             slice_hdr->delta_pic_order_cnt_bottom;
1303         break;
1304     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
1305         // (8-4)
1306         priv->field_poc[TOP_FIELD] = temp_poc;
1307         break;
1308     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
1309         // (8-5)
1310         priv->field_poc[BOTTOM_FIELD] = temp_poc;
1311         break;
1312     }
1313 }
1314
1315 /* 8.2.1.2 - Decoding process for picture order count type 1 */
1316 static void
1317 init_picture_poc_1(
1318     GstVaapiDecoderH264 *decoder,
1319     GstVaapiPictureH264 *picture,
1320     GstH264SliceHdr     *slice_hdr
1321 )
1322 {
1323     GstVaapiDecoderH264Private * const priv = decoder->priv;
1324     GstH264PPS * const pps = slice_hdr->pps;
1325     GstH264SPS * const sps = pps->sequence;
1326     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1327     gint32 prev_frame_num_offset, abs_frame_num, expected_poc;
1328     guint i;
1329
1330     GST_DEBUG("decode picture order count type 1");
1331
1332     if (priv->prev_pic_has_mmco5)
1333         prev_frame_num_offset = 0;
1334     else
1335         prev_frame_num_offset = priv->frame_num_offset;
1336
1337     // (8-6)
1338     if (GST_VAAPI_PICTURE_IS_IDR(picture))
1339         priv->frame_num_offset = 0;
1340     else if (priv->prev_frame_num > priv->frame_num)
1341         priv->frame_num_offset = prev_frame_num_offset + MaxFrameNum;
1342     else
1343         priv->frame_num_offset = prev_frame_num_offset;
1344
1345     // (8-7)
1346     if (sps->num_ref_frames_in_pic_order_cnt_cycle != 0)
1347         abs_frame_num = priv->frame_num_offset + priv->frame_num;
1348     else
1349         abs_frame_num = 0;
1350     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture) && abs_frame_num > 0)
1351         abs_frame_num = abs_frame_num - 1;
1352
1353     if (abs_frame_num > 0) {
1354         gint32 expected_delta_per_poc_cycle;
1355         gint32 poc_cycle_cnt, frame_num_in_poc_cycle;
1356
1357         expected_delta_per_poc_cycle = 0;
1358         for (i = 0; i < sps->num_ref_frames_in_pic_order_cnt_cycle; i++)
1359             expected_delta_per_poc_cycle += sps->offset_for_ref_frame[i];
1360
1361         // (8-8)
1362         poc_cycle_cnt = (abs_frame_num - 1) /
1363             sps->num_ref_frames_in_pic_order_cnt_cycle;
1364         frame_num_in_poc_cycle = (abs_frame_num - 1) %
1365             sps->num_ref_frames_in_pic_order_cnt_cycle;
1366
1367         // (8-9)
1368         expected_poc = poc_cycle_cnt * expected_delta_per_poc_cycle;
1369         for (i = 0; i <= frame_num_in_poc_cycle; i++)
1370             expected_poc += sps->offset_for_ref_frame[i];
1371     }
1372     else
1373         expected_poc = 0;
1374     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
1375         expected_poc += sps->offset_for_non_ref_pic;
1376
1377     // (8-10)
1378     switch (picture->structure) {
1379     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
1380         priv->field_poc[TOP_FIELD] = expected_poc +
1381             slice_hdr->delta_pic_order_cnt[0];
1382         priv->field_poc[BOTTOM_FIELD] = priv->field_poc[TOP_FIELD] +
1383             sps->offset_for_top_to_bottom_field +
1384             slice_hdr->delta_pic_order_cnt[1];
1385         break;
1386     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
1387         priv->field_poc[TOP_FIELD] = expected_poc +
1388             slice_hdr->delta_pic_order_cnt[0];
1389         break;
1390     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
1391         priv->field_poc[BOTTOM_FIELD] = expected_poc +
1392             sps->offset_for_top_to_bottom_field +
1393             slice_hdr->delta_pic_order_cnt[0];
1394         break;
1395     }
1396 }
1397
1398 /* 8.2.1.3 - Decoding process for picture order count type 2 */
1399 static void
1400 init_picture_poc_2(
1401     GstVaapiDecoderH264 *decoder,
1402     GstVaapiPictureH264 *picture,
1403     GstH264SliceHdr     *slice_hdr
1404 )
1405 {
1406     GstVaapiDecoderH264Private * const priv = decoder->priv;
1407     GstH264PPS * const pps = slice_hdr->pps;
1408     GstH264SPS * const sps = pps->sequence;
1409     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1410     gint32 prev_frame_num_offset, temp_poc;
1411
1412     GST_DEBUG("decode picture order count type 2");
1413
1414     if (priv->prev_pic_has_mmco5)
1415         prev_frame_num_offset = 0;
1416     else
1417         prev_frame_num_offset = priv->frame_num_offset;
1418
1419     // (8-11)
1420     if (GST_VAAPI_PICTURE_IS_IDR(picture))
1421         priv->frame_num_offset = 0;
1422     else if (priv->prev_frame_num > priv->frame_num)
1423         priv->frame_num_offset = prev_frame_num_offset + MaxFrameNum;
1424     else
1425         priv->frame_num_offset = prev_frame_num_offset;
1426
1427     // (8-12)
1428     if (GST_VAAPI_PICTURE_IS_IDR(picture))
1429         temp_poc = 0;
1430     else if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
1431         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num) - 1;
1432     else
1433         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num);
1434
1435     // (8-13)
1436     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
1437         priv->field_poc[TOP_FIELD] = temp_poc;
1438     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
1439         priv->field_poc[BOTTOM_FIELD] = temp_poc;
1440 }
1441
1442 /* 8.2.1 - Decoding process for picture order count */
1443 static void
1444 init_picture_poc(
1445     GstVaapiDecoderH264 *decoder,
1446     GstVaapiPictureH264 *picture,
1447     GstH264SliceHdr     *slice_hdr
1448 )
1449 {
1450     GstVaapiDecoderH264Private * const priv = decoder->priv;
1451     GstH264PPS * const pps = slice_hdr->pps;
1452     GstH264SPS * const sps = pps->sequence;
1453
1454     switch (sps->pic_order_cnt_type) {
1455     case 0:
1456         init_picture_poc_0(decoder, picture, slice_hdr);
1457         break;
1458     case 1:
1459         init_picture_poc_1(decoder, picture, slice_hdr);
1460         break;
1461     case 2:
1462         init_picture_poc_2(decoder, picture, slice_hdr);
1463         break;
1464     }
1465
1466     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
1467         picture->field_poc[TOP_FIELD] = priv->field_poc[TOP_FIELD];
1468     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
1469         picture->field_poc[BOTTOM_FIELD] = priv->field_poc[BOTTOM_FIELD];
1470     picture->base.poc = MIN(picture->field_poc[0], picture->field_poc[1]);
1471 }
1472
1473 static int
1474 compare_picture_pic_num_dec(const void *a, const void *b)
1475 {
1476     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1477     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1478
1479     return picB->pic_num - picA->pic_num;
1480 }
1481
1482 static int
1483 compare_picture_long_term_pic_num_inc(const void *a, const void *b)
1484 {
1485     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1486     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1487
1488     return picA->long_term_pic_num - picB->long_term_pic_num;
1489 }
1490
1491 static int
1492 compare_picture_poc_dec(const void *a, const void *b)
1493 {
1494     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1495     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1496
1497     return picB->base.poc - picA->base.poc;
1498 }
1499
1500 static int
1501 compare_picture_poc_inc(const void *a, const void *b)
1502 {
1503     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1504     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1505
1506     return picA->base.poc - picB->base.poc;
1507 }
1508
1509 static int
1510 compare_picture_frame_num_wrap_dec(const void *a, const void *b)
1511 {
1512     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1513     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1514
1515     return picB->frame_num_wrap - picA->frame_num_wrap;
1516 }
1517
1518 static int
1519 compare_picture_long_term_frame_idx_inc(const void *a, const void *b)
1520 {
1521     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1522     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1523
1524     return picA->long_term_frame_idx - picB->long_term_frame_idx;
1525 }
1526
1527 /* 8.2.4.1 - Decoding process for picture numbers */
1528 static void
1529 init_picture_refs_pic_num(
1530     GstVaapiDecoderH264 *decoder,
1531     GstVaapiPictureH264 *picture,
1532     GstH264SliceHdr     *slice_hdr
1533 )
1534 {
1535     GstVaapiDecoderH264Private * const priv = decoder->priv;
1536     GstH264PPS * const pps = slice_hdr->pps;
1537     GstH264SPS * const sps = pps->sequence;
1538     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1539     guint i;
1540
1541     GST_DEBUG("decode picture numbers");
1542
1543     for (i = 0; i < priv->short_ref_count; i++) {
1544         GstVaapiPictureH264 * const pic = priv->short_ref[i];
1545
1546         // (8-27)
1547         if (pic->frame_num > priv->frame_num)
1548             pic->frame_num_wrap = pic->frame_num - MaxFrameNum;
1549         else
1550             pic->frame_num_wrap = pic->frame_num;
1551
1552         // (8-28, 8-30, 8-31)
1553         if (GST_VAAPI_PICTURE_IS_FRAME(picture))
1554             pic->pic_num = pic->frame_num_wrap;
1555         else {
1556             if (pic->structure == picture->structure)
1557                 pic->pic_num = 2 * pic->frame_num_wrap + 1;
1558             else
1559                 pic->pic_num = 2 * pic->frame_num_wrap;
1560         }
1561     }
1562
1563     for (i = 0; i < priv->long_ref_count; i++) {
1564         GstVaapiPictureH264 * const pic = priv->long_ref[i];
1565
1566         // (8-29, 8-32, 8-33)
1567         if (GST_VAAPI_PICTURE_IS_FRAME(picture))
1568             pic->long_term_pic_num = pic->long_term_frame_idx;
1569         else {
1570             if (pic->structure == picture->structure)
1571                 pic->long_term_pic_num = 2 * pic->long_term_frame_idx + 1;
1572             else
1573                 pic->long_term_pic_num = 2 * pic->long_term_frame_idx;
1574         }
1575     }
1576 }
1577
1578 #define SORT_REF_LIST(list, n, compare_func) \
1579     qsort(list, n, sizeof(*(list)), compare_picture_##compare_func)
1580
1581 static void
1582 init_picture_refs_fields_1(
1583     guint                picture_structure,
1584     GstVaapiPictureH264 *RefPicList[32],
1585     guint               *RefPicList_count,
1586     GstVaapiPictureH264 *ref_list[32],
1587     guint                ref_list_count
1588 )
1589 {
1590     guint i, j, n;
1591
1592     i = 0;
1593     j = 0;
1594     n = *RefPicList_count;
1595     do {
1596         g_assert(n < 32);
1597         for (; i < ref_list_count; i++) {
1598             if (ref_list[i]->structure == picture_structure) {
1599                 RefPicList[n++] = ref_list[i++];
1600                 break;
1601             }
1602         }
1603         for (; j < ref_list_count; j++) {
1604             if (ref_list[j]->structure != picture_structure) {
1605                 RefPicList[n++] = ref_list[j++];
1606                 break;
1607             }
1608         }
1609     } while (i < ref_list_count || j < ref_list_count);
1610     *RefPicList_count = n;
1611 }
1612
1613 static inline void
1614 init_picture_refs_fields(
1615     GstVaapiPictureH264 *picture,
1616     GstVaapiPictureH264 *RefPicList[32],
1617     guint               *RefPicList_count,
1618     GstVaapiPictureH264 *short_ref[32],
1619     guint                short_ref_count,
1620     GstVaapiPictureH264 *long_ref[32],
1621     guint                long_ref_count
1622 )
1623 {
1624     guint n = 0;
1625
1626     /* 8.2.4.2.5 - reference picture lists in fields */
1627     init_picture_refs_fields_1(picture->structure, RefPicList, &n,
1628         short_ref, short_ref_count);
1629     init_picture_refs_fields_1(picture->structure, RefPicList, &n,
1630         long_ref, long_ref_count);
1631     *RefPicList_count = n;
1632 }
1633
1634 static void
1635 init_picture_refs_p_slice(
1636     GstVaapiDecoderH264 *decoder,
1637     GstVaapiPictureH264 *picture,
1638     GstH264SliceHdr     *slice_hdr
1639 )
1640 {
1641     GstVaapiDecoderH264Private * const priv = decoder->priv;
1642     GstVaapiPictureH264 **ref_list;
1643     guint i;
1644
1645     GST_DEBUG("decode reference picture list for P and SP slices");
1646
1647     if (GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1648         /* 8.2.4.2.1 - P and SP slices in frames */
1649         if (priv->short_ref_count > 0) {
1650             ref_list = priv->RefPicList0;
1651             for (i = 0; i < priv->short_ref_count; i++)
1652                 ref_list[i] = priv->short_ref[i];
1653             SORT_REF_LIST(ref_list, i, pic_num_dec);
1654             priv->RefPicList0_count += i;
1655         }
1656
1657         if (priv->long_ref_count > 0) {
1658             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1659             for (i = 0; i < priv->long_ref_count; i++)
1660                 ref_list[i] = priv->long_ref[i];
1661             SORT_REF_LIST(ref_list, i, long_term_pic_num_inc);
1662             priv->RefPicList0_count += i;
1663         }
1664     }
1665     else {
1666         /* 8.2.4.2.2 - P and SP slices in fields */
1667         GstVaapiPictureH264 *short_ref[32];
1668         guint short_ref_count = 0;
1669         GstVaapiPictureH264 *long_ref[32];
1670         guint long_ref_count = 0;
1671
1672         if (priv->short_ref_count > 0) {
1673             for (i = 0; i < priv->short_ref_count; i++)
1674                 short_ref[i] = priv->short_ref[i];
1675             SORT_REF_LIST(short_ref, i, frame_num_wrap_dec);
1676             short_ref_count = i;
1677         }
1678
1679         if (priv->long_ref_count > 0) {
1680             for (i = 0; i < priv->long_ref_count; i++)
1681                 long_ref[i] = priv->long_ref[i];
1682             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1683             long_ref_count = i;
1684         }
1685
1686         init_picture_refs_fields(
1687             picture,
1688             priv->RefPicList0, &priv->RefPicList0_count,
1689             short_ref,          short_ref_count,
1690             long_ref,           long_ref_count
1691         );
1692     }
1693 }
1694
1695 static void
1696 init_picture_refs_b_slice(
1697     GstVaapiDecoderH264 *decoder,
1698     GstVaapiPictureH264 *picture,
1699     GstH264SliceHdr     *slice_hdr
1700 )
1701 {
1702     GstVaapiDecoderH264Private * const priv = decoder->priv;
1703     GstVaapiPictureH264 **ref_list;
1704     guint i, n;
1705
1706     GST_DEBUG("decode reference picture list for B slices");
1707
1708     if (GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1709         /* 8.2.4.2.3 - B slices in frames */
1710
1711         /* RefPicList0 */
1712         if (priv->short_ref_count > 0) {
1713             // 1. Short-term references
1714             ref_list = priv->RefPicList0;
1715             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1716                 if (priv->short_ref[i]->base.poc < picture->base.poc)
1717                     ref_list[n++] = priv->short_ref[i];
1718             }
1719             SORT_REF_LIST(ref_list, n, poc_dec);
1720             priv->RefPicList0_count += n;
1721
1722             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1723             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1724                 if (priv->short_ref[i]->base.poc >= picture->base.poc)
1725                     ref_list[n++] = priv->short_ref[i];
1726             }
1727             SORT_REF_LIST(ref_list, n, poc_inc);
1728             priv->RefPicList0_count += n;
1729         }
1730
1731         if (priv->long_ref_count > 0) {
1732             // 2. Long-term references
1733             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1734             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1735                 ref_list[n++] = priv->long_ref[i];
1736             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1737             priv->RefPicList0_count += n;
1738         }
1739
1740         /* RefPicList1 */
1741         if (priv->short_ref_count > 0) {
1742             // 1. Short-term references
1743             ref_list = priv->RefPicList1;
1744             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1745                 if (priv->short_ref[i]->base.poc > picture->base.poc)
1746                     ref_list[n++] = priv->short_ref[i];
1747             }
1748             SORT_REF_LIST(ref_list, n, poc_inc);
1749             priv->RefPicList1_count += n;
1750
1751             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1752             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1753                 if (priv->short_ref[i]->base.poc <= picture->base.poc)
1754                     ref_list[n++] = priv->short_ref[i];
1755             }
1756             SORT_REF_LIST(ref_list, n, poc_dec);
1757             priv->RefPicList1_count += n;
1758         }
1759
1760         if (priv->long_ref_count > 0) {
1761             // 2. Long-term references
1762             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1763             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1764                 ref_list[n++] = priv->long_ref[i];
1765             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1766             priv->RefPicList1_count += n;
1767         }
1768     }
1769     else {
1770         /* 8.2.4.2.4 - B slices in fields */
1771         GstVaapiPictureH264 *short_ref0[32];
1772         guint short_ref0_count = 0;
1773         GstVaapiPictureH264 *short_ref1[32];
1774         guint short_ref1_count = 0;
1775         GstVaapiPictureH264 *long_ref[32];
1776         guint long_ref_count = 0;
1777
1778         /* refFrameList0ShortTerm */
1779         if (priv->short_ref_count > 0) {
1780             ref_list = short_ref0;
1781             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1782                 if (priv->short_ref[i]->base.poc <= picture->base.poc)
1783                     ref_list[n++] = priv->short_ref[i];
1784             }
1785             SORT_REF_LIST(ref_list, n, poc_dec);
1786             short_ref0_count += n;
1787
1788             ref_list = &short_ref0[short_ref0_count];
1789             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1790                 if (priv->short_ref[i]->base.poc > picture->base.poc)
1791                     ref_list[n++] = priv->short_ref[i];
1792             }
1793             SORT_REF_LIST(ref_list, n, poc_inc);
1794             short_ref0_count += n;
1795         }
1796
1797         /* refFrameList1ShortTerm */
1798         if (priv->short_ref_count > 0) {
1799             ref_list = short_ref1;
1800             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1801                 if (priv->short_ref[i]->base.poc > picture->base.poc)
1802                     ref_list[n++] = priv->short_ref[i];
1803             }
1804             SORT_REF_LIST(ref_list, n, poc_inc);
1805             short_ref1_count += n;
1806
1807             ref_list = &short_ref1[short_ref1_count];
1808             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1809                 if (priv->short_ref[i]->base.poc <= picture->base.poc)
1810                     ref_list[n++] = priv->short_ref[i];
1811             }
1812             SORT_REF_LIST(ref_list, n, poc_dec);
1813             short_ref1_count += n;
1814         }
1815
1816         /* refFrameListLongTerm */
1817         if (priv->long_ref_count > 0) {
1818             for (i = 0; i < priv->long_ref_count; i++)
1819                 long_ref[i] = priv->long_ref[i];
1820             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1821             long_ref_count = i;
1822         }
1823
1824         init_picture_refs_fields(
1825             picture,
1826             priv->RefPicList0, &priv->RefPicList0_count,
1827             short_ref0,         short_ref0_count,
1828             long_ref,           long_ref_count
1829         );
1830
1831         init_picture_refs_fields(
1832             picture,
1833             priv->RefPicList1, &priv->RefPicList1_count,
1834             short_ref1,         short_ref1_count,
1835             long_ref,           long_ref_count
1836         );
1837    }
1838
1839     /* Check whether RefPicList1 is identical to RefPicList0, then
1840        swap if necessary */
1841     if (priv->RefPicList1_count > 1 &&
1842         priv->RefPicList1_count == priv->RefPicList0_count &&
1843         memcmp(priv->RefPicList0, priv->RefPicList1,
1844                priv->RefPicList0_count * sizeof(priv->RefPicList0[0])) == 0) {
1845         GstVaapiPictureH264 * const tmp = priv->RefPicList1[0];
1846         priv->RefPicList1[0] = priv->RefPicList1[1];
1847         priv->RefPicList1[1] = tmp;
1848     }
1849 }
1850
1851 #undef SORT_REF_LIST
1852
1853 static gint
1854 find_short_term_reference(GstVaapiDecoderH264 *decoder, gint32 pic_num)
1855 {
1856     GstVaapiDecoderH264Private * const priv = decoder->priv;
1857     guint i;
1858
1859     for (i = 0; i < priv->short_ref_count; i++) {
1860         if (priv->short_ref[i]->pic_num == pic_num)
1861             return i;
1862     }
1863     GST_ERROR("found no short-term reference picture with PicNum = %d",
1864               pic_num);
1865     return -1;
1866 }
1867
1868 static gint
1869 find_long_term_reference(GstVaapiDecoderH264 *decoder, gint32 long_term_pic_num)
1870 {
1871     GstVaapiDecoderH264Private * const priv = decoder->priv;
1872     guint i;
1873
1874     for (i = 0; i < priv->long_ref_count; i++) {
1875         if (priv->long_ref[i]->long_term_pic_num == long_term_pic_num)
1876             return i;
1877     }
1878     GST_ERROR("found no long-term reference picture with LongTermPicNum = %d",
1879               long_term_pic_num);
1880     return -1;
1881 }
1882
1883 static void
1884 exec_picture_refs_modification_1(
1885     GstVaapiDecoderH264           *decoder,
1886     GstVaapiPictureH264           *picture,
1887     GstH264SliceHdr               *slice_hdr,
1888     guint                          list
1889 )
1890 {
1891     GstVaapiDecoderH264Private * const priv = decoder->priv;
1892     GstH264PPS * const pps = slice_hdr->pps;
1893     GstH264SPS * const sps = pps->sequence;
1894     GstH264RefPicListModification *ref_pic_list_modification;
1895     guint num_ref_pic_list_modifications;
1896     GstVaapiPictureH264 **ref_list;
1897     guint *ref_list_count_ptr, ref_list_count, ref_list_idx = 0;
1898     guint i, j, n, num_refs;
1899     gint found_ref_idx;
1900     gint32 MaxPicNum, CurrPicNum, picNumPred;
1901
1902     GST_DEBUG("modification process of reference picture list %u", list);
1903
1904     if (list == 0) {
1905         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l0;
1906         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l0;
1907         ref_list                       = priv->RefPicList0;
1908         ref_list_count_ptr             = &priv->RefPicList0_count;
1909         num_refs                       = slice_hdr->num_ref_idx_l0_active_minus1 + 1;
1910     }
1911     else {
1912         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l1;
1913         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l1;
1914         ref_list                       = priv->RefPicList1;
1915         ref_list_count_ptr             = &priv->RefPicList1_count;
1916         num_refs                       = slice_hdr->num_ref_idx_l1_active_minus1 + 1;
1917     }
1918     ref_list_count = *ref_list_count_ptr;
1919
1920     if (!GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1921         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 5); // 2 * MaxFrameNum
1922         CurrPicNum = 2 * slice_hdr->frame_num + 1;              // 2 * frame_num + 1
1923     }
1924     else {
1925         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 4); // MaxFrameNum
1926         CurrPicNum = slice_hdr->frame_num;                      // frame_num
1927     }
1928
1929     picNumPred = CurrPicNum;
1930
1931     for (i = 0; i < num_ref_pic_list_modifications; i++) {
1932         GstH264RefPicListModification * const l = &ref_pic_list_modification[i];
1933         if (l->modification_of_pic_nums_idc == 3)
1934             break;
1935
1936         /* 8.2.4.3.1 - Short-term reference pictures */
1937         if (l->modification_of_pic_nums_idc == 0 || l->modification_of_pic_nums_idc == 1) {
1938             gint32 abs_diff_pic_num = l->value.abs_diff_pic_num_minus1 + 1;
1939             gint32 picNum, picNumNoWrap;
1940
1941             // (8-34)
1942             if (l->modification_of_pic_nums_idc == 0) {
1943                 picNumNoWrap = picNumPred - abs_diff_pic_num;
1944                 if (picNumNoWrap < 0)
1945                     picNumNoWrap += MaxPicNum;
1946             }
1947
1948             // (8-35)
1949             else {
1950                 picNumNoWrap = picNumPred + abs_diff_pic_num;
1951                 if (picNumNoWrap >= MaxPicNum)
1952                     picNumNoWrap -= MaxPicNum;
1953             }
1954             picNumPred = picNumNoWrap;
1955
1956             // (8-36)
1957             picNum = picNumNoWrap;
1958             if (picNum > CurrPicNum)
1959                 picNum -= MaxPicNum;
1960
1961             // (8-37)
1962             for (j = num_refs; j > ref_list_idx; j--)
1963                 ref_list[j] = ref_list[j - 1];
1964             found_ref_idx = find_short_term_reference(decoder, picNum);
1965             ref_list[ref_list_idx++] =
1966                 found_ref_idx >= 0 ? priv->short_ref[found_ref_idx] : NULL;
1967             n = ref_list_idx;
1968             for (j = ref_list_idx; j <= num_refs; j++) {
1969                 gint32 PicNumF;
1970                 if (!ref_list[j])
1971                     continue;
1972                 PicNumF =
1973                     GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(ref_list[j]) ?
1974                     ref_list[j]->pic_num : MaxPicNum;
1975                 if (PicNumF != picNum)
1976                     ref_list[n++] = ref_list[j];
1977             }
1978         }
1979
1980         /* 8.2.4.3.2 - Long-term reference pictures */
1981         else {
1982
1983             for (j = num_refs; j > ref_list_idx; j--)
1984                 ref_list[j] = ref_list[j - 1];
1985             found_ref_idx =
1986                 find_long_term_reference(decoder, l->value.long_term_pic_num);
1987             ref_list[ref_list_idx++] =
1988                 found_ref_idx >= 0 ? priv->long_ref[found_ref_idx] : NULL;
1989             n = ref_list_idx;
1990             for (j = ref_list_idx; j <= num_refs; j++) {
1991                 gint32 LongTermPicNumF;
1992                 if (!ref_list[j])
1993                     continue;
1994                 LongTermPicNumF =
1995                     GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(ref_list[j]) ?
1996                     ref_list[j]->long_term_pic_num : INT_MAX;
1997                 if (LongTermPicNumF != l->value.long_term_pic_num)
1998                     ref_list[n++] = ref_list[j];
1999             }
2000         }
2001     }
2002
2003 #if DEBUG
2004     for (i = 0; i < num_refs; i++)
2005         if (!ref_list[i])
2006             GST_ERROR("list %u entry %u is empty", list, i);
2007 #endif
2008     *ref_list_count_ptr = num_refs;
2009 }
2010
2011 /* 8.2.4.3 - Modification process for reference picture lists */
2012 static void
2013 exec_picture_refs_modification(
2014     GstVaapiDecoderH264 *decoder,
2015     GstVaapiPictureH264 *picture,
2016     GstH264SliceHdr     *slice_hdr
2017 )
2018 {
2019     GST_DEBUG("execute ref_pic_list_modification()");
2020
2021     /* RefPicList0 */
2022     if (!GST_H264_IS_I_SLICE(slice_hdr) && !GST_H264_IS_SI_SLICE(slice_hdr) &&
2023         slice_hdr->ref_pic_list_modification_flag_l0)
2024         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 0);
2025
2026     /* RefPicList1 */
2027     if (GST_H264_IS_B_SLICE(slice_hdr) &&
2028         slice_hdr->ref_pic_list_modification_flag_l1)
2029         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 1);
2030 }
2031
2032 static void
2033 init_picture_ref_lists(GstVaapiDecoderH264 *decoder)
2034 {
2035     GstVaapiDecoderH264Private * const priv = decoder->priv;
2036     guint i, j, short_ref_count, long_ref_count;
2037
2038     short_ref_count = 0;
2039     long_ref_count  = 0;
2040     if (GST_VAAPI_PICTURE_IS_FRAME(priv->current_picture)) {
2041         for (i = 0; i < priv->dpb_count; i++) {
2042             GstVaapiFrameStore * const fs = priv->dpb[i];
2043             GstVaapiPictureH264 *picture;
2044             if (!gst_vaapi_frame_store_has_frame(fs))
2045                 continue;
2046             picture = fs->buffers[0];
2047             if (GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture))
2048                 priv->short_ref[short_ref_count++] = picture;
2049             else if (GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture))
2050                 priv->long_ref[long_ref_count++] = picture;
2051             picture->structure = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
2052             picture->other_field = fs->buffers[1];
2053         }
2054     }
2055     else {
2056         for (i = 0; i < priv->dpb_count; i++) {
2057             GstVaapiFrameStore * const fs = priv->dpb[i];
2058             for (j = 0; j < fs->num_buffers; j++) {
2059                 GstVaapiPictureH264 * const picture = fs->buffers[j];
2060                 if (GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture))
2061                     priv->short_ref[short_ref_count++] = picture;
2062                 else if (GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture))
2063                     priv->long_ref[long_ref_count++] = picture;
2064                 picture->structure = picture->base.structure;
2065                 picture->other_field = fs->buffers[j ^ 1];
2066             }
2067         }
2068     }
2069
2070     for (i = short_ref_count; i < priv->short_ref_count; i++)
2071         priv->short_ref[i] = NULL;
2072     priv->short_ref_count = short_ref_count;
2073
2074     for (i = long_ref_count; i < priv->long_ref_count; i++)
2075         priv->long_ref[i] = NULL;
2076     priv->long_ref_count = long_ref_count;
2077 }
2078
2079 static void
2080 init_picture_refs(
2081     GstVaapiDecoderH264 *decoder,
2082     GstVaapiPictureH264 *picture,
2083     GstH264SliceHdr     *slice_hdr
2084 )
2085 {
2086     GstVaapiDecoderH264Private * const priv = decoder->priv;
2087     GstVaapiPicture * const base_picture = &picture->base;
2088     guint i, num_refs;
2089
2090     init_picture_ref_lists(decoder);
2091     init_picture_refs_pic_num(decoder, picture, slice_hdr);
2092
2093     priv->RefPicList0_count = 0;
2094     priv->RefPicList1_count = 0;
2095
2096     switch (base_picture->type) {
2097     case GST_VAAPI_PICTURE_TYPE_P:
2098     case GST_VAAPI_PICTURE_TYPE_SP:
2099         init_picture_refs_p_slice(decoder, picture, slice_hdr);
2100         break;
2101     case GST_VAAPI_PICTURE_TYPE_B:
2102         init_picture_refs_b_slice(decoder, picture, slice_hdr);
2103         break;
2104     default:
2105         break;
2106     }
2107
2108     exec_picture_refs_modification(decoder, picture, slice_hdr);
2109
2110     switch (base_picture->type) {
2111     case GST_VAAPI_PICTURE_TYPE_B:
2112         num_refs = 1 + slice_hdr->num_ref_idx_l1_active_minus1;
2113         for (i = priv->RefPicList1_count; i < num_refs; i++)
2114             priv->RefPicList1[i] = NULL;
2115         priv->RefPicList1_count = num_refs;
2116
2117         // fall-through
2118     case GST_VAAPI_PICTURE_TYPE_P:
2119     case GST_VAAPI_PICTURE_TYPE_SP:
2120         num_refs = 1 + slice_hdr->num_ref_idx_l0_active_minus1;
2121         for (i = priv->RefPicList0_count; i < num_refs; i++)
2122             priv->RefPicList0[i] = NULL;
2123         priv->RefPicList0_count = num_refs;
2124         break;
2125     default:
2126         break;
2127     }
2128 }
2129
2130 static gboolean
2131 init_picture(
2132     GstVaapiDecoderH264 *decoder,
2133     GstVaapiPictureH264 *picture,
2134     GstH264SliceHdr     *slice_hdr,
2135     GstH264NalUnit      *nalu
2136 )
2137 {
2138     GstVaapiDecoderH264Private * const priv = decoder->priv;
2139     GstVaapiPicture * const base_picture = &picture->base;
2140
2141     priv->prev_frame_num        = priv->frame_num;
2142     priv->frame_num             = slice_hdr->frame_num;
2143     picture->frame_num          = priv->frame_num;
2144     picture->frame_num_wrap     = priv->frame_num;
2145     picture->output_flag        = TRUE; /* XXX: conformant to Annex A only */
2146     base_picture->pts           = gst_adapter_prev_timestamp(priv->adapter, NULL);
2147
2148     /* Reset decoder state for IDR pictures */
2149     if (nalu->type == GST_H264_NAL_SLICE_IDR) {
2150         GST_DEBUG("<IDR>");
2151         GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_IDR);
2152         dpb_flush(decoder);
2153     }
2154
2155     /* Initialize slice type */
2156     switch (slice_hdr->type % 5) {
2157     case GST_H264_P_SLICE:
2158         base_picture->type = GST_VAAPI_PICTURE_TYPE_P;
2159         break;
2160     case GST_H264_B_SLICE:
2161         base_picture->type = GST_VAAPI_PICTURE_TYPE_B;
2162         break;
2163     case GST_H264_I_SLICE:
2164         base_picture->type = GST_VAAPI_PICTURE_TYPE_I;
2165         break;
2166     case GST_H264_SP_SLICE:
2167         base_picture->type = GST_VAAPI_PICTURE_TYPE_SP;
2168         break;
2169     case GST_H264_SI_SLICE:
2170         base_picture->type = GST_VAAPI_PICTURE_TYPE_SI;
2171         break;
2172     }
2173
2174     /* Initialize picture structure */
2175     if (!slice_hdr->field_pic_flag)
2176         base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
2177     else {
2178         GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_INTERLACED);
2179         if (!slice_hdr->bottom_field_flag)
2180             base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD;
2181         else
2182             base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD;
2183     }
2184     picture->structure = base_picture->structure;
2185
2186     /* Initialize reference flags */
2187     if (nalu->ref_idc) {
2188         GstH264DecRefPicMarking * const dec_ref_pic_marking =
2189             &slice_hdr->dec_ref_pic_marking;
2190
2191         if (GST_VAAPI_PICTURE_IS_IDR(picture) &&
2192             dec_ref_pic_marking->long_term_reference_flag)
2193             GST_VAAPI_PICTURE_FLAG_SET(picture,
2194                 GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE);
2195         else
2196             GST_VAAPI_PICTURE_FLAG_SET(picture,
2197                 GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE);
2198     }
2199
2200     init_picture_poc(decoder, picture, slice_hdr);
2201     init_picture_refs(decoder, picture, slice_hdr);
2202     return TRUE;
2203 }
2204
2205 /* 8.2.5.3 - Sliding window decoded reference picture marking process */
2206 static gboolean
2207 exec_ref_pic_marking_sliding_window(GstVaapiDecoderH264 *decoder)
2208 {
2209     GstVaapiDecoderH264Private * const priv = decoder->priv;
2210     GstH264PPS * const pps = priv->current_picture->pps;
2211     GstH264SPS * const sps = pps->sequence;
2212     GstVaapiPictureH264 *ref_picture;
2213     guint i, m, max_num_ref_frames;
2214
2215     GST_DEBUG("reference picture marking process (sliding window)");
2216
2217     if (!GST_VAAPI_PICTURE_IS_FIRST_FIELD(priv->current_picture))
2218         return TRUE;
2219
2220     max_num_ref_frames = sps->num_ref_frames;
2221     if (max_num_ref_frames == 0)
2222         max_num_ref_frames = 1;
2223     if (!GST_VAAPI_PICTURE_IS_FRAME(priv->current_picture))
2224         max_num_ref_frames <<= 1;
2225
2226     if (priv->short_ref_count + priv->long_ref_count < max_num_ref_frames)
2227         return TRUE;
2228     if (priv->short_ref_count < 1)
2229         return FALSE;
2230
2231     for (m = 0, i = 1; i < priv->short_ref_count; i++) {
2232         GstVaapiPictureH264 * const picture = priv->short_ref[i];
2233         if (picture->frame_num_wrap < priv->short_ref[m]->frame_num_wrap)
2234             m = i;
2235     }
2236
2237     ref_picture = priv->short_ref[m];
2238     gst_vaapi_picture_h264_set_reference(ref_picture, 0, TRUE);
2239     ARRAY_REMOVE_INDEX(priv->short_ref, m);
2240
2241     /* Both fields need to be marked as "unused for reference", so
2242        remove the other field from the short_ref[] list as well */
2243     if (!GST_VAAPI_PICTURE_IS_FRAME(priv->current_picture) && ref_picture->other_field) {
2244         for (i = 0; i < priv->short_ref_count; i++) {
2245             if (priv->short_ref[i] == ref_picture->other_field) {
2246                 ARRAY_REMOVE_INDEX(priv->short_ref, i);
2247                 break;
2248             }
2249         }
2250     }
2251     return TRUE;
2252 }
2253
2254 static inline gint32
2255 get_picNumX(GstVaapiPictureH264 *picture, GstH264RefPicMarking *ref_pic_marking)
2256 {
2257     gint32 pic_num;
2258
2259     if (GST_VAAPI_PICTURE_IS_FRAME(picture))
2260         pic_num = picture->frame_num_wrap;
2261     else
2262         pic_num = 2 * picture->frame_num_wrap + 1;
2263     pic_num -= ref_pic_marking->difference_of_pic_nums_minus1 + 1;
2264     return pic_num;
2265 }
2266
2267 /* 8.2.5.4.1. Mark short-term reference picture as "unused for reference" */
2268 static void
2269 exec_ref_pic_marking_adaptive_mmco_1(
2270     GstVaapiDecoderH264  *decoder,
2271     GstVaapiPictureH264  *picture,
2272     GstH264RefPicMarking *ref_pic_marking
2273 )
2274 {
2275     GstVaapiDecoderH264Private * const priv = decoder->priv;
2276     gint32 i, picNumX;
2277
2278     picNumX = get_picNumX(picture, ref_pic_marking);
2279     i = find_short_term_reference(decoder, picNumX);
2280     if (i < 0)
2281         return;
2282
2283     gst_vaapi_picture_h264_set_reference(priv->short_ref[i], 0,
2284         GST_VAAPI_PICTURE_IS_FRAME(picture));
2285     ARRAY_REMOVE_INDEX(priv->short_ref, i);
2286 }
2287
2288 /* 8.2.5.4.2. Mark long-term reference picture as "unused for reference" */
2289 static void
2290 exec_ref_pic_marking_adaptive_mmco_2(
2291     GstVaapiDecoderH264  *decoder,
2292     GstVaapiPictureH264  *picture,
2293     GstH264RefPicMarking *ref_pic_marking
2294 )
2295 {
2296     GstVaapiDecoderH264Private * const priv = decoder->priv;
2297     gint32 i;
2298
2299     i = find_long_term_reference(decoder, ref_pic_marking->long_term_pic_num);
2300     if (i < 0)
2301         return;
2302
2303     gst_vaapi_picture_h264_set_reference(priv->long_ref[i], 0,
2304         GST_VAAPI_PICTURE_IS_FRAME(picture));
2305     ARRAY_REMOVE_INDEX(priv->long_ref, i);
2306 }
2307
2308 /* 8.2.5.4.3. Assign LongTermFrameIdx to a short-term reference picture */
2309 static void
2310 exec_ref_pic_marking_adaptive_mmco_3(
2311     GstVaapiDecoderH264  *decoder,
2312     GstVaapiPictureH264  *picture,
2313     GstH264RefPicMarking *ref_pic_marking
2314 )
2315 {
2316     GstVaapiDecoderH264Private * const priv = decoder->priv;
2317     GstVaapiPictureH264 *ref_picture;
2318     gint32 i, picNumX;
2319
2320     for (i = 0; i < priv->long_ref_count; i++) {
2321         if (priv->long_ref[i]->long_term_frame_idx == ref_pic_marking->long_term_frame_idx)
2322             break;
2323     }
2324     if (i != priv->long_ref_count) {
2325         gst_vaapi_picture_h264_set_reference(priv->long_ref[i], 0, TRUE);
2326         ARRAY_REMOVE_INDEX(priv->long_ref, i);
2327     }
2328
2329     picNumX = get_picNumX(picture, ref_pic_marking);
2330     i = find_short_term_reference(decoder, picNumX);
2331     if (i < 0)
2332         return;
2333
2334     ref_picture = priv->short_ref[i];
2335     ARRAY_REMOVE_INDEX(priv->short_ref, i);
2336     priv->long_ref[priv->long_ref_count++] = ref_picture;
2337
2338     ref_picture->long_term_frame_idx = ref_pic_marking->long_term_frame_idx;
2339     gst_vaapi_picture_h264_set_reference(ref_picture,
2340         GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE,
2341         GST_VAAPI_PICTURE_IS_FRAME(picture));
2342 }
2343
2344 /* 8.2.5.4.4. Mark pictures with LongTermFramIdx > max_long_term_frame_idx
2345  * as "unused for reference" */
2346 static void
2347 exec_ref_pic_marking_adaptive_mmco_4(
2348     GstVaapiDecoderH264  *decoder,
2349     GstVaapiPictureH264  *picture,
2350     GstH264RefPicMarking *ref_pic_marking
2351 )
2352 {
2353     GstVaapiDecoderH264Private * const priv = decoder->priv;
2354     gint32 i, long_term_frame_idx;
2355
2356     long_term_frame_idx = ref_pic_marking->max_long_term_frame_idx_plus1 - 1;
2357
2358     for (i = 0; i < priv->long_ref_count; i++) {
2359         if (priv->long_ref[i]->long_term_frame_idx <= long_term_frame_idx)
2360             continue;
2361         gst_vaapi_picture_h264_set_reference(priv->long_ref[i], 0, FALSE);
2362         ARRAY_REMOVE_INDEX(priv->long_ref, i);
2363         i--;
2364     }
2365 }
2366
2367 /* 8.2.5.4.5. Mark all reference pictures as "unused for reference" */
2368 static void
2369 exec_ref_pic_marking_adaptive_mmco_5(
2370     GstVaapiDecoderH264  *decoder,
2371     GstVaapiPictureH264  *picture,
2372     GstH264RefPicMarking *ref_pic_marking
2373 )
2374 {
2375     GstVaapiDecoderH264Private * const priv = decoder->priv;
2376
2377     dpb_flush(decoder);
2378
2379     priv->prev_pic_has_mmco5 = TRUE;
2380
2381     /* The picture shall be inferred to have had frame_num equal to 0 (7.4.3) */
2382     priv->frame_num = 0;
2383     priv->frame_num_offset = 0;
2384     picture->frame_num = 0;
2385
2386     /* Update TopFieldOrderCnt and BottomFieldOrderCnt (8.2.1) */
2387     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
2388         picture->field_poc[TOP_FIELD] -= picture->base.poc;
2389     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
2390         picture->field_poc[BOTTOM_FIELD] -= picture->base.poc;
2391     picture->base.poc = 0;
2392 }
2393
2394 /* 8.2.5.4.6. Assign a long-term frame index to the current picture */
2395 static void
2396 exec_ref_pic_marking_adaptive_mmco_6(
2397     GstVaapiDecoderH264  *decoder,
2398     GstVaapiPictureH264  *picture,
2399     GstH264RefPicMarking *ref_pic_marking
2400 )
2401 {
2402     picture->long_term_frame_idx = ref_pic_marking->long_term_frame_idx;
2403     gst_vaapi_picture_h264_set_reference(picture,
2404         GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE, FALSE);
2405 }
2406
2407 /* 8.2.5.4. Adaptive memory control decoded reference picture marking process */
2408 static gboolean
2409 exec_ref_pic_marking_adaptive(
2410     GstVaapiDecoderH264     *decoder,
2411     GstVaapiPictureH264     *picture,
2412     GstH264DecRefPicMarking *dec_ref_pic_marking
2413 )
2414 {
2415     guint i;
2416
2417     GST_DEBUG("reference picture marking process (adaptive memory control)");
2418
2419     typedef void (*exec_ref_pic_marking_adaptive_mmco_func)(
2420         GstVaapiDecoderH264  *decoder,
2421         GstVaapiPictureH264  *picture,
2422         GstH264RefPicMarking *ref_pic_marking
2423     );
2424
2425     static const exec_ref_pic_marking_adaptive_mmco_func mmco_funcs[] = {
2426         NULL,
2427         exec_ref_pic_marking_adaptive_mmco_1,
2428         exec_ref_pic_marking_adaptive_mmco_2,
2429         exec_ref_pic_marking_adaptive_mmco_3,
2430         exec_ref_pic_marking_adaptive_mmco_4,
2431         exec_ref_pic_marking_adaptive_mmco_5,
2432         exec_ref_pic_marking_adaptive_mmco_6,
2433     };
2434
2435     for (i = 0; i < dec_ref_pic_marking->n_ref_pic_marking; i++) {
2436         GstH264RefPicMarking * const ref_pic_marking =
2437             &dec_ref_pic_marking->ref_pic_marking[i];
2438
2439         const guint mmco = ref_pic_marking->memory_management_control_operation;
2440         if (mmco < G_N_ELEMENTS(mmco_funcs) && mmco_funcs[mmco])
2441             mmco_funcs[mmco](decoder, picture, ref_pic_marking);
2442         else {
2443             GST_ERROR("unhandled MMCO %u", mmco);
2444             return FALSE;
2445         }
2446     }
2447     return TRUE;
2448 }
2449
2450 /* 8.2.5 - Execute reference picture marking process */
2451 static gboolean
2452 exec_ref_pic_marking(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
2453 {
2454     GstVaapiDecoderH264Private * const priv = decoder->priv;
2455
2456     priv->prev_pic_has_mmco5 = FALSE;
2457     priv->prev_pic_structure = picture->structure;
2458
2459     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
2460         return TRUE;
2461
2462     if (!GST_VAAPI_PICTURE_IS_IDR(picture)) {
2463         GstVaapiSliceH264 * const slice =
2464             gst_vaapi_picture_h264_get_last_slice(picture);
2465         GstH264DecRefPicMarking * const dec_ref_pic_marking =
2466             &slice->slice_hdr.dec_ref_pic_marking;
2467         if (dec_ref_pic_marking->adaptive_ref_pic_marking_mode_flag) {
2468             if (!exec_ref_pic_marking_adaptive(decoder, picture, dec_ref_pic_marking))
2469                 return FALSE;
2470         }
2471         else {
2472             if (!exec_ref_pic_marking_sliding_window(decoder))
2473                 return FALSE;
2474         }
2475     }
2476     return TRUE;
2477 }
2478
2479 static void
2480 vaapi_init_picture(VAPictureH264 *pic)
2481 {
2482     pic->picture_id           = VA_INVALID_ID;
2483     pic->frame_idx            = 0;
2484     pic->flags                = VA_PICTURE_H264_INVALID;
2485     pic->TopFieldOrderCnt     = 0;
2486     pic->BottomFieldOrderCnt  = 0;
2487 }
2488
2489 static void
2490 vaapi_fill_picture(VAPictureH264 *pic, GstVaapiPictureH264 *picture,
2491     guint picture_structure)
2492 {
2493     if (!picture_structure)
2494         picture_structure = picture->structure;
2495
2496     pic->picture_id = picture->base.surface_id;
2497     pic->flags = 0;
2498
2499     if (GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture)) {
2500         pic->flags |= VA_PICTURE_H264_LONG_TERM_REFERENCE;
2501         pic->frame_idx = picture->long_term_frame_idx;
2502     }
2503     else {
2504         if (GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture))
2505             pic->flags |= VA_PICTURE_H264_SHORT_TERM_REFERENCE;
2506         pic->frame_idx = picture->frame_num;
2507     }
2508
2509     switch (picture_structure) {
2510     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
2511         pic->TopFieldOrderCnt = picture->field_poc[TOP_FIELD];
2512         pic->BottomFieldOrderCnt = picture->field_poc[BOTTOM_FIELD];
2513         break;
2514     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
2515         pic->flags |= VA_PICTURE_H264_TOP_FIELD;
2516         pic->TopFieldOrderCnt = picture->field_poc[TOP_FIELD];
2517         pic->BottomFieldOrderCnt = 0;
2518         break;
2519     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
2520         pic->flags |= VA_PICTURE_H264_BOTTOM_FIELD;
2521         pic->BottomFieldOrderCnt = picture->field_poc[BOTTOM_FIELD];
2522         pic->TopFieldOrderCnt = 0;
2523         break;
2524     }
2525 }
2526
2527 static gboolean
2528 fill_picture(
2529     GstVaapiDecoderH264 *decoder,
2530     GstVaapiPictureH264 *picture,
2531     GstH264SliceHdr     *slice_hdr,
2532     GstH264NalUnit      *nalu
2533 )
2534 {
2535     GstVaapiDecoderH264Private * const priv = decoder->priv;
2536     GstVaapiPicture * const base_picture = &picture->base;
2537     GstH264PPS * const pps = picture->pps;
2538     GstH264SPS * const sps = pps->sequence;
2539     VAPictureParameterBufferH264 * const pic_param = base_picture->param;
2540     guint i, n;
2541
2542     /* Fill in VAPictureParameterBufferH264 */
2543     vaapi_fill_picture(&pic_param->CurrPic, picture, 0);
2544
2545     for (i = 0, n = 0; i < priv->dpb_count; i++) {
2546         GstVaapiFrameStore * const fs = priv->dpb[i];
2547         if (gst_vaapi_frame_store_has_reference(fs))
2548             vaapi_fill_picture(&pic_param->ReferenceFrames[n++],
2549                 fs->buffers[0], fs->structure);
2550     }
2551     for (; n < G_N_ELEMENTS(pic_param->ReferenceFrames); n++)
2552         vaapi_init_picture(&pic_param->ReferenceFrames[n]);
2553
2554 #define COPY_FIELD(s, f) \
2555     pic_param->f = (s)->f
2556
2557 #define COPY_BFM(a, s, f) \
2558     pic_param->a.bits.f = (s)->f
2559
2560     pic_param->picture_width_in_mbs_minus1  = priv->mb_width - 1;
2561     pic_param->picture_height_in_mbs_minus1 = priv->mb_height - 1;
2562     pic_param->frame_num                    = priv->frame_num;
2563
2564     COPY_FIELD(sps, bit_depth_luma_minus8);
2565     COPY_FIELD(sps, bit_depth_chroma_minus8);
2566     COPY_FIELD(sps, num_ref_frames);
2567     COPY_FIELD(pps, num_slice_groups_minus1);
2568     COPY_FIELD(pps, slice_group_map_type);
2569     COPY_FIELD(pps, slice_group_change_rate_minus1);
2570     COPY_FIELD(pps, pic_init_qp_minus26);
2571     COPY_FIELD(pps, pic_init_qs_minus26);
2572     COPY_FIELD(pps, chroma_qp_index_offset);
2573     COPY_FIELD(pps, second_chroma_qp_index_offset);
2574
2575     pic_param->seq_fields.value                                         = 0; /* reset all bits */
2576     pic_param->seq_fields.bits.residual_colour_transform_flag           = sps->separate_colour_plane_flag;
2577     pic_param->seq_fields.bits.MinLumaBiPredSize8x8                     = sps->level_idc >= 31; /* A.3.3.2 */
2578
2579     COPY_BFM(seq_fields, sps, chroma_format_idc);
2580     COPY_BFM(seq_fields, sps, gaps_in_frame_num_value_allowed_flag);
2581     COPY_BFM(seq_fields, sps, frame_mbs_only_flag);
2582     COPY_BFM(seq_fields, sps, mb_adaptive_frame_field_flag);
2583     COPY_BFM(seq_fields, sps, direct_8x8_inference_flag);
2584     COPY_BFM(seq_fields, sps, log2_max_frame_num_minus4);
2585     COPY_BFM(seq_fields, sps, pic_order_cnt_type);
2586     COPY_BFM(seq_fields, sps, log2_max_pic_order_cnt_lsb_minus4);
2587     COPY_BFM(seq_fields, sps, delta_pic_order_always_zero_flag);
2588
2589     pic_param->pic_fields.value                                         = 0; /* reset all bits */
2590     pic_param->pic_fields.bits.field_pic_flag                           = slice_hdr->field_pic_flag;
2591     pic_param->pic_fields.bits.reference_pic_flag                       = GST_VAAPI_PICTURE_IS_REFERENCE(picture);
2592
2593     COPY_BFM(pic_fields, pps, entropy_coding_mode_flag);
2594     COPY_BFM(pic_fields, pps, weighted_pred_flag);
2595     COPY_BFM(pic_fields, pps, weighted_bipred_idc);
2596     COPY_BFM(pic_fields, pps, transform_8x8_mode_flag);
2597     COPY_BFM(pic_fields, pps, constrained_intra_pred_flag);
2598     COPY_BFM(pic_fields, pps, pic_order_present_flag);
2599     COPY_BFM(pic_fields, pps, deblocking_filter_control_present_flag);
2600     COPY_BFM(pic_fields, pps, redundant_pic_cnt_present_flag);
2601     return TRUE;
2602 }
2603
2604 /* Detection of the first VCL NAL unit of a primary coded picture (7.4.1.2.4) */
2605 static gboolean
2606 is_new_picture(
2607     GstVaapiDecoderH264 *decoder,
2608     GstH264NalUnit      *nalu,
2609     GstH264SliceHdr     *slice_hdr
2610 )
2611 {
2612     GstVaapiDecoderH264Private * const priv = decoder->priv;
2613     GstH264PPS * const pps = slice_hdr->pps;
2614     GstH264SPS * const sps = pps->sequence;
2615     GstVaapiSliceH264 *slice;
2616     GstH264SliceHdr *prev_slice_hdr;
2617
2618     if (!priv->current_picture)
2619         return TRUE;
2620
2621     slice = gst_vaapi_picture_h264_get_last_slice(priv->current_picture);
2622     if (!slice)
2623         return FALSE;
2624     prev_slice_hdr = &slice->slice_hdr;
2625
2626 #define CHECK_EXPR(expr, field_name) do {              \
2627         if (!(expr)) {                                 \
2628             GST_DEBUG(field_name " differs in value"); \
2629             return TRUE;                               \
2630         }                                              \
2631     } while (0)
2632
2633 #define CHECK_VALUE(new_slice_hdr, old_slice_hdr, field) \
2634     CHECK_EXPR(((new_slice_hdr)->field == (old_slice_hdr)->field), #field)
2635
2636     /* frame_num differs in value, regardless of inferred values to 0 */
2637     CHECK_VALUE(slice_hdr, prev_slice_hdr, frame_num);
2638
2639     /* pic_parameter_set_id differs in value */
2640     CHECK_VALUE(slice_hdr, prev_slice_hdr, pps);
2641
2642     /* field_pic_flag differs in value */
2643     CHECK_VALUE(slice_hdr, prev_slice_hdr, field_pic_flag);
2644
2645     /* bottom_field_flag is present in both and differs in value */
2646     if (slice_hdr->field_pic_flag && prev_slice_hdr->field_pic_flag)
2647         CHECK_VALUE(slice_hdr, prev_slice_hdr, bottom_field_flag);
2648
2649     /* nal_ref_idc differs in value with one of the nal_ref_idc values is 0 */
2650     CHECK_EXPR(((GST_VAAPI_PICTURE_IS_REFERENCE(priv->current_picture) ^
2651                  (nalu->ref_idc != 0)) == 0), "nal_ref_idc");
2652
2653     /* POC type is 0 for both and either pic_order_cnt_lsb differs in
2654        value or delta_pic_order_cnt_bottom differs in value */
2655     if (sps->pic_order_cnt_type == 0) {
2656         CHECK_VALUE(slice_hdr, prev_slice_hdr, pic_order_cnt_lsb);
2657         if (pps->pic_order_present_flag && !slice_hdr->field_pic_flag)
2658             CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt_bottom);
2659     }
2660
2661     /* POC type is 1 for both and either delta_pic_order_cnt[0]
2662        differs in value or delta_pic_order_cnt[1] differs in value */
2663     else if (sps->pic_order_cnt_type == 1) {
2664         CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt[0]);
2665         CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt[1]);
2666     }
2667
2668     /* IdrPicFlag differs in value */
2669     CHECK_EXPR(((GST_VAAPI_PICTURE_IS_IDR(priv->current_picture) ^
2670                  (nalu->type == GST_H264_NAL_SLICE_IDR)) == 0), "IdrPicFlag");
2671
2672     /* IdrPicFlag is equal to 1 for both and idr_pic_id differs in value */
2673     if (GST_VAAPI_PICTURE_IS_IDR(priv->current_picture))
2674         CHECK_VALUE(slice_hdr, prev_slice_hdr, idr_pic_id);
2675
2676 #undef CHECK_EXPR
2677 #undef CHECK_VALUE
2678     return FALSE;
2679 }
2680
2681 static GstVaapiDecoderStatus
2682 decode_picture(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu, GstH264SliceHdr *slice_hdr)
2683 {
2684     GstVaapiDecoderH264Private * const priv = decoder->priv;
2685     GstVaapiPictureH264 *picture;
2686     GstVaapiDecoderStatus status;
2687     GstH264PPS * const pps = slice_hdr->pps;
2688     GstH264SPS * const sps = pps->sequence;
2689
2690     status = decode_current_picture(decoder);
2691     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2692         return status;
2693
2694     status = ensure_context(decoder, sps);
2695     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2696         return status;
2697
2698     if (priv->current_picture) {
2699         /* Re-use current picture where the first field was decoded */
2700         picture = gst_vaapi_picture_h264_new_field(priv->current_picture);
2701         if (!picture) {
2702             GST_ERROR("failed to allocate field picture");
2703             return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2704         }
2705     }
2706     else {
2707         /* Create new picture */
2708         picture = gst_vaapi_picture_h264_new(decoder);
2709         if (!picture) {
2710             GST_ERROR("failed to allocate picture");
2711             return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2712         }
2713     }
2714     gst_vaapi_picture_replace(&priv->current_picture, picture);
2715     gst_vaapi_picture_unref(picture);
2716
2717     picture->pps = pps;
2718
2719     status = ensure_quant_matrix(decoder, picture);
2720     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS) {
2721         GST_ERROR("failed to reset quantizer matrix");
2722         return status;
2723     }
2724
2725     if (!init_picture(decoder, picture, slice_hdr, nalu))
2726         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2727     if (!fill_picture(decoder, picture, slice_hdr, nalu))
2728         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2729     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2730 }
2731
2732 static inline guint
2733 get_slice_data_bit_offset(GstH264SliceHdr *slice_hdr, GstH264NalUnit *nalu)
2734 {
2735     guint epb_count;
2736
2737     epb_count = slice_hdr->n_emulation_prevention_bytes;
2738     return 8 /* nal_unit_type */ + slice_hdr->header_size - epb_count * 8;
2739 }
2740
2741 static gboolean
2742 fill_pred_weight_table(GstVaapiDecoderH264 *decoder, GstVaapiSliceH264 *slice)
2743 {
2744     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2745     GstH264PPS * const pps = slice_hdr->pps;
2746     GstH264SPS * const sps = pps->sequence;
2747     GstH264PredWeightTable * const w = &slice_hdr->pred_weight_table;
2748     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2749     guint num_weight_tables = 0;
2750     gint i, j;
2751
2752     if (pps->weighted_pred_flag &&
2753         (GST_H264_IS_P_SLICE(slice_hdr) || GST_H264_IS_SP_SLICE(slice_hdr)))
2754         num_weight_tables = 1;
2755     else if (pps->weighted_bipred_idc == 1 && GST_H264_IS_B_SLICE(slice_hdr))
2756         num_weight_tables = 2;
2757     else
2758         num_weight_tables = 0;
2759
2760     slice_param->luma_log2_weight_denom   = w->luma_log2_weight_denom;
2761     slice_param->chroma_log2_weight_denom = w->chroma_log2_weight_denom;
2762     slice_param->luma_weight_l0_flag      = 0;
2763     slice_param->chroma_weight_l0_flag    = 0;
2764     slice_param->luma_weight_l1_flag      = 0;
2765     slice_param->chroma_weight_l1_flag    = 0;
2766
2767     if (num_weight_tables < 1)
2768         return TRUE;
2769
2770     slice_param->luma_weight_l0_flag = 1;
2771     for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2772         slice_param->luma_weight_l0[i] = w->luma_weight_l0[i];
2773         slice_param->luma_offset_l0[i] = w->luma_offset_l0[i];
2774     }
2775
2776     slice_param->chroma_weight_l0_flag = sps->chroma_array_type != 0;
2777     if (slice_param->chroma_weight_l0_flag) {
2778         for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2779             for (j = 0; j < 2; j++) {
2780                 slice_param->chroma_weight_l0[i][j] = w->chroma_weight_l0[i][j];
2781                 slice_param->chroma_offset_l0[i][j] = w->chroma_offset_l0[i][j];
2782             }
2783         }
2784     }
2785
2786     if (num_weight_tables < 2)
2787         return TRUE;
2788
2789     slice_param->luma_weight_l1_flag = 1;
2790     for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2791         slice_param->luma_weight_l1[i] = w->luma_weight_l1[i];
2792         slice_param->luma_offset_l1[i] = w->luma_offset_l1[i];
2793     }
2794
2795     slice_param->chroma_weight_l1_flag = sps->chroma_array_type != 0;
2796     if (slice_param->chroma_weight_l1_flag) {
2797         for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2798             for (j = 0; j < 2; j++) {
2799                 slice_param->chroma_weight_l1[i][j] = w->chroma_weight_l1[i][j];
2800                 slice_param->chroma_offset_l1[i][j] = w->chroma_offset_l1[i][j];
2801             }
2802         }
2803     }
2804     return TRUE;
2805 }
2806
2807 static gboolean
2808 fill_RefPicList(GstVaapiDecoderH264 *decoder, GstVaapiSliceH264 *slice)
2809 {
2810     GstVaapiDecoderH264Private * const priv = decoder->priv;
2811     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2812     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2813     guint i, num_ref_lists = 0;
2814
2815     slice_param->num_ref_idx_l0_active_minus1 = 0;
2816     slice_param->num_ref_idx_l1_active_minus1 = 0;
2817
2818     if (GST_H264_IS_B_SLICE(slice_hdr))
2819         num_ref_lists = 2;
2820     else if (GST_H264_IS_I_SLICE(slice_hdr))
2821         num_ref_lists = 0;
2822     else
2823         num_ref_lists = 1;
2824
2825     if (num_ref_lists < 1)
2826         return TRUE;
2827
2828     slice_param->num_ref_idx_l0_active_minus1 =
2829         slice_hdr->num_ref_idx_l0_active_minus1;
2830
2831     for (i = 0; i < priv->RefPicList0_count && priv->RefPicList0[i]; i++)
2832         vaapi_fill_picture(&slice_param->RefPicList0[i], priv->RefPicList0[i], 0);
2833     for (; i <= slice_param->num_ref_idx_l0_active_minus1; i++)
2834         vaapi_init_picture(&slice_param->RefPicList0[i]);
2835
2836     if (num_ref_lists < 2)
2837         return TRUE;
2838
2839     slice_param->num_ref_idx_l1_active_minus1 =
2840         slice_hdr->num_ref_idx_l1_active_minus1;
2841
2842     for (i = 0; i < priv->RefPicList1_count && priv->RefPicList1[i]; i++)
2843         vaapi_fill_picture(&slice_param->RefPicList1[i], priv->RefPicList1[i], 0);
2844     for (; i <= slice_param->num_ref_idx_l1_active_minus1; i++)
2845         vaapi_init_picture(&slice_param->RefPicList1[i]);
2846     return TRUE;
2847 }
2848
2849 static gboolean
2850 fill_slice(
2851     GstVaapiDecoderH264 *decoder,
2852     GstVaapiSliceH264   *slice,
2853     GstH264NalUnit      *nalu
2854 )
2855 {
2856     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2857     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2858
2859     /* Fill in VASliceParameterBufferH264 */
2860     slice_param->slice_data_bit_offset          = get_slice_data_bit_offset(slice_hdr, nalu);
2861     slice_param->first_mb_in_slice              = slice_hdr->first_mb_in_slice;
2862     slice_param->slice_type                     = slice_hdr->type % 5;
2863     slice_param->direct_spatial_mv_pred_flag    = slice_hdr->direct_spatial_mv_pred_flag;
2864     slice_param->cabac_init_idc                 = slice_hdr->cabac_init_idc;
2865     slice_param->slice_qp_delta                 = slice_hdr->slice_qp_delta;
2866     slice_param->disable_deblocking_filter_idc  = slice_hdr->disable_deblocking_filter_idc;
2867     slice_param->slice_alpha_c0_offset_div2     = slice_hdr->slice_alpha_c0_offset_div2;
2868     slice_param->slice_beta_offset_div2         = slice_hdr->slice_beta_offset_div2;
2869
2870     if (!fill_RefPicList(decoder, slice))
2871         return FALSE;
2872     if (!fill_pred_weight_table(decoder, slice))
2873         return FALSE;
2874     return TRUE;
2875 }
2876
2877 static GstVaapiDecoderStatus
2878 decode_slice(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
2879 {
2880     GstVaapiDecoderH264Private * const priv = decoder->priv;
2881     GstVaapiDecoderStatus status;
2882     GstVaapiPictureH264 *picture;
2883     GstVaapiSliceH264 *slice = NULL;
2884     GstH264SliceHdr *slice_hdr;
2885     GstH264ParserResult result;
2886     gboolean is_first_slice = !priv->has_context;
2887
2888     GST_DEBUG("slice (%u bytes)", nalu->size);
2889
2890     if (is_first_slice) {
2891         slice_hdr = &priv->temp_slice_hdr;
2892         memset(slice_hdr, 0, sizeof(*slice_hdr));
2893         result = gst_h264_parser_parse_slice_hdr(priv->parser, nalu,
2894             slice_hdr, TRUE, TRUE);
2895         if (result != GST_H264_PARSER_OK) {
2896             status = get_status(result);
2897             goto error;
2898         }
2899
2900         status = ensure_context(decoder, slice_hdr->pps->sequence);
2901         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2902             return status;
2903     }
2904
2905     slice = gst_vaapi_slice_h264_new(
2906         decoder,
2907         nalu->data + nalu->offset,
2908         nalu->size
2909     );
2910     if (!slice) {
2911         GST_ERROR("failed to allocate slice");
2912         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2913     }
2914
2915     slice_hdr = &slice->slice_hdr;
2916     if (is_first_slice)
2917         memcpy(slice_hdr, &priv->temp_slice_hdr, sizeof(*slice_hdr));
2918     else {
2919         memset(slice_hdr, 0, sizeof(*slice_hdr));
2920         result = gst_h264_parser_parse_slice_hdr(priv->parser, nalu,
2921             slice_hdr, TRUE, TRUE);
2922         if (result != GST_H264_PARSER_OK) {
2923             status = get_status(result);
2924             goto error;
2925         }
2926     }
2927
2928     if (is_new_picture(decoder, nalu, slice_hdr)) {
2929         status = decode_picture(decoder, nalu, slice_hdr);
2930         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2931             goto error;
2932     }
2933     picture = priv->current_picture;
2934
2935     if (!fill_slice(decoder, slice, nalu)) {
2936         status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2937         goto error;
2938     }
2939     gst_vaapi_picture_add_slice(
2940         GST_VAAPI_PICTURE_CAST(picture),
2941         GST_VAAPI_SLICE_CAST(slice)
2942     );
2943     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2944
2945 error:
2946     if (slice)
2947         gst_mini_object_unref(GST_MINI_OBJECT(slice));
2948     return status;
2949 }
2950
2951 static inline gint
2952 scan_for_start_code(GstAdapter *adapter, guint ofs, guint size, guint32 *scp)
2953 {
2954     return (gint)gst_adapter_masked_scan_uint32_peek(adapter,
2955                                                      0xffffff00, 0x00000100,
2956                                                      ofs, size,
2957                                                      scp);
2958 }
2959
2960 static GstVaapiDecoderStatus
2961 decode_nalu(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
2962 {
2963     GstVaapiDecoderH264Private * const priv = decoder->priv;
2964     GstVaapiDecoderStatus status;
2965
2966     switch (nalu->type) {
2967     case GST_H264_NAL_SLICE_IDR:
2968         /* fall-through. IDR specifics are handled in init_picture() */
2969     case GST_H264_NAL_SLICE:
2970         if (!priv->got_sps || !priv->got_pps)
2971             return GST_VAAPI_DECODER_STATUS_SUCCESS;
2972         status = decode_slice(decoder, nalu);
2973         break;
2974     case GST_H264_NAL_SPS:
2975         status = decode_sps(decoder, nalu);
2976         break;
2977     case GST_H264_NAL_PPS:
2978         status = decode_pps(decoder, nalu);
2979         break;
2980     case GST_H264_NAL_SEI:
2981         status = decode_sei(decoder, nalu);
2982         break;
2983     case GST_H264_NAL_SEQ_END:
2984         status = decode_sequence_end(decoder);
2985         break;
2986     case GST_H264_NAL_AU_DELIMITER:
2987         /* skip all Access Unit NALs */
2988         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2989         break;
2990     case GST_H264_NAL_FILLER_DATA:
2991         /* skip all Filler Data NALs */
2992         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2993         break;
2994     default:
2995         GST_WARNING("unsupported NAL unit type %d", nalu->type);
2996         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
2997         break;
2998     }
2999     return status;
3000 }
3001
3002 static GstVaapiDecoderStatus
3003 decode_buffer(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
3004 {
3005     GstVaapiDecoderH264Private * const priv = decoder->priv;
3006     GstVaapiDecoderStatus status;
3007     GstH264ParserResult result;
3008     GstH264NalUnit nalu;
3009     gboolean is_eos;
3010     const guchar *buf;
3011     guint i, buf_size, nalu_size, size;
3012     guint32 start_code;
3013     gint ofs;
3014
3015     buf      = GST_BUFFER_DATA(buffer);
3016     buf_size = GST_BUFFER_SIZE(buffer);
3017     is_eos   = GST_BUFFER_IS_EOS(buffer);
3018     if (buf && buf_size > 0)
3019         gst_adapter_push(priv->adapter, gst_buffer_ref(buffer));
3020
3021     size = gst_adapter_available(priv->adapter);
3022     do {
3023         if (size == 0) {
3024             status = GST_VAAPI_DECODER_STATUS_SUCCESS;
3025             break;
3026         }
3027
3028         status = gst_vaapi_decoder_check_status(GST_VAAPI_DECODER(decoder));
3029         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
3030             break;
3031
3032         status = GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
3033         if (priv->is_avc) {
3034             if (size < priv->nal_length_size)
3035                 break;
3036             buf = gst_adapter_peek(priv->adapter, priv->nal_length_size);
3037
3038             nalu_size = 0;
3039             for (i = 0; i < priv->nal_length_size; i++)
3040                 nalu_size = (nalu_size << 8) | buf[i];
3041
3042             buf_size = priv->nal_length_size + nalu_size;
3043             if (size < buf_size)
3044                 break;
3045             buffer = gst_adapter_take_buffer(priv->adapter, buf_size);
3046             size -= buf_size;
3047
3048             buf      = GST_BUFFER_DATA(buffer);
3049             buf_size = GST_BUFFER_SIZE(buffer);
3050
3051             result = gst_h264_parser_identify_nalu_avc(
3052                 priv->parser,
3053                 buf, 0, buf_size, priv->nal_length_size,
3054                 &nalu
3055             );
3056         }
3057         else {
3058             if (size < 4)
3059                 break;
3060             ofs = scan_for_start_code(priv->adapter, 0, size, &start_code);
3061             if (ofs < 0)
3062                 break;
3063             gst_adapter_flush(priv->adapter, ofs);
3064             size -= ofs;
3065
3066             ofs = G_UNLIKELY(size < 8) ? -1 :
3067                 scan_for_start_code(priv->adapter, 4, size - 4, NULL);
3068             if (ofs < 0) {
3069                 // Assume the whole NAL unit is present if end-of-stream
3070                 if (!is_eos)
3071                     break;
3072                 ofs = size;
3073             }
3074             buffer = gst_adapter_take_buffer(priv->adapter, ofs);
3075             size -= ofs;
3076
3077             buf      = GST_BUFFER_DATA(buffer);
3078             buf_size = GST_BUFFER_SIZE(buffer);
3079
3080             result = gst_h264_parser_identify_nalu_unchecked(
3081                 priv->parser,
3082                 buf, 0, buf_size,
3083                 &nalu
3084             );
3085         }
3086         status = get_status(result);
3087         if (status == GST_VAAPI_DECODER_STATUS_SUCCESS)
3088             status = decode_nalu(decoder, &nalu);
3089         gst_buffer_unref(buffer);
3090     } while (status == GST_VAAPI_DECODER_STATUS_SUCCESS);
3091
3092     if (is_eos && (status == GST_VAAPI_DECODER_STATUS_SUCCESS ||
3093                    status == GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA))
3094         status = decode_sequence_end(decoder);
3095     return status;
3096 }
3097
3098 static GstVaapiDecoderStatus
3099 decode_codec_data(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
3100 {
3101     GstVaapiDecoderH264Private * const priv = decoder->priv;
3102     GstVaapiDecoderStatus status;
3103     GstH264NalUnit nalu;
3104     GstH264ParserResult result;
3105     guchar *buf;
3106     guint buf_size;
3107     guint i, ofs, num_sps, num_pps;
3108
3109     buf      = GST_BUFFER_DATA(buffer);
3110     buf_size = GST_BUFFER_SIZE(buffer);
3111     if (!buf || buf_size == 0)
3112         return GST_VAAPI_DECODER_STATUS_SUCCESS;
3113
3114     if (buf_size < 8)
3115         return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
3116
3117     if (buf[0] != 1) {
3118         GST_ERROR("failed to decode codec-data, not in avcC format");
3119         return GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
3120     }
3121
3122     priv->nal_length_size = (buf[4] & 0x03) + 1;
3123
3124     num_sps = buf[5] & 0x1f;
3125     ofs = 6;
3126
3127     for (i = 0; i < num_sps; i++) {
3128         result = gst_h264_parser_identify_nalu_avc(
3129             priv->parser,
3130             buf, ofs, buf_size, 2,
3131             &nalu
3132         );
3133         if (result != GST_H264_PARSER_OK)
3134             return get_status(result);
3135
3136         status = decode_sps(decoder, &nalu);
3137         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
3138             return status;
3139         ofs = nalu.offset + nalu.size;
3140     }
3141
3142     num_pps = buf[ofs];
3143     ofs++;
3144
3145     for (i = 0; i < num_pps; i++) {
3146         result = gst_h264_parser_identify_nalu_avc(
3147             priv->parser,
3148             buf, ofs, buf_size, 2,
3149             &nalu
3150         );
3151         if (result != GST_H264_PARSER_OK)
3152             return get_status(result);
3153
3154         status = decode_pps(decoder, &nalu);
3155         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
3156             return status;
3157         ofs = nalu.offset + nalu.size;
3158     }
3159
3160     priv->is_avc = TRUE;
3161     return status;
3162 }
3163
3164 GstVaapiDecoderStatus
3165 gst_vaapi_decoder_h264_decode(GstVaapiDecoder *base, GstBuffer *buffer)
3166 {
3167     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264(base);
3168     GstVaapiDecoderH264Private * const priv = decoder->priv;
3169     GstVaapiDecoderStatus status;
3170     GstBuffer *codec_data;
3171
3172     g_return_val_if_fail(priv->is_constructed,
3173                          GST_VAAPI_DECODER_STATUS_ERROR_INIT_FAILED);
3174
3175     if (!priv->is_opened) {
3176         priv->is_opened = gst_vaapi_decoder_h264_open(decoder, buffer);
3177         if (!priv->is_opened)
3178             return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CODEC;
3179
3180         codec_data = GST_VAAPI_DECODER_CODEC_DATA(decoder);
3181         if (codec_data) {
3182             status = decode_codec_data(decoder, codec_data);
3183             if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
3184                 return status;
3185         }
3186      }
3187      return decode_buffer(decoder, buffer);
3188 }
3189
3190 static void
3191 gst_vaapi_decoder_h264_finalize(GObject *object)
3192 {
3193     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264(object);
3194
3195     gst_vaapi_decoder_h264_destroy(decoder);
3196
3197     G_OBJECT_CLASS(gst_vaapi_decoder_h264_parent_class)->finalize(object);
3198 }
3199
3200 static void
3201 gst_vaapi_decoder_h264_constructed(GObject *object)
3202 {
3203     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264(object);
3204     GstVaapiDecoderH264Private * const priv = decoder->priv;
3205     GObjectClass *parent_class;
3206
3207     parent_class = G_OBJECT_CLASS(gst_vaapi_decoder_h264_parent_class);
3208     if (parent_class->constructed)
3209         parent_class->constructed(object);
3210
3211     priv->is_constructed = gst_vaapi_decoder_h264_create(decoder);
3212 }
3213
3214 static void
3215 gst_vaapi_decoder_h264_class_init(GstVaapiDecoderH264Class *klass)
3216 {
3217     GObjectClass * const object_class = G_OBJECT_CLASS(klass);
3218     GstVaapiDecoderClass * const decoder_class = GST_VAAPI_DECODER_CLASS(klass);
3219
3220     g_type_class_add_private(klass, sizeof(GstVaapiDecoderH264Private));
3221
3222     object_class->finalize      = gst_vaapi_decoder_h264_finalize;
3223     object_class->constructed   = gst_vaapi_decoder_h264_constructed;
3224
3225     decoder_class->decode       = gst_vaapi_decoder_h264_decode;
3226     decoder_class->clear_buffer = gst_vaapi_decoder_h264_clear_buffer;
3227 }
3228
3229 static void
3230 gst_vaapi_decoder_h264_init(GstVaapiDecoderH264 *decoder)
3231 {
3232     GstVaapiDecoderH264Private *priv;
3233
3234     priv                        = GST_VAAPI_DECODER_H264_GET_PRIVATE(decoder);
3235     decoder->priv               = priv;
3236     priv->parser                = NULL;
3237     priv->current_picture       = NULL;
3238     priv->dpb_count             = 0;
3239     priv->dpb_size              = 0;
3240     priv->profile               = GST_VAAPI_PROFILE_UNKNOWN;
3241     priv->entrypoint            = GST_VAAPI_ENTRYPOINT_VLD;
3242     priv->chroma_type           = GST_VAAPI_CHROMA_TYPE_YUV420;
3243     priv->short_ref_count       = 0;
3244     priv->long_ref_count        = 0;
3245     priv->RefPicList0_count     = 0;
3246     priv->RefPicList1_count     = 0;
3247     priv->nal_length_size       = 0;
3248     priv->adapter               = NULL;
3249     priv->field_poc[0]          = 0;
3250     priv->field_poc[1]          = 0;
3251     priv->poc_msb               = 0;
3252     priv->poc_lsb               = 0;
3253     priv->prev_poc_msb          = 0;
3254     priv->prev_poc_lsb          = 0;
3255     priv->frame_num_offset      = 0;
3256     priv->frame_num             = 0;
3257     priv->prev_frame_num        = 0;
3258     priv->prev_pic_has_mmco5    = FALSE;
3259     priv->prev_pic_structure    = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
3260     priv->is_constructed        = FALSE;
3261     priv->is_opened             = FALSE;
3262     priv->is_avc                = FALSE;
3263     priv->has_context           = FALSE;
3264     priv->progressive_sequence  = TRUE;
3265
3266     memset(priv->dpb, 0, sizeof(priv->dpb));
3267     memset(priv->short_ref, 0, sizeof(priv->short_ref));
3268     memset(priv->long_ref, 0, sizeof(priv->long_ref));
3269     memset(priv->RefPicList0, 0, sizeof(priv->RefPicList0));
3270     memset(priv->RefPicList1, 0, sizeof(priv->RefPicList1));
3271 }
3272
3273 /**
3274  * gst_vaapi_decoder_h264_new:
3275  * @display: a #GstVaapiDisplay
3276  * @caps: a #GstCaps holding codec information
3277  *
3278  * Creates a new #GstVaapiDecoder for MPEG-2 decoding.  The @caps can
3279  * hold extra information like codec-data and pictured coded size.
3280  *
3281  * Return value: the newly allocated #GstVaapiDecoder object
3282  */
3283 GstVaapiDecoder *
3284 gst_vaapi_decoder_h264_new(GstVaapiDisplay *display, GstCaps *caps)
3285 {
3286     GstVaapiDecoderH264 *decoder;
3287
3288     g_return_val_if_fail(GST_VAAPI_IS_DISPLAY(display), NULL);
3289     g_return_val_if_fail(GST_IS_CAPS(caps), NULL);
3290
3291     decoder = g_object_new(
3292         GST_VAAPI_TYPE_DECODER_H264,
3293         "display",      display,
3294         "caps",         caps,
3295         NULL
3296     );
3297     if (!decoder->priv->is_constructed) {
3298         g_object_unref(decoder);
3299         return NULL;
3300     }
3301     return GST_VAAPI_DECODER_CAST(decoder);
3302 }