gst-libs/gst/vaapi/gstvaapidecoder_h264.c

   1 /*
   2  *  gstvaapidecoder_h264.c - H.264 decoder
   3  *
   4  *  Copyright (C) 2011-2012 Intel Corporation
   5  *
   6  *  This library is free software; you can redistribute it and/or
   7  *  modify it under the terms of the GNU Lesser General Public License
   8  *  as published by the Free Software Foundation; either version 2.1
   9  *  of the License, or (at your option) any later version.
  10  *
  11  *  This library is distributed in the hope that it will be useful,
  12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  *  Lesser General Public License for more details.
  15  *
  16  *  You should have received a copy of the GNU Lesser General Public
  17  *  License along with this library; if not, write to the Free
  18  *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  19  *  Boston, MA 02110-1301 USA
  20  */
  21
  22 /**
  23  * SECTION:gstvaapidecoder_h264
  24  * @short_description: H.264 decoder
  25  */
  26
  27 #include "sysdeps.h"
  28 #include <string.h>
  29 #include <stdlib.h>
  30 #include <gst/base/gstadapter.h>
  31 #include <gst/codecparsers/gsth264parser.h>
  32 #include "gstvaapidecoder_h264.h"
  33 #include "gstvaapidecoder_objects.h"
  34 #include "gstvaapidecoder_priv.h"
  35 #include "gstvaapidisplay_priv.h"
  36 #include "gstvaapiobject_priv.h"
  37
  38 #define DEBUG 1
  39 #include "gstvaapidebug.h"
  40
  41 /* Defined to 1 if strict ordering of DPB is needed. Only useful for debug */
  42 #define USE_STRICT_DPB_ORDERING 0
  43
  44 typedef struct _GstVaapiFrameStore              GstVaapiFrameStore;
  45 typedef struct _GstVaapiFrameStoreClass         GstVaapiFrameStoreClass;
  46 typedef struct _GstVaapiPictureH264             GstVaapiPictureH264;
  47 typedef struct _GstVaapiPictureH264Class        GstVaapiPictureH264Class;
  48 typedef struct _GstVaapiSliceH264               GstVaapiSliceH264;
  49 typedef struct _GstVaapiSliceH264Class          GstVaapiSliceH264Class;
  50
  51 // Used for field_poc[]
  52 #define TOP_FIELD       0
  53 #define BOTTOM_FIELD    1
  54
  55 /* ------------------------------------------------------------------------- */
  56 /* --- H.264 Pictures                                                    --- */
  57 /* ------------------------------------------------------------------------- */
  58
  59 #define GST_VAAPI_TYPE_PICTURE_H264 \
  60     (gst_vaapi_picture_h264_get_type())
  61
  62 #define GST_VAAPI_PICTURE_H264_CAST(obj) \
  63     ((GstVaapiPictureH264 *)(obj))
  64
  65 #define GST_VAAPI_PICTURE_H264(obj)                             \
  66     (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
  67                                 GST_VAAPI_TYPE_PICTURE_H264,    \
  68                                 GstVaapiPictureH264))
  69
  70 #define GST_VAAPI_PICTURE_H264_CLASS(klass)                     \
  71     (G_TYPE_CHECK_CLASS_CAST((klass),                           \
  72                              GST_VAAPI_TYPE_PICTURE_H264,       \
  73                              GstVaapiPictureH264Class))
  74
  75 #define GST_VAAPI_IS_PICTURE_H264(obj) \
  76     (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_VAAPI_TYPE_PICTURE_H264))
  77
  78 #define GST_VAAPI_IS_PICTURE_H264_CLASS(klass) \
  79     (G_TYPE_CHECK_CLASS_TYPE((klass), GST_VAAPI_TYPE_PICTURE_H264))
  80
  81 #define GST_VAAPI_PICTURE_H264_GET_CLASS(obj)                   \
  82     (G_TYPE_INSTANCE_GET_CLASS((obj),                           \
  83                                GST_VAAPI_TYPE_PICTURE_H264,     \
  84                                GstVaapiPictureH264Class))
  85
  86 /*
  87  * Extended picture flags:
  88  *
  89  * @GST_VAAPI_PICTURE_FLAG_IDR: flag that specifies an IDR picture
  90  * @GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE: flag that specifies
  91  *     "used for short-term reference"
  92  * @GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE: flag that specifies
  93  *     "used for long-term reference"
  94  * @GST_VAAPI_PICTURE_FLAGS_REFERENCE: mask covering any kind of
  95  *     reference picture (short-term reference or long-term reference)
  96  */
  97 enum {
  98     GST_VAAPI_PICTURE_FLAG_IDR = (GST_VAAPI_PICTURE_FLAG_LAST << 0),
  99
 100     GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE = (
 101         GST_VAAPI_PICTURE_FLAG_REFERENCE),
 102     GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE = (
 103         GST_VAAPI_PICTURE_FLAG_REFERENCE | (GST_VAAPI_PICTURE_FLAG_LAST << 1)),
 104     GST_VAAPI_PICTURE_FLAGS_REFERENCE = (
 105         GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE |
 106         GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE),
 107 };
 108
 109 #define GST_VAAPI_PICTURE_IS_IDR(picture) \
 110     (GST_VAAPI_PICTURE_FLAG_IS_SET(picture, GST_VAAPI_PICTURE_FLAG_IDR))
 111
 112 #define GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture)      \
 113     ((GST_VAAPI_PICTURE_FLAGS(picture) &                        \
 114       GST_VAAPI_PICTURE_FLAGS_REFERENCE) ==                     \
 115      GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE)
 116
 117 #define GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture)       \
 118     ((GST_VAAPI_PICTURE_FLAGS(picture) &                        \
 119       GST_VAAPI_PICTURE_FLAGS_REFERENCE) ==                     \
 120      GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE)
 121
 122 struct _GstVaapiPictureH264 {
 123     GstVaapiPicture             base;
 124     GstH264PPS                 *pps;
 125     guint                       structure;
 126     gint32                      field_poc[2];
 127     gint32                      frame_num;              // Original frame_num from slice_header()
 128     gint32                      frame_num_wrap;         // Temporary for ref pic marking: FrameNumWrap
 129     gint32                      long_term_frame_idx;    // Temporary for ref pic marking: LongTermFrameIdx
 130     gint32                      pic_num;                // Temporary for ref pic marking: PicNum
 131     gint32                      long_term_pic_num;      // Temporary for ref pic marking: LongTermPicNum
 132     GstVaapiPictureH264        *other_field;            // Temporary for ref pic marking: other field in the same frame store
 133     guint                       output_flag             : 1;
 134     guint                       output_needed           : 1;
 135 };
 136
 137 struct _GstVaapiPictureH264Class {
 138     /*< private >*/
 139     GstVaapiPictureClass        parent_class;
 140 };
 141
 142 GST_VAAPI_CODEC_DEFINE_TYPE(GstVaapiPictureH264,
 143                             gst_vaapi_picture_h264,
 144                             GST_VAAPI_TYPE_PICTURE)
 145
 146 static void
 147 gst_vaapi_picture_h264_destroy(GstVaapiPictureH264 *decoder)
 148 {
 149 }
 150
 151 static gboolean
 152 gst_vaapi_picture_h264_create(
 153     GstVaapiPictureH264                      *picture,
 154     const GstVaapiCodecObjectConstructorArgs *args
 155 )
 156 {
 157     return TRUE;
 158 }
 159
 160 static void
 161 gst_vaapi_picture_h264_init(GstVaapiPictureH264 *picture)
 162 {
 163     picture->field_poc[0]       = G_MAXINT32;
 164     picture->field_poc[1]       = G_MAXINT32;
 165     picture->output_needed      = FALSE;
 166 }
 167
 168 static inline GstVaapiPictureH264 *
 169 gst_vaapi_picture_h264_new(GstVaapiDecoderH264 *decoder)
 170 {
 171     GstVaapiCodecObject *object;
 172
 173     g_return_val_if_fail(GST_VAAPI_IS_DECODER(decoder), NULL);
 174
 175     object = gst_vaapi_codec_object_new(
 176         GST_VAAPI_TYPE_PICTURE_H264,
 177         GST_VAAPI_CODEC_BASE(decoder),
 178         NULL, sizeof(VAPictureParameterBufferH264),
 179         NULL, 0
 180     );
 181     if (!object)
 182         return NULL;
 183     return GST_VAAPI_PICTURE_H264_CAST(object);
 184 }
 185
 186 static inline void
 187 gst_vaapi_picture_h264_set_reference(
 188     GstVaapiPictureH264 *picture,
 189     guint                reference_flags,
 190     gboolean             other_field
 191 )
 192 {
 193     g_return_if_fail(GST_VAAPI_IS_PICTURE_H264(picture));
 194
 195     GST_VAAPI_PICTURE_FLAG_UNSET(picture, GST_VAAPI_PICTURE_FLAGS_REFERENCE);
 196     GST_VAAPI_PICTURE_FLAG_SET(picture, reference_flags);
 197
 198     if (!other_field || !(picture = picture->other_field))
 199         return;
 200     GST_VAAPI_PICTURE_FLAG_UNSET(picture, GST_VAAPI_PICTURE_FLAGS_REFERENCE);
 201     GST_VAAPI_PICTURE_FLAG_SET(picture, reference_flags);
 202 }
 203
 204 static inline GstVaapiPictureH264 *
 205 gst_vaapi_picture_h264_new_field(GstVaapiPictureH264 *picture)
 206 {
 207     GstVaapiPicture *base_picture;
 208
 209     g_return_val_if_fail(GST_VAAPI_IS_PICTURE_H264(picture), NULL);
 210
 211     base_picture = gst_vaapi_picture_new_field(&picture->base);
 212     if (!base_picture)
 213         return NULL;
 214     return GST_VAAPI_PICTURE_H264_CAST(base_picture);
 215 }
 216
 217 static inline GstVaapiSliceH264 *
 218 gst_vaapi_picture_h264_get_last_slice(GstVaapiPictureH264 *picture)
 219 {
 220     g_return_val_if_fail(GST_VAAPI_IS_PICTURE_H264(picture), NULL);
 221
 222     if (G_UNLIKELY(picture->base.slices->len < 1))
 223         return NULL;
 224     return g_ptr_array_index(picture->base.slices,
 225         picture->base.slices->len - 1);
 226 }
 227
 228 /* ------------------------------------------------------------------------- */
 229 /* --- Slices                                                            --- */
 230 /* ------------------------------------------------------------------------- */
 231
 232 #define GST_VAAPI_TYPE_SLICE_H264 \
 233     (gst_vaapi_slice_h264_get_type())
 234
 235 #define GST_VAAPI_SLICE_H264_CAST(obj) \
 236     ((GstVaapiSliceH264 *)(obj))
 237
 238 #define GST_VAAPI_SLICE_H264(obj)                               \
 239     (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
 240                                 GST_VAAPI_TYPE_SLICE_H264,      \
 241                                 GstVaapiSliceH264))
 242
 243 #define GST_VAAPI_SLICE_H264_CLASS(klass)                       \
 244     (G_TYPE_CHECK_CLASS_CAST((klass),                           \
 245                              GST_VAAPI_TYPE_SLICE_H264,         \
 246                              GstVaapiSliceH264Class))
 247
 248 #define GST_VAAPI_IS_SLICE_H264(obj) \
 249     (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_VAAPI_TYPE_SLICE_H264))
 250
 251 #define GST_VAAPI_IS_SLICE_H264_CLASS(klass) \
 252     (G_TYPE_CHECK_CLASS_TYPE((klass), GST_VAAPI_TYPE_SLICE_H264))
 253
 254 #define GST_VAAPI_SLICE_H264_GET_CLASS(obj)                     \
 255     (G_TYPE_INSTANCE_GET_CLASS((obj),                           \
 256                                GST_VAAPI_TYPE_SLICE_H264,       \
 257                                GstVaapiSliceH264Class))
 258
 259 struct _GstVaapiSliceH264 {
 260     GstVaapiSlice               base;
 261     GstH264SliceHdr             slice_hdr;              // parsed slice_header()
 262 };
 263
 264 struct _GstVaapiSliceH264Class {
 265     /*< private >*/
 266     GstVaapiSliceClass          parent_class;
 267 };
 268
 269 GST_VAAPI_CODEC_DEFINE_TYPE(GstVaapiSliceH264,
 270                             gst_vaapi_slice_h264,
 271                             GST_VAAPI_TYPE_SLICE)
 272
 273 static void
 274 gst_vaapi_slice_h264_destroy(GstVaapiSliceH264 *slice)
 275 {
 276 }
 277
 278 static gboolean
 279 gst_vaapi_slice_h264_create(
 280     GstVaapiSliceH264                        *slice,
 281     const GstVaapiCodecObjectConstructorArgs *args
 282 )
 283 {
 284     return TRUE;
 285 }
 286
 287 static void
 288 gst_vaapi_slice_h264_init(GstVaapiSliceH264 *slice)
 289 {
 290 }
 291
 292 static inline GstVaapiSliceH264 *
 293 gst_vaapi_slice_h264_new(
 294     GstVaapiDecoderH264 *decoder,
 295     const guint8        *data,
 296     guint                data_size
 297 )
 298 {
 299     GstVaapiCodecObject *object;
 300
 301     g_return_val_if_fail(GST_VAAPI_IS_DECODER(decoder), NULL);
 302
 303     object = gst_vaapi_codec_object_new(
 304         GST_VAAPI_TYPE_SLICE_H264,
 305         GST_VAAPI_CODEC_BASE(decoder),
 306         NULL, sizeof(VASliceParameterBufferH264),
 307         data, data_size
 308     );
 309     if (!object)
 310         return NULL;
 311     return GST_VAAPI_SLICE_H264_CAST(object);
 312 }
 313
 314 /* ------------------------------------------------------------------------- */
 315 /* --- Frame Buffers (DPB)                                               --- */
 316 /* ------------------------------------------------------------------------- */
 317
 318 #define GST_VAAPI_TYPE_FRAME_STORE \
 319     (gst_vaapi_frame_store_get_type())
 320
 321 #define GST_VAAPI_FRAME_STORE_CAST(obj) \
 322     ((GstVaapiFrameStore *)(obj))
 323
 324 #define GST_VAAPI_FRAME_STORE(obj)                              \
 325     (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
 326                                 GST_VAAPI_TYPE_FRAME_STORE,     \
 327                                 GstVaapiFrameStore))
 328
 329 #define GST_VAAPI_FRAME_STORE_CLASS(klass)                      \
 330     (G_TYPE_CHECK_CLASS_CAST((klass),                           \
 331                              GST_VAAPI_TYPE_FRAME_STORE,        \
 332                              GstVaapiFrameStoreClass))
 333
 334 #define GST_VAAPI_IS_FRAME_STORE(obj) \
 335     (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_VAAPI_TYPE_FRAME_STORE))
 336
 337 #define GST_VAAPI_IS_FRAME_STORE_CLASS(klass) \
 338     (G_TYPE_CHECK_CLASS_TYPE((klass), GST_VAAPI_TYPE_FRAME_STORE))
 339
 340 #define GST_VAAPI_FRAME_STORE_GET_CLASS(obj)                   \
 341     (G_TYPE_INSTANCE_GET_CLASS((obj),                          \
 342                                GST_VAAPI_TYPE_FRAME_STORE,     \
 343                                GstVaapiFrameStoreClass))
 344
 345 struct _GstVaapiFrameStore {
 346     /*< private >*/
 347     GstMiniObject               parent_instance;
 348
 349     guint                       structure;
 350     GstVaapiPictureH264        *buffers[2];
 351     guint                       num_buffers;
 352     guint                       output_needed;
 353 };
 354
 355 struct _GstVaapiFrameStoreClass {
 356     /*< private >*/
 357     GstMiniObjectClass          parent_class;
 358 };
 359
 360 G_DEFINE_TYPE(GstVaapiFrameStore, gst_vaapi_frame_store, GST_TYPE_MINI_OBJECT)
 361
 362 static void
 363 gst_vaapi_frame_store_finalize(GstMiniObject *object)
 364 {
 365     GstVaapiFrameStore * const fs = GST_VAAPI_FRAME_STORE_CAST(object);
 366     GstMiniObjectClass *parent_class;
 367     guint i;
 368
 369     for (i = 0; i < fs->num_buffers; i++)
 370         gst_vaapi_picture_replace(&fs->buffers[i], NULL);
 371
 372     parent_class = GST_MINI_OBJECT_CLASS(gst_vaapi_frame_store_parent_class);
 373     if (parent_class->finalize)
 374         parent_class->finalize(object);
 375 }
 376
 377 static void
 378 gst_vaapi_frame_store_init(GstVaapiFrameStore *fs)
 379 {
 380 }
 381
 382 static void
 383 gst_vaapi_frame_store_class_init(GstVaapiFrameStoreClass *klass)
 384 {
 385     GstMiniObjectClass * const object_class = GST_MINI_OBJECT_CLASS(klass);
 386
 387     object_class->finalize = gst_vaapi_frame_store_finalize;
 388 }
 389
 390 static inline gpointer
 391 _gst_vaapi_frame_store_new(void)
 392 {
 393     return gst_mini_object_new(GST_VAAPI_TYPE_FRAME_STORE);
 394 }
 395
 396 static GstVaapiFrameStore *
 397 gst_vaapi_frame_store_new(GstVaapiPictureH264 *picture)
 398 {
 399     GstVaapiFrameStore *fs;
 400
 401     g_return_val_if_fail(GST_VAAPI_IS_PICTURE_H264(picture), NULL);
 402
 403     fs = _gst_vaapi_frame_store_new();
 404     if (!fs)
 405         return NULL;
 406
 407     fs->structure       = picture->structure;
 408     fs->buffers[0]      = gst_vaapi_picture_ref(picture);
 409     fs->num_buffers     = 1;
 410     fs->output_needed   = picture->output_needed;
 411     return fs;
 412 }
 413
 414 static gboolean
 415 gst_vaapi_frame_store_add(GstVaapiFrameStore *fs, GstVaapiPictureH264 *picture)
 416 {
 417     guint field;
 418
 419     g_return_val_if_fail(GST_VAAPI_IS_FRAME_STORE(fs), FALSE);
 420     g_return_val_if_fail(fs->num_buffers == 1, FALSE);
 421     g_return_val_if_fail(GST_VAAPI_IS_PICTURE_H264(picture), FALSE);
 422     g_return_val_if_fail(!GST_VAAPI_PICTURE_IS_FRAME(picture), FALSE);
 423
 424     gst_vaapi_picture_replace(&fs->buffers[fs->num_buffers++], picture);
 425     if (picture->output_flag) {
 426         picture->output_needed = TRUE;
 427         fs->output_needed++;
 428     }
 429
 430     fs->structure = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
 431
 432     field = picture->structure == GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD ?
 433         TOP_FIELD : BOTTOM_FIELD;
 434     g_return_val_if_fail(fs->buffers[0]->field_poc[field] == G_MAXINT32, FALSE);
 435     fs->buffers[0]->field_poc[field] = picture->field_poc[field];
 436     g_return_val_if_fail(picture->field_poc[!field] == G_MAXINT32, FALSE);
 437     picture->field_poc[!field] = fs->buffers[0]->field_poc[!field];
 438     return TRUE;
 439 }
 440
 441 static gboolean
 442 gst_vaapi_frame_store_split_fields(GstVaapiFrameStore *fs)
 443 {
 444     GstVaapiPictureH264 * const first_field = fs->buffers[0];
 445     GstVaapiPictureH264 *second_field;
 446
 447     g_return_val_if_fail(fs->num_buffers == 1, FALSE);
 448
 449     first_field->base.structure = GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD;
 450     GST_VAAPI_PICTURE_FLAG_SET(first_field, GST_VAAPI_PICTURE_FLAG_INTERLACED);
 451
 452     second_field = gst_vaapi_picture_h264_new_field(first_field);
 453     if (!second_field)
 454         return FALSE;
 455     gst_vaapi_picture_replace(&fs->buffers[fs->num_buffers++], second_field);
 456     gst_vaapi_picture_unref(second_field);
 457
 458     second_field->frame_num    = first_field->frame_num;
 459     second_field->field_poc[0] = first_field->field_poc[0];
 460     second_field->field_poc[1] = first_field->field_poc[1];
 461     second_field->output_flag  = first_field->output_flag;
 462     if (second_field->output_flag) {
 463         second_field->output_needed = TRUE;
 464         fs->output_needed++;
 465     }
 466     return TRUE;
 467 }
 468
 469 static inline gboolean
 470 gst_vaapi_frame_store_has_frame(GstVaapiFrameStore *fs)
 471 {
 472     return fs->structure == GST_VAAPI_PICTURE_STRUCTURE_FRAME;
 473 }
 474
 475 static inline gboolean
 476 gst_vaapi_frame_store_has_reference(GstVaapiFrameStore *fs)
 477 {
 478     guint i;
 479
 480     for (i = 0; i < fs->num_buffers; i++) {
 481         if (GST_VAAPI_PICTURE_IS_REFERENCE(fs->buffers[i]))
 482             return TRUE;
 483     }
 484     return FALSE;
 485 }
 486
 487 #define gst_vaapi_frame_store_ref(fs) \
 488     gst_mini_object_ref(GST_MINI_OBJECT(fs))
 489
 490 #define gst_vaapi_frame_store_unref(fs) \
 491     gst_mini_object_unref(GST_MINI_OBJECT(fs))
 492
 493 #define gst_vaapi_frame_store_replace(old_fs_p, new_fs)         \
 494     gst_mini_object_replace((GstMiniObject **)(old_fs_p),       \
 495                             (GstMiniObject *)(new_fs))
 496
 497 /* ------------------------------------------------------------------------- */
 498 /* --- H.264 Decoder                                                     --- */
 499 /* ------------------------------------------------------------------------- */
 500
 501 G_DEFINE_TYPE(GstVaapiDecoderH264,
 502               gst_vaapi_decoder_h264,
 503               GST_VAAPI_TYPE_DECODER)
 504
 505 #define GST_VAAPI_DECODER_H264_GET_PRIVATE(obj)                 \
 506     (G_TYPE_INSTANCE_GET_PRIVATE((obj),                         \
 507                                  GST_VAAPI_TYPE_DECODER_H264,   \
 508                                  GstVaapiDecoderH264Private))
 509
 510 struct _GstVaapiDecoderH264Private {
 511     GstAdapter                 *adapter;
 512     GstH264NalParser           *parser;
 513     /* Last decoded SPS. May not be the last activated one. Just here because
 514        it may not fit stack memory allocation in decode_sps() */
 515     GstH264SPS                  last_sps;
 516     /* Last decoded PPS. May not be the last activated one. Just here because
 517        it may not fit stack memory allocation in decode_pps() */
 518     GstH264PPS                  last_pps;
 519     /* Temporary slice header. Just here because it may not fit stack
 520        memory allocation in decode_slice() */
 521     GstH264SliceHdr             temp_slice_hdr;
 522     GstVaapiPictureH264        *current_picture;
 523     GstVaapiFrameStore         *prev_frame;
 524     GstVaapiFrameStore         *dpb[16];
 525     guint                       dpb_count;
 526     guint                       dpb_size;
 527     GstVaapiProfile             profile;
 528     GstVaapiEntrypoint          entrypoint;
 529     GstVaapiChromaType          chroma_type;
 530     GstVaapiPictureH264        *short_ref[32];
 531     guint                       short_ref_count;
 532     GstVaapiPictureH264        *long_ref[32];
 533     guint                       long_ref_count;
 534     GstVaapiPictureH264        *RefPicList0[32];
 535     guint                       RefPicList0_count;
 536     GstVaapiPictureH264        *RefPicList1[32];
 537     guint                       RefPicList1_count;
 538     guint                       nal_length_size;
 539     guint                       mb_width;
 540     guint                       mb_height;
 541     gint32                      field_poc[2];           // 0:TopFieldOrderCnt / 1:BottomFieldOrderCnt
 542     gint32                      poc_msb;                // PicOrderCntMsb
 543     gint32                      poc_lsb;                // pic_order_cnt_lsb (from slice_header())
 544     gint32                      prev_poc_msb;           // prevPicOrderCntMsb
 545     gint32                      prev_poc_lsb;           // prevPicOrderCntLsb
 546     gint32                      frame_num_offset;       // FrameNumOffset
 547     gint32                      frame_num;              // frame_num (from slice_header())
 548     gint32                      prev_frame_num;         // prevFrameNum
 549     gboolean                    prev_pic_has_mmco5;     // prevMmco5Pic
 550     gboolean                    prev_pic_structure;     // previous picture structure
 551     guint                       is_constructed          : 1;
 552     guint                       is_opened               : 1;
 553     guint                       is_avc                  : 1;
 554     guint                       got_sps                 : 1;
 555     guint                       got_pps                 : 1;
 556     guint                       has_context             : 1;
 557     guint                       progressive_sequence    : 1;
 558 };
 559
 560 static gboolean
 561 exec_ref_pic_marking(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture);
 562
 563 /* Get number of reference frames to use */
 564 static guint
 565 get_max_dec_frame_buffering(GstH264SPS *sps)
 566 {
 567     guint max_dec_frame_buffering, MaxDpbMbs, PicSizeMbs;
 568
 569     /* Table A-1 - Level limits */
 570     switch (sps->level_idc) {
 571     case 10: MaxDpbMbs = 396;    break;
 572     case 11: MaxDpbMbs = 900;    break;
 573     case 12: MaxDpbMbs = 2376;   break;
 574     case 13: MaxDpbMbs = 2376;   break;
 575     case 20: MaxDpbMbs = 2376;   break;
 576     case 21: MaxDpbMbs = 4752;   break;
 577     case 22: MaxDpbMbs = 8100;   break;
 578     case 30: MaxDpbMbs = 8100;   break;
 579     case 31: MaxDpbMbs = 18000;  break;
 580     case 32: MaxDpbMbs = 20480;  break;
 581     case 40: MaxDpbMbs = 32768;  break;
 582     case 41: MaxDpbMbs = 32768;  break;
 583     case 42: MaxDpbMbs = 34816;  break;
 584     case 50: MaxDpbMbs = 110400; break;
 585     case 51: MaxDpbMbs = 184320; break;
 586     default:
 587         g_assert(0 && "unhandled level");
 588         break;
 589     }
 590
 591     PicSizeMbs = ((sps->pic_width_in_mbs_minus1 + 1) *
 592                   (sps->pic_height_in_map_units_minus1 + 1) *
 593                   (sps->frame_mbs_only_flag ? 1 : 2));
 594     max_dec_frame_buffering = MaxDpbMbs / PicSizeMbs;
 595
 596     /* VUI parameters */
 597     if (sps->vui_parameters_present_flag) {
 598         GstH264VUIParams * const vui_params = &sps->vui_parameters;
 599         if (vui_params->bitstream_restriction_flag)
 600             max_dec_frame_buffering = vui_params->max_dec_frame_buffering;
 601         else {
 602             switch (sps->profile_idc) {
 603             case 44:  // CAVLC 4:4:4 Intra profile
 604             case 86:  // Scalable High profile
 605             case 100: // High profile
 606             case 110: // High 10 profile
 607             case 122: // High 4:2:2 profile
 608             case 244: // High 4:4:4 Predictive profile
 609                 if (sps->constraint_set3_flag)
 610                     max_dec_frame_buffering = 0;
 611                 break;
 612             }
 613         }
 614     }
 615
 616     if (max_dec_frame_buffering > 16)
 617         max_dec_frame_buffering = 16;
 618     else if (max_dec_frame_buffering < sps->num_ref_frames)
 619         max_dec_frame_buffering = sps->num_ref_frames;
 620     return MAX(1, max_dec_frame_buffering);
 621 }
 622
 623 static void
 624 array_remove_index_fast(void *array, guint *array_length_ptr, guint index)
 625 {
 626     gpointer * const entries = array;
 627     guint num_entries = *array_length_ptr;
 628
 629     g_return_if_fail(index < num_entries);
 630
 631     if (index != --num_entries)
 632         entries[index] = entries[num_entries];
 633     entries[num_entries] = NULL;
 634     *array_length_ptr = num_entries;
 635 }
 636
 637 #if 1
 638 static inline void
 639 array_remove_index(void *array, guint *array_length_ptr, guint index)
 640 {
 641     array_remove_index_fast(array, array_length_ptr, index);
 642 }
 643 #else
 644 static void
 645 array_remove_index(void *array, guint *array_length_ptr, guint index)
 646 {
 647     gpointer * const entries = array;
 648     const guint num_entries = *array_length_ptr - 1;
 649     guint i;
 650
 651     g_return_if_fail(index <= num_entries);
 652
 653     for (i = index; i < num_entries; i++)
 654         entries[i] = entries[i + 1];
 655     entries[num_entries] = NULL;
 656     *array_length_ptr = num_entries;
 657 }
 658 #endif
 659
 660 #define ARRAY_REMOVE_INDEX(array, index) \
 661     array_remove_index(array, &array##_count, index)
 662
 663 static void
 664 dpb_remove_index(GstVaapiDecoderH264 *decoder, guint index)
 665 {
 666     GstVaapiDecoderH264Private * const priv = decoder->priv;
 667     guint i, num_frames = --priv->dpb_count;
 668
 669     if (USE_STRICT_DPB_ORDERING) {
 670         for (i = index; i < num_frames; i++)
 671             gst_vaapi_frame_store_replace(&priv->dpb[i], priv->dpb[i + 1]);
 672     }
 673     else if (index != num_frames)
 674         gst_vaapi_frame_store_replace(&priv->dpb[index], priv->dpb[num_frames]);
 675     gst_vaapi_frame_store_replace(&priv->dpb[num_frames], NULL);
 676 }
 677
 678 static gboolean
 679 dpb_output(
 680     GstVaapiDecoderH264 *decoder,
 681     GstVaapiFrameStore  *fs,
 682     GstVaapiPictureH264 *picture
 683 )
 684 {
 685     picture->output_needed = FALSE;
 686
 687     if (fs) {
 688         if (--fs->output_needed > 0)
 689             return TRUE;
 690         picture = fs->buffers[0];
 691     }
 692
 693     /* XXX: update cropping rectangle */
 694     return gst_vaapi_picture_output(GST_VAAPI_PICTURE_CAST(picture));
 695 }
 696
 697 static inline void
 698 dpb_evict(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture, guint i)
 699 {
 700     GstVaapiFrameStore * const fs = decoder->priv->dpb[i];
 701
 702     if (!fs->output_needed && !gst_vaapi_frame_store_has_reference(fs))
 703         dpb_remove_index(decoder, i);
 704 }
 705
 706 static gboolean
 707 dpb_bump(GstVaapiDecoderH264 *decoder)
 708 {
 709     GstVaapiDecoderH264Private * const priv = decoder->priv;
 710     GstVaapiPictureH264 *found_picture = NULL;
 711     guint i, j, found_index;
 712     gboolean success;
 713
 714     for (i = 0; i < priv->dpb_count; i++) {
 715         GstVaapiFrameStore * const fs = priv->dpb[i];
 716         if (!fs->output_needed)
 717             continue;
 718         for (j = 0; j < fs->num_buffers; j++) {
 719             GstVaapiPictureH264 * const picture = fs->buffers[j];
 720             if (!picture->output_needed)
 721                 continue;
 722             if (!found_picture || found_picture->base.poc > picture->base.poc)
 723                 found_picture = picture, found_index = i;
 724         }
 725     }
 726     if (!found_picture)
 727         return FALSE;
 728
 729     success = dpb_output(decoder, priv->dpb[found_index], found_picture);
 730     dpb_evict(decoder, found_picture, found_index);
 731     return success;
 732 }
 733
 734 static void
 735 dpb_clear(GstVaapiDecoderH264 *decoder)
 736 {
 737     GstVaapiDecoderH264Private * const priv = decoder->priv;
 738     guint i;
 739
 740     for (i = 0; i < priv->dpb_count; i++)
 741         gst_vaapi_frame_store_replace(&priv->dpb[i], NULL);
 742     priv->dpb_count = 0;
 743
 744     gst_vaapi_frame_store_replace(&priv->prev_frame, NULL);
 745 }
 746
 747 static void
 748 dpb_flush(GstVaapiDecoderH264 *decoder)
 749 {
 750     while (dpb_bump(decoder))
 751         ;
 752     dpb_clear(decoder);
 753 }
 754
 755 static gboolean
 756 dpb_add(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
 757 {
 758     GstVaapiDecoderH264Private * const priv = decoder->priv;
 759     GstVaapiFrameStore *fs;
 760     guint i, j;
 761
 762     // Remove all unused pictures
 763     if (!GST_VAAPI_PICTURE_IS_IDR(picture)) {
 764         i = 0;
 765         while (i < priv->dpb_count) {
 766             GstVaapiFrameStore * const fs = priv->dpb[i];
 767             if (!fs->output_needed && !gst_vaapi_frame_store_has_reference(fs))
 768                 dpb_remove_index(decoder, i);
 769             else
 770                 i++;
 771         }
 772     }
 773
 774     // Check if picture is the second field and the first field is still in DPB
 775     fs = priv->prev_frame;
 776     if (fs && !gst_vaapi_frame_store_has_frame(fs)) {
 777         g_return_val_if_fail(fs->num_buffers == 1, FALSE);
 778         g_return_val_if_fail(!GST_VAAPI_PICTURE_IS_FRAME(picture), FALSE);
 779         g_return_val_if_fail(!GST_VAAPI_PICTURE_IS_FIRST_FIELD(picture), FALSE);
 780         return gst_vaapi_frame_store_add(fs, picture);
 781     }
 782
 783     // Create new frame store, and split fields if necessary
 784     fs = gst_vaapi_frame_store_new(picture);
 785     if (!fs)
 786         return FALSE;
 787     gst_vaapi_frame_store_replace(&priv->prev_frame, fs);
 788     gst_vaapi_frame_store_unref(fs);
 789
 790     if (!priv->progressive_sequence && gst_vaapi_frame_store_has_frame(fs)) {
 791         if (!gst_vaapi_frame_store_split_fields(fs))
 792             return FALSE;
 793     }
 794
 795     // C.4.5.1 - Storage and marking of a reference decoded picture into the DPB
 796     if (GST_VAAPI_PICTURE_IS_REFERENCE(picture)) {
 797         while (priv->dpb_count == priv->dpb_size) {
 798             if (!dpb_bump(decoder))
 799                 return FALSE;
 800         }
 801         gst_vaapi_frame_store_replace(&priv->dpb[priv->dpb_count++], fs);
 802         if (picture->output_flag) {
 803             picture->output_needed = TRUE;
 804             fs->output_needed++;
 805         }
 806     }
 807
 808     // C.4.5.2 - Storage and marking of a non-reference decoded picture into the DPB
 809     else {
 810         if (!picture->output_flag)
 811             return TRUE;
 812         while (priv->dpb_count == priv->dpb_size) {
 813             gboolean found_picture = FALSE;
 814             for (i = 0; !found_picture && i < priv->dpb_count; i++) {
 815                 GstVaapiFrameStore * const fs = priv->dpb[i];
 816                 if (!fs->output_needed)
 817                     continue;
 818                 for (j = 0; !found_picture && j < fs->num_buffers; j++)
 819                     found_picture = fs->buffers[j]->output_needed &&
 820                         fs->buffers[j]->base.poc < picture->base.poc;
 821             }
 822             if (!found_picture)
 823                 return dpb_output(decoder, NULL, picture);
 824             if (!dpb_bump(decoder))
 825                 return FALSE;
 826         }
 827         gst_vaapi_frame_store_replace(&priv->dpb[priv->dpb_count++], fs);
 828         picture->output_needed = TRUE;
 829         fs->output_needed++;
 830     }
 831     return TRUE;
 832 }
 833
 834 static inline void
 835 dpb_reset(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 836 {
 837     GstVaapiDecoderH264Private * const priv = decoder->priv;
 838
 839     priv->dpb_size = get_max_dec_frame_buffering(sps);
 840     GST_DEBUG("DPB size %u", priv->dpb_size);
 841 }
 842
 843 static GstVaapiDecoderStatus
 844 get_status(GstH264ParserResult result)
 845 {
 846     GstVaapiDecoderStatus status;
 847
 848     switch (result) {
 849     case GST_H264_PARSER_OK:
 850         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
 851         break;
 852     case GST_H264_PARSER_NO_NAL_END:
 853         status = GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
 854         break;
 855     case GST_H264_PARSER_ERROR:
 856         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
 857         break;
 858     default:
 859         status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 860         break;
 861     }
 862     return status;
 863 }
 864
 865 static void
 866 gst_vaapi_decoder_h264_clear_buffer(GstVaapiDecoder *base)
 867 {
 868     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264(base);
 869     GstVaapiDecoderH264Private * const priv = decoder->priv;
 870
 871     gst_vaapi_picture_replace(&priv->current_picture, NULL);
 872
 873     dpb_clear(decoder);
 874
 875     if (priv->adapter) {
 876         gst_adapter_clear(priv->adapter);
 877     }
 878 }
 879
 880 static void
 881 gst_vaapi_decoder_h264_close(GstVaapiDecoderH264 *decoder)
 882 {
 883     GstVaapiDecoderH264Private * const priv = decoder->priv;
 884
 885     gst_vaapi_decoder_h264_clear_buffer(GST_VAAPI_DECODER_CAST(decoder));
 886
 887     if (priv->parser) {
 888         gst_h264_nal_parser_free(priv->parser);
 889         priv->parser = NULL;
 890     }
 891
 892     if (priv->adapter) {
 893         g_object_unref(priv->adapter);
 894         priv->adapter = NULL;
 895     }
 896 }
 897
 898 static gboolean
 899 gst_vaapi_decoder_h264_open(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
 900 {
 901     GstVaapiDecoderH264Private * const priv = decoder->priv;
 902
 903     gst_vaapi_decoder_h264_close(decoder);
 904
 905     priv->adapter = gst_adapter_new();
 906     if (!priv->adapter)
 907         return FALSE;
 908
 909     priv->parser = gst_h264_nal_parser_new();
 910     if (!priv->parser)
 911         return FALSE;
 912     return TRUE;
 913 }
 914
 915 static void
 916 gst_vaapi_decoder_h264_destroy(GstVaapiDecoderH264 *decoder)
 917 {
 918     gst_vaapi_decoder_h264_close(decoder);
 919 }
 920
 921 static gboolean
 922 gst_vaapi_decoder_h264_create(GstVaapiDecoderH264 *decoder)
 923 {
 924     if (!GST_VAAPI_DECODER_CODEC(decoder))
 925         return FALSE;
 926     return TRUE;
 927 }
 928
 929 static guint
 930 h264_get_profile(GstH264SPS *sps)
 931 {
 932     guint profile = 0;
 933
 934     switch (sps->profile_idc) {
 935     case 66:
 936         profile = GST_VAAPI_PROFILE_H264_BASELINE;
 937         break;
 938     case 77:
 939         profile = GST_VAAPI_PROFILE_H264_MAIN;
 940         break;
 941     case 100:
 942         profile = GST_VAAPI_PROFILE_H264_HIGH;
 943         break;
 944     }
 945     return profile;
 946 }
 947
 948 static guint
 949 h264_get_chroma_type(GstH264SPS *sps)
 950 {
 951     guint chroma_type = 0;
 952
 953     switch (sps->chroma_format_idc) {
 954     case 1:
 955         chroma_type = GST_VAAPI_CHROMA_TYPE_YUV420;
 956         break;
 957     case 2:
 958         chroma_type = GST_VAAPI_CHROMA_TYPE_YUV422;
 959         break;
 960     case 3:
 961         if (!sps->separate_colour_plane_flag)
 962             chroma_type = GST_VAAPI_CHROMA_TYPE_YUV444;
 963         break;
 964     }
 965     return chroma_type;
 966 }
 967
 968 static GstVaapiProfile
 969 get_profile(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 970 {
 971     GstVaapiDecoderH264Private * const priv = decoder->priv;
 972     GstVaapiDisplay * const display = GST_VAAPI_DECODER_DISPLAY(decoder);
 973     GstVaapiProfile profile, profiles[2];
 974     guint i, n_profiles = 0;
 975
 976     profile = h264_get_profile(sps);
 977     if (!profile)
 978         return GST_VAAPI_PROFILE_UNKNOWN;
 979
 980     profiles[n_profiles++] = profile;
 981     switch (profile) {
 982     case GST_VAAPI_PROFILE_H264_MAIN:
 983         profiles[n_profiles++] = GST_VAAPI_PROFILE_H264_HIGH;
 984         break;
 985     default:
 986         break;
 987     }
 988
 989     /* If the preferred profile (profiles[0]) matches one that we already
 990        found, then just return it now instead of searching for it again */
 991     if (profiles[0] == priv->profile)
 992         return priv->profile;
 993
 994     for (i = 0; i < n_profiles; i++) {
 995         if (gst_vaapi_display_has_decoder(display, profiles[i], priv->entrypoint))
 996             return profiles[i];
 997     }
 998     return GST_VAAPI_PROFILE_UNKNOWN;
 999 }
1000
1001 static GstVaapiDecoderStatus
1002 ensure_context(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
1003 {
1004     GstVaapiDecoder * const base_decoder = GST_VAAPI_DECODER_CAST(decoder);
1005     GstVaapiDecoderH264Private * const priv = decoder->priv;
1006     GstVaapiContextInfo info;
1007     GstVaapiProfile profile;
1008     GstVaapiChromaType chroma_type;
1009     gboolean reset_context = FALSE;
1010     guint mb_width, mb_height;
1011
1012     profile = get_profile(decoder, sps);
1013     if (!profile) {
1014         GST_ERROR("unsupported profile_idc %u", sps->profile_idc);
1015         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_PROFILE;
1016     }
1017
1018     if (priv->profile != profile) {
1019         GST_DEBUG("profile changed");
1020         reset_context = TRUE;
1021         priv->profile = profile;
1022     }
1023
1024     chroma_type = h264_get_chroma_type(sps);
1025     if (!chroma_type || chroma_type != GST_VAAPI_CHROMA_TYPE_YUV420) {
1026         GST_ERROR("unsupported chroma_format_idc %u", sps->chroma_format_idc);
1027         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;
1028     }
1029
1030     if (priv->chroma_type != chroma_type) {
1031         GST_DEBUG("chroma format changed");
1032         reset_context     = TRUE;
1033         priv->chroma_type = chroma_type;
1034     }
1035
1036     mb_width  = sps->pic_width_in_mbs_minus1 + 1;
1037     mb_height = (sps->pic_height_in_map_units_minus1 + 1) <<
1038         !sps->frame_mbs_only_flag;
1039     if (priv->mb_width != mb_width || priv->mb_height != mb_height) {
1040         GST_DEBUG("size changed");
1041         reset_context   = TRUE;
1042         priv->mb_width  = mb_width;
1043         priv->mb_height = mb_height;
1044     }
1045
1046     priv->progressive_sequence = sps->frame_mbs_only_flag;
1047 #if 0
1048     /* XXX: we only output complete frames for now */
1049     gst_vaapi_decoder_set_interlaced(base_decoder, !priv->progressive_sequence);
1050 #endif
1051
1052     gst_vaapi_decoder_set_pixel_aspect_ratio(
1053         base_decoder,
1054         sps->vui_parameters.par_n,
1055         sps->vui_parameters.par_d
1056     );
1057
1058     if (!reset_context && priv->has_context)
1059         return GST_VAAPI_DECODER_STATUS_SUCCESS;
1060
1061     /* XXX: fix surface size when cropping is implemented */
1062     info.profile    = priv->profile;
1063     info.entrypoint = priv->entrypoint;
1064     info.width      = sps->width;
1065     info.height     = sps->height;
1066     info.ref_frames = get_max_dec_frame_buffering(sps);
1067
1068     if (!gst_vaapi_decoder_ensure_context(GST_VAAPI_DECODER(decoder), &info))
1069         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
1070     priv->has_context = TRUE;
1071
1072     /* Reset DPB */
1073     dpb_reset(decoder, sps);
1074     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1075 }
1076
1077 static void
1078 fill_iq_matrix_4x4(VAIQMatrixBufferH264 *iq_matrix, const GstH264PPS *pps)
1079 {
1080     const guint8 (* const ScalingList4x4)[6][16] = &pps->scaling_lists_4x4;
1081     guint i, j;
1082
1083     /* There are always 6 4x4 scaling lists */
1084     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList4x4) == 6);
1085     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList4x4[0]) == 16);
1086
1087     if (sizeof(iq_matrix->ScalingList4x4[0][0]) == 1)
1088         memcpy(iq_matrix->ScalingList4x4, *ScalingList4x4,
1089                sizeof(iq_matrix->ScalingList4x4));
1090     else {
1091         for (i = 0; i < G_N_ELEMENTS(iq_matrix->ScalingList4x4); i++) {
1092             for (j = 0; j < G_N_ELEMENTS(iq_matrix->ScalingList4x4[i]); j++)
1093                 iq_matrix->ScalingList4x4[i][j] = (*ScalingList4x4)[i][j];
1094         }
1095     }
1096 }
1097
1098 static void
1099 fill_iq_matrix_8x8(VAIQMatrixBufferH264 *iq_matrix, const GstH264PPS *pps)
1100 {
1101     const guint8 (* const ScalingList8x8)[6][64] = &pps->scaling_lists_8x8;
1102     const GstH264SPS * const sps = pps->sequence;
1103     guint i, j, n;
1104
1105     /* If chroma_format_idc != 3, there are up to 2 8x8 scaling lists */
1106     if (!pps->transform_8x8_mode_flag)
1107         return;
1108
1109     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList8x8) >= 2);
1110     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList8x8[0]) == 64);
1111
1112     if (sizeof(iq_matrix->ScalingList8x8[0][0]) == 1)
1113         memcpy(iq_matrix->ScalingList8x8, *ScalingList8x8,
1114                sizeof(iq_matrix->ScalingList8x8));
1115     else {
1116         n = (sps->chroma_format_idc != 3) ? 2 : 6;
1117         for (i = 0; i < n; i++) {
1118             for (j = 0; j < G_N_ELEMENTS(iq_matrix->ScalingList8x8[i]); j++)
1119                 iq_matrix->ScalingList8x8[i][j] = (*ScalingList8x8)[i][j];
1120         }
1121     }
1122 }
1123
1124 static GstVaapiDecoderStatus
1125 ensure_quant_matrix(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
1126 {
1127     GstVaapiPicture * const base_picture = &picture->base;
1128     GstH264PPS * const pps = picture->pps;
1129     GstH264SPS * const sps = pps->sequence;
1130     VAIQMatrixBufferH264 *iq_matrix;
1131
1132     base_picture->iq_matrix = GST_VAAPI_IQ_MATRIX_NEW(H264, decoder);
1133     if (!base_picture->iq_matrix) {
1134         GST_ERROR("failed to allocate IQ matrix");
1135         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
1136     }
1137     iq_matrix = base_picture->iq_matrix->param;
1138
1139     /* XXX: we can only support 4:2:0 or 4:2:2 since ScalingLists8x8[]
1140        is not large enough to hold lists for 4:4:4 */
1141     if (sps->chroma_format_idc == 3)
1142         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;
1143
1144     fill_iq_matrix_4x4(iq_matrix, pps);
1145     fill_iq_matrix_8x8(iq_matrix, pps);
1146
1147     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1148 }
1149
1150 static GstVaapiDecoderStatus
1151 decode_current_picture(GstVaapiDecoderH264 *decoder)
1152 {
1153     GstVaapiDecoderH264Private * const priv = decoder->priv;
1154     GstVaapiPictureH264 * const picture = priv->current_picture;
1155     GstVaapiDecoderStatus status;
1156
1157     if (!picture)
1158         return GST_VAAPI_DECODER_STATUS_SUCCESS;
1159
1160     status = ensure_context(decoder, picture->pps->sequence);
1161     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
1162         return status;
1163
1164     if (!exec_ref_pic_marking(decoder, picture))
1165         goto error;
1166     if (!dpb_add(decoder, picture))
1167         goto error;
1168     if (!gst_vaapi_picture_decode(GST_VAAPI_PICTURE_CAST(picture)))
1169         goto error;
1170     if (priv->prev_frame && gst_vaapi_frame_store_has_frame(priv->prev_frame))
1171         gst_vaapi_picture_replace(&priv->current_picture, NULL);
1172     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1173
1174 error:
1175     /* XXX: fix for cases where first field failed to be decoded */
1176     gst_vaapi_picture_replace(&priv->current_picture, NULL);
1177     return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
1178 }
1179
1180 static GstVaapiDecoderStatus
1181 decode_sps(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
1182 {
1183     GstVaapiDecoderH264Private * const priv = decoder->priv;
1184     GstH264SPS * const sps = &priv->last_sps;
1185     GstH264ParserResult result;
1186
1187     GST_DEBUG("decode SPS");
1188
1189     memset(sps, 0, sizeof(*sps));
1190     result = gst_h264_parser_parse_sps(priv->parser, nalu, sps, TRUE);
1191     if (result != GST_H264_PARSER_OK) {
1192         priv->got_sps = FALSE;
1193         return get_status(result);
1194     }
1195
1196     priv->got_sps = TRUE;
1197     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1198 }
1199
1200 static GstVaapiDecoderStatus
1201 decode_pps(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
1202 {
1203     GstVaapiDecoderH264Private * const priv = decoder->priv;
1204     GstH264PPS * const pps = &priv->last_pps;
1205     GstH264ParserResult result;
1206
1207     GST_DEBUG("decode PPS");
1208
1209     memset(pps, 0, sizeof(*pps));
1210     result = gst_h264_parser_parse_pps(priv->parser, nalu, pps);
1211     if (result != GST_H264_PARSER_OK) {
1212         priv->got_pps = FALSE;
1213         return get_status(result);
1214     }
1215
1216     priv->got_pps = TRUE;
1217     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1218 }
1219
1220 static GstVaapiDecoderStatus
1221 decode_sei(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
1222 {
1223     GstVaapiDecoderH264Private * const priv = decoder->priv;
1224     GstH264SEIMessage sei;
1225     GstH264ParserResult result;
1226
1227     GST_DEBUG("decode SEI");
1228
1229     memset(&sei, 0, sizeof(sei));
1230     result = gst_h264_parser_parse_sei(priv->parser, nalu, &sei);
1231     if (result != GST_H264_PARSER_OK) {
1232         GST_WARNING("failed to decode SEI, payload type:%d", sei.payloadType);
1233         return get_status(result);
1234     }
1235
1236     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1237 }
1238
1239 static GstVaapiDecoderStatus
1240 decode_sequence_end(GstVaapiDecoderH264 *decoder)
1241 {
1242     GstVaapiDecoderStatus status;
1243
1244     GST_DEBUG("decode sequence-end");
1245
1246     status = decode_current_picture(decoder);
1247     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
1248         return status;
1249
1250     dpb_flush(decoder);
1251     return GST_VAAPI_DECODER_STATUS_END_OF_STREAM;
1252 }
1253
1254 /* 8.2.1.1 - Decoding process for picture order count type 0 */
1255 static void
1256 init_picture_poc_0(
1257     GstVaapiDecoderH264 *decoder,
1258     GstVaapiPictureH264 *picture,
1259     GstH264SliceHdr     *slice_hdr
1260 )
1261 {
1262     GstVaapiDecoderH264Private * const priv = decoder->priv;
1263     GstH264PPS * const pps = slice_hdr->pps;
1264     GstH264SPS * const sps = pps->sequence;
1265     const gint32 MaxPicOrderCntLsb = 1 << (sps->log2_max_pic_order_cnt_lsb_minus4 + 4);
1266     gint32 temp_poc;
1267
1268     GST_DEBUG("decode picture order count type 0");
1269
1270     if (GST_VAAPI_PICTURE_IS_IDR(picture)) {
1271         priv->prev_poc_msb = 0;
1272         priv->prev_poc_lsb = 0;
1273     }
1274     else if (priv->prev_pic_has_mmco5) {
1275         priv->prev_poc_msb = 0;
1276         priv->prev_poc_lsb =
1277             (priv->prev_pic_structure == GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD ?
1278              0 : priv->field_poc[TOP_FIELD]);
1279     }
1280     else {
1281         priv->prev_poc_msb = priv->poc_msb;
1282         priv->prev_poc_lsb = priv->poc_lsb;
1283     }
1284
1285     // (8-3)
1286     priv->poc_lsb = slice_hdr->pic_order_cnt_lsb;
1287     if (priv->poc_lsb < priv->prev_poc_lsb &&
1288         (priv->prev_poc_lsb - priv->poc_lsb) >= (MaxPicOrderCntLsb / 2))
1289         priv->poc_msb = priv->prev_poc_msb + MaxPicOrderCntLsb;
1290     else if (priv->poc_lsb > priv->prev_poc_lsb &&
1291              (priv->poc_lsb - priv->prev_poc_lsb) > (MaxPicOrderCntLsb / 2))
1292         priv->poc_msb = priv->prev_poc_msb - MaxPicOrderCntLsb;
1293     else
1294         priv->poc_msb = priv->prev_poc_msb;
1295
1296     temp_poc = priv->poc_msb + priv->poc_lsb;
1297     switch (picture->structure) {
1298     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
1299         // (8-4, 8-5)
1300         priv->field_poc[TOP_FIELD] = temp_poc;
1301         priv->field_poc[BOTTOM_FIELD] = temp_poc +
1302             slice_hdr->delta_pic_order_cnt_bottom;
1303         break;
1304     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
1305         // (8-4)
1306         priv->field_poc[TOP_FIELD] = temp_poc;
1307         break;
1308     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
1309         // (8-5)
1310         priv->field_poc[BOTTOM_FIELD] = temp_poc;
1311         break;
1312     }
1313 }
1314
1315 /* 8.2.1.2 - Decoding process for picture order count type 1 */
1316 static void
1317 init_picture_poc_1(
1318     GstVaapiDecoderH264 *decoder,
1319     GstVaapiPictureH264 *picture,
1320     GstH264SliceHdr     *slice_hdr
1321 )
1322 {
1323     GstVaapiDecoderH264Private * const priv = decoder->priv;
1324     GstH264PPS * const pps = slice_hdr->pps;
1325     GstH264SPS * const sps = pps->sequence;
1326     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1327     gint32 prev_frame_num_offset, abs_frame_num, expected_poc;
1328     guint i;
1329
1330     GST_DEBUG("decode picture order count type 1");
1331
1332     if (priv->prev_pic_has_mmco5)
1333         prev_frame_num_offset = 0;
1334     else
1335         prev_frame_num_offset = priv->frame_num_offset;
1336
1337     // (8-6)
1338     if (GST_VAAPI_PICTURE_IS_IDR(picture))
1339         priv->frame_num_offset = 0;
1340     else if (priv->prev_frame_num > priv->frame_num)
1341         priv->frame_num_offset = prev_frame_num_offset + MaxFrameNum;
1342     else
1343         priv->frame_num_offset = prev_frame_num_offset;
1344
1345     // (8-7)
1346     if (sps->num_ref_frames_in_pic_order_cnt_cycle != 0)
1347         abs_frame_num = priv->frame_num_offset + priv->frame_num;
1348     else
1349         abs_frame_num = 0;
1350     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture) && abs_frame_num > 0)
1351         abs_frame_num = abs_frame_num - 1;
1352
1353     if (abs_frame_num > 0) {
1354         gint32 expected_delta_per_poc_cycle;
1355         gint32 poc_cycle_cnt, frame_num_in_poc_cycle;
1356
1357         expected_delta_per_poc_cycle = 0;
1358         for (i = 0; i < sps->num_ref_frames_in_pic_order_cnt_cycle; i++)
1359             expected_delta_per_poc_cycle += sps->offset_for_ref_frame[i];
1360
1361         // (8-8)
1362         poc_cycle_cnt = (abs_frame_num - 1) /
1363             sps->num_ref_frames_in_pic_order_cnt_cycle;
1364         frame_num_in_poc_cycle = (abs_frame_num - 1) %
1365             sps->num_ref_frames_in_pic_order_cnt_cycle;
1366
1367         // (8-9)
1368         expected_poc = poc_cycle_cnt * expected_delta_per_poc_cycle;
1369         for (i = 0; i <= frame_num_in_poc_cycle; i++)
1370             expected_poc += sps->offset_for_ref_frame[i];
1371     }
1372     else
1373         expected_poc = 0;
1374     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
1375         expected_poc += sps->offset_for_non_ref_pic;
1376
1377     // (8-10)
1378     switch (picture->structure) {
1379     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
1380         priv->field_poc[TOP_FIELD] = expected_poc +
1381             slice_hdr->delta_pic_order_cnt[0];
1382         priv->field_poc[BOTTOM_FIELD] = priv->field_poc[TOP_FIELD] +
1383             sps->offset_for_top_to_bottom_field +
1384             slice_hdr->delta_pic_order_cnt[1];
1385         break;
1386     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
1387         priv->field_poc[TOP_FIELD] = expected_poc +
1388             slice_hdr->delta_pic_order_cnt[0];
1389         break;
1390     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
1391         priv->field_poc[BOTTOM_FIELD] = expected_poc +
1392             sps->offset_for_top_to_bottom_field +
1393             slice_hdr->delta_pic_order_cnt[0];
1394         break;
1395     }
1396 }
1397
1398 /* 8.2.1.3 - Decoding process for picture order count type 2 */
1399 static void
1400 init_picture_poc_2(
1401     GstVaapiDecoderH264 *decoder,
1402     GstVaapiPictureH264 *picture,
1403     GstH264SliceHdr     *slice_hdr
1404 )
1405 {
1406     GstVaapiDecoderH264Private * const priv = decoder->priv;
1407     GstH264PPS * const pps = slice_hdr->pps;
1408     GstH264SPS * const sps = pps->sequence;
1409     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1410     gint32 prev_frame_num_offset, temp_poc;
1411
1412     GST_DEBUG("decode picture order count type 2");
1413
1414     if (priv->prev_pic_has_mmco5)
1415         prev_frame_num_offset = 0;
1416     else
1417         prev_frame_num_offset = priv->frame_num_offset;
1418
1419     // (8-11)
1420     if (GST_VAAPI_PICTURE_IS_IDR(picture))
1421         priv->frame_num_offset = 0;
1422     else if (priv->prev_frame_num > priv->frame_num)
1423         priv->frame_num_offset = prev_frame_num_offset + MaxFrameNum;
1424     else
1425         priv->frame_num_offset = prev_frame_num_offset;
1426
1427     // (8-12)
1428     if (GST_VAAPI_PICTURE_IS_IDR(picture))
1429         temp_poc = 0;
1430     else if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
1431         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num) - 1;
1432     else
1433         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num);
1434
1435     // (8-13)
1436     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
1437         priv->field_poc[TOP_FIELD] = temp_poc;
1438     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
1439         priv->field_poc[BOTTOM_FIELD] = temp_poc;
1440 }
1441
1442 /* 8.2.1 - Decoding process for picture order count */
1443 static void
1444 init_picture_poc(
1445     GstVaapiDecoderH264 *decoder,
1446     GstVaapiPictureH264 *picture,
1447     GstH264SliceHdr     *slice_hdr
1448 )
1449 {
1450     GstVaapiDecoderH264Private * const priv = decoder->priv;
1451     GstH264PPS * const pps = slice_hdr->pps;
1452     GstH264SPS * const sps = pps->sequence;
1453
1454     switch (sps->pic_order_cnt_type) {
1455     case 0:
1456         init_picture_poc_0(decoder, picture, slice_hdr);
1457         break;
1458     case 1:
1459         init_picture_poc_1(decoder, picture, slice_hdr);
1460         break;
1461     case 2:
1462         init_picture_poc_2(decoder, picture, slice_hdr);
1463         break;
1464     }
1465
1466     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
1467         picture->field_poc[TOP_FIELD] = priv->field_poc[TOP_FIELD];
1468     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
1469         picture->field_poc[BOTTOM_FIELD] = priv->field_poc[BOTTOM_FIELD];
1470     picture->base.poc = MIN(picture->field_poc[0], picture->field_poc[1]);
1471 }
1472
1473 static int
1474 compare_picture_pic_num_dec(const void *a, const void *b)
1475 {
1476     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1477     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1478
1479     return picB->pic_num - picA->pic_num;
1480 }
1481
1482 static int
1483 compare_picture_long_term_pic_num_inc(const void *a, const void *b)
1484 {
1485     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1486     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1487
1488     return picA->long_term_pic_num - picB->long_term_pic_num;
1489 }
1490
1491 static int
1492 compare_picture_poc_dec(const void *a, const void *b)
1493 {
1494     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1495     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1496
1497     return picB->base.poc - picA->base.poc;
1498 }
1499
1500 static int
1501 compare_picture_poc_inc(const void *a, const void *b)
1502 {
1503     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1504     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1505
1506     return picA->base.poc - picB->base.poc;
1507 }
1508
1509 static int
1510 compare_picture_frame_num_wrap_dec(const void *a, const void *b)
1511 {
1512     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1513     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1514
1515     return picB->frame_num_wrap - picA->frame_num_wrap;
1516 }
1517
1518 static int
1519 compare_picture_long_term_frame_idx_inc(const void *a, const void *b)
1520 {
1521     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1522     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1523
1524     return picA->long_term_frame_idx - picB->long_term_frame_idx;
1525 }
1526
1527 /* 8.2.4.1 - Decoding process for picture numbers */
1528 static void
1529 init_picture_refs_pic_num(
1530     GstVaapiDecoderH264 *decoder,
1531     GstVaapiPictureH264 *picture,
1532     GstH264SliceHdr     *slice_hdr
1533 )
1534 {
1535     GstVaapiDecoderH264Private * const priv = decoder->priv;
1536     GstH264PPS * const pps = slice_hdr->pps;
1537     GstH264SPS * const sps = pps->sequence;
1538     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1539     guint i;
1540
1541     GST_DEBUG("decode picture numbers");
1542
1543     for (i = 0; i < priv->short_ref_count; i++) {
1544         GstVaapiPictureH264 * const pic = priv->short_ref[i];
1545
1546         // (8-27)
1547         if (pic->frame_num > priv->frame_num)
1548             pic->frame_num_wrap = pic->frame_num - MaxFrameNum;
1549         else
1550             pic->frame_num_wrap = pic->frame_num;
1551
1552         // (8-28, 8-30, 8-31)
1553         if (GST_VAAPI_PICTURE_IS_FRAME(picture))
1554             pic->pic_num = pic->frame_num_wrap;
1555         else {
1556             if (pic->structure == picture->structure)
1557                 pic->pic_num = 2 * pic->frame_num_wrap + 1;
1558             else
1559                 pic->pic_num = 2 * pic->frame_num_wrap;
1560         }
1561     }
1562
1563     for (i = 0; i < priv->long_ref_count; i++) {
1564         GstVaapiPictureH264 * const pic = priv->long_ref[i];
1565
1566         // (8-29, 8-32, 8-33)
1567         if (GST_VAAPI_PICTURE_IS_FRAME(picture))
1568             pic->long_term_pic_num = pic->long_term_frame_idx;
1569         else {
1570             if (pic->structure == picture->structure)
1571                 pic->long_term_pic_num = 2 * pic->long_term_frame_idx + 1;
1572             else
1573                 pic->long_term_pic_num = 2 * pic->long_term_frame_idx;
1574         }
1575     }
1576 }
1577
1578 #define SORT_REF_LIST(list, n, compare_func) \
1579     qsort(list, n, sizeof(*(list)), compare_picture_##compare_func)
1580
1581 static void
1582 init_picture_refs_fields_1(
1583     guint                picture_structure,
1584     GstVaapiPictureH264 *RefPicList[32],
1585     guint               *RefPicList_count,
1586     GstVaapiPictureH264 *ref_list[32],
1587     guint                ref_list_count
1588 )
1589 {
1590     guint i, j, n;
1591
1592     i = 0;
1593     j = 0;
1594     n = *RefPicList_count;
1595     do {
1596         g_assert(n < 32);
1597         for (; i < ref_list_count; i++) {
1598             if (ref_list[i]->structure == picture_structure) {
1599                 RefPicList[n++] = ref_list[i++];
1600                 break;
1601             }
1602         }
1603         for (; j < ref_list_count; j++) {
1604             if (ref_list[j]->structure != picture_structure) {
1605                 RefPicList[n++] = ref_list[j++];
1606                 break;
1607             }
1608         }
1609     } while (i < ref_list_count || j < ref_list_count);
1610     *RefPicList_count = n;
1611 }
1612
1613 static inline void
1614 init_picture_refs_fields(
1615     GstVaapiPictureH264 *picture,
1616     GstVaapiPictureH264 *RefPicList[32],
1617     guint               *RefPicList_count,
1618     GstVaapiPictureH264 *short_ref[32],
1619     guint                short_ref_count,
1620     GstVaapiPictureH264 *long_ref[32],
1621     guint                long_ref_count
1622 )
1623 {
1624     guint n = 0;
1625
1626     /* 8.2.4.2.5 - reference picture lists in fields */
1627     init_picture_refs_fields_1(picture->structure, RefPicList, &n,
1628         short_ref, short_ref_count);
1629     init_picture_refs_fields_1(picture->structure, RefPicList, &n,
1630         long_ref, long_ref_count);
1631     *RefPicList_count = n;
1632 }
1633
1634 static void
1635 init_picture_refs_p_slice(
1636     GstVaapiDecoderH264 *decoder,
1637     GstVaapiPictureH264 *picture,
1638     GstH264SliceHdr     *slice_hdr
1639 )
1640 {
1641     GstVaapiDecoderH264Private * const priv = decoder->priv;
1642     GstVaapiPictureH264 **ref_list;
1643     guint i;
1644
1645     GST_DEBUG("decode reference picture list for P and SP slices");
1646
1647     if (GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1648         /* 8.2.4.2.1 - P and SP slices in frames */
1649         if (priv->short_ref_count > 0) {
1650             ref_list = priv->RefPicList0;
1651             for (i = 0; i < priv->short_ref_count; i++)
1652                 ref_list[i] = priv->short_ref[i];
1653             SORT_REF_LIST(ref_list, i, pic_num_dec);
1654             priv->RefPicList0_count += i;
1655         }
1656
1657         if (priv->long_ref_count > 0) {
1658             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1659             for (i = 0; i < priv->long_ref_count; i++)
1660                 ref_list[i] = priv->long_ref[i];
1661             SORT_REF_LIST(ref_list, i, long_term_pic_num_inc);
1662             priv->RefPicList0_count += i;
1663         }
1664     }
1665     else {
1666         /* 8.2.4.2.2 - P and SP slices in fields */
1667         GstVaapiPictureH264 *short_ref[32];
1668         guint short_ref_count = 0;
1669         GstVaapiPictureH264 *long_ref[32];
1670         guint long_ref_count = 0;
1671
1672         if (priv->short_ref_count > 0) {
1673             for (i = 0; i < priv->short_ref_count; i++)
1674                 short_ref[i] = priv->short_ref[i];
1675             SORT_REF_LIST(short_ref, i, frame_num_wrap_dec);
1676             short_ref_count = i;
1677         }
1678
1679         if (priv->long_ref_count > 0) {
1680             for (i = 0; i < priv->long_ref_count; i++)
1681                 long_ref[i] = priv->long_ref[i];
1682             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1683             long_ref_count = i;
1684         }
1685
1686         init_picture_refs_fields(
1687             picture,
1688             priv->RefPicList0, &priv->RefPicList0_count,
1689             short_ref,          short_ref_count,
1690             long_ref,           long_ref_count
1691         );
1692     }
1693 }
1694
1695 static void
1696 init_picture_refs_b_slice(
1697     GstVaapiDecoderH264 *decoder,
1698     GstVaapiPictureH264 *picture,
1699     GstH264SliceHdr     *slice_hdr
1700 )
1701 {
1702     GstVaapiDecoderH264Private * const priv = decoder->priv;
1703     GstVaapiPictureH264 **ref_list;
1704     guint i, n;
1705
1706     GST_DEBUG("decode reference picture list for B slices");
1707
1708     if (GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1709         /* 8.2.4.2.3 - B slices in frames */
1710
1711         /* RefPicList0 */
1712         if (priv->short_ref_count > 0) {
1713             // 1. Short-term references
1714             ref_list = priv->RefPicList0;
1715             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1716                 if (priv->short_ref[i]->base.poc < picture->base.poc)
1717                     ref_list[n++] = priv->short_ref[i];
1718             }
1719             SORT_REF_LIST(ref_list, n, poc_dec);
1720             priv->RefPicList0_count += n;
1721
1722             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1723             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1724                 if (priv->short_ref[i]->base.poc >= picture->base.poc)
1725                     ref_list[n++] = priv->short_ref[i];
1726             }
1727             SORT_REF_LIST(ref_list, n, poc_inc);
1728             priv->RefPicList0_count += n;
1729         }
1730
1731         if (priv->long_ref_count > 0) {
1732             // 2. Long-term references
1733             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1734             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1735                 ref_list[n++] = priv->long_ref[i];
1736             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1737             priv->RefPicList0_count += n;
1738         }
1739
1740         /* RefPicList1 */
1741         if (priv->short_ref_count > 0) {
1742             // 1. Short-term references
1743             ref_list = priv->RefPicList1;
1744             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1745                 if (priv->short_ref[i]->base.poc > picture->base.poc)
1746                     ref_list[n++] = priv->short_ref[i];
1747             }
1748             SORT_REF_LIST(ref_list, n, poc_inc);
1749             priv->RefPicList1_count += n;
1750
1751             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1752             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1753                 if (priv->short_ref[i]->base.poc <= picture->base.poc)
1754                     ref_list[n++] = priv->short_ref[i];
1755             }
1756             SORT_REF_LIST(ref_list, n, poc_dec);
1757             priv->RefPicList1_count += n;
1758         }
1759
1760         if (priv->long_ref_count > 0) {
1761             // 2. Long-term references
1762             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1763             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1764                 ref_list[n++] = priv->long_ref[i];
1765             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1766             priv->RefPicList1_count += n;
1767         }
1768     }
1769     else {
1770         /* 8.2.4.2.4 - B slices in fields */
1771         GstVaapiPictureH264 *short_ref0[32];
1772         guint short_ref0_count = 0;
1773         GstVaapiPictureH264 *short_ref1[32];
1774         guint short_ref1_count = 0;
1775         GstVaapiPictureH264 *long_ref[32];
1776         guint long_ref_count = 0;
1777
1778         /* refFrameList0ShortTerm */
1779         if (priv->short_ref_count > 0) {
1780             ref_list = short_ref0;
1781             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1782                 if (priv->short_ref[i]->base.poc <= picture->base.poc)
1783                     ref_list[n++] = priv->short_ref[i];
1784             }
1785             SORT_REF_LIST(ref_list, n, poc_dec);
1786             short_ref0_count += n;
1787
1788             ref_list = &short_ref0[short_ref0_count];
1789             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1790                 if (priv->short_ref[i]->base.poc > picture->base.poc)
1791                     ref_list[n++] = priv->short_ref[i];
1792             }
1793             SORT_REF_LIST(ref_list, n, poc_inc);
1794             short_ref0_count += n;
1795         }
1796
1797         /* refFrameList1ShortTerm */
1798         if (priv->short_ref_count > 0) {
1799             ref_list = short_ref1;
1800             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1801                 if (priv->short_ref[i]->base.poc > picture->base.poc)
1802                     ref_list[n++] = priv->short_ref[i];
1803             }
1804             SORT_REF_LIST(ref_list, n, poc_inc);
1805             short_ref1_count += n;
1806
1807             ref_list = &short_ref1[short_ref1_count];
1808             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1809                 if (priv->short_ref[i]->base.poc <= picture->base.poc)
1810                     ref_list[n++] = priv->short_ref[i];
1811             }
1812             SORT_REF_LIST(ref_list, n, poc_dec);
1813             short_ref1_count += n;
1814         }
1815
1816         /* refFrameListLongTerm */
1817         if (priv->long_ref_count > 0) {
1818             for (i = 0; i < priv->long_ref_count; i++)
1819                 long_ref[i] = priv->long_ref[i];
1820             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1821             long_ref_count = i;
1822         }
1823
1824         init_picture_refs_fields(
1825             picture,
1826             priv->RefPicList0, &priv->RefPicList0_count,
1827             short_ref0,         short_ref0_count,
1828             long_ref,           long_ref_count
1829         );
1830
1831         init_picture_refs_fields(
1832             picture,
1833             priv->RefPicList1, &priv->RefPicList1_count,
1834             short_ref1,         short_ref1_count,
1835             long_ref,           long_ref_count
1836         );
1837    }
1838
1839     /* Check whether RefPicList1 is identical to RefPicList0, then
1840        swap if necessary */
1841     if (priv->RefPicList1_count > 1 &&
1842         priv->RefPicList1_count == priv->RefPicList0_count &&
1843         memcmp(priv->RefPicList0, priv->RefPicList1,
1844                priv->RefPicList0_count * sizeof(priv->RefPicList0[0])) == 0) {
1845         GstVaapiPictureH264 * const tmp = priv->RefPicList1[0];
1846         priv->RefPicList1[0] = priv->RefPicList1[1];
1847         priv->RefPicList1[1] = tmp;
1848     }
1849 }
1850
1851 #undef SORT_REF_LIST
1852
1853 static gint
1854 find_short_term_reference(GstVaapiDecoderH264 *decoder, gint32 pic_num)
1855 {
1856     GstVaapiDecoderH264Private * const priv = decoder->priv;
1857     guint i;
1858
1859     for (i = 0; i < priv->short_ref_count; i++) {
1860         if (priv->short_ref[i]->pic_num == pic_num)
1861             return i;
1862     }
1863     GST_ERROR("found no short-term reference picture with PicNum = %d",
1864               pic_num);
1865     return -1;
1866 }
1867
1868 static gint
1869 find_long_term_reference(GstVaapiDecoderH264 *decoder, gint32 long_term_pic_num)
1870 {
1871     GstVaapiDecoderH264Private * const priv = decoder->priv;
1872     guint i;
1873
1874     for (i = 0; i < priv->long_ref_count; i++) {
1875         if (priv->long_ref[i]->long_term_pic_num == long_term_pic_num)
1876             return i;
1877     }
1878     GST_ERROR("found no long-term reference picture with LongTermPicNum = %d",
1879               long_term_pic_num);
1880     return -1;
1881 }
1882
1883 static void
1884 exec_picture_refs_modification_1(
1885     GstVaapiDecoderH264           *decoder,
1886     GstVaapiPictureH264           *picture,
1887     GstH264SliceHdr               *slice_hdr,
1888     guint                          list
1889 )
1890 {
1891     GstVaapiDecoderH264Private * const priv = decoder->priv;
1892     GstH264PPS * const pps = slice_hdr->pps;
1893     GstH264SPS * const sps = pps->sequence;
1894     GstH264RefPicListModification *ref_pic_list_modification;
1895     guint num_ref_pic_list_modifications;
1896     GstVaapiPictureH264 **ref_list;
1897     guint *ref_list_count_ptr, ref_list_count, ref_list_idx = 0;
1898     guint i, j, n, num_refs;
1899     gint found_ref_idx;
1900     gint32 MaxPicNum, CurrPicNum, picNumPred;
1901
1902     GST_DEBUG("modification process of reference picture list %u", list);
1903
1904     if (list == 0) {
1905         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l0;
1906         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l0;
1907         ref_list                       = priv->RefPicList0;
1908         ref_list_count_ptr             = &priv->RefPicList0_count;
1909         num_refs                       = slice_hdr->num_ref_idx_l0_active_minus1 + 1;
1910     }
1911     else {
1912         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l1;
1913         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l1;
1914         ref_list                       = priv->RefPicList1;
1915         ref_list_count_ptr             = &priv->RefPicList1_count;
1916         num_refs                       = slice_hdr->num_ref_idx_l1_active_minus1 + 1;
1917     }
1918     ref_list_count = *ref_list_count_ptr;
1919     if (num_refs > ref_list_count)
1920         num_refs = ref_list_count;
1921
1922     if (!GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1923         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 5); // 2 * MaxFrameNum
1924         CurrPicNum = 2 * slice_hdr->frame_num + 1;              // 2 * frame_num + 1
1925     }
1926     else {
1927         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 4); // MaxFrameNum
1928         CurrPicNum = slice_hdr->frame_num;                      // frame_num
1929     }
1930
1931     picNumPred = CurrPicNum;
1932
1933     for (i = 0; i < num_ref_pic_list_modifications; i++) {
1934         GstH264RefPicListModification * const l = &ref_pic_list_modification[i];
1935         if (l->modification_of_pic_nums_idc == 3)
1936             break;
1937
1938         /* 8.2.4.3.1 - Short-term reference pictures */
1939         if (l->modification_of_pic_nums_idc == 0 || l->modification_of_pic_nums_idc == 1) {
1940             gint32 abs_diff_pic_num = l->value.abs_diff_pic_num_minus1 + 1;
1941             gint32 picNum, picNumNoWrap;
1942
1943             // (8-34)
1944             if (l->modification_of_pic_nums_idc == 0) {
1945                 picNumNoWrap = picNumPred - abs_diff_pic_num;
1946                 if (picNumNoWrap < 0)
1947                     picNumNoWrap += MaxPicNum;
1948             }
1949
1950             // (8-35)
1951             else {
1952                 picNumNoWrap = picNumPred + abs_diff_pic_num;
1953                 if (picNumNoWrap >= MaxPicNum)
1954                     picNumNoWrap -= MaxPicNum;
1955             }
1956             picNumPred = picNumNoWrap;
1957
1958             // (8-36)
1959             picNum = picNumNoWrap;
1960             if (picNum > CurrPicNum)
1961                 picNum -= MaxPicNum;
1962
1963             // (8-37)
1964             for (j = num_refs; j > ref_list_idx; j--)
1965                 ref_list[j] = ref_list[j - 1];
1966             found_ref_idx = find_short_term_reference(decoder, picNum);
1967             ref_list[ref_list_idx++] =
1968                 found_ref_idx >= 0 ? priv->short_ref[found_ref_idx] : NULL;
1969             n = ref_list_idx;
1970             for (j = ref_list_idx; j <= num_refs; j++) {
1971                 gint32 PicNumF;
1972                 if (!ref_list[j])
1973                     continue;
1974                 PicNumF =
1975                     GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(ref_list[j]) ?
1976                     ref_list[j]->pic_num : MaxPicNum;
1977                 if (PicNumF != picNum)
1978                     ref_list[n++] = ref_list[j];
1979             }
1980         }
1981
1982         /* 8.2.4.3.2 - Long-term reference pictures */
1983         else {
1984
1985             for (j = num_refs; j > ref_list_idx; j--)
1986                 ref_list[j] = ref_list[j - 1];
1987             found_ref_idx =
1988                 find_long_term_reference(decoder, l->value.long_term_pic_num);
1989             ref_list[ref_list_idx++] =
1990                 found_ref_idx >= 0 ? priv->long_ref[found_ref_idx] : NULL;
1991             n = ref_list_idx;
1992             for (j = ref_list_idx; j <= num_refs; j++) {
1993                 gint32 LongTermPicNumF;
1994                 if (!ref_list[j])
1995                     continue;
1996                 LongTermPicNumF =
1997                     GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(ref_list[j]) ?
1998                     ref_list[j]->long_term_pic_num : INT_MAX;
1999                 if (LongTermPicNumF != l->value.long_term_pic_num)
2000                     ref_list[n++] = ref_list[j];
2001             }
2002         }
2003     }
2004
2005 #if DEBUG
2006     for (i = 0; i < num_refs; i++)
2007         if (!ref_list[i])
2008             GST_ERROR("list %u entry %u is empty", list, i);
2009 #endif
2010     *ref_list_count_ptr = num_refs;
2011 }
2012
2013 /* 8.2.4.3 - Modification process for reference picture lists */
2014 static void
2015 exec_picture_refs_modification(
2016     GstVaapiDecoderH264 *decoder,
2017     GstVaapiPictureH264 *picture,
2018     GstH264SliceHdr     *slice_hdr
2019 )
2020 {
2021     GST_DEBUG("execute ref_pic_list_modification()");
2022
2023     /* RefPicList0 */
2024     if (!GST_H264_IS_I_SLICE(slice_hdr) && !GST_H264_IS_SI_SLICE(slice_hdr) &&
2025         slice_hdr->ref_pic_list_modification_flag_l0)
2026         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 0);
2027
2028     /* RefPicList1 */
2029     if (GST_H264_IS_B_SLICE(slice_hdr) &&
2030         slice_hdr->ref_pic_list_modification_flag_l1)
2031         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 1);
2032 }
2033
2034 static void
2035 init_picture_ref_lists(GstVaapiDecoderH264 *decoder)
2036 {
2037     GstVaapiDecoderH264Private * const priv = decoder->priv;
2038     guint i, j, short_ref_count, long_ref_count;
2039
2040     short_ref_count = 0;
2041     long_ref_count  = 0;
2042     if (GST_VAAPI_PICTURE_IS_FRAME(priv->current_picture)) {
2043         for (i = 0; i < priv->dpb_count; i++) {
2044             GstVaapiFrameStore * const fs = priv->dpb[i];
2045             GstVaapiPictureH264 *picture;
2046             if (!gst_vaapi_frame_store_has_frame(fs))
2047                 continue;
2048             picture = fs->buffers[0];
2049             if (GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture))
2050                 priv->short_ref[short_ref_count++] = picture;
2051             else if (GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture))
2052                 priv->long_ref[long_ref_count++] = picture;
2053             picture->structure = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
2054             picture->other_field = fs->buffers[1];
2055         }
2056     }
2057     else {
2058         for (i = 0; i < priv->dpb_count; i++) {
2059             GstVaapiFrameStore * const fs = priv->dpb[i];
2060             for (j = 0; j < fs->num_buffers; j++) {
2061                 GstVaapiPictureH264 * const picture = fs->buffers[j];
2062                 if (GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture))
2063                     priv->short_ref[short_ref_count++] = picture;
2064                 else if (GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture))
2065                     priv->long_ref[long_ref_count++] = picture;
2066                 picture->structure = picture->base.structure;
2067                 picture->other_field = fs->buffers[j ^ 1];
2068             }
2069         }
2070     }
2071
2072     for (i = short_ref_count; i < priv->short_ref_count; i++)
2073         priv->short_ref[i] = NULL;
2074     priv->short_ref_count = short_ref_count;
2075
2076     for (i = long_ref_count; i < priv->long_ref_count; i++)
2077         priv->long_ref[i] = NULL;
2078     priv->long_ref_count = long_ref_count;
2079 }
2080
2081 static void
2082 init_picture_refs(
2083     GstVaapiDecoderH264 *decoder,
2084     GstVaapiPictureH264 *picture,
2085     GstH264SliceHdr     *slice_hdr
2086 )
2087 {
2088     GstVaapiDecoderH264Private * const priv = decoder->priv;
2089     GstVaapiPicture * const base_picture = &picture->base;
2090     guint i, num_refs;
2091
2092     init_picture_ref_lists(decoder);
2093     init_picture_refs_pic_num(decoder, picture, slice_hdr);
2094
2095     priv->RefPicList0_count = 0;
2096     priv->RefPicList1_count = 0;
2097
2098     switch (base_picture->type) {
2099     case GST_VAAPI_PICTURE_TYPE_P:
2100     case GST_VAAPI_PICTURE_TYPE_SP:
2101         init_picture_refs_p_slice(decoder, picture, slice_hdr);
2102         break;
2103     case GST_VAAPI_PICTURE_TYPE_B:
2104         init_picture_refs_b_slice(decoder, picture, slice_hdr);
2105         break;
2106     default:
2107         break;
2108     }
2109
2110     exec_picture_refs_modification(decoder, picture, slice_hdr);
2111
2112     switch (base_picture->type) {
2113     case GST_VAAPI_PICTURE_TYPE_B:
2114         num_refs = 1 + slice_hdr->num_ref_idx_l1_active_minus1;
2115         for (i = priv->RefPicList1_count; i < num_refs; i++)
2116             priv->RefPicList1[i] = NULL;
2117         //priv->RefPicList1_count = num_refs;
2118
2119         // fall-through
2120     case GST_VAAPI_PICTURE_TYPE_P:
2121     case GST_VAAPI_PICTURE_TYPE_SP:
2122         num_refs = 1 + slice_hdr->num_ref_idx_l0_active_minus1;
2123         for (i = priv->RefPicList0_count; i < num_refs; i++)
2124             priv->RefPicList0[i] = NULL;
2125         //priv->RefPicList0_count = num_refs;
2126         break;
2127     default:
2128         break;
2129     }
2130 }
2131
2132 static gboolean
2133 init_picture(
2134     GstVaapiDecoderH264 *decoder,
2135     GstVaapiPictureH264 *picture,
2136     GstH264SliceHdr     *slice_hdr,
2137     GstH264NalUnit      *nalu
2138 )
2139 {
2140     GstVaapiDecoderH264Private * const priv = decoder->priv;
2141     GstVaapiPicture * const base_picture = &picture->base;
2142
2143     priv->prev_frame_num        = priv->frame_num;
2144     priv->frame_num             = slice_hdr->frame_num;
2145     picture->frame_num          = priv->frame_num;
2146     picture->frame_num_wrap     = priv->frame_num;
2147     picture->output_flag        = TRUE; /* XXX: conformant to Annex A only */
2148     base_picture->pts           = gst_adapter_prev_timestamp(priv->adapter, NULL);
2149
2150     /* Reset decoder state for IDR pictures */
2151     if (nalu->type == GST_H264_NAL_SLICE_IDR) {
2152         GST_DEBUG("<IDR>");
2153         GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_IDR);
2154         dpb_flush(decoder);
2155     }
2156
2157     /* Initialize slice type */
2158     switch (slice_hdr->type % 5) {
2159     case GST_H264_P_SLICE:
2160         base_picture->type = GST_VAAPI_PICTURE_TYPE_P;
2161         break;
2162     case GST_H264_B_SLICE:
2163         base_picture->type = GST_VAAPI_PICTURE_TYPE_B;
2164         break;
2165     case GST_H264_I_SLICE:
2166         base_picture->type = GST_VAAPI_PICTURE_TYPE_I;
2167         break;
2168     case GST_H264_SP_SLICE:
2169         base_picture->type = GST_VAAPI_PICTURE_TYPE_SP;
2170         break;
2171     case GST_H264_SI_SLICE:
2172         base_picture->type = GST_VAAPI_PICTURE_TYPE_SI;
2173         break;
2174     }
2175
2176     /* Initialize picture structure */
2177     if (!slice_hdr->field_pic_flag)
2178         base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
2179     else {
2180         GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_INTERLACED);
2181         if (!slice_hdr->bottom_field_flag)
2182             base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD;
2183         else
2184             base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD;
2185     }
2186     picture->structure = base_picture->structure;
2187
2188     /* Initialize reference flags */
2189     if (nalu->ref_idc) {
2190         GstH264DecRefPicMarking * const dec_ref_pic_marking =
2191             &slice_hdr->dec_ref_pic_marking;
2192
2193         if (GST_VAAPI_PICTURE_IS_IDR(picture) &&
2194             dec_ref_pic_marking->long_term_reference_flag)
2195             GST_VAAPI_PICTURE_FLAG_SET(picture,
2196                 GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE);
2197         else
2198             GST_VAAPI_PICTURE_FLAG_SET(picture,
2199                 GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE);
2200     }
2201
2202     init_picture_poc(decoder, picture, slice_hdr);
2203     init_picture_refs(decoder, picture, slice_hdr);
2204     return TRUE;
2205 }
2206
2207 /* 8.2.5.3 - Sliding window decoded reference picture marking process */
2208 static gboolean
2209 exec_ref_pic_marking_sliding_window(GstVaapiDecoderH264 *decoder)
2210 {
2211     GstVaapiDecoderH264Private * const priv = decoder->priv;
2212     GstH264PPS * const pps = priv->current_picture->pps;
2213     GstH264SPS * const sps = pps->sequence;
2214     GstVaapiPictureH264 *ref_picture;
2215     guint i, m, max_num_ref_frames;
2216
2217     GST_DEBUG("reference picture marking process (sliding window)");
2218
2219     if (!GST_VAAPI_PICTURE_IS_FIRST_FIELD(priv->current_picture))
2220         return TRUE;
2221
2222     max_num_ref_frames = sps->num_ref_frames;
2223     if (max_num_ref_frames == 0)
2224         max_num_ref_frames = 1;
2225     if (!GST_VAAPI_PICTURE_IS_FRAME(priv->current_picture))
2226         max_num_ref_frames <<= 1;
2227
2228     if (priv->short_ref_count + priv->long_ref_count < max_num_ref_frames)
2229         return TRUE;
2230     if (priv->short_ref_count < 1)
2231         return FALSE;
2232
2233     for (m = 0, i = 1; i < priv->short_ref_count; i++) {
2234         GstVaapiPictureH264 * const picture = priv->short_ref[i];
2235         if (picture->frame_num_wrap < priv->short_ref[m]->frame_num_wrap)
2236             m = i;
2237     }
2238
2239     ref_picture = priv->short_ref[m];
2240     gst_vaapi_picture_h264_set_reference(ref_picture, 0, TRUE);
2241     ARRAY_REMOVE_INDEX(priv->short_ref, m);
2242
2243     /* Both fields need to be marked as "unused for reference", so
2244        remove the other field from the short_ref[] list as well */
2245     if (!GST_VAAPI_PICTURE_IS_FRAME(priv->current_picture) && ref_picture->other_field) {
2246         for (i = 0; i < priv->short_ref_count; i++) {
2247             if (priv->short_ref[i] == ref_picture->other_field) {
2248                 ARRAY_REMOVE_INDEX(priv->short_ref, i);
2249                 break;
2250             }
2251         }
2252     }
2253     return TRUE;
2254 }
2255
2256 static inline gint32
2257 get_picNumX(GstVaapiPictureH264 *picture, GstH264RefPicMarking *ref_pic_marking)
2258 {
2259     gint32 pic_num;
2260
2261     if (GST_VAAPI_PICTURE_IS_FRAME(picture))
2262         pic_num = picture->frame_num_wrap;
2263     else
2264         pic_num = 2 * picture->frame_num_wrap + 1;
2265     pic_num -= ref_pic_marking->difference_of_pic_nums_minus1 + 1;
2266     return pic_num;
2267 }
2268
2269 /* 8.2.5.4.1. Mark short-term reference picture as "unused for reference" */
2270 static void
2271 exec_ref_pic_marking_adaptive_mmco_1(
2272     GstVaapiDecoderH264  *decoder,
2273     GstVaapiPictureH264  *picture,
2274     GstH264RefPicMarking *ref_pic_marking
2275 )
2276 {
2277     GstVaapiDecoderH264Private * const priv = decoder->priv;
2278     gint32 i, picNumX;
2279
2280     picNumX = get_picNumX(picture, ref_pic_marking);
2281     i = find_short_term_reference(decoder, picNumX);
2282     if (i < 0)
2283         return;
2284
2285     gst_vaapi_picture_h264_set_reference(priv->short_ref[i], 0,
2286         GST_VAAPI_PICTURE_IS_FRAME(picture));
2287     ARRAY_REMOVE_INDEX(priv->short_ref, i);
2288 }
2289
2290 /* 8.2.5.4.2. Mark long-term reference picture as "unused for reference" */
2291 static void
2292 exec_ref_pic_marking_adaptive_mmco_2(
2293     GstVaapiDecoderH264  *decoder,
2294     GstVaapiPictureH264  *picture,
2295     GstH264RefPicMarking *ref_pic_marking
2296 )
2297 {
2298     GstVaapiDecoderH264Private * const priv = decoder->priv;
2299     gint32 i;
2300
2301     i = find_long_term_reference(decoder, ref_pic_marking->long_term_pic_num);
2302     if (i < 0)
2303         return;
2304
2305     gst_vaapi_picture_h264_set_reference(priv->long_ref[i], 0,
2306         GST_VAAPI_PICTURE_IS_FRAME(picture));
2307     ARRAY_REMOVE_INDEX(priv->long_ref, i);
2308 }
2309
2310 /* 8.2.5.4.3. Assign LongTermFrameIdx to a short-term reference picture */
2311 static void
2312 exec_ref_pic_marking_adaptive_mmco_3(
2313     GstVaapiDecoderH264  *decoder,
2314     GstVaapiPictureH264  *picture,
2315     GstH264RefPicMarking *ref_pic_marking
2316 )
2317 {
2318     GstVaapiDecoderH264Private * const priv = decoder->priv;
2319     GstVaapiPictureH264 *ref_picture;
2320     gint32 i, picNumX;
2321
2322     for (i = 0; i < priv->long_ref_count; i++) {
2323         if (priv->long_ref[i]->long_term_frame_idx == ref_pic_marking->long_term_frame_idx)
2324             break;
2325     }
2326     if (i != priv->long_ref_count) {
2327         gst_vaapi_picture_h264_set_reference(priv->long_ref[i], 0, TRUE);
2328         ARRAY_REMOVE_INDEX(priv->long_ref, i);
2329     }
2330
2331     picNumX = get_picNumX(picture, ref_pic_marking);
2332     i = find_short_term_reference(decoder, picNumX);
2333     if (i < 0)
2334         return;
2335
2336     ref_picture = priv->short_ref[i];
2337     ARRAY_REMOVE_INDEX(priv->short_ref, i);
2338     priv->long_ref[priv->long_ref_count++] = ref_picture;
2339
2340     ref_picture->long_term_frame_idx = ref_pic_marking->long_term_frame_idx;
2341     gst_vaapi_picture_h264_set_reference(ref_picture,
2342         GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE,
2343         GST_VAAPI_PICTURE_IS_FRAME(picture));
2344 }
2345
2346 /* 8.2.5.4.4. Mark pictures with LongTermFramIdx > max_long_term_frame_idx
2347  * as "unused for reference" */
2348 static void
2349 exec_ref_pic_marking_adaptive_mmco_4(
2350     GstVaapiDecoderH264  *decoder,
2351     GstVaapiPictureH264  *picture,
2352     GstH264RefPicMarking *ref_pic_marking
2353 )
2354 {
2355     GstVaapiDecoderH264Private * const priv = decoder->priv;
2356     gint32 i, long_term_frame_idx;
2357
2358     long_term_frame_idx = ref_pic_marking->max_long_term_frame_idx_plus1 - 1;
2359
2360     for (i = 0; i < priv->long_ref_count; i++) {
2361         if (priv->long_ref[i]->long_term_frame_idx <= long_term_frame_idx)
2362             continue;
2363         gst_vaapi_picture_h264_set_reference(priv->long_ref[i], 0, FALSE);
2364         ARRAY_REMOVE_INDEX(priv->long_ref, i);
2365         i--;
2366     }
2367 }
2368
2369 /* 8.2.5.4.5. Mark all reference pictures as "unused for reference" */
2370 static void
2371 exec_ref_pic_marking_adaptive_mmco_5(
2372     GstVaapiDecoderH264  *decoder,
2373     GstVaapiPictureH264  *picture,
2374     GstH264RefPicMarking *ref_pic_marking
2375 )
2376 {
2377     GstVaapiDecoderH264Private * const priv = decoder->priv;
2378
2379     dpb_flush(decoder);
2380
2381     priv->prev_pic_has_mmco5 = TRUE;
2382
2383     /* The picture shall be inferred to have had frame_num equal to 0 (7.4.3) */
2384     priv->frame_num = 0;
2385     priv->frame_num_offset = 0;
2386     picture->frame_num = 0;
2387
2388     /* Update TopFieldOrderCnt and BottomFieldOrderCnt (8.2.1) */
2389     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
2390         picture->field_poc[TOP_FIELD] -= picture->base.poc;
2391     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
2392         picture->field_poc[BOTTOM_FIELD] -= picture->base.poc;
2393     picture->base.poc = 0;
2394 }
2395
2396 /* 8.2.5.4.6. Assign a long-term frame index to the current picture */
2397 static void
2398 exec_ref_pic_marking_adaptive_mmco_6(
2399     GstVaapiDecoderH264  *decoder,
2400     GstVaapiPictureH264  *picture,
2401     GstH264RefPicMarking *ref_pic_marking
2402 )
2403 {
2404     picture->long_term_frame_idx = ref_pic_marking->long_term_frame_idx;
2405     gst_vaapi_picture_h264_set_reference(picture,
2406         GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE, FALSE);
2407 }
2408
2409 /* 8.2.5.4. Adaptive memory control decoded reference picture marking process */
2410 static gboolean
2411 exec_ref_pic_marking_adaptive(
2412     GstVaapiDecoderH264     *decoder,
2413     GstVaapiPictureH264     *picture,
2414     GstH264DecRefPicMarking *dec_ref_pic_marking
2415 )
2416 {
2417     guint i;
2418
2419     GST_DEBUG("reference picture marking process (adaptive memory control)");
2420
2421     typedef void (*exec_ref_pic_marking_adaptive_mmco_func)(
2422         GstVaapiDecoderH264  *decoder,
2423         GstVaapiPictureH264  *picture,
2424         GstH264RefPicMarking *ref_pic_marking
2425     );
2426
2427     static const exec_ref_pic_marking_adaptive_mmco_func mmco_funcs[] = {
2428         NULL,
2429         exec_ref_pic_marking_adaptive_mmco_1,
2430         exec_ref_pic_marking_adaptive_mmco_2,
2431         exec_ref_pic_marking_adaptive_mmco_3,
2432         exec_ref_pic_marking_adaptive_mmco_4,
2433         exec_ref_pic_marking_adaptive_mmco_5,
2434         exec_ref_pic_marking_adaptive_mmco_6,
2435     };
2436
2437     for (i = 0; i < dec_ref_pic_marking->n_ref_pic_marking; i++) {
2438         GstH264RefPicMarking * const ref_pic_marking =
2439             &dec_ref_pic_marking->ref_pic_marking[i];
2440
2441         const guint mmco = ref_pic_marking->memory_management_control_operation;
2442         if (mmco < G_N_ELEMENTS(mmco_funcs) && mmco_funcs[mmco])
2443             mmco_funcs[mmco](decoder, picture, ref_pic_marking);
2444         else {
2445             GST_ERROR("unhandled MMCO %u", mmco);
2446             return FALSE;
2447         }
2448     }
2449     return TRUE;
2450 }
2451
2452 /* 8.2.5 - Execute reference picture marking process */
2453 static gboolean
2454 exec_ref_pic_marking(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
2455 {
2456     GstVaapiDecoderH264Private * const priv = decoder->priv;
2457
2458     priv->prev_pic_has_mmco5 = FALSE;
2459     priv->prev_pic_structure = picture->structure;
2460
2461     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
2462         return TRUE;
2463
2464     if (!GST_VAAPI_PICTURE_IS_IDR(picture)) {
2465         GstVaapiSliceH264 * const slice =
2466             gst_vaapi_picture_h264_get_last_slice(picture);
2467         GstH264DecRefPicMarking * const dec_ref_pic_marking =
2468             &slice->slice_hdr.dec_ref_pic_marking;
2469         if (dec_ref_pic_marking->adaptive_ref_pic_marking_mode_flag) {
2470             if (!exec_ref_pic_marking_adaptive(decoder, picture, dec_ref_pic_marking))
2471                 return FALSE;
2472         }
2473         else {
2474             if (!exec_ref_pic_marking_sliding_window(decoder))
2475                 return FALSE;
2476         }
2477     }
2478     return TRUE;
2479 }
2480
2481 static void
2482 vaapi_init_picture(VAPictureH264 *pic)
2483 {
2484     pic->picture_id           = VA_INVALID_ID;
2485     pic->frame_idx            = 0;
2486     pic->flags                = VA_PICTURE_H264_INVALID;
2487     pic->TopFieldOrderCnt     = 0;
2488     pic->BottomFieldOrderCnt  = 0;
2489 }
2490
2491 static void
2492 vaapi_fill_picture(VAPictureH264 *pic, GstVaapiPictureH264 *picture,
2493     guint picture_structure)
2494 {
2495     if (!picture_structure)
2496         picture_structure = picture->structure;
2497
2498     pic->picture_id = picture->base.surface_id;
2499     pic->flags = 0;
2500
2501     if (GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture)) {
2502         pic->flags |= VA_PICTURE_H264_LONG_TERM_REFERENCE;
2503         pic->frame_idx = picture->long_term_frame_idx;
2504     }
2505     else {
2506         if (GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture))
2507             pic->flags |= VA_PICTURE_H264_SHORT_TERM_REFERENCE;
2508         pic->frame_idx = picture->frame_num;
2509     }
2510
2511     switch (picture_structure) {
2512     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
2513         pic->TopFieldOrderCnt = picture->field_poc[TOP_FIELD];
2514         pic->BottomFieldOrderCnt = picture->field_poc[BOTTOM_FIELD];
2515         break;
2516     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
2517         pic->flags |= VA_PICTURE_H264_TOP_FIELD;
2518         pic->TopFieldOrderCnt = picture->field_poc[TOP_FIELD];
2519         pic->BottomFieldOrderCnt = 0;
2520         break;
2521     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
2522         pic->flags |= VA_PICTURE_H264_BOTTOM_FIELD;
2523         pic->BottomFieldOrderCnt = picture->field_poc[BOTTOM_FIELD];
2524         pic->TopFieldOrderCnt = 0;
2525         break;
2526     }
2527 }
2528
2529 static gboolean
2530 fill_picture(
2531     GstVaapiDecoderH264 *decoder,
2532     GstVaapiPictureH264 *picture,
2533     GstH264SliceHdr     *slice_hdr,
2534     GstH264NalUnit      *nalu
2535 )
2536 {
2537     GstVaapiDecoderH264Private * const priv = decoder->priv;
2538     GstVaapiPicture * const base_picture = &picture->base;
2539     GstH264PPS * const pps = picture->pps;
2540     GstH264SPS * const sps = pps->sequence;
2541     VAPictureParameterBufferH264 * const pic_param = base_picture->param;
2542     guint i, n;
2543
2544     /* Fill in VAPictureParameterBufferH264 */
2545     vaapi_fill_picture(&pic_param->CurrPic, picture, 0);
2546
2547     for (i = 0, n = 0; i < priv->dpb_count; i++) {
2548         GstVaapiFrameStore * const fs = priv->dpb[i];
2549         if (gst_vaapi_frame_store_has_reference(fs))
2550             vaapi_fill_picture(&pic_param->ReferenceFrames[n++],
2551                 fs->buffers[0], fs->structure);
2552     }
2553     for (; n < G_N_ELEMENTS(pic_param->ReferenceFrames); n++)
2554         vaapi_init_picture(&pic_param->ReferenceFrames[n]);
2555
2556 #define COPY_FIELD(s, f) \
2557     pic_param->f = (s)->f
2558
2559 #define COPY_BFM(a, s, f) \
2560     pic_param->a.bits.f = (s)->f
2561
2562     pic_param->picture_width_in_mbs_minus1  = priv->mb_width - 1;
2563     pic_param->picture_height_in_mbs_minus1 = priv->mb_height - 1;
2564     pic_param->frame_num                    = priv->frame_num;
2565
2566     COPY_FIELD(sps, bit_depth_luma_minus8);
2567     COPY_FIELD(sps, bit_depth_chroma_minus8);
2568     COPY_FIELD(sps, num_ref_frames);
2569     COPY_FIELD(pps, num_slice_groups_minus1);
2570     COPY_FIELD(pps, slice_group_map_type);
2571     COPY_FIELD(pps, slice_group_change_rate_minus1);
2572     COPY_FIELD(pps, pic_init_qp_minus26);
2573     COPY_FIELD(pps, pic_init_qs_minus26);
2574     COPY_FIELD(pps, chroma_qp_index_offset);
2575     COPY_FIELD(pps, second_chroma_qp_index_offset);
2576
2577     pic_param->seq_fields.value                                         = 0; /* reset all bits */
2578     pic_param->seq_fields.bits.residual_colour_transform_flag           = sps->separate_colour_plane_flag;
2579     pic_param->seq_fields.bits.MinLumaBiPredSize8x8                     = sps->level_idc >= 31; /* A.3.3.2 */
2580
2581     COPY_BFM(seq_fields, sps, chroma_format_idc);
2582     COPY_BFM(seq_fields, sps, gaps_in_frame_num_value_allowed_flag);
2583     COPY_BFM(seq_fields, sps, frame_mbs_only_flag);
2584     COPY_BFM(seq_fields, sps, mb_adaptive_frame_field_flag);
2585     COPY_BFM(seq_fields, sps, direct_8x8_inference_flag);
2586     COPY_BFM(seq_fields, sps, log2_max_frame_num_minus4);
2587     COPY_BFM(seq_fields, sps, pic_order_cnt_type);
2588     COPY_BFM(seq_fields, sps, log2_max_pic_order_cnt_lsb_minus4);
2589     COPY_BFM(seq_fields, sps, delta_pic_order_always_zero_flag);
2590
2591     pic_param->pic_fields.value                                         = 0; /* reset all bits */
2592     pic_param->pic_fields.bits.field_pic_flag                           = slice_hdr->field_pic_flag;
2593     pic_param->pic_fields.bits.reference_pic_flag                       = GST_VAAPI_PICTURE_IS_REFERENCE(picture);
2594
2595     COPY_BFM(pic_fields, pps, entropy_coding_mode_flag);
2596     COPY_BFM(pic_fields, pps, weighted_pred_flag);
2597     COPY_BFM(pic_fields, pps, weighted_bipred_idc);
2598     COPY_BFM(pic_fields, pps, transform_8x8_mode_flag);
2599     COPY_BFM(pic_fields, pps, constrained_intra_pred_flag);
2600     COPY_BFM(pic_fields, pps, pic_order_present_flag);
2601     COPY_BFM(pic_fields, pps, deblocking_filter_control_present_flag);
2602     COPY_BFM(pic_fields, pps, redundant_pic_cnt_present_flag);
2603     return TRUE;
2604 }
2605
2606 /* Detection of the first VCL NAL unit of a primary coded picture (7.4.1.2.4) */
2607 static gboolean
2608 is_new_picture(
2609     GstVaapiDecoderH264 *decoder,
2610     GstH264NalUnit      *nalu,
2611     GstH264SliceHdr     *slice_hdr
2612 )
2613 {
2614     GstVaapiDecoderH264Private * const priv = decoder->priv;
2615     GstH264PPS * const pps = slice_hdr->pps;
2616     GstH264SPS * const sps = pps->sequence;
2617     GstVaapiSliceH264 *slice;
2618     GstH264SliceHdr *prev_slice_hdr;
2619
2620     if (!priv->current_picture)
2621         return TRUE;
2622
2623     slice = gst_vaapi_picture_h264_get_last_slice(priv->current_picture);
2624     if (!slice)
2625         return FALSE;
2626     prev_slice_hdr = &slice->slice_hdr;
2627
2628 #define CHECK_EXPR(expr, field_name) do {              \
2629         if (!(expr)) {                                 \
2630             GST_DEBUG(field_name " differs in value"); \
2631             return TRUE;                               \
2632         }                                              \
2633     } while (0)
2634
2635 #define CHECK_VALUE(new_slice_hdr, old_slice_hdr, field) \
2636     CHECK_EXPR(((new_slice_hdr)->field == (old_slice_hdr)->field), #field)
2637
2638     /* frame_num differs in value, regardless of inferred values to 0 */
2639     CHECK_VALUE(slice_hdr, prev_slice_hdr, frame_num);
2640
2641     /* pic_parameter_set_id differs in value */
2642     CHECK_VALUE(slice_hdr, prev_slice_hdr, pps);
2643
2644     /* field_pic_flag differs in value */
2645     CHECK_VALUE(slice_hdr, prev_slice_hdr, field_pic_flag);
2646
2647     /* bottom_field_flag is present in both and differs in value */
2648     if (slice_hdr->field_pic_flag && prev_slice_hdr->field_pic_flag)
2649         CHECK_VALUE(slice_hdr, prev_slice_hdr, bottom_field_flag);
2650
2651     /* nal_ref_idc differs in value with one of the nal_ref_idc values is 0 */
2652     CHECK_EXPR(((GST_VAAPI_PICTURE_IS_REFERENCE(priv->current_picture) ^
2653                  (nalu->ref_idc != 0)) == 0), "nal_ref_idc");
2654
2655     /* POC type is 0 for both and either pic_order_cnt_lsb differs in
2656        value or delta_pic_order_cnt_bottom differs in value */
2657     if (sps->pic_order_cnt_type == 0) {
2658         CHECK_VALUE(slice_hdr, prev_slice_hdr, pic_order_cnt_lsb);
2659         if (pps->pic_order_present_flag && !slice_hdr->field_pic_flag)
2660             CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt_bottom);
2661     }
2662
2663     /* POC type is 1 for both and either delta_pic_order_cnt[0]
2664        differs in value or delta_pic_order_cnt[1] differs in value */
2665     else if (sps->pic_order_cnt_type == 1) {
2666         CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt[0]);
2667         CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt[1]);
2668     }
2669
2670     /* IdrPicFlag differs in value */
2671     CHECK_EXPR(((GST_VAAPI_PICTURE_IS_IDR(priv->current_picture) ^
2672                  (nalu->type == GST_H264_NAL_SLICE_IDR)) == 0), "IdrPicFlag");
2673
2674     /* IdrPicFlag is equal to 1 for both and idr_pic_id differs in value */
2675     if (GST_VAAPI_PICTURE_IS_IDR(priv->current_picture))
2676         CHECK_VALUE(slice_hdr, prev_slice_hdr, idr_pic_id);
2677
2678 #undef CHECK_EXPR
2679 #undef CHECK_VALUE
2680     return FALSE;
2681 }
2682
2683 static GstVaapiDecoderStatus
2684 decode_picture(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu, GstH264SliceHdr *slice_hdr)
2685 {
2686     GstVaapiDecoderH264Private * const priv = decoder->priv;
2687     GstVaapiPictureH264 *picture;
2688     GstVaapiDecoderStatus status;
2689     GstH264PPS * const pps = slice_hdr->pps;
2690     GstH264SPS * const sps = pps->sequence;
2691
2692     status = decode_current_picture(decoder);
2693     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2694         return status;
2695
2696     status = ensure_context(decoder, sps);
2697     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2698         return status;
2699
2700     if (priv->current_picture) {
2701         /* Re-use current picture where the first field was decoded */
2702         picture = gst_vaapi_picture_h264_new_field(priv->current_picture);
2703         if (!picture) {
2704             GST_ERROR("failed to allocate field picture");
2705             return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2706         }
2707     }
2708     else {
2709         /* Create new picture */
2710         picture = gst_vaapi_picture_h264_new(decoder);
2711         if (!picture) {
2712             GST_ERROR("failed to allocate picture");
2713             return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2714         }
2715     }
2716     gst_vaapi_picture_replace(&priv->current_picture, picture);
2717     gst_vaapi_picture_unref(picture);
2718
2719     picture->pps = pps;
2720
2721     status = ensure_quant_matrix(decoder, picture);
2722     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS) {
2723         GST_ERROR("failed to reset quantizer matrix");
2724         return status;
2725     }
2726
2727     if (!init_picture(decoder, picture, slice_hdr, nalu))
2728         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2729     if (!fill_picture(decoder, picture, slice_hdr, nalu))
2730         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2731     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2732 }
2733
2734 static inline guint
2735 get_slice_data_bit_offset(GstH264SliceHdr *slice_hdr, GstH264NalUnit *nalu)
2736 {
2737     guint epb_count;
2738
2739     epb_count = slice_hdr->n_emulation_prevention_bytes;
2740     return 8 /* nal_unit_type */ + slice_hdr->header_size - epb_count * 8;
2741 }
2742
2743 static gboolean
2744 fill_pred_weight_table(GstVaapiDecoderH264 *decoder, GstVaapiSliceH264 *slice)
2745 {
2746     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2747     GstH264PPS * const pps = slice_hdr->pps;
2748     GstH264SPS * const sps = pps->sequence;
2749     GstH264PredWeightTable * const w = &slice_hdr->pred_weight_table;
2750     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2751     guint num_weight_tables = 0;
2752     gint i, j;
2753
2754     if (pps->weighted_pred_flag &&
2755         (GST_H264_IS_P_SLICE(slice_hdr) || GST_H264_IS_SP_SLICE(slice_hdr)))
2756         num_weight_tables = 1;
2757     else if (pps->weighted_bipred_idc == 1 && GST_H264_IS_B_SLICE(slice_hdr))
2758         num_weight_tables = 2;
2759     else
2760         num_weight_tables = 0;
2761
2762     slice_param->luma_log2_weight_denom   = w->luma_log2_weight_denom;
2763     slice_param->chroma_log2_weight_denom = w->chroma_log2_weight_denom;
2764     slice_param->luma_weight_l0_flag      = 0;
2765     slice_param->chroma_weight_l0_flag    = 0;
2766     slice_param->luma_weight_l1_flag      = 0;
2767     slice_param->chroma_weight_l1_flag    = 0;
2768
2769     if (num_weight_tables < 1)
2770         return TRUE;
2771
2772     slice_param->luma_weight_l0_flag = 1;
2773     for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2774         slice_param->luma_weight_l0[i] = w->luma_weight_l0[i];
2775         slice_param->luma_offset_l0[i] = w->luma_offset_l0[i];
2776     }
2777
2778     slice_param->chroma_weight_l0_flag = sps->chroma_array_type != 0;
2779     if (slice_param->chroma_weight_l0_flag) {
2780         for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2781             for (j = 0; j < 2; j++) {
2782                 slice_param->chroma_weight_l0[i][j] = w->chroma_weight_l0[i][j];
2783                 slice_param->chroma_offset_l0[i][j] = w->chroma_offset_l0[i][j];
2784             }
2785         }
2786     }
2787
2788     if (num_weight_tables < 2)
2789         return TRUE;
2790
2791     slice_param->luma_weight_l1_flag = 1;
2792     for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2793         slice_param->luma_weight_l1[i] = w->luma_weight_l1[i];
2794         slice_param->luma_offset_l1[i] = w->luma_offset_l1[i];
2795     }
2796
2797     slice_param->chroma_weight_l1_flag = sps->chroma_array_type != 0;
2798     if (slice_param->chroma_weight_l1_flag) {
2799         for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2800             for (j = 0; j < 2; j++) {
2801                 slice_param->chroma_weight_l1[i][j] = w->chroma_weight_l1[i][j];
2802                 slice_param->chroma_offset_l1[i][j] = w->chroma_offset_l1[i][j];
2803             }
2804         }
2805     }
2806     return TRUE;
2807 }
2808
2809 static gboolean
2810 fill_RefPicList(GstVaapiDecoderH264 *decoder, GstVaapiSliceH264 *slice)
2811 {
2812     GstVaapiDecoderH264Private * const priv = decoder->priv;
2813     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2814     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2815     guint i, num_ref_lists = 0;
2816
2817     slice_param->num_ref_idx_l0_active_minus1 = 0;
2818     slice_param->num_ref_idx_l1_active_minus1 = 0;
2819
2820     if (GST_H264_IS_B_SLICE(slice_hdr))
2821         num_ref_lists = 2;
2822     else if (GST_H264_IS_I_SLICE(slice_hdr))
2823         num_ref_lists = 0;
2824     else
2825         num_ref_lists = 1;
2826
2827     if (num_ref_lists < 1)
2828         return TRUE;
2829
2830     slice_param->num_ref_idx_l0_active_minus1 =
2831         slice_hdr->num_ref_idx_l0_active_minus1;
2832
2833     for (i = 0; i < priv->RefPicList0_count && priv->RefPicList0[i]; i++)
2834         vaapi_fill_picture(&slice_param->RefPicList0[i], priv->RefPicList0[i], 0);
2835     for (; i <= slice_param->num_ref_idx_l0_active_minus1; i++)
2836         vaapi_init_picture(&slice_param->RefPicList0[i]);
2837
2838     if (num_ref_lists < 2)
2839         return TRUE;
2840
2841     slice_param->num_ref_idx_l1_active_minus1 =
2842         slice_hdr->num_ref_idx_l1_active_minus1;
2843
2844     for (i = 0; i < priv->RefPicList1_count && priv->RefPicList1[i]; i++)
2845         vaapi_fill_picture(&slice_param->RefPicList1[i], priv->RefPicList1[i], 0);
2846     for (; i <= slice_param->num_ref_idx_l1_active_minus1; i++)
2847         vaapi_init_picture(&slice_param->RefPicList1[i]);
2848     return TRUE;
2849 }
2850
2851 static gboolean
2852 fill_slice(
2853     GstVaapiDecoderH264 *decoder,
2854     GstVaapiSliceH264   *slice,
2855     GstH264NalUnit      *nalu
2856 )
2857 {
2858     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2859     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2860
2861     /* Fill in VASliceParameterBufferH264 */
2862     slice_param->slice_data_bit_offset          = get_slice_data_bit_offset(slice_hdr, nalu);
2863     slice_param->first_mb_in_slice              = slice_hdr->first_mb_in_slice;
2864     slice_param->slice_type                     = slice_hdr->type % 5;
2865     slice_param->direct_spatial_mv_pred_flag    = slice_hdr->direct_spatial_mv_pred_flag;
2866     slice_param->cabac_init_idc                 = slice_hdr->cabac_init_idc;
2867     slice_param->slice_qp_delta                 = slice_hdr->slice_qp_delta;
2868     slice_param->disable_deblocking_filter_idc  = slice_hdr->disable_deblocking_filter_idc;
2869     slice_param->slice_alpha_c0_offset_div2     = slice_hdr->slice_alpha_c0_offset_div2;
2870     slice_param->slice_beta_offset_div2         = slice_hdr->slice_beta_offset_div2;
2871
2872     if (!fill_RefPicList(decoder, slice))
2873         return FALSE;
2874     if (!fill_pred_weight_table(decoder, slice))
2875         return FALSE;
2876     return TRUE;
2877 }
2878
2879 static GstVaapiDecoderStatus
2880 decode_slice(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
2881 {
2882     GstVaapiDecoderH264Private * const priv = decoder->priv;
2883     GstVaapiDecoderStatus status;
2884     GstVaapiPictureH264 *picture;
2885     GstVaapiSliceH264 *slice = NULL;
2886     GstH264SliceHdr *slice_hdr;
2887     GstH264ParserResult result;
2888     gboolean is_first_slice = !priv->has_context;
2889
2890     GST_DEBUG("slice (%u bytes)", nalu->size);
2891
2892     if (is_first_slice) {
2893         slice_hdr = &priv->temp_slice_hdr;
2894         memset(slice_hdr, 0, sizeof(*slice_hdr));
2895         result = gst_h264_parser_parse_slice_hdr(priv->parser, nalu,
2896             slice_hdr, TRUE, TRUE);
2897         if (result != GST_H264_PARSER_OK) {
2898             status = get_status(result);
2899             goto error;
2900         }
2901
2902         status = ensure_context(decoder, slice_hdr->pps->sequence);
2903         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2904             return status;
2905     }
2906
2907     slice = gst_vaapi_slice_h264_new(
2908         decoder,
2909         nalu->data + nalu->offset,
2910         nalu->size
2911     );
2912     if (!slice) {
2913         GST_ERROR("failed to allocate slice");
2914         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2915     }
2916
2917     slice_hdr = &slice->slice_hdr;
2918     if (is_first_slice)
2919         memcpy(slice_hdr, &priv->temp_slice_hdr, sizeof(*slice_hdr));
2920     else {
2921         memset(slice_hdr, 0, sizeof(*slice_hdr));
2922         result = gst_h264_parser_parse_slice_hdr(priv->parser, nalu,
2923             slice_hdr, TRUE, TRUE);
2924         if (result != GST_H264_PARSER_OK) {
2925             status = get_status(result);
2926             goto error;
2927         }
2928     }
2929
2930     if (is_new_picture(decoder, nalu, slice_hdr)) {
2931         status = decode_picture(decoder, nalu, slice_hdr);
2932         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2933             goto error;
2934     }
2935     picture = priv->current_picture;
2936
2937     if (!fill_slice(decoder, slice, nalu)) {
2938         status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2939         goto error;
2940     }
2941     gst_vaapi_picture_add_slice(
2942         GST_VAAPI_PICTURE_CAST(picture),
2943         GST_VAAPI_SLICE_CAST(slice)
2944     );
2945     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2946
2947 error:
2948     if (slice)
2949         gst_mini_object_unref(GST_MINI_OBJECT(slice));
2950     return status;
2951 }
2952
2953 static inline gint
2954 scan_for_start_code(GstAdapter *adapter, guint ofs, guint size, guint32 *scp)
2955 {
2956     return (gint)gst_adapter_masked_scan_uint32_peek(adapter,
2957                                                      0xffffff00, 0x00000100,
2958                                                      ofs, size,
2959                                                      scp);
2960 }
2961
2962 static GstVaapiDecoderStatus
2963 decode_nalu(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
2964 {
2965     GstVaapiDecoderH264Private * const priv = decoder->priv;
2966     GstVaapiDecoderStatus status;
2967
2968     switch (nalu->type) {
2969     case GST_H264_NAL_SLICE_IDR:
2970         /* fall-through. IDR specifics are handled in init_picture() */
2971     case GST_H264_NAL_SLICE:
2972         if (!priv->got_sps || !priv->got_pps)
2973             return GST_VAAPI_DECODER_STATUS_SUCCESS;
2974         status = decode_slice(decoder, nalu);
2975         break;
2976     case GST_H264_NAL_SPS:
2977         status = decode_sps(decoder, nalu);
2978         break;
2979     case GST_H264_NAL_PPS:
2980         status = decode_pps(decoder, nalu);
2981         break;
2982     case GST_H264_NAL_SEI:
2983         status = decode_sei(decoder, nalu);
2984         break;
2985     case GST_H264_NAL_SEQ_END:
2986         status = decode_sequence_end(decoder);
2987         break;
2988     case GST_H264_NAL_AU_DELIMITER:
2989         /* skip all Access Unit NALs */
2990         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2991         break;
2992     case GST_H264_NAL_FILLER_DATA:
2993         /* skip all Filler Data NALs */
2994         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2995         break;
2996     default:
2997         GST_WARNING("unsupported NAL unit type %d", nalu->type);
2998         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
2999         break;
3000     }
3001     return status;
3002 }
3003
3004 static GstVaapiDecoderStatus
3005 decode_buffer(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
3006 {
3007     GstVaapiDecoderH264Private * const priv = decoder->priv;
3008     GstVaapiDecoderStatus status;
3009     GstH264ParserResult result;
3010     GstH264NalUnit nalu;
3011     gboolean is_eos;
3012     const guchar *buf;
3013     guint i, buf_size, nalu_size, size;
3014     guint32 start_code;
3015     gint ofs;
3016
3017     buf      = GST_BUFFER_DATA(buffer);
3018     buf_size = GST_BUFFER_SIZE(buffer);
3019     is_eos   = GST_BUFFER_IS_EOS(buffer);
3020     if (buf && buf_size > 0)
3021         gst_adapter_push(priv->adapter, gst_buffer_ref(buffer));
3022
3023     size = gst_adapter_available(priv->adapter);
3024     do {
3025         if (size == 0) {
3026             status = GST_VAAPI_DECODER_STATUS_SUCCESS;
3027             break;
3028         }
3029
3030         status = gst_vaapi_decoder_check_status(GST_VAAPI_DECODER(decoder));
3031         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
3032             break;
3033
3034         status = GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
3035         if (priv->is_avc) {
3036             if (size < priv->nal_length_size)
3037                 break;
3038             buf = gst_adapter_peek(priv->adapter, priv->nal_length_size);
3039
3040             nalu_size = 0;
3041             for (i = 0; i < priv->nal_length_size; i++)
3042                 nalu_size = (nalu_size << 8) | buf[i];
3043
3044             buf_size = priv->nal_length_size + nalu_size;
3045             if (size < buf_size)
3046                 break;
3047             buffer = gst_adapter_take_buffer(priv->adapter, buf_size);
3048             size -= buf_size;
3049
3050             buf      = GST_BUFFER_DATA(buffer);
3051             buf_size = GST_BUFFER_SIZE(buffer);
3052
3053             result = gst_h264_parser_identify_nalu_avc(
3054                 priv->parser,
3055                 buf, 0, buf_size, priv->nal_length_size,
3056                 &nalu
3057             );
3058         }
3059         else {
3060             if (size < 4)
3061                 break;
3062             ofs = scan_for_start_code(priv->adapter, 0, size, &start_code);
3063             if (ofs < 0)
3064                 break;
3065             gst_adapter_flush(priv->adapter, ofs);
3066             size -= ofs;
3067
3068             ofs = G_UNLIKELY(size < 8) ? -1 :
3069                 scan_for_start_code(priv->adapter, 4, size - 4, NULL);
3070             if (ofs < 0) {
3071                 // Assume the whole NAL unit is present if end-of-stream
3072                 if (!is_eos)
3073                     break;
3074                 ofs = size;
3075             }
3076             buffer = gst_adapter_take_buffer(priv->adapter, ofs);
3077             size -= ofs;
3078
3079             buf      = GST_BUFFER_DATA(buffer);
3080             buf_size = GST_BUFFER_SIZE(buffer);
3081
3082             result = gst_h264_parser_identify_nalu_unchecked(
3083                 priv->parser,
3084                 buf, 0, buf_size,
3085                 &nalu
3086             );
3087         }
3088         status = get_status(result);
3089         if (status == GST_VAAPI_DECODER_STATUS_SUCCESS)
3090             status = decode_nalu(decoder, &nalu);
3091         gst_buffer_unref(buffer);
3092     } while (status == GST_VAAPI_DECODER_STATUS_SUCCESS);
3093
3094     if (is_eos && (status == GST_VAAPI_DECODER_STATUS_SUCCESS ||
3095                    status == GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA))
3096         status = decode_sequence_end(decoder);
3097     return status;
3098 }
3099
3100 static GstVaapiDecoderStatus
3101 decode_codec_data(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
3102 {
3103     GstVaapiDecoderH264Private * const priv = decoder->priv;
3104     GstVaapiDecoderStatus status;
3105     GstH264NalUnit nalu;
3106     GstH264ParserResult result;
3107     guchar *buf;
3108     guint buf_size;
3109     guint i, ofs, num_sps, num_pps;
3110
3111     buf      = GST_BUFFER_DATA(buffer);
3112     buf_size = GST_BUFFER_SIZE(buffer);
3113     if (!buf || buf_size == 0)
3114         return GST_VAAPI_DECODER_STATUS_SUCCESS;
3115
3116     if (buf_size < 8)
3117         return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
3118
3119     if (buf[0] != 1) {
3120         GST_ERROR("failed to decode codec-data, not in avcC format");
3121         return GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
3122     }
3123
3124     priv->nal_length_size = (buf[4] & 0x03) + 1;
3125
3126     num_sps = buf[5] & 0x1f;
3127     ofs = 6;
3128
3129     for (i = 0; i < num_sps; i++) {
3130         result = gst_h264_parser_identify_nalu_avc(
3131             priv->parser,
3132             buf, ofs, buf_size, 2,
3133             &nalu
3134         );
3135         if (result != GST_H264_PARSER_OK)
3136             return get_status(result);
3137
3138         status = decode_sps(decoder, &nalu);
3139         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
3140             return status;
3141         ofs = nalu.offset + nalu.size;
3142     }
3143
3144     num_pps = buf[ofs];
3145     ofs++;
3146
3147     for (i = 0; i < num_pps; i++) {
3148         result = gst_h264_parser_identify_nalu_avc(
3149             priv->parser,
3150             buf, ofs, buf_size, 2,
3151             &nalu
3152         );
3153         if (result != GST_H264_PARSER_OK)
3154             return get_status(result);
3155
3156         status = decode_pps(decoder, &nalu);
3157         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
3158             return status;
3159         ofs = nalu.offset + nalu.size;
3160     }
3161
3162     priv->is_avc = TRUE;
3163     return status;
3164 }
3165
3166 GstVaapiDecoderStatus
3167 gst_vaapi_decoder_h264_decode(GstVaapiDecoder *base, GstBuffer *buffer)
3168 {
3169     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264(base);
3170     GstVaapiDecoderH264Private * const priv = decoder->priv;
3171     GstVaapiDecoderStatus status;
3172     GstBuffer *codec_data;
3173
3174     g_return_val_if_fail(priv->is_constructed,
3175                          GST_VAAPI_DECODER_STATUS_ERROR_INIT_FAILED);
3176
3177     if (!priv->is_opened) {
3178         priv->is_opened = gst_vaapi_decoder_h264_open(decoder, buffer);
3179         if (!priv->is_opened)
3180             return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CODEC;
3181
3182         codec_data = GST_VAAPI_DECODER_CODEC_DATA(decoder);
3183         if (codec_data) {
3184             status = decode_codec_data(decoder, codec_data);
3185             if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
3186                 return status;
3187         }
3188      }
3189      return decode_buffer(decoder, buffer);
3190 }
3191
3192 static void
3193 gst_vaapi_decoder_h264_finalize(GObject *object)
3194 {
3195     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264(object);
3196
3197     gst_vaapi_decoder_h264_destroy(decoder);
3198
3199     G_OBJECT_CLASS(gst_vaapi_decoder_h264_parent_class)->finalize(object);
3200 }
3201
3202 static void
3203 gst_vaapi_decoder_h264_constructed(GObject *object)
3204 {
3205     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264(object);
3206     GstVaapiDecoderH264Private * const priv = decoder->priv;
3207     GObjectClass *parent_class;
3208
3209     parent_class = G_OBJECT_CLASS(gst_vaapi_decoder_h264_parent_class);
3210     if (parent_class->constructed)
3211         parent_class->constructed(object);
3212
3213     priv->is_constructed = gst_vaapi_decoder_h264_create(decoder);
3214 }
3215
3216 static void
3217 gst_vaapi_decoder_h264_class_init(GstVaapiDecoderH264Class *klass)
3218 {
3219     GObjectClass * const object_class = G_OBJECT_CLASS(klass);
3220     GstVaapiDecoderClass * const decoder_class = GST_VAAPI_DECODER_CLASS(klass);
3221
3222     g_type_class_add_private(klass, sizeof(GstVaapiDecoderH264Private));
3223
3224     object_class->finalize      = gst_vaapi_decoder_h264_finalize;
3225     object_class->constructed   = gst_vaapi_decoder_h264_constructed;
3226
3227     decoder_class->decode       = gst_vaapi_decoder_h264_decode;
3228     decoder_class->clear_buffer = gst_vaapi_decoder_h264_clear_buffer;
3229 }
3230
3231 static void
3232 gst_vaapi_decoder_h264_init(GstVaapiDecoderH264 *decoder)
3233 {
3234     GstVaapiDecoderH264Private *priv;
3235
3236     priv                        = GST_VAAPI_DECODER_H264_GET_PRIVATE(decoder);
3237     decoder->priv               = priv;
3238     priv->parser                = NULL;
3239     priv->current_picture       = NULL;
3240     priv->dpb_count             = 0;
3241     priv->dpb_size              = 0;
3242     priv->profile               = GST_VAAPI_PROFILE_UNKNOWN;
3243     priv->entrypoint            = GST_VAAPI_ENTRYPOINT_VLD;
3244     priv->chroma_type           = GST_VAAPI_CHROMA_TYPE_YUV420;
3245     priv->short_ref_count       = 0;
3246     priv->long_ref_count        = 0;
3247     priv->RefPicList0_count     = 0;
3248     priv->RefPicList1_count     = 0;
3249     priv->nal_length_size       = 0;
3250     priv->adapter               = NULL;
3251     priv->field_poc[0]          = 0;
3252     priv->field_poc[1]          = 0;
3253     priv->poc_msb               = 0;
3254     priv->poc_lsb               = 0;
3255     priv->prev_poc_msb          = 0;
3256     priv->prev_poc_lsb          = 0;
3257     priv->frame_num_offset      = 0;
3258     priv->frame_num             = 0;
3259     priv->prev_frame_num        = 0;
3260     priv->prev_pic_has_mmco5    = FALSE;
3261     priv->prev_pic_structure    = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
3262     priv->is_constructed        = FALSE;
3263     priv->is_opened             = FALSE;
3264     priv->is_avc                = FALSE;
3265     priv->has_context           = FALSE;
3266     priv->progressive_sequence  = TRUE;
3267
3268     memset(priv->dpb, 0, sizeof(priv->dpb));
3269     memset(priv->short_ref, 0, sizeof(priv->short_ref));
3270     memset(priv->long_ref, 0, sizeof(priv->long_ref));
3271     memset(priv->RefPicList0, 0, sizeof(priv->RefPicList0));
3272     memset(priv->RefPicList1, 0, sizeof(priv->RefPicList1));
3273 }
3274
3275 /**
3276  * gst_vaapi_decoder_h264_new:
3277  * @display: a #GstVaapiDisplay
3278  * @caps: a #GstCaps holding codec information
3279  *
3280  * Creates a new #GstVaapiDecoder for MPEG-2 decoding.  The @caps can
3281  * hold extra information like codec-data and pictured coded size.
3282  *
3283  * Return value: the newly allocated #GstVaapiDecoder object
3284  */
3285 GstVaapiDecoder *
3286 gst_vaapi_decoder_h264_new(GstVaapiDisplay *display, GstCaps *caps)
3287 {
3288     GstVaapiDecoderH264 *decoder;
3289
3290     g_return_val_if_fail(GST_VAAPI_IS_DISPLAY(display), NULL);
3291     g_return_val_if_fail(GST_IS_CAPS(caps), NULL);
3292
3293     decoder = g_object_new(
3294         GST_VAAPI_TYPE_DECODER_H264,
3295         "display",      display,
3296         "caps",         caps,
3297         NULL
3298     );
3299     if (!decoder->priv->is_constructed) {
3300         g_object_unref(decoder);
3301         return NULL;
3302     }
3303     return GST_VAAPI_DECODER_CAST(decoder);
3304 }