gst-libs/gst/vaapi/gstvaapidecoder_h264.c

   1 /*
   2  *  gstvaapidecoder_h264.c - H.264 decoder
   3  *
   4  *  Copyright (C) 2011-2012 Intel Corporation
   5  *
   6  *  This library is free software; you can redistribute it and/or
   7  *  modify it under the terms of the GNU Lesser General Public License
   8  *  as published by the Free Software Foundation; either version 2.1
   9  *  of the License, or (at your option) any later version.
  10  *
  11  *  This library is distributed in the hope that it will be useful,
  12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  *  Lesser General Public License for more details.
  15  *
  16  *  You should have received a copy of the GNU Lesser General Public
  17  *  License along with this library; if not, write to the Free
  18  *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  19  *  Boston, MA 02110-1301 USA
  20  */
  21
  22 /**
  23  * SECTION:gstvaapidecoder_h264
  24  * @short_description: H.264 decoder
  25  */
  26
  27 #include "sysdeps.h"
  28 #include <string.h>
  29 #include <gst/base/gstadapter.h>
  30 #include <gst/codecparsers/gsth264parser.h>
  31 #include "gstvaapidecoder_h264.h"
  32 #include "gstvaapidecoder_objects.h"
  33 #include "gstvaapidecoder_priv.h"
  34 #include "gstvaapidisplay_priv.h"
  35 #include "gstvaapiobject_priv.h"
  36
  37 #define DEBUG 1
  38 #include "gstvaapidebug.h"
  39
  40 /* Defined to 1 if strict ordering of DPB is needed. Only useful for debug */
  41 #define USE_STRICT_DPB_ORDERING 0
  42
  43 typedef struct _GstVaapiPictureH264             GstVaapiPictureH264;
  44 typedef struct _GstVaapiPictureH264Class        GstVaapiPictureH264Class;
  45 typedef struct _GstVaapiSliceH264               GstVaapiSliceH264;
  46 typedef struct _GstVaapiSliceH264Class          GstVaapiSliceH264Class;
  47
  48 /* ------------------------------------------------------------------------- */
  49 /* --- H.264 Pictures                                                    --- */
  50 /* ------------------------------------------------------------------------- */
  51
  52 #define GST_VAAPI_TYPE_PICTURE_H264 \
  53     (gst_vaapi_picture_h264_get_type())
  54
  55 #define GST_VAAPI_PICTURE_H264_CAST(obj) \
  56     ((GstVaapiPictureH264 *)(obj))
  57
  58 #define GST_VAAPI_PICTURE_H264(obj)                             \
  59     (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
  60                                 GST_VAAPI_TYPE_PICTURE_H264,    \
  61                                 GstVaapiPictureH264))
  62
  63 #define GST_VAAPI_PICTURE_H264_CLASS(klass)                     \
  64     (G_TYPE_CHECK_CLASS_CAST((klass),                           \
  65                              GST_VAAPI_TYPE_PICTURE_H264,       \
  66                              GstVaapiPictureH264Class))
  67
  68 #define GST_VAAPI_IS_PICTURE_H264(obj) \
  69     (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_VAAPI_TYPE_PICTURE_H264))
  70
  71 #define GST_VAAPI_IS_PICTURE_H264_CLASS(klass) \
  72     (G_TYPE_CHECK_CLASS_TYPE((klass), GST_VAAPI_TYPE_PICTURE_H264))
  73
  74 #define GST_VAAPI_PICTURE_H264_GET_CLASS(obj)                   \
  75     (G_TYPE_INSTANCE_GET_CLASS((obj),                           \
  76                                GST_VAAPI_TYPE_PICTURE_H264,     \
  77                                GstVaapiPictureH264Class))
  78
  79 struct _GstVaapiPictureH264 {
  80     GstVaapiPicture             base;
  81     VAPictureH264               info;
  82     gint32                      poc;
  83     gint32                      frame_num;              // Original frame_num from slice_header()
  84     gint32                      frame_num_wrap;         // Temporary for ref pic marking: FrameNumWrap
  85     gint32                      pic_num;                // Temporary for ref pic marking: PicNum
  86     gint32                      long_term_pic_num;      // Temporary for ref pic marking: LongTermPicNum
  87     guint                       is_idr                  : 1;
  88     guint                       is_long_term            : 1;
  89     guint                       field_pic_flag          : 1;
  90     guint                       bottom_field_flag       : 1;
  91     guint                       output_flag             : 1;
  92     guint                       output_needed           : 1;
  93 };
  94
  95 struct _GstVaapiPictureH264Class {
  96     /*< private >*/
  97     GstVaapiPictureClass        parent_class;
  98 };
  99
 100 GST_VAAPI_CODEC_DEFINE_TYPE(GstVaapiPictureH264,
 101                             gst_vaapi_picture_h264,
 102                             GST_VAAPI_TYPE_PICTURE)
 103
 104 static void
 105 gst_vaapi_picture_h264_destroy(GstVaapiPictureH264 *decoder)
 106 {
 107 }
 108
 109 static gboolean
 110 gst_vaapi_picture_h264_create(
 111     GstVaapiPictureH264                      *picture,
 112     const GstVaapiCodecObjectConstructorArgs *args
 113 )
 114 {
 115     return TRUE;
 116 }
 117
 118 static void
 119 gst_vaapi_picture_h264_init(GstVaapiPictureH264 *picture)
 120 {
 121     VAPictureH264 *va_pic;
 122
 123     va_pic                      = &picture->info;
 124     va_pic->flags               = 0;
 125     va_pic->TopFieldOrderCnt    = 0;
 126     va_pic->BottomFieldOrderCnt = 0;
 127
 128     picture->poc                = 0;
 129     picture->is_long_term       = FALSE;
 130     picture->is_idr             = FALSE;
 131     picture->output_needed      = FALSE;
 132 }
 133
 134 static inline GstVaapiPictureH264 *
 135 gst_vaapi_picture_h264_new(GstVaapiDecoderH264 *decoder)
 136 {
 137     GstVaapiCodecObject *object;
 138
 139     g_return_val_if_fail(GST_VAAPI_IS_DECODER(decoder), NULL);
 140
 141     object = gst_vaapi_codec_object_new(
 142         GST_VAAPI_TYPE_PICTURE_H264,
 143         GST_VAAPI_CODEC_BASE(decoder),
 144         NULL, sizeof(VAPictureParameterBufferH264),
 145         NULL, 0
 146     );
 147     if (!object)
 148         return NULL;
 149     return GST_VAAPI_PICTURE_H264_CAST(object);
 150 }
 151
 152 static inline GstVaapiSliceH264 *
 153 gst_vaapi_picture_h264_get_last_slice(GstVaapiPictureH264 *picture)
 154 {
 155     g_return_val_if_fail(GST_VAAPI_IS_PICTURE_H264(picture), NULL);
 156
 157     if (G_UNLIKELY(picture->base.slices->len < 1))
 158         return NULL;
 159     return g_ptr_array_index(picture->base.slices,
 160         picture->base.slices->len - 1);
 161 }
 162
 163 /* ------------------------------------------------------------------------- */
 164 /* --- Slices                                                            --- */
 165 /* ------------------------------------------------------------------------- */
 166
 167 #define GST_VAAPI_TYPE_SLICE_H264 \
 168     (gst_vaapi_slice_h264_get_type())
 169
 170 #define GST_VAAPI_SLICE_H264_CAST(obj) \
 171     ((GstVaapiSliceH264 *)(obj))
 172
 173 #define GST_VAAPI_SLICE_H264(obj)                               \
 174     (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
 175                                 GST_VAAPI_TYPE_SLICE_H264,      \
 176                                 GstVaapiSliceH264))
 177
 178 #define GST_VAAPI_SLICE_H264_CLASS(klass)                       \
 179     (G_TYPE_CHECK_CLASS_CAST((klass),                           \
 180                              GST_VAAPI_TYPE_SLICE_H264,         \
 181                              GstVaapiSliceH264Class))
 182
 183 #define GST_VAAPI_IS_SLICE_H264(obj) \
 184     (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_VAAPI_TYPE_SLICE_H264))
 185
 186 #define GST_VAAPI_IS_SLICE_H264_CLASS(klass) \
 187     (G_TYPE_CHECK_CLASS_TYPE((klass), GST_VAAPI_TYPE_SLICE_H264))
 188
 189 #define GST_VAAPI_SLICE_H264_GET_CLASS(obj)                     \
 190     (G_TYPE_INSTANCE_GET_CLASS((obj),                           \
 191                                GST_VAAPI_TYPE_SLICE_H264,       \
 192                                GstVaapiSliceH264Class))
 193
 194 struct _GstVaapiSliceH264 {
 195     GstVaapiSlice               base;
 196     GstH264SliceHdr             slice_hdr;              // parsed slice_header()
 197 };
 198
 199 struct _GstVaapiSliceH264Class {
 200     /*< private >*/
 201     GstVaapiSliceClass          parent_class;
 202 };
 203
 204 GST_VAAPI_CODEC_DEFINE_TYPE(GstVaapiSliceH264,
 205                             gst_vaapi_slice_h264,
 206                             GST_VAAPI_TYPE_SLICE)
 207
 208 static void
 209 gst_vaapi_slice_h264_destroy(GstVaapiSliceH264 *slice)
 210 {
 211 }
 212
 213 static gboolean
 214 gst_vaapi_slice_h264_create(
 215     GstVaapiSliceH264                        *slice,
 216     const GstVaapiCodecObjectConstructorArgs *args
 217 )
 218 {
 219     return TRUE;
 220 }
 221
 222 static void
 223 gst_vaapi_slice_h264_init(GstVaapiSliceH264 *slice)
 224 {
 225 }
 226
 227 static inline GstVaapiSliceH264 *
 228 gst_vaapi_slice_h264_new(
 229     GstVaapiDecoderH264 *decoder,
 230     const guint8        *data,
 231     guint                data_size
 232 )
 233 {
 234     GstVaapiCodecObject *object;
 235
 236     g_return_val_if_fail(GST_VAAPI_IS_DECODER(decoder), NULL);
 237
 238     object = gst_vaapi_codec_object_new(
 239         GST_VAAPI_TYPE_SLICE_H264,
 240         GST_VAAPI_CODEC_BASE(decoder),
 241         NULL, sizeof(VASliceParameterBufferH264),
 242         data, data_size
 243     );
 244     if (!object)
 245         return NULL;
 246     return GST_VAAPI_SLICE_H264_CAST(object);
 247 }
 248
 249 /* ------------------------------------------------------------------------- */
 250 /* --- H.264 Decoder                                                     --- */
 251 /* ------------------------------------------------------------------------- */
 252
 253 G_DEFINE_TYPE(GstVaapiDecoderH264,
 254               gst_vaapi_decoder_h264,
 255               GST_VAAPI_TYPE_DECODER)
 256
 257 #define GST_VAAPI_DECODER_H264_GET_PRIVATE(obj)                 \
 258     (G_TYPE_INSTANCE_GET_PRIVATE((obj),                         \
 259                                  GST_VAAPI_TYPE_DECODER_H264,   \
 260                                  GstVaapiDecoderH264Private))
 261
 262 // Used for field_poc[]
 263 #define TOP_FIELD       0
 264 #define BOTTOM_FIELD    1
 265
 266 struct _GstVaapiDecoderH264Private {
 267     GstAdapter                 *adapter;
 268     GstH264NalParser           *parser;
 269     GstH264SPS                 *sps;
 270     GstH264SPS                  last_sps;
 271     GstH264PPS                 *pps;
 272     GstH264PPS                  last_pps;
 273     GstVaapiPictureH264        *current_picture;
 274     GstVaapiPictureH264        *dpb[16];
 275     guint                       dpb_count;
 276     guint                       dpb_size;
 277     GstVaapiProfile             profile;
 278     GstVaapiPictureH264        *short_ref[32];
 279     guint                       short_ref_count;
 280     GstVaapiPictureH264        *long_ref[32];
 281     guint                       long_ref_count;
 282     GstVaapiPictureH264        *RefPicList0[32];
 283     guint                       RefPicList0_count;
 284     GstVaapiPictureH264        *RefPicList1[32];
 285     guint                       RefPicList1_count;
 286     guint                       nal_length_size;
 287     guint                       width;
 288     guint                       height;
 289     guint                       mb_x;
 290     guint                       mb_y;
 291     guint                       mb_width;
 292     guint                       mb_height;
 293     gint32                      field_poc[2];           // 0:TopFieldOrderCnt / 1:BottomFieldOrderCnt
 294     gint32                      poc_msb;                // PicOrderCntMsb
 295     gint32                      poc_lsb;                // pic_order_cnt_lsb (from slice_header())
 296     gint32                      prev_poc_msb;           // prevPicOrderCntMsb
 297     gint32                      prev_poc_lsb;           // prevPicOrderCntLsb
 298     gint32                      frame_num_offset;       // FrameNumOffset
 299     gint32                      frame_num;              // frame_num (from slice_header())
 300     gint32                      prev_frame_num;         // prevFrameNum
 301     gboolean                    prev_pic_has_mmco5;     // prevMmco5Pic
 302     gboolean                    prev_pic_bottom_field;  // Flag: previous picture is a bottom field
 303     guint                       is_constructed          : 1;
 304     guint                       is_opened               : 1;
 305     guint                       is_avc                  : 1;
 306     guint                       has_context             : 1;
 307 };
 308
 309 static gboolean
 310 decode_picture_end(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture);
 311
 312 static void
 313 clear_references(
 314     GstVaapiDecoderH264  *decoder,
 315     GstVaapiPictureH264 **pictures,
 316     guint                *picture_count
 317 );
 318
 319 /* Get number of reference frames to use */
 320 static guint
 321 get_max_dec_frame_buffering(GstH264SPS *sps)
 322 {
 323     guint max_dec_frame_buffering, MaxDpbMbs, PicSizeMbs;
 324
 325     /* Table A-1 - Level limits */
 326     switch (sps->level_idc) {
 327     case 10: MaxDpbMbs = 396;    break;
 328     case 11: MaxDpbMbs = 900;    break;
 329     case 12: MaxDpbMbs = 2376;   break;
 330     case 13: MaxDpbMbs = 2376;   break;
 331     case 20: MaxDpbMbs = 2376;   break;
 332     case 21: MaxDpbMbs = 4752;   break;
 333     case 22: MaxDpbMbs = 8100;   break;
 334     case 30: MaxDpbMbs = 8100;   break;
 335     case 31: MaxDpbMbs = 18000;  break;
 336     case 32: MaxDpbMbs = 20480;  break;
 337     case 40: MaxDpbMbs = 32768;  break;
 338     case 41: MaxDpbMbs = 32768;  break;
 339     case 42: MaxDpbMbs = 34816;  break;
 340     case 50: MaxDpbMbs = 110400; break;
 341     case 51: MaxDpbMbs = 184320; break;
 342     default:
 343         g_assert(0 && "unhandled level");
 344         break;
 345     }
 346
 347     PicSizeMbs = ((sps->pic_width_in_mbs_minus1 + 1) *
 348                   (sps->pic_height_in_map_units_minus1 + 1) *
 349                   (sps->frame_mbs_only_flag ? 1 : 2));
 350     max_dec_frame_buffering = MaxDpbMbs / PicSizeMbs;
 351
 352     /* VUI parameters */
 353     if (sps->vui_parameters_present_flag) {
 354         GstH264VUIParams * const vui_params = &sps->vui_parameters;
 355         if (vui_params->bitstream_restriction_flag)
 356             max_dec_frame_buffering = vui_params->max_dec_frame_buffering;
 357         else {
 358             switch (sps->profile_idc) {
 359             case 44:  // CAVLC 4:4:4 Intra profile
 360             case 86:  // Scalable High profile
 361             case 100: // High profile
 362             case 110: // High 10 profile
 363             case 122: // High 4:2:2 profile
 364             case 244: // High 4:4:4 Predictive profile
 365                 if (sps->constraint_set3_flag)
 366                     max_dec_frame_buffering = 0;
 367                 break;
 368             }
 369         }
 370     }
 371
 372     if (max_dec_frame_buffering > 16)
 373         max_dec_frame_buffering = 16;
 374     else if (max_dec_frame_buffering < sps->num_ref_frames)
 375         max_dec_frame_buffering = sps->num_ref_frames;
 376     return MAX(1, max_dec_frame_buffering);
 377 }
 378
 379 static void
 380 dpb_remove_index(GstVaapiDecoderH264 *decoder, guint index)
 381 {
 382     GstVaapiDecoderH264Private * const priv = decoder->priv;
 383     guint i, num_pictures = --priv->dpb_count;
 384
 385     if (USE_STRICT_DPB_ORDERING) {
 386         for (i = index; i < num_pictures; i++)
 387             gst_vaapi_picture_replace(&priv->dpb[i], priv->dpb[i + 1]);
 388     }
 389     else if (index != num_pictures)
 390         gst_vaapi_picture_replace(&priv->dpb[index], priv->dpb[num_pictures]);
 391     gst_vaapi_picture_replace(&priv->dpb[num_pictures], NULL);
 392 }
 393
 394 static inline gboolean
 395 dpb_output(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
 396 {
 397     /* XXX: update cropping rectangle */
 398     picture->output_needed = FALSE;
 399     return gst_vaapi_picture_output(GST_VAAPI_PICTURE_CAST(picture));
 400 }
 401
 402 static gboolean
 403 dpb_bump(GstVaapiDecoderH264 *decoder)
 404 {
 405     GstVaapiDecoderH264Private * const priv = decoder->priv;
 406     guint i, lowest_poc_index;
 407     gboolean success;
 408
 409     for (i = 0; i < priv->dpb_count; i++) {
 410         if (priv->dpb[i]->output_needed)
 411             break;
 412     }
 413     if (i == priv->dpb_count)
 414         return FALSE;
 415
 416     lowest_poc_index = i++;
 417     for (; i < priv->dpb_count; i++) {
 418         GstVaapiPictureH264 * const picture = priv->dpb[i];
 419         if (picture->output_needed && picture->poc < priv->dpb[lowest_poc_index]->poc)
 420             lowest_poc_index = i;
 421     }
 422
 423     success = dpb_output(decoder, priv->dpb[lowest_poc_index]);
 424     if (!GST_VAAPI_PICTURE_IS_REFERENCE(priv->dpb[lowest_poc_index]))
 425         dpb_remove_index(decoder, lowest_poc_index);
 426     return success;
 427 }
 428
 429 static void
 430 dpb_flush(GstVaapiDecoderH264 *decoder)
 431 {
 432     GstVaapiDecoderH264Private * const priv = decoder->priv;
 433
 434     while (dpb_bump(decoder))
 435         ;
 436     clear_references(decoder, priv->dpb, &priv->dpb_count);
 437 }
 438
 439 static gboolean
 440 dpb_add(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
 441 {
 442     GstVaapiDecoderH264Private * const priv = decoder->priv;
 443     guint i;
 444
 445     // Remove all unused pictures
 446     if (picture->is_idr)
 447         dpb_flush(decoder);
 448     else {
 449         i = 0;
 450         while (i < priv->dpb_count) {
 451             GstVaapiPictureH264 * const picture = priv->dpb[i];
 452             if (!picture->output_needed &&
 453                 !GST_VAAPI_PICTURE_IS_REFERENCE(picture))
 454                 dpb_remove_index(decoder, i);
 455             else
 456                 i++;
 457         }
 458     }
 459
 460     // C.4.5.1 - Storage and marking of a reference decoded picture into the DPB
 461     if (GST_VAAPI_PICTURE_IS_REFERENCE(picture)) {
 462         while (priv->dpb_count == priv->dpb_size) {
 463             if (!dpb_bump(decoder))
 464                 return FALSE;
 465         }
 466         gst_vaapi_picture_replace(&priv->dpb[priv->dpb_count++], picture);
 467         if (picture->output_flag)
 468             picture->output_needed = TRUE;
 469     }
 470
 471     // C.4.5.2 - Storage and marking of a non-reference decoded picture into the DPB
 472     else {
 473         if (!picture->output_flag)
 474             return TRUE;
 475         while (priv->dpb_count == priv->dpb_size) {
 476             for (i = 0; i < priv->dpb_count; i++) {
 477                 if (priv->dpb[i]->output_needed &&
 478                     priv->dpb[i]->poc < picture->poc)
 479                     break;
 480             }
 481             if (i == priv->dpb_count)
 482                 return dpb_output(decoder, picture);
 483             if (!dpb_bump(decoder))
 484                 return FALSE;
 485         }
 486         gst_vaapi_picture_replace(&priv->dpb[priv->dpb_count++], picture);
 487         picture->output_needed = TRUE;
 488     }
 489     return TRUE;
 490 }
 491
 492 static inline void
 493 dpb_reset(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 494 {
 495     GstVaapiDecoderH264Private * const priv = decoder->priv;
 496
 497     priv->dpb_size = get_max_dec_frame_buffering(sps);
 498     GST_DEBUG("DPB size %u", priv->dpb_size);
 499 }
 500
 501 static GstVaapiDecoderStatus
 502 get_status(GstH264ParserResult result)
 503 {
 504     GstVaapiDecoderStatus status;
 505
 506     switch (result) {
 507     case GST_H264_PARSER_OK:
 508         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
 509         break;
 510     case GST_H264_PARSER_NO_NAL_END:
 511         status = GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
 512         break;
 513     case GST_H264_PARSER_ERROR:
 514         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
 515         break;
 516     default:
 517         status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 518         break;
 519     }
 520     return status;
 521 }
 522
 523 static void
 524 gst_vaapi_decoder_h264_close(GstVaapiDecoderH264 *decoder)
 525 {
 526     GstVaapiDecoderH264Private * const priv = decoder->priv;
 527
 528     gst_vaapi_picture_replace(&priv->current_picture, NULL);
 529     clear_references(decoder, priv->short_ref, &priv->short_ref_count);
 530     clear_references(decoder, priv->long_ref,  &priv->long_ref_count );
 531     clear_references(decoder, priv->dpb,       &priv->dpb_count      );
 532
 533     if (priv->parser) {
 534         gst_h264_nal_parser_free(priv->parser);
 535         priv->parser = NULL;
 536     }
 537
 538     if (priv->adapter) {
 539         gst_adapter_clear(priv->adapter);
 540         g_object_unref(priv->adapter);
 541         priv->adapter = NULL;
 542     }
 543 }
 544
 545 static gboolean
 546 gst_vaapi_decoder_h264_open(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
 547 {
 548     GstVaapiDecoderH264Private * const priv = decoder->priv;
 549
 550     gst_vaapi_decoder_h264_close(decoder);
 551
 552     priv->adapter = gst_adapter_new();
 553     if (!priv->adapter)
 554         return FALSE;
 555
 556     priv->parser = gst_h264_nal_parser_new();
 557     if (!priv->parser)
 558         return FALSE;
 559     return TRUE;
 560 }
 561
 562 static void
 563 gst_vaapi_decoder_h264_destroy(GstVaapiDecoderH264 *decoder)
 564 {
 565     gst_vaapi_decoder_h264_close(decoder);
 566 }
 567
 568 static gboolean
 569 gst_vaapi_decoder_h264_create(GstVaapiDecoderH264 *decoder)
 570 {
 571     if (!GST_VAAPI_DECODER_CODEC(decoder))
 572         return FALSE;
 573     return TRUE;
 574 }
 575
 576 static GstVaapiDecoderStatus
 577 ensure_context(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 578 {
 579     GstVaapiDecoderH264Private * const priv = decoder->priv;
 580     GstVaapiProfile profiles[2];
 581     GstVaapiEntrypoint entrypoint = GST_VAAPI_ENTRYPOINT_VLD;
 582     guint i, n_profiles = 0;
 583     gboolean success, reset_context = FALSE;
 584
 585     if (!priv->has_context || priv->sps->profile_idc != sps->profile_idc) {
 586         GST_DEBUG("profile changed");
 587         reset_context = TRUE;
 588
 589         switch (sps->profile_idc) {
 590         case 66:
 591             profiles[n_profiles++] = GST_VAAPI_PROFILE_H264_BASELINE;
 592             break;
 593         case 77:
 594             profiles[n_profiles++] = GST_VAAPI_PROFILE_H264_MAIN;
 595             // fall-through
 596         case 100:
 597             profiles[n_profiles++] = GST_VAAPI_PROFILE_H264_HIGH;
 598             break;
 599         default:
 600             GST_DEBUG("unsupported profile %d", sps->profile_idc);
 601             return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_PROFILE;
 602         }
 603
 604         for (i = 0; i < n_profiles; i++) {
 605             success = gst_vaapi_display_has_decoder(
 606                 GST_VAAPI_DECODER_DISPLAY(decoder),
 607                 profiles[i],
 608                 entrypoint
 609             );
 610             if (success)
 611                 break;
 612         }
 613         if (i == n_profiles)
 614             return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_PROFILE;
 615         priv->profile = profiles[i];
 616     }
 617
 618     if (!priv->has_context ||
 619         priv->sps->chroma_format_idc != sps->chroma_format_idc) {
 620         GST_DEBUG("chroma format changed");
 621         reset_context = TRUE;
 622
 623         /* XXX: theoritically, we could handle 4:2:2 format */
 624         if (sps->chroma_format_idc != 1)
 625             return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;
 626     }
 627
 628     if (!priv->has_context              ||
 629         priv->sps->width  != sps->width ||
 630         priv->sps->height != sps->height) {
 631         GST_DEBUG("size changed");
 632         reset_context      = TRUE;
 633
 634         priv->width      = sps->width;
 635         priv->height     = sps->height;
 636         priv->mb_width   = sps->pic_width_in_mbs_minus1 + 1;
 637         priv->mb_height  = sps->pic_height_in_map_units_minus1 + 1;
 638         priv->mb_height *= 2 - sps->frame_mbs_only_flag;
 639     }
 640
 641     if (reset_context) {
 642         GstVaapiContextInfo info;
 643
 644         info.profile    = priv->profile;
 645         info.entrypoint = entrypoint;
 646         info.width      = priv->width;
 647         info.height     = priv->height;
 648         info.ref_frames = get_max_dec_frame_buffering(sps);
 649
 650         if (!gst_vaapi_decoder_ensure_context(GST_VAAPI_DECODER(decoder), &info))
 651             return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 652         priv->has_context = TRUE;
 653
 654         /* Reset DPB */
 655         dpb_reset(decoder, sps);
 656     }
 657     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 658 }
 659
 660 static GstVaapiDecoderStatus
 661 ensure_quant_matrix(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
 662 {
 663     GstVaapiDecoderH264Private * const priv = decoder->priv;
 664     GstH264SPS * const sps = priv->sps;
 665     GstH264PPS * const pps = priv->pps;
 666     GstVaapiPicture * const base_picture = &picture->base;
 667     VAIQMatrixBufferH264 *iq_matrix;
 668     guint i, j, n;
 669
 670     base_picture->iq_matrix = GST_VAAPI_IQ_MATRIX_NEW(H264, decoder);
 671     if (!base_picture->iq_matrix) {
 672         GST_ERROR("failed to allocate IQ matrix");
 673         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
 674     }
 675     iq_matrix = base_picture->iq_matrix->param;
 676
 677     /* XXX: we can only support 4:2:0 or 4:2:2 since ScalingLists8x8[]
 678        is not large enough to hold lists for 4:4:4 */
 679     if (sps->chroma_format_idc == 3)
 680         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;
 681
 682     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList4x4[0]) == 16);
 683     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList8x8[0]) == 64);
 684
 685     if (sizeof(iq_matrix->ScalingList4x4) == sizeof(pps->scaling_lists_4x4))
 686         memcpy(iq_matrix->ScalingList4x4, pps->scaling_lists_4x4,
 687                sizeof(iq_matrix->ScalingList4x4));
 688     else {
 689         n = MIN(G_N_ELEMENTS(iq_matrix->ScalingList4x4),
 690                 G_N_ELEMENTS(pps->scaling_lists_4x4));
 691         for (i = 0; i < n; i++) {
 692             for (j = 0; j < 16; j++)
 693                 iq_matrix->ScalingList4x4[i][j] = pps->scaling_lists_4x4[i][j];
 694         }
 695     }
 696
 697     if (sizeof(iq_matrix->ScalingList8x8) == sizeof(pps->scaling_lists_8x8))
 698         memcpy(iq_matrix->ScalingList8x8, pps->scaling_lists_8x8,
 699                sizeof(iq_matrix->ScalingList8x8));
 700     else {
 701         n = MIN(G_N_ELEMENTS(iq_matrix->ScalingList8x8),
 702                 G_N_ELEMENTS(pps->scaling_lists_8x8));
 703         for (i = 0; i < n; i++) {
 704             for (j = 0; j < 16; j++)
 705                 iq_matrix->ScalingList8x8[i][j] = pps->scaling_lists_8x8[i][j];
 706         }
 707     }
 708     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 709 }
 710
 711 static gboolean
 712 decode_current_picture(GstVaapiDecoderH264 *decoder)
 713 {
 714     GstVaapiDecoderH264Private * const priv = decoder->priv;
 715     GstVaapiPictureH264 * const picture = priv->current_picture;
 716     gboolean success = FALSE;
 717
 718     if (!picture)
 719         return TRUE;
 720
 721     if (!decode_picture_end(decoder, picture))
 722         goto end;
 723     if (!gst_vaapi_picture_decode(GST_VAAPI_PICTURE_CAST(picture)))
 724         goto end;
 725     success = TRUE;
 726 end:
 727     gst_vaapi_picture_replace(&priv->current_picture, NULL);
 728     return success;
 729 }
 730
 731 static GstVaapiDecoderStatus
 732 decode_sps(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
 733 {
 734     GstVaapiDecoder * const base_decoder = GST_VAAPI_DECODER(decoder);
 735     GstVaapiDecoderH264Private * const priv = decoder->priv;
 736     GstH264SPS * const sps = &priv->last_sps;
 737     GstH264ParserResult result;
 738
 739     GST_DEBUG("decode SPS");
 740
 741     if (priv->current_picture && !decode_current_picture(decoder))
 742         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 743
 744     memset(sps, 0, sizeof(*sps));
 745     result = gst_h264_parser_parse_sps(priv->parser, nalu, sps, TRUE);
 746     if (result != GST_H264_PARSER_OK)
 747         return get_status(result);
 748
 749     gst_vaapi_decoder_set_pixel_aspect_ratio(
 750         base_decoder,
 751         sps->vui_parameters.par_n,
 752         sps->vui_parameters.par_d
 753     );
 754     return ensure_context(decoder, sps);
 755 }
 756
 757 static GstVaapiDecoderStatus
 758 decode_pps(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
 759 {
 760     GstVaapiDecoderH264Private * const priv = decoder->priv;
 761     GstH264PPS * const pps = &priv->last_pps;
 762     GstH264ParserResult result;
 763
 764     GST_DEBUG("decode PPS");
 765
 766     if (priv->current_picture && !decode_current_picture(decoder))
 767         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 768
 769     memset(pps, 0, sizeof(*pps));
 770     result = gst_h264_parser_parse_pps(priv->parser, nalu, pps);
 771     if (result != GST_H264_PARSER_OK)
 772         return get_status(result);
 773
 774     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 775 }
 776
 777 static GstVaapiDecoderStatus
 778 decode_sei(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
 779 {
 780     GstVaapiDecoderH264Private * const priv = decoder->priv;
 781     GstH264SEIMessage sei;
 782     GstH264ParserResult result;
 783
 784     GST_DEBUG("decode SEI");
 785
 786     memset(&sei, 0, sizeof(sei));
 787     result = gst_h264_parser_parse_sei(priv->parser, nalu, &sei);
 788     if (result != GST_H264_PARSER_OK) {
 789         GST_WARNING("failed to decode SEI, payload type:%d", sei.payloadType);
 790         return get_status(result);
 791     }
 792
 793     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 794 }
 795
 796 static GstVaapiDecoderStatus
 797 decode_sequence_end(GstVaapiDecoderH264 *decoder)
 798 {
 799     GstVaapiDecoderH264Private * const priv = decoder->priv;
 800
 801     GST_DEBUG("decode sequence-end");
 802
 803     if (priv->current_picture && !decode_current_picture(decoder))
 804         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 805     dpb_flush(decoder);
 806     return GST_VAAPI_DECODER_STATUS_END_OF_STREAM;
 807 }
 808
 809 /* 8.2.1.1 - Decoding process for picture order count type 0 */
 810 static void
 811 init_picture_poc_0(
 812     GstVaapiDecoderH264 *decoder,
 813     GstVaapiPictureH264 *picture,
 814     GstH264SliceHdr     *slice_hdr
 815 )
 816 {
 817     GstVaapiDecoderH264Private * const priv = decoder->priv;
 818     GstH264PPS * const pps = slice_hdr->pps;
 819     GstH264SPS * const sps = pps->sequence;
 820     const gint32 MaxPicOrderCntLsb = 1 << (sps->log2_max_pic_order_cnt_lsb_minus4 + 4);
 821
 822     GST_DEBUG("decode picture order count type 0");
 823
 824     if (picture->is_idr) {
 825         priv->prev_poc_msb = 0;
 826         priv->prev_poc_lsb = 0;
 827     }
 828     else if (priv->prev_pic_has_mmco5) {
 829         priv->prev_poc_msb = 0;
 830         priv->prev_poc_lsb = priv->prev_pic_bottom_field ? 0 :
 831             priv->field_poc[TOP_FIELD];
 832     }
 833     else {
 834         priv->prev_poc_msb = priv->poc_msb;
 835         priv->prev_poc_lsb = priv->poc_lsb;
 836     }
 837
 838     // (8-3)
 839     priv->poc_lsb = slice_hdr->pic_order_cnt_lsb;
 840     if (priv->poc_lsb < priv->prev_poc_lsb &&
 841         (priv->prev_poc_lsb - priv->poc_lsb) >= (MaxPicOrderCntLsb / 2))
 842         priv->poc_msb = priv->prev_poc_msb + MaxPicOrderCntLsb;
 843     else if (priv->poc_lsb > priv->prev_poc_lsb &&
 844              (priv->poc_lsb - priv->prev_poc_lsb) > (MaxPicOrderCntLsb / 2))
 845         priv->poc_msb = priv->prev_poc_msb - MaxPicOrderCntLsb;
 846     else
 847         priv->poc_msb = priv->prev_poc_msb;
 848
 849     // (8-4)
 850     if (!slice_hdr->field_pic_flag || !slice_hdr->bottom_field_flag)
 851         priv->field_poc[TOP_FIELD] = priv->poc_msb + priv->poc_lsb;
 852
 853     // (8-5)
 854     if (!slice_hdr->field_pic_flag)
 855         priv->field_poc[BOTTOM_FIELD] = priv->field_poc[TOP_FIELD] +
 856             slice_hdr->delta_pic_order_cnt_bottom;
 857     else if (slice_hdr->bottom_field_flag)
 858         priv->field_poc[BOTTOM_FIELD] = priv->poc_msb + priv->poc_lsb;
 859 }
 860
 861 /* 8.2.1.2 - Decoding process for picture order count type 1 */
 862 static void
 863 init_picture_poc_1(
 864     GstVaapiDecoderH264 *decoder,
 865     GstVaapiPictureH264 *picture,
 866     GstH264SliceHdr     *slice_hdr
 867 )
 868 {
 869     GstVaapiDecoderH264Private * const priv = decoder->priv;
 870     GstH264PPS * const pps = slice_hdr->pps;
 871     GstH264SPS * const sps = pps->sequence;
 872     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
 873     gint32 prev_frame_num_offset, abs_frame_num, expected_poc;
 874     guint i;
 875
 876     GST_DEBUG("decode picture order count type 1");
 877
 878     if (priv->prev_pic_has_mmco5)
 879         prev_frame_num_offset = 0;
 880     else
 881         prev_frame_num_offset = priv->frame_num_offset;
 882
 883     // (8-6)
 884     if (picture->is_idr)
 885         priv->frame_num_offset = 0;
 886     else if (priv->prev_frame_num > priv->frame_num)
 887         priv->frame_num_offset = prev_frame_num_offset + MaxFrameNum;
 888     else
 889         priv->frame_num_offset = prev_frame_num_offset;
 890
 891     // (8-7)
 892     if (sps->num_ref_frames_in_pic_order_cnt_cycle != 0)
 893         abs_frame_num = priv->frame_num_offset + priv->frame_num;
 894     else
 895         abs_frame_num = 0;
 896     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture) && abs_frame_num > 0)
 897         abs_frame_num = abs_frame_num - 1;
 898
 899     if (abs_frame_num > 0) {
 900         gint32 expected_delta_per_poc_cycle;
 901         gint32 poc_cycle_cnt, frame_num_in_poc_cycle;
 902
 903         expected_delta_per_poc_cycle = 0;
 904         for (i = 0; i < sps->num_ref_frames_in_pic_order_cnt_cycle; i++)
 905             expected_delta_per_poc_cycle += sps->offset_for_ref_frame[i];
 906
 907         // (8-8)
 908         poc_cycle_cnt = (abs_frame_num - 1) /
 909             sps->num_ref_frames_in_pic_order_cnt_cycle;
 910         frame_num_in_poc_cycle = (abs_frame_num - 1) %
 911             sps->num_ref_frames_in_pic_order_cnt_cycle;
 912
 913         // (8-9)
 914         expected_poc = poc_cycle_cnt * expected_delta_per_poc_cycle;
 915         for (i = 0; i <= frame_num_in_poc_cycle; i++)
 916             expected_poc += sps->offset_for_ref_frame[i];
 917     }
 918     else
 919         expected_poc = 0;
 920     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
 921         expected_poc += sps->offset_for_non_ref_pic;
 922
 923     // (8-10)
 924     if (!slice_hdr->field_pic_flag) {
 925         priv->field_poc[TOP_FIELD] = expected_poc +
 926             slice_hdr->delta_pic_order_cnt[0];
 927         priv->field_poc[BOTTOM_FIELD] = priv->field_poc[TOP_FIELD] +
 928             sps->offset_for_top_to_bottom_field +
 929             slice_hdr->delta_pic_order_cnt[1];
 930     }
 931     else if (!slice_hdr->bottom_field_flag)
 932         priv->field_poc[TOP_FIELD] = expected_poc +
 933             slice_hdr->delta_pic_order_cnt[0];
 934     else
 935         priv->field_poc[BOTTOM_FIELD] = expected_poc +
 936             sps->offset_for_top_to_bottom_field + slice_hdr->delta_pic_order_cnt[0];
 937 }
 938
 939 /* 8.2.1.3 - Decoding process for picture order count type 2 */
 940 static void
 941 init_picture_poc_2(
 942     GstVaapiDecoderH264 *decoder,
 943     GstVaapiPictureH264 *picture,
 944     GstH264SliceHdr     *slice_hdr
 945 )
 946 {
 947     GstVaapiDecoderH264Private * const priv = decoder->priv;
 948     GstH264PPS * const pps = slice_hdr->pps;
 949     GstH264SPS * const sps = pps->sequence;
 950     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
 951     gint32 prev_frame_num_offset, temp_poc;
 952
 953     GST_DEBUG("decode picture order count type 2");
 954
 955     if (priv->prev_pic_has_mmco5)
 956         prev_frame_num_offset = 0;
 957     else
 958         prev_frame_num_offset = priv->frame_num_offset;
 959
 960     // (8-11)
 961     if (picture->is_idr)
 962         priv->frame_num_offset = 0;
 963     else if (priv->prev_frame_num > priv->frame_num)
 964         priv->frame_num_offset = prev_frame_num_offset + MaxFrameNum;
 965     else
 966         priv->frame_num_offset = prev_frame_num_offset;
 967
 968     // (8-12)
 969     if (picture->is_idr)
 970         temp_poc = 0;
 971     else if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
 972         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num) - 1;
 973     else
 974         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num);
 975
 976     // (8-13)
 977     if (!slice_hdr->field_pic_flag) {
 978         priv->field_poc[TOP_FIELD] = temp_poc;
 979         priv->field_poc[BOTTOM_FIELD] = temp_poc;
 980     }
 981     else if (slice_hdr->bottom_field_flag)
 982         priv->field_poc[BOTTOM_FIELD] = temp_poc;
 983     else
 984         priv->field_poc[TOP_FIELD] = temp_poc;
 985 }
 986
 987 /* 8.2.1 - Decoding process for picture order count */
 988 static void
 989 init_picture_poc(
 990     GstVaapiDecoderH264 *decoder,
 991     GstVaapiPictureH264 *picture,
 992     GstH264SliceHdr     *slice_hdr
 993 )
 994 {
 995     GstVaapiDecoderH264Private * const priv = decoder->priv;
 996     VAPictureH264 * const pic = &picture->info;
 997     GstH264PPS * const pps = slice_hdr->pps;
 998     GstH264SPS * const sps = pps->sequence;
 999
1000     switch (sps->pic_order_cnt_type) {
1001     case 0:
1002         init_picture_poc_0(decoder, picture, slice_hdr);
1003         break;
1004     case 1:
1005         init_picture_poc_1(decoder, picture, slice_hdr);
1006         break;
1007     case 2:
1008         init_picture_poc_2(decoder, picture, slice_hdr);
1009         break;
1010     }
1011
1012     if (!(pic->flags & VA_PICTURE_H264_BOTTOM_FIELD))
1013         pic->TopFieldOrderCnt = priv->field_poc[TOP_FIELD];
1014     if (!(pic->flags & VA_PICTURE_H264_TOP_FIELD))
1015         pic->BottomFieldOrderCnt = priv->field_poc[BOTTOM_FIELD];
1016     picture->poc = MIN(pic->TopFieldOrderCnt, pic->BottomFieldOrderCnt);
1017 }
1018
1019 static int
1020 compare_picture_pic_num_dec(const void *a, const void *b)
1021 {
1022     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1023     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1024
1025     return picB->pic_num - picA->pic_num;
1026 }
1027
1028 static int
1029 compare_picture_long_term_pic_num_inc(const void *a, const void *b)
1030 {
1031     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1032     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1033
1034     return picA->long_term_pic_num - picB->long_term_pic_num;
1035 }
1036
1037 static int
1038 compare_picture_poc_dec(const void *a, const void *b)
1039 {
1040     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1041     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1042
1043     return picB->poc - picA->poc;
1044 }
1045
1046 static int
1047 compare_picture_poc_inc(const void *a, const void *b)
1048 {
1049     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1050     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1051
1052     return picA->poc - picB->poc;
1053 }
1054
1055 static int
1056 compare_picture_frame_num_wrap_dec(const void *a, const void *b)
1057 {
1058     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1059     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1060
1061     return picB->frame_num_wrap - picA->frame_num_wrap;
1062 }
1063
1064 static int
1065 compare_picture_long_term_frame_idx_inc(const void *a, const void *b)
1066 {
1067     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1068     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1069
1070     return picA->info.frame_idx - picB->info.frame_idx;
1071 }
1072
1073 /* 8.2.4.1 - Decoding process for picture numbers */
1074 static void
1075 init_picture_refs_pic_num(
1076     GstVaapiDecoderH264 *decoder,
1077     GstVaapiPictureH264 *picture,
1078     GstH264SliceHdr     *slice_hdr
1079 )
1080 {
1081     GstVaapiDecoderH264Private * const priv = decoder->priv;
1082     GstH264PPS * const pps = slice_hdr->pps;
1083     GstH264SPS * const sps = pps->sequence;
1084     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1085     const guint field_flags = VA_PICTURE_H264_TOP_FIELD | VA_PICTURE_H264_BOTTOM_FIELD;
1086     guint i;
1087
1088     GST_DEBUG("decode picture numbers");
1089
1090     for (i = 0; i < priv->short_ref_count; i++) {
1091         GstVaapiPictureH264 * const pic = priv->short_ref[i];
1092
1093         // (8-27)
1094         if (pic->frame_num > priv->frame_num)
1095             pic->frame_num_wrap = pic->frame_num - MaxFrameNum;
1096         else
1097             pic->frame_num_wrap = pic->frame_num;
1098
1099         // (8-28, 8-30, 8-31)
1100         if (!pic->field_pic_flag)
1101             pic->pic_num = pic->frame_num_wrap;
1102         else {
1103             if (((picture->info.flags ^ pic->info.flags) & field_flags) == 0)
1104                 pic->pic_num = 2 * pic->frame_num_wrap + 1;
1105             else
1106                 pic->pic_num = 2 * pic->frame_num_wrap;
1107         }
1108     }
1109
1110     for (i = 0; i < priv->long_ref_count; i++) {
1111         GstVaapiPictureH264 * const pic = priv->long_ref[i];
1112
1113         // (8-29, 8-32, 8-33)
1114         if (!pic->field_pic_flag)
1115             pic->long_term_pic_num = pic->info.frame_idx;
1116         else {
1117             if (((picture->info.flags ^ pic->info.flags) & field_flags) == 0)
1118                 pic->long_term_pic_num = 2 * pic->info.frame_idx + 1;
1119             else
1120                 pic->long_term_pic_num = 2 * pic->info.frame_idx;
1121         }
1122     }
1123 }
1124
1125 #define SORT_REF_LIST(list, n, compare_func) \
1126     qsort(list, n, sizeof(*(list)), compare_picture_##compare_func)
1127
1128 static void
1129 init_picture_refs_p_slice(
1130     GstVaapiDecoderH264 *decoder,
1131     GstVaapiPictureH264 *picture,
1132     GstH264SliceHdr     *slice_hdr
1133 )
1134 {
1135     GstVaapiDecoderH264Private * const priv = decoder->priv;
1136     GstVaapiPictureH264 **ref_list;
1137     guint i;
1138
1139     GST_DEBUG("decode reference picture list for P and SP slices");
1140
1141     if (!picture->field_pic_flag) {
1142         /* 8.2.4.2.1 - P and SP slices in frames */
1143         if (priv->short_ref_count > 0) {
1144             ref_list = priv->RefPicList0;
1145             for (i = 0; i < priv->short_ref_count; i++)
1146                 ref_list[i] = priv->short_ref[i];
1147             SORT_REF_LIST(ref_list, i, pic_num_dec);
1148             priv->RefPicList0_count += i;
1149         }
1150
1151         if (priv->long_ref_count > 0) {
1152             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1153             for (i = 0; i < priv->long_ref_count; i++)
1154                 ref_list[i] = priv->long_ref[i];
1155             SORT_REF_LIST(ref_list, i, long_term_pic_num_inc);
1156             priv->RefPicList0_count += i;
1157         }
1158     }
1159     else {
1160         /* 8.2.4.2.2 - P and SP slices in fields */
1161         GstVaapiPictureH264 *short_ref[32];
1162         guint short_ref_count = 0;
1163         GstVaapiPictureH264 *long_ref[32];
1164         guint long_ref_count = 0;
1165
1166         // XXX: handle second field if current field is marked as
1167         // "used for short-term reference"
1168         if (priv->short_ref_count > 0) {
1169             for (i = 0; i < priv->short_ref_count; i++)
1170                 short_ref[i] = priv->short_ref[i];
1171             SORT_REF_LIST(short_ref, i, frame_num_wrap_dec);
1172             short_ref_count = i;
1173         }
1174
1175         // XXX: handle second field if current field is marked as
1176         // "used for long-term reference"
1177         if (priv->long_ref_count > 0) {
1178             for (i = 0; i < priv->long_ref_count; i++)
1179                 long_ref[i] = priv->long_ref[i];
1180             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1181             long_ref_count = i;
1182         }
1183
1184         // XXX: handle 8.2.4.2.5
1185     }
1186 }
1187
1188 static void
1189 init_picture_refs_b_slice(
1190     GstVaapiDecoderH264 *decoder,
1191     GstVaapiPictureH264 *picture,
1192     GstH264SliceHdr     *slice_hdr
1193 )
1194 {
1195     GstVaapiDecoderH264Private * const priv = decoder->priv;
1196     GstVaapiPictureH264 **ref_list;
1197     guint i, n;
1198
1199     GST_DEBUG("decode reference picture list for B slices");
1200
1201     if (!picture->field_pic_flag) {
1202         /* 8.2.4.2.3 - B slices in frames */
1203
1204         /* RefPicList0 */
1205         if (priv->short_ref_count > 0) {
1206             // 1. Short-term references
1207             ref_list = priv->RefPicList0;
1208             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1209                 if (priv->short_ref[i]->poc < picture->poc)
1210                     ref_list[n++] = priv->short_ref[i];
1211             }
1212             SORT_REF_LIST(ref_list, n, poc_dec);
1213             priv->RefPicList0_count += n;
1214
1215             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1216             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1217                 if (priv->short_ref[i]->poc >= picture->poc)
1218                     ref_list[n++] = priv->short_ref[i];
1219             }
1220             SORT_REF_LIST(ref_list, n, poc_inc);
1221             priv->RefPicList0_count += n;
1222         }
1223
1224         if (priv->long_ref_count > 0) {
1225             // 2. Long-term references
1226             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1227             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1228                 ref_list[n++] = priv->long_ref[i];
1229             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1230             priv->RefPicList0_count += n;
1231         }
1232
1233         /* RefPicList1 */
1234         if (priv->short_ref_count > 0) {
1235             // 1. Short-term references
1236             ref_list = priv->RefPicList1;
1237             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1238                 if (priv->short_ref[i]->poc > picture->poc)
1239                     ref_list[n++] = priv->short_ref[i];
1240             }
1241             SORT_REF_LIST(ref_list, n, poc_inc);
1242             priv->RefPicList1_count += n;
1243
1244             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1245             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1246                 if (priv->short_ref[i]->poc <= picture->poc)
1247                     ref_list[n++] = priv->short_ref[i];
1248             }
1249             SORT_REF_LIST(ref_list, n, poc_dec);
1250             priv->RefPicList1_count += n;
1251         }
1252
1253         if (priv->long_ref_count > 0) {
1254             // 2. Long-term references
1255             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1256             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1257                 ref_list[n++] = priv->long_ref[i];
1258             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1259             priv->RefPicList1_count += n;
1260         }
1261     }
1262     else {
1263         /* 8.2.4.2.4 - B slices in fields */
1264         GstVaapiPictureH264 *short_ref0[32];
1265         guint short_ref0_count = 0;
1266         GstVaapiPictureH264 *short_ref1[32];
1267         guint short_ref1_count = 0;
1268         GstVaapiPictureH264 *long_ref[32];
1269         guint long_ref_count = 0;
1270
1271         /* refFrameList0ShortTerm */
1272         if (priv->short_ref_count > 0) {
1273             ref_list = short_ref0;
1274             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1275                 if (priv->short_ref[i]->poc <= picture->poc)
1276                     ref_list[n++] = priv->short_ref[i];
1277             }
1278             SORT_REF_LIST(ref_list, n, poc_dec);
1279             short_ref0_count += n;
1280
1281             ref_list = &short_ref0[short_ref0_count];
1282             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1283                 if (priv->short_ref[i]->poc > picture->poc)
1284                     ref_list[n++] = priv->short_ref[i];
1285             }
1286             SORT_REF_LIST(ref_list, n, poc_inc);
1287             short_ref0_count += n;
1288         }
1289
1290         /* refFrameList1ShortTerm */
1291         if (priv->short_ref_count > 0) {
1292             ref_list = short_ref1;
1293             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1294                 if (priv->short_ref[i]->poc > picture->poc)
1295                     ref_list[n++] = priv->short_ref[i];
1296             }
1297             SORT_REF_LIST(ref_list, n, poc_inc);
1298             short_ref1_count += n;
1299
1300             ref_list = &short_ref1[short_ref1_count];
1301             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1302                 if (priv->short_ref[i]->poc <= picture->poc)
1303                     ref_list[n++] = priv->short_ref[i];
1304             }
1305             SORT_REF_LIST(ref_list, n, poc_dec);
1306             short_ref1_count += n;
1307         }
1308
1309         /* refFrameListLongTerm */
1310         if (priv->long_ref_count > 0) {
1311             for (i = 0; i < priv->long_ref_count; i++)
1312                 long_ref[i] = priv->long_ref[i];
1313             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1314             long_ref_count = i;
1315         }
1316
1317         // XXX: handle 8.2.4.2.5
1318     }
1319
1320     /* Check whether RefPicList1 is identical to RefPicList0, then
1321        swap if necessary */
1322     if (priv->RefPicList1_count > 1 &&
1323         priv->RefPicList1_count == priv->RefPicList0_count &&
1324         memcmp(priv->RefPicList0, priv->RefPicList1,
1325                priv->RefPicList0_count * sizeof(priv->RefPicList0[0])) == 0) {
1326         GstVaapiPictureH264 * const tmp = priv->RefPicList1[0];
1327         priv->RefPicList1[0] = priv->RefPicList1[1];
1328         priv->RefPicList1[1] = tmp;
1329     }
1330 }
1331
1332 #undef SORT_REF_LIST
1333
1334 static void
1335 clear_references(
1336     GstVaapiDecoderH264  *decoder,
1337     GstVaapiPictureH264 **pictures,
1338     guint                *picture_count
1339 )
1340 {
1341     const guint num_pictures = *picture_count;
1342     guint i;
1343
1344     for (i = 0; i < num_pictures; i++)
1345         gst_vaapi_picture_replace(&pictures[i], NULL);
1346     *picture_count = 0;
1347 }
1348
1349 static gboolean
1350 remove_reference_at(
1351     GstVaapiDecoderH264  *decoder,
1352     GstVaapiPictureH264 **pictures,
1353     guint                *picture_count,
1354     guint                 index
1355 )
1356 {
1357     guint num_pictures = *picture_count;
1358     GstVaapiPictureH264 *picture;
1359
1360     g_return_val_if_fail(index < num_pictures, FALSE);
1361
1362     picture = pictures[index];
1363     GST_VAAPI_PICTURE_FLAG_UNSET(picture, GST_VAAPI_PICTURE_FLAG_REFERENCE);
1364     picture->is_long_term = FALSE;
1365     picture->info.flags &= ~(VA_PICTURE_H264_SHORT_TERM_REFERENCE |
1366                              VA_PICTURE_H264_LONG_TERM_REFERENCE);
1367
1368     if (index != --num_pictures)
1369         gst_vaapi_picture_replace(&pictures[index], pictures[num_pictures]);
1370     gst_vaapi_picture_replace(&pictures[num_pictures], NULL);
1371     *picture_count = num_pictures;
1372     return TRUE;
1373 }
1374
1375 static gint
1376 find_short_term_reference(GstVaapiDecoderH264 *decoder, gint32 pic_num)
1377 {
1378     GstVaapiDecoderH264Private * const priv = decoder->priv;
1379     guint i;
1380
1381     for (i = 0; i < priv->short_ref_count; i++) {
1382         if (priv->short_ref[i]->pic_num == pic_num)
1383             return i;
1384     }
1385     GST_ERROR("found no short-term reference picture with PicNum = %d",
1386               pic_num);
1387     return -1;
1388 }
1389
1390 static gint
1391 find_long_term_reference(GstVaapiDecoderH264 *decoder, gint32 long_term_pic_num)
1392 {
1393     GstVaapiDecoderH264Private * const priv = decoder->priv;
1394     guint i;
1395
1396     for (i = 0; i < priv->long_ref_count; i++) {
1397         if (priv->long_ref[i]->long_term_pic_num == long_term_pic_num)
1398             return i;
1399     }
1400     GST_ERROR("found no long-term reference picture with LongTermPicNum = %d",
1401               long_term_pic_num);
1402     return -1;
1403 }
1404
1405 static void
1406 exec_picture_refs_modification_1(
1407     GstVaapiDecoderH264           *decoder,
1408     GstVaapiPictureH264           *picture,
1409     GstH264SliceHdr               *slice_hdr,
1410     guint                          list
1411 )
1412 {
1413     GstVaapiDecoderH264Private * const priv = decoder->priv;
1414     GstH264PPS * const pps = slice_hdr->pps;
1415     GstH264SPS * const sps = pps->sequence;
1416     GstH264RefPicListModification *ref_pic_list_modification;
1417     guint num_ref_pic_list_modifications;
1418     GstVaapiPictureH264 **ref_list;
1419     guint *ref_list_count_ptr, ref_list_count, ref_list_idx = 0;
1420     guint i, j, n, num_refs;
1421     gint found_ref_idx;
1422     gint32 MaxPicNum, CurrPicNum, picNumPred;
1423
1424     GST_DEBUG("modification process of reference picture list %u", list);
1425
1426     if (list == 0) {
1427         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l0;
1428         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l0;
1429         ref_list                       = priv->RefPicList0;
1430         ref_list_count_ptr             = &priv->RefPicList0_count;
1431         num_refs                       = slice_hdr->num_ref_idx_l0_active_minus1 + 1;
1432     }
1433     else {
1434         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l1;
1435         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l1;
1436         ref_list                       = priv->RefPicList1;
1437         ref_list_count_ptr             = &priv->RefPicList1_count;
1438         num_refs                       = slice_hdr->num_ref_idx_l1_active_minus1 + 1;
1439     }
1440     ref_list_count = *ref_list_count_ptr;
1441
1442     if (picture->field_pic_flag) {
1443         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 5); // 2 * MaxFrameNum
1444         CurrPicNum = 2 * slice_hdr->frame_num + 1;              // 2 * frame_num + 1
1445     }
1446     else {
1447         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 4); // MaxFrameNum
1448         CurrPicNum = slice_hdr->frame_num;                      // frame_num
1449     }
1450
1451     picNumPred = CurrPicNum;
1452
1453     for (i = 0; i < num_ref_pic_list_modifications; i++) {
1454         GstH264RefPicListModification * const l = &ref_pic_list_modification[i];
1455         if (l->modification_of_pic_nums_idc == 3)
1456             break;
1457
1458         /* 8.2.4.3.1 - Short-term reference pictures */
1459         if (l->modification_of_pic_nums_idc == 0 || l->modification_of_pic_nums_idc == 1) {
1460             gint32 abs_diff_pic_num = l->value.abs_diff_pic_num_minus1 + 1;
1461             gint32 picNum, picNumNoWrap;
1462
1463             // (8-34)
1464             if (l->modification_of_pic_nums_idc == 0) {
1465                 picNumNoWrap = picNumPred - abs_diff_pic_num;
1466                 if (picNumNoWrap < 0)
1467                     picNumNoWrap += MaxPicNum;
1468             }
1469
1470             // (8-35)
1471             else {
1472                 picNumNoWrap = picNumPred + abs_diff_pic_num;
1473                 if (picNumNoWrap >= MaxPicNum)
1474                     picNumNoWrap -= MaxPicNum;
1475             }
1476             picNumPred = picNumNoWrap;
1477
1478             // (8-36)
1479             picNum = picNumNoWrap;
1480             if (picNum > CurrPicNum)
1481                 picNum -= MaxPicNum;
1482
1483             // (8-37)
1484             for (j = num_refs; j > ref_list_idx; j--)
1485                 ref_list[j] = ref_list[j - 1];
1486             found_ref_idx = find_short_term_reference(decoder, picNum);
1487             ref_list[ref_list_idx++] =
1488                 found_ref_idx >= 0 ? priv->short_ref[found_ref_idx] : NULL;
1489             n = ref_list_idx;
1490             for (j = ref_list_idx; j <= num_refs; j++) {
1491                 gint32 PicNumF;
1492                 if (!ref_list[j])
1493                     continue;
1494                 PicNumF = ref_list[j]->is_long_term ?
1495                     MaxPicNum : ref_list[j]->pic_num;
1496                 if (PicNumF != picNum)
1497                     ref_list[n++] = ref_list[j];
1498             }
1499         }
1500
1501         /* 8.2.4.3.2 - Long-term reference pictures */
1502         else {
1503
1504             for (j = num_refs; j > ref_list_idx; j--)
1505                 ref_list[j] = ref_list[j - 1];
1506             found_ref_idx =
1507                 find_long_term_reference(decoder, l->value.long_term_pic_num);
1508             ref_list[ref_list_idx++] =
1509                 found_ref_idx >= 0 ? priv->long_ref[found_ref_idx] : NULL;
1510             n = ref_list_idx;
1511             for (j = ref_list_idx; j <= num_refs; j++) {
1512                 gint32 LongTermPicNumF;
1513                 if (!ref_list[j])
1514                     continue;
1515                 LongTermPicNumF = ref_list[j]->is_long_term ?
1516                     ref_list[j]->long_term_pic_num : INT_MAX;
1517                 if (LongTermPicNumF != l->value.long_term_pic_num)
1518                     ref_list[n++] = ref_list[j];
1519             }
1520         }
1521     }
1522
1523 #if DEBUG
1524     for (i = 0; i < num_refs; i++)
1525         if (!ref_list[i])
1526             GST_ERROR("list %u entry %u is empty", list, i);
1527 #endif
1528     *ref_list_count_ptr = num_refs;
1529 }
1530
1531 /* 8.2.4.3 - Modification process for reference picture lists */
1532 static void
1533 exec_picture_refs_modification(
1534     GstVaapiDecoderH264 *decoder,
1535     GstVaapiPictureH264 *picture,
1536     GstH264SliceHdr     *slice_hdr
1537 )
1538 {
1539     GST_DEBUG("execute ref_pic_list_modification()");
1540
1541     /* RefPicList0 */
1542     if (!GST_H264_IS_I_SLICE(slice_hdr) && !GST_H264_IS_SI_SLICE(slice_hdr) &&
1543         slice_hdr->ref_pic_list_modification_flag_l0)
1544         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 0);
1545
1546     /* RefPicList1 */
1547     if (GST_H264_IS_B_SLICE(slice_hdr) &&
1548         slice_hdr->ref_pic_list_modification_flag_l1)
1549         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 1);
1550 }
1551
1552 static gboolean
1553 init_picture_refs(
1554     GstVaapiDecoderH264 *decoder,
1555     GstVaapiPictureH264 *picture,
1556     GstH264SliceHdr     *slice_hdr
1557 )
1558 {
1559     GstVaapiDecoderH264Private * const priv = decoder->priv;
1560     GstVaapiPicture * const base_picture = &picture->base;
1561     guint i, num_refs;
1562
1563     init_picture_refs_pic_num(decoder, picture, slice_hdr);
1564
1565     priv->RefPicList0_count = 0;
1566     priv->RefPicList1_count = 0;
1567
1568     switch (base_picture->type) {
1569     case GST_VAAPI_PICTURE_TYPE_P:
1570     case GST_VAAPI_PICTURE_TYPE_SP:
1571         init_picture_refs_p_slice(decoder, picture, slice_hdr);
1572         break;
1573     case GST_VAAPI_PICTURE_TYPE_B:
1574         init_picture_refs_b_slice(decoder, picture, slice_hdr);
1575         break;
1576     default:
1577         break;
1578     }
1579
1580     exec_picture_refs_modification(decoder, picture, slice_hdr);
1581
1582     switch (base_picture->type) {
1583     case GST_VAAPI_PICTURE_TYPE_B:
1584         num_refs = 1 + slice_hdr->num_ref_idx_l1_active_minus1;
1585         for (i = priv->RefPicList1_count; i < num_refs; i++)
1586             priv->RefPicList1[i] = NULL;
1587         priv->RefPicList1_count = num_refs;
1588
1589         // fall-through
1590     case GST_VAAPI_PICTURE_TYPE_P:
1591     case GST_VAAPI_PICTURE_TYPE_SP:
1592         num_refs = 1 + slice_hdr->num_ref_idx_l0_active_minus1;
1593         for (i = priv->RefPicList0_count; i < num_refs; i++)
1594             priv->RefPicList0[i] = NULL;
1595         priv->RefPicList0_count = num_refs;
1596         break;
1597     default:
1598         break;
1599     }
1600     return TRUE;
1601 }
1602
1603 static gboolean
1604 init_picture(
1605     GstVaapiDecoderH264 *decoder,
1606     GstVaapiPictureH264 *picture,
1607     GstH264SliceHdr     *slice_hdr,
1608     GstH264NalUnit      *nalu
1609 )
1610 {
1611     GstVaapiDecoderH264Private * const priv = decoder->priv;
1612     GstVaapiPicture * const base_picture = &picture->base;
1613     VAPictureH264 *pic;
1614
1615     priv->prev_frame_num        = priv->frame_num;
1616     priv->frame_num             = slice_hdr->frame_num;
1617     picture->frame_num          = priv->frame_num;
1618     picture->frame_num_wrap     = priv->frame_num;
1619     picture->is_idr             = nalu->type == GST_H264_NAL_SLICE_IDR;
1620     picture->field_pic_flag     = slice_hdr->field_pic_flag;
1621     picture->bottom_field_flag  = slice_hdr->bottom_field_flag;
1622     picture->output_flag        = TRUE; /* XXX: conformant to Annex A only */
1623     base_picture->pts           = gst_adapter_prev_timestamp(priv->adapter, NULL);
1624
1625     /* Reset decoder state for IDR pictures */
1626     if (picture->is_idr) {
1627         GST_DEBUG("<IDR>");
1628         clear_references(decoder, priv->short_ref, &priv->short_ref_count);
1629         clear_references(decoder, priv->long_ref,  &priv->long_ref_count );
1630     }
1631
1632     /* Initialize VA picture info */
1633     pic = &picture->info;
1634     pic->picture_id = picture->base.surface_id;
1635     pic->frame_idx  = priv->frame_num;
1636     if (picture->field_pic_flag) {
1637         if (picture->bottom_field_flag)
1638             pic->flags |= VA_PICTURE_H264_BOTTOM_FIELD;
1639         else
1640             pic->flags |= VA_PICTURE_H264_TOP_FIELD;
1641     }
1642
1643     /* Initialize base picture */
1644     switch (slice_hdr->type % 5) {
1645     case GST_H264_P_SLICE:
1646         base_picture->type = GST_VAAPI_PICTURE_TYPE_P;
1647         break;
1648     case GST_H264_B_SLICE:
1649         base_picture->type = GST_VAAPI_PICTURE_TYPE_B;
1650         break;
1651     case GST_H264_I_SLICE:
1652         base_picture->type = GST_VAAPI_PICTURE_TYPE_I;
1653         break;
1654     case GST_H264_SP_SLICE:
1655         base_picture->type = GST_VAAPI_PICTURE_TYPE_SP;
1656         break;
1657     case GST_H264_SI_SLICE:
1658         base_picture->type = GST_VAAPI_PICTURE_TYPE_SI;
1659         break;
1660     }
1661
1662     if (nalu->ref_idc) {
1663         GstH264DecRefPicMarking * const dec_ref_pic_marking =
1664             &slice_hdr->dec_ref_pic_marking;
1665         GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_REFERENCE);
1666         if (picture->is_idr) {
1667             if (dec_ref_pic_marking->long_term_reference_flag)
1668                 picture->is_long_term = TRUE;
1669         }
1670     }
1671
1672     init_picture_poc(decoder, picture, slice_hdr);
1673     if (!init_picture_refs(decoder, picture, slice_hdr)) {
1674         GST_ERROR("failed to initialize references");
1675         return FALSE;
1676     }
1677     return TRUE;
1678 }
1679
1680 /* 8.2.5.3 - Sliding window decoded reference picture marking process */
1681 static gboolean
1682 exec_ref_pic_marking_sliding_window(GstVaapiDecoderH264 *decoder)
1683 {
1684     GstVaapiDecoderH264Private * const priv = decoder->priv;
1685     GstH264SPS * const sps = priv->sps;
1686     guint i, max_num_ref_frames, lowest_frame_num_index;
1687     gint32 lowest_frame_num;
1688
1689     GST_DEBUG("reference picture marking process (sliding window)");
1690
1691     max_num_ref_frames = sps->num_ref_frames;
1692     if (max_num_ref_frames == 0)
1693         max_num_ref_frames = 1;
1694
1695     if (priv->short_ref_count + priv->long_ref_count < max_num_ref_frames)
1696         return TRUE;
1697     if (priv->short_ref_count < 1)
1698         return FALSE;
1699
1700     lowest_frame_num = priv->short_ref[0]->frame_num_wrap;
1701     lowest_frame_num_index = 0;
1702     for (i = 1; i < priv->short_ref_count; i++) {
1703         if (priv->short_ref[i]->frame_num_wrap < lowest_frame_num) {
1704             lowest_frame_num = priv->short_ref[i]->frame_num_wrap;
1705             lowest_frame_num_index = i;
1706         }
1707     }
1708
1709     remove_reference_at(
1710         decoder,
1711         priv->short_ref, &priv->short_ref_count,
1712         lowest_frame_num_index
1713     );
1714     return TRUE;
1715 }
1716
1717 static inline gint32
1718 get_picNumX(GstVaapiPictureH264 *picture, GstH264RefPicMarking *ref_pic_marking)
1719 {
1720     gint32 pic_num;
1721
1722     if (!picture->field_pic_flag)
1723         pic_num = picture->frame_num_wrap;
1724     else
1725         pic_num = 2 * picture->frame_num_wrap + 1;
1726     pic_num -= ref_pic_marking->difference_of_pic_nums_minus1 + 1;
1727     return pic_num;
1728 }
1729
1730 /* 8.2.5.4.1. Mark-term reference picture as "unused for reference" */
1731 static void
1732 exec_ref_pic_marking_adaptive_mmco_1(
1733     GstVaapiDecoderH264  *decoder,
1734     GstVaapiPictureH264  *picture,
1735     GstH264RefPicMarking *ref_pic_marking
1736 )
1737 {
1738     GstVaapiDecoderH264Private * const priv = decoder->priv;
1739     gint32 i, picNumX;
1740
1741     picNumX = get_picNumX(picture, ref_pic_marking);
1742     i = find_short_term_reference(decoder, picNumX);
1743     if (i < 0)
1744         return;
1745     remove_reference_at(decoder, priv->short_ref, &priv->short_ref_count, i);
1746 }
1747
1748 /* 8.2.5.4.2. Mark long-term reference picture as "unused for reference" */
1749 static void
1750 exec_ref_pic_marking_adaptive_mmco_2(
1751     GstVaapiDecoderH264  *decoder,
1752     GstVaapiPictureH264  *picture,
1753     GstH264RefPicMarking *ref_pic_marking
1754 )
1755 {
1756     GstVaapiDecoderH264Private * const priv = decoder->priv;
1757     gint32 i;
1758
1759     i = find_long_term_reference(decoder, ref_pic_marking->long_term_pic_num);
1760     if (i < 0)
1761         return;
1762     remove_reference_at(decoder, priv->long_ref, &priv->long_ref_count, i);
1763 }
1764
1765 /* 8.2.5.4.3. Assign LongTermFrameIdx to a short-term reference picture */
1766 static void
1767 exec_ref_pic_marking_adaptive_mmco_3(
1768     GstVaapiDecoderH264  *decoder,
1769     GstVaapiPictureH264  *picture,
1770     GstH264RefPicMarking *ref_pic_marking
1771 )
1772 {
1773     GstVaapiDecoderH264Private * const priv = decoder->priv;
1774     VAPictureH264 *pic;
1775     gint32 i, picNumX;
1776
1777     for (i = 0; i < priv->long_ref_count; i++) {
1778         if (priv->long_ref[i]->info.frame_idx == ref_pic_marking->long_term_frame_idx)
1779             break;
1780     }
1781     if (i != priv->long_ref_count)
1782         remove_reference_at(decoder, priv->long_ref, &priv->long_ref_count, i);
1783
1784     picNumX = get_picNumX(picture, ref_pic_marking);
1785     i = find_short_term_reference(decoder, picNumX);
1786     if (i < 0)
1787         return;
1788
1789     picture = gst_vaapi_picture_ref(priv->short_ref[i]);
1790     remove_reference_at(decoder, priv->short_ref, &priv->short_ref_count, i);
1791     gst_vaapi_picture_replace(&priv->long_ref[priv->long_ref_count++], picture);
1792     gst_vaapi_picture_unref(picture);
1793
1794     picture->is_long_term = TRUE;
1795     pic = &picture->info;
1796     pic->frame_idx = ref_pic_marking->long_term_frame_idx;
1797     pic->flags &= ~VA_PICTURE_H264_SHORT_TERM_REFERENCE;
1798     pic->flags |= VA_PICTURE_H264_LONG_TERM_REFERENCE;
1799     GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_REFERENCE);
1800 }
1801
1802 /* 8.2.5.4.4. Mark pictures with LongTermFramIdx > max_long_term_frame_idx
1803  * as "unused for reference" */
1804 static void
1805 exec_ref_pic_marking_adaptive_mmco_4(
1806     GstVaapiDecoderH264  *decoder,
1807     GstVaapiPictureH264  *picture,
1808     GstH264RefPicMarking *ref_pic_marking
1809 )
1810 {
1811     GstVaapiDecoderH264Private * const priv = decoder->priv;
1812     gint32 i, long_term_frame_idx;
1813
1814     long_term_frame_idx = ref_pic_marking->max_long_term_frame_idx_plus1 - 1;
1815
1816     for (i = 0; i < priv->long_ref_count; i++) {
1817         if ((gint32)priv->long_ref[i]->info.frame_idx <= long_term_frame_idx)
1818             continue;
1819         remove_reference_at(decoder, priv->long_ref, &priv->long_ref_count, i);
1820         i--;
1821     }
1822 }
1823
1824 /* 8.2.5.4.5. Mark all reference pictures as "unused for reference" */
1825 static void
1826 exec_ref_pic_marking_adaptive_mmco_5(
1827     GstVaapiDecoderH264  *decoder,
1828     GstVaapiPictureH264  *picture,
1829     GstH264RefPicMarking *ref_pic_marking
1830 )
1831 {
1832     GstVaapiDecoderH264Private * const priv = decoder->priv;
1833     VAPictureH264 * const pic = &picture->info;
1834
1835     clear_references(decoder, priv->short_ref, &priv->short_ref_count);
1836     clear_references(decoder, priv->long_ref,  &priv->long_ref_count );
1837     dpb_flush(decoder);
1838
1839     priv->prev_pic_has_mmco5 = TRUE;
1840
1841     /* The picture shall be inferred to have had frame_num equal to 0 (7.4.3) */
1842     priv->frame_num = 0;
1843     priv->frame_num_offset = 0;
1844     picture->frame_num = 0;
1845
1846     /* Update TopFieldOrderCnt and BottomFieldOrderCnt (8.2.1) */
1847     if (!(pic->flags & VA_PICTURE_H264_BOTTOM_FIELD))
1848         pic->TopFieldOrderCnt -= picture->poc;
1849     if (!(pic->flags & VA_PICTURE_H264_TOP_FIELD))
1850         pic->BottomFieldOrderCnt -= picture->poc;
1851     picture->poc = 0;
1852 }
1853
1854 /* 8.2.5.4.6. Assign a long-term frame index to the current picture */
1855 static void
1856 exec_ref_pic_marking_adaptive_mmco_6(
1857     GstVaapiDecoderH264  *decoder,
1858     GstVaapiPictureH264  *picture,
1859     GstH264RefPicMarking *ref_pic_marking
1860 )
1861 {
1862     picture->is_long_term = TRUE;
1863     picture->info.frame_idx = ref_pic_marking->long_term_frame_idx;
1864 }
1865
1866 /* 8.2.5.4. Adaptive memory control decoded reference picture marking process */
1867 static gboolean
1868 exec_ref_pic_marking_adaptive(
1869     GstVaapiDecoderH264     *decoder,
1870     GstVaapiPictureH264     *picture,
1871     GstH264DecRefPicMarking *dec_ref_pic_marking
1872 )
1873 {
1874     guint i;
1875
1876     GST_DEBUG("reference picture marking process (adaptive memory control)");
1877
1878     typedef void (*exec_ref_pic_marking_adaptive_mmco_func)(
1879         GstVaapiDecoderH264  *decoder,
1880         GstVaapiPictureH264  *picture,
1881         GstH264RefPicMarking *ref_pic_marking
1882     );
1883
1884     static const exec_ref_pic_marking_adaptive_mmco_func mmco_funcs[] = {
1885         NULL,
1886         exec_ref_pic_marking_adaptive_mmco_1,
1887         exec_ref_pic_marking_adaptive_mmco_2,
1888         exec_ref_pic_marking_adaptive_mmco_3,
1889         exec_ref_pic_marking_adaptive_mmco_4,
1890         exec_ref_pic_marking_adaptive_mmco_5,
1891         exec_ref_pic_marking_adaptive_mmco_6,
1892     };
1893
1894     for (i = 0; i < dec_ref_pic_marking->n_ref_pic_marking; i++) {
1895         GstH264RefPicMarking * const ref_pic_marking =
1896             &dec_ref_pic_marking->ref_pic_marking[i];
1897
1898         const guint mmco = ref_pic_marking->memory_management_control_operation;
1899         if (mmco < G_N_ELEMENTS(mmco_funcs) && mmco_funcs[mmco])
1900             mmco_funcs[mmco](decoder, picture, ref_pic_marking);
1901         else {
1902             GST_ERROR("unhandled MMCO %u", mmco);
1903             return FALSE;
1904         }
1905     }
1906     return TRUE;
1907 }
1908
1909 /* 8.2.5 - Execute reference picture marking process */
1910 static gboolean
1911 exec_ref_pic_marking(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
1912 {
1913     GstVaapiDecoderH264Private * const priv = decoder->priv;
1914     GstVaapiPictureH264 **picture_ptr;
1915
1916     priv->prev_pic_has_mmco5 = FALSE;
1917     priv->prev_pic_bottom_field =
1918         picture->field_pic_flag && picture->bottom_field_flag;
1919
1920     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
1921         return TRUE;
1922
1923     if (!picture->is_idr) {
1924         GstVaapiSliceH264 * const slice =
1925             gst_vaapi_picture_h264_get_last_slice(picture);
1926         GstH264DecRefPicMarking * const dec_ref_pic_marking =
1927             &slice->slice_hdr.dec_ref_pic_marking;
1928         if (dec_ref_pic_marking->adaptive_ref_pic_marking_mode_flag) {
1929             if (!exec_ref_pic_marking_adaptive(decoder, picture, dec_ref_pic_marking))
1930                 return FALSE;
1931         }
1932         else {
1933             if (!exec_ref_pic_marking_sliding_window(decoder))
1934                 return FALSE;
1935         }
1936     }
1937
1938     if (picture->is_long_term) {
1939         picture_ptr = &priv->long_ref[priv->long_ref_count++];
1940         picture->info.flags |= VA_PICTURE_H264_LONG_TERM_REFERENCE;
1941     }
1942     else {
1943         picture_ptr = &priv->short_ref[priv->short_ref_count++];
1944         picture->info.flags |= VA_PICTURE_H264_SHORT_TERM_REFERENCE;
1945     }
1946     gst_vaapi_picture_replace(picture_ptr, picture);
1947     return TRUE;
1948 }
1949
1950 static void
1951 vaapi_init_picture(VAPictureH264 *pic)
1952 {
1953     pic->picture_id           = VA_INVALID_ID;
1954     pic->frame_idx            = 0;
1955     pic->flags                = VA_PICTURE_H264_INVALID;
1956     pic->TopFieldOrderCnt     = 0;
1957     pic->BottomFieldOrderCnt  = 0;
1958 }
1959
1960 static gboolean
1961 fill_picture(
1962     GstVaapiDecoderH264 *decoder,
1963     GstVaapiPictureH264 *picture,
1964     GstH264SliceHdr     *slice_hdr,
1965     GstH264NalUnit      *nalu
1966 )
1967 {
1968     GstVaapiDecoderH264Private * const priv = decoder->priv;
1969     GstVaapiPicture * const base_picture = &picture->base;
1970     GstH264SPS * const sps = priv->sps;
1971     GstH264PPS * const pps = priv->pps;
1972     VAPictureParameterBufferH264 * const pic_param = base_picture->param;
1973     guint i, n;
1974
1975     /* Fill in VAPictureParameterBufferH264 */
1976     pic_param->CurrPic = picture->info;
1977     for (i = 0, n = 0; i < priv->short_ref_count; i++)
1978         pic_param->ReferenceFrames[n++] = priv->short_ref[i]->info;
1979     for (i = 0; i < priv->long_ref_count; i++)
1980         pic_param->ReferenceFrames[n++] = priv->long_ref[i]->info;
1981     for (; n < G_N_ELEMENTS(pic_param->ReferenceFrames); n++)
1982         vaapi_init_picture(&pic_param->ReferenceFrames[n]);
1983
1984 #define COPY_FIELD(s, f) \
1985     pic_param->f = (s)->f
1986
1987 #define COPY_BFM(a, s, f) \
1988     pic_param->a.bits.f = (s)->f
1989
1990     pic_param->picture_width_in_mbs_minus1      = priv->mb_width  - 1;
1991     pic_param->picture_height_in_mbs_minus1     = priv->mb_height - 1;
1992     pic_param->frame_num                        = priv->frame_num;
1993
1994     COPY_FIELD(sps, bit_depth_luma_minus8);
1995     COPY_FIELD(sps, bit_depth_chroma_minus8);
1996     COPY_FIELD(sps, num_ref_frames);
1997     COPY_FIELD(pps, num_slice_groups_minus1);
1998     COPY_FIELD(pps, slice_group_map_type);
1999     COPY_FIELD(pps, slice_group_change_rate_minus1);
2000     COPY_FIELD(pps, pic_init_qp_minus26);
2001     COPY_FIELD(pps, pic_init_qs_minus26);
2002     COPY_FIELD(pps, chroma_qp_index_offset);
2003     COPY_FIELD(pps, second_chroma_qp_index_offset);
2004
2005     pic_param->seq_fields.value                                         = 0; /* reset all bits */
2006     pic_param->seq_fields.bits.residual_colour_transform_flag           = sps->separate_colour_plane_flag;
2007     pic_param->seq_fields.bits.MinLumaBiPredSize8x8                     = sps->level_idc >= 31; /* A.3.3.2 */
2008
2009     COPY_BFM(seq_fields, sps, chroma_format_idc);
2010     COPY_BFM(seq_fields, sps, gaps_in_frame_num_value_allowed_flag);
2011     COPY_BFM(seq_fields, sps, frame_mbs_only_flag);
2012     COPY_BFM(seq_fields, sps, mb_adaptive_frame_field_flag);
2013     COPY_BFM(seq_fields, sps, direct_8x8_inference_flag);
2014     COPY_BFM(seq_fields, sps, log2_max_frame_num_minus4);
2015     COPY_BFM(seq_fields, sps, pic_order_cnt_type);
2016     COPY_BFM(seq_fields, sps, log2_max_pic_order_cnt_lsb_minus4);
2017     COPY_BFM(seq_fields, sps, delta_pic_order_always_zero_flag);
2018
2019     pic_param->pic_fields.value                                         = 0; /* reset all bits */
2020     pic_param->pic_fields.bits.field_pic_flag                           = slice_hdr->field_pic_flag;
2021     pic_param->pic_fields.bits.reference_pic_flag                       = GST_VAAPI_PICTURE_IS_REFERENCE(picture);
2022
2023     COPY_BFM(pic_fields, pps, entropy_coding_mode_flag);
2024     COPY_BFM(pic_fields, pps, weighted_pred_flag);
2025     COPY_BFM(pic_fields, pps, weighted_bipred_idc);
2026     COPY_BFM(pic_fields, pps, transform_8x8_mode_flag);
2027     COPY_BFM(pic_fields, pps, constrained_intra_pred_flag);
2028     COPY_BFM(pic_fields, pps, pic_order_present_flag);
2029     COPY_BFM(pic_fields, pps, deblocking_filter_control_present_flag);
2030     COPY_BFM(pic_fields, pps, redundant_pic_cnt_present_flag);
2031     return TRUE;
2032 }
2033
2034 /* Detection of the first VCL NAL unit of a primary coded picture (7.4.1.2.4) */
2035 static gboolean
2036 is_new_picture(
2037     GstVaapiDecoderH264 *decoder,
2038     GstH264NalUnit      *nalu,
2039     GstH264SliceHdr     *slice_hdr
2040 )
2041 {
2042     GstVaapiDecoderH264Private * const priv = decoder->priv;
2043     GstH264PPS * const pps = slice_hdr->pps;
2044     GstH264SPS * const sps = pps->sequence;
2045     GstVaapiSliceH264 *slice;
2046     GstH264SliceHdr *prev_slice_hdr;
2047
2048     if (!priv->current_picture)
2049         return TRUE;
2050
2051     slice = gst_vaapi_picture_h264_get_last_slice(priv->current_picture);
2052     if (!slice)
2053         return FALSE;
2054     prev_slice_hdr = &slice->slice_hdr;
2055
2056 #define CHECK_EXPR(expr, field_name) do {              \
2057         if (!(expr)) {                                 \
2058             GST_DEBUG(field_name " differs in value"); \
2059             return TRUE;                               \
2060         }                                              \
2061     } while (0)
2062
2063 #define CHECK_VALUE(new_slice_hdr, old_slice_hdr, field) \
2064     CHECK_EXPR(((new_slice_hdr)->field == (old_slice_hdr)->field), #field)
2065
2066     /* frame_num differs in value, regardless of inferred values to 0 */
2067     CHECK_VALUE(slice_hdr, prev_slice_hdr, frame_num);
2068
2069     /* pic_parameter_set_id differs in value */
2070     CHECK_VALUE(slice_hdr, prev_slice_hdr, pps);
2071
2072     /* field_pic_flag differs in value */
2073     CHECK_VALUE(slice_hdr, prev_slice_hdr, field_pic_flag);
2074
2075     /* bottom_field_flag is present in both and differs in value */
2076     if (slice_hdr->field_pic_flag && prev_slice_hdr->field_pic_flag)
2077         CHECK_VALUE(slice_hdr, prev_slice_hdr, bottom_field_flag);
2078
2079     /* nal_ref_idc differs in value with one of the nal_ref_idc values is 0 */
2080     CHECK_EXPR(((GST_VAAPI_PICTURE_IS_REFERENCE(priv->current_picture) ^
2081                  (nalu->ref_idc != 0)) == 0), "nal_ref_idc");
2082
2083     /* POC type is 0 for both and either pic_order_cnt_lsb differs in
2084        value or delta_pic_order_cnt_bottom differs in value */
2085     if (sps->pic_order_cnt_type == 0) {
2086         CHECK_VALUE(slice_hdr, prev_slice_hdr, pic_order_cnt_lsb);
2087         if (pps->pic_order_present_flag && !slice_hdr->field_pic_flag)
2088             CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt_bottom);
2089     }
2090
2091     /* POC type is 1 for both and either delta_pic_order_cnt[0]
2092        differs in value or delta_pic_order_cnt[1] differs in value */
2093     else if (sps->pic_order_cnt_type == 1) {
2094         CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt[0]);
2095         CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt[1]);
2096     }
2097
2098     /* IdrPicFlag differs in value */
2099     CHECK_EXPR(((priv->current_picture->is_idr ^
2100                  (nalu->type == GST_H264_NAL_SLICE_IDR)) == 0), "IdrPicFlag");
2101
2102     /* IdrPicFlag is equal to 1 for both and idr_pic_id differs in value */
2103     if (priv->current_picture->is_idr)
2104         CHECK_VALUE(slice_hdr, prev_slice_hdr, idr_pic_id);
2105
2106 #undef CHECK_EXPR
2107 #undef CHECK_VALUE
2108     return FALSE;
2109 }
2110
2111 static GstVaapiDecoderStatus
2112 decode_picture(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu, GstH264SliceHdr *slice_hdr)
2113 {
2114     GstVaapiDecoderH264Private * const priv = decoder->priv;
2115     GstVaapiPictureH264 *picture;
2116     GstVaapiDecoderStatus status;
2117     GstH264PPS * const pps = slice_hdr->pps;
2118     GstH264SPS * const sps = pps->sequence;
2119
2120     status = ensure_context(decoder, sps);
2121     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS) {
2122         GST_ERROR("failed to reset context");
2123         return status;
2124     }
2125
2126     if (priv->current_picture && !decode_current_picture(decoder))
2127         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2128
2129     picture = gst_vaapi_picture_h264_new(decoder);
2130     if (!picture) {
2131         GST_ERROR("failed to allocate picture");
2132         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2133     }
2134     priv->current_picture = picture;
2135
2136     priv->sps = sps;
2137     priv->pps = pps;
2138
2139     status = ensure_quant_matrix(decoder, picture);
2140     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS) {
2141         GST_ERROR("failed to reset quantizer matrix");
2142         return status;
2143     }
2144
2145     if (!init_picture(decoder, picture, slice_hdr, nalu))
2146         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2147     if (!fill_picture(decoder, picture, slice_hdr, nalu))
2148         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2149     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2150 }
2151
2152 static gboolean
2153 decode_picture_end(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
2154 {
2155     if (!exec_ref_pic_marking(decoder, picture))
2156         return FALSE;
2157     if (!dpb_add(decoder, picture))
2158         return FALSE;
2159     return TRUE;
2160 }
2161
2162 static inline guint
2163 get_slice_data_bit_offset(GstH264SliceHdr *slice_hdr, GstH264NalUnit *nalu)
2164 {
2165     guint epb_count;
2166
2167     epb_count = slice_hdr->n_emulation_prevention_bytes;
2168     return 8 /* nal_unit_type */ + slice_hdr->header_size - epb_count * 8;
2169 }
2170
2171 static gboolean
2172 fill_pred_weight_table(GstVaapiDecoderH264 *decoder, GstVaapiSliceH264 *slice)
2173 {
2174     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2175     GstH264PPS * const pps = slice_hdr->pps;
2176     GstH264SPS * const sps = pps->sequence;
2177     GstH264PredWeightTable * const w = &slice_hdr->pred_weight_table;
2178     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2179     guint num_weight_tables = 0;
2180     gint i, j;
2181
2182     if (pps->weighted_pred_flag &&
2183         (GST_H264_IS_P_SLICE(slice_hdr) || GST_H264_IS_SP_SLICE(slice_hdr)))
2184         num_weight_tables = 1;
2185     else if (pps->weighted_bipred_idc == 1 && GST_H264_IS_B_SLICE(slice_hdr))
2186         num_weight_tables = 2;
2187     else
2188         num_weight_tables = 0;
2189
2190     slice_param->luma_log2_weight_denom   = w->luma_log2_weight_denom;
2191     slice_param->chroma_log2_weight_denom = w->chroma_log2_weight_denom;
2192     slice_param->luma_weight_l0_flag      = 0;
2193     slice_param->chroma_weight_l0_flag    = 0;
2194     slice_param->luma_weight_l1_flag      = 0;
2195     slice_param->chroma_weight_l1_flag    = 0;
2196
2197     if (num_weight_tables < 1)
2198         return TRUE;
2199
2200     slice_param->luma_weight_l0_flag = 1;
2201     for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2202         slice_param->luma_weight_l0[i] = w->luma_weight_l0[i];
2203         slice_param->luma_offset_l0[i] = w->luma_offset_l0[i];
2204     }
2205
2206     slice_param->chroma_weight_l0_flag = sps->chroma_array_type != 0;
2207     if (slice_param->chroma_weight_l0_flag) {
2208         for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2209             for (j = 0; j < 2; j++) {
2210                 slice_param->chroma_weight_l0[i][j] = w->chroma_weight_l0[i][j];
2211                 slice_param->chroma_offset_l0[i][j] = w->chroma_offset_l0[i][j];
2212             }
2213         }
2214     }
2215
2216     if (num_weight_tables < 2)
2217         return TRUE;
2218
2219     slice_param->luma_weight_l1_flag = 1;
2220     for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2221         slice_param->luma_weight_l1[i] = w->luma_weight_l1[i];
2222         slice_param->luma_offset_l1[i] = w->luma_offset_l1[i];
2223     }
2224
2225     slice_param->chroma_weight_l1_flag = sps->chroma_array_type != 0;
2226     if (slice_param->chroma_weight_l1_flag) {
2227         for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2228             for (j = 0; j < 2; j++) {
2229                 slice_param->chroma_weight_l1[i][j] = w->chroma_weight_l1[i][j];
2230                 slice_param->chroma_offset_l1[i][j] = w->chroma_offset_l1[i][j];
2231             }
2232         }
2233     }
2234     return TRUE;
2235 }
2236
2237 static gboolean
2238 fill_RefPicList(GstVaapiDecoderH264 *decoder, GstVaapiSliceH264 *slice)
2239 {
2240     GstVaapiDecoderH264Private * const priv = decoder->priv;
2241     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2242     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2243     guint i, num_ref_lists = 0;
2244
2245     slice_param->num_ref_idx_l0_active_minus1 = 0;
2246     slice_param->num_ref_idx_l1_active_minus1 = 0;
2247
2248     if (GST_H264_IS_B_SLICE(slice_hdr))
2249         num_ref_lists = 2;
2250     else if (GST_H264_IS_I_SLICE(slice_hdr))
2251         num_ref_lists = 0;
2252     else
2253         num_ref_lists = 1;
2254
2255     if (num_ref_lists < 1)
2256         return TRUE;
2257
2258     slice_param->num_ref_idx_l0_active_minus1 =
2259         slice_hdr->num_ref_idx_l0_active_minus1;
2260
2261     for (i = 0; i < priv->RefPicList0_count && priv->RefPicList0[i]; i++)
2262         slice_param->RefPicList0[i] = priv->RefPicList0[i]->info;
2263     for (; i <= slice_param->num_ref_idx_l0_active_minus1; i++)
2264         vaapi_init_picture(&slice_param->RefPicList0[i]);
2265
2266     if (num_ref_lists < 2)
2267         return TRUE;
2268
2269     slice_param->num_ref_idx_l1_active_minus1 =
2270         slice_hdr->num_ref_idx_l1_active_minus1;
2271
2272     for (i = 0; i < priv->RefPicList1_count && priv->RefPicList1[i]; i++)
2273         slice_param->RefPicList1[i] = priv->RefPicList1[i]->info;
2274     for (; i <= slice_param->num_ref_idx_l1_active_minus1; i++)
2275         vaapi_init_picture(&slice_param->RefPicList1[i]);
2276     return TRUE;
2277 }
2278
2279 static gboolean
2280 fill_slice(
2281     GstVaapiDecoderH264 *decoder,
2282     GstVaapiSliceH264   *slice,
2283     GstH264NalUnit      *nalu
2284 )
2285 {
2286     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2287     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2288
2289     /* Fill in VASliceParameterBufferH264 */
2290     slice_param->slice_data_bit_offset          = get_slice_data_bit_offset(slice_hdr, nalu);
2291     slice_param->first_mb_in_slice              = slice_hdr->first_mb_in_slice;
2292     slice_param->slice_type                     = slice_hdr->type % 5;
2293     slice_param->direct_spatial_mv_pred_flag    = slice_hdr->direct_spatial_mv_pred_flag;
2294     slice_param->cabac_init_idc                 = slice_hdr->cabac_init_idc;
2295     slice_param->slice_qp_delta                 = slice_hdr->slice_qp_delta;
2296     slice_param->disable_deblocking_filter_idc  = slice_hdr->disable_deblocking_filter_idc;
2297     slice_param->slice_alpha_c0_offset_div2     = slice_hdr->slice_alpha_c0_offset_div2;
2298     slice_param->slice_beta_offset_div2         = slice_hdr->slice_beta_offset_div2;
2299
2300     if (!fill_RefPicList(decoder, slice))
2301         return FALSE;
2302     if (!fill_pred_weight_table(decoder, slice))
2303         return FALSE;
2304     return TRUE;
2305 }
2306
2307 static GstVaapiDecoderStatus
2308 decode_slice(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
2309 {
2310     GstVaapiDecoderH264Private * const priv = decoder->priv;
2311     GstVaapiDecoderStatus status;
2312     GstVaapiPictureH264 *picture;
2313     GstVaapiSliceH264 *slice = NULL;
2314     GstH264SliceHdr *slice_hdr;
2315     GstH264ParserResult result;
2316
2317     GST_DEBUG("slice (%u bytes)", nalu->size);
2318
2319     slice = gst_vaapi_slice_h264_new(
2320         decoder,
2321         nalu->data + nalu->offset,
2322         nalu->size
2323     );
2324     if (!slice) {
2325         GST_ERROR("failed to allocate slice");
2326         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2327     }
2328
2329     slice_hdr = &slice->slice_hdr;
2330     memset(slice_hdr, 0, sizeof(*slice_hdr));
2331     result = gst_h264_parser_parse_slice_hdr(priv->parser, nalu, slice_hdr, TRUE, TRUE);
2332     if (result != GST_H264_PARSER_OK) {
2333         status = get_status(result);
2334         goto error;
2335     }
2336
2337     if (is_new_picture(decoder, nalu, slice_hdr)) {
2338         status = decode_picture(decoder, nalu, slice_hdr);
2339         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2340             goto error;
2341     }
2342     picture = priv->current_picture;
2343
2344     priv->mb_x = slice_hdr->first_mb_in_slice % priv->mb_width;
2345     priv->mb_y = slice_hdr->first_mb_in_slice / priv->mb_width; // FIXME: MBAFF or field
2346
2347     if (!fill_slice(decoder, slice, nalu)) {
2348         status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2349         goto error;
2350     }
2351     gst_vaapi_picture_add_slice(
2352         GST_VAAPI_PICTURE_CAST(picture),
2353         GST_VAAPI_SLICE_CAST(slice)
2354     );
2355
2356     /* Commit picture for decoding if we reached the last slice */
2357     if (++priv->mb_y >= priv->mb_height) {
2358         if (!decode_current_picture(decoder)) {
2359             status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2360             goto error;
2361         }
2362         GST_DEBUG("done");
2363     }
2364     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2365
2366 error:
2367     if (slice)
2368         gst_mini_object_unref(GST_MINI_OBJECT(slice));
2369     return status;
2370 }
2371
2372 static inline gint
2373 scan_for_start_code(GstAdapter *adapter, guint ofs, guint size, guint32 *scp)
2374 {
2375     return (gint)gst_adapter_masked_scan_uint32_peek(adapter,
2376                                                      0xffffff00, 0x00000100,
2377                                                      ofs, size,
2378                                                      scp);
2379 }
2380
2381 static GstVaapiDecoderStatus
2382 decode_nalu(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
2383 {
2384     GstVaapiDecoderStatus status;
2385
2386     switch (nalu->type) {
2387     case GST_H264_NAL_SLICE_IDR:
2388         /* fall-through. IDR specifics are handled in init_picture() */
2389     case GST_H264_NAL_SLICE:
2390         status = decode_slice(decoder, nalu);
2391         break;
2392     case GST_H264_NAL_SPS:
2393         status = decode_sps(decoder, nalu);
2394         break;
2395     case GST_H264_NAL_PPS:
2396         status = decode_pps(decoder, nalu);
2397         break;
2398     case GST_H264_NAL_SEI:
2399         status = decode_sei(decoder, nalu);
2400         break;
2401     case GST_H264_NAL_SEQ_END:
2402         status = decode_sequence_end(decoder);
2403         break;
2404     case GST_H264_NAL_AU_DELIMITER:
2405         /* skip all Access Unit NALs */
2406         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2407         break;
2408     case GST_H264_NAL_FILLER_DATA:
2409         /* skip all Filler Data NALs */
2410         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2411         break;
2412     default:
2413         GST_WARNING("unsupported NAL unit type %d", nalu->type);
2414         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
2415         break;
2416     }
2417     return status;
2418 }
2419
2420 static GstVaapiDecoderStatus
2421 decode_buffer(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
2422 {
2423     GstVaapiDecoderH264Private * const priv = decoder->priv;
2424     GstVaapiDecoderStatus status;
2425     GstH264ParserResult result;
2426     GstH264NalUnit nalu;
2427     gboolean is_eos;
2428     const guchar *buf;
2429     guint i, buf_size, nalu_size, size;
2430     guint32 start_code;
2431     gint ofs;
2432
2433     buf      = GST_BUFFER_DATA(buffer);
2434     buf_size = GST_BUFFER_SIZE(buffer);
2435     is_eos   = GST_BUFFER_IS_EOS(buffer);
2436     if (buf && buf_size > 0)
2437         gst_adapter_push(priv->adapter, gst_buffer_ref(buffer));
2438
2439     size = gst_adapter_available(priv->adapter);
2440     do {
2441         if (size == 0) {
2442             status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2443             break;
2444         }
2445
2446         status = gst_vaapi_decoder_check_status(GST_VAAPI_DECODER(decoder));
2447         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2448             break;
2449
2450         status = GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2451         if (priv->is_avc) {
2452             if (size < priv->nal_length_size)
2453                 break;
2454             buf = gst_adapter_peek(priv->adapter, priv->nal_length_size);
2455
2456             nalu_size = 0;
2457             for (i = 0; i < priv->nal_length_size; i++)
2458                 nalu_size = (nalu_size << 8) | buf[i];
2459
2460             buf_size = priv->nal_length_size + nalu_size;
2461             if (size < buf_size)
2462                 break;
2463             buffer = gst_adapter_take_buffer(priv->adapter, buf_size);
2464             size -= buf_size;
2465
2466             buf      = GST_BUFFER_DATA(buffer);
2467             buf_size = GST_BUFFER_SIZE(buffer);
2468
2469             result = gst_h264_parser_identify_nalu_avc(
2470                 priv->parser,
2471                 buf, 0, buf_size, priv->nal_length_size,
2472                 &nalu
2473             );
2474         }
2475         else {
2476             if (size < 4)
2477                 break;
2478             ofs = scan_for_start_code(priv->adapter, 0, size, &start_code);
2479             if (ofs < 0)
2480                 break;
2481             gst_adapter_flush(priv->adapter, ofs);
2482             size -= ofs;
2483
2484             ofs = G_UNLIKELY(size < 8) ? -1 :
2485                 scan_for_start_code(priv->adapter, 4, size - 4, NULL);
2486             if (ofs < 0) {
2487                 // Assume the whole NAL unit is present if end-of-stream
2488                 if (!is_eos)
2489                     break;
2490                 ofs = size;
2491             }
2492             buffer = gst_adapter_take_buffer(priv->adapter, ofs);
2493             size -= ofs;
2494
2495             buf      = GST_BUFFER_DATA(buffer);
2496             buf_size = GST_BUFFER_SIZE(buffer);
2497
2498             result = gst_h264_parser_identify_nalu_unchecked(
2499                 priv->parser,
2500                 buf, 0, buf_size,
2501                 &nalu
2502             );
2503         }
2504         status = get_status(result);
2505         if (status == GST_VAAPI_DECODER_STATUS_SUCCESS)
2506             status = decode_nalu(decoder, &nalu);
2507         gst_buffer_unref(buffer);
2508     } while (status == GST_VAAPI_DECODER_STATUS_SUCCESS);
2509
2510     if (is_eos && (status == GST_VAAPI_DECODER_STATUS_SUCCESS ||
2511                    status == GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA))
2512         status = decode_sequence_end(decoder);
2513     return status;
2514 }
2515
2516 static GstVaapiDecoderStatus
2517 decode_codec_data(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
2518 {
2519     GstVaapiDecoderH264Private * const priv = decoder->priv;
2520     GstVaapiDecoderStatus status;
2521     GstH264NalUnit nalu;
2522     GstH264ParserResult result;
2523     guchar *buf;
2524     guint buf_size;
2525     guint i, ofs, num_sps, num_pps;
2526
2527     buf      = GST_BUFFER_DATA(buffer);
2528     buf_size = GST_BUFFER_SIZE(buffer);
2529     if (!buf || buf_size == 0)
2530         return GST_VAAPI_DECODER_STATUS_SUCCESS;
2531
2532     if (buf_size < 8)
2533         return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2534
2535     if (buf[0] != 1) {
2536         GST_ERROR("failed to decode codec-data, not in avcC format");
2537         return GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
2538     }
2539
2540     priv->nal_length_size = (buf[4] & 0x03) + 1;
2541
2542     num_sps = buf[5] & 0x1f;
2543     ofs = 6;
2544
2545     for (i = 0; i < num_sps; i++) {
2546         result = gst_h264_parser_identify_nalu_avc(
2547             priv->parser,
2548             buf, ofs, buf_size, 2,
2549             &nalu
2550         );
2551         if (result != GST_H264_PARSER_OK)
2552             return get_status(result);
2553
2554         status = decode_sps(decoder, &nalu);
2555         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2556             return status;
2557         ofs = nalu.offset + nalu.size;
2558     }
2559
2560     num_pps = buf[ofs];
2561     ofs++;
2562
2563     for (i = 0; i < num_pps; i++) {
2564         result = gst_h264_parser_identify_nalu_avc(
2565             priv->parser,
2566             buf, ofs, buf_size, 2,
2567             &nalu
2568         );
2569         if (result != GST_H264_PARSER_OK)
2570             return get_status(result);
2571
2572         status = decode_pps(decoder, &nalu);
2573         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2574             return status;
2575         ofs = nalu.offset + nalu.size;
2576     }
2577
2578     priv->is_avc = TRUE;
2579     return status;
2580 }
2581
2582 GstVaapiDecoderStatus
2583 gst_vaapi_decoder_h264_decode(GstVaapiDecoder *base, GstBuffer *buffer)
2584 {
2585     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264(base);
2586     GstVaapiDecoderH264Private * const priv = decoder->priv;
2587     GstVaapiDecoderStatus status;
2588     GstBuffer *codec_data;
2589
2590     g_return_val_if_fail(priv->is_constructed,
2591                          GST_VAAPI_DECODER_STATUS_ERROR_INIT_FAILED);
2592
2593     if (!priv->is_opened) {
2594         priv->is_opened = gst_vaapi_decoder_h264_open(decoder, buffer);
2595         if (!priv->is_opened)
2596             return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CODEC;
2597
2598         codec_data = GST_VAAPI_DECODER_CODEC_DATA(decoder);
2599         if (codec_data) {
2600             status = decode_codec_data(decoder, codec_data);
2601             if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2602                 return status;
2603         }
2604      }
2605      return decode_buffer(decoder, buffer);
2606 }
2607
2608 static void
2609 gst_vaapi_decoder_h264_finalize(GObject *object)
2610 {
2611     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264(object);
2612
2613     gst_vaapi_decoder_h264_destroy(decoder);
2614
2615     G_OBJECT_CLASS(gst_vaapi_decoder_h264_parent_class)->finalize(object);
2616 }
2617
2618 static void
2619 gst_vaapi_decoder_h264_constructed(GObject *object)
2620 {
2621     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264(object);
2622     GstVaapiDecoderH264Private * const priv = decoder->priv;
2623     GObjectClass *parent_class;
2624
2625     parent_class = G_OBJECT_CLASS(gst_vaapi_decoder_h264_parent_class);
2626     if (parent_class->constructed)
2627         parent_class->constructed(object);
2628
2629     priv->is_constructed = gst_vaapi_decoder_h264_create(decoder);
2630 }
2631
2632 static void
2633 gst_vaapi_decoder_h264_class_init(GstVaapiDecoderH264Class *klass)
2634 {
2635     GObjectClass * const object_class = G_OBJECT_CLASS(klass);
2636     GstVaapiDecoderClass * const decoder_class = GST_VAAPI_DECODER_CLASS(klass);
2637
2638     g_type_class_add_private(klass, sizeof(GstVaapiDecoderH264Private));
2639
2640     object_class->finalize      = gst_vaapi_decoder_h264_finalize;
2641     object_class->constructed   = gst_vaapi_decoder_h264_constructed;
2642
2643     decoder_class->decode       = gst_vaapi_decoder_h264_decode;
2644 }
2645
2646 static void
2647 gst_vaapi_decoder_h264_init(GstVaapiDecoderH264 *decoder)
2648 {
2649     GstVaapiDecoderH264Private *priv;
2650
2651     priv                        = GST_VAAPI_DECODER_H264_GET_PRIVATE(decoder);
2652     decoder->priv               = priv;
2653     priv->parser                = NULL;
2654     priv->sps                   = &priv->last_sps;
2655     priv->pps                   = &priv->last_pps;
2656     priv->current_picture       = NULL;
2657     priv->dpb_count             = 0;
2658     priv->dpb_size              = 0;
2659     priv->profile               = GST_VAAPI_PROFILE_H264_HIGH;
2660     priv->short_ref_count       = 0;
2661     priv->long_ref_count        = 0;
2662     priv->RefPicList0_count     = 0;
2663     priv->RefPicList1_count     = 0;
2664     priv->nal_length_size       = 0;
2665     priv->width                 = 0;
2666     priv->height                = 0;
2667     priv->mb_x                  = 0;
2668     priv->mb_y                  = 0;
2669     priv->mb_width              = 0;
2670     priv->mb_height             = 0;
2671     priv->adapter               = NULL;
2672     priv->field_poc[0]          = 0;
2673     priv->field_poc[1]          = 0;
2674     priv->poc_msb               = 0;
2675     priv->poc_lsb               = 0;
2676     priv->prev_poc_msb          = 0;
2677     priv->prev_poc_lsb          = 0;
2678     priv->frame_num_offset      = 0;
2679     priv->frame_num             = 0;
2680     priv->prev_frame_num        = 0;
2681     priv->prev_pic_has_mmco5    = FALSE;
2682     priv->prev_pic_bottom_field = FALSE;
2683     priv->is_constructed        = FALSE;
2684     priv->is_opened             = FALSE;
2685     priv->is_avc                = FALSE;
2686     priv->has_context           = FALSE;
2687
2688     memset(priv->dpb, 0, sizeof(priv->dpb));
2689     memset(priv->short_ref, 0, sizeof(priv->short_ref));
2690     memset(priv->long_ref, 0, sizeof(priv->long_ref));
2691     memset(priv->RefPicList0, 0, sizeof(priv->RefPicList0));
2692     memset(priv->RefPicList1, 0, sizeof(priv->RefPicList1));
2693 }
2694
2695 /**
2696  * gst_vaapi_decoder_h264_new:
2697  * @display: a #GstVaapiDisplay
2698  * @caps: a #GstCaps holding codec information
2699  *
2700  * Creates a new #GstVaapiDecoder for MPEG-2 decoding.  The @caps can
2701  * hold extra information like codec-data and pictured coded size.
2702  *
2703  * Return value: the newly allocated #GstVaapiDecoder object
2704  */
2705 GstVaapiDecoder *
2706 gst_vaapi_decoder_h264_new(GstVaapiDisplay *display, GstCaps *caps)
2707 {
2708     GstVaapiDecoderH264 *decoder;
2709
2710     g_return_val_if_fail(GST_VAAPI_IS_DISPLAY(display), NULL);
2711     g_return_val_if_fail(GST_IS_CAPS(caps), NULL);
2712
2713     decoder = g_object_new(
2714         GST_VAAPI_TYPE_DECODER_H264,
2715         "display",      display,
2716         "caps",         caps,
2717         NULL
2718     );
2719     if (!decoder->priv->is_constructed) {
2720         g_object_unref(decoder);
2721         return NULL;
2722     }
2723     return GST_VAAPI_DECODER_CAST(decoder);
2724 }