gst-libs/gst/vaapi/gstvaapidecoder_h264.c

   1 /*
   2  *  gstvaapidecoder_h264.c - H.264 decoder
   3  *
   4  *  Copyright (C) 2011-2012 Intel Corporation
   5  *
   6  *  This library is free software; you can redistribute it and/or
   7  *  modify it under the terms of the GNU Lesser General Public License
   8  *  as published by the Free Software Foundation; either version 2.1
   9  *  of the License, or (at your option) any later version.
  10  *
  11  *  This library is distributed in the hope that it will be useful,
  12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  *  Lesser General Public License for more details.
  15  *
  16  *  You should have received a copy of the GNU Lesser General Public
  17  *  License along with this library; if not, write to the Free
  18  *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  19  *  Boston, MA 02110-1301 USA
  20  */
  21
  22 /**
  23  * SECTION:gstvaapidecoder_h264
  24  * @short_description: H.264 decoder
  25  */
  26
  27 #include "sysdeps.h"
  28 #include <string.h>
  29 #include <gst/base/gstadapter.h>
  30 #include <gst/codecparsers/gsth264parser.h>
  31 #include "gstvaapidecoder_h264.h"
  32 #include "gstvaapidecoder_objects.h"
  33 #include "gstvaapidecoder_priv.h"
  34 #include "gstvaapidisplay_priv.h"
  35 #include "gstvaapiobject_priv.h"
  36
  37 #define DEBUG 1
  38 #include "gstvaapidebug.h"
  39
  40 /* Defined to 1 if strict ordering of DPB is needed. Only useful for debug */
  41 #define USE_STRICT_DPB_ORDERING 0
  42
  43 typedef struct _GstVaapiPictureH264             GstVaapiPictureH264;
  44 typedef struct _GstVaapiPictureH264Class        GstVaapiPictureH264Class;
  45 typedef struct _GstVaapiSliceH264               GstVaapiSliceH264;
  46 typedef struct _GstVaapiSliceH264Class          GstVaapiSliceH264Class;
  47
  48 /* ------------------------------------------------------------------------- */
  49 /* --- H.264 Pictures                                                    --- */
  50 /* ------------------------------------------------------------------------- */
  51
  52 #define GST_VAAPI_TYPE_PICTURE_H264 \
  53     (gst_vaapi_picture_h264_get_type())
  54
  55 #define GST_VAAPI_PICTURE_H264_CAST(obj) \
  56     ((GstVaapiPictureH264 *)(obj))
  57
  58 #define GST_VAAPI_PICTURE_H264(obj)                             \
  59     (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
  60                                 GST_VAAPI_TYPE_PICTURE_H264,    \
  61                                 GstVaapiPictureH264))
  62
  63 #define GST_VAAPI_PICTURE_H264_CLASS(klass)                     \
  64     (G_TYPE_CHECK_CLASS_CAST((klass),                           \
  65                              GST_VAAPI_TYPE_PICTURE_H264,       \
  66                              GstVaapiPictureH264Class))
  67
  68 #define GST_VAAPI_IS_PICTURE_H264(obj) \
  69     (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_VAAPI_TYPE_PICTURE_H264))
  70
  71 #define GST_VAAPI_IS_PICTURE_H264_CLASS(klass) \
  72     (G_TYPE_CHECK_CLASS_TYPE((klass), GST_VAAPI_TYPE_PICTURE_H264))
  73
  74 #define GST_VAAPI_PICTURE_H264_GET_CLASS(obj)                   \
  75     (G_TYPE_INSTANCE_GET_CLASS((obj),                           \
  76                                GST_VAAPI_TYPE_PICTURE_H264,     \
  77                                GstVaapiPictureH264Class))
  78
  79 struct _GstVaapiPictureH264 {
  80     GstVaapiPicture             base;
  81     VAPictureH264               info;
  82     gint32                      poc;
  83     gint32                      frame_num;              // Original frame_num from slice_header()
  84     gint32                      frame_num_wrap;         // Temporary for ref pic marking: FrameNumWrap
  85     gint32                      pic_num;                // Temporary for ref pic marking: PicNum
  86     gint32                      long_term_pic_num;      // Temporary for ref pic marking: LongTermPicNum
  87     guint                       is_idr                  : 1;
  88     guint                       is_long_term            : 1;
  89     guint                       field_pic_flag          : 1;
  90     guint                       bottom_field_flag       : 1;
  91     guint                       output_flag             : 1;
  92     guint                       output_needed           : 1;
  93 };
  94
  95 struct _GstVaapiPictureH264Class {
  96     /*< private >*/
  97     GstVaapiPictureClass        parent_class;
  98 };
  99
 100 GST_VAAPI_CODEC_DEFINE_TYPE(GstVaapiPictureH264,
 101                             gst_vaapi_picture_h264,
 102                             GST_VAAPI_TYPE_PICTURE)
 103
 104 static void
 105 gst_vaapi_picture_h264_destroy(GstVaapiPictureH264 *decoder)
 106 {
 107 }
 108
 109 static gboolean
 110 gst_vaapi_picture_h264_create(
 111     GstVaapiPictureH264                      *picture,
 112     const GstVaapiCodecObjectConstructorArgs *args
 113 )
 114 {
 115     return TRUE;
 116 }
 117
 118 static void
 119 gst_vaapi_picture_h264_init(GstVaapiPictureH264 *picture)
 120 {
 121     VAPictureH264 *va_pic;
 122
 123     va_pic                      = &picture->info;
 124     va_pic->flags               = 0;
 125     va_pic->TopFieldOrderCnt    = 0;
 126     va_pic->BottomFieldOrderCnt = 0;
 127
 128     picture->poc                = 0;
 129     picture->is_long_term       = FALSE;
 130     picture->is_idr             = FALSE;
 131     picture->output_needed      = FALSE;
 132 }
 133
 134 static inline GstVaapiPictureH264 *
 135 gst_vaapi_picture_h264_new(GstVaapiDecoderH264 *decoder)
 136 {
 137     GstVaapiCodecObject *object;
 138
 139     g_return_val_if_fail(GST_VAAPI_IS_DECODER(decoder), NULL);
 140
 141     object = gst_vaapi_codec_object_new(
 142         GST_VAAPI_TYPE_PICTURE_H264,
 143         GST_VAAPI_CODEC_BASE(decoder),
 144         NULL, sizeof(VAPictureParameterBufferH264),
 145         NULL, 0
 146     );
 147     if (!object)
 148         return NULL;
 149     return GST_VAAPI_PICTURE_H264_CAST(object);
 150 }
 151
 152 /* ------------------------------------------------------------------------- */
 153 /* --- Slices                                                            --- */
 154 /* ------------------------------------------------------------------------- */
 155
 156 #define GST_VAAPI_TYPE_SLICE_H264 \
 157     (gst_vaapi_slice_h264_get_type())
 158
 159 #define GST_VAAPI_SLICE_H264_CAST(obj) \
 160     ((GstVaapiSliceH264 *)(obj))
 161
 162 #define GST_VAAPI_SLICE_H264(obj)                               \
 163     (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
 164                                 GST_VAAPI_TYPE_SLICE_H264,      \
 165                                 GstVaapiSliceH264))
 166
 167 #define GST_VAAPI_SLICE_H264_CLASS(klass)                       \
 168     (G_TYPE_CHECK_CLASS_CAST((klass),                           \
 169                              GST_VAAPI_TYPE_SLICE_H264,         \
 170                              GstVaapiSliceH264Class))
 171
 172 #define GST_VAAPI_IS_SLICE_H264(obj) \
 173     (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_VAAPI_TYPE_SLICE_H264))
 174
 175 #define GST_VAAPI_IS_SLICE_H264_CLASS(klass) \
 176     (G_TYPE_CHECK_CLASS_TYPE((klass), GST_VAAPI_TYPE_SLICE_H264))
 177
 178 #define GST_VAAPI_SLICE_H264_GET_CLASS(obj)                     \
 179     (G_TYPE_INSTANCE_GET_CLASS((obj),                           \
 180                                GST_VAAPI_TYPE_SLICE_H264,       \
 181                                GstVaapiSliceH264Class))
 182
 183 struct _GstVaapiSliceH264 {
 184     GstVaapiSlice               base;
 185     GstH264SliceHdr             slice_hdr;              // parsed slice_header()
 186 };
 187
 188 struct _GstVaapiSliceH264Class {
 189     /*< private >*/
 190     GstVaapiSliceClass          parent_class;
 191 };
 192
 193 GST_VAAPI_CODEC_DEFINE_TYPE(GstVaapiSliceH264,
 194                             gst_vaapi_slice_h264,
 195                             GST_VAAPI_TYPE_SLICE)
 196
 197 static void
 198 gst_vaapi_slice_h264_destroy(GstVaapiSliceH264 *slice)
 199 {
 200 }
 201
 202 static gboolean
 203 gst_vaapi_slice_h264_create(
 204     GstVaapiSliceH264                        *slice,
 205     const GstVaapiCodecObjectConstructorArgs *args
 206 )
 207 {
 208     return TRUE;
 209 }
 210
 211 static void
 212 gst_vaapi_slice_h264_init(GstVaapiSliceH264 *slice)
 213 {
 214 }
 215
 216 static inline GstVaapiSliceH264 *
 217 gst_vaapi_slice_h264_new(
 218     GstVaapiDecoderH264 *decoder,
 219     const guint8        *data,
 220     guint                data_size
 221 )
 222 {
 223     GstVaapiCodecObject *object;
 224
 225     g_return_val_if_fail(GST_VAAPI_IS_DECODER(decoder), NULL);
 226
 227     object = gst_vaapi_codec_object_new(
 228         GST_VAAPI_TYPE_SLICE_H264,
 229         GST_VAAPI_CODEC_BASE(decoder),
 230         NULL, sizeof(VASliceParameterBufferH264),
 231         data, data_size
 232     );
 233     if (!object)
 234         return NULL;
 235     return GST_VAAPI_SLICE_H264_CAST(object);
 236 }
 237
 238 /* ------------------------------------------------------------------------- */
 239 /* --- H.264 Decoder                                                     --- */
 240 /* ------------------------------------------------------------------------- */
 241
 242 G_DEFINE_TYPE(GstVaapiDecoderH264,
 243               gst_vaapi_decoder_h264,
 244               GST_VAAPI_TYPE_DECODER)
 245
 246 #define GST_VAAPI_DECODER_H264_GET_PRIVATE(obj)                 \
 247     (G_TYPE_INSTANCE_GET_PRIVATE((obj),                         \
 248                                  GST_VAAPI_TYPE_DECODER_H264,   \
 249                                  GstVaapiDecoderH264Private))
 250
 251 // Used for field_poc[]
 252 #define TOP_FIELD       0
 253 #define BOTTOM_FIELD    1
 254
 255 struct _GstVaapiDecoderH264Private {
 256     GstAdapter                 *adapter;
 257     GstH264NalParser           *parser;
 258     GstH264SPS                 *sps;
 259     GstH264SPS                  last_sps;
 260     GstH264PPS                 *pps;
 261     GstH264PPS                  last_pps;
 262     GstVaapiPictureH264        *current_picture;
 263     GstVaapiPictureH264        *dpb[16];
 264     guint                       dpb_count;
 265     guint                       dpb_size;
 266     GstVaapiProfile             profile;
 267     GstVaapiPictureH264        *short_ref[32];
 268     guint                       short_ref_count;
 269     GstVaapiPictureH264        *long_ref[32];
 270     guint                       long_ref_count;
 271     GstVaapiPictureH264        *RefPicList0[32];
 272     guint                       RefPicList0_count;
 273     GstVaapiPictureH264        *RefPicList1[32];
 274     guint                       RefPicList1_count;
 275     guint                       nal_length_size;
 276     guint                       width;
 277     guint                       height;
 278     guint                       mb_x;
 279     guint                       mb_y;
 280     guint                       mb_width;
 281     guint                       mb_height;
 282     guint8                      scaling_list_4x4[6][16];
 283     guint8                      scaling_list_8x8[6][64];
 284     gint32                      field_poc[2];           // 0:TopFieldOrderCnt / 1:BottomFieldOrderCnt
 285     gint32                      poc_msb;                // PicOrderCntMsb
 286     gint32                      poc_lsb;                // pic_order_cnt_lsb (from slice_header())
 287     gint32                      prev_poc_msb;           // prevPicOrderCntMsb
 288     gint32                      prev_poc_lsb;           // prevPicOrderCntLsb
 289     gint32                      frame_num_offset;       // FrameNumOffset
 290     gint32                      frame_num;              // frame_num (from slice_header())
 291     gint32                      prev_frame_num;         // prevFrameNum
 292     gboolean                    prev_pic_has_mmco5;     // prevMmco5Pic
 293     gboolean                    prev_pic_bottom_field;  // Flag: previous picture is a bottom field
 294     guint                       is_constructed          : 1;
 295     guint                       is_opened               : 1;
 296     guint                       is_avc                  : 1;
 297     guint                       has_context             : 1;
 298 };
 299
 300 static gboolean
 301 decode_picture_end(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture);
 302
 303 static void
 304 clear_references(
 305     GstVaapiDecoderH264  *decoder,
 306     GstVaapiPictureH264 **pictures,
 307     guint                *picture_count
 308 );
 309
 310 /* Get number of reference frames to use */
 311 static guint
 312 get_max_dec_frame_buffering(GstH264SPS *sps)
 313 {
 314     guint max_dec_frame_buffering, MaxDpbMbs, PicSizeMbs;
 315
 316     /* Table A-1 - Level limits */
 317     switch (sps->level_idc) {
 318     case 10: MaxDpbMbs = 396;    break;
 319     case 11: MaxDpbMbs = 900;    break;
 320     case 12: MaxDpbMbs = 2376;   break;
 321     case 13: MaxDpbMbs = 2376;   break;
 322     case 20: MaxDpbMbs = 2376;   break;
 323     case 21: MaxDpbMbs = 4752;   break;
 324     case 22: MaxDpbMbs = 8100;   break;
 325     case 30: MaxDpbMbs = 8100;   break;
 326     case 31: MaxDpbMbs = 18000;  break;
 327     case 32: MaxDpbMbs = 20480;  break;
 328     case 40: MaxDpbMbs = 32768;  break;
 329     case 41: MaxDpbMbs = 32768;  break;
 330     case 42: MaxDpbMbs = 34816;  break;
 331     case 50: MaxDpbMbs = 110400; break;
 332     case 51: MaxDpbMbs = 184320; break;
 333     default:
 334         g_assert(0 && "unhandled level");
 335         break;
 336     }
 337
 338     PicSizeMbs = ((sps->pic_width_in_mbs_minus1 + 1) *
 339                   (sps->pic_height_in_map_units_minus1 + 1) *
 340                   (sps->frame_mbs_only_flag ? 1 : 2));
 341     max_dec_frame_buffering = MaxDpbMbs / PicSizeMbs;
 342
 343     /* VUI parameters */
 344     if (sps->vui_parameters_present_flag) {
 345         GstH264VUIParams * const vui_params = &sps->vui_parameters;
 346         if (vui_params->bitstream_restriction_flag)
 347             max_dec_frame_buffering = vui_params->max_dec_frame_buffering;
 348         else {
 349             switch (sps->profile_idc) {
 350             case 44:  // CAVLC 4:4:4 Intra profile
 351             case 86:  // Scalable High profile
 352             case 100: // High profile
 353             case 110: // High 10 profile
 354             case 122: // High 4:2:2 profile
 355             case 244: // High 4:4:4 Predictive profile
 356                 if (sps->constraint_set3_flag)
 357                     max_dec_frame_buffering = 0;
 358                 break;
 359             }
 360         }
 361     }
 362
 363     if (max_dec_frame_buffering > 16)
 364         max_dec_frame_buffering = 16;
 365     else if (max_dec_frame_buffering < sps->num_ref_frames)
 366         max_dec_frame_buffering = sps->num_ref_frames;
 367     return MAX(1, max_dec_frame_buffering);
 368 }
 369
 370 static void
 371 dpb_remove_index(GstVaapiDecoderH264 *decoder, guint index)
 372 {
 373     GstVaapiDecoderH264Private * const priv = decoder->priv;
 374     guint i, num_pictures = --priv->dpb_count;
 375
 376     if (USE_STRICT_DPB_ORDERING) {
 377         for (i = index; i < num_pictures; i++)
 378             gst_vaapi_picture_replace(&priv->dpb[i], priv->dpb[i + 1]);
 379     }
 380     else if (index != num_pictures)
 381         gst_vaapi_picture_replace(&priv->dpb[index], priv->dpb[num_pictures]);
 382     gst_vaapi_picture_replace(&priv->dpb[num_pictures], NULL);
 383 }
 384
 385 static inline gboolean
 386 dpb_output(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
 387 {
 388     /* XXX: update cropping rectangle */
 389     picture->output_needed = FALSE;
 390     return gst_vaapi_picture_output(GST_VAAPI_PICTURE_CAST(picture));
 391 }
 392
 393 static gboolean
 394 dpb_bump(GstVaapiDecoderH264 *decoder)
 395 {
 396     GstVaapiDecoderH264Private * const priv = decoder->priv;
 397     guint i, lowest_poc_index;
 398     gboolean success;
 399
 400     for (i = 0; i < priv->dpb_count; i++) {
 401         if (priv->dpb[i]->output_needed)
 402             break;
 403     }
 404     if (i == priv->dpb_count)
 405         return FALSE;
 406
 407     lowest_poc_index = i++;
 408     for (; i < priv->dpb_count; i++) {
 409         GstVaapiPictureH264 * const picture = priv->dpb[i];
 410         if (picture->output_needed && picture->poc < priv->dpb[lowest_poc_index]->poc)
 411             lowest_poc_index = i;
 412     }
 413
 414     success = dpb_output(decoder, priv->dpb[lowest_poc_index]);
 415     if (!GST_VAAPI_PICTURE_IS_REFERENCE(priv->dpb[lowest_poc_index]))
 416         dpb_remove_index(decoder, lowest_poc_index);
 417     return success;
 418 }
 419
 420 static void
 421 dpb_flush(GstVaapiDecoderH264 *decoder)
 422 {
 423     GstVaapiDecoderH264Private * const priv = decoder->priv;
 424
 425     while (dpb_bump(decoder))
 426         ;
 427     clear_references(decoder, priv->dpb, &priv->dpb_count);
 428 }
 429
 430 static gboolean
 431 dpb_add(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
 432 {
 433     GstVaapiDecoderH264Private * const priv = decoder->priv;
 434     guint i;
 435
 436     // Remove all unused pictures
 437     if (picture->is_idr)
 438         dpb_flush(decoder);
 439     else {
 440         i = 0;
 441         while (i < priv->dpb_count) {
 442             GstVaapiPictureH264 * const picture = priv->dpb[i];
 443             if (!picture->output_needed &&
 444                 !GST_VAAPI_PICTURE_IS_REFERENCE(picture))
 445                 dpb_remove_index(decoder, i);
 446             else
 447                 i++;
 448         }
 449     }
 450
 451     // C.4.5.1 - Storage and marking of a reference decoded picture into the DPB
 452     if (GST_VAAPI_PICTURE_IS_REFERENCE(picture)) {
 453         while (priv->dpb_count == priv->dpb_size) {
 454             if (!dpb_bump(decoder))
 455                 return FALSE;
 456         }
 457         gst_vaapi_picture_replace(&priv->dpb[priv->dpb_count++], picture);
 458         if (picture->output_flag)
 459             picture->output_needed = TRUE;
 460     }
 461
 462     // C.4.5.2 - Storage and marking of a non-reference decoded picture into the DPB
 463     else {
 464         if (!picture->output_flag)
 465             return TRUE;
 466         while (priv->dpb_count == priv->dpb_size) {
 467             for (i = 0; i < priv->dpb_count; i++) {
 468                 if (priv->dpb[i]->output_needed &&
 469                     priv->dpb[i]->poc < picture->poc)
 470                     break;
 471             }
 472             if (i == priv->dpb_count)
 473                 return dpb_output(decoder, picture);
 474             if (!dpb_bump(decoder))
 475                 return FALSE;
 476         }
 477         gst_vaapi_picture_replace(&priv->dpb[priv->dpb_count++], picture);
 478         picture->output_needed = TRUE;
 479     }
 480     return TRUE;
 481 }
 482
 483 static inline void
 484 dpb_reset(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 485 {
 486     GstVaapiDecoderH264Private * const priv = decoder->priv;
 487
 488     priv->dpb_size = get_max_dec_frame_buffering(sps);
 489     GST_DEBUG("DPB size %u", priv->dpb_size);
 490 }
 491
 492 static GstVaapiDecoderStatus
 493 get_status(GstH264ParserResult result)
 494 {
 495     GstVaapiDecoderStatus status;
 496
 497     switch (result) {
 498     case GST_H264_PARSER_OK:
 499         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
 500         break;
 501     case GST_H264_PARSER_NO_NAL_END:
 502         status = GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
 503         break;
 504     case GST_H264_PARSER_ERROR:
 505         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
 506         break;
 507     default:
 508         status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 509         break;
 510     }
 511     return status;
 512 }
 513
 514 static inline GstH264DecRefPicMarking *
 515 get_dec_ref_pic_marking(GstVaapiPictureH264 *picture_h264)
 516 {
 517     GstVaapiPicture * const picture = GST_VAAPI_PICTURE_CAST(picture_h264);
 518     GstVaapiSliceH264 *slice;
 519
 520     slice = g_ptr_array_index(picture->slices, picture->slices->len - 1);
 521     return &slice->slice_hdr.dec_ref_pic_marking;
 522 }
 523
 524 static void
 525 gst_vaapi_decoder_h264_close(GstVaapiDecoderH264 *decoder)
 526 {
 527     GstVaapiDecoderH264Private * const priv = decoder->priv;
 528
 529     gst_vaapi_picture_replace(&priv->current_picture, NULL);
 530     clear_references(decoder, priv->short_ref, &priv->short_ref_count);
 531     clear_references(decoder, priv->long_ref,  &priv->long_ref_count );
 532     clear_references(decoder, priv->dpb,       &priv->dpb_count      );
 533
 534     if (priv->parser) {
 535         gst_h264_nal_parser_free(priv->parser);
 536         priv->parser = NULL;
 537     }
 538
 539     if (priv->adapter) {
 540         gst_adapter_clear(priv->adapter);
 541         g_object_unref(priv->adapter);
 542         priv->adapter = NULL;
 543     }
 544 }
 545
 546 static gboolean
 547 gst_vaapi_decoder_h264_open(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
 548 {
 549     GstVaapiDecoderH264Private * const priv = decoder->priv;
 550
 551     gst_vaapi_decoder_h264_close(decoder);
 552
 553     priv->adapter = gst_adapter_new();
 554     if (!priv->adapter)
 555         return FALSE;
 556
 557     priv->parser = gst_h264_nal_parser_new();
 558     if (!priv->parser)
 559         return FALSE;
 560     return TRUE;
 561 }
 562
 563 static void
 564 gst_vaapi_decoder_h264_destroy(GstVaapiDecoderH264 *decoder)
 565 {
 566     gst_vaapi_decoder_h264_close(decoder);
 567 }
 568
 569 static gboolean
 570 gst_vaapi_decoder_h264_create(GstVaapiDecoderH264 *decoder)
 571 {
 572     if (!GST_VAAPI_DECODER_CODEC(decoder))
 573         return FALSE;
 574     return TRUE;
 575 }
 576
 577 static GstVaapiDecoderStatus
 578 ensure_context(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 579 {
 580     GstVaapiDecoderH264Private * const priv = decoder->priv;
 581     GstVaapiProfile profiles[2];
 582     GstVaapiEntrypoint entrypoint = GST_VAAPI_ENTRYPOINT_VLD;
 583     guint i, n_profiles = 0;
 584     gboolean success, reset_context = FALSE;
 585
 586     if (!priv->has_context || priv->sps->profile_idc != sps->profile_idc) {
 587         GST_DEBUG("profile changed");
 588         reset_context = TRUE;
 589
 590         switch (sps->profile_idc) {
 591         case 66:
 592             profiles[n_profiles++] = GST_VAAPI_PROFILE_H264_BASELINE;
 593             break;
 594         case 77:
 595             profiles[n_profiles++] = GST_VAAPI_PROFILE_H264_MAIN;
 596             // fall-through
 597         case 100:
 598             profiles[n_profiles++] = GST_VAAPI_PROFILE_H264_HIGH;
 599             break;
 600         default:
 601             GST_DEBUG("unsupported profile %d", sps->profile_idc);
 602             return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_PROFILE;
 603         }
 604
 605         for (i = 0; i < n_profiles; i++) {
 606             success = gst_vaapi_display_has_decoder(
 607                 GST_VAAPI_DECODER_DISPLAY(decoder),
 608                 profiles[i],
 609                 entrypoint
 610             );
 611             if (success)
 612                 break;
 613         }
 614         if (i == n_profiles)
 615             return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_PROFILE;
 616         priv->profile = profiles[i];
 617     }
 618
 619     if (!priv->has_context ||
 620         priv->sps->chroma_format_idc != sps->chroma_format_idc) {
 621         GST_DEBUG("chroma format changed");
 622         reset_context = TRUE;
 623
 624         /* XXX: theoritically, we could handle 4:2:2 format */
 625         if (sps->chroma_format_idc != 1)
 626             return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;
 627     }
 628
 629     if (!priv->has_context              ||
 630         priv->sps->width  != sps->width ||
 631         priv->sps->height != sps->height) {
 632         GST_DEBUG("size changed");
 633         reset_context      = TRUE;
 634
 635         priv->width      = sps->width;
 636         priv->height     = sps->height;
 637         priv->mb_width   = sps->pic_width_in_mbs_minus1 + 1;
 638         priv->mb_height  = sps->pic_height_in_map_units_minus1 + 1;
 639         priv->mb_height *= 2 - sps->frame_mbs_only_flag;
 640     }
 641
 642     if (reset_context) {
 643         GstVaapiContextInfo info;
 644
 645         info.profile    = priv->profile;
 646         info.entrypoint = entrypoint;
 647         info.width      = priv->width;
 648         info.height     = priv->height;
 649         info.ref_frames = get_max_dec_frame_buffering(sps);
 650
 651         if (!gst_vaapi_decoder_ensure_context(GST_VAAPI_DECODER(decoder), &info))
 652             return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 653         priv->has_context = TRUE;
 654
 655         /* Reset DPB */
 656         dpb_reset(decoder, sps);
 657     }
 658     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 659 }
 660
 661 static GstVaapiDecoderStatus
 662 ensure_quant_matrix(GstVaapiDecoderH264 *decoder, GstH264PPS *pps)
 663 {
 664     GstVaapiDecoderH264Private * const priv = decoder->priv;
 665
 666     if (priv->pps != pps) {
 667         memcpy(priv->scaling_list_4x4, pps->scaling_lists_4x4,
 668                sizeof(priv->scaling_list_4x4));
 669         memcpy(priv->scaling_list_8x8, pps->scaling_lists_8x8,
 670                sizeof(priv->scaling_list_8x8));
 671     }
 672     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 673 }
 674
 675 static gboolean
 676 decode_current_picture(GstVaapiDecoderH264 *decoder)
 677 {
 678     GstVaapiDecoderH264Private * const priv = decoder->priv;
 679     GstVaapiPictureH264 * const picture = priv->current_picture;
 680     gboolean success = FALSE;
 681
 682     if (!picture)
 683         return TRUE;
 684
 685     if (!decode_picture_end(decoder, picture))
 686         goto end;
 687     if (!gst_vaapi_picture_decode(GST_VAAPI_PICTURE_CAST(picture)))
 688         goto end;
 689     success = TRUE;
 690 end:
 691     gst_vaapi_picture_replace(&priv->current_picture, NULL);
 692     return success;
 693 }
 694
 695 static GstVaapiDecoderStatus
 696 decode_sps(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
 697 {
 698     GstVaapiDecoder * const base_decoder = GST_VAAPI_DECODER(decoder);
 699     GstVaapiDecoderH264Private * const priv = decoder->priv;
 700     GstH264SPS * const sps = &priv->last_sps;
 701     GstH264ParserResult result;
 702
 703     GST_DEBUG("decode SPS");
 704
 705     if (priv->current_picture && !decode_current_picture(decoder))
 706         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 707
 708     memset(sps, 0, sizeof(*sps));
 709     result = gst_h264_parser_parse_sps(priv->parser, nalu, sps, TRUE);
 710     if (result != GST_H264_PARSER_OK)
 711         return get_status(result);
 712
 713     gst_vaapi_decoder_set_pixel_aspect_ratio(
 714         base_decoder,
 715         sps->vui_parameters.par_n,
 716         sps->vui_parameters.par_d
 717     );
 718     return ensure_context(decoder, sps);
 719 }
 720
 721 static GstVaapiDecoderStatus
 722 decode_pps(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
 723 {
 724     GstVaapiDecoderH264Private * const priv = decoder->priv;
 725     GstH264PPS * const pps = &priv->last_pps;
 726     GstH264ParserResult result;
 727
 728     GST_DEBUG("decode PPS");
 729
 730     if (priv->current_picture && !decode_current_picture(decoder))
 731         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 732
 733     memset(pps, 0, sizeof(*pps));
 734     result = gst_h264_parser_parse_pps(priv->parser, nalu, pps);
 735     if (result != GST_H264_PARSER_OK)
 736         return get_status(result);
 737
 738     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 739 }
 740
 741 static GstVaapiDecoderStatus
 742 decode_sei(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
 743 {
 744     GstVaapiDecoderH264Private * const priv = decoder->priv;
 745     GstH264SEIMessage sei;
 746     GstH264ParserResult result;
 747
 748     GST_DEBUG("decode SEI");
 749
 750     memset(&sei, 0, sizeof(sei));
 751     result = gst_h264_parser_parse_sei(priv->parser, nalu, &sei);
 752     if (result != GST_H264_PARSER_OK) {
 753         GST_WARNING("failed to decode SEI, payload type:%d", sei.payloadType);
 754         return get_status(result);
 755     }
 756
 757     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 758 }
 759
 760 static GstVaapiDecoderStatus
 761 decode_sequence_end(GstVaapiDecoderH264 *decoder)
 762 {
 763     GstVaapiDecoderH264Private * const priv = decoder->priv;
 764
 765     GST_DEBUG("decode sequence-end");
 766
 767     if (priv->current_picture && !decode_current_picture(decoder))
 768         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 769     dpb_flush(decoder);
 770     return GST_VAAPI_DECODER_STATUS_END_OF_STREAM;
 771 }
 772
 773 /* 8.2.1.1 - Decoding process for picture order count type 0 */
 774 static void
 775 init_picture_poc_0(
 776     GstVaapiDecoderH264 *decoder,
 777     GstVaapiPictureH264 *picture,
 778     GstH264SliceHdr     *slice_hdr
 779 )
 780 {
 781     GstVaapiDecoderH264Private * const priv = decoder->priv;
 782     GstH264PPS * const pps = slice_hdr->pps;
 783     GstH264SPS * const sps = pps->sequence;
 784     const gint32 MaxPicOrderCntLsb = 1 << (sps->log2_max_pic_order_cnt_lsb_minus4 + 4);
 785
 786     GST_DEBUG("decode picture order count type 0");
 787
 788     if (picture->is_idr) {
 789         priv->prev_poc_msb = 0;
 790         priv->prev_poc_lsb = 0;
 791     }
 792     else if (priv->prev_pic_has_mmco5) {
 793         priv->prev_poc_msb = 0;
 794         priv->prev_poc_lsb = priv->prev_pic_bottom_field ? 0 :
 795             priv->field_poc[TOP_FIELD];
 796     }
 797     else {
 798         priv->prev_poc_msb = priv->poc_msb;
 799         priv->prev_poc_lsb = priv->poc_lsb;
 800     }
 801
 802     // (8-3)
 803     priv->poc_lsb = slice_hdr->pic_order_cnt_lsb;
 804     if (priv->poc_lsb < priv->prev_poc_lsb &&
 805         (priv->prev_poc_lsb - priv->poc_lsb) >= (MaxPicOrderCntLsb / 2))
 806         priv->poc_msb = priv->prev_poc_msb + MaxPicOrderCntLsb;
 807     else if (priv->poc_lsb > priv->prev_poc_lsb &&
 808              (priv->poc_lsb - priv->prev_poc_lsb) > (MaxPicOrderCntLsb / 2))
 809         priv->poc_msb = priv->prev_poc_msb - MaxPicOrderCntLsb;
 810     else
 811         priv->poc_msb = priv->prev_poc_msb;
 812
 813     // (8-4)
 814     if (!slice_hdr->field_pic_flag || !slice_hdr->bottom_field_flag)
 815         priv->field_poc[TOP_FIELD] = priv->poc_msb + priv->poc_lsb;
 816
 817     // (8-5)
 818     if (!slice_hdr->field_pic_flag)
 819         priv->field_poc[BOTTOM_FIELD] = priv->field_poc[TOP_FIELD] +
 820             slice_hdr->delta_pic_order_cnt_bottom;
 821     else if (slice_hdr->bottom_field_flag)
 822         priv->field_poc[BOTTOM_FIELD] = priv->poc_msb + priv->poc_lsb;
 823 }
 824
 825 /* 8.2.1.2 - Decoding process for picture order count type 1 */
 826 static void
 827 init_picture_poc_1(
 828     GstVaapiDecoderH264 *decoder,
 829     GstVaapiPictureH264 *picture,
 830     GstH264SliceHdr     *slice_hdr
 831 )
 832 {
 833     GstVaapiDecoderH264Private * const priv = decoder->priv;
 834     GstH264PPS * const pps = slice_hdr->pps;
 835     GstH264SPS * const sps = pps->sequence;
 836     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
 837     gint32 prev_frame_num_offset, abs_frame_num, expected_poc;
 838     guint i;
 839
 840     GST_DEBUG("decode picture order count type 1");
 841
 842     if (priv->prev_pic_has_mmco5)
 843         prev_frame_num_offset = 0;
 844     else
 845         prev_frame_num_offset = priv->frame_num_offset;
 846
 847     // (8-6)
 848     if (picture->is_idr)
 849         priv->frame_num_offset = 0;
 850     else if (priv->prev_frame_num > priv->frame_num)
 851         priv->frame_num_offset = prev_frame_num_offset + MaxFrameNum;
 852     else
 853         priv->frame_num_offset = prev_frame_num_offset;
 854
 855     // (8-7)
 856     if (sps->num_ref_frames_in_pic_order_cnt_cycle != 0)
 857         abs_frame_num = priv->frame_num_offset + priv->frame_num;
 858     else
 859         abs_frame_num = 0;
 860     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture) && abs_frame_num > 0)
 861         abs_frame_num = abs_frame_num - 1;
 862
 863     if (abs_frame_num > 0) {
 864         gint32 expected_delta_per_poc_cycle;
 865         gint32 poc_cycle_cnt, frame_num_in_poc_cycle;
 866
 867         expected_delta_per_poc_cycle = 0;
 868         for (i = 0; i < sps->num_ref_frames_in_pic_order_cnt_cycle; i++)
 869             expected_delta_per_poc_cycle += sps->offset_for_ref_frame[i];
 870
 871         // (8-8)
 872         poc_cycle_cnt = (abs_frame_num - 1) /
 873             sps->num_ref_frames_in_pic_order_cnt_cycle;
 874         frame_num_in_poc_cycle = (abs_frame_num - 1) %
 875             sps->num_ref_frames_in_pic_order_cnt_cycle;
 876
 877         // (8-9)
 878         expected_poc = poc_cycle_cnt * expected_delta_per_poc_cycle;
 879         for (i = 0; i <= frame_num_in_poc_cycle; i++)
 880             expected_poc += sps->offset_for_ref_frame[i];
 881     }
 882     else
 883         expected_poc = 0;
 884     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
 885         expected_poc += sps->offset_for_non_ref_pic;
 886
 887     // (8-10)
 888     if (!slice_hdr->field_pic_flag) {
 889         priv->field_poc[TOP_FIELD] = expected_poc +
 890             slice_hdr->delta_pic_order_cnt[0];
 891         priv->field_poc[BOTTOM_FIELD] = priv->field_poc[TOP_FIELD] +
 892             sps->offset_for_top_to_bottom_field +
 893             slice_hdr->delta_pic_order_cnt[1];
 894     }
 895     else if (!slice_hdr->bottom_field_flag)
 896         priv->field_poc[TOP_FIELD] = expected_poc +
 897             slice_hdr->delta_pic_order_cnt[0];
 898     else
 899         priv->field_poc[BOTTOM_FIELD] = expected_poc +
 900             sps->offset_for_top_to_bottom_field + slice_hdr->delta_pic_order_cnt[0];
 901 }
 902
 903 /* 8.2.1.3 - Decoding process for picture order count type 2 */
 904 static void
 905 init_picture_poc_2(
 906     GstVaapiDecoderH264 *decoder,
 907     GstVaapiPictureH264 *picture,
 908     GstH264SliceHdr     *slice_hdr
 909 )
 910 {
 911     GstVaapiDecoderH264Private * const priv = decoder->priv;
 912     GstH264PPS * const pps = slice_hdr->pps;
 913     GstH264SPS * const sps = pps->sequence;
 914     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
 915     gint32 prev_frame_num_offset, temp_poc;
 916
 917     GST_DEBUG("decode picture order count type 2");
 918
 919     if (priv->prev_pic_has_mmco5)
 920         prev_frame_num_offset = 0;
 921     else
 922         prev_frame_num_offset = priv->frame_num_offset;
 923
 924     // (8-11)
 925     if (picture->is_idr)
 926         priv->frame_num_offset = 0;
 927     else if (priv->prev_frame_num > priv->frame_num)
 928         priv->frame_num_offset = prev_frame_num_offset + MaxFrameNum;
 929     else
 930         priv->frame_num_offset = prev_frame_num_offset;
 931
 932     // (8-12)
 933     if (picture->is_idr)
 934         temp_poc = 0;
 935     else if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
 936         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num) - 1;
 937     else
 938         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num);
 939
 940     // (8-13)
 941     if (!slice_hdr->field_pic_flag) {
 942         priv->field_poc[TOP_FIELD] = temp_poc;
 943         priv->field_poc[BOTTOM_FIELD] = temp_poc;
 944     }
 945     else if (slice_hdr->bottom_field_flag)
 946         priv->field_poc[BOTTOM_FIELD] = temp_poc;
 947     else
 948         priv->field_poc[TOP_FIELD] = temp_poc;
 949 }
 950
 951 /* 8.2.1 - Decoding process for picture order count */
 952 static void
 953 init_picture_poc(
 954     GstVaapiDecoderH264 *decoder,
 955     GstVaapiPictureH264 *picture,
 956     GstH264SliceHdr     *slice_hdr
 957 )
 958 {
 959     GstVaapiDecoderH264Private * const priv = decoder->priv;
 960     VAPictureH264 * const pic = &picture->info;
 961     GstH264PPS * const pps = slice_hdr->pps;
 962     GstH264SPS * const sps = pps->sequence;
 963
 964     switch (sps->pic_order_cnt_type) {
 965     case 0:
 966         init_picture_poc_0(decoder, picture, slice_hdr);
 967         break;
 968     case 1:
 969         init_picture_poc_1(decoder, picture, slice_hdr);
 970         break;
 971     case 2:
 972         init_picture_poc_2(decoder, picture, slice_hdr);
 973         break;
 974     }
 975
 976     if (!(pic->flags & VA_PICTURE_H264_BOTTOM_FIELD))
 977         pic->TopFieldOrderCnt = priv->field_poc[TOP_FIELD];
 978     if (!(pic->flags & VA_PICTURE_H264_TOP_FIELD))
 979         pic->BottomFieldOrderCnt = priv->field_poc[BOTTOM_FIELD];
 980     picture->poc = MIN(pic->TopFieldOrderCnt, pic->BottomFieldOrderCnt);
 981 }
 982
 983 static int
 984 compare_picture_pic_num_dec(const void *a, const void *b)
 985 {
 986     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
 987     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
 988
 989     return picB->pic_num - picA->pic_num;
 990 }
 991
 992 static int
 993 compare_picture_long_term_pic_num_inc(const void *a, const void *b)
 994 {
 995     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
 996     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
 997
 998     return picA->long_term_pic_num - picB->long_term_pic_num;
 999 }
1000
1001 static int
1002 compare_picture_poc_dec(const void *a, const void *b)
1003 {
1004     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1005     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1006
1007     return picB->poc - picA->poc;
1008 }
1009
1010 static int
1011 compare_picture_poc_inc(const void *a, const void *b)
1012 {
1013     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1014     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1015
1016     return picA->poc - picB->poc;
1017 }
1018
1019 static int
1020 compare_picture_frame_num_wrap_dec(const void *a, const void *b)
1021 {
1022     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1023     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1024
1025     return picB->frame_num_wrap - picA->frame_num_wrap;
1026 }
1027
1028 static int
1029 compare_picture_long_term_frame_idx_inc(const void *a, const void *b)
1030 {
1031     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1032     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1033
1034     return picA->info.frame_idx - picB->info.frame_idx;
1035 }
1036
1037 /* 8.2.4.1 - Decoding process for picture numbers */
1038 static void
1039 init_picture_refs_pic_num(
1040     GstVaapiDecoderH264 *decoder,
1041     GstVaapiPictureH264 *picture,
1042     GstH264SliceHdr     *slice_hdr
1043 )
1044 {
1045     GstVaapiDecoderH264Private * const priv = decoder->priv;
1046     GstH264PPS * const pps = slice_hdr->pps;
1047     GstH264SPS * const sps = pps->sequence;
1048     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1049     const guint field_flags = VA_PICTURE_H264_TOP_FIELD | VA_PICTURE_H264_BOTTOM_FIELD;
1050     guint i;
1051
1052     GST_DEBUG("decode picture numbers");
1053
1054     for (i = 0; i < priv->short_ref_count; i++) {
1055         GstVaapiPictureH264 * const pic = priv->short_ref[i];
1056
1057         // (8-27)
1058         if (pic->frame_num > priv->frame_num)
1059             pic->frame_num_wrap = pic->frame_num - MaxFrameNum;
1060         else
1061             pic->frame_num_wrap = pic->frame_num;
1062
1063         // (8-28, 8-30, 8-31)
1064         if (!pic->field_pic_flag)
1065             pic->pic_num = pic->frame_num_wrap;
1066         else {
1067             if (((picture->info.flags ^ pic->info.flags) & field_flags) == 0)
1068                 pic->pic_num = 2 * pic->frame_num_wrap + 1;
1069             else
1070                 pic->pic_num = 2 * pic->frame_num_wrap;
1071         }
1072     }
1073
1074     for (i = 0; i < priv->long_ref_count; i++) {
1075         GstVaapiPictureH264 * const pic = priv->long_ref[i];
1076
1077         // (8-29, 8-32, 8-33)
1078         if (!pic->field_pic_flag)
1079             pic->long_term_pic_num = pic->info.frame_idx;
1080         else {
1081             if (((picture->info.flags ^ pic->info.flags) & field_flags) == 0)
1082                 pic->long_term_pic_num = 2 * pic->info.frame_idx + 1;
1083             else
1084                 pic->long_term_pic_num = 2 * pic->info.frame_idx;
1085         }
1086     }
1087 }
1088
1089 #define SORT_REF_LIST(list, n, compare_func) \
1090     qsort(list, n, sizeof(*(list)), compare_picture_##compare_func)
1091
1092 static void
1093 init_picture_refs_p_slice(
1094     GstVaapiDecoderH264 *decoder,
1095     GstVaapiPictureH264 *picture,
1096     GstH264SliceHdr     *slice_hdr
1097 )
1098 {
1099     GstVaapiDecoderH264Private * const priv = decoder->priv;
1100     GstVaapiPictureH264 **ref_list;
1101     guint i;
1102
1103     GST_DEBUG("decode reference picture list for P and SP slices");
1104
1105     if (!picture->field_pic_flag) {
1106         /* 8.2.4.2.1 - P and SP slices in frames */
1107         if (priv->short_ref_count > 0) {
1108             ref_list = priv->RefPicList0;
1109             for (i = 0; i < priv->short_ref_count; i++)
1110                 ref_list[i] = priv->short_ref[i];
1111             SORT_REF_LIST(ref_list, i, pic_num_dec);
1112             priv->RefPicList0_count += i;
1113         }
1114
1115         if (priv->long_ref_count > 0) {
1116             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1117             for (i = 0; i < priv->long_ref_count; i++)
1118                 ref_list[i] = priv->long_ref[i];
1119             SORT_REF_LIST(ref_list, i, long_term_pic_num_inc);
1120             priv->RefPicList0_count += i;
1121         }
1122     }
1123     else {
1124         /* 8.2.4.2.2 - P and SP slices in fields */
1125         GstVaapiPictureH264 *short_ref[32];
1126         guint short_ref_count = 0;
1127         GstVaapiPictureH264 *long_ref[32];
1128         guint long_ref_count = 0;
1129
1130         // XXX: handle second field if current field is marked as
1131         // "used for short-term reference"
1132         if (priv->short_ref_count > 0) {
1133             for (i = 0; i < priv->short_ref_count; i++)
1134                 short_ref[i] = priv->short_ref[i];
1135             SORT_REF_LIST(short_ref, i, frame_num_wrap_dec);
1136             short_ref_count = i;
1137         }
1138
1139         // XXX: handle second field if current field is marked as
1140         // "used for long-term reference"
1141         if (priv->long_ref_count > 0) {
1142             for (i = 0; i < priv->long_ref_count; i++)
1143                 long_ref[i] = priv->long_ref[i];
1144             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1145             long_ref_count = i;
1146         }
1147
1148         // XXX: handle 8.2.4.2.5
1149     }
1150 }
1151
1152 static void
1153 init_picture_refs_b_slice(
1154     GstVaapiDecoderH264 *decoder,
1155     GstVaapiPictureH264 *picture,
1156     GstH264SliceHdr     *slice_hdr
1157 )
1158 {
1159     GstVaapiDecoderH264Private * const priv = decoder->priv;
1160     GstVaapiPictureH264 **ref_list;
1161     guint i, n;
1162
1163     GST_DEBUG("decode reference picture list for B slices");
1164
1165     if (!picture->field_pic_flag) {
1166         /* 8.2.4.2.3 - B slices in frames */
1167
1168         /* RefPicList0 */
1169         if (priv->short_ref_count > 0) {
1170             // 1. Short-term references
1171             ref_list = priv->RefPicList0;
1172             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1173                 if (priv->short_ref[i]->poc < picture->poc)
1174                     ref_list[n++] = priv->short_ref[i];
1175             }
1176             SORT_REF_LIST(ref_list, n, poc_dec);
1177             priv->RefPicList0_count += n;
1178
1179             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1180             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1181                 if (priv->short_ref[i]->poc >= picture->poc)
1182                     ref_list[n++] = priv->short_ref[i];
1183             }
1184             SORT_REF_LIST(ref_list, n, poc_inc);
1185             priv->RefPicList0_count += n;
1186         }
1187
1188         if (priv->long_ref_count > 0) {
1189             // 2. Long-term references
1190             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1191             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1192                 ref_list[n++] = priv->long_ref[i];
1193             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1194             priv->RefPicList0_count += n;
1195         }
1196
1197         /* RefPicList1 */
1198         if (priv->short_ref_count > 0) {
1199             // 1. Short-term references
1200             ref_list = priv->RefPicList1;
1201             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1202                 if (priv->short_ref[i]->poc > picture->poc)
1203                     ref_list[n++] = priv->short_ref[i];
1204             }
1205             SORT_REF_LIST(ref_list, n, poc_inc);
1206             priv->RefPicList1_count += n;
1207
1208             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1209             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1210                 if (priv->short_ref[i]->poc <= picture->poc)
1211                     ref_list[n++] = priv->short_ref[i];
1212             }
1213             SORT_REF_LIST(ref_list, n, poc_dec);
1214             priv->RefPicList1_count += n;
1215         }
1216
1217         if (priv->long_ref_count > 0) {
1218             // 2. Long-term references
1219             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1220             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1221                 ref_list[n++] = priv->long_ref[i];
1222             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1223             priv->RefPicList1_count += n;
1224         }
1225     }
1226     else {
1227         /* 8.2.4.2.4 - B slices in fields */
1228         GstVaapiPictureH264 *short_ref0[32];
1229         guint short_ref0_count = 0;
1230         GstVaapiPictureH264 *short_ref1[32];
1231         guint short_ref1_count = 0;
1232         GstVaapiPictureH264 *long_ref[32];
1233         guint long_ref_count = 0;
1234
1235         /* refFrameList0ShortTerm */
1236         if (priv->short_ref_count > 0) {
1237             ref_list = short_ref0;
1238             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1239                 if (priv->short_ref[i]->poc <= picture->poc)
1240                     ref_list[n++] = priv->short_ref[i];
1241             }
1242             SORT_REF_LIST(ref_list, n, poc_dec);
1243             short_ref0_count += n;
1244
1245             ref_list = &short_ref0[short_ref0_count];
1246             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1247                 if (priv->short_ref[i]->poc > picture->poc)
1248                     ref_list[n++] = priv->short_ref[i];
1249             }
1250             SORT_REF_LIST(ref_list, n, poc_inc);
1251             short_ref0_count += n;
1252         }
1253
1254         /* refFrameList1ShortTerm */
1255         if (priv->short_ref_count > 0) {
1256             ref_list = short_ref1;
1257             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1258                 if (priv->short_ref[i]->poc > picture->poc)
1259                     ref_list[n++] = priv->short_ref[i];
1260             }
1261             SORT_REF_LIST(ref_list, n, poc_inc);
1262             short_ref1_count += n;
1263
1264             ref_list = &short_ref1[short_ref1_count];
1265             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1266                 if (priv->short_ref[i]->poc <= picture->poc)
1267                     ref_list[n++] = priv->short_ref[i];
1268             }
1269             SORT_REF_LIST(ref_list, n, poc_dec);
1270             short_ref1_count += n;
1271         }
1272
1273         /* refFrameListLongTerm */
1274         if (priv->long_ref_count > 0) {
1275             for (i = 0; i < priv->long_ref_count; i++)
1276                 long_ref[i] = priv->long_ref[i];
1277             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1278             long_ref_count = i;
1279         }
1280
1281         // XXX: handle 8.2.4.2.5
1282     }
1283
1284     /* Check whether RefPicList1 is identical to RefPicList0, then
1285        swap if necessary */
1286     if (priv->RefPicList1_count > 1 &&
1287         priv->RefPicList1_count == priv->RefPicList0_count &&
1288         memcmp(priv->RefPicList0, priv->RefPicList1,
1289                priv->RefPicList0_count * sizeof(priv->RefPicList0[0])) == 0) {
1290         GstVaapiPictureH264 * const tmp = priv->RefPicList1[0];
1291         priv->RefPicList1[0] = priv->RefPicList1[1];
1292         priv->RefPicList1[1] = tmp;
1293     }
1294 }
1295
1296 #undef SORT_REF_LIST
1297
1298 static void
1299 clear_references(
1300     GstVaapiDecoderH264  *decoder,
1301     GstVaapiPictureH264 **pictures,
1302     guint                *picture_count
1303 )
1304 {
1305     const guint num_pictures = *picture_count;
1306     guint i;
1307
1308     for (i = 0; i < num_pictures; i++)
1309         gst_vaapi_picture_replace(&pictures[i], NULL);
1310     *picture_count = 0;
1311 }
1312
1313 static gboolean
1314 remove_reference_at(
1315     GstVaapiDecoderH264  *decoder,
1316     GstVaapiPictureH264 **pictures,
1317     guint                *picture_count,
1318     guint                 index
1319 )
1320 {
1321     guint num_pictures = *picture_count;
1322     GstVaapiPictureH264 *picture;
1323
1324     g_return_val_if_fail(index < num_pictures, FALSE);
1325
1326     picture = pictures[index];
1327     GST_VAAPI_PICTURE_FLAG_UNSET(picture, GST_VAAPI_PICTURE_FLAG_REFERENCE);
1328     picture->is_long_term = FALSE;
1329     picture->info.flags &= ~(VA_PICTURE_H264_SHORT_TERM_REFERENCE |
1330                              VA_PICTURE_H264_LONG_TERM_REFERENCE);
1331
1332     if (index != --num_pictures)
1333         gst_vaapi_picture_replace(&pictures[index], pictures[num_pictures]);
1334     gst_vaapi_picture_replace(&pictures[num_pictures], NULL);
1335     *picture_count = num_pictures;
1336     return TRUE;
1337 }
1338
1339 static gint
1340 find_short_term_reference(GstVaapiDecoderH264 *decoder, gint32 pic_num)
1341 {
1342     GstVaapiDecoderH264Private * const priv = decoder->priv;
1343     guint i;
1344
1345     for (i = 0; i < priv->short_ref_count; i++) {
1346         if (priv->short_ref[i]->pic_num == pic_num)
1347             return i;
1348     }
1349     GST_ERROR("found no short-term reference picture with PicNum = %d",
1350               pic_num);
1351     return -1;
1352 }
1353
1354 static gint
1355 find_long_term_reference(GstVaapiDecoderH264 *decoder, gint32 long_term_pic_num)
1356 {
1357     GstVaapiDecoderH264Private * const priv = decoder->priv;
1358     guint i;
1359
1360     for (i = 0; i < priv->long_ref_count; i++) {
1361         if (priv->long_ref[i]->long_term_pic_num == long_term_pic_num)
1362             return i;
1363     }
1364     GST_ERROR("found no long-term reference picture with LongTermPicNum = %d",
1365               long_term_pic_num);
1366     return -1;
1367 }
1368
1369 static void
1370 exec_picture_refs_modification_1(
1371     GstVaapiDecoderH264           *decoder,
1372     GstVaapiPictureH264           *picture,
1373     GstH264SliceHdr               *slice_hdr,
1374     guint                          list
1375 )
1376 {
1377     GstVaapiDecoderH264Private * const priv = decoder->priv;
1378     GstH264PPS * const pps = slice_hdr->pps;
1379     GstH264SPS * const sps = pps->sequence;
1380     GstH264RefPicListModification *ref_pic_list_modification;
1381     guint num_ref_pic_list_modifications;
1382     GstVaapiPictureH264 **ref_list;
1383     guint *ref_list_count_ptr, ref_list_count, ref_list_idx = 0;
1384     guint i, j, n, num_refs;
1385     gint found_ref_idx;
1386     gint32 MaxPicNum, CurrPicNum, picNumPred;
1387
1388     GST_DEBUG("modification process of reference picture list %u", list);
1389
1390     if (list == 0) {
1391         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l0;
1392         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l0;
1393         ref_list                       = priv->RefPicList0;
1394         ref_list_count_ptr             = &priv->RefPicList0_count;
1395         num_refs                       = slice_hdr->num_ref_idx_l0_active_minus1 + 1;
1396     }
1397     else {
1398         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l1;
1399         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l1;
1400         ref_list                       = priv->RefPicList1;
1401         ref_list_count_ptr             = &priv->RefPicList1_count;
1402         num_refs                       = slice_hdr->num_ref_idx_l1_active_minus1 + 1;
1403     }
1404     ref_list_count = *ref_list_count_ptr;
1405
1406     if (picture->field_pic_flag) {
1407         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 5); // 2 * MaxFrameNum
1408         CurrPicNum = 2 * slice_hdr->frame_num + 1;              // 2 * frame_num + 1
1409     }
1410     else {
1411         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 4); // MaxFrameNum
1412         CurrPicNum = slice_hdr->frame_num;                      // frame_num
1413     }
1414
1415     picNumPred = CurrPicNum;
1416
1417     for (i = 0; i < num_ref_pic_list_modifications; i++) {
1418         GstH264RefPicListModification * const l = &ref_pic_list_modification[i];
1419         if (l->modification_of_pic_nums_idc == 3)
1420             break;
1421
1422         /* 8.2.4.3.1 - Short-term reference pictures */
1423         if (l->modification_of_pic_nums_idc == 0 || l->modification_of_pic_nums_idc == 1) {
1424             gint32 abs_diff_pic_num = l->value.abs_diff_pic_num_minus1 + 1;
1425             gint32 picNum, picNumNoWrap;
1426
1427             // (8-34)
1428             if (l->modification_of_pic_nums_idc == 0) {
1429                 picNumNoWrap = picNumPred - abs_diff_pic_num;
1430                 if (picNumNoWrap < 0)
1431                     picNumNoWrap += MaxPicNum;
1432             }
1433
1434             // (8-35)
1435             else {
1436                 picNumNoWrap = picNumPred + abs_diff_pic_num;
1437                 if (picNumNoWrap >= MaxPicNum)
1438                     picNumNoWrap -= MaxPicNum;
1439             }
1440             picNumPred = picNumNoWrap;
1441
1442             // (8-36)
1443             picNum = picNumNoWrap;
1444             if (picNum > CurrPicNum)
1445                 picNum -= MaxPicNum;
1446
1447             // (8-37)
1448             for (j = num_refs; j > ref_list_idx; j--)
1449                 ref_list[j] = ref_list[j - 1];
1450             found_ref_idx = find_short_term_reference(decoder, picNum);
1451             ref_list[ref_list_idx++] =
1452                 found_ref_idx >= 0 ? priv->short_ref[found_ref_idx] : NULL;
1453             n = ref_list_idx;
1454             for (j = ref_list_idx; j <= num_refs; j++) {
1455                 gint32 PicNumF;
1456                 if (!ref_list[j])
1457                     continue;
1458                 PicNumF = ref_list[j]->is_long_term ?
1459                     MaxPicNum : ref_list[j]->pic_num;
1460                 if (PicNumF != picNum)
1461                     ref_list[n++] = ref_list[j];
1462             }
1463         }
1464
1465         /* 8.2.4.3.2 - Long-term reference pictures */
1466         else {
1467
1468             for (j = num_refs; j > ref_list_idx; j--)
1469                 ref_list[j] = ref_list[j - 1];
1470             found_ref_idx =
1471                 find_long_term_reference(decoder, l->value.long_term_pic_num);
1472             ref_list[ref_list_idx++] =
1473                 found_ref_idx >= 0 ? priv->long_ref[found_ref_idx] : NULL;
1474             n = ref_list_idx;
1475             for (j = ref_list_idx; j <= num_refs; j++) {
1476                 gint32 LongTermPicNumF;
1477                 if (!ref_list[j])
1478                     continue;
1479                 LongTermPicNumF = ref_list[j]->is_long_term ?
1480                     ref_list[j]->long_term_pic_num : INT_MAX;
1481                 if (LongTermPicNumF != l->value.long_term_pic_num)
1482                     ref_list[n++] = ref_list[j];
1483             }
1484         }
1485     }
1486
1487 #if DEBUG
1488     for (i = 0; i < num_refs; i++)
1489         if (!ref_list[i])
1490             GST_ERROR("list %u entry %u is empty", list, i);
1491 #endif
1492     *ref_list_count_ptr = num_refs;
1493 }
1494
1495 /* 8.2.4.3 - Modification process for reference picture lists */
1496 static void
1497 exec_picture_refs_modification(
1498     GstVaapiDecoderH264 *decoder,
1499     GstVaapiPictureH264 *picture,
1500     GstH264SliceHdr     *slice_hdr
1501 )
1502 {
1503     GST_DEBUG("execute ref_pic_list_modification()");
1504
1505     /* RefPicList0 */
1506     if (!GST_H264_IS_I_SLICE(slice_hdr) && !GST_H264_IS_SI_SLICE(slice_hdr) &&
1507         slice_hdr->ref_pic_list_modification_flag_l0)
1508         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 0);
1509
1510     /* RefPicList1 */
1511     if (GST_H264_IS_B_SLICE(slice_hdr) &&
1512         slice_hdr->ref_pic_list_modification_flag_l1)
1513         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 1);
1514 }
1515
1516 static gboolean
1517 init_picture_refs(
1518     GstVaapiDecoderH264 *decoder,
1519     GstVaapiPictureH264 *picture,
1520     GstH264SliceHdr     *slice_hdr
1521 )
1522 {
1523     GstVaapiDecoderH264Private * const priv = decoder->priv;
1524     GstVaapiPicture * const base_picture = &picture->base;
1525     guint i, num_refs;
1526
1527     init_picture_refs_pic_num(decoder, picture, slice_hdr);
1528
1529     priv->RefPicList0_count = 0;
1530     priv->RefPicList1_count = 0;
1531
1532     switch (base_picture->type) {
1533     case GST_VAAPI_PICTURE_TYPE_P:
1534     case GST_VAAPI_PICTURE_TYPE_SP:
1535         init_picture_refs_p_slice(decoder, picture, slice_hdr);
1536         break;
1537     case GST_VAAPI_PICTURE_TYPE_B:
1538         init_picture_refs_b_slice(decoder, picture, slice_hdr);
1539         break;
1540     default:
1541         break;
1542     }
1543
1544     exec_picture_refs_modification(decoder, picture, slice_hdr);
1545
1546     switch (base_picture->type) {
1547     case GST_VAAPI_PICTURE_TYPE_B:
1548         num_refs = 1 + slice_hdr->num_ref_idx_l1_active_minus1;
1549         for (i = priv->RefPicList1_count; i < num_refs; i++)
1550             priv->RefPicList1[i] = NULL;
1551         priv->RefPicList1_count = num_refs;
1552
1553         // fall-through
1554     case GST_VAAPI_PICTURE_TYPE_P:
1555     case GST_VAAPI_PICTURE_TYPE_SP:
1556         num_refs = 1 + slice_hdr->num_ref_idx_l0_active_minus1;
1557         for (i = priv->RefPicList0_count; i < num_refs; i++)
1558             priv->RefPicList0[i] = NULL;
1559         priv->RefPicList0_count = num_refs;
1560         break;
1561     default:
1562         break;
1563     }
1564     return TRUE;
1565 }
1566
1567 static gboolean
1568 init_picture(
1569     GstVaapiDecoderH264 *decoder,
1570     GstVaapiPictureH264 *picture,
1571     GstH264SliceHdr     *slice_hdr,
1572     GstH264NalUnit      *nalu
1573 )
1574 {
1575     GstVaapiDecoderH264Private * const priv = decoder->priv;
1576     GstVaapiPicture * const base_picture = &picture->base;
1577     VAPictureH264 *pic;
1578
1579     priv->prev_frame_num        = priv->frame_num;
1580     priv->frame_num             = slice_hdr->frame_num;
1581     picture->frame_num          = priv->frame_num;
1582     picture->frame_num_wrap     = priv->frame_num;
1583     picture->is_idr             = nalu->type == GST_H264_NAL_SLICE_IDR;
1584     picture->field_pic_flag     = slice_hdr->field_pic_flag;
1585     picture->bottom_field_flag  = slice_hdr->bottom_field_flag;
1586     picture->output_flag        = TRUE; /* XXX: conformant to Annex A only */
1587     base_picture->pts           = gst_adapter_prev_timestamp(priv->adapter, NULL);
1588
1589     /* Reset decoder state for IDR pictures */
1590     if (picture->is_idr) {
1591         GST_DEBUG("<IDR>");
1592         clear_references(decoder, priv->short_ref, &priv->short_ref_count);
1593         clear_references(decoder, priv->long_ref,  &priv->long_ref_count );
1594     }
1595
1596     /* Initialize VA picture info */
1597     pic = &picture->info;
1598     pic->picture_id = picture->base.surface_id;
1599     pic->frame_idx  = priv->frame_num;
1600     if (picture->field_pic_flag) {
1601         if (picture->bottom_field_flag)
1602             pic->flags |= VA_PICTURE_H264_BOTTOM_FIELD;
1603         else
1604             pic->flags |= VA_PICTURE_H264_TOP_FIELD;
1605     }
1606
1607     /* Initialize base picture */
1608     switch (slice_hdr->type % 5) {
1609     case GST_H264_P_SLICE:
1610         base_picture->type = GST_VAAPI_PICTURE_TYPE_P;
1611         break;
1612     case GST_H264_B_SLICE:
1613         base_picture->type = GST_VAAPI_PICTURE_TYPE_B;
1614         break;
1615     case GST_H264_I_SLICE:
1616         base_picture->type = GST_VAAPI_PICTURE_TYPE_I;
1617         break;
1618     case GST_H264_SP_SLICE:
1619         base_picture->type = GST_VAAPI_PICTURE_TYPE_SP;
1620         break;
1621     case GST_H264_SI_SLICE:
1622         base_picture->type = GST_VAAPI_PICTURE_TYPE_SI;
1623         break;
1624     }
1625
1626     if (nalu->ref_idc) {
1627         GstH264DecRefPicMarking * const dec_ref_pic_marking =
1628             &slice_hdr->dec_ref_pic_marking;
1629         GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_REFERENCE);
1630         if (picture->is_idr) {
1631             if (dec_ref_pic_marking->long_term_reference_flag)
1632                 picture->is_long_term = TRUE;
1633         }
1634     }
1635
1636     init_picture_poc(decoder, picture, slice_hdr);
1637     if (!init_picture_refs(decoder, picture, slice_hdr)) {
1638         GST_ERROR("failed to initialize references");
1639         return FALSE;
1640     }
1641     return TRUE;
1642 }
1643
1644 /* 8.2.5.3 - Sliding window decoded reference picture marking process */
1645 static gboolean
1646 exec_ref_pic_marking_sliding_window(GstVaapiDecoderH264 *decoder)
1647 {
1648     GstVaapiDecoderH264Private * const priv = decoder->priv;
1649     GstH264SPS * const sps = priv->sps;
1650     guint i, max_num_ref_frames, lowest_frame_num_index;
1651     gint32 lowest_frame_num;
1652
1653     GST_DEBUG("reference picture marking process (sliding window)");
1654
1655     max_num_ref_frames = sps->num_ref_frames;
1656     if (max_num_ref_frames == 0)
1657         max_num_ref_frames = 1;
1658
1659     if (priv->short_ref_count + priv->long_ref_count < max_num_ref_frames)
1660         return TRUE;
1661     if (priv->short_ref_count < 1)
1662         return FALSE;
1663
1664     lowest_frame_num = priv->short_ref[0]->frame_num_wrap;
1665     lowest_frame_num_index = 0;
1666     for (i = 1; i < priv->short_ref_count; i++) {
1667         if (priv->short_ref[i]->frame_num_wrap < lowest_frame_num) {
1668             lowest_frame_num = priv->short_ref[i]->frame_num_wrap;
1669             lowest_frame_num_index = i;
1670         }
1671     }
1672
1673     remove_reference_at(
1674         decoder,
1675         priv->short_ref, &priv->short_ref_count,
1676         lowest_frame_num_index
1677     );
1678     return TRUE;
1679 }
1680
1681 static inline gint32
1682 get_picNumX(GstVaapiPictureH264 *picture, GstH264RefPicMarking *ref_pic_marking)
1683 {
1684     gint32 pic_num;
1685
1686     if (!picture->field_pic_flag)
1687         pic_num = picture->frame_num_wrap;
1688     else
1689         pic_num = 2 * picture->frame_num_wrap + 1;
1690     pic_num -= ref_pic_marking->difference_of_pic_nums_minus1 + 1;
1691     return pic_num;
1692 }
1693
1694 /* 8.2.5.4.1. Mark-term reference picture as "unused for reference" */
1695 static void
1696 exec_ref_pic_marking_adaptive_mmco_1(
1697     GstVaapiDecoderH264  *decoder,
1698     GstVaapiPictureH264  *picture,
1699     GstH264RefPicMarking *ref_pic_marking
1700 )
1701 {
1702     GstVaapiDecoderH264Private * const priv = decoder->priv;
1703     gint32 i, picNumX;
1704
1705     picNumX = get_picNumX(picture, ref_pic_marking);
1706     i = find_short_term_reference(decoder, picNumX);
1707     if (i < 0)
1708         return;
1709     remove_reference_at(decoder, priv->short_ref, &priv->short_ref_count, i);
1710 }
1711
1712 /* 8.2.5.4.2. Mark long-term reference picture as "unused for reference" */
1713 static void
1714 exec_ref_pic_marking_adaptive_mmco_2(
1715     GstVaapiDecoderH264  *decoder,
1716     GstVaapiPictureH264  *picture,
1717     GstH264RefPicMarking *ref_pic_marking
1718 )
1719 {
1720     GstVaapiDecoderH264Private * const priv = decoder->priv;
1721     gint32 i;
1722
1723     i = find_long_term_reference(decoder, ref_pic_marking->long_term_pic_num);
1724     if (i < 0)
1725         return;
1726     remove_reference_at(decoder, priv->long_ref, &priv->long_ref_count, i);
1727 }
1728
1729 /* 8.2.5.4.3. Assign LongTermFrameIdx to a short-term reference picture */
1730 static void
1731 exec_ref_pic_marking_adaptive_mmco_3(
1732     GstVaapiDecoderH264  *decoder,
1733     GstVaapiPictureH264  *picture,
1734     GstH264RefPicMarking *ref_pic_marking
1735 )
1736 {
1737     GstVaapiDecoderH264Private * const priv = decoder->priv;
1738     VAPictureH264 *pic;
1739     gint32 i, picNumX;
1740
1741     for (i = 0; i < priv->long_ref_count; i++) {
1742         if (priv->long_ref[i]->info.frame_idx == ref_pic_marking->long_term_frame_idx)
1743             break;
1744     }
1745     if (i != priv->long_ref_count)
1746         remove_reference_at(decoder, priv->long_ref, &priv->long_ref_count, i);
1747
1748     picNumX = get_picNumX(picture, ref_pic_marking);
1749     i = find_short_term_reference(decoder, picNumX);
1750     if (i < 0)
1751         return;
1752
1753     picture = gst_vaapi_picture_ref(priv->short_ref[i]);
1754     remove_reference_at(decoder, priv->short_ref, &priv->short_ref_count, i);
1755     gst_vaapi_picture_replace(&priv->long_ref[priv->long_ref_count++], picture);
1756     gst_vaapi_picture_unref(picture);
1757
1758     picture->is_long_term = TRUE;
1759     pic = &picture->info;
1760     pic->frame_idx = ref_pic_marking->long_term_frame_idx;
1761     pic->flags &= ~VA_PICTURE_H264_SHORT_TERM_REFERENCE;
1762     pic->flags |= VA_PICTURE_H264_LONG_TERM_REFERENCE;
1763     GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_REFERENCE);
1764 }
1765
1766 /* 8.2.5.4.4. Mark pictures with LongTermFramIdx > max_long_term_frame_idx
1767  * as "unused for reference" */
1768 static void
1769 exec_ref_pic_marking_adaptive_mmco_4(
1770     GstVaapiDecoderH264  *decoder,
1771     GstVaapiPictureH264  *picture,
1772     GstH264RefPicMarking *ref_pic_marking
1773 )
1774 {
1775     GstVaapiDecoderH264Private * const priv = decoder->priv;
1776     gint32 i, long_term_frame_idx;
1777
1778     long_term_frame_idx = ref_pic_marking->max_long_term_frame_idx_plus1 - 1;
1779
1780     for (i = 0; i < priv->long_ref_count; i++) {
1781         if ((gint32)priv->long_ref[i]->info.frame_idx <= long_term_frame_idx)
1782             continue;
1783         remove_reference_at(decoder, priv->long_ref, &priv->long_ref_count, i);
1784         i--;
1785     }
1786 }
1787
1788 /* 8.2.5.4.5. Mark all reference pictures as "unused for reference" */
1789 static void
1790 exec_ref_pic_marking_adaptive_mmco_5(
1791     GstVaapiDecoderH264  *decoder,
1792     GstVaapiPictureH264  *picture,
1793     GstH264RefPicMarking *ref_pic_marking
1794 )
1795 {
1796     GstVaapiDecoderH264Private * const priv = decoder->priv;
1797     VAPictureH264 * const pic = &picture->info;
1798
1799     clear_references(decoder, priv->short_ref, &priv->short_ref_count);
1800     clear_references(decoder, priv->long_ref,  &priv->long_ref_count );
1801     dpb_flush(decoder);
1802
1803     priv->prev_pic_has_mmco5 = TRUE;
1804
1805     /* The picture shall be inferred to have had frame_num equal to 0 (7.4.3) */
1806     priv->frame_num = 0;
1807     priv->frame_num_offset = 0;
1808     picture->frame_num = 0;
1809
1810     /* Update TopFieldOrderCnt and BottomFieldOrderCnt (8.2.1) */
1811     if (!(pic->flags & VA_PICTURE_H264_BOTTOM_FIELD))
1812         pic->TopFieldOrderCnt -= picture->poc;
1813     if (!(pic->flags & VA_PICTURE_H264_TOP_FIELD))
1814         pic->BottomFieldOrderCnt -= picture->poc;
1815     picture->poc = 0;
1816 }
1817
1818 /* 8.2.5.4.6. Assign a long-term frame index to the current picture */
1819 static void
1820 exec_ref_pic_marking_adaptive_mmco_6(
1821     GstVaapiDecoderH264  *decoder,
1822     GstVaapiPictureH264  *picture,
1823     GstH264RefPicMarking *ref_pic_marking
1824 )
1825 {
1826     picture->is_long_term = TRUE;
1827     picture->info.frame_idx = ref_pic_marking->long_term_frame_idx;
1828 }
1829
1830 /* 8.2.5.4. Adaptive memory control decoded reference picture marking process */
1831 static gboolean
1832 exec_ref_pic_marking_adaptive(
1833     GstVaapiDecoderH264     *decoder,
1834     GstVaapiPictureH264     *picture,
1835     GstH264DecRefPicMarking *dec_ref_pic_marking
1836 )
1837 {
1838     guint i;
1839
1840     GST_DEBUG("reference picture marking process (adaptive memory control)");
1841
1842     typedef void (*exec_ref_pic_marking_adaptive_mmco_func)(
1843         GstVaapiDecoderH264  *decoder,
1844         GstVaapiPictureH264  *picture,
1845         GstH264RefPicMarking *ref_pic_marking
1846     );
1847
1848     static const exec_ref_pic_marking_adaptive_mmco_func mmco_funcs[] = {
1849         NULL,
1850         exec_ref_pic_marking_adaptive_mmco_1,
1851         exec_ref_pic_marking_adaptive_mmco_2,
1852         exec_ref_pic_marking_adaptive_mmco_3,
1853         exec_ref_pic_marking_adaptive_mmco_4,
1854         exec_ref_pic_marking_adaptive_mmco_5,
1855         exec_ref_pic_marking_adaptive_mmco_6,
1856     };
1857
1858     for (i = 0; i < dec_ref_pic_marking->n_ref_pic_marking; i++) {
1859         GstH264RefPicMarking * const ref_pic_marking =
1860             &dec_ref_pic_marking->ref_pic_marking[i];
1861
1862         const guint mmco = ref_pic_marking->memory_management_control_operation;
1863         if (mmco < G_N_ELEMENTS(mmco_funcs) && mmco_funcs[mmco])
1864             mmco_funcs[mmco](decoder, picture, ref_pic_marking);
1865         else {
1866             GST_ERROR("unhandled MMCO %u", mmco);
1867             return FALSE;
1868         }
1869     }
1870     return TRUE;
1871 }
1872
1873 /* 8.2.5 - Execute reference picture marking process */
1874 static gboolean
1875 exec_ref_pic_marking(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
1876 {
1877     GstVaapiDecoderH264Private * const priv = decoder->priv;
1878     GstVaapiPictureH264 **picture_ptr;
1879
1880     priv->prev_pic_has_mmco5 = FALSE;
1881     priv->prev_pic_bottom_field =
1882         picture->field_pic_flag && picture->bottom_field_flag;
1883
1884     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
1885         return TRUE;
1886
1887     if (!picture->is_idr) {
1888         GstH264DecRefPicMarking * const dec_ref_pic_marking =
1889             get_dec_ref_pic_marking(picture);
1890         if (dec_ref_pic_marking->adaptive_ref_pic_marking_mode_flag) {
1891             if (!exec_ref_pic_marking_adaptive(decoder, picture, dec_ref_pic_marking))
1892                 return FALSE;
1893         }
1894         else {
1895             if (!exec_ref_pic_marking_sliding_window(decoder))
1896                 return FALSE;
1897         }
1898     }
1899
1900     if (picture->is_long_term) {
1901         picture_ptr = &priv->long_ref[priv->long_ref_count++];
1902         picture->info.flags |= VA_PICTURE_H264_LONG_TERM_REFERENCE;
1903     }
1904     else {
1905         picture_ptr = &priv->short_ref[priv->short_ref_count++];
1906         picture->info.flags |= VA_PICTURE_H264_SHORT_TERM_REFERENCE;
1907     }
1908     gst_vaapi_picture_replace(picture_ptr, picture);
1909     return TRUE;
1910 }
1911
1912 static void
1913 vaapi_init_picture(VAPictureH264 *pic)
1914 {
1915     pic->picture_id           = VA_INVALID_ID;
1916     pic->frame_idx            = 0;
1917     pic->flags                = VA_PICTURE_H264_INVALID;
1918     pic->TopFieldOrderCnt     = 0;
1919     pic->BottomFieldOrderCnt  = 0;
1920 }
1921
1922 static gboolean
1923 fill_picture(
1924     GstVaapiDecoderH264 *decoder,
1925     GstVaapiPictureH264 *picture,
1926     GstH264SliceHdr     *slice_hdr,
1927     GstH264NalUnit      *nalu
1928 )
1929 {
1930     GstVaapiDecoderH264Private * const priv = decoder->priv;
1931     GstVaapiPicture * const base_picture = &picture->base;
1932     GstH264SPS * const sps = priv->sps;
1933     GstH264PPS * const pps = priv->pps;
1934     VAPictureParameterBufferH264 * const pic_param = base_picture->param;
1935     guint i, n;
1936
1937     /* Fill in VAPictureParameterBufferH264 */
1938     pic_param->CurrPic = picture->info;
1939     for (i = 0, n = 0; i < priv->short_ref_count; i++)
1940         pic_param->ReferenceFrames[n++] = priv->short_ref[i]->info;
1941     for (i = 0; i < priv->long_ref_count; i++)
1942         pic_param->ReferenceFrames[n++] = priv->long_ref[i]->info;
1943     for (; n < G_N_ELEMENTS(pic_param->ReferenceFrames); n++)
1944         vaapi_init_picture(&pic_param->ReferenceFrames[n]);
1945
1946 #define COPY_FIELD(s, f) \
1947     pic_param->f = (s)->f
1948
1949 #define COPY_BFM(a, s, f) \
1950     pic_param->a.bits.f = (s)->f
1951
1952     pic_param->picture_width_in_mbs_minus1      = priv->mb_width  - 1;
1953     pic_param->picture_height_in_mbs_minus1     = priv->mb_height - 1;
1954     pic_param->frame_num                        = priv->frame_num;
1955
1956     COPY_FIELD(sps, bit_depth_luma_minus8);
1957     COPY_FIELD(sps, bit_depth_chroma_minus8);
1958     COPY_FIELD(sps, num_ref_frames);
1959     COPY_FIELD(pps, num_slice_groups_minus1);
1960     COPY_FIELD(pps, slice_group_map_type);
1961     COPY_FIELD(pps, slice_group_change_rate_minus1);
1962     COPY_FIELD(pps, pic_init_qp_minus26);
1963     COPY_FIELD(pps, pic_init_qs_minus26);
1964     COPY_FIELD(pps, chroma_qp_index_offset);
1965     COPY_FIELD(pps, second_chroma_qp_index_offset);
1966
1967     pic_param->seq_fields.value                                         = 0; /* reset all bits */
1968     pic_param->seq_fields.bits.residual_colour_transform_flag           = sps->separate_colour_plane_flag;
1969     pic_param->seq_fields.bits.MinLumaBiPredSize8x8                     = sps->level_idc >= 31; /* A.3.3.2 */
1970
1971     COPY_BFM(seq_fields, sps, chroma_format_idc);
1972     COPY_BFM(seq_fields, sps, gaps_in_frame_num_value_allowed_flag);
1973     COPY_BFM(seq_fields, sps, frame_mbs_only_flag);
1974     COPY_BFM(seq_fields, sps, mb_adaptive_frame_field_flag);
1975     COPY_BFM(seq_fields, sps, direct_8x8_inference_flag);
1976     COPY_BFM(seq_fields, sps, log2_max_frame_num_minus4);
1977     COPY_BFM(seq_fields, sps, pic_order_cnt_type);
1978     COPY_BFM(seq_fields, sps, log2_max_pic_order_cnt_lsb_minus4);
1979     COPY_BFM(seq_fields, sps, delta_pic_order_always_zero_flag);
1980
1981     pic_param->pic_fields.value                                         = 0; /* reset all bits */
1982     pic_param->pic_fields.bits.field_pic_flag                           = slice_hdr->field_pic_flag;
1983     pic_param->pic_fields.bits.reference_pic_flag                       = GST_VAAPI_PICTURE_IS_REFERENCE(picture);
1984
1985     COPY_BFM(pic_fields, pps, entropy_coding_mode_flag);
1986     COPY_BFM(pic_fields, pps, weighted_pred_flag);
1987     COPY_BFM(pic_fields, pps, weighted_bipred_idc);
1988     COPY_BFM(pic_fields, pps, transform_8x8_mode_flag);
1989     COPY_BFM(pic_fields, pps, constrained_intra_pred_flag);
1990     COPY_BFM(pic_fields, pps, pic_order_present_flag);
1991     COPY_BFM(pic_fields, pps, deblocking_filter_control_present_flag);
1992     COPY_BFM(pic_fields, pps, redundant_pic_cnt_present_flag);
1993     return TRUE;
1994 }
1995
1996 static gboolean
1997 fill_quant_matrix(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
1998 {
1999     GstVaapiDecoderH264Private * const priv = decoder->priv;
2000     VAIQMatrixBufferH264 * const iq_matrix = picture->base.iq_matrix->param;
2001
2002     /* XXX: we can only support 4:2:0 or 4:2:2 since ScalingLists8x8[]
2003        is not large enough to hold lists for 4:4:4 */
2004     if (priv->sps->chroma_format_idc == 3 &&
2005         sizeof(iq_matrix->ScalingList8x8) != sizeof(priv->scaling_list_8x8))
2006         return FALSE;
2007
2008     /* Fill in VAIQMatrixBufferH264 */
2009     memcpy(iq_matrix->ScalingList4x4, priv->scaling_list_4x4,
2010            sizeof(iq_matrix->ScalingList4x4));
2011     memcpy(iq_matrix->ScalingList8x8, priv->scaling_list_8x8,
2012            sizeof(iq_matrix->ScalingList8x8));
2013     return TRUE;
2014 }
2015
2016 static GstVaapiDecoderStatus
2017 decode_picture(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu, GstH264SliceHdr *slice_hdr)
2018 {
2019     GstVaapiDecoderH264Private * const priv = decoder->priv;
2020     GstVaapiPictureH264 *picture;
2021     GstVaapiDecoderStatus status;
2022     GstH264PPS * const pps = slice_hdr->pps;
2023     GstH264SPS * const sps = pps->sequence;
2024
2025     status = ensure_context(decoder, sps);
2026     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS) {
2027         GST_ERROR("failed to reset context");
2028         return status;
2029     }
2030
2031     if (priv->current_picture && !decode_current_picture(decoder))
2032         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2033
2034     picture = gst_vaapi_picture_h264_new(decoder);
2035     if (!picture) {
2036         GST_ERROR("failed to allocate picture");
2037         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2038     }
2039     priv->current_picture = picture;
2040
2041     picture->base.iq_matrix = GST_VAAPI_IQ_MATRIX_NEW(H264, decoder);
2042     if (!picture->base.iq_matrix) {
2043         GST_ERROR("failed to allocate IQ matrix");
2044         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2045     }
2046
2047     status = ensure_quant_matrix(decoder, pps);
2048     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS) {
2049         GST_ERROR("failed to reset quantizer matrix");
2050         return status;
2051     }
2052
2053     priv->sps = sps;
2054     priv->pps = pps;
2055
2056     if (!init_picture(decoder, picture, slice_hdr, nalu))
2057         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2058     if (!fill_picture(decoder, picture, slice_hdr, nalu))
2059         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2060     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2061 }
2062
2063 static gboolean
2064 decode_picture_end(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
2065 {
2066     if (!fill_quant_matrix(decoder, picture))
2067         return FALSE;
2068     if (!exec_ref_pic_marking(decoder, picture))
2069         return FALSE;
2070     if (!dpb_add(decoder, picture))
2071         return FALSE;
2072     return TRUE;
2073 }
2074
2075 static inline guint
2076 get_slice_data_bit_offset(GstH264SliceHdr *slice_hdr, GstH264NalUnit *nalu)
2077 {
2078     guint epb_count;
2079
2080     epb_count = slice_hdr->n_emulation_prevention_bytes;
2081     return 8 /* nal_unit_type */ + slice_hdr->header_size - epb_count * 8;
2082 }
2083
2084 static gboolean
2085 fill_pred_weight_table(GstVaapiDecoderH264 *decoder, GstVaapiSliceH264 *slice)
2086 {
2087     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2088     GstH264PPS * const pps = slice_hdr->pps;
2089     GstH264SPS * const sps = pps->sequence;
2090     GstH264PredWeightTable * const w = &slice_hdr->pred_weight_table;
2091     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2092     guint num_weight_tables = 0;
2093     gint i, j;
2094
2095     if (pps->weighted_pred_flag &&
2096         (GST_H264_IS_P_SLICE(slice_hdr) || GST_H264_IS_SP_SLICE(slice_hdr)))
2097         num_weight_tables = 1;
2098     else if (pps->weighted_bipred_idc == 1 && GST_H264_IS_B_SLICE(slice_hdr))
2099         num_weight_tables = 2;
2100     else
2101         num_weight_tables = 0;
2102
2103     slice_param->luma_log2_weight_denom   = w->luma_log2_weight_denom;
2104     slice_param->chroma_log2_weight_denom = w->chroma_log2_weight_denom;
2105     slice_param->luma_weight_l0_flag      = 0;
2106     slice_param->chroma_weight_l0_flag    = 0;
2107     slice_param->luma_weight_l1_flag      = 0;
2108     slice_param->chroma_weight_l1_flag    = 0;
2109
2110     if (num_weight_tables < 1)
2111         return TRUE;
2112
2113     slice_param->luma_weight_l0_flag = 1;
2114     for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2115         slice_param->luma_weight_l0[i] = w->luma_weight_l0[i];
2116         slice_param->luma_offset_l0[i] = w->luma_offset_l0[i];
2117     }
2118
2119     slice_param->chroma_weight_l0_flag = sps->chroma_array_type != 0;
2120     if (slice_param->chroma_weight_l0_flag) {
2121         for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2122             for (j = 0; j < 2; j++) {
2123                 slice_param->chroma_weight_l0[i][j] = w->chroma_weight_l0[i][j];
2124                 slice_param->chroma_offset_l0[i][j] = w->chroma_offset_l0[i][j];
2125             }
2126         }
2127     }
2128
2129     if (num_weight_tables < 2)
2130         return TRUE;
2131
2132     slice_param->luma_weight_l1_flag = 1;
2133     for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2134         slice_param->luma_weight_l1[i] = w->luma_weight_l1[i];
2135         slice_param->luma_offset_l1[i] = w->luma_offset_l1[i];
2136     }
2137
2138     slice_param->chroma_weight_l1_flag = sps->chroma_array_type != 0;
2139     if (slice_param->chroma_weight_l1_flag) {
2140         for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2141             for (j = 0; j < 2; j++) {
2142                 slice_param->chroma_weight_l1[i][j] = w->chroma_weight_l1[i][j];
2143                 slice_param->chroma_offset_l1[i][j] = w->chroma_offset_l1[i][j];
2144             }
2145         }
2146     }
2147     return TRUE;
2148 }
2149
2150 static gboolean
2151 fill_RefPicList(GstVaapiDecoderH264 *decoder, GstVaapiSliceH264 *slice)
2152 {
2153     GstVaapiDecoderH264Private * const priv = decoder->priv;
2154     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2155     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2156     guint i, num_ref_lists = 0;
2157
2158     slice_param->num_ref_idx_l0_active_minus1 = 0;
2159     slice_param->num_ref_idx_l1_active_minus1 = 0;
2160
2161     if (GST_H264_IS_B_SLICE(slice_hdr))
2162         num_ref_lists = 2;
2163     else if (GST_H264_IS_I_SLICE(slice_hdr))
2164         num_ref_lists = 0;
2165     else
2166         num_ref_lists = 1;
2167
2168     if (num_ref_lists < 1)
2169         return TRUE;
2170
2171     slice_param->num_ref_idx_l0_active_minus1 =
2172         slice_hdr->num_ref_idx_l0_active_minus1;
2173
2174     for (i = 0; i < priv->RefPicList0_count && priv->RefPicList0[i]; i++)
2175         slice_param->RefPicList0[i] = priv->RefPicList0[i]->info;
2176     for (; i <= slice_param->num_ref_idx_l0_active_minus1; i++)
2177         vaapi_init_picture(&slice_param->RefPicList0[i]);
2178
2179     if (num_ref_lists < 2)
2180         return TRUE;
2181
2182     slice_param->num_ref_idx_l1_active_minus1 =
2183         slice_hdr->num_ref_idx_l1_active_minus1;
2184
2185     for (i = 0; i < priv->RefPicList1_count && priv->RefPicList1[i]; i++)
2186         slice_param->RefPicList1[i] = priv->RefPicList1[i]->info;
2187     for (; i <= slice_param->num_ref_idx_l1_active_minus1; i++)
2188         vaapi_init_picture(&slice_param->RefPicList1[i]);
2189     return TRUE;
2190 }
2191
2192 static gboolean
2193 fill_slice(
2194     GstVaapiDecoderH264 *decoder,
2195     GstVaapiSliceH264   *slice,
2196     GstH264NalUnit      *nalu
2197 )
2198 {
2199     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2200     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2201
2202     /* Fill in VASliceParameterBufferH264 */
2203     slice_param->slice_data_bit_offset          = get_slice_data_bit_offset(slice_hdr, nalu);
2204     slice_param->first_mb_in_slice              = slice_hdr->first_mb_in_slice;
2205     slice_param->slice_type                     = slice_hdr->type % 5;
2206     slice_param->direct_spatial_mv_pred_flag    = slice_hdr->direct_spatial_mv_pred_flag;
2207     slice_param->cabac_init_idc                 = slice_hdr->cabac_init_idc;
2208     slice_param->slice_qp_delta                 = slice_hdr->slice_qp_delta;
2209     slice_param->disable_deblocking_filter_idc  = slice_hdr->disable_deblocking_filter_idc;
2210     slice_param->slice_alpha_c0_offset_div2     = slice_hdr->slice_alpha_c0_offset_div2;
2211     slice_param->slice_beta_offset_div2         = slice_hdr->slice_beta_offset_div2;
2212
2213     if (!fill_RefPicList(decoder, slice))
2214         return FALSE;
2215     if (!fill_pred_weight_table(decoder, slice))
2216         return FALSE;
2217     return TRUE;
2218 }
2219
2220 static GstVaapiDecoderStatus
2221 decode_slice(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
2222 {
2223     GstVaapiDecoderH264Private * const priv = decoder->priv;
2224     GstVaapiDecoderStatus status;
2225     GstVaapiPictureH264 *picture;
2226     GstVaapiSliceH264 *slice = NULL;
2227     GstH264SliceHdr *slice_hdr;
2228     GstH264ParserResult result;
2229
2230     GST_DEBUG("slice (%u bytes)", nalu->size);
2231
2232     slice = gst_vaapi_slice_h264_new(
2233         decoder,
2234         nalu->data + nalu->offset,
2235         nalu->size
2236     );
2237     if (!slice) {
2238         GST_ERROR("failed to allocate slice");
2239         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2240     }
2241
2242     slice_hdr = &slice->slice_hdr;
2243     memset(slice_hdr, 0, sizeof(*slice_hdr));
2244     result = gst_h264_parser_parse_slice_hdr(priv->parser, nalu, slice_hdr, TRUE, TRUE);
2245     if (result != GST_H264_PARSER_OK) {
2246         status = get_status(result);
2247         goto error;
2248     }
2249
2250     if (slice_hdr->first_mb_in_slice == 0) {
2251         status = decode_picture(decoder, nalu, slice_hdr);
2252         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2253             goto error;
2254     }
2255     picture = priv->current_picture;
2256
2257     priv->mb_x = slice_hdr->first_mb_in_slice % priv->mb_width;
2258     priv->mb_y = slice_hdr->first_mb_in_slice / priv->mb_width; // FIXME: MBAFF or field
2259
2260     if (!fill_slice(decoder, slice, nalu)) {
2261         status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2262         goto error;
2263     }
2264     gst_vaapi_picture_add_slice(
2265         GST_VAAPI_PICTURE_CAST(picture),
2266         GST_VAAPI_SLICE_CAST(slice)
2267     );
2268
2269     /* Commit picture for decoding if we reached the last slice */
2270     if (++priv->mb_y >= priv->mb_height) {
2271         if (!decode_current_picture(decoder)) {
2272             status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2273             goto error;
2274         }
2275         GST_DEBUG("done");
2276     }
2277     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2278
2279 error:
2280     if (slice)
2281         gst_mini_object_unref(GST_MINI_OBJECT(slice));
2282     return status;
2283 }
2284
2285 static inline gint
2286 scan_for_start_code(GstAdapter *adapter, guint ofs, guint size, guint32 *scp)
2287 {
2288     return (gint)gst_adapter_masked_scan_uint32_peek(adapter,
2289                                                      0xffffff00, 0x00000100,
2290                                                      ofs, size,
2291                                                      scp);
2292 }
2293
2294 static GstVaapiDecoderStatus
2295 decode_nalu(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
2296 {
2297     GstVaapiDecoderStatus status;
2298
2299     switch (nalu->type) {
2300     case GST_H264_NAL_SLICE_IDR:
2301         /* fall-through. IDR specifics are handled in init_picture() */
2302     case GST_H264_NAL_SLICE:
2303         status = decode_slice(decoder, nalu);
2304         break;
2305     case GST_H264_NAL_SPS:
2306         status = decode_sps(decoder, nalu);
2307         break;
2308     case GST_H264_NAL_PPS:
2309         status = decode_pps(decoder, nalu);
2310         break;
2311     case GST_H264_NAL_SEI:
2312         status = decode_sei(decoder, nalu);
2313         break;
2314     case GST_H264_NAL_SEQ_END:
2315         status = decode_sequence_end(decoder);
2316         break;
2317     case GST_H264_NAL_AU_DELIMITER:
2318         /* skip all Access Unit NALs */
2319         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2320         break;
2321     case GST_H264_NAL_FILLER_DATA:
2322         /* skip all Filler Data NALs */
2323         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2324         break;
2325     default:
2326         GST_WARNING("unsupported NAL unit type %d", nalu->type);
2327         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
2328         break;
2329     }
2330     return status;
2331 }
2332
2333 static GstVaapiDecoderStatus
2334 decode_buffer(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
2335 {
2336     GstVaapiDecoderH264Private * const priv = decoder->priv;
2337     GstVaapiDecoderStatus status;
2338     GstH264ParserResult result;
2339     GstH264NalUnit nalu;
2340     gboolean is_eos;
2341     const guchar *buf;
2342     guint i, buf_size, nalu_size, size;
2343     guint32 start_code;
2344     gint ofs;
2345
2346     buf      = GST_BUFFER_DATA(buffer);
2347     buf_size = GST_BUFFER_SIZE(buffer);
2348     is_eos   = GST_BUFFER_IS_EOS(buffer);
2349     if (buf && buf_size > 0)
2350         gst_adapter_push(priv->adapter, gst_buffer_ref(buffer));
2351
2352     size = gst_adapter_available(priv->adapter);
2353     do {
2354         if (size == 0) {
2355             status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2356             break;
2357         }
2358
2359         status = gst_vaapi_decoder_check_status(GST_VAAPI_DECODER(decoder));
2360         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2361             break;
2362
2363         status = GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2364         if (priv->is_avc) {
2365             if (size < priv->nal_length_size)
2366                 break;
2367             buf = gst_adapter_peek(priv->adapter, priv->nal_length_size);
2368
2369             nalu_size = 0;
2370             for (i = 0; i < priv->nal_length_size; i++)
2371                 nalu_size = (nalu_size << 8) | buf[i];
2372
2373             buf_size = priv->nal_length_size + nalu_size;
2374             if (size < buf_size)
2375                 break;
2376             buffer = gst_adapter_take_buffer(priv->adapter, buf_size);
2377             size -= buf_size;
2378
2379             buf      = GST_BUFFER_DATA(buffer);
2380             buf_size = GST_BUFFER_SIZE(buffer);
2381
2382             result = gst_h264_parser_identify_nalu_avc(
2383                 priv->parser,
2384                 buf, 0, buf_size, priv->nal_length_size,
2385                 &nalu
2386             );
2387         }
2388         else {
2389             if (size < 4)
2390                 break;
2391             ofs = scan_for_start_code(priv->adapter, 0, size, &start_code);
2392             if (ofs < 0)
2393                 break;
2394             gst_adapter_flush(priv->adapter, ofs);
2395             size -= ofs;
2396
2397             ofs = G_UNLIKELY(size < 8) ? -1 :
2398                 scan_for_start_code(priv->adapter, 4, size - 4, NULL);
2399             if (ofs < 0) {
2400                 // Assume the whole NAL unit is present if end-of-stream
2401                 if (!is_eos)
2402                     break;
2403                 ofs = size;
2404             }
2405             buffer = gst_adapter_take_buffer(priv->adapter, ofs);
2406             size -= ofs;
2407
2408             buf      = GST_BUFFER_DATA(buffer);
2409             buf_size = GST_BUFFER_SIZE(buffer);
2410
2411             result = gst_h264_parser_identify_nalu_unchecked(
2412                 priv->parser,
2413                 buf, 0, buf_size,
2414                 &nalu
2415             );
2416         }
2417         status = get_status(result);
2418         if (status == GST_VAAPI_DECODER_STATUS_SUCCESS)
2419             status = decode_nalu(decoder, &nalu);
2420         gst_buffer_unref(buffer);
2421     } while (status == GST_VAAPI_DECODER_STATUS_SUCCESS);
2422
2423     if (is_eos && (status == GST_VAAPI_DECODER_STATUS_SUCCESS ||
2424                    status == GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA))
2425         status = decode_sequence_end(decoder);
2426     return status;
2427 }
2428
2429 static GstVaapiDecoderStatus
2430 decode_codec_data(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
2431 {
2432     GstVaapiDecoderH264Private * const priv = decoder->priv;
2433     GstVaapiDecoderStatus status;
2434     GstH264NalUnit nalu;
2435     GstH264ParserResult result;
2436     guchar *buf;
2437     guint buf_size;
2438     guint i, ofs, num_sps, num_pps;
2439
2440     buf      = GST_BUFFER_DATA(buffer);
2441     buf_size = GST_BUFFER_SIZE(buffer);
2442     if (!buf || buf_size == 0)
2443         return GST_VAAPI_DECODER_STATUS_SUCCESS;
2444
2445     if (buf_size < 8)
2446         return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2447
2448     if (buf[0] != 1) {
2449         GST_ERROR("failed to decode codec-data, not in avcC format");
2450         return GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
2451     }
2452
2453     priv->nal_length_size = (buf[4] & 0x03) + 1;
2454
2455     num_sps = buf[5] & 0x1f;
2456     ofs = 6;
2457
2458     for (i = 0; i < num_sps; i++) {
2459         result = gst_h264_parser_identify_nalu_avc(
2460             priv->parser,
2461             buf, ofs, buf_size, 2,
2462             &nalu
2463         );
2464         if (result != GST_H264_PARSER_OK)
2465             return get_status(result);
2466
2467         status = decode_sps(decoder, &nalu);
2468         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2469             return status;
2470         ofs = nalu.offset + nalu.size;
2471     }
2472
2473     num_pps = buf[ofs];
2474     ofs++;
2475
2476     for (i = 0; i < num_pps; i++) {
2477         result = gst_h264_parser_identify_nalu_avc(
2478             priv->parser,
2479             buf, ofs, buf_size, 2,
2480             &nalu
2481         );
2482         if (result != GST_H264_PARSER_OK)
2483             return get_status(result);
2484
2485         status = decode_pps(decoder, &nalu);
2486         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2487             return status;
2488         ofs = nalu.offset + nalu.size;
2489     }
2490
2491     priv->is_avc = TRUE;
2492     return status;
2493 }
2494
2495 GstVaapiDecoderStatus
2496 gst_vaapi_decoder_h264_decode(GstVaapiDecoder *base, GstBuffer *buffer)
2497 {
2498     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264(base);
2499     GstVaapiDecoderH264Private * const priv = decoder->priv;
2500     GstVaapiDecoderStatus status;
2501     GstBuffer *codec_data;
2502
2503     g_return_val_if_fail(priv->is_constructed,
2504                          GST_VAAPI_DECODER_STATUS_ERROR_INIT_FAILED);
2505
2506     if (!priv->is_opened) {
2507         priv->is_opened = gst_vaapi_decoder_h264_open(decoder, buffer);
2508         if (!priv->is_opened)
2509             return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CODEC;
2510
2511         codec_data = GST_VAAPI_DECODER_CODEC_DATA(decoder);
2512         if (codec_data) {
2513             status = decode_codec_data(decoder, codec_data);
2514             if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2515                 return status;
2516         }
2517      }
2518      return decode_buffer(decoder, buffer);
2519 }
2520
2521 static void
2522 gst_vaapi_decoder_h264_finalize(GObject *object)
2523 {
2524     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264(object);
2525
2526     gst_vaapi_decoder_h264_destroy(decoder);
2527
2528     G_OBJECT_CLASS(gst_vaapi_decoder_h264_parent_class)->finalize(object);
2529 }
2530
2531 static void
2532 gst_vaapi_decoder_h264_constructed(GObject *object)
2533 {
2534     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264(object);
2535     GstVaapiDecoderH264Private * const priv = decoder->priv;
2536     GObjectClass *parent_class;
2537
2538     parent_class = G_OBJECT_CLASS(gst_vaapi_decoder_h264_parent_class);
2539     if (parent_class->constructed)
2540         parent_class->constructed(object);
2541
2542     priv->is_constructed = gst_vaapi_decoder_h264_create(decoder);
2543 }
2544
2545 static void
2546 gst_vaapi_decoder_h264_class_init(GstVaapiDecoderH264Class *klass)
2547 {
2548     GObjectClass * const object_class = G_OBJECT_CLASS(klass);
2549     GstVaapiDecoderClass * const decoder_class = GST_VAAPI_DECODER_CLASS(klass);
2550
2551     g_type_class_add_private(klass, sizeof(GstVaapiDecoderH264Private));
2552
2553     object_class->finalize      = gst_vaapi_decoder_h264_finalize;
2554     object_class->constructed   = gst_vaapi_decoder_h264_constructed;
2555
2556     decoder_class->decode       = gst_vaapi_decoder_h264_decode;
2557 }
2558
2559 static void
2560 gst_vaapi_decoder_h264_init(GstVaapiDecoderH264 *decoder)
2561 {
2562     GstVaapiDecoderH264Private *priv;
2563
2564     priv                        = GST_VAAPI_DECODER_H264_GET_PRIVATE(decoder);
2565     decoder->priv               = priv;
2566     priv->parser                = NULL;
2567     priv->sps                   = &priv->last_sps;
2568     priv->pps                   = &priv->last_pps;
2569     priv->current_picture       = NULL;
2570     priv->dpb_count             = 0;
2571     priv->dpb_size              = 0;
2572     priv->profile               = GST_VAAPI_PROFILE_H264_HIGH;
2573     priv->short_ref_count       = 0;
2574     priv->long_ref_count        = 0;
2575     priv->RefPicList0_count     = 0;
2576     priv->RefPicList1_count     = 0;
2577     priv->nal_length_size       = 0;
2578     priv->width                 = 0;
2579     priv->height                = 0;
2580     priv->mb_x                  = 0;
2581     priv->mb_y                  = 0;
2582     priv->mb_width              = 0;
2583     priv->mb_height             = 0;
2584     priv->adapter               = NULL;
2585     priv->field_poc[0]          = 0;
2586     priv->field_poc[1]          = 0;
2587     priv->poc_msb               = 0;
2588     priv->poc_lsb               = 0;
2589     priv->prev_poc_msb          = 0;
2590     priv->prev_poc_lsb          = 0;
2591     priv->frame_num_offset      = 0;
2592     priv->frame_num             = 0;
2593     priv->prev_frame_num        = 0;
2594     priv->prev_pic_has_mmco5    = FALSE;
2595     priv->prev_pic_bottom_field = FALSE;
2596     priv->is_constructed        = FALSE;
2597     priv->is_opened             = FALSE;
2598     priv->is_avc                = FALSE;
2599     priv->has_context           = FALSE;
2600
2601     memset(priv->dpb, 0, sizeof(priv->dpb));
2602     memset(priv->short_ref, 0, sizeof(priv->short_ref));
2603     memset(priv->long_ref, 0, sizeof(priv->long_ref));
2604     memset(priv->RefPicList0, 0, sizeof(priv->RefPicList0));
2605     memset(priv->RefPicList1, 0, sizeof(priv->RefPicList1));
2606 }
2607
2608 /**
2609  * gst_vaapi_decoder_h264_new:
2610  * @display: a #GstVaapiDisplay
2611  * @caps: a #GstCaps holding codec information
2612  *
2613  * Creates a new #GstVaapiDecoder for MPEG-2 decoding.  The @caps can
2614  * hold extra information like codec-data and pictured coded size.
2615  *
2616  * Return value: the newly allocated #GstVaapiDecoder object
2617  */
2618 GstVaapiDecoder *
2619 gst_vaapi_decoder_h264_new(GstVaapiDisplay *display, GstCaps *caps)
2620 {
2621     GstVaapiDecoderH264 *decoder;
2622
2623     g_return_val_if_fail(GST_VAAPI_IS_DISPLAY(display), NULL);
2624     g_return_val_if_fail(GST_IS_CAPS(caps), NULL);
2625
2626     decoder = g_object_new(
2627         GST_VAAPI_TYPE_DECODER_H264,
2628         "display",      display,
2629         "caps",         caps,
2630         NULL
2631     );
2632     if (!decoder->priv->is_constructed) {
2633         g_object_unref(decoder);
2634         return NULL;
2635     }
2636     return GST_VAAPI_DECODER_CAST(decoder);
2637 }