gst-libs/gst/vaapi/gstvaapidecoder_h264.c

   1 /*
   2  *  gstvaapidecoder_h264.c - H.264 decoder
   3  *
   4  *  Copyright (C) 2011-2012 Intel Corporation
   5  *
   6  *  This library is free software; you can redistribute it and/or
   7  *  modify it under the terms of the GNU Lesser General Public License
   8  *  as published by the Free Software Foundation; either version 2.1
   9  *  of the License, or (at your option) any later version.
  10  *
  11  *  This library is distributed in the hope that it will be useful,
  12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  *  Lesser General Public License for more details.
  15  *
  16  *  You should have received a copy of the GNU Lesser General Public
  17  *  License along with this library; if not, write to the Free
  18  *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  19  *  Boston, MA 02110-1301 USA
  20  */
  21
  22 /**
  23  * SECTION:gstvaapidecoder_h264
  24  * @short_description: H.264 decoder
  25  */
  26
  27 #include "sysdeps.h"
  28 #include <string.h>
  29 #include <gst/base/gstadapter.h>
  30 #include <gst/codecparsers/gsth264parser.h>
  31 #include "gstvaapidecoder_h264.h"
  32 #include "gstvaapidecoder_objects.h"
  33 #include "gstvaapidecoder_priv.h"
  34 #include "gstvaapidisplay_priv.h"
  35 #include "gstvaapiobject_priv.h"
  36
  37 #define DEBUG 1
  38 #include "gstvaapidebug.h"
  39
  40 typedef struct _GstVaapiPictureH264             GstVaapiPictureH264;
  41 typedef struct _GstVaapiPictureH264Class        GstVaapiPictureH264Class;
  42 typedef struct _GstVaapiSliceH264               GstVaapiSliceH264;
  43 typedef struct _GstVaapiSliceH264Class          GstVaapiSliceH264Class;
  44
  45 /* ------------------------------------------------------------------------- */
  46 /* --- H.264 Pictures                                                    --- */
  47 /* ------------------------------------------------------------------------- */
  48
  49 #define GST_VAAPI_TYPE_PICTURE_H264 \
  50     (gst_vaapi_picture_h264_get_type())
  51
  52 #define GST_VAAPI_PICTURE_H264_CAST(obj) \
  53     ((GstVaapiPictureH264 *)(obj))
  54
  55 #define GST_VAAPI_PICTURE_H264(obj)                             \
  56     (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
  57                                 GST_VAAPI_TYPE_PICTURE_H264,    \
  58                                 GstVaapiPictureH264))
  59
  60 #define GST_VAAPI_PICTURE_H264_CLASS(klass)                     \
  61     (G_TYPE_CHECK_CLASS_CAST((klass),                           \
  62                              GST_VAAPI_TYPE_PICTURE_H264,       \
  63                              GstVaapiPictureH264Class))
  64
  65 #define GST_VAAPI_IS_PICTURE_H264(obj) \
  66     (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_VAAPI_TYPE_PICTURE_H264))
  67
  68 #define GST_VAAPI_IS_PICTURE_H264_CLASS(klass) \
  69     (G_TYPE_CHECK_CLASS_TYPE((klass), GST_VAAPI_TYPE_PICTURE_H264))
  70
  71 #define GST_VAAPI_PICTURE_H264_GET_CLASS(obj)                   \
  72     (G_TYPE_INSTANCE_GET_CLASS((obj),                           \
  73                                GST_VAAPI_TYPE_PICTURE_H264,     \
  74                                GstVaapiPictureH264Class))
  75
  76 struct _GstVaapiPictureH264 {
  77     GstVaapiPicture             base;
  78     VAPictureH264               info;
  79     gint32                      poc;
  80     gint32                      frame_num;              // Original frame_num from slice_header()
  81     gint32                      frame_num_wrap;         // Temporary for ref pic marking: FrameNumWrap
  82     gint32                      pic_num;                // Temporary for ref pic marking: PicNum
  83     gint32                      long_term_pic_num;      // Temporary for ref pic marking: LongTermPicNum
  84     guint                       is_idr                  : 1;
  85     guint                       is_long_term            : 1;
  86     guint                       field_pic_flag          : 1;
  87     guint                       bottom_field_flag       : 1;
  88     guint                       has_mmco_5              : 1;
  89     guint                       output_flag             : 1;
  90     guint                       output_needed           : 1;
  91 };
  92
  93 struct _GstVaapiPictureH264Class {
  94     /*< private >*/
  95     GstVaapiPictureClass        parent_class;
  96 };
  97
  98 GST_VAAPI_CODEC_DEFINE_TYPE(GstVaapiPictureH264,
  99                             gst_vaapi_picture_h264,
 100                             GST_VAAPI_TYPE_PICTURE)
 101
 102 static void
 103 gst_vaapi_picture_h264_destroy(GstVaapiPictureH264 *decoder)
 104 {
 105 }
 106
 107 static gboolean
 108 gst_vaapi_picture_h264_create(
 109     GstVaapiPictureH264                      *picture,
 110     const GstVaapiCodecObjectConstructorArgs *args
 111 )
 112 {
 113     return TRUE;
 114 }
 115
 116 static void
 117 gst_vaapi_picture_h264_init(GstVaapiPictureH264 *picture)
 118 {
 119     VAPictureH264 *va_pic;
 120
 121     va_pic                      = &picture->info;
 122     va_pic->flags               = 0;
 123     va_pic->TopFieldOrderCnt    = 0;
 124     va_pic->BottomFieldOrderCnt = 0;
 125
 126     picture->poc                = 0;
 127     picture->is_long_term       = FALSE;
 128     picture->is_idr             = FALSE;
 129     picture->has_mmco_5         = FALSE;
 130     picture->output_needed      = FALSE;
 131 }
 132
 133 static inline GstVaapiPictureH264 *
 134 gst_vaapi_picture_h264_new(GstVaapiDecoderH264 *decoder)
 135 {
 136     GstVaapiCodecObject *object;
 137
 138     g_return_val_if_fail(GST_VAAPI_IS_DECODER(decoder), NULL);
 139
 140     object = gst_vaapi_codec_object_new(
 141         GST_VAAPI_TYPE_PICTURE_H264,
 142         GST_VAAPI_CODEC_BASE(decoder),
 143         NULL, sizeof(VAPictureParameterBufferH264),
 144         NULL, 0
 145     );
 146     if (!object)
 147         return NULL;
 148     return GST_VAAPI_PICTURE_H264_CAST(object);
 149 }
 150
 151 /* ------------------------------------------------------------------------- */
 152 /* --- Slices                                                            --- */
 153 /* ------------------------------------------------------------------------- */
 154
 155 #define GST_VAAPI_TYPE_SLICE_H264 \
 156     (gst_vaapi_slice_h264_get_type())
 157
 158 #define GST_VAAPI_SLICE_H264_CAST(obj) \
 159     ((GstVaapiSliceH264 *)(obj))
 160
 161 #define GST_VAAPI_SLICE_H264(obj)                               \
 162     (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
 163                                 GST_VAAPI_TYPE_SLICE_H264,      \
 164                                 GstVaapiSliceH264))
 165
 166 #define GST_VAAPI_SLICE_H264_CLASS(klass)                       \
 167     (G_TYPE_CHECK_CLASS_CAST((klass),                           \
 168                              GST_VAAPI_TYPE_SLICE_H264,         \
 169                              GstVaapiSliceH264Class))
 170
 171 #define GST_VAAPI_IS_SLICE_H264(obj) \
 172     (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_VAAPI_TYPE_SLICE_H264))
 173
 174 #define GST_VAAPI_IS_SLICE_H264_CLASS(klass) \
 175     (G_TYPE_CHECK_CLASS_TYPE((klass), GST_VAAPI_TYPE_SLICE_H264))
 176
 177 #define GST_VAAPI_SLICE_H264_GET_CLASS(obj)                     \
 178     (G_TYPE_INSTANCE_GET_CLASS((obj),                           \
 179                                GST_VAAPI_TYPE_SLICE_H264,       \
 180                                GstVaapiSliceH264Class))
 181
 182 struct _GstVaapiSliceH264 {
 183     GstVaapiSlice               base;
 184     GstH264SliceHdr             slice_hdr;              // parsed slice_header()
 185 };
 186
 187 struct _GstVaapiSliceH264Class {
 188     /*< private >*/
 189     GstVaapiSliceClass          parent_class;
 190 };
 191
 192 GST_VAAPI_CODEC_DEFINE_TYPE(GstVaapiSliceH264,
 193                             gst_vaapi_slice_h264,
 194                             GST_VAAPI_TYPE_SLICE)
 195
 196 static void
 197 gst_vaapi_slice_h264_destroy(GstVaapiSliceH264 *slice)
 198 {
 199 }
 200
 201 static gboolean
 202 gst_vaapi_slice_h264_create(
 203     GstVaapiSliceH264                        *slice,
 204     const GstVaapiCodecObjectConstructorArgs *args
 205 )
 206 {
 207     return TRUE;
 208 }
 209
 210 static void
 211 gst_vaapi_slice_h264_init(GstVaapiSliceH264 *slice)
 212 {
 213 }
 214
 215 static inline GstVaapiSliceH264 *
 216 gst_vaapi_slice_h264_new(
 217     GstVaapiDecoderH264 *decoder,
 218     const guint8        *data,
 219     guint                data_size
 220 )
 221 {
 222     GstVaapiCodecObject *object;
 223
 224     g_return_val_if_fail(GST_VAAPI_IS_DECODER(decoder), NULL);
 225
 226     object = gst_vaapi_codec_object_new(
 227         GST_VAAPI_TYPE_SLICE_H264,
 228         GST_VAAPI_CODEC_BASE(decoder),
 229         NULL, sizeof(VASliceParameterBufferH264),
 230         data, data_size
 231     );
 232     if (!object)
 233         return NULL;
 234     return GST_VAAPI_SLICE_H264_CAST(object);
 235 }
 236
 237 /* ------------------------------------------------------------------------- */
 238 /* --- H.264 Decoder                                                     --- */
 239 /* ------------------------------------------------------------------------- */
 240
 241 G_DEFINE_TYPE(GstVaapiDecoderH264,
 242               gst_vaapi_decoder_h264,
 243               GST_VAAPI_TYPE_DECODER)
 244
 245 #define GST_VAAPI_DECODER_H264_GET_PRIVATE(obj)                 \
 246     (G_TYPE_INSTANCE_GET_PRIVATE((obj),                         \
 247                                  GST_VAAPI_TYPE_DECODER_H264,   \
 248                                  GstVaapiDecoderH264Private))
 249
 250 // Used for field_poc[]
 251 #define TOP_FIELD       0
 252 #define BOTTOM_FIELD    1
 253
 254 struct _GstVaapiDecoderH264Private {
 255     GstAdapter                 *adapter;
 256     GstH264NalParser           *parser;
 257     GstH264SPS                 *sps;
 258     GstH264SPS                  last_sps;
 259     GstH264PPS                 *pps;
 260     GstH264PPS                  last_pps;
 261     GstVaapiPictureH264        *current_picture;
 262     GstVaapiPictureH264        *dpb[16];
 263     guint                       dpb_count;
 264     guint                       dpb_size;
 265     GstVaapiProfile             profile;
 266     GstVaapiPictureH264        *short_ref[32];
 267     guint                       short_ref_count;
 268     GstVaapiPictureH264        *long_ref[32];
 269     guint                       long_ref_count;
 270     GstVaapiPictureH264        *RefPicList0[32];
 271     guint                       RefPicList0_count;
 272     GstVaapiPictureH264        *RefPicList1[32];
 273     guint                       RefPicList1_count;
 274     guint                       nal_length_size;
 275     guint                       width;
 276     guint                       height;
 277     guint                       mb_x;
 278     guint                       mb_y;
 279     guint                       mb_width;
 280     guint                       mb_height;
 281     guint8                      scaling_list_4x4[6][16];
 282     guint8                      scaling_list_8x8[6][64];
 283     gint32                      field_poc[2];           // 0:TopFieldOrderCnt / 1:BottomFieldOrderCnt
 284     gint32                      poc_msb;                // PicOrderCntMsb
 285     gint32                      poc_lsb;                // pic_order_cnt_lsb (from slice_header())
 286     gint32                      prev_poc_msb;           // prevPicOrderCntMsb
 287     gint32                      prev_poc_lsb;           // prevPicOrderCntLsb
 288     gint32                      frame_num_offset;       // FrameNumOffset
 289     gint32                      prev_frame_num_offset;  // prevFrameNumOffset
 290     gint32                      frame_num;              // frame_num (from slice_header())
 291     gint32                      prev_frame_num;         // prevFrameNum
 292     guint                       is_constructed          : 1;
 293     guint                       is_opened               : 1;
 294     guint                       is_avc                  : 1;
 295     guint                       has_context             : 1;
 296 };
 297
 298 static gboolean
 299 decode_picture_end(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture);
 300
 301 static void
 302 clear_references(
 303     GstVaapiDecoderH264  *decoder,
 304     GstVaapiPictureH264 **pictures,
 305     guint                *picture_count
 306 );
 307
 308 /* Get number of reference frames to use */
 309 static guint
 310 get_max_dec_frame_buffering(GstH264SPS *sps)
 311 {
 312     guint max_dec_frame_buffering, MaxDpbMbs, PicSizeMbs;
 313
 314     /* Table A-1 - Level limits */
 315     switch (sps->level_idc) {
 316     case 10: MaxDpbMbs = 396;    break;
 317     case 11: MaxDpbMbs = 900;    break;
 318     case 12: MaxDpbMbs = 2376;   break;
 319     case 13: MaxDpbMbs = 2376;   break;
 320     case 20: MaxDpbMbs = 2376;   break;
 321     case 21: MaxDpbMbs = 4752;   break;
 322     case 22: MaxDpbMbs = 8100;   break;
 323     case 30: MaxDpbMbs = 8100;   break;
 324     case 31: MaxDpbMbs = 18000;  break;
 325     case 32: MaxDpbMbs = 20480;  break;
 326     case 40: MaxDpbMbs = 32768;  break;
 327     case 41: MaxDpbMbs = 32768;  break;
 328     case 42: MaxDpbMbs = 34816;  break;
 329     case 50: MaxDpbMbs = 110400; break;
 330     case 51: MaxDpbMbs = 184320; break;
 331     default:
 332         g_assert(0 && "unhandled level");
 333         break;
 334     }
 335
 336     PicSizeMbs = ((sps->pic_width_in_mbs_minus1 + 1) *
 337                   (sps->pic_height_in_map_units_minus1 + 1) *
 338                   (sps->frame_mbs_only_flag ? 1 : 2));
 339     max_dec_frame_buffering = MaxDpbMbs / PicSizeMbs;
 340
 341     /* VUI parameters */
 342     if (sps->vui_parameters_present_flag) {
 343         GstH264VUIParams * const vui_params = &sps->vui_parameters;
 344         if (vui_params->bitstream_restriction_flag)
 345             max_dec_frame_buffering = vui_params->max_dec_frame_buffering;
 346         else {
 347             switch (sps->profile_idc) {
 348             case 44:  // CAVLC 4:4:4 Intra profile
 349             case 86:  // Scalable High profile
 350             case 100: // High profile
 351             case 110: // High 10 profile
 352             case 122: // High 4:2:2 profile
 353             case 244: // High 4:4:4 Predictive profile
 354                 if (sps->constraint_set3_flag)
 355                     max_dec_frame_buffering = 0;
 356                 break;
 357             }
 358         }
 359     }
 360
 361     if (max_dec_frame_buffering > 16)
 362         max_dec_frame_buffering = 16;
 363     else if (max_dec_frame_buffering < sps->num_ref_frames)
 364         max_dec_frame_buffering = sps->num_ref_frames;
 365     return MAX(1, max_dec_frame_buffering);
 366 }
 367
 368 static void
 369 dpb_remove_index(GstVaapiDecoderH264 *decoder, guint index)
 370 {
 371     GstVaapiDecoderH264Private * const priv = decoder->priv;
 372     guint num_pictures = --priv->dpb_count;
 373
 374     if (index != num_pictures)
 375         gst_vaapi_picture_replace(&priv->dpb[index], priv->dpb[num_pictures]);
 376     gst_vaapi_picture_replace(&priv->dpb[num_pictures], NULL);
 377 }
 378
 379 static inline gboolean
 380 dpb_output(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
 381 {
 382     /* XXX: update cropping rectangle */
 383     picture->output_needed = FALSE;
 384     return gst_vaapi_picture_output(GST_VAAPI_PICTURE_CAST(picture));
 385 }
 386
 387 static gboolean
 388 dpb_bump(GstVaapiDecoderH264 *decoder)
 389 {
 390     GstVaapiDecoderH264Private * const priv = decoder->priv;
 391     guint i, lowest_poc_index;
 392     gboolean success;
 393
 394     for (i = 0; i < priv->dpb_count; i++) {
 395         if (priv->dpb[i]->output_needed)
 396             break;
 397     }
 398     if (i == priv->dpb_count)
 399         return FALSE;
 400
 401     lowest_poc_index = i++;
 402     for (; i < priv->dpb_count; i++) {
 403         GstVaapiPictureH264 * const picture = priv->dpb[i];
 404         if (picture->output_needed && picture->poc < priv->dpb[lowest_poc_index]->poc)
 405             lowest_poc_index = i;
 406     }
 407
 408     success = dpb_output(decoder, priv->dpb[lowest_poc_index]);
 409     if (!GST_VAAPI_PICTURE_IS_REFERENCE(priv->dpb[lowest_poc_index]))
 410         dpb_remove_index(decoder, lowest_poc_index);
 411     return success;
 412 }
 413
 414 static void
 415 dpb_flush(GstVaapiDecoderH264 *decoder)
 416 {
 417     GstVaapiDecoderH264Private * const priv = decoder->priv;
 418
 419     while (dpb_bump(decoder))
 420         ;
 421     clear_references(decoder, priv->dpb, &priv->dpb_count);
 422 }
 423
 424 static gboolean
 425 dpb_add(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
 426 {
 427     GstVaapiDecoderH264Private * const priv = decoder->priv;
 428     guint i;
 429
 430     // Remove all unused pictures
 431     if (picture->is_idr)
 432         dpb_flush(decoder);
 433     else {
 434         i = 0;
 435         while (i < priv->dpb_count) {
 436             GstVaapiPictureH264 * const picture = priv->dpb[i];
 437             if (!picture->output_needed &&
 438                 !GST_VAAPI_PICTURE_IS_REFERENCE(picture))
 439                 dpb_remove_index(decoder, i);
 440             else
 441                 i++;
 442         }
 443     }
 444
 445     // C.4.5.1 - Storage and marking of a reference decoded picture into the DPB
 446     if (GST_VAAPI_PICTURE_IS_REFERENCE(picture)) {
 447         while (priv->dpb_count == priv->dpb_size) {
 448             if (!dpb_bump(decoder))
 449                 return FALSE;
 450         }
 451         gst_vaapi_picture_replace(&priv->dpb[priv->dpb_count++], picture);
 452         if (picture->output_flag)
 453             picture->output_needed = TRUE;
 454     }
 455
 456     // C.4.5.2 - Storage and marking of a non-reference decoded picture into the DPB
 457     else {
 458         if (!picture->output_flag)
 459             return TRUE;
 460         while (priv->dpb_count == priv->dpb_size) {
 461             for (i = 0; i < priv->dpb_count; i++) {
 462                 if (priv->dpb[i]->output_needed &&
 463                     priv->dpb[i]->poc < picture->poc)
 464                     break;
 465             }
 466             if (i == priv->dpb_count)
 467                 return dpb_output(decoder, picture);
 468             if (!dpb_bump(decoder))
 469                 return FALSE;
 470         }
 471         gst_vaapi_picture_replace(&priv->dpb[priv->dpb_count++], picture);
 472         picture->output_needed = TRUE;
 473     }
 474     return TRUE;
 475 }
 476
 477 static inline void
 478 dpb_reset(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 479 {
 480     GstVaapiDecoderH264Private * const priv = decoder->priv;
 481
 482     priv->dpb_size = get_max_dec_frame_buffering(sps);
 483     GST_DEBUG("DPB size %u", priv->dpb_size);
 484 }
 485
 486 static GstVaapiDecoderStatus
 487 get_status(GstH264ParserResult result)
 488 {
 489     GstVaapiDecoderStatus status;
 490
 491     switch (result) {
 492     case GST_H264_PARSER_OK:
 493         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
 494         break;
 495     case GST_H264_PARSER_NO_NAL_END:
 496         status = GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
 497         break;
 498     case GST_H264_PARSER_ERROR:
 499         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
 500         break;
 501     default:
 502         status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 503         break;
 504     }
 505     return status;
 506 }
 507
 508 static inline GstH264DecRefPicMarking *
 509 get_dec_ref_pic_marking(GstVaapiPictureH264 *picture_h264)
 510 {
 511     GstVaapiPicture * const picture = GST_VAAPI_PICTURE_CAST(picture_h264);
 512     GstVaapiSliceH264 *slice;
 513
 514     slice = g_ptr_array_index(picture->slices, picture->slices->len - 1);
 515     return &slice->slice_hdr.dec_ref_pic_marking;
 516 }
 517
 518 static void
 519 gst_vaapi_decoder_h264_close(GstVaapiDecoderH264 *decoder)
 520 {
 521     GstVaapiDecoderH264Private * const priv = decoder->priv;
 522
 523     gst_vaapi_picture_replace(&priv->current_picture, NULL);
 524     clear_references(decoder, priv->short_ref, &priv->short_ref_count);
 525     clear_references(decoder, priv->long_ref,  &priv->long_ref_count );
 526     clear_references(decoder, priv->dpb,       &priv->dpb_count      );
 527
 528     if (priv->parser) {
 529         gst_h264_nal_parser_free(priv->parser);
 530         priv->parser = NULL;
 531     }
 532
 533     if (priv->adapter) {
 534         gst_adapter_clear(priv->adapter);
 535         g_object_unref(priv->adapter);
 536         priv->adapter = NULL;
 537     }
 538 }
 539
 540 static gboolean
 541 gst_vaapi_decoder_h264_open(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
 542 {
 543     GstVaapiDecoderH264Private * const priv = decoder->priv;
 544
 545     gst_vaapi_decoder_h264_close(decoder);
 546
 547     priv->adapter = gst_adapter_new();
 548     if (!priv->adapter)
 549         return FALSE;
 550
 551     priv->parser = gst_h264_nal_parser_new();
 552     if (!priv->parser)
 553         return FALSE;
 554     return TRUE;
 555 }
 556
 557 static void
 558 gst_vaapi_decoder_h264_destroy(GstVaapiDecoderH264 *decoder)
 559 {
 560     gst_vaapi_decoder_h264_close(decoder);
 561 }
 562
 563 static gboolean
 564 gst_vaapi_decoder_h264_create(GstVaapiDecoderH264 *decoder)
 565 {
 566     if (!GST_VAAPI_DECODER_CODEC(decoder))
 567         return FALSE;
 568     return TRUE;
 569 }
 570
 571 static GstVaapiDecoderStatus
 572 ensure_context(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 573 {
 574     GstVaapiDecoderH264Private * const priv = decoder->priv;
 575     GstVaapiProfile profiles[2];
 576     GstVaapiEntrypoint entrypoint = GST_VAAPI_ENTRYPOINT_VLD;
 577     guint i, n_profiles = 0;
 578     gboolean success, reset_context = FALSE;
 579
 580     if (!priv->has_context || priv->sps->profile_idc != sps->profile_idc) {
 581         GST_DEBUG("profile changed");
 582         reset_context = TRUE;
 583
 584         switch (sps->profile_idc) {
 585         case 66:
 586             profiles[n_profiles++] = GST_VAAPI_PROFILE_H264_BASELINE;
 587             break;
 588         case 77:
 589             profiles[n_profiles++] = GST_VAAPI_PROFILE_H264_MAIN;
 590             // fall-through
 591         case 100:
 592             profiles[n_profiles++] = GST_VAAPI_PROFILE_H264_HIGH;
 593             break;
 594         default:
 595             GST_DEBUG("unsupported profile %d", sps->profile_idc);
 596             return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_PROFILE;
 597         }
 598
 599         for (i = 0; i < n_profiles; i++) {
 600             success = gst_vaapi_display_has_decoder(
 601                 GST_VAAPI_DECODER_DISPLAY(decoder),
 602                 profiles[i],
 603                 entrypoint
 604             );
 605             if (success)
 606                 break;
 607         }
 608         if (i == n_profiles)
 609             return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_PROFILE;
 610         priv->profile = profiles[i];
 611     }
 612
 613     if (!priv->has_context ||
 614         priv->sps->chroma_format_idc != sps->chroma_format_idc) {
 615         GST_DEBUG("chroma format changed");
 616         reset_context = TRUE;
 617
 618         /* XXX: theoritically, we could handle 4:2:2 format */
 619         if (sps->chroma_format_idc != 1)
 620             return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;
 621     }
 622
 623     if (!priv->has_context              ||
 624         priv->sps->width  != sps->width ||
 625         priv->sps->height != sps->height) {
 626         GST_DEBUG("size changed");
 627         reset_context      = TRUE;
 628
 629         priv->width      = sps->width;
 630         priv->height     = sps->height;
 631         priv->mb_width   = sps->pic_width_in_mbs_minus1 + 1;
 632         priv->mb_height  = sps->pic_height_in_map_units_minus1 + 1;
 633         priv->mb_height *= 2 - sps->frame_mbs_only_flag;
 634     }
 635
 636     if (reset_context) {
 637         GstVaapiContextInfo info;
 638
 639         info.profile    = priv->profile;
 640         info.entrypoint = entrypoint;
 641         info.width      = priv->width;
 642         info.height     = priv->height;
 643         info.ref_frames = get_max_dec_frame_buffering(sps);
 644
 645         if (!gst_vaapi_decoder_ensure_context(GST_VAAPI_DECODER(decoder), &info))
 646             return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 647         priv->has_context = TRUE;
 648
 649         /* Reset DPB */
 650         dpb_reset(decoder, sps);
 651     }
 652     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 653 }
 654
 655 static GstVaapiDecoderStatus
 656 ensure_quant_matrix(GstVaapiDecoderH264 *decoder, GstH264PPS *pps)
 657 {
 658     GstVaapiDecoderH264Private * const priv = decoder->priv;
 659
 660     if (priv->pps != pps) {
 661         memcpy(priv->scaling_list_4x4, pps->scaling_lists_4x4,
 662                sizeof(priv->scaling_list_4x4));
 663         memcpy(priv->scaling_list_8x8, pps->scaling_lists_8x8,
 664                sizeof(priv->scaling_list_8x8));
 665     }
 666     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 667 }
 668
 669 static gboolean
 670 decode_current_picture(GstVaapiDecoderH264 *decoder)
 671 {
 672     GstVaapiDecoderH264Private * const priv = decoder->priv;
 673     GstVaapiPictureH264 * const picture = priv->current_picture;
 674     gboolean success = FALSE;
 675
 676     if (!picture)
 677         return TRUE;
 678
 679     if (!decode_picture_end(decoder, picture))
 680         goto end;
 681     if (!gst_vaapi_picture_decode(GST_VAAPI_PICTURE_CAST(picture)))
 682         goto end;
 683     success = TRUE;
 684 end:
 685     gst_vaapi_picture_replace(&priv->current_picture, NULL);
 686     return success;
 687 }
 688
 689 static GstVaapiDecoderStatus
 690 decode_sps(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
 691 {
 692     GstVaapiDecoderH264Private * const priv = decoder->priv;
 693     GstH264SPS * const sps = &priv->last_sps;
 694     GstH264ParserResult result;
 695
 696     GST_DEBUG("decode SPS");
 697
 698     if (priv->current_picture && !decode_current_picture(decoder))
 699         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 700
 701     memset(sps, 0, sizeof(*sps));
 702     result = gst_h264_parser_parse_sps(priv->parser, nalu, sps, TRUE);
 703     if (result != GST_H264_PARSER_OK)
 704         return get_status(result);
 705
 706     return ensure_context(decoder, sps);
 707 }
 708
 709 static GstVaapiDecoderStatus
 710 decode_pps(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
 711 {
 712     GstVaapiDecoderH264Private * const priv = decoder->priv;
 713     GstH264PPS * const pps = &priv->last_pps;
 714     GstH264ParserResult result;
 715
 716     GST_DEBUG("decode PPS");
 717
 718     if (priv->current_picture && !decode_current_picture(decoder))
 719         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 720
 721     memset(pps, 0, sizeof(*pps));
 722     result = gst_h264_parser_parse_pps(priv->parser, nalu, pps);
 723     if (result != GST_H264_PARSER_OK)
 724         return get_status(result);
 725
 726     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 727 }
 728
 729 static GstVaapiDecoderStatus
 730 decode_sei(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
 731 {
 732     GstVaapiDecoderH264Private * const priv = decoder->priv;
 733     GstH264SEIMessage sei;
 734     GstH264ParserResult result;
 735
 736     GST_DEBUG("decode SEI");
 737
 738     memset(&sei, 0, sizeof(sei));
 739     result = gst_h264_parser_parse_sei(priv->parser, nalu, &sei);
 740     if (result != GST_H264_PARSER_OK) {
 741         GST_WARNING("failed to decode SEI, payload type:%d", sei.payloadType);
 742         return get_status(result);
 743     }
 744
 745     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 746 }
 747
 748 static GstVaapiDecoderStatus
 749 decode_sequence_end(GstVaapiDecoderH264 *decoder)
 750 {
 751     GstVaapiDecoderH264Private * const priv = decoder->priv;
 752
 753     GST_DEBUG("decode sequence-end");
 754
 755     if (priv->current_picture && !decode_current_picture(decoder))
 756         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 757     dpb_flush(decoder);
 758     return GST_VAAPI_DECODER_STATUS_END_OF_STREAM;
 759 }
 760
 761 /* 8.2.1.1 - Decoding process for picture order count type 0 */
 762 static void
 763 init_picture_poc_0(
 764     GstVaapiDecoderH264 *decoder,
 765     GstVaapiPictureH264 *picture,
 766     GstH264SliceHdr     *slice_hdr
 767 )
 768 {
 769     GstVaapiDecoderH264Private * const priv = decoder->priv;
 770     GstH264PPS * const pps = slice_hdr->pps;
 771     GstH264SPS * const sps = pps->sequence;
 772     const gint32 MaxPicOrderCntLsb = 1 << (sps->log2_max_pic_order_cnt_lsb_minus4 + 4);
 773
 774     GST_DEBUG("decode picture order count type 0");
 775
 776     // (8-3)
 777     priv->poc_lsb = slice_hdr->pic_order_cnt_lsb;
 778     if (priv->poc_lsb < priv->prev_poc_lsb &&
 779         (priv->prev_poc_lsb - priv->poc_lsb) >= (MaxPicOrderCntLsb / 2))
 780         priv->poc_msb = priv->prev_poc_msb + MaxPicOrderCntLsb;
 781     else if (priv->poc_lsb > priv->prev_poc_lsb &&
 782              (priv->poc_lsb - priv->prev_poc_lsb) > (MaxPicOrderCntLsb / 2))
 783         priv->poc_msb = priv->prev_poc_msb - MaxPicOrderCntLsb;
 784     else
 785         priv->poc_msb = priv->prev_poc_msb;
 786
 787     // (8-4)
 788     if (!slice_hdr->field_pic_flag || !slice_hdr->bottom_field_flag)
 789         priv->field_poc[TOP_FIELD] = priv->poc_msb + priv->poc_lsb;
 790
 791     // (8-5)
 792     if (!slice_hdr->field_pic_flag)
 793         priv->field_poc[BOTTOM_FIELD] = priv->field_poc[TOP_FIELD] +
 794             slice_hdr->delta_pic_order_cnt_bottom;
 795     else if (slice_hdr->bottom_field_flag)
 796         priv->field_poc[BOTTOM_FIELD] = priv->poc_msb + priv->poc_lsb;
 797 }
 798
 799 /* 8.2.1.2 - Decoding process for picture order count type 1 */
 800 static void
 801 init_picture_poc_1(
 802     GstVaapiDecoderH264 *decoder,
 803     GstVaapiPictureH264 *picture,
 804     GstH264SliceHdr     *slice_hdr
 805 )
 806 {
 807     GstVaapiDecoderH264Private * const priv = decoder->priv;
 808     GstH264PPS * const pps = slice_hdr->pps;
 809     GstH264SPS * const sps = pps->sequence;
 810     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
 811     gint32 abs_frame_num, expected_poc;
 812     guint i;
 813
 814     GST_DEBUG("decode picture order count type 1");
 815
 816     // (8-6)
 817     if (picture->is_idr)
 818         priv->frame_num_offset = 0;
 819     else if (priv->prev_frame_num > priv->frame_num)
 820         priv->frame_num_offset = priv->prev_frame_num_offset + MaxFrameNum;
 821     else
 822         priv->frame_num_offset = priv->prev_frame_num_offset;
 823
 824     // (8-7)
 825     if (sps->num_ref_frames_in_pic_order_cnt_cycle != 0)
 826         abs_frame_num = priv->frame_num_offset + priv->frame_num;
 827     else
 828         abs_frame_num = 0;
 829     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture) && abs_frame_num > 0)
 830         abs_frame_num = abs_frame_num - 1;
 831
 832     if (abs_frame_num > 0) {
 833         gint32 expected_delta_per_poc_cycle;
 834         gint32 poc_cycle_cnt, frame_num_in_poc_cycle;
 835
 836         expected_delta_per_poc_cycle = 0;
 837         for (i = 0; i < sps->num_ref_frames_in_pic_order_cnt_cycle; i++)
 838             expected_delta_per_poc_cycle += sps->offset_for_ref_frame[i];
 839
 840         // (8-8)
 841         poc_cycle_cnt = (abs_frame_num - 1) /
 842             sps->num_ref_frames_in_pic_order_cnt_cycle;
 843         frame_num_in_poc_cycle = (abs_frame_num - 1) %
 844             sps->num_ref_frames_in_pic_order_cnt_cycle;
 845
 846         // (8-9)
 847         expected_poc = poc_cycle_cnt * expected_delta_per_poc_cycle;
 848         for (i = 0; i <= frame_num_in_poc_cycle; i++)
 849             expected_poc += sps->offset_for_ref_frame[i];
 850     }
 851     else
 852         expected_poc = 0;
 853     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
 854         expected_poc += sps->offset_for_non_ref_pic;
 855
 856     // (8-10)
 857     if (!slice_hdr->field_pic_flag) {
 858         priv->field_poc[TOP_FIELD] = expected_poc +
 859             slice_hdr->delta_pic_order_cnt[0];
 860         priv->field_poc[BOTTOM_FIELD] = priv->field_poc[TOP_FIELD] +
 861             sps->offset_for_top_to_bottom_field +
 862             slice_hdr->delta_pic_order_cnt[1];
 863     }
 864     else if (!slice_hdr->bottom_field_flag)
 865         priv->field_poc[TOP_FIELD] = expected_poc +
 866             slice_hdr->delta_pic_order_cnt[0];
 867     else
 868         priv->field_poc[BOTTOM_FIELD] = expected_poc +
 869             sps->offset_for_top_to_bottom_field + slice_hdr->delta_pic_order_cnt[0];
 870 }
 871
 872 /* 8.2.1.3 - Decoding process for picture order count type 2 */
 873 static void
 874 init_picture_poc_2(
 875     GstVaapiDecoderH264 *decoder,
 876     GstVaapiPictureH264 *picture,
 877     GstH264SliceHdr     *slice_hdr
 878 )
 879 {
 880     GstVaapiDecoderH264Private * const priv = decoder->priv;
 881     GstH264PPS * const pps = slice_hdr->pps;
 882     GstH264SPS * const sps = pps->sequence;
 883     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
 884     guint temp_poc;
 885
 886     GST_DEBUG("decode picture order count type 2");
 887
 888     // (8-11)
 889     if (picture->is_idr)
 890         priv->frame_num_offset = 0;
 891     else if (priv->prev_frame_num > priv->frame_num)
 892         priv->frame_num_offset = priv->prev_frame_num_offset + MaxFrameNum;
 893     else
 894         priv->frame_num_offset = priv->prev_frame_num_offset;
 895
 896     // (8-12)
 897     if (picture->is_idr)
 898         temp_poc = 0;
 899     else if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
 900         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num) - 1;
 901     else
 902         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num);
 903
 904     // (8-13)
 905     if (!slice_hdr->field_pic_flag) {
 906         priv->field_poc[TOP_FIELD] = temp_poc;
 907         priv->field_poc[BOTTOM_FIELD] = temp_poc;
 908     }
 909     else if (slice_hdr->bottom_field_flag)
 910         priv->field_poc[BOTTOM_FIELD] = temp_poc;
 911     else
 912         priv->field_poc[TOP_FIELD] = temp_poc;
 913 }
 914
 915 /* 8.2.1 - Decoding process for picture order count */
 916 static void
 917 init_picture_poc(
 918     GstVaapiDecoderH264 *decoder,
 919     GstVaapiPictureH264 *picture,
 920     GstH264SliceHdr     *slice_hdr
 921 )
 922 {
 923     GstVaapiDecoderH264Private * const priv = decoder->priv;
 924     VAPictureH264 * const pic = &picture->info;
 925     GstH264PPS * const pps = slice_hdr->pps;
 926     GstH264SPS * const sps = pps->sequence;
 927
 928     switch (sps->pic_order_cnt_type) {
 929     case 0:
 930         init_picture_poc_0(decoder, picture, slice_hdr);
 931         break;
 932     case 1:
 933         init_picture_poc_1(decoder, picture, slice_hdr);
 934         break;
 935     case 2:
 936         init_picture_poc_2(decoder, picture, slice_hdr);
 937         break;
 938     }
 939
 940     if (!(pic->flags & VA_PICTURE_H264_BOTTOM_FIELD))
 941         pic->TopFieldOrderCnt = priv->field_poc[TOP_FIELD];
 942     if (!(pic->flags & VA_PICTURE_H264_TOP_FIELD))
 943         pic->BottomFieldOrderCnt = priv->field_poc[BOTTOM_FIELD];
 944     picture->poc = MIN(pic->TopFieldOrderCnt, pic->BottomFieldOrderCnt);
 945 }
 946
 947 static int
 948 compare_picture_pic_num_dec(const void *a, const void *b)
 949 {
 950     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
 951     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
 952
 953     return picB->pic_num - picA->pic_num;
 954 }
 955
 956 static int
 957 compare_picture_long_term_pic_num_inc(const void *a, const void *b)
 958 {
 959     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
 960     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
 961
 962     return picA->long_term_pic_num - picB->long_term_pic_num;
 963 }
 964
 965 static int
 966 compare_picture_poc_dec(const void *a, const void *b)
 967 {
 968     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
 969     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
 970
 971     return picB->poc - picA->poc;
 972 }
 973
 974 static int
 975 compare_picture_poc_inc(const void *a, const void *b)
 976 {
 977     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
 978     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
 979
 980     return picA->poc - picB->poc;
 981 }
 982
 983 static int
 984 compare_picture_frame_num_wrap_dec(const void *a, const void *b)
 985 {
 986     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
 987     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
 988
 989     return picB->frame_num_wrap - picA->frame_num_wrap;
 990 }
 991
 992 static int
 993 compare_picture_long_term_frame_idx_inc(const void *a, const void *b)
 994 {
 995     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
 996     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
 997
 998     return picA->info.frame_idx - picB->info.frame_idx;
 999 }
1000
1001 /* 8.2.4.1 - Decoding process for picture numbers */
1002 static void
1003 init_picture_refs_pic_num(
1004     GstVaapiDecoderH264 *decoder,
1005     GstVaapiPictureH264 *picture,
1006     GstH264SliceHdr     *slice_hdr
1007 )
1008 {
1009     GstVaapiDecoderH264Private * const priv = decoder->priv;
1010     GstH264PPS * const pps = slice_hdr->pps;
1011     GstH264SPS * const sps = pps->sequence;
1012     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1013     const guint field_flags = VA_PICTURE_H264_TOP_FIELD | VA_PICTURE_H264_BOTTOM_FIELD;
1014     guint i;
1015
1016     GST_DEBUG("decode picture numbers");
1017
1018     for (i = 0; i < priv->short_ref_count; i++) {
1019         GstVaapiPictureH264 * const pic = priv->short_ref[i];
1020
1021         // (8-27)
1022         if (pic->frame_num > priv->frame_num)
1023             pic->frame_num_wrap = pic->frame_num - MaxFrameNum;
1024         else
1025             pic->frame_num_wrap = pic->frame_num;
1026
1027         // (8-28, 8-30, 8-31)
1028         if (!pic->field_pic_flag)
1029             pic->pic_num = pic->frame_num_wrap;
1030         else {
1031             if (((picture->info.flags ^ pic->info.flags) & field_flags) == 0)
1032                 pic->pic_num = 2 * pic->frame_num_wrap + 1;
1033             else
1034                 pic->pic_num = 2 * pic->frame_num_wrap;
1035         }
1036     }
1037
1038     for (i = 0; i < priv->long_ref_count; i++) {
1039         GstVaapiPictureH264 * const pic = priv->long_ref[i];
1040
1041         // (8-29, 8-32, 8-33)
1042         if (!pic->field_pic_flag)
1043             pic->long_term_pic_num = pic->info.frame_idx;
1044         else {
1045             if (((picture->info.flags ^ pic->info.flags) & field_flags) == 0)
1046                 pic->long_term_pic_num = 2 * pic->info.frame_idx + 1;
1047             else
1048                 pic->long_term_pic_num = 2 * pic->info.frame_idx;
1049         }
1050     }
1051 }
1052
1053 #define SORT_REF_LIST(list, n, compare_func) \
1054     qsort(list, n, sizeof(*(list)), compare_picture_##compare_func)
1055
1056 static void
1057 init_picture_refs_p_slice(
1058     GstVaapiDecoderH264 *decoder,
1059     GstVaapiPictureH264 *picture,
1060     GstH264SliceHdr     *slice_hdr
1061 )
1062 {
1063     GstVaapiDecoderH264Private * const priv = decoder->priv;
1064     GstVaapiPictureH264 **ref_list;
1065     guint i;
1066
1067     GST_DEBUG("decode reference picture list for P and SP slices");
1068
1069     if (!picture->field_pic_flag) {
1070         /* 8.2.4.2.1 - P and SP slices in frames */
1071         if (priv->short_ref_count > 0) {
1072             ref_list = priv->RefPicList0;
1073             for (i = 0; i < priv->short_ref_count; i++)
1074                 ref_list[i] = priv->short_ref[i];
1075             SORT_REF_LIST(ref_list, i, pic_num_dec);
1076             priv->RefPicList0_count += i;
1077         }
1078
1079         if (priv->long_ref_count > 0) {
1080             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1081             for (i = 0; i < priv->long_ref_count; i++)
1082                 ref_list[i] = priv->long_ref[i];
1083             SORT_REF_LIST(ref_list, i, long_term_pic_num_inc);
1084             priv->RefPicList0_count += i;
1085         }
1086     }
1087     else {
1088         /* 8.2.4.2.2 - P and SP slices in fields */
1089         GstVaapiPictureH264 *short_ref[32];
1090         guint short_ref_count = 0;
1091         GstVaapiPictureH264 *long_ref[32];
1092         guint long_ref_count = 0;
1093
1094         // XXX: handle second field if current field is marked as
1095         // "used for short-term reference"
1096         if (priv->short_ref_count > 0) {
1097             for (i = 0; i < priv->short_ref_count; i++)
1098                 short_ref[i] = priv->short_ref[i];
1099             SORT_REF_LIST(short_ref, i, frame_num_wrap_dec);
1100             short_ref_count = i;
1101         }
1102
1103         // XXX: handle second field if current field is marked as
1104         // "used for long-term reference"
1105         if (priv->long_ref_count > 0) {
1106             for (i = 0; i < priv->long_ref_count; i++)
1107                 long_ref[i] = priv->long_ref[i];
1108             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1109             long_ref_count = i;
1110         }
1111
1112         // XXX: handle 8.2.4.2.5
1113     }
1114 }
1115
1116 static void
1117 init_picture_refs_b_slice(
1118     GstVaapiDecoderH264 *decoder,
1119     GstVaapiPictureH264 *picture,
1120     GstH264SliceHdr     *slice_hdr
1121 )
1122 {
1123     GstVaapiDecoderH264Private * const priv = decoder->priv;
1124     GstVaapiPictureH264 **ref_list;
1125     guint i, n;
1126
1127     GST_DEBUG("decode reference picture list for B slices");
1128
1129     if (!picture->field_pic_flag) {
1130         /* 8.2.4.2.3 - B slices in frames */
1131
1132         /* RefPicList0 */
1133         if (priv->short_ref_count > 0) {
1134             // 1. Short-term references
1135             ref_list = priv->RefPicList0;
1136             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1137                 if (priv->short_ref[i]->poc < picture->poc)
1138                     ref_list[n++] = priv->short_ref[i];
1139             }
1140             SORT_REF_LIST(ref_list, n, poc_dec);
1141             priv->RefPicList0_count += n;
1142
1143             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1144             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1145                 if (priv->short_ref[i]->poc >= picture->poc)
1146                     ref_list[n++] = priv->short_ref[i];
1147             }
1148             SORT_REF_LIST(ref_list, n, poc_inc);
1149             priv->RefPicList0_count += n;
1150         }
1151
1152         if (priv->long_ref_count > 0) {
1153             // 2. Long-term references
1154             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1155             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1156                 ref_list[n++] = priv->long_ref[i];
1157             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1158             priv->RefPicList0_count += n;
1159         }
1160
1161         /* RefPicList1 */
1162         if (priv->short_ref_count > 0) {
1163             // 1. Short-term references
1164             ref_list = priv->RefPicList1;
1165             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1166                 if (priv->short_ref[i]->poc > picture->poc)
1167                     ref_list[n++] = priv->short_ref[i];
1168             }
1169             SORT_REF_LIST(ref_list, n, poc_inc);
1170             priv->RefPicList1_count += n;
1171
1172             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1173             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1174                 if (priv->short_ref[i]->poc <= picture->poc)
1175                     ref_list[n++] = priv->short_ref[i];
1176             }
1177             SORT_REF_LIST(ref_list, n, poc_dec);
1178             priv->RefPicList1_count += n;
1179         }
1180
1181         if (priv->long_ref_count > 0) {
1182             // 2. Long-term references
1183             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1184             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1185                 ref_list[n++] = priv->long_ref[i];
1186             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1187             priv->RefPicList1_count += n;
1188         }
1189     }
1190     else {
1191         /* 8.2.4.2.4 - B slices in fields */
1192         GstVaapiPictureH264 *short_ref0[32];
1193         guint short_ref0_count = 0;
1194         GstVaapiPictureH264 *short_ref1[32];
1195         guint short_ref1_count = 0;
1196         GstVaapiPictureH264 *long_ref[32];
1197         guint long_ref_count = 0;
1198
1199         /* refFrameList0ShortTerm */
1200         if (priv->short_ref_count > 0) {
1201             ref_list = short_ref0;
1202             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1203                 if (priv->short_ref[i]->poc <= picture->poc)
1204                     ref_list[n++] = priv->short_ref[i];
1205             }
1206             SORT_REF_LIST(ref_list, n, poc_dec);
1207             short_ref0_count += n;
1208
1209             ref_list = &short_ref0[short_ref0_count];
1210             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1211                 if (priv->short_ref[i]->poc > picture->poc)
1212                     ref_list[n++] = priv->short_ref[i];
1213             }
1214             SORT_REF_LIST(ref_list, n, poc_inc);
1215             short_ref0_count += n;
1216         }
1217
1218         /* refFrameList1ShortTerm */
1219         if (priv->short_ref_count > 0) {
1220             ref_list = short_ref1;
1221             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1222                 if (priv->short_ref[i]->poc > picture->poc)
1223                     ref_list[n++] = priv->short_ref[i];
1224             }
1225             SORT_REF_LIST(ref_list, n, poc_inc);
1226             short_ref1_count += n;
1227
1228             ref_list = &short_ref1[short_ref1_count];
1229             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1230                 if (priv->short_ref[i]->poc <= picture->poc)
1231                     ref_list[n++] = priv->short_ref[i];
1232             }
1233             SORT_REF_LIST(ref_list, n, poc_dec);
1234             short_ref1_count += n;
1235         }
1236
1237         /* refFrameListLongTerm */
1238         if (priv->long_ref_count > 0) {
1239             for (i = 0; i < priv->long_ref_count; i++)
1240                 long_ref[i] = priv->long_ref[i];
1241             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1242             long_ref_count = i;
1243         }
1244
1245         // XXX: handle 8.2.4.2.5
1246     }
1247
1248     /* Check whether RefPicList1 is identical to RefPicList0, then
1249        swap if necessary */
1250     if (priv->RefPicList1_count > 1 &&
1251         priv->RefPicList1_count == priv->RefPicList0_count &&
1252         memcmp(priv->RefPicList0, priv->RefPicList1,
1253                priv->RefPicList0_count * sizeof(priv->RefPicList0[0])) == 0) {
1254         GstVaapiPictureH264 * const tmp = priv->RefPicList1[0];
1255         priv->RefPicList1[0] = priv->RefPicList1[1];
1256         priv->RefPicList1[1] = tmp;
1257     }
1258 }
1259
1260 #undef SORT_REF_LIST
1261
1262 static void
1263 clear_references(
1264     GstVaapiDecoderH264  *decoder,
1265     GstVaapiPictureH264 **pictures,
1266     guint                *picture_count
1267 )
1268 {
1269     const guint num_pictures = *picture_count;
1270     guint i;
1271
1272     for (i = 0; i < num_pictures; i++)
1273         gst_vaapi_picture_replace(&pictures[i], NULL);
1274     *picture_count = 0;
1275 }
1276
1277 static gboolean
1278 remove_reference_at(
1279     GstVaapiDecoderH264  *decoder,
1280     GstVaapiPictureH264 **pictures,
1281     guint                *picture_count,
1282     guint                 index
1283 )
1284 {
1285     guint num_pictures = *picture_count;
1286
1287     g_return_val_if_fail(index < num_pictures, FALSE);
1288
1289     GST_VAAPI_PICTURE_FLAG_UNSET(pictures[index], GST_VAAPI_PICTURE_FLAG_REFERENCE);
1290     if (index != --num_pictures)
1291         gst_vaapi_picture_replace(&pictures[index], pictures[num_pictures]);
1292     gst_vaapi_picture_replace(&pictures[num_pictures], NULL);
1293     *picture_count = num_pictures;
1294     return TRUE;
1295 }
1296
1297 static gint
1298 find_short_term_reference(GstVaapiDecoderH264 *decoder, gint32 pic_num)
1299 {
1300     GstVaapiDecoderH264Private * const priv = decoder->priv;
1301     guint i;
1302
1303     for (i = 0; i < priv->short_ref_count; i++) {
1304         if (priv->short_ref[i]->pic_num == pic_num)
1305             return i;
1306     }
1307     GST_ERROR("found no short-term reference picture with PicNum = %d",
1308               pic_num);
1309     return -1;
1310 }
1311
1312 static gint
1313 find_long_term_reference(GstVaapiDecoderH264 *decoder, gint32 long_term_pic_num)
1314 {
1315     GstVaapiDecoderH264Private * const priv = decoder->priv;
1316     guint i;
1317
1318     for (i = 0; i < priv->long_ref_count; i++) {
1319         if (priv->long_ref[i]->long_term_pic_num == long_term_pic_num)
1320             return i;
1321     }
1322     GST_ERROR("found no long-term reference picture with LongTermPicNum = %d",
1323               long_term_pic_num);
1324     return -1;
1325 }
1326
1327 static void
1328 exec_picture_refs_modification_1(
1329     GstVaapiDecoderH264           *decoder,
1330     GstVaapiPictureH264           *picture,
1331     GstH264SliceHdr               *slice_hdr,
1332     guint                          list
1333 )
1334 {
1335     GstVaapiDecoderH264Private * const priv = decoder->priv;
1336     GstH264PPS * const pps = slice_hdr->pps;
1337     GstH264SPS * const sps = pps->sequence;
1338     GstH264RefPicListModification *ref_pic_list_modification;
1339     guint num_ref_pic_list_modifications;
1340     GstVaapiPictureH264 **ref_list;
1341     guint *ref_list_count_ptr, ref_list_count, ref_list_idx = 0;
1342     guint i, j, n, num_refs;
1343     gint found_ref_idx;
1344     gint32 MaxPicNum, CurrPicNum, picNumPred;
1345
1346     GST_DEBUG("modification process of reference picture list %u", list);
1347
1348     if (list == 0) {
1349         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l0;
1350         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l0;
1351         ref_list                       = priv->RefPicList0;
1352         ref_list_count_ptr             = &priv->RefPicList0_count;
1353         num_refs                       = slice_hdr->num_ref_idx_l0_active_minus1 + 1;
1354     }
1355     else {
1356         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l1;
1357         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l1;
1358         ref_list                       = priv->RefPicList1;
1359         ref_list_count_ptr             = &priv->RefPicList1_count;
1360         num_refs                       = slice_hdr->num_ref_idx_l1_active_minus1 + 1;
1361     }
1362     ref_list_count = *ref_list_count_ptr;
1363
1364     if (picture->field_pic_flag) {
1365         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 5); // 2 * MaxFrameNum
1366         CurrPicNum = 2 * slice_hdr->frame_num + 1;              // 2 * frame_num + 1
1367     }
1368     else {
1369         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 4); // MaxFrameNum
1370         CurrPicNum = slice_hdr->frame_num;                      // frame_num
1371     }
1372
1373     picNumPred = CurrPicNum;
1374
1375     for (i = 0; i < num_ref_pic_list_modifications; i++) {
1376         GstH264RefPicListModification * const l = &ref_pic_list_modification[i];
1377         if (l->modification_of_pic_nums_idc == 3)
1378             break;
1379
1380         /* 8.2.4.3.1 - Short-term reference pictures */
1381         if (l->modification_of_pic_nums_idc == 0 || l->modification_of_pic_nums_idc == 1) {
1382             gint32 abs_diff_pic_num = l->value.abs_diff_pic_num_minus1 + 1;
1383             gint32 picNum, picNumNoWrap;
1384
1385             // (8-34)
1386             if (l->modification_of_pic_nums_idc == 0) {
1387                 picNumNoWrap = picNumPred - abs_diff_pic_num;
1388                 if (picNumNoWrap < 0)
1389                     picNumNoWrap += MaxPicNum;
1390             }
1391
1392             // (8-35)
1393             else {
1394                 picNumNoWrap = picNumPred + abs_diff_pic_num;
1395                 if (picNumNoWrap >= MaxPicNum)
1396                     picNumNoWrap -= MaxPicNum;
1397             }
1398             picNumPred = picNumNoWrap;
1399
1400             // (8-36)
1401             picNum = picNumNoWrap;
1402             if (picNum > CurrPicNum)
1403                 picNum -= MaxPicNum;
1404
1405             // (8-37)
1406             for (j = num_refs; j > ref_list_idx; j--)
1407                 ref_list[j] = ref_list[j - 1];
1408             found_ref_idx = find_short_term_reference(decoder, picNum);
1409             ref_list[ref_list_idx++] =
1410                 found_ref_idx >= 0 ? priv->short_ref[found_ref_idx] : NULL;
1411             n = ref_list_idx;
1412             for (j = ref_list_idx; j <= num_refs; j++) {
1413                 gint32 PicNumF;
1414                 if (!ref_list[j])
1415                     continue;
1416                 PicNumF = ref_list[j]->is_long_term ?
1417                     MaxPicNum : ref_list[j]->pic_num;
1418                 if (PicNumF != picNum)
1419                     ref_list[n++] = ref_list[j];
1420             }
1421         }
1422
1423         /* 8.2.4.3.2 - Long-term reference pictures */
1424         else {
1425
1426             for (j = num_refs; j > ref_list_idx; j--)
1427                 ref_list[j] = ref_list[j - 1];
1428             found_ref_idx =
1429                 find_long_term_reference(decoder, l->value.long_term_pic_num);
1430             ref_list[ref_list_idx++] =
1431                 found_ref_idx >= 0 ? priv->long_ref[found_ref_idx] : NULL;
1432             n = ref_list_idx;
1433             for (j = ref_list_idx; j <= num_refs; j++) {
1434                 gint32 LongTermPicNumF;
1435                 if (!ref_list[j])
1436                     continue;
1437                 LongTermPicNumF = ref_list[j]->is_long_term ?
1438                     ref_list[j]->long_term_pic_num : INT_MAX;
1439                 if (LongTermPicNumF != l->value.long_term_pic_num)
1440                     ref_list[n++] = ref_list[j];
1441             }
1442         }
1443     }
1444
1445 #if DEBUG
1446     for (i = 0; i < num_refs; i++)
1447         if (!ref_list[i])
1448             GST_ERROR("list %u entry %u is empty", list, i);
1449 #endif
1450     *ref_list_count_ptr = num_refs;
1451 }
1452
1453 /* 8.2.4.3 - Modification process for reference picture lists */
1454 static void
1455 exec_picture_refs_modification(
1456     GstVaapiDecoderH264 *decoder,
1457     GstVaapiPictureH264 *picture,
1458     GstH264SliceHdr     *slice_hdr
1459 )
1460 {
1461     GST_DEBUG("execute ref_pic_list_modification()");
1462
1463     /* RefPicList0 */
1464     if (!GST_H264_IS_I_SLICE(slice_hdr) && !GST_H264_IS_SI_SLICE(slice_hdr) &&
1465         slice_hdr->ref_pic_list_modification_flag_l0)
1466         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 0);
1467
1468     /* RefPicList1 */
1469     if (GST_H264_IS_B_SLICE(slice_hdr) &&
1470         slice_hdr->ref_pic_list_modification_flag_l1)
1471         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 1);
1472 }
1473
1474 static gboolean
1475 init_picture_refs(
1476     GstVaapiDecoderH264 *decoder,
1477     GstVaapiPictureH264 *picture,
1478     GstH264SliceHdr     *slice_hdr
1479 )
1480 {
1481     GstVaapiDecoderH264Private * const priv = decoder->priv;
1482     GstVaapiPicture * const base_picture = &picture->base;
1483     guint i, num_refs;
1484
1485     init_picture_refs_pic_num(decoder, picture, slice_hdr);
1486
1487     priv->RefPicList0_count = 0;
1488     priv->RefPicList1_count = 0;
1489
1490     switch (base_picture->type) {
1491     case GST_VAAPI_PICTURE_TYPE_P:
1492     case GST_VAAPI_PICTURE_TYPE_SP:
1493         init_picture_refs_p_slice(decoder, picture, slice_hdr);
1494         break;
1495     case GST_VAAPI_PICTURE_TYPE_B:
1496         init_picture_refs_b_slice(decoder, picture, slice_hdr);
1497         break;
1498     default:
1499         break;
1500     }
1501
1502     exec_picture_refs_modification(decoder, picture, slice_hdr);
1503
1504     switch (base_picture->type) {
1505     case GST_VAAPI_PICTURE_TYPE_B:
1506         num_refs = 1 + slice_hdr->num_ref_idx_l1_active_minus1;
1507         for (i = priv->RefPicList1_count; i < num_refs; i++)
1508             priv->RefPicList1[i] = NULL;
1509         priv->RefPicList1_count = num_refs;
1510
1511         // fall-through
1512     case GST_VAAPI_PICTURE_TYPE_P:
1513     case GST_VAAPI_PICTURE_TYPE_SP:
1514         num_refs = 1 + slice_hdr->num_ref_idx_l0_active_minus1;
1515         for (i = priv->RefPicList0_count; i < num_refs; i++)
1516             priv->RefPicList0[i] = NULL;
1517         priv->RefPicList0_count = num_refs;
1518         break;
1519     default:
1520         break;
1521     }
1522     return TRUE;
1523 }
1524
1525 static gboolean
1526 init_picture(
1527     GstVaapiDecoderH264 *decoder,
1528     GstVaapiPictureH264 *picture,
1529     GstH264SliceHdr     *slice_hdr,
1530     GstH264NalUnit      *nalu
1531 )
1532 {
1533     GstVaapiDecoderH264Private * const priv = decoder->priv;
1534     GstVaapiPicture * const base_picture = &picture->base;
1535     VAPictureH264 *pic;
1536     guint i;
1537
1538     priv->frame_num             = slice_hdr->frame_num;
1539     picture->frame_num          = priv->frame_num;
1540     picture->frame_num_wrap     = priv->frame_num;
1541     picture->is_idr             = nalu->type == GST_H264_NAL_SLICE_IDR;
1542     picture->field_pic_flag     = slice_hdr->field_pic_flag;
1543     picture->bottom_field_flag  = slice_hdr->bottom_field_flag;
1544     picture->output_flag        = TRUE; /* XXX: conformant to Annex A only */
1545     base_picture->pts           = gst_adapter_prev_timestamp(priv->adapter, NULL);
1546
1547     /* Reset decoder state for IDR pictures */
1548     if (picture->is_idr) {
1549         GST_DEBUG("<IDR>");
1550         clear_references(decoder, priv->short_ref, &priv->short_ref_count);
1551         clear_references(decoder, priv->long_ref,  &priv->long_ref_count );
1552         priv->prev_poc_msb = 0;
1553         priv->prev_poc_lsb = 0;
1554     }
1555
1556     /* Initialize VA picture info */
1557     pic = &picture->info;
1558     pic->picture_id = picture->base.surface_id;
1559     pic->frame_idx  = priv->frame_num;
1560     if (picture->field_pic_flag) {
1561         if (picture->bottom_field_flag)
1562             pic->flags |= VA_PICTURE_H264_BOTTOM_FIELD;
1563         else
1564             pic->flags |= VA_PICTURE_H264_TOP_FIELD;
1565     }
1566
1567     /* Initialize base picture */
1568     switch (slice_hdr->type % 5) {
1569     case GST_H264_P_SLICE:
1570         base_picture->type = GST_VAAPI_PICTURE_TYPE_P;
1571         break;
1572     case GST_H264_B_SLICE:
1573         base_picture->type = GST_VAAPI_PICTURE_TYPE_B;
1574         break;
1575     case GST_H264_I_SLICE:
1576         base_picture->type = GST_VAAPI_PICTURE_TYPE_I;
1577         break;
1578     case GST_H264_SP_SLICE:
1579         base_picture->type = GST_VAAPI_PICTURE_TYPE_SP;
1580         break;
1581     case GST_H264_SI_SLICE:
1582         base_picture->type = GST_VAAPI_PICTURE_TYPE_SI;
1583         break;
1584     }
1585
1586     if (nalu->ref_idc) {
1587         GstH264DecRefPicMarking * const dec_ref_pic_marking =
1588             &slice_hdr->dec_ref_pic_marking;
1589         GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_REFERENCE);
1590         if (picture->is_idr) {
1591             if (dec_ref_pic_marking->long_term_reference_flag)
1592                 picture->is_long_term = TRUE;
1593         }
1594         else if (dec_ref_pic_marking->adaptive_ref_pic_marking_mode_flag) {
1595             for (i = 0; i < dec_ref_pic_marking->n_ref_pic_marking; i++) {
1596                 GstH264RefPicMarking * const ref_pic_marking =
1597                     &dec_ref_pic_marking->ref_pic_marking[i];
1598                 switch (ref_pic_marking->memory_management_control_operation) {
1599                 case 3:
1600                 case 6:
1601                     picture->is_long_term = TRUE;
1602                     pic->frame_idx = ref_pic_marking->long_term_frame_idx;
1603                     break;
1604                 case 5:
1605                     picture->has_mmco_5 = TRUE;
1606                     break;
1607                 }
1608             }
1609         }
1610         if (picture->is_long_term)
1611             pic->flags |= VA_PICTURE_H264_LONG_TERM_REFERENCE;
1612         else
1613             pic->flags |= VA_PICTURE_H264_SHORT_TERM_REFERENCE;
1614     }
1615
1616     init_picture_poc(decoder, picture, slice_hdr);
1617     if (!init_picture_refs(decoder, picture, slice_hdr)) {
1618         GST_ERROR("failed to initialize references");
1619         return FALSE;
1620     }
1621     return TRUE;
1622 }
1623
1624 /* 8.2.5.3 - Sliding window decoded reference picture marking process */
1625 static gboolean
1626 exec_ref_pic_marking_sliding_window(GstVaapiDecoderH264 *decoder)
1627 {
1628     GstVaapiDecoderH264Private * const priv = decoder->priv;
1629     GstH264SPS * const sps = priv->sps;
1630     guint i, max_num_ref_frames, lowest_frame_num_index;
1631     gint32 lowest_frame_num;
1632
1633     GST_DEBUG("reference picture marking process (sliding window)");
1634
1635     max_num_ref_frames = sps->num_ref_frames;
1636     if (max_num_ref_frames == 0)
1637         max_num_ref_frames = 1;
1638
1639     if (priv->short_ref_count + priv->long_ref_count < max_num_ref_frames)
1640         return TRUE;
1641     if (priv->short_ref_count < 1)
1642         return FALSE;
1643
1644     lowest_frame_num = priv->short_ref[0]->frame_num_wrap;
1645     lowest_frame_num_index = 0;
1646     for (i = 1; i < priv->short_ref_count; i++) {
1647         if (priv->short_ref[i]->frame_num_wrap < lowest_frame_num) {
1648             lowest_frame_num = priv->short_ref[i]->frame_num_wrap;
1649             lowest_frame_num_index = i;
1650         }
1651     }
1652
1653     remove_reference_at(
1654         decoder,
1655         priv->short_ref, &priv->short_ref_count,
1656         lowest_frame_num_index
1657     );
1658     return TRUE;
1659 }
1660
1661 /* 8.2.5.4. Adaptive memory control decoded reference picture marking process */
1662 static gboolean
1663 exec_ref_pic_marking_adaptive(
1664     GstVaapiDecoderH264     *decoder,
1665     GstVaapiPictureH264     *picture,
1666     GstH264DecRefPicMarking *dec_ref_pic_marking
1667 )
1668 {
1669     GstVaapiDecoderH264Private * const priv = decoder->priv;
1670     gint32 pic_num, ref_idx;
1671     guint i;
1672
1673     GST_DEBUG("reference picture marking process (adaptive memory control)");
1674
1675     for (i = 0; i < dec_ref_pic_marking->n_ref_pic_marking; i++) {
1676         GstH264RefPicMarking * const ref_pic_marking =
1677             &dec_ref_pic_marking->ref_pic_marking[i];
1678
1679         switch (ref_pic_marking->memory_management_control_operation) {
1680         case 1:
1681             // Mark short-term reference picture as "unused for reference"
1682             if (!picture->field_pic_flag)
1683                 pic_num = picture->frame_num_wrap;
1684             else
1685                 pic_num = 2 * picture->frame_num_wrap + 1;
1686             pic_num -= ref_pic_marking->difference_of_pic_nums_minus1 + 1;
1687             ref_idx = find_short_term_reference(decoder, pic_num);
1688             if (ref_idx < 0)
1689                 break;
1690             remove_reference_at(
1691                 decoder,
1692                 priv->short_ref, &priv->short_ref_count,
1693                 ref_idx
1694             );
1695             break;
1696         case 2:
1697             // Mark long-term reference picture as "unused for reference"
1698             pic_num = picture->long_term_pic_num;
1699             ref_idx = find_long_term_reference(decoder, pic_num);
1700             if (ref_idx < 0)
1701                 break;
1702             remove_reference_at(
1703                 decoder,
1704                 priv->long_ref, &priv->long_ref_count,
1705                 ref_idx
1706             );
1707             break;
1708         case 3:
1709             // Assign LongTermFrameIdx to a short-term reference picture
1710             if (!picture->field_pic_flag)
1711                 pic_num = picture->frame_num_wrap;
1712             else
1713                 pic_num = 2 * picture->frame_num_wrap + 1;
1714             pic_num -= ref_pic_marking->difference_of_pic_nums_minus1 + 1;
1715             ref_idx = find_short_term_reference(decoder, pic_num);
1716             if (ref_idx < 0)
1717                 break;
1718             break;
1719         case 5:
1720             // Mark all reference pictures as "unused for reference"
1721             clear_references(decoder, priv->short_ref, &priv->short_ref_count);
1722             clear_references(decoder, priv->long_ref,  &priv->long_ref_count );
1723             break;
1724         default:
1725             g_assert(0 && "unhandled MMCO");
1726             break;
1727         }
1728     }
1729     return TRUE;
1730 }
1731
1732 /* 8.2.5 - Execute reference picture marking process */
1733 static gboolean
1734 exec_ref_pic_marking(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
1735 {
1736     GstVaapiDecoderH264Private * const priv = decoder->priv;
1737     GstVaapiPictureH264 **picture_ptr;
1738
1739     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
1740         return TRUE;
1741
1742     if (!picture->is_idr) {
1743         GstH264DecRefPicMarking * const dec_ref_pic_marking =
1744             get_dec_ref_pic_marking(picture);
1745         if (dec_ref_pic_marking->adaptive_ref_pic_marking_mode_flag) {
1746             if (!exec_ref_pic_marking_adaptive(decoder, picture, dec_ref_pic_marking))
1747                 return FALSE;
1748         }
1749         else {
1750             if (!exec_ref_pic_marking_sliding_window(decoder))
1751                 return FALSE;
1752         }
1753     }
1754
1755     if (picture->is_long_term)
1756         picture_ptr = &priv->long_ref[priv->long_ref_count++];
1757     else
1758         picture_ptr = &priv->short_ref[priv->short_ref_count++];
1759     gst_vaapi_picture_replace(picture_ptr, picture);
1760     return TRUE;
1761 }
1762
1763 /* Update picture order count */
1764 static void
1765 exit_picture_poc(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
1766 {
1767     GstVaapiDecoderH264Private * const priv = decoder->priv;
1768     GstH264SPS * const sps = priv->sps;
1769
1770     switch (sps->pic_order_cnt_type) {
1771     case 0:
1772         if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
1773             break;
1774         if (picture->has_mmco_5) {
1775             priv->prev_poc_msb = 0;
1776             if (!picture->field_pic_flag || !picture->bottom_field_flag)
1777                 priv->prev_poc_lsb = picture->info.TopFieldOrderCnt;
1778             else
1779                 priv->prev_poc_lsb = 0;
1780         }
1781         else {
1782             priv->prev_poc_msb = priv->poc_msb;
1783             priv->prev_poc_lsb = priv->poc_lsb;
1784         }
1785         break;
1786     case 1:
1787     case 2:
1788         priv->prev_frame_num = priv->frame_num;
1789         if (picture->has_mmco_5)
1790             priv->prev_frame_num_offset = 0;
1791         else
1792             priv->prev_frame_num_offset = priv->frame_num_offset;
1793         break;
1794     }
1795 }
1796
1797 static inline gboolean
1798 exit_picture(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
1799 {
1800     /* Update picture order count */
1801     exit_picture_poc(decoder, picture);
1802
1803     /* Decoded reference picture marking process */
1804     if (!exec_ref_pic_marking(decoder, picture))
1805         return FALSE;
1806     return TRUE;
1807 }
1808
1809 static void
1810 vaapi_init_picture(VAPictureH264 *pic)
1811 {
1812     pic->picture_id           = VA_INVALID_ID;
1813     pic->frame_idx            = 0;
1814     pic->flags                = VA_PICTURE_H264_INVALID;
1815     pic->TopFieldOrderCnt     = 0;
1816     pic->BottomFieldOrderCnt  = 0;
1817 }
1818
1819 static gboolean
1820 fill_picture(
1821     GstVaapiDecoderH264 *decoder,
1822     GstVaapiPictureH264 *picture,
1823     GstH264SliceHdr     *slice_hdr,
1824     GstH264NalUnit      *nalu
1825 )
1826 {
1827     GstVaapiDecoderH264Private * const priv = decoder->priv;
1828     GstVaapiPicture * const base_picture = &picture->base;
1829     GstH264SPS * const sps = priv->sps;
1830     GstH264PPS * const pps = priv->pps;
1831     VAPictureParameterBufferH264 * const pic_param = base_picture->param;
1832     guint i, n;
1833
1834     /* Fill in VAPictureParameterBufferH264 */
1835     pic_param->CurrPic = picture->info;
1836     for (i = 0, n = 0; i < priv->short_ref_count; i++)
1837         pic_param->ReferenceFrames[n++] = priv->short_ref[i]->info;
1838     for (i = 0; i < priv->long_ref_count; i++)
1839         pic_param->ReferenceFrames[n++] = priv->long_ref[i]->info;
1840     for (; n < G_N_ELEMENTS(pic_param->ReferenceFrames); n++)
1841         vaapi_init_picture(&pic_param->ReferenceFrames[n]);
1842
1843 #define COPY_FIELD(s, f) \
1844     pic_param->f = (s)->f
1845
1846 #define COPY_BFM(a, s, f) \
1847     pic_param->a.bits.f = (s)->f
1848
1849     pic_param->picture_width_in_mbs_minus1      = priv->mb_width  - 1;
1850     pic_param->picture_height_in_mbs_minus1     = priv->mb_height - 1;
1851     pic_param->frame_num                        = priv->frame_num;
1852
1853     COPY_FIELD(sps, bit_depth_luma_minus8);
1854     COPY_FIELD(sps, bit_depth_chroma_minus8);
1855     COPY_FIELD(sps, num_ref_frames);
1856     COPY_FIELD(pps, num_slice_groups_minus1);
1857     COPY_FIELD(pps, slice_group_map_type);
1858     COPY_FIELD(pps, slice_group_change_rate_minus1);
1859     COPY_FIELD(pps, pic_init_qp_minus26);
1860     COPY_FIELD(pps, pic_init_qs_minus26);
1861     COPY_FIELD(pps, chroma_qp_index_offset);
1862     COPY_FIELD(pps, second_chroma_qp_index_offset);
1863
1864     pic_param->seq_fields.value                                         = 0; /* reset all bits */
1865     pic_param->seq_fields.bits.residual_colour_transform_flag           = sps->separate_colour_plane_flag;
1866     pic_param->seq_fields.bits.MinLumaBiPredSize8x8                     = sps->level_idc >= 31; /* A.3.3.2 */
1867
1868     COPY_BFM(seq_fields, sps, chroma_format_idc);
1869     COPY_BFM(seq_fields, sps, gaps_in_frame_num_value_allowed_flag);
1870     COPY_BFM(seq_fields, sps, frame_mbs_only_flag);
1871     COPY_BFM(seq_fields, sps, mb_adaptive_frame_field_flag);
1872     COPY_BFM(seq_fields, sps, direct_8x8_inference_flag);
1873     COPY_BFM(seq_fields, sps, log2_max_frame_num_minus4);
1874     COPY_BFM(seq_fields, sps, pic_order_cnt_type);
1875     COPY_BFM(seq_fields, sps, log2_max_pic_order_cnt_lsb_minus4);
1876     COPY_BFM(seq_fields, sps, delta_pic_order_always_zero_flag);
1877
1878     pic_param->pic_fields.value                                         = 0; /* reset all bits */
1879     pic_param->pic_fields.bits.field_pic_flag                           = slice_hdr->field_pic_flag;
1880     pic_param->pic_fields.bits.reference_pic_flag                       = GST_VAAPI_PICTURE_IS_REFERENCE(picture);
1881
1882     COPY_BFM(pic_fields, pps, entropy_coding_mode_flag);
1883     COPY_BFM(pic_fields, pps, weighted_pred_flag);
1884     COPY_BFM(pic_fields, pps, weighted_bipred_idc);
1885     COPY_BFM(pic_fields, pps, transform_8x8_mode_flag);
1886     COPY_BFM(pic_fields, pps, constrained_intra_pred_flag);
1887     COPY_BFM(pic_fields, pps, pic_order_present_flag);
1888     COPY_BFM(pic_fields, pps, deblocking_filter_control_present_flag);
1889     COPY_BFM(pic_fields, pps, redundant_pic_cnt_present_flag);
1890     return TRUE;
1891 }
1892
1893 static gboolean
1894 fill_quant_matrix(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
1895 {
1896     GstVaapiDecoderH264Private * const priv = decoder->priv;
1897     VAIQMatrixBufferH264 * const iq_matrix = picture->base.iq_matrix->param;
1898
1899     /* XXX: we can only support 4:2:0 or 4:2:2 since ScalingLists8x8[]
1900        is not large enough to hold lists for 4:4:4 */
1901     if (priv->sps->chroma_format_idc == 3 &&
1902         sizeof(iq_matrix->ScalingList8x8) != sizeof(priv->scaling_list_8x8))
1903         return FALSE;
1904
1905     /* Fill in VAIQMatrixBufferH264 */
1906     memcpy(iq_matrix->ScalingList4x4, priv->scaling_list_4x4,
1907            sizeof(iq_matrix->ScalingList4x4));
1908     memcpy(iq_matrix->ScalingList8x8, priv->scaling_list_8x8,
1909            sizeof(iq_matrix->ScalingList8x8));
1910     return TRUE;
1911 }
1912
1913 static GstVaapiDecoderStatus
1914 decode_picture(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu, GstH264SliceHdr *slice_hdr)
1915 {
1916     GstVaapiDecoderH264Private * const priv = decoder->priv;
1917     GstVaapiPictureH264 *picture;
1918     GstVaapiDecoderStatus status;
1919     GstH264PPS * const pps = slice_hdr->pps;
1920     GstH264SPS * const sps = pps->sequence;
1921
1922     status = ensure_context(decoder, sps);
1923     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS) {
1924         GST_ERROR("failed to reset context");
1925         return status;
1926     }
1927
1928     if (priv->current_picture && !decode_current_picture(decoder))
1929         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
1930
1931     picture = gst_vaapi_picture_h264_new(decoder);
1932     if (!picture) {
1933         GST_ERROR("failed to allocate picture");
1934         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
1935     }
1936     priv->current_picture = picture;
1937
1938     picture->base.iq_matrix = GST_VAAPI_IQ_MATRIX_NEW(H264, decoder);
1939     if (!picture->base.iq_matrix) {
1940         GST_ERROR("failed to allocate IQ matrix");
1941         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
1942     }
1943
1944     status = ensure_quant_matrix(decoder, pps);
1945     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS) {
1946         GST_ERROR("failed to reset quantizer matrix");
1947         return status;
1948     }
1949
1950     priv->sps = sps;
1951     priv->pps = pps;
1952
1953     if (!init_picture(decoder, picture, slice_hdr, nalu))
1954         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
1955     if (!fill_picture(decoder, picture, slice_hdr, nalu))
1956         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
1957     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1958 }
1959
1960 static gboolean
1961 decode_picture_end(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
1962 {
1963     if (!fill_quant_matrix(decoder, picture))
1964         return FALSE;
1965     if (!exit_picture(decoder, picture))
1966         return FALSE;
1967     if (!dpb_add(decoder, picture))
1968         return FALSE;
1969     return TRUE;
1970 }
1971
1972 #ifndef HAVE_GST_H264_SLICE_HDR_EPB_COUNT
1973 static guint
1974 get_epb_count(const guint8 *buf, guint buf_size, guint header_size)
1975 {
1976     guint i, n = 0;
1977
1978     if (buf_size > header_size)
1979         buf_size = header_size;
1980
1981     for (i = 2; i < buf_size; i++) {
1982         if (!buf[i - 2] && !buf[i - 1] && buf[i] == 0x03)
1983             i += 2, n++;
1984     }
1985     return n;
1986 }
1987 #endif
1988
1989 static inline guint
1990 get_slice_data_bit_offset(GstH264SliceHdr *slice_hdr, GstH264NalUnit *nalu)
1991 {
1992     guint epb_count;
1993
1994 #ifdef HAVE_GST_H264_SLICE_HDR_EPB_COUNT
1995     epb_count = slice_hdr->n_emulation_prevention_bytes;
1996 #else
1997     epb_count = get_epb_count(
1998         nalu->data + nalu->offset,
1999         nalu->size,
2000         slice_hdr->header_size / 8
2001     );
2002 #endif
2003     return 8 /* nal_unit_type */ + slice_hdr->header_size - epb_count * 8;
2004 }
2005
2006 static gboolean
2007 fill_pred_weight_table(GstVaapiDecoderH264 *decoder, GstVaapiSliceH264 *slice)
2008 {
2009     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2010     GstH264PPS * const pps = slice_hdr->pps;
2011     GstH264SPS * const sps = pps->sequence;
2012     GstH264PredWeightTable * const w = &slice_hdr->pred_weight_table;
2013     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2014     guint num_weight_tables = 0;
2015     gint i, j;
2016
2017     if (pps->weighted_pred_flag &&
2018         (GST_H264_IS_P_SLICE(slice_hdr) || GST_H264_IS_SP_SLICE(slice_hdr)))
2019         num_weight_tables = 1;
2020     else if (pps->weighted_bipred_idc == 1 && GST_H264_IS_B_SLICE(slice_hdr))
2021         num_weight_tables = 2;
2022     else
2023         num_weight_tables = 0;
2024
2025     slice_param->luma_log2_weight_denom   = w->luma_log2_weight_denom;
2026     slice_param->chroma_log2_weight_denom = w->chroma_log2_weight_denom;
2027     slice_param->luma_weight_l0_flag      = 0;
2028     slice_param->chroma_weight_l0_flag    = 0;
2029     slice_param->luma_weight_l1_flag      = 0;
2030     slice_param->chroma_weight_l1_flag    = 0;
2031
2032     if (num_weight_tables < 1)
2033         return TRUE;
2034
2035     slice_param->luma_weight_l0_flag = 1;
2036     for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2037         slice_param->luma_weight_l0[i] = w->luma_weight_l0[i];
2038         slice_param->luma_offset_l0[i] = w->luma_offset_l0[i];
2039     }
2040
2041     slice_param->chroma_weight_l0_flag = sps->chroma_array_type != 0;
2042     if (slice_param->chroma_weight_l0_flag) {
2043         for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2044             for (j = 0; j < 2; j++) {
2045                 slice_param->chroma_weight_l0[i][j] = w->chroma_weight_l0[i][j];
2046                 slice_param->chroma_offset_l0[i][j] = w->chroma_offset_l0[i][j];
2047             }
2048         }
2049     }
2050
2051     if (num_weight_tables < 2)
2052         return TRUE;
2053
2054     slice_param->luma_weight_l1_flag = 1;
2055     for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2056         slice_param->luma_weight_l1[i] = w->luma_weight_l1[i];
2057         slice_param->luma_offset_l1[i] = w->luma_offset_l1[i];
2058     }
2059
2060     slice_param->chroma_weight_l1_flag = sps->chroma_array_type != 0;
2061     if (slice_param->chroma_weight_l1_flag) {
2062         for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2063             for (j = 0; j < 2; j++) {
2064                 slice_param->chroma_weight_l1[i][j] = w->chroma_weight_l1[i][j];
2065                 slice_param->chroma_offset_l1[i][j] = w->chroma_offset_l1[i][j];
2066             }
2067         }
2068     }
2069     return TRUE;
2070 }
2071
2072 static gboolean
2073 fill_RefPicList(GstVaapiDecoderH264 *decoder, GstVaapiSliceH264 *slice)
2074 {
2075     GstVaapiDecoderH264Private * const priv = decoder->priv;
2076     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2077     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2078     guint i, num_ref_lists = 0;
2079
2080     slice_param->num_ref_idx_l0_active_minus1 = 0;
2081     slice_param->num_ref_idx_l1_active_minus1 = 0;
2082
2083     if (GST_H264_IS_B_SLICE(slice_hdr))
2084         num_ref_lists = 2;
2085     else if (GST_H264_IS_I_SLICE(slice_hdr))
2086         num_ref_lists = 0;
2087     else
2088         num_ref_lists = 1;
2089
2090     if (num_ref_lists < 1)
2091         return TRUE;
2092
2093     slice_param->num_ref_idx_l0_active_minus1 =
2094         slice_hdr->num_ref_idx_l0_active_minus1;
2095
2096     for (i = 0; i < priv->RefPicList0_count && priv->RefPicList0[i]; i++)
2097         slice_param->RefPicList0[i] = priv->RefPicList0[i]->info;
2098     for (; i <= slice_param->num_ref_idx_l0_active_minus1; i++)
2099         vaapi_init_picture(&slice_param->RefPicList0[i]);
2100
2101     if (num_ref_lists < 2)
2102         return TRUE;
2103
2104     slice_param->num_ref_idx_l1_active_minus1 =
2105         slice_hdr->num_ref_idx_l1_active_minus1;
2106
2107     for (i = 0; i < priv->RefPicList1_count && priv->RefPicList1[i]; i++)
2108         slice_param->RefPicList1[i] = priv->RefPicList1[i]->info;
2109     for (; i <= slice_param->num_ref_idx_l1_active_minus1; i++)
2110         vaapi_init_picture(&slice_param->RefPicList1[i]);
2111     return TRUE;
2112 }
2113
2114 static gboolean
2115 fill_slice(
2116     GstVaapiDecoderH264 *decoder,
2117     GstVaapiSliceH264   *slice,
2118     GstH264NalUnit      *nalu
2119 )
2120 {
2121     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2122     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2123
2124     /* Fill in VASliceParameterBufferH264 */
2125     slice_param->slice_data_bit_offset          = get_slice_data_bit_offset(slice_hdr, nalu);
2126     slice_param->first_mb_in_slice              = slice_hdr->first_mb_in_slice;
2127     slice_param->slice_type                     = slice_hdr->type % 5;
2128     slice_param->direct_spatial_mv_pred_flag    = slice_hdr->direct_spatial_mv_pred_flag;
2129     slice_param->cabac_init_idc                 = slice_hdr->cabac_init_idc;
2130     slice_param->slice_qp_delta                 = slice_hdr->slice_qp_delta;
2131     slice_param->disable_deblocking_filter_idc  = slice_hdr->disable_deblocking_filter_idc;
2132     slice_param->slice_alpha_c0_offset_div2     = slice_hdr->slice_alpha_c0_offset_div2;
2133     slice_param->slice_beta_offset_div2         = slice_hdr->slice_beta_offset_div2;
2134
2135     if (!fill_RefPicList(decoder, slice))
2136         return FALSE;
2137     if (!fill_pred_weight_table(decoder, slice))
2138         return FALSE;
2139     return TRUE;
2140 }
2141
2142 static GstVaapiDecoderStatus
2143 decode_slice(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
2144 {
2145     GstVaapiDecoderH264Private * const priv = decoder->priv;
2146     GstVaapiDecoderStatus status;
2147     GstVaapiPictureH264 *picture;
2148     GstVaapiSliceH264 *slice = NULL;
2149     GstH264SliceHdr *slice_hdr;
2150     GstH264ParserResult result;
2151
2152     GST_DEBUG("slice (%u bytes)", nalu->size);
2153
2154     slice = gst_vaapi_slice_h264_new(
2155         decoder,
2156         nalu->data + nalu->offset,
2157         nalu->size
2158     );
2159     if (!slice) {
2160         GST_ERROR("failed to allocate slice");
2161         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2162     }
2163
2164     slice_hdr = &slice->slice_hdr;
2165     memset(slice_hdr, 0, sizeof(*slice_hdr));
2166     result = gst_h264_parser_parse_slice_hdr(priv->parser, nalu, slice_hdr, TRUE, TRUE);
2167     if (result != GST_H264_PARSER_OK) {
2168         status = get_status(result);
2169         goto error;
2170     }
2171
2172     if (slice_hdr->first_mb_in_slice == 0) {
2173         status = decode_picture(decoder, nalu, slice_hdr);
2174         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2175             goto error;
2176     }
2177     picture = priv->current_picture;
2178
2179     priv->mb_x = slice_hdr->first_mb_in_slice % priv->mb_width;
2180     priv->mb_y = slice_hdr->first_mb_in_slice / priv->mb_width; // FIXME: MBAFF or field
2181
2182     if (!fill_slice(decoder, slice, nalu)) {
2183         status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2184         goto error;
2185     }
2186     gst_vaapi_picture_add_slice(
2187         GST_VAAPI_PICTURE_CAST(picture),
2188         GST_VAAPI_SLICE_CAST(slice)
2189     );
2190
2191     /* Commit picture for decoding if we reached the last slice */
2192     if (++priv->mb_y >= priv->mb_height) {
2193         if (!decode_current_picture(decoder)) {
2194             status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2195             goto error;
2196         }
2197         GST_DEBUG("done");
2198     }
2199     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2200
2201 error:
2202     if (slice)
2203         gst_mini_object_unref(GST_MINI_OBJECT(slice));
2204     return status;
2205 }
2206
2207 static inline gint
2208 scan_for_start_code(GstAdapter *adapter, guint ofs, guint size, guint32 *scp)
2209 {
2210     return (gint)gst_adapter_masked_scan_uint32_peek(adapter,
2211                                                      0xffffff00, 0x00000100,
2212                                                      ofs, size,
2213                                                      scp);
2214 }
2215
2216 static GstVaapiDecoderStatus
2217 decode_nalu(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
2218 {
2219     GstVaapiDecoderStatus status;
2220
2221     switch (nalu->type) {
2222     case GST_H264_NAL_SLICE_IDR:
2223         /* fall-through. IDR specifics are handled in init_picture() */
2224     case GST_H264_NAL_SLICE:
2225         status = decode_slice(decoder, nalu);
2226         break;
2227     case GST_H264_NAL_SPS:
2228         status = decode_sps(decoder, nalu);
2229         break;
2230     case GST_H264_NAL_PPS:
2231         status = decode_pps(decoder, nalu);
2232         break;
2233     case GST_H264_NAL_SEI:
2234         status = decode_sei(decoder, nalu);
2235         break;
2236     case GST_H264_NAL_SEQ_END:
2237         status = decode_sequence_end(decoder);
2238         break;
2239     case GST_H264_NAL_AU_DELIMITER:
2240         /* skip all Access Unit NALs */
2241         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2242         break;
2243     case GST_H264_NAL_FILLER_DATA:
2244         /* skip all Filler Data NALs */
2245         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2246         break;
2247     default:
2248         GST_WARNING("unsupported NAL unit type %d", nalu->type);
2249         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
2250         break;
2251     }
2252     return status;
2253 }
2254
2255 static GstVaapiDecoderStatus
2256 decode_buffer(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
2257 {
2258     GstVaapiDecoderH264Private * const priv = decoder->priv;
2259     GstVaapiDecoderStatus status;
2260     GstH264ParserResult result;
2261     GstH264NalUnit nalu;
2262     gboolean is_eos;
2263     const guchar *buf;
2264     guint i, buf_size, nalu_size, size;
2265     guint32 start_code;
2266     gint ofs;
2267
2268     buf      = GST_BUFFER_DATA(buffer);
2269     buf_size = GST_BUFFER_SIZE(buffer);
2270     is_eos   = GST_BUFFER_IS_EOS(buffer);
2271     if (buf && buf_size > 0)
2272         gst_adapter_push(priv->adapter, gst_buffer_ref(buffer));
2273
2274     size = gst_adapter_available(priv->adapter);
2275     do {
2276         if (size == 0) {
2277             status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2278             break;
2279         }
2280
2281         status = gst_vaapi_decoder_check_status(GST_VAAPI_DECODER(decoder));
2282         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2283             break;
2284
2285         status = GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2286         if (priv->is_avc) {
2287             if (size < priv->nal_length_size)
2288                 break;
2289             buf = gst_adapter_peek(priv->adapter, priv->nal_length_size);
2290
2291             nalu_size = 0;
2292             for (i = 0; i < priv->nal_length_size; i++)
2293                 nalu_size = (nalu_size << 8) | buf[i];
2294
2295             buf_size = priv->nal_length_size + nalu_size;
2296             if (size < buf_size)
2297                 break;
2298             buffer = gst_adapter_take_buffer(priv->adapter, buf_size);
2299             size -= buf_size;
2300
2301             buf      = GST_BUFFER_DATA(buffer);
2302             buf_size = GST_BUFFER_SIZE(buffer);
2303
2304             result = gst_h264_parser_identify_nalu_avc(
2305                 priv->parser,
2306                 buf, 0, buf_size, priv->nal_length_size,
2307                 &nalu
2308             );
2309         }
2310         else {
2311             if (size < 4)
2312                 break;
2313             ofs = scan_for_start_code(priv->adapter, 0, size, &start_code);
2314             if (ofs < 0)
2315                 break;
2316             gst_adapter_flush(priv->adapter, ofs);
2317             size -= ofs;
2318
2319             ofs = G_UNLIKELY(size < 8) ? -1 :
2320                 scan_for_start_code(priv->adapter, 4, size - 4, NULL);
2321             if (ofs < 0) {
2322                 // Assume the whole NAL unit is present if end-of-stream
2323                 if (!is_eos)
2324                     break;
2325                 ofs = size;
2326             }
2327             buffer = gst_adapter_take_buffer(priv->adapter, ofs);
2328             size -= ofs;
2329
2330             buf      = GST_BUFFER_DATA(buffer);
2331             buf_size = GST_BUFFER_SIZE(buffer);
2332
2333             result = gst_h264_parser_identify_nalu_unchecked(
2334                 priv->parser,
2335                 buf, 0, buf_size,
2336                 &nalu
2337             );
2338         }
2339         status = get_status(result);
2340         if (status == GST_VAAPI_DECODER_STATUS_SUCCESS)
2341             status = decode_nalu(decoder, &nalu);
2342         gst_buffer_unref(buffer);
2343     } while (status == GST_VAAPI_DECODER_STATUS_SUCCESS);
2344
2345     if (is_eos && (status == GST_VAAPI_DECODER_STATUS_SUCCESS ||
2346                    status == GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA))
2347         status = decode_sequence_end(decoder);
2348     return status;
2349 }
2350
2351 static GstVaapiDecoderStatus
2352 decode_codec_data(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
2353 {
2354     GstVaapiDecoderH264Private * const priv = decoder->priv;
2355     GstVaapiDecoderStatus status;
2356     GstH264NalUnit nalu;
2357     GstH264ParserResult result;
2358     guchar *buf;
2359     guint buf_size;
2360     guint i, ofs, num_sps, num_pps;
2361
2362     buf      = GST_BUFFER_DATA(buffer);
2363     buf_size = GST_BUFFER_SIZE(buffer);
2364     if (!buf || buf_size == 0)
2365         return GST_VAAPI_DECODER_STATUS_SUCCESS;
2366
2367     if (buf_size < 8)
2368         return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2369
2370     if (buf[0] != 1) {
2371         GST_ERROR("failed to decode codec-data, not in avcC format");
2372         return GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
2373     }
2374
2375     priv->nal_length_size = (buf[4] & 0x03) + 1;
2376
2377     num_sps = buf[5] & 0x1f;
2378     ofs = 6;
2379
2380     for (i = 0; i < num_sps; i++) {
2381         result = gst_h264_parser_identify_nalu_avc(
2382             priv->parser,
2383             buf, ofs, buf_size, 2,
2384             &nalu
2385         );
2386         if (result != GST_H264_PARSER_OK)
2387             return get_status(result);
2388
2389         status = decode_sps(decoder, &nalu);
2390         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2391             return status;
2392         ofs = nalu.offset + nalu.size;
2393     }
2394
2395     num_pps = buf[ofs];
2396     ofs++;
2397
2398     for (i = 0; i < num_pps; i++) {
2399         result = gst_h264_parser_identify_nalu_avc(
2400             priv->parser,
2401             buf, ofs, buf_size, 2,
2402             &nalu
2403         );
2404         if (result != GST_H264_PARSER_OK)
2405             return get_status(result);
2406
2407         status = decode_pps(decoder, &nalu);
2408         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2409             return status;
2410         ofs = nalu.offset + nalu.size;
2411     }
2412
2413     priv->is_avc = TRUE;
2414     return status;
2415 }
2416
2417 GstVaapiDecoderStatus
2418 gst_vaapi_decoder_h264_decode(GstVaapiDecoder *base, GstBuffer *buffer)
2419 {
2420     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264(base);
2421     GstVaapiDecoderH264Private * const priv = decoder->priv;
2422     GstVaapiDecoderStatus status;
2423     GstBuffer *codec_data;
2424
2425     g_return_val_if_fail(priv->is_constructed,
2426                          GST_VAAPI_DECODER_STATUS_ERROR_INIT_FAILED);
2427
2428     if (!priv->is_opened) {
2429         priv->is_opened = gst_vaapi_decoder_h264_open(decoder, buffer);
2430         if (!priv->is_opened)
2431             return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CODEC;
2432
2433         codec_data = GST_VAAPI_DECODER_CODEC_DATA(decoder);
2434         if (codec_data) {
2435             status = decode_codec_data(decoder, codec_data);
2436             if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2437                 return status;
2438         }
2439      }
2440      return decode_buffer(decoder, buffer);
2441 }
2442
2443 static void
2444 gst_vaapi_decoder_h264_finalize(GObject *object)
2445 {
2446     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264(object);
2447
2448     gst_vaapi_decoder_h264_destroy(decoder);
2449
2450     G_OBJECT_CLASS(gst_vaapi_decoder_h264_parent_class)->finalize(object);
2451 }
2452
2453 static void
2454 gst_vaapi_decoder_h264_constructed(GObject *object)
2455 {
2456     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264(object);
2457     GstVaapiDecoderH264Private * const priv = decoder->priv;
2458     GObjectClass *parent_class;
2459
2460     parent_class = G_OBJECT_CLASS(gst_vaapi_decoder_h264_parent_class);
2461     if (parent_class->constructed)
2462         parent_class->constructed(object);
2463
2464     priv->is_constructed = gst_vaapi_decoder_h264_create(decoder);
2465 }
2466
2467 static void
2468 gst_vaapi_decoder_h264_class_init(GstVaapiDecoderH264Class *klass)
2469 {
2470     GObjectClass * const object_class = G_OBJECT_CLASS(klass);
2471     GstVaapiDecoderClass * const decoder_class = GST_VAAPI_DECODER_CLASS(klass);
2472
2473     g_type_class_add_private(klass, sizeof(GstVaapiDecoderH264Private));
2474
2475     object_class->finalize      = gst_vaapi_decoder_h264_finalize;
2476     object_class->constructed   = gst_vaapi_decoder_h264_constructed;
2477
2478     decoder_class->decode       = gst_vaapi_decoder_h264_decode;
2479 }
2480
2481 static void
2482 gst_vaapi_decoder_h264_init(GstVaapiDecoderH264 *decoder)
2483 {
2484     GstVaapiDecoderH264Private *priv;
2485
2486     priv                        = GST_VAAPI_DECODER_H264_GET_PRIVATE(decoder);
2487     decoder->priv               = priv;
2488     priv->parser                = NULL;
2489     priv->sps                   = &priv->last_sps;
2490     priv->pps                   = &priv->last_pps;
2491     priv->current_picture       = NULL;
2492     priv->dpb_count             = 0;
2493     priv->dpb_size              = 0;
2494     priv->profile               = GST_VAAPI_PROFILE_H264_HIGH;
2495     priv->short_ref_count       = 0;
2496     priv->long_ref_count        = 0;
2497     priv->RefPicList0_count     = 0;
2498     priv->RefPicList1_count     = 0;
2499     priv->nal_length_size       = 0;
2500     priv->width                 = 0;
2501     priv->height                = 0;
2502     priv->mb_x                  = 0;
2503     priv->mb_y                  = 0;
2504     priv->mb_width              = 0;
2505     priv->mb_height             = 0;
2506     priv->adapter               = NULL;
2507     priv->field_poc[0]          = 0;
2508     priv->field_poc[1]          = 0;
2509     priv->poc_msb               = 0;
2510     priv->poc_lsb               = 0;
2511     priv->prev_poc_msb          = 0;
2512     priv->prev_poc_lsb          = 0;
2513     priv->frame_num_offset      = 0;
2514     priv->prev_frame_num_offset = 0;
2515     priv->frame_num             = 0;
2516     priv->prev_frame_num        = 0;
2517     priv->is_constructed        = FALSE;
2518     priv->is_opened             = FALSE;
2519     priv->is_avc                = FALSE;
2520     priv->has_context           = FALSE;
2521
2522     memset(priv->dpb, 0, sizeof(priv->dpb));
2523     memset(priv->short_ref, 0, sizeof(priv->short_ref));
2524     memset(priv->long_ref, 0, sizeof(priv->long_ref));
2525     memset(priv->RefPicList0, 0, sizeof(priv->RefPicList0));
2526     memset(priv->RefPicList1, 0, sizeof(priv->RefPicList1));
2527 }
2528
2529 /**
2530  * gst_vaapi_decoder_h264_new:
2531  * @display: a #GstVaapiDisplay
2532  * @caps: a #GstCaps holding codec information
2533  *
2534  * Creates a new #GstVaapiDecoder for MPEG-2 decoding.  The @caps can
2535  * hold extra information like codec-data and pictured coded size.
2536  *
2537  * Return value: the newly allocated #GstVaapiDecoder object
2538  */
2539 GstVaapiDecoder *
2540 gst_vaapi_decoder_h264_new(GstVaapiDisplay *display, GstCaps *caps)
2541 {
2542     GstVaapiDecoderH264 *decoder;
2543
2544     g_return_val_if_fail(GST_VAAPI_IS_DISPLAY(display), NULL);
2545     g_return_val_if_fail(GST_IS_CAPS(caps), NULL);
2546
2547     decoder = g_object_new(
2548         GST_VAAPI_TYPE_DECODER_H264,
2549         "display",      display,
2550         "caps",         caps,
2551         NULL
2552     );
2553     if (!decoder->priv->is_constructed) {
2554         g_object_unref(decoder);
2555         return NULL;
2556     }
2557     return GST_VAAPI_DECODER_CAST(decoder);
2558 }