gst-libs/gst/vaapi/gstvaapidecoder_h264.c

   1 /*
   2  *  gstvaapidecoder_h264.c - H.264 decoder
   3  *
   4  *  Copyright (C) 2011-2012 Intel Corporation
   5  *
   6  *  This library is free software; you can redistribute it and/or
   7  *  modify it under the terms of the GNU Lesser General Public License
   8  *  as published by the Free Software Foundation; either version 2.1
   9  *  of the License, or (at your option) any later version.
  10  *
  11  *  This library is distributed in the hope that it will be useful,
  12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  *  Lesser General Public License for more details.
  15  *
  16  *  You should have received a copy of the GNU Lesser General Public
  17  *  License along with this library; if not, write to the Free
  18  *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  19  *  Boston, MA 02110-1301 USA
  20  */
  21
  22 /**
  23  * SECTION:gstvaapidecoder_h264
  24  * @short_description: H.264 decoder
  25  */
  26
  27 #include "sysdeps.h"
  28 #include <string.h>
  29 #include <gst/base/gstadapter.h>
  30 #include <gst/codecparsers/gsth264parser.h>
  31 #include "gstvaapidecoder_h264.h"
  32 #include "gstvaapidecoder_objects.h"
  33 #include "gstvaapidecoder_priv.h"
  34 #include "gstvaapidisplay_priv.h"
  35 #include "gstvaapiobject_priv.h"
  36
  37 #define DEBUG 1
  38 #include "gstvaapidebug.h"
  39
  40 /* Defined to 1 if strict ordering of DPB is needed. Only useful for debug */
  41 #define USE_STRICT_DPB_ORDERING 0
  42
  43 typedef struct _GstVaapiPictureH264             GstVaapiPictureH264;
  44 typedef struct _GstVaapiPictureH264Class        GstVaapiPictureH264Class;
  45 typedef struct _GstVaapiSliceH264               GstVaapiSliceH264;
  46 typedef struct _GstVaapiSliceH264Class          GstVaapiSliceH264Class;
  47
  48 /* ------------------------------------------------------------------------- */
  49 /* --- H.264 Pictures                                                    --- */
  50 /* ------------------------------------------------------------------------- */
  51
  52 #define GST_VAAPI_TYPE_PICTURE_H264 \
  53     (gst_vaapi_picture_h264_get_type())
  54
  55 #define GST_VAAPI_PICTURE_H264_CAST(obj) \
  56     ((GstVaapiPictureH264 *)(obj))
  57
  58 #define GST_VAAPI_PICTURE_H264(obj)                             \
  59     (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
  60                                 GST_VAAPI_TYPE_PICTURE_H264,    \
  61                                 GstVaapiPictureH264))
  62
  63 #define GST_VAAPI_PICTURE_H264_CLASS(klass)                     \
  64     (G_TYPE_CHECK_CLASS_CAST((klass),                           \
  65                              GST_VAAPI_TYPE_PICTURE_H264,       \
  66                              GstVaapiPictureH264Class))
  67
  68 #define GST_VAAPI_IS_PICTURE_H264(obj) \
  69     (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_VAAPI_TYPE_PICTURE_H264))
  70
  71 #define GST_VAAPI_IS_PICTURE_H264_CLASS(klass) \
  72     (G_TYPE_CHECK_CLASS_TYPE((klass), GST_VAAPI_TYPE_PICTURE_H264))
  73
  74 #define GST_VAAPI_PICTURE_H264_GET_CLASS(obj)                   \
  75     (G_TYPE_INSTANCE_GET_CLASS((obj),                           \
  76                                GST_VAAPI_TYPE_PICTURE_H264,     \
  77                                GstVaapiPictureH264Class))
  78
  79 struct _GstVaapiPictureH264 {
  80     GstVaapiPicture             base;
  81     VAPictureH264               info;
  82     gint32                      poc;
  83     gint32                      frame_num;              // Original frame_num from slice_header()
  84     gint32                      frame_num_wrap;         // Temporary for ref pic marking: FrameNumWrap
  85     gint32                      pic_num;                // Temporary for ref pic marking: PicNum
  86     gint32                      long_term_pic_num;      // Temporary for ref pic marking: LongTermPicNum
  87     guint                       is_idr                  : 1;
  88     guint                       is_long_term            : 1;
  89     guint                       field_pic_flag          : 1;
  90     guint                       bottom_field_flag       : 1;
  91     guint                       output_flag             : 1;
  92     guint                       output_needed           : 1;
  93 };
  94
  95 struct _GstVaapiPictureH264Class {
  96     /*< private >*/
  97     GstVaapiPictureClass        parent_class;
  98 };
  99
 100 GST_VAAPI_CODEC_DEFINE_TYPE(GstVaapiPictureH264,
 101                             gst_vaapi_picture_h264,
 102                             GST_VAAPI_TYPE_PICTURE)
 103
 104 static void
 105 gst_vaapi_picture_h264_destroy(GstVaapiPictureH264 *decoder)
 106 {
 107 }
 108
 109 static gboolean
 110 gst_vaapi_picture_h264_create(
 111     GstVaapiPictureH264                      *picture,
 112     const GstVaapiCodecObjectConstructorArgs *args
 113 )
 114 {
 115     return TRUE;
 116 }
 117
 118 static void
 119 gst_vaapi_picture_h264_init(GstVaapiPictureH264 *picture)
 120 {
 121     VAPictureH264 *va_pic;
 122
 123     va_pic                      = &picture->info;
 124     va_pic->flags               = 0;
 125     va_pic->TopFieldOrderCnt    = 0;
 126     va_pic->BottomFieldOrderCnt = 0;
 127
 128     picture->poc                = 0;
 129     picture->is_long_term       = FALSE;
 130     picture->is_idr             = FALSE;
 131     picture->output_needed      = FALSE;
 132 }
 133
 134 static inline GstVaapiPictureH264 *
 135 gst_vaapi_picture_h264_new(GstVaapiDecoderH264 *decoder)
 136 {
 137     GstVaapiCodecObject *object;
 138
 139     g_return_val_if_fail(GST_VAAPI_IS_DECODER(decoder), NULL);
 140
 141     object = gst_vaapi_codec_object_new(
 142         GST_VAAPI_TYPE_PICTURE_H264,
 143         GST_VAAPI_CODEC_BASE(decoder),
 144         NULL, sizeof(VAPictureParameterBufferH264),
 145         NULL, 0
 146     );
 147     if (!object)
 148         return NULL;
 149     return GST_VAAPI_PICTURE_H264_CAST(object);
 150 }
 151
 152 /* ------------------------------------------------------------------------- */
 153 /* --- Slices                                                            --- */
 154 /* ------------------------------------------------------------------------- */
 155
 156 #define GST_VAAPI_TYPE_SLICE_H264 \
 157     (gst_vaapi_slice_h264_get_type())
 158
 159 #define GST_VAAPI_SLICE_H264_CAST(obj) \
 160     ((GstVaapiSliceH264 *)(obj))
 161
 162 #define GST_VAAPI_SLICE_H264(obj)                               \
 163     (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
 164                                 GST_VAAPI_TYPE_SLICE_H264,      \
 165                                 GstVaapiSliceH264))
 166
 167 #define GST_VAAPI_SLICE_H264_CLASS(klass)                       \
 168     (G_TYPE_CHECK_CLASS_CAST((klass),                           \
 169                              GST_VAAPI_TYPE_SLICE_H264,         \
 170                              GstVaapiSliceH264Class))
 171
 172 #define GST_VAAPI_IS_SLICE_H264(obj) \
 173     (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_VAAPI_TYPE_SLICE_H264))
 174
 175 #define GST_VAAPI_IS_SLICE_H264_CLASS(klass) \
 176     (G_TYPE_CHECK_CLASS_TYPE((klass), GST_VAAPI_TYPE_SLICE_H264))
 177
 178 #define GST_VAAPI_SLICE_H264_GET_CLASS(obj)                     \
 179     (G_TYPE_INSTANCE_GET_CLASS((obj),                           \
 180                                GST_VAAPI_TYPE_SLICE_H264,       \
 181                                GstVaapiSliceH264Class))
 182
 183 struct _GstVaapiSliceH264 {
 184     GstVaapiSlice               base;
 185     GstH264SliceHdr             slice_hdr;              // parsed slice_header()
 186 };
 187
 188 struct _GstVaapiSliceH264Class {
 189     /*< private >*/
 190     GstVaapiSliceClass          parent_class;
 191 };
 192
 193 GST_VAAPI_CODEC_DEFINE_TYPE(GstVaapiSliceH264,
 194                             gst_vaapi_slice_h264,
 195                             GST_VAAPI_TYPE_SLICE)
 196
 197 static void
 198 gst_vaapi_slice_h264_destroy(GstVaapiSliceH264 *slice)
 199 {
 200 }
 201
 202 static gboolean
 203 gst_vaapi_slice_h264_create(
 204     GstVaapiSliceH264                        *slice,
 205     const GstVaapiCodecObjectConstructorArgs *args
 206 )
 207 {
 208     return TRUE;
 209 }
 210
 211 static void
 212 gst_vaapi_slice_h264_init(GstVaapiSliceH264 *slice)
 213 {
 214 }
 215
 216 static inline GstVaapiSliceH264 *
 217 gst_vaapi_slice_h264_new(
 218     GstVaapiDecoderH264 *decoder,
 219     const guint8        *data,
 220     guint                data_size
 221 )
 222 {
 223     GstVaapiCodecObject *object;
 224
 225     g_return_val_if_fail(GST_VAAPI_IS_DECODER(decoder), NULL);
 226
 227     object = gst_vaapi_codec_object_new(
 228         GST_VAAPI_TYPE_SLICE_H264,
 229         GST_VAAPI_CODEC_BASE(decoder),
 230         NULL, sizeof(VASliceParameterBufferH264),
 231         data, data_size
 232     );
 233     if (!object)
 234         return NULL;
 235     return GST_VAAPI_SLICE_H264_CAST(object);
 236 }
 237
 238 /* ------------------------------------------------------------------------- */
 239 /* --- H.264 Decoder                                                     --- */
 240 /* ------------------------------------------------------------------------- */
 241
 242 G_DEFINE_TYPE(GstVaapiDecoderH264,
 243               gst_vaapi_decoder_h264,
 244               GST_VAAPI_TYPE_DECODER)
 245
 246 #define GST_VAAPI_DECODER_H264_GET_PRIVATE(obj)                 \
 247     (G_TYPE_INSTANCE_GET_PRIVATE((obj),                         \
 248                                  GST_VAAPI_TYPE_DECODER_H264,   \
 249                                  GstVaapiDecoderH264Private))
 250
 251 // Used for field_poc[]
 252 #define TOP_FIELD       0
 253 #define BOTTOM_FIELD    1
 254
 255 struct _GstVaapiDecoderH264Private {
 256     GstAdapter                 *adapter;
 257     GstH264NalParser           *parser;
 258     GstH264SPS                 *sps;
 259     GstH264SPS                  last_sps;
 260     GstH264PPS                 *pps;
 261     GstH264PPS                  last_pps;
 262     GstVaapiPictureH264        *current_picture;
 263     GstVaapiPictureH264        *dpb[16];
 264     guint                       dpb_count;
 265     guint                       dpb_size;
 266     GstVaapiProfile             profile;
 267     GstVaapiPictureH264        *short_ref[32];
 268     guint                       short_ref_count;
 269     GstVaapiPictureH264        *long_ref[32];
 270     guint                       long_ref_count;
 271     GstVaapiPictureH264        *RefPicList0[32];
 272     guint                       RefPicList0_count;
 273     GstVaapiPictureH264        *RefPicList1[32];
 274     guint                       RefPicList1_count;
 275     guint                       nal_length_size;
 276     guint                       width;
 277     guint                       height;
 278     guint                       mb_x;
 279     guint                       mb_y;
 280     guint                       mb_width;
 281     guint                       mb_height;
 282     gint32                      field_poc[2];           // 0:TopFieldOrderCnt / 1:BottomFieldOrderCnt
 283     gint32                      poc_msb;                // PicOrderCntMsb
 284     gint32                      poc_lsb;                // pic_order_cnt_lsb (from slice_header())
 285     gint32                      prev_poc_msb;           // prevPicOrderCntMsb
 286     gint32                      prev_poc_lsb;           // prevPicOrderCntLsb
 287     gint32                      frame_num_offset;       // FrameNumOffset
 288     gint32                      frame_num;              // frame_num (from slice_header())
 289     gint32                      prev_frame_num;         // prevFrameNum
 290     gboolean                    prev_pic_has_mmco5;     // prevMmco5Pic
 291     gboolean                    prev_pic_bottom_field;  // Flag: previous picture is a bottom field
 292     guint                       is_constructed          : 1;
 293     guint                       is_opened               : 1;
 294     guint                       is_avc                  : 1;
 295     guint                       has_context             : 1;
 296 };
 297
 298 static gboolean
 299 decode_picture_end(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture);
 300
 301 static void
 302 clear_references(
 303     GstVaapiDecoderH264  *decoder,
 304     GstVaapiPictureH264 **pictures,
 305     guint                *picture_count
 306 );
 307
 308 /* Get number of reference frames to use */
 309 static guint
 310 get_max_dec_frame_buffering(GstH264SPS *sps)
 311 {
 312     guint max_dec_frame_buffering, MaxDpbMbs, PicSizeMbs;
 313
 314     /* Table A-1 - Level limits */
 315     switch (sps->level_idc) {
 316     case 10: MaxDpbMbs = 396;    break;
 317     case 11: MaxDpbMbs = 900;    break;
 318     case 12: MaxDpbMbs = 2376;   break;
 319     case 13: MaxDpbMbs = 2376;   break;
 320     case 20: MaxDpbMbs = 2376;   break;
 321     case 21: MaxDpbMbs = 4752;   break;
 322     case 22: MaxDpbMbs = 8100;   break;
 323     case 30: MaxDpbMbs = 8100;   break;
 324     case 31: MaxDpbMbs = 18000;  break;
 325     case 32: MaxDpbMbs = 20480;  break;
 326     case 40: MaxDpbMbs = 32768;  break;
 327     case 41: MaxDpbMbs = 32768;  break;
 328     case 42: MaxDpbMbs = 34816;  break;
 329     case 50: MaxDpbMbs = 110400; break;
 330     case 51: MaxDpbMbs = 184320; break;
 331     default:
 332         g_assert(0 && "unhandled level");
 333         break;
 334     }
 335
 336     PicSizeMbs = ((sps->pic_width_in_mbs_minus1 + 1) *
 337                   (sps->pic_height_in_map_units_minus1 + 1) *
 338                   (sps->frame_mbs_only_flag ? 1 : 2));
 339     max_dec_frame_buffering = MaxDpbMbs / PicSizeMbs;
 340
 341     /* VUI parameters */
 342     if (sps->vui_parameters_present_flag) {
 343         GstH264VUIParams * const vui_params = &sps->vui_parameters;
 344         if (vui_params->bitstream_restriction_flag)
 345             max_dec_frame_buffering = vui_params->max_dec_frame_buffering;
 346         else {
 347             switch (sps->profile_idc) {
 348             case 44:  // CAVLC 4:4:4 Intra profile
 349             case 86:  // Scalable High profile
 350             case 100: // High profile
 351             case 110: // High 10 profile
 352             case 122: // High 4:2:2 profile
 353             case 244: // High 4:4:4 Predictive profile
 354                 if (sps->constraint_set3_flag)
 355                     max_dec_frame_buffering = 0;
 356                 break;
 357             }
 358         }
 359     }
 360
 361     if (max_dec_frame_buffering > 16)
 362         max_dec_frame_buffering = 16;
 363     else if (max_dec_frame_buffering < sps->num_ref_frames)
 364         max_dec_frame_buffering = sps->num_ref_frames;
 365     return MAX(1, max_dec_frame_buffering);
 366 }
 367
 368 static void
 369 dpb_remove_index(GstVaapiDecoderH264 *decoder, guint index)
 370 {
 371     GstVaapiDecoderH264Private * const priv = decoder->priv;
 372     guint i, num_pictures = --priv->dpb_count;
 373
 374     if (USE_STRICT_DPB_ORDERING) {
 375         for (i = index; i < num_pictures; i++)
 376             gst_vaapi_picture_replace(&priv->dpb[i], priv->dpb[i + 1]);
 377     }
 378     else if (index != num_pictures)
 379         gst_vaapi_picture_replace(&priv->dpb[index], priv->dpb[num_pictures]);
 380     gst_vaapi_picture_replace(&priv->dpb[num_pictures], NULL);
 381 }
 382
 383 static inline gboolean
 384 dpb_output(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
 385 {
 386     /* XXX: update cropping rectangle */
 387     picture->output_needed = FALSE;
 388     return gst_vaapi_picture_output(GST_VAAPI_PICTURE_CAST(picture));
 389 }
 390
 391 static gboolean
 392 dpb_bump(GstVaapiDecoderH264 *decoder)
 393 {
 394     GstVaapiDecoderH264Private * const priv = decoder->priv;
 395     guint i, lowest_poc_index;
 396     gboolean success;
 397
 398     for (i = 0; i < priv->dpb_count; i++) {
 399         if (priv->dpb[i]->output_needed)
 400             break;
 401     }
 402     if (i == priv->dpb_count)
 403         return FALSE;
 404
 405     lowest_poc_index = i++;
 406     for (; i < priv->dpb_count; i++) {
 407         GstVaapiPictureH264 * const picture = priv->dpb[i];
 408         if (picture->output_needed && picture->poc < priv->dpb[lowest_poc_index]->poc)
 409             lowest_poc_index = i;
 410     }
 411
 412     success = dpb_output(decoder, priv->dpb[lowest_poc_index]);
 413     if (!GST_VAAPI_PICTURE_IS_REFERENCE(priv->dpb[lowest_poc_index]))
 414         dpb_remove_index(decoder, lowest_poc_index);
 415     return success;
 416 }
 417
 418 static void
 419 dpb_flush(GstVaapiDecoderH264 *decoder)
 420 {
 421     GstVaapiDecoderH264Private * const priv = decoder->priv;
 422
 423     while (dpb_bump(decoder))
 424         ;
 425     clear_references(decoder, priv->dpb, &priv->dpb_count);
 426 }
 427
 428 static gboolean
 429 dpb_add(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
 430 {
 431     GstVaapiDecoderH264Private * const priv = decoder->priv;
 432     guint i;
 433
 434     // Remove all unused pictures
 435     if (picture->is_idr)
 436         dpb_flush(decoder);
 437     else {
 438         i = 0;
 439         while (i < priv->dpb_count) {
 440             GstVaapiPictureH264 * const picture = priv->dpb[i];
 441             if (!picture->output_needed &&
 442                 !GST_VAAPI_PICTURE_IS_REFERENCE(picture))
 443                 dpb_remove_index(decoder, i);
 444             else
 445                 i++;
 446         }
 447     }
 448
 449     // C.4.5.1 - Storage and marking of a reference decoded picture into the DPB
 450     if (GST_VAAPI_PICTURE_IS_REFERENCE(picture)) {
 451         while (priv->dpb_count == priv->dpb_size) {
 452             if (!dpb_bump(decoder))
 453                 return FALSE;
 454         }
 455         gst_vaapi_picture_replace(&priv->dpb[priv->dpb_count++], picture);
 456         if (picture->output_flag)
 457             picture->output_needed = TRUE;
 458     }
 459
 460     // C.4.5.2 - Storage and marking of a non-reference decoded picture into the DPB
 461     else {
 462         if (!picture->output_flag)
 463             return TRUE;
 464         while (priv->dpb_count == priv->dpb_size) {
 465             for (i = 0; i < priv->dpb_count; i++) {
 466                 if (priv->dpb[i]->output_needed &&
 467                     priv->dpb[i]->poc < picture->poc)
 468                     break;
 469             }
 470             if (i == priv->dpb_count)
 471                 return dpb_output(decoder, picture);
 472             if (!dpb_bump(decoder))
 473                 return FALSE;
 474         }
 475         gst_vaapi_picture_replace(&priv->dpb[priv->dpb_count++], picture);
 476         picture->output_needed = TRUE;
 477     }
 478     return TRUE;
 479 }
 480
 481 static inline void
 482 dpb_reset(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 483 {
 484     GstVaapiDecoderH264Private * const priv = decoder->priv;
 485
 486     priv->dpb_size = get_max_dec_frame_buffering(sps);
 487     GST_DEBUG("DPB size %u", priv->dpb_size);
 488 }
 489
 490 static GstVaapiDecoderStatus
 491 get_status(GstH264ParserResult result)
 492 {
 493     GstVaapiDecoderStatus status;
 494
 495     switch (result) {
 496     case GST_H264_PARSER_OK:
 497         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
 498         break;
 499     case GST_H264_PARSER_NO_NAL_END:
 500         status = GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
 501         break;
 502     case GST_H264_PARSER_ERROR:
 503         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
 504         break;
 505     default:
 506         status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 507         break;
 508     }
 509     return status;
 510 }
 511
 512 static inline GstH264DecRefPicMarking *
 513 get_dec_ref_pic_marking(GstVaapiPictureH264 *picture_h264)
 514 {
 515     GstVaapiPicture * const picture = GST_VAAPI_PICTURE_CAST(picture_h264);
 516     GstVaapiSliceH264 *slice;
 517
 518     slice = g_ptr_array_index(picture->slices, picture->slices->len - 1);
 519     return &slice->slice_hdr.dec_ref_pic_marking;
 520 }
 521
 522 static void
 523 gst_vaapi_decoder_h264_close(GstVaapiDecoderH264 *decoder)
 524 {
 525     GstVaapiDecoderH264Private * const priv = decoder->priv;
 526
 527     gst_vaapi_picture_replace(&priv->current_picture, NULL);
 528     clear_references(decoder, priv->short_ref, &priv->short_ref_count);
 529     clear_references(decoder, priv->long_ref,  &priv->long_ref_count );
 530     clear_references(decoder, priv->dpb,       &priv->dpb_count      );
 531
 532     if (priv->parser) {
 533         gst_h264_nal_parser_free(priv->parser);
 534         priv->parser = NULL;
 535     }
 536
 537     if (priv->adapter) {
 538         gst_adapter_clear(priv->adapter);
 539         g_object_unref(priv->adapter);
 540         priv->adapter = NULL;
 541     }
 542 }
 543
 544 static gboolean
 545 gst_vaapi_decoder_h264_open(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
 546 {
 547     GstVaapiDecoderH264Private * const priv = decoder->priv;
 548
 549     gst_vaapi_decoder_h264_close(decoder);
 550
 551     priv->adapter = gst_adapter_new();
 552     if (!priv->adapter)
 553         return FALSE;
 554
 555     priv->parser = gst_h264_nal_parser_new();
 556     if (!priv->parser)
 557         return FALSE;
 558     return TRUE;
 559 }
 560
 561 static void
 562 gst_vaapi_decoder_h264_destroy(GstVaapiDecoderH264 *decoder)
 563 {
 564     gst_vaapi_decoder_h264_close(decoder);
 565 }
 566
 567 static gboolean
 568 gst_vaapi_decoder_h264_create(GstVaapiDecoderH264 *decoder)
 569 {
 570     if (!GST_VAAPI_DECODER_CODEC(decoder))
 571         return FALSE;
 572     return TRUE;
 573 }
 574
 575 static GstVaapiDecoderStatus
 576 ensure_context(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 577 {
 578     GstVaapiDecoderH264Private * const priv = decoder->priv;
 579     GstVaapiProfile profiles[2];
 580     GstVaapiEntrypoint entrypoint = GST_VAAPI_ENTRYPOINT_VLD;
 581     guint i, n_profiles = 0;
 582     gboolean success, reset_context = FALSE;
 583
 584     if (!priv->has_context || priv->sps->profile_idc != sps->profile_idc) {
 585         GST_DEBUG("profile changed");
 586         reset_context = TRUE;
 587
 588         switch (sps->profile_idc) {
 589         case 66:
 590             profiles[n_profiles++] = GST_VAAPI_PROFILE_H264_BASELINE;
 591             break;
 592         case 77:
 593             profiles[n_profiles++] = GST_VAAPI_PROFILE_H264_MAIN;
 594             // fall-through
 595         case 100:
 596             profiles[n_profiles++] = GST_VAAPI_PROFILE_H264_HIGH;
 597             break;
 598         default:
 599             GST_DEBUG("unsupported profile %d", sps->profile_idc);
 600             return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_PROFILE;
 601         }
 602
 603         for (i = 0; i < n_profiles; i++) {
 604             success = gst_vaapi_display_has_decoder(
 605                 GST_VAAPI_DECODER_DISPLAY(decoder),
 606                 profiles[i],
 607                 entrypoint
 608             );
 609             if (success)
 610                 break;
 611         }
 612         if (i == n_profiles)
 613             return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_PROFILE;
 614         priv->profile = profiles[i];
 615     }
 616
 617     if (!priv->has_context ||
 618         priv->sps->chroma_format_idc != sps->chroma_format_idc) {
 619         GST_DEBUG("chroma format changed");
 620         reset_context = TRUE;
 621
 622         /* XXX: theoritically, we could handle 4:2:2 format */
 623         if (sps->chroma_format_idc != 1)
 624             return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;
 625     }
 626
 627     if (!priv->has_context              ||
 628         priv->sps->width  != sps->width ||
 629         priv->sps->height != sps->height) {
 630         GST_DEBUG("size changed");
 631         reset_context      = TRUE;
 632
 633         priv->width      = sps->width;
 634         priv->height     = sps->height;
 635         priv->mb_width   = sps->pic_width_in_mbs_minus1 + 1;
 636         priv->mb_height  = sps->pic_height_in_map_units_minus1 + 1;
 637         priv->mb_height *= 2 - sps->frame_mbs_only_flag;
 638     }
 639
 640     if (reset_context) {
 641         GstVaapiContextInfo info;
 642
 643         info.profile    = priv->profile;
 644         info.entrypoint = entrypoint;
 645         info.width      = priv->width;
 646         info.height     = priv->height;
 647         info.ref_frames = get_max_dec_frame_buffering(sps);
 648
 649         if (!gst_vaapi_decoder_ensure_context(GST_VAAPI_DECODER(decoder), &info))
 650             return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 651         priv->has_context = TRUE;
 652
 653         /* Reset DPB */
 654         dpb_reset(decoder, sps);
 655     }
 656     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 657 }
 658
 659 static GstVaapiDecoderStatus
 660 ensure_quant_matrix(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
 661 {
 662     GstVaapiDecoderH264Private * const priv = decoder->priv;
 663     GstH264SPS * const sps = priv->sps;
 664     GstH264PPS * const pps = priv->pps;
 665     GstVaapiPicture * const base_picture = &picture->base;
 666     VAIQMatrixBufferH264 *iq_matrix;
 667     guint i, j, n;
 668
 669     base_picture->iq_matrix = GST_VAAPI_IQ_MATRIX_NEW(H264, decoder);
 670     if (!base_picture->iq_matrix) {
 671         GST_ERROR("failed to allocate IQ matrix");
 672         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
 673     }
 674     iq_matrix = base_picture->iq_matrix->param;
 675
 676     /* XXX: we can only support 4:2:0 or 4:2:2 since ScalingLists8x8[]
 677        is not large enough to hold lists for 4:4:4 */
 678     if (sps->chroma_format_idc == 3)
 679         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;
 680
 681     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList4x4[0]) == 16);
 682     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList8x8[0]) == 64);
 683
 684     if (sizeof(iq_matrix->ScalingList4x4) == sizeof(pps->scaling_lists_4x4))
 685         memcpy(iq_matrix->ScalingList4x4, pps->scaling_lists_4x4,
 686                sizeof(iq_matrix->ScalingList4x4));
 687     else {
 688         n = MIN(G_N_ELEMENTS(iq_matrix->ScalingList4x4),
 689                 G_N_ELEMENTS(pps->scaling_lists_4x4));
 690         for (i = 0; i < n; i++) {
 691             for (j = 0; j < 16; j++)
 692                 iq_matrix->ScalingList4x4[i][j] = pps->scaling_lists_4x4[i][j];
 693         }
 694     }
 695
 696     if (sizeof(iq_matrix->ScalingList8x8) == sizeof(pps->scaling_lists_8x8))
 697         memcpy(iq_matrix->ScalingList8x8, pps->scaling_lists_8x8,
 698                sizeof(iq_matrix->ScalingList8x8));
 699     else {
 700         n = MIN(G_N_ELEMENTS(iq_matrix->ScalingList8x8),
 701                 G_N_ELEMENTS(pps->scaling_lists_8x8));
 702         for (i = 0; i < n; i++) {
 703             for (j = 0; j < 16; j++)
 704                 iq_matrix->ScalingList8x8[i][j] = pps->scaling_lists_8x8[i][j];
 705         }
 706     }
 707     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 708 }
 709
 710 static gboolean
 711 decode_current_picture(GstVaapiDecoderH264 *decoder)
 712 {
 713     GstVaapiDecoderH264Private * const priv = decoder->priv;
 714     GstVaapiPictureH264 * const picture = priv->current_picture;
 715     gboolean success = FALSE;
 716
 717     if (!picture)
 718         return TRUE;
 719
 720     if (!decode_picture_end(decoder, picture))
 721         goto end;
 722     if (!gst_vaapi_picture_decode(GST_VAAPI_PICTURE_CAST(picture)))
 723         goto end;
 724     success = TRUE;
 725 end:
 726     gst_vaapi_picture_replace(&priv->current_picture, NULL);
 727     return success;
 728 }
 729
 730 static GstVaapiDecoderStatus
 731 decode_sps(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
 732 {
 733     GstVaapiDecoder * const base_decoder = GST_VAAPI_DECODER(decoder);
 734     GstVaapiDecoderH264Private * const priv = decoder->priv;
 735     GstH264SPS * const sps = &priv->last_sps;
 736     GstH264ParserResult result;
 737
 738     GST_DEBUG("decode SPS");
 739
 740     if (priv->current_picture && !decode_current_picture(decoder))
 741         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 742
 743     memset(sps, 0, sizeof(*sps));
 744     result = gst_h264_parser_parse_sps(priv->parser, nalu, sps, TRUE);
 745     if (result != GST_H264_PARSER_OK)
 746         return get_status(result);
 747
 748     gst_vaapi_decoder_set_pixel_aspect_ratio(
 749         base_decoder,
 750         sps->vui_parameters.par_n,
 751         sps->vui_parameters.par_d
 752     );
 753     return ensure_context(decoder, sps);
 754 }
 755
 756 static GstVaapiDecoderStatus
 757 decode_pps(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
 758 {
 759     GstVaapiDecoderH264Private * const priv = decoder->priv;
 760     GstH264PPS * const pps = &priv->last_pps;
 761     GstH264ParserResult result;
 762
 763     GST_DEBUG("decode PPS");
 764
 765     if (priv->current_picture && !decode_current_picture(decoder))
 766         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 767
 768     memset(pps, 0, sizeof(*pps));
 769     result = gst_h264_parser_parse_pps(priv->parser, nalu, pps);
 770     if (result != GST_H264_PARSER_OK)
 771         return get_status(result);
 772
 773     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 774 }
 775
 776 static GstVaapiDecoderStatus
 777 decode_sei(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
 778 {
 779     GstVaapiDecoderH264Private * const priv = decoder->priv;
 780     GstH264SEIMessage sei;
 781     GstH264ParserResult result;
 782
 783     GST_DEBUG("decode SEI");
 784
 785     memset(&sei, 0, sizeof(sei));
 786     result = gst_h264_parser_parse_sei(priv->parser, nalu, &sei);
 787     if (result != GST_H264_PARSER_OK) {
 788         GST_WARNING("failed to decode SEI, payload type:%d", sei.payloadType);
 789         return get_status(result);
 790     }
 791
 792     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 793 }
 794
 795 static GstVaapiDecoderStatus
 796 decode_sequence_end(GstVaapiDecoderH264 *decoder)
 797 {
 798     GstVaapiDecoderH264Private * const priv = decoder->priv;
 799
 800     GST_DEBUG("decode sequence-end");
 801
 802     if (priv->current_picture && !decode_current_picture(decoder))
 803         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 804     dpb_flush(decoder);
 805     return GST_VAAPI_DECODER_STATUS_END_OF_STREAM;
 806 }
 807
 808 /* 8.2.1.1 - Decoding process for picture order count type 0 */
 809 static void
 810 init_picture_poc_0(
 811     GstVaapiDecoderH264 *decoder,
 812     GstVaapiPictureH264 *picture,
 813     GstH264SliceHdr     *slice_hdr
 814 )
 815 {
 816     GstVaapiDecoderH264Private * const priv = decoder->priv;
 817     GstH264PPS * const pps = slice_hdr->pps;
 818     GstH264SPS * const sps = pps->sequence;
 819     const gint32 MaxPicOrderCntLsb = 1 << (sps->log2_max_pic_order_cnt_lsb_minus4 + 4);
 820
 821     GST_DEBUG("decode picture order count type 0");
 822
 823     if (picture->is_idr) {
 824         priv->prev_poc_msb = 0;
 825         priv->prev_poc_lsb = 0;
 826     }
 827     else if (priv->prev_pic_has_mmco5) {
 828         priv->prev_poc_msb = 0;
 829         priv->prev_poc_lsb = priv->prev_pic_bottom_field ? 0 :
 830             priv->field_poc[TOP_FIELD];
 831     }
 832     else {
 833         priv->prev_poc_msb = priv->poc_msb;
 834         priv->prev_poc_lsb = priv->poc_lsb;
 835     }
 836
 837     // (8-3)
 838     priv->poc_lsb = slice_hdr->pic_order_cnt_lsb;
 839     if (priv->poc_lsb < priv->prev_poc_lsb &&
 840         (priv->prev_poc_lsb - priv->poc_lsb) >= (MaxPicOrderCntLsb / 2))
 841         priv->poc_msb = priv->prev_poc_msb + MaxPicOrderCntLsb;
 842     else if (priv->poc_lsb > priv->prev_poc_lsb &&
 843              (priv->poc_lsb - priv->prev_poc_lsb) > (MaxPicOrderCntLsb / 2))
 844         priv->poc_msb = priv->prev_poc_msb - MaxPicOrderCntLsb;
 845     else
 846         priv->poc_msb = priv->prev_poc_msb;
 847
 848     // (8-4)
 849     if (!slice_hdr->field_pic_flag || !slice_hdr->bottom_field_flag)
 850         priv->field_poc[TOP_FIELD] = priv->poc_msb + priv->poc_lsb;
 851
 852     // (8-5)
 853     if (!slice_hdr->field_pic_flag)
 854         priv->field_poc[BOTTOM_FIELD] = priv->field_poc[TOP_FIELD] +
 855             slice_hdr->delta_pic_order_cnt_bottom;
 856     else if (slice_hdr->bottom_field_flag)
 857         priv->field_poc[BOTTOM_FIELD] = priv->poc_msb + priv->poc_lsb;
 858 }
 859
 860 /* 8.2.1.2 - Decoding process for picture order count type 1 */
 861 static void
 862 init_picture_poc_1(
 863     GstVaapiDecoderH264 *decoder,
 864     GstVaapiPictureH264 *picture,
 865     GstH264SliceHdr     *slice_hdr
 866 )
 867 {
 868     GstVaapiDecoderH264Private * const priv = decoder->priv;
 869     GstH264PPS * const pps = slice_hdr->pps;
 870     GstH264SPS * const sps = pps->sequence;
 871     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
 872     gint32 prev_frame_num_offset, abs_frame_num, expected_poc;
 873     guint i;
 874
 875     GST_DEBUG("decode picture order count type 1");
 876
 877     if (priv->prev_pic_has_mmco5)
 878         prev_frame_num_offset = 0;
 879     else
 880         prev_frame_num_offset = priv->frame_num_offset;
 881
 882     // (8-6)
 883     if (picture->is_idr)
 884         priv->frame_num_offset = 0;
 885     else if (priv->prev_frame_num > priv->frame_num)
 886         priv->frame_num_offset = prev_frame_num_offset + MaxFrameNum;
 887     else
 888         priv->frame_num_offset = prev_frame_num_offset;
 889
 890     // (8-7)
 891     if (sps->num_ref_frames_in_pic_order_cnt_cycle != 0)
 892         abs_frame_num = priv->frame_num_offset + priv->frame_num;
 893     else
 894         abs_frame_num = 0;
 895     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture) && abs_frame_num > 0)
 896         abs_frame_num = abs_frame_num - 1;
 897
 898     if (abs_frame_num > 0) {
 899         gint32 expected_delta_per_poc_cycle;
 900         gint32 poc_cycle_cnt, frame_num_in_poc_cycle;
 901
 902         expected_delta_per_poc_cycle = 0;
 903         for (i = 0; i < sps->num_ref_frames_in_pic_order_cnt_cycle; i++)
 904             expected_delta_per_poc_cycle += sps->offset_for_ref_frame[i];
 905
 906         // (8-8)
 907         poc_cycle_cnt = (abs_frame_num - 1) /
 908             sps->num_ref_frames_in_pic_order_cnt_cycle;
 909         frame_num_in_poc_cycle = (abs_frame_num - 1) %
 910             sps->num_ref_frames_in_pic_order_cnt_cycle;
 911
 912         // (8-9)
 913         expected_poc = poc_cycle_cnt * expected_delta_per_poc_cycle;
 914         for (i = 0; i <= frame_num_in_poc_cycle; i++)
 915             expected_poc += sps->offset_for_ref_frame[i];
 916     }
 917     else
 918         expected_poc = 0;
 919     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
 920         expected_poc += sps->offset_for_non_ref_pic;
 921
 922     // (8-10)
 923     if (!slice_hdr->field_pic_flag) {
 924         priv->field_poc[TOP_FIELD] = expected_poc +
 925             slice_hdr->delta_pic_order_cnt[0];
 926         priv->field_poc[BOTTOM_FIELD] = priv->field_poc[TOP_FIELD] +
 927             sps->offset_for_top_to_bottom_field +
 928             slice_hdr->delta_pic_order_cnt[1];
 929     }
 930     else if (!slice_hdr->bottom_field_flag)
 931         priv->field_poc[TOP_FIELD] = expected_poc +
 932             slice_hdr->delta_pic_order_cnt[0];
 933     else
 934         priv->field_poc[BOTTOM_FIELD] = expected_poc +
 935             sps->offset_for_top_to_bottom_field + slice_hdr->delta_pic_order_cnt[0];
 936 }
 937
 938 /* 8.2.1.3 - Decoding process for picture order count type 2 */
 939 static void
 940 init_picture_poc_2(
 941     GstVaapiDecoderH264 *decoder,
 942     GstVaapiPictureH264 *picture,
 943     GstH264SliceHdr     *slice_hdr
 944 )
 945 {
 946     GstVaapiDecoderH264Private * const priv = decoder->priv;
 947     GstH264PPS * const pps = slice_hdr->pps;
 948     GstH264SPS * const sps = pps->sequence;
 949     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
 950     gint32 prev_frame_num_offset, temp_poc;
 951
 952     GST_DEBUG("decode picture order count type 2");
 953
 954     if (priv->prev_pic_has_mmco5)
 955         prev_frame_num_offset = 0;
 956     else
 957         prev_frame_num_offset = priv->frame_num_offset;
 958
 959     // (8-11)
 960     if (picture->is_idr)
 961         priv->frame_num_offset = 0;
 962     else if (priv->prev_frame_num > priv->frame_num)
 963         priv->frame_num_offset = prev_frame_num_offset + MaxFrameNum;
 964     else
 965         priv->frame_num_offset = prev_frame_num_offset;
 966
 967     // (8-12)
 968     if (picture->is_idr)
 969         temp_poc = 0;
 970     else if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
 971         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num) - 1;
 972     else
 973         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num);
 974
 975     // (8-13)
 976     if (!slice_hdr->field_pic_flag) {
 977         priv->field_poc[TOP_FIELD] = temp_poc;
 978         priv->field_poc[BOTTOM_FIELD] = temp_poc;
 979     }
 980     else if (slice_hdr->bottom_field_flag)
 981         priv->field_poc[BOTTOM_FIELD] = temp_poc;
 982     else
 983         priv->field_poc[TOP_FIELD] = temp_poc;
 984 }
 985
 986 /* 8.2.1 - Decoding process for picture order count */
 987 static void
 988 init_picture_poc(
 989     GstVaapiDecoderH264 *decoder,
 990     GstVaapiPictureH264 *picture,
 991     GstH264SliceHdr     *slice_hdr
 992 )
 993 {
 994     GstVaapiDecoderH264Private * const priv = decoder->priv;
 995     VAPictureH264 * const pic = &picture->info;
 996     GstH264PPS * const pps = slice_hdr->pps;
 997     GstH264SPS * const sps = pps->sequence;
 998
 999     switch (sps->pic_order_cnt_type) {
1000     case 0:
1001         init_picture_poc_0(decoder, picture, slice_hdr);
1002         break;
1003     case 1:
1004         init_picture_poc_1(decoder, picture, slice_hdr);
1005         break;
1006     case 2:
1007         init_picture_poc_2(decoder, picture, slice_hdr);
1008         break;
1009     }
1010
1011     if (!(pic->flags & VA_PICTURE_H264_BOTTOM_FIELD))
1012         pic->TopFieldOrderCnt = priv->field_poc[TOP_FIELD];
1013     if (!(pic->flags & VA_PICTURE_H264_TOP_FIELD))
1014         pic->BottomFieldOrderCnt = priv->field_poc[BOTTOM_FIELD];
1015     picture->poc = MIN(pic->TopFieldOrderCnt, pic->BottomFieldOrderCnt);
1016 }
1017
1018 static int
1019 compare_picture_pic_num_dec(const void *a, const void *b)
1020 {
1021     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1022     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1023
1024     return picB->pic_num - picA->pic_num;
1025 }
1026
1027 static int
1028 compare_picture_long_term_pic_num_inc(const void *a, const void *b)
1029 {
1030     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1031     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1032
1033     return picA->long_term_pic_num - picB->long_term_pic_num;
1034 }
1035
1036 static int
1037 compare_picture_poc_dec(const void *a, const void *b)
1038 {
1039     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1040     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1041
1042     return picB->poc - picA->poc;
1043 }
1044
1045 static int
1046 compare_picture_poc_inc(const void *a, const void *b)
1047 {
1048     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1049     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1050
1051     return picA->poc - picB->poc;
1052 }
1053
1054 static int
1055 compare_picture_frame_num_wrap_dec(const void *a, const void *b)
1056 {
1057     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1058     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1059
1060     return picB->frame_num_wrap - picA->frame_num_wrap;
1061 }
1062
1063 static int
1064 compare_picture_long_term_frame_idx_inc(const void *a, const void *b)
1065 {
1066     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1067     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1068
1069     return picA->info.frame_idx - picB->info.frame_idx;
1070 }
1071
1072 /* 8.2.4.1 - Decoding process for picture numbers */
1073 static void
1074 init_picture_refs_pic_num(
1075     GstVaapiDecoderH264 *decoder,
1076     GstVaapiPictureH264 *picture,
1077     GstH264SliceHdr     *slice_hdr
1078 )
1079 {
1080     GstVaapiDecoderH264Private * const priv = decoder->priv;
1081     GstH264PPS * const pps = slice_hdr->pps;
1082     GstH264SPS * const sps = pps->sequence;
1083     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1084     const guint field_flags = VA_PICTURE_H264_TOP_FIELD | VA_PICTURE_H264_BOTTOM_FIELD;
1085     guint i;
1086
1087     GST_DEBUG("decode picture numbers");
1088
1089     for (i = 0; i < priv->short_ref_count; i++) {
1090         GstVaapiPictureH264 * const pic = priv->short_ref[i];
1091
1092         // (8-27)
1093         if (pic->frame_num > priv->frame_num)
1094             pic->frame_num_wrap = pic->frame_num - MaxFrameNum;
1095         else
1096             pic->frame_num_wrap = pic->frame_num;
1097
1098         // (8-28, 8-30, 8-31)
1099         if (!pic->field_pic_flag)
1100             pic->pic_num = pic->frame_num_wrap;
1101         else {
1102             if (((picture->info.flags ^ pic->info.flags) & field_flags) == 0)
1103                 pic->pic_num = 2 * pic->frame_num_wrap + 1;
1104             else
1105                 pic->pic_num = 2 * pic->frame_num_wrap;
1106         }
1107     }
1108
1109     for (i = 0; i < priv->long_ref_count; i++) {
1110         GstVaapiPictureH264 * const pic = priv->long_ref[i];
1111
1112         // (8-29, 8-32, 8-33)
1113         if (!pic->field_pic_flag)
1114             pic->long_term_pic_num = pic->info.frame_idx;
1115         else {
1116             if (((picture->info.flags ^ pic->info.flags) & field_flags) == 0)
1117                 pic->long_term_pic_num = 2 * pic->info.frame_idx + 1;
1118             else
1119                 pic->long_term_pic_num = 2 * pic->info.frame_idx;
1120         }
1121     }
1122 }
1123
1124 #define SORT_REF_LIST(list, n, compare_func) \
1125     qsort(list, n, sizeof(*(list)), compare_picture_##compare_func)
1126
1127 static void
1128 init_picture_refs_p_slice(
1129     GstVaapiDecoderH264 *decoder,
1130     GstVaapiPictureH264 *picture,
1131     GstH264SliceHdr     *slice_hdr
1132 )
1133 {
1134     GstVaapiDecoderH264Private * const priv = decoder->priv;
1135     GstVaapiPictureH264 **ref_list;
1136     guint i;
1137
1138     GST_DEBUG("decode reference picture list for P and SP slices");
1139
1140     if (!picture->field_pic_flag) {
1141         /* 8.2.4.2.1 - P and SP slices in frames */
1142         if (priv->short_ref_count > 0) {
1143             ref_list = priv->RefPicList0;
1144             for (i = 0; i < priv->short_ref_count; i++)
1145                 ref_list[i] = priv->short_ref[i];
1146             SORT_REF_LIST(ref_list, i, pic_num_dec);
1147             priv->RefPicList0_count += i;
1148         }
1149
1150         if (priv->long_ref_count > 0) {
1151             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1152             for (i = 0; i < priv->long_ref_count; i++)
1153                 ref_list[i] = priv->long_ref[i];
1154             SORT_REF_LIST(ref_list, i, long_term_pic_num_inc);
1155             priv->RefPicList0_count += i;
1156         }
1157     }
1158     else {
1159         /* 8.2.4.2.2 - P and SP slices in fields */
1160         GstVaapiPictureH264 *short_ref[32];
1161         guint short_ref_count = 0;
1162         GstVaapiPictureH264 *long_ref[32];
1163         guint long_ref_count = 0;
1164
1165         // XXX: handle second field if current field is marked as
1166         // "used for short-term reference"
1167         if (priv->short_ref_count > 0) {
1168             for (i = 0; i < priv->short_ref_count; i++)
1169                 short_ref[i] = priv->short_ref[i];
1170             SORT_REF_LIST(short_ref, i, frame_num_wrap_dec);
1171             short_ref_count = i;
1172         }
1173
1174         // XXX: handle second field if current field is marked as
1175         // "used for long-term reference"
1176         if (priv->long_ref_count > 0) {
1177             for (i = 0; i < priv->long_ref_count; i++)
1178                 long_ref[i] = priv->long_ref[i];
1179             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1180             long_ref_count = i;
1181         }
1182
1183         // XXX: handle 8.2.4.2.5
1184     }
1185 }
1186
1187 static void
1188 init_picture_refs_b_slice(
1189     GstVaapiDecoderH264 *decoder,
1190     GstVaapiPictureH264 *picture,
1191     GstH264SliceHdr     *slice_hdr
1192 )
1193 {
1194     GstVaapiDecoderH264Private * const priv = decoder->priv;
1195     GstVaapiPictureH264 **ref_list;
1196     guint i, n;
1197
1198     GST_DEBUG("decode reference picture list for B slices");
1199
1200     if (!picture->field_pic_flag) {
1201         /* 8.2.4.2.3 - B slices in frames */
1202
1203         /* RefPicList0 */
1204         if (priv->short_ref_count > 0) {
1205             // 1. Short-term references
1206             ref_list = priv->RefPicList0;
1207             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1208                 if (priv->short_ref[i]->poc < picture->poc)
1209                     ref_list[n++] = priv->short_ref[i];
1210             }
1211             SORT_REF_LIST(ref_list, n, poc_dec);
1212             priv->RefPicList0_count += n;
1213
1214             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1215             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1216                 if (priv->short_ref[i]->poc >= picture->poc)
1217                     ref_list[n++] = priv->short_ref[i];
1218             }
1219             SORT_REF_LIST(ref_list, n, poc_inc);
1220             priv->RefPicList0_count += n;
1221         }
1222
1223         if (priv->long_ref_count > 0) {
1224             // 2. Long-term references
1225             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1226             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1227                 ref_list[n++] = priv->long_ref[i];
1228             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1229             priv->RefPicList0_count += n;
1230         }
1231
1232         /* RefPicList1 */
1233         if (priv->short_ref_count > 0) {
1234             // 1. Short-term references
1235             ref_list = priv->RefPicList1;
1236             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1237                 if (priv->short_ref[i]->poc > picture->poc)
1238                     ref_list[n++] = priv->short_ref[i];
1239             }
1240             SORT_REF_LIST(ref_list, n, poc_inc);
1241             priv->RefPicList1_count += n;
1242
1243             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1244             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1245                 if (priv->short_ref[i]->poc <= picture->poc)
1246                     ref_list[n++] = priv->short_ref[i];
1247             }
1248             SORT_REF_LIST(ref_list, n, poc_dec);
1249             priv->RefPicList1_count += n;
1250         }
1251
1252         if (priv->long_ref_count > 0) {
1253             // 2. Long-term references
1254             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1255             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1256                 ref_list[n++] = priv->long_ref[i];
1257             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1258             priv->RefPicList1_count += n;
1259         }
1260     }
1261     else {
1262         /* 8.2.4.2.4 - B slices in fields */
1263         GstVaapiPictureH264 *short_ref0[32];
1264         guint short_ref0_count = 0;
1265         GstVaapiPictureH264 *short_ref1[32];
1266         guint short_ref1_count = 0;
1267         GstVaapiPictureH264 *long_ref[32];
1268         guint long_ref_count = 0;
1269
1270         /* refFrameList0ShortTerm */
1271         if (priv->short_ref_count > 0) {
1272             ref_list = short_ref0;
1273             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1274                 if (priv->short_ref[i]->poc <= picture->poc)
1275                     ref_list[n++] = priv->short_ref[i];
1276             }
1277             SORT_REF_LIST(ref_list, n, poc_dec);
1278             short_ref0_count += n;
1279
1280             ref_list = &short_ref0[short_ref0_count];
1281             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1282                 if (priv->short_ref[i]->poc > picture->poc)
1283                     ref_list[n++] = priv->short_ref[i];
1284             }
1285             SORT_REF_LIST(ref_list, n, poc_inc);
1286             short_ref0_count += n;
1287         }
1288
1289         /* refFrameList1ShortTerm */
1290         if (priv->short_ref_count > 0) {
1291             ref_list = short_ref1;
1292             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1293                 if (priv->short_ref[i]->poc > picture->poc)
1294                     ref_list[n++] = priv->short_ref[i];
1295             }
1296             SORT_REF_LIST(ref_list, n, poc_inc);
1297             short_ref1_count += n;
1298
1299             ref_list = &short_ref1[short_ref1_count];
1300             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1301                 if (priv->short_ref[i]->poc <= picture->poc)
1302                     ref_list[n++] = priv->short_ref[i];
1303             }
1304             SORT_REF_LIST(ref_list, n, poc_dec);
1305             short_ref1_count += n;
1306         }
1307
1308         /* refFrameListLongTerm */
1309         if (priv->long_ref_count > 0) {
1310             for (i = 0; i < priv->long_ref_count; i++)
1311                 long_ref[i] = priv->long_ref[i];
1312             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1313             long_ref_count = i;
1314         }
1315
1316         // XXX: handle 8.2.4.2.5
1317     }
1318
1319     /* Check whether RefPicList1 is identical to RefPicList0, then
1320        swap if necessary */
1321     if (priv->RefPicList1_count > 1 &&
1322         priv->RefPicList1_count == priv->RefPicList0_count &&
1323         memcmp(priv->RefPicList0, priv->RefPicList1,
1324                priv->RefPicList0_count * sizeof(priv->RefPicList0[0])) == 0) {
1325         GstVaapiPictureH264 * const tmp = priv->RefPicList1[0];
1326         priv->RefPicList1[0] = priv->RefPicList1[1];
1327         priv->RefPicList1[1] = tmp;
1328     }
1329 }
1330
1331 #undef SORT_REF_LIST
1332
1333 static void
1334 clear_references(
1335     GstVaapiDecoderH264  *decoder,
1336     GstVaapiPictureH264 **pictures,
1337     guint                *picture_count
1338 )
1339 {
1340     const guint num_pictures = *picture_count;
1341     guint i;
1342
1343     for (i = 0; i < num_pictures; i++)
1344         gst_vaapi_picture_replace(&pictures[i], NULL);
1345     *picture_count = 0;
1346 }
1347
1348 static gboolean
1349 remove_reference_at(
1350     GstVaapiDecoderH264  *decoder,
1351     GstVaapiPictureH264 **pictures,
1352     guint                *picture_count,
1353     guint                 index
1354 )
1355 {
1356     guint num_pictures = *picture_count;
1357     GstVaapiPictureH264 *picture;
1358
1359     g_return_val_if_fail(index < num_pictures, FALSE);
1360
1361     picture = pictures[index];
1362     GST_VAAPI_PICTURE_FLAG_UNSET(picture, GST_VAAPI_PICTURE_FLAG_REFERENCE);
1363     picture->is_long_term = FALSE;
1364     picture->info.flags &= ~(VA_PICTURE_H264_SHORT_TERM_REFERENCE |
1365                              VA_PICTURE_H264_LONG_TERM_REFERENCE);
1366
1367     if (index != --num_pictures)
1368         gst_vaapi_picture_replace(&pictures[index], pictures[num_pictures]);
1369     gst_vaapi_picture_replace(&pictures[num_pictures], NULL);
1370     *picture_count = num_pictures;
1371     return TRUE;
1372 }
1373
1374 static gint
1375 find_short_term_reference(GstVaapiDecoderH264 *decoder, gint32 pic_num)
1376 {
1377     GstVaapiDecoderH264Private * const priv = decoder->priv;
1378     guint i;
1379
1380     for (i = 0; i < priv->short_ref_count; i++) {
1381         if (priv->short_ref[i]->pic_num == pic_num)
1382             return i;
1383     }
1384     GST_ERROR("found no short-term reference picture with PicNum = %d",
1385               pic_num);
1386     return -1;
1387 }
1388
1389 static gint
1390 find_long_term_reference(GstVaapiDecoderH264 *decoder, gint32 long_term_pic_num)
1391 {
1392     GstVaapiDecoderH264Private * const priv = decoder->priv;
1393     guint i;
1394
1395     for (i = 0; i < priv->long_ref_count; i++) {
1396         if (priv->long_ref[i]->long_term_pic_num == long_term_pic_num)
1397             return i;
1398     }
1399     GST_ERROR("found no long-term reference picture with LongTermPicNum = %d",
1400               long_term_pic_num);
1401     return -1;
1402 }
1403
1404 static void
1405 exec_picture_refs_modification_1(
1406     GstVaapiDecoderH264           *decoder,
1407     GstVaapiPictureH264           *picture,
1408     GstH264SliceHdr               *slice_hdr,
1409     guint                          list
1410 )
1411 {
1412     GstVaapiDecoderH264Private * const priv = decoder->priv;
1413     GstH264PPS * const pps = slice_hdr->pps;
1414     GstH264SPS * const sps = pps->sequence;
1415     GstH264RefPicListModification *ref_pic_list_modification;
1416     guint num_ref_pic_list_modifications;
1417     GstVaapiPictureH264 **ref_list;
1418     guint *ref_list_count_ptr, ref_list_count, ref_list_idx = 0;
1419     guint i, j, n, num_refs;
1420     gint found_ref_idx;
1421     gint32 MaxPicNum, CurrPicNum, picNumPred;
1422
1423     GST_DEBUG("modification process of reference picture list %u", list);
1424
1425     if (list == 0) {
1426         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l0;
1427         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l0;
1428         ref_list                       = priv->RefPicList0;
1429         ref_list_count_ptr             = &priv->RefPicList0_count;
1430         num_refs                       = slice_hdr->num_ref_idx_l0_active_minus1 + 1;
1431     }
1432     else {
1433         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l1;
1434         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l1;
1435         ref_list                       = priv->RefPicList1;
1436         ref_list_count_ptr             = &priv->RefPicList1_count;
1437         num_refs                       = slice_hdr->num_ref_idx_l1_active_minus1 + 1;
1438     }
1439     ref_list_count = *ref_list_count_ptr;
1440
1441     if (picture->field_pic_flag) {
1442         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 5); // 2 * MaxFrameNum
1443         CurrPicNum = 2 * slice_hdr->frame_num + 1;              // 2 * frame_num + 1
1444     }
1445     else {
1446         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 4); // MaxFrameNum
1447         CurrPicNum = slice_hdr->frame_num;                      // frame_num
1448     }
1449
1450     picNumPred = CurrPicNum;
1451
1452     for (i = 0; i < num_ref_pic_list_modifications; i++) {
1453         GstH264RefPicListModification * const l = &ref_pic_list_modification[i];
1454         if (l->modification_of_pic_nums_idc == 3)
1455             break;
1456
1457         /* 8.2.4.3.1 - Short-term reference pictures */
1458         if (l->modification_of_pic_nums_idc == 0 || l->modification_of_pic_nums_idc == 1) {
1459             gint32 abs_diff_pic_num = l->value.abs_diff_pic_num_minus1 + 1;
1460             gint32 picNum, picNumNoWrap;
1461
1462             // (8-34)
1463             if (l->modification_of_pic_nums_idc == 0) {
1464                 picNumNoWrap = picNumPred - abs_diff_pic_num;
1465                 if (picNumNoWrap < 0)
1466                     picNumNoWrap += MaxPicNum;
1467             }
1468
1469             // (8-35)
1470             else {
1471                 picNumNoWrap = picNumPred + abs_diff_pic_num;
1472                 if (picNumNoWrap >= MaxPicNum)
1473                     picNumNoWrap -= MaxPicNum;
1474             }
1475             picNumPred = picNumNoWrap;
1476
1477             // (8-36)
1478             picNum = picNumNoWrap;
1479             if (picNum > CurrPicNum)
1480                 picNum -= MaxPicNum;
1481
1482             // (8-37)
1483             for (j = num_refs; j > ref_list_idx; j--)
1484                 ref_list[j] = ref_list[j - 1];
1485             found_ref_idx = find_short_term_reference(decoder, picNum);
1486             ref_list[ref_list_idx++] =
1487                 found_ref_idx >= 0 ? priv->short_ref[found_ref_idx] : NULL;
1488             n = ref_list_idx;
1489             for (j = ref_list_idx; j <= num_refs; j++) {
1490                 gint32 PicNumF;
1491                 if (!ref_list[j])
1492                     continue;
1493                 PicNumF = ref_list[j]->is_long_term ?
1494                     MaxPicNum : ref_list[j]->pic_num;
1495                 if (PicNumF != picNum)
1496                     ref_list[n++] = ref_list[j];
1497             }
1498         }
1499
1500         /* 8.2.4.3.2 - Long-term reference pictures */
1501         else {
1502
1503             for (j = num_refs; j > ref_list_idx; j--)
1504                 ref_list[j] = ref_list[j - 1];
1505             found_ref_idx =
1506                 find_long_term_reference(decoder, l->value.long_term_pic_num);
1507             ref_list[ref_list_idx++] =
1508                 found_ref_idx >= 0 ? priv->long_ref[found_ref_idx] : NULL;
1509             n = ref_list_idx;
1510             for (j = ref_list_idx; j <= num_refs; j++) {
1511                 gint32 LongTermPicNumF;
1512                 if (!ref_list[j])
1513                     continue;
1514                 LongTermPicNumF = ref_list[j]->is_long_term ?
1515                     ref_list[j]->long_term_pic_num : INT_MAX;
1516                 if (LongTermPicNumF != l->value.long_term_pic_num)
1517                     ref_list[n++] = ref_list[j];
1518             }
1519         }
1520     }
1521
1522 #if DEBUG
1523     for (i = 0; i < num_refs; i++)
1524         if (!ref_list[i])
1525             GST_ERROR("list %u entry %u is empty", list, i);
1526 #endif
1527     *ref_list_count_ptr = num_refs;
1528 }
1529
1530 /* 8.2.4.3 - Modification process for reference picture lists */
1531 static void
1532 exec_picture_refs_modification(
1533     GstVaapiDecoderH264 *decoder,
1534     GstVaapiPictureH264 *picture,
1535     GstH264SliceHdr     *slice_hdr
1536 )
1537 {
1538     GST_DEBUG("execute ref_pic_list_modification()");
1539
1540     /* RefPicList0 */
1541     if (!GST_H264_IS_I_SLICE(slice_hdr) && !GST_H264_IS_SI_SLICE(slice_hdr) &&
1542         slice_hdr->ref_pic_list_modification_flag_l0)
1543         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 0);
1544
1545     /* RefPicList1 */
1546     if (GST_H264_IS_B_SLICE(slice_hdr) &&
1547         slice_hdr->ref_pic_list_modification_flag_l1)
1548         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 1);
1549 }
1550
1551 static gboolean
1552 init_picture_refs(
1553     GstVaapiDecoderH264 *decoder,
1554     GstVaapiPictureH264 *picture,
1555     GstH264SliceHdr     *slice_hdr
1556 )
1557 {
1558     GstVaapiDecoderH264Private * const priv = decoder->priv;
1559     GstVaapiPicture * const base_picture = &picture->base;
1560     guint i, num_refs;
1561
1562     init_picture_refs_pic_num(decoder, picture, slice_hdr);
1563
1564     priv->RefPicList0_count = 0;
1565     priv->RefPicList1_count = 0;
1566
1567     switch (base_picture->type) {
1568     case GST_VAAPI_PICTURE_TYPE_P:
1569     case GST_VAAPI_PICTURE_TYPE_SP:
1570         init_picture_refs_p_slice(decoder, picture, slice_hdr);
1571         break;
1572     case GST_VAAPI_PICTURE_TYPE_B:
1573         init_picture_refs_b_slice(decoder, picture, slice_hdr);
1574         break;
1575     default:
1576         break;
1577     }
1578
1579     exec_picture_refs_modification(decoder, picture, slice_hdr);
1580
1581     switch (base_picture->type) {
1582     case GST_VAAPI_PICTURE_TYPE_B:
1583         num_refs = 1 + slice_hdr->num_ref_idx_l1_active_minus1;
1584         for (i = priv->RefPicList1_count; i < num_refs; i++)
1585             priv->RefPicList1[i] = NULL;
1586         priv->RefPicList1_count = num_refs;
1587
1588         // fall-through
1589     case GST_VAAPI_PICTURE_TYPE_P:
1590     case GST_VAAPI_PICTURE_TYPE_SP:
1591         num_refs = 1 + slice_hdr->num_ref_idx_l0_active_minus1;
1592         for (i = priv->RefPicList0_count; i < num_refs; i++)
1593             priv->RefPicList0[i] = NULL;
1594         priv->RefPicList0_count = num_refs;
1595         break;
1596     default:
1597         break;
1598     }
1599     return TRUE;
1600 }
1601
1602 static gboolean
1603 init_picture(
1604     GstVaapiDecoderH264 *decoder,
1605     GstVaapiPictureH264 *picture,
1606     GstH264SliceHdr     *slice_hdr,
1607     GstH264NalUnit      *nalu
1608 )
1609 {
1610     GstVaapiDecoderH264Private * const priv = decoder->priv;
1611     GstVaapiPicture * const base_picture = &picture->base;
1612     VAPictureH264 *pic;
1613
1614     priv->prev_frame_num        = priv->frame_num;
1615     priv->frame_num             = slice_hdr->frame_num;
1616     picture->frame_num          = priv->frame_num;
1617     picture->frame_num_wrap     = priv->frame_num;
1618     picture->is_idr             = nalu->type == GST_H264_NAL_SLICE_IDR;
1619     picture->field_pic_flag     = slice_hdr->field_pic_flag;
1620     picture->bottom_field_flag  = slice_hdr->bottom_field_flag;
1621     picture->output_flag        = TRUE; /* XXX: conformant to Annex A only */
1622     base_picture->pts           = gst_adapter_prev_timestamp(priv->adapter, NULL);
1623
1624     /* Reset decoder state for IDR pictures */
1625     if (picture->is_idr) {
1626         GST_DEBUG("<IDR>");
1627         clear_references(decoder, priv->short_ref, &priv->short_ref_count);
1628         clear_references(decoder, priv->long_ref,  &priv->long_ref_count );
1629     }
1630
1631     /* Initialize VA picture info */
1632     pic = &picture->info;
1633     pic->picture_id = picture->base.surface_id;
1634     pic->frame_idx  = priv->frame_num;
1635     if (picture->field_pic_flag) {
1636         if (picture->bottom_field_flag)
1637             pic->flags |= VA_PICTURE_H264_BOTTOM_FIELD;
1638         else
1639             pic->flags |= VA_PICTURE_H264_TOP_FIELD;
1640     }
1641
1642     /* Initialize base picture */
1643     switch (slice_hdr->type % 5) {
1644     case GST_H264_P_SLICE:
1645         base_picture->type = GST_VAAPI_PICTURE_TYPE_P;
1646         break;
1647     case GST_H264_B_SLICE:
1648         base_picture->type = GST_VAAPI_PICTURE_TYPE_B;
1649         break;
1650     case GST_H264_I_SLICE:
1651         base_picture->type = GST_VAAPI_PICTURE_TYPE_I;
1652         break;
1653     case GST_H264_SP_SLICE:
1654         base_picture->type = GST_VAAPI_PICTURE_TYPE_SP;
1655         break;
1656     case GST_H264_SI_SLICE:
1657         base_picture->type = GST_VAAPI_PICTURE_TYPE_SI;
1658         break;
1659     }
1660
1661     if (nalu->ref_idc) {
1662         GstH264DecRefPicMarking * const dec_ref_pic_marking =
1663             &slice_hdr->dec_ref_pic_marking;
1664         GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_REFERENCE);
1665         if (picture->is_idr) {
1666             if (dec_ref_pic_marking->long_term_reference_flag)
1667                 picture->is_long_term = TRUE;
1668         }
1669     }
1670
1671     init_picture_poc(decoder, picture, slice_hdr);
1672     if (!init_picture_refs(decoder, picture, slice_hdr)) {
1673         GST_ERROR("failed to initialize references");
1674         return FALSE;
1675     }
1676     return TRUE;
1677 }
1678
1679 /* 8.2.5.3 - Sliding window decoded reference picture marking process */
1680 static gboolean
1681 exec_ref_pic_marking_sliding_window(GstVaapiDecoderH264 *decoder)
1682 {
1683     GstVaapiDecoderH264Private * const priv = decoder->priv;
1684     GstH264SPS * const sps = priv->sps;
1685     guint i, max_num_ref_frames, lowest_frame_num_index;
1686     gint32 lowest_frame_num;
1687
1688     GST_DEBUG("reference picture marking process (sliding window)");
1689
1690     max_num_ref_frames = sps->num_ref_frames;
1691     if (max_num_ref_frames == 0)
1692         max_num_ref_frames = 1;
1693
1694     if (priv->short_ref_count + priv->long_ref_count < max_num_ref_frames)
1695         return TRUE;
1696     if (priv->short_ref_count < 1)
1697         return FALSE;
1698
1699     lowest_frame_num = priv->short_ref[0]->frame_num_wrap;
1700     lowest_frame_num_index = 0;
1701     for (i = 1; i < priv->short_ref_count; i++) {
1702         if (priv->short_ref[i]->frame_num_wrap < lowest_frame_num) {
1703             lowest_frame_num = priv->short_ref[i]->frame_num_wrap;
1704             lowest_frame_num_index = i;
1705         }
1706     }
1707
1708     remove_reference_at(
1709         decoder,
1710         priv->short_ref, &priv->short_ref_count,
1711         lowest_frame_num_index
1712     );
1713     return TRUE;
1714 }
1715
1716 static inline gint32
1717 get_picNumX(GstVaapiPictureH264 *picture, GstH264RefPicMarking *ref_pic_marking)
1718 {
1719     gint32 pic_num;
1720
1721     if (!picture->field_pic_flag)
1722         pic_num = picture->frame_num_wrap;
1723     else
1724         pic_num = 2 * picture->frame_num_wrap + 1;
1725     pic_num -= ref_pic_marking->difference_of_pic_nums_minus1 + 1;
1726     return pic_num;
1727 }
1728
1729 /* 8.2.5.4.1. Mark-term reference picture as "unused for reference" */
1730 static void
1731 exec_ref_pic_marking_adaptive_mmco_1(
1732     GstVaapiDecoderH264  *decoder,
1733     GstVaapiPictureH264  *picture,
1734     GstH264RefPicMarking *ref_pic_marking
1735 )
1736 {
1737     GstVaapiDecoderH264Private * const priv = decoder->priv;
1738     gint32 i, picNumX;
1739
1740     picNumX = get_picNumX(picture, ref_pic_marking);
1741     i = find_short_term_reference(decoder, picNumX);
1742     if (i < 0)
1743         return;
1744     remove_reference_at(decoder, priv->short_ref, &priv->short_ref_count, i);
1745 }
1746
1747 /* 8.2.5.4.2. Mark long-term reference picture as "unused for reference" */
1748 static void
1749 exec_ref_pic_marking_adaptive_mmco_2(
1750     GstVaapiDecoderH264  *decoder,
1751     GstVaapiPictureH264  *picture,
1752     GstH264RefPicMarking *ref_pic_marking
1753 )
1754 {
1755     GstVaapiDecoderH264Private * const priv = decoder->priv;
1756     gint32 i;
1757
1758     i = find_long_term_reference(decoder, ref_pic_marking->long_term_pic_num);
1759     if (i < 0)
1760         return;
1761     remove_reference_at(decoder, priv->long_ref, &priv->long_ref_count, i);
1762 }
1763
1764 /* 8.2.5.4.3. Assign LongTermFrameIdx to a short-term reference picture */
1765 static void
1766 exec_ref_pic_marking_adaptive_mmco_3(
1767     GstVaapiDecoderH264  *decoder,
1768     GstVaapiPictureH264  *picture,
1769     GstH264RefPicMarking *ref_pic_marking
1770 )
1771 {
1772     GstVaapiDecoderH264Private * const priv = decoder->priv;
1773     VAPictureH264 *pic;
1774     gint32 i, picNumX;
1775
1776     for (i = 0; i < priv->long_ref_count; i++) {
1777         if (priv->long_ref[i]->info.frame_idx == ref_pic_marking->long_term_frame_idx)
1778             break;
1779     }
1780     if (i != priv->long_ref_count)
1781         remove_reference_at(decoder, priv->long_ref, &priv->long_ref_count, i);
1782
1783     picNumX = get_picNumX(picture, ref_pic_marking);
1784     i = find_short_term_reference(decoder, picNumX);
1785     if (i < 0)
1786         return;
1787
1788     picture = gst_vaapi_picture_ref(priv->short_ref[i]);
1789     remove_reference_at(decoder, priv->short_ref, &priv->short_ref_count, i);
1790     gst_vaapi_picture_replace(&priv->long_ref[priv->long_ref_count++], picture);
1791     gst_vaapi_picture_unref(picture);
1792
1793     picture->is_long_term = TRUE;
1794     pic = &picture->info;
1795     pic->frame_idx = ref_pic_marking->long_term_frame_idx;
1796     pic->flags &= ~VA_PICTURE_H264_SHORT_TERM_REFERENCE;
1797     pic->flags |= VA_PICTURE_H264_LONG_TERM_REFERENCE;
1798     GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_REFERENCE);
1799 }
1800
1801 /* 8.2.5.4.4. Mark pictures with LongTermFramIdx > max_long_term_frame_idx
1802  * as "unused for reference" */
1803 static void
1804 exec_ref_pic_marking_adaptive_mmco_4(
1805     GstVaapiDecoderH264  *decoder,
1806     GstVaapiPictureH264  *picture,
1807     GstH264RefPicMarking *ref_pic_marking
1808 )
1809 {
1810     GstVaapiDecoderH264Private * const priv = decoder->priv;
1811     gint32 i, long_term_frame_idx;
1812
1813     long_term_frame_idx = ref_pic_marking->max_long_term_frame_idx_plus1 - 1;
1814
1815     for (i = 0; i < priv->long_ref_count; i++) {
1816         if ((gint32)priv->long_ref[i]->info.frame_idx <= long_term_frame_idx)
1817             continue;
1818         remove_reference_at(decoder, priv->long_ref, &priv->long_ref_count, i);
1819         i--;
1820     }
1821 }
1822
1823 /* 8.2.5.4.5. Mark all reference pictures as "unused for reference" */
1824 static void
1825 exec_ref_pic_marking_adaptive_mmco_5(
1826     GstVaapiDecoderH264  *decoder,
1827     GstVaapiPictureH264  *picture,
1828     GstH264RefPicMarking *ref_pic_marking
1829 )
1830 {
1831     GstVaapiDecoderH264Private * const priv = decoder->priv;
1832     VAPictureH264 * const pic = &picture->info;
1833
1834     clear_references(decoder, priv->short_ref, &priv->short_ref_count);
1835     clear_references(decoder, priv->long_ref,  &priv->long_ref_count );
1836     dpb_flush(decoder);
1837
1838     priv->prev_pic_has_mmco5 = TRUE;
1839
1840     /* The picture shall be inferred to have had frame_num equal to 0 (7.4.3) */
1841     priv->frame_num = 0;
1842     priv->frame_num_offset = 0;
1843     picture->frame_num = 0;
1844
1845     /* Update TopFieldOrderCnt and BottomFieldOrderCnt (8.2.1) */
1846     if (!(pic->flags & VA_PICTURE_H264_BOTTOM_FIELD))
1847         pic->TopFieldOrderCnt -= picture->poc;
1848     if (!(pic->flags & VA_PICTURE_H264_TOP_FIELD))
1849         pic->BottomFieldOrderCnt -= picture->poc;
1850     picture->poc = 0;
1851 }
1852
1853 /* 8.2.5.4.6. Assign a long-term frame index to the current picture */
1854 static void
1855 exec_ref_pic_marking_adaptive_mmco_6(
1856     GstVaapiDecoderH264  *decoder,
1857     GstVaapiPictureH264  *picture,
1858     GstH264RefPicMarking *ref_pic_marking
1859 )
1860 {
1861     picture->is_long_term = TRUE;
1862     picture->info.frame_idx = ref_pic_marking->long_term_frame_idx;
1863 }
1864
1865 /* 8.2.5.4. Adaptive memory control decoded reference picture marking process */
1866 static gboolean
1867 exec_ref_pic_marking_adaptive(
1868     GstVaapiDecoderH264     *decoder,
1869     GstVaapiPictureH264     *picture,
1870     GstH264DecRefPicMarking *dec_ref_pic_marking
1871 )
1872 {
1873     guint i;
1874
1875     GST_DEBUG("reference picture marking process (adaptive memory control)");
1876
1877     typedef void (*exec_ref_pic_marking_adaptive_mmco_func)(
1878         GstVaapiDecoderH264  *decoder,
1879         GstVaapiPictureH264  *picture,
1880         GstH264RefPicMarking *ref_pic_marking
1881     );
1882
1883     static const exec_ref_pic_marking_adaptive_mmco_func mmco_funcs[] = {
1884         NULL,
1885         exec_ref_pic_marking_adaptive_mmco_1,
1886         exec_ref_pic_marking_adaptive_mmco_2,
1887         exec_ref_pic_marking_adaptive_mmco_3,
1888         exec_ref_pic_marking_adaptive_mmco_4,
1889         exec_ref_pic_marking_adaptive_mmco_5,
1890         exec_ref_pic_marking_adaptive_mmco_6,
1891     };
1892
1893     for (i = 0; i < dec_ref_pic_marking->n_ref_pic_marking; i++) {
1894         GstH264RefPicMarking * const ref_pic_marking =
1895             &dec_ref_pic_marking->ref_pic_marking[i];
1896
1897         const guint mmco = ref_pic_marking->memory_management_control_operation;
1898         if (mmco < G_N_ELEMENTS(mmco_funcs) && mmco_funcs[mmco])
1899             mmco_funcs[mmco](decoder, picture, ref_pic_marking);
1900         else {
1901             GST_ERROR("unhandled MMCO %u", mmco);
1902             return FALSE;
1903         }
1904     }
1905     return TRUE;
1906 }
1907
1908 /* 8.2.5 - Execute reference picture marking process */
1909 static gboolean
1910 exec_ref_pic_marking(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
1911 {
1912     GstVaapiDecoderH264Private * const priv = decoder->priv;
1913     GstVaapiPictureH264 **picture_ptr;
1914
1915     priv->prev_pic_has_mmco5 = FALSE;
1916     priv->prev_pic_bottom_field =
1917         picture->field_pic_flag && picture->bottom_field_flag;
1918
1919     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
1920         return TRUE;
1921
1922     if (!picture->is_idr) {
1923         GstH264DecRefPicMarking * const dec_ref_pic_marking =
1924             get_dec_ref_pic_marking(picture);
1925         if (dec_ref_pic_marking->adaptive_ref_pic_marking_mode_flag) {
1926             if (!exec_ref_pic_marking_adaptive(decoder, picture, dec_ref_pic_marking))
1927                 return FALSE;
1928         }
1929         else {
1930             if (!exec_ref_pic_marking_sliding_window(decoder))
1931                 return FALSE;
1932         }
1933     }
1934
1935     if (picture->is_long_term) {
1936         picture_ptr = &priv->long_ref[priv->long_ref_count++];
1937         picture->info.flags |= VA_PICTURE_H264_LONG_TERM_REFERENCE;
1938     }
1939     else {
1940         picture_ptr = &priv->short_ref[priv->short_ref_count++];
1941         picture->info.flags |= VA_PICTURE_H264_SHORT_TERM_REFERENCE;
1942     }
1943     gst_vaapi_picture_replace(picture_ptr, picture);
1944     return TRUE;
1945 }
1946
1947 static void
1948 vaapi_init_picture(VAPictureH264 *pic)
1949 {
1950     pic->picture_id           = VA_INVALID_ID;
1951     pic->frame_idx            = 0;
1952     pic->flags                = VA_PICTURE_H264_INVALID;
1953     pic->TopFieldOrderCnt     = 0;
1954     pic->BottomFieldOrderCnt  = 0;
1955 }
1956
1957 static gboolean
1958 fill_picture(
1959     GstVaapiDecoderH264 *decoder,
1960     GstVaapiPictureH264 *picture,
1961     GstH264SliceHdr     *slice_hdr,
1962     GstH264NalUnit      *nalu
1963 )
1964 {
1965     GstVaapiDecoderH264Private * const priv = decoder->priv;
1966     GstVaapiPicture * const base_picture = &picture->base;
1967     GstH264SPS * const sps = priv->sps;
1968     GstH264PPS * const pps = priv->pps;
1969     VAPictureParameterBufferH264 * const pic_param = base_picture->param;
1970     guint i, n;
1971
1972     /* Fill in VAPictureParameterBufferH264 */
1973     pic_param->CurrPic = picture->info;
1974     for (i = 0, n = 0; i < priv->short_ref_count; i++)
1975         pic_param->ReferenceFrames[n++] = priv->short_ref[i]->info;
1976     for (i = 0; i < priv->long_ref_count; i++)
1977         pic_param->ReferenceFrames[n++] = priv->long_ref[i]->info;
1978     for (; n < G_N_ELEMENTS(pic_param->ReferenceFrames); n++)
1979         vaapi_init_picture(&pic_param->ReferenceFrames[n]);
1980
1981 #define COPY_FIELD(s, f) \
1982     pic_param->f = (s)->f
1983
1984 #define COPY_BFM(a, s, f) \
1985     pic_param->a.bits.f = (s)->f
1986
1987     pic_param->picture_width_in_mbs_minus1      = priv->mb_width  - 1;
1988     pic_param->picture_height_in_mbs_minus1     = priv->mb_height - 1;
1989     pic_param->frame_num                        = priv->frame_num;
1990
1991     COPY_FIELD(sps, bit_depth_luma_minus8);
1992     COPY_FIELD(sps, bit_depth_chroma_minus8);
1993     COPY_FIELD(sps, num_ref_frames);
1994     COPY_FIELD(pps, num_slice_groups_minus1);
1995     COPY_FIELD(pps, slice_group_map_type);
1996     COPY_FIELD(pps, slice_group_change_rate_minus1);
1997     COPY_FIELD(pps, pic_init_qp_minus26);
1998     COPY_FIELD(pps, pic_init_qs_minus26);
1999     COPY_FIELD(pps, chroma_qp_index_offset);
2000     COPY_FIELD(pps, second_chroma_qp_index_offset);
2001
2002     pic_param->seq_fields.value                                         = 0; /* reset all bits */
2003     pic_param->seq_fields.bits.residual_colour_transform_flag           = sps->separate_colour_plane_flag;
2004     pic_param->seq_fields.bits.MinLumaBiPredSize8x8                     = sps->level_idc >= 31; /* A.3.3.2 */
2005
2006     COPY_BFM(seq_fields, sps, chroma_format_idc);
2007     COPY_BFM(seq_fields, sps, gaps_in_frame_num_value_allowed_flag);
2008     COPY_BFM(seq_fields, sps, frame_mbs_only_flag);
2009     COPY_BFM(seq_fields, sps, mb_adaptive_frame_field_flag);
2010     COPY_BFM(seq_fields, sps, direct_8x8_inference_flag);
2011     COPY_BFM(seq_fields, sps, log2_max_frame_num_minus4);
2012     COPY_BFM(seq_fields, sps, pic_order_cnt_type);
2013     COPY_BFM(seq_fields, sps, log2_max_pic_order_cnt_lsb_minus4);
2014     COPY_BFM(seq_fields, sps, delta_pic_order_always_zero_flag);
2015
2016     pic_param->pic_fields.value                                         = 0; /* reset all bits */
2017     pic_param->pic_fields.bits.field_pic_flag                           = slice_hdr->field_pic_flag;
2018     pic_param->pic_fields.bits.reference_pic_flag                       = GST_VAAPI_PICTURE_IS_REFERENCE(picture);
2019
2020     COPY_BFM(pic_fields, pps, entropy_coding_mode_flag);
2021     COPY_BFM(pic_fields, pps, weighted_pred_flag);
2022     COPY_BFM(pic_fields, pps, weighted_bipred_idc);
2023     COPY_BFM(pic_fields, pps, transform_8x8_mode_flag);
2024     COPY_BFM(pic_fields, pps, constrained_intra_pred_flag);
2025     COPY_BFM(pic_fields, pps, pic_order_present_flag);
2026     COPY_BFM(pic_fields, pps, deblocking_filter_control_present_flag);
2027     COPY_BFM(pic_fields, pps, redundant_pic_cnt_present_flag);
2028     return TRUE;
2029 }
2030
2031 static GstVaapiDecoderStatus
2032 decode_picture(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu, GstH264SliceHdr *slice_hdr)
2033 {
2034     GstVaapiDecoderH264Private * const priv = decoder->priv;
2035     GstVaapiPictureH264 *picture;
2036     GstVaapiDecoderStatus status;
2037     GstH264PPS * const pps = slice_hdr->pps;
2038     GstH264SPS * const sps = pps->sequence;
2039
2040     status = ensure_context(decoder, sps);
2041     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS) {
2042         GST_ERROR("failed to reset context");
2043         return status;
2044     }
2045
2046     if (priv->current_picture && !decode_current_picture(decoder))
2047         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2048
2049     picture = gst_vaapi_picture_h264_new(decoder);
2050     if (!picture) {
2051         GST_ERROR("failed to allocate picture");
2052         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2053     }
2054     priv->current_picture = picture;
2055
2056     priv->sps = sps;
2057     priv->pps = pps;
2058
2059     status = ensure_quant_matrix(decoder, picture);
2060     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS) {
2061         GST_ERROR("failed to reset quantizer matrix");
2062         return status;
2063     }
2064
2065     if (!init_picture(decoder, picture, slice_hdr, nalu))
2066         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2067     if (!fill_picture(decoder, picture, slice_hdr, nalu))
2068         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2069     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2070 }
2071
2072 static gboolean
2073 decode_picture_end(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
2074 {
2075     if (!exec_ref_pic_marking(decoder, picture))
2076         return FALSE;
2077     if (!dpb_add(decoder, picture))
2078         return FALSE;
2079     return TRUE;
2080 }
2081
2082 static inline guint
2083 get_slice_data_bit_offset(GstH264SliceHdr *slice_hdr, GstH264NalUnit *nalu)
2084 {
2085     guint epb_count;
2086
2087     epb_count = slice_hdr->n_emulation_prevention_bytes;
2088     return 8 /* nal_unit_type */ + slice_hdr->header_size - epb_count * 8;
2089 }
2090
2091 static gboolean
2092 fill_pred_weight_table(GstVaapiDecoderH264 *decoder, GstVaapiSliceH264 *slice)
2093 {
2094     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2095     GstH264PPS * const pps = slice_hdr->pps;
2096     GstH264SPS * const sps = pps->sequence;
2097     GstH264PredWeightTable * const w = &slice_hdr->pred_weight_table;
2098     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2099     guint num_weight_tables = 0;
2100     gint i, j;
2101
2102     if (pps->weighted_pred_flag &&
2103         (GST_H264_IS_P_SLICE(slice_hdr) || GST_H264_IS_SP_SLICE(slice_hdr)))
2104         num_weight_tables = 1;
2105     else if (pps->weighted_bipred_idc == 1 && GST_H264_IS_B_SLICE(slice_hdr))
2106         num_weight_tables = 2;
2107     else
2108         num_weight_tables = 0;
2109
2110     slice_param->luma_log2_weight_denom   = w->luma_log2_weight_denom;
2111     slice_param->chroma_log2_weight_denom = w->chroma_log2_weight_denom;
2112     slice_param->luma_weight_l0_flag      = 0;
2113     slice_param->chroma_weight_l0_flag    = 0;
2114     slice_param->luma_weight_l1_flag      = 0;
2115     slice_param->chroma_weight_l1_flag    = 0;
2116
2117     if (num_weight_tables < 1)
2118         return TRUE;
2119
2120     slice_param->luma_weight_l0_flag = 1;
2121     for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2122         slice_param->luma_weight_l0[i] = w->luma_weight_l0[i];
2123         slice_param->luma_offset_l0[i] = w->luma_offset_l0[i];
2124     }
2125
2126     slice_param->chroma_weight_l0_flag = sps->chroma_array_type != 0;
2127     if (slice_param->chroma_weight_l0_flag) {
2128         for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2129             for (j = 0; j < 2; j++) {
2130                 slice_param->chroma_weight_l0[i][j] = w->chroma_weight_l0[i][j];
2131                 slice_param->chroma_offset_l0[i][j] = w->chroma_offset_l0[i][j];
2132             }
2133         }
2134     }
2135
2136     if (num_weight_tables < 2)
2137         return TRUE;
2138
2139     slice_param->luma_weight_l1_flag = 1;
2140     for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2141         slice_param->luma_weight_l1[i] = w->luma_weight_l1[i];
2142         slice_param->luma_offset_l1[i] = w->luma_offset_l1[i];
2143     }
2144
2145     slice_param->chroma_weight_l1_flag = sps->chroma_array_type != 0;
2146     if (slice_param->chroma_weight_l1_flag) {
2147         for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2148             for (j = 0; j < 2; j++) {
2149                 slice_param->chroma_weight_l1[i][j] = w->chroma_weight_l1[i][j];
2150                 slice_param->chroma_offset_l1[i][j] = w->chroma_offset_l1[i][j];
2151             }
2152         }
2153     }
2154     return TRUE;
2155 }
2156
2157 static gboolean
2158 fill_RefPicList(GstVaapiDecoderH264 *decoder, GstVaapiSliceH264 *slice)
2159 {
2160     GstVaapiDecoderH264Private * const priv = decoder->priv;
2161     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2162     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2163     guint i, num_ref_lists = 0;
2164
2165     slice_param->num_ref_idx_l0_active_minus1 = 0;
2166     slice_param->num_ref_idx_l1_active_minus1 = 0;
2167
2168     if (GST_H264_IS_B_SLICE(slice_hdr))
2169         num_ref_lists = 2;
2170     else if (GST_H264_IS_I_SLICE(slice_hdr))
2171         num_ref_lists = 0;
2172     else
2173         num_ref_lists = 1;
2174
2175     if (num_ref_lists < 1)
2176         return TRUE;
2177
2178     slice_param->num_ref_idx_l0_active_minus1 =
2179         slice_hdr->num_ref_idx_l0_active_minus1;
2180
2181     for (i = 0; i < priv->RefPicList0_count && priv->RefPicList0[i]; i++)
2182         slice_param->RefPicList0[i] = priv->RefPicList0[i]->info;
2183     for (; i <= slice_param->num_ref_idx_l0_active_minus1; i++)
2184         vaapi_init_picture(&slice_param->RefPicList0[i]);
2185
2186     if (num_ref_lists < 2)
2187         return TRUE;
2188
2189     slice_param->num_ref_idx_l1_active_minus1 =
2190         slice_hdr->num_ref_idx_l1_active_minus1;
2191
2192     for (i = 0; i < priv->RefPicList1_count && priv->RefPicList1[i]; i++)
2193         slice_param->RefPicList1[i] = priv->RefPicList1[i]->info;
2194     for (; i <= slice_param->num_ref_idx_l1_active_minus1; i++)
2195         vaapi_init_picture(&slice_param->RefPicList1[i]);
2196     return TRUE;
2197 }
2198
2199 static gboolean
2200 fill_slice(
2201     GstVaapiDecoderH264 *decoder,
2202     GstVaapiSliceH264   *slice,
2203     GstH264NalUnit      *nalu
2204 )
2205 {
2206     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2207     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2208
2209     /* Fill in VASliceParameterBufferH264 */
2210     slice_param->slice_data_bit_offset          = get_slice_data_bit_offset(slice_hdr, nalu);
2211     slice_param->first_mb_in_slice              = slice_hdr->first_mb_in_slice;
2212     slice_param->slice_type                     = slice_hdr->type % 5;
2213     slice_param->direct_spatial_mv_pred_flag    = slice_hdr->direct_spatial_mv_pred_flag;
2214     slice_param->cabac_init_idc                 = slice_hdr->cabac_init_idc;
2215     slice_param->slice_qp_delta                 = slice_hdr->slice_qp_delta;
2216     slice_param->disable_deblocking_filter_idc  = slice_hdr->disable_deblocking_filter_idc;
2217     slice_param->slice_alpha_c0_offset_div2     = slice_hdr->slice_alpha_c0_offset_div2;
2218     slice_param->slice_beta_offset_div2         = slice_hdr->slice_beta_offset_div2;
2219
2220     if (!fill_RefPicList(decoder, slice))
2221         return FALSE;
2222     if (!fill_pred_weight_table(decoder, slice))
2223         return FALSE;
2224     return TRUE;
2225 }
2226
2227 static GstVaapiDecoderStatus
2228 decode_slice(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
2229 {
2230     GstVaapiDecoderH264Private * const priv = decoder->priv;
2231     GstVaapiDecoderStatus status;
2232     GstVaapiPictureH264 *picture;
2233     GstVaapiSliceH264 *slice = NULL;
2234     GstH264SliceHdr *slice_hdr;
2235     GstH264ParserResult result;
2236
2237     GST_DEBUG("slice (%u bytes)", nalu->size);
2238
2239     slice = gst_vaapi_slice_h264_new(
2240         decoder,
2241         nalu->data + nalu->offset,
2242         nalu->size
2243     );
2244     if (!slice) {
2245         GST_ERROR("failed to allocate slice");
2246         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2247     }
2248
2249     slice_hdr = &slice->slice_hdr;
2250     memset(slice_hdr, 0, sizeof(*slice_hdr));
2251     result = gst_h264_parser_parse_slice_hdr(priv->parser, nalu, slice_hdr, TRUE, TRUE);
2252     if (result != GST_H264_PARSER_OK) {
2253         status = get_status(result);
2254         goto error;
2255     }
2256
2257     if (slice_hdr->first_mb_in_slice == 0) {
2258         status = decode_picture(decoder, nalu, slice_hdr);
2259         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2260             goto error;
2261     }
2262     picture = priv->current_picture;
2263
2264     priv->mb_x = slice_hdr->first_mb_in_slice % priv->mb_width;
2265     priv->mb_y = slice_hdr->first_mb_in_slice / priv->mb_width; // FIXME: MBAFF or field
2266
2267     if (!fill_slice(decoder, slice, nalu)) {
2268         status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2269         goto error;
2270     }
2271     gst_vaapi_picture_add_slice(
2272         GST_VAAPI_PICTURE_CAST(picture),
2273         GST_VAAPI_SLICE_CAST(slice)
2274     );
2275
2276     /* Commit picture for decoding if we reached the last slice */
2277     if (++priv->mb_y >= priv->mb_height) {
2278         if (!decode_current_picture(decoder)) {
2279             status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2280             goto error;
2281         }
2282         GST_DEBUG("done");
2283     }
2284     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2285
2286 error:
2287     if (slice)
2288         gst_mini_object_unref(GST_MINI_OBJECT(slice));
2289     return status;
2290 }
2291
2292 static inline gint
2293 scan_for_start_code(GstAdapter *adapter, guint ofs, guint size, guint32 *scp)
2294 {
2295     return (gint)gst_adapter_masked_scan_uint32_peek(adapter,
2296                                                      0xffffff00, 0x00000100,
2297                                                      ofs, size,
2298                                                      scp);
2299 }
2300
2301 static GstVaapiDecoderStatus
2302 decode_nalu(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
2303 {
2304     GstVaapiDecoderStatus status;
2305
2306     switch (nalu->type) {
2307     case GST_H264_NAL_SLICE_IDR:
2308         /* fall-through. IDR specifics are handled in init_picture() */
2309     case GST_H264_NAL_SLICE:
2310         status = decode_slice(decoder, nalu);
2311         break;
2312     case GST_H264_NAL_SPS:
2313         status = decode_sps(decoder, nalu);
2314         break;
2315     case GST_H264_NAL_PPS:
2316         status = decode_pps(decoder, nalu);
2317         break;
2318     case GST_H264_NAL_SEI:
2319         status = decode_sei(decoder, nalu);
2320         break;
2321     case GST_H264_NAL_SEQ_END:
2322         status = decode_sequence_end(decoder);
2323         break;
2324     case GST_H264_NAL_AU_DELIMITER:
2325         /* skip all Access Unit NALs */
2326         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2327         break;
2328     case GST_H264_NAL_FILLER_DATA:
2329         /* skip all Filler Data NALs */
2330         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2331         break;
2332     default:
2333         GST_WARNING("unsupported NAL unit type %d", nalu->type);
2334         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
2335         break;
2336     }
2337     return status;
2338 }
2339
2340 static GstVaapiDecoderStatus
2341 decode_buffer(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
2342 {
2343     GstVaapiDecoderH264Private * const priv = decoder->priv;
2344     GstVaapiDecoderStatus status;
2345     GstH264ParserResult result;
2346     GstH264NalUnit nalu;
2347     gboolean is_eos;
2348     const guchar *buf;
2349     guint i, buf_size, nalu_size, size;
2350     guint32 start_code;
2351     gint ofs;
2352
2353     buf      = GST_BUFFER_DATA(buffer);
2354     buf_size = GST_BUFFER_SIZE(buffer);
2355     is_eos   = GST_BUFFER_IS_EOS(buffer);
2356     if (buf && buf_size > 0)
2357         gst_adapter_push(priv->adapter, gst_buffer_ref(buffer));
2358
2359     size = gst_adapter_available(priv->adapter);
2360     do {
2361         if (size == 0) {
2362             status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2363             break;
2364         }
2365
2366         status = gst_vaapi_decoder_check_status(GST_VAAPI_DECODER(decoder));
2367         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2368             break;
2369
2370         status = GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2371         if (priv->is_avc) {
2372             if (size < priv->nal_length_size)
2373                 break;
2374             buf = gst_adapter_peek(priv->adapter, priv->nal_length_size);
2375
2376             nalu_size = 0;
2377             for (i = 0; i < priv->nal_length_size; i++)
2378                 nalu_size = (nalu_size << 8) | buf[i];
2379
2380             buf_size = priv->nal_length_size + nalu_size;
2381             if (size < buf_size)
2382                 break;
2383             buffer = gst_adapter_take_buffer(priv->adapter, buf_size);
2384             size -= buf_size;
2385
2386             buf      = GST_BUFFER_DATA(buffer);
2387             buf_size = GST_BUFFER_SIZE(buffer);
2388
2389             result = gst_h264_parser_identify_nalu_avc(
2390                 priv->parser,
2391                 buf, 0, buf_size, priv->nal_length_size,
2392                 &nalu
2393             );
2394         }
2395         else {
2396             if (size < 4)
2397                 break;
2398             ofs = scan_for_start_code(priv->adapter, 0, size, &start_code);
2399             if (ofs < 0)
2400                 break;
2401             gst_adapter_flush(priv->adapter, ofs);
2402             size -= ofs;
2403
2404             ofs = G_UNLIKELY(size < 8) ? -1 :
2405                 scan_for_start_code(priv->adapter, 4, size - 4, NULL);
2406             if (ofs < 0) {
2407                 // Assume the whole NAL unit is present if end-of-stream
2408                 if (!is_eos)
2409                     break;
2410                 ofs = size;
2411             }
2412             buffer = gst_adapter_take_buffer(priv->adapter, ofs);
2413             size -= ofs;
2414
2415             buf      = GST_BUFFER_DATA(buffer);
2416             buf_size = GST_BUFFER_SIZE(buffer);
2417
2418             result = gst_h264_parser_identify_nalu_unchecked(
2419                 priv->parser,
2420                 buf, 0, buf_size,
2421                 &nalu
2422             );
2423         }
2424         status = get_status(result);
2425         if (status == GST_VAAPI_DECODER_STATUS_SUCCESS)
2426             status = decode_nalu(decoder, &nalu);
2427         gst_buffer_unref(buffer);
2428     } while (status == GST_VAAPI_DECODER_STATUS_SUCCESS);
2429
2430     if (is_eos && (status == GST_VAAPI_DECODER_STATUS_SUCCESS ||
2431                    status == GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA))
2432         status = decode_sequence_end(decoder);
2433     return status;
2434 }
2435
2436 static GstVaapiDecoderStatus
2437 decode_codec_data(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
2438 {
2439     GstVaapiDecoderH264Private * const priv = decoder->priv;
2440     GstVaapiDecoderStatus status;
2441     GstH264NalUnit nalu;
2442     GstH264ParserResult result;
2443     guchar *buf;
2444     guint buf_size;
2445     guint i, ofs, num_sps, num_pps;
2446
2447     buf      = GST_BUFFER_DATA(buffer);
2448     buf_size = GST_BUFFER_SIZE(buffer);
2449     if (!buf || buf_size == 0)
2450         return GST_VAAPI_DECODER_STATUS_SUCCESS;
2451
2452     if (buf_size < 8)
2453         return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2454
2455     if (buf[0] != 1) {
2456         GST_ERROR("failed to decode codec-data, not in avcC format");
2457         return GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
2458     }
2459
2460     priv->nal_length_size = (buf[4] & 0x03) + 1;
2461
2462     num_sps = buf[5] & 0x1f;
2463     ofs = 6;
2464
2465     for (i = 0; i < num_sps; i++) {
2466         result = gst_h264_parser_identify_nalu_avc(
2467             priv->parser,
2468             buf, ofs, buf_size, 2,
2469             &nalu
2470         );
2471         if (result != GST_H264_PARSER_OK)
2472             return get_status(result);
2473
2474         status = decode_sps(decoder, &nalu);
2475         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2476             return status;
2477         ofs = nalu.offset + nalu.size;
2478     }
2479
2480     num_pps = buf[ofs];
2481     ofs++;
2482
2483     for (i = 0; i < num_pps; i++) {
2484         result = gst_h264_parser_identify_nalu_avc(
2485             priv->parser,
2486             buf, ofs, buf_size, 2,
2487             &nalu
2488         );
2489         if (result != GST_H264_PARSER_OK)
2490             return get_status(result);
2491
2492         status = decode_pps(decoder, &nalu);
2493         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2494             return status;
2495         ofs = nalu.offset + nalu.size;
2496     }
2497
2498     priv->is_avc = TRUE;
2499     return status;
2500 }
2501
2502 GstVaapiDecoderStatus
2503 gst_vaapi_decoder_h264_decode(GstVaapiDecoder *base, GstBuffer *buffer)
2504 {
2505     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264(base);
2506     GstVaapiDecoderH264Private * const priv = decoder->priv;
2507     GstVaapiDecoderStatus status;
2508     GstBuffer *codec_data;
2509
2510     g_return_val_if_fail(priv->is_constructed,
2511                          GST_VAAPI_DECODER_STATUS_ERROR_INIT_FAILED);
2512
2513     if (!priv->is_opened) {
2514         priv->is_opened = gst_vaapi_decoder_h264_open(decoder, buffer);
2515         if (!priv->is_opened)
2516             return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CODEC;
2517
2518         codec_data = GST_VAAPI_DECODER_CODEC_DATA(decoder);
2519         if (codec_data) {
2520             status = decode_codec_data(decoder, codec_data);
2521             if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2522                 return status;
2523         }
2524      }
2525      return decode_buffer(decoder, buffer);
2526 }
2527
2528 static void
2529 gst_vaapi_decoder_h264_finalize(GObject *object)
2530 {
2531     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264(object);
2532
2533     gst_vaapi_decoder_h264_destroy(decoder);
2534
2535     G_OBJECT_CLASS(gst_vaapi_decoder_h264_parent_class)->finalize(object);
2536 }
2537
2538 static void
2539 gst_vaapi_decoder_h264_constructed(GObject *object)
2540 {
2541     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264(object);
2542     GstVaapiDecoderH264Private * const priv = decoder->priv;
2543     GObjectClass *parent_class;
2544
2545     parent_class = G_OBJECT_CLASS(gst_vaapi_decoder_h264_parent_class);
2546     if (parent_class->constructed)
2547         parent_class->constructed(object);
2548
2549     priv->is_constructed = gst_vaapi_decoder_h264_create(decoder);
2550 }
2551
2552 static void
2553 gst_vaapi_decoder_h264_class_init(GstVaapiDecoderH264Class *klass)
2554 {
2555     GObjectClass * const object_class = G_OBJECT_CLASS(klass);
2556     GstVaapiDecoderClass * const decoder_class = GST_VAAPI_DECODER_CLASS(klass);
2557
2558     g_type_class_add_private(klass, sizeof(GstVaapiDecoderH264Private));
2559
2560     object_class->finalize      = gst_vaapi_decoder_h264_finalize;
2561     object_class->constructed   = gst_vaapi_decoder_h264_constructed;
2562
2563     decoder_class->decode       = gst_vaapi_decoder_h264_decode;
2564 }
2565
2566 static void
2567 gst_vaapi_decoder_h264_init(GstVaapiDecoderH264 *decoder)
2568 {
2569     GstVaapiDecoderH264Private *priv;
2570
2571     priv                        = GST_VAAPI_DECODER_H264_GET_PRIVATE(decoder);
2572     decoder->priv               = priv;
2573     priv->parser                = NULL;
2574     priv->sps                   = &priv->last_sps;
2575     priv->pps                   = &priv->last_pps;
2576     priv->current_picture       = NULL;
2577     priv->dpb_count             = 0;
2578     priv->dpb_size              = 0;
2579     priv->profile               = GST_VAAPI_PROFILE_H264_HIGH;
2580     priv->short_ref_count       = 0;
2581     priv->long_ref_count        = 0;
2582     priv->RefPicList0_count     = 0;
2583     priv->RefPicList1_count     = 0;
2584     priv->nal_length_size       = 0;
2585     priv->width                 = 0;
2586     priv->height                = 0;
2587     priv->mb_x                  = 0;
2588     priv->mb_y                  = 0;
2589     priv->mb_width              = 0;
2590     priv->mb_height             = 0;
2591     priv->adapter               = NULL;
2592     priv->field_poc[0]          = 0;
2593     priv->field_poc[1]          = 0;
2594     priv->poc_msb               = 0;
2595     priv->poc_lsb               = 0;
2596     priv->prev_poc_msb          = 0;
2597     priv->prev_poc_lsb          = 0;
2598     priv->frame_num_offset      = 0;
2599     priv->frame_num             = 0;
2600     priv->prev_frame_num        = 0;
2601     priv->prev_pic_has_mmco5    = FALSE;
2602     priv->prev_pic_bottom_field = FALSE;
2603     priv->is_constructed        = FALSE;
2604     priv->is_opened             = FALSE;
2605     priv->is_avc                = FALSE;
2606     priv->has_context           = FALSE;
2607
2608     memset(priv->dpb, 0, sizeof(priv->dpb));
2609     memset(priv->short_ref, 0, sizeof(priv->short_ref));
2610     memset(priv->long_ref, 0, sizeof(priv->long_ref));
2611     memset(priv->RefPicList0, 0, sizeof(priv->RefPicList0));
2612     memset(priv->RefPicList1, 0, sizeof(priv->RefPicList1));
2613 }
2614
2615 /**
2616  * gst_vaapi_decoder_h264_new:
2617  * @display: a #GstVaapiDisplay
2618  * @caps: a #GstCaps holding codec information
2619  *
2620  * Creates a new #GstVaapiDecoder for MPEG-2 decoding.  The @caps can
2621  * hold extra information like codec-data and pictured coded size.
2622  *
2623  * Return value: the newly allocated #GstVaapiDecoder object
2624  */
2625 GstVaapiDecoder *
2626 gst_vaapi_decoder_h264_new(GstVaapiDisplay *display, GstCaps *caps)
2627 {
2628     GstVaapiDecoderH264 *decoder;
2629
2630     g_return_val_if_fail(GST_VAAPI_IS_DISPLAY(display), NULL);
2631     g_return_val_if_fail(GST_IS_CAPS(caps), NULL);
2632
2633     decoder = g_object_new(
2634         GST_VAAPI_TYPE_DECODER_H264,
2635         "display",      display,
2636         "caps",         caps,
2637         NULL
2638     );
2639     if (!decoder->priv->is_constructed) {
2640         g_object_unref(decoder);
2641         return NULL;
2642     }
2643     return GST_VAAPI_DECODER_CAST(decoder);
2644 }