gst-libs/gst/vaapi/gstvaapidecoder_h264.c

   1 /*
   2  *  gstvaapidecoder_h264.c - H.264 decoder
   3  *
   4  *  Copyright (C) 2011-2012 Intel Corporation
   5  *
   6  *  This library is free software; you can redistribute it and/or
   7  *  modify it under the terms of the GNU Lesser General Public License
   8  *  as published by the Free Software Foundation; either version 2.1
   9  *  of the License, or (at your option) any later version.
  10  *
  11  *  This library is distributed in the hope that it will be useful,
  12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  *  Lesser General Public License for more details.
  15  *
  16  *  You should have received a copy of the GNU Lesser General Public
  17  *  License along with this library; if not, write to the Free
  18  *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  19  *  Boston, MA 02110-1301 USA
  20  */
  21
  22 /**
  23  * SECTION:gstvaapidecoder_h264
  24  * @short_description: H.264 decoder
  25  */
  26
  27 #include "sysdeps.h"
  28 #include <string.h>
  29 #include <gst/base/gstadapter.h>
  30 #include <gst/codecparsers/gsth264parser.h>
  31 #include "gstvaapidecoder_h264.h"
  32 #include "gstvaapidecoder_objects.h"
  33 #include "gstvaapidecoder_priv.h"
  34 #include "gstvaapidisplay_priv.h"
  35 #include "gstvaapiobject_priv.h"
  36
  37 #define DEBUG 1
  38 #include "gstvaapidebug.h"
  39
  40 /* Defined to 1 if strict ordering of DPB is needed. Only useful for debug */
  41 #define USE_STRICT_DPB_ORDERING 0
  42
  43 typedef struct _GstVaapiPictureH264             GstVaapiPictureH264;
  44 typedef struct _GstVaapiPictureH264Class        GstVaapiPictureH264Class;
  45 typedef struct _GstVaapiSliceH264               GstVaapiSliceH264;
  46 typedef struct _GstVaapiSliceH264Class          GstVaapiSliceH264Class;
  47
  48 /* ------------------------------------------------------------------------- */
  49 /* --- H.264 Pictures                                                    --- */
  50 /* ------------------------------------------------------------------------- */
  51
  52 #define GST_VAAPI_TYPE_PICTURE_H264 \
  53     (gst_vaapi_picture_h264_get_type())
  54
  55 #define GST_VAAPI_PICTURE_H264_CAST(obj) \
  56     ((GstVaapiPictureH264 *)(obj))
  57
  58 #define GST_VAAPI_PICTURE_H264(obj)                             \
  59     (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
  60                                 GST_VAAPI_TYPE_PICTURE_H264,    \
  61                                 GstVaapiPictureH264))
  62
  63 #define GST_VAAPI_PICTURE_H264_CLASS(klass)                     \
  64     (G_TYPE_CHECK_CLASS_CAST((klass),                           \
  65                              GST_VAAPI_TYPE_PICTURE_H264,       \
  66                              GstVaapiPictureH264Class))
  67
  68 #define GST_VAAPI_IS_PICTURE_H264(obj) \
  69     (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_VAAPI_TYPE_PICTURE_H264))
  70
  71 #define GST_VAAPI_IS_PICTURE_H264_CLASS(klass) \
  72     (G_TYPE_CHECK_CLASS_TYPE((klass), GST_VAAPI_TYPE_PICTURE_H264))
  73
  74 #define GST_VAAPI_PICTURE_H264_GET_CLASS(obj)                   \
  75     (G_TYPE_INSTANCE_GET_CLASS((obj),                           \
  76                                GST_VAAPI_TYPE_PICTURE_H264,     \
  77                                GstVaapiPictureH264Class))
  78
  79 struct _GstVaapiPictureH264 {
  80     GstVaapiPicture             base;
  81     VAPictureH264               info;
  82     GstH264PPS                 *pps;
  83     gint32                      poc;
  84     gint32                      frame_num;              // Original frame_num from slice_header()
  85     gint32                      frame_num_wrap;         // Temporary for ref pic marking: FrameNumWrap
  86     gint32                      pic_num;                // Temporary for ref pic marking: PicNum
  87     gint32                      long_term_pic_num;      // Temporary for ref pic marking: LongTermPicNum
  88     guint                       is_idr                  : 1;
  89     guint                       is_long_term            : 1;
  90     guint                       field_pic_flag          : 1;
  91     guint                       bottom_field_flag       : 1;
  92     guint                       output_flag             : 1;
  93     guint                       output_needed           : 1;
  94 };
  95
  96 struct _GstVaapiPictureH264Class {
  97     /*< private >*/
  98     GstVaapiPictureClass        parent_class;
  99 };
 100
 101 GST_VAAPI_CODEC_DEFINE_TYPE(GstVaapiPictureH264,
 102                             gst_vaapi_picture_h264,
 103                             GST_VAAPI_TYPE_PICTURE)
 104
 105 static void
 106 gst_vaapi_picture_h264_destroy(GstVaapiPictureH264 *decoder)
 107 {
 108 }
 109
 110 static gboolean
 111 gst_vaapi_picture_h264_create(
 112     GstVaapiPictureH264                      *picture,
 113     const GstVaapiCodecObjectConstructorArgs *args
 114 )
 115 {
 116     return TRUE;
 117 }
 118
 119 static void
 120 gst_vaapi_picture_h264_init(GstVaapiPictureH264 *picture)
 121 {
 122     VAPictureH264 *va_pic;
 123
 124     va_pic                      = &picture->info;
 125     va_pic->flags               = 0;
 126     va_pic->TopFieldOrderCnt    = 0;
 127     va_pic->BottomFieldOrderCnt = 0;
 128
 129     picture->poc                = 0;
 130     picture->is_long_term       = FALSE;
 131     picture->is_idr             = FALSE;
 132     picture->output_needed      = FALSE;
 133 }
 134
 135 static inline GstVaapiPictureH264 *
 136 gst_vaapi_picture_h264_new(GstVaapiDecoderH264 *decoder)
 137 {
 138     GstVaapiCodecObject *object;
 139
 140     g_return_val_if_fail(GST_VAAPI_IS_DECODER(decoder), NULL);
 141
 142     object = gst_vaapi_codec_object_new(
 143         GST_VAAPI_TYPE_PICTURE_H264,
 144         GST_VAAPI_CODEC_BASE(decoder),
 145         NULL, sizeof(VAPictureParameterBufferH264),
 146         NULL, 0
 147     );
 148     if (!object)
 149         return NULL;
 150     return GST_VAAPI_PICTURE_H264_CAST(object);
 151 }
 152
 153 static inline GstVaapiSliceH264 *
 154 gst_vaapi_picture_h264_get_last_slice(GstVaapiPictureH264 *picture)
 155 {
 156     g_return_val_if_fail(GST_VAAPI_IS_PICTURE_H264(picture), NULL);
 157
 158     if (G_UNLIKELY(picture->base.slices->len < 1))
 159         return NULL;
 160     return g_ptr_array_index(picture->base.slices,
 161         picture->base.slices->len - 1);
 162 }
 163
 164 /* ------------------------------------------------------------------------- */
 165 /* --- Slices                                                            --- */
 166 /* ------------------------------------------------------------------------- */
 167
 168 #define GST_VAAPI_TYPE_SLICE_H264 \
 169     (gst_vaapi_slice_h264_get_type())
 170
 171 #define GST_VAAPI_SLICE_H264_CAST(obj) \
 172     ((GstVaapiSliceH264 *)(obj))
 173
 174 #define GST_VAAPI_SLICE_H264(obj)                               \
 175     (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
 176                                 GST_VAAPI_TYPE_SLICE_H264,      \
 177                                 GstVaapiSliceH264))
 178
 179 #define GST_VAAPI_SLICE_H264_CLASS(klass)                       \
 180     (G_TYPE_CHECK_CLASS_CAST((klass),                           \
 181                              GST_VAAPI_TYPE_SLICE_H264,         \
 182                              GstVaapiSliceH264Class))
 183
 184 #define GST_VAAPI_IS_SLICE_H264(obj) \
 185     (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_VAAPI_TYPE_SLICE_H264))
 186
 187 #define GST_VAAPI_IS_SLICE_H264_CLASS(klass) \
 188     (G_TYPE_CHECK_CLASS_TYPE((klass), GST_VAAPI_TYPE_SLICE_H264))
 189
 190 #define GST_VAAPI_SLICE_H264_GET_CLASS(obj)                     \
 191     (G_TYPE_INSTANCE_GET_CLASS((obj),                           \
 192                                GST_VAAPI_TYPE_SLICE_H264,       \
 193                                GstVaapiSliceH264Class))
 194
 195 struct _GstVaapiSliceH264 {
 196     GstVaapiSlice               base;
 197     GstH264SliceHdr             slice_hdr;              // parsed slice_header()
 198 };
 199
 200 struct _GstVaapiSliceH264Class {
 201     /*< private >*/
 202     GstVaapiSliceClass          parent_class;
 203 };
 204
 205 GST_VAAPI_CODEC_DEFINE_TYPE(GstVaapiSliceH264,
 206                             gst_vaapi_slice_h264,
 207                             GST_VAAPI_TYPE_SLICE)
 208
 209 static void
 210 gst_vaapi_slice_h264_destroy(GstVaapiSliceH264 *slice)
 211 {
 212 }
 213
 214 static gboolean
 215 gst_vaapi_slice_h264_create(
 216     GstVaapiSliceH264                        *slice,
 217     const GstVaapiCodecObjectConstructorArgs *args
 218 )
 219 {
 220     return TRUE;
 221 }
 222
 223 static void
 224 gst_vaapi_slice_h264_init(GstVaapiSliceH264 *slice)
 225 {
 226 }
 227
 228 static inline GstVaapiSliceH264 *
 229 gst_vaapi_slice_h264_new(
 230     GstVaapiDecoderH264 *decoder,
 231     const guint8        *data,
 232     guint                data_size
 233 )
 234 {
 235     GstVaapiCodecObject *object;
 236
 237     g_return_val_if_fail(GST_VAAPI_IS_DECODER(decoder), NULL);
 238
 239     object = gst_vaapi_codec_object_new(
 240         GST_VAAPI_TYPE_SLICE_H264,
 241         GST_VAAPI_CODEC_BASE(decoder),
 242         NULL, sizeof(VASliceParameterBufferH264),
 243         data, data_size
 244     );
 245     if (!object)
 246         return NULL;
 247     return GST_VAAPI_SLICE_H264_CAST(object);
 248 }
 249
 250 /* ------------------------------------------------------------------------- */
 251 /* --- H.264 Decoder                                                     --- */
 252 /* ------------------------------------------------------------------------- */
 253
 254 G_DEFINE_TYPE(GstVaapiDecoderH264,
 255               gst_vaapi_decoder_h264,
 256               GST_VAAPI_TYPE_DECODER)
 257
 258 #define GST_VAAPI_DECODER_H264_GET_PRIVATE(obj)                 \
 259     (G_TYPE_INSTANCE_GET_PRIVATE((obj),                         \
 260                                  GST_VAAPI_TYPE_DECODER_H264,   \
 261                                  GstVaapiDecoderH264Private))
 262
 263 // Used for field_poc[]
 264 #define TOP_FIELD       0
 265 #define BOTTOM_FIELD    1
 266
 267 struct _GstVaapiDecoderH264Private {
 268     GstAdapter                 *adapter;
 269     GstH264NalParser           *parser;
 270     /* Last decoded SPS. May not be the last activated one. Just here because
 271        it may not fit stack memory allocation in decode_sps() */
 272     GstH264SPS                  last_sps;
 273     /* Last decoded PPS. May not be the last activated one. Just here because
 274        it may not fit stack memory allocation in decode_pps() */
 275     GstH264PPS                  last_pps;
 276     GstVaapiPictureH264        *current_picture;
 277     GstVaapiPictureH264        *dpb[16];
 278     guint                       dpb_count;
 279     guint                       dpb_size;
 280     GstVaapiProfile             profile;
 281     GstVaapiEntrypoint          entrypoint;
 282     GstVaapiChromaType          chroma_type;
 283     GstVaapiPictureH264        *short_ref[32];
 284     guint                       short_ref_count;
 285     GstVaapiPictureH264        *long_ref[32];
 286     guint                       long_ref_count;
 287     GstVaapiPictureH264        *RefPicList0[32];
 288     guint                       RefPicList0_count;
 289     GstVaapiPictureH264        *RefPicList1[32];
 290     guint                       RefPicList1_count;
 291     guint                       nal_length_size;
 292     guint                       width;
 293     guint                       height;
 294     gint32                      field_poc[2];           // 0:TopFieldOrderCnt / 1:BottomFieldOrderCnt
 295     gint32                      poc_msb;                // PicOrderCntMsb
 296     gint32                      poc_lsb;                // pic_order_cnt_lsb (from slice_header())
 297     gint32                      prev_poc_msb;           // prevPicOrderCntMsb
 298     gint32                      prev_poc_lsb;           // prevPicOrderCntLsb
 299     gint32                      frame_num_offset;       // FrameNumOffset
 300     gint32                      frame_num;              // frame_num (from slice_header())
 301     gint32                      prev_frame_num;         // prevFrameNum
 302     gboolean                    prev_pic_has_mmco5;     // prevMmco5Pic
 303     gboolean                    prev_pic_bottom_field;  // Flag: previous picture is a bottom field
 304     guint                       is_constructed          : 1;
 305     guint                       is_opened               : 1;
 306     guint                       is_avc                  : 1;
 307     guint                       has_context             : 1;
 308 };
 309
 310 static gboolean
 311 decode_picture_end(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture);
 312
 313 static void
 314 clear_references(
 315     GstVaapiDecoderH264  *decoder,
 316     GstVaapiPictureH264 **pictures,
 317     guint                *picture_count
 318 );
 319
 320 /* Get number of reference frames to use */
 321 static guint
 322 get_max_dec_frame_buffering(GstH264SPS *sps)
 323 {
 324     guint max_dec_frame_buffering, MaxDpbMbs, PicSizeMbs;
 325
 326     /* Table A-1 - Level limits */
 327     switch (sps->level_idc) {
 328     case 10: MaxDpbMbs = 396;    break;
 329     case 11: MaxDpbMbs = 900;    break;
 330     case 12: MaxDpbMbs = 2376;   break;
 331     case 13: MaxDpbMbs = 2376;   break;
 332     case 20: MaxDpbMbs = 2376;   break;
 333     case 21: MaxDpbMbs = 4752;   break;
 334     case 22: MaxDpbMbs = 8100;   break;
 335     case 30: MaxDpbMbs = 8100;   break;
 336     case 31: MaxDpbMbs = 18000;  break;
 337     case 32: MaxDpbMbs = 20480;  break;
 338     case 40: MaxDpbMbs = 32768;  break;
 339     case 41: MaxDpbMbs = 32768;  break;
 340     case 42: MaxDpbMbs = 34816;  break;
 341     case 50: MaxDpbMbs = 110400; break;
 342     case 51: MaxDpbMbs = 184320; break;
 343     default:
 344         g_assert(0 && "unhandled level");
 345         break;
 346     }
 347
 348     PicSizeMbs = ((sps->pic_width_in_mbs_minus1 + 1) *
 349                   (sps->pic_height_in_map_units_minus1 + 1) *
 350                   (sps->frame_mbs_only_flag ? 1 : 2));
 351     max_dec_frame_buffering = MaxDpbMbs / PicSizeMbs;
 352
 353     /* VUI parameters */
 354     if (sps->vui_parameters_present_flag) {
 355         GstH264VUIParams * const vui_params = &sps->vui_parameters;
 356         if (vui_params->bitstream_restriction_flag)
 357             max_dec_frame_buffering = vui_params->max_dec_frame_buffering;
 358         else {
 359             switch (sps->profile_idc) {
 360             case 44:  // CAVLC 4:4:4 Intra profile
 361             case 86:  // Scalable High profile
 362             case 100: // High profile
 363             case 110: // High 10 profile
 364             case 122: // High 4:2:2 profile
 365             case 244: // High 4:4:4 Predictive profile
 366                 if (sps->constraint_set3_flag)
 367                     max_dec_frame_buffering = 0;
 368                 break;
 369             }
 370         }
 371     }
 372
 373     if (max_dec_frame_buffering > 16)
 374         max_dec_frame_buffering = 16;
 375     else if (max_dec_frame_buffering < sps->num_ref_frames)
 376         max_dec_frame_buffering = sps->num_ref_frames;
 377     return MAX(1, max_dec_frame_buffering);
 378 }
 379
 380 static void
 381 dpb_remove_index(GstVaapiDecoderH264 *decoder, guint index)
 382 {
 383     GstVaapiDecoderH264Private * const priv = decoder->priv;
 384     guint i, num_pictures = --priv->dpb_count;
 385
 386     if (USE_STRICT_DPB_ORDERING) {
 387         for (i = index; i < num_pictures; i++)
 388             gst_vaapi_picture_replace(&priv->dpb[i], priv->dpb[i + 1]);
 389     }
 390     else if (index != num_pictures)
 391         gst_vaapi_picture_replace(&priv->dpb[index], priv->dpb[num_pictures]);
 392     gst_vaapi_picture_replace(&priv->dpb[num_pictures], NULL);
 393 }
 394
 395 static inline gboolean
 396 dpb_output(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
 397 {
 398     /* XXX: update cropping rectangle */
 399     picture->output_needed = FALSE;
 400     return gst_vaapi_picture_output(GST_VAAPI_PICTURE_CAST(picture));
 401 }
 402
 403 static gboolean
 404 dpb_bump(GstVaapiDecoderH264 *decoder)
 405 {
 406     GstVaapiDecoderH264Private * const priv = decoder->priv;
 407     guint i, lowest_poc_index;
 408     gboolean success;
 409
 410     for (i = 0; i < priv->dpb_count; i++) {
 411         if (priv->dpb[i]->output_needed)
 412             break;
 413     }
 414     if (i == priv->dpb_count)
 415         return FALSE;
 416
 417     lowest_poc_index = i++;
 418     for (; i < priv->dpb_count; i++) {
 419         GstVaapiPictureH264 * const picture = priv->dpb[i];
 420         if (picture->output_needed && picture->poc < priv->dpb[lowest_poc_index]->poc)
 421             lowest_poc_index = i;
 422     }
 423
 424     success = dpb_output(decoder, priv->dpb[lowest_poc_index]);
 425     if (!GST_VAAPI_PICTURE_IS_REFERENCE(priv->dpb[lowest_poc_index]))
 426         dpb_remove_index(decoder, lowest_poc_index);
 427     return success;
 428 }
 429
 430 static void
 431 dpb_flush(GstVaapiDecoderH264 *decoder)
 432 {
 433     GstVaapiDecoderH264Private * const priv = decoder->priv;
 434
 435     while (dpb_bump(decoder))
 436         ;
 437     clear_references(decoder, priv->dpb, &priv->dpb_count);
 438 }
 439
 440 static gboolean
 441 dpb_add(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
 442 {
 443     GstVaapiDecoderH264Private * const priv = decoder->priv;
 444     guint i;
 445
 446     // Remove all unused pictures
 447     if (picture->is_idr)
 448         dpb_flush(decoder);
 449     else {
 450         i = 0;
 451         while (i < priv->dpb_count) {
 452             GstVaapiPictureH264 * const picture = priv->dpb[i];
 453             if (!picture->output_needed &&
 454                 !GST_VAAPI_PICTURE_IS_REFERENCE(picture))
 455                 dpb_remove_index(decoder, i);
 456             else
 457                 i++;
 458         }
 459     }
 460
 461     // C.4.5.1 - Storage and marking of a reference decoded picture into the DPB
 462     if (GST_VAAPI_PICTURE_IS_REFERENCE(picture)) {
 463         while (priv->dpb_count == priv->dpb_size) {
 464             if (!dpb_bump(decoder))
 465                 return FALSE;
 466         }
 467         gst_vaapi_picture_replace(&priv->dpb[priv->dpb_count++], picture);
 468         if (picture->output_flag)
 469             picture->output_needed = TRUE;
 470     }
 471
 472     // C.4.5.2 - Storage and marking of a non-reference decoded picture into the DPB
 473     else {
 474         if (!picture->output_flag)
 475             return TRUE;
 476         while (priv->dpb_count == priv->dpb_size) {
 477             for (i = 0; i < priv->dpb_count; i++) {
 478                 if (priv->dpb[i]->output_needed &&
 479                     priv->dpb[i]->poc < picture->poc)
 480                     break;
 481             }
 482             if (i == priv->dpb_count)
 483                 return dpb_output(decoder, picture);
 484             if (!dpb_bump(decoder))
 485                 return FALSE;
 486         }
 487         gst_vaapi_picture_replace(&priv->dpb[priv->dpb_count++], picture);
 488         picture->output_needed = TRUE;
 489     }
 490     return TRUE;
 491 }
 492
 493 static inline void
 494 dpb_reset(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 495 {
 496     GstVaapiDecoderH264Private * const priv = decoder->priv;
 497
 498     priv->dpb_size = get_max_dec_frame_buffering(sps);
 499     GST_DEBUG("DPB size %u", priv->dpb_size);
 500 }
 501
 502 static GstVaapiDecoderStatus
 503 get_status(GstH264ParserResult result)
 504 {
 505     GstVaapiDecoderStatus status;
 506
 507     switch (result) {
 508     case GST_H264_PARSER_OK:
 509         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
 510         break;
 511     case GST_H264_PARSER_NO_NAL_END:
 512         status = GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
 513         break;
 514     case GST_H264_PARSER_ERROR:
 515         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
 516         break;
 517     default:
 518         status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 519         break;
 520     }
 521     return status;
 522 }
 523
 524 static void
 525 gst_vaapi_decoder_h264_close(GstVaapiDecoderH264 *decoder)
 526 {
 527     GstVaapiDecoderH264Private * const priv = decoder->priv;
 528
 529     gst_vaapi_picture_replace(&priv->current_picture, NULL);
 530     clear_references(decoder, priv->short_ref, &priv->short_ref_count);
 531     clear_references(decoder, priv->long_ref,  &priv->long_ref_count );
 532     clear_references(decoder, priv->dpb,       &priv->dpb_count      );
 533
 534     if (priv->parser) {
 535         gst_h264_nal_parser_free(priv->parser);
 536         priv->parser = NULL;
 537     }
 538
 539     if (priv->adapter) {
 540         gst_adapter_clear(priv->adapter);
 541         g_object_unref(priv->adapter);
 542         priv->adapter = NULL;
 543     }
 544 }
 545
 546 static gboolean
 547 gst_vaapi_decoder_h264_open(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
 548 {
 549     GstVaapiDecoderH264Private * const priv = decoder->priv;
 550
 551     gst_vaapi_decoder_h264_close(decoder);
 552
 553     priv->adapter = gst_adapter_new();
 554     if (!priv->adapter)
 555         return FALSE;
 556
 557     priv->parser = gst_h264_nal_parser_new();
 558     if (!priv->parser)
 559         return FALSE;
 560     return TRUE;
 561 }
 562
 563 static void
 564 gst_vaapi_decoder_h264_destroy(GstVaapiDecoderH264 *decoder)
 565 {
 566     gst_vaapi_decoder_h264_close(decoder);
 567 }
 568
 569 static gboolean
 570 gst_vaapi_decoder_h264_create(GstVaapiDecoderH264 *decoder)
 571 {
 572     if (!GST_VAAPI_DECODER_CODEC(decoder))
 573         return FALSE;
 574     return TRUE;
 575 }
 576
 577 static guint
 578 h264_get_profile(GstH264SPS *sps)
 579 {
 580     guint profile = 0;
 581
 582     switch (sps->profile_idc) {
 583     case 66:
 584         profile = GST_VAAPI_PROFILE_H264_BASELINE;
 585         break;
 586     case 77:
 587         profile = GST_VAAPI_PROFILE_H264_MAIN;
 588         break;
 589     case 100:
 590         profile = GST_VAAPI_PROFILE_H264_HIGH;
 591         break;
 592     }
 593     return profile;
 594 }
 595
 596 static guint
 597 h264_get_chroma_type(GstH264SPS *sps)
 598 {
 599     guint chroma_type = 0;
 600
 601     switch (sps->chroma_format_idc) {
 602     case 1:
 603         chroma_type = GST_VAAPI_CHROMA_TYPE_YUV420;
 604         break;
 605     case 2:
 606         chroma_type = GST_VAAPI_CHROMA_TYPE_YUV422;
 607         break;
 608     case 3:
 609         if (!sps->separate_colour_plane_flag)
 610             chroma_type = GST_VAAPI_CHROMA_TYPE_YUV444;
 611         break;
 612     }
 613     return chroma_type;
 614 }
 615
 616 static GstVaapiProfile
 617 get_profile(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 618 {
 619     GstVaapiDecoderH264Private * const priv = decoder->priv;
 620     GstVaapiDisplay * const display = GST_VAAPI_DECODER_DISPLAY(decoder);
 621     GstVaapiProfile profile, profiles[2];
 622     guint i, n_profiles = 0;
 623
 624     profile = h264_get_profile(sps);
 625     if (!profile)
 626         return GST_VAAPI_PROFILE_UNKNOWN;
 627
 628     profiles[n_profiles++] = profile;
 629     switch (profile) {
 630     case GST_VAAPI_PROFILE_H264_MAIN:
 631         profiles[n_profiles++] = GST_VAAPI_PROFILE_H264_HIGH;
 632         break;
 633     default:
 634         break;
 635     }
 636
 637     /* If the preferred profile (profiles[0]) matches one that we already
 638        found, then just return it now instead of searching for it again */
 639     if (profiles[0] == priv->profile)
 640         return priv->profile;
 641
 642     for (i = 0; i < n_profiles; i++) {
 643         if (gst_vaapi_display_has_decoder(display, profiles[i], priv->entrypoint))
 644             return profiles[i];
 645     }
 646     return GST_VAAPI_PROFILE_UNKNOWN;
 647 }
 648
 649 static GstVaapiDecoderStatus
 650 ensure_context(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
 651 {
 652     GstVaapiDecoder * const base_decoder = GST_VAAPI_DECODER_CAST(decoder);
 653     GstVaapiDecoderH264Private * const priv = decoder->priv;
 654     GstVaapiContextInfo info;
 655     GstVaapiProfile profile;
 656     GstVaapiChromaType chroma_type;
 657     gboolean reset_context = FALSE;
 658
 659     profile = get_profile(decoder, sps);
 660     if (!profile) {
 661         GST_ERROR("unsupported profile_idc %u", sps->profile_idc);
 662         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_PROFILE;
 663     }
 664
 665     if (priv->profile != profile) {
 666         GST_DEBUG("profile changed");
 667         reset_context = TRUE;
 668         priv->profile = profile;
 669     }
 670
 671     chroma_type = h264_get_chroma_type(sps);
 672     if (!chroma_type || chroma_type != GST_VAAPI_CHROMA_TYPE_YUV420) {
 673         GST_ERROR("unsupported chroma_format_idc %u", sps->chroma_format_idc);
 674         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;
 675     }
 676
 677     if (priv->chroma_type != chroma_type) {
 678         GST_DEBUG("chroma format changed");
 679         reset_context     = TRUE;
 680         priv->chroma_type = chroma_type;
 681     }
 682
 683     if (priv->width != sps->width || priv->height != sps->height) {
 684         GST_DEBUG("size changed");
 685         reset_context = TRUE;
 686         priv->width   = sps->width;
 687         priv->height  = sps->height;
 688     }
 689
 690     gst_vaapi_decoder_set_pixel_aspect_ratio(
 691         base_decoder,
 692         sps->vui_parameters.par_n,
 693         sps->vui_parameters.par_d
 694     );
 695
 696     if (!reset_context && priv->has_context)
 697         return GST_VAAPI_DECODER_STATUS_SUCCESS;
 698
 699     info.profile    = priv->profile;
 700     info.entrypoint = priv->entrypoint;
 701     info.width      = priv->width;
 702     info.height     = priv->height;
 703     info.ref_frames = get_max_dec_frame_buffering(sps);
 704
 705     if (!gst_vaapi_decoder_ensure_context(GST_VAAPI_DECODER(decoder), &info))
 706         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 707     priv->has_context = TRUE;
 708
 709     /* Reset DPB */
 710     dpb_reset(decoder, sps);
 711     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 712 }
 713
 714 static void
 715 fill_iq_matrix_4x4(VAIQMatrixBufferH264 *iq_matrix, const GstH264PPS *pps)
 716 {
 717     const guint8 (* const ScalingList4x4)[6][16] = &pps->scaling_lists_4x4;
 718     guint i, j;
 719
 720     /* There are always 6 4x4 scaling lists */
 721     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList4x4) == 6);
 722     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList4x4[0]) == 16);
 723
 724     if (sizeof(iq_matrix->ScalingList4x4[0][0]) == 1)
 725         memcpy(iq_matrix->ScalingList4x4, *ScalingList4x4,
 726                sizeof(iq_matrix->ScalingList4x4));
 727     else {
 728         for (i = 0; i < G_N_ELEMENTS(iq_matrix->ScalingList4x4); i++) {
 729             for (j = 0; j < G_N_ELEMENTS(iq_matrix->ScalingList4x4[i]); j++)
 730                 iq_matrix->ScalingList4x4[i][j] = (*ScalingList4x4)[i][j];
 731         }
 732     }
 733 }
 734
 735 static void
 736 fill_iq_matrix_8x8(VAIQMatrixBufferH264 *iq_matrix, const GstH264PPS *pps)
 737 {
 738     const guint8 (* const ScalingList8x8)[6][64] = &pps->scaling_lists_8x8;
 739     const GstH264SPS * const sps = pps->sequence;
 740     guint i, j, n;
 741
 742     /* If chroma_format_idc != 3, there are up to 2 8x8 scaling lists */
 743     if (!pps->transform_8x8_mode_flag)
 744         return;
 745
 746     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList8x8) >= 2);
 747     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList8x8[0]) == 64);
 748
 749     if (sizeof(iq_matrix->ScalingList8x8[0][0]) == 1)
 750         memcpy(iq_matrix->ScalingList8x8, *ScalingList8x8,
 751                sizeof(iq_matrix->ScalingList8x8));
 752     else {
 753         n = (sps->chroma_format_idc != 3) ? 2 : 6;
 754         for (i = 0; i < n; i++) {
 755             for (j = 0; j < G_N_ELEMENTS(iq_matrix->ScalingList8x8[i]); j++)
 756                 iq_matrix->ScalingList8x8[i][j] = (*ScalingList8x8)[i][j];
 757         }
 758     }
 759 }
 760
 761 static GstVaapiDecoderStatus
 762 ensure_quant_matrix(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
 763 {
 764     GstVaapiPicture * const base_picture = &picture->base;
 765     GstH264PPS * const pps = picture->pps;
 766     GstH264SPS * const sps = pps->sequence;
 767     VAIQMatrixBufferH264 *iq_matrix;
 768
 769     base_picture->iq_matrix = GST_VAAPI_IQ_MATRIX_NEW(H264, decoder);
 770     if (!base_picture->iq_matrix) {
 771         GST_ERROR("failed to allocate IQ matrix");
 772         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
 773     }
 774     iq_matrix = base_picture->iq_matrix->param;
 775
 776     /* XXX: we can only support 4:2:0 or 4:2:2 since ScalingLists8x8[]
 777        is not large enough to hold lists for 4:4:4 */
 778     if (sps->chroma_format_idc == 3)
 779         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;
 780
 781     fill_iq_matrix_4x4(iq_matrix, pps);
 782     fill_iq_matrix_8x8(iq_matrix, pps);
 783
 784     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 785 }
 786
 787 static GstVaapiDecoderStatus
 788 decode_current_picture(GstVaapiDecoderH264 *decoder)
 789 {
 790     GstVaapiDecoderH264Private * const priv = decoder->priv;
 791     GstVaapiPictureH264 * const picture = priv->current_picture;
 792     GstVaapiDecoderStatus status;
 793
 794     if (!picture)
 795         return GST_VAAPI_DECODER_STATUS_SUCCESS;
 796
 797     status = ensure_context(decoder, picture->pps->sequence);
 798     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
 799         return status;
 800
 801     if (!decode_picture_end(decoder, picture))
 802         goto error;
 803     if (!gst_vaapi_picture_decode(GST_VAAPI_PICTURE_CAST(picture)))
 804         goto error;
 805     gst_vaapi_picture_replace(&priv->current_picture, NULL);
 806     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 807
 808 error:
 809     gst_vaapi_picture_replace(&priv->current_picture, NULL);
 810     return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
 811 }
 812
 813 static GstVaapiDecoderStatus
 814 decode_sps(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
 815 {
 816     GstVaapiDecoderH264Private * const priv = decoder->priv;
 817     GstH264SPS * const sps = &priv->last_sps;
 818     GstH264ParserResult result;
 819
 820     GST_DEBUG("decode SPS");
 821
 822     memset(sps, 0, sizeof(*sps));
 823     result = gst_h264_parser_parse_sps(priv->parser, nalu, sps, TRUE);
 824     if (result != GST_H264_PARSER_OK)
 825         return get_status(result);
 826
 827     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 828 }
 829
 830 static GstVaapiDecoderStatus
 831 decode_pps(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
 832 {
 833     GstVaapiDecoderH264Private * const priv = decoder->priv;
 834     GstH264PPS * const pps = &priv->last_pps;
 835     GstH264ParserResult result;
 836
 837     GST_DEBUG("decode PPS");
 838
 839     memset(pps, 0, sizeof(*pps));
 840     result = gst_h264_parser_parse_pps(priv->parser, nalu, pps);
 841     if (result != GST_H264_PARSER_OK)
 842         return get_status(result);
 843
 844     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 845 }
 846
 847 static GstVaapiDecoderStatus
 848 decode_sei(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
 849 {
 850     GstVaapiDecoderH264Private * const priv = decoder->priv;
 851     GstH264SEIMessage sei;
 852     GstH264ParserResult result;
 853
 854     GST_DEBUG("decode SEI");
 855
 856     memset(&sei, 0, sizeof(sei));
 857     result = gst_h264_parser_parse_sei(priv->parser, nalu, &sei);
 858     if (result != GST_H264_PARSER_OK) {
 859         GST_WARNING("failed to decode SEI, payload type:%d", sei.payloadType);
 860         return get_status(result);
 861     }
 862
 863     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 864 }
 865
 866 static GstVaapiDecoderStatus
 867 decode_sequence_end(GstVaapiDecoderH264 *decoder)
 868 {
 869     GstVaapiDecoderStatus status;
 870
 871     GST_DEBUG("decode sequence-end");
 872
 873     status = decode_current_picture(decoder);
 874     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
 875         return status;
 876
 877     dpb_flush(decoder);
 878     return GST_VAAPI_DECODER_STATUS_END_OF_STREAM;
 879 }
 880
 881 /* 8.2.1.1 - Decoding process for picture order count type 0 */
 882 static void
 883 init_picture_poc_0(
 884     GstVaapiDecoderH264 *decoder,
 885     GstVaapiPictureH264 *picture,
 886     GstH264SliceHdr     *slice_hdr
 887 )
 888 {
 889     GstVaapiDecoderH264Private * const priv = decoder->priv;
 890     GstH264PPS * const pps = slice_hdr->pps;
 891     GstH264SPS * const sps = pps->sequence;
 892     const gint32 MaxPicOrderCntLsb = 1 << (sps->log2_max_pic_order_cnt_lsb_minus4 + 4);
 893
 894     GST_DEBUG("decode picture order count type 0");
 895
 896     if (picture->is_idr) {
 897         priv->prev_poc_msb = 0;
 898         priv->prev_poc_lsb = 0;
 899     }
 900     else if (priv->prev_pic_has_mmco5) {
 901         priv->prev_poc_msb = 0;
 902         priv->prev_poc_lsb = priv->prev_pic_bottom_field ? 0 :
 903             priv->field_poc[TOP_FIELD];
 904     }
 905     else {
 906         priv->prev_poc_msb = priv->poc_msb;
 907         priv->prev_poc_lsb = priv->poc_lsb;
 908     }
 909
 910     // (8-3)
 911     priv->poc_lsb = slice_hdr->pic_order_cnt_lsb;
 912     if (priv->poc_lsb < priv->prev_poc_lsb &&
 913         (priv->prev_poc_lsb - priv->poc_lsb) >= (MaxPicOrderCntLsb / 2))
 914         priv->poc_msb = priv->prev_poc_msb + MaxPicOrderCntLsb;
 915     else if (priv->poc_lsb > priv->prev_poc_lsb &&
 916              (priv->poc_lsb - priv->prev_poc_lsb) > (MaxPicOrderCntLsb / 2))
 917         priv->poc_msb = priv->prev_poc_msb - MaxPicOrderCntLsb;
 918     else
 919         priv->poc_msb = priv->prev_poc_msb;
 920
 921     // (8-4)
 922     if (!slice_hdr->field_pic_flag || !slice_hdr->bottom_field_flag)
 923         priv->field_poc[TOP_FIELD] = priv->poc_msb + priv->poc_lsb;
 924
 925     // (8-5)
 926     if (!slice_hdr->field_pic_flag)
 927         priv->field_poc[BOTTOM_FIELD] = priv->field_poc[TOP_FIELD] +
 928             slice_hdr->delta_pic_order_cnt_bottom;
 929     else if (slice_hdr->bottom_field_flag)
 930         priv->field_poc[BOTTOM_FIELD] = priv->poc_msb + priv->poc_lsb;
 931 }
 932
 933 /* 8.2.1.2 - Decoding process for picture order count type 1 */
 934 static void
 935 init_picture_poc_1(
 936     GstVaapiDecoderH264 *decoder,
 937     GstVaapiPictureH264 *picture,
 938     GstH264SliceHdr     *slice_hdr
 939 )
 940 {
 941     GstVaapiDecoderH264Private * const priv = decoder->priv;
 942     GstH264PPS * const pps = slice_hdr->pps;
 943     GstH264SPS * const sps = pps->sequence;
 944     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
 945     gint32 prev_frame_num_offset, abs_frame_num, expected_poc;
 946     guint i;
 947
 948     GST_DEBUG("decode picture order count type 1");
 949
 950     if (priv->prev_pic_has_mmco5)
 951         prev_frame_num_offset = 0;
 952     else
 953         prev_frame_num_offset = priv->frame_num_offset;
 954
 955     // (8-6)
 956     if (picture->is_idr)
 957         priv->frame_num_offset = 0;
 958     else if (priv->prev_frame_num > priv->frame_num)
 959         priv->frame_num_offset = prev_frame_num_offset + MaxFrameNum;
 960     else
 961         priv->frame_num_offset = prev_frame_num_offset;
 962
 963     // (8-7)
 964     if (sps->num_ref_frames_in_pic_order_cnt_cycle != 0)
 965         abs_frame_num = priv->frame_num_offset + priv->frame_num;
 966     else
 967         abs_frame_num = 0;
 968     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture) && abs_frame_num > 0)
 969         abs_frame_num = abs_frame_num - 1;
 970
 971     if (abs_frame_num > 0) {
 972         gint32 expected_delta_per_poc_cycle;
 973         gint32 poc_cycle_cnt, frame_num_in_poc_cycle;
 974
 975         expected_delta_per_poc_cycle = 0;
 976         for (i = 0; i < sps->num_ref_frames_in_pic_order_cnt_cycle; i++)
 977             expected_delta_per_poc_cycle += sps->offset_for_ref_frame[i];
 978
 979         // (8-8)
 980         poc_cycle_cnt = (abs_frame_num - 1) /
 981             sps->num_ref_frames_in_pic_order_cnt_cycle;
 982         frame_num_in_poc_cycle = (abs_frame_num - 1) %
 983             sps->num_ref_frames_in_pic_order_cnt_cycle;
 984
 985         // (8-9)
 986         expected_poc = poc_cycle_cnt * expected_delta_per_poc_cycle;
 987         for (i = 0; i <= frame_num_in_poc_cycle; i++)
 988             expected_poc += sps->offset_for_ref_frame[i];
 989     }
 990     else
 991         expected_poc = 0;
 992     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
 993         expected_poc += sps->offset_for_non_ref_pic;
 994
 995     // (8-10)
 996     if (!slice_hdr->field_pic_flag) {
 997         priv->field_poc[TOP_FIELD] = expected_poc +
 998             slice_hdr->delta_pic_order_cnt[0];
 999         priv->field_poc[BOTTOM_FIELD] = priv->field_poc[TOP_FIELD] +
1000             sps->offset_for_top_to_bottom_field +
1001             slice_hdr->delta_pic_order_cnt[1];
1002     }
1003     else if (!slice_hdr->bottom_field_flag)
1004         priv->field_poc[TOP_FIELD] = expected_poc +
1005             slice_hdr->delta_pic_order_cnt[0];
1006     else
1007         priv->field_poc[BOTTOM_FIELD] = expected_poc +
1008             sps->offset_for_top_to_bottom_field + slice_hdr->delta_pic_order_cnt[0];
1009 }
1010
1011 /* 8.2.1.3 - Decoding process for picture order count type 2 */
1012 static void
1013 init_picture_poc_2(
1014     GstVaapiDecoderH264 *decoder,
1015     GstVaapiPictureH264 *picture,
1016     GstH264SliceHdr     *slice_hdr
1017 )
1018 {
1019     GstVaapiDecoderH264Private * const priv = decoder->priv;
1020     GstH264PPS * const pps = slice_hdr->pps;
1021     GstH264SPS * const sps = pps->sequence;
1022     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1023     gint32 prev_frame_num_offset, temp_poc;
1024
1025     GST_DEBUG("decode picture order count type 2");
1026
1027     if (priv->prev_pic_has_mmco5)
1028         prev_frame_num_offset = 0;
1029     else
1030         prev_frame_num_offset = priv->frame_num_offset;
1031
1032     // (8-11)
1033     if (picture->is_idr)
1034         priv->frame_num_offset = 0;
1035     else if (priv->prev_frame_num > priv->frame_num)
1036         priv->frame_num_offset = prev_frame_num_offset + MaxFrameNum;
1037     else
1038         priv->frame_num_offset = prev_frame_num_offset;
1039
1040     // (8-12)
1041     if (picture->is_idr)
1042         temp_poc = 0;
1043     else if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
1044         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num) - 1;
1045     else
1046         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num);
1047
1048     // (8-13)
1049     if (!slice_hdr->field_pic_flag) {
1050         priv->field_poc[TOP_FIELD] = temp_poc;
1051         priv->field_poc[BOTTOM_FIELD] = temp_poc;
1052     }
1053     else if (slice_hdr->bottom_field_flag)
1054         priv->field_poc[BOTTOM_FIELD] = temp_poc;
1055     else
1056         priv->field_poc[TOP_FIELD] = temp_poc;
1057 }
1058
1059 /* 8.2.1 - Decoding process for picture order count */
1060 static void
1061 init_picture_poc(
1062     GstVaapiDecoderH264 *decoder,
1063     GstVaapiPictureH264 *picture,
1064     GstH264SliceHdr     *slice_hdr
1065 )
1066 {
1067     GstVaapiDecoderH264Private * const priv = decoder->priv;
1068     VAPictureH264 * const pic = &picture->info;
1069     GstH264PPS * const pps = slice_hdr->pps;
1070     GstH264SPS * const sps = pps->sequence;
1071
1072     switch (sps->pic_order_cnt_type) {
1073     case 0:
1074         init_picture_poc_0(decoder, picture, slice_hdr);
1075         break;
1076     case 1:
1077         init_picture_poc_1(decoder, picture, slice_hdr);
1078         break;
1079     case 2:
1080         init_picture_poc_2(decoder, picture, slice_hdr);
1081         break;
1082     }
1083
1084     if (!(pic->flags & VA_PICTURE_H264_BOTTOM_FIELD))
1085         pic->TopFieldOrderCnt = priv->field_poc[TOP_FIELD];
1086     if (!(pic->flags & VA_PICTURE_H264_TOP_FIELD))
1087         pic->BottomFieldOrderCnt = priv->field_poc[BOTTOM_FIELD];
1088     picture->poc = MIN(pic->TopFieldOrderCnt, pic->BottomFieldOrderCnt);
1089 }
1090
1091 static int
1092 compare_picture_pic_num_dec(const void *a, const void *b)
1093 {
1094     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1095     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1096
1097     return picB->pic_num - picA->pic_num;
1098 }
1099
1100 static int
1101 compare_picture_long_term_pic_num_inc(const void *a, const void *b)
1102 {
1103     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1104     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1105
1106     return picA->long_term_pic_num - picB->long_term_pic_num;
1107 }
1108
1109 static int
1110 compare_picture_poc_dec(const void *a, const void *b)
1111 {
1112     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1113     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1114
1115     return picB->poc - picA->poc;
1116 }
1117
1118 static int
1119 compare_picture_poc_inc(const void *a, const void *b)
1120 {
1121     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1122     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1123
1124     return picA->poc - picB->poc;
1125 }
1126
1127 static int
1128 compare_picture_frame_num_wrap_dec(const void *a, const void *b)
1129 {
1130     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1131     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1132
1133     return picB->frame_num_wrap - picA->frame_num_wrap;
1134 }
1135
1136 static int
1137 compare_picture_long_term_frame_idx_inc(const void *a, const void *b)
1138 {
1139     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1140     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1141
1142     return picA->info.frame_idx - picB->info.frame_idx;
1143 }
1144
1145 /* 8.2.4.1 - Decoding process for picture numbers */
1146 static void
1147 init_picture_refs_pic_num(
1148     GstVaapiDecoderH264 *decoder,
1149     GstVaapiPictureH264 *picture,
1150     GstH264SliceHdr     *slice_hdr
1151 )
1152 {
1153     GstVaapiDecoderH264Private * const priv = decoder->priv;
1154     GstH264PPS * const pps = slice_hdr->pps;
1155     GstH264SPS * const sps = pps->sequence;
1156     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1157     const guint field_flags = VA_PICTURE_H264_TOP_FIELD | VA_PICTURE_H264_BOTTOM_FIELD;
1158     guint i;
1159
1160     GST_DEBUG("decode picture numbers");
1161
1162     for (i = 0; i < priv->short_ref_count; i++) {
1163         GstVaapiPictureH264 * const pic = priv->short_ref[i];
1164
1165         // (8-27)
1166         if (pic->frame_num > priv->frame_num)
1167             pic->frame_num_wrap = pic->frame_num - MaxFrameNum;
1168         else
1169             pic->frame_num_wrap = pic->frame_num;
1170
1171         // (8-28, 8-30, 8-31)
1172         if (!pic->field_pic_flag)
1173             pic->pic_num = pic->frame_num_wrap;
1174         else {
1175             if (((picture->info.flags ^ pic->info.flags) & field_flags) == 0)
1176                 pic->pic_num = 2 * pic->frame_num_wrap + 1;
1177             else
1178                 pic->pic_num = 2 * pic->frame_num_wrap;
1179         }
1180     }
1181
1182     for (i = 0; i < priv->long_ref_count; i++) {
1183         GstVaapiPictureH264 * const pic = priv->long_ref[i];
1184
1185         // (8-29, 8-32, 8-33)
1186         if (!pic->field_pic_flag)
1187             pic->long_term_pic_num = pic->info.frame_idx;
1188         else {
1189             if (((picture->info.flags ^ pic->info.flags) & field_flags) == 0)
1190                 pic->long_term_pic_num = 2 * pic->info.frame_idx + 1;
1191             else
1192                 pic->long_term_pic_num = 2 * pic->info.frame_idx;
1193         }
1194     }
1195 }
1196
1197 #define SORT_REF_LIST(list, n, compare_func) \
1198     qsort(list, n, sizeof(*(list)), compare_picture_##compare_func)
1199
1200 static void
1201 init_picture_refs_p_slice(
1202     GstVaapiDecoderH264 *decoder,
1203     GstVaapiPictureH264 *picture,
1204     GstH264SliceHdr     *slice_hdr
1205 )
1206 {
1207     GstVaapiDecoderH264Private * const priv = decoder->priv;
1208     GstVaapiPictureH264 **ref_list;
1209     guint i;
1210
1211     GST_DEBUG("decode reference picture list for P and SP slices");
1212
1213     if (!picture->field_pic_flag) {
1214         /* 8.2.4.2.1 - P and SP slices in frames */
1215         if (priv->short_ref_count > 0) {
1216             ref_list = priv->RefPicList0;
1217             for (i = 0; i < priv->short_ref_count; i++)
1218                 ref_list[i] = priv->short_ref[i];
1219             SORT_REF_LIST(ref_list, i, pic_num_dec);
1220             priv->RefPicList0_count += i;
1221         }
1222
1223         if (priv->long_ref_count > 0) {
1224             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1225             for (i = 0; i < priv->long_ref_count; i++)
1226                 ref_list[i] = priv->long_ref[i];
1227             SORT_REF_LIST(ref_list, i, long_term_pic_num_inc);
1228             priv->RefPicList0_count += i;
1229         }
1230     }
1231     else {
1232         /* 8.2.4.2.2 - P and SP slices in fields */
1233         GstVaapiPictureH264 *short_ref[32];
1234         guint short_ref_count = 0;
1235         GstVaapiPictureH264 *long_ref[32];
1236         guint long_ref_count = 0;
1237
1238         // XXX: handle second field if current field is marked as
1239         // "used for short-term reference"
1240         if (priv->short_ref_count > 0) {
1241             for (i = 0; i < priv->short_ref_count; i++)
1242                 short_ref[i] = priv->short_ref[i];
1243             SORT_REF_LIST(short_ref, i, frame_num_wrap_dec);
1244             short_ref_count = i;
1245         }
1246
1247         // XXX: handle second field if current field is marked as
1248         // "used for long-term reference"
1249         if (priv->long_ref_count > 0) {
1250             for (i = 0; i < priv->long_ref_count; i++)
1251                 long_ref[i] = priv->long_ref[i];
1252             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1253             long_ref_count = i;
1254         }
1255
1256         // XXX: handle 8.2.4.2.5
1257     }
1258 }
1259
1260 static void
1261 init_picture_refs_b_slice(
1262     GstVaapiDecoderH264 *decoder,
1263     GstVaapiPictureH264 *picture,
1264     GstH264SliceHdr     *slice_hdr
1265 )
1266 {
1267     GstVaapiDecoderH264Private * const priv = decoder->priv;
1268     GstVaapiPictureH264 **ref_list;
1269     guint i, n;
1270
1271     GST_DEBUG("decode reference picture list for B slices");
1272
1273     if (!picture->field_pic_flag) {
1274         /* 8.2.4.2.3 - B slices in frames */
1275
1276         /* RefPicList0 */
1277         if (priv->short_ref_count > 0) {
1278             // 1. Short-term references
1279             ref_list = priv->RefPicList0;
1280             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1281                 if (priv->short_ref[i]->poc < picture->poc)
1282                     ref_list[n++] = priv->short_ref[i];
1283             }
1284             SORT_REF_LIST(ref_list, n, poc_dec);
1285             priv->RefPicList0_count += n;
1286
1287             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1288             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1289                 if (priv->short_ref[i]->poc >= picture->poc)
1290                     ref_list[n++] = priv->short_ref[i];
1291             }
1292             SORT_REF_LIST(ref_list, n, poc_inc);
1293             priv->RefPicList0_count += n;
1294         }
1295
1296         if (priv->long_ref_count > 0) {
1297             // 2. Long-term references
1298             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1299             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1300                 ref_list[n++] = priv->long_ref[i];
1301             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1302             priv->RefPicList0_count += n;
1303         }
1304
1305         /* RefPicList1 */
1306         if (priv->short_ref_count > 0) {
1307             // 1. Short-term references
1308             ref_list = priv->RefPicList1;
1309             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1310                 if (priv->short_ref[i]->poc > picture->poc)
1311                     ref_list[n++] = priv->short_ref[i];
1312             }
1313             SORT_REF_LIST(ref_list, n, poc_inc);
1314             priv->RefPicList1_count += n;
1315
1316             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1317             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1318                 if (priv->short_ref[i]->poc <= picture->poc)
1319                     ref_list[n++] = priv->short_ref[i];
1320             }
1321             SORT_REF_LIST(ref_list, n, poc_dec);
1322             priv->RefPicList1_count += n;
1323         }
1324
1325         if (priv->long_ref_count > 0) {
1326             // 2. Long-term references
1327             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1328             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1329                 ref_list[n++] = priv->long_ref[i];
1330             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1331             priv->RefPicList1_count += n;
1332         }
1333     }
1334     else {
1335         /* 8.2.4.2.4 - B slices in fields */
1336         GstVaapiPictureH264 *short_ref0[32];
1337         guint short_ref0_count = 0;
1338         GstVaapiPictureH264 *short_ref1[32];
1339         guint short_ref1_count = 0;
1340         GstVaapiPictureH264 *long_ref[32];
1341         guint long_ref_count = 0;
1342
1343         /* refFrameList0ShortTerm */
1344         if (priv->short_ref_count > 0) {
1345             ref_list = short_ref0;
1346             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1347                 if (priv->short_ref[i]->poc <= picture->poc)
1348                     ref_list[n++] = priv->short_ref[i];
1349             }
1350             SORT_REF_LIST(ref_list, n, poc_dec);
1351             short_ref0_count += n;
1352
1353             ref_list = &short_ref0[short_ref0_count];
1354             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1355                 if (priv->short_ref[i]->poc > picture->poc)
1356                     ref_list[n++] = priv->short_ref[i];
1357             }
1358             SORT_REF_LIST(ref_list, n, poc_inc);
1359             short_ref0_count += n;
1360         }
1361
1362         /* refFrameList1ShortTerm */
1363         if (priv->short_ref_count > 0) {
1364             ref_list = short_ref1;
1365             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1366                 if (priv->short_ref[i]->poc > picture->poc)
1367                     ref_list[n++] = priv->short_ref[i];
1368             }
1369             SORT_REF_LIST(ref_list, n, poc_inc);
1370             short_ref1_count += n;
1371
1372             ref_list = &short_ref1[short_ref1_count];
1373             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1374                 if (priv->short_ref[i]->poc <= picture->poc)
1375                     ref_list[n++] = priv->short_ref[i];
1376             }
1377             SORT_REF_LIST(ref_list, n, poc_dec);
1378             short_ref1_count += n;
1379         }
1380
1381         /* refFrameListLongTerm */
1382         if (priv->long_ref_count > 0) {
1383             for (i = 0; i < priv->long_ref_count; i++)
1384                 long_ref[i] = priv->long_ref[i];
1385             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1386             long_ref_count = i;
1387         }
1388
1389         // XXX: handle 8.2.4.2.5
1390     }
1391
1392     /* Check whether RefPicList1 is identical to RefPicList0, then
1393        swap if necessary */
1394     if (priv->RefPicList1_count > 1 &&
1395         priv->RefPicList1_count == priv->RefPicList0_count &&
1396         memcmp(priv->RefPicList0, priv->RefPicList1,
1397                priv->RefPicList0_count * sizeof(priv->RefPicList0[0])) == 0) {
1398         GstVaapiPictureH264 * const tmp = priv->RefPicList1[0];
1399         priv->RefPicList1[0] = priv->RefPicList1[1];
1400         priv->RefPicList1[1] = tmp;
1401     }
1402 }
1403
1404 #undef SORT_REF_LIST
1405
1406 static void
1407 clear_references(
1408     GstVaapiDecoderH264  *decoder,
1409     GstVaapiPictureH264 **pictures,
1410     guint                *picture_count
1411 )
1412 {
1413     const guint num_pictures = *picture_count;
1414     guint i;
1415
1416     for (i = 0; i < num_pictures; i++)
1417         gst_vaapi_picture_replace(&pictures[i], NULL);
1418     *picture_count = 0;
1419 }
1420
1421 static gboolean
1422 remove_reference_at(
1423     GstVaapiDecoderH264  *decoder,
1424     GstVaapiPictureH264 **pictures,
1425     guint                *picture_count,
1426     guint                 index
1427 )
1428 {
1429     guint num_pictures = *picture_count;
1430     GstVaapiPictureH264 *picture;
1431
1432     g_return_val_if_fail(index < num_pictures, FALSE);
1433
1434     picture = pictures[index];
1435     GST_VAAPI_PICTURE_FLAG_UNSET(picture, GST_VAAPI_PICTURE_FLAG_REFERENCE);
1436     picture->is_long_term = FALSE;
1437     picture->info.flags &= ~(VA_PICTURE_H264_SHORT_TERM_REFERENCE |
1438                              VA_PICTURE_H264_LONG_TERM_REFERENCE);
1439
1440     if (index != --num_pictures)
1441         gst_vaapi_picture_replace(&pictures[index], pictures[num_pictures]);
1442     gst_vaapi_picture_replace(&pictures[num_pictures], NULL);
1443     *picture_count = num_pictures;
1444     return TRUE;
1445 }
1446
1447 static gint
1448 find_short_term_reference(GstVaapiDecoderH264 *decoder, gint32 pic_num)
1449 {
1450     GstVaapiDecoderH264Private * const priv = decoder->priv;
1451     guint i;
1452
1453     for (i = 0; i < priv->short_ref_count; i++) {
1454         if (priv->short_ref[i]->pic_num == pic_num)
1455             return i;
1456     }
1457     GST_ERROR("found no short-term reference picture with PicNum = %d",
1458               pic_num);
1459     return -1;
1460 }
1461
1462 static gint
1463 find_long_term_reference(GstVaapiDecoderH264 *decoder, gint32 long_term_pic_num)
1464 {
1465     GstVaapiDecoderH264Private * const priv = decoder->priv;
1466     guint i;
1467
1468     for (i = 0; i < priv->long_ref_count; i++) {
1469         if (priv->long_ref[i]->long_term_pic_num == long_term_pic_num)
1470             return i;
1471     }
1472     GST_ERROR("found no long-term reference picture with LongTermPicNum = %d",
1473               long_term_pic_num);
1474     return -1;
1475 }
1476
1477 static void
1478 exec_picture_refs_modification_1(
1479     GstVaapiDecoderH264           *decoder,
1480     GstVaapiPictureH264           *picture,
1481     GstH264SliceHdr               *slice_hdr,
1482     guint                          list
1483 )
1484 {
1485     GstVaapiDecoderH264Private * const priv = decoder->priv;
1486     GstH264PPS * const pps = slice_hdr->pps;
1487     GstH264SPS * const sps = pps->sequence;
1488     GstH264RefPicListModification *ref_pic_list_modification;
1489     guint num_ref_pic_list_modifications;
1490     GstVaapiPictureH264 **ref_list;
1491     guint *ref_list_count_ptr, ref_list_count, ref_list_idx = 0;
1492     guint i, j, n, num_refs;
1493     gint found_ref_idx;
1494     gint32 MaxPicNum, CurrPicNum, picNumPred;
1495
1496     GST_DEBUG("modification process of reference picture list %u", list);
1497
1498     if (list == 0) {
1499         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l0;
1500         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l0;
1501         ref_list                       = priv->RefPicList0;
1502         ref_list_count_ptr             = &priv->RefPicList0_count;
1503         num_refs                       = slice_hdr->num_ref_idx_l0_active_minus1 + 1;
1504     }
1505     else {
1506         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l1;
1507         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l1;
1508         ref_list                       = priv->RefPicList1;
1509         ref_list_count_ptr             = &priv->RefPicList1_count;
1510         num_refs                       = slice_hdr->num_ref_idx_l1_active_minus1 + 1;
1511     }
1512     ref_list_count = *ref_list_count_ptr;
1513
1514     if (picture->field_pic_flag) {
1515         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 5); // 2 * MaxFrameNum
1516         CurrPicNum = 2 * slice_hdr->frame_num + 1;              // 2 * frame_num + 1
1517     }
1518     else {
1519         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 4); // MaxFrameNum
1520         CurrPicNum = slice_hdr->frame_num;                      // frame_num
1521     }
1522
1523     picNumPred = CurrPicNum;
1524
1525     for (i = 0; i < num_ref_pic_list_modifications; i++) {
1526         GstH264RefPicListModification * const l = &ref_pic_list_modification[i];
1527         if (l->modification_of_pic_nums_idc == 3)
1528             break;
1529
1530         /* 8.2.4.3.1 - Short-term reference pictures */
1531         if (l->modification_of_pic_nums_idc == 0 || l->modification_of_pic_nums_idc == 1) {
1532             gint32 abs_diff_pic_num = l->value.abs_diff_pic_num_minus1 + 1;
1533             gint32 picNum, picNumNoWrap;
1534
1535             // (8-34)
1536             if (l->modification_of_pic_nums_idc == 0) {
1537                 picNumNoWrap = picNumPred - abs_diff_pic_num;
1538                 if (picNumNoWrap < 0)
1539                     picNumNoWrap += MaxPicNum;
1540             }
1541
1542             // (8-35)
1543             else {
1544                 picNumNoWrap = picNumPred + abs_diff_pic_num;
1545                 if (picNumNoWrap >= MaxPicNum)
1546                     picNumNoWrap -= MaxPicNum;
1547             }
1548             picNumPred = picNumNoWrap;
1549
1550             // (8-36)
1551             picNum = picNumNoWrap;
1552             if (picNum > CurrPicNum)
1553                 picNum -= MaxPicNum;
1554
1555             // (8-37)
1556             for (j = num_refs; j > ref_list_idx; j--)
1557                 ref_list[j] = ref_list[j - 1];
1558             found_ref_idx = find_short_term_reference(decoder, picNum);
1559             ref_list[ref_list_idx++] =
1560                 found_ref_idx >= 0 ? priv->short_ref[found_ref_idx] : NULL;
1561             n = ref_list_idx;
1562             for (j = ref_list_idx; j <= num_refs; j++) {
1563                 gint32 PicNumF;
1564                 if (!ref_list[j])
1565                     continue;
1566                 PicNumF = ref_list[j]->is_long_term ?
1567                     MaxPicNum : ref_list[j]->pic_num;
1568                 if (PicNumF != picNum)
1569                     ref_list[n++] = ref_list[j];
1570             }
1571         }
1572
1573         /* 8.2.4.3.2 - Long-term reference pictures */
1574         else {
1575
1576             for (j = num_refs; j > ref_list_idx; j--)
1577                 ref_list[j] = ref_list[j - 1];
1578             found_ref_idx =
1579                 find_long_term_reference(decoder, l->value.long_term_pic_num);
1580             ref_list[ref_list_idx++] =
1581                 found_ref_idx >= 0 ? priv->long_ref[found_ref_idx] : NULL;
1582             n = ref_list_idx;
1583             for (j = ref_list_idx; j <= num_refs; j++) {
1584                 gint32 LongTermPicNumF;
1585                 if (!ref_list[j])
1586                     continue;
1587                 LongTermPicNumF = ref_list[j]->is_long_term ?
1588                     ref_list[j]->long_term_pic_num : INT_MAX;
1589                 if (LongTermPicNumF != l->value.long_term_pic_num)
1590                     ref_list[n++] = ref_list[j];
1591             }
1592         }
1593     }
1594
1595 #if DEBUG
1596     for (i = 0; i < num_refs; i++)
1597         if (!ref_list[i])
1598             GST_ERROR("list %u entry %u is empty", list, i);
1599 #endif
1600     *ref_list_count_ptr = num_refs;
1601 }
1602
1603 /* 8.2.4.3 - Modification process for reference picture lists */
1604 static void
1605 exec_picture_refs_modification(
1606     GstVaapiDecoderH264 *decoder,
1607     GstVaapiPictureH264 *picture,
1608     GstH264SliceHdr     *slice_hdr
1609 )
1610 {
1611     GST_DEBUG("execute ref_pic_list_modification()");
1612
1613     /* RefPicList0 */
1614     if (!GST_H264_IS_I_SLICE(slice_hdr) && !GST_H264_IS_SI_SLICE(slice_hdr) &&
1615         slice_hdr->ref_pic_list_modification_flag_l0)
1616         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 0);
1617
1618     /* RefPicList1 */
1619     if (GST_H264_IS_B_SLICE(slice_hdr) &&
1620         slice_hdr->ref_pic_list_modification_flag_l1)
1621         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 1);
1622 }
1623
1624 static gboolean
1625 init_picture_refs(
1626     GstVaapiDecoderH264 *decoder,
1627     GstVaapiPictureH264 *picture,
1628     GstH264SliceHdr     *slice_hdr
1629 )
1630 {
1631     GstVaapiDecoderH264Private * const priv = decoder->priv;
1632     GstVaapiPicture * const base_picture = &picture->base;
1633     guint i, num_refs;
1634
1635     init_picture_refs_pic_num(decoder, picture, slice_hdr);
1636
1637     priv->RefPicList0_count = 0;
1638     priv->RefPicList1_count = 0;
1639
1640     switch (base_picture->type) {
1641     case GST_VAAPI_PICTURE_TYPE_P:
1642     case GST_VAAPI_PICTURE_TYPE_SP:
1643         init_picture_refs_p_slice(decoder, picture, slice_hdr);
1644         break;
1645     case GST_VAAPI_PICTURE_TYPE_B:
1646         init_picture_refs_b_slice(decoder, picture, slice_hdr);
1647         break;
1648     default:
1649         break;
1650     }
1651
1652     exec_picture_refs_modification(decoder, picture, slice_hdr);
1653
1654     switch (base_picture->type) {
1655     case GST_VAAPI_PICTURE_TYPE_B:
1656         num_refs = 1 + slice_hdr->num_ref_idx_l1_active_minus1;
1657         for (i = priv->RefPicList1_count; i < num_refs; i++)
1658             priv->RefPicList1[i] = NULL;
1659         priv->RefPicList1_count = num_refs;
1660
1661         // fall-through
1662     case GST_VAAPI_PICTURE_TYPE_P:
1663     case GST_VAAPI_PICTURE_TYPE_SP:
1664         num_refs = 1 + slice_hdr->num_ref_idx_l0_active_minus1;
1665         for (i = priv->RefPicList0_count; i < num_refs; i++)
1666             priv->RefPicList0[i] = NULL;
1667         priv->RefPicList0_count = num_refs;
1668         break;
1669     default:
1670         break;
1671     }
1672     return TRUE;
1673 }
1674
1675 static gboolean
1676 init_picture(
1677     GstVaapiDecoderH264 *decoder,
1678     GstVaapiPictureH264 *picture,
1679     GstH264SliceHdr     *slice_hdr,
1680     GstH264NalUnit      *nalu
1681 )
1682 {
1683     GstVaapiDecoderH264Private * const priv = decoder->priv;
1684     GstVaapiPicture * const base_picture = &picture->base;
1685     VAPictureH264 *pic;
1686
1687     priv->prev_frame_num        = priv->frame_num;
1688     priv->frame_num             = slice_hdr->frame_num;
1689     picture->frame_num          = priv->frame_num;
1690     picture->frame_num_wrap     = priv->frame_num;
1691     picture->is_idr             = nalu->type == GST_H264_NAL_SLICE_IDR;
1692     picture->field_pic_flag     = slice_hdr->field_pic_flag;
1693     picture->bottom_field_flag  = slice_hdr->bottom_field_flag;
1694     picture->output_flag        = TRUE; /* XXX: conformant to Annex A only */
1695     base_picture->pts           = gst_adapter_prev_timestamp(priv->adapter, NULL);
1696
1697     /* Reset decoder state for IDR pictures */
1698     if (picture->is_idr) {
1699         GST_DEBUG("<IDR>");
1700         clear_references(decoder, priv->short_ref, &priv->short_ref_count);
1701         clear_references(decoder, priv->long_ref,  &priv->long_ref_count );
1702     }
1703
1704     /* Initialize VA picture info */
1705     pic = &picture->info;
1706     pic->picture_id = picture->base.surface_id;
1707     pic->frame_idx  = priv->frame_num;
1708     if (picture->field_pic_flag) {
1709         if (picture->bottom_field_flag)
1710             pic->flags |= VA_PICTURE_H264_BOTTOM_FIELD;
1711         else
1712             pic->flags |= VA_PICTURE_H264_TOP_FIELD;
1713     }
1714
1715     /* Initialize base picture */
1716     switch (slice_hdr->type % 5) {
1717     case GST_H264_P_SLICE:
1718         base_picture->type = GST_VAAPI_PICTURE_TYPE_P;
1719         break;
1720     case GST_H264_B_SLICE:
1721         base_picture->type = GST_VAAPI_PICTURE_TYPE_B;
1722         break;
1723     case GST_H264_I_SLICE:
1724         base_picture->type = GST_VAAPI_PICTURE_TYPE_I;
1725         break;
1726     case GST_H264_SP_SLICE:
1727         base_picture->type = GST_VAAPI_PICTURE_TYPE_SP;
1728         break;
1729     case GST_H264_SI_SLICE:
1730         base_picture->type = GST_VAAPI_PICTURE_TYPE_SI;
1731         break;
1732     }
1733
1734     if (nalu->ref_idc) {
1735         GstH264DecRefPicMarking * const dec_ref_pic_marking =
1736             &slice_hdr->dec_ref_pic_marking;
1737         GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_REFERENCE);
1738         if (picture->is_idr) {
1739             if (dec_ref_pic_marking->long_term_reference_flag)
1740                 picture->is_long_term = TRUE;
1741         }
1742     }
1743
1744     init_picture_poc(decoder, picture, slice_hdr);
1745     if (!init_picture_refs(decoder, picture, slice_hdr)) {
1746         GST_ERROR("failed to initialize references");
1747         return FALSE;
1748     }
1749     return TRUE;
1750 }
1751
1752 /* 8.2.5.3 - Sliding window decoded reference picture marking process */
1753 static gboolean
1754 exec_ref_pic_marking_sliding_window(GstVaapiDecoderH264 *decoder)
1755 {
1756     GstVaapiDecoderH264Private * const priv = decoder->priv;
1757     GstH264PPS * const pps = priv->current_picture->pps;
1758     GstH264SPS * const sps = pps->sequence;
1759     guint i, max_num_ref_frames, lowest_frame_num_index;
1760     gint32 lowest_frame_num;
1761
1762     GST_DEBUG("reference picture marking process (sliding window)");
1763
1764     max_num_ref_frames = sps->num_ref_frames;
1765     if (max_num_ref_frames == 0)
1766         max_num_ref_frames = 1;
1767
1768     if (priv->short_ref_count + priv->long_ref_count < max_num_ref_frames)
1769         return TRUE;
1770     if (priv->short_ref_count < 1)
1771         return FALSE;
1772
1773     lowest_frame_num = priv->short_ref[0]->frame_num_wrap;
1774     lowest_frame_num_index = 0;
1775     for (i = 1; i < priv->short_ref_count; i++) {
1776         if (priv->short_ref[i]->frame_num_wrap < lowest_frame_num) {
1777             lowest_frame_num = priv->short_ref[i]->frame_num_wrap;
1778             lowest_frame_num_index = i;
1779         }
1780     }
1781
1782     remove_reference_at(
1783         decoder,
1784         priv->short_ref, &priv->short_ref_count,
1785         lowest_frame_num_index
1786     );
1787     return TRUE;
1788 }
1789
1790 static inline gint32
1791 get_picNumX(GstVaapiPictureH264 *picture, GstH264RefPicMarking *ref_pic_marking)
1792 {
1793     gint32 pic_num;
1794
1795     if (!picture->field_pic_flag)
1796         pic_num = picture->frame_num_wrap;
1797     else
1798         pic_num = 2 * picture->frame_num_wrap + 1;
1799     pic_num -= ref_pic_marking->difference_of_pic_nums_minus1 + 1;
1800     return pic_num;
1801 }
1802
1803 /* 8.2.5.4.1. Mark-term reference picture as "unused for reference" */
1804 static void
1805 exec_ref_pic_marking_adaptive_mmco_1(
1806     GstVaapiDecoderH264  *decoder,
1807     GstVaapiPictureH264  *picture,
1808     GstH264RefPicMarking *ref_pic_marking
1809 )
1810 {
1811     GstVaapiDecoderH264Private * const priv = decoder->priv;
1812     gint32 i, picNumX;
1813
1814     picNumX = get_picNumX(picture, ref_pic_marking);
1815     i = find_short_term_reference(decoder, picNumX);
1816     if (i < 0)
1817         return;
1818     remove_reference_at(decoder, priv->short_ref, &priv->short_ref_count, i);
1819 }
1820
1821 /* 8.2.5.4.2. Mark long-term reference picture as "unused for reference" */
1822 static void
1823 exec_ref_pic_marking_adaptive_mmco_2(
1824     GstVaapiDecoderH264  *decoder,
1825     GstVaapiPictureH264  *picture,
1826     GstH264RefPicMarking *ref_pic_marking
1827 )
1828 {
1829     GstVaapiDecoderH264Private * const priv = decoder->priv;
1830     gint32 i;
1831
1832     i = find_long_term_reference(decoder, ref_pic_marking->long_term_pic_num);
1833     if (i < 0)
1834         return;
1835     remove_reference_at(decoder, priv->long_ref, &priv->long_ref_count, i);
1836 }
1837
1838 /* 8.2.5.4.3. Assign LongTermFrameIdx to a short-term reference picture */
1839 static void
1840 exec_ref_pic_marking_adaptive_mmco_3(
1841     GstVaapiDecoderH264  *decoder,
1842     GstVaapiPictureH264  *picture,
1843     GstH264RefPicMarking *ref_pic_marking
1844 )
1845 {
1846     GstVaapiDecoderH264Private * const priv = decoder->priv;
1847     VAPictureH264 *pic;
1848     gint32 i, picNumX;
1849
1850     for (i = 0; i < priv->long_ref_count; i++) {
1851         if (priv->long_ref[i]->info.frame_idx == ref_pic_marking->long_term_frame_idx)
1852             break;
1853     }
1854     if (i != priv->long_ref_count)
1855         remove_reference_at(decoder, priv->long_ref, &priv->long_ref_count, i);
1856
1857     picNumX = get_picNumX(picture, ref_pic_marking);
1858     i = find_short_term_reference(decoder, picNumX);
1859     if (i < 0)
1860         return;
1861
1862     picture = gst_vaapi_picture_ref(priv->short_ref[i]);
1863     remove_reference_at(decoder, priv->short_ref, &priv->short_ref_count, i);
1864     gst_vaapi_picture_replace(&priv->long_ref[priv->long_ref_count++], picture);
1865     gst_vaapi_picture_unref(picture);
1866
1867     picture->is_long_term = TRUE;
1868     pic = &picture->info;
1869     pic->frame_idx = ref_pic_marking->long_term_frame_idx;
1870     pic->flags &= ~VA_PICTURE_H264_SHORT_TERM_REFERENCE;
1871     pic->flags |= VA_PICTURE_H264_LONG_TERM_REFERENCE;
1872     GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_REFERENCE);
1873 }
1874
1875 /* 8.2.5.4.4. Mark pictures with LongTermFramIdx > max_long_term_frame_idx
1876  * as "unused for reference" */
1877 static void
1878 exec_ref_pic_marking_adaptive_mmco_4(
1879     GstVaapiDecoderH264  *decoder,
1880     GstVaapiPictureH264  *picture,
1881     GstH264RefPicMarking *ref_pic_marking
1882 )
1883 {
1884     GstVaapiDecoderH264Private * const priv = decoder->priv;
1885     gint32 i, long_term_frame_idx;
1886
1887     long_term_frame_idx = ref_pic_marking->max_long_term_frame_idx_plus1 - 1;
1888
1889     for (i = 0; i < priv->long_ref_count; i++) {
1890         if ((gint32)priv->long_ref[i]->info.frame_idx <= long_term_frame_idx)
1891             continue;
1892         remove_reference_at(decoder, priv->long_ref, &priv->long_ref_count, i);
1893         i--;
1894     }
1895 }
1896
1897 /* 8.2.5.4.5. Mark all reference pictures as "unused for reference" */
1898 static void
1899 exec_ref_pic_marking_adaptive_mmco_5(
1900     GstVaapiDecoderH264  *decoder,
1901     GstVaapiPictureH264  *picture,
1902     GstH264RefPicMarking *ref_pic_marking
1903 )
1904 {
1905     GstVaapiDecoderH264Private * const priv = decoder->priv;
1906     VAPictureH264 * const pic = &picture->info;
1907
1908     clear_references(decoder, priv->short_ref, &priv->short_ref_count);
1909     clear_references(decoder, priv->long_ref,  &priv->long_ref_count );
1910     dpb_flush(decoder);
1911
1912     priv->prev_pic_has_mmco5 = TRUE;
1913
1914     /* The picture shall be inferred to have had frame_num equal to 0 (7.4.3) */
1915     priv->frame_num = 0;
1916     priv->frame_num_offset = 0;
1917     picture->frame_num = 0;
1918
1919     /* Update TopFieldOrderCnt and BottomFieldOrderCnt (8.2.1) */
1920     if (!(pic->flags & VA_PICTURE_H264_BOTTOM_FIELD))
1921         pic->TopFieldOrderCnt -= picture->poc;
1922     if (!(pic->flags & VA_PICTURE_H264_TOP_FIELD))
1923         pic->BottomFieldOrderCnt -= picture->poc;
1924     picture->poc = 0;
1925 }
1926
1927 /* 8.2.5.4.6. Assign a long-term frame index to the current picture */
1928 static void
1929 exec_ref_pic_marking_adaptive_mmco_6(
1930     GstVaapiDecoderH264  *decoder,
1931     GstVaapiPictureH264  *picture,
1932     GstH264RefPicMarking *ref_pic_marking
1933 )
1934 {
1935     picture->is_long_term = TRUE;
1936     picture->info.frame_idx = ref_pic_marking->long_term_frame_idx;
1937 }
1938
1939 /* 8.2.5.4. Adaptive memory control decoded reference picture marking process */
1940 static gboolean
1941 exec_ref_pic_marking_adaptive(
1942     GstVaapiDecoderH264     *decoder,
1943     GstVaapiPictureH264     *picture,
1944     GstH264DecRefPicMarking *dec_ref_pic_marking
1945 )
1946 {
1947     guint i;
1948
1949     GST_DEBUG("reference picture marking process (adaptive memory control)");
1950
1951     typedef void (*exec_ref_pic_marking_adaptive_mmco_func)(
1952         GstVaapiDecoderH264  *decoder,
1953         GstVaapiPictureH264  *picture,
1954         GstH264RefPicMarking *ref_pic_marking
1955     );
1956
1957     static const exec_ref_pic_marking_adaptive_mmco_func mmco_funcs[] = {
1958         NULL,
1959         exec_ref_pic_marking_adaptive_mmco_1,
1960         exec_ref_pic_marking_adaptive_mmco_2,
1961         exec_ref_pic_marking_adaptive_mmco_3,
1962         exec_ref_pic_marking_adaptive_mmco_4,
1963         exec_ref_pic_marking_adaptive_mmco_5,
1964         exec_ref_pic_marking_adaptive_mmco_6,
1965     };
1966
1967     for (i = 0; i < dec_ref_pic_marking->n_ref_pic_marking; i++) {
1968         GstH264RefPicMarking * const ref_pic_marking =
1969             &dec_ref_pic_marking->ref_pic_marking[i];
1970
1971         const guint mmco = ref_pic_marking->memory_management_control_operation;
1972         if (mmco < G_N_ELEMENTS(mmco_funcs) && mmco_funcs[mmco])
1973             mmco_funcs[mmco](decoder, picture, ref_pic_marking);
1974         else {
1975             GST_ERROR("unhandled MMCO %u", mmco);
1976             return FALSE;
1977         }
1978     }
1979     return TRUE;
1980 }
1981
1982 /* 8.2.5 - Execute reference picture marking process */
1983 static gboolean
1984 exec_ref_pic_marking(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
1985 {
1986     GstVaapiDecoderH264Private * const priv = decoder->priv;
1987     GstVaapiPictureH264 **picture_ptr;
1988
1989     priv->prev_pic_has_mmco5 = FALSE;
1990     priv->prev_pic_bottom_field =
1991         picture->field_pic_flag && picture->bottom_field_flag;
1992
1993     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
1994         return TRUE;
1995
1996     if (!picture->is_idr) {
1997         GstVaapiSliceH264 * const slice =
1998             gst_vaapi_picture_h264_get_last_slice(picture);
1999         GstH264DecRefPicMarking * const dec_ref_pic_marking =
2000             &slice->slice_hdr.dec_ref_pic_marking;
2001         if (dec_ref_pic_marking->adaptive_ref_pic_marking_mode_flag) {
2002             if (!exec_ref_pic_marking_adaptive(decoder, picture, dec_ref_pic_marking))
2003                 return FALSE;
2004         }
2005         else {
2006             if (!exec_ref_pic_marking_sliding_window(decoder))
2007                 return FALSE;
2008         }
2009     }
2010
2011     if (picture->is_long_term) {
2012         picture_ptr = &priv->long_ref[priv->long_ref_count++];
2013         picture->info.flags |= VA_PICTURE_H264_LONG_TERM_REFERENCE;
2014     }
2015     else {
2016         picture_ptr = &priv->short_ref[priv->short_ref_count++];
2017         picture->info.flags |= VA_PICTURE_H264_SHORT_TERM_REFERENCE;
2018     }
2019     gst_vaapi_picture_replace(picture_ptr, picture);
2020     return TRUE;
2021 }
2022
2023 static void
2024 vaapi_init_picture(VAPictureH264 *pic)
2025 {
2026     pic->picture_id           = VA_INVALID_ID;
2027     pic->frame_idx            = 0;
2028     pic->flags                = VA_PICTURE_H264_INVALID;
2029     pic->TopFieldOrderCnt     = 0;
2030     pic->BottomFieldOrderCnt  = 0;
2031 }
2032
2033 static void
2034 vaapi_fill_picture(VAPictureH264 *pic, GstVaapiPictureH264 *picture)
2035 {
2036     const guint field_flags = (VA_PICTURE_H264_TOP_FIELD |
2037                                VA_PICTURE_H264_BOTTOM_FIELD);
2038
2039     pic->picture_id = picture->base.surface_id;
2040     pic->flags = 0;
2041
2042     if (picture->info.flags & VA_PICTURE_H264_LONG_TERM_REFERENCE) {
2043         pic->flags |= VA_PICTURE_H264_LONG_TERM_REFERENCE;
2044         pic->frame_idx = picture->info.frame_idx;
2045     }
2046     else {
2047         if (picture->info.flags & VA_PICTURE_H264_SHORT_TERM_REFERENCE)
2048             pic->flags |= VA_PICTURE_H264_SHORT_TERM_REFERENCE;
2049         pic->frame_idx = picture->frame_num;
2050     }
2051
2052     switch (picture->info.flags & field_flags) {
2053     case 0:
2054         pic->TopFieldOrderCnt = picture->info.TopFieldOrderCnt;
2055         pic->BottomFieldOrderCnt = picture->info.BottomFieldOrderCnt;
2056         break;
2057     case VA_PICTURE_H264_TOP_FIELD:
2058         pic->flags |= VA_PICTURE_H264_TOP_FIELD;
2059         pic->TopFieldOrderCnt = picture->info.BottomFieldOrderCnt;
2060         pic->BottomFieldOrderCnt = 0;
2061         break;
2062     case VA_PICTURE_H264_BOTTOM_FIELD:
2063         pic->flags |= VA_PICTURE_H264_BOTTOM_FIELD;
2064         pic->BottomFieldOrderCnt = picture->info.BottomFieldOrderCnt;
2065         pic->TopFieldOrderCnt = 0;
2066         break;
2067     }
2068 }
2069
2070 static gboolean
2071 fill_picture(
2072     GstVaapiDecoderH264 *decoder,
2073     GstVaapiPictureH264 *picture,
2074     GstH264SliceHdr     *slice_hdr,
2075     GstH264NalUnit      *nalu
2076 )
2077 {
2078     GstVaapiDecoderH264Private * const priv = decoder->priv;
2079     GstVaapiPicture * const base_picture = &picture->base;
2080     GstH264PPS * const pps = picture->pps;
2081     GstH264SPS * const sps = pps->sequence;
2082     VAPictureParameterBufferH264 * const pic_param = base_picture->param;
2083     guint i, n;
2084
2085     /* Fill in VAPictureParameterBufferH264 */
2086     vaapi_fill_picture(&pic_param->CurrPic, picture);
2087     for (i = 0, n = 0; i < priv->short_ref_count; i++, n++)
2088         vaapi_fill_picture(&pic_param->ReferenceFrames[n], priv->short_ref[i]);
2089     for (i = 0; i < priv->long_ref_count; i++, n++)
2090         vaapi_fill_picture(&pic_param->ReferenceFrames[n], priv->long_ref[i]);
2091     for (; n < G_N_ELEMENTS(pic_param->ReferenceFrames); n++)
2092         vaapi_init_picture(&pic_param->ReferenceFrames[n]);
2093
2094 #define COPY_FIELD(s, f) \
2095     pic_param->f = (s)->f
2096
2097 #define COPY_BFM(a, s, f) \
2098     pic_param->a.bits.f = (s)->f
2099
2100     pic_param->picture_width_in_mbs_minus1  = ((priv->width + 15) >> 4)  - 1;
2101     pic_param->picture_height_in_mbs_minus1 = ((priv->height + 15) >> 4) - 1;
2102     pic_param->frame_num                    = priv->frame_num;
2103
2104     COPY_FIELD(sps, bit_depth_luma_minus8);
2105     COPY_FIELD(sps, bit_depth_chroma_minus8);
2106     COPY_FIELD(sps, num_ref_frames);
2107     COPY_FIELD(pps, num_slice_groups_minus1);
2108     COPY_FIELD(pps, slice_group_map_type);
2109     COPY_FIELD(pps, slice_group_change_rate_minus1);
2110     COPY_FIELD(pps, pic_init_qp_minus26);
2111     COPY_FIELD(pps, pic_init_qs_minus26);
2112     COPY_FIELD(pps, chroma_qp_index_offset);
2113     COPY_FIELD(pps, second_chroma_qp_index_offset);
2114
2115     pic_param->seq_fields.value                                         = 0; /* reset all bits */
2116     pic_param->seq_fields.bits.residual_colour_transform_flag           = sps->separate_colour_plane_flag;
2117     pic_param->seq_fields.bits.MinLumaBiPredSize8x8                     = sps->level_idc >= 31; /* A.3.3.2 */
2118
2119     COPY_BFM(seq_fields, sps, chroma_format_idc);
2120     COPY_BFM(seq_fields, sps, gaps_in_frame_num_value_allowed_flag);
2121     COPY_BFM(seq_fields, sps, frame_mbs_only_flag);
2122     COPY_BFM(seq_fields, sps, mb_adaptive_frame_field_flag);
2123     COPY_BFM(seq_fields, sps, direct_8x8_inference_flag);
2124     COPY_BFM(seq_fields, sps, log2_max_frame_num_minus4);
2125     COPY_BFM(seq_fields, sps, pic_order_cnt_type);
2126     COPY_BFM(seq_fields, sps, log2_max_pic_order_cnt_lsb_minus4);
2127     COPY_BFM(seq_fields, sps, delta_pic_order_always_zero_flag);
2128
2129     pic_param->pic_fields.value                                         = 0; /* reset all bits */
2130     pic_param->pic_fields.bits.field_pic_flag                           = slice_hdr->field_pic_flag;
2131     pic_param->pic_fields.bits.reference_pic_flag                       = GST_VAAPI_PICTURE_IS_REFERENCE(picture);
2132
2133     COPY_BFM(pic_fields, pps, entropy_coding_mode_flag);
2134     COPY_BFM(pic_fields, pps, weighted_pred_flag);
2135     COPY_BFM(pic_fields, pps, weighted_bipred_idc);
2136     COPY_BFM(pic_fields, pps, transform_8x8_mode_flag);
2137     COPY_BFM(pic_fields, pps, constrained_intra_pred_flag);
2138     COPY_BFM(pic_fields, pps, pic_order_present_flag);
2139     COPY_BFM(pic_fields, pps, deblocking_filter_control_present_flag);
2140     COPY_BFM(pic_fields, pps, redundant_pic_cnt_present_flag);
2141     return TRUE;
2142 }
2143
2144 /* Detection of the first VCL NAL unit of a primary coded picture (7.4.1.2.4) */
2145 static gboolean
2146 is_new_picture(
2147     GstVaapiDecoderH264 *decoder,
2148     GstH264NalUnit      *nalu,
2149     GstH264SliceHdr     *slice_hdr
2150 )
2151 {
2152     GstVaapiDecoderH264Private * const priv = decoder->priv;
2153     GstH264PPS * const pps = slice_hdr->pps;
2154     GstH264SPS * const sps = pps->sequence;
2155     GstVaapiSliceH264 *slice;
2156     GstH264SliceHdr *prev_slice_hdr;
2157
2158     if (!priv->current_picture)
2159         return TRUE;
2160
2161     slice = gst_vaapi_picture_h264_get_last_slice(priv->current_picture);
2162     if (!slice)
2163         return FALSE;
2164     prev_slice_hdr = &slice->slice_hdr;
2165
2166 #define CHECK_EXPR(expr, field_name) do {              \
2167         if (!(expr)) {                                 \
2168             GST_DEBUG(field_name " differs in value"); \
2169             return TRUE;                               \
2170         }                                              \
2171     } while (0)
2172
2173 #define CHECK_VALUE(new_slice_hdr, old_slice_hdr, field) \
2174     CHECK_EXPR(((new_slice_hdr)->field == (old_slice_hdr)->field), #field)
2175
2176     /* frame_num differs in value, regardless of inferred values to 0 */
2177     CHECK_VALUE(slice_hdr, prev_slice_hdr, frame_num);
2178
2179     /* pic_parameter_set_id differs in value */
2180     CHECK_VALUE(slice_hdr, prev_slice_hdr, pps);
2181
2182     /* field_pic_flag differs in value */
2183     CHECK_VALUE(slice_hdr, prev_slice_hdr, field_pic_flag);
2184
2185     /* bottom_field_flag is present in both and differs in value */
2186     if (slice_hdr->field_pic_flag && prev_slice_hdr->field_pic_flag)
2187         CHECK_VALUE(slice_hdr, prev_slice_hdr, bottom_field_flag);
2188
2189     /* nal_ref_idc differs in value with one of the nal_ref_idc values is 0 */
2190     CHECK_EXPR(((GST_VAAPI_PICTURE_IS_REFERENCE(priv->current_picture) ^
2191                  (nalu->ref_idc != 0)) == 0), "nal_ref_idc");
2192
2193     /* POC type is 0 for both and either pic_order_cnt_lsb differs in
2194        value or delta_pic_order_cnt_bottom differs in value */
2195     if (sps->pic_order_cnt_type == 0) {
2196         CHECK_VALUE(slice_hdr, prev_slice_hdr, pic_order_cnt_lsb);
2197         if (pps->pic_order_present_flag && !slice_hdr->field_pic_flag)
2198             CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt_bottom);
2199     }
2200
2201     /* POC type is 1 for both and either delta_pic_order_cnt[0]
2202        differs in value or delta_pic_order_cnt[1] differs in value */
2203     else if (sps->pic_order_cnt_type == 1) {
2204         CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt[0]);
2205         CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt[1]);
2206     }
2207
2208     /* IdrPicFlag differs in value */
2209     CHECK_EXPR(((priv->current_picture->is_idr ^
2210                  (nalu->type == GST_H264_NAL_SLICE_IDR)) == 0), "IdrPicFlag");
2211
2212     /* IdrPicFlag is equal to 1 for both and idr_pic_id differs in value */
2213     if (priv->current_picture->is_idr)
2214         CHECK_VALUE(slice_hdr, prev_slice_hdr, idr_pic_id);
2215
2216 #undef CHECK_EXPR
2217 #undef CHECK_VALUE
2218     return FALSE;
2219 }
2220
2221 static GstVaapiDecoderStatus
2222 decode_picture(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu, GstH264SliceHdr *slice_hdr)
2223 {
2224     GstVaapiDecoderH264Private * const priv = decoder->priv;
2225     GstVaapiPictureH264 *picture;
2226     GstVaapiDecoderStatus status;
2227     GstH264PPS * const pps = slice_hdr->pps;
2228     GstH264SPS * const sps = pps->sequence;
2229
2230     status = decode_current_picture(decoder);
2231     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2232         return status;
2233
2234     status = ensure_context(decoder, sps);
2235     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2236         return status;
2237
2238     picture = gst_vaapi_picture_h264_new(decoder);
2239     if (!picture) {
2240         GST_ERROR("failed to allocate picture");
2241         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2242     }
2243     priv->current_picture = picture;
2244
2245     picture->pps = pps;
2246
2247     status = ensure_quant_matrix(decoder, picture);
2248     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS) {
2249         GST_ERROR("failed to reset quantizer matrix");
2250         return status;
2251     }
2252
2253     if (!init_picture(decoder, picture, slice_hdr, nalu))
2254         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2255     if (!fill_picture(decoder, picture, slice_hdr, nalu))
2256         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2257     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2258 }
2259
2260 static gboolean
2261 decode_picture_end(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
2262 {
2263     if (!exec_ref_pic_marking(decoder, picture))
2264         return FALSE;
2265     if (!dpb_add(decoder, picture))
2266         return FALSE;
2267     return TRUE;
2268 }
2269
2270 static inline guint
2271 get_slice_data_bit_offset(GstH264SliceHdr *slice_hdr, GstH264NalUnit *nalu)
2272 {
2273     guint epb_count;
2274
2275     epb_count = slice_hdr->n_emulation_prevention_bytes;
2276     return 8 /* nal_unit_type */ + slice_hdr->header_size - epb_count * 8;
2277 }
2278
2279 static gboolean
2280 fill_pred_weight_table(GstVaapiDecoderH264 *decoder, GstVaapiSliceH264 *slice)
2281 {
2282     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2283     GstH264PPS * const pps = slice_hdr->pps;
2284     GstH264SPS * const sps = pps->sequence;
2285     GstH264PredWeightTable * const w = &slice_hdr->pred_weight_table;
2286     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2287     guint num_weight_tables = 0;
2288     gint i, j;
2289
2290     if (pps->weighted_pred_flag &&
2291         (GST_H264_IS_P_SLICE(slice_hdr) || GST_H264_IS_SP_SLICE(slice_hdr)))
2292         num_weight_tables = 1;
2293     else if (pps->weighted_bipred_idc == 1 && GST_H264_IS_B_SLICE(slice_hdr))
2294         num_weight_tables = 2;
2295     else
2296         num_weight_tables = 0;
2297
2298     slice_param->luma_log2_weight_denom   = w->luma_log2_weight_denom;
2299     slice_param->chroma_log2_weight_denom = w->chroma_log2_weight_denom;
2300     slice_param->luma_weight_l0_flag      = 0;
2301     slice_param->chroma_weight_l0_flag    = 0;
2302     slice_param->luma_weight_l1_flag      = 0;
2303     slice_param->chroma_weight_l1_flag    = 0;
2304
2305     if (num_weight_tables < 1)
2306         return TRUE;
2307
2308     slice_param->luma_weight_l0_flag = 1;
2309     for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2310         slice_param->luma_weight_l0[i] = w->luma_weight_l0[i];
2311         slice_param->luma_offset_l0[i] = w->luma_offset_l0[i];
2312     }
2313
2314     slice_param->chroma_weight_l0_flag = sps->chroma_array_type != 0;
2315     if (slice_param->chroma_weight_l0_flag) {
2316         for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2317             for (j = 0; j < 2; j++) {
2318                 slice_param->chroma_weight_l0[i][j] = w->chroma_weight_l0[i][j];
2319                 slice_param->chroma_offset_l0[i][j] = w->chroma_offset_l0[i][j];
2320             }
2321         }
2322     }
2323
2324     if (num_weight_tables < 2)
2325         return TRUE;
2326
2327     slice_param->luma_weight_l1_flag = 1;
2328     for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2329         slice_param->luma_weight_l1[i] = w->luma_weight_l1[i];
2330         slice_param->luma_offset_l1[i] = w->luma_offset_l1[i];
2331     }
2332
2333     slice_param->chroma_weight_l1_flag = sps->chroma_array_type != 0;
2334     if (slice_param->chroma_weight_l1_flag) {
2335         for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2336             for (j = 0; j < 2; j++) {
2337                 slice_param->chroma_weight_l1[i][j] = w->chroma_weight_l1[i][j];
2338                 slice_param->chroma_offset_l1[i][j] = w->chroma_offset_l1[i][j];
2339             }
2340         }
2341     }
2342     return TRUE;
2343 }
2344
2345 static gboolean
2346 fill_RefPicList(GstVaapiDecoderH264 *decoder, GstVaapiSliceH264 *slice)
2347 {
2348     GstVaapiDecoderH264Private * const priv = decoder->priv;
2349     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2350     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2351     guint i, num_ref_lists = 0;
2352
2353     slice_param->num_ref_idx_l0_active_minus1 = 0;
2354     slice_param->num_ref_idx_l1_active_minus1 = 0;
2355
2356     if (GST_H264_IS_B_SLICE(slice_hdr))
2357         num_ref_lists = 2;
2358     else if (GST_H264_IS_I_SLICE(slice_hdr))
2359         num_ref_lists = 0;
2360     else
2361         num_ref_lists = 1;
2362
2363     if (num_ref_lists < 1)
2364         return TRUE;
2365
2366     slice_param->num_ref_idx_l0_active_minus1 =
2367         slice_hdr->num_ref_idx_l0_active_minus1;
2368
2369     for (i = 0; i < priv->RefPicList0_count && priv->RefPicList0[i]; i++)
2370         vaapi_fill_picture(&slice_param->RefPicList0[i], priv->RefPicList0[i]);
2371     for (; i <= slice_param->num_ref_idx_l0_active_minus1; i++)
2372         vaapi_init_picture(&slice_param->RefPicList0[i]);
2373
2374     if (num_ref_lists < 2)
2375         return TRUE;
2376
2377     slice_param->num_ref_idx_l1_active_minus1 =
2378         slice_hdr->num_ref_idx_l1_active_minus1;
2379
2380     for (i = 0; i < priv->RefPicList1_count && priv->RefPicList1[i]; i++)
2381         vaapi_fill_picture(&slice_param->RefPicList1[i], priv->RefPicList1[i]);
2382     for (; i <= slice_param->num_ref_idx_l1_active_minus1; i++)
2383         vaapi_init_picture(&slice_param->RefPicList1[i]);
2384     return TRUE;
2385 }
2386
2387 static gboolean
2388 fill_slice(
2389     GstVaapiDecoderH264 *decoder,
2390     GstVaapiSliceH264   *slice,
2391     GstH264NalUnit      *nalu
2392 )
2393 {
2394     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2395     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2396
2397     /* Fill in VASliceParameterBufferH264 */
2398     slice_param->slice_data_bit_offset          = get_slice_data_bit_offset(slice_hdr, nalu);
2399     slice_param->first_mb_in_slice              = slice_hdr->first_mb_in_slice;
2400     slice_param->slice_type                     = slice_hdr->type % 5;
2401     slice_param->direct_spatial_mv_pred_flag    = slice_hdr->direct_spatial_mv_pred_flag;
2402     slice_param->cabac_init_idc                 = slice_hdr->cabac_init_idc;
2403     slice_param->slice_qp_delta                 = slice_hdr->slice_qp_delta;
2404     slice_param->disable_deblocking_filter_idc  = slice_hdr->disable_deblocking_filter_idc;
2405     slice_param->slice_alpha_c0_offset_div2     = slice_hdr->slice_alpha_c0_offset_div2;
2406     slice_param->slice_beta_offset_div2         = slice_hdr->slice_beta_offset_div2;
2407
2408     if (!fill_RefPicList(decoder, slice))
2409         return FALSE;
2410     if (!fill_pred_weight_table(decoder, slice))
2411         return FALSE;
2412     return TRUE;
2413 }
2414
2415 static GstVaapiDecoderStatus
2416 decode_slice(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
2417 {
2418     GstVaapiDecoderH264Private * const priv = decoder->priv;
2419     GstVaapiDecoderStatus status;
2420     GstVaapiPictureH264 *picture;
2421     GstVaapiSliceH264 *slice = NULL;
2422     GstH264SliceHdr *slice_hdr;
2423     GstH264ParserResult result;
2424
2425     GST_DEBUG("slice (%u bytes)", nalu->size);
2426
2427     slice = gst_vaapi_slice_h264_new(
2428         decoder,
2429         nalu->data + nalu->offset,
2430         nalu->size
2431     );
2432     if (!slice) {
2433         GST_ERROR("failed to allocate slice");
2434         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2435     }
2436
2437     slice_hdr = &slice->slice_hdr;
2438     memset(slice_hdr, 0, sizeof(*slice_hdr));
2439     result = gst_h264_parser_parse_slice_hdr(priv->parser, nalu, slice_hdr, TRUE, TRUE);
2440     if (result != GST_H264_PARSER_OK) {
2441         status = get_status(result);
2442         goto error;
2443     }
2444
2445     if (is_new_picture(decoder, nalu, slice_hdr)) {
2446         status = decode_picture(decoder, nalu, slice_hdr);
2447         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2448             goto error;
2449     }
2450     picture = priv->current_picture;
2451
2452     if (!fill_slice(decoder, slice, nalu)) {
2453         status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2454         goto error;
2455     }
2456     gst_vaapi_picture_add_slice(
2457         GST_VAAPI_PICTURE_CAST(picture),
2458         GST_VAAPI_SLICE_CAST(slice)
2459     );
2460     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2461
2462 error:
2463     if (slice)
2464         gst_mini_object_unref(GST_MINI_OBJECT(slice));
2465     return status;
2466 }
2467
2468 static inline gint
2469 scan_for_start_code(GstAdapter *adapter, guint ofs, guint size, guint32 *scp)
2470 {
2471     return (gint)gst_adapter_masked_scan_uint32_peek(adapter,
2472                                                      0xffffff00, 0x00000100,
2473                                                      ofs, size,
2474                                                      scp);
2475 }
2476
2477 static GstVaapiDecoderStatus
2478 decode_nalu(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
2479 {
2480     GstVaapiDecoderStatus status;
2481
2482     switch (nalu->type) {
2483     case GST_H264_NAL_SLICE_IDR:
2484         /* fall-through. IDR specifics are handled in init_picture() */
2485     case GST_H264_NAL_SLICE:
2486         status = decode_slice(decoder, nalu);
2487         break;
2488     case GST_H264_NAL_SPS:
2489         status = decode_sps(decoder, nalu);
2490         break;
2491     case GST_H264_NAL_PPS:
2492         status = decode_pps(decoder, nalu);
2493         break;
2494     case GST_H264_NAL_SEI:
2495         status = decode_sei(decoder, nalu);
2496         break;
2497     case GST_H264_NAL_SEQ_END:
2498         status = decode_sequence_end(decoder);
2499         break;
2500     case GST_H264_NAL_AU_DELIMITER:
2501         /* skip all Access Unit NALs */
2502         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2503         break;
2504     case GST_H264_NAL_FILLER_DATA:
2505         /* skip all Filler Data NALs */
2506         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2507         break;
2508     default:
2509         GST_WARNING("unsupported NAL unit type %d", nalu->type);
2510         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
2511         break;
2512     }
2513     return status;
2514 }
2515
2516 static GstVaapiDecoderStatus
2517 decode_buffer(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
2518 {
2519     GstVaapiDecoderH264Private * const priv = decoder->priv;
2520     GstVaapiDecoderStatus status;
2521     GstH264ParserResult result;
2522     GstH264NalUnit nalu;
2523     gboolean is_eos;
2524     const guchar *buf;
2525     guint i, buf_size, nalu_size, size;
2526     guint32 start_code;
2527     gint ofs;
2528
2529     buf      = GST_BUFFER_DATA(buffer);
2530     buf_size = GST_BUFFER_SIZE(buffer);
2531     is_eos   = GST_BUFFER_IS_EOS(buffer);
2532     if (buf && buf_size > 0)
2533         gst_adapter_push(priv->adapter, gst_buffer_ref(buffer));
2534
2535     size = gst_adapter_available(priv->adapter);
2536     do {
2537         if (size == 0) {
2538             status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2539             break;
2540         }
2541
2542         status = gst_vaapi_decoder_check_status(GST_VAAPI_DECODER(decoder));
2543         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2544             break;
2545
2546         status = GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2547         if (priv->is_avc) {
2548             if (size < priv->nal_length_size)
2549                 break;
2550             buf = gst_adapter_peek(priv->adapter, priv->nal_length_size);
2551
2552             nalu_size = 0;
2553             for (i = 0; i < priv->nal_length_size; i++)
2554                 nalu_size = (nalu_size << 8) | buf[i];
2555
2556             buf_size = priv->nal_length_size + nalu_size;
2557             if (size < buf_size)
2558                 break;
2559             buffer = gst_adapter_take_buffer(priv->adapter, buf_size);
2560             size -= buf_size;
2561
2562             buf      = GST_BUFFER_DATA(buffer);
2563             buf_size = GST_BUFFER_SIZE(buffer);
2564
2565             result = gst_h264_parser_identify_nalu_avc(
2566                 priv->parser,
2567                 buf, 0, buf_size, priv->nal_length_size,
2568                 &nalu
2569             );
2570         }
2571         else {
2572             if (size < 4)
2573                 break;
2574             ofs = scan_for_start_code(priv->adapter, 0, size, &start_code);
2575             if (ofs < 0)
2576                 break;
2577             gst_adapter_flush(priv->adapter, ofs);
2578             size -= ofs;
2579
2580             ofs = G_UNLIKELY(size < 8) ? -1 :
2581                 scan_for_start_code(priv->adapter, 4, size - 4, NULL);
2582             if (ofs < 0) {
2583                 // Assume the whole NAL unit is present if end-of-stream
2584                 if (!is_eos)
2585                     break;
2586                 ofs = size;
2587             }
2588             buffer = gst_adapter_take_buffer(priv->adapter, ofs);
2589             size -= ofs;
2590
2591             buf      = GST_BUFFER_DATA(buffer);
2592             buf_size = GST_BUFFER_SIZE(buffer);
2593
2594             result = gst_h264_parser_identify_nalu_unchecked(
2595                 priv->parser,
2596                 buf, 0, buf_size,
2597                 &nalu
2598             );
2599         }
2600         status = get_status(result);
2601         if (status == GST_VAAPI_DECODER_STATUS_SUCCESS)
2602             status = decode_nalu(decoder, &nalu);
2603         gst_buffer_unref(buffer);
2604     } while (status == GST_VAAPI_DECODER_STATUS_SUCCESS);
2605
2606     if (is_eos && (status == GST_VAAPI_DECODER_STATUS_SUCCESS ||
2607                    status == GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA))
2608         status = decode_sequence_end(decoder);
2609     return status;
2610 }
2611
2612 static GstVaapiDecoderStatus
2613 decode_codec_data(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
2614 {
2615     GstVaapiDecoderH264Private * const priv = decoder->priv;
2616     GstVaapiDecoderStatus status;
2617     GstH264NalUnit nalu;
2618     GstH264ParserResult result;
2619     guchar *buf;
2620     guint buf_size;
2621     guint i, ofs, num_sps, num_pps;
2622
2623     buf      = GST_BUFFER_DATA(buffer);
2624     buf_size = GST_BUFFER_SIZE(buffer);
2625     if (!buf || buf_size == 0)
2626         return GST_VAAPI_DECODER_STATUS_SUCCESS;
2627
2628     if (buf_size < 8)
2629         return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2630
2631     if (buf[0] != 1) {
2632         GST_ERROR("failed to decode codec-data, not in avcC format");
2633         return GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
2634     }
2635
2636     priv->nal_length_size = (buf[4] & 0x03) + 1;
2637
2638     num_sps = buf[5] & 0x1f;
2639     ofs = 6;
2640
2641     for (i = 0; i < num_sps; i++) {
2642         result = gst_h264_parser_identify_nalu_avc(
2643             priv->parser,
2644             buf, ofs, buf_size, 2,
2645             &nalu
2646         );
2647         if (result != GST_H264_PARSER_OK)
2648             return get_status(result);
2649
2650         status = decode_sps(decoder, &nalu);
2651         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2652             return status;
2653         ofs = nalu.offset + nalu.size;
2654     }
2655
2656     num_pps = buf[ofs];
2657     ofs++;
2658
2659     for (i = 0; i < num_pps; i++) {
2660         result = gst_h264_parser_identify_nalu_avc(
2661             priv->parser,
2662             buf, ofs, buf_size, 2,
2663             &nalu
2664         );
2665         if (result != GST_H264_PARSER_OK)
2666             return get_status(result);
2667
2668         status = decode_pps(decoder, &nalu);
2669         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2670             return status;
2671         ofs = nalu.offset + nalu.size;
2672     }
2673
2674     priv->is_avc = TRUE;
2675     return status;
2676 }
2677
2678 GstVaapiDecoderStatus
2679 gst_vaapi_decoder_h264_decode(GstVaapiDecoder *base, GstBuffer *buffer)
2680 {
2681     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264(base);
2682     GstVaapiDecoderH264Private * const priv = decoder->priv;
2683     GstVaapiDecoderStatus status;
2684     GstBuffer *codec_data;
2685
2686     g_return_val_if_fail(priv->is_constructed,
2687                          GST_VAAPI_DECODER_STATUS_ERROR_INIT_FAILED);
2688
2689     if (!priv->is_opened) {
2690         priv->is_opened = gst_vaapi_decoder_h264_open(decoder, buffer);
2691         if (!priv->is_opened)
2692             return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CODEC;
2693
2694         codec_data = GST_VAAPI_DECODER_CODEC_DATA(decoder);
2695         if (codec_data) {
2696             status = decode_codec_data(decoder, codec_data);
2697             if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2698                 return status;
2699         }
2700      }
2701      return decode_buffer(decoder, buffer);
2702 }
2703
2704 static void
2705 gst_vaapi_decoder_h264_finalize(GObject *object)
2706 {
2707     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264(object);
2708
2709     gst_vaapi_decoder_h264_destroy(decoder);
2710
2711     G_OBJECT_CLASS(gst_vaapi_decoder_h264_parent_class)->finalize(object);
2712 }
2713
2714 static void
2715 gst_vaapi_decoder_h264_constructed(GObject *object)
2716 {
2717     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264(object);
2718     GstVaapiDecoderH264Private * const priv = decoder->priv;
2719     GObjectClass *parent_class;
2720
2721     parent_class = G_OBJECT_CLASS(gst_vaapi_decoder_h264_parent_class);
2722     if (parent_class->constructed)
2723         parent_class->constructed(object);
2724
2725     priv->is_constructed = gst_vaapi_decoder_h264_create(decoder);
2726 }
2727
2728 static void
2729 gst_vaapi_decoder_h264_class_init(GstVaapiDecoderH264Class *klass)
2730 {
2731     GObjectClass * const object_class = G_OBJECT_CLASS(klass);
2732     GstVaapiDecoderClass * const decoder_class = GST_VAAPI_DECODER_CLASS(klass);
2733
2734     g_type_class_add_private(klass, sizeof(GstVaapiDecoderH264Private));
2735
2736     object_class->finalize      = gst_vaapi_decoder_h264_finalize;
2737     object_class->constructed   = gst_vaapi_decoder_h264_constructed;
2738
2739     decoder_class->decode       = gst_vaapi_decoder_h264_decode;
2740 }
2741
2742 static void
2743 gst_vaapi_decoder_h264_init(GstVaapiDecoderH264 *decoder)
2744 {
2745     GstVaapiDecoderH264Private *priv;
2746
2747     priv                        = GST_VAAPI_DECODER_H264_GET_PRIVATE(decoder);
2748     decoder->priv               = priv;
2749     priv->parser                = NULL;
2750     priv->current_picture       = NULL;
2751     priv->dpb_count             = 0;
2752     priv->dpb_size              = 0;
2753     priv->profile               = GST_VAAPI_PROFILE_UNKNOWN;
2754     priv->entrypoint            = GST_VAAPI_ENTRYPOINT_VLD;
2755     priv->chroma_type           = GST_VAAPI_CHROMA_TYPE_YUV420;
2756     priv->short_ref_count       = 0;
2757     priv->long_ref_count        = 0;
2758     priv->RefPicList0_count     = 0;
2759     priv->RefPicList1_count     = 0;
2760     priv->nal_length_size       = 0;
2761     priv->width                 = 0;
2762     priv->height                = 0;
2763     priv->adapter               = NULL;
2764     priv->field_poc[0]          = 0;
2765     priv->field_poc[1]          = 0;
2766     priv->poc_msb               = 0;
2767     priv->poc_lsb               = 0;
2768     priv->prev_poc_msb          = 0;
2769     priv->prev_poc_lsb          = 0;
2770     priv->frame_num_offset      = 0;
2771     priv->frame_num             = 0;
2772     priv->prev_frame_num        = 0;
2773     priv->prev_pic_has_mmco5    = FALSE;
2774     priv->prev_pic_bottom_field = FALSE;
2775     priv->is_constructed        = FALSE;
2776     priv->is_opened             = FALSE;
2777     priv->is_avc                = FALSE;
2778     priv->has_context           = FALSE;
2779
2780     memset(priv->dpb, 0, sizeof(priv->dpb));
2781     memset(priv->short_ref, 0, sizeof(priv->short_ref));
2782     memset(priv->long_ref, 0, sizeof(priv->long_ref));
2783     memset(priv->RefPicList0, 0, sizeof(priv->RefPicList0));
2784     memset(priv->RefPicList1, 0, sizeof(priv->RefPicList1));
2785 }
2786
2787 /**
2788  * gst_vaapi_decoder_h264_new:
2789  * @display: a #GstVaapiDisplay
2790  * @caps: a #GstCaps holding codec information
2791  *
2792  * Creates a new #GstVaapiDecoder for MPEG-2 decoding.  The @caps can
2793  * hold extra information like codec-data and pictured coded size.
2794  *
2795  * Return value: the newly allocated #GstVaapiDecoder object
2796  */
2797 GstVaapiDecoder *
2798 gst_vaapi_decoder_h264_new(GstVaapiDisplay *display, GstCaps *caps)
2799 {
2800     GstVaapiDecoderH264 *decoder;
2801
2802     g_return_val_if_fail(GST_VAAPI_IS_DISPLAY(display), NULL);
2803     g_return_val_if_fail(GST_IS_CAPS(caps), NULL);
2804
2805     decoder = g_object_new(
2806         GST_VAAPI_TYPE_DECODER_H264,
2807         "display",      display,
2808         "caps",         caps,
2809         NULL
2810     );
2811     if (!decoder->priv->is_constructed) {
2812         g_object_unref(decoder);
2813         return NULL;
2814     }
2815     return GST_VAAPI_DECODER_CAST(decoder);
2816 }