subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codech264dec.c

   1 /* GStreamer
   2  * Copyright (C) 2020 Nicolas Dufresne <nicolas.dufresne@collabora.com>
   3  *
   4  * This library is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU Library General Public
   6  * License as published by the Free Software Foundation; either
   7  * version 2 of the License, or (at your option) any later version.
   8  *
   9  * This library is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12  * Library General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU Library General Public
  15  * License along with this library; if not, write to the
  16  * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
  17  * Boston, MA 02110-1301, USA.
  18  */
  19
  20 #ifdef HAVE_CONFIG_H
  21 #include <config.h>
  22 #endif
  23
  24 #include "gstv4l2codecallocator.h"
  25 #include "gstv4l2codech264dec.h"
  26 #include "gstv4l2codecpool.h"
  27 #include "gstv4l2format.h"
  28 #include "linux/v4l2-controls.h"
  29
  30 #define KERNEL_VERSION(a,b,c) (((a) << 16) + ((b) << 8) + (c))
  31
  32 #define V4L2_MIN_KERNEL_VER_MAJOR 5
  33 #define V4L2_MIN_KERNEL_VER_MINOR 11
  34 #define V4L2_MIN_KERNEL_VERSION KERNEL_VERSION(V4L2_MIN_KERNEL_VER_MAJOR, V4L2_MIN_KERNEL_VER_MINOR, 0)
  35
  36 GST_DEBUG_CATEGORY_STATIC (v4l2_h264dec_debug);
  37 #define GST_CAT_DEFAULT v4l2_h264dec_debug
  38
  39 enum
  40 {
  41   PROP_0,
  42   PROP_LAST = PROP_0
  43 };
  44
  45 static GstStaticPadTemplate sink_template =
  46 GST_STATIC_PAD_TEMPLATE (GST_VIDEO_DECODER_SINK_NAME,
  47     GST_PAD_SINK, GST_PAD_ALWAYS,
  48     GST_STATIC_CAPS ("video/x-h264, "
  49         "stream-format=(string) { avc, avc3, byte-stream }, "
  50         "alignment=(string) au")
  51     );
  52
  53 static GstStaticPadTemplate src_template =
  54 GST_STATIC_PAD_TEMPLATE (GST_VIDEO_DECODER_SRC_NAME,
  55     GST_PAD_SRC, GST_PAD_ALWAYS,
  56     GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE (GST_V4L2_DEFAULT_VIDEO_FORMATS)));
  57
  58 struct _GstV4l2CodecH264Dec
  59 {
  60   GstH264Decoder parent;
  61   GstV4l2Decoder *decoder;
  62   GstVideoCodecState *output_state;
  63   GstVideoInfo vinfo;
  64   gint display_width;
  65   gint display_height;
  66   gint coded_width;
  67   gint coded_height;
  68   guint bitdepth;
  69   guint chroma_format_idc;
  70   guint num_slices;
  71   gboolean first_slice;
  72
  73   GstV4l2CodecAllocator *sink_allocator;
  74   GstV4l2CodecAllocator *src_allocator;
  75   GstV4l2CodecPool *src_pool;
  76   gint min_pool_size;
  77   gboolean has_videometa;
  78   gboolean need_negotiation;
  79   gboolean interlaced;
  80   gboolean need_sequence;
  81   gboolean copy_frames;
  82   gboolean scaling_matrix_present;
  83
  84   struct v4l2_ctrl_h264_sps sps;
  85   struct v4l2_ctrl_h264_pps pps;
  86   struct v4l2_ctrl_h264_scaling_matrix scaling_matrix;
  87   struct v4l2_ctrl_h264_decode_params decode_params;
  88   struct v4l2_ctrl_h264_pred_weights pred_weight;
  89   GArray *slice_params;
  90
  91   enum v4l2_stateless_h264_decode_mode decode_mode;
  92   enum v4l2_stateless_h264_start_code start_code;
  93
  94   GstMemory *bitstream;
  95   GstMapInfo bitstream_map;
  96 };
  97
  98 G_DEFINE_ABSTRACT_TYPE (GstV4l2CodecH264Dec, gst_v4l2_codec_h264_dec,
  99     GST_TYPE_H264_DECODER);
 100
 101 #define parent_class gst_v4l2_codec_h264_dec_parent_class
 102
 103 static gboolean
 104 is_frame_based (GstV4l2CodecH264Dec * self)
 105 {
 106   return self->decode_mode == V4L2_STATELESS_H264_DECODE_MODE_FRAME_BASED;
 107 }
 108
 109 static gboolean
 110 is_slice_based (GstV4l2CodecH264Dec * self)
 111 {
 112   return self->decode_mode == V4L2_STATELESS_H264_DECODE_MODE_SLICE_BASED;
 113 }
 114
 115 static gboolean
 116 needs_start_codes (GstV4l2CodecH264Dec * self)
 117 {
 118   return self->start_code == V4L2_STATELESS_H264_START_CODE_ANNEX_B;
 119 }
 120
 121 static gboolean
 122 gst_v4l2_decoder_h264_api_check (GstV4l2Decoder * decoder)
 123 {
 124   guint i, ret_size;
 125   /* *INDENT-OFF* */
 126   #define SET_ID(cid) .id = (cid), .name = #cid
 127   struct
 128   {
 129     const gchar *name;
 130     unsigned int id;
 131     unsigned int size;
 132     gboolean optional;
 133   } controls[] = {
 134     {
 135       SET_ID (V4L2_CID_STATELESS_H264_SPS),
 136       .size = sizeof(struct v4l2_ctrl_h264_sps),
 137     }, {
 138       SET_ID (V4L2_CID_STATELESS_H264_PPS),
 139       .size = sizeof(struct v4l2_ctrl_h264_pps),
 140     }, {
 141       SET_ID (V4L2_CID_STATELESS_H264_SCALING_MATRIX),
 142       .size = sizeof(struct v4l2_ctrl_h264_scaling_matrix),
 143     }, {
 144       SET_ID (V4L2_CID_STATELESS_H264_DECODE_PARAMS),
 145       .size = sizeof(struct v4l2_ctrl_h264_decode_params),
 146     }, {
 147       SET_ID (V4L2_CID_STATELESS_H264_SLICE_PARAMS),
 148       .size = sizeof(struct v4l2_ctrl_h264_slice_params),
 149       .optional = TRUE,
 150     }, {
 151       SET_ID (V4L2_CID_STATELESS_H264_PRED_WEIGHTS),
 152       .size = sizeof(struct v4l2_ctrl_h264_pred_weights),
 153       .optional = TRUE,
 154     }
 155   };
 156   #undef SET_ID
 157   /* *INDENT-ON* */
 158
 159   /*
 160    * Compatibility check: make sure the pointer controls are
 161    * the right size.
 162    */
 163   for (i = 0; i < G_N_ELEMENTS (controls); i++) {
 164     gboolean control_found;
 165
 166     control_found = gst_v4l2_decoder_query_control_size (decoder,
 167         controls[i].id, &ret_size);
 168
 169     if (!controls[i].optional && !control_found) {
 170       GST_WARNING ("Driver is missing %s support.", controls[i].name);
 171       return FALSE;
 172     }
 173
 174     if (control_found && ret_size != controls[i].size) {
 175       GST_WARNING ("%s control size mismatch: got %d bytes but %d expected.",
 176           controls[i].name, ret_size, controls[i].size);
 177       return FALSE;
 178     }
 179   }
 180
 181   return TRUE;
 182 }
 183
 184 static gboolean
 185 gst_v4l2_codec_h264_dec_open (GstVideoDecoder * decoder)
 186 {
 187   GstV4l2CodecH264Dec *self = GST_V4L2_CODEC_H264_DEC (decoder);
 188
 189   /* *INDENT-OFF* */
 190   struct v4l2_ext_control control[] = {
 191     {
 192       .id = V4L2_CID_STATELESS_H264_DECODE_MODE,
 193     },
 194     {
 195       .id = V4L2_CID_STATELESS_H264_START_CODE,
 196     },
 197   };
 198   /* *INDENT-ON* */
 199
 200   if (!gst_v4l2_decoder_open (self->decoder)) {
 201     GST_ELEMENT_ERROR (self, RESOURCE, OPEN_READ_WRITE,
 202         ("Failed to open H264 decoder"),
 203         ("gst_v4l2_decoder_open() failed: %s", g_strerror (errno)));
 204     return FALSE;
 205   }
 206
 207   if (!gst_v4l2_decoder_get_controls (self->decoder, control,
 208           G_N_ELEMENTS (control))) {
 209     GST_ELEMENT_ERROR (self, RESOURCE, OPEN_READ_WRITE,
 210         ("Driver did not report framing and start code method."),
 211         ("gst_v4l2_decoder_get_controls() failed: %s", g_strerror (errno)));
 212     return FALSE;
 213   }
 214
 215   self->decode_mode = control[0].value;
 216   self->start_code = control[1].value;
 217
 218   GST_INFO_OBJECT (self, "Opened H264 %s decoder %s",
 219       is_frame_based (self) ? "frame based" : "slice based",
 220       needs_start_codes (self) ? "using start-codes" : "without start-codes");
 221   gst_h264_decoder_set_process_ref_pic_lists (GST_H264_DECODER (self),
 222       is_slice_based (self));
 223
 224   return TRUE;
 225 }
 226
 227 static gboolean
 228 gst_v4l2_codec_h264_dec_close (GstVideoDecoder * decoder)
 229 {
 230   GstV4l2CodecH264Dec *self = GST_V4L2_CODEC_H264_DEC (decoder);
 231   gst_v4l2_decoder_close (self->decoder);
 232   return TRUE;
 233 }
 234
 235 static void
 236 gst_v4l2_codec_h264_dec_reset_allocation (GstV4l2CodecH264Dec * self)
 237 {
 238   if (self->sink_allocator) {
 239     gst_v4l2_codec_allocator_detach (self->sink_allocator);
 240     g_clear_object (&self->sink_allocator);
 241   }
 242
 243   if (self->src_allocator) {
 244     gst_v4l2_codec_allocator_detach (self->src_allocator);
 245     g_clear_object (&self->src_allocator);
 246     g_clear_object (&self->src_pool);
 247   }
 248 }
 249
 250 static gboolean
 251 gst_v4l2_codec_h264_dec_stop (GstVideoDecoder * decoder)
 252 {
 253   GstV4l2CodecH264Dec *self = GST_V4L2_CODEC_H264_DEC (decoder);
 254
 255   gst_v4l2_decoder_streamoff (self->decoder, GST_PAD_SINK);
 256   gst_v4l2_decoder_streamoff (self->decoder, GST_PAD_SRC);
 257
 258   gst_v4l2_codec_h264_dec_reset_allocation (self);
 259
 260   if (self->output_state)
 261     gst_video_codec_state_unref (self->output_state);
 262   self->output_state = NULL;
 263
 264   return GST_VIDEO_DECODER_CLASS (parent_class)->stop (decoder);
 265 }
 266
 267 static gint
 268 get_pixel_bitdepth (GstV4l2CodecH264Dec * self)
 269 {
 270   gint depth;
 271
 272   switch (self->chroma_format_idc) {
 273     case 0:
 274       /* 4:0:0 */
 275       depth = self->bitdepth;
 276       break;
 277     case 1:
 278       /* 4:2:0 */
 279       depth = self->bitdepth + self->bitdepth / 2;
 280       break;
 281     case 2:
 282       /* 4:2:2 */
 283       depth = 2 * self->bitdepth;
 284       break;
 285     case 3:
 286       /* 4:4:4 */
 287       depth = 3 * self->bitdepth;
 288       break;
 289     default:
 290       GST_WARNING_OBJECT (self, "Unsupported chroma format %i",
 291           self->chroma_format_idc);
 292       depth = 0;
 293       break;
 294   }
 295
 296   return depth;
 297 }
 298
 299 static gboolean
 300 gst_v4l2_codec_h264_dec_negotiate (GstVideoDecoder * decoder)
 301 {
 302   GstV4l2CodecH264Dec *self = GST_V4L2_CODEC_H264_DEC (decoder);
 303   GstH264Decoder *h264dec = GST_H264_DECODER (decoder);
 304   /* *INDENT-OFF* */
 305   struct v4l2_ext_control control[] = {
 306     {
 307       .id = V4L2_CID_STATELESS_H264_SPS,
 308       .ptr = &self->sps,
 309       .size = sizeof (self->sps),
 310     },
 311   };
 312   /* *INDENT-ON* */
 313   GstCaps *filter, *caps;
 314
 315   /* Ignore downstream renegotiation request. */
 316   if (!self->need_negotiation)
 317     return TRUE;
 318   self->need_negotiation = FALSE;
 319
 320   GST_DEBUG_OBJECT (self, "Negotiate");
 321
 322   gst_v4l2_decoder_streamoff (self->decoder, GST_PAD_SINK);
 323   gst_v4l2_decoder_streamoff (self->decoder, GST_PAD_SRC);
 324
 325   gst_v4l2_codec_h264_dec_reset_allocation (self);
 326
 327   if (!gst_v4l2_decoder_set_sink_fmt (self->decoder, V4L2_PIX_FMT_H264_SLICE,
 328           self->coded_width, self->coded_height, get_pixel_bitdepth (self))) {
 329     GST_ELEMENT_ERROR (self, CORE, NEGOTIATION,
 330         ("Failed to configure H264 decoder"),
 331         ("gst_v4l2_decoder_set_sink_fmt() failed: %s", g_strerror (errno)));
 332     gst_v4l2_decoder_close (self->decoder);
 333     return FALSE;
 334   }
 335
 336   if (!gst_v4l2_decoder_set_controls (self->decoder, NULL, control,
 337           G_N_ELEMENTS (control))) {
 338     GST_ELEMENT_ERROR (decoder, RESOURCE, WRITE,
 339         ("Driver does not support the selected stream."), (NULL));
 340     return FALSE;
 341   }
 342
 343   filter = gst_v4l2_decoder_enum_src_formats (self->decoder);
 344   if (!filter) {
 345     GST_ELEMENT_ERROR (self, CORE, NEGOTIATION,
 346         ("No supported decoder output formats"), (NULL));
 347     return FALSE;
 348   }
 349   GST_DEBUG_OBJECT (self, "Supported output formats: %" GST_PTR_FORMAT, filter);
 350
 351   caps = gst_pad_peer_query_caps (decoder->srcpad, filter);
 352   gst_caps_unref (filter);
 353   GST_DEBUG_OBJECT (self, "Peer supported formats: %" GST_PTR_FORMAT, caps);
 354
 355   if (!gst_v4l2_decoder_select_src_format (self->decoder, caps, &self->vinfo)) {
 356     GST_ELEMENT_ERROR (self, CORE, NEGOTIATION,
 357         ("Unsupported bitdepth/chroma format"),
 358         ("No support for %ux%u %ubit chroma IDC %i", self->coded_width,
 359             self->coded_height, self->bitdepth, self->chroma_format_idc));
 360     gst_caps_unref (caps);
 361     return FALSE;
 362   }
 363   gst_caps_unref (caps);
 364
 365   if (self->output_state)
 366     gst_video_codec_state_unref (self->output_state);
 367
 368   self->output_state =
 369       gst_video_decoder_set_output_state (GST_VIDEO_DECODER (self),
 370       self->vinfo.finfo->format, self->display_width,
 371       self->display_height, h264dec->input_state);
 372
 373   if (self->interlaced)
 374     self->output_state->info.interlace_mode = GST_VIDEO_INTERLACE_MODE_MIXED;
 375
 376   self->output_state->caps = gst_video_info_to_caps (&self->output_state->info);
 377
 378   if (GST_VIDEO_DECODER_CLASS (parent_class)->negotiate (decoder)) {
 379     if (!gst_v4l2_decoder_streamon (self->decoder, GST_PAD_SINK)) {
 380       GST_ELEMENT_ERROR (self, RESOURCE, FAILED,
 381           ("Could not enable the decoder driver."),
 382           ("VIDIOC_STREAMON(SINK) failed: %s", g_strerror (errno)));
 383       return FALSE;
 384     }
 385
 386     if (!gst_v4l2_decoder_streamon (self->decoder, GST_PAD_SRC)) {
 387       GST_ELEMENT_ERROR (self, RESOURCE, FAILED,
 388           ("Could not enable the decoder driver."),
 389           ("VIDIOC_STREAMON(SRC) failed: %s", g_strerror (errno)));
 390       return FALSE;
 391     }
 392
 393     return TRUE;
 394   }
 395
 396   return FALSE;
 397 }
 398
 399 static gboolean
 400 gst_v4l2_codec_h264_dec_decide_allocation (GstVideoDecoder * decoder,
 401     GstQuery * query)
 402 {
 403   GstV4l2CodecH264Dec *self = GST_V4L2_CODEC_H264_DEC (decoder);
 404   guint min = 0, num_bitstream;
 405
 406   self->has_videometa = gst_query_find_allocation_meta (query,
 407       GST_VIDEO_META_API_TYPE, NULL);
 408
 409   g_clear_object (&self->src_pool);
 410   g_clear_object (&self->src_allocator);
 411
 412   if (gst_query_get_n_allocation_pools (query) > 0)
 413     gst_query_parse_nth_allocation_pool (query, 0, NULL, NULL, &min, NULL);
 414
 415   min = MAX (2, min);
 416
 417   num_bitstream = 1 +
 418       MAX (1, gst_v4l2_decoder_get_render_delay (self->decoder));
 419
 420   self->sink_allocator = gst_v4l2_codec_allocator_new (self->decoder,
 421       GST_PAD_SINK, num_bitstream);
 422   if (!self->sink_allocator) {
 423     GST_ELEMENT_ERROR (self, RESOURCE, NO_SPACE_LEFT,
 424         ("Not enough memory to allocate sink buffers."), (NULL));
 425     return FALSE;
 426   }
 427
 428   self->src_allocator = gst_v4l2_codec_allocator_new (self->decoder,
 429       GST_PAD_SRC, self->min_pool_size + min + 4);
 430   if (!self->src_allocator) {
 431     GST_ELEMENT_ERROR (self, RESOURCE, NO_SPACE_LEFT,
 432         ("Not enough memory to allocate source buffers."), (NULL));
 433     g_clear_object (&self->sink_allocator);
 434     return FALSE;
 435   }
 436
 437   self->src_pool = gst_v4l2_codec_pool_new (self->src_allocator, &self->vinfo);
 438
 439   /* Our buffer pool is internal, we will let the base class create a video
 440    * pool, and use it if we are running out of buffers or if downstream does
 441    * not support GstVideoMeta */
 442   return GST_VIDEO_DECODER_CLASS (parent_class)->decide_allocation
 443       (decoder, query);
 444 }
 445
 446 static void
 447 gst_v4l2_codec_h264_dec_fill_sequence (GstV4l2CodecH264Dec * self,
 448     const GstH264SPS * sps)
 449 {
 450   gint i;
 451
 452   /* *INDENT-OFF* */
 453   self->sps = (struct v4l2_ctrl_h264_sps) {
 454     .profile_idc = sps->profile_idc,
 455     .constraint_set_flags = (sps->constraint_set0_flag)
 456         | (sps->constraint_set1_flag << 1) | (sps->constraint_set2_flag << 2)
 457         | (sps->constraint_set3_flag << 3) | (sps->constraint_set4_flag << 4)
 458         | (sps->constraint_set5_flag << 5),
 459     .level_idc = sps->level_idc,
 460     .seq_parameter_set_id = sps->id,
 461     .chroma_format_idc = sps->chroma_format_idc,
 462     .bit_depth_luma_minus8 = sps->bit_depth_luma_minus8,
 463     .bit_depth_chroma_minus8 = sps->bit_depth_chroma_minus8,
 464     .log2_max_frame_num_minus4 = sps->log2_max_frame_num_minus4,
 465     .pic_order_cnt_type = sps->pic_order_cnt_type,
 466     .log2_max_pic_order_cnt_lsb_minus4 = sps->log2_max_pic_order_cnt_lsb_minus4,
 467     .max_num_ref_frames = sps->num_ref_frames,
 468     .num_ref_frames_in_pic_order_cnt_cycle = sps->num_ref_frames_in_pic_order_cnt_cycle,
 469     .offset_for_non_ref_pic = sps->offset_for_non_ref_pic,
 470     .offset_for_top_to_bottom_field = sps->offset_for_top_to_bottom_field,
 471     .pic_width_in_mbs_minus1 = sps->pic_width_in_mbs_minus1,
 472     .pic_height_in_map_units_minus1 = sps->pic_height_in_map_units_minus1,
 473     .flags = (sps->separate_colour_plane_flag ? V4L2_H264_SPS_FLAG_SEPARATE_COLOUR_PLANE : 0)
 474         | (sps->qpprime_y_zero_transform_bypass_flag ? V4L2_H264_SPS_FLAG_QPPRIME_Y_ZERO_TRANSFORM_BYPASS : 0)
 475         | (sps->delta_pic_order_always_zero_flag ? V4L2_H264_SPS_FLAG_DELTA_PIC_ORDER_ALWAYS_ZERO : 0)
 476         | (sps->gaps_in_frame_num_value_allowed_flag ? V4L2_H264_SPS_FLAG_GAPS_IN_FRAME_NUM_VALUE_ALLOWED : 0)
 477         | (sps->frame_mbs_only_flag ? V4L2_H264_SPS_FLAG_FRAME_MBS_ONLY : 0)
 478         | (sps->mb_adaptive_frame_field_flag ? V4L2_H264_SPS_FLAG_MB_ADAPTIVE_FRAME_FIELD : 0)
 479         | (sps->direct_8x8_inference_flag ? V4L2_H264_SPS_FLAG_DIRECT_8X8_INFERENCE : 0),
 480   };
 481   /* *INDENT-ON* */
 482
 483   for (i = 0; i < sps->num_ref_frames_in_pic_order_cnt_cycle; i++)
 484     self->sps.offset_for_ref_frame[i] = sps->offset_for_ref_frame[i];
 485 }
 486
 487 static void
 488 gst_v4l2_codec_h264_dec_fill_pps (GstV4l2CodecH264Dec * self, GstH264PPS * pps)
 489 {
 490   /* *INDENT-OFF* */
 491   self->pps = (struct v4l2_ctrl_h264_pps) {
 492     .pic_parameter_set_id = pps->id,
 493     .seq_parameter_set_id = pps->sequence->id,
 494     .num_slice_groups_minus1 = pps->num_slice_groups_minus1,
 495     .num_ref_idx_l0_default_active_minus1 = pps->num_ref_idx_l0_active_minus1,
 496     .num_ref_idx_l1_default_active_minus1 = pps->num_ref_idx_l1_active_minus1,
 497     .weighted_bipred_idc = pps->weighted_bipred_idc,
 498     .pic_init_qp_minus26 = pps->pic_init_qp_minus26,
 499     .pic_init_qs_minus26 = pps->pic_init_qs_minus26,
 500     .chroma_qp_index_offset = pps->chroma_qp_index_offset,
 501     .second_chroma_qp_index_offset = pps->second_chroma_qp_index_offset,
 502     .flags = 0
 503         | (pps->entropy_coding_mode_flag ? V4L2_H264_PPS_FLAG_ENTROPY_CODING_MODE : 0)
 504         | (pps->pic_order_present_flag ? V4L2_H264_PPS_FLAG_BOTTOM_FIELD_PIC_ORDER_IN_FRAME_PRESENT : 0)
 505         | (pps->weighted_pred_flag ? V4L2_H264_PPS_FLAG_WEIGHTED_PRED : 0)
 506         | (pps->deblocking_filter_control_present_flag ? V4L2_H264_PPS_FLAG_DEBLOCKING_FILTER_CONTROL_PRESENT : 0)
 507         | (pps->constrained_intra_pred_flag ? V4L2_H264_PPS_FLAG_CONSTRAINED_INTRA_PRED : 0)
 508         | (pps->redundant_pic_cnt_present_flag ? V4L2_H264_PPS_FLAG_REDUNDANT_PIC_CNT_PRESENT : 0)
 509         | (pps->transform_8x8_mode_flag ? V4L2_H264_PPS_FLAG_TRANSFORM_8X8_MODE : 0)
 510         | (self->scaling_matrix_present ? V4L2_H264_PPS_FLAG_SCALING_MATRIX_PRESENT : 0),
 511   };
 512   /* *INDENT-ON* */
 513 }
 514
 515 static void
 516 gst_v4l2_codec_h264_dec_fill_scaling_matrix (GstV4l2CodecH264Dec * self,
 517     GstH264PPS * pps)
 518 {
 519   gint i, n;
 520
 521   for (i = 0; i < G_N_ELEMENTS (pps->scaling_lists_4x4); i++)
 522     gst_h264_quant_matrix_4x4_get_raster_from_zigzag (self->
 523         scaling_matrix.scaling_list_4x4[i], pps->scaling_lists_4x4[i]);
 524
 525   /* Avoid uninitialize data passed into ioctl() */
 526   memset (self->scaling_matrix.scaling_list_8x8, 0,
 527       sizeof (self->scaling_matrix.scaling_list_8x8));
 528
 529   /* We need the first 2 entries (Y intra and Y inter for YCbCr 4:2:2 and
 530    * less, and the full 6 entries for 4:4:4, see Table 7-2 of the spec for
 531    * more details */
 532   n = (pps->sequence->chroma_format_idc == 3) ? 6 : 2;
 533   for (i = 0; i < n; i++)
 534     gst_h264_quant_matrix_8x8_get_raster_from_zigzag (self->
 535         scaling_matrix.scaling_list_8x8[i], pps->scaling_lists_8x8[i]);
 536 }
 537
 538 static void
 539 gst_v4l2_codec_h264_dec_fill_decoder_params (GstV4l2CodecH264Dec * self,
 540     GstH264SliceHdr * slice_hdr, GstH264Picture * picture, GstH264Dpb * dpb)
 541 {
 542   GArray *refs = gst_h264_dpb_get_pictures_all (dpb);
 543   gint i, entry_id = 0;
 544
 545   /* *INDENT-OFF* */
 546   self->decode_params = (struct v4l2_ctrl_h264_decode_params) {
 547     .nal_ref_idc = picture->nal_ref_idc,
 548     .frame_num = slice_hdr->frame_num,
 549     .idr_pic_id = slice_hdr->idr_pic_id,
 550     .pic_order_cnt_lsb = slice_hdr->pic_order_cnt_lsb,
 551     .delta_pic_order_cnt_bottom = slice_hdr->delta_pic_order_cnt_bottom,
 552     .delta_pic_order_cnt0 = slice_hdr->delta_pic_order_cnt[0],
 553     .delta_pic_order_cnt1 = slice_hdr->delta_pic_order_cnt[1],
 554     .dec_ref_pic_marking_bit_size = slice_hdr->dec_ref_pic_marking.bit_size,
 555     .pic_order_cnt_bit_size = slice_hdr->pic_order_cnt_bit_size,
 556     .slice_group_change_cycle = slice_hdr->slice_group_change_cycle,
 557     .flags = (picture->idr ? V4L2_H264_DECODE_PARAM_FLAG_IDR_PIC : 0) |
 558              (slice_hdr->field_pic_flag ? V4L2_H264_DECODE_PARAM_FLAG_FIELD_PIC : 0) |
 559              (slice_hdr->bottom_field_flag ? V4L2_H264_DECODE_PARAM_FLAG_BOTTOM_FIELD : 0),
 560   };
 561
 562   switch (picture->field) {
 563     case GST_H264_PICTURE_FIELD_FRAME:
 564       self->decode_params.top_field_order_cnt = picture->top_field_order_cnt;
 565       self->decode_params.bottom_field_order_cnt =
 566         picture->bottom_field_order_cnt;
 567       break;
 568     case GST_H264_PICTURE_FIELD_TOP_FIELD:
 569       self->decode_params.top_field_order_cnt = picture->top_field_order_cnt;
 570       self->decode_params.bottom_field_order_cnt = 0;
 571       break;
 572     case GST_H264_PICTURE_FIELD_BOTTOM_FIELD:
 573       self->decode_params.top_field_order_cnt = 0;
 574       self->decode_params.bottom_field_order_cnt =
 575           picture->bottom_field_order_cnt;
 576       break;
 577   }
 578
 579   for (i = 0; i < refs->len; i++) {
 580     GstH264Picture *ref_pic = g_array_index (refs, GstH264Picture *, i);
 581     gint pic_num = ref_pic->pic_num;
 582     gint frame_num = ref_pic->frame_num;
 583     struct v4l2_h264_dpb_entry *entry;
 584
 585     /* Skip non-reference as they are not useful to decoding */
 586     if (!GST_H264_PICTURE_IS_REF (ref_pic))
 587       continue;
 588
 589     /* The second field picture will be handled differently */
 590     if (ref_pic->second_field)
 591       continue;
 592
 593     /* V4L2 uAPI uses pic_num for both PicNum and LongTermPicNum, and
 594      * frame_num for both FrameNum and LongTermFrameIdx */
 595     if (GST_H264_PICTURE_IS_LONG_TERM_REF (ref_pic)) {
 596       pic_num = ref_pic->long_term_pic_num;
 597       frame_num = ref_pic->long_term_frame_idx;
 598     }
 599
 600     entry = &self->decode_params.dpb[entry_id++];
 601     *entry = (struct v4l2_h264_dpb_entry) {
 602       /*
 603        * The reference is multiplied by 1000 because it's was set as micro
 604        * seconds and this TS is nanosecond.
 605        */
 606       .reference_ts = (guint64) ref_pic->system_frame_number * 1000,
 607       .frame_num = frame_num,
 608       .pic_num = pic_num,
 609       .flags = V4L2_H264_DPB_ENTRY_FLAG_VALID
 610           | (GST_H264_PICTURE_IS_REF (ref_pic) ? V4L2_H264_DPB_ENTRY_FLAG_ACTIVE : 0)
 611           | (GST_H264_PICTURE_IS_LONG_TERM_REF (ref_pic) ? V4L2_H264_DPB_ENTRY_FLAG_LONG_TERM : 0),
 612     };
 613
 614     switch (ref_pic->field) {
 615       case GST_H264_PICTURE_FIELD_FRAME:
 616         entry->top_field_order_cnt = ref_pic->top_field_order_cnt;
 617         entry->bottom_field_order_cnt = ref_pic->bottom_field_order_cnt;
 618         entry->fields = V4L2_H264_FRAME_REF;
 619         break;
 620       case GST_H264_PICTURE_FIELD_TOP_FIELD:
 621         entry->top_field_order_cnt = ref_pic->top_field_order_cnt;
 622         entry->fields = V4L2_H264_TOP_FIELD_REF;
 623
 624         if (ref_pic->other_field) {
 625           entry->bottom_field_order_cnt =
 626               ref_pic->other_field->bottom_field_order_cnt;
 627           entry->fields |= V4L2_H264_BOTTOM_FIELD_REF;
 628         } else {
 629           entry->flags |= V4L2_H264_DPB_ENTRY_FLAG_FIELD;
 630         }
 631         break;
 632       case GST_H264_PICTURE_FIELD_BOTTOM_FIELD:
 633         entry->bottom_field_order_cnt = ref_pic->bottom_field_order_cnt;
 634         entry->fields = V4L2_H264_BOTTOM_FIELD_REF;
 635
 636         if (ref_pic->other_field) {
 637           entry->top_field_order_cnt =
 638             ref_pic->other_field->top_field_order_cnt;
 639           entry->fields |= V4L2_H264_TOP_FIELD_REF;
 640         } else {
 641           entry->flags |= V4L2_H264_DPB_ENTRY_FLAG_FIELD;
 642         }
 643         break;
 644     }
 645   }
 646   /* *INDENT-ON* */
 647
 648   g_array_unref (refs);
 649 }
 650
 651 static void
 652 gst_v4l2_codec_h264_dec_fill_pred_weight (GstV4l2CodecH264Dec * self,
 653     GstH264SliceHdr * slice_hdr)
 654 {
 655   gint i, j;
 656
 657   /* *INDENT-OFF* */
 658   self->pred_weight = (struct v4l2_ctrl_h264_pred_weights) {
 659     .luma_log2_weight_denom = slice_hdr->pred_weight_table.luma_log2_weight_denom,
 660     .chroma_log2_weight_denom = slice_hdr->pred_weight_table.chroma_log2_weight_denom,
 661   };
 662   /* *INDENT-ON* */
 663
 664   for (i = 0; i <= slice_hdr->num_ref_idx_l0_active_minus1; i++) {
 665     self->pred_weight.weight_factors[0].luma_weight[i] =
 666         slice_hdr->pred_weight_table.luma_weight_l0[i];
 667     self->pred_weight.weight_factors[0].luma_offset[i] =
 668         slice_hdr->pred_weight_table.luma_offset_l0[i];
 669   }
 670
 671   if (slice_hdr->pps->sequence->chroma_array_type != 0) {
 672     for (i = 0; i <= slice_hdr->num_ref_idx_l0_active_minus1; i++) {
 673       for (j = 0; j < 2; j++) {
 674         self->pred_weight.weight_factors[0].chroma_weight[i][j] =
 675             slice_hdr->pred_weight_table.chroma_weight_l0[i][j];
 676         self->pred_weight.weight_factors[0].chroma_offset[i][j] =
 677             slice_hdr->pred_weight_table.chroma_offset_l0[i][j];
 678       }
 679     }
 680   }
 681
 682   /* Skip l1 if this is not a B-Frames. */
 683   if (slice_hdr->type % 5 != GST_H264_B_SLICE)
 684     return;
 685
 686   for (i = 0; i <= slice_hdr->num_ref_idx_l1_active_minus1; i++) {
 687     self->pred_weight.weight_factors[1].luma_weight[i] =
 688         slice_hdr->pred_weight_table.luma_weight_l1[i];
 689     self->pred_weight.weight_factors[1].luma_offset[i] =
 690         slice_hdr->pred_weight_table.luma_offset_l1[i];
 691   }
 692
 693   if (slice_hdr->pps->sequence->chroma_array_type != 0) {
 694     for (i = 0; i <= slice_hdr->num_ref_idx_l1_active_minus1; i++) {
 695       for (j = 0; j < 2; j++) {
 696         self->pred_weight.weight_factors[1].chroma_weight[i][j] =
 697             slice_hdr->pred_weight_table.chroma_weight_l1[i][j];
 698         self->pred_weight.weight_factors[1].chroma_offset[i][j] =
 699             slice_hdr->pred_weight_table.chroma_offset_l1[i][j];
 700       }
 701     }
 702   }
 703 }
 704
 705 static guint
 706 get_slice_header_bit_size (GstH264Slice * slice)
 707 {
 708   return 8 * slice->nalu.header_bytes + slice->header.header_size
 709       - 8 * slice->header.n_emulation_prevention_bytes;
 710 }
 711
 712 static void
 713 gst_v4l2_codec_h264_dec_fill_slice_params (GstV4l2CodecH264Dec * self,
 714     GstH264Slice * slice)
 715 {
 716   gint n = self->num_slices++;
 717   gsize slice_size = slice->nalu.size;
 718   struct v4l2_ctrl_h264_slice_params *params;
 719
 720   /* Ensure array is large enough */
 721   if (self->slice_params->len < self->num_slices)
 722     g_array_set_size (self->slice_params, self->slice_params->len * 2);
 723
 724   if (needs_start_codes (self))
 725     slice_size += 3;
 726
 727   /* *INDENT-OFF* */
 728   params = &g_array_index (self->slice_params, struct v4l2_ctrl_h264_slice_params, n);
 729   *params = (struct v4l2_ctrl_h264_slice_params) {
 730     .header_bit_size = get_slice_header_bit_size (slice),
 731     .first_mb_in_slice = slice->header.first_mb_in_slice,
 732     .slice_type = slice->header.type % 5,
 733     .colour_plane_id = slice->header.colour_plane_id,
 734     .redundant_pic_cnt = slice->header.redundant_pic_cnt,
 735     .cabac_init_idc = slice->header.cabac_init_idc,
 736     .slice_qp_delta = slice->header.slice_qp_delta,
 737     .slice_qs_delta = slice->header.slice_qs_delta,
 738     .disable_deblocking_filter_idc = slice->header.disable_deblocking_filter_idc,
 739     .slice_alpha_c0_offset_div2 = slice->header.slice_alpha_c0_offset_div2,
 740     .slice_beta_offset_div2 = slice->header.slice_beta_offset_div2,
 741     .num_ref_idx_l0_active_minus1 = slice->header.num_ref_idx_l0_active_minus1,
 742     .num_ref_idx_l1_active_minus1 = slice->header.num_ref_idx_l1_active_minus1,
 743     .flags = (slice->header.direct_spatial_mv_pred_flag ? V4L2_H264_SLICE_FLAG_DIRECT_SPATIAL_MV_PRED : 0) |
 744              (slice->header.sp_for_switch_flag ? V4L2_H264_SLICE_FLAG_SP_FOR_SWITCH : 0),
 745   };
 746   /* *INDENT-ON* */
 747 }
 748
 749 static guint8
 750 lookup_dpb_index (struct v4l2_h264_dpb_entry dpb[16], GstH264Picture * ref_pic)
 751 {
 752   guint64 ref_ts;
 753   gint i;
 754
 755   /* Reference list may have wholes in case a ref is missing, we should mark
 756    * the whole and avoid moving items in the list */
 757   if (!ref_pic)
 758     return 0xff;
 759
 760   /* DPB entries only stores first field in a merged fashion */
 761   if (ref_pic->second_field && ref_pic->other_field)
 762     ref_pic = ref_pic->other_field;
 763
 764   ref_ts = (guint64) ref_pic->system_frame_number * 1000;
 765   for (i = 0; i < 16; i++) {
 766     if (dpb[i].flags & V4L2_H264_DPB_ENTRY_FLAG_ACTIVE
 767         && dpb[i].reference_ts == ref_ts)
 768       return i;
 769   }
 770
 771   return 0xff;
 772 }
 773
 774 static guint
 775 _get_v4l2_fields_ref (GstH264Picture * ref_pic, gboolean merge)
 776 {
 777   if (merge && ref_pic->other_field)
 778     return V4L2_H264_FRAME_REF;
 779
 780   switch (ref_pic->field) {
 781     case GST_H264_PICTURE_FIELD_FRAME:
 782       return V4L2_H264_FRAME_REF;
 783       break;
 784     case GST_H264_PICTURE_FIELD_TOP_FIELD:
 785       return V4L2_H264_TOP_FIELD_REF;
 786       break;
 787     case GST_H264_PICTURE_FIELD_BOTTOM_FIELD:
 788       return V4L2_H264_BOTTOM_FIELD_REF;
 789       break;
 790   }
 791
 792   return V4L2_H264_FRAME_REF;
 793 }
 794
 795 static void
 796 gst_v4l2_codec_h264_dec_fill_references (GstV4l2CodecH264Dec * self,
 797     gboolean cur_is_frame, GArray * ref_pic_list0, GArray * ref_pic_list1)
 798 {
 799   struct v4l2_ctrl_h264_slice_params *slice_params;
 800   gint i;
 801
 802   slice_params = &g_array_index (self->slice_params,
 803       struct v4l2_ctrl_h264_slice_params, 0);
 804
 805   memset (slice_params->ref_pic_list0, 0xff,
 806       sizeof (slice_params->ref_pic_list0));
 807   memset (slice_params->ref_pic_list1, 0xff,
 808       sizeof (slice_params->ref_pic_list1));
 809
 810   for (i = 0; i < ref_pic_list0->len; i++) {
 811     GstH264Picture *ref_pic =
 812         g_array_index (ref_pic_list0, GstH264Picture *, i);
 813     slice_params->ref_pic_list0[i].index =
 814         lookup_dpb_index (self->decode_params.dpb, ref_pic);
 815     slice_params->ref_pic_list0[i].fields =
 816         _get_v4l2_fields_ref (ref_pic, cur_is_frame);
 817   }
 818
 819   for (i = 0; i < ref_pic_list1->len; i++) {
 820     GstH264Picture *ref_pic =
 821         g_array_index (ref_pic_list1, GstH264Picture *, i);
 822     slice_params->ref_pic_list1[i].index =
 823         lookup_dpb_index (self->decode_params.dpb, ref_pic);
 824     slice_params->ref_pic_list1[i].fields =
 825         _get_v4l2_fields_ref (ref_pic, cur_is_frame);
 826   }
 827 }
 828
 829 static GstFlowReturn
 830 gst_v4l2_codec_h264_dec_new_sequence (GstH264Decoder * decoder,
 831     const GstH264SPS * sps, gint max_dpb_size)
 832 {
 833   GstV4l2CodecH264Dec *self = GST_V4L2_CODEC_H264_DEC (decoder);
 834   gint crop_width = sps->width;
 835   gint crop_height = sps->height;
 836   gboolean negotiation_needed = FALSE;
 837   gboolean interlaced;
 838
 839   if (self->vinfo.finfo->format == GST_VIDEO_FORMAT_UNKNOWN)
 840     negotiation_needed = TRUE;
 841
 842   /* TODO check if CREATE_BUFS is supported, and simply grow the pool */
 843   if (self->min_pool_size < max_dpb_size) {
 844     self->min_pool_size = max_dpb_size;
 845     negotiation_needed = TRUE;
 846   }
 847
 848   if (sps->frame_cropping_flag) {
 849     crop_width = sps->crop_rect_width;
 850     crop_height = sps->crop_rect_height;
 851   }
 852
 853   /* TODO Check if current buffers are large enough, and reuse them */
 854   if (self->display_width != crop_width || self->display_height != crop_height
 855       || self->coded_width != sps->width || self->coded_height != sps->height) {
 856     self->display_width = crop_width;
 857     self->display_height = crop_height;
 858     self->coded_width = sps->width;
 859     self->coded_height = sps->height;
 860     negotiation_needed = TRUE;
 861     GST_INFO_OBJECT (self, "Resolution changed to %dx%d (%ix%i)",
 862         self->display_width, self->display_height,
 863         self->coded_width, self->coded_height);
 864   }
 865
 866   interlaced = !sps->frame_mbs_only_flag;
 867   if (self->interlaced != interlaced) {
 868     self->interlaced = interlaced;
 869
 870     negotiation_needed = TRUE;
 871     GST_INFO_OBJECT (self, "Interlaced mode changed to %d", interlaced);
 872   }
 873
 874   if (self->bitdepth != sps->bit_depth_luma_minus8 + 8) {
 875     self->bitdepth = sps->bit_depth_luma_minus8 + 8;
 876     negotiation_needed = TRUE;
 877     GST_INFO_OBJECT (self, "Bitdepth changed to %u", self->bitdepth);
 878   }
 879
 880   if (self->chroma_format_idc != sps->chroma_format_idc) {
 881     self->chroma_format_idc = sps->chroma_format_idc;
 882     negotiation_needed = TRUE;
 883     GST_INFO_OBJECT (self, "Chroma format changed to %i",
 884         self->chroma_format_idc);
 885   }
 886
 887   gst_v4l2_codec_h264_dec_fill_sequence (self, sps);
 888   self->need_sequence = TRUE;
 889
 890   if (negotiation_needed) {
 891     self->need_negotiation = TRUE;
 892     if (!gst_video_decoder_negotiate (GST_VIDEO_DECODER (self))) {
 893       GST_ERROR_OBJECT (self, "Failed to negotiate with downstream");
 894       return GST_FLOW_NOT_NEGOTIATED;
 895     }
 896   }
 897
 898   /* Check if we can zero-copy buffers */
 899   if (!self->has_videometa) {
 900     GstVideoInfo ref_vinfo;
 901     gint i;
 902
 903     gst_video_info_set_format (&ref_vinfo, GST_VIDEO_INFO_FORMAT (&self->vinfo),
 904         self->display_width, self->display_height);
 905
 906     for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&self->vinfo); i++) {
 907       if (self->vinfo.stride[i] != ref_vinfo.stride[i] ||
 908           self->vinfo.offset[i] != ref_vinfo.offset[i]) {
 909         GST_WARNING_OBJECT (self,
 910             "GstVideoMeta support required, copying frames.");
 911         self->copy_frames = TRUE;
 912         break;
 913       }
 914     }
 915   } else {
 916     self->copy_frames = FALSE;
 917   }
 918
 919   return GST_FLOW_OK;
 920 }
 921
 922 static gboolean
 923 gst_v4l2_codec_h264_dec_ensure_bitstream (GstV4l2CodecH264Dec * self)
 924 {
 925   if (self->bitstream)
 926     goto done;
 927
 928   self->bitstream = gst_v4l2_codec_allocator_alloc (self->sink_allocator);
 929
 930   if (!self->bitstream) {
 931     GST_ELEMENT_ERROR (self, RESOURCE, NO_SPACE_LEFT,
 932         ("Not enough memory to decode H264 stream."), (NULL));
 933     return FALSE;
 934   }
 935
 936   if (!gst_memory_map (self->bitstream, &self->bitstream_map, GST_MAP_WRITE)) {
 937     GST_ELEMENT_ERROR (self, RESOURCE, WRITE,
 938         ("Could not access bitstream memory for writing"), (NULL));
 939     g_clear_pointer (&self->bitstream, gst_memory_unref);
 940     return FALSE;
 941   }
 942
 943 done:
 944   /* We use this field to track how much we have written */
 945   self->bitstream_map.size = 0;
 946
 947   return TRUE;
 948 }
 949
 950 static GstFlowReturn
 951 gst_v4l2_codec_h264_dec_start_picture (GstH264Decoder * decoder,
 952     GstH264Picture * picture, GstH264Slice * slice, GstH264Dpb * dpb)
 953 {
 954   GstV4l2CodecH264Dec *self = GST_V4L2_CODEC_H264_DEC (decoder);
 955
 956   /* FIXME base class should not call us if negotiation failed */
 957   if (!self->sink_allocator)
 958     return GST_FLOW_NOT_NEGOTIATED;
 959
 960   if (!gst_v4l2_codec_h264_dec_ensure_bitstream (self))
 961     return GST_FLOW_ERROR;
 962
 963   /*
 964    * Scaling matrix is present if there's one provided
 965    * by either the SPS or the PPS. This flag must be
 966    * set to true or false, before filling the PPS V4L2 control.
 967    */
 968   self->scaling_matrix_present =
 969       slice->header.pps->sequence->scaling_matrix_present_flag ||
 970       slice->header.pps->pic_scaling_matrix_present_flag;
 971
 972   gst_v4l2_codec_h264_dec_fill_pps (self, slice->header.pps);
 973
 974   if (self->scaling_matrix_present)
 975     gst_v4l2_codec_h264_dec_fill_scaling_matrix (self, slice->header.pps);
 976
 977   gst_v4l2_codec_h264_dec_fill_decoder_params (self, &slice->header, picture,
 978       dpb);
 979
 980   self->first_slice = TRUE;
 981
 982   return GST_FLOW_OK;
 983 }
 984
 985 static gboolean
 986 gst_v4l2_codec_h264_dec_copy_output_buffer (GstV4l2CodecH264Dec * self,
 987     GstVideoCodecFrame * codec_frame)
 988 {
 989   GstVideoFrame src_frame;
 990   GstVideoFrame dest_frame;
 991   GstVideoInfo dest_vinfo;
 992   GstBuffer *buffer;
 993
 994   gst_video_info_set_format (&dest_vinfo, GST_VIDEO_INFO_FORMAT (&self->vinfo),
 995       self->display_width, self->display_height);
 996
 997   buffer = gst_video_decoder_allocate_output_buffer (GST_VIDEO_DECODER (self));
 998   if (!buffer)
 999     goto fail;
1000
1001   if (!gst_video_frame_map (&src_frame, &self->vinfo,
1002           codec_frame->output_buffer, GST_MAP_READ))
1003     goto fail;
1004
1005   if (!gst_video_frame_map (&dest_frame, &dest_vinfo, buffer, GST_MAP_WRITE)) {
1006     gst_video_frame_unmap (&dest_frame);
1007     goto fail;
1008   }
1009
1010   /* gst_video_frame_copy can crop this, but does not know, so let make it
1011    * think it's all right */
1012   GST_VIDEO_INFO_WIDTH (&src_frame.info) = self->display_width;
1013   GST_VIDEO_INFO_HEIGHT (&src_frame.info) = self->display_height;
1014
1015   if (!gst_video_frame_copy (&dest_frame, &src_frame)) {
1016     gst_video_frame_unmap (&src_frame);
1017     gst_video_frame_unmap (&dest_frame);
1018     goto fail;
1019   }
1020
1021   gst_video_frame_unmap (&src_frame);
1022   gst_video_frame_unmap (&dest_frame);
1023   gst_buffer_replace (&codec_frame->output_buffer, buffer);
1024   gst_buffer_unref (buffer);
1025
1026   return TRUE;
1027
1028 fail:
1029   GST_ERROR_OBJECT (self, "Failed copy output buffer.");
1030   return FALSE;
1031 }
1032
1033 static GstFlowReturn
1034 gst_v4l2_codec_h264_dec_output_picture (GstH264Decoder * decoder,
1035     GstVideoCodecFrame * frame, GstH264Picture * picture)
1036 {
1037   GstV4l2CodecH264Dec *self = GST_V4L2_CODEC_H264_DEC (decoder);
1038   GstVideoDecoder *vdec = GST_VIDEO_DECODER (decoder);
1039   GstV4l2Request *request = gst_h264_picture_get_user_data (picture);
1040   gint ret;
1041
1042   GST_DEBUG_OBJECT (self, "Output picture %u", picture->system_frame_number);
1043
1044   ret = gst_v4l2_request_set_done (request);
1045   if (ret == 0) {
1046     GST_ELEMENT_ERROR (self, STREAM, DECODE,
1047         ("Decoding frame %u took too long", picture->system_frame_number),
1048         (NULL));
1049     goto error;
1050   } else if (ret < 0) {
1051     GST_ELEMENT_ERROR (self, STREAM, DECODE,
1052         ("Decoding request failed: %s", g_strerror (errno)), (NULL));
1053     goto error;
1054   }
1055   g_return_val_if_fail (frame->output_buffer, GST_FLOW_ERROR);
1056
1057   if (gst_v4l2_request_failed (request)) {
1058     GST_ELEMENT_ERROR (self, STREAM, DECODE,
1059         ("Failed to decode frame %u", picture->system_frame_number), (NULL));
1060     goto error;
1061   }
1062
1063   /* Hold on reference buffers for the rest of the picture lifetime */
1064   gst_h264_picture_set_user_data (picture,
1065       gst_buffer_ref (frame->output_buffer), (GDestroyNotify) gst_buffer_unref);
1066
1067   if (self->copy_frames)
1068     gst_v4l2_codec_h264_dec_copy_output_buffer (self, frame);
1069
1070   gst_h264_picture_unref (picture);
1071
1072   return gst_video_decoder_finish_frame (vdec, frame);
1073
1074 error:
1075   gst_video_decoder_drop_frame (vdec, frame);
1076   gst_h264_picture_unref (picture);
1077
1078   return GST_FLOW_ERROR;
1079 }
1080
1081 static void
1082 gst_v4l2_codec_h264_dec_reset_picture (GstV4l2CodecH264Dec * self)
1083 {
1084   if (self->bitstream) {
1085     if (self->bitstream_map.memory)
1086       gst_memory_unmap (self->bitstream, &self->bitstream_map);
1087     g_clear_pointer (&self->bitstream, gst_memory_unref);
1088     self->bitstream_map = (GstMapInfo) GST_MAP_INFO_INIT;
1089   }
1090
1091   self->num_slices = 0;
1092 }
1093
1094 static gboolean
1095 gst_v4l2_codec_h264_dec_ensure_output_buffer (GstV4l2CodecH264Dec * self,
1096     GstVideoCodecFrame * frame)
1097 {
1098   GstBuffer *buffer;
1099   GstFlowReturn flow_ret;
1100
1101   if (frame->output_buffer)
1102     return TRUE;
1103
1104   flow_ret = gst_buffer_pool_acquire_buffer (GST_BUFFER_POOL (self->src_pool),
1105       &buffer, NULL);
1106   if (flow_ret != GST_FLOW_OK) {
1107     if (flow_ret == GST_FLOW_FLUSHING)
1108       GST_DEBUG_OBJECT (self, "Frame decoding aborted, we are flushing.");
1109     else
1110       GST_ELEMENT_ERROR (self, RESOURCE, WRITE,
1111           ("No more picture buffer available."), (NULL));
1112     return FALSE;
1113   }
1114
1115   frame->output_buffer = buffer;
1116   return TRUE;
1117 }
1118
1119 static gboolean
1120 gst_v4l2_codec_h264_dec_submit_bitstream (GstV4l2CodecH264Dec * self,
1121     GstH264Picture * picture, guint flags)
1122 {
1123   GstV4l2Request *prev_request, *request = NULL;
1124   gsize bytesused;
1125   gboolean ret = FALSE;
1126   guint count = 0;
1127
1128   /* *INDENT-OFF* */
1129   /* Reserve space for controls */
1130   struct v4l2_ext_control control[] = {
1131     { }, /* SPS */
1132     { }, /* PPS */
1133     { }, /* DECODE_PARAMS */
1134     { }, /* SLICE_PARAMS */
1135     { }, /* SCALING_MATRIX */
1136     { }, /* PRED_WEIGHTS */
1137   };
1138   /* *INDENT-ON* */
1139
1140   prev_request = gst_h264_picture_get_user_data (picture);
1141
1142   bytesused = self->bitstream_map.size;
1143   gst_memory_unmap (self->bitstream, &self->bitstream_map);
1144   self->bitstream_map = (GstMapInfo) GST_MAP_INFO_INIT;
1145   gst_memory_resize (self->bitstream, 0, bytesused);
1146
1147   if (prev_request) {
1148     request = gst_v4l2_decoder_alloc_sub_request (self->decoder, prev_request,
1149         self->bitstream);
1150   } else {
1151     GstVideoCodecFrame *frame;
1152
1153     frame = gst_video_decoder_get_frame (GST_VIDEO_DECODER (self),
1154         picture->system_frame_number);
1155     g_return_val_if_fail (frame, FALSE);
1156
1157     if (!gst_v4l2_codec_h264_dec_ensure_output_buffer (self, frame))
1158       goto done;
1159
1160     request = gst_v4l2_decoder_alloc_request (self->decoder,
1161         picture->system_frame_number, self->bitstream, frame->output_buffer);
1162
1163     gst_video_codec_frame_unref (frame);
1164   }
1165
1166   if (!request) {
1167     GST_ELEMENT_ERROR (self, RESOURCE, NO_SPACE_LEFT,
1168         ("Failed to allocate a media request object."), (NULL));
1169     goto done;
1170   }
1171
1172   if (self->need_sequence) {
1173     control[count].id = V4L2_CID_STATELESS_H264_SPS;
1174     control[count].ptr = &self->sps;
1175     control[count].size = sizeof (self->sps);
1176     count++;
1177     self->need_sequence = FALSE;
1178   }
1179
1180   if (self->first_slice) {
1181     control[count].id = V4L2_CID_STATELESS_H264_PPS;
1182     control[count].ptr = &self->pps;
1183     control[count].size = sizeof (self->pps);
1184     count++;
1185
1186     if (self->scaling_matrix_present) {
1187       control[count].id = V4L2_CID_STATELESS_H264_SCALING_MATRIX;
1188       control[count].ptr = &self->scaling_matrix;
1189       control[count].size = sizeof (self->scaling_matrix);
1190       count++;
1191     }
1192
1193     control[count].id = V4L2_CID_STATELESS_H264_DECODE_PARAMS;
1194     control[count].ptr = &self->decode_params;
1195     control[count].size = sizeof (self->decode_params);
1196     count++;
1197
1198     self->first_slice = FALSE;
1199   }
1200
1201   /* If it's not slice-based then it doesn't support per-slice controls. */
1202   if (is_slice_based (self)) {
1203     control[count].id = V4L2_CID_STATELESS_H264_SLICE_PARAMS;
1204     control[count].ptr = self->slice_params->data;
1205     control[count].size = g_array_get_element_size (self->slice_params)
1206         * self->num_slices;
1207     count++;
1208
1209     control[count].id = V4L2_CID_STATELESS_H264_PRED_WEIGHTS;
1210     control[count].ptr = &self->pred_weight;
1211     control[count].size = sizeof (self->pred_weight);
1212     count++;
1213   }
1214
1215   if (!gst_v4l2_decoder_set_controls (self->decoder, request, control, count)) {
1216     GST_ELEMENT_ERROR (self, RESOURCE, WRITE,
1217         ("Driver did not accept the bitstream parameters."), (NULL));
1218     goto done;
1219   }
1220
1221   if (!gst_v4l2_request_queue (request, flags)) {
1222     GST_ELEMENT_ERROR (self, RESOURCE, WRITE,
1223         ("Driver did not accept the decode request."), (NULL));
1224     goto done;
1225   }
1226
1227   gst_h264_picture_set_user_data (picture, g_steal_pointer (&request),
1228       (GDestroyNotify) gst_v4l2_request_unref);
1229   ret = TRUE;
1230
1231 done:
1232   if (request)
1233     gst_v4l2_request_unref (request);
1234
1235   gst_v4l2_codec_h264_dec_reset_picture (self);
1236
1237   return ret;
1238 }
1239
1240 static GstFlowReturn
1241 gst_v4l2_codec_h264_dec_decode_slice (GstH264Decoder * decoder,
1242     GstH264Picture * picture, GstH264Slice * slice, GArray * ref_pic_list0,
1243     GArray * ref_pic_list1)
1244 {
1245   GstV4l2CodecH264Dec *self = GST_V4L2_CODEC_H264_DEC (decoder);
1246   gsize sc_off = 0;
1247   gsize nal_size;
1248   guint8 *bitstream_data;
1249
1250   if (is_slice_based (self)) {
1251     if (self->bitstream_map.size) {
1252       /* In slice mode, we submit the pending slice asking the accelerator to
1253        * hold on the picture */
1254       if (!gst_v4l2_codec_h264_dec_submit_bitstream (self, picture,
1255               V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF)
1256           || !gst_v4l2_codec_h264_dec_ensure_bitstream (self))
1257         return GST_FLOW_ERROR;
1258     }
1259
1260     gst_v4l2_codec_h264_dec_fill_slice_params (self, slice);
1261     gst_v4l2_codec_h264_dec_fill_pred_weight (self, &slice->header);
1262     gst_v4l2_codec_h264_dec_fill_references (self,
1263         GST_H264_PICTURE_IS_FRAME (picture), ref_pic_list0, ref_pic_list1);
1264   }
1265
1266   bitstream_data = self->bitstream_map.data + self->bitstream_map.size;
1267
1268   if (needs_start_codes (self))
1269     sc_off = 3;
1270   nal_size = sc_off + slice->nalu.size;
1271
1272   if (self->bitstream_map.size + nal_size > self->bitstream_map.maxsize) {
1273     GST_ELEMENT_ERROR (decoder, RESOURCE, NO_SPACE_LEFT,
1274         ("Not enough space to send all slice of an H264 frame."), (NULL));
1275     return GST_FLOW_ERROR;
1276   }
1277
1278   if (needs_start_codes (self)) {
1279     bitstream_data[0] = 0x00;
1280     bitstream_data[1] = 0x00;
1281     bitstream_data[2] = 0x01;
1282   }
1283
1284   memcpy (bitstream_data + sc_off, slice->nalu.data + slice->nalu.offset,
1285       slice->nalu.size);
1286   self->bitstream_map.size += nal_size;
1287
1288   return GST_FLOW_OK;
1289 }
1290
1291 static GstFlowReturn
1292 gst_v4l2_codec_h264_dec_end_picture (GstH264Decoder * decoder,
1293     GstH264Picture * picture)
1294 {
1295   GstV4l2CodecH264Dec *self = GST_V4L2_CODEC_H264_DEC (decoder);
1296   guint flags = 0;
1297
1298   /* Hold on the output frame if this is first field of a pair */
1299   if (picture->field != GST_H264_PICTURE_FIELD_FRAME && !picture->second_field)
1300     flags = V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF;
1301
1302   if (!gst_v4l2_codec_h264_dec_submit_bitstream (self, picture, flags))
1303     return GST_FLOW_ERROR;
1304
1305   return GST_FLOW_OK;
1306 }
1307
1308 static GstFlowReturn
1309 gst_v4l2_codec_h264_dec_new_field_picture (GstH264Decoder * decoder,
1310     GstH264Picture * first_field, GstH264Picture * second_field)
1311 {
1312   GstV4l2CodecH264Dec *self = GST_V4L2_CODEC_H264_DEC (decoder);
1313   GstV4l2Request *request = gst_h264_picture_get_user_data (first_field);
1314
1315   if (!request) {
1316     GST_WARNING_OBJECT (self,
1317         "First picture does not have an associated request");
1318     return GST_FLOW_OK;
1319   }
1320
1321   GST_DEBUG_OBJECT (self, "Assigned request %p to second field.", request);
1322
1323   /* Associate the previous request with the new picture so that
1324    * submit_bitstream can create sub-request */
1325   gst_h264_picture_set_user_data (second_field, gst_v4l2_request_ref (request),
1326       (GDestroyNotify) gst_v4l2_request_unref);
1327
1328   return GST_FLOW_OK;
1329 }
1330
1331 static guint
1332 gst_v4l2_codec_h264_dec_get_preferred_output_delay (GstH264Decoder * decoder,
1333     gboolean live)
1334 {
1335   GstV4l2CodecH264Dec *self = GST_V4L2_CODEC_H264_DEC (decoder);
1336   guint delay;
1337
1338   if (live)
1339     delay = 0;
1340   else
1341     /* Just one for now, perhaps we can make this configurable in the future. */
1342     delay = 1;
1343
1344   gst_v4l2_decoder_set_render_delay (self->decoder, delay);
1345
1346   return delay;
1347 }
1348
1349 static void
1350 gst_v4l2_codec_h264_dec_set_flushing (GstV4l2CodecH264Dec * self,
1351     gboolean flushing)
1352 {
1353   if (self->sink_allocator)
1354     gst_v4l2_codec_allocator_set_flushing (self->sink_allocator, flushing);
1355   if (self->src_allocator)
1356     gst_v4l2_codec_allocator_set_flushing (self->src_allocator, flushing);
1357 }
1358
1359 static gboolean
1360 gst_v4l2_codec_h264_dec_flush (GstVideoDecoder * decoder)
1361 {
1362   GstV4l2CodecH264Dec *self = GST_V4L2_CODEC_H264_DEC (decoder);
1363
1364   GST_DEBUG_OBJECT (self, "Flushing decoder state.");
1365
1366   gst_v4l2_decoder_flush (self->decoder);
1367   gst_v4l2_codec_h264_dec_set_flushing (self, FALSE);
1368
1369   return GST_VIDEO_DECODER_CLASS (parent_class)->flush (decoder);
1370 }
1371
1372 static gboolean
1373 gst_v4l2_codec_h264_dec_sink_event (GstVideoDecoder * decoder, GstEvent * event)
1374 {
1375   GstV4l2CodecH264Dec *self = GST_V4L2_CODEC_H264_DEC (decoder);
1376
1377   switch (GST_EVENT_TYPE (event)) {
1378     case GST_EVENT_FLUSH_START:
1379       GST_DEBUG_OBJECT (self, "flush start");
1380       gst_v4l2_codec_h264_dec_set_flushing (self, TRUE);
1381       break;
1382     default:
1383       break;
1384   }
1385
1386   return GST_VIDEO_DECODER_CLASS (parent_class)->sink_event (decoder, event);
1387 }
1388
1389 static GstStateChangeReturn
1390 gst_v4l2_codec_h264_dec_change_state (GstElement * element,
1391     GstStateChange transition)
1392 {
1393   GstV4l2CodecH264Dec *self = GST_V4L2_CODEC_H264_DEC (element);
1394
1395   if (transition == GST_STATE_CHANGE_PAUSED_TO_READY)
1396     gst_v4l2_codec_h264_dec_set_flushing (self, TRUE);
1397
1398   return GST_ELEMENT_CLASS (parent_class)->change_state (element, transition);
1399 }
1400
1401 static void
1402 gst_v4l2_codec_h264_dec_set_property (GObject * object, guint prop_id,
1403     const GValue * value, GParamSpec * pspec)
1404 {
1405   GstV4l2CodecH264Dec *self = GST_V4L2_CODEC_H264_DEC (object);
1406   GObject *dec = G_OBJECT (self->decoder);
1407
1408   switch (prop_id) {
1409     default:
1410       gst_v4l2_decoder_set_property (dec, prop_id - PROP_LAST, value, pspec);
1411       break;
1412   }
1413 }
1414
1415 static void
1416 gst_v4l2_codec_h264_dec_get_property (GObject * object, guint prop_id,
1417     GValue * value, GParamSpec * pspec)
1418 {
1419   GstV4l2CodecH264Dec *self = GST_V4L2_CODEC_H264_DEC (object);
1420   GObject *dec = G_OBJECT (self->decoder);
1421
1422   switch (prop_id) {
1423     default:
1424       gst_v4l2_decoder_get_property (dec, prop_id - PROP_LAST, value, pspec);
1425       break;
1426   }
1427 }
1428
1429 static void
1430 gst_v4l2_codec_h264_dec_init (GstV4l2CodecH264Dec * self)
1431 {
1432 }
1433
1434 static void
1435 gst_v4l2_codec_h264_dec_subinit (GstV4l2CodecH264Dec * self,
1436     GstV4l2CodecH264DecClass * klass)
1437 {
1438   self->decoder = gst_v4l2_decoder_new (klass->device);
1439   gst_video_info_init (&self->vinfo);
1440   self->slice_params = g_array_sized_new (FALSE, TRUE,
1441       sizeof (struct v4l2_ctrl_h264_slice_params), 4);
1442 }
1443
1444 static void
1445 gst_v4l2_codec_h264_dec_dispose (GObject * object)
1446 {
1447   GstV4l2CodecH264Dec *self = GST_V4L2_CODEC_H264_DEC (object);
1448
1449   g_clear_object (&self->decoder);
1450   g_clear_pointer (&self->slice_params, g_array_unref);
1451
1452   G_OBJECT_CLASS (parent_class)->dispose (object);
1453 }
1454
1455 static void
1456 gst_v4l2_codec_h264_dec_class_init (GstV4l2CodecH264DecClass * klass)
1457 {
1458 }
1459
1460 static void
1461 gst_v4l2_codec_h264_dec_subclass_init (GstV4l2CodecH264DecClass * klass,
1462     GstV4l2CodecDevice * device)
1463 {
1464   GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
1465   GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
1466   GstVideoDecoderClass *decoder_class = GST_VIDEO_DECODER_CLASS (klass);
1467   GstH264DecoderClass *h264decoder_class = GST_H264_DECODER_CLASS (klass);
1468
1469   gobject_class->set_property = gst_v4l2_codec_h264_dec_set_property;
1470   gobject_class->get_property = gst_v4l2_codec_h264_dec_get_property;
1471   gobject_class->dispose = gst_v4l2_codec_h264_dec_dispose;
1472
1473   gst_element_class_set_static_metadata (element_class,
1474       "V4L2 Stateless H.264 Video Decoder",
1475       "Codec/Decoder/Video/Hardware",
1476       "A V4L2 based H.264 video decoder",
1477       "Nicolas Dufresne <nicolas.dufresne@collabora.com>");
1478
1479   gst_element_class_add_static_pad_template (element_class, &sink_template);
1480   gst_element_class_add_static_pad_template (element_class, &src_template);
1481   element_class->change_state =
1482       GST_DEBUG_FUNCPTR (gst_v4l2_codec_h264_dec_change_state);
1483
1484   decoder_class->open = GST_DEBUG_FUNCPTR (gst_v4l2_codec_h264_dec_open);
1485   decoder_class->close = GST_DEBUG_FUNCPTR (gst_v4l2_codec_h264_dec_close);
1486   decoder_class->stop = GST_DEBUG_FUNCPTR (gst_v4l2_codec_h264_dec_stop);
1487   decoder_class->negotiate =
1488       GST_DEBUG_FUNCPTR (gst_v4l2_codec_h264_dec_negotiate);
1489   decoder_class->decide_allocation =
1490       GST_DEBUG_FUNCPTR (gst_v4l2_codec_h264_dec_decide_allocation);
1491   decoder_class->flush = GST_DEBUG_FUNCPTR (gst_v4l2_codec_h264_dec_flush);
1492   decoder_class->sink_event =
1493       GST_DEBUG_FUNCPTR (gst_v4l2_codec_h264_dec_sink_event);
1494
1495   h264decoder_class->new_sequence =
1496       GST_DEBUG_FUNCPTR (gst_v4l2_codec_h264_dec_new_sequence);
1497   h264decoder_class->output_picture =
1498       GST_DEBUG_FUNCPTR (gst_v4l2_codec_h264_dec_output_picture);
1499   h264decoder_class->start_picture =
1500       GST_DEBUG_FUNCPTR (gst_v4l2_codec_h264_dec_start_picture);
1501   h264decoder_class->decode_slice =
1502       GST_DEBUG_FUNCPTR (gst_v4l2_codec_h264_dec_decode_slice);
1503   h264decoder_class->end_picture =
1504       GST_DEBUG_FUNCPTR (gst_v4l2_codec_h264_dec_end_picture);
1505   h264decoder_class->new_field_picture =
1506       GST_DEBUG_FUNCPTR (gst_v4l2_codec_h264_dec_new_field_picture);
1507   h264decoder_class->get_preferred_output_delay =
1508       GST_DEBUG_FUNCPTR (gst_v4l2_codec_h264_dec_get_preferred_output_delay);
1509
1510   klass->device = device;
1511   gst_v4l2_decoder_install_properties (gobject_class, PROP_LAST, device);
1512 }
1513
1514 void
1515 gst_v4l2_codec_h264_dec_register (GstPlugin * plugin, GstV4l2Decoder * decoder,
1516     GstV4l2CodecDevice * device, guint rank)
1517 {
1518   GstCaps *src_caps;
1519   guint version;
1520
1521   GST_DEBUG_CATEGORY_INIT (v4l2_h264dec_debug, "v4l2codecs-h264dec", 0,
1522       "V4L2 stateless h264 decoder");
1523
1524   if (!gst_v4l2_decoder_set_sink_fmt (decoder, V4L2_PIX_FMT_H264_SLICE,
1525           320, 240, 8))
1526     return;
1527   src_caps = gst_v4l2_decoder_enum_src_formats (decoder);
1528
1529   if (gst_caps_is_empty (src_caps)) {
1530     GST_WARNING ("Not registering H264 decoder since it produces no "
1531         "supported format");
1532     goto done;
1533   }
1534
1535   version = gst_v4l2_decoder_get_version (decoder);
1536   if (version < V4L2_MIN_KERNEL_VERSION)
1537     GST_WARNING ("V4L2 API v%u.%u too old, at least v%u.%u required",
1538         (version >> 16) & 0xff, (version >> 8) & 0xff,
1539         V4L2_MIN_KERNEL_VER_MAJOR, V4L2_MIN_KERNEL_VER_MINOR);
1540
1541   if (!gst_v4l2_decoder_h264_api_check (decoder)) {
1542     GST_WARNING ("Not registering H264 decoder as it failed ABI check.");
1543     goto done;
1544   }
1545
1546   gst_v4l2_decoder_register (plugin,
1547       GST_TYPE_V4L2_CODEC_H264_DEC,
1548       (GClassInitFunc) gst_v4l2_codec_h264_dec_subclass_init,
1549       gst_mini_object_ref (GST_MINI_OBJECT (device)),
1550       (GInstanceInitFunc) gst_v4l2_codec_h264_dec_subinit,
1551       "v4l2sl%sh264dec", device, rank, NULL);
1552
1553 done:
1554   gst_caps_unref (src_caps);
1555 }