1 // Copyright 2015 The Chromium Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 // This file contains an implementation of an H265 Annex-B video stream parser.
7 #ifndef MEDIA_VIDEO_H265_PARSER_H_
8 #define MEDIA_VIDEO_H265_PARSER_H_
11 #include <sys/types.h>
15 #include "base/containers/flat_map.h"
16 #include "media/base/media_export.h"
17 #include "media/base/ranges.h"
18 #include "media/base/video_color_space.h"
19 #include "media/base/video_types.h"
20 #include "media/video/h264_bit_reader.h"
21 #include "media/video/h264_parser.h"
22 #include "media/video/h265_nalu_parser.h"
25 struct HdrMetadataCta861_3;
26 struct HdrMetadataSmpteSt2086;
31 // For explanations of each struct and its members, see H.265 specification
32 // at http://www.itu.int/rec/T-REC-H.265.
34 kMaxLongTermRefPicSets = 32, // 7.4.3.2.1
35 kMaxShortTermRefPicSets = 64, // 7.4.3.2.1
36 kMaxSubLayers = 7, // 7.4.3.1 & 7.4.3.2.1 [v|s]ps_max_sub_layers_minus1 + 1
37 kMaxDpbSize = 16, // A.4.2
38 kMaxRefIdxActive = 15, // 7.4.7.1 num_ref_idx_l{0,1}_active_minus1 + 1
41 struct MEDIA_EXPORT H265ProfileTierLevel {
42 H265ProfileTierLevel();
46 kProfileIdcMain10 = 2,
47 kProfileIdcMainStill = 3,
48 kProfileIdcRangeExtensions = 4,
49 kProfileIdcHighThroughput = 5,
50 kProfileIdcMultiviewMain = 6,
51 kProfileIdcScalableMain = 7,
52 kProfileIdc3dMain = 8,
53 kProfileIdcScreenContentCoding = 9,
54 kProfileIdcScalableRangeExtensions = 10,
55 kProfileIdcHighThroughputScreenContentCoding = 11,
59 int general_profile_idc;
60 int general_level_idc; // 30x the actual level.
61 uint32_t general_profile_compatibility_flags;
62 bool general_progressive_source_flag;
63 bool general_interlaced_source_flag;
64 bool general_non_packed_constraint_flag;
65 bool general_frame_only_constraint_flag;
66 bool general_one_picture_only_constraint_flag;
68 // From Table A.8 - General tier and level limits.
69 int GetMaxLumaPs() const;
70 // From A.4.2 - Profile-specific level limits for the video profiles.
71 size_t GetDpbMaxPicBuf() const;
74 struct MEDIA_EXPORT H265ScalingListData {
75 H265ScalingListData();
78 kDefaultScalingListSize0Values = 16, // Table 7-5, all values are 16
79 kScalingListSizeId0Count = 16, // 7.4.5
80 kScalingListSizeId1To3Count = 64, // 7.4.5
81 kNumScalingListMatrices = 6,
84 // TODO(jkardatzke): Optimize storage of the 32x32 since only indices 0 and 3
85 // are actually used. Also change it in the accelerator delegate if that is
88 uint8_t scaling_list_dc_coef_16x16[kNumScalingListMatrices];
89 uint8_t scaling_list_dc_coef_32x32[kNumScalingListMatrices];
90 uint8_t scaling_list_4x4[kNumScalingListMatrices][kScalingListSizeId0Count];
91 uint8_t scaling_list_8x8[kNumScalingListMatrices]
92 [kScalingListSizeId1To3Count];
93 uint8_t scaling_list_16x16[kNumScalingListMatrices]
94 [kScalingListSizeId1To3Count];
95 uint8_t scaling_list_32x32[kNumScalingListMatrices]
96 [kScalingListSizeId1To3Count];
99 struct MEDIA_EXPORT H265StRefPicSet {
103 int num_negative_pics;
104 int num_positive_pics;
105 int delta_poc_s0[kMaxShortTermRefPicSets];
106 int used_by_curr_pic_s0[kMaxShortTermRefPicSets];
107 int delta_poc_s1[kMaxShortTermRefPicSets];
108 int used_by_curr_pic_s1[kMaxShortTermRefPicSets];
110 // Calculated fields.
112 int rps_idx_num_delta_pocs;
115 struct MEDIA_EXPORT H265VUIParameters {
121 bool video_full_range_flag;
122 bool colour_description_present_flag;
123 int colour_primaries;
124 int transfer_characteristics;
126 int def_disp_win_left_offset;
127 int def_disp_win_right_offset;
128 int def_disp_win_top_offset;
129 int def_disp_win_bottom_offset;
131 bool timing_info_present_flag = false;
132 // Not part of standard, used only for replacing timing_info data
133 int timing_info_bit_offset_from_nal_end = 0;
135 bool bitstream_restriction_flag;
136 int min_spatial_segmentation_idc;
137 int max_bytes_per_pic_denom;
138 int max_bits_per_min_cu_denom;
139 int log2_max_mv_length_horizontal;
140 int log2_max_mv_length_vertical;
143 struct MEDIA_EXPORT H265VPS {
146 int vps_video_parameter_set_id;
147 bool vps_base_layer_internal_flag;
148 bool vps_base_layer_available_flag;
149 int vps_max_layers_minus1;
150 int vps_max_sub_layers_minus1;
151 bool vps_temporal_id_nesting_flag;
152 H265ProfileTierLevel profile_tier_level;
153 int vps_max_dec_pic_buffering_minus1[kMaxSubLayers];
154 int vps_max_num_reorder_pics[kMaxSubLayers];
155 int vps_max_latency_increase_plus1[kMaxSubLayers];
156 int vps_max_layer_id;
157 int vps_num_layer_sets_minus1;
159 // Computed from ScalabilityId
160 int aux_alpha_layer_id;
165 struct MEDIA_EXPORT H265SPS {
169 int sps_video_parameter_set_id;
170 int sps_max_sub_layers_minus1;
171 bool sps_temporal_id_nesting_flag;
172 H265ProfileTierLevel profile_tier_level;
173 int sps_seq_parameter_set_id;
174 int chroma_format_idc;
175 bool separate_colour_plane_flag;
176 int pic_width_in_luma_samples;
177 int pic_height_in_luma_samples;
178 int conf_win_left_offset;
179 int conf_win_right_offset;
180 int conf_win_top_offset;
181 int conf_win_bottom_offset;
182 int bit_depth_luma_minus8;
183 int bit_depth_chroma_minus8;
184 int log2_max_pic_order_cnt_lsb_minus4;
185 int sps_max_dec_pic_buffering_minus1[kMaxSubLayers];
186 int sps_max_num_reorder_pics[kMaxSubLayers];
187 uint32_t sps_max_latency_increase_plus1[kMaxSubLayers];
188 int log2_min_luma_coding_block_size_minus3;
189 int log2_diff_max_min_luma_coding_block_size;
190 int log2_min_luma_transform_block_size_minus2;
191 int log2_diff_max_min_luma_transform_block_size;
192 int max_transform_hierarchy_depth_inter;
193 int max_transform_hierarchy_depth_intra;
194 bool scaling_list_enabled_flag;
195 bool sps_scaling_list_data_present_flag;
196 H265ScalingListData scaling_list_data;
197 bool amp_enabled_flag;
198 bool sample_adaptive_offset_enabled_flag;
199 bool pcm_enabled_flag;
200 int pcm_sample_bit_depth_luma_minus1;
201 int pcm_sample_bit_depth_chroma_minus1;
202 int log2_min_pcm_luma_coding_block_size_minus3;
203 int log2_diff_max_min_pcm_luma_coding_block_size;
204 bool pcm_loop_filter_disabled_flag;
205 int num_short_term_ref_pic_sets;
206 H265StRefPicSet st_ref_pic_set[kMaxShortTermRefPicSets];
207 bool long_term_ref_pics_present_flag;
208 int num_long_term_ref_pics_sps;
209 int lt_ref_pic_poc_lsb_sps[kMaxLongTermRefPicSets];
210 bool used_by_curr_pic_lt_sps_flag[kMaxLongTermRefPicSets];
211 bool sps_temporal_mvp_enabled_flag;
212 bool strong_intra_smoothing_enabled_flag;
213 H265VUIParameters vui_parameters;
215 // Extension extra elements.
216 bool sps_extension_present_flag;
217 bool sps_range_extension_flag;
218 bool sps_multilayer_extension_flag;
219 bool sps_3d_extension_flag;
220 bool sps_scc_extension_flag;
221 bool transform_skip_rotation_enabled_flag;
222 bool transform_skip_context_enabled_flag;
223 bool implicit_rdpcm_enabled_flag;
224 bool explicit_rdpcm_enabled_flag;
225 bool extended_precision_processing_flag;
226 bool intra_smoothing_disabled_flag;
227 bool high_precision_offsets_enabled_flag;
228 bool persistent_rice_adaptation_enabled_flag;
229 bool cabac_bypass_alignment_enabled_flag;
231 // Calculated fields.
232 int chroma_array_type;
238 int max_pic_order_cnt_lsb;
240 int pic_width_in_ctbs_y;
241 int pic_height_in_ctbs_y;
242 int pic_size_in_ctbs_y;
243 int wp_offset_half_range_y;
244 int wp_offset_half_range_c;
245 bool vui_parameters_present_flag = false;
246 uint32_t sps_max_latency_pictures[kMaxSubLayers];
248 // Helpers to compute frequently-used values. They do not verify that the
249 // results are in-spec for the given profile or level.
250 gfx::Size GetCodedSize() const;
251 gfx::Rect GetVisibleRect() const;
252 VideoColorSpace GetColorSpace() const;
253 VideoChromaSampling GetChromaSampling() const;
256 struct MEDIA_EXPORT H265PPS {
260 kMaxNumTileColumnWidth = 19, // From VAAPI.
261 kMaxNumTileRowHeight = 21, // From VAAPI.
264 int temporal_id; // calculated from NALU
267 int pps_pic_parameter_set_id;
268 int pps_seq_parameter_set_id;
269 bool dependent_slice_segments_enabled_flag;
270 bool output_flag_present_flag;
271 int num_extra_slice_header_bits;
272 bool sign_data_hiding_enabled_flag;
273 bool cabac_init_present_flag;
274 int num_ref_idx_l0_default_active_minus1;
275 int num_ref_idx_l1_default_active_minus1;
277 bool constrained_intra_pred_flag;
278 bool transform_skip_enabled_flag;
279 bool cu_qp_delta_enabled_flag;
280 int diff_cu_qp_delta_depth;
281 int pps_cb_qp_offset;
282 int pps_cr_qp_offset;
283 bool pps_slice_chroma_qp_offsets_present_flag;
284 bool weighted_pred_flag;
285 bool weighted_bipred_flag;
286 bool transquant_bypass_enabled_flag;
287 bool tiles_enabled_flag;
288 bool entropy_coding_sync_enabled_flag;
289 int num_tile_columns_minus1;
290 int num_tile_rows_minus1;
291 bool uniform_spacing_flag;
292 int column_width_minus1[kMaxNumTileColumnWidth];
293 int row_height_minus1[kMaxNumTileRowHeight];
294 bool loop_filter_across_tiles_enabled_flag;
295 bool pps_loop_filter_across_slices_enabled_flag;
296 bool deblocking_filter_control_present_flag;
297 bool deblocking_filter_override_enabled_flag;
298 bool pps_deblocking_filter_disabled_flag;
299 int pps_beta_offset_div2;
300 int pps_tc_offset_div2;
301 bool pps_scaling_list_data_present_flag;
302 H265ScalingListData scaling_list_data;
303 bool lists_modification_present_flag;
304 int log2_parallel_merge_level_minus2;
305 bool slice_segment_header_extension_present_flag;
307 // Extension extra elements.
308 bool pps_extension_present_flag;
309 bool pps_range_extension_flag;
310 bool pps_multilayer_extension_flag;
311 bool pps_3d_extension_flag;
312 bool pps_scc_extension_flag;
313 int log2_max_transform_skip_block_size_minus2;
314 bool cross_component_prediction_enabled_flag;
315 bool chroma_qp_offset_list_enabled_flag;
316 int diff_cu_chroma_qp_offset_depth;
317 int chroma_qp_offset_list_len_minus1;
318 int cb_qp_offset_list[6];
319 int cr_qp_offset_list[6];
320 int log2_sao_offset_scale_luma;
321 int log2_sao_offset_scale_chroma;
323 // Calculated fields.
327 struct MEDIA_EXPORT H265RefPicListsModifications {
328 H265RefPicListsModifications();
331 bool ref_pic_list_modification_flag_l0;
332 int list_entry_l0[kMaxRefIdxActive];
333 bool ref_pic_list_modification_flag_l1;
334 int list_entry_l1[kMaxRefIdxActive];
337 struct MEDIA_EXPORT H265PredWeightTable {
338 H265PredWeightTable();
341 int luma_log2_weight_denom;
342 int delta_chroma_log2_weight_denom;
343 int chroma_log2_weight_denom;
344 int delta_luma_weight_l0[kMaxRefIdxActive];
345 int luma_offset_l0[kMaxRefIdxActive];
346 int delta_chroma_weight_l0[kMaxRefIdxActive][2];
347 int delta_chroma_offset_l0[kMaxRefIdxActive][2];
348 int delta_luma_weight_l1[kMaxRefIdxActive];
349 int luma_offset_l1[kMaxRefIdxActive];
350 int delta_chroma_weight_l1[kMaxRefIdxActive][2];
351 int delta_chroma_offset_l1[kMaxRefIdxActive][2];
354 struct MEDIA_EXPORT H265SliceHeader {
358 kSliceTypeB = 0, // Table 7-7
359 kSliceTypeP = 1, // Table 7-7
360 kSliceTypeI = 2, // Table 7-7
363 int nal_unit_type; // from NAL header
364 const uint8_t* nalu_data; // from NAL header
365 size_t nalu_size; // from NAL header
366 size_t header_size; // calculated, not including emulation prevention bytes
367 size_t header_emulation_prevention_bytes;
369 // Calculated, but needs to be preserved when we copy slice dependent data
370 // so put it at the front.
374 bool first_slice_segment_in_pic_flag;
375 bool no_output_of_prior_pics_flag;
376 int slice_pic_parameter_set_id;
377 bool dependent_slice_segment_flag;
378 int slice_segment_address;
379 // Do not move any of the above fields below or vice-versa, everything after
380 // this is copied as a block.
382 bool pic_output_flag;
384 int slice_pic_order_cnt_lsb;
385 bool short_term_ref_pic_set_sps_flag;
386 H265StRefPicSet st_ref_pic_set;
387 // Do not change the order of the following fields up through
388 // slice_sao_luma_flag. They are compared as a block.
389 int short_term_ref_pic_set_idx;
390 int num_long_term_sps;
391 int num_long_term_pics;
392 int poc_lsb_lt[kMaxLongTermRefPicSets];
393 bool used_by_curr_pic_lt[kMaxLongTermRefPicSets];
394 bool delta_poc_msb_present_flag[kMaxLongTermRefPicSets];
395 int delta_poc_msb_cycle_lt[kMaxLongTermRefPicSets];
396 bool slice_temporal_mvp_enabled_flag;
397 bool slice_sao_luma_flag;
398 bool slice_sao_chroma_flag;
399 bool num_ref_idx_active_override_flag;
400 int num_ref_idx_l0_active_minus1;
401 int num_ref_idx_l1_active_minus1;
402 H265RefPicListsModifications ref_pic_lists_modification;
403 bool mvd_l1_zero_flag;
404 bool cabac_init_flag;
405 bool collocated_from_l0_flag;
406 int collocated_ref_idx;
407 H265PredWeightTable pred_weight_table;
408 int five_minus_max_num_merge_cand;
410 int slice_cb_qp_offset;
411 int slice_cr_qp_offset;
412 bool slice_deblocking_filter_disabled_flag;
413 int slice_beta_offset_div2;
414 int slice_tc_offset_div2;
415 bool slice_loop_filter_across_slices_enabled_flag;
419 int num_pic_total_curr;
420 // Number of bits st_ref_pic_set takes after removing emulation prevention
423 // Number of bits lt_ref_pic_set takes after removing emulation prevention
427 bool IsISlice() const;
428 bool IsPSlice() const;
429 bool IsBSlice() const;
431 const H265StRefPicSet& GetStRefPicSet(const H265SPS* sps) const {
432 if (curr_rps_idx == sps->num_short_term_ref_pic_sets)
433 return st_ref_pic_set;
435 return sps->st_ref_pic_set[curr_rps_idx];
439 struct MEDIA_EXPORT H265SEIAlphaChannelInfo {
440 bool alpha_channel_cancel_flag;
441 int alpha_channel_use_idc;
442 int alpha_channel_bit_depth_minus8;
443 int alpha_transparent_value;
444 int alpha_opaque_value;
445 bool alpha_channel_incr_flag;
446 bool alpha_channel_clip_flag;
447 bool alpha_channel_clip_type_flag;
450 struct MEDIA_EXPORT H265SEIContentLightLevelInfo {
451 uint16_t max_content_light_level;
452 uint16_t max_picture_average_light_level;
454 gfx::HdrMetadataCta861_3 ToGfx() const;
457 struct MEDIA_EXPORT H265SEIMasteringDisplayInfo {
459 kNumDisplayPrimaries = 3,
460 kDisplayPrimaryComponents = 2,
463 uint16_t display_primaries[kNumDisplayPrimaries][kDisplayPrimaryComponents];
464 uint16_t white_points[2];
465 uint32_t max_luminance;
466 uint32_t min_luminance;
468 gfx::HdrMetadataSmpteSt2086 ToGfx() const;
471 struct MEDIA_EXPORT H265SEIMessage {
475 kSEIMasteringDisplayInfo = 137,
476 kSEIContentLightLevelInfo = 144,
477 kSEIAlphaChannelInfo = 165,
483 // Placeholder; in future more supported types will contribute to more
484 // union members here.
485 H265SEIAlphaChannelInfo alpha_channel_info;
486 H265SEIContentLightLevelInfo content_light_level_info;
487 H265SEIMasteringDisplayInfo mastering_display_info;
491 struct MEDIA_EXPORT H265SEI {
495 std::vector<H265SEIMessage> msgs;
498 // Class to parse an Annex-B H.265 stream.
499 class MEDIA_EXPORT H265Parser : public H265NaluParser {
503 H265Parser(const H265Parser&) = delete;
504 H265Parser& operator=(const H265Parser&) = delete;
506 ~H265Parser() override;
508 // NALU-specific parsing functions.
509 // These should be called after AdvanceToNextNALU().
511 // VPSes, SPSes and PPSes are owned by the parser class and the memory for
512 // their structures is managed here, not by the caller, as they are
513 // reused across NALUs.
515 // Parse an VPS/SPS/PPS NALU and save their data in the parser, returning id
516 // of the parsed structure in |*pps_id|/|*sps_id|/|*vps_id|. To get a pointer
517 // to a given VPS/SPS/PPS structure, use GetVPS()/GetSPS()/GetPPS(), passing
518 // the returned |*vps_id|/|*sps_id|/|*pps_id| as parameter.
519 Result ParseVPS(int* vps_id);
520 Result ParseSPS(int* sps_id);
521 Result ParsePPS(const H265NALU& nalu, int* pps_id);
523 // Return a pointer to VPS/SPS/PPS with given |*vps_id|/|sps_id|/|pps_id| or
524 // null if not present.
525 const H265VPS* GetVPS(int vps_id) const;
526 const H265SPS* GetSPS(int sps_id) const;
527 const H265PPS* GetPPS(int pps_id) const;
529 // Slice headers are not used across NALUs by the parser and can be discarded
530 // after current NALU, so the parser does not store them, nor does it manage
531 // their memory. The caller has to provide and manage it instead.
533 // Parse a slice header, returning it in |*shdr|. |*nalu| must be set to
534 // the NALU returned from AdvanceToNextNALU() and corresponding to |*shdr|.
535 // |prior_shdr| should be the last parsed header in decoding order for
536 // handling dependent slice segments. If |prior_shdr| is null and this is a
537 // dependent slice segment, an error will be returned.
538 Result ParseSliceHeader(const H265NALU& nalu,
539 H265SliceHeader* shdr,
540 H265SliceHeader* prior_shdr);
542 // Parse a slice header and return the associated picture parameter set ID.
543 Result ParseSliceHeaderForPictureParameterSets(const H265NALU& nalu,
546 // Parse a SEI, returning it in |*sei|, provided and managed by the caller.
547 Result ParseSEI(H265SEI* sei);
549 static VideoCodecProfile ProfileIDCToVideoCodecProfile(int profile_idc);
551 void UpdateTimingInfoIfNeeded(std::vector<std::vector<uint8_t>>* spses_array,
552 int num_units_in_tick,
556 Result ParseProfileTierLevel(bool profile_present,
557 int max_num_sub_layers_minus1,
558 H265ProfileTierLevel* profile_tier_level);
559 Result ParseScalingListData(H265ScalingListData* scaling_list_data);
560 Result ParseStRefPicSet(int st_rps_idx,
562 H265StRefPicSet* st_ref_pic_set,
563 bool is_slice_hdr = false);
564 Result ParseVuiParameters(const H265SPS& sps, H265VUIParameters* vui);
565 Result ParseAndIgnoreHrdParameters(bool common_inf_present_flag,
566 int max_num_sub_layers_minus1);
567 Result ParseAndIgnoreSubLayerHrdParameters(
569 bool sub_pic_hrd_params_present_flag);
570 Result ParseRefPicListsModifications(const H265SliceHeader& shdr,
571 H265RefPicListsModifications* rpl_mod);
572 Result ParsePredWeightTable(const H265SPS& sps,
573 const H265SliceHeader& shdr,
574 H265PredWeightTable* pred_weight_table);
575 std::vector<uint8_t> PrepareSPSWithTimingInfo(
576 const std::vector<uint8_t>& old_sps,
578 int num_units_in_tick,
581 // VPSes, PPSes and SPSes stored for future reference.
582 base::flat_map<int, std::unique_ptr<H265VPS>> active_vps_;
583 base::flat_map<int, std::unique_ptr<H265SPS>> active_sps_;
584 base::flat_map<int, std::unique_ptr<H265PPS>> active_pps_;
589 #endif // MEDIA_VIDEO_H265_PARSER_H_