From adebf364cb7ea098a366108acae8cf01595388fa Mon Sep 17 00:00:00 2001 From: Marco Paniconi Date: Tue, 21 Nov 2023 14:00:16 -0800 Subject: [PATCH] rtc: Set nonrd keyframe under dynamic change of deadline For realtime mode: if the deadline mode (good/best/realtime) is changed on the fly (via codec_encode() call), force a key frame and set the speed feature nonrd_keyframe = 1 to avoid entering the rd pickmode. nonrd_pickmode=0/off is the only feature in realtime mode that involves rd pickmode, so by forcing it on/1 we can cleanly separate nonrd (realtime) from rd (good/best), so we can avoid possible issues on this dynamic mode switch, such as in bug listed below. Dynamic change of deadline, in particular for realtime mode, involves a lot of coding/speed feature changes, so best to also force reset with keyframe. Added unitest that triggers the issue in the bug. Bug: b/310663186 Change-Id: Idf8fd7c9ee54b301968184be5481ee9faa06468d --- test/encode_api_test.cc | 94 ++++++++++++++++++++++++++++++++++++++++ vp9/encoder/vp9_encoder.c | 5 +++ vp9/encoder/vp9_encoder.h | 4 ++ vp9/encoder/vp9_ratectrl.c | 15 ++++--- vp9/encoder/vp9_speed_features.c | 5 +++ 5 files changed, 118 insertions(+), 5 deletions(-) diff --git a/test/encode_api_test.cc b/test/encode_api_test.cc index aa2d28b..f48c9a1 100644 --- a/test/encode_api_test.cc +++ b/test/encode_api_test.cc @@ -667,6 +667,100 @@ TEST(EncodeAPI, MultipleChangeConfigResize) { ASSERT_EQ(vpx_codec_destroy(&enc), VPX_CODEC_OK); } +// This is a test case from clusterfuzz: based on b/310663186. +// Encode set of frames while varying the deadline on the fly from +// good to realtime to best and back to realtime. +TEST(EncodeAPI, DynamicDeadlineChange) { + vpx_codec_iface_t *const iface = vpx_codec_vp9_cx(); + vpx_codec_enc_cfg_t cfg; + + struct Config { + unsigned int thread; + unsigned int width; + unsigned int height; + vpx_rc_mode end_usage; + unsigned long deadline; + }; + + // Set initial config, in particular set deadline to GOOD mode. + struct Config config = { 0, 1, 1, VPX_VBR, VPX_DL_GOOD_QUALITY }; + unsigned long deadline = config.deadline; + ASSERT_EQ(vpx_codec_enc_config_default(iface, &cfg, /*usage=*/0), + VPX_CODEC_OK); + cfg.g_threads = config.thread; + cfg.g_w = config.width; + cfg.g_h = config.height; + cfg.g_timebase.num = 1; + cfg.g_timebase.den = 1000 * 1000; // microseconds + cfg.g_pass = VPX_RC_ONE_PASS; + cfg.g_lag_in_frames = 0; + cfg.rc_end_usage = config.end_usage; + cfg.rc_min_quantizer = 2; + cfg.rc_max_quantizer = 58; + + vpx_codec_ctx_t enc; + ASSERT_EQ(vpx_codec_enc_init(&enc, iface, &cfg, 0), VPX_CODEC_OK); + // Use realtime speed: 5 to 9. + ASSERT_EQ(vpx_codec_control(&enc, VP8E_SET_CPUUSED, 5), VPX_CODEC_OK); + int frame_index = 0; + + // Encode 1st frame. + CodecEncodeFrame(&enc, cfg.g_w, cfg.g_h, frame_index, deadline, true); + frame_index++; + + // Encode 2nd frame, delta frame. + CodecEncodeFrame(&enc, cfg.g_w, cfg.g_h, frame_index, deadline, false); + frame_index++; + + // Change config: change deadline to REALTIME. + config = { 0, 1, 1, VPX_VBR, VPX_DL_REALTIME }; + deadline = config.deadline; + ASSERT_EQ(vpx_codec_enc_config_set(&enc, &cfg), VPX_CODEC_OK) + << vpx_codec_error_detail(&enc); + + // Encode 3rd frame with new config, set key frame. + CodecEncodeFrame(&enc, cfg.g_w, cfg.g_h, frame_index, deadline, true); + frame_index++; + + // Encode 4th frame with same config, delta frame. + CodecEncodeFrame(&enc, cfg.g_w, cfg.g_h, frame_index, deadline, false); + frame_index++; + + // Encode 5th frame with same config, key frame. + CodecEncodeFrame(&enc, cfg.g_w, cfg.g_h, frame_index, deadline, true); + frame_index++; + + // Change config: change deadline to BEST. + config = { 0, 1, 1, VPX_VBR, VPX_DL_BEST_QUALITY }; + deadline = config.deadline; + ASSERT_EQ(vpx_codec_enc_config_set(&enc, &cfg), VPX_CODEC_OK) + << vpx_codec_error_detail(&enc); + + // Encode 6th frame with new config, set delta frame. + CodecEncodeFrame(&enc, cfg.g_w, cfg.g_h, frame_index, deadline, false); + frame_index++; + + // Change config: change deadline to REALTIME. + config = { 0, 1, 1, VPX_VBR, VPX_DL_REALTIME }; + deadline = config.deadline; + ASSERT_EQ(vpx_codec_enc_config_set(&enc, &cfg), VPX_CODEC_OK) + << vpx_codec_error_detail(&enc); + + // Encode 7th frame with new config, set delta frame. + CodecEncodeFrame(&enc, cfg.g_w, cfg.g_h, frame_index, deadline, false); + frame_index++; + + // Encode 8th frame with new config, set key frame. + CodecEncodeFrame(&enc, cfg.g_w, cfg.g_h, frame_index, deadline, true); + frame_index++; + + // Encode 9th frame with new config, set delta frame. + CodecEncodeFrame(&enc, cfg.g_w, cfg.g_h, frame_index, deadline, false); + frame_index++; + + ASSERT_EQ(vpx_codec_destroy(&enc), VPX_CODEC_OK); +} + class EncodeApiGetTplStatsTest : public ::libvpx_test::EncoderTest, public ::testing::TestWithParam { diff --git a/vp9/encoder/vp9_encoder.c b/vp9/encoder/vp9_encoder.c index e1a4d98..e27a77e 100644 --- a/vp9/encoder/vp9_encoder.c +++ b/vp9/encoder/vp9_encoder.c @@ -5535,6 +5535,11 @@ static void encode_frame_to_data_rate( set_ref_sign_bias(cpi); } + // On the very first frame set the deadline_mode_previous_frame to + // the current mode. + if (cpi->common.current_video_frame == 0) + cpi->deadline_mode_previous_frame = cpi->oxcf.mode; + // Set default state for segment based loop filter update flags. cm->lf.mode_ref_delta_update = 0; diff --git a/vp9/encoder/vp9_encoder.h b/vp9/encoder/vp9_encoder.h index 7b02fe7..1714893 100644 --- a/vp9/encoder/vp9_encoder.h +++ b/vp9/encoder/vp9_encoder.h @@ -1037,6 +1037,10 @@ typedef struct VP9_COMP { int fixed_qp_onepass; + // Flag to keep track of dynamic change in deadline mode + // (good/best/realtime). + MODE deadline_mode_previous_frame; + #if CONFIG_COLLECT_COMPONENT_TIMING /*! * component_time[] are initialized to zero while encoder starts. diff --git a/vp9/encoder/vp9_ratectrl.c b/vp9/encoder/vp9_ratectrl.c index e02b289..aa77b7c 100644 --- a/vp9/encoder/vp9_ratectrl.c +++ b/vp9/encoder/vp9_ratectrl.c @@ -1991,6 +1991,7 @@ void vp9_rc_postencode_update(VP9_COMP *cpi, uint64_t bytes_used) { rc->last_avg_frame_bandwidth = rc->avg_frame_bandwidth; if (cpi->use_svc && svc->spatial_layer_id < svc->number_spatial_layers - 1) svc->lower_layer_qindex = cm->base_qindex; + cpi->deadline_mode_previous_frame = cpi->oxcf.mode; } void vp9_rc_postencode_update_drop_frame(VP9_COMP *cpi) { @@ -2011,6 +2012,7 @@ void vp9_rc_postencode_update_drop_frame(VP9_COMP *cpi) { cpi->rc.buffer_level = cpi->rc.optimal_buffer_level; cpi->rc.bits_off_target = cpi->rc.optimal_buffer_level; } + cpi->deadline_mode_previous_frame = cpi->oxcf.mode; } int vp9_calc_pframe_target_size_one_pass_vbr(const VP9_COMP *cpi) { @@ -2118,7 +2120,8 @@ void vp9_rc_get_one_pass_vbr_params(VP9_COMP *cpi) { int target; if (!cpi->refresh_alt_ref_frame && (cm->current_video_frame == 0 || (cpi->frame_flags & FRAMEFLAGS_KEY) || - rc->frames_to_key == 0)) { + rc->frames_to_key == 0 || + (cpi->oxcf.mode != cpi->deadline_mode_previous_frame))) { cm->frame_type = KEY_FRAME; rc->this_key_frame_forced = cm->current_video_frame != 0 && rc->frames_to_key == 0; @@ -2284,14 +2287,15 @@ void vp9_rc_get_svc_params(VP9_COMP *cpi) { // Periodic key frames is based on the super-frame counter // (svc.current_superframe), also only base spatial layer is key frame. // Key frame is set for any of the following: very first frame, frame flags - // indicates key, superframe counter hits key frequency, or (non-intra) sync - // flag is set for spatial layer 0. + // indicates key, superframe counter hits key frequency,(non-intra) sync + // flag is set for spatial layer 0, or deadline mode changes. if ((cm->current_video_frame == 0 && !svc->previous_frame_is_intra_only) || (cpi->frame_flags & FRAMEFLAGS_KEY) || (cpi->oxcf.auto_key && (svc->current_superframe % cpi->oxcf.key_freq == 0) && !svc->previous_frame_is_intra_only && svc->spatial_layer_id == 0) || - (svc->spatial_layer_sync[0] == 1 && svc->spatial_layer_id == 0)) { + (svc->spatial_layer_sync[0] == 1 && svc->spatial_layer_id == 0) || + (cpi->oxcf.mode != cpi->deadline_mode_previous_frame)) { cm->frame_type = KEY_FRAME; rc->source_alt_ref_active = 0; if (is_one_pass_svc(cpi)) { @@ -2490,7 +2494,8 @@ void vp9_rc_get_one_pass_cbr_params(VP9_COMP *cpi) { RATE_CONTROL *const rc = &cpi->rc; int target; if ((cm->current_video_frame == 0) || (cpi->frame_flags & FRAMEFLAGS_KEY) || - (cpi->oxcf.auto_key && rc->frames_to_key == 0)) { + (cpi->oxcf.auto_key && rc->frames_to_key == 0) || + (cpi->oxcf.mode != cpi->deadline_mode_previous_frame)) { cm->frame_type = KEY_FRAME; rc->frames_to_key = cpi->oxcf.key_freq; rc->kf_boost = DEFAULT_KF_BOOST; diff --git a/vp9/encoder/vp9_speed_features.c b/vp9/encoder/vp9_speed_features.c index 4a71721..56fb5f9 100644 --- a/vp9/encoder/vp9_speed_features.c +++ b/vp9/encoder/vp9_speed_features.c @@ -859,6 +859,11 @@ static void set_rt_speed_feature_framesize_independent( // off for now. if (speed <= 3 && cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ) cpi->oxcf.aq_mode = 0; + // For all speeds for rt mode: if the deadline mode changed (was good/best + // quality on previous frame and now is realtime) set nonrd_keyframe to 1 to + // avoid entering rd pickmode. This causes issues, such as: b/310663186. + if (cpi->oxcf.mode != cpi->deadline_mode_previous_frame) + sf->nonrd_keyframe = 1; } void vp9_set_speed_features_framesize_dependent(VP9_COMP *cpi, int speed) { -- 2.7.4