2 * Copyright (c) 2012 The WebM project authors. All Rights Reserved.
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
10 #include "./vpx_config.h"
11 #include "third_party/googletest/src/include/gtest/gtest.h"
12 #include "test/codec_factory.h"
13 #include "test/encode_test_driver.h"
14 #include "test/i420_video_source.h"
15 #include "test/util.h"
16 #include "test/y4m_video_source.h"
20 class DatarateTest : public ::libvpx_test::EncoderTest,
21 public ::libvpx_test::CodecTestWithParam<libvpx_test::TestMode> {
23 DatarateTest() : EncoderTest(GET_PARAM(0)) {}
26 virtual void SetUp() {
28 SetMode(GET_PARAM(1));
32 virtual void ResetModel() {
34 bits_in_buffer_model_ = cfg_.rc_target_bitrate * cfg_.rc_buf_initial_sz;
41 virtual void PreEncodeFrameHook(::libvpx_test::VideoSource *video,
42 ::libvpx_test::Encoder *encoder) {
43 const vpx_rational_t tb = video->timebase();
44 timebase_ = static_cast<double>(tb.num) / tb.den;
48 virtual void FramePktHook(const vpx_codec_cx_pkt_t *pkt) {
49 // Time since last timestamp = duration.
50 vpx_codec_pts_t duration = pkt->data.frame.pts - last_pts_;
52 // TODO(jimbankoski): Remove these lines when the issue:
53 // http://code.google.com/p/webm/issues/detail?id=496 is fixed.
54 // For now the codec assumes buffer starts at starting buffer rate
55 // plus one frame's time.
59 // Add to the buffer the bits we'd expect from a constant bitrate server.
60 bits_in_buffer_model_ += static_cast<int64_t>(
61 duration * timebase_ * cfg_.rc_target_bitrate * 1000);
63 /* Test the buffer model here before subtracting the frame. Do so because
64 * the way the leaky bucket model works in libvpx is to allow the buffer to
65 * empty - and then stop showing frames until we've got enough bits to
66 * show one. As noted in comment below (issue 495), this does not currently
67 * apply to key frames. For now exclude key frames in condition below. */
68 const bool key_frame = (pkt->data.frame.flags & VPX_FRAME_IS_KEY)
71 ASSERT_GE(bits_in_buffer_model_, 0) << "Buffer Underrun at frame "
72 << pkt->data.frame.pts;
75 const size_t frame_size_in_bits = pkt->data.frame.sz * 8;
77 // Subtract from the buffer the bits associated with a played back frame.
78 bits_in_buffer_model_ -= frame_size_in_bits;
80 // Update the running total of bits for end of test datarate checks.
81 bits_total_ += frame_size_in_bits;
83 // If first drop not set and we have a drop set it to this time.
84 if (!first_drop_ && duration > 1)
85 first_drop_ = last_pts_ + 1;
87 // Update the most recent pts.
88 last_pts_ = pkt->data.frame.pts;
90 // We update this so that we can calculate the datarate minus the last
91 // frame encoded in the file.
92 bits_in_last_frame_ = frame_size_in_bits;
97 virtual void EndPassHook(void) {
99 const double file_size_in_kb = bits_total_ / 1000.; // bits per kilobit
101 duration_ = (last_pts_ + 1) * timebase_;
103 // Effective file datarate includes the time spent prebuffering.
104 effective_datarate_ = (bits_total_ - bits_in_last_frame_) / 1000.0
105 / (cfg_.rc_buf_initial_sz / 1000.0 + duration_);
107 file_datarate_ = file_size_in_kb / duration_;
111 vpx_codec_pts_t last_pts_;
112 int64_t bits_in_buffer_model_;
115 vpx_codec_pts_t first_drop_;
118 double file_datarate_;
119 double effective_datarate_;
120 size_t bits_in_last_frame_;
123 TEST_P(DatarateTest, BasicBufferModel) {
124 cfg_.rc_buf_initial_sz = 500;
125 cfg_.rc_dropframe_thresh = 1;
126 cfg_.rc_max_quantizer = 56;
127 cfg_.rc_end_usage = VPX_CBR;
128 // 2 pass cbr datarate control has a bug hidden by the small # of
129 // frames selected in this encode. The problem is that even if the buffer is
130 // negative we produce a keyframe on a cutscene. Ignoring datarate
132 // TODO(jimbankoski): ( Fix when issue
133 // http://code.google.com/p/webm/issues/detail?id=495 is addressed. )
134 ::libvpx_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288,
137 // There is an issue for low bitrates in real-time mode, where the
138 // effective_datarate slightly overshoots the target bitrate.
139 // This is same the issue as noted about (#495).
140 // TODO(jimbankoski/marpan): Update test to run for lower bitrates (< 100),
141 // when the issue is resolved.
142 for (int i = 100; i < 800; i += 200) {
143 cfg_.rc_target_bitrate = i;
145 ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
146 ASSERT_GE(cfg_.rc_target_bitrate, effective_datarate_)
147 << " The datarate for the file exceeds the target!";
149 ASSERT_LE(cfg_.rc_target_bitrate, file_datarate_ * 1.3)
150 << " The datarate for the file missed the target!";
154 TEST_P(DatarateTest, ChangingDropFrameThresh) {
155 cfg_.rc_buf_initial_sz = 500;
156 cfg_.rc_max_quantizer = 36;
157 cfg_.rc_end_usage = VPX_CBR;
158 cfg_.rc_target_bitrate = 200;
159 cfg_.kf_mode = VPX_KF_DISABLED;
161 const int frame_count = 40;
162 ::libvpx_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288,
163 30, 1, 0, frame_count);
165 // Here we check that the first dropped frame gets earlier and earlier
166 // as the drop frame threshold is increased.
168 const int kDropFrameThreshTestStep = 30;
169 vpx_codec_pts_t last_drop = frame_count;
170 for (int i = 1; i < 91; i += kDropFrameThreshTestStep) {
171 cfg_.rc_dropframe_thresh = i;
173 ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
174 ASSERT_LE(first_drop_, last_drop)
175 << " The first dropped frame for drop_thresh " << i
176 << " > first dropped frame for drop_thresh "
177 << i - kDropFrameThreshTestStep;
178 last_drop = first_drop_;
182 class DatarateTestVP9 : public ::libvpx_test::EncoderTest,
183 public ::libvpx_test::CodecTestWith2Params<libvpx_test::TestMode, int> {
185 DatarateTestVP9() : EncoderTest(GET_PARAM(0)) {}
188 virtual ~DatarateTestVP9() {}
190 virtual void SetUp() {
192 SetMode(GET_PARAM(1));
193 set_cpu_used_ = GET_PARAM(2);
197 virtual void ResetModel() {
199 bits_in_buffer_model_ = cfg_.rc_target_bitrate * cfg_.rc_buf_initial_sz;
201 tot_frame_number_ = 0;
204 // For testing up to 3 layers.
205 for (int i = 0; i < 3; ++i) {
211 // Frame flags and layer id for temporal layers.
214 // For two layers, test pattern is:
217 // For three layers, test pattern is:
221 // LAST is always update on base/layer 0, GOLDEN is updated on layer 1.
222 // For this 3 layer example, the 2nd enhancement layer (layer 2) does not
223 // update any reference frames.
224 int SetFrameFlags(int frame_num, int num_temp_layers) {
226 if (num_temp_layers == 2) {
227 if (frame_num % 2 == 0) {
228 // Layer 0: predict from L and ARF, update L.
229 frame_flags = VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_GF |
230 VP8_EFLAG_NO_UPD_ARF;
232 // Layer 1: predict from L, G and ARF, and update G.
233 frame_flags = VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST |
234 VP8_EFLAG_NO_UPD_ENTROPY;
236 } else if (num_temp_layers == 3) {
237 if (frame_num % 4 == 0) {
238 // Layer 0: predict from L and ARF; update L.
239 frame_flags = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF |
241 } else if ((frame_num - 2) % 4 == 0) {
242 // Layer 1: predict from L, G, ARF; update G.
243 frame_flags = VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST;
244 } else if ((frame_num - 1) % 2 == 0) {
245 // Layer 2: predict from L, G, ARF; update none.
246 frame_flags = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF |
247 VP8_EFLAG_NO_UPD_LAST;
253 int SetLayerId(int frame_num, int num_temp_layers) {
255 if (num_temp_layers == 2) {
256 if (frame_num % 2 == 0) {
261 } else if (num_temp_layers == 3) {
262 if (frame_num % 4 == 0) {
264 } else if ((frame_num - 2) % 4 == 0) {
266 } else if ((frame_num - 1) % 2 == 0) {
273 virtual void PreEncodeFrameHook(::libvpx_test::VideoSource *video,
274 ::libvpx_test::Encoder *encoder) {
275 if (video->frame() == 1) {
276 encoder->Control(VP8E_SET_CPUUSED, set_cpu_used_);
278 if (cfg_.ts_number_layers > 1) {
279 if (video->frame() == 1) {
280 encoder->Control(VP9E_SET_SVC, 1);
282 vpx_svc_layer_id_t layer_id = {0, 0};
283 layer_id.spatial_layer_id = 0;
284 frame_flags_ = SetFrameFlags(video->frame(), cfg_.ts_number_layers);
285 layer_id.temporal_layer_id = SetLayerId(video->frame(),
286 cfg_.ts_number_layers);
287 if (video->frame() > 0) {
288 encoder->Control(VP9E_SET_SVC_LAYER_ID, &layer_id);
291 const vpx_rational_t tb = video->timebase();
292 timebase_ = static_cast<double>(tb.num) / tb.den;
297 virtual void FramePktHook(const vpx_codec_cx_pkt_t *pkt) {
298 // Time since last timestamp = duration.
299 vpx_codec_pts_t duration = pkt->data.frame.pts - last_pts_;
302 // If first drop not set and we have a drop set it to this time.
304 first_drop_ = last_pts_ + 1;
305 // Update the number of frame drops.
306 num_drops_ += static_cast<int>(duration - 1);
307 // Update counter for total number of frames (#frames input to encoder).
308 // Needed for setting the proper layer_id below.
309 tot_frame_number_ += static_cast<int>(duration - 1);
312 int layer = SetLayerId(tot_frame_number_, cfg_.ts_number_layers);
314 // Add to the buffer the bits we'd expect from a constant bitrate server.
315 bits_in_buffer_model_ += static_cast<int64_t>(
316 duration * timebase_ * cfg_.rc_target_bitrate * 1000);
318 // Buffer should not go negative.
319 ASSERT_GE(bits_in_buffer_model_, 0) << "Buffer Underrun at frame "
320 << pkt->data.frame.pts;
322 const size_t frame_size_in_bits = pkt->data.frame.sz * 8;
324 // Update the total encoded bits. For temporal layers, update the cumulative
325 // encoded bits per layer.
326 for (int i = layer; i < static_cast<int>(cfg_.ts_number_layers); ++i) {
327 bits_total_[i] += frame_size_in_bits;
330 // Update the most recent pts.
331 last_pts_ = pkt->data.frame.pts;
336 virtual void EndPassHook(void) {
337 for (int layer = 0; layer < static_cast<int>(cfg_.ts_number_layers);
339 duration_ = (last_pts_ + 1) * timebase_;
340 if (bits_total_[layer]) {
341 // Effective file datarate:
342 effective_datarate_[layer] = (bits_total_[layer] / 1000.0) / duration_;
347 vpx_codec_pts_t last_pts_;
349 int frame_number_; // Counter for number of non-dropped/encoded frames.
350 int tot_frame_number_; // Counter for total number of input frames.
351 int64_t bits_total_[3];
353 double effective_datarate_[3];
355 int64_t bits_in_buffer_model_;
356 vpx_codec_pts_t first_drop_;
360 // Check basic rate targeting,
361 TEST_P(DatarateTestVP9, BasicRateTargeting) {
362 cfg_.rc_buf_initial_sz = 500;
363 cfg_.rc_buf_optimal_sz = 500;
364 cfg_.rc_buf_sz = 1000;
365 cfg_.rc_dropframe_thresh = 1;
366 cfg_.rc_min_quantizer = 0;
367 cfg_.rc_max_quantizer = 63;
368 cfg_.rc_end_usage = VPX_CBR;
369 cfg_.g_lag_in_frames = 0;
371 ::libvpx_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288,
373 for (int i = 150; i < 800; i += 200) {
374 cfg_.rc_target_bitrate = i;
376 ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
377 ASSERT_GE(effective_datarate_[0], cfg_.rc_target_bitrate * 0.85)
378 << " The datarate for the file is lower than target by too much!";
379 ASSERT_LE(effective_datarate_[0], cfg_.rc_target_bitrate * 1.15)
380 << " The datarate for the file is greater than target by too much!";
385 // Check basic rate targeting,
386 TEST_P(DatarateTestVP9, BasicRateTargeting444) {
387 ::libvpx_test::Y4mVideoSource video("rush_hour_444.y4m", 0, 140);
390 cfg_.g_timebase = video.timebase();
392 cfg_.rc_buf_initial_sz = 500;
393 cfg_.rc_buf_optimal_sz = 500;
394 cfg_.rc_buf_sz = 1000;
395 cfg_.rc_dropframe_thresh = 1;
396 cfg_.rc_min_quantizer = 0;
397 cfg_.rc_max_quantizer = 63;
398 cfg_.rc_end_usage = VPX_CBR;
400 for (int i = 250; i < 900; i += 200) {
401 cfg_.rc_target_bitrate = i;
403 ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
404 ASSERT_GE(static_cast<double>(cfg_.rc_target_bitrate),
405 effective_datarate_[0] * 0.85)
406 << " The datarate for the file exceeds the target by too much!";
407 ASSERT_LE(static_cast<double>(cfg_.rc_target_bitrate),
408 effective_datarate_[0] * 1.15)
409 << " The datarate for the file missed the target!"
410 << cfg_.rc_target_bitrate << " "<< effective_datarate_;
415 // Check that (1) the first dropped frame gets earlier and earlier
416 // as the drop frame threshold is increased, and (2) that the total number of
417 // frame drops does not decrease as we increase frame drop threshold.
418 // Use a lower qp-max to force some frame drops.
419 TEST_P(DatarateTestVP9, ChangingDropFrameThresh) {
420 cfg_.rc_buf_initial_sz = 500;
421 cfg_.rc_buf_optimal_sz = 500;
422 cfg_.rc_buf_sz = 1000;
423 cfg_.rc_undershoot_pct = 20;
424 cfg_.rc_undershoot_pct = 20;
425 cfg_.rc_dropframe_thresh = 10;
426 cfg_.rc_min_quantizer = 0;
427 cfg_.rc_max_quantizer = 50;
428 cfg_.rc_end_usage = VPX_CBR;
429 cfg_.rc_target_bitrate = 200;
430 cfg_.g_lag_in_frames = 0;
432 ::libvpx_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288,
435 const int kDropFrameThreshTestStep = 30;
436 vpx_codec_pts_t last_drop = 140;
437 int last_num_drops = 0;
438 for (int i = 10; i < 100; i += kDropFrameThreshTestStep) {
439 cfg_.rc_dropframe_thresh = i;
441 ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
442 ASSERT_GE(effective_datarate_[0], cfg_.rc_target_bitrate * 0.85)
443 << " The datarate for the file is lower than target by too much!";
444 ASSERT_LE(effective_datarate_[0], cfg_.rc_target_bitrate * 1.15)
445 << " The datarate for the file is greater than target by too much!";
446 ASSERT_LE(first_drop_, last_drop)
447 << " The first dropped frame for drop_thresh " << i
448 << " > first dropped frame for drop_thresh "
449 << i - kDropFrameThreshTestStep;
450 ASSERT_GE(num_drops_, last_num_drops)
451 << " The number of dropped frames for drop_thresh " << i
452 << " < number of dropped frames for drop_thresh "
453 << i - kDropFrameThreshTestStep;
454 last_drop = first_drop_;
455 last_num_drops = num_drops_;
459 // Check basic rate targeting for 2 temporal layers.
460 TEST_P(DatarateTestVP9, BasicRateTargeting2TemporalLayers) {
461 cfg_.rc_buf_initial_sz = 500;
462 cfg_.rc_buf_optimal_sz = 500;
463 cfg_.rc_buf_sz = 1000;
464 cfg_.rc_dropframe_thresh = 1;
465 cfg_.rc_min_quantizer = 0;
466 cfg_.rc_max_quantizer = 63;
467 cfg_.rc_end_usage = VPX_CBR;
468 cfg_.g_lag_in_frames = 0;
470 // 2 Temporal layers, no spatial layers: Framerate decimation (2, 1).
471 cfg_.ss_number_layers = 1;
472 cfg_.ts_number_layers = 2;
473 cfg_.ts_rate_decimator[0] = 2;
474 cfg_.ts_rate_decimator[1] = 1;
476 ::libvpx_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288,
478 for (int i = 200; i <= 800; i += 200) {
479 cfg_.rc_target_bitrate = i;
481 // 60-40 bitrate allocation for 2 temporal layers.
482 cfg_.ts_target_bitrate[0] = 60 * cfg_.rc_target_bitrate / 100;
483 cfg_.ts_target_bitrate[1] = cfg_.rc_target_bitrate;
484 ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
485 for (int j = 0; j < static_cast<int>(cfg_.ts_number_layers); ++j) {
486 ASSERT_GE(effective_datarate_[j], cfg_.ts_target_bitrate[j] * 0.85)
487 << " The datarate for the file is lower than target by too much, "
489 ASSERT_LE(effective_datarate_[j], cfg_.ts_target_bitrate[j] * 1.15)
490 << " The datarate for the file is greater than target by too much, "
496 // Check basic rate targeting for 3 temporal layers.
497 TEST_P(DatarateTestVP9, BasicRateTargeting3TemporalLayers) {
498 cfg_.rc_buf_initial_sz = 500;
499 cfg_.rc_buf_optimal_sz = 500;
500 cfg_.rc_buf_sz = 1000;
501 cfg_.rc_dropframe_thresh = 1;
502 cfg_.rc_min_quantizer = 0;
503 cfg_.rc_max_quantizer = 63;
504 cfg_.rc_end_usage = VPX_CBR;
505 cfg_.g_lag_in_frames = 0;
507 // 3 Temporal layers, no spatial layers: Framerate decimation (4, 2, 1).
508 cfg_.ss_number_layers = 1;
509 cfg_.ts_number_layers = 3;
510 cfg_.ts_rate_decimator[0] = 4;
511 cfg_.ts_rate_decimator[1] = 2;
512 cfg_.ts_rate_decimator[2] = 1;
514 ::libvpx_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288,
516 for (int i = 200; i <= 800; i += 200) {
517 cfg_.rc_target_bitrate = i;
519 // 40-20-40 bitrate allocation for 3 temporal layers.
520 cfg_.ts_target_bitrate[0] = 40 * cfg_.rc_target_bitrate / 100;
521 cfg_.ts_target_bitrate[1] = 60 * cfg_.rc_target_bitrate / 100;
522 cfg_.ts_target_bitrate[2] = cfg_.rc_target_bitrate;
523 ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
524 for (int j = 0; j < static_cast<int>(cfg_.ts_number_layers); ++j) {
525 ASSERT_GE(effective_datarate_[j], cfg_.ts_target_bitrate[j] * 0.85)
526 << " The datarate for the file is lower than target by too much, "
528 ASSERT_LE(effective_datarate_[j], cfg_.ts_target_bitrate[j] * 1.15)
529 << " The datarate for the file is greater than target by too much, "
535 // Check basic rate targeting for 3 temporal layers, with frame dropping.
536 // Only for one (low) bitrate with lower max_quantizer, and somewhat higher
537 // frame drop threshold, to force frame dropping.
538 TEST_P(DatarateTestVP9, BasicRateTargeting3TemporalLayersFrameDropping) {
539 cfg_.rc_buf_initial_sz = 500;
540 cfg_.rc_buf_optimal_sz = 500;
541 cfg_.rc_buf_sz = 1000;
542 // Set frame drop threshold and rc_max_quantizer to force some frame drops.
543 cfg_.rc_dropframe_thresh = 20;
544 cfg_.rc_max_quantizer = 45;
545 cfg_.rc_min_quantizer = 0;
546 cfg_.rc_end_usage = VPX_CBR;
547 cfg_.g_lag_in_frames = 0;
549 // 3 Temporal layers, no spatial layers: Framerate decimation (4, 2, 1).
550 cfg_.ss_number_layers = 1;
551 cfg_.ts_number_layers = 3;
552 cfg_.ts_rate_decimator[0] = 4;
553 cfg_.ts_rate_decimator[1] = 2;
554 cfg_.ts_rate_decimator[2] = 1;
556 ::libvpx_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288,
558 cfg_.rc_target_bitrate = 200;
560 // 40-20-40 bitrate allocation for 3 temporal layers.
561 cfg_.ts_target_bitrate[0] = 40 * cfg_.rc_target_bitrate / 100;
562 cfg_.ts_target_bitrate[1] = 60 * cfg_.rc_target_bitrate / 100;
563 cfg_.ts_target_bitrate[2] = cfg_.rc_target_bitrate;
564 ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
565 for (int j = 0; j < static_cast<int>(cfg_.ts_number_layers); ++j) {
566 ASSERT_GE(effective_datarate_[j], cfg_.ts_target_bitrate[j] * 0.85)
567 << " The datarate for the file is lower than target by too much, "
569 ASSERT_LE(effective_datarate_[j], cfg_.ts_target_bitrate[j] * 1.15)
570 << " The datarate for the file is greater than target by too much, "
572 // Expect some frame drops in this test: for this 200 frames test,
573 // expect at least 10% and not more than 50% drops.
574 ASSERT_GE(num_drops_, 20);
575 ASSERT_LE(num_drops_, 100);
579 VP8_INSTANTIATE_TEST_CASE(DatarateTest, ALL_TEST_MODES);
580 VP9_INSTANTIATE_TEST_CASE(DatarateTestVP9,
581 ::testing::Values(::libvpx_test::kOnePassGood),
582 ::testing::Range(2, 5));