Unit Test for Error Resilience Mode

This unit test compares the difference in quality with
error resilience enabled and disabled. The test runs
for all of the one-pass encoding modes.

The test ensures that the effect of turning on error
resilience makes less than a 10% difference in PSNR.

Further cases should be added to do a more comprehensive
test.

Change-Id: I1fc747fc78c9459bc6c74494f4b38308dbed0c32

diff --git a/test/error_resilience_test.cc b/test/error_resilience_test.cc
new file mode 100644 (file)
index 0000000..25c6731
--- /dev/null
+++ b/test/error_resilience_test.cc
@@ -0,0 +1,90 @@
+/*
+  Copyright (c) 2012 The WebM project authors. All Rights Reserved.
+
+  Use of this source code is governed by a BSD-style license
+  that can be found in the LICENSE file in the root of the source
+  tree. An additional intellectual property rights grant can be found
+  in the file PATENTS.  All contributing project authors may
+  be found in the AUTHORS file in the root of the source tree.
+*/
+#include "third_party/googletest/src/include/gtest/gtest.h"
+#include "test/encode_test_driver.h"
+#include "test/i420_video_source.h"
+
+namespace {
+
+class ErrorResilienceTest : public libvpx_test::EncoderTest,
+    public ::testing::TestWithParam<int> {
+ protected:
+  ErrorResilienceTest() {
+    psnr_ = 0.0;
+    nframes_ = 0;
+    encoding_mode_ = static_cast<libvpx_test::TestMode>(GetParam());
+  }
+  virtual ~ErrorResilienceTest() {}
+
+  virtual void SetUp() {
+    InitializeConfig();
+    SetMode(encoding_mode_);
+  }
+
+  virtual void BeginPassHook(unsigned int /*pass*/) {
+    psnr_ = 0.0;
+    nframes_ = 0;
+  }
+
+  virtual bool Continue() const {
+    return !HasFatalFailure() && !abort_;
+  }
+
+  virtual void PSNRPktHook(const vpx_codec_cx_pkt_t *pkt) {
+    psnr_ += pkt->data.psnr.psnr[0];
+    nframes_++;
+  }
+
+  double GetAveragePsnr() const {
+    if (nframes_)
+      return psnr_ / nframes_;
+    return 0.0;
+  }
+
+ private:
+  double psnr_;
+  unsigned int nframes_;
+  libvpx_test::TestMode encoding_mode_;
+};
+
+TEST_P(ErrorResilienceTest, OnVersusOff) {
+  const vpx_rational timebase = { 33333333, 1000000000 };
+  cfg_.g_timebase = timebase;
+  cfg_.rc_target_bitrate = 2000;
+  cfg_.g_lag_in_frames = 25;
+
+  init_flags_ = VPX_CODEC_USE_PSNR;
+
+  libvpx_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288,
+                                     timebase.den, timebase.num, 0, 30);
+
+  // Error resilient mode OFF.
+  cfg_.g_error_resilient = 0;
+  ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
+  const double psnr_resilience_off = GetAveragePsnr();
+  EXPECT_GT(psnr_resilience_off, 25.0);
+
+  // Error resilient mode ON.
+  cfg_.g_error_resilient = 1;
+  ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
+  const double psnr_resilience_on = GetAveragePsnr();
+  EXPECT_GT(psnr_resilience_on, 25.0);
+
+  // Test that turning on error resilient mode hurts by 10% at most.
+  if (psnr_resilience_off > 0.0) {
+    const double psnr_ratio = psnr_resilience_on / psnr_resilience_off;
+    EXPECT_GE(psnr_ratio, 0.9);
+    EXPECT_LE(psnr_ratio, 1.1);
+  }
+}
+
+INSTANTIATE_TEST_CASE_P(OnOffTest, ErrorResilienceTest,
+                        ONE_PASS_TEST_MODES);
+}  // namespace

diff --git a/test/test.mk b/test/test.mk
index c0aabb9..bf49bd2 100644 (file)
--- a/test/test.mk
+++ b/test/test.mk
@@ -12,6 +12,7 @@ LIBVPX_TEST_SRCS-$(CONFIG_VP8_ENCODER) += altref_test.cc
 LIBVPX_TEST_SRCS-$(CONFIG_VP8_ENCODER) += config_test.cc
 LIBVPX_TEST_SRCS-$(CONFIG_VP8_ENCODER) += encode_test_driver.cc
 LIBVPX_TEST_SRCS-$(CONFIG_VP8_ENCODER) += encode_test_driver.h
+LIBVPX_TEST_SRCS-$(CONFIG_VP8_ENCODER) += error_resilience_test.cc
 LIBVPX_TEST_SRCS-$(CONFIG_VP8_ENCODER) += i420_video_source.h
 LIBVPX_TEST_SRCS-$(CONFIG_VP8_ENCODER) += keyframe_test.cc
 LIBVPX_TEST_SRCS-$(CONFIG_VP8_ENCODER) += resize_test.cc