src/third_party/webrtc/video_engine/test/auto_test/automated/vie_video_verification_test.cc

   1 /*
   2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
   3  *
   4  *  Use of this source code is governed by a BSD-style license
   5  *  that can be found in the LICENSE file in the root of the source
   6  *  tree. An additional intellectual property rights grant can be found
   7  *  in the file PATENTS.  All contributing project authors may
   8  *  be found in the AUTHORS file in the root of the source tree.
   9  */
  10
  11 #include <sstream>
  12 #include <string>
  13 #include <vector>
  14
  15 #include "testing/gtest/include/gtest/gtest.h"
  16 #include "webrtc/test/testsupport/fileutils.h"
  17 #include "webrtc/test/testsupport/metrics/video_metrics.h"
  18 #include "webrtc/test/testsupport/perf_test.h"
  19 #include "webrtc/video_engine/test/auto_test/interface/vie_autotest.h"
  20 #include "webrtc/video_engine/test/auto_test/interface/vie_file_based_comparison_tests.h"
  21 #include "webrtc/video_engine/test/auto_test/primitives/framedrop_primitives.h"
  22 #include "webrtc/video_engine/test/libvietest/include/tb_external_transport.h"
  23 #include "webrtc/video_engine/test/libvietest/include/vie_to_file_renderer.h"
  24
  25 namespace {
  26
  27 // The input file must be QCIF since I420 gets scaled to that in the tests
  28 // (it is so bandwidth-heavy we have no choice). Our comparison algorithms
  29 // wouldn't like scaling, so this will work when we compare with the original.
  30 const int kInputWidth = 176;
  31 const int kInputHeight = 144;
  32
  33 class ViEVideoVerificationTest : public testing::Test {
  34  protected:
  35   void SetUp() {
  36     input_file_ = webrtc::test::ResourcePath("paris_qcif", "yuv");
  37     local_file_renderer_ = NULL;
  38     remote_file_renderer_ = NULL;
  39   }
  40
  41   void InitializeFileRenderers() {
  42     local_file_renderer_ = new ViEToFileRenderer();
  43     remote_file_renderer_ = new ViEToFileRenderer();
  44     SetUpLocalFileRenderer(local_file_renderer_);
  45     SetUpRemoteFileRenderer(remote_file_renderer_);
  46   }
  47
  48   void SetUpLocalFileRenderer(ViEToFileRenderer* file_renderer) {
  49     SetUpFileRenderer(file_renderer, "-local-preview.yuv");
  50   }
  51
  52   void SetUpRemoteFileRenderer(ViEToFileRenderer* file_renderer) {
  53     SetUpFileRenderer(file_renderer, "-remote.yuv");
  54   }
  55
  56   // Must be called manually inside the tests.
  57   void StopRenderers() {
  58     local_file_renderer_->StopRendering();
  59     remote_file_renderer_->StopRendering();
  60   }
  61
  62   void CompareFiles(const std::string& reference_file,
  63                     const std::string& test_file,
  64                     double* psnr_result, double *ssim_result) {
  65     webrtc::test::QualityMetricsResult psnr;
  66     int error = I420PSNRFromFiles(reference_file.c_str(), test_file.c_str(),
  67                                   kInputWidth, kInputHeight, &psnr);
  68
  69     EXPECT_EQ(0, error) << "PSNR routine failed - output files missing?";
  70     *psnr_result = psnr.average;
  71
  72     webrtc::test::QualityMetricsResult ssim;
  73     error = I420SSIMFromFiles(reference_file.c_str(), test_file.c_str(),
  74                               kInputWidth, kInputHeight, &ssim);
  75     EXPECT_EQ(0, error) << "SSIM routine failed - output files missing?";
  76     *ssim_result = ssim.average;
  77
  78     ViETest::Log("Results: PSNR is %f (dB; 48 is max), "
  79                  "SSIM is %f (1 is perfect)",
  80                  psnr.average, ssim.average);
  81   }
  82
  83   // Note: must call AFTER CompareFiles.
  84   void TearDownFileRenderers() {
  85     TearDownFileRenderer(local_file_renderer_);
  86     TearDownFileRenderer(remote_file_renderer_);
  87   }
  88
  89   std::string input_file_;
  90   ViEToFileRenderer* local_file_renderer_;
  91   ViEToFileRenderer* remote_file_renderer_;
  92   ViEFileBasedComparisonTests tests_;
  93
  94  private:
  95   void SetUpFileRenderer(ViEToFileRenderer* file_renderer,
  96                          const std::string& suffix) {
  97     std::string output_path = ViETest::GetResultOutputPath();
  98     std::string filename = "render_output" + suffix;
  99
 100     if (!file_renderer->PrepareForRendering(output_path, filename)) {
 101       FAIL() << "Could not open output file " << filename <<
 102           " for writing.";
 103     }
 104   }
 105
 106   void TearDownFileRenderer(ViEToFileRenderer* file_renderer) {
 107       assert(file_renderer);
 108       bool test_failed = ::testing::UnitTest::GetInstance()->
 109           current_test_info()->result()->Failed();
 110       if (test_failed) {
 111         // Leave the files for analysis if the test failed.
 112         file_renderer->SaveOutputFile("failed-");
 113       }
 114       delete file_renderer;
 115     }
 116 };
 117
 118 TEST_F(ViEVideoVerificationTest, RunsBaseStandardTestWithoutErrors) {
 119   // I420 is lossless, so the I420 test should obviously get perfect results -
 120   // the local preview and remote output files should be bit-exact. This test
 121   // runs on external transport to ensure we do not drop packets.
 122   // However, it's hard to make 100% stringent requirements on the video engine
 123   // since for instance the jitter buffer has non-deterministic elements. If it
 124   // breaks five times in a row though, you probably introduced a bug.
 125   const double kReasonablePsnr = webrtc::test::kMetricsPerfectPSNR - 2.0f;
 126   const double kReasonableSsim = 0.99f;
 127   const int kNumAttempts = 5;
 128   for (int attempt = 0; attempt < kNumAttempts; ++attempt) {
 129     InitializeFileRenderers();
 130     ASSERT_TRUE(tests_.TestCallSetup(input_file_, kInputWidth, kInputHeight,
 131                                      local_file_renderer_,
 132                                      remote_file_renderer_));
 133     std::string remote_file = remote_file_renderer_->GetFullOutputPath();
 134     std::string local_preview = local_file_renderer_->GetFullOutputPath();
 135     StopRenderers();
 136
 137     double actual_psnr = 0;
 138     double actual_ssim = 0;
 139     CompareFiles(local_preview, remote_file, &actual_psnr, &actual_ssim);
 140
 141     TearDownFileRenderers();
 142
 143     if (actual_psnr > kReasonablePsnr && actual_ssim > kReasonableSsim) {
 144       // Test successful.
 145       return;
 146     } else {
 147       ViETest::Log("Retrying; attempt %d of %d.", attempt + 1, kNumAttempts);
 148     }
 149   }
 150
 151   FAIL() << "Failed to achieve near-perfect PSNR and SSIM results after " <<
 152       kNumAttempts << " attempts.";
 153 }
 154
 155 }  // namespace