src/chrome/browser/media/chrome_webrtc_video_quality_browsertest.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "base/command_line.h"
   6 #include "base/environment.h"
   7 #include "base/file_util.h"
   8 #include "base/path_service.h"
   9 #include "base/process/launch.h"
  10 #include "base/strings/string_split.h"
  11 #include "base/strings/stringprintf.h"
  12 #include "base/test/test_timeouts.h"
  13 #include "base/time/time.h"
  14 #include "chrome/browser/chrome_notification_types.h"
  15 #include "chrome/browser/infobars/infobar_service.h"
  16 #include "chrome/browser/media/media_stream_infobar_delegate.h"
  17 #include "chrome/browser/media/webrtc_browsertest_base.h"
  18 #include "chrome/browser/media/webrtc_browsertest_common.h"
  19 #include "chrome/browser/profiles/profile.h"
  20 #include "chrome/browser/ui/browser.h"
  21 #include "chrome/browser/ui/browser_tabstrip.h"
  22 #include "chrome/browser/ui/tabs/tab_strip_model.h"
  23 #include "chrome/common/chrome_switches.h"
  24 #include "chrome/test/base/in_process_browser_test.h"
  25 #include "chrome/test/base/ui_test_utils.h"
  26 #include "components/infobars/core/infobar.h"
  27 #include "content/public/browser/notification_service.h"
  28 #include "content/public/test/browser_test_utils.h"
  29 #include "media/base/media_switches.h"
  30 #include "net/test/embedded_test_server/embedded_test_server.h"
  31 #include "net/test/python_utils.h"
  32 #include "testing/perf/perf_test.h"
  33 #include "ui/gl/gl_switches.h"
  34
  35 // For fine-grained suppression on flaky tests.
  36 #if defined(OS_WIN)
  37 #include "base/win/windows_version.h"
  38 #endif
  39
  40 static const base::FilePath::CharType kFrameAnalyzerExecutable[] =
  41 #if defined(OS_WIN)
  42     FILE_PATH_LITERAL("frame_analyzer.exe");
  43 #else
  44     FILE_PATH_LITERAL("frame_analyzer");
  45 #endif
  46
  47 static const base::FilePath::CharType kArgbToI420ConverterExecutable[] =
  48 #if defined(OS_WIN)
  49     FILE_PATH_LITERAL("rgba_to_i420_converter.exe");
  50 #else
  51     FILE_PATH_LITERAL("rgba_to_i420_converter");
  52 #endif
  53
  54 static const char kHomeEnvName[] =
  55 #if defined(OS_WIN)
  56     "USERPROFILE";
  57 #else
  58     "HOME";
  59 #endif
  60
  61 // The working dir should be in the user's home folder.
  62 static const base::FilePath::CharType kWorkingDirName[] =
  63     FILE_PATH_LITERAL("webrtc_video_quality");
  64 static const base::FilePath::CharType kCapturedYuvFileName[] =
  65     FILE_PATH_LITERAL("captured_video.yuv");
  66 static const base::FilePath::CharType kStatsFileName[] =
  67     FILE_PATH_LITERAL("stats.txt");
  68 static const char kMainWebrtcTestHtmlPage[] =
  69     "/webrtc/webrtc_jsep01_test.html";
  70
  71 // If you change the port number, don't forget to modify video_extraction.js
  72 // too!
  73 static const char kPyWebSocketPortNumber[] = "12221";
  74
  75 static const struct VideoQualityTestConfig {
  76   const char* test_name;
  77   int width;
  78   int height;
  79   const char* capture_page;
  80   const base::FilePath::CharType* reference_video;
  81   const char* constraints;
  82 } kVideoConfigurations[] = {
  83   { "360p", 640, 360,
  84     "/webrtc/webrtc_video_quality_test.html",
  85     test::kReferenceFileName360p,
  86     WebRtcTestBase::kAudioVideoCallConstraints360p },
  87   { "720p", 1280, 720,
  88     "/webrtc/webrtc_video_quality_test_hd.html",
  89     test::kReferenceFileName720p,
  90     WebRtcTestBase::kAudioVideoCallConstraints720p },
  91 };
  92
  93 // Test the video quality of the WebRTC output.
  94 //
  95 // Prerequisites: This test case must run on a machine with a chrome playing
  96 // the video from the reference files located in GetReferenceFilesDir().
  97 // The file kReferenceY4mFileName.kY4mFileExtension is played using a
  98 // FileVideoCaptureDevice and its sibling with kYuvFileExtension is used for
  99 // comparison.
 100 //
 101 // You must also compile the chromium_builder_webrtc target before you run this
 102 // test to get all the tools built.
 103 //
 104 // The external compare_videos.py script also depends on two external
 105 // executables which must be located in the PATH when running this test.
 106 // * zxing (see the CPP version at https://code.google.com/p/zxing)
 107 // * ffmpeg 0.11.1 or compatible version (see http://www.ffmpeg.org)
 108 //
 109 // The test runs several custom binaries - rgba_to_i420 converter and
 110 // frame_analyzer. Both tools can be found under third_party/webrtc/tools. The
 111 // test also runs a stand alone Python implementation of a WebSocket server
 112 // (pywebsocket) and a barcode_decoder script.
 113 class WebRtcVideoQualityBrowserTest : public WebRtcTestBase,
 114     public testing::WithParamInterface<VideoQualityTestConfig> {
 115  public:
 116   WebRtcVideoQualityBrowserTest()
 117       : pywebsocket_server_(0),
 118         environment_(base::Environment::Create()) {
 119     test_config_ = GetParam();
 120   }
 121
 122   virtual void SetUpInProcessBrowserTestFixture() OVERRIDE {
 123     DetectErrorsInJavaScript();  // Look for errors in our rather complex js.
 124   }
 125
 126   virtual void SetUpCommandLine(CommandLine* command_line) OVERRIDE {
 127     // Set up the command line option with the expected file name. We will check
 128     // its existence in HasAllRequiredResources().
 129     webrtc_reference_video_y4m_ = test::GetReferenceFilesDir()
 130         .Append(test_config_.reference_video)
 131         .AddExtension(test::kY4mFileExtension);
 132     command_line->AppendSwitchPath(switches::kUseFileForFakeVideoCapture,
 133                                    webrtc_reference_video_y4m_);
 134     command_line->AppendSwitch(switches::kUseFakeDeviceForMediaStream);
 135
 136     // The video playback will not work without a GPU, so force its use here.
 137     command_line->AppendSwitch(switches::kUseGpuInTests);
 138   }
 139
 140   bool HasAllRequiredResources() {
 141     if (!base::PathExists(GetWorkingDir())) {
 142       LOG(ERROR) << "Cannot find the working directory for the temporary "
 143           "files:" << GetWorkingDir().value();
 144       return false;
 145     }
 146
 147     // Ensure we have the required input files.
 148     return test::HasReferenceFilesInCheckout();
 149   }
 150
 151   bool StartPyWebSocketServer() {
 152     base::FilePath path_pywebsocket_dir =
 153         GetSourceDir().Append(FILE_PATH_LITERAL("third_party/pywebsocket/src"));
 154     base::FilePath pywebsocket_server = path_pywebsocket_dir.Append(
 155         FILE_PATH_LITERAL("mod_pywebsocket/standalone.py"));
 156     base::FilePath path_to_data_handler =
 157         GetSourceDir().Append(FILE_PATH_LITERAL("chrome/test/data/webrtc/wsh"));
 158
 159     if (!base::PathExists(pywebsocket_server)) {
 160       LOG(ERROR) << "Missing pywebsocket server.";
 161       return false;
 162     }
 163     if (!base::PathExists(path_to_data_handler)) {
 164       LOG(ERROR) << "Missing data handler for pywebsocket server.";
 165       return false;
 166     }
 167
 168     AppendToPythonPath(path_pywebsocket_dir);
 169
 170     // Note: don't append switches to this command since it will mess up the
 171     // -u in the python invocation!
 172     CommandLine pywebsocket_command(CommandLine::NO_PROGRAM);
 173     EXPECT_TRUE(GetPythonCommand(&pywebsocket_command));
 174
 175     pywebsocket_command.AppendArgPath(pywebsocket_server);
 176     pywebsocket_command.AppendArg("-p");
 177     pywebsocket_command.AppendArg(kPyWebSocketPortNumber);
 178     pywebsocket_command.AppendArg("-d");
 179     pywebsocket_command.AppendArgPath(path_to_data_handler);
 180
 181     VLOG(0) << "Running " << pywebsocket_command.GetCommandLineString();
 182     return base::LaunchProcess(pywebsocket_command, base::LaunchOptions(),
 183                                &pywebsocket_server_);
 184   }
 185
 186   bool ShutdownPyWebSocketServer() {
 187     return base::KillProcess(pywebsocket_server_, 0, false);
 188   }
 189
 190   // Runs the RGBA to I420 converter on the video in |capture_video_filename|,
 191   // which should contain frames of size |width| x |height|.
 192   //
 193   // The rgba_to_i420_converter is part of the webrtc_test_tools target which
 194   // should be build prior to running this test. The resulting binary should
 195   // live next to Chrome.
 196   bool RunARGBtoI420Converter(int width,
 197                               int height,
 198                               const base::FilePath& captured_video_filename) {
 199     base::FilePath path_to_converter = base::MakeAbsoluteFilePath(
 200         GetBrowserDir().Append(kArgbToI420ConverterExecutable));
 201
 202     if (!base::PathExists(path_to_converter)) {
 203       LOG(ERROR) << "Missing ARGB->I420 converter: should be in "
 204           << path_to_converter.value();
 205       return false;
 206     }
 207
 208     CommandLine converter_command(path_to_converter);
 209     converter_command.AppendSwitchPath("--frames_dir", GetWorkingDir());
 210     converter_command.AppendSwitchPath("--output_file",
 211                                        captured_video_filename);
 212     converter_command.AppendSwitchASCII("--width",
 213                                         base::StringPrintf("%d", width));
 214     converter_command.AppendSwitchASCII("--height",
 215                                         base::StringPrintf("%d", height));
 216     converter_command.AppendSwitchASCII("--delete_frames", "true");
 217
 218     // We produce an output file that will later be used as an input to the
 219     // barcode decoder and frame analyzer tools.
 220     VLOG(0) << "Running " << converter_command.GetCommandLineString();
 221     std::string result;
 222     bool ok = base::GetAppOutput(converter_command, &result);
 223     VLOG(0) << "Output was:\n\n" << result;
 224     return ok;
 225   }
 226
 227   // Compares the |captured_video_filename| with the |reference_video_filename|.
 228   //
 229   // The barcode decoder decodes the captured video containing barcodes overlaid
 230   // into every frame of the video (produced by rgba_to_i420_converter). It
 231   // produces a set of PNG images and a |stats_file| that maps each captured
 232   // frame to a frame in the reference video. The frames should be of size
 233   // |width| x |height|.
 234   // All measurements calculated are printed as perf parsable numbers to stdout.
 235   bool CompareVideosAndPrintResult(
 236       const char* test_label,
 237       int width,
 238       int height,
 239       const base::FilePath& captured_video_filename,
 240       const base::FilePath& reference_video_filename,
 241       const base::FilePath& stats_file) {
 242
 243     base::FilePath path_to_analyzer = base::MakeAbsoluteFilePath(
 244         GetBrowserDir().Append(kFrameAnalyzerExecutable));
 245     base::FilePath path_to_compare_script = GetSourceDir().Append(
 246         FILE_PATH_LITERAL("third_party/webrtc/tools/compare_videos.py"));
 247
 248     if (!base::PathExists(path_to_analyzer)) {
 249       LOG(ERROR) << "Missing frame analyzer: should be in "
 250           << path_to_analyzer.value();
 251       return false;
 252     }
 253     if (!base::PathExists(path_to_compare_script)) {
 254       LOG(ERROR) << "Missing video compare script: should be in "
 255           << path_to_compare_script.value();
 256       return false;
 257     }
 258
 259     // Note: don't append switches to this command since it will mess up the
 260     // -u in the python invocation!
 261     CommandLine compare_command(CommandLine::NO_PROGRAM);
 262     EXPECT_TRUE(GetPythonCommand(&compare_command));
 263
 264     compare_command.AppendArgPath(path_to_compare_script);
 265     compare_command.AppendArg(base::StringPrintf("--label=%s", test_label));
 266     compare_command.AppendArg("--ref_video");
 267     compare_command.AppendArgPath(reference_video_filename);
 268     compare_command.AppendArg("--test_video");
 269     compare_command.AppendArgPath(captured_video_filename);
 270     compare_command.AppendArg("--frame_analyzer");
 271     compare_command.AppendArgPath(path_to_analyzer);
 272     compare_command.AppendArg("--yuv_frame_width");
 273     compare_command.AppendArg(base::StringPrintf("%d", width));
 274     compare_command.AppendArg("--yuv_frame_height");
 275     compare_command.AppendArg(base::StringPrintf("%d", height));
 276     compare_command.AppendArg("--stats_file");
 277     compare_command.AppendArgPath(stats_file);
 278
 279     VLOG(0) << "Running " << compare_command.GetCommandLineString();
 280     std::string output;
 281     bool ok = base::GetAppOutput(compare_command, &output);
 282     // Print to stdout to ensure the perf numbers are parsed properly by the
 283     // buildbot step.
 284     printf("Output was:\n\n%s\n", output.c_str());
 285     return ok;
 286   }
 287
 288   base::FilePath GetWorkingDir() {
 289     std::string home_dir;
 290     environment_->GetVar(kHomeEnvName, &home_dir);
 291     base::FilePath::StringType native_home_dir(home_dir.begin(),
 292                                                home_dir.end());
 293     return base::FilePath(native_home_dir).Append(kWorkingDirName);
 294   }
 295
 296  protected:
 297   VideoQualityTestConfig test_config_;
 298
 299  private:
 300   base::FilePath GetSourceDir() {
 301     base::FilePath source_dir;
 302     PathService::Get(base::DIR_SOURCE_ROOT, &source_dir);
 303     return source_dir;
 304   }
 305
 306   base::FilePath GetBrowserDir() {
 307     base::FilePath browser_dir;
 308     EXPECT_TRUE(PathService::Get(base::DIR_MODULE, &browser_dir));
 309     return browser_dir;
 310   }
 311
 312   base::ProcessHandle pywebsocket_server_;
 313   scoped_ptr<base::Environment> environment_;
 314   base::FilePath webrtc_reference_video_y4m_;
 315 };
 316
 317 INSTANTIATE_TEST_CASE_P(
 318     WebRtcVideoQualityBrowserTests,
 319     WebRtcVideoQualityBrowserTest,
 320     testing::ValuesIn(kVideoConfigurations));
 321
 322 IN_PROC_BROWSER_TEST_P(WebRtcVideoQualityBrowserTest,
 323                        MANUAL_TestVideoQuality) {
 324
 325 #if defined(OS_WIN)
 326   // Fails on XP. http://crbug.com/353078
 327   if (base::win::GetVersion() <= base::win::VERSION_XP)
 328     return;
 329 #endif
 330
 331   ASSERT_GE(TestTimeouts::action_max_timeout().InSeconds(), 150) <<
 332       "This is a long-running test; you must specify "
 333       "--ui-test-action-max-timeout to have a value of at least 150000.";
 334   ASSERT_TRUE(HasAllRequiredResources());
 335   ASSERT_TRUE(embedded_test_server()->InitializeAndWaitUntilReady());
 336   ASSERT_TRUE(StartPyWebSocketServer());
 337
 338   content::WebContents* left_tab =
 339       OpenPageAndGetUserMediaInNewTabWithConstraints(
 340           embedded_test_server()->GetURL(kMainWebrtcTestHtmlPage),
 341           test_config_.constraints);
 342   content::WebContents* right_tab =
 343       OpenPageAndGetUserMediaInNewTabWithConstraints(
 344           embedded_test_server()->GetURL(test_config_.capture_page),
 345           test_config_.constraints);
 346
 347   SetupPeerconnectionWithLocalStream(left_tab);
 348   SetupPeerconnectionWithLocalStream(right_tab);
 349
 350   NegotiateCall(left_tab, right_tab);
 351
 352   // Poll slower here to avoid flooding the log with messages: capturing and
 353   // sending frames take quite a bit of time.
 354   int polling_interval_msec = 1000;
 355
 356   EXPECT_TRUE(test::PollingWaitUntil(
 357       "doneFrameCapturing()", "done-capturing", right_tab,
 358       polling_interval_msec));
 359
 360   HangUp(left_tab);
 361
 362   EXPECT_TRUE(test::PollingWaitUntil(
 363       "haveMoreFramesToSend()", "no-more-frames", right_tab,
 364       polling_interval_msec));
 365
 366   // Shut everything down to avoid having the javascript race with the analysis
 367   // tools. For instance, dont have console log printouts interleave with the
 368   // RESULT lines from the analysis tools (crbug.com/323200).
 369   ASSERT_TRUE(ShutdownPyWebSocketServer());
 370
 371   chrome::CloseWebContents(browser(), left_tab, false);
 372   chrome::CloseWebContents(browser(), right_tab, false);
 373
 374   RunARGBtoI420Converter(
 375       test_config_.width, test_config_.height,
 376       GetWorkingDir().Append(kCapturedYuvFileName));
 377   ASSERT_TRUE(CompareVideosAndPrintResult(
 378       test_config_.test_name,
 379       test_config_.width,
 380       test_config_.height,
 381       GetWorkingDir().Append(kCapturedYuvFileName),
 382       test::GetReferenceFilesDir()
 383           .Append(test_config_.reference_video)
 384           .AddExtension(test::kYuvFileExtension),
 385       GetWorkingDir().Append(kStatsFileName)));
 386 }