src/chrome/browser/media/chrome_webrtc_video_quality_browsertest.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "base/environment.h"
   6 #include "base/file_util.h"
   7 #include "base/path_service.h"
   8 #include "base/process/launch.h"
   9 #include "base/strings/string_split.h"
  10 #include "base/strings/stringprintf.h"
  11 #include "base/test/test_timeouts.h"
  12 #include "base/time/time.h"
  13 #include "chrome/browser/chrome_notification_types.h"
  14 #include "chrome/browser/infobars/infobar.h"
  15 #include "chrome/browser/infobars/infobar_service.h"
  16 #include "chrome/browser/media/media_stream_infobar_delegate.h"
  17 #include "chrome/browser/media/webrtc_browsertest_base.h"
  18 #include "chrome/browser/media/webrtc_browsertest_common.h"
  19 #include "chrome/browser/profiles/profile.h"
  20 #include "chrome/browser/ui/browser.h"
  21 #include "chrome/browser/ui/browser_tabstrip.h"
  22 #include "chrome/browser/ui/tabs/tab_strip_model.h"
  23 #include "chrome/common/chrome_switches.h"
  24 #include "chrome/test/base/in_process_browser_test.h"
  25 #include "chrome/test/base/ui_test_utils.h"
  26 #include "chrome/test/ui/ui_test.h"
  27 #include "content/public/browser/notification_service.h"
  28 #include "content/public/test/browser_test_utils.h"
  29 #include "media/base/media_switches.h"
  30 #include "net/test/embedded_test_server/embedded_test_server.h"
  31 #include "net/test/python_utils.h"
  32 #include "testing/perf/perf_test.h"
  33 #include "ui/gl/gl_switches.h"
  34
  35 static const base::FilePath::CharType kFrameAnalyzerExecutable[] =
  36 #if defined(OS_WIN)
  37     FILE_PATH_LITERAL("frame_analyzer.exe");
  38 #else
  39     FILE_PATH_LITERAL("frame_analyzer");
  40 #endif
  41
  42 static const base::FilePath::CharType kArgbToI420ConverterExecutable[] =
  43 #if defined(OS_WIN)
  44     FILE_PATH_LITERAL("rgba_to_i420_converter.exe");
  45 #else
  46     FILE_PATH_LITERAL("rgba_to_i420_converter");
  47 #endif
  48
  49 static const char kHomeEnvName[] =
  50 #if defined(OS_WIN)
  51     "HOMEPATH";
  52 #else
  53     "HOME";
  54 #endif
  55
  56 // The working dir should be in the user's home folder.
  57 static const base::FilePath::CharType kWorkingDirName[] =
  58     FILE_PATH_LITERAL("webrtc_video_quality");
  59 static const base::FilePath::CharType kReferenceYuvFileName[] =
  60     FILE_PATH_LITERAL("reference_video.yuv");
  61 static const base::FilePath::CharType kCapturedYuvFileName[] =
  62     FILE_PATH_LITERAL("captured_video.yuv");
  63 static const base::FilePath::CharType kStatsFileName[] =
  64     FILE_PATH_LITERAL("stats.txt");
  65 static const char kMainWebrtcTestHtmlPage[] =
  66     "/webrtc/webrtc_jsep01_test.html";
  67 static const char kCapturingWebrtcHtmlPage[] =
  68     "/webrtc/webrtc_video_quality_test.html";
  69 static const int kVgaWidth = 640;
  70 static const int kVgaHeight = 480;
  71
  72 // If you change the port number, don't forget to modify video_extraction.js
  73 // too!
  74 static const char kPyWebSocketPortNumber[] = "12221";
  75
  76 // Test the video quality of the WebRTC output.
  77 //
  78 // Prerequisites: This test case must run on a machine with a virtual webcam
  79 // that plays video from the reference file located in <the running user's home
  80 // folder>/kWorkingDirName/kReferenceYuvFileName.
  81 //
  82 // You must also compile the chromium_builder_webrtc target before you run this
  83 // test to get all the tools built.
  84 //
  85 // The external compare_videos.py script also depends on two external
  86 // executables which must be located in the PATH when running this test.
  87 // * zxing (see the CPP version at https://code.google.com/p/zxing)
  88 // * ffmpeg 0.11.1 or compatible version (see http://www.ffmpeg.org)
  89 //
  90 // The test case will launch a custom binary (peerconnection_server) which will
  91 // allow two WebRTC clients to find each other.
  92 //
  93 // The test also runs several other custom binaries - rgba_to_i420 converter and
  94 // frame_analyzer. Both tools can be found under third_party/webrtc/tools. The
  95 // test also runs a stand alone Python implementation of a WebSocket server
  96 // (pywebsocket) and a barcode_decoder script.
  97 class WebRtcVideoQualityBrowserTest : public WebRtcTestBase {
  98  public:
  99   WebRtcVideoQualityBrowserTest()
 100       : pywebsocket_server_(0),
 101         environment_(base::Environment::Create()) {}
 102
 103   virtual void SetUpInProcessBrowserTestFixture() OVERRIDE {
 104     PeerConnectionServerRunner::KillAllPeerConnectionServersOnCurrentSystem();
 105     DetectErrorsInJavaScript();  // Look for errors in our rather complex js.
 106   }
 107
 108   virtual void SetUpCommandLine(CommandLine* command_line) OVERRIDE {
 109     // This test expects real device handling and requires a real webcam / audio
 110     // device; it will not work with fake devices.
 111     EXPECT_FALSE(
 112         command_line->HasSwitch(switches::kUseFakeDeviceForMediaStream))
 113         << "You cannot run this test with fake devices.";
 114
 115 #if defined(OS_MACOSX)
 116     // TODO(mcasas): Remove this switch when ManyCam virtual video capture
 117     // device starts supporting AVFoundation, see http://crbug.com/327618.
 118     command_line->AppendSwitch(switches::kDisableAVFoundation);
 119 #endif
 120
 121     // The video playback will not work without a GPU, so force its use here.
 122     command_line->AppendSwitch(switches::kUseGpuInTests);
 123   }
 124
 125   bool HasAllRequiredResources() {
 126     if (!base::PathExists(GetWorkingDir())) {
 127       LOG(ERROR) << "Cannot find the working directory for the reference video "
 128           "and the temporary files:" << GetWorkingDir().value();
 129       return false;
 130     }
 131     base::FilePath reference_file =
 132         GetWorkingDir().Append(kReferenceYuvFileName);
 133     if (!base::PathExists(reference_file)) {
 134       LOG(ERROR) << "Cannot find the reference file to be used for video "
 135           << "quality comparison: " << reference_file.value();
 136       return false;
 137     }
 138     return true;
 139   }
 140
 141   bool StartPyWebSocketServer() {
 142     base::FilePath path_pywebsocket_dir =
 143         GetSourceDir().Append(FILE_PATH_LITERAL("third_party/pywebsocket/src"));
 144     base::FilePath pywebsocket_server = path_pywebsocket_dir.Append(
 145         FILE_PATH_LITERAL("mod_pywebsocket/standalone.py"));
 146     base::FilePath path_to_data_handler =
 147         GetSourceDir().Append(FILE_PATH_LITERAL("chrome/test/functional"));
 148
 149     if (!base::PathExists(pywebsocket_server)) {
 150       LOG(ERROR) << "Missing pywebsocket server.";
 151       return false;
 152     }
 153     if (!base::PathExists(path_to_data_handler)) {
 154       LOG(ERROR) << "Missing data handler for pywebsocket server.";
 155       return false;
 156     }
 157
 158     AppendToPythonPath(path_pywebsocket_dir);
 159
 160     // Note: don't append switches to this command since it will mess up the
 161     // -u in the python invocation!
 162     CommandLine pywebsocket_command(CommandLine::NO_PROGRAM);
 163     EXPECT_TRUE(GetPythonCommand(&pywebsocket_command));
 164
 165     pywebsocket_command.AppendArgPath(pywebsocket_server);
 166     pywebsocket_command.AppendArg("-p");
 167     pywebsocket_command.AppendArg(kPyWebSocketPortNumber);
 168     pywebsocket_command.AppendArg("-d");
 169     pywebsocket_command.AppendArgPath(path_to_data_handler);
 170
 171     VLOG(0) << "Running " << pywebsocket_command.GetCommandLineString();
 172     return base::LaunchProcess(pywebsocket_command, base::LaunchOptions(),
 173                                &pywebsocket_server_);
 174   }
 175
 176   bool ShutdownPyWebSocketServer() {
 177     return base::KillProcess(pywebsocket_server_, 0, false);
 178   }
 179
 180   void EstablishCall(content::WebContents* from_tab,
 181                      content::WebContents* to_tab) {
 182     EXPECT_EQ("ok-peerconnection-created",
 183               ExecuteJavascript("preparePeerConnection()", from_tab));
 184     EXPECT_EQ("ok-added", ExecuteJavascript("addLocalStream()", from_tab));
 185     EXPECT_EQ("ok-negotiating", ExecuteJavascript("negotiateCall()", from_tab));
 186
 187     // Ensure the call gets up on both sides.
 188     EXPECT_TRUE(PollingWaitUntil(
 189         "getPeerConnectionReadyState()", "active", from_tab));
 190     EXPECT_TRUE(PollingWaitUntil(
 191         "getPeerConnectionReadyState()", "active", to_tab));
 192   }
 193
 194   void HangUp(content::WebContents* from_tab) {
 195     EXPECT_EQ("ok-call-hung-up", ExecuteJavascript("hangUp()", from_tab));
 196   }
 197
 198   void WaitUntilHangupVerified(content::WebContents* tab_contents) {
 199     EXPECT_TRUE(PollingWaitUntil(
 200         "getPeerConnectionReadyState()", "no-peer-connection", tab_contents));
 201   }
 202
 203   // Runs the RGBA to I420 converter on the video in |capture_video_filename|,
 204   // which should contain frames of size |width| x |height|.
 205   //
 206   // The rgba_to_i420_converter is part of the webrtc_test_tools target which
 207   // should be build prior to running this test. The resulting binary should
 208   // live next to Chrome.
 209   bool RunARGBtoI420Converter(int width,
 210                               int height,
 211                               const base::FilePath& captured_video_filename) {
 212     base::FilePath path_to_converter = base::MakeAbsoluteFilePath(
 213         GetBrowserDir().Append(kArgbToI420ConverterExecutable));
 214
 215     if (!base::PathExists(path_to_converter)) {
 216       LOG(ERROR) << "Missing ARGB->I420 converter: should be in "
 217           << path_to_converter.value();
 218       return false;
 219     }
 220
 221     CommandLine converter_command(path_to_converter);
 222     converter_command.AppendSwitchPath("--frames_dir", GetWorkingDir());
 223     converter_command.AppendSwitchPath("--output_file",
 224                                        captured_video_filename);
 225     converter_command.AppendSwitchASCII("--width",
 226                                         base::StringPrintf("%d", width));
 227     converter_command.AppendSwitchASCII("--height",
 228                                         base::StringPrintf("%d", height));
 229
 230     // We produce an output file that will later be used as an input to the
 231     // barcode decoder and frame analyzer tools.
 232     VLOG(0) << "Running " << converter_command.GetCommandLineString();
 233     std::string result;
 234     bool ok = base::GetAppOutput(converter_command, &result);
 235     VLOG(0) << "Output was:\n\n" << result;
 236     return ok;
 237   }
 238
 239   // Compares the |captured_video_filename| with the |reference_video_filename|.
 240   //
 241   // The barcode decoder decodes the captured video containing barcodes overlaid
 242   // into every frame of the video (produced by rgba_to_i420_converter). It
 243   // produces a set of PNG images and a |stats_file| that maps each captured
 244   // frame to a frame in the reference video. The frames should be of size
 245   // |width| x |height|.
 246   // All measurements calculated are printed as perf parsable numbers to stdout.
 247   bool CompareVideosAndPrintResult(
 248       int width,
 249       int height,
 250       const base::FilePath& captured_video_filename,
 251       const base::FilePath& reference_video_filename,
 252       const base::FilePath& stats_file) {
 253
 254     base::FilePath path_to_analyzer = base::MakeAbsoluteFilePath(
 255         GetBrowserDir().Append(kFrameAnalyzerExecutable));
 256     base::FilePath path_to_compare_script = GetSourceDir().Append(
 257         FILE_PATH_LITERAL("third_party/webrtc/tools/compare_videos.py"));
 258
 259     if (!base::PathExists(path_to_analyzer)) {
 260       LOG(ERROR) << "Missing frame analyzer: should be in "
 261           << path_to_analyzer.value();
 262       return false;
 263     }
 264     if (!base::PathExists(path_to_compare_script)) {
 265       LOG(ERROR) << "Missing video compare script: should be in "
 266           << path_to_compare_script.value();
 267       return false;
 268     }
 269
 270     // Note: don't append switches to this command since it will mess up the
 271     // -u in the python invocation!
 272     CommandLine compare_command(CommandLine::NO_PROGRAM);
 273     EXPECT_TRUE(GetPythonCommand(&compare_command));
 274
 275     compare_command.AppendArgPath(path_to_compare_script);
 276     compare_command.AppendArg("--label=VGA");
 277     compare_command.AppendArg("--ref_video");
 278     compare_command.AppendArgPath(reference_video_filename);
 279     compare_command.AppendArg("--test_video");
 280     compare_command.AppendArgPath(captured_video_filename);
 281     compare_command.AppendArg("--frame_analyzer");
 282     compare_command.AppendArgPath(path_to_analyzer);
 283     compare_command.AppendArg("--yuv_frame_width");
 284     compare_command.AppendArg(base::StringPrintf("%d", width));
 285     compare_command.AppendArg("--yuv_frame_height");
 286     compare_command.AppendArg(base::StringPrintf("%d", height));
 287     compare_command.AppendArg("--stats_file");
 288     compare_command.AppendArgPath(stats_file);
 289
 290     VLOG(0) << "Running " << compare_command.GetCommandLineString();
 291     std::string output;
 292     bool ok = base::GetAppOutput(compare_command, &output);
 293     // Print to stdout to ensure the perf numbers are parsed properly by the
 294     // buildbot step.
 295     printf("Output was:\n\n%s\n", output.c_str());
 296     return ok;
 297   }
 298
 299   base::FilePath GetWorkingDir() {
 300     std::string home_dir;
 301     environment_->GetVar(kHomeEnvName, &home_dir);
 302     base::FilePath::StringType native_home_dir(home_dir.begin(),
 303                                                home_dir.end());
 304     return base::FilePath(native_home_dir).Append(kWorkingDirName);
 305   }
 306
 307   PeerConnectionServerRunner peerconnection_server_;
 308
 309  private:
 310   base::FilePath GetSourceDir() {
 311     base::FilePath source_dir;
 312     PathService::Get(base::DIR_SOURCE_ROOT, &source_dir);
 313     return source_dir;
 314   }
 315
 316   base::FilePath GetBrowserDir() {
 317     base::FilePath browser_dir;
 318     EXPECT_TRUE(PathService::Get(base::DIR_MODULE, &browser_dir));
 319     return browser_dir;
 320   }
 321
 322   base::ProcessHandle pywebsocket_server_;
 323   scoped_ptr<base::Environment> environment_;
 324 };
 325
 326 IN_PROC_BROWSER_TEST_F(WebRtcVideoQualityBrowserTest,
 327                        MANUAL_TestVGAVideoQuality) {
 328   ASSERT_GE(TestTimeouts::action_max_timeout().InSeconds(), 150) <<
 329       "This is a long-running test; you must specify "
 330       "--ui-test-action-max-timeout to have a value of at least 150000.";
 331
 332   ASSERT_TRUE(HasAllRequiredResources());
 333   ASSERT_TRUE(embedded_test_server()->InitializeAndWaitUntilReady());
 334   ASSERT_TRUE(StartPyWebSocketServer());
 335   ASSERT_TRUE(peerconnection_server_.Start());
 336
 337   content::WebContents* left_tab =
 338       OpenPageAndGetUserMediaInNewTab(
 339           embedded_test_server()->GetURL(kMainWebrtcTestHtmlPage));
 340   content::WebContents* right_tab =
 341       OpenPageAndGetUserMediaInNewTab(
 342           embedded_test_server()->GetURL(kCapturingWebrtcHtmlPage));
 343
 344   ConnectToPeerConnectionServer("peer 1", left_tab);
 345   ConnectToPeerConnectionServer("peer 2", right_tab);
 346
 347   EstablishCall(left_tab, right_tab);
 348
 349   // Poll slower here to avoid flooding the log with messages: capturing and
 350   // sending frames take quite a bit of time.
 351   int polling_interval_msec = 1000;
 352
 353   EXPECT_TRUE(PollingWaitUntil(
 354       "doneFrameCapturing()", "done-capturing", right_tab,
 355       polling_interval_msec));
 356
 357   HangUp(left_tab);
 358   WaitUntilHangupVerified(left_tab);
 359   WaitUntilHangupVerified(right_tab);
 360
 361   EXPECT_TRUE(PollingWaitUntil(
 362       "haveMoreFramesToSend()", "no-more-frames", right_tab,
 363       polling_interval_msec));
 364
 365   // Shut everything down to avoid having the javascript race with the analysis
 366   // tools. For instance, dont have console log printouts interleave with the
 367   // RESULT lines from the analysis tools (crbug.com/323200).
 368   ASSERT_TRUE(peerconnection_server_.Stop());
 369   ASSERT_TRUE(ShutdownPyWebSocketServer());
 370
 371   chrome::CloseWebContents(browser(), left_tab, false);
 372   chrome::CloseWebContents(browser(), right_tab, false);
 373
 374   RunARGBtoI420Converter(
 375       kVgaWidth, kVgaHeight, GetWorkingDir().Append(kCapturedYuvFileName));
 376   ASSERT_TRUE(
 377       CompareVideosAndPrintResult(kVgaWidth,
 378                                   kVgaHeight,
 379                                   GetWorkingDir().Append(kCapturedYuvFileName),
 380                                   GetWorkingDir().Append(kReferenceYuvFileName),
 381                                   GetWorkingDir().Append(kStatsFileName)));
 382 }