src/chrome/browser/media/chrome_webrtc_audio_quality_browsertest.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include <ctime>
   6
   7 #include "base/command_line.h"
   8 #include "base/file_util.h"
   9 #include "base/path_service.h"
  10 #include "base/process/launch.h"
  11 #include "base/scoped_native_library.h"
  12 #include "base/strings/stringprintf.h"
  13 #include "base/win/windows_version.h"
  14 #include "chrome/browser/media/webrtc_browsertest_base.h"
  15 #include "chrome/browser/media/webrtc_browsertest_common.h"
  16 #include "chrome/browser/profiles/profile.h"
  17 #include "chrome/browser/ui/browser.h"
  18 #include "chrome/browser/ui/browser_tabstrip.h"
  19 #include "chrome/browser/ui/tabs/tab_strip_model.h"
  20 #include "chrome/common/chrome_paths.h"
  21 #include "chrome/common/chrome_switches.h"
  22 #include "chrome/test/base/ui_test_utils.h"
  23 #include "chrome/test/ui/ui_test.h"
  24 #include "content/public/test/browser_test_utils.h"
  25 #include "net/test/embedded_test_server/embedded_test_server.h"
  26 #include "testing/perf/perf_test.h"
  27
  28 static const base::FilePath::CharType kReferenceFile[] =
  29 #if defined (OS_WIN)
  30     FILE_PATH_LITERAL("pyauto_private/webrtc/human-voice-win.wav");
  31 #else
  32     FILE_PATH_LITERAL("pyauto_private/webrtc/human-voice-linux.wav");
  33 #endif
  34
  35 // The javascript will load the reference file relative to its location,
  36 // which is in /webrtc on the web server. Therefore, prepend a '..' traversal.
  37 static const char kReferenceFileRelativeUrl[] =
  38 #if defined (OS_WIN)
  39     "../pyauto_private/webrtc/human-voice-win.wav";
  40 #else
  41     "../pyauto_private/webrtc/human-voice-linux.wav";
  42 #endif
  43
  44 static const base::FilePath::CharType kToolsPath[] =
  45     FILE_PATH_LITERAL("pyauto_private/media/tools");
  46
  47 static const char kMainWebrtcTestHtmlPage[] =
  48     "files/webrtc/webrtc_audio_quality_test.html";
  49
  50 static base::FilePath GetTestDataDir() {
  51   base::FilePath source_dir;
  52   PathService::Get(chrome::DIR_TEST_DATA, &source_dir);
  53   return source_dir;
  54 }
  55
  56 // Test we can set up a WebRTC call and play audio through it.
  57 //
  58 // You must have the src-internal solution in your .gclient to put the required
  59 // pyauto_private directory into chrome/test/data/.
  60 //
  61 // This test will only work on machines that have been configured to record
  62 // their own input.
  63 //
  64 // On Linux:
  65 // 1. # sudo apt-get install pavucontrol
  66 // 2. For the user who will run the test: # pavucontrol
  67 // 3. In a separate terminal, # arecord dummy
  68 // 4. In pavucontrol, go to the recording tab.
  69 // 5. For the ALSA plug-in [aplay]: ALSA Capture from, change from <x> to
  70 //    <Monitor of x>, where x is whatever your primary sound device is called.
  71 // 6. Try launching chrome as the target user on the target machine, try
  72 //    playing, say, a YouTube video, and record with # arecord -f dat tmp.dat.
  73 //    Verify the recording with aplay (should have recorded what you played
  74 //    from chrome).
  75 //
  76 // Note: the volume for ALL your input devices will be forced to 100% by
  77 //       running this test on Linux.
  78 //
  79 // On Windows 7:
  80 // 1. Control panel > Sound > Manage audio devices.
  81 // 2. In the recording tab, right-click in an empty space in the pane with the
  82 //    devices. Tick 'show disabled devices'.
  83 // 3. You should see a 'stero mix' device - this is what your speakers output.
  84 //    Right click > Properties.
  85 // 4. In the Listen tab for the mix device, check the 'listen to this device'
  86 //    checkbox. Ensure the mix device is the default recording device.
  87 // 5. Launch chrome and try playing a video with sound. You should see
  88 //    in the volume meter for the mix device. Configure the mix device to have
  89 //    50 / 100 in level. Also go into the playback tab, right-click Speakers,
  90 //    and set that level to 50 / 100. Otherwise you will get distortion in
  91 //    the recording.
  92 class WebrtcAudioQualityBrowserTest : public WebRtcTestBase {
  93  public:
  94   virtual void SetUpInProcessBrowserTestFixture() OVERRIDE {
  95     PeerConnectionServerRunner::KillAllPeerConnectionServersOnCurrentSystem();
  96   }
  97
  98   virtual void SetUpCommandLine(CommandLine* command_line) OVERRIDE {
  99     // This test expects real device handling and requires a real webcam / audio
 100     // device; it will not work with fake devices.
 101     EXPECT_FALSE(command_line->HasSwitch(
 102         switches::kUseFakeDeviceForMediaStream));
 103     EXPECT_FALSE(command_line->HasSwitch(
 104         switches::kUseFakeUIForMediaStream));
 105   }
 106
 107   bool HasAllRequiredResources() {
 108     base::FilePath reference_file =
 109         GetTestDataDir().Append(kReferenceFile);
 110     if (!base::PathExists(reference_file)) {
 111       LOG(ERROR) << "Cannot find the reference file to be used for audio "
 112           << "quality comparison: " << reference_file.value();
 113       return false;
 114     }
 115     return true;
 116   }
 117
 118   void AddAudioFile(const std::string& input_file_relative_url,
 119                     content::WebContents* tab_contents) {
 120     EXPECT_EQ("ok-added", ExecuteJavascript(
 121         "addAudioFile('" + input_file_relative_url + "')", tab_contents));
 122   }
 123
 124   void PlayAudioFile(content::WebContents* tab_contents) {
 125     EXPECT_EQ("ok-playing", ExecuteJavascript("playAudioFile()", tab_contents));
 126   }
 127
 128   // Ensures we didn't get any errors asynchronously (e.g. while no javascript
 129   // call from this test was outstanding).
 130   // TODO(phoglund): this becomes obsolete when we switch to communicating with
 131   // the DOM message queue.
 132   void AssertNoAsynchronousErrors(content::WebContents* tab_contents) {
 133     EXPECT_EQ("ok-no-errors",
 134               ExecuteJavascript("getAnyTestFailures()", tab_contents));
 135   }
 136
 137   void EstablishCall(content::WebContents* from_tab,
 138                      content::WebContents* to_tab) {
 139     EXPECT_EQ("ok-negotiating",
 140               ExecuteJavascript("negotiateCall()", from_tab));
 141
 142     // Ensure the call gets up on both sides.
 143     EXPECT_TRUE(PollingWaitUntil("getPeerConnectionReadyState()",
 144                                  "active", from_tab));
 145     EXPECT_TRUE(PollingWaitUntil("getPeerConnectionReadyState()",
 146                                  "active", to_tab));
 147   }
 148
 149   void HangUp(content::WebContents* from_tab) {
 150     EXPECT_EQ("ok-call-hung-up", ExecuteJavascript("hangUp()", from_tab));
 151   }
 152
 153   void WaitUntilHangupVerified(content::WebContents* tab_contents) {
 154     EXPECT_TRUE(PollingWaitUntil("getPeerConnectionReadyState()",
 155                                  "no-peer-connection", tab_contents));
 156   }
 157
 158   base::FilePath CreateTemporaryWaveFile() {
 159     base::FilePath filename;
 160     EXPECT_TRUE(file_util::CreateTemporaryFile(&filename));
 161     base::FilePath wav_filename =
 162         filename.AddExtension(FILE_PATH_LITERAL(".wav"));
 163     EXPECT_TRUE(base::Move(filename, wav_filename));
 164     return wav_filename;
 165   }
 166
 167   PeerConnectionServerRunner peerconnection_server_;
 168 };
 169
 170 class AudioRecorder {
 171  public:
 172   AudioRecorder(): recording_application_(base::kNullProcessHandle) {}
 173   ~AudioRecorder() {}
 174
 175   // Starts the recording program for the specified duration. Returns true
 176   // on success.
 177   bool StartRecording(int duration_sec, const base::FilePath& output_file,
 178                       bool mono) {
 179     EXPECT_EQ(base::kNullProcessHandle, recording_application_)
 180         << "Tried to record, but is already recording.";
 181
 182     CommandLine command_line(CommandLine::NO_PROGRAM);
 183 #if defined(OS_WIN)
 184     // This disable is required to run SoundRecorder.exe on 64-bit Windows
 185     // from a 32-bit binary. We need to load the wow64 disable function from
 186     // the DLL since it doesn't exist on Windows XP.
 187     // TODO(phoglund): find some cleaner solution than using SoundRecorder.exe.
 188     base::ScopedNativeLibrary kernel32_lib(base::FilePath(L"kernel32"));
 189     if (kernel32_lib.is_valid()) {
 190       typedef BOOL (WINAPI* Wow64DisableWow64FSRedirection)(PVOID*);
 191       Wow64DisableWow64FSRedirection wow_64_disable_wow_64_fs_redirection;
 192       wow_64_disable_wow_64_fs_redirection =
 193           reinterpret_cast<Wow64DisableWow64FSRedirection>(
 194               kernel32_lib.GetFunctionPointer(
 195                   "Wow64DisableWow64FsRedirection"));
 196       if (wow_64_disable_wow_64_fs_redirection != NULL) {
 197         PVOID* ignored = NULL;
 198         wow_64_disable_wow_64_fs_redirection(ignored);
 199       }
 200     }
 201
 202     char duration_in_hms[128] = {0};
 203     struct tm duration_tm = {0};
 204     duration_tm.tm_sec = duration_sec;
 205     EXPECT_NE(0u, strftime(duration_in_hms, arraysize(duration_in_hms),
 206                            "%H:%M:%S", &duration_tm));
 207
 208     command_line.SetProgram(
 209         base::FilePath(FILE_PATH_LITERAL("SoundRecorder.exe")));
 210     command_line.AppendArg("/FILE");
 211     command_line.AppendArgPath(output_file);
 212     command_line.AppendArg("/DURATION");
 213     command_line.AppendArg(duration_in_hms);
 214 #else
 215     int num_channels = mono ? 1 : 2;
 216     command_line.SetProgram(base::FilePath("arecord"));
 217     command_line.AppendArg("-d");
 218     command_line.AppendArg(base::StringPrintf("%d", duration_sec));
 219     command_line.AppendArg("-f");
 220     command_line.AppendArg("dat");
 221     command_line.AppendArg("-c");
 222     command_line.AppendArg(base::StringPrintf("%d", num_channels));
 223     command_line.AppendArgPath(output_file);
 224 #endif
 225
 226     LOG(INFO) << "Running " << command_line.GetCommandLineString();
 227     return base::LaunchProcess(command_line, base::LaunchOptions(),
 228                                &recording_application_);
 229   }
 230
 231   // Joins the recording program. Returns true on success.
 232   bool WaitForRecordingToEnd() {
 233     int exit_code = -1;
 234     base::WaitForExitCode(recording_application_, &exit_code);
 235     return exit_code == 0;
 236   }
 237  private:
 238   base::ProcessHandle recording_application_;
 239 };
 240
 241 bool ForceMicrophoneVolumeTo100Percent() {
 242 #if defined(OS_WIN)
 243   CommandLine command_line(GetTestDataDir().Append(kToolsPath).Append(
 244       FILE_PATH_LITERAL("force_mic_volume_max.exe")));
 245   LOG(INFO) << "Running " << command_line.GetCommandLineString();
 246   std::string result;
 247   if (!base::GetAppOutput(command_line, &result)) {
 248     LOG(ERROR) << "Failed to set source volume: output was " << result;
 249     return false;
 250   }
 251 #else
 252   // Just force the volume of, say the first 5 devices. A machine will rarely
 253   // have more input sources than that. This is way easier than finding the
 254   // input device we happen to be using.
 255   for (int device_index = 0; device_index < 5; ++device_index) {
 256     std::string result;
 257     const std::string kHundredPercentVolume = "65536";
 258     CommandLine command_line(base::FilePath(FILE_PATH_LITERAL("pacmd")));
 259     command_line.AppendArg("set-source-volume");
 260     command_line.AppendArg(base::StringPrintf("%d", device_index));
 261     command_line.AppendArg(kHundredPercentVolume);
 262     LOG(INFO) << "Running " << command_line.GetCommandLineString();
 263     if (!base::GetAppOutput(command_line, &result)) {
 264       LOG(ERROR) << "Failed to set source volume: output was " << result;
 265       return false;
 266     }
 267   }
 268 #endif
 269   return true;
 270 }
 271
 272 // Removes silence from beginning and end of the |input_audio_file| and writes
 273 // the result to the |output_audio_file|. Returns true on success.
 274 bool RemoveSilence(const base::FilePath& input_file,
 275                    const base::FilePath& output_file) {
 276   // SOX documentation for silence command: http://sox.sourceforge.net/sox.html
 277   // To remove the silence from both beginning and end of the audio file, we
 278   // call sox silence command twice: once on normal file and again on its
 279   // reverse, then we reverse the final output.
 280   // Silence parameters are (in sequence):
 281   // ABOVE_PERIODS: The period for which silence occurs. Value 1 is used for
 282   //                 silence at beginning of audio.
 283   // DURATION: the amount of time in seconds that non-silence must be detected
 284   //           before sox stops trimming audio.
 285   // THRESHOLD: value used to indicate what sample value is treates as silence.
 286   const char* kAbovePeriods = "1";
 287   const char* kDuration = "2";
 288   const char* kTreshold = "5%";
 289
 290 #if defined(OS_WIN)
 291   CommandLine command_line(GetTestDataDir().Append(kToolsPath).Append(
 292       FILE_PATH_LITERAL("sox.exe")));
 293 #else
 294   CommandLine command_line(base::FilePath(FILE_PATH_LITERAL("sox")));
 295 #endif
 296   command_line.AppendArgPath(input_file);
 297   command_line.AppendArgPath(output_file);
 298   command_line.AppendArg("silence");
 299   command_line.AppendArg(kAbovePeriods);
 300   command_line.AppendArg(kDuration);
 301   command_line.AppendArg(kTreshold);
 302   command_line.AppendArg("reverse");
 303   command_line.AppendArg("silence");
 304   command_line.AppendArg(kAbovePeriods);
 305   command_line.AppendArg(kDuration);
 306   command_line.AppendArg(kTreshold);
 307   command_line.AppendArg("reverse");
 308
 309   LOG(INFO) << "Running " << command_line.GetCommandLineString();
 310   std::string result;
 311   bool ok = base::GetAppOutput(command_line, &result);
 312   LOG(INFO) << "Output was:\n\n" << result;
 313   return ok;
 314 }
 315
 316 bool CanParseAsFloat(const std::string& value) {
 317   return atof(value.c_str()) != 0 || value == "0";
 318 }
 319
 320 // Runs PESQ to compare |reference_file| to a |actual_file|. The |sample_rate|
 321 // can be either 16000 or 8000.
 322 //
 323 // PESQ is only mono-aware, so the files should preferably be recorded in mono.
 324 // Furthermore it expects the file to be 16 rather than 32 bits, even though
 325 // 32 bits might work. The audio bandwidth of the two files should be the same
 326 // e.g. don't compare a 32 kHz file to a 8 kHz file.
 327 //
 328 // The raw score in MOS is written to |raw_mos|, whereas the MOS-LQO score is
 329 // written to mos_lqo. The scores are returned as floats in string form (e.g.
 330 // "3.145", etc). Returns true on success.
 331 bool RunPesq(const base::FilePath& reference_file,
 332              const base::FilePath& actual_file,
 333              int sample_rate, std::string* raw_mos, std::string* mos_lqo) {
 334   // PESQ will break if the paths are too long (!).
 335   EXPECT_LT(reference_file.value().length(), 128u);
 336   EXPECT_LT(actual_file.value().length(), 128u);
 337
 338 #if defined(OS_WIN)
 339   base::FilePath pesq_path =
 340       GetTestDataDir().Append(kToolsPath).Append(FILE_PATH_LITERAL("pesq.exe"));
 341 #else
 342   base::FilePath pesq_path =
 343       GetTestDataDir().Append(kToolsPath).Append(FILE_PATH_LITERAL("pesq"));
 344 #endif
 345
 346   if (!base::PathExists(pesq_path)) {
 347     LOG(ERROR) << "Missing PESQ binary in " << pesq_path.value();
 348     return false;
 349   }
 350
 351   CommandLine command_line(pesq_path);
 352   command_line.AppendArg(base::StringPrintf("+%d", sample_rate));
 353   command_line.AppendArgPath(reference_file);
 354   command_line.AppendArgPath(actual_file);
 355
 356   LOG(INFO) << "Running " << command_line.GetCommandLineString();
 357   std::string result;
 358   if (!base::GetAppOutput(command_line, &result)) {
 359     LOG(ERROR) << "Failed to run PESQ.";
 360     return false;
 361   }
 362   LOG(INFO) << "Output was:\n\n" << result;
 363
 364   const std::string result_anchor = "Prediction (Raw MOS, MOS-LQO):  = ";
 365   std::size_t anchor_pos = result.find(result_anchor);
 366   if (anchor_pos == std::string::npos) {
 367     LOG(ERROR) << "PESQ was not able to compute a score; we probably recorded "
 368         << "only silence.";
 369     return false;
 370   }
 371
 372   // There are two tab-separated numbers on the format x.xxx, e.g. 5 chars each.
 373   std::size_t first_number_pos = anchor_pos + result_anchor.length();
 374   *raw_mos = result.substr(first_number_pos, 5);
 375   EXPECT_TRUE(CanParseAsFloat(*raw_mos)) << "Failed to parse raw MOS number.";
 376   *mos_lqo = result.substr(first_number_pos + 5 + 1, 5);
 377   EXPECT_TRUE(CanParseAsFloat(*mos_lqo)) << "Failed to parse MOS LQO number.";
 378
 379   return true;
 380 }
 381
 382 #if defined(OS_LINUX) || defined(OS_WIN)
 383 // Only implemented on Linux and Windows for now.
 384 #define MAYBE_MANUAL_TestAudioQuality MANUAL_TestAudioQuality
 385 #else
 386 #define MAYBE_MANUAL_TestAudioQuality DISABLED_MANUAL_TestAudioQuality
 387 #endif
 388
 389 IN_PROC_BROWSER_TEST_F(WebrtcAudioQualityBrowserTest,
 390                        MAYBE_MANUAL_TestAudioQuality) {
 391 #if defined(OS_WIN)
 392   if (base::win::GetVersion() < base::win::VERSION_VISTA) {
 393     // It would take work to implement this on XP; not prioritized right now.
 394     LOG(ERROR) << "This test is not implemented for Windows XP.";
 395     return;
 396   }
 397 #endif
 398   ASSERT_TRUE(HasAllRequiredResources());
 399   // TODO(phoglund): make this use embedded_test_server when that test server
 400   // can handle files > ~400Kb.
 401   ASSERT_TRUE(test_server()->Start());
 402   ASSERT_TRUE(peerconnection_server_.Start());
 403
 404   ASSERT_TRUE(ForceMicrophoneVolumeTo100Percent());
 405
 406   ui_test_utils::NavigateToURL(
 407       browser(), test_server()->GetURL(kMainWebrtcTestHtmlPage));
 408   content::WebContents* left_tab =
 409       browser()->tab_strip_model()->GetActiveWebContents();
 410
 411   chrome::AddBlankTabAt(browser(), -1, true);
 412   content::WebContents* right_tab =
 413       browser()->tab_strip_model()->GetActiveWebContents();
 414   ui_test_utils::NavigateToURL(
 415       browser(), test_server()->GetURL(kMainWebrtcTestHtmlPage));
 416
 417   ConnectToPeerConnectionServer("peer 1", left_tab);
 418   ConnectToPeerConnectionServer("peer 2", right_tab);
 419
 420   EXPECT_EQ("ok-peerconnection-created",
 421             ExecuteJavascript("preparePeerConnection()", left_tab));
 422
 423   AddAudioFile(kReferenceFileRelativeUrl, left_tab);
 424
 425   EstablishCall(left_tab, right_tab);
 426
 427   // Note: the media flow isn't necessarily established on the connection just
 428   // because the ready state is ok on both sides. We sleep a bit between call
 429   // establishment and playing to avoid cutting of the beginning of the audio
 430   // file.
 431   SleepInJavascript(left_tab, 2000);
 432
 433   base::FilePath recording = CreateTemporaryWaveFile();
 434
 435   // Note: the sound clip is about 10 seconds: record for 15 seconds to get some
 436   // safety margins on each side.
 437   AudioRecorder recorder;
 438   static int kRecordingTimeSeconds = 15;
 439   ASSERT_TRUE(recorder.StartRecording(kRecordingTimeSeconds, recording, true));
 440
 441   PlayAudioFile(left_tab);
 442
 443   ASSERT_TRUE(recorder.WaitForRecordingToEnd());
 444   LOG(INFO) << "Done recording to " << recording.value() << std::endl;
 445
 446   AssertNoAsynchronousErrors(left_tab);
 447   AssertNoAsynchronousErrors(right_tab);
 448
 449   HangUp(left_tab);
 450   WaitUntilHangupVerified(left_tab);
 451   WaitUntilHangupVerified(right_tab);
 452
 453   AssertNoAsynchronousErrors(left_tab);
 454   AssertNoAsynchronousErrors(right_tab);
 455
 456   base::FilePath trimmed_recording = CreateTemporaryWaveFile();
 457
 458   ASSERT_TRUE(RemoveSilence(recording, trimmed_recording));
 459   LOG(INFO) << "Trimmed silence: " << trimmed_recording.value() << std::endl;
 460
 461   std::string raw_mos;
 462   std::string mos_lqo;
 463   base::FilePath reference_file_in_test_dir =
 464       GetTestDataDir().Append(kReferenceFile);
 465   ASSERT_TRUE(RunPesq(reference_file_in_test_dir, trimmed_recording, 16000,
 466                       &raw_mos, &mos_lqo));
 467
 468   perf_test::PrintResult("audio_pesq", "", "raw_mos", raw_mos, "score", true);
 469   perf_test::PrintResult("audio_pesq", "", "mos_lqo", mos_lqo, "score", true);
 470
 471   EXPECT_TRUE(base::DeleteFile(recording, false));
 472   EXPECT_TRUE(base::DeleteFile(trimmed_recording, false));
 473
 474   ASSERT_TRUE(peerconnection_server_.Stop());
 475 }