src/content/renderer/media/webrtc_audio_device_impl.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/renderer/media/webrtc_audio_device_impl.h"
   6
   7 #include "base/bind.h"
   8 #include "base/metrics/histogram.h"
   9 #include "base/strings/string_util.h"
  10 #include "base/win/windows_version.h"
  11 #include "content/renderer/media/media_stream_audio_processor.h"
  12 #include "content/renderer/media/webrtc_audio_capturer.h"
  13 #include "content/renderer/media/webrtc_audio_renderer.h"
  14 #include "content/renderer/render_thread_impl.h"
  15 #include "media/audio/audio_parameters.h"
  16 #include "media/audio/sample_rates.h"
  17
  18 using media::AudioParameters;
  19 using media::ChannelLayout;
  20
  21 namespace content {
  22
  23 WebRtcAudioDeviceImpl::WebRtcAudioDeviceImpl()
  24     : ref_count_(0),
  25       audio_transport_callback_(NULL),
  26       input_delay_ms_(0),
  27       output_delay_ms_(0),
  28       initialized_(false),
  29       playing_(false),
  30       recording_(false),
  31       microphone_volume_(0),
  32       is_audio_track_processing_enabled_(
  33           MediaStreamAudioProcessor::IsAudioTrackProcessingEnabled()) {
  34   DVLOG(1) << "WebRtcAudioDeviceImpl::WebRtcAudioDeviceImpl()";
  35 }
  36
  37 WebRtcAudioDeviceImpl::~WebRtcAudioDeviceImpl() {
  38   DVLOG(1) << "WebRtcAudioDeviceImpl::~WebRtcAudioDeviceImpl()";
  39   DCHECK(thread_checker_.CalledOnValidThread());
  40   Terminate();
  41 }
  42
  43 int32_t WebRtcAudioDeviceImpl::AddRef() {
  44   DCHECK(thread_checker_.CalledOnValidThread());
  45   return base::subtle::Barrier_AtomicIncrement(&ref_count_, 1);
  46 }
  47
  48 int32_t WebRtcAudioDeviceImpl::Release() {
  49   DCHECK(thread_checker_.CalledOnValidThread());
  50   int ret = base::subtle::Barrier_AtomicIncrement(&ref_count_, -1);
  51   if (ret == 0) {
  52     delete this;
  53   }
  54   return ret;
  55 }
  56 int WebRtcAudioDeviceImpl::OnData(const int16* audio_data,
  57                                   int sample_rate,
  58                                   int number_of_channels,
  59                                   int number_of_frames,
  60                                   const std::vector<int>& channels,
  61                                   int audio_delay_milliseconds,
  62                                   int current_volume,
  63                                   bool need_audio_processing,
  64                                   bool key_pressed) {
  65   int total_delay_ms = 0;
  66   {
  67     base::AutoLock auto_lock(lock_);
  68     // Return immediately when not recording or |channels| is empty.
  69     // See crbug.com/274017: renderer crash dereferencing invalid channels[0].
  70     if (!recording_ || channels.empty())
  71       return 0;
  72
  73     // Store the reported audio delay locally.
  74     input_delay_ms_ = audio_delay_milliseconds;
  75     total_delay_ms = input_delay_ms_ + output_delay_ms_;
  76     DVLOG(2) << "total delay: " << input_delay_ms_ + output_delay_ms_;
  77   }
  78
  79   // Write audio frames in blocks of 10 milliseconds to the registered
  80   // webrtc::AudioTransport sink. Keep writing until our internal byte
  81   // buffer is empty.
  82   const int16* audio_buffer = audio_data;
  83   const int frames_per_10_ms = (sample_rate / 100);
  84   CHECK_EQ(number_of_frames % frames_per_10_ms, 0);
  85   int accumulated_audio_frames = 0;
  86   uint32_t new_volume = 0;
  87
  88   // The lock here is to protect a race in the resampler inside webrtc when
  89   // there are more than one input stream calling OnData(), which can happen
  90   // when the users setup two getUserMedia, one for the microphone, another
  91   // for WebAudio. Currently we don't have a better way to fix it except for
  92   // adding a lock here to sequence the call.
  93   // TODO(xians): Remove this workaround after we move the
  94   // webrtc::AudioProcessing module to Chrome. See http://crbug/264611 for
  95   // details.
  96   base::AutoLock auto_lock(capture_callback_lock_);
  97   while (accumulated_audio_frames < number_of_frames) {
  98     // Deliver 10ms of recorded 16-bit linear PCM audio.
  99     int new_mic_level = audio_transport_callback_->OnDataAvailable(
 100         &channels[0],
 101         channels.size(),
 102         audio_buffer,
 103         sample_rate,
 104         number_of_channels,
 105         frames_per_10_ms,
 106         total_delay_ms,
 107         current_volume,
 108         key_pressed,
 109         need_audio_processing);
 110
 111     accumulated_audio_frames += frames_per_10_ms;
 112     audio_buffer += frames_per_10_ms * number_of_channels;
 113
 114     // The latest non-zero new microphone level will be returned.
 115     if (new_mic_level)
 116       new_volume = new_mic_level;
 117   }
 118
 119   return new_volume;
 120 }
 121
 122 void WebRtcAudioDeviceImpl::OnSetFormat(
 123     const media::AudioParameters& params) {
 124   DVLOG(1) << "WebRtcAudioDeviceImpl::OnSetFormat()";
 125 }
 126
 127 void WebRtcAudioDeviceImpl::RenderData(media::AudioBus* audio_bus,
 128                                        int sample_rate,
 129                                        int audio_delay_milliseconds) {
 130   render_buffer_.resize(audio_bus->frames() * audio_bus->channels());
 131
 132   {
 133     base::AutoLock auto_lock(lock_);
 134     DCHECK(audio_transport_callback_);
 135     // Store the reported audio delay locally.
 136     output_delay_ms_ = audio_delay_milliseconds;
 137   }
 138
 139   int frames_per_10_ms = (sample_rate / 100);
 140   int bytes_per_sample = sizeof(render_buffer_[0]);
 141   const int bytes_per_10_ms =
 142       audio_bus->channels() * frames_per_10_ms * bytes_per_sample;
 143   DCHECK_EQ(audio_bus->frames() % frames_per_10_ms, 0);
 144
 145   // Get audio frames in blocks of 10 milliseconds from the registered
 146   // webrtc::AudioTransport source. Keep reading until our internal buffer
 147   // is full.
 148   uint32_t num_audio_frames = 0;
 149   int accumulated_audio_frames = 0;
 150   int16* audio_data = &render_buffer_[0];
 151   while (accumulated_audio_frames < audio_bus->frames()) {
 152     // Get 10ms and append output to temporary byte buffer.
 153     if (is_audio_track_processing_enabled_) {
 154       // When audio processing is enabled in the audio track, we use
 155       // PullRenderData() instead of NeedMorePlayData() to avoid passing the
 156       // render data to the APM in WebRTC as reference signal for echo
 157       // cancellation.
 158       static const int kBitsPerByte = 8;
 159       audio_transport_callback_->PullRenderData(bytes_per_sample * kBitsPerByte,
 160                                                 sample_rate,
 161                                                 audio_bus->channels(),
 162                                                 frames_per_10_ms,
 163                                                 audio_data);
 164       accumulated_audio_frames += frames_per_10_ms;
 165     } else {
 166       // TODO(xians): Remove the following code after the APM in WebRTC is
 167       // deprecated.
 168       audio_transport_callback_->NeedMorePlayData(frames_per_10_ms,
 169                                                   bytes_per_sample,
 170                                                   audio_bus->channels(),
 171                                                   sample_rate,
 172                                                   audio_data,
 173                                                   num_audio_frames);
 174       accumulated_audio_frames += num_audio_frames;
 175     }
 176
 177     audio_data += bytes_per_10_ms;
 178   }
 179
 180   // De-interleave each channel and convert to 32-bit floating-point
 181   // with nominal range -1.0 -> +1.0 to match the callback format.
 182   audio_bus->FromInterleaved(&render_buffer_[0],
 183                              audio_bus->frames(),
 184                              bytes_per_sample);
 185
 186   // Pass the render data to the playout sinks.
 187   base::AutoLock auto_lock(lock_);
 188   for (PlayoutDataSinkList::const_iterator it = playout_sinks_.begin();
 189        it != playout_sinks_.end(); ++it) {
 190     (*it)->OnPlayoutData(audio_bus, sample_rate, audio_delay_milliseconds);
 191   }
 192 }
 193
 194 void WebRtcAudioDeviceImpl::RemoveAudioRenderer(WebRtcAudioRenderer* renderer) {
 195   DCHECK(thread_checker_.CalledOnValidThread());
 196   DCHECK_EQ(renderer, renderer_);
 197   base::AutoLock auto_lock(lock_);
 198   // Notify the playout sink of the change.
 199   for (PlayoutDataSinkList::const_iterator it = playout_sinks_.begin();
 200        it != playout_sinks_.end(); ++it) {
 201     (*it)->OnPlayoutDataSourceChanged();
 202   }
 203
 204   renderer_ = NULL;
 205   playing_ = false;
 206 }
 207
 208 int32_t WebRtcAudioDeviceImpl::RegisterAudioCallback(
 209     webrtc::AudioTransport* audio_callback) {
 210   DVLOG(1) << "WebRtcAudioDeviceImpl::RegisterAudioCallback()";
 211   DCHECK(thread_checker_.CalledOnValidThread());
 212   DCHECK_EQ(audio_transport_callback_ == NULL, audio_callback != NULL);
 213   audio_transport_callback_ = audio_callback;
 214   return 0;
 215 }
 216
 217 int32_t WebRtcAudioDeviceImpl::Init() {
 218   DVLOG(1) << "WebRtcAudioDeviceImpl::Init()";
 219   DCHECK(thread_checker_.CalledOnValidThread());
 220
 221   // We need to return a success to continue the initialization of WebRtc VoE
 222   // because failure on the capturer_ initialization should not prevent WebRTC
 223   // from working. See issue http://crbug.com/144421 for details.
 224   initialized_ = true;
 225
 226   return 0;
 227 }
 228
 229 int32_t WebRtcAudioDeviceImpl::Terminate() {
 230   DVLOG(1) << "WebRtcAudioDeviceImpl::Terminate()";
 231   DCHECK(thread_checker_.CalledOnValidThread());
 232
 233   // Calling Terminate() multiple times in a row is OK.
 234   if (!initialized_)
 235     return 0;
 236
 237   StopRecording();
 238   StopPlayout();
 239
 240   DCHECK(!renderer_.get() || !renderer_->IsStarted())
 241       << "The shared audio renderer shouldn't be running";
 242
 243   DisableAecDump();
 244
 245   capturers_.clear();
 246
 247   initialized_ = false;
 248   return 0;
 249 }
 250
 251 bool WebRtcAudioDeviceImpl::Initialized() const {
 252   return initialized_;
 253 }
 254
 255 int32_t WebRtcAudioDeviceImpl::PlayoutIsAvailable(bool* available) {
 256   *available = initialized_;
 257   return 0;
 258 }
 259
 260 bool WebRtcAudioDeviceImpl::PlayoutIsInitialized() const {
 261   return initialized_;
 262 }
 263
 264 int32_t WebRtcAudioDeviceImpl::RecordingIsAvailable(bool* available) {
 265   *available = (!capturers_.empty());
 266   return 0;
 267 }
 268
 269 bool WebRtcAudioDeviceImpl::RecordingIsInitialized() const {
 270   DVLOG(1) << "WebRtcAudioDeviceImpl::RecordingIsInitialized()";
 271   DCHECK(thread_checker_.CalledOnValidThread());
 272   return (!capturers_.empty());
 273 }
 274
 275 int32_t WebRtcAudioDeviceImpl::StartPlayout() {
 276   DVLOG(1) << "WebRtcAudioDeviceImpl::StartPlayout()";
 277   LOG_IF(ERROR, !audio_transport_callback_) << "Audio transport is missing";
 278   {
 279     base::AutoLock auto_lock(lock_);
 280     if (!audio_transport_callback_)
 281       return 0;
 282   }
 283
 284   if (playing_) {
 285     // webrtc::VoiceEngine assumes that it is OK to call Start() twice and
 286     // that the call is ignored the second time.
 287     return 0;
 288   }
 289
 290   playing_ = true;
 291   return 0;
 292 }
 293
 294 int32_t WebRtcAudioDeviceImpl::StopPlayout() {
 295   DVLOG(1) << "WebRtcAudioDeviceImpl::StopPlayout()";
 296   if (!playing_) {
 297     // webrtc::VoiceEngine assumes that it is OK to call Stop() just in case.
 298     return 0;
 299   }
 300
 301   playing_ = false;
 302   return 0;
 303 }
 304
 305 bool WebRtcAudioDeviceImpl::Playing() const {
 306   return playing_;
 307 }
 308
 309 int32_t WebRtcAudioDeviceImpl::StartRecording() {
 310   DVLOG(1) << "WebRtcAudioDeviceImpl::StartRecording()";
 311   DCHECK(initialized_);
 312   LOG_IF(ERROR, !audio_transport_callback_) << "Audio transport is missing";
 313   if (!audio_transport_callback_) {
 314     return -1;
 315   }
 316
 317   {
 318     base::AutoLock auto_lock(lock_);
 319     if (recording_)
 320       return 0;
 321
 322     recording_ = true;
 323   }
 324
 325   return 0;
 326 }
 327
 328 int32_t WebRtcAudioDeviceImpl::StopRecording() {
 329   DVLOG(1) << "WebRtcAudioDeviceImpl::StopRecording()";
 330   {
 331     base::AutoLock auto_lock(lock_);
 332     if (!recording_)
 333       return 0;
 334
 335     recording_ = false;
 336   }
 337
 338   return 0;
 339 }
 340
 341 bool WebRtcAudioDeviceImpl::Recording() const {
 342   base::AutoLock auto_lock(lock_);
 343   return recording_;
 344 }
 345
 346 int32_t WebRtcAudioDeviceImpl::SetMicrophoneVolume(uint32_t volume) {
 347   DVLOG(1) << "WebRtcAudioDeviceImpl::SetMicrophoneVolume(" << volume << ")";
 348   DCHECK(initialized_);
 349
 350   // Only one microphone is supported at the moment, which is represented by
 351   // the default capturer.
 352   scoped_refptr<WebRtcAudioCapturer> capturer(GetDefaultCapturer());
 353   if (!capturer.get())
 354     return -1;
 355
 356   capturer->SetVolume(volume);
 357   return 0;
 358 }
 359
 360 // TODO(henrika): sort out calling thread once we start using this API.
 361 int32_t WebRtcAudioDeviceImpl::MicrophoneVolume(uint32_t* volume) const {
 362   DVLOG(1) << "WebRtcAudioDeviceImpl::MicrophoneVolume()";
 363   // We only support one microphone now, which is accessed via the default
 364   // capturer.
 365   DCHECK(initialized_);
 366   scoped_refptr<WebRtcAudioCapturer> capturer(GetDefaultCapturer());
 367   if (!capturer.get())
 368     return -1;
 369
 370   *volume = static_cast<uint32_t>(capturer->Volume());
 371
 372   return 0;
 373 }
 374
 375 int32_t WebRtcAudioDeviceImpl::MaxMicrophoneVolume(uint32_t* max_volume) const {
 376   DCHECK(initialized_);
 377   *max_volume = kMaxVolumeLevel;
 378   return 0;
 379 }
 380
 381 int32_t WebRtcAudioDeviceImpl::MinMicrophoneVolume(uint32_t* min_volume) const {
 382   *min_volume = 0;
 383   return 0;
 384 }
 385
 386 int32_t WebRtcAudioDeviceImpl::StereoPlayoutIsAvailable(bool* available) const {
 387   DCHECK(initialized_);
 388   *available = renderer_ && renderer_->channels() == 2;
 389   return 0;
 390 }
 391
 392 int32_t WebRtcAudioDeviceImpl::StereoRecordingIsAvailable(
 393     bool* available) const {
 394   DCHECK(initialized_);
 395   // TODO(xians): These kind of hardware methods do not make much sense since we
 396   // support multiple sources. Remove or figure out new APIs for such methods.
 397   scoped_refptr<WebRtcAudioCapturer> capturer(GetDefaultCapturer());
 398   if (!capturer.get())
 399     return -1;
 400
 401   *available = (capturer->source_audio_parameters().channels() == 2);
 402   return 0;
 403 }
 404
 405 int32_t WebRtcAudioDeviceImpl::PlayoutDelay(uint16_t* delay_ms) const {
 406   base::AutoLock auto_lock(lock_);
 407   *delay_ms = static_cast<uint16_t>(output_delay_ms_);
 408   return 0;
 409 }
 410
 411 int32_t WebRtcAudioDeviceImpl::RecordingDelay(uint16_t* delay_ms) const {
 412   base::AutoLock auto_lock(lock_);
 413   *delay_ms = static_cast<uint16_t>(input_delay_ms_);
 414   return 0;
 415 }
 416
 417 int32_t WebRtcAudioDeviceImpl::RecordingSampleRate(
 418     uint32_t* sample_rate) const {
 419   // We use the default capturer as the recording sample rate.
 420   scoped_refptr<WebRtcAudioCapturer> capturer(GetDefaultCapturer());
 421   if (!capturer.get())
 422     return -1;
 423
 424   *sample_rate = static_cast<uint32_t>(
 425       capturer->source_audio_parameters().sample_rate());
 426   return 0;
 427 }
 428
 429 int32_t WebRtcAudioDeviceImpl::PlayoutSampleRate(
 430     uint32_t* sample_rate) const {
 431   *sample_rate = renderer_ ? renderer_->sample_rate() : 0;
 432   return 0;
 433 }
 434
 435 bool WebRtcAudioDeviceImpl::SetAudioRenderer(WebRtcAudioRenderer* renderer) {
 436   DCHECK(thread_checker_.CalledOnValidThread());
 437   DCHECK(renderer);
 438
 439   base::AutoLock auto_lock(lock_);
 440   if (renderer_.get())
 441     return false;
 442
 443   if (!renderer->Initialize(this))
 444     return false;
 445
 446   renderer_ = renderer;
 447   return true;
 448 }
 449
 450 void WebRtcAudioDeviceImpl::AddAudioCapturer(
 451     const scoped_refptr<WebRtcAudioCapturer>& capturer) {
 452   DVLOG(1) << "WebRtcAudioDeviceImpl::AddAudioCapturer()";
 453   DCHECK(thread_checker_.CalledOnValidThread());
 454   DCHECK(capturer.get());
 455   DCHECK(!capturer->device_id().empty());
 456   {
 457     base::AutoLock auto_lock(lock_);
 458     DCHECK(std::find(capturers_.begin(), capturers_.end(), capturer) ==
 459         capturers_.end());
 460     capturers_.push_back(capturer);
 461   }
 462
 463   // Start the Aec dump if the Aec dump has been enabled and has not been
 464   // started.
 465   if (aec_dump_file_.IsValid())
 466     MaybeStartAecDump();
 467 }
 468
 469 void WebRtcAudioDeviceImpl::RemoveAudioCapturer(
 470     const scoped_refptr<WebRtcAudioCapturer>& capturer) {
 471   DVLOG(1) << "WebRtcAudioDeviceImpl::AddAudioCapturer()";
 472   DCHECK(thread_checker_.CalledOnValidThread());
 473   DCHECK(capturer.get());
 474   base::AutoLock auto_lock(lock_);
 475   capturers_.remove(capturer);
 476 }
 477
 478 scoped_refptr<WebRtcAudioCapturer>
 479 WebRtcAudioDeviceImpl::GetDefaultCapturer() const {
 480   base::AutoLock auto_lock(lock_);
 481   // Use the last |capturer| which is from the latest getUserMedia call as
 482   // the default capture device.
 483   return capturers_.empty() ? NULL : capturers_.back();
 484 }
 485
 486 void WebRtcAudioDeviceImpl::AddPlayoutSink(
 487     WebRtcPlayoutDataSource::Sink* sink) {
 488   DCHECK(thread_checker_.CalledOnValidThread());
 489   DCHECK(sink);
 490   base::AutoLock auto_lock(lock_);
 491   DCHECK(std::find(playout_sinks_.begin(), playout_sinks_.end(), sink) ==
 492       playout_sinks_.end());
 493   playout_sinks_.push_back(sink);
 494 }
 495
 496 void WebRtcAudioDeviceImpl::RemovePlayoutSink(
 497     WebRtcPlayoutDataSource::Sink* sink) {
 498   DCHECK(thread_checker_.CalledOnValidThread());
 499   DCHECK(sink);
 500   base::AutoLock auto_lock(lock_);
 501   playout_sinks_.remove(sink);
 502 }
 503
 504 bool WebRtcAudioDeviceImpl::GetAuthorizedDeviceInfoForAudioRenderer(
 505     int* session_id,
 506     int* output_sample_rate,
 507     int* output_frames_per_buffer) {
 508   DCHECK(thread_checker_.CalledOnValidThread());
 509   // If there is no capturer or there are more than one open capture devices,
 510   // return false.
 511   if (capturers_.empty() || capturers_.size() > 1)
 512     return false;
 513
 514   return GetDefaultCapturer()->GetPairedOutputParameters(
 515       session_id, output_sample_rate, output_frames_per_buffer);
 516 }
 517
 518 void WebRtcAudioDeviceImpl::EnableAecDump(base::File aec_dump_file) {
 519   DCHECK(thread_checker_.CalledOnValidThread());
 520   DCHECK(aec_dump_file.IsValid());
 521   DCHECK(!aec_dump_file_.IsValid());
 522   aec_dump_file_ = aec_dump_file.Pass();
 523   MaybeStartAecDump();
 524 }
 525
 526 void WebRtcAudioDeviceImpl::DisableAecDump() {
 527   DCHECK(thread_checker_.CalledOnValidThread());
 528   // Simply invalidate the |aec_dump_file_| if we have not pass the ownership
 529   // to WebRtc.
 530   if (aec_dump_file_.IsValid()) {
 531     aec_dump_file_.Close();
 532     return;
 533   }
 534
 535   // We might have call StartAecDump() on one of the capturer. Loop
 536   // through all the capturers and call StopAecDump() on each of them.
 537   for (CapturerList::const_iterator iter = capturers_.begin();
 538        iter != capturers_.end(); ++iter) {
 539     (*iter)->StopAecDump();
 540   }
 541 }
 542
 543 void WebRtcAudioDeviceImpl::MaybeStartAecDump() {
 544   DCHECK(thread_checker_.CalledOnValidThread());
 545   DCHECK(aec_dump_file_.IsValid());
 546
 547   // Start the Aec dump on the current default capturer.
 548   scoped_refptr<WebRtcAudioCapturer> default_capturer(GetDefaultCapturer());
 549   if (!default_capturer)
 550     return;
 551
 552   default_capturer->StartAecDump(aec_dump_file_.Pass());
 553 }
 554
 555 }  // namespace content