src/content/renderer/media/webrtc_audio_renderer.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/renderer/media/webrtc_audio_renderer.h"
   6
   7 #include "base/logging.h"
   8 #include "base/metrics/histogram.h"
   9 #include "base/strings/string_util.h"
  10 #include "base/strings/stringprintf.h"
  11 #include "content/renderer/media/audio_device_factory.h"
  12 #include "content/renderer/media/media_stream_dispatcher.h"
  13 #include "content/renderer/media/webrtc_audio_device_impl.h"
  14 #include "content/renderer/media/webrtc_logging.h"
  15 #include "content/renderer/render_frame_impl.h"
  16 #include "media/audio/audio_output_device.h"
  17 #include "media/audio/audio_parameters.h"
  18 #include "media/audio/sample_rates.h"
  19 #include "third_party/libjingle/source/talk/app/webrtc/mediastreaminterface.h"
  20 #include "third_party/libjingle/source/talk/media/base/audiorenderer.h"
  21
  22
  23 #if defined(OS_WIN)
  24 #include "base/win/windows_version.h"
  25 #include "media/audio/win/core_audio_util_win.h"
  26 #endif
  27
  28 namespace content {
  29
  30 namespace {
  31
  32 // We add a UMA histogram measuring the execution time of the Render() method
  33 // every |kNumCallbacksBetweenRenderTimeHistograms| callback. Assuming 10ms
  34 // between each callback leads to one UMA update each 100ms.
  35 const int kNumCallbacksBetweenRenderTimeHistograms = 10;
  36
  37 // This is a simple wrapper class that's handed out to users of a shared
  38 // WebRtcAudioRenderer instance.  This class maintains the per-user 'playing'
  39 // and 'started' states to avoid problems related to incorrect usage which
  40 // might violate the implementation assumptions inside WebRtcAudioRenderer
  41 // (see the play reference count).
  42 class SharedAudioRenderer : public MediaStreamAudioRenderer {
  43  public:
  44   // Callback definition for a callback that is called when when Play(), Pause()
  45   // or SetVolume are called (whenever the internal |playing_state_| changes).
  46   typedef base::Callback<
  47       void(const scoped_refptr<webrtc::MediaStreamInterface>&,
  48            WebRtcAudioRenderer::PlayingState*)> OnPlayStateChanged;
  49
  50   SharedAudioRenderer(
  51       const scoped_refptr<MediaStreamAudioRenderer>& delegate,
  52       const scoped_refptr<webrtc::MediaStreamInterface>& media_stream,
  53       const OnPlayStateChanged& on_play_state_changed)
  54       : delegate_(delegate), media_stream_(media_stream), started_(false),
  55         on_play_state_changed_(on_play_state_changed) {
  56     DCHECK(!on_play_state_changed_.is_null());
  57     DCHECK(media_stream_.get());
  58   }
  59
  60  protected:
  61   virtual ~SharedAudioRenderer() {
  62     DCHECK(thread_checker_.CalledOnValidThread());
  63     DVLOG(1) << __FUNCTION__;
  64     Stop();
  65   }
  66
  67   virtual void Start() OVERRIDE {
  68     DCHECK(thread_checker_.CalledOnValidThread());
  69     if (started_)
  70       return;
  71     started_ = true;
  72     delegate_->Start();
  73   }
  74
  75   virtual void Play() OVERRIDE {
  76     DCHECK(thread_checker_.CalledOnValidThread());
  77     DCHECK(started_);
  78     if (playing_state_.playing())
  79       return;
  80     playing_state_.set_playing(true);
  81     on_play_state_changed_.Run(media_stream_, &playing_state_);
  82   }
  83
  84   virtual void Pause() OVERRIDE {
  85     DCHECK(thread_checker_.CalledOnValidThread());
  86     DCHECK(started_);
  87     if (!playing_state_.playing())
  88       return;
  89     playing_state_.set_playing(false);
  90     on_play_state_changed_.Run(media_stream_, &playing_state_);
  91   }
  92
  93   virtual void Stop() OVERRIDE {
  94     DCHECK(thread_checker_.CalledOnValidThread());
  95     if (!started_)
  96       return;
  97     Pause();
  98     started_ = false;
  99     delegate_->Stop();
 100   }
 101
 102   virtual void SetVolume(float volume) OVERRIDE {
 103     DCHECK(thread_checker_.CalledOnValidThread());
 104     DCHECK(volume >= 0.0f && volume <= 1.0f);
 105     playing_state_.set_volume(volume);
 106     on_play_state_changed_.Run(media_stream_, &playing_state_);
 107   }
 108
 109   virtual base::TimeDelta GetCurrentRenderTime() const OVERRIDE {
 110     DCHECK(thread_checker_.CalledOnValidThread());
 111     return delegate_->GetCurrentRenderTime();
 112   }
 113
 114   virtual bool IsLocalRenderer() const OVERRIDE {
 115     DCHECK(thread_checker_.CalledOnValidThread());
 116     return delegate_->IsLocalRenderer();
 117   }
 118
 119  private:
 120   base::ThreadChecker thread_checker_;
 121   const scoped_refptr<MediaStreamAudioRenderer> delegate_;
 122   const scoped_refptr<webrtc::MediaStreamInterface> media_stream_;
 123   bool started_;
 124   WebRtcAudioRenderer::PlayingState playing_state_;
 125   OnPlayStateChanged on_play_state_changed_;
 126 };
 127
 128 // Returns either AudioParameters::NO_EFFECTS or AudioParameters::DUCKING
 129 // depending on whether or not an input element is currently open with
 130 // ducking enabled.
 131 int GetCurrentDuckingFlag(int render_frame_id) {
 132   RenderFrameImpl* const frame =
 133       RenderFrameImpl::FromRoutingID(render_frame_id);
 134   MediaStreamDispatcher* const dispatcher = frame ?
 135       frame->GetMediaStreamDispatcher() : NULL;
 136   if (dispatcher && dispatcher->IsAudioDuckingActive()) {
 137     return media::AudioParameters::DUCKING;
 138   }
 139
 140   return media::AudioParameters::NO_EFFECTS;
 141 }
 142
 143 // Helper method to get platform specific optimal buffer size.
 144 int GetOptimalBufferSize(int sample_rate, int hardware_buffer_size) {
 145   // Use native hardware buffer size as default. On Windows, we strive to open
 146   // up using this native hardware buffer size to achieve best
 147   // possible performance and to ensure that no FIFO is needed on the browser
 148   // side to match the client request. That is why there is no #if case for
 149   // Windows below.
 150   int frames_per_buffer = hardware_buffer_size;
 151
 152 #if defined(OS_LINUX) || defined(OS_MACOSX)
 153   // On Linux and MacOS, the low level IO implementations on the browser side
 154   // supports all buffer size the clients want. We use the native peer
 155   // connection buffer size (10ms) to achieve best possible performance.
 156   frames_per_buffer = sample_rate / 100;
 157 #elif defined(OS_ANDROID)
 158   // TODO(henrika): Keep tuning this scheme and espcicially for low-latency
 159   // cases. Might not be possible to come up with the perfect solution using
 160   // the render side only.
 161   int frames_per_10ms = sample_rate / 100;
 162   if (frames_per_buffer < 2 * frames_per_10ms) {
 163     // Examples of low-latency frame sizes and the resulting |buffer_size|:
 164     //  Nexus 7     : 240 audio frames => 2*480 = 960
 165     //  Nexus 10    : 256              => 2*441 = 882
 166     //  Galaxy Nexus: 144              => 2*441 = 882
 167     frames_per_buffer = 2 * frames_per_10ms;
 168     DVLOG(1) << "Low-latency output detected on Android";
 169   }
 170 #endif
 171
 172   DVLOG(1) << "Using sink output buffer size: " << frames_per_buffer;
 173   return frames_per_buffer;
 174 }
 175
 176 }  // namespace
 177
 178 WebRtcAudioRenderer::WebRtcAudioRenderer(
 179     const scoped_refptr<webrtc::MediaStreamInterface>& media_stream,
 180     int source_render_view_id,
 181     int source_render_frame_id,
 182     int session_id,
 183     int sample_rate,
 184     int frames_per_buffer)
 185     : state_(UNINITIALIZED),
 186       source_render_view_id_(source_render_view_id),
 187       source_render_frame_id_(source_render_frame_id),
 188       session_id_(session_id),
 189       media_stream_(media_stream),
 190       source_(NULL),
 191       play_ref_count_(0),
 192       start_ref_count_(0),
 193       audio_delay_milliseconds_(0),
 194       fifo_delay_milliseconds_(0),
 195       sink_params_(media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
 196                    media::CHANNEL_LAYOUT_STEREO, sample_rate, 16,
 197                    frames_per_buffer,
 198                    GetCurrentDuckingFlag(source_render_frame_id)),
 199       render_callback_count_(0) {
 200   WebRtcLogMessage(base::StringPrintf(
 201       "WAR::WAR. source_render_view_id=%d"
 202       ", session_id=%d, sample_rate=%d, frames_per_buffer=%d, effects=%i",
 203       source_render_view_id,
 204       session_id,
 205       sample_rate,
 206       frames_per_buffer,
 207       sink_params_.effects()));
 208 }
 209
 210 WebRtcAudioRenderer::~WebRtcAudioRenderer() {
 211   DCHECK(thread_checker_.CalledOnValidThread());
 212   DCHECK_EQ(state_, UNINITIALIZED);
 213 }
 214
 215 bool WebRtcAudioRenderer::Initialize(WebRtcAudioRendererSource* source) {
 216   DVLOG(1) << "WebRtcAudioRenderer::Initialize()";
 217   DCHECK(thread_checker_.CalledOnValidThread());
 218   base::AutoLock auto_lock(lock_);
 219   DCHECK_EQ(state_, UNINITIALIZED);
 220   DCHECK(source);
 221   DCHECK(!sink_.get());
 222   DCHECK(!source_);
 223
 224   // WebRTC does not yet support higher rates than 96000 on the client side
 225   // and 48000 is the preferred sample rate. Therefore, if 192000 is detected,
 226   // we change the rate to 48000 instead. The consequence is that the native
 227   // layer will be opened up at 192kHz but WebRTC will provide data at 48kHz
 228   // which will then be resampled by the audio converted on the browser side
 229   // to match the native audio layer.
 230   int sample_rate = sink_params_.sample_rate();
 231   DVLOG(1) << "Audio output hardware sample rate: " << sample_rate;
 232   if (sample_rate == 192000) {
 233     DVLOG(1) << "Resampling from 48000 to 192000 is required";
 234     sample_rate = 48000;
 235   }
 236   media::AudioSampleRate asr;
 237   if (media::ToAudioSampleRate(sample_rate, &asr)) {
 238     UMA_HISTOGRAM_ENUMERATION(
 239         "WebRTC.AudioOutputSampleRate", asr, media::kAudioSampleRateMax + 1);
 240   } else {
 241     UMA_HISTOGRAM_COUNTS("WebRTC.AudioOutputSampleRateUnexpected",
 242                          sample_rate);
 243   }
 244
 245   // Set up audio parameters for the source, i.e., the WebRTC client.
 246
 247   // The WebRTC client only supports multiples of 10ms as buffer size where
 248   // 10ms is preferred for lowest possible delay.
 249   media::AudioParameters source_params;
 250   const int frames_per_10ms = (sample_rate / 100);
 251   DVLOG(1) << "Using WebRTC output buffer size: " << frames_per_10ms;
 252
 253   source_params.Reset(media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
 254                       sink_params_.channel_layout(), sink_params_.channels(),
 255                       sample_rate, 16, frames_per_10ms);
 256
 257   const int frames_per_buffer =
 258       GetOptimalBufferSize(sample_rate, sink_params_.frames_per_buffer());
 259
 260   sink_params_.Reset(sink_params_.format(), sink_params_.channel_layout(),
 261                      sink_params_.channels(), sample_rate, 16,
 262                      frames_per_buffer);
 263
 264   // Create a FIFO if re-buffering is required to match the source input with
 265   // the sink request. The source acts as provider here and the sink as
 266   // consumer.
 267   fifo_delay_milliseconds_ = 0;
 268   if (source_params.frames_per_buffer() != sink_params_.frames_per_buffer()) {
 269     DVLOG(1) << "Rebuffering from " << source_params.frames_per_buffer()
 270              << " to " << sink_params_.frames_per_buffer();
 271     audio_fifo_.reset(new media::AudioPullFifo(
 272         source_params.channels(),
 273         source_params.frames_per_buffer(),
 274         base::Bind(
 275             &WebRtcAudioRenderer::SourceCallback,
 276             base::Unretained(this))));
 277
 278     if (sink_params_.frames_per_buffer() > source_params.frames_per_buffer()) {
 279       int frame_duration_milliseconds = base::Time::kMillisecondsPerSecond /
 280           static_cast<double>(source_params.sample_rate());
 281       fifo_delay_milliseconds_ = (sink_params_.frames_per_buffer() -
 282         source_params.frames_per_buffer()) * frame_duration_milliseconds;
 283     }
 284   }
 285
 286   source_ = source;
 287
 288   // Configure the audio rendering client and start rendering.
 289   sink_ = AudioDeviceFactory::NewOutputDevice(
 290       source_render_view_id_, source_render_frame_id_);
 291
 292   DCHECK_GE(session_id_, 0);
 293   sink_->InitializeWithSessionId(sink_params_, this, session_id_);
 294
 295   sink_->Start();
 296
 297   // User must call Play() before any audio can be heard.
 298   state_ = PAUSED;
 299
 300   return true;
 301 }
 302
 303 scoped_refptr<MediaStreamAudioRenderer>
 304 WebRtcAudioRenderer::CreateSharedAudioRendererProxy(
 305     const scoped_refptr<webrtc::MediaStreamInterface>& media_stream) {
 306   content::SharedAudioRenderer::OnPlayStateChanged on_play_state_changed =
 307       base::Bind(&WebRtcAudioRenderer::OnPlayStateChanged, this);
 308   return new SharedAudioRenderer(this, media_stream, on_play_state_changed);
 309 }
 310
 311 bool WebRtcAudioRenderer::IsStarted() const {
 312   DCHECK(thread_checker_.CalledOnValidThread());
 313   return start_ref_count_ != 0;
 314 }
 315
 316 void WebRtcAudioRenderer::Start() {
 317   DVLOG(1) << "WebRtcAudioRenderer::Start()";
 318   DCHECK(thread_checker_.CalledOnValidThread());
 319   ++start_ref_count_;
 320 }
 321
 322 void WebRtcAudioRenderer::Play() {
 323   DVLOG(1) << "WebRtcAudioRenderer::Play()";
 324   DCHECK(thread_checker_.CalledOnValidThread());
 325
 326   if (playing_state_.playing())
 327     return;
 328
 329   playing_state_.set_playing(true);
 330   render_callback_count_ = 0;
 331
 332   OnPlayStateChanged(media_stream_, &playing_state_);
 333 }
 334
 335 void WebRtcAudioRenderer::EnterPlayState() {
 336   DVLOG(1) << "WebRtcAudioRenderer::EnterPlayState()";
 337   DCHECK(thread_checker_.CalledOnValidThread());
 338   DCHECK_GT(start_ref_count_, 0) << "Did you forget to call Start()?";
 339   base::AutoLock auto_lock(lock_);
 340   if (state_ == UNINITIALIZED)
 341     return;
 342
 343   DCHECK(play_ref_count_ == 0 || state_ == PLAYING);
 344   ++play_ref_count_;
 345
 346   if (state_ != PLAYING) {
 347     state_ = PLAYING;
 348
 349     if (audio_fifo_) {
 350       audio_delay_milliseconds_ = 0;
 351       audio_fifo_->Clear();
 352     }
 353   }
 354 }
 355
 356 void WebRtcAudioRenderer::Pause() {
 357   DVLOG(1) << "WebRtcAudioRenderer::Pause()";
 358   DCHECK(thread_checker_.CalledOnValidThread());
 359   if (!playing_state_.playing())
 360     return;
 361
 362   playing_state_.set_playing(false);
 363
 364   OnPlayStateChanged(media_stream_, &playing_state_);
 365 }
 366
 367 void WebRtcAudioRenderer::EnterPauseState() {
 368   DVLOG(1) << "WebRtcAudioRenderer::EnterPauseState()";
 369   DCHECK(thread_checker_.CalledOnValidThread());
 370   DCHECK_GT(start_ref_count_, 0) << "Did you forget to call Start()?";
 371   base::AutoLock auto_lock(lock_);
 372   if (state_ == UNINITIALIZED)
 373     return;
 374
 375   DCHECK_EQ(state_, PLAYING);
 376   DCHECK_GT(play_ref_count_, 0);
 377   if (!--play_ref_count_)
 378     state_ = PAUSED;
 379 }
 380
 381 void WebRtcAudioRenderer::Stop() {
 382   DVLOG(1) << "WebRtcAudioRenderer::Stop()";
 383   DCHECK(thread_checker_.CalledOnValidThread());
 384   {
 385     base::AutoLock auto_lock(lock_);
 386     if (state_ == UNINITIALIZED)
 387       return;
 388
 389     if (--start_ref_count_)
 390       return;
 391
 392     DVLOG(1) << "Calling RemoveAudioRenderer and Stop().";
 393
 394     source_->RemoveAudioRenderer(this);
 395     source_ = NULL;
 396     state_ = UNINITIALIZED;
 397   }
 398
 399   // Make sure to stop the sink while _not_ holding the lock since the Render()
 400   // callback may currently be executing and try to grab the lock while we're
 401   // stopping the thread on which it runs.
 402   sink_->Stop();
 403 }
 404
 405 void WebRtcAudioRenderer::SetVolume(float volume) {
 406   DCHECK(thread_checker_.CalledOnValidThread());
 407   DCHECK(volume >= 0.0f && volume <= 1.0f);
 408
 409   playing_state_.set_volume(volume);
 410   OnPlayStateChanged(media_stream_, &playing_state_);
 411 }
 412
 413 base::TimeDelta WebRtcAudioRenderer::GetCurrentRenderTime() const {
 414   DCHECK(thread_checker_.CalledOnValidThread());
 415   base::AutoLock auto_lock(lock_);
 416   return current_time_;
 417 }
 418
 419 bool WebRtcAudioRenderer::IsLocalRenderer() const {
 420   return false;
 421 }
 422
 423 int WebRtcAudioRenderer::Render(media::AudioBus* audio_bus,
 424                                 int audio_delay_milliseconds) {
 425   base::AutoLock auto_lock(lock_);
 426   if (!source_)
 427     return 0;
 428
 429   DVLOG(2) << "WebRtcAudioRenderer::Render()";
 430   DVLOG(2) << "audio_delay_milliseconds: " << audio_delay_milliseconds;
 431
 432   audio_delay_milliseconds_ = audio_delay_milliseconds;
 433
 434   if (audio_fifo_)
 435     audio_fifo_->Consume(audio_bus, audio_bus->frames());
 436   else
 437     SourceCallback(0, audio_bus);
 438
 439   return (state_ == PLAYING) ? audio_bus->frames() : 0;
 440 }
 441
 442 void WebRtcAudioRenderer::OnRenderError() {
 443   NOTIMPLEMENTED();
 444   LOG(ERROR) << "OnRenderError()";
 445 }
 446
 447 // Called by AudioPullFifo when more data is necessary.
 448 void WebRtcAudioRenderer::SourceCallback(
 449     int fifo_frame_delay, media::AudioBus* audio_bus) {
 450   base::TimeTicks start_time = base::TimeTicks::Now() ;
 451   DVLOG(2) << "WebRtcAudioRenderer::SourceCallback("
 452            << fifo_frame_delay << ", "
 453            << audio_bus->frames() << ")";
 454
 455   int output_delay_milliseconds = audio_delay_milliseconds_;
 456   output_delay_milliseconds += fifo_delay_milliseconds_;
 457   DVLOG(2) << "output_delay_milliseconds: " << output_delay_milliseconds;
 458
 459   // We need to keep render data for the |source_| regardless of |state_|,
 460   // otherwise the data will be buffered up inside |source_|.
 461   source_->RenderData(audio_bus, sink_params_.sample_rate(),
 462                       output_delay_milliseconds,
 463                       &current_time_);
 464
 465   // Avoid filling up the audio bus if we are not playing; instead
 466   // return here and ensure that the returned value in Render() is 0.
 467   if (state_ != PLAYING)
 468     audio_bus->Zero();
 469
 470   if (++render_callback_count_ == kNumCallbacksBetweenRenderTimeHistograms) {
 471     base::TimeDelta elapsed = base::TimeTicks::Now() - start_time;
 472     render_callback_count_ = 0;
 473     UMA_HISTOGRAM_TIMES("WebRTC.AudioRenderTimes", elapsed);
 474   }
 475 }
 476
 477 void WebRtcAudioRenderer::UpdateSourceVolume(
 478     webrtc::AudioSourceInterface* source) {
 479   DCHECK(thread_checker_.CalledOnValidThread());
 480
 481   // Note: If there are no playing audio renderers, then the volume will be
 482   // set to 0.0.
 483   float volume = 0.0f;
 484
 485   SourcePlayingStates::iterator entry = source_playing_states_.find(source);
 486   if (entry != source_playing_states_.end()) {
 487     PlayingStates& states = entry->second;
 488     for (PlayingStates::const_iterator it = states.begin();
 489          it != states.end(); ++it) {
 490       if ((*it)->playing())
 491         volume += (*it)->volume();
 492     }
 493   }
 494
 495   // The valid range for volume scaling of a remote webrtc source is
 496   // 0.0-10.0 where 1.0 is no attenuation/boost.
 497   DCHECK(volume >= 0.0f);
 498   if (volume > 10.0f)
 499     volume = 10.0f;
 500
 501   DVLOG(1) << "Setting remote source volume: " << volume;
 502   source->SetVolume(volume);
 503 }
 504
 505 bool WebRtcAudioRenderer::AddPlayingState(
 506     webrtc::AudioSourceInterface* source,
 507     PlayingState* state) {
 508   DCHECK(thread_checker_.CalledOnValidThread());
 509   DCHECK(state->playing());
 510   // Look up or add the |source| to the map.
 511   PlayingStates& array = source_playing_states_[source];
 512   if (std::find(array.begin(), array.end(), state) != array.end())
 513     return false;
 514
 515   array.push_back(state);
 516
 517   return true;
 518 }
 519
 520 bool WebRtcAudioRenderer::RemovePlayingState(
 521     webrtc::AudioSourceInterface* source,
 522     PlayingState* state) {
 523   DCHECK(thread_checker_.CalledOnValidThread());
 524   DCHECK(!state->playing());
 525   SourcePlayingStates::iterator found = source_playing_states_.find(source);
 526   if (found == source_playing_states_.end())
 527     return false;
 528
 529   PlayingStates& array = found->second;
 530   PlayingStates::iterator state_it =
 531       std::find(array.begin(), array.end(), state);
 532   if (state_it == array.end())
 533     return false;
 534
 535   array.erase(state_it);
 536
 537   if (array.empty())
 538     source_playing_states_.erase(found);
 539
 540   return true;
 541 }
 542
 543 void WebRtcAudioRenderer::OnPlayStateChanged(
 544     const scoped_refptr<webrtc::MediaStreamInterface>& media_stream,
 545     PlayingState* state) {
 546   webrtc::AudioTrackVector tracks(media_stream->GetAudioTracks());
 547   for (webrtc::AudioTrackVector::iterator it = tracks.begin();
 548        it != tracks.end(); ++it) {
 549     webrtc::AudioSourceInterface* source = (*it)->GetSource();
 550     DCHECK(source);
 551     if (!state->playing()) {
 552       if (RemovePlayingState(source, state))
 553         EnterPauseState();
 554     } else if (AddPlayingState(source, state)) {
 555       EnterPlayState();
 556     }
 557     UpdateSourceVolume(source);
 558   }
 559 }
 560
 561 }  // namespace content