src/media/audio/audio_input_controller.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/audio/audio_input_controller.h"
   6
   7 #include "base/bind.h"
   8 #include "base/strings/stringprintf.h"
   9 #include "base/threading/thread_restrictions.h"
  10 #include "base/time/time.h"
  11 #include "media/base/limits.h"
  12 #include "media/base/scoped_histogram_timer.h"
  13 #include "media/base/user_input_monitor.h"
  14
  15 using base::TimeDelta;
  16
  17 namespace {
  18 const int kMaxInputChannels = 3;
  19
  20 // TODO(henrika): remove usage of timers and add support for proper
  21 // notification of when the input device is removed.  This was originally added
  22 // to resolve http://crbug.com/79936 for Windows platforms.  This then caused
  23 // breakage (very hard to repro bugs!) on other platforms: See
  24 // http://crbug.com/226327 and http://crbug.com/230972.
  25 // See also that the timer has been disabled on Mac now due to
  26 // crbug.com/357501.
  27 const int kTimerResetIntervalSeconds = 1;
  28 // We have received reports that the timer can be too trigger happy on some
  29 // Mac devices and the initial timer interval has therefore been increased
  30 // from 1 second to 5 seconds.
  31 const int kTimerInitialIntervalSeconds = 5;
  32
  33 #if defined(AUDIO_POWER_MONITORING)
  34 // Time constant for AudioPowerMonitor.
  35 // The utilized smoothing factor (alpha) in the exponential filter is given
  36 // by 1-exp(-1/(fs*ts)), where fs is the sample rate in Hz and ts is the time
  37 // constant given by |kPowerMeasurementTimeConstantMilliseconds|.
  38 // Example: fs=44100, ts=10e-3 => alpha~0.022420
  39 //          fs=44100, ts=20e-3 => alpha~0.165903
  40 // A large smoothing factor corresponds to a faster filter response to input
  41 // changes since y(n)=alpha*x(n)+(1-alpha)*y(n-1), where x(n) is the input
  42 // and y(n) is the output.
  43 const int kPowerMeasurementTimeConstantMilliseconds = 10;
  44
  45 // Time in seconds between two successive measurements of audio power levels.
  46 const int kPowerMonitorLogIntervalSeconds = 5;
  47 #endif
  48 }
  49
  50 // Used to log the result of capture startup.
  51 // This was previously logged as a boolean with only the no callback and OK
  52 // options. The enum order is kept to ensure backwards compatibility.
  53 // Elements in this enum should not be deleted or rearranged; the only
  54 // permitted operation is to add new elements before CAPTURE_STARTUP_RESULT_MAX
  55 // and update CAPTURE_STARTUP_RESULT_MAX.
  56 enum CaptureStartupResult {
  57   CAPTURE_STARTUP_NO_DATA_CALLBACK = 0,
  58   CAPTURE_STARTUP_OK = 1,
  59   CAPTURE_STARTUP_CREATE_STREAM_FAILED = 2,
  60   CAPTURE_STARTUP_OPEN_STREAM_FAILED = 3,
  61   CAPTURE_STARTUP_RESULT_MAX = CAPTURE_STARTUP_OPEN_STREAM_FAILED
  62 };
  63
  64 void LogCaptureStartupResult(CaptureStartupResult result) {
  65   UMA_HISTOGRAM_ENUMERATION("Media.AudioInputControllerCaptureStartupSuccess",
  66                             result,
  67                             CAPTURE_STARTUP_RESULT_MAX + 1);
  68
  69 }
  70
  71 namespace media {
  72
  73 // static
  74 AudioInputController::Factory* AudioInputController::factory_ = NULL;
  75
  76 AudioInputController::AudioInputController(EventHandler* handler,
  77                                            SyncWriter* sync_writer,
  78                                            UserInputMonitor* user_input_monitor)
  79     : creator_task_runner_(base::MessageLoopProxy::current()),
  80       handler_(handler),
  81       stream_(NULL),
  82       data_is_active_(false),
  83       state_(CLOSED),
  84       sync_writer_(sync_writer),
  85       max_volume_(0.0),
  86       user_input_monitor_(user_input_monitor),
  87 #if defined(AUDIO_POWER_MONITORING)
  88       silence_state_(SILENCE_STATE_NO_MEASUREMENT),
  89 #endif
  90       prev_key_down_count_(0) {
  91   DCHECK(creator_task_runner_.get());
  92 }
  93
  94 AudioInputController::~AudioInputController() {
  95   DCHECK_EQ(state_, CLOSED);
  96 }
  97
  98 // static
  99 scoped_refptr<AudioInputController> AudioInputController::Create(
 100     AudioManager* audio_manager,
 101     EventHandler* event_handler,
 102     const AudioParameters& params,
 103     const std::string& device_id,
 104     UserInputMonitor* user_input_monitor) {
 105   DCHECK(audio_manager);
 106
 107   if (!params.IsValid() || (params.channels() > kMaxInputChannels))
 108     return NULL;
 109
 110   if (factory_) {
 111     return factory_->Create(
 112         audio_manager, event_handler, params, user_input_monitor);
 113   }
 114   scoped_refptr<AudioInputController> controller(
 115       new AudioInputController(event_handler, NULL, user_input_monitor));
 116
 117   controller->task_runner_ = audio_manager->GetTaskRunner();
 118
 119   // Create and open a new audio input stream from the existing
 120   // audio-device thread.
 121   if (!controller->task_runner_->PostTask(FROM_HERE,
 122           base::Bind(&AudioInputController::DoCreate, controller,
 123                      base::Unretained(audio_manager), params, device_id))) {
 124     controller = NULL;
 125   }
 126
 127   return controller;
 128 }
 129
 130 // static
 131 scoped_refptr<AudioInputController> AudioInputController::CreateLowLatency(
 132     AudioManager* audio_manager,
 133     EventHandler* event_handler,
 134     const AudioParameters& params,
 135     const std::string& device_id,
 136     SyncWriter* sync_writer,
 137     UserInputMonitor* user_input_monitor) {
 138   DCHECK(audio_manager);
 139   DCHECK(sync_writer);
 140
 141   if (!params.IsValid() || (params.channels() > kMaxInputChannels))
 142     return NULL;
 143
 144   // Create the AudioInputController object and ensure that it runs on
 145   // the audio-manager thread.
 146   scoped_refptr<AudioInputController> controller(
 147       new AudioInputController(event_handler, sync_writer, user_input_monitor));
 148   controller->task_runner_ = audio_manager->GetTaskRunner();
 149
 150   // Create and open a new audio input stream from the existing
 151   // audio-device thread. Use the provided audio-input device.
 152   if (!controller->task_runner_->PostTask(FROM_HERE,
 153           base::Bind(&AudioInputController::DoCreate, controller,
 154                      base::Unretained(audio_manager), params, device_id))) {
 155     controller = NULL;
 156   }
 157
 158   return controller;
 159 }
 160
 161 // static
 162 scoped_refptr<AudioInputController> AudioInputController::CreateForStream(
 163     const scoped_refptr<base::SingleThreadTaskRunner>& task_runner,
 164     EventHandler* event_handler,
 165     AudioInputStream* stream,
 166     SyncWriter* sync_writer,
 167     UserInputMonitor* user_input_monitor) {
 168   DCHECK(sync_writer);
 169   DCHECK(stream);
 170
 171   // Create the AudioInputController object and ensure that it runs on
 172   // the audio-manager thread.
 173   scoped_refptr<AudioInputController> controller(
 174       new AudioInputController(event_handler, sync_writer, user_input_monitor));
 175   controller->task_runner_ = task_runner;
 176
 177   // TODO(miu): See TODO at top of file.  Until that's resolved, we need to
 178   // disable the error auto-detection here (since the audio mirroring
 179   // implementation will reliably report error and close events).  Note, of
 180   // course, that we're assuming CreateForStream() has been called for the audio
 181   // mirroring use case only.
 182   if (!controller->task_runner_->PostTask(
 183           FROM_HERE,
 184           base::Bind(&AudioInputController::DoCreateForStream,
 185                      controller,
 186                      stream))) {
 187     controller = NULL;
 188   }
 189
 190   return controller;
 191 }
 192
 193 void AudioInputController::Record() {
 194   task_runner_->PostTask(FROM_HERE, base::Bind(
 195       &AudioInputController::DoRecord, this));
 196 }
 197
 198 void AudioInputController::Close(const base::Closure& closed_task) {
 199   DCHECK(!closed_task.is_null());
 200   DCHECK(creator_task_runner_->BelongsToCurrentThread());
 201
 202   task_runner_->PostTaskAndReply(
 203       FROM_HERE, base::Bind(&AudioInputController::DoClose, this), closed_task);
 204 }
 205
 206 void AudioInputController::SetVolume(double volume) {
 207   task_runner_->PostTask(FROM_HERE, base::Bind(
 208       &AudioInputController::DoSetVolume, this, volume));
 209 }
 210
 211 void AudioInputController::SetAutomaticGainControl(bool enabled) {
 212   task_runner_->PostTask(FROM_HERE, base::Bind(
 213       &AudioInputController::DoSetAutomaticGainControl, this, enabled));
 214 }
 215
 216 void AudioInputController::DoCreate(AudioManager* audio_manager,
 217                                     const AudioParameters& params,
 218                                     const std::string& device_id) {
 219   DCHECK(task_runner_->BelongsToCurrentThread());
 220   SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.CreateTime");
 221
 222 #if defined(AUDIO_POWER_MONITORING)
 223   // Create the audio (power) level meter given the provided audio parameters.
 224   // An AudioBus is also needed to wrap the raw data buffer from the native
 225   // layer to match AudioPowerMonitor::Scan().
 226   // TODO(henrika): Remove use of extra AudioBus. See http://crbug.com/375155.
 227   audio_level_.reset(new media::AudioPowerMonitor(
 228       params.sample_rate(),
 229       TimeDelta::FromMilliseconds(kPowerMeasurementTimeConstantMilliseconds)));
 230   audio_params_ = params;
 231   silence_state_ = SILENCE_STATE_NO_MEASUREMENT;
 232 #endif
 233
 234   // TODO(miu): See TODO at top of file.  Until that's resolved, assume all
 235   // platform audio input requires the |no_data_timer_| be used to auto-detect
 236   // errors.  In reality, probably only Windows needs to be treated as
 237   // unreliable here.
 238   DoCreateForStream(audio_manager->MakeAudioInputStream(params, device_id));
 239 }
 240
 241 void AudioInputController::DoCreateForStream(
 242     AudioInputStream* stream_to_control) {
 243   DCHECK(task_runner_->BelongsToCurrentThread());
 244
 245   DCHECK(!stream_);
 246   stream_ = stream_to_control;
 247
 248   if (!stream_) {
 249     if (handler_)
 250       handler_->OnError(this, STREAM_CREATE_ERROR);
 251     LogCaptureStartupResult(CAPTURE_STARTUP_CREATE_STREAM_FAILED);
 252     return;
 253   }
 254
 255   if (stream_ && !stream_->Open()) {
 256     stream_->Close();
 257     stream_ = NULL;
 258     if (handler_)
 259       handler_->OnError(this, STREAM_OPEN_ERROR);
 260     LogCaptureStartupResult(CAPTURE_STARTUP_OPEN_STREAM_FAILED);
 261     return;
 262   }
 263
 264   DCHECK(!no_data_timer_.get());
 265
 266   // Create the data timer which will call FirstCheckForNoData(). The timer
 267   // is started in DoRecord() and restarted in each DoCheckForNoData()
 268   // callback.
 269   // The timer is enabled for logging purposes. The NO_DATA_ERROR triggered
 270   // from the timer must be ignored by the EventHandler.
 271   // TODO(henrika): remove usage of timer when it has been verified on Canary
 272   // that we are safe doing so. Goal is to get rid of |no_data_timer_| and
 273   // everything that is tied to it. crbug.com/357569.
 274   no_data_timer_.reset(new base::Timer(
 275       FROM_HERE, base::TimeDelta::FromSeconds(kTimerInitialIntervalSeconds),
 276       base::Bind(&AudioInputController::FirstCheckForNoData,
 277                  base::Unretained(this)), false));
 278
 279   state_ = CREATED;
 280   if (handler_)
 281     handler_->OnCreated(this);
 282
 283   if (user_input_monitor_) {
 284     user_input_monitor_->EnableKeyPressMonitoring();
 285     prev_key_down_count_ = user_input_monitor_->GetKeyPressCount();
 286   }
 287 }
 288
 289 void AudioInputController::DoRecord() {
 290   DCHECK(task_runner_->BelongsToCurrentThread());
 291   SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.RecordTime");
 292
 293   if (state_ != CREATED)
 294     return;
 295
 296   {
 297     base::AutoLock auto_lock(lock_);
 298     state_ = RECORDING;
 299   }
 300
 301   if (no_data_timer_) {
 302     // Start the data timer. Once |kTimerResetIntervalSeconds| have passed,
 303     // a callback to FirstCheckForNoData() is made.
 304     no_data_timer_->Reset();
 305   }
 306
 307   stream_->Start(this);
 308   if (handler_)
 309     handler_->OnRecording(this);
 310 }
 311
 312 void AudioInputController::DoClose() {
 313   DCHECK(task_runner_->BelongsToCurrentThread());
 314   SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.CloseTime");
 315
 316   if (state_ == CLOSED)
 317     return;
 318
 319   // Delete the timer on the same thread that created it.
 320   no_data_timer_.reset();
 321
 322   DoStopCloseAndClearStream();
 323   SetDataIsActive(false);
 324
 325   if (SharedMemoryAndSyncSocketMode())
 326     sync_writer_->Close();
 327
 328   if (user_input_monitor_)
 329     user_input_monitor_->DisableKeyPressMonitoring();
 330
 331 #if defined(AUDIO_POWER_MONITORING)
 332   // Send UMA stats if we have enabled power monitoring.
 333   if (audio_level_) {
 334     LogSilenceState(silence_state_);
 335   }
 336 #endif
 337
 338   state_ = CLOSED;
 339 }
 340
 341 void AudioInputController::DoReportError() {
 342   DCHECK(task_runner_->BelongsToCurrentThread());
 343   if (handler_)
 344     handler_->OnError(this, STREAM_ERROR);
 345 }
 346
 347 void AudioInputController::DoSetVolume(double volume) {
 348   DCHECK(task_runner_->BelongsToCurrentThread());
 349   DCHECK_GE(volume, 0);
 350   DCHECK_LE(volume, 1.0);
 351
 352   if (state_ != CREATED && state_ != RECORDING)
 353     return;
 354
 355   // Only ask for the maximum volume at first call and use cached value
 356   // for remaining function calls.
 357   if (!max_volume_) {
 358     max_volume_ = stream_->GetMaxVolume();
 359   }
 360
 361   if (max_volume_ == 0.0) {
 362     DLOG(WARNING) << "Failed to access input volume control";
 363     return;
 364   }
 365
 366   // Set the stream volume and scale to a range matched to the platform.
 367   stream_->SetVolume(max_volume_ * volume);
 368 }
 369
 370 void AudioInputController::DoSetAutomaticGainControl(bool enabled) {
 371   DCHECK(task_runner_->BelongsToCurrentThread());
 372   DCHECK_NE(state_, RECORDING);
 373
 374   // Ensure that the AGC state only can be modified before streaming starts.
 375   if (state_ != CREATED)
 376     return;
 377
 378   stream_->SetAutomaticGainControl(enabled);
 379 }
 380
 381 void AudioInputController::FirstCheckForNoData() {
 382   DCHECK(task_runner_->BelongsToCurrentThread());
 383   LogCaptureStartupResult(GetDataIsActive() ?
 384                           CAPTURE_STARTUP_OK :
 385                           CAPTURE_STARTUP_NO_DATA_CALLBACK);
 386   DoCheckForNoData();
 387 }
 388
 389 void AudioInputController::DoCheckForNoData() {
 390   DCHECK(task_runner_->BelongsToCurrentThread());
 391
 392   if (!GetDataIsActive()) {
 393     // The data-is-active marker will be false only if it has been more than
 394     // one second since a data packet was recorded. This can happen if a
 395     // capture device has been removed or disabled.
 396     if (handler_)
 397       handler_->OnError(this, NO_DATA_ERROR);
 398   }
 399
 400   // Mark data as non-active. The flag will be re-enabled in OnData() each
 401   // time a data packet is received. Hence, under normal conditions, the
 402   // flag will only be disabled during a very short period.
 403   SetDataIsActive(false);
 404
 405   // Restart the timer to ensure that we check the flag again in
 406   // |kTimerResetIntervalSeconds|.
 407   no_data_timer_->Start(
 408       FROM_HERE, base::TimeDelta::FromSeconds(kTimerResetIntervalSeconds),
 409       base::Bind(&AudioInputController::DoCheckForNoData,
 410       base::Unretained(this)));
 411 }
 412
 413 void AudioInputController::OnData(AudioInputStream* stream,
 414                                   const AudioBus* source,
 415                                   uint32 hardware_delay_bytes,
 416                                   double volume) {
 417   // Mark data as active to ensure that the periodic calls to
 418   // DoCheckForNoData() does not report an error to the event handler.
 419   SetDataIsActive(true);
 420
 421   {
 422     base::AutoLock auto_lock(lock_);
 423     if (state_ != RECORDING)
 424       return;
 425   }
 426
 427   bool key_pressed = false;
 428   if (user_input_monitor_) {
 429     size_t current_count = user_input_monitor_->GetKeyPressCount();
 430     key_pressed = current_count != prev_key_down_count_;
 431     prev_key_down_count_ = current_count;
 432     DVLOG_IF(6, key_pressed) << "Detected keypress.";
 433   }
 434
 435   // Use SharedMemory and SyncSocket if the client has created a SyncWriter.
 436   // Used by all low-latency clients except WebSpeech.
 437   if (SharedMemoryAndSyncSocketMode()) {
 438     sync_writer_->Write(source, volume, key_pressed);
 439     sync_writer_->UpdateRecordedBytes(hardware_delay_bytes);
 440
 441 #if defined(AUDIO_POWER_MONITORING)
 442     // Only do power-level measurements if an AudioPowerMonitor object has
 443     // been created. Done in DoCreate() but not DoCreateForStream(), hence
 444     // logging will mainly be done for WebRTC and WebSpeech clients.
 445     if (!audio_level_)
 446       return;
 447
 448     // Perform periodic audio (power) level measurements.
 449     if ((base::TimeTicks::Now() - last_audio_level_log_time_).InSeconds() >
 450         kPowerMonitorLogIntervalSeconds) {
 451       // Wrap data into an AudioBus to match AudioPowerMonitor::Scan.
 452       // TODO(henrika): remove this section when capture side uses AudioBus.
 453       // See http://crbug.com/375155 for details.
 454       audio_level_->Scan(*source, source->frames());
 455
 456       // Get current average power level and add it to the log.
 457       // Possible range is given by [-inf, 0] dBFS.
 458       std::pair<float, bool> result = audio_level_->ReadCurrentPowerAndClip();
 459
 460       // Use event handler on the audio thread to relay a message to the ARIH
 461       // in content which does the actual logging on the IO thread.
 462       task_runner_->PostTask(
 463           FROM_HERE,
 464           base::Bind(
 465               &AudioInputController::DoLogAudioLevel, this, result.first));
 466
 467       last_audio_level_log_time_ = base::TimeTicks::Now();
 468
 469       // Reset the average power level (since we don't log continuously).
 470       audio_level_->Reset();
 471     }
 472 #endif
 473     return;
 474   }
 475
 476   // TODO(henrika): Investigate if we can avoid the extra copy here.
 477   // (see http://crbug.com/249316 for details). AFAIK, this scope is only
 478   // active for WebSpeech clients.
 479   scoped_ptr<AudioBus> audio_data =
 480       AudioBus::Create(source->channels(), source->frames());
 481   source->CopyTo(audio_data.get());
 482
 483   // Ownership of the audio buffer will be with the callback until it is run,
 484   // when ownership is passed to the callback function.
 485   task_runner_->PostTask(
 486       FROM_HERE,
 487       base::Bind(
 488           &AudioInputController::DoOnData, this, base::Passed(&audio_data)));
 489 }
 490
 491 void AudioInputController::DoOnData(scoped_ptr<AudioBus> data) {
 492   DCHECK(task_runner_->BelongsToCurrentThread());
 493   if (handler_)
 494     handler_->OnData(this, data.get());
 495 }
 496
 497 void AudioInputController::DoLogAudioLevel(float level_dbfs) {
 498 #if defined(AUDIO_POWER_MONITORING)
 499   DCHECK(task_runner_->BelongsToCurrentThread());
 500   if (!handler_)
 501     return;
 502
 503   std::string log_string = base::StringPrintf(
 504       "AIC::OnData: average audio level=%.2f dBFS", level_dbfs);
 505   static const float kSilenceThresholdDBFS = -72.24719896f;
 506   if (level_dbfs < kSilenceThresholdDBFS) {
 507     log_string += " <=> no audio input!";
 508     if (silence_state_ == SILENCE_STATE_NO_MEASUREMENT)
 509       silence_state_ = SILENCE_STATE_ONLY_SILENCE;
 510     else if (silence_state_ == SILENCE_STATE_ONLY_AUDIO)
 511       silence_state_ = SILENCE_STATE_AUDIO_AND_SILENCE;
 512     else
 513       DCHECK(silence_state_ == SILENCE_STATE_ONLY_SILENCE ||
 514              silence_state_ == SILENCE_STATE_AUDIO_AND_SILENCE);
 515   } else {
 516     if (silence_state_ == SILENCE_STATE_NO_MEASUREMENT)
 517       silence_state_ = SILENCE_STATE_ONLY_AUDIO;
 518     else if (silence_state_ == SILENCE_STATE_ONLY_SILENCE)
 519       silence_state_ = SILENCE_STATE_AUDIO_AND_SILENCE;
 520     else
 521       DCHECK(silence_state_ == SILENCE_STATE_ONLY_AUDIO ||
 522              silence_state_ == SILENCE_STATE_AUDIO_AND_SILENCE);
 523   }
 524
 525   handler_->OnLog(this, log_string);
 526 #endif
 527 }
 528
 529 void AudioInputController::OnError(AudioInputStream* stream) {
 530   // Handle error on the audio-manager thread.
 531   task_runner_->PostTask(FROM_HERE, base::Bind(
 532       &AudioInputController::DoReportError, this));
 533 }
 534
 535 void AudioInputController::DoStopCloseAndClearStream() {
 536   DCHECK(task_runner_->BelongsToCurrentThread());
 537
 538   // Allow calling unconditionally and bail if we don't have a stream to close.
 539   if (stream_ != NULL) {
 540     stream_->Stop();
 541     stream_->Close();
 542     stream_ = NULL;
 543   }
 544
 545   // The event handler should not be touched after the stream has been closed.
 546   handler_ = NULL;
 547 }
 548
 549 void AudioInputController::SetDataIsActive(bool enabled) {
 550   base::subtle::Release_Store(&data_is_active_, enabled);
 551 }
 552
 553 bool AudioInputController::GetDataIsActive() {
 554   return (base::subtle::Acquire_Load(&data_is_active_) != false);
 555 }
 556
 557 #if defined(AUDIO_POWER_MONITORING)
 558 void AudioInputController::LogSilenceState(SilenceState value) {
 559   UMA_HISTOGRAM_ENUMERATION("Media.AudioInputControllerSessionSilenceReport",
 560                             value,
 561                             SILENCE_STATE_MAX + 1);
 562 }
 563 #endif
 564
 565 }  // namespace media