1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "media/audio/audio_input_controller.h"
8 #include "base/strings/stringprintf.h"
9 #include "base/threading/thread_restrictions.h"
10 #include "base/time/time.h"
11 #include "media/audio/audio_parameters.h"
12 #include "media/base/limits.h"
13 #include "media/base/scoped_histogram_timer.h"
14 #include "media/base/user_input_monitor.h"
16 using base::TimeDelta;
19 const int kMaxInputChannels = 3;
21 // TODO(henrika): remove usage of timers and add support for proper
22 // notification of when the input device is removed. This was originally added
23 // to resolve http://crbug.com/79936 for Windows platforms. This then caused
24 // breakage (very hard to repro bugs!) on other platforms: See
25 // http://crbug.com/226327 and http://crbug.com/230972.
26 // See also that the timer has been disabled on Mac now due to
28 const int kTimerResetIntervalSeconds = 1;
29 // We have received reports that the timer can be too trigger happy on some
30 // Mac devices and the initial timer interval has therefore been increased
31 // from 1 second to 5 seconds.
32 const int kTimerInitialIntervalSeconds = 5;
34 #if defined(AUDIO_POWER_MONITORING)
35 // Time constant for AudioPowerMonitor.
36 // The utilized smoothing factor (alpha) in the exponential filter is given
37 // by 1-exp(-1/(fs*ts)), where fs is the sample rate in Hz and ts is the time
38 // constant given by |kPowerMeasurementTimeConstantMilliseconds|.
39 // Example: fs=44100, ts=10e-3 => alpha~0.022420
40 // fs=44100, ts=20e-3 => alpha~0.165903
41 // A large smoothing factor corresponds to a faster filter response to input
42 // changes since y(n)=alpha*x(n)+(1-alpha)*y(n-1), where x(n) is the input
43 // and y(n) is the output.
44 const int kPowerMeasurementTimeConstantMilliseconds = 10;
46 // Time in seconds between two successive measurements of audio power levels.
47 const int kPowerMonitorLogIntervalSeconds = 15;
49 // A warning will be logged when the microphone audio volume is below this
51 const int kLowLevelMicrophoneLevelPercent = 10;
55 // Used to log the result of capture startup.
56 // This was previously logged as a boolean with only the no callback and OK
57 // options. The enum order is kept to ensure backwards compatibility.
58 // Elements in this enum should not be deleted or rearranged; the only
59 // permitted operation is to add new elements before CAPTURE_STARTUP_RESULT_MAX
60 // and update CAPTURE_STARTUP_RESULT_MAX.
61 enum CaptureStartupResult {
62 CAPTURE_STARTUP_NO_DATA_CALLBACK = 0,
63 CAPTURE_STARTUP_OK = 1,
64 CAPTURE_STARTUP_CREATE_STREAM_FAILED = 2,
65 CAPTURE_STARTUP_OPEN_STREAM_FAILED = 3,
66 CAPTURE_STARTUP_RESULT_MAX = CAPTURE_STARTUP_OPEN_STREAM_FAILED
69 void LogCaptureStartupResult(CaptureStartupResult result) {
70 UMA_HISTOGRAM_ENUMERATION("Media.AudioInputControllerCaptureStartupSuccess",
72 CAPTURE_STARTUP_RESULT_MAX + 1);
79 AudioInputController::Factory* AudioInputController::factory_ = NULL;
81 AudioInputController::AudioInputController(EventHandler* handler,
82 SyncWriter* sync_writer,
83 UserInputMonitor* user_input_monitor)
84 : creator_task_runner_(base::MessageLoopProxy::current()),
87 data_is_active_(false),
89 sync_writer_(sync_writer),
91 user_input_monitor_(user_input_monitor),
92 #if defined(AUDIO_POWER_MONITORING)
93 log_silence_state_(false),
94 silence_state_(SILENCE_STATE_NO_MEASUREMENT),
96 prev_key_down_count_(0) {
97 DCHECK(creator_task_runner_.get());
100 AudioInputController::~AudioInputController() {
101 DCHECK_EQ(state_, CLOSED);
105 scoped_refptr<AudioInputController> AudioInputController::Create(
106 AudioManager* audio_manager,
107 EventHandler* event_handler,
108 const AudioParameters& params,
109 const std::string& device_id,
110 UserInputMonitor* user_input_monitor) {
111 DCHECK(audio_manager);
113 if (!params.IsValid() || (params.channels() > kMaxInputChannels))
117 return factory_->Create(
118 audio_manager, event_handler, params, user_input_monitor);
120 scoped_refptr<AudioInputController> controller(
121 new AudioInputController(event_handler, NULL, user_input_monitor));
123 controller->task_runner_ = audio_manager->GetTaskRunner();
125 // Create and open a new audio input stream from the existing
126 // audio-device thread.
127 if (!controller->task_runner_->PostTask(
129 base::Bind(&AudioInputController::DoCreate,
131 base::Unretained(audio_manager),
141 scoped_refptr<AudioInputController> AudioInputController::CreateLowLatency(
142 AudioManager* audio_manager,
143 EventHandler* event_handler,
144 const AudioParameters& params,
145 const std::string& device_id,
146 SyncWriter* sync_writer,
147 UserInputMonitor* user_input_monitor) {
148 DCHECK(audio_manager);
151 if (!params.IsValid() || (params.channels() > kMaxInputChannels))
154 // Create the AudioInputController object and ensure that it runs on
155 // the audio-manager thread.
156 scoped_refptr<AudioInputController> controller(
157 new AudioInputController(event_handler, sync_writer, user_input_monitor));
158 controller->task_runner_ = audio_manager->GetTaskRunner();
160 // Create and open a new audio input stream from the existing
161 // audio-device thread. Use the provided audio-input device.
162 if (!controller->task_runner_->PostTask(
164 base::Bind(&AudioInputController::DoCreateForLowLatency,
166 base::Unretained(audio_manager),
176 scoped_refptr<AudioInputController> AudioInputController::CreateForStream(
177 const scoped_refptr<base::SingleThreadTaskRunner>& task_runner,
178 EventHandler* event_handler,
179 AudioInputStream* stream,
180 SyncWriter* sync_writer,
181 UserInputMonitor* user_input_monitor) {
185 // Create the AudioInputController object and ensure that it runs on
186 // the audio-manager thread.
187 scoped_refptr<AudioInputController> controller(
188 new AudioInputController(event_handler, sync_writer, user_input_monitor));
189 controller->task_runner_ = task_runner;
191 // TODO(miu): See TODO at top of file. Until that's resolved, we need to
192 // disable the error auto-detection here (since the audio mirroring
193 // implementation will reliably report error and close events). Note, of
194 // course, that we're assuming CreateForStream() has been called for the audio
195 // mirroring use case only.
196 if (!controller->task_runner_->PostTask(
198 base::Bind(&AudioInputController::DoCreateForStream,
207 void AudioInputController::Record() {
208 task_runner_->PostTask(FROM_HERE, base::Bind(
209 &AudioInputController::DoRecord, this));
212 void AudioInputController::Close(const base::Closure& closed_task) {
213 DCHECK(!closed_task.is_null());
214 DCHECK(creator_task_runner_->BelongsToCurrentThread());
216 task_runner_->PostTaskAndReply(
217 FROM_HERE, base::Bind(&AudioInputController::DoClose, this), closed_task);
220 void AudioInputController::SetVolume(double volume) {
221 task_runner_->PostTask(FROM_HERE, base::Bind(
222 &AudioInputController::DoSetVolume, this, volume));
225 void AudioInputController::SetAutomaticGainControl(bool enabled) {
226 task_runner_->PostTask(FROM_HERE, base::Bind(
227 &AudioInputController::DoSetAutomaticGainControl, this, enabled));
230 void AudioInputController::DoCreate(AudioManager* audio_manager,
231 const AudioParameters& params,
232 const std::string& device_id) {
233 DCHECK(task_runner_->BelongsToCurrentThread());
234 SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.CreateTime");
236 handler_->OnLog(this, "AIC::DoCreate");
238 #if defined(AUDIO_POWER_MONITORING)
239 // Create the audio (power) level meter given the provided audio parameters.
240 // An AudioBus is also needed to wrap the raw data buffer from the native
241 // layer to match AudioPowerMonitor::Scan().
242 // TODO(henrika): Remove use of extra AudioBus. See http://crbug.com/375155.
243 last_audio_level_log_time_ = base::TimeTicks::Now();
244 audio_level_.reset(new media::AudioPowerMonitor(
245 params.sample_rate(),
246 TimeDelta::FromMilliseconds(kPowerMeasurementTimeConstantMilliseconds)));
247 audio_params_ = params;
248 silence_state_ = SILENCE_STATE_NO_MEASUREMENT;
251 // TODO(miu): See TODO at top of file. Until that's resolved, assume all
252 // platform audio input requires the |no_data_timer_| be used to auto-detect
253 // errors. In reality, probably only Windows needs to be treated as
255 DoCreateForStream(audio_manager->MakeAudioInputStream(params, device_id));
258 void AudioInputController::DoCreateForLowLatency(AudioManager* audio_manager,
259 const AudioParameters& params,
260 const std::string& device_id) {
261 DCHECK(task_runner_->BelongsToCurrentThread());
263 #if defined(AUDIO_POWER_MONITORING)
264 // We only log silence state UMA stats for low latency mode and if we use a
266 if (params.format() != AudioParameters::AUDIO_FAKE)
267 log_silence_state_ = true;
270 DoCreate(audio_manager, params, device_id);
273 void AudioInputController::DoCreateForStream(
274 AudioInputStream* stream_to_control) {
275 DCHECK(task_runner_->BelongsToCurrentThread());
278 stream_ = stream_to_control;
282 handler_->OnError(this, STREAM_CREATE_ERROR);
283 LogCaptureStartupResult(CAPTURE_STARTUP_CREATE_STREAM_FAILED);
287 if (stream_ && !stream_->Open()) {
291 handler_->OnError(this, STREAM_OPEN_ERROR);
292 LogCaptureStartupResult(CAPTURE_STARTUP_OPEN_STREAM_FAILED);
296 DCHECK(!no_data_timer_.get());
298 // Create the data timer which will call FirstCheckForNoData(). The timer
299 // is started in DoRecord() and restarted in each DoCheckForNoData()
301 // The timer is enabled for logging purposes. The NO_DATA_ERROR triggered
302 // from the timer must be ignored by the EventHandler.
303 // TODO(henrika): remove usage of timer when it has been verified on Canary
304 // that we are safe doing so. Goal is to get rid of |no_data_timer_| and
305 // everything that is tied to it. crbug.com/357569.
306 no_data_timer_.reset(new base::Timer(
307 FROM_HERE, base::TimeDelta::FromSeconds(kTimerInitialIntervalSeconds),
308 base::Bind(&AudioInputController::FirstCheckForNoData,
309 base::Unretained(this)), false));
313 handler_->OnCreated(this);
315 if (user_input_monitor_) {
316 user_input_monitor_->EnableKeyPressMonitoring();
317 prev_key_down_count_ = user_input_monitor_->GetKeyPressCount();
321 void AudioInputController::DoRecord() {
322 DCHECK(task_runner_->BelongsToCurrentThread());
323 SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.RecordTime");
325 if (state_ != CREATED)
329 base::AutoLock auto_lock(lock_);
334 handler_->OnLog(this, "AIC::DoRecord");
336 if (no_data_timer_) {
337 // Start the data timer. Once |kTimerResetIntervalSeconds| have passed,
338 // a callback to FirstCheckForNoData() is made.
339 no_data_timer_->Reset();
342 stream_->Start(this);
344 handler_->OnRecording(this);
347 void AudioInputController::DoClose() {
348 DCHECK(task_runner_->BelongsToCurrentThread());
349 SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.CloseTime");
351 if (state_ == CLOSED)
355 handler_->OnLog(this, "AIC::DoClose");
357 // Delete the timer on the same thread that created it.
358 no_data_timer_.reset();
360 DoStopCloseAndClearStream();
361 SetDataIsActive(false);
363 if (SharedMemoryAndSyncSocketMode())
364 sync_writer_->Close();
366 if (user_input_monitor_)
367 user_input_monitor_->DisableKeyPressMonitoring();
369 #if defined(AUDIO_POWER_MONITORING)
370 // Send UMA stats if enabled.
371 if (log_silence_state_)
372 LogSilenceState(silence_state_);
373 log_silence_state_ = false;
379 void AudioInputController::DoReportError() {
380 DCHECK(task_runner_->BelongsToCurrentThread());
382 handler_->OnError(this, STREAM_ERROR);
385 void AudioInputController::DoSetVolume(double volume) {
386 DCHECK(task_runner_->BelongsToCurrentThread());
387 DCHECK_GE(volume, 0);
388 DCHECK_LE(volume, 1.0);
390 if (state_ != CREATED && state_ != RECORDING)
393 // Only ask for the maximum volume at first call and use cached value
394 // for remaining function calls.
396 max_volume_ = stream_->GetMaxVolume();
399 if (max_volume_ == 0.0) {
400 DLOG(WARNING) << "Failed to access input volume control";
404 // Set the stream volume and scale to a range matched to the platform.
405 stream_->SetVolume(max_volume_ * volume);
408 void AudioInputController::DoSetAutomaticGainControl(bool enabled) {
409 DCHECK(task_runner_->BelongsToCurrentThread());
410 DCHECK_NE(state_, RECORDING);
412 // Ensure that the AGC state only can be modified before streaming starts.
413 if (state_ != CREATED)
416 stream_->SetAutomaticGainControl(enabled);
419 void AudioInputController::FirstCheckForNoData() {
420 DCHECK(task_runner_->BelongsToCurrentThread());
421 LogCaptureStartupResult(GetDataIsActive() ?
423 CAPTURE_STARTUP_NO_DATA_CALLBACK);
425 handler_->OnLog(this, GetDataIsActive() ?
426 "AIC::FirstCheckForNoData => data is active" :
427 "AIC::FirstCheckForNoData => data is NOT active");
432 void AudioInputController::DoCheckForNoData() {
433 DCHECK(task_runner_->BelongsToCurrentThread());
435 if (!GetDataIsActive()) {
436 // The data-is-active marker will be false only if it has been more than
437 // one second since a data packet was recorded. This can happen if a
438 // capture device has been removed or disabled.
440 handler_->OnError(this, NO_DATA_ERROR);
443 // Mark data as non-active. The flag will be re-enabled in OnData() each
444 // time a data packet is received. Hence, under normal conditions, the
445 // flag will only be disabled during a very short period.
446 SetDataIsActive(false);
448 // Restart the timer to ensure that we check the flag again in
449 // |kTimerResetIntervalSeconds|.
450 no_data_timer_->Start(
451 FROM_HERE, base::TimeDelta::FromSeconds(kTimerResetIntervalSeconds),
452 base::Bind(&AudioInputController::DoCheckForNoData,
453 base::Unretained(this)));
456 void AudioInputController::OnData(AudioInputStream* stream,
457 const AudioBus* source,
458 uint32 hardware_delay_bytes,
460 // Mark data as active to ensure that the periodic calls to
461 // DoCheckForNoData() does not report an error to the event handler.
462 SetDataIsActive(true);
465 base::AutoLock auto_lock(lock_);
466 if (state_ != RECORDING)
470 bool key_pressed = false;
471 if (user_input_monitor_) {
472 size_t current_count = user_input_monitor_->GetKeyPressCount();
473 key_pressed = current_count != prev_key_down_count_;
474 prev_key_down_count_ = current_count;
475 DVLOG_IF(6, key_pressed) << "Detected keypress.";
478 // Use SharedMemory and SyncSocket if the client has created a SyncWriter.
479 // Used by all low-latency clients except WebSpeech.
480 if (SharedMemoryAndSyncSocketMode()) {
481 sync_writer_->Write(source, volume, key_pressed);
482 sync_writer_->UpdateRecordedBytes(hardware_delay_bytes);
484 #if defined(AUDIO_POWER_MONITORING)
485 // Only do power-level measurements if an AudioPowerMonitor object has
486 // been created. Done in DoCreate() but not DoCreateForStream(), hence
487 // logging will mainly be done for WebRTC and WebSpeech clients.
491 // Perform periodic audio (power) level measurements.
492 if ((base::TimeTicks::Now() - last_audio_level_log_time_).InSeconds() >
493 kPowerMonitorLogIntervalSeconds) {
494 // Wrap data into an AudioBus to match AudioPowerMonitor::Scan.
495 // TODO(henrika): remove this section when capture side uses AudioBus.
496 // See http://crbug.com/375155 for details.
497 audio_level_->Scan(*source, source->frames());
499 // Get current average power level and add it to the log.
500 // Possible range is given by [-inf, 0] dBFS.
501 std::pair<float, bool> result = audio_level_->ReadCurrentPowerAndClip();
503 // Add current microphone volume to log and UMA histogram.
504 const int mic_volume_percent = static_cast<int>(100.0 * volume);
506 // Use event handler on the audio thread to relay a message to the ARIH
507 // in content which does the actual logging on the IO thread.
508 task_runner_->PostTask(FROM_HERE,
509 base::Bind(&AudioInputController::DoLogAudioLevels,
512 mic_volume_percent));
514 last_audio_level_log_time_ = base::TimeTicks::Now();
516 // Reset the average power level (since we don't log continuously).
517 audio_level_->Reset();
523 // TODO(henrika): Investigate if we can avoid the extra copy here.
524 // (see http://crbug.com/249316 for details). AFAIK, this scope is only
525 // active for WebSpeech clients.
526 scoped_ptr<AudioBus> audio_data =
527 AudioBus::Create(source->channels(), source->frames());
528 source->CopyTo(audio_data.get());
530 // Ownership of the audio buffer will be with the callback until it is run,
531 // when ownership is passed to the callback function.
532 task_runner_->PostTask(
535 &AudioInputController::DoOnData, this, base::Passed(&audio_data)));
538 void AudioInputController::DoOnData(scoped_ptr<AudioBus> data) {
539 DCHECK(task_runner_->BelongsToCurrentThread());
541 handler_->OnData(this, data.get());
544 void AudioInputController::DoLogAudioLevels(float level_dbfs,
545 int microphone_volume_percent) {
546 #if defined(AUDIO_POWER_MONITORING)
547 DCHECK(task_runner_->BelongsToCurrentThread());
551 std::string log_string = base::StringPrintf(
552 "AIC::OnData: average audio level=%.2f dBFS", level_dbfs);
553 static const float kSilenceThresholdDBFS = -72.24719896f;
554 if (level_dbfs < kSilenceThresholdDBFS)
555 log_string += " <=> no audio input!";
556 handler_->OnLog(this, log_string);
558 UpdateSilenceState(level_dbfs < kSilenceThresholdDBFS);
560 UMA_HISTOGRAM_PERCENTAGE("Media.MicrophoneVolume", microphone_volume_percent);
561 log_string = base::StringPrintf(
562 "AIC::OnData: microphone volume=%d%%", microphone_volume_percent);
563 if (microphone_volume_percent < kLowLevelMicrophoneLevelPercent)
564 log_string += " <=> low microphone level!";
565 handler_->OnLog(this, log_string);
569 void AudioInputController::OnError(AudioInputStream* stream) {
570 // Handle error on the audio-manager thread.
571 task_runner_->PostTask(FROM_HERE, base::Bind(
572 &AudioInputController::DoReportError, this));
575 void AudioInputController::DoStopCloseAndClearStream() {
576 DCHECK(task_runner_->BelongsToCurrentThread());
578 // Allow calling unconditionally and bail if we don't have a stream to close.
579 if (stream_ != NULL) {
585 // The event handler should not be touched after the stream has been closed.
589 void AudioInputController::SetDataIsActive(bool enabled) {
590 base::subtle::Release_Store(&data_is_active_, enabled);
593 bool AudioInputController::GetDataIsActive() {
594 return (base::subtle::Acquire_Load(&data_is_active_) != false);
597 #if defined(AUDIO_POWER_MONITORING)
598 void AudioInputController::UpdateSilenceState(bool silence) {
600 if (silence_state_ == SILENCE_STATE_NO_MEASUREMENT) {
601 silence_state_ = SILENCE_STATE_ONLY_SILENCE;
602 } else if (silence_state_ == SILENCE_STATE_ONLY_AUDIO) {
603 silence_state_ = SILENCE_STATE_AUDIO_AND_SILENCE;
605 DCHECK(silence_state_ == SILENCE_STATE_ONLY_SILENCE ||
606 silence_state_ == SILENCE_STATE_AUDIO_AND_SILENCE);
609 if (silence_state_ == SILENCE_STATE_NO_MEASUREMENT) {
610 silence_state_ = SILENCE_STATE_ONLY_AUDIO;
611 } else if (silence_state_ == SILENCE_STATE_ONLY_SILENCE) {
612 silence_state_ = SILENCE_STATE_AUDIO_AND_SILENCE;
614 DCHECK(silence_state_ == SILENCE_STATE_ONLY_AUDIO ||
615 silence_state_ == SILENCE_STATE_AUDIO_AND_SILENCE);
620 void AudioInputController::LogSilenceState(SilenceState value) {
621 UMA_HISTOGRAM_ENUMERATION("Media.AudioInputControllerSessionSilenceReport",
623 SILENCE_STATE_MAX + 1);