1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "media/audio/audio_input_controller.h"
8 #include "base/strings/string_number_conversions.h"
9 #include "base/strings/stringprintf.h"
10 #include "base/threading/thread_restrictions.h"
11 #include "base/time/time.h"
12 #include "media/audio/audio_parameters.h"
13 #include "media/base/scoped_histogram_timer.h"
14 #include "media/base/user_input_monitor.h"
16 using base::TimeDelta;
20 const int kMaxInputChannels = 3;
22 // TODO(henrika): remove usage of timers and add support for proper
23 // notification of when the input device is removed. This was originally added
24 // to resolve http://crbug.com/79936 for Windows platforms. This then caused
25 // breakage (very hard to repro bugs!) on other platforms: See
26 // http://crbug.com/226327 and http://crbug.com/230972.
27 // See also that the timer has been disabled on Mac now due to
29 const int kTimerResetIntervalSeconds = 1;
30 // We have received reports that the timer can be too trigger happy on some
31 // Mac devices and the initial timer interval has therefore been increased
32 // from 1 second to 5 seconds.
33 const int kTimerInitialIntervalSeconds = 5;
35 #if defined(AUDIO_POWER_MONITORING)
36 // Time constant for AudioPowerMonitor.
37 // The utilized smoothing factor (alpha) in the exponential filter is given
38 // by 1-exp(-1/(fs*ts)), where fs is the sample rate in Hz and ts is the time
39 // constant given by |kPowerMeasurementTimeConstantMilliseconds|.
40 // Example: fs=44100, ts=10e-3 => alpha~0.022420
41 // fs=44100, ts=20e-3 => alpha~0.165903
42 // A large smoothing factor corresponds to a faster filter response to input
43 // changes since y(n)=alpha*x(n)+(1-alpha)*y(n-1), where x(n) is the input
44 // and y(n) is the output.
45 const int kPowerMeasurementTimeConstantMilliseconds = 10;
47 // Time in seconds between two successive measurements of audio power levels.
48 const int kPowerMonitorLogIntervalSeconds = 15;
50 // A warning will be logged when the microphone audio volume is below this
52 const int kLowLevelMicrophoneLevelPercent = 10;
54 // Logs if the user has enabled the microphone mute or not. This is normally
55 // done by marking a checkbox in an audio-settings UI which is unique for each
56 // platform. Elements in this enum should not be added, deleted or rearranged.
57 enum MicrophoneMuteResult {
58 MICROPHONE_IS_MUTED = 0,
59 MICROPHONE_IS_NOT_MUTED = 1,
60 MICROPHONE_MUTE_MAX = MICROPHONE_IS_NOT_MUTED
63 void LogMicrophoneMuteResult(MicrophoneMuteResult result) {
64 UMA_HISTOGRAM_ENUMERATION("Media.MicrophoneMuted",
66 MICROPHONE_MUTE_MAX + 1);
71 // Used to log the result of capture startup.
72 // This was previously logged as a boolean with only the no callback and OK
73 // options. The enum order is kept to ensure backwards compatibility.
74 // Elements in this enum should not be deleted or rearranged; the only
75 // permitted operation is to add new elements before CAPTURE_STARTUP_RESULT_MAX
76 // and update CAPTURE_STARTUP_RESULT_MAX.
77 enum CaptureStartupResult {
78 CAPTURE_STARTUP_NO_DATA_CALLBACK = 0,
79 CAPTURE_STARTUP_OK = 1,
80 CAPTURE_STARTUP_CREATE_STREAM_FAILED = 2,
81 CAPTURE_STARTUP_OPEN_STREAM_FAILED = 3,
82 CAPTURE_STARTUP_RESULT_MAX = CAPTURE_STARTUP_OPEN_STREAM_FAILED
85 void LogCaptureStartupResult(CaptureStartupResult result) {
86 UMA_HISTOGRAM_ENUMERATION("Media.AudioInputControllerCaptureStartupSuccess",
88 CAPTURE_STARTUP_RESULT_MAX + 1);
95 AudioInputController::Factory* AudioInputController::factory_ = NULL;
97 AudioInputController::AudioInputController(EventHandler* handler,
98 SyncWriter* sync_writer,
99 UserInputMonitor* user_input_monitor)
100 : creator_task_runner_(base::MessageLoopProxy::current()),
103 data_is_active_(false),
105 sync_writer_(sync_writer),
107 user_input_monitor_(user_input_monitor),
108 #if defined(AUDIO_POWER_MONITORING)
109 log_silence_state_(false),
110 silence_state_(SILENCE_STATE_NO_MEASUREMENT),
112 prev_key_down_count_(0) {
113 DCHECK(creator_task_runner_.get());
116 AudioInputController::~AudioInputController() {
117 DCHECK_EQ(state_, CLOSED);
121 scoped_refptr<AudioInputController> AudioInputController::Create(
122 AudioManager* audio_manager,
123 EventHandler* event_handler,
124 const AudioParameters& params,
125 const std::string& device_id,
126 UserInputMonitor* user_input_monitor) {
127 DCHECK(audio_manager);
129 if (!params.IsValid() || (params.channels() > kMaxInputChannels))
133 return factory_->Create(
134 audio_manager, event_handler, params, user_input_monitor);
136 scoped_refptr<AudioInputController> controller(
137 new AudioInputController(event_handler, NULL, user_input_monitor));
139 controller->task_runner_ = audio_manager->GetTaskRunner();
141 // Create and open a new audio input stream from the existing
142 // audio-device thread.
143 if (!controller->task_runner_->PostTask(
145 base::Bind(&AudioInputController::DoCreate,
147 base::Unretained(audio_manager),
157 scoped_refptr<AudioInputController> AudioInputController::CreateLowLatency(
158 AudioManager* audio_manager,
159 EventHandler* event_handler,
160 const AudioParameters& params,
161 const std::string& device_id,
162 SyncWriter* sync_writer,
163 UserInputMonitor* user_input_monitor) {
164 DCHECK(audio_manager);
167 if (!params.IsValid() || (params.channels() > kMaxInputChannels))
170 // Create the AudioInputController object and ensure that it runs on
171 // the audio-manager thread.
172 scoped_refptr<AudioInputController> controller(
173 new AudioInputController(event_handler, sync_writer, user_input_monitor));
174 controller->task_runner_ = audio_manager->GetTaskRunner();
176 // Create and open a new audio input stream from the existing
177 // audio-device thread. Use the provided audio-input device.
178 if (!controller->task_runner_->PostTask(
180 base::Bind(&AudioInputController::DoCreateForLowLatency,
182 base::Unretained(audio_manager),
192 scoped_refptr<AudioInputController> AudioInputController::CreateForStream(
193 const scoped_refptr<base::SingleThreadTaskRunner>& task_runner,
194 EventHandler* event_handler,
195 AudioInputStream* stream,
196 SyncWriter* sync_writer,
197 UserInputMonitor* user_input_monitor) {
201 // Create the AudioInputController object and ensure that it runs on
202 // the audio-manager thread.
203 scoped_refptr<AudioInputController> controller(
204 new AudioInputController(event_handler, sync_writer, user_input_monitor));
205 controller->task_runner_ = task_runner;
207 // TODO(miu): See TODO at top of file. Until that's resolved, we need to
208 // disable the error auto-detection here (since the audio mirroring
209 // implementation will reliably report error and close events). Note, of
210 // course, that we're assuming CreateForStream() has been called for the audio
211 // mirroring use case only.
212 if (!controller->task_runner_->PostTask(
214 base::Bind(&AudioInputController::DoCreateForStream,
223 void AudioInputController::Record() {
224 task_runner_->PostTask(FROM_HERE, base::Bind(
225 &AudioInputController::DoRecord, this));
228 void AudioInputController::Close(const base::Closure& closed_task) {
229 DCHECK(!closed_task.is_null());
230 DCHECK(creator_task_runner_->BelongsToCurrentThread());
232 task_runner_->PostTaskAndReply(
233 FROM_HERE, base::Bind(&AudioInputController::DoClose, this), closed_task);
236 void AudioInputController::SetVolume(double volume) {
237 task_runner_->PostTask(FROM_HERE, base::Bind(
238 &AudioInputController::DoSetVolume, this, volume));
241 void AudioInputController::SetAutomaticGainControl(bool enabled) {
242 task_runner_->PostTask(FROM_HERE, base::Bind(
243 &AudioInputController::DoSetAutomaticGainControl, this, enabled));
246 void AudioInputController::DoCreate(AudioManager* audio_manager,
247 const AudioParameters& params,
248 const std::string& device_id) {
249 DCHECK(task_runner_->BelongsToCurrentThread());
250 SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.CreateTime");
252 handler_->OnLog(this, "AIC::DoCreate");
254 #if defined(AUDIO_POWER_MONITORING)
255 // Create the audio (power) level meter given the provided audio parameters.
256 // An AudioBus is also needed to wrap the raw data buffer from the native
257 // layer to match AudioPowerMonitor::Scan().
258 // TODO(henrika): Remove use of extra AudioBus. See http://crbug.com/375155.
259 last_audio_level_log_time_ = base::TimeTicks::Now();
260 audio_level_.reset(new media::AudioPowerMonitor(
261 params.sample_rate(),
262 TimeDelta::FromMilliseconds(kPowerMeasurementTimeConstantMilliseconds)));
263 audio_params_ = params;
264 silence_state_ = SILENCE_STATE_NO_MEASUREMENT;
267 // TODO(miu): See TODO at top of file. Until that's resolved, assume all
268 // platform audio input requires the |no_data_timer_| be used to auto-detect
269 // errors. In reality, probably only Windows needs to be treated as
271 DoCreateForStream(audio_manager->MakeAudioInputStream(params, device_id));
274 void AudioInputController::DoCreateForLowLatency(AudioManager* audio_manager,
275 const AudioParameters& params,
276 const std::string& device_id) {
277 DCHECK(task_runner_->BelongsToCurrentThread());
279 #if defined(AUDIO_POWER_MONITORING)
280 // We only log silence state UMA stats for low latency mode and if we use a
282 if (params.format() != AudioParameters::AUDIO_FAKE)
283 log_silence_state_ = true;
286 low_latency_create_time_ = base::TimeTicks::Now();
287 DoCreate(audio_manager, params, device_id);
290 void AudioInputController::DoCreateForStream(
291 AudioInputStream* stream_to_control) {
292 DCHECK(task_runner_->BelongsToCurrentThread());
295 stream_ = stream_to_control;
299 handler_->OnError(this, STREAM_CREATE_ERROR);
300 LogCaptureStartupResult(CAPTURE_STARTUP_CREATE_STREAM_FAILED);
304 if (stream_ && !stream_->Open()) {
308 handler_->OnError(this, STREAM_OPEN_ERROR);
309 LogCaptureStartupResult(CAPTURE_STARTUP_OPEN_STREAM_FAILED);
313 DCHECK(!no_data_timer_.get());
315 // Create the data timer which will call FirstCheckForNoData(). The timer
316 // is started in DoRecord() and restarted in each DoCheckForNoData()
318 // The timer is enabled for logging purposes. The NO_DATA_ERROR triggered
319 // from the timer must be ignored by the EventHandler.
320 // TODO(henrika): remove usage of timer when it has been verified on Canary
321 // that we are safe doing so. Goal is to get rid of |no_data_timer_| and
322 // everything that is tied to it. crbug.com/357569.
323 no_data_timer_.reset(new base::Timer(
324 FROM_HERE, base::TimeDelta::FromSeconds(kTimerInitialIntervalSeconds),
325 base::Bind(&AudioInputController::FirstCheckForNoData,
326 base::Unretained(this)), false));
330 handler_->OnCreated(this);
332 if (user_input_monitor_) {
333 user_input_monitor_->EnableKeyPressMonitoring();
334 prev_key_down_count_ = user_input_monitor_->GetKeyPressCount();
338 void AudioInputController::DoRecord() {
339 DCHECK(task_runner_->BelongsToCurrentThread());
340 SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.RecordTime");
342 if (state_ != CREATED)
346 base::AutoLock auto_lock(lock_);
351 handler_->OnLog(this, "AIC::DoRecord");
353 if (no_data_timer_) {
354 // Start the data timer. Once |kTimerResetIntervalSeconds| have passed,
355 // a callback to FirstCheckForNoData() is made.
356 no_data_timer_->Reset();
359 stream_->Start(this);
361 handler_->OnRecording(this);
364 void AudioInputController::DoClose() {
365 DCHECK(task_runner_->BelongsToCurrentThread());
366 SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.CloseTime");
368 if (state_ == CLOSED)
371 // If this is a low-latency stream, log the total duration (since DoCreate)
372 // and add it to a UMA histogram.
373 if (!low_latency_create_time_.is_null()) {
374 base::TimeDelta duration =
375 base::TimeTicks::Now() - low_latency_create_time_;
376 UMA_HISTOGRAM_LONG_TIMES("Media.InputStreamDuration", duration);
378 std::string log_string =
379 base::StringPrintf("AIC::DoClose: stream duration=");
380 log_string += base::Int64ToString(duration.InSeconds());
381 log_string += " seconds";
382 handler_->OnLog(this, log_string);
386 // Delete the timer on the same thread that created it.
387 no_data_timer_.reset();
389 DoStopCloseAndClearStream();
390 SetDataIsActive(false);
392 if (SharedMemoryAndSyncSocketMode())
393 sync_writer_->Close();
395 if (user_input_monitor_)
396 user_input_monitor_->DisableKeyPressMonitoring();
398 #if defined(AUDIO_POWER_MONITORING)
399 // Send UMA stats if enabled.
400 if (log_silence_state_)
401 LogSilenceState(silence_state_);
402 log_silence_state_ = false;
408 void AudioInputController::DoReportError() {
409 DCHECK(task_runner_->BelongsToCurrentThread());
411 handler_->OnError(this, STREAM_ERROR);
414 void AudioInputController::DoSetVolume(double volume) {
415 DCHECK(task_runner_->BelongsToCurrentThread());
416 DCHECK_GE(volume, 0);
417 DCHECK_LE(volume, 1.0);
419 if (state_ != CREATED && state_ != RECORDING)
422 // Only ask for the maximum volume at first call and use cached value
423 // for remaining function calls.
425 max_volume_ = stream_->GetMaxVolume();
428 if (max_volume_ == 0.0) {
429 DLOG(WARNING) << "Failed to access input volume control";
433 // Set the stream volume and scale to a range matched to the platform.
434 stream_->SetVolume(max_volume_ * volume);
437 void AudioInputController::DoSetAutomaticGainControl(bool enabled) {
438 DCHECK(task_runner_->BelongsToCurrentThread());
439 DCHECK_NE(state_, RECORDING);
441 // Ensure that the AGC state only can be modified before streaming starts.
442 if (state_ != CREATED)
445 stream_->SetAutomaticGainControl(enabled);
448 void AudioInputController::FirstCheckForNoData() {
449 DCHECK(task_runner_->BelongsToCurrentThread());
450 LogCaptureStartupResult(GetDataIsActive() ?
452 CAPTURE_STARTUP_NO_DATA_CALLBACK);
454 handler_->OnLog(this, GetDataIsActive() ?
455 "AIC::FirstCheckForNoData => data is active" :
456 "AIC::FirstCheckForNoData => data is NOT active");
461 void AudioInputController::DoCheckForNoData() {
462 DCHECK(task_runner_->BelongsToCurrentThread());
464 if (!GetDataIsActive()) {
465 // The data-is-active marker will be false only if it has been more than
466 // one second since a data packet was recorded. This can happen if a
467 // capture device has been removed or disabled.
469 handler_->OnError(this, NO_DATA_ERROR);
472 // Mark data as non-active. The flag will be re-enabled in OnData() each
473 // time a data packet is received. Hence, under normal conditions, the
474 // flag will only be disabled during a very short period.
475 SetDataIsActive(false);
477 // Restart the timer to ensure that we check the flag again in
478 // |kTimerResetIntervalSeconds|.
479 no_data_timer_->Start(
480 FROM_HERE, base::TimeDelta::FromSeconds(kTimerResetIntervalSeconds),
481 base::Bind(&AudioInputController::DoCheckForNoData,
482 base::Unretained(this)));
485 void AudioInputController::OnData(AudioInputStream* stream,
486 const AudioBus* source,
487 uint32 hardware_delay_bytes,
489 // Mark data as active to ensure that the periodic calls to
490 // DoCheckForNoData() does not report an error to the event handler.
491 SetDataIsActive(true);
494 base::AutoLock auto_lock(lock_);
495 if (state_ != RECORDING)
499 bool key_pressed = false;
500 if (user_input_monitor_) {
501 size_t current_count = user_input_monitor_->GetKeyPressCount();
502 key_pressed = current_count != prev_key_down_count_;
503 prev_key_down_count_ = current_count;
504 DVLOG_IF(6, key_pressed) << "Detected keypress.";
507 // Use SharedMemory and SyncSocket if the client has created a SyncWriter.
508 // Used by all low-latency clients except WebSpeech.
509 if (SharedMemoryAndSyncSocketMode()) {
510 sync_writer_->Write(source, volume, key_pressed);
511 sync_writer_->UpdateRecordedBytes(hardware_delay_bytes);
513 #if defined(AUDIO_POWER_MONITORING)
514 // Only do power-level measurements if an AudioPowerMonitor object has
515 // been created. Done in DoCreate() but not DoCreateForStream(), hence
516 // logging will mainly be done for WebRTC and WebSpeech clients.
520 // Perform periodic audio (power) level measurements.
521 if ((base::TimeTicks::Now() - last_audio_level_log_time_).InSeconds() >
522 kPowerMonitorLogIntervalSeconds) {
523 // Wrap data into an AudioBus to match AudioPowerMonitor::Scan.
524 // TODO(henrika): remove this section when capture side uses AudioBus.
525 // See http://crbug.com/375155 for details.
526 audio_level_->Scan(*source, source->frames());
528 // Get current average power level and add it to the log.
529 // Possible range is given by [-inf, 0] dBFS.
530 std::pair<float, bool> result = audio_level_->ReadCurrentPowerAndClip();
532 // Add current microphone volume to log and UMA histogram.
533 const int mic_volume_percent = static_cast<int>(100.0 * volume);
535 // Use event handler on the audio thread to relay a message to the ARIH
536 // in content which does the actual logging on the IO thread.
537 task_runner_->PostTask(FROM_HERE,
538 base::Bind(&AudioInputController::DoLogAudioLevels,
541 mic_volume_percent));
543 last_audio_level_log_time_ = base::TimeTicks::Now();
545 // Reset the average power level (since we don't log continuously).
546 audio_level_->Reset();
552 // TODO(henrika): Investigate if we can avoid the extra copy here.
553 // (see http://crbug.com/249316 for details). AFAIK, this scope is only
554 // active for WebSpeech clients.
555 scoped_ptr<AudioBus> audio_data =
556 AudioBus::Create(source->channels(), source->frames());
557 source->CopyTo(audio_data.get());
559 // Ownership of the audio buffer will be with the callback until it is run,
560 // when ownership is passed to the callback function.
561 task_runner_->PostTask(
564 &AudioInputController::DoOnData, this, base::Passed(&audio_data)));
567 void AudioInputController::DoOnData(scoped_ptr<AudioBus> data) {
568 DCHECK(task_runner_->BelongsToCurrentThread());
570 handler_->OnData(this, data.get());
573 void AudioInputController::DoLogAudioLevels(float level_dbfs,
574 int microphone_volume_percent) {
575 #if defined(AUDIO_POWER_MONITORING)
576 DCHECK(task_runner_->BelongsToCurrentThread());
580 std::string log_string = base::StringPrintf(
581 "AIC::OnData: average audio level=%.2f dBFS", level_dbfs);
582 static const float kSilenceThresholdDBFS = -72.24719896f;
583 if (level_dbfs < kSilenceThresholdDBFS)
584 log_string += " <=> no audio input!";
585 handler_->OnLog(this, log_string);
587 UpdateSilenceState(level_dbfs < kSilenceThresholdDBFS);
589 UMA_HISTOGRAM_PERCENTAGE("Media.MicrophoneVolume", microphone_volume_percent);
590 log_string = base::StringPrintf(
591 "AIC::OnData: microphone volume=%d%%", microphone_volume_percent);
592 if (microphone_volume_percent < kLowLevelMicrophoneLevelPercent)
593 log_string += " <=> low microphone level!";
594 handler_->OnLog(this, log_string);
596 // Try to detect if the user has enabled hardware mute by pressing the mute
597 // button in audio settings for the selected microphone. The idea here is to
598 // detect when all input samples are zeros but the actual volume slider is
599 // larger than zero. It should correspond to a hardware mute state.
600 if (level_dbfs == -std::numeric_limits<float>::infinity() &&
601 microphone_volume_percent > 0) {
602 LogMicrophoneMuteResult(MICROPHONE_IS_MUTED);
603 log_string = base::StringPrintf(
604 "AIC::OnData: microphone is muted!");
605 handler_->OnLog(this, log_string);
607 LogMicrophoneMuteResult(MICROPHONE_IS_NOT_MUTED);
612 void AudioInputController::OnError(AudioInputStream* stream) {
613 // Handle error on the audio-manager thread.
614 task_runner_->PostTask(FROM_HERE, base::Bind(
615 &AudioInputController::DoReportError, this));
618 void AudioInputController::DoStopCloseAndClearStream() {
619 DCHECK(task_runner_->BelongsToCurrentThread());
621 // Allow calling unconditionally and bail if we don't have a stream to close.
622 if (stream_ != NULL) {
628 // The event handler should not be touched after the stream has been closed.
632 void AudioInputController::SetDataIsActive(bool enabled) {
633 base::subtle::Release_Store(&data_is_active_, enabled);
636 bool AudioInputController::GetDataIsActive() {
637 return (base::subtle::Acquire_Load(&data_is_active_) != false);
640 #if defined(AUDIO_POWER_MONITORING)
641 void AudioInputController::UpdateSilenceState(bool silence) {
643 if (silence_state_ == SILENCE_STATE_NO_MEASUREMENT) {
644 silence_state_ = SILENCE_STATE_ONLY_SILENCE;
645 } else if (silence_state_ == SILENCE_STATE_ONLY_AUDIO) {
646 silence_state_ = SILENCE_STATE_AUDIO_AND_SILENCE;
648 DCHECK(silence_state_ == SILENCE_STATE_ONLY_SILENCE ||
649 silence_state_ == SILENCE_STATE_AUDIO_AND_SILENCE);
652 if (silence_state_ == SILENCE_STATE_NO_MEASUREMENT) {
653 silence_state_ = SILENCE_STATE_ONLY_AUDIO;
654 } else if (silence_state_ == SILENCE_STATE_ONLY_SILENCE) {
655 silence_state_ = SILENCE_STATE_AUDIO_AND_SILENCE;
657 DCHECK(silence_state_ == SILENCE_STATE_ONLY_AUDIO ||
658 silence_state_ == SILENCE_STATE_AUDIO_AND_SILENCE);
663 void AudioInputController::LogSilenceState(SilenceState value) {
664 UMA_HISTOGRAM_ENUMERATION("Media.AudioInputControllerSessionSilenceReport",
666 SILENCE_STATE_MAX + 1);