#include "media/audio/audio_input_controller.h"
#include "base/bind.h"
+#include "base/strings/stringprintf.h"
#include "base/threading/thread_restrictions.h"
+#include "base/time/time.h"
+#include "media/audio/audio_parameters.h"
#include "media/base/limits.h"
#include "media/base/scoped_histogram_timer.h"
#include "media/base/user_input_monitor.h"
+using base::TimeDelta;
+
namespace {
-const int kMaxInputChannels = 2;
+const int kMaxInputChannels = 3;
// TODO(henrika): remove usage of timers and add support for proper
// notification of when the input device is removed. This was originally added
// to resolve http://crbug.com/79936 for Windows platforms. This then caused
// breakage (very hard to repro bugs!) on other platforms: See
// http://crbug.com/226327 and http://crbug.com/230972.
+// See also that the timer has been disabled on Mac now due to
+// crbug.com/357501.
const int kTimerResetIntervalSeconds = 1;
// We have received reports that the timer can be too trigger happy on some
// Mac devices and the initial timer interval has therefore been increased
// from 1 second to 5 seconds.
const int kTimerInitialIntervalSeconds = 5;
+
+#if defined(AUDIO_POWER_MONITORING)
+// Time constant for AudioPowerMonitor.
+// The utilized smoothing factor (alpha) in the exponential filter is given
+// by 1-exp(-1/(fs*ts)), where fs is the sample rate in Hz and ts is the time
+// constant given by |kPowerMeasurementTimeConstantMilliseconds|.
+// Example: fs=44100, ts=10e-3 => alpha~0.022420
+// fs=44100, ts=20e-3 => alpha~0.165903
+// A large smoothing factor corresponds to a faster filter response to input
+// changes since y(n)=alpha*x(n)+(1-alpha)*y(n-1), where x(n) is the input
+// and y(n) is the output.
+const int kPowerMeasurementTimeConstantMilliseconds = 10;
+
+// Time in seconds between two successive measurements of audio power levels.
+const int kPowerMonitorLogIntervalSeconds = 5;
+#endif
+}
+
+// Used to log the result of capture startup.
+// This was previously logged as a boolean with only the no callback and OK
+// options. The enum order is kept to ensure backwards compatibility.
+// Elements in this enum should not be deleted or rearranged; the only
+// permitted operation is to add new elements before CAPTURE_STARTUP_RESULT_MAX
+// and update CAPTURE_STARTUP_RESULT_MAX.
+enum CaptureStartupResult {
+ CAPTURE_STARTUP_NO_DATA_CALLBACK = 0,
+ CAPTURE_STARTUP_OK = 1,
+ CAPTURE_STARTUP_CREATE_STREAM_FAILED = 2,
+ CAPTURE_STARTUP_OPEN_STREAM_FAILED = 3,
+ CAPTURE_STARTUP_RESULT_MAX = CAPTURE_STARTUP_OPEN_STREAM_FAILED
+};
+
+void LogCaptureStartupResult(CaptureStartupResult result) {
+ UMA_HISTOGRAM_ENUMERATION("Media.AudioInputControllerCaptureStartupSuccess",
+ result,
+ CAPTURE_STARTUP_RESULT_MAX + 1);
+
}
namespace media {
AudioInputController::AudioInputController(EventHandler* handler,
SyncWriter* sync_writer,
UserInputMonitor* user_input_monitor)
- : creator_loop_(base::MessageLoopProxy::current()),
+ : creator_task_runner_(base::MessageLoopProxy::current()),
handler_(handler),
stream_(NULL),
data_is_active_(false),
- state_(kEmpty),
+ state_(CLOSED),
sync_writer_(sync_writer),
max_volume_(0.0),
user_input_monitor_(user_input_monitor),
+#if defined(AUDIO_POWER_MONITORING)
+ log_silence_state_(false),
+ silence_state_(SILENCE_STATE_NO_MEASUREMENT),
+#endif
prev_key_down_count_(0) {
- DCHECK(creator_loop_.get());
+ DCHECK(creator_task_runner_.get());
}
AudioInputController::~AudioInputController() {
- DCHECK(kClosed == state_ || kCreated == state_ || kEmpty == state_);
+ DCHECK_EQ(state_, CLOSED);
}
// static
scoped_refptr<AudioInputController> controller(
new AudioInputController(event_handler, NULL, user_input_monitor));
- controller->message_loop_ = audio_manager->GetMessageLoop();
+ controller->task_runner_ = audio_manager->GetTaskRunner();
// Create and open a new audio input stream from the existing
// audio-device thread.
- if (!controller->message_loop_->PostTask(FROM_HERE,
- base::Bind(&AudioInputController::DoCreate, controller,
- base::Unretained(audio_manager), params, device_id))) {
+ if (!controller->task_runner_->PostTask(
+ FROM_HERE,
+ base::Bind(&AudioInputController::DoCreate,
+ controller,
+ base::Unretained(audio_manager),
+ params,
+ device_id))) {
controller = NULL;
}
// the audio-manager thread.
scoped_refptr<AudioInputController> controller(
new AudioInputController(event_handler, sync_writer, user_input_monitor));
- controller->message_loop_ = audio_manager->GetMessageLoop();
+ controller->task_runner_ = audio_manager->GetTaskRunner();
// Create and open a new audio input stream from the existing
// audio-device thread. Use the provided audio-input device.
- if (!controller->message_loop_->PostTask(FROM_HERE,
- base::Bind(&AudioInputController::DoCreate, controller,
- base::Unretained(audio_manager), params, device_id))) {
+ if (!controller->task_runner_->PostTask(
+ FROM_HERE,
+ base::Bind(&AudioInputController::DoCreateForLowLatency,
+ controller,
+ base::Unretained(audio_manager),
+ params,
+ device_id))) {
controller = NULL;
}
// static
scoped_refptr<AudioInputController> AudioInputController::CreateForStream(
- const scoped_refptr<base::MessageLoopProxy>& message_loop,
+ const scoped_refptr<base::SingleThreadTaskRunner>& task_runner,
EventHandler* event_handler,
AudioInputStream* stream,
SyncWriter* sync_writer,
// the audio-manager thread.
scoped_refptr<AudioInputController> controller(
new AudioInputController(event_handler, sync_writer, user_input_monitor));
- controller->message_loop_ = message_loop;
+ controller->task_runner_ = task_runner;
// TODO(miu): See TODO at top of file. Until that's resolved, we need to
// disable the error auto-detection here (since the audio mirroring
// implementation will reliably report error and close events). Note, of
// course, that we're assuming CreateForStream() has been called for the audio
// mirroring use case only.
- if (!controller->message_loop_->PostTask(
+ if (!controller->task_runner_->PostTask(
FROM_HERE,
- base::Bind(&AudioInputController::DoCreateForStream, controller,
- stream, false))) {
+ base::Bind(&AudioInputController::DoCreateForStream,
+ controller,
+ stream))) {
controller = NULL;
}
}
void AudioInputController::Record() {
- message_loop_->PostTask(FROM_HERE, base::Bind(
+ task_runner_->PostTask(FROM_HERE, base::Bind(
&AudioInputController::DoRecord, this));
}
void AudioInputController::Close(const base::Closure& closed_task) {
DCHECK(!closed_task.is_null());
- DCHECK(creator_loop_->BelongsToCurrentThread());
+ DCHECK(creator_task_runner_->BelongsToCurrentThread());
- message_loop_->PostTaskAndReply(
+ task_runner_->PostTaskAndReply(
FROM_HERE, base::Bind(&AudioInputController::DoClose, this), closed_task);
}
void AudioInputController::SetVolume(double volume) {
- message_loop_->PostTask(FROM_HERE, base::Bind(
+ task_runner_->PostTask(FROM_HERE, base::Bind(
&AudioInputController::DoSetVolume, this, volume));
}
void AudioInputController::SetAutomaticGainControl(bool enabled) {
- message_loop_->PostTask(FROM_HERE, base::Bind(
+ task_runner_->PostTask(FROM_HERE, base::Bind(
&AudioInputController::DoSetAutomaticGainControl, this, enabled));
}
void AudioInputController::DoCreate(AudioManager* audio_manager,
const AudioParameters& params,
const std::string& device_id) {
- DCHECK(message_loop_->BelongsToCurrentThread());
+ DCHECK(task_runner_->BelongsToCurrentThread());
SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.CreateTime");
+ if (handler_)
+ handler_->OnLog(this, "AIC::DoCreate");
+
+#if defined(AUDIO_POWER_MONITORING)
+ // Create the audio (power) level meter given the provided audio parameters.
+ // An AudioBus is also needed to wrap the raw data buffer from the native
+ // layer to match AudioPowerMonitor::Scan().
+ // TODO(henrika): Remove use of extra AudioBus. See http://crbug.com/375155.
+ last_audio_level_log_time_ = base::TimeTicks::Now();
+ audio_level_.reset(new media::AudioPowerMonitor(
+ params.sample_rate(),
+ TimeDelta::FromMilliseconds(kPowerMeasurementTimeConstantMilliseconds)));
+ audio_params_ = params;
+ silence_state_ = SILENCE_STATE_NO_MEASUREMENT;
+#endif
+
// TODO(miu): See TODO at top of file. Until that's resolved, assume all
// platform audio input requires the |no_data_timer_| be used to auto-detect
// errors. In reality, probably only Windows needs to be treated as
// unreliable here.
- DoCreateForStream(audio_manager->MakeAudioInputStream(params, device_id),
- true);
+ DoCreateForStream(audio_manager->MakeAudioInputStream(params, device_id));
+}
+
+void AudioInputController::DoCreateForLowLatency(AudioManager* audio_manager,
+ const AudioParameters& params,
+ const std::string& device_id) {
+ DCHECK(task_runner_->BelongsToCurrentThread());
+
+#if defined(AUDIO_POWER_MONITORING)
+ // We only log silence state UMA stats for low latency mode and if we use a
+ // real device.
+ if (params.format() != AudioParameters::AUDIO_FAKE)
+ log_silence_state_ = true;
+#endif
+
+ DoCreate(audio_manager, params, device_id);
}
void AudioInputController::DoCreateForStream(
- AudioInputStream* stream_to_control, bool enable_nodata_timer) {
- DCHECK(message_loop_->BelongsToCurrentThread());
+ AudioInputStream* stream_to_control) {
+ DCHECK(task_runner_->BelongsToCurrentThread());
DCHECK(!stream_);
stream_ = stream_to_control;
if (!stream_) {
- handler_->OnError(this);
+ if (handler_)
+ handler_->OnError(this, STREAM_CREATE_ERROR);
+ LogCaptureStartupResult(CAPTURE_STARTUP_CREATE_STREAM_FAILED);
return;
}
if (stream_ && !stream_->Open()) {
stream_->Close();
stream_ = NULL;
- handler_->OnError(this);
+ if (handler_)
+ handler_->OnError(this, STREAM_OPEN_ERROR);
+ LogCaptureStartupResult(CAPTURE_STARTUP_OPEN_STREAM_FAILED);
return;
}
DCHECK(!no_data_timer_.get());
- if (enable_nodata_timer) {
- // Create the data timer which will call DoCheckForNoData(). The timer
- // is started in DoRecord() and restarted in each DoCheckForNoData()
- // callback.
- no_data_timer_.reset(new base::Timer(
- FROM_HERE, base::TimeDelta::FromSeconds(kTimerInitialIntervalSeconds),
- base::Bind(&AudioInputController::DoCheckForNoData,
- base::Unretained(this)), false));
- } else {
- DVLOG(1) << "Disabled: timer check for no data.";
- }
- state_ = kCreated;
- handler_->OnCreated(this);
+ // Create the data timer which will call FirstCheckForNoData(). The timer
+ // is started in DoRecord() and restarted in each DoCheckForNoData()
+ // callback.
+ // The timer is enabled for logging purposes. The NO_DATA_ERROR triggered
+ // from the timer must be ignored by the EventHandler.
+ // TODO(henrika): remove usage of timer when it has been verified on Canary
+ // that we are safe doing so. Goal is to get rid of |no_data_timer_| and
+ // everything that is tied to it. crbug.com/357569.
+ no_data_timer_.reset(new base::Timer(
+ FROM_HERE, base::TimeDelta::FromSeconds(kTimerInitialIntervalSeconds),
+ base::Bind(&AudioInputController::FirstCheckForNoData,
+ base::Unretained(this)), false));
+
+ state_ = CREATED;
+ if (handler_)
+ handler_->OnCreated(this);
if (user_input_monitor_) {
user_input_monitor_->EnableKeyPressMonitoring();
}
void AudioInputController::DoRecord() {
- DCHECK(message_loop_->BelongsToCurrentThread());
+ DCHECK(task_runner_->BelongsToCurrentThread());
SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.RecordTime");
- if (state_ != kCreated)
+ if (state_ != CREATED)
return;
{
base::AutoLock auto_lock(lock_);
- state_ = kRecording;
+ state_ = RECORDING;
}
+ if (handler_)
+ handler_->OnLog(this, "AIC::DoRecord");
+
if (no_data_timer_) {
// Start the data timer. Once |kTimerResetIntervalSeconds| have passed,
- // a callback to DoCheckForNoData() is made.
+ // a callback to FirstCheckForNoData() is made.
no_data_timer_->Reset();
}
stream_->Start(this);
- handler_->OnRecording(this);
+ if (handler_)
+ handler_->OnRecording(this);
}
void AudioInputController::DoClose() {
- DCHECK(message_loop_->BelongsToCurrentThread());
+ DCHECK(task_runner_->BelongsToCurrentThread());
SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.CloseTime");
+ if (state_ == CLOSED)
+ return;
+
+ if (handler_)
+ handler_->OnLog(this, "AIC::DoClose");
+
// Delete the timer on the same thread that created it.
no_data_timer_.reset();
- if (state_ != kClosed) {
- DoStopCloseAndClearStream(NULL);
- SetDataIsActive(false);
+ DoStopCloseAndClearStream();
+ SetDataIsActive(false);
- if (LowLatencyMode()) {
- sync_writer_->Close();
- }
+ if (SharedMemoryAndSyncSocketMode())
+ sync_writer_->Close();
- state_ = kClosed;
+ if (user_input_monitor_)
+ user_input_monitor_->DisableKeyPressMonitoring();
- if (user_input_monitor_)
- user_input_monitor_->DisableKeyPressMonitoring();
- }
+#if defined(AUDIO_POWER_MONITORING)
+ // Send UMA stats if enabled.
+ if (log_silence_state_)
+ LogSilenceState(silence_state_);
+ log_silence_state_ = false;
+#endif
+
+ state_ = CLOSED;
}
void AudioInputController::DoReportError() {
- DCHECK(message_loop_->BelongsToCurrentThread());
- handler_->OnError(this);
+ DCHECK(task_runner_->BelongsToCurrentThread());
+ if (handler_)
+ handler_->OnError(this, STREAM_ERROR);
}
void AudioInputController::DoSetVolume(double volume) {
- DCHECK(message_loop_->BelongsToCurrentThread());
+ DCHECK(task_runner_->BelongsToCurrentThread());
DCHECK_GE(volume, 0);
DCHECK_LE(volume, 1.0);
- if (state_ != kCreated && state_ != kRecording)
+ if (state_ != CREATED && state_ != RECORDING)
return;
// Only ask for the maximum volume at first call and use cached value
}
void AudioInputController::DoSetAutomaticGainControl(bool enabled) {
- DCHECK(message_loop_->BelongsToCurrentThread());
- DCHECK_NE(state_, kRecording);
+ DCHECK(task_runner_->BelongsToCurrentThread());
+ DCHECK_NE(state_, RECORDING);
// Ensure that the AGC state only can be modified before streaming starts.
- if (state_ != kCreated || state_ == kRecording)
+ if (state_ != CREATED)
return;
stream_->SetAutomaticGainControl(enabled);
}
+void AudioInputController::FirstCheckForNoData() {
+ DCHECK(task_runner_->BelongsToCurrentThread());
+ LogCaptureStartupResult(GetDataIsActive() ?
+ CAPTURE_STARTUP_OK :
+ CAPTURE_STARTUP_NO_DATA_CALLBACK);
+ if (handler_) {
+ handler_->OnLog(this, GetDataIsActive() ?
+ "AIC::FirstCheckForNoData => data is active" :
+ "AIC::FirstCheckForNoData => data is NOT active");
+ }
+ DoCheckForNoData();
+}
+
void AudioInputController::DoCheckForNoData() {
- DCHECK(message_loop_->BelongsToCurrentThread());
+ DCHECK(task_runner_->BelongsToCurrentThread());
if (!GetDataIsActive()) {
// The data-is-active marker will be false only if it has been more than
// one second since a data packet was recorded. This can happen if a
// capture device has been removed or disabled.
- handler_->OnError(this);
- return;
+ if (handler_)
+ handler_->OnError(this, NO_DATA_ERROR);
}
// Mark data as non-active. The flag will be re-enabled in OnData() each
}
void AudioInputController::OnData(AudioInputStream* stream,
- const uint8* data,
- uint32 size,
+ const AudioBus* source,
uint32 hardware_delay_bytes,
double volume) {
+ // Mark data as active to ensure that the periodic calls to
+ // DoCheckForNoData() does not report an error to the event handler.
+ SetDataIsActive(true);
+
{
base::AutoLock auto_lock(lock_);
- if (state_ != kRecording)
+ if (state_ != RECORDING)
return;
}
DVLOG_IF(6, key_pressed) << "Detected keypress.";
}
- // Mark data as active to ensure that the periodic calls to
- // DoCheckForNoData() does not report an error to the event handler.
- SetDataIsActive(true);
-
- // Use SyncSocket if we are in a low-latency mode.
- if (LowLatencyMode()) {
- sync_writer_->Write(data, size, volume, key_pressed);
+ // Use SharedMemory and SyncSocket if the client has created a SyncWriter.
+ // Used by all low-latency clients except WebSpeech.
+ if (SharedMemoryAndSyncSocketMode()) {
+ sync_writer_->Write(source, volume, key_pressed);
sync_writer_->UpdateRecordedBytes(hardware_delay_bytes);
+
+#if defined(AUDIO_POWER_MONITORING)
+ // Only do power-level measurements if an AudioPowerMonitor object has
+ // been created. Done in DoCreate() but not DoCreateForStream(), hence
+ // logging will mainly be done for WebRTC and WebSpeech clients.
+ if (!audio_level_)
+ return;
+
+ // Perform periodic audio (power) level measurements.
+ if ((base::TimeTicks::Now() - last_audio_level_log_time_).InSeconds() >
+ kPowerMonitorLogIntervalSeconds) {
+ // Wrap data into an AudioBus to match AudioPowerMonitor::Scan.
+ // TODO(henrika): remove this section when capture side uses AudioBus.
+ // See http://crbug.com/375155 for details.
+ audio_level_->Scan(*source, source->frames());
+
+ // Get current average power level and add it to the log.
+ // Possible range is given by [-inf, 0] dBFS.
+ std::pair<float, bool> result = audio_level_->ReadCurrentPowerAndClip();
+
+ // Use event handler on the audio thread to relay a message to the ARIH
+ // in content which does the actual logging on the IO thread.
+ task_runner_->PostTask(
+ FROM_HERE,
+ base::Bind(
+ &AudioInputController::DoLogAudioLevel, this, result.first));
+
+ last_audio_level_log_time_ = base::TimeTicks::Now();
+
+ // Reset the average power level (since we don't log continuously).
+ audio_level_->Reset();
+ }
+#endif
return;
}
- handler_->OnData(this, data, size);
+ // TODO(henrika): Investigate if we can avoid the extra copy here.
+ // (see http://crbug.com/249316 for details). AFAIK, this scope is only
+ // active for WebSpeech clients.
+ scoped_ptr<AudioBus> audio_data =
+ AudioBus::Create(source->channels(), source->frames());
+ source->CopyTo(audio_data.get());
+
+ // Ownership of the audio buffer will be with the callback until it is run,
+ // when ownership is passed to the callback function.
+ task_runner_->PostTask(
+ FROM_HERE,
+ base::Bind(
+ &AudioInputController::DoOnData, this, base::Passed(&audio_data)));
}
-void AudioInputController::OnClose(AudioInputStream* stream) {
- DVLOG(1) << "AudioInputController::OnClose()";
- // TODO(satish): Sometimes the device driver closes the input stream without
- // us asking for it (may be if the device was unplugged?). Check how to handle
- // such cases here.
+void AudioInputController::DoOnData(scoped_ptr<AudioBus> data) {
+ DCHECK(task_runner_->BelongsToCurrentThread());
+ if (handler_)
+ handler_->OnData(this, data.get());
+}
+
+void AudioInputController::DoLogAudioLevel(float level_dbfs) {
+#if defined(AUDIO_POWER_MONITORING)
+ DCHECK(task_runner_->BelongsToCurrentThread());
+ if (!handler_)
+ return;
+
+ std::string log_string = base::StringPrintf(
+ "AIC::OnData: average audio level=%.2f dBFS", level_dbfs);
+ static const float kSilenceThresholdDBFS = -72.24719896f;
+ if (level_dbfs < kSilenceThresholdDBFS)
+ log_string += " <=> no audio input!";
+ handler_->OnLog(this, log_string);
+
+ UpdateSilenceState(level_dbfs < kSilenceThresholdDBFS);
+#endif
}
void AudioInputController::OnError(AudioInputStream* stream) {
// Handle error on the audio-manager thread.
- message_loop_->PostTask(FROM_HERE, base::Bind(
+ task_runner_->PostTask(FROM_HERE, base::Bind(
&AudioInputController::DoReportError, this));
}
-void AudioInputController::DoStopCloseAndClearStream(
- base::WaitableEvent* done) {
- DCHECK(message_loop_->BelongsToCurrentThread());
+void AudioInputController::DoStopCloseAndClearStream() {
+ DCHECK(task_runner_->BelongsToCurrentThread());
// Allow calling unconditionally and bail if we don't have a stream to close.
if (stream_ != NULL) {
stream_ = NULL;
}
- // Should be last in the method, do not touch "this" from here on.
- if (done != NULL)
- done->Signal();
+ // The event handler should not be touched after the stream has been closed.
+ handler_ = NULL;
}
void AudioInputController::SetDataIsActive(bool enabled) {
return (base::subtle::Acquire_Load(&data_is_active_) != false);
}
+#if defined(AUDIO_POWER_MONITORING)
+void AudioInputController::UpdateSilenceState(bool silence) {
+ if (silence) {
+ if (silence_state_ == SILENCE_STATE_NO_MEASUREMENT) {
+ silence_state_ = SILENCE_STATE_ONLY_SILENCE;
+ } else if (silence_state_ == SILENCE_STATE_ONLY_AUDIO) {
+ silence_state_ = SILENCE_STATE_AUDIO_AND_SILENCE;
+ } else {
+ DCHECK(silence_state_ == SILENCE_STATE_ONLY_SILENCE ||
+ silence_state_ == SILENCE_STATE_AUDIO_AND_SILENCE);
+ }
+ } else {
+ if (silence_state_ == SILENCE_STATE_NO_MEASUREMENT) {
+ silence_state_ = SILENCE_STATE_ONLY_AUDIO;
+ } else if (silence_state_ == SILENCE_STATE_ONLY_SILENCE) {
+ silence_state_ = SILENCE_STATE_AUDIO_AND_SILENCE;
+ } else {
+ DCHECK(silence_state_ == SILENCE_STATE_ONLY_AUDIO ||
+ silence_state_ == SILENCE_STATE_AUDIO_AND_SILENCE);
+ }
+ }
+}
+
+void AudioInputController::LogSilenceState(SilenceState value) {
+ UMA_HISTOGRAM_ENUMERATION("Media.AudioInputControllerSessionSilenceReport",
+ value,
+ SILENCE_STATE_MAX + 1);
+}
+#endif
+
} // namespace media
projects / platform / framework / web / crosswalk.git / blobdiff