#include "media/audio/audio_input_controller.h"
#include "base/bind.h"
+#include "base/strings/stringprintf.h"
#include "base/threading/thread_restrictions.h"
+#include "base/time/time.h"
+#include "media/audio/audio_parameters.h"
#include "media/base/limits.h"
#include "media/base/scoped_histogram_timer.h"
#include "media/base/user_input_monitor.h"
+using base::TimeDelta;
+
namespace {
const int kMaxInputChannels = 3;
// Mac devices and the initial timer interval has therefore been increased
// from 1 second to 5 seconds.
const int kTimerInitialIntervalSeconds = 5;
+
+#if defined(AUDIO_POWER_MONITORING)
+// Time constant for AudioPowerMonitor.
+// The utilized smoothing factor (alpha) in the exponential filter is given
+// by 1-exp(-1/(fs*ts)), where fs is the sample rate in Hz and ts is the time
+// constant given by |kPowerMeasurementTimeConstantMilliseconds|.
+// Example: fs=44100, ts=10e-3 => alpha~0.022420
+// fs=44100, ts=20e-3 => alpha~0.165903
+// A large smoothing factor corresponds to a faster filter response to input
+// changes since y(n)=alpha*x(n)+(1-alpha)*y(n-1), where x(n) is the input
+// and y(n) is the output.
+const int kPowerMeasurementTimeConstantMilliseconds = 10;
+
+// Time in seconds between two successive measurements of audio power levels.
+const int kPowerMonitorLogIntervalSeconds = 5;
+#endif
+}
+
+// Used to log the result of capture startup.
+// This was previously logged as a boolean with only the no callback and OK
+// options. The enum order is kept to ensure backwards compatibility.
+// Elements in this enum should not be deleted or rearranged; the only
+// permitted operation is to add new elements before CAPTURE_STARTUP_RESULT_MAX
+// and update CAPTURE_STARTUP_RESULT_MAX.
+enum CaptureStartupResult {
+ CAPTURE_STARTUP_NO_DATA_CALLBACK = 0,
+ CAPTURE_STARTUP_OK = 1,
+ CAPTURE_STARTUP_CREATE_STREAM_FAILED = 2,
+ CAPTURE_STARTUP_OPEN_STREAM_FAILED = 3,
+ CAPTURE_STARTUP_RESULT_MAX = CAPTURE_STARTUP_OPEN_STREAM_FAILED
+};
+
+void LogCaptureStartupResult(CaptureStartupResult result) {
+ UMA_HISTOGRAM_ENUMERATION("Media.AudioInputControllerCaptureStartupSuccess",
+ result,
+ CAPTURE_STARTUP_RESULT_MAX + 1);
+
}
namespace media {
sync_writer_(sync_writer),
max_volume_(0.0),
user_input_monitor_(user_input_monitor),
+#if defined(AUDIO_POWER_MONITORING)
+ log_silence_state_(false),
+ silence_state_(SILENCE_STATE_NO_MEASUREMENT),
+#endif
prev_key_down_count_(0) {
DCHECK(creator_task_runner_.get());
}
// Create and open a new audio input stream from the existing
// audio-device thread.
- if (!controller->task_runner_->PostTask(FROM_HERE,
- base::Bind(&AudioInputController::DoCreate, controller,
- base::Unretained(audio_manager), params, device_id))) {
+ if (!controller->task_runner_->PostTask(
+ FROM_HERE,
+ base::Bind(&AudioInputController::DoCreate,
+ controller,
+ base::Unretained(audio_manager),
+ params,
+ device_id))) {
controller = NULL;
}
// Create and open a new audio input stream from the existing
// audio-device thread. Use the provided audio-input device.
- if (!controller->task_runner_->PostTask(FROM_HERE,
- base::Bind(&AudioInputController::DoCreate, controller,
- base::Unretained(audio_manager), params, device_id))) {
+ if (!controller->task_runner_->PostTask(
+ FROM_HERE,
+ base::Bind(&AudioInputController::DoCreateForLowLatency,
+ controller,
+ base::Unretained(audio_manager),
+ params,
+ device_id))) {
controller = NULL;
}
// mirroring use case only.
if (!controller->task_runner_->PostTask(
FROM_HERE,
- base::Bind(&AudioInputController::DoCreateForStream, controller,
- stream, false))) {
+ base::Bind(&AudioInputController::DoCreateForStream,
+ controller,
+ stream))) {
controller = NULL;
}
const std::string& device_id) {
DCHECK(task_runner_->BelongsToCurrentThread());
SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.CreateTime");
+ if (handler_)
+ handler_->OnLog(this, "AIC::DoCreate");
+
+#if defined(AUDIO_POWER_MONITORING)
+ // Create the audio (power) level meter given the provided audio parameters.
+ // An AudioBus is also needed to wrap the raw data buffer from the native
+ // layer to match AudioPowerMonitor::Scan().
+ // TODO(henrika): Remove use of extra AudioBus. See http://crbug.com/375155.
+ last_audio_level_log_time_ = base::TimeTicks::Now();
+ audio_level_.reset(new media::AudioPowerMonitor(
+ params.sample_rate(),
+ TimeDelta::FromMilliseconds(kPowerMeasurementTimeConstantMilliseconds)));
+ audio_params_ = params;
+ silence_state_ = SILENCE_STATE_NO_MEASUREMENT;
+#endif
+
// TODO(miu): See TODO at top of file. Until that's resolved, assume all
// platform audio input requires the |no_data_timer_| be used to auto-detect
// errors. In reality, probably only Windows needs to be treated as
// unreliable here.
- DoCreateForStream(audio_manager->MakeAudioInputStream(params, device_id),
- true);
+ DoCreateForStream(audio_manager->MakeAudioInputStream(params, device_id));
+}
+
+void AudioInputController::DoCreateForLowLatency(AudioManager* audio_manager,
+ const AudioParameters& params,
+ const std::string& device_id) {
+ DCHECK(task_runner_->BelongsToCurrentThread());
+
+#if defined(AUDIO_POWER_MONITORING)
+ // We only log silence state UMA stats for low latency mode and if we use a
+ // real device.
+ if (params.format() != AudioParameters::AUDIO_FAKE)
+ log_silence_state_ = true;
+#endif
+
+ DoCreate(audio_manager, params, device_id);
}
void AudioInputController::DoCreateForStream(
- AudioInputStream* stream_to_control, bool enable_nodata_timer) {
+ AudioInputStream* stream_to_control) {
DCHECK(task_runner_->BelongsToCurrentThread());
DCHECK(!stream_);
if (!stream_) {
if (handler_)
handler_->OnError(this, STREAM_CREATE_ERROR);
+ LogCaptureStartupResult(CAPTURE_STARTUP_CREATE_STREAM_FAILED);
return;
}
stream_ = NULL;
if (handler_)
handler_->OnError(this, STREAM_OPEN_ERROR);
+ LogCaptureStartupResult(CAPTURE_STARTUP_OPEN_STREAM_FAILED);
return;
}
DCHECK(!no_data_timer_.get());
+ // Create the data timer which will call FirstCheckForNoData(). The timer
+ // is started in DoRecord() and restarted in each DoCheckForNoData()
+ // callback.
// The timer is enabled for logging purposes. The NO_DATA_ERROR triggered
// from the timer must be ignored by the EventHandler.
// TODO(henrika): remove usage of timer when it has been verified on Canary
// that we are safe doing so. Goal is to get rid of |no_data_timer_| and
// everything that is tied to it. crbug.com/357569.
- enable_nodata_timer = true;
-
- if (enable_nodata_timer) {
- // Create the data timer which will call DoCheckForNoData(). The timer
- // is started in DoRecord() and restarted in each DoCheckForNoData()
- // callback.
- no_data_timer_.reset(new base::Timer(
- FROM_HERE, base::TimeDelta::FromSeconds(kTimerInitialIntervalSeconds),
- base::Bind(&AudioInputController::DoCheckForNoData,
- base::Unretained(this)), false));
- } else {
- DVLOG(1) << "Disabled: timer check for no data.";
- }
+ no_data_timer_.reset(new base::Timer(
+ FROM_HERE, base::TimeDelta::FromSeconds(kTimerInitialIntervalSeconds),
+ base::Bind(&AudioInputController::FirstCheckForNoData,
+ base::Unretained(this)), false));
state_ = CREATED;
if (handler_)
state_ = RECORDING;
}
+ if (handler_)
+ handler_->OnLog(this, "AIC::DoRecord");
+
if (no_data_timer_) {
// Start the data timer. Once |kTimerResetIntervalSeconds| have passed,
- // a callback to DoCheckForNoData() is made.
+ // a callback to FirstCheckForNoData() is made.
no_data_timer_->Reset();
}
if (state_ == CLOSED)
return;
+ if (handler_)
+ handler_->OnLog(this, "AIC::DoClose");
+
// Delete the timer on the same thread that created it.
no_data_timer_.reset();
if (user_input_monitor_)
user_input_monitor_->DisableKeyPressMonitoring();
+#if defined(AUDIO_POWER_MONITORING)
+ // Send UMA stats if enabled.
+ if (log_silence_state_)
+ LogSilenceState(silence_state_);
+ log_silence_state_ = false;
+#endif
+
state_ = CLOSED;
}
stream_->SetAutomaticGainControl(enabled);
}
+void AudioInputController::FirstCheckForNoData() {
+ DCHECK(task_runner_->BelongsToCurrentThread());
+ LogCaptureStartupResult(GetDataIsActive() ?
+ CAPTURE_STARTUP_OK :
+ CAPTURE_STARTUP_NO_DATA_CALLBACK);
+ if (handler_) {
+ handler_->OnLog(this, GetDataIsActive() ?
+ "AIC::FirstCheckForNoData => data is active" :
+ "AIC::FirstCheckForNoData => data is NOT active");
+ }
+ DoCheckForNoData();
+}
+
void AudioInputController::DoCheckForNoData() {
DCHECK(task_runner_->BelongsToCurrentThread());
}
void AudioInputController::OnData(AudioInputStream* stream,
- const uint8* data,
- uint32 size,
+ const AudioBus* source,
uint32 hardware_delay_bytes,
double volume) {
+ // Mark data as active to ensure that the periodic calls to
+ // DoCheckForNoData() does not report an error to the event handler.
+ SetDataIsActive(true);
+
{
base::AutoLock auto_lock(lock_);
if (state_ != RECORDING)
DVLOG_IF(6, key_pressed) << "Detected keypress.";
}
- // Mark data as active to ensure that the periodic calls to
- // DoCheckForNoData() does not report an error to the event handler.
- SetDataIsActive(true);
-
// Use SharedMemory and SyncSocket if the client has created a SyncWriter.
// Used by all low-latency clients except WebSpeech.
if (SharedMemoryAndSyncSocketMode()) {
- sync_writer_->Write(data, size, volume, key_pressed);
+ sync_writer_->Write(source, volume, key_pressed);
sync_writer_->UpdateRecordedBytes(hardware_delay_bytes);
+
+#if defined(AUDIO_POWER_MONITORING)
+ // Only do power-level measurements if an AudioPowerMonitor object has
+ // been created. Done in DoCreate() but not DoCreateForStream(), hence
+ // logging will mainly be done for WebRTC and WebSpeech clients.
+ if (!audio_level_)
+ return;
+
+ // Perform periodic audio (power) level measurements.
+ if ((base::TimeTicks::Now() - last_audio_level_log_time_).InSeconds() >
+ kPowerMonitorLogIntervalSeconds) {
+ // Wrap data into an AudioBus to match AudioPowerMonitor::Scan.
+ // TODO(henrika): remove this section when capture side uses AudioBus.
+ // See http://crbug.com/375155 for details.
+ audio_level_->Scan(*source, source->frames());
+
+ // Get current average power level and add it to the log.
+ // Possible range is given by [-inf, 0] dBFS.
+ std::pair<float, bool> result = audio_level_->ReadCurrentPowerAndClip();
+
+ // Use event handler on the audio thread to relay a message to the ARIH
+ // in content which does the actual logging on the IO thread.
+ task_runner_->PostTask(
+ FROM_HERE,
+ base::Bind(
+ &AudioInputController::DoLogAudioLevel, this, result.first));
+
+ last_audio_level_log_time_ = base::TimeTicks::Now();
+
+ // Reset the average power level (since we don't log continuously).
+ audio_level_->Reset();
+ }
+#endif
return;
}
// TODO(henrika): Investigate if we can avoid the extra copy here.
// (see http://crbug.com/249316 for details). AFAIK, this scope is only
// active for WebSpeech clients.
- scoped_ptr<uint8[]> audio_data(new uint8[size]);
- memcpy(audio_data.get(), data, size);
+ scoped_ptr<AudioBus> audio_data =
+ AudioBus::Create(source->channels(), source->frames());
+ source->CopyTo(audio_data.get());
// Ownership of the audio buffer will be with the callback until it is run,
// when ownership is passed to the callback function.
- task_runner_->PostTask(FROM_HERE, base::Bind(
- &AudioInputController::DoOnData, this, base::Passed(&audio_data), size));
+ task_runner_->PostTask(
+ FROM_HERE,
+ base::Bind(
+ &AudioInputController::DoOnData, this, base::Passed(&audio_data)));
}
-void AudioInputController::DoOnData(scoped_ptr<uint8[]> data, uint32 size) {
+void AudioInputController::DoOnData(scoped_ptr<AudioBus> data) {
DCHECK(task_runner_->BelongsToCurrentThread());
if (handler_)
- handler_->OnData(this, data.get(), size);
+ handler_->OnData(this, data.get());
+}
+
+void AudioInputController::DoLogAudioLevel(float level_dbfs) {
+#if defined(AUDIO_POWER_MONITORING)
+ DCHECK(task_runner_->BelongsToCurrentThread());
+ if (!handler_)
+ return;
+
+ std::string log_string = base::StringPrintf(
+ "AIC::OnData: average audio level=%.2f dBFS", level_dbfs);
+ static const float kSilenceThresholdDBFS = -72.24719896f;
+ if (level_dbfs < kSilenceThresholdDBFS)
+ log_string += " <=> no audio input!";
+ handler_->OnLog(this, log_string);
+
+ UpdateSilenceState(level_dbfs < kSilenceThresholdDBFS);
+#endif
}
void AudioInputController::OnError(AudioInputStream* stream) {
return (base::subtle::Acquire_Load(&data_is_active_) != false);
}
+#if defined(AUDIO_POWER_MONITORING)
+void AudioInputController::UpdateSilenceState(bool silence) {
+ if (silence) {
+ if (silence_state_ == SILENCE_STATE_NO_MEASUREMENT) {
+ silence_state_ = SILENCE_STATE_ONLY_SILENCE;
+ } else if (silence_state_ == SILENCE_STATE_ONLY_AUDIO) {
+ silence_state_ = SILENCE_STATE_AUDIO_AND_SILENCE;
+ } else {
+ DCHECK(silence_state_ == SILENCE_STATE_ONLY_SILENCE ||
+ silence_state_ == SILENCE_STATE_AUDIO_AND_SILENCE);
+ }
+ } else {
+ if (silence_state_ == SILENCE_STATE_NO_MEASUREMENT) {
+ silence_state_ = SILENCE_STATE_ONLY_AUDIO;
+ } else if (silence_state_ == SILENCE_STATE_ONLY_SILENCE) {
+ silence_state_ = SILENCE_STATE_AUDIO_AND_SILENCE;
+ } else {
+ DCHECK(silence_state_ == SILENCE_STATE_ONLY_AUDIO ||
+ silence_state_ == SILENCE_STATE_AUDIO_AND_SILENCE);
+ }
+ }
+}
+
+void AudioInputController::LogSilenceState(SilenceState value) {
+ UMA_HISTOGRAM_ENUMERATION("Media.AudioInputControllerSessionSilenceReport",
+ value,
+ SILENCE_STATE_MAX + 1);
+}
+#endif
+
} // namespace media
projects / platform / framework / web / crosswalk.git / blobdiff