e98ead8afc4a97458f178af6dd03804b0389d60f
[platform/framework/web/crosswalk.git] / src / media / audio / audio_input_controller.cc
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "media/audio/audio_input_controller.h"
6
7 #include "base/bind.h"
8 #include "base/strings/stringprintf.h"
9 #include "base/threading/thread_restrictions.h"
10 #include "base/time/time.h"
11 #include "media/audio/audio_parameters.h"
12 #include "media/base/limits.h"
13 #include "media/base/scoped_histogram_timer.h"
14 #include "media/base/user_input_monitor.h"
15
16 using base::TimeDelta;
17
18 namespace {
19 const int kMaxInputChannels = 3;
20
21 // TODO(henrika): remove usage of timers and add support for proper
22 // notification of when the input device is removed.  This was originally added
23 // to resolve http://crbug.com/79936 for Windows platforms.  This then caused
24 // breakage (very hard to repro bugs!) on other platforms: See
25 // http://crbug.com/226327 and http://crbug.com/230972.
26 // See also that the timer has been disabled on Mac now due to
27 // crbug.com/357501.
28 const int kTimerResetIntervalSeconds = 1;
29 // We have received reports that the timer can be too trigger happy on some
30 // Mac devices and the initial timer interval has therefore been increased
31 // from 1 second to 5 seconds.
32 const int kTimerInitialIntervalSeconds = 5;
33
34 #if defined(AUDIO_POWER_MONITORING)
35 // Time constant for AudioPowerMonitor.
36 // The utilized smoothing factor (alpha) in the exponential filter is given
37 // by 1-exp(-1/(fs*ts)), where fs is the sample rate in Hz and ts is the time
38 // constant given by |kPowerMeasurementTimeConstantMilliseconds|.
39 // Example: fs=44100, ts=10e-3 => alpha~0.022420
40 //          fs=44100, ts=20e-3 => alpha~0.165903
41 // A large smoothing factor corresponds to a faster filter response to input
42 // changes since y(n)=alpha*x(n)+(1-alpha)*y(n-1), where x(n) is the input
43 // and y(n) is the output.
44 const int kPowerMeasurementTimeConstantMilliseconds = 10;
45
46 // Time in seconds between two successive measurements of audio power levels.
47 const int kPowerMonitorLogIntervalSeconds = 15;
48
49 // A warning will be logged when the microphone audio volume is below this
50 // threshold.
51 const int kLowLevelMicrophoneLevelPercent = 10;
52 #endif
53 }
54
55 // Used to log the result of capture startup.
56 // This was previously logged as a boolean with only the no callback and OK
57 // options. The enum order is kept to ensure backwards compatibility.
58 // Elements in this enum should not be deleted or rearranged; the only
59 // permitted operation is to add new elements before CAPTURE_STARTUP_RESULT_MAX
60 // and update CAPTURE_STARTUP_RESULT_MAX.
61 enum CaptureStartupResult {
62   CAPTURE_STARTUP_NO_DATA_CALLBACK = 0,
63   CAPTURE_STARTUP_OK = 1,
64   CAPTURE_STARTUP_CREATE_STREAM_FAILED = 2,
65   CAPTURE_STARTUP_OPEN_STREAM_FAILED = 3,
66   CAPTURE_STARTUP_RESULT_MAX = CAPTURE_STARTUP_OPEN_STREAM_FAILED
67 };
68
69 void LogCaptureStartupResult(CaptureStartupResult result) {
70   UMA_HISTOGRAM_ENUMERATION("Media.AudioInputControllerCaptureStartupSuccess",
71                             result,
72                             CAPTURE_STARTUP_RESULT_MAX + 1);
73
74 }
75
76 namespace media {
77
78 // static
79 AudioInputController::Factory* AudioInputController::factory_ = NULL;
80
81 AudioInputController::AudioInputController(EventHandler* handler,
82                                            SyncWriter* sync_writer,
83                                            UserInputMonitor* user_input_monitor)
84     : creator_task_runner_(base::MessageLoopProxy::current()),
85       handler_(handler),
86       stream_(NULL),
87       data_is_active_(false),
88       state_(CLOSED),
89       sync_writer_(sync_writer),
90       max_volume_(0.0),
91       user_input_monitor_(user_input_monitor),
92 #if defined(AUDIO_POWER_MONITORING)
93       log_silence_state_(false),
94       silence_state_(SILENCE_STATE_NO_MEASUREMENT),
95 #endif
96       prev_key_down_count_(0) {
97   DCHECK(creator_task_runner_.get());
98 }
99
100 AudioInputController::~AudioInputController() {
101   DCHECK_EQ(state_, CLOSED);
102 }
103
104 // static
105 scoped_refptr<AudioInputController> AudioInputController::Create(
106     AudioManager* audio_manager,
107     EventHandler* event_handler,
108     const AudioParameters& params,
109     const std::string& device_id,
110     UserInputMonitor* user_input_monitor) {
111   DCHECK(audio_manager);
112
113   if (!params.IsValid() || (params.channels() > kMaxInputChannels))
114     return NULL;
115
116   if (factory_) {
117     return factory_->Create(
118         audio_manager, event_handler, params, user_input_monitor);
119   }
120   scoped_refptr<AudioInputController> controller(
121       new AudioInputController(event_handler, NULL, user_input_monitor));
122
123   controller->task_runner_ = audio_manager->GetTaskRunner();
124
125   // Create and open a new audio input stream from the existing
126   // audio-device thread.
127   if (!controller->task_runner_->PostTask(
128           FROM_HERE,
129           base::Bind(&AudioInputController::DoCreate,
130                      controller,
131                      base::Unretained(audio_manager),
132                      params,
133                      device_id))) {
134     controller = NULL;
135   }
136
137   return controller;
138 }
139
140 // static
141 scoped_refptr<AudioInputController> AudioInputController::CreateLowLatency(
142     AudioManager* audio_manager,
143     EventHandler* event_handler,
144     const AudioParameters& params,
145     const std::string& device_id,
146     SyncWriter* sync_writer,
147     UserInputMonitor* user_input_monitor) {
148   DCHECK(audio_manager);
149   DCHECK(sync_writer);
150
151   if (!params.IsValid() || (params.channels() > kMaxInputChannels))
152     return NULL;
153
154   // Create the AudioInputController object and ensure that it runs on
155   // the audio-manager thread.
156   scoped_refptr<AudioInputController> controller(
157       new AudioInputController(event_handler, sync_writer, user_input_monitor));
158   controller->task_runner_ = audio_manager->GetTaskRunner();
159
160   // Create and open a new audio input stream from the existing
161   // audio-device thread. Use the provided audio-input device.
162   if (!controller->task_runner_->PostTask(
163           FROM_HERE,
164           base::Bind(&AudioInputController::DoCreateForLowLatency,
165                      controller,
166                      base::Unretained(audio_manager),
167                      params,
168                      device_id))) {
169     controller = NULL;
170   }
171
172   return controller;
173 }
174
175 // static
176 scoped_refptr<AudioInputController> AudioInputController::CreateForStream(
177     const scoped_refptr<base::SingleThreadTaskRunner>& task_runner,
178     EventHandler* event_handler,
179     AudioInputStream* stream,
180     SyncWriter* sync_writer,
181     UserInputMonitor* user_input_monitor) {
182   DCHECK(sync_writer);
183   DCHECK(stream);
184
185   // Create the AudioInputController object and ensure that it runs on
186   // the audio-manager thread.
187   scoped_refptr<AudioInputController> controller(
188       new AudioInputController(event_handler, sync_writer, user_input_monitor));
189   controller->task_runner_ = task_runner;
190
191   // TODO(miu): See TODO at top of file.  Until that's resolved, we need to
192   // disable the error auto-detection here (since the audio mirroring
193   // implementation will reliably report error and close events).  Note, of
194   // course, that we're assuming CreateForStream() has been called for the audio
195   // mirroring use case only.
196   if (!controller->task_runner_->PostTask(
197           FROM_HERE,
198           base::Bind(&AudioInputController::DoCreateForStream,
199                      controller,
200                      stream))) {
201     controller = NULL;
202   }
203
204   return controller;
205 }
206
207 void AudioInputController::Record() {
208   task_runner_->PostTask(FROM_HERE, base::Bind(
209       &AudioInputController::DoRecord, this));
210 }
211
212 void AudioInputController::Close(const base::Closure& closed_task) {
213   DCHECK(!closed_task.is_null());
214   DCHECK(creator_task_runner_->BelongsToCurrentThread());
215
216   task_runner_->PostTaskAndReply(
217       FROM_HERE, base::Bind(&AudioInputController::DoClose, this), closed_task);
218 }
219
220 void AudioInputController::SetVolume(double volume) {
221   task_runner_->PostTask(FROM_HERE, base::Bind(
222       &AudioInputController::DoSetVolume, this, volume));
223 }
224
225 void AudioInputController::SetAutomaticGainControl(bool enabled) {
226   task_runner_->PostTask(FROM_HERE, base::Bind(
227       &AudioInputController::DoSetAutomaticGainControl, this, enabled));
228 }
229
230 void AudioInputController::DoCreate(AudioManager* audio_manager,
231                                     const AudioParameters& params,
232                                     const std::string& device_id) {
233   DCHECK(task_runner_->BelongsToCurrentThread());
234   SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.CreateTime");
235   if (handler_)
236     handler_->OnLog(this, "AIC::DoCreate");
237
238 #if defined(AUDIO_POWER_MONITORING)
239   // Create the audio (power) level meter given the provided audio parameters.
240   // An AudioBus is also needed to wrap the raw data buffer from the native
241   // layer to match AudioPowerMonitor::Scan().
242   // TODO(henrika): Remove use of extra AudioBus. See http://crbug.com/375155.
243   last_audio_level_log_time_ = base::TimeTicks::Now();
244   audio_level_.reset(new media::AudioPowerMonitor(
245       params.sample_rate(),
246       TimeDelta::FromMilliseconds(kPowerMeasurementTimeConstantMilliseconds)));
247   audio_params_ = params;
248   silence_state_ = SILENCE_STATE_NO_MEASUREMENT;
249 #endif
250
251   // TODO(miu): See TODO at top of file.  Until that's resolved, assume all
252   // platform audio input requires the |no_data_timer_| be used to auto-detect
253   // errors.  In reality, probably only Windows needs to be treated as
254   // unreliable here.
255   DoCreateForStream(audio_manager->MakeAudioInputStream(params, device_id));
256 }
257
258 void AudioInputController::DoCreateForLowLatency(AudioManager* audio_manager,
259                                                  const AudioParameters& params,
260                                                  const std::string& device_id) {
261   DCHECK(task_runner_->BelongsToCurrentThread());
262
263 #if defined(AUDIO_POWER_MONITORING)
264   // We only log silence state UMA stats for low latency mode and if we use a
265   // real device.
266   if (params.format() != AudioParameters::AUDIO_FAKE)
267     log_silence_state_ = true;
268 #endif
269
270   DoCreate(audio_manager, params, device_id);
271 }
272
273 void AudioInputController::DoCreateForStream(
274     AudioInputStream* stream_to_control) {
275   DCHECK(task_runner_->BelongsToCurrentThread());
276
277   DCHECK(!stream_);
278   stream_ = stream_to_control;
279
280   if (!stream_) {
281     if (handler_)
282       handler_->OnError(this, STREAM_CREATE_ERROR);
283     LogCaptureStartupResult(CAPTURE_STARTUP_CREATE_STREAM_FAILED);
284     return;
285   }
286
287   if (stream_ && !stream_->Open()) {
288     stream_->Close();
289     stream_ = NULL;
290     if (handler_)
291       handler_->OnError(this, STREAM_OPEN_ERROR);
292     LogCaptureStartupResult(CAPTURE_STARTUP_OPEN_STREAM_FAILED);
293     return;
294   }
295
296   DCHECK(!no_data_timer_.get());
297
298   // Create the data timer which will call FirstCheckForNoData(). The timer
299   // is started in DoRecord() and restarted in each DoCheckForNoData()
300   // callback.
301   // The timer is enabled for logging purposes. The NO_DATA_ERROR triggered
302   // from the timer must be ignored by the EventHandler.
303   // TODO(henrika): remove usage of timer when it has been verified on Canary
304   // that we are safe doing so. Goal is to get rid of |no_data_timer_| and
305   // everything that is tied to it. crbug.com/357569.
306   no_data_timer_.reset(new base::Timer(
307       FROM_HERE, base::TimeDelta::FromSeconds(kTimerInitialIntervalSeconds),
308       base::Bind(&AudioInputController::FirstCheckForNoData,
309                  base::Unretained(this)), false));
310
311   state_ = CREATED;
312   if (handler_)
313     handler_->OnCreated(this);
314
315   if (user_input_monitor_) {
316     user_input_monitor_->EnableKeyPressMonitoring();
317     prev_key_down_count_ = user_input_monitor_->GetKeyPressCount();
318   }
319 }
320
321 void AudioInputController::DoRecord() {
322   DCHECK(task_runner_->BelongsToCurrentThread());
323   SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.RecordTime");
324
325   if (state_ != CREATED)
326     return;
327
328   {
329     base::AutoLock auto_lock(lock_);
330     state_ = RECORDING;
331   }
332
333   if (handler_)
334     handler_->OnLog(this, "AIC::DoRecord");
335
336   if (no_data_timer_) {
337     // Start the data timer. Once |kTimerResetIntervalSeconds| have passed,
338     // a callback to FirstCheckForNoData() is made.
339     no_data_timer_->Reset();
340   }
341
342   stream_->Start(this);
343   if (handler_)
344     handler_->OnRecording(this);
345 }
346
347 void AudioInputController::DoClose() {
348   DCHECK(task_runner_->BelongsToCurrentThread());
349   SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.CloseTime");
350
351   if (state_ == CLOSED)
352     return;
353
354   if (handler_)
355     handler_->OnLog(this, "AIC::DoClose");
356
357   // Delete the timer on the same thread that created it.
358   no_data_timer_.reset();
359
360   DoStopCloseAndClearStream();
361   SetDataIsActive(false);
362
363   if (SharedMemoryAndSyncSocketMode())
364     sync_writer_->Close();
365
366   if (user_input_monitor_)
367     user_input_monitor_->DisableKeyPressMonitoring();
368
369 #if defined(AUDIO_POWER_MONITORING)
370   // Send UMA stats if enabled.
371   if (log_silence_state_)
372     LogSilenceState(silence_state_);
373   log_silence_state_ = false;
374 #endif
375
376   state_ = CLOSED;
377 }
378
379 void AudioInputController::DoReportError() {
380   DCHECK(task_runner_->BelongsToCurrentThread());
381   if (handler_)
382     handler_->OnError(this, STREAM_ERROR);
383 }
384
385 void AudioInputController::DoSetVolume(double volume) {
386   DCHECK(task_runner_->BelongsToCurrentThread());
387   DCHECK_GE(volume, 0);
388   DCHECK_LE(volume, 1.0);
389
390   if (state_ != CREATED && state_ != RECORDING)
391     return;
392
393   // Only ask for the maximum volume at first call and use cached value
394   // for remaining function calls.
395   if (!max_volume_) {
396     max_volume_ = stream_->GetMaxVolume();
397   }
398
399   if (max_volume_ == 0.0) {
400     DLOG(WARNING) << "Failed to access input volume control";
401     return;
402   }
403
404   // Set the stream volume and scale to a range matched to the platform.
405   stream_->SetVolume(max_volume_ * volume);
406 }
407
408 void AudioInputController::DoSetAutomaticGainControl(bool enabled) {
409   DCHECK(task_runner_->BelongsToCurrentThread());
410   DCHECK_NE(state_, RECORDING);
411
412   // Ensure that the AGC state only can be modified before streaming starts.
413   if (state_ != CREATED)
414     return;
415
416   stream_->SetAutomaticGainControl(enabled);
417 }
418
419 void AudioInputController::FirstCheckForNoData() {
420   DCHECK(task_runner_->BelongsToCurrentThread());
421   LogCaptureStartupResult(GetDataIsActive() ?
422                           CAPTURE_STARTUP_OK :
423                           CAPTURE_STARTUP_NO_DATA_CALLBACK);
424   if (handler_) {
425     handler_->OnLog(this, GetDataIsActive() ?
426                     "AIC::FirstCheckForNoData => data is active" :
427                     "AIC::FirstCheckForNoData => data is NOT active");
428   }
429   DoCheckForNoData();
430 }
431
432 void AudioInputController::DoCheckForNoData() {
433   DCHECK(task_runner_->BelongsToCurrentThread());
434
435   if (!GetDataIsActive()) {
436     // The data-is-active marker will be false only if it has been more than
437     // one second since a data packet was recorded. This can happen if a
438     // capture device has been removed or disabled.
439     if (handler_)
440       handler_->OnError(this, NO_DATA_ERROR);
441   }
442
443   // Mark data as non-active. The flag will be re-enabled in OnData() each
444   // time a data packet is received. Hence, under normal conditions, the
445   // flag will only be disabled during a very short period.
446   SetDataIsActive(false);
447
448   // Restart the timer to ensure that we check the flag again in
449   // |kTimerResetIntervalSeconds|.
450   no_data_timer_->Start(
451       FROM_HERE, base::TimeDelta::FromSeconds(kTimerResetIntervalSeconds),
452       base::Bind(&AudioInputController::DoCheckForNoData,
453       base::Unretained(this)));
454 }
455
456 void AudioInputController::OnData(AudioInputStream* stream,
457                                   const AudioBus* source,
458                                   uint32 hardware_delay_bytes,
459                                   double volume) {
460   // Mark data as active to ensure that the periodic calls to
461   // DoCheckForNoData() does not report an error to the event handler.
462   SetDataIsActive(true);
463
464   {
465     base::AutoLock auto_lock(lock_);
466     if (state_ != RECORDING)
467       return;
468   }
469
470   bool key_pressed = false;
471   if (user_input_monitor_) {
472     size_t current_count = user_input_monitor_->GetKeyPressCount();
473     key_pressed = current_count != prev_key_down_count_;
474     prev_key_down_count_ = current_count;
475     DVLOG_IF(6, key_pressed) << "Detected keypress.";
476   }
477
478   // Use SharedMemory and SyncSocket if the client has created a SyncWriter.
479   // Used by all low-latency clients except WebSpeech.
480   if (SharedMemoryAndSyncSocketMode()) {
481     sync_writer_->Write(source, volume, key_pressed);
482     sync_writer_->UpdateRecordedBytes(hardware_delay_bytes);
483
484 #if defined(AUDIO_POWER_MONITORING)
485     // Only do power-level measurements if an AudioPowerMonitor object has
486     // been created. Done in DoCreate() but not DoCreateForStream(), hence
487     // logging will mainly be done for WebRTC and WebSpeech clients.
488     if (!audio_level_)
489       return;
490
491     // Perform periodic audio (power) level measurements.
492     if ((base::TimeTicks::Now() - last_audio_level_log_time_).InSeconds() >
493         kPowerMonitorLogIntervalSeconds) {
494       // Wrap data into an AudioBus to match AudioPowerMonitor::Scan.
495       // TODO(henrika): remove this section when capture side uses AudioBus.
496       // See http://crbug.com/375155 for details.
497       audio_level_->Scan(*source, source->frames());
498
499       // Get current average power level and add it to the log.
500       // Possible range is given by [-inf, 0] dBFS.
501       std::pair<float, bool> result = audio_level_->ReadCurrentPowerAndClip();
502
503       // Add current microphone volume to log and UMA histogram.
504       const int mic_volume_percent = static_cast<int>(100.0 * volume);
505
506       // Use event handler on the audio thread to relay a message to the ARIH
507       // in content which does the actual logging on the IO thread.
508       task_runner_->PostTask(FROM_HERE,
509                              base::Bind(&AudioInputController::DoLogAudioLevels,
510                                         this,
511                                         result.first,
512                                         mic_volume_percent));
513
514       last_audio_level_log_time_ = base::TimeTicks::Now();
515
516       // Reset the average power level (since we don't log continuously).
517       audio_level_->Reset();
518     }
519 #endif
520     return;
521   }
522
523   // TODO(henrika): Investigate if we can avoid the extra copy here.
524   // (see http://crbug.com/249316 for details). AFAIK, this scope is only
525   // active for WebSpeech clients.
526   scoped_ptr<AudioBus> audio_data =
527       AudioBus::Create(source->channels(), source->frames());
528   source->CopyTo(audio_data.get());
529
530   // Ownership of the audio buffer will be with the callback until it is run,
531   // when ownership is passed to the callback function.
532   task_runner_->PostTask(
533       FROM_HERE,
534       base::Bind(
535           &AudioInputController::DoOnData, this, base::Passed(&audio_data)));
536 }
537
538 void AudioInputController::DoOnData(scoped_ptr<AudioBus> data) {
539   DCHECK(task_runner_->BelongsToCurrentThread());
540   if (handler_)
541     handler_->OnData(this, data.get());
542 }
543
544 void AudioInputController::DoLogAudioLevels(float level_dbfs,
545                                             int microphone_volume_percent) {
546 #if defined(AUDIO_POWER_MONITORING)
547   DCHECK(task_runner_->BelongsToCurrentThread());
548   if (!handler_)
549     return;
550
551   std::string log_string = base::StringPrintf(
552       "AIC::OnData: average audio level=%.2f dBFS", level_dbfs);
553   static const float kSilenceThresholdDBFS = -72.24719896f;
554   if (level_dbfs < kSilenceThresholdDBFS)
555     log_string += " <=> no audio input!";
556   handler_->OnLog(this, log_string);
557
558   UpdateSilenceState(level_dbfs < kSilenceThresholdDBFS);
559
560   UMA_HISTOGRAM_PERCENTAGE("Media.MicrophoneVolume", microphone_volume_percent);
561   log_string = base::StringPrintf(
562       "AIC::OnData: microphone volume=%d%%", microphone_volume_percent);
563   if (microphone_volume_percent < kLowLevelMicrophoneLevelPercent)
564     log_string += " <=> low microphone level!";
565   handler_->OnLog(this, log_string);
566 #endif
567 }
568
569 void AudioInputController::OnError(AudioInputStream* stream) {
570   // Handle error on the audio-manager thread.
571   task_runner_->PostTask(FROM_HERE, base::Bind(
572       &AudioInputController::DoReportError, this));
573 }
574
575 void AudioInputController::DoStopCloseAndClearStream() {
576   DCHECK(task_runner_->BelongsToCurrentThread());
577
578   // Allow calling unconditionally and bail if we don't have a stream to close.
579   if (stream_ != NULL) {
580     stream_->Stop();
581     stream_->Close();
582     stream_ = NULL;
583   }
584
585   // The event handler should not be touched after the stream has been closed.
586   handler_ = NULL;
587 }
588
589 void AudioInputController::SetDataIsActive(bool enabled) {
590   base::subtle::Release_Store(&data_is_active_, enabled);
591 }
592
593 bool AudioInputController::GetDataIsActive() {
594   return (base::subtle::Acquire_Load(&data_is_active_) != false);
595 }
596
597 #if defined(AUDIO_POWER_MONITORING)
598 void AudioInputController::UpdateSilenceState(bool silence) {
599   if (silence) {
600     if (silence_state_ == SILENCE_STATE_NO_MEASUREMENT) {
601       silence_state_ = SILENCE_STATE_ONLY_SILENCE;
602     } else if (silence_state_ == SILENCE_STATE_ONLY_AUDIO) {
603       silence_state_ = SILENCE_STATE_AUDIO_AND_SILENCE;
604     } else {
605       DCHECK(silence_state_ == SILENCE_STATE_ONLY_SILENCE ||
606              silence_state_ == SILENCE_STATE_AUDIO_AND_SILENCE);
607     }
608   } else {
609     if (silence_state_ == SILENCE_STATE_NO_MEASUREMENT) {
610       silence_state_ = SILENCE_STATE_ONLY_AUDIO;
611     } else if (silence_state_ == SILENCE_STATE_ONLY_SILENCE) {
612       silence_state_ = SILENCE_STATE_AUDIO_AND_SILENCE;
613     } else {
614       DCHECK(silence_state_ == SILENCE_STATE_ONLY_AUDIO ||
615              silence_state_ == SILENCE_STATE_AUDIO_AND_SILENCE);
616     }
617   }
618 }
619
620 void AudioInputController::LogSilenceState(SilenceState value) {
621   UMA_HISTOGRAM_ENUMERATION("Media.AudioInputControllerSessionSilenceReport",
622                             value,
623                             SILENCE_STATE_MAX + 1);
624 }
625 #endif
626
627 }  // namespace media