Upstream version 10.38.208.0
[platform/framework/web/crosswalk.git] / src / media / audio / audio_input_controller.cc
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "media/audio/audio_input_controller.h"
6
7 #include "base/bind.h"
8 #include "base/strings/stringprintf.h"
9 #include "base/threading/thread_restrictions.h"
10 #include "base/time/time.h"
11 #include "media/audio/audio_parameters.h"
12 #include "media/base/limits.h"
13 #include "media/base/scoped_histogram_timer.h"
14 #include "media/base/user_input_monitor.h"
15
16 using base::TimeDelta;
17
18 namespace {
19 const int kMaxInputChannels = 3;
20
21 // TODO(henrika): remove usage of timers and add support for proper
22 // notification of when the input device is removed.  This was originally added
23 // to resolve http://crbug.com/79936 for Windows platforms.  This then caused
24 // breakage (very hard to repro bugs!) on other platforms: See
25 // http://crbug.com/226327 and http://crbug.com/230972.
26 // See also that the timer has been disabled on Mac now due to
27 // crbug.com/357501.
28 const int kTimerResetIntervalSeconds = 1;
29 // We have received reports that the timer can be too trigger happy on some
30 // Mac devices and the initial timer interval has therefore been increased
31 // from 1 second to 5 seconds.
32 const int kTimerInitialIntervalSeconds = 5;
33
34 #if defined(AUDIO_POWER_MONITORING)
35 // Time constant for AudioPowerMonitor.
36 // The utilized smoothing factor (alpha) in the exponential filter is given
37 // by 1-exp(-1/(fs*ts)), where fs is the sample rate in Hz and ts is the time
38 // constant given by |kPowerMeasurementTimeConstantMilliseconds|.
39 // Example: fs=44100, ts=10e-3 => alpha~0.022420
40 //          fs=44100, ts=20e-3 => alpha~0.165903
41 // A large smoothing factor corresponds to a faster filter response to input
42 // changes since y(n)=alpha*x(n)+(1-alpha)*y(n-1), where x(n) is the input
43 // and y(n) is the output.
44 const int kPowerMeasurementTimeConstantMilliseconds = 10;
45
46 // Time in seconds between two successive measurements of audio power levels.
47 const int kPowerMonitorLogIntervalSeconds = 5;
48 #endif
49 }
50
51 // Used to log the result of capture startup.
52 // This was previously logged as a boolean with only the no callback and OK
53 // options. The enum order is kept to ensure backwards compatibility.
54 // Elements in this enum should not be deleted or rearranged; the only
55 // permitted operation is to add new elements before CAPTURE_STARTUP_RESULT_MAX
56 // and update CAPTURE_STARTUP_RESULT_MAX.
57 enum CaptureStartupResult {
58   CAPTURE_STARTUP_NO_DATA_CALLBACK = 0,
59   CAPTURE_STARTUP_OK = 1,
60   CAPTURE_STARTUP_CREATE_STREAM_FAILED = 2,
61   CAPTURE_STARTUP_OPEN_STREAM_FAILED = 3,
62   CAPTURE_STARTUP_RESULT_MAX = CAPTURE_STARTUP_OPEN_STREAM_FAILED
63 };
64
65 void LogCaptureStartupResult(CaptureStartupResult result) {
66   UMA_HISTOGRAM_ENUMERATION("Media.AudioInputControllerCaptureStartupSuccess",
67                             result,
68                             CAPTURE_STARTUP_RESULT_MAX + 1);
69
70 }
71
72 namespace media {
73
74 // static
75 AudioInputController::Factory* AudioInputController::factory_ = NULL;
76
77 AudioInputController::AudioInputController(EventHandler* handler,
78                                            SyncWriter* sync_writer,
79                                            UserInputMonitor* user_input_monitor)
80     : creator_task_runner_(base::MessageLoopProxy::current()),
81       handler_(handler),
82       stream_(NULL),
83       data_is_active_(false),
84       state_(CLOSED),
85       sync_writer_(sync_writer),
86       max_volume_(0.0),
87       user_input_monitor_(user_input_monitor),
88 #if defined(AUDIO_POWER_MONITORING)
89       log_silence_state_(false),
90       silence_state_(SILENCE_STATE_NO_MEASUREMENT),
91 #endif
92       prev_key_down_count_(0) {
93   DCHECK(creator_task_runner_.get());
94 }
95
96 AudioInputController::~AudioInputController() {
97   DCHECK_EQ(state_, CLOSED);
98 }
99
100 // static
101 scoped_refptr<AudioInputController> AudioInputController::Create(
102     AudioManager* audio_manager,
103     EventHandler* event_handler,
104     const AudioParameters& params,
105     const std::string& device_id,
106     UserInputMonitor* user_input_monitor) {
107   DCHECK(audio_manager);
108
109   if (!params.IsValid() || (params.channels() > kMaxInputChannels))
110     return NULL;
111
112   if (factory_) {
113     return factory_->Create(
114         audio_manager, event_handler, params, user_input_monitor);
115   }
116   scoped_refptr<AudioInputController> controller(
117       new AudioInputController(event_handler, NULL, user_input_monitor));
118
119   controller->task_runner_ = audio_manager->GetTaskRunner();
120
121   // Create and open a new audio input stream from the existing
122   // audio-device thread.
123   if (!controller->task_runner_->PostTask(
124           FROM_HERE,
125           base::Bind(&AudioInputController::DoCreate,
126                      controller,
127                      base::Unretained(audio_manager),
128                      params,
129                      device_id))) {
130     controller = NULL;
131   }
132
133   return controller;
134 }
135
136 // static
137 scoped_refptr<AudioInputController> AudioInputController::CreateLowLatency(
138     AudioManager* audio_manager,
139     EventHandler* event_handler,
140     const AudioParameters& params,
141     const std::string& device_id,
142     SyncWriter* sync_writer,
143     UserInputMonitor* user_input_monitor) {
144   DCHECK(audio_manager);
145   DCHECK(sync_writer);
146
147   if (!params.IsValid() || (params.channels() > kMaxInputChannels))
148     return NULL;
149
150   // Create the AudioInputController object and ensure that it runs on
151   // the audio-manager thread.
152   scoped_refptr<AudioInputController> controller(
153       new AudioInputController(event_handler, sync_writer, user_input_monitor));
154   controller->task_runner_ = audio_manager->GetTaskRunner();
155
156   // Create and open a new audio input stream from the existing
157   // audio-device thread. Use the provided audio-input device.
158   if (!controller->task_runner_->PostTask(
159           FROM_HERE,
160           base::Bind(&AudioInputController::DoCreateForLowLatency,
161                      controller,
162                      base::Unretained(audio_manager),
163                      params,
164                      device_id))) {
165     controller = NULL;
166   }
167
168   return controller;
169 }
170
171 // static
172 scoped_refptr<AudioInputController> AudioInputController::CreateForStream(
173     const scoped_refptr<base::SingleThreadTaskRunner>& task_runner,
174     EventHandler* event_handler,
175     AudioInputStream* stream,
176     SyncWriter* sync_writer,
177     UserInputMonitor* user_input_monitor) {
178   DCHECK(sync_writer);
179   DCHECK(stream);
180
181   // Create the AudioInputController object and ensure that it runs on
182   // the audio-manager thread.
183   scoped_refptr<AudioInputController> controller(
184       new AudioInputController(event_handler, sync_writer, user_input_monitor));
185   controller->task_runner_ = task_runner;
186
187   // TODO(miu): See TODO at top of file.  Until that's resolved, we need to
188   // disable the error auto-detection here (since the audio mirroring
189   // implementation will reliably report error and close events).  Note, of
190   // course, that we're assuming CreateForStream() has been called for the audio
191   // mirroring use case only.
192   if (!controller->task_runner_->PostTask(
193           FROM_HERE,
194           base::Bind(&AudioInputController::DoCreateForStream,
195                      controller,
196                      stream))) {
197     controller = NULL;
198   }
199
200   return controller;
201 }
202
203 void AudioInputController::Record() {
204   task_runner_->PostTask(FROM_HERE, base::Bind(
205       &AudioInputController::DoRecord, this));
206 }
207
208 void AudioInputController::Close(const base::Closure& closed_task) {
209   DCHECK(!closed_task.is_null());
210   DCHECK(creator_task_runner_->BelongsToCurrentThread());
211
212   task_runner_->PostTaskAndReply(
213       FROM_HERE, base::Bind(&AudioInputController::DoClose, this), closed_task);
214 }
215
216 void AudioInputController::SetVolume(double volume) {
217   task_runner_->PostTask(FROM_HERE, base::Bind(
218       &AudioInputController::DoSetVolume, this, volume));
219 }
220
221 void AudioInputController::SetAutomaticGainControl(bool enabled) {
222   task_runner_->PostTask(FROM_HERE, base::Bind(
223       &AudioInputController::DoSetAutomaticGainControl, this, enabled));
224 }
225
226 void AudioInputController::DoCreate(AudioManager* audio_manager,
227                                     const AudioParameters& params,
228                                     const std::string& device_id) {
229   DCHECK(task_runner_->BelongsToCurrentThread());
230   SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.CreateTime");
231   if (handler_)
232     handler_->OnLog(this, "AIC::DoCreate");
233
234 #if defined(AUDIO_POWER_MONITORING)
235   // Create the audio (power) level meter given the provided audio parameters.
236   // An AudioBus is also needed to wrap the raw data buffer from the native
237   // layer to match AudioPowerMonitor::Scan().
238   // TODO(henrika): Remove use of extra AudioBus. See http://crbug.com/375155.
239   last_audio_level_log_time_ = base::TimeTicks::Now();
240   audio_level_.reset(new media::AudioPowerMonitor(
241       params.sample_rate(),
242       TimeDelta::FromMilliseconds(kPowerMeasurementTimeConstantMilliseconds)));
243   audio_params_ = params;
244   silence_state_ = SILENCE_STATE_NO_MEASUREMENT;
245 #endif
246
247   // TODO(miu): See TODO at top of file.  Until that's resolved, assume all
248   // platform audio input requires the |no_data_timer_| be used to auto-detect
249   // errors.  In reality, probably only Windows needs to be treated as
250   // unreliable here.
251   DoCreateForStream(audio_manager->MakeAudioInputStream(params, device_id));
252 }
253
254 void AudioInputController::DoCreateForLowLatency(AudioManager* audio_manager,
255                                                  const AudioParameters& params,
256                                                  const std::string& device_id) {
257   DCHECK(task_runner_->BelongsToCurrentThread());
258
259 #if defined(AUDIO_POWER_MONITORING)
260   // We only log silence state UMA stats for low latency mode and if we use a
261   // real device.
262   if (params.format() != AudioParameters::AUDIO_FAKE)
263     log_silence_state_ = true;
264 #endif
265
266   DoCreate(audio_manager, params, device_id);
267 }
268
269 void AudioInputController::DoCreateForStream(
270     AudioInputStream* stream_to_control) {
271   DCHECK(task_runner_->BelongsToCurrentThread());
272
273   DCHECK(!stream_);
274   stream_ = stream_to_control;
275
276   if (!stream_) {
277     if (handler_)
278       handler_->OnError(this, STREAM_CREATE_ERROR);
279     LogCaptureStartupResult(CAPTURE_STARTUP_CREATE_STREAM_FAILED);
280     return;
281   }
282
283   if (stream_ && !stream_->Open()) {
284     stream_->Close();
285     stream_ = NULL;
286     if (handler_)
287       handler_->OnError(this, STREAM_OPEN_ERROR);
288     LogCaptureStartupResult(CAPTURE_STARTUP_OPEN_STREAM_FAILED);
289     return;
290   }
291
292   DCHECK(!no_data_timer_.get());
293
294   // Create the data timer which will call FirstCheckForNoData(). The timer
295   // is started in DoRecord() and restarted in each DoCheckForNoData()
296   // callback.
297   // The timer is enabled for logging purposes. The NO_DATA_ERROR triggered
298   // from the timer must be ignored by the EventHandler.
299   // TODO(henrika): remove usage of timer when it has been verified on Canary
300   // that we are safe doing so. Goal is to get rid of |no_data_timer_| and
301   // everything that is tied to it. crbug.com/357569.
302   no_data_timer_.reset(new base::Timer(
303       FROM_HERE, base::TimeDelta::FromSeconds(kTimerInitialIntervalSeconds),
304       base::Bind(&AudioInputController::FirstCheckForNoData,
305                  base::Unretained(this)), false));
306
307   state_ = CREATED;
308   if (handler_)
309     handler_->OnCreated(this);
310
311   if (user_input_monitor_) {
312     user_input_monitor_->EnableKeyPressMonitoring();
313     prev_key_down_count_ = user_input_monitor_->GetKeyPressCount();
314   }
315 }
316
317 void AudioInputController::DoRecord() {
318   DCHECK(task_runner_->BelongsToCurrentThread());
319   SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.RecordTime");
320
321   if (state_ != CREATED)
322     return;
323
324   {
325     base::AutoLock auto_lock(lock_);
326     state_ = RECORDING;
327   }
328
329   if (handler_)
330     handler_->OnLog(this, "AIC::DoRecord");
331
332   if (no_data_timer_) {
333     // Start the data timer. Once |kTimerResetIntervalSeconds| have passed,
334     // a callback to FirstCheckForNoData() is made.
335     no_data_timer_->Reset();
336   }
337
338   stream_->Start(this);
339   if (handler_)
340     handler_->OnRecording(this);
341 }
342
343 void AudioInputController::DoClose() {
344   DCHECK(task_runner_->BelongsToCurrentThread());
345   SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.CloseTime");
346
347   if (state_ == CLOSED)
348     return;
349
350   if (handler_)
351     handler_->OnLog(this, "AIC::DoClose");
352
353   // Delete the timer on the same thread that created it.
354   no_data_timer_.reset();
355
356   DoStopCloseAndClearStream();
357   SetDataIsActive(false);
358
359   if (SharedMemoryAndSyncSocketMode())
360     sync_writer_->Close();
361
362   if (user_input_monitor_)
363     user_input_monitor_->DisableKeyPressMonitoring();
364
365 #if defined(AUDIO_POWER_MONITORING)
366   // Send UMA stats if enabled.
367   if (log_silence_state_)
368     LogSilenceState(silence_state_);
369   log_silence_state_ = false;
370 #endif
371
372   state_ = CLOSED;
373 }
374
375 void AudioInputController::DoReportError() {
376   DCHECK(task_runner_->BelongsToCurrentThread());
377   if (handler_)
378     handler_->OnError(this, STREAM_ERROR);
379 }
380
381 void AudioInputController::DoSetVolume(double volume) {
382   DCHECK(task_runner_->BelongsToCurrentThread());
383   DCHECK_GE(volume, 0);
384   DCHECK_LE(volume, 1.0);
385
386   if (state_ != CREATED && state_ != RECORDING)
387     return;
388
389   // Only ask for the maximum volume at first call and use cached value
390   // for remaining function calls.
391   if (!max_volume_) {
392     max_volume_ = stream_->GetMaxVolume();
393   }
394
395   if (max_volume_ == 0.0) {
396     DLOG(WARNING) << "Failed to access input volume control";
397     return;
398   }
399
400   // Set the stream volume and scale to a range matched to the platform.
401   stream_->SetVolume(max_volume_ * volume);
402 }
403
404 void AudioInputController::DoSetAutomaticGainControl(bool enabled) {
405   DCHECK(task_runner_->BelongsToCurrentThread());
406   DCHECK_NE(state_, RECORDING);
407
408   // Ensure that the AGC state only can be modified before streaming starts.
409   if (state_ != CREATED)
410     return;
411
412   stream_->SetAutomaticGainControl(enabled);
413 }
414
415 void AudioInputController::FirstCheckForNoData() {
416   DCHECK(task_runner_->BelongsToCurrentThread());
417   LogCaptureStartupResult(GetDataIsActive() ?
418                           CAPTURE_STARTUP_OK :
419                           CAPTURE_STARTUP_NO_DATA_CALLBACK);
420   if (handler_) {
421     handler_->OnLog(this, GetDataIsActive() ?
422                     "AIC::FirstCheckForNoData => data is active" :
423                     "AIC::FirstCheckForNoData => data is NOT active");
424   }
425   DoCheckForNoData();
426 }
427
428 void AudioInputController::DoCheckForNoData() {
429   DCHECK(task_runner_->BelongsToCurrentThread());
430
431   if (!GetDataIsActive()) {
432     // The data-is-active marker will be false only if it has been more than
433     // one second since a data packet was recorded. This can happen if a
434     // capture device has been removed or disabled.
435     if (handler_)
436       handler_->OnError(this, NO_DATA_ERROR);
437   }
438
439   // Mark data as non-active. The flag will be re-enabled in OnData() each
440   // time a data packet is received. Hence, under normal conditions, the
441   // flag will only be disabled during a very short period.
442   SetDataIsActive(false);
443
444   // Restart the timer to ensure that we check the flag again in
445   // |kTimerResetIntervalSeconds|.
446   no_data_timer_->Start(
447       FROM_HERE, base::TimeDelta::FromSeconds(kTimerResetIntervalSeconds),
448       base::Bind(&AudioInputController::DoCheckForNoData,
449       base::Unretained(this)));
450 }
451
452 void AudioInputController::OnData(AudioInputStream* stream,
453                                   const AudioBus* source,
454                                   uint32 hardware_delay_bytes,
455                                   double volume) {
456   // Mark data as active to ensure that the periodic calls to
457   // DoCheckForNoData() does not report an error to the event handler.
458   SetDataIsActive(true);
459
460   {
461     base::AutoLock auto_lock(lock_);
462     if (state_ != RECORDING)
463       return;
464   }
465
466   bool key_pressed = false;
467   if (user_input_monitor_) {
468     size_t current_count = user_input_monitor_->GetKeyPressCount();
469     key_pressed = current_count != prev_key_down_count_;
470     prev_key_down_count_ = current_count;
471     DVLOG_IF(6, key_pressed) << "Detected keypress.";
472   }
473
474   // Use SharedMemory and SyncSocket if the client has created a SyncWriter.
475   // Used by all low-latency clients except WebSpeech.
476   if (SharedMemoryAndSyncSocketMode()) {
477     sync_writer_->Write(source, volume, key_pressed);
478     sync_writer_->UpdateRecordedBytes(hardware_delay_bytes);
479
480 #if defined(AUDIO_POWER_MONITORING)
481     // Only do power-level measurements if an AudioPowerMonitor object has
482     // been created. Done in DoCreate() but not DoCreateForStream(), hence
483     // logging will mainly be done for WebRTC and WebSpeech clients.
484     if (!audio_level_)
485       return;
486
487     // Perform periodic audio (power) level measurements.
488     if ((base::TimeTicks::Now() - last_audio_level_log_time_).InSeconds() >
489         kPowerMonitorLogIntervalSeconds) {
490       // Wrap data into an AudioBus to match AudioPowerMonitor::Scan.
491       // TODO(henrika): remove this section when capture side uses AudioBus.
492       // See http://crbug.com/375155 for details.
493       audio_level_->Scan(*source, source->frames());
494
495       // Get current average power level and add it to the log.
496       // Possible range is given by [-inf, 0] dBFS.
497       std::pair<float, bool> result = audio_level_->ReadCurrentPowerAndClip();
498
499       // Use event handler on the audio thread to relay a message to the ARIH
500       // in content which does the actual logging on the IO thread.
501       task_runner_->PostTask(
502           FROM_HERE,
503           base::Bind(
504               &AudioInputController::DoLogAudioLevel, this, result.first));
505
506       last_audio_level_log_time_ = base::TimeTicks::Now();
507
508       // Reset the average power level (since we don't log continuously).
509       audio_level_->Reset();
510     }
511 #endif
512     return;
513   }
514
515   // TODO(henrika): Investigate if we can avoid the extra copy here.
516   // (see http://crbug.com/249316 for details). AFAIK, this scope is only
517   // active for WebSpeech clients.
518   scoped_ptr<AudioBus> audio_data =
519       AudioBus::Create(source->channels(), source->frames());
520   source->CopyTo(audio_data.get());
521
522   // Ownership of the audio buffer will be with the callback until it is run,
523   // when ownership is passed to the callback function.
524   task_runner_->PostTask(
525       FROM_HERE,
526       base::Bind(
527           &AudioInputController::DoOnData, this, base::Passed(&audio_data)));
528 }
529
530 void AudioInputController::DoOnData(scoped_ptr<AudioBus> data) {
531   DCHECK(task_runner_->BelongsToCurrentThread());
532   if (handler_)
533     handler_->OnData(this, data.get());
534 }
535
536 void AudioInputController::DoLogAudioLevel(float level_dbfs) {
537 #if defined(AUDIO_POWER_MONITORING)
538   DCHECK(task_runner_->BelongsToCurrentThread());
539   if (!handler_)
540     return;
541
542   std::string log_string = base::StringPrintf(
543       "AIC::OnData: average audio level=%.2f dBFS", level_dbfs);
544   static const float kSilenceThresholdDBFS = -72.24719896f;
545   if (level_dbfs < kSilenceThresholdDBFS)
546     log_string += " <=> no audio input!";
547   handler_->OnLog(this, log_string);
548
549   UpdateSilenceState(level_dbfs < kSilenceThresholdDBFS);
550 #endif
551 }
552
553 void AudioInputController::OnError(AudioInputStream* stream) {
554   // Handle error on the audio-manager thread.
555   task_runner_->PostTask(FROM_HERE, base::Bind(
556       &AudioInputController::DoReportError, this));
557 }
558
559 void AudioInputController::DoStopCloseAndClearStream() {
560   DCHECK(task_runner_->BelongsToCurrentThread());
561
562   // Allow calling unconditionally and bail if we don't have a stream to close.
563   if (stream_ != NULL) {
564     stream_->Stop();
565     stream_->Close();
566     stream_ = NULL;
567   }
568
569   // The event handler should not be touched after the stream has been closed.
570   handler_ = NULL;
571 }
572
573 void AudioInputController::SetDataIsActive(bool enabled) {
574   base::subtle::Release_Store(&data_is_active_, enabled);
575 }
576
577 bool AudioInputController::GetDataIsActive() {
578   return (base::subtle::Acquire_Load(&data_is_active_) != false);
579 }
580
581 #if defined(AUDIO_POWER_MONITORING)
582 void AudioInputController::UpdateSilenceState(bool silence) {
583   if (silence) {
584     if (silence_state_ == SILENCE_STATE_NO_MEASUREMENT) {
585       silence_state_ = SILENCE_STATE_ONLY_SILENCE;
586     } else if (silence_state_ == SILENCE_STATE_ONLY_AUDIO) {
587       silence_state_ = SILENCE_STATE_AUDIO_AND_SILENCE;
588     } else {
589       DCHECK(silence_state_ == SILENCE_STATE_ONLY_SILENCE ||
590              silence_state_ == SILENCE_STATE_AUDIO_AND_SILENCE);
591     }
592   } else {
593     if (silence_state_ == SILENCE_STATE_NO_MEASUREMENT) {
594       silence_state_ = SILENCE_STATE_ONLY_AUDIO;
595     } else if (silence_state_ == SILENCE_STATE_ONLY_SILENCE) {
596       silence_state_ = SILENCE_STATE_AUDIO_AND_SILENCE;
597     } else {
598       DCHECK(silence_state_ == SILENCE_STATE_ONLY_AUDIO ||
599              silence_state_ == SILENCE_STATE_AUDIO_AND_SILENCE);
600     }
601   }
602 }
603
604 void AudioInputController::LogSilenceState(SilenceState value) {
605   UMA_HISTOGRAM_ENUMERATION("Media.AudioInputControllerSessionSilenceReport",
606                             value,
607                             SILENCE_STATE_MAX + 1);
608 }
609 #endif
610
611 }  // namespace media