Upstream version 10.38.208.0
[platform/framework/web/crosswalk.git] / src / content / renderer / media / webrtc_audio_capturer.cc
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "content/renderer/media/webrtc_audio_capturer.h"
6
7 #include "base/bind.h"
8 #include "base/logging.h"
9 #include "base/metrics/histogram.h"
10 #include "base/strings/string_util.h"
11 #include "base/strings/stringprintf.h"
12 #include "content/child/child_process.h"
13 #include "content/renderer/media/audio_device_factory.h"
14 #include "content/renderer/media/media_stream_audio_processor.h"
15 #include "content/renderer/media/media_stream_audio_processor_options.h"
16 #include "content/renderer/media/media_stream_audio_source.h"
17 #include "content/renderer/media/webrtc_audio_device_impl.h"
18 #include "content/renderer/media/webrtc_local_audio_track.h"
19 #include "content/renderer/media/webrtc_logging.h"
20 #include "media/audio/sample_rates.h"
21
22 namespace content {
23
24 namespace {
25
26 // Time constant for AudioPowerMonitor.  See AudioPowerMonitor ctor comments
27 // for semantics.  This value was arbitrarily chosen, but seems to work well.
28 const int kPowerMonitorTimeConstantMs = 10;
29
30 // The time between two audio power level samples.
31 const int kPowerMonitorLogIntervalSeconds = 10;
32
33 }  // namespace
34
35 // Reference counted container of WebRtcLocalAudioTrack delegate.
36 // TODO(xians): Switch to MediaStreamAudioSinkOwner.
37 class WebRtcAudioCapturer::TrackOwner
38     : public base::RefCountedThreadSafe<WebRtcAudioCapturer::TrackOwner> {
39  public:
40   explicit TrackOwner(WebRtcLocalAudioTrack* track)
41       : delegate_(track) {}
42
43   void Capture(const int16* audio_data,
44                base::TimeDelta delay,
45                double volume,
46                bool key_pressed,
47                bool need_audio_processing) {
48     base::AutoLock lock(lock_);
49     if (delegate_) {
50       delegate_->Capture(audio_data,
51                          delay,
52                          volume,
53                          key_pressed,
54                          need_audio_processing);
55     }
56   }
57
58   void OnSetFormat(const media::AudioParameters& params) {
59     base::AutoLock lock(lock_);
60     if (delegate_)
61       delegate_->OnSetFormat(params);
62   }
63
64   void SetAudioProcessor(
65       const scoped_refptr<MediaStreamAudioProcessor>& processor) {
66     base::AutoLock lock(lock_);
67     if (delegate_)
68       delegate_->SetAudioProcessor(processor);
69   }
70
71   void Reset() {
72     base::AutoLock lock(lock_);
73     delegate_ = NULL;
74   }
75
76   void Stop() {
77     base::AutoLock lock(lock_);
78     DCHECK(delegate_);
79
80     // This can be reentrant so reset |delegate_| before calling out.
81     WebRtcLocalAudioTrack* temp = delegate_;
82     delegate_ = NULL;
83     temp->Stop();
84   }
85
86   // Wrapper which allows to use std::find_if() when adding and removing
87   // sinks to/from the list.
88   struct TrackWrapper {
89     TrackWrapper(WebRtcLocalAudioTrack* track) : track_(track) {}
90     bool operator()(
91         const scoped_refptr<WebRtcAudioCapturer::TrackOwner>& owner) const {
92       return owner->IsEqual(track_);
93     }
94     WebRtcLocalAudioTrack* track_;
95   };
96
97  protected:
98   virtual ~TrackOwner() {}
99
100  private:
101   friend class base::RefCountedThreadSafe<WebRtcAudioCapturer::TrackOwner>;
102
103   bool IsEqual(const WebRtcLocalAudioTrack* other) const {
104     base::AutoLock lock(lock_);
105     return (other == delegate_);
106   }
107
108   // Do NOT reference count the |delegate_| to avoid cyclic reference counting.
109   WebRtcLocalAudioTrack* delegate_;
110   mutable base::Lock lock_;
111
112   DISALLOW_COPY_AND_ASSIGN(TrackOwner);
113 };
114
115 // static
116 scoped_refptr<WebRtcAudioCapturer> WebRtcAudioCapturer::CreateCapturer(
117     int render_view_id, const StreamDeviceInfo& device_info,
118     const blink::WebMediaConstraints& constraints,
119     WebRtcAudioDeviceImpl* audio_device,
120     MediaStreamAudioSource* audio_source) {
121   scoped_refptr<WebRtcAudioCapturer> capturer = new WebRtcAudioCapturer(
122       render_view_id, device_info, constraints, audio_device, audio_source);
123   if (capturer->Initialize())
124     return capturer;
125
126   return NULL;
127 }
128
129 bool WebRtcAudioCapturer::Initialize() {
130   DCHECK(thread_checker_.CalledOnValidThread());
131   DVLOG(1) << "WebRtcAudioCapturer::Initialize()";
132   WebRtcLogMessage(base::StringPrintf(
133       "WAC::Initialize. render_view_id=%d"
134       ", channel_layout=%d, sample_rate=%d, buffer_size=%d"
135       ", session_id=%d, paired_output_sample_rate=%d"
136       ", paired_output_frames_per_buffer=%d, effects=%d. ",
137       render_view_id_,
138       device_info_.device.input.channel_layout,
139       device_info_.device.input.sample_rate,
140       device_info_.device.input.frames_per_buffer,
141       device_info_.session_id,
142       device_info_.device.matched_output.sample_rate,
143       device_info_.device.matched_output.frames_per_buffer,
144       device_info_.device.input.effects));
145
146   if (render_view_id_ == -1) {
147     // Return true here to allow injecting a new source via
148     // SetCapturerSourceForTesting() at a later state.
149     return true;
150   }
151
152   MediaAudioConstraints audio_constraints(constraints_,
153                                           device_info_.device.input.effects);
154   if (!audio_constraints.IsValid())
155     return false;
156
157   media::ChannelLayout channel_layout = static_cast<media::ChannelLayout>(
158       device_info_.device.input.channel_layout);
159   DVLOG(1) << "Audio input hardware channel layout: " << channel_layout;
160   UMA_HISTOGRAM_ENUMERATION("WebRTC.AudioInputChannelLayout",
161                             channel_layout, media::CHANNEL_LAYOUT_MAX + 1);
162
163   // Verify that the reported input channel configuration is supported.
164   if (channel_layout != media::CHANNEL_LAYOUT_MONO &&
165       channel_layout != media::CHANNEL_LAYOUT_STEREO &&
166       channel_layout != media::CHANNEL_LAYOUT_STEREO_AND_KEYBOARD_MIC) {
167     DLOG(ERROR) << channel_layout
168                 << " is not a supported input channel configuration.";
169     return false;
170   }
171
172   DVLOG(1) << "Audio input hardware sample rate: "
173            << device_info_.device.input.sample_rate;
174   media::AudioSampleRate asr;
175   if (media::ToAudioSampleRate(device_info_.device.input.sample_rate, &asr)) {
176     UMA_HISTOGRAM_ENUMERATION(
177         "WebRTC.AudioInputSampleRate", asr, media::kAudioSampleRateMax + 1);
178   } else {
179     UMA_HISTOGRAM_COUNTS("WebRTC.AudioInputSampleRateUnexpected",
180                          device_info_.device.input.sample_rate);
181   }
182
183   // Create and configure the default audio capturing source.
184   SetCapturerSource(AudioDeviceFactory::NewInputDevice(render_view_id_),
185                     channel_layout,
186                     static_cast<float>(device_info_.device.input.sample_rate));
187
188   // Add the capturer to the WebRtcAudioDeviceImpl since it needs some hardware
189   // information from the capturer.
190   if (audio_device_)
191     audio_device_->AddAudioCapturer(this);
192
193   return true;
194 }
195
196 WebRtcAudioCapturer::WebRtcAudioCapturer(
197     int render_view_id,
198     const StreamDeviceInfo& device_info,
199     const blink::WebMediaConstraints& constraints,
200     WebRtcAudioDeviceImpl* audio_device,
201     MediaStreamAudioSource* audio_source)
202     : constraints_(constraints),
203       audio_processor_(
204           new rtc::RefCountedObject<MediaStreamAudioProcessor>(
205               constraints, device_info.device.input.effects, audio_device)),
206       running_(false),
207       render_view_id_(render_view_id),
208       device_info_(device_info),
209       volume_(0),
210       peer_connection_mode_(false),
211       key_pressed_(false),
212       need_audio_processing_(false),
213       audio_device_(audio_device),
214       audio_source_(audio_source),
215       audio_power_monitor_(
216           device_info_.device.input.sample_rate,
217           base::TimeDelta::FromMilliseconds(kPowerMonitorTimeConstantMs)) {
218   DVLOG(1) << "WebRtcAudioCapturer::WebRtcAudioCapturer()";
219 }
220
221 WebRtcAudioCapturer::~WebRtcAudioCapturer() {
222   DCHECK(thread_checker_.CalledOnValidThread());
223   DCHECK(tracks_.IsEmpty());
224   DVLOG(1) << "WebRtcAudioCapturer::~WebRtcAudioCapturer()";
225   Stop();
226 }
227
228 void WebRtcAudioCapturer::AddTrack(WebRtcLocalAudioTrack* track) {
229   DCHECK(track);
230   DVLOG(1) << "WebRtcAudioCapturer::AddTrack()";
231
232   {
233     base::AutoLock auto_lock(lock_);
234     // Verify that |track| is not already added to the list.
235     DCHECK(!tracks_.Contains(TrackOwner::TrackWrapper(track)));
236
237     // Add with a tag, so we remember to call OnSetFormat() on the new
238     // track.
239     scoped_refptr<TrackOwner> track_owner(new TrackOwner(track));
240     tracks_.AddAndTag(track_owner);
241   }
242 }
243
244 void WebRtcAudioCapturer::RemoveTrack(WebRtcLocalAudioTrack* track) {
245   DCHECK(thread_checker_.CalledOnValidThread());
246   DVLOG(1) << "WebRtcAudioCapturer::RemoveTrack()";
247   bool stop_source = false;
248   {
249     base::AutoLock auto_lock(lock_);
250
251     scoped_refptr<TrackOwner> removed_item =
252         tracks_.Remove(TrackOwner::TrackWrapper(track));
253
254     // Clear the delegate to ensure that no more capture callbacks will
255     // be sent to this sink. Also avoids a possible crash which can happen
256     // if this method is called while capturing is active.
257     if (removed_item.get()) {
258       removed_item->Reset();
259       stop_source = tracks_.IsEmpty();
260     }
261   }
262   if (stop_source) {
263     // Since WebRtcAudioCapturer does not inherit MediaStreamAudioSource,
264     // and instead MediaStreamAudioSource is composed of a WebRtcAudioCapturer,
265     // we have to call StopSource on the MediaStreamSource. This will call
266     // MediaStreamAudioSource::DoStopSource which in turn call
267     // WebRtcAudioCapturerer::Stop();
268     audio_source_->StopSource();
269   }
270 }
271
272 void WebRtcAudioCapturer::SetCapturerSource(
273     const scoped_refptr<media::AudioCapturerSource>& source,
274     media::ChannelLayout channel_layout,
275     float sample_rate) {
276   DCHECK(thread_checker_.CalledOnValidThread());
277   DVLOG(1) << "SetCapturerSource(channel_layout=" << channel_layout << ","
278            << "sample_rate=" << sample_rate << ")";
279   scoped_refptr<media::AudioCapturerSource> old_source;
280   {
281     base::AutoLock auto_lock(lock_);
282     if (source_.get() == source.get())
283       return;
284
285     source_.swap(old_source);
286     source_ = source;
287
288     // Reset the flag to allow starting the new source.
289     running_ = false;
290   }
291
292   DVLOG(1) << "Switching to a new capture source.";
293   if (old_source.get())
294     old_source->Stop();
295
296   // Dispatch the new parameters both to the sink(s) and to the new source,
297   // also apply the new |constraints|.
298   // The idea is to get rid of any dependency of the microphone parameters
299   // which would normally be used by default.
300   // bits_per_sample is always 16 for now.
301   int buffer_size = GetBufferSize(sample_rate);
302   media::AudioParameters params(media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
303                                 channel_layout, 0, sample_rate,
304                                 16, buffer_size,
305                                 device_info_.device.input.effects);
306
307   {
308     base::AutoLock auto_lock(lock_);
309     // Notify the |audio_processor_| of the new format.
310     audio_processor_->OnCaptureFormatChanged(params);
311
312     MediaAudioConstraints audio_constraints(constraints_,
313                                             device_info_.device.input.effects);
314     need_audio_processing_ = audio_constraints.NeedsAudioProcessing();
315     // Notify all tracks about the new format.
316     tracks_.TagAll();
317   }
318
319   if (source.get())
320     source->Initialize(params, this, session_id());
321
322   Start();
323 }
324
325 void WebRtcAudioCapturer::EnablePeerConnectionMode() {
326   DCHECK(thread_checker_.CalledOnValidThread());
327   DVLOG(1) << "EnablePeerConnectionMode";
328   // Do nothing if the peer connection mode has been enabled.
329   if (peer_connection_mode_)
330     return;
331
332   peer_connection_mode_ = true;
333   int render_view_id = -1;
334   media::AudioParameters input_params;
335   {
336     base::AutoLock auto_lock(lock_);
337     // Simply return if there is no existing source or the |render_view_id_| is
338     // not valid.
339     if (!source_.get() || render_view_id_== -1)
340       return;
341
342     render_view_id = render_view_id_;
343     input_params = audio_processor_->InputFormat();
344   }
345
346   // Do nothing if the current buffer size is the WebRtc native buffer size.
347   if (GetBufferSize(input_params.sample_rate()) ==
348           input_params.frames_per_buffer()) {
349     return;
350   }
351
352   // Create a new audio stream as source which will open the hardware using
353   // WebRtc native buffer size.
354   SetCapturerSource(AudioDeviceFactory::NewInputDevice(render_view_id),
355                     input_params.channel_layout(),
356                     static_cast<float>(input_params.sample_rate()));
357 }
358
359 void WebRtcAudioCapturer::Start() {
360   DCHECK(thread_checker_.CalledOnValidThread());
361   DVLOG(1) << "WebRtcAudioCapturer::Start()";
362   base::AutoLock auto_lock(lock_);
363   if (running_ || !source_)
364     return;
365
366   // Start the data source, i.e., start capturing data from the current source.
367   // We need to set the AGC control before starting the stream.
368   source_->SetAutomaticGainControl(true);
369   source_->Start();
370   running_ = true;
371 }
372
373 void WebRtcAudioCapturer::Stop() {
374   DCHECK(thread_checker_.CalledOnValidThread());
375   DVLOG(1) << "WebRtcAudioCapturer::Stop()";
376   scoped_refptr<media::AudioCapturerSource> source;
377   TrackList::ItemList tracks;
378   {
379     base::AutoLock auto_lock(lock_);
380     if (!running_)
381       return;
382
383     source = source_;
384     tracks = tracks_.Items();
385     tracks_.Clear();
386     running_ = false;
387   }
388
389   // Remove the capturer object from the WebRtcAudioDeviceImpl.
390   if (audio_device_)
391     audio_device_->RemoveAudioCapturer(this);
392
393   for (TrackList::ItemList::const_iterator it = tracks.begin();
394        it != tracks.end();
395        ++it) {
396     (*it)->Stop();
397   }
398
399   if (source.get())
400     source->Stop();
401
402   // Stop the audio processor to avoid feeding render data into the processor.
403   audio_processor_->Stop();
404 }
405
406 void WebRtcAudioCapturer::SetVolume(int volume) {
407   DVLOG(1) << "WebRtcAudioCapturer::SetVolume()";
408   DCHECK_LE(volume, MaxVolume());
409   double normalized_volume = static_cast<double>(volume) / MaxVolume();
410   base::AutoLock auto_lock(lock_);
411   if (source_.get())
412     source_->SetVolume(normalized_volume);
413 }
414
415 int WebRtcAudioCapturer::Volume() const {
416   base::AutoLock auto_lock(lock_);
417   return volume_;
418 }
419
420 int WebRtcAudioCapturer::MaxVolume() const {
421   return WebRtcAudioDeviceImpl::kMaxVolumeLevel;
422 }
423
424 void WebRtcAudioCapturer::Capture(const media::AudioBus* audio_source,
425                                   int audio_delay_milliseconds,
426                                   double volume,
427                                   bool key_pressed) {
428 // This callback is driven by AudioInputDevice::AudioThreadCallback if
429 // |source_| is AudioInputDevice, otherwise it is driven by client's
430 // CaptureCallback.
431 #if defined(OS_WIN) || defined(OS_MACOSX)
432   DCHECK_LE(volume, 1.0);
433 #elif (defined(OS_LINUX) && !defined(OS_CHROMEOS)) || defined(OS_OPENBSD)
434   // We have a special situation on Linux where the microphone volume can be
435   // "higher than maximum". The input volume slider in the sound preference
436   // allows the user to set a scaling that is higher than 100%. It means that
437   // even if the reported maximum levels is N, the actual microphone level can
438   // go up to 1.5x*N and that corresponds to a normalized |volume| of 1.5x.
439   DCHECK_LE(volume, 1.6);
440 #endif
441
442   TrackList::ItemList tracks;
443   TrackList::ItemList tracks_to_notify_format;
444   int current_volume = 0;
445   base::TimeDelta audio_delay;
446   bool need_audio_processing = true;
447   {
448     base::AutoLock auto_lock(lock_);
449     if (!running_)
450       return;
451
452     // Map internal volume range of [0.0, 1.0] into [0, 255] used by AGC.
453     // The volume can be higher than 255 on Linux, and it will be cropped to
454     // 255 since AGC does not allow values out of range.
455     volume_ = static_cast<int>((volume * MaxVolume()) + 0.5);
456     current_volume = volume_ > MaxVolume() ? MaxVolume() : volume_;
457     audio_delay = base::TimeDelta::FromMilliseconds(audio_delay_milliseconds);
458     audio_delay_ = audio_delay;
459     key_pressed_ = key_pressed;
460     tracks = tracks_.Items();
461     tracks_.RetrieveAndClearTags(&tracks_to_notify_format);
462
463     // Set the flag to turn on the audio processing in PeerConnection level.
464     // Note that, we turn off the audio processing in PeerConnection if the
465     // processor has already processed the data.
466     need_audio_processing = need_audio_processing_ ?
467         !MediaStreamAudioProcessor::IsAudioTrackProcessingEnabled() : false;
468   }
469
470   DCHECK(audio_processor_->InputFormat().IsValid());
471   DCHECK_EQ(audio_source->channels(),
472             audio_processor_->InputFormat().channels());
473   DCHECK_EQ(audio_source->frames(),
474             audio_processor_->InputFormat().frames_per_buffer());
475
476   // Notify the tracks on when the format changes. This will do nothing if
477   // |tracks_to_notify_format| is empty.
478   media::AudioParameters output_params = audio_processor_->OutputFormat();
479   for (TrackList::ItemList::const_iterator it = tracks_to_notify_format.begin();
480        it != tracks_to_notify_format.end(); ++it) {
481     (*it)->OnSetFormat(output_params);
482     (*it)->SetAudioProcessor(audio_processor_);
483   }
484
485   if ((base::TimeTicks::Now() - last_audio_level_log_time_).InSeconds() >
486           kPowerMonitorLogIntervalSeconds) {
487     audio_power_monitor_.Scan(*audio_source, audio_source->frames());
488
489     last_audio_level_log_time_ = base::TimeTicks::Now();
490
491     std::pair<float, bool> result =
492         audio_power_monitor_.ReadCurrentPowerAndClip();
493     WebRtcLogMessage(base::StringPrintf(
494         "WAC::Capture: current_audio_power=%.2fdBFS.", result.first));
495
496     audio_power_monitor_.Reset();
497   }
498
499   // Push the data to the processor for processing.
500   audio_processor_->PushCaptureData(audio_source);
501
502   // Process and consume the data in the processor until there is not enough
503   // data in the processor.
504   int16* output = NULL;
505   int new_volume = 0;
506   while (audio_processor_->ProcessAndConsumeData(
507       audio_delay, current_volume, key_pressed, &new_volume, &output)) {
508     // Feed the post-processed data to the tracks.
509     for (TrackList::ItemList::const_iterator it = tracks.begin();
510          it != tracks.end(); ++it) {
511       (*it)->Capture(output, audio_delay, current_volume, key_pressed,
512                      need_audio_processing);
513     }
514
515     if (new_volume) {
516       SetVolume(new_volume);
517
518       // Update the |current_volume| to avoid passing the old volume to AGC.
519       current_volume = new_volume;
520     }
521   }
522 }
523
524 void WebRtcAudioCapturer::OnCaptureError() {
525   NOTIMPLEMENTED();
526 }
527
528 media::AudioParameters WebRtcAudioCapturer::source_audio_parameters() const {
529   base::AutoLock auto_lock(lock_);
530   return audio_processor_ ?
531       audio_processor_->InputFormat() : media::AudioParameters();
532 }
533
534 bool WebRtcAudioCapturer::GetPairedOutputParameters(
535     int* session_id,
536     int* output_sample_rate,
537     int* output_frames_per_buffer) const {
538   // Don't set output parameters unless all of them are valid.
539   if (device_info_.session_id <= 0 ||
540       !device_info_.device.matched_output.sample_rate ||
541       !device_info_.device.matched_output.frames_per_buffer)
542     return false;
543
544   *session_id = device_info_.session_id;
545   *output_sample_rate = device_info_.device.matched_output.sample_rate;
546   *output_frames_per_buffer =
547       device_info_.device.matched_output.frames_per_buffer;
548
549   return true;
550 }
551
552 int WebRtcAudioCapturer::GetBufferSize(int sample_rate) const {
553   DCHECK(thread_checker_.CalledOnValidThread());
554 #if defined(OS_ANDROID)
555   // TODO(henrika): Tune and adjust buffer size on Android.
556   return (2 * sample_rate / 100);
557 #endif
558
559   // PeerConnection is running at a buffer size of 10ms data. A multiple of
560   // 10ms as the buffer size can give the best performance to PeerConnection.
561   int peer_connection_buffer_size = sample_rate / 100;
562
563   // Use the native hardware buffer size in non peer connection mode when the
564   // platform is using a native buffer size smaller than the PeerConnection
565   // buffer size.
566   int hardware_buffer_size = device_info_.device.input.frames_per_buffer;
567   if (!peer_connection_mode_ && hardware_buffer_size &&
568       hardware_buffer_size <= peer_connection_buffer_size) {
569     return hardware_buffer_size;
570   }
571
572   return (sample_rate / 100);
573 }
574
575 void WebRtcAudioCapturer::GetAudioProcessingParams(
576     base::TimeDelta* delay, int* volume, bool* key_pressed) {
577   base::AutoLock auto_lock(lock_);
578   *delay = audio_delay_;
579   *volume = volume_;
580   *key_pressed = key_pressed_;
581 }
582
583 void WebRtcAudioCapturer::SetCapturerSourceForTesting(
584     const scoped_refptr<media::AudioCapturerSource>& source,
585     media::AudioParameters params) {
586   // Create a new audio stream as source which uses the new source.
587   SetCapturerSource(source, params.channel_layout(),
588                     static_cast<float>(params.sample_rate()));
589 }
590
591 }  // namespace content