1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/renderer/media/webrtc_audio_capturer.h"
8 #include "base/logging.h"
9 #include "base/metrics/histogram.h"
10 #include "base/strings/string_util.h"
11 #include "base/strings/stringprintf.h"
12 #include "content/child/child_process.h"
13 #include "content/renderer/media/audio_device_factory.h"
14 #include "content/renderer/media/media_stream_audio_processor.h"
15 #include "content/renderer/media/media_stream_audio_processor_options.h"
16 #include "content/renderer/media/media_stream_audio_source.h"
17 #include "content/renderer/media/webrtc_audio_device_impl.h"
18 #include "content/renderer/media/webrtc_local_audio_track.h"
19 #include "content/renderer/media/webrtc_logging.h"
20 #include "media/audio/sample_rates.h"
26 // Time constant for AudioPowerMonitor. See AudioPowerMonitor ctor comments
27 // for semantics. This value was arbitrarily chosen, but seems to work well.
28 const int kPowerMonitorTimeConstantMs = 10;
30 // The time between two audio power level samples.
31 const int kPowerMonitorLogIntervalSeconds = 10;
35 // Reference counted container of WebRtcLocalAudioTrack delegate.
36 // TODO(xians): Switch to MediaStreamAudioSinkOwner.
37 class WebRtcAudioCapturer::TrackOwner
38 : public base::RefCountedThreadSafe<WebRtcAudioCapturer::TrackOwner> {
40 explicit TrackOwner(WebRtcLocalAudioTrack* track)
43 void Capture(const int16* audio_data,
44 base::TimeDelta delay,
47 bool need_audio_processing) {
48 base::AutoLock lock(lock_);
50 delegate_->Capture(audio_data,
54 need_audio_processing);
58 void OnSetFormat(const media::AudioParameters& params) {
59 base::AutoLock lock(lock_);
61 delegate_->OnSetFormat(params);
64 void SetAudioProcessor(
65 const scoped_refptr<MediaStreamAudioProcessor>& processor) {
66 base::AutoLock lock(lock_);
68 delegate_->SetAudioProcessor(processor);
72 base::AutoLock lock(lock_);
77 base::AutoLock lock(lock_);
80 // This can be reentrant so reset |delegate_| before calling out.
81 WebRtcLocalAudioTrack* temp = delegate_;
86 // Wrapper which allows to use std::find_if() when adding and removing
87 // sinks to/from the list.
89 TrackWrapper(WebRtcLocalAudioTrack* track) : track_(track) {}
91 const scoped_refptr<WebRtcAudioCapturer::TrackOwner>& owner) const {
92 return owner->IsEqual(track_);
94 WebRtcLocalAudioTrack* track_;
98 virtual ~TrackOwner() {}
101 friend class base::RefCountedThreadSafe<WebRtcAudioCapturer::TrackOwner>;
103 bool IsEqual(const WebRtcLocalAudioTrack* other) const {
104 base::AutoLock lock(lock_);
105 return (other == delegate_);
108 // Do NOT reference count the |delegate_| to avoid cyclic reference counting.
109 WebRtcLocalAudioTrack* delegate_;
110 mutable base::Lock lock_;
112 DISALLOW_COPY_AND_ASSIGN(TrackOwner);
116 scoped_refptr<WebRtcAudioCapturer> WebRtcAudioCapturer::CreateCapturer(
117 int render_view_id, const StreamDeviceInfo& device_info,
118 const blink::WebMediaConstraints& constraints,
119 WebRtcAudioDeviceImpl* audio_device,
120 MediaStreamAudioSource* audio_source) {
121 scoped_refptr<WebRtcAudioCapturer> capturer = new WebRtcAudioCapturer(
122 render_view_id, device_info, constraints, audio_device, audio_source);
123 if (capturer->Initialize())
129 bool WebRtcAudioCapturer::Initialize() {
130 DCHECK(thread_checker_.CalledOnValidThread());
131 DVLOG(1) << "WebRtcAudioCapturer::Initialize()";
132 WebRtcLogMessage(base::StringPrintf(
133 "WAC::Initialize. render_view_id=%d"
134 ", channel_layout=%d, sample_rate=%d, buffer_size=%d"
135 ", session_id=%d, paired_output_sample_rate=%d"
136 ", paired_output_frames_per_buffer=%d, effects=%d. ",
138 device_info_.device.input.channel_layout,
139 device_info_.device.input.sample_rate,
140 device_info_.device.input.frames_per_buffer,
141 device_info_.session_id,
142 device_info_.device.matched_output.sample_rate,
143 device_info_.device.matched_output.frames_per_buffer,
144 device_info_.device.input.effects));
146 if (render_view_id_ == -1) {
147 // Return true here to allow injecting a new source via
148 // SetCapturerSourceForTesting() at a later state.
152 MediaAudioConstraints audio_constraints(constraints_,
153 device_info_.device.input.effects);
154 if (!audio_constraints.IsValid())
157 media::ChannelLayout channel_layout = static_cast<media::ChannelLayout>(
158 device_info_.device.input.channel_layout);
159 DVLOG(1) << "Audio input hardware channel layout: " << channel_layout;
160 UMA_HISTOGRAM_ENUMERATION("WebRTC.AudioInputChannelLayout",
161 channel_layout, media::CHANNEL_LAYOUT_MAX + 1);
163 // Verify that the reported input channel configuration is supported.
164 if (channel_layout != media::CHANNEL_LAYOUT_MONO &&
165 channel_layout != media::CHANNEL_LAYOUT_STEREO &&
166 channel_layout != media::CHANNEL_LAYOUT_STEREO_AND_KEYBOARD_MIC) {
167 DLOG(ERROR) << channel_layout
168 << " is not a supported input channel configuration.";
172 DVLOG(1) << "Audio input hardware sample rate: "
173 << device_info_.device.input.sample_rate;
174 media::AudioSampleRate asr;
175 if (media::ToAudioSampleRate(device_info_.device.input.sample_rate, &asr)) {
176 UMA_HISTOGRAM_ENUMERATION(
177 "WebRTC.AudioInputSampleRate", asr, media::kAudioSampleRateMax + 1);
179 UMA_HISTOGRAM_COUNTS("WebRTC.AudioInputSampleRateUnexpected",
180 device_info_.device.input.sample_rate);
183 // Create and configure the default audio capturing source.
184 SetCapturerSource(AudioDeviceFactory::NewInputDevice(render_view_id_),
186 static_cast<float>(device_info_.device.input.sample_rate));
188 // Add the capturer to the WebRtcAudioDeviceImpl since it needs some hardware
189 // information from the capturer.
191 audio_device_->AddAudioCapturer(this);
196 WebRtcAudioCapturer::WebRtcAudioCapturer(
198 const StreamDeviceInfo& device_info,
199 const blink::WebMediaConstraints& constraints,
200 WebRtcAudioDeviceImpl* audio_device,
201 MediaStreamAudioSource* audio_source)
202 : constraints_(constraints),
204 new rtc::RefCountedObject<MediaStreamAudioProcessor>(
205 constraints, device_info.device.input.effects, audio_device)),
207 render_view_id_(render_view_id),
208 device_info_(device_info),
210 peer_connection_mode_(false),
212 need_audio_processing_(false),
213 audio_device_(audio_device),
214 audio_source_(audio_source),
215 audio_power_monitor_(
216 device_info_.device.input.sample_rate,
217 base::TimeDelta::FromMilliseconds(kPowerMonitorTimeConstantMs)) {
218 DVLOG(1) << "WebRtcAudioCapturer::WebRtcAudioCapturer()";
221 WebRtcAudioCapturer::~WebRtcAudioCapturer() {
222 DCHECK(thread_checker_.CalledOnValidThread());
223 DCHECK(tracks_.IsEmpty());
224 DVLOG(1) << "WebRtcAudioCapturer::~WebRtcAudioCapturer()";
228 void WebRtcAudioCapturer::AddTrack(WebRtcLocalAudioTrack* track) {
230 DVLOG(1) << "WebRtcAudioCapturer::AddTrack()";
233 base::AutoLock auto_lock(lock_);
234 // Verify that |track| is not already added to the list.
235 DCHECK(!tracks_.Contains(TrackOwner::TrackWrapper(track)));
237 // Add with a tag, so we remember to call OnSetFormat() on the new
239 scoped_refptr<TrackOwner> track_owner(new TrackOwner(track));
240 tracks_.AddAndTag(track_owner);
244 void WebRtcAudioCapturer::RemoveTrack(WebRtcLocalAudioTrack* track) {
245 DCHECK(thread_checker_.CalledOnValidThread());
246 DVLOG(1) << "WebRtcAudioCapturer::RemoveTrack()";
247 bool stop_source = false;
249 base::AutoLock auto_lock(lock_);
251 scoped_refptr<TrackOwner> removed_item =
252 tracks_.Remove(TrackOwner::TrackWrapper(track));
254 // Clear the delegate to ensure that no more capture callbacks will
255 // be sent to this sink. Also avoids a possible crash which can happen
256 // if this method is called while capturing is active.
257 if (removed_item.get()) {
258 removed_item->Reset();
259 stop_source = tracks_.IsEmpty();
263 // Since WebRtcAudioCapturer does not inherit MediaStreamAudioSource,
264 // and instead MediaStreamAudioSource is composed of a WebRtcAudioCapturer,
265 // we have to call StopSource on the MediaStreamSource. This will call
266 // MediaStreamAudioSource::DoStopSource which in turn call
267 // WebRtcAudioCapturerer::Stop();
268 audio_source_->StopSource();
272 void WebRtcAudioCapturer::SetCapturerSource(
273 const scoped_refptr<media::AudioCapturerSource>& source,
274 media::ChannelLayout channel_layout,
276 DCHECK(thread_checker_.CalledOnValidThread());
277 DVLOG(1) << "SetCapturerSource(channel_layout=" << channel_layout << ","
278 << "sample_rate=" << sample_rate << ")";
279 scoped_refptr<media::AudioCapturerSource> old_source;
281 base::AutoLock auto_lock(lock_);
282 if (source_.get() == source.get())
285 source_.swap(old_source);
288 // Reset the flag to allow starting the new source.
292 DVLOG(1) << "Switching to a new capture source.";
293 if (old_source.get())
296 // Dispatch the new parameters both to the sink(s) and to the new source,
297 // also apply the new |constraints|.
298 // The idea is to get rid of any dependency of the microphone parameters
299 // which would normally be used by default.
300 // bits_per_sample is always 16 for now.
301 int buffer_size = GetBufferSize(sample_rate);
302 media::AudioParameters params(media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
303 channel_layout, 0, sample_rate,
305 device_info_.device.input.effects);
308 base::AutoLock auto_lock(lock_);
309 // Notify the |audio_processor_| of the new format.
310 audio_processor_->OnCaptureFormatChanged(params);
312 MediaAudioConstraints audio_constraints(constraints_,
313 device_info_.device.input.effects);
314 need_audio_processing_ = audio_constraints.NeedsAudioProcessing();
315 // Notify all tracks about the new format.
320 source->Initialize(params, this, session_id());
325 void WebRtcAudioCapturer::EnablePeerConnectionMode() {
326 DCHECK(thread_checker_.CalledOnValidThread());
327 DVLOG(1) << "EnablePeerConnectionMode";
328 // Do nothing if the peer connection mode has been enabled.
329 if (peer_connection_mode_)
332 peer_connection_mode_ = true;
333 int render_view_id = -1;
334 media::AudioParameters input_params;
336 base::AutoLock auto_lock(lock_);
337 // Simply return if there is no existing source or the |render_view_id_| is
339 if (!source_.get() || render_view_id_== -1)
342 render_view_id = render_view_id_;
343 input_params = audio_processor_->InputFormat();
346 // Do nothing if the current buffer size is the WebRtc native buffer size.
347 if (GetBufferSize(input_params.sample_rate()) ==
348 input_params.frames_per_buffer()) {
352 // Create a new audio stream as source which will open the hardware using
353 // WebRtc native buffer size.
354 SetCapturerSource(AudioDeviceFactory::NewInputDevice(render_view_id),
355 input_params.channel_layout(),
356 static_cast<float>(input_params.sample_rate()));
359 void WebRtcAudioCapturer::Start() {
360 DCHECK(thread_checker_.CalledOnValidThread());
361 DVLOG(1) << "WebRtcAudioCapturer::Start()";
362 base::AutoLock auto_lock(lock_);
363 if (running_ || !source_)
366 // Start the data source, i.e., start capturing data from the current source.
367 // We need to set the AGC control before starting the stream.
368 source_->SetAutomaticGainControl(true);
373 void WebRtcAudioCapturer::Stop() {
374 DCHECK(thread_checker_.CalledOnValidThread());
375 DVLOG(1) << "WebRtcAudioCapturer::Stop()";
376 scoped_refptr<media::AudioCapturerSource> source;
377 TrackList::ItemList tracks;
379 base::AutoLock auto_lock(lock_);
384 tracks = tracks_.Items();
389 // Remove the capturer object from the WebRtcAudioDeviceImpl.
391 audio_device_->RemoveAudioCapturer(this);
393 for (TrackList::ItemList::const_iterator it = tracks.begin();
402 // Stop the audio processor to avoid feeding render data into the processor.
403 audio_processor_->Stop();
406 void WebRtcAudioCapturer::SetVolume(int volume) {
407 DVLOG(1) << "WebRtcAudioCapturer::SetVolume()";
408 DCHECK_LE(volume, MaxVolume());
409 double normalized_volume = static_cast<double>(volume) / MaxVolume();
410 base::AutoLock auto_lock(lock_);
412 source_->SetVolume(normalized_volume);
415 int WebRtcAudioCapturer::Volume() const {
416 base::AutoLock auto_lock(lock_);
420 int WebRtcAudioCapturer::MaxVolume() const {
421 return WebRtcAudioDeviceImpl::kMaxVolumeLevel;
424 void WebRtcAudioCapturer::Capture(const media::AudioBus* audio_source,
425 int audio_delay_milliseconds,
428 // This callback is driven by AudioInputDevice::AudioThreadCallback if
429 // |source_| is AudioInputDevice, otherwise it is driven by client's
431 #if defined(OS_WIN) || defined(OS_MACOSX)
432 DCHECK_LE(volume, 1.0);
433 #elif (defined(OS_LINUX) && !defined(OS_CHROMEOS)) || defined(OS_OPENBSD)
434 // We have a special situation on Linux where the microphone volume can be
435 // "higher than maximum". The input volume slider in the sound preference
436 // allows the user to set a scaling that is higher than 100%. It means that
437 // even if the reported maximum levels is N, the actual microphone level can
438 // go up to 1.5x*N and that corresponds to a normalized |volume| of 1.5x.
439 DCHECK_LE(volume, 1.6);
442 TrackList::ItemList tracks;
443 TrackList::ItemList tracks_to_notify_format;
444 int current_volume = 0;
445 base::TimeDelta audio_delay;
446 bool need_audio_processing = true;
448 base::AutoLock auto_lock(lock_);
452 // Map internal volume range of [0.0, 1.0] into [0, 255] used by AGC.
453 // The volume can be higher than 255 on Linux, and it will be cropped to
454 // 255 since AGC does not allow values out of range.
455 volume_ = static_cast<int>((volume * MaxVolume()) + 0.5);
456 current_volume = volume_ > MaxVolume() ? MaxVolume() : volume_;
457 audio_delay = base::TimeDelta::FromMilliseconds(audio_delay_milliseconds);
458 audio_delay_ = audio_delay;
459 key_pressed_ = key_pressed;
460 tracks = tracks_.Items();
461 tracks_.RetrieveAndClearTags(&tracks_to_notify_format);
463 // Set the flag to turn on the audio processing in PeerConnection level.
464 // Note that, we turn off the audio processing in PeerConnection if the
465 // processor has already processed the data.
466 need_audio_processing = need_audio_processing_ ?
467 !MediaStreamAudioProcessor::IsAudioTrackProcessingEnabled() : false;
470 DCHECK(audio_processor_->InputFormat().IsValid());
471 DCHECK_EQ(audio_source->channels(),
472 audio_processor_->InputFormat().channels());
473 DCHECK_EQ(audio_source->frames(),
474 audio_processor_->InputFormat().frames_per_buffer());
476 // Notify the tracks on when the format changes. This will do nothing if
477 // |tracks_to_notify_format| is empty.
478 media::AudioParameters output_params = audio_processor_->OutputFormat();
479 for (TrackList::ItemList::const_iterator it = tracks_to_notify_format.begin();
480 it != tracks_to_notify_format.end(); ++it) {
481 (*it)->OnSetFormat(output_params);
482 (*it)->SetAudioProcessor(audio_processor_);
485 if ((base::TimeTicks::Now() - last_audio_level_log_time_).InSeconds() >
486 kPowerMonitorLogIntervalSeconds) {
487 audio_power_monitor_.Scan(*audio_source, audio_source->frames());
489 last_audio_level_log_time_ = base::TimeTicks::Now();
491 std::pair<float, bool> result =
492 audio_power_monitor_.ReadCurrentPowerAndClip();
493 WebRtcLogMessage(base::StringPrintf(
494 "WAC::Capture: current_audio_power=%.2fdBFS.", result.first));
496 audio_power_monitor_.Reset();
499 // Push the data to the processor for processing.
500 audio_processor_->PushCaptureData(audio_source);
502 // Process and consume the data in the processor until there is not enough
503 // data in the processor.
504 int16* output = NULL;
506 while (audio_processor_->ProcessAndConsumeData(
507 audio_delay, current_volume, key_pressed, &new_volume, &output)) {
508 // Feed the post-processed data to the tracks.
509 for (TrackList::ItemList::const_iterator it = tracks.begin();
510 it != tracks.end(); ++it) {
511 (*it)->Capture(output, audio_delay, current_volume, key_pressed,
512 need_audio_processing);
516 SetVolume(new_volume);
518 // Update the |current_volume| to avoid passing the old volume to AGC.
519 current_volume = new_volume;
524 void WebRtcAudioCapturer::OnCaptureError() {
528 media::AudioParameters WebRtcAudioCapturer::source_audio_parameters() const {
529 base::AutoLock auto_lock(lock_);
530 return audio_processor_ ?
531 audio_processor_->InputFormat() : media::AudioParameters();
534 bool WebRtcAudioCapturer::GetPairedOutputParameters(
536 int* output_sample_rate,
537 int* output_frames_per_buffer) const {
538 // Don't set output parameters unless all of them are valid.
539 if (device_info_.session_id <= 0 ||
540 !device_info_.device.matched_output.sample_rate ||
541 !device_info_.device.matched_output.frames_per_buffer)
544 *session_id = device_info_.session_id;
545 *output_sample_rate = device_info_.device.matched_output.sample_rate;
546 *output_frames_per_buffer =
547 device_info_.device.matched_output.frames_per_buffer;
552 int WebRtcAudioCapturer::GetBufferSize(int sample_rate) const {
553 DCHECK(thread_checker_.CalledOnValidThread());
554 #if defined(OS_ANDROID)
555 // TODO(henrika): Tune and adjust buffer size on Android.
556 return (2 * sample_rate / 100);
559 // PeerConnection is running at a buffer size of 10ms data. A multiple of
560 // 10ms as the buffer size can give the best performance to PeerConnection.
561 int peer_connection_buffer_size = sample_rate / 100;
563 // Use the native hardware buffer size in non peer connection mode when the
564 // platform is using a native buffer size smaller than the PeerConnection
566 int hardware_buffer_size = device_info_.device.input.frames_per_buffer;
567 if (!peer_connection_mode_ && hardware_buffer_size &&
568 hardware_buffer_size <= peer_connection_buffer_size) {
569 return hardware_buffer_size;
572 return (sample_rate / 100);
575 void WebRtcAudioCapturer::GetAudioProcessingParams(
576 base::TimeDelta* delay, int* volume, bool* key_pressed) {
577 base::AutoLock auto_lock(lock_);
578 *delay = audio_delay_;
580 *key_pressed = key_pressed_;
583 void WebRtcAudioCapturer::SetCapturerSourceForTesting(
584 const scoped_refptr<media::AudioCapturerSource>& source,
585 media::AudioParameters params) {
586 // Create a new audio stream as source which uses the new source.
587 SetCapturerSource(source, params.channel_layout(),
588 static_cast<float>(params.sample_rate()));
591 } // namespace content