1 // Copyright 2018 The Chromium Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/browser/renderer_host/media/render_frame_audio_input_stream_factory.h"
11 #include "base/check_op.h"
12 #include "base/functional/bind.h"
13 #include "base/functional/callback.h"
14 #include "base/functional/callback_helpers.h"
15 #include "base/location.h"
16 #include "base/memory/raw_ptr.h"
17 #include "base/memory/weak_ptr.h"
18 #include "base/trace_event/trace_event.h"
19 #include "base/unguessable_token.h"
20 #include "content/browser/media/capture/desktop_capture_device_uma_types.h"
21 #include "content/browser/media/forwarding_audio_stream_factory.h"
22 #include "content/browser/media/media_devices_permission_checker.h"
23 #include "content/browser/media/media_devices_util.h"
24 #include "content/browser/renderer_host/media/audio_input_device_manager.h"
25 #include "content/browser/renderer_host/media/media_devices_manager.h"
26 #include "content/browser/renderer_host/media/media_stream_manager.h"
27 #include "content/public/browser/audio_stream_broker.h"
28 #include "content/public/browser/browser_task_traits.h"
29 #include "content/public/browser/browser_thread.h"
30 #include "content/public/browser/global_routing_id.h"
31 #include "content/public/browser/media_device_id.h"
32 #include "content/public/browser/render_frame_host.h"
33 #include "content/public/browser/render_process_host.h"
34 #include "content/public/browser/web_contents_media_capture_id.h"
35 #include "media/audio/audio_device_description.h"
36 #include "media/base/audio_parameters.h"
37 #include "mojo/public/cpp/bindings/pending_remote.h"
38 #include "mojo/public/cpp/bindings/receiver.h"
39 #include "third_party/blink/public/common/mediastream/media_stream_request.h"
40 #include "url/origin.h"
42 #if BUILDFLAG(IS_TIZEN_TV)
43 #include "content/browser/browser_main_loop.h"
46 using blink::mojom::MediaDeviceType;
52 AudioStreamBroker::LoopbackSource* GetLoopbackSourceOnUIThread(
53 int render_process_id,
54 int render_frame_id) {
55 DCHECK_CURRENTLY_ON(BrowserThread::UI);
56 auto* source = ForwardingAudioStreamFactory::CoreForFrame(
57 (RenderFrameHost::FromID(render_process_id, render_frame_id)));
59 // The source of the capture has already been destroyed, so fail early.
62 // Note: this pointer is sent over to the IO thread. This is safe since the
63 // destruction of |source| is posted to the IO thread and it hasn't been
68 void EnumerateOutputDevices(MediaStreamManager* media_stream_manager,
69 MediaDevicesManager::EnumerationCallback cb) {
70 DCHECK_CURRENTLY_ON(BrowserThread::IO);
71 MediaDevicesManager::BoolDeviceTypes device_types;
72 device_types[static_cast<size_t>(MediaDeviceType::kMediaAudioOuput)] = true;
73 media_stream_manager->media_devices_manager()->EnumerateDevices(
74 device_types, std::move(cb));
77 void TranslateDeviceId(const std::string& device_id,
78 const MediaDeviceSaltAndOrigin& salt_and_origin,
79 base::RepeatingCallback<void(const std::string&)> cb,
80 const MediaDeviceEnumeration& device_array) {
81 DCHECK_CURRENTLY_ON(BrowserThread::IO);
82 for (const auto& device_info :
83 device_array[static_cast<size_t>(MediaDeviceType::kMediaAudioOuput)]) {
84 if (DoesRawMediaDeviceIDMatchHMAC(salt_and_origin, device_id,
85 device_info.device_id)) {
86 cb.Run(device_info.device_id);
90 // If we're unable to translate the device id, |cb| will not be run.
93 void GotSaltAndOrigin(
96 base::OnceCallback<void(const MediaDeviceSaltAndOrigin& salt_and_origin,
98 const MediaDeviceSaltAndOrigin& salt_and_origin) {
99 bool access = MediaDevicesPermissionChecker().CheckPermissionOnUIThread(
100 MediaDeviceType::kMediaAudioOuput, process_id, frame_id);
101 GetIOThreadTaskRunner({})->PostTask(
102 FROM_HERE, base::BindOnce(std::move(cb), salt_and_origin, access));
105 void GetSaltOriginAndPermissionsOnUIThread(
108 base::OnceCallback<void(const MediaDeviceSaltAndOrigin& salt_and_origin,
109 bool has_access)> cb) {
110 GetMediaDeviceSaltAndOrigin(
111 GlobalRenderFrameHostId(process_id, frame_id),
112 base::BindOnce(&GotSaltAndOrigin, process_id, frame_id, std::move(cb)));
117 class RenderFrameAudioInputStreamFactory::Core final
118 : public blink::mojom::RendererAudioInputStreamFactory {
120 Core(mojo::PendingReceiver<blink::mojom::RendererAudioInputStreamFactory>
122 MediaStreamManager* media_stream_manager,
123 RenderFrameHost* render_frame_host);
125 Core(const Core&) = delete;
126 Core& operator=(const Core&) = delete;
130 void Init(mojo::PendingReceiver<blink::mojom::RendererAudioInputStreamFactory>
133 // mojom::RendererAudioInputStreamFactory implementation.
135 mojo::PendingRemote<blink::mojom::RendererAudioInputStreamFactoryClient>
137 const base::UnguessableToken& session_id,
138 const media::AudioParameters& audio_params,
139 bool automatic_gain_control,
140 uint32_t shared_memory_count,
141 media::mojom::AudioProcessingConfigPtr processing_config) final;
143 void AssociateInputAndOutputForAec(
144 const base::UnguessableToken& input_stream_id,
145 const std::string& output_device_id) final;
147 void CreateLoopbackStream(
148 mojo::PendingRemote<blink::mojom::RendererAudioInputStreamFactoryClient>
150 const media::AudioParameters& audio_params,
151 uint32_t shared_memory_count,
152 bool disable_local_echo,
153 AudioStreamBroker::LoopbackSource* loopback_source);
155 void AssociateInputAndOutputForAecAfterCheckingAccess(
156 const base::UnguessableToken& input_stream_id,
157 const std::string& output_device_id,
158 const MediaDeviceSaltAndOrigin& salt_and_origin,
159 bool access_granted);
161 void AssociateTranslatedOutputDeviceForAec(
162 const base::UnguessableToken& input_stream_id,
163 const std::string& raw_output_device_id);
165 #if BUILDFLAG(IS_TIZEN_TV)
166 void OnMediaStateChanged(uint32_t previous, uint32_t current) final;
167 void NotifyMediaStateChanged(uint32_t previous, uint32_t current);
170 const raw_ptr<MediaStreamManager> media_stream_manager_;
171 const int process_id_;
174 mojo::Receiver<RendererAudioInputStreamFactory> receiver_{this};
175 // Always null-check this weak pointer before dereferencing it.
176 base::WeakPtr<ForwardingAudioStreamFactory::Core> forwarding_factory_;
178 base::WeakPtrFactory<Core> weak_ptr_factory_{this};
181 RenderFrameAudioInputStreamFactory::RenderFrameAudioInputStreamFactory(
182 mojo::PendingReceiver<blink::mojom::RendererAudioInputStreamFactory>
184 MediaStreamManager* media_stream_manager,
185 RenderFrameHost* render_frame_host)
186 : core_(new Core(std::move(receiver),
187 media_stream_manager,
188 render_frame_host)) {
189 DCHECK_CURRENTLY_ON(BrowserThread::UI);
192 RenderFrameAudioInputStreamFactory::~RenderFrameAudioInputStreamFactory() {
193 DCHECK_CURRENTLY_ON(BrowserThread::UI);
194 // Ensure |core_| is deleted on the right thread. DeleteOnIOThread isn't used
195 // as it doesn't post in case it is already executed on the right thread. That
196 // causes issues in unit tests where the UI thread and the IO thread are the
198 GetIOThreadTaskRunner({})->PostTask(
199 FROM_HERE, base::DoNothingWithBoundArgs(std::move(core_)));
202 RenderFrameAudioInputStreamFactory::Core::Core(
203 mojo::PendingReceiver<blink::mojom::RendererAudioInputStreamFactory>
205 MediaStreamManager* media_stream_manager,
206 RenderFrameHost* render_frame_host)
207 : media_stream_manager_(media_stream_manager),
208 process_id_(render_frame_host->GetProcess()->GetID()),
209 frame_id_(render_frame_host->GetRoutingID()) {
210 DCHECK_CURRENTLY_ON(BrowserThread::UI);
212 ForwardingAudioStreamFactory::Core* tmp_factory =
213 ForwardingAudioStreamFactory::CoreForFrame(render_frame_host);
216 // The only case when we not have a forwarding factory at this point is when
217 // the frame belongs to an interstitial. Interstitials don't need audio, so
218 // it's fine to drop the receiver.
222 forwarding_factory_ = tmp_factory->AsWeakPtr();
224 // Unretained is safe since the destruction of |this| is posted to the IO
226 GetIOThreadTaskRunner({})->PostTask(
228 base::BindOnce(&Core::Init, base::Unretained(this), std::move(receiver)));
231 RenderFrameAudioInputStreamFactory::Core::~Core() {
232 DCHECK_CURRENTLY_ON(BrowserThread::IO);
235 void RenderFrameAudioInputStreamFactory::Core::Init(
236 mojo::PendingReceiver<blink::mojom::RendererAudioInputStreamFactory>
238 DCHECK_CURRENTLY_ON(BrowserThread::IO);
239 receiver_.Bind(std::move(receiver));
242 void RenderFrameAudioInputStreamFactory::Core::CreateStream(
243 mojo::PendingRemote<blink::mojom::RendererAudioInputStreamFactoryClient>
245 const base::UnguessableToken& session_id,
246 const media::AudioParameters& audio_params,
247 bool automatic_gain_control,
248 uint32_t shared_memory_count,
249 media::mojom::AudioProcessingConfigPtr processing_config) {
250 DCHECK_CURRENTLY_ON(BrowserThread::IO);
251 TRACE_EVENT1("audio", "RenderFrameAudioInputStreamFactory::CreateStream",
252 "session id", session_id.ToString());
254 if (!forwarding_factory_)
257 const blink::MediaStreamDevice* device =
258 media_stream_manager_->audio_input_device_manager()->GetOpenedDeviceById(
262 TRACE_EVENT_INSTANT0("audio", "device not found", TRACE_EVENT_SCOPE_THREAD);
266 WebContentsMediaCaptureId capture_id;
267 if (WebContentsMediaCaptureId::Parse(device->id, &capture_id)) {
268 // For MEDIA_GUM_DESKTOP_AUDIO_CAPTURE, the source is selected from
269 // picker window, we do not mute the source audio. For
270 // MEDIA_GUM_TAB_AUDIO_CAPTURE, the probable use case is Cast, we mute
272 // TODO(qiangchen): Analyze audio constraints to make a duplicating or
273 // diverting decision. It would give web developer more flexibility.
275 GetUIThreadTaskRunner({})->PostTaskAndReplyWithResult(
277 base::BindOnce(&GetLoopbackSourceOnUIThread,
278 capture_id.render_process_id,
279 capture_id.main_render_frame_id),
281 &RenderFrameAudioInputStreamFactory::Core::CreateLoopbackStream,
282 weak_ptr_factory_.GetWeakPtr(), std::move(client), audio_params,
283 shared_memory_count, capture_id.disable_local_echo));
286 blink::mojom::MediaStreamType::GUM_DESKTOP_AUDIO_CAPTURE)
287 IncrementDesktopCaptureCounter(SYSTEM_LOOPBACK_AUDIO_CAPTURER_CREATED);
290 forwarding_factory_->CreateInputStream(
291 process_id_, frame_id_, device->id, audio_params, shared_memory_count,
292 automatic_gain_control, std::move(processing_config),
295 // Only count for captures from desktop media picker dialog and system loop
298 blink::mojom::MediaStreamType::GUM_DESKTOP_AUDIO_CAPTURE &&
299 (media::AudioDeviceDescription::IsLoopbackDevice(device->id))) {
300 IncrementDesktopCaptureCounter(SYSTEM_LOOPBACK_AUDIO_CAPTURER_CREATED);
305 void RenderFrameAudioInputStreamFactory::Core::CreateLoopbackStream(
306 mojo::PendingRemote<blink::mojom::RendererAudioInputStreamFactoryClient>
308 const media::AudioParameters& audio_params,
309 uint32_t shared_memory_count,
310 bool disable_local_echo,
311 AudioStreamBroker::LoopbackSource* loopback_source) {
312 if (!loopback_source || !forwarding_factory_)
315 forwarding_factory_->CreateLoopbackStream(
316 process_id_, frame_id_, loopback_source, audio_params,
317 shared_memory_count, disable_local_echo, std::move(client));
320 void RenderFrameAudioInputStreamFactory::Core::AssociateInputAndOutputForAec(
321 const base::UnguessableToken& input_stream_id,
322 const std::string& output_device_id) {
323 DCHECK_CURRENTLY_ON(BrowserThread::IO);
324 if (!IsValidDeviceId(output_device_id))
327 GetUIThreadTaskRunner({})->PostTask(
330 &GetSaltOriginAndPermissionsOnUIThread, process_id_, frame_id_,
332 &Core::AssociateInputAndOutputForAecAfterCheckingAccess,
333 weak_ptr_factory_.GetWeakPtr(), input_stream_id,
337 void RenderFrameAudioInputStreamFactory::Core::
338 AssociateInputAndOutputForAecAfterCheckingAccess(
339 const base::UnguessableToken& input_stream_id,
340 const std::string& output_device_id,
341 const MediaDeviceSaltAndOrigin& salt_and_origin,
342 bool access_granted) {
343 DCHECK_CURRENTLY_ON(BrowserThread::IO);
345 if (!forwarding_factory_ || !access_granted)
348 if (media::AudioDeviceDescription::IsDefaultDevice(output_device_id) ||
349 media::AudioDeviceDescription::IsCommunicationsDevice(output_device_id)) {
350 forwarding_factory_->AssociateInputAndOutputForAec(input_stream_id,
353 EnumerateOutputDevices(
354 media_stream_manager_,
355 base::BindOnce(&TranslateDeviceId, output_device_id, salt_and_origin,
357 &RenderFrameAudioInputStreamFactory::Core::
358 AssociateTranslatedOutputDeviceForAec,
359 weak_ptr_factory_.GetWeakPtr(), input_stream_id)));
363 void RenderFrameAudioInputStreamFactory::Core::
364 AssociateTranslatedOutputDeviceForAec(
365 const base::UnguessableToken& input_stream_id,
366 const std::string& raw_output_device_id) {
367 DCHECK_CURRENTLY_ON(BrowserThread::IO);
368 if (!forwarding_factory_)
370 forwarding_factory_->AssociateInputAndOutputForAec(input_stream_id,
371 raw_output_device_id);
374 #if BUILDFLAG(IS_TIZEN_TV)
375 void RenderFrameAudioInputStreamFactory::Core::OnMediaStateChanged(
378 DCHECK_CURRENTLY_ON(BrowserThread::IO);
379 GetUIThreadTaskRunner({})->PostTask(
381 base::BindOnce(&Core::NotifyMediaStateChanged,
382 weak_ptr_factory_.GetWeakPtr(), previous, current));
385 void RenderFrameAudioInputStreamFactory::Core::NotifyMediaStateChanged(
388 content::BrowserMainLoop* browser_main_loop =
389 content::BrowserMainLoop::GetInstance();
390 if (!browser_main_loop) {
391 LOG(ERROR) << "browser_main_loop is null";
394 content::MediaStreamManager* msm = browser_main_loop->media_stream_manager();
396 LOG(ERROR) << "MediaStreamManager is null";
399 msm->NotifyMediaStateChanged(content::MediaInputStreamType::AUDIO_CAPTURE,
403 } // namespace content