X-Git-Url: http://review.tizen.org/git/?a=blobdiff_plain;f=content%2Fbrowser%2Frenderer_host%2Fmedia%2Frender_frame_audio_input_stream_factory.cc;h=0e0488e6b113dc563649c859f97902cc714083ec;hb=dc040ada5512bfacb5a6c65399e22d0f70659bbc;hp=a07394101ea0ab046762040bb4dfa1aeae4a4772;hpb=6bb4df8e136ee3a6cc82dabb6d0743605295a5fb;p=platform%2Fframework%2Fweb%2Fchromium-efl.git diff --git a/content/browser/renderer_host/media/render_frame_audio_input_stream_factory.cc b/content/browser/renderer_host/media/render_frame_audio_input_stream_factory.cc index a073941..0e0488e 100644 --- a/content/browser/renderer_host/media/render_frame_audio_input_stream_factory.cc +++ b/content/browser/renderer_host/media/render_frame_audio_input_stream_factory.cc @@ -1,51 +1,75 @@ -// Copyright 2018 The Chromium Authors. All rights reserved. +// Copyright 2018 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. #include "content/browser/renderer_host/media/render_frame_audio_input_stream_factory.h" +#include #include #include +#include "base/check_op.h" +#include "base/functional/bind.h" +#include "base/functional/callback.h" +#include "base/functional/callback_helpers.h" +#include "base/location.h" +#include "base/memory/raw_ptr.h" +#include "base/memory/weak_ptr.h" #include "base/trace_event/trace_event.h" -#include "content/browser/browser_main_loop.h" +#include "base/unguessable_token.h" #include "content/browser/media/capture/desktop_capture_device_uma_types.h" #include "content/browser/media/forwarding_audio_stream_factory.h" #include "content/browser/media/media_devices_permission_checker.h" +#include "content/browser/media/media_devices_util.h" +#include "content/browser/renderer_host/media/audio_input_device_manager.h" +#include "content/browser/renderer_host/media/media_devices_manager.h" #include "content/browser/renderer_host/media/media_stream_manager.h" +#include "content/public/browser/audio_stream_broker.h" +#include "content/public/browser/browser_task_traits.h" #include "content/public/browser/browser_thread.h" +#include "content/public/browser/global_routing_id.h" #include "content/public/browser/media_device_id.h" #include "content/public/browser/render_frame_host.h" #include "content/public/browser/render_process_host.h" #include "content/public/browser/web_contents_media_capture_id.h" -#include "content/public/common/media_stream_request.h" #include "media/audio/audio_device_description.h" -#include "media/audio/audio_input_device.h" #include "media/base/audio_parameters.h" +#include "mojo/public/cpp/bindings/pending_remote.h" +#include "mojo/public/cpp/bindings/receiver.h" +#include "third_party/blink/public/common/mediastream/media_stream_request.h" +#include "url/origin.h" + +#if BUILDFLAG(IS_TIZEN_TV) +#include "content/browser/browser_main_loop.h" +#endif + +using blink::mojom::MediaDeviceType; namespace content { namespace { -void LookUpDeviceAndRespondIfFound( - scoped_refptr audio_input_device_manager, - int32_t session_id, - base::OnceCallback response) { - DCHECK_CURRENTLY_ON(BrowserThread::IO); - const MediaStreamDevice* device = - audio_input_device_manager->GetOpenedDeviceById(session_id); - if (device) { - // Copies device. - BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, - base::BindOnce(std::move(response), *device)); +AudioStreamBroker::LoopbackSource* GetLoopbackSourceOnUIThread( + int render_process_id, + int render_frame_id) { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + auto* source = ForwardingAudioStreamFactory::CoreForFrame( + (RenderFrameHost::FromID(render_process_id, render_frame_id))); + if (!source) { + // The source of the capture has already been destroyed, so fail early. + return nullptr; } + // Note: this pointer is sent over to the IO thread. This is safe since the + // destruction of |source| is posted to the IO thread and it hasn't been + // posted yet. + return source; } void EnumerateOutputDevices(MediaStreamManager* media_stream_manager, MediaDevicesManager::EnumerationCallback cb) { DCHECK_CURRENTLY_ON(BrowserThread::IO); MediaDevicesManager::BoolDeviceTypes device_types; - device_types[MEDIA_DEVICE_TYPE_AUDIO_OUTPUT] = true; + device_types[static_cast(MediaDeviceType::kMediaAudioOuput)] = true; media_stream_manager->media_devices_manager()->EnumerateDevices( device_types, std::move(cb)); } @@ -55,162 +79,325 @@ void TranslateDeviceId(const std::string& device_id, base::RepeatingCallback cb, const MediaDeviceEnumeration& device_array) { DCHECK_CURRENTLY_ON(BrowserThread::IO); - for (const auto& device_info : device_array[MEDIA_DEVICE_TYPE_AUDIO_OUTPUT]) { - if (MediaStreamManager::DoesMediaDeviceIDMatchHMAC( - salt_and_origin.device_id_salt, salt_and_origin.origin, device_id, - device_info.device_id)) { - BrowserThread::PostTask( - BrowserThread::UI, FROM_HERE, - base::BindOnce(std::move(cb), device_info.device_id)); + for (const auto& device_info : + device_array[static_cast(MediaDeviceType::kMediaAudioOuput)]) { + if (DoesRawMediaDeviceIDMatchHMAC(salt_and_origin, device_id, + device_info.device_id)) { + cb.Run(device_info.device_id); break; } } // If we're unable to translate the device id, |cb| will not be run. } +void GotSaltAndOrigin( + int process_id, + int frame_id, + base::OnceCallback cb, + const MediaDeviceSaltAndOrigin& salt_and_origin) { + bool access = MediaDevicesPermissionChecker().CheckPermissionOnUIThread( + MediaDeviceType::kMediaAudioOuput, process_id, frame_id); + GetIOThreadTaskRunner({})->PostTask( + FROM_HERE, base::BindOnce(std::move(cb), salt_and_origin, access)); +} + +void GetSaltOriginAndPermissionsOnUIThread( + int process_id, + int frame_id, + base::OnceCallback cb) { + GetMediaDeviceSaltAndOrigin( + GlobalRenderFrameHostId(process_id, frame_id), + base::BindOnce(&GotSaltAndOrigin, process_id, frame_id, std::move(cb))); +} + } // namespace +class RenderFrameAudioInputStreamFactory::Core final + : public blink::mojom::RendererAudioInputStreamFactory { + public: + Core(mojo::PendingReceiver + receiver, + MediaStreamManager* media_stream_manager, + RenderFrameHost* render_frame_host); + + Core(const Core&) = delete; + Core& operator=(const Core&) = delete; + + ~Core() final; + + void Init(mojo::PendingReceiver + receiver); + + // mojom::RendererAudioInputStreamFactory implementation. + void CreateStream( + mojo::PendingRemote + client, + const base::UnguessableToken& session_id, + const media::AudioParameters& audio_params, + bool automatic_gain_control, + uint32_t shared_memory_count, + media::mojom::AudioProcessingConfigPtr processing_config) final; + + void AssociateInputAndOutputForAec( + const base::UnguessableToken& input_stream_id, + const std::string& output_device_id) final; + + void CreateLoopbackStream( + mojo::PendingRemote + client, + const media::AudioParameters& audio_params, + uint32_t shared_memory_count, + bool disable_local_echo, + AudioStreamBroker::LoopbackSource* loopback_source); + + void AssociateInputAndOutputForAecAfterCheckingAccess( + const base::UnguessableToken& input_stream_id, + const std::string& output_device_id, + const MediaDeviceSaltAndOrigin& salt_and_origin, + bool access_granted); + + void AssociateTranslatedOutputDeviceForAec( + const base::UnguessableToken& input_stream_id, + const std::string& raw_output_device_id); + +#if BUILDFLAG(IS_TIZEN_TV) + void OnMediaStateChanged(uint32_t previous, uint32_t current) final; + void NotifyMediaStateChanged(uint32_t previous, uint32_t current); +#endif + + const raw_ptr media_stream_manager_; + const int process_id_; + const int frame_id_; + + mojo::Receiver receiver_{this}; + // Always null-check this weak pointer before dereferencing it. + base::WeakPtr forwarding_factory_; + + base::WeakPtrFactory weak_ptr_factory_{this}; +}; + RenderFrameAudioInputStreamFactory::RenderFrameAudioInputStreamFactory( - mojom::RendererAudioInputStreamFactoryRequest request, - scoped_refptr audio_input_device_manager, + mojo::PendingReceiver + receiver, + MediaStreamManager* media_stream_manager, RenderFrameHost* render_frame_host) - : binding_(this, std::move(request)), - audio_input_device_manager_(std::move(audio_input_device_manager)), - render_frame_host_(render_frame_host), - weak_ptr_factory_(this) { + : core_(new Core(std::move(receiver), + media_stream_manager, + render_frame_host)) { DCHECK_CURRENTLY_ON(BrowserThread::UI); } RenderFrameAudioInputStreamFactory::~RenderFrameAudioInputStreamFactory() { DCHECK_CURRENTLY_ON(BrowserThread::UI); + // Ensure |core_| is deleted on the right thread. DeleteOnIOThread isn't used + // as it doesn't post in case it is already executed on the right thread. That + // causes issues in unit tests where the UI thread and the IO thread are the + // same. + GetIOThreadTaskRunner({})->PostTask( + FROM_HERE, base::DoNothingWithBoundArgs(std::move(core_))); } -void RenderFrameAudioInputStreamFactory::CreateStream( - mojom::RendererAudioInputStreamFactoryClientPtr client, - int32_t session_id, - const media::AudioParameters& audio_params, - bool automatic_gain_control, - uint32_t shared_memory_count) { +RenderFrameAudioInputStreamFactory::Core::Core( + mojo::PendingReceiver + receiver, + MediaStreamManager* media_stream_manager, + RenderFrameHost* render_frame_host) + : media_stream_manager_(media_stream_manager), + process_id_(render_frame_host->GetProcess()->GetID()), + frame_id_(render_frame_host->GetRoutingID()) { DCHECK_CURRENTLY_ON(BrowserThread::UI); - TRACE_EVENT_INSTANT1("audio", - "RenderFrameAudioInputStreamFactory::CreateStream", - TRACE_EVENT_SCOPE_THREAD, "session id", session_id); - BrowserThread::PostTask( - BrowserThread::IO, FROM_HERE, - base::BindOnce( - &LookUpDeviceAndRespondIfFound, audio_input_device_manager_, - session_id, - base::BindOnce(&RenderFrameAudioInputStreamFactory:: - CreateStreamAfterLookingUpDevice, - weak_ptr_factory_.GetWeakPtr(), std::move(client), - audio_params, automatic_gain_control, - shared_memory_count))); + ForwardingAudioStreamFactory::Core* tmp_factory = + ForwardingAudioStreamFactory::CoreForFrame(render_frame_host); + + if (!tmp_factory) { + // The only case when we not have a forwarding factory at this point is when + // the frame belongs to an interstitial. Interstitials don't need audio, so + // it's fine to drop the receiver. + return; + } + + forwarding_factory_ = tmp_factory->AsWeakPtr(); + + // Unretained is safe since the destruction of |this| is posted to the IO + // thread. + GetIOThreadTaskRunner({})->PostTask( + FROM_HERE, + base::BindOnce(&Core::Init, base::Unretained(this), std::move(receiver))); +} + +RenderFrameAudioInputStreamFactory::Core::~Core() { + DCHECK_CURRENTLY_ON(BrowserThread::IO); +} + +void RenderFrameAudioInputStreamFactory::Core::Init( + mojo::PendingReceiver + receiver) { + DCHECK_CURRENTLY_ON(BrowserThread::IO); + receiver_.Bind(std::move(receiver)); } -void RenderFrameAudioInputStreamFactory::CreateStreamAfterLookingUpDevice( - mojom::RendererAudioInputStreamFactoryClientPtr client, +void RenderFrameAudioInputStreamFactory::Core::CreateStream( + mojo::PendingRemote + client, + const base::UnguessableToken& session_id, const media::AudioParameters& audio_params, bool automatic_gain_control, uint32_t shared_memory_count, - const MediaStreamDevice& device) { - TRACE_EVENT1( - "audio", - "RenderFrameAudioInputStreamFactory::CreateStreamAfterLookingUpDevice", - "device id", device.id); - DCHECK_CURRENTLY_ON(BrowserThread::UI); - ForwardingAudioStreamFactory* factory = - ForwardingAudioStreamFactory::ForFrame(render_frame_host_); - if (!factory) + media::mojom::AudioProcessingConfigPtr processing_config) { + DCHECK_CURRENTLY_ON(BrowserThread::IO); + TRACE_EVENT1("audio", "RenderFrameAudioInputStreamFactory::CreateStream", + "session id", session_id.ToString()); + + if (!forwarding_factory_) + return; + + const blink::MediaStreamDevice* device = + media_stream_manager_->audio_input_device_manager()->GetOpenedDeviceById( + session_id); + + if (!device) { + TRACE_EVENT_INSTANT0("audio", "device not found", TRACE_EVENT_SCOPE_THREAD); return; + } WebContentsMediaCaptureId capture_id; - if (WebContentsMediaCaptureId::Parse(device.id, &capture_id)) { - // For MEDIA_DESKTOP_AUDIO_CAPTURE, the source is selected from picker - // window, we do not mute the source audio. - // For MEDIA_TAB_AUDIO_CAPTURE, the probable use case is Cast, we mute + if (WebContentsMediaCaptureId::Parse(device->id, &capture_id)) { + // For MEDIA_GUM_DESKTOP_AUDIO_CAPTURE, the source is selected from + // picker window, we do not mute the source audio. For + // MEDIA_GUM_TAB_AUDIO_CAPTURE, the probable use case is Cast, we mute // the source audio. // TODO(qiangchen): Analyze audio constraints to make a duplicating or // diverting decision. It would give web developer more flexibility. - RenderFrameHost* source_host = RenderFrameHost::FromID( - capture_id.render_process_id, capture_id.main_render_frame_id); - if (!source_host) { - // The source of the capture has already been destroyed, so fail early. - return; - } - - factory->CreateLoopbackStream( - render_frame_host_, source_host, audio_params, shared_memory_count, - capture_id.disable_local_echo, std::move(client)); + GetUIThreadTaskRunner({})->PostTaskAndReplyWithResult( + FROM_HERE, + base::BindOnce(&GetLoopbackSourceOnUIThread, + capture_id.render_process_id, + capture_id.main_render_frame_id), + base::BindOnce( + &RenderFrameAudioInputStreamFactory::Core::CreateLoopbackStream, + weak_ptr_factory_.GetWeakPtr(), std::move(client), audio_params, + shared_memory_count, capture_id.disable_local_echo)); - if (device.type == MEDIA_DESKTOP_AUDIO_CAPTURE) + if (device->type == + blink::mojom::MediaStreamType::GUM_DESKTOP_AUDIO_CAPTURE) IncrementDesktopCaptureCounter(SYSTEM_LOOPBACK_AUDIO_CAPTURER_CREATED); + return; } else { - factory->CreateInputStream(render_frame_host_, device.id, audio_params, - shared_memory_count, automatic_gain_control, - std::move(client)); + forwarding_factory_->CreateInputStream( + process_id_, frame_id_, device->id, audio_params, shared_memory_count, + automatic_gain_control, std::move(processing_config), + std::move(client)); // Only count for captures from desktop media picker dialog and system loop // back audio. - if (device.type == MEDIA_DESKTOP_AUDIO_CAPTURE && - (media::AudioDeviceDescription::IsLoopbackDevice(device.id))) { + if (device->type == + blink::mojom::MediaStreamType::GUM_DESKTOP_AUDIO_CAPTURE && + (media::AudioDeviceDescription::IsLoopbackDevice(device->id))) { IncrementDesktopCaptureCounter(SYSTEM_LOOPBACK_AUDIO_CAPTURER_CREATED); } } } -void RenderFrameAudioInputStreamFactory::AssociateInputAndOutputForAec( +void RenderFrameAudioInputStreamFactory::Core::CreateLoopbackStream( + mojo::PendingRemote + client, + const media::AudioParameters& audio_params, + uint32_t shared_memory_count, + bool disable_local_echo, + AudioStreamBroker::LoopbackSource* loopback_source) { + if (!loopback_source || !forwarding_factory_) + return; + + forwarding_factory_->CreateLoopbackStream( + process_id_, frame_id_, loopback_source, audio_params, + shared_memory_count, disable_local_echo, std::move(client)); +} + +void RenderFrameAudioInputStreamFactory::Core::AssociateInputAndOutputForAec( const base::UnguessableToken& input_stream_id, const std::string& output_device_id) { - DCHECK_CURRENTLY_ON(BrowserThread::UI); + DCHECK_CURRENTLY_ON(BrowserThread::IO); if (!IsValidDeviceId(output_device_id)) return; - ForwardingAudioStreamFactory* factory = - ForwardingAudioStreamFactory::ForFrame(render_frame_host_); - if (!factory) - return; + GetUIThreadTaskRunner({})->PostTask( + FROM_HERE, + base::BindOnce( + &GetSaltOriginAndPermissionsOnUIThread, process_id_, frame_id_, + base::BindOnce( + &Core::AssociateInputAndOutputForAecAfterCheckingAccess, + weak_ptr_factory_.GetWeakPtr(), input_stream_id, + output_device_id))); +} - const int process_id = render_frame_host_->GetProcess()->GetID(); - const int frame_id = render_frame_host_->GetRoutingID(); - auto salt_and_origin = GetMediaDeviceSaltAndOrigin(process_id, frame_id); +void RenderFrameAudioInputStreamFactory::Core:: + AssociateInputAndOutputForAecAfterCheckingAccess( + const base::UnguessableToken& input_stream_id, + const std::string& output_device_id, + const MediaDeviceSaltAndOrigin& salt_and_origin, + bool access_granted) { + DCHECK_CURRENTLY_ON(BrowserThread::IO); - // Check permissions for everything but the default device - if (!media::AudioDeviceDescription::IsDefaultDevice(output_device_id) && - !MediaDevicesPermissionChecker().CheckPermissionOnUIThread( - MEDIA_DEVICE_TYPE_AUDIO_OUTPUT, process_id, frame_id)) { + if (!forwarding_factory_ || !access_granted) return; - } if (media::AudioDeviceDescription::IsDefaultDevice(output_device_id) || media::AudioDeviceDescription::IsCommunicationsDevice(output_device_id)) { - factory->AssociateInputAndOutputForAec(input_stream_id, output_device_id); + forwarding_factory_->AssociateInputAndOutputForAec(input_stream_id, + output_device_id); } else { - auto* media_stream_manager = - BrowserMainLoop::GetInstance()->media_stream_manager(); - BrowserThread::PostTask( - BrowserThread::IO, FROM_HERE, - base::BindOnce( - EnumerateOutputDevices, media_stream_manager, - base::BindRepeating( - &TranslateDeviceId, output_device_id, salt_and_origin, - base::BindRepeating(&RenderFrameAudioInputStreamFactory:: - AssociateTranslatedOutputDeviceForAec, - weak_ptr_factory_.GetWeakPtr(), - input_stream_id)))); + EnumerateOutputDevices( + media_stream_manager_, + base::BindOnce(&TranslateDeviceId, output_device_id, salt_and_origin, + base::BindRepeating( + &RenderFrameAudioInputStreamFactory::Core:: + AssociateTranslatedOutputDeviceForAec, + weak_ptr_factory_.GetWeakPtr(), input_stream_id))); } } -void RenderFrameAudioInputStreamFactory::AssociateTranslatedOutputDeviceForAec( - const base::UnguessableToken& input_stream_id, - const std::string& raw_output_device_id) { - DCHECK_CURRENTLY_ON(BrowserThread::UI); - ForwardingAudioStreamFactory* factory = - ForwardingAudioStreamFactory::ForFrame(render_frame_host_); - if (factory) - factory->AssociateInputAndOutputForAec(input_stream_id, - raw_output_device_id); +void RenderFrameAudioInputStreamFactory::Core:: + AssociateTranslatedOutputDeviceForAec( + const base::UnguessableToken& input_stream_id, + const std::string& raw_output_device_id) { + DCHECK_CURRENTLY_ON(BrowserThread::IO); + if (!forwarding_factory_) + return; + forwarding_factory_->AssociateInputAndOutputForAec(input_stream_id, + raw_output_device_id); } +#if BUILDFLAG(IS_TIZEN_TV) +void RenderFrameAudioInputStreamFactory::Core::OnMediaStateChanged( + uint32_t previous, + uint32_t current) { + DCHECK_CURRENTLY_ON(BrowserThread::IO); + GetUIThreadTaskRunner({})->PostTask( + FROM_HERE, + base::BindOnce(&Core::NotifyMediaStateChanged, + weak_ptr_factory_.GetWeakPtr(), previous, current)); +} + +void RenderFrameAudioInputStreamFactory::Core::NotifyMediaStateChanged( + uint32_t previous, + uint32_t current) { + content::BrowserMainLoop* browser_main_loop = + content::BrowserMainLoop::GetInstance(); + if (!browser_main_loop) { + LOG(ERROR) << "browser_main_loop is null"; + return; + } + content::MediaStreamManager* msm = browser_main_loop->media_stream_manager(); + if (!msm) { + LOG(ERROR) << "MediaStreamManager is null"; + return; + } + msm->NotifyMediaStateChanged(content::MediaInputStreamType::AUDIO_CAPTURE, + previous, current); +} +#endif } // namespace content