content/browser/renderer_host/media/render_frame_audio_input_stream_factory.cc

   1 // Copyright 2018 The Chromium Authors
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/browser/renderer_host/media/render_frame_audio_input_stream_factory.h"
   6
   7 #include <cstdint>
   8 #include <string>
   9 #include <utility>
  10
  11 #include "base/check_op.h"
  12 #include "base/functional/bind.h"
  13 #include "base/functional/callback.h"
  14 #include "base/functional/callback_helpers.h"
  15 #include "base/location.h"
  16 #include "base/memory/raw_ptr.h"
  17 #include "base/memory/weak_ptr.h"
  18 #include "base/trace_event/trace_event.h"
  19 #include "base/unguessable_token.h"
  20 #include "content/browser/media/capture/desktop_capture_device_uma_types.h"
  21 #include "content/browser/media/forwarding_audio_stream_factory.h"
  22 #include "content/browser/media/media_devices_permission_checker.h"
  23 #include "content/browser/media/media_devices_util.h"
  24 #include "content/browser/renderer_host/media/audio_input_device_manager.h"
  25 #include "content/browser/renderer_host/media/media_devices_manager.h"
  26 #include "content/browser/renderer_host/media/media_stream_manager.h"
  27 #include "content/public/browser/audio_stream_broker.h"
  28 #include "content/public/browser/browser_task_traits.h"
  29 #include "content/public/browser/browser_thread.h"
  30 #include "content/public/browser/global_routing_id.h"
  31 #include "content/public/browser/media_device_id.h"
  32 #include "content/public/browser/render_frame_host.h"
  33 #include "content/public/browser/render_process_host.h"
  34 #include "content/public/browser/web_contents_media_capture_id.h"
  35 #include "media/audio/audio_device_description.h"
  36 #include "media/base/audio_parameters.h"
  37 #include "mojo/public/cpp/bindings/pending_remote.h"
  38 #include "mojo/public/cpp/bindings/receiver.h"
  39 #include "third_party/blink/public/common/mediastream/media_stream_request.h"
  40 #include "url/origin.h"
  41
  42 using blink::mojom::MediaDeviceType;
  43
  44 namespace content {
  45
  46 namespace {
  47
  48 AudioStreamBroker::LoopbackSource* GetLoopbackSourceOnUIThread(
  49     int render_process_id,
  50     int render_frame_id) {
  51   DCHECK_CURRENTLY_ON(BrowserThread::UI);
  52   auto* source = ForwardingAudioStreamFactory::CoreForFrame(
  53       (RenderFrameHost::FromID(render_process_id, render_frame_id)));
  54   if (!source) {
  55     // The source of the capture has already been destroyed, so fail early.
  56     return nullptr;
  57   }
  58   // Note: this pointer is sent over to the IO thread. This is safe since the
  59   // destruction of |source| is posted to the IO thread and it hasn't been
  60   // posted yet.
  61   return source;
  62 }
  63
  64 void EnumerateOutputDevices(MediaStreamManager* media_stream_manager,
  65                             MediaDevicesManager::EnumerationCallback cb) {
  66   DCHECK_CURRENTLY_ON(BrowserThread::IO);
  67   MediaDevicesManager::BoolDeviceTypes device_types;
  68   device_types[static_cast<size_t>(MediaDeviceType::kMediaAudioOuput)] = true;
  69   media_stream_manager->media_devices_manager()->EnumerateDevices(
  70       device_types, std::move(cb));
  71 }
  72
  73 void TranslateDeviceId(const std::string& device_id,
  74                        const MediaDeviceSaltAndOrigin& salt_and_origin,
  75                        base::RepeatingCallback<void(const std::string&)> cb,
  76                        const MediaDeviceEnumeration& device_array) {
  77   DCHECK_CURRENTLY_ON(BrowserThread::IO);
  78   for (const auto& device_info :
  79        device_array[static_cast<size_t>(MediaDeviceType::kMediaAudioOuput)]) {
  80     if (DoesRawMediaDeviceIDMatchHMAC(salt_and_origin, device_id,
  81                                       device_info.device_id)) {
  82       cb.Run(device_info.device_id);
  83       break;
  84     }
  85   }
  86   // If we're unable to translate the device id, |cb| will not be run.
  87 }
  88
  89 void GotSaltAndOrigin(
  90     int process_id,
  91     int frame_id,
  92     base::OnceCallback<void(const MediaDeviceSaltAndOrigin& salt_and_origin,
  93                             bool has_access)> cb,
  94     const MediaDeviceSaltAndOrigin& salt_and_origin) {
  95   bool access = MediaDevicesPermissionChecker().CheckPermissionOnUIThread(
  96       MediaDeviceType::kMediaAudioOuput, process_id, frame_id);
  97   GetIOThreadTaskRunner({})->PostTask(
  98       FROM_HERE, base::BindOnce(std::move(cb), salt_and_origin, access));
  99 }
 100
 101 void GetSaltOriginAndPermissionsOnUIThread(
 102     int process_id,
 103     int frame_id,
 104     base::OnceCallback<void(const MediaDeviceSaltAndOrigin& salt_and_origin,
 105                             bool has_access)> cb) {
 106   GetMediaDeviceSaltAndOrigin(
 107       GlobalRenderFrameHostId(process_id, frame_id),
 108       base::BindOnce(&GotSaltAndOrigin, process_id, frame_id, std::move(cb)));
 109 }
 110
 111 }  // namespace
 112
 113 class RenderFrameAudioInputStreamFactory::Core final
 114     : public blink::mojom::RendererAudioInputStreamFactory {
 115  public:
 116   Core(mojo::PendingReceiver<blink::mojom::RendererAudioInputStreamFactory>
 117            receiver,
 118        MediaStreamManager* media_stream_manager,
 119        RenderFrameHost* render_frame_host);
 120
 121   Core(const Core&) = delete;
 122   Core& operator=(const Core&) = delete;
 123
 124   ~Core() final;
 125
 126   void Init(mojo::PendingReceiver<blink::mojom::RendererAudioInputStreamFactory>
 127                 receiver);
 128
 129   // mojom::RendererAudioInputStreamFactory implementation.
 130   void CreateStream(
 131       mojo::PendingRemote<blink::mojom::RendererAudioInputStreamFactoryClient>
 132           client,
 133       const base::UnguessableToken& session_id,
 134       const media::AudioParameters& audio_params,
 135       bool automatic_gain_control,
 136       uint32_t shared_memory_count,
 137       media::mojom::AudioProcessingConfigPtr processing_config) final;
 138
 139   void AssociateInputAndOutputForAec(
 140       const base::UnguessableToken& input_stream_id,
 141       const std::string& output_device_id) final;
 142
 143   void CreateLoopbackStream(
 144       mojo::PendingRemote<blink::mojom::RendererAudioInputStreamFactoryClient>
 145           client,
 146       const media::AudioParameters& audio_params,
 147       uint32_t shared_memory_count,
 148       bool disable_local_echo,
 149       AudioStreamBroker::LoopbackSource* loopback_source);
 150
 151   void AssociateInputAndOutputForAecAfterCheckingAccess(
 152       const base::UnguessableToken& input_stream_id,
 153       const std::string& output_device_id,
 154       const MediaDeviceSaltAndOrigin& salt_and_origin,
 155       bool access_granted);
 156
 157   void AssociateTranslatedOutputDeviceForAec(
 158       const base::UnguessableToken& input_stream_id,
 159       const std::string& raw_output_device_id);
 160
 161   const raw_ptr<MediaStreamManager> media_stream_manager_;
 162   const int process_id_;
 163   const int frame_id_;
 164
 165   mojo::Receiver<RendererAudioInputStreamFactory> receiver_{this};
 166   // Always null-check this weak pointer before dereferencing it.
 167   base::WeakPtr<ForwardingAudioStreamFactory::Core> forwarding_factory_;
 168
 169   base::WeakPtrFactory<Core> weak_ptr_factory_{this};
 170 };
 171
 172 RenderFrameAudioInputStreamFactory::RenderFrameAudioInputStreamFactory(
 173     mojo::PendingReceiver<blink::mojom::RendererAudioInputStreamFactory>
 174         receiver,
 175     MediaStreamManager* media_stream_manager,
 176     RenderFrameHost* render_frame_host)
 177     : core_(new Core(std::move(receiver),
 178                      media_stream_manager,
 179                      render_frame_host)) {
 180   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 181 }
 182
 183 RenderFrameAudioInputStreamFactory::~RenderFrameAudioInputStreamFactory() {
 184   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 185   // Ensure |core_| is deleted on the right thread. DeleteOnIOThread isn't used
 186   // as it doesn't post in case it is already executed on the right thread. That
 187   // causes issues in unit tests where the UI thread and the IO thread are the
 188   // same.
 189   GetIOThreadTaskRunner({})->PostTask(
 190       FROM_HERE, base::DoNothingWithBoundArgs(std::move(core_)));
 191 }
 192
 193 RenderFrameAudioInputStreamFactory::Core::Core(
 194     mojo::PendingReceiver<blink::mojom::RendererAudioInputStreamFactory>
 195         receiver,
 196     MediaStreamManager* media_stream_manager,
 197     RenderFrameHost* render_frame_host)
 198     : media_stream_manager_(media_stream_manager),
 199       process_id_(render_frame_host->GetProcess()->GetID()),
 200       frame_id_(render_frame_host->GetRoutingID()) {
 201   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 202
 203   ForwardingAudioStreamFactory::Core* tmp_factory =
 204       ForwardingAudioStreamFactory::CoreForFrame(render_frame_host);
 205
 206   if (!tmp_factory) {
 207     // The only case when we not have a forwarding factory at this point is when
 208     // the frame belongs to an interstitial. Interstitials don't need audio, so
 209     // it's fine to drop the receiver.
 210     return;
 211   }
 212
 213   forwarding_factory_ = tmp_factory->AsWeakPtr();
 214
 215   // Unretained is safe since the destruction of |this| is posted to the IO
 216   // thread.
 217   GetIOThreadTaskRunner({})->PostTask(
 218       FROM_HERE,
 219       base::BindOnce(&Core::Init, base::Unretained(this), std::move(receiver)));
 220 }
 221
 222 RenderFrameAudioInputStreamFactory::Core::~Core() {
 223   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 224 }
 225
 226 void RenderFrameAudioInputStreamFactory::Core::Init(
 227     mojo::PendingReceiver<blink::mojom::RendererAudioInputStreamFactory>
 228         receiver) {
 229   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 230   receiver_.Bind(std::move(receiver));
 231 }
 232
 233 void RenderFrameAudioInputStreamFactory::Core::CreateStream(
 234     mojo::PendingRemote<blink::mojom::RendererAudioInputStreamFactoryClient>
 235         client,
 236     const base::UnguessableToken& session_id,
 237     const media::AudioParameters& audio_params,
 238     bool automatic_gain_control,
 239     uint32_t shared_memory_count,
 240     media::mojom::AudioProcessingConfigPtr processing_config) {
 241   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 242   TRACE_EVENT1("audio", "RenderFrameAudioInputStreamFactory::CreateStream",
 243                "session id", session_id.ToString());
 244
 245   if (!forwarding_factory_)
 246     return;
 247
 248   const blink::MediaStreamDevice* device =
 249       media_stream_manager_->audio_input_device_manager()->GetOpenedDeviceById(
 250           session_id);
 251
 252   if (!device) {
 253     TRACE_EVENT_INSTANT0("audio", "device not found", TRACE_EVENT_SCOPE_THREAD);
 254     return;
 255   }
 256
 257   WebContentsMediaCaptureId capture_id;
 258   if (WebContentsMediaCaptureId::Parse(device->id, &capture_id)) {
 259     // For MEDIA_GUM_DESKTOP_AUDIO_CAPTURE, the source is selected from
 260     // picker window, we do not mute the source audio. For
 261     // MEDIA_GUM_TAB_AUDIO_CAPTURE, the probable use case is Cast, we mute
 262     // the source audio.
 263     // TODO(qiangchen): Analyze audio constraints to make a duplicating or
 264     // diverting decision. It would give web developer more flexibility.
 265
 266     GetUIThreadTaskRunner({})->PostTaskAndReplyWithResult(
 267         FROM_HERE,
 268         base::BindOnce(&GetLoopbackSourceOnUIThread,
 269                        capture_id.render_process_id,
 270                        capture_id.main_render_frame_id),
 271         base::BindOnce(
 272             &RenderFrameAudioInputStreamFactory::Core::CreateLoopbackStream,
 273             weak_ptr_factory_.GetWeakPtr(), std::move(client), audio_params,
 274             shared_memory_count, capture_id.disable_local_echo));
 275
 276     if (device->type ==
 277         blink::mojom::MediaStreamType::GUM_DESKTOP_AUDIO_CAPTURE)
 278       IncrementDesktopCaptureCounter(SYSTEM_LOOPBACK_AUDIO_CAPTURER_CREATED);
 279     return;
 280   } else {
 281     forwarding_factory_->CreateInputStream(
 282         process_id_, frame_id_, device->id, audio_params, shared_memory_count,
 283         automatic_gain_control, std::move(processing_config),
 284         std::move(client));
 285
 286     // Only count for captures from desktop media picker dialog and system loop
 287     // back audio.
 288     if (device->type ==
 289             blink::mojom::MediaStreamType::GUM_DESKTOP_AUDIO_CAPTURE &&
 290         (media::AudioDeviceDescription::IsLoopbackDevice(device->id))) {
 291       IncrementDesktopCaptureCounter(SYSTEM_LOOPBACK_AUDIO_CAPTURER_CREATED);
 292     }
 293   }
 294 }
 295
 296 void RenderFrameAudioInputStreamFactory::Core::CreateLoopbackStream(
 297     mojo::PendingRemote<blink::mojom::RendererAudioInputStreamFactoryClient>
 298         client,
 299     const media::AudioParameters& audio_params,
 300     uint32_t shared_memory_count,
 301     bool disable_local_echo,
 302     AudioStreamBroker::LoopbackSource* loopback_source) {
 303   if (!loopback_source || !forwarding_factory_)
 304     return;
 305
 306   forwarding_factory_->CreateLoopbackStream(
 307       process_id_, frame_id_, loopback_source, audio_params,
 308       shared_memory_count, disable_local_echo, std::move(client));
 309 }
 310
 311 void RenderFrameAudioInputStreamFactory::Core::AssociateInputAndOutputForAec(
 312     const base::UnguessableToken& input_stream_id,
 313     const std::string& output_device_id) {
 314   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 315   if (!IsValidDeviceId(output_device_id))
 316     return;
 317
 318   GetUIThreadTaskRunner({})->PostTask(
 319       FROM_HERE,
 320       base::BindOnce(
 321           &GetSaltOriginAndPermissionsOnUIThread, process_id_, frame_id_,
 322           base::BindOnce(
 323               &Core::AssociateInputAndOutputForAecAfterCheckingAccess,
 324               weak_ptr_factory_.GetWeakPtr(), input_stream_id,
 325               output_device_id)));
 326 }
 327
 328 void RenderFrameAudioInputStreamFactory::Core::
 329     AssociateInputAndOutputForAecAfterCheckingAccess(
 330         const base::UnguessableToken& input_stream_id,
 331         const std::string& output_device_id,
 332         const MediaDeviceSaltAndOrigin& salt_and_origin,
 333         bool access_granted) {
 334   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 335
 336   if (!forwarding_factory_ || !access_granted)
 337     return;
 338
 339   if (media::AudioDeviceDescription::IsDefaultDevice(output_device_id) ||
 340       media::AudioDeviceDescription::IsCommunicationsDevice(output_device_id)) {
 341     forwarding_factory_->AssociateInputAndOutputForAec(input_stream_id,
 342                                                        output_device_id);
 343   } else {
 344     EnumerateOutputDevices(
 345         media_stream_manager_,
 346         base::BindOnce(&TranslateDeviceId, output_device_id, salt_and_origin,
 347                        base::BindRepeating(
 348                            &RenderFrameAudioInputStreamFactory::Core::
 349                                AssociateTranslatedOutputDeviceForAec,
 350                            weak_ptr_factory_.GetWeakPtr(), input_stream_id)));
 351   }
 352 }
 353
 354 void RenderFrameAudioInputStreamFactory::Core::
 355     AssociateTranslatedOutputDeviceForAec(
 356         const base::UnguessableToken& input_stream_id,
 357         const std::string& raw_output_device_id) {
 358   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 359   if (!forwarding_factory_)
 360     return;
 361   forwarding_factory_->AssociateInputAndOutputForAec(input_stream_id,
 362                                                      raw_output_device_id);
 363 }
 364
 365 }  // namespace content