content/browser/renderer_host/media/render_frame_audio_input_stream_factory.cc

   1 // Copyright 2018 The Chromium Authors
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/browser/renderer_host/media/render_frame_audio_input_stream_factory.h"
   6
   7 #include <cstdint>
   8 #include <string>
   9 #include <utility>
  10
  11 #include "base/check_op.h"
  12 #include "base/functional/bind.h"
  13 #include "base/functional/callback.h"
  14 #include "base/functional/callback_helpers.h"
  15 #include "base/location.h"
  16 #include "base/memory/raw_ptr.h"
  17 #include "base/memory/weak_ptr.h"
  18 #include "base/trace_event/trace_event.h"
  19 #include "base/unguessable_token.h"
  20 #include "content/browser/media/capture/desktop_capture_device_uma_types.h"
  21 #include "content/browser/media/forwarding_audio_stream_factory.h"
  22 #include "content/browser/media/media_devices_permission_checker.h"
  23 #include "content/browser/media/media_devices_util.h"
  24 #include "content/browser/renderer_host/media/audio_input_device_manager.h"
  25 #include "content/browser/renderer_host/media/media_devices_manager.h"
  26 #include "content/browser/renderer_host/media/media_stream_manager.h"
  27 #include "content/public/browser/audio_stream_broker.h"
  28 #include "content/public/browser/browser_task_traits.h"
  29 #include "content/public/browser/browser_thread.h"
  30 #include "content/public/browser/media_device_id.h"
  31 #include "content/public/browser/render_frame_host.h"
  32 #include "content/public/browser/render_process_host.h"
  33 #include "content/public/browser/web_contents_media_capture_id.h"
  34 #include "media/audio/audio_device_description.h"
  35 #include "media/base/audio_parameters.h"
  36 #include "mojo/public/cpp/bindings/pending_remote.h"
  37 #include "mojo/public/cpp/bindings/receiver.h"
  38 #include "third_party/blink/public/common/mediastream/media_stream_request.h"
  39 #include "url/origin.h"
  40
  41 using blink::mojom::MediaDeviceType;
  42
  43 namespace content {
  44
  45 namespace {
  46
  47 AudioStreamBroker::LoopbackSource* GetLoopbackSourceOnUIThread(
  48     int render_process_id,
  49     int render_frame_id) {
  50   DCHECK_CURRENTLY_ON(BrowserThread::UI);
  51   auto* source = ForwardingAudioStreamFactory::CoreForFrame(
  52       (RenderFrameHost::FromID(render_process_id, render_frame_id)));
  53   if (!source) {
  54     // The source of the capture has already been destroyed, so fail early.
  55     return nullptr;
  56   }
  57   // Note: this pointer is sent over to the IO thread. This is safe since the
  58   // destruction of |source| is posted to the IO thread and it hasn't been
  59   // posted yet.
  60   return source;
  61 }
  62
  63 void EnumerateOutputDevices(MediaStreamManager* media_stream_manager,
  64                             MediaDevicesManager::EnumerationCallback cb) {
  65   DCHECK_CURRENTLY_ON(BrowserThread::IO);
  66   MediaDevicesManager::BoolDeviceTypes device_types;
  67   device_types[static_cast<size_t>(MediaDeviceType::MEDIA_AUDIO_OUTPUT)] = true;
  68   media_stream_manager->media_devices_manager()->EnumerateDevices(
  69       device_types, std::move(cb));
  70 }
  71
  72 void TranslateDeviceId(const std::string& device_id,
  73                        const MediaDeviceSaltAndOrigin& salt_and_origin,
  74                        base::RepeatingCallback<void(const std::string&)> cb,
  75                        const MediaDeviceEnumeration& device_array) {
  76   DCHECK_CURRENTLY_ON(BrowserThread::IO);
  77   for (const auto& device_info :
  78        device_array[static_cast<size_t>(MediaDeviceType::MEDIA_AUDIO_OUTPUT)]) {
  79     if (MediaStreamManager::DoesMediaDeviceIDMatchHMAC(
  80             salt_and_origin.device_id_salt, salt_and_origin.origin, device_id,
  81             device_info.device_id)) {
  82       cb.Run(device_info.device_id);
  83       break;
  84     }
  85   }
  86   // If we're unable to translate the device id, |cb| will not be run.
  87 }
  88
  89 void GetSaltOriginAndPermissionsOnUIThread(
  90     int process_id,
  91     int frame_id,
  92     base::OnceCallback<void(MediaDeviceSaltAndOrigin salt_and_origin,
  93                             bool has_access)> cb) {
  94   auto salt_and_origin = GetMediaDeviceSaltAndOrigin(process_id, frame_id);
  95   bool access = MediaDevicesPermissionChecker().CheckPermissionOnUIThread(
  96       MediaDeviceType::MEDIA_AUDIO_OUTPUT, process_id, frame_id);
  97   GetIOThreadTaskRunner({})->PostTask(
  98       FROM_HERE,
  99       base::BindOnce(std::move(cb), std::move(salt_and_origin), access));
 100 }
 101
 102 }  // namespace
 103
 104 class RenderFrameAudioInputStreamFactory::Core final
 105     : public blink::mojom::RendererAudioInputStreamFactory {
 106  public:
 107   Core(mojo::PendingReceiver<blink::mojom::RendererAudioInputStreamFactory>
 108            receiver,
 109        MediaStreamManager* media_stream_manager,
 110        RenderFrameHost* render_frame_host);
 111
 112   Core(const Core&) = delete;
 113   Core& operator=(const Core&) = delete;
 114
 115   ~Core() final;
 116
 117   void Init(mojo::PendingReceiver<blink::mojom::RendererAudioInputStreamFactory>
 118                 receiver);
 119
 120   // mojom::RendererAudioInputStreamFactory implementation.
 121   void CreateStream(
 122       mojo::PendingRemote<blink::mojom::RendererAudioInputStreamFactoryClient>
 123           client,
 124       const base::UnguessableToken& session_id,
 125       const media::AudioParameters& audio_params,
 126       bool automatic_gain_control,
 127       uint32_t shared_memory_count,
 128       media::mojom::AudioProcessingConfigPtr processing_config) final;
 129
 130   void AssociateInputAndOutputForAec(
 131       const base::UnguessableToken& input_stream_id,
 132       const std::string& output_device_id) final;
 133
 134   void CreateLoopbackStream(
 135       mojo::PendingRemote<blink::mojom::RendererAudioInputStreamFactoryClient>
 136           client,
 137       const media::AudioParameters& audio_params,
 138       uint32_t shared_memory_count,
 139       bool disable_local_echo,
 140       AudioStreamBroker::LoopbackSource* loopback_source);
 141
 142   void AssociateInputAndOutputForAecAfterCheckingAccess(
 143       const base::UnguessableToken& input_stream_id,
 144       const std::string& output_device_id,
 145       MediaDeviceSaltAndOrigin salt_and_origin,
 146       bool access_granted);
 147
 148   void AssociateTranslatedOutputDeviceForAec(
 149       const base::UnguessableToken& input_stream_id,
 150       const std::string& raw_output_device_id);
 151
 152   const raw_ptr<MediaStreamManager> media_stream_manager_;
 153   const int process_id_;
 154   const int frame_id_;
 155   const url::Origin origin_;
 156
 157   mojo::Receiver<RendererAudioInputStreamFactory> receiver_{this};
 158   // Always null-check this weak pointer before dereferencing it.
 159   base::WeakPtr<ForwardingAudioStreamFactory::Core> forwarding_factory_;
 160
 161   base::WeakPtrFactory<Core> weak_ptr_factory_{this};
 162 };
 163
 164 RenderFrameAudioInputStreamFactory::RenderFrameAudioInputStreamFactory(
 165     mojo::PendingReceiver<blink::mojom::RendererAudioInputStreamFactory>
 166         receiver,
 167     MediaStreamManager* media_stream_manager,
 168     RenderFrameHost* render_frame_host)
 169     : core_(new Core(std::move(receiver),
 170                      media_stream_manager,
 171                      render_frame_host)) {
 172   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 173 }
 174
 175 RenderFrameAudioInputStreamFactory::~RenderFrameAudioInputStreamFactory() {
 176   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 177   // Ensure |core_| is deleted on the right thread. DeleteOnIOThread isn't used
 178   // as it doesn't post in case it is already executed on the right thread. That
 179   // causes issues in unit tests where the UI thread and the IO thread are the
 180   // same.
 181   GetIOThreadTaskRunner({})->PostTask(
 182       FROM_HERE, base::DoNothingWithBoundArgs(std::move(core_)));
 183 }
 184
 185 RenderFrameAudioInputStreamFactory::Core::Core(
 186     mojo::PendingReceiver<blink::mojom::RendererAudioInputStreamFactory>
 187         receiver,
 188     MediaStreamManager* media_stream_manager,
 189     RenderFrameHost* render_frame_host)
 190     : media_stream_manager_(media_stream_manager),
 191       process_id_(render_frame_host->GetProcess()->GetID()),
 192       frame_id_(render_frame_host->GetRoutingID()),
 193       origin_(render_frame_host->GetLastCommittedOrigin()) {
 194   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 195
 196   ForwardingAudioStreamFactory::Core* tmp_factory =
 197       ForwardingAudioStreamFactory::CoreForFrame(render_frame_host);
 198
 199   if (!tmp_factory) {
 200     // The only case when we not have a forwarding factory at this point is when
 201     // the frame belongs to an interstitial. Interstitials don't need audio, so
 202     // it's fine to drop the receiver.
 203     return;
 204   }
 205
 206   forwarding_factory_ = tmp_factory->AsWeakPtr();
 207
 208   // Unretained is safe since the destruction of |this| is posted to the IO
 209   // thread.
 210   GetIOThreadTaskRunner({})->PostTask(
 211       FROM_HERE,
 212       base::BindOnce(&Core::Init, base::Unretained(this), std::move(receiver)));
 213 }
 214
 215 RenderFrameAudioInputStreamFactory::Core::~Core() {
 216   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 217 }
 218
 219 void RenderFrameAudioInputStreamFactory::Core::Init(
 220     mojo::PendingReceiver<blink::mojom::RendererAudioInputStreamFactory>
 221         receiver) {
 222   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 223   receiver_.Bind(std::move(receiver));
 224 }
 225
 226 void RenderFrameAudioInputStreamFactory::Core::CreateStream(
 227     mojo::PendingRemote<blink::mojom::RendererAudioInputStreamFactoryClient>
 228         client,
 229     const base::UnguessableToken& session_id,
 230     const media::AudioParameters& audio_params,
 231     bool automatic_gain_control,
 232     uint32_t shared_memory_count,
 233     media::mojom::AudioProcessingConfigPtr processing_config) {
 234   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 235   TRACE_EVENT1("audio", "RenderFrameAudioInputStreamFactory::CreateStream",
 236                "session id", session_id.ToString());
 237
 238   if (!forwarding_factory_)
 239     return;
 240
 241   const blink::MediaStreamDevice* device =
 242       media_stream_manager_->audio_input_device_manager()->GetOpenedDeviceById(
 243           session_id);
 244
 245   if (!device) {
 246     TRACE_EVENT_INSTANT0("audio", "device not found", TRACE_EVENT_SCOPE_THREAD);
 247     return;
 248   }
 249
 250   WebContentsMediaCaptureId capture_id;
 251   if (WebContentsMediaCaptureId::Parse(device->id, &capture_id)) {
 252     // For MEDIA_GUM_DESKTOP_AUDIO_CAPTURE, the source is selected from
 253     // picker window, we do not mute the source audio. For
 254     // MEDIA_GUM_TAB_AUDIO_CAPTURE, the probable use case is Cast, we mute
 255     // the source audio.
 256     // TODO(qiangchen): Analyze audio constraints to make a duplicating or
 257     // diverting decision. It would give web developer more flexibility.
 258
 259     GetUIThreadTaskRunner({})->PostTaskAndReplyWithResult(
 260         FROM_HERE,
 261         base::BindOnce(&GetLoopbackSourceOnUIThread,
 262                        capture_id.render_process_id,
 263                        capture_id.main_render_frame_id),
 264         base::BindOnce(
 265             &RenderFrameAudioInputStreamFactory::Core::CreateLoopbackStream,
 266             weak_ptr_factory_.GetWeakPtr(), std::move(client), audio_params,
 267             shared_memory_count, capture_id.disable_local_echo));
 268
 269     if (device->type ==
 270         blink::mojom::MediaStreamType::GUM_DESKTOP_AUDIO_CAPTURE)
 271       IncrementDesktopCaptureCounter(SYSTEM_LOOPBACK_AUDIO_CAPTURER_CREATED);
 272     return;
 273   } else {
 274     forwarding_factory_->CreateInputStream(
 275         process_id_, frame_id_, device->id, audio_params, shared_memory_count,
 276         automatic_gain_control, std::move(processing_config),
 277         std::move(client));
 278
 279     // Only count for captures from desktop media picker dialog and system loop
 280     // back audio.
 281     if (device->type ==
 282             blink::mojom::MediaStreamType::GUM_DESKTOP_AUDIO_CAPTURE &&
 283         (media::AudioDeviceDescription::IsLoopbackDevice(device->id))) {
 284       IncrementDesktopCaptureCounter(SYSTEM_LOOPBACK_AUDIO_CAPTURER_CREATED);
 285     }
 286   }
 287 }
 288
 289 void RenderFrameAudioInputStreamFactory::Core::CreateLoopbackStream(
 290     mojo::PendingRemote<blink::mojom::RendererAudioInputStreamFactoryClient>
 291         client,
 292     const media::AudioParameters& audio_params,
 293     uint32_t shared_memory_count,
 294     bool disable_local_echo,
 295     AudioStreamBroker::LoopbackSource* loopback_source) {
 296   if (!loopback_source || !forwarding_factory_)
 297     return;
 298
 299   forwarding_factory_->CreateLoopbackStream(
 300       process_id_, frame_id_, loopback_source, audio_params,
 301       shared_memory_count, disable_local_echo, std::move(client));
 302 }
 303
 304 void RenderFrameAudioInputStreamFactory::Core::AssociateInputAndOutputForAec(
 305     const base::UnguessableToken& input_stream_id,
 306     const std::string& output_device_id) {
 307   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 308   if (!IsValidDeviceId(output_device_id))
 309     return;
 310
 311   GetUIThreadTaskRunner({})->PostTask(
 312       FROM_HERE,
 313       base::BindOnce(
 314           &GetSaltOriginAndPermissionsOnUIThread, process_id_, frame_id_,
 315           base::BindOnce(
 316               &Core::AssociateInputAndOutputForAecAfterCheckingAccess,
 317               weak_ptr_factory_.GetWeakPtr(), input_stream_id,
 318               output_device_id)));
 319 }
 320
 321 void RenderFrameAudioInputStreamFactory::Core::
 322     AssociateInputAndOutputForAecAfterCheckingAccess(
 323         const base::UnguessableToken& input_stream_id,
 324         const std::string& output_device_id,
 325         MediaDeviceSaltAndOrigin salt_and_origin,
 326         bool access_granted) {
 327   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 328
 329   if (!forwarding_factory_ || !access_granted)
 330     return;
 331
 332   if (media::AudioDeviceDescription::IsDefaultDevice(output_device_id) ||
 333       media::AudioDeviceDescription::IsCommunicationsDevice(output_device_id)) {
 334     forwarding_factory_->AssociateInputAndOutputForAec(input_stream_id,
 335                                                        output_device_id);
 336   } else {
 337     EnumerateOutputDevices(
 338         media_stream_manager_,
 339         base::BindOnce(&TranslateDeviceId, output_device_id, salt_and_origin,
 340                        base::BindRepeating(
 341                            &RenderFrameAudioInputStreamFactory::Core::
 342                                AssociateTranslatedOutputDeviceForAec,
 343                            weak_ptr_factory_.GetWeakPtr(), input_stream_id)));
 344   }
 345 }
 346
 347 void RenderFrameAudioInputStreamFactory::Core::
 348     AssociateTranslatedOutputDeviceForAec(
 349         const base::UnguessableToken& input_stream_id,
 350         const std::string& raw_output_device_id) {
 351   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 352   if (!forwarding_factory_)
 353     return;
 354   forwarding_factory_->AssociateInputAndOutputForAec(input_stream_id,
 355                                                      raw_output_device_id);
 356 }
 357
 358 }  // namespace content