content/browser/renderer_host/media/render_frame_audio_input_stream_factory.cc

   1 // Copyright 2018 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/browser/renderer_host/media/render_frame_audio_input_stream_factory.h"
   6
   7 #include <cstdint>
   8 #include <string>
   9 #include <utility>
  10
  11 #include "base/bind.h"
  12 #include "base/callback.h"
  13 #include "base/location.h"
  14 #include "base/logging.h"
  15 #include "base/memory/weak_ptr.h"
  16 #include "base/task/post_task.h"
  17 #include "base/trace_event/trace_event.h"
  18 #include "base/unguessable_token.h"
  19 #include "content/browser/media/audio_stream_broker.h"
  20 #include "content/browser/media/capture/desktop_capture_device_uma_types.h"
  21 #include "content/browser/media/forwarding_audio_stream_factory.h"
  22 #include "content/browser/media/media_devices_permission_checker.h"
  23 #include "content/browser/media/media_devices_util.h"
  24 #include "content/browser/renderer_host/media/audio_input_device_manager.h"
  25 #include "content/browser/renderer_host/media/media_devices_manager.h"
  26 #include "content/browser/renderer_host/media/media_stream_manager.h"
  27 #include "content/public/browser/browser_task_traits.h"
  28 #include "content/public/browser/browser_thread.h"
  29 #include "content/public/browser/media_device_id.h"
  30 #include "content/public/browser/render_frame_host.h"
  31 #include "content/public/browser/render_process_host.h"
  32 #include "content/public/browser/web_contents_media_capture_id.h"
  33 #include "media/audio/audio_device_description.h"
  34 #include "media/base/audio_parameters.h"
  35 #include "mojo/public/cpp/bindings/binding.h"
  36 #include "services/audio/public/mojom/audio_processing.mojom.h"
  37 #include "third_party/blink/public/common/mediastream/media_stream_request.h"
  38 #include "url/origin.h"
  39
  40 namespace content {
  41
  42 namespace {
  43
  44 AudioStreamBroker::LoopbackSource* GetLoopbackSourceOnUIThread(
  45     int render_process_id,
  46     int render_frame_id) {
  47   DCHECK_CURRENTLY_ON(BrowserThread::UI);
  48   auto* source = ForwardingAudioStreamFactory::CoreForFrame(
  49       (RenderFrameHost::FromID(render_process_id, render_frame_id)));
  50   if (!source) {
  51     // The source of the capture has already been destroyed, so fail early.
  52     return nullptr;
  53   }
  54   // Note: this pointer is sent over to the IO thread. This is safe since the
  55   // destruction of |source| is posted to the IO thread and it hasn't been
  56   // posted yet.
  57   return source;
  58 }
  59
  60 void EnumerateOutputDevices(MediaStreamManager* media_stream_manager,
  61                             MediaDevicesManager::EnumerationCallback cb) {
  62   DCHECK_CURRENTLY_ON(BrowserThread::IO);
  63   MediaDevicesManager::BoolDeviceTypes device_types;
  64   device_types[blink::MEDIA_DEVICE_TYPE_AUDIO_OUTPUT] = true;
  65   media_stream_manager->media_devices_manager()->EnumerateDevices(
  66       device_types, std::move(cb));
  67 }
  68
  69 void TranslateDeviceId(const std::string& device_id,
  70                        const MediaDeviceSaltAndOrigin& salt_and_origin,
  71                        base::RepeatingCallback<void(const std::string&)> cb,
  72                        const MediaDeviceEnumeration& device_array) {
  73   DCHECK_CURRENTLY_ON(BrowserThread::IO);
  74   for (const auto& device_info :
  75        device_array[blink::MEDIA_DEVICE_TYPE_AUDIO_OUTPUT]) {
  76     if (MediaStreamManager::DoesMediaDeviceIDMatchHMAC(
  77             salt_and_origin.device_id_salt, salt_and_origin.origin, device_id,
  78             device_info.device_id)) {
  79       cb.Run(device_info.device_id);
  80       break;
  81     }
  82   }
  83   // If we're unable to translate the device id, |cb| will not be run.
  84 }
  85
  86 void GetSaltOriginAndPermissionsOnUIThread(
  87     int process_id,
  88     int frame_id,
  89     base::OnceCallback<void(MediaDeviceSaltAndOrigin salt_and_origin,
  90                             bool has_access)> cb) {
  91   auto salt_and_origin = GetMediaDeviceSaltAndOrigin(process_id, frame_id);
  92   bool access = MediaDevicesPermissionChecker().CheckPermissionOnUIThread(
  93       blink::MEDIA_DEVICE_TYPE_AUDIO_OUTPUT, process_id, frame_id);
  94   base::PostTaskWithTraits(
  95       FROM_HERE, {BrowserThread::IO},
  96       base::BindOnce(std::move(cb), std::move(salt_and_origin), access));
  97 }
  98
  99 }  // namespace
 100
 101 class RenderFrameAudioInputStreamFactory::Core final
 102     : public mojom::RendererAudioInputStreamFactory {
 103  public:
 104   Core(mojom::RendererAudioInputStreamFactoryRequest request,
 105        MediaStreamManager* media_stream_manager,
 106        RenderFrameHost* render_frame_host);
 107
 108   ~Core() final;
 109
 110   void Init(mojom::RendererAudioInputStreamFactoryRequest request);
 111
 112   // mojom::RendererAudioInputStreamFactory implementation.
 113   void CreateStream(
 114       mojom::RendererAudioInputStreamFactoryClientPtr client,
 115       int32_t session_id,
 116       const media::AudioParameters& audio_params,
 117       bool automatic_gain_control,
 118       uint32_t shared_memory_count,
 119       audio::mojom::AudioProcessingConfigPtr processing_config) final;
 120
 121   void AssociateInputAndOutputForAec(
 122       const base::UnguessableToken& input_stream_id,
 123       const std::string& output_device_id) final;
 124
 125   void CreateLoopbackStream(
 126       mojom::RendererAudioInputStreamFactoryClientPtr client,
 127       const media::AudioParameters& audio_params,
 128       uint32_t shared_memory_count,
 129       bool disable_local_echo,
 130       AudioStreamBroker::LoopbackSource* loopback_source);
 131
 132   void AssociateInputAndOutputForAecAfterCheckingAccess(
 133       const base::UnguessableToken& input_stream_id,
 134       const std::string& output_device_id,
 135       MediaDeviceSaltAndOrigin salt_and_origin,
 136       bool access_granted);
 137
 138   void AssociateTranslatedOutputDeviceForAec(
 139       const base::UnguessableToken& input_stream_id,
 140       const std::string& raw_output_device_id);
 141
 142   MediaStreamManager* const media_stream_manager_;
 143   const int process_id_;
 144   const int frame_id_;
 145   const url::Origin origin_;
 146
 147   mojo::Binding<RendererAudioInputStreamFactory> binding_;
 148   // Always null-check this weak pointer before dereferencing it.
 149   base::WeakPtr<ForwardingAudioStreamFactory::Core> forwarding_factory_;
 150
 151   base::WeakPtrFactory<Core> weak_ptr_factory_;
 152
 153   DISALLOW_COPY_AND_ASSIGN(Core);
 154 };
 155
 156 RenderFrameAudioInputStreamFactory::RenderFrameAudioInputStreamFactory(
 157     mojom::RendererAudioInputStreamFactoryRequest request,
 158     MediaStreamManager* media_stream_manager,
 159     RenderFrameHost* render_frame_host)
 160     : core_(new Core(std::move(request),
 161                      media_stream_manager,
 162                      render_frame_host)) {
 163   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 164 }
 165
 166 RenderFrameAudioInputStreamFactory::~RenderFrameAudioInputStreamFactory() {
 167   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 168   // Ensure |core_| is deleted on the right thread. DeleteOnIOThread isn't used
 169   // as it doesn't post in case it is already executed on the right thread. That
 170   // causes issues in unit tests where the UI thread and the IO thread are the
 171   // same.
 172   base::PostTaskWithTraits(
 173       FROM_HERE, {BrowserThread::IO},
 174       base::BindOnce([](std::unique_ptr<Core>) {}, std::move(core_)));
 175 }
 176
 177 RenderFrameAudioInputStreamFactory::Core::Core(
 178     mojom::RendererAudioInputStreamFactoryRequest request,
 179     MediaStreamManager* media_stream_manager,
 180     RenderFrameHost* render_frame_host)
 181     : media_stream_manager_(media_stream_manager),
 182       process_id_(render_frame_host->GetProcess()->GetID()),
 183       frame_id_(render_frame_host->GetRoutingID()),
 184       origin_(render_frame_host->GetLastCommittedOrigin()),
 185       binding_(this),
 186       weak_ptr_factory_(this) {
 187   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 188
 189   ForwardingAudioStreamFactory::Core* tmp_factory =
 190       ForwardingAudioStreamFactory::CoreForFrame(render_frame_host);
 191
 192   if (!tmp_factory) {
 193     // The only case when we not have a forwarding factory at this point is when
 194     // the frame belongs to an interstitial. Interstitials don't need audio, so
 195     // it's fine to drop the request.
 196     return;
 197   }
 198
 199   forwarding_factory_ = tmp_factory->AsWeakPtr();
 200
 201   // Unretained is safe since the destruction of |this| is posted to the IO
 202   // thread.
 203   base::PostTaskWithTraits(
 204       FROM_HERE, {BrowserThread::IO},
 205       base::BindOnce(&Core::Init, base::Unretained(this), std::move(request)));
 206 }
 207
 208 RenderFrameAudioInputStreamFactory::Core::~Core() {
 209   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 210 }
 211
 212 void RenderFrameAudioInputStreamFactory::Core::Init(
 213     mojom::RendererAudioInputStreamFactoryRequest request) {
 214   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 215   binding_.Bind(std::move(request));
 216 }
 217
 218 void RenderFrameAudioInputStreamFactory::Core::CreateStream(
 219     mojom::RendererAudioInputStreamFactoryClientPtr client,
 220     int32_t session_id,
 221     const media::AudioParameters& audio_params,
 222     bool automatic_gain_control,
 223     uint32_t shared_memory_count,
 224     audio::mojom::AudioProcessingConfigPtr processing_config) {
 225   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 226   TRACE_EVENT1("audio", "RenderFrameAudioInputStreamFactory::CreateStream",
 227                "session id", session_id);
 228
 229   if (!forwarding_factory_)
 230     return;
 231
 232   const blink::MediaStreamDevice* device =
 233       media_stream_manager_->audio_input_device_manager()->GetOpenedDeviceById(
 234           session_id);
 235
 236   if (!device) {
 237     TRACE_EVENT_INSTANT0("audio", "device not found", TRACE_EVENT_SCOPE_THREAD);
 238     return;
 239   }
 240
 241   WebContentsMediaCaptureId capture_id;
 242   if (WebContentsMediaCaptureId::Parse(device->id, &capture_id)) {
 243     // For MEDIA_GUM_DESKTOP_AUDIO_CAPTURE, the source is selected from
 244     // picker window, we do not mute the source audio. For
 245     // MEDIA_GUM_TAB_AUDIO_CAPTURE, the probable use case is Cast, we mute
 246     // the source audio.
 247     // TODO(qiangchen): Analyze audio constraints to make a duplicating or
 248     // diverting decision. It would give web developer more flexibility.
 249
 250     base::PostTaskWithTraitsAndReplyWithResult(
 251         FROM_HERE, {BrowserThread::UI},
 252         base::BindOnce(&GetLoopbackSourceOnUIThread,
 253                        capture_id.render_process_id,
 254                        capture_id.main_render_frame_id),
 255         base::BindOnce(
 256             &RenderFrameAudioInputStreamFactory::Core::CreateLoopbackStream,
 257             weak_ptr_factory_.GetWeakPtr(), std::move(client), audio_params,
 258             shared_memory_count, capture_id.disable_local_echo));
 259
 260     if (device->type == blink::MEDIA_GUM_DESKTOP_AUDIO_CAPTURE)
 261       IncrementDesktopCaptureCounter(SYSTEM_LOOPBACK_AUDIO_CAPTURER_CREATED);
 262     return;
 263   } else {
 264     forwarding_factory_->CreateInputStream(
 265         process_id_, frame_id_, device->id, audio_params, shared_memory_count,
 266         automatic_gain_control, std::move(processing_config),
 267         std::move(client));
 268
 269     // Only count for captures from desktop media picker dialog and system loop
 270     // back audio.
 271     if (device->type == blink::MEDIA_GUM_DESKTOP_AUDIO_CAPTURE &&
 272         (media::AudioDeviceDescription::IsLoopbackDevice(device->id))) {
 273       IncrementDesktopCaptureCounter(SYSTEM_LOOPBACK_AUDIO_CAPTURER_CREATED);
 274     }
 275   }
 276 }
 277
 278 void RenderFrameAudioInputStreamFactory::Core::CreateLoopbackStream(
 279     mojom::RendererAudioInputStreamFactoryClientPtr client,
 280     const media::AudioParameters& audio_params,
 281     uint32_t shared_memory_count,
 282     bool disable_local_echo,
 283     AudioStreamBroker::LoopbackSource* loopback_source) {
 284   if (!loopback_source || !forwarding_factory_)
 285     return;
 286
 287   forwarding_factory_->CreateLoopbackStream(
 288       process_id_, frame_id_, loopback_source, audio_params,
 289       shared_memory_count, disable_local_echo, std::move(client));
 290 }
 291
 292 void RenderFrameAudioInputStreamFactory::Core::AssociateInputAndOutputForAec(
 293     const base::UnguessableToken& input_stream_id,
 294     const std::string& output_device_id) {
 295   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 296   if (!IsValidDeviceId(output_device_id))
 297     return;
 298
 299   base::PostTaskWithTraits(
 300       FROM_HERE, {BrowserThread::UI},
 301       base::BindOnce(
 302           &GetSaltOriginAndPermissionsOnUIThread, process_id_, frame_id_,
 303           base::BindOnce(
 304               &Core::AssociateInputAndOutputForAecAfterCheckingAccess,
 305               weak_ptr_factory_.GetWeakPtr(), input_stream_id,
 306               output_device_id)));
 307 }
 308
 309 void RenderFrameAudioInputStreamFactory::Core::
 310     AssociateInputAndOutputForAecAfterCheckingAccess(
 311         const base::UnguessableToken& input_stream_id,
 312         const std::string& output_device_id,
 313         MediaDeviceSaltAndOrigin salt_and_origin,
 314         bool access_granted) {
 315   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 316
 317   if (!forwarding_factory_ || !access_granted)
 318     return;
 319
 320   if (media::AudioDeviceDescription::IsDefaultDevice(output_device_id) ||
 321       media::AudioDeviceDescription::IsCommunicationsDevice(output_device_id)) {
 322     forwarding_factory_->AssociateInputAndOutputForAec(input_stream_id,
 323                                                        output_device_id);
 324   } else {
 325     EnumerateOutputDevices(
 326         media_stream_manager_,
 327         base::BindRepeating(
 328             &TranslateDeviceId, output_device_id, salt_and_origin,
 329             base::BindRepeating(&RenderFrameAudioInputStreamFactory::Core::
 330                                     AssociateTranslatedOutputDeviceForAec,
 331                                 weak_ptr_factory_.GetWeakPtr(),
 332                                 input_stream_id)));
 333   }
 334 }
 335
 336 void RenderFrameAudioInputStreamFactory::Core::
 337     AssociateTranslatedOutputDeviceForAec(
 338         const base::UnguessableToken& input_stream_id,
 339         const std::string& raw_output_device_id) {
 340   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 341   if (!forwarding_factory_)
 342     return;
 343   forwarding_factory_->AssociateInputAndOutputForAec(input_stream_id,
 344                                                      raw_output_device_id);
 345 }
 346
 347 }  // namespace content