content/browser/renderer_host/media/render_frame_audio_input_stream_factory.cc

   1 // Copyright 2018 The Chromium Authors
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/browser/renderer_host/media/render_frame_audio_input_stream_factory.h"
   6
   7 #include <cstdint>
   8 #include <string>
   9 #include <utility>
  10
  11 #include "base/check_op.h"
  12 #include "base/functional/bind.h"
  13 #include "base/functional/callback.h"
  14 #include "base/functional/callback_helpers.h"
  15 #include "base/location.h"
  16 #include "base/memory/raw_ptr.h"
  17 #include "base/memory/weak_ptr.h"
  18 #include "base/trace_event/trace_event.h"
  19 #include "base/unguessable_token.h"
  20 #include "content/browser/media/capture/desktop_capture_device_uma_types.h"
  21 #include "content/browser/media/forwarding_audio_stream_factory.h"
  22 #include "content/browser/media/media_devices_permission_checker.h"
  23 #include "content/browser/media/media_devices_util.h"
  24 #include "content/browser/renderer_host/media/audio_input_device_manager.h"
  25 #include "content/browser/renderer_host/media/media_devices_manager.h"
  26 #include "content/browser/renderer_host/media/media_stream_manager.h"
  27 #include "content/public/browser/audio_stream_broker.h"
  28 #include "content/public/browser/browser_task_traits.h"
  29 #include "content/public/browser/browser_thread.h"
  30 #include "content/public/browser/global_routing_id.h"
  31 #include "content/public/browser/media_device_id.h"
  32 #include "content/public/browser/render_frame_host.h"
  33 #include "content/public/browser/render_process_host.h"
  34 #include "content/public/browser/web_contents_media_capture_id.h"
  35 #include "media/audio/audio_device_description.h"
  36 #include "media/base/audio_parameters.h"
  37 #include "mojo/public/cpp/bindings/pending_remote.h"
  38 #include "mojo/public/cpp/bindings/receiver.h"
  39 #include "third_party/blink/public/common/mediastream/media_stream_request.h"
  40 #include "url/origin.h"
  41
  42 #if BUILDFLAG(IS_TIZEN_TV)
  43 #include "content/browser/browser_main_loop.h"
  44 #endif
  45
  46 using blink::mojom::MediaDeviceType;
  47
  48 namespace content {
  49
  50 namespace {
  51
  52 AudioStreamBroker::LoopbackSource* GetLoopbackSourceOnUIThread(
  53     int render_process_id,
  54     int render_frame_id) {
  55   DCHECK_CURRENTLY_ON(BrowserThread::UI);
  56   auto* source = ForwardingAudioStreamFactory::CoreForFrame(
  57       (RenderFrameHost::FromID(render_process_id, render_frame_id)));
  58   if (!source) {
  59     // The source of the capture has already been destroyed, so fail early.
  60     return nullptr;
  61   }
  62   // Note: this pointer is sent over to the IO thread. This is safe since the
  63   // destruction of |source| is posted to the IO thread and it hasn't been
  64   // posted yet.
  65   return source;
  66 }
  67
  68 void EnumerateOutputDevices(MediaStreamManager* media_stream_manager,
  69                             MediaDevicesManager::EnumerationCallback cb) {
  70   DCHECK_CURRENTLY_ON(BrowserThread::IO);
  71   MediaDevicesManager::BoolDeviceTypes device_types;
  72   device_types[static_cast<size_t>(MediaDeviceType::kMediaAudioOuput)] = true;
  73   media_stream_manager->media_devices_manager()->EnumerateDevices(
  74       device_types, std::move(cb));
  75 }
  76
  77 void TranslateDeviceId(const std::string& device_id,
  78                        const MediaDeviceSaltAndOrigin& salt_and_origin,
  79                        base::RepeatingCallback<void(const std::string&)> cb,
  80                        const MediaDeviceEnumeration& device_array) {
  81   DCHECK_CURRENTLY_ON(BrowserThread::IO);
  82   for (const auto& device_info :
  83        device_array[static_cast<size_t>(MediaDeviceType::kMediaAudioOuput)]) {
  84     if (DoesRawMediaDeviceIDMatchHMAC(salt_and_origin, device_id,
  85                                       device_info.device_id)) {
  86       cb.Run(device_info.device_id);
  87       break;
  88     }
  89   }
  90   // If we're unable to translate the device id, |cb| will not be run.
  91 }
  92
  93 void GotSaltAndOrigin(
  94     int process_id,
  95     int frame_id,
  96     base::OnceCallback<void(const MediaDeviceSaltAndOrigin& salt_and_origin,
  97                             bool has_access)> cb,
  98     const MediaDeviceSaltAndOrigin& salt_and_origin) {
  99   bool access = MediaDevicesPermissionChecker().CheckPermissionOnUIThread(
 100       MediaDeviceType::kMediaAudioOuput, process_id, frame_id);
 101   GetIOThreadTaskRunner({})->PostTask(
 102       FROM_HERE, base::BindOnce(std::move(cb), salt_and_origin, access));
 103 }
 104
 105 void GetSaltOriginAndPermissionsOnUIThread(
 106     int process_id,
 107     int frame_id,
 108     base::OnceCallback<void(const MediaDeviceSaltAndOrigin& salt_and_origin,
 109                             bool has_access)> cb) {
 110   GetMediaDeviceSaltAndOrigin(
 111       GlobalRenderFrameHostId(process_id, frame_id),
 112       base::BindOnce(&GotSaltAndOrigin, process_id, frame_id, std::move(cb)));
 113 }
 114
 115 }  // namespace
 116
 117 class RenderFrameAudioInputStreamFactory::Core final
 118     : public blink::mojom::RendererAudioInputStreamFactory {
 119  public:
 120   Core(mojo::PendingReceiver<blink::mojom::RendererAudioInputStreamFactory>
 121            receiver,
 122        MediaStreamManager* media_stream_manager,
 123        RenderFrameHost* render_frame_host);
 124
 125   Core(const Core&) = delete;
 126   Core& operator=(const Core&) = delete;
 127
 128   ~Core() final;
 129
 130   void Init(mojo::PendingReceiver<blink::mojom::RendererAudioInputStreamFactory>
 131                 receiver);
 132
 133   // mojom::RendererAudioInputStreamFactory implementation.
 134   void CreateStream(
 135       mojo::PendingRemote<blink::mojom::RendererAudioInputStreamFactoryClient>
 136           client,
 137       const base::UnguessableToken& session_id,
 138       const media::AudioParameters& audio_params,
 139       bool automatic_gain_control,
 140       uint32_t shared_memory_count,
 141       media::mojom::AudioProcessingConfigPtr processing_config) final;
 142
 143   void AssociateInputAndOutputForAec(
 144       const base::UnguessableToken& input_stream_id,
 145       const std::string& output_device_id) final;
 146
 147   void CreateLoopbackStream(
 148       mojo::PendingRemote<blink::mojom::RendererAudioInputStreamFactoryClient>
 149           client,
 150       const media::AudioParameters& audio_params,
 151       uint32_t shared_memory_count,
 152       bool disable_local_echo,
 153       AudioStreamBroker::LoopbackSource* loopback_source);
 154
 155   void AssociateInputAndOutputForAecAfterCheckingAccess(
 156       const base::UnguessableToken& input_stream_id,
 157       const std::string& output_device_id,
 158       const MediaDeviceSaltAndOrigin& salt_and_origin,
 159       bool access_granted);
 160
 161   void AssociateTranslatedOutputDeviceForAec(
 162       const base::UnguessableToken& input_stream_id,
 163       const std::string& raw_output_device_id);
 164
 165 #if BUILDFLAG(IS_TIZEN_TV)
 166   void OnMediaStateChanged(uint32_t previous, uint32_t current);
 167   void NotifyMediaStateChanged(uint32_t previous, uint32_t current);
 168 #endif
 169
 170   const raw_ptr<MediaStreamManager> media_stream_manager_;
 171   const int process_id_;
 172   const int frame_id_;
 173
 174   mojo::Receiver<RendererAudioInputStreamFactory> receiver_{this};
 175   // Always null-check this weak pointer before dereferencing it.
 176   base::WeakPtr<ForwardingAudioStreamFactory::Core> forwarding_factory_;
 177
 178   base::WeakPtrFactory<Core> weak_ptr_factory_{this};
 179 };
 180
 181 RenderFrameAudioInputStreamFactory::RenderFrameAudioInputStreamFactory(
 182     mojo::PendingReceiver<blink::mojom::RendererAudioInputStreamFactory>
 183         receiver,
 184     MediaStreamManager* media_stream_manager,
 185     RenderFrameHost* render_frame_host)
 186     : core_(new Core(std::move(receiver),
 187                      media_stream_manager,
 188                      render_frame_host)) {
 189   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 190 }
 191
 192 RenderFrameAudioInputStreamFactory::~RenderFrameAudioInputStreamFactory() {
 193   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 194   // Ensure |core_| is deleted on the right thread. DeleteOnIOThread isn't used
 195   // as it doesn't post in case it is already executed on the right thread. That
 196   // causes issues in unit tests where the UI thread and the IO thread are the
 197   // same.
 198   GetIOThreadTaskRunner({})->PostTask(
 199       FROM_HERE, base::DoNothingWithBoundArgs(std::move(core_)));
 200 }
 201
 202 RenderFrameAudioInputStreamFactory::Core::Core(
 203     mojo::PendingReceiver<blink::mojom::RendererAudioInputStreamFactory>
 204         receiver,
 205     MediaStreamManager* media_stream_manager,
 206     RenderFrameHost* render_frame_host)
 207     : media_stream_manager_(media_stream_manager),
 208       process_id_(render_frame_host->GetProcess()->GetID()),
 209       frame_id_(render_frame_host->GetRoutingID()) {
 210   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 211
 212   ForwardingAudioStreamFactory::Core* tmp_factory =
 213       ForwardingAudioStreamFactory::CoreForFrame(render_frame_host);
 214
 215   if (!tmp_factory) {
 216     // The only case when we not have a forwarding factory at this point is when
 217     // the frame belongs to an interstitial. Interstitials don't need audio, so
 218     // it's fine to drop the receiver.
 219     return;
 220   }
 221
 222   forwarding_factory_ = tmp_factory->AsWeakPtr();
 223
 224   // Unretained is safe since the destruction of |this| is posted to the IO
 225   // thread.
 226   GetIOThreadTaskRunner({})->PostTask(
 227       FROM_HERE,
 228       base::BindOnce(&Core::Init, base::Unretained(this), std::move(receiver)));
 229 }
 230
 231 RenderFrameAudioInputStreamFactory::Core::~Core() {
 232   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 233 }
 234
 235 void RenderFrameAudioInputStreamFactory::Core::Init(
 236     mojo::PendingReceiver<blink::mojom::RendererAudioInputStreamFactory>
 237         receiver) {
 238   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 239   receiver_.Bind(std::move(receiver));
 240 }
 241
 242 void RenderFrameAudioInputStreamFactory::Core::CreateStream(
 243     mojo::PendingRemote<blink::mojom::RendererAudioInputStreamFactoryClient>
 244         client,
 245     const base::UnguessableToken& session_id,
 246     const media::AudioParameters& audio_params,
 247     bool automatic_gain_control,
 248     uint32_t shared_memory_count,
 249     media::mojom::AudioProcessingConfigPtr processing_config) {
 250   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 251   TRACE_EVENT1("audio", "RenderFrameAudioInputStreamFactory::CreateStream",
 252                "session id", session_id.ToString());
 253
 254   if (!forwarding_factory_)
 255     return;
 256
 257   const blink::MediaStreamDevice* device =
 258       media_stream_manager_->audio_input_device_manager()->GetOpenedDeviceById(
 259           session_id);
 260
 261   if (!device) {
 262     TRACE_EVENT_INSTANT0("audio", "device not found", TRACE_EVENT_SCOPE_THREAD);
 263     return;
 264   }
 265
 266   WebContentsMediaCaptureId capture_id;
 267   if (WebContentsMediaCaptureId::Parse(device->id, &capture_id)) {
 268     // For MEDIA_GUM_DESKTOP_AUDIO_CAPTURE, the source is selected from
 269     // picker window, we do not mute the source audio. For
 270     // MEDIA_GUM_TAB_AUDIO_CAPTURE, the probable use case is Cast, we mute
 271     // the source audio.
 272     // TODO(qiangchen): Analyze audio constraints to make a duplicating or
 273     // diverting decision. It would give web developer more flexibility.
 274
 275     GetUIThreadTaskRunner({})->PostTaskAndReplyWithResult(
 276         FROM_HERE,
 277         base::BindOnce(&GetLoopbackSourceOnUIThread,
 278                        capture_id.render_process_id,
 279                        capture_id.main_render_frame_id),
 280         base::BindOnce(
 281             &RenderFrameAudioInputStreamFactory::Core::CreateLoopbackStream,
 282             weak_ptr_factory_.GetWeakPtr(), std::move(client), audio_params,
 283             shared_memory_count, capture_id.disable_local_echo));
 284
 285     if (device->type ==
 286         blink::mojom::MediaStreamType::GUM_DESKTOP_AUDIO_CAPTURE)
 287       IncrementDesktopCaptureCounter(SYSTEM_LOOPBACK_AUDIO_CAPTURER_CREATED);
 288     return;
 289   } else {
 290     forwarding_factory_->CreateInputStream(
 291         process_id_, frame_id_, device->id, audio_params, shared_memory_count,
 292         automatic_gain_control, std::move(processing_config),
 293         std::move(client));
 294
 295     // Only count for captures from desktop media picker dialog and system loop
 296     // back audio.
 297     if (device->type ==
 298             blink::mojom::MediaStreamType::GUM_DESKTOP_AUDIO_CAPTURE &&
 299         (media::AudioDeviceDescription::IsLoopbackDevice(device->id))) {
 300       IncrementDesktopCaptureCounter(SYSTEM_LOOPBACK_AUDIO_CAPTURER_CREATED);
 301     }
 302   }
 303 }
 304
 305 void RenderFrameAudioInputStreamFactory::Core::CreateLoopbackStream(
 306     mojo::PendingRemote<blink::mojom::RendererAudioInputStreamFactoryClient>
 307         client,
 308     const media::AudioParameters& audio_params,
 309     uint32_t shared_memory_count,
 310     bool disable_local_echo,
 311     AudioStreamBroker::LoopbackSource* loopback_source) {
 312   if (!loopback_source || !forwarding_factory_)
 313     return;
 314
 315   forwarding_factory_->CreateLoopbackStream(
 316       process_id_, frame_id_, loopback_source, audio_params,
 317       shared_memory_count, disable_local_echo, std::move(client));
 318 }
 319
 320 void RenderFrameAudioInputStreamFactory::Core::AssociateInputAndOutputForAec(
 321     const base::UnguessableToken& input_stream_id,
 322     const std::string& output_device_id) {
 323   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 324   if (!IsValidDeviceId(output_device_id))
 325     return;
 326
 327   GetUIThreadTaskRunner({})->PostTask(
 328       FROM_HERE,
 329       base::BindOnce(
 330           &GetSaltOriginAndPermissionsOnUIThread, process_id_, frame_id_,
 331           base::BindOnce(
 332               &Core::AssociateInputAndOutputForAecAfterCheckingAccess,
 333               weak_ptr_factory_.GetWeakPtr(), input_stream_id,
 334               output_device_id)));
 335 }
 336
 337 void RenderFrameAudioInputStreamFactory::Core::
 338     AssociateInputAndOutputForAecAfterCheckingAccess(
 339         const base::UnguessableToken& input_stream_id,
 340         const std::string& output_device_id,
 341         const MediaDeviceSaltAndOrigin& salt_and_origin,
 342         bool access_granted) {
 343   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 344
 345   if (!forwarding_factory_ || !access_granted)
 346     return;
 347
 348   if (media::AudioDeviceDescription::IsDefaultDevice(output_device_id) ||
 349       media::AudioDeviceDescription::IsCommunicationsDevice(output_device_id)) {
 350     forwarding_factory_->AssociateInputAndOutputForAec(input_stream_id,
 351                                                        output_device_id);
 352   } else {
 353     EnumerateOutputDevices(
 354         media_stream_manager_,
 355         base::BindOnce(&TranslateDeviceId, output_device_id, salt_and_origin,
 356                        base::BindRepeating(
 357                            &RenderFrameAudioInputStreamFactory::Core::
 358                                AssociateTranslatedOutputDeviceForAec,
 359                            weak_ptr_factory_.GetWeakPtr(), input_stream_id)));
 360   }
 361 }
 362
 363 void RenderFrameAudioInputStreamFactory::Core::
 364     AssociateTranslatedOutputDeviceForAec(
 365         const base::UnguessableToken& input_stream_id,
 366         const std::string& raw_output_device_id) {
 367   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 368   if (!forwarding_factory_)
 369     return;
 370   forwarding_factory_->AssociateInputAndOutputForAec(input_stream_id,
 371                                                      raw_output_device_id);
 372 }
 373
 374 #if BUILDFLAG(IS_TIZEN_TV)
 375 void RenderFrameAudioInputStreamFactory::Core::OnMediaStateChanged(
 376     uint32_t previous,
 377     uint32_t current) {
 378   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 379   GetUIThreadTaskRunner({})->PostTask(
 380       FROM_HERE,
 381       base::BindOnce(&Core::NotifyMediaStateChanged,
 382                      weak_ptr_factory_.GetWeakPtr(), previous, current));
 383 }
 384
 385 void RenderFrameAudioInputStreamFactory::Core::NotifyMediaStateChanged(
 386     uint32_t previous,
 387     uint32_t current) {
 388   content::BrowserMainLoop* browser_main_loop =
 389       content::BrowserMainLoop::GetInstance();
 390   if (!browser_main_loop) {
 391     LOG(ERROR) << "browser_main_loop is null";
 392     return;
 393   }
 394   content::MediaStreamManager* msm = browser_main_loop->media_stream_manager();
 395   if (!msm) {
 396     LOG(ERROR) << "MediaStreamManager is null";
 397     return;
 398   }
 399   msm->NotifyMediaStateChanged(content::MediaInputStreamType::AUDIO_CAPTURE,
 400                                previous, current);
 401 }
 402 #endif
 403 }  // namespace content