[WebRTC] Tizen system output audio capture for screen share feature

to capture tizen system output audio for webrtc screen share
feature, a new audio forwarding device is created by audio io team.
the job of the our part is to convert the getDisplayMedia audio
into requesting this forwarding device instead of the upstream logic.

Change-Id: I5bd678041e9c3cbcd647b2d85ee6d9c5d5459562
Signed-off-by: peng.yin <peng8.yin@samsung.com>

diff --git a/content/browser/media/media_devices_util.cc b/content/browser/media/media_devices_util.cc
index f2bf2606b51703a372b70bab2954d8a17c48d6fb..c42b1710ecbbd61c09a1229fed60a096be41e2de 100644 (file)
--- a/content/browser/media/media_devices_util.cc
+++ b/content/browser/media/media_devices_util.cc
@@ -397,4 +397,8 @@ void GetRawDeviceIDForMediaDeviceHMAC(
                      std::move(task_runner), std::move(callback)));
 }
 
+bool is_forwarding_device(const std::string string) {
+  return string.find("forwarding") != std::string::npos;
+}
+
 }  // namespace content

diff --git a/content/browser/media/media_devices_util.h b/content/browser/media/media_devices_util.h
index b3350f2af47f41c5666b015e9e22443b9e877589..d57c821fd53212a223f34b26a008b3b996ad2939 100644 (file)
--- a/content/browser/media/media_devices_util.h
+++ b/content/browser/media/media_devices_util.h
@@ -186,6 +186,8 @@ CONTENT_EXPORT void GetRawDeviceIDForMediaDeviceHMAC(
     scoped_refptr<base::SequencedTaskRunner> task_runner,
     OptionalDeviceIdCallback callback);
 
+bool is_forwarding_device(const std::string string);
+
 }  // namespace content
 
 #endif  // CONTENT_BROWSER_MEDIA_MEDIA_DEVICES_UTIL_H_

diff --git a/content/browser/renderer_host/media/media_capture_devices_impl.cc b/content/browser/renderer_host/media/media_capture_devices_impl.cc
index f42078f5f32a5f3829e2fe701cd5da0b3767ca0e..597ed5d1cfb3d33dee190950b1ebe6680784e4ea 100644 (file)
--- a/content/browser/renderer_host/media/media_capture_devices_impl.cc
+++ b/content/browser/renderer_host/media/media_capture_devices_impl.cc
@@ -44,6 +44,16 @@ MediaCaptureDevicesImpl::GetAudioCaptureDevices() {
   return audio_devices_;
 }
 
+const blink::MediaStreamDevices&
+MediaCaptureDevicesImpl::GetAudioForwardDevices() {
+  DCHECK_CURRENTLY_ON(BrowserThread::UI);
+  if (!devices_enumerated_) {
+    EnsureMonitorCaptureDevices();
+    devices_enumerated_ = true;
+  }
+  return audio_forward_devices_;
+}
+
 const blink::MediaStreamDevices&
 MediaCaptureDevicesImpl::GetVideoCaptureDevices() {
   DCHECK_CURRENTLY_ON(BrowserThread::UI);
@@ -112,11 +122,32 @@ MediaCaptureDevicesImpl::MediaCaptureDevicesImpl()
 MediaCaptureDevicesImpl::~MediaCaptureDevicesImpl() {
 }
 
+blink::MediaStreamDevices MediaCaptureDevicesImpl::ExtractAudioForwardDevices(
+    blink::MediaStreamDevices& devices) {
+  // Find out tizen forwarding device, and classify it as
+  // |DISPLAY_AUDIO_CAPTURE| device.
+  blink::MediaStreamDevices forward_devices;
+  for (auto device = devices.begin(); device != devices.end(); device++) {
+    if (is_forwarding_device(device->name)) {
+      device->type = blink::mojom::MediaStreamType::DISPLAY_AUDIO_CAPTURE;
+      forward_devices.push_back(*device);
+      devices.erase(device);
+    }
+  }
+  return forward_devices;
+}
+
 void MediaCaptureDevicesImpl::UpdateAudioDevicesOnUIThread(
     const blink::MediaStreamDevices& devices) {
   DCHECK_CURRENTLY_ON(BrowserThread::UI);
   devices_enumerated_ = true;
+#if BUILDFLAG(IS_TIZEN_TV)
+  blink::MediaStreamDevices all_devices = devices;
+  audio_forward_devices_ = ExtractAudioForwardDevices(all_devices);
+  audio_devices_ = all_devices;
+#else
   audio_devices_ = devices;
+#endif
 }
 
 void MediaCaptureDevicesImpl::UpdateVideoDevicesOnUIThread(

diff --git a/content/browser/renderer_host/media/media_capture_devices_impl.h b/content/browser/renderer_host/media/media_capture_devices_impl.h
index 3c9db080f0cb2bbbc040ec98dd082e923cc2d2ab..a0b17df62b57d3a5ce9b26da3f20f9c835c0785e 100644 (file)
--- a/content/browser/renderer_host/media/media_capture_devices_impl.h
+++ b/content/browser/renderer_host/media/media_capture_devices_impl.h
@@ -20,6 +20,7 @@ class MediaCaptureDevicesImpl : public MediaCaptureDevices {
 
   // Overriden from MediaCaptureDevices
   const blink::MediaStreamDevices& GetAudioCaptureDevices() override;
+  const blink::MediaStreamDevices& GetAudioForwardDevices() override;
   const blink::MediaStreamDevices& GetVideoCaptureDevices() override;
   void AddVideoCaptureObserver(media::VideoCaptureObserver* observer) override;
   void RemoveAllVideoCaptureObservers() override;
@@ -40,6 +41,8 @@ class MediaCaptureDevicesImpl : public MediaCaptureDevices {
   MediaCaptureDevicesImpl();
   ~MediaCaptureDevicesImpl() override;
 
+  blink::MediaStreamDevices ExtractAudioForwardDevices(
+      blink::MediaStreamDevices& devices);
   void UpdateAudioDevicesOnUIThread(const blink::MediaStreamDevices& devices);
   void UpdateVideoDevicesOnUIThread(const blink::MediaStreamDevices& devices);
 
@@ -49,6 +52,7 @@ class MediaCaptureDevicesImpl : public MediaCaptureDevices {
 
   // A list of cached audio capture devices.
   blink::MediaStreamDevices audio_devices_;
+  blink::MediaStreamDevices audio_forward_devices_;
 
   // A list of cached video capture devices.
   blink::MediaStreamDevices video_devices_;

diff --git a/content/browser/renderer_host/media/media_devices_dispatcher_host.cc b/content/browser/renderer_host/media/media_devices_dispatcher_host.cc
index 8d21e8761b3a6b10abb28e995b010c2a8828984a..bf7a3b4cbd47614ad76b4d24fb335d3ae0175db8 100644 (file)
--- a/content/browser/renderer_host/media/media_devices_dispatcher_host.cc
+++ b/content/browser/renderer_host/media/media_devices_dispatcher_host.cc
@@ -507,6 +507,17 @@ void MediaDevicesDispatcherHost::GotAudioInputEnumeration(
         parameters.GetBufferDuration());
     LOG(INFO) << "audio device id:" << device_info.device_id
               << ",default id:" << default_device_id;
+
+#if BUILDFLAG(IS_TIZEN_TV)
+    // Tizen forwarding device is used to capture all tizen output audio,
+    // only available for getDisplayMedia. make it's capibility be invisible
+    // for other case.
+    if (is_forwarding_device(device_info.label)) {
+      LOG(INFO) << "ignore tizen forwarding device capability.";
+      continue;
+    }
+#endif
+
     if (device_info.device_id == default_device_id)
       current_audio_input_capabilities_.insert(
           current_audio_input_capabilities_.begin(), std::move(capabilities));

diff --git a/content/public/browser/media_capture_devices.h b/content/public/browser/media_capture_devices.h
index 1260f66ff0715c1cfefcc4940dbe151109b5c0bd..940bb7c14c42ea45a585b8ae48c29eeffb5237ce 100644 (file)
--- a/content/public/browser/media_capture_devices.h
+++ b/content/public/browser/media_capture_devices.h
@@ -24,6 +24,7 @@ class CONTENT_EXPORT  MediaCaptureDevices {
 
   // Return all Audio/Video devices.
   virtual const blink::MediaStreamDevices& GetAudioCaptureDevices() = 0;
+  virtual const blink::MediaStreamDevices& GetAudioForwardDevices() = 0;
   virtual const blink::MediaStreamDevices& GetVideoCaptureDevices() = 0;
 
   virtual void AddVideoCaptureObserver(

diff --git a/tizen_src/chromium_impl/media/audio/tizen/audio_manager_capi.cc b/tizen_src/chromium_impl/media/audio/tizen/audio_manager_capi.cc
index fd5d6aee82f75ca5df5a73669c8347748feaeac1..68995260c1a818392b04315db14ea3d8edf7b2a9 100644 (file)
--- a/tizen_src/chromium_impl/media/audio/tizen/audio_manager_capi.cc
+++ b/tizen_src/chromium_impl/media/audio/tizen/audio_manager_capi.cc
@@ -201,25 +201,27 @@ void AudioManagerCapi::GetAudioDeviceNames(
       continue;
     }
 
-    if (input && device_type == SOUND_DEVICE_FORWARDING) {
-      LOG(ERROR) << "Device with type SOUND_DEVICE_FORWARDING is audio input "
-                    "device but it doesn't provide microphone data. ignore it:"
-                 << "Device - ID:" << id << ". Name:" << name
-                 << ". type:" << device_type;
-      continue;
+    std::string modified_name{name};
+    if (device_type == SOUND_DEVICE_FORWARDING) {
+      // Used to distinguish whether it is a forwarding device by name, or we
+      // need to deliver the device type.
+      modified_name += "(forwarding)";
     }
 
     LOG(INFO) << "Device - ID:" << id << ". Name:" << name
+              << ". modified name:" << modified_name
               << ". type:" << device_type;
 #if TIZEN_VERSION_AT_LEAST(9, 0, 0)
-    device_names->push_front(AudioDeviceName(name, std::to_string(id)));
+    device_names->push_front(
+        AudioDeviceName(modified_name, std::to_string(id)));
   }
 #else
     if (device_type == SOUND_DEVICE_BUILTIN_MIC) {
       device_names_built_in.push_back(
           AudioDeviceName(name, std::to_string(id)));
     } else {
-      device_names->push_back(AudioDeviceName(name, std::to_string(id)));
+      device_names->push_back(
+          AudioDeviceName(modified_name, std::to_string(id)));
     }
   }
 

diff --git a/tizen_src/chromium_impl/media/audio/tizen/capi_usb_audio_input_stream.cc b/tizen_src/chromium_impl/media/audio/tizen/capi_usb_audio_input_stream.cc
index a5653b01363354758c55f023cbd85227fdd7c46a..25d0ed13830348de98339e70a84defbfcf3fb045 100644 (file)
--- a/tizen_src/chromium_impl/media/audio/tizen/capi_usb_audio_input_stream.cc
+++ b/tizen_src/chromium_impl/media/audio/tizen/capi_usb_audio_input_stream.cc
@@ -105,7 +105,8 @@ bool CapiUsbAudioInputStream::OpenMic() {
     return false;
   }
 
-  if (device_type != SOUND_DEVICE_BUILTIN_MIC) {
+  if (device_type != SOUND_DEVICE_BUILTIN_MIC &&
+      device_type != SOUND_DEVICE_FORWARDING) {
     ret = sound_manager_add_device_for_stream_routing(stream_info_,
                                                       sound_device_);
     if (ret != SOUND_MANAGER_ERROR_NONE) {

diff --git a/tizen_src/ewk/efl_integration/web_contents_delegate_efl.cc b/tizen_src/ewk/efl_integration/web_contents_delegate_efl.cc
index d16f6a3837f2f0625dbf007814bcc60ac1e98780..37f21cb7fe233927af5245e7c56d2f5583ff44da 100644 (file)
--- a/tizen_src/ewk/efl_integration/web_contents_delegate_efl.cc
+++ b/tizen_src/ewk/efl_integration/web_contents_delegate_efl.cc
@@ -105,6 +105,18 @@ static const blink::MediaStreamDevice* GetRequestedAudioDevice(
   return nullptr;
 }
 
+static const blink::MediaStreamDevice* GetForwardingAudioDevice() {
+  const blink::MediaStreamDevices& audio_devices =
+      MediaCaptureDevices::GetInstance()->GetAudioForwardDevices();
+  for (const auto& device : audio_devices) {
+    if (device.type == blink::mojom::MediaStreamType::DISPLAY_AUDIO_CAPTURE) {
+      return &device;
+    }
+  }
+
+  return nullptr;
+}
+
 static const blink::MediaStreamDevice* GetRequestedVideoDevice(
     const std::string& device_id) {
   const blink::MediaStreamDevices& video_devices =
@@ -397,19 +409,26 @@ void WebContentsDelegateEfl::RequestMediaAccessAllow(
   blink::mojom::StreamDevices& stream_devices =
       *stream_devices_set.stream_devices[0];
 
+  const blink::MediaStreamDevice* audio_device = nullptr;
+  bool has_audio = true;
   if (request.audio_type ==
       blink::mojom::MediaStreamType::DEVICE_AUDIO_CAPTURE) {
-    const blink::MediaStreamDevice* audio_device =
-        GetRequestedAudioDevice(request.requested_audio_device_id);
-    if (audio_device) {
-      stream_devices.audio_device = *audio_device;
-    } else {
+    audio_device = GetRequestedAudioDevice(request.requested_audio_device_id);
+  } else if (request.audio_type ==
+             blink::mojom::MediaStreamType::DISPLAY_AUDIO_CAPTURE) {
+    audio_device = GetForwardingAudioDevice();
+  } else {
+    has_audio = false;
+  }
+  if (has_audio) {
+    if (!audio_device) {
       std::move(callback).Run(
           blink::mojom::StreamDevicesSet(),
           blink::mojom::MediaStreamRequestResult::NOT_SUPPORTED,
           std::unique_ptr<MediaStreamUI>());
       return;
     }
+    stream_devices.audio_device = *audio_device;
   }
 
   if (request.video_type ==