wasapisrc: Make sure that wasapisrc produces data in loopback mode

An oddness of wasapi loopback feature is that capture client will not
produce any data if there's no outputting sound to corresponding
render client. In other words, if there's no sound to render,
capture task will stall. As an option to solve such issue, we can
add timeout to wake up from capture thread if there's no incoming data
within given time interval. But it seems to be glitch prone.
Another approach is that we can keep pushing silence data into
render client so that capture client can keep capturing data
(even if it's just silence).

This patch will choose the latter one because it's more straightforward
way and it's likely produce glitchless sound than former approach.

A bonus point of this approach is that loopback capture on Windows7/8
will work with this patch. Note that there's an OS bug prior to Windows10
when loopback capture client is running with event-driven mode.
To work around the bug, event signalling should be handled manually
for read thread to wake up.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/1588>

diff --git a/sys/wasapi/gstwasapisrc.c b/sys/wasapi/gstwasapisrc.c
index 00364df..db69170 100644 (file)
--- a/sys/wasapi/gstwasapisrc.c
+++ b/sys/wasapi/gstwasapisrc.c
@@ -195,6 +195,10 @@ gst_wasapi_src_init (GstWasapiSrc * self)
   self->client_needs_restart = FALSE;
   self->adapter = gst_adapter_new ();
 
+  /* Extra event handles used for loopback */
+  self->loopback_event_handle = CreateEvent (NULL, FALSE, FALSE, NULL);
+  self->loopback_cancellable = CreateEvent (NULL, TRUE, FALSE, NULL);
+
   CoInitializeEx (NULL, COINIT_MULTITHREADED);
 }
 
@@ -228,6 +232,21 @@ gst_wasapi_src_dispose (GObject * object)
     self->capture_client = NULL;
   }
 
+  if (self->loopback_client != NULL) {
+    IUnknown_Release (self->loopback_client);
+    self->loopback_client = NULL;
+  }
+
+  if (self->loopback_event_handle != NULL) {
+    CloseHandle (self->loopback_event_handle);
+    self->loopback_event_handle = NULL;
+  }
+
+  if (self->loopback_cancellable != NULL) {
+    CloseHandle (self->loopback_cancellable);
+    self->loopback_cancellable = NULL;
+  }
+
   G_OBJECT_CLASS (parent_class)->dispose (object);
 }
 
@@ -398,6 +417,7 @@ gst_wasapi_src_open (GstAudioSrc * asrc)
   gboolean res = FALSE;
   IAudioClient *client = NULL;
   IMMDevice *device = NULL;
+  IMMDevice *loopback_device = NULL;
 
   if (self->client)
     return TRUE;
@@ -418,6 +438,28 @@ gst_wasapi_src_open (GstAudioSrc * asrc)
     goto beach;
   }
 
+  /* An oddness of wasapi loopback feature is that capture client will not
+   * provide any audio data if there is no outputting sound.
+   * To workaround this problem, probably we can add timeout around loop
+   * in this case but it's glitch prone. So, instead of timeout,
+   * we will keep pusing silence data to into wasapi client so that make audio
+   * client report audio data in any case
+   */
+  if (!gst_wasapi_util_get_device_client (GST_ELEMENT (self),
+          eRender, self->role, self->device_strid,
+          &loopback_device, &self->loopback_client)) {
+    if (!self->device_strid)
+      GST_ELEMENT_ERROR (self, RESOURCE, OPEN_READ, (NULL),
+          ("Failed to get default device for loopback"));
+    else
+      GST_ELEMENT_ERROR (self, RESOURCE, OPEN_READ, (NULL),
+          ("Failed to open device %S", self->device_strid));
+    goto beach;
+
+    /* no need to hold this object */
+    IUnknown_Release (loopback_device);
+  }
+
   self->client = client;
   self->device = device;
   res = TRUE;
@@ -442,9 +484,114 @@ gst_wasapi_src_close (GstAudioSrc * asrc)
     self->client = NULL;
   }
 
+  if (self->loopback_client != NULL) {
+    IUnknown_Release (self->loopback_client);
+    self->loopback_client = NULL;
+  }
+
   return TRUE;
 }
 
+static gpointer
+gst_wasapi_src_loopback_silence_feeding_thread (GstWasapiSrc * self)
+{
+  HRESULT hr;
+  UINT32 buffer_frames;
+  gboolean res G_GNUC_UNUSED = FALSE;
+  BYTE *data;
+  DWORD dwWaitResult;
+  HANDLE event_handle[2];
+  UINT32 padding;
+  UINT32 n_frames;
+
+  /* NOTE: if this task cause glitch, we need to consider thread priority
+   * adjusing. See gstaudioutilsprivate.c (e.g., AvSetMmThreadCharacteristics)
+   * for this context */
+
+  GST_INFO_OBJECT (self, "Run loopback silence feeding thread");
+
+  event_handle[0] = self->loopback_event_handle;
+  event_handle[1] = self->loopback_cancellable;
+
+  hr = IAudioClient_GetBufferSize (self->loopback_client, &buffer_frames);
+  HR_FAILED_GOTO (hr, IAudioClient::GetBufferSize, beach);
+
+  hr = IAudioClient_SetEventHandle (self->loopback_client,
+      self->loopback_event_handle);
+  HR_FAILED_GOTO (hr, IAudioClient::SetEventHandle, beach);
+
+  /* To avoid start-up glitches, before starting the streaming, we fill the
+   * buffer with silence as recommended by the documentation:
+   * https://msdn.microsoft.com/en-us/library/windows/desktop/dd370879%28v=vs.85%29.aspx */
+  hr = IAudioRenderClient_GetBuffer (self->loopback_render_client,
+      buffer_frames, &data);
+  HR_FAILED_GOTO (hr, IAudioRenderClient::GetBuffer, beach);
+
+  hr = IAudioRenderClient_ReleaseBuffer (self->loopback_render_client,
+      buffer_frames, AUDCLNT_BUFFERFLAGS_SILENT);
+  HR_FAILED_GOTO (hr, IAudioRenderClient::ReleaseBuffer, beach);
+
+  hr = IAudioClient_Start (self->loopback_client);
+  HR_FAILED_GOTO (hr, IAudioClock::Start, beach);
+
+  /* There is an OS bug prior to Windows 10, that is loopback capture client
+   * will not receive event (in case of event-driven mode).
+   * A guide for workaround this case is that signal it whenever render client
+   * writes data.
+   * See https://docs.microsoft.com/en-us/windows/win32/api/audioclient/nf-audioclient-iaudioclient-initialize
+   */
+
+  /* Signal for read thread to wakeup */
+  SetEvent (self->event_handle);
+
+  /* Ok, now we are ready for running for feeding silence data */
+  while (1) {
+    dwWaitResult = WaitForMultipleObjects (2, event_handle, FALSE, INFINITE);
+    if (dwWaitResult != WAIT_OBJECT_0 && dwWaitResult != WAIT_OBJECT_0 + 1) {
+      GST_ERROR_OBJECT (self, "Error waiting for event handle: %x",
+          (guint) dwWaitResult);
+      goto stop;
+    }
+
+    /* Stopping was requested from unprepare() */
+    if (dwWaitResult == WAIT_OBJECT_0 + 1) {
+      GST_DEBUG_OBJECT (self, "operation was cancelled");
+      goto stop;
+    }
+
+    hr = IAudioClient_GetCurrentPadding (self->loopback_client, &padding);
+    HR_FAILED_GOTO (hr, IAudioClock::Start, stop);
+
+    if (buffer_frames < padding) {
+      GST_WARNING_OBJECT (self,
+          "Current padding %d is too large (buffer size %d)",
+          padding, buffer_frames);
+      n_frames = 0;
+    } else {
+      n_frames = buffer_frames - padding;
+    }
+
+    hr = IAudioRenderClient_GetBuffer (self->loopback_render_client, n_frames,
+        &data);
+    HR_FAILED_GOTO (hr, IAudioRenderClient::GetBuffer, stop);
+
+    hr = IAudioRenderClient_ReleaseBuffer (self->loopback_render_client,
+        n_frames, AUDCLNT_BUFFERFLAGS_SILENT);
+    HR_FAILED_GOTO (hr, IAudioRenderClient::ReleaseBuffer, stop);
+
+    /* Signal for read thread to wakeup */
+    SetEvent (self->event_handle);
+  }
+
+stop:
+  IAudioClient_Stop (self->loopback_client);
+
+beach:
+  GST_INFO_OBJECT (self, "Terminate loopback silence feeding thread");
+
+  return NULL;
+}
+
 static gboolean
 gst_wasapi_src_prepare (GstAudioSrc * asrc, GstAudioRingBufferSpec * spec)
 {
@@ -516,6 +663,25 @@ gst_wasapi_src_prepare (GstAudioSrc * asrc, GstAudioRingBufferSpec * spec)
     goto beach;
   }
 
+  /* In case loopback, spawn another dedicated thread for feeding silence data
+   * into wasapi render client */
+  if (self->loopback) {
+    /* don't need to be audioclient3 or low-latency since we will keep pushing
+     * silence data which is not varying over entire playback */
+    if (!gst_wasapi_util_initialize_audioclient (GST_ELEMENT (self), spec,
+            self->loopback_client, self->mix_format, self->sharemode,
+            FALSE, FALSE, &devicep_frames))
+      goto beach;
+
+    if (!gst_wasapi_util_get_render_client (GST_ELEMENT (self),
+            self->loopback_client, &self->loopback_render_client)) {
+      goto beach;
+    }
+
+    self->loopback_thread = g_thread_new ("wasapi-loopback",
+        (GThreadFunc) gst_wasapi_src_loopback_silence_feeding_thread, self);
+  }
+
   hr = IAudioClient_Start (self->client);
   HR_FAILED_GOTO (hr, IAudioClock::Start, beach);
   self->client_needs_restart = FALSE;
@@ -557,6 +723,22 @@ gst_wasapi_src_unprepare (GstAudioSrc * asrc)
     self->client_clock = NULL;
   }
 
+  if (self->loopback_thread) {
+    GST_DEBUG_OBJECT (self, "loopback task thread is stopping");
+
+    SetEvent (self->loopback_cancellable);
+
+    g_thread_join (self->loopback_thread);
+    self->loopback_thread = NULL;
+    ResetEvent (self->loopback_cancellable);
+    GST_DEBUG_OBJECT (self, "loopback task thread has been stopped");
+  }
+
+  if (self->loopback_render_client != NULL) {
+    IUnknown_Release (self->loopback_render_client);
+    self->loopback_render_client = NULL;
+  }
+
   self->client_clock_freq = 0;
 
   CoUninitialize ();

diff --git a/sys/wasapi/gstwasapisrc.h b/sys/wasapi/gstwasapisrc.h
index 3e2c968..53dacd9 100644 (file)
--- a/sys/wasapi/gstwasapisrc.h
+++ b/sys/wasapi/gstwasapisrc.h
@@ -61,6 +61,13 @@ struct _GstWasapiSrc
    * translate it to the native GStreamer channel layout. */
   GstAudioChannelPosition *positions;
 
+  /* Used for loopback use case in order to keep feeding silence into client */
+  IAudioClient *loopback_client;
+  IAudioRenderClient *loopback_render_client;
+  GThread *loopback_thread;
+  HANDLE loopback_event_handle;
+  HANDLE loopback_cancellable;
+
   /* properties */
   gint role;
   gint sharemode;