2 * Copyright (C) 2008 Ole André Vadla Ravnås <ole.andre.ravnas@tandberg.com>
3 * Copyright (C) 2018 Centricular Ltd.
4 * Author: Nirbheek Chauhan <nirbheek@centricular.com>
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Library General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Library General Public License for more details.
16 * You should have received a copy of the GNU Library General Public
17 * License along with this library; if not, write to the
18 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
19 * Boston, MA 02110-1301, USA.
23 * SECTION:element-wasapisrc
26 * Provides audio capture from the Windows Audio Session API available with
29 * ## Example pipelines
31 * gst-launch-1.0 -v wasapisrc ! fakesink
32 * ]| Capture from the default audio device and render to fakesink.
35 * gst-launch-1.0 -v wasapisrc low-latency=true ! fakesink
36 * ]| Capture from the default audio device with the minimum possible latency and render to fakesink.
43 #include "gstwasapisrc.h"
47 GST_DEBUG_CATEGORY_STATIC (gst_wasapi_src_debug);
48 #define GST_CAT_DEFAULT gst_wasapi_src_debug
50 static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src",
53 GST_STATIC_CAPS (GST_WASAPI_STATIC_CAPS));
55 #define DEFAULT_ROLE GST_WASAPI_DEVICE_ROLE_CONSOLE
56 #define DEFAULT_LOOPBACK FALSE
57 #define DEFAULT_EXCLUSIVE FALSE
58 #define DEFAULT_LOW_LATENCY FALSE
59 #define DEFAULT_AUDIOCLIENT3 FALSE
60 /* The clock provided by WASAPI is always off and causes buffers to be late
61 * very quickly on the sink. Disable pending further investigation. */
62 #define DEFAULT_PROVIDE_CLOCK FALSE
75 static void gst_wasapi_src_dispose (GObject * object);
76 static void gst_wasapi_src_finalize (GObject * object);
77 static void gst_wasapi_src_set_property (GObject * object, guint prop_id,
78 const GValue * value, GParamSpec * pspec);
79 static void gst_wasapi_src_get_property (GObject * object, guint prop_id,
80 GValue * value, GParamSpec * pspec);
82 static GstCaps *gst_wasapi_src_get_caps (GstBaseSrc * bsrc, GstCaps * filter);
84 static gboolean gst_wasapi_src_open (GstAudioSrc * asrc);
85 static gboolean gst_wasapi_src_close (GstAudioSrc * asrc);
86 static gboolean gst_wasapi_src_prepare (GstAudioSrc * asrc,
87 GstAudioRingBufferSpec * spec);
88 static gboolean gst_wasapi_src_unprepare (GstAudioSrc * asrc);
89 static guint gst_wasapi_src_read (GstAudioSrc * asrc, gpointer data,
90 guint length, GstClockTime * timestamp);
91 static guint gst_wasapi_src_delay (GstAudioSrc * asrc);
92 static void gst_wasapi_src_reset (GstAudioSrc * asrc);
94 #if DEFAULT_PROVIDE_CLOCK
95 static GstClockTime gst_wasapi_src_get_time (GstClock * clock,
99 #define gst_wasapi_src_parent_class parent_class
100 G_DEFINE_TYPE (GstWasapiSrc, gst_wasapi_src, GST_TYPE_AUDIO_SRC);
103 gst_wasapi_src_class_init (GstWasapiSrcClass * klass)
105 GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
106 GstElementClass *gstelement_class = GST_ELEMENT_CLASS (klass);
107 GstBaseSrcClass *gstbasesrc_class = GST_BASE_SRC_CLASS (klass);
108 GstAudioSrcClass *gstaudiosrc_class = GST_AUDIO_SRC_CLASS (klass);
110 gobject_class->dispose = gst_wasapi_src_dispose;
111 gobject_class->finalize = gst_wasapi_src_finalize;
112 gobject_class->set_property = gst_wasapi_src_set_property;
113 gobject_class->get_property = gst_wasapi_src_get_property;
115 g_object_class_install_property (gobject_class,
117 g_param_spec_enum ("role", "Role",
118 "Role of the device: communications, multimedia, etc",
119 GST_WASAPI_DEVICE_TYPE_ROLE, DEFAULT_ROLE, G_PARAM_READWRITE |
120 G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_READY));
122 g_object_class_install_property (gobject_class,
124 g_param_spec_string ("device", "Device",
125 "WASAPI playback device as a GUID string",
126 NULL, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
128 g_object_class_install_property (gobject_class,
130 g_param_spec_boolean ("loopback", "Loopback recording",
131 "Open the sink device for loopback recording",
132 DEFAULT_LOOPBACK, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
134 g_object_class_install_property (gobject_class,
136 g_param_spec_boolean ("exclusive", "Exclusive mode",
137 "Open the device in exclusive mode",
138 DEFAULT_EXCLUSIVE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
140 g_object_class_install_property (gobject_class,
142 g_param_spec_boolean ("low-latency", "Low latency",
143 "Optimize all settings for lowest latency. Always safe to enable.",
144 DEFAULT_LOW_LATENCY, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
146 g_object_class_install_property (gobject_class,
148 g_param_spec_boolean ("use-audioclient3", "Use the AudioClient3 API",
149 "Whether to use the Windows 10 AudioClient3 API when available",
150 DEFAULT_AUDIOCLIENT3, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
152 gst_element_class_add_static_pad_template (gstelement_class, &src_template);
153 gst_element_class_set_static_metadata (gstelement_class, "WasapiSrc",
154 "Source/Audio/Hardware",
155 "Stream audio from an audio capture device through WASAPI",
156 "Nirbheek Chauhan <nirbheek@centricular.com>, "
157 "Ole André Vadla Ravnås <ole.andre.ravnas@tandberg.com>");
159 gstbasesrc_class->get_caps = GST_DEBUG_FUNCPTR (gst_wasapi_src_get_caps);
161 gstaudiosrc_class->open = GST_DEBUG_FUNCPTR (gst_wasapi_src_open);
162 gstaudiosrc_class->close = GST_DEBUG_FUNCPTR (gst_wasapi_src_close);
163 gstaudiosrc_class->read = GST_DEBUG_FUNCPTR (gst_wasapi_src_read);
164 gstaudiosrc_class->prepare = GST_DEBUG_FUNCPTR (gst_wasapi_src_prepare);
165 gstaudiosrc_class->unprepare = GST_DEBUG_FUNCPTR (gst_wasapi_src_unprepare);
166 gstaudiosrc_class->delay = GST_DEBUG_FUNCPTR (gst_wasapi_src_delay);
167 gstaudiosrc_class->reset = GST_DEBUG_FUNCPTR (gst_wasapi_src_reset);
169 GST_DEBUG_CATEGORY_INIT (gst_wasapi_src_debug, "wasapisrc",
170 0, "Windows audio session API source");
174 gst_wasapi_src_init (GstWasapiSrc * self)
176 #if DEFAULT_PROVIDE_CLOCK
177 /* override with a custom clock */
178 if (GST_AUDIO_BASE_SRC (self)->clock)
179 gst_object_unref (GST_AUDIO_BASE_SRC (self)->clock);
181 GST_AUDIO_BASE_SRC (self)->clock = gst_audio_clock_new ("GstWasapiSrcClock",
182 gst_wasapi_src_get_time, gst_object_ref (self),
183 (GDestroyNotify) gst_object_unref);
186 self->role = DEFAULT_ROLE;
187 self->sharemode = AUDCLNT_SHAREMODE_SHARED;
188 self->loopback = DEFAULT_LOOPBACK;
189 self->low_latency = DEFAULT_LOW_LATENCY;
190 self->try_audioclient3 = DEFAULT_AUDIOCLIENT3;
191 self->event_handle = CreateEvent (NULL, FALSE, FALSE, NULL);
192 self->client_needs_restart = FALSE;
194 CoInitializeEx (NULL, COINIT_MULTITHREADED);
198 gst_wasapi_src_dispose (GObject * object)
200 GstWasapiSrc *self = GST_WASAPI_SRC (object);
202 if (self->event_handle != NULL) {
203 CloseHandle (self->event_handle);
204 self->event_handle = NULL;
207 if (self->client_clock != NULL) {
208 IUnknown_Release (self->client_clock);
209 self->client_clock = NULL;
212 if (self->client != NULL) {
213 IUnknown_Release (self->client);
217 if (self->capture_client != NULL) {
218 IUnknown_Release (self->capture_client);
219 self->capture_client = NULL;
222 G_OBJECT_CLASS (parent_class)->dispose (object);
226 gst_wasapi_src_finalize (GObject * object)
228 GstWasapiSrc *self = GST_WASAPI_SRC (object);
230 CoTaskMemFree (self->mix_format);
231 self->mix_format = NULL;
235 g_clear_pointer (&self->cached_caps, gst_caps_unref);
236 g_clear_pointer (&self->positions, g_free);
237 g_clear_pointer (&self->device_strid, g_free);
239 G_OBJECT_CLASS (parent_class)->finalize (object);
243 gst_wasapi_src_set_property (GObject * object, guint prop_id,
244 const GValue * value, GParamSpec * pspec)
246 GstWasapiSrc *self = GST_WASAPI_SRC (object);
250 self->role = gst_wasapi_device_role_to_erole (g_value_get_enum (value));
254 const gchar *device = g_value_get_string (value);
255 g_free (self->device_strid);
257 device ? g_utf8_to_utf16 (device, -1, NULL, NULL, NULL) : NULL;
261 self->loopback = g_value_get_boolean (value);
264 self->sharemode = g_value_get_boolean (value)
265 ? AUDCLNT_SHAREMODE_EXCLUSIVE : AUDCLNT_SHAREMODE_SHARED;
267 case PROP_LOW_LATENCY:
268 self->low_latency = g_value_get_boolean (value);
270 case PROP_AUDIOCLIENT3:
271 self->try_audioclient3 = g_value_get_boolean (value);
274 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
280 gst_wasapi_src_get_property (GObject * object, guint prop_id,
281 GValue * value, GParamSpec * pspec)
283 GstWasapiSrc *self = GST_WASAPI_SRC (object);
287 g_value_set_enum (value, gst_wasapi_erole_to_device_role (self->role));
290 g_value_take_string (value, self->device_strid ?
291 g_utf16_to_utf8 (self->device_strid, -1, NULL, NULL, NULL) : NULL);
294 g_value_set_boolean (value, self->loopback);
297 g_value_set_boolean (value,
298 self->sharemode == AUDCLNT_SHAREMODE_EXCLUSIVE);
300 case PROP_LOW_LATENCY:
301 g_value_set_boolean (value, self->low_latency);
303 case PROP_AUDIOCLIENT3:
304 g_value_set_boolean (value, self->try_audioclient3);
307 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
313 gst_wasapi_src_can_audioclient3 (GstWasapiSrc * self)
315 if (self->sharemode == AUDCLNT_SHAREMODE_SHARED &&
316 self->try_audioclient3 && gst_wasapi_util_have_audioclient3 ())
322 gst_wasapi_src_get_caps (GstBaseSrc * bsrc, GstCaps * filter)
324 GstWasapiSrc *self = GST_WASAPI_SRC (bsrc);
325 WAVEFORMATEX *format = NULL;
326 GstCaps *caps = NULL;
328 GST_DEBUG_OBJECT (self, "entering get caps");
330 if (self->cached_caps) {
331 caps = gst_caps_ref (self->cached_caps);
333 GstCaps *template_caps;
336 template_caps = gst_pad_get_pad_template_caps (bsrc->srcpad);
339 caps = template_caps;
343 ret = gst_wasapi_util_get_device_format (GST_ELEMENT (self),
344 self->sharemode, self->device, self->client, &format);
346 GST_ELEMENT_ERROR (self, STREAM, FORMAT, (NULL),
347 ("failed to detect format"));
348 gst_caps_unref (template_caps);
352 gst_wasapi_util_parse_waveformatex ((WAVEFORMATEXTENSIBLE *) format,
353 template_caps, &caps, &self->positions);
355 GST_ELEMENT_ERROR (self, STREAM, FORMAT, (NULL), ("unknown format"));
356 gst_caps_unref (template_caps);
361 gchar *pos_str = gst_audio_channel_positions_to_string (self->positions,
363 GST_INFO_OBJECT (self, "positions are: %s", pos_str);
367 self->mix_format = format;
368 gst_caps_replace (&self->cached_caps, caps);
369 gst_caps_unref (template_caps);
374 gst_caps_intersect_full (filter, caps, GST_CAPS_INTERSECT_FIRST);
375 gst_caps_unref (caps);
380 GST_DEBUG_OBJECT (self, "returning caps %" GST_PTR_FORMAT, caps);
385 gst_wasapi_src_open (GstAudioSrc * asrc)
387 GstWasapiSrc *self = GST_WASAPI_SRC (asrc);
388 gboolean res = FALSE;
389 IAudioClient *client = NULL;
390 IMMDevice *device = NULL;
395 /* FIXME: Switching the default device does not switch the stream to it,
396 * even if the old device was unplugged. We need to handle this somehow.
397 * For example, perhaps we should automatically switch to the new device if
398 * the default device is changed and a device isn't explicitly selected. */
399 if (!gst_wasapi_util_get_device_client (GST_ELEMENT (self),
400 self->loopback ? eRender : eCapture, self->role, self->device_strid,
402 if (!self->device_strid)
403 GST_ELEMENT_ERROR (self, RESOURCE, OPEN_READ, (NULL),
404 ("Failed to get default device"));
406 GST_ELEMENT_ERROR (self, RESOURCE, OPEN_READ, (NULL),
407 ("Failed to open device %S", self->device_strid));
411 self->client = client;
412 self->device = device;
421 gst_wasapi_src_close (GstAudioSrc * asrc)
423 GstWasapiSrc *self = GST_WASAPI_SRC (asrc);
425 if (self->device != NULL) {
426 IUnknown_Release (self->device);
430 if (self->client != NULL) {
431 IUnknown_Release (self->client);
439 gst_wasapi_src_prepare (GstAudioSrc * asrc, GstAudioRingBufferSpec * spec)
441 GstWasapiSrc *self = GST_WASAPI_SRC (asrc);
442 gboolean res = FALSE;
443 REFERENCE_TIME latency_rt;
444 guint bpf, rate, devicep_frames, buffer_frames;
447 CoInitializeEx (NULL, COINIT_MULTITHREADED);
449 if (gst_wasapi_src_can_audioclient3 (self)) {
450 if (!gst_wasapi_util_initialize_audioclient3 (GST_ELEMENT (self), spec,
451 (IAudioClient3 *) self->client, self->mix_format, self->low_latency,
452 self->loopback, &devicep_frames))
455 if (!gst_wasapi_util_initialize_audioclient (GST_ELEMENT (self), spec,
456 self->client, self->mix_format, self->sharemode, self->low_latency,
457 self->loopback, &devicep_frames))
461 bpf = GST_AUDIO_INFO_BPF (&spec->info);
462 rate = GST_AUDIO_INFO_RATE (&spec->info);
464 /* Total size in frames of the allocated buffer that we will read from */
465 hr = IAudioClient_GetBufferSize (self->client, &buffer_frames);
466 HR_FAILED_GOTO (hr, IAudioClient::GetBufferSize, beach);
468 GST_INFO_OBJECT (self, "buffer size is %i frames, device period is %i "
469 "frames, bpf is %i bytes, rate is %i Hz", buffer_frames,
470 devicep_frames, bpf, rate);
472 /* Actual latency-time/buffer-time will be different now */
473 spec->segsize = devicep_frames * bpf;
475 /* We need a minimum of 2 segments to ensure glitch-free playback */
476 spec->segtotal = MAX (buffer_frames * bpf / spec->segsize, 2);
478 GST_INFO_OBJECT (self, "segsize is %i, segtotal is %i", spec->segsize,
481 /* Get WASAPI latency for logging */
482 hr = IAudioClient_GetStreamLatency (self->client, &latency_rt);
483 HR_FAILED_GOTO (hr, IAudioClient::GetStreamLatency, beach);
485 GST_INFO_OBJECT (self, "wasapi stream latency: %" G_GINT64_FORMAT " (%"
486 G_GINT64_FORMAT " ms)", latency_rt, latency_rt / 10000);
488 /* Set the event handler which will trigger reads */
489 hr = IAudioClient_SetEventHandle (self->client, self->event_handle);
490 HR_FAILED_GOTO (hr, IAudioClient::SetEventHandle, beach);
492 /* Get the clock and the clock freq */
493 if (!gst_wasapi_util_get_clock (GST_ELEMENT (self), self->client,
494 &self->client_clock))
497 hr = IAudioClock_GetFrequency (self->client_clock, &self->client_clock_freq);
498 HR_FAILED_GOTO (hr, IAudioClock::GetFrequency, beach);
500 GST_INFO_OBJECT (self, "wasapi clock freq is %" G_GUINT64_FORMAT,
501 self->client_clock_freq);
503 /* Get capture source client and start it up */
504 if (!gst_wasapi_util_get_capture_client (GST_ELEMENT (self), self->client,
505 &self->capture_client)) {
509 hr = IAudioClient_Start (self->client);
510 HR_FAILED_GOTO (hr, IAudioClock::Start, beach);
511 self->client_needs_restart = FALSE;
513 gst_audio_ring_buffer_set_channel_positions (GST_AUDIO_BASE_SRC
514 (self)->ringbuffer, self->positions);
518 /* unprepare() is not called if prepare() fails, but we want it to be, so call
519 * it manually when needed */
521 gst_wasapi_src_unprepare (asrc);
527 gst_wasapi_src_unprepare (GstAudioSrc * asrc)
529 GstWasapiSrc *self = GST_WASAPI_SRC (asrc);
531 if (self->client != NULL) {
532 IAudioClient_Stop (self->client);
535 if (self->capture_client != NULL) {
536 IUnknown_Release (self->capture_client);
537 self->capture_client = NULL;
540 if (self->client_clock != NULL) {
541 IUnknown_Release (self->client_clock);
542 self->client_clock = NULL;
545 self->client_clock_freq = 0;
553 gst_wasapi_src_read (GstAudioSrc * asrc, gpointer data, guint length,
554 GstClockTime * timestamp)
556 GstWasapiSrc *self = GST_WASAPI_SRC (asrc);
559 guint wanted = length;
563 GST_OBJECT_LOCK (self);
564 if (self->client_needs_restart) {
565 hr = IAudioClient_Start (self->client);
566 HR_FAILED_ELEMENT_ERROR_AND (hr, IAudioClient::Start, self,
567 GST_OBJECT_UNLOCK (self); goto err);
568 self->client_needs_restart = FALSE;
571 bpf = self->mix_format->nBlockAlign;
572 GST_OBJECT_UNLOCK (self);
576 guint have_frames, n_frames, want_frames, read_len;
578 /* Wait for data to become available */
579 dwWaitResult = WaitForSingleObject (self->event_handle, INFINITE);
580 if (dwWaitResult != WAIT_OBJECT_0) {
581 GST_ERROR_OBJECT (self, "Error waiting for event handle: %x",
582 (guint) dwWaitResult);
586 hr = IAudioCaptureClient_GetBuffer (self->capture_client,
587 (BYTE **) & from, &have_frames, &flags, NULL, NULL);
589 if (hr == AUDCLNT_S_BUFFER_EMPTY) {
590 gchar *msg = gst_wasapi_util_hresult_to_string (hr);
591 GST_WARNING_OBJECT (self, "IAudioCaptureClient::GetBuffer failed: %s"
597 HR_FAILED_ELEMENT_ERROR_AND (hr, IAudioCaptureClient::GetBuffer, self,
602 GST_INFO_OBJECT (self, "buffer flags=%#08x", (guint) flags);
604 /* XXX: How do we handle AUDCLNT_BUFFERFLAGS_SILENT? We're supposed to write
605 * out silence when that flag is set? See:
606 * https://msdn.microsoft.com/en-us/library/windows/desktop/dd370800(v=vs.85).aspx */
608 if (flags & AUDCLNT_BUFFERFLAGS_DATA_DISCONTINUITY)
609 GST_WARNING_OBJECT (self, "WASAPI reported glitch in buffer");
611 want_frames = wanted / bpf;
613 /* If GetBuffer is returning more frames than we can handle, all we can do is
614 * hope that this is temporary and that things will settle down later. */
615 if (G_UNLIKELY (have_frames > want_frames))
616 GST_WARNING_OBJECT (self, "captured too many frames: have %i, want %i",
617 have_frames, want_frames);
619 /* Only copy data that will fit into the allocated buffer of size @length */
620 n_frames = MIN (have_frames, want_frames);
621 read_len = n_frames * bpf;
623 GST_DEBUG_OBJECT (self, "have: %i (%i bytes), can read: %i (%i bytes), "
624 "will read: %i (%i bytes)", have_frames, have_frames * bpf,
625 want_frames, wanted, n_frames, read_len);
627 memcpy (data, from, read_len);
630 /* Always release all captured buffers if we've captured any at all */
631 hr = IAudioCaptureClient_ReleaseBuffer (self->capture_client, have_frames);
632 HR_FAILED_ELEMENT_ERROR_AND (hr, IAudioCaptureClient::ReleaseBuffer, self,
646 gst_wasapi_src_delay (GstAudioSrc * asrc)
648 GstWasapiSrc *self = GST_WASAPI_SRC (asrc);
652 hr = IAudioClient_GetCurrentPadding (self->client, &delay);
653 HR_FAILED_RET (hr, IAudioClock::GetCurrentPadding, 0);
659 gst_wasapi_src_reset (GstAudioSrc * asrc)
661 GstWasapiSrc *self = GST_WASAPI_SRC (asrc);
667 GST_OBJECT_LOCK (self);
668 hr = IAudioClient_Stop (self->client);
669 HR_FAILED_RET (hr, IAudioClock::Stop,);
671 hr = IAudioClient_Reset (self->client);
672 HR_FAILED_RET (hr, IAudioClock::Reset,);
674 self->client_needs_restart = TRUE;
675 GST_OBJECT_UNLOCK (self);
678 #if DEFAULT_PROVIDE_CLOCK
680 gst_wasapi_src_get_time (GstClock * clock, gpointer user_data)
682 GstWasapiSrc *self = GST_WASAPI_SRC (user_data);
687 if (G_UNLIKELY (self->client_clock == NULL))
688 return GST_CLOCK_TIME_NONE;
690 hr = IAudioClock_GetPosition (self->client_clock, &devpos, NULL);
691 HR_FAILED_RET (hr, IAudioClock::GetPosition, GST_CLOCK_TIME_NONE);
693 result = gst_util_uint64_scale_int (devpos, GST_SECOND,
694 self->client_clock_freq);
697 GST_DEBUG_OBJECT (self, "devpos = %" G_GUINT64_FORMAT
698 " frequency = %" G_GUINT64_FORMAT
699 " result = %" G_GUINT64_FORMAT " ms",
700 devpos, self->client_clock_freq, GST_TIME_AS_MSECONDS (result));