From 3b7f397863f0e792086c34de7acb3962ef0ec889 Mon Sep 17 00:00:00 2001 From: Vivienne Watermeier Date: Thu, 3 Feb 2022 15:01:46 +0100 Subject: [PATCH] ximagesink: Add touch event support Send touch events for XI_TouchBegin, XI_TouchEnd, and XI_TouchUpdate events, grouping events with identical timestamps into one TOUCH_FRAME. Part-of: --- subprojects/gst-plugins-base/meson_options.txt | 1 + subprojects/gst-plugins-base/sys/meson.build | 2 + .../gst-plugins-base/sys/ximage/meson.build | 2 +- .../gst-plugins-base/sys/ximage/ximagesink.c | 287 ++++++++++++++++++++- .../gst-plugins-base/sys/ximage/ximagesink.h | 33 +++ 5 files changed, 318 insertions(+), 7 deletions(-) diff --git a/subprojects/gst-plugins-base/meson_options.txt b/subprojects/gst-plugins-base/meson_options.txt index 7010b91..3c7641c 100644 --- a/subprojects/gst-plugins-base/meson_options.txt +++ b/subprojects/gst-plugins-base/meson_options.txt @@ -65,6 +65,7 @@ option('vorbis', type : 'feature', value : 'auto', description : 'Vorbis audio p option('x11', type : 'feature', value : 'auto', description : 'X11 ximagesink plugin, and X11 support in libraries, plugins, examples') option('xshm', type : 'feature', value : 'auto', description : 'X11 shared memory support for X11 plugins') option('xvideo', type : 'feature', value : 'auto', description : 'X11 XVideo xvimagesink plugin') +option('xi', type : 'feature', value : 'auto', description : 'X11 input extension for touch support in ximagesink') # Common feature options option('examples', type : 'feature', value : 'auto', yield : true) diff --git a/subprojects/gst-plugins-base/sys/meson.build b/subprojects/gst-plugins-base/sys/meson.build index 7637c1c..a93e075 100644 --- a/subprojects/gst-plugins-base/sys/meson.build +++ b/subprojects/gst-plugins-base/sys/meson.build @@ -2,7 +2,9 @@ if x11_dep.found() # XShm is only used by ximage and xvimage # FIXME: Need to check for XShmAttach inside libXext xshm_dep = dependency('xext', required : get_option('xshm')) + xi_dep = dependency('xi', required : get_option('xi')) core_conf.set('HAVE_XSHM', xshm_dep.found()) + core_conf.set('HAVE_XI2', xi_dep.found()) subdir('ximage') subdir('xvimage') diff --git a/subprojects/gst-plugins-base/sys/ximage/meson.build b/subprojects/gst-plugins-base/sys/ximage/meson.build index c2ce795..b96b4f1 100644 --- a/subprojects/gst-plugins-base/sys/ximage/meson.build +++ b/subprojects/gst-plugins-base/sys/ximage/meson.build @@ -8,7 +8,7 @@ gstximage = library('gstximagesink', 'ximagesink.c', 'ximage.c', 'ximagepool.c', c_args : gst_plugins_base_args + no_warn_args, include_directories: [configinc, libsinc], - dependencies : glib_deps + [video_dep, gst_base_dep, gst_dep, x11_dep, xshm_dep], + dependencies : glib_deps + [video_dep, gst_base_dep, gst_dep, x11_dep, xshm_dep, xi_dep], install : true, install_dir : plugins_install_dir, ) diff --git a/subprojects/gst-plugins-base/sys/ximage/ximagesink.c b/subprojects/gst-plugins-base/sys/ximage/ximagesink.c index 8bf8464..f983a3c 100644 --- a/subprojects/gst-plugins-base/sys/ximage/ximagesink.c +++ b/subprojects/gst-plugins-base/sys/ximage/ximagesink.c @@ -115,6 +115,11 @@ /* for XkbKeycodeToKeysym */ #include +/* used for touchscreen events */ +#ifdef HAVE_XI2 +#include +#endif + GST_DEBUG_CATEGORY_EXTERN (gst_debug_x_image_pool); GST_DEBUG_CATEGORY (gst_debug_x_image_sink); GST_DEBUG_CATEGORY_STATIC (CAT_PERFORMANCE); @@ -432,6 +437,98 @@ gst_x_image_sink_xwindow_set_title (GstXImageSink * ximagesink, } } +#ifdef HAVE_XI2 +static void +gst_x_image_sink_xtouchdevice_free (GstXTouchDevice * device) +{ + g_free (device->name); +} + +static void +gst_x_image_sink_xwindow_select_touch_events (GstXImageSink * ximagesink, + GstXWindow * xwindow) +{ + XIDeviceInfo *devices; + int ndevices, i, j, mask_len; + unsigned char *mask; + + ximagesink->touch_devices = g_array_remove_range (ximagesink->touch_devices, + 0, ximagesink->touch_devices->len); + + mask_len = (XI_LASTEVENT + 7) << 3; + mask = g_new0 (unsigned char, mask_len); + XISetMask (mask, XI_TouchBegin); + XISetMask (mask, XI_TouchUpdate); + XISetMask (mask, XI_TouchEnd); + + devices = XIQueryDevice (ximagesink->xcontext->disp, XIAllDevices, &ndevices); + + /* Find suitable touch screen devices, and select touch events for each */ + for (i = 0; i < ndevices; i++) { + XIEventMask mask_data; + GstXTouchDevice temp, *device; + gboolean has_touch = FALSE; + + if (devices[i].use != XISlavePointer) + continue; + + temp.pressure_valuator = -1; + temp.id = devices[i].deviceid; + temp.name = devices[i].name; + + for (j = 0; j < devices[i].num_classes; j++) { + XIAnyClassInfo *class = devices[i].classes[j]; + + /* only pick devices with direct touch, to avoid touchpads and similar */ + if (class->type == XITouchClass && + ((XITouchClassInfo *) class)->mode == XIDirectTouch) { + has_touch = TRUE; + } + + /* Find the valuator representing pressure, if present */ + if (class->type == XIValuatorClass) { + XIValuatorClassInfo *val_info = (XIValuatorClassInfo *) class; + + if (val_info->label == XInternAtom (ximagesink->xcontext->disp, + "Abs Pressure", TRUE)) + temp.abs_pressure = TRUE; + else if (val_info->label == XInternAtom (ximagesink->xcontext->disp, + "Rel Pressure", TRUE)) + temp.abs_pressure = FALSE; + else + continue; + + temp.pressure_valuator = i; + temp.pressure_min = val_info->min; + temp.pressure_max = val_info->max; + temp.current_pressure = temp.pressure_min; + } + } + + if (has_touch) { + GST_DEBUG ("found%s touch screen with id %d: %s", + temp.pressure_valuator < 0 ? "" : (temp.abs_pressure ? + "pressure-sensitive (abs)" : "pressure-sensitive (rel)"), + temp.id, temp.name); + + device = g_new (GstXTouchDevice, 1); + *device = temp; + device->name = g_strdup (device->name); + ximagesink->touch_devices = + g_array_append_vals (ximagesink->touch_devices, device, 1); + + mask_data.deviceid = temp.id; + mask_data.mask_len = mask_len; + mask_data.mask = mask; + XISelectEvents (ximagesink->xcontext->disp, xwindow->win, &mask_data, 1); + } + } + + g_free (mask); + XIFreeDeviceInfo (devices); +} +#endif + /* This function handles a GstXWindow creation */ static GstXWindow * gst_x_image_sink_xwindow_new (GstXImageSink * ximagesink, gint width, @@ -474,6 +571,23 @@ gst_x_image_sink_xwindow_new (GstXImageSink * ximagesink, gint width, "WM_DELETE_WINDOW", False); (void) XSetWMProtocols (ximagesink->xcontext->disp, xwindow->win, &wm_delete, 1); + +#ifdef HAVE_XI2 + if (ximagesink->xcontext->use_xi2) { + XIEventMask mask_data; + unsigned char mask[2]; + + gst_x_image_sink_xwindow_select_touch_events (ximagesink, xwindow); + + XISetMask (mask, XI_HierarchyChanged); + mask_data.deviceid = XIAllDevices; + mask_data.mask_len = sizeof (mask); + mask_data.mask = mask; + + /* register for HierarchyChanged events to see device changes */ + XISelectEvents (ximagesink->xcontext->disp, xwindow->win, &mask_data, 1); + } +#endif } xwindow->gc = XCreateGC (ximagesink->xcontext->disp, xwindow->win, @@ -577,7 +691,7 @@ gst_x_image_sink_handle_xevents (GstXImageSink * ximagesink) { XEvent e; gint pointer_x = 0, pointer_y = 0; - gboolean pointer_moved = FALSE; + gboolean pointer_moved = FALSE, touch_frame_open = FALSE; gboolean exposed = FALSE, configured = FALSE; g_return_if_fail (GST_IS_X_IMAGE_SINK (ximagesink)); @@ -708,11 +822,11 @@ gst_x_image_sink_handle_xevents (GstXImageSink * ximagesink) g_mutex_lock (&ximagesink->x_lock); } - /* Handle Display events */ while (XPending (ximagesink->xcontext->disp)) { XNextEvent (ximagesink->xcontext->disp, &e); switch (e.type) { + /* Handle Display events */ case ClientMessage:{ Atom wm_delete; @@ -730,13 +844,137 @@ gst_x_image_sink_handle_xevents (GstXImageSink * ximagesink) } break; } - default: + default:{ +#ifdef HAVE_XI2 + /* Handle XInput2 touch screen events */ + if (ximagesink->xcontext->use_xi2 + && XGetEventData (ximagesink->xcontext->disp, + (XGenericEventCookie *) & e) + && e.xcookie.extension == ximagesink->xcontext->xi_opcode) { + XGenericEventCookie *cookie; + XIDeviceEvent *touch; + GstXTouchDevice device; + GstEvent *nav; + gdouble pressure; + gboolean device_found; + unsigned int ev_id, i; + + cookie = &e.xcookie; + nav = NULL; + + if (cookie->evtype == XI_HierarchyChanged) { + GST_DEBUG ("ximagesink devices changed, searching for " + "touch devices again"); + gst_x_image_sink_xwindow_select_touch_events (ximagesink, + ximagesink->xwindow); + break; + } + + if (cookie->evtype != XI_TouchBegin + && cookie->evtype != XI_TouchUpdate + && cookie->evtype != XI_TouchEnd) + break; + + touch = cookie->data; + ev_id = ((unsigned int) touch->deviceid) << 16 | + (((unsigned int) touch->detail) & 0x00ff); + + /* find device that the event belongs to */ + device_found = FALSE; + for (i = 0; i < ximagesink->touch_devices->len; i++) { + device = g_array_index (ximagesink->touch_devices, + GstXTouchDevice, i); + if (device.id == touch->deviceid) { + device_found = TRUE; + break; + } + } + if (!device_found) + break; + + if (device.pressure_valuator >= 0) { + pressure = touch->valuators.values[device.pressure_valuator]; + pressure -= device.pressure_min; + pressure /= device.pressure_max - device.pressure_min; + if (!device.abs_pressure) + pressure += device.current_pressure; + } else + pressure = NAN; + + device.current_pressure = pressure; + + g_mutex_unlock (&ximagesink->x_lock); + g_mutex_unlock (&ximagesink->flow_lock); + + /* assume the event queue is ordered chronologically, and end */ + /* the previous touch event frame when the timestamp increases */ + if (touch->time != ximagesink->last_touch && touch_frame_open) { + if (touch->time < ximagesink->last_touch) { + GST_WARNING ("ximagesink out of order touch event " + "with timestamp %lu, not ending touch frame", touch->time); + } else { + GST_DEBUG ("ximagesink ending touch frame for %lu", + ximagesink->last_touch); + gst_navigation_send_event_simple (GST_NAVIGATION (ximagesink), + gst_navigation_event_new_touch_frame ()); + } + } + + switch (cookie->evtype) { + case XI_TouchBegin:{ + GST_DEBUG ("ximagesink new touch point %d on device %d " + "at %.0f,%.0f", touch->detail, touch->deviceid, + touch->event_x, touch->event_y); + nav = gst_navigation_event_new_touch_down (ev_id, touch->event_x, + touch->event_y, pressure); + break; + } + case XI_TouchEnd:{ + GST_DEBUG ("ximagesink removed touch point %d from device %d " + "at %.0f,%.0f", touch->detail, touch->deviceid, + touch->event_x, touch->event_y); + nav = gst_navigation_event_new_touch_up (ev_id, touch->event_x, + touch->event_y); + break; + } + case XI_TouchUpdate:{ + GST_DEBUG ("ximagesink touch point %d on device %d moved " + "to %.0f,%.0f", touch->detail, touch->deviceid, + touch->event_x, touch->event_y); + nav = gst_navigation_event_new_touch_motion (ev_id, + touch->event_x, touch->event_y, pressure); + break; + } + default: + break; + } + + gst_navigation_send_event_simple (GST_NAVIGATION (ximagesink), nav); + ximagesink->last_touch = touch->time; + touch_frame_open = TRUE; + + g_mutex_lock (&ximagesink->flow_lock); + g_mutex_lock (&ximagesink->x_lock); + XFreeEventData (ximagesink->xcontext->disp, cookie); + } +#endif break; + } } } g_mutex_unlock (&ximagesink->x_lock); g_mutex_unlock (&ximagesink->flow_lock); + +#ifdef HAVE_XI2 + /* send a touch-frame event now to avoid delay from having to wait for the */ + /* next invocation */ + if (touch_frame_open) { + GST_DEBUG ("ximagesink ending touch frame for %lu", ximagesink->last_touch); + gst_navigation_send_event_simple (GST_NAVIGATION (ximagesink), + gst_navigation_event_new_touch_frame ()); + } +#endif } static gpointer @@ -872,8 +1110,12 @@ gst_x_image_sink_xcontext_get (GstXImageSink * ximagesink) GstVideoFormat vformat; guint32 alpha_mask; int opcode, event, err; - int major = XkbMajorVersion; - int minor = XkbMinorVersion; + int xkb_major = XkbMajorVersion; + int xkb_minor = XkbMinorVersion; +#ifdef HAVE_XI2 + int xi2_major = XI_2_Major; + int xi2_minor = XI_2_Minor; +#endif g_return_val_if_fail (GST_IS_X_IMAGE_SINK (ximagesink), NULL); @@ -945,7 +1187,8 @@ gst_x_image_sink_xcontext_get (GstXImageSink * ximagesink) xcontext->use_xshm = FALSE; GST_DEBUG ("ximagesink is not using XShm extension"); } - if (XkbQueryExtension (xcontext->disp, &opcode, &event, &err, &major, &minor)) { + if (XkbQueryExtension (xcontext->disp, &opcode, &event, &err, + &xkb_major, &xkb_minor)) { xcontext->use_xkb = TRUE; GST_DEBUG ("ximagesink is using Xkb extension"); } else { @@ -953,6 +1196,22 @@ gst_x_image_sink_xcontext_get (GstXImageSink * ximagesink) GST_DEBUG ("ximagesink is not using Xkb extension"); } + /* Search for XInput extension support */ +#ifdef HAVE_XI2 + if (XQueryExtension (xcontext->disp, "XInputExtension", + &(xcontext->xi_opcode), &event, &err) + && (XIQueryVersion (xcontext->disp, &xi2_major, &xi2_minor) == Success + && (xi2_major > 2 || (xi2_major == 2 && xi2_minor >= 2)))) { + xcontext->use_xi2 = TRUE; + GST_DEBUG ("ximagesink is using XInput extension"); + } else +#endif /* HAVE_XI2 */ + { + xcontext->use_xi2 = FALSE; + GST_DEBUG ("ximagesink is not using XInput extension: " + "XInput is not present"); + } + /* extrapolate alpha mask */ if (xcontext->depth == 32) { alpha_mask = ~(xcontext->visual->red_mask @@ -1865,6 +2124,11 @@ gst_x_image_sink_reset (GstXImageSink * ximagesink) g_mutex_lock (&ximagesink->flow_lock); +#ifdef HAVE_XI2 + ximagesink->touch_devices = g_array_remove_range (ximagesink->touch_devices, + 0, ximagesink->touch_devices->len); +#endif + if (ximagesink->pool) { gst_object_unref (ximagesink->pool); ximagesink->pool = NULL; @@ -1897,6 +2161,9 @@ gst_x_image_sink_finalize (GObject * object) g_free (ximagesink->par); ximagesink->par = NULL; } +#ifdef HAVE_XI2 + g_array_free (ximagesink->touch_devices, TRUE); +#endif g_mutex_clear (&ximagesink->x_lock); g_mutex_clear (&ximagesink->flow_lock); @@ -1919,6 +2186,14 @@ gst_x_image_sink_init (GstXImageSink * ximagesink) ximagesink->fps_n = 0; ximagesink->fps_d = 1; +#ifdef HAVE_XI2 + ximagesink->last_touch = 0; + ximagesink->touch_devices = g_array_new (FALSE, FALSE, + sizeof (GstXTouchDevice)); + g_array_set_clear_func (ximagesink->touch_devices, + (GDestroyNotify) gst_x_image_sink_xtouchdevice_free); +#endif + g_mutex_init (&ximagesink->x_lock); g_mutex_init (&ximagesink->flow_lock); diff --git a/subprojects/gst-plugins-base/sys/ximage/ximagesink.h b/subprojects/gst-plugins-base/sys/ximage/ximagesink.h index 98dce61..9150c5a 100644 --- a/subprojects/gst-plugins-base/sys/ximage/ximagesink.h +++ b/subprojects/gst-plugins-base/sys/ximage/ximagesink.h @@ -55,6 +55,7 @@ G_BEGIN_DECLS typedef struct _GstXContext GstXContext; typedef struct _GstXWindow GstXWindow; +typedef struct _GstXTouchDevice GstXTouchDevice; typedef struct _GstXImageSink GstXImageSink; typedef struct _GstXImageSinkClass GstXImageSinkClass; @@ -83,6 +84,9 @@ typedef struct _GstXImageSinkClass GstXImageSinkClass; * if the Extension is present * @use_xkb: used to known wether of not Xkb extension is usable or not even * if the Extension is present + * @use_xi2: used to known wether of not XInput2 extension is usable or not even + * if the Extension is present + * @xi_opcode: XInput opcode * @caps: the #GstCaps that Display @disp can accept * * Structure used to store various information collected/calculated for a @@ -110,6 +114,9 @@ struct _GstXContext gboolean use_xshm; gboolean use_xkb; + gboolean use_xi2; + + int xi_opcode; GstCaps *caps; GstCaps *last_caps; @@ -134,6 +141,25 @@ struct _GstXWindow GC gc; }; +/* + * GstXTouchDevice: + * @name: the name of this decive + * @id: the device ID of this device + * @pressure_valuator: index of the valuator that encodes pressure data, if present + * @abs_pressure: if pressure is reported in absolute or relative values + * @current_pressure: stores the most recent pressure value for this device + * @pressure_min: lowest possible pressure value + * @pressure_max: highest possible pressure value + * + * Structure used to store information about a touchscreen device. + */ +struct _GstXTouchDevice { + gchar *name; + gint id, pressure_valuator; + gboolean abs_pressure; + gdouble current_pressure, pressure_min, pressure_max; +}; + /** * GstXImageSink: * @display_name: the name of the Display we want to render to @@ -147,6 +173,8 @@ struct _GstXWindow * @running: used to inform @event_thread if it should run/shutdown * @fps_n: the framerate fraction numerator * @fps_d: the framerate fraction denominator + * @last_touch: timestamp of the last received touch event + * @touch_devices: list of known touchscreen devices * @x_lock: used to protect X calls as we are not using the XLib in threaded * mode * @flow_lock: used to protect data flow routines from external calls such as @@ -183,6 +211,11 @@ struct _GstXImageSink gint fps_n; gint fps_d; +#ifdef HAVE_XI2 + Time last_touch; + GArray *touch_devices; +#endif + GMutex x_lock; GMutex flow_lock; -- 2.7.4