From a3cc71ae5f6b2280d2fb9a2702f706dced7f368b Mon Sep 17 00:00:00 2001 From: Jonathan Matthew Date: Fri, 25 Nov 2011 16:51:59 +0100 Subject: [PATCH] [482/906] ext: Add a libvisual plugin wrapping opengl libvisual Fix bug #202069 --- ext/libvisual/CMakeLists.txt | 15 + ext/libvisual/Makefile.am | 15 + ext/libvisual/visual-gl.c | 1143 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 1173 insertions(+) create mode 100644 ext/libvisual/CMakeLists.txt create mode 100644 ext/libvisual/Makefile.am create mode 100644 ext/libvisual/visual-gl.c diff --git a/ext/libvisual/CMakeLists.txt b/ext/libvisual/CMakeLists.txt new file mode 100644 index 0000000..490ed89 --- /dev/null +++ b/ext/libvisual/CMakeLists.txt @@ -0,0 +1,15 @@ +add_definitions(-DHAVE_CONFIG_H) + +include_directories(AFTER + ${GSTREAMER_INCLUDE_DIR} + ${GLIB2_INCLUDE_DIR} + ../../gst-libs/gst/gl + ../../gst-libs + ../.. + .) + +add_library (gstlibvisualgl SHARED + visual-gl.c) + +target_link_libraries(gstlibvisualgl + general gstgl) diff --git a/ext/libvisual/Makefile.am b/ext/libvisual/Makefile.am new file mode 100644 index 0000000..9868083 --- /dev/null +++ b/ext/libvisual/Makefile.am @@ -0,0 +1,15 @@ +plugin_LTLIBRARIES = libgstlibvisualgl.la + +libgstlibvisualgl_la_SOURCES = visual-gl.c + +libgstlibvisualgl_la_CFLAGS = $(GST_PLUGINS_BASE_CFLAGS) $(GST_CFLAGS) $(LIBVISUAL_CFLAGS) \ + -I$(top_srcdir)/gst-libs \ + -I$(top_srcdir)/gst-libs/gst/gl + +libgstlibvisualgl_la_LIBADD = \ + $(top_builddir)/gst-libs/gst/gl/libgstgl-$(GST_MAJORMINOR).la \ + $(GST_PLUGINS_BASE_LIBS) $(GST_LIBS) $(LIBVISUAL_LIBS) + +libgstlibvisualgl_la_LDFLAGS = $(GST_PLUGIN_LDFLAGS) +libgstlibvisualgl_la_LIBTOOLFLAGS = --tag=disable-static + diff --git a/ext/libvisual/visual-gl.c b/ext/libvisual/visual-gl.c new file mode 100644 index 0000000..294dfc0 --- /dev/null +++ b/ext/libvisual/visual-gl.c @@ -0,0 +1,1143 @@ +/* GStreamer + * Copyright (C) 2004 Benjamin Otte + * Copyright (C) 2009 Jonathan Matthew + * Copyright (C) 2011 Julien Isorce + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +/** + * SECTION:element-libvisualgl + * + * Wrapper for libvisual plugins that use OpenGL + * + * + * Examples + * |[ + * gst-launch -v audiotestsrc ! libvisual_gl_lv_flower ! glimagesink + * ]| + * + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include + +#include + +#define GST_TYPE_VISUAL_GL (gst_visual_gl_get_type()) +#define GST_IS_VISUAL_GL(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_VISUAL_GL)) +#define GST_VISUAL_GL(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_VISUAL_GL,GstVisualGL)) +#define GST_IS_VISUAL_GL_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_VISUAL_GL)) +#define GST_VISUAL_GL_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_VISUAL_GL,GstVisualGLClass)) +#define GST_VISUAL_GL_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS ((obj), GST_TYPE_VISUAL_GL, GstVisualGLClass)) + +typedef struct _GstVisualGL GstVisualGL; +typedef struct _GstVisualGLClass GstVisualGLClass; + +/* XXX use same category as libvisual plugin in -base? */ +GST_DEBUG_CATEGORY_STATIC (libvisual_debug); +#define GST_CAT_DEFAULT (libvisual_debug) + +/* amounf of samples before we can feed libvisual */ +#define VISUAL_SAMPLES 512 + +#define DEFAULT_WIDTH 320 +#define DEFAULT_HEIGHT 240 +#define DEFAULT_FPS_N 25 +#define DEFAULT_FPS_D 1 + +struct _GstVisualGL +{ + GstElement element; + + /* pads */ + GstPad *sinkpad; + GstPad *srcpad; + GstSegment segment; + + /* GL stuff */ + GstGLDisplay *display; + GLuint fbo; + GLuint depthbuffer; + GLdouble actor_projection_matrix[16]; + GLdouble actor_modelview_matrix[16]; + GLboolean is_enabled_gl_depth_test; + GLint gl_depth_func; + GLboolean is_enabled_gl_blend; + GLint gl_blend_src_alpha; + + /* libvisual stuff */ + VisAudio *audio; + VisVideo *video; + VisActor *actor; + int actor_setup_result; + + /* audio/video state */ + gint channels; + gint rate; /* Input samplerate */ + gint bps; + VisAudioSampleRateType libvisual_rate; + + /* framerate numerator & denominator */ + gint fps_n; + gint fps_d; + gint width; + gint height; + GstClockTime duration; + guint outsize; + + /* samples per frame based on caps */ + guint spf; + + /* state stuff */ + GstAdapter *adapter; + guint count; + + /* QoS stuff *//* with LOCK */ + gdouble proportion; + GstClockTime earliest_time; +}; + +struct _GstVisualGLClass +{ + GstElementClass parent_class; + + VisPluginRef *plugin; +}; + +GType gst_visual_gl_get_type (void); + + +static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS (GST_GL_VIDEO_CAPS) + ); + +static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS ("audio/x-raw-int, " + "width = (int) 16, " + "depth = (int) 16, " + "endianness = (int) BYTE_ORDER, " + "signed = (boolean) TRUE, " "channels = (int) { 1, 2 }, " + "rate = (int) { 8000, 11250, 22500, 32000, 44100, 48000, 96000 }") + ); + + +static void gst_visual_gl_class_init (gpointer g_class, gpointer class_data); +static void gst_visual_gl_init (GstVisualGL * visual); +static void gst_visual_gl_dispose (GObject * object); + +static GstStateChangeReturn gst_visual_gl_change_state (GstElement * element, + GstStateChange transition); +static GstFlowReturn gst_visual_gl_chain (GstPad * pad, GstBuffer * buffer); +static gboolean gst_visual_gl_sink_event (GstPad * pad, GstEvent * event); +static gboolean gst_visual_gl_src_event (GstPad * pad, GstEvent * event); + +static gboolean gst_visual_gl_src_query (GstPad * pad, GstQuery * query); + +static gboolean gst_visual_gl_sink_setcaps (GstPad * pad, GstCaps * caps); +static gboolean gst_visual_gl_src_setcaps (GstPad * pad, GstCaps * caps); +static GstCaps *gst_visual_gl_getcaps (GstPad * pad); +static void libvisual_log_handler (const char *message, const char *funcname, + void *priv); + +static GstElementClass *parent_class = NULL; + +GType +gst_visual_gl_get_type (void) +{ + static GType type = 0; + + if (G_UNLIKELY (type == 0)) { + static const GTypeInfo info = { + sizeof (GstVisualGLClass), + NULL, + NULL, + gst_visual_gl_class_init, + NULL, + NULL, + sizeof (GstVisualGL), + 0, + (GInstanceInitFunc) gst_visual_gl_init, + }; + + type = g_type_register_static (GST_TYPE_ELEMENT, "GstVisualGL", &info, 0); + } + return type; +} + +static void +libvisual_log_handler (const char *message, const char *funcname, void *priv) +{ + GST_CAT_LEVEL_LOG (libvisual_debug, (GstDebugLevel) (priv), NULL, "%s - %s", + funcname, message); +} + +static void +gst_visual_gl_class_init (gpointer g_class, gpointer class_data) +{ + GstVisualGLClass *klass = GST_VISUAL_GL_CLASS (g_class); + GstElementClass *element_class = GST_ELEMENT_CLASS (g_class); + GObjectClass *object = G_OBJECT_CLASS (g_class); + + klass->plugin = class_data; + + element_class->change_state = gst_visual_gl_change_state; + + if (class_data == NULL) { + parent_class = g_type_class_peek_parent (g_class); + } else { + char *longname = g_strdup_printf ("libvisual %s plugin v.%s", + klass->plugin->info->name, klass->plugin->info->version); + + /* FIXME: improve to only register what plugin supports? */ + gst_element_class_add_pad_template (element_class, + gst_static_pad_template_get (&src_template)); + gst_element_class_add_pad_template (element_class, + gst_static_pad_template_get (&sink_template)); + + gst_element_class_set_details_simple (element_class, + longname, "Visualization", klass->plugin->info->about, + "Benjamin Otte "); + + g_free (longname); + } + + object->dispose = gst_visual_gl_dispose; +} + +static void +gst_visual_gl_init (GstVisualGL * visual) +{ + /* create the sink and src pads */ + visual->sinkpad = gst_pad_new_from_static_template (&sink_template, "sink"); + gst_pad_set_setcaps_function (visual->sinkpad, gst_visual_gl_sink_setcaps); + gst_pad_set_chain_function (visual->sinkpad, gst_visual_gl_chain); + gst_pad_set_event_function (visual->sinkpad, gst_visual_gl_sink_event); + gst_element_add_pad (GST_ELEMENT (visual), visual->sinkpad); + + visual->srcpad = gst_pad_new_from_static_template (&src_template, "src"); + gst_pad_set_setcaps_function (visual->srcpad, gst_visual_gl_src_setcaps); + gst_pad_set_getcaps_function (visual->srcpad, gst_visual_gl_getcaps); + gst_pad_set_event_function (visual->srcpad, gst_visual_gl_src_event); + gst_pad_set_query_function (visual->srcpad, gst_visual_gl_src_query); + gst_element_add_pad (GST_ELEMENT (visual), visual->srcpad); + + visual->adapter = gst_adapter_new (); + + visual->actor = NULL; + + visual->display = NULL; + visual->fbo = 0; + visual->depthbuffer = 0; + + visual->is_enabled_gl_depth_test = GL_FALSE; + visual->gl_depth_func = GL_LESS; + visual->is_enabled_gl_blend = GL_FALSE; + visual->gl_blend_src_alpha = GL_ONE; +} + +static void +gst_visual_gl_clear_actors (GstVisualGL * visual) +{ + if (visual->actor) { + visual_object_unref (VISUAL_OBJECT (visual->actor)); + visual->actor = NULL; + } + if (visual->video) { + visual_object_unref (VISUAL_OBJECT (visual->video)); + visual->video = NULL; + } + if (visual->audio) { + visual_object_unref (VISUAL_OBJECT (visual->audio)); + visual->audio = NULL; + } +} + +static void +gst_visual_gl_dispose (GObject * object) +{ + GstVisualGL *visual = GST_VISUAL_GL (object); + + if (visual->adapter) { + g_object_unref (visual->adapter); + visual->adapter = NULL; + } + + GST_CALL_PARENT (G_OBJECT_CLASS, dispose, (object)); +} + +static void +gst_visual_gl_reset (GstVisualGL * visual) +{ + gst_adapter_clear (visual->adapter); + gst_segment_init (&visual->segment, GST_FORMAT_UNDEFINED); + + GST_OBJECT_LOCK (visual); + visual->proportion = 1.0; + visual->earliest_time = -1; + GST_OBJECT_UNLOCK (visual); +} + +static GstCaps * +gst_visual_gl_getcaps (GstPad * pad) +{ + GstCaps *ret; + GstVisualGL *visual = GST_VISUAL_GL (gst_pad_get_parent (pad)); + int depths; + + if (!visual->actor) { + ret = gst_caps_copy (gst_pad_get_pad_template_caps (visual->srcpad)); + goto beach; + } + + ret = gst_caps_new_empty (); + depths = visual_actor_get_supported_depth (visual->actor); + if (depths < 0) { + /* FIXME: set an error */ + goto beach; + } + if ((depths & VISUAL_VIDEO_DEPTH_GL) == 0) { + /* We don't handle non-GL plugins */ + goto beach; + } + + GST_DEBUG_OBJECT (visual, "libvisual-gl plugin supports depths %u (0x%04x)", + depths, depths); + /* only do GL output */ + gst_caps_append (ret, gst_caps_from_string (GST_GL_VIDEO_CAPS)); + +beach: + + GST_DEBUG_OBJECT (visual, "returning caps %" GST_PTR_FORMAT, ret); + gst_object_unref (visual); + return ret; +} + +static gboolean +gst_visual_gl_src_setcaps (GstPad * pad, GstCaps * caps) +{ + GstVisualGL *visual = GST_VISUAL_GL (gst_pad_get_parent (pad)); + GstStructure *structure; + + structure = gst_caps_get_structure (caps, 0); + + GST_DEBUG_OBJECT (visual, "src pad got caps %" GST_PTR_FORMAT, caps); + + if (!gst_structure_get_int (structure, "width", &visual->width)) + goto error; + if (!gst_structure_get_int (structure, "height", &visual->height)) + goto error; + if (!gst_structure_get_fraction (structure, "framerate", &visual->fps_n, + &visual->fps_d)) + goto error; + + /* precalc some values */ + visual->spf = + gst_util_uint64_scale_int (visual->rate, visual->fps_d, visual->fps_n); + visual->duration = + gst_util_uint64_scale_int (GST_SECOND, visual->fps_d, visual->fps_n); + + gst_gl_display_gen_fbo (visual->display, visual->width, visual->height, + &visual->fbo, &visual->depthbuffer); + + gst_object_unref (visual); + return TRUE; + + /* ERRORS */ +error: + { + GST_DEBUG_OBJECT (visual, "error parsing caps"); + gst_object_unref (visual); + return FALSE; + } +} + +static gboolean +gst_visual_gl_sink_setcaps (GstPad * pad, GstCaps * caps) +{ + GstVisualGL *visual = GST_VISUAL_GL (gst_pad_get_parent (pad)); + GstStructure *structure; + + structure = gst_caps_get_structure (caps, 0); + + gst_structure_get_int (structure, "channels", &visual->channels); + gst_structure_get_int (structure, "rate", &visual->rate); + + switch (visual->rate) { + case 8000: + visual->libvisual_rate = VISUAL_AUDIO_SAMPLE_RATE_8000; + break; + case 11250: + visual->libvisual_rate = VISUAL_AUDIO_SAMPLE_RATE_11250; + break; + case 22500: + visual->libvisual_rate = VISUAL_AUDIO_SAMPLE_RATE_22500; + break; + case 32000: + visual->libvisual_rate = VISUAL_AUDIO_SAMPLE_RATE_32000; + break; + case 44100: + visual->libvisual_rate = VISUAL_AUDIO_SAMPLE_RATE_44100; + break; + case 48000: + visual->libvisual_rate = VISUAL_AUDIO_SAMPLE_RATE_48000; + break; + case 96000: + visual->libvisual_rate = VISUAL_AUDIO_SAMPLE_RATE_96000; + break; + default: + gst_object_unref (visual); + return FALSE; + } + + /* this is how many samples we need to fill one frame at the requested + * framerate. */ + if (visual->fps_n != 0) { + visual->spf = + gst_util_uint64_scale_int (visual->rate, visual->fps_d, visual->fps_n); + } + visual->bps = visual->channels * sizeof (gint16); + + gst_object_unref (visual); + return TRUE; +} + +static gboolean +gst_vis_gl_src_negotiate (GstVisualGL * visual) +{ + GstCaps *othercaps, *target; + GstStructure *structure; + GstCaps *caps; + + caps = gst_pad_get_caps (visual->srcpad); + + /* see what the peer can do */ + othercaps = gst_pad_peer_get_caps (visual->srcpad); + if (othercaps) { + target = gst_caps_intersect (othercaps, caps); + gst_caps_unref (othercaps); + gst_caps_unref (caps); + + if (gst_caps_is_empty (target)) + goto no_format; + + gst_caps_truncate (target); + } else { + /* need a copy, we'll be modifying it when fixating */ + target = gst_caps_copy (caps); + gst_caps_unref (caps); + } + + /* fixate in case something is not fixed. This does nothing if the value is + * already fixed. For video we always try to fixate to something like + * 320x240x25 by convention. */ + structure = gst_caps_get_structure (target, 0); + gst_structure_fixate_field_nearest_int (structure, "width", DEFAULT_WIDTH); + gst_structure_fixate_field_nearest_int (structure, "height", DEFAULT_HEIGHT); + gst_structure_fixate_field_nearest_fraction (structure, "framerate", + DEFAULT_FPS_N, DEFAULT_FPS_D); + + gst_pad_set_caps (visual->srcpad, target); + gst_caps_unref (target); + + return TRUE; + + /* ERRORS */ +no_format: + { + GST_ELEMENT_ERROR (visual, STREAM, FORMAT, (NULL), + ("could not negotiate output format")); + gst_caps_unref (target); + return FALSE; + } +} + +static gboolean +gst_visual_gl_sink_event (GstPad * pad, GstEvent * event) +{ + GstVisualGL *visual; + gboolean res; + + visual = GST_VISUAL_GL (gst_pad_get_parent (pad)); + + switch (GST_EVENT_TYPE (event)) { + case GST_EVENT_FLUSH_START: + res = gst_pad_push_event (visual->srcpad, event); + break; + case GST_EVENT_FLUSH_STOP: + /* reset QoS and adapter. */ + gst_visual_gl_reset (visual); + res = gst_pad_push_event (visual->srcpad, event); + break; + case GST_EVENT_NEWSEGMENT: + { + GstFormat format; + gdouble rate, arate; + gint64 start, stop, time; + gboolean update; + + /* the newsegment values are used to clip the input samples + * and to convert the incomming timestamps to running time so + * we can do QoS */ + gst_event_parse_new_segment_full (event, &update, &rate, &arate, &format, + &start, &stop, &time); + + /* now configure the values */ + gst_segment_set_newsegment_full (&visual->segment, update, + rate, arate, format, start, stop, time); + + /* and forward */ + res = gst_pad_push_event (visual->srcpad, event); + break; + } + default: + res = gst_pad_push_event (visual->srcpad, event); + break; + } + + gst_object_unref (visual); + return res; +} + +static gboolean +gst_visual_gl_src_event (GstPad * pad, GstEvent * event) +{ + GstVisualGL *visual; + gboolean res; + + visual = GST_VISUAL_GL (gst_pad_get_parent (pad)); + + switch (GST_EVENT_TYPE (event)) { + case GST_EVENT_QOS: + { + gdouble proportion; + GstClockTimeDiff diff; + GstClockTime timestamp; + + gst_event_parse_qos (event, &proportion, &diff, ×tamp); + + /* save stuff for the _chain function */ + GST_OBJECT_LOCK (visual); + visual->proportion = proportion; + if (diff >= 0) + /* we're late, this is a good estimate for next displayable + * frame (see part-qos.txt) */ + visual->earliest_time = timestamp + 2 * diff + visual->duration; + else + visual->earliest_time = timestamp + diff; + + GST_OBJECT_UNLOCK (visual); + + res = gst_pad_push_event (visual->sinkpad, event); + break; + } + default: + res = gst_pad_push_event (visual->sinkpad, event); + break; + } + + gst_object_unref (visual); + return res; +} + +static gboolean +gst_visual_gl_src_query (GstPad * pad, GstQuery * query) +{ + gboolean res; + GstVisualGL *visual; + + visual = GST_VISUAL_GL (gst_pad_get_parent (pad)); + + switch (GST_QUERY_TYPE (query)) { + case GST_QUERY_LATENCY: + { + /* We need to send the query upstream and add the returned latency to our + * own */ + GstClockTime min_latency, max_latency; + gboolean us_live; + GstClockTime our_latency; + guint max_samples; + + if ((res = gst_pad_peer_query (visual->sinkpad, query))) { + gst_query_parse_latency (query, &us_live, &min_latency, &max_latency); + + GST_DEBUG_OBJECT (visual, "Peer latency: min %" + GST_TIME_FORMAT " max %" GST_TIME_FORMAT, + GST_TIME_ARGS (min_latency), GST_TIME_ARGS (max_latency)); + + /* the max samples we must buffer buffer */ + max_samples = MAX (VISUAL_SAMPLES, visual->spf); + our_latency = + gst_util_uint64_scale_int (max_samples, GST_SECOND, visual->rate); + + GST_DEBUG_OBJECT (visual, "Our latency: %" GST_TIME_FORMAT, + GST_TIME_ARGS (our_latency)); + + /* we add some latency but only if we need to buffer more than what + * upstream gives us */ + min_latency += our_latency; + if (max_latency != -1) + max_latency += our_latency; + + GST_DEBUG_OBJECT (visual, "Calculated total latency : min %" + GST_TIME_FORMAT " max %" GST_TIME_FORMAT, + GST_TIME_ARGS (min_latency), GST_TIME_ARGS (max_latency)); + + gst_query_set_latency (query, TRUE, min_latency, max_latency); + } + break; + } + case GST_QUERY_CUSTOM: + { + GstStructure *structure = gst_query_get_structure (query); + res = + g_strcmp0 (gst_element_get_name (visual), + gst_structure_get_name (structure)) == 0; + if (!res) + res = gst_pad_query_default (pad, query); + break; + } + default: + res = gst_pad_peer_query (visual->sinkpad, query); + break; + } + + gst_object_unref (visual); + + return res; +} + +/* allocate and output buffer, if no format was negotiated, this + * function will negotiate one. After calling this function, a + * reverse negotiation could have happened. */ +static GstFlowReturn +get_buffer (GstVisualGL * visual, GstGLBuffer ** outbuf) +{ + /* we don't know an output format yet, pick one */ + if (GST_PAD_CAPS (visual->srcpad) == NULL) { + if (!gst_vis_gl_src_negotiate (visual)) + return GST_FLOW_NOT_NEGOTIATED; + } + + GST_DEBUG_OBJECT (visual, "allocating output buffer with caps %" + GST_PTR_FORMAT, GST_PAD_CAPS (visual->srcpad)); + + *outbuf = gst_gl_buffer_new (visual->display, visual->width, visual->height); + if (*outbuf == NULL) + return GST_FLOW_ERROR; + + gst_buffer_set_caps (GST_BUFFER (*outbuf), GST_PAD_CAPS (visual->srcpad)); + return GST_FLOW_OK; +} + +static void +actor_setup (GstGLDisplay * display, GstVisualGL * visual) +{ + /* save and clear top of the stack */ + glPushAttrib (GL_ALL_ATTRIB_BITS); + + glMatrixMode (GL_PROJECTION); + glPushMatrix (); + glLoadIdentity (); + + glMatrixMode (GL_MODELVIEW); + glPushMatrix (); + glLoadIdentity (); + + visual->actor_setup_result = visual_actor_realize (visual->actor); + if (visual->actor_setup_result == 0) { + /* store the actor's matrices for rendering the first frame */ + glGetDoublev (GL_MODELVIEW_MATRIX, visual->actor_modelview_matrix); + glGetDoublev (GL_PROJECTION_MATRIX, visual->actor_projection_matrix); + + visual->is_enabled_gl_depth_test = glIsEnabled (GL_DEPTH_TEST); + glGetIntegerv (GL_DEPTH_FUNC, &visual->gl_depth_func); + + visual->is_enabled_gl_blend = glIsEnabled (GL_BLEND); + glGetIntegerv (GL_BLEND_SRC_ALPHA, &visual->gl_blend_src_alpha); + + /* retore matrix */ + glMatrixMode (GL_PROJECTION); + glPopMatrix (); + + glMatrixMode (GL_MODELVIEW); + glPopMatrix (); + + glPopAttrib (); + } +} + +static void +actor_negotiate (GstGLDisplay * display, GstVisualGL * visual) +{ + gint err = VISUAL_OK; + + err = visual_video_set_depth (visual->video, VISUAL_VIDEO_DEPTH_GL); + if (err != VISUAL_OK) + g_warning ("failed to visual_video_set_depth\n"); + + err = + visual_video_set_dimension (visual->video, visual->width, visual->height); + if (err != VISUAL_OK) + g_warning ("failed to visual_video_set_depth\n"); + + err = visual_actor_video_negotiate (visual->actor, 0, FALSE, FALSE); + if (err != VISUAL_OK) + g_warning ("failed to visual_video_set_depth\n"); +} + +static void +render_frame (gint width, gint height, guint texture, GstVisualGL * visual) +{ + const guint16 *data; + VisBuffer *lbuf, *rbuf; + guint16 ldata[VISUAL_SAMPLES], rdata[VISUAL_SAMPLES]; + guint i; + + /* Read VISUAL_SAMPLES samples per channel */ + data = + (const guint16 *) gst_adapter_peek (visual->adapter, + VISUAL_SAMPLES * visual->bps); + + lbuf = visual_buffer_new_with_buffer (ldata, sizeof (ldata), NULL); + rbuf = visual_buffer_new_with_buffer (rdata, sizeof (rdata), NULL); + + if (visual->channels == 2) { + for (i = 0; i < VISUAL_SAMPLES; i++) { + ldata[i] = *data++; + rdata[i] = *data++; + } + } else { + for (i = 0; i < VISUAL_SAMPLES; i++) { + ldata[i] = *data; + rdata[i] = *data++; + } + } + + visual_audio_samplepool_input_channel (visual->audio->samplepool, + lbuf, visual->libvisual_rate, VISUAL_AUDIO_SAMPLE_FORMAT_S16, + VISUAL_AUDIO_CHANNEL_LEFT); + visual_audio_samplepool_input_channel (visual->audio->samplepool, + rbuf, visual->libvisual_rate, VISUAL_AUDIO_SAMPLE_FORMAT_S16, + VISUAL_AUDIO_CHANNEL_RIGHT); + + visual_object_unref (VISUAL_OBJECT (lbuf)); + visual_object_unref (VISUAL_OBJECT (rbuf)); + + visual_audio_analyze (visual->audio); + + /* apply the matrices that the actor set up */ + glPushAttrib (GL_ALL_ATTRIB_BITS); + + glMatrixMode (GL_PROJECTION); + glPushMatrix (); + glLoadMatrixd (visual->actor_projection_matrix); + + glMatrixMode (GL_MODELVIEW); + glPushMatrix (); + glLoadMatrixd (visual->actor_modelview_matrix); + + actor_negotiate (visual->display, visual); + + glMatrixMode (GL_MODELVIEW); + glScaled (1.0, -1.0, 1.0); + + visual_actor_run (visual->actor, visual->audio); + + glMatrixMode (GL_PROJECTION); + glPopMatrix (); + + glMatrixMode (GL_MODELVIEW); + glPopMatrix (); + + glPopAttrib (); + + GST_DEBUG_OBJECT (visual, "rendered one frame"); +} + +static GstFlowReturn +gst_visual_gl_chain (GstPad * pad, GstBuffer * buffer) +{ + GstGLBuffer *outbuf = NULL; + GstVisualGL *visual = GST_VISUAL_GL (gst_pad_get_parent (pad)); + GstFlowReturn ret = GST_FLOW_OK; + guint avail; + + GST_DEBUG_OBJECT (visual, "chain function called"); + + /* If we don't have an output format yet, preallocate a buffer to try and + * set one */ + if (GST_PAD_CAPS (visual->srcpad) == NULL) { + ret = get_buffer (visual, &outbuf); + if (ret != GST_FLOW_OK) { + gst_buffer_unref (buffer); + goto beach; + } + } + + /* resync on DISCONT */ + if (GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_DISCONT)) { + gst_adapter_clear (visual->adapter); + } + + GST_DEBUG_OBJECT (visual, + "Input buffer has %d samples, time=%" G_GUINT64_FORMAT, + GST_BUFFER_SIZE (buffer) / visual->bps, GST_BUFFER_TIMESTAMP (buffer)); + + gst_adapter_push (visual->adapter, buffer); + + while (TRUE) { + gboolean need_skip; + guint64 dist, timestamp; + + GST_DEBUG_OBJECT (visual, "processing buffer"); + + avail = gst_adapter_available (visual->adapter); + GST_DEBUG_OBJECT (visual, "avail now %u", avail); + + /* we need at least VISUAL_SAMPLES samples */ + if (avail < VISUAL_SAMPLES * visual->bps) + break; + + /* we need at least enough samples to make one frame */ + if (avail < visual->spf * visual->bps) + break; + + /* get timestamp of the current adapter byte */ + timestamp = gst_adapter_prev_timestamp (visual->adapter, &dist); + if (GST_CLOCK_TIME_IS_VALID (timestamp)) { + /* convert bytes to time */ + dist /= visual->bps; + timestamp += gst_util_uint64_scale_int (dist, GST_SECOND, visual->rate); + } + + if (timestamp != -1) { + gint64 qostime; + + /* QoS is done on running time */ + qostime = gst_segment_to_running_time (&visual->segment, GST_FORMAT_TIME, + timestamp); + qostime += visual->duration; + + GST_OBJECT_LOCK (visual); + /* check for QoS, don't compute buffers that are known to be late */ + need_skip = visual->earliest_time != -1 && + qostime <= visual->earliest_time; + GST_OBJECT_UNLOCK (visual); + + if (need_skip) { + GST_WARNING_OBJECT (visual, + "QoS: skip ts: %" GST_TIME_FORMAT ", earliest: %" GST_TIME_FORMAT, + GST_TIME_ARGS (qostime), GST_TIME_ARGS (visual->earliest_time)); + goto skip; + } + } + + /* alloc a buffer if we don't have one yet, this happens + * when we pushed a buffer in this while loop before */ + if (outbuf == NULL) { + ret = get_buffer (visual, &outbuf); + if (ret != GST_FLOW_OK) { + goto beach; + } + } + + /* this is a pretty weird API */ + gst_gl_display_use_fbo (visual->display, + visual->width, visual->height, visual->fbo, visual->depthbuffer, + outbuf->texture, (GLCB) render_frame, 0, 0, 0, + 0, visual->width, 0, visual->height, GST_GL_DISPLAY_PROJECTION_ORTHO2D, + (gpointer *) visual); + + GST_BUFFER_TIMESTAMP (outbuf) = timestamp; + GST_BUFFER_DURATION (outbuf) = visual->duration; + + ret = gst_pad_push (visual->srcpad, GST_BUFFER (outbuf)); + outbuf = NULL; + + skip: + GST_DEBUG_OBJECT (visual, "finished frame, flushing %u samples from input", + visual->spf); + + /* Flush out the number of samples per frame */ + gst_adapter_flush (visual->adapter, visual->spf * visual->bps); + + /* quit the loop if something was wrong */ + if (ret != GST_FLOW_OK) + break; + } + +beach: + + if (outbuf != NULL) + gst_gl_buffer_unref (outbuf); + + gst_object_unref (visual); + + return ret; +} + +static GstStateChangeReturn +gst_visual_gl_change_state (GstElement * element, GstStateChange transition) +{ + GstVisualGL *visual = GST_VISUAL_GL (element); + GstStateChangeReturn ret; + + switch (transition) { + case GST_STATE_CHANGE_NULL_TO_READY: + break; + case GST_STATE_CHANGE_READY_TO_PAUSED: + { + GstElement *parent = GST_ELEMENT (gst_element_get_parent (visual)); + GstStructure *structure = NULL; + GstQuery *query = NULL; + gboolean isPerformed = FALSE; + + if (!parent) { + GST_ELEMENT_ERROR (visual, CORE, STATE_CHANGE, (NULL), + ("A parent bin is required")); + return FALSE; + } + + structure = gst_structure_new (gst_element_get_name (visual), NULL); + query = gst_query_new_application (GST_QUERY_CUSTOM, structure); + + isPerformed = gst_element_query (parent, query); + + if (isPerformed) { + const GValue *id_value = + gst_structure_get_value (structure, "gstgldisplay"); + if (G_VALUE_HOLDS_POINTER (id_value)) + /* at least one gl element is after in our gl chain */ + visual->display = + g_object_ref (GST_GL_DISPLAY (g_value_get_pointer (id_value))); + else { + /* this gl filter is a sink in terms of the gl chain */ + visual->display = gst_gl_display_new (); + gst_gl_display_create_context (visual->display, 0); + //TODO visual->external_gl_context); + } + + gst_visual_gl_reset (visual); + + visual->actor = + visual_actor_new (GST_VISUAL_GL_GET_CLASS (visual)->plugin-> + info->plugname); + visual->video = visual_video_new (); + visual->audio = visual_audio_new (); + + if (!visual->actor || !visual->video) + goto actor_setup_failed; + + gst_gl_display_thread_add (visual->display, + (GstGLDisplayThreadFunc) actor_setup, visual); + + if (visual->actor_setup_result != 0) + goto actor_setup_failed; + else + visual_actor_set_video (visual->actor, visual->video); + } + + gst_query_unref (query); + gst_object_unref (GST_OBJECT (parent)); + + if (!isPerformed) + return GST_STATE_CHANGE_FAILURE; + } + break; + case GST_STATE_CHANGE_PAUSED_TO_PLAYING: + break; + default: + break; + } + + ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition); + + switch (transition) { + case GST_STATE_CHANGE_PLAYING_TO_PAUSED: + break; + case GST_STATE_CHANGE_PAUSED_TO_READY: + { + if (visual->fbo) { + gst_gl_display_del_fbo (visual->display, visual->fbo, + visual->depthbuffer); + visual->fbo = 0; + visual->depthbuffer = 0; + } + if (visual->display) { + g_object_unref (visual->display); + visual->display = NULL; + } + + gst_visual_gl_clear_actors (visual); + } + break; + case GST_STATE_CHANGE_READY_TO_NULL: + break; + default: + break; + } + + return ret; + + /* ERRORS */ +actor_setup_failed: + { + GST_ELEMENT_ERROR (visual, LIBRARY, INIT, (NULL), + ("could not set up actor")); + gst_visual_gl_clear_actors (visual); + return GST_STATE_CHANGE_FAILURE; + } +} + +static void +make_valid_name (char *name) +{ + /* + * Replace invalid chars with _ in the type name + */ + static const gchar extra_chars[] = "-_+"; + gchar *p = name; + + for (; *p; p++) { + int valid = ((p[0] >= 'A' && p[0] <= 'Z') || + (p[0] >= 'a' && p[0] <= 'z') || + (p[0] >= '0' && p[0] <= '9') || strchr (extra_chars, p[0])); + if (!valid) + *p = '_'; + } +} + +static gboolean +gst_visual_gl_actor_plugin_is_gl (VisObject * plugin, const gchar * name) +{ + gboolean is_gl; + gint depth; + + depth = VISUAL_ACTOR_PLUGIN (plugin)->vidoptions.depth; + is_gl = (depth & VISUAL_VIDEO_DEPTH_GL) != 0; + + if (!is_gl) { + GST_DEBUG ("plugin %s is not a GL plugin (%d), ignoring", name, depth); + } else { + GST_DEBUG ("plugin %s is a GL plugin (%d), registering", name, depth); + } + + return is_gl; +} + +static gboolean +plugin_init (GstPlugin * plugin) +{ + guint i, count; + VisList *list; + + GST_DEBUG_CATEGORY_INIT (libvisual_debug, "libvisual", 0, + "libvisual audio visualisations"); + +#ifdef LIBVISUAL_PLUGINSBASEDIR + gst_plugin_add_dependency_simple (plugin, "HOME/.libvisual/actor", + LIBVISUAL_PLUGINSBASEDIR "/actor", NULL, GST_PLUGIN_DEPENDENCY_FLAG_NONE); +#endif + + visual_log_set_verboseness (VISUAL_LOG_VERBOSENESS_LOW); + visual_log_set_info_handler (libvisual_log_handler, (void *) GST_LEVEL_INFO); + visual_log_set_warning_handler (libvisual_log_handler, + (void *) GST_LEVEL_WARNING); + visual_log_set_critical_handler (libvisual_log_handler, + (void *) GST_LEVEL_ERROR); + visual_log_set_error_handler (libvisual_log_handler, + (void *) GST_LEVEL_ERROR); + + if (!visual_is_initialized ()) + if (visual_init (NULL, NULL) != 0) + return FALSE; + + list = visual_actor_get_list (); + + count = visual_collection_size (VISUAL_COLLECTION (list)); + + for (i = 0; i < count; i++) { + VisPluginRef *ref = visual_list_get (list, i); + VisPluginData *visplugin = NULL; + gboolean skip = FALSE; + GType type; + gchar *name; + GTypeInfo info = { + sizeof (GstVisualGLClass), + NULL, + NULL, + gst_visual_gl_class_init, + NULL, + ref, + sizeof (GstVisualGL), + 0, + NULL + }; + + visplugin = visual_plugin_load (ref); + + if (ref->info->plugname == NULL) + continue; + + /* Blacklist some plugins */ + if (strcmp (ref->info->plugname, "gstreamer") == 0 || + strcmp (ref->info->plugname, "gdkpixbuf") == 0) { + skip = TRUE; + } else { + /* only register plugins that support GL */ + skip = !(gst_visual_gl_actor_plugin_is_gl (visplugin->info->plugin, + visplugin->info->plugname)); + } + + visual_plugin_unload (visplugin); + + if (!skip) { + name = g_strdup_printf ("GstVisualGL%s", ref->info->plugname); + make_valid_name (name); + type = g_type_register_static (GST_TYPE_VISUAL_GL, name, &info, 0); + g_free (name); + + name = g_strdup_printf ("libvisual_gl_%s", ref->info->plugname); + make_valid_name (name); + if (!gst_element_register (plugin, name, GST_RANK_NONE, type)) { + g_free (name); + return FALSE; + } + g_free (name); + } + } + + return TRUE; +} + +GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, + GST_VERSION_MINOR, + "libvisual-gl", + "libvisual-gl visualization plugins", + plugin_init, VERSION, "LGPL", GST_PACKAGE_NAME, GST_PACKAGE_ORIGIN) -- 2.7.4