From 6f7d4ac52556a8f7b9c6c663ea654246e021ad2d Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Fri, 3 Jul 2020 18:16:13 -0400 Subject: [PATCH] ges: Fix smart rendering Smart rendering has been broken since, mostly forever, but some code was there pretending it was supported... let's try to stop pretending. We now keep track of the smart rendering state in the timeline, track and sources to be able to: * tell decodebin to stop plugging more (decoding elements) as soon as downstream supports the format. * avoid plugging converters after the source element when smart rendering. Part-of: --- ges/ges-enums.h | 4 +++- ges/ges-internal.h | 12 ++++++++++++ ges/ges-pipeline.c | 5 +++++ ges/ges-source.c | 31 ++++++++++++++++++++++++++++--- ges/ges-timeline-tree.c | 18 ++++++++++++++++++ ges/ges-timeline-tree.h | 1 + ges/ges-timeline.c | 32 ++++++++++++++++++++++++++++++++ ges/ges-track.c | 19 ++++++++++++++++++- ges/ges-uri-source.c | 32 ++++++++++++++++++++++++++++++++ 9 files changed, 149 insertions(+), 5 deletions(-) diff --git a/ges/ges-enums.h b/ges/ges-enums.h index 91d1568..edcd84e 100644 --- a/ges/ges-enums.h +++ b/ges/ges-enums.h @@ -344,7 +344,9 @@ GType ges_video_test_pattern_get_type (void); * #encodebin:avoid-reencoding property set to %FALSE) * @GES_PIPELINE_MODE_SMART_RENDER: Render the #GESPipeline:timeline, * avoiding decoding/reencoding (the underlying #encodebin has its - * #encodebin:avoid-reencoding property set to %TRUE) + * #encodebin:avoid-reencoding property set to %TRUE). + * > NOTE: Smart rendering can not work in tracks where #GESTrack:mixing + * > is enabled. * * The various modes a #GESPipeline can be configured to. */ diff --git a/ges/ges-internal.h b/ges/ges-internal.h index 7884c8e..186da68 100644 --- a/ges/ges-internal.h +++ b/ges/ges-internal.h @@ -177,6 +177,12 @@ G_GNUC_INTERNAL void ges_timeline_remove_clip (GESTimeline * timeline, GESClip * clip); G_GNUC_INTERNAL void +ges_timeline_set_smart_rendering (GESTimeline * timeline, gboolean rendering_smartly); + +G_GNUC_INTERNAL gboolean +ges_timeline_get_smart_rendering (GESTimeline *timeline); + +G_GNUC_INTERNAL void ges_auto_transition_set_previous_source (GESAutoTransition * self, GESTrackElement * source); @@ -475,6 +481,12 @@ G_GNUC_INTERNAL GstElement* ges_source_create_topbin (GESSource *source, const gchar* bin_name, GstElement* sub_element, GPtrArray* elements); +G_GNUC_INTERNAL void ges_source_set_rendering_smartly (GESSource *source, + gboolean rendering_smartly); +G_GNUC_INTERNAL gboolean +ges_source_get_rendering_smartly (GESSource *source); + +G_GNUC_INTERNAL void ges_track_set_smart_rendering (GESTrack* track, gboolean rendering_smartly); G_GNUC_INTERNAL GstElement * ges_track_get_composition (GESTrack *track); diff --git a/ges/ges-pipeline.c b/ges/ges-pipeline.c index 3e76dd1..e714aa5 100644 --- a/ges/ges-pipeline.c +++ b/ges/ges-pipeline.c @@ -1268,6 +1268,11 @@ ges_pipeline_set_mode (GESPipeline * pipeline, GESPipelineFlags mode) pipeline->priv->urisink, "sink", GST_PAD_LINK_CHECK_NOTHING); } + if (pipeline->priv->timeline) { + ges_timeline_set_smart_rendering (pipeline->priv->timeline, + (mode & GES_PIPELINE_MODE_SMART_RENDER) != 0); + } + /* FIXUPS */ /* FIXME * If we are rendering, set playsink to sync=False, diff --git a/ges/ges-source.c b/ges/ges-source.c index f9355c5..0ca5e11 100644 --- a/ges/ges-source.c +++ b/ges/ges-source.c @@ -40,6 +40,7 @@ struct _GESSourcePrivate GstElement *last_converter; GstPad *ghostpad; + gboolean is_rendering_smartly; }; G_DEFINE_TYPE_WITH_PRIVATE (GESSource, ges_source, GES_TYPE_TRACK_ELEMENT); @@ -76,11 +77,11 @@ link_elements (GstElement * bin, GPtrArray * elements) static void _set_ghost_pad_target (GESSource * self, GstPad * srcpad, GstElement * element) { - GESSourcePrivate *priv = self->priv; - gboolean use_converter = FALSE; GstPadLinkReturn link_return; + GESSourcePrivate *priv = self->priv; + gboolean use_converter = ! !priv->first_converter; - if (priv->first_converter) { + if (use_converter && priv->is_rendering_smartly) { GstPad *pad = gst_element_get_static_pad (priv->first_converter, "sink"); use_converter = gst_pad_can_link (srcpad, pad); gst_object_unref (pad); @@ -168,6 +169,30 @@ ges_source_create_topbin (GESSource * source, const gchar * bin_name, } +void +ges_source_set_rendering_smartly (GESSource * source, + gboolean is_rendering_smartly) +{ + + if (is_rendering_smartly) { + GESTrack *track = ges_track_element_get_track (GES_TRACK_ELEMENT (source)); + + if (track && ges_track_get_mixing (track)) { + GST_DEBUG_OBJECT (source, "Not rendering smartly as track is mixing!"); + + source->priv->is_rendering_smartly = FALSE; + return; + } + } + source->priv->is_rendering_smartly = is_rendering_smartly; +} + +gboolean +ges_source_get_rendering_smartly (GESSource * source) +{ + return source->priv->is_rendering_smartly; +} + static void ges_source_dispose (GObject * object) { diff --git a/ges/ges-timeline-tree.c b/ges/ges-timeline-tree.c index 09f405a..8f92d23 100644 --- a/ges/ges-timeline-tree.c +++ b/ges/ges-timeline-tree.c @@ -2481,3 +2481,21 @@ timeline_tree_reset_layer_active (GNode * root, GESLayer * layer) g_node_traverse (root, G_PRE_ORDER, G_TRAVERSE_LEAFS, -1, (GNodeTraverseFunc) reset_layer_activness, layer); } + +static gboolean +set_is_smart_rendering (GNode * node, gboolean * is_rendering_smartly) +{ + if (!GES_IS_SOURCE (node->data)) + return FALSE; + + ges_source_set_rendering_smartly (GES_SOURCE (node->data), + *is_rendering_smartly); + return FALSE; +} + +void +timeline_tree_set_smart_rendering (GNode * root, gboolean rendering_smartly) +{ + g_node_traverse (root, G_PRE_ORDER, G_TRAVERSE_LEAFS, -1, + (GNodeTraverseFunc) set_is_smart_rendering, &rendering_smartly); +} diff --git a/ges/ges-timeline-tree.h b/ges/ges-timeline-tree.h index 5fafba3..2ab4703 100644 --- a/ges/ges-timeline-tree.h +++ b/ges/ges-timeline-tree.h @@ -82,5 +82,6 @@ void timeline_update_duration (GESTimeline * timeline); void timeline_tree_reset_layer_active (GNode *root, GESLayer *layer); +void timeline_tree_set_smart_rendering (GNode * root, gboolean rendering_smartly); void timeline_tree_init_debug (void); diff --git a/ges/ges-timeline.c b/ges/ges-timeline.c index 8883921..23227cb 100644 --- a/ges/ges-timeline.c +++ b/ges/ges-timeline.c @@ -230,6 +230,8 @@ struct _GESTimelinePrivate gboolean disposed; GstStreamCollection *stream_collection; + + gboolean rendering_smartly; }; /* private structure to contain our track-related information */ @@ -2051,6 +2053,10 @@ timeline_add_element (GESTimeline * timeline, GESTimelineElement * element) ges_timeline_element_get_name (element), gst_object_ref (element)); timeline_tree_track_element (timeline->priv->tree, element); + if (GES_IS_SOURCE (element)) { + ges_source_set_rendering_smartly (GES_SOURCE (element), + timeline->priv->rendering_smartly); + } return TRUE; } @@ -2085,6 +2091,32 @@ timeline_get_tree (GESTimeline * timeline) return timeline->priv->tree; } +void +ges_timeline_set_smart_rendering (GESTimeline * timeline, + gboolean rendering_smartly) +{ + if (rendering_smartly) { + GList *tmp; + + for (tmp = timeline->tracks; tmp; tmp = tmp->next) { + if (ges_track_get_mixing (tmp->data)) { + GST_INFO_OBJECT (timeline, "Smart rendering will not" + " work as track %" GST_PTR_FORMAT " is doing mixing", tmp->data); + } else { + ges_track_set_smart_rendering (tmp->data, rendering_smartly); + } + } + } + timeline_tree_set_smart_rendering (timeline->priv->tree, rendering_smartly); + timeline->priv->rendering_smartly = rendering_smartly; +} + +gboolean +ges_timeline_get_smart_rendering (GESTimeline * timeline) +{ + return timeline->priv->rendering_smartly; +} + /**** API *****/ /** * ges_timeline_new: diff --git a/ges/ges-track.c b/ges/ges-track.c index 8ac2868..d3bd2ef 100644 --- a/ges/ges-track.c +++ b/ges/ges-track.c @@ -375,6 +375,15 @@ ges_track_get_composition (GESTrack * track) return track->priv->composition; } +void +ges_track_set_smart_rendering (GESTrack * track, gboolean rendering_smartly) +{ + GESTrackPrivate *priv = track->priv; + + g_object_set (priv->capsfilter, "caps", + rendering_smartly ? NULL : priv->restriction_caps, NULL); +} + /* FIXME: Find out how to avoid doing this "hack" using the GDestroyNotify * function pointer in the trackelements_by_start GSequence * @@ -985,6 +994,9 @@ ges_track_set_caps (GESTrack * track, const GstCaps * caps) * @caps: The new restriction-caps for @track * * Sets the #GESTrack:restriction-caps for the track. + * + * > **NOTE**: Restriction caps are **not** taken into account when + * > using #GESPipeline:mode=#GES_PIPELINE_MODE_SMART_RENDER. */ void ges_track_set_restriction_caps (GESTrack * track, const GstCaps * caps) @@ -1003,7 +1015,9 @@ ges_track_set_restriction_caps (GESTrack * track, const GstCaps * caps) gst_caps_unref (priv->restriction_caps); priv->restriction_caps = gst_caps_copy (caps); - g_object_set (priv->capsfilter, "caps", caps, NULL); + if (!track->priv->timeline || + !ges_timeline_get_smart_rendering (track->priv->timeline)) + g_object_set (priv->capsfilter, "caps", caps, NULL); g_object_notify (G_OBJECT (track), "restriction-caps"); } @@ -1105,6 +1119,9 @@ ges_track_set_mixing (GESTrack * track, gboolean mixing) notify: track->priv->mixing = mixing; + if (track->priv->timeline) + ges_timeline_set_smart_rendering (track->priv->timeline, + ges_timeline_get_smart_rendering (track->priv->timeline)); g_object_notify_by_pspec (G_OBJECT (track), properties[ARG_MIXING]); GST_DEBUG_OBJECT (track, "The track has been set to mixing = %d", mixing); diff --git a/ges/ges-uri-source.c b/ges/ges-uri-source.c index 138b8b9..8c78926 100644 --- a/ges/ges-uri-source.c +++ b/ges/ges-uri-source.c @@ -29,6 +29,36 @@ GST_DEBUG_CATEGORY_STATIC (uri_source_debug); #undef GST_CAT_DEFAULT #define GST_CAT_DEFAULT uri_source_debug +typedef enum +{ + GST_AUTOPLUG_SELECT_TRY, + GST_AUTOPLUG_SELECT_EXPOSE, + GST_AUTOPLUG_SELECT_SKIP, +} GstAutoplugSelectResult; + +static gint +autoplug_select_cb (GstElement * bin, GstPad * pad, GstCaps * caps, + GstElementFactory * factory, GESUriSource * self) +{ + GstElement *nlesrc; + GstCaps *downstream_caps; + GstAutoplugSelectResult res = GST_AUTOPLUG_SELECT_TRY; + + if (!ges_source_get_rendering_smartly (GES_SOURCE (self->element))) { + GST_LOG_OBJECT (self->element, "Not being smart here"); + return res; + } + + nlesrc = ges_track_element_get_nleobject (self->element); + downstream_caps = gst_pad_peer_query_caps (nlesrc->srcpads->data, NULL); + if (downstream_caps && gst_caps_can_intersect (downstream_caps, caps)) { + GST_DEBUG_OBJECT (self, "Exposing %s", GST_OBJECT_NAME (factory)); + res = GST_AUTOPLUG_SELECT_EXPOSE; + } + gst_clear_caps (&downstream_caps); + + return res; +} GstElement * ges_uri_source_create_source (GESUriSource * self) @@ -48,6 +78,8 @@ ges_uri_source_create_source (GESUriSource * self) g_object_set (decodebin, "caps", caps, "expose-all-streams", FALSE, "uri", self->uri, NULL); + g_signal_connect (decodebin, "autoplug-select", + G_CALLBACK (autoplug_select_cb), self); return decodebin; -- 2.7.4