ges: Fix smart rendering

Smart rendering has been broken since, mostly forever, but some code
was there pretending it was supported... let's try to stop pretending.

We now keep track of the smart rendering state in the timeline, track
and sources to be able to:

 * tell decodebin to stop plugging more (decoding elements) as soon as
   downstream supports the format.

 * avoid plugging converters after the source element when smart
   rendering.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-editing-services/-/merge_requests/198>

diff --git a/ges/ges-enums.h b/ges/ges-enums.h
index 91d1568..edcd84e 100644 (file)
--- a/ges/ges-enums.h
+++ b/ges/ges-enums.h
@@ -344,7 +344,9 @@ GType ges_video_test_pattern_get_type (void);
  * #encodebin:avoid-reencoding property set to %FALSE)
  * @GES_PIPELINE_MODE_SMART_RENDER: Render the #GESPipeline:timeline,
  * avoiding decoding/reencoding (the underlying #encodebin has its
- * #encodebin:avoid-reencoding property set to %TRUE)
+ * #encodebin:avoid-reencoding property set to %TRUE).
+ * > NOTE: Smart rendering can not work in tracks where #GESTrack:mixing
+ * > is enabled.
  *
  * The various modes a #GESPipeline can be configured to.
  */

diff --git a/ges/ges-internal.h b/ges/ges-internal.h
index 7884c8e..186da68 100644 (file)
--- a/ges/ges-internal.h
+++ b/ges/ges-internal.h
@@ -177,6 +177,12 @@ G_GNUC_INTERNAL void
 ges_timeline_remove_clip (GESTimeline * timeline, GESClip * clip);
 
 G_GNUC_INTERNAL void
+ges_timeline_set_smart_rendering (GESTimeline * timeline, gboolean rendering_smartly);
+
+G_GNUC_INTERNAL gboolean
+ges_timeline_get_smart_rendering (GESTimeline *timeline);
+
+G_GNUC_INTERNAL void
 ges_auto_transition_set_previous_source (GESAutoTransition * self, GESTrackElement * source);
 
 
@@ -475,6 +481,12 @@ G_GNUC_INTERNAL GstElement* ges_source_create_topbin  (GESSource *source,
                                                        const gchar* bin_name,
                                                        GstElement* sub_element,
                                                        GPtrArray* elements);
+G_GNUC_INTERNAL void ges_source_set_rendering_smartly (GESSource *source,
+                                                       gboolean rendering_smartly);
+G_GNUC_INTERNAL gboolean
+ges_source_get_rendering_smartly                      (GESSource *source);
+
+G_GNUC_INTERNAL void ges_track_set_smart_rendering     (GESTrack* track, gboolean rendering_smartly);
 G_GNUC_INTERNAL GstElement * ges_track_get_composition (GESTrack *track);
 
 

diff --git a/ges/ges-pipeline.c b/ges/ges-pipeline.c
index 3e76dd1..e714aa5 100644 (file)
--- a/ges/ges-pipeline.c
+++ b/ges/ges-pipeline.c
@@ -1268,6 +1268,11 @@ ges_pipeline_set_mode (GESPipeline * pipeline, GESPipelineFlags mode)
         pipeline->priv->urisink, "sink", GST_PAD_LINK_CHECK_NOTHING);
   }
 
+  if (pipeline->priv->timeline) {
+    ges_timeline_set_smart_rendering (pipeline->priv->timeline,
+        (mode & GES_PIPELINE_MODE_SMART_RENDER) != 0);
+  }
+
   /* FIXUPS */
   /* FIXME
    * If we are rendering, set playsink to sync=False,

diff --git a/ges/ges-source.c b/ges/ges-source.c
index f9355c5..0ca5e11 100644 (file)
--- a/ges/ges-source.c
+++ b/ges/ges-source.c
@@ -40,6 +40,7 @@ struct _GESSourcePrivate
   GstElement *last_converter;
   GstPad *ghostpad;
 
+  gboolean is_rendering_smartly;
 };
 
 G_DEFINE_TYPE_WITH_PRIVATE (GESSource, ges_source, GES_TYPE_TRACK_ELEMENT);
@@ -76,11 +77,11 @@ link_elements (GstElement * bin, GPtrArray * elements)
 static void
 _set_ghost_pad_target (GESSource * self, GstPad * srcpad, GstElement * element)
 {
-  GESSourcePrivate *priv = self->priv;
-  gboolean use_converter = FALSE;
   GstPadLinkReturn link_return;
+  GESSourcePrivate *priv = self->priv;
+  gboolean use_converter = ! !priv->first_converter;
 
-  if (priv->first_converter) {
+  if (use_converter && priv->is_rendering_smartly) {
     GstPad *pad = gst_element_get_static_pad (priv->first_converter, "sink");
     use_converter = gst_pad_can_link (srcpad, pad);
     gst_object_unref (pad);
@@ -168,6 +169,30 @@ ges_source_create_topbin (GESSource * source, const gchar * bin_name,
 }
 
 
+void
+ges_source_set_rendering_smartly (GESSource * source,
+    gboolean is_rendering_smartly)
+{
+
+  if (is_rendering_smartly) {
+    GESTrack *track = ges_track_element_get_track (GES_TRACK_ELEMENT (source));
+
+    if (track && ges_track_get_mixing (track)) {
+      GST_DEBUG_OBJECT (source, "Not rendering smartly as track is mixing!");
+
+      source->priv->is_rendering_smartly = FALSE;
+      return;
+    }
+  }
+  source->priv->is_rendering_smartly = is_rendering_smartly;
+}
+
+gboolean
+ges_source_get_rendering_smartly (GESSource * source)
+{
+  return source->priv->is_rendering_smartly;
+}
+
 static void
 ges_source_dispose (GObject * object)
 {

diff --git a/ges/ges-timeline-tree.c b/ges/ges-timeline-tree.c
index 09f405a..8f92d23 100644 (file)
--- a/ges/ges-timeline-tree.c
+++ b/ges/ges-timeline-tree.c
@@ -2481,3 +2481,21 @@ timeline_tree_reset_layer_active (GNode * root, GESLayer * layer)
   g_node_traverse (root, G_PRE_ORDER, G_TRAVERSE_LEAFS, -1,
       (GNodeTraverseFunc) reset_layer_activness, layer);
 }
+
+static gboolean
+set_is_smart_rendering (GNode * node, gboolean * is_rendering_smartly)
+{
+  if (!GES_IS_SOURCE (node->data))
+    return FALSE;
+
+  ges_source_set_rendering_smartly (GES_SOURCE (node->data),
+      *is_rendering_smartly);
+  return FALSE;
+}
+
+void
+timeline_tree_set_smart_rendering (GNode * root, gboolean rendering_smartly)
+{
+  g_node_traverse (root, G_PRE_ORDER, G_TRAVERSE_LEAFS, -1,
+      (GNodeTraverseFunc) set_is_smart_rendering, &rendering_smartly);
+}

diff --git a/ges/ges-timeline-tree.h b/ges/ges-timeline-tree.h
index 5fafba3..2ab4703 100644 (file)
--- a/ges/ges-timeline-tree.h
+++ b/ges/ges-timeline-tree.h
@@ -82,5 +82,6 @@ void
 timeline_update_duration                  (GESTimeline * timeline);
 
 void timeline_tree_reset_layer_active     (GNode *root, GESLayer *layer);
+void timeline_tree_set_smart_rendering    (GNode * root, gboolean rendering_smartly);
 
 void timeline_tree_init_debug             (void);

diff --git a/ges/ges-timeline.c b/ges/ges-timeline.c
index 8883921..23227cb 100644 (file)
--- a/ges/ges-timeline.c
+++ b/ges/ges-timeline.c
@@ -230,6 +230,8 @@ struct _GESTimelinePrivate
   gboolean disposed;
 
   GstStreamCollection *stream_collection;
+
+  gboolean rendering_smartly;
 };
 
 /* private structure to contain our track-related information */
@@ -2051,6 +2053,10 @@ timeline_add_element (GESTimeline * timeline, GESTimelineElement * element)
       ges_timeline_element_get_name (element), gst_object_ref (element));
 
   timeline_tree_track_element (timeline->priv->tree, element);
+  if (GES_IS_SOURCE (element)) {
+    ges_source_set_rendering_smartly (GES_SOURCE (element),
+        timeline->priv->rendering_smartly);
+  }
 
   return TRUE;
 }
@@ -2085,6 +2091,32 @@ timeline_get_tree (GESTimeline * timeline)
   return timeline->priv->tree;
 }
 
+void
+ges_timeline_set_smart_rendering (GESTimeline * timeline,
+    gboolean rendering_smartly)
+{
+  if (rendering_smartly) {
+    GList *tmp;
+
+    for (tmp = timeline->tracks; tmp; tmp = tmp->next) {
+      if (ges_track_get_mixing (tmp->data)) {
+        GST_INFO_OBJECT (timeline, "Smart rendering will not"
+            " work as track %" GST_PTR_FORMAT " is doing mixing", tmp->data);
+      } else {
+        ges_track_set_smart_rendering (tmp->data, rendering_smartly);
+      }
+    }
+  }
+  timeline_tree_set_smart_rendering (timeline->priv->tree, rendering_smartly);
+  timeline->priv->rendering_smartly = rendering_smartly;
+}
+
+gboolean
+ges_timeline_get_smart_rendering (GESTimeline * timeline)
+{
+  return timeline->priv->rendering_smartly;
+}
+
 /**** API *****/
 /**
  * ges_timeline_new:

diff --git a/ges/ges-track.c b/ges/ges-track.c
index 8ac2868..d3bd2ef 100644 (file)
--- a/ges/ges-track.c
+++ b/ges/ges-track.c
@@ -375,6 +375,15 @@ ges_track_get_composition (GESTrack * track)
   return track->priv->composition;
 }
 
+void
+ges_track_set_smart_rendering (GESTrack * track, gboolean rendering_smartly)
+{
+  GESTrackPrivate *priv = track->priv;
+
+  g_object_set (priv->capsfilter, "caps",
+      rendering_smartly ? NULL : priv->restriction_caps, NULL);
+}
+
 /* FIXME: Find out how to avoid doing this "hack" using the GDestroyNotify
  * function pointer in the trackelements_by_start GSequence
  *
@@ -985,6 +994,9 @@ ges_track_set_caps (GESTrack * track, const GstCaps * caps)
  * @caps: The new restriction-caps for @track
  *
  * Sets the #GESTrack:restriction-caps for the track.
+ *
+ * > **NOTE**: Restriction caps are **not** taken into account when
+ * > using #GESPipeline:mode=#GES_PIPELINE_MODE_SMART_RENDER.
  */
 void
 ges_track_set_restriction_caps (GESTrack * track, const GstCaps * caps)
@@ -1003,7 +1015,9 @@ ges_track_set_restriction_caps (GESTrack * track, const GstCaps * caps)
     gst_caps_unref (priv->restriction_caps);
   priv->restriction_caps = gst_caps_copy (caps);
 
-  g_object_set (priv->capsfilter, "caps", caps, NULL);
+  if (!track->priv->timeline ||
+      !ges_timeline_get_smart_rendering (track->priv->timeline))
+    g_object_set (priv->capsfilter, "caps", caps, NULL);
 
   g_object_notify (G_OBJECT (track), "restriction-caps");
 }
@@ -1105,6 +1119,9 @@ ges_track_set_mixing (GESTrack * track, gboolean mixing)
 notify:
   track->priv->mixing = mixing;
 
+  if (track->priv->timeline)
+    ges_timeline_set_smart_rendering (track->priv->timeline,
+        ges_timeline_get_smart_rendering (track->priv->timeline));
   g_object_notify_by_pspec (G_OBJECT (track), properties[ARG_MIXING]);
 
   GST_DEBUG_OBJECT (track, "The track has been set to mixing = %d", mixing);

diff --git a/ges/ges-uri-source.c b/ges/ges-uri-source.c
index 138b8b9..8c78926 100644 (file)
--- a/ges/ges-uri-source.c
+++ b/ges/ges-uri-source.c
@@ -29,6 +29,36 @@ GST_DEBUG_CATEGORY_STATIC (uri_source_debug);
 #undef GST_CAT_DEFAULT
 #define GST_CAT_DEFAULT uri_source_debug
 
+typedef enum
+{
+  GST_AUTOPLUG_SELECT_TRY,
+  GST_AUTOPLUG_SELECT_EXPOSE,
+  GST_AUTOPLUG_SELECT_SKIP,
+} GstAutoplugSelectResult;
+
+static gint
+autoplug_select_cb (GstElement * bin, GstPad * pad, GstCaps * caps,
+    GstElementFactory * factory, GESUriSource * self)
+{
+  GstElement *nlesrc;
+  GstCaps *downstream_caps;
+  GstAutoplugSelectResult res = GST_AUTOPLUG_SELECT_TRY;
+
+  if (!ges_source_get_rendering_smartly (GES_SOURCE (self->element))) {
+    GST_LOG_OBJECT (self->element, "Not being smart here");
+    return res;
+  }
+
+  nlesrc = ges_track_element_get_nleobject (self->element);
+  downstream_caps = gst_pad_peer_query_caps (nlesrc->srcpads->data, NULL);
+  if (downstream_caps && gst_caps_can_intersect (downstream_caps, caps)) {
+    GST_DEBUG_OBJECT (self, "Exposing %s", GST_OBJECT_NAME (factory));
+    res = GST_AUTOPLUG_SELECT_EXPOSE;
+  }
+  gst_clear_caps (&downstream_caps);
+
+  return res;
+}
 
 GstElement *
 ges_uri_source_create_source (GESUriSource * self)
@@ -48,6 +78,8 @@ ges_uri_source_create_source (GESUriSource * self)
 
   g_object_set (decodebin, "caps", caps,
       "expose-all-streams", FALSE, "uri", self->uri, NULL);
+  g_signal_connect (decodebin, "autoplug-select",
+      G_CALLBACK (autoplug_select_cb), self);
 
   return decodebin;