gtkglsink: Avoid assertion when applying "auto" rotation method

Guard against the orientation not coming from an inexistant tag, nor
from the application (rotation set to "auto") which caused an assertion.

When the application requests the auto rotation method, make sure it is
resolved to a rotation that's applicable.

ERROR:gstreamer/subprojects/gst-plugins-good/ext/gtk/gtkgstglwidget.c:745:gtk_gst_gl_widget_set_rotate_method: code should not be reached

Fixes: 103ceb853a5f7dade07a1ac4aa517e9df6ed5ded
Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/1134>

diff --git a/subprojects/gst-plugins-good/ext/gtk/gtkgstglwidget.c b/subprojects/gst-plugins-good/ext/gtk/gtkgstglwidget.c
index c2d564d..1f24400 100644 (file)
--- a/subprojects/gst-plugins-good/ext/gtk/gtkgstglwidget.c
+++ b/subprojects/gst-plugins-good/ext/gtk/gtkgstglwidget.c
@@ -693,6 +693,9 @@ gtk_gst_gl_widget_set_rotate_method (GtkGstGLWidget * gst_widget,
   if (method == GST_VIDEO_ORIENTATION_CUSTOM) {
     GST_WARNING_OBJECT (gst_widget, "unsupported custom orientation");
     return;
+  } else if (method == GST_VIDEO_ORIENTATION_AUTO && from_tag) {
+    GST_WARNING_OBJECT (gst_widget, "auto orientation cannot come from a tag");
+    return;
   }
 
   GTK_GST_BASE_WIDGET_LOCK (gst_widget);
@@ -706,6 +709,12 @@ gtk_gst_gl_widget_set_rotate_method (GtkGstGLWidget * gst_widget,
   else
     method = priv->rotate_method;
 
+  /* We can't apply an AUTO orientation if we don't have an
+   * orientation coming from a tag, so reset to identity */
+  if (method != priv->current_rotate_method &&
+      method == GST_VIDEO_ORIENTATION_AUTO)
+    method = GST_VIDEO_ORIENTATION_IDENTITY;
+
   if (method != priv->current_rotate_method) {
     GST_DEBUG ("Changing method from %d to %d",
         priv->current_rotate_method, method);