glimagesink: only update the output info iff there's a valid display_rect
authorMatthew Waters <matthew@centricular.com>
Fri, 28 Sep 2018 02:09:16 +0000 (12:09 +1000)
committerMatthew Waters <matthew@centricular.com>
Thu, 4 Oct 2018 04:28:35 +0000 (14:28 +1000)
Attempting to use the MAX(1, display_rect) would result in the overlay
composition attempting to draw into 1x1 buffer and calculate some
grossly incorrect sizes.

previously failing case:
gltestsrc ! textoverlay text=GStreamer ! glimagesinkelement

ext/gl/gstglimagesink.c

index 860bcfd..c70cd55 100644 (file)
@@ -1426,27 +1426,29 @@ update_output_format (GstGLImageSink * glimage_sink)
      * view parity properly for line-by-line modes, because that
      * depends on the window being placed correctly.
      * FIXME: Should this rescaling be configurable? */
-    new_w = MAX (1, glimage_sink->display_rect.w);
-    new_h = MAX (1, glimage_sink->display_rect.h);
-    if (new_w != out_info->width || new_h != out_info->height) {
-      /* Recalculate PAR if rescaling */
-      gint from_dar_n, from_dar_d;
-      if (!gst_util_fraction_multiply (out_info->width, out_info->height,
-              out_info->par_n, out_info->par_d, &from_dar_n,
-              &from_dar_d) ||
-          !gst_util_fraction_multiply (from_dar_n, from_dar_d, new_h, new_w,
-              &par_n, &par_d)) {
-        par_n = glimage_sink->par_n;
-        par_d = glimage_sink->par_d;
+    if (glimage_sink->display_rect.w > 0 && glimage_sink->display_rect.h > 0) {
+      new_w = glimage_sink->display_rect.w;
+      new_h = glimage_sink->display_rect.h;
+      if (new_w != out_info->width || new_h != out_info->height) {
+        /* Recalculate PAR if rescaling */
+        gint from_dar_n, from_dar_d;
+        if (!gst_util_fraction_multiply (out_info->width, out_info->height,
+                out_info->par_n, out_info->par_d, &from_dar_n,
+                &from_dar_d) ||
+            !gst_util_fraction_multiply (from_dar_n, from_dar_d, new_h, new_w,
+                &par_n, &par_d)) {
+          par_n = glimage_sink->par_n;
+          par_d = glimage_sink->par_d;
+        }
+        out_info->par_n = par_n;
+        out_info->par_d = par_d;
+        out_info->width = new_w;
+        out_info->height = new_h;
       }
-      out_info->par_n = par_n;
-      out_info->par_d = par_d;
-      out_info->width = new_w;
-      out_info->height = new_h;
-    }
 
-    GST_LOG_OBJECT (glimage_sink, "Set 3D output scale to %d,%d PAR %d/%d",
-        out_info->width, out_info->height, out_info->par_n, out_info->par_d);
+      GST_LOG_OBJECT (glimage_sink, "Set 3D output scale to %dx%d PAR %d/%d",
+          out_info->width, out_info->height, out_info->par_n, out_info->par_d);
+    }
   }
 
   s = gst_caps_get_structure (glimage_sink->in_caps, 0);