videoconvert: clamp intermediates when dithering

Port from the colorspace plugin in -bad.

diff --git a/gst/videoconvert/videoconvert.c b/gst/videoconvert/videoconvert.c
index 543f2a9..8533cf6 100644 (file)
--- a/gst/videoconvert/videoconvert.c
+++ b/gst/videoconvert/videoconvert.c
@@ -1783,10 +1783,14 @@ videoconvert_dither_verterr (VideoConvert * convert, int j)
   int i;
   guint16 *tmpline = convert->tmpline16;
   guint16 *errline = convert->errline;
+  unsigned int mask = 0xff;
 
   for (i = 0; i < 4 * convert->width; i++) {
-    tmpline[i] += errline[i];
-    errline[i] = tmpline[i] & 0xff;
+    int x = tmpline[i] + errline[i];
+    if (x > 65535)
+      x = 65535;
+    tmpline[i] = x;
+    errline[i] = x & mask;
   }
 }
 
@@ -1807,7 +1811,11 @@ videoconvert_dither_halftone (VideoConvert * convert, int j)
   };
 
   for (i = 0; i < convert->width * 4; i++) {
-    tmpline[i] += halftone[(i >> 2) & 7][j & 7];
+    int x;
+    x = tmpline[i] + halftone[(i >> 2) & 7][j & 7];
+    if (x > 65535)
+      x = 65535;
+    tmpline[i] = x;
   }
 }