From: David Schleef <ds@schleef.org>
Date: Mon, 20 Feb 2012 15:10:22 +0000 (+0100)
Subject: videoconvert: clamp intermediates when dithering
X-Git-Tag: 1.19.3~511^2~6766
X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=bb19c41081f8a4204a4a5a4ec38f7b730039d238;p=platform%2Fupstream%2Fgstreamer.git

videoconvert: clamp intermediates when dithering

Port from the colorspace plugin in -bad.
---

diff --git a/gst/videoconvert/videoconvert.c b/gst/videoconvert/videoconvert.c
index 543f2a9..8533cf6 100644
--- a/gst/videoconvert/videoconvert.c
+++ b/gst/videoconvert/videoconvert.c
@@ -1783,10 +1783,14 @@ videoconvert_dither_verterr (VideoConvert * convert, int j)
   int i;
   guint16 *tmpline = convert->tmpline16;
   guint16 *errline = convert->errline;
+  unsigned int mask = 0xff;
 
   for (i = 0; i < 4 * convert->width; i++) {
-    tmpline[i] += errline[i];
-    errline[i] = tmpline[i] & 0xff;
+    int x = tmpline[i] + errline[i];
+    if (x > 65535)
+      x = 65535;
+    tmpline[i] = x;
+    errline[i] = x & mask;
   }
 }
 
@@ -1807,7 +1811,11 @@ videoconvert_dither_halftone (VideoConvert * convert, int j)
   };
 
   for (i = 0; i < convert->width * 4; i++) {
-    tmpline[i] += halftone[(i >> 2) & 7][j & 7];
+    int x;
+    x = tmpline[i] + halftone[(i >> 2) & 7][j & 7];
+    if (x > 65535)
+      x = 65535;
+    tmpline[i] = x;
   }
 }