gstutils: Fix linear regression comparision

The check for dropping precision was wrong when sxx and syy were negative.

if they are negative then "G_MAXINT64 - val" would always overflow

The check was meant to use G_MININT64 (like in the loop contained just
after).

diff --git a/gst/gstutils.c b/gst/gstutils.c
index 9372e4e..143bed7 100644 (file)
--- a/gst/gstutils.c
+++ b/gst/gstutils.c
@@ -4368,7 +4368,7 @@ gst_calculate_linear_regression (const GstClockTime * xy,
           tmp /= 4;
         } while (G_MAXINT64 - sxx <= tmp);
         break;
-      } else if (G_UNLIKELY (tmp < 0 && sxx < 0 && (G_MAXINT64 - sxx >= tmp))) {
+      } else if (G_UNLIKELY (tmp < 0 && sxx < 0 && (G_MININT64 - sxx >= tmp))) {
         do {
           /* Drop some precision and restart */
           pshift++;
@@ -4387,7 +4387,7 @@ gst_calculate_linear_regression (const GstClockTime * xy,
           tmp /= 4;
         } while (G_MAXINT64 - syy <= tmp);
         break;
-      } else if (G_UNLIKELY (tmp < 0 && syy < 0 && (G_MAXINT64 - syy >= tmp))) {
+      } else if (G_UNLIKELY (tmp < 0 && syy < 0 && (G_MININT64 - syy >= tmp))) {
         do {
           pshift++;
           syy /= 4;