Avoid signed integer overflows in numeric conversions.

Reported by Miroslav Lichvar <mlichvar@redhat.com>
in <https://lists.gnu.org/archive/html/guile-devel/2016-02/msg00045.html>

* libguile/conv-integer.i.c: Avoid signed overflow.
* libguile/numbers.c (scm_is_signed_integer): Avoid signed overflow.

diff --git a/libguile/conv-integer.i.c b/libguile/conv-integer.i.c
index a5362d39cc411daea32903b991668bfc0af92d4a..0aa81dc74694d4d094af9b39a8ddba47a2fcd297 100644 (file)
--- a/libguile/conv-integer.i.c
+++ b/libguile/conv-integer.i.c
@@ -64,25 +64,30 @@ SCM_TO_TYPE_PROTO (SCM val)
        }
       else
        {
-         scm_t_intmax n;
+         scm_t_uintmax abs_n;
+         TYPE n;
          size_t count;
 
          if (mpz_sizeinbase (SCM_I_BIG_MPZ (val), 2)
              > CHAR_BIT*sizeof (scm_t_uintmax))
            goto out_of_range;
          
-         mpz_export (&n, &count, 1, sizeof (scm_t_uintmax), 0, 0,
+         mpz_export (&abs_n, &count, 1, sizeof (scm_t_uintmax), 0, 0,
                      SCM_I_BIG_MPZ (val));
 
          if (mpz_sgn (SCM_I_BIG_MPZ (val)) >= 0)
            {
-             if (n < 0)
+             if (abs_n <= TYPE_MAX)
+               n = abs_n;
+             else
                goto out_of_range;
            }
          else
            {
-             n = -n;
-             if (n >= 0)
+             /* Carefully avoid signed integer overflow. */
+             if (TYPE_MIN < 0 && abs_n - 1 <= -(TYPE_MIN + 1))
+               n = -1 - (TYPE)(abs_n - 1);
+             else
                goto out_of_range;
            }
 

diff --git a/libguile/numbers.c b/libguile/numbers.c
index 0d053c867f557c35ea1daf08db59c82e27caa415..34980c6a9935bd1851416a205ebe2e53e98c583a 100644 (file)
--- a/libguile/numbers.c
+++ b/libguile/numbers.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 1995-2015 Free Software Foundation, Inc.
+/* Copyright (C) 1995-2016 Free Software Foundation, Inc.
  *
  * Portions Copyright 1990, 1991, 1992, 1993 by AT&T Bell Laboratories
  * and Bellcore.  See scm_divide.
@@ -9641,6 +9641,7 @@ scm_is_signed_integer (SCM val, scm_t_intmax min, scm_t_intmax max)
        }
       else
        {
+         scm_t_uintmax abs_n;
          scm_t_intmax n;
          size_t count;
 
@@ -9648,18 +9649,22 @@ scm_is_signed_integer (SCM val, scm_t_intmax min, scm_t_intmax max)
              > CHAR_BIT*sizeof (scm_t_uintmax))
            return 0;
          
-         mpz_export (&n, &count, 1, sizeof (scm_t_uintmax), 0, 0,
+         mpz_export (&abs_n, &count, 1, sizeof (scm_t_uintmax), 0, 0,
                      SCM_I_BIG_MPZ (val));
 
          if (mpz_sgn (SCM_I_BIG_MPZ (val)) >= 0)
            {
-             if (n < 0)
+             if (abs_n <= max)
+               n = abs_n;
+             else
                return 0;
            }
          else
            {
-             n = -n;
-             if (n >= 0)
+             /* Carefully avoid signed integer overflow. */
+             if (min < 0 && abs_n - 1 <= -(min + 1))
+               n = -1 - (scm_t_intmax)(abs_n - 1);
+             else
                return 0;
            }