regexec.c: avoid arithmetic overflow in buffer size calculation

diff --git a/ChangeLog b/ChangeLog
index c4fb74f..9b3fe33 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,9 @@
 2010-01-22  Jim Meyering  <jim@meyering.net>
 
+       [BZ #11188]
+       * posix/regexec.c (build_trtable): Avoid arithmetic overflow
+       in size calculation.
+
        [BZ #11187]
        * posix/regexec.c (re_search_2_stub): Use simpler method than
        boolean for freeing internal storage.

diff --git a/posix/regexec.c b/posix/regexec.c
index c7d0b37..3765d00 100644 (file)
--- a/posix/regexec.c
+++ b/posix/regexec.c
@@ -3359,6 +3359,13 @@ build_trtable (const re_dfa_t *dfa, re_dfastate_t *state)
   if (BE (err != REG_NOERROR, 0))
     goto out_free;
 
+  /* Avoid arithmetic overflow in size calculation.  */
+  if (BE ((((SIZE_MAX - (sizeof (re_node_set) + sizeof (bitset_t)) * SBC_MAX)
+           / (3 * sizeof (re_dfastate_t *)))
+          < ndests),
+         0))
+    goto out_free;
+
   if (__libc_use_alloca ((sizeof (re_node_set) + sizeof (bitset_t)) * SBC_MAX
                         + ndests * 3 * sizeof (re_dfastate_t *)))
     dest_states = (re_dfastate_t **)