wcswidth, mbswidth: add overflow check

[Bruno Haible]

Hi,

On 2005-04-09, Jim noticed that mbswidth() can provoke integer overflow [1].
wcswidth() can do the same in gnulib, but the replacement in coreutils has
a safety check against it. Let's do the same in gnulib:

[1] http://lists.gnu.org/archive/html/bug-gnulib/2005-04/msg00022.html


2011-03-12  Bruno Haible  <address@hidden>

        wcswidth, mbswidth: Avoid integer overflow.
        * lib/wcswidth.c: Include <limits.h>.
        * lib/wcswidth-impl.h (wcswidth): Avoid 'int' overflow.
        * lib/mbswidth.c: Include <limits.h>.
        (mbsnwidth): Avoid 'int' overflow.
        Reported by Jim Meyering.

--- lib/mbswidth.c.orig Sat Mar 12 13:50:16 2011
+++ lib/mbswidth.c      Sat Mar 12 13:50:11 2011
@@ -35,12 +35,14 @@
 /* Get iswcntrl().  */
 #include <wctype.h>
 
+/* Get INT_MAX.  */
+#include <limits.h>
+
 /* Returns the number of columns needed to represent the multibyte
    character string pointed to by STRING.  If a non-printable character
    occurs, and MBSW_REJECT_UNPRINTABLE is specified, -1 is returned.
    With flags = MBSW_REJECT_INVALID | MBSW_REJECT_UNPRINTABLE, this is
-   the multibyte analogue of the wcswidth function.
-   If STRING is not of length < INT_MAX / 2, integer overflow can occur.  */
+   the multibyte analogue of the wcswidth function.  */
 int
 mbswidth (const char *string, int flags)
 {
@@ -50,8 +52,7 @@
 /* Returns the number of columns needed to represent the multibyte
    character string pointed to by STRING of length NBYTES.  If a
    non-printable character occurs, and MBSW_REJECT_UNPRINTABLE is
-   specified, -1 is returned.
-   If NBYTES is not < INT_MAX / 2, integer overflow can occur.  */
+   specified, -1 is returned.  */
 int
 mbsnwidth (const char *string, size_t nbytes, int flags)
 {
@@ -135,11 +136,22 @@
                     w = wcwidth (wc);
                     if (w >= 0)
                       /* A printable multibyte character.  */
-                      width += w;
+                      {
+                        if (w > INT_MAX - width)
+                          goto overflow;
+                        width += w;
+                      }
                     else
                       /* An unprintable multibyte character.  */
                       if (!(flags & MBSW_REJECT_UNPRINTABLE))
-                        width += (iswcntrl (wc) ? 0 : 1);
+                        {
+                          if (!iswcntrl (wc))
+                            {
+                              if (width == INT_MAX)
+                                goto overflow;
+                              width++;
+                            }
+                        }
                       else
                         return -1;
 
@@ -157,11 +169,25 @@
       unsigned char c = (unsigned char) *p++;
 
       if (isprint (c))
-        width++;
+        {
+          if (width == INT_MAX)
+            goto overflow;
+          width++;
+        }
       else if (!(flags & MBSW_REJECT_UNPRINTABLE))
-        width += (iscntrl (c) ? 0 : 1);
+        {
+          if (!iscntrl (c))
+            {
+              if (width == INT_MAX)
+                goto overflow;
+              width++;
+            }
+        }
       else
         return -1;
     }
   return width;
+
+ overflow:
+  return INT_MAX;
 }
--- lib/wcswidth-impl.h.orig    Sat Mar 12 13:50:16 2011
+++ lib/wcswidth-impl.h Sat Mar 12 13:19:58 2011
@@ -28,6 +28,8 @@
         int width = wcwidth (c);
         if (width < 0)
           goto found_nonprinting;
+        if (width > INT_MAX - count)
+          goto overflow;
         count += width;
       }
     }
@@ -35,4 +37,7 @@
 
  found_nonprinting:
   return -1;
+
+ overflow:
+  return INT_MAX;
 }
--- lib/wcswidth.c.orig Sat Mar 12 13:50:16 2011
+++ lib/wcswidth.c      Sat Mar 12 13:20:10 2011
@@ -20,4 +20,6 @@
 /* Specification.  */
 #include <wchar.h>
 
+#include <limits.h>
+
 #include "wcswidth-impl.h"
-- 
In memoriam Zoran Djindjić <http://en.wikipedia.org/wiki/Zoran_Đinđić>