Re: xmalloc.c's xcalloc performs unnecessary test for N*S overflow

[Paul Eggert]

I installed the following (slightly different) patch for that problem,
into gnulib.  It's tested with GNU tar.

2005-06-22  Paul Eggert  <address@hidden>

        * xmalloc.c (HAVE_GNU_CALLOC): New constant.
        (xcalloc): Use it to avoid needless tests.
        Problem reported by Jim Meyering.

--- xmalloc.c   14 May 2005 06:03:58 -0000      1.36
+++ xmalloc.c   22 Jun 2005 07:12:04 -0000      1.37
@@ -30,6 +30,15 @@
 # define SIZE_MAX ((size_t) -1)
 #endif
 
+/* 1 if calloc is known to be compatible with GNU calloc.  This
+   matters if we are not also using the calloc module, which defines
+   HAVE_CALLOC and supports the GNU API even on non-GNU platforms.  */
+#if defined HAVE_CALLOC || defined __GLIBC__
+enum { HAVE_GNU_CALLOC = 1 };
+#else
+enum { HAVE_GNU_CALLOC = 0 };
+#endif
+
 /* Allocate an array of N objects, each with S bytes of memory,
    dynamically, with error checking.  S must be nonzero.  */
 
@@ -204,8 +213,11 @@ xcalloc (size_t n, size_t s)
 {
   void *p;
   /* Test for overflow, since some calloc implementations don't have
-     proper overflow checks.  */
-  if (xalloc_oversized (n, s) || (! (p = calloc (n, s)) && n != 0))
+     proper overflow checks.  But omit overflow and size-zero tests if
+     HAVE_GNU_CALLOC, since GNU calloc catches overflow and never
+     returns NULL if successful.  */
+  if ((! HAVE_GNU_CALLOC && xalloc_oversized (n, s))
+      || (! (p = calloc (n, s)) && (HAVE_GNU_CALLOC || n != 0)))
     xalloc_die ();
   return p;
 }