[PATCH] reallocarray: don’t crash if item size is 0

[Paul Eggert]

This problem affects only platforms where xalloc_oversized
divides a number by the size arg.  Fix this by defining
xalloc_oversized (n, s) to work even if s == 0.
* lib/malloca.h, lib/xalloc-oversized.h: Document new behavior.
* lib/xalloc-oversized.h (__xalloc_oversized): Do not crash if S==0.
* tests/test-reallocarray.c (main): Test for the bug.
---
 ChangeLog                 | 10 ++++++++++
 lib/malloca.h             |  2 +-
 lib/xalloc-oversized.h    |  8 +++++---
 tests/test-reallocarray.c |  9 +++++++--
 4 files changed, 23 insertions(+), 6 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 3a21db4f4..03b01b3d7 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,13 @@
+2021-04-25  Paul Eggert  <eggert@cs.ucla.edu>
+
+       reallocarray: don’t crash if item size is 0
+       This problem affects only platforms where xalloc_oversized
+       divides a number by the size arg.  Fix this by defining
+       xalloc_oversized (n, s) to work even if s == 0.
+       * lib/malloca.h, lib/xalloc-oversized.h: Document new behavior.
+       * lib/xalloc-oversized.h (__xalloc_oversized): Do not crash if S==0.
+       * tests/test-reallocarray.c (main): Test for the bug.
+
 2021-04-24  Paul Eggert  <eggert@cs.ucla.edu>
 
        xmalloca, etc.: avoid unlikely trap
diff --git a/lib/malloca.h b/lib/malloca.h
index a255e3f04..d646d4102 100644
--- a/lib/malloca.h
+++ b/lib/malloca.h
@@ -76,7 +76,7 @@ extern void freea (void *p);
 
 /* nmalloca(N,S) is an overflow-safe variant of malloca (N * S).
    It allocates an array of N objects, each with S bytes of memory,
-   on the stack.  S must be positive and N must be nonnegative.
+   on the stack.  N and S should be nonnegative and free of side effects.
    The array must be freed using freea() before the function returns.  */
 #define nmalloca(n, s) \
   (xalloc_oversized (n, s) ? NULL : malloca ((n) * (size_t) (s)))
diff --git a/lib/xalloc-oversized.h b/lib/xalloc-oversized.h
index 2619a2aab..dbaee4a29 100644
--- a/lib/xalloc-oversized.h
+++ b/lib/xalloc-oversized.h
@@ -22,19 +22,21 @@
 #include <stdint.h>
 
 /* True if N * S does not fit into both ptrdiff_t and size_t.
-   S must be positive and N must be nonnegative.
+   N and S should be nonnegative and free of side effects.
    This expands to a constant expression if N and S are both constants.
    By gnulib convention, SIZE_MAX represents overflow in size_t
    calculations, so the conservative size_t-based dividend to use here
    is SIZE_MAX - 1.  */
 #define __xalloc_oversized(n, s) \
-  ((size_t) (PTRDIFF_MAX < SIZE_MAX ? PTRDIFF_MAX : SIZE_MAX - 1) / (s) < (n))
+  ((s) != 0 \
+   && ((size_t) (PTRDIFF_MAX < SIZE_MAX ? PTRDIFF_MAX : SIZE_MAX - 1) / (s) \
+       < (n)))
 
 /* Return 1 if and only if an array of N objects, each of size S,
    cannot exist reliably because its total size in bytes would exceed
    MIN (PTRDIFF_MAX, SIZE_MAX - 1).
 
-   N must be nonnegative and S must be positive.
+   N and S should be nonnegative and free of side effects.
 
    Warning: (xalloc_oversized (N, S) ? NULL : malloc (N * S)) can
    misbehave if N and S are both narrower than ptrdiff_t and size_t,
diff --git a/tests/test-reallocarray.c b/tests/test-reallocarray.c
index 973f22ba6..6de355e6b 100644
--- a/tests/test-reallocarray.c
+++ b/tests/test-reallocarray.c
@@ -26,12 +26,12 @@ SIGNATURE_CHECK (reallocarray, void *, (void *, size_t, 
size_t));
 int
 main ()
 {
-  void *volatile p = NULL;
-
   /* Check that reallocarray fails when requested to allocate a block
      of memory larger than PTRDIFF_MAX or SIZE_MAX bytes.  */
   for (size_t n = 2; n != 0; n <<= 1)
     {
+      void *volatile p = NULL;
+
       p = reallocarray (p, PTRDIFF_MAX / n + 1, n);
       if (p)
         return 1;
@@ -43,6 +43,11 @@ main ()
         return 3;
       if (errno != ENOMEM)
         return 4;
+
+      /* Reallocarray should not crash with zero sizes.  */
+      p = reallocarray (p, 0, n);
+      p = reallocarray (p, n, 0);
+      free (p);
     }
 
   return 0;
-- 
2.27.0