vasnprintf: fix out-of-memory handling

[Bruno Haible]

The out-of-memory handling of gnulib's *printf replacement is broken: it
sometimes sets errno to EINVAL instead of ENOMEM. This fixes it.

2007-11-03  Bruno Haible  <address@hidden>

        Fix out-of-memory handling of vasnprintf.
        * lib/printf-parse.c: Include <errno.h>.
        (PRINTF_PARSE): When failing, set errno to EINVAL or ENOMEM.
        * lib/vasnprintf.c (VASNPRINTF): When PRINTF_PARSE fails, assume errno
        is already set.

*** lib/printf-parse.c.orig     2007-11-03 12:28:31.000000000 +0100
--- lib/printf-parse.c  2007-11-03 12:28:29.000000000 +0100
***************
*** 63,68 ****
--- 63,71 ----
  /* malloc(), realloc(), free().  */
  #include <stdlib.h>
  
+ /* errno.  */
+ #include <errno.h>
+ 
  /* Checked size_t computations.  */
  #include "xsize.h"
  
***************
*** 89,95 ****
    d->dir = (DIRECTIVE *) malloc (d_allocated * sizeof (DIRECTIVE));
    if (d->dir == NULL)
      /* Out of memory.  */
!     return -1;
  
    a->count = 0;
    a_allocated = 0;
--- 92,98 ----
    d->dir = (DIRECTIVE *) malloc (d_allocated * sizeof (DIRECTIVE));
    if (d->dir == NULL)
      /* Out of memory.  */
!     goto out_of_memory_1;
  
    a->count = 0;
    a_allocated = 0;
***************
*** 109,121 ****
        memory_size = xtimes (a_allocated, sizeof (argument));          \
        if (size_overflow_p (memory_size))                              \
          /* Overflow, would lead to out of memory.  */                 \
!         goto error;                                                   \
        memory = (argument *) (a->arg                                   \
                               ? realloc (a->arg, memory_size)          \
                               : malloc (memory_size));                 \
        if (memory == NULL)                                             \
          /* Out of memory.  */                                         \
!         goto error;                                                   \
        a->arg = memory;                                                \
        }                                                                       
\
      while (a->count <= n)                                             \
--- 112,124 ----
        memory_size = xtimes (a_allocated, sizeof (argument));          \
        if (size_overflow_p (memory_size))                              \
          /* Overflow, would lead to out of memory.  */                 \
!         goto out_of_memory;                                           \
        memory = (argument *) (a->arg                                   \
                               ? realloc (a->arg, memory_size)          \
                               : malloc (memory_size));                 \
        if (memory == NULL)                                             \
          /* Out of memory.  */                                         \
!         goto out_of_memory;                                           \
        a->arg = memory;                                                \
        }                                                                       
\
      while (a->count <= n)                                             \
***************
*** 539,549 ****
              memory_size = xtimes (d_allocated, sizeof (DIRECTIVE));
              if (size_overflow_p (memory_size))
                /* Overflow, would lead to out of memory.  */
!               goto error;
              memory = (DIRECTIVE *) realloc (d->dir, memory_size);
              if (memory == NULL)
                /* Out of memory.  */
!               goto error;
              d->dir = memory;
            }
        }
--- 542,552 ----
              memory_size = xtimes (d_allocated, sizeof (DIRECTIVE));
              if (size_overflow_p (memory_size))
                /* Overflow, would lead to out of memory.  */
!               goto out_of_memory;
              memory = (DIRECTIVE *) realloc (d->dir, memory_size);
              if (memory == NULL)
                /* Out of memory.  */
!               goto out_of_memory;
              d->dir = memory;
            }
        }
***************
*** 566,571 ****
--- 569,584 ----
      free (a->arg);
    if (d->dir)
      free (d->dir);
+   errno = EINVAL;
+   return -1;
+ 
+ out_of_memory:
+   if (a->arg)
+     free (a->arg);
+   if (d->dir)
+     free (d->dir);
+ out_of_memory_1:
+   errno = ENOMEM;
    return -1;
  }
  
*** lib/vasnprintf.c.orig       2007-11-03 12:28:31.000000000 +0100
--- lib/vasnprintf.c    2007-11-03 12:14:59.000000000 +0100
***************
*** 1196,1205 ****
    arguments a;
  
    if (PRINTF_PARSE (format, &d, &a) < 0)
!     {
!       errno = EINVAL;
!       return NULL;
!     }
  
  #define CLEANUP() \
    free (d.dir);                                                               
\
--- 1412,1419 ----
    arguments a;
  
    if (PRINTF_PARSE (format, &d, &a) < 0)
!     /* errno is already set.  */
!     return NULL;
  
  #define CLEANUP() \
    free (d.dir);                                                               
\