(make-vector 100000000000 nil) shouldn't force a quick exit

[Paul Eggert]

Currently, when the user allocates a too-large vector, e.g.,
(make-vector 100000000000 nil), the malloc failure causes Emacs to go
into "I'm about to die!" mode, because it assumes that memory is so
low that the user should simply exit as soon as possible.  But there's
typically plenty of memory available.  Here's a proposed patch to
cause Emacs to go into low-memory mode only when memory is really low.

This proposed patch also would affect gnulib (lib/allocator.h and
lib/careadlinkat.c) so I'm CC'ing bug-gnulib.


[ChangeLog]
Malloc failure behavior now depends on size of allocation.
* lib/allocator.h (struct allocator.die): New size arg.
* lib/careadlinkat.c (careadlinkat): Pass size to 'die' function.
If the actual problem is an ssize_t limitation, not a size_t or
malloc failure, fail with errno == ENAMETOOLONG instead of calling 'die'.
[src/ChangeLog]
Malloc failure behavior now depends on size of allocation.
* alloc.c (buffer_memory_full, memory_full): New arg NBYTES.
* lisp.h: Change signatures accordingly.
* alloc.c, buffer.c, editfns.c, menu.c, minibuf.c, xterm.c:
All callers changed.
=== modified file 'lib/allocator.h'
--- lib/allocator.h     2011-04-09 18:44:05 +0000
+++ lib/allocator.h     2011-05-30 16:12:20 +0000
@@ -45,10 +45,11 @@
   /* Call FREE to free memory, like 'free'.  */
   void (*free) (void *);

-  /* If nonnull, call DIE if MALLOC or REALLOC fails.  DIE should not
-     return.  DIE can be used by code that detects memory overflow
-     while calculating sizes to be passed to MALLOC or REALLOC.  */
-  void (*die) (void);
+  /* If nonnull, call DIE (SIZE) if MALLOC (SIZE) or REALLOC (...,
+     SIZE) fails.  DIE should not return.  SIZE should equal SIZE_MAX
+     if size_t overflow was detected while calculating sizes to be
+     passed to MALLOC or REALLOC.  */
+  void (*die) (size_t);
 };

 /* An allocator using the stdlib functions and a null DIE function.  */

=== modified file 'lib/careadlinkat.c'
--- lib/careadlinkat.c  2011-04-10 16:00:46 +0000
+++ lib/careadlinkat.c  2011-05-30 16:12:20 +0000
@@ -135,6 +135,7 @@
           if (buf == stack_buf)
             {
               char *b = (char *) alloc->allocate (link_size);
+              buf_size = link_size;
               if (! b)
                 break;
               memcpy (b, buf, link_size);
@@ -158,6 +159,11 @@
         buf_size *= 2;
       else if (buf_size < buf_size_max)
         buf_size = buf_size_max;
+      else if (buf_size_max < SIZE_MAX)
+        {
+          errno = ENAMETOOLONG;
+          return NULL;
+        }
       else
         break;
       buf = (char *) alloc->allocate (buf_size);
@@ -165,7 +171,7 @@
   while (buf);

   if (alloc->die)
-    alloc->die ();
+    alloc->die (buf_size);
   errno = ENOMEM;
   return NULL;
 }

=== modified file 'src/alloc.c'
--- src/alloc.c 2011-05-30 16:09:29 +0000
+++ src/alloc.c 2011-05-30 16:23:20 +0000
@@ -471,7 +471,7 @@
 /* Called if we can't allocate relocatable space for a buffer.  */

 void
-buffer_memory_full (void)
+buffer_memory_full (EMACS_INT nbytes)
 {
   /* If buffers use the relocating allocator, no need to free
      spare_memory, because we may have plenty of malloc space left
@@ -481,7 +481,7 @@
      malloc.  */

 #ifndef REL_ALLOC
-  memory_full ();
+  memory_full (nbytes);
 #endif

   /* This used to call error, but if we've run out of memory, we could
@@ -677,7 +677,7 @@
   MALLOC_UNBLOCK_INPUT;

   if (!val && size)
-    memory_full ();
+    memory_full (size);
   return val;
 }

@@ -698,7 +698,8 @@
     val = (POINTER_TYPE *) realloc (block, size);
   MALLOC_UNBLOCK_INPUT;

-  if (!val && size) memory_full ();
+  if (!val && size)
+    memory_full (size);
   return val;
 }

@@ -791,7 +792,7 @@

   MALLOC_UNBLOCK_INPUT;
   if (!val && nbytes)
-    memory_full ();
+    memory_full (nbytes);
   return val;
 }

@@ -938,7 +939,7 @@
       if (base == 0)
        {
          MALLOC_UNBLOCK_INPUT;
-         memory_full ();
+         memory_full (ABLOCKS_BYTES);
        }

       aligned = (base == abase);
@@ -964,7 +965,7 @@
              lisp_malloc_loser = base;
              free (base);
              MALLOC_UNBLOCK_INPUT;
-             memory_full ();
+             memory_full (SIZE_MAX);
            }
        }
 #endif
@@ -3270,35 +3271,58 @@
  ************************************************************************/


-/* Called if malloc returns zero.  */
+/* Called if malloc (NBYTES) returns zero.  If NBYTES == SIZE_MAX,
+   there may have been size_t overflow so that malloc was never
+   called, or perhaps malloc was invoked successfully but the
+   resulting pointer had problems fitting into a tagged EMACS_INT.  In
+   either case this counts as memory being full even though malloc did
+   not fail.  */

 void
-memory_full (void)
+memory_full (size_t nbytes)
 {
-  int i;
-
-  Vmemory_full = Qt;
-
-  memory_full_cons_threshold = sizeof (struct cons_block);
-
-  /* The first time we get here, free the spare memory.  */
-  for (i = 0; i < sizeof (spare_memory) / sizeof (char *); i++)
-    if (spare_memory[i])
-      {
-       if (i == 0)
-         free (spare_memory[i]);
-       else if (i >= 1 && i <= 4)
-         lisp_align_free (spare_memory[i]);
-       else
-         lisp_free (spare_memory[i]);
-       spare_memory[i] = 0;
-      }
-
-  /* Record the space now used.  When it decreases substantially,
-     we can refill the memory reserve.  */
+  /* Do not go into hysterics merely because a large request failed.  */
+  int enough_free_memory = 0;
+  if (SPARE_MEMORY < nbytes)
+    {
+      void *p = malloc (SPARE_MEMORY);
+      if (p)
+       {
+         if (spare_memory[0])
+           free (p);
+         else
+           spare_memory[0] = p;
+         enough_free_memory = 1;
+       }
+    }
+
+  if (! enough_free_memory)
+    {
+      int i;
+
+      Vmemory_full = Qt;
+
+      memory_full_cons_threshold = sizeof (struct cons_block);
+
+      /* The first time we get here, free the spare memory.  */
+      for (i = 0; i < sizeof (spare_memory) / sizeof (char *); i++)
+       if (spare_memory[i])
+         {
+           if (i == 0)
+             free (spare_memory[i]);
+           else if (i >= 1 && i <= 4)
+             lisp_align_free (spare_memory[i]);
+           else
+             lisp_free (spare_memory[i]);
+           spare_memory[i] = 0;
+         }
+
+      /* Record the space now used.  When it decreases substantially,
+        we can refill the memory reserve.  */
 #if !defined SYSTEM_MALLOC && !defined SYNC_INPUT
-  bytes_used_when_full = BYTES_USED;
+      bytes_used_when_full = BYTES_USED;
 #endif
+    }

   /* This used to call error, but if we've run out of memory, we could
      get infinite recursion trying to build the string.  */

=== modified file 'src/buffer.c'
--- src/buffer.c        2011-05-12 07:07:06 +0000
+++ src/buffer.c        2011-05-30 16:12:20 +0000
@@ -328,7 +328,7 @@
   alloc_buffer_text (b, BUF_GAP_SIZE (b) + 1);
   UNBLOCK_INPUT;
   if (! BUF_BEG_ADDR (b))
-    buffer_memory_full ();
+    buffer_memory_full (BUF_GAP_SIZE (b) + 1);

   b->pt = BEG;
   b->begv = BEG;
@@ -4892,7 +4892,7 @@
   if (p == NULL)
     {
       UNBLOCK_INPUT;
-      memory_full ();
+      memory_full (nbytes);
     }

   b->text->beg = (unsigned char *) p;
@@ -4920,7 +4920,7 @@
   if (p == NULL)
     {
       UNBLOCK_INPUT;
-      memory_full ();
+      memory_full (nbytes);
     }

   BUF_BEG_ADDR (b) = (unsigned char *) p;

=== modified file 'src/editfns.c'
--- src/editfns.c       2011-05-27 19:37:32 +0000
+++ src/editfns.c       2011-05-30 16:12:20 +0000
@@ -3636,7 +3636,7 @@
   {
     EMACS_INT i;
     if ((SIZE_MAX - formatlen) / sizeof (struct info) <= nargs)
-      memory_full ();
+      memory_full (SIZE_MAX);
     SAFE_ALLOCA (info, struct info *, (nargs + 1) * sizeof *info + formatlen);
     discarded = (char *) &info[nargs + 1];
     for (i = 0; i < nargs + 1; i++)

=== modified file 'src/lisp.h'
--- src/lisp.h  2011-05-30 05:39:59 +0000
+++ src/lisp.h  2011-05-30 16:12:20 +0000
@@ -2685,8 +2685,8 @@
 extern void reset_malloc_hooks (void);
 extern void uninterrupt_malloc (void);
 extern void malloc_warning (const char *);
-extern void memory_full (void) NO_RETURN;
-extern void buffer_memory_full (void) NO_RETURN;
+extern void memory_full (size_t) NO_RETURN;
+extern void buffer_memory_full (EMACS_INT) NO_RETURN;
 extern int survives_gc_p (Lisp_Object);
 extern void mark_object (Lisp_Object);
 #if defined REL_ALLOC && !defined SYSTEM_MALLOC

=== modified file 'src/menu.c'
--- src/menu.c  2011-05-18 03:03:15 +0000
+++ src/menu.c  2011-05-30 16:12:20 +0000
@@ -178,7 +178,7 @@
 grow_menu_items (void)
 {
   if ((INT_MAX - MENU_ITEMS_PANE_LENGTH) / 2 < menu_items_allocated)
-    memory_full ();
+    memory_full (SIZE_MAX);
   menu_items_allocated *= 2;
   menu_items = larger_vector (menu_items, menu_items_allocated, Qnil);
 }

=== modified file 'src/minibuf.c'
--- src/minibuf.c       2011-05-12 07:07:06 +0000
+++ src/minibuf.c       2011-05-30 16:12:20 +0000
@@ -245,7 +245,7 @@
             len == size - 1 && line[len - 1] != '\n'))
     {
       if ((size_t) -1 / 2 < size)
-       memory_full ();
+       memory_full (SIZE_MAX);
       size *= 2;
       line = (char *) xrealloc (line, size);
     }

=== modified file 'src/xterm.c'
--- src/xterm.c 2011-05-27 16:17:59 +0000
+++ src/xterm.c 2011-05-30 16:12:20 +0000
@@ -4225,7 +4225,7 @@
       size_t old_nbytes = scroll_bar_windows_size * sizeof *scroll_bar_windows;

       if ((size_t) -1 / sizeof *scroll_bar_windows < new_size)
-       memory_full ();
+       memory_full (SIZE_MAX);
       scroll_bar_windows = (struct window **) xrealloc (scroll_bar_windows,
                                                        nbytes);
       memset (&scroll_bar_windows[i], 0, nbytes - old_nbytes);