combining large xmalloc and setvbuf

[Sami Kerola]

Hello,

While adding a gnulib (git describe v0.0-7694-g0dc4348) to a project I
noticed that one of the commands did not work after malloc() calls
where replaced with xmalloc(). After a bit of debugging I were able to
write a small program which demonstrates what went wrong.

-- snip
#include "config.h"
#include <stdio.h>

#define PROBLEM_VALUE 131049

int main(int argc, char **argv)
{
        char *buf;
        FILE *f;
        int i;
        if (argc < 2)
                return 1;
        buf = xmalloc(PROBLEM_VALUE);
        f = fopen(argv[1], "r");
        setvbuf(f, buf, _IOFBF, PROBLEM_VALUE);
        while (!feof(f))
                fgetc(f);
        return 0;
}
-- snip

The command produces

execve("./test", ["./test", "Makefile"], [/* 48 vars */]) = 0
brk(0)                                  = 0x1ef6000
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=81365, ...}) = 0
mmap(NULL, 81365, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f8bb6eee000
close(3)                                = 0
open("/usr/lib/libm.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0V\0\0\0\0\0\0"...,
832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1022344, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7f8bb6eed000
mmap(NULL, 3117368, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
0) = 0x7f8bb69e7000
mprotect(0x7f8bb6ae0000, 2093056, PROT_NONE) = 0
mmap(0x7f8bb6cdf000, 8192, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xf8000) = 0x7f8bb6cdf000
close(3)                                = 0
open("/usr/lib/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0000\30\2\0\0\0\0\0"...,
832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=2010709, ...}) = 0
mmap(NULL, 3828848, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
0) = 0x7f8bb6640000
mprotect(0x7f8bb67dd000, 2097152, PROT_NONE) = 0
mmap(0x7f8bb69dd000, 24576, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x19d000) = 0x7f8bb69dd000
mmap(0x7f8bb69e3000, 15472, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f8bb69e3000
close(3)                                = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7f8bb6eec000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7f8bb6eeb000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7f8bb6eea000
arch_prctl(ARCH_SET_FS, 0x7f8bb6eeb700) = 0
mprotect(0x7f8bb69dd000, 16384, PROT_READ) = 0
mprotect(0x7f8bb6cdf000, 4096, PROT_READ) = 0
mprotect(0x7f8bb6f02000, 4096, PROT_READ) = 0
munmap(0x7f8bb6eee000, 81365)           = 0
mmap(NULL, 1003520, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS,
-1, 0) = 0x7f8bb6df5000
brk(0)                                  = 0x1ef6000
brk(0x1f17000)                          = 0x1f17000
open("Makefile", O_RDONLY)              = 3
read(3, 0xffffffffb6df5010, 1000000)    = -1 EFAULT (Bad address)
read(3, 0xffffffffb6df5010, 1000000)    = -1 EFAULT (Bad address)
read(3, 0xffffffffb6df5010, 1000000)    = -1 EFAULT (Bad address)
read(3, 0xffffffffb6df5010, 1000000)    = -1 EFAULT (Bad address)
read(3, 0xffffffffb6df5010, 1000000)    = -1 EFAULT (Bad address)
read(3, 0xffffffffb6df5010, 1000000)    = -1 EFAULT (Bad address)
[to infinity...]

The problem disappers if I remove setvbuf() call, or make xmalloc() to
malloc(). It does not seem to make any difference what is the size of
input file. Quite disturbingly the break point with the code I
debugged this phenomena is 135097. In case you have difficulties to
reproduce the issue please try with greater value, perhaps something
odd happens to others as well.

The glib version is fairly recent; Archlinux version 2.16.0-5. I did
not specify any optioons for gcc (4.7.2). For full config.log see url
below.

http://ut3.org/~kerolasa/config.log

And of course one of the options is that this is not a bug at all, but
my misexpectation. If so please point me to right direction where
correct usage of these functions is explained.

--
   Sami Kerola
   http://www.iki.fi/kerolasa/