fopen: Fix the trailing slash workaround

[Bruno Haible]

When I add a couple of more unit tests to the 'fopen' tests, I see a test
failure on Solaris 9:

$ ./test-fopen
../../gltests/test-fopen.h:74: assertion 'fopen ("./", "r+") == NULL' failed
Abort (core dumped)

This patch fixes it.


2020-05-24  Bruno Haible  <address@hidden>

        fopen: Fix the trailing slash workaround.
        * lib/fopen.c (rpl_fopen): Parse the mode string. Recognize "r+" as a
        write access. Pass the right flags to open().
        * tests/test-fopen.h (test_fopen): Add a few more tests on directories.

diff --git a/lib/fopen.c b/lib/fopen.c
index a3128fa..ad6511d 100644
--- a/lib/fopen.c
+++ b/lib/fopen.c
@@ -47,11 +47,47 @@ orig_fopen (const char *filename, const char *mode)
 FILE *
 rpl_fopen (const char *filename, const char *mode)
 {
+  int open_direction;
+  int open_flags_standard;
+
 #if defined _WIN32 && ! defined __CYGWIN__
   if (strcmp (filename, "/dev/null") == 0)
     filename = "NUL";
 #endif
 
+  /* Parse the mode.  */
+  open_direction = 0;
+  open_flags_standard = 0;
+  {
+    const char *m;
+
+    for (m = mode; *m != '\0'; m++)
+      {
+        switch (*m)
+          {
+          case 'r':
+            open_direction = O_RDONLY;
+            continue;
+          case 'w':
+            open_direction = O_WRONLY;
+            open_flags_standard |= O_CREAT | O_TRUNC;
+            continue;
+          case 'a':
+            open_direction = O_WRONLY;
+            open_flags_standard |= O_CREAT | O_APPEND;
+            continue;
+          case 'b':
+            continue;
+          case '+':
+            open_direction = O_RDWR;
+            continue;
+          default:
+            break;
+          }
+        break;
+      }
+  }
+
 #if FOPEN_TRAILING_SLASH_BUG
   /* Fail if the mode requires write access and the filename ends in a slash,
      as POSIX says such a filename must name a directory
@@ -74,13 +110,13 @@ rpl_fopen (const char *filename, const char *mode)
         struct stat statbuf;
         FILE *fp;
 
-        if (mode[0] == 'w' || mode[0] == 'a')
+        if (open_direction != O_RDONLY)
           {
             errno = EISDIR;
             return NULL;
           }
 
-        fd = open (filename, O_RDONLY);
+        fd = open (filename, open_direction | open_flags_standard);
         if (fd < 0)
           return NULL;
 
@@ -101,7 +137,7 @@ rpl_fopen (const char *filename, const char *mode)
         return fp;
       }
   }
-# endif
+#endif
 
   return orig_fopen (filename, mode);
 }
diff --git a/tests/test-fopen.h b/tests/test-fopen.h
index fb036af..875cdaf 100644
--- a/tests/test-fopen.h
+++ b/tests/test-fopen.h
@@ -47,6 +47,10 @@ test_fopen (void)
   ASSERT (fopen (BASE "file/", "r") == NULL);
   ASSERT (errno == ENOTDIR || errno == EISDIR || errno == EINVAL);
 
+  errno = 0;
+  ASSERT (fopen (BASE "file/", "r+") == NULL);
+  ASSERT (errno == ENOTDIR || errno == EISDIR || errno == EINVAL);
+
   /* Cannot create a directory.  */
   errno = 0;
   ASSERT (fopen ("nonexist.ent/", "w") == NULL);
@@ -58,6 +62,18 @@ test_fopen (void)
   ASSERT (fopen (".", "w") == NULL);
   ASSERT (errno == EISDIR || errno == EINVAL || errno == EACCES);
 
+  errno = 0;
+  ASSERT (fopen ("./", "w") == NULL);
+  ASSERT (errno == EISDIR || errno == EINVAL || errno == EACCES);
+
+  errno = 0;
+  ASSERT (fopen (".", "r+") == NULL);
+  ASSERT (errno == EISDIR || errno == EINVAL || errno == EACCES);
+
+  errno = 0;
+  ASSERT (fopen ("./", "r+") == NULL);
+  ASSERT (errno == EISDIR || errno == EINVAL || errno == EACCES);
+
   /* /dev/null must exist, and be writable.  */
   f = fopen ("/dev/null", "r");
   ASSERT (f);