Re: portability of fopen and 'e' (O_CLOEXEC) flag

[Daiki Ueno]

Bruno Haible <bruno@clisp.org> writes:

>> Here are proposed patches for other modules. Does this look right?
>
> There were no objections. I pushed the changes.

Thank you for this.  I have rebased GnuTLS on top of it, but noticed a
strange test failures on Windows CI, which involve reading binary files
(OCSP response):
https://gitlab.com/gnutls/gnutls/-/jobs/569815031

It seems that the fopen module ignores a 'b' flag.  The attached patch
fixes the failures.

Regards,
-- 
Daiki Ueno

>From 17fbb2560a05e3006125f8793c8e814ef5baa847 Mon Sep 17 00:00:00 2001
From: Daiki Ueno <ueno@gnu.org>
Date: Thu, 28 May 2020 11:40:49 +0200
Subject: [PATCH] fopen-gnu: make 'b' flag can be used with 'e' on Windows

* lib/fopen.c (rpl_fopen): Pass O_BINARY to open, if a 'b' flag is
specified on Windows.
* tests/test-fopen-gnu.c (DATA): New define.
(main): Add test for reading binary files with an 'e' flag.
---
 ChangeLog              |  8 ++++++++
 lib/fopen.c            |  9 +++++++--
 tests/test-fopen-gnu.c | 17 +++++++++++++++++
 3 files changed, 32 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index c17b76b72..ea2716b2f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,11 @@
+2020-05-28  Daiki Ueno  <ueno@gnu.org>
+
+       fopen-gnu: make 'b' flag can be used with 'e' on Windows
+       * lib/fopen.c (rpl_fopen): Pass O_BINARY to open, if a 'b' flag is
+       specified on Windows.
+       * tests/test-fopen-gnu.c (DATA): New define.
+       (main): Add test for reading binary files with an 'e' flag.
+
 2020-05-27  Bruno Haible  <bruno@clisp.org>
 
        Don't assume that UNICODE is not defined.
diff --git a/lib/fopen.c b/lib/fopen.c
index 20065e4c6..f60c51f95 100644
--- a/lib/fopen.c
+++ b/lib/fopen.c
@@ -51,6 +51,7 @@ rpl_fopen (const char *filename, const char *mode)
   int open_flags_standard;
 #if GNULIB_FOPEN_GNU
   int open_flags_gnu;
+  int open_flags_other;
 # define BUF_SIZE 80
   char fdopen_mode_buf[BUF_SIZE + 1];
 #endif
@@ -66,6 +67,7 @@ rpl_fopen (const char *filename, const char *mode)
   open_flags_standard = 0;
 #if GNULIB_FOPEN_GNU
   open_flags_gnu = 0;
+  open_flags_other = 0;
 #endif
   {
     const char *p = mode;
@@ -101,6 +103,9 @@ rpl_fopen (const char *filename, const char *mode)
 #endif
             continue;
           case 'b':
+#if defined _WIN32 && ! defined __CYGWIN__
+           open_flags_other |= O_BINARY;
+#endif
 #if GNULIB_FOPEN_GNU
             if (q < fdopen_mode_buf + BUF_SIZE)
               *q++ = *p;
@@ -142,9 +147,9 @@ rpl_fopen (const char *filename, const char *mode)
 #endif
   }
 #if GNULIB_FOPEN_GNU
-  open_flags = open_flags_standard | open_flags_gnu;
+  open_flags = open_flags_standard | open_flags_other | open_flags_gnu;
 #else
-  open_flags = open_flags_standard;
+  open_flags = open_flags_standard | open_flags_other;
 #endif
 
 #if FOPEN_TRAILING_SLASH_BUG
diff --git a/tests/test-fopen-gnu.c b/tests/test-fopen-gnu.c
index cae40421a..eeb1712c7 100644
--- a/tests/test-fopen-gnu.c
+++ b/tests/test-fopen-gnu.c
@@ -29,15 +29,20 @@
 
 #define BASE "test-fopen-gnu.t"
 
+/* 0x1a is an EOF on Windows.  */
+#define DATA "abc\x1adef"
+
 int
 main (void)
 {
   FILE *f;
   int fd;
   int flags;
+  char buf[16];
 
   /* Remove anything from prior partial run.  */
   unlink (BASE "file");
+  unlink (BASE "binary");
 
   /* Create the file.  */
   f = fopen (BASE "file", "w");
@@ -64,8 +69,20 @@ main (void)
   ASSERT (f == NULL);
   ASSERT (errno == EEXIST);
 
+  /* Open a binary file and check that the 'e' mode doesn't interfere.  */
+  f = fopen (BASE "binary", "wbe");
+  ASSERT (f);
+  ASSERT (fwrite (DATA, 1, sizeof(DATA)-1, f) == sizeof(DATA)-1);
+  ASSERT (fclose (f) == 0);
+
+  f = fopen (BASE "binary", "rbe");
+  ASSERT (f);
+  ASSERT (fread (buf, 1, sizeof(buf), f) == sizeof(DATA)-1);
+  ASSERT (fclose (f) == 0);
+
   /* Cleanup.  */
   ASSERT (unlink (BASE "file") == 0);
+  ASSERT (unlink (BASE "binary") == 0);
 
   return 0;
 }
-- 
2.26.2