[PATCH] popen-safer: test O_CLOEXEC at run-time.

[Paolo Bonzini]

This patch tests for the presence of O_CLOEXEC at run-time.

The technique I use allows to use only two system calls instead of
three on system where O_CLOEXEC does not work, at least after the
first call to open_noinherit.

The code can be moved to an emulation of open by doing something
like:

-#ifdef O_CLOEXEC
+#if NEED_O_CLOEXEC_EMULATION
   /* 0 = unknown, 1 = yes, -1 = no.  */
   static bool have_cloexec;
-  if (have_cloexec >= 0)
+  if (have_cloexec >= 0 && (flags & O_CLOEXEC))
     {
-      fd = open (name, flags | O_CLOEXEC);
+      fd = open (name, flags);
 ...
-  fd = open (name, flags);
-  if (0 <= fd && set_cloexec_flag (fd, true) != 0)
+  fd = open (name, flags & ~O_CLOEXEC);
+  if (0 <= fd && (flags & O_CLOEXEC) && set_cloexec_flag (fd, true) != 0)

(Actually when this code moves to open there will be a small corner
case in which it will use three system calls; that is, if the
first call to open will fail with EINVAL for unrelated reasons).

Ok?

Paolo

2009-08-21  Paolo Bonzini  <address@hidden>

        popen-safer: test O_CLOEXEC at run-time.
        * lib/popen-safer.c: Test O_CLOEXEC at run-time.
---
 ChangeLog         |    5 +++++
 lib/popen-safer.c |   22 ++++++++++++++++------
 2 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/lib/popen-safer.c b/lib/popen-safer.c
index 1905be5..a261b71 100644
--- a/lib/popen-safer.c
+++ b/lib/popen-safer.c
@@ -27,18 +27,28 @@
 
 #include "cloexec.h"
 
-#ifndef O_CLOEXEC
-# define O_CLOEXEC 0
-#endif
-
 /* Like open (name, flags | O_CLOEXEC), although not necessarily
    atomic.  FLAGS must not include O_CREAT.  */
 
 static int
 open_noinherit (char const *name, int flags)
 {
-  int fd = open (name, flags | O_CLOEXEC);
-  if (0 <= fd && !O_CLOEXEC && set_cloexec_flag (fd, true) != 0)
+  int fd;
+#ifdef O_CLOEXEC
+  /* 0 = unknown, 1 = yes, -1 = no.  */
+  static bool have_cloexec;
+  if (have_cloexec >= 0)
+    {
+      fd = open (name, flags | O_CLOEXEC);
+      if (have_cloexec == 0 && (0 <= fd || errno == EINVAL))
+       have_cloexec = (0 <= fd ? 1 : -1);
+      if (have_cloexec == 1)
+       return fd;
+    }
+#endif
+
+  fd = open (name, flags);
+  if (0 <= fd && set_cloexec_flag (fd, true) != 0)
     {
       int saved_errno = errno;
       close (fd);
-- 
1.6.2.5