posix_spawn_file_actions_addclose on Solaris 11

[Bruno Haible]

On Solaris 11 2011-11, I'm seeing these test failures:

  test-posix_spawn_file_actions_addclose.c:43: assertion failed
  FAIL: test-posix_spawn_file_actions_addclose

  test-posix_spawn_file_actions_adddup2.c:43: assertion failed
  FAIL: test-posix_spawn_file_actions_adddup2

  test-posix_spawn_file_actions_addopen.c:49: assertion failed
  FAIL: test-posix_spawn_file_actions_addopen

The bug is that Solaris does not reject out-of-range file descriptors.
This patch adds a workaround for the first one. Similarly for the other two
functions.


2012-01-08  Bruno Haible  <address@hidden>

        posix_spawn_file_actions_addclose: Work around Solaris 11 2011-11 bug.
        * m4/spawn_h.m4 (gl_SPAWN_H_DEFAULTS): Initialize
        REPLACE_POSIX_SPAWN_FILE_ACTIONS_ADDCLOSE.
        * m4/posix_spawn.m4 (gl_POSIX_SPAWN_BODY): Define
        HAVE_WORKING_POSIX_SPAWN.
        (gl_FUNC_POSIX_SPAWN_FILE_ACTIONS_ADDCLOSE): New macro.
        * lib/spawn.in.h (posix_spawn_file_actions_addclose): Test
        REPLACE_POSIX_SPAWN_FILE_ACTIONS_ADDCLOSE, not REPLACE_POSIX_SPAWN.
        * lib/spawn_faction_addclose.c: Add workaround implementation if
        HAVE_WORKING_POSIX_SPAWN.
        * modules/spawn (Makefile): Substitute
        REPLACE_POSIX_SPAWN_FILE_ACTIONS_ADDCLOSE.
        * modules/posix_spawn_file_actions_addclose (configure.ac): Invoke
        gl_FUNC_POSIX_SPAWN_FILE_ACTIONS_ADDCLOSE. Test
        REPLACE_POSIX_SPAWN_FILE_ACTIONS_ADDCLOSE, not REPLACE_POSIX_SPAWN.
        (Depends-on): Update conditions.
        * doc/posix-functions/posix_spawn_file_actions_addclose.texi: Mention
        the Solaris 11 bug.

--- doc/posix-functions/posix_spawn_file_actions_addclose.texi.orig     Sun Jan 
 8 21:53:37 2012
+++ doc/posix-functions/posix_spawn_file_actions_addclose.texi  Sun Jan  8 
20:53:12 2012
@@ -11,6 +11,9 @@
 @item
 This function is missing on some platforms:
 MacOS X 10.4, FreeBSD 6.0, NetBSD 5.0, OpenBSD 3.8, Minix 3.1.8, AIX 5.1, 
HP-UX 11, IRIX 6.5, OSF/1 5.1, Solaris 9, Cygwin, mingw, MSVC 9, Interix 3.5, 
BeOS.
address@hidden
+This function does not reject a too large file descriptor on some platforms:
+Solaris 11 2011-11. 
 @end itemize
 
 Portability problems not fixed by Gnulib:
--- lib/spawn.in.h.orig Sun Jan  8 21:53:37 2012
+++ lib/spawn.in.h      Sun Jan  8 20:59:24 2012
@@ -813,7 +813,7 @@
 #if @GNULIB_POSIX_SPAWN_FILE_ACTIONS_ADDCLOSE@
 /* Add an action to FILE-ACTIONS which tells the implementation to call
    'close' for the given file descriptor during the 'spawn' call.  */
-# if @REPLACE_POSIX_SPAWN@
+# if @REPLACE_POSIX_SPAWN_FILE_ACTIONS_ADDCLOSE@
 #  if !(defined __cplusplus && defined GNULIB_NAMESPACE)
 #   define posix_spawn_file_actions_addclose 
rpl_posix_spawn_file_actions_addclose
 #  endif
--- lib/spawn_faction_addclose.c.orig   Sun Jan  8 21:53:38 2012
+++ lib/spawn_faction_addclose.c        Sun Jan  8 20:51:02 2012
@@ -26,34 +26,44 @@
 # define __sysconf(open_max) getdtablesize ()
 #endif
 
-#include "spawn_int.h"
+#if !HAVE_WORKING_POSIX_SPAWN
+# include "spawn_int.h"
+#endif
 
 /* Add an action to FILE-ACTIONS which tells the implementation to call
    'close' for the given file descriptor during the 'spawn' call.  */
 int
 posix_spawn_file_actions_addclose (posix_spawn_file_actions_t *file_actions,
                                    int fd)
+#undef posix_spawn_file_actions_addclose
 {
   int maxfd = __sysconf (_SC_OPEN_MAX);
-  struct __spawn_action *rec;
 
   /* Test for the validity of the file descriptor.  */
   if (fd < 0 || fd >= maxfd)
     return EBADF;
 
-  /* Allocate more memory if needed.  */
-  if (file_actions->_used == file_actions->_allocated
-      && __posix_spawn_file_actions_realloc (file_actions) != 0)
-    /* This can only mean we ran out of memory.  */
-    return ENOMEM;
-
-  /* Add the new value.  */
-  rec = &file_actions->_actions[file_actions->_used];
-  rec->tag = spawn_do_close;
-  rec->action.open_action.fd = fd;
+#if HAVE_WORKING_POSIX_SPAWN
+  return posix_spawn_file_actions_addclose (file_actions, fd);
+#else
+  {
+    struct __spawn_action *rec;
+
+    /* Allocate more memory if needed.  */
+    if (file_actions->_used == file_actions->_allocated
+        && __posix_spawn_file_actions_realloc (file_actions) != 0)
+      /* This can only mean we ran out of memory.  */
+      return ENOMEM;
+
+    /* Add the new value.  */
+    rec = &file_actions->_actions[file_actions->_used];
+    rec->tag = spawn_do_close;
+    rec->action.open_action.fd = fd;
 
-  /* Account for the new entry.  */
-  ++file_actions->_used;
+    /* Account for the new entry.  */
+    ++file_actions->_used;
 
-  return 0;
+    return 0;
+  }
+#endif
 }
--- m4/posix_spawn.m4.orig      Sun Jan  8 21:53:38 2012
+++ m4/posix_spawn.m4   Sun Jan  8 21:10:22 2012
@@ -1,4 +1,4 @@
-# posix_spawn.m4 serial 9
+# posix_spawn.m4 serial 10
 dnl Copyright (C) 2008-2012 Free Software Foundation, Inc.
 dnl This file is free software; the Free Software Foundation
 dnl gives unlimited permission to copy and/or distribute it,
@@ -36,6 +36,9 @@
     gl_POSIX_SPAWN_WORKS
     case "$gl_cv_func_posix_spawn_works" in
       *yes)
+        AC_DEFINE([HAVE_WORKING_POSIX_SPAWN], [1],
+          [Define if you have the posix_spawn and posix_spawnp functions and
+           they work.])
         dnl Assume that these functions are available if 
POSIX_SPAWN_SETSCHEDULER
         dnl evaluates to nonzero.
         dnl AC_CHECK_FUNCS_ONCE([posix_spawnattr_getschedpolicy])
@@ -397,3 +400,44 @@
   AC_CHECK_HEADERS([paths.h])
   AC_CHECK_FUNCS([confstr sched_setparam sched_setscheduler setegid seteuid 
vfork])
 ])
+
+AC_DEFUN([gl_FUNC_POSIX_SPAWN_FILE_ACTIONS_ADDCLOSE],
+[
+  AC_REQUIRE([gl_SPAWN_H_DEFAULTS])
+  AC_REQUIRE([AC_PROG_CC])
+  AC_REQUIRE([AC_CANONICAL_HOST]) dnl for cross-compiles
+  gl_POSIX_SPAWN
+  if test $REPLACE_POSIX_SPAWN = 1; then
+    REPLACE_POSIX_SPAWN_FILE_ACTIONS_ADDCLOSE=1
+  else
+    dnl On Solaris 11 2011-11, posix_spawn_file_actions_addclose succeeds even
+    dnl if the fd argument is out of range.
+    AC_CACHE_CHECK([whether posix_spawn_file_actions_addclose works],
+      [gl_cv_func_posix_spawn_file_actions_addclose_works],
+      [AC_RUN_IFELSE(
+         [AC_LANG_SOURCE([[
+#include <spawn.h>
+int main ()
+{
+  posix_spawn_file_actions_t actions;
+  if (posix_spawn_file_actions_init (&actions) != 0)
+    return 1;
+  if (posix_spawn_file_actions_addclose (&actions, 10000000) == 0)
+    return 2;
+  return 0;
+}]])],
+         [gl_cv_func_posix_spawn_file_actions_addclose_works=yes],
+         [gl_cv_func_posix_spawn_file_actions_addclose_works=no],
+         [# Guess no on Solaris, yes otherwise.
+          case "$host_os" in
+            solaris*) 
gl_cv_func_posix_spawn_file_actions_addclose_works="guessing no";;
+            *)        
gl_cv_func_posix_spawn_file_actions_addclose_works="guessing yes";;
+          esac
+         ])
+      ])
+    case "$gl_cv_func_posix_spawn_file_actions_addclose_works" in
+      *yes) ;;
+      *) REPLACE_POSIX_SPAWN_FILE_ACTIONS_ADDCLOSE=1 ;;
+    esac
+  fi
+])
--- m4/spawn_h.m4.orig  Sun Jan  8 21:53:38 2012
+++ m4/spawn_h.m4       Sun Jan  8 20:59:16 2012
@@ -1,4 +1,4 @@
-# spawn_h.m4 serial 15
+# spawn_h.m4 serial 16
 dnl Copyright (C) 2008-2012 Free Software Foundation, Inc.
 dnl This file is free software; the Free Software Foundation
 dnl gives unlimited permission to copy and/or distribute it,
@@ -108,4 +108,6 @@
   HAVE_POSIX_SPAWN_FILE_ACTIONS_T=1;
                              AC_SUBST([HAVE_POSIX_SPAWN_FILE_ACTIONS_T])
   REPLACE_POSIX_SPAWN=0;     AC_SUBST([REPLACE_POSIX_SPAWN])
+  REPLACE_POSIX_SPAWN_FILE_ACTIONS_ADDCLOSE=0;
+                             
AC_SUBST([REPLACE_POSIX_SPAWN_FILE_ACTIONS_ADDCLOSE])
 ])
--- modules/posix_spawn_file_actions_addclose.orig      Sun Jan  8 21:53:38 2012
+++ modules/posix_spawn_file_actions_addclose   Sun Jan  8 20:57:17 2012
@@ -9,12 +9,12 @@
 
 Depends-on:
 spawn
-getdtablesize                 [test $HAVE_POSIX_SPAWN = 0 || test 
$REPLACE_POSIX_SPAWN = 1]
-posix_spawn_file_actions_init [test $HAVE_POSIX_SPAWN = 0 || test 
$REPLACE_POSIX_SPAWN = 1]
+getdtablesize                 [test $HAVE_POSIX_SPAWN = 0 || test 
$REPLACE_POSIX_SPAWN_FILE_ACTIONS_ADDCLOSE = 1]
+posix_spawn_file_actions_init [test $HAVE_POSIX_SPAWN = 0 || test 
$REPLACE_POSIX_SPAWN_FILE_ACTIONS_ADDCLOSE = 1]
 
 configure.ac:
-gl_POSIX_SPAWN
-if test $HAVE_POSIX_SPAWN = 0 || test $REPLACE_POSIX_SPAWN = 1; then
+gl_FUNC_POSIX_SPAWN_FILE_ACTIONS_ADDCLOSE
+if test $HAVE_POSIX_SPAWN = 0 || test 
$REPLACE_POSIX_SPAWN_FILE_ACTIONS_ADDCLOSE = 1; then
   AC_LIBOBJ([spawn_faction_addclose])
 fi
 gl_SPAWN_MODULE_INDICATOR([posix_spawn_file_actions_addclose])
--- modules/spawn.orig  Sun Jan  8 21:53:38 2012
+++ modules/spawn       Sun Jan  8 20:58:54 2012
@@ -54,6 +54,7 @@
              -e 's|@''HAVE_POSIX_SPAWNATTR_T''@|$(HAVE_POSIX_SPAWNATTR_T)|g' \
              -e 
's|@''HAVE_POSIX_SPAWN_FILE_ACTIONS_T''@|$(HAVE_POSIX_SPAWN_FILE_ACTIONS_T)|g' \
              -e 's|@''REPLACE_POSIX_SPAWN''@|$(REPLACE_POSIX_SPAWN)|g' \
+             -e 
's|@''REPLACE_POSIX_SPAWN_FILE_ACTIONS_ADDCLOSE''@|$(REPLACE_POSIX_SPAWN_FILE_ACTIONS_ADDCLOSE)|g'
 \
              -e '/definitions of _GL_FUNCDECL_RPL/r $(CXXDEFS_H)' \
              -e '/definition of _GL_ARG_NONNULL/r $(ARG_NONNULL_H)' \
              -e '/definition of _GL_WARN_ON_USE/r $(WARN_ON_USE_H)' \