[PATCH] dup2, fcntl, fcntl-h: port to AIX 7.1

[Paul Eggert]

This fixes some porting problems discovered when testing the latest
grep snapshot on AIX 7.1.  I don't think if fixes any bugs
in grep but it could be important for other applications.
* doc/posix-functions/dup2.texi:
* doc/posix-functions/fcntl.texi:
* doc/posix-headers/fcntl.texi:
Document AIX bugs.
* lib/fcntl.in.h (O_CLOEXEC, O_NOFOLLOW, O_TTY_INIT) [_AIX]:
Define to 0 if outside 'int' range.
* m4/dup2.m4 (gl_FUNC_DUP2):
* m4/fcntl.m4 (gl_FUNC_FCNTL):
Check for getdtablesize.  If it's available, test a value just
outside its range instead of testing 1000000.  When cross-compiling,
guess that AIX will fail this improved test.
---
 ChangeLog                      | 18 ++++++++++++++++++
 doc/posix-functions/dup2.texi  |  4 ++--
 doc/posix-functions/fcntl.texi |  4 ++--
 doc/posix-headers/fcntl.texi   |  5 +++++
 lib/fcntl.in.h                 | 16 ++++++++++++++++
 m4/dup2.m4                     | 12 ++++++++++--
 m4/fcntl.m4                    | 12 +++++++++++-
 7 files changed, 64 insertions(+), 7 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 30df129..0c8782b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,21 @@
+2014-05-31  Paul Eggert  <address@hidden>
+
+       dup2, fcntl, fcntl-h: port to AIX 7.1
+       This fixes some porting problems discovered when testing the latest
+       grep snapshot on AIX 7.1.  I don't think if fixes any bugs
+       in grep but it could be important for other applications.
+       * doc/posix-functions/dup2.texi:
+       * doc/posix-functions/fcntl.texi:
+       * doc/posix-headers/fcntl.texi:
+       Document AIX bugs.
+       * lib/fcntl.in.h (O_CLOEXEC, O_NOFOLLOW, O_TTY_INIT) [_AIX]:
+       Define to 0 if outside 'int' range.
+       * m4/dup2.m4 (gl_FUNC_DUP2):
+       * m4/fcntl.m4 (gl_FUNC_FCNTL):
+       Check for getdtablesize.  If it's available, test a value just
+       outside its range instead of testing 1000000.  When cross-compiling,
+       guess that AIX will fail this improved test.
+
 2014-05-30  Paul Eggert  <address@hidden>
 
        printf, config.rpath: Port to FreeBSD 10.
diff --git a/doc/posix-functions/dup2.texi b/doc/posix-functions/dup2.texi
index cab793f..6e501b5 100644
--- a/doc/posix-functions/dup2.texi
+++ b/doc/posix-functions/dup2.texi
@@ -39,9 +39,9 @@ Linux releases between July 2008 and May 2009 (versions 
2.6.27 to 2.6.29).
 
 @item
 This function returns @code{EMFILE} instead of @code{EBADF} for
-extremely large targets, which interferes with using
+large targets, which interferes with using
 @code{dup2(fd,fd)==fd)} as the minimal @code{EBADF} filter:
-FreeBSD 6.1, Cygwin 1.5.
+AIX 7.1, FreeBSD 6.1, Cygwin 1.5.
 @end itemize
 
 Portability problems fixed by Gnulib module @code{dup2-obsolete}:
diff --git a/doc/posix-functions/fcntl.texi b/doc/posix-functions/fcntl.texi
index 0de983e..dc9fd78 100644
--- a/doc/posix-functions/fcntl.texi
+++ b/doc/posix-functions/fcntl.texi
@@ -16,7 +16,7 @@ mingw, MSVC 9.
 This function does not support @code{F_DUPFD_CLOEXEC} on some
 platforms:
 glibc with Linux kernels before 2.6.24,
-Mac OS X 10.5, FreeBSD 6.0, NetBSD 5.0, OpenBSD 3.8, AIX 5.1, HP-UX 11,
+Mac OS X 10.5, FreeBSD 6.0, NetBSD 5.0, OpenBSD 3.8, AIX 7.1, HP-UX 11,
 IRIX 6.5, OSF/1 5.1, Solaris 11 2010-11, Cygwin 1.7.1, Interix 3.5,
 BeOS.
 Note that the gnulib replacement code is functional but not atomic.
@@ -24,7 +24,7 @@ Note that the gnulib replacement code is functional but not 
atomic.
 @item
 The @code{F_DUPFD} action of this function does not reject
 out-of-range targets properly on some platforms:
-Cygwin 1.5.x, Haiku.
+AIX 7.1, Cygwin 1.5.x, Haiku.
 
 @item
 The @code{F_DUPFD} action of this function mistakenly clears
diff --git a/doc/posix-headers/fcntl.texi b/doc/posix-headers/fcntl.texi
index 7366513..3d3ade9 100644
--- a/doc/posix-headers/fcntl.texi
+++ b/doc/posix-headers/fcntl.texi
@@ -48,6 +48,11 @@ portability to native Windows platforms) are defined on some 
platforms but
 not on others.
 
 @item
address@hidden, @samp{O_NOFOLLOW}, and @samp{O_TTY_INIT}
+are defined to values that are too large for an @code{int} on some platforms:
+AIX 7.1 with XL C 12.1.
+
address@hidden
 @samp{O_DIRECT}, @samp{O_IGNORE_CTTY}, @samp{O_NDELAY},
 @samp{O_NOATIME}, @samp{O_NOLINK}, @samp{O_NOLINKS}, and
 @samp{O_NOTRANS} (not specified by POSIX) are defined
diff --git a/lib/fcntl.in.h b/lib/fcntl.in.h
index 99f75e6..e23b4b2 100644
--- a/lib/fcntl.in.h
+++ b/lib/fcntl.in.h
@@ -186,6 +186,22 @@ _GL_WARN_ON_USE (openat, "openat is not portable - "
 
 /* Fix up the O_* macros.  */
 
+/* AIX 7.1 with XL C 12.1 defines O_CLOEXEC, O_NOFOLLOW, and O_TTY_INIT
+   to values outside 'int' range, so omit these misdefinitions.
+   But avoid namespace pollution on non-AIX systems.  */
+#ifdef _AIX
+# include <limits.h>
+# if defined O_CLOEXEC && ! (INT_MIN <= O_CLOEXEC && O_CLOEXEC <= INT_MAX)
+#  undef O_CLOEXEC
+# endif
+# if defined O_NOFOLLOW && ! (INT_MIN <= O_NOFOLLOW && O_NOFOLLOW <= INT_MAX)
+#  undef O_NOFOLLOW
+# endif
+# if defined O_TTY_INIT && ! (INT_MIN <= O_TTY_INIT && O_TTY_INIT <= INT_MAX)
+#  undef O_TTY_INIT
+# endif
+#endif
+
 #if !defined O_DIRECT && defined O_DIRECTIO
 /* Tru64 spells it 'O_DIRECTIO'.  */
 # define O_DIRECT O_DIRECTIO
diff --git a/m4/dup2.m4 b/m4/dup2.m4
index 89638a0..6498fc2 100644
--- a/m4/dup2.m4
+++ b/m4/dup2.m4
@@ -8,6 +8,7 @@ AC_DEFUN([gl_FUNC_DUP2],
 [
   AC_REQUIRE([gl_UNISTD_H_DEFAULTS])
   AC_REQUIRE([AC_CANONICAL_HOST])
+  AC_CHECK_FUNCS_ONCE([getdtablesize])
   m4_ifdef([gl_FUNC_DUP2_OBSOLETE], [
     AC_CHECK_FUNCS_ONCE([dup2])
     if test $ac_cv_func_dup2 = no; then
@@ -23,6 +24,11 @@ AC_DEFUN([gl_FUNC_DUP2],
 #include <fcntl.h>
 #include <errno.h>]],
            [int result = 0;
+#ifdef HAVE_GETDTABLESIZE
+            int bad_fd = getdtablesize ();
+#else
+            int bad_fd = 1000000;
+#endif
 #ifdef FD_CLOEXEC
             if (fcntl (1, F_SETFD, FD_CLOEXEC) == -1)
               result |= 1;
@@ -37,7 +43,7 @@ AC_DEFUN([gl_FUNC_DUP2],
             if (dup2 (0, 0) != -1)
               result |= 8;
             /* Many gnulib modules require POSIX conformance of EBADF.  */
-            if (dup2 (2, 1000000) == -1 && errno != EBADF)
+            if (dup2 (2, bad_fd) == -1 && errno != EBADF)
               result |= 16;
             /* Flush out some cygwin core dumps.  */
             if (dup2 (2, -1) != -1 || errno != EBADF)
@@ -56,7 +62,9 @@ AC_DEFUN([gl_FUNC_DUP2],
            linux*) # On linux between 2008-07-27 and 2009-05-11, dup2 of a
                    # closed fd may yield -EBADF instead of -1 / errno=EBADF.
              gl_cv_func_dup2_works="guessing no" ;;
-           freebsd*) # on FreeBSD 6.1, dup2(1,1000000) gives EMFILE, not EBADF.
+           aix* | freebsd*)
+                   # on AIX 7.1 and FreeBSD 6.1, dup2 (1,toobig) gives EMFILE,
+                   # not EBADF.
              gl_cv_func_dup2_works="guessing no" ;;
            haiku*) # on Haiku alpha 2, dup2(1, 1) resets FD_CLOEXEC.
              gl_cv_func_dup2_works="guessing no" ;;
diff --git a/m4/fcntl.m4 b/m4/fcntl.m4
index 4a2771f..f824beb 100644
--- a/m4/fcntl.m4
+++ b/m4/fcntl.m4
@@ -19,7 +19,7 @@ AC_DEFUN([gl_FUNC_FCNTL],
   AC_REQUIRE([gl_USE_SYSTEM_EXTENSIONS])
   AC_REQUIRE([gl_FCNTL_H_DEFAULTS])
   AC_REQUIRE([AC_CANONICAL_HOST])
-  AC_CHECK_FUNCS_ONCE([fcntl])
+  AC_CHECK_FUNCS_ONCE([fcntl getdtablesize])
   if test $ac_cv_func_fcntl = no; then
     gl_REPLACE_FCNTL
   else
@@ -28,11 +28,21 @@ AC_DEFUN([gl_FUNC_FCNTL],
     AC_CACHE_CHECK([whether fcntl handles F_DUPFD correctly],
       [gl_cv_func_fcntl_f_dupfd_works],
       [AC_RUN_IFELSE([AC_LANG_PROGRAM([[
+#ifdef HAVE_GETDTABLESIZE
+# include <unistd.h>
+#endif
 #include <fcntl.h>
 #include <errno.h>
 ]], [[int result = 0;
+#ifdef HAVE_GETDTABLESIZE
+      int bad_fd = getdtablesize ();
+#else
+      int bad_fd = 1000000;
+#endif
       if (fcntl (0, F_DUPFD, -1) != -1) result |= 1;
       if (errno != EINVAL) result |= 2;
+      if (fcntl (0, F_DUPFD, bad_fd) != -1) result |= 4;
+      if (errno != EINVAL) result |= 8;
       return result;
          ]])],
          [gl_cv_func_fcntl_f_dupfd_works=yes],
-- 
1.9.3