[PATCH] ptsname: reject invalid file descriptors

[Eric Blake]

POSIX left errno undefined on ptsname() failure, although there
has at least been an effort to specify reasonable values to use:
http://www.austingroupbugs.net/view.php?id=503

However, our tests for ptsname and ptsname_r already require errno
to be set to useful values (as in glibc), so it is worth replacing
ptsname on FreeBSD 8.2 in order to get better QoI and pass the test.

* m4/ptsname.m4 (gl_FUNC_PTSNAME): Probe for FreeBSD bug.
* m4/stdlib_h.m4 (gl_STDLIB_H_DEFAULTS): Add new witness.
* modules/stdlib (Makefile.am): Replace witness.
* lib/stdlib.in.h (ptsname): Allow for replacement.
* modules/ptsname (configure.ac): Trigger replacement.
* doc/posix-functions/ptsname.texi (ptsname): Document this.
---

FreeBSD was passing test-ptsname_r test but failing test-ptsname,
without this patch.

 ChangeLog                        | 11 +++++++++++
 doc/posix-functions/ptsname.texi |  3 +++
 lib/stdlib.in.h                  | 13 +++++++++++--
 m4/ptsname.m4                    | 22 +++++++++++++++++++++-
 m4/stdlib_h.m4                   |  1 +
 modules/ptsname                  |  2 +-
 modules/stdlib                   |  1 +
 7 files changed, 49 insertions(+), 4 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 741c48d..c27db0d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,14 @@
+2012-10-02  Eric Blake  <address@hidden>
+
+       ptsname: reject invalid file descriptors
+       http://www.austingroupbugs.net/view.php?id=503
+       * m4/ptsname.m4 (gl_FUNC_PTSNAME): Probe for FreeBSD bug.
+       * m4/stdlib_h.m4 (gl_STDLIB_H_DEFAULTS): Add new witness.
+       * modules/stdlib (Makefile.am): Replace witness.
+       * lib/stdlib.in.h (ptsname): Allow for replacement.
+       * modules/ptsname (configure.ac): Trigger replacement.
+       * doc/posix-functions/ptsname.texi (ptsname): Document this.
+
 2012-10-02:  Nikos Mavrogiannopoulos  <address@hidden>  (tiny change)

        hash-pjw-bare: new module
diff --git a/doc/posix-functions/ptsname.texi b/doc/posix-functions/ptsname.texi
index c19ad4f..f145aed 100644
--- a/doc/posix-functions/ptsname.texi
+++ b/doc/posix-functions/ptsname.texi
@@ -11,6 +11,9 @@ ptsname
 @item
 This function is missing on some platforms:
 Mac OS X 10.3, OpenBSD 3.8, Minix 3.1.8, mingw, MSVC 9, BeOS.
address@hidden
+This function fails to set errno on failure on some platforms:
+FreeBSD 8.2.
 @end itemize

 Portability problems not fixed by Gnulib:
diff --git a/lib/stdlib.in.h b/lib/stdlib.in.h
index 1d67ec6..8311a28 100644
--- a/lib/stdlib.in.h
+++ b/lib/stdlib.in.h
@@ -457,10 +457,19 @@ _GL_WARN_ON_USE (posix_openpt, "posix_openpt is not 
portable - "
 #if @GNULIB_PTSNAME@
 /* Return the pathname of the pseudo-terminal slave associated with
    the master FD is open on, or NULL on errors.  */
-# if address@hidden@
+# if @REPLACE_PTSNAME@
+#  if !(defined __cplusplus && defined GNULIB_NAMESPCE)
+#   undef ptsname
+#   define ptsname rpl_ptsname
+#  endif
+_GL_FUNCDECL_RPL (ptsname, char *, (int fd));
+_GL_CXXALIAS_RPL (ptsname, char *, (int fd));
+# else
+#  if address@hidden@
 _GL_FUNCDECL_SYS (ptsname, char *, (int fd));
-# endif
+#  endif
 _GL_CXXALIAS_SYS (ptsname, char *, (int fd));
+# endif
 _GL_CXXALIASWARN (ptsname);
 #elif defined GNULIB_POSIXCHECK
 # undef ptsname
diff --git a/m4/ptsname.m4 b/m4/ptsname.m4
index ab105be..08c9c94 100644
--- a/m4/ptsname.m4
+++ b/m4/ptsname.m4
@@ -1,4 +1,4 @@
-# ptsname.m4 serial 2
+# ptsname.m4 serial 3
 dnl Copyright (C) 2010-2012 Free Software Foundation, Inc.
 dnl This file is free software; the Free Software Foundation
 dnl gives unlimited permission to copy and/or distribute it,
@@ -14,6 +14,26 @@ AC_DEFUN([gl_FUNC_PTSNAME],
   AC_CHECK_FUNCS([ptsname])
   if test $ac_cv_func_ptsname = no; then
     HAVE_PTSNAME=0
+  else
+    AC_CACHE_CHECK([whether ptsname sets errno on failure],
+      [gl_cv_func_ptsname_sets_errno],
+      [AC_RUN_IFELSE(
+         [AC_LANG_PROGRAM([[#include <errno.h>
+      ]], [[
+      return ptsname (-1) || !errno;
+           ]])],
+         [gl_cv_func_ptsname_sets_errno=yes],
+         [gl_cv_func_ptsname_sets_errno=no],
+         [case "$host_os" in
+                    # Guess yes on glibc systems.
+            *-gnu*) gl_cv_func_ptsname_sets_errno="guessing yes" ;;
+                    # If we don't know, assume the worst.
+            *)      gl_cv_func_ptsname_sets_errno="guessing no" ;;
+          esac
+         ])])
+    case $gl_cv_func_ptsname_sets_errno in
+      *no) REPLACE_PTSNAME=1 ;;
+    esac
   fi
 ])

diff --git a/m4/stdlib_h.m4 b/m4/stdlib_h.m4
index ab43728..9c69f2e 100644
--- a/m4/stdlib_h.m4
+++ b/m4/stdlib_h.m4
@@ -102,6 +102,7 @@ AC_DEFUN([gl_STDLIB_H_DEFAULTS],
   REPLACE_MALLOC=0;          AC_SUBST([REPLACE_MALLOC])
   REPLACE_MBTOWC=0;          AC_SUBST([REPLACE_MBTOWC])
   REPLACE_MKSTEMP=0;         AC_SUBST([REPLACE_MKSTEMP])
+  REPLACE_PTSNAME=0;         AC_SUBST([REPLACE_PTSNAME])
   REPLACE_PTSNAME_R=0;       AC_SUBST([REPLACE_PTSNAME_R])
   REPLACE_PUTENV=0;          AC_SUBST([REPLACE_PUTENV])
   REPLACE_RANDOM_R=0;        AC_SUBST([REPLACE_RANDOM_R])
diff --git a/modules/ptsname b/modules/ptsname
index f0ce256..2f67966 100644
--- a/modules/ptsname
+++ b/modules/ptsname
@@ -12,7 +12,7 @@ ptsname_r       [test $HAVE_PTSNAME = 0]

 configure.ac:
 gl_FUNC_PTSNAME
-if test $HAVE_PTSNAME = 0; then
+if test $HAVE_PTSNAME = 0 || test $REPLACE_PTSNAME = 1; then
   AC_LIBOBJ([ptsname])
   gl_PREREQ_PTSNAME
 fi
diff --git a/modules/stdlib b/modules/stdlib
index f967ef6..8164477 100644
--- a/modules/stdlib
+++ b/modules/stdlib
@@ -95,6 +95,7 @@ stdlib.h: stdlib.in.h $(top_builddir)/config.status 
$(CXXDEFS_H) \
              -e 's|@''REPLACE_MALLOC''@|$(REPLACE_MALLOC)|g' \
              -e 's|@''REPLACE_MBTOWC''@|$(REPLACE_MBTOWC)|g' \
              -e 's|@''REPLACE_MKSTEMP''@|$(REPLACE_MKSTEMP)|g' \
+             -e 's|@''REPLACE_PTSNAME''@|$(REPLACE_PTSNAME)|g' \
              -e 's|@''REPLACE_PTSNAME_R''@|$(REPLACE_PTSNAME_R)|g' \
              -e 's|@''REPLACE_PUTENV''@|$(REPLACE_PUTENV)|g' \
              -e 's|@''REPLACE_RANDOM_R''@|$(REPLACE_RANDOM_R)|g' \
-- 
1.7.11.4