popen binary mode patch

[Bruce Korb]

This patch adds "binary mode" handling for popen() on platforms where
the native popen() will not.  This is a problem especially on
platforms where fopen() will ignore the 'b' but popen() will not.
Hence, the test for "does fopen() accept a 'b'" won't apply to popen().
GNU libc ought to just ignore the 'b', but it doesn't.

2010-02-21  Bruce Korb  <address@hidden>

        * m4/popen.m4: check to see if popen() can handle a "rb" mode string.
        * lib/popen.c (rpl_popen): strip out any 'b' from the mode string.
        * lib/popen-safer.c (popen_safer): ditto

--- lib/popen.c 2010-02-17 16:03:53.000000000 -0800
+++ lib/popen-new.c     2010-02-22 09:45:11.000000000 -0800
@@ -47,6 +47,30 @@ rpl_popen (const char *filename, const c
   int cloexec1 = fcntl (STDOUT_FILENO, F_GETFD);
   int saved_errno;
 
+#if ! BINARY_MODE_POPEN
+  char tmp_mode[8], *tmp_pz = tmp_mode;
+
+  for (;;)
+    {
+      int ch = *(mode++);
+      *tmp_pz = ch;
+      if (ch == '\0')
+        break;
+      if (ch != 'b') {
+        tmp_pz++;
+
+        /* Silently truncate overlong mode strings.  Let the popen() funciton
+           complain about the issue.  It will. */
+        if (tmp_pz == tmp_mode + sizeof (tmp_mode) - 1)
+          {
+            *tmp_pz = '\0';
+            break;
+          }
+      }
+    }
+  mode = tmp_mode;
+#endif
+
   /* If either stdin or stdout was closed (that is, fcntl failed),
      then we open a dummy close-on-exec fd to occupy that slot.  That
      way, popen's internal use of pipe() will not contain either fd 0
--- lib/popen-safer.c   2010-02-17 16:03:53.000000000 -0800
+++ lib/popen-safer-new.c       2010-02-22 09:47:01.000000000 -0800
@@ -73,6 +73,7 @@ popen_safer (char const *cmd, char const
      the original fd created by popen is safe.  */
   FILE *fp;
   int fd = open_noinherit ("/dev/null", O_RDONLY);
+
   if (0 <= fd && fd <= STDERR_FILENO)
     {
       /* Maximum recursion depth is 3.  */
@@ -84,6 +85,30 @@ popen_safer (char const *cmd, char const
     }
   else
     {
+#if ! BINARY_MODE_POPEN
+      char tmp_mode[8], *tmp_pz = tmp_mode;
+
+      for (;;)
+        {
+          int ch = *(mode++);
+          *tmp_pz = ch;
+          if (ch == '\0')
+            break;
+          if (ch != 'b') {
+            tmp_pz++;
+
+            /* Silently truncate overlong mode strings.  Let the popen()
+               funciton complain about the issue.  It will. */
+            if (tmp_pz == tmp_mode + sizeof (tmp_mode) - 1)
+              {
+                *tmp_pz = '\0';
+                break;
+              }
+          }
+        }
+      mode = tmp_mode;
+#endif
+
       /* Either all fd's are tied up, or fd is safe and the real popen
          will reuse it.  */
       close (fd);
--- m4/popen.m4 2010-02-17 16:03:53.000000000 -0800
+++ m4/popen-new.m4     2010-02-22 09:36:56.000000000 -0800
@@ -25,6 +25,12 @@ AC_DEFUN([gl_FUNC_POPEN],
     AC_LIBOBJ([popen])
     gl_PREREQ_POPEN
   fi
+  AC_RUN_IFELSE(
+    [AC_LANG_PROGRAM([], [dnl
+      FILE * fp = popen ("date", "rb");
+      exit (fp == NULL);])],
+    [AC_DEFINE([BINARY_MODE_POPEN], [1], [define if popen supports "rb" mode])]
+  )
 ])
 
 # Prerequisites of lib/popen.c.