[PATCH] xsetmode: new module

[Paul Eggert]

This is to fix a problem noted by Eric Blake.
Code was using xfreopen to change files to binary mode, but this
fails for stdout when in append mode.  Such code should use
xsetmode instead.
* NEWS: Document incompatible changes to binary-io module.
* lib/binary-io.c (__gl_setmode_check) [__DJGPP__ || __EMX__]:
New function.
* lib/binary-io.h (__gl_setmode): Rename from set_binary_mode.
(set_binary_mode): New function, which also checks for tty.
* lib/xsetmode.c, lib/xsetmode.h, modules/xsetmode: New files.
---
 ChangeLog        | 14 ++++++++++++++
 NEWS             |  4 ++++
 lib/binary-io.c  | 35 ++++++++++++++++++++++++++++++++++-
 lib/binary-io.h  | 30 ++++++++++++++++--------------
 lib/xsetmode.c   | 38 ++++++++++++++++++++++++++++++++++++++
 lib/xsetmode.h   | 45 +++++++++++++++++++++++++++++++++++++++++++++
 modules/xsetmode | 28 ++++++++++++++++++++++++++++
 7 files changed, 179 insertions(+), 15 deletions(-)
 create mode 100644 lib/xsetmode.c
 create mode 100644 lib/xsetmode.h
 create mode 100644 modules/xsetmode

diff --git a/ChangeLog b/ChangeLog
index 10e5240..756858d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,17 @@
+2017-02-15  Paul Eggert  <address@hidden>
+
+       xsetmode: new module
+       This is to fix a problem noted by Eric Blake.
+       Code was using xfreopen to change files to binary mode, but this
+       fails for stdout when in append mode.  Such code should use
+       xsetmode instead.
+       * NEWS: Document incompatible changes to binary-io module.
+       * lib/binary-io.c (__gl_setmode_check) [__DJGPP__ || __EMX__]:
+       New function.
+       * lib/binary-io.h (__gl_setmode): Rename from set_binary_mode.
+       (set_binary_mode): New function, which also checks for tty.
+       * lib/xsetmode.c, lib/xsetmode.h, modules/xsetmode: New files.
+
 2017-02-14  Paul Eggert  <address@hidden>
 
        headers: fix begin-end typos
diff --git a/NEWS b/NEWS
index 266275a..294aa22 100644
--- a/NEWS
+++ b/NEWS
@@ -42,6 +42,10 @@ User visible incompatible changes
 
 Date        Modules         Changes
 
+2017-02-15  binary-io       On MS-DOS and OS/2, set_binary_mode now fails
+                            on ttys, and sets errno == ENOTTY.  SET_BINARY
+                            has been removed; use set_binary_mode instead.
+
 2017-01-20  parse-datetime  The parse_datetime2 function now takes two
                             more arguments TZ and TZSTRING, for the
                             time zone and its name.
diff --git a/lib/binary-io.c b/lib/binary-io.c
index d828bcd..c721650 100644
--- a/lib/binary-io.c
+++ b/lib/binary-io.c
@@ -1,4 +1,37 @@
+/* Binary mode I/O.
+   Copyright 2017 Free Software Foundation, Inc.
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
+
 #include <config.h>
+
 #define BINARY_IO_INLINE _GL_EXTERN_INLINE
 #include "binary-io.h"
-typedef int dummy;
+
+#if defined __DJGPP__ || defined __EMX__
+# include <errno.h>
+# include <unistd.h>
+
+int
+__gl_setmode_check (int fd)
+{
+  if (isatty (fd))
+    {
+      errno = ENOTTY;
+      return -1;
+    }
+  else
+    return 0;
+}
+#endif
diff --git a/lib/binary-io.h b/lib/binary-io.h
index f766439..ae7690d 100644
--- a/lib/binary-io.h
+++ b/lib/binary-io.h
@@ -33,15 +33,12 @@ _GL_INLINE_HEADER_BEGIN
 # define BINARY_IO_INLINE _GL_INLINE
 #endif
 
-/* set_binary_mode (fd, mode)
-   sets the binary/text I/O mode of file descriptor fd to the given mode
-   (must be O_BINARY or O_TEXT) and returns the previous mode.  */
 #if O_BINARY
 # if defined __EMX__ || defined __DJGPP__ || defined __CYGWIN__
 #  include <io.h> /* declares setmode() */
-#  define set_binary_mode setmode
+#  define __gl_setmode setmode
 # else
-#  define set_binary_mode _setmode
+#  define __gl_setmode _setmode
 #  undef fileno
 #  define fileno _fileno
 # endif
@@ -50,7 +47,7 @@ _GL_INLINE_HEADER_BEGIN
   /* Use a function rather than a macro, to avoid gcc warnings
      "warning: statement with no effect".  */
 BINARY_IO_INLINE int
-set_binary_mode (int fd, int mode)
+__gl_setmode (int fd, int mode)
 {
   (void) fd;
   (void) mode;
@@ -58,18 +55,23 @@ set_binary_mode (int fd, int mode)
 }
 #endif
 
-/* SET_BINARY (fd);
-   changes the file descriptor fd to perform binary I/O.  */
+/* Set FD's mode to MODE.  Return the old mode if successful, -1
+   (setting errno) on failure.  */
+
 #if defined __DJGPP__ || defined __EMX__
-# include <unistd.h> /* declares isatty() */
-  /* Avoid putting stdin/stdout in binary mode if it is connected to
-     the console, because that would make it impossible for the user
-     to interrupt the program through Ctrl-C or Ctrl-Break.  */
-# define SET_BINARY(fd) ((void) (!isatty (fd) ? (set_binary_mode (fd, 
O_BINARY), 0) : 0))
+extern int __gl_setmode_check (int);
 #else
-# define SET_BINARY(fd) ((void) set_binary_mode (fd, O_BINARY))
+BINARY_IO_INLINE int
+__gl_setmode_check (int fd) { return 0; }
 #endif
 
+BINARY_IO_INLINE int
+set_binary_mode (int fd, int mode)
+{
+  int r = __gl_setmode_check (fd);
+  return r != 0 ? r : __gl_setmode (fd, mode);
+}
+
 _GL_INLINE_HEADER_END
 
 #endif /* _BINARY_H */
diff --git a/lib/xsetmode.c b/lib/xsetmode.c
new file mode 100644
index 0000000..628f4f0
--- /dev/null
+++ b/lib/xsetmode.c
@@ -0,0 +1,38 @@
+/* Binary mode I/O with checking
+   Copyright 2017 Free Software Foundation, Inc.
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
+
+#include <config.h>
+
+#define XSETMODE_INLINE _GL_EXTERN_INLINE
+#include "xsetmode.h"
+
+#include <errno.h>
+#include <error.h>
+#include <stdbool.h>
+#include "exitfail.h"
+#include "verify.h"
+
+#if O_BINARY
+
+_Noreturn void
+xsetmode_error (void)
+{
+  error (exit_failure, errno,
+         _("failed to set file descriptor text/binary mode"));
+  assume (false);
+}
+
+#endif
diff --git a/lib/xsetmode.h b/lib/xsetmode.h
new file mode 100644
index 0000000..c88d915
--- /dev/null
+++ b/lib/xsetmode.h
@@ -0,0 +1,45 @@
+/* Binary mode I/O with checking
+   Copyright 2017 Free Software Foundation, Inc.
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
+
+#ifndef _XSETMODE_H
+#define _XSETMODE_H
+
+#include "binary-io.h"
+
+#ifndef _GL_INLINE_HEADER_BEGIN
+ #error "Please include config.h first."
+#endif
+_GL_INLINE_HEADER_BEGIN
+#ifndef XSETMODE_INLINE
+# define XSETMODE_INLINE _GL_INLINE
+#endif
+
+#if O_BINARY
+extern _Noreturn void xsetmode_error (void);
+#else
+XSETMODE_INLINE void xsetmode_error (void) {}
+#endif
+
+XSETMODE_INLINE void
+xsetmode (int fd, int mode)
+{
+  if (set_binary_mode (fd, mode) < 0)
+    xsetmode_error ();
+}
+
+_GL_INLINE_HEADER_END
+
+#endif /* _XSETMODE_H */
diff --git a/modules/xsetmode b/modules/xsetmode
new file mode 100644
index 0000000..360a054
--- /dev/null
+++ b/modules/xsetmode
@@ -0,0 +1,28 @@
+Description:
+Checked Binary mode I/O.
+
+Files:
+lib/xsetmode.h
+lib/xsetmode.c
+
+Depends-on:
+binary-io
+error
+exitfail
+extern-inline
+stdbool
+verify
+
+configure.ac:
+
+Makefile.am:
+lib_SOURCES += xsetmode.h xsetmode.c
+
+Include:
+"xsetmode.h"
+
+License:
+LGPLv2+
+
+Maintainer:
+all
-- 
2.9.3