From: Bruno Haible <bruno@clisp.org>
Date: Sat, 4 Feb 2012 18:13:34 +0000 (+0100)
Subject: isatty: Fix test failure of ptsname_r on native Windows.
X-Git-Tag: v0.1~1170
X-Git-Url: http://erislabs.org.uk/gitweb/?a=commitdiff_plain;h=5eb934dfd78a8ff086ffef87f5d4ec18e2d45cf7;p=gnulib.git

isatty: Fix test failure of ptsname_r on native Windows.

* lib/isatty.c (_isatty_nothrow): Upon exception, return 0, not -1,
and don't set errno.
(isatty): Test first whether fd is valid. Set errno when returning 0.
---

diff --git a/ChangeLog b/ChangeLog
index 8f0854396..d5b72cad7 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,12 @@
 2012-02-04  Bruno Haible  <bruno@clisp.org>
 
+	isatty: Fix test failure of ptsname_r on native Windows.
+	* lib/isatty.c (_isatty_nothrow): Upon exception, return 0, not -1,
+	and don't set errno.
+	(isatty): Test first whether fd is valid. Set errno when returning 0.
+
+2012-02-04  Bruno Haible  <bruno@clisp.org>
+
 	spawn-pipe tests: Fix a NULL program name in a diagnostic.
 	* tests/test-spawn-pipe-main.c: Include progname.h.
 	(main): Invoke set_program_name.
diff --git a/lib/isatty.c b/lib/isatty.c
index 424245358..2ecdd5494 100644
--- a/lib/isatty.c
+++ b/lib/isatty.c
@@ -48,8 +48,7 @@ _isatty_nothrow (int fd)
     }
   CATCH_MSVC_INVAL
     {
-      result = -1;
-      errno = EBADF;
+      result = 0;
     }
   DONE_MSVC_INVAL;
 
@@ -59,15 +58,24 @@ _isatty_nothrow (int fd)
 # define _isatty_nothrow _isatty
 #endif
 
+/* Determine whether FD refers to a console device.  Return 1 if yes.
+   Return 0 and set errno if no. (ptsname_r relies on the errno value.)  */
 int
 isatty (int fd)
 {
-  /* _isatty (fd) tests whether GetFileType of the handle is FILE_TYPE_CHAR.  */
+  HANDLE h = (HANDLE) _get_osfhandle (fd);
+  if (h == INVALID_HANDLE_VALUE)
+    {
+      errno = EBADF;
+      return 0;
+    }
+  /* _isatty (fd) tests whether GetFileType of the handle is FILE_TYPE_CHAR.
+     But it does not set errno when it returns 0.  */
   if (_isatty_nothrow (fd))
     {
-      HANDLE h = (HANDLE) _get_osfhandle (fd);
-      return IsConsoleHandle (h);
+      if (IsConsoleHandle (h))
+        return 1;
     }
-  else
-    return 0;
+  errno = ENOTTY;
+  return 0;
 }