1 /* isatty() replacement.
2 Copyright (C) 2012-2023 Free Software Foundation, Inc.
3
4 This file is free software: you can redistribute it and/or modify
5 it under the terms of the GNU Lesser General Public License as
6 published by the Free Software Foundation; either version 2.1 of the
7 License, or (at your option) any later version.
8
9 This file is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU Lesser General Public License for more details.
13
14 You should have received a copy of the GNU Lesser General Public License
15 along with this program. If not, see <https://www.gnu.org/licenses/>. */
16
17 #include <config.h>
18
19 /* Specification. */
20 #include <unistd.h>
21
22 /* This replacement is enabled on native Windows. */
23
24 #include <errno.h>
25 #include <string.h>
26
27 /* Get declarations of the Win32 API functions. */
28 #define WIN32_LEAN_AND_MEAN
29 #include <windows.h>
30
31 #if HAVE_MSVC_INVALID_PARAMETER_HANDLER
32 # include "msvc-inval.h"
33 #endif
34
35 /* Get _get_osfhandle(). */
36 #if GNULIB_MSVC_NOTHROW
37 # include "msvc-nothrow.h"
38 #else
39 # include <io.h>
40 #endif
41
42 /* Don't assume that UNICODE is not defined. */
43 #undef LoadLibrary
44 #define LoadLibrary LoadLibraryA
45 #undef QueryFullProcessImageName
46 #define QueryFullProcessImageName QueryFullProcessImageNameA
47
48 #if !(_WIN32_WINNT >= _WIN32_WINNT_VISTA)
49
50 /* Avoid warnings from gcc -Wcast-function-type. */
51 # define GetProcAddress \
52 (void *) GetProcAddress
53
54 /* GetNamedPipeClientProcessId was introduced only in Windows Vista. */
55 typedef BOOL (WINAPI * GetNamedPipeClientProcessIdFuncType) (HANDLE hPipe,
56 PULONG pClientProcessId);
57 static GetNamedPipeClientProcessIdFuncType GetNamedPipeClientProcessIdFunc = NULL;
58 /* QueryFullProcessImageName was introduced only in Windows Vista. */
59 typedef BOOL (WINAPI * QueryFullProcessImageNameFuncType) (HANDLE hProcess,
60 DWORD dwFlags,
61 LPSTR lpExeName,
62 PDWORD pdwSize);
63 static QueryFullProcessImageNameFuncType QueryFullProcessImageNameFunc = NULL;
64 static BOOL initialized = FALSE;
65
66 static void
67 initialize (void)
68 {
69 HMODULE kernel32 = LoadLibrary ("kernel32.dll");
70 if (kernel32 != NULL)
71 {
72 GetNamedPipeClientProcessIdFunc =
73 (GetNamedPipeClientProcessIdFuncType) GetProcAddress (kernel32, "GetNamedPipeClientProcessId");
74 QueryFullProcessImageNameFunc =
75 (QueryFullProcessImageNameFuncType) GetProcAddress (kernel32, "QueryFullProcessImageNameA");
76 }
77 initialized = TRUE;
78 }
79
80 #else
81
82 # define GetNamedPipeClientProcessIdFunc GetNamedPipeClientProcessId
83 # define QueryFullProcessImageNameFunc QueryFullProcessImageName
84
85 #endif
86
87 static BOOL IsConsoleHandle (HANDLE h)
88 {
89 DWORD mode;
90 /* GetConsoleMode
91 <https://docs.microsoft.com/en-us/windows/console/getconsolemode> */
92 return GetConsoleMode (h, &mode) != 0;
93 }
94
95 static BOOL IsCygwinConsoleHandle (HANDLE h)
96 {
97 /* A handle to a Cygwin console is in fact a named pipe whose client process
98 and server process is <CYGWIN_INSTALL_DIR>\bin\mintty.exe. */
99 BOOL result = FALSE;
100 ULONG processId;
101
102 #if !(_WIN32_WINNT >= _WIN32_WINNT_VISTA)
103 if (!initialized)
104 initialize ();
105 #endif
106
107 /* GetNamedPipeClientProcessId
108 <https://docs.microsoft.com/en-us/windows/desktop/api/winbase/nf-winbase-getnamedpipeclientprocessid>
109 It requires -D_WIN32_WINNT=_WIN32_WINNT_VISTA or higher. */
110 if (GetNamedPipeClientProcessIdFunc && QueryFullProcessImageNameFunc
111 && GetNamedPipeClientProcessIdFunc (h, &processId))
112 {
113 /* OpenProcess
114 <https://docs.microsoft.com/en-us/windows/desktop/api/processthreadsapi/nf-processthreadsapi-openprocess> */
115 HANDLE processHandle =
116 OpenProcess (PROCESS_QUERY_LIMITED_INFORMATION, FALSE, processId);
117 if (processHandle != NULL)
118 {
119 char buf[1024];
120 DWORD bufsize = sizeof (buf);
121 /* The file name can be determined through
122 GetProcessImageFileName
123 <https://docs.microsoft.com/en-us/windows/desktop/api/psapi/nf-psapi-getprocessimagefilenamea>
124 or
125 QueryFullProcessImageName
126 <https://docs.microsoft.com/en-us/windows/desktop/api/winbase/nf-winbase-queryfullprocessimagenamea>
127 The former returns a file name in non-standard notation (it starts
128 with '\Device\') and may require linking with psapi.dll.
129 The latter is better, but requires -D_WIN32_WINNT=_WIN32_WINNT_VISTA
130 or higher. */
131 if (QueryFullProcessImageNameFunc (processHandle, 0, buf, &bufsize))
132 {
133 if (strlen (buf) >= 11
134 && strcmp (buf + strlen (buf) - 11, "\\mintty.exe") == 0)
135 result = TRUE;
136 }
137 CloseHandle (processHandle);
138 }
139 }
140 return result;
141 }
142
143 #if HAVE_MSVC_INVALID_PARAMETER_HANDLER
144 static int
145 _isatty_nothrow (int fd)
146 {
147 int result;
148
149 TRY_MSVC_INVAL
150 {
151 result = _isatty (fd);
152 }
153 CATCH_MSVC_INVAL
154 {
155 result = 0;
156 }
157 DONE_MSVC_INVAL;
158
159 return result;
160 }
161 #else
162 # define _isatty_nothrow _isatty
163 #endif
164
165 /* Determine whether FD refers to a console device. Return 1 if yes.
166 Return 0 and set errno if no. (ptsname_r relies on the errno value.) */
167 int
168 isatty (int fd)
169 {
170 HANDLE h = (HANDLE) _get_osfhandle (fd);
171 if (h == INVALID_HANDLE_VALUE)
172 {
173 errno = EBADF;
174 return 0;
175 }
176 /* _isatty (fd) tests whether GetFileType of the handle is FILE_TYPE_CHAR.
177 But it does not set errno when it returns 0. */
178 if (_isatty_nothrow (fd))
179 {
180 if (IsConsoleHandle (h))
181 return 1;
182 }
183 if (IsCygwinConsoleHandle (h))
184 return 1;
185 errno = ENOTTY;
186 return 0;
187 }