1 #include "Python.h"
2 #include "pycore_fileutils.h" // fileutils definitions
3 #include "pycore_runtime.h" // _PyRuntime
4 #include "osdefs.h" // SEP
5 #include <locale.h>
6 #include <stdlib.h> // mbstowcs()
7
8 #ifdef MS_WINDOWS
9 # include <malloc.h>
10 # include <windows.h>
11 # include <pathcch.h> // PathCchCombineEx
12 extern int winerror_to_errno(int);
13 #endif
14
15 #ifdef HAVE_LANGINFO_H
16 #include <langinfo.h>
17 #endif
18
19 #ifdef HAVE_SYS_IOCTL_H
20 #include <sys/ioctl.h>
21 #endif
22
23 #ifdef HAVE_NON_UNICODE_WCHAR_T_REPRESENTATION
24 #include <iconv.h>
25 #endif
26
27 #ifdef HAVE_FCNTL_H
28 #include <fcntl.h>
29 #endif /* HAVE_FCNTL_H */
30
31 #ifdef O_CLOEXEC
32 /* Does open() support the O_CLOEXEC flag? Possible values:
33
34 -1: unknown
35 0: open() ignores O_CLOEXEC flag, ex: Linux kernel older than 2.6.23
36 1: open() supports O_CLOEXEC flag, close-on-exec is set
37
38 The flag is used by _Py_open(), _Py_open_noraise(), io.FileIO
39 and os.open(). */
40 int _Py_open_cloexec_works = -1;
41 #endif
42
43 // The value must be the same in unicodeobject.c.
44 #define MAX_UNICODE 0x10ffff
45
46 // mbstowcs() and mbrtowc() errors
47 static const size_t DECODE_ERROR = ((size_t)-1);
48 static const size_t INCOMPLETE_CHARACTER = (size_t)-2;
49
50
51 static int
52 get_surrogateescape(_Py_error_handler errors, int *surrogateescape)
53 {
54 switch (errors)
55 {
56 case _Py_ERROR_STRICT:
57 *surrogateescape = 0;
58 return 0;
59 case _Py_ERROR_SURROGATEESCAPE:
60 *surrogateescape = 1;
61 return 0;
62 default:
63 return -1;
64 }
65 }
66
67
68 PyObject *
69 _Py_device_encoding(int fd)
70 {
71 int valid;
72 Py_BEGIN_ALLOW_THREADS
73 _Py_BEGIN_SUPPRESS_IPH
74 valid = isatty(fd);
75 _Py_END_SUPPRESS_IPH
76 Py_END_ALLOW_THREADS
77 if (!valid)
78 Py_RETURN_NONE;
79
80 #if defined(MS_WINDOWS)
81 UINT cp;
82 if (fd == 0)
83 cp = GetConsoleCP();
84 else if (fd == 1 || fd == 2)
85 cp = GetConsoleOutputCP();
86 else
87 cp = 0;
88 /* GetConsoleCP() and GetConsoleOutputCP() return 0 if the application
89 has no console */
90 if (cp == 0) {
91 Py_RETURN_NONE;
92 }
93
94 return PyUnicode_FromFormat("cp%u", (unsigned int)cp);
95 #else
96 if (_PyRuntime.preconfig.utf8_mode) {
97 _Py_DECLARE_STR(utf_8, "utf-8");
98 return Py_NewRef(&_Py_STR(utf_8));
99 }
100 return _Py_GetLocaleEncodingObject();
101 #endif
102 }
103
104
105 static size_t
106 is_valid_wide_char(wchar_t ch)
107 {
108 #ifdef HAVE_NON_UNICODE_WCHAR_T_REPRESENTATION
109 /* Oracle Solaris doesn't use Unicode code points as wchar_t encoding
110 for non-Unicode locales, which makes values higher than MAX_UNICODE
111 possibly valid. */
112 return 1;
113 #endif
114 if (Py_UNICODE_IS_SURROGATE(ch)) {
115 // Reject lone surrogate characters
116 return 0;
117 }
118 if (ch > MAX_UNICODE) {
119 // bpo-35883: Reject characters outside [U+0000; U+10ffff] range.
120 // The glibc mbstowcs() UTF-8 decoder does not respect the RFC 3629,
121 // it creates characters outside the [U+0000; U+10ffff] range:
122 // https://sourceware.org/bugzilla/show_bug.cgi?id=2373
123 return 0;
124 }
125 return 1;
126 }
127
128
129 static size_t
130 _Py_mbstowcs(wchar_t *dest, const char *src, size_t n)
131 {
132 size_t count = mbstowcs(dest, src, n);
133 if (dest != NULL && count != DECODE_ERROR) {
134 for (size_t i=0; i < count; i++) {
135 wchar_t ch = dest[i];
136 if (!is_valid_wide_char(ch)) {
137 return DECODE_ERROR;
138 }
139 }
140 }
141 return count;
142 }
143
144
145 #ifdef HAVE_MBRTOWC
146 static size_t
147 _Py_mbrtowc(wchar_t *pwc, const char *str, size_t len, mbstate_t *pmbs)
148 {
149 assert(pwc != NULL);
150 size_t count = mbrtowc(pwc, str, len, pmbs);
151 if (count != 0 && count != DECODE_ERROR && count != INCOMPLETE_CHARACTER) {
152 if (!is_valid_wide_char(*pwc)) {
153 return DECODE_ERROR;
154 }
155 }
156 return count;
157 }
158 #endif
159
160
161 #if !defined(_Py_FORCE_UTF8_FS_ENCODING) && !defined(MS_WINDOWS)
162
163 #define USE_FORCE_ASCII
164
165 extern int _Py_normalize_encoding(const char *, char *, size_t);
166
167 /* Workaround FreeBSD and OpenIndiana locale encoding issue with the C locale
168 and POSIX locale. nl_langinfo(CODESET) announces an alias of the
169 ASCII encoding, whereas mbstowcs() and wcstombs() functions use the
170 ISO-8859-1 encoding. The problem is that os.fsencode() and os.fsdecode() use
171 locale.getpreferredencoding() codec. For example, if command line arguments
172 are decoded by mbstowcs() and encoded back by os.fsencode(), we get a
173 UnicodeEncodeError instead of retrieving the original byte string.
174
175 The workaround is enabled if setlocale(LC_CTYPE, NULL) returns "C",
176 nl_langinfo(CODESET) announces "ascii" (or an alias to ASCII), and at least
177 one byte in range 0x80-0xff can be decoded from the locale encoding. The
178 workaround is also enabled on error, for example if getting the locale
179 failed.
180
181 On HP-UX with the C locale or the POSIX locale, nl_langinfo(CODESET)
182 announces "roman8" but mbstowcs() uses Latin1 in practice. Force also the
183 ASCII encoding in this case.
184
185 Values of force_ascii:
186
187 1: the workaround is used: Py_EncodeLocale() uses
188 encode_ascii_surrogateescape() and Py_DecodeLocale() uses
189 decode_ascii()
190 0: the workaround is not used: Py_EncodeLocale() uses wcstombs() and
191 Py_DecodeLocale() uses mbstowcs()
192 -1: unknown, need to call check_force_ascii() to get the value
193 */
194 static int force_ascii = -1;
195
196 static int
197 check_force_ascii(void)
198 {
199 char *loc = setlocale(LC_CTYPE, NULL);
200 if (loc == NULL) {
201 goto error;
202 }
203 if (strcmp(loc, "C") != 0 && strcmp(loc, "POSIX") != 0) {
204 /* the LC_CTYPE locale is different than C and POSIX */
205 return 0;
206 }
207
208 #if defined(HAVE_LANGINFO_H) && defined(CODESET)
209 const char *codeset = nl_langinfo(CODESET);
210 if (!codeset || codeset[0] == '\0') {
211 /* CODESET is not set or empty */
212 goto error;
213 }
214
215 char encoding[20]; /* longest name: "iso_646.irv_1991\0" */
216 if (!_Py_normalize_encoding(codeset, encoding, sizeof(encoding))) {
217 goto error;
218 }
219
220 #ifdef __hpux
221 if (strcmp(encoding, "roman8") == 0) {
222 unsigned char ch;
223 wchar_t wch;
224 size_t res;
225
226 ch = (unsigned char)0xA7;
227 res = _Py_mbstowcs(&wch, (char*)&ch, 1);
228 if (res != DECODE_ERROR && wch == L'\xA7') {
229 /* On HP-UX with C locale or the POSIX locale,
230 nl_langinfo(CODESET) announces "roman8", whereas mbstowcs() uses
231 Latin1 encoding in practice. Force ASCII in this case.
232
233 Roman8 decodes 0xA7 to U+00CF. Latin1 decodes 0xA7 to U+00A7. */
234 return 1;
235 }
236 }
237 #else
238 const char* ascii_aliases[] = {
239 "ascii",
240 /* Aliases from Lib/encodings/aliases.py */
241 "646",
242 "ansi_x3.4_1968",
243 "ansi_x3.4_1986",
244 "ansi_x3_4_1968",
245 "cp367",
246 "csascii",
247 "ibm367",
248 "iso646_us",
249 "iso_646.irv_1991",
250 "iso_ir_6",
251 "us",
252 "us_ascii",
253 NULL
254 };
255
256 int is_ascii = 0;
257 for (const char **alias=ascii_aliases; *alias != NULL; alias++) {
258 if (strcmp(encoding, *alias) == 0) {
259 is_ascii = 1;
260 break;
261 }
262 }
263 if (!is_ascii) {
264 /* nl_langinfo(CODESET) is not "ascii" or an alias of ASCII */
265 return 0;
266 }
267
268 for (unsigned int i=0x80; i<=0xff; i++) {
269 char ch[1];
270 wchar_t wch[1];
271 size_t res;
272
273 unsigned uch = (unsigned char)i;
274 ch[0] = (char)uch;
275 res = _Py_mbstowcs(wch, ch, 1);
276 if (res != DECODE_ERROR) {
277 /* decoding a non-ASCII character from the locale encoding succeed:
278 the locale encoding is not ASCII, force ASCII */
279 return 1;
280 }
281 }
282 /* None of the bytes in the range 0x80-0xff can be decoded from the locale
283 encoding: the locale encoding is really ASCII */
284 #endif /* !defined(__hpux) */
285 return 0;
286 #else
287 /* nl_langinfo(CODESET) is not available: always force ASCII */
288 return 1;
289 #endif /* defined(HAVE_LANGINFO_H) && defined(CODESET) */
290
291 error:
292 /* if an error occurred, force the ASCII encoding */
293 return 1;
294 }
295
296
297 int
298 _Py_GetForceASCII(void)
299 {
300 if (force_ascii == -1) {
301 force_ascii = check_force_ascii();
302 }
303 return force_ascii;
304 }
305
306
307 void
308 _Py_ResetForceASCII(void)
309 {
310 force_ascii = -1;
311 }
312
313
314 static int
315 encode_ascii(const wchar_t *text, char **str,
316 size_t *error_pos, const char **reason,
317 int raw_malloc, _Py_error_handler errors)
318 {
319 char *result = NULL, *out;
320 size_t len, i;
321 wchar_t ch;
322
323 int surrogateescape;
324 if (get_surrogateescape(errors, &surrogateescape) < 0) {
325 return -3;
326 }
327
328 len = wcslen(text);
329
330 /* +1 for NULL byte */
331 if (raw_malloc) {
332 result = PyMem_RawMalloc(len + 1);
333 }
334 else {
335 result = PyMem_Malloc(len + 1);
336 }
337 if (result == NULL) {
338 return -1;
339 }
340
341 out = result;
342 for (i=0; i<len; i++) {
343 ch = text[i];
344
345 if (ch <= 0x7f) {
346 /* ASCII character */
347 *out++ = (char)ch;
348 }
349 else if (surrogateescape && 0xdc80 <= ch && ch <= 0xdcff) {
350 /* UTF-8b surrogate */
351 *out++ = (char)(ch - 0xdc00);
352 }
353 else {
354 if (raw_malloc) {
355 PyMem_RawFree(result);
356 }
357 else {
358 PyMem_Free(result);
359 }
360 if (error_pos != NULL) {
361 *error_pos = i;
362 }
363 if (reason) {
364 *reason = "encoding error";
365 }
366 return -2;
367 }
368 }
369 *out = '\0';
370 *str = result;
371 return 0;
372 }
373 #else
374 int
375 _Py_GetForceASCII(void)
376 {
377 return 0;
378 }
379
380 void
381 _Py_ResetForceASCII(void)
382 {
383 /* nothing to do */
384 }
385 #endif /* !defined(_Py_FORCE_UTF8_FS_ENCODING) && !defined(MS_WINDOWS) */
386
387
388 #if !defined(HAVE_MBRTOWC) || defined(USE_FORCE_ASCII)
389 static int
390 decode_ascii(const char *arg, wchar_t **wstr, size_t *wlen,
391 const char **reason, _Py_error_handler errors)
392 {
393 wchar_t *res;
394 unsigned char *in;
395 wchar_t *out;
396 size_t argsize = strlen(arg) + 1;
397
398 int surrogateescape;
399 if (get_surrogateescape(errors, &surrogateescape) < 0) {
400 return -3;
401 }
402
403 if (argsize > PY_SSIZE_T_MAX / sizeof(wchar_t)) {
404 return -1;
405 }
406 res = PyMem_RawMalloc(argsize * sizeof(wchar_t));
407 if (!res) {
408 return -1;
409 }
410
411 out = res;
412 for (in = (unsigned char*)arg; *in; in++) {
413 unsigned char ch = *in;
414 if (ch < 128) {
415 *out++ = ch;
416 }
417 else {
418 if (!surrogateescape) {
419 PyMem_RawFree(res);
420 if (wlen) {
421 *wlen = in - (unsigned char*)arg;
422 }
423 if (reason) {
424 *reason = "decoding error";
425 }
426 return -2;
427 }
428 *out++ = 0xdc00 + ch;
429 }
430 }
431 *out = 0;
432
433 if (wlen != NULL) {
434 *wlen = out - res;
435 }
436 *wstr = res;
437 return 0;
438 }
439 #endif /* !HAVE_MBRTOWC */
440
441 static int
442 decode_current_locale(const char* arg, wchar_t **wstr, size_t *wlen,
443 const char **reason, _Py_error_handler errors)
444 {
445 wchar_t *res;
446 size_t argsize;
447 size_t count;
448 #ifdef HAVE_MBRTOWC
449 unsigned char *in;
450 wchar_t *out;
451 mbstate_t mbs;
452 #endif
453
454 int surrogateescape;
455 if (get_surrogateescape(errors, &surrogateescape) < 0) {
456 return -3;
457 }
458
459 #ifdef HAVE_BROKEN_MBSTOWCS
460 /* Some platforms have a broken implementation of
461 * mbstowcs which does not count the characters that
462 * would result from conversion. Use an upper bound.
463 */
464 argsize = strlen(arg);
465 #else
466 argsize = _Py_mbstowcs(NULL, arg, 0);
467 #endif
468 if (argsize != DECODE_ERROR) {
469 if (argsize > PY_SSIZE_T_MAX / sizeof(wchar_t) - 1) {
470 return -1;
471 }
472 res = (wchar_t *)PyMem_RawMalloc((argsize + 1) * sizeof(wchar_t));
473 if (!res) {
474 return -1;
475 }
476
477 count = _Py_mbstowcs(res, arg, argsize + 1);
478 if (count != DECODE_ERROR) {
479 *wstr = res;
480 if (wlen != NULL) {
481 *wlen = count;
482 }
483 return 0;
484 }
485 PyMem_RawFree(res);
486 }
487
488 /* Conversion failed. Fall back to escaping with surrogateescape. */
489 #ifdef HAVE_MBRTOWC
490 /* Try conversion with mbrtwoc (C99), and escape non-decodable bytes. */
491
492 /* Overallocate; as multi-byte characters are in the argument, the
493 actual output could use less memory. */
494 argsize = strlen(arg) + 1;
495 if (argsize > PY_SSIZE_T_MAX / sizeof(wchar_t)) {
496 return -1;
497 }
498 res = (wchar_t*)PyMem_RawMalloc(argsize * sizeof(wchar_t));
499 if (!res) {
500 return -1;
501 }
502
503 in = (unsigned char*)arg;
504 out = res;
505 memset(&mbs, 0, sizeof mbs);
506 while (argsize) {
507 size_t converted = _Py_mbrtowc(out, (char*)in, argsize, &mbs);
508 if (converted == 0) {
509 /* Reached end of string; null char stored. */
510 break;
511 }
512
513 if (converted == INCOMPLETE_CHARACTER) {
514 /* Incomplete character. This should never happen,
515 since we provide everything that we have -
516 unless there is a bug in the C library, or I
517 misunderstood how mbrtowc works. */
518 goto decode_error;
519 }
520
521 if (converted == DECODE_ERROR) {
522 if (!surrogateescape) {
523 goto decode_error;
524 }
525
526 /* Decoding error. Escape as UTF-8b, and start over in the initial
527 shift state. */
528 *out++ = 0xdc00 + *in++;
529 argsize--;
530 memset(&mbs, 0, sizeof mbs);
531 continue;
532 }
533
534 // _Py_mbrtowc() reject lone surrogate characters
535 assert(!Py_UNICODE_IS_SURROGATE(*out));
536
537 /* successfully converted some bytes */
538 in += converted;
539 argsize -= converted;
540 out++;
541 }
542 if (wlen != NULL) {
543 *wlen = out - res;
544 }
545 *wstr = res;
546 return 0;
547
548 decode_error:
549 PyMem_RawFree(res);
550 if (wlen) {
551 *wlen = in - (unsigned char*)arg;
552 }
553 if (reason) {
554 *reason = "decoding error";
555 }
556 return -2;
557 #else /* HAVE_MBRTOWC */
558 /* Cannot use C locale for escaping; manually escape as if charset
559 is ASCII (i.e. escape all bytes > 128. This will still roundtrip
560 correctly in the locale's charset, which must be an ASCII superset. */
561 return decode_ascii(arg, wstr, wlen, reason, errors);
562 #endif /* HAVE_MBRTOWC */
563 }
564
565
566 /* Decode a byte string from the locale encoding.
567
568 Use the strict error handler if 'surrogateescape' is zero. Use the
569 surrogateescape error handler if 'surrogateescape' is non-zero: undecodable
570 bytes are decoded as characters in range U+DC80..U+DCFF. If a byte sequence
571 can be decoded as a surrogate character, escape the bytes using the
572 surrogateescape error handler instead of decoding them.
573
574 On success, return 0 and write the newly allocated wide character string into
575 *wstr (use PyMem_RawFree() to free the memory). If wlen is not NULL, write
576 the number of wide characters excluding the null character into *wlen.
577
578 On memory allocation failure, return -1.
579
580 On decoding error, return -2. If wlen is not NULL, write the start of
581 invalid byte sequence in the input string into *wlen. If reason is not NULL,
582 write the decoding error message into *reason.
583
584 Return -3 if the error handler 'errors' is not supported.
585
586 Use the Py_EncodeLocaleEx() function to encode the character string back to
587 a byte string. */
588 int
589 _Py_DecodeLocaleEx(const char* arg, wchar_t **wstr, size_t *wlen,
590 const char **reason,
591 int current_locale, _Py_error_handler errors)
592 {
593 if (current_locale) {
594 #ifdef _Py_FORCE_UTF8_LOCALE
595 return _Py_DecodeUTF8Ex(arg, strlen(arg), wstr, wlen, reason,
596 errors);
597 #else
598 return decode_current_locale(arg, wstr, wlen, reason, errors);
599 #endif
600 }
601
602 #ifdef _Py_FORCE_UTF8_FS_ENCODING
603 return _Py_DecodeUTF8Ex(arg, strlen(arg), wstr, wlen, reason,
604 errors);
605 #else
606 int use_utf8 = (Py_UTF8Mode == 1);
607 #ifdef MS_WINDOWS
608 use_utf8 |= !Py_LegacyWindowsFSEncodingFlag;
609 #endif
610 if (use_utf8) {
611 return _Py_DecodeUTF8Ex(arg, strlen(arg), wstr, wlen, reason,
612 errors);
613 }
614
615 #ifdef USE_FORCE_ASCII
616 if (force_ascii == -1) {
617 force_ascii = check_force_ascii();
618 }
619
620 if (force_ascii) {
621 /* force ASCII encoding to workaround mbstowcs() issue */
622 return decode_ascii(arg, wstr, wlen, reason, errors);
623 }
624 #endif
625
626 return decode_current_locale(arg, wstr, wlen, reason, errors);
627 #endif /* !_Py_FORCE_UTF8_FS_ENCODING */
628 }
629
630
631 /* Decode a byte string from the locale encoding with the
632 surrogateescape error handler: undecodable bytes are decoded as characters
633 in range U+DC80..U+DCFF. If a byte sequence can be decoded as a surrogate
634 character, escape the bytes using the surrogateescape error handler instead
635 of decoding them.
636
637 Return a pointer to a newly allocated wide character string, use
638 PyMem_RawFree() to free the memory. If size is not NULL, write the number of
639 wide characters excluding the null character into *size
640
641 Return NULL on decoding error or memory allocation error. If *size* is not
642 NULL, *size is set to (size_t)-1 on memory error or set to (size_t)-2 on
643 decoding error.
644
645 Decoding errors should never happen, unless there is a bug in the C
646 library.
647
648 Use the Py_EncodeLocale() function to encode the character string back to a
649 byte string. */
650 wchar_t*
651 Py_DecodeLocale(const char* arg, size_t *wlen)
652 {
653 wchar_t *wstr;
654 int res = _Py_DecodeLocaleEx(arg, &wstr, wlen,
655 NULL, 0,
656 _Py_ERROR_SURROGATEESCAPE);
657 if (res != 0) {
658 assert(res != -3);
659 if (wlen != NULL) {
660 *wlen = (size_t)res;
661 }
662 return NULL;
663 }
664 return wstr;
665 }
666
667
668 static int
669 encode_current_locale(const wchar_t *text, char **str,
670 size_t *error_pos, const char **reason,
671 int raw_malloc, _Py_error_handler errors)
672 {
673 const size_t len = wcslen(text);
674 char *result = NULL, *bytes = NULL;
675 size_t i, size, converted;
676 wchar_t c, buf[2];
677
678 int surrogateescape;
679 if (get_surrogateescape(errors, &surrogateescape) < 0) {
680 return -3;
681 }
682
683 /* The function works in two steps:
684 1. compute the length of the output buffer in bytes (size)
685 2. outputs the bytes */
686 size = 0;
687 buf[1] = 0;
688 while (1) {
689 for (i=0; i < len; i++) {
690 c = text[i];
691 if (c >= 0xdc80 && c <= 0xdcff) {
692 if (!surrogateescape) {
693 goto encode_error;
694 }
695 /* UTF-8b surrogate */
696 if (bytes != NULL) {
697 *bytes++ = c - 0xdc00;
698 size--;
699 }
700 else {
701 size++;
702 }
703 continue;
704 }
705 else {
706 buf[0] = c;
707 if (bytes != NULL) {
708 converted = wcstombs(bytes, buf, size);
709 }
710 else {
711 converted = wcstombs(NULL, buf, 0);
712 }
713 if (converted == DECODE_ERROR) {
714 goto encode_error;
715 }
716 if (bytes != NULL) {
717 bytes += converted;
718 size -= converted;
719 }
720 else {
721 size += converted;
722 }
723 }
724 }
725 if (result != NULL) {
726 *bytes = '\0';
727 break;
728 }
729
730 size += 1; /* nul byte at the end */
731 if (raw_malloc) {
732 result = PyMem_RawMalloc(size);
733 }
734 else {
735 result = PyMem_Malloc(size);
736 }
737 if (result == NULL) {
738 return -1;
739 }
740 bytes = result;
741 }
742 *str = result;
743 return 0;
744
745 encode_error:
746 if (raw_malloc) {
747 PyMem_RawFree(result);
748 }
749 else {
750 PyMem_Free(result);
751 }
752 if (error_pos != NULL) {
753 *error_pos = i;
754 }
755 if (reason) {
756 *reason = "encoding error";
757 }
758 return -2;
759 }
760
761
762 /* Encode a string to the locale encoding.
763
764 Parameters:
765
766 * raw_malloc: if non-zero, allocate memory using PyMem_RawMalloc() instead
767 of PyMem_Malloc().
768 * current_locale: if non-zero, use the current LC_CTYPE, otherwise use
769 Python filesystem encoding.
770 * errors: error handler like "strict" or "surrogateescape".
771
772 Return value:
773
774 0: success, *str is set to a newly allocated decoded string.
775 -1: memory allocation failure
776 -2: encoding error, set *error_pos and *reason (if set).
777 -3: the error handler 'errors' is not supported.
778 */
779 static int
780 encode_locale_ex(const wchar_t *text, char **str, size_t *error_pos,
781 const char **reason,
782 int raw_malloc, int current_locale, _Py_error_handler errors)
783 {
784 if (current_locale) {
785 #ifdef _Py_FORCE_UTF8_LOCALE
786 return _Py_EncodeUTF8Ex(text, str, error_pos, reason,
787 raw_malloc, errors);
788 #else
789 return encode_current_locale(text, str, error_pos, reason,
790 raw_malloc, errors);
791 #endif
792 }
793
794 #ifdef _Py_FORCE_UTF8_FS_ENCODING
795 return _Py_EncodeUTF8Ex(text, str, error_pos, reason,
796 raw_malloc, errors);
797 #else
798 int use_utf8 = (Py_UTF8Mode == 1);
799 #ifdef MS_WINDOWS
800 use_utf8 |= !Py_LegacyWindowsFSEncodingFlag;
801 #endif
802 if (use_utf8) {
803 return _Py_EncodeUTF8Ex(text, str, error_pos, reason,
804 raw_malloc, errors);
805 }
806
807 #ifdef USE_FORCE_ASCII
808 if (force_ascii == -1) {
809 force_ascii = check_force_ascii();
810 }
811
812 if (force_ascii) {
813 return encode_ascii(text, str, error_pos, reason,
814 raw_malloc, errors);
815 }
816 #endif
817
818 return encode_current_locale(text, str, error_pos, reason,
819 raw_malloc, errors);
820 #endif /* _Py_FORCE_UTF8_FS_ENCODING */
821 }
822
823 static char*
824 encode_locale(const wchar_t *text, size_t *error_pos,
825 int raw_malloc, int current_locale)
826 {
827 char *str;
828 int res = encode_locale_ex(text, &str, error_pos, NULL,
829 raw_malloc, current_locale,
830 _Py_ERROR_SURROGATEESCAPE);
831 if (res != -2 && error_pos) {
832 *error_pos = (size_t)-1;
833 }
834 if (res != 0) {
835 return NULL;
836 }
837 return str;
838 }
839
840 /* Encode a wide character string to the locale encoding with the
841 surrogateescape error handler: surrogate characters in the range
842 U+DC80..U+DCFF are converted to bytes 0x80..0xFF.
843
844 Return a pointer to a newly allocated byte string, use PyMem_Free() to free
845 the memory. Return NULL on encoding or memory allocation error.
846
847 If error_pos is not NULL, *error_pos is set to (size_t)-1 on success, or set
848 to the index of the invalid character on encoding error.
849
850 Use the Py_DecodeLocale() function to decode the bytes string back to a wide
851 character string. */
852 char*
853 Py_EncodeLocale(const wchar_t *text, size_t *error_pos)
854 {
855 return encode_locale(text, error_pos, 0, 0);
856 }
857
858
859 /* Similar to Py_EncodeLocale(), but result must be freed by PyMem_RawFree()
860 instead of PyMem_Free(). */
861 char*
862 _Py_EncodeLocaleRaw(const wchar_t *text, size_t *error_pos)
863 {
864 return encode_locale(text, error_pos, 1, 0);
865 }
866
867
868 int
869 _Py_EncodeLocaleEx(const wchar_t *text, char **str,
870 size_t *error_pos, const char **reason,
871 int current_locale, _Py_error_handler errors)
872 {
873 return encode_locale_ex(text, str, error_pos, reason, 1,
874 current_locale, errors);
875 }
876
877
878 // Get the current locale encoding name:
879 //
880 // - Return "utf-8" if _Py_FORCE_UTF8_LOCALE macro is defined (ex: on Android)
881 // - Return "utf-8" if the UTF-8 Mode is enabled
882 // - On Windows, return the ANSI code page (ex: "cp1250")
883 // - Return "utf-8" if nl_langinfo(CODESET) returns an empty string.
884 // - Otherwise, return nl_langinfo(CODESET).
885 //
886 // Return NULL on memory allocation failure.
887 //
888 // See also config_get_locale_encoding()
889 wchar_t*
890 _Py_GetLocaleEncoding(void)
891 {
892 #ifdef _Py_FORCE_UTF8_LOCALE
893 // On Android langinfo.h and CODESET are missing,
894 // and UTF-8 is always used in mbstowcs() and wcstombs().
895 return _PyMem_RawWcsdup(L"utf-8");
896 #else
897
898 #ifdef MS_WINDOWS
899 wchar_t encoding[23];
900 unsigned int ansi_codepage = GetACP();
901 swprintf(encoding, Py_ARRAY_LENGTH(encoding), L"cp%u", ansi_codepage);
902 encoding[Py_ARRAY_LENGTH(encoding) - 1] = 0;
903 return _PyMem_RawWcsdup(encoding);
904 #else
905 const char *encoding = nl_langinfo(CODESET);
906 if (!encoding || encoding[0] == '\0') {
907 // Use UTF-8 if nl_langinfo() returns an empty string. It can happen on
908 // macOS if the LC_CTYPE locale is not supported.
909 return _PyMem_RawWcsdup(L"utf-8");
910 }
911
912 wchar_t *wstr;
913 int res = decode_current_locale(encoding, &wstr, NULL,
914 NULL, _Py_ERROR_SURROGATEESCAPE);
915 if (res < 0) {
916 return NULL;
917 }
918 return wstr;
919 #endif // !MS_WINDOWS
920
921 #endif // !_Py_FORCE_UTF8_LOCALE
922 }
923
924
925 PyObject *
926 _Py_GetLocaleEncodingObject(void)
927 {
928 wchar_t *encoding = _Py_GetLocaleEncoding();
929 if (encoding == NULL) {
930 PyErr_NoMemory();
931 return NULL;
932 }
933
934 PyObject *str = PyUnicode_FromWideChar(encoding, -1);
935 PyMem_RawFree(encoding);
936 return str;
937 }
938
939 #ifdef HAVE_NON_UNICODE_WCHAR_T_REPRESENTATION
940
941 /* Check whether current locale uses Unicode as internal wchar_t form. */
942 int
943 _Py_LocaleUsesNonUnicodeWchar(void)
944 {
945 /* Oracle Solaris uses non-Unicode internal wchar_t form for
946 non-Unicode locales and hence needs conversion to UTF first. */
947 char* codeset = nl_langinfo(CODESET);
948 if (!codeset) {
949 return 0;
950 }
951 /* 646 refers to ISO/IEC 646 standard that corresponds to ASCII encoding */
952 return (strcmp(codeset, "UTF-8") != 0 && strcmp(codeset, "646") != 0);
953 }
954
955 static wchar_t *
956 _Py_ConvertWCharForm(const wchar_t *source, Py_ssize_t size,
957 const char *tocode, const char *fromcode)
958 {
959 static_assert(sizeof(wchar_t) == 4, "wchar_t must be 32-bit");
960
961 /* Ensure we won't overflow the size. */
962 if (size > (PY_SSIZE_T_MAX / (Py_ssize_t)sizeof(wchar_t))) {
963 PyErr_NoMemory();
964 return NULL;
965 }
966
967 /* the string doesn't have to be NULL terminated */
968 wchar_t* target = PyMem_Malloc(size * sizeof(wchar_t));
969 if (target == NULL) {
970 PyErr_NoMemory();
971 return NULL;
972 }
973
974 iconv_t cd = iconv_open(tocode, fromcode);
975 if (cd == (iconv_t)-1) {
976 PyErr_Format(PyExc_ValueError, "iconv_open() failed");
977 PyMem_Free(target);
978 return NULL;
979 }
980
981 char *inbuf = (char *) source;
982 char *outbuf = (char *) target;
983 size_t inbytesleft = sizeof(wchar_t) * size;
984 size_t outbytesleft = inbytesleft;
985
986 size_t ret = iconv(cd, &inbuf, &inbytesleft, &outbuf, &outbytesleft);
987 if (ret == DECODE_ERROR) {
988 PyErr_Format(PyExc_ValueError, "iconv() failed");
989 PyMem_Free(target);
990 iconv_close(cd);
991 return NULL;
992 }
993
994 iconv_close(cd);
995 return target;
996 }
997
998 /* Convert a wide character string to the UCS-4 encoded string. This
999 is necessary on systems where internal form of wchar_t are not Unicode
1000 code points (e.g. Oracle Solaris).
1001
1002 Return a pointer to a newly allocated string, use PyMem_Free() to free
1003 the memory. Return NULL and raise exception on conversion or memory
1004 allocation error. */
1005 wchar_t *
1006 _Py_DecodeNonUnicodeWchar(const wchar_t *native, Py_ssize_t size)
1007 {
1008 return _Py_ConvertWCharForm(native, size, "UCS-4-INTERNAL", "wchar_t");
1009 }
1010
1011 /* Convert a UCS-4 encoded string to native wide character string. This
1012 is necessary on systems where internal form of wchar_t are not Unicode
1013 code points (e.g. Oracle Solaris).
1014
1015 The conversion is done in place. This can be done because both wchar_t
1016 and UCS-4 use 4-byte encoding, and one wchar_t symbol always correspond
1017 to a single UCS-4 symbol and vice versa. (This is true for Oracle Solaris,
1018 which is currently the only system using these functions; it doesn't have
1019 to be for other systems).
1020
1021 Return 0 on success. Return -1 and raise exception on conversion
1022 or memory allocation error. */
1023 int
1024 _Py_EncodeNonUnicodeWchar_InPlace(wchar_t *unicode, Py_ssize_t size)
1025 {
1026 wchar_t* result = _Py_ConvertWCharForm(unicode, size, "wchar_t", "UCS-4-INTERNAL");
1027 if (!result) {
1028 return -1;
1029 }
1030 memcpy(unicode, result, size * sizeof(wchar_t));
1031 PyMem_Free(result);
1032 return 0;
1033 }
1034 #endif /* HAVE_NON_UNICODE_WCHAR_T_REPRESENTATION */
1035
1036 #ifdef MS_WINDOWS
1037 static __int64 secs_between_epochs = 11644473600; /* Seconds between 1.1.1601 and 1.1.1970 */
1038
1039 static void
1040 FILE_TIME_to_time_t_nsec(FILETIME *in_ptr, time_t *time_out, int* nsec_out)
1041 {
1042 /* XXX endianness. Shouldn't matter, as all Windows implementations are little-endian */
1043 /* Cannot simply cast and dereference in_ptr,
1044 since it might not be aligned properly */
1045 __int64 in;
1046 memcpy(&in, in_ptr, sizeof(in));
1047 *nsec_out = (int)(in % 10000000) * 100; /* FILETIME is in units of 100 nsec. */
1048 *time_out = Py_SAFE_DOWNCAST((in / 10000000) - secs_between_epochs, __int64, time_t);
1049 }
1050
1051 void
1052 _Py_time_t_to_FILE_TIME(time_t time_in, int nsec_in, FILETIME *out_ptr)
1053 {
1054 /* XXX endianness */
1055 __int64 out;
1056 out = time_in + secs_between_epochs;
1057 out = out * 10000000 + nsec_in / 100;
1058 memcpy(out_ptr, &out, sizeof(out));
1059 }
1060
1061 /* Below, we *know* that ugo+r is 0444 */
1062 #if _S_IREAD != 0400
1063 #error Unsupported C library
1064 #endif
1065 static int
1066 attributes_to_mode(DWORD attr)
1067 {
1068 int m = 0;
1069 if (attr & FILE_ATTRIBUTE_DIRECTORY)
1070 m |= _S_IFDIR | 0111; /* IFEXEC for user,group,other */
1071 else
1072 m |= _S_IFREG;
1073 if (attr & FILE_ATTRIBUTE_READONLY)
1074 m |= 0444;
1075 else
1076 m |= 0666;
1077 return m;
1078 }
1079
1080 void
1081 _Py_attribute_data_to_stat(BY_HANDLE_FILE_INFORMATION *info, ULONG reparse_tag,
1082 struct _Py_stat_struct *result)
1083 {
1084 memset(result, 0, sizeof(*result));
1085 result->st_mode = attributes_to_mode(info->dwFileAttributes);
1086 result->st_size = (((__int64)info->nFileSizeHigh)<<32) + info->nFileSizeLow;
1087 result->st_dev = info->dwVolumeSerialNumber;
1088 result->st_rdev = result->st_dev;
1089 FILE_TIME_to_time_t_nsec(&info->ftCreationTime, &result->st_ctime, &result->st_ctime_nsec);
1090 FILE_TIME_to_time_t_nsec(&info->ftLastWriteTime, &result->st_mtime, &result->st_mtime_nsec);
1091 FILE_TIME_to_time_t_nsec(&info->ftLastAccessTime, &result->st_atime, &result->st_atime_nsec);
1092 result->st_nlink = info->nNumberOfLinks;
1093 result->st_ino = (((uint64_t)info->nFileIndexHigh) << 32) + info->nFileIndexLow;
1094 /* bpo-37834: Only actual symlinks set the S_IFLNK flag. But lstat() will
1095 open other name surrogate reparse points without traversing them. To
1096 detect/handle these, check st_file_attributes and st_reparse_tag. */
1097 result->st_reparse_tag = reparse_tag;
1098 if (info->dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT &&
1099 reparse_tag == IO_REPARSE_TAG_SYMLINK) {
1100 /* first clear the S_IFMT bits */
1101 result->st_mode ^= (result->st_mode & S_IFMT);
1102 /* now set the bits that make this a symlink */
1103 result->st_mode |= S_IFLNK;
1104 }
1105 result->st_file_attributes = info->dwFileAttributes;
1106 }
1107 #endif
1108
1109 /* Return information about a file.
1110
1111 On POSIX, use fstat().
1112
1113 On Windows, use GetFileType() and GetFileInformationByHandle() which support
1114 files larger than 2 GiB. fstat() may fail with EOVERFLOW on files larger
1115 than 2 GiB because the file size type is a signed 32-bit integer: see issue
1116 #23152.
1117
1118 On Windows, set the last Windows error and return nonzero on error. On
1119 POSIX, set errno and return nonzero on error. Fill status and return 0 on
1120 success. */
1121 int
1122 _Py_fstat_noraise(int fd, struct _Py_stat_struct *status)
1123 {
1124 #ifdef MS_WINDOWS
1125 BY_HANDLE_FILE_INFORMATION info;
1126 HANDLE h;
1127 int type;
1128
1129 h = _Py_get_osfhandle_noraise(fd);
1130
1131 if (h == INVALID_HANDLE_VALUE) {
1132 /* errno is already set by _get_osfhandle, but we also set
1133 the Win32 error for callers who expect that */
1134 SetLastError(ERROR_INVALID_HANDLE);
1135 return -1;
1136 }
1137 memset(status, 0, sizeof(*status));
1138
1139 type = GetFileType(h);
1140 if (type == FILE_TYPE_UNKNOWN) {
1141 DWORD error = GetLastError();
1142 if (error != 0) {
1143 errno = winerror_to_errno(error);
1144 return -1;
1145 }
1146 /* else: valid but unknown file */
1147 }
1148
1149 if (type != FILE_TYPE_DISK) {
1150 if (type == FILE_TYPE_CHAR)
1151 status->st_mode = _S_IFCHR;
1152 else if (type == FILE_TYPE_PIPE)
1153 status->st_mode = _S_IFIFO;
1154 return 0;
1155 }
1156
1157 if (!GetFileInformationByHandle(h, &info)) {
1158 /* The Win32 error is already set, but we also set errno for
1159 callers who expect it */
1160 errno = winerror_to_errno(GetLastError());
1161 return -1;
1162 }
1163
1164 _Py_attribute_data_to_stat(&info, 0, status);
1165 /* specific to fstat() */
1166 status->st_ino = (((uint64_t)info.nFileIndexHigh) << 32) + info.nFileIndexLow;
1167 return 0;
1168 #else
1169 return fstat(fd, status);
1170 #endif
1171 }
1172
1173 /* Return information about a file.
1174
1175 On POSIX, use fstat().
1176
1177 On Windows, use GetFileType() and GetFileInformationByHandle() which support
1178 files larger than 2 GiB. fstat() may fail with EOVERFLOW on files larger
1179 than 2 GiB because the file size type is a signed 32-bit integer: see issue
1180 #23152.
1181
1182 Raise an exception and return -1 on error. On Windows, set the last Windows
1183 error on error. On POSIX, set errno on error. Fill status and return 0 on
1184 success.
1185
1186 Release the GIL to call GetFileType() and GetFileInformationByHandle(), or
1187 to call fstat(). The caller must hold the GIL. */
1188 int
1189 _Py_fstat(int fd, struct _Py_stat_struct *status)
1190 {
1191 int res;
1192
1193 assert(PyGILState_Check());
1194
1195 Py_BEGIN_ALLOW_THREADS
1196 res = _Py_fstat_noraise(fd, status);
1197 Py_END_ALLOW_THREADS
1198
1199 if (res != 0) {
1200 #ifdef MS_WINDOWS
1201 PyErr_SetFromWindowsErr(0);
1202 #else
1203 PyErr_SetFromErrno(PyExc_OSError);
1204 #endif
1205 return -1;
1206 }
1207 return 0;
1208 }
1209
1210 /* Like _Py_stat() but with a raw filename. */
1211 int
1212 _Py_wstat(const wchar_t* path, struct stat *buf)
1213 {
1214 int err;
1215 #ifdef MS_WINDOWS
1216 struct _stat wstatbuf;
1217 err = _wstat(path, &wstatbuf);
1218 if (!err) {
1219 buf->st_mode = wstatbuf.st_mode;
1220 }
1221 #else
1222 char *fname;
1223 fname = _Py_EncodeLocaleRaw(path, NULL);
1224 if (fname == NULL) {
1225 errno = EINVAL;
1226 return -1;
1227 }
1228 err = stat(fname, buf);
1229 PyMem_RawFree(fname);
1230 #endif
1231 return err;
1232 }
1233
1234
1235 /* Call _wstat() on Windows, or encode the path to the filesystem encoding and
1236 call stat() otherwise. Only fill st_mode attribute on Windows.
1237
1238 Return 0 on success, -1 on _wstat() / stat() error, -2 if an exception was
1239 raised. */
1240
1241 int
1242 _Py_stat(PyObject *path, struct stat *statbuf)
1243 {
1244 #ifdef MS_WINDOWS
1245 int err;
1246
1247 #if USE_UNICODE_WCHAR_CACHE
1248 const wchar_t *wpath = _PyUnicode_AsUnicode(path);
1249 #else /* USE_UNICODE_WCHAR_CACHE */
1250 wchar_t *wpath = PyUnicode_AsWideCharString(path, NULL);
1251 #endif /* USE_UNICODE_WCHAR_CACHE */
1252 if (wpath == NULL)
1253 return -2;
1254
1255 err = _Py_wstat(wpath, statbuf);
1256 #if !USE_UNICODE_WCHAR_CACHE
1257 PyMem_Free(wpath);
1258 #endif /* USE_UNICODE_WCHAR_CACHE */
1259 return err;
1260 #else
1261 int ret;
1262 PyObject *bytes;
1263 char *cpath;
1264
1265 bytes = PyUnicode_EncodeFSDefault(path);
1266 if (bytes == NULL)
1267 return -2;
1268
1269 /* check for embedded null bytes */
1270 if (PyBytes_AsStringAndSize(bytes, &cpath, NULL) == -1) {
1271 Py_DECREF(bytes);
1272 return -2;
1273 }
1274
1275 ret = stat(cpath, statbuf);
1276 Py_DECREF(bytes);
1277 return ret;
1278 #endif
1279 }
1280
1281
1282 /* This function MUST be kept async-signal-safe on POSIX when raise=0. */
1283 static int
1284 get_inheritable(int fd, int raise)
1285 {
1286 #ifdef MS_WINDOWS
1287 HANDLE handle;
1288 DWORD flags;
1289
1290 handle = _Py_get_osfhandle_noraise(fd);
1291 if (handle == INVALID_HANDLE_VALUE) {
1292 if (raise)
1293 PyErr_SetFromErrno(PyExc_OSError);
1294 return -1;
1295 }
1296
1297 if (!GetHandleInformation(handle, &flags)) {
1298 if (raise)
1299 PyErr_SetFromWindowsErr(0);
1300 return -1;
1301 }
1302
1303 return (flags & HANDLE_FLAG_INHERIT);
1304 #else
1305 int flags;
1306
1307 flags = fcntl(fd, F_GETFD, 0);
1308 if (flags == -1) {
1309 if (raise)
1310 PyErr_SetFromErrno(PyExc_OSError);
1311 return -1;
1312 }
1313 return !(flags & FD_CLOEXEC);
1314 #endif
1315 }
1316
1317 /* Get the inheritable flag of the specified file descriptor.
1318 Return 1 if the file descriptor can be inherited, 0 if it cannot,
1319 raise an exception and return -1 on error. */
1320 int
1321 _Py_get_inheritable(int fd)
1322 {
1323 return get_inheritable(fd, 1);
1324 }
1325
1326
1327 /* This function MUST be kept async-signal-safe on POSIX when raise=0. */
1328 static int
1329 set_inheritable(int fd, int inheritable, int raise, int *atomic_flag_works)
1330 {
1331 #ifdef MS_WINDOWS
1332 HANDLE handle;
1333 DWORD flags;
1334 #else
1335 #if defined(HAVE_SYS_IOCTL_H) && defined(FIOCLEX) && defined(FIONCLEX)
1336 static int ioctl_works = -1;
1337 int request;
1338 int err;
1339 #endif
1340 int flags, new_flags;
1341 int res;
1342 #endif
1343
1344 /* atomic_flag_works can only be used to make the file descriptor
1345 non-inheritable */
1346 assert(!(atomic_flag_works != NULL && inheritable));
1347
1348 if (atomic_flag_works != NULL && !inheritable) {
1349 if (*atomic_flag_works == -1) {
1350 int isInheritable = get_inheritable(fd, raise);
1351 if (isInheritable == -1)
1352 return -1;
1353 *atomic_flag_works = !isInheritable;
1354 }
1355
1356 if (*atomic_flag_works)
1357 return 0;
1358 }
1359
1360 #ifdef MS_WINDOWS
1361 handle = _Py_get_osfhandle_noraise(fd);
1362 if (handle == INVALID_HANDLE_VALUE) {
1363 if (raise)
1364 PyErr_SetFromErrno(PyExc_OSError);
1365 return -1;
1366 }
1367
1368 if (inheritable)
1369 flags = HANDLE_FLAG_INHERIT;
1370 else
1371 flags = 0;
1372
1373 /* This check can be removed once support for Windows 7 ends. */
1374 #define CONSOLE_PSEUDOHANDLE(handle) (((ULONG_PTR)(handle) & 0x3) == 0x3 && \
1375 GetFileType(handle) == FILE_TYPE_CHAR)
1376
1377 if (!CONSOLE_PSEUDOHANDLE(handle) &&
1378 !SetHandleInformation(handle, HANDLE_FLAG_INHERIT, flags)) {
1379 if (raise)
1380 PyErr_SetFromWindowsErr(0);
1381 return -1;
1382 }
1383 #undef CONSOLE_PSEUDOHANDLE
1384 return 0;
1385
1386 #else
1387
1388 #if defined(HAVE_SYS_IOCTL_H) && defined(FIOCLEX) && defined(FIONCLEX)
1389 if (ioctl_works != 0 && raise != 0) {
1390 /* fast-path: ioctl() only requires one syscall */
1391 /* caveat: raise=0 is an indicator that we must be async-signal-safe
1392 * thus avoid using ioctl() so we skip the fast-path. */
1393 if (inheritable)
1394 request = FIONCLEX;
1395 else
1396 request = FIOCLEX;
1397 err = ioctl(fd, request, NULL);
1398 if (!err) {
1399 ioctl_works = 1;
1400 return 0;
1401 }
1402
1403 #ifdef O_PATH
1404 if (errno == EBADF) {
1405 // bpo-44849: On Linux and FreeBSD, ioctl(FIOCLEX) fails with EBADF
1406 // on O_PATH file descriptors. Fall through to the fcntl()
1407 // implementation.
1408 }
1409 else
1410 #endif
1411 if (errno != ENOTTY && errno != EACCES) {
1412 if (raise)
1413 PyErr_SetFromErrno(PyExc_OSError);
1414 return -1;
1415 }
1416 else {
1417 /* Issue #22258: Here, ENOTTY means "Inappropriate ioctl for
1418 device". The ioctl is declared but not supported by the kernel.
1419 Remember that ioctl() doesn't work. It is the case on
1420 Illumos-based OS for example.
1421
1422 Issue #27057: When SELinux policy disallows ioctl it will fail
1423 with EACCES. While FIOCLEX is safe operation it may be
1424 unavailable because ioctl was denied altogether.
1425 This can be the case on Android. */
1426 ioctl_works = 0;
1427 }
1428 /* fallback to fcntl() if ioctl() does not work */
1429 }
1430 #endif
1431
1432 /* slow-path: fcntl() requires two syscalls */
1433 flags = fcntl(fd, F_GETFD);
1434 if (flags < 0) {
1435 if (raise)
1436 PyErr_SetFromErrno(PyExc_OSError);
1437 return -1;
1438 }
1439
1440 if (inheritable) {
1441 new_flags = flags & ~FD_CLOEXEC;
1442 }
1443 else {
1444 new_flags = flags | FD_CLOEXEC;
1445 }
1446
1447 if (new_flags == flags) {
1448 /* FD_CLOEXEC flag already set/cleared: nothing to do */
1449 return 0;
1450 }
1451
1452 res = fcntl(fd, F_SETFD, new_flags);
1453 if (res < 0) {
1454 if (raise)
1455 PyErr_SetFromErrno(PyExc_OSError);
1456 return -1;
1457 }
1458 return 0;
1459 #endif
1460 }
1461
1462 /* Make the file descriptor non-inheritable.
1463 Return 0 on success, set errno and return -1 on error. */
1464 static int
1465 make_non_inheritable(int fd)
1466 {
1467 return set_inheritable(fd, 0, 0, NULL);
1468 }
1469
1470 /* Set the inheritable flag of the specified file descriptor.
1471 On success: return 0, on error: raise an exception and return -1.
1472
1473 If atomic_flag_works is not NULL:
1474
1475 * if *atomic_flag_works==-1, check if the inheritable is set on the file
1476 descriptor: if yes, set *atomic_flag_works to 1, otherwise set to 0 and
1477 set the inheritable flag
1478 * if *atomic_flag_works==1: do nothing
1479 * if *atomic_flag_works==0: set inheritable flag to False
1480
1481 Set atomic_flag_works to NULL if no atomic flag was used to create the
1482 file descriptor.
1483
1484 atomic_flag_works can only be used to make a file descriptor
1485 non-inheritable: atomic_flag_works must be NULL if inheritable=1. */
1486 int
1487 _Py_set_inheritable(int fd, int inheritable, int *atomic_flag_works)
1488 {
1489 return set_inheritable(fd, inheritable, 1, atomic_flag_works);
1490 }
1491
1492 /* Same as _Py_set_inheritable() but on error, set errno and
1493 don't raise an exception.
1494 This function is async-signal-safe. */
1495 int
1496 _Py_set_inheritable_async_safe(int fd, int inheritable, int *atomic_flag_works)
1497 {
1498 return set_inheritable(fd, inheritable, 0, atomic_flag_works);
1499 }
1500
1501 static int
1502 _Py_open_impl(const char *pathname, int flags, int gil_held)
1503 {
1504 int fd;
1505 int async_err = 0;
1506 #ifndef MS_WINDOWS
1507 int *atomic_flag_works;
1508 #endif
1509
1510 #ifdef MS_WINDOWS
1511 flags |= O_NOINHERIT;
1512 #elif defined(O_CLOEXEC)
1513 atomic_flag_works = &_Py_open_cloexec_works;
1514 flags |= O_CLOEXEC;
1515 #else
1516 atomic_flag_works = NULL;
1517 #endif
1518
1519 if (gil_held) {
1520 PyObject *pathname_obj = PyUnicode_DecodeFSDefault(pathname);
1521 if (pathname_obj == NULL) {
1522 return -1;
1523 }
1524 if (PySys_Audit("open", "OOi", pathname_obj, Py_None, flags) < 0) {
1525 Py_DECREF(pathname_obj);
1526 return -1;
1527 }
1528
1529 do {
1530 Py_BEGIN_ALLOW_THREADS
1531 fd = open(pathname, flags);
1532 Py_END_ALLOW_THREADS
1533 } while (fd < 0
1534 && errno == EINTR && !(async_err = PyErr_CheckSignals()));
1535 if (async_err) {
1536 Py_DECREF(pathname_obj);
1537 return -1;
1538 }
1539 if (fd < 0) {
1540 PyErr_SetFromErrnoWithFilenameObjects(PyExc_OSError, pathname_obj, NULL);
1541 Py_DECREF(pathname_obj);
1542 return -1;
1543 }
1544 Py_DECREF(pathname_obj);
1545 }
1546 else {
1547 fd = open(pathname, flags);
1548 if (fd < 0)
1549 return -1;
1550 }
1551
1552 #ifndef MS_WINDOWS
1553 if (set_inheritable(fd, 0, gil_held, atomic_flag_works) < 0) {
1554 close(fd);
1555 return -1;
1556 }
1557 #endif
1558
1559 return fd;
1560 }
1561
1562 /* Open a file with the specified flags (wrapper to open() function).
1563 Return a file descriptor on success. Raise an exception and return -1 on
1564 error.
1565
1566 The file descriptor is created non-inheritable.
1567
1568 When interrupted by a signal (open() fails with EINTR), retry the syscall,
1569 except if the Python signal handler raises an exception.
1570
1571 Release the GIL to call open(). The caller must hold the GIL. */
1572 int
1573 _Py_open(const char *pathname, int flags)
1574 {
1575 /* _Py_open() must be called with the GIL held. */
1576 assert(PyGILState_Check());
1577 return _Py_open_impl(pathname, flags, 1);
1578 }
1579
1580 /* Open a file with the specified flags (wrapper to open() function).
1581 Return a file descriptor on success. Set errno and return -1 on error.
1582
1583 The file descriptor is created non-inheritable.
1584
1585 If interrupted by a signal, fail with EINTR. */
1586 int
1587 _Py_open_noraise(const char *pathname, int flags)
1588 {
1589 return _Py_open_impl(pathname, flags, 0);
1590 }
1591
1592 /* Open a file. Use _wfopen() on Windows, encode the path to the locale
1593 encoding and use fopen() otherwise.
1594
1595 The file descriptor is created non-inheritable.
1596
1597 If interrupted by a signal, fail with EINTR. */
1598 FILE *
1599 _Py_wfopen(const wchar_t *path, const wchar_t *mode)
1600 {
1601 FILE *f;
1602 if (PySys_Audit("open", "uui", path, mode, 0) < 0) {
1603 return NULL;
1604 }
1605 #ifndef MS_WINDOWS
1606 char *cpath;
1607 char cmode[10];
1608 size_t r;
1609 r = wcstombs(cmode, mode, 10);
1610 if (r == DECODE_ERROR || r >= 10) {
1611 errno = EINVAL;
1612 return NULL;
1613 }
1614 cpath = _Py_EncodeLocaleRaw(path, NULL);
1615 if (cpath == NULL) {
1616 return NULL;
1617 }
1618 f = fopen(cpath, cmode);
1619 PyMem_RawFree(cpath);
1620 #else
1621 f = _wfopen(path, mode);
1622 #endif
1623 if (f == NULL)
1624 return NULL;
1625 if (make_non_inheritable(fileno(f)) < 0) {
1626 fclose(f);
1627 return NULL;
1628 }
1629 return f;
1630 }
1631
1632
1633 /* Open a file. Call _wfopen() on Windows, or encode the path to the filesystem
1634 encoding and call fopen() otherwise.
1635
1636 Return the new file object on success. Raise an exception and return NULL
1637 on error.
1638
1639 The file descriptor is created non-inheritable.
1640
1641 When interrupted by a signal (open() fails with EINTR), retry the syscall,
1642 except if the Python signal handler raises an exception.
1643
1644 Release the GIL to call _wfopen() or fopen(). The caller must hold
1645 the GIL. */
1646 FILE*
1647 _Py_fopen_obj(PyObject *path, const char *mode)
1648 {
1649 FILE *f;
1650 int async_err = 0;
1651 #ifdef MS_WINDOWS
1652 wchar_t wmode[10];
1653 int usize;
1654
1655 assert(PyGILState_Check());
1656
1657 if (PySys_Audit("open", "Osi", path, mode, 0) < 0) {
1658 return NULL;
1659 }
1660 if (!PyUnicode_Check(path)) {
1661 PyErr_Format(PyExc_TypeError,
1662 "str file path expected under Windows, got %R",
1663 Py_TYPE(path));
1664 return NULL;
1665 }
1666 #if USE_UNICODE_WCHAR_CACHE
1667 const wchar_t *wpath = _PyUnicode_AsUnicode(path);
1668 #else /* USE_UNICODE_WCHAR_CACHE */
1669 wchar_t *wpath = PyUnicode_AsWideCharString(path, NULL);
1670 #endif /* USE_UNICODE_WCHAR_CACHE */
1671 if (wpath == NULL)
1672 return NULL;
1673
1674 usize = MultiByteToWideChar(CP_ACP, 0, mode, -1,
1675 wmode, Py_ARRAY_LENGTH(wmode));
1676 if (usize == 0) {
1677 PyErr_SetFromWindowsErr(0);
1678 #if !USE_UNICODE_WCHAR_CACHE
1679 PyMem_Free(wpath);
1680 #endif /* USE_UNICODE_WCHAR_CACHE */
1681 return NULL;
1682 }
1683
1684 do {
1685 Py_BEGIN_ALLOW_THREADS
1686 f = _wfopen(wpath, wmode);
1687 Py_END_ALLOW_THREADS
1688 } while (f == NULL
1689 && errno == EINTR && !(async_err = PyErr_CheckSignals()));
1690 int saved_errno = errno;
1691 #if !USE_UNICODE_WCHAR_CACHE
1692 PyMem_Free(wpath);
1693 #endif /* USE_UNICODE_WCHAR_CACHE */
1694 #else
1695 PyObject *bytes;
1696 const char *path_bytes;
1697
1698 assert(PyGILState_Check());
1699
1700 if (!PyUnicode_FSConverter(path, &bytes))
1701 return NULL;
1702 path_bytes = PyBytes_AS_STRING(bytes);
1703
1704 if (PySys_Audit("open", "Osi", path, mode, 0) < 0) {
1705 Py_DECREF(bytes);
1706 return NULL;
1707 }
1708
1709 do {
1710 Py_BEGIN_ALLOW_THREADS
1711 f = fopen(path_bytes, mode);
1712 Py_END_ALLOW_THREADS
1713 } while (f == NULL
1714 && errno == EINTR && !(async_err = PyErr_CheckSignals()));
1715 int saved_errno = errno;
1716 Py_DECREF(bytes);
1717 #endif
1718 if (async_err)
1719 return NULL;
1720
1721 if (f == NULL) {
1722 errno = saved_errno;
1723 PyErr_SetFromErrnoWithFilenameObject(PyExc_OSError, path);
1724 return NULL;
1725 }
1726
1727 if (set_inheritable(fileno(f), 0, 1, NULL) < 0) {
1728 fclose(f);
1729 return NULL;
1730 }
1731 return f;
1732 }
1733
1734 /* Read count bytes from fd into buf.
1735
1736 On success, return the number of read bytes, it can be lower than count.
1737 If the current file offset is at or past the end of file, no bytes are read,
1738 and read() returns zero.
1739
1740 On error, raise an exception, set errno and return -1.
1741
1742 When interrupted by a signal (read() fails with EINTR), retry the syscall.
1743 If the Python signal handler raises an exception, the function returns -1
1744 (the syscall is not retried).
1745
1746 Release the GIL to call read(). The caller must hold the GIL. */
1747 Py_ssize_t
1748 _Py_read(int fd, void *buf, size_t count)
1749 {
1750 Py_ssize_t n;
1751 int err;
1752 int async_err = 0;
1753
1754 assert(PyGILState_Check());
1755
1756 /* _Py_read() must not be called with an exception set, otherwise the
1757 * caller may think that read() was interrupted by a signal and the signal
1758 * handler raised an exception. */
1759 assert(!PyErr_Occurred());
1760
1761 if (count > _PY_READ_MAX) {
1762 count = _PY_READ_MAX;
1763 }
1764
1765 _Py_BEGIN_SUPPRESS_IPH
1766 do {
1767 Py_BEGIN_ALLOW_THREADS
1768 errno = 0;
1769 #ifdef MS_WINDOWS
1770 n = read(fd, buf, (int)count);
1771 #else
1772 n = read(fd, buf, count);
1773 #endif
1774 /* save/restore errno because PyErr_CheckSignals()
1775 * and PyErr_SetFromErrno() can modify it */
1776 err = errno;
1777 Py_END_ALLOW_THREADS
1778 } while (n < 0 && err == EINTR &&
1779 !(async_err = PyErr_CheckSignals()));
1780 _Py_END_SUPPRESS_IPH
1781
1782 if (async_err) {
1783 /* read() was interrupted by a signal (failed with EINTR)
1784 * and the Python signal handler raised an exception */
1785 errno = err;
1786 assert(errno == EINTR && PyErr_Occurred());
1787 return -1;
1788 }
1789 if (n < 0) {
1790 PyErr_SetFromErrno(PyExc_OSError);
1791 errno = err;
1792 return -1;
1793 }
1794
1795 return n;
1796 }
1797
1798 static Py_ssize_t
1799 _Py_write_impl(int fd, const void *buf, size_t count, int gil_held)
1800 {
1801 Py_ssize_t n;
1802 int err;
1803 int async_err = 0;
1804
1805 _Py_BEGIN_SUPPRESS_IPH
1806 #ifdef MS_WINDOWS
1807 if (count > 32767) {
1808 /* Issue #11395: the Windows console returns an error (12: not
1809 enough space error) on writing into stdout if stdout mode is
1810 binary and the length is greater than 66,000 bytes (or less,
1811 depending on heap usage). */
1812 if (gil_held) {
1813 Py_BEGIN_ALLOW_THREADS
1814 if (isatty(fd)) {
1815 count = 32767;
1816 }
1817 Py_END_ALLOW_THREADS
1818 } else {
1819 if (isatty(fd)) {
1820 count = 32767;
1821 }
1822 }
1823 }
1824 #endif
1825 if (count > _PY_WRITE_MAX) {
1826 count = _PY_WRITE_MAX;
1827 }
1828
1829 if (gil_held) {
1830 do {
1831 Py_BEGIN_ALLOW_THREADS
1832 errno = 0;
1833 #ifdef MS_WINDOWS
1834 n = write(fd, buf, (int)count);
1835 #else
1836 n = write(fd, buf, count);
1837 #endif
1838 /* save/restore errno because PyErr_CheckSignals()
1839 * and PyErr_SetFromErrno() can modify it */
1840 err = errno;
1841 Py_END_ALLOW_THREADS
1842 } while (n < 0 && err == EINTR &&
1843 !(async_err = PyErr_CheckSignals()));
1844 }
1845 else {
1846 do {
1847 errno = 0;
1848 #ifdef MS_WINDOWS
1849 n = write(fd, buf, (int)count);
1850 #else
1851 n = write(fd, buf, count);
1852 #endif
1853 err = errno;
1854 } while (n < 0 && err == EINTR);
1855 }
1856 _Py_END_SUPPRESS_IPH
1857
1858 if (async_err) {
1859 /* write() was interrupted by a signal (failed with EINTR)
1860 and the Python signal handler raised an exception (if gil_held is
1861 nonzero). */
1862 errno = err;
1863 assert(errno == EINTR && (!gil_held || PyErr_Occurred()));
1864 return -1;
1865 }
1866 if (n < 0) {
1867 if (gil_held)
1868 PyErr_SetFromErrno(PyExc_OSError);
1869 errno = err;
1870 return -1;
1871 }
1872
1873 return n;
1874 }
1875
1876 /* Write count bytes of buf into fd.
1877
1878 On success, return the number of written bytes, it can be lower than count
1879 including 0. On error, raise an exception, set errno and return -1.
1880
1881 When interrupted by a signal (write() fails with EINTR), retry the syscall.
1882 If the Python signal handler raises an exception, the function returns -1
1883 (the syscall is not retried).
1884
1885 Release the GIL to call write(). The caller must hold the GIL. */
1886 Py_ssize_t
1887 _Py_write(int fd, const void *buf, size_t count)
1888 {
1889 assert(PyGILState_Check());
1890
1891 /* _Py_write() must not be called with an exception set, otherwise the
1892 * caller may think that write() was interrupted by a signal and the signal
1893 * handler raised an exception. */
1894 assert(!PyErr_Occurred());
1895
1896 return _Py_write_impl(fd, buf, count, 1);
1897 }
1898
1899 /* Write count bytes of buf into fd.
1900 *
1901 * On success, return the number of written bytes, it can be lower than count
1902 * including 0. On error, set errno and return -1.
1903 *
1904 * When interrupted by a signal (write() fails with EINTR), retry the syscall
1905 * without calling the Python signal handler. */
1906 Py_ssize_t
1907 _Py_write_noraise(int fd, const void *buf, size_t count)
1908 {
1909 return _Py_write_impl(fd, buf, count, 0);
1910 }
1911
1912 #ifdef HAVE_READLINK
1913
1914 /* Read value of symbolic link. Encode the path to the locale encoding, decode
1915 the result from the locale encoding.
1916
1917 Return -1 on encoding error, on readlink() error, if the internal buffer is
1918 too short, on decoding error, or if 'buf' is too short. */
1919 int
1920 _Py_wreadlink(const wchar_t *path, wchar_t *buf, size_t buflen)
1921 {
1922 char *cpath;
1923 char cbuf[MAXPATHLEN];
1924 size_t cbuf_len = Py_ARRAY_LENGTH(cbuf);
1925 wchar_t *wbuf;
1926 Py_ssize_t res;
1927 size_t r1;
1928
1929 cpath = _Py_EncodeLocaleRaw(path, NULL);
1930 if (cpath == NULL) {
1931 errno = EINVAL;
1932 return -1;
1933 }
1934 res = readlink(cpath, cbuf, cbuf_len);
1935 PyMem_RawFree(cpath);
1936 if (res == -1) {
1937 return -1;
1938 }
1939 if ((size_t)res == cbuf_len) {
1940 errno = EINVAL;
1941 return -1;
1942 }
1943 cbuf[res] = '\0'; /* buf will be null terminated */
1944 wbuf = Py_DecodeLocale(cbuf, &r1);
1945 if (wbuf == NULL) {
1946 errno = EINVAL;
1947 return -1;
1948 }
1949 /* wbuf must have space to store the trailing NUL character */
1950 if (buflen <= r1) {
1951 PyMem_RawFree(wbuf);
1952 errno = EINVAL;
1953 return -1;
1954 }
1955 wcsncpy(buf, wbuf, buflen);
1956 PyMem_RawFree(wbuf);
1957 return (int)r1;
1958 }
1959 #endif
1960
1961 #ifdef HAVE_REALPATH
1962
1963 /* Return the canonicalized absolute pathname. Encode path to the locale
1964 encoding, decode the result from the locale encoding.
1965
1966 Return NULL on encoding error, realpath() error, decoding error
1967 or if 'resolved_path' is too short. */
1968 wchar_t*
1969 _Py_wrealpath(const wchar_t *path,
1970 wchar_t *resolved_path, size_t resolved_path_len)
1971 {
1972 char *cpath;
1973 char cresolved_path[MAXPATHLEN];
1974 wchar_t *wresolved_path;
1975 char *res;
1976 size_t r;
1977 cpath = _Py_EncodeLocaleRaw(path, NULL);
1978 if (cpath == NULL) {
1979 errno = EINVAL;
1980 return NULL;
1981 }
1982 res = realpath(cpath, cresolved_path);
1983 PyMem_RawFree(cpath);
1984 if (res == NULL)
1985 return NULL;
1986
1987 wresolved_path = Py_DecodeLocale(cresolved_path, &r);
1988 if (wresolved_path == NULL) {
1989 errno = EINVAL;
1990 return NULL;
1991 }
1992 /* wresolved_path must have space to store the trailing NUL character */
1993 if (resolved_path_len <= r) {
1994 PyMem_RawFree(wresolved_path);
1995 errno = EINVAL;
1996 return NULL;
1997 }
1998 wcsncpy(resolved_path, wresolved_path, resolved_path_len);
1999 PyMem_RawFree(wresolved_path);
2000 return resolved_path;
2001 }
2002 #endif
2003
2004
2005 int
2006 _Py_isabs(const wchar_t *path)
2007 {
2008 #ifdef MS_WINDOWS
2009 const wchar_t *tail;
2010 HRESULT hr = PathCchSkipRoot(path, &tail);
2011 if (FAILED(hr) || path == tail) {
2012 return 0;
2013 }
2014 if (tail == &path[1] && (path[0] == SEP || path[0] == ALTSEP)) {
2015 // Exclude paths with leading SEP
2016 return 0;
2017 }
2018 if (tail == &path[2] && path[1] == L':') {
2019 // Exclude drive-relative paths (e.g. C:filename.ext)
2020 return 0;
2021 }
2022 return 1;
2023 #else
2024 return (path[0] == SEP);
2025 #endif
2026 }
2027
2028
2029 /* Get an absolute path.
2030 On error (ex: fail to get the current directory), return -1.
2031 On memory allocation failure, set *abspath_p to NULL and return 0.
2032 On success, return a newly allocated to *abspath_p to and return 0.
2033 The string must be freed by PyMem_RawFree(). */
2034 int
2035 _Py_abspath(const wchar_t *path, wchar_t **abspath_p)
2036 {
2037 if (path[0] == '\0' || !wcscmp(path, L".")) {
2038 wchar_t cwd[MAXPATHLEN + 1];
2039 cwd[Py_ARRAY_LENGTH(cwd) - 1] = 0;
2040 if (!_Py_wgetcwd(cwd, Py_ARRAY_LENGTH(cwd) - 1)) {
2041 /* unable to get the current directory */
2042 return -1;
2043 }
2044 *abspath_p = _PyMem_RawWcsdup(cwd);
2045 return 0;
2046 }
2047
2048 if (_Py_isabs(path)) {
2049 *abspath_p = _PyMem_RawWcsdup(path);
2050 return 0;
2051 }
2052
2053 #ifdef MS_WINDOWS
2054 return _PyOS_getfullpathname(path, abspath_p);
2055 #else
2056 wchar_t cwd[MAXPATHLEN + 1];
2057 cwd[Py_ARRAY_LENGTH(cwd) - 1] = 0;
2058 if (!_Py_wgetcwd(cwd, Py_ARRAY_LENGTH(cwd) - 1)) {
2059 /* unable to get the current directory */
2060 return -1;
2061 }
2062
2063 size_t cwd_len = wcslen(cwd);
2064 size_t path_len = wcslen(path);
2065 size_t len = cwd_len + 1 + path_len + 1;
2066 if (len <= (size_t)PY_SSIZE_T_MAX / sizeof(wchar_t)) {
2067 *abspath_p = PyMem_RawMalloc(len * sizeof(wchar_t));
2068 }
2069 else {
2070 *abspath_p = NULL;
2071 }
2072 if (*abspath_p == NULL) {
2073 return 0;
2074 }
2075
2076 wchar_t *abspath = *abspath_p;
2077 memcpy(abspath, cwd, cwd_len * sizeof(wchar_t));
2078 abspath += cwd_len;
2079
2080 *abspath = (wchar_t)SEP;
2081 abspath++;
2082
2083 memcpy(abspath, path, path_len * sizeof(wchar_t));
2084 abspath += path_len;
2085
2086 *abspath = 0;
2087 return 0;
2088 #endif
2089 }
2090
2091
2092 // The caller must ensure "buffer" is big enough.
2093 static int
2094 join_relfile(wchar_t *buffer, size_t bufsize,
2095 const wchar_t *dirname, const wchar_t *relfile)
2096 {
2097 #ifdef MS_WINDOWS
2098 if (FAILED(PathCchCombineEx(buffer, bufsize, dirname, relfile,
2099 PATHCCH_ALLOW_LONG_PATHS))) {
2100 return -1;
2101 }
2102 #else
2103 assert(!_Py_isabs(relfile));
2104 size_t dirlen = wcslen(dirname);
2105 size_t rellen = wcslen(relfile);
2106 size_t maxlen = bufsize - 1;
2107 if (maxlen > MAXPATHLEN || dirlen >= maxlen || rellen >= maxlen - dirlen) {
2108 return -1;
2109 }
2110 if (dirlen == 0) {
2111 // We do not add a leading separator.
2112 wcscpy(buffer, relfile);
2113 }
2114 else {
2115 if (dirname != buffer) {
2116 wcscpy(buffer, dirname);
2117 }
2118 size_t relstart = dirlen;
2119 if (dirlen > 1 && dirname[dirlen - 1] != SEP) {
2120 buffer[dirlen] = SEP;
2121 relstart += 1;
2122 }
2123 wcscpy(&buffer[relstart], relfile);
2124 }
2125 #endif
2126 return 0;
2127 }
2128
2129 /* Join the two paths together, like os.path.join(). Return NULL
2130 if memory could not be allocated. The caller is responsible
2131 for calling PyMem_RawFree() on the result. */
2132 wchar_t *
2133 _Py_join_relfile(const wchar_t *dirname, const wchar_t *relfile)
2134 {
2135 assert(dirname != NULL && relfile != NULL);
2136 #ifndef MS_WINDOWS
2137 assert(!_Py_isabs(relfile));
2138 #endif
2139 size_t maxlen = wcslen(dirname) + 1 + wcslen(relfile);
2140 size_t bufsize = maxlen + 1;
2141 wchar_t *filename = PyMem_RawMalloc(bufsize * sizeof(wchar_t));
2142 if (filename == NULL) {
2143 return NULL;
2144 }
2145 assert(wcslen(dirname) < MAXPATHLEN);
2146 assert(wcslen(relfile) < MAXPATHLEN - wcslen(dirname));
2147 if (join_relfile(filename, bufsize, dirname, relfile) < 0) {
2148 PyMem_RawFree(filename);
2149 return NULL;
2150 }
2151 return filename;
2152 }
2153
2154 /* Join the two paths together, like os.path.join().
2155 dirname: the target buffer with the dirname already in place,
2156 including trailing NUL
2157 relfile: this must be a relative path
2158 bufsize: total allocated size of the buffer
2159 Return -1 if anything is wrong with the path lengths. */
2160 int
2161 _Py_add_relfile(wchar_t *dirname, const wchar_t *relfile, size_t bufsize)
2162 {
2163 assert(dirname != NULL && relfile != NULL);
2164 assert(bufsize > 0);
2165 return join_relfile(dirname, bufsize, dirname, relfile);
2166 }
2167
2168
2169 size_t
2170 _Py_find_basename(const wchar_t *filename)
2171 {
2172 for (size_t i = wcslen(filename); i > 0; --i) {
2173 if (filename[i] == SEP) {
2174 return i + 1;
2175 }
2176 }
2177 return 0;
2178 }
2179
2180 /* In-place path normalisation. Returns the start of the normalized
2181 path, which will be within the original buffer. Guaranteed to not
2182 make the path longer, and will not fail. 'size' is the length of
2183 the path, if known. If -1, the first null character will be assumed
2184 to be the end of the path. 'normsize' will be set to contain the
2185 length of the resulting normalized path. */
2186 wchar_t *
2187 _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize)
2188 {
2189 assert(path != NULL);
2190 if ((size < 0 && !path[0]) || size == 0) {
2191 *normsize = 0;
2192 return path;
2193 }
2194 wchar_t *pEnd = size >= 0 ? &path[size] : NULL;
2195 wchar_t *p1 = path; // sequentially scanned address in the path
2196 wchar_t *p2 = path; // destination of a scanned character to be ljusted
2197 wchar_t *minP2 = path; // the beginning of the destination range
2198 wchar_t lastC = L'\0'; // the last ljusted character, p2[-1] in most cases
2199
2200 #define IS_END(x) (pEnd ? (x) == pEnd : !*(x))
2201 #ifdef ALTSEP
2202 #define IS_SEP(x) (*(x) == SEP || *(x) == ALTSEP)
2203 #else
2204 #define IS_SEP(x) (*(x) == SEP)
2205 #endif
2206 #define SEP_OR_END(x) (IS_SEP(x) || IS_END(x))
2207
2208 // Skip leading '.\'
2209 if (p1[0] == L'.' && IS_SEP(&p1[1])) {
2210 path = &path[2];
2211 while (IS_SEP(path) && !IS_END(path)) {
2212 path++;
2213 }
2214 p1 = p2 = minP2 = path;
2215 lastC = SEP;
2216 }
2217 #ifdef MS_WINDOWS
2218 // Skip past drive segment and update minP2
2219 else if (p1[0] && p1[1] == L':') {
2220 *p2++ = *p1++;
2221 *p2++ = *p1++;
2222 minP2 = p2;
2223 lastC = L':';
2224 }
2225 // Skip past all \\-prefixed paths, including \\?\, \\.\,
2226 // and network paths, including the first segment.
2227 else if (IS_SEP(&p1[0]) && IS_SEP(&p1[1])) {
2228 int sepCount = 2;
2229 *p2++ = SEP;
2230 *p2++ = SEP;
2231 p1 += 2;
2232 for (; !IS_END(p1) && sepCount; ++p1) {
2233 if (IS_SEP(p1)) {
2234 --sepCount;
2235 *p2++ = lastC = SEP;
2236 } else {
2237 *p2++ = lastC = *p1;
2238 }
2239 }
2240 minP2 = p2 - 1;
2241 }
2242 #else
2243 // Skip past two leading SEPs
2244 else if (IS_SEP(&p1[0]) && IS_SEP(&p1[1]) && !IS_SEP(&p1[2])) {
2245 *p2++ = *p1++;
2246 *p2++ = *p1++;
2247 minP2 = p2 - 1; // Absolute path has SEP at minP2
2248 lastC = SEP;
2249 }
2250 #endif /* MS_WINDOWS */
2251
2252 /* if pEnd is specified, check that. Else, check for null terminator */
2253 for (; !IS_END(p1); ++p1) {
2254 wchar_t c = *p1;
2255 #ifdef ALTSEP
2256 if (c == ALTSEP) {
2257 c = SEP;
2258 }
2259 #endif
2260 if (lastC == SEP) {
2261 if (c == L'.') {
2262 int sep_at_1 = SEP_OR_END(&p1[1]);
2263 int sep_at_2 = !sep_at_1 && SEP_OR_END(&p1[2]);
2264 if (sep_at_2 && p1[1] == L'.') {
2265 wchar_t *p3 = p2;
2266 while (p3 != minP2 && *--p3 == SEP) { }
2267 while (p3 != minP2 && *(p3 - 1) != SEP) { --p3; }
2268 if (p2 == minP2
2269 || (p3[0] == L'.' && p3[1] == L'.' && IS_SEP(&p3[2])))
2270 {
2271 // Previous segment is also ../, so append instead.
2272 // Relative path does not absorb ../ at minP2 as well.
2273 *p2++ = L'.';
2274 *p2++ = L'.';
2275 lastC = L'.';
2276 } else if (p3[0] == SEP) {
2277 // Absolute path, so absorb segment
2278 p2 = p3 + 1;
2279 } else {
2280 p2 = p3;
2281 }
2282 p1 += 1;
2283 } else if (sep_at_1) {
2284 } else {
2285 *p2++ = lastC = c;
2286 }
2287 } else if (c == SEP) {
2288 } else {
2289 *p2++ = lastC = c;
2290 }
2291 } else {
2292 *p2++ = lastC = c;
2293 }
2294 }
2295 *p2 = L'\0';
2296 if (p2 != minP2) {
2297 while (--p2 != minP2 && *p2 == SEP) {
2298 *p2 = L'\0';
2299 }
2300 } else {
2301 --p2;
2302 }
2303 *normsize = p2 - path + 1;
2304 #undef SEP_OR_END
2305 #undef IS_SEP
2306 #undef IS_END
2307 return path;
2308 }
2309
2310 /* In-place path normalisation. Returns the start of the normalized
2311 path, which will be within the original buffer. Guaranteed to not
2312 make the path longer, and will not fail. 'size' is the length of
2313 the path, if known. If -1, the first null character will be assumed
2314 to be the end of the path. */
2315 wchar_t *
2316 _Py_normpath(wchar_t *path, Py_ssize_t size)
2317 {
2318 Py_ssize_t norm_length;
2319 return _Py_normpath_and_size(path, size, &norm_length);
2320 }
2321
2322
2323 /* Get the current directory. buflen is the buffer size in wide characters
2324 including the null character. Decode the path from the locale encoding.
2325
2326 Return NULL on getcwd() error, on decoding error, or if 'buf' is
2327 too short. */
2328 wchar_t*
2329 _Py_wgetcwd(wchar_t *buf, size_t buflen)
2330 {
2331 #ifdef MS_WINDOWS
2332 int ibuflen = (int)Py_MIN(buflen, INT_MAX);
2333 return _wgetcwd(buf, ibuflen);
2334 #else
2335 char fname[MAXPATHLEN];
2336 wchar_t *wname;
2337 size_t len;
2338
2339 if (getcwd(fname, Py_ARRAY_LENGTH(fname)) == NULL)
2340 return NULL;
2341 wname = Py_DecodeLocale(fname, &len);
2342 if (wname == NULL)
2343 return NULL;
2344 /* wname must have space to store the trailing NUL character */
2345 if (buflen <= len) {
2346 PyMem_RawFree(wname);
2347 return NULL;
2348 }
2349 wcsncpy(buf, wname, buflen);
2350 PyMem_RawFree(wname);
2351 return buf;
2352 #endif
2353 }
2354
2355 /* Duplicate a file descriptor. The new file descriptor is created as
2356 non-inheritable. Return a new file descriptor on success, raise an OSError
2357 exception and return -1 on error.
2358
2359 The GIL is released to call dup(). The caller must hold the GIL. */
2360 int
2361 _Py_dup(int fd)
2362 {
2363 #ifdef MS_WINDOWS
2364 HANDLE handle;
2365 #endif
2366
2367 assert(PyGILState_Check());
2368
2369 #ifdef MS_WINDOWS
2370 handle = _Py_get_osfhandle(fd);
2371 if (handle == INVALID_HANDLE_VALUE)
2372 return -1;
2373
2374 Py_BEGIN_ALLOW_THREADS
2375 _Py_BEGIN_SUPPRESS_IPH
2376 fd = dup(fd);
2377 _Py_END_SUPPRESS_IPH
2378 Py_END_ALLOW_THREADS
2379 if (fd < 0) {
2380 PyErr_SetFromErrno(PyExc_OSError);
2381 return -1;
2382 }
2383
2384 if (_Py_set_inheritable(fd, 0, NULL) < 0) {
2385 _Py_BEGIN_SUPPRESS_IPH
2386 close(fd);
2387 _Py_END_SUPPRESS_IPH
2388 return -1;
2389 }
2390 #elif defined(HAVE_FCNTL_H) && defined(F_DUPFD_CLOEXEC)
2391 Py_BEGIN_ALLOW_THREADS
2392 _Py_BEGIN_SUPPRESS_IPH
2393 fd = fcntl(fd, F_DUPFD_CLOEXEC, 0);
2394 _Py_END_SUPPRESS_IPH
2395 Py_END_ALLOW_THREADS
2396 if (fd < 0) {
2397 PyErr_SetFromErrno(PyExc_OSError);
2398 return -1;
2399 }
2400
2401 #elif HAVE_DUP
2402 Py_BEGIN_ALLOW_THREADS
2403 _Py_BEGIN_SUPPRESS_IPH
2404 fd = dup(fd);
2405 _Py_END_SUPPRESS_IPH
2406 Py_END_ALLOW_THREADS
2407 if (fd < 0) {
2408 PyErr_SetFromErrno(PyExc_OSError);
2409 return -1;
2410 }
2411
2412 if (_Py_set_inheritable(fd, 0, NULL) < 0) {
2413 _Py_BEGIN_SUPPRESS_IPH
2414 close(fd);
2415 _Py_END_SUPPRESS_IPH
2416 return -1;
2417 }
2418 #else
2419 errno = ENOTSUP;
2420 PyErr_SetFromErrno(PyExc_OSError);
2421 return -1;
2422 #endif
2423 return fd;
2424 }
2425
2426 #ifndef MS_WINDOWS
2427 /* Get the blocking mode of the file descriptor.
2428 Return 0 if the O_NONBLOCK flag is set, 1 if the flag is cleared,
2429 raise an exception and return -1 on error. */
2430 int
2431 _Py_get_blocking(int fd)
2432 {
2433 int flags;
2434 _Py_BEGIN_SUPPRESS_IPH
2435 flags = fcntl(fd, F_GETFL, 0);
2436 _Py_END_SUPPRESS_IPH
2437 if (flags < 0) {
2438 PyErr_SetFromErrno(PyExc_OSError);
2439 return -1;
2440 }
2441
2442 return !(flags & O_NONBLOCK);
2443 }
2444
2445 /* Set the blocking mode of the specified file descriptor.
2446
2447 Set the O_NONBLOCK flag if blocking is False, clear the O_NONBLOCK flag
2448 otherwise.
2449
2450 Return 0 on success, raise an exception and return -1 on error. */
2451 int
2452 _Py_set_blocking(int fd, int blocking)
2453 {
2454 /* bpo-41462: On VxWorks, ioctl(FIONBIO) only works on sockets.
2455 Use fcntl() instead. */
2456 #if defined(HAVE_SYS_IOCTL_H) && defined(FIONBIO) && !defined(__VXWORKS__)
2457 int arg = !blocking;
2458 if (ioctl(fd, FIONBIO, &arg) < 0)
2459 goto error;
2460 #else
2461 int flags, res;
2462
2463 _Py_BEGIN_SUPPRESS_IPH
2464 flags = fcntl(fd, F_GETFL, 0);
2465 if (flags >= 0) {
2466 if (blocking)
2467 flags = flags & (~O_NONBLOCK);
2468 else
2469 flags = flags | O_NONBLOCK;
2470
2471 res = fcntl(fd, F_SETFL, flags);
2472 } else {
2473 res = -1;
2474 }
2475 _Py_END_SUPPRESS_IPH
2476
2477 if (res < 0)
2478 goto error;
2479 #endif
2480 return 0;
2481
2482 error:
2483 PyErr_SetFromErrno(PyExc_OSError);
2484 return -1;
2485 }
2486 #else /* MS_WINDOWS */
2487 void*
2488 _Py_get_osfhandle_noraise(int fd)
2489 {
2490 void *handle;
2491 _Py_BEGIN_SUPPRESS_IPH
2492 handle = (void*)_get_osfhandle(fd);
2493 _Py_END_SUPPRESS_IPH
2494 return handle;
2495 }
2496
2497 void*
2498 _Py_get_osfhandle(int fd)
2499 {
2500 void *handle = _Py_get_osfhandle_noraise(fd);
2501 if (handle == INVALID_HANDLE_VALUE)
2502 PyErr_SetFromErrno(PyExc_OSError);
2503
2504 return handle;
2505 }
2506
2507 int
2508 _Py_open_osfhandle_noraise(void *handle, int flags)
2509 {
2510 int fd;
2511 _Py_BEGIN_SUPPRESS_IPH
2512 fd = _open_osfhandle((intptr_t)handle, flags);
2513 _Py_END_SUPPRESS_IPH
2514 return fd;
2515 }
2516
2517 int
2518 _Py_open_osfhandle(void *handle, int flags)
2519 {
2520 int fd = _Py_open_osfhandle_noraise(handle, flags);
2521 if (fd == -1)
2522 PyErr_SetFromErrno(PyExc_OSError);
2523
2524 return fd;
2525 }
2526 #endif /* MS_WINDOWS */
2527
2528 int
2529 _Py_GetLocaleconvNumeric(struct lconv *lc,
2530 PyObject **decimal_point, PyObject **thousands_sep)
2531 {
2532 assert(decimal_point != NULL);
2533 assert(thousands_sep != NULL);
2534
2535 #ifndef MS_WINDOWS
2536 int change_locale = 0;
2537 if ((strlen(lc->decimal_point) > 1 || ((unsigned char)lc->decimal_point[0]) > 127)) {
2538 change_locale = 1;
2539 }
2540 if ((strlen(lc->thousands_sep) > 1 || ((unsigned char)lc->thousands_sep[0]) > 127)) {
2541 change_locale = 1;
2542 }
2543
2544 /* Keep a copy of the LC_CTYPE locale */
2545 char *oldloc = NULL, *loc = NULL;
2546 if (change_locale) {
2547 oldloc = setlocale(LC_CTYPE, NULL);
2548 if (!oldloc) {
2549 PyErr_SetString(PyExc_RuntimeWarning,
2550 "failed to get LC_CTYPE locale");
2551 return -1;
2552 }
2553
2554 oldloc = _PyMem_Strdup(oldloc);
2555 if (!oldloc) {
2556 PyErr_NoMemory();
2557 return -1;
2558 }
2559
2560 loc = setlocale(LC_NUMERIC, NULL);
2561 if (loc != NULL && strcmp(loc, oldloc) == 0) {
2562 loc = NULL;
2563 }
2564
2565 if (loc != NULL) {
2566 /* Only set the locale temporarily the LC_CTYPE locale
2567 if LC_NUMERIC locale is different than LC_CTYPE locale and
2568 decimal_point and/or thousands_sep are non-ASCII or longer than
2569 1 byte */
2570 setlocale(LC_CTYPE, loc);
2571 }
2572 }
2573
2574 #define GET_LOCALE_STRING(ATTR) PyUnicode_DecodeLocale(lc->ATTR, NULL)
2575 #else /* MS_WINDOWS */
2576 /* Use _W_* fields of Windows strcut lconv */
2577 #define GET_LOCALE_STRING(ATTR) PyUnicode_FromWideChar(lc->_W_ ## ATTR, -1)
2578 #endif /* MS_WINDOWS */
2579
2580 int res = -1;
2581
2582 *decimal_point = GET_LOCALE_STRING(decimal_point);
2583 if (*decimal_point == NULL) {
2584 goto done;
2585 }
2586
2587 *thousands_sep = GET_LOCALE_STRING(thousands_sep);
2588 if (*thousands_sep == NULL) {
2589 goto done;
2590 }
2591
2592 res = 0;
2593
2594 done:
2595 #ifndef MS_WINDOWS
2596 if (loc != NULL) {
2597 setlocale(LC_CTYPE, oldloc);
2598 }
2599 PyMem_Free(oldloc);
2600 #endif
2601 return res;
2602
2603 #undef GET_LOCALE_STRING
2604 }
2605
2606 /* Our selection logic for which function to use is as follows:
2607 * 1. If close_range(2) is available, always prefer that; it's better for
2608 * contiguous ranges like this than fdwalk(3) which entails iterating over
2609 * the entire fd space and simply doing nothing for those outside the range.
2610 * 2. If closefrom(2) is available, we'll attempt to use that next if we're
2611 * closing up to sysconf(_SC_OPEN_MAX).
2612 * 2a. Fallback to fdwalk(3) if we're not closing up to sysconf(_SC_OPEN_MAX),
2613 * as that will be more performant if the range happens to have any chunk of
2614 * non-opened fd in the middle.
2615 * 2b. If fdwalk(3) isn't available, just do a plain close(2) loop.
2616 */
2617 #ifdef __FreeBSD__
2618 # define USE_CLOSEFROM
2619 #endif /* __FreeBSD__ */
2620
2621 #ifdef HAVE_FDWALK
2622 # define USE_FDWALK
2623 #endif /* HAVE_FDWALK */
2624
2625 #ifdef USE_FDWALK
2626 static int
2627 _fdwalk_close_func(void *lohi, int fd)
2628 {
2629 int lo = ((int *)lohi)[0];
2630 int hi = ((int *)lohi)[1];
2631
2632 if (fd >= hi) {
2633 return 1;
2634 }
2635 else if (fd >= lo) {
2636 /* Ignore errors */
2637 (void)close(fd);
2638 }
2639 return 0;
2640 }
2641 #endif /* USE_FDWALK */
2642
2643 /* Closes all file descriptors in [first, last], ignoring errors. */
2644 void
2645 _Py_closerange(int first, int last)
2646 {
2647 first = Py_MAX(first, 0);
2648 _Py_BEGIN_SUPPRESS_IPH
2649 #ifdef HAVE_CLOSE_RANGE
2650 if (close_range(first, last, 0) == 0) {
2651 /* close_range() ignores errors when it closes file descriptors.
2652 * Possible reasons of an error return are lack of kernel support
2653 * or denial of the underlying syscall by a seccomp sandbox on Linux.
2654 * Fallback to other methods in case of any error. */
2655 }
2656 else
2657 #endif /* HAVE_CLOSE_RANGE */
2658 #ifdef USE_CLOSEFROM
2659 if (last >= sysconf(_SC_OPEN_MAX)) {
2660 /* Any errors encountered while closing file descriptors are ignored */
2661 closefrom(first);
2662 }
2663 else
2664 #endif /* USE_CLOSEFROM */
2665 #ifdef USE_FDWALK
2666 {
2667 int lohi[2];
2668 lohi[0] = first;
2669 lohi[1] = last + 1;
2670 fdwalk(_fdwalk_close_func, lohi);
2671 }
2672 #else
2673 {
2674 for (int i = first; i <= last; i++) {
2675 /* Ignore errors */
2676 (void)close(i);
2677 }
2678 }
2679 #endif /* USE_FDWALK */
2680 _Py_END_SUPPRESS_IPH
2681 }