1 /*
2 * C Extension module to test Python internal C APIs (Include/internal).
3 */
4
5 #ifndef Py_BUILD_CORE_BUILTIN
6 # define Py_BUILD_CORE_MODULE 1
7 #endif
8
9 /* Always enable assertions */
10 #undef NDEBUG
11
12 #define PY_SSIZE_T_CLEAN
13
14 #include "Python.h"
15 #include "pycore_atomic_funcs.h" // _Py_atomic_int_get()
16 #include "pycore_bitutils.h" // _Py_bswap32()
17 #include "pycore_bytesobject.h" // _PyBytes_Find()
18 #include "pycore_fileutils.h" // _Py_normpath
19 #include "pycore_frame.h" // _PyInterpreterFrame
20 #include "pycore_gc.h" // PyGC_Head
21 #include "pycore_hashtable.h" // _Py_hashtable_new()
22 #include "pycore_initconfig.h" // _Py_GetConfigsAsDict()
23 #include "pycore_pathconfig.h" // _PyPathConfig_ClearGlobal()
24 #include "pycore_interp.h" // _PyInterpreterState_GetConfigCopy()
25 #include "pycore_pyerrors.h" // _Py_UTF8_Edit_Cost()
26 #include "pycore_pystate.h" // _PyThreadState_GET()
27 #include "osdefs.h" // MAXPATHLEN
28
29
30 static PyObject *
31 get_configs(PyObject *self, PyObject *Py_UNUSED(args))
32 {
33 return _Py_GetConfigsAsDict();
34 }
35
36
37 static PyObject*
38 get_recursion_depth(PyObject *self, PyObject *Py_UNUSED(args))
39 {
40 PyThreadState *tstate = _PyThreadState_GET();
41
42 /* subtract one to ignore the frame of the get_recursion_depth() call */
43
44 return PyLong_FromLong(tstate->recursion_limit - tstate->recursion_remaining - 1);
45 }
46
47
48 static PyObject*
49 test_bswap(PyObject *self, PyObject *Py_UNUSED(args))
50 {
51 uint16_t u16 = _Py_bswap16(UINT16_C(0x3412));
52 if (u16 != UINT16_C(0x1234)) {
53 PyErr_Format(PyExc_AssertionError,
54 "_Py_bswap16(0x3412) returns %u", u16);
55 return NULL;
56 }
57
58 uint32_t u32 = _Py_bswap32(UINT32_C(0x78563412));
59 if (u32 != UINT32_C(0x12345678)) {
60 PyErr_Format(PyExc_AssertionError,
61 "_Py_bswap32(0x78563412) returns %lu", u32);
62 return NULL;
63 }
64
65 uint64_t u64 = _Py_bswap64(UINT64_C(0xEFCDAB9078563412));
66 if (u64 != UINT64_C(0x1234567890ABCDEF)) {
67 PyErr_Format(PyExc_AssertionError,
68 "_Py_bswap64(0xEFCDAB9078563412) returns %llu", u64);
69 return NULL;
70 }
71
72 Py_RETURN_NONE;
73 }
74
75
76 static int
77 check_popcount(uint32_t x, int expected)
78 {
79 // Use volatile to prevent the compiler to optimize out the whole test
80 volatile uint32_t u = x;
81 int bits = _Py_popcount32(u);
82 if (bits != expected) {
83 PyErr_Format(PyExc_AssertionError,
84 "_Py_popcount32(%lu) returns %i, expected %i",
85 (unsigned long)x, bits, expected);
86 return -1;
87 }
88 return 0;
89 }
90
91
92 static PyObject*
93 test_popcount(PyObject *self, PyObject *Py_UNUSED(args))
94 {
95 #define CHECK(X, RESULT) \
96 do { \
97 if (check_popcount(X, RESULT) < 0) { \
98 return NULL; \
99 } \
100 } while (0)
101
102 CHECK(0, 0);
103 CHECK(1, 1);
104 CHECK(0x08080808, 4);
105 CHECK(0x10000001, 2);
106 CHECK(0x10101010, 4);
107 CHECK(0x10204080, 4);
108 CHECK(0xDEADCAFE, 22);
109 CHECK(0xFFFFFFFF, 32);
110 Py_RETURN_NONE;
111
112 #undef CHECK
113 }
114
115
116 static int
117 check_bit_length(unsigned long x, int expected)
118 {
119 // Use volatile to prevent the compiler to optimize out the whole test
120 volatile unsigned long u = x;
121 int len = _Py_bit_length(u);
122 if (len != expected) {
123 PyErr_Format(PyExc_AssertionError,
124 "_Py_bit_length(%lu) returns %i, expected %i",
125 x, len, expected);
126 return -1;
127 }
128 return 0;
129 }
130
131
132 static PyObject*
133 test_bit_length(PyObject *self, PyObject *Py_UNUSED(args))
134 {
135 #define CHECK(X, RESULT) \
136 do { \
137 if (check_bit_length(X, RESULT) < 0) { \
138 return NULL; \
139 } \
140 } while (0)
141
142 CHECK(0, 0);
143 CHECK(1, 1);
144 CHECK(0x1000, 13);
145 CHECK(0x1234, 13);
146 CHECK(0x54321, 19);
147 CHECK(0x7FFFFFFF, 31);
148 CHECK(0xFFFFFFFF, 32);
149 Py_RETURN_NONE;
150
151 #undef CHECK
152 }
153
154
155 #define TO_PTR(ch) ((void*)(uintptr_t)ch)
156 #define FROM_PTR(ptr) ((uintptr_t)ptr)
157 #define VALUE(key) (1 + ((int)(key) - 'a'))
158
159 static Py_uhash_t
160 hash_char(const void *key)
161 {
162 char ch = (char)FROM_PTR(key);
163 return ch;
164 }
165
166
167 static int
168 hashtable_cb(_Py_hashtable_t *table,
169 const void *key_ptr, const void *value_ptr,
170 void *user_data)
171 {
172 int *count = (int *)user_data;
173 char key = (char)FROM_PTR(key_ptr);
174 int value = (int)FROM_PTR(value_ptr);
175 assert(value == VALUE(key));
176 *count += 1;
177 return 0;
178 }
179
180
181 static PyObject*
182 test_hashtable(PyObject *self, PyObject *Py_UNUSED(args))
183 {
184 _Py_hashtable_t *table = _Py_hashtable_new(hash_char,
185 _Py_hashtable_compare_direct);
186 if (table == NULL) {
187 return PyErr_NoMemory();
188 }
189
190 // Using an newly allocated table must not crash
191 assert(table->nentries == 0);
192 assert(table->nbuckets > 0);
193 assert(_Py_hashtable_get(table, TO_PTR('x')) == NULL);
194
195 // Test _Py_hashtable_set()
196 char key;
197 for (key='a'; key <= 'z'; key++) {
198 int value = VALUE(key);
199 if (_Py_hashtable_set(table, TO_PTR(key), TO_PTR(value)) < 0) {
200 _Py_hashtable_destroy(table);
201 return PyErr_NoMemory();
202 }
203 }
204 assert(table->nentries == 26);
205 assert(table->nbuckets > table->nentries);
206
207 // Test _Py_hashtable_get_entry()
208 for (key='a'; key <= 'z'; key++) {
209 _Py_hashtable_entry_t *entry = _Py_hashtable_get_entry(table, TO_PTR(key));
210 assert(entry != NULL);
211 assert(entry->key == TO_PTR(key));
212 assert(entry->value == TO_PTR(VALUE(key)));
213 }
214
215 // Test _Py_hashtable_get()
216 for (key='a'; key <= 'z'; key++) {
217 void *value_ptr = _Py_hashtable_get(table, TO_PTR(key));
218 assert((int)FROM_PTR(value_ptr) == VALUE(key));
219 }
220
221 // Test _Py_hashtable_steal()
222 key = 'p';
223 void *value_ptr = _Py_hashtable_steal(table, TO_PTR(key));
224 assert((int)FROM_PTR(value_ptr) == VALUE(key));
225 assert(table->nentries == 25);
226 assert(_Py_hashtable_get_entry(table, TO_PTR(key)) == NULL);
227
228 // Test _Py_hashtable_foreach()
229 int count = 0;
230 int res = _Py_hashtable_foreach(table, hashtable_cb, &count);
231 assert(res == 0);
232 assert(count == 25);
233
234 // Test _Py_hashtable_clear()
235 _Py_hashtable_clear(table);
236 assert(table->nentries == 0);
237 assert(table->nbuckets > 0);
238 assert(_Py_hashtable_get(table, TO_PTR('x')) == NULL);
239
240 _Py_hashtable_destroy(table);
241 Py_RETURN_NONE;
242 }
243
244
245 static PyObject *
246 test_get_config(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args))
247 {
248 PyConfig config;
249 PyConfig_InitIsolatedConfig(&config);
250 if (_PyInterpreterState_GetConfigCopy(&config) < 0) {
251 PyConfig_Clear(&config);
252 return NULL;
253 }
254 PyObject *dict = _PyConfig_AsDict(&config);
255 PyConfig_Clear(&config);
256 return dict;
257 }
258
259
260 static PyObject *
261 test_set_config(PyObject *Py_UNUSED(self), PyObject *dict)
262 {
263 PyConfig config;
264 PyConfig_InitIsolatedConfig(&config);
265 if (_PyConfig_FromDict(&config, dict) < 0) {
266 goto error;
267 }
268 if (_PyInterpreterState_SetConfig(&config) < 0) {
269 goto error;
270 }
271 PyConfig_Clear(&config);
272 Py_RETURN_NONE;
273
274 error:
275 PyConfig_Clear(&config);
276 return NULL;
277 }
278
279
280 static PyObject *
281 test_reset_path_config(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(arg))
282 {
283 _PyPathConfig_ClearGlobal();
284 Py_RETURN_NONE;
285 }
286
287
288 static PyObject*
289 test_atomic_funcs(PyObject *self, PyObject *Py_UNUSED(args))
290 {
291 // Test _Py_atomic_size_get() and _Py_atomic_size_set()
292 Py_ssize_t var = 1;
293 _Py_atomic_size_set(&var, 2);
294 assert(_Py_atomic_size_get(&var) == 2);
295 Py_RETURN_NONE;
296 }
297
298
299 static int
300 check_edit_cost(const char *a, const char *b, Py_ssize_t expected)
301 {
302 int ret = -1;
303 PyObject *a_obj = NULL;
304 PyObject *b_obj = NULL;
305
306 a_obj = PyUnicode_FromString(a);
307 if (a_obj == NULL) {
308 goto exit;
309 }
310 b_obj = PyUnicode_FromString(b);
311 if (b_obj == NULL) {
312 goto exit;
313 }
314 Py_ssize_t result = _Py_UTF8_Edit_Cost(a_obj, b_obj, -1);
315 if (result != expected) {
316 PyErr_Format(PyExc_AssertionError,
317 "Edit cost from '%s' to '%s' returns %zd, expected %zd",
318 a, b, result, expected);
319 goto exit;
320 }
321 // Check that smaller max_edits thresholds are exceeded.
322 Py_ssize_t max_edits = result;
323 while (max_edits > 0) {
324 max_edits /= 2;
325 Py_ssize_t result2 = _Py_UTF8_Edit_Cost(a_obj, b_obj, max_edits);
326 if (result2 <= max_edits) {
327 PyErr_Format(PyExc_AssertionError,
328 "Edit cost from '%s' to '%s' (threshold %zd) "
329 "returns %zd, expected greater than %zd",
330 a, b, max_edits, result2, max_edits);
331 goto exit;
332 }
333 }
334 // Check that bigger max_edits thresholds don't change anything
335 Py_ssize_t result3 = _Py_UTF8_Edit_Cost(a_obj, b_obj, result * 2 + 1);
336 if (result3 != result) {
337 PyErr_Format(PyExc_AssertionError,
338 "Edit cost from '%s' to '%s' (threshold %zd) "
339 "returns %zd, expected %zd",
340 a, b, result * 2, result3, result);
341 goto exit;
342 }
343 ret = 0;
344 exit:
345 Py_XDECREF(a_obj);
346 Py_XDECREF(b_obj);
347 return ret;
348 }
349
350 static PyObject *
351 test_edit_cost(PyObject *self, PyObject *Py_UNUSED(args))
352 {
353 #define CHECK(a, b, n) do { \
354 if (check_edit_cost(a, b, n) < 0) { \
355 return NULL; \
356 } \
357 } while (0) \
358
359 CHECK("", "", 0);
360 CHECK("", "a", 2);
361 CHECK("a", "A", 1);
362 CHECK("Apple", "Aple", 2);
363 CHECK("Banana", "B@n@n@", 6);
364 CHECK("Cherry", "Cherry!", 2);
365 CHECK("---0---", "------", 2);
366 CHECK("abc", "y", 6);
367 CHECK("aa", "bb", 4);
368 CHECK("aaaaa", "AAAAA", 5);
369 CHECK("wxyz", "wXyZ", 2);
370 CHECK("wxyz", "wXyZ123", 8);
371 CHECK("Python", "Java", 12);
372 CHECK("Java", "C#", 8);
373 CHECK("AbstractFoobarManager", "abstract_foobar_manager", 3+2*2);
374 CHECK("CPython", "PyPy", 10);
375 CHECK("CPython", "pypy", 11);
376 CHECK("AttributeError", "AttributeErrop", 2);
377 CHECK("AttributeError", "AttributeErrorTests", 10);
378
379 #undef CHECK
380 Py_RETURN_NONE;
381 }
382
383
384 static int
385 check_bytes_find(const char *haystack0, const char *needle0,
386 int offset, Py_ssize_t expected)
387 {
388 Py_ssize_t len_haystack = strlen(haystack0);
389 Py_ssize_t len_needle = strlen(needle0);
390 Py_ssize_t result_1 = _PyBytes_Find(haystack0, len_haystack,
391 needle0, len_needle, offset);
392 if (result_1 != expected) {
393 PyErr_Format(PyExc_AssertionError,
394 "Incorrect result_1: '%s' in '%s' (offset=%zd)",
395 needle0, haystack0, offset);
396 return -1;
397 }
398 // Allocate new buffer with no NULL terminator.
399 char *haystack = PyMem_Malloc(len_haystack);
400 if (haystack == NULL) {
401 PyErr_NoMemory();
402 return -1;
403 }
404 char *needle = PyMem_Malloc(len_needle);
405 if (needle == NULL) {
406 PyMem_Free(haystack);
407 PyErr_NoMemory();
408 return -1;
409 }
410 memcpy(haystack, haystack0, len_haystack);
411 memcpy(needle, needle0, len_needle);
412 Py_ssize_t result_2 = _PyBytes_Find(haystack, len_haystack,
413 needle, len_needle, offset);
414 PyMem_Free(haystack);
415 PyMem_Free(needle);
416 if (result_2 != expected) {
417 PyErr_Format(PyExc_AssertionError,
418 "Incorrect result_2: '%s' in '%s' (offset=%zd)",
419 needle0, haystack0, offset);
420 return -1;
421 }
422 return 0;
423 }
424
425 static int
426 check_bytes_find_large(Py_ssize_t len_haystack, Py_ssize_t len_needle,
427 const char *needle)
428 {
429 char *zeros = PyMem_RawCalloc(len_haystack, 1);
430 if (zeros == NULL) {
431 PyErr_NoMemory();
432 return -1;
433 }
434 Py_ssize_t res = _PyBytes_Find(zeros, len_haystack, needle, len_needle, 0);
435 PyMem_RawFree(zeros);
436 if (res != -1) {
437 PyErr_Format(PyExc_AssertionError,
438 "check_bytes_find_large(%zd, %zd) found %zd",
439 len_haystack, len_needle, res);
440 return -1;
441 }
442 return 0;
443 }
444
445 static PyObject *
446 test_bytes_find(PyObject *self, PyObject *Py_UNUSED(args))
447 {
448 #define CHECK(H, N, O, E) do { \
449 if (check_bytes_find(H, N, O, E) < 0) { \
450 return NULL; \
451 } \
452 } while (0)
453
454 CHECK("", "", 0, 0);
455 CHECK("Python", "", 0, 0);
456 CHECK("Python", "", 3, 3);
457 CHECK("Python", "", 6, 6);
458 CHECK("Python", "yth", 0, 1);
459 CHECK("ython", "yth", 1, 1);
460 CHECK("thon", "yth", 2, -1);
461 CHECK("Python", "thon", 0, 2);
462 CHECK("ython", "thon", 1, 2);
463 CHECK("thon", "thon", 2, 2);
464 CHECK("hon", "thon", 3, -1);
465 CHECK("Pytho", "zz", 0, -1);
466 CHECK("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "ab", 0, -1);
467 CHECK("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "ba", 0, -1);
468 CHECK("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "bb", 0, -1);
469 CHECK("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab", "ab", 0, 30);
470 CHECK("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaba", "ba", 0, 30);
471 CHECK("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaabb", "bb", 0, 30);
472 #undef CHECK
473
474 // Hunt for segfaults
475 // n, m chosen here so that (n - m) % (m + 1) == 0
476 // This would make default_find in fastsearch.h access haystack[n].
477 if (check_bytes_find_large(2048, 2, "ab") < 0) {
478 return NULL;
479 }
480 if (check_bytes_find_large(4096, 16, "0123456789abcdef") < 0) {
481 return NULL;
482 }
483 if (check_bytes_find_large(8192, 2, "ab") < 0) {
484 return NULL;
485 }
486 if (check_bytes_find_large(16384, 4, "abcd") < 0) {
487 return NULL;
488 }
489 if (check_bytes_find_large(32768, 2, "ab") < 0) {
490 return NULL;
491 }
492 Py_RETURN_NONE;
493 }
494
495
496 static PyObject *
497 normalize_path(PyObject *self, PyObject *filename)
498 {
499 Py_ssize_t size = -1;
500 wchar_t *encoded = PyUnicode_AsWideCharString(filename, &size);
501 if (encoded == NULL) {
502 return NULL;
503 }
504
505 PyObject *result = PyUnicode_FromWideChar(_Py_normpath(encoded, size), -1);
506 PyMem_Free(encoded);
507
508 return result;
509 }
510
511 static PyObject *
512 get_getpath_codeobject(PyObject *self, PyObject *Py_UNUSED(args)) {
513 return _Py_Get_Getpath_CodeObject();
514 }
515
516
517 static PyObject *
518 encode_locale_ex(PyObject *self, PyObject *args)
519 {
520 PyObject *unicode;
521 int current_locale = 0;
522 wchar_t *wstr;
523 PyObject *res = NULL;
524 const char *errors = NULL;
525
526 if (!PyArg_ParseTuple(args, "U|is", &unicode, ¤t_locale, &errors)) {
527 return NULL;
528 }
529 wstr = PyUnicode_AsWideCharString(unicode, NULL);
530 if (wstr == NULL) {
531 return NULL;
532 }
533 _Py_error_handler error_handler = _Py_GetErrorHandler(errors);
534
535 char *str = NULL;
536 size_t error_pos;
537 const char *reason = NULL;
538 int ret = _Py_EncodeLocaleEx(wstr,
539 &str, &error_pos, &reason,
540 current_locale, error_handler);
541 PyMem_Free(wstr);
542
543 switch(ret) {
544 case 0:
545 res = PyBytes_FromString(str);
546 PyMem_RawFree(str);
547 break;
548 case -1:
549 PyErr_NoMemory();
550 break;
551 case -2:
552 PyErr_Format(PyExc_RuntimeError, "encode error: pos=%zu, reason=%s",
553 error_pos, reason);
554 break;
555 case -3:
556 PyErr_SetString(PyExc_ValueError, "unsupported error handler");
557 break;
558 default:
559 PyErr_SetString(PyExc_ValueError, "unknown error code");
560 break;
561 }
562 return res;
563 }
564
565
566 static PyObject *
567 decode_locale_ex(PyObject *self, PyObject *args)
568 {
569 char *str;
570 int current_locale = 0;
571 PyObject *res = NULL;
572 const char *errors = NULL;
573
574 if (!PyArg_ParseTuple(args, "y|is", &str, ¤t_locale, &errors)) {
575 return NULL;
576 }
577 _Py_error_handler error_handler = _Py_GetErrorHandler(errors);
578
579 wchar_t *wstr = NULL;
580 size_t wlen = 0;
581 const char *reason = NULL;
582 int ret = _Py_DecodeLocaleEx(str,
583 &wstr, &wlen, &reason,
584 current_locale, error_handler);
585
586 switch(ret) {
587 case 0:
588 res = PyUnicode_FromWideChar(wstr, wlen);
589 PyMem_RawFree(wstr);
590 break;
591 case -1:
592 PyErr_NoMemory();
593 break;
594 case -2:
595 PyErr_Format(PyExc_RuntimeError, "decode error: pos=%zu, reason=%s",
596 wlen, reason);
597 break;
598 case -3:
599 PyErr_SetString(PyExc_ValueError, "unsupported error handler");
600 break;
601 default:
602 PyErr_SetString(PyExc_ValueError, "unknown error code");
603 break;
604 }
605 return res;
606 }
607
608 static PyObject *record_list = NULL;
609
610 static PyObject *
611 set_eval_frame_default(PyObject *self, PyObject *Py_UNUSED(args))
612 {
613 _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState_Get(), _PyEval_EvalFrameDefault);
614 Py_CLEAR(record_list);
615 Py_RETURN_NONE;
616 }
617
618 static PyObject *
619 record_eval(PyThreadState *tstate, struct _PyInterpreterFrame *f, int exc)
620 {
621 PyList_Append(record_list, f->f_func->func_name);
622 return _PyEval_EvalFrameDefault(tstate, f, exc);
623 }
624
625
626 static PyObject *
627 set_eval_frame_record(PyObject *self, PyObject *list)
628 {
629 if (!PyList_Check(list)) {
630 PyErr_SetString(PyExc_TypeError, "argument must be a list");
631 return NULL;
632 }
633 Py_CLEAR(record_list);
634 Py_INCREF(list);
635 record_list = list;
636 _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState_Get(), record_eval);
637 Py_RETURN_NONE;
638 }
639
640
641 static PyMethodDef TestMethods[] = {
642 {"get_configs", get_configs, METH_NOARGS},
643 {"get_recursion_depth", get_recursion_depth, METH_NOARGS},
644 {"test_bswap", test_bswap, METH_NOARGS},
645 {"test_popcount", test_popcount, METH_NOARGS},
646 {"test_bit_length", test_bit_length, METH_NOARGS},
647 {"test_hashtable", test_hashtable, METH_NOARGS},
648 {"get_config", test_get_config, METH_NOARGS},
649 {"set_config", test_set_config, METH_O},
650 {"reset_path_config", test_reset_path_config, METH_NOARGS},
651 {"test_atomic_funcs", test_atomic_funcs, METH_NOARGS},
652 {"test_edit_cost", test_edit_cost, METH_NOARGS},
653 {"test_bytes_find", test_bytes_find, METH_NOARGS},
654 {"normalize_path", normalize_path, METH_O, NULL},
655 {"get_getpath_codeobject", get_getpath_codeobject, METH_NOARGS, NULL},
656 {"EncodeLocaleEx", encode_locale_ex, METH_VARARGS},
657 {"DecodeLocaleEx", decode_locale_ex, METH_VARARGS},
658 {"set_eval_frame_default", set_eval_frame_default, METH_NOARGS, NULL},
659 {"set_eval_frame_record", set_eval_frame_record, METH_O, NULL},
660 {NULL, NULL} /* sentinel */
661 };
662
663
664 static struct PyModuleDef _testcapimodule = {
665 PyModuleDef_HEAD_INIT,
666 "_testinternalcapi",
667 NULL,
668 -1,
669 TestMethods,
670 NULL,
671 NULL,
672 NULL,
673 NULL
674 };
675
676
677 PyMODINIT_FUNC
678 PyInit__testinternalcapi(void)
679 {
680 PyObject *module = PyModule_Create(&_testcapimodule);
681 if (module == NULL) {
682 return NULL;
683 }
684
685 if (_PyModule_Add(module, "SIZEOF_PYGC_HEAD",
686 PyLong_FromSsize_t(sizeof(PyGC_Head))) < 0) {
687 goto error;
688 }
689
690 return module;
691
692 error:
693 Py_DECREF(module);
694 return NULL;
695 }