1
2 /* Traceback implementation */
3
4 #include "Python.h"
5
6 #include "pycore_ast.h" // asdl_seq_*
7 #include "pycore_call.h" // _PyObject_CallMethodFormat()
8 #include "pycore_compile.h" // _PyAST_Optimize
9 #include "pycore_fileutils.h" // _Py_BEGIN_SUPPRESS_IPH
10 #include "pycore_frame.h" // _PyFrame_GetCode()
11 #include "pycore_interp.h" // PyInterpreterState.gc
12 #include "pycore_parser.h" // _PyParser_ASTFromString
13 #include "pycore_pyarena.h" // _PyArena_Free()
14 #include "pycore_pyerrors.h" // _PyErr_Fetch()
15 #include "pycore_pymem.h" // _PyMem_IsPtrFreed()
16 #include "pycore_pystate.h" // _PyThreadState_GET()
17 #include "pycore_traceback.h" // EXCEPTION_TB_HEADER
18
19 #include "../Parser/pegen.h" // _PyPegen_byte_offset_to_character_offset()
20 #include "frameobject.h" // PyFrame_New()
21 #include "structmember.h" // PyMemberDef
22 #include "osdefs.h" // SEP
23 #ifdef HAVE_FCNTL_H
24 # include <fcntl.h>
25 #endif
26
27 #define OFF(x) offsetof(PyTracebackObject, x)
28
29 #define PUTS(fd, str) _Py_write_noraise(fd, str, (int)strlen(str))
30 #define MAX_STRING_LENGTH 500
31 #define MAX_FRAME_DEPTH 100
32 #define MAX_NTHREADS 100
33
34 /* Function from Parser/tokenizer.c */
35 extern char* _PyTokenizer_FindEncodingFilename(int, PyObject *);
36
37 /*[clinic input]
38 class TracebackType "PyTracebackObject *" "&PyTraceback_Type"
39 [clinic start generated code]*/
40 /*[clinic end generated code: output=da39a3ee5e6b4b0d input=928fa06c10151120]*/
41
42 #include "clinic/traceback.c.h"
43
44 static PyObject *
45 tb_create_raw(PyTracebackObject *next, PyFrameObject *frame, int lasti,
46 int lineno)
47 {
48 PyTracebackObject *tb;
49 if ((next != NULL && !PyTraceBack_Check(next)) ||
50 frame == NULL || !PyFrame_Check(frame)) {
51 PyErr_BadInternalCall();
52 return NULL;
53 }
54 tb = PyObject_GC_New(PyTracebackObject, &PyTraceBack_Type);
55 if (tb != NULL) {
56 Py_XINCREF(next);
57 tb->tb_next = next;
58 Py_XINCREF(frame);
59 tb->tb_frame = frame;
60 tb->tb_lasti = lasti;
61 tb->tb_lineno = lineno;
62 PyObject_GC_Track(tb);
63 }
64 return (PyObject *)tb;
65 }
66
67 /*[clinic input]
68 @classmethod
69 TracebackType.__new__ as tb_new
70
71 tb_next: object
72 tb_frame: object(type='PyFrameObject *', subclass_of='&PyFrame_Type')
73 tb_lasti: int
74 tb_lineno: int
75
76 Create a new traceback object.
77 [clinic start generated code]*/
78
79 static PyObject *
80 tb_new_impl(PyTypeObject *type, PyObject *tb_next, PyFrameObject *tb_frame,
81 int tb_lasti, int tb_lineno)
82 /*[clinic end generated code: output=fa077debd72d861a input=01cbe8ec8783fca7]*/
83 {
84 if (tb_next == Py_None) {
85 tb_next = NULL;
86 } else if (!PyTraceBack_Check(tb_next)) {
87 return PyErr_Format(PyExc_TypeError,
88 "expected traceback object or None, got '%s'",
89 Py_TYPE(tb_next)->tp_name);
90 }
91
92 return tb_create_raw((PyTracebackObject *)tb_next, tb_frame, tb_lasti,
93 tb_lineno);
94 }
95
96 static PyObject *
97 tb_dir(PyTracebackObject *self, PyObject *Py_UNUSED(ignored))
98 {
99 return Py_BuildValue("[ssss]", "tb_frame", "tb_next",
100 "tb_lasti", "tb_lineno");
101 }
102
103 static PyObject *
104 tb_next_get(PyTracebackObject *self, void *Py_UNUSED(_))
105 {
106 PyObject* ret = (PyObject*)self->tb_next;
107 if (!ret) {
108 ret = Py_None;
109 }
110 Py_INCREF(ret);
111 return ret;
112 }
113
114 static int
115 tb_get_lineno(PyTracebackObject* tb) {
116 PyFrameObject* frame = tb->tb_frame;
117 assert(frame != NULL);
118 PyCodeObject *code = PyFrame_GetCode(frame);
119 int lineno = PyCode_Addr2Line(code, tb->tb_lasti);
120 Py_DECREF(code);
121 return lineno;
122 }
123
124 static PyObject *
125 tb_lineno_get(PyTracebackObject *self, void *Py_UNUSED(_))
126 {
127 int lineno = self->tb_lineno;
128 if (lineno == -1) {
129 lineno = tb_get_lineno(self);
130 if (lineno < 0) {
131 Py_RETURN_NONE;
132 }
133 }
134 return PyLong_FromLong(lineno);
135 }
136
137 static int
138 tb_next_set(PyTracebackObject *self, PyObject *new_next, void *Py_UNUSED(_))
139 {
140 if (!new_next) {
141 PyErr_Format(PyExc_TypeError, "can't delete tb_next attribute");
142 return -1;
143 }
144
145 /* We accept None or a traceback object, and map None -> NULL (inverse of
146 tb_next_get) */
147 if (new_next == Py_None) {
148 new_next = NULL;
149 } else if (!PyTraceBack_Check(new_next)) {
150 PyErr_Format(PyExc_TypeError,
151 "expected traceback object, got '%s'",
152 Py_TYPE(new_next)->tp_name);
153 return -1;
154 }
155
156 /* Check for loops */
157 PyTracebackObject *cursor = (PyTracebackObject *)new_next;
158 while (cursor) {
159 if (cursor == self) {
160 PyErr_Format(PyExc_ValueError, "traceback loop detected");
161 return -1;
162 }
163 cursor = cursor->tb_next;
164 }
165
166 PyObject *old_next = (PyObject*)self->tb_next;
167 Py_XINCREF(new_next);
168 self->tb_next = (PyTracebackObject *)new_next;
169 Py_XDECREF(old_next);
170
171 return 0;
172 }
173
174
175 static PyMethodDef tb_methods[] = {
176 {"__dir__", _PyCFunction_CAST(tb_dir), METH_NOARGS},
177 {NULL, NULL, 0, NULL},
178 };
179
180 static PyMemberDef tb_memberlist[] = {
181 {"tb_frame", T_OBJECT, OFF(tb_frame), READONLY|PY_AUDIT_READ},
182 {"tb_lasti", T_INT, OFF(tb_lasti), READONLY},
183 {NULL} /* Sentinel */
184 };
185
186 static PyGetSetDef tb_getsetters[] = {
187 {"tb_next", (getter)tb_next_get, (setter)tb_next_set, NULL, NULL},
188 {"tb_lineno", (getter)tb_lineno_get, NULL, NULL, NULL},
189 {NULL} /* Sentinel */
190 };
191
192 static void
193 tb_dealloc(PyTracebackObject *tb)
194 {
195 PyObject_GC_UnTrack(tb);
196 Py_TRASHCAN_BEGIN(tb, tb_dealloc)
197 Py_XDECREF(tb->tb_next);
198 Py_XDECREF(tb->tb_frame);
199 PyObject_GC_Del(tb);
200 Py_TRASHCAN_END
201 }
202
203 static int
204 tb_traverse(PyTracebackObject *tb, visitproc visit, void *arg)
205 {
206 Py_VISIT(tb->tb_next);
207 Py_VISIT(tb->tb_frame);
208 return 0;
209 }
210
211 static int
212 tb_clear(PyTracebackObject *tb)
213 {
214 Py_CLEAR(tb->tb_next);
215 Py_CLEAR(tb->tb_frame);
216 return 0;
217 }
218
219 PyTypeObject PyTraceBack_Type = {
220 PyVarObject_HEAD_INIT(&PyType_Type, 0)
221 "traceback",
222 sizeof(PyTracebackObject),
223 0,
224 (destructor)tb_dealloc, /*tp_dealloc*/
225 0, /*tp_vectorcall_offset*/
226 0, /*tp_getattr*/
227 0, /*tp_setattr*/
228 0, /*tp_as_async*/
229 0, /*tp_repr*/
230 0, /*tp_as_number*/
231 0, /*tp_as_sequence*/
232 0, /*tp_as_mapping*/
233 0, /* tp_hash */
234 0, /* tp_call */
235 0, /* tp_str */
236 PyObject_GenericGetAttr, /* tp_getattro */
237 0, /* tp_setattro */
238 0, /* tp_as_buffer */
239 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */
240 tb_new__doc__, /* tp_doc */
241 (traverseproc)tb_traverse, /* tp_traverse */
242 (inquiry)tb_clear, /* tp_clear */
243 0, /* tp_richcompare */
244 0, /* tp_weaklistoffset */
245 0, /* tp_iter */
246 0, /* tp_iternext */
247 tb_methods, /* tp_methods */
248 tb_memberlist, /* tp_members */
249 tb_getsetters, /* tp_getset */
250 0, /* tp_base */
251 0, /* tp_dict */
252 0, /* tp_descr_get */
253 0, /* tp_descr_set */
254 0, /* tp_dictoffset */
255 0, /* tp_init */
256 0, /* tp_alloc */
257 tb_new, /* tp_new */
258 };
259
260
261 PyObject*
262 _PyTraceBack_FromFrame(PyObject *tb_next, PyFrameObject *frame)
263 {
264 assert(tb_next == NULL || PyTraceBack_Check(tb_next));
265 assert(frame != NULL);
266 int addr = _PyInterpreterFrame_LASTI(frame->f_frame) * sizeof(_Py_CODEUNIT);
267 return tb_create_raw((PyTracebackObject *)tb_next, frame, addr, -1);
268 }
269
270
271 int
272 PyTraceBack_Here(PyFrameObject *frame)
273 {
274 PyObject *exc, *val, *tb, *newtb;
275 PyErr_Fetch(&exc, &val, &tb);
276 newtb = _PyTraceBack_FromFrame(tb, frame);
277 if (newtb == NULL) {
278 _PyErr_ChainExceptions(exc, val, tb);
279 return -1;
280 }
281 PyErr_Restore(exc, val, newtb);
282 Py_XDECREF(tb);
283 return 0;
284 }
285
286 /* Insert a frame into the traceback for (funcname, filename, lineno). */
287 void _PyTraceback_Add(const char *funcname, const char *filename, int lineno)
288 {
289 PyObject *globals;
290 PyCodeObject *code;
291 PyFrameObject *frame;
292 PyObject *exc, *val, *tb;
293 PyThreadState *tstate = _PyThreadState_GET();
294
295 /* Save and clear the current exception. Python functions must not be
296 called with an exception set. Calling Python functions happens when
297 the codec of the filesystem encoding is implemented in pure Python. */
298 _PyErr_Fetch(tstate, &exc, &val, &tb);
299
300 globals = PyDict_New();
301 if (!globals)
302 goto error;
303 code = PyCode_NewEmpty(filename, funcname, lineno);
304 if (!code) {
305 Py_DECREF(globals);
306 goto error;
307 }
308 frame = PyFrame_New(tstate, code, globals, NULL);
309 Py_DECREF(globals);
310 Py_DECREF(code);
311 if (!frame)
312 goto error;
313 frame->f_lineno = lineno;
314
315 _PyErr_Restore(tstate, exc, val, tb);
316 PyTraceBack_Here(frame);
317 Py_DECREF(frame);
318 return;
319
320 error:
321 _PyErr_ChainExceptions(exc, val, tb);
322 }
323
324 static PyObject *
325 _Py_FindSourceFile(PyObject *filename, char* namebuf, size_t namelen, PyObject *io)
326 {
327 Py_ssize_t i;
328 PyObject *binary;
329 PyObject *v;
330 Py_ssize_t npath;
331 size_t taillen;
332 PyObject *syspath;
333 PyObject *path;
334 const char* tail;
335 PyObject *filebytes;
336 const char* filepath;
337 Py_ssize_t len;
338 PyObject* result;
339 PyObject *open = NULL;
340
341 filebytes = PyUnicode_EncodeFSDefault(filename);
342 if (filebytes == NULL) {
343 PyErr_Clear();
344 return NULL;
345 }
346 filepath = PyBytes_AS_STRING(filebytes);
347
348 /* Search tail of filename in sys.path before giving up */
349 tail = strrchr(filepath, SEP);
350 if (tail == NULL)
351 tail = filepath;
352 else
353 tail++;
354 taillen = strlen(tail);
355
356 PyThreadState *tstate = _PyThreadState_GET();
357 syspath = _PySys_GetAttr(tstate, &_Py_ID(path));
358 if (syspath == NULL || !PyList_Check(syspath))
359 goto error;
360 npath = PyList_Size(syspath);
361
362 open = PyObject_GetAttr(io, &_Py_ID(open));
363 for (i = 0; i < npath; i++) {
364 v = PyList_GetItem(syspath, i);
365 if (v == NULL) {
366 PyErr_Clear();
367 break;
368 }
369 if (!PyUnicode_Check(v))
370 continue;
371 path = PyUnicode_EncodeFSDefault(v);
372 if (path == NULL) {
373 PyErr_Clear();
374 continue;
375 }
376 len = PyBytes_GET_SIZE(path);
377 if (len + 1 + (Py_ssize_t)taillen >= (Py_ssize_t)namelen - 1) {
378 Py_DECREF(path);
379 continue; /* Too long */
380 }
381 strcpy(namebuf, PyBytes_AS_STRING(path));
382 Py_DECREF(path);
383 if (strlen(namebuf) != (size_t)len)
384 continue; /* v contains '\0' */
385 if (len > 0 && namebuf[len-1] != SEP)
386 namebuf[len++] = SEP;
387 strcpy(namebuf+len, tail);
388
389 binary = _PyObject_CallMethodFormat(tstate, open, "ss", namebuf, "rb");
390 if (binary != NULL) {
391 result = binary;
392 goto finally;
393 }
394 PyErr_Clear();
395 }
396 goto error;
397
398 error:
399 result = NULL;
400 finally:
401 Py_XDECREF(open);
402 Py_DECREF(filebytes);
403 return result;
404 }
405
406 /* Writes indent spaces. Returns 0 on success and non-zero on failure.
407 */
408 int
409 _Py_WriteIndent(int indent, PyObject *f)
410 {
411 char buf[11] = " ";
412 assert(strlen(buf) == 10);
413 while (indent > 0) {
414 if (indent < 10) {
415 buf[indent] = '\0';
416 }
417 if (PyFile_WriteString(buf, f) < 0) {
418 return -1;
419 }
420 indent -= 10;
421 }
422 return 0;
423 }
424
425 /* Writes indent spaces, followed by the margin if it is not `\0`.
426 Returns 0 on success and non-zero on failure.
427 */
428 int
429 _Py_WriteIndentedMargin(int indent, const char *margin, PyObject *f)
430 {
431 if (_Py_WriteIndent(indent, f) < 0) {
432 return -1;
433 }
434 if (margin) {
435 if (PyFile_WriteString(margin, f) < 0) {
436 return -1;
437 }
438 }
439 return 0;
440 }
441
442 static int
443 display_source_line_with_margin(PyObject *f, PyObject *filename, int lineno, int indent,
444 int margin_indent, const char *margin,
445 int *truncation, PyObject **line)
446 {
447 int fd;
448 int i;
449 char *found_encoding;
450 const char *encoding;
451 PyObject *io;
452 PyObject *binary;
453 PyObject *fob = NULL;
454 PyObject *lineobj = NULL;
455 PyObject *res;
456 char buf[MAXPATHLEN+1];
457 int kind;
458 const void *data;
459
460 /* open the file */
461 if (filename == NULL)
462 return 0;
463
464 /* Do not attempt to open things like <string> or <stdin> */
465 assert(PyUnicode_Check(filename));
466 if (PyUnicode_READ_CHAR(filename, 0) == '<') {
467 Py_ssize_t len = PyUnicode_GET_LENGTH(filename);
468 if (len > 0 && PyUnicode_READ_CHAR(filename, len - 1) == '>') {
469 return 0;
470 }
471 }
472
473 io = PyImport_ImportModule("io");
474 if (io == NULL) {
475 return -1;
476 }
477
478 binary = _PyObject_CallMethod(io, &_Py_ID(open), "Os", filename, "rb");
479 if (binary == NULL) {
480 PyErr_Clear();
481
482 binary = _Py_FindSourceFile(filename, buf, sizeof(buf), io);
483 if (binary == NULL) {
484 Py_DECREF(io);
485 return -1;
486 }
487 }
488
489 /* use the right encoding to decode the file as unicode */
490 fd = PyObject_AsFileDescriptor(binary);
491 if (fd < 0) {
492 Py_DECREF(io);
493 Py_DECREF(binary);
494 return 0;
495 }
496 found_encoding = _PyTokenizer_FindEncodingFilename(fd, filename);
497 if (found_encoding == NULL)
498 PyErr_Clear();
499 encoding = (found_encoding != NULL) ? found_encoding : "utf-8";
500 /* Reset position */
501 if (lseek(fd, 0, SEEK_SET) == (off_t)-1) {
502 Py_DECREF(io);
503 Py_DECREF(binary);
504 PyMem_Free(found_encoding);
505 return 0;
506 }
507 fob = _PyObject_CallMethod(io, &_Py_ID(TextIOWrapper),
508 "Os", binary, encoding);
509 Py_DECREF(io);
510 PyMem_Free(found_encoding);
511
512 if (fob == NULL) {
513 PyErr_Clear();
514
515 res = PyObject_CallMethodNoArgs(binary, &_Py_ID(close));
516 Py_DECREF(binary);
517 if (res)
518 Py_DECREF(res);
519 else
520 PyErr_Clear();
521 return 0;
522 }
523 Py_DECREF(binary);
524
525 /* get the line number lineno */
526 for (i = 0; i < lineno; i++) {
527 Py_XDECREF(lineobj);
528 lineobj = PyFile_GetLine(fob, -1);
529 if (!lineobj) {
530 PyErr_Clear();
531 break;
532 }
533 }
534 res = PyObject_CallMethodNoArgs(fob, &_Py_ID(close));
535 if (res) {
536 Py_DECREF(res);
537 }
538 else {
539 PyErr_Clear();
540 }
541 Py_DECREF(fob);
542 if (!lineobj || !PyUnicode_Check(lineobj)) {
543 Py_XDECREF(lineobj);
544 return -1;
545 }
546
547 if (line) {
548 Py_INCREF(lineobj);
549 *line = lineobj;
550 }
551
552 /* remove the indentation of the line */
553 kind = PyUnicode_KIND(lineobj);
554 data = PyUnicode_DATA(lineobj);
555 for (i=0; i < PyUnicode_GET_LENGTH(lineobj); i++) {
556 Py_UCS4 ch = PyUnicode_READ(kind, data, i);
557 if (ch != ' ' && ch != '\t' && ch != '\014')
558 break;
559 }
560 if (i) {
561 PyObject *truncated;
562 truncated = PyUnicode_Substring(lineobj, i, PyUnicode_GET_LENGTH(lineobj));
563 if (truncated) {
564 Py_DECREF(lineobj);
565 lineobj = truncated;
566 } else {
567 PyErr_Clear();
568 }
569 }
570
571 if (truncation != NULL) {
572 *truncation = i - indent;
573 }
574
575 if (_Py_WriteIndentedMargin(margin_indent, margin, f) < 0) {
576 goto error;
577 }
578
579 /* Write some spaces before the line */
580 if (_Py_WriteIndent(indent, f) < 0) {
581 goto error;
582 }
583
584 /* finally display the line */
585 if (PyFile_WriteObject(lineobj, f, Py_PRINT_RAW) < 0) {
586 goto error;
587 }
588
589 if (PyFile_WriteString("\n", f) < 0) {
590 goto error;
591 }
592
593 Py_DECREF(lineobj);
594 return 0;
595 error:
596 Py_DECREF(lineobj);
597 return -1;
598 }
599
600 int
601 _Py_DisplaySourceLine(PyObject *f, PyObject *filename, int lineno, int indent,
602 int *truncation, PyObject **line)
603 {
604 return display_source_line_with_margin(f, filename, lineno, indent, 0,
605 NULL, truncation, line);
606 }
607
608 /* AST based Traceback Specialization
609 *
610 * When displaying a new traceback line, for certain syntactical constructs
611 * (e.g a subscript, an arithmetic operation) we try to create a representation
612 * that separates the primary source of error from the rest.
613 *
614 * Example specialization of BinOp nodes:
615 * Traceback (most recent call last):
616 * File "/home/isidentical/cpython/cpython/t.py", line 10, in <module>
617 * add_values(1, 2, 'x', 3, 4)
618 * File "/home/isidentical/cpython/cpython/t.py", line 2, in add_values
619 * return a + b + c + d + e
620 * ~~~~~~^~~
621 * TypeError: 'NoneType' object is not subscriptable
622 */
623
624 #define IS_WHITESPACE(c) (((c) == ' ') || ((c) == '\t') || ((c) == '\f'))
625
626 static int
627 extract_anchors_from_expr(const char *segment_str, expr_ty expr, Py_ssize_t *left_anchor, Py_ssize_t *right_anchor,
628 char** primary_error_char, char** secondary_error_char)
629 {
630 switch (expr->kind) {
631 case BinOp_kind: {
632 expr_ty left = expr->v.BinOp.left;
633 expr_ty right = expr->v.BinOp.right;
634 for (int i = left->end_col_offset; i < right->col_offset; i++) {
635 if (IS_WHITESPACE(segment_str[i])) {
636 continue;
637 }
638
639 *left_anchor = i;
640 *right_anchor = i + 1;
641
642 // Check whether if this a two-character operator (e.g //)
643 if (i + 1 < right->col_offset && !IS_WHITESPACE(segment_str[i + 1])) {
644 ++*right_anchor;
645 }
646
647 // Keep going if the current char is not ')'
648 if (i+1 < right->col_offset && (segment_str[i] == ')')) {
649 continue;
650 }
651
652 // Set the error characters
653 *primary_error_char = "~";
654 *secondary_error_char = "^";
655 break;
656 }
657 return 1;
658 }
659 case Subscript_kind: {
660 *left_anchor = expr->v.Subscript.value->end_col_offset;
661 *right_anchor = expr->v.Subscript.slice->end_col_offset + 1;
662 Py_ssize_t str_len = strlen(segment_str);
663
664 // Move right_anchor and left_anchor forward to the first non-whitespace character that is not ']' and '['
665 while (*left_anchor < str_len && (IS_WHITESPACE(segment_str[*left_anchor]) || segment_str[*left_anchor] != '[')) {
666 ++*left_anchor;
667 }
668 while (*right_anchor < str_len && (IS_WHITESPACE(segment_str[*right_anchor]) || segment_str[*right_anchor] != ']')) {
669 ++*right_anchor;
670 }
671 if (*right_anchor < str_len){
672 *right_anchor += 1;
673 }
674
675 // Set the error characters
676 *primary_error_char = "~";
677 *secondary_error_char = "^";
678 return 1;
679 }
680 default:
681 return 0;
682 }
683 }
684
685 static int
686 extract_anchors_from_stmt(const char *segment_str, stmt_ty statement, Py_ssize_t *left_anchor, Py_ssize_t *right_anchor,
687 char** primary_error_char, char** secondary_error_char)
688 {
689 switch (statement->kind) {
690 case Expr_kind: {
691 return extract_anchors_from_expr(segment_str, statement->v.Expr.value, left_anchor, right_anchor,
692 primary_error_char, secondary_error_char);
693 }
694 default:
695 return 0;
696 }
697 }
698
699 static int
700 extract_anchors_from_line(PyObject *filename, PyObject *line,
701 Py_ssize_t start_offset, Py_ssize_t end_offset,
702 Py_ssize_t *left_anchor, Py_ssize_t *right_anchor,
703 char** primary_error_char, char** secondary_error_char)
704 {
705 int res = -1;
706 PyArena *arena = NULL;
707 PyObject *segment = PyUnicode_Substring(line, start_offset, end_offset);
708 if (!segment) {
709 goto done;
710 }
711
712 const char *segment_str = PyUnicode_AsUTF8(segment);
713 if (!segment_str) {
714 goto done;
715 }
716
717 arena = _PyArena_New();
718 if (!arena) {
719 goto done;
720 }
721
722 PyCompilerFlags flags = _PyCompilerFlags_INIT;
723
724 _PyASTOptimizeState state;
725 state.optimize = _Py_GetConfig()->optimization_level;
726 state.ff_features = 0;
727
728 mod_ty module = _PyParser_ASTFromString(segment_str, filename, Py_file_input,
729 &flags, arena);
730 if (!module) {
731 goto done;
732 }
733 if (!_PyAST_Optimize(module, arena, &state)) {
734 goto done;
735 }
736
737 assert(module->kind == Module_kind);
738 if (asdl_seq_LEN(module->v.Module.body) == 1) {
739 stmt_ty statement = asdl_seq_GET(module->v.Module.body, 0);
740 res = extract_anchors_from_stmt(segment_str, statement, left_anchor, right_anchor,
741 primary_error_char, secondary_error_char);
742 } else {
743 res = 0;
744 }
745
746 done:
747 if (res > 0) {
748 // Normalize the AST offsets to byte offsets and adjust them with the
749 // start of the actual line (instead of the source code segment).
750 assert(segment != NULL);
751 assert(*left_anchor >= 0);
752 assert(*right_anchor >= 0);
753 *left_anchor = _PyPegen_byte_offset_to_character_offset(segment, *left_anchor) + start_offset;
754 *right_anchor = _PyPegen_byte_offset_to_character_offset(segment, *right_anchor) + start_offset;
755 }
756 Py_XDECREF(segment);
757 if (arena) {
758 _PyArena_Free(arena);
759 }
760 return res;
761 }
762
763 #define _TRACEBACK_SOURCE_LINE_INDENT 4
764
765 static inline int
766 ignore_source_errors(void) {
767 if (PyErr_Occurred()) {
768 if (PyErr_ExceptionMatches(PyExc_KeyboardInterrupt)) {
769 return -1;
770 }
771 PyErr_Clear();
772 }
773 return 0;
774 }
775
776 static inline int
777 print_error_location_carets(PyObject *f, int offset, Py_ssize_t start_offset, Py_ssize_t end_offset,
778 Py_ssize_t right_start_offset, Py_ssize_t left_end_offset,
779 const char *primary, const char *secondary) {
780 int special_chars = (left_end_offset != -1 || right_start_offset != -1);
781 const char *str;
782 while (++offset <= end_offset) {
783 if (offset <= start_offset) {
784 str = " ";
785 } else if (special_chars && left_end_offset < offset && offset <= right_start_offset) {
786 str = secondary;
787 } else {
788 str = primary;
789 }
790 if (PyFile_WriteString(str, f) < 0) {
791 return -1;
792 }
793 }
794 if (PyFile_WriteString("\n", f) < 0) {
795 return -1;
796 }
797 return 0;
798 }
799
800 static int
801 tb_displayline(PyTracebackObject* tb, PyObject *f, PyObject *filename, int lineno,
802 PyFrameObject *frame, PyObject *name, int margin_indent, const char *margin)
803 {
804 if (filename == NULL || name == NULL) {
805 return -1;
806 }
807
808 if (_Py_WriteIndentedMargin(margin_indent, margin, f) < 0) {
809 return -1;
810 }
811
812 PyObject *line = PyUnicode_FromFormat(" File \"%U\", line %d, in %U\n",
813 filename, lineno, name);
814 if (line == NULL) {
815 return -1;
816 }
817
818 int res = PyFile_WriteObject(line, f, Py_PRINT_RAW);
819 Py_DECREF(line);
820 if (res < 0) {
821 return -1;
822 }
823
824 int err = 0;
825
826 int truncation = _TRACEBACK_SOURCE_LINE_INDENT;
827 PyObject* source_line = NULL;
828 int rc = display_source_line_with_margin(
829 f, filename, lineno, _TRACEBACK_SOURCE_LINE_INDENT,
830 margin_indent, margin, &truncation, &source_line);
831 if (rc != 0 || !source_line) {
832 /* ignore errors since we can't report them, can we? */
833 err = ignore_source_errors();
834 goto done;
835 }
836
837 int code_offset = tb->tb_lasti;
838 PyCodeObject* code = frame->f_frame->f_code;
839 const Py_ssize_t source_line_len = PyUnicode_GET_LENGTH(source_line);
840
841 int start_line;
842 int end_line;
843 int start_col_byte_offset;
844 int end_col_byte_offset;
845 if (!PyCode_Addr2Location(code, code_offset, &start_line, &start_col_byte_offset,
846 &end_line, &end_col_byte_offset)) {
847 goto done;
848 }
849
850 if (start_line < 0 || end_line < 0
851 || start_col_byte_offset < 0
852 || end_col_byte_offset < 0)
853 {
854 goto done;
855 }
856
857 // When displaying errors, we will use the following generic structure:
858 //
859 // ERROR LINE ERROR LINE ERROR LINE ERROR LINE ERROR LINE ERROR LINE ERROR LINE
860 // ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^~~~~~~~~~~~~~~~~~~~
861 // | |-> left_end_offset | |-> end_offset
862 // |-> start_offset |-> right_start_offset
863 //
864 // In general we will only have (start_offset, end_offset) but we can gather more information
865 // by analyzing the AST of the text between *start_offset* and *end_offset*. If this succeeds
866 // we could get *left_end_offset* and *right_start_offset* and some selection of characters for
867 // the different ranges (primary_error_char and secondary_error_char). If we cannot obtain the
868 // AST information or we cannot identify special ranges within it, then left_end_offset and
869 // right_end_offset will be set to -1.
870 //
871 // To keep the column indicators pertinent, they are not shown when the primary character
872 // spans the whole line.
873
874 // Convert the utf-8 byte offset to the actual character offset so we print the right number of carets.
875 assert(source_line);
876 Py_ssize_t start_offset = _PyPegen_byte_offset_to_character_offset(source_line, start_col_byte_offset);
877 if (start_offset < 0) {
878 err = ignore_source_errors() < 0;
879 goto done;
880 }
881
882 Py_ssize_t end_offset = _PyPegen_byte_offset_to_character_offset(source_line, end_col_byte_offset);
883 if (end_offset < 0) {
884 err = ignore_source_errors() < 0;
885 goto done;
886 }
887
888 Py_ssize_t left_end_offset = -1;
889 Py_ssize_t right_start_offset = -1;
890
891 char *primary_error_char = "^";
892 char *secondary_error_char = primary_error_char;
893
894 if (start_line == end_line) {
895 int res = extract_anchors_from_line(filename, source_line, start_offset, end_offset,
896 &left_end_offset, &right_start_offset,
897 &primary_error_char, &secondary_error_char);
898 if (res < 0 && ignore_source_errors() < 0) {
899 goto done;
900 }
901 }
902 else {
903 // If this is a multi-line expression, then we will highlight until
904 // the last non-whitespace character.
905 const char *source_line_str = PyUnicode_AsUTF8(source_line);
906 if (!source_line_str) {
907 goto done;
908 }
909
910 Py_ssize_t i = source_line_len;
911 while (--i >= 0) {
912 if (!IS_WHITESPACE(source_line_str[i])) {
913 break;
914 }
915 }
916
917 end_offset = i + 1;
918 }
919
920 // Elide indicators if primary char spans the frame line
921 Py_ssize_t stripped_line_len = source_line_len - truncation - _TRACEBACK_SOURCE_LINE_INDENT;
922 bool has_secondary_ranges = (left_end_offset != -1 || right_start_offset != -1);
923 if (end_offset - start_offset == stripped_line_len && !has_secondary_ranges) {
924 goto done;
925 }
926
927 if (_Py_WriteIndentedMargin(margin_indent, margin, f) < 0) {
928 err = -1;
929 goto done;
930 }
931
932 // Convert all offsets to display offsets (e.g. the space they would take up if printed
933 // on the screen).
934 Py_ssize_t dp_start = _PyPegen_calculate_display_width(source_line, start_offset);
935 if (dp_start < 0) {
936 err = ignore_source_errors() < 0;
937 goto done;
938 }
939
940 Py_ssize_t dp_end = _PyPegen_calculate_display_width(source_line, end_offset);
941 if (dp_end < 0) {
942 err = ignore_source_errors() < 0;
943 goto done;
944 }
945
946 Py_ssize_t dp_left_end = -1;
947 Py_ssize_t dp_right_start = -1;
948 if (has_secondary_ranges) {
949 dp_left_end = _PyPegen_calculate_display_width(source_line, left_end_offset);
950 if (dp_left_end < 0) {
951 err = ignore_source_errors() < 0;
952 goto done;
953 }
954
955 dp_right_start = _PyPegen_calculate_display_width(source_line, right_start_offset);
956 if (dp_right_start < 0) {
957 err = ignore_source_errors() < 0;
958 goto done;
959 }
960 }
961
962
963 if (print_error_location_carets(f, truncation, dp_start, dp_end,
964 dp_right_start, dp_left_end,
965 primary_error_char, secondary_error_char) < 0) {
966 err = -1;
967 goto done;
968 }
969
970 done:
971 Py_XDECREF(source_line);
972 return err;
973 }
974
975 static const int TB_RECURSIVE_CUTOFF = 3; // Also hardcoded in traceback.py.
976
977 static int
978 tb_print_line_repeated(PyObject *f, long cnt)
979 {
980 cnt -= TB_RECURSIVE_CUTOFF;
981 PyObject *line = PyUnicode_FromFormat(
982 (cnt > 1)
983 ? " [Previous line repeated %ld more times]\n"
984 : " [Previous line repeated %ld more time]\n",
985 cnt);
986 if (line == NULL) {
987 return -1;
988 }
989 int err = PyFile_WriteObject(line, f, Py_PRINT_RAW);
990 Py_DECREF(line);
991 return err;
992 }
993
994 static int
995 tb_printinternal(PyTracebackObject *tb, PyObject *f, long limit,
996 int indent, const char *margin)
997 {
998 PyCodeObject *code = NULL;
999 Py_ssize_t depth = 0;
1000 PyObject *last_file = NULL;
1001 int last_line = -1;
1002 PyObject *last_name = NULL;
1003 long cnt = 0;
1004 PyTracebackObject *tb1 = tb;
1005 while (tb1 != NULL) {
1006 depth++;
1007 tb1 = tb1->tb_next;
1008 }
1009 while (tb != NULL && depth > limit) {
1010 depth--;
1011 tb = tb->tb_next;
1012 }
1013 while (tb != NULL) {
1014 code = PyFrame_GetCode(tb->tb_frame);
1015 int tb_lineno = tb->tb_lineno;
1016 if (tb_lineno == -1) {
1017 tb_lineno = tb_get_lineno(tb);
1018 }
1019 if (last_file == NULL ||
1020 code->co_filename != last_file ||
1021 last_line == -1 || tb_lineno != last_line ||
1022 last_name == NULL || code->co_name != last_name) {
1023 if (cnt > TB_RECURSIVE_CUTOFF) {
1024 if (tb_print_line_repeated(f, cnt) < 0) {
1025 goto error;
1026 }
1027 }
1028 last_file = code->co_filename;
1029 last_line = tb_lineno;
1030 last_name = code->co_name;
1031 cnt = 0;
1032 }
1033 cnt++;
1034 if (cnt <= TB_RECURSIVE_CUTOFF) {
1035 if (tb_displayline(tb, f, code->co_filename, tb_lineno,
1036 tb->tb_frame, code->co_name, indent, margin) < 0) {
1037 goto error;
1038 }
1039
1040 if (PyErr_CheckSignals() < 0) {
1041 goto error;
1042 }
1043 }
1044 Py_CLEAR(code);
1045 tb = tb->tb_next;
1046 }
1047 if (cnt > TB_RECURSIVE_CUTOFF) {
1048 if (tb_print_line_repeated(f, cnt) < 0) {
1049 goto error;
1050 }
1051 }
1052 return 0;
1053 error:
1054 Py_XDECREF(code);
1055 return -1;
1056 }
1057
1058 #define PyTraceBack_LIMIT 1000
1059
1060 int
1061 _PyTraceBack_Print_Indented(PyObject *v, int indent, const char *margin,
1062 const char *header_margin, const char *header, PyObject *f)
1063 {
1064 PyObject *limitv;
1065 long limit = PyTraceBack_LIMIT;
1066
1067 if (v == NULL) {
1068 return 0;
1069 }
1070 if (!PyTraceBack_Check(v)) {
1071 PyErr_BadInternalCall();
1072 return -1;
1073 }
1074 limitv = PySys_GetObject("tracebacklimit");
1075 if (limitv && PyLong_Check(limitv)) {
1076 int overflow;
1077 limit = PyLong_AsLongAndOverflow(limitv, &overflow);
1078 if (overflow > 0) {
1079 limit = LONG_MAX;
1080 }
1081 else if (limit <= 0) {
1082 return 0;
1083 }
1084 }
1085 if (_Py_WriteIndentedMargin(indent, header_margin, f) < 0) {
1086 return -1;
1087 }
1088
1089 if (PyFile_WriteString(header, f) < 0) {
1090 return -1;
1091 }
1092
1093 if (tb_printinternal((PyTracebackObject *)v, f, limit, indent, margin) < 0) {
1094 return -1;
1095 }
1096
1097 return 0;
1098 }
1099
1100 int
1101 PyTraceBack_Print(PyObject *v, PyObject *f)
1102 {
1103 int indent = 0;
1104 const char *margin = NULL;
1105 const char *header_margin = NULL;
1106 const char *header = EXCEPTION_TB_HEADER;
1107
1108 return _PyTraceBack_Print_Indented(v, indent, margin, header_margin, header, f);
1109 }
1110
1111 /* Format an integer in range [0; 0xffffffff] to decimal and write it
1112 into the file fd.
1113
1114 This function is signal safe. */
1115
1116 void
1117 _Py_DumpDecimal(int fd, size_t value)
1118 {
1119 /* maximum number of characters required for output of %lld or %p.
1120 We need at most ceil(log10(256)*SIZEOF_LONG_LONG) digits,
1121 plus 1 for the null byte. 53/22 is an upper bound for log10(256). */
1122 char buffer[1 + (sizeof(size_t)*53-1) / 22 + 1];
1123 char *ptr, *end;
1124
1125 end = &buffer[Py_ARRAY_LENGTH(buffer) - 1];
1126 ptr = end;
1127 *ptr = '\0';
1128 do {
1129 --ptr;
1130 assert(ptr >= buffer);
1131 *ptr = '0' + (value % 10);
1132 value /= 10;
1133 } while (value);
1134
1135 _Py_write_noraise(fd, ptr, end - ptr);
1136 }
1137
1138 /* Format an integer as hexadecimal with width digits into fd file descriptor.
1139 The function is signal safe. */
1140 void
1141 _Py_DumpHexadecimal(int fd, uintptr_t value, Py_ssize_t width)
1142 {
1143 char buffer[sizeof(uintptr_t) * 2 + 1], *ptr, *end;
1144 const Py_ssize_t size = Py_ARRAY_LENGTH(buffer) - 1;
1145
1146 if (width > size)
1147 width = size;
1148 /* it's ok if width is negative */
1149
1150 end = &buffer[size];
1151 ptr = end;
1152 *ptr = '\0';
1153 do {
1154 --ptr;
1155 assert(ptr >= buffer);
1156 *ptr = Py_hexdigits[value & 15];
1157 value >>= 4;
1158 } while ((end - ptr) < width || value);
1159
1160 _Py_write_noraise(fd, ptr, end - ptr);
1161 }
1162
1163 void
1164 _Py_DumpASCII(int fd, PyObject *text)
1165 {
1166 PyASCIIObject *ascii = _PyASCIIObject_CAST(text);
1167 Py_ssize_t i, size;
1168 int truncated;
1169 int kind;
1170 void *data = NULL;
1171 wchar_t *wstr = NULL;
1172 Py_UCS4 ch;
1173
1174 if (!PyUnicode_Check(text))
1175 return;
1176
1177 size = ascii->length;
1178 kind = ascii->state.kind;
1179 if (kind == PyUnicode_WCHAR_KIND) {
1180 wstr = ascii->wstr;
1181 if (wstr == NULL)
1182 return;
1183 size = _PyCompactUnicodeObject_CAST(text)->wstr_length;
1184 }
1185 else if (ascii->state.compact) {
1186 if (ascii->state.ascii)
1187 data = ascii + 1;
1188 else
1189 data = _PyCompactUnicodeObject_CAST(text) + 1;
1190 }
1191 else {
1192 data = _PyUnicodeObject_CAST(text)->data.any;
1193 if (data == NULL)
1194 return;
1195 }
1196
1197 if (MAX_STRING_LENGTH < size) {
1198 size = MAX_STRING_LENGTH;
1199 truncated = 1;
1200 }
1201 else {
1202 truncated = 0;
1203 }
1204
1205 // Is an ASCII string?
1206 if (ascii->state.ascii) {
1207 assert(kind == PyUnicode_1BYTE_KIND);
1208 char *str = data;
1209
1210 int need_escape = 0;
1211 for (i=0; i < size; i++) {
1212 ch = str[i];
1213 if (!(' ' <= ch && ch <= 126)) {
1214 need_escape = 1;
1215 break;
1216 }
1217 }
1218 if (!need_escape) {
1219 // The string can be written with a single write() syscall
1220 _Py_write_noraise(fd, str, size);
1221 goto done;
1222 }
1223 }
1224
1225 for (i=0; i < size; i++) {
1226 if (kind != PyUnicode_WCHAR_KIND)
1227 ch = PyUnicode_READ(kind, data, i);
1228 else
1229 ch = wstr[i];
1230 if (' ' <= ch && ch <= 126) {
1231 /* printable ASCII character */
1232 char c = (char)ch;
1233 _Py_write_noraise(fd, &c, 1);
1234 }
1235 else if (ch <= 0xff) {
1236 PUTS(fd, "\\x");
1237 _Py_DumpHexadecimal(fd, ch, 2);
1238 }
1239 else if (ch <= 0xffff) {
1240 PUTS(fd, "\\u");
1241 _Py_DumpHexadecimal(fd, ch, 4);
1242 }
1243 else {
1244 PUTS(fd, "\\U");
1245 _Py_DumpHexadecimal(fd, ch, 8);
1246 }
1247 }
1248
1249 done:
1250 if (truncated) {
1251 PUTS(fd, "...");
1252 }
1253 }
1254
1255 /* Write a frame into the file fd: "File "xxx", line xxx in xxx".
1256
1257 This function is signal safe. */
1258
1259 static void
1260 dump_frame(int fd, _PyInterpreterFrame *frame)
1261 {
1262 PyCodeObject *code = frame->f_code;
1263 PUTS(fd, " File ");
1264 if (code->co_filename != NULL
1265 && PyUnicode_Check(code->co_filename))
1266 {
1267 PUTS(fd, "\"");
1268 _Py_DumpASCII(fd, code->co_filename);
1269 PUTS(fd, "\"");
1270 } else {
1271 PUTS(fd, "???");
1272 }
1273
1274 int lineno = _PyInterpreterFrame_GetLine(frame);
1275 PUTS(fd, ", line ");
1276 if (lineno >= 0) {
1277 _Py_DumpDecimal(fd, (size_t)lineno);
1278 }
1279 else {
1280 PUTS(fd, "???");
1281 }
1282 PUTS(fd, " in ");
1283
1284 if (code->co_name != NULL
1285 && PyUnicode_Check(code->co_name)) {
1286 _Py_DumpASCII(fd, code->co_name);
1287 }
1288 else {
1289 PUTS(fd, "???");
1290 }
1291
1292 PUTS(fd, "\n");
1293 }
1294
1295 static int
1296 tstate_is_freed(PyThreadState *tstate)
1297 {
1298 if (_PyMem_IsPtrFreed(tstate)) {
1299 return 1;
1300 }
1301 if (_PyMem_IsPtrFreed(tstate->interp)) {
1302 return 1;
1303 }
1304 return 0;
1305 }
1306
1307
1308 static int
1309 interp_is_freed(PyInterpreterState *interp)
1310 {
1311 return _PyMem_IsPtrFreed(interp);
1312 }
1313
1314
1315 static void
1316 dump_traceback(int fd, PyThreadState *tstate, int write_header)
1317 {
1318 if (write_header) {
1319 PUTS(fd, "Stack (most recent call first):\n");
1320 }
1321
1322 if (tstate_is_freed(tstate)) {
1323 PUTS(fd, " <tstate is freed>\n");
1324 return;
1325 }
1326
1327 _PyInterpreterFrame *frame = tstate->cframe->current_frame;
1328 if (frame == NULL) {
1329 PUTS(fd, " <no Python frame>\n");
1330 return;
1331 }
1332
1333 unsigned int depth = 0;
1334 while (1) {
1335 if (MAX_FRAME_DEPTH <= depth) {
1336 PUTS(fd, " ...\n");
1337 break;
1338 }
1339 dump_frame(fd, frame);
1340 frame = frame->previous;
1341 if (frame == NULL) {
1342 break;
1343 }
1344 depth++;
1345 }
1346 }
1347
1348 /* Dump the traceback of a Python thread into fd. Use write() to write the
1349 traceback and retry if write() is interrupted by a signal (failed with
1350 EINTR), but don't call the Python signal handler.
1351
1352 The caller is responsible to call PyErr_CheckSignals() to call Python signal
1353 handlers if signals were received. */
1354 void
1355 _Py_DumpTraceback(int fd, PyThreadState *tstate)
1356 {
1357 dump_traceback(fd, tstate, 1);
1358 }
1359
1360 /* Write the thread identifier into the file 'fd': "Current thread 0xHHHH:\" if
1361 is_current is true, "Thread 0xHHHH:\n" otherwise.
1362
1363 This function is signal safe. */
1364
1365 static void
1366 write_thread_id(int fd, PyThreadState *tstate, int is_current)
1367 {
1368 if (is_current)
1369 PUTS(fd, "Current thread 0x");
1370 else
1371 PUTS(fd, "Thread 0x");
1372 _Py_DumpHexadecimal(fd,
1373 tstate->thread_id,
1374 sizeof(unsigned long) * 2);
1375 PUTS(fd, " (most recent call first):\n");
1376 }
1377
1378 /* Dump the traceback of all Python threads into fd. Use write() to write the
1379 traceback and retry if write() is interrupted by a signal (failed with
1380 EINTR), but don't call the Python signal handler.
1381
1382 The caller is responsible to call PyErr_CheckSignals() to call Python signal
1383 handlers if signals were received. */
1384 const char*
1385 _Py_DumpTracebackThreads(int fd, PyInterpreterState *interp,
1386 PyThreadState *current_tstate)
1387 {
1388 if (current_tstate == NULL) {
1389 /* _Py_DumpTracebackThreads() is called from signal handlers by
1390 faulthandler.
1391
1392 SIGSEGV, SIGFPE, SIGABRT, SIGBUS and SIGILL are synchronous signals
1393 and are thus delivered to the thread that caused the fault. Get the
1394 Python thread state of the current thread.
1395
1396 PyThreadState_Get() doesn't give the state of the thread that caused
1397 the fault if the thread released the GIL, and so
1398 _PyThreadState_GET() cannot be used. Read the thread specific
1399 storage (TSS) instead: call PyGILState_GetThisThreadState(). */
1400 current_tstate = PyGILState_GetThisThreadState();
1401 }
1402
1403 if (current_tstate != NULL && tstate_is_freed(current_tstate)) {
1404 return "tstate is freed";
1405 }
1406
1407 if (interp == NULL) {
1408 if (current_tstate == NULL) {
1409 interp = _PyGILState_GetInterpreterStateUnsafe();
1410 if (interp == NULL) {
1411 /* We need the interpreter state to get Python threads */
1412 return "unable to get the interpreter state";
1413 }
1414 }
1415 else {
1416 interp = current_tstate->interp;
1417 }
1418 }
1419 assert(interp != NULL);
1420
1421 if (interp_is_freed(interp)) {
1422 return "interp is freed";
1423 }
1424
1425 /* Get the current interpreter from the current thread */
1426 PyThreadState *tstate = PyInterpreterState_ThreadHead(interp);
1427 if (tstate == NULL)
1428 return "unable to get the thread head state";
1429
1430 /* Dump the traceback of each thread */
1431 tstate = PyInterpreterState_ThreadHead(interp);
1432 unsigned int nthreads = 0;
1433 _Py_BEGIN_SUPPRESS_IPH
1434 do
1435 {
1436 if (nthreads != 0)
1437 PUTS(fd, "\n");
1438 if (nthreads >= MAX_NTHREADS) {
1439 PUTS(fd, "...\n");
1440 break;
1441 }
1442 write_thread_id(fd, tstate, tstate == current_tstate);
1443 if (tstate == current_tstate && tstate->interp->gc.collecting) {
1444 PUTS(fd, " Garbage-collecting\n");
1445 }
1446 dump_traceback(fd, tstate, 0);
1447 tstate = PyThreadState_Next(tstate);
1448 nthreads++;
1449 } while (tstate != NULL);
1450 _Py_END_SUPPRESS_IPH
1451
1452 return NULL;
1453 }
1454