1 #ifndef Py_INTERNAL_CODE_H
2 #define Py_INTERNAL_CODE_H
3 #ifdef __cplusplus
4 extern "C" {
5 #endif
6
7 #define CODE_MAX_WATCHERS 8
8
9 /* PEP 659
10 * Specialization and quickening structs and helper functions
11 */
12
13
14 // Inline caches. If you change the number of cache entries for an instruction,
15 // you must *also* update the number of cache entries in Lib/opcode.py and bump
16 // the magic number in Lib/importlib/_bootstrap_external.py!
17
18 #define CACHE_ENTRIES(cache) (sizeof(cache)/sizeof(_Py_CODEUNIT))
19
20 typedef struct {
21 uint16_t counter;
22 uint16_t index;
23 uint16_t module_keys_version;
24 uint16_t builtin_keys_version;
25 } _PyLoadGlobalCache;
26
27 #define INLINE_CACHE_ENTRIES_LOAD_GLOBAL CACHE_ENTRIES(_PyLoadGlobalCache)
28
29 typedef struct {
30 uint16_t counter;
31 } _PyBinaryOpCache;
32
33 #define INLINE_CACHE_ENTRIES_BINARY_OP CACHE_ENTRIES(_PyBinaryOpCache)
34
35 typedef struct {
36 uint16_t counter;
37 } _PyUnpackSequenceCache;
38
39 #define INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE \
40 CACHE_ENTRIES(_PyUnpackSequenceCache)
41
42 typedef struct {
43 uint16_t counter;
44 } _PyCompareOpCache;
45
46 #define INLINE_CACHE_ENTRIES_COMPARE_OP CACHE_ENTRIES(_PyCompareOpCache)
47
48 typedef struct {
49 uint16_t counter;
50 } _PyBinarySubscrCache;
51
52 #define INLINE_CACHE_ENTRIES_BINARY_SUBSCR CACHE_ENTRIES(_PyBinarySubscrCache)
53
54 typedef struct {
55 uint16_t counter;
56 } _PySuperAttrCache;
57
58 #define INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR CACHE_ENTRIES(_PySuperAttrCache)
59
60 typedef struct {
61 uint16_t counter;
62 uint16_t version[2];
63 uint16_t index;
64 } _PyAttrCache;
65
66 typedef struct {
67 uint16_t counter;
68 uint16_t type_version[2];
69 uint16_t keys_version[2];
70 uint16_t descr[4];
71 } _PyLoadMethodCache;
72
73
74 // MUST be the max(_PyAttrCache, _PyLoadMethodCache)
75 #define INLINE_CACHE_ENTRIES_LOAD_ATTR CACHE_ENTRIES(_PyLoadMethodCache)
76
77 #define INLINE_CACHE_ENTRIES_STORE_ATTR CACHE_ENTRIES(_PyAttrCache)
78
79 typedef struct {
80 uint16_t counter;
81 uint16_t func_version[2];
82 } _PyCallCache;
83
84 #define INLINE_CACHE_ENTRIES_CALL CACHE_ENTRIES(_PyCallCache)
85
86 typedef struct {
87 uint16_t counter;
88 } _PyStoreSubscrCache;
89
90 #define INLINE_CACHE_ENTRIES_STORE_SUBSCR CACHE_ENTRIES(_PyStoreSubscrCache)
91
92 typedef struct {
93 uint16_t counter;
94 } _PyForIterCache;
95
96 #define INLINE_CACHE_ENTRIES_FOR_ITER CACHE_ENTRIES(_PyForIterCache)
97
98 typedef struct {
99 uint16_t counter;
100 } _PySendCache;
101
102 #define INLINE_CACHE_ENTRIES_SEND CACHE_ENTRIES(_PySendCache)
103
104 // Borrowed references to common callables:
105 struct callable_cache {
106 PyObject *isinstance;
107 PyObject *len;
108 PyObject *list_append;
109 PyObject *object__getattribute__;
110 };
111
112 /* "Locals plus" for a code object is the set of locals + cell vars +
113 * free vars. This relates to variable names as well as offsets into
114 * the "fast locals" storage array of execution frames. The compiler
115 * builds the list of names, their offsets, and the corresponding
116 * kind of local.
117 *
118 * Those kinds represent the source of the initial value and the
119 * variable's scope (as related to closures). A "local" is an
120 * argument or other variable defined in the current scope. A "free"
121 * variable is one that is defined in an outer scope and comes from
122 * the function's closure. A "cell" variable is a local that escapes
123 * into an inner function as part of a closure, and thus must be
124 * wrapped in a cell. Any "local" can also be a "cell", but the
125 * "free" kind is mutually exclusive with both.
126 */
127
128 // Note that these all fit within a byte, as do combinations.
129 // Later, we will use the smaller numbers to differentiate the different
130 // kinds of locals (e.g. pos-only arg, varkwargs, local-only).
131 #define CO_FAST_HIDDEN 0x10
132 #define CO_FAST_LOCAL 0x20
133 #define CO_FAST_CELL 0x40
134 #define CO_FAST_FREE 0x80
135
136 typedef unsigned char _PyLocals_Kind;
137
138 static inline _PyLocals_Kind
139 _PyLocals_GetKind(PyObject *kinds, int i)
140 {
141 assert(PyBytes_Check(kinds));
142 assert(0 <= i && i < PyBytes_GET_SIZE(kinds));
143 char *ptr = PyBytes_AS_STRING(kinds);
144 return (_PyLocals_Kind)(ptr[i]);
145 }
146
147 static inline void
148 _PyLocals_SetKind(PyObject *kinds, int i, _PyLocals_Kind kind)
149 {
150 assert(PyBytes_Check(kinds));
151 assert(0 <= i && i < PyBytes_GET_SIZE(kinds));
152 char *ptr = PyBytes_AS_STRING(kinds);
153 ptr[i] = (char) kind;
154 }
155
156
157 struct _PyCodeConstructor {
158 /* metadata */
159 PyObject *filename;
160 PyObject *name;
161 PyObject *qualname;
162 int flags;
163
164 /* the code */
165 PyObject *code;
166 int firstlineno;
167 PyObject *linetable;
168
169 /* used by the code */
170 PyObject *consts;
171 PyObject *names;
172
173 /* mapping frame offsets to information */
174 PyObject *localsplusnames; // Tuple of strings
175 PyObject *localspluskinds; // Bytes object, one byte per variable
176
177 /* args (within varnames) */
178 int argcount;
179 int posonlyargcount;
180 // XXX Replace argcount with posorkwargcount (argcount - posonlyargcount).
181 int kwonlyargcount;
182
183 /* needed to create the frame */
184 int stacksize;
185
186 /* used by the eval loop */
187 PyObject *exceptiontable;
188 };
189
190 // Using an "arguments struct" like this is helpful for maintainability
191 // in a case such as this with many parameters. It does bear a risk:
192 // if the struct changes and callers are not updated properly then the
193 // compiler will not catch problems (like a missing argument). This can
194 // cause hard-to-debug problems. The risk is mitigated by the use of
195 // check_code() in codeobject.c. However, we may decide to switch
196 // back to a regular function signature. Regardless, this approach
197 // wouldn't be appropriate if this weren't a strictly internal API.
198 // (See the comments in https://github.com/python/cpython/pull/26258.)
199 PyAPI_FUNC(int) _PyCode_Validate(struct _PyCodeConstructor *);
200 PyAPI_FUNC(PyCodeObject *) _PyCode_New(struct _PyCodeConstructor *);
201
202
203 /* Private API */
204
205 /* Getters for internal PyCodeObject data. */
206 extern PyObject* _PyCode_GetVarnames(PyCodeObject *);
207 extern PyObject* _PyCode_GetCellvars(PyCodeObject *);
208 extern PyObject* _PyCode_GetFreevars(PyCodeObject *);
209 extern PyObject* _PyCode_GetCode(PyCodeObject *);
210
211 /** API for initializing the line number tables. */
212 extern int _PyCode_InitAddressRange(PyCodeObject* co, PyCodeAddressRange *bounds);
213
214 /** Out of process API for initializing the location table. */
215 extern void _PyLineTable_InitAddressRange(
216 const char *linetable,
217 Py_ssize_t length,
218 int firstlineno,
219 PyCodeAddressRange *range);
220
221 /** API for traversing the line number table. */
222 extern int _PyLineTable_NextAddressRange(PyCodeAddressRange *range);
223 extern int _PyLineTable_PreviousAddressRange(PyCodeAddressRange *range);
224
225 /* Specialization functions */
226
227 extern void _Py_Specialize_LoadSuperAttr(PyObject *global_super, PyObject *cls,
228 _Py_CODEUNIT *instr, int load_method);
229 extern void _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr,
230 PyObject *name);
231 extern void _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr,
232 PyObject *name);
233 extern void _Py_Specialize_LoadGlobal(PyObject *globals, PyObject *builtins,
234 _Py_CODEUNIT *instr, PyObject *name);
235 extern void _Py_Specialize_BinarySubscr(PyObject *sub, PyObject *container,
236 _Py_CODEUNIT *instr);
237 extern void _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub,
238 _Py_CODEUNIT *instr);
239 extern void _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr,
240 int nargs, PyObject *kwnames);
241 extern void _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
242 int oparg, PyObject **locals);
243 extern void _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs,
244 _Py_CODEUNIT *instr, int oparg);
245 extern void _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr,
246 int oparg);
247 extern void _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg);
248 extern void _Py_Specialize_Send(PyObject *receiver, _Py_CODEUNIT *instr);
249
250 /* Finalizer function for static codeobjects used in deepfreeze.py */
251 extern void _PyStaticCode_Fini(PyCodeObject *co);
252 /* Function to intern strings of codeobjects and quicken the bytecode */
253 extern int _PyStaticCode_Init(PyCodeObject *co);
254
255 #ifdef Py_STATS
256
257
258 #define STAT_INC(opname, name) do { if (_py_stats) _py_stats->opcode_stats[opname].specialization.name++; } while (0)
259 #define STAT_DEC(opname, name) do { if (_py_stats) _py_stats->opcode_stats[opname].specialization.name--; } while (0)
260 #define OPCODE_EXE_INC(opname) do { if (_py_stats) _py_stats->opcode_stats[opname].execution_count++; } while (0)
261 #define CALL_STAT_INC(name) do { if (_py_stats) _py_stats->call_stats.name++; } while (0)
262 #define OBJECT_STAT_INC(name) do { if (_py_stats) _py_stats->object_stats.name++; } while (0)
263 #define OBJECT_STAT_INC_COND(name, cond) \
264 do { if (_py_stats && cond) _py_stats->object_stats.name++; } while (0)
265 #define EVAL_CALL_STAT_INC(name) do { if (_py_stats) _py_stats->call_stats.eval_calls[name]++; } while (0)
266 #define EVAL_CALL_STAT_INC_IF_FUNCTION(name, callable) \
267 do { if (_py_stats && PyFunction_Check(callable)) _py_stats->call_stats.eval_calls[name]++; } while (0)
268
269 // Used by the _opcode extension which is built as a shared library
270 PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void);
271
272 #else
273 #define STAT_INC(opname, name) ((void)0)
274 #define STAT_DEC(opname, name) ((void)0)
275 #define OPCODE_EXE_INC(opname) ((void)0)
276 #define CALL_STAT_INC(name) ((void)0)
277 #define OBJECT_STAT_INC(name) ((void)0)
278 #define OBJECT_STAT_INC_COND(name, cond) ((void)0)
279 #define EVAL_CALL_STAT_INC(name) ((void)0)
280 #define EVAL_CALL_STAT_INC_IF_FUNCTION(name, callable) ((void)0)
281 #endif // !Py_STATS
282
283 // Utility functions for reading/writing 32/64-bit values in the inline caches.
284 // Great care should be taken to ensure that these functions remain correct and
285 // performant! They should compile to just "move" instructions on all supported
286 // compilers and platforms.
287
288 // We use memcpy to let the C compiler handle unaligned accesses and endianness
289 // issues for us. It also seems to produce better code than manual copying for
290 // most compilers (see https://blog.regehr.org/archives/959 for more info).
291
292 static inline void
293 write_u32(uint16_t *p, uint32_t val)
294 {
295 memcpy(p, &val, sizeof(val));
296 }
297
298 static inline void
299 write_u64(uint16_t *p, uint64_t val)
300 {
301 memcpy(p, &val, sizeof(val));
302 }
303
304 static inline void
305 write_obj(uint16_t *p, PyObject *val)
306 {
307 memcpy(p, &val, sizeof(val));
308 }
309
310 static inline uint16_t
311 read_u16(uint16_t *p)
312 {
313 return *p;
314 }
315
316 static inline uint32_t
317 read_u32(uint16_t *p)
318 {
319 uint32_t val;
320 memcpy(&val, p, sizeof(val));
321 return val;
322 }
323
324 static inline uint64_t
325 read_u64(uint16_t *p)
326 {
327 uint64_t val;
328 memcpy(&val, p, sizeof(val));
329 return val;
330 }
331
332 static inline PyObject *
333 read_obj(uint16_t *p)
334 {
335 PyObject *val;
336 memcpy(&val, p, sizeof(val));
337 return val;
338 }
339
340 /* See Objects/exception_handling_notes.txt for details.
341 */
342 static inline unsigned char *
343 parse_varint(unsigned char *p, int *result) {
344 int val = p[0] & 63;
345 while (p[0] & 64) {
346 p++;
347 val = (val << 6) | (p[0] & 63);
348 }
349 *result = val;
350 return p+1;
351 }
352
353 static inline int
354 write_varint(uint8_t *ptr, unsigned int val)
355 {
356 int written = 1;
357 while (val >= 64) {
358 *ptr++ = 64 | (val & 63);
359 val >>= 6;
360 written++;
361 }
362 *ptr = val;
363 return written;
364 }
365
366 static inline int
367 write_signed_varint(uint8_t *ptr, int val)
368 {
369 if (val < 0) {
370 val = ((-val)<<1) | 1;
371 }
372 else {
373 val = val << 1;
374 }
375 return write_varint(ptr, val);
376 }
377
378 static inline int
379 write_location_entry_start(uint8_t *ptr, int code, int length)
380 {
381 assert((code & 15) == code);
382 *ptr = 128 | (code << 3) | (length - 1);
383 return 1;
384 }
385
386
387 /** Counters
388 * The first 16-bit value in each inline cache is a counter.
389 * When counting misses, the counter is treated as a simple unsigned value.
390 *
391 * When counting executions until the next specialization attempt,
392 * exponential backoff is used to reduce the number of specialization failures.
393 * The high 12 bits store the counter, the low 4 bits store the backoff exponent.
394 * On a specialization failure, the backoff exponent is incremented and the
395 * counter set to (2**backoff - 1).
396 * Backoff == 6 -> starting counter == 63, backoff == 10 -> starting counter == 1023.
397 */
398
399 /* With a 16-bit counter, we have 12 bits for the counter value, and 4 bits for the backoff */
400 #define ADAPTIVE_BACKOFF_BITS 4
401
402 // A value of 1 means that we attempt to specialize the *second* time each
403 // instruction is executed. Executing twice is a much better indicator of
404 // "hotness" than executing once, but additional warmup delays only prevent
405 // specialization. Most types stabilize by the second execution, too:
406 #define ADAPTIVE_WARMUP_VALUE 1
407 #define ADAPTIVE_WARMUP_BACKOFF 1
408
409 // A value of 52 means that we attempt to re-specialize after 53 misses (a prime
410 // number, useful for avoiding artifacts if every nth value is a different type
411 // or something). Setting the backoff to 0 means that the counter is reset to
412 // the same state as a warming-up instruction (value == 1, backoff == 1) after
413 // deoptimization. This isn't strictly necessary, but it is bit easier to reason
414 // about when thinking about the opcode transitions as a state machine:
415 #define ADAPTIVE_COOLDOWN_VALUE 52
416 #define ADAPTIVE_COOLDOWN_BACKOFF 0
417
418 #define MAX_BACKOFF_VALUE (16 - ADAPTIVE_BACKOFF_BITS)
419
420
421 static inline uint16_t
422 adaptive_counter_bits(int value, int backoff) {
423 return (value << ADAPTIVE_BACKOFF_BITS) |
424 (backoff & ((1<<ADAPTIVE_BACKOFF_BITS)-1));
425 }
426
427 static inline uint16_t
428 adaptive_counter_warmup(void) {
429 return adaptive_counter_bits(ADAPTIVE_WARMUP_VALUE,
430 ADAPTIVE_WARMUP_BACKOFF);
431 }
432
433 static inline uint16_t
434 adaptive_counter_cooldown(void) {
435 return adaptive_counter_bits(ADAPTIVE_COOLDOWN_VALUE,
436 ADAPTIVE_COOLDOWN_BACKOFF);
437 }
438
439 static inline uint16_t
440 adaptive_counter_backoff(uint16_t counter) {
441 unsigned int backoff = counter & ((1<<ADAPTIVE_BACKOFF_BITS)-1);
442 backoff++;
443 if (backoff > MAX_BACKOFF_VALUE) {
444 backoff = MAX_BACKOFF_VALUE;
445 }
446 unsigned int value = (1 << backoff) - 1;
447 return adaptive_counter_bits(value, backoff);
448 }
449
450
451 /* Line array cache for tracing */
452
453 typedef struct _PyShimCodeDef {
454 const uint8_t *code;
455 int codelen;
456 int stacksize;
457 const char *cname;
458 } _PyShimCodeDef;
459
460 extern PyCodeObject *
461 _Py_MakeShimCode(const _PyShimCodeDef *code);
462
463 extern uint32_t _Py_next_func_version;
464
465
466 /* Comparison bit masks. */
467
468 /* Note this evaluates its arguments twice each */
469 #define COMPARISON_BIT(x, y) (1 << (2 * ((x) >= (y)) + ((x) <= (y))))
470
471 /*
472 * The following bits are chosen so that the value of
473 * COMPARSION_BIT(left, right)
474 * masked by the values below will be non-zero if the
475 * comparison is true, and zero if it is false */
476
477 /* This is for values that are unordered, ie. NaN, not types that are unordered, e.g. sets */
478 #define COMPARISON_UNORDERED 1
479
480 #define COMPARISON_LESS_THAN 2
481 #define COMPARISON_GREATER_THAN 4
482 #define COMPARISON_EQUALS 8
483
484 #define COMPARISON_NOT_EQUALS (COMPARISON_UNORDERED | COMPARISON_LESS_THAN | COMPARISON_GREATER_THAN)
485
486 extern int _Py_Instrument(PyCodeObject *co, PyInterpreterState *interp);
487
488 extern int _Py_GetBaseOpcode(PyCodeObject *code, int offset);
489
490
491 #ifdef __cplusplus
492 }
493 #endif
494 #endif /* !Py_INTERNAL_CODE_H */