python (3.11.7)
e' R d dl mZ d dlmZ d dlmZmZmZmZmZm Z m
Z
m Z m Z m
Z
mZmZmZmZmZmZmZ d dlmZmZ d dlZd dlmZ d dlm Z m Z d dl m Z m Z d dlmZ d d l!m"Z"m#Z# d dl$Z$d dl%Z%d
Z& G d d e Z' G d
de' Z(d Z) G d de Z* G d de Z+ G d de Z, G d de Z- G d de Z. G d de Z/ G d d e Z0 G d d e Z1 G d d!ej Z2e3d"k r ej4 dS dS )# )support) os_helper)tokenize _tokenize
untokenizeNUMBERNAMEOPSTRING ENDMARKERENCODINGtok_namedetect_encodingopen Untokenizergenerate_tokensNEWLINE!_generate_tokens_from_c_tokenizerDEDENT)BytesIOStringION)dedent)TestCasemock)VALID_UNDERSCORE_LITERALSINVALID_UNDERSCORE_LITERALS)run_test_script make_scriptc $ g }t | }|d dv}| D ]_\ }}}}} |t k r nK|r|t k r
|d |k r/t | }| d|dd|dd| d| `|S )Nz
r 10 13)len
splitlinesr r r append)
token_generator
source_stringresult num_linesmissing_trailing_nltypetokenstartendlines
F/BuggyBox/python/3.11.7/bootstrap/lib/python3.11/test/test_tokenize.py stringify_tokens_from_sourcer3 s
FM,,..//I'+69)8 C C%eUC 9 E 47??s1v7J7J ~
ATAAAuAAAEAACAABBBB M c e Zd Zd Zd Zd Zd Zd Zd Zd Z d Z
d Z d
Z d Z
d Zd
Zd Zd Zd Zd Zd Zd Zd ZdS ) TokenizeTestc
t | d }t t |j | }| |dg| z d S )Nutf-8z* ENCODING 'utf-8' (0, 0) (0, 0))r encoder3 r readline assertEqualrstripr& selfsexpectedfr* s r2 check_tokenizezTokenizeTest.check_tokenize- s~
AHHW%% & &-hqz.B.BAFF FG! **55778 9 9 9 9 9r4 c $ t d d }t t |j }| |d j t | |d j t d S )Nxr8 r ) r r9 listr r: r; r- r r )r> rA tokenss r2 test_implicit_newlinez"TokenizeTest.test_implicit_newline6 sr
CJJw'' ( (hqz**++ '222 )44444r4 c | dd | dd d}t | j }| t d 5 t | D ]} d d d d S # 1 swxY w Y d S )Nz1 + 1z NUMBER '1' (1, 0) (1, 1)
OP '+' (1, 2) (1, 3)
NUMBER '1' (1, 4) (1, 5)
z3if False:
# NL
True = False # NEWLINE
af NAME 'if' (1, 0) (1, 2)
NAME 'False' (1, 3) (1, 8)
OP ':' (1, 8) (1, 9)
NEWLINE '\n' (1, 9) (1, 10)
COMMENT '# NL' (2, 4) (2, 8)
NL '\n' (2, 8) (2, 9)
NL '\n' (3, 4) (3, 5)
INDENT ' ' (4, 0) (4, 4)
NAME 'True' (4, 4) (4, 8)
OP '=' (4, 9) (4, 10)
NAME 'False' (4, 11) (4, 16)
COMMENT '# NEWLINE' (4, 17) (4, 26)
NEWLINE '\n' (4, 26) (4, 27)
DEDENT '' (5, 0) (5, 0)
s def k(x):
x += 2
x += 5
z3unindent does not match any outer indentation level)rB r r: assertRaisesRegexIndentationErrorr )r> indent_error_filer: toks r2
test_basiczTokenizeTest.test_basic> s G &
;= &
,--6
#
#$4%>? ? ))
s A??BBc | dd | dd | dd | dd | d d
| d d | d
d | dd | dd d S )N 0xff <= 255z NUMBER '0xff' (1, 0) (1, 4)
OP '<=' (1, 5) (1, 7)
NUMBER '255' (1, 8) (1, 11)
0b10 <= 255z NUMBER '0b10' (1, 0) (1, 4)
OP '<=' (1, 5) (1, 7)
NUMBER '255' (1, 8) (1, 11)
0o123 <= 0O123z NUMBER '0o123' (1, 0) (1, 5)
OP '<=' (1, 6) (1, 8)
NUMBER '0O123' (1, 9) (1, 14)
1234567 > ~0x15z NUMBER '1234567' (1, 0) (1, 7)
OP '>' (1, 8) (1, 9)
OP '~' (1, 10) (1, 11)
NUMBER '0x15' (1, 11) (1, 15)
2134568 != 1231515z NUMBER '2134568' (1, 0) (1, 7)
OP '!=' (1, 8) (1, 10)
NUMBER '1231515' (1, 11) (1, 18)
(-124561-1) & 200000000ac OP '(' (1, 0) (1, 1)
OP '-' (1, 1) (1, 2)
NUMBER '124561' (1, 2) (1, 8)
OP '-' (1, 8) (1, 9)
NUMBER '1' (1, 9) (1, 10)
OP ')' (1, 10) (1, 11)
OP '&' (1, 12) (1, 13)
NUMBER '200000000' (1, 14) (1, 23)
0xdeadbeef != -1z NUMBER '0xdeadbeef' (1, 0) (1, 10)
OP '!=' (1, 11) (1, 13)
OP '-' (1, 14) (1, 15)
NUMBER '1' (1, 15) (1, 16)
0xdeadc0de & 12345z NUMBER '0xdeadc0de' (1, 0) (1, 10)
OP '&' (1, 11) (1, 12)
NUMBER '12345' (1, 13) (1, 18)
0xFF & 0x15 | 1234z NUMBER '0xFF' (1, 0) (1, 4)
OP '&' (1, 5) (1, 6)
NUMBER '0x15' (1, 7) (1, 11)
OP '|' (1, 12) (1, 13)
NUMBER '1234' (1, 14) (1, 18)
rB r> s r2 test_intzTokenizeTest.test_intc s: M ,
M ,
, /
- 0
0 3
5 8
. 1
0 3
0 3 r4 c | dd | dd | dd | dd d S ) Nzx = 0z NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '0' (1, 4) (1, 5)
zx = 0xfffffffffffz NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '0xfffffffffff' (1, 4) (1, 17)
zx = 123141242151251616110z NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '123141242151251616110' (1, 4) (1, 25)
zx = -15921590215012591z NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
OP '-' (1, 4) (1, 5)
NUMBER '15921590215012591' (1, 5) (1, 22)
rY rZ s r2 test_longzTokenizeTest.test_long s G &
/ 2
7 :
4 7 r4 c : | dd | dd | dd | dd | d d
| d d | d
d d S )N x = 3.14159z NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '3.14159' (1, 4) (1, 11)
x = 314159.z NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '314159.' (1, 4) (1, 11)
x = .314159z NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '.314159' (1, 4) (1, 11)
x = 3e14159z NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '3e14159' (1, 4) (1, 11)
x = 3E123z NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '3E123' (1, 4) (1, 9)
x+y = 3e-1230z NAME 'x' (1, 0) (1, 1)
OP '+' (1, 1) (1, 2)
NAME 'y' (1, 2) (1, 3)
OP '=' (1, 4) (1, 5)
NUMBER '3e-1230' (1, 6) (1, 13)
x = 3.14e159z NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '3.14e159' (1, 4) (1, 12)
rY rZ s r2
test_floatzTokenizeTest.test_float s M ,
M ,
M ,
M ,
K *
O .
N - r4 c d }t D ]&}d|v r| || | 't D ]!}| || | "d S )Nc t | d }t |j D ]\ }}}}}|t k r|c S dS )Nr8 z
invalid token)r r9 r r: r )r? rA toktyper. r/ r0 r1 s r2 number_tokenz;TokenizeTest.test_underscore_literals.<locals>.number_token s] ))**A4<QZ4H4H
!
!0sDf$$ LLL %"?r4 ()r r; r assertNotEqual)r> rj lits r2 test_underscore_literalsz%TokenizeTest.test_underscore_literals s # # # - 5 5Cczz \\# . . 4 4 4 4. 8 8C S 1 13 7 7 7 7 8 8r4 c ~ | dd | dd | dd | dd | d d
| d d | d
d | dd | dd | dd | dd | dd | dd | dd | d d | d d | d!d" | d#d$ | d%d& | d'd( | d)d* | d+d, | d-d. | d/d0 | d1d2 | d3d4 d S )5Nx = ''; y = ""a5 NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
STRING "''" (1, 4) (1, 6)
OP ';' (1, 6) (1, 7)
NAME 'y' (1, 8) (1, 9)
OP '=' (1, 10) (1, 11)
STRING '""' (1, 12) (1, 14)
x = '"'; y = "'"a6 NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
STRING '\'"\'' (1, 4) (1, 7)
OP ';' (1, 7) (1, 8)
NAME 'y' (1, 9) (1, 10)
OP '=' (1, 11) (1, 12)
STRING '"\'"' (1, 13) (1, 16)
x = "doesn't "shrink", does it"z NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
STRING '"doesn\'t "' (1, 4) (1, 14)
NAME 'shrink' (1, 14) (1, 20)
STRING '", does it"' (1, 20) (1, 31)
x = 'abc' + 'ABC'z NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
STRING "'abc'" (1, 4) (1, 9)
OP '+' (1, 10) (1, 11)
STRING "'ABC'" (1, 12) (1, 17)
y = "ABC" + "ABC"z NAME 'y' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
STRING '"ABC"' (1, 4) (1, 9)
OP '+' (1, 10) (1, 11)
STRING '"ABC"' (1, 12) (1, 17)
%x = r'abc' + r'ABC' + R'ABC' + R'ABC'a NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
STRING "r'abc'" (1, 4) (1, 10)
OP '+' (1, 11) (1, 12)
STRING "r'ABC'" (1, 13) (1, 19)
OP '+' (1, 20) (1, 21)
STRING "R'ABC'" (1, 22) (1, 28)
OP '+' (1, 29) (1, 30)
STRING "R'ABC'" (1, 31) (1, 37)
%y = r"abc" + r"ABC" + R"ABC" + R"ABC"a NAME 'y' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
STRING 'r"abc"' (1, 4) (1, 10)
OP '+' (1, 11) (1, 12)
STRING 'r"ABC"' (1, 13) (1, 19)
OP '+' (1, 20) (1, 21)
STRING 'R"ABC"' (1, 22) (1, 28)
OP '+' (1, 29) (1, 30)
STRING 'R"ABC"' (1, 31) (1, 37)
u'abc' + U'abc'z STRING "u'abc'" (1, 0) (1, 6)
OP '+' (1, 7) (1, 8)
STRING "U'abc'" (1, 9) (1, 15)
u"abc" + U"abc"z STRING 'u"abc"' (1, 0) (1, 6)
OP '+' (1, 7) (1, 8)
STRING 'U"abc"' (1, 9) (1, 15)
b'abc' + B'abc'z STRING "b'abc'" (1, 0) (1, 6)
OP '+' (1, 7) (1, 8)
STRING "B'abc'" (1, 9) (1, 15)
b"abc" + B"abc"z STRING 'b"abc"' (1, 0) (1, 6)
OP '+' (1, 7) (1, 8)
STRING 'B"abc"' (1, 9) (1, 15)
%br'abc' + bR'abc' + Br'abc' + BR'abc'a; STRING "br'abc'" (1, 0) (1, 7)
OP '+' (1, 8) (1, 9)
STRING "bR'abc'" (1, 10) (1, 17)
OP '+' (1, 18) (1, 19)
STRING "Br'abc'" (1, 20) (1, 27)
OP '+' (1, 28) (1, 29)
STRING "BR'abc'" (1, 30) (1, 37)
%br"abc" + bR"abc" + Br"abc" + BR"abc"a; STRING 'br"abc"' (1, 0) (1, 7)
OP '+' (1, 8) (1, 9)
STRING 'bR"abc"' (1, 10) (1, 17)
OP '+' (1, 18) (1, 19)
STRING 'Br"abc"' (1, 20) (1, 27)
OP '+' (1, 28) (1, 29)
STRING 'BR"abc"' (1, 30) (1, 37)
%rb'abc' + rB'abc' + Rb'abc' + RB'abc'a; STRING "rb'abc'" (1, 0) (1, 7)
OP '+' (1, 8) (1, 9)
STRING "rB'abc'" (1, 10) (1, 17)
OP '+' (1, 18) (1, 19)
STRING "Rb'abc'" (1, 20) (1, 27)
OP '+' (1, 28) (1, 29)
STRING "RB'abc'" (1, 30) (1, 37)
%rb"abc" + rB"abc" + Rb"abc" + RB"abc"a; STRING 'rb"abc"' (1, 0) (1, 7)
OP '+' (1, 8) (1, 9)
STRING 'rB"abc"' (1, 10) (1, 17)
OP '+' (1, 18) (1, 19)
STRING 'Rb"abc"' (1, 20) (1, 27)
OP '+' (1, 28) (1, 29)
STRING 'RB"abc"' (1, 30) (1, 37)
"a\
de\
fg"3 STRING '"a\\\nde\\\nfg"' (1, 0) (3, 3)
u"a\
de"/ STRING 'u"a\\\nde"' (1, 0) (2, 3)
rb"a\
d"/ STRING 'rb"a\\\nd"' (1, 0) (2, 2)
"""a\
b"""0 STRING '"""a\\\nb"""' (1, 0) (2, 4)
u"""a\
b"""1 STRING 'u"""a\\\nb"""' (1, 0) (2, 4)
rb"""a\
b\
c"""7 STRING 'rb"""a\\\nb\\\nc"""' (1, 0) (3, 4)
f"abc"/ STRING 'f"abc"' (1, 0) (1, 6)
fR"a{b}c"/ STRING 'fR"a{b}c"' (1, 0) (1, 9)
f"""abc"""0 STRING 'f"""abc"""' (1, 0) (1, 10)
f"abc\
def"1 STRING 'f"abc\\\ndef"' (1, 0) (2, 4)
Rf"abc\
def"2 STRING 'Rf"abc\\\ndef"' (1, 0) (2, 4)
rY rZ s r2 test_stringzTokenizeTest.test_string s . 1
1 4
A D
/ 2
/ 2
C
F
C
F
- 0
- 0
- 0
- 0
C F
C F
C F
C F
H '
K *
L +
r4 c | dd | dd | dd d S )N!def d22(a, b, c=2, d=2, *k): passa NAME 'def' (1, 0) (1, 3)
NAME 'd22' (1, 4) (1, 7)
OP '(' (1, 7) (1, 8)
NAME 'a' (1, 8) (1, 9)
OP ',' (1, 9) (1, 10)
NAME 'b' (1, 11) (1, 12)
OP ',' (1, 12) (1, 13)
NAME 'c' (1, 14) (1, 15)
OP '=' (1, 15) (1, 16)
NUMBER '2' (1, 16) (1, 17)
OP ',' (1, 17) (1, 18)
NAME 'd' (1, 19) (1, 20)
OP '=' (1, 20) (1, 21)
NUMBER '2' (1, 21) (1, 22)
OP ',' (1, 22) (1, 23)
OP '*' (1, 24) (1, 25)
NAME 'k' (1, 25) (1, 26)
OP ')' (1, 26) (1, 27)
OP ':' (1, 27) (1, 28)
NAME 'pass' (1, 29) (1, 33)
def d01v_(a=1, *k, **w): passa NAME 'def' (1, 0) (1, 3)
NAME 'd01v_' (1, 4) (1, 9)
OP '(' (1, 9) (1, 10)
NAME 'a' (1, 10) (1, 11)
OP '=' (1, 11) (1, 12)
NUMBER '1' (1, 12) (1, 13)
OP ',' (1, 13) (1, 14)
OP '*' (1, 15) (1, 16)
NAME 'k' (1, 16) (1, 17)
OP ',' (1, 17) (1, 18)
OP '**' (1, 19) (1, 21)
NAME 'w' (1, 21) (1, 22)
OP ')' (1, 22) (1, 23)
OP ':' (1, 23) (1, 24)
NAME 'pass' (1, 25) (1, 29)
&def d23(a: str, b: int=3) -> int: passa NAME 'def' (1, 0) (1, 3)
NAME 'd23' (1, 4) (1, 7)
OP '(' (1, 7) (1, 8)
NAME 'a' (1, 8) (1, 9)
OP ':' (1, 9) (1, 10)
NAME 'str' (1, 11) (1, 14)
OP ',' (1, 14) (1, 15)
NAME 'b' (1, 16) (1, 17)
OP ':' (1, 17) (1, 18)
NAME 'int' (1, 19) (1, 22)
OP '=' (1, 22) (1, 23)
NUMBER '3' (1, 23) (1, 24)
OP ')' (1, 24) (1, 25)
OP '->' (1, 26) (1, 28)
NAME 'int' (1, 29) (1, 32)
OP ':' (1, 32) (1, 33)
NAME 'pass' (1, 34) (1, 38)
rY rZ s r2
test_functionzTokenizeTest.test_function sr ? B ,
; > "
D G r4 c 2 | dd d S )NXif 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != 1 and 5 in 1 not in 1 is 1 or 5 is not 1: passa NAME 'if' (1, 0) (1, 2)
NUMBER '1' (1, 3) (1, 4)
OP '<' (1, 5) (1, 6)
NUMBER '1' (1, 7) (1, 8)
OP '>' (1, 9) (1, 10)
NUMBER '1' (1, 11) (1, 12)
OP '==' (1, 13) (1, 15)
NUMBER '1' (1, 16) (1, 17)
OP '>=' (1, 18) (1, 20)
NUMBER '5' (1, 21) (1, 22)
OP '<=' (1, 23) (1, 25)
NUMBER '0x15' (1, 26) (1, 30)
OP '<=' (1, 31) (1, 33)
NUMBER '0x12' (1, 34) (1, 38)
OP '!=' (1, 39) (1, 41)
NUMBER '1' (1, 42) (1, 43)
NAME 'and' (1, 44) (1, 47)
NUMBER '5' (1, 48) (1, 49)
NAME 'in' (1, 50) (1, 52)
NUMBER '1' (1, 53) (1, 54)
NAME 'not' (1, 55) (1, 58)
NAME 'in' (1, 59) (1, 61)
NUMBER '1' (1, 62) (1, 63)
NAME 'is' (1, 64) (1, 66)
NUMBER '1' (1, 67) (1, 68)
NAME 'or' (1, 69) (1, 71)
NUMBER '5' (1, 72) (1, 73)
NAME 'is' (1, 74) (1, 76)
NAME 'not' (1, 77) (1, 80)
NUMBER '1' (1, 81) (1, 82)
OP ':' (1, 82) (1, 83)
NAME 'pass' (1, 84) (1, 88)
rY rZ s r2 test_comparisonz TokenizeTest.test_comparison 0 M!O" " " " " r4 c 2 | dd d S )Nzx = 1 << 1 >> 5a6 NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '1' (1, 4) (1, 5)
OP '<<' (1, 6) (1, 8)
NUMBER '1' (1, 9) (1, 10)
OP '>>' (1, 11) (1, 13)
NUMBER '5' (1, 14) (1, 15)
rY rZ s r2
test_shiftzTokenizeTest.test_shift s, - 0 r4 c 2 | dd d S )N%x = 1 - y + 15 - 1 + 0x124 + z + a[5]a$ NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '1' (1, 4) (1, 5)
OP '-' (1, 6) (1, 7)
NAME 'y' (1, 8) (1, 9)
OP '+' (1, 10) (1, 11)
NUMBER '15' (1, 12) (1, 14)
OP '-' (1, 15) (1, 16)
NUMBER '1' (1, 17) (1, 18)
OP '+' (1, 19) (1, 20)
NUMBER '0x124' (1, 21) (1, 26)
OP '+' (1, 27) (1, 28)
NAME 'z' (1, 29) (1, 30)
OP '+' (1, 31) (1, 32)
NAME 'a' (1, 33) (1, 34)
OP '[' (1, 34) (1, 35)
NUMBER '5' (1, 35) (1, 36)
OP ']' (1, 36) (1, 37)
rY rZ s r2
test_additivezTokenizeTest.test_additive - C F r4 c 2 | dd d S )Nx = 1//1*1/5*12%0x12@42a NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '1' (1, 4) (1, 5)
OP '//' (1, 5) (1, 7)
NUMBER '1' (1, 7) (1, 8)
OP '*' (1, 8) (1, 9)
NUMBER '1' (1, 9) (1, 10)
OP '/' (1, 10) (1, 11)
NUMBER '5' (1, 11) (1, 12)
OP '*' (1, 12) (1, 13)
NUMBER '12' (1, 13) (1, 15)
OP '%' (1, 15) (1, 16)
NUMBER '0x12' (1, 16) (1, 20)
OP '@' (1, 20) (1, 21)
NUMBER '42' (1, 21) (1, 23)
rY rZ s r2 test_multiplicativez TokenizeTest.test_multiplicative , 5 8 r4 c ^ | dd | dd d S )N~1 ^ 1 & 1 |1 ^ -1a OP '~' (1, 0) (1, 1)
NUMBER '1' (1, 1) (1, 2)
OP '^' (1, 3) (1, 4)
NUMBER '1' (1, 5) (1, 6)
OP '&' (1, 7) (1, 8)
NUMBER '1' (1, 9) (1, 10)
OP '|' (1, 11) (1, 12)
NUMBER '1' (1, 12) (1, 13)
OP '^' (1, 14) (1, 15)
OP '-' (1, 16) (1, 17)
NUMBER '1' (1, 17) (1, 18)
-1*1/1+1*1//1 - ---1**1aH OP '-' (1, 0) (1, 1)
NUMBER '1' (1, 1) (1, 2)
OP '*' (1, 2) (1, 3)
NUMBER '1' (1, 3) (1, 4)
OP '/' (1, 4) (1, 5)
NUMBER '1' (1, 5) (1, 6)
OP '+' (1, 6) (1, 7)
NUMBER '1' (1, 7) (1, 8)
OP '*' (1, 8) (1, 9)
NUMBER '1' (1, 9) (1, 10)
OP '//' (1, 10) (1, 12)
NUMBER '1' (1, 12) (1, 13)
OP '-' (1, 14) (1, 15)
OP '-' (1, 16) (1, 17)
OP '-' (1, 17) (1, 18)
OP '-' (1, 18) (1, 19)
NUMBER '1' (1, 19) (1, 20)
OP '**' (1, 20) (1, 22)
NUMBER '1' (1, 22) (1, 23)
rY rZ s r2
test_unaryzTokenizeTest.test_unary* sN 0 3
5 8 r4 c 2 | dd d S )N/import sys, time
x = sys.modules['time'].time()a NAME 'import' (1, 0) (1, 6)
NAME 'sys' (1, 7) (1, 10)
OP ',' (1, 10) (1, 11)
NAME 'time' (1, 12) (1, 16)
NEWLINE '\n' (1, 16) (1, 17)
NAME 'x' (2, 0) (2, 1)
OP '=' (2, 2) (2, 3)
NAME 'sys' (2, 4) (2, 7)
OP '.' (2, 7) (2, 8)
NAME 'modules' (2, 8) (2, 15)
OP '[' (2, 15) (2, 16)
STRING "'time'" (2, 16) (2, 22)
OP ']' (2, 22) (2, 23)
OP '.' (2, 23) (2, 24)
NAME 'time' (2, 24) (2, 28)
OP '(' (2, 28) (2, 29)
OP ')' (2, 29) (2, 30)
rY rZ s r2
test_selectorzTokenizeTest.test_selectorO - N Q r4 c 2 | dd d S )N @staticmethod
def foo(x,y): passa OP '@' (1, 0) (1, 1)
NAME 'staticmethod' (1, 1) (1, 13)
NEWLINE '\n' (1, 13) (1, 14)
NAME 'def' (2, 0) (2, 3)
NAME 'foo' (2, 4) (2, 7)
OP '(' (2, 7) (2, 8)
NAME 'x' (2, 8) (2, 9)
OP ',' (2, 9) (2, 10)
NAME 'y' (2, 10) (2, 11)
OP ')' (2, 11) (2, 12)
OP ':' (2, 12) (2, 13)
NAME 'pass' (2, 14) (2, 18)
rY rZ s r2 test_methodzTokenizeTest.test_methode - ?
B
r4 c 2 | dd d S )Nz def f():
if x
passa_ NAME 'def' (1, 0) (1, 3)
NAME 'f' (1, 4) (1, 5)
OP '(' (1, 5) (1, 6)
OP ')' (1, 6) (1, 7)
OP ':' (1, 7) (1, 8)
NEWLINE '\n' (1, 8) (1, 9)
INDENT '\t' (2, 0) (2, 1)
NAME 'if' (2, 1) (2, 3)
NAME 'x' (2, 4) (2, 5)
NEWLINE '\n' (2, 5) (2, 6)
INDENT ' \t' (3, 0) (3, 9)
NAME 'pass' (3, 9) (3, 13)
DEDENT '' (4, 0) (4, 0)
DEDENT '' (4, 0) (4, 0)
rY rZ s r2 test_tabszTokenizeTest.test_tabsv s. -/ r4 c 2 | dd d S )Nu! Örter = 'places'
grün = 'green'u7 NAME 'Örter' (1, 0) (1, 5)
OP '=' (1, 6) (1, 7)
STRING "'places'" (1, 8) (1, 16)
NEWLINE '\n' (1, 16) (1, 17)
NAME 'grün' (2, 0) (2, 4)
OP '=' (2, 5) (2, 6)
STRING "'green'" (2, 7) (2, 14)
rY rZ s r2 test_non_ascii_identifiersz'TokenizeTest.test_non_ascii_identifiers s- @ C r4 c 2 | dd d S )N# Örter = u'places'
grün = U'green'u7 NAME 'Örter' (1, 0) (1, 5)
OP '=' (1, 6) (1, 7)
STRING "u'places'" (1, 8) (1, 17)
NEWLINE '\n' (1, 17) (1, 18)
NAME 'grün' (2, 0) (2, 4)
OP '=' (2, 5) (2, 6)
STRING "U'green'" (2, 7) (2, 15)
rY rZ s r2 test_unicodezTokenizeTest.test_unicode - B E r4 c | dd | dd | dd | dd | d d
| d d | d
d | dd | dd | dd | dd | dd | dd | dd | d d | d d | d!d" | d#d$ | d%d& | d'd( | d)d* d S )+N async = 1z NAME 'async' (1, 0) (1, 5)
OP '=' (1, 6) (1, 7)
NUMBER '1' (1, 8) (1, 9)
a = (async = 1)a8 NAME 'a' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
OP '(' (1, 4) (1, 5)
NAME 'async' (1, 5) (1, 10)
OP '=' (1, 11) (1, 12)
NUMBER '1' (1, 13) (1, 14)
OP ')' (1, 14) (1, 15)
async()z NAME 'async' (1, 0) (1, 5)
OP '(' (1, 5) (1, 6)
OP ')' (1, 6) (1, 7)
class async(Bar):passa< NAME 'class' (1, 0) (1, 5)
NAME 'async' (1, 6) (1, 11)
OP '(' (1, 11) (1, 12)
NAME 'Bar' (1, 12) (1, 15)
OP ')' (1, 15) (1, 16)
OP ':' (1, 16) (1, 17)
NAME 'pass' (1, 17) (1, 21)
class async:passz NAME 'class' (1, 0) (1, 5)
NAME 'async' (1, 6) (1, 11)
OP ':' (1, 11) (1, 12)
NAME 'pass' (1, 12) (1, 16)
await = 1z NAME 'await' (1, 0) (1, 5)
OP '=' (1, 6) (1, 7)
NUMBER '1' (1, 8) (1, 9)
foo.asyncz NAME 'foo' (1, 0) (1, 3)
OP '.' (1, 3) (1, 4)
NAME 'async' (1, 4) (1, 9)
async for a in b: passa; NAME 'async' (1, 0) (1, 5)
NAME 'for' (1, 6) (1, 9)
NAME 'a' (1, 10) (1, 11)
NAME 'in' (1, 12) (1, 14)
NAME 'b' (1, 15) (1, 16)
OP ':' (1, 16) (1, 17)
NAME 'pass' (1, 18) (1, 22)
async with a as b: passa< NAME 'async' (1, 0) (1, 5)
NAME 'with' (1, 6) (1, 10)
NAME 'a' (1, 11) (1, 12)
NAME 'as' (1, 13) (1, 15)
NAME 'b' (1, 16) (1, 17)
OP ':' (1, 17) (1, 18)
NAME 'pass' (1, 19) (1, 23)
async.fooz NAME 'async' (1, 0) (1, 5)
OP '.' (1, 5) (1, 6)
NAME 'foo' (1, 6) (1, 9)
asyncz/ NAME 'async' (1, 0) (1, 5)
async
#comment
awaitz NAME 'async' (1, 0) (1, 5)
NEWLINE '\n' (1, 5) (1, 6)
COMMENT '#comment' (2, 0) (2, 8)
NL '\n' (2, 8) (2, 9)
NAME 'await' (3, 0) (3, 5)
async
...
awaitz NAME 'async' (1, 0) (1, 5)
NEWLINE '\n' (1, 5) (1, 6)
OP '...' (2, 0) (2, 3)
NEWLINE '\n' (2, 3) (2, 4)
NAME 'await' (3, 0) (3, 5)
async
awaitz NAME 'async' (1, 0) (1, 5)
NEWLINE '\n' (1, 5) (1, 6)
NAME 'await' (2, 0) (2, 5)
foo.async + 1z NAME 'foo' (1, 0) (1, 3)
OP '.' (1, 3) (1, 4)
NAME 'async' (1, 4) (1, 9)
OP '+' (1, 10) (1, 11)
NUMBER '1' (1, 12) (1, 13)
async def foo(): passa; NAME 'async' (1, 0) (1, 5)
NAME 'def' (1, 6) (1, 9)
NAME 'foo' (1, 10) (1, 13)
OP '(' (1, 13) (1, 14)
OP ')' (1, 14) (1, 15)
OP ':' (1, 15) (1, 16)
NAME 'pass' (1, 17) (1, 21)
Nasync def foo():
def foo(await):
await = 1
if 1:
await
async += 1
a NAME 'async' (1, 0) (1, 5)
NAME 'def' (1, 6) (1, 9)
NAME 'foo' (1, 10) (1, 13)
OP '(' (1, 13) (1, 14)
OP ')' (1, 14) (1, 15)
OP ':' (1, 15) (1, 16)
NEWLINE '\n' (1, 16) (1, 17)
INDENT ' ' (2, 0) (2, 2)
NAME 'def' (2, 2) (2, 5)
NAME 'foo' (2, 6) (2, 9)
OP '(' (2, 9) (2, 10)
NAME 'await' (2, 10) (2, 15)
OP ')' (2, 15) (2, 16)
OP ':' (2, 16) (2, 17)
NEWLINE '\n' (2, 17) (2, 18)
INDENT ' ' (3, 0) (3, 4)
NAME 'await' (3, 4) (3, 9)
OP '=' (3, 10) (3, 11)
NUMBER '1' (3, 12) (3, 13)
NEWLINE '\n' (3, 13) (3, 14)
DEDENT '' (4, 2) (4, 2)
NAME 'if' (4, 2) (4, 4)
NUMBER '1' (4, 5) (4, 6)
OP ':' (4, 6) (4, 7)
NEWLINE '\n' (4, 7) (4, 8)
INDENT ' ' (5, 0) (5, 4)
NAME 'await' (5, 4) (5, 9)
NEWLINE '\n' (5, 9) (5, 10)
DEDENT '' (6, 0) (6, 0)
DEDENT '' (6, 0) (6, 0)
NAME 'async' (6, 0) (6, 5)
OP '+=' (6, 6) (6, 8)
NUMBER '1' (6, 9) (6, 10)
NEWLINE '\n' (6, 10) (6, 11)
)async def foo():
async for i in 1: passa NAME 'async' (1, 0) (1, 5)
NAME 'def' (1, 6) (1, 9)
NAME 'foo' (1, 10) (1, 13)
OP '(' (1, 13) (1, 14)
OP ')' (1, 14) (1, 15)
OP ':' (1, 15) (1, 16)
NEWLINE '\n' (1, 16) (1, 17)
INDENT ' ' (2, 0) (2, 2)
NAME 'async' (2, 2) (2, 7)
NAME 'for' (2, 8) (2, 11)
NAME 'i' (2, 12) (2, 13)
NAME 'in' (2, 14) (2, 16)
NUMBER '1' (2, 17) (2, 18)
OP ':' (2, 18) (2, 19)
NAME 'pass' (2, 20) (2, 24)
DEDENT '' (3, 0) (3, 0)
async def foo(async): awaitah NAME 'async' (1, 0) (1, 5)
NAME 'def' (1, 6) (1, 9)
NAME 'foo' (1, 10) (1, 13)
OP '(' (1, 13) (1, 14)
NAME 'async' (1, 14) (1, 19)
OP ')' (1, 19) (1, 20)
OP ':' (1, 20) (1, 21)
NAME 'await' (1, 22) (1, 27)
@def f():
def baz(): pass
async def bar(): pass
await = 2a NAME 'def' (1, 0) (1, 3)
NAME 'f' (1, 4) (1, 5)
OP '(' (1, 5) (1, 6)
OP ')' (1, 6) (1, 7)
OP ':' (1, 7) (1, 8)
NEWLINE '\n' (1, 8) (1, 9)
NL '\n' (2, 0) (2, 1)
INDENT ' ' (3, 0) (3, 2)
NAME 'def' (3, 2) (3, 5)
NAME 'baz' (3, 6) (3, 9)
OP '(' (3, 9) (3, 10)
OP ')' (3, 10) (3, 11)
OP ':' (3, 11) (3, 12)
NAME 'pass' (3, 13) (3, 17)
NEWLINE '\n' (3, 17) (3, 18)
NAME 'async' (4, 2) (4, 7)
NAME 'def' (4, 8) (4, 11)
NAME 'bar' (4, 12) (4, 15)
OP '(' (4, 15) (4, 16)
OP ')' (4, 16) (4, 17)
OP ':' (4, 17) (4, 18)
NAME 'pass' (4, 19) (4, 23)
NEWLINE '\n' (4, 23) (4, 24)
NL '\n' (5, 0) (5, 1)
NAME 'await' (6, 2) (6, 7)
OP '=' (6, 8) (6, 9)
NUMBER '2' (6, 10) (6, 11)
DEDENT '' (7, 0) (7, 0)
Fasync def f():
def baz(): pass
async def bar(): pass
await = 2a NAME 'async' (1, 0) (1, 5)
NAME 'def' (1, 6) (1, 9)
NAME 'f' (1, 10) (1, 11)
OP '(' (1, 11) (1, 12)
OP ')' (1, 12) (1, 13)
OP ':' (1, 13) (1, 14)
NEWLINE '\n' (1, 14) (1, 15)
NL '\n' (2, 0) (2, 1)
INDENT ' ' (3, 0) (3, 2)
NAME 'def' (3, 2) (3, 5)
NAME 'baz' (3, 6) (3, 9)
OP '(' (3, 9) (3, 10)
OP ')' (3, 10) (3, 11)
OP ':' (3, 11) (3, 12)
NAME 'pass' (3, 13) (3, 17)
NEWLINE '\n' (3, 17) (3, 18)
NAME 'async' (4, 2) (4, 7)
NAME 'def' (4, 8) (4, 11)
NAME 'bar' (4, 12) (4, 15)
OP '(' (4, 15) (4, 16)
OP ')' (4, 16) (4, 17)
OP ':' (4, 17) (4, 18)
NAME 'pass' (4, 19) (4, 23)
NEWLINE '\n' (4, 23) (4, 24)
NL '\n' (5, 0) (5, 1)
NAME 'await' (6, 2) (6, 7)
OP '=' (6, 8) (6, 9)
NUMBER '2' (6, 10) (6, 11)
DEDENT '' (7, 0) (7, 0)
rY rZ s r2
test_asynczTokenizeTest.test_async s K *
- 0
I (
3 6
. 1
K *
K *
4 7
5 8
K *
G &
4 7
/ 2
N -
O .
3 6
#* * * X
*
= @
# # # J
$ $ $ $ $ r4 N)__name__
__module__ __qualname__rB rH rN r[ r] rf rn r r r r r r r r r r r r r r4 r2 r6 r6 & sO 9 9 95 5 5# # #J7 7 7 r 0& & & P
8
8
8 d d d L: : : x$ $ $ L
. (# # # J , " *
P P P P P r4 r6 c e Zd Zd ZdS )GenerateTokensTestc t | }t t |j | }| || d S N)r r3 r r: r; r< r& r= s r2 rB z!GenerateTokensTest.check_tokenize sZ
QKK-oaj.I.I1MM !2!2!=!=!?!?@@@@@r4 N)r r r rB r r4 r2 r r s( A A A A Ar4 r c g }t t | d j }|D ]q\ }}}}}|t k rGd|v rC| t dft dft t | ft dfg Z|
||f rt | d S )Nr8 .Decimalrk ))
r r r9 r: r extendr r
r reprr' r decode)r? r* gtoknumtokval_s r2 decistmtr s
F'**++455A$% , , 1a V v
MMy!S f &S
MM66* + + + + f $ $W - --r4 c e Zd Zd ZdS )TestMiscc . ddl m} d}| t | d | t t
| d | t
t | |d d S )Nr )r z+21.3e-5*-.1234/81.7z8+Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7')z-3.2171603427[0-9]*e-0+7z!-3.217160342717258261933904529E-7)decimalr r; r assertRegexr eval)r> r r? s r2
test_decistmtzTestMisc.test_decistmt s $##### " !S U U U
d1gg (BCCC
hqkk** !DEE G G G G Gr4 N)r r r r r r4 r2 r r s( G G G G Gr4 r c 6 e Zd ZdZd Zd Zd Zd Zd Zd Z dS ) TestTokenizerAdheresToPep0263zU
Test that tokenizer adheres to the coding behaviour stipulated in PEP 0263.
c t j t j t d| }t
| t |d d S )N tokenizedatarb)ospathjoindirname__file__
TestRoundtripcheck_roundtripr )r> filenamer s r2 _testFilez'TestTokenizerAdheresToPep0263._testFile sM w||BGOOH 5 5~xPP%%dDt,<,<=====r4 c 4 d}| | d S )Nz9tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txtr r> rA s r2 'test_utf8_coding_cookie_and_no_utf8_bomzETestTokenizerAdheresToPep0263.test_utf8_coding_cookie_and_no_utf8_bom G qr4 c L d}| t | j | dS )a
As per PEP 0263, if a file starts with a utf-8 BOM signature, the only
allowed encoding for the comment is 'utf-8'. The text file used in
this test starts with a BOM signature, but specifies latin1 as the
coding, so verify that a SyntaxError is raised, which matches the
behaviour of the interpreter when it encounters a similar condition.
z8tokenize_tests-latin1-coding-cookie-and-utf8-bom-sig.txtN assertRaises SyntaxErrorr r s r2 &test_latin1_coding_cookie_and_utf8_bomzDTestTokenizerAdheresToPep0263.test_latin1_coding_cookie_and_utf8_bom s*
G +t~q99999r4 c 4 d}| | d S )Nz9tokenize_tests-no-coding-cookie-and-utf8-bom-sig-only.txtr r s r2 "test_no_coding_cookie_and_utf8_bomz@TestTokenizerAdheresToPep0263.test_no_coding_cookie_and_utf8_bom r r4 c 4 d}| | d S )Nz6tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txtr r s r2 $test_utf8_coding_cookie_and_utf8_bomzBTestTokenizerAdheresToPep0263.test_utf8_coding_cookie_and_utf8_bom s D qr4 c | t | j d | t | j d d S )Nz
bad_coding.pyzbad_coding2.pyr rZ s r2 test_bad_coding_cookiez4TestTokenizerAdheresToPep0263.test_bad_coding_cookie s= +t~GGG +t~7GHHHHHr4 N)
r r r __doc__r r r r r
r r r4 r2 r r s} > > > : : : I I I I Ir4 r c e Zd Zd Zd ZdS )
Test_Tokenizec d}| d dfd}t t |d dd }dg}| ||d d S )
N "ЉЊЈЁЂ"r8 Fc sd S dS NTr4 r )firstr1 s r2 r: zNTest_Tokenize.test__tokenize_decodes_with_specified_encoding.<locals>.readline s
sr4 encoding rE r )r r )r r z bytes not decoded with encoding)r9 rF r r; )r> literalr: rG expected_tokensr r1 s @@r2 .test__tokenize_decodes_with_specified_encodingz<Test_Tokenize.test__tokenize_decodes_with_specified_encoding s ~~g&& i7;;;<<QrTBNO : < < < < <r4 c ddfd}t t |d d d }dg}| ||d d S )Nr Fc sd S dS r r )r r s r2 r: zQTest_Tokenize.test__tokenize_does_not_decode_with_encoding_none.<locals>.readline s
sr4 r rE r z*string not tokenized when encoding is None)rF r r; )r> r: rG r r r s @@r2 1test__tokenize_does_not_decode_with_encoding_nonez?Test_Tokenize.test__tokenize_does_not_decode_with_encoding_none s i488899#2#>NO E G G G G Gr4 N)r r r r r r r4 r2 r r s7 < < <$G G G G Gr4 r c e Zd Zd Zd Zd Zd Zd Zd Zd Z d Z
d Z d
Z d Z
d Zd
Zd Zd Zd Zd Zd Zd Zd ZdS )TestDetectEncodingc dfd}|S )Nr c V t k rt } dz
| S Nr )r%
StopIteration)r1 indexliness r2 r: z1TestDetectEncoding.get_readline.<locals>.readline3 s1 E
""##<D QJEKr4 r )r> r( r: r' s ` @r2 get_readlinez TestDetectEncoding.get_readline1 s. r4 c d}t | | \ }}| |d | |t |d d d S )N) # something
print(something)
do_something(else)
r8 )r r) r; rF r> r( r consumed_liness r2 test_no_bom_no_encoding_cookiez1TestDetectEncoding.test_no_bom_no_encoding_cookie< sk
$343D3DU3K3K#L#L . 7+++ eBQBi 99999r4 c d}t | | \ }}| |d | |ddg d S )N)s # something
r, r- utf-8-sigr+ r, r r) r; r/ s r2 test_bom_no_cookiez%TestDetectEncoding.test_bom_no_cookieF ss
$343D3DU3K3K#L#L . ;/// *,AB D D D D Dr4 c d}t | | \ }}| |d | |dg d S )N) # -*- coding: latin-1 -*-
r, r-
iso-8859-1r7 r4 r/ s r2 test_cookie_first_line_no_bomz0TestDetectEncoding.test_cookie_first_line_no_bomQ sa
$343D3DU3K3K#L#L . <000 *H)IJJJJJr4 c d}t | | \ }}| |d | |dg d S )N)s # coding=utf-8
r, r- r3 s # coding=utf-8
r4 r/ s r2 &test_matched_bom_and_cookie_first_linez9TestDetectEncoding.test_matched_bom_and_cookie_first_line[ sa
$343D3DU3K3K#L#L . ;/// *=)>?????r4 c v d}| | }| t t | d S )N)s# # vim: set fileencoding=ascii :
r, r- r) r r r r> r( r: s r2 <test_mismatched_bom_and_cookie_first_line_raises_syntaxerrorzOTestDetectEncoding.test_mismatched_bom_and_cookie_first_line_raises_syntaxerrore s<
$$U++ +AAAAAr4 c d}t | | \ }}| |d ddg}| || d S )N)
#! something
# vim: set fileencoding=ascii :
r, r- asciirA rB r4 r> r( r r0 r@ s r2 test_cookie_second_line_no_bomz1TestDetectEncoding.test_cookie_second_line_no_bomn sh
$343D3DU3K3K#L#L . 7+++%'KL 22222r4 c d}t | | \ }}| |d | |ddg d S )N) #! something
f# coding=utf-8
r, r- r3 rA rH r4 r/ s r2 'test_matched_bom_and_cookie_second_linez:TestDetectEncoding.test_matched_bom_and_cookie_second_linez ss
$343D3DU3K3K#L#L . ;/// +-AB D D D D Dr4 c v d}| | }| t t | d S )N)rG rB r, r- r= r> s r2 =test_mismatched_bom_and_cookie_second_line_raises_syntaxerrorzPTestDetectEncoding.test_mismatched_bom_and_cookie_second_line_raises_syntaxerror s<
$$U++ +AAAAAr4 c d}t | | \ }}| |d dg}| || d S )N) print('£')
% # vim: set fileencoding=iso8859-15 :
print('€')
r8 rM r4 rD s r2 /test_cookie_second_line_noncommented_first_linezBTestDetectEncoding.test_cookie_second_line_noncommented_first_line se
$343D3DU3K3K#L#L . 7+++*+ 22222r4 c d}t | | \ }}| |d ddg}| || d S )N)
#print('£')
rN rO
iso8859-15rR rN r4 rD s r2 ,test_cookie_second_line_commented_first_linez?TestDetectEncoding.test_cookie_second_line_commented_first_line sh
$343D3DU3K3K#L#L . <000+-VW 22222r4 c d}t | | \ }}| |d ddg}| || d S )N)
rN rO rS rV rN r4 rD s r2 (test_cookie_second_line_empty_first_linez;TestDetectEncoding.test_cookie_second_line_empty_first_line sh
$343D3DU3K3K#L#L . <000DE 22222r4 c d}|D ]y}dD ]t}| d| }dd| d z dz dd f}| | }t | \ }}| |d
uzd S ) N)zlatin-1r8 z iso-latin-1z latin-1-unixziso-8859-1-unixziso-latin-1-mac-r rZ #!/usr/bin/python
# coding: rC rV s print(things)
s do_something += 4
r8 replacer9 r) r r; r> encodingsr repencr( rlfoundr0 s r2 test_latin1_normalizationz,TestDetectEncoding.test_latin1_normalization s ; ! 6 6H!
6
6 &&sC00/&G)<)<<uD+/1 &&u--(7(;(;%~ 5555
6 6 6r4 c v d}| | }| t t | d S )Ns
print("")r= r> s r2 test_syntaxerror_latin1z*TestDetectEncoding.test_syntaxerror_latin1 s< $$U++ +AAAAAr4 c d}|D ]x}dD ]s}| d| }dd| d z dz df}| | }t | \ }}| |d tyd S )
N)r8 z utf-8-macz
utf-8-unixrY rZ r[ r\ rC rV s 1 + 3
r8 r] r_ s r2 test_utf8_normalizationz*TestDetectEncoding.test_utf8_normalization s 8 ! 1 1H!
1
1 &&sC00/&G)<)<<uD#% &&u--(7(;(;%~ 0000
1 1 1r4 c
| d }t | \ }}| |d | |dg t | d \ }}| |d | |g | d }t | \ }}| |d | |dg | d }t | \ }}| |d | |g | d }| t t | d S ) N)r, r8 r, r )s print(something)
r3 )s )s # coding: bad
)r) r r; r r r> r: r r0 s r2 test_short_filesz#TestDetectEncoding.test_short_files s~ $$%=>>#28#<#< . 7+++ *?)@AAA#243D3DR3H3H#I#I . 7+++ ,,,$$%IJJ#28#<#< . ;/// *?)@AAA$$%788#28#<#< . ;/// ,,,$$%:;; +AAAAAr4 c | d }t | \ }}| |d | |dg d S )N) print("#coding=fake")r8 ro )r) r r; rl s r2 test_false_encodingz&TestDetectEncoding.test_false_encoding s^ $$%@AA#28#<#< . 7+++ *B)CDDDDDr4 c t j dz }| t j | dD ]}t |d| 5 }t d|z | t d| d d d n # 1 swxY w Y t
| 5 }| |j | | |j d d d d n # 1 swxY w Y t |dd 5 }t d| d d d n # 1 swxY w Y t
| 5 }| |j d | |j d d d d d S # 1 swxY w Y d S )
Nz.py)z iso-8859-15r8 wr z # coding: %sfileu print('euro:€')rr3 )
r TESTFN
addCleanupunlinkr print
tokenize_openr; r mode)r> r r fps r2 test_openz TestDetectEncoding.test_open sy #e+ ((333 1 / /Hhh777
72nx/b9999,26666
7
7
7
7
7
7
7
7
7
7
7
7
7
7
7 x((
/B h777 #...
/
/
/
/
/
/
/
/
/
/
/
/
/
/
/
(C+
6
6
6 3" (r 2 2 2 2 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3
8
$
$ + R[+ 6 6 6 RWc * * * + + + + + + + + + + + + + + + + + +sH &A88A< ?A< 7CC C 3DDD+7E//E36E3c d}d G fdd }| t 5 || }|`t |j d d d n # 1 swxY w Y | t d | 5 || }t |j d d d d S # 1 swxY w Y d S )Nsome_file_pathrg c e Zd Zd Z fdZdS );TestDetectEncoding.test_filename_in_exception.<locals>.Bunkc 0 || _ || _ d| _ d S )Nr )name_lines_index)r> r( r s r2 __init__zDTestDetectEncoding.test_filename_in_exception.<locals>.Bunk.__init__ s # r4 c | j t k rt | j }| xj dz
c_ |S r% )r r% r& )r> r1 r( s r2 r: zDTestDetectEncoding.test_filename_in_exception.<locals>.Bunk.readline s= ;#e**,,'' T[) q r4 N)r r r r r: )r( s r2 Bunkr s=
r4 r z.*{})r r r r r: rJ format)r> r r insr( s @r2 test_filename_in_exceptionz-TestDetectEncoding.test_filename_in_exception s{
{
+
+ * *$ud##C CL ) ) ) * * * * * * * * * * * * * * *
#
#K t1D1D
E
E * *$ud##C CL ) ) ) * * * * * * * * * * * * * * * * * *s# #A A"%A"!CC C c t d }t j d| 5 | t t
d d d d n # 1 swxY w Y | |j d S )Ns #coding:xxxztokenize._builtin_open) return_valuefoobar)r r patchr r rz
assertTrueclosed)r> ms r2 test_open_errorz"TestDetectEncoding.test_open_error% s N # #
Z0q
A
A
A D D k=( C C C D D D D D D D D D D D D D D D !!!!!s "AAAN)r r r r) r1 r5 r9 r; r? rE rI rK rP rT rW re rh rj rm rp r} r r r r4 r2 r" r" / s[ : : : D D DK K K@ @ @B B B
3
3
3
D
D
DB B B 3 3 3 3 3 3 3 3 3
6
6
6 B B B 1 1 1B B B.E E E+ + +(* * *8" " " " "r4 r" c 2 e Zd Zd Zd Zd Zd Zd Zd ZdS ) TestTokenizec h
dd l }t d
fd}
fd}dfd}|j }|j }||_ ||_ t | }| t | g d ||_ ||_ n# ||_ ||_ w xY w|
d S )Nr c ddgfS )N first secondr )r: r s r2 mock_detect_encodingz8TestTokenize.test_tokenize.<locals>.mock_detect_encoding3 s h 22 2r4 c V |g } | }|r| | #|S r )r' )r: r out next_line
encoding_useds r2 mock__tokenizez2TestTokenize.test_tokenize.<locals>.mock__tokenize6 s? $MC
$HJJ JJy)))
r4 c ` dz
dk rdS t S )Nr r4 )strr9 )counters r2
mock_readlinez1TestTokenize.test_tokenize.<locals>.mock_readlineB s2 qLG!||sw<<&&(( (r4 )r r 1 2 3 4)r objectr r r; rF ) r> tokenize_moduler r r orig_detect_encodingorig__tokenizeresultsr r r s @@@r2
test_tokenizezTestTokenize.test_tokenize/ s$ ****88
3 3 3 3 3 ) ) ) ) ) / > (2*>'$2! 7 }--G T']] J J J
L
L
L /CO +(6O % % /CO +(6O % 6 6 6 6 11111s 4B Bc g }t d D ]+}| d | ,| d d | }t t t
| d j }| |d j
d d S )Ni zdef i{i}(): return {i})iOK
r8 ) ranger' r r rF r r r9 r: r; string)r> bufr tokss r2 test_oneline_defsz TestTokenize.test_oneline_defsW s s = =A JJ/666;; < < < <
4iinn HWSZZ%8%8 9 9 BCCDD b $/////r4 c t t t | d j }t | }| t | d|z | t |d j t t t | D ]B}| t ||dz j t || C| t |d|z j t t j | t |d|z j t t j
d S )Nr8 r r r r. )rF r r r9 r: r% r; r
exact_typer
r r. r r )r> opstroptypesrG num_optypesr s r2 assertExactTypeEqualz!TestTokenize.assertExactTypeEqualc sM hwu||G'<'< = = FGGHH'll Va+o666 &)"67!(+ - - -{## 3 3A XfQUm&> ? %gaj 1
3
3
3
3 &[ "9"DE!%-0 2 2 2 &[ "9"DE!%/2 4 4 4 4 4r4 c
| dt j t j | dt j t j | dt j | dt j | dt j | dt j | dt j
| dt j | d t j | d
t j
| d t j | d t j | d
t j | dt j | dt j | dt j | dt j t j | dt j | dt j | dt j | dt j | dt j | dt j | dt j | dt j | dt j | dt j | d t j | d t j! | d t j" | d t j# | d t j$ | d!t j% | d"t j& | d"t j& | d#t j' | d$t j( | d%t j) | d&t j* | d't j+ | d(t j, | d)t j- | d*t j. | d+t j/ | d,t j0 | d-tb t j td t j tb t j td t j tb t j td | d.t j t j2 t j t j2 t j t j2 t j | d/t j t j t j1 t j t j2 t j d S )0Nz()z[]:,;+rZ */|&<>=r %z{}z==z!=z<=z>=~^z<<z>>z**z+=z-=z*=z/=z%=z&=z|=z^=z<<=z>>=z**=z//z//=z:=z...z->@z@=za**2+b**2==c**2z {1, 2, 3}z
^(x & 0x1))3r r. LPARRPARLSQBRSQBCOLONCOMMASEMIPLUSMINUSSTARSLASHVBARAMPERLESSGREATEREQUALDOTPERCENTLBRACERBRACEEQEQUALNOTEQUAL LESSEQUAL GREATEREQUALTILDE
CIRCUMFLEX LEFTSHIFT
RIGHTSHIFT
DOUBLESTAR PLUSEQUALMINEQUAL STAREQUAL
SLASHEQUAL PERCENTEQUAL
AMPEREQUAL VBAREQUALCIRCUMFLEXEQUALLEFTSHIFTEQUALRIGHTSHIFTEQUALDOUBLESTAREQUAL DOUBLESLASHDOUBLESLASHEQUAL
COLONEQUALELLIPSISRARROWATATEQUALr r rZ s r2 test_exact_typez TestTokenize.test_exact_typeq s !!$
EJ??? !!$
EJ??? !!#u{333 !!#u{333 !!#uz222 !!#uz222 !!#u{333 !!#uz222 !!#u{333 !!#uz222 !!#u{333 !!#uz222 !!#u}555 !!#u{333 !!#uy111 !!#u}555 !!$ elCCC !!$
666 !!$777 !!$888 !!$(:;;; !!#u{333 !!#u'7888 !!$888 !!$(8999 !!$(8999 !!$888 !!$777 !!$888 !!$(8999 !!$(:;;; !!$(8999 !!$888 !!$(=>>> !!$(=>>> !!%)=>>> !!%)>??? !!%)>??? !!$(9::: !!%)?@@@ !!$(8999 !!% 888 !!$ 555 !!#ux000 !!$
666 !!"3"&(8&"'*"&(8&"'-"&(8& B B B
!!+"',"', "', "',"', 0 0 0
!!,"'"2"'*"'*ek5<"'* . . . . .r4 c F | dt j d S )Nz @ )r r. r rZ s r2 %test_pathological_trailing_whitespacez2TestTokenize.test_pathological_trailing_whitespace s !!-:::::r4 c d}t j t j t j t j t j t j g}t t t |
d j }| t |d j t t t! d D ]B}| t ||dz j t || C| t |d j t t j d S )Nz b = 1
#testr8 r r r )r. r r r r NLCOMMENTrF r r r9 r: r; r r r
r r )r> sourcer rG r s r2 5test_comment_at_the_end_of_the_source_without_newlinezBTestTokenize.test_comment_at_the_end_of_the_source_without_newline s ! :u{EL%-QVQY[`[hihwv}}W'='= > > GHHII &)"67(9KLLLq _ _A XfQUm&> ?/Z[J\A] ^ ^ ^ ^ &*"78(5?:STTTTTr4 N) r r r r r r r r r r r4 r2 r r - su &2 &2 &2P 0 0 0 4 4 4 ?. ?. ?.B; ; ; U U U U Ur4 r c e Zd Zd Zd Zd ZdS )UntokenizeTestc Z t }d|_ d|_ | t 5 }| d d d d n # 1 swxY w Y | |j j d d | t |j d d S )Nr. )r r r z'start (1,3) precedes previous end (2,2))r. r ) r prev_rowprev_colr
ValueErroradd_whitespacer; exceptionargs)r> ucms r2 test_bad_input_orderz#UntokenizeTest.test_bad_input_order s MM
z
*
* $b
U # # # $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ *1-9 ; ; ;
*a&6>>>>>s AA A c V t }d|_ d|_ g |_ | d | |j dg d|_ | d | |j g d t | d d S )Nr )r. r \
r. ) r )r z\
\
r! za
b
c
\
c
)r r
r rG r
r; r r )r> r s r2 test_backslash_continuationz*UntokenizeTest.test_backslash_continuation s MM
F8,,,
#?#?#?@@@%%d,HIIIIIr4 c p t }t df}t df|g}| |t g | |j dg t }| | t |g d t }| | t | d | |j d | t t | d d S )NHellor8 zHello s Hello ) r r r
compatiterr; rG r r )r> r r. rG s r2 test_iter_compatz UntokenizeTest.test_iter_compat s MMwW%u- R!!! H:... MM dE7mm44h??? MM d6ll33X>>> W--- DLL119=====r4 N)r r r r r r r r4 r2 r r sD
?
?
? J J J > > > > >r4 r c > e Zd Zd Zd Zd Zd Zd Zd Zd Z d Z
d S )
r c t |t r| d }n(| }| t | d j }t t | }d |D }t | }t | d j }d t | D }| || t | } t | d j }
d t |
D } | | | dS )a)
Test roundtrip for `untokenize`. `f` is an open file or a string.
The source code in f is tokenized to both 5- and 2-tuples.
Both sequences are converted back to source code via
tokenize.untokenize(), and the latter tokenized again to 2-tuples.
The test fails if the 3 pair tokenizations do not match.
When untokenize bugs are fixed, untokenize with 5-tuples should
reproduce code that does not contain a backslash continuation
following spaces. A proper test should test this.
r8 T)keependsc " g | ] }|d d
S Nr. r .0rM s r2
<listcomp>z1TestRoundtrip.check_roundtrip.<locals>.<listcomp> s ...s3rr7...r4 c " g | ] }|d d
S r r r! s r2 r# z1TestRoundtrip.check_roundtrip.<locals>.<listcomp> @@@SRaR@@@r4 c " g | ] }|d d
S r r r! s r2 r# z1TestRoundtrip.check_roundtrip.<locals>.<listcomp> r% r4 N)
isinstancer r9 readcloser r& __next__rF r r r; ) r> rA coder: tokens5tokens2 bytes_from2 readline2
tokens2_from2 bytes_from5 readline5
tokens2_from5s r2 r z TestRoundtrip.check_roundtrip sM a 88G$$DD6688D
GGIII6677@x))**..g... )) ///>>??H @@HY,?,?@@@
000 )) ///>>??H @@HY,?,?@@@
00000r4 c | d | d | d t j dd }t |d 5 }| | d d d n # 1 swxY w Y | d | d | d
| d d S ) Nzif x == 1:
print(x)
z # This is a comment
# This also
zif x == 1 :
print(x)
ztokenize_tests.txtr )subdirr zoif x == 1:
# A comment by itself.
print(x) # Comment here, too.
# Another comment.
after_if = True
zPif (x # The comments need to go in the right place
== 1):
print('x==1')
zclass Test: # A comment here
# A comment with weird indent
after_com = 5
def x(m): return m*5 # a one liner
def y(m): # A whitespace after the colon
return y*4 # 3-space indent
zttry: import somemodule
except ImportError: # comment
print('Can not import' # comment2
)else: print('Loaded')
)r r findfiler )r> fnrA s r2 test_roundtripz TestRoundtrip.test_roundtrip so
. / / / - . . .
, - - -
2>
J
J
J
"d^^ $q # # # $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ 1 2 2 2
3 4 4 4
B C C C
9 : : : : :s &BB B c 0 | d d S )Nzna = (3,4,
5,6)
y = [3, 4,
5]
z = {'a': 5,
'b':15, 'c':True}
x = len(y) + 5 - a[
3] - a[2]
+ len(z) - z[
'b']
r rZ s r2 test_continuationz TestRoundtrip.test_continuation3 s) & ' ' ' ' 'r4 c Z | d | d d S )Nz)x=1+\
1
# This is a comment\
# This also
z# Comment \
x = 0r: rZ s r2 r z)TestRoundtrip.test_backslash_continuation@ sH - . . .
% & & & & &r4 c 0 | d d S )Nz'' ''r: rZ s r2 test_string_concatenationz'TestRoundtrip.test_string_concatenationI s W%%%%%r4 c R dd l }dd l}t j t
p t j }| t j | | d }| t j |d dD ]8}| t j |d |z 9t j d s| |d }|D ]}t j
dk rt d | t |d
5 }| | 5 | | d d d n # 1 swxY w Y d d d n # 1 swxY w Y d S ) Nr ztest*.pyztest_unicode_identifiers.py)bufferbuiltinfileior platformsysz
test_%s.pycpu
r. r r rs )globrandomr r r r curdirr escaperemover is_resource_enabledsampleverbosery r subTestr )r> rG rH tempdir testfilesrA testfiles r2 test_random_filesz TestRoundtrip.test_random_filesM s '//(++8ryIIbgll4;;w+?+? L LMM g/LMMNNNI F FA RW\\'< @ @1 D E E E E*511 5
i44I! , ,H!##j(+++h%%
,\\x\00 , ,((+++, , , , , , , , , , , , , , ,
,
,
,
,
,
,
,
,
,
,
,
,
,
,
, , ,s6 F"F8 FF F F FF "F c t |t r| d }t t t | j d S )Nr8 )r' r r9 r r r r: r )r> r+ s r2 roundtripzTestRoundtrip.roundtripi sR dC (;;w''D(74==#9::;;BB7KKKr4 c d}| | d }| |d |d | | dS )z
Ensure that although whitespace might be mutated in a roundtrip,
the semantic meaning of the indentation remains consistent.
zif False:
x=3
x=3
r r r. N)rU splitr; r )r> r+ codeliness r2 #test_indentation_semantics_retainedz1TestRoundtrip.test_indentation_semantics_retainedn s`
+NN4((..t44 1 y|444 T"""""r4 N) r r r r r8 r; r r> rS rU rY r r4 r2 r r s 1 1 1@%: %: %:N ' ' '& & && & &, , ,8L L L
# # # # #r4 r c t e Zd Zd Zd Zd Zd Zd Zd Zd Z d Z
d Z d
Z d Z
d Zd
Zd Zd Zd Zd ZdS )
CTokenizeTestc | | 5 t t | | }| || d d d d S # 1 swxY w Y d S )N)r )rO r3 r r; r< r& )r> r? r@ r* s r2 rB z CTokenizeTest.check_tokenizez s \\\
#
# E E11!44a F
VX__%6%6%A%A%C%C D D D E E E E E E E E E E E E E E E E E Es AA<<B B c | dd | dd | dd | dd | d d
| d d | d
d | dd | dd d S )NrP z NUMBER '0xff' (1, 0) (1, 4)
LESSEQUAL '<=' (1, 5) (1, 7)
NUMBER '255' (1, 8) (1, 11)
rQ z NUMBER '0b10' (1, 0) (1, 4)
LESSEQUAL '<=' (1, 5) (1, 7)
NUMBER '255' (1, 8) (1, 11)
rR z NUMBER '0o123' (1, 0) (1, 5)
LESSEQUAL '<=' (1, 6) (1, 8)
NUMBER '0O123' (1, 9) (1, 14)
rS z NUMBER '1234567' (1, 0) (1, 7)
GREATER '>' (1, 8) (1, 9)
TILDE '~' (1, 10) (1, 11)
NUMBER '0x15' (1, 11) (1, 15)
rT z NUMBER '2134568' (1, 0) (1, 7)
NOTEQUAL '!=' (1, 8) (1, 10)
NUMBER '1231515' (1, 11) (1, 18)
rU ac LPAR '(' (1, 0) (1, 1)
MINUS '-' (1, 1) (1, 2)
NUMBER '124561' (1, 2) (1, 8)
MINUS '-' (1, 8) (1, 9)
NUMBER '1' (1, 9) (1, 10)
RPAR ')' (1, 10) (1, 11)
AMPER '&' (1, 12) (1, 13)
NUMBER '200000000' (1, 14) (1, 23)
rV z NUMBER '0xdeadbeef' (1, 0) (1, 10)
NOTEQUAL '!=' (1, 11) (1, 13)
MINUS '-' (1, 14) (1, 15)
NUMBER '1' (1, 15) (1, 16)
rW z NUMBER '0xdeadc0de' (1, 0) (1, 10)
AMPER '&' (1, 11) (1, 12)
NUMBER '12345' (1, 13) (1, 18)
rX z NUMBER '0xFF' (1, 0) (1, 4)
AMPER '&' (1, 5) (1, 6)
NUMBER '0x15' (1, 7) (1, 11)
VBAR '|' (1, 12) (1, 13)
NUMBER '1234' (1, 14) (1, 18)
rY rZ s r2 r[ zCTokenizeTest.test_int s: M ,
M ,
, /
- 0
0 3
5 8
. 1
0 3
0 3 r4 c : | dd | dd | dd | dd | d d
| d d | d
d d S )Nr_ z NAME 'x' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
NUMBER '3.14159' (1, 4) (1, 11)
r` z NAME 'x' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
NUMBER '314159.' (1, 4) (1, 11)
ra z NAME 'x' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
NUMBER '.314159' (1, 4) (1, 11)
rb z NAME 'x' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
NUMBER '3e14159' (1, 4) (1, 11)
rc z NAME 'x' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
NUMBER '3E123' (1, 4) (1, 9)
rd z NAME 'x' (1, 0) (1, 1)
PLUS '+' (1, 1) (1, 2)
NAME 'y' (1, 2) (1, 3)
EQUAL '=' (1, 4) (1, 5)
NUMBER '3e-1230' (1, 6) (1, 13)
re z NAME 'x' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
NUMBER '3.14e159' (1, 4) (1, 12)
rY rZ s r2 rf zCTokenizeTest.test_float s M ,
M ,
M ,
M ,
K *
O .
N - r4 c ~ | dd | dd | dd | dd | d d
| d d | d
d | dd | dd | dd | dd | dd | dd | dd | d d | d d | d!d" | d#d$ | d%d& | d'd( | d)d* | d+d, | d-d. | d/d0 | d1d2 | d3d4 d S )5Nrp a5 NAME 'x' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
STRING "''" (1, 4) (1, 6)
SEMI ';' (1, 6) (1, 7)
NAME 'y' (1, 8) (1, 9)
EQUAL '=' (1, 10) (1, 11)
STRING '""' (1, 12) (1, 14)
rq a6 NAME 'x' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
STRING '\'"\'' (1, 4) (1, 7)
SEMI ';' (1, 7) (1, 8)
NAME 'y' (1, 9) (1, 10)
EQUAL '=' (1, 11) (1, 12)
STRING '"\'"' (1, 13) (1, 16)
rr z NAME 'x' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
STRING '"doesn\'t "' (1, 4) (1, 14)
NAME 'shrink' (1, 14) (1, 20)
STRING '", does it"' (1, 20) (1, 31)
rs z NAME 'x' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
STRING "'abc'" (1, 4) (1, 9)
PLUS '+' (1, 10) (1, 11)
STRING "'ABC'" (1, 12) (1, 17)
rt z NAME 'y' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
STRING '"ABC"' (1, 4) (1, 9)
PLUS '+' (1, 10) (1, 11)
STRING '"ABC"' (1, 12) (1, 17)
ru a NAME 'x' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
STRING "r'abc'" (1, 4) (1, 10)
PLUS '+' (1, 11) (1, 12)
STRING "r'ABC'" (1, 13) (1, 19)
PLUS '+' (1, 20) (1, 21)
STRING "R'ABC'" (1, 22) (1, 28)
PLUS '+' (1, 29) (1, 30)
STRING "R'ABC'" (1, 31) (1, 37)
rv a NAME 'y' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
STRING 'r"abc"' (1, 4) (1, 10)
PLUS '+' (1, 11) (1, 12)
STRING 'r"ABC"' (1, 13) (1, 19)
PLUS '+' (1, 20) (1, 21)
STRING 'R"ABC"' (1, 22) (1, 28)
PLUS '+' (1, 29) (1, 30)
STRING 'R"ABC"' (1, 31) (1, 37)
rw z STRING "u'abc'" (1, 0) (1, 6)
PLUS '+' (1, 7) (1, 8)
STRING "U'abc'" (1, 9) (1, 15)
rx z STRING 'u"abc"' (1, 0) (1, 6)
PLUS '+' (1, 7) (1, 8)
STRING 'U"abc"' (1, 9) (1, 15)
ry z STRING "b'abc'" (1, 0) (1, 6)
PLUS '+' (1, 7) (1, 8)
STRING "B'abc'" (1, 9) (1, 15)
rz z STRING 'b"abc"' (1, 0) (1, 6)
PLUS '+' (1, 7) (1, 8)
STRING 'B"abc"' (1, 9) (1, 15)
r{ a; STRING "br'abc'" (1, 0) (1, 7)
PLUS '+' (1, 8) (1, 9)
STRING "bR'abc'" (1, 10) (1, 17)
PLUS '+' (1, 18) (1, 19)
STRING "Br'abc'" (1, 20) (1, 27)
PLUS '+' (1, 28) (1, 29)
STRING "BR'abc'" (1, 30) (1, 37)
r| a; STRING 'br"abc"' (1, 0) (1, 7)
PLUS '+' (1, 8) (1, 9)
STRING 'bR"abc"' (1, 10) (1, 17)
PLUS '+' (1, 18) (1, 19)
STRING 'Br"abc"' (1, 20) (1, 27)
PLUS '+' (1, 28) (1, 29)
STRING 'BR"abc"' (1, 30) (1, 37)
r} a; STRING "rb'abc'" (1, 0) (1, 7)
PLUS '+' (1, 8) (1, 9)
STRING "rB'abc'" (1, 10) (1, 17)
PLUS '+' (1, 18) (1, 19)
STRING "Rb'abc'" (1, 20) (1, 27)
PLUS '+' (1, 28) (1, 29)
STRING "RB'abc'" (1, 30) (1, 37)
r~ a; STRING 'rb"abc"' (1, 0) (1, 7)
PLUS '+' (1, 8) (1, 9)
STRING 'rB"abc"' (1, 10) (1, 17)
PLUS '+' (1, 18) (1, 19)
STRING 'Rb"abc"' (1, 20) (1, 27)
PLUS '+' (1, 28) (1, 29)
STRING 'RB"abc"' (1, 30) (1, 37)
r r r r r r r r r r r r r r r r r r r r r r rY rZ s r2 r zCTokenizeTest.test_string s . 1
1 4
> A
/ 2
/ 2
C
F
C
F
- 0
- 0
- 0
- 0
C F
C F
C F
C F
- 0
L +
L +
H '
K *
L +
r4 c | dd | dd | dd d S )Nr a NAME 'def' (1, 0) (1, 3)
NAME 'd22' (1, 4) (1, 7)
LPAR '(' (1, 7) (1, 8)
NAME 'a' (1, 8) (1, 9)
COMMA ',' (1, 9) (1, 10)
NAME 'b' (1, 11) (1, 12)
COMMA ',' (1, 12) (1, 13)
NAME 'c' (1, 14) (1, 15)
EQUAL '=' (1, 15) (1, 16)
NUMBER '2' (1, 16) (1, 17)
COMMA ',' (1, 17) (1, 18)
NAME 'd' (1, 19) (1, 20)
EQUAL '=' (1, 20) (1, 21)
NUMBER '2' (1, 21) (1, 22)
COMMA ',' (1, 22) (1, 23)
STAR '*' (1, 24) (1, 25)
NAME 'k' (1, 25) (1, 26)
RPAR ')' (1, 26) (1, 27)
COLON ':' (1, 27) (1, 28)
NAME 'pass' (1, 29) (1, 33)
r a NAME 'def' (1, 0) (1, 3)
NAME 'd01v_' (1, 4) (1, 9)
LPAR '(' (1, 9) (1, 10)
NAME 'a' (1, 10) (1, 11)
EQUAL '=' (1, 11) (1, 12)
NUMBER '1' (1, 12) (1, 13)
COMMA ',' (1, 13) (1, 14)
STAR '*' (1, 15) (1, 16)
NAME 'k' (1, 16) (1, 17)
COMMA ',' (1, 17) (1, 18)
DOUBLESTAR '**' (1, 19) (1, 21)
NAME 'w' (1, 21) (1, 22)
RPAR ')' (1, 22) (1, 23)
COLON ':' (1, 23) (1, 24)
NAME 'pass' (1, 25) (1, 29)
r a NAME 'def' (1, 0) (1, 3)
NAME 'd23' (1, 4) (1, 7)
LPAR '(' (1, 7) (1, 8)
NAME 'a' (1, 8) (1, 9)
COLON ':' (1, 9) (1, 10)
NAME 'str' (1, 11) (1, 14)
COMMA ',' (1, 14) (1, 15)
NAME 'b' (1, 16) (1, 17)
COLON ':' (1, 17) (1, 18)
NAME 'int' (1, 19) (1, 22)
EQUAL '=' (1, 22) (1, 23)
NUMBER '3' (1, 23) (1, 24)
RPAR ')' (1, 24) (1, 25)
RARROW '->' (1, 26) (1, 28)
NAME 'int' (1, 29) (1, 32)
COLON ':' (1, 32) (1, 33)
NAME 'pass' (1, 34) (1, 38)
rY rZ s r2 r zCTokenizeTest.test_function sr ? B .
; > $
D G r4 c 2 | dd d S )Nr a NAME 'if' (1, 0) (1, 2)
NUMBER '1' (1, 3) (1, 4)
LESS '<' (1, 5) (1, 6)
NUMBER '1' (1, 7) (1, 8)
GREATER '>' (1, 9) (1, 10)
NUMBER '1' (1, 11) (1, 12)
EQEQUAL '==' (1, 13) (1, 15)
NUMBER '1' (1, 16) (1, 17)
GREATEREQUAL '>=' (1, 18) (1, 20)
NUMBER '5' (1, 21) (1, 22)
LESSEQUAL '<=' (1, 23) (1, 25)
NUMBER '0x15' (1, 26) (1, 30)
LESSEQUAL '<=' (1, 31) (1, 33)
NUMBER '0x12' (1, 34) (1, 38)
NOTEQUAL '!=' (1, 39) (1, 41)
NUMBER '1' (1, 42) (1, 43)
NAME 'and' (1, 44) (1, 47)
NUMBER '5' (1, 48) (1, 49)
NAME 'in' (1, 50) (1, 52)
NUMBER '1' (1, 53) (1, 54)
NAME 'not' (1, 55) (1, 58)
NAME 'in' (1, 59) (1, 61)
NUMBER '1' (1, 62) (1, 63)
NAME 'is' (1, 64) (1, 66)
NUMBER '1' (1, 67) (1, 68)
NAME 'or' (1, 69) (1, 71)
NUMBER '5' (1, 72) (1, 73)
NAME 'is' (1, 74) (1, 76)
NAME 'not' (1, 77) (1, 80)
NUMBER '1' (1, 81) (1, 82)
COLON ':' (1, 82) (1, 83)
NAME 'pass' (1, 84) (1, 88)
rY rZ s r2 r z CTokenizeTest.test_comparison r r4 c 2 | dd d S )Nr a$ NAME 'x' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
NUMBER '1' (1, 4) (1, 5)
MINUS '-' (1, 6) (1, 7)
NAME 'y' (1, 8) (1, 9)
PLUS '+' (1, 10) (1, 11)
NUMBER '15' (1, 12) (1, 14)
MINUS '-' (1, 15) (1, 16)
NUMBER '1' (1, 17) (1, 18)
PLUS '+' (1, 19) (1, 20)
NUMBER '0x124' (1, 21) (1, 26)
PLUS '+' (1, 27) (1, 28)
NAME 'z' (1, 29) (1, 30)
PLUS '+' (1, 31) (1, 32)
NAME 'a' (1, 33) (1, 34)
LSQB '[' (1, 34) (1, 35)
NUMBER '5' (1, 35) (1, 36)
RSQB ']' (1, 36) (1, 37)
rY rZ s r2 r zCTokenizeTest.test_additive
r r4 c 2 | dd d S )Nr a NAME 'x' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
NUMBER '1' (1, 4) (1, 5)
DOUBLESLASH '//' (1, 5) (1, 7)
NUMBER '1' (1, 7) (1, 8)
STAR '*' (1, 8) (1, 9)
NUMBER '1' (1, 9) (1, 10)
SLASH '/' (1, 10) (1, 11)
NUMBER '5' (1, 11) (1, 12)
STAR '*' (1, 12) (1, 13)
NUMBER '12' (1, 13) (1, 15)
PERCENT '%' (1, 15) (1, 16)
NUMBER '0x12' (1, 16) (1, 20)
AT '@' (1, 20) (1, 21)
NUMBER '42' (1, 21) (1, 23)
rY rZ s r2 r z!CTokenizeTest.test_multiplicative$ r r4 c ^ | dd | dd d S )Nr a TILDE '~' (1, 0) (1, 1)
NUMBER '1' (1, 1) (1, 2)
CIRCUMFLEX '^' (1, 3) (1, 4)
NUMBER '1' (1, 5) (1, 6)
AMPER '&' (1, 7) (1, 8)
NUMBER '1' (1, 9) (1, 10)
VBAR '|' (1, 11) (1, 12)
NUMBER '1' (1, 12) (1, 13)
CIRCUMFLEX '^' (1, 14) (1, 15)
MINUS '-' (1, 16) (1, 17)
NUMBER '1' (1, 17) (1, 18)
r aI MINUS '-' (1, 0) (1, 1)
NUMBER '1' (1, 1) (1, 2)
STAR '*' (1, 2) (1, 3)
NUMBER '1' (1, 3) (1, 4)
SLASH '/' (1, 4) (1, 5)
NUMBER '1' (1, 5) (1, 6)
PLUS '+' (1, 6) (1, 7)
NUMBER '1' (1, 7) (1, 8)
STAR '*' (1, 8) (1, 9)
NUMBER '1' (1, 9) (1, 10)
DOUBLESLASH '//' (1, 10) (1, 12)
NUMBER '1' (1, 12) (1, 13)
MINUS '-' (1, 14) (1, 15)
MINUS '-' (1, 16) (1, 17)
MINUS '-' (1, 17) (1, 18)
MINUS '-' (1, 18) (1, 19)
NUMBER '1' (1, 19) (1, 20)
DOUBLESTAR '**' (1, 20) (1, 22)
NUMBER '1' (1, 22) (1, 23)
rY rZ s r2 r zCTokenizeTest.test_unary8 sN 0 3
5 8 r4 c 2 | dd d S )Nr a NAME 'import' (1, 0) (1, 6)
NAME 'sys' (1, 7) (1, 10)
COMMA ',' (1, 10) (1, 11)
NAME 'time' (1, 12) (1, 16)
NEWLINE '' (1, 16) (1, 16)
NAME 'x' (2, 0) (2, 1)
EQUAL '=' (2, 2) (2, 3)
NAME 'sys' (2, 4) (2, 7)
DOT '.' (2, 7) (2, 8)
NAME 'modules' (2, 8) (2, 15)
LSQB '[' (2, 15) (2, 16)
STRING "'time'" (2, 16) (2, 22)
RSQB ']' (2, 22) (2, 23)
DOT '.' (2, 23) (2, 24)
NAME 'time' (2, 24) (2, 28)
LPAR '(' (2, 28) (2, 29)
RPAR ')' (2, 29) (2, 30)
rY rZ s r2 r zCTokenizeTest.test_selector^ r r4 c 2 | dd d S Nr a AT '@' (1, 0) (1, 1)
NAME 'staticmethod' (1, 1) (1, 13)
NEWLINE '' (1, 13) (1, 13)
NAME 'def' (2, 0) (2, 3)
NAME 'foo' (2, 4) (2, 7)
LPAR '(' (2, 7) (2, 8)
NAME 'x' (2, 8) (2, 9)
COMMA ',' (2, 9) (2, 10)
NAME 'y' (2, 10) (2, 11)
RPAR ')' (2, 11) (2, 12)
COLON ':' (2, 12) (2, 13)
NAME 'pass' (2, 14) (2, 18)
rY rZ s r2 r zCTokenizeTest.test_methodt r r4 c 2 | dd d S rg rY rZ s r2 r zCTokenizeTest.test_tabs r r4 c | dd | dd | dd | dd | d d
| d d | d
d | dd | dd | dd | dd | dd | dd | dd | d d | d d | d!d" | d#d$ | d%d& | d'd( | d)d* d S )+Nr z ASYNC 'async' (1, 0) (1, 5)
EQUAL '=' (1, 6) (1, 7)
NUMBER '1' (1, 8) (1, 9)
r a8 NAME 'a' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
LPAR '(' (1, 4) (1, 5)
ASYNC 'async' (1, 5) (1, 10)
EQUAL '=' (1, 11) (1, 12)
NUMBER '1' (1, 13) (1, 14)
RPAR ')' (1, 14) (1, 15)
r z ASYNC 'async' (1, 0) (1, 5)
LPAR '(' (1, 5) (1, 6)
RPAR ')' (1, 6) (1, 7)
r a< NAME 'class' (1, 0) (1, 5)
ASYNC 'async' (1, 6) (1, 11)
LPAR '(' (1, 11) (1, 12)
NAME 'Bar' (1, 12) (1, 15)
RPAR ')' (1, 15) (1, 16)
COLON ':' (1, 16) (1, 17)
NAME 'pass' (1, 17) (1, 21)
r z NAME 'class' (1, 0) (1, 5)
ASYNC 'async' (1, 6) (1, 11)
COLON ':' (1, 11) (1, 12)
NAME 'pass' (1, 12) (1, 16)
r z AWAIT 'await' (1, 0) (1, 5)
EQUAL '=' (1, 6) (1, 7)
NUMBER '1' (1, 8) (1, 9)
r z NAME 'foo' (1, 0) (1, 3)
DOT '.' (1, 3) (1, 4)
ASYNC 'async' (1, 4) (1, 9)
r a; ASYNC 'async' (1, 0) (1, 5)
NAME 'for' (1, 6) (1, 9)
NAME 'a' (1, 10) (1, 11)
NAME 'in' (1, 12) (1, 14)
NAME 'b' (1, 15) (1, 16)
COLON ':' (1, 16) (1, 17)
NAME 'pass' (1, 18) (1, 22)
r a< ASYNC 'async' (1, 0) (1, 5)
NAME 'with' (1, 6) (1, 10)
NAME 'a' (1, 11) (1, 12)
NAME 'as' (1, 13) (1, 15)
NAME 'b' (1, 16) (1, 17)
COLON ':' (1, 17) (1, 18)
NAME 'pass' (1, 19) (1, 23)
r z ASYNC 'async' (1, 0) (1, 5)
DOT '.' (1, 5) (1, 6)
NAME 'foo' (1, 6) (1, 9)
r z/ ASYNC 'async' (1, 0) (1, 5)
r z ASYNC 'async' (1, 0) (1, 5)
NEWLINE '' (1, 5) (1, 5)
AWAIT 'await' (3, 0) (3, 5)
r z ASYNC 'async' (1, 0) (1, 5)
NEWLINE '' (1, 5) (1, 5)
ELLIPSIS '...' (2, 0) (2, 3)
NEWLINE '' (2, 3) (2, 3)
AWAIT 'await' (3, 0) (3, 5)
r z ASYNC 'async' (1, 0) (1, 5)
NEWLINE '' (1, 5) (1, 5)
AWAIT 'await' (2, 0) (2, 5)
r z NAME 'foo' (1, 0) (1, 3)
DOT '.' (1, 3) (1, 4)
ASYNC 'async' (1, 4) (1, 9)
PLUS '+' (1, 10) (1, 11)
NUMBER '1' (1, 12) (1, 13)
r a; ASYNC 'async' (1, 0) (1, 5)
NAME 'def' (1, 6) (1, 9)
NAME 'foo' (1, 10) (1, 13)
LPAR '(' (1, 13) (1, 14)
RPAR ')' (1, 14) (1, 15)
COLON ':' (1, 15) (1, 16)
NAME 'pass' (1, 17) (1, 21)
r a ASYNC 'async' (1, 0) (1, 5)
NAME 'def' (1, 6) (1, 9)
NAME 'foo' (1, 10) (1, 13)
LPAR '(' (1, 13) (1, 14)
RPAR ')' (1, 14) (1, 15)
COLON ':' (1, 15) (1, 16)
NEWLINE '' (1, 16) (1, 16)
INDENT '' (2, -1) (2, -1)
NAME 'def' (2, 2) (2, 5)
NAME 'foo' (2, 6) (2, 9)
LPAR '(' (2, 9) (2, 10)
AWAIT 'await' (2, 10) (2, 15)
RPAR ')' (2, 15) (2, 16)
COLON ':' (2, 16) (2, 17)
NEWLINE '' (2, 17) (2, 17)
INDENT '' (3, -1) (3, -1)
AWAIT 'await' (3, 4) (3, 9)
EQUAL '=' (3, 10) (3, 11)
NUMBER '1' (3, 12) (3, 13)
NEWLINE '' (3, 13) (3, 13)
DEDENT '' (4, -1) (4, -1)
NAME 'if' (4, 2) (4, 4)
NUMBER '1' (4, 5) (4, 6)
COLON ':' (4, 6) (4, 7)
NEWLINE '' (4, 7) (4, 7)
INDENT '' (5, -1) (5, -1)
AWAIT 'await' (5, 4) (5, 9)
NEWLINE '' (5, 9) (5, 9)
DEDENT '' (6, -1) (6, -1)
DEDENT '' (6, -1) (6, -1)
ASYNC 'async' (6, 0) (6, 5)
PLUSEQUAL '+=' (6, 6) (6, 8)
NUMBER '1' (6, 9) (6, 10)
NEWLINE '' (6, 10) (6, 10)
r a ASYNC 'async' (1, 0) (1, 5)
NAME 'def' (1, 6) (1, 9)
NAME 'foo' (1, 10) (1, 13)
LPAR '(' (1, 13) (1, 14)
RPAR ')' (1, 14) (1, 15)
COLON ':' (1, 15) (1, 16)
NEWLINE '' (1, 16) (1, 16)
INDENT '' (2, -1) (2, -1)
ASYNC 'async' (2, 2) (2, 7)
NAME 'for' (2, 8) (2, 11)
NAME 'i' (2, 12) (2, 13)
NAME 'in' (2, 14) (2, 16)
NUMBER '1' (2, 17) (2, 18)
COLON ':' (2, 18) (2, 19)
NAME 'pass' (2, 20) (2, 24)
DEDENT '' (2, -1) (2, -1)
r ah ASYNC 'async' (1, 0) (1, 5)
NAME 'def' (1, 6) (1, 9)
NAME 'foo' (1, 10) (1, 13)
LPAR '(' (1, 13) (1, 14)
ASYNC 'async' (1, 14) (1, 19)
RPAR ')' (1, 19) (1, 20)
COLON ':' (1, 20) (1, 21)
AWAIT 'await' (1, 22) (1, 27)
r a~ NAME 'def' (1, 0) (1, 3)
NAME 'f' (1, 4) (1, 5)
LPAR '(' (1, 5) (1, 6)
RPAR ')' (1, 6) (1, 7)
COLON ':' (1, 7) (1, 8)
NEWLINE '' (1, 8) (1, 8)
INDENT '' (3, -1) (3, -1)
NAME 'def' (3, 2) (3, 5)
NAME 'baz' (3, 6) (3, 9)
LPAR '(' (3, 9) (3, 10)
RPAR ')' (3, 10) (3, 11)
COLON ':' (3, 11) (3, 12)
NAME 'pass' (3, 13) (3, 17)
NEWLINE '' (3, 17) (3, 17)
ASYNC 'async' (4, 2) (4, 7)
NAME 'def' (4, 8) (4, 11)
NAME 'bar' (4, 12) (4, 15)
LPAR '(' (4, 15) (4, 16)
RPAR ')' (4, 16) (4, 17)
COLON ':' (4, 17) (4, 18)
NAME 'pass' (4, 19) (4, 23)
NEWLINE '' (4, 23) (4, 23)
AWAIT 'await' (6, 2) (6, 7)
EQUAL '=' (6, 8) (6, 9)
NUMBER '2' (6, 10) (6, 11)
DEDENT '' (6, -1) (6, -1)
r a ASYNC 'async' (1, 0) (1, 5)
NAME 'def' (1, 6) (1, 9)
NAME 'f' (1, 10) (1, 11)
LPAR '(' (1, 11) (1, 12)
RPAR ')' (1, 12) (1, 13)
COLON ':' (1, 13) (1, 14)
NEWLINE '' (1, 14) (1, 14)
INDENT '' (3, -1) (3, -1)
NAME 'def' (3, 2) (3, 5)
NAME 'baz' (3, 6) (3, 9)
LPAR '(' (3, 9) (3, 10)
RPAR ')' (3, 10) (3, 11)
COLON ':' (3, 11) (3, 12)
NAME 'pass' (3, 13) (3, 17)
NEWLINE '' (3, 17) (3, 17)
ASYNC 'async' (4, 2) (4, 7)
NAME 'def' (4, 8) (4, 11)
NAME 'bar' (4, 12) (4, 15)
LPAR '(' (4, 15) (4, 16)
RPAR ')' (4, 16) (4, 17)
COLON ':' (4, 17) (4, 18)
NAME 'pass' (4, 19) (4, 23)
NEWLINE '' (4, 23) (4, 23)
AWAIT 'await' (6, 2) (6, 7)
EQUAL '=' (6, 8) (6, 9)
NUMBER '2' (6, 10) (6, 11)
DEDENT '' (6, -1) (6, -1)
rY rZ s r2 r zCTokenizeTest.test_async s K *
- 0
I (
3 6
. 1
K *
K *
4 7
5 8
K *
G &
4 7
/ 2
N -
O .
3 6
#* * * X
H K &
9 <
! ! ! F
" " " " " r4 c 2 | dd d S )Nr u7 NAME 'Örter' (1, 0) (1, 6)
EQUAL '=' (1, 7) (1, 8)
STRING "u'places'" (1, 9) (1, 18)
NEWLINE '' (1, 18) (1, 18)
NAME 'grün' (2, 0) (2, 5)
EQUAL '=' (2, 6) (2, 7)
STRING "U'green'" (2, 8) (2, 16)
rY rZ s r2 r zCTokenizeTest.test_unicode r r4 c d }| t |d | t |d | t |d | t |d | t |d | t |d | t |d | t |d | t |d
| t |d | t |d | t |d
| t |d | t |d | t |d | t |d | t |d | t |d | t |d | t |d | t |d | t |d | t |d | t |d | t |d | t |d | t |d | t |d | t |d | t |d d S ) Nc : t t | S r )rF r r s r2
get_tokensz5CTokenizeTest.test_invalid_syntax.<locals>.get_tokens s 9&AABB Br4 z(1+2]z(1+2}z{1+2]1_z1.2_1e2_z1e+ u €0b120b1_20b20b1_0b0o180o1_80o80o1_0o0x1_0x012z'sdfsdfz '''sdfsdf''a ((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((a))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))])r r )r> rn s r2 test_invalid_syntaxz!CTokenizeTest.test_invalid_syntax s C C C
+z7;;; +z7;;; +z7;;; +z4888 +z6::: +z6::: +z5999 +z6::: +z5999 +z6::: +z7;;; +z5999 +z6::: +z4888 +z6::: +z7;;; +z5999 +z6::: +z4888 +z6::: +z4888 +z4888 +z5999 +z6::: +z6::: +z5999 +z9=== +z=AAA +z3HIII +z377777r4 c d}d } ||dz
}t t | }| |d j t t |dd || }t t | }| |d j t | t t
|dd d S )Nd c t d d t | D }|d| z dz z
}|S )N c 3 & K | ] }d |z dz V
dS ) z if True:
Nr )r" rD s r2 <genexpr>zICTokenizeTest.test_max_indent.<locals>.generate_source.<locals>.<genexpr> s+ O O1dQh, 6 O O O O O Or4 r zpass
)r r )indentsr s r2 generate_sourcez6CTokenizeTest.test_max_indent.<locals>.generate_source s@ WW O Og O O OOOF dWnx/ /FMr4 r r z<string>exec) rF r r; r- r compiler r rK )r> MAXINDENTr validrG invalids r2 test_max_indentz CTokenizeTest.test_max_indent s
A
..7>>?? &111z6***!/),,7@@AA '222 gw
F
r4 c 4 d }t d }| |d t d }| || || t d }| |d t d }| || || t d }| |d t d
}| || || d S ) Nc 4 d t | D S )Nc g | ] ^}}}||f
S r r )r" kindr r s r2 r# zYCTokenizeTest.test_continuation_lines_indentation.<locals>.get_tokens.<locals>.<listcomp> s# ggg'9fqT6Ngggr4 )r rm s r2 rn zECTokenizeTest.test_continuation_lines_indentation.<locals>.get_tokens s gg=^_e=f=fggg gr4 z
def fib(n):
\
'''Print a Fibonacci series up to n.'''
\
a, b = 0, 1
ae NAME 'def' (2, 0) (2, 3)
NAME 'fib' (2, 4) (2, 7)
LPAR '(' (2, 7) (2, 8)
NAME 'n' (2, 8) (2, 9)
RPAR ')' (2, 9) (2, 10)
COLON ':' (2, 10) (2, 11)
NEWLINE '' (2, 11) (2, 11)
INDENT '' (4, -1) (4, -1)
STRING "'''Print a Fibonacci series up to n.'''" (4, 0) (4, 39)
NEWLINE '' (4, 39) (4, 39)
NAME 'a' (6, 0) (6, 1)
COMMA ',' (6, 1) (6, 2)
NAME 'b' (6, 3) (6, 4)
EQUAL '=' (6, 5) (6, 6)
NUMBER '0' (6, 7) (6, 8)
COMMA ',' (6, 8) (6, 9)
NUMBER '1' (6, 10) (6, 11)
NEWLINE '' (6, 11) (6, 11)
DEDENT '' (6, -1) (6, -1)
zu
def fib(n):
'''Print a Fibonacci series up to n.'''
a, b = 0, 1
z>
pass
\
pass
z NAME 'pass' (2, 0) (2, 4)
NEWLINE '' (2, 4) (2, 4)
NAME 'pass' (5, 0) (5, 4)
NEWLINE '' (5, 4) (5, 4)
z+
pass
pass
z
if x:
y = 1
\
\
\
\
foo = 1
aj NAME 'if' (2, 0) (2, 2)
NAME 'x' (2, 3) (2, 4)
COLON ':' (2, 4) (2, 5)
NEWLINE '' (2, 5) (2, 5)
INDENT '' (3, -1) (3, -1)
NAME 'y' (3, 4) (3, 5)
EQUAL '=' (3, 6) (3, 7)
NUMBER '1' (3, 8) (3, 9)
NEWLINE '' (3, 9) (3, 9)
NAME 'foo' (8, 4) (8, 7)
EQUAL '=' (8, 8) (8, 9)
NUMBER '1' (8, 10) (8, 11)
NEWLINE '' (8, 11) (8, 11)
DEDENT '' (8, -1) (8, -1)
zI
if x:
y = 1
foo = 1
)r rB r; )r> rn r+ code_no_conts r2 #test_continuation_lines_indentationz1CTokenizeTest.test_continuation_lines_indentation s h h h
D #
,
D))::l+C+CDDD
D #
D))::l+C+CDDD
D #
"
D))::l+C+CDDDDDr4 N)r r r rB r[ rf r r r r r r r r r r r r r r r r4 r2 r[ r[ y s. E E E? ? ? B, , , \t t t l= = = ~$ $ $ L . ($ $ $ L , " "H H H T
&8 &8 &8P
(`E `E `E `E `Er4 r[ c e Zd Zd ZdS )CTokenizerBufferTestsc dddz dddz }t j 5 }t |d| }t | d d d d S # 1 swxY w Y d S )Nz" #coding: latin-1
#ai' z
#i' foo)r temp_dirr r )r> test_scriptr file_names r2 !test_newline_at_the_end_of_bufferz7CTokenizerBufferTests.test_newline_at_the_end_of_bufferL
s
e) e)
!
! 'X#He[AAI I & & & ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' 's !AAAN)r r r r r r4 r2 r r K
s#
'
'
'
'
'r4 r __main__)5testr test.supportr r r r r r r
r r r
r r r rz r r r r r ior r unittesttextwrapr r r test.test_grammarr r test.support.script_helperr r r r. r3 r6 r r r r r r" r r r r[ r r mainr r4 r2 <module>r s3 " " " " " "I I I I I I I I I I I I I I I I I I I I I I I I I I I I I I I I I I I I I I ! # # # # # # # #< < < < < < < < " " " " " " C C C C C C C C M M M M M 8 M M M ^ A A A A A A A A
.
.
. G G G G Gx G G G."I "I "I "I "IH "I "I "IJ#G #G #G #G #GH #G #G #GL{" {" {" {" {" {" {" {"|RU RU RU RU RU8 RU RU RUh&> &> &> &> &>X &> &> &>RL# L# L# L# L#H L# L# L#^OE OE OE OE OEH OE OE OEd ' ' ' ' 'H - ' ' ' zHMOOOOO r4 
Browse
Usage
Versions