python (3.12.0)
Αes 6 d dl mZ d dlmZ d dlmZmZmZmZmZm Z m
Z
m Z m Z m
Z
mZmZmZmZmZmZmZmZ d dlmZmZ d dlZd dlm Z d dlm Z m Z d dl m Z m!Z! d dlmZ d d l"m#Z#m$Z$m%Z% d dl&Z&d dl'Z'd
Z( G d d e Z) G d
de) Z*d Z+ G d de Z, G d de Z- G d de Z. G d de Z/ G d de Z0 G d de Z1 G d d e Z2 G d d e Z3 G d d!e Z4 G d" d#ej: Z5e6d$k( r ejn yy)% )support) os_helper)tokenize
untokenizeNUMBERNAMEOPSTRING ENDMARKERENCODINGtok_namedetect_encodingopen Untokenizergenerate_tokensNEWLINE!_generate_tokens_from_c_tokenizerDEDENT TokenInfo
TokenError)BytesIOStringION)dedent)TestCasemock)VALID_UNDERSCORE_LITERALSINVALID_UNDERSCORE_LITERALS)run_test_script make_scriptrun_python_until_endc g }t |j }|d dv}| D ]P \ }}}}} |t k( r |S |r|t k( r |d |k( r)t | }|j d|dd|dd| d| R |S )Nz
r 10 13)len
splitlinesr r r
append)
token_generator
source_stringresult num_linesmissing_trailing_nltypetokenstartendlines
F/BuggyBox/python/3.12.0/bootstrap/lib/python3.12/test/test_tokenize.py stringify_tokens_from_sourcer5 s
FM,,./I'+69)8 C%eUC 9 M 47?s1v7J ~
T"IQurl!E7!C5ABC M c e Zd Zd Zd Zd Zd Zd Zd Zd Z d Z
d Z d
Z d Z
d Zd
Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zy) TokenizeTestc t |j d }t t |j | }| j |dg|j
j z y )Nutf-8z* ENCODING 'utf-8' (0, 0) (0, 0))r encoder5 r readline assertEqualrstripr( selfsexpectedfr, s r4 check_tokenizezTokenizeTest.check_tokenize. sZ
AHHW% &-hqzz.BAF FG! *5578 9r6 c d }| j t 5 t t | j d d d d }| j t 5 t t | j d d d d }| j t 5 t t | j d d d y # 1 sw Y xY w# 1 sw Y ZxY w# 1 sw Y y xY w)Nc 3 K d d y w)Nsdfosdg rH r6 r4 genz/TokenizeTest.test_invalid_readline.<locals>.gen8 s OO
c 3 K d d y w)Ns sdfosdgrH rH r6 r4 rI z/TokenizeTest.test_invalid_readline.<locals>.gen> s rJ c 3 K d ddz y w)NrG r rH rH r6 r4 rI z/TokenizeTest.test_invalid_readline.<locals>.genD s O
aCs
) assertRaises TypeErrorlistr __next__r ZeroDivisionError)r@ rI s r4 test_invalid_readlinez"TokenizeTest.test_invalid_readline7 s
y
) + #%..) * +
y
) 2 0 1 2
0
1 2 0 1 2 2 + + 2 2 2 2s# $C $C!$C&CC#&C/c t dj d }t t |j }| j |d j t | j |d j t y )Nxr: r" ) r r; rP r r< r= r/ r r )r@ rC tokenss r4 test_implicit_newlinez"TokenizeTest.test_implicit_newlineJ sZ
CJJw' (hqzz*+ '2 )4r6 c P | j dd | j dd | j dd | j dd d }t | j }| j t d
5 }t | D ] } d d d | j
j j d | j
|j j d | j
|j j d | j
|j j d | j
|j j d
| j
|j j d
| j
|j j d y # 1 sw Y xY w)Nz1 + 1z NUMBER '1' (1, 0) (1, 1)
OP '+' (1, 2) (1, 3)
NUMBER '1' (1, 4) (1, 5)
z3if False:
# NL
True = False # NEWLINE
af NAME 'if' (1, 0) (1, 2)
NAME 'False' (1, 3) (1, 8)
OP ':' (1, 8) (1, 9)
NEWLINE '\n' (1, 9) (1, 10)
COMMENT '# NL' (2, 4) (2, 8)
NL '\n' (2, 8) (2, 9)
NL '\n' (3, 4) (3, 5)
INDENT ' ' (4, 0) (4, 4)
NAME 'True' (4, 4) (4, 8)
OP '=' (4, 9) (4, 10)
NAME 'False' (4, 11) (4, 16)
COMMENT '# NEWLINE' (4, 17) (4, 26)
NEWLINE '\n' (4, 26) (4, 27)
DEDENT '' (5, 0) (5, 0)
z%if True:
# NL
foo='bar'
a@ NAME 'if' (1, 0) (1, 2)
NAME 'True' (1, 3) (1, 7)
OP ':' (1, 7) (1, 8)
NEWLINE '\r\n' (1, 8) (1, 10)
COMMENT '# NL' (2, 4) (2, 8)
NL '\r\n' (2, 8) (2, 10)
INDENT ' ' (3, 0) (3, 4)
NAME 'foo' (3, 4) (3, 7)
OP '=' (3, 7) (3, 8)
STRING "'bar'" (3, 8) (3, 13)
NEWLINE '\r\n' (3, 13) (3, 15)
NL '\r\n' (4, 0) (4, 2)
DEDENT '' (5, 0) (5, 0)
zx = 1 + \
1
a NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '1' (1, 4) (1, 5)
OP '+' (1, 6) (1, 7)
NUMBER '1' (2, 0) (2, 1)
NEWLINE '\r\n' (2, 1) (2, 3)
s def k(x):
x += 2
x += 5
z3unindent does not match any outer indentation level <string> z x += 5)rD r r< assertRaisesRegexIndentationErrorr r= exceptionlinenofilename
end_lineno
end_offsetmsgoffsettext)r@ indent_error_filer< etoks r4
test_basiczTokenizeTest.test_basicR s G &
;= (
K N
1 4
,-66
#
#$4%>? BC )
++Q/ --z: //6 //6
KKOO A C
++Q/ )):6 s 6FF%c H | j dd | j dd | j dd | j dd | j d d
| j d d | j d
d | j dd | j dd y )N 0xff <= 255z NUMBER '0xff' (1, 0) (1, 4)
OP '<=' (1, 5) (1, 7)
NUMBER '255' (1, 8) (1, 11)
0b10 <= 255z NUMBER '0b10' (1, 0) (1, 4)
OP '<=' (1, 5) (1, 7)
NUMBER '255' (1, 8) (1, 11)
0o123 <= 0O123z NUMBER '0o123' (1, 0) (1, 5)
OP '<=' (1, 6) (1, 8)
NUMBER '0O123' (1, 9) (1, 14)
1234567 > ~0x15z NUMBER '1234567' (1, 0) (1, 7)
OP '>' (1, 8) (1, 9)
OP '~' (1, 10) (1, 11)
NUMBER '0x15' (1, 11) (1, 15)
2134568 != 1231515z NUMBER '2134568' (1, 0) (1, 7)
OP '!=' (1, 8) (1, 10)
NUMBER '1231515' (1, 11) (1, 18)
(-124561-1) & 200000000ac OP '(' (1, 0) (1, 1)
OP '-' (1, 1) (1, 2)
NUMBER '124561' (1, 2) (1, 8)
OP '-' (1, 8) (1, 9)
NUMBER '1' (1, 9) (1, 10)
OP ')' (1, 10) (1, 11)
OP '&' (1, 12) (1, 13)
NUMBER '200000000' (1, 14) (1, 23)
0xdeadbeef != -1z NUMBER '0xdeadbeef' (1, 0) (1, 10)
OP '!=' (1, 11) (1, 13)
OP '-' (1, 14) (1, 15)
NUMBER '1' (1, 15) (1, 16)
0xdeadc0de & 12345z NUMBER '0xdeadc0de' (1, 0) (1, 10)
OP '&' (1, 11) (1, 12)
NUMBER '12345' (1, 13) (1, 18)
0xFF & 0x15 | 1234z NUMBER '0xFF' (1, 0) (1, 4)
OP '&' (1, 5) (1, 6)
NUMBER '0x15' (1, 7) (1, 11)
OP '|' (1, 12) (1, 13)
NUMBER '1234' (1, 14) (1, 18)
rD r@ s r4 test_intzTokenizeTest.test_int s M ,
M ,
, /
- 0
0 3
5 8
. 1
0 3
0 3 r6 c | j dd | j dd | j dd | j dd y ) Nzx = 0z NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '0' (1, 4) (1, 5)
zx = 0xfffffffffffz NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '0xfffffffffff' (1, 4) (1, 17)
zx = 123141242151251616110z NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '123141242151251616110' (1, 4) (1, 25)
zx = -15921590215012591z NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
OP '-' (1, 4) (1, 5)
NUMBER '15921590215012591' (1, 5) (1, 22)
ru rv s r4 test_longzTokenizeTest.test_long s_ G &
/ 2
7 :
4 7 r6 c | j dd | j dd | j dd | j dd | j d d
| j d d | j d
d y )N x = 3.14159z NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '3.14159' (1, 4) (1, 11)
x = 314159.z NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '314159.' (1, 4) (1, 11)
x = .314159z NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '.314159' (1, 4) (1, 11)
x = 3e14159z NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '3e14159' (1, 4) (1, 11)
x = 3E123z NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '3E123' (1, 4) (1, 9)
x+y = 3e-1230z NAME 'x' (1, 0) (1, 1)
OP '+' (1, 1) (1, 2)
NAME 'y' (1, 2) (1, 3)
OP '=' (1, 4) (1, 5)
NUMBER '3e-1230' (1, 6) (1, 13)
x = 3.14e159z NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '3.14e159' (1, 4) (1, 12)
ru rv s r4
test_floatzTokenizeTest.test_float s M ,
M ,
M ,
M ,
K *
O .
N - r6 c d }t D ] }d|v r| j || | ! ddh}t D ]( }||v r || | j || | * y # t $ r Y 7w xY w)Nc t | j d }t |j D ] \ }}}}}|t k( s|c S y)Nr: z
invalid token)r r; r r< r )rA rC toktyper0 r1 r2 r3 s r4 number_tokenz;TokenizeTest.test_underscore_literals.<locals>.number_token sJ )*A4<QZZ4H
!0sDf$ L
! #r6 (0_709_99)r r= r r assertNotEqual)r@ r litextra_valid_casess r4 test_underscore_literalsz%TokenizeTest.test_underscore_literals s # - 5Ccz \# . 4 5 #G ,. 8C''
S!
S 13 7 8
s A"" A.-A.c | j dd | j dd | j dd | j dd | j d d
| j d d | j d
d | j dd | j dd | j dd | j dd | j dd | j dd | j dd | j d d | j d d | j d!d" | j d#d$ | j d%d& | j d'd( | j d)d* | j d+d, | j d-d. | j d/d0 | j d1d2 | j d3d4 | j d5d6 | j d7d8 | j d9d: | j d;d< | j d=d> | j d?d@ | j dAdB y )CNx = ''; y = ""a5 NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
STRING "''" (1, 4) (1, 6)
OP ';' (1, 6) (1, 7)
NAME 'y' (1, 8) (1, 9)
OP '=' (1, 10) (1, 11)
STRING '""' (1, 12) (1, 14)
x = '"'; y = "'"a6 NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
STRING '\'"\'' (1, 4) (1, 7)
OP ';' (1, 7) (1, 8)
NAME 'y' (1, 9) (1, 10)
OP '=' (1, 11) (1, 12)
STRING '"\'"' (1, 13) (1, 16)
x = "doesn't "shrink", does it"z NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
STRING '"doesn\'t "' (1, 4) (1, 14)
NAME 'shrink' (1, 14) (1, 20)
STRING '", does it"' (1, 20) (1, 31)
x = 'abc' + 'ABC'z NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
STRING "'abc'" (1, 4) (1, 9)
OP '+' (1, 10) (1, 11)
STRING "'ABC'" (1, 12) (1, 17)
y = "ABC" + "ABC"z NAME 'y' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
STRING '"ABC"' (1, 4) (1, 9)
OP '+' (1, 10) (1, 11)
STRING '"ABC"' (1, 12) (1, 17)
%x = r'abc' + r'ABC' + R'ABC' + R'ABC'a NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
STRING "r'abc'" (1, 4) (1, 10)
OP '+' (1, 11) (1, 12)
STRING "r'ABC'" (1, 13) (1, 19)
OP '+' (1, 20) (1, 21)
STRING "R'ABC'" (1, 22) (1, 28)
OP '+' (1, 29) (1, 30)
STRING "R'ABC'" (1, 31) (1, 37)
%y = r"abc" + r"ABC" + R"ABC" + R"ABC"a NAME 'y' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
STRING 'r"abc"' (1, 4) (1, 10)
OP '+' (1, 11) (1, 12)
STRING 'r"ABC"' (1, 13) (1, 19)
OP '+' (1, 20) (1, 21)
STRING 'R"ABC"' (1, 22) (1, 28)
OP '+' (1, 29) (1, 30)
STRING 'R"ABC"' (1, 31) (1, 37)
u'abc' + U'abc'z STRING "u'abc'" (1, 0) (1, 6)
OP '+' (1, 7) (1, 8)
STRING "U'abc'" (1, 9) (1, 15)
u"abc" + U"abc"z STRING 'u"abc"' (1, 0) (1, 6)
OP '+' (1, 7) (1, 8)
STRING 'U"abc"' (1, 9) (1, 15)
b'abc' + B'abc'z STRING "b'abc'" (1, 0) (1, 6)
OP '+' (1, 7) (1, 8)
STRING "B'abc'" (1, 9) (1, 15)
b"abc" + B"abc"z STRING 'b"abc"' (1, 0) (1, 6)
OP '+' (1, 7) (1, 8)
STRING 'B"abc"' (1, 9) (1, 15)
%br'abc' + bR'abc' + Br'abc' + BR'abc'a; STRING "br'abc'" (1, 0) (1, 7)
OP '+' (1, 8) (1, 9)
STRING "bR'abc'" (1, 10) (1, 17)
OP '+' (1, 18) (1, 19)
STRING "Br'abc'" (1, 20) (1, 27)
OP '+' (1, 28) (1, 29)
STRING "BR'abc'" (1, 30) (1, 37)
%br"abc" + bR"abc" + Br"abc" + BR"abc"a; STRING 'br"abc"' (1, 0) (1, 7)
OP '+' (1, 8) (1, 9)
STRING 'bR"abc"' (1, 10) (1, 17)
OP '+' (1, 18) (1, 19)
STRING 'Br"abc"' (1, 20) (1, 27)
OP '+' (1, 28) (1, 29)
STRING 'BR"abc"' (1, 30) (1, 37)
%rb'abc' + rB'abc' + Rb'abc' + RB'abc'a; STRING "rb'abc'" (1, 0) (1, 7)
OP '+' (1, 8) (1, 9)
STRING "rB'abc'" (1, 10) (1, 17)
OP '+' (1, 18) (1, 19)
STRING "Rb'abc'" (1, 20) (1, 27)
OP '+' (1, 28) (1, 29)
STRING "RB'abc'" (1, 30) (1, 37)
%rb"abc" + rB"abc" + Rb"abc" + RB"abc"a; STRING 'rb"abc"' (1, 0) (1, 7)
OP '+' (1, 8) (1, 9)
STRING 'rB"abc"' (1, 10) (1, 17)
OP '+' (1, 18) (1, 19)
STRING 'Rb"abc"' (1, 20) (1, 27)
OP '+' (1, 28) (1, 29)
STRING 'RB"abc"' (1, 30) (1, 37)
"a\
de\
fg"3 STRING '"a\\\nde\\\nfg"' (1, 0) (3, 3)
u"a\
de"/ STRING 'u"a\\\nde"' (1, 0) (2, 3)
rb"a\
d"/ STRING 'rb"a\\\nd"' (1, 0) (2, 2)
"""a\
b"""0 STRING '"""a\\\nb"""' (1, 0) (2, 4)
u"""a\
b"""1 STRING 'u"""a\\\nb"""' (1, 0) (2, 4)
rb"""a\
b\
c"""7 STRING 'rb"""a\\\nb\\\nc"""' (1, 0) (3, 4)
f"abc" FSTRING_START 'f"' (1, 0) (1, 2)
FSTRING_MIDDLE 'abc' (1, 2) (1, 5)
FSTRING_END '"' (1, 5) (1, 6)
fR"a{b}c"a= FSTRING_START 'fR"' (1, 0) (1, 3)
FSTRING_MIDDLE 'a' (1, 3) (1, 4)
OP '{' (1, 4) (1, 5)
NAME 'b' (1, 5) (1, 6)
OP '}' (1, 6) (1, 7)
FSTRING_MIDDLE 'c' (1, 7) (1, 8)
FSTRING_END '"' (1, 8) (1, 9)
zfR"a{{{b!r}}}c"a FSTRING_START 'fR"' (1, 0) (1, 3)
FSTRING_MIDDLE 'a{' (1, 3) (1, 5)
OP '{' (1, 6) (1, 7)
NAME 'b' (1, 7) (1, 8)
OP '!' (1, 8) (1, 9)
NAME 'r' (1, 9) (1, 10)
OP '}' (1, 10) (1, 11)
FSTRING_MIDDLE '}' (1, 11) (1, 12)
FSTRING_MIDDLE 'c' (1, 13) (1, 14)
FSTRING_END '"' (1, 14) (1, 15)
z f"{{{1+1}}}"a FSTRING_START 'f"' (1, 0) (1, 2)
FSTRING_MIDDLE '{' (1, 2) (1, 3)
OP '{' (1, 4) (1, 5)
NUMBER '1' (1, 5) (1, 6)
OP '+' (1, 6) (1, 7)
NUMBER '1' (1, 7) (1, 8)
OP '}' (1, 8) (1, 9)
FSTRING_MIDDLE '}' (1, 9) (1, 10)
FSTRING_END '"' (1, 11) (1, 12)
z f"""{f'''{f'{f"{1+1}"}'}'''}"""ad FSTRING_START 'f"""' (1, 0) (1, 4)
OP '{' (1, 4) (1, 5)
FSTRING_START "f'''" (1, 5) (1, 9)
OP '{' (1, 9) (1, 10)
FSTRING_START "f'" (1, 10) (1, 12)
OP '{' (1, 12) (1, 13)
FSTRING_START 'f"' (1, 13) (1, 15)
OP '{' (1, 15) (1, 16)
NUMBER '1' (1, 16) (1, 17)
OP '+' (1, 17) (1, 18)
NUMBER '1' (1, 18) (1, 19)
OP '}' (1, 19) (1, 20)
FSTRING_END '"' (1, 20) (1, 21)
OP '}' (1, 21) (1, 22)
FSTRING_END "'" (1, 22) (1, 23)
OP '}' (1, 23) (1, 24)
FSTRING_END "'''" (1, 24) (1, 27)
OP '}' (1, 27) (1, 28)
FSTRING_END '"""' (1, 28) (1, 31)
z.f""" x
str(data, encoding={invalid!r})
"""a FSTRING_START 'f"""' (1, 0) (1, 4)
FSTRING_MIDDLE ' x\nstr(data, encoding=' (1, 4) (2, 19)
OP '{' (2, 19) (2, 20)
NAME 'invalid' (2, 20) (2, 27)
OP '!' (2, 27) (2, 28)
NAME 'r' (2, 28) (2, 29)
OP '}' (2, 29) (2, 30)
FSTRING_MIDDLE ')\n' (2, 30) (3, 0)
FSTRING_END '"""' (3, 0) (3, 3)
z#f"""123456789
something{None}bad"""aO FSTRING_START 'f"""' (1, 0) (1, 4)
FSTRING_MIDDLE '123456789\nsomething' (1, 4) (2, 9)
OP '{' (2, 9) (2, 10)
NAME 'None' (2, 10) (2, 14)
OP '}' (2, 14) (2, 15)
FSTRING_MIDDLE 'bad' (2, 15) (2, 18)
FSTRING_END '"""' (2, 18) (2, 21)
f"""abc""" FSTRING_START 'f"""' (1, 0) (1, 4)
FSTRING_MIDDLE 'abc' (1, 4) (1, 7)
FSTRING_END '"""' (1, 7) (1, 10)
f"abc\
def" FSTRING_START 'f"' (1, 0) (1, 2)
FSTRING_MIDDLE 'abc\\\ndef' (1, 2) (2, 3)
FSTRING_END '"' (2, 3) (2, 4)
Rf"abc\
def" FSTRING_START 'Rf"' (1, 0) (1, 3)
FSTRING_MIDDLE 'abc\\\ndef' (1, 3) (2, 3)
FSTRING_END '"' (2, 3) (2, 4)
z5f'some words {a+b:.3f} more words {c+d=} final words'aA FSTRING_START "f'" (1, 0) (1, 2)
FSTRING_MIDDLE 'some words ' (1, 2) (1, 13)
OP '{' (1, 13) (1, 14)
NAME 'a' (1, 14) (1, 15)
OP '+' (1, 15) (1, 16)
NAME 'b' (1, 16) (1, 17)
OP ':' (1, 17) (1, 18)
FSTRING_MIDDLE '.3f' (1, 18) (1, 21)
OP '}' (1, 21) (1, 22)
FSTRING_MIDDLE ' more words ' (1, 22) (1, 34)
OP '{' (1, 34) (1, 35)
NAME 'c' (1, 35) (1, 36)
OP '+' (1, 36) (1, 37)
NAME 'd' (1, 37) (1, 38)
OP '=' (1, 38) (1, 39)
OP '}' (1, 39) (1, 40)
FSTRING_MIDDLE ' final words' (1, 40) (1, 52)
FSTRING_END "'" (1, 52) (1, 53)
z
f'''{
3
=}'''a` FSTRING_START "f'''" (1, 0) (1, 4)
OP '{' (1, 4) (1, 5)
NL '\n' (1, 5) (1, 6)
NUMBER '3' (2, 0) (2, 1)
NL '\n' (2, 1) (2, 2)
OP '=' (3, 0) (3, 1)
OP '}' (3, 1) (3, 2)
FSTRING_END "'''" (3, 2) (3, 5)
ru rv s r4 test_stringzTokenizeTest.test_string+ s1 . 1
1 4
A D
/ 2
/ 2
C
F
C
F
- 0
- 0
- 0
- 0
C F
C F
C F
C F
H '
K *
- 0
N
-
E H *
N
Q
B E
L +
S V (
r6 c p | j dd | j dd | j dd y )N!def d22(a, b, c=2, d=2, *k): passa NAME 'def' (1, 0) (1, 3)
NAME 'd22' (1, 4) (1, 7)
OP '(' (1, 7) (1, 8)
NAME 'a' (1, 8) (1, 9)
OP ',' (1, 9) (1, 10)
NAME 'b' (1, 11) (1, 12)
OP ',' (1, 12) (1, 13)
NAME 'c' (1, 14) (1, 15)
OP '=' (1, 15) (1, 16)
NUMBER '2' (1, 16) (1, 17)
OP ',' (1, 17) (1, 18)
NAME 'd' (1, 19) (1, 20)
OP '=' (1, 20) (1, 21)
NUMBER '2' (1, 21) (1, 22)
OP ',' (1, 22) (1, 23)
OP '*' (1, 24) (1, 25)
NAME 'k' (1, 25) (1, 26)
OP ')' (1, 26) (1, 27)
OP ':' (1, 27) (1, 28)
NAME 'pass' (1, 29) (1, 33)
def d01v_(a=1, *k, **w): passa NAME 'def' (1, 0) (1, 3)
NAME 'd01v_' (1, 4) (1, 9)
OP '(' (1, 9) (1, 10)
NAME 'a' (1, 10) (1, 11)
OP '=' (1, 11) (1, 12)
NUMBER '1' (1, 12) (1, 13)
OP ',' (1, 13) (1, 14)
OP '*' (1, 15) (1, 16)
NAME 'k' (1, 16) (1, 17)
OP ',' (1, 17) (1, 18)
OP '**' (1, 19) (1, 21)
NAME 'w' (1, 21) (1, 22)
OP ')' (1, 22) (1, 23)
OP ':' (1, 23) (1, 24)
NAME 'pass' (1, 25) (1, 29)
&def d23(a: str, b: int=3) -> int: passa NAME 'def' (1, 0) (1, 3)
NAME 'd23' (1, 4) (1, 7)
OP '(' (1, 7) (1, 8)
NAME 'a' (1, 8) (1, 9)
OP ':' (1, 9) (1, 10)
NAME 'str' (1, 11) (1, 14)
OP ',' (1, 14) (1, 15)
NAME 'b' (1, 16) (1, 17)
OP ':' (1, 17) (1, 18)
NAME 'int' (1, 19) (1, 22)
OP '=' (1, 22) (1, 23)
NUMBER '3' (1, 23) (1, 24)
OP ')' (1, 24) (1, 25)
OP '->' (1, 26) (1, 28)
NAME 'int' (1, 29) (1, 32)
OP ':' (1, 32) (1, 33)
NAME 'pass' (1, 34) (1, 38)
ru rv s r4
test_functionzTokenizeTest.test_function@ sJ ? B ,
; > "
D G r6 c ( | j dd y )NXif 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != 1 and 5 in 1 not in 1 is 1 or 5 is not 1: passa NAME 'if' (1, 0) (1, 2)
NUMBER '1' (1, 3) (1, 4)
OP '<' (1, 5) (1, 6)
NUMBER '1' (1, 7) (1, 8)
OP '>' (1, 9) (1, 10)
NUMBER '1' (1, 11) (1, 12)
OP '==' (1, 13) (1, 15)
NUMBER '1' (1, 16) (1, 17)
OP '>=' (1, 18) (1, 20)
NUMBER '5' (1, 21) (1, 22)
OP '<=' (1, 23) (1, 25)
NUMBER '0x15' (1, 26) (1, 30)
OP '<=' (1, 31) (1, 33)
NUMBER '0x12' (1, 34) (1, 38)
OP '!=' (1, 39) (1, 41)
NUMBER '1' (1, 42) (1, 43)
NAME 'and' (1, 44) (1, 47)
NUMBER '5' (1, 48) (1, 49)
NAME 'in' (1, 50) (1, 52)
NUMBER '1' (1, 53) (1, 54)
NAME 'not' (1, 55) (1, 58)
NAME 'in' (1, 59) (1, 61)
NUMBER '1' (1, 62) (1, 63)
NAME 'is' (1, 64) (1, 66)
NUMBER '1' (1, 67) (1, 68)
NAME 'or' (1, 69) (1, 71)
NUMBER '5' (1, 72) (1, 73)
NAME 'is' (1, 74) (1, 76)
NAME 'not' (1, 77) (1, 80)
NUMBER '1' (1, 81) (1, 82)
OP ':' (1, 82) (1, 83)
NAME 'pass' (1, 84) (1, 88)
ru rv s r4 test_comparisonz TokenizeTest.test_comparison| M!O" r6 c ( | j dd y )Nzx = 1 << 1 >> 5a6 NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '1' (1, 4) (1, 5)
OP '<<' (1, 6) (1, 8)
NUMBER '1' (1, 9) (1, 10)
OP '>>' (1, 11) (1, 13)
NUMBER '5' (1, 14) (1, 15)
ru rv s r4
test_shiftzTokenizeTest.test_shift s - 0 r6 c ( | j dd y )N%x = 1 - y + 15 - 1 + 0x124 + z + a[5]a$ NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '1' (1, 4) (1, 5)
OP '-' (1, 6) (1, 7)
NAME 'y' (1, 8) (1, 9)
OP '+' (1, 10) (1, 11)
NUMBER '15' (1, 12) (1, 14)
OP '-' (1, 15) (1, 16)
NUMBER '1' (1, 17) (1, 18)
OP '+' (1, 19) (1, 20)
NUMBER '0x124' (1, 21) (1, 26)
OP '+' (1, 27) (1, 28)
NAME 'z' (1, 29) (1, 30)
OP '+' (1, 31) (1, 32)
NAME 'a' (1, 33) (1, 34)
OP '[' (1, 34) (1, 35)
NUMBER '5' (1, 35) (1, 36)
OP ']' (1, 36) (1, 37)
ru rv s r4
test_additivezTokenizeTest.test_additive C F r6 c ( | j dd y )Nx = 1//1*1/5*12%0x12@42a NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '1' (1, 4) (1, 5)
OP '//' (1, 5) (1, 7)
NUMBER '1' (1, 7) (1, 8)
OP '*' (1, 8) (1, 9)
NUMBER '1' (1, 9) (1, 10)
OP '/' (1, 10) (1, 11)
NUMBER '5' (1, 11) (1, 12)
OP '*' (1, 12) (1, 13)
NUMBER '12' (1, 13) (1, 15)
OP '%' (1, 15) (1, 16)
NUMBER '0x12' (1, 16) (1, 20)
OP '@' (1, 20) (1, 21)
NUMBER '42' (1, 21) (1, 23)
ru rv s r4 test_multiplicativez TokenizeTest.test_multiplicative 5 8 r6 c L | j dd | j dd y )N~1 ^ 1 & 1 |1 ^ -1a OP '~' (1, 0) (1, 1)
NUMBER '1' (1, 1) (1, 2)
OP '^' (1, 3) (1, 4)
NUMBER '1' (1, 5) (1, 6)
OP '&' (1, 7) (1, 8)
NUMBER '1' (1, 9) (1, 10)
OP '|' (1, 11) (1, 12)
NUMBER '1' (1, 12) (1, 13)
OP '^' (1, 14) (1, 15)
OP '-' (1, 16) (1, 17)
NUMBER '1' (1, 17) (1, 18)
-1*1/1+1*1//1 - ---1**1aH OP '-' (1, 0) (1, 1)
NUMBER '1' (1, 1) (1, 2)
OP '*' (1, 2) (1, 3)
NUMBER '1' (1, 3) (1, 4)
OP '/' (1, 4) (1, 5)
NUMBER '1' (1, 5) (1, 6)
OP '+' (1, 6) (1, 7)
NUMBER '1' (1, 7) (1, 8)
OP '*' (1, 8) (1, 9)
NUMBER '1' (1, 9) (1, 10)
OP '//' (1, 10) (1, 12)
NUMBER '1' (1, 12) (1, 13)
OP '-' (1, 14) (1, 15)
OP '-' (1, 16) (1, 17)
OP '-' (1, 17) (1, 18)
OP '-' (1, 18) (1, 19)
NUMBER '1' (1, 19) (1, 20)
OP '**' (1, 20) (1, 22)
NUMBER '1' (1, 22) (1, 23)
ru rv s r4
test_unaryzTokenizeTest.test_unary s0 0 3
5 8 r6 c ( | j dd y )N/import sys, time
x = sys.modules['time'].time()a NAME 'import' (1, 0) (1, 6)
NAME 'sys' (1, 7) (1, 10)
OP ',' (1, 10) (1, 11)
NAME 'time' (1, 12) (1, 16)
NEWLINE '\n' (1, 16) (1, 17)
NAME 'x' (2, 0) (2, 1)
OP '=' (2, 2) (2, 3)
NAME 'sys' (2, 4) (2, 7)
OP '.' (2, 7) (2, 8)
NAME 'modules' (2, 8) (2, 15)
OP '[' (2, 15) (2, 16)
STRING "'time'" (2, 16) (2, 22)
OP ']' (2, 22) (2, 23)
OP '.' (2, 23) (2, 24)
NAME 'time' (2, 24) (2, 28)
OP '(' (2, 28) (2, 29)
OP ')' (2, 29) (2, 30)
ru rv s r4
test_selectorzTokenizeTest.test_selector N Q r6 c ( | j dd y )N @staticmethod
def foo(x,y): passa OP '@' (1, 0) (1, 1)
NAME 'staticmethod' (1, 1) (1, 13)
NEWLINE '\n' (1, 13) (1, 14)
NAME 'def' (2, 0) (2, 3)
NAME 'foo' (2, 4) (2, 7)
OP '(' (2, 7) (2, 8)
NAME 'x' (2, 8) (2, 9)
OP ',' (2, 9) (2, 10)
NAME 'y' (2, 10) (2, 11)
OP ')' (2, 11) (2, 12)
OP ':' (2, 12) (2, 13)
NAME 'pass' (2, 14) (2, 18)
ru rv s r4 test_methodzTokenizeTest.test_method ?
B
r6 c ( | j dd y )Nz def f():
if x
passa_ NAME 'def' (1, 0) (1, 3)
NAME 'f' (1, 4) (1, 5)
OP '(' (1, 5) (1, 6)
OP ')' (1, 6) (1, 7)
OP ':' (1, 7) (1, 8)
NEWLINE '\n' (1, 8) (1, 9)
INDENT '\t' (2, 0) (2, 1)
NAME 'if' (2, 1) (2, 3)
NAME 'x' (2, 4) (2, 5)
NEWLINE '\n' (2, 5) (2, 6)
INDENT ' \t' (3, 0) (3, 9)
NAME 'pass' (3, 9) (3, 13)
DEDENT '' (4, 0) (4, 0)
DEDENT '' (4, 0) (4, 0)
ru rv s r4 test_tabszTokenizeTest.test_tabs% s -/ r6 c ( | j dd y )Nu! Örter = 'places'
grün = 'green'u7 NAME 'Örter' (1, 0) (1, 5)
OP '=' (1, 6) (1, 7)
STRING "'places'" (1, 8) (1, 16)
NEWLINE '\n' (1, 16) (1, 17)
NAME 'grün' (2, 0) (2, 4)
OP '=' (2, 5) (2, 6)
STRING "'green'" (2, 7) (2, 14)
ru rv s r4 test_non_ascii_identifiersz'TokenizeTest.test_non_ascii_identifiers: s @ C r6 c ( | j dd y )N# Örter = u'places'
grün = U'green'u7 NAME 'Örter' (1, 0) (1, 5)
OP '=' (1, 6) (1, 7)
STRING "u'places'" (1, 8) (1, 17)
NEWLINE '\n' (1, 17) (1, 18)
NAME 'grün' (2, 0) (2, 4)
OP '=' (2, 5) (2, 6)
STRING "U'green'" (2, 7) (2, 15)
ru rv s r4 test_unicodezTokenizeTest.test_unicodeF B E r6 c | j dd | j dd | j dd | j dd | j d d
| j d d | j d
d | j dd | j dd | j dd | j dd | j dd | j dd | j dd | j d d | j d d | j d!d" | j d#d$ | j d%d& | j d'd( | j d)d* y )+N async = 1z NAME 'async' (1, 0) (1, 5)
OP '=' (1, 6) (1, 7)
NUMBER '1' (1, 8) (1, 9)
a = (async = 1)a8 NAME 'a' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
OP '(' (1, 4) (1, 5)
NAME 'async' (1, 5) (1, 10)
OP '=' (1, 11) (1, 12)
NUMBER '1' (1, 13) (1, 14)
OP ')' (1, 14) (1, 15)
async()z NAME 'async' (1, 0) (1, 5)
OP '(' (1, 5) (1, 6)
OP ')' (1, 6) (1, 7)
class async(Bar):passa< NAME 'class' (1, 0) (1, 5)
NAME 'async' (1, 6) (1, 11)
OP '(' (1, 11) (1, 12)
NAME 'Bar' (1, 12) (1, 15)
OP ')' (1, 15) (1, 16)
OP ':' (1, 16) (1, 17)
NAME 'pass' (1, 17) (1, 21)
class async:passz NAME 'class' (1, 0) (1, 5)
NAME 'async' (1, 6) (1, 11)
OP ':' (1, 11) (1, 12)
NAME 'pass' (1, 12) (1, 16)
await = 1z NAME 'await' (1, 0) (1, 5)
OP '=' (1, 6) (1, 7)
NUMBER '1' (1, 8) (1, 9)
foo.asyncz NAME 'foo' (1, 0) (1, 3)
OP '.' (1, 3) (1, 4)
NAME 'async' (1, 4) (1, 9)
async for a in b: passa; NAME 'async' (1, 0) (1, 5)
NAME 'for' (1, 6) (1, 9)
NAME 'a' (1, 10) (1, 11)
NAME 'in' (1, 12) (1, 14)
NAME 'b' (1, 15) (1, 16)
OP ':' (1, 16) (1, 17)
NAME 'pass' (1, 18) (1, 22)
async with a as b: passa< NAME 'async' (1, 0) (1, 5)
NAME 'with' (1, 6) (1, 10)
NAME 'a' (1, 11) (1, 12)
NAME 'as' (1, 13) (1, 15)
NAME 'b' (1, 16) (1, 17)
OP ':' (1, 17) (1, 18)
NAME 'pass' (1, 19) (1, 23)
async.fooz NAME 'async' (1, 0) (1, 5)
OP '.' (1, 5) (1, 6)
NAME 'foo' (1, 6) (1, 9)
asyncz/ NAME 'async' (1, 0) (1, 5)
async
#comment
awaitz NAME 'async' (1, 0) (1, 5)
NEWLINE '\n' (1, 5) (1, 6)
COMMENT '#comment' (2, 0) (2, 8)
NL '\n' (2, 8) (2, 9)
NAME 'await' (3, 0) (3, 5)
async
...
awaitz NAME 'async' (1, 0) (1, 5)
NEWLINE '\n' (1, 5) (1, 6)
OP '...' (2, 0) (2, 3)
NEWLINE '\n' (2, 3) (2, 4)
NAME 'await' (3, 0) (3, 5)
async
awaitz NAME 'async' (1, 0) (1, 5)
NEWLINE '\n' (1, 5) (1, 6)
NAME 'await' (2, 0) (2, 5)
foo.async + 1z NAME 'foo' (1, 0) (1, 3)
OP '.' (1, 3) (1, 4)
NAME 'async' (1, 4) (1, 9)
OP '+' (1, 10) (1, 11)
NUMBER '1' (1, 12) (1, 13)
async def foo(): passa; NAME 'async' (1, 0) (1, 5)
NAME 'def' (1, 6) (1, 9)
NAME 'foo' (1, 10) (1, 13)
OP '(' (1, 13) (1, 14)
OP ')' (1, 14) (1, 15)
OP ':' (1, 15) (1, 16)
NAME 'pass' (1, 17) (1, 21)
Nasync def foo():
def foo(await):
await = 1
if 1:
await
async += 1
a NAME 'async' (1, 0) (1, 5)
NAME 'def' (1, 6) (1, 9)
NAME 'foo' (1, 10) (1, 13)
OP '(' (1, 13) (1, 14)
OP ')' (1, 14) (1, 15)
OP ':' (1, 15) (1, 16)
NEWLINE '\n' (1, 16) (1, 17)
INDENT ' ' (2, 0) (2, 2)
NAME 'def' (2, 2) (2, 5)
NAME 'foo' (2, 6) (2, 9)
OP '(' (2, 9) (2, 10)
NAME 'await' (2, 10) (2, 15)
OP ')' (2, 15) (2, 16)
OP ':' (2, 16) (2, 17)
NEWLINE '\n' (2, 17) (2, 18)
INDENT ' ' (3, 0) (3, 4)
NAME 'await' (3, 4) (3, 9)
OP '=' (3, 10) (3, 11)
NUMBER '1' (3, 12) (3, 13)
NEWLINE '\n' (3, 13) (3, 14)
DEDENT '' (4, 2) (4, 2)
NAME 'if' (4, 2) (4, 4)
NUMBER '1' (4, 5) (4, 6)
OP ':' (4, 6) (4, 7)
NEWLINE '\n' (4, 7) (4, 8)
INDENT ' ' (5, 0) (5, 4)
NAME 'await' (5, 4) (5, 9)
NEWLINE '\n' (5, 9) (5, 10)
DEDENT '' (6, 0) (6, 0)
DEDENT '' (6, 0) (6, 0)
NAME 'async' (6, 0) (6, 5)
OP '+=' (6, 6) (6, 8)
NUMBER '1' (6, 9) (6, 10)
NEWLINE '\n' (6, 10) (6, 11)
)async def foo():
async for i in 1: passa NAME 'async' (1, 0) (1, 5)
NAME 'def' (1, 6) (1, 9)
NAME 'foo' (1, 10) (1, 13)
OP '(' (1, 13) (1, 14)
OP ')' (1, 14) (1, 15)
OP ':' (1, 15) (1, 16)
NEWLINE '\n' (1, 16) (1, 17)
INDENT ' ' (2, 0) (2, 2)
NAME 'async' (2, 2) (2, 7)
NAME 'for' (2, 8) (2, 11)
NAME 'i' (2, 12) (2, 13)
NAME 'in' (2, 14) (2, 16)
NUMBER '1' (2, 17) (2, 18)
OP ':' (2, 18) (2, 19)
NAME 'pass' (2, 20) (2, 24)
DEDENT '' (3, 0) (3, 0)
async def foo(async): awaitah NAME 'async' (1, 0) (1, 5)
NAME 'def' (1, 6) (1, 9)
NAME 'foo' (1, 10) (1, 13)
OP '(' (1, 13) (1, 14)
NAME 'async' (1, 14) (1, 19)
OP ')' (1, 19) (1, 20)
OP ':' (1, 20) (1, 21)
NAME 'await' (1, 22) (1, 27)
@def f():
def baz(): pass
async def bar(): pass
await = 2a NAME 'def' (1, 0) (1, 3)
NAME 'f' (1, 4) (1, 5)
OP '(' (1, 5) (1, 6)
OP ')' (1, 6) (1, 7)
OP ':' (1, 7) (1, 8)
NEWLINE '\n' (1, 8) (1, 9)
NL '\n' (2, 0) (2, 1)
INDENT ' ' (3, 0) (3, 2)
NAME 'def' (3, 2) (3, 5)
NAME 'baz' (3, 6) (3, 9)
OP '(' (3, 9) (3, 10)
OP ')' (3, 10) (3, 11)
OP ':' (3, 11) (3, 12)
NAME 'pass' (3, 13) (3, 17)
NEWLINE '\n' (3, 17) (3, 18)
NAME 'async' (4, 2) (4, 7)
NAME 'def' (4, 8) (4, 11)
NAME 'bar' (4, 12) (4, 15)
OP '(' (4, 15) (4, 16)
OP ')' (4, 16) (4, 17)
OP ':' (4, 17) (4, 18)
NAME 'pass' (4, 19) (4, 23)
NEWLINE '\n' (4, 23) (4, 24)
NL '\n' (5, 0) (5, 1)
NAME 'await' (6, 2) (6, 7)
OP '=' (6, 8) (6, 9)
NUMBER '2' (6, 10) (6, 11)
DEDENT '' (7, 0) (7, 0)
Fasync def f():
def baz(): pass
async def bar(): pass
await = 2a NAME 'async' (1, 0) (1, 5)
NAME 'def' (1, 6) (1, 9)
NAME 'f' (1, 10) (1, 11)
OP '(' (1, 11) (1, 12)
OP ')' (1, 12) (1, 13)
OP ':' (1, 13) (1, 14)
NEWLINE '\n' (1, 14) (1, 15)
NL '\n' (2, 0) (2, 1)
INDENT ' ' (3, 0) (3, 2)
NAME 'def' (3, 2) (3, 5)
NAME 'baz' (3, 6) (3, 9)
OP '(' (3, 9) (3, 10)
OP ')' (3, 10) (3, 11)
OP ':' (3, 11) (3, 12)
NAME 'pass' (3, 13) (3, 17)
NEWLINE '\n' (3, 17) (3, 18)
NAME 'async' (4, 2) (4, 7)
NAME 'def' (4, 8) (4, 11)
NAME 'bar' (4, 12) (4, 15)
OP '(' (4, 15) (4, 16)
OP ')' (4, 16) (4, 17)
OP ':' (4, 17) (4, 18)
NAME 'pass' (4, 19) (4, 23)
NEWLINE '\n' (4, 23) (4, 24)
NL '\n' (5, 0) (5, 1)
NAME 'await' (6, 2) (6, 7)
OP '=' (6, 8) (6, 9)
NUMBER '2' (6, 10) (6, 11)
DEDENT '' (7, 0) (7, 0)
ru rv s r4
test_asynczTokenizeTest.test_asyncR s K *
- 0
I (
3 6
. 1
K *
K *
4 7
5 8
K *
G &
4 7
/ 2
N -
O .
3 6
#* X
*
= @
# J
$ r6 c ( | j dd y )Nz [
# A comment here
1
]
ad OP '[' (1, 0) (1, 1)
NL '\n' (1, 1) (1, 2)
COMMENT '# A comment here' (2, 4) (2, 20)
NL '\n' (2, 20) (2, 21)
NUMBER '1' (3, 4) (3, 5)
NL '\n' (3, 5) (3, 6)
OP ']' (4, 0) (4, 1)
NEWLINE '\n' (4, 1) (4, 2)
ru rv s r4 3test_newline_after_parenthesized_block_with_commentz@TokenizeTest.test_newline_after_parenthesized_block_with_commentd s
r6 c ( | j dd y )Nz); xz OP ')' (1, 0) (1, 1)
OP ';' (1, 1) (1, 2)
NAME 'x' (1, 3) (1, 4)
ru rv s r4 ,test_closing_parenthesis_from_different_linez9TokenizeTest.test_closing_parenthesis_from_different_lineu s F % r6 N)__name__
__module__ __qualname__rD rS rX rj rw ry r r r r r r r r r r r r r r r r r rH r6 r4 r8 r8 ' s 92&5F7P7 r 0& P80S j: x$ L
. (# J , " *
P d " r6 r8 c e Zd Zd Zy)GenerateTokensTestc t | }t t |j | }| j ||j j
y N)r r5 r r< r= r> r( r? s r4 rD z!GenerateTokensTest.check_tokenize} sB
QK-oajj.I1M !2!=!=!?@r6 N)r r r rD rH r6 r4 r r | s Ar6 r c x g }t t | j d j }|D ]_ \ }}}}}|t k( r;d|v r7|j t dft dft t | ft dfg M|j ||f a t | j d j S )Nr: .Decimalr ))r r r; r< r extendr r r
reprr) r decodestrip)rA r, gtoknumtokval_s r4 decistmtr s
F'*+445A$% , 1a V v
MMy!S f &S
MM66* + , f $ $W - 3 3 55r6 c e Zd Zd Zy)TestMiscc ddl m} d}| j t | d | j t t
| d | j t
t | |d y )Nr )r z+21.3e-5*-.1234/81.7z8+Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7')z-3.2171603427[0-9]*e-0+7z!-3.217160342717258261933904529E-7)decimalr r= r assertRegexr eval)r@ r rA s r4
test_decistmtzTestMisc.test_decistmt sd $ " !S U
d1g (BC
hqk* !DE Gr6 N)r r r r rH r6 r4 r r s Gr6 r c 4 e Zd ZdZd Zd Zd Zd Zd Zd Z y) TestTokenizerAdheresToPep0263zU
Test that tokenizer adheres to the coding behaviour stipulated in PEP 0263.
c t j j t j j t | }t |d 5 }t j | | d d d y # 1 sw Y y xY w)Nrb)ospathjoindirname__file__r
TestRoundtripcheck_roundtrip)r@ ra r rC s r4 _testFilez'TestTokenizerAdheresToPep0263._testFile sT ww||BGGOOH 5x@
$
3 ) )$ 2 3 3 3s A..A7c * d}| j | y )Nz9tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txtr r@ rC s r4 'test_utf8_coding_cookie_and_no_utf8_bomzETestTokenizerAdheresToPep0263.test_utf8_coding_cookie_and_no_utf8_bom G qr6 c J d}| j t | j | y)a
As per PEP 0263, if a file starts with a utf-8 BOM signature, the only
allowed encoding for the comment is 'utf-8'. The text file used in
this test starts with a BOM signature, but specifies latin1 as the
coding, so verify that a SyntaxError is raised, which matches the
behaviour of the interpreter when it encounters a similar condition.
z8tokenize_tests-latin1-coding-cookie-and-utf8-bom-sig.txtNrN SyntaxErrorr r s r4 &test_latin1_coding_cookie_and_utf8_bomzDTestTokenizerAdheresToPep0263.test_latin1_coding_cookie_and_utf8_bom s
G +t~~q9r6 c * d}| j | y )Nz9tokenize_tests-no-coding-cookie-and-utf8-bom-sig-only.txtr r s r4 "test_no_coding_cookie_and_utf8_bomz@TestTokenizerAdheresToPep0263.test_no_coding_cookie_and_utf8_bom r! r6 c * d}| j | y )Nz6tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txtr r s r4 $test_utf8_coding_cookie_and_utf8_bomzBTestTokenizerAdheresToPep0263.test_utf8_coding_cookie_and_utf8_bom s D qr6 c | j t | j d | j t | j d y )Nz
bad_coding.pyzbad_coding2.pyr# rv s r4 test_bad_coding_cookiez4TestTokenizerAdheresToPep0263.test_bad_coding_cookie s/ +t~~G +t~~7GHr6 N)
r r r __doc__r r r% r' r) r+ rH r6 r4 r r s& 3
:Ir6 r c e Zd Zd Zy)
Test_Tokenizec d}|j d dfd}t t | j dd d d }t ddd d
d g}| j ||d y ) Nu "ЉЊЈЁЂ"r: Fc 3 ( K sd y d y w)NTr6 rH )firstr3 s r4 r< zNTest_Tokenize.test__tokenize_decodes_with_specified_encoding.<locals>.readline s
s T)encoding extra_tokensrV rZ rM r rM z bytes not decoded with encoding)r; rP r rQ r r= )r@ literalr< rW expected_tokensr1 r3 s @@r4 .test__tokenize_decodes_with_specified_encodingz<Test_Tokenize.test__tokenize_decodes_with_specified_encoding sx ~~g& 7
8K8KV]#') **-2/$QWX : <r6 N)r r r r9 rH r6 r4 r. r. s <r6 r. c e Zd Zd Zd Zd Zd Zd Zd Zd Z d Z
d Z d
Z d Z
d Zd
Zd Zd Zd Zd Zd Zd Zd Zy)TestDetectEncodingc dfd}|S )Nr c D t k( rt } dz
| S NrM )r'
StopIteration)r3 indexliness r4 r< z1TestDetectEncoding.get_readline.<locals>.readline s, E
"##<D QJEKr6 rH )r@ rA r< r@ s ` @r4 get_readlinez TestDetectEncoding.get_readline s r6 c d}t | j | \ }}| j |d | j |t |d d y )N) # something
print(something)
do_something(else)
r: )r rB r= rP r@ rA r2 consumed_liness r4 test_no_bom_no_encoding_cookiez1TestDetectEncoding.test_no_bom_no_encoding_cookie sO
$343D3DU3K#L . 7+ eBQi 9r6 c d}t | j | \ }}| j |d | j |ddg y )N)s # something
rE rF utf-8-sigrD rE r rB r= rH s r4 test_bom_no_cookiez%TestDetectEncoding.test_bom_no_cookie sO
$343D3DU3K#L . ;/ *,AB Dr6 c d}t | j | \ }}| j |d | j |dg y )N) # -*- coding: latin-1 -*-
rE rF
iso-8859-1rP rM rH s r4 test_cookie_first_line_no_bomz0TestDetectEncoding.test_cookie_first_line_no_bom sI
$343D3DU3K#L . <0 *H)IJr6 c d}t | j | \ }}| j |d | j |dg y )N)s # coding=utf-8
rE rF rL s # coding=utf-8
rM rH s r4 &test_matched_bom_and_cookie_first_linez9TestDetectEncoding.test_matched_bom_and_cookie_first_line sI
$343D3DU3K#L . ;/ *=)>?r6 c ` d}| j | }| j t t | y )N)s# # vim: set fileencoding=ascii :
rE rF rB rN r$ r r@ rA r< s r4 <test_mismatched_bom_and_cookie_first_line_raises_syntaxerrorzOTestDetectEncoding.test_mismatched_bom_and_cookie_first_line_raises_syntaxerror s-
$$U+ +Ar6 c d}t | j | \ }}| j |d ddg}| j || y )N)
#! something
# vim: set fileencoding=ascii :
rE rF asciirZ r[ rM r@ rA r2 rI rB s r4 test_cookie_second_line_no_bomz1TestDetectEncoding.test_cookie_second_line_no_bom& sP
$343D3DU3K#L . 7+%'KL 2r6 c d}t | j | \ }}| j |d | j |ddg y )N) #! something
f# coding=utf-8
rE rF rL rZ ra rM rH s r4 'test_matched_bom_and_cookie_second_linez:TestDetectEncoding.test_matched_bom_and_cookie_second_line2 sO
$343D3DU3K#L . ;/ +-AB Dr6 c ` d}| j | }| j t t | y )N)r` r[ rE rF rV rW s r4 =test_mismatched_bom_and_cookie_second_line_raises_syntaxerrorzPTestDetectEncoding.test_mismatched_bom_and_cookie_second_line_raises_syntaxerror> s-
$$U+ +Ar6 c d}t | j | \ }}| j |d dg}| j || y )N) print('£')
% # vim: set fileencoding=iso8859-15 :
print('€')
r: rf rM r] s r4 /test_cookie_second_line_noncommented_first_linezBTestDetectEncoding.test_cookie_second_line_noncommented_first_lineH sM
$343D3DU3K#L . 7+*+ 2r6 c d}t | j | \ }}| j |d ddg}| j || y )N)
#print('£')
rg rh
iso8859-15rk rg rM r] s r4 ,test_cookie_second_line_commented_first_linez?TestDetectEncoding.test_cookie_second_line_commented_first_lineS sP
$343D3DU3K#L . <0+-VW 2r6 c d}t | j | \ }}| j |d ddg}| j || y )N)
rg rh rl ro rg rM r] s r4 (test_cookie_second_line_empty_first_linez;TestDetectEncoding.test_cookie_second_line_empty_first_line^ sP
$343D3DU3K#L . <0DE 2r6 c d}|D ]g }dD ]` }|j d| }dd|j d z dz dd f}| j | }t | \ }}| j |d
b i y ) N)latin-1rQ z iso-latin-1z latin-1-unixziso-8859-1-unixziso-latin-1-mac-r rt #!/usr/bin/python
# coding: r\ ro s print(things)
s do_something += 4
rQ replacer; rB r r= r@ encodingsr2 repencrA rlfoundrI s r4 test_latin1_normalizationz,TestDetectEncoding.test_latin1_normalizationi s ; ! 6H!
6 &&sC0/&G)<<uD+/1 &&u-(7(;%~ 5
6 6r6 c ` d}| j | }| j t t | y )Ns
print("")rV rW s r4 test_syntaxerror_latin1z*TestDetectEncoding.test_syntaxerror_latin1x s- $$U+ +Ar6 c d}|D ]f }dD ]_ }|j d| }dd|j d z dz df}| j | }t | \ }}| j |d a h y )
N)r: z utf-8-macz
utf-8-unixrs rt ru rv r\ ro s 1 + 3
r: rw ry s r4 test_utf8_normalizationz*TestDetectEncoding.test_utf8_normalization s 8 ! 1H!
1 &&sC0/&G)<<uD#% &&u-(7(;%~ 0
1 1r6 c t | j d }t | \ }}| j |d | j |dg t | j d \ }}| j |d | j |g | j d }t | \ }}| j |d | j |dg | j d }t | \ }}| j |d | j |g | j d }| j t t | y ) N)rE r: rE rH )s print(something)
rL )s )s # coding: bad
)rB r r= rN r$ r@ r< r2 rI s r4 test_short_filesz#TestDetectEncoding.test_short_files s' $$%=>#28#< . 7+ *?)@A#243D3DR3H#I . 7+ ,$$%IJ#28#< . ;/ *?)@A$$%78#28#< . ;/ ,$$%:; +Ar6 c | j d }t | \ }}| j |d | j |dg y )N) print("#coding=fake")r: r )rB r r= r s r4 test_false_encodingz&TestDetectEncoding.test_false_encoding sF $$%@A#28#< . 7+ *B)CDr6 c t j dz }| j t j | dD ] }t |d| 5 }t d|z | t d| d d d t
| 5 }| j |j | | j |j d d d d t |dd 5 }t d| d d d t
| 5 }| j |j d | j |j d d d d y # 1 sw Y xY w# 1 sw Y xY w# 1 sw Y oxY w# 1 sw Y y xY w)
Nz.py)z iso-8859-15r: wr2 z # coding: %sfileu print('euro:€')rrL )
r TESTFN
addCleanupunlinkr print
tokenize_openr= r2 mode)r@ ra r2 fps r4 test_openz TestDetectEncoding.test_open s> ##e+ (((3 1 /Hhh7
72nx/b9,26
7 x(
/B h7 #.
/
/ / (C+
6 3" (r 2 3
8
$ + R[[+ 6 RWWc * + +
7
7
/
/
3 3 + +s0 D,89D8 E*9E,D5 8E EEc j d}d G fdd }| j t 5 || }|`t |j d d d | j t dj
| 5 || }t |j d d d y # 1 sw Y VxY w# 1 sw Y y xY w)Nsome_file_pathr c e Zd Zd Z fdZy);TestDetectEncoding.test_filename_in_exception.<locals>.Bunkc . || _ || _ d| _ y )Nr )name_lines_index)r@ rA r s r4 __init__zDTestDetectEncoding.test_filename_in_exception.<locals>.Bunk.__init__ s # r6 c | j t k( rt | j }| xj dz
c_ |S r> )r r' r? )r@ r3 rA s r4 r< zDTestDetectEncoding.test_filename_in_exception.<locals>.Bunk.readline s: ;;#e*,'' T[[) q r6 N)r r r r r< )rA s r4 Bunkr s
r6 r z.*{})rN r$ r r r< r] format)r@ r r insrA s @r4 test_filename_in_exceptionz-TestDetectEncoding.test_filename_in_exception s
{
+ *ud#C CLL ) *
#
#K t1D
E *ud#C CLL ) * * * *
* *s !B 5 B) B&)B2c t d }t j d| 5 | j t t
d d d d | j
|j y # 1 sw Y %xY w)Ns #coding:xxxztokenize._builtin_open) return_valuefoobar)r r patchrN r$ r
assertTrueclosed)r@ ms r4 test_open_errorz"TestDetectEncoding.test_open_error sV N #
ZZ0q
A D k=( C D ! D Ds A##A,N)r r r rB rJ rN rR rT rX r^ rb rd ri rm rp r r r r r r r r rH r6 r4 r; r; so : DK@B
3
DB 3 3 3
6 B 1B.E+(*8"r6 r; c < e Zd Zd Zd Zd Zd Zd Zd Zd Z d Z
y )
TestTokenizec F
dd l }d d
fd}
fd}dfd}|j }|j }||_ ||_ t | }| j t | dd g d ||_ ||_ | j
y # ||_ ||_ w xY w)Nr r: c ddgfS )N first secondrH )r< r2 s r4 mock_detect_encodingz8TestTokenize.test_tokenize.<locals>.mock_detect_encoding s h 22 2r6 c l |g } | }|r|j | |S # t $ r |cY S w xY wr )r? r) )r< r2 kwargsout next_line
encoding_useds r4 mock__tokenizez2TestTokenize.test_tokenize.<locals>.mock__tokenize sK $MC (
I JJy)
% J s % 33c L dz
dk( ryt j S )NrM r6 )strr; )counters r4
mock_readlinez1TestTokenize.test_tokenize.<locals>.mock_readline s) qLG!|w<&&( (r6 rM )r r 1 2 3 4)r r r r= rP ) r@ tokenize_moduler r r orig_detect_encoding orig_c_tokenresultsr r2 r s @@@r4
test_tokenizezTestTokenize.test_tokenize s *
3 ) / > > &HH *>'<J9 M }-G T']12 . J
L /CO +@LO = 1 /CO +@LO =s +B B c X g }t d D ]# }|j dj | % |j d dj | }t t t
|j d j }| j |d j d y )Ni zdef i{i}(): return {i})iOK
r: ) ranger) r r rP r r r; r< r= string)r@ bufr tokss r4 test_oneline_defsz TestTokenize.test_oneline_defs s s =A JJ/666; < =
4iin HWSZZ%8 9 B BCD b $/r6 c t t t |j d j }t | }| j
t | d|z | j
t |d j t t t | D ]5 }| j
t ||dz j t || 7 | j
t |d|z j t t j | j
t |d|z j t t j y )Nr: rZ r rM rG )rP r r r; r< r' r= r
exact_typer r r0 r r )r@ opstroptypesrW num_optypesr s r4 assertExactTypeEqualz!TestTokenize.assertExactTypeEqual s hwu||G'< = F FGH'l Va+o6 &)"6"67!(+ -{# 3A XfQUm&>&> ? %gaj 1
3 3
&[ "9"D"DE!%--0 2 &[ "9"D"DE!%//2 4r6 c \ | j dt j t j | j dt j t j
| j dt j | j dt j | j dt j | j dt j | j dt j | j dt j | j d t j | j d
t j | j d t j | j d t j | j d
t j | j dt j" | j dt j$ | j dt j& | j dt j( t j* | j dt j, | j dt j. | j dt j0 | j dt j2 | j dt j4 | j dt j6 | j dt j8 | j dt j: | j dt j< | j dt j> | j d t j@ | j d t jB | j d t jD | j d t jF | j d t jH | j d!t jJ | j d"t jL | j d"t jL | j d#t jN | j d$t jP | j d%t jR | j d&t jT | j d't jV | j d(t jX | j d)t jZ | j d*t j\ | j d+t j^ | j d,t j` | j d-tb t j< td t j tb t j< td t j, tb t j< td | j d.t j( t jd t j t jd t j t jd t j* | j d/t j6 t j t jb t j t jd t j y )0Nz()z[]:,;+rt */|&<>=r %z{}z==z!=z<=z>=~^z<<z>>z**z+=z-=z*=z/=z%=z&=z|=z^=z<<=z>>=z**=z//z//=z:=z...z->@z@=za**2+b**2==c**2z {1, 2, 3}z
^(x & 0x1))3r r0 LPARRPARLSQBRSQBCOLONCOMMASEMIPLUSMINUSSTARSLASHVBARAMPERLESSGREATEREQUALDOTPERCENTLBRACERBRACEEQEQUALNOTEQUAL LESSEQUAL GREATEREQUALTILDE
CIRCUMFLEX LEFTSHIFT
RIGHTSHIFT
DOUBLESTAR PLUSEQUALMINEQUAL STAREQUAL
SLASHEQUAL PERCENTEQUAL
AMPEREQUAL VBAREQUALCIRCUMFLEXEQUALLEFTSHIFTEQUALRIGHTSHIFTEQUALDOUBLESTAREQUAL DOUBLESLASHDOUBLESLASHEQUAL
COLONEQUALELLIPSISRARROWATATEQUALr r rv s r4 test_exact_typez TestTokenize.test_exact_type, sz !!$
EJJ? !!$
EJJ? !!#u{{3 !!#u{{3 !!#uzz2 !!#uzz2 !!#u{{3 !!#uzz2 !!#u{{3 !!#uzz2 !!#u{{3 !!#uzz2 !!#u}}5 !!#u{{3 !!#uyy1 !!#u}}5 !!$ ellC !!$
6 !!$7 !!$8 !!$(:(:; !!#u{{3 !!#u'7'78 !!$8 !!$(8(89 !!$(8(89 !!$8 !!$7 !!$8 !!$(8(89 !!$(:(:; !!$(8(89 !!$8 !!$(=(=> !!$(=(=> !!%)=)=> !!%)>)>? !!%)>)>? !!$(9(9: !!%)?)?@ !!$(8(89 !!% 8 !!$ 5 !!#uxx0 !!$
6 !!"3"&(8(8&"'**"&(8(8&"'--"&(8(8& B
!!+"',,"',, "',, "',,"',, 0
!!,"'"2"2"'**"'**ekk5<<"'** .r6 c D | j dt j y )Nz @ )r r0 r rv s r4 %test_pathological_trailing_whitespacez2TestTokenize.test_pathological_trailing_whitespacem s !!-:r6 c d}t t j dddd t t j dddd t t j d
d d d t t j
d
ddd t t j dddd t t j dddd t t j dddd t t j dddd t t j dddd g }t t t |j d j }| j || y )Nz b = 1
#testr: r r r/ r r1 r2 r3 br4 rM rM zb = 1
r rM rG rM rZ 1rM rM r r rM rG r rG rM z#testrZ r )rZ r )rZ r) )r& r )r r0 r r r r r NLCOMMENTr rP r r r; r< r= r@ sourcer8 rW s r4 5test_comment_at_the_end_of_the_source_without_newlinezBTestTokenize.test_comment_at_the_end_of_the_source_without_newlineq s ! 5>>'VZ\ ] 5::cVR[ \ 588Cv6PY Z 5<<6vT] ^ 5==VV_ ` 588DFQU V 5==FY` a 588Bf&w W 5??2VVX Y
hwv}}W'= > G GHI 1r6 c d}t t j dddd t t j dddd t t j d
dd d t t j
dd d
d t t j dddd g}t t t |j d j }| j || y )Nza
r: r r )r r1 r2 r3 ar4 r! za
r r" r+ )rG rG r% r, )
r r0 r r r r- r rP r r r; r< r= r/ s r4 ?test_newline_and_space_at_the_end_of_the_source_without_newlinezLTestTokenize.test_newline_and_space_at_the_end_of_the_source_without_newline s ennWFUW X ejjFU S emmDFQV W ehhrV# N eoobFQS T
hwv}}W'= > G GHI 1r6 c R d}t j 5 }t j j |d }t |d 5 }|j
| d d d t | \ }}| j d|j d d d y # 1 sw Y <xY w# 1 sw Y y xY w)Ns F"""
"""z script.pywbs SyntaxError)
r temp_dirr r r r writer assertInerr)r@ scriptr7 ra r rsr s r4 (test_invalid_character_in_fstring_middlez5TestTokenize.test_invalid_character_in_fstring_middle s
! 2Xww||Hk:Hh%
#
6"
#(2EB MM."&& 1 2 2
#
# 2 2s# -B B2B B B B&N) r r r r r r r r r1 r4 r= rH r6 r4 r r s, )2V 0 4 ?.B;2$ 2
2r6 r c e Zd Zd Zd Zd Zy)UntokenizeTestc > t }d|_ d|_ | j t 5 }|j d d d d | j
j j d d | j t |j
d y # 1 sw Y TxY w)NrG r# r z'start (1,3) precedes previous end (2,2)r+ ) r prev_rowprev_colrN
ValueErroradd_whitespacer= r_ args)r@ ucms r4 test_bad_input_orderz#UntokenizeTest.test_bad_input_order s M
z
* $b
U # $ **1-9 ;
*a&6&6> $ $s BB c 6 t }d|_ d|_ g |_ |j d | j |j dg d|_ |j d | j |j g d t j | d y )NrM r* \
rG )r& r& )rJ z\
\
r# za
b
c
\
c
)r rA rB rW rD r= r r )r@ rF s r4 test_backslash_continuationz*UntokenizeTest.test_backslash_continuation s} M
F8,
#?@%%d,HIr6 c t }t df}t df|g}|j |t g | j |j dg t }| j |j t |g d t }| j |j t | d | j |j d | j t t | d y )NHellor: zHello s Hello ) r r r compatiterr= rW r r2 )r@ rF r0 rW s r4 test_iter_compatz UntokenizeTest.test_iter_compat s MwW%u- R! H:. M dE7m4h? M d6l3X> W- DL19=r6 N)r r r rH rK rP rH r6 r4 r? r? s
? J >r6 r? c B e Zd Zd Zd Zd Zd Zd Zd Zd Z d Z
d Z y
) r c t |t r|j d }n|j }t |j d j }t t | }|D cg c] }|dd }}t | }t |j d j }t | D cg c] }|dd } }| j | | t | }
t |
j d j } t | D cg c] }|dd } }| j | | yc c}w c c}w c c}w )a)
Test roundtrip for `untokenize`. `f` is an open file or a string.
The source code in f is tokenized to both 5- and 2-tuples.
Both sequences are converted back to source code via
tokenize.untokenize(), and the latter tokenized again to 2-tuples.
The test fails if the 3 pair tokenizations do not match.
When untokenize bugs are fixed, untokenize with 5-tuples should
reproduce code that does not contain a backslash continuation
following spaces. A proper test should test this.
r: TkeependsNrG )
isinstancer r; readrO r( rQ rP r r r= )
r@ rC coder< tokens5ri tokens2 bytes_from2 readline2
tokens2_from2 bytes_from5 readline5
tokens2_from5s
r4 r z TestRoundtrip.check_roundtrip s' a 88G$D668D67@@x)*&-.s3r7.. ) ///>?HH ,4Y,?@SRa@
@ 0 ) ///>?HH ,4Y,?@SRa@
@ 0 / A
As 0 D9; D> Ec t |t r|j d }n|j }t |j d j }t | D ]^ }|j t t hv r | j |j |j |j d |j d ` y )Nr: TrS rM )rU r r; rV rO r( rQ r r/ r r r= r r3 r1 r2 )r@ rC rW r< ri s r4 check_line_extractionz#TestRoundtrip.check_line_extraction s a 88G$D668D67@@H% MCxxXy 11 SZZ#))A,
)K L Mr6 c | j d | j d | j d t j d }t |d 5 }| j | d d d | j d | j d | j d | j d y # 1 sw Y NxY w)
Nzif x == 1:
print(x)
z # This is a comment
# This also
zif x == 1 :
print(x)
tokenize_tests.txtr zoif x == 1:
# A comment by itself.
print(x) # Comment here, too.
# Another comment.
after_if = True
zPif (x # The comments need to go in the right place
== 1):
print('x==1')
zclass Test: # A comment here
# A comment with weird indent
after_com = 5
def x(m): return m*5 # a one liner
def y(m): # A whitespace after the colon
return y*4 # 3-space indent
zttry: import somemodule
except ImportError: # comment
print('Can not import' # comment2
)else: print('Loaded')
)r r findfiler )r@ fnrC s r4 test_roundtripz TestRoundtrip.test_roundtrip s
. / - .
, -
2
3
"d^ $q # $ 1 2
3 4
B C
9 :% $ $s B44B=c & | j d y )Nzna = (3,4,
5,6)
y = [3, 4,
5]
z = {'a': 5,
'b':15, 'c':True}
x = len(y) + 5 - a[
3] - a[2]
+ len(z) - z[
'b']
r rv s r4 test_continuationz TestRoundtrip.test_continuation s & 'r6 c H | j d | j d y )Nz)x=1+\
1
# This is a comment\
# This also
z# Comment \
x = 0rh rv s r4 rK z)TestRoundtrip.test_backslash_continuation' s* - .
% &r6 c & | j d y )Nz'' ''rh rv s r4 test_string_concatenationz'TestRoundtrip.test_string_concatenation0 s W%r6 c dd l }dd l}t j d }t j
j
| xs t j }|j t j
j |j | d }|j t j
j |d |j t j
j |d dD ]4 }|j t j
j |d |z 6 t j d s|j |d }|D ]s }t j d
k\ r t d | t |d 5 }| j! |
5 | j# | | j% | d d d d d d u y # 1 sw Y xY w# 1 sw Y xY w)Nr rc ztest*.pyztest_unicode_identifiers.pyztest_fstring.py)bufferbuiltinfileioinspectr platformsysz
test_%s.pycpu
rG r r r )globrandomr rd r r r curdirr escaperemoveis_resource_enabledsampleverboser r subTestr ra )r@ rv rw re tempdir testfilesrC testfiles r4 test_random_filesz TestRoundtrip.test_random_files4 sv
2
3''//"%2IIbggll4;;w+? LM g/LMN g/@ABT FA RWW\\'< @1 D E F **51
i4I! 2H!#j(+h%
2\\x\0 2((+..q12
2
2 22 2
2
2s$ ;G#G1GG
GG c t |t r|j d }t t t | j j d S )Nr: )rU r r; r r r r< r )r@ rW s r4 roundtripzTestRoundtrip.roundtripV s? dC ;;w'D(74=#9#9:;BB7KKr6 c d}| j | j d }| j |d |d | j | y)z
Ensure that although whitespace might be mutated in a roundtrip,
the semantic meaning of the indentation remains consistent.
zif False:
x=3
x=3
r rM rG N)r splitr= r )r@ rW codeliness r4 #test_indentation_semantics_retainedz1TestRoundtrip.test_indentation_semantics_retained[ sH
+NN4(..t4 1 y|4 T"r6 N) r r r r ra rf ri rK rl r r r rH r6 r4 r r s3 1> M%:N '&& 2DL
#r6 r c e Zd Zd Zd Zy)InvalidPythonTestsc
d}t t j dddd t t j dddd t t j ddd d t t j d
d d d t t j d d d
d t t j
dd
dd t t j dddd g}t t t | j }| j || y )Nz2sin(x)2r4 r! r sinr% r r' rU r( r r5 r )rM r* ) r r0 r r r r r rP r r r< r= r/ s r4 test_number_followed_by_namez/InvalidPythonTests.test_number_followed_by_nameg s 5<<6vT] ^ 5::e6vT] ^ 588Cv6PY Z 5::cVR[ \ 588Cv6PY Z 5==6vT] ^ 5??2VVX Y
ohv&6&?&?@A 1r6 c , d}t t j dddd t t j dddd t t j dddd g}t t
t | j }| j || y )N01234r4 r' r r r( r* )
r r0 r r r rP r r r< r= r/ s r4 test_number_starting_with_zeroz1InvalidPythonTests.test_number_starting_with_zerow sy 5<<v6X_ ` 5==6vT[ \ 5??2VVX Y
ohv&6&?&?@A 1r6 N)r r r r r rH r6 r4 r r f s 2 2r6 r c x e Zd Zd Zd Zd Zd Zd Zd Zd Z d Z
d Z d
Z d Z
d Zd
Zd Zd Zd Zd Zd Zy)
CTokenizeTestc t | }| j | 5 t t |j | }| j ||j
j d d d y # 1 sw Y y xY w)N)r0 )r r~ r5 r r< r= r> r( r? s r4 rD z CTokenizeTest.check_tokenize sn
QK
\\\
# E11!**=qF
VX__%6%A%A%C D E E Es AA55A>c d }t t dddd t t dddd t t ddd d t t d
d d d t t d
d d d
g}d
D ]V }| j | 5 t
t || j d| }| j || d d d X y # 1 sw Y cxY w)Nc 3 2 K dj | y w)N1+1)r; r s r4 r< z-CTokenizeTest.test_encoding.<locals>.readline s ,,x( (s r$ r4 r! r r r r" r# r r% r* )r: rr zutf-16r T)r3 r2 )
r r r r r r~ rP r rQ r= )r@ r< rB r2 rW s r4
test_encodingzCTokenizeTest.test_encoding s )
6#Ve T 2cV% P 6#Ve T 72Ve T 9Rv6PR S
7 3H x 0
3 ? X&//!% %
2
3
3 3
3
3s <:CC
c H | j dd | j dd | j dd | j dd | j d d
| j d d | j d
d | j dd | j dd y )Nrl z NUMBER '0xff' (1, 0) (1, 4)
LESSEQUAL '<=' (1, 5) (1, 7)
NUMBER '255' (1, 8) (1, 11)
rm z NUMBER '0b10' (1, 0) (1, 4)
LESSEQUAL '<=' (1, 5) (1, 7)
NUMBER '255' (1, 8) (1, 11)
rn z NUMBER '0o123' (1, 0) (1, 5)
LESSEQUAL '<=' (1, 6) (1, 8)
NUMBER '0O123' (1, 9) (1, 14)
ro z NUMBER '1234567' (1, 0) (1, 7)
GREATER '>' (1, 8) (1, 9)
TILDE '~' (1, 10) (1, 11)
NUMBER '0x15' (1, 11) (1, 15)
rp z NUMBER '2134568' (1, 0) (1, 7)
NOTEQUAL '!=' (1, 8) (1, 10)
NUMBER '1231515' (1, 11) (1, 18)
rq ac LPAR '(' (1, 0) (1, 1)
MINUS '-' (1, 1) (1, 2)
NUMBER '124561' (1, 2) (1, 8)
MINUS '-' (1, 8) (1, 9)
NUMBER '1' (1, 9) (1, 10)
RPAR ')' (1, 10) (1, 11)
AMPER '&' (1, 12) (1, 13)
NUMBER '200000000' (1, 14) (1, 23)
rr z NUMBER '0xdeadbeef' (1, 0) (1, 10)
NOTEQUAL '!=' (1, 11) (1, 13)
MINUS '-' (1, 14) (1, 15)
NUMBER '1' (1, 15) (1, 16)
rs z NUMBER '0xdeadc0de' (1, 0) (1, 10)
AMPER '&' (1, 11) (1, 12)
NUMBER '12345' (1, 13) (1, 18)
rt z NUMBER '0xFF' (1, 0) (1, 4)
AMPER '&' (1, 5) (1, 6)
NUMBER '0x15' (1, 7) (1, 11)
VBAR '|' (1, 12) (1, 13)
NUMBER '1234' (1, 14) (1, 18)
ru rv s r4 rw zCTokenizeTest.test_int s M ,
M ,
, /
- 0
0 3
5 8
. 1
0 3
0 3 r6 c | j dd | j dd | j dd | j dd | j d d
| j d d | j d
d y )Nr{ z NAME 'x' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
NUMBER '3.14159' (1, 4) (1, 11)
r| z NAME 'x' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
NUMBER '314159.' (1, 4) (1, 11)
r} z NAME 'x' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
NUMBER '.314159' (1, 4) (1, 11)
r~ z NAME 'x' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
NUMBER '3e14159' (1, 4) (1, 11)
r z NAME 'x' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
NUMBER '3E123' (1, 4) (1, 9)
r z NAME 'x' (1, 0) (1, 1)
PLUS '+' (1, 1) (1, 2)
NAME 'y' (1, 2) (1, 3)
EQUAL '=' (1, 4) (1, 5)
NUMBER '3e-1230' (1, 6) (1, 13)
r z NAME 'x' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
NUMBER '3.14e159' (1, 4) (1, 12)
ru rv s r4 r zCTokenizeTest.test_float s M ,
M ,
M ,
M ,
K *
O .
N - r6 c | j dd | j dd | j dd | j dd | j d d
| j d d | j d
d | j dd | j dd | j dd | j dd | j dd | j dd | j dd | j d d | j d d | j d!d" | j d#d$ | j d%d& | j d'd( | j d)d* | j d+d, | j d-d. | j d/d0 | j d1d2 | j d3d4 | j d5d6 | j d7d8 | j d9d: y );Nr a5 NAME 'x' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
STRING "''" (1, 4) (1, 6)
SEMI ';' (1, 6) (1, 7)
NAME 'y' (1, 8) (1, 9)
EQUAL '=' (1, 10) (1, 11)
STRING '""' (1, 12) (1, 14)
r a6 NAME 'x' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
STRING '\'"\'' (1, 4) (1, 7)
SEMI ';' (1, 7) (1, 8)
NAME 'y' (1, 9) (1, 10)
EQUAL '=' (1, 11) (1, 12)
STRING '"\'"' (1, 13) (1, 16)
r z NAME 'x' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
STRING '"doesn\'t "' (1, 4) (1, 14)
NAME 'shrink' (1, 14) (1, 20)
STRING '", does it"' (1, 20) (1, 31)
r z NAME 'x' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
STRING "'abc'" (1, 4) (1, 9)
PLUS '+' (1, 10) (1, 11)
STRING "'ABC'" (1, 12) (1, 17)
r z NAME 'y' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
STRING '"ABC"' (1, 4) (1, 9)
PLUS '+' (1, 10) (1, 11)
STRING '"ABC"' (1, 12) (1, 17)
r a NAME 'x' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
STRING "r'abc'" (1, 4) (1, 10)
PLUS '+' (1, 11) (1, 12)
STRING "r'ABC'" (1, 13) (1, 19)
PLUS '+' (1, 20) (1, 21)
STRING "R'ABC'" (1, 22) (1, 28)
PLUS '+' (1, 29) (1, 30)
STRING "R'ABC'" (1, 31) (1, 37)
r a NAME 'y' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
STRING 'r"abc"' (1, 4) (1, 10)
PLUS '+' (1, 11) (1, 12)
STRING 'r"ABC"' (1, 13) (1, 19)
PLUS '+' (1, 20) (1, 21)
STRING 'R"ABC"' (1, 22) (1, 28)
PLUS '+' (1, 29) (1, 30)
STRING 'R"ABC"' (1, 31) (1, 37)
r z STRING "u'abc'" (1, 0) (1, 6)
PLUS '+' (1, 7) (1, 8)
STRING "U'abc'" (1, 9) (1, 15)
r z STRING 'u"abc"' (1, 0) (1, 6)
PLUS '+' (1, 7) (1, 8)
STRING 'U"abc"' (1, 9) (1, 15)
r z STRING "b'abc'" (1, 0) (1, 6)
PLUS '+' (1, 7) (1, 8)
STRING "B'abc'" (1, 9) (1, 15)
r z STRING 'b"abc"' (1, 0) (1, 6)
PLUS '+' (1, 7) (1, 8)
STRING 'B"abc"' (1, 9) (1, 15)
r a; STRING "br'abc'" (1, 0) (1, 7)
PLUS '+' (1, 8) (1, 9)
STRING "bR'abc'" (1, 10) (1, 17)
PLUS '+' (1, 18) (1, 19)
STRING "Br'abc'" (1, 20) (1, 27)
PLUS '+' (1, 28) (1, 29)
STRING "BR'abc'" (1, 30) (1, 37)
r a; STRING 'br"abc"' (1, 0) (1, 7)
PLUS '+' (1, 8) (1, 9)
STRING 'bR"abc"' (1, 10) (1, 17)
PLUS '+' (1, 18) (1, 19)
STRING 'Br"abc"' (1, 20) (1, 27)
PLUS '+' (1, 28) (1, 29)
STRING 'BR"abc"' (1, 30) (1, 37)
r a; STRING "rb'abc'" (1, 0) (1, 7)
PLUS '+' (1, 8) (1, 9)
STRING "rB'abc'" (1, 10) (1, 17)
PLUS '+' (1, 18) (1, 19)
STRING "Rb'abc'" (1, 20) (1, 27)
PLUS '+' (1, 28) (1, 29)
STRING "RB'abc'" (1, 30) (1, 37)
r a; STRING 'rb"abc"' (1, 0) (1, 7)
PLUS '+' (1, 8) (1, 9)
STRING 'rB"abc"' (1, 10) (1, 17)
PLUS '+' (1, 18) (1, 19)
STRING 'Rb"abc"' (1, 20) (1, 27)
PLUS '+' (1, 28) (1, 29)
STRING 'RB"abc"' (1, 30) (1, 37)
r r r r r r r r r r r r z"hola\\\r\ndfgf"z9 STRING '"hola\\\\\\r\\ndfgf"' (1, 0) (1, 16)
r r r a= FSTRING_START 'fR"' (1, 0) (1, 3)
FSTRING_MIDDLE 'a' (1, 3) (1, 4)
LBRACE '{' (1, 4) (1, 5)
NAME 'b' (1, 5) (1, 6)
RBRACE '}' (1, 6) (1, 7)
FSTRING_MIDDLE 'c' (1, 7) (1, 8)
FSTRING_END '"' (1, 8) (1, 9)
r r r r zf"{
a}"z FSTRING_START 'f"' (1, 0) (1, 2)
LBRACE '{' (1, 2) (1, 3)
NAME 'a' (2, 0) (2, 1)
RBRACE '}' (2, 1) (2, 2)
FSTRING_END '"' (2, 2) (2, 3)
r r zf"hola\\\r\ndfgf"z FSTRING_START 'f"' (1, 0) (1, 2)
FSTRING_MIDDLE 'hola\\\\\\r\\ndfgf' (1, 2) (1, 16)
FSTRING_END '"' (1, 16) (1, 17)
ru rv s r4 r zCTokenizeTest.test_string s . 1
1 4
> A
/ 2
/ 2
C
F
C
F
- 0
- 0
- 0
- 0
C F
C F
C F
C F
- 0
L +
L +
/ 2
H '
K *
L +
0 3 r6 c p | j dd | j dd | j dd y )Nr a NAME 'def' (1, 0) (1, 3)
NAME 'd22' (1, 4) (1, 7)
LPAR '(' (1, 7) (1, 8)
NAME 'a' (1, 8) (1, 9)
COMMA ',' (1, 9) (1, 10)
NAME 'b' (1, 11) (1, 12)
COMMA ',' (1, 12) (1, 13)
NAME 'c' (1, 14) (1, 15)
EQUAL '=' (1, 15) (1, 16)
NUMBER '2' (1, 16) (1, 17)
COMMA ',' (1, 17) (1, 18)
NAME 'd' (1, 19) (1, 20)
EQUAL '=' (1, 20) (1, 21)
NUMBER '2' (1, 21) (1, 22)
COMMA ',' (1, 22) (1, 23)
STAR '*' (1, 24) (1, 25)
NAME 'k' (1, 25) (1, 26)
RPAR ')' (1, 26) (1, 27)
COLON ':' (1, 27) (1, 28)
NAME 'pass' (1, 29) (1, 33)
r a NAME 'def' (1, 0) (1, 3)
NAME 'd01v_' (1, 4) (1, 9)
LPAR '(' (1, 9) (1, 10)
NAME 'a' (1, 10) (1, 11)
EQUAL '=' (1, 11) (1, 12)
NUMBER '1' (1, 12) (1, 13)
COMMA ',' (1, 13) (1, 14)
STAR '*' (1, 15) (1, 16)
NAME 'k' (1, 16) (1, 17)
COMMA ',' (1, 17) (1, 18)
DOUBLESTAR '**' (1, 19) (1, 21)
NAME 'w' (1, 21) (1, 22)
RPAR ')' (1, 22) (1, 23)
COLON ':' (1, 23) (1, 24)
NAME 'pass' (1, 25) (1, 29)
r a NAME 'def' (1, 0) (1, 3)
NAME 'd23' (1, 4) (1, 7)
LPAR '(' (1, 7) (1, 8)
NAME 'a' (1, 8) (1, 9)
COLON ':' (1, 9) (1, 10)
NAME 'str' (1, 11) (1, 14)
COMMA ',' (1, 14) (1, 15)
NAME 'b' (1, 16) (1, 17)
COLON ':' (1, 17) (1, 18)
NAME 'int' (1, 19) (1, 22)
EQUAL '=' (1, 22) (1, 23)
NUMBER '3' (1, 23) (1, 24)
RPAR ')' (1, 24) (1, 25)
RARROW '->' (1, 26) (1, 28)
NAME 'int' (1, 29) (1, 32)
COLON ':' (1, 32) (1, 33)
NAME 'pass' (1, 34) (1, 38)
ru rv s r4 r zCTokenizeTest.test_function sJ ? B .
; > $
D G r6 c ( | j dd y )Nr a NAME 'if' (1, 0) (1, 2)
NUMBER '1' (1, 3) (1, 4)
LESS '<' (1, 5) (1, 6)
NUMBER '1' (1, 7) (1, 8)
GREATER '>' (1, 9) (1, 10)
NUMBER '1' (1, 11) (1, 12)
EQEQUAL '==' (1, 13) (1, 15)
NUMBER '1' (1, 16) (1, 17)
GREATEREQUAL '>=' (1, 18) (1, 20)
NUMBER '5' (1, 21) (1, 22)
LESSEQUAL '<=' (1, 23) (1, 25)
NUMBER '0x15' (1, 26) (1, 30)
LESSEQUAL '<=' (1, 31) (1, 33)
NUMBER '0x12' (1, 34) (1, 38)
NOTEQUAL '!=' (1, 39) (1, 41)
NUMBER '1' (1, 42) (1, 43)
NAME 'and' (1, 44) (1, 47)
NUMBER '5' (1, 48) (1, 49)
NAME 'in' (1, 50) (1, 52)
NUMBER '1' (1, 53) (1, 54)
NAME 'not' (1, 55) (1, 58)
NAME 'in' (1, 59) (1, 61)
NUMBER '1' (1, 62) (1, 63)
NAME 'is' (1, 64) (1, 66)
NUMBER '1' (1, 67) (1, 68)
NAME 'or' (1, 69) (1, 71)
NUMBER '5' (1, 72) (1, 73)
NAME 'is' (1, 74) (1, 76)
NAME 'not' (1, 77) (1, 80)
NUMBER '1' (1, 81) (1, 82)
COLON ':' (1, 82) (1, 83)
NAME 'pass' (1, 84) (1, 88)
ru rv s r4 r z CTokenizeTest.test_comparison' r r6 c ( | j dd y )Nr a$ NAME 'x' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
NUMBER '1' (1, 4) (1, 5)
MINUS '-' (1, 6) (1, 7)
NAME 'y' (1, 8) (1, 9)
PLUS '+' (1, 10) (1, 11)
NUMBER '15' (1, 12) (1, 14)
MINUS '-' (1, 15) (1, 16)
NUMBER '1' (1, 17) (1, 18)
PLUS '+' (1, 19) (1, 20)
NUMBER '0x124' (1, 21) (1, 26)
PLUS '+' (1, 27) (1, 28)
NAME 'z' (1, 29) (1, 30)
PLUS '+' (1, 31) (1, 32)
NAME 'a' (1, 33) (1, 34)
LSQB '[' (1, 34) (1, 35)
NUMBER '5' (1, 35) (1, 36)
RSQB ']' (1, 36) (1, 37)
ru rv s r4 r zCTokenizeTest.test_additiveM r r6 c ( | j dd y )Nr a NAME 'x' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
NUMBER '1' (1, 4) (1, 5)
DOUBLESLASH '//' (1, 5) (1, 7)
NUMBER '1' (1, 7) (1, 8)
STAR '*' (1, 8) (1, 9)
NUMBER '1' (1, 9) (1, 10)
SLASH '/' (1, 10) (1, 11)
NUMBER '5' (1, 11) (1, 12)
STAR '*' (1, 12) (1, 13)
NUMBER '12' (1, 13) (1, 15)
PERCENT '%' (1, 15) (1, 16)
NUMBER '0x12' (1, 16) (1, 20)
AT '@' (1, 20) (1, 21)
NUMBER '42' (1, 21) (1, 23)
ru rv s r4 r z!CTokenizeTest.test_multiplicatived r r6 c L | j dd | j dd y )Nr a TILDE '~' (1, 0) (1, 1)
NUMBER '1' (1, 1) (1, 2)
CIRCUMFLEX '^' (1, 3) (1, 4)
NUMBER '1' (1, 5) (1, 6)
AMPER '&' (1, 7) (1, 8)
NUMBER '1' (1, 9) (1, 10)
VBAR '|' (1, 11) (1, 12)
NUMBER '1' (1, 12) (1, 13)
CIRCUMFLEX '^' (1, 14) (1, 15)
MINUS '-' (1, 16) (1, 17)
NUMBER '1' (1, 17) (1, 18)
r aI MINUS '-' (1, 0) (1, 1)
NUMBER '1' (1, 1) (1, 2)
STAR '*' (1, 2) (1, 3)
NUMBER '1' (1, 3) (1, 4)
SLASH '/' (1, 4) (1, 5)
NUMBER '1' (1, 5) (1, 6)
PLUS '+' (1, 6) (1, 7)
NUMBER '1' (1, 7) (1, 8)
STAR '*' (1, 8) (1, 9)
NUMBER '1' (1, 9) (1, 10)
DOUBLESLASH '//' (1, 10) (1, 12)
NUMBER '1' (1, 12) (1, 13)
MINUS '-' (1, 14) (1, 15)
MINUS '-' (1, 16) (1, 17)
MINUS '-' (1, 17) (1, 18)
MINUS '-' (1, 18) (1, 19)
NUMBER '1' (1, 19) (1, 20)
DOUBLESTAR '**' (1, 20) (1, 22)
NUMBER '1' (1, 22) (1, 23)
ru rv s r4 r zCTokenizeTest.test_unaryx s0 0 3
5 8 r6 c ( | j dd y )Nr a NAME 'import' (1, 0) (1, 6)
NAME 'sys' (1, 7) (1, 10)
COMMA ',' (1, 10) (1, 11)
NAME 'time' (1, 12) (1, 16)
NEWLINE '' (1, 16) (1, 16)
NAME 'x' (2, 0) (2, 1)
EQUAL '=' (2, 2) (2, 3)
NAME 'sys' (2, 4) (2, 7)
DOT '.' (2, 7) (2, 8)
NAME 'modules' (2, 8) (2, 15)
LSQB '[' (2, 15) (2, 16)
STRING "'time'" (2, 16) (2, 22)
RSQB ']' (2, 22) (2, 23)
DOT '.' (2, 23) (2, 24)
NAME 'time' (2, 24) (2, 28)
LPAR '(' (2, 28) (2, 29)
RPAR ')' (2, 29) (2, 30)
ru rv s r4 r zCTokenizeTest.test_selector r r6 c ( | j dd y Nr a AT '@' (1, 0) (1, 1)
NAME 'staticmethod' (1, 1) (1, 13)
NEWLINE '' (1, 13) (1, 13)
NAME 'def' (2, 0) (2, 3)
NAME 'foo' (2, 4) (2, 7)
LPAR '(' (2, 7) (2, 8)
NAME 'x' (2, 8) (2, 9)
COMMA ',' (2, 9) (2, 10)
NAME 'y' (2, 10) (2, 11)
RPAR ')' (2, 11) (2, 12)
COLON ':' (2, 12) (2, 13)
NAME 'pass' (2, 14) (2, 18)
ru rv s r4 r zCTokenizeTest.test_method r r6 c ( | j dd y r ru rv s r4 r zCTokenizeTest.test_tabs r r6 c | j dd | j dd | j dd | j dd | j d d
| j d d | j d
d | j dd | j dd | j dd | j dd | j dd | j dd | j dd | j d d | j d d | j d!d" | j d#d$ | j d%d& | j d'd( | j d)d* y )+Nr z ASYNC 'async' (1, 0) (1, 5)
EQUAL '=' (1, 6) (1, 7)
NUMBER '1' (1, 8) (1, 9)
r a8 NAME 'a' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
LPAR '(' (1, 4) (1, 5)
ASYNC 'async' (1, 5) (1, 10)
EQUAL '=' (1, 11) (1, 12)
NUMBER '1' (1, 13) (1, 14)
RPAR ')' (1, 14) (1, 15)
r z ASYNC 'async' (1, 0) (1, 5)
LPAR '(' (1, 5) (1, 6)
RPAR ')' (1, 6) (1, 7)
r a< NAME 'class' (1, 0) (1, 5)
ASYNC 'async' (1, 6) (1, 11)
LPAR '(' (1, 11) (1, 12)
NAME 'Bar' (1, 12) (1, 15)
RPAR ')' (1, 15) (1, 16)
COLON ':' (1, 16) (1, 17)
NAME 'pass' (1, 17) (1, 21)
r z NAME 'class' (1, 0) (1, 5)
ASYNC 'async' (1, 6) (1, 11)
COLON ':' (1, 11) (1, 12)
NAME 'pass' (1, 12) (1, 16)
r z AWAIT 'await' (1, 0) (1, 5)
EQUAL '=' (1, 6) (1, 7)
NUMBER '1' (1, 8) (1, 9)
r z NAME 'foo' (1, 0) (1, 3)
DOT '.' (1, 3) (1, 4)
ASYNC 'async' (1, 4) (1, 9)
r a; ASYNC 'async' (1, 0) (1, 5)
NAME 'for' (1, 6) (1, 9)
NAME 'a' (1, 10) (1, 11)
NAME 'in' (1, 12) (1, 14)
NAME 'b' (1, 15) (1, 16)
COLON ':' (1, 16) (1, 17)
NAME 'pass' (1, 18) (1, 22)
r a< ASYNC 'async' (1, 0) (1, 5)
NAME 'with' (1, 6) (1, 10)
NAME 'a' (1, 11) (1, 12)
NAME 'as' (1, 13) (1, 15)
NAME 'b' (1, 16) (1, 17)
COLON ':' (1, 17) (1, 18)
NAME 'pass' (1, 19) (1, 23)
r z ASYNC 'async' (1, 0) (1, 5)
DOT '.' (1, 5) (1, 6)
NAME 'foo' (1, 6) (1, 9)
r z/ ASYNC 'async' (1, 0) (1, 5)
r z ASYNC 'async' (1, 0) (1, 5)
NEWLINE '' (1, 5) (1, 5)
AWAIT 'await' (3, 0) (3, 5)
r z ASYNC 'async' (1, 0) (1, 5)
NEWLINE '' (1, 5) (1, 5)
ELLIPSIS '...' (2, 0) (2, 3)
NEWLINE '' (2, 3) (2, 3)
AWAIT 'await' (3, 0) (3, 5)
r z ASYNC 'async' (1, 0) (1, 5)
NEWLINE '' (1, 5) (1, 5)
AWAIT 'await' (2, 0) (2, 5)
r z NAME 'foo' (1, 0) (1, 3)
DOT '.' (1, 3) (1, 4)
ASYNC 'async' (1, 4) (1, 9)
PLUS '+' (1, 10) (1, 11)
NUMBER '1' (1, 12) (1, 13)
r a; ASYNC 'async' (1, 0) (1, 5)
NAME 'def' (1, 6) (1, 9)
NAME 'foo' (1, 10) (1, 13)
LPAR '(' (1, 13) (1, 14)
RPAR ')' (1, 14) (1, 15)
COLON ':' (1, 15) (1, 16)
NAME 'pass' (1, 17) (1, 21)
r a ASYNC 'async' (1, 0) (1, 5)
NAME 'def' (1, 6) (1, 9)
NAME 'foo' (1, 10) (1, 13)
LPAR '(' (1, 13) (1, 14)
RPAR ')' (1, 14) (1, 15)
COLON ':' (1, 15) (1, 16)
NEWLINE '' (1, 16) (1, 16)
INDENT '' (2, -1) (2, -1)
NAME 'def' (2, 2) (2, 5)
NAME 'foo' (2, 6) (2, 9)
LPAR '(' (2, 9) (2, 10)
AWAIT 'await' (2, 10) (2, 15)
RPAR ')' (2, 15) (2, 16)
COLON ':' (2, 16) (2, 17)
NEWLINE '' (2, 17) (2, 17)
INDENT '' (3, -1) (3, -1)
AWAIT 'await' (3, 4) (3, 9)
EQUAL '=' (3, 10) (3, 11)
NUMBER '1' (3, 12) (3, 13)
NEWLINE '' (3, 13) (3, 13)
DEDENT '' (4, -1) (4, -1)
NAME 'if' (4, 2) (4, 4)
NUMBER '1' (4, 5) (4, 6)
COLON ':' (4, 6) (4, 7)
NEWLINE '' (4, 7) (4, 7)
INDENT '' (5, -1) (5, -1)
AWAIT 'await' (5, 4) (5, 9)
NEWLINE '' (5, 9) (5, 9)
DEDENT '' (6, -1) (6, -1)
DEDENT '' (6, -1) (6, -1)
ASYNC 'async' (6, 0) (6, 5)
PLUSEQUAL '+=' (6, 6) (6, 8)
NUMBER '1' (6, 9) (6, 10)
NEWLINE '' (6, 10) (6, 10)
r a ASYNC 'async' (1, 0) (1, 5)
NAME 'def' (1, 6) (1, 9)
NAME 'foo' (1, 10) (1, 13)
LPAR '(' (1, 13) (1, 14)
RPAR ')' (1, 14) (1, 15)
COLON ':' (1, 15) (1, 16)
NEWLINE '' (1, 16) (1, 16)
INDENT '' (2, -1) (2, -1)
ASYNC 'async' (2, 2) (2, 7)
NAME 'for' (2, 8) (2, 11)
NAME 'i' (2, 12) (2, 13)
NAME 'in' (2, 14) (2, 16)
NUMBER '1' (2, 17) (2, 18)
COLON ':' (2, 18) (2, 19)
NAME 'pass' (2, 20) (2, 24)
DEDENT '' (2, -1) (2, -1)
r ah ASYNC 'async' (1, 0) (1, 5)
NAME 'def' (1, 6) (1, 9)
NAME 'foo' (1, 10) (1, 13)
LPAR '(' (1, 13) (1, 14)
ASYNC 'async' (1, 14) (1, 19)
RPAR ')' (1, 19) (1, 20)
COLON ':' (1, 20) (1, 21)
AWAIT 'await' (1, 22) (1, 27)
r a~ NAME 'def' (1, 0) (1, 3)
NAME 'f' (1, 4) (1, 5)
LPAR '(' (1, 5) (1, 6)
RPAR ')' (1, 6) (1, 7)
COLON ':' (1, 7) (1, 8)
NEWLINE '' (1, 8) (1, 8)
INDENT '' (3, -1) (3, -1)
NAME 'def' (3, 2) (3, 5)
NAME 'baz' (3, 6) (3, 9)
LPAR '(' (3, 9) (3, 10)
RPAR ')' (3, 10) (3, 11)
COLON ':' (3, 11) (3, 12)
NAME 'pass' (3, 13) (3, 17)
NEWLINE '' (3, 17) (3, 17)
ASYNC 'async' (4, 2) (4, 7)
NAME 'def' (4, 8) (4, 11)
NAME 'bar' (4, 12) (4, 15)
LPAR '(' (4, 15) (4, 16)
RPAR ')' (4, 16) (4, 17)
COLON ':' (4, 17) (4, 18)
NAME 'pass' (4, 19) (4, 23)
NEWLINE '' (4, 23) (4, 23)
AWAIT 'await' (6, 2) (6, 7)
EQUAL '=' (6, 8) (6, 9)
NUMBER '2' (6, 10) (6, 11)
DEDENT '' (6, -1) (6, -1)
r a ASYNC 'async' (1, 0) (1, 5)
NAME 'def' (1, 6) (1, 9)
NAME 'f' (1, 10) (1, 11)
LPAR '(' (1, 11) (1, 12)
RPAR ')' (1, 12) (1, 13)
COLON ':' (1, 13) (1, 14)
NEWLINE '' (1, 14) (1, 14)
INDENT '' (3, -1) (3, -1)
NAME 'def' (3, 2) (3, 5)
NAME 'baz' (3, 6) (3, 9)
LPAR '(' (3, 9) (3, 10)
RPAR ')' (3, 10) (3, 11)
COLON ':' (3, 11) (3, 12)
NAME 'pass' (3, 13) (3, 17)
NEWLINE '' (3, 17) (3, 17)
ASYNC 'async' (4, 2) (4, 7)
NAME 'def' (4, 8) (4, 11)
NAME 'bar' (4, 12) (4, 15)
LPAR '(' (4, 15) (4, 16)
RPAR ')' (4, 16) (4, 17)
COLON ':' (4, 17) (4, 18)
NAME 'pass' (4, 19) (4, 23)
NEWLINE '' (4, 23) (4, 23)
AWAIT 'await' (6, 2) (6, 7)
EQUAL '=' (6, 8) (6, 9)
NUMBER '2' (6, 10) (6, 11)
DEDENT '' (6, -1) (6, -1)
ru rv s r4 r zCTokenizeTest.test_async s K *
- 0
I (
3 6
. 1
K *
K *
4 7
5 8
K *
G &
4 7
/ 2
N -
O .
3 6
#* X
H K &
9 <
! F
" r6 c ( | j dd y )Nr u7 NAME 'Örter' (1, 0) (1, 5)
EQUAL '=' (1, 6) (1, 7)
STRING "u'places'" (1, 8) (1, 17)
NEWLINE '' (1, 17) (1, 17)
NAME 'grün' (2, 0) (2, 4)
EQUAL '=' (2, 5) (2, 6)
STRING "U'green'" (2, 7) (2, 15)
ru rv s r4 r zCTokenizeTest.test_unicode
r r6 c d }dD ]4 }| j | 5 | j t || d d d 6 y # 1 sw Y AxY w)Nc T t | }t t |j S r )r rP r r< )r
the_strings r4
get_tokensz5CTokenizeTest.test_invalid_syntax.<locals>.get_tokens
s# !&)J9*:M:MNO Or6 ) z(1+2]z(1+2}z{1+2]1_1.2_1e2_1e+ u €0b120b1_20b20b1_0b0o180o1_80o80o1_0o0x1_0xr 012r r r z'sdfsdfz '''sdfsdf''a ((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((a))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))])case)r~ rN r )r@ r r s r4 test_invalid_syntaxz!CTokenizeTest.test_invalid_syntax
s_ P
" @DB 4 (
@!!*j$?
@
@C" @B
@
@s >A c d}d } ||dz
}t | t t j }| j |d j
t | j |d j
t t |dd || }t | | j t fd | j t t |dd y ) Nd c ^ dj d t | D }|d| z dz z
}|S )Nr c 3 , K | ] }d |z dz yw) z if True:
NrH ).0rU s r4 <genexpr>zICTokenizeTest.test_max_indent.<locals>.generate_source.<locals>.<genexpr> s O1dQh, 6 Os r zpass
)r r )indentsr0 s r4 generate_sourcez6CTokenizeTest.test_max_indent.<locals>.generate_source s2 WW Og OOF dWnx/ /FMr6 rM rV r" r[ execc @ t t j S r )rP r r< ) the_inputs r4 <lambda>z/CTokenizeTest.test_max_indent.<locals>.<lambda>& s D9Z[d[m[m9n4o r6 ) r rP r r< r= r/ r r compilerN r^ )r@ MAXINDENTr validrW invalidr s @r4 test_max_indentz CTokenizeTest.test_max_indent s
A
. UO 7 8J8JKL &1 )4z6*!), W% *,op gw
F
r6 c d }t d }| j |d t d }| j || || t d }| j |d t d }| j || || t d }| j |d t d
}| j || || y ) Nc ~ t | }t |j D cg c] ^}} }|| f c}} }S c c}} }w r )r r r< )r r kindr s r4 r zECTokenizeTest.test_continuation_lines_indentation.<locals>.get_tokens, sH !&)J89L9LMO O'9fqT6N O
O Os 8z
def fib(n):
\
'''Print a Fibonacci series up to n.'''
\
a, b = 0, 1
ae NAME 'def' (2, 0) (2, 3)
NAME 'fib' (2, 4) (2, 7)
LPAR '(' (2, 7) (2, 8)
NAME 'n' (2, 8) (2, 9)
RPAR ')' (2, 9) (2, 10)
COLON ':' (2, 10) (2, 11)
NEWLINE '' (2, 11) (2, 11)
INDENT '' (4, -1) (4, -1)
STRING "'''Print a Fibonacci series up to n.'''" (4, 0) (4, 39)
NEWLINE '' (4, 39) (4, 39)
NAME 'a' (6, 0) (6, 1)
COMMA ',' (6, 1) (6, 2)
NAME 'b' (6, 3) (6, 4)
EQUAL '=' (6, 5) (6, 6)
NUMBER '0' (6, 7) (6, 8)
COMMA ',' (6, 8) (6, 9)
NUMBER '1' (6, 10) (6, 11)
NEWLINE '' (6, 11) (6, 11)
DEDENT '' (6, -1) (6, -1)
zu
def fib(n):
'''Print a Fibonacci series up to n.'''
a, b = 0, 1
z>
pass
\
pass
z NAME 'pass' (2, 0) (2, 4)
NEWLINE '' (2, 4) (2, 4)
NAME 'pass' (5, 0) (5, 4)
NEWLINE '' (5, 4) (5, 4)
z+
pass
pass
z
if x:
y = 1
\
\
\
\
foo = 1
aj NAME 'if' (2, 0) (2, 2)
NAME 'x' (2, 3) (2, 4)
COLON ':' (2, 4) (2, 5)
NEWLINE '' (2, 5) (2, 5)
INDENT '' (3, -1) (3, -1)
NAME 'y' (3, 4) (3, 5)
EQUAL '=' (3, 6) (3, 7)
NUMBER '1' (3, 8) (3, 9)
NEWLINE '' (3, 9) (3, 9)
NAME 'foo' (8, 4) (8, 7)
EQUAL '=' (8, 8) (8, 9)
NUMBER '1' (8, 10) (8, 11)
NEWLINE '' (8, 11) (8, 11)
DEDENT '' (8, -1) (8, -1)
zI
if x:
y = 1
foo = 1
)r rD r= )r@ r rW code_no_conts r4 #test_continuation_lines_indentationz1CTokenizeTest.test_continuation_lines_indentation+ s O
D #
,
D):l+CD
D #
D):l+CD
D #
"
D):l+CDr6 N)r r r rD r rw r r r r r r r r r r r r r r r rH r6 r4 r r sj E3(? B, \V p= ~$ L . ($ L , " "H T
'@R
,bEr6 r c e Zd Zd Zy)CTokenizerBufferTestsc dddz dddz }t j 5 }t |d| }t | d d d y # 1 sw Y y xY w)Nz" #coding: latin-1
#r3 i' z
#i' foo)r r7 r r )r@ test_scriptr7 file_names r4 !test_newline_at_the_end_of_bufferz7CTokenizerBufferTests.test_newline_at_the_end_of_buffer s`
e)
e)
! 'X#He[AI I & ' ' 's AAN)r r r r rH r6 r4 r r s
'r6 r __main__)8testr test.supportr r r r r r r
r r r
r r r r r r r r r r ior r unittesttextwrapr r r test.test_grammarr r test.support.script_helperr r r r r0 r5 r8 r r r r r. r; r r? r r r r r mainrH r6 r4 <module>r s "! ! ! ! !
! #< " Y Y S 8 S j"A A
6 Gx G.#IH #IL<H <,{" {"|v28 v2r&>X &>R\#H \#~2 28KEH KE\ 'H - - ' zHMMO r6 
Browse
Usage
Versions