python (3.12.0)
Browse
Usage
Versions
̑e$ d dl Z d dlZd dlZdgZ e j
dd Z G d d Z G d d Z G d d Z y)
NRobotFileParser RequestRatezrequests secondsc X e Zd Z ddZd Zd Zd Zd Zd Zd Z d Z
d Z d
Z d Z
d Zy
)r c z g | _ g | _ d | _ d| _ d| _ | j | d| _ y )NFr )entriessitemaps
default_entry disallow_all allow_allset_url last_checkedselfurls F/BuggyBox/python/3.12.0/bootstrap/lib/python3.12/urllib/robotparser.py__init__zRobotFileParser.__init__ s;
! ! S c | j S N)r
r s r mtimezRobotFileParser.mtime% s r c 8 dd l }|j | _ y )Nr )timer
)r r s r modifiedzRobotFileParser.modified. s IIKr c r || _ t j j | dd \ | _ | _ y )N )r urllibparseurlparsehostpathr s r r zRobotFileParser.set_url6 s0 : %|| 4 4S 9!A > 49r c t j j | j }|j }| j |j
d j y # t j j $ rT}|j dv rd| _ n4|j dk\ r |j dk r d| _ Y d }~y Y d }~y Y d }~y Y d }~y d }~ww xY w)Nzutf-8)i i Ti i )
r requesturlopenr readr decode
splitlineserror HTTPErrorcoder
r )r frawerrs r r& zRobotFileParser.read; s B 9&&txx0A &&(C JJszz'*557 8 ||%% &xx:%$(!S SXX^!% &4 " &s )A+ + C ;CC c d|j v r| j || _ y y | j j | y N*)
useragentsr r append)r entrys r
_add_entryzRobotFileParser._add_entryH s= %"" "!!)%*" *
LL &r c d}t }| j |D ] }|s4|dk( r
t }d}n"|dk( r | j | t }d}|j d }|dk\ r|d | }|j }|sh|j dd }t
| dk( s|d j j |d< t j j |d j |d< |d dk( rB|dk( r| j | t }|j j |d d}*|d dk( r3|dk7 s9|j j t |d d d}e|d d k( r3|dk7 st|j j t |d d
d}|d d k( r?|dk7 s|d j j rt! |d |_ d}|d d k( r|dk7 s|d j d
}t
| dk( rk|d j j rJ|d j j r)t% t! |d t! |d |_ d}|d dk( s| j( j |d |dk( r| j | y y )Nr r #:z
user-agentdisallowFallowTz crawl-delayz request-rate/sitemap)Entryr r5 findstripsplitlenlowerr r unquoter2 r3 rulelinesRuleLineisdigitintdelayr req_rater )r linesstater4 lineinumberss r r zRobotFileParser.parseQ s
7 2DA: !GE EaZOOE* !GE E #AAvBQx::<D::c1%D4yA~ q'--///1Q ,,..tAw}}?Q7l* z . %$$++DG4 E!W
* z ..xQ/GH !!W' z ..xQ/FG !!W
- z 7==?224*-d1g,EK !!W. z"&q'--"4 LA -'!*2B2B2D2L2L2N '
0 0 2 : : <-8WQZ #gVWj/-ZEN !!W )
MM((a1o7 2p A: OOE " r c d | j ry| j ry| j syt j j t j j
| }t j j dd|j |j |j |j f }t j j | }|sd}| j D ]& }|j | s|j | c S | j r| j j | S y)NFT r< )r
r r
r r r rD
urlunparser" paramsqueryfragmentquoter
applies_to allowancer )r useragentr
parsed_urlr4 s r can_fetchzRobotFileParser.can_fetch s K >>
\\**6<<+?+?+DE
ll%%r"Z__ j . .
0C0C'E Fll %C\\ ,E * s++ , %%//4 4r c | j sy | j D ]! }|j | s|j c S | j r| j j S y r )r r rW rI r r rY r4 s r crawl_delayzRobotFileParser.crawl_delay sY zz|\\ #E * {{" # %%++ +r c | j sy | j D ]! }|j | s|j c S | j r| j j S y r )r r rW rJ r r] s r request_ratez RobotFileParser.request_rate sY zz|\\ &E * ~~% & %%.. .r c 4 | j sy | j S r )r r s r site_mapszRobotFileParser.site_maps s }}}} r c | j }| j || j gz }dj t t | S )Nz
)r r joinmapstr)r r s r __str__zRobotFileParser.__str__ s@ ,, ) !3!3 44G{{3sG,--r N)rQ )__name__
__module__ __qualname__r r r r r& r5 r r[ r^ r` rb rg r r r r sE
!(?
9'G#R:
.r c e Zd Z d Zd Zd Zy)rF c |dk( r|sd}t j j t j j | }t j j | | _ || _ y )NrQ T)r r rR r rV r" rX )r r" rX s r r zRuleLine.__init__ sP 2:i I||&&v||'<'<T'BCLL&&t, "r c Z | j dk( xs |j | j S r0 )r"
startswith)r filenames r rW zRuleLine.applies_to s% yyC A8#6#6tyy#AAr c B | j rdnddz | j z S )NAllowDisallowz: )rX r" r s r rg zRuleLine.__str__ s >>zTADIIMMr N)rh ri rj r rW rg rk r r rF rF s 1#BNr rF c & e Zd Z d Zd Zd Zd Zy)r> c < g | _ g | _ d | _ d | _ y r )r2 rE rI rJ r s r r zEntry.__init__ s
r c g }| j D ] }|j d| | j |j d| j | j 7| j }|j d|j d|j
|j
t t | j dj | S )Nz User-agent: z
Crawl-delay: zRequest-rate: r<
) r2 r3 rI rJ requestssecondsextendre rf rE rd )r retagentrates r rg z
Entry.__str__ s __ /E JJ eW- . / :: ! JJ tzzl3 4 == $==D JJ
a ~F G
3sDNN+,yy~ r c |j d d j }| j D ] }|dk( r y|j }||v s y y)Nr< r r1 TF)rA rC r2 )r rY r| s r rW zEntry.applies_to sY @ OOC(+113 __ E|KKME !
r c f | j D ]! }|j | s|j c S y)NT)rE rW rX )r rp rM s r rX zEntry.allowance
s7 % NN &Dx(~~% & r N)rh ri rj r rg rW rX rk r r r> r> s I
r r> )
collections urllib.parser urllib.request__all__
namedtupler r rF r> rk r r <module>r sU
$k$$]4FG ~. ~.BN N$( (r