python (3.11.7)
e, H d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlm Z d dlm
Z
d dl m Z m
Z
G d d Z G d d eej Z G d
d eej Z G d d
eej Z G d deej Z G d de Z G d deej Z G d deej Z G d de Z G d deej Z G d deej Z G d d eej Z G d d eej Z G d d!e Z G d" d#eej Z G d$ d%eej Z G d& d'eej Z G d( d)eej Z G d* d+eej Z! G d, d-eej Z" G d. d/e Z# ej$ ej% d0 G d1 d2ej Z& ej' G d3 d4ej Z(e)d5k r ej* dS dS )6 N)support)
socket_helper)threading_helper)BaseHTTPRequestHandler
HTTPServerc @ e Zd ZdZdZg Zg ZdZd Zd Z d Z
d Z d Z dS )
BaseRobotTest test_robotparserNc t j | j }t j | _ | j | d S N) ioStringIO
robots_txt readlinesurllib robotparserRobotFileParserparserparse)selfliness I/BuggyBox/python/3.11.7/bootstrap/lib/python3.11/test/test_robotparser.pysetUpzBaseRobotTest.setUp sQ DO,,6688 (88:: % c P t |t r |\ }}||fS | j |fS r
)
isinstancetupleagentr urlr s r get_agent_and_urlz BaseRobotTest.get_agent_and_url s3 c5 ! ! JE3#: z3 r c | j D ]w}| | \ }}| || 5 | | j || d d d n # 1 swxY w Y xd S N)r! r )goodr" subTest
assertTruer can_fetchr s r test_good_urlsz BaseRobotTest.test_good_urls s 9 C CC //44JE3 #U 33
C
C 5 5eS A ABBB
C
C
C
C
C
C
C
C
C
C
C
C
C
C
C C C /A44A8 ;A8 c | j D ]w}| | \ }}| || 5 | | j || d d d n # 1 swxY w Y xd S r$ )badr" r& assertFalser r( r s r
test_bad_urlszBaseRobotTest.test_bad_urls$ s 8 D DC //44JE3 #U 33
D
D !6!6uc!B!BCCC
D
D
D
D
D
D
D
D
D
D
D
D
D
D
D D Dr* c j | | j | j d S r
) assertEqualr site_mapsr s r test_site_mapsz BaseRobotTest.test_site_maps* s. ..00$.AAAAAr )
__name__
__module__ __qualname__r r r% r, r1 r r" r) r. r3 r r r r s J E
D
CI! ! !
C C C D D D B B B B Br r c " e Zd ZdZddgZg dZdS )UserAgentWildcardTestzUser-agent: *
Disallow: /cyberworld/map/ # This is an infinite virtual URL space
Disallow: /tmp/ # these will soon disappear
Disallow: /foo.html
/
/test.html)/cyberworld/map/index.htmlz/tmp/xxx /foo.htmlNr4 r5 r6 r r% r, r7 r r r9 r9 . s, J
D
A
A
ACCCr r9 c e Zd ZdZg dZdgZdS ) CrawlDelayAndCustomAgentTestz# robots.txt for http://www.example.com/
User-agent: *
Crawl-delay: 1
Request-rate: 3/15
Disallow: /cyberworld/map/ # This is an infinite virtual URL space
# Cybermapper knows where to go.
User-agent: cybermapper
Disallow:
)r: r; ) cybermapperr< r< Nr> r7 r r r@ r@ 9 s+ J N M MD '
(CCCr r@ c ( e Zd ZdZddgZdgZddgZdS ) SitemapTesta # robots.txt for http://www.example.com/
User-agent: *
Sitemap: http://www.gstatic.com/s2/sitemaps/profiles-sitemap.xml
Sitemap: http://www.google.com/hostednews/sitemap_index.xml
Request-rate: 3/15
Disallow: /cyberworld/map/ # This is an infinite virtual URL space
r: r; r< z7http://www.gstatic.com/s2/sitemaps/profiles-sitemap.xmlz2http://www.google.com/hostednews/sitemap_index.xmlN)r4 r5 r6 r r% r, r1 r7 r r rC rC J s7 J
D '
(CJEGIIIr rC c e Zd ZdZg Zg dZdS )RejectAllRobotsTestz(# go away
User-agent: *
Disallow: /
)r< r: /tmp/Nr> r7 r r rE rE [ s' J
D
6
6
6CCCr rE c e Zd ZdZdZd ZdS )BaseRequestRateTestNc r | j }| j | j z D ] }| | \ }}| || 5 | | | | j | | }| || j | j o| |t j
j | |j | j j | |j
| j j
d d d n # 1 swxY w Y d S r$ )r r% r, r" r& r0 crawl_delay request_rateassertIsInstancer r RequestRaterequestsseconds)r r r! r parsed_request_rates r test_request_ratez%BaseRequestRateTest.test_request_ratei s 9tx' C //44JE3 #U 33
!3!3E!:!:D<LMMM&,&9&9%&@&@# !4d6GHHH$0))+ *6 $$+4 )2 $$+3 )1
s CD**D. 1D. )r4 r5 r6 rK rJ rQ r7 r r rH rH e s- LK r rH c e Zd ZdZdgZdS )
EmptyFileTestr
z/fooN)r4 r5 r6 r r% r7 r r rS rS s J 8DDDr rS c ^ e Zd ZdZdZej dd ZdZ dgZ
g dZ dS ) CrawlDelayAndRequestRateTestzUser-agent: figtree
Crawl-delay: 3
Request-rate: 9/30
Disallow: /tmp
Disallow: /a%3cd.html
Disallow: /a%2fb.html
Disallow: /%7ejoe/index.html
figtree )rV r= )/tmpz /tmp.html /tmp/a.html /a%3cd.html /a%3Cd.htmlz /a%2fb.htmlz/~joe/index.htmlN) r4 r5 r6 r r r r rM rK rJ r% r, r7 r r rU rU sR J
E%11!R88LK $ %D . . .CCCr rU c e Zd ZdZdS )DifferentAgentTestz FigTree Robot libwww-perl/5.04Nr4 r5 r6 r r7 r r r_ r_ s ,EEEr r_ c $ e Zd ZdZdgZg dZdZdS )InvalidRequestRateTestzUser-agent: *
Disallow: /tmp/
Disallow: /a%3Cd.html
Disallow: /a/b.html
Disallow: /%7ejoe/index.html
Crawl-delay: 3
Request-rate: 9/banana
rZ )rF r[ r\ r] z /a/b.htmlz/%7Ejoe/index.htmlrY N)r4 r5 r6 r r% r, rJ r7 r r rb rb s4 J
8D ! ! !CKKKr rb c e Zd ZdZdgZg ZdS )InvalidCrawlDelayTestz2User-Agent: *
Disallow: /.
Crawl-delay: pears
r= Nr> r7 r r rd rd s# J
=D
CCCr rd c " e Zd ZdZdZdgZdgZdS ) AnotherInvalidRequestRateTestzeUser-agent: Googlebot
Allow: /folder1/myfile.html
Disallow: /folder1/
Request-rate: whale/banana
Googlebot/folder1/myfile.html/folder1/anotherfile.htmlNr4 r5 r6 r r r% r, r7 r r rf rf s, J
E " #D &
'CCCr rf c e Zd ZdZdZdgZdS )UserAgentOrderingTestzMUser-agent: Googlebot
Disallow: /
User-agent: Googlebot-Mobile
Allow: /
rg z/something.jpgN)r4 r5 r6 r r r, r7 r r rl rl s$ J
E
CCCr rl c e Zd ZdZdS )UserAgentGoogleMobileTestzGooglebot-MobileNr` r7 r r rn rn s EEEr rn c " e Zd ZdZdZdgZdgZdS )GoogleURLOrderingTestzJUser-agent: Googlebot
Allow: /folder1/myfile.html
Disallow: /folder1/
googlebotrh ri Nrj r7 r r rp rp s, J
E " #D &
'CCCr rp c e Zd ZdZdgZdgZdS )DisallowQueryStringTestz2User-agent: *
Disallow: /some/path?name=value
/some/pathz/some/path?name=valueNr> r7 r r rs rs s& J
>D "
#CCCr rs c e Zd ZdZdgZdgZdS ) UseFirstUserAgentWildcardTestzNUser-agent: *
Disallow: /some/path
User-agent: *
Disallow: /another/path
z
/another/pathrt Nr> r7 r r rv rv s& J
D .CCCr rv c e Zd ZdZdgZdgZdS )EmptyQueryStringTestz>User-agent: *
Allow: /some/path?
Disallow: /another/path?
z /some/path?z/another/path?Nr> r7 r r rx rx s& J
?D
CCCr rx c Z e Zd ZdZej dd ZdZddgZ dgZ
dS ) DefaultEntryTestzOUser-agent: *
Crawl-delay: 1
Request-rate: 3/15
Disallow: /cyberworld/map/
rY r: r; r< N) r4 r5 r6 r r r rM rK rJ r% r, r7 r r rz rz sF J %11!R88LK D '
(CCCr rz c e Zd ZdZdZd ZdS )StringFormattingTestzUser-agent: *
Crawl-delay: 1
Request-rate: 3/15
Disallow: /cyberworld/map/ # This is an infinite virtual URL space
# Cybermapper knows where to go.
User-agent: cybermapper
Disallow: /some/path
zxUser-agent: cybermapper
Disallow: /some/path
User-agent: *
Crawl-delay: 1
Request-rate: 3/15
Disallow: /cyberworld/map/c ` | t | j | j d S r
)r0 strr expected_outputr2 s r test_string_formattingz+StringFormattingTest.test_string_formatting* s+ T[))4+?@@@@@r N)r4 r5 r6 r r r r7 r r r~ r~ s6 JOA A A A Ar r~ c e Zd Zd Zd ZdS ) RobotHandlerc 2 | dd d S )Ni zForbidden access)
send_errorr2 s r do_GETzRobotHandler.do_GET0 s /00000r c d S r
r7 )r formatargss r log_messagezRobotHandler.log_message3 s r N)r4 r5 r6 r r r7 r r r r . s2 1 1 1
r r z&Socket server requires working socket.c : e Zd Zd Zd Zej d ZdS ) PasswordProtectedSiteTestCasec 4 | t j j t t
j dft | _ t j
d| j j ddi | _ d| j _
| j d S )Nr zHTTPServer serving
poll_intervalg{Gz?)nametargetkwargsT)
addCleanupr request
urlcleanupr r HOSTr server threadingThread
serve_forevertdaemonstartr2 s r r z#PasswordProtectedSiteTestCase.setUp= s~ 1222 -"4a!8,GG !%;, $D)
+ + +
r c | j | j | j d S r
)r shutdownr join server_closer2 s r tearDownz&PasswordProtectedSiteTestCase.tearDownM s?
"""""r c X | j j }dt j z dz t |d z }|dz }t
j }| | | |
| d| d S )Nzhttp://:r| z /robots.txt*) r server_addressr r r r r r set_urlreadr- r( )r addrr!
robots_urlr s r testPasswordProtectedSitez7PasswordProtectedSiteTestCase.testPasswordProtectedSiteR s {)-,,s2Sa\\A=(
#3355s
))#z::;;;;;r N)r4 r5 r6 r r r reap_threadsr r7 r r r r 7 sQ # # #
"< < #"< < <r r c j e Zd ZdZd e Zed Zd Zd Z d Z
d Z dS ) NetworkTestCasezhttp://www.pythontest.net/z{}elsewhere/robots.txtc t j d t j | j 5 t
j | j | _ | j
d d d d S # 1 swxY w Y d S )Nnetwork) r requiresr transient_internetbase_urlr r r r r r )clss r
setUpClasszNetworkTestCase.setUpClassc s ###
-cl
;
; +;;CNKKCJ JOO s AA>>BBc d | j |t j | d sdnd S )Nz{}{}{}r| r: r
)r r ospathsplitext)r r s r r! zNetworkTestCase.urlj s? M4BG,<,<T,B,B1,E!M2
r c | | j j | | j j | | j d | | j d | | j d d S )Nr r )r- r disallow_all allow_all
assertGreatermtimerJ rK r2 s r
test_basiczNetworkTestCase.test_basico s 1222 ./// 4;,,..222 0055666 11#6677777r c | | j d| d | | j d| j | | j d| d | | j d| d | | j d| d | | j d| j d S )Nr elsewhereNutchbrianwebstats)r' r r( r! r- r r2 s r test_can_fetchz NetworkTestCase.test_can_fetchv s --c488K3H3HIIJJJ ..w
FFGGG ..w8I8IJJKKK ..w8L8LMMNNN ..sDHHZ4H4HIIJJJ --c4=AABBBBBr c t j | d }| | |j | |j | |
d | | d | |
d d S )Nz i-robot.txtr r )r r r r! r r' r r- r r0 r assertIsNonerJ rK )r r s r
test_read_404z NetworkTestCase.test_read_404~ s #33DHH]4K4KLL
())) ,--- +++ &,,S11222 &--c2233333r N) r4 r5 r6 r r r classmethodr r! r r r r7 r r r r ] s ,H)00::J [
8 8 8C C C4 4 4 4 4r r __main__)+r r r unittesturllib.robotparserr testr test.supportr r http.serverr r r TestCaser9 r@ rC rE rH rS rU r_ rb rd rf rl rn rp rs rv rx rz r~ r
skipUnlesshas_socket_supportr requires_working_socketr r4 mainr7 r r <module>r s. & & & & & & ) ) ) ) ) ) : : : : : : : : B B B B B B B BDB B B B BM8+< B B B) ) ) ) )=(2C ) ) )"G G G G G-!2 G G G"7 7 7 7 7-): 7 7 7 - 8 '):
. . . . .#68I . . .$- - - - -5 - - -
]H,=
M8+<
(
(
(
(
(M83D
(
(
( M8+< 5
(
(
(
(
(M8+<
(
(
($ $ $ $ $mX-> $ $ $
M83D
=(*;
)
)
)
)
)*H,=
)
)
)A A A A A=(*; A A A4
)
, < < < < <H$5 < < <D ! ""'4 '4 '4 '4 '4h' '4 '4 #"'4R ZHMOOOOO r 
Browse
Usage
Versions