python (3.12.0)
Αe, v d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlm Z d dlm
Z
d dl m Z m
Z
G d d Z G d d eej Z G d
d eej Z G d d
eej Z G d deej Z G d de Z G d deej Z G d deej Z G d de Z G d deej Z G d deej Z G d d eej Z G d d eej Z G d d!e Z G d" d#eej Z G d$ d%eej Z G d& d'eej Z G d( d)eej Z G d* d+eej Z! G d, d-eej Z" G d. d/e Z# ejH ejJ d0 G d1 d2ej Z& ejN G d3 d4ej Z(e)d5k( r ejT yy)6 N)support)
socket_helper)threading_helper)BaseHTTPRequestHandler
HTTPServerc > e Zd ZdZdZg Zg ZdZd Zd Z d Z
d Z d Z y)
BaseRobotTest test_robotparserNc t j | j j }t j
j
| _ | j j | y N) ioStringIO
robots_txt readlinesurllib robotparserRobotFileParserparserparse)selfliness I/BuggyBox/python/3.12.0/bootstrap/lib/python3.12/test/test_robotparser.pysetUpzBaseRobotTest.setUp sE DOO,668 ((88: % c P t |t r |\ }}||fS | j |fS r
)
isinstancetupleagentr urlr s r get_agent_and_urlz BaseRobotTest.get_agent_and_url s- c5 ! JE3#: zz3 r c | j D ]] }| j | \ }}| j || 5 | j | j j || d d d _ y # 1 sw Y jxY wN)r! r )goodr" subTest
assertTruer can_fetchr s r test_good_urlsz BaseRobotTest.test_good_urls sw 99 CC //4JE3 #U 3
C 5 5eS AB
C
C C
C
C ,A..A7 c | j D ]] }| j | \ }}| j || 5 | j | j j || d d d _ y # 1 sw Y jxY wr$ )badr" r& assertFalser r( r s r
test_bad_urlszBaseRobotTest.test_bad_urls$ sy 88 DC //4JE3 #U 3
D !6!6uc!BC
D
D D
D
Dr* c l | j | j j | j y r
) assertEqualr site_mapsr s r test_site_mapsz BaseRobotTest.test_site_maps* s# ..0$..Ar )
__name__
__module__ __qualname__r r r% r, r1 r r" r) r. r3 r r r r s7 J E
D
CI!
C D Br r c e Zd ZdZddgZg dZy)UserAgentWildcardTestzUser-agent: *
Disallow: /cyberworld/map/ # This is an infinite virtual URL space
Disallow: /tmp/ # these will soon disappear
Disallow: /foo.html
/
/test.html)/cyberworld/map/index.htmlz/tmp/xxx /foo.htmlNr4 r5 r6 r r% r, r7 r r r9 r9 . s J
D
ACr r9 c e Zd ZdZg dZdgZy) CrawlDelayAndCustomAgentTestz# robots.txt for http://www.example.com/
User-agent: *
Crawl-delay: 1
Request-rate: 3/15
Disallow: /cyberworld/map/ # This is an infinite virtual URL space
# Cybermapper knows where to go.
User-agent: cybermapper
Disallow:
)r: r; ) cybermapperr< r< Nr> r7 r r r@ r@ 9 s J ND '
(Cr r@ c & e Zd ZdZddgZdgZddgZy) SitemapTesta # robots.txt for http://www.example.com/
User-agent: *
Sitemap: http://www.gstatic.com/s2/sitemaps/profiles-sitemap.xml
Sitemap: http://www.google.com/hostednews/sitemap_index.xml
Request-rate: 3/15
Disallow: /cyberworld/map/ # This is an infinite virtual URL space
r: r; r< z7http://www.gstatic.com/s2/sitemaps/profiles-sitemap.xmlz2http://www.google.com/hostednews/sitemap_index.xmlN)r4 r5 r6 r r% r, r1 r7 r r rC rC J s+ J
D '
(CJEGIr rC c e Zd ZdZg Zg dZy)RejectAllRobotsTestz(# go away
User-agent: *
Disallow: /
)r< r: /tmp/Nr> r7 r r rE rE [ s J
D
6Cr rE c e Zd ZdZdZd Zy)BaseRequestRateTestNc | j }| j | j z D ]! }| j | \ }}| j || 5 | j |j
| | j |j | }| j || j | j | j |t j j | j |j | j j | j |j | j j d d d $ y # 1 sw Y 0xY wr$ )r r% r, r" r& r0 crawl_delay request_rateassertIsInstancer r RequestRaterequestsseconds)r r r! r parsed_request_rates r test_request_ratez%BaseRequestRateTest.test_request_ratei s* 99txx' C //4JE3 #U 3
!3!3E!:D<L<LM&,&9&9%&@# !4d6G6GH$$0))+ **66 $$+44 ))22 $$+33 ))11
s
C/E E )r4 r5 r6 rK rJ rQ r7 r r rH rH e s LKr rH c e Zd ZdZdgZy)
EmptyFileTestr
z/fooN)r4 r5 r6 r r% r7 r r rS rS s J 8Dr rS c ^ e Zd ZdZdZej j dd ZdZ dgZ
g dZ y) CrawlDelayAndRequestRateTestzUser-agent: figtree
Crawl-delay: 3
Request-rate: 9/30
Disallow: /tmp
Disallow: /a%3cd.html
Disallow: /a%2fb.html
Disallow: /%7ejoe/index.html
figtree )rV r= )/tmpz /tmp.html /tmp/a.html /a%3cd.html /a%3Cd.htmlz /a%2fb.htmlz/~joe/index.htmlN) r4 r5 r6 r r r r rM rK rJ r% r, r7 r r rU rU s< J
E%%11!R8LK $ %D .Cr rU c e Zd ZdZy)DifferentAgentTestz FigTree Robot libwww-perl/5.04Nr4 r5 r6 r r7 r r r_ r_ s ,Er r_ c " e Zd ZdZdgZg dZdZy)InvalidRequestRateTestzUser-agent: *
Disallow: /tmp/
Disallow: /a%3Cd.html
Disallow: /a/b.html
Disallow: /%7ejoe/index.html
Crawl-delay: 3
Request-rate: 9/banana
rZ )rF r[ r\ r] z /a/b.htmlz/%7Ejoe/index.htmlrY N)r4 r5 r6 r r% r, rJ r7 r r rb rb s J
8D !CKr rb c e Zd ZdZdgZg Zy)InvalidCrawlDelayTestz2User-Agent: *
Disallow: /.
Crawl-delay: pears
r= Nr> r7 r r rd rd s J
=D
Cr rd c e Zd ZdZdZdgZdgZy) AnotherInvalidRequestRateTestzeUser-agent: Googlebot
Allow: /folder1/myfile.html
Disallow: /folder1/
Request-rate: whale/banana
Googlebot/folder1/myfile.html/folder1/anotherfile.htmlNr4 r5 r6 r r r% r, r7 r r rf rf s J
E " #D &
'Cr rf c e Zd ZdZdZdgZy)UserAgentOrderingTestzMUser-agent: Googlebot
Disallow: /
User-agent: Googlebot-Mobile
Allow: /
rg z/something.jpgN)r4 r5 r6 r r r, r7 r r rl rl s J
E
Cr rl c e Zd ZdZy)UserAgentGoogleMobileTestzGooglebot-MobileNr` r7 r r rn rn s Er rn c e Zd ZdZdZdgZdgZy)GoogleURLOrderingTestzJUser-agent: Googlebot
Allow: /folder1/myfile.html
Disallow: /folder1/
googlebotrh ri Nrj r7 r r rp rp s J
E " #D &
'Cr rp c e Zd ZdZdgZdgZy)DisallowQueryStringTestz2User-agent: *
Disallow: /some/path?name=value
/some/pathz/some/path?name=valueNr> r7 r r rs rs s J
>D "
#Cr rs c e Zd ZdZdgZdgZy) UseFirstUserAgentWildcardTestzNUser-agent: *
Disallow: /some/path
User-agent: *
Disallow: /another/path
z
/another/pathrt Nr> r7 r r rv rv s J
D .Cr rv c e Zd ZdZdgZdgZy)EmptyQueryStringTestz>User-agent: *
Allow: /some/path?
Disallow: /another/path?
z /some/path?z/another/path?Nr> r7 r r rx rx s J
?D
Cr rx c Z e Zd ZdZej
j
dd ZdZddgZ dgZ
y) DefaultEntryTestzOUser-agent: *
Crawl-delay: 1
Request-rate: 3/15
Disallow: /cyberworld/map/
rY r: r; r< N) r4 r5 r6 r r r rM rK rJ r% r, r7 r r rz rz s: J %%11!R8LK D '
(Cr rz c e Zd ZdZdZd Zy)StringFormattingTestzUser-agent: *
Crawl-delay: 1
Request-rate: 3/15
Disallow: /cyberworld/map/ # This is an infinite virtual URL space
# Cybermapper knows where to go.
User-agent: cybermapper
Disallow: /some/path
zxUser-agent: cybermapper
Disallow: /some/path
User-agent: *
Crawl-delay: 1
Request-rate: 3/15
Disallow: /cyberworld/map/c b | j t | j | j y r
)r0 strr expected_outputr2 s r test_string_formattingz+StringFormattingTest.test_string_formatting* s! T[[)4+?+?@r N)r4 r5 r6 r r r r7 r r r~ r~ s JOAr r~ c e Zd Zd Zd Zy) RobotHandlerc ( | j dd y )Ni zForbidden access)
send_errorr2 s r do_GETzRobotHandler.do_GET0 s /0r c y r
r7 )r formatargss r log_messagezRobotHandler.log_message3 s r N)r4 r5 r6 r r r7 r r r r . s 1
r r z&Socket server requires working socket.c < e Zd Zd Zd Zej d Zy) PasswordProtectedSiteTestCasec \ | j t j j t t
j dft | _ t j d| j j ddi | _ d| j _
| j j y )Nr zHTTPServer serving
poll_intervalg{Gz?)nametargetkwargsT)
addCleanupr request
urlcleanupr r HOSTr server threadingThread
serve_forevertdaemonstartr2 s r r z#PasswordProtectedSiteTestCase.setUp= sw 112 -"4"4a!8,G !!%;;,, $D)
+
r c | j j | j j | j j y r
)r shutdownr join server_closer2 s r tearDownz&PasswordProtectedSiteTestCase.tearDownM s/
"r c D | j j }dt j z dz t |d z }|dz }t
j j }|j | |j | j |j d| y )Nzhttp://:r| z /robots.txt*) r server_addressr r r r r r set_urlreadr- r( )r addrr!
robots_urlr s r testPasswordProtectedSitez7PasswordProtectedSiteTestCase.testPasswordProtectedSiteR s {{))-,,,s2Sa\A=(
##335s
))#z:;r N)r4 r5 r6 r r r reap_threadsr r7 r r r r 7 s& #
""< #<r r c Z e Zd ZdZdj e Zed Zd Zd Z d Z
d Z y) NetworkTestCasezhttp://www.pythontest.net/z{}elsewhere/robots.txtc & t j d t j | j 5 t
j j | j | _ | j j d d d y # 1 sw Y y xY w)Nnetwork) r requiresr transient_internetbase_urlr r r r r r )clss r
setUpClasszNetworkTestCase.setUpClassc sa #
-
-cll
; ++;;CNNKCJ JJOO s A BBc dj | j |t j j | d sd S d S )Nz{}{}{}r| r: r
)r r ospathsplitext)r r s r r! zNetworkTestCase.urlj sA MM4BGG,<,<T,B1,E
KM
r c | j | j j | j | j j | j | j j d | j | j j
d | j | j j d y )Nr r )r- r disallow_all allow_all
assertGreatermtimerJ rK r2 s r
test_basiczNetworkTestCase.test_basico s 112 ../ 4;;,,.2 0056 11#67r c | j | j j d| j d | j | j j d| j
| j | j j d| j d | j | j j d| j d | j | j j d| j d | j | j j d| j
y )Nr elsewhereNutchbrianwebstats)r' r r( r! r- r r2 s r test_can_fetchz NetworkTestCase.test_can_fetchv s --c488K3HIJ ..w
FG ..w8IJK ..w8LMN ..sDHHZ4HIJ --c4==ABr c t j j | j d }|j | j |j | j |j | j |j d | j |j d | j |j d y )Nz i-robot.txtr r )r r r r! r r' r r- r r0 r assertIsNonerJ rK )r r s r
test_read_404z NetworkTestCase.test_read_404~ s ##33DHH]4KL
(() ,,- + &,,S12 &--c23r N) r4 r5 r6 r r r classmethodr r! r r r r7 r r r r ] s@ ,H)00:J
8C4r r __main__)+r r r unittesturllib.robotparserr testr test.supportr r http.serverr r r TestCaser9 r@ rC rE rH rS rU r_ rb rd rf rl rn rp rs rv rx rz r~ r
skipUnlesshas_socket_supportr requires_working_socketr r4 mainr7 r r <module>r s+ & ) : B BDBM8+<+< B)=(2C2C )"G-!2!2 G"7-):): 7- 8'):):
.#68I8I .$-5 -
]H,=,=
M8+<+<
(M83D3D
( M8+<+< 5
(M8+<+<
($mX->-> $
M83D3D
=(*;*;
)*H,=,=
)A=(*;*; A4
)
, <H$5$5 < <D ! "'4h'' '4 #'4R ZHMMO r 
Browse
Usage
Versions