B 2*™\P)ã@säddlZddlZddlZddlZddlZddlmZddlm Z m Z Gdd„dƒZ Gdd„de ej ƒZ Gdd „d e ej ƒZGd d „d e ej ƒZGd d „d e ƒZGdd„deej ƒZGdd„deƒZGdd„de ej ƒZGdd„de ej ƒZGdd„de ej ƒZGdd„de ej ƒZGdd„deƒZGdd„de ej ƒZGdd„de ej ƒZGd d!„d!e ej ƒZGd"d#„d#e ej ƒZGd$d%„d%eej ƒZGd&d'„d'e ej ƒZGd(d)„d)e ƒZGd*d+„d+ej ƒZGd,d-„d-ej ƒZ e!d.kràe "¡dS)/éN)Úsupport)ÚBaseHTTPRequestHandlerÚ HTTPServerc@s<eZdZdZdZgZgZdd„Zdd„Zdd„Z d d „Z d S) Ú BaseRobotTestÚZtest_robotparsercCs,t |j¡ ¡}tj ¡|_|j |¡dS)N) ÚioÚStringIOÚ robots_txtÚ readlinesÚurllibÚ robotparserÚRobotFileParserÚparserÚparse)ÚselfÚlines©rú@C:\ALexclude\prg\programme\Python37\Lib\test\test_robotparser.pyÚsetUps zBaseRobotTest.setUpcCs$t|tƒr|\}}||fS|j|fS)N)Ú isinstanceÚtupleÚagent)rÚurlrrrrÚget_agent_and_urls zBaseRobotTest.get_agent_and_urlc CsPxJ|jD]@}| |¡\}}|j||d| |j ||¡¡WdQRXqWdS)N)rr)ÚgoodrÚsubTestÚ assertTruerÚ can_fetch)rrrrrrÚtest_good_urlss zBaseRobotTest.test_good_urlsc CsPxJ|jD]@}| |¡\}}|j||d| |j ||¡¡WdQRXqWdS)N)rr)ÚbadrrÚ assertFalserr)rrrrrrÚ test_bad_urls!s zBaseRobotTest.test_bad_urlsN) Ú__name__Ú __module__Ú __qualname__r rrrrrrr!rrrrr src@s"eZdZdZddgZdddgZdS)ÚUserAgentWildcardTestz•User-agent: * Disallow: /cyberworld/map/ # This is an infinite virtual URL space Disallow: /tmp/ # these will soon disappear Disallow: /foo.html ú/z /test.htmlz/cyberworld/map/index.htmlz/tmp/xxxz /foo.htmlN)r"r#r$r rrrrrrr%(sr%c@s eZdZdZdddgZdgZdS)ÚCrawlDelayAndCustomAgentTestzå# robots.txt for http://www.example.com/ User-agent: * Crawl-delay: 1 Request-rate: 3/15 Disallow: /cyberworld/map/ # This is an infinite virtual URL space # Cybermapper knows where to go. User-agent: cybermapper Disallow: r&z /test.html)Z cybermapperz/cyberworld/map/index.htmlz/cyberworld/map/index.htmlN)r"r#r$r rrrrrrr'3s  r'c@seZdZdZgZdddgZdS)ÚRejectAllRobotsTestz(# go away User-agent: * Disallow: / z/cyberworld/map/index.htmlr&z/tmp/N)r"r#r$r rrrrrrr(Dsr(c@seZdZdd„ZdS)ÚBaseRequestRateTestc Cs°xª|j|jD]š}| |¡\}}|j||dt|jrL| |j |¡|j¡|jrž| |j |¡t j j ¡| |j |¡j |jj ¡| |j |¡j |jj ¡WdQRXqWdS)N)rr)rrrrÚ crawl_delayÚ assertEqualrÚ request_rateZassertIsInstancer r Ú RequestRateZrequestsZseconds)rrrrrrÚtest_request_ratePs      z%BaseRequestRateTest.test_request_rateN)r"r#r$r.rrrrr)Nsr)c@s>eZdZdZdZej dd¡ZdZ dgZ ddd d d d d gZ dS)ÚCrawlDelayAndRequestRateTestz’User-agent: figtree Crawl-delay: 3 Request-rate: 9/30 Disallow: /tmp Disallow: /a%3cd.html Disallow: /a%2fb.html Disallow: /%7ejoe/index.html Úfigtreeé éé)r0z /foo.htmlz/tmpz /tmp.htmlz /tmp/a.htmlz /a%3cd.htmlz /a%3Cd.htmlz /a%2fb.htmlz/~joe/index.htmlN) r"r#r$r rr r r-r,r*rrrrrrr/gs  r/c@seZdZdZdZdZdS)ÚDifferentAgentTestzFigTree Robot libwww-perl/5.04N)r"r#r$rr,r*rrrrr4ysr4c@s*eZdZdZdgZddddddgZd Zd S) ÚInvalidRequestRateTestzUser-agent: * Disallow: /tmp/ Disallow: /a%3Cd.html Disallow: /a/b.html Disallow: /%7ejoe/index.html Crawl-delay: 3 Request-rate: 9/banana z/tmpz/tmp/z /tmp/a.htmlz /a%3cd.htmlz /a%3Cd.htmlz /a/b.htmlz/%7Ejoe/index.htmlr3N)r"r#r$r rrr*rrrrr5s   r5c@seZdZdZdgZgZdS)ÚInvalidCrawlDelayTestz2User-Agent: * Disallow: /. Crawl-delay: pears z /foo.htmlN)r"r#r$r rrrrrrr6‘sr6c@s eZdZdZdZdgZdgZdS)ÚAnotherInvalidRequestRateTestzeUser-agent: Googlebot Allow: /folder1/myfile.html Disallow: /folder1/ Request-rate: whale/banana Ú Googlebotz/folder1/myfile.htmlz/folder1/anotherfile.htmlN)r"r#r$r rrrrrrrr7sr7c@seZdZdZdZdgZdS)ÚUserAgentOrderingTestzMUser-agent: Googlebot Disallow: / User-agent: Googlebot-Mobile Allow: / r8z/something.jpgN)r"r#r$r rrrrrrr9ªs r9c@seZdZdZdS)ÚUserAgentGoogleMobileTestzGooglebot-MobileN)r"r#r$rrrrrr:¹sr:c@s eZdZdZdZdgZdgZdS)ÚGoogleURLOrderingTestzJUser-agent: Googlebot Allow: /folder1/myfile.html Disallow: /folder1/ Z googlebotz/folder1/myfile.htmlz/folder1/anotherfile.htmlN)r"r#r$r rrrrrrrr;½sr;c@seZdZdZdgZdgZdS)ÚDisallowQueryStringTestz2User-agent: * Disallow: /some/path?name=value z /some/pathz/some/path?name=valueN)r"r#r$r rrrrrrr<Êsr<c@seZdZdZdgZdgZdS)ÚUseFirstUserAgentWildcardTestzNUser-agent: * Disallow: /some/path User-agent: * Disallow: /another/path z /another/pathz /some/pathN)r"r#r$r rrrrrrr=Ôsr=c@seZdZdZdgZdgZdS)ÚEmptyQueryStringTestz>User-agent: * Allow: /some/path? Disallow: /another/path? z /some/path?z/another/path?N)r"r#r$r rrrrrrr>ásr>c@s0eZdZdZej dd¡ZdZddgZ dgZ dS) ÚDefaultEntryTestzOUser-agent: * Crawl-delay: 1 Request-rate: 3/15 Disallow: /cyberworld/map/ r3éér&z /test.htmlz/cyberworld/map/index.htmlN) r"r#r$r r r r-r,r*rrrrrrr?ìs r?c@seZdZdZdZdd„ZdS)ÚStringFormattingTestzÆUser-agent: * Crawl-delay: 1 Request-rate: 3/15 Disallow: /cyberworld/map/ # This is an infinite virtual URL space # Cybermapper knows where to go. User-agent: cybermapper Disallow: /some/path zzUser-agent: cybermapper Disallow: /some/path User-agent: * Crawl-delay: 1 Request-rate: 3/15 Disallow: /cyberworld/map/ cCs| t|jƒ|j¡dS)N)r+ÚstrrÚexpected_output)rrrrÚtest_string_formattingsz+StringFormattingTest.test_string_formattingN)r"r#r$r rDrErrrrrBùs  rBc@seZdZdd„Zdd„ZdS)Ú RobotHandlercCs| dd¡dS)Ni“zForbidden access)Z send_error)rrrrÚdo_GETszRobotHandler.do_GETcGsdS)Nr)rÚformatÚargsrrrÚ log_messageszRobotHandler.log_messageN)r"r#r$rGrJrrrrrFsrFc@s*eZdZdd„Zdd„Zejdd„ƒZdS)ÚPasswordProtectedSiteTestCasecCsBttjdftƒ|_tjd|jjddid|_d|j_ |j  ¡dS)NrzHTTPServer servingZ poll_intervalg{®Gáz„?)ÚnameÚtargetÚkwargsT) rrÚHOSTrFÚserverÚ threadingZThreadZ serve_foreverÚtZdaemonÚstart)rrrrrsz#PasswordProtectedSiteTestCase.setUpcCs"|j ¡|j ¡|j ¡dS)N)rPZshutdownrRÚjoinZ server_close)rrrrÚtearDown,s  z&PasswordProtectedSiteTestCase.tearDowncCs\|jj}dtjdt|dƒ}|d}tj ¡}| |¡|  ¡|  |  d|¡¡dS)Nzhttp://ú:rAz /robots.txtÚ*) rPZserver_addressrrOrCr r r Zset_urlÚreadr r)rZaddrrZ robots_urlrrrrÚtestPasswordProtectedSite1s  z7PasswordProtectedSiteTestCase.testPasswordProtectedSiteN)r"r#r$rrUrZ reap_threadsrYrrrrrKs rKc@sFeZdZdZd e¡Zedd„ƒZdd„Zdd„Z d d „Z d d „Z d S)ÚNetworkTestCasezhttp://www.pythontest.net/z{}elsewhere/robots.txtc Cs@t d¡t |j¡ tj |j¡|_|j  ¡WdQRXdS)NZnetwork) rZrequiresZtransient_internetÚbase_urlr r r r rrX)ÚclsrrrÚ setUpClassAs zNetworkTestCase.setUpClasscCs$d |j|tj |¡dsdnd¡S)Nz{}{}{}rAr&r)rHr[ÚosÚpathÚsplitext)rr_rrrrHszNetworkTestCase.urlcCsV| |jj¡| |jj¡| |j ¡d¡| |j d¡¡| |j d¡¡dS)NrrW)r rÚ disallow_allÚ allow_allZ assertGreaterÚmtimer*r,)rrrrÚ test_basicMs zNetworkTestCase.test_basiccCs˜| |j d| d¡¡¡| |j d|j¡¡| |j d| d¡¡¡| |j d| d¡¡¡| |j d| d¡¡¡| |j d|j¡¡dS)NrWZ elsewhereZNutchZbrianZwebstats)rrrrr r[)rrrrÚtest_can_fetchTs zNetworkTestCase.test_can_fetchcCsftj | d¡¡}| ¡| |j¡| |j¡|  |  ¡d¡|  |  d¡¡|  |  d¡¡dS)Nz i-robot.txtrrW)r r r rrXrrbr rar+rcZ assertIsNoner*r,)rrrrrÚ test_read_404\s  zNetworkTestCase.test_read_404N) r"r#r$r[rHr Ú classmethodr]rrdrerfrrrrrZ<s  rZÚ__main__)#rr^rQZunittestZurllib.robotparserr ZtestrZ http.serverrrrZTestCaser%r'r(r)r/r4r5r6r7r9r:r;r<r=r>r?rBrFrKrZr"ÚmainrrrrÚs:           )