| D | test_robotparser.py | 14 bad = [] variable in BaseRobotTest
35 for url in self.bad:
52 bad = ['/cyberworld/map/index.html', '/tmp/xxx', '/foo.html'] variable in UserAgentWildcardTest
69 bad = ['/cyberworld/map/index.html'] variable in CrawlDelayAndCustomAgentTest
84 bad = ['/cyberworld/map/index.html'] variable in SitemapTest
96 bad = ['/cyberworld/map/index.html', '/', '/tmp/'] variable in RejectAllRobotsTest
105 for url in self.good + self.bad:
146 bad = ['/tmp', '/tmp.html', '/tmp/a.html', '/a%3cd.html', '/a%3Cd.html', variable in CrawlDelayAndRequestRateTest
165 bad = ['/tmp/', '/tmp/a.html', '/a%3cd.html', '/a%3Cd.html', '/a/b.html', variable in InvalidRequestRateTest
179 bad = [] variable in InvalidCrawlDelayTest
[all …]
|