| D | test_robotparser.py | 14 bad = [] variable in BaseRobotTest
34 for url in self.bad:
48 bad = ['/cyberworld/map/index.html', '/tmp/xxx', '/foo.html'] variable in UserAgentWildcardTest
65 bad = ['/cyberworld/map/index.html'] variable in CrawlDelayAndCustomAgentTest
75 bad = ['/cyberworld/map/index.html', '/', '/tmp/'] variable in RejectAllRobotsTest
81 for url in self.good + self.bad:
117 bad = ['/tmp', '/tmp.html', '/tmp/a.html', '/a%3cd.html', '/a%3Cd.html', variable in CrawlDelayAndRequestRateTest
140 bad = ['/tmp/', '/tmp/a.html', '/a%3cd.html', '/a%3Cd.html', '/a/b.html', variable in InvalidRequestRateTest
154 bad = [] variable in InvalidCrawlDelayTest
167 bad = ['/folder1/anotherfile.html'] variable in AnotherInvalidRequestRateTest
[all …]
|