• Home
  • Raw
  • Download

Lines Matching defs:robots_txt

13     robots_txt = ''  variable in BaseRobotTest
47 robots_txt = """\ variable in UserAgentWildcardTest
58 robots_txt = """\ variable in CrawlDelayAndCustomAgentTest
75 robots_txt = """\ variable in SitemapTest
92 robots_txt = """\ variable in RejectAllRobotsTest
130 robots_txt = '' variable in EmptyFileTest
135 robots_txt = """\ variable in CrawlDelayAndRequestRateTest
157 robots_txt = """\ variable in InvalidRequestRateTest
174 robots_txt = """\ variable in InvalidCrawlDelayTest
186 robots_txt = """\ variable in AnotherInvalidRequestRateTest
201 robots_txt = """\ variable in UserAgentOrderingTest
219 robots_txt = """\ variable in GoogleURLOrderingTest
231 robots_txt = """\ variable in DisallowQueryStringTest
241 robots_txt = """\ variable in UseFirstUserAgentWildcardTest
254 robots_txt = """\ variable in EmptyQueryStringTest
264 robots_txt = """\ variable in DefaultEntryTest
277 robots_txt = """\ variable in StringFormattingTest
353 robots_txt = '{}elsewhere/robots.txt'.format(base_url) variable in NetworkTestCase