# robots.txt for https://www.finda.co.nz

# Site scrapers that are completely disallowed

User-agent: *libwww* 
Disallow: /

User-agent: HttpTool* 
Disallow: /

User-agent: *Fetch* 
Disallow: /

User-agent: http://www.almaden.ibm.com/cs/crawler 
Disallow: /

User-agent: BorderManager*
Disallow: /

User-agent: Maxthon
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: Bilbo/2.3b-UNIX
Disallow: /

User-agent: FAST-WebCrawler/3.6/FirstPage
Disallow: /

User-agent: TurnitinBot
Disallow: /

User-agent: Java*
Disallow: /

User-agent: Offline Explorer/1.9
Disallow: /

User-agent: Missigua Locator 1.9
Disallow: /

User-agent: grub-client 
Disallow: /

User-agent: msrbot
Disallow: /

User-agent: *
Disallow: /clickheat/
Disallow: /approve/
Disallow: /business/listing/*/claim/
Disallow: /afro/

Sitemap: https://www.finda.co.nz/sitemap_index.xml