# /robots.txt file for http://webcrawler.com/ # mail webmaster@webcrawler.com for constructive criticism # disallow all user-agents: User-agent: * or you can specify # the user-agents that are allowed, e.g webcrawler: User-agent: webcrawler User-agent: * Disallow: /automail Disallow: /admin