# robots.txt for search engines User-agent: BecomeBot Crawl-Delay: 30 User-agent:* Disallow: /cgi-bin/