# All robots will spider the domain
User-agent: *
Disallow: /cgi-bin/
Disallow: /awstats/
Disallow: /head.htm
Disallow: /home.htm
Disallow: /ages/
Disallow: /mp/
Disallow: /vid/
Disallow: /robots.txt