# robots.txt for Slashdot.org
# $Id$
# "Any empty [Disallow] value, indicates that all URLs can be retrieved.
# At least one Disallow field needs to be present in a record."

User-agent: Mediapartners-Google
Disallow:

User-agent: Googlebot
Crawl-delay: 100
Disallow: /red/
Disallow: /widescreen/
Disallow: /press/

User-agent: Slurp
Crawl-delay: 100
Disallow:

User-agent: Yahoo-NewsCrawler
Disallow:

User-Agent: msnbot
Crawl-delay: 100
Disallow:

User-agent: *
Crawl-delay: 100
Disallow:

Sitemap: http://www.the-mill.com/sitemap/sitemap.xml