User-agent: *
Disallow: /cgi-bin/
#
User-agent: Googlebot
Disallow: /cgi-bin/
#
User-agent: msnbot
Disallow: /cgi-bin/
#
User-agent: Slurp
Disallow: /cgi-bin/
#
# allow W3C
## User-agent: W3C-checklink
## Disallow:
#
# block the following bots
User-agent: MJ12bot
Disallow: /
User-agent: ICCrawler - ICjobs
Disallow: /
User-agent: jobs.de-Robot
Disallow: /
User-agent: cometrics-bot
Disallow: /
User-agent: gonzo2
Disallow: /
User-agent: gonzo2[P]
Disallow: /
User-agent: Openbot
Disallow: /
User-agent: sdcresearchlabs-testbot
Disallow: /
#