#
# Nobody should archive any of these
#

User-agent: htdig
Disallow:


#
# Disallow all access by The Internet Archive
#

User-agent: ia_archiver
Disallow: /

# google search
User-agent: Googlebot
Disallow: /
Allow: /~ark/

# For now, disallow any web archiving outside of CS
User-agent: *
Disallow: /
Disallow: /images/
Disallow: /flash/
Disallow: /five_minutes
Disallow: /csdocs
Disallow: /usr/
Disallow: /gifs/
Disallow: /doc/
Disallow: /photo-album/
Disallow: /slurm.html