# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
#
# To ban all spiders from the entire site uncomment the next two lines:
# User-Agent: *
# Disallow: /

# Disallow alexa crawler since it doesn't handle rails URLs properly
User-Agent: ia_archiver
User-Agent: ia_archiver (+http://www.alexa.com/site/help/webmasters; crawler@alexa.com)
Disallow: /