# robots.txt for http://www.fredberardi.com/ User-agent: Speedy Disallow: /journal/ # User-agent: msnbot # Disallow: / # User-agent: slurp # Disallow: / User-agent: Googlebot Disallow: /.ged$ User-agent: GingerCrawler/1.0 Disallow:/ User-agent: GingerCrawler Disallow:/ User-agent: betaBot Disallow:/ User-agent: Linguee Bot Disallow:/ # To make all robots go away-decomment the two lines below to disallow ALL Robots # User-agent: * # Disallow: / User-agent: * Disallow: /rootsindex/ #too many html documents # Disallow: /images/ # This is a large image space and contains proprietary images Disallow: /journal/ # This is private rantings Disallow: /personal.html