# Robots.txt # All robots will spider the domain User-agent: * Disallow: /_notes Disallow: /cgi-bin Disallow: /images Disallow: /includes Disallow: /gallery2 Disallow: /geog Disallow: /video