# Reference: http://www.robotstxt.org/wc/faq.html # Validator: http://www.searchengineworld.com/cgi-bin/robotcheck.cgi # User-agent: * Allow: /admin/searches/ Allow: /dept/csbr/pdf/ Allow: /dept/csbr/ Allow: /dept/csbr/index.html Allow: /csbr Disallow: /faculty/ Disallow: /images/ Disallow: /include/ Disallow: /movies/ Disallow: /tech/cams/ Disallow: /tech/docs/ Disallow: /tech/labs/ Disallow: /tech/csbs/mmclass/ Disallow: /users/turner/ # Reference: http://www.google.com/webmasters/faq.html#nocrawl # #User-agent: Googlebot #Disallow: /*.js$