# # robots.txt # # This file is to prevent the crawling and indexing of certain parts # of your site by web crawlers and spiders run by sites like Yahoo! # and Google. By telling these "robots" where not to go on your site, # you save bandwidth and server resources. # # This file will be ignored unless it is at the root of your host. # # For more information about the robots.txt standard, see: # http://www.robotstxt.org/wc/robots.html # # For syntax checking, see: # http://www.sxw.org.uk/computing/robots/check.html # # Don't list hidden directories here. No need to inform bots about them. User-agent: * Crawl-delay: 10 # Directories # Bookstore Disallow: /bookstore2/ # Article manager Disallow: /artman2/css/ Disallow: /artman2/images/ Disallow: /artman2/javascript/ Disallow: /artman2/style/ Disallow: /artman2/tsc_pages/ Disallow: /artman2/uploads/ Disallow: /artman2/wysiwyg/ # Note: we do want to allow /artman2/publish/ # Basic website Disallow: /css/ Disallow: /images/ Disallow: /audio/ Disallow: /journal_article_images/ Disallow: /journal_cover_images/ Disallow: /media/ Disallow: /photos/ Disallow: /css/ Disallow: /scripts/ # Files # Paths