# small robots.txt # more information about this file can be found at # http://www.robotstxt.org/ # lines beginning with the pund ("#") sign are comments and can be deleted. # if case your drupal site is in a directory # lower than your docroot (e.g. /drupal) # please add this before the /-es below # to stop a polite robot indexing an example dir # add a line like (delete the #'s) # user-agent: polite-bot # Disallow: /example-dir/ User-agent: * Crawl-Delay: 10 Disallow: /node/add Disallow: /node/*/edit # Directories Disallow: /database/ Disallow: /includes/ Disallow: /misc/ Disallow: /modules/ Disallow: /sites/ Disallow: /themes/ Disallow: /scripts/ Disallow: /updates/ Disallow: /profiles/ Disallow: /tracker/ Disallow: /xtracker/ # Disallow: /user/ Disallow: /myuserpoints/* Disallow: /book/export/ Disallow: /book/print Disallow: /forward/ Disallow: /comment/ Disallow: /taxonomy/ Disallow: /popular/ Disallow: */feed* Disallow: */comment/reply/ Disallow: /popular/* Disallow: /comments* Disallow: /frontpage* Disallow: /comments* Disallow: /aggregator* Disallow: /aggregator2* Disallow: *comment-* Disallow: *?sort=asc&order=Time* Disallow: *?filter0=* Disallow: *?page=* Disallow: /?q= # Files Disallow: /files Disallow: /rss.xml Disallow: /feed Disallow: /xmlrpc.php Disallow: /cron.php Disallow: /update.php Disallow: /install.php Disallow: /INSTALL.mysql.txt Disallow: /INSTALL.pgsql.txt Disallow: /CHANGELOG.txt Disallow: /MAINTAINERS.txt Disallow: /LICENSE.txt Disallow: /UPGRADE.txt # Paths (clean URLs) Disallow: /admin/ Disallow: /comment/reply/ Disallow: /contact/ Disallow: /logout/ Disallow: /node/add/ Disallow: /search/ Disallow: /user/register/ Disallow: /user/password/ Disallow: /user/login/ Disallow: /ka2* # Views and Forum module problem: Disallow: /\*sort= # Favorite nodes: Disallow: /favorite_nodes/* # Allow images indexing: Allow: /sites/default/files/images Allow: /sites/default/files/imagecache Allow: /sites/default/files/photogal