home *** CD-ROM | disk | FTP | other *** search
/ linuxmafia.com 2016 / linuxmafia.com.tar / linuxmafia.com / jordan / robots.txt < prev    next >
Text File  |  2010-11-05  |  2KB  |  62 lines
  1. # $Id: robots.txt,v 1.9.2.1 2008/12/10 20:12:19 goba Exp $
  2. #
  3. # robots.txt
  4. #
  5. # This file is to prevent the crawling and indexing of certain parts
  6. # of your site by web crawlers and spiders run by sites like Yahoo!
  7. # and Google. By telling these "robots" where not to go on your site,
  8. # you save bandwidth and server resources.
  9. #
  10. # This file will be ignored unless it is at the root of your host:
  11. # Used:    http://example.com/robots.txt
  12. # Ignored: http://example.com/site/robots.txt
  13. #
  14. # For more information about the robots.txt standard, see:
  15. # http://www.robotstxt.org/wc/robots.html
  16. #
  17. # For syntax checking, see:
  18. # http://www.sxw.org.uk/computing/robots/check.html
  19.  
  20. User-agent: *
  21. Crawl-delay: 10
  22. # Directories
  23. Disallow: /includes/
  24. Disallow: /misc/
  25. Disallow: /modules/
  26. Disallow: /profiles/
  27. Disallow: /scripts/
  28. Disallow: /sites/
  29. Disallow: /themes/
  30. # Files
  31. Disallow: /CHANGELOG.txt
  32. Disallow: /cron.php
  33. Disallow: /INSTALL.mysql.txt
  34. Disallow: /INSTALL.pgsql.txt
  35. Disallow: /install.php
  36. Disallow: /INSTALL.txt
  37. Disallow: /LICENSE.txt
  38. Disallow: /MAINTAINERS.txt
  39. Disallow: /update.php
  40. Disallow: /UPGRADE.txt
  41. Disallow: /xmlrpc.php
  42. # Paths (clean URLs)
  43. Disallow: /admin/
  44. Disallow: /comment/reply/
  45. Disallow: /contact/
  46. Disallow: /logout/
  47. Disallow: /node/add/
  48. Disallow: /search/
  49. Disallow: /user/register/
  50. Disallow: /user/password/
  51. Disallow: /user/login/
  52. # Paths (no clean URLs)
  53. Disallow: /?q=admin/
  54. Disallow: /?q=comment/reply/
  55. Disallow: /?q=contact/
  56. Disallow: /?q=logout/
  57. Disallow: /?q=node/add/
  58. Disallow: /?q=search/
  59. Disallow: /?q=user/password/
  60. Disallow: /?q=user/register/
  61. Disallow: /?q=user/login/
  62.