# HG changeset patch
# User Oleksandr Gavenko <gavenkoa@gmail.com>
# Date 1290103239 -7200
# Node ID fae2370e94fd9dca5fa5dd993646892dd5abb57c
# Parent  4ce0f742db1a11bf329e2b708a434731a467b0ef
robots.txt

diff -r 4ce0f742db1a -r fae2370e94fd web-site.rst
--- a/web-site.rst	Tue Nov 16 22:39:51 2010 +0200
+++ b/web-site.rst	Thu Nov 18 20:00:39 2010 +0200
@@ -4,3 +4,36 @@
 
   http://developer.yahoo.com/performance/rules.html
 
+* robots.txt.
+
+To exclude all robots from the entire server
+
+  User-agent: *
+  Disallow: /
+
+To exclude all robots from part of the server:
+
+  User-agent: *
+  Disallow: /cgi-bin/
+  Disallow: /tmp/
+  Disallow: /junk/
+
+To allow a single robot:
+
+  User-agent: Google
+  Disallow:
+
+  User-agent: *
+  Disallow: /
+
+To allow all robots complete access:
+
+  User-agent: *
+  Disallow:
+
+  http://www.robotstxt.org/
+                home page
+  http://www.robotstxt.org/robotstxt.html
+                About /robots.txt
+
+