--- a/web-site.rst	Tue Nov 16 22:39:51 2010 +0200
+++ b/web-site.rst	Thu Nov 18 20:00:39 2010 +0200
@@ -4,3 +4,36 @@
 
   http://developer.yahoo.com/performance/rules.html
 
+* robots.txt.
+
+To exclude all robots from the entire server
+
+  User-agent: *
+  Disallow: /
+
+To exclude all robots from part of the server:
+
+  User-agent: *
+  Disallow: /cgi-bin/
+  Disallow: /tmp/
+  Disallow: /junk/
+
+To allow a single robot:
+
+  User-agent: Google
+  Disallow:
+
+  User-agent: *
+  Disallow: /
+
+To allow all robots complete access:
+
+  User-agent: *
+  Disallow:
+
+  http://www.robotstxt.org/
+                home page
+  http://www.robotstxt.org/robotstxt.html
+                About /robots.txt
+
+