From 284dfdee3a3bc7d3ff8303c9f6d2b2715725eb76 Mon Sep 17 00:00:00 2001 From: Stefan Wintermeyer Date: Fri, 23 Jan 2015 14:11:45 +0100 Subject: [PATCH] Added robots.txt --- robots.txt | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 robots.txt diff --git a/robots.txt b/robots.txt new file mode 100644 index 0000000..141f197 --- /dev/null +++ b/robots.txt @@ -0,0 +1,8 @@ +# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file +# +# To ban all spiders from the entire site uncomment the next two lines: +# User-Agent: * +# Disallow: / + +User-Agent: * +Disallow: /downloads