From 27c02f5ef5346b44a0449d9b0cebeebb99b3b182 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 22 Sep 2015 14:16:58 +0100 Subject: [PATCH] added some robots to the blocked list --- services/web/public/robots.txt | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/services/web/public/robots.txt b/services/web/public/robots.txt index 4b7aa5be91..071ef0cddc 100644 --- a/services/web/public/robots.txt +++ b/services/web/public/robots.txt @@ -3,3 +3,16 @@ User-agent: * Disallow: /project/* Disallow: /recurly.com +Allow: / + +User-Agent: AhrefsBot +Disallow: / + +User-Agent: XoviBot +Disallow: / + +User-Agent: RankSonicBot +Disallow: / + +User-Agent: SMTBot +Disallow: / \ No newline at end of file