From e74e79360c18d6ba72aff94699956efb8b84ed6f Mon Sep 17 00:00:00 2001 From: MaertHaekkinen Date: Fri, 25 Oct 2024 02:17:06 +0300 Subject: [PATCH] Add crawl-delay to robots.txt --- templates/robots.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/robots.txt b/templates/robots.txt index 7b604472d..e03099c45 100755 --- a/templates/robots.txt +++ b/templates/robots.txt @@ -1,4 +1,5 @@ User-agent: * +Crawl-delay: 10 Disallow: /facebook/ Disallow: /photo-upload-modal/ Disallow: /submit-dating/