Update robots.txt

ethanniser · Oct 22, 2024 · 6f084ef · 6f084ef
1 parent 594e64a
commit 6f084ef
Showing 1 changed file with 6 additions and 2 deletions.
diff --git a/src/app/robots.txt b/src/app/robots.txt
@@ -1,6 +1,10 @@
-# Block all crawlers from accessing the site
+# Block all crawlers from accessing /products
 User-agent: *
-Disallow: /
+Disallow: /products
+
+# Allow crawling of the homepage
+User-agent: *
+Allow: /
 
 # Allow Twitter to fetch Open Graph data
 User-agent: Twitterbot