Merge pull request #57 from ethanniser/RhysSullivan-patch-2

robots 2 electric boogaloo
ethanniser · Oct 22, 2024 · f685659 · f685659
2 parents 594e64a + 393bdcc
commit f685659
Showing 1 changed file with 12 additions and 5 deletions.
diff --git a/src/app/robots.txt b/src/app/robots.txt
@@ -1,15 +1,22 @@
-# Block all crawlers from accessing the site
+# Block all crawlers from accessing /products
 User-agent: *
-Disallow: /
+Disallow: /products
 
-# Allow Twitter to fetch Open Graph data
+# Allow Twitter to fetch Open Graph data from /products
 User-agent: Twitterbot
+Allow: /products
 Allow: /
 
-# Allow Facebook to fetch Open Graph data
+# Allow Facebook to fetch Open Graph data from /products
 User-agent: facebookexternalhit
+Allow: /products
 Allow: /
 
-# Allow LinkedIn to fetch Open Graph data
+# Allow LinkedIn to fetch Open Graph data from /products
 User-agent: LinkedInBot
+Allow: /products
+Allow: /
+
+# Allow crawling of the homepage for all crawlers
+User-agent: *
 Allow: /