From 6f084ef922577c30f87e2ac47f2ab6abfbb1a876 Mon Sep 17 00:00:00 2001
From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com>
Date: Tue, 22 Oct 2024 11:02:32 -0700
Subject: [PATCH 1/2] Update robots.txt

---
 src/app/robots.txt | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/app/robots.txt b/src/app/robots.txt
index 383664c..e22d78f 100644
--- a/src/app/robots.txt
+++ b/src/app/robots.txt
@@ -1,6 +1,10 @@
-# Block all crawlers from accessing the site
+# Block all crawlers from accessing /products
 User-agent: *
-Disallow: /
+Disallow: /products
+
+# Allow crawling of the homepage
+User-agent: *
+Allow: /
 
 # Allow Twitter to fetch Open Graph data
 User-agent: Twitterbot

From 393bdcc7a28258b273dea8ab6aa20cc46e7adaa0 Mon Sep 17 00:00:00 2001
From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com>
Date: Tue, 22 Oct 2024 11:04:06 -0700
Subject: [PATCH 2/2] Update robots.txt

---
 src/app/robots.txt | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/src/app/robots.txt b/src/app/robots.txt
index e22d78f..ab41d20 100644
--- a/src/app/robots.txt
+++ b/src/app/robots.txt
@@ -2,18 +2,21 @@
 User-agent: *
 Disallow: /products
 
-# Allow crawling of the homepage
-User-agent: *
-Allow: /
-
-# Allow Twitter to fetch Open Graph data
+# Allow Twitter to fetch Open Graph data from /products
 User-agent: Twitterbot
+Allow: /products
 Allow: /
 
-# Allow Facebook to fetch Open Graph data
+# Allow Facebook to fetch Open Graph data from /products
 User-agent: facebookexternalhit
+Allow: /products
 Allow: /
 
-# Allow LinkedIn to fetch Open Graph data
+# Allow LinkedIn to fetch Open Graph data from /products
 User-agent: LinkedInBot
+Allow: /products
+Allow: /
+
+# Allow crawling of the homepage for all crawlers
+User-agent: *
 Allow: /