diff --git a/src/app/robots.txt b/src/app/robots.txt index 383664c..e22d78f 100644 --- a/src/app/robots.txt +++ b/src/app/robots.txt @@ -1,6 +1,10 @@ -# Block all crawlers from accessing the site +# Block all crawlers from accessing /products User-agent: * -Disallow: / +Disallow: /products + +# Allow crawling of the homepage +User-agent: * +Allow: / # Allow Twitter to fetch Open Graph data User-agent: Twitterbot