Merge pull request #298 from alphagov/deny-crawler-non-prod

Deny web crawlers from scraping non production environments
alphagov · Sep 17, 2024 · 2bac5b0 · 2bac5b0
2 parents 715d74a + 8d4f415
commit 2bac5b0
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 1 deletion.
diff --git a/charts/app-of-apps/values-production.yaml b/charts/app-of-apps/values-production.yaml
@@ -52,6 +52,8 @@ ckanHelmValues:
       enabled: true
       name: ckan
       iamRoleARN: arn:aws:iam::172025368201:role/ckan-govuk
+    nginx:
+      denyCrawlers: false
   pycsw:
     replicaCount: 1
     args: ["gunicorn --bind 0.0.0.0:8000 wsgi:application --timeout 120"]

diff --git a/charts/ckan/values.yaml b/charts/ckan/values.yaml
@@ -83,7 +83,7 @@ ckan:
       create: true
       name: ""
     extraVolumeMounts: []
-    denyCrawlers: false
+    denyCrawlers: true
     safelist:
       # Used by: ckan.publishing.service.gov.uk
       # Used for: internationalisation of form controls