From 65719fbf40d96510a6201119b91dcdd1231f5d3a Mon Sep 17 00:00:00 2001 From: kentsanggds Date: Tue, 17 Sep 2024 16:57:36 +0100 Subject: [PATCH 1/2] Deny web crawlers by default from scraping ckan --- charts/ckan/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/ckan/values.yaml b/charts/ckan/values.yaml index ccebc08d..dec29f53 100644 --- a/charts/ckan/values.yaml +++ b/charts/ckan/values.yaml @@ -83,7 +83,7 @@ ckan: create: true name: "" extraVolumeMounts: [] - denyCrawlers: false + denyCrawlers: true safelist: # Used by: ckan.publishing.service.gov.uk # Used for: internationalisation of form controls From 8d4f415767113160bdc7ac272bef3fa2ec109325 Mon Sep 17 00:00:00 2001 From: kentsanggds Date: Tue, 17 Sep 2024 16:58:00 +0100 Subject: [PATCH 2/2] Allow web crawlers on ckan production --- charts/app-of-apps/values-production.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/charts/app-of-apps/values-production.yaml b/charts/app-of-apps/values-production.yaml index b0423362..f3310d70 100644 --- a/charts/app-of-apps/values-production.yaml +++ b/charts/app-of-apps/values-production.yaml @@ -52,6 +52,8 @@ ckanHelmValues: enabled: true name: ckan iamRoleARN: arn:aws:iam::172025368201:role/ckan-govuk + nginx: + denyCrawlers: false pycsw: replicaCount: 1 args: ["gunicorn --bind 0.0.0.0:8000 wsgi:application --timeout 120"]