From e1aab2c4ecc206a725bf8e66e9585ac2879837ba Mon Sep 17 00:00:00 2001 From: George Taylor Date: Tue, 6 Aug 2024 16:09:09 +0100 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Helm=20chart=20configuration=20for?= =?UTF-8?q?=20Opensearch=20and=20enterprise=20search=20connector=20(#70)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * update values for opensearch * add makefile +chart changes * Update values.yaml * Update values.yaml * update makefile * Update values.yaml * Update values.yaml * Update values.yaml * vars * Update values.yaml * Update values.yaml * images * config map * Revert "config map" This reverts commit e714ada49d662efc0f1975ab82d74d1c380d5592. * Reapply "config map" This reverts commit c47acb8b104e93bacb2fe666b342cb1e5b9aa28d. * config map * images * Squashed commit of the following: commit 6cda4f5909b44b8079126ed1c9ba53260a6e53fd Merge: a682b79 ca20a8c Author: George Taylor Date: Tue Jul 16 15:03:01 2024 +0100 Merge pull request #75 from ministryofjustice/fix-ingress-share-real Revert share ingress to dedicated hostname + enable redirect to /share commit ca20a8ce61135215f6a9446366c1685e4a44ee39 Author: George Taylor Date: Tue Jul 16 13:15:37 2024 +0100 redirect to /share commit a682b79d5b3357fc69bbb51ec98451ff7e7f7391 Merge: 066cecc 1780e23 Author: George Taylor Date: Tue Jul 16 11:23:05 2024 +0100 Merge pull request #74 from ministryofjustice/remove-share-sub-domain Remove share subdomain commit 1780e23e1d16e1f930fb70bb673952aeb6f74ad4 Author: George Taylor Date: Tue Jul 16 11:21:18 2024 +0100 remove share subdomain commit 066cecce0b371960c18c44a1df65a20ae3f4f3f9 Merge: 4d23e4b b21f656 Author: George Taylor Date: Thu Jul 11 15:36:49 2024 +0100 Merge pull request #73 from ministryofjustice/workflow-changes Update cloud-platform-deploy-release.yml commit b21f656110c1056f3bc502a429952e67e4a0ccd4 Author: George Taylor Date: Thu Jul 11 15:35:57 2024 +0100 Update cloud-platform-deploy-release.yml commit 4d23e4b37106f7124e5bab722e0e685bc1574127 Merge: 6aeb967 ca1cfb2 Author: George Taylor Date: Thu Jul 11 15:09:57 2024 +0100 Merge pull request #72 from ministryofjustice/add-airflow-ips chore: Add templated directory to .gitignore and update IP whitelist in values files commit ca1cfb2c87cbf0cb5e1903bef89a637f91dd537c Author: George Taylor Date: Thu Jul 11 15:08:14 2024 +0100 chore: Add templated directory to .gitignore and update IP whitelist in values files commit 6aeb96763b76dba46d040a826b35cf5253788e6a Merge: f6a7c0e 540fe64 Author: Prem Basumatary <142879429+pbasumatary@users.noreply.github.com> Date: Fri Jul 5 09:10:27 2024 +0100 Merge pull request #71 from ministryofjustice/NIT-1305-custom-scaling-times NIT-1305 fix error in script commit 540fe64dded67378e5e22b9ecd24f6cf50805e26 Author: Prem Basumatary Date: Fri Jul 5 09:03:12 2024 +0100 NIT-1305 fix error in script * Update ingress-share.yaml * Use multi live indexing vals indexing Update values.yaml Update liveindexing-deployment.yaml Update values.yaml --- .../templates/prefixes-file-config-map.yaml | 78 +++++++++++++++++++ .../templates/reindexing-config.yaml | 10 +-- .../templates/reindexing-job.yaml | 17 ++++ alfresco-content-services/values.yaml | 69 +++++++++------- alfresco-content-services/values_poc.yaml | 2 +- makefile | 44 +++++++++++ 6 files changed, 188 insertions(+), 32 deletions(-) create mode 100644 alfresco-content-services/charts/alfresco-search-enterprise/templates/prefixes-file-config-map.yaml create mode 100644 makefile diff --git a/alfresco-content-services/charts/alfresco-search-enterprise/templates/prefixes-file-config-map.yaml b/alfresco-content-services/charts/alfresco-search-enterprise/templates/prefixes-file-config-map.yaml new file mode 100644 index 0000000..382c731 --- /dev/null +++ b/alfresco-content-services/charts/alfresco-search-enterprise/templates/prefixes-file-config-map.yaml @@ -0,0 +1,78 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: prefixes-file +data: + reindex.prefixes-file.json: | + { + "prefixUriMap":{ + "":"", + "http://www.alfresco.org/model/aos/1.0":"aos", + "http://www.alfresco.org/model/workflow/invite/nominated/1.0":"inwf", + "http://www.alfresco.org/model/solrfacetcustomproperty/1.0":"srftcustom", + "http://www.alfresco.org/model/datalist/1.0":"dl", + "http://www.alfresco.org/model/webdav/1.0":"webdav", + "http://www.alfresco.org/model/cmis/1.0/cs01ext":"cmisext", + "http://www.alfresco.org/model/distributionpolicies/1.0/model":"dp", + "http://www.alfresco.org/view/repository/1.0":"view", + "http://www.alfresco.org/model/download/1.0":"download", + "http://www.alfresco.org/model/publishing/twitter/1.0":"twitter", + "http://www.alfresco.org/model/action/1.0":"act", + "http://www.alfresco.org/system/registry/1.0":"reg", + "http://www.alfresco.org/model/user/1.0":"usr", + "http://www.alfresco.org/model/calendar":"ia", + "http://www.alfresco.org":"alf", + "http://www.alfresco.org/model/content/metadata/IPTCXMP/1.0":"iptcxmp", + "http://www.alfresco.org/model/application/1.0":"app", + "http://www.alfresco.org/model/surf/1.0":"surf", + "http://www.alfresco.org/model/versionstore/1.0":"ver", + "http://www.alfresco.org/system/modules/1.0":"module", + "http://www.alfresco.org/model/linksmodel/1.0":"lnk", + "http://iptc.org/std/Iptc4xmpExt/2008-02-29/":"Iptc4xmpExt", + "http://ns.adobe.com/photoshop/1.0/":"photoshop", + "http://www.alfresco.org/model/sync/1.0":"sync", + "http://ns.useplus.org/ldf/xmp/1.0/":"plus", + "http://www.alfresco.org/model/zaizi/gdpr/1.0":"gdpr", + "http://www.alfresco.org/model/remotecredentials/1.0":"rc", + "http://www.alfresco.org/model/emailserver/1.0":"emailserver", + "http://www.alfresco.org/model/hybridworkflow/1.0":"hwf", + "http://www.alfresco.org/model/sitecustomproperty/1.0":"stcp", + "http://www.alfresco.org/model/cmis/1.0/cs01":"cmis", + "http://www.alfresco.org/model/transfer/1.0":"trx", + "http://www.alfresco.org/model/rendition/1.0":"rn", + "http://www.alfresco.org/model/exif/1.0":"exif", + "http://www.alfresco.org/model/publishing/youtube/1.0":"youtube", + "http://ns.adobe.com/xap/1.0/rights/":"xmpRights", + "http://www.alfresco.org/model/custommodelmanagement/1.0":"cmm", + "http://www.alfresco.org/model/workflow/invite/moderated/1.0":"imwf", + "http://www.alfresco.org/model/forum/1.0":"fm", + "http://www.alfresco.org/model/rule/1.0":"rule", + "http://www.alfresco.org/model/publishing/linkedin/1.0":"linkedin", + "http://www.alfresco.org/model/publishing/slideshare/1.0":"slideshare", + "http://www.alfresco.org/model/system/1.0":"sys", + "http://www.alfresco.org/model/content/smartfolder/1.0":"smf", + "http://www.alfresco.org/model/zaizi/nomsspg/1.0":"nspg", + "http://www.alfresco.org/model/workflow/1.0":"wf", + "http://www.alfresco.org/model/qshare/1.0":"qshare", + "http://www.alfresco.org/model/versionstore/2.0":"ver2", + "http://www.alfresco.org/model/solrfacet/1.0":"srft", + "http://www.alfresco.org/model/audio/1.0":"audio", + "http://www.alfresco.org/model/blogintegration/1.0":"blg", + "http://www.alfresco.org/model/bpm/1.0":"bpm", + "http://www.alfresco.org/model/site/1.0":"st", + "http://www.alfresco.org/model/imap/1.0":"imap", + "http://www.alfresco.org/model/dictionary/1.0":"d", + "custom.model":"custom", + "http://www.alfresco.org/model/publishing/facebook/1.0":"facebook", + "http://www.alfresco.org/model/content/1.0":"cm", + "http://www.alfresco.org/model/cmis/custom":"cmiscustom", + "http://www.alfresco.org/model/devicesync/1.0":"devicesync", + "http://iptc.org/std/Iptc4xmpCore/1.0/xmlns/":"Iptc4xmpCore", + "http://www.alfresco.org/model/googledocs/2.0":"gd2", + "http://www.alfresco.org/model/publishing/flickr/1.0":"flickr", + "http://www.alfresco.org/model/workflow/resetpassword/1.0":"resetpasswordwf", + "http://www.alfresco.org/model/cmis/1.0/alfcmis":"alfcmis", + "http://www.alfresco.org/model/publishing/1.0":"pub", + "http://purl.org/dc/elements/1.1/":"dc" + } + } diff --git a/alfresco-content-services/charts/alfresco-search-enterprise/templates/reindexing-config.yaml b/alfresco-content-services/charts/alfresco-search-enterprise/templates/reindexing-config.yaml index 82e78be..94e2ea8 100644 --- a/alfresco-content-services/charts/alfresco-search-enterprise/templates/reindexing-config.yaml +++ b/alfresco-content-services/charts/alfresco-search-enterprise/templates/reindexing-config.yaml @@ -10,11 +10,11 @@ data: ALFRESCO_SHAREDFILESTORE_BASEURL: http://{{ template "alfresco.shortname" . }}-filestore:80/alfresco/api/-default-/private/sfs/versions/1/file/ ALFRESCO_ACCEPTEDCONTENTMEDIATYPESCACHE_BASEURL: http://{{ template "alfresco.shortname" . }}-router/transform/config ALFRESCO_REINDEX_PATHINDEXINGENABLED: {{ .Values.reindexing.pathIndexingEnabled | quote }} - {{- if .Values.reindexing.postgresql.url }} - SPRING_DATASOURCE_URL: {{ .Values.reindexing.postgresql.url }} - {{- else }} - SPRING_DATASOURCE_URL: jdbc:postgresql://{{ .Release.Name }}-{{ .Values.reindexing.postgresql.hostname }}:{{ .Values.reindexing.postgresql.port | default 5432 }}/{{ .Values.reindexing.postgresql.database }} - {{- end }} + # {{- if .Values.reindexing.postgresql.url }} + # SPRING_DATASOURCE_URL: {{ .Values.reindexing.postgresql.url }} + # {{- else }} + # SPRING_DATASOURCE_URL: jdbc:postgresql://{{ .Release.Name }}-{{ .Values.reindexing.postgresql.hostname }}:{{ .Values.reindexing.postgresql.port | default 5432 }}/{{ .Values.reindexing.postgresql.database }} + # {{- end }} {{- if .Values.reindexing.environment }} {{- range $key, $val := .Values.reindexing.environment }} {{ $key }}: {{ $val | quote }} diff --git a/alfresco-content-services/charts/alfresco-search-enterprise/templates/reindexing-job.yaml b/alfresco-content-services/charts/alfresco-search-enterprise/templates/reindexing-job.yaml index f223b16..c8fc4f6 100644 --- a/alfresco-content-services/charts/alfresco-search-enterprise/templates/reindexing-job.yaml +++ b/alfresco-content-services/charts/alfresco-search-enterprise/templates/reindexing-job.yaml @@ -29,6 +29,8 @@ spec: name: {{ template "alfresco-search-enterprise.fullName" . }}-reindexing-configmap - secretRef: name: {{ default (printf "%s-messagebroker-secret" (include "alfresco-search-enterprise.fullName" .)) .Values.messageBroker.existingSecretName }} + - secretRef: + name: {{ .Values.reindexing.postgresql.existingSecretName }} env: - name: SPRING_DATASOURCE_PASSWORD valueFrom: @@ -40,12 +42,27 @@ spec: secretKeyRef: name: {{ default (printf "%s-postgresql-secret" (include "alfresco-search-enterprise.fullName" $)) $.Values.reindexing.postgresql.existingSecretName }} key: DATABASE_USERNAME + - name: SPRING_DATASOURCE_URL + valueFrom: + secretKeyRef: + name: {{ default (printf "%s-postgresql-secret" (include "alfresco-search-enterprise.fullName" $)) $.Values.reindexing.postgresql.existingSecretName }} + key: RDS_JDBC_URL {{- include "spring.activemq.env" . | nindent 12 }} {{- include "alfresco-search-enterprise.config.spring.envCredentials" $ | nindent 12 }} + - name: ALFRESCO_REINDEX_PREFIXES_FILE + value: file:///alf/reindex.prefixes-file.json ports: - name: http containerPort: 8080 protocol: TCP + volumeMounts: + - name: prefixes-file-volume + mountPath: /alf/reindex.prefixes-file.json + subPath: reindex.prefixes-file.json + volumes: + - name: prefixes-file-volume + configMap: + name: prefixes-file initContainers: - name: wait-for-repository image: curlimages/curl:7.79.1 diff --git a/alfresco-content-services/values.yaml b/alfresco-content-services/values.yaml index bec04fa..2cba1af 100644 --- a/alfresco-content-services/values.yaml +++ b/alfresco-content-services/values.yaml @@ -29,7 +29,7 @@ repository: type: Recreate image: repository: 754256621582.dkr.ecr.eu-west-2.amazonaws.com/hmpps-migration/alfresco-content-repository - tag: 0.1.0-content-beta.1 + tag: content-9895398808-1 pullPolicy: IfNotPresent internalPort: 8080 hazelcastPort: 5701 @@ -59,6 +59,9 @@ repository: hostName: apps.live.cloud-platform.service.justice.gov.uk environment: JAVA_OPTS: >- + -Dindex.subsystem.name=elasticsearch + -Delasticsearch.indexName=alfresco + -Delasticsearch.createIndexIfNotExists=true -Dtransform.service.enabled=true -XX:MinRAMPercentage=50 -XX:MaxRAMPercentage=80 @@ -558,7 +561,7 @@ aiTransformer: # -- Declares the alfresco-shared-file-store used by the content repository # and transform service filestore: - replicaCount: 0 + replicaCount: 1 nodeSelector: {} image: repository: quay.io/alfresco/alfresco-shared-file-store @@ -634,7 +637,7 @@ share: image: # repository: quay.io/alfresco/alfresco-share repository: 754256621582.dkr.ecr.eu-west-2.amazonaws.com/hmpps-migration/alfresco-share - tag: 0.1.0-share-beta.3 + tag: share-9895398808-1 pullPolicy: IfNotPresent internalPort: 8080 service: @@ -712,12 +715,14 @@ messageBroker: &acs_messageBroker # -- Alternatively, provide credentials via an existing secret that contains # BROKER_URL, BROKER_USERNAME and BROKER_PASSWORD keys existingSecretName: null +alfresco-elasticsearch-connector: + enabled: false alfresco-search: searchServicesImage: repository: alfresco/alfresco-search-services tag: 2.0.7 nodeSelector: {} - enabled: true + enabled: false # If an external Solr service is to be used then enabled must be set to false # and external has to be configured accordingly. external: @@ -765,34 +770,46 @@ database: # When using embedded postgres you need to also set `postgresql.existingSecret`. existingSecretName: rds-instance-output alfresco-search-enterprise: - enabled: false - liveIndexing: - mediation: - image: - tag: 3.2.0 - content: - image: - tag: 3.2.0 - metadata: - image: - tag: 3.2.0 - path: - image: - tag: 3.2.0 + indexName: alfresco + enabled: true elasticsearch: # -- Enables the embedded elasticsearch cluster enabled: false messageBroker: existingSecretName: *acs_messageBroker_secretName + searchIndex: + # -- The host where service is available + host: opensearch-proxy-service-cloud-platform-62a206e0.hmpps-delius-alfrsco-poc.svc.cluster.local + # -- The port where service is available + port: 8080 + # -- Valid values are http or https + protocol: http + # -- The username required to access the service, if any + user: null + # -- The password required to access the service, if any + password: null reindexing: - image: - tag: 3.2.0 enabled: true postgresql: url: - hostname: postgresql-acs - database: alfresco - # existingSecretName: *acs_database_secretName + hostname: + database: + existingSecretName: rds-instance-output + image: + tag: 4.0.1 + liveIndexing: + mediation: + image: + tag: 4.0.1 + content: + image: + tag: 4.0.1 + metadata: + image: + tag: 4.0.1 + path: + image: + tag: 4.0.1 alfresco-digital-workspace: nodeSelector: {} enabled: false @@ -1088,13 +1105,13 @@ global: # s3Bucket: # comprehendRoleARN: # -- Shared connections details for Elasticsearch/Opensearch, required when - # alfresco-search-enterprise.enabled is true + # .enabled is true elasticsearch: # -- The host where service is available. The provided default is for when # elasticsearch.enabled is true - host: elasticsearch-master + host: opensearch-proxy-service-cloud-platform-62a206e0.hmpps-delius-alfrsco-poc.svc.cluster.local # -- The port where service is available - port: 9200 + port: 8080 # -- Valid values are http or https protocol: http # -- The username required to access the service, if any diff --git a/alfresco-content-services/values_poc.yaml b/alfresco-content-services/values_poc.yaml index 85c73a3..337dbef 100644 --- a/alfresco-content-services/values_poc.yaml +++ b/alfresco-content-services/values_poc.yaml @@ -1,6 +1,6 @@ # this file overrides values defined in ./values.yaml repository: - replicaCount: 2 + replicaCount: 1 share: replicaCount: 1 global: diff --git a/makefile b/makefile new file mode 100644 index 0000000..80de682 --- /dev/null +++ b/makefile @@ -0,0 +1,44 @@ +# Define the Helm chart name and release name +CHART_NAME := alfresco-content-services +VALUES := values.yaml +VALUES_ENV := values_$(ENV).yaml +DEBUG := false +ATOMIC := true + +# Helm upgrade/install command +helm_upgrade: + $(eval BUCKET_NAME := $(shell kubectl get secrets s3-bucket-output -o jsonpath='{.data.BUCKET_NAME}' | base64 -d)) + + @SECRET=$$(kubectl get secrets alfresco-content-services-alfresco-repository-properties-secret -o jsonpath='{.data.alfresco-global\.properties}' | base64 -d | awk '{print substr($$0, 19)}'); \ + if [ -z "$$SECRET" ]; then \ + SECRET=$$(openssl rand -base64 20); \ + fi; \ + if [ "$(ENV)" = "poc" ]; then \ + NAMESPACE=hmpps-delius-alfrsco-$(ENV); \ + else \ + NAMESPACE=hmpps-delius-alfresco-$(ENV); \ + fi; \ + echo "Using namespace: $${NAMESPACE}"; \ + if [ "$(DEBUG)" = "true" ]; then \ + DEBUG_FLAG="--debug"; \ + else \ + DEBUG_FLAG=""; \ + fi; \ + if [ "$(ATOMIC)" = "true" ]; then \ + ATOMIC_FLAG="--atomic"; \ + else \ + ATOMIC_FLAG=""; \ + fi; \ + echo "BUCKET_NAME: $(BUCKET_NAME)"; \ + helm upgrade --install $(CHART_NAME) ./$(CHART_NAME) --namespace $${NAMESPACE} \ + --values=./$(CHART_NAME)/$(VALUES) --values=./$(CHART_NAME)/$(VALUES_ENV) \ + --set s3connector.config.bucketName=$(BUCKET_NAME) \ + --set global.tracking.sharedsecret=$${SECRET} $${ATOMIC_FLAG} $${DEBUG_FLAG} --wait --timeout=20m + + +# Default target +.PHONY: default +default: helm_upgrade + +# Phony targets +.PHONY: helm_upgrade