-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
🔧 make all config dynamic, create taskfile and add index task
- Loading branch information
1 parent
f34a8e7
commit 282ef4c
Showing
7 changed files
with
193 additions
and
75 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,169 @@ | ||
| version: "3" | ||
|
|
||
| vars: | ||
| ENV: "{{.ENV}}" | ||
| NAMESPACE: | ||
| sh: echo "hmpps-delius-alfrsco-{{.ENV}}" | ||
| BUCKET_NAME: | ||
| sh: kubectl get secret s3-bucket-output -n {{.NAMESPACE}} -o jsonpath='{.data.BUCKET_NAME}' | base64 --decode | ||
| OPEN_SEARCH_PREFIX: | ||
| sh: kubectl get svc --namespace {{.NAMESPACE}} | grep 'opensearch-proxy-service-cloud-platform' | awk '{ print $1 }' | ||
| OPENSEARCH_HOST: | ||
| sh: echo "{{.OPEN_SEARCH_PREFIX}}.{{.NAMESPACE}}.svc.cluster.local" | ||
| ALLOWLIST: | ||
| sh: yq 'join(",")' ./kustomize/{{.ENV}}/allowlist.yaml | ||
| RDS_JDBC_URL: | ||
| sh: kubectl get secrets rds-instance-output --namespace {{.NAMESPACE}} -o json | jq -r ".data | map_values(@base64d) | .RDS_JDBC_URL" | ||
| DEBUG: "false" | ||
| DEBUG_FLAG: | ||
| sh: if [ "{{.DEBUG}}" = "true" ]; then echo "--debug"; else echo ""; fi | ||
| HELM_POST_RENDERER_ARGS: | ||
| sh: if [ "{{.DEBUG}}" = "true" ]; then echo "-d true"; else echo "-d false"; fi | ||
| ATOMIC: "true" | ||
| ATOMIC_FLAG: | ||
| sh: if [ "{{.ATOMIC}}" = "true" ]; then echo "--atomic"; else echo ""; fi | ||
| CHART_VERSION: "6.0.2" | ||
|
|
||
| tasks: | ||
| helm_upgrade: | ||
| cmds: | ||
| - echo "NAMESPACE set to {{.NAMESPACE}}" | ||
| - echo "BUCKET_NAME set to {{.BUCKET_NAME}}" | ||
| - echo "OPEN_SEARCH_PREFIX set to {{.OPEN_SEARCH_PREFIX}}" | ||
| - echo "OPENSEARCH_HOST set to {{.OPENSEARCH_HOST}}" | ||
| - echo "RDS_JDBC_URL set to {{.RDS_JDBC_URL}}" | ||
| - echo "DEBUG set to {{.DEBUG}}" | ||
| - task: helm_repo_add | ||
| - task: update_allowlist | ||
| - task: helm_upgrade_install | ||
| vars: | ||
| NAMESPACE: "{{.NAMESPACE}}" | ||
| BUCKET_NAME: "{{.BUCKET_NAME}}" | ||
| OPENSEARCH_HOST: "{{.OPENSEARCH_HOST}}" | ||
| RDS_JDBC_URL: "{{.RDS_JDBC_URL}}" | ||
| DEBUG_FLAG: "{{.DEBUG_FLAG}}" | ||
| ATOMIC_FLAG: "{{.ATOMIC_FLAG}}" | ||
| - task: reset_allowlist | ||
| silent: true | ||
|
|
||
| prepare_namespace: | ||
| internal: true | ||
| cmds: | ||
| - | | ||
| export BUCKET_NAME=$(kubectl get secrets s3-bucket-output -o jsonpath='{.data.BUCKET_NAME}' | base64 -d) | ||
| if [ "${ENV}" = "poc" ]; then | ||
| export NAMESPACE=hmpps-delius-alfrsco-${ENV} | ||
| else | ||
| export NAMESPACE=hmpps-delius-alfresco-${ENV} | ||
| fi | ||
| export OPENSEARCH_PREFIX=$(kubectl get svc --namespace ${NAMESPACE} | grep 'opensearch-proxy-service-cloud-platform' | awk '{ print $1 }') | ||
| export OPENSEARCH_HOST=${OPENSEARCH_PREFIX}.${NAMESPACE}.svc.cluster.local | ||
| export RDS_JDBC_URL=$(kubectl get secrets rds-instance-output --namespace ${NAMESPACE} -o json | jq -r ".data | map_values(@base64d) | .RDS_JDBC_URL") | ||
| export EXTRACTED=$(yq 'join(",")' ./kustomize/${ENV}/allowlist.yaml) | ||
| echo "Using namespace: ${NAMESPACE}" | ||
| update_allowlist: | ||
| internal: true | ||
| dir: ./kustomize/{{.ENV}} | ||
| cmds: | ||
| - | | ||
| export ALLOWLIST={{.ALLOWLIST}} | ||
| yq '.metadata.annotations."nginx.ingress.kubernetes.io/whitelist-source-range" = strenv(ALLOWLIST)' -i ./patch-ingress-repository.yaml | ||
| yq '.metadata.annotations."nginx.ingress.kubernetes.io/whitelist-source-range" = strenv(ALLOWLIST)' -i ./patch-ingress-share.yaml | ||
| helm_repo_add: | ||
| internal: true | ||
| cmds: | ||
| - helm repo add alfresco https://kubernetes-charts.alfresco.com/stable --force-update | ||
|
|
||
| helm_upgrade_install: | ||
| internal: true | ||
| dir: ./kustomize/{{.ENV}} | ||
| cmds: | ||
| - | | ||
| helm upgrade --install alfresco-content-services alfresco/alfresco-content-services --version {{.CHART_VERSION}} --namespace {{.NAMESPACE}} \ | ||
| --values=../base/values.yaml --values=values.yaml \ | ||
| --set s3connector.config.bucketName={{.BUCKET_NAME}} \ | ||
| --set database.url={{.RDS_JDBC_URL}} \ | ||
| --set global.elasticsearch.host={{.OPENSEARCH_HOST}} \ | ||
| --set alfresco-search-enterprise.searchIndex.host={{.OPENSEARCH_HOST}} \ | ||
| --wait --timeout=20m \ | ||
| --post-renderer ../kustomizer.sh --post-renderer-args "{{.HELM_POST_RENDERER_ARGS}}" \ | ||
| {{.DEBUG_FLAG}} {{.ATOMIC_FLAG}} | ||
| reset_allowlist: | ||
| internal: true | ||
| dir: ./kustomize/{{.ENV}} | ||
| cmds: | ||
| - yq '.metadata.annotations."nginx.ingress.kubernetes.io/whitelist-source-range" = "placeholder"' -i patch-ingress-repository.yaml | ||
| - yq '.metadata.annotations."nginx.ingress.kubernetes.io/whitelist-source-range" = "placeholder"' -i patch-ingress-share.yaml | ||
|
|
||
| batch_reindex: | ||
| vars: | ||
| START: 0 | ||
| END: 10050 | ||
| CHUNK: 1000 | ||
| CONCURRENCY: 5 | ||
| HELM_ARRAY: | ||
| sh: | | ||
| yml=$( | ||
| for i in $(seq {{.START}} {{.CHUNK}} {{.END}}); do | ||
| new_start=$i | ||
| end=$((i + {{.CHUNK}} - 1)) | ||
| if [ $end -gt {{.END}} ]; then | ||
| end={{.END}} | ||
| fi | ||
| cat << EOF | ||
| - fromId: ${new_start} | ||
| toId: ${end} | ||
| EOF | ||
| done | ||
| ) | ||
| echo "$yml" | ||
| cmds: | ||
| - echo "Starting batch reindex from {{.START}} to {{.END}} in chunks of {{.CHUNK}}" | ||
| - task: run_reindex_batches | ||
| vars: | ||
| OPENSEARCH_HOST: "{{.OPENSEARCH_HOST}}" | ||
| NAMESPACE: "{{.NAMESPACE}}" | ||
| HELM_ARRAY: "{{.HELM_ARRAY}}" | ||
| CONCURRENCY: "{{.CONCURRENCY}}" | ||
|
|
||
| run_reindex_batches: | ||
| cmds: | ||
| - | | ||
| # Get the total number of items | ||
| export yaml_content="{{.HELM_ARRAY}}" | ||
| total_items=$(echo "$yaml_content" | yq eval 'length' -) | ||
| echo "Total items: $total_items" | ||
| # Loop through the items in batches of {{.CONCURRENCY}} | ||
| for ((i=0; i<total_items; i+={{.CONCURRENCY}})); do | ||
| echo "Batch $((i/{{.CONCURRENCY}} + 1)):" | ||
| yml_batch=$(echo "$yaml_content" | yq eval ".[$i:$((i+{{.CONCURRENCY}}))]" -) | ||
| echo "$yml_batch" | ||
| date=$(date '+%Y%m%d-%H%M%S') | ||
| release_name="reindex-$((i/{{.CONCURRENCY}} + 1))-${date}" | ||
| touch ./jobs/reindex/values-${release_name}.yaml | ||
| cat << EOF > ./jobs/reindex/values-${release_name}.yaml | ||
| reindex_batches: | ||
| $(echo "$yml_batch" | sed 's/^/ /' | sed 's/^ -/ -/') | ||
| EOF | ||
| cat ./jobs/reindex/values-${release_name}.yaml | ||
| echo "helm command: helm install $release_name jobs/reindex --set "global.elasticsearch.host={{.OPENSEARCH_HOST}}" --set "release_name=${release_name}" -f ./jobs/reindex/values.yaml -f ./jobs/reindex/values-${release_name}.yaml --namespace {{.NAMESPACE}}" | ||
| helm install "$release_name" jobs/reindex --set "global.elasticsearch.host={{.OPENSEARCH_HOST}}" --set "release_name=${release_name}" -f ./jobs/reindex/values.yaml -f ./jobs/reindex/values-${release_name}.yaml --namespace {{.NAMESPACE}} | ||
| kubectl wait --for=condition=complete jobs --namespace {{.NAMESPACE}} -l "reindex-job=$release_name" --timeout=600s | ||
| echo "Batch $((i/{{.CONCURRENCY}} + 1)) completed" | ||
| # Uninstall the job and remove the values file | ||
| helm uninstall "$release_name" --namespace {{.NAMESPACE}} | ||
| rm -f ./jobs/reindex/values-${release_name}.yaml | ||
| done | ||
| echo "All batches completed" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters