Add option to create indices from existingConfigmap

Currently the Helm Chart supports creating new repositories via an existing ConfigMap. For deploying GraphDB as IaC this is quite useful. Unfortunately, there is no way to create indices at install time. This adds the option to provide a existing ConfigMap via values and create an index for one or multiple repositories. The ConfigMap has to consist out of keys that have the same name as the repository. The value is the SPARQL query.
Ontotext-AD · Jan 8, 2025 · 4eaa602 · 4eaa602
1 parent 8b55cb4
commit 4eaa602
Show file tree

Hide file tree

Showing 5 changed files with 152 additions and 0 deletions.
diff --git a/README.md b/README.md
@@ -387,6 +387,7 @@ IMPORTANT: This is generated by helm-docs, do not attempt modifying it on hand a
 | import.volumeMount.volumeClaimTemplate.name | string | `"import"` |  |
 | import.volumeMount.volumeClaimTemplate.spec.accessModes[0] | string | `"ReadWriteOnce"` |  |
 | import.volumeMount.volumeClaimTemplate.spec.resources.requests.storage | string | `"10Gi"` |  |
+| indices.existingConfigmap | string | `""` |  |
 | ingress.annotations | object | `{}` |  |
 | ingress.className | string | `""` |  |
 | ingress.enabled | bool | `true` |  |

diff --git a/files/scripts/graphdb.sh b/files/scripts/graphdb.sh
@@ -155,6 +155,44 @@ function cloudBackup {
   log "Backup ${BACKUP_NAME} completed successfully!"
 }
 
+function createIndicesFromFiles() {
+  local node_count=$1
+  local indicesConfigsLocation=$2
+  local success=true
+  local response
+  local repository
+
+  set -v
+
+  waitAllNodes "$node_count"
+
+  for filename in "${indicesConfigsLocation}"/*; do
+    repository="$(basename "$filename")"
+    response=$(
+      curl --connect-timeout 60 --retry 3 --retry-all-errors --retry-delay 10 \
+        -H "Authorization: Basic ${GRAPHDB_AUTH_TOKEN}" \
+        -H 'Accept: application/json' \
+        -H 'Content-Type: application/sparql-update' \
+        --data-binary @"${filename}" \
+        "http://${GRAPHDB_POD_NAME}-0.${GRAPHDB_SERVICE_NAME}:${GRAPHDB_SERVICE_PORT}/repositories/${repository}/statements"
+    )
+
+    if [ -z "$response" ]; then
+      log "Successfully created index for ${filename}"
+    else
+      log "Could not create index for ${filename}, response:"
+      log "$response"
+      if ! grep -q "already exists." <<<$response; then
+        success=false
+      fi
+    fi
+  done
+
+  if [ $success != true ]; then
+    exit 1
+  fi
+}
+
 function localBackup() {
   BACKUP_TIMESTAMP="$(date +'%Y-%m-%d_%H-%M-%S')"
   BACKUP_NAME="graphdb-backup-${BACKUP_TIMESTAMP}.tar"

diff --git a/templates/jobs/_labels.tpl b/templates/jobs/_labels.tpl
@@ -30,6 +30,10 @@ Helper functions for labels related to Job and provisioning resources
   {{- printf "%s-%s" (include "graphdb.fullname" .) "provision-repositories" -}}
 {{- end -}}
 
+{{- define "graphdb.fullname.job.provision-indices" -}}
+  {{- printf "%s-%s" (include "graphdb.fullname" .) "provision-indices" -}}
+{{- end -}}
+
 {{- define "graphdb.fullname.job.scale-down-cluster" -}}
   {{- printf "%s-%s" (include "graphdb.fullname" .) "scale-down-cluster" -}}
 {{- end -}}

diff --git a/templates/jobs/job-create-indices.yaml b/templates/jobs/job-create-indices.yaml
@@ -0,0 +1,99 @@
+{{- if .Values.indices.existingConfigmap }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ include "graphdb.fullname.job.provision-indices" . }}
+  namespace: {{ include "graphdb.namespace" . }}
+  labels:
+    {{- include "graphdb.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": post-install, post-upgrade, post-rollback
+    "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed
+    "helm.sh/hook-weight": "5"
+    {{- with .Values.annotations }}
+      {{- tpl (toYaml .) $ | nindent 4 }}
+    {{- end }}
+spec:
+  backoffLimit: {{ .Values.jobs.backoffLimit }}
+  ttlSecondsAfterFinished: {{ .Values.jobs.ttlSecondsAfterFinished }}
+  template:
+    spec:
+      restartPolicy: Never
+      automountServiceAccountToken: false
+      {{- if .Values.jobs.schedulerName }}
+      schedulerName: {{ .Values.jobs.schedulerName }}
+      {{- end }}
+      {{- with .Values.jobs.dnsConfig }}
+      dnsConfig: {{ toYaml . | nindent 8 }}
+      {{- end }}
+      {{- if .Values.jobs.dnsPolicy }}
+      dnsPolicy: {{ .Values.jobs.dnsPolicy }}
+      {{- end }}
+      {{- if .Values.jobs.priorityClassName }}
+      priorityClassName: {{ .Values.jobs.priorityClassName }}
+      {{- end }}
+      {{- with .Values.jobs.nodeSelector }}
+      nodeSelector: {{- tpl (toYaml .) $ | nindent 8 }}
+      {{- end }}
+      {{- with .Values.jobs.affinity }}
+      affinity: {{- tpl (toYaml .) $ | nindent 8 }}
+      {{- end }}
+      {{- with .Values.jobs.tolerations }}
+      tolerations: {{- tpl (toYaml .) $ | nindent 8 }}
+      {{- end }}
+      {{- with .Values.jobs.topologySpreadConstraints }}
+      topologySpreadConstraints: {{- tpl (toYaml .) $ | nindent 8 }}
+      {{- end }}
+      {{- if or .Values.global.imagePullSecrets .Values.image.pullSecrets }}
+      imagePullSecrets:
+        {{- include "graphdb.combinedImagePullSecrets" . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.jobs.podSecurityContext }}
+      securityContext: {{ toYaml . | nindent 8 }}
+      {{- end }}
+      volumes:
+        - name: job-temp
+          emptyDir: {{ .Values.jobs.persistence.emptyDir | toYaml | nindent 12 }}
+        - name: indices-config
+          configMap:
+            name: {{ .Values.indices.existingConfigmap }}
+        - name: graphdb-utils
+          configMap:
+            name: {{ include "graphdb.fullname.configmap.utils" . }}
+      containers:
+        - name: provision-indices
+          image: {{ include "graphdb.image" . }}
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          env:
+            - name: GRAPHDB_POD_NAME
+              value: {{ include "graphdb.fullname" . }}
+            - name: GRAPHDB_SERVICE_NAME
+              value: {{ include "graphdb.fullname.service.headless" . }}
+            - name: GRAPHDB_SERVICE_PORT
+              value: {{ .Values.headlessService.ports.http | quote }}
+            - name: GRAPHDB_AUTH_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: {{ (tpl .Values.security.provisioner.existingSecret .) | default (include "graphdb.fullname.secret.provisioning-user" .) }}
+                  key: {{ .Values.security.provisioner.tokenKey }}
+          {{- with .Values.jobs.securityContext }}
+          securityContext: {{ toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.jobs.resources }}
+          resources: {{ toYaml . | nindent 12 }}
+          {{- end }}
+          volumeMounts:
+            - name: job-temp
+              mountPath: /tmp
+            - name: graphdb-utils
+              mountPath: /tmp/utils
+            - name: indices-config
+              mountPath: /tmp/indices-config
+          workingDir: /tmp
+          command: ["bash"]
+          args:
+            - "/tmp/utils/graphdb.sh"
+            - "createIndicesFromFiles"
+            - "{{ .Values.replicas }}"
+            - "/tmp/indices-config"
+{{- end }}
diff --git a/values.yaml b/values.yaml
@@ -969,6 +969,16 @@ repositories:
   # Each key in the existing configmap will be treated as the config.ttl of a repository.
   existingConfigmap: ""
 
+################################
+# Custom Lucene Configurations #
+################################
+
+indices:
+  # Optional configmap containing indices configuration .ttl file(s).
+  # GraphDB will automatically create indices with the provided repositories configuration files.
+  # Each key in the existing configmap will be treated as the config.ttl of a repository.
+  existingConfigmap: ""
+
 ################################################
 # Persistent Volume Permissions Configurations #
 ################################################