publish task (#109)

Element84 · Sep 1, 2023 · 3a995bf · 3a995bf
1 parent ccb24cf
commit 3a995bf
Show file tree

Hide file tree

Showing 4 changed files with 95 additions and 3 deletions.
diff --git a/charts/workflow-config/templates/copy-stac-asset-s3-secret.yaml b/charts/workflow-config/templates/copy-stac-asset-s3-secret.yaml
@@ -9,4 +9,6 @@ data:
   secret_access_key: {{ .Values.s3.secretAccessKey }}
   region: {{ .Values.s3.region }}
   session_token: {{ .Values.s3.sessionToken }}
+  data_bucket: {{ .Values.s3.data_bucket }}
+
 {{- end }}
diff --git a/charts/workflow-config/templates/tasks/publish-template.yaml b/charts/workflow-config/templates/tasks/publish-template.yaml
@@ -0,0 +1,73 @@
+apiVersion: argoproj.io/v1alpha1
+kind: WorkflowTemplate
+metadata:
+  name: publish-template
+spec:
+# Start of variable mappings
+{{- $accessKeyIdSecretName := .Values.s3.accessKeyIdSecret.name -}}
+{{- $accessKeyIdSecretKey := .Values.s3.accessKeyIdSecret.key -}}
+{{- $secretAccessKeySecretName := .Values.s3.secretAccessKeySecret.name -}}
+{{- $secretAccessKeySecretKey := .Values.s3.secretAccessKeySecret.key -}}
+{{- $regionSecretName := .Values.s3.regionSecret.name -}}
+{{- $regionSecretKey := .Values.s3.regionSecret.key -}}
+{{- $sessionTokenSecretName := .Values.s3.sessionTokenSecret.name -}}
+{{- $sessionTokenSecretKey := .Values.s3.sessionTokenSecret.key -}}
+{{- $dataBucketSecretName := .Values.s3.dataBucketSecret.name -}}
+{{- $dataBucketSecretKey := .Values.s3.dataBucketSecret.key -}}
+{{- $serviceAccountName := .Values.publishTemplateTaskServiceAccountName -}}
+# End of global variable mappings
+  serviceAccountName: {{ $serviceAccountName }}
+  templates:
+    - name: publish
+      inputs:
+        artifacts:
+          - name: input
+            path: /mnt/input.json
+      outputs:
+        artifacts:
+          - name: output
+            path: /mnt/output.json
+            archive:
+              none: {}
+            s3:
+              key: "executions/{{`{{workflow.name}}`}}/output.json"
+      container:
+        env:
+        - name: AWS_ACCESS_KEY_ID
+          valueFrom:
+            secretKeyRef:
+              name:  {{ $accessKeyIdSecretName }}
+              key: {{ $accessKeyIdSecretKey }}
+        - name: AWS_SECRET_ACCESS_KEY
+          valueFrom:
+            secretKeyRef:
+              name: {{ $secretAccessKeySecretName }}
+              key: {{ $secretAccessKeySecretKey }}
+        {{- if .Values.s3.region }}
+        - name: AWS_DEFAULT_REGION
+          valueFrom:
+            secretKeyRef:
+              name: {{ $regionSecretName }}
+              key: {{ $regionSecretKey }}
+        {{- end }}
+        {{- if .Values.s3.sessionToken }}
+        - name: AWS_SESSION_TOKEN
+          valueFrom:
+            secretKeyRef:
+              name: {{ $sessionTokenSecretName }}
+              key: {{ $sessionTokenSecretKey }}
+        {{- end }}
+        - name: SWOOP_DATA_BUCKET
+          valueFrom:
+            secretKeyRef:
+              name: {{ $dataBucketSecretName }}
+              key: {{ $dataBucketSecretKey }}
+
+        # should pin to a specific version or label here
+        image: quay.io/element84/publish-stac-task:latest
+        imagePullPolicy: IfNotPresent
+        args:
+          - run
+          - /mnt/input.json
+          - --output
+          - /mnt/output.json
diff --git a/charts/workflow-config/templates/workflows/mirror-workflow.yaml b/charts/workflow-config/templates/workflows/mirror-workflow.yaml
@@ -9,10 +9,10 @@ spec:
 # Start of variable mappings
 {{- $serviceAccountName := .Values.mirrorWorkflowServiceAccountName -}}
 # End of global variable mappings
-  entrypoint: copy-assets
+  entrypoint: workflow
   serviceAccountName: {{ $serviceAccountName }} # set this so the proper permissions are assigned
   templates:
-    - name: copy-assets
+    - name: workflow
       dag:
         tasks:
           - name: copy-assets
@@ -25,4 +25,16 @@ spec:
                   path: /mnt/input.json
                   s3:
                     key: "executions/{{`{{workflow.name}}`}}/input.json"
-
+          - name: publish
+            dependencies: [copy-assets]
+            templateRef:
+              name: publish-template
+              template: publish
+            arguments:
+              artifacts:
+              - name: input
+                from: "{{`{{tasks.copy-assets.outputs.artifacts.output}}`}}"
+      outputs:
+        artifacts:
+          - name: output
+            from: {{`"{{tasks.publish.outputs.artifacts.output}}"`}}
diff --git a/charts/workflow-config/values.yaml b/charts/workflow-config/values.yaml
@@ -6,6 +6,7 @@ s3:
   accessKeyId: YWNjZXNza2V5aWQ=
   secretAccessKey: c2VjcmV0YWNjZXNza2V5
   region: cmVnaW9u
+  data_bucket: YnVja2V0
   sessionToken:
   accessKeyIdSecret:
     name: copy-stac-asset-s3-secret
@@ -19,6 +20,9 @@ s3:
   sessionTokenSecret:
     name: copy-stac-asset-s3-secret
     key: session_token
+  dataBucketSecret:
+    name: copy-stac-asset-s3-secret
+    key: data_bucket
 
 minio:
   service:
@@ -34,3 +38,4 @@ minio:
 serviceAccountName: argo
 mirrorWorkflowServiceAccountName: argo
 copyAssetsTemplateTaskServiceAccountName: argo
+publishTemplateTaskServiceAccountName: argo