Adding changes to support latest swoop-config.yaml and conductor call…

…back functionality

charts/swoop-caboose/templates/swoop-caboose-configmap.yaml

@@ -0,0 +1,78 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Release.Name }}-configmap
+data:
+{{- $stacApiEndpoint := .Values.service.stacApiEndpoint -}}
+{{- if (hasKey .Values "global")  }}
+  {{- if (hasKey .Values.global "stac")  }}
+    {{- if (hasKey .Values.global.stac "endpoint")  }}
+      {{- $stacApiEndpoint := .Values.global.stac.endpoint -}}
+    {{- end -}}
+  {{- end -}}
+{{- end -}}
+  swoop-config.yaml: |
+    handlers:
+      argoHandler:
+        type: argoWorkflows
+        argoConf:
+          k8sOptions:
+            configOverrides:
+              context:
+                namespace: {{ .Release.Namespace | quote }}
+
+      publishStacItemHandler:
+        type: synchttp
+        parameters:
+          feature:
+            type: object
+            required:
+              - collection
+            properties:
+              collection:
+                type: string
+        request:
+          url: {{ $stacApiEndpoint }}/collections/{{`{{ .parameters.feature.collection }}`}}/items
+          method: POST
+          body: "{{`{{ .parameters.feature }}`}}"
+          headers:
+            Content-Type: "application/json"
+          followRedirects: true
+          responses:
+            - statusCode: 400
+              message: ".*timeout.*"
+              result: error
+            - statusCode: 404
+              result: fatal
+
+    conductors:
+      main:
+        handlers:
+          - argoHandler
+          - stacItemHandler
+
+    callbacks:
+      publishStacItem: &callbacksPublishStacItem
+        handler: publishStacItemHandler
+        type: perFeature
+        when:
+          - "successful"
+        parameters:
+          feature:
+            path: .feature
+        enabled: true
+
+    workflows:
+      mirror:
+        description: "A workflow to copy STAC items into a local mirror"
+        version: 2
+        handler: argoHandler
+        argoOpts:
+          template: workflowtemplate/mirror-workflow
+        cacheKeyHashIncludes:
+          - .features[].id
+          - .features[].collection
+        cacheKeyHashExcludes: []
+        callbacks:
+          publishStacItem:
+            <<: *callbacksPublishStacItem

charts/swoop-caboose/templates/swoop-caboose-deployment.yaml

@@ -194,6 +194,10 @@ spec:
           command: ["swoop", "caboose", "argo"]
           ports:
             - containerPort: {{ .Values.container.port }}
+          volumeMounts:
+            - name: swoop-config
+              mountPath: {{ $swoopConfigFile }}
+              subPath: swoop-config.yaml
           resources: {}
       restartPolicy: Always
 status: {}

charts/swoop-caboose/values.yaml

@@ -63,6 +63,7 @@ service:
   configFile: "/opt/swoop-go/fixtures/swoop-config.yml"
   name: swoop-caboose
   serviceAccount: argo
+  stacApiEndpoint: http://stac-fastapi-pgstac.default:8080
 deployment:
   name: swoop-caboose
 argoWorkflows:

charts/swoop-conductor/templates/swoop-conductor-configmap.yaml

@@ -3,99 +3,63 @@ kind: ConfigMap
 metadata:
   name: {{ .Release.Name }}-configmap
 data:
+{{- $stacApiEndpoint := .Values.service.stacApiEndpoint -}}
+{{- if (hasKey .Values "global")  }}
+  {{- if (hasKey .Values.global "stac")  }}
+    {{- if (hasKey .Values.global.stac "endpoint")  }}
+      {{- $stacApiEndpoint := .Values.global.stac.endpoint -}}
+    {{- end -}}
+  {{- end -}}
+{{- end -}}
   swoop-config.yaml: |
     handlers:
       argoHandler:
         type: argoWorkflows
         argoConf:
           k8sOptions:
-            # relative paths must be relative to cwd of process when it runs
-            # kubeconfig: ./kubeconfig.yaml
             configOverrides:
               context:
                 namespace: {{ .Release.Namespace | quote }}
 
-      cirrusHandler:
-        type: cirrus
-        cirrusConf:
-          sqsUrl: https://sqs.aws.com/0142354653636::sqs::queue
-
-      noopHandler:
-        type: noop
-        parameters:
-          workflowUuid:
-            type: string
-            format: uuid
-
-      # We believe it is best to re-template the request
-      # (at least with regard to secrets) JIT for every request execution
-      publishS3Handler:
-        type: http
-        backoff:
-          retries: 10
-          seconds: 5
-          factor: 2
-          max: 25
+      publishStacItemHandler:
+        type: synchttp
         parameters:
-          workflowName:
-            type: string
-            default: a_value
           feature:
             type: object
-        secrets:
-          # simplest approach for secret renewal is to have a ttl
-          # and periodically re-resolve the secret values, but does mean
-          # invalid secrets may be used for some time until the ttl expires
-          #
-          # how to handle secrets that need to resolve a token,
-          # have some other initialization step, or are of some built-in
-          # limited duration?
-          - name: user
-            type: file
-            path: /secrets-mount/username-secret
-            ttl: 1200
+            required:
+              - collection
+            properties:
+              collection:
+                type: string
         request:
-          url: https://{secrets.minio-user}:{secrets.minio-password}@our-minio:9000
+          url: {{ $stacApiEndpoint }}/collections/{{`{{ .parameters.feature.collection }}`}}/items
           method: POST
-          body: |
-            {
-              "fixed": "a_value",
-              "name": "{{`{{ .parameters.workflowName -}}`}}",
-              "date": "{{`{{ .parameters.feature.properties.datetime -}}`}}"
-            }
+          body: "{{`{{ .parameters.feature }}`}}"
           headers:
-            Authorization: "{{`Basic {{ .secrets.user }} {{ .secrets.password}}`}}"
             Content-Type: "application/json"
-            X-Workflow-Name: "{{`{{ .parameters.workflowName }}`}}"
           followRedirects: true
           responses:
-            # first matched wins
-            # by default any 2xx is success and anything else will be retried
-            - status: 400
-              message: ".*timed out.*"
-              fatal: false
-            - status: 400
-              fatal: true
+            - statusCode: 400
+              message: ".*timeout.*"
+              result: error
+            - statusCode: 404
+              result: fatal
 
     conductors:
-      instance-a:
+      main:
         handlers:
           - argoHandler
-          - publishS3Handler
-      instance-b: {}
+          - stacItemHandler
 
     callbacks:
-      publishS3Push: &callbacksPublishS3Push
-        handler: publishS3Handler
+      publishStacItem: &callbacksPublishStacItem
+        handler: publishStacItemHandler
         type: perFeature
         when:
           - "successful"
-        featureFilter: "@.id =~ 'fake*' & @.properties.gsd <= 0"
         parameters:
-          workflowName:
-            path: .input.features[0].id
           feature:
-            value: { "feature": true }
+            path: .feature
         enabled: true
 
     workflows:
@@ -110,39 +74,5 @@ data:
           - .features[].collection
         cacheKeyHashExcludes: []
         callbacks:
-          publishS3Push:
-            <<: *callbacksPublishS3Push
-          failed:
-            handler: noopHandler
-            type: single
-            when:
-              - "!successful"
-            parameters:
-              workflowUuid:
-                path: .workflow.uuid
-      cirrus-example:
-        callbacks:
-          publishS3Push:
-            <<: *callbacksPublishS3Push
-            when:
-              - "!failed"
-              - "successful"
-        title: "Cirrus example workflow"
-        description: "An example workflow config for a cirrus workflow"
-        version: 1
-        handler: cirrusHandler
-        cirrusOpts:
-          sfnArn: arn:aws:states:us-west-2:09876543210:stateMachine:cirrus-example
-        cacheKeyHashIncludes:
-          - .features[].id
-          - .features[].collection
-        cacheKeyHashExcludes: []
-        links:
-          - href: https://example.com/repo
-            rel: external
-            type: text/html
-            title: "source repository"
-          - href: https://example.com/docs
-            rel: external
-            type: text/html
-            title: "process documentation"
+          publishStacItem:
+            <<: *callbacksPublishStacItem

charts/swoop-conductor/values.yaml

@@ -63,7 +63,9 @@ service:
   configFile: "/opt/swoop-go/fixtures/swoop-config.yml"
   name: swoop-conductor
   serviceAccount: argo
-  instanceName: instance-a
+  instanceName: main
+  stacApiEndpoint: http://stac-fastapi-pgstac.default:8080
+
 deployment:
   name: swoop-conductor
 argoWorkflows: