From 85c2c5546796041c05ddac6dc7ad5063c72fadbc Mon Sep 17 00:00:00 2001 From: Hector Machin Date: Thu, 14 Sep 2023 17:20:17 -0400 Subject: [PATCH] Adding changes to support latest swoop-config.yaml and conductor callback functionality --- .../templates/swoop-caboose-configmap.yaml | 78 +++++++++++ .../templates/swoop-caboose-deployment.yaml | 4 + charts/swoop-caboose/values.yaml | 1 + .../templates/swoop-conductor-configmap.yaml | 128 ++++-------------- charts/swoop-conductor/values.yaml | 4 +- 5 files changed, 115 insertions(+), 100 deletions(-) create mode 100644 charts/swoop-caboose/templates/swoop-caboose-configmap.yaml diff --git a/charts/swoop-caboose/templates/swoop-caboose-configmap.yaml b/charts/swoop-caboose/templates/swoop-caboose-configmap.yaml new file mode 100644 index 0000000..9b1c47e --- /dev/null +++ b/charts/swoop-caboose/templates/swoop-caboose-configmap.yaml @@ -0,0 +1,78 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-configmap +data: +{{- $stacApiEndpoint := .Values.service.stacApiEndpoint -}} +{{- if (hasKey .Values "global") }} + {{- if (hasKey .Values.global "stac") }} + {{- if (hasKey .Values.global.stac "endpoint") }} + {{- $stacApiEndpoint := .Values.global.stac.endpoint -}} + {{- end -}} + {{- end -}} +{{- end -}} + swoop-config.yaml: | + handlers: + argoHandler: + type: argoWorkflows + argoConf: + k8sOptions: + configOverrides: + context: + namespace: {{ .Release.Namespace | quote }} + + publishStacItemHandler: + type: synchttp + parameters: + feature: + type: object + required: + - collection + properties: + collection: + type: string + request: + url: {{ $stacApiEndpoint }}/collections/{{`{{ .parameters.feature.collection }}`}}/items + method: POST + body: "{{`{{ .parameters.feature }}`}}" + headers: + Content-Type: "application/json" + followRedirects: true + responses: + - statusCode: 400 + message: ".*timeout.*" + result: error + - statusCode: 404 + result: fatal + + conductors: + main: + handlers: + - argoHandler + - stacItemHandler + + callbacks: + publishStacItem: &callbacksPublishStacItem + handler: publishStacItemHandler + type: perFeature + when: + - "successful" + parameters: + feature: + path: .feature + enabled: true + + workflows: + mirror: + description: "A workflow to copy STAC items into a local mirror" + version: 2 + handler: argoHandler + argoOpts: + template: workflowtemplate/mirror-workflow + cacheKeyHashIncludes: + - .features[].id + - .features[].collection + cacheKeyHashExcludes: [] + callbacks: + publishStacItem: + <<: *callbacksPublishStacItem diff --git a/charts/swoop-caboose/templates/swoop-caboose-deployment.yaml b/charts/swoop-caboose/templates/swoop-caboose-deployment.yaml index df0c4d0..0a19d80 100644 --- a/charts/swoop-caboose/templates/swoop-caboose-deployment.yaml +++ b/charts/swoop-caboose/templates/swoop-caboose-deployment.yaml @@ -194,6 +194,10 @@ spec: command: ["swoop", "caboose", "argo"] ports: - containerPort: {{ .Values.container.port }} + volumeMounts: + - name: swoop-config + mountPath: {{ $swoopConfigFile }} + subPath: swoop-config.yaml resources: {} restartPolicy: Always status: {} diff --git a/charts/swoop-caboose/values.yaml b/charts/swoop-caboose/values.yaml index df03f41..761044f 100644 --- a/charts/swoop-caboose/values.yaml +++ b/charts/swoop-caboose/values.yaml @@ -63,6 +63,7 @@ service: configFile: "/opt/swoop-go/fixtures/swoop-config.yml" name: swoop-caboose serviceAccount: argo + stacApiEndpoint: http://stac-fastapi-pgstac.default:8080 deployment: name: swoop-caboose argoWorkflows: diff --git a/charts/swoop-conductor/templates/swoop-conductor-configmap.yaml b/charts/swoop-conductor/templates/swoop-conductor-configmap.yaml index 934952b..9b1c47e 100644 --- a/charts/swoop-conductor/templates/swoop-conductor-configmap.yaml +++ b/charts/swoop-conductor/templates/swoop-conductor-configmap.yaml @@ -3,99 +3,63 @@ kind: ConfigMap metadata: name: {{ .Release.Name }}-configmap data: +{{- $stacApiEndpoint := .Values.service.stacApiEndpoint -}} +{{- if (hasKey .Values "global") }} + {{- if (hasKey .Values.global "stac") }} + {{- if (hasKey .Values.global.stac "endpoint") }} + {{- $stacApiEndpoint := .Values.global.stac.endpoint -}} + {{- end -}} + {{- end -}} +{{- end -}} swoop-config.yaml: | handlers: argoHandler: type: argoWorkflows argoConf: k8sOptions: - # relative paths must be relative to cwd of process when it runs - # kubeconfig: ./kubeconfig.yaml configOverrides: context: namespace: {{ .Release.Namespace | quote }} - cirrusHandler: - type: cirrus - cirrusConf: - sqsUrl: https://sqs.aws.com/0142354653636::sqs::queue - - noopHandler: - type: noop - parameters: - workflowUuid: - type: string - format: uuid - - # We believe it is best to re-template the request - # (at least with regard to secrets) JIT for every request execution - publishS3Handler: - type: http - backoff: - retries: 10 - seconds: 5 - factor: 2 - max: 25 + publishStacItemHandler: + type: synchttp parameters: - workflowName: - type: string - default: a_value feature: type: object - secrets: - # simplest approach for secret renewal is to have a ttl - # and periodically re-resolve the secret values, but does mean - # invalid secrets may be used for some time until the ttl expires - # - # how to handle secrets that need to resolve a token, - # have some other initialization step, or are of some built-in - # limited duration? - - name: user - type: file - path: /secrets-mount/username-secret - ttl: 1200 + required: + - collection + properties: + collection: + type: string request: - url: https://{secrets.minio-user}:{secrets.minio-password}@our-minio:9000 + url: {{ $stacApiEndpoint }}/collections/{{`{{ .parameters.feature.collection }}`}}/items method: POST - body: | - { - "fixed": "a_value", - "name": "{{`{{ .parameters.workflowName -}}`}}", - "date": "{{`{{ .parameters.feature.properties.datetime -}}`}}" - } + body: "{{`{{ .parameters.feature }}`}}" headers: - Authorization: "{{`Basic {{ .secrets.user }} {{ .secrets.password}}`}}" Content-Type: "application/json" - X-Workflow-Name: "{{`{{ .parameters.workflowName }}`}}" followRedirects: true responses: - # first matched wins - # by default any 2xx is success and anything else will be retried - - status: 400 - message: ".*timed out.*" - fatal: false - - status: 400 - fatal: true + - statusCode: 400 + message: ".*timeout.*" + result: error + - statusCode: 404 + result: fatal conductors: - instance-a: + main: handlers: - argoHandler - - publishS3Handler - instance-b: {} + - stacItemHandler callbacks: - publishS3Push: &callbacksPublishS3Push - handler: publishS3Handler + publishStacItem: &callbacksPublishStacItem + handler: publishStacItemHandler type: perFeature when: - "successful" - featureFilter: "@.id =~ 'fake*' & @.properties.gsd <= 0" parameters: - workflowName: - path: .input.features[0].id feature: - value: { "feature": true } + path: .feature enabled: true workflows: @@ -110,39 +74,5 @@ data: - .features[].collection cacheKeyHashExcludes: [] callbacks: - publishS3Push: - <<: *callbacksPublishS3Push - failed: - handler: noopHandler - type: single - when: - - "!successful" - parameters: - workflowUuid: - path: .workflow.uuid - cirrus-example: - callbacks: - publishS3Push: - <<: *callbacksPublishS3Push - when: - - "!failed" - - "successful" - title: "Cirrus example workflow" - description: "An example workflow config for a cirrus workflow" - version: 1 - handler: cirrusHandler - cirrusOpts: - sfnArn: arn:aws:states:us-west-2:09876543210:stateMachine:cirrus-example - cacheKeyHashIncludes: - - .features[].id - - .features[].collection - cacheKeyHashExcludes: [] - links: - - href: https://example.com/repo - rel: external - type: text/html - title: "source repository" - - href: https://example.com/docs - rel: external - type: text/html - title: "process documentation" + publishStacItem: + <<: *callbacksPublishStacItem diff --git a/charts/swoop-conductor/values.yaml b/charts/swoop-conductor/values.yaml index e720dc5..0091c23 100644 --- a/charts/swoop-conductor/values.yaml +++ b/charts/swoop-conductor/values.yaml @@ -63,7 +63,9 @@ service: configFile: "/opt/swoop-go/fixtures/swoop-config.yml" name: swoop-conductor serviceAccount: argo - instanceName: instance-a + instanceName: main + stacApiEndpoint: http://stac-fastapi-pgstac.default:8080 + deployment: name: swoop-conductor argoWorkflows: