fix: use aws-list rather than list within convert step

linz · Aug 23, 2023 · b2b00b8 · b2b00b8
1 parent c4ee40a
commit b2b00b8
Showing 1 changed file with 36 additions and 5 deletions.
diff --git a/workflows/imagery/ascii-standardising.yaml b/workflows/imagery/ascii-standardising.yaml
@@ -213,15 +213,19 @@ spec:
           - name: generate-ulid
             template: generate-ulid
 
+          - name: aws-list
+            template: aws-list
+
           - name: convert-asc-files
             template: convert-asc-files
             arguments:
               parameters:
-                - name: source
-                  value: "{{=sprig.trim(workflow.parameters.source)}}"
                 - name: target
                   value: "{{tasks.get-location.outputs.parameters.location}}translated-asc-files/"
-            depends: "get-location"
+              artifacts:
+                - name: files
+                  from: "{{ tasks.aws-list.outputs.artifacts.files }}"
+            depends: "get-location && aws-list"
 
           - name: tile-index-validate
             templateRef:
@@ -340,15 +344,42 @@ spec:
             valueFrom:
               path: "/tmp/ulid"
 
+    - name: aws-list
+      inputs:
+      container:
+        image: 019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:argo-tasks-{{workflow.parameters.version-argo-tasks}}
+        command: [node, /app/index.js]
+        env:
+          - name: AWS_ROLE_CONFIG_PATH
+            value: s3://linz-bucket-config/config.json
+        args:
+          [
+            "list",
+            "--verbose",
+            "--include",
+            ".asc?$",
+            "--group",
+            "999999",
+            "--output",
+            "/tmp/file_list.json",
+            "{{workflow.parameters.source}}",
+          ]
+      outputs:
+        artifacts:
+          - name: files
+            path: /tmp/file_list.json
+
     - name: convert-asc-files
       retryStrategy:
         limit: "2"
       nodeSelector:
         karpenter.sh/capacity-type: "spot"
       inputs:
         parameters:
-          - name: source
           - name: target
+        artifacts:
+          - name: files
+            path: /tmp/file_list.json
       container:
         image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:topo-imagery-{{=sprig.trim(workflow.parameters['version-topo-imagery'])}}"
         resources:
@@ -364,7 +395,7 @@ spec:
           - "/app/scripts/translate_ascii.py"
         args:
           - "--source"
-          - "{{inputs.parameters.source}}"
+          - "/tmp/file_list.json"
           - "--target"
           - "{{inputs.parameters.target}}"