feat: add publish copy steps

workflows/util/README.md

@@ -12,9 +12,12 @@ Upon completion all standardised TIFF and STAC files will be located within the
 
 ## Workflow Input Parameters
 
-| Parameter | Type | Default                           | Description                       |
-| --------- | ---- | --------------------------------- | --------------------------------- |
-| source    | str  | s3://linz-topgraphic/maps/topo50/ | the uri (path) to the input tiffs |
+| Parameter   | Type | Default                           | Description                                                                                                                                                      |
+| ----------- | ---- | --------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| source      | str  | s3://linz-topgraphic/maps/topo50/ | the uri (path) to the input tiffs                                                                                                                                |
+| target      | str  | s3://linz-                        | the target uri (path) to copy the input and thumbnails                                                                                                           |
+| transform   | str  | f                                 | String to be transformed from source to target to renamed filenames, e.g. `f.replace("text to replace", "new_text_to_use")`. Leave as `f` for no transformation. |
+| copy-option | str  | no-clobber                        | `--no-clobber` Skip overwriting existing files. `--force` Overwrite all files. `--force-no-clobber` Overwrite only changed files, skip unchanged files. tiffs    |
 
 ## Workflow Outputs
 

workflows/util/create-thumbnails.yaml

@@ -17,7 +17,13 @@ spec:
       - name: version-topo-imagery
         value: "v3"
       - name: source
-        value: "s3://linz-topographic/maps/topo50/"
+        value: "s3://linz-topographic-upload/maps/topo50/"
+      - name: target
+        value: "s3://linz-"
+      - name: transform
+        value: "f"
+      - name: copy-option
+        value: "--no-clobber"
   templateDefaults:
     container:
       imagePullPolicy: Always
@@ -28,6 +34,9 @@ spec:
           - name: aws-list
             template: aws-list
 
+          - name: get-location
+            template: get-location
+
           - name: thumbnails
             template: thumbnails
             arguments:
@@ -39,8 +48,37 @@ spec:
                   from: "{{ tasks.aws-list.outputs.artifacts.files }}"
             depends: "aws-list && get-location"
 
-          - name: get-location
-            template: get-location
+          - name: publish-source
+            templateRef:
+              name: publish-copy
+              template: main
+            arguments:
+              parameters:
+                - name: source
+                  value: "{{workflow.parameters.source}}"
+                - name: include
+                  value: ".*.*.tif?$"
+                - name: group
+                  value: "1000"
+                - name: group-size
+                  value: "100Gi"
+            depends: "thumbnails"
+
+          - name: publish-thumbnails
+            templateRef:
+              name: publish-copy
+              template: main
+            arguments:
+              parameters:
+                - name: source
+                  value: "{{tasks.get-location.outputs.parameters.location}}thumbnails/"
+                - name: include
+                  value: ".jpg?$"
+                - name: group
+                  value: "1000"
+                - name: group-size
+                  value: "100Gi"
+            depends: "thumbnails"
 
       outputs:
         parameters: