feat: add include regex

linz · Aug 31, 2023 · 5e5c4a5 · 5e5c4a5
1 parent f52b71b
commit 5e5c4a5
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 8 deletions.
diff --git a/workflows/util/README.md b/workflows/util/README.md
@@ -12,12 +12,13 @@ Upon completion all standardised TIFF and STAC files will be located within the
 
 ## Workflow Input Parameters
 
-| Parameter   | Type | Default                           | Description                                                                                                                                                      |
-| ----------- | ---- | --------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| source      | str  | s3://linz-topgraphic/maps/topo50/ | the uri (path) to the input tiffs                                                                                                                                |
-| target      | str  | s3://linz-                        | the target uri (path) to copy the input and thumbnails                                                                                                           |
-| transform   | str  | f                                 | String to be transformed from source to target to renamed filenames, e.g. `f.replace("text to replace", "new_text_to_use")`. Leave as `f` for no transformation. |
-| copy-option | str  | no-clobber                        | `--no-clobber` Skip overwriting existing files. `--force` Overwrite all files. `--force-no-clobber` Overwrite only changed files, skip unchanged files. tiffs    |
+| Parameter   | Type  | Default                           | Description                                                                                                                                                      |
+| ----------- | ----- | --------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| source      | str   | s3://linz-topgraphic/maps/topo50/ | the uri (path) to the input tiffs                                                                                                                                |
+| target      | str   | s3://linz-                        | the target uri (path) to copy the input and thumbnails                                                                                                           |
+| include     | regex | .\*.\*.tif?$                      | A regular expression to match object path(s) or name(s) from within the source path to include.                                                                  |
+| transform   | str   | f                                 | String to be transformed from source to target to renamed filenames, e.g. `f.replace("text to replace", "new_text_to_use")`. Leave as `f` for no transformation. |
+| copy-option | str   | no-clobber                        | `--no-clobber` Skip overwriting existing files. `--force` Overwrite all files. `--force-no-clobber` Overwrite only changed files, skip unchanged files. tiffs    |
 
 ## Workflow Outputs
 

diff --git a/workflows/util/create-thumbnails.yaml b/workflows/util/create-thumbnails.yaml
@@ -20,6 +20,8 @@ spec:
         value: "s3://linz-topographic-upload/maps/topo50/"
       - name: target
         value: "s3://linz-"
+      - name: include
+        value: ".*.*.tif?$"
       - name: transform
         value: "f"
       - name: copy-option
@@ -57,7 +59,7 @@ spec:
                 - name: source
                   value: "{{workflow.parameters.source}}"
                 - name: include
-                  value: ".*.*.tif?$"
+                  value: "{{workflow.parameters.include}}"
                 - name: group
                   value: "1000"
                 - name: group-size
@@ -94,7 +96,7 @@ spec:
             "list",
             "--verbose",
             "--include",
-            ".*.*.tif?$",
+            "{{workflow.parameters.include}}",
             "--group",
             "2000",
             "--output",