From 22fcdcd21ab73e472fb11c30fb5f70093b1114d7 Mon Sep 17 00:00:00 2001 From: Paul Fouquet Date: Tue, 11 Jul 2023 12:02:05 +1200 Subject: [PATCH 01/14] wip: add grouping (using templates) to standardising workflow --- templates/argo-tasks/tile-index-validate.yml | 10 ++ workflows/imagery/standardising.yaml | 110 ++++++++++--------- 2 files changed, 70 insertions(+), 50 deletions(-) diff --git a/templates/argo-tasks/tile-index-validate.yml b/templates/argo-tasks/tile-index-validate.yml index 85560fd64..0372229a9 100644 --- a/templates/argo-tasks/tile-index-validate.yml +++ b/templates/argo-tasks/tile-index-validate.yml @@ -25,6 +25,14 @@ spec: description: Limit the input list to this regexp default: "" + - name: validate + description: Validate that all input tiffs perfectly align to tile grid + default: "true" + + - name: retile + description: Output tile configuration for retiling + default: "false" + - name: version description: container version to use default: "v2" @@ -62,6 +70,8 @@ spec: - "tileindex-validate" - "--verbose" - "--scale={{= inputs.parameters.scale }}" + - "--validate={{= inputs.parameters.validate }}" + - "--retile={{= inputs.parameters.retile }}" - "{{= sprig.empty(inputs.parameters.source_epsg) ? '' : '--source-epsg=' + inputs.parameters.source_epsg }}" - "{{= sprig.empty(inputs.parameters.include) ? '' : '--include=' + inputs.parameters.include }}" - "{{= sprig.trim(inputs.parameters.source) }}" diff --git a/workflows/imagery/standardising.yaml b/workflows/imagery/standardising.yaml index 525302eaa..29b5a0335 100644 --- a/workflows/imagery/standardising.yaml +++ b/workflows/imagery/standardising.yaml @@ -2,22 +2,18 @@ apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate metadata: - name: imagery-standardising + name: test-imagery-standardising-retile namespace: argo spec: parallelism: 50 nodeSelector: karpenter.sh/capacity-type: "spot" entrypoint: main - synchronization: - semaphore: - configMapKeyRef: - name: semaphores - key: standardising arguments: parameters: + # FIXME: Should use camelCase or underscore? - name: version-argo-tasks - value: "v2.9" + value: "latest" # TODO: use latest version for test purposes - name: version-basemaps-cli value: "v6.39.0-15-g3e982390" - name: version-topo-imagery @@ -36,6 +32,16 @@ spec: - "10000" - "50000" - "None" + - name: validate + value: true + enum: + - "false" + - "true" + - name: retile + value: false + enum: + - "true" + - "false" - name: source-epsg value: "2193" - name: target-epsg @@ -47,6 +53,7 @@ spec: enum: - "webp" - "lzw" + - "dem_lerc" - name: cutline description: "(Optional) location of a cutline file to cut the imagery to .fgb or .geojson" value: "" @@ -198,7 +205,47 @@ spec: - name: generate-ulid template: generate-ulid - name: tile-index-validate - template: tile-index-validate + templateRef: + name: tpl-at-tile-index-validate + template: main + arguments: + parameters: + - name: scale + value: "{{workflow.parameters.scale}}" + + - name: source + value: "{{=sprig.trim(workflow.parameters.source)}}" + + - name: source_epsg + value: 2193 # FIXME: should be defaulted to this value? + + - name: validate + value: "{{= workflow.parameters.validate}}" + + - name: retile + value: "{{= workflow.parameters.retile}}" + + - name: version + value: "{{= workflow.parameters['version-argo-tasks']}}" + + - name: group + templateRef: + name: tpl-at-group + template: main + + arguments: + artifacts: + - name: input + from: "{{ tasks.tile-index-validate.outputs.artifacts.files }}" + + parameters: + - name: size + value: 1 + + - name: version + value: "{{= workflow.parameters['version-argo-tasks']}}" + + depends: "tile-index-validate" - name: standardise-validate template: standardise-validate arguments: @@ -209,8 +256,8 @@ spec: value: "{{tasks.generate-ulid.outputs.parameters.ulid}}" - name: target value: "{{tasks.get-location.outputs.parameters.location}}flat/" - depends: "tile-index-validate && generate-ulid && get-location" - withParam: "{{tasks.tile-index-validate.outputs.parameters.files}}" + depends: "group && generate-ulid && get-location" + withParam: "{{ tasks.group.outputs.parameters.output }}" - name: create-collection template: create-collection arguments: @@ -256,7 +303,7 @@ spec: parameter: "{{tasks.get-location.outputs.parameters.location}}" - name: generate-ulid script: - image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:topo-imagery-{{=sprig.trim(workflow.parameters['version-topo-imagery'])}}" + image: "ghcr.io/paulfouquet/topo-imagery-test:latest" command: [python] source: | import ulid @@ -267,43 +314,6 @@ spec: - name: ulid valueFrom: path: "/tmp/ulid" - - name: tile-index-validate - nodeSelector: - karpenter.sh/capacity-type: "spot" - container: - image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:argo-tasks-{{=sprig.trim(workflow.parameters['version-argo-tasks'])}}" - command: [node, /app/index.js] - env: - - name: AWS_ROLE_CONFIG_PATH - value: s3://linz-bucket-config/config.json - args: - [ - "tileindex-validate", - "--verbose", - "--include", - "{{=sprig.trim(workflow.parameters.include)}}", - "--group", - "{{=sprig.trim(workflow.parameters.group)}}", - "--output", - "/tmp/file_list.json", - "--duplicates-output", - "/tmp/duplicate_list.json", - "--scale", - "{{workflow.parameters.scale}}", - "{{=sprig.trim(workflow.parameters.source)}}", - ] - outputs: - parameters: - - name: files - valueFrom: - path: /tmp/file_list.json - artifacts: - - name: list - path: /tmp/file_list.json - optional: true - - name: duplicates - path: /tmp/duplicate_list.json - optional: true - name: standardise-validate retryStrategy: limit: "2" @@ -315,7 +325,7 @@ spec: - name: collection-id - name: target container: - image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:topo-imagery-{{=sprig.trim(workflow.parameters['version-topo-imagery'])}}" + image: "ghcr.io/paulfouquet/topo-imagery-test:latest" resources: requests: memory: 7.8Gi @@ -358,7 +368,7 @@ spec: - name: collection-id - name: location container: - image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:topo-imagery-{{=sprig.trim(workflow.parameters['version-topo-imagery'])}}" + image: "ghcr.io/paulfouquet/topo-imagery-test:latest" resources: requests: memory: 7.8Gi From 4fbfa2f91f5c4187770cec5669a041ec885b138f Mon Sep 17 00:00:00 2001 From: Paul Fouquet Date: Tue, 11 Jul 2023 12:03:38 +1200 Subject: [PATCH 02/14] wip: add semaphore (commented out) --- workflows/imagery/standardising.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/workflows/imagery/standardising.yaml b/workflows/imagery/standardising.yaml index 29b5a0335..4a7e9742d 100644 --- a/workflows/imagery/standardising.yaml +++ b/workflows/imagery/standardising.yaml @@ -9,6 +9,11 @@ spec: nodeSelector: karpenter.sh/capacity-type: "spot" entrypoint: main + # synchronization: + # semaphore: + # configMapKeyRef: + # name: semaphores + # key: standardising arguments: parameters: # FIXME: Should use camelCase or underscore? From 1bba94dcd02d7b239f7096aa6bd5490c6b23a5ce Mon Sep 17 00:00:00 2001 From: Megan Davidson Date: Wed, 12 Jul 2023 15:01:19 +1200 Subject: [PATCH 03/14] WIP: don't run basemaps when dem_lerc preset can be removed when basemaps is ready --- workflows/imagery/standardising.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/workflows/imagery/standardising.yaml b/workflows/imagery/standardising.yaml index 4a7e9742d..3a28563a9 100644 --- a/workflows/imagery/standardising.yaml +++ b/workflows/imagery/standardising.yaml @@ -222,7 +222,7 @@ spec: value: "{{=sprig.trim(workflow.parameters.source)}}" - name: source_epsg - value: 2193 # FIXME: should be defaulted to this value? + value: "{{=sprig.trim(workflow.parameters['source-epsg'])}}" - name: validate value: "{{= workflow.parameters.validate}}" @@ -286,7 +286,7 @@ spec: - name: get-location template: get-location - name: create-overview - when: "'{{workflow.parameters.target-epsg}}' =~ '2193|3857'" + when: "'{{workflow.parameters.target-epsg}}' =~ '2193|3857' && '{{workflow.parameters.compression}}' != 'dem_lerc'" arguments: parameters: - name: location @@ -294,7 +294,7 @@ spec: template: create-overview depends: "standardise-validate" - name: create-config - when: "'{{workflow.parameters.target-epsg}}' =~ '2193|3857'" + when: "'{{workflow.parameters.target-epsg}}' =~ '2193|3857' && '{{workflow.parameters.compression}}' != 'dem_lerc'" arguments: parameters: - name: location From 028695057d0c2386626ac4d7a5c0a05086fb8347 Mon Sep 17 00:00:00 2001 From: Megan Davidson Date: Wed, 19 Jul 2023 14:25:32 +1200 Subject: [PATCH 04/14] fix: add back semaphore and group remove scale from standardising --- workflows/imagery/standardising.yaml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/workflows/imagery/standardising.yaml b/workflows/imagery/standardising.yaml index 3a28563a9..f7213b5ad 100644 --- a/workflows/imagery/standardising.yaml +++ b/workflows/imagery/standardising.yaml @@ -9,11 +9,11 @@ spec: nodeSelector: karpenter.sh/capacity-type: "spot" entrypoint: main - # synchronization: - # semaphore: - # configMapKeyRef: - # name: semaphores - # key: standardising + synchronization: + semaphore: + configMapKeyRef: + name: semaphores + key: standardising arguments: parameters: # FIXME: Should use camelCase or underscore? @@ -245,7 +245,7 @@ spec: parameters: - name: size - value: 1 + value: "{{workflow.parameters.group}}" - name: version value: "{{= workflow.parameters['version-argo-tasks']}}" @@ -347,8 +347,6 @@ spec: - "{{inputs.parameters.file}}" - "--target" - "{{inputs.parameters.target}}" - - "--scale" - - "{{workflow.parameters.scale}}" - "--preset" - "{{workflow.parameters.compression}}" - "--start-datetime" From 937ec446933583a1f884bfd60c0e2c549e575b5d Mon Sep 17 00:00:00 2001 From: Alice Fage Date: Wed, 19 Jul 2023 15:27:29 +1200 Subject: [PATCH 05/14] chore: add producers and licensors for elevation --- workflows/imagery/standardising.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/workflows/imagery/standardising.yaml b/workflows/imagery/standardising.yaml index f7213b5ad..a872cde46 100644 --- a/workflows/imagery/standardising.yaml +++ b/workflows/imagery/standardising.yaml @@ -80,7 +80,9 @@ spec: "Landpro", "Maxar", "NZ Aerial Mapping", + "Ocean Infinity", "Recon", + "RPS", "SkyCan", "Terralink International", "UAV Mapping NZ", @@ -140,6 +142,7 @@ spec: "Ministry of Primary Industries", "NZ Aerial Mapping", "Napier City Council", + "National Emergency Management Agency", "Nelson City Council", "New Plymouth District Council", "Northland Regional Council", From 486db963b4e90d8ccab6fda8803dba3829f139e5 Mon Sep 17 00:00:00 2001 From: Alice Fage Date: Wed, 19 Jul 2023 15:47:05 +1200 Subject: [PATCH 06/14] docs: add retile to description --- templates/argo-tasks/tile-index-validate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/argo-tasks/tile-index-validate.yml b/templates/argo-tasks/tile-index-validate.yml index 0372229a9..3341c0c45 100644 --- a/templates/argo-tasks/tile-index-validate.yml +++ b/templates/argo-tasks/tile-index-validate.yml @@ -11,7 +11,7 @@ spec: inputs: parameters: - name: scale - description: Tile Index scale to validate against + description: Tile Index scale to validate against or target scale for retiling - name: source_epsg description: Force source EPSG, "" will attempt to load EPSG from the source tiffs From 608deedf9273380fa0082e7796ae3f43f66f328d Mon Sep 17 00:00:00 2001 From: Alice Fage Date: Wed, 19 Jul 2023 15:52:18 +1200 Subject: [PATCH 07/14] fix: semaphone indentation --- workflows/imagery/standardising.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/workflows/imagery/standardising.yaml b/workflows/imagery/standardising.yaml index a872cde46..cb2fed911 100644 --- a/workflows/imagery/standardising.yaml +++ b/workflows/imagery/standardising.yaml @@ -10,10 +10,10 @@ spec: karpenter.sh/capacity-type: "spot" entrypoint: main synchronization: - semaphore: - configMapKeyRef: - name: semaphores - key: standardising + semaphore: + configMapKeyRef: + name: semaphores + key: standardising arguments: parameters: # FIXME: Should use camelCase or underscore? From 35a4a8a1f834bce487350db4b28b6459fb9dfc70 Mon Sep 17 00:00:00 2001 From: Alice Fage Date: Fri, 21 Jul 2023 11:48:08 +1200 Subject: [PATCH 08/14] fix: standardising workflowtemplate name and container versions --- workflows/imagery/standardising.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/workflows/imagery/standardising.yaml b/workflows/imagery/standardising.yaml index cb2fed911..fa832b3fa 100644 --- a/workflows/imagery/standardising.yaml +++ b/workflows/imagery/standardising.yaml @@ -2,7 +2,7 @@ apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate metadata: - name: test-imagery-standardising-retile + name: imagery-standardising namespace: argo spec: parallelism: 50 @@ -18,11 +18,11 @@ spec: parameters: # FIXME: Should use camelCase or underscore? - name: version-argo-tasks - value: "latest" # TODO: use latest version for test purposes + value: "v2" - name: version-basemaps-cli value: "v6.39.0-15-g3e982390" - name: version-topo-imagery - value: "v2" + value: "v3" - name: source value: "s3://linz-imagery-staging/test/sample/" - name: include @@ -311,7 +311,7 @@ spec: parameter: "{{tasks.get-location.outputs.parameters.location}}" - name: generate-ulid script: - image: "ghcr.io/paulfouquet/topo-imagery-test:latest" + image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:topo-imagery-{{=sprig.trim(workflow.parameters['version-topo-imagery'])}}" command: [python] source: | import ulid @@ -333,7 +333,7 @@ spec: - name: collection-id - name: target container: - image: "ghcr.io/paulfouquet/topo-imagery-test:latest" + image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:topo-imagery-{{=sprig.trim(workflow.parameters['version-topo-imagery'])}}" resources: requests: memory: 7.8Gi @@ -374,7 +374,7 @@ spec: - name: collection-id - name: location container: - image: "ghcr.io/paulfouquet/topo-imagery-test:latest" + image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:topo-imagery-{{=sprig.trim(workflow.parameters['version-topo-imagery'])}}" resources: requests: memory: 7.8Gi From 3cfcc33e249ec6deac458785ff543351198b3ad6 Mon Sep 17 00:00:00 2001 From: Alice Fage Date: Fri, 21 Jul 2023 14:10:21 +1200 Subject: [PATCH 09/14] fix: add include to tile-index-validate step --- workflows/imagery/standardising.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/workflows/imagery/standardising.yaml b/workflows/imagery/standardising.yaml index fa832b3fa..8f444f18b 100644 --- a/workflows/imagery/standardising.yaml +++ b/workflows/imagery/standardising.yaml @@ -221,6 +221,9 @@ spec: - name: scale value: "{{workflow.parameters.scale}}" + - name: include + value: "{{workflow.parameters.include}}" + - name: source value: "{{=sprig.trim(workflow.parameters.source)}}" From fd8d93ff2594d03cd09145a314885e1198454d12 Mon Sep 17 00:00:00 2001 From: Megan Davidson Date: Tue, 25 Jul 2023 13:30:59 +1200 Subject: [PATCH 10/14] fix: standardise publish workflow update --- workflows/imagery/standardising-publish-import.yaml | 4 ++++ workflows/imagery/standardising.yaml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/workflows/imagery/standardising-publish-import.yaml b/workflows/imagery/standardising-publish-import.yaml index 803f44dbc..bebe9d8ae 100644 --- a/workflows/imagery/standardising-publish-import.yaml +++ b/workflows/imagery/standardising-publish-import.yaml @@ -32,6 +32,10 @@ spec: value: ".tiff?$" - name: transform value: "f" + - name: validate + value: "true" + - name: retile + value: "false" - name: version-argo-tasks value: "v2.9" - name: version-basemaps-cli diff --git a/workflows/imagery/standardising.yaml b/workflows/imagery/standardising.yaml index 8f444f18b..7ed8a6082 100644 --- a/workflows/imagery/standardising.yaml +++ b/workflows/imagery/standardising.yaml @@ -227,7 +227,7 @@ spec: - name: source value: "{{=sprig.trim(workflow.parameters.source)}}" - - name: source_epsg + - name: source-epsg value: "{{=sprig.trim(workflow.parameters['source-epsg'])}}" - name: validate From 4a868f60fceed22723c6a0e57fe2d1686f5a5a62 Mon Sep 17 00:00:00 2001 From: Paul Fouquet Date: Wed, 26 Jul 2023 15:26:08 +1200 Subject: [PATCH 11/14] feat: group use artifact output files --- templates/argo-tasks/group.yml | 33 +++++++++++++++++------ workflows/imagery/standardising.yaml | 40 ++++++++++++++++++---------- 2 files changed, 51 insertions(+), 22 deletions(-) diff --git a/templates/argo-tasks/group.yml b/templates/argo-tasks/group.yml index 5816341bc..b66ab01c5 100644 --- a/templates/argo-tasks/group.yml +++ b/templates/argo-tasks/group.yml @@ -10,31 +10,48 @@ spec: - name: main inputs: artifacts: + # + # JSON array of things to group + # + # @example + # ```json + # ["a.json", "b.json", "c.json"] + # ``` - name: input path: /tmp/group/input.json - optional: true parameters: - name: size - description: group into this number of records per group + description: Group into this number of records per group - name: version - description: container version to use + description: argo-task Container version to use default: "v2" outputs: parameters: - # Grouped output of the input + # + # JSON array of all the group ids, which correspond to a output artifact file + # + # @example + # ```json + # ["000", "001", "002", "003"] + # ``` + # + # Workflows should use the "000" string to access the data from the output artifact folder - name: output valueFrom: path: /tmp/group/output.json artifacts: - # Grouped output of the input + # Grouped output of the input as one file per output groupId + # + # - /output/000.json + # - /output/001.json + # - ... + # - name: output - path: /tmp/group/output.json - archive: - none: {} + path: /tmp/group/output/ container: image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:argo-tasks-{{= inputs.parameters.version }}" diff --git a/workflows/imagery/standardising.yaml b/workflows/imagery/standardising.yaml index 7ed8a6082..041dca610 100644 --- a/workflows/imagery/standardising.yaml +++ b/workflows/imagery/standardising.yaml @@ -220,22 +220,16 @@ spec: parameters: - name: scale value: "{{workflow.parameters.scale}}" - - name: include value: "{{workflow.parameters.include}}" - - name: source value: "{{=sprig.trim(workflow.parameters.source)}}" - - name: source-epsg value: "{{=sprig.trim(workflow.parameters['source-epsg'])}}" - - name: validate value: "{{= workflow.parameters.validate}}" - - name: retile value: "{{= workflow.parameters.retile}}" - - name: version value: "{{= workflow.parameters['version-argo-tasks']}}" @@ -243,32 +237,33 @@ spec: templateRef: name: tpl-at-group template: main - arguments: artifacts: - name: input from: "{{ tasks.tile-index-validate.outputs.artifacts.files }}" - parameters: - name: size value: "{{workflow.parameters.group}}" - - name: version value: "{{= workflow.parameters['version-argo-tasks']}}" - depends: "tile-index-validate" + - name: standardise-validate template: standardise-validate arguments: parameters: - - name: file + - name: group_id value: "{{item}}" - name: collection-id value: "{{tasks.generate-ulid.outputs.parameters.ulid}}" - name: target value: "{{tasks.get-location.outputs.parameters.location}}flat/" + artifacts: + - name: group_data + from: "{{ tasks.group.outputs.artifacts.output }}" depends: "group && generate-ulid && get-location" withParam: "{{ tasks.group.outputs.parameters.output }}" + - name: create-collection template: create-collection arguments: @@ -278,6 +273,7 @@ spec: - name: location value: "{{tasks.get-location.outputs.parameters.location}}" depends: "standardise-validate" + - name: stac-validate template: stac-validate arguments: @@ -289,8 +285,10 @@ spec: raw: data: "{{tasks.stac-validate.outputs.result}}" depends: "create-collection" + - name: get-location template: get-location + - name: create-overview when: "'{{workflow.parameters.target-epsg}}' =~ '2193|3857' && '{{workflow.parameters.compression}}' != 'dem_lerc'" arguments: @@ -299,6 +297,7 @@ spec: value: "{{tasks.get-location.outputs.parameters.location}}" template: create-overview depends: "standardise-validate" + - name: create-config when: "'{{workflow.parameters.target-epsg}}' =~ '2193|3857' && '{{workflow.parameters.compression}}' != 'dem_lerc'" arguments: @@ -307,11 +306,14 @@ spec: value: "{{tasks.get-location.outputs.parameters.location}}" template: create-config depends: "create-overview" + outputs: parameters: - name: target valueFrom: parameter: "{{tasks.get-location.outputs.parameters.location}}" + # END TEMPLATE `main` + - name: generate-ulid script: image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:topo-imagery-{{=sprig.trim(workflow.parameters['version-topo-imagery'])}}" @@ -325,6 +327,7 @@ spec: - name: ulid valueFrom: path: "/tmp/ulid" + - name: standardise-validate retryStrategy: limit: "2" @@ -332,9 +335,12 @@ spec: karpenter.sh/capacity-type: "spot" inputs: parameters: - - name: file + - name: group_id - name: collection-id - name: target + artifacts: + - name: group_data + path: /tmp/input/ container: image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:topo-imagery-{{=sprig.trim(workflow.parameters['version-topo-imagery'])}}" resources: @@ -349,8 +355,8 @@ spec: - python - "/app/scripts/standardise_validate.py" args: - - "--source" - - "{{inputs.parameters.file}}" + - "--from-file" + - "/tmp/input/{{inputs.parameters.group_id}}.json" - "--target" - "{{inputs.parameters.target}}" - "--preset" @@ -367,6 +373,7 @@ spec: - "{{=sprig.trim(workflow.parameters['source-epsg'])}}" - "--target-epsg" - "{{=sprig.trim(workflow.parameters['target-epsg'])}}" + - name: create-collection retryStrategy: limit: "2" @@ -404,6 +411,7 @@ spec: - "{{=sprig.trim(workflow.parameters['licensor-list'])}}" - "--concurrency" - "25" + - name: stac-validate inputs: parameters: @@ -421,6 +429,7 @@ spec: "--recursive", "{{inputs.parameters.location}}flat/collection.json", ] + - name: get-location script: image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:argo-tasks-{{=sprig.trim(workflow.parameters['version-argo-tasks'])}}" @@ -435,6 +444,7 @@ spec: - name: location valueFrom: path: "/tmp/location" + - name: create-overview retryStrategy: limit: "2" @@ -460,6 +470,7 @@ spec: "--output", "{{inputs.parameters.location}}flat/", ] + - name: create-config inputs: parameters: @@ -487,6 +498,7 @@ spec: - name: url valueFrom: path: "/tmp/url" + volumes: - name: ephemeral emptyDir: {} From 75b206fe17695a23f40efd3a5b42ff816db976a0 Mon Sep 17 00:00:00 2001 From: Paul Fouquet Date: Mon, 31 Jul 2023 11:09:44 +1200 Subject: [PATCH 12/14] docs: update README --- workflows/imagery/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/workflows/imagery/README.md b/workflows/imagery/README.md index fdf6d15cc..5eb50bf6a 100644 --- a/workflows/imagery/README.md +++ b/workflows/imagery/README.md @@ -19,6 +19,8 @@ In addition, a Basemaps link is produced enabling visual QA. | source | str | s3://linz-imagery-staging/test/sample | the uri (path) to the input tiffs | | include | regex | .tiff?$ | A regular expression to match object path(s) or name(s) from within the source path to include in standardising\*. | | scale | enum | 500 | The scale of the TIFFs | +| validate | enum | true | Validate the TIFFs files with `tileindex-validate`. | +| retile | enum | false | Prepare the data for retiling TIFFs files to `scale` with `tileindex-validate`. | | group | int | 50 | The number of files to group into the pods (testing has recommended using 50 for large datasets). | | compression | enum | webp | Standardised file format | | cutline | str | | (Optional) location of a cutline file to cut the imagery to `.fgb` or `.geojson` (leave blank if no cutline) | @@ -43,6 +45,8 @@ In addition, a Basemaps link is produced enabling visual QA. | source | s3://linz-imagery-upload/PRJ39741_BOPLASS_Imagery_2021-22/PRJ39741_03/01_GeoTiff/ | | include | .tiff?$ | | scale | 2000 | +| validate | true | +| retile | false | | group | 50 | | compression | webp | | cutline | s3://linz-imagery-staging/cutline/bay-of-plenty_2021-2022.fgb | From cde64db1d38895d6feb8a3e858ac2bf38e55dc54 Mon Sep 17 00:00:00 2001 From: Paul Fouquet Date: Mon, 31 Jul 2023 11:23:34 +1200 Subject: [PATCH 13/14] docs: fix typo --- workflows/imagery/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflows/imagery/README.md b/workflows/imagery/README.md index 5eb50bf6a..6fd1ccdfb 100644 --- a/workflows/imagery/README.md +++ b/workflows/imagery/README.md @@ -33,8 +33,8 @@ In addition, a Basemaps link is produced enabling visual QA. | start-datetime | str | YYYY-MM-DD | Imagery start date (flown from), must be in default formatting | | end-datetime | str | YYYY-MM-DD | Imagery end date (flown to), must be in default formatting | | copy-option | enum | --no-clobber | `--no-clobber` Skip overwriting existing files. `--force` Overwrite all files. `--force-no-clobber` Overwrite only changed files, skip unchanged files. | -| source-espg | str | 2193 | The EPSP code of the source imagery | -| target-espg | str | 2193 | The target EPSP code - if different to source the imagery will be reprojected | +| source-espg | str | 2193 | The EPSG code of the source imagery | +| target-espg | str | 2193 | The target EPSG code - if different to source the imagery will be reprojected | \* This regex can be used to exclude paths as well, e.g. if there are RBG and RGBI directories, the following regex will only include TIFF files in the RGB directory: `RGB(?!I).*.tiff?$`. For more complicated exclusions, there is an `--exclude` parameter, which would need to be added to the Argo WorkflowTemplate. From d4217684c41bda88fcf7a3ef9b56eec5044e8faf Mon Sep 17 00:00:00 2001 From: Paul Fouquet Date: Mon, 31 Jul 2023 11:24:57 +1200 Subject: [PATCH 14/14] docs: fix typo --- workflows/imagery/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflows/imagery/README.md b/workflows/imagery/README.md index 6fd1ccdfb..efc0ae835 100644 --- a/workflows/imagery/README.md +++ b/workflows/imagery/README.md @@ -33,8 +33,8 @@ In addition, a Basemaps link is produced enabling visual QA. | start-datetime | str | YYYY-MM-DD | Imagery start date (flown from), must be in default formatting | | end-datetime | str | YYYY-MM-DD | Imagery end date (flown to), must be in default formatting | | copy-option | enum | --no-clobber | `--no-clobber` Skip overwriting existing files. `--force` Overwrite all files. `--force-no-clobber` Overwrite only changed files, skip unchanged files. | -| source-espg | str | 2193 | The EPSG code of the source imagery | -| target-espg | str | 2193 | The target EPSG code - if different to source the imagery will be reprojected | +| source-epsg | str | 2193 | The EPSG code of the source imagery | +| target-epsg | str | 2193 | The target EPSG code - if different to source the imagery will be reprojected | \* This regex can be used to exclude paths as well, e.g. if there are RBG and RGBI directories, the following regex will only include TIFF files in the RGB directory: `RGB(?!I).*.tiff?$`. For more complicated exclusions, there is an `--exclude` parameter, which would need to be added to the Argo WorkflowTemplate.