diff --git a/workflows/util/README.md b/workflows/util/README.md index b996115af..3aeaa83fb 100644 --- a/workflows/util/README.md +++ b/workflows/util/README.md @@ -12,9 +12,12 @@ Upon completion all standardised TIFF and STAC files will be located within the ## Workflow Input Parameters -| Parameter | Type | Default | Description | -| --------- | ---- | --------------------------------- | --------------------------------- | -| source | str | s3://linz-topgraphic/maps/topo50/ | the uri (path) to the input tiffs | +| Parameter | Type | Default | Description | +| ----------- | ---- | --------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| source | str | s3://linz-topgraphic/maps/topo50/ | the uri (path) to the input tiffs | +| target | str | s3://linz- | the target uri (path) to copy the input and thumbnails | +| transform | str | f | String to be transformed from source to target to renamed filenames, e.g. `f.replace("text to replace", "new_text_to_use")`. Leave as `f` for no transformation. | +| copy-option | str | no-clobber | `--no-clobber` Skip overwriting existing files. `--force` Overwrite all files. `--force-no-clobber` Overwrite only changed files, skip unchanged files. tiffs | ## Workflow Outputs diff --git a/workflows/util/create-thumbnails.yaml b/workflows/util/create-thumbnails.yaml index bb5d9d2db..5997416dc 100644 --- a/workflows/util/create-thumbnails.yaml +++ b/workflows/util/create-thumbnails.yaml @@ -17,7 +17,13 @@ spec: - name: version-topo-imagery value: "v3" - name: source - value: "s3://linz-topographic/maps/topo50/" + value: "s3://linz-topographic-upload/maps/topo50/" + - name: target + value: "s3://linz-" + - name: transform + value: "f" + - name: copy-option + value: "--no-clobber" templateDefaults: container: imagePullPolicy: Always @@ -28,6 +34,9 @@ spec: - name: aws-list template: aws-list + - name: get-location + template: get-location + - name: thumbnails template: thumbnails arguments: @@ -39,8 +48,37 @@ spec: from: "{{ tasks.aws-list.outputs.artifacts.files }}" depends: "aws-list && get-location" - - name: get-location - template: get-location + - name: publish-source + templateRef: + name: publish-copy + template: main + arguments: + parameters: + - name: source + value: "{{workflow.parameters.source}}" + - name: include + value: ".*.*.tif?$" + - name: group + value: "1000" + - name: group-size + value: "100Gi" + depends: "thumbnails" + + - name: publish-thumbnails + templateRef: + name: publish-copy + template: main + arguments: + parameters: + - name: source + value: "{{tasks.get-location.outputs.parameters.location}}thumbnails/" + - name: include + value: ".jpg?$" + - name: group + value: "1000" + - name: group-size + value: "100Gi" + depends: "thumbnails" outputs: parameters: