From 983721542b5d0d60bb5a79f5745b6b15b568ec09 Mon Sep 17 00:00:00 2001 From: Blayne Chard Date: Tue, 25 Jul 2023 15:35:05 +1200 Subject: [PATCH 1/4] feat: group using artifact files --- templates/argo-tasks/README.md | 79 ++++++++++++++++++++++++++++++++++ templates/argo-tasks/group.yml | 21 +++++---- 2 files changed, 92 insertions(+), 8 deletions(-) create mode 100644 templates/argo-tasks/README.md diff --git a/templates/argo-tasks/README.md b/templates/argo-tasks/README.md new file mode 100644 index 000000000..6a8e38934 --- /dev/null +++ b/templates/argo-tasks/README.md @@ -0,0 +1,79 @@ +# ArgoTasks Templates + +## argo-tasks/group - `tpl-at-group` + +Group inputs into outputs to be used with `withParam` to run one task per grouping + +### Template usage + +Group the output of `tile-index-validate` into groups of size 5 + +```yaml +- name: group + templateRef: + name: tpl-at-group + template: main + + arguments: + artifacts: + - name: input + from: "{{ tasks.tile-index-validate.outputs.artifacts.files }}" + + parameters: + - name: size + value: 5 + + depends: "tile-index-validate" +``` + +### Consumer usage + +Using `withParams` to spin up a number of tasks from + +```yaml +# ... + +steps: + # consume the output of a grouper + - - name: consume + template: consume + arguments: + parameters: + - name: group_id + value: "{{item}}" # groupId eg "000", "001", "002" etc.. + + # All the grouped data as a folder + artifacts: + - name: group_data + from: "{{ steps.group.outputs.artifacts.output }}" + + withParam: "{{ steps.group.outputs.parameters.output }}" + +# ... + +- name: consume + inputs: + # Id of the grouping file to consume + # to be used with `group_data` + parameters: + - name: group_id # "000", "001" ... etc + + # grouped input data for the consumer, this will be a folder full of JSON files + # one file per groupId + artifacts: + - name: group_data + path: /tmp/input/ + + parameters: + - name: group_id # "000", "001" ... etc + + script: + image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:argo-tasks-latest" + command: [bash] + source: | + echo {{ inputs.parameters.group_id}} + ls -alh /tmp/input/ + + # for example using with a --from-file + # ./test-cli --from-file=/tmp/group/input/{{inputs.parameters.group_id}}.json +``` diff --git a/templates/argo-tasks/group.yml b/templates/argo-tasks/group.yml index 5816341bc..e3fb8e166 100644 --- a/templates/argo-tasks/group.yml +++ b/templates/argo-tasks/group.yml @@ -10,31 +10,36 @@ spec: - name: main inputs: artifacts: + # JSON array of things to group + # eg ["a.json", "b.json", "c.json"] - name: input + description: JSON array of items to group path: /tmp/group/input.json - optional: true parameters: - name: size - description: group into this number of records per group + description: Group into this number of records per group - name: version - description: container version to use + description: argo-task Container version to use default: "v2" outputs: parameters: - # Grouped output of the input + # Ids of output files + # eg "000", "001", "002", "003" + # Workflows should use the "000" string to access the data from the output artifact folder - name: output valueFrom: path: /tmp/group/output.json artifacts: - # Grouped output of the input + # Grouped output of the input as one file per output groupId + # - /output/000.json + # - /output/001.json + # - ... - name: output - path: /tmp/group/output.json - archive: - none: {} + path: /tmp/group/output/ container: image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:argo-tasks-{{= inputs.parameters.version }}" From 451aacfd1248fc2a045d8894a41b5775d619cfc7 Mon Sep 17 00:00:00 2001 From: Blayne Chard Date: Tue, 25 Jul 2023 15:36:06 +1200 Subject: [PATCH 2/4] refactor: not allowed iunput description :sad-panda; --- templates/argo-tasks/group.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/templates/argo-tasks/group.yml b/templates/argo-tasks/group.yml index e3fb8e166..e587e3b93 100644 --- a/templates/argo-tasks/group.yml +++ b/templates/argo-tasks/group.yml @@ -13,7 +13,6 @@ spec: # JSON array of things to group # eg ["a.json", "b.json", "c.json"] - name: input - description: JSON array of items to group path: /tmp/group/input.json parameters: From 684de8162ec3f5ddfcbafbc24cd4106d9779973d Mon Sep 17 00:00:00 2001 From: Blayne Chard Date: Tue, 25 Jul 2023 15:43:15 +1200 Subject: [PATCH 3/4] refactor: improve comments --- templates/argo-tasks/group.yml | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/templates/argo-tasks/group.yml b/templates/argo-tasks/group.yml index e587e3b93..d3a75d309 100644 --- a/templates/argo-tasks/group.yml +++ b/templates/argo-tasks/group.yml @@ -10,8 +10,13 @@ spec: - name: main inputs: artifacts: + # # JSON array of things to group - # eg ["a.json", "b.json", "c.json"] + # + # @example + # ```json + # ["a.json", "b.json", "c.json"] + # ``` - name: input path: /tmp/group/input.json @@ -25,8 +30,14 @@ spec: outputs: parameters: - # Ids of output files - # eg "000", "001", "002", "003" + # + # JSON array of all the group ids, which correspond to a output artifact file + # + # @example + # ```json + # ["000", "001", "002", "003"] + # ``` + # # Workflows should use the "000" string to access the data from the output artifact folder - name: output valueFrom: @@ -34,9 +45,11 @@ spec: artifacts: # Grouped output of the input as one file per output groupId + # # - /output/000.json # - /output/001.json # - ... + # - name: output path: /tmp/group/output/ From d98a560220adc723f352c8cbe4399e5feaf0c492 Mon Sep 17 00:00:00 2001 From: Blayne Chard Date: Tue, 25 Jul 2023 15:52:52 +1200 Subject: [PATCH 4/4] refactor: apply formatter --- templates/argo-tasks/group.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/argo-tasks/group.yml b/templates/argo-tasks/group.yml index d3a75d309..b66ab01c5 100644 --- a/templates/argo-tasks/group.yml +++ b/templates/argo-tasks/group.yml @@ -12,7 +12,7 @@ spec: artifacts: # # JSON array of things to group - # + # # @example # ```json # ["a.json", "b.json", "c.json"] @@ -33,7 +33,7 @@ spec: # # JSON array of all the group ids, which correspond to a output artifact file # - # @example + # @example # ```json # ["000", "001", "002", "003"] # ```