Use input arguments to specify table name and path prefix

lsst-dm · Nov 12, 2021 · df361e9 · df361e9
1 parent 3df2256
commit df361e9
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 4 deletions.
diff --git a/argo-submit-preingest.sh b/argo-submit-preingest.sh
@@ -0,0 +1,4 @@
+
+argo submit   --serviceaccount=argo-workflow -p 'workflow-table-name=objectTable_tract' -p 'prefix=PREOPS-863'  manifests/preingest.yaml
+argo submit   --serviceaccount=argo-workflow -p 'workflow-table-name=sourceTable_visit' -p 'prefix=PREOPS-863'  manifests/preingest.yaml
+argo submit   --serviceaccount=argo-workflow -p 'workflow-table-name=forcedSourceTable' -p 'prefix=PREOPS-863'  manifests/preingest.yaml
diff --git a/manifests/preingest.yaml b/manifests/preingest.yaml
@@ -12,6 +12,10 @@ metadata:
     workflows.argoproj.io/version: ">= 3.0.0"
 spec:
   entrypoint: main
+  arguments:
+    parameters:
+    - name: workflow-table-name
+    - name: prefix
   volumes:
   - name: secret-vol
     secret:
@@ -27,7 +31,7 @@ spec:
                 - name: input-folder
                   gcs:
                     bucket: qserv-us-central1-argo-artifact
-                    key: dc2/w_2021_40/input-parquet-source
+                    key: "dp02/{{workflow.parameters.prefix}}/{{workflow.parameters.workflow-table-name}}/input-parquet"
                     serviceAccountKeySecret:
                       name: secret-argo-gcs-artifact
                       key: qserv-dev-argo.json
@@ -41,21 +45,21 @@ spec:
                 - name: inparq
                   gcs:
                     bucket: qserv-us-central1-argo-artifact
-                    key: "dc2/w_2021_40/input-parquet-source/{{item}}"
+                    key: "dp02/{{workflow.parameters.prefix}}/{{workflow.parameters.workflow-table-name}}/input-parquet/{{item}}"
                     serviceAccountKeySecret:
                       name: secret-argo-gcs-artifact
                       key: qserv-dev-argo.json
                 - name: schema_abh
                   gcs:
                     bucket: qserv-us-central1-argo-artifact
-                    key: dc2/w_2021_40/configs-source/schema.abh.source
+                    key: "dp02/{{workflow.parameters.prefix}}/{{workflow.parameters.workflow-table-name}}/configs/schema.abh"
                     serviceAccountKeySecret:
                       name: secret-argo-gcs-artifact
                       key: qserv-dev-argo.json
                 - name: partition_config
                   gcs:
                     bucket: qserv-us-central1-argo-artifact
-                    key: dc2/w_2021_40/configs-source/partition.json
+                    key: "dp02/{{workflow.parameters.prefix}}/{{workflow.parameters.workflow-table-name}}/configs/partition.json"
                     serviceAccountKeySecret:
                       name: secret-argo-gcs-artifact
                       key: qserv-dev-argo.json