Why isn't the output working?

catalyst-cooperative · Jan 5, 2024 · 2a922bc · 2a922bc
1 parent 6d8314f
commit 2a922bc
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 12 deletions.
diff --git a/.github/workflows/build-deploy-pudl.yml b/.github/workflows/build-deploy-pudl.yml
@@ -136,6 +136,7 @@ jobs:
           PUDL_OUTPUT_PATH: ${{ env.GCS_OUTPUT_BUCKET }}/${{ env.BUILD_ID }}
         run: |-
           ./devtools/generate_batch_config.py \
+            --output ${{ env.BATCH_JOB_JSON }} \
             --container-image "docker.io/catalystcoop/pudl-etl@${{ steps.docker-build.outputs.digest }}" \
             --container-command "micromamba" \
             --container-arg="run" \
@@ -166,7 +167,6 @@ jobs:
             --container-env PUDL_SETTINGS_YML="/home/mambauser/pudl/src/pudl/package_data/settings/etl_full.yml" \
             --container-env SLACK_TOKEN=${{ secrets.PUDL_DEPLOY_SLACK_TOKEN }} \
             --container-env ZENODO_SANDBOX_TOKEN_PUBLISH=${{ secrets.ZENODO_SANDBOX_TOKEN_PUBLISH }} \
-            --output ${{ env.BATCH_JOB_JSON }}
 
       # Start the batch job
       - name: Kick off batch job

diff --git a/devtools/generate_batch_config.py b/devtools/generate_batch_config.py
@@ -12,9 +12,13 @@
 import argparse
 import itertools
 import json
+import logging
 from collections import OrderedDict
 from pathlib import Path
 
+logging.basicConfig()
+logger = logging.getLogger()
+
 
 def _flat(ls: list[list]) -> list:
     return list(itertools.chain.from_iterable(ls))
@@ -23,18 +27,12 @@ def _flat(ls: list[list]) -> list:
 def to_config(
     *,
     container_image: str,
-    container_env_file: Path,
     container_env: str,
     container_command: str,
     container_arg: str,
 ) -> dict:
     """Munge arguments into a configuration dictionary."""
-    if container_env_file is None or not container_env_file.is_file():
-        env_from_file = []
-    else:
-        with container_env_file.open("r") as f:
-            env_from_file = [line.strip() for line in f.readlines()]
-    complete_env = sorted(env_from_file + _flat(container_env))
+    complete_env = sorted(_flat(container_env))
     env_dict = OrderedDict(
         (name, value.strip('"'))
         for name, value in (pair.split("=", maxsplit=1) for pair in complete_env)
@@ -75,21 +73,19 @@ def generate_batch_config():
     parser = argparse.ArgumentParser()
     parser.add_argument("--container-image")
     parser.add_argument("--container-command")
-    parser.add_argument("--container-env-file", type=Path)
     parser.add_argument("--container-env", action="append", nargs="*", default=[])
     parser.add_argument("--container-arg", action="append", nargs="*", default=[])
     parser.add_argument("--output", type=Path)
     args = parser.parse_args()
+
     config = to_config(
         container_image=args.container_image,
         container_command=args.container_command,
         container_arg=args.container_arg,
-        container_env_file=args.container_env_file,
         container_env=args.container_env,
     )
-    # no-op change to see if the two jobs both get kicked off with different
-    # digests
 
+    logger.info(f"Writing to {args.output}")
     with args.output.open("w") as f:
         f.write(json.dumps(config, indent=2))