diff --git a/.github/workflows/build-deploy-pudl.yml b/.github/workflows/build-deploy-pudl.yml index 7348e0bb3a..1b59220fd9 100644 --- a/.github/workflows/build-deploy-pudl.yml +++ b/.github/workflows/build-deploy-pudl.yml @@ -136,6 +136,7 @@ jobs: PUDL_OUTPUT_PATH: ${{ env.GCS_OUTPUT_BUCKET }}/${{ env.BUILD_ID }} run: |- ./devtools/generate_batch_config.py \ + --output ${{ env.BATCH_JOB_JSON }} \ --container-image "docker.io/catalystcoop/pudl-etl@${{ steps.docker-build.outputs.digest }}" \ --container-command "micromamba" \ --container-arg="run" \ @@ -166,7 +167,6 @@ jobs: --container-env PUDL_SETTINGS_YML="/home/mambauser/pudl/src/pudl/package_data/settings/etl_full.yml" \ --container-env SLACK_TOKEN=${{ secrets.PUDL_DEPLOY_SLACK_TOKEN }} \ --container-env ZENODO_SANDBOX_TOKEN_PUBLISH=${{ secrets.ZENODO_SANDBOX_TOKEN_PUBLISH }} \ - --output ${{ env.BATCH_JOB_JSON }} # Start the batch job - name: Kick off batch job diff --git a/devtools/generate_batch_config.py b/devtools/generate_batch_config.py index bccad68148..896d78bf14 100755 --- a/devtools/generate_batch_config.py +++ b/devtools/generate_batch_config.py @@ -12,9 +12,13 @@ import argparse import itertools import json +import logging from collections import OrderedDict from pathlib import Path +logging.basicConfig() +logger = logging.getLogger() + def _flat(ls: list[list]) -> list: return list(itertools.chain.from_iterable(ls)) @@ -23,18 +27,12 @@ def _flat(ls: list[list]) -> list: def to_config( *, container_image: str, - container_env_file: Path, container_env: str, container_command: str, container_arg: str, ) -> dict: """Munge arguments into a configuration dictionary.""" - if container_env_file is None or not container_env_file.is_file(): - env_from_file = [] - else: - with container_env_file.open("r") as f: - env_from_file = [line.strip() for line in f.readlines()] - complete_env = sorted(env_from_file + _flat(container_env)) + complete_env = sorted(_flat(container_env)) env_dict = OrderedDict( (name, value.strip('"')) for name, value in (pair.split("=", maxsplit=1) for pair in complete_env) @@ -75,21 +73,19 @@ def generate_batch_config(): parser = argparse.ArgumentParser() parser.add_argument("--container-image") parser.add_argument("--container-command") - parser.add_argument("--container-env-file", type=Path) parser.add_argument("--container-env", action="append", nargs="*", default=[]) parser.add_argument("--container-arg", action="append", nargs="*", default=[]) parser.add_argument("--output", type=Path) args = parser.parse_args() + config = to_config( container_image=args.container_image, container_command=args.container_command, container_arg=args.container_arg, - container_env_file=args.container_env_file, container_env=args.container_env, ) - # no-op change to see if the two jobs both get kicked off with different - # digests + logger.info(f"Writing to {args.output}") with args.output.open("w") as f: f.write(json.dumps(config, indent=2))