From 87eeadcbd39f51c47a8400e6c43915d8ac3bbf11 Mon Sep 17 00:00:00 2001 From: James Meakin <12661555+jmsmkn@users.noreply.github.com> Date: Fri, 3 Mar 2023 16:12:34 +0100 Subject: [PATCH] Unify the container templates (#342) Working with pretty much a duplicate of the container template directory was annoying, so this PR unifies the templates by introducing a `template_kind` option. --- evalutils/cli.py | 22 ++-- evalutils/evalutils.py | 2 +- .../algorithm/hooks/post_gen_project.py | 42 ------- .../.gitattributes.j2 | 1 - .../Dockerfile | 24 ---- .../{{ cookiecutter.package_name }}/README.md | 6 - .../{algorithm => container}/__init__.py | 0 .../cookiecutter.json | 10 +- .../hooks/__init__.py | 0 .../container/hooks/post_gen_project.py | 77 +++++++++++++ .../hooks/pre_gen_project.py | 0 .../.dockerignore | 0 .../.gitattributes.j2 | 0 .../.github/workflows/ci.yml | 2 +- .../.gitignore | 0 .../Dockerfile | 33 ++++++ .../{{ cookiecutter.package_name }}/README.md | 6 + ....0.00.0.0000000000.0000.0000000000.000.mhd | 0 ...0.00.0.0000000000.0000.0000000000.000.zraw | Bin .../results_classification.json | 0 .../algorithm_test}/results_detection.json | 0 .../algorithm_test}/results_segmentation.json | 0 .../{{ cookiecutter.package_name }}/build.sh | 0 .../evaluation.py.j2 | 14 +-- ....0.00.0.0000000000.0000.0000000000.000.mhd | 0 ...0.00.0.0000000000.0000.0000000000.000.zraw | Bin ....276.0.28.3.0.14.4.0.20090213134050413.mhd | 0 ...276.0.28.3.0.14.4.0.20090213134050413.zraw | Bin .../evaluation_test}/detection-submission.csv | 0 .../evaluation_test}/submission.csv | 0 .../{{ cookiecutter.package_name }}/export.sh | 0 ....0.00.0.0000000000.0000.0000000000.000.mhd | 0 ...0.00.0.0000000000.0000.0000000000.000.zraw | Bin ....276.0.28.3.0.14.4.0.20090213134050413.mhd | 0 ...276.0.28.3.0.14.4.0.20090213134050413.zraw | Bin .../ground-truth/detection-reference.csv | 0 .../ground-truth/reference.csv | 0 .../process.py.j2 | 14 +-- .../requirements.in | 0 .../{{ cookiecutter.package_name }}/test.sh | 13 ++- evalutils/templates/evaluation/__init__.py | 0 .../templates/evaluation/cookiecutter.json | 14 --- .../templates/evaluation/hooks/__init__.py | 0 .../evaluation/hooks/post_gen_project.py | 54 --------- .../evaluation/hooks/pre_gen_project.py | 11 -- .../.dockerignore | 3 - .../.github/workflows/ci.yml | 23 ---- .../.gitignore | 105 ------------------ .../Dockerfile | 26 ----- .../{{ cookiecutter.package_name }}/README.md | 6 - .../{{ cookiecutter.package_name }}/build.sh | 4 - .../{{ cookiecutter.package_name }}/export.sh | 5 - .../requirements.in | 3 - .../{{ cookiecutter.package_name }}/test.sh | 28 ----- tests/test_algorithm.py | 4 +- tests/test_templates.py | 4 +- 56 files changed, 167 insertions(+), 389 deletions(-) delete mode 100644 evalutils/templates/algorithm/hooks/post_gen_project.py delete mode 100644 evalutils/templates/algorithm/{{ cookiecutter.package_name }}/.gitattributes.j2 delete mode 100644 evalutils/templates/algorithm/{{ cookiecutter.package_name }}/Dockerfile delete mode 100644 evalutils/templates/algorithm/{{ cookiecutter.package_name }}/README.md rename evalutils/templates/{algorithm => container}/__init__.py (100%) rename evalutils/templates/{algorithm => container}/cookiecutter.json (52%) rename evalutils/templates/{algorithm => container}/hooks/__init__.py (100%) create mode 100644 evalutils/templates/container/hooks/post_gen_project.py rename evalutils/templates/{algorithm => container}/hooks/pre_gen_project.py (100%) rename evalutils/templates/{algorithm => container}/{{ cookiecutter.package_name }}/.dockerignore (100%) rename evalutils/templates/{evaluation => container}/{{ cookiecutter.package_name }}/.gitattributes.j2 (100%) rename evalutils/templates/{algorithm => container}/{{ cookiecutter.package_name }}/.github/workflows/ci.yml (96%) rename evalutils/templates/{algorithm => container}/{{ cookiecutter.package_name }}/.gitignore (100%) create mode 100644 evalutils/templates/container/{{ cookiecutter.package_name }}/Dockerfile create mode 100644 evalutils/templates/container/{{ cookiecutter.package_name }}/README.md rename evalutils/templates/{algorithm/{{ cookiecutter.package_name }}/test => container/{{ cookiecutter.package_name }}/algorithm_test}/1.0.000.000000.0.00.0.0000000000.0000.0000000000.000.mhd (100%) rename evalutils/templates/{algorithm/{{ cookiecutter.package_name }}/test => container/{{ cookiecutter.package_name }}/algorithm_test}/1.0.000.000000.0.00.0.0000000000.0000.0000000000.000.zraw (100%) rename evalutils/templates/{algorithm/{{ cookiecutter.package_name }}/test => container/{{ cookiecutter.package_name }}/algorithm_test}/results_classification.json (100%) rename evalutils/templates/{algorithm/{{ cookiecutter.package_name }}/test => container/{{ cookiecutter.package_name }}/algorithm_test}/results_detection.json (100%) rename evalutils/templates/{algorithm/{{ cookiecutter.package_name }}/test => container/{{ cookiecutter.package_name }}/algorithm_test}/results_segmentation.json (100%) rename evalutils/templates/{algorithm => container}/{{ cookiecutter.package_name }}/build.sh (100%) rename evalutils/templates/{evaluation => container}/{{ cookiecutter.package_name }}/evaluation.py.j2 (91%) rename evalutils/templates/{evaluation/{{ cookiecutter.package_name }}/test => container/{{ cookiecutter.package_name }}/evaluation_test}/1.0.000.000000.0.00.0.0000000000.0000.0000000000.000.mhd (100%) rename evalutils/templates/{evaluation/{{ cookiecutter.package_name }}/test => container/{{ cookiecutter.package_name }}/evaluation_test}/1.0.000.000000.0.00.0.0000000000.0000.0000000000.000.zraw (100%) rename evalutils/templates/{evaluation/{{ cookiecutter.package_name }}/test => container/{{ cookiecutter.package_name }}/evaluation_test}/1.2.276.0.28.3.0.14.4.0.20090213134050413.mhd (100%) rename evalutils/templates/{evaluation/{{ cookiecutter.package_name }}/test => container/{{ cookiecutter.package_name }}/evaluation_test}/1.2.276.0.28.3.0.14.4.0.20090213134050413.zraw (100%) rename evalutils/templates/{evaluation/{{ cookiecutter.package_name }}/test => container/{{ cookiecutter.package_name }}/evaluation_test}/detection-submission.csv (100%) rename evalutils/templates/{evaluation/{{ cookiecutter.package_name }}/test => container/{{ cookiecutter.package_name }}/evaluation_test}/submission.csv (100%) rename evalutils/templates/{algorithm => container}/{{ cookiecutter.package_name }}/export.sh (100%) rename evalutils/templates/{evaluation => container}/{{ cookiecutter.package_name }}/ground-truth/1.0.000.000000.0.00.0.0000000000.0000.0000000000.000.mhd (100%) rename evalutils/templates/{evaluation => container}/{{ cookiecutter.package_name }}/ground-truth/1.0.000.000000.0.00.0.0000000000.0000.0000000000.000.zraw (100%) rename evalutils/templates/{evaluation => container}/{{ cookiecutter.package_name }}/ground-truth/1.2.276.0.28.3.0.14.4.0.20090213134050413.mhd (100%) rename evalutils/templates/{evaluation => container}/{{ cookiecutter.package_name }}/ground-truth/1.2.276.0.28.3.0.14.4.0.20090213134050413.zraw (100%) rename evalutils/templates/{evaluation => container}/{{ cookiecutter.package_name }}/ground-truth/detection-reference.csv (100%) rename evalutils/templates/{evaluation => container}/{{ cookiecutter.package_name }}/ground-truth/reference.csv (100%) rename evalutils/templates/{algorithm => container}/{{ cookiecutter.package_name }}/process.py.j2 (86%) rename evalutils/templates/{algorithm => container}/{{ cookiecutter.package_name }}/requirements.in (100%) rename evalutils/templates/{algorithm => container}/{{ cookiecutter.package_name }}/test.sh (73%) delete mode 100644 evalutils/templates/evaluation/__init__.py delete mode 100644 evalutils/templates/evaluation/cookiecutter.json delete mode 100644 evalutils/templates/evaluation/hooks/__init__.py delete mode 100644 evalutils/templates/evaluation/hooks/post_gen_project.py delete mode 100644 evalutils/templates/evaluation/hooks/pre_gen_project.py delete mode 100644 evalutils/templates/evaluation/{{ cookiecutter.package_name }}/.dockerignore delete mode 100644 evalutils/templates/evaluation/{{ cookiecutter.package_name }}/.github/workflows/ci.yml delete mode 100644 evalutils/templates/evaluation/{{ cookiecutter.package_name }}/.gitignore delete mode 100644 evalutils/templates/evaluation/{{ cookiecutter.package_name }}/Dockerfile delete mode 100644 evalutils/templates/evaluation/{{ cookiecutter.package_name }}/README.md delete mode 100755 evalutils/templates/evaluation/{{ cookiecutter.package_name }}/build.sh delete mode 100755 evalutils/templates/evaluation/{{ cookiecutter.package_name }}/export.sh delete mode 100644 evalutils/templates/evaluation/{{ cookiecutter.package_name }}/requirements.in delete mode 100755 evalutils/templates/evaluation/{{ cookiecutter.package_name }}/test.sh diff --git a/evalutils/cli.py b/evalutils/cli.py index 63a30d5..51fe013 100644 --- a/evalutils/cli.py +++ b/evalutils/cli.py @@ -30,12 +30,12 @@ def init(): def validate_python_module_name_fn(option): def validate_python_module_name_string(ctx, param, arg): if len(arg.strip()) == 0: - click.echo(f"{option.upper()} should be non empty. Aborting...") - exit(1) + raise click.BadParameter(f"{option.upper()} should be non empty") if not re.match(MODULE_REGEX, arg) or arg in FORBIDDEN_NAMES: - click.echo(f"ERROR: {arg!r} is not a valid Python module name!") - exit(1) + raise click.BadParameter( + f"{arg!r} is not a valid Python module name" + ) return arg @@ -79,14 +79,15 @@ def convert(self, value, param, ctx): ) @click.option("--dev", is_flag=True) def init_evaluation(challenge_name, kind, dev): - template_dir = Path(__file__).parent / "templates" / "evaluation" + template_dir = Path(__file__).parent / "templates" / "container" try: cookiecutter( template=str(template_dir.absolute()), no_input=True, extra_context={ - "challenge_name": challenge_name, - "challenge_kind": kind, + "full_project_name": challenge_name, + "task_kind": kind, + "template_kind": "Evaluation", **_get_cookiecutter_base_context(dev_build=dev), }, ) @@ -167,14 +168,15 @@ def req_gpu_prompt(ctx, param, req_gpu_count): ) @click.option("--dev", is_flag=True) def init_algorithm(algorithm_name, kind, dev): - template_dir = Path(__file__).parent / "templates" / "algorithm" + template_dir = Path(__file__).parent / "templates" / "container" try: cookiecutter( template=str(template_dir.absolute()), no_input=True, extra_context={ - "algorithm_name": algorithm_name, - "algorithm_kind": kind, + "full_project_name": algorithm_name, + "task_kind": kind, + "template_kind": "Algorithm", **_get_cookiecutter_base_context(dev_build=dev), }, ) diff --git a/evalutils/evalutils.py b/evalutils/evalutils.py index 645d273..4d63d40 100644 --- a/evalutils/evalutils.py +++ b/evalutils/evalutils.py @@ -36,7 +36,7 @@ DEFAULT_INPUT_PATH = Path("/input/") DEFAULT_ALGORITHM_OUTPUT_IMAGES_PATH = Path("/output/images/") DEFAULT_ALGORITHM_OUTPUT_FILE_PATH = Path("/output/results.json") -DEFAULT_GROUND_TRUTH_PATH = Path("/opt/evaluation/ground-truth/") +DEFAULT_GROUND_TRUTH_PATH = Path("/opt/app/ground-truth/") DEFAULT_EVALUATION_OUTPUT_FILE_PATH = Path("/output/metrics.json") diff --git a/evalutils/templates/algorithm/hooks/post_gen_project.py b/evalutils/templates/algorithm/hooks/post_gen_project.py deleted file mode 100644 index 9fc9608..0000000 --- a/evalutils/templates/algorithm/hooks/post_gen_project.py +++ /dev/null @@ -1,42 +0,0 @@ -import os -import shutil -from pathlib import Path - -from evalutils.utils import ( - convert_line_endings, - generate_requirements_txt, - generate_source_wheel, -) - -ALGORITHM_KIND = "{{ cookiecutter.algorithm_kind }}" -ALGORITHM_NAME = "{{ cookiecutter.algorithm_name }}" -IS_DEV_BUILD = int("{{ cookiecutter.dev_build }}") == 1 - -template_dir = Path(os.getcwd()) -template_test_dir = template_dir / "test" - -templated_files = template_dir.glob("*.j2") -for f in templated_files: - shutil.move(f.name, f.stem) - - -def remove_result_files(): - for algorithm_kind in ["segmentation", "detection", "classification"]: - os.remove(template_test_dir / f"results_{algorithm_kind}.json") - - -expected_output_file = ( - template_test_dir / f"results_{ALGORITHM_KIND.lower()}.json" -) - -shutil.copy( - str(expected_output_file), template_test_dir / "expected_output.json" -) - -remove_result_files() - -if IS_DEV_BUILD: - generate_source_wheel(template_dir / "vendor") - -generate_requirements_txt() -convert_line_endings() diff --git a/evalutils/templates/algorithm/{{ cookiecutter.package_name }}/.gitattributes.j2 b/evalutils/templates/algorithm/{{ cookiecutter.package_name }}/.gitattributes.j2 deleted file mode 100644 index 66c8e07..0000000 --- a/evalutils/templates/algorithm/{{ cookiecutter.package_name }}/.gitattributes.j2 +++ /dev/null @@ -1 +0,0 @@ -test/* filter=lfs diff=lfs merge=lfs -text diff --git a/evalutils/templates/algorithm/{{ cookiecutter.package_name }}/Dockerfile b/evalutils/templates/algorithm/{{ cookiecutter.package_name }}/Dockerfile deleted file mode 100644 index a48f437..0000000 --- a/evalutils/templates/algorithm/{{ cookiecutter.package_name }}/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -FROM python:{{ cookiecutter.python_major_version }}.{{ cookiecutter.python_minor_version }}-slim - -RUN groupadd -r algorithm && useradd -m --no-log-init -r -g algorithm algorithm - -RUN mkdir -p /opt/algorithm /input /output \ - && chown algorithm:algorithm /opt/algorithm /input /output - -USER algorithm -WORKDIR /opt/algorithm - -ENV PATH="/home/algorithm/.local/bin:${PATH}" - -RUN python -m pip install --user -U pip && python -m pip install --user pip-tools - -{% if cookiecutter.dev_build|int -%} -COPY --chown=algorithm:algorithm vendor /opt/algorithm/vendor -{%- endif %} - -COPY --chown=algorithm:algorithm requirements.txt /opt/algorithm/ -RUN python -m piptools sync requirements.txt - -COPY --chown=algorithm:algorithm process.py /opt/algorithm/ - -ENTRYPOINT [ "python", "-m", "process" ] diff --git a/evalutils/templates/algorithm/{{ cookiecutter.package_name }}/README.md b/evalutils/templates/algorithm/{{ cookiecutter.package_name }}/README.md deleted file mode 100644 index bc1833d..0000000 --- a/evalutils/templates/algorithm/{{ cookiecutter.package_name }}/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# {{ cookiecutter.algorithm_name }} Algorithm - -The source code for the algorithm container for -{{ cookiecutter.algorithm_name }}, generated with -evalutils version {{ cookiecutter.evalutils_version }} -using Python {{ cookiecutter.python_major_version }}.{{ cookiecutter.python_minor_version }}. diff --git a/evalutils/templates/algorithm/__init__.py b/evalutils/templates/container/__init__.py similarity index 100% rename from evalutils/templates/algorithm/__init__.py rename to evalutils/templates/container/__init__.py diff --git a/evalutils/templates/algorithm/cookiecutter.json b/evalutils/templates/container/cookiecutter.json similarity index 52% rename from evalutils/templates/algorithm/cookiecutter.json rename to evalutils/templates/container/cookiecutter.json index 4f86618..817a4e5 100644 --- a/evalutils/templates/algorithm/cookiecutter.json +++ b/evalutils/templates/container/cookiecutter.json @@ -1,11 +1,15 @@ { - "algorithm_name": "", - "algorithm_kind": [ + "template_kind": [ + "Algorithm", + "Evaluation" + ], + "full_project_name": "", + "task_kind": [ "Classification", "Segmentation", "Detection" ], - "package_name": "{{ cookiecutter.algorithm_name|replace(' ', '') }}", + "package_name": "{{ cookiecutter.full_project_name|replace(' ', '') }}", "evalutils_version": "", "dev_build": 0, "python_major_version": "", diff --git a/evalutils/templates/algorithm/hooks/__init__.py b/evalutils/templates/container/hooks/__init__.py similarity index 100% rename from evalutils/templates/algorithm/hooks/__init__.py rename to evalutils/templates/container/hooks/__init__.py diff --git a/evalutils/templates/container/hooks/post_gen_project.py b/evalutils/templates/container/hooks/post_gen_project.py new file mode 100644 index 0000000..dda0ea9 --- /dev/null +++ b/evalutils/templates/container/hooks/post_gen_project.py @@ -0,0 +1,77 @@ +import os +import shutil +from pathlib import Path + +from evalutils.utils import ( + convert_line_endings, + generate_requirements_txt, + generate_source_wheel, +) + +TASK_KIND = "{{ cookiecutter.task_kind }}" +TEMPLATE_KIND = "{{ cookiecutter.template_kind }}" +IS_DEV_BUILD = int("{{ cookiecutter.dev_build }}") == 1 + +template_dir = Path(os.getcwd()) + +templated_files = template_dir.glob("*.j2") +for f in templated_files: + shutil.move(f.name, f.stem) + +if TEMPLATE_KIND == "Evaluation": # noqa: C901 + shutil.rmtree("algorithm_test") + os.remove("process.py") + os.rename("evaluation_test", "test") + + def remove_classification_files(): + os.remove(Path("ground-truth") / "reference.csv") + os.remove(Path("test") / "submission.csv") + + def remove_segmentation_files(): + files = [] + for ext in ["mhd", "zraw"]: + files.extend(Path(".").glob(f"**/*.{ext}")) + + for file in files: + os.remove(str(file)) + + def remove_detection_files(): + os.remove(Path("ground-truth") / "detection-reference.csv") + os.remove(Path("test") / "detection-submission.csv") + + if TASK_KIND.lower() != "segmentation": + remove_segmentation_files() + + if TASK_KIND.lower() != "detection": + remove_detection_files() + + if TASK_KIND.lower() != "classification": + remove_classification_files() + +elif TEMPLATE_KIND == "Algorithm": + shutil.rmtree("evaluation_test") + shutil.rmtree("ground-truth") + os.remove("evaluation.py") + os.rename("algorithm_test", "test") + + template_test_dir = template_dir / "test" + + def remove_result_files(): + for task_kind in ["segmentation", "detection", "classification"]: + os.remove(template_test_dir / f"results_{task_kind}.json") + + expected_output_file = ( + template_test_dir / f"results_{TASK_KIND.lower()}.json" + ) + + shutil.copy( + str(expected_output_file), template_test_dir / "expected_output.json" + ) + + remove_result_files() + +if IS_DEV_BUILD: + generate_source_wheel(template_dir / "vendor") + +generate_requirements_txt() +convert_line_endings() diff --git a/evalutils/templates/algorithm/hooks/pre_gen_project.py b/evalutils/templates/container/hooks/pre_gen_project.py similarity index 100% rename from evalutils/templates/algorithm/hooks/pre_gen_project.py rename to evalutils/templates/container/hooks/pre_gen_project.py diff --git a/evalutils/templates/algorithm/{{ cookiecutter.package_name }}/.dockerignore b/evalutils/templates/container/{{ cookiecutter.package_name }}/.dockerignore similarity index 100% rename from evalutils/templates/algorithm/{{ cookiecutter.package_name }}/.dockerignore rename to evalutils/templates/container/{{ cookiecutter.package_name }}/.dockerignore diff --git a/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/.gitattributes.j2 b/evalutils/templates/container/{{ cookiecutter.package_name }}/.gitattributes.j2 similarity index 100% rename from evalutils/templates/evaluation/{{ cookiecutter.package_name }}/.gitattributes.j2 rename to evalutils/templates/container/{{ cookiecutter.package_name }}/.gitattributes.j2 diff --git a/evalutils/templates/algorithm/{{ cookiecutter.package_name }}/.github/workflows/ci.yml b/evalutils/templates/container/{{ cookiecutter.package_name }}/.github/workflows/ci.yml similarity index 96% rename from evalutils/templates/algorithm/{{ cookiecutter.package_name }}/.github/workflows/ci.yml rename to evalutils/templates/container/{{ cookiecutter.package_name }}/.github/workflows/ci.yml index dd0570c..b30b489 100644 --- a/evalutils/templates/algorithm/{{ cookiecutter.package_name }}/.github/workflows/ci.yml +++ b/evalutils/templates/container/{{ cookiecutter.package_name }}/.github/workflows/ci.yml @@ -7,7 +7,7 @@ env: jobs: - algorithm-tests: + tests: runs-on: ubuntu-latest steps: - name: Install Python ${{ "{{" }} env.PYTHON_VERSION {{ "}}" }} diff --git a/evalutils/templates/algorithm/{{ cookiecutter.package_name }}/.gitignore b/evalutils/templates/container/{{ cookiecutter.package_name }}/.gitignore similarity index 100% rename from evalutils/templates/algorithm/{{ cookiecutter.package_name }}/.gitignore rename to evalutils/templates/container/{{ cookiecutter.package_name }}/.gitignore diff --git a/evalutils/templates/container/{{ cookiecutter.package_name }}/Dockerfile b/evalutils/templates/container/{{ cookiecutter.package_name }}/Dockerfile new file mode 100644 index 0000000..bfb39de --- /dev/null +++ b/evalutils/templates/container/{{ cookiecutter.package_name }}/Dockerfile @@ -0,0 +1,33 @@ +FROM python:{{ cookiecutter.python_major_version }}.{{ cookiecutter.python_minor_version }}-slim + +RUN groupadd -r user && useradd -m --no-log-init -r -g user user + +RUN mkdir -p /opt/app /input /output \ + && chown user:user /opt/app /input /output + +USER user +WORKDIR /opt/app + +ENV PATH="/home/user/.local/bin:${PATH}" + +RUN python -m pip install --user -U pip && python -m pip install --user pip-tools + +{% if cookiecutter.dev_build|int -%} +COPY --chown=user:user vendor /opt/app/vendor +{%- endif %} + +COPY --chown=user:user requirements.txt /opt/app/ +RUN python -m piptools sync requirements.txt + +{% if cookiecutter.template_kind == "Evaluation" -%} +COPY --chown=user:user ground-truth /opt/app/ground-truth +COPY --chown=user:user evaluation.py /opt/app/ + +ENTRYPOINT [ "python", "-m", "evaluation" ] +{%- endif %} + +{% if cookiecutter.template_kind == "Algorithm" -%} +COPY --chown=user:user process.py /opt/app/ + +ENTRYPOINT [ "python", "-m", "process" ] +{%- endif %} diff --git a/evalutils/templates/container/{{ cookiecutter.package_name }}/README.md b/evalutils/templates/container/{{ cookiecutter.package_name }}/README.md new file mode 100644 index 0000000..9815a24 --- /dev/null +++ b/evalutils/templates/container/{{ cookiecutter.package_name }}/README.md @@ -0,0 +1,6 @@ +# {{ cookiecutter.full_project_name }} {{ cookiecutter.template_kind }} + +The source code for the {{ cookiecutter.template_kind.lower() }} container for +{{ cookiecutter.full_project_name }}, generated with +evalutils version {{ cookiecutter.evalutils_version }} +using Python {{ cookiecutter.python_major_version }}.{{ cookiecutter.python_minor_version }}. diff --git a/evalutils/templates/algorithm/{{ cookiecutter.package_name }}/test/1.0.000.000000.0.00.0.0000000000.0000.0000000000.000.mhd b/evalutils/templates/container/{{ cookiecutter.package_name }}/algorithm_test/1.0.000.000000.0.00.0.0000000000.0000.0000000000.000.mhd similarity index 100% rename from evalutils/templates/algorithm/{{ cookiecutter.package_name }}/test/1.0.000.000000.0.00.0.0000000000.0000.0000000000.000.mhd rename to evalutils/templates/container/{{ cookiecutter.package_name }}/algorithm_test/1.0.000.000000.0.00.0.0000000000.0000.0000000000.000.mhd diff --git a/evalutils/templates/algorithm/{{ cookiecutter.package_name }}/test/1.0.000.000000.0.00.0.0000000000.0000.0000000000.000.zraw b/evalutils/templates/container/{{ cookiecutter.package_name }}/algorithm_test/1.0.000.000000.0.00.0.0000000000.0000.0000000000.000.zraw similarity index 100% rename from evalutils/templates/algorithm/{{ cookiecutter.package_name }}/test/1.0.000.000000.0.00.0.0000000000.0000.0000000000.000.zraw rename to evalutils/templates/container/{{ cookiecutter.package_name }}/algorithm_test/1.0.000.000000.0.00.0.0000000000.0000.0000000000.000.zraw diff --git a/evalutils/templates/algorithm/{{ cookiecutter.package_name }}/test/results_classification.json b/evalutils/templates/container/{{ cookiecutter.package_name }}/algorithm_test/results_classification.json similarity index 100% rename from evalutils/templates/algorithm/{{ cookiecutter.package_name }}/test/results_classification.json rename to evalutils/templates/container/{{ cookiecutter.package_name }}/algorithm_test/results_classification.json diff --git a/evalutils/templates/algorithm/{{ cookiecutter.package_name }}/test/results_detection.json b/evalutils/templates/container/{{ cookiecutter.package_name }}/algorithm_test/results_detection.json similarity index 100% rename from evalutils/templates/algorithm/{{ cookiecutter.package_name }}/test/results_detection.json rename to evalutils/templates/container/{{ cookiecutter.package_name }}/algorithm_test/results_detection.json diff --git a/evalutils/templates/algorithm/{{ cookiecutter.package_name }}/test/results_segmentation.json b/evalutils/templates/container/{{ cookiecutter.package_name }}/algorithm_test/results_segmentation.json similarity index 100% rename from evalutils/templates/algorithm/{{ cookiecutter.package_name }}/test/results_segmentation.json rename to evalutils/templates/container/{{ cookiecutter.package_name }}/algorithm_test/results_segmentation.json diff --git a/evalutils/templates/algorithm/{{ cookiecutter.package_name }}/build.sh b/evalutils/templates/container/{{ cookiecutter.package_name }}/build.sh similarity index 100% rename from evalutils/templates/algorithm/{{ cookiecutter.package_name }}/build.sh rename to evalutils/templates/container/{{ cookiecutter.package_name }}/build.sh diff --git a/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/evaluation.py.j2 b/evalutils/templates/container/{{ cookiecutter.package_name }}/evaluation.py.j2 similarity index 91% rename from evalutils/templates/evaluation/{{ cookiecutter.package_name }}/evaluation.py.j2 rename to evalutils/templates/container/{{ cookiecutter.package_name }}/evaluation.py.j2 index 91533cf..3cbea9f 100644 --- a/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/evaluation.py.j2 +++ b/evalutils/templates/container/{{ cookiecutter.package_name }}/evaluation.py.j2 @@ -1,4 +1,4 @@ -{%- if cookiecutter.challenge_kind == "Classification" -%} +{%- if cookiecutter.task_kind == "Classification" -%} from sklearn.metrics import accuracy_score from evalutils import ClassificationEvaluation @@ -6,7 +6,7 @@ from evalutils.io import CSVLoader from evalutils.validators import ( NumberOfCasesValidator, ExpectedColumnNamesValidator ) -{%- elif cookiecutter.challenge_kind == "Segmentation" -%} +{%- elif cookiecutter.task_kind == "Segmentation" -%} import SimpleITK from evalutils import ClassificationEvaluation @@ -14,7 +14,7 @@ from evalutils.io import SimpleITKLoader from evalutils.validators import ( NumberOfCasesValidator, UniquePathIndicesValidator, UniqueImagesValidator ) -{%- elif cookiecutter.challenge_kind == "Detection" -%} +{%- elif cookiecutter.task_kind == "Detection" -%} from evalutils import DetectionEvaluation from evalutils.io import CSVLoader from evalutils.validators import ExpectedColumnNamesValidator @@ -23,14 +23,14 @@ from evalutils.validators import ExpectedColumnNamesValidator class {{ cookiecutter.package_name|capitalize }}( -{%- if cookiecutter.challenge_kind == "Detection" -%} +{%- if cookiecutter.task_kind == "Detection" -%} DetectionEvaluation {%- else -%} ClassificationEvaluation {%- endif -%} ): -{%- if cookiecutter.challenge_kind == "Classification" %} +{%- if cookiecutter.task_kind == "Classification" %} def __init__(self): super().__init__( file_loader=CSVLoader(), @@ -48,7 +48,7 @@ class {{ cookiecutter.package_name|capitalize }}( self._cases["class_prediction"], ), } -{% elif cookiecutter.challenge_kind == "Detection" %} +{% elif cookiecutter.task_kind == "Detection" %} def __init__(self): super().__init__( file_loader=CSVLoader(), @@ -78,7 +78,7 @@ class {{ cookiecutter.package_name|capitalize }}( for _, p in points.iterrows() if p["score"] > self._detection_threshold ] -{% elif cookiecutter.challenge_kind == "Segmentation" %} +{% elif cookiecutter.task_kind == "Segmentation" %} def __init__(self): super().__init__( file_loader=SimpleITKLoader(), diff --git a/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/test/1.0.000.000000.0.00.0.0000000000.0000.0000000000.000.mhd b/evalutils/templates/container/{{ cookiecutter.package_name }}/evaluation_test/1.0.000.000000.0.00.0.0000000000.0000.0000000000.000.mhd similarity index 100% rename from evalutils/templates/evaluation/{{ cookiecutter.package_name }}/test/1.0.000.000000.0.00.0.0000000000.0000.0000000000.000.mhd rename to evalutils/templates/container/{{ cookiecutter.package_name }}/evaluation_test/1.0.000.000000.0.00.0.0000000000.0000.0000000000.000.mhd diff --git a/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/test/1.0.000.000000.0.00.0.0000000000.0000.0000000000.000.zraw b/evalutils/templates/container/{{ cookiecutter.package_name }}/evaluation_test/1.0.000.000000.0.00.0.0000000000.0000.0000000000.000.zraw similarity index 100% rename from evalutils/templates/evaluation/{{ cookiecutter.package_name }}/test/1.0.000.000000.0.00.0.0000000000.0000.0000000000.000.zraw rename to evalutils/templates/container/{{ cookiecutter.package_name }}/evaluation_test/1.0.000.000000.0.00.0.0000000000.0000.0000000000.000.zraw diff --git a/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/test/1.2.276.0.28.3.0.14.4.0.20090213134050413.mhd b/evalutils/templates/container/{{ cookiecutter.package_name }}/evaluation_test/1.2.276.0.28.3.0.14.4.0.20090213134050413.mhd similarity index 100% rename from evalutils/templates/evaluation/{{ cookiecutter.package_name }}/test/1.2.276.0.28.3.0.14.4.0.20090213134050413.mhd rename to evalutils/templates/container/{{ cookiecutter.package_name }}/evaluation_test/1.2.276.0.28.3.0.14.4.0.20090213134050413.mhd diff --git a/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/test/1.2.276.0.28.3.0.14.4.0.20090213134050413.zraw b/evalutils/templates/container/{{ cookiecutter.package_name }}/evaluation_test/1.2.276.0.28.3.0.14.4.0.20090213134050413.zraw similarity index 100% rename from evalutils/templates/evaluation/{{ cookiecutter.package_name }}/test/1.2.276.0.28.3.0.14.4.0.20090213134050413.zraw rename to evalutils/templates/container/{{ cookiecutter.package_name }}/evaluation_test/1.2.276.0.28.3.0.14.4.0.20090213134050413.zraw diff --git a/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/test/detection-submission.csv b/evalutils/templates/container/{{ cookiecutter.package_name }}/evaluation_test/detection-submission.csv similarity index 100% rename from evalutils/templates/evaluation/{{ cookiecutter.package_name }}/test/detection-submission.csv rename to evalutils/templates/container/{{ cookiecutter.package_name }}/evaluation_test/detection-submission.csv diff --git a/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/test/submission.csv b/evalutils/templates/container/{{ cookiecutter.package_name }}/evaluation_test/submission.csv similarity index 100% rename from evalutils/templates/evaluation/{{ cookiecutter.package_name }}/test/submission.csv rename to evalutils/templates/container/{{ cookiecutter.package_name }}/evaluation_test/submission.csv diff --git a/evalutils/templates/algorithm/{{ cookiecutter.package_name }}/export.sh b/evalutils/templates/container/{{ cookiecutter.package_name }}/export.sh similarity index 100% rename from evalutils/templates/algorithm/{{ cookiecutter.package_name }}/export.sh rename to evalutils/templates/container/{{ cookiecutter.package_name }}/export.sh diff --git a/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/ground-truth/1.0.000.000000.0.00.0.0000000000.0000.0000000000.000.mhd b/evalutils/templates/container/{{ cookiecutter.package_name }}/ground-truth/1.0.000.000000.0.00.0.0000000000.0000.0000000000.000.mhd similarity index 100% rename from evalutils/templates/evaluation/{{ cookiecutter.package_name }}/ground-truth/1.0.000.000000.0.00.0.0000000000.0000.0000000000.000.mhd rename to evalutils/templates/container/{{ cookiecutter.package_name }}/ground-truth/1.0.000.000000.0.00.0.0000000000.0000.0000000000.000.mhd diff --git a/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/ground-truth/1.0.000.000000.0.00.0.0000000000.0000.0000000000.000.zraw b/evalutils/templates/container/{{ cookiecutter.package_name }}/ground-truth/1.0.000.000000.0.00.0.0000000000.0000.0000000000.000.zraw similarity index 100% rename from evalutils/templates/evaluation/{{ cookiecutter.package_name }}/ground-truth/1.0.000.000000.0.00.0.0000000000.0000.0000000000.000.zraw rename to evalutils/templates/container/{{ cookiecutter.package_name }}/ground-truth/1.0.000.000000.0.00.0.0000000000.0000.0000000000.000.zraw diff --git a/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/ground-truth/1.2.276.0.28.3.0.14.4.0.20090213134050413.mhd b/evalutils/templates/container/{{ cookiecutter.package_name }}/ground-truth/1.2.276.0.28.3.0.14.4.0.20090213134050413.mhd similarity index 100% rename from evalutils/templates/evaluation/{{ cookiecutter.package_name }}/ground-truth/1.2.276.0.28.3.0.14.4.0.20090213134050413.mhd rename to evalutils/templates/container/{{ cookiecutter.package_name }}/ground-truth/1.2.276.0.28.3.0.14.4.0.20090213134050413.mhd diff --git a/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/ground-truth/1.2.276.0.28.3.0.14.4.0.20090213134050413.zraw b/evalutils/templates/container/{{ cookiecutter.package_name }}/ground-truth/1.2.276.0.28.3.0.14.4.0.20090213134050413.zraw similarity index 100% rename from evalutils/templates/evaluation/{{ cookiecutter.package_name }}/ground-truth/1.2.276.0.28.3.0.14.4.0.20090213134050413.zraw rename to evalutils/templates/container/{{ cookiecutter.package_name }}/ground-truth/1.2.276.0.28.3.0.14.4.0.20090213134050413.zraw diff --git a/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/ground-truth/detection-reference.csv b/evalutils/templates/container/{{ cookiecutter.package_name }}/ground-truth/detection-reference.csv similarity index 100% rename from evalutils/templates/evaluation/{{ cookiecutter.package_name }}/ground-truth/detection-reference.csv rename to evalutils/templates/container/{{ cookiecutter.package_name }}/ground-truth/detection-reference.csv diff --git a/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/ground-truth/reference.csv b/evalutils/templates/container/{{ cookiecutter.package_name }}/ground-truth/reference.csv similarity index 100% rename from evalutils/templates/evaluation/{{ cookiecutter.package_name }}/ground-truth/reference.csv rename to evalutils/templates/container/{{ cookiecutter.package_name }}/ground-truth/reference.csv diff --git a/evalutils/templates/algorithm/{{ cookiecutter.package_name }}/process.py.j2 b/evalutils/templates/container/{{ cookiecutter.package_name }}/process.py.j2 similarity index 86% rename from evalutils/templates/algorithm/{{ cookiecutter.package_name }}/process.py.j2 rename to evalutils/templates/container/{{ cookiecutter.package_name }}/process.py.j2 index c2d57ac..a36fe36 100644 --- a/evalutils/templates/algorithm/{{ cookiecutter.package_name }}/process.py.j2 +++ b/evalutils/templates/container/{{ cookiecutter.package_name }}/process.py.j2 @@ -1,20 +1,20 @@ -{% if cookiecutter.algorithm_kind == "Classification" %} +{% if cookiecutter.task_kind == "Classification" %} from typing import Dict {% endif %} import SimpleITK import numpy as np -{% if cookiecutter.algorithm_kind == "Detection" %} +{% if cookiecutter.task_kind == "Detection" %} from pandas import DataFrame from scipy.ndimage import center_of_mass, label {% endif %} -from evalutils import {{ cookiecutter.algorithm_kind }}Algorithm +from evalutils import {{ cookiecutter.task_kind }}Algorithm from evalutils.validators import ( UniquePathIndicesValidator, UniqueImagesValidator, ) -class {{ cookiecutter.package_name|capitalize }}({{ cookiecutter.algorithm_kind }}Algorithm): +class {{ cookiecutter.package_name|capitalize }}({{ cookiecutter.task_kind }}Algorithm): def __init__(self): super().__init__( validators=dict( @@ -24,7 +24,7 @@ class {{ cookiecutter.package_name|capitalize }}({{ cookiecutter.algorithm_kind ) ), ) -{% if cookiecutter.algorithm_kind == "Detection" %} +{% if cookiecutter.task_kind == "Detection" %} def predict(self, *, input_image: SimpleITK.Image) -> DataFrame: # Extract a numpy array with image data from the SimpleITK Image image_data = SimpleITK.GetArrayFromImage(input_image) @@ -52,13 +52,13 @@ class {{ cookiecutter.package_name|capitalize }}({{ cookiecutter.algorithm_kind # Convert serialized candidates to a pandas.DataFrame return DataFrame(data) -{% elif cookiecutter.algorithm_kind == "Segmentation" %} +{% elif cookiecutter.task_kind == "Segmentation" %} def predict(self, *, input_image: SimpleITK.Image) -> SimpleITK.Image: # Segment all values greater than 2 in the input image return SimpleITK.BinaryThreshold( image1=input_image, lowerThreshold=2, insideValue=1, outsideValue=0 ) -{% elif cookiecutter.algorithm_kind == "Classification" %} +{% elif cookiecutter.task_kind == "Classification" %} def predict(self, *, input_image: SimpleITK.Image) -> Dict: # Checks if there are any nodules voxels (> 1) in the input image return dict( diff --git a/evalutils/templates/algorithm/{{ cookiecutter.package_name }}/requirements.in b/evalutils/templates/container/{{ cookiecutter.package_name }}/requirements.in similarity index 100% rename from evalutils/templates/algorithm/{{ cookiecutter.package_name }}/requirements.in rename to evalutils/templates/container/{{ cookiecutter.package_name }}/requirements.in diff --git a/evalutils/templates/algorithm/{{ cookiecutter.package_name }}/test.sh b/evalutils/templates/container/{{ cookiecutter.package_name }}/test.sh similarity index 73% rename from evalutils/templates/algorithm/{{ cookiecutter.package_name }}/test.sh rename to evalutils/templates/container/{{ cookiecutter.package_name }}/test.sh index 8d00def..aa97400 100755 --- a/evalutils/templates/algorithm/{{ cookiecutter.package_name }}/test.sh +++ b/evalutils/templates/container/{{ cookiecutter.package_name }}/test.sh @@ -5,7 +5,10 @@ SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" ./build.sh VOLUME_SUFFIX=$(dd if=/dev/urandom bs=32 count=1 | md5sum | cut --delimiter=' ' --fields=1) -MEM_LIMIT="4g" # Maximum is currently 30g, configurable in your algorithm image settings on grand challenge +{% if cookiecutter.template_kind == "Algorithm" -%} +# Maximum is currently 30g, configurable in your algorithm image settings on grand challenge +{%- endif %} +MEM_LIMIT="4g" docker volume create {{ cookiecutter.package_name|lower }}-output-$VOLUME_SUFFIX @@ -22,6 +25,13 @@ docker run --rm \ -v {{ cookiecutter.package_name|lower }}-output-$VOLUME_SUFFIX:/output/ \ {{ cookiecutter.package_name|lower }} +{% if cookiecutter.template_kind == "Evaluation" -%} +docker run --rm \ + -v {{ cookiecutter.package_name|lower }}-output-$VOLUME_SUFFIX:/output/ \ + python:{{ cookiecutter.python_major_version }}.{{ cookiecutter.python_minor_version }}-slim cat /output/metrics.json | python -m json.tool +{%- endif %} + +{% if cookiecutter.template_kind == "Algorithm" -%} docker run --rm \ -v {{ cookiecutter.package_name|lower }}-output-$VOLUME_SUFFIX:/output/ \ python:{{ cookiecutter.python_major_version }}.{{ cookiecutter.python_minor_version }}-slim cat /output/results.json | python -m json.tool @@ -36,5 +46,6 @@ if [ $? -eq 0 ]; then else echo "Expected output was not found..." fi +{%- endif %} docker volume rm {{ cookiecutter.package_name|lower }}-output-$VOLUME_SUFFIX diff --git a/evalutils/templates/evaluation/__init__.py b/evalutils/templates/evaluation/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/evalutils/templates/evaluation/cookiecutter.json b/evalutils/templates/evaluation/cookiecutter.json deleted file mode 100644 index cb9125b..0000000 --- a/evalutils/templates/evaluation/cookiecutter.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "challenge_name": "", - "challenge_kind": [ - "Classification", - "Segmentation", - "Detection" - ], - "package_name": "{{ cookiecutter.challenge_name|replace(' ', '') }}", - "evalutils_version": "", - "dev_build": 0, - "python_major_version": "", - "python_minor_version": "", - "requirements": {} -} diff --git a/evalutils/templates/evaluation/hooks/__init__.py b/evalutils/templates/evaluation/hooks/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/evalutils/templates/evaluation/hooks/post_gen_project.py b/evalutils/templates/evaluation/hooks/post_gen_project.py deleted file mode 100644 index 0ce5346..0000000 --- a/evalutils/templates/evaluation/hooks/post_gen_project.py +++ /dev/null @@ -1,54 +0,0 @@ -import os -import shutil -from pathlib import Path - -from evalutils.utils import ( - convert_line_endings, - generate_requirements_txt, - generate_source_wheel, -) - -CHALLENGE_KIND = "{{ cookiecutter.challenge_kind }}" -CHALLENGE_NAME = "{{ cookiecutter.challenge_name }}" -IS_DEV_BUILD = int("{{ cookiecutter.dev_build }}") == 1 - -template_dir = Path(os.getcwd()) - -templated_files = template_dir.glob("*.j2") -for f in templated_files: - shutil.move(f.name, f.stem) - - -def remove_classification_files(): - os.remove(Path("ground-truth") / "reference.csv") - os.remove(Path("test") / "submission.csv") - - -def remove_segmentation_files(): - files = [] - for ext in ["mhd", "zraw"]: - files.extend(Path(".").glob(f"**/*.{ext}")) - - for file in files: - os.remove(str(file)) - - -def remove_detection_files(): - os.remove(Path("ground-truth") / "detection-reference.csv") - os.remove(Path("test") / "detection-submission.csv") - - -if CHALLENGE_KIND.lower() != "segmentation": - remove_segmentation_files() - -if CHALLENGE_KIND.lower() != "detection": - remove_detection_files() - -if CHALLENGE_KIND.lower() != "classification": - remove_classification_files() - -if IS_DEV_BUILD: - generate_source_wheel(template_dir / "vendor") - -generate_requirements_txt() -convert_line_endings() diff --git a/evalutils/templates/evaluation/hooks/pre_gen_project.py b/evalutils/templates/evaluation/hooks/pre_gen_project.py deleted file mode 100644 index 264caf4..0000000 --- a/evalutils/templates/evaluation/hooks/pre_gen_project.py +++ /dev/null @@ -1,11 +0,0 @@ -import re - -FORBIDDEN_NAMES = ["evalutils", "pandas", "Evaluation", "Algorithm"] - -MODULE_REGEX = r"^[_a-zA-Z][_a-zA-Z0-9]+$" - -package_name = "{{ cookiecutter.package_name }}" - -if not re.match(MODULE_REGEX, package_name) or package_name in FORBIDDEN_NAMES: - print(f"ERROR: {package_name!r} is not a valid Python module name!") - exit(1) diff --git a/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/.dockerignore b/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/.dockerignore deleted file mode 100644 index 1068f81..0000000 --- a/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -test/ -.git/ -*.tar.gz diff --git a/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/.github/workflows/ci.yml b/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/.github/workflows/ci.yml deleted file mode 100644 index c18166c..0000000 --- a/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/.github/workflows/ci.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: CI - -on: [push, pull_request] - -env: - PYTHON_VERSION: '{{ cookiecutter.python_major_version }}.{{ cookiecutter.python_minor_version }}' - -jobs: - - evaluation-tests: - runs-on: ubuntu-latest - steps: - - name: Install Python ${{ "{{" }} env.PYTHON_VERSION {{ "}}" }} - uses: actions/setup-python@v4 - with: - python-version: ${{ "{{" }} env.PYTHON_VERSION {{ "}}" }} - - uses: actions/checkout@v3 - - name: Build the containers - run: | - ./build.sh - - name: Run the tests - run: | - ./test.sh diff --git a/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/.gitignore b/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/.gitignore deleted file mode 100644 index 13f175c..0000000 --- a/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/.gitignore +++ /dev/null @@ -1,105 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -env/ -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -*.egg-info/ -.installed.cfg -*.egg - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -.hypothesis/ -.pytest_cache/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# pyenv -.python-version - -# celery beat schedule file -celerybeat-schedule - -# SageMath parsed files -*.sage.py - -# dotenv -.env - -# virtualenv -.venv -venv/ -ENV/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ - -# Pycharm -.idea/ diff --git a/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/Dockerfile b/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/Dockerfile deleted file mode 100644 index cc63534..0000000 --- a/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -FROM python:{{ cookiecutter.python_major_version }}.{{ cookiecutter.python_minor_version }}-slim - -RUN groupadd -r evaluator && useradd -m --no-log-init -r -g evaluator evaluator - -RUN mkdir -p /opt/evaluation /input /output \ - && chown evaluator:evaluator /opt/evaluation /input /output - -USER evaluator -WORKDIR /opt/evaluation - -ENV PATH="/home/evaluator/.local/bin:${PATH}" - -RUN python -m pip install --user -U pip && python -m pip install --user pip-tools - -{% if cookiecutter.dev_build|int -%} -COPY --chown=evaluator:evaluator vendor /opt/evaluation/vendor -{%- endif %} - -COPY --chown=evaluator:evaluator requirements.txt /opt/evaluation/ -RUN python -m piptools sync requirements.txt - -COPY --chown=evaluator:evaluator ground-truth /opt/evaluation/ground-truth - -COPY --chown=evaluator:evaluator evaluation.py /opt/evaluation/ - -ENTRYPOINT [ "python", "-m", "evaluation" ] diff --git a/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/README.md b/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/README.md deleted file mode 100644 index 953a4e3..0000000 --- a/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# {{ cookiecutter.challenge_name }} Evaluation - -The source code for the evaluation container for -{{ cookiecutter.challenge_name }}, generated with -evalutils version {{ cookiecutter.evalutils_version }} -using Python {{ cookiecutter.python_major_version }}.{{ cookiecutter.python_minor_version }}. diff --git a/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/build.sh b/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/build.sh deleted file mode 100755 index ac80049..0000000 --- a/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/build.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash -SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" - -docker build -t {{ cookiecutter.package_name|lower }} "$SCRIPTPATH" diff --git a/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/export.sh b/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/export.sh deleted file mode 100755 index aa1d9a5..0000000 --- a/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/export.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -./build.sh - -docker save {{ cookiecutter.package_name|lower }} | gzip -c > {{ cookiecutter.package_name }}.tar.gz diff --git a/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/requirements.in b/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/requirements.in deleted file mode 100644 index 9d9b709..0000000 --- a/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/requirements.in +++ /dev/null @@ -1,3 +0,0 @@ -{%- for package, version in cookiecutter.requirements.items() %} -{{ package }}{{ version }} -{%- endfor %} diff --git a/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/test.sh b/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/test.sh deleted file mode 100755 index 72c111a..0000000 --- a/evalutils/templates/evaluation/{{ cookiecutter.package_name }}/test.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash - -SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" - -./build.sh - -VOLUME_SUFFIX=$(dd if=/dev/urandom bs=32 count=1 | md5sum | cut --delimiter=' ' --fields=1) - -docker volume create {{ cookiecutter.package_name|lower }}-output-$VOLUME_SUFFIX - -# Do not change any of the parameters to docker run, these are fixed -docker run --rm \ - --memory="4g" \ - --memory-swap="4g" \ - --network="none" \ - --cap-drop="ALL" \ - --security-opt="no-new-privileges" \ - --shm-size="128m" \ - --pids-limit="256" \ - -v $SCRIPTPATH/test/:/input/ \ - -v {{ cookiecutter.package_name|lower }}-output-$VOLUME_SUFFIX:/output/ \ - {{ cookiecutter.package_name|lower }} - -docker run --rm \ - -v {{ cookiecutter.package_name|lower }}-output-$VOLUME_SUFFIX:/output/ \ - python:{{ cookiecutter.python_major_version }}.{{ cookiecutter.python_minor_version }}-slim cat /output/metrics.json | python -m json.tool - -docker volume rm {{ cookiecutter.package_name|lower }}-output-$VOLUME_SUFFIX diff --git a/tests/test_algorithm.py b/tests/test_algorithm.py index 923e6a1..ca5ec40 100644 --- a/tests/test_algorithm.py +++ b/tests/test_algorithm.py @@ -19,9 +19,9 @@ Path(__file__).parent.parent / "evalutils" / "templates" - / "algorithm" + / "container" / "{{ cookiecutter.package_name }}" - / "test" + / "algorithm_test" ) diff --git a/tests/test_templates.py b/tests/test_templates.py index dc4631e..17a045d 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -133,9 +133,9 @@ def test_algorithm_cli(tmpdir, kind): Path(__file__).parent.parent / "evalutils" / "templates" - / "algorithm" + / "container" / "{{ cookiecutter.package_name }}" - / "test" + / "algorithm_test" / f"results_{kind.lower()}.json" ) as f: expected = json.load(f)