Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: automatic conda environment handling #1469

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
+115 −120
Open

feat: automatic conda environment handling #1469

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a1b9b1f
feat: automatic conda environment handling
lkstrp Dec 20, 2024
ac08dc4
remove `retrieve.yaml`
lkstrp Dec 20, 2024
0865849
fix: typo
lkstrp Dec 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/test.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ jobs:

- name: Run snakemake test workflows
run: |
make test
make test envs=pinned # TODO: Remove custom env handling

- name: Run unit tests
run: |
Expand Down
20 changes: 10 additions & 10 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,23 +54,23 @@ install-pinned-macos: _conda_check
# Run default tests
test:
set -e
snakemake solve_elec_networks --configfile config/test/config.electricity.yaml --rerun-triggers=mtime
snakemake --configfile config/test/config.overnight.yaml --rerun-triggers=mtime
snakemake --configfile config/test/config.myopic.yaml --rerun-triggers=mtime
snakemake make_summary_perfect --configfile config/test/config.perfect.yaml --rerun-triggers=mtime
snakemake --configfile config/test/config.scenarios.yaml --rerun-triggers=mtime -n
snakemake solve_elec_networks --configfile config/test/config.electricity.yaml --rerun-triggers=mtime $(if $(filter pinned,$(envs)),--sdm conda)
snakemake --configfile config/test/config.overnight.yaml --rerun-triggers=mtime $(if $(filter pinned,$(envs)),--sdm conda)
snakemake --configfile config/test/config.myopic.yaml --rerun-triggers=mtime $(if $(filter pinned,$(envs)),--sdm conda)
snakemake make_summary_perfect --configfile config/test/config.perfect.yaml --rerun-triggers=mtime $(if $(filter pinned,$(envs)),--sdm conda)
snakemake --configfile config/test/config.scenarios.yaml --rerun-triggers=mtime -n $(if $(filter pinned,$(envs)),--sdm conda)
echo "All tests completed successfully."

unit-test:
pytest test

# Cleans all output files from tests
clean-tests:
snakemake solve_elec_networks --configfile config/test/config.electricity.yaml --rerun-triggers=mtime --delete-all-output
snakemake --configfile config/test/config.overnight.yaml --rerun-triggers=mtime --delete-all-output
snakemake --configfile config/test/config.myopic.yaml --rerun-triggers=mtime --delete-all-output
snakemake make_summary_perfect --configfile config/test/config.perfect.yaml --rerun-triggers=mtime --delete-all-output
snakemake --configfile config/test/config.scenarios.yaml --rerun-triggers=mtime -n --delete-all-output
snakemake solve_elec_networks --configfile config/test/config.electricity.yaml --rerun-triggers=mtime --delete-all-output $(if $(filter pinned,$(envs)),--sdm conda)
snakemake --configfile config/test/config.overnight.yaml --rerun-triggers=mtime --delete-all-output $(if $(filter pinned,$(envs)),--sdm conda)
snakemake --configfile config/test/config.myopic.yaml --rerun-triggers=mtime --delete-all-output $(if $(filter pinned,$(envs)),--sdm conda)
snakemake make_summary_perfect --configfile config/test/config.perfect.yaml --rerun-triggers=mtime --delete-all-output $(if $(filter pinned,$(envs)),--sdm conda)
snakemake --configfile config/test/config.scenarios.yaml --rerun-triggers=mtime -n --delete-all-output $(if $(filter pinned,$(envs)),--sdm conda)

# Removes all created files except for large cutout files (similar to fresh clone)
reset:
Expand Down
4 changes: 2 additions & 2 deletions Snakefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ rule create_scenarios:
output:
config["run"]["scenarios"]["file"],
conda:
"envs/retrieve.yaml"
conda_env_provider()
script:
"config/create_scenarios.py"

Expand Down Expand Up @@ -110,7 +110,7 @@ rule dag:
pdf=resources("dag.pdf"),
png=resources("dag.png"),
conda:
"envs/environment.yaml"
conda_env_provider()
shell:
r"""
snakemake --rulegraph all | sed -n "/digraph/,\$p" > {output.dot}
Expand Down
18 changes: 0 additions & 18 deletions envs/retrieve.yaml
View file
Open in desktop

This file was deleted.

46 changes: 23 additions & 23 deletions rules/build_electricity.smk
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ rule build_electricity_demand:
resources:
mem_mb=5000,
conda:
"../envs/environment.yaml"
conda_env_provider("..")
script:
"../scripts/build_electricity_demand.py"

Expand All @@ -47,7 +47,7 @@ rule build_powerplants:
resources:
mem_mb=7000,
conda:
"../envs/environment.yaml"
conda_env_provider("..")
script:
"../scripts/build_powerplants.py"

Expand Down Expand Up @@ -93,7 +93,7 @@ rule base_network:
resources:
mem_mb=1500,
conda:
"../envs/environment.yaml"
conda_env_provider("..")
script:
"../scripts/base_network.py"

Expand All @@ -120,7 +120,7 @@ rule build_shapes:
resources:
mem_mb=1500,
conda:
"../envs/environment.yaml"
conda_env_provider("..")
script:
"../scripts/build_shapes.py"

Expand All @@ -144,7 +144,7 @@ if config["enable"].get("build_cutout", False):
resources:
mem_mb=config["atlite"].get("nprocesses", 4) * 1000,
conda:
"../envs/environment.yaml"
conda_env_provider("..")
script:
"../scripts/build_cutout.py"

Expand All @@ -165,7 +165,7 @@ rule build_ship_raster:
benchmark:
benchmarks("build_ship_raster")
conda:
"../envs/environment.yaml"
conda_env_provider("..")
script:
"../scripts/build_ship_raster.py"

Expand Down Expand Up @@ -208,7 +208,7 @@ rule determine_availability_matrix_MD_UA:
resources:
mem_mb=config["atlite"].get("nprocesses", 4) * 5000,
conda:
"../envs/environment.yaml"
conda_env_provider("..")
script:
"../scripts/determine_availability_matrix_MD_UA.py"

Expand Down Expand Up @@ -277,7 +277,7 @@ rule determine_availability_matrix:
resources:
mem_mb=config["atlite"].get("nprocesses", 4) * 5000,
conda:
"../envs/environment.yaml"
conda_env_provider("..")
script:
"../scripts/determine_availability_matrix.py"

Expand Down Expand Up @@ -307,7 +307,7 @@ rule build_renewable_profiles:
wildcard_constraints:
technology="(?!hydro).*", # Any technology other than hydro
conda:
"../envs/environment.yaml"
conda_env_provider("..")
script:
"../scripts/build_renewable_profiles.py"

Expand All @@ -325,7 +325,7 @@ rule build_monthly_prices:
resources:
mem_mb=5000,
conda:
"../envs/environment.yaml"
conda_env_provider("..")
script:
"../scripts/build_monthly_prices.py"

Expand All @@ -352,7 +352,7 @@ rule build_hydro_profile:
resources:
mem_mb=5000,
conda:
"../envs/environment.yaml"
conda_env_provider("..")
script:
"../scripts/build_hydro_profile.py"

Expand All @@ -377,7 +377,7 @@ rule build_line_rating:
resources:
mem_mb=config["atlite"].get("nprocesses", 4) * 1000,
conda:
"../envs/environment.yaml"
conda_env_provider("..")
script:
"../scripts/build_line_rating.py"

Expand Down Expand Up @@ -412,7 +412,7 @@ rule build_transmission_projects:
mem_mb=4000,
threads: 1
conda:
"../envs/environment.yaml"
conda_env_provider("..")
script:
"../scripts/build_transmission_projects.py"

Expand Down Expand Up @@ -450,7 +450,7 @@ rule add_transmission_projects_and_dlr:
resources:
mem_mb=4000,
conda:
"../envs/environment.yaml"
conda_env_provider("..")
script:
"../scripts/add_transmission_projects_and_dlr.py"

Expand Down Expand Up @@ -498,7 +498,7 @@ rule build_gdp_pop_non_nuts3:
resources:
mem_mb=8000,
conda:
"../envs/environment.yaml"
conda_env_provider("..")
script:
"../scripts/build_gdp_pop_non_nuts3.py"

Expand All @@ -521,7 +521,7 @@ rule build_electricity_demand_base:
resources:
mem_mb=5000,
conda:
"../envs/environment.yaml"
conda_env_provider("..")
script:
"../scripts/build_electricity_demand_base.py"

Expand All @@ -547,7 +547,7 @@ rule build_hac_features:
resources:
mem_mb=10000,
conda:
"../envs/environment.yaml"
conda_env_provider("..")
script:
"../scripts/build_hac_features.py"

Expand Down Expand Up @@ -577,7 +577,7 @@ rule simplify_network:
resources:
mem_mb=12000,
conda:
"../envs/environment.yaml"
conda_env_provider("..")
script:
"../scripts/simplify_network.py"

Expand Down Expand Up @@ -632,7 +632,7 @@ rule cluster_network:
resources:
mem_mb=10000,
conda:
"../envs/environment.yaml"
conda_env_provider("..")
script:
"../scripts/cluster_network.py"

Expand Down Expand Up @@ -704,7 +704,7 @@ rule add_electricity:
resources:
mem_mb=10000,
conda:
"../envs/environment.yaml"
conda_env_provider("..")
script:
"../scripts/add_electricity.py"

Expand Down Expand Up @@ -740,7 +740,7 @@ rule prepare_network:
resources:
mem_mb=4000,
conda:
"../envs/environment.yaml"
conda_env_provider("..")
script:
"../scripts/prepare_network.py"

Expand Down Expand Up @@ -785,7 +785,7 @@ if config["electricity"]["base_network"] == "osm-raw":
resources:
mem_mb=4000,
conda:
"../envs/environment.yaml"
conda_env_provider("..")
script:
"../scripts/clean_osm_data.py"

Expand Down Expand Up @@ -825,6 +825,6 @@ if config["electricity"]["base_network"] == "osm-raw":
resources:
mem_mb=4000,
conda:
"../envs/environment.yaml"
conda_env_provider("..")
script:
"../scripts/build_osm_network.py"
Loading
Loading