From c4553247d102dfb2896842bdf9c566357598dd79 Mon Sep 17 00:00:00 2001 From: ollie-bell <56110893+ollie-bell@users.noreply.github.com> Date: Sat, 10 Aug 2024 20:55:30 +0100 Subject: [PATCH 1/7] fix: prevent unwanted subprocess termination --- Snakefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Snakefile b/Snakefile index 9610490a5..ff61762f2 100644 --- a/Snakefile +++ b/Snakefile @@ -346,7 +346,9 @@ def terminate_if_cutout_exists(config=config): if config["enable"].get("build_cutout", False): - terminate_if_cutout_exists(config) + cmd = " ".join(sys.argv) + if "--mode 1" not in cmd or "build_cutout" in cmd: + terminate_if_cutout_exists(config) rule build_cutout: params: From d6e2cf69365202f9411f3547936ae62912d84dfe Mon Sep 17 00:00:00 2001 From: ollie-bell <56110893+ollie-bell@users.noreply.github.com> Date: Sat, 10 Aug 2024 21:01:34 +0100 Subject: [PATCH 2/7] Update release_notes.rst --- doc/release_notes.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 3abf89322..0bd4821e5 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -19,6 +19,7 @@ E.g. if a new rule becomes available describe how to use it `snakemake -j1 run_t **Minor Changes and bug-fixing** * Remove unused `countries_codes` argument from `load_GDP` function in `build_shapes.py` script, which was not being called as intended with positional arguments `PR #1069 `__ +* Fix unwanted termination due to existing cutout during snakemake subprocess calls `PR #1079 `__ PyPSA-Earth 0.4.0 From a5e7d6037f12b738abcea55a94114589716450b9 Mon Sep 17 00:00:00 2001 From: ollie-bell <56110893+ollie-bell@users.noreply.github.com> Date: Sat, 10 Aug 2024 22:34:51 +0100 Subject: [PATCH 3/7] refactor: more robust subprocess detection --- Snakefile | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/Snakefile b/Snakefile index ff61762f2..26406c2c6 100644 --- a/Snakefile +++ b/Snakefile @@ -331,6 +331,17 @@ def terminate_if_cutout_exists(config=config): Check if any of the requested cutout files exist. If that's the case, terminate execution to avoid data loss. """ + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("--mode", type=int, default=snakemake.Mode.default) + parser.add_argument("--target-jobs", type=str, nargs="+", default=[]) + args, _ = parser.parse_known_args(sys.argv) + if args.mode == snakemake.Mode.subprocess and "build_cutout" not in " ".join( + args.target_jobs + ): + return + config_cutouts = [ d_value["cutout"] for tc, d_value in config["renewable"].items() ] + list(config["atlite"]["cutouts"].keys()) @@ -346,9 +357,7 @@ def terminate_if_cutout_exists(config=config): if config["enable"].get("build_cutout", False): - cmd = " ".join(sys.argv) - if "--mode 1" not in cmd or "build_cutout" in cmd: - terminate_if_cutout_exists(config) + terminate_if_cutout_exists(config) rule build_cutout: params: From ba132e8f2bcddc8e5abf688761dda2ffce033926 Mon Sep 17 00:00:00 2001 From: ollie-bell <56110893+ollie-bell@users.noreply.github.com> Date: Sun, 29 Sep 2024 15:42:25 +0100 Subject: [PATCH 4/7] test: snakemake subprocess termination fix --- test/test_cutout_termination | 45 ++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 test/test_cutout_termination diff --git a/test/test_cutout_termination b/test/test_cutout_termination new file mode 100644 index 000000000..47539ce5e --- /dev/null +++ b/test/test_cutout_termination @@ -0,0 +1,45 @@ +import os +import sys + + +def terminate_if_cutout_exists(subprocess_check=False): + """ + Check if any of the requested cutout files exist. + If that's the case, terminate execution to avoid data loss. + """ + if subprocess_check: + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("--mode", type=int, default=snakemake.Mode.default) + parser.add_argument("--target-jobs", type=str, nargs="+", default=[]) + args, _ = parser.parse_known_args(sys.argv) + if args.mode == snakemake.Mode.subprocess and "build_cutout" not in " ".join( + args.target_jobs + ): + return + + cutout_fl = "cutout.txt" + if os.path.exists(cutout_fl): + os.remove(cutout_fl) + raise Exception( + f"A cutout file {cutout_fl} exists and risks to be overwritten. " + "If this is an intended behavior, please move or delete this file and re-run the rule." + ) + + +terminate_if_cutout_exists(subprocess_check=config.get("subprocess_check", False)) + + +rule remove_cutout: + input: + "cutout.txt", + run: + os.remove(input[0]) + + +rule build_cutout: + output: + "cutout.txt", + shell: + "echo build cutout > {output}" From ba4bb0cb8fd23fc0fd7a2dce68505594c3815bb7 Mon Sep 17 00:00:00 2001 From: ollie-bell <56110893+ollie-bell@users.noreply.github.com> Date: Sun, 29 Sep 2024 19:26:11 +0100 Subject: [PATCH 5/7] test: snakemake subprocess termination fix - update snakefile --- test/test_cutout_termination | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/test/test_cutout_termination b/test/test_cutout_termination index 47539ce5e..58dbe1190 100644 --- a/test/test_cutout_termination +++ b/test/test_cutout_termination @@ -1,6 +1,8 @@ import os import sys +CUTOUT = "cutout.nc" + def terminate_if_cutout_exists(subprocess_check=False): """ @@ -19,11 +21,10 @@ def terminate_if_cutout_exists(subprocess_check=False): ): return - cutout_fl = "cutout.txt" - if os.path.exists(cutout_fl): - os.remove(cutout_fl) + if os.path.exists(CUTOUT): + os.remove(CUTOUT) raise Exception( - f"A cutout file {cutout_fl} exists and risks to be overwritten. " + f"A cutout file {CUTOUT} exists and risks to be overwritten. " "If this is an intended behavior, please move or delete this file and re-run the rule." ) @@ -33,13 +34,13 @@ terminate_if_cutout_exists(subprocess_check=config.get("subprocess_check", False rule remove_cutout: input: - "cutout.txt", + cutout=CUTOUT, run: - os.remove(input[0]) + os.remove(input.cutout) rule build_cutout: output: - "cutout.txt", + cutout=CUTOUT, shell: - "echo build cutout > {output}" + "echo build_cutout > {output.cutout}" From 2b451e4353a53e7090fd6a65833d464d3832a703 Mon Sep 17 00:00:00 2001 From: ollie-bell <56110893+ollie-bell@users.noreply.github.com> Date: Sun, 29 Sep 2024 19:26:38 +0100 Subject: [PATCH 6/7] chore: add license info for pre-commit --- test/test_cutout_termination | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/test_cutout_termination b/test/test_cutout_termination index 58dbe1190..edced08ef 100644 --- a/test/test_cutout_termination +++ b/test/test_cutout_termination @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors +# +# SPDX-License-Identifier: AGPL-3.0-or-later + import os import sys From 909c22084b88e9ff7273c79270ddbff0e3af06d4 Mon Sep 17 00:00:00 2001 From: ollie-bell <56110893+ollie-bell@users.noreply.github.com> Date: Mon, 30 Sep 2024 10:15:23 +0100 Subject: [PATCH 7/7] Rename test_cutout_termination to test_cutout_termination.smk --- test/{test_cutout_termination => test_cutout_termination.smk} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/{test_cutout_termination => test_cutout_termination.smk} (100%) diff --git a/test/test_cutout_termination b/test/test_cutout_termination.smk similarity index 100% rename from test/test_cutout_termination rename to test/test_cutout_termination.smk