From 9b68d2b06fbd247fc10a8c00429d31e61b88b271 Mon Sep 17 00:00:00 2001 From: Shettland Date: Fri, 1 Mar 2024 12:23:53 +0100 Subject: [PATCH 1/6] Updated iskylims api url to v2 --- bu_isciii/conf/configuration.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bu_isciii/conf/configuration.json b/bu_isciii/conf/configuration.json index a1316823..7a39ff99 100755 --- a/bu_isciii/conf/configuration.json +++ b/bu_isciii/conf/configuration.json @@ -21,10 +21,10 @@ }, "xtutatis_api_settings": { "api_url": "/drylab/api/", - "server": "http://iskylims.isciiides.es" + "server": "https://iskylims.isciii.es" }, "api_settings": { - "server": "http://iskylims.isciiides.es", + "server": "https://iskylims.isciii.es", "api_url": "/drylab/api/" }, "bioinfo_doc": { From 3e45c61a84cd683acd37a94f56860ed54a57d12b Mon Sep 17 00:00:00 2001 From: Shettland Date: Fri, 1 Mar 2024 12:24:19 +0100 Subject: [PATCH 2/6] Included IRMA template --- bu_isciii/templates/services.json | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/bu_isciii/templates/services.json b/bu_isciii/templates/services.json index 59a451a7..a2007838 100755 --- a/bu_isciii/templates/services.json +++ b/bu_isciii/templates/services.json @@ -304,5 +304,22 @@ "last_folder":"RESULTS", "delivery_md": "", "results_md": "" - } + }, + "IRMA": { + "label": "", + "template": "IRMA", + "url": "", + "order": 1, + "begin": "", + "end": "", + "description": "", + "clean": { + "folders":["02-preprocessing"], + "files":[] + }, + "no_copy": ["RAW", "TMP"], + "last_folder":"REFERENCES", + "delivery_md": "", + "results_md": "" + } } From ac12b979b3e24e4162c220ef28e8af10c831640f Mon Sep 17 00:00:00 2001 From: Shettland Date: Fri, 1 Mar 2024 12:25:10 +0100 Subject: [PATCH 3/6] Fixed bug when executing rsync through srun --- bu_isciii/scratch.py | 44 ++++++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/bu_isciii/scratch.py b/bu_isciii/scratch.py index a43b228d..68ebd3c9 100755 --- a/bu_isciii/scratch.py +++ b/bu_isciii/scratch.py @@ -71,9 +71,6 @@ def __init__( "resolution_full_number" ] self.scratch_tmp_path = os.path.join(self.tmp_dir, self.service_folder) - self.out_file = os.path.join( - self.tmp_dir, self.scratch_tmp_path, "DOC", "service_info.txt" - ) # params like --chdir, --partition and --time srun_params = self.conf["srun_settings"].items() self.srun_settings = [arg for param in srun_params for arg in param] @@ -98,12 +95,15 @@ def __init__( ) self.full_path = os.path.join(self.path, self.service_folder) + self.out_file = os.path.join( + self.full_path, "DOC", "service_info.txt" + ) def srun_command(self, srun_settings, command): command_list = [["srun"], srun_settings, command] srun_command = [arg for command in command_list for arg in command] - subprocess.run(srun_command, check=True, shell=True) - return + exit_code = subprocess.call(srun_command) + return exit_code def copy_scratch(self): stderr.print("[blue]I will copy the service from %s" % self.full_path) @@ -114,31 +114,35 @@ def copy_scratch(self): if protocol == "rsync": rsync_command = sysrsync.get_rsync_command( source=self.full_path, - destination=self.conf["scratch_path"], + destination=self.conf["scratch_path"]+"/", options=self.conf["options"], exclusions=self.conf["exclusions"], sync_source_contents=False, ) - self.srun_command(self.srun_settings, rsync_command) + exit_code = self.srun_command(self.srun_settings, rsync_command) + + else: + stderr.print( + "[ref] This protocol is not allowed at the moment", + highlight=False, + ) + sys.exit() + if exit_code == 0: stderr.print( "[green] Data copied to the sftp folder successfully", highlight=False, ) - else: + f = open(self.out_file, "w") + f.write("Temporal directory: " + self.scratch_tmp_path + "\n") + f.write("Origin service directory: " + self.full_path + "\n") + f.close() stderr.print( - "[ref] This protocol is not allowed at the moment", + "[green]Successfully copied the directory to %s" + % self.scratch_tmp_path, highlight=False, ) - sys.exit() - f = open(self.out_file, "w") - f.write("Temporal directory: " + self.scratch_tmp_path + "\n") - f.write("Origin service directory: " + self.full_path + "\n") - f.close() - stderr.print( - "[green]Successfully copied the directory to %s" - % self.scratch_tmp_path, - highlight=False, - ) + else: + stderr.print(f"[red]Errors while copying {self.full_path}") except Exception as e: stderr.print(e) stderr.print( @@ -173,7 +177,7 @@ def revert_copy_scratch(self): if self.conf["protocol"] == "rsync": # scratch_tmp cannot be used due to permission issues scratch_bi_path = "".join( - self.conf["scratch_path"], self.service_folder + [self.conf["scratch_path"], self.service_folder] ) rsync_command = sysrsync.get_rsync_command( source=scratch_bi_path, From d71ffbea1a59025b5fadaabb858f3229c732be06 Mon Sep 17 00:00:00 2001 From: Shettland Date: Fri, 1 Mar 2024 12:25:30 +0100 Subject: [PATCH 4/6] Prepared CHANGELOG for new release 2.0.0 --- CHANGELOG.md | 42 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e50d8bbc..393eb16c 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,47 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [2.0.0] - 2024-02-28 : https://github.com/BU-ISCIII/buisciii-tools/releases/tag/2.0.0 +## [2.1.0dev] - 2024-0X-0X : https://github.com/BU-ISCIII/buisciii-tools/releases/tag/2.1.X + +### Credits + +Code contributions to the hotfix: + +### Template fixes and updates + +### Modules + +#### Added enhancements + +#### Fixes + +#### Changed + +#### Removed + +### Requirements + +## [2.X.1hot] - 2024-0X-0X : https://github.com/BU-ISCIII/buisciii-tools/releases/tag/2.X.1 + +### Credits + +Code contributions to the hotfix: + +### Template fixes and updates + +### Modules + +#### Added enhancements + +#### Fixes + +#### Changed + +#### Removed + +### Requirements + +## [2.0.0] - 2024-03-01 : https://github.com/BU-ISCIII/buisciii-tools/releases/tag/2.0.0 ### Credits From 758ae7481790cdb5b0a85000bb05c467c81f955e Mon Sep 17 00:00:00 2001 From: Shettland Date: Fri, 1 Mar 2024 12:27:47 +0100 Subject: [PATCH 5/6] Fixed bug when executing rsync through srun --- bu_isciii/scratch.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/bu_isciii/scratch.py b/bu_isciii/scratch.py index 68ebd3c9..d31ca0cd 100755 --- a/bu_isciii/scratch.py +++ b/bu_isciii/scratch.py @@ -95,9 +95,7 @@ def __init__( ) self.full_path = os.path.join(self.path, self.service_folder) - self.out_file = os.path.join( - self.full_path, "DOC", "service_info.txt" - ) + self.out_file = os.path.join(self.full_path, "DOC", "service_info.txt") def srun_command(self, srun_settings, command): command_list = [["srun"], srun_settings, command] @@ -114,7 +112,7 @@ def copy_scratch(self): if protocol == "rsync": rsync_command = sysrsync.get_rsync_command( source=self.full_path, - destination=self.conf["scratch_path"]+"/", + destination=self.conf["scratch_path"] + "/", options=self.conf["options"], exclusions=self.conf["exclusions"], sync_source_contents=False, From 92079353b130520e007c3c211d5815727af353d1 Mon Sep 17 00:00:00 2001 From: Shettland Date: Fri, 1 Mar 2024 12:39:57 +0100 Subject: [PATCH 6/6] Updated CHANGELOG.md with latest changes --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 393eb16c..d706f7ca 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,6 +71,8 @@ Code contributions to the release: - Added finish module - Added json files: sftp_user.json - Added delivery jinja templates +- Added IRMA template to services.json +- Scratch module now executes rsync using SLURM's srun #### Fixes