Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fixed iskylims isciiides #162

Merged
merged 10 commits into from
Sep 27, 2023
Merged

Fixed iskylims isciiides #162

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
c61dbb4
Fixed some variables strings
svarona Sep 27, 2023
1e1a9c0
Fixed for new rsync command format
svarona Sep 27, 2023
450ddb2
Fixed variables to fit in DB
svarona Sep 27, 2023
2b70e67
Clean some functions and code
svarona Sep 27, 2023
0e1ae7c
Fixed database variables
svarona Sep 27, 2023
c0e0fb7
Cleaning and fixes
svarona Sep 27, 2023
f40e0e3
Fixed new rsync command
svarona Sep 27, 2023
c8b8fc9
Removed duplicated entry
svarona Sep 27, 2023
6aaa008
Fixed black
svarona Sep 27, 2023
436868e
Resolved suggestions
svarona Sep 27, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 9 additions & 30 deletions bu_isciii/clean.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,17 +31,6 @@ def __init__(
option=None,
api_token=None,
):
"""
Description:
Class to perform the cleaning.

Usage:

Attributes:

Methods:

"""
# access the api with the resolution name to obtain the data
# ask away if no input given
if resolution_id is None:
Expand All @@ -58,13 +47,13 @@ def __init__(
conf_api["server"], conf_api["api_url"], api_token
)
self.resolution_info = rest_api.get_request(
request_info="serviceFullData", safe=False, resolution=self.resolution_id
request_info="service-data", safe=False, resolution=self.resolution_id
)
self.service_folder = self.resolution_info["resolutions"][0][
"resolutionFullNumber"
"resolution_full_number"
]
self.services_requested = self.resolution_info["resolutions"][0][
"availableServices"
"available_services"
]
self.service_samples = self.resolution_info["samples"]

Expand All @@ -86,7 +75,9 @@ def __init__(
)
sys.exit()
else:
self.path = bu_isciii.utils.get_service_paths(self.resolution_info)
self.path = bu_isciii.utils.get_service_paths(
"services_and_colaborations", self.resolution_info, "non_archived_path"
)

self.full_path = os.path.join(self.path, self.service_folder)

Expand Down Expand Up @@ -297,18 +288,6 @@ def rename(self, to_find, add, verbose=True):
print(f"Renamed {directory_to_rename} to {newpath}.")
return

def rename_nocopy(self, verbose=True):
"""
Description:

Usage:

Params:

"""
self.rename(to_find=self.nocopy, add="_NC", verbose=verbose)
return

def purge_files(self):
"""
Description:
Expand All @@ -323,7 +302,7 @@ def purge_files(self):
files_to_delete = []
for sample_info in self.service_samples:
for file in self.delete_files:
file_to_delete = file.replace("sample_name", sample_info["sampleName"])
file_to_delete = file.replace("sample_name", sample_info["sample_name"])
files_to_delete.append(file_to_delete)
path_content = self.scan_dirs(to_find=files_to_delete)
for file in path_content:
Expand Down Expand Up @@ -442,7 +421,7 @@ def full_clean(self):
"""

self.delete_rename()
self.rename_nocopy()
self.rename(to_find=self.nocopy, add="_NC", verbose=True)

def handle_clean(self):
"""
Expand All @@ -455,7 +434,7 @@ def handle_clean(self):
if self.option == "full_clean":
self.full_clean()
if self.option == "rename_nocopy":
self.rename_nocopy()
self.rename(to_find=self.nocopy, add="_NC", verbose=True)
if self.option == "clean":
self.delete_rename()
if self.option == "revert_renaming":
Expand Down
54 changes: 24 additions & 30 deletions bu_isciii/copy_sftp.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,18 +33,6 @@ def __init__(
sftp_folder=None,
api_token=None,
):
"""
Description:
Class to perform the copy of the service to sftp folfer.

Usage:

Attributes:

Methods:

"""

if resolution_id is None:
self.resolution_id = bu_isciii.utils.prompt_resolution_id()
else:
Expand All @@ -62,25 +50,22 @@ def __init__(
)

self.resolution_info = rest_api.get_request(
request_info="serviceFullData", safe=False, resolution=self.resolution_id
request_info="service-data", safe=False, resolution=self.resolution_id
)
if sftp_folder is None:
self.sftp_folder = bu_isciii.utils.get_sftp_folder(self.resolution_info)[0]
else:
self.sftp_folder = sftp_folder

self.service_folder = self.resolution_info["resolutions"][0][
"resolutionFullNumber"
"resolution_full_number"
]
self.services_requested = self.resolution_info["resolutions"][0][
"availableServices"
"available_services"
]
self.sftp_options = bu_isciii.config_json.ConfigJson().get_find(
"sftp_copy", "options"
)
self.sftp_exclusions = bu_isciii.config_json.ConfigJson().get_find(
"sftp_copy", "exclusions"
)
self.services_to_copy = bu_isciii.utils.get_service_ids(self.services_requested)

self.last_folders = self.get_last_folders(
Expand All @@ -102,7 +87,9 @@ def __init__(
)
sys.exit()
else:
self.path = bu_isciii.utils.get_service_paths(self.resolution_info)
self.path = bu_isciii.utils.get_service_paths(
"services_and_colaborations", self.resolution_info, "non_archived_path"
)

self.full_path = os.path.join(self.path, self.service_folder)

Expand Down Expand Up @@ -147,17 +134,24 @@ def copy_sftp(self):
)
self.sftp_options.append(log_command)
try:
sysrsync.run(
source=self.full_path,
destination=self.sftp_folder,
options=self.sftp_options,
exclusions=self.sftp_exclusions,
sync_source_contents=False,
)
stderr.print(
"[green] Data copied to the sftp folder successfully",
highlight=False,
)
if self.conf["protocol"] == "rsync":
sysrsync.run(
source=self.full_path,
destination=self.sftp_folder,
options=self.sftp_options,
exclusions=self.conf["exclusions"],
sync_source_contents=False,
)
stderr.print(
"[green] Data copied to the sftp folder successfully",
highlight=False,
)
else:
stderr.print(
"[ref] This protocol is not allowd for the moment",
svarona marked this conversation as resolved.
Show resolved Hide resolved
highlight=False,
)
sys.exit()
except RsyncError as e:
stderr.print(e)
stderr.print(
Expand Down
74 changes: 56 additions & 18 deletions bu_isciii/scratch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,10 @@
# Generic imports
import sys
import os
import subprocess
import logging
import re
import sysrsync
from sysrsync.exceptions import RsyncError

import rich
import shutil
Expand Down Expand Up @@ -54,22 +55,22 @@ def __init__(
self.direction = direction

# Load conf
conf_api = bu_isciii.config_json.ConfigJson().get_configuration("api_settings")
conf_api = bu_isciii.config_json.ConfigJson().get_configuration(
"xtutatis_api_settings"
)
# Obtain info from iskylims api
rest_api = bu_isciii.drylab_api.RestServiceApi(
conf_api["server"],
conf_api["api_url"],
api_token,
)
self.conf = bu_isciii.config_json.ConfigJson().get_configuration("scratch_copy")
self.rsync_command = self.conf["command"]

self.resolution_info = rest_api.get_request(
request_info="resolutionFullData", safe=False, resolution=self.resolution_id
request_info="service-data", safe=False, resolution=self.resolution_id
)

self.service_folder = self.resolution_info["resolutions"][
"resolutionFullNumber"
self.service_folder = self.resolution_info["resolutions"][0][
"resolution_full_number"
]
self.scratch_path = os.path.join(self.tmp_dir, self.service_folder)
self.out_file = os.path.join(
Expand Down Expand Up @@ -102,9 +103,26 @@ def copy_scratch(self):
stderr.print("[blue]I will copy the service from %s" % self.full_path)
stderr.print("[blue]to %s" % self.scratch_path)
if self.service_folder in self.full_path:
rsync_command = self.rsync_command + self.full_path + " " + self.tmp_dir
protocol = self.conf["protocol"]
try:
subprocess.run(rsync_command, shell=True, check=True)
if protocol == "rsync":
sysrsync.run(
source=self.full_path,
destination=self.tmp_dir,
options=self.conf["options"],
exclusions=self.conf["exclusions"],
sync_source_contents=False,
)
stderr.print(
"[green] Data copied to the sftp folder successfully",
highlight=False,
)
else:
stderr.print(
"[ref] This protocol is not allowd for the moment",
highlight=False,
)
sys.exit()
f = open(self.out_file, "w")
f.write("Temporal directory: " + self.scratch_path + "\n")
f.write("Origin service directory: " + self.full_path + "\n")
Expand All @@ -114,13 +132,12 @@ def copy_scratch(self):
% self.scratch_path,
highlight=False,
)
except subprocess.CalledProcessError as e:
except RsyncError as e:
stderr.print(e)
stderr.print(
"[red]ERROR: Copy of the directory %s failed" % self.full_path,
highlight=False,
)
log.exception("Unable to create pdf.", exc_info=e)
sys.exit()
else:
log.error(
f"Directory path not the same as service resolution. Skip folder copy '{self.full_path}'"
Expand All @@ -144,12 +161,33 @@ def revert_copy_scratch(self):
dest_dir = os.path.normpath("/".join(dest_folder.split("/")[:-1]))
stderr.print("[blue]to %s" % dest_folder)
if self.service_folder in dest_folder:
rsync_command = self.rsync_command + self.scratch_path + " " + dest_dir
subprocess.run(rsync_command, shell=True, check=True)
stderr.print(
"[green]Successfully copyed the directory to %s" % dest_folder,
highlight=False,
)
try:
if self.conf["protocol"] == "rsync":
sysrsync.run(
source=self.scratch_path,
destination=dest_dir,
options=self.conf["options"],
exclusions=self.conf["exclusions"],
sync_source_contents=False,
)
stderr.print(
"[green]Successfully copyed the directory to %s"
% dest_folder,
highlight=False,
)
else:
stderr.print(
"[ref] This protocol is not allowd for the moment",
svarona marked this conversation as resolved.
Show resolved Hide resolved
highlight=False,
)
sys.exit()
except RsyncError as e:
stderr.print(e)
stderr.print(
"[red]ERROR: Copy of the directory %s failed"
% self.scratch_path,
highlight=False,
)
else:
log.error(
f"Directory path not the same as service resolution. Skip folder copy '{dest_folder}'"
Expand Down
1 change: 0 additions & 1 deletion bu_isciii/templates/sftp_user.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
"imago": ["rnaseqrsv"],
"miglesias": ["Labvirusres"],
"bmartinezd": ["SpainUDP", "SpainUDP_ext","GeneticDiagnosis","GeneticDiagnosis_ext"],
"sresino": ["Labvirushep"],
"a.vazquez": ["Labarbovirus"],
"aavellon": ["Labvirushep"],
"adelavieja": ["LabEndocrineTumors"],
Expand Down
Loading