diff --git a/src/pzserver/communicate.py b/src/pzserver/communicate.py index 97df643..c6a698b 100644 --- a/src/pzserver/communicate.py +++ b/src/pzserver/communicate.py @@ -649,6 +649,39 @@ def download_product(self, _id, save_in="."): f"{self._base_api_url}products/{_id}/download", save_in ) + def start_process(self, data): + """ + Start process in Pz Server + + Args: + data (dict): data process + + Returns: + dict: record data + """ + + process = self._post_request(f"{self._base_api_url}processes/", payload=data) + + if "success" in process and process["success"] is False: + raise requests.exceptions.RequestException(process["message"]) + + return process.get("data") + + def stop_process(self, process_id): + """ + Stop process in Pz Server + + Args: + process_id (int): process ID + """ + + data = self._get_request(f"{self._base_api_url}processes/{process_id}/stop") + + if "success" in data and data["success"] is False: + raise requests.exceptions.RequestException(data["message"]) + + return data.get("data") + def upload_basic_info( self, name, product_type, release=None, pz_code=None, description=None ): diff --git a/src/pzserver/core.py b/src/pzserver/core.py index 999eb4a..2819823 100644 --- a/src/pzserver/core.py +++ b/src/pzserver/core.py @@ -3,6 +3,7 @@ """ import tempfile +import time import pandas as pd import tables_io @@ -11,6 +12,7 @@ from .catalog import SpeczCatalog, TrainingSet from .communicate import PzRequests +from .process import TSMProcess from .upload import PzUpload, UploadData pd.options.display.max_colwidth = None @@ -31,7 +33,7 @@ def __init__(self, token=None, host="pz"): PzServer class constructor Args: - token: (str): user's token generated on the PZ Server website + token (str): user's token generated on the PZ Server website host (str): "pz" (production) or "pz-dev" (test environment) or "localhost" (dev environment) or @@ -68,12 +70,15 @@ def display_product_types(self): descriptions (optimized for use in Jupyter Notebook). """ results_dict = self.get_product_types() - dataframe = pd.DataFrame(results_dict, columns=["display_name", "name", "description"]) + dataframe = pd.DataFrame( + results_dict, columns=["display_name", "name", "description"] + ) dataframe.rename( columns={ - "display_name": "Product Type", - "name": "product_type", - "description": "Description"}, + "display_name": "Product Type", + "name": "product_type", + "description": "Description", + }, inplace=True, ) display(dataframe.style.hide(axis="index")) @@ -398,10 +403,17 @@ def get_product(self, product_id=None, fmt=None): print("Done!") return results - def upload(self, name: str, product_type: str, main_file: str, - release: str = None, pz_code: str = None, - auxiliary_files: list = None, description: str = None): - """ Make upload + def upload( + self, + name: str, + product_type: str, + main_file: str, + release: str = None, + pz_code: str = None, + auxiliary_files: list = None, + description: str = None, + ): + """Make upload Args: name (str): name @@ -423,7 +435,7 @@ def upload(self, name: str, product_type: str, main_file: str, "main_file": main_file, "auxiliary_files": auxiliary_files, "pz_code": pz_code, - "description": description + "description": description, } prod = UploadData(**data) @@ -468,26 +480,34 @@ def combine_specz_catalogs(self, catalog_list, duplicates_critera="smallest flag # return SpeczCatalog object raise NotImplementedError - def make_training_set( - self, - specz_catalog=None, - photo_catalog=None, - search_radius=1.0, - multiple_match_criteria="select closest", - ): - """_summary_ + def make_training_set(self, name): + """ + Make training set Args: - specz_catalog (_type_, optional): _description_. Defaults to None. - photo_catalog (_type_, optional): _description_. Defaults to None. - search_radius (float, optional): _description_. Defaults to 1.0. - multiple_match_criteria (str, optional): _description_. Defaults to "select closest". + name (str): training set name - Raises: - NotImplementedError: _description_ + Return: + TSMProcess: TSMProcess object """ - # "select closest" - # keep all - # show progress bar - # return - raise NotImplementedError + return TSMProcess(name, self.api) + + def wait_processing(self, process): + """ + Wait for processing to finish (30 minute tolerance time) + + Args: + process (TSMProcess or CombSpeczProcess): process object + + Return: + dict: process status + """ + + retry = 30 + process.run() + + while process.check_status() in ("Running", "Pending") and retry: + time.sleep(60) + retry = retry - 1 + + return process.check_status() diff --git a/src/pzserver/pipeline.py b/src/pzserver/pipeline.py new file mode 100644 index 0000000..7d4d7f4 --- /dev/null +++ b/src/pzserver/pipeline.py @@ -0,0 +1,92 @@ +"""Pipeline package""" + + +class Pipeline: + """Pipeline data class""" + + def __init__(self, name, api): + """Initialize pipeline data class + + Args: + name: pipeline name + api: PzRequests + """ + + self.__api = api + self.__data = self.__api.get_by_name("pipelines", name) + self.__data["name"] = name + + acceptable_product_types = [] + + for typeid in self.__data.get("product_types_accepted", []): + typeobj = self.__api.get("product-types", typeid) + acceptable_product_types.append(typeobj) + + self.acceptable_product_types = tuple(acceptable_product_types) + + self.output_product_type = self.__api.get( + "product-types", self.__data.get("output_product_type") + ) + + @property + def pipeline_id(self): + """Get pipeline ID + + Returns: + int: pipeline ID + """ + return self.__data.get("id") + + @property + def name(self): + """Get pipeline name + + Returns: + str: pipeline name + """ + return self.__data.get("name") + + @property + def display_name(self): + """Get pipeline display name + + Returns: + str: pipeline display name + """ + return self.__data.get("display_name", None) + + @property + def system_config(self): + """Get pipeline config + + Returns: + dict: pipeline config + """ + return self.__data.get("system_config", {}) + + @property + def parameters(self): + """Get pipeline parameters + + Returns: + dict: pipeline parameters + """ + return self.system_config.get("param", {}) + + @property + def version(self): + """Get pipeline version + + Returns: + str: pipeline version + """ + return self.__data.get("version", None) + + @property + def description(self): + """Get pipeline description + + Returns: + str: pipeline description + """ + return self.__data.get("description", None) diff --git a/src/pzserver/process.py b/src/pzserver/process.py new file mode 100644 index 0000000..96361e9 --- /dev/null +++ b/src/pzserver/process.py @@ -0,0 +1,254 @@ +""" +Classes responsible for managing user interaction with processes +""" + +from .pipeline import Pipeline + +FONTCOLORERR = "\033[38;2;255;0;0m" +FONTCOLOREND = "\033[0m" + + +class TSMProcess: + """Responsible for managing user interactions with TSM process.""" + + # pylint: disable=too-many-instance-attributes + # Eight is reasonable in this case. + + def __init__(self, name, api): + """TSM process class constructor + + Args: + name (str): TSM name + api (PzRequests): PzRequests + """ + + self.__api = api + self.name = name + self.comment = None + self.pipeline = Pipeline("training_set_maker", self.__api) + self.__config = self.pipeline.parameters + self.__release = None + self.__specz = None + self.__process = None + self.__upload = None + + def __available_product_types_by_attribute(self, attr): + """List the product types available for TSM by attribute + + Returns: + list: product types available + """ + available_inputs = [] + + for producttype in self.pipeline.acceptable_product_types: + available_inputs.append(producttype.get(attr)) + + return available_inputs + + def available_product_types(self): + """List the product types available for TSM + + Returns: + list: product types available + """ + return self.__available_product_types_by_attribute("name") + + def available_product_types_id(self): + """List the product types id available for TSM + + Returns: + list: product types id available + """ + return self.__available_product_types_by_attribute("id") + + @property + def output(self): + """TSM output info + + Returns: + dict: output info + """ + if not self.__upload: + return None + + return { + "id": self.__upload.get("id"), + "display_name": self.__upload.get("display_name"), + "internal_name": self.__upload.get("internal_name"), + } + + @property + def process(self): + """TSM process info + + Returns: + dict: process info + """ + if not self.__process: + return None + + return { + "output": self.output, + "id": self.__process.get("id"), + "status": self.check_status(), + } + + @property + def release(self): + """Gets release info + + Returns: + dict: release info + """ + return self.__release + + def set_release(self, release_id=None, name=None): + """Set release + + Args: + release_id (int, optional): release ID. Defaults to None. + name (str, optional): release name. Defaults to None. + + Raises: + ValueError: when neither release_id nor name is informed, the raise is triggered + """ + if release_id: + release = self.__api.get("releases", release_id) + elif name: + release = self.__api.get_by_name("releases", name) + else: + raise ValueError(f"{FONTCOLORERR}No release selected{FONTCOLOREND}") + self.__release = release + + @property + def specz(self): + """Gets specz info + + Returns: + dict: specz info + """ + return self.__specz + + def set_specz(self, specz_id=None, internal_name=None): + """Set specz + + Args: + specz_id (int, optional): product ID. Defaults to None. + internal_name (str, optional): internal name. Defaults to None. + + Raises: + ValueError: when neither specz_id nor internal_name is informed, the raise is triggered + """ + if specz_id: + specz = self.__api.get("products", specz_id) + elif internal_name: + specz = self.__api.get_by_attribute( + "products", "internal_name", internal_name + ) + specz = specz.get("results")[0] + else: + raise ValueError(f"{FONTCOLORERR}No specz selected{FONTCOLOREND}") + + specz_pt = specz.get("product_type") + + if not specz_pt in self.available_product_types_id(): + raise ValueError( + f"{FONTCOLORERR}Input is not of the expected type.{FONTCOLOREND}" + ) + + self.__specz = specz + + @property + def config(self): + """Gets config + + Returns: + dict: config + """ + return self.__config + + def set_config(self, config): + """Set config + + Args: + config (dict): config + """ + self.__config.update(config) + + @property + def summary(self): + """Summary of what will be executed""" + + dn_specz = { + "name": self.specz.get("display_name"), + "internal_name": self.specz.get("internal_name"), + "id": self.specz.get("id"), + } + + print('-'*30) + print(f"Training Set Maker: {self.name}") + print(f"Release: {self.release.get('display_name')}") + print(f"Specz: {dn_specz}") + print(f"Configuration: {self.config}") + if self.output: + print(f"Output: {self.output}") + print(f"Status: {self.check_status()}") + print('-'*30) + + def run(self): + """Starts TSM processing + + Raises: + ValueError: Fired when no release is set + ValueError: Fired when no specz is set + + Returns: + dict: process info + """ + if self.__process: + print(f"Process has already been initialized: {self.process}") + return {} + + if not self.release: + raise ValueError(f"{FONTCOLORERR}No release selected{FONTCOLOREND}") + + if not self.specz: + raise ValueError(f"{FONTCOLORERR}No specz selected{FONTCOLOREND}") + + data_process = { + "release": self.release.get("id"), + "display_name": self.name, + "used_config": {"param": self.config}, + "pipeline": self.pipeline.pipeline_id, + "inputs": [self.specz.get("id")], + } + + process = self.__api.start_process(data_process) + data_process["id"] = process.get("id") + self.__process = data_process + self.__upload = self.__api.get("products", process.get("upload")) + + return self.process + + def check_status(self): + """Checks process status + + Returns: + dict: process status + """ + if not self.__process: + return "The process has not been started" + + process = self.__api.get("processes", self.__process.get("id")) + return f"{process.get('status')}" + + def stop(self): + """Stop process + + Returns: + dict: process info + """ + if not self.__process: + return "No process is running" + + return self.__api.stop_process(self.__process.get("id")) diff --git a/src/pzserver/upload.py b/src/pzserver/upload.py index 7104743..93cc471 100644 --- a/src/pzserver/upload.py +++ b/src/pzserver/upload.py @@ -8,9 +8,6 @@ from pydantic import BaseModel, validator -FONTCOLORERR = "\033[38;2;255;0;0m" -FONTCOLOREND = "\033[0m" - class UploadData(BaseModel): """Upload data"""