Merge pull request #105 from linea-it/99-tsm-from-pzserver-lib

Implemented TSM processing engine

src/pzserver/communicate.py

@@ -649,6 +649,39 @@ def download_product(self, _id, save_in="."):
             f"{self._base_api_url}products/{_id}/download", save_in
         )
 
+    def start_process(self, data):
+        """
+        Start process in Pz Server
+
+        Args:
+            data (dict): data process
+
+        Returns:
+            dict: record data
+        """
+
+        process = self._post_request(f"{self._base_api_url}processes/", payload=data)
+
+        if "success" in process and process["success"] is False:
+            raise requests.exceptions.RequestException(process["message"])
+
+        return process.get("data")
+
+    def stop_process(self, process_id):
+        """
+        Stop process in Pz Server
+
+        Args:
+            process_id (int): process ID
+        """
+
+        data = self._get_request(f"{self._base_api_url}processes/{process_id}/stop")
+
+        if "success" in data and data["success"] is False:
+            raise requests.exceptions.RequestException(data["message"])
+
+        return data.get("data")
+
     def upload_basic_info(
         self, name, product_type, release=None, pz_code=None, description=None
     ):

src/pzserver/core.py

@@ -3,6 +3,7 @@
 """
 
 import tempfile
+import time
 
 import pandas as pd
 import tables_io
@@ -11,6 +12,7 @@
 
 from .catalog import SpeczCatalog, TrainingSet
 from .communicate import PzRequests
+from .process import TSMProcess
 from .upload import PzUpload, UploadData
 
 pd.options.display.max_colwidth = None
@@ -31,7 +33,7 @@ def __init__(self, token=None, host="pz"):
         PzServer class constructor
 
         Args:
-            token: (str): user's token generated on the PZ Server website
+            token (str): user's token generated on the PZ Server website
             host (str): "pz" (production) or
                         "pz-dev" (test environment) or
                         "localhost" (dev environment) or
@@ -68,12 +70,15 @@ def display_product_types(self):
         descriptions (optimized for use in Jupyter Notebook).
         """
         results_dict = self.get_product_types()
-        dataframe = pd.DataFrame(results_dict, columns=["display_name", "name", "description"])
+        dataframe = pd.DataFrame(
+            results_dict, columns=["display_name", "name", "description"]
+        )
         dataframe.rename(
             columns={
-                     "display_name": "Product Type", 
-                     "name": "product_type",
-                     "description": "Description"},
+                "display_name": "Product Type",
+                "name": "product_type",
+                "description": "Description",
+            },
             inplace=True,
         )
         display(dataframe.style.hide(axis="index"))
@@ -398,10 +403,17 @@ def get_product(self, product_id=None, fmt=None):
         print("Done!")
         return results
 
-    def upload(self, name: str, product_type: str, main_file: str,
-                    release: str = None, pz_code: str = None,
-                    auxiliary_files: list = None, description: str = None):
-        """ Make upload
+    def upload(
+        self,
+        name: str,
+        product_type: str,
+        main_file: str,
+        release: str = None,
+        pz_code: str = None,
+        auxiliary_files: list = None,
+        description: str = None,
+    ):
+        """Make upload
 
         Args:
             name (str): name
@@ -423,7 +435,7 @@ def upload(self, name: str, product_type: str, main_file: str,
             "main_file": main_file,
             "auxiliary_files": auxiliary_files,
             "pz_code": pz_code,
-            "description": description
+            "description": description,
         }
 
         prod = UploadData(**data)
@@ -468,26 +480,34 @@ def combine_specz_catalogs(self, catalog_list, duplicates_critera="smallest flag
         # return SpeczCatalog object
         raise NotImplementedError
 
-    def make_training_set(
-        self,
-        specz_catalog=None,
-        photo_catalog=None,
-        search_radius=1.0,
-        multiple_match_criteria="select closest",
-    ):
-        """_summary_
+    def make_training_set(self, name):
+        """
+        Make training set
 
         Args:
-            specz_catalog (_type_, optional): _description_. Defaults to None.
-            photo_catalog (_type_, optional): _description_. Defaults to None.
-            search_radius (float, optional): _description_. Defaults to 1.0.
-            multiple_match_criteria (str, optional): _description_. Defaults to "select closest".
+            name (str): training set name
 
-        Raises:
-            NotImplementedError: _description_
+        Return:
+            TSMProcess: TSMProcess object
         """
-        # "select closest"
-        # keep all
-        # show progress bar
-        # return
-        raise NotImplementedError
+        return TSMProcess(name, self.api)
+
+    def wait_processing(self, process):
+        """
+        Wait for processing to finish (30 minute tolerance time)
+
+        Args:
+            process (TSMProcess or CombSpeczProcess): process object
+
+        Return:
+            dict: process status
+        """
+
+        retry = 30
+        process.run()
+
+        while process.check_status() in ("Running", "Pending") and retry:
+            time.sleep(60)
+            retry = retry - 1
+
+        return process.check_status()

src/pzserver/pipeline.py

@@ -0,0 +1,92 @@
+"""Pipeline package"""
+
+
+class Pipeline:
+    """Pipeline data class"""
+
+    def __init__(self, name, api):
+        """Initialize pipeline data class
+
+        Args:
+            name: pipeline name
+            api: PzRequests
+        """
+
+        self.__api = api
+        self.__data = self.__api.get_by_name("pipelines", name)
+        self.__data["name"] = name
+
+        acceptable_product_types = []
+
+        for typeid in self.__data.get("product_types_accepted", []):
+            typeobj = self.__api.get("product-types", typeid)
+            acceptable_product_types.append(typeobj)
+
+        self.acceptable_product_types = tuple(acceptable_product_types)
+
+        self.output_product_type = self.__api.get(
+            "product-types", self.__data.get("output_product_type")
+        )
+
+    @property
+    def pipeline_id(self):
+        """Get pipeline ID
+
+        Returns:
+            int: pipeline ID
+        """
+        return self.__data.get("id")
+
+    @property
+    def name(self):
+        """Get pipeline name
+
+        Returns:
+            str: pipeline name
+        """
+        return self.__data.get("name")
+
+    @property
+    def display_name(self):
+        """Get pipeline display name
+
+        Returns:
+            str: pipeline display name
+        """
+        return self.__data.get("display_name", None)
+
+    @property
+    def system_config(self):
+        """Get pipeline config
+
+        Returns:
+            dict: pipeline config
+        """
+        return self.__data.get("system_config", {})
+
+    @property
+    def parameters(self):
+        """Get pipeline parameters
+
+        Returns:
+            dict: pipeline parameters
+        """
+        return self.system_config.get("param", {})
+
+    @property
+    def version(self):
+        """Get pipeline version
+
+        Returns:
+            str: pipeline version
+        """
+        return self.__data.get("version", None)
+
+    @property
+    def description(self):
+        """Get pipeline description
+
+        Returns:
+            str: pipeline description
+        """
+        return self.__data.get("description", None)