diff --git a/README.md b/README.md index 2504b8d80..4b8987ae3 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,7 @@ deployment.start() Make predictions with a deployed model ```python -data = { "instances": model.input_example } +data = { "instances": [ model.input_example ] } predictions = deployment.predict(data) ``` diff --git a/python/hsml/connection.py b/python/hsml/connection.py index a25fe83be..621842ad6 100644 --- a/python/hsml/connection.py +++ b/python/hsml/connection.py @@ -131,6 +131,16 @@ def get_model_registry(self, project: str = None): def get_model_serving(self): """Get a reference to model serving to perform operations on. Model serving operates on top of a model registry, defaulting to the project's default model registry. + !!! example + ```python + + import hopsworks + + project = hopsworks.login() + + ms = project.get_model_serving() + ``` + # Returns `ModelServing`. A model serving handle object to perform operations on. """ diff --git a/python/hsml/deployment.py b/python/hsml/deployment.py index 44af62ff0..c057b1484 100644 --- a/python/hsml/deployment.py +++ b/python/hsml/deployment.py @@ -143,17 +143,41 @@ def is_stopped(self, or_created=True) -> bool: or_created and status == PREDICTOR_STATE.STATUS_CREATED ) - def predict(self, data: dict): - """Send inference requests to the deployment + def predict(self, data: dict = None, inputs: list = None): + """Send inference requests to the deployment. + One of data or inputs parameters must be set. If both are set, inputs will be ignored. + + !!! example + ```python + # login into Hopsworks using hopsworks.login() + + # get Hopsworks Model Serving handle + ms = project.get_model_serving() + + # retrieve deployment by name + my_deployment = ms.get_deployment("my_deployment") + + # (optional) retrieve model input example + my_model = project.get_model_registry() \ + .get_model(my_deployment.model_name, my_deployment.model_version) + + # make predictions using model inputs (single or batch) + predictions = my_deployment.predict(inputs=my_model.input_example) + + # or using more sophisticated inference request payloads + data = { "instances": [ my_model.input_example ], "key2": "value2" } + predictions = my_deployment.predict(data) + ``` # Arguments - data: Payload of the inference request. + data: Payload dictionary for the inference request including the model input(s) + inputs: Model inputs used in the inference requests # Returns `dict`. Inference response. """ - return self._serving_engine.predict(self, data) + return self._serving_engine.predict(self, data, inputs) def download_artifact(self): """Download the model artifact served by the deployment""" diff --git a/python/hsml/engine/serving_engine.py b/python/hsml/engine/serving_engine.py index 47b1c96b7..07788e56d 100644 --- a/python/hsml/engine/serving_engine.py +++ b/python/hsml/engine/serving_engine.py @@ -166,14 +166,16 @@ def update_progress(state, num_instances): update_progress, ) - def predict(self, deployment_instance, data: dict): + def predict(self, deployment_instance, data, inputs): + payload = self._build_inference_payload(data, inputs) + serving_tool = deployment_instance.predictor.serving_tool through_hopsworks = ( serving_tool != PREDICTOR.SERVING_TOOL_KSERVE ) # if not KServe, send request to Hopsworks try: return self._serving_api.send_inference_request( - deployment_instance, data, through_hopsworks + deployment_instance, payload, through_hopsworks ) except RestAPIError as re: if ( @@ -190,6 +192,33 @@ def predict(self, deployment_instance, data: dict): ) raise re + def _build_inference_payload(self, data, inputs): + """Build or check the payload for an inference request. If the 'data' parameter is provided, this method ensures + it contains one of 'instances' or 'inputs' keys needed by the model server. Otherwise, if the 'inputs' parameter + is provided, this method builds the correct request payload using the 'instances' key. + While the 'inputs' key is only supported by default deployments, the 'instances' key is supported in all types of deployments. + """ + if data is not None: # check data + if not isinstance(data, dict): + raise ModelServingException( + "Inference data must be a dictionary. Otherwise, use the inputs parameter." + ) + if "instances" not in data and "inputs" not in data: + raise ModelServingException("Inference data is missing 'instances' key") + else: # parse inputs + if not isinstance(inputs, list): + data = {"instances": [inputs]} # wrap inputs in a list + else: + data = {"instances": inputs} # use given inputs list by default + # check depth of the list: at least two levels are required for batch inference + # if the content is neither a list or dict, wrap it in an additional list + for i in inputs: + if not isinstance(i, list) and not isinstance(i, dict): + # if there are no two levels, wrap inputs in a list + data = {"instances": [inputs]} + break + return data + def _check_status(self, deployment_instance, desired_status): state = deployment_instance.get_state() if state is None: diff --git a/python/hsml/model.py b/python/hsml/model.py index 1d18d8f03..859df1172 100644 --- a/python/hsml/model.py +++ b/python/hsml/model.py @@ -83,7 +83,15 @@ def __init__( self._model_engine = model_engine.ModelEngine() def save(self, model_path, await_registration=480): - """Persist this model including model files and metadata to the model registry.""" + """Persist this model including model files and metadata to the model registry. + + # Arguments + model_path: Local or remote (Hopsworks file system) path to the folder where the model files are located, or path to a specific model file. + await_registration: Awaiting time for the model to be registered in Hopsworks. + + # Returns + `Model`. The model metadata object. + """ return self._model_engine.save( self, model_path, await_registration=await_registration ) @@ -118,6 +126,21 @@ def deploy( ): """Deploy the model. + !!! example + ```python + + import hopsworks + + project = hopsworks.login() + + # get Hopsworks Model Registry handle + mr = project.get_model_registry() + + # retrieve the trained model you want to deploy + my_model = mr.get_model("my_model", version=1) + + my_deployment = my_model.deploy() + ``` # Arguments name: Name of the deployment. description: Description of the deployment. diff --git a/python/hsml/model_serving.py b/python/hsml/model_serving.py index e9866f3d5..0e9946dca 100644 --- a/python/hsml/model_serving.py +++ b/python/hsml/model_serving.py @@ -43,6 +43,14 @@ def get_deployment_by_id(self, id: int): Getting a deployment from Model Serving means getting its metadata handle so you can subsequently operate on it (e.g., start or stop). + !!! example + ```python + # login and get Hopsworks Model Serving handle using .login() and .get_model_serving() + + # get a deployment by id + my_deployment = ms.get_deployment_by_id(1) + ``` + # Arguments id: Id of the deployment to get. # Returns @@ -55,6 +63,15 @@ def get_deployment_by_id(self, id: int): def get_deployment(self, name: str): """Get a deployment by name from Model Serving. + + !!! example + ```python + # login and get Hopsworks Model Serving handle using .login() and .get_model_serving() + + # get a deployment by name + my_deployment = ms.get_deployment('deployment_name') + ``` + Getting a deployment from Model Serving means getting its metadata handle so you can subsequently operate on it (e.g., start or stop). @@ -70,7 +87,24 @@ def get_deployment(self, name: str): def get_deployments(self, model: Model = None, status: str = None): """Get all deployments from model serving. + !!! example + ```python + # login into Hopsworks using hopsworks.login() + + # get Hopsworks Model Registry handle + mr = project.get_model_registry() + # get Hopsworks Model Serving handle + ms = project.get_model_serving() + + # retrieve the trained model you want to deploy + my_model = mr.get_model("my_model", version=1) + + list_deployments = ms.get_deployment(my_model) + + for deployment in list_deployments: + print(deployment.get_state()) + ``` # Arguments model: Filter by model served in the deployments status: Filter by status of the deployments @@ -120,6 +154,24 @@ def create_predictor( ): """Create a Predictor metadata object. + !!! example + ```python + # login into Hopsworks using hopsworks.login() + + # get Hopsworks Model Registry handle + mr = project.get_model_registry() + + # retrieve the trained model you want to deploy + my_model = mr.get_model("my_model", version=1) + + # get Hopsworks Model Serving handle + ms = project.get_model_serving() + + my_predictor = ms.create_predictor(my_model) + + my_deployment = my_predictor.deploy() + ``` + !!! note "Lazy" This method is lazy and does not persist any metadata or deploy any model on its own. To create a deployment using this predictor, call the `deploy()` method. @@ -162,6 +214,53 @@ def create_transformer( ): """Create a Transformer metadata object. + !!! example + ```python + # login into Hopsworks using hopsworks.login() + + # get Dataset API instance + dataset_api = project.get_dataset_api() + + # get Hopsworks Model Serving handle + ms = project.get_model_serving() + + # create my_transformer.py Python script + class Transformer(object): + def __init__(self): + ''' Initialization code goes here ''' + pass + + def preprocess(self, inputs): + ''' Transform the requests inputs here. The object returned by this method will be used as model input to make predictions. ''' + return inputs + + def postprocess(self, outputs): + ''' Transform the predictions computed by the model before returning a response ''' + return outputs + + uploaded_file_path = dataset_api.upload("my_transformer.py", "Resources", overwrite=True) + transformer_script_path = os.path.join("/Projects", project.name, uploaded_file_path) + + my_transformer = ms.create_transformer(script_file=uploaded_file_path) + + # or + + from hsml.transformer import Transformer + + my_transformer = Transformer(script_file) + ``` + + !!! example "Create a deployment with the transformer" + ```python + + my_predictor = ms.create_predictor(transformer=my_transformer) + my_deployment = my_predictor.deploy() + + # or + my_deployment = ms.create_deployment(my_predictor, transformer=my_transformer) + my_deployment.save() + ``` + !!! note "Lazy" This method is lazy and does not persist any metadata or deploy any transformer. To create a deployment using this transformer, set it in the `predictor.transformer` property. @@ -178,6 +277,60 @@ def create_transformer( def create_deployment(self, predictor: Predictor, name: Optional[str] = None): """Create a Deployment metadata object. + !!! example + ```python + # login into Hopsworks using hopsworks.login() + + # get Hopsworks Model Registry handle + mr = project.get_model_registry() + + # retrieve the trained model you want to deploy + my_model = mr.get_model("my_model", version=1) + + # get Hopsworks Model Serving handle + ms = project.get_model_serving() + + my_predictor = ms.create_predictor(my_model) + + my_deployment = ms.create_deployment(my_predictor) + my_deployment.save() + ``` + + !!! example "Using the model object" + ```python + # login into Hopsworks using hopsworks.login() + + # get Hopsworks Model Registry handle + mr = project.get_model_registry() + + # retrieve the trained model you want to deploy + my_model = mr.get_model("my_model", version=1) + + my_deployment = my_model.deploy() + + my_deployment.get_state().describe() + ``` + + !!! example "Using the Model Serving handle" + ```python + # login into Hopsworks using hopsworks.login() + + # get Hopsworks Model Registry handle + mr = project.get_model_registry() + + # retrieve the trained model you want to deploy + my_model = mr.get_model("my_model", version=1) + + # get Hopsworks Model Serving handle + ms = project.get_model_serving() + + my_predictor = ms.create_predictor(my_model) + + my_deployment = my_predictor.deploy() + + my_deployment.get_state().describe() + ``` + !!! note "Lazy" This method is lazy and does not persist any metadata or deploy any model. To create a deployment, call the `save()` method. diff --git a/python/hsml/predictor.py b/python/hsml/predictor.py index 5d2a5394e..05c4faf2d 100644 --- a/python/hsml/predictor.py +++ b/python/hsml/predictor.py @@ -89,6 +89,28 @@ def __init__( def deploy(self): """Create a deployment for this predictor and persists it in the Model Serving. + !!! example + ```python + + import hopsworks + + project = hopsworks.login() + + # get Hopsworks Model Registry handle + mr = project.get_model_registry() + + # retrieve the trained model you want to deploy + my_model = mr.get_model("my_model", version=1) + + # get Hopsworks Model Serving handle + ms = project.get_model_serving() + + my_predictor = ms.create_predictor(my_model) + my_deployment = my_predictor.deploy() + + print(my_deployment.get_state()) + ``` + # Returns `Deployment`. The deployment metadata object of a new or existing deployment. """ @@ -106,7 +128,6 @@ def describe(self): def _set_state(self, state: PredictorState): """Set the state of the predictor""" - self._state = state @classmethod diff --git a/python/hsml/python/signature.py b/python/hsml/python/signature.py index f572096ab..80e6ff9ca 100644 --- a/python/hsml/python/signature.py +++ b/python/hsml/python/signature.py @@ -47,7 +47,7 @@ def create_model( version in the model registry. description: Optionally a string describing the model, defaults to empty string `""`. - input_example: Optionally an input example that represents inputs for the model, defaults to `None`. + input_example: Optionally an input example that represents a single input for the model, defaults to `None`. model_schema: Optionally a model schema for the model inputs and/or outputs. # Returns diff --git a/python/hsml/sklearn/signature.py b/python/hsml/sklearn/signature.py index 27adb6f5f..4af911f40 100644 --- a/python/hsml/sklearn/signature.py +++ b/python/hsml/sklearn/signature.py @@ -47,7 +47,7 @@ def create_model( version in the model registry. description: Optionally a string describing the model, defaults to empty string `""`. - input_example: Optionally an input example that represents inputs for the model, defaults to `None`. + input_example: Optionally an input example that represents a single input for the model, defaults to `None`. model_schema: Optionally a model schema for the model inputs and/or outputs. # Returns diff --git a/python/hsml/tensorflow/signature.py b/python/hsml/tensorflow/signature.py index f1d1635f4..5ea2d9754 100644 --- a/python/hsml/tensorflow/signature.py +++ b/python/hsml/tensorflow/signature.py @@ -47,7 +47,7 @@ def create_model( version in the model registry. description: Optionally a string describing the model, defaults to empty string `""`. - input_example: Optionally an input example that represents inputs for the model, defaults to `None`. + input_example: Optionally an input example that represents a single input for the model, defaults to `None`. model_schema: Optionally a model schema for the model inputs and/or outputs. # Returns diff --git a/python/hsml/torch/signature.py b/python/hsml/torch/signature.py index 32f359be8..9cff09d53 100644 --- a/python/hsml/torch/signature.py +++ b/python/hsml/torch/signature.py @@ -47,7 +47,7 @@ def create_model( version in the model registry. description: Optionally a string describing the model, defaults to empty string `""`. - input_example: Optionally an input example that represents inputs for the model, defaults to `None`. + input_example: Optionally an input example that represents a single input for the model, defaults to `None`. model_schema: Optionally a model schema for the model inputs and/or outputs. # Returns diff --git a/python/hsml/util.py b/python/hsml/util.py index 646415cc0..aa5309b94 100644 --- a/python/hsml/util.py +++ b/python/hsml/util.py @@ -102,10 +102,6 @@ def default(self, obj): # pylint: disable=E0202 # - schema and types -def _is_numpy_scalar(x): - return np.isscalar(x) or x is None - - def set_model_class(model): if "href" in model: _ = model.pop("href") @@ -128,10 +124,6 @@ def set_model_class(model): return PyModel(**model) -def _handle_tensor_input(input_tensor): - return input_tensor.tolist() - - def input_example_to_json(input_example): if isinstance(input_example, np.ndarray): if input_example.size > 0: @@ -140,10 +132,16 @@ def input_example_to_json(input_example): raise ValueError( "input_example of type {} can not be empty".format(type(input_example)) ) + elif isinstance(input_example, dict): + return _handle_dict_input(input_example) else: return _handle_dataframe_input(input_example) +def _handle_tensor_input(input_tensor): + return input_tensor.tolist() + + def _handle_dataframe_input(input_ex): if isinstance(input_ex, pd.DataFrame): if not input_ex.empty: @@ -154,7 +152,7 @@ def _handle_dataframe_input(input_ex): ) elif isinstance(input_ex, pd.Series): if not input_ex.empty: - return input_ex.iloc[0] + return input_ex.tolist() else: raise ValueError( "input_example of type {} can not be empty".format(type(input_ex)) @@ -172,6 +170,10 @@ def _handle_dataframe_input(input_ex): ) +def _handle_dict_input(input_ex): + return input_ex + + # - artifacts