From 1661b18a0eb98201453c9be4e857957c82dfd2fe Mon Sep 17 00:00:00 2001 From: Robin Andersson Date: Wed, 24 Apr 2024 10:21:52 +0200 Subject: [PATCH] [HWORKS-1048] Support multiple modularized project environments --- python/hopsworks/core/environment_api.py | 52 +++++++++++++------ python/hopsworks/core/library_api.py | 4 +- python/hopsworks/engine/environment_engine.py | 20 +++---- python/hopsworks/environment.py | 50 ++++++++++++------ python/pyproject.toml | 2 +- 5 files changed, 82 insertions(+), 46 deletions(-) diff --git a/python/hopsworks/core/environment_api.py b/python/hopsworks/core/environment_api.py index 1946d5180..971af194c 100644 --- a/python/hopsworks/core/environment_api.py +++ b/python/hopsworks/core/environment_api.py @@ -14,6 +14,9 @@ # limitations under the License. # +import json +from typing import List, Optional + from hopsworks import client, environment from hopsworks.engine import environment_engine @@ -29,7 +32,7 @@ def __init__( self._environment_engine = environment_engine.EnvironmentEngine(project_id) - def create_environment(self, await_creation=True): + def create_environment(self, name: str, description: Optional[str] = None, base_environment_name: Optional[str] = "python-feature-pipeline", await_creation: Optional[bool] = True) -> environment.Environment: """Create Python environment for the project ```python @@ -40,10 +43,13 @@ def create_environment(self, await_creation=True): env_api = project.get_environment_api() - env = env_api.create_environment() + new_env = env_api.create_environment("my_custom_environment", base_environment_name="python-feature-pipeline") + ``` # Arguments + name: name of the environment + base_environment_name: the name of the environment to copy from await_creation: bool. If True the method returns only when the creation is finished. Default True # Returns `Environment`: The Environment object @@ -57,21 +63,26 @@ def create_environment(self, await_creation=True): self._project_id, "python", "environments", - client.get_python_version(), + name, ] headers = {"content-type": "application/json"} + data = {"name": name, + "baseImage": { + "name": base_environment_name, + "description": description + }} env = environment.Environment.from_response_json( - _client._send_request("POST", path_params, headers=headers), + _client._send_request("POST", path_params, headers=headers, data=json.dumps(data)), self._project_id, self._project_name, ) if await_creation: - self._environment_engine.await_environment_command() + self._environment_engine.await_environment_command(name) return env - def _get_environments(self): + def get_environments(self) -> List[environment.Environment]: """ Get all available python environments in the project """ @@ -88,7 +99,7 @@ def _get_environments(self): self._project_name, ) - def get_environment(self): + def get_environment(self, name: str) -> environment.Environment: """Get handle for the Python environment for the project ```python @@ -99,7 +110,7 @@ def get_environment(self): env_api = project.get_environment_api() - env = env_api.get_environment() + env = env_api.get_environment("my_custom_environment") ``` # Returns @@ -107,14 +118,21 @@ def get_environment(self): # Raises `RestAPIError`: If unable to get the environment """ - project_envs = self._get_environments() - if len(project_envs) == 0: - return None - elif len(project_envs) > 0: - return project_envs[0] - - def _delete(self, python_version): - """Delete the project Python environment""" + _client = client.get_instance() + + path_params = ["project", self._project_id, "python", "environments", name] + query_params = {"expand": ["libraries", "commands"]} + headers = {"content-type": "application/json"} + return environment.Environment.from_response_json( + _client._send_request( + "GET", path_params, query_params=query_params, headers=headers + ), + self._project_id, + self._project_name, + ) + + def _delete(self, name): + """Delete the Python environment""" _client = client.get_instance() path_params = [ @@ -122,7 +140,7 @@ def _delete(self, python_version): self._project_id, "python", "environments", - python_version, + name, ] headers = {"content-type": "application/json"} _client._send_request("DELETE", path_params, headers=headers), diff --git a/python/hopsworks/core/library_api.py b/python/hopsworks/core/library_api.py index dd430a5d9..d80b75ea4 100644 --- a/python/hopsworks/core/library_api.py +++ b/python/hopsworks/core/library_api.py @@ -28,7 +28,7 @@ def __init__( self._project_id = project_id self._project_name = project_name - def install(self, library_name: str, python_version: str, library_spec: dict): + def install(self, library_name: str, name: str, library_spec: dict): """Create Python environment for the project""" _client = client.get_instance() @@ -37,7 +37,7 @@ def install(self, library_name: str, python_version: str, library_spec: dict): self._project_id, "python", "environments", - python_version, + name, "libraries", library_name, ] diff --git a/python/hopsworks/engine/environment_engine.py b/python/hopsworks/engine/environment_engine.py index 56f605504..6941334ec 100644 --- a/python/hopsworks/engine/environment_engine.py +++ b/python/hopsworks/engine/environment_engine.py @@ -16,29 +16,29 @@ import time -from hopsworks import client, library, environment, command -from hopsworks.client.exceptions import RestAPIError, EnvironmentException +from hopsworks import client, command, environment, library +from hopsworks.client.exceptions import EnvironmentException, RestAPIError class EnvironmentEngine: def __init__(self, project_id): self._project_id = project_id - def await_library_command(self, library_name=None): + def await_library_command(self, environment_name, library_name): commands = [command.Command(status="ONGOING")] while len(commands) > 0 and not self._is_final_status(commands[0]): time.sleep(5) - library = self._poll_commands_library(library_name) + library = self._poll_commands_library(environment_name, library_name) if library is None: commands = [] else: commands = library._commands - def await_environment_command(self): + def await_environment_command(self, environment_name): commands = [command.Command(status="ONGOING")] while len(commands) > 0 and not self._is_final_status(commands[0]): time.sleep(5) - environment = self._poll_commands_environment() + environment = self._poll_commands_environment(environment_name) if environment is None: commands = [] else: @@ -54,7 +54,7 @@ def _is_final_status(self, command): else: return False - def _poll_commands_library(self, library_name): + def _poll_commands_library(self, environment_name, library_name): _client = client.get_instance() path_params = [ @@ -62,7 +62,7 @@ def _poll_commands_library(self, library_name): self._project_id, "python", "environments", - client.get_python_version(), + environment_name, "libraries", library_name, ] @@ -85,7 +85,7 @@ def _poll_commands_library(self, library_name): ): return None - def _poll_commands_environment(self): + def _poll_commands_environment(self, environment_name): _client = client.get_instance() path_params = [ @@ -93,7 +93,7 @@ def _poll_commands_environment(self): self._project_id, "python", "environments", - client.get_python_version(), + environment_name, ] query_params = {"expand": "commands"} diff --git a/python/hopsworks/environment.py b/python/hopsworks/environment.py index 0c93d560e..5f760729b 100644 --- a/python/hopsworks/environment.py +++ b/python/hopsworks/environment.py @@ -14,9 +14,10 @@ # limitations under the License. # -import humps import os +from typing import Optional +import humps from hopsworks import command, util from hopsworks.core import environment_api, library_api from hopsworks.engine import environment_engine @@ -25,9 +26,11 @@ class Environment: def __init__( self, - python_version, - python_conflicts, - pip_search_enabled, + name=None, + description=None, + python_version=None, + python_conflicts=None, + pip_search_enabled=None, conflicts=None, conda_channel=None, libraries=None, @@ -38,6 +41,8 @@ def __init__( project_name=None, **kwargs, ): + self._name = name + self._description = description self._python_version = python_version self._python_conflicts = python_conflicts self._pip_search_enabled = pip_search_enabled @@ -77,7 +82,17 @@ def python_version(self): """Python version of the environment""" return self._python_version - def install_wheel(self, path, await_installation=True): + @property + def name(self): + """Name of the environment""" + return self._name + + @property + def description(self): + """Description of the environment""" + return self._description + + def install_wheel(self, path, await_installation: Optional[bool] = True): """Install a python library packaged in a wheel file ```python @@ -92,7 +107,8 @@ def install_wheel(self, path, await_installation=True): # Install env_api = project.get_environment_api() - env = env_api.get_environment() + env = env_api.get_environment("my_custom_environment") + env.install_wheel(whl_path) ``` @@ -103,7 +119,7 @@ def install_wheel(self, path, await_installation=True): """ # Wait for any ongoing environment operations - self._environment_engine.await_environment_command() + self._environment_engine.await_environment_command(self.name) library_name = os.path.basename(path) @@ -116,15 +132,15 @@ def install_wheel(self, path, await_installation=True): } library_rest = self._library_api.install( - library_name, self.python_version, library_spec + library_name, self.name, library_spec ) if await_installation: - return self._environment_engine.await_library_command(library_name) + return self._environment_engine.await_library_command(self.name, library_name) return library_rest - def install_requirements(self, path, await_installation=True): + def install_requirements(self, path, await_installation: Optional[bool] = True): """Install libraries specified in a requirements.txt file ```python @@ -139,7 +155,9 @@ def install_requirements(self, path, await_installation=True): # Install env_api = project.get_environment_api() - env = env_api.get_environment() + env = env_api.get_environment("my_custom_environment") + + env.install_requirements(requirements_path) ``` @@ -150,7 +168,7 @@ def install_requirements(self, path, await_installation=True): """ # Wait for any ongoing environment operations - self._environment_engine.await_environment_command() + self._environment_engine.await_environment_command(self.name) library_name = os.path.basename(path) @@ -163,11 +181,11 @@ def install_requirements(self, path, await_installation=True): } library_rest = self._library_api.install( - library_name, self.python_version, library_spec + library_name, self.name, library_spec ) if await_installation: - return self._environment_engine.await_library_command(library_name) + return self._environment_engine.await_library_command(self.name, library_name) return library_rest @@ -178,7 +196,7 @@ def delete(self): # Raises `RestAPIError`. """ - self._environment_api._delete(self.python_version) + self._environment_api._delete(self.name) def __repr__(self): - return f"Environment({self._python_version!r})" + return f"Environment({self.name!r})" diff --git a/python/pyproject.toml b/python/pyproject.toml index b5c356526..0b2a30892 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -23,7 +23,7 @@ classifiers = [ dependencies = [ "hsfs[python] @ git+https://git@github.com/logicalclocks/feature-store-api@master#subdirectory=python", - "hsml @ git+https://git@github.com/logicalclocks/machine-learning-api@main#subdirectory=python", + "hsml @ git+https://git@github.com/robzor92/machine-learning-api@HWORKS-1048#subdirectory=python", "pyhumps==1.6.1", "requests", "furl",