diff --git a/auto_doc.py b/auto_doc.py index b5a42c1a1..9b2a6ad00 100644 --- a/auto_doc.py +++ b/auto_doc.py @@ -22,6 +22,7 @@ PAGES = { "api/login.md": { "login": ["hopsworks.login"], + "get_current_project": ["hopsworks.get_current_project"], "fs_api": ["hopsworks.project.Project.get_feature_store"], "mr_api": ["hopsworks.project.Project.get_model_registry"], "ms_api": ["hopsworks.project.Project.get_model_serving"], @@ -36,9 +37,7 @@ ), }, "api/projects.md": { - "project_create": ["hopsworks.connection.Connection.create_project"], - "project_get": ["hopsworks.connection.Connection.get_project"], - "project_get_all": ["hopsworks.connection.Connection.get_projects"], + "project_create": ["hopsworks.create_project"], "project_properties": keras_autodoc.get_properties("hopsworks.project.Project"), "project_methods": keras_autodoc.get_methods( "hopsworks.project.Project", exclude=["from_response_json", "json"] @@ -163,9 +162,10 @@ ), }, "api/secrets.md": { - "secret_api_handle": ["hopsworks.connection.Connection.get_secrets_api"], + "secret_api_handle": ["hopsworks.get_secrets_api"], "secret_create": ["hopsworks.core.secret_api.SecretsApi.create_secret"], "secret_get": ["hopsworks.core.secret_api.SecretsApi.get_secret"], + "secret_get_simplified": ["hopsworks.core.secret_api.SecretsApi.get"], "secret_get_all": ["hopsworks.core.secret_api.SecretsApi.get_secrets"], "secret_properties": keras_autodoc.get_properties("hopsworks.secret.Secret"), "secret_methods": keras_autodoc.get_methods( diff --git a/docs/templates/api/login.md b/docs/templates/api/login.md index 05368ba49..9a2d73e45 100644 --- a/docs/templates/api/login.md +++ b/docs/templates/api/login.md @@ -2,6 +2,8 @@ {{login}} +{{get_current_project}} + ## Feature Store API {{fs_api}} diff --git a/docs/templates/api/projects.md b/docs/templates/api/projects.md index 170a63b60..a39282d15 100644 --- a/docs/templates/api/projects.md +++ b/docs/templates/api/projects.md @@ -4,12 +4,6 @@ {{project_create}} -## Retrieval - -{{project_get}} - -{{project_get_all}} - ## Properties {{project_properties}} diff --git a/docs/templates/api/secrets.md b/docs/templates/api/secrets.md index ca7042613..ab186ab44 100644 --- a/docs/templates/api/secrets.md +++ b/docs/templates/api/secrets.md @@ -10,6 +10,8 @@ ## Retrieval +{{secret_get_simplified}} + {{secret_get}} {{secret_get_all}} diff --git a/python/hopsworks/__init__.py b/python/hopsworks/__init__.py index a0c6920d8..e1d09688b 100644 --- a/python/hopsworks/__init__.py +++ b/python/hopsworks/__init__.py @@ -26,6 +26,8 @@ from hopsworks import client, constants, project, version from hopsworks.client.exceptions import ProjectException, RestAPIError from hopsworks.connection import Connection +from hopsworks.core import project_api, secret_api +from hopsworks.decorators import NoHopsworksConnectionError # Needs to run before import of hsml and hsfs @@ -42,6 +44,8 @@ _hw_connection = Connection.connection _connected_project = None +_secrets_api = None +_project_api = None def hw_formatwarning(message, category, filename, lineno, line=None): @@ -113,6 +117,7 @@ def login( if "REST_ENDPOINT" in os.environ: _hw_connection = _hw_connection() _connected_project = _hw_connection.get_project() + _initialize_module_apis() print("\nLogged in to project, explore it here " + _connected_project.get_url()) return _connected_project @@ -140,6 +145,8 @@ def login( elif host is None: # Always do a fallback to Serverless Hopsworks if not defined host = constants.HOSTS.APP_HOST + is_app = host == constants.HOSTS.APP_HOST + # If port same as default, get HOPSWORKS_HOST environment variable if port == 443 and "HOPSWORKS_PORT" in os.environ: port = os.environ["HOPSWORKS_PORT"] @@ -166,23 +173,24 @@ def login( "Could not find api key file on path: {}".format(api_key_file) ) # If user connected to Serverless Hopsworks, and the cached .hw_api_key exists, then use it. - elif os.path.exists(api_key_path) and host == constants.HOSTS.APP_HOST: + elif os.path.exists(api_key_path) and is_app: try: _hw_connection = _hw_connection( host=host, port=port, api_key_file=api_key_path ) - _connected_project = _prompt_project(_hw_connection, project) + _connected_project = _prompt_project(_hw_connection, project, is_app) print( "\nLogged in to project, explore it here " + _connected_project.get_url() ) + _initialize_module_apis() return _connected_project except RestAPIError: logout() # API Key may be invalid, have the user supply it again os.remove(api_key_path) - if api_key is None and host == constants.HOSTS.APP_HOST: + if api_key is None and is_app: print( "Copy your Api Key (first register/login): https://c.app.hopsworks.ai/account/api/generated" ) @@ -198,12 +206,19 @@ def login( try: _hw_connection = _hw_connection(host=host, port=port, api_key_value=api_key) - _connected_project = _prompt_project(_hw_connection, project) + _connected_project = _prompt_project(_hw_connection, project, is_app) except RestAPIError as e: logout() raise e - print("\nLogged in to project, explore it here " + _connected_project.get_url()) + if _connected_project is None: + print( + "Could not find any project, use hopsworks.create_project('my_project') to create one" + ) + else: + print("\nLogged in to project, explore it here " + _connected_project.get_url()) + + _initialize_module_apis() return _connected_project @@ -245,11 +260,14 @@ def _get_cached_api_key_path(): return api_key_path -def _prompt_project(valid_connection, project): +def _prompt_project(valid_connection, project, is_app): saas_projects = valid_connection.get_projects() if project is None: if len(saas_projects) == 0: - raise ProjectException("Could not find any project") + if is_app: + raise ProjectException("Could not find any project") + else: + return None elif len(saas_projects) == 1: return saas_projects[0] else: @@ -258,7 +276,9 @@ def _prompt_project(valid_connection, project): for index in range(len(saas_projects)): print("\t (" + str(index + 1) + ") " + saas_projects[index].name) while True: - project_index = input("\nEnter project to access: ") + project_index = input( + "\nEnter number corresponding to the project to use: " + ) # Handle invalid input type try: project_index = int(project_index) @@ -285,8 +305,110 @@ def _prompt_project(valid_connection, project): def logout(): + """Cleans up and closes the connection for the hopsworks, hsfs and hsml libraries.""" global _hw_connection - if isinstance(_hw_connection, Connection): + global _project_api + global _secrets_api + + if _is_connection_active(): _hw_connection.close() + client.stop() + _project_api = None + _secrets_api = None _hw_connection = Connection.connection + + +def _is_connection_active(): + global _hw_connection + return isinstance(_hw_connection, Connection) + + +def get_current_project() -> project.Project: + """Get a reference to the current logged in project. + + Example for creating a new project + + ```python + + import hopsworks + + hopsworks.login() + + project = hopsworks.get_current_project() + + ``` + + # Returns + `Project`. A project handle object to perform operations on. + """ + global _connected_project + if _connected_project is None: + raise ProjectException("No project is set for this session") + return _connected_project + + +def _initialize_module_apis(): + global _project_api + global _secrets_api + _project_api = project_api.ProjectApi() + _secrets_api = secret_api.SecretsApi() + + +def create_project(name: str, description: str = None, feature_store_topic: str = None): + """Create a new project. + + Example for creating a new project + + ```python + + import hopsworks + + hopsworks.login() + + hopsworks.create_project("my_hopsworks_project", description="An example Hopsworks project") + + ``` + # Arguments + name: The name of the project. + description: optional description of the project + feature_store_topic: optional feature store topic name + + # Returns + `Project`. A project handle object to perform operations on. + """ + global _hw_connection + global _connected_project + + if not _is_connection_active(): + raise NoHopsworksConnectionError() + + new_project = _hw_connection._project_api._create_project( + name, description, feature_store_topic + ) + if _connected_project is None: + _connected_project = new_project + print( + "Setting {} as the current project, a reference can be retrieved by calling hopsworks.get_current_project()".format( + _connected_project.name + ) + ) + return _connected_project + else: + print( + "You are already using the project {}, to access the new project use hopsworks.login(..., project='{}')".format( + _connected_project.name, new_project.name + ) + ) + + +def get_secrets_api(): + """Get the secrets api. + + # Returns + `SecretsApi`: The Secrets Api handle + """ + global _secrets_api + if not _is_connection_active(): + raise NoHopsworksConnectionError() + return _secrets_api diff --git a/python/hopsworks/core/secret_api.py b/python/hopsworks/core/secret_api.py index a39a2dca0..40e5340d2 100644 --- a/python/hopsworks/core/secret_api.py +++ b/python/hopsworks/core/secret_api.py @@ -14,10 +14,13 @@ # limitations under the License. # -from hopsworks import client, secret -from hopsworks.core import project_api +import getpass import json +from hopsworks import client, secret, util +from hopsworks.client.exceptions import RestAPIError +from hopsworks.core import project_api + class SecretsApi: def __init__( @@ -42,11 +45,11 @@ def get_secrets(self): _client._send_request("GET", path_params) ) - def get_secret(self, name: str, owner: str = None): + def get_secret(self, name: str, owner: str = None) -> secret.Secret: """Get a secret. # Arguments - name: Name of the project. + name: Name of the secret. owner: email of the owner for a secret shared with the current project. # Returns `Secret`: The Secret object @@ -69,11 +72,34 @@ def get_secret(self, name: str, owner: str = None): "shared", ] - return secret.Secret.from_response_json( - _client._send_request("GET", path_params, query_params=query_params) - )[0] + return secret.Secret.from_response_json(_client._send_request("GET", path_params, query_params=query_params))[0] + + def get(self, name: str, owner: str = None) -> str: + """Get the secret's value. + If the secret does not exist, it prompts the user to create the secret if the application is running interactively - def create_secret(self, name: str, value: str, project: str = None): + # Arguments + name: Name of the secret. + owner: email of the owner for a secret shared with the current project. + # Returns + `str`: The secret value + # Raises + `RestAPIError`: If unable to get the secret + """ + try: + return self.get_secret(name=name, owner=owner).value + except RestAPIError as e: + if ( + e.response.json().get("errorCode", "") == 160048 + and e.response.status_code == 404 + and util.is_interactive() + ): + secret_input = getpass.getpass(prompt="\nCould not find secret, enter value here to create it: ") + return self.create_secret(name, secret_input).value + else: + raise e + + def create_secret(self, name: str, value: str, project: str = None) -> secret.Secret: """Create a new secret. ```python diff --git a/python/hopsworks/project.py b/python/hopsworks/project.py index 80cd387be..7e1fd1664 100644 --- a/python/hopsworks/project.py +++ b/python/hopsworks/project.py @@ -109,6 +109,8 @@ def get_feature_store(self, name: str = None) -> feature_store.FeatureStore: Defaulting to the project name of default feature store. To get a shared feature store, the project name of the feature store is required. + # Arguments + name: Project name of the feature store. # Returns `hsfs.feature_store.FeatureStore`: The Feature Store API # Raises diff --git a/python/hopsworks/secret.py b/python/hopsworks/secret.py index 4583e233c..701488e5d 100644 --- a/python/hopsworks/secret.py +++ b/python/hopsworks/secret.py @@ -15,8 +15,8 @@ # import json -import humps +import humps from hopsworks import util from hopsworks.core import secret_api @@ -48,7 +48,7 @@ def __init__( @classmethod def from_response_json(cls, json_dict): json_decamelized = humps.decamelize(json_dict) - if len(json_decamelized["items"]) == 0: + if "items" not in json_decamelized or len(json_decamelized["items"]) == 0: return [] return [cls(**secret) for secret in json_decamelized["items"]] diff --git a/python/hopsworks/util.py b/python/hopsworks/util.py index 70712431a..35785783f 100644 --- a/python/hopsworks/util.py +++ b/python/hopsworks/util.py @@ -15,10 +15,11 @@ # from json import JSONEncoder +from urllib.parse import urljoin, urlparse + +from hopsworks import client from hopsworks.client.exceptions import JobException from hopsworks.git_file_status import GitFileStatus -from hopsworks import client -from urllib.parse import urljoin, urlparse class Encoder(JSONEncoder): @@ -79,3 +80,7 @@ def get_hostname_replaced_url(sub_path: str): href = urljoin(client.get_instance()._base_url, sub_path) url_parsed = client.get_instance().replace_public_host(urlparse(href)) return url_parsed.geturl() + +def is_interactive(): + import __main__ as main + return not hasattr(main, '__file__')