[FSTORE-1389] get_secrets_api and create_project should be accessible…

… as hopsworks module functions (#202)

auto_doc.py

@@ -22,6 +22,7 @@
 PAGES = {
     "api/login.md": {
         "login": ["hopsworks.login"],
+        "get_current_project": ["hopsworks.get_current_project"],
         "fs_api": ["hopsworks.project.Project.get_feature_store"],
         "mr_api": ["hopsworks.project.Project.get_model_registry"],
         "ms_api": ["hopsworks.project.Project.get_model_serving"],
@@ -36,9 +37,7 @@
         ),
     },
     "api/projects.md": {
-        "project_create": ["hopsworks.connection.Connection.create_project"],
-        "project_get": ["hopsworks.connection.Connection.get_project"],
-        "project_get_all": ["hopsworks.connection.Connection.get_projects"],
+        "project_create": ["hopsworks.create_project"],
         "project_properties": keras_autodoc.get_properties("hopsworks.project.Project"),
         "project_methods": keras_autodoc.get_methods(
             "hopsworks.project.Project", exclude=["from_response_json", "json"]
@@ -163,9 +162,10 @@
         ),
     },
     "api/secrets.md": {
-        "secret_api_handle": ["hopsworks.connection.Connection.get_secrets_api"],
+        "secret_api_handle": ["hopsworks.get_secrets_api"],
         "secret_create": ["hopsworks.core.secret_api.SecretsApi.create_secret"],
         "secret_get": ["hopsworks.core.secret_api.SecretsApi.get_secret"],
+        "secret_get_simplified": ["hopsworks.core.secret_api.SecretsApi.get"],
         "secret_get_all": ["hopsworks.core.secret_api.SecretsApi.get_secrets"],
         "secret_properties": keras_autodoc.get_properties("hopsworks.secret.Secret"),
         "secret_methods": keras_autodoc.get_methods(

docs/templates/api/login.md

@@ -2,6 +2,8 @@
 
 {{login}}
 
+{{get_current_project}}
+
 ## Feature Store API
 
 {{fs_api}}

docs/templates/api/projects.md

@@ -4,12 +4,6 @@
 
 {{project_create}}
 
-## Retrieval
-
-{{project_get}}
-
-{{project_get_all}}
-
 ## Properties
 
 {{project_properties}}

docs/templates/api/secrets.md

@@ -10,6 +10,8 @@
 
 ## Retrieval
 
+{{secret_get_simplified}}
+
 {{secret_get}}
 
 {{secret_get_all}}

python/hopsworks/__init__.py

@@ -26,6 +26,8 @@
 from hopsworks import client, constants, project, version
 from hopsworks.client.exceptions import ProjectException, RestAPIError
 from hopsworks.connection import Connection
+from hopsworks.core import project_api, secret_api
+from hopsworks.decorators import NoHopsworksConnectionError
 
 
 # Needs to run before import of hsml and hsfs
@@ -42,6 +44,8 @@
 _hw_connection = Connection.connection
 
 _connected_project = None
+_secrets_api = None
+_project_api = None
 
 
 def hw_formatwarning(message, category, filename, lineno, line=None):
@@ -68,21 +72,27 @@ def login(
 ) -> project.Project:
     """Connect to [Serverless Hopsworks](https://app.hopsworks.ai) by calling the `hopsworks.login()` function with no arguments.
 
-    ```python
+    !!! example "Connect to Serverless"
+        ```python
 
-    project = hopsworks.login()
+        import hopsworks
 
-    ```
+        project = hopsworks.login()
+
+        ```
 
     Alternatively, connect to your own Hopsworks installation by specifying the host, port and api key.
 
-    ```python
+    !!! example "Connect to your Hopsworks cluster"
+        ```python
+
+        import hopsworks
 
-    project = hopsworks.login(host="my.hopsworks.server",
-                              port=8181,
-                              api_key_value="DKN8DndwaAjdf98FFNSxwdVKx")
+        project = hopsworks.login(host="my.hopsworks.server",
+                                  port=8181,
+                                  api_key_value="DKN8DndwaAjdf98FFNSxwdVKx")
 
-    ```
+        ```
 
     In addition to setting function arguments directly, `hopsworks.login()` also reads the environment variables:
     HOPSWORKS_HOST, HOPSWORKS_PORT, HOPSWORKS_PROJECT and HOPSWORKS_API_KEY.
@@ -97,7 +107,7 @@ def login(
         api_key_value: Value of the Api Key
         api_key_file: Path to file wih Api Key
     # Returns
-        `Project`: The Project object
+        `Project`: The Project object to perform operations on
     # Raises
         `RestAPIError`: If unable to connect to Hopsworks
     """
@@ -113,6 +123,7 @@ def login(
     if "REST_ENDPOINT" in os.environ:
         _hw_connection = _hw_connection()
         _connected_project = _hw_connection.get_project()
+        _initialize_module_apis()
         print("\nLogged in to project, explore it here " + _connected_project.get_url())
         return _connected_project
 
@@ -140,6 +151,8 @@ def login(
     elif host is None:  # Always do a fallback to Serverless Hopsworks if not defined
         host = constants.HOSTS.APP_HOST
 
+    is_app = host == constants.HOSTS.APP_HOST
+
     # If port same as default, get HOPSWORKS_HOST environment variable
     if port == 443 and "HOPSWORKS_PORT" in os.environ:
         port = os.environ["HOPSWORKS_PORT"]
@@ -166,23 +179,24 @@ def login(
                 "Could not find api key file on path: {}".format(api_key_file)
             )
     # If user connected to Serverless Hopsworks, and the cached .hw_api_key exists, then use it.
-    elif os.path.exists(api_key_path) and host == constants.HOSTS.APP_HOST:
+    elif os.path.exists(api_key_path) and is_app:
         try:
             _hw_connection = _hw_connection(
                 host=host, port=port, api_key_file=api_key_path
             )
-            _connected_project = _prompt_project(_hw_connection, project)
+            _connected_project = _prompt_project(_hw_connection, project, is_app)
             print(
                 "\nLogged in to project, explore it here "
                 + _connected_project.get_url()
             )
+            _initialize_module_apis()
             return _connected_project
         except RestAPIError:
             logout()
             # API Key may be invalid, have the user supply it again
             os.remove(api_key_path)
 
-    if api_key is None and host == constants.HOSTS.APP_HOST:
+    if api_key is None and is_app:
         print(
             "Copy your Api Key (first register/login): https://c.app.hopsworks.ai/account/api/generated"
         )
@@ -198,12 +212,19 @@ def login(
 
     try:
         _hw_connection = _hw_connection(host=host, port=port, api_key_value=api_key)
-        _connected_project = _prompt_project(_hw_connection, project)
+        _connected_project = _prompt_project(_hw_connection, project, is_app)
     except RestAPIError as e:
         logout()
         raise e
 
-    print("\nLogged in to project, explore it here " + _connected_project.get_url())
+    if _connected_project is None:
+        print(
+            "Could not find any project, use hopsworks.create_project('my_project') to create one"
+        )
+    else:
+        print("\nLogged in to project, explore it here " + _connected_project.get_url())
+
+    _initialize_module_apis()
     return _connected_project
 
 
@@ -245,11 +266,14 @@ def _get_cached_api_key_path():
     return api_key_path
 
 
-def _prompt_project(valid_connection, project):
+def _prompt_project(valid_connection, project, is_app):
     saas_projects = valid_connection.get_projects()
     if project is None:
         if len(saas_projects) == 0:
-            raise ProjectException("Could not find any project")
+            if is_app:
+                raise ProjectException("Could not find any project")
+            else:
+                return None
         elif len(saas_projects) == 1:
             return saas_projects[0]
         else:
@@ -258,7 +282,9 @@ def _prompt_project(valid_connection, project):
                 for index in range(len(saas_projects)):
                     print("\t (" + str(index + 1) + ") " + saas_projects[index].name)
                 while True:
-                    project_index = input("\nEnter project to access: ")
+                    project_index = input(
+                        "\nEnter number corresponding to the project to use: "
+                    )
                     # Handle invalid input type
                     try:
                         project_index = int(project_index)
@@ -285,8 +311,111 @@ def _prompt_project(valid_connection, project):
 
 
 def logout():
+    """Cleans up and closes the connection for the hopsworks, hsfs and hsml libraries."""
     global _hw_connection
-    if isinstance(_hw_connection, Connection):
+    global _project_api
+    global _secrets_api
+
+    if _is_connection_active():
         _hw_connection.close()
+
     client.stop()
+    _project_api = None
+    _secrets_api = None
     _hw_connection = Connection.connection
+
+
+def _is_connection_active():
+    global _hw_connection
+    return isinstance(_hw_connection, Connection)
+
+
+def get_current_project() -> project.Project:
+    """Get a reference to the current logged in project.
+
+    !!! example "Example for getting the project reference"
+        ```python
+
+        import hopsworks
+
+        hopsworks.login()
+
+        project = hopsworks.get_current_project()
+
+        ```
+
+    # Returns
+        `Project`. The Project object to perform operations on
+    """
+    global _connected_project
+    if _connected_project is None:
+        raise ProjectException("No project is set for this session")
+    return _connected_project
+
+
+def _initialize_module_apis():
+    global _project_api
+    global _secrets_api
+    _project_api = project_api.ProjectApi()
+    _secrets_api = secret_api.SecretsApi()
+
+
+def create_project(name: str, description: str = None, feature_store_topic: str = None):
+    """Create a new project.
+
+    !!! warning "Not supported"
+        This is not supported if you are connected to [Serverless Hopsworks](https://app.hopsworks.ai)
+
+    !!! example "Example for creating a new project"
+        ```python
+
+        import hopsworks
+
+        hopsworks.login(...)
+
+        hopsworks.create_project("my_project", description="An example Hopsworks project")
+
+        ```
+    # Arguments
+        name: The name of the project.
+        description: optional description of the project
+        feature_store_topic: optional feature store topic name
+
+    # Returns
+        `Project`. The Project object to perform operations on
+    """
+    global _hw_connection
+    global _connected_project
+
+    if not _is_connection_active():
+        raise NoHopsworksConnectionError()
+
+    new_project = _hw_connection._project_api._create_project(
+        name, description, feature_store_topic
+    )
+    if _connected_project is None:
+        _connected_project = new_project
+        print(
+            "Setting {} as the current project, a reference can be retrieved by calling hopsworks.get_current_project()".format(
+                _connected_project.name
+            )
+        )
+        return _connected_project
+    else:
+        print(
+            "You are already using the project {}, to access the new project use hopsworks.login(..., project='{}')".format(
+                _connected_project.name, new_project.name
+            )
+        )
+
+
+def get_secrets_api():
+    """Get the secrets api.
+
+    # Returns
+        `SecretsApi`: The Secrets Api handle
+    """
+    global _secrets_api
+    if not _is_connection_active():
+        raise NoHopsworksConnectionError()
+    return _secrets_api

python/hopsworks/core/opensearch_api.py

@@ -15,10 +15,9 @@
 #
 
 from furl import furl
-
 from hopsworks import client, constants
-from hopsworks.core import variable_api
 from hopsworks.client.exceptions import OpenSearchException
+from hopsworks.core import variable_api
 
 
 class OpenSearchApi: