Merge remote-tracking branch 'origin/main' into dedup-hsml

logicalclocks · Oct 7, 2024 · 7479664 · 7479664
2 parents b30f8b0 + a5c28e7
commit 7479664
Show file tree

Hide file tree

Showing 66 changed files with 918 additions and 491 deletions.
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -36,7 +36,7 @@
 
 We follow a few best practices for writing the Python documentation:
 
-1. Use the google docstring style:
+1. Use the Google docstring style:
 
    ```python
    """[One Line Summary]
@@ -72,16 +72,17 @@ We use `mkdocs` together with `mike` ([for versioning](https://github.com/jimpor
 **Background about `mike`:**
 `mike` builds the documentation and commits it as a new directory to the gh-pages branch. Each directory corresponds to one version of the documentation. Additionally, `mike` maintains a json in the root of gh-pages with the mappings of versions/aliases for each of the directories available. With aliases you can define extra names like `dev` or `latest`, to indicate stable and unstable releases.
 
-1. Install Hopsworks with `dev-docs` extras:
+1. Install Hopsworks with `requirements-docs.txt`:
 
    ```bash
-   pip install -e ".[dev-docs]"
+   pip install -r requirements-docs.txt
+   pip install -e "python[dev]"
    ```
 
 2. To build the docs, first run the auto doc script:
 
    ```bash
-   python auto_doc.py
+   python python/auto_doc.py
    ```
 
 ##### Option 1: Build only current version of docs

diff --git a/docs/templates/api/connection.md b/docs/templates/api/connection.md
@@ -1,13 +1,11 @@
-# Connection API
+# Connection
 
-## Creation
-
-{{connection_create}}
+{{connection}}
 
 ## Properties
 
 {{connection_properties}}
 
 ## Methods
 
-{{connection_methods}}
+{{connection_methods}}
diff --git a/docs/templates/api/connection_api.md b/docs/templates/api/connection_api.md
diff --git a/docs/templates/api/job.md b/docs/templates/api/job.md
diff --git a/docs/templates/api/jobs.md b/docs/templates/api/jobs.md
@@ -1,6 +1,6 @@
 # Jobs API
 
-## Handle 
+## Handle
 
 {{job_api_handle}}
 

diff --git a/docs/templates/connection_api.md b/docs/templates/connection_api.md
diff --git a/java/beam/pom.xml b/java/beam/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <artifactId>hsfs-parent</artifactId>
     <groupId>com.logicalclocks</groupId>
-    <version>4.0.0-SNAPSHOT</version>
+    <version>4.1.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

diff --git a/java/flink/pom.xml b/java/flink/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <artifactId>hsfs-parent</artifactId>
     <groupId>com.logicalclocks</groupId>
-    <version>4.0.0-SNAPSHOT</version>
+    <version>4.1.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

diff --git a/java/hsfs/pom.xml b/java/hsfs/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <artifactId>hsfs-parent</artifactId>
     <groupId>com.logicalclocks</groupId>
-    <version>4.0.0-SNAPSHOT</version>
+    <version>4.1.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

diff --git a/java/pom.xml b/java/pom.xml
@@ -7,7 +7,7 @@
     <groupId>com.logicalclocks</groupId>
     <artifactId>hsfs-parent</artifactId>
     <packaging>pom</packaging>
-    <version>4.0.0-SNAPSHOT</version>
+    <version>4.1.0-SNAPSHOT</version>
     <modules>
         <module>hsfs</module>
         <module>spark</module>

diff --git a/java/spark/pom.xml b/java/spark/pom.xml
@@ -22,7 +22,7 @@
     <parent>
         <artifactId>hsfs-parent</artifactId>
         <groupId>com.logicalclocks</groupId>
-        <version>4.0.0-SNAPSHOT</version>
+        <version>4.1.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/locust_benchmark/create_feature_group.py b/locust_benchmark/create_feature_group.py
@@ -1,7 +1,6 @@
 from common.hopsworks_client import HopsworksClient
 
 if __name__ == "__main__":
-
     hopsworks_client = HopsworksClient()
     fg = hopsworks_client.get_or_create_fg()
     hopsworks_client.insert_data(fg)

diff --git a/mkdocs.yml b/mkdocs.yml
@@ -16,8 +16,8 @@ nav:
   - Guides: https://docs.hopsworks.ai/
   - Setup and Installation: https://docs.hopsworks.ai/
   - Administration: https://docs.hopsworks.ai/
-  - API<div class="dropdown"><button class="dropbtn"> API </button> <div id="myDropdown" class="dropdown-content"> <a id="hopsworks_api_link" href="https://docs.hopsworks.ai/hopsworks-api/latest">Hopsworks API</a> <a id="hsfs_api_link" href="https://docs.hopsworks.ai/feature-store-api/latest">Feature Store API</a> <a id="hsfs_javadoc_link" href="https://docs.hopsworks.ai/feature-store-api/latest/javadoc">Feature Store JavaDoc</a> <a id="hsml_api_link" href="https://docs.hopsworks.ai/machine-learning-api/latest">MLOps API</a> </div></div>:
-    - API Reference:
+  - API<div class="dropdown"><button class="dropbtn"> API </button> <div id="myDropdown" class="dropdown-content"> <a id="hopsworks_api_link" href="https://docs.hopsworks.ai/hopsworks-api/latest">Python API</a> <a id="hsfs_javadoc_link" href="https://docs.hopsworks.ai/hopsworks-api/latest/javadoc">Feature Store JavaDoc</a> </div></div>:
+    - Platform API:
       - Login: generated/api/login.md
       - Connection: generated/api/connection.md
       - Projects: generated/api/projects.md
@@ -33,7 +33,7 @@ nav:
       - KafkaSchema: generated/api/kafka_schema.md
       - Secrets: generated/api/secrets.md
       - OpenSearch: generated/api/opensearch.md
-      - Connection (HSFS): generated/api/connection_api.md
+    - Feature Store API:
       - ExpectationSuite: generated/api/expectation_suite_api.md
       - FeatureStore: generated/api/feature_store_api.md
       - FeatureGroup: generated/api/feature_group_api.md
@@ -48,11 +48,10 @@ nav:
         - UDF: generated/api/udf.md
         - HopsworksUDF: generated/api/hopsworks_udf.md
         - TransformationFunction: generated/api/transformation_functions_api.md
-        - Transformation Statistics: 
+        - Transformation Statistics:
           - TransformationStatistics: generated/api/transformation_statistics.md
           - FeatureTransformationStatistics: generated/api/feature_transformation_statistics.md
       - ValidationReport: generated/api/validation_report_api.md
-      - Job: generated/api/job.md
       - Provenance Links: generated/api/links.md
       - Statistics:
           - Statistics: generated/api/statistics_api.md
@@ -66,7 +65,7 @@ nav:
           - EmbeddingIndex: generated/api/embedding_index_api.md
           - EmbeddingFeature: generated/api/embedding_feature_api.md
           - SimilarityFunctionType: generated/api/similarity_function_type_api.md
-      - Connection (HSML): generated/connection_api.md
+    - Machine Learning API:
       - Model Registry:
         - Model Registry: generated/model-registry/model_registry_api.md
         - Model: generated/model-registry/model_api.md
@@ -82,7 +81,6 @@ nav:
         - Inference Batcher: generated/model-serving/inference_batcher_api.md
         - Resources: generated/model-serving/resources_api.md
     # Added to allow navigation using the side drawer
-    - Hopsworks API: https://docs.hopsworks.ai/hopsworks-api/latest/
     - Feature Store JavaDoc: https://docs.hopsworks.ai/feature-store-javadoc/latest/
     - Contributing: CONTRIBUTING.md
   - Community ↗: https://community.hopsworks.ai/

diff --git a/python/auto_doc.py b/python/auto_doc.py
@@ -40,7 +40,7 @@
         "udf": ["hopsworks.udf"],
     },
     "api/connection.md": {
-        "connection_create": ["hopsworks.connection.Connection.connection"],
+        "connection": ["hopsworks.connection.Connection.connection"],
         "connection_properties": keras_autodoc.get_properties(
             "hopsworks.connection.Connection"
         ),
@@ -61,7 +61,10 @@
         "job_get": ["hopsworks.core.job_api.JobApi.get_job"],
         "job_get_all": ["hopsworks.core.job_api.JobApi.get_jobs"],
         "job_properties": keras_autodoc.get_properties("hopsworks.job.Job"),
-        "job_config": ["hopsworks.core.job_api.JobApi.get_configuration"],
+        "job_config": [
+            "hopsworks.core.job_api.JobApi.get_configuration",
+            "hopsworks_common.core.job_configuration.JobConfiguration",
+        ],
         "job_methods": keras_autodoc.get_methods(
             "hopsworks.job.Job", exclude=["from_response_json", "json"]
         ),
@@ -190,13 +193,6 @@
             "hopsworks.core.opensearch_api.OpenSearchApi"
         ),
     },
-    "api/connection_api.md": {
-        "connection": ["hsfs.connection.Connection"],
-        "connection_properties": keras_autodoc.get_properties(
-            "hsfs.connection.Connection"
-        ),
-        "connection_methods": keras_autodoc.get_methods("hsfs.connection.Connection"),
-    },
     "api/spine_group_api.md": {
         "fg": ["hsfs.feature_group.SpineGroup"],
         "fg_create": ["hsfs.feature_store.FeatureStore.get_or_create_spine_group"],
@@ -413,14 +409,6 @@
             "hsfs.validation_report.ValidationReport"
         ),
     },
-    "api/job.md": {
-        "job_configuration": ["hsfs.core.job_configuration.JobConfiguration"],
-        "job": ["hsfs.core.job.Job"],
-        "job_methods": [
-            "hsfs.core.job.Job.get_state",
-            "hsfs.core.job.Job.get_final_state",
-        ],
-    },
     "api/query_api.md": {
         "query_methods": keras_autodoc.get_methods(
             "hsfs.constructor.query.Query",
@@ -550,13 +538,6 @@
         "similarity_function_type": ["hsfs.embedding.SimilarityFunctionType"],
     },
     # Model registry
-    "connection_api.md": {
-        "connection": ["hsml.connection.Connection"],
-        "connection_properties": keras_autodoc.get_properties(
-            "hsml.connection.Connection", exclude=["trust_store_path"]
-        ),
-        "connection_methods": keras_autodoc.get_methods("hsml.connection.Connection"),
-    },
     "model-registry/model_registry_api.md": {
         "mr_get": ["hsml.connection.Connection.get_model_registry"],
         "mr_modules": keras_autodoc.get_properties(

diff --git a/python/hopsworks/__init__.py b/python/hopsworks/__init__.py
@@ -81,7 +81,7 @@ def login(
     project: str = None,
     api_key_value: str = None,
     api_key_file: str = None,
-    hostname_verification: bool = True,
+    hostname_verification: bool = False,
     trust_store_path: str = None,
 ) -> project.Project:
     """Connect to [Serverless Hopsworks](https://app.hopsworks.ai) by calling the `hopsworks.login()` function with no arguments.

diff --git a/python/hopsworks_common/client/exceptions.py b/python/hopsworks_common/client/exceptions.py
@@ -50,6 +50,7 @@ def __init__(self, url: str, response: requests.Response) -> None:
                 error_object = {"errorMsg": error_object}
         except Exception:
             error_object = {}
+            self.error_code = None
         message = (
             "Metadata operation error: (url: {}). Server response: \n"
             "HTTP code: {}, HTTP reason: {}, body: {}, error code: {}, error msg: {}, user "
@@ -63,6 +64,8 @@ def __init__(self, url: str, response: requests.Response) -> None:
                 error_object.get("usrMsg", ""),
             )
         )
+        if len(error_object) != 0:
+            self.error_code = error_object.get("errorCode", "")
         super().__init__(message)
         self.url = url
         self.response = response
@@ -118,14 +121,10 @@ def __init__(self, missing_argument: str) -> None:
         super().__init__(message)
 
 
-class HopsworksClientError(TypeError):
-    """Raised when hopsworks internal client cannot be initialized due to missing arguments."""
+class InternalClientError(TypeError):
+    """Raised when hopsworks internal client is missing some necessary configuration."""
 
-    def __init__(self, missing_argument):
-        message = (
-            "{0} cannot be of type NoneType, {0} is a non-optional "
-            "argument to connect to hopsworks from an internal environment."
-        ).format(missing_argument)
+    def __init__(self, message: str) -> None:
         super().__init__(message)
 
 

diff --git a/python/hopsworks_common/client/hopsworks.py b/python/hopsworks_common/client/hopsworks.py
@@ -56,7 +56,7 @@ def __init__(self, hostname_verification):
         self._hostname_verification = os.environ.get(
             self.HOPSWORKS_HOSTNAME_VERIFICATION, "{}".format(hostname_verification)
         ).lower() in ("true", "1", "y", "yes")
-        self._hopsworks_ca_trust_store_path = self._get_trust_store_path()
+        self._hopsworks_ca_trust_store_path = self._materialize_ca_chain()
 
         self._project_id = os.environ[self.PROJECT_ID]
         self._project_name = self._project_name()
@@ -73,17 +73,12 @@ def __init__(self, hostname_verification):
 
         credentials = self._get_credentials(self._project_id)
 
-        self._write_pem_file(credentials["caChain"], self._get_ca_chain_path())
         self._write_pem_file(credentials["clientCert"], self._get_client_cert_path())
         self._write_pem_file(credentials["clientKey"], self._get_client_key_path())
 
-    def _get_hopsworks_rest_endpoint(self):
-        """Get the hopsworks REST endpoint for making requests to the REST API."""
-        return os.environ[self.REST_ENDPOINT]
-
-    def _get_trust_store_path(self):
+    def _materialize_ca_chain(self):
         """Convert truststore from jks to pem and return the location"""
-        ca_chain_path = Path(self.PEM_CA_CHAIN)
+        ca_chain_path = Path(self._get_ca_chain_path())
         if not ca_chain_path.exists():
             keystore_pw = self._cert_key
             ks = jks.KeyStore.load(
@@ -95,6 +90,10 @@ def _get_trust_store_path(self):
             self._write_ca_chain(ks, ts, ca_chain_path)
         return str(ca_chain_path)
 
+    def _get_hopsworks_rest_endpoint(self):
+        """Get the hopsworks REST endpoint for making requests to the REST API."""
+        return os.environ[self.REST_ENDPOINT]
+
     def _get_ca_chain_path(self) -> str:
         return os.path.join("/tmp", "ca_chain.pem")