From 4a683b014b5f5ac686effb4fc3ff531127c30768 Mon Sep 17 00:00:00 2001 From: Aleksey Veresov Date: Wed, 3 Jul 2024 10:00:28 +0200 Subject: [PATCH 1/8] Merge variable_api of hsfs --- python/hopsworks/core/variable_api.py | 48 ++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/python/hopsworks/core/variable_api.py b/python/hopsworks/core/variable_api.py index 6c8e02a80..02041c1bd 100644 --- a/python/hopsworks/core/variable_api.py +++ b/python/hopsworks/core/variable_api.py @@ -1,5 +1,5 @@ # -# Copyright 2022 Logical Clocks AB +# Copyright 2022 Hopsworks AB # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,8 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. # +from __future__ import annotations + +import re from hopsworks import client +from hopsworks.client.exceptions import RestAPIError class VariableApi: @@ -43,3 +47,45 @@ def get_version(self, software: str): if entry["software"] == software: return entry["version"] return None + + def parse_major_and_minor(self, backend_version): + version_pattern = r"(\d+)\.(\d+)" + matches = re.match(version_pattern, backend_version) + + return matches.group(1), matches.group(2) + + def get_flyingduck_enabled(self): + _client = client.get_instance() + path_params = [ + "variables", + "enable_flyingduck", + ] + + resp = _client._send_request("GET", path_params) + return resp["successMessage"] == "true" + + def get_loadbalancer_external_domain(self): + _client = client.get_instance() + path_params = [ + "variables", + "loadbalancer_external_domain", + ] + + try: + resp = _client._send_request("GET", path_params) + return resp["successMessage"] + except RestAPIError: + return "" + + def get_service_discovery_domain(self): + _client = client.get_instance() + path_params = [ + "variables", + "service_discovery_domain", + ] + + try: + resp = _client._send_request("GET", path_params) + return resp["successMessage"] + except RestAPIError: + return "" From 11b7446c7b88716ff3267ed1357ce17e86f735b5 Mon Sep 17 00:00:00 2001 From: Aleksey Veresov Date: Wed, 3 Jul 2024 10:10:15 +0200 Subject: [PATCH 2/8] Merge project_api of hsfs --- python/hopsworks/core/project_api.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/python/hopsworks/core/project_api.py b/python/hopsworks/core/project_api.py index 7795dd3e0..2ac28c045 100644 --- a/python/hopsworks/core/project_api.py +++ b/python/hopsworks/core/project_api.py @@ -111,3 +111,12 @@ def _create_project( project = self._get_project(name) print("Project created successfully, explore it at " + project.get_url()) return project + + def get_client(self): + _client = client.get_instance() + path_params = [ + "project", + _client._project_id, + "client", + ] + return _client._send_request("GET", path_params, stream=True) From 2f2e380be0d2f01a4bd20a77e9b6f1b152741a37 Mon Sep 17 00:00:00 2001 From: Aleksey Veresov Date: Wed, 3 Jul 2024 10:12:28 +0200 Subject: [PATCH 3/8] Ruff project_api --- python/hopsworks/core/project_api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/hopsworks/core/project_api.py b/python/hopsworks/core/project_api.py index 2ac28c045..db36a5935 100644 --- a/python/hopsworks/core/project_api.py +++ b/python/hopsworks/core/project_api.py @@ -14,8 +14,9 @@ # limitations under the License. # -from hopsworks import client, project, constants import json + +from hopsworks import client, constants, project from hopsworks.client.exceptions import RestAPIError From a7a6388d66800411f323b14dd0e1b3a8bc54cfc4 Mon Sep 17 00:00:00 2001 From: Aleksey Veresov Date: Wed, 3 Jul 2024 10:46:00 +0200 Subject: [PATCH 4/8] Fix client/external so that Project API get_client works That is, set _project_id. --- python/hopsworks/client/external.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/python/hopsworks/client/external.py b/python/hopsworks/client/external.py index 171c2ba7b..d88beb5c5 100644 --- a/python/hopsworks/client/external.py +++ b/python/hopsworks/client/external.py @@ -41,6 +41,11 @@ def __init__( self._port = port self._base_url = "https://" + self._host + ":" + str(self._port) self._project_name = project + if project is not None: + project_info = self._get_project_info(project) + self._project_id = str(project_info["projectId"]) + else: + self._project_id = None if api_key_value is not None: api_key = api_key_value From c6f7ce04eec12e43f49c1939f04c844be89fa364 Mon Sep 17 00:00:00 2001 From: Aleksey Veresov Date: Wed, 3 Jul 2024 10:46:16 +0200 Subject: [PATCH 5/8] Fix Project API docs --- python/hopsworks/core/project_api.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/hopsworks/core/project_api.py b/python/hopsworks/core/project_api.py index db36a5935..46e8a3df9 100644 --- a/python/hopsworks/core/project_api.py +++ b/python/hopsworks/core/project_api.py @@ -28,8 +28,6 @@ def _exists(self, name: str): name: Name of the project. # Returns `bool`: True if project exists, otherwise False - # Raises - `RestAPIError`: If unable to check the existence of the project """ try: self._get_project(name) From 6efbb50d1a59ad4224468840879894d7b8c0c637 Mon Sep 17 00:00:00 2001 From: Aleksey Veresov Date: Wed, 3 Jul 2024 11:04:08 +0200 Subject: [PATCH 6/8] Add docs to variable_api --- python/hopsworks/core/variable_api.py | 61 ++++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 6 deletions(-) diff --git a/python/hopsworks/core/variable_api.py b/python/hopsworks/core/variable_api.py index 02041c1bd..301657346 100644 --- a/python/hopsworks/core/variable_api.py +++ b/python/hopsworks/core/variable_api.py @@ -16,6 +16,7 @@ from __future__ import annotations import re +from typing import Optional, Tuple from hopsworks import client from hopsworks.client.exceptions import RestAPIError @@ -26,7 +27,15 @@ def __init__(self): pass def get_variable(self, variable: str): - """Get the configured value for a variable""" + """Get the configured value of a variable. + + # Arguments + vairable: Name of the variable. + # Returns + The vairable's value + # Raises + `RestAPIError`: If unable to get the variable + """ _client = client.get_instance() @@ -35,7 +44,17 @@ def get_variable(self, variable: str): return domain["successMessage"] - def get_version(self, software: str): + def get_version(self, software: str) -> Optional[str]: + """Get version of a software component. + + # Arguments + software: Name of the software. + # Returns + The software's version, if the software is available, otherwise `None`. + # Raises + `RestAPIError`: If unable to get the version + """ + _client = client.get_instance() path_params = [ "variables", @@ -48,13 +67,31 @@ def get_version(self, software: str): return entry["version"] return None - def parse_major_and_minor(self, backend_version): + def parse_major_and_minor(self, backend_version: str) -> Tuple[Optional[str], Optional[str]]: + """Extract major and minor version from full version. + + # Arguments + backend_version: The full version. + # Returns + (major, minor): The pair of major and minor parts of the version, or (None, None) if the version format is incorrect. + """ + version_pattern = r"(\d+)\.(\d+)" matches = re.match(version_pattern, backend_version) + if matches is None: + return (None, None) return matches.group(1), matches.group(2) - def get_flyingduck_enabled(self): + def get_flyingduck_enabled(self) -> bool: + """Check if Flying Duck is enabled on the backend. + + # Returns + `True`: If flying duck is availalbe, `False` otherwise. + # Raises + `RestAPIError`: If unable to obtain the flag's value. + """ + _client = client.get_instance() path_params = [ "variables", @@ -64,7 +101,13 @@ def get_flyingduck_enabled(self): resp = _client._send_request("GET", path_params) return resp["successMessage"] == "true" - def get_loadbalancer_external_domain(self): + def get_loadbalancer_external_domain(self) -> str: + """Get domain of external loadbalancer. + + # Returns + `str`: The domain of external loadbalancer, if it is set up, otherwise empty string `""`. + """ + _client = client.get_instance() path_params = [ "variables", @@ -77,7 +120,13 @@ def get_loadbalancer_external_domain(self): except RestAPIError: return "" - def get_service_discovery_domain(self): + def get_service_discovery_domain(self) -> str: + """Get domain of service discovery server. + + # Returns + `str`: The domain of service discovery server, if it is set up, otherwise empty string `""`. + """ + _client = client.get_instance() path_params = [ "variables", From 9d7e8481650d7838e9572a8801176b1b06fb2233 Mon Sep 17 00:00:00 2001 From: Aleksey Veresov Date: Wed, 3 Jul 2024 11:04:38 +0200 Subject: [PATCH 7/8] Ruff client/external.py --- python/hopsworks/client/external.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/hopsworks/client/external.py b/python/hopsworks/client/external.py index d88beb5c5..d0a277e71 100644 --- a/python/hopsworks/client/external.py +++ b/python/hopsworks/client/external.py @@ -14,11 +14,11 @@ # limitations under the License. # -import os import base64 -import requests +import os -from hopsworks.client import base, auth, exceptions +import requests +from hopsworks.client import auth, base, exceptions class Client(base.Client): From 76c31afb12babe1cae12fc4e4e0743d3f37fa400 Mon Sep 17 00:00:00 2001 From: Aleksey Veresov Date: Tue, 16 Jul 2024 08:51:52 +0200 Subject: [PATCH 8/8] Refactor variable_api --- python/hopsworks/core/variable_api.py | 40 ++++++--------------------- 1 file changed, 8 insertions(+), 32 deletions(-) diff --git a/python/hopsworks/core/variable_api.py b/python/hopsworks/core/variable_api.py index 301657346..d4e8d188c 100644 --- a/python/hopsworks/core/variable_api.py +++ b/python/hopsworks/core/variable_api.py @@ -56,18 +56,18 @@ def get_version(self, software: str) -> Optional[str]: """ _client = client.get_instance() - path_params = [ - "variables", - "versions", - ] + path_params = ["variables", "versions"] resp = _client._send_request("GET", path_params) + for entry in resp: if entry["software"] == software: return entry["version"] return None - def parse_major_and_minor(self, backend_version: str) -> Tuple[Optional[str], Optional[str]]: + def parse_major_and_minor( + self, backend_version: str + ) -> Tuple[Optional[str], Optional[str]]: """Extract major and minor version from full version. # Arguments @@ -91,15 +91,7 @@ def get_flyingduck_enabled(self) -> bool: # Raises `RestAPIError`: If unable to obtain the flag's value. """ - - _client = client.get_instance() - path_params = [ - "variables", - "enable_flyingduck", - ] - - resp = _client._send_request("GET", path_params) - return resp["successMessage"] == "true" + return self.get_variable("enable_flyingduck") == "true" def get_loadbalancer_external_domain(self) -> str: """Get domain of external loadbalancer. @@ -107,16 +99,8 @@ def get_loadbalancer_external_domain(self) -> str: # Returns `str`: The domain of external loadbalancer, if it is set up, otherwise empty string `""`. """ - - _client = client.get_instance() - path_params = [ - "variables", - "loadbalancer_external_domain", - ] - try: - resp = _client._send_request("GET", path_params) - return resp["successMessage"] + return self.get_variable("loadbalancer_external_domain") except RestAPIError: return "" @@ -126,15 +110,7 @@ def get_service_discovery_domain(self) -> str: # Returns `str`: The domain of service discovery server, if it is set up, otherwise empty string `""`. """ - - _client = client.get_instance() - path_params = [ - "variables", - "service_discovery_domain", - ] - try: - resp = _client._send_request("GET", path_params) - return resp["successMessage"] + return self.get_variable("service_discovery_domain") except RestAPIError: return ""