Skip to content

Commit

Permalink
Feature/SK-344 | Add compute package registry (#495)
Browse files Browse the repository at this point in the history 
* Started on new set compute package

* set compute package updated

* list compute packages added

* added properties to /get_package

* name and desription added to compute package

* simplyfied compute package object response (get_package)

* include name and desc in compute package list response

* include id in result for compute packages

* added id property to compute packages to enable tracking active easier

* set active compute package

* list_models can include active property

* select correct file name

* set active model

* set CURRENT model

* Removed comment

* lint fix

* Added api test for list_compute_packages
  • Loading branch information
@niklastheman
niklastheman authored Jan 4, 2024
1 parent 2cb4c13 commit c4b3c4e
Show file tree
Hide file tree
Showing 6 changed files with 394 additions and 74 deletions.
14 changes: 12 additions & 2 deletions fedn/fedn/network/api/client.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,7 @@ def get_session(self, session_id):
response = requests.get(self._get_url(f'get_session?session_id={session_id}'), self.verify)
return response.json()

def set_package(self, path, helper):
def set_package(self, path: str, helper: str, name: str = None, description: str = None):
""" Set the compute package in the statestore.
:param path: The file path of the compute package to set.
Expand All @@ -180,7 +180,8 @@ def set_package(self, path, helper):
:rtype: dict
"""
with open(path, 'rb') as file:
response = requests.post(self._get_url('set_package'), files={'file': file}, data={'helper': helper}, verify=self.verify)
response = requests.post(self._get_url('set_package'), files={'file': file}, data={
'helper': helper, 'name': name, 'description': description}, verify=self.verify)
return response.json()

def get_package(self):
Expand All @@ -192,6 +193,15 @@ def get_package(self):
response = requests.get(self._get_url('get_package'), verify=self.verify)
return response.json()

def list_compute_packages(self):
""" Get all compute packages from the statestore.
:return: All compute packages with info.
:rtype: dict
"""
response = requests.get(self._get_url('list_compute_packages'), verify=self.verify)
return response.json()

def download_package(self, path):
""" Download the compute package.
Expand Down
236 changes: 193 additions & 43 deletions fedn/fedn/network/api/interface.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,14 +42,15 @@ def _allowed_file_extension(
:param filename: The filename to check.
:type filename: str
:return: True if file extension is allowed, else False.
:rtype: bool
:return: True and extension str if file extension is allowed, else False and None.
:rtype: Tuple (bool, str)
"""
if "." in filename:
extension = filename.rsplit(".", 1)[1].lower()
if extension in ALLOWED_EXTENSIONS:
return (True, extension)

return (
"." in filename
and filename.rsplit(".", 1)[1].lower() in ALLOWED_EXTENSIONS
)
return (False, None)

def get_clients(self, limit=None, skip=None, status=False):
"""Get all clients from the statestore.
Expand Down Expand Up @@ -169,7 +170,24 @@ def get_session(self, session_id):
payload[id] = info
return jsonify(payload)

def set_compute_package(self, file, helper_type):
def set_active_compute_package(self, id: str):

success = self.statestore.set_active_compute_package(id)

if not success:
return (
jsonify(
{
"success": False,
"message": "Failed to set compute package.",
}
),
400,
)

return jsonify({"success": True, "message": "Compute package set."})

def set_compute_package(self, file, helper_type: str, name: str = None, description: str = None):
"""Set the compute package in the statestore.
:param file: The compute package to set.
Expand All @@ -178,31 +196,54 @@ def set_compute_package(self, file, helper_type):
:rtype: :class:`flask.Response`
"""

if file and self._allowed_file_extension(file.filename):
filename = secure_filename(file.filename)
# TODO: make configurable, perhaps in config.py or package.py
file_path = os.path.join("/app/client/package/", filename)
file.save(file_path)
if (
self.control.state() == ReducerState.instructing
or self.control.state() == ReducerState.monitoring
):
return (
jsonify(
{
"success": False,
"message": "Reducer is in instructing or monitoring state."
"Cannot set compute package.",
}
),
400,
)

if (
self.control.state() == ReducerState.instructing
or self.control.state() == ReducerState.monitoring
):
return (
jsonify(
{
"success": False,
"message": "Reducer is in instructing or monitoring state."
"Cannot set compute package.",
}
),
400,
)
if file is None:
return (
jsonify(
{
"success": False,
"message": "No file provided.",
}
),
404,
)

success, extension = self._allowed_file_extension(file.filename)

if not success:
return (
jsonify(
{
"success": False,
"message": f"File extension {extension} not allowed.",
}
),
404,
)

file_name = file.filename
storage_file_name = secure_filename(f"{str(uuid.uuid4())}.{extension}")

file_path = os.path.join("/app/client/package/", storage_file_name)
file.save(file_path)

self.control.set_compute_package(filename, file_path)
self.statestore.set_helper(helper_type)
self.control.set_compute_package(storage_file_name, file_path)
success = self.statestore.set_compute_package(file_name, storage_file_name, helper_type, name, description)

success = self.statestore.set_compute_package(filename)
if not success:
return (
jsonify(
Expand All @@ -213,6 +254,7 @@ def set_compute_package(self, file, helper_type):
),
400,
)

return jsonify({"success": True, "message": "Compute package set."})

def _get_compute_package_name(self):
Expand All @@ -227,7 +269,7 @@ def _get_compute_package_name(self):
return None, message
else:
try:
name = package_objects["filename"]
name = package_objects["storage_file_name"]
except KeyError as e:
message = "No compute package found. Key error."
print(e)
Expand All @@ -240,22 +282,87 @@ def get_compute_package(self):
:return: The compute package as a json response.
:rtype: :class:`flask.Response`
"""
package_object = self.statestore.get_compute_package()
if package_object is None:
result = self.statestore.get_compute_package()
if result is None:
return (
jsonify(
{"success": False, "message": "No compute package found."}
),
404,
)
payload = {}
id = str(package_object["_id"])
info = {
"filename": package_object["filename"],
"helper": package_object["helper"],

obj = {
"id": result["id"] if "id" in result else "",
"file_name": result["file_name"],
"helper": result["helper"],
"committed_at": result["committed_at"],
"storage_file_name": result["storage_file_name"] if "storage_file_name" in result else "",
"name": result["name"] if "name" in result else "",
"description": result["description"] if "description" in result else "",
}
payload[id] = info
return jsonify(payload)

return jsonify(obj)

def list_compute_packages(self, limit: str = None, skip: str = None, include_active: str = None):
"""Get paginated list of compute packages from the statestore.
:return: All compute packages as a json response.
:rtype: :class:`flask.Response`
"""

if limit is not None and skip is not None:
limit = int(limit)
skip = int(skip)

include_active: bool = include_active == "true"

result = self.statestore.list_compute_packages(limit, skip)
if result is None:
return (
jsonify(
{"success": False, "message": "No compute packages found."}
),
404,
)

active_package_id: str = None

if include_active:
active_package = self.statestore.get_compute_package()

if active_package is not None:
active_package_id = active_package["id"] if "id" in active_package else ""

if include_active:
arr = [
{
"id": element["id"] if "id" in element else "",
"file_name": element["file_name"],
"helper": element["helper"],
"committed_at": element["committed_at"],
"storage_file_name": element["storage_file_name"] if "storage_file_name" in element else "",
"name": element["name"] if "name" in element else "",
"description": element["description"] if "description" in element else "",
"active": "id" in element and element["id"] == active_package_id,
}
for element in result["result"]
]
else:
arr = [
{
"id": element["id"] if "id" in element else "",
"file_name": element["file_name"],
"helper": element["helper"],
"committed_at": element["committed_at"],
"storage_file_name": element["storage_file_name"] if "storage_file_name" in element else "",
"name": element["name"] if "name" in element else "",
"description": element["description"] if "description" in element else "",
}
for element in result["result"]
]

result = {"result": arr, "count": result["count"]}
return jsonify(result)

def download_compute_package(self, name):
"""Download the compute package.
Expand Down Expand Up @@ -595,7 +702,30 @@ def get_latest_model(self):
{"success": False, "message": "No initial model set."}
)

def get_models(self, session_id=None, limit=None, skip=None):
def set_current_model(self, model_id: str):
"""Set the active model in the statestore.
:param model_id: The model id to set.
:type model_id: str
:return: A json response with success or failure message.
:rtype: :class:`flask.Response`
"""
success = self.statestore.set_current_model(model_id)

if not success:
return (
jsonify(
{
"success": False,
"message": "Failed to set active model.",
}
),
400,
)

return jsonify({"success": True, "message": "Active model set."})

def get_models(self, session_id: str = None, limit: str = None, skip: str = None, include_active: str = None):
result = self.statestore.list_models(session_id, limit, skip)

if result is None:
Expand All @@ -604,10 +734,30 @@ def get_models(self, session_id=None, limit=None, skip=None):
404,
)

arr = []
include_active: bool = include_active == "true"

for model in result["result"]:
arr.append(model)
if include_active:

latest_model = self.statestore.get_latest_model()

arr = [
{
"committed_at": element["committed_at"],
"model": element["model"],
"session_id": element["session_id"],
"active": element["model"] == latest_model,
}
for element in result["result"]
]
else:
arr = [
{
"committed_at": element["committed_at"],
"model": element["model"],
"session_id": element["session_id"],
}
for element in result["result"]
]

result = {"result": arr, "count": result["count"]}

Expand Down
Loading

0 comments on commit c4b3c4e

Please sign in to comment.