From 8d292509e8226e9d9cf3ee8d3ebe88ee9185d242 Mon Sep 17 00:00:00 2001 From: newbyvector Date: Thu, 30 Nov 2023 21:43:15 +0800 Subject: [PATCH 1/3] feat: use dataset handle sqlite data store --- requirements-dev.lock | 7 ++++++- requirements.lock | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/requirements-dev.lock b/requirements-dev.lock index 3d38e628..4385db92 100644 --- a/requirements-dev.lock +++ b/requirements-dev.lock @@ -212,7 +212,7 @@ snakeviz==2.2.0 sniffio==1.3.0 soupsieve==2.5 spu==0.6.0b0 -sqlalchemy==1.4.50 +sqlparse==0.4.4 stack-data==0.6.3 starlette==0.27.0 statsmodels==0.14.0 @@ -251,6 +251,11 @@ widgetsnbextension==4.0.9 wrapt==1.16.0 yarl==1.9.3 zipp==3.17.0 +dataset==1.6.2 +alembic==1.12.1 +sqlalchemy==1.4.50 +banal==1.0.6 +mako==1.3.0 # The following packages are considered to be unsafe in a requirements file: pip==23.3.1 setuptools==69.0.2 diff --git a/requirements.lock b/requirements.lock index 692180c9..72686940 100644 --- a/requirements.lock +++ b/requirements.lock @@ -132,7 +132,7 @@ six==1.16.0 smart-open==6.4.0 sniffio==1.3.0 soupsieve==2.5 -sqlalchemy==1.4.50 +sqlparse==0.4.4 stack-data==0.6.3 terminado==0.18.0 tinycss2==1.2.1 @@ -153,3 +153,8 @@ widgetsnbextension==4.0.9 wrapt==1.16.0 yarl==1.9.3 zipp==3.17.0 +dataset==1.6.2 +alembic==1.12.1 +sqlalchemy==1.4.50 +banal==1.0.6 +mako==1.3.0 From c7b5469b5932b3233197d066e66acfe63f07d7f6 Mon Sep 17 00:00:00 2001 From: newbyvector Date: Fri, 1 Dec 2023 10:42:15 +0800 Subject: [PATCH 2/3] chore: update requirements --- requirements-dev.lock | 7 +------ requirements.lock | 7 +------ 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/requirements-dev.lock b/requirements-dev.lock index 4385db92..3d38e628 100644 --- a/requirements-dev.lock +++ b/requirements-dev.lock @@ -212,7 +212,7 @@ snakeviz==2.2.0 sniffio==1.3.0 soupsieve==2.5 spu==0.6.0b0 -sqlparse==0.4.4 +sqlalchemy==1.4.50 stack-data==0.6.3 starlette==0.27.0 statsmodels==0.14.0 @@ -251,11 +251,6 @@ widgetsnbextension==4.0.9 wrapt==1.16.0 yarl==1.9.3 zipp==3.17.0 -dataset==1.6.2 -alembic==1.12.1 -sqlalchemy==1.4.50 -banal==1.0.6 -mako==1.3.0 # The following packages are considered to be unsafe in a requirements file: pip==23.3.1 setuptools==69.0.2 diff --git a/requirements.lock b/requirements.lock index 72686940..692180c9 100644 --- a/requirements.lock +++ b/requirements.lock @@ -132,7 +132,7 @@ six==1.16.0 smart-open==6.4.0 sniffio==1.3.0 soupsieve==2.5 -sqlparse==0.4.4 +sqlalchemy==1.4.50 stack-data==0.6.3 terminado==0.18.0 tinycss2==1.2.1 @@ -153,8 +153,3 @@ widgetsnbextension==4.0.9 wrapt==1.16.0 yarl==1.9.3 zipp==3.17.0 -dataset==1.6.2 -alembic==1.12.1 -sqlalchemy==1.4.50 -banal==1.0.6 -mako==1.3.0 From 07cea6eee1e501ba5a1f1029e66a3623249804c4 Mon Sep 17 00:00:00 2001 From: newbyvector Date: Tue, 5 Dec 2023 16:10:28 +0800 Subject: [PATCH 3/3] feat: add broker api --- .../src/modules/server/server-manager.ts | 2 +- .../server/services/broker/__init__.py | 0 .../server/services/broker/handlers.py | 44 +++ .../server/services/broker/manager.py | 316 ++++++++++++++++++ .../server/services/nodes/handlers.py | 2 +- .../nodes/{nodemanager.py => manager.py} | 0 6 files changed, 362 insertions(+), 2 deletions(-) create mode 100644 pyprojects/secretnote/src/secretnote/server/services/broker/__init__.py create mode 100644 pyprojects/secretnote/src/secretnote/server/services/broker/handlers.py create mode 100644 pyprojects/secretnote/src/secretnote/server/services/broker/manager.py rename pyprojects/secretnote/src/secretnote/server/services/nodes/{nodemanager.py => manager.py} (100%) diff --git a/packages/secretnote/src/modules/server/server-manager.ts b/packages/secretnote/src/modules/server/server-manager.ts index 3756aa2c..de61410f 100644 --- a/packages/secretnote/src/modules/server/server-manager.ts +++ b/packages/secretnote/src/modules/server/server-manager.ts @@ -1,4 +1,4 @@ -import { ServerConnection, URL } from '@difizen/libro-jupyter'; +import { ServerConnection } from '@difizen/libro-jupyter'; import { Emitter, inject, prop, singleton } from '@difizen/mana-app'; import { RequestService } from '@/utils'; diff --git a/pyprojects/secretnote/src/secretnote/server/services/broker/__init__.py b/pyprojects/secretnote/src/secretnote/server/services/broker/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pyprojects/secretnote/src/secretnote/server/services/broker/handlers.py b/pyprojects/secretnote/src/secretnote/server/services/broker/handlers.py new file mode 100644 index 00000000..46259ffa --- /dev/null +++ b/pyprojects/secretnote/src/secretnote/server/services/broker/handlers.py @@ -0,0 +1,44 @@ +import json +from typing import List, Tuple, Type + +from jupyter_client.jsonutil import json_default +from jupyter_server.base.handlers import APIHandler, JupyterHandler +from tornado import web + +from .manager import node_manager + + +class NodeHandler(APIHandler): + @web.authenticated + async def get(self, node_id): + node = node_manager.get_node(id=node_id) + if node is None: + raise web.HTTPError(404, "node not found.") + self.finish(json.dumps(node, default=json_default)) + + @web.authenticated + async def patch(self, node_id): + model = self.get_json_body() + + if model is None: + raise web.HTTPError(400, "no request body provided.") + + try: + node_manager.update_node(node_id, model) + except Exception as e: + raise web.HTTPError(400, str(e)) # noqa: B904 + + self.finish(json.dumps(model, default=json_default)) + + @web.authenticated + async def delete(self, node_id): + node_manager.remove_node(node_id) + self.set_status(204) + self.finish() + + +_node_id_regex = r"(?P\d+)" + +broker_handlers: List[Tuple[str, Type[JupyterHandler]]] = [ + (r"/api/broker", NodeHandler), +] diff --git a/pyprojects/secretnote/src/secretnote/server/services/broker/manager.py b/pyprojects/secretnote/src/secretnote/server/services/broker/manager.py new file mode 100644 index 00000000..e195e8e4 --- /dev/null +++ b/pyprojects/secretnote/src/secretnote/server/services/broker/manager.py @@ -0,0 +1,316 @@ +import json +from typing import Any, List + +from tornado import httpclient + +BROKER_SERVICE_PATH = { + "query": "/intra/query", + "submit_query": "/intra/query/submit", + "fetch_result": "/intra/query/fetch", + "create_project": "/intra/project/create", + "list_projects": "/intra/project/list", + "invite_member": "/intra/member/invite", + "list_invitations": "/intra/invitation/list", + "process_invitation": "/intra/invitation/process", + "create_table": "/intra/table/create", + "list_tables": "/intra/table/list", + "drop_table": "/intra/table/drop", + "grant_ccl": "/intra/ccl/grant", + "revoke_ccl": "/intra/ccl/revoke", + "show_ccl": "/intra/ccl/show", +} + + +class BrokerManager: + global_project_id = "secretnote" + + def __init__(self): + pass + + def request(self, url: str, method="GET", body=None): + if body is None: + body = {} + http_client = httpclient.HTTPClient() + http_request_body = json.dumps(body) + + try: + http_request = httpclient.HTTPRequest( + url=url, + method=method, + body=http_request_body, + headers={"Content-Type": "application/json"}, + ) + response = http_client.fetch(http_request) + return json.loads(response.body) + except httpclient.HTTPError as e: + # HTTPError is raised for non-200 responses; the response + # can be found in e.response. + print("Error: " + str(e)) + except Exception as e: + # Other errors are possible, such as IOError. + print("Error: " + str(e)) + http_client.close() + + def get_request_status(self, response): + code = 0 + message = "" + status = response.get("status", None) + + if status is not None: + code = status.get("code", 0) + message = status.get("message", "") + else: + message = "no status found." + code = 500 + + return code, message + + def create_project(self, project_id: str, address: str): + url = f"{address}{BROKER_SERVICE_PATH['create_project']}" + body = { + "project_id": project_id, + "conf": {"spu_runtime_cfg": {"protocol": "SEMI2K", "field": "FM64"}}, + } + response = self.request( + url=url, + method="POST", + body=body, + ) + code, message = self.get_request_status(response) + + if code != 0: + raise Exception(message) + + return response.get("project_id", "") + + def get_project_list(self, address: str): + url = f"{address}{BROKER_SERVICE_PATH['list_projects']}" + body = {"ids": []} + response = self.request( + url=url, + method="POST", + body=body, + ) + code, message = self.get_request_status(response) + + if code != 0: + raise Exception(message) + + return response.get("projects", []) + + def invite_member(self, invitee: str, address: str, project_id=global_project_id): + url = f"{address}{BROKER_SERVICE_PATH['invite_member']}" + body = { + "project_id": project_id, + "invitee": invitee, + "postscript": "", + "method": "PUSH", + } + response = self.request( + url=url, + method="POST", + body=body, + ) + code, message = self.get_request_status(response) + + if code != 0: + raise Exception(message) + + return response + + def get_invite_list(self, address: str): + url = f"{address}{BROKER_SERVICE_PATH['list_invitations']}" + body = {} + response = self.request( + url=url, + method="POST", + body=body, + ) + code, message = self.get_request_status(response) + + if code != 0: + raise Exception(message) + + return response.get("invitations", []) + + def process_invite(self, invitation_id: str, respond: str, address: str): + url = f"{address}{BROKER_SERVICE_PATH['process_invitation']}" + body = { + "invitation_id": invitation_id, + "respond": respond, + "respond_comment": "", + } + response = self.request( + url=url, + method="POST", + body=body, + ) + code, message = self.get_request_status(response) + + if code != 0: + raise Exception(message) + + def create_table( + self, + table_name: str, + ref_table: str, + columns: List[Any], + address: str, + project_id=global_project_id, + ): + url = f"{address}{BROKER_SERVICE_PATH['create_table']}" + body = { + "project_id": project_id, + "table_name": table_name, + "ref_table": ref_table, + "db_type": "mysql", + "columns": columns, + } + response = self.request( + url=url, + method="POST", + body=body, + ) + code, message = self.get_request_status(response) + + if code != 0: + raise Exception(message) + + def get_table_list(self, address: str, project_id=global_project_id): + url = f"{address}{BROKER_SERVICE_PATH['list_tables']}" + body = {"project_id": project_id, "names": []} + response = self.request( + url=url, + method="POST", + body=body, + ) + code, message = self.get_request_status(response) + + if code != 0: + raise Exception(message) + + return response.get("tables", []) + + def delete_table(self, table_name: str, address: str, project_id=global_project_id): + url = f"{address}{BROKER_SERVICE_PATH['drop_table']}" + body = {"project_id": project_id, "table_name": table_name} + response = self.request( + url=url, + method="POST", + body=body, + ) + code, message = self.get_request_status(response) + + if code != 0: + raise Exception(message) + + def grant_ccl( + self, ccl_list: List[Any], address: str, project_id=global_project_id + ): + url = f"{address}{BROKER_SERVICE_PATH['grant_ccl']}" + body = { + "project_id": project_id, + "column_control_list": ccl_list, + } + response = self.request( + url=url, + method="POST", + body=body, + ) + code, message = self.get_request_status(response) + + if code != 0: + raise Exception(message) + + def revoke_ccl( + self, ccl_list: List[Any], address: str, project_id=global_project_id + ): + url = f"{address}{BROKER_SERVICE_PATH['revoke_ccl']}" + body = { + "project_id": project_id, + "column_control_list": ccl_list, + } + response = self.request( + url=url, + method="POST", + body=body, + ) + code, message = self.get_request_status(response) + + if code != 0: + raise Exception(message) + + def get_ccl_list( + self, + party: List[str], + table_name: List[str], + address: str, + project_id=global_project_id, + ): + url = f"{address}{BROKER_SERVICE_PATH['show_ccl']}" + body = {"project_id": project_id, "tables": table_name, "dest_parties": party} + response = self.request( + url=url, + method="POST", + body=body, + ) + code, message = self.get_request_status(response) + + if code != 0: + raise Exception(message) + + return response.get("column_control_list", []) + + def query(self, query: str, address: str, project_id=global_project_id): + url = f"{address}{BROKER_SERVICE_PATH['query']}" + body = { + "project_id": project_id, + "query": query, + } + response = self.request( + url=url, + method="POST", + body=body, + ) + code, message = self.get_request_status(response) + + if code != 0: + raise Exception(message) + + return response.get("out_columns", []) + + def create_query_job(self, query: str, address: str, project_id=global_project_id): + url = f"{address}{BROKER_SERVICE_PATH['submit_query']}" + body = { + "project_id": project_id, + "query": query, + } + response = self.request( + url=url, + method="POST", + body=body, + ) + code, message = self.get_request_status(response) + + if code != 0: + raise Exception(message) + + return response.get("job_id", "") + + def get_job_result(self, job_id: str, address: str): + url = f"{address}{BROKER_SERVICE_PATH['fetch_result']}" + body = {"job_id": job_id} + response = self.request( + url=url, + method="POST", + body=body, + ) + code, message = self.get_request_status(response) + + if code != 0: + raise Exception(message) + + return response.get("out_columns", []) + + +broker_manager = BrokerManager() diff --git a/pyprojects/secretnote/src/secretnote/server/services/nodes/handlers.py b/pyprojects/secretnote/src/secretnote/server/services/nodes/handlers.py index a03d69b1..974694ac 100644 --- a/pyprojects/secretnote/src/secretnote/server/services/nodes/handlers.py +++ b/pyprojects/secretnote/src/secretnote/server/services/nodes/handlers.py @@ -5,7 +5,7 @@ from jupyter_server.base.handlers import APIHandler, JupyterHandler from tornado import web -from .nodemanager import node_manager +from .manager import node_manager class NodeRootHandler(APIHandler): diff --git a/pyprojects/secretnote/src/secretnote/server/services/nodes/nodemanager.py b/pyprojects/secretnote/src/secretnote/server/services/nodes/manager.py similarity index 100% rename from pyprojects/secretnote/src/secretnote/server/services/nodes/nodemanager.py rename to pyprojects/secretnote/src/secretnote/server/services/nodes/manager.py