From 2085325eb038523cf13bdb51707da92855173f5b Mon Sep 17 00:00:00 2001 From: Peter Chang Date: Wed, 6 Nov 2024 12:31:02 +0000 Subject: [PATCH] Reorganise server files (#178) Move optional client endpoint to /, bundle example client in package, update documentation and setup.py with scripts --- DEVELOP.md | 58 ++++++++++++++++++ README.md | 36 ++++------- client/example/vite.config.ts | 3 + server/davidia/generate_openapi.py | 4 +- server/davidia/main.py | 79 +++++++++++++++---------- server/davidia/models/messages.py | 1 - server/davidia/models/parameters.py | 4 -- server/davidia/server/fastapi_utils.py | 1 + server/davidia/server/plotserver.py | 6 +- server/davidia/tests/test_api.py | 10 ++-- server/demos/__init__.py | 0 server/demos/__main__.py | 3 + server/{davidia => }/demos/benchmark.py | 0 server/{davidia => }/demos/simple.py | 20 +++++++ server/setup.py | 13 ++++ 15 files changed, 172 insertions(+), 66 deletions(-) create mode 100644 DEVELOP.md create mode 100644 server/demos/__init__.py create mode 100644 server/demos/__main__.py rename server/{davidia => }/demos/benchmark.py (100%) rename server/{davidia => }/demos/simple.py (94%) diff --git a/DEVELOP.md b/DEVELOP.md new file mode 100644 index 00000000..19421f7a --- /dev/null +++ b/DEVELOP.md @@ -0,0 +1,58 @@ +# Davidia development + +To get started with Davidia development, create a conda environment called `davidia` + +### `conda env create --file environment.yml` + +Activate it: + +### `conda activate davidia` + +Install Typescript dependencies (read pnpm's [installation guide](https://pnpm.io/installation), if needed) + +### `pnpm install` + +Build web client + +### `pnpm build` + +## Running Python plot server + +From the top level of the repository, you can run: + +### `cd server && uvicorn --factory davidia.main:create_app` or `PYTHONPATH=server python server/davidia/main.py -c` + +Open [localhost:8000](http://localhost:8000) to view it in the browser. Now test plot server with, + +### `PYTHONPATH=server python server/demos/simple.py` + +## Benchmarking the plot client + +Set the environment variable `DVD_BENCHMARK` as `on` or add a `-b` argument: + +### `cd server && DVD_BENCHMARK=on uvicorn --factory davidia.main:create_app` +### `DVD_BENCHMARK=on PYTHONPATH=server python server/davidia/main.py` +### `PYTHONPATH=server python server/davidia/main.py -c -b` + +Run the script to trigger benchmarks: + +### `PYTHONPATH=server python server/demos/benchmark.py` + +See its builtin help using the `-h` argument. + +## Storybook + +View the Storybook [here](https://diamondlightsource.github.io/davidia). + +To build and run the Storybook locally: + +### `pnpm build:storybook` +### `pnpm start:storybook` + +## Documentation + +View the documentation [here](https://diamondlightsource.github.io/davidia/typedocs/index.html). + +## API documentation + +View the API documentation [here](https://diamondlightsource.github.io/davidia/?path=/docs/api-documentation--docs). diff --git a/README.md b/README.md index 62bcfd6c..f9d03c68 100644 --- a/README.md +++ b/README.md @@ -1,41 +1,36 @@ # Davidia -Create a conda environment called `davidia` +Davidia comprises two parts: a plot server and a set of React components including a connected plot. The plot server has a REST api that allows clients to visualize data in connected plots in React applications. -### `conda env create --file environment.yml` +## Demonstrating Davidia -Activate it: +Install the Davidia Python package with -### `conda activate davidia` +### `pip install davidia[all]` -Install Typescript dependencies (read pnpm's [installation guide](https://pnpm.io/installation), if needed) +To start the demo, run -### `pnpm install` +### `dvd-demo` -Build web client +This starts the plot server, opens a browser window and runs a demo script that shows different plots. -### `pnpm build` +## Running Python plot server (with bundled example client) -## Running Python plot server +### `dvd-server -c` -From the top level of the repository, you can run: +Open [localhost:8000](http://localhost:8000) to view it in the browser. Now test plot server with, -### `cd server && uvicorn --factory davidia.main:create_app` or `PYTHONPATH=server python server/davidia/main.py -c` - -Open [localhost:8000/client](http://localhost:8000/client) to view it in the browser. Now test plot server with, - -### `PYTHONPATH=server python server/davidia/simple.py` +### `python -m davidia.demos` ## Benchmarking the plot client Set the environment variable `DVD_BENCHMARK` as `on` or add a `-b` argument: -### `DVD_BENCHMARK=on PYTHONPATH=server python server/davidia/main.py` -### `PYTHONPATH=server python server/davidia/main.py -c -b` +### `dvd-server -c -b` Run the script to trigger benchmarks: -### `PYTHONPATH=server python server/davidia/demos/benchmark.py` +### `dvd-benchmark` See its builtin help using the `-h` argument. @@ -43,11 +38,6 @@ See its builtin help using the `-h` argument. View the Storybook [here](https://diamondlightsource.github.io/davidia). -To build and run the Storybook locally: - -### `pnpm build:storybook` -### `pnpm start:storybook` - ## Documentation View the documentation [here](https://diamondlightsource.github.io/davidia/typedocs/index.html). diff --git a/client/example/vite.config.ts b/client/example/vite.config.ts index af912f3e..9abcc8f1 100644 --- a/client/example/vite.config.ts +++ b/client/example/vite.config.ts @@ -8,4 +8,7 @@ export default defineConfig({ define: { global: 'window', // this fixes global is not defined }, + build: { + outDir: "../../server/davidia/client", + }, }); diff --git a/server/davidia/generate_openapi.py b/server/davidia/generate_openapi.py index b4f98f19..614b96fe 100644 --- a/server/davidia/generate_openapi.py +++ b/server/davidia/generate_openapi.py @@ -1,9 +1,9 @@ import json from pathlib import Path -from main import _create_bare_app +from davidia.main import _create_bare_app -app, _ = _create_bare_app() +app = _create_bare_app() public_path = Path("storybook/public/openapi.json") public_path.parent.mkdir(exist_ok=True) diff --git a/server/davidia/main.py b/server/davidia/main.py index 3178211c..71ad1601 100644 --- a/server/davidia/main.py +++ b/server/davidia/main.py @@ -23,7 +23,7 @@ logger = logging.getLogger("main") -def _create_bare_app(): +def _create_bare_app(add_benchmark=False): app = FastAPI() def customize_openapi(): @@ -88,7 +88,16 @@ def get_plot_ids() -> list[str]: async def get_regions(plot_id: str) -> list[AnySelection]: return await ps.get_regions(plot_id) - return app, ps + if add_benchmark: + + @app.post("/benchmark/{plot_id}") + async def benchmark(plot_id: str, params: BenchmarkParams) -> str: + """ + Benchmark davidia + """ + return await ps.benchmark(plot_id, params) + + return app def _setup_logger(): @@ -100,36 +109,36 @@ def _setup_logger(): logger.setLevel(logging.DEBUG) -def add_benchmark_endpoint(app, ps): - @app.post("/benchmark/{plot_id}") - async def benchmark(plot_id: str, params: BenchmarkParams) -> str: - """ - Benchmark davidia - """ - return await ps.benchmark(plot_id, params) - - def add_client_endpoint(app, client_path): index_path = client_path / "index.html" if index_path.is_file() and os.access(index_path, os.R_OK): logger.debug("Adding /client endpoint which uses %s", client_path) - app.mount( - "/client", StaticFiles(directory=client_path, html=True), name="webui" - ) + app.mount("/", StaticFiles(directory=client_path, html=True), name="webui") else: logger.warning( - "%s not readable so `/client` endpoint will not be available", index_path + "%s not readable so '/' endpoint will not be available", index_path ) -CLIENT_BUILD_DIR = str( - pathlib.Path(__file__).parent.parent.parent.joinpath("client/example/dist") -) +def find_client_build(): + dvd_pkg_dir = pathlib.Path(__file__).parent + # example client packaged or developing in source tree + client_dir = dvd_pkg_dir / "client" + if client_dir.is_dir(): + return client_dir + logger.warning( + "Client directory not found in %s (if developing, then build example app)", + dvd_pkg_dir, + ) + + +CLIENT_BUILD_PATH = find_client_build() def create_parser(): from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, SUPPRESS + CLIENT_BUILD_DIR = str(CLIENT_BUILD_PATH) parser = ArgumentParser( description="Davidia plot server", formatter_class=ArgumentDefaultsHelpFormatter ) @@ -144,27 +153,37 @@ def create_parser(): const=CLIENT_BUILD_DIR, default=SUPPRESS, ) + parser.add_argument( + "-p", "--port", help="Set the port number for server", type=int, default=8000 + ) return parser -def create_app(client_pathname=CLIENT_BUILD_DIR, benchmark=False): +def create_app(client_path=CLIENT_BUILD_PATH, benchmark=False): _setup_logger() - app, ps = _create_bare_app() - if client_pathname: - client_path = pathlib.Path(client_pathname) + app = _create_bare_app( + benchmark or os.getenv("DVD_BENCHMARK", "off").lower() == "on" + ) + if client_path: if client_path.is_dir(): add_client_endpoint(app, client_path) - - if benchmark: - add_benchmark_endpoint(app, ps) return app -if __name__ == "__main__": +def run_app(client_path=CLIENT_BUILD_PATH, benchmark=False, port=8000): + app = create_app(client_path=client_path, benchmark=benchmark) + uvicorn.run(app, host="0.0.0.0", port=port, log_level="info", access_log=False) + + +def main(): args = create_parser().parse_args() - app = create_app( - client_pathname=getattr(args, "client", None), - benchmark=args.benchmark or os.getenv("DVD_BENCHMARK", "off").lower() == "on", + client_path = getattr(args, "client", None) + run_app( + client_path=pathlib.Path(client_path) if client_path else None, + benchmark=args.benchmark, + port=args.port, ) - uvicorn.run(app, host="0.0.0.0", port=8000, log_level="info", access_log=False) + +if __name__ == "__main__": + main() diff --git a/server/davidia/models/messages.py b/server/davidia/models/messages.py index 2ca6343c..dc89f6d1 100644 --- a/server/davidia/models/messages.py +++ b/server/davidia/models/messages.py @@ -185,7 +185,6 @@ class ImageData(DvDNpModel): ) # need this to prevent any dict validating as all fields have default values - def validate_colour_map(v: ColourMap | str): if isinstance(v, str): return ColourMap[v] diff --git a/server/davidia/models/parameters.py b/server/davidia/models/parameters.py index ad84e207..bd5d1ebf 100644 --- a/server/davidia/models/parameters.py +++ b/server/davidia/models/parameters.py @@ -16,10 +16,6 @@ ), ] -class AutoNameEnum(str, Enum): - @staticmethod - def _generate_next_value_(name, start, count, last_values): - return name class AutoNameEnum(str, Enum): @staticmethod diff --git a/server/davidia/server/fastapi_utils.py b/server/davidia/server/fastapi_utils.py index acb6303c..1169a85d 100644 --- a/server/davidia/server/fastapi_utils.py +++ b/server/davidia/server/fastapi_utils.py @@ -14,6 +14,7 @@ logger = logging.getLogger("main") + def as_model(raw: dict) -> BaseModel | None: for m in ALL_MODELS: try: diff --git a/server/davidia/server/plotserver.py b/server/davidia/server/plotserver.py index 92d50cb4..e07628c8 100644 --- a/server/davidia/server/plotserver.py +++ b/server/davidia/server/plotserver.py @@ -777,7 +777,11 @@ async def handle_client(server: PlotServer, plot_id: str, socket: WebSocket, uui or mtype == MsgType.client_update_line_parameters ): logger.debug( - "Got from %s (%s) from client %s: %s", plot_id, mtype, client.uuid, received_message.params + "Got from %s (%s) from client %s: %s", + plot_id, + mtype, + client.uuid, + received_message.params, ) is_valid = client.uuid == server.baton if is_valid: diff --git a/server/davidia/tests/test_api.py b/server/davidia/tests/test_api.py index 009f4751..31dbfd9e 100644 --- a/server/davidia/tests/test_api.py +++ b/server/davidia/tests/test_api.py @@ -107,7 +107,7 @@ def test_status_ws(): ) msg_2 = plot_msg_2.model_dump(by_alias=True) - app, _ = _create_bare_app() + app = _create_bare_app() with TestClient(app) as client: with client.websocket_connect("/plot/30064551/plot_0") as ws_0: @@ -238,7 +238,7 @@ async def test_get_data(send, receive): plot_id="plot_0", type=MsgType.new_multiline_data, params=[line] ) - app, _ = _create_bare_app() + app = _create_bare_app() async with AsyncClient(app=app, base_url="http://test") as ac: headers = {} @@ -254,7 +254,7 @@ async def test_get_data(send, receive): @pytest.mark.asyncio async def test_clear_data_via_message(): - app, _ = _create_bare_app() + app = _create_bare_app() with TestClient(app) as client: ps = getattr(app, "_plot_server") @@ -304,7 +304,7 @@ async def test_push_points(): "Content-Type": "application/x-msgpack", "Accept": "application/x-msgpack", } - app, _ = _create_bare_app() + app = _create_bare_app() with TestClient(app) as client: with client.websocket_connect("/plot/99a81b01/plot_0"): @@ -368,7 +368,7 @@ async def test_post_test_pydantic(send, receive): array=np.array([-3.75, 10]), original=testa, ) - app, _ = _create_bare_app() + app = _create_bare_app() @app.post("/test_pydantic") @message_unpack diff --git a/server/demos/__init__.py b/server/demos/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/server/demos/__main__.py b/server/demos/__main__.py new file mode 100644 index 00000000..6e72ed3c --- /dev/null +++ b/server/demos/__main__.py @@ -0,0 +1,3 @@ +from .simple import run_all_demos + +run_all_demos() diff --git a/server/davidia/demos/benchmark.py b/server/demos/benchmark.py similarity index 100% rename from server/davidia/demos/benchmark.py rename to server/demos/benchmark.py diff --git a/server/davidia/demos/simple.py b/server/demos/simple.py similarity index 94% rename from server/davidia/demos/simple.py rename to server/demos/simple.py index 7fae2140..48418fa7 100644 --- a/server/davidia/demos/simple.py +++ b/server/demos/simple.py @@ -212,6 +212,7 @@ def run_line_demos(p, wait=3, no_x=False, verbose=False): print("append line 2") sleep(wait) + def run_all_demos(wait=3, repeats=5): """Run all demos in example app which has plot_0 and plot_1 @@ -269,5 +270,24 @@ def run_all_demos(wait=3, repeats=5): clear(f"plot_{p}") +def start_and_run_all_demos(port=8000): + from threading import Thread + from time import sleep + import webbrowser + from davidia.main import run_app + + def browser(): + sleep(2) + webbrowser.open(f"http://localhost:{port}") + + def demo(): + sleep(5) + run_all_demos() + + Thread(target=browser).start() + Thread(target=demo).start() + run_app(port=port) + + if __name__ == "__main__": run_all_demos() diff --git a/server/setup.py b/server/setup.py index 05048490..fd78d343 100644 --- a/server/setup.py +++ b/server/setup.py @@ -36,8 +36,21 @@ def find_readme(): long_description=open(readme_path).read(), long_description_content_type="text/markdown", author_email="dataanalysis@diamond.ac.uk", + package_dir={ + "davidia": "davidia", + "davidia.demos": "demos", + }, package_data={ "davidia.data": ["*.png"], + "davidia.client": ["*.png", "*.ico", "*.html", "*.txt"], + "davidia.client.assets": ["*.css", "*.js"], + }, + entry_points={ + "console_scripts": [ + "dvd-server = davidia.main:main", + "dvd-demo = davidia.demos.simple:start_and_run_all_demos", + "dvd-benchmark = davidia.demos.benchmark:main", + ], }, python_requires=">=3.10", install_requires=[