From 2ac1fc28ba155e96997609bebdfc3857865a0a4a Mon Sep 17 00:00:00 2001 From: Tony Wu Date: Thu, 23 Nov 2023 17:33:02 +0800 Subject: [PATCH] chore: merge and make CI buildable (#5) --- .changeset/blue-crabs-argue.md | 4 +- .changeset/loud-kids-dream.md | 9 + .changeset/pre.json | 6 +- .devcontainer/devcontainer.json | 14 +- .devcontainer/scripts/create-venv.sh | 15 +- .editorconfig | 6 + .eslintrc.js => .eslintrc.cjs | 10 +- .eslintrc.node.js => .eslintrc.node.cjs | 2 +- .github/workflows/ci-docker.yml | 2 +- .github/workflows/ci.yml | 6 +- .github/workflows/release.yml | 2 +- .gitignore | 24 +- .lintstagedrc.js | 3 +- .prettierignore | 1 + .python-version | 1 - .stylelintrc.js | 6 + .vscode/cspell.json | 23 +- .vscode/extensions.json | 4 +- .vscode/settings.json | 128 +- Makefile | 10 - README.md | 1 + docker/ci.Dockerfile | 44 + .../devcontainer.Dockerfile | 35 +- docker/notebook.Dockerfile | 1 - examples/debug.ipynb | 24 + examples/millionaires.ipynb | 138 + examples/psi.ipynb | 54 +- examples/real.ipynb | 89 + .../sf-2pc-local-ray-instances/alice.ipynb | 238 - examples/sf-2pc-local-ray-instances/bob.ipynb | 228 - examples/sim.ipynb | 97 + examples/spu-over-local-network.ipynb | 325 - examples/spu_lr.ipynb | 236 + examples/spu_lr_unrolled.ipynb | 257 + examples/start-ray.sh | 4 +- nx.json | 102 +- package.json | 58 +- packages/repo-utils/.eslintrc.cjs | 2 +- packages/repo-utils/package.json | 10 +- packages/repo-utils/src/cli/index.mts | 8 +- packages/repo-utils/src/tsup.mts | 4 +- packages/repo-utils/tsconfig.json | 2 +- packages/secretnote-ui/.eslintrc.cjs | 17 + packages/secretnote-ui/.storybook/main.ts | 19 + packages/secretnote-ui/.storybook/preview.ts | 15 + packages/secretnote-ui/README.md | 27 + packages/secretnote-ui/package.json | 81 + .../secretnote-ui/src/.openapi-stubs/index.ts | 43 + .../src/.openapi-stubs/models/ArgumentEdge.ts | 12 + .../src/.openapi-stubs/models/DictSnapshot.ts | 19 + .../.openapi-stubs/models/ExecExpression.ts | 18 + .../src/.openapi-stubs/models/Frame.ts | 25 + .../models/FrameInfoSnapshot.ts | 19 + .../.openapi-stubs/models/FrameSnapshot.ts | 22 + .../models/FunctionCheckpoint.ts | 13 + .../src/.openapi-stubs/models/FunctionInfo.ts | 11 + .../src/.openapi-stubs/models/FunctionNode.ts | 18 + .../models/FunctionParameter.ts | 13 + .../models/FunctionSignature.ts | 12 + .../.openapi-stubs/models/FunctionSnapshot.ts | 29 + .../src/.openapi-stubs/models/Graph.ts | 20 + .../models/HTTPValidationError.ts | 11 + .../src/.openapi-stubs/models/ListSnapshot.ts | 19 + .../src/.openapi-stubs/models/LocalObject.ts | 11 + .../.openapi-stubs/models/LocalObjectNode.ts | 15 + .../.openapi-stubs/models/LogicalLocation.ts | 12 + .../.openapi-stubs/models/MoveExpression.ts | 14 + .../src/.openapi-stubs/models/NoneSnapshot.ts | 14 + .../.openapi-stubs/models/ObjectSnapshot.ts | 16 + .../src/.openapi-stubs/models/Reference.ts | 9 + .../.openapi-stubs/models/ReferenceEdge.ts | 11 + .../src/.openapi-stubs/models/ReferenceMap.ts | 9 + .../models/RemoteLocationSnapshot.ts | 18 + .../src/.openapi-stubs/models/RemoteObject.ts | 15 + .../.openapi-stubs/models/RemoteObjectNode.ts | 15 + .../models/RemoteObjectSnapshot.ts | 19 + .../src/.openapi-stubs/models/ReturnEdge.ts | 12 + .../src/.openapi-stubs/models/RevealEdge.ts | 12 + .../.openapi-stubs/models/RevealExpression.ts | 14 + .../src/.openapi-stubs/models/RevealNode.ts | 12 + .../src/.openapi-stubs/models/Semantics.ts | 10 + .../src/.openapi-stubs/models/Timeline.ts | 23 + .../.openapi-stubs/models/TransformEdge.ts | 14 + .../.openapi-stubs/models/ValidationError.ts | 11 + .../.openapi-stubs/models/Visualization.ts | 11 + .../.openapi-stubs/models/_ParameterKind.ts | 9 + .../src/components/DataProvider/context.ts | 7 + .../src/components/DataProvider/index.tsx | 24 + .../src/components/DataProvider/utils.ts | 7 + .../components/ExecutionGraph/colorization.ts | 217 + .../src/components/ExecutionGraph/index.tsx | 308 + .../src/components/ExecutionGraph/shapes.ts | 549 + .../src/components/ExecutionGraph/tooltip.ts | 201 + .../src/components/ExecutionGraph/types.ts | 80 + .../src/components/ExecutionGraph/utils.ts | 224 + .../src/components/Visualization.tsx | 16 + packages/secretnote-ui/src/index.ts | 2 + packages/secretnote-ui/src/render.tsx | 18 + packages/secretnote-ui/src/utils/drawer.ts | 37 + packages/secretnote-ui/src/utils/reify.ts | 169 + packages/secretnote-ui/src/utils/string.ts | 67 + packages/secretnote-ui/src/utils/typing.ts | 1 + packages/secretnote-ui/src/vite-env.d.ts | 1 + packages/secretnote-ui/tsconfig.json | 8 + packages/secretnote-ui/tsconfig.node.json | 4 + packages/secretnote-ui/vite.config.ts | 78 + packages/secretnote/.eslintrc.cjs | 2 +- packages/secretnote/.umirc.ts | 18 +- packages/secretnote/package.json | 35 +- packages/secretnote/public/favicon.svg | 25 + .../secretnote/src/assets/notebook/psi.ipynb | 48 +- .../src/components/log-viewer/index.less | 2 +- packages/secretnote/src/global.ts | 7 + .../secretnote/src/modules/editor/model.ts | 5 +- .../src/modules/integration/service.ts | 5 +- .../src/modules/server/server-manager.ts | 10 +- packages/secretnote/tsconfig.json | 2 - pnpm-lock.yaml | 13563 +++++++++++----- pyproject.toml | 27 +- pyprojects/secretnote/.jupyter/config_dev.py | 40 + pyprojects/secretnote/CHANGELOG.md | 2 +- .../dev-config/jupyter_server_config.py | 1767 -- .../jupyter_server_config.d/libro-server.json | 7 - .../secretnote/libro_server/__init__.py | 9 - pyprojects/secretnote/libro_server/app.py | 73 - pyprojects/secretnote/libro_server/handler.py | 29 - .../libro_server/static/favicon.ico | Bin 4286 -> 0 bytes .../libro_server/template/error.html | 21 - .../libro_server/template/page.html | 20 - .../libro_server/template/secretnote.html | 38 - pyprojects/secretnote/package.json | 19 +- pyprojects/secretnote/proto/buf.gen.yaml | 5 + pyprojects/secretnote/proto/buf.lock | 8 + pyprojects/secretnote/proto/buf.yaml | 9 + .../proto/secretnote/v1/secretnote.proto | 9 + pyprojects/secretnote/pyproject-poetry.toml | 38 - pyprojects/secretnote/pyproject.toml | 37 +- pyprojects/secretnote/ruff.toml | 3 - .../secretnote/scripts/copy_static_files.py | 7 + .../secretnote/scripts/generate_proto.sh | 18 + pyprojects/secretnote/setup.py | 1 - .../src/secretnote/display/__init__.py | 3 + .../secretnote/src/secretnote/display/api.py | 10 + .../secretnote/src/secretnote/display/app.py | 16 + .../src/secretnote/display/core/__init__.py | 0 .../src/secretnote/display/core/renderer.py | 36 + .../display/core/templates/component.html | 11 + .../src/secretnote/display/models.py | 12 + .../secretnote/display/parsers/__init__.py | 0 .../src/secretnote/display/parsers/base.py | 46 + .../secretnote/display/parsers/expression.py | 242 + .../src/secretnote/display/parsers/graph.py | 267 + .../secretnote/display/parsers/timeline.py | 169 + .../src/secretnote/formal/__init__.py | 0 .../secretnote/formal/locations/__init__.py | 22 + .../src/secretnote/formal/locations/pyu.py | 8 + .../src/secretnote/formal/locations/spu.py | 50 + .../src/secretnote/formal/locations/utils.py | 53 + .../src/secretnote/formal/locations/world.py | 62 + .../src/secretnote/formal/symbols.py | 165 + .../secretnote/instrumentation/__init__.py | 0 .../secretnote/instrumentation/checkpoint.py | 28 + .../src/secretnote/instrumentation/envvars.py | 3 + .../secretnote/instrumentation/exporters.py | 87 + .../src/secretnote/instrumentation/models.py | 599 + .../src/secretnote/instrumentation/package.py | 3 + .../secretnote/instrumentation/profiler.py | 309 + .../src/secretnote/instrumentation/sdk.py | 258 + .../secretnote/instrumentation/snapshot.py | 252 + .../src/secretnote/instrumentation/version.py | 1 + pyprojects/secretnote/src/secretnote/py.typed | 0 .../src/secretnote/server/__init__.py | 9 + .../src/secretnote/server/__main__.py | 4 + .../secretnote/src/secretnote/server/app.py | 53 + .../secretnote/server}/db.py | 0 .../src/secretnote/server/handlers.py | 28 + .../secretnote/server}/node/handler.py | 12 +- .../secretnote/server}/node/nodemanager.py | 3 +- .../src/secretnote/typing/__init__.py | 0 .../src/secretnote/typing/tree_util.py | 22 + .../src/secretnote/utils/itertools.py | 6 + .../src/secretnote/utils/logging.py | 94 + .../secretnote/src/secretnote/utils/node.py | 197 + .../src/secretnote/utils/pydantic.py | 187 + .../src/secretnote/utils/version.py | 31 + .../src/secretnote/utils/warnings.py | 11 + .../secretnote/tests/secretnote/test_dummy.py | 2 - .../tests/secretnote/utils/test_version.py | 49 + pyprojects/spu-stubs/CHANGELOG.md | 2 +- pyprojects/spu-stubs/package.json | 13 +- pyprojects/spu-stubs/pyproject.toml | 3 +- requirements-dev.lock | 219 +- requirements.lock | 145 +- scripts/setup_all.sh | 4 +- scripts/setup_python.sh | 5 +- tsconfig.base.json | 3 +- tsconfig.node.json | 2 +- 197 files changed, 17402 insertions(+), 7343 deletions(-) create mode 100644 .changeset/loud-kids-dream.md rename .eslintrc.js => .eslintrc.cjs (95%) rename .eslintrc.node.js => .eslintrc.node.cjs (61%) delete mode 100644 .python-version delete mode 100644 Makefile create mode 100644 docker/ci.Dockerfile rename .devcontainer/Dockerfile => docker/devcontainer.Dockerfile (52%) delete mode 100644 docker/notebook.Dockerfile create mode 100644 examples/debug.ipynb create mode 100644 examples/millionaires.ipynb create mode 100644 examples/real.ipynb delete mode 100644 examples/sf-2pc-local-ray-instances/alice.ipynb delete mode 100644 examples/sf-2pc-local-ray-instances/bob.ipynb create mode 100644 examples/sim.ipynb delete mode 100644 examples/spu-over-local-network.ipynb create mode 100644 examples/spu_lr.ipynb create mode 100644 examples/spu_lr_unrolled.ipynb create mode 100644 packages/secretnote-ui/.eslintrc.cjs create mode 100644 packages/secretnote-ui/.storybook/main.ts create mode 100644 packages/secretnote-ui/.storybook/preview.ts create mode 100644 packages/secretnote-ui/README.md create mode 100644 packages/secretnote-ui/package.json create mode 100644 packages/secretnote-ui/src/.openapi-stubs/index.ts create mode 100644 packages/secretnote-ui/src/.openapi-stubs/models/ArgumentEdge.ts create mode 100644 packages/secretnote-ui/src/.openapi-stubs/models/DictSnapshot.ts create mode 100644 packages/secretnote-ui/src/.openapi-stubs/models/ExecExpression.ts create mode 100644 packages/secretnote-ui/src/.openapi-stubs/models/Frame.ts create mode 100644 packages/secretnote-ui/src/.openapi-stubs/models/FrameInfoSnapshot.ts create mode 100644 packages/secretnote-ui/src/.openapi-stubs/models/FrameSnapshot.ts create mode 100644 packages/secretnote-ui/src/.openapi-stubs/models/FunctionCheckpoint.ts create mode 100644 packages/secretnote-ui/src/.openapi-stubs/models/FunctionInfo.ts create mode 100644 packages/secretnote-ui/src/.openapi-stubs/models/FunctionNode.ts create mode 100644 packages/secretnote-ui/src/.openapi-stubs/models/FunctionParameter.ts create mode 100644 packages/secretnote-ui/src/.openapi-stubs/models/FunctionSignature.ts create mode 100644 packages/secretnote-ui/src/.openapi-stubs/models/FunctionSnapshot.ts create mode 100644 packages/secretnote-ui/src/.openapi-stubs/models/Graph.ts create mode 100644 packages/secretnote-ui/src/.openapi-stubs/models/HTTPValidationError.ts create mode 100644 packages/secretnote-ui/src/.openapi-stubs/models/ListSnapshot.ts create mode 100644 packages/secretnote-ui/src/.openapi-stubs/models/LocalObject.ts create mode 100644 packages/secretnote-ui/src/.openapi-stubs/models/LocalObjectNode.ts create mode 100644 packages/secretnote-ui/src/.openapi-stubs/models/LogicalLocation.ts create mode 100644 packages/secretnote-ui/src/.openapi-stubs/models/MoveExpression.ts create mode 100644 packages/secretnote-ui/src/.openapi-stubs/models/NoneSnapshot.ts create mode 100644 packages/secretnote-ui/src/.openapi-stubs/models/ObjectSnapshot.ts create mode 100644 packages/secretnote-ui/src/.openapi-stubs/models/Reference.ts create mode 100644 packages/secretnote-ui/src/.openapi-stubs/models/ReferenceEdge.ts create mode 100644 packages/secretnote-ui/src/.openapi-stubs/models/ReferenceMap.ts create mode 100644 packages/secretnote-ui/src/.openapi-stubs/models/RemoteLocationSnapshot.ts create mode 100644 packages/secretnote-ui/src/.openapi-stubs/models/RemoteObject.ts create mode 100644 packages/secretnote-ui/src/.openapi-stubs/models/RemoteObjectNode.ts create mode 100644 packages/secretnote-ui/src/.openapi-stubs/models/RemoteObjectSnapshot.ts create mode 100644 packages/secretnote-ui/src/.openapi-stubs/models/ReturnEdge.ts create mode 100644 packages/secretnote-ui/src/.openapi-stubs/models/RevealEdge.ts create mode 100644 packages/secretnote-ui/src/.openapi-stubs/models/RevealExpression.ts create mode 100644 packages/secretnote-ui/src/.openapi-stubs/models/RevealNode.ts create mode 100644 packages/secretnote-ui/src/.openapi-stubs/models/Semantics.ts create mode 100644 packages/secretnote-ui/src/.openapi-stubs/models/Timeline.ts create mode 100644 packages/secretnote-ui/src/.openapi-stubs/models/TransformEdge.ts create mode 100644 packages/secretnote-ui/src/.openapi-stubs/models/ValidationError.ts create mode 100644 packages/secretnote-ui/src/.openapi-stubs/models/Visualization.ts create mode 100644 packages/secretnote-ui/src/.openapi-stubs/models/_ParameterKind.ts create mode 100644 packages/secretnote-ui/src/components/DataProvider/context.ts create mode 100644 packages/secretnote-ui/src/components/DataProvider/index.tsx create mode 100644 packages/secretnote-ui/src/components/DataProvider/utils.ts create mode 100644 packages/secretnote-ui/src/components/ExecutionGraph/colorization.ts create mode 100644 packages/secretnote-ui/src/components/ExecutionGraph/index.tsx create mode 100644 packages/secretnote-ui/src/components/ExecutionGraph/shapes.ts create mode 100644 packages/secretnote-ui/src/components/ExecutionGraph/tooltip.ts create mode 100644 packages/secretnote-ui/src/components/ExecutionGraph/types.ts create mode 100644 packages/secretnote-ui/src/components/ExecutionGraph/utils.ts create mode 100644 packages/secretnote-ui/src/components/Visualization.tsx create mode 100644 packages/secretnote-ui/src/index.ts create mode 100644 packages/secretnote-ui/src/render.tsx create mode 100644 packages/secretnote-ui/src/utils/drawer.ts create mode 100644 packages/secretnote-ui/src/utils/reify.ts create mode 100644 packages/secretnote-ui/src/utils/string.ts create mode 100644 packages/secretnote-ui/src/utils/typing.ts create mode 100644 packages/secretnote-ui/src/vite-env.d.ts create mode 100644 packages/secretnote-ui/tsconfig.json create mode 100644 packages/secretnote-ui/tsconfig.node.json create mode 100644 packages/secretnote-ui/vite.config.ts create mode 100644 packages/secretnote/public/favicon.svg create mode 100644 packages/secretnote/src/global.ts create mode 100644 pyprojects/secretnote/.jupyter/config_dev.py delete mode 100644 pyprojects/secretnote/dev-config/jupyter_server_config.py delete mode 100644 pyprojects/secretnote/jupyter-config/jupyter_server_config.d/libro-server.json delete mode 100644 pyprojects/secretnote/libro_server/__init__.py delete mode 100644 pyprojects/secretnote/libro_server/app.py delete mode 100644 pyprojects/secretnote/libro_server/handler.py delete mode 100644 pyprojects/secretnote/libro_server/static/favicon.ico delete mode 100644 pyprojects/secretnote/libro_server/template/error.html delete mode 100644 pyprojects/secretnote/libro_server/template/page.html delete mode 100644 pyprojects/secretnote/libro_server/template/secretnote.html create mode 100644 pyprojects/secretnote/proto/buf.gen.yaml create mode 100644 pyprojects/secretnote/proto/buf.lock create mode 100644 pyprojects/secretnote/proto/buf.yaml create mode 100644 pyprojects/secretnote/proto/secretnote/v1/secretnote.proto delete mode 100644 pyprojects/secretnote/pyproject-poetry.toml delete mode 100644 pyprojects/secretnote/ruff.toml create mode 100644 pyprojects/secretnote/scripts/copy_static_files.py create mode 100755 pyprojects/secretnote/scripts/generate_proto.sh delete mode 100644 pyprojects/secretnote/setup.py create mode 100644 pyprojects/secretnote/src/secretnote/display/__init__.py create mode 100644 pyprojects/secretnote/src/secretnote/display/api.py create mode 100644 pyprojects/secretnote/src/secretnote/display/app.py create mode 100644 pyprojects/secretnote/src/secretnote/display/core/__init__.py create mode 100644 pyprojects/secretnote/src/secretnote/display/core/renderer.py create mode 100644 pyprojects/secretnote/src/secretnote/display/core/templates/component.html create mode 100644 pyprojects/secretnote/src/secretnote/display/models.py create mode 100644 pyprojects/secretnote/src/secretnote/display/parsers/__init__.py create mode 100644 pyprojects/secretnote/src/secretnote/display/parsers/base.py create mode 100644 pyprojects/secretnote/src/secretnote/display/parsers/expression.py create mode 100644 pyprojects/secretnote/src/secretnote/display/parsers/graph.py create mode 100644 pyprojects/secretnote/src/secretnote/display/parsers/timeline.py create mode 100644 pyprojects/secretnote/src/secretnote/formal/__init__.py create mode 100644 pyprojects/secretnote/src/secretnote/formal/locations/__init__.py create mode 100644 pyprojects/secretnote/src/secretnote/formal/locations/pyu.py create mode 100644 pyprojects/secretnote/src/secretnote/formal/locations/spu.py create mode 100644 pyprojects/secretnote/src/secretnote/formal/locations/utils.py create mode 100644 pyprojects/secretnote/src/secretnote/formal/locations/world.py create mode 100644 pyprojects/secretnote/src/secretnote/formal/symbols.py create mode 100644 pyprojects/secretnote/src/secretnote/instrumentation/__init__.py create mode 100644 pyprojects/secretnote/src/secretnote/instrumentation/checkpoint.py create mode 100644 pyprojects/secretnote/src/secretnote/instrumentation/envvars.py create mode 100644 pyprojects/secretnote/src/secretnote/instrumentation/exporters.py create mode 100644 pyprojects/secretnote/src/secretnote/instrumentation/models.py create mode 100644 pyprojects/secretnote/src/secretnote/instrumentation/package.py create mode 100644 pyprojects/secretnote/src/secretnote/instrumentation/profiler.py create mode 100644 pyprojects/secretnote/src/secretnote/instrumentation/sdk.py create mode 100644 pyprojects/secretnote/src/secretnote/instrumentation/snapshot.py create mode 100644 pyprojects/secretnote/src/secretnote/instrumentation/version.py create mode 100644 pyprojects/secretnote/src/secretnote/py.typed create mode 100644 pyprojects/secretnote/src/secretnote/server/__init__.py create mode 100644 pyprojects/secretnote/src/secretnote/server/__main__.py create mode 100644 pyprojects/secretnote/src/secretnote/server/app.py rename pyprojects/secretnote/{libro_server => src/secretnote/server}/db.py (100%) create mode 100644 pyprojects/secretnote/src/secretnote/server/handlers.py rename pyprojects/secretnote/{libro_server => src/secretnote/server}/node/handler.py (92%) rename pyprojects/secretnote/{libro_server => src/secretnote/server}/node/nodemanager.py (96%) create mode 100644 pyprojects/secretnote/src/secretnote/typing/__init__.py create mode 100644 pyprojects/secretnote/src/secretnote/typing/tree_util.py create mode 100644 pyprojects/secretnote/src/secretnote/utils/itertools.py create mode 100644 pyprojects/secretnote/src/secretnote/utils/logging.py create mode 100644 pyprojects/secretnote/src/secretnote/utils/node.py create mode 100644 pyprojects/secretnote/src/secretnote/utils/pydantic.py create mode 100644 pyprojects/secretnote/src/secretnote/utils/version.py create mode 100644 pyprojects/secretnote/src/secretnote/utils/warnings.py delete mode 100644 pyprojects/secretnote/tests/secretnote/test_dummy.py create mode 100644 pyprojects/secretnote/tests/secretnote/utils/test_version.py diff --git a/.changeset/blue-crabs-argue.md b/.changeset/blue-crabs-argue.md index 391526d6..7fb5c39d 100644 --- a/.changeset/blue-crabs-argue.md +++ b/.changeset/blue-crabs-argue.md @@ -1,6 +1,6 @@ --- -'@secretflow/notebook-pyproject-secretnote': patch -'@secretflow/notebook-pyproject-spu-stubs': patch +'secretnote': patch +'spu-stubs': patch --- Initial publish diff --git a/.changeset/loud-kids-dream.md b/.changeset/loud-kids-dream.md new file mode 100644 index 00000000..3c44aee8 --- /dev/null +++ b/.changeset/loud-kids-dream.md @@ -0,0 +1,9 @@ +--- +'@secretflow/secretnote-ui': minor +'secretnote': minor +'spu-stubs': minor +'@secretflow/repo-utils': minor +'@secretflow/secretnote': minor +--- + +Include SecretNote server application diff --git a/.changeset/pre.json b/.changeset/pre.json index d626a294..c5c75bee 100644 --- a/.changeset/pre.json +++ b/.changeset/pre.json @@ -2,9 +2,9 @@ "mode": "pre", "tag": "dev", "initialVersions": { - "tsup-utils": "0.1.0", - "@secretflow/notebook-pyproject-secretnote": "0.0.0", - "@secretflow/notebook-pyproject-spu-stubs": "0.0.0" + "@secretflow/repo-utils": "0.1.0", + "secretnote": "0.0.0", + "spu-stubs": "0.0.0" }, "changesets": ["blue-crabs-argue"] } diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index f20b538d..9504916e 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,7 +1,7 @@ { "name": "notebook", "build": { - "dockerfile": "./Dockerfile" + "dockerfile": "../docker/devcontainer.Dockerfile" }, "mounts": [ { @@ -21,7 +21,6 @@ "waitFor": "postCreateCommand", "features": { "ghcr.io/devcontainers/features/common-utils:2": {}, - "ghcr.io/devcontainers/features/docker-in-docker:2": {}, "ghcr.io/devcontainers/features/github-cli:1": {} }, "customizations": { @@ -39,16 +38,5 @@ "tamasfe.even-better-toml" ] } - }, - "forwardPorts": [3000, 8000], - "portsAttributes": { - "3000": { - "label": "Docusaurus", - "onAutoForward": "notify" - }, - "8000": { - "label": "Umi", - "onAutoForward": "notify" - } } } diff --git a/.devcontainer/scripts/create-venv.sh b/.devcontainer/scripts/create-venv.sh index f9a0c58a..e0a40cbf 100755 --- a/.devcontainer/scripts/create-venv.sh +++ b/.devcontainer/scripts/create-venv.sh @@ -1,11 +1,18 @@ #!/usr/bin/env bash -PYTHON_VENV_VERSION=$(cat .python-version | tr -d " \t\n\r") +PYTHON_VERSION="3.8.17" REPO_ROOT=$(git rev-parse --show-toplevel) -rye fetch $PYTHON_VENV_VERSION +# Download Python +rye fetch $PYTHON_VERSION +# Remove current virtual environment +find .venv -exec rm -rf {} + 2> /dev/null + +# Create new virtual environment cd "$REPO_ROOT"/.. +python +$PYTHON_VERSION -m venv "$REPO_ROOT/.venv" -python +$PYTHON_VENV_VERSION -m venv "$REPO_ROOT/.venv" -printf '{"python": "%s"}' "$PYTHON_VENV_VERSION" > "$REPO_ROOT/.venv/rye-venv.json" +# Write auxiliary files for Rye +printf '{"python": "%s"}\n' "$PYTHON_VERSION" > "$REPO_ROOT/.venv/rye-venv.json" +printf '%s\n' "$PYTHON_VERSION" > "$REPO_ROOT/.python-version" diff --git a/.editorconfig b/.editorconfig index 276a4d43..9c56036b 100644 --- a/.editorconfig +++ b/.editorconfig @@ -16,5 +16,11 @@ indent_size = 4 [*.ipynb] indent_size = 4 +[*.bazel] +indent_size = 4 + +[*.BUILD] +indent_size = 4 + [Makefile] indent_style = tab diff --git a/.eslintrc.js b/.eslintrc.cjs similarity index 95% rename from .eslintrc.js rename to .eslintrc.cjs index e35d26f9..f6c2d0c5 100644 --- a/.eslintrc.js +++ b/.eslintrc.cjs @@ -1,6 +1,7 @@ module.exports = { root: true, + env: { browser: true, es2020: true }, extends: [ 'eslint:recommended', 'plugin:@typescript-eslint/recommended', @@ -11,14 +12,7 @@ module.exports = { ], plugins: ['import'], - env: { - node: true, - }, - settings: { - react: { - version: '18', - }, - }, + ignorePatterns: ['dist', '.eslintrc.*'], rules: { // common pitfalls diff --git a/.eslintrc.node.js b/.eslintrc.node.cjs similarity index 61% rename from .eslintrc.node.js rename to .eslintrc.node.cjs index 3a2f015a..0aeeb742 100644 --- a/.eslintrc.node.js +++ b/.eslintrc.node.cjs @@ -1,6 +1,6 @@ module.exports = { root: true, - extends: [require.resolve('./.eslintrc.js')], + extends: [require.resolve('./.eslintrc.cjs')], rules: { 'no-console': 'off', }, diff --git a/.github/workflows/ci-docker.yml b/.github/workflows/ci-docker.yml index 6581fe82..e0b367fc 100644 --- a/.github/workflows/ci-docker.yml +++ b/.github/workflows/ci-docker.yml @@ -21,4 +21,4 @@ jobs: - uses: docker/build-push-action@v4 with: push: false - file: ./docker/notebook.Dockerfile + file: ./docker/ci.Dockerfile diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 51b4a10b..6ba7e080 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,17 +19,19 @@ jobs: uses: secretflow/web-ci/.github/workflows/ci-javascript.yml@main with: node-version: ${{ matrix.node-version }} + python-version: '3.8' strategy: fail-fast: false matrix: - node-version: ['16.20.x'] + node-version: ['18', '20'] ci-python: name: 'CI: Python' uses: secretflow/web-ci/.github/workflows/ci-python.yml@main with: + node-version: '18' python-version: ${{ matrix.python-version }} strategy: fail-fast: false matrix: - python-version: ['3.8.x'] + python-version: ['3.8'] diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0037d4e1..30701ddb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,7 +10,7 @@ permissions: {} jobs: changesets: # prevents this action from running on forks - if: github.repository == 'secretflow/notebook' + if: github.repository == 'secretflow/secretnote' name: Changesets uses: secretflow/web-ci/.github/workflows/release-changesets.yml@main diff --git a/.gitignore b/.gitignore index bbd3722e..b6df38dd 100644 --- a/.gitignore +++ b/.gitignore @@ -136,7 +136,7 @@ venv/ ENV/ env.bak/ venv.bak/ -# .python-version +.python-version # Env files .env.* @@ -303,13 +303,27 @@ dist **/.dumi/tmp **/.dumi/tmp-production +# Umi +.umi/ +.umi-production/ +.umi-test/ + # macOS stuff .DS_Store # Personalization .vscode/launch.json -#umi -.umi/ -.umi-production/ -.umi-test/ +# gitignore template for Bazel build system +# website: https://bazel.build/ + +# Ignore all bazel-* symlinks. There is no full list since this can change +# based on the name of the directory bazel is cloned into. +/bazel-* + +# Directories for the Bazel IntelliJ plugin containing the generated +# IntelliJ project files and plugin configuration. Seperate directories are +# for the IntelliJ, Android Studio and CLion versions of the plugin. +/.ijwb/ +/.aswb/ +/.clwb/ diff --git a/.lintstagedrc.js b/.lintstagedrc.js index 323fe774..ccf7f710 100644 --- a/.lintstagedrc.js +++ b/.lintstagedrc.js @@ -1,6 +1,7 @@ module.exports = { - '*': ['prettier --ignore-unknown --check'], + '*.*': ['prettier --ignore-unknown --check'], '*.{css,less}': ['stylelint'], '*.{js,jsx,ts,tsx,mjs,mts,cjs,cts,mtsx,ctsx,mjsx,cjsx}': ['eslint'], '*.{py,pyi}': ['black --check', 'ruff check', 'pyright'], + '*.ipynb': ['black --check', 'ruff check' /* 'pyright' */], }; diff --git a/.prettierignore b/.prettierignore index fdc9aaa3..04a3d280 100644 --- a/.prettierignore +++ b/.prettierignore @@ -12,6 +12,7 @@ pnpm-lock.yaml **/.dumi/tmp **/.dumi/tmp-production **/.docusaurus +**/.openapi-stubs # protobuf **/*_pb.ts diff --git a/.python-version b/.python-version deleted file mode 100644 index 202cfa46..00000000 --- a/.python-version +++ /dev/null @@ -1 +0,0 @@ -cpython@3.8.17 diff --git a/.stylelintrc.js b/.stylelintrc.js index ee7e3ae7..a2837491 100644 --- a/.stylelintrc.js +++ b/.stylelintrc.js @@ -8,4 +8,10 @@ module.exports = { 'selector-class-pattern': null, 'no-invalid-double-slash-comments': null, }, + overrides: [ + { + files: ['*.less', '**/*.less'], + customSyntax: 'postcss-less', + }, + ], }; diff --git a/.vscode/cspell.json b/.vscode/cspell.json index 77671edd..7171803b 100644 --- a/.vscode/cspell.json +++ b/.vscode/cspell.json @@ -12,29 +12,43 @@ "arange", "astd", "awatch", + "consts", "coro", "cpus", + "dagre", "dbaeumer", "difizen", + "elkjs", + "elts", "ename", "endent", "fdpexpect", "fdspawn", + "getipython", + "grpcio", "ifaddresses", "iloc", "INET", + "infeed", + "ipywidgets", + "jaxpr", + "jsonlines", "kuscia", + "lasti", "libro", "libspu", "lucide", "lumino", "mfsu", + "nbclient", "nbformat", "ndarray", "netifaces", + "networkx", "noopener", "noreferrer", "nrwl", + "otlp", "pbar", "pbars", "pphlo", @@ -49,14 +63,21 @@ "pyright", "pytest", "remarkrc", + "retvals", "rjsf", + "Roboto", "scql", + "SPUIO", "tablefmt", + "teeu", "tqdm", "tsup", + "ultratb", + "unflatten", "vercel", "vitest", - "webp" + "webp", + "Yao's" ], // flagWords - list of words to be always considered incorrect // This is useful for offensive words and common spelling errors. diff --git a/.vscode/extensions.json b/.vscode/extensions.json index b1b46342..31307aa2 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -6,10 +6,10 @@ "streetsidesoftware.code-spell-checker", "ms-python.python", "ms-python.vscode-pylance", - // "ms-python.flake8", "charliermarsh.ruff", "ms-python.black-formatter", - "ms-vscode-remote.remote-containers" + "ms-vscode-remote.remote-containers", + "nrwl.angular-console" ], "unwantedRecommendations": [] } diff --git a/.vscode/settings.json b/.vscode/settings.json index e436723a..4398bc9b 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,97 +1,91 @@ { - "files.associations": { - // Turborepo - "turbo.json": "jsonc", - // Nx - "nx.json": "jsonc", - "project.json": "jsonc", - // tsconfig - "**/tooling/tsconfig/*.json": "jsonc", - "**/tooling/tsconfig/package.json": "json", - "tsconfig.*.json": "jsonc", - "tsconfig.json": "jsonc", - // rye - "requirements.lock": "pip-requirements", - "requirements-dev.lock": "pip-requirements" - }, - "cSpell.allowCompoundWords": true, - "cSpell.enabled": true, - "editor.rulers": [88], - "editor.formatOnSave": true, - "editor.insertSpaces": true, - "files.insertFinalNewline": true, - "black-formatter.args": ["--config", "pyproject.toml"], - "notebook.formatOnSave.enabled": true, - "typescript.tsdk": "node_modules/typescript/lib", - "eslint.validate": [ - "javascript", - "javascriptreact", - "typescript", - "typescriptreact", - "mdx" - ], - "yaml.schemas": { - "https://json.schemastore.org/github-issue-config.json": ".github/ISSUE_TEMPLATE/config.yml" - }, - "[python]": { + "[css]": { "editor.codeActionsOnSave": { - "source.organizeImports": false, - "source.fixAll": true + "source.fixAll.stylelint": true }, - "editor.defaultFormatter": "ms-python.black-formatter" - }, - "[typescriptreact]": { - "editor.tabSize": 2, "editor.defaultFormatter": "esbenp.prettier-vscode", - "editor.codeActionsOnSave": { - "source.fixAll.eslint": true - } + "editor.tabSize": 2 }, - "[typescript]": { - "editor.tabSize": 2, - "editor.defaultFormatter": "esbenp.prettier-vscode", + "[javascript]": { "editor.codeActionsOnSave": { "source.fixAll.eslint": true - } + }, + "editor.defaultFormatter": "esbenp.prettier-vscode", + "editor.tabSize": 2 }, "[javascriptreact]": { - "editor.tabSize": 2, - "editor.defaultFormatter": "esbenp.prettier-vscode", "editor.codeActionsOnSave": { "source.fixAll.eslint": true - } - }, - "[javascript]": { - "editor.tabSize": 2, + }, "editor.defaultFormatter": "esbenp.prettier-vscode", - "editor.codeActionsOnSave": { - "source.fixAll.eslint": true - } + "editor.tabSize": 2 }, "[json]": { - "editor.tabSize": 2, - "editor.defaultFormatter": "esbenp.prettier-vscode" + "editor.defaultFormatter": "esbenp.prettier-vscode", + "editor.tabSize": 2 }, "[jsonc]": { - "editor.tabSize": 2, - "editor.defaultFormatter": "esbenp.prettier-vscode" + "editor.defaultFormatter": "esbenp.prettier-vscode", + "editor.tabSize": 2 }, "[markdown]": { "editor.quickSuggestions": { - "other": true, "comments": true, + "other": true, "strings": true } }, "[mdx]": { "editor.defaultFormatter": "esbenp.prettier-vscode" }, - "[css]": { - "editor.tabSize": 2, + "[python]": { + "editor.codeActionsOnSave": { + "source.fixAll": true, + "source.organizeImports": true + }, + "editor.defaultFormatter": "ms-python.black-formatter", + "editor.formatOnSave": true + }, + "[typescript]": { + "editor.codeActionsOnSave": { + "source.fixAll.eslint": true + }, "editor.defaultFormatter": "esbenp.prettier-vscode", + "editor.tabSize": 2 + }, + "[typescriptreact]": { "editor.codeActionsOnSave": { - "source.fixAll.stylelint": true - } + "source.fixAll.eslint": true + }, + "editor.defaultFormatter": "esbenp.prettier-vscode", + "editor.tabSize": 2 }, - "cSpell.words": ["ahooks"] + "black-formatter.args": ["--config", "pyproject.toml"], + "cSpell.allowCompoundWords": true, + "cSpell.enabled": true, + "editor.formatOnSave": true, + "editor.insertSpaces": true, + "editor.rulers": [88], + "eslint.validate": [ + "javascript", + "javascriptreact", + "typescript", + "typescriptreact", + "mdx" + ], + "files.associations": { + "**/tooling/tsconfig/*.json": "jsonc", + "**/tooling/tsconfig/package.json": "json", + "nx.json": "jsonc", + "requirements-dev.lock": "pip-requirements", + "requirements.lock": "pip-requirements", + "tsconfig.*.json": "jsonc", + "tsconfig.json": "jsonc" + }, + "python.analysis.diagnosticMode": "workspace", + "python.analysis.include": ["pyprojects/**", "examples/**"], + "typescript.tsdk": "node_modules/typescript/lib", + "python.testing.pytestArgs": ["pyprojects"], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true } diff --git a/Makefile b/Makefile deleted file mode 100644 index 4d3a4d73..00000000 --- a/Makefile +++ /dev/null @@ -1,10 +0,0 @@ -DOCKER_USERNAME ?= $(shell whoami) -APPLICATION_NAME ?= notebook - -GIT_HASH ?= $(shell git log --format="%h" -n 1) - -image: - docker buildx build \ - --platform linux/amd64 \ - -t ${DOCKER_USERNAME}/${APPLICATION_NAME}:${GIT_HASH} \ - -f ./docker/notebook.Dockerfile . diff --git a/README.md b/README.md index 13f720b7..5f77fc54 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ - [ ] Type stubs: SecretFlow, SPU, HEU - [ ] Unified configuration (Pydantic) - [ ] Pydantic-Protobuf interop + - [ ] Endpoint Projection (choreographic) ## 本地开发 diff --git a/docker/ci.Dockerfile b/docker/ci.Dockerfile new file mode 100644 index 00000000..b2ddc9c6 --- /dev/null +++ b/docker/ci.Dockerfile @@ -0,0 +1,44 @@ +FROM rust:bullseye + +ARG FNM_VERSION=1.35.1 +ARG RYE_VERSION=0.15.1 + +ARG NODE_VERSION=18.18 +ARG PYTHON_VERSION=3.8.18 + +# Install fnm (for Node) +RUN cargo install fnm \ + --version ${FNM_VERSION} + +# Install Rye (for Python) +RUN cargo install rye \ + --git https://github.com/mitsuhiko/rye \ + --tag ${RYE_VERSION} + +# Install Node +RUN fnm install ${NODE_VERSION} \ + && fnm default ${NODE_VERSION} + +# Install PNPM +RUN eval $(fnm env) \ + && npm install -g pnpm \ + && SHELL=bash pnpm setup \ + && echo "" >> ~/.bashrc + +# Install Python +RUN rye self install --yes + +# Copy workspace +COPY . /home/vscode/workspace +WORKDIR /home/vscode/workspace + +ENV PATH="/root/.cargo/bin:/root/.rye/shims:${PATH}" + +RUN eval $(fnm env) \ + && pnpm run bootstrap + +RUN eval $(fnm env) \ + && pnpm run ci:javascript \ + && pnpm run ci:python + +ENTRYPOINT [ "/bin/bash" ] diff --git a/.devcontainer/Dockerfile b/docker/devcontainer.Dockerfile similarity index 52% rename from .devcontainer/Dockerfile rename to docker/devcontainer.Dockerfile index 192544e9..e0ff71e8 100644 --- a/.devcontainer/Dockerfile +++ b/docker/devcontainer.Dockerfile @@ -1,31 +1,30 @@ -FROM mcr.microsoft.com/devcontainers/base:bullseye +FROM mcr.microsoft.com/devcontainers/rust:bullseye ARG FNM_VERSION=1.35.1 -ARG RYE_VERSION=0.13.0 -ARG NODE_VERSION=16.20 +ARG RYE_VERSION=0.15.1 + +ARG NODE_VERSION=18.18 +ARG PYTHON_VERSION=3.8.18 RUN apt-get update \ && apt-get install -y \ - sudo \ - unzip \ - curl - -# Install fnm (for Node) -RUN curl -sSL https://github.com/Schniz/fnm/releases/download/v${FNM_VERSION}/fnm-linux.zip \ - -o fnm.zip \ - && unzip -d /usr/local/bin fnm.zip fnm \ - && chmod +x /usr/local/bin/fnm \ - && rm fnm.zip - -# Install Rye (for Python) -RUN curl -sSL https://github.com/mitsuhiko/rye/releases/download/${RYE_VERSION}/rye-x86_64-linux.gz \ - | gunzip > /usr/local/bin/rye \ - && chmod +x /usr/local/bin/rye + sudo # Default non-root user USER vscode WORKDIR /home/vscode +ENV PATH="/home/vscode/.cargo/bin:${PATH}" + +# Install fnm (for Node) +RUN cargo install fnm \ + --version ${FNM_VERSION} + +# Install Rye (for Python) +RUN cargo install rye \ + --git https://github.com/mitsuhiko/rye \ + --tag ${RYE_VERSION} + # Install Node RUN fnm install ${NODE_VERSION} \ && fnm default ${NODE_VERSION} diff --git a/docker/notebook.Dockerfile b/docker/notebook.Dockerfile deleted file mode 100644 index 5972614c..00000000 --- a/docker/notebook.Dockerfile +++ /dev/null @@ -1 +0,0 @@ -FROM python:3.10-slim-bookworm AS node-setup diff --git a/examples/debug.ipynb b/examples/debug.ipynb new file mode 100644 index 00000000..f855afdb --- /dev/null +++ b/examples/debug.ipynb @@ -0,0 +1,24 @@ +{ + "cells": [], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.17" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/examples/millionaires.ipynb b/examples/millionaires.ipynb new file mode 100644 index 00000000..c3b30786 --- /dev/null +++ b/examples/millionaires.ipynb @@ -0,0 +1,138 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from secretnote.formal.locations import (\n", + " SPUFieldType,\n", + " SPUProtocolKind,\n", + " SymbolicPYU,\n", + " SymbolicSPU,\n", + " SymbolicWorld,\n", + ")\n", + "\n", + "sym_world = SymbolicWorld(world=frozenset((\"alice\", \"bob\")))\n", + "sym_alice = SymbolicPYU(\"alice\")\n", + "sym_bob = SymbolicPYU(\"bob\")\n", + "sym_spu = SymbolicSPU(\n", + " world=frozenset((\"alice\", \"bob\")),\n", + " protocol=SPUProtocolKind.SEMI2K,\n", + " field=SPUFieldType.FM128,\n", + " fxp_fraction_bits=0,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import ray\n", + "from secretnote.formal.locations import SFConfigSimulation, PortBinding\n", + "\n", + "ray.shutdown()\n", + "\n", + "sym_world.reify(SFConfigSimulation())\n", + "\n", + "alice = sym_alice.reify()\n", + "bob = sym_bob.reify()\n", + "spu = sym_spu.reify(\n", + " alice=PortBinding(announced_as=\"127.0.0.1:32767\"),\n", + " bob=PortBinding(announced_as=\"127.0.0.1:32768\"),\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from secretnote.instrumentation.sdk import create_profiler, setup_tracing\n", + "\n", + "setup_tracing()\n", + "\n", + "profiler = create_profiler()\n", + "profiler.start()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\"\"\"Yao's Millionaires' Problem in SecretFlow\"\"\"\n", + "\n", + "import jax\n", + "import secretflow\n", + "\n", + "key = jax.random.PRNGKey(42)\n", + "\n", + "\n", + "def make_money(seed: jax.random.KeyArray, generation: int) -> jax.Array:\n", + " for _ in range(generation):\n", + " seed, subkey = jax.random.split(seed)\n", + " return jax.random.randint(seed, shape=(), minval=10**6, maxval=10**9)\n", + "\n", + "\n", + "def compare(a: jax.Array, b: jax.Array) -> jax.Array:\n", + " return a > b\n", + "\n", + "\n", + "balance_alice = alice(make_money)(key, 3)\n", + "balance_bob = bob(make_money)(key, 2)\n", + "\n", + "balance_alice = balance_alice.to(spu)\n", + "balance_bob = balance_bob.to(spu)\n", + "\n", + "alice_is_richer = spu(compare)(balance_alice, balance_bob)\n", + "\n", + "alice_is_richer = secretflow.reveal(alice_is_richer)\n", + "print(f\"{alice_is_richer=}\")\n", + "\n", + "account_alice, account_bob = secretflow.reveal((balance_alice, balance_bob))\n", + "\n", + "print(f\"{account_alice=}\")\n", + "print(f\"{account_bob=}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from secretnote.display import visualize_run\n", + "\n", + "profiler.stop()\n", + "visualize_run(profiler)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.17" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/examples/psi.ipynb b/examples/psi.ipynb index 705c8410..e51211be 100644 --- a/examples/psi.ipynb +++ b/examples/psi.ipynb @@ -13,7 +13,7 @@ "ray start --head --node-ip-address=\"ip\" --port=\"port\" --include-dashboard=False --disable-usage-stats\n", "```\n", "\n", - "屏幕输出中显示 Ray runtime started，则说明 Ray 的主节点启动成功。" + "屏幕输出中显示 Ray runtime started，则说明 Ray 的主节点启动成功。\n" ] }, { @@ -26,7 +26,7 @@ "alice 和 bob 节点都需要初始化 secretflow，初始化语句只有在 self_party 上有区别。\n", "\n", "1. 分别使用 Ray 主节点的 node-ip-address 和 port 填充 sf.init 的 address 参数。\n", - "2. parties 中 alice 和 bob 的 address 分别填可以被对方访问的地址，并且选择一个**未被占用的端口。**" + "2. parties 中 alice 和 bob 的 address 分别填可以被对方访问的地址，并且选择一个**未被占用的端口。**\n" ] }, { @@ -42,12 +42,12 @@ "import spu\n", "\n", "network_conf = {\n", - " 'parties': {\n", - " 'alice': {\n", - " 'address': 'alice:8080',\n", + " \"parties\": {\n", + " \"alice\": {\n", + " \"address\": \"alice:8080\",\n", " },\n", - " 'bob': {\n", - " 'address': 'bob:8080',\n", + " \"bob\": {\n", + " \"address\": \"bob:8080\",\n", " },\n", " },\n", "}" @@ -60,7 +60,7 @@ "source": [ "### alice 和 bob 初始化\n", "\n", - "注意：下面初始化语句是分别只在对应的 alice 或 bob 节点执行的。除了初始化语句，后面所有代码，会被同时下发到 alice 和 bob 节点执行。" + "注意：下面初始化语句是分别只在对应的 alice 或 bob 节点执行的。除了初始化语句，后面所有代码，会被同时下发到 alice 和 bob 节点执行。\n" ] }, { @@ -74,8 +74,8 @@ "source": [ "sf.shutdown()\n", "sf.init(\n", - " address='127.0.0.1:6379',\n", - " cluster_config={**network_conf, 'self_party': 'alice'},\n", + " address=\"127.0.0.1:6379\",\n", + " cluster_config={**network_conf, \"self_party\": \"alice\"},\n", " log_to_driver=True,\n", ")" ] @@ -91,8 +91,8 @@ "source": [ "sf.shutdown()\n", "sf.init(\n", - " address='127.0.0.1:6379',\n", - " cluster_config={**network_conf, 'self_party': 'bob'},\n", + " address=\"127.0.0.1:6379\",\n", + " cluster_config={**network_conf, \"self_party\": \"bob\"},\n", " log_to_driver=True,\n", ")" ] @@ -107,7 +107,7 @@ "1. alice 的 address 请填写可以被 bob 访通的地址，并且选择一个**未被占用的端口** ，注意不要和 Ray 端口冲突。\n", "2. alice 的 listen_addr 可以和 alice address 里的端口一样。\n", "3. bob 的 address 请填写可以被 alice 访通的地址，并且选择一个**未被占用的端口** ，注意不要和 Ray 端口冲突。\n", - "4. bob 的 listen_addr 可以和 bob address 里的端口一样。" + "4. bob 的 listen_addr 可以和 bob address 里的端口一样。\n" ] }, { @@ -119,19 +119,11 @@ }, "outputs": [], "source": [ - "alice, bob = sf.PYU('alice'), sf.PYU('bob')\n", + "alice, bob = sf.PYU(\"alice\"), sf.PYU(\"bob\")\n", "spu_conf = {\n", " \"nodes\": [\n", - " {\n", - " \"party\": \"alice\",\n", - " \"address\": \"alice:8081\",\n", - " \"listen_addr\": \"alice:8081\"\n", - " },\n", - " {\n", - " \"party\": \"bob\",\n", - " \"address\": \"bob:8081\",\n", - " \"listen_addr\": \"alice:8081\"\n", - " },\n", + " {\"party\": \"alice\", \"address\": \"alice:8081\", \"listen_addr\": \"alice:8081\"},\n", + " {\"party\": \"bob\", \"address\": \"bob:8081\", \"listen_addr\": \"alice:8081\"},\n", " ],\n", " \"runtime_config\": {\n", " \"protocol\": spu.spu_pb2.SEMI2K,\n", @@ -149,7 +141,7 @@ "source": [ "## 隐私求交\n", "\n", - "提供 `psi_csv` 函数， `psi_csv` 将 csv 文件作为输入，并在求交后生成 csv 文件。默认协议为 [**KKRT**](https://eprint.iacr.org/2016/799.pdf)。" + "提供 `psi_csv` 函数， `psi_csv` 将 csv 文件作为输入，并在求交后生成 csv 文件。默认协议为 [**KKRT**](https://eprint.iacr.org/2016/799.pdf)。\n" ] }, { @@ -165,9 +157,15 @@ "\n", "current_dir = os.getcwd()\n", "\n", - "input_path = {alice: f'{current_dir}/iris_alice.csv', bob: f'{current_dir}/iris_bob.csv'}\n", - "output_path = {alice: f'{current_dir}/iris_alice_psi.csv', bob: f'{current_dir}/iris_alice_psi.csv'}\n", - "spu.psi_csv('uid', input_path, output_path, 'alice')" + "input_path = {\n", + " alice: f\"{current_dir}/iris_alice.csv\",\n", + " bob: f\"{current_dir}/iris_bob.csv\",\n", + "}\n", + "output_path = {\n", + " alice: f\"{current_dir}/iris_alice_psi.csv\",\n", + " bob: f\"{current_dir}/iris_alice_psi.csv\",\n", + "}\n", + "spu.psi_csv(\"uid\", input_path, output_path, \"alice\")" ] } ], diff --git a/examples/real.ipynb b/examples/real.ipynb new file mode 100644 index 00000000..9420c249 --- /dev/null +++ b/examples/real.ipynb @@ -0,0 +1,89 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "PRESET = os.environ.setdefault(\"PRESET\", \"millionaires\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import asyncio\n", + "from opentelemetry import trace\n", + "from opentelemetry.sdk.environment_variables import OTEL_SERVICE_NAME\n", + "\n", + "from secretnote.instrumentation.sdk import (\n", + " dump_tracing_context,\n", + " setup_debug_exporter,\n", + " setup_tracing,\n", + ")\n", + "\n", + "os.environ[OTEL_SERVICE_NAME] = f\"{PRESET}:driver\"\n", + "setup_tracing()\n", + "setup_debug_exporter()\n", + "\n", + "tracer = trace.get_tracer(__name__)\n", + "\n", + "with tracer.start_as_current_span(\"caller\"):\n", + " tracing_context = dump_tracing_context()\n", + "\n", + " p1, p2 = await asyncio.gather(\n", + " asyncio.create_subprocess_exec(\n", + " \"jupyter\",\n", + " \"execute\",\n", + " \"sim.ipynb\",\n", + " env={\n", + " **os.environ,\n", + " **tracing_context,\n", + " \"SELF_PARTY\": \"alice\",\n", + " \"RAY_ADDRESS\": \"127.0.0.1:32400\",\n", + " },\n", + " ),\n", + " asyncio.create_subprocess_exec(\n", + " \"jupyter\",\n", + " \"execute\",\n", + " \"sim.ipynb\",\n", + " env={\n", + " **os.environ,\n", + " **tracing_context,\n", + " \"SELF_PARTY\": \"bob\",\n", + " \"RAY_ADDRESS\": \"127.0.0.1:32401\",\n", + " },\n", + " ),\n", + " )\n", + "\n", + " await asyncio.gather(p1.wait(), p2.wait())" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.17" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/sf-2pc-local-ray-instances/alice.ipynb b/examples/sf-2pc-local-ray-instances/alice.ipynb deleted file mode 100644 index a4c25793..00000000 --- a/examples/sf-2pc-local-ray-instances/alice.ipynb +++ /dev/null @@ -1,238 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "b5b2d474", - "metadata": { - "papermill": { - "duration": 0.002938, - "end_time": "2023-09-18T00:12:41.900146", - "exception": false, - "start_time": "2023-09-18T00:12:41.897208", - "status": "completed" - }, - "tags": [] - }, - "source": [ - "# SecretFlow is like React Hooks [alice]\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3c3e6bbd", - "metadata": { - "execution": { - "iopub.execute_input": "2023-09-18T00:12:42.900256Z", - "iopub.status.busy": "2023-09-18T00:12:42.900077Z", - "iopub.status.idle": "2023-09-18T00:12:42.901838Z", - "shell.execute_reply": "2023-09-18T00:12:42.901575Z" - }, - "papermill": { - "duration": 0.003828, - "end_time": "2023-09-18T00:12:42.902739", - "exception": false, - "start_time": "2023-09-18T00:12:42.898911", - "status": "completed" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "self_party: str = \"alice\"\n", - "peer_party: str = \"bob\"\n", - "self_ray_port: int = 32400\n", - "peer_ray_port: int = 32401\n", - "self_secretflow_port: int = 8080\n", - "peer_secretflow_port: int = 8081\n", - "self_spu_port: int = 8090\n", - "peer_spu_port: int = 8091" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "dc6dfd39", - "metadata": { - "execution": { - "iopub.execute_input": "2023-09-18T00:12:42.909843Z", - "iopub.status.busy": "2023-09-18T00:12:42.909717Z", - "iopub.status.idle": "2023-09-18T00:12:42.947263Z", - "shell.execute_reply": "2023-09-18T00:12:42.946964Z" - }, - "papermill": { - "duration": 0.040205, - "end_time": "2023-09-18T00:12:42.948467", - "exception": false, - "start_time": "2023-09-18T00:12:42.908262", - "status": "completed" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "from secretnote.compat.secretflow.device.driver import (\n", - " SFConfigNetworked,\n", - " SFClusterConfig,\n", - " SFClusterParty,\n", - ")\n", - "\n", - "secretflow_config = SFConfigNetworked(\n", - " address=f\"localhost:{self_ray_port}\",\n", - " cluster_config=SFClusterConfig(\n", - " parties={\n", - " self_party: SFClusterParty(address=f\"127.0.0.1:{self_secretflow_port}\"),\n", - " peer_party: SFClusterParty(address=f\"127.0.0.1:{peer_secretflow_port}\"),\n", - " },\n", - " self_party=self_party,\n", - " ),\n", - " logging_level=\"debug\",\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "764bd227", - "metadata": {}, - "outputs": [], - "source": [ - "from secretnote.compat.spu import (\n", - " SPUConfig,\n", - " SPUClusterDef,\n", - " SPUNode,\n", - " SPURuntimeConfig,\n", - " SPUProtocolKind,\n", - " SPUFieldType,\n", - ")\n", - "\n", - "spu_config = SPUConfig(\n", - " cluster_def=SPUClusterDef(\n", - " nodes=[\n", - " SPUNode(party=self_party, address=f\"127.0.0.1:{self_spu_port}\"),\n", - " SPUNode(party=peer_party, address=f\"127.0.0.1:{peer_spu_port}\"),\n", - " ],\n", - " runtime_config=SPURuntimeConfig(\n", - " protocol=SPUProtocolKind.SEMI2K,\n", - " field=SPUFieldType.FM128,\n", - " ),\n", - " )\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0603e573", - "metadata": {}, - "outputs": [], - "source": [ - "import secretflow\n", - "\n", - "secretflow.shutdown()\n", - "secretflow.init(**secretflow_config.dict())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6065a829", - "metadata": {}, - "outputs": [], - "source": [ - "alice = secretflow.PYU(self_party)\n", - "bob = secretflow.PYU(peer_party)\n", - "spu = secretflow.SPU(**spu_config.dict())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7d19304c", - "metadata": {}, - "outputs": [], - "source": [ - "import jax.numpy as jnp\n", - "import numpy as np\n", - "import secretnote.functional as sfx\n", - "\n", - "\n", - "def multiply(x: np.ndarray, y: float):\n", - " return jnp.multiply(x, y)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "638a39e1", - "metadata": {}, - "outputs": [], - "source": [ - "array = sfx.use_cleartext(alice)(jnp.arange(10))\n", - "array = sfx.use_relocation(spu, array)()\n", - "\n", - "multiplier = sfx.use_cleartext(bob)(...)\n", - "multiplier = sfx.use_relocation(spu, multiplier)()\n", - "\n", - "result = sfx.use_function(spu, array, multiplier)(multiply)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "99bcaa32", - "metadata": {}, - "outputs": [], - "source": [ - "result_for_alice = sfx.use_relocation(alice, result)()\n", - "result_for_bob = sfx.use_relocation(bob, result)()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cd32acbf", - "metadata": {}, - "outputs": [], - "source": [ - "secretflow.wait(result_for_alice)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": ".venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.17" - }, - "papermill": { - "default_parameters": {}, - "duration": 2.249147, - "end_time": "2023-09-18T00:12:43.271623", - "environment_variables": {}, - "exception": null, - "input_path": "examples/sf-2pt-local-ray-instances.ipynb", - "output_path": "examples/sf-2pt-local-ray-instances.ipynb", - "parameters": { - "peer_party": "alice", - "self_party": "bob" - }, - "start_time": "2023-09-18T00:12:41.022476", - "version": "2.4.0" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/examples/sf-2pc-local-ray-instances/bob.ipynb b/examples/sf-2pc-local-ray-instances/bob.ipynb deleted file mode 100644 index 88c05748..00000000 --- a/examples/sf-2pc-local-ray-instances/bob.ipynb +++ /dev/null @@ -1,228 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "b5b2d474", - "metadata": { - "papermill": { - "duration": 0.002938, - "end_time": "2023-09-18T00:12:41.900146", - "exception": false, - "start_time": "2023-09-18T00:12:41.897208", - "status": "completed" - }, - "tags": [] - }, - "source": [ - "# SecretFlow is like React Hooks [bob]\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3c3e6bbd", - "metadata": { - "execution": { - "iopub.execute_input": "2023-09-18T00:12:42.900256Z", - "iopub.status.busy": "2023-09-18T00:12:42.900077Z", - "iopub.status.idle": "2023-09-18T00:12:42.901838Z", - "shell.execute_reply": "2023-09-18T00:12:42.901575Z" - }, - "papermill": { - "duration": 0.003828, - "end_time": "2023-09-18T00:12:42.902739", - "exception": false, - "start_time": "2023-09-18T00:12:42.898911", - "status": "completed" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "self_party: str = \"bob\"\n", - "peer_party: str = \"alice\"\n", - "self_ray_port: int = 32401\n", - "peer_ray_port: int = 32400\n", - "self_secretflow_port: int = 8081\n", - "peer_secretflow_port: int = 8080\n", - "self_spu_port: int = 8091\n", - "peer_spu_port: int = 8090" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "dc6dfd39", - "metadata": { - "execution": { - "iopub.execute_input": "2023-09-18T00:12:42.909843Z", - "iopub.status.busy": "2023-09-18T00:12:42.909717Z", - "iopub.status.idle": "2023-09-18T00:12:42.947263Z", - "shell.execute_reply": "2023-09-18T00:12:42.946964Z" - }, - "papermill": { - "duration": 0.040205, - "end_time": "2023-09-18T00:12:42.948467", - "exception": false, - "start_time": "2023-09-18T00:12:42.908262", - "status": "completed" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "from secretnote.compat.secretflow.device.driver import (\n", - " SFConfigNetworked,\n", - " SFClusterConfig,\n", - " SFClusterParty,\n", - ")\n", - "\n", - "secretflow_config = SFConfigNetworked(\n", - " address=f\"localhost:{self_ray_port}\",\n", - " cluster_config=SFClusterConfig(\n", - " parties={\n", - " self_party: SFClusterParty(address=f\"127.0.0.1:{self_secretflow_port}\"),\n", - " peer_party: SFClusterParty(address=f\"127.0.0.1:{peer_secretflow_port}\"),\n", - " },\n", - " self_party=self_party,\n", - " ),\n", - " logging_level=\"debug\",\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "764bd227", - "metadata": {}, - "outputs": [], - "source": [ - "from secretnote.compat.spu import (\n", - " SPUConfig,\n", - " SPUClusterDef,\n", - " SPUNode,\n", - " SPURuntimeConfig,\n", - " SPUProtocolKind,\n", - " SPUFieldType,\n", - ")\n", - "\n", - "spu_config = SPUConfig(\n", - " cluster_def=SPUClusterDef(\n", - " nodes=[\n", - " SPUNode(party=self_party, address=f\"127.0.0.1:{self_spu_port}\"),\n", - " SPUNode(party=peer_party, address=f\"127.0.0.1:{peer_spu_port}\"),\n", - " ],\n", - " runtime_config=SPURuntimeConfig(\n", - " protocol=SPUProtocolKind.SEMI2K,\n", - " field=SPUFieldType.FM128,\n", - " ),\n", - " )\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0603e573", - "metadata": {}, - "outputs": [], - "source": [ - "import secretflow\n", - "\n", - "secretflow.shutdown()\n", - "secretflow.init(**secretflow_config.dict())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6065a829", - "metadata": {}, - "outputs": [], - "source": [ - "bob = secretflow.PYU(self_party)\n", - "alice = secretflow.PYU(peer_party)\n", - "spu = secretflow.SPU(**spu_config.dict())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0c57b3af", - "metadata": {}, - "outputs": [], - "source": [ - "import jax.numpy as jnp\n", - "import numpy as np\n", - "import secretnote.functional as sfx\n", - "\n", - "\n", - "def multiply(x: np.ndarray, y: float):\n", - " return jnp.multiply(x, y)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "629a9fed", - "metadata": {}, - "outputs": [], - "source": [ - "array = sfx.use_cleartext(alice)(...)\n", - "array = sfx.use_relocation(spu, array)()\n", - "\n", - "multiplier = sfx.use_cleartext(bob)(0.5)\n", - "multiplier = sfx.use_relocation(spu, multiplier)()\n", - "\n", - "result = sfx.use_function(spu, array, multiplier)(multiply)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "67103d66", - "metadata": {}, - "outputs": [], - "source": [ - "result_for_bob = sfx.use_relocation(bob, result)()\n", - "result_for_alice = sfx.use_relocation(alice, result)()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": ".venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.17" - }, - "papermill": { - "default_parameters": {}, - "duration": 2.249147, - "end_time": "2023-09-18T00:12:43.271623", - "environment_variables": {}, - "exception": null, - "input_path": "examples/sf-2pt-local-ray-instances.ipynb", - "output_path": "examples/sf-2pt-local-ray-instances.ipynb", - "parameters": { - "peer_party": "alice", - "self_party": "bob" - }, - "start_time": "2023-09-18T00:12:41.022476", - "version": "2.4.0" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/examples/sim.ipynb b/examples/sim.ipynb new file mode 100644 index 00000000..0cb076a0 --- /dev/null +++ b/examples/sim.ipynb @@ -0,0 +1,97 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "PRESET = os.environ.setdefault(\"PRESET\", \"test_spu_aggregator\")\n", + "SELF_PARTY = os.environ.setdefault(\"SELF_PARTY\", \"sim\")\n", + "os.environ[\"NODE_ENV\"] = \"development\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "import ray\n", + "from opentelemetry import trace\n", + "from opentelemetry.sdk.environment_variables import OTEL_SERVICE_NAME\n", + "\n", + "from secretnote.instrumentation.sdk import (\n", + " create_profiler,\n", + " inherit_tracing_context,\n", + " setup_tracing,\n", + ")\n", + "\n", + "ray.shutdown()\n", + "\n", + "os.environ[OTEL_SERVICE_NAME] = f\"{PRESET}:{SELF_PARTY}\"\n", + "\n", + "setup_tracing()\n", + "# setup_debug_exporter()\n", + "\n", + "\n", + "with trace.get_tracer(__name__).start_as_current_span(\n", + " \"sim_trace\",\n", + " context=inherit_tracing_context(),\n", + "):\n", + " with create_profiler() as profiler:\n", + " with open(f\"./presets/{PRESET}/_world.py\") as f:\n", + " _world = compile(f.read(), str(Path(f.name).resolve()), \"exec\")\n", + " with open(f\"./presets/{PRESET}/_algorithm.py\") as f:\n", + " _algorithm = compile(f.read(), str(Path(f.name).resolve()), \"exec\")\n", + " exec(_world, globals())\n", + " exec(_algorithm, globals())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from secretnote.display import visualize_run\n", + "\n", + "visualize_run(profiler)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.17" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/examples/spu-over-local-network.ipynb b/examples/spu-over-local-network.ipynb deleted file mode 100644 index 86706cee..00000000 --- a/examples/spu-over-local-network.ipynb +++ /dev/null @@ -1,325 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# SPU over local network\n", - "\n", - "This example spins up a 2-party SPU network. Both parties listen on 0.0.0.0, and will connect to each other over the local network instead of through the loopback interface (localhost).\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "First, start an example 2-party SecretFlow network in simulation mode:\n" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2023-09-18 07:46:13,277\tINFO worker.py:1529 -- Started a local Ray instance. View the dashboard at \u001b[1m\u001b[32mhttp://127.0.0.1:8265 \u001b[39m\u001b[22m\n" - ] - } - ], - "source": [ - "import secretflow\n", - "from secretnote.compat.secretflow.device.driver import SFConfigSimulationFullyManaged\n", - "\n", - "secretflow.shutdown()\n", - "\n", - "secretflow_config = SFConfigSimulationFullyManaged(parties=[\"alice\", \"bob\"])\n", - "secretflow.init(**secretflow_config.dict())\n", - "\n", - "alice = secretflow.PYU(\"alice\")\n", - "bob = secretflow.PYU(\"bob\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Figure out what the local IP address is for this machine. Depending on your network configuration, there may be several applicable addresses (e.g. one on Wireless and one on Ethernet). We will be using [`netifaces`][netifaces] to find the first interface that is up and has an `AF_INET` address assigned to it:\n", - "\n", - "[netifaces]: https://pypi.org/project/netifaces/\n" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "import netifaces" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Using interface en0 with address 30.231.120.44\n" - ] - } - ], - "source": [ - "for iface in netifaces.interfaces():\n", - " if iface.startswith(\"lo\"):\n", - " # skip loopback interface\n", - " continue\n", - " if iface.startswith(\"vbox\"):\n", - " # skip virtualbox interfaces\n", - " continue\n", - " details = netifaces.ifaddresses(iface)\n", - " if netifaces.AF_INET not in details:\n", - " # skip interfaces without an IP address\n", - " continue\n", - " inet_addr = details[netifaces.AF_INET][0][\"addr\"]\n", - " break\n", - "else:\n", - " raise RuntimeError(\"No suitable network interface found\")\n", - "\n", - "print(f\"Using interface {iface} with address {inet_addr}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Find two ports that are available on this machine. We will be using [`portpicker`][portpicker]:\n", - "\n", - "[portpicker]: https://pypi.org/project/portpicker/\n" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "import portpicker" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Network layout:\n", - "alice: 127.0.0.1:54238 -> 30.231.120.44:54238\n", - "bob: 127.0.0.1:54239 -> 30.231.120.44:54239\n" - ] - } - ], - "source": [ - "port_for_alice = portpicker.pick_unused_port()\n", - "port_for_bob = portpicker.pick_unused_port()\n", - "\n", - "alice_bind_to = f\"127.0.0.1:{port_for_alice}\"\n", - "bob_bind_to = f\"127.0.0.1:{port_for_bob}\"\n", - "\n", - "connect_to_alice_via = f\"{inet_addr}:{port_for_alice}\"\n", - "connect_to_bob_via = f\"{inet_addr}:{port_for_bob}\"\n", - "\n", - "print(\n", - " f\"\"\"Network layout:\n", - "alice: {alice_bind_to} -> {connect_to_alice_via}\n", - "bob: {bob_bind_to} -> {connect_to_bob_via}\"\"\"\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Spin up the SPU network:\n" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "from secretnote.compat.spu import (\n", - " SPUClusterDef,\n", - " SPUConfig,\n", - " SPURuntimeConfig,\n", - " SPUProtocolKind,\n", - " SPUFieldType,\n", - " SPUNode,\n", - ")\n", - "\n", - "mpc_config = SPUConfig(\n", - " cluster_def=SPUClusterDef(\n", - " nodes=[\n", - " SPUNode(\n", - " party=\"alice\",\n", - " listen_addr=alice_bind_to,\n", - " address=connect_to_alice_via,\n", - " ),\n", - " SPUNode(\n", - " party=\"bob\",\n", - " listen_addr=bob_bind_to,\n", - " address=connect_to_bob_via,\n", - " ),\n", - " ],\n", - " runtime_config=SPURuntimeConfig(\n", - " protocol=SPUProtocolKind.SEMI2K,\n", - " field=SPUFieldType.FM128,\n", - " ),\n", - " ),\n", - ")\n", - "\n", - "mpc = secretflow.SPU(**mpc_config.dict())\n", - "mpc.init()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Run an example JAX program on the SPU network:\n" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "import jax.numpy as jnp" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "def dot(x, y):\n", - " return jnp.dot(x, y)" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:jax._src.xla_bridge:Unable to initialize backend 'cuda': module 'jaxlib.xla_extension' has no attribute 'GpuAllocatorConfig'\n", - "INFO:jax._src.xla_bridge:Unable to initialize backend 'rocm': module 'jaxlib.xla_extension' has no attribute 'GpuAllocatorConfig'\n", - "INFO:jax._src.xla_bridge:Unable to initialize backend 'tpu': module 'jaxlib.xla_extension' has no attribute 'get_tpu_client'\n", - "INFO:jax._src.xla_bridge:Unable to initialize backend 'plugin': xla_extension has no attributes named get_plugin_device_client. Compile TensorFlow with //tensorflow/compiler/xla/python:enable_plugin_device set to true (defaults to false) to enable this.\n", - "\u001b[2m\u001b[36m(_run pid=15138)\u001b[0m INFO:jax._src.xla_bridge:Unable to initialize backend 'cuda': module 'jaxlib.xla_extension' has no attribute 'GpuAllocatorConfig'\n", - "\u001b[2m\u001b[36m(_run pid=15138)\u001b[0m INFO:jax._src.xla_bridge:Unable to initialize backend 'rocm': module 'jaxlib.xla_extension' has no attribute 'GpuAllocatorConfig'\n", - "\u001b[2m\u001b[36m(_run pid=15138)\u001b[0m INFO:jax._src.xla_bridge:Unable to initialize backend 'tpu': module 'jaxlib.xla_extension' has no attribute 'get_tpu_client'\n", - "\u001b[2m\u001b[36m(_run pid=15138)\u001b[0m INFO:jax._src.xla_bridge:Unable to initialize backend 'plugin': xla_extension has no attributes named get_plugin_device_client. Compile TensorFlow with //tensorflow/compiler/xla/python:enable_plugin_device set to true (defaults to false) to enable this.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(SPURuntime pid=15173)\u001b[0m 2023-09-18 07:46:16.382 [info] [default_brpc_retry_policy.cc:DoRetry:52] socket error, sleep=1000000us and retry\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(_run pid=15169)\u001b[0m INFO:jax._src.xla_bridge:Unable to initialize backend 'cuda': module 'jaxlib.xla_extension' has no attribute 'GpuAllocatorConfig'\n", - "\u001b[2m\u001b[36m(_run pid=15169)\u001b[0m INFO:jax._src.xla_bridge:Unable to initialize backend 'rocm': module 'jaxlib.xla_extension' has no attribute 'GpuAllocatorConfig'\n", - "\u001b[2m\u001b[36m(_run pid=15169)\u001b[0m INFO:jax._src.xla_bridge:Unable to initialize backend 'tpu': module 'jaxlib.xla_extension' has no attribute 'get_tpu_client'\n", - "\u001b[2m\u001b[36m(_run pid=15169)\u001b[0m INFO:jax._src.xla_bridge:Unable to initialize backend 'plugin': xla_extension has no attributes named get_plugin_device_client. Compile TensorFlow with //tensorflow/compiler/xla/python:enable_plugin_device set to true (defaults to false) to enable this.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[2m\u001b[36m(SPURuntime pid=15173)\u001b[0m 2023-09-18 07:46:17.382 [info] [default_brpc_retry_policy.cc:LogHttpDetail:33] cntl ErrorCode '64', http status code '200', response header '', error msg '[E61]Fail to connect Socket{id=0 addr=30.231.120.44:54238} (0x0x1380f8000): Connection refused [R1][E64]Not connected to 30.231.120.44:54238 yet, server_id=0'\n", - "\u001b[2m\u001b[36m(SPURuntime pid=15173)\u001b[0m 2023-09-18 07:46:17.383 [info] [default_brpc_retry_policy.cc:DoRetry:75] aggressive retry, sleep=1000000us and retry\n", - "\u001b[2m\u001b[36m(SPURuntime pid=15173)\u001b[0m 2023-09-18 07:46:18.383 [info] [default_brpc_retry_policy.cc:LogHttpDetail:33] cntl ErrorCode '64', http status code '200', response header '', error msg '[E61]Fail to connect Socket{id=0 addr=30.231.120.44:54238} (0x0x1380f8000): Connection refused [R1][E64]Not connected to 30.231.120.44:54238 yet, server_id=0 [R2][E64]Not connected to 30.231.120.44:54238 yet, server_id=0'\n", - "\u001b[2m\u001b[36m(SPURuntime pid=15173)\u001b[0m 2023-09-18 07:46:18.383 [info] [default_brpc_retry_policy.cc:DoRetry:75] aggressive retry, sleep=1000000us and retry\n", - "\u001b[2m\u001b[36m(SPURuntime pid=15171)\u001b[0m 2023-09-18 07:46:18.419 [info] [default_brpc_retry_policy.cc:DoRetry:71] not retry for reached rcp timeout, ErrorCode '1008', error msg '[E1008]Reached timeout=2000ms @30.231.120.44:54239'\n" - ] - }, - { - "data": { - "text/plain": [ - "array(735, dtype=int32)" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "product = mpc(dot)(\n", - " secretflow.to(alice, jnp.arange(10)).to(mpc),\n", - " secretflow.to(bob, jnp.arange(10, 20)).to(mpc),\n", - ")\n", - "secretflow.reveal(product)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Stop everything:\n" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "mpc.shutdown()\n", - "secretflow.shutdown()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": ".venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.17" - }, - "orig_nbformat": 4 - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/examples/spu_lr.ipynb b/examples/spu_lr.ipynb new file mode 100644 index 00000000..3694fea5 --- /dev/null +++ b/examples/spu_lr.ipynb @@ -0,0 +1,236 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from secretnote.formal.locations import (\n", + " SPUFieldType,\n", + " SPUProtocolKind,\n", + " SymbolicPYU,\n", + " SymbolicSPU,\n", + " SymbolicWorld,\n", + ")\n", + "from secretflow.device import SPUCompilerNumReturnsPolicy\n", + "\n", + "sym_world = SymbolicWorld(world=frozenset((\"alice\", \"bob\")))\n", + "sym_alice = SymbolicPYU(\"alice\")\n", + "sym_bob = SymbolicPYU(\"bob\")\n", + "sym_spu = SymbolicSPU(\n", + " world=frozenset((\"alice\", \"bob\")),\n", + " protocol=SPUProtocolKind.SEMI2K,\n", + " field=SPUFieldType.FM128,\n", + " fxp_fraction_bits=0,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import ray\n", + "from secretnote.formal.locations import SFConfigSimulation, PortBinding\n", + "\n", + "ray.shutdown()\n", + "\n", + "sym_world.reify(SFConfigSimulation())\n", + "\n", + "alice = sym_alice.reify()\n", + "bob = sym_bob.reify()\n", + "spu = sym_spu.reify(\n", + " alice=PortBinding(announced_as=\"127.0.0.1:32767\"),\n", + " bob=PortBinding(announced_as=\"127.0.0.1:32768\"),\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from secretnote.instrumentation.sdk import create_profiler\n", + "\n", + "profiler = create_profiler()\n", + "profiler.start()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "from sklearn.datasets import load_breast_cancer\n", + "from sklearn.model_selection import train_test_split\n", + "\n", + "\n", + "def load_dataset_for_training(rand_key: int, party_id: int):\n", + " x, y = load_breast_cancer(return_X_y=True)\n", + " x = (x - np.min(x)) / (np.max(x) - np.min(x))\n", + " x_train, x_test, y_train, y_test = train_test_split(\n", + " x,\n", + " y,\n", + " test_size=0.2,\n", + " random_state=rand_key,\n", + " )\n", + " if party_id == 0:\n", + " return x_train[:, :15], None\n", + " else:\n", + " return x_train[:, 15:], y_train\n", + "\n", + "\n", + "def load_dataset_for_testing(rand_key: int):\n", + " x, y = load_breast_cancer(return_X_y=True)\n", + " x = (x - np.min(x)) / (np.max(x) - np.min(x))\n", + " x_train, x_test, y_train, y_test = train_test_split(\n", + " x,\n", + " y,\n", + " test_size=0.2,\n", + " random_state=rand_key,\n", + " )\n", + " return x_test, y_test" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import jax\n", + "from sklearn.metrics import roc_auc_score\n", + "\n", + "\n", + "def sigmoid(x):\n", + " return 1 / (1 + jax.numpy.exp(-x))\n", + "\n", + "\n", + "def predict(W, b, inputs):\n", + " \"\"\"Outputs probability of a label being true.\"\"\"\n", + " return sigmoid(jax.numpy.dot(inputs, W) + b)\n", + "\n", + "\n", + "def loss(W, b, inputs, targets):\n", + " \"\"\"Training loss is the negative log-likelihood of the training examples.\"\"\"\n", + " preds = predict(W, b, inputs)\n", + " label_probs = preds * targets + (1 - preds) * (1 - targets)\n", + " return -jax.numpy.mean(jax.numpy.log(label_probs))\n", + "\n", + "\n", + "def concatenate_samples(x1, x2):\n", + " return jax.numpy.concatenate([x1, x2], axis=1)\n", + "\n", + "\n", + "def validate_model(weights, bias, X_test, y_test):\n", + " y_pred = predict(weights, bias, X_test)\n", + " return roc_auc_score(y_test, y_pred)\n", + "\n", + "\n", + "def logistic_regression(weights, bias, x1, x2, y, epochs, learning_rate):\n", + " x = concatenate_samples(x1, x2)\n", + " for _ in range(10):\n", + " gradients = jax.grad(loss, (0, 1))(weights, bias, x, y)\n", + " weights -= learning_rate * gradients[0]\n", + " bias -= learning_rate * gradients[1]\n", + " return weights, bias" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "x1, _ = alice(load_dataset_for_training, num_returns=2)(rand_key=42, party_id=0)\n", + "x2, y = bob(load_dataset_for_training, num_returns=2)(rand_key=42, party_id=1)\n", + "\n", + "x1 = x1.to(spu)\n", + "x2 = x2.to(spu)\n", + "y = y.to(spu)\n", + "\n", + "weights = jax.numpy.zeros((30,))\n", + "bias = 0.0\n", + "\n", + "weights = alice(lambda x: x)(weights)\n", + "weights = weights.to(spu)\n", + "\n", + "bias = alice(lambda x: x)(bias)\n", + "bias = bias.to(spu)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "epochs = 10\n", + "\n", + "weights, bias = spu(\n", + " logistic_regression,\n", + " static_argnames=[\"epochs\"],\n", + " num_returns_policy=SPUCompilerNumReturnsPolicy.FROM_USER,\n", + " user_specified_num_returns=2,\n", + ")(weights, bias, x1, x2, y, epochs=10, learning_rate=1e-2)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from secretflow import reveal\n", + "\n", + "weights = weights.to(bob)\n", + "bias = bias.to(bob)\n", + "x_test, y_test = bob(load_dataset_for_testing, num_returns=2)(rand_key=42)\n", + "\n", + "auc = reveal(bob(validate_model)(weights, bias, x_test, y_test))\n", + "weights = reveal(weights)\n", + "bias = reveal(bias)\n", + "\n", + "print(f\"{weights=}\")\n", + "print(f\"{bias=}\")\n", + "print(f\"{auc=}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "profiler.stop()\n", + "profiler.visualize()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.18" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/examples/spu_lr_unrolled.ipynb b/examples/spu_lr_unrolled.ipynb new file mode 100644 index 00000000..aac3fff0 --- /dev/null +++ b/examples/spu_lr_unrolled.ipynb @@ -0,0 +1,257 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from secretnote.formal.locations import (\n", + " SPUFieldType,\n", + " SPUProtocolKind,\n", + " SymbolicPYU,\n", + " SymbolicSPU,\n", + " SymbolicWorld,\n", + ")\n", + "from secretflow.device import SPUCompilerNumReturnsPolicy\n", + "\n", + "sym_world = SymbolicWorld(world=frozenset((\"alice\", \"bob\")))\n", + "sym_alice = SymbolicPYU(\"alice\")\n", + "sym_bob = SymbolicPYU(\"bob\")\n", + "sym_spu = SymbolicSPU(\n", + " world=frozenset((\"alice\", \"bob\")),\n", + " protocol=SPUProtocolKind.SEMI2K,\n", + " field=SPUFieldType.FM128,\n", + " fxp_fraction_bits=0,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import ray\n", + "from secretnote.formal.locations import SFConfigSimulation, PortBinding\n", + "\n", + "ray.shutdown()\n", + "\n", + "sym_world.reify(SFConfigSimulation())\n", + "\n", + "alice = sym_alice.reify()\n", + "bob = sym_bob.reify()\n", + "spu = sym_spu.reify(\n", + " alice=PortBinding(announced_as=\"127.0.0.1:32767\"),\n", + " bob=PortBinding(announced_as=\"127.0.0.1:32768\"),\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from secretnote.instrumentation.sdk import create_profiler, setup_tracing\n", + "\n", + "setup_tracing()\n", + "\n", + "profiler = create_profiler()\n", + "profiler.start()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "from sklearn.datasets import load_breast_cancer\n", + "from sklearn.model_selection import train_test_split\n", + "\n", + "\n", + "def load_dataset_for_training(rand_key: int, party_id: int):\n", + " x, y = load_breast_cancer(return_X_y=True)\n", + " x = (x - np.min(x)) / (np.max(x) - np.min(x))\n", + " x_train, x_test, y_train, y_test = train_test_split(\n", + " x,\n", + " y,\n", + " test_size=0.2,\n", + " random_state=rand_key,\n", + " )\n", + " if party_id == 0:\n", + " return x_train[:, :15], None\n", + " else:\n", + " return x_train[:, 15:], y_train\n", + "\n", + "\n", + "def load_dataset_for_testing(rand_key: int):\n", + " x, y = load_breast_cancer(return_X_y=True)\n", + " x = (x - np.min(x)) / (np.max(x) - np.min(x))\n", + " x_train, x_test, y_train, y_test = train_test_split(\n", + " x,\n", + " y,\n", + " test_size=0.2,\n", + " random_state=rand_key,\n", + " )\n", + " return x_test, y_test" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import jax\n", + "from sklearn.metrics import roc_auc_score\n", + "\n", + "\n", + "def sigmoid(x):\n", + " return 1 / (1 + jax.numpy.exp(-x))\n", + "\n", + "\n", + "def predict(W, b, inputs):\n", + " \"\"\"Outputs probability of a label being true.\"\"\"\n", + " return sigmoid(jax.numpy.dot(inputs, W) + b)\n", + "\n", + "\n", + "def loss(W, b, inputs, targets):\n", + " \"\"\"Training loss is the negative log-likelihood of the training examples.\"\"\"\n", + " preds = predict(W, b, inputs)\n", + " label_probs = preds * targets + (1 - preds) * (1 - targets)\n", + " return -jax.numpy.mean(jax.numpy.log(label_probs))\n", + "\n", + "\n", + "def train_step(W, b, x1, x2, y, learning_rate):\n", + " x = jax.numpy.concatenate([x1, x2], axis=1)\n", + " Wb_grad = grad(loss, (0, 1))(W, b, x, y)\n", + " W -= learning_rate * Wb_grad[0]\n", + " b -= learning_rate * Wb_grad[1]\n", + " return W, b\n", + "\n", + "\n", + "def fit(W, b, x1, x2, y, epochs=1, learning_rate=1e-2):\n", + " for _ in range(epochs):\n", + " W, b = train_step(W, b, x1, x2, y, learning_rate=learning_rate)\n", + " return W, b\n", + "\n", + "\n", + "def concatenate_samples(x1, x2):\n", + " return jax.numpy.concatenate([x1, x2], axis=1)\n", + "\n", + "\n", + "def grad(weights, bias, x, y, *, learning_rate):\n", + " gradients = jax.grad(loss, (0, 1))(weights, bias, x, y)\n", + " weights -= learning_rate * gradients[0]\n", + " bias -= learning_rate * gradients[1]\n", + " return weights, bias\n", + "\n", + "\n", + "def validate_model(weights, bias, X_test, y_test):\n", + " y_pred = predict(weights, bias, X_test)\n", + " return roc_auc_score(y_test, y_pred)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "x1, _ = alice(load_dataset_for_training, num_returns=2)(rand_key=42, party_id=0)\n", + "x2, y = bob(load_dataset_for_training, num_returns=2)(rand_key=42, party_id=1)\n", + "\n", + "x1 = x1.to(spu)\n", + "x2 = x2.to(spu)\n", + "y = y.to(spu)\n", + "\n", + "weights = jax.numpy.zeros((30,))\n", + "bias = 0.0\n", + "\n", + "weights = alice(lambda x: x)(weights)\n", + "weights = weights.to(spu)\n", + "\n", + "bias = alice(lambda x: x)(bias)\n", + "bias = bias.to(spu)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "epochs = 10\n", + "\n", + "x = spu(concatenate_samples)(x1, x2)\n", + "\n", + "for _ in range(10):\n", + " weights, bias = spu(\n", + " grad,\n", + " num_returns_policy=SPUCompilerNumReturnsPolicy.FROM_USER,\n", + " user_specified_num_returns=2,\n", + " )(\n", + " weights,\n", + " bias,\n", + " x,\n", + " y,\n", + " learning_rate=1e-2,\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from secretflow import reveal\n", + "\n", + "weights = weights.to(bob)\n", + "bias = bias.to(bob)\n", + "x_test, y_test = bob(load_dataset_for_testing, num_returns=2)(rand_key=42)\n", + "auc = reveal(bob(validate_model)(weights, bias, x_test, y_test))\n", + "\n", + "print(f\"{weights=}\")\n", + "print(f\"{bias=}\")\n", + "print(f\"{auc=}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from secretnote.display import visualize_run\n", + "\n", + "profiler.stop()\n", + "visualize_run(profiler)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.17" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/examples/start-ray.sh b/examples/start-ray.sh index 9f4a0ccf..b190ce11 100755 --- a/examples/start-ray.sh +++ b/examples/start-ray.sh @@ -1,12 +1,12 @@ #!/usr/bin/env bash ray start --block --head --port 32400 \ - --dashboard-port 8265 \ + --dashboard-port 8266 \ --include-dashboard True & PID1=$! ray start --block --head --port 32401 \ - --dashboard-port 8266 \ + --dashboard-port 8267 \ --include-dashboard True & PID2=$! diff --git a/nx.json b/nx.json index 12027666..a3fded99 100644 --- a/nx.json +++ b/nx.json @@ -15,80 +15,66 @@ // ${projectRoot} = root of package in which the task is defined "namedInputs": { ":default": ["{projectRoot}/**/*"], - ":eslint": [ - "{workspaceRoot}/.eslintrc.{js,cjs}", - "{workspaceRoot}/.eslintrc.*.{js,cjs}", - "{workspaceRoot}/.eslintignore", - "{workspaceRoot}/.remarkrc.mjs", - "{projectRoot}/.eslintrc.{js,cjs}" - ], + ":eslint": ["{workspaceRoot}/.eslintrc.*", "{workspaceRoot}/.eslintignore"], ":stylelint": [ - "{workspaceRoot}/.stylelintrc.{js,cjs}", - "{workspaceRoot}/.stylelintignore", - "{projectRoot}/.stylelint.{js,cjs}" - ], - ":prettier": [ - "{workspaceRoot}/.prettierrc.json", - "{workspaceRoot}/.prettierignore" + "{workspaceRoot}/.stylelintrc.*", + "{workspaceRoot}/.stylelintignore" ], - ":tsc": [ - "{workspaceRoot}/tsconfig.json", - "{workspaceRoot}/tsconfig.*.json", - "{projectRoot}/**/tsconfig.json", - "{projectRoot}/**/tsconfig.*.json" - ], - ":vitest": [ - "{workspaceRoot}/vitest.config.mts", - "{workspaceRoot}/vitest.config.mjs" - ] + ":prettier": ["{workspaceRoot}/.prettierrc.*", "{workspaceRoot}/.prettierignore"], + ":tsc": ["{workspaceRoot}/tsconfig.*"], + ":vitest": ["{workspaceRoot}/vitest.config.*", "{workspaceRoot}/vitest.config.mjs"], + ":python": ["{workspaceRoot}/pyproject.toml"] }, // targetDefaults configure default options for each task // such as inputs, outputs and dependencies // See https://nx.dev/reference/nx-json#target-defaults // The actual commands are defined in the `scripts` section in each package's package.json "targetDefaults": { - "setup": { - "inputs": [":default", ":tsc"], - "outputs": ["{projectRoot}/dist"], - "dependsOn": ["^setup"] - }, - "build": { - "inputs": [":default", ":tsc"], - "outputs": ["{projectRoot}/dist"], - "dependsOn": ["^build"] - }, - "release": { - "dependsOn": ["^build", "^release", "build"] - }, "lint:eslint": { "inputs": [":default", ":eslint"] }, "lint:stylelint": { "inputs": [":default", ":stylelint"] }, - "lint:prettier": { + "format:black": { + "inputs": [":default", ":python"] + }, + "format:prettier": { "inputs": [":default", ":prettier"] }, - "lint:tsc": { - "inputs": [":default", ":tsc"] + "lint:ruff": { + "inputs": [":default", ":python"] }, - "lint:black": { - "inputs": [":default"] + "typecheck:tsc": { + "inputs": ["^:default", ":default", ":tsc"] }, - "lint:pyright": { - "inputs": [":default"] - }, - "lint:ruff": { - "inputs": [":default"] + "typecheck:pyright": { + "inputs": ["^:default", ":default", ":python"] }, "test:vitest": { - "inputs": [":default", ":vitest"], + "inputs": ["^:default", ":default", ":vitest"], "outputs": ["{projectRoot}/coverage"] }, "test:pytest": { - "inputs": [":default"], - "outputs": ["{projectRoot}/coverage"] - } + "inputs": ["^:default", ":default", ":python"] + }, + "setup:umi": { + "inputs": [":default", ":tsc"], + "outputs": ["{projectRoot}/src/.umi"] + }, + "build": { + "inputs": ["^:default", ":default", ":tsc", ":python"], + "outputs": ["{projectRoot}/build", "{projectRoot}/dist"], + "dependsOn": ["^build"] + }, + "publish": { + "dependsOn": ["^build", "build", "^publish"] + }, + "ci:setup": {}, + "ci:javascript": {}, + "ci:python": {}, + "ci:publish": {}, + "dev": {} }, "tasksRunnerOptions": { "default": { @@ -96,17 +82,17 @@ "options": { "parallel": 8, "cacheableOperations": [ - "setup", "build", + "format:black", + "format:prettier", "lint:eslint", - "lint:stylelint", - "lint:prettier", - "lint:tsc", - "lint:black", - "lint:pyright", "lint:ruff", + "lint:stylelint", + "setup:umi", + "test:pytest", "test:vitest", - "test:pytest" + "typecheck:pyright", + "typecheck:tsc" ] } } diff --git a/package.json b/package.json index 316e6fb4..349474c8 100644 --- a/package.json +++ b/package.json @@ -1,55 +1,57 @@ { "name": "monorepo", "private": true, - "license": "MIT", + "license": "Apache-2.0", "author": "Tony Wu ", - "repository": "https://github.com/secretflow/notebook", + "repository": "https://github.com/secretflow/secretnote", "description": "Notebook suite for SecretFlow", "scripts": { "bootstrap": "./scripts/setup_all.sh", "postinstall": "is-ci || husky install", "changeset": "changeset", "clean": "git clean -fX .", - "lint:prettier": "prettier --check --write --ignore-unknown .", - "ci:javascript": "nx run-many -t lint:prettier -t lint:eslint -t lint:stylelint -t typecheck:tsc -t test:vitest", - "ci:python": "nx run-many -t lint:black -t lint:ruff -t test:pytest -t typecheck:pyright", - "ci:release": "nx run-many --nx-bail -t release" - }, - "nx": { - "includedScripts": [ - "lint:prettier" - ] + "format:black": "python -m black --check examples", + "format:prettier": "prettier --check --ignore-unknown .", + "ci:setup": "nx run-many -p tag:postinstall -t setup:umi -t build", + "ci:javascript": "nx run-many -t format:prettier -t lint:eslint -t lint:stylelint -t test:vitest -t typecheck:tsc", + "ci:python": "nx run-many -t format:black -t lint:ruff -t test:pytest -t typecheck:pyright", + "ci:publish": "nx run-many --nx-bail -t build && changeset publish && nx run-many --nx-bail -t publish" }, + "nx": {}, "devDependencies": { + "@bufbuild/buf": "^1.27.1", "@changesets/cli": "^2.26.2", - "@commitlint/cli": "^17.6.6", - "@commitlint/config-conventional": "^17.6.6", + "@commitlint/cli": "^17.8.1", + "@commitlint/config-conventional": "^17.8.1", "@secretflow/repo-utils": "workspace:^", + "@types/node": "^18.16.19", "@typescript-eslint/eslint-plugin": "^5.62.0", "@typescript-eslint/parser": "^5.62.0", "@vitest/coverage-v8": "^0.32.4", - "dotenv-cli": "^7.2.1", - "eslint": "^8.45.0", - "eslint-config-prettier": "^8.8.0", - "eslint-plugin-import": "^2.27.5", - "eslint-plugin-mdx": "^2.1.0", + "dotenv-cli": "^7.3.0", + "eslint": "^8.52.0", + "eslint-config-prettier": "^8.10.0", + "eslint-plugin-import": "^2.28.1", + "eslint-plugin-mdx": "^2.2.0", "eslint-plugin-promise": "^6.1.1", - "eslint-plugin-react": "^7.32.2", + "eslint-plugin-react": "^7.33.2", "eslint-plugin-react-hooks": "^4.6.0", "husky": "^8.0.3", "is-ci": "^3.0.1", - "jest": "^29.6.1", - "lint-staged": "^13.2.3", - "nx": "^16.5.2", - "postcss": "^8.4.26", - "prettier": "^3.0.0", - "pyright": "^1.1.323", + "jest": "^29.7.0", + "lint-staged": "^13.3.0", + "nx": "^16.10.0", + "openapi-typescript-codegen": "^0.25.0", + "postcss": "^8.4.31", + "postcss-less": "^6.0.0", + "prettier": "^3.0.3", + "pyright": "^1.1.332", "remark-directive": "^2.0.1", "remark-frontmatter": "^4.0.1", "remark-gfm": "^3.0.1", "remark-math": "^5.1.1", - "stylelint": "^15.10.1", - "stylelint-config-css-modules": "^4.2.0", + "stylelint": "^15.11.0", + "stylelint-config-css-modules": "^4.3.0", "stylelint-config-idiomatic-order": "^9.0.0", "stylelint-config-prettier": "^9.0.5", "stylelint-config-standard": "^30.0.1", @@ -60,7 +62,7 @@ "vitest": "^0.32.4" }, "engines": { - "node": ">=16.20.0" + "node": ">=18.18.0" }, "packageManager": "pnpm@8.7.6" } diff --git a/packages/repo-utils/.eslintrc.cjs b/packages/repo-utils/.eslintrc.cjs index 3c2bee46..ae4d29da 100644 --- a/packages/repo-utils/.eslintrc.cjs +++ b/packages/repo-utils/.eslintrc.cjs @@ -1,4 +1,4 @@ module.exports = { root: true, - extends: [require.resolve('../../.eslintrc.node.js')], + extends: [require.resolve('../../.eslintrc.node.cjs')], }; diff --git a/packages/repo-utils/package.json b/packages/repo-utils/package.json index df8290fe..821c2306 100644 --- a/packages/repo-utils/package.json +++ b/packages/repo-utils/package.json @@ -17,10 +17,9 @@ "repo-utils": "./bin/cli.mjs" }, "scripts": { - "setup": "tsup", - "build": "tsup", "lint:eslint": "eslint src", - "lint:tsc": "tsc --noEmit" + "typecheck:tsc": "tsc --noEmit", + "build": "tsup" }, "publishConfig": { "access": "public" @@ -41,5 +40,10 @@ "devDependencies": { "@types/node": "^18.16.19", "@types/validate-npm-package-name": "^4.0.0" + }, + "nx": { + "tags": [ + "postinstall" + ] } } diff --git a/packages/repo-utils/src/cli/index.mts b/packages/repo-utils/src/cli/index.mts index 7c3c9fde..fc148a44 100644 --- a/packages/repo-utils/src/cli/index.mts +++ b/packages/repo-utils/src/cli/index.mts @@ -3,13 +3,7 @@ import { Command } from 'commander'; import { version } from '../../package.json'; import { clone } from './clone.mjs'; -import { pyproject } from './pyproject.mjs'; const program = new Command(); -program - .name('repo-utils') - .version(version) - .addCommand(clone) - .addCommand(pyproject) - .parse(); +program.name('repo-utils').version(version).addCommand(clone).parse(); diff --git a/packages/repo-utils/src/tsup.mts b/packages/repo-utils/src/tsup.mts index 0a0f3562..080e457a 100644 --- a/packages/repo-utils/src/tsup.mts +++ b/packages/repo-utils/src/tsup.mts @@ -81,13 +81,13 @@ export function signal() { signalInstalled = true; } -export function defineOptions(options: Options): Options { +export function preset(options: Options): Options { if (options.watch) { signal(); } return { outDir: 'dist', - format: ['esm', 'cjs'], + format: ['esm'], outExtension, sourcemap: true, dts: false, diff --git a/packages/repo-utils/tsconfig.json b/packages/repo-utils/tsconfig.json index 23500130..9fd8e61a 100644 --- a/packages/repo-utils/tsconfig.json +++ b/packages/repo-utils/tsconfig.json @@ -1,4 +1,4 @@ { "extends": "../../tsconfig.node.json", - "include": ["src/**/*.mts"] + "include": ["src/**/*.mts", "package.json"] } diff --git a/packages/secretnote-ui/.eslintrc.cjs b/packages/secretnote-ui/.eslintrc.cjs new file mode 100644 index 00000000..6a5791d3 --- /dev/null +++ b/packages/secretnote-ui/.eslintrc.cjs @@ -0,0 +1,17 @@ +module.exports = { + root: true, + env: { browser: true, es2020: true }, + extends: [ + 'eslint:recommended', + 'plugin:@typescript-eslint/recommended', + 'plugin:react-hooks/recommended', + 'plugin:storybook/recommended', + require.resolve('../../.eslintrc.cjs'), + ], + ignorePatterns: ['dist', '.eslintrc.cjs'], + parser: '@typescript-eslint/parser', + plugins: ['react-refresh'], + rules: { + 'react-refresh/only-export-components': ['warn', { allowConstantExport: true }], + }, +}; diff --git a/packages/secretnote-ui/.storybook/main.ts b/packages/secretnote-ui/.storybook/main.ts new file mode 100644 index 00000000..1a6f22fc --- /dev/null +++ b/packages/secretnote-ui/.storybook/main.ts @@ -0,0 +1,19 @@ +import type { StorybookConfig } from '@storybook/react-vite'; + +const config: StorybookConfig = { + stories: ['../src/**/*.mdx', '../src/**/*.stories.@(js|jsx|mjs|ts|tsx)'], + addons: [ + '@storybook/addon-links', + '@storybook/addon-essentials', + '@storybook/addon-interactions', + ], + framework: { + name: '@storybook/react-vite', + options: {}, + }, + docs: { + autodocs: 'tag', + }, +}; + +export default config; diff --git a/packages/secretnote-ui/.storybook/preview.ts b/packages/secretnote-ui/.storybook/preview.ts new file mode 100644 index 00000000..817ac3ce --- /dev/null +++ b/packages/secretnote-ui/.storybook/preview.ts @@ -0,0 +1,15 @@ +import type { Preview } from '@storybook/react'; + +const preview: Preview = { + parameters: { + actions: { argTypesRegex: '^on[A-Z].*' }, + controls: { + matchers: { + color: /(background|color)$/i, + date: /Date$/i, + }, + }, + }, +}; + +export default preview; diff --git a/packages/secretnote-ui/README.md b/packages/secretnote-ui/README.md new file mode 100644 index 00000000..1ebe379f --- /dev/null +++ b/packages/secretnote-ui/README.md @@ -0,0 +1,27 @@ +# React + TypeScript + Vite + +This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules. + +Currently, two official plugins are available: + +- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react/README.md) uses [Babel](https://babeljs.io/) for Fast Refresh +- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh + +## Expanding the ESLint configuration + +If you are developing a production application, we recommend updating the configuration to enable type aware lint rules: + +- Configure the top-level `parserOptions` property like this: + +```js + parserOptions: { + ecmaVersion: 'latest', + sourceType: 'module', + project: ['./tsconfig.json', './tsconfig.node.json'], + tsconfigRootDir: __dirname, + }, +``` + +- Replace `plugin:@typescript-eslint/recommended` to `plugin:@typescript-eslint/recommended-type-checked` or `plugin:@typescript-eslint/strict-type-checked` +- Optionally add `plugin:@typescript-eslint/stylistic-type-checked` +- Install [eslint-plugin-react](https://github.com/jsx-eslint/eslint-plugin-react) and add `plugin:react/recommended` & `plugin:react/jsx-runtime` to the `extends` list diff --git a/packages/secretnote-ui/package.json b/packages/secretnote-ui/package.json new file mode 100644 index 00000000..8eb9e0ad --- /dev/null +++ b/packages/secretnote-ui/package.json @@ -0,0 +1,81 @@ +{ + "name": "@secretflow/secretnote-ui", + "private": true, + "version": "0.0.0", + "license": "Apache-2.0", + "author": "Tony Wu ", + "repository": "https://github.com/secretflow/secretnote/tree/main/packages/secretnote-ui", + "bugs": "https://github.com/secretflow/secretnote/issues", + "files": [ + "dist" + ], + "type": "module", + "exports": { + ".": { + "import": "./dist/esm/index.js", + "default": "./dist/esm/index.js" + }, + "./bundled": { + "import": "./dist/esm-bundled/index.js", + "default": "./dist/esm-bundled/index.js" + }, + "./browser": { + "import": "./dist/browser/index.js", + "default": "./dist/browser/index.js" + } + }, + "scripts": { + "lint:eslint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0", + "typecheck:tsc": "tsc --noEmit", + "build": "vite build --mode=esm && vite build --mode=browser && vite build --mode=esm-bundled", + "dev": "NODE_ENV=development vite build --mode=development --watch" + }, + "dependencies": { + "@antv/algorithm": "^0.1.26", + "@antv/g6": "^4.8.23", + "@antv/graphlib": "^2.0.2", + "color": "^4.2.3", + "d3": "^7.8.5", + "lodash": "^4.17.21", + "styled-components": "^6.1.0", + "yaml": "^2.3.4" + }, + "peerDependencies": { + "antd": "^5.10.2", + "react": "^18.2.0", + "react-dom": "^18.2.0" + }, + "devDependencies": { + "@storybook/addon-essentials": "^7.5.1", + "@storybook/addon-interactions": "^7.5.1", + "@storybook/addon-links": "^7.5.1", + "@storybook/blocks": "^7.5.1", + "@storybook/react": "^7.5.1", + "@storybook/react-vite": "^7.5.1", + "@storybook/testing-library": "^0.2.2", + "@types/color": "^3.0.5", + "@types/d3": "^7.4.2", + "@types/lodash": "^4.14.200", + "@types/react": "^18.2.15", + "@types/react-dom": "^18.2.7", + "@typescript-eslint/eslint-plugin": "^6.0.0", + "@typescript-eslint/parser": "^6.0.0", + "@vitejs/plugin-react-swc": "^3.3.2", + "eslint-plugin-react-hooks": "^4.6.0", + "eslint-plugin-react-refresh": "^0.4.3", + "eslint-plugin-storybook": "^0.6.15", + "prop-types": "^15.8.1", + "react": "^18.2.0", + "react-dom": "^18.2.0", + "storybook": "^7.5.1", + "vite": "^4.4.5" + }, + "publishConfig": { + "access": "public" + }, + "nx": { + "tags": [ + "postinstall" + ] + } +} diff --git a/packages/secretnote-ui/src/.openapi-stubs/index.ts b/packages/secretnote-ui/src/.openapi-stubs/index.ts new file mode 100644 index 00000000..effa4968 --- /dev/null +++ b/packages/secretnote-ui/src/.openapi-stubs/index.ts @@ -0,0 +1,43 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +export type { _ParameterKind } from './models/_ParameterKind'; +export type { ArgumentEdge } from './models/ArgumentEdge'; +export type { DictSnapshot } from './models/DictSnapshot'; +export type { ExecExpression } from './models/ExecExpression'; +export type { Frame } from './models/Frame'; +export type { FrameInfoSnapshot } from './models/FrameInfoSnapshot'; +export type { FrameSnapshot } from './models/FrameSnapshot'; +export type { FunctionCheckpoint } from './models/FunctionCheckpoint'; +export type { FunctionInfo } from './models/FunctionInfo'; +export type { FunctionNode } from './models/FunctionNode'; +export type { FunctionParameter } from './models/FunctionParameter'; +export type { FunctionSignature } from './models/FunctionSignature'; +export type { FunctionSnapshot } from './models/FunctionSnapshot'; +export type { Graph } from './models/Graph'; +export type { HTTPValidationError } from './models/HTTPValidationError'; +export type { ListSnapshot } from './models/ListSnapshot'; +export type { LocalObject } from './models/LocalObject'; +export type { LocalObjectNode } from './models/LocalObjectNode'; +export type { LogicalLocation } from './models/LogicalLocation'; +export type { MoveExpression } from './models/MoveExpression'; +export type { NoneSnapshot } from './models/NoneSnapshot'; +export type { ObjectSnapshot } from './models/ObjectSnapshot'; +export type { Reference } from './models/Reference'; +export type { ReferenceEdge } from './models/ReferenceEdge'; +export type { ReferenceMap } from './models/ReferenceMap'; +export type { RemoteLocationSnapshot } from './models/RemoteLocationSnapshot'; +export type { RemoteObject } from './models/RemoteObject'; +export type { RemoteObjectNode } from './models/RemoteObjectNode'; +export type { RemoteObjectSnapshot } from './models/RemoteObjectSnapshot'; +export type { ReturnEdge } from './models/ReturnEdge'; +export type { RevealEdge } from './models/RevealEdge'; +export type { RevealExpression } from './models/RevealExpression'; +export type { RevealNode } from './models/RevealNode'; +export type { Semantics } from './models/Semantics'; +export type { Timeline } from './models/Timeline'; +export type { TransformEdge } from './models/TransformEdge'; +export type { ValidationError } from './models/ValidationError'; +export type { Visualization } from './models/Visualization'; diff --git a/packages/secretnote-ui/src/.openapi-stubs/models/ArgumentEdge.ts b/packages/secretnote-ui/src/.openapi-stubs/models/ArgumentEdge.ts new file mode 100644 index 00000000..52ae586a --- /dev/null +++ b/packages/secretnote-ui/src/.openapi-stubs/models/ArgumentEdge.ts @@ -0,0 +1,12 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +export type ArgumentEdge = { + source: string; + target: string; + kind?: 'argument'; + name?: string; +}; + diff --git a/packages/secretnote-ui/src/.openapi-stubs/models/DictSnapshot.ts b/packages/secretnote-ui/src/.openapi-stubs/models/DictSnapshot.ts new file mode 100644 index 00000000..840de673 --- /dev/null +++ b/packages/secretnote-ui/src/.openapi-stubs/models/DictSnapshot.ts @@ -0,0 +1,19 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { ReferenceMap } from './ReferenceMap'; + +/** + * Helper class that provides a standard way to create an ABC using + * inheritance. + */ +export type DictSnapshot = { + ref: string; + kind?: 'dict'; + type: string; + snapshot: string; + values?: ReferenceMap; +}; + diff --git a/packages/secretnote-ui/src/.openapi-stubs/models/ExecExpression.ts b/packages/secretnote-ui/src/.openapi-stubs/models/ExecExpression.ts new file mode 100644 index 00000000..d24cc7b4 --- /dev/null +++ b/packages/secretnote-ui/src/.openapi-stubs/models/ExecExpression.ts @@ -0,0 +1,18 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { LocalObject } from './LocalObject'; +import type { LogicalLocation } from './LogicalLocation'; +import type { RemoteObject } from './RemoteObject'; + +export type ExecExpression = { + kind?: 'exec'; + function: LocalObject; + location: LogicalLocation; + boundvars?: Array<(LocalObject | RemoteObject)>; + freevars?: Array<(LocalObject | RemoteObject)>; + results?: Array<(LocalObject | RemoteObject)>; +}; + diff --git a/packages/secretnote-ui/src/.openapi-stubs/models/Frame.ts b/packages/secretnote-ui/src/.openapi-stubs/models/Frame.ts new file mode 100644 index 00000000..69785552 --- /dev/null +++ b/packages/secretnote-ui/src/.openapi-stubs/models/Frame.ts @@ -0,0 +1,25 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { ExecExpression } from './ExecExpression'; +import type { FunctionCheckpoint } from './FunctionCheckpoint'; +import type { MoveExpression } from './MoveExpression'; +import type { Reference } from './Reference'; +import type { RevealExpression } from './RevealExpression'; + +export type Frame = { + span_id: string; + parent_span_id?: string; + start_time: string; + end_time: string; + epoch?: number; + checkpoints?: Array; + function?: Reference; + frame?: Reference; + retval?: Reference; + expression?: (ExecExpression | MoveExpression | RevealExpression); + inner_frames?: Array; +}; + diff --git a/packages/secretnote-ui/src/.openapi-stubs/models/FrameInfoSnapshot.ts b/packages/secretnote-ui/src/.openapi-stubs/models/FrameInfoSnapshot.ts new file mode 100644 index 00000000..7fbded69 --- /dev/null +++ b/packages/secretnote-ui/src/.openapi-stubs/models/FrameInfoSnapshot.ts @@ -0,0 +1,19 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +/** + * Helper class that provides a standard way to create an ABC using + * inheritance. + */ +export type FrameInfoSnapshot = { + ref: string; + kind?: 'frame_info'; + type: string; + filename: string; + lineno: number; + func: string; + code?: string; +}; + diff --git a/packages/secretnote-ui/src/.openapi-stubs/models/FrameSnapshot.ts b/packages/secretnote-ui/src/.openapi-stubs/models/FrameSnapshot.ts new file mode 100644 index 00000000..4b05bbde --- /dev/null +++ b/packages/secretnote-ui/src/.openapi-stubs/models/FrameSnapshot.ts @@ -0,0 +1,22 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { ReferenceMap } from './ReferenceMap'; + +/** + * Helper class that provides a standard way to create an ABC using + * inheritance. + */ +export type FrameSnapshot = { + ref: string; + kind?: 'frame'; + type: string; + local_vars?: ReferenceMap; + global_vars?: ReferenceMap; + outer_frames?: ReferenceMap; + module?: string; + func: string; +}; + diff --git a/packages/secretnote-ui/src/.openapi-stubs/models/FunctionCheckpoint.ts b/packages/secretnote-ui/src/.openapi-stubs/models/FunctionCheckpoint.ts new file mode 100644 index 00000000..5b67ab88 --- /dev/null +++ b/packages/secretnote-ui/src/.openapi-stubs/models/FunctionCheckpoint.ts @@ -0,0 +1,13 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { FunctionInfo } from './FunctionInfo'; +import type { Semantics } from './Semantics'; + +export type FunctionCheckpoint = { + function: FunctionInfo; + semantics?: Semantics; +}; + diff --git a/packages/secretnote-ui/src/.openapi-stubs/models/FunctionInfo.ts b/packages/secretnote-ui/src/.openapi-stubs/models/FunctionInfo.ts new file mode 100644 index 00000000..3fb15c06 --- /dev/null +++ b/packages/secretnote-ui/src/.openapi-stubs/models/FunctionInfo.ts @@ -0,0 +1,11 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +export type FunctionInfo = { + code_hash: string; + module: string; + name: string; +}; + diff --git a/packages/secretnote-ui/src/.openapi-stubs/models/FunctionNode.ts b/packages/secretnote-ui/src/.openapi-stubs/models/FunctionNode.ts new file mode 100644 index 00000000..3e0230d1 --- /dev/null +++ b/packages/secretnote-ui/src/.openapi-stubs/models/FunctionNode.ts @@ -0,0 +1,18 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { LocalObject } from './LocalObject'; +import type { LogicalLocation } from './LogicalLocation'; + +export type FunctionNode = { + id: string; + epoch: number; + order?: number; + kind?: 'function'; + function: LocalObject; + stackframe?: LocalObject; + location: LogicalLocation; +}; + diff --git a/packages/secretnote-ui/src/.openapi-stubs/models/FunctionParameter.ts b/packages/secretnote-ui/src/.openapi-stubs/models/FunctionParameter.ts new file mode 100644 index 00000000..68f1af71 --- /dev/null +++ b/packages/secretnote-ui/src/.openapi-stubs/models/FunctionParameter.ts @@ -0,0 +1,13 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { _ParameterKind } from './_ParameterKind'; + +export type FunctionParameter = { + name: string; + kind: _ParameterKind; + annotation?: string; +}; + diff --git a/packages/secretnote-ui/src/.openapi-stubs/models/FunctionSignature.ts b/packages/secretnote-ui/src/.openapi-stubs/models/FunctionSignature.ts new file mode 100644 index 00000000..96e3b7fb --- /dev/null +++ b/packages/secretnote-ui/src/.openapi-stubs/models/FunctionSignature.ts @@ -0,0 +1,12 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { FunctionParameter } from './FunctionParameter'; + +export type FunctionSignature = { + parameters?: Array; + return_annotation?: string; +}; + diff --git a/packages/secretnote-ui/src/.openapi-stubs/models/FunctionSnapshot.ts b/packages/secretnote-ui/src/.openapi-stubs/models/FunctionSnapshot.ts new file mode 100644 index 00000000..39f9894f --- /dev/null +++ b/packages/secretnote-ui/src/.openapi-stubs/models/FunctionSnapshot.ts @@ -0,0 +1,29 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { FunctionSignature } from './FunctionSignature'; +import type { ReferenceMap } from './ReferenceMap'; + +/** + * Helper class that provides a standard way to create an ABC using + * inheritance. + */ +export type FunctionSnapshot = { + ref: string; + kind?: 'function'; + type: string; + bytecode_hash?: string; + module?: string; + name: string; + signature?: FunctionSignature; + filename?: string; + firstlineno?: number; + source?: string; + docstring?: string; + default_args?: ReferenceMap; + closure_vars?: ReferenceMap; + global_vars?: ReferenceMap; +}; + diff --git a/packages/secretnote-ui/src/.openapi-stubs/models/Graph.ts b/packages/secretnote-ui/src/.openapi-stubs/models/Graph.ts new file mode 100644 index 00000000..71ba4aae --- /dev/null +++ b/packages/secretnote-ui/src/.openapi-stubs/models/Graph.ts @@ -0,0 +1,20 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { ArgumentEdge } from './ArgumentEdge'; +import type { FunctionNode } from './FunctionNode'; +import type { LocalObjectNode } from './LocalObjectNode'; +import type { ReferenceEdge } from './ReferenceEdge'; +import type { RemoteObjectNode } from './RemoteObjectNode'; +import type { ReturnEdge } from './ReturnEdge'; +import type { RevealEdge } from './RevealEdge'; +import type { RevealNode } from './RevealNode'; +import type { TransformEdge } from './TransformEdge'; + +export type Graph = { + nodes?: Array<(LocalObjectNode | RemoteObjectNode | FunctionNode | RevealNode)>; + edges?: Array<(ArgumentEdge | ReturnEdge | TransformEdge | ReferenceEdge | RevealEdge)>; +}; + diff --git a/packages/secretnote-ui/src/.openapi-stubs/models/HTTPValidationError.ts b/packages/secretnote-ui/src/.openapi-stubs/models/HTTPValidationError.ts new file mode 100644 index 00000000..193eeb3f --- /dev/null +++ b/packages/secretnote-ui/src/.openapi-stubs/models/HTTPValidationError.ts @@ -0,0 +1,11 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { ValidationError } from './ValidationError'; + +export type HTTPValidationError = { + detail?: Array; +}; + diff --git a/packages/secretnote-ui/src/.openapi-stubs/models/ListSnapshot.ts b/packages/secretnote-ui/src/.openapi-stubs/models/ListSnapshot.ts new file mode 100644 index 00000000..dae7dcd5 --- /dev/null +++ b/packages/secretnote-ui/src/.openapi-stubs/models/ListSnapshot.ts @@ -0,0 +1,19 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { ReferenceMap } from './ReferenceMap'; + +/** + * Helper class that provides a standard way to create an ABC using + * inheritance. + */ +export type ListSnapshot = { + ref: string; + kind?: 'list'; + type: string; + snapshot: string; + values?: ReferenceMap; +}; + diff --git a/packages/secretnote-ui/src/.openapi-stubs/models/LocalObject.ts b/packages/secretnote-ui/src/.openapi-stubs/models/LocalObject.ts new file mode 100644 index 00000000..6a95b933 --- /dev/null +++ b/packages/secretnote-ui/src/.openapi-stubs/models/LocalObject.ts @@ -0,0 +1,11 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +export type LocalObject = { + kind?: 'local_object'; + ref: string; + name?: string; +}; + diff --git a/packages/secretnote-ui/src/.openapi-stubs/models/LocalObjectNode.ts b/packages/secretnote-ui/src/.openapi-stubs/models/LocalObjectNode.ts new file mode 100644 index 00000000..f711e165 --- /dev/null +++ b/packages/secretnote-ui/src/.openapi-stubs/models/LocalObjectNode.ts @@ -0,0 +1,15 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { LocalObject } from './LocalObject'; + +export type LocalObjectNode = { + id: string; + epoch: number; + order?: number; + kind?: 'local'; + data: LocalObject; +}; + diff --git a/packages/secretnote-ui/src/.openapi-stubs/models/LogicalLocation.ts b/packages/secretnote-ui/src/.openapi-stubs/models/LogicalLocation.ts new file mode 100644 index 00000000..c54575df --- /dev/null +++ b/packages/secretnote-ui/src/.openapi-stubs/models/LogicalLocation.ts @@ -0,0 +1,12 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +export type LogicalLocation = { + kind?: 'location'; + type: string; + parties: Array; + parameters?: Record; +}; + diff --git a/packages/secretnote-ui/src/.openapi-stubs/models/MoveExpression.ts b/packages/secretnote-ui/src/.openapi-stubs/models/MoveExpression.ts new file mode 100644 index 00000000..1be63460 --- /dev/null +++ b/packages/secretnote-ui/src/.openapi-stubs/models/MoveExpression.ts @@ -0,0 +1,14 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { LocalObject } from './LocalObject'; +import type { RemoteObject } from './RemoteObject'; + +export type MoveExpression = { + kind?: 'move'; + source: (RemoteObject | LocalObject); + target: RemoteObject; +}; + diff --git a/packages/secretnote-ui/src/.openapi-stubs/models/NoneSnapshot.ts b/packages/secretnote-ui/src/.openapi-stubs/models/NoneSnapshot.ts new file mode 100644 index 00000000..5edb9d40 --- /dev/null +++ b/packages/secretnote-ui/src/.openapi-stubs/models/NoneSnapshot.ts @@ -0,0 +1,14 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +/** + * Helper class that provides a standard way to create an ABC using + * inheritance. + */ +export type NoneSnapshot = { + ref: string; + kind?: 'none'; +}; + diff --git a/packages/secretnote-ui/src/.openapi-stubs/models/ObjectSnapshot.ts b/packages/secretnote-ui/src/.openapi-stubs/models/ObjectSnapshot.ts new file mode 100644 index 00000000..a58784ce --- /dev/null +++ b/packages/secretnote-ui/src/.openapi-stubs/models/ObjectSnapshot.ts @@ -0,0 +1,16 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +/** + * Helper class that provides a standard way to create an ABC using + * inheritance. + */ +export type ObjectSnapshot = { + ref: string; + kind?: 'object'; + type: string; + snapshot: string; +}; + diff --git a/packages/secretnote-ui/src/.openapi-stubs/models/Reference.ts b/packages/secretnote-ui/src/.openapi-stubs/models/Reference.ts new file mode 100644 index 00000000..c1b03f1b --- /dev/null +++ b/packages/secretnote-ui/src/.openapi-stubs/models/Reference.ts @@ -0,0 +1,9 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +export type Reference = { + ref: string; +}; + diff --git a/packages/secretnote-ui/src/.openapi-stubs/models/ReferenceEdge.ts b/packages/secretnote-ui/src/.openapi-stubs/models/ReferenceEdge.ts new file mode 100644 index 00000000..1c7a3bd6 --- /dev/null +++ b/packages/secretnote-ui/src/.openapi-stubs/models/ReferenceEdge.ts @@ -0,0 +1,11 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +export type ReferenceEdge = { + source: string; + target: string; + kind?: 'reference'; +}; + diff --git a/packages/secretnote-ui/src/.openapi-stubs/models/ReferenceMap.ts b/packages/secretnote-ui/src/.openapi-stubs/models/ReferenceMap.ts new file mode 100644 index 00000000..209e4ae8 --- /dev/null +++ b/packages/secretnote-ui/src/.openapi-stubs/models/ReferenceMap.ts @@ -0,0 +1,9 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { Reference } from './Reference'; + +export type ReferenceMap = (Array | Record); + diff --git a/packages/secretnote-ui/src/.openapi-stubs/models/RemoteLocationSnapshot.ts b/packages/secretnote-ui/src/.openapi-stubs/models/RemoteLocationSnapshot.ts new file mode 100644 index 00000000..0953f8cf --- /dev/null +++ b/packages/secretnote-ui/src/.openapi-stubs/models/RemoteLocationSnapshot.ts @@ -0,0 +1,18 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { LogicalLocation } from './LogicalLocation'; + +/** + * Helper class that provides a standard way to create an ABC using + * inheritance. + */ +export type RemoteLocationSnapshot = { + ref: string; + kind?: 'remote_location'; + type: string; + location: LogicalLocation; +}; + diff --git a/packages/secretnote-ui/src/.openapi-stubs/models/RemoteObject.ts b/packages/secretnote-ui/src/.openapi-stubs/models/RemoteObject.ts new file mode 100644 index 00000000..952f5003 --- /dev/null +++ b/packages/secretnote-ui/src/.openapi-stubs/models/RemoteObject.ts @@ -0,0 +1,15 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { LogicalLocation } from './LogicalLocation'; + +export type RemoteObject = { + kind?: 'remote_object'; + numbering?: number; + ref: string; + location: LogicalLocation; + name?: string; +}; + diff --git a/packages/secretnote-ui/src/.openapi-stubs/models/RemoteObjectNode.ts b/packages/secretnote-ui/src/.openapi-stubs/models/RemoteObjectNode.ts new file mode 100644 index 00000000..6b7e8dfa --- /dev/null +++ b/packages/secretnote-ui/src/.openapi-stubs/models/RemoteObjectNode.ts @@ -0,0 +1,15 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { RemoteObject } from './RemoteObject'; + +export type RemoteObjectNode = { + id: string; + epoch: number; + order?: number; + kind?: 'remote'; + data: RemoteObject; +}; + diff --git a/packages/secretnote-ui/src/.openapi-stubs/models/RemoteObjectSnapshot.ts b/packages/secretnote-ui/src/.openapi-stubs/models/RemoteObjectSnapshot.ts new file mode 100644 index 00000000..a543d79e --- /dev/null +++ b/packages/secretnote-ui/src/.openapi-stubs/models/RemoteObjectSnapshot.ts @@ -0,0 +1,19 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { LogicalLocation } from './LogicalLocation'; + +/** + * Helper class that provides a standard way to create an ABC using + * inheritance. + */ +export type RemoteObjectSnapshot = { + ref: string; + kind?: 'remote_object'; + type: string; + location: LogicalLocation; + refs: Array; +}; + diff --git a/packages/secretnote-ui/src/.openapi-stubs/models/ReturnEdge.ts b/packages/secretnote-ui/src/.openapi-stubs/models/ReturnEdge.ts new file mode 100644 index 00000000..35ca9808 --- /dev/null +++ b/packages/secretnote-ui/src/.openapi-stubs/models/ReturnEdge.ts @@ -0,0 +1,12 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +export type ReturnEdge = { + source: string; + target: string; + kind?: 'return'; + assignment?: string; +}; + diff --git a/packages/secretnote-ui/src/.openapi-stubs/models/RevealEdge.ts b/packages/secretnote-ui/src/.openapi-stubs/models/RevealEdge.ts new file mode 100644 index 00000000..0b42556c --- /dev/null +++ b/packages/secretnote-ui/src/.openapi-stubs/models/RevealEdge.ts @@ -0,0 +1,12 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +export type RevealEdge = { + source: string; + target: string; + kind?: 'reveal'; + name?: string; +}; + diff --git a/packages/secretnote-ui/src/.openapi-stubs/models/RevealExpression.ts b/packages/secretnote-ui/src/.openapi-stubs/models/RevealExpression.ts new file mode 100644 index 00000000..be45f1b7 --- /dev/null +++ b/packages/secretnote-ui/src/.openapi-stubs/models/RevealExpression.ts @@ -0,0 +1,14 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { LocalObject } from './LocalObject'; +import type { RemoteObject } from './RemoteObject'; + +export type RevealExpression = { + kind?: 'reveal'; + items: Array<(LocalObject | RemoteObject)>; + results: Array; +}; + diff --git a/packages/secretnote-ui/src/.openapi-stubs/models/RevealNode.ts b/packages/secretnote-ui/src/.openapi-stubs/models/RevealNode.ts new file mode 100644 index 00000000..ffd26d2c --- /dev/null +++ b/packages/secretnote-ui/src/.openapi-stubs/models/RevealNode.ts @@ -0,0 +1,12 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +export type RevealNode = { + id: string; + epoch: number; + order?: number; + kind?: 'reveal'; +}; + diff --git a/packages/secretnote-ui/src/.openapi-stubs/models/Semantics.ts b/packages/secretnote-ui/src/.openapi-stubs/models/Semantics.ts new file mode 100644 index 00000000..2e3f8984 --- /dev/null +++ b/packages/secretnote-ui/src/.openapi-stubs/models/Semantics.ts @@ -0,0 +1,10 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +export type Semantics = { + api_level?: number; + description?: string; +}; + diff --git a/packages/secretnote-ui/src/.openapi-stubs/models/Timeline.ts b/packages/secretnote-ui/src/.openapi-stubs/models/Timeline.ts new file mode 100644 index 00000000..32a0ef74 --- /dev/null +++ b/packages/secretnote-ui/src/.openapi-stubs/models/Timeline.ts @@ -0,0 +1,23 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { DictSnapshot } from './DictSnapshot'; +import type { Frame } from './Frame'; +import type { FrameInfoSnapshot } from './FrameInfoSnapshot'; +import type { FrameSnapshot } from './FrameSnapshot'; +import type { FunctionSnapshot } from './FunctionSnapshot'; +import type { Graph } from './Graph'; +import type { ListSnapshot } from './ListSnapshot'; +import type { NoneSnapshot } from './NoneSnapshot'; +import type { ObjectSnapshot } from './ObjectSnapshot'; +import type { RemoteLocationSnapshot } from './RemoteLocationSnapshot'; +import type { RemoteObjectSnapshot } from './RemoteObjectSnapshot'; + +export type Timeline = { + variables?: Record; + timeline?: Array; + graph: Graph; +}; + diff --git a/packages/secretnote-ui/src/.openapi-stubs/models/TransformEdge.ts b/packages/secretnote-ui/src/.openapi-stubs/models/TransformEdge.ts new file mode 100644 index 00000000..796b0919 --- /dev/null +++ b/packages/secretnote-ui/src/.openapi-stubs/models/TransformEdge.ts @@ -0,0 +1,14 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { LogicalLocation } from './LogicalLocation'; + +export type TransformEdge = { + source: string; + target: string; + kind?: 'transform'; + destination: LogicalLocation; +}; + diff --git a/packages/secretnote-ui/src/.openapi-stubs/models/ValidationError.ts b/packages/secretnote-ui/src/.openapi-stubs/models/ValidationError.ts new file mode 100644 index 00000000..b91764c9 --- /dev/null +++ b/packages/secretnote-ui/src/.openapi-stubs/models/ValidationError.ts @@ -0,0 +1,11 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +export type ValidationError = { + loc: Array<(string | number)>; + msg: string; + type: string; +}; + diff --git a/packages/secretnote-ui/src/.openapi-stubs/models/Visualization.ts b/packages/secretnote-ui/src/.openapi-stubs/models/Visualization.ts new file mode 100644 index 00000000..4b1de7a2 --- /dev/null +++ b/packages/secretnote-ui/src/.openapi-stubs/models/Visualization.ts @@ -0,0 +1,11 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { Timeline } from './Timeline'; + +export type Visualization = { + timeline: Timeline; +}; + diff --git a/packages/secretnote-ui/src/.openapi-stubs/models/_ParameterKind.ts b/packages/secretnote-ui/src/.openapi-stubs/models/_ParameterKind.ts new file mode 100644 index 00000000..1212b758 --- /dev/null +++ b/packages/secretnote-ui/src/.openapi-stubs/models/_ParameterKind.ts @@ -0,0 +1,9 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +/** + * An enumeration. + */ +export type _ParameterKind = 0 | 1 | 2 | 3 | 4; diff --git a/packages/secretnote-ui/src/components/DataProvider/context.ts b/packages/secretnote-ui/src/components/DataProvider/context.ts new file mode 100644 index 00000000..fc12c7c8 --- /dev/null +++ b/packages/secretnote-ui/src/components/DataProvider/context.ts @@ -0,0 +1,7 @@ +import { createContext } from 'react'; + +import type { SnapshotReifier } from '../../utils/reify'; + +export const DataProviderContext = createContext<{ reify: SnapshotReifier }>({ + reify: () => undefined, +}); diff --git a/packages/secretnote-ui/src/components/DataProvider/index.tsx b/packages/secretnote-ui/src/components/DataProvider/index.tsx new file mode 100644 index 00000000..b218f00e --- /dev/null +++ b/packages/secretnote-ui/src/components/DataProvider/index.tsx @@ -0,0 +1,24 @@ +import type { ContextType } from 'react'; +import { useMemo } from 'react'; + +import type { Timeline } from '../../.openapi-stubs'; +import { reify } from '../../utils/reify'; + +import { DataProviderContext } from './context'; + +export const DataProvider = ({ + timeline, + children, +}: React.PropsWithChildren<{ timeline?: Timeline }>) => { + const value: ContextType = useMemo( + () => ({ + reify: (kind, ref) => reify(kind, ref, timeline?.variables), + }), + [timeline?.variables], + ); + return ( + + {children} + + ); +}; diff --git a/packages/secretnote-ui/src/components/DataProvider/utils.ts b/packages/secretnote-ui/src/components/DataProvider/utils.ts new file mode 100644 index 00000000..c664cc09 --- /dev/null +++ b/packages/secretnote-ui/src/components/DataProvider/utils.ts @@ -0,0 +1,7 @@ +import { useContext } from 'react'; + +import { DataProviderContext } from './context'; + +export function useDataProvider() { + return useContext(DataProviderContext); +} diff --git a/packages/secretnote-ui/src/components/ExecutionGraph/colorization.ts b/packages/secretnote-ui/src/components/ExecutionGraph/colorization.ts new file mode 100644 index 00000000..52e1331a --- /dev/null +++ b/packages/secretnote-ui/src/components/ExecutionGraph/colorization.ts @@ -0,0 +1,217 @@ +import type * as G6 from '@antv/g6'; +import type * as graphlib from '@antv/graphlib'; +import Color from 'color'; +import { useCallback, useMemo, useState } from 'react'; + +import type { LogicalLocation } from '../../.openapi-stubs'; + +import type { TrustedModel } from './types'; +import { isTrusted } from './types'; +import type { PartitionFunction } from './utils'; +import { completePartition, toPureGraph } from './utils'; + +export type ColorizeFunction = (item: T) => { + background: string; + foreground: string; +}; + +export interface Colorizer { + colors(): Map; + colorize: ColorizeFunction; +} + +export class LocationColorizer implements Colorizer { + private readonly palette: string[]; + + private readonly cache = new Map(); + private readonly names = new Map(); + + constructor(palette: string[]) { + this.palette = palette; + } + + public colorize(location: LogicalLocation) { + const key = LocationColorizer.locationKey(location); + let color = this.cache.get(key); + if (!color) { + color = this.makeColor(); + this.cache.set(key, color); + } + this.names.set(key, this.locationName(location)); + return { background: color, foreground: this.foreground(color) }; + } + + public colors() { + return new Map( + [...this.names.entries()].map(([k, name]) => [ + k, + { name, color: this.cache.get(k)! }, + ]), + ); + } + + private locationName(location: LogicalLocation) { + return `${location.type}[${location.parties.join(', ')}]`; + } + + public static locationKey(location: LogicalLocation) { + return [ + location.type, + ...location.parties, + ...Object.entries(location.parameters ?? {}).map(([k, v]) => `${k}=${v}`), + ].join(':'); + } + + protected makeColor() { + const currentColorCount = this.cache.size; + const position = currentColorCount % this.palette.length; + const generation = Math.floor(currentColorCount / this.palette.length); + if (generation === 0) { + return this.palette[position]; + } + const hueShifts = [ + // triadic + 120, 240, + // tetradic + 90, 180, 270, + ]; + const hueShift = hueShifts[generation - 1]; + if (hueShift === undefined) { + throw new Error('Too many colors'); + } + return new Color(this.palette[position]).rotate(hueShift).hex(); + } + + protected foreground( + color: string, + darken = 0.2, + light = '#ffffff', + dark = '#1d1d1d', + ) { + return new Color(color).darken(darken).isDark() ? light : dark; + } +} + +export function colorizeByLocation(colorize: ColorizeFunction) { + return (node: TrustedModel) => { + switch (node.data.kind) { + case 'function': + return colorize(node.data.location); + case 'local': + return { background: '#1d1d1d', foreground: '#ffffff' }; + case 'remote': + return colorize(node.data.data.location); + case 'reveal': + return { background: '#f04654', foreground: '#ffffff' }; + case 'argument': + return { background: '#a5aab5', foreground: '#ffffff' }; + case 'return': + return { background: '#a5aab5', foreground: '#ffffff' }; + case 'transform': + return colorize(node.data.destination); + default: + throw new Error(`Unknown shape kind: ${node.data.kind}`); + } + }; +} + +export function recolorOnHover({ + partition, + colorize, +}: { + partition: PartitionFunction; + colorize: (node: TrustedModel) => { background: string; foreground: string }; +}) { + return (graph: G6.Graph) => { + const highlight = (id: graphlib.ID) => { + const g = toPureGraph(graph); + const { matched, unmatched } = completePartition(g, partition(g, id)); + + matched.forEach((k) => { + const shape = graph.findById(String(k)); + const model = shape.getModel(); + if (isTrusted(model)) { + const { background, foreground } = colorize(model); + graph.updateItem(shape, { + colors: { background, foreground }, + }); + } + }); + + unmatched.forEach((k) => { + const shape = graph.findById(String(k)); + const model = shape.getModel(); + if (isTrusted(model)) { + graph.updateItem(shape, { + colors: { background: '#d3d3d3', foreground: '#ffffff' }, + }); + } + }); + }; + + const reset = () => { + [...graph.getNodes(), ...graph.getEdges()].forEach((shape) => { + const model = shape.getModel(); + if (isTrusted(model)) { + const { background, foreground } = colorize(model); + graph.updateItem(shape, { + colors: { background, foreground }, + }); + } + }); + }; + + const onEnter = ({ item }: { item: G6.Item | null }) => { + if (!item) { + return; + } + highlight(item.getID()); + }; + + return { + enable: () => { + graph.on('node:mouseenter', onEnter); + graph.on('node:mouseleave', reset); + }, + disable: () => { + graph.off('node:mouseenter', onEnter); + graph.off('node:mouseleave', reset); + }, + highlight: (target: graphlib.ID | null) => { + if (target) { + highlight(target); + } else { + reset(); + } + }, + }; + }; +} + +export function useColorizer( + factory: () => Colorizer, +): Colorizer & { reset: () => void } { + const [colorizer, setColorizer] = useState(factory); + const [, setColorCount] = useState(0); + + const colorize = useCallback>( + (...args) => { + const color = colorizer.colorize(...args); + setColorCount(colorizer.colors().size); + return color; + }, + [colorizer], + ); + + return useMemo( + () => ({ + colorize, + colors: colorizer.colors.bind(colorizer), + reset: () => { + setColorCount(0); + setColorizer(factory); + }, + }), + [colorize, colorizer, factory], + ); +} diff --git a/packages/secretnote-ui/src/components/ExecutionGraph/index.tsx b/packages/secretnote-ui/src/components/ExecutionGraph/index.tsx new file mode 100644 index 00000000..f6bbd0d5 --- /dev/null +++ b/packages/secretnote-ui/src/components/ExecutionGraph/index.tsx @@ -0,0 +1,308 @@ +import * as G6 from '@antv/g6'; +import { Card, ConfigProvider, Divider, Form, Switch } from 'antd'; +import type { MouseEventHandler, MutableRefObject } from 'react'; +import { Fragment, useCallback, useEffect, useMemo, useRef, useState } from 'react'; + +import type { Graph as GraphProps, LogicalLocation } from '../../.openapi-stubs'; +import { useDataProvider } from '../DataProvider/utils'; + +import type { Colorizer } from './colorization'; +import { colorizeByLocation, recolorOnHover, useColorizer } from './colorization'; +import { LocationColorizer } from './colorization'; +import { setupG6 } from './shapes'; +import { tooltip } from './tooltip'; +import { isTrusted, type GraphNodeType } from './types'; +import { partitionByEntityType, partitionByLocation } from './utils'; + +type GraphRef = MutableRefObject; + +const { fromGraph } = setupG6(); + +const defaultColorizer = () => + new LocationColorizer([ + '#79a25c', + '#de4c8b', + '#8271df', + '#3398a6', + '#c47d3a', + '#b45dcb', + '#4c99d8', + '#df6a72', + ]); + +function Legend({ + graph, + colorizer, +}: { + graph: GraphRef; + colorizer: Colorizer; +}) { + const locationColorizer = useMemo( + () => + recolorOnHover({ + partition: partitionByLocation, + colorize: colorizeByLocation(colorizer.colorize), + }), + [colorizer.colorize], + ); + + const resetColors = useCallback(() => { + if (!graph.current) { + return; + } + locationColorizer(graph.current).highlight(null); + }, [graph, locationColorizer]); + + const highlight = useCallback( + (locationKey: string): MouseEventHandler => + () => { + if (!graph.current) { + return; + } + const target = graph.current.getNodes().find((v) => { + const model = v.getModel(); + if (!isTrusted(model)) { + return false; + } + switch (model.data.kind) { + case 'function': + return LocationColorizer.locationKey(model.data.location) === locationKey; + case 'remote': + return ( + LocationColorizer.locationKey(model.data.data.location) === locationKey + ); + default: + return false; + } + }); + if (target) { + locationColorizer(graph.current).highlight(target.getID()); + } + }, + [graph, locationColorizer], + ); + + const [hovered, setHovered] = useState(); + + return ( +
{ + setHovered(undefined); + resetColors(e); + }} + > + {[...colorizer.colors()].map(([key, { name, color }]) => ( + +
{ + highlight(key)(e); + setHovered(key); + }} + /> +
{ + highlight(key)(e); + setHovered(key); + }} + > + + {name} + +
+ + ))} +
+ ); +} + +function useExecutionGraph() { + const { reify } = useDataProvider(); + + const colorizer = useColorizer(defaultColorizer); + + const entityColorizer = useMemo( + () => + recolorOnHover({ + partition: partitionByEntityType, + colorize: colorizeByLocation(colorizer.colorize), + }), + [colorizer.colorize], + ); + + const containerRef = useRef(null); + const graphRef = useRef(); + const tooltipEnabledRef = useRef(true); + + useEffect(() => { + if (!containerRef.current) { + graphRef.current = undefined; + return; + } + + const graph = new G6.Graph({ + container: containerRef.current, + width: containerRef.current.clientWidth, + height: containerRef.current.clientHeight, + layout: { + type: 'dagre', + ranksepFunc: (node: { data: GraphNodeType }) => { + if (node.data?.kind === 'reveal' || node.data?.kind === 'remote') { + return 2.5; + } + if (node.data?.kind === 'local') { + return 5; + } + return 10; + }, + nodesep: 10, + }, + modes: { + default: [ + { type: 'scroll-canvas' }, + { type: 'drag-canvas' }, + { + type: 'tooltip', + formatText: (model) => { + if (!tooltipEnabledRef.current) { + return ''; + } + return tooltip(model, reify); + }, + offset: 10, + }, + ], + highlighting: [], + }, + minZoom: 0.2, + maxZoom: 3, + }); + + graph.on('node:click', ({ item }) => { + if (item) { + graph.focusItem(item); + } + }); + + entityColorizer(graph).enable(); + + graphRef.current = graph; + + return () => { + graph.destroy(); + }; + }, [entityColorizer, reify]); + + useEffect(() => { + const outputView = containerRef.current?.closest('.jp-LinkedOutputView'); + const resizeObserver = new ResizeObserver(() => { + if (!containerRef.current) { + return; + } + const [height, cssHeight] = (() => { + if (outputView && outputView.clientHeight !== 0) { + return [outputView.clientHeight, `${outputView.clientHeight}px`]; + } + return [ + containerRef.current.clientHeight, + `${containerRef.current.clientHeight}px`, + ]; + })(); + containerRef.current.style.height = cssHeight; + graphRef.current?.changeSize(containerRef.current.clientWidth, height); + }); + if (containerRef.current) { + resizeObserver.observe(containerRef.current); + } + if (outputView) { + resizeObserver.observe(outputView); + } + return () => { + resizeObserver.disconnect(); + }; + }, []); + + return { + container: containerRef, + graph: graphRef, + colorizer, + tooltipEnabled: tooltipEnabledRef, + load: (data: GraphProps) => { + graphRef.current?.data(fromGraph(data, { reify, colorize: colorizer.colorize })); + graphRef.current?.render(); + }, + }; +} + +export function ExecutionGraph(data: GraphProps) { + const { container, load, graph, colorizer, tooltipEnabled } = useExecutionGraph(); + + useEffect(() => { + load(data); + }, [data, load]); + + return ( +
+
+
+ + + + DEVELOPER PREVIEW + + + + + + + + + { + tooltipEnabled.current = checked; + }} + /> + + +
+
+ ); +} diff --git a/packages/secretnote-ui/src/components/ExecutionGraph/shapes.ts b/packages/secretnote-ui/src/components/ExecutionGraph/shapes.ts new file mode 100644 index 00000000..b5018d22 --- /dev/null +++ b/packages/secretnote-ui/src/components/ExecutionGraph/shapes.ts @@ -0,0 +1,549 @@ +import type { + ArgumentEdge, + ReturnEdge, + RevealEdge, + TransformEdge, +} from '../../.openapi-stubs'; +import { truncate, truncateLines, wrap } from '../../utils/string'; + +import { defineShape, registerShapes } from './utils'; + +declare module '@antv/g6' { + interface IShape { + getPoint(ratio: number): { + x: number; + y: number; + }; + } +} + +const LOCAL_DATA_NODE = defineShape({ + kind: 'local', + render: ({ + item, + renderer, + config: { + colors: { foreground, background } = { + foreground: '#ffffff', + background: '#1d1d1d', + }, + }, + utils: { reify }, + }) => { + const value = reify(undefined, item.data); + + let content: string = ''; + + switch (value?.kind) { + case 'dict': + case 'list': + case 'object': + content = `${value.snapshot}`; + break; + case 'function': + content = `${value.name}()`; + break; + default: + break; + } + + content = content.trim(); + + let label: string; + + if (item.data.name && content) { + label = `${item.data.name} = ${content}`; + if (label.length > 12) { + label = `${item.data.name}\n= ${content}`; + } + } else { + label = content; + } + + label = truncateLines(label, { maxLines: 3, maxWidth: 15 }); + + const rect = renderer.addShape('rect', { + name: 'background', + attrs: { + anchorPoints: [ + [0.5, 0], + [0.5, 1], + ], + stroke: null, + fill: background, + }, + }); + + const text = renderer.addShape('text', { + name: 'label', + attrs: { + text: label, + x: 0, + y: 0, + fontFamily: 'Roboto Mono, monospace', + fontSize: 12, + lineHeight: 16.8, + textAlign: 'center', + textBaseline: 'middle', + fill: foreground, + }, + }); + + const { width, height, x, y } = text.getBBox(); + + rect.attr('width', width + 10); + rect.attr('height', height + 10); + rect.attr('x', x - 5); + rect.attr('y', y - 5); + + return rect; + }, +}); + +const REMOTE_DATA_NODE = defineShape({ + kind: 'remote', + render: ({ item, renderer, config: { colors }, utils: { colorize } }) => { + const { background, foreground } = colors || colorize(item.data.location); + const label = `${item.data.location.type[0]}${item.data.numbering}`; + + const rect = renderer.addShape('circle', { + name: 'background', + attrs: { + x: 0, + y: 0, + anchorPoints: [ + [0.5, 0], + [0.5, 1], + ], + stroke: null, + fill: background, + }, + }); + + renderer.addShape('text', { + name: 'label', + attrs: { + text: label, + x: 0, + y: 0, + fontFamily: 'Inter, sans-serif', + fontWeight: 700, + fontSize: 12, + lineHeight: 16, + textAlign: 'center', + textBaseline: 'middle', + fill: foreground, + }, + }); + + const diameter = 40 + Math.floor(Math.log10(item.data.numbering || 0) / 2) * 5; + + rect.attr('width', diameter); + rect.attr('height', diameter); + rect.attr('x', 0); + rect.attr('y', 0); + rect.attr('r', diameter / 2); + + return rect; + }, +}); + +const FUNCTION_NODE = defineShape({ + kind: 'function', + render: ({ item, renderer, config: { colors }, utils: { reify, colorize } }) => { + const { background, foreground } = colors || colorize(item.location); + + const label = (() => { + const parties = item.location.parties.map((d) => d[0].toUpperCase()).join(','); + if (item.function) { + const value = reify('function', item.function); + if (value) { + return truncateLines(wrap(`let ${parties} in ${value.name}`, '.', 24), { + maxWidth: 24, + maxLines: 2, + }); + } + } + return `let ${parties} in (anonymous)`; + })(); + + const rect = renderer.addShape('rect', { + name: 'background', + attrs: { + radius: 8, + anchorPoints: [ + [0.5, 0], + [0.5, 1], + ], + stroke: null, + fill: background, + lineWidth: 2, + }, + }); + + const text = renderer.addShape('text', { + name: 'label', + attrs: { + text: label, + x: 0, + y: 0, + fontFamily: 'Roboto Mono, monospace', + fontSize: 12, + fontWeight: 600, + lineHeight: 14.4, + letterSpacing: 0.5, + textAlign: 'center', + textBaseline: 'middle', + fill: foreground, + }, + }); + + const { width, height, x, y } = text.getBBox(); + + rect.attr('width', width + 30); + rect.attr('height', height + 15); + rect.attr('x', x - 15); + rect.attr('y', y - 7.5); + + return rect; + }, +}); + +const REVEAL_NODE = defineShape({ + kind: 'reveal', + render: ({ + renderer, + config: { + colors: { background, foreground } = { + background: '#f04654', + foreground: '#ffffff', + }, + }, + }) => { + const rect = renderer.addShape('rect', { + name: 'background', + attrs: { + anchorPoints: [ + [0.5, 0], + [0.5, 1], + ], + stroke: null, + fill: background, + }, + }); + const text = renderer.addShape('text', { + name: 'label', + attrs: { + text: 'reveal', + x: 0, + y: 0, + fontFamily: 'Inter, sans-serif', + fontSize: 12, + fontWeight: 500, + lineHeight: 16.8, + textAlign: 'center', + textBaseline: 'middle', + fill: foreground, + }, + }); + + const { width, height, x, y } = text.getBBox(); + + rect.attr('width', width + 10); + rect.attr('height', height + 10); + rect.attr('x', x - 5); + rect.attr('y', y - 5); + + return rect; + }, +}); + +function distance(x1: number, y1: number, x2: number, y2: number): number { + return Math.sqrt(Math.pow(x2 - x1, 2) + Math.pow(y2 - y1, 2)); +} + +// Thank you GPT (and Desmos) +function textRotation(x1: number, y1: number, x2: number, y2: number): number { + const dx = x2 - x1; + const dy = y2 - y1; + let theta = Math.atan2(dy, dx); + if (dx < 0) { + theta -= Math.PI; + } + if (theta > (70 / 180) * Math.PI) { + return theta - (1 / 2) * Math.PI; + } + if (theta < (-70 / 180) * Math.PI) { + return theta + (1 / 2) * Math.PI; + } + return theta; +} + +const ARGUMENT_EDGE = defineShape({ + kind: 'argument', + render: ({ + item, + renderer, + config: { + startPoint = { x: 0, y: 0 }, + endPoint = { x: 0, y: 0 }, + colors: { background, foreground } = { + background: '#a5aab5', + foreground: '#ffffff', + }, + }, + }) => { + const shape = renderer.addShape('path', { + name: 'line', + attrs: { + stroke: background, + lineWidth: 1, + path: [ + ['M', startPoint.x, startPoint.y], + ['L', endPoint.x, endPoint.y], + ], + endArrow: { + path: 'M 5 -5 L 0 0 L 5 5', + lineWidth: 1, + }, + }, + }); + if (item.name) { + const label = truncate(item.name, 20); + const midPoint = shape.getPoint(0.5); + const rect = renderer.addShape('rect', { + name: 'label-background', + attrs: { + radius: 0, + anchorPoints: [ + [0.5, 0], + [0.5, 1], + ], + stroke: null, + fill: background, + }, + }); + const text = renderer.addShape('text', { + name: 'label', + attrs: { + text: label, + x: midPoint.x, + y: midPoint.y, + fontFamily: 'Roboto Mono, monospace', + fontStyle: 'italic', + fontSize: 11, + textAlign: 'center', + textBaseline: 'middle', + fill: foreground, + }, + }); + const { width, height, x, y } = text.getBBox(); + rect.attr('width', width + 5); + rect.attr('height', height + 5); + const rotation = textRotation(startPoint.x, startPoint.y, endPoint.x, endPoint.y); + if ( + width > 30 && + width < distance(startPoint.x, startPoint.y, endPoint.x, endPoint.y) - 20 + ) { + rect.rotateAtPoint(midPoint.x, midPoint.y, rotation); + text.rotateAtPoint(midPoint.x, midPoint.y, rotation); + } + rect.attr('x', x - 2.5); + rect.attr('y', y - 2.5); + } + return shape; + }, +}); + +const REVEAL_EDGE = defineShape({ + kind: 'reveal', + render: ({ + item, + renderer, + config: { + startPoint = { x: 0, y: 0 }, + endPoint = { x: 0, y: 0 }, + colors: { background, foreground } = { + background: '#f04654', + foreground: '#ffffff', + }, + }, + }) => { + const shape = renderer.addShape('path', { + name: 'line', + attrs: { + stroke: background, + lineWidth: 1, + path: [ + ['M', startPoint.x, startPoint.y], + ['L', endPoint.x, endPoint.y], + ], + lineDash: [2], + }, + }); + if (item.name) { + const label = truncate(item.name, 20); + const midPoint = shape.getPoint(0.5); + const rect = renderer.addShape('rect', { + name: 'label-background', + attrs: { + radius: 0, + anchorPoints: [ + [0.5, 0], + [0.5, 1], + ], + stroke: null, + fill: background, + }, + }); + const text = renderer.addShape('text', { + name: 'label', + attrs: { + text: label, + x: midPoint.x, + y: midPoint.y, + fontFamily: 'Roboto Mono, monospace', + fontStyle: 'italic', + fontSize: 11, + textAlign: 'center', + textBaseline: 'middle', + fill: foreground, + }, + }); + const { width, height, x, y } = text.getBBox(); + rect.attr('width', width + 5); + rect.attr('height', height + 5); + const rotation = textRotation(startPoint.x, startPoint.y, endPoint.x, endPoint.y); + if ( + width > 30 && + width < distance(startPoint.x, startPoint.y, endPoint.x, endPoint.y) - 20 + ) { + rect.rotateAtPoint(midPoint.x, midPoint.y, rotation); + text.rotateAtPoint(midPoint.x, midPoint.y, rotation); + } + rect.attr('x', x - 2.5); + rect.attr('y', y - 2.5); + } + return shape; + }, +}); + +const RETURN_EDGE = defineShape({ + kind: 'return', + render: ({ + item, + renderer, + config: { + startPoint = { x: 0, y: 0 }, + endPoint = { x: 0, y: 0 }, + colors: { background, foreground } = { + background: '#a5aab5', + foreground: '#ffffff', + }, + }, + }) => { + const shape = renderer.addShape('path', { + name: 'line', + attrs: { + stroke: background, + lineWidth: 1, + path: [ + ['M', startPoint.x, startPoint.y], + ['L', endPoint.x, endPoint.y], + ], + endArrow: { + path: 'M 5 -5 L 0 0 L 5 5', + lineWidth: 1, + }, + }, + }); + if (item.assignment) { + const label = truncate(item.assignment, 20); + const midPoint = shape.getPoint(0.5); + const rect = renderer.addShape('rect', { + name: 'label-background', + attrs: { + radius: 0, + anchorPoints: [ + [0.5, 0], + [0.5, 1], + ], + stroke: null, + fill: background, + }, + }); + const text = renderer.addShape('text', { + name: 'label', + attrs: { + text: label, + x: midPoint.x, + y: midPoint.y, + fontFamily: 'Roboto Mono, monospace', + fontStyle: 'italic', + fontSize: 11, + textAlign: 'center', + textBaseline: 'middle', + fill: foreground, + }, + }); + const { width, height, x, y } = text.getBBox(); + rect.attr('width', width + 5); + rect.attr('height', height + 5); + const rotation = textRotation(startPoint.x, startPoint.y, endPoint.x, endPoint.y); + if ( + width > 30 && + width < distance(startPoint.x, startPoint.y, endPoint.x, endPoint.y) - 20 + ) { + rect.rotateAtPoint(midPoint.x, midPoint.y, rotation); + text.rotateAtPoint(midPoint.x, midPoint.y, rotation); + } + rect.attr('x', x - 2.5); + rect.attr('y', y - 2.5); + } + return shape; + }, +}); + +const TRANSFORM_EDGE = defineShape({ + kind: 'transform', + render: ({ + item, + renderer, + config: { startPoint = { x: 0, y: 0 }, endPoint = { x: 0, y: 0 }, colors }, + utils: { colorize }, + }) => { + const { background } = colors || colorize(item.destination); + const shape = renderer.addShape('path', { + name: 'line-background', + attrs: { + stroke: background, + lineWidth: 3, + path: [ + ['M', startPoint.x, startPoint.y], + ['L', endPoint.x, endPoint.y], + ], + }, + }); + renderer.addShape('path', { + name: 'line-foreground', + attrs: { + stroke: '#ffffff', + lineWidth: 1.5, + path: [ + ['M', startPoint.x, startPoint.y], + ['L', endPoint.x, endPoint.y], + ], + }, + }); + return shape; + }, +}); + +export function setupG6() { + return { + fromGraph: registerShapes({ + nodes: [LOCAL_DATA_NODE, REMOTE_DATA_NODE, FUNCTION_NODE, REVEAL_NODE], + edges: [ARGUMENT_EDGE, RETURN_EDGE, TRANSFORM_EDGE, REVEAL_EDGE], + }), + }; +} diff --git a/packages/secretnote-ui/src/components/ExecutionGraph/tooltip.ts b/packages/secretnote-ui/src/components/ExecutionGraph/tooltip.ts new file mode 100644 index 00000000..946960ca --- /dev/null +++ b/packages/secretnote-ui/src/components/ExecutionGraph/tooltip.ts @@ -0,0 +1,201 @@ +import type * as G6 from '@antv/g6'; +import * as d3 from 'd3'; +import YAML from 'yaml'; + +import type { + RemoteObjectNode, + LocalObjectNode, + FunctionNode, +} from '../../.openapi-stubs'; +import type { SnapshotReifier } from '../../utils/reify'; +import { wrap } from '../../utils/string'; + +import { isTrusted } from './types'; + +type Selection = d3.Selection< + E, + T, + null, + undefined +>; + +type TooltipProps = { + root: Selection; + reify: SnapshotReifier; +}; + +function tooltipHeader( + root: TooltipProps['root'], + text: string | d3.ValueFn, +) { + root.append('strong').text(text).style('font-size', '0.9rem'); + root + .append('hr') + .style('margin', '3px 0') + .style('border', 0) + .style('border-top', '1px solid #d3d3d3'); +} + +function attributes( + root: TooltipProps['root'], + items: [ + string | d3.ValueFn, + string | d3.ValueFn, + ][], +) { + const container = root + .append('div') + .style('display', 'grid') + .style('gap', '.3rem') + .style('min-width', '0') + .style('grid-template-columns', '2fr 8fr') + .style('grid-auto-flow', 'row') + .style('align-items', 'baseline'); + items.forEach(([name, value]) => { + container.append('span').text(name); + container + .append('code') + .style('font-weight', 700) + .style('word-break', 'break-all') + .style('background', 'none') + .text(value); + }); +} + +function codeBlock( + root: TooltipProps['root'], + value: string | d3.ValueFn, +) { + root + .append('div') + .style('background', '#f5f5f5') + .style('margin', '6px 0 0') + .style('max-height', '10vh') + .style('overflow', 'auto') + .style('padding', '6px') + .append('pre') + .style('background', 'none') + .style('overflow', 'auto') + .style('white-space', 'pre') + .style('word-break', 'break-all') + .text(value); +} + +export function remoteObjectTooltip({ root }: TooltipProps) { + tooltipHeader(root, (d) => `Remote object #${d.data.numbering || 'numbering ?'}`); + attributes(root, [ + ['Device', (d) => d.data.location.type], + [ + (d) => (d.data.location.parties.length > 1 ? 'Parties' : 'Party'), + (d) => d.data.location.parties.join(', '), + ], + ]); + const params = root.datum().data.location.parameters || {}; + if (Object.keys(params).length > 0) { + codeBlock(root, () => YAML.stringify({ properties: params }, { indent: 2 })); + } +} + +export function localObjectTooltip({ root, reify }: TooltipProps) { + tooltipHeader(root, 'Local value'); + + const value = reify(undefined, root.datum().data); + const node = root.datum(); + + switch (value?.kind) { + case 'object': + case 'list': + case 'dict': + attributes(root.datum(value), [ + ['Name', node.data.name || '?'], + ['Type', (d) => wrap(d.type, '.', 30)], + ]); + codeBlock(root.datum(value), (d) => d.snapshot); + break; + case 'none': + attributes(root.datum(value), [ + ['Name', node.data.name || '?'], + ['Value', 'None'], + ]); + break; + case 'function': + attributes(root.datum(value), [ + ['Function', (d) => wrap(d.name, '.', 32)], + ['Module', (d) => wrap(d.module || '?', '.', 32)], + ['File', (d) => wrap(d.filename || '?', '/', 32)], + ['Line', (d) => d.firstlineno || '?'], + ]); + codeBlock(root.datum(value), (d) => d.source || '(no source)'); + break; + } +} + +export function functionTooltip({ root, reify }: TooltipProps) { + tooltipHeader(root, 'Code execution'); + + attributes(root, [ + ['Device', (d) => `${d.location.type}[${d.location.parties.join(', ')}]`], + ['Frame #', (d) => d.epoch], + ]); + + const func = reify('function', root.datum().function); + + if (!func) { + return; + } + + attributes(root.datum(func), [ + ['Function', (d) => wrap(d.name, '.', 32)], + ['Module', (d) => wrap(d.module || '?', '.', 32)], + [ + 'File', + (d) => `${wrap(d.filename || '?', '/', 32)}, line ${d.firstlineno || '?'}`, + ], + ]); + + codeBlock(root.datum(func), (d) => d.source || '(no source, likely a C function)'); +} + +export function tooltip(model: G6.ModelConfig, reify: SnapshotReifier): string { + if (!isTrusted(model)) { + return ''; + } + const div = document.createElement('div'); + const root = d3.select(div); + const { data } = model; + switch (data.kind) { + case 'remote': + remoteObjectTooltip({ root: root.datum(data), reify }); + break; + case 'local': + localObjectTooltip({ root: root.datum(data), reify }); + break; + case 'function': + functionTooltip({ root: root.datum(data), reify }); + break; + default: + break; + } + root + .style('box-sizing', 'border-box') + .style('padding', '10px') + .style('margin', '0') + .style('display', 'flex') + .style('flex-direction', 'column') + .style('align-items', 'stretch') + .style('gap', '.3rem') + .style('font-size', '0.8rem') + .style('color', '#333') + .style('border-radius', '4px') + .style('background-color', '#fff') + .style('min-width', '200px') + .style('max-width', '25vw') + .style( + 'box-shadow', + '0px 1px 2px -2px rgba(0,0,0,0.08), 0px 3px 6px 0px rgba(0,0,0,0.06), 0px 5px 12px 4px rgba(0,0,0,0.03)', + ); + if (div.childNodes.length === 0) { + return ''; + } + return div.outerHTML; +} diff --git a/packages/secretnote-ui/src/components/ExecutionGraph/types.ts b/packages/secretnote-ui/src/components/ExecutionGraph/types.ts new file mode 100644 index 00000000..6013411c --- /dev/null +++ b/packages/secretnote-ui/src/components/ExecutionGraph/types.ts @@ -0,0 +1,80 @@ +import type * as G6 from '@antv/g6'; + +import type { LogicalLocation, Timeline } from '../../.openapi-stubs'; +import type { SnapshotReifier } from '../../utils/reify'; +import type { ElementOf } from '../../utils/typing'; + +import type { ColorizeFunction } from './colorization'; + +export type GraphNodeType = ElementOf< + NonNullable['nodes']> +>; + +export type GraphEdgeType = ElementOf< + NonNullable['edges']> +>; + +export type GraphElementType = GraphNodeType | GraphEdgeType; + +export type GraphUtils = { + reify: SnapshotReifier; + colorize: ColorizeFunction; +}; + +export type TrustedModel = + G6.ModelConfig & { + data: T; + colors?: { foreground: string; background: string }; + _utils: GraphUtils; + }; + +export type TrustedNode = G6.NodeConfig & { + data: T; + _utils: GraphUtils; +}; + +export type TrustedEdge = G6.EdgeConfig & { + id: string; + source: string; + target: string; + data: T; + _utils: GraphUtils; +}; + +export function isTrusted( + data: G6.NodeConfig, +): data is TrustedNode; + +export function isTrusted( + data: G6.EdgeConfig, +): data is TrustedEdge; + +export function isTrusted( + data: G6.ModelConfig, +): data is TrustedModel; + +export function isTrusted(data: unknown) { + return ( + typeof data === 'object' && + data !== null && + 'data' in data && + typeof data['data'] === 'object' && + data['data'] !== null && + 'kind' in data['data'] && + typeof data['data']['kind'] === 'string' + ); +} + +export function isOfKind( + kind: K, + item: G6.ModelConfig, +): item is TrustedModel> { + if (!isTrusted(item)) { + return false; + } + const data = item.data; + if (kind !== undefined && data.kind !== kind) { + return false; + } + return true; +} diff --git a/packages/secretnote-ui/src/components/ExecutionGraph/utils.ts b/packages/secretnote-ui/src/components/ExecutionGraph/utils.ts new file mode 100644 index 00000000..b6baa671 --- /dev/null +++ b/packages/secretnote-ui/src/components/ExecutionGraph/utils.ts @@ -0,0 +1,224 @@ +import type * as G6 from '@antv/g6'; +import { registerNode, registerEdge } from '@antv/g6'; +import type * as graphlib from '@antv/graphlib'; +import { Graph as PureGraph } from '@antv/graphlib'; +import isEqual from 'lodash/isEqual'; + +import type { Graph, LogicalLocation } from '../../.openapi-stubs'; + +import type { + GraphUtils, + GraphElementType, + TrustedNode, + TrustedEdge, + TrustedModel, + GraphNodeType, + GraphEdgeType, +} from './types'; +import { isTrusted } from './types'; + +type RenderingContext = { + item: Extract; + config: TrustedModel>; + renderer: G6.IGroup; + utils: GraphUtils; +}; + +type ShapeDefinition = { + kind: NonNullable; + render: (ctx: RenderingContext) => G6.IShape; + options?: Omit; +}; + +type ShapeOptions = G6.ShapeOptions & { kind: string }; + +export function defineShape({ + kind, + render, + options, +}: ShapeDefinition): ShapeOptions { + return { + kind, + draw: (config, renderer) => { + if (!isTrusted>(config)) { + throw new Error( + `Unexpected model for shape ${kind}: ${JSON.stringify(config)}`, + ); + } + const item = config.data; + const utils = config._utils; + const shape = render({ item, renderer, config, utils }); + config.size = [shape.attr('width'), shape.attr('height')]; + return shape; + }, + ...options, + }; +} + +export function registerShapes({ + nodes, + edges, +}: { + nodes: ShapeOptions[]; + edges: ShapeOptions[]; +}) { + const shapeIdentifier = (type: string, data: { kind?: string }) => + `${type}:${data.kind}`; + + nodes.forEach((node) => registerNode(shapeIdentifier('node', node), node)); + edges.forEach((edge) => registerEdge(shapeIdentifier('edge', edge), edge)); + + return function fromGraph(graph: Graph, _utils: GraphUtils): G6.GraphData { + return { + nodes: + graph.nodes?.map( + (node) => + ({ + id: node.id, + type: shapeIdentifier('node', node), + data: node, + _utils, + }) satisfies TrustedNode, + ) ?? [], + edges: + graph.edges?.map( + (edge) => + ({ + id: `${edge.source}-${edge.target}`, + source: edge.source, + target: edge.target, + type: shapeIdentifier('edge', edge), + data: edge, + _utils, + }) satisfies TrustedEdge, + ) ?? [], + }; + }; +} + +export function toPureGraph(graph: G6.Graph) { + const { nodes = [], edges = [] } = graph.save() as G6.GraphData; + return new PureGraph({ + nodes: nodes.filter(isTrusted) as TrustedNode[], + edges: edges.filter(isTrusted) as TrustedEdge[], + }); +} + +export function recursive< + V extends graphlib.PlainObject, + E extends graphlib.PlainObject, +>( + graph: graphlib.Graph, + origin: graphlib.ID, + filterer: (id: graphlib.ID) => graphlib.Node[], + stopWhen?: (v: graphlib.Node) => boolean, +): graphlib.Node[] { + const queue: graphlib.Node[] = [...filterer.bind(graph)(origin)]; + const all: graphlib.Node[] = [...queue]; + const seen = new Set(queue.map((n) => n.id)); + while (queue.length > 0) { + const node = queue.shift(); + if (!node) { + break; + } + if (stopWhen && stopWhen(graph.getNode(node.id))) { + continue; + } + const successors = filterer + .bind(graph)(node.id) + .filter((n) => !seen.has(n.id)); + successors.forEach((n) => { + seen.add(n.id); + queue.push(n); + all.push(n); + }); + } + return all; +} + +export function completePartition( + graph: graphlib.Graph, + matched: Set, +) { + [...matched].forEach((v) => + graph.getRelatedEdges(v, 'both').forEach((e) => { + if (matched.has(e.source) && matched.has(e.target)) { + matched.add(e.id); + } + }), + ); + const unmatched = new Set([ + ...graph + .getAllNodes() + .filter((n) => !matched.has(n.id)) + .map((n) => n.id), + ...graph + .getAllEdges() + .filter((e) => !matched.has(e.id)) + .map((e) => e.id), + ]); + return { matched, unmatched }; +} + +export type PartitionFunction = ( + graph: graphlib.Graph, + id: graphlib.ID, +) => Set; + +export const partitionByEntityType: PartitionFunction = (graph, id) => { + const matched = new Set( + (() => { + switch (graph.getNode(id).data.kind) { + case 'function': + return graph.getNeighbors(id); + case 'reveal': + return graph.getNeighbors(id); + case 'remote': + case 'local': + return [ + ...recursive(graph, id, graph.getPredecessors), + ...recursive(graph, id, graph.getSuccessors), + ]; + default: + return []; + } + // I miss Rust + })().map((v) => v.id), + ); + matched.add(id); + return matched; +}; + +export const partitionByLocation: PartitionFunction = (graph, id) => { + const byLocation: ( + location: LogicalLocation, + ) => (v: graphlib.Node) => boolean = (location) => (node) => { + switch (node.data.kind) { + case 'function': + return isEqual(node.data.location, location); + case 'remote': + return isEqual(node.data.data.location, location); + case 'local': + return graph.getSuccessors(node.id).some((v) => byLocation(location)(v)); + default: + return false; + } + }; + const matched = new Set( + (() => { + const node = graph.getNode(id); + switch (node.data.kind) { + case 'function': + return graph.getAllNodes().filter(byLocation(node.data.location)); + case 'remote': + return graph.getAllNodes().filter(byLocation(node.data.data.location)); + case 'local': + return graph.getAllNodes().filter((v) => v.data.kind === 'local'); + default: + return []; + } + })().map((v) => v.id), + ); + matched.add(id); + return matched; +}; diff --git a/packages/secretnote-ui/src/components/Visualization.tsx b/packages/secretnote-ui/src/components/Visualization.tsx new file mode 100644 index 00000000..b5eb11e9 --- /dev/null +++ b/packages/secretnote-ui/src/components/Visualization.tsx @@ -0,0 +1,16 @@ +import { Alert } from 'antd'; + +import type { Visualization as VisualizationProps } from '../.openapi-stubs'; + +import { DataProvider } from './DataProvider'; +import { ExecutionGraph } from './ExecutionGraph'; + +export function Visualization({ timeline }: VisualizationProps) { + return ( + Exception in cell output:}> + + + + + ); +} diff --git a/packages/secretnote-ui/src/index.ts b/packages/secretnote-ui/src/index.ts new file mode 100644 index 00000000..bd1a7c0f --- /dev/null +++ b/packages/secretnote-ui/src/index.ts @@ -0,0 +1,2 @@ +export { Visualization } from './components/Visualization'; +export { render } from './render'; diff --git a/packages/secretnote-ui/src/render.tsx b/packages/secretnote-ui/src/render.tsx new file mode 100644 index 00000000..de552498 --- /dev/null +++ b/packages/secretnote-ui/src/render.tsx @@ -0,0 +1,18 @@ +import { StrictMode } from 'react'; +import { createRoot } from 'react-dom/client'; + +export function render({ + elem, + Component, + props, +}: { + elem: HTMLElement; + Component: React.FC; + props?: Record; +}) { + createRoot(elem).render( + + + , + ); +} diff --git a/packages/secretnote-ui/src/utils/drawer.ts b/packages/secretnote-ui/src/utils/drawer.ts new file mode 100644 index 00000000..39b146ad --- /dev/null +++ b/packages/secretnote-ui/src/utils/drawer.ts @@ -0,0 +1,37 @@ +interface ShapeBuilder { + attr: (key: K, value: S[N][K]) => ShapeBuilder; + align: (value: string) => ShapeBuilder; + margin: (side: string, value: number) => ShapeBuilder; + floating: (enabled: boolean) => ShapeBuilder; + sibling: () => Drawer; + container: () => Drawer; +} + +interface Drawer { + shape: (name: N) => ShapeBuilder; +} + +interface Shapes { + rect: { width: number; height: number }; +} + +export function draw(): Drawer { + throw new Error('Not implemented'); + // const drawer: Drawer = { + // shape: (name) => { + // const builder: ShapeBuilder = { + // attr: (key, value) => { + // return builder; + // }, + // sibling: () => { + // return drawer; + // }, + // container: () => { + // return drawer; + // }, + // }; + // return builder; + // }, + // }; + // return drawer; +} diff --git a/packages/secretnote-ui/src/utils/reify.ts b/packages/secretnote-ui/src/utils/reify.ts new file mode 100644 index 00000000..4877a288 --- /dev/null +++ b/packages/secretnote-ui/src/utils/reify.ts @@ -0,0 +1,169 @@ +import type { Reference, ReferenceMap, Timeline } from '../.openapi-stubs'; + +type TaggedUnion = { kind?: Tag }; + +export type UnionMember, K extends T['kind']> = Extract< + T, + { kind?: K | undefined } +>; + +export type ReferenceResolver> = { + get: (key: string | number) => Reified | undefined; + ofKind: >( + kind: K, + key: string | number, + ) => Reified, T> | undefined; + items: () => Iterable<[string | number, Reified]>; + itemsOfKind: >( + kind: K, + ) => Iterable<[string | number, Reified, T>]>; +}; + +type ReferenceKeys = { + [K in keyof T]: T[K] extends ReferenceMap | undefined ? NonNullable : never; +}[keyof T]; + +// infer T_ distributes StaticRecord/DeferredRecord over the union members of T + +export type StaticRecord = T extends infer T_ ? Omit> : never; + +export type DeferredRecord> = T extends infer T_ + ? Record, ReferenceResolver> + : never; + +export type Reified> = StaticRecord & + DeferredRecord; + +export type SnapshotType = NonNullable[string]; + +export type SnapshotDiscriminator = NonNullable; + +export type SnapshotReifier = ( + kind: K | undefined, + ref: Reference | undefined, +) => Reified, SnapshotType> | undefined; + +// FIXME: improve these types + +function isReferenceList(value: unknown): value is Reference[] { + return Array.isArray(value); +} + +function isReferenceMap(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Object.hasOwn(value, 'kind'); +} + +export function reify( + rootKind: K | undefined, + rootRef: Reference | undefined, + variables: Record | undefined, +): Reified, SnapshotType> | undefined { + if (rootRef?.ref === undefined) { + return undefined; + } + + const root = variables?.[rootRef.ref]; + + if (root === undefined) { + return undefined; + } + + if (rootKind !== undefined && root.kind !== rootKind) { + return undefined; + } + + const staticItems = Object.fromEntries( + Object.entries(root).filter( + ([, value]) => !isReferenceList(value) && !isReferenceMap(value), + ), + ) as StaticRecord>; + + const deferredItems = Object.fromEntries( + Object.entries(root) + .filter(([, value]) => isReferenceMap(value) || isReferenceList(value)) + .map(([rootKey, _]) => { + const lookup: ReferenceMap = _; + + let getReference: (k: unknown) => Reference | undefined; + + let iterKeys: () => Generator< + readonly [string | number, Reference], + void, + unknown + >; + + if (isReferenceList(lookup)) { + getReference = (k) => lookup[Number(k)]; + + iterKeys = function* () { + for (let i = 0; i < lookup.length; i++) { + yield [i, lookup[i]] as const; + } + }; + } else { + getReference = (k) => lookup[String(k)]; + + iterKeys = function* () { + for (const [k, v] of Object.entries(lookup)) { + yield [k, v] as const; + } + }; + } + + const resolver: ReferenceResolver = { + get: (item) => { + const ref = getReference(item); + if (!ref) { + return undefined; + } + const value = variables?.[ref.ref]; + if (!value?.kind) { + return undefined; + } + return reify(value.kind, ref, variables); + }, + ofKind: (kind, item) => { + const ref = getReference(item); + if (!ref) { + return undefined; + } + const value = variables?.[ref.ref]; + if (value?.kind !== kind) { + return undefined; + } + return reify(kind, ref, variables); + }, + items: function* () { + for (const [key, ref] of iterKeys()) { + const value = variables?.[ref.ref]; + if (!value?.kind) { + continue; + } + const reified = reify(value.kind, ref, variables); + if (!reified) { + continue; + } + yield [key, reified]; + } + }, + itemsOfKind: function* (key) { + for (const [subkey, ref] of iterKeys()) { + const value = variables?.[ref.ref]; + if (value?.kind !== key) { + continue; + } + const reified = reify(key, ref, variables); + if (!reified) { + continue; + } + yield [subkey, reified]; + } + }, + }; + + return [rootKey, resolver]; + }), + ) as DeferredRecord, SnapshotType>; + + return { ...staticItems, ...deferredItems }; +} diff --git a/packages/secretnote-ui/src/utils/string.ts b/packages/secretnote-ui/src/utils/string.ts new file mode 100644 index 00000000..fee90485 --- /dev/null +++ b/packages/secretnote-ui/src/utils/string.ts @@ -0,0 +1,67 @@ +export function truncate( + text: string, + maxLength = 20, + placeholder = '...', + keep: 'start' | 'end' = 'start', +) { + const trimmed = text.trim(); + if (trimmed.length > maxLength) { + if (keep === 'start') { + return `${trimmed.slice(0, maxLength)}${placeholder}`; + } else { + return `${placeholder}${trimmed.slice(trimmed.length - maxLength)}`; + } + } + return text; +} + +export function truncateLines( + text: string, + { + maxWidth = 20, + maxLines = Infinity, + placeholder = '...', + }: { maxWidth?: number; maxLines?: number; placeholder?: string } = {}, +) { + const lines = text.split('\n'); + if (lines.length > maxLines) { + lines.splice(maxLines, lines.length - maxLines); + lines[maxLines - 1] = lines[maxLines - 1] + placeholder; + } + return lines.map((line) => truncate(line, maxWidth, placeholder)).join('\n'); +} + +export function wrap(text: string, breakOn: string, maxWidth = 20): string { + const parts = text.split(breakOn); + + if (!parts.length) { + return ''; + } + + const lines: string[] = []; + + let currentLine: string = parts.shift()!; + if (currentLine === undefined) { + return ''; + } + + parts.forEach((part) => { + if ((part + breakOn).length > maxWidth) { + if (currentLine) { + lines.push(currentLine); + } + lines.push(`${breakOn}${part}`); + } else if ((currentLine + breakOn + part).length > maxWidth) { + lines.push(currentLine); + currentLine = `${breakOn}${part}`; + } else { + currentLine += `${breakOn}${part}`; + } + }); + + if (currentLine) { + lines.push(currentLine); + } + + return lines.join('\n'); +} diff --git a/packages/secretnote-ui/src/utils/typing.ts b/packages/secretnote-ui/src/utils/typing.ts new file mode 100644 index 00000000..ee547d33 --- /dev/null +++ b/packages/secretnote-ui/src/utils/typing.ts @@ -0,0 +1 @@ +export type ElementOf = T extends (infer E)[] | undefined ? E : never; diff --git a/packages/secretnote-ui/src/vite-env.d.ts b/packages/secretnote-ui/src/vite-env.d.ts new file mode 100644 index 00000000..11f02fe2 --- /dev/null +++ b/packages/secretnote-ui/src/vite-env.d.ts @@ -0,0 +1 @@ +/// diff --git a/packages/secretnote-ui/tsconfig.json b/packages/secretnote-ui/tsconfig.json new file mode 100644 index 00000000..e993bba9 --- /dev/null +++ b/packages/secretnote-ui/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "build" + }, + "include": ["./src/**/*", "./src/.openapi-stubs/**/*"], + "references": [{ "path": "./tsconfig.node.json" }] +} diff --git a/packages/secretnote-ui/tsconfig.node.json b/packages/secretnote-ui/tsconfig.node.json new file mode 100644 index 00000000..cb112c7c --- /dev/null +++ b/packages/secretnote-ui/tsconfig.node.json @@ -0,0 +1,4 @@ +{ + "extends": "../../tsconfig.node.json", + "include": ["vite.config.ts", "package.json"] +} diff --git a/packages/secretnote-ui/vite.config.ts b/packages/secretnote-ui/vite.config.ts new file mode 100644 index 00000000..40251198 --- /dev/null +++ b/packages/secretnote-ui/vite.config.ts @@ -0,0 +1,78 @@ +import react from '@vitejs/plugin-react-swc'; +import { defineConfig, mergeConfig } from 'vite'; + +import { dependencies, peerDependencies } from './package.json'; + +const isDev = process.env['NODE_ENV'] === 'development'; + +const define = { 'process.env.NODE_ENV': JSON.stringify(process.env['NODE_ENV']) }; + +export default defineConfig(({ mode }) => { + const dependencyConfig = (() => { + switch (mode) { + case 'browser': + case 'deno': + case 'development': + // zero bundle for - - - -
-

Libro

- - -
- diff --git a/pyprojects/secretnote/package.json b/pyprojects/secretnote/package.json index 8ae441c9..d3e07108 100644 --- a/pyprojects/secretnote/package.json +++ b/pyprojects/secretnote/package.json @@ -2,13 +2,18 @@ "name": "secretnote", "private": true, "files": [], - "version": "0.0.1-dev.0", + "version": "0.0.0", "scripts": { - "build": "hatch clean && hatch build -t sdist -t wheel", - "test:pytest": "pytest", - "lint:black": "black --check src tests", - "lint:ruff": "ruff check src tests", - "lint:pyright": "pyright src", - "release": "hatch publish ./dist" + "format:black": "python -m black --check src tests", + "lint:ruff": "python -m ruff check src tests", + "publish": "python -m hatch publish ./dist", + "test:pytest": "python -m pytest", + "typecheck:pyright": "pyright --project ../.. src tests", + "dev": "NODE_ENV=development python -m secretnote.server --config=./.jupyter/config_dev.py --debug --no-browser", + "build": "python scripts/copy_static_files.py && python -m hatch clean && python -m hatch build -t sdist -t wheel" + }, + "devDependencies": { + "@secretflow/secretnote": "workspace:^", + "@secretflow/secretnote-ui": "workspace:^" } } diff --git a/pyprojects/secretnote/proto/buf.gen.yaml b/pyprojects/secretnote/proto/buf.gen.yaml new file mode 100644 index 00000000..e4d67801 --- /dev/null +++ b/pyprojects/secretnote/proto/buf.gen.yaml @@ -0,0 +1,5 @@ +version: v1 +plugins: + - plugin: buf.build/community/chrusty-jsonschema:v1.4.1 + opt: prefix_schema_files_with_package + out: generated/proto/jsonschema diff --git a/pyprojects/secretnote/proto/buf.lock b/pyprojects/secretnote/proto/buf.lock new file mode 100644 index 00000000..ada80d9e --- /dev/null +++ b/pyprojects/secretnote/proto/buf.lock @@ -0,0 +1,8 @@ +# Generated by buf. DO NOT EDIT. +version: v1 +deps: + - remote: buf.build + owner: opentelemetry + repository: opentelemetry + commit: c5370fbbc76844b595972771b6888e08 + digest: shake256:1cd6aa4b458ae4874f645bc35cb667c19e1edcced751a0f409c96b85110c7cbb52aab415898a9034c591fa0f3d92086824bba3b6f923cd72c26a24a44c5fa3f8 diff --git a/pyprojects/secretnote/proto/buf.yaml b/pyprojects/secretnote/proto/buf.yaml new file mode 100644 index 00000000..3f62967f --- /dev/null +++ b/pyprojects/secretnote/proto/buf.yaml @@ -0,0 +1,9 @@ +version: v1 +breaking: + use: + - FILE +lint: + use: + - DEFAULT +deps: + - buf.build/opentelemetry/opentelemetry diff --git a/pyprojects/secretnote/proto/secretnote/v1/secretnote.proto b/pyprojects/secretnote/proto/secretnote/v1/secretnote.proto new file mode 100644 index 00000000..b4ca2f79 --- /dev/null +++ b/pyprojects/secretnote/proto/secretnote/v1/secretnote.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; + +package secretnote.v1; + +import "opentelemetry/proto/trace/v1/trace.proto"; + +message Trace { + opentelemetry.proto.trace.v1.TracesData data = 1; +} diff --git a/pyprojects/secretnote/pyproject-poetry.toml b/pyprojects/secretnote/pyproject-poetry.toml deleted file mode 100644 index 54c9d05d..00000000 --- a/pyprojects/secretnote/pyproject-poetry.toml +++ /dev/null @@ -1,38 +0,0 @@ -[tool.poetry] -name = "secretnote" -version = "0.4.0" -description = "" -authors = ["wenyu.jqq "] -readme = "README.md" -packages = [{include = "libro_server"}] -include = ['README.md', 'setup.py', 'jupyter-config', 'libro_server/static/**/*', 'libro_server/template', 'libro_server/*.py'] - -[tool.poetry.scripts] -secretnote = 'libro_server.app:launch' - -[tool.poe.tasks] -dev = "secretnote --config=./dev-config/jupyter_server_config.py --debug" -publish = "twine upload dist/*" -gateway = 'jupyter lab --config=./dev-config/jupyter_server_config.py --debug --gateway-url=http://127.0.0.1:8888' - -[tool.poetry.dependencies] -python = ">=3.8,<3.12" -jupyter-server = "^2.7.0" -ipykernel = "^6.24.0" -jupyter-resource-usage = "^1.0.1" -ipython-sql = "^0.5.0" -pymysql = "^1.1.0" - - -[tool.poetry.group.dev.dependencies] -jupyter-lsp = "^2.2.0" -pandas = "^2.0.1" -transformers = "^4.31.0" -matplotlib = "^3.7.2" -poethepoet = "^0.21.1" -twine = "^4.0.2" - - -[build-system] -requires = ["poetry-core"] -build-backend = "poetry.core.masonry.api" diff --git a/pyprojects/secretnote/pyproject.toml b/pyprojects/secretnote/pyproject.toml index f4fd7bc0..c3ed1010 100644 --- a/pyprojects/secretnote/pyproject.toml +++ b/pyprojects/secretnote/pyproject.toml @@ -1,23 +1,42 @@ [project] -authors = [{name = "Tony Wu", email = "tonywu6@protonmail.com"}] +authors = [ + {name = "wenyu.jqq", email = "wenyu.jqq@antgroup.com"}, + {name = "Tony Wu", email = "tonywu6@protonmail.com"}, +] classifiers = [ - "Development Status :: 1 - Planning", + "Development Status :: 3 - Alpha", "License :: OSI Approved :: Apache Software License", ] dependencies = [ + "astunparse>=1.6.3", + "importlib-resources>=5.12", + "ipykernel>=6.24.0", + "ipython-sql~=0.5", + "ipywidgets>=8.1.1", + "jupyter-resource-usage~=1.0", + "jupyter-server~=2.7", "loguru>=0.7.2", + "more-itertools>=10.1.0", + "networkx>=2.8.8", + "opentelemetry-api>=1.20.0", + "opentelemetry-sdk>=1.20.0", + "orjson>=3.9.9", "packaging>=23.1", "pydantic>=1.10, <2", - "spu-stubs>=0.0.0", - "ruamel.yaml>=0.17.32", + "pymysql~=1.1", + "ray[default]~=2.6.3", # FIXME: ideally from upstream + "stack_data>=0.6.3", + "tqdm>=4.66.1", ] description = "Notebook suite for SecretFlow" dynamic = ["version"] +license = "Apache-2.0" name = "secretnote" readme = "README.md" -requires-python = ">=3.8, <3.9" +requires-python = ">=3.8, <3.12" [project.scripts] +secretnote = "secretnote.server.app:SecretNoteApp.launch_instance" [build-system] build-backend = "hatchling.build" @@ -32,7 +51,7 @@ path = "package.json" pattern = '"version": "(?P[^"]+)"' [tool.hatch.build] -artifacts = [] +artifacts = ["src/secretnote/**/dist/**/*"] directory = "dist" only-include = ["src", "package.json"] @@ -41,3 +60,9 @@ sources = ["src"] [tool.pytest.ini_options] pythonpath = ["src"] + +[tool.ruff] +extend = "../../pyproject.toml" + +[tool.ruff.extend-per-file-ignores] +"src/secretnote/proto/**/*" = ["E501", "I001"] diff --git a/pyprojects/secretnote/ruff.toml b/pyprojects/secretnote/ruff.toml deleted file mode 100644 index 45ffa636..00000000 --- a/pyprojects/secretnote/ruff.toml +++ /dev/null @@ -1,3 +0,0 @@ -exclude = [ - "jupyter_server_config.py" -] diff --git a/pyprojects/secretnote/scripts/copy_static_files.py b/pyprojects/secretnote/scripts/copy_static_files.py new file mode 100644 index 00000000..22cba7f8 --- /dev/null +++ b/pyprojects/secretnote/scripts/copy_static_files.py @@ -0,0 +1,7 @@ +import secretnote.display.core.renderer +import secretnote.server.app +from secretnote.utils.node import copy_static_files + +if __name__ == "__main__": + copy_static_files(secretnote.display.core.renderer.require) + copy_static_files(secretnote.server.app.require) diff --git a/pyprojects/secretnote/scripts/generate_proto.sh b/pyprojects/secretnote/scripts/generate_proto.sh new file mode 100755 index 00000000..ac24eace --- /dev/null +++ b/pyprojects/secretnote/scripts/generate_proto.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +PROTO_SOURCE=proto +PROTO_TEMPFILES=generated/proto +PROTO_TARGET=src/secretnote/proto + +rm -rf $PROTO_TEMPFILES $PROTO_TARGET + +buf generate \ + --template $PROTO_SOURCE/buf.gen.yaml \ + $PROTO_SOURCE + +python -m datamodel_code_generator \ + --input $PROTO_TEMPFILES/jsonschema/secretnote.v1/Trace.json \ + --input-file-type jsonschema \ + --base-class secretnote.utils.pydantic.ProtoModel \ + --output-model-type pydantic.BaseModel \ + --output $PROTO_TARGET diff --git a/pyprojects/secretnote/setup.py b/pyprojects/secretnote/setup.py deleted file mode 100644 index aefdf20d..00000000 --- a/pyprojects/secretnote/setup.py +++ /dev/null @@ -1 +0,0 @@ -__import__("setuptools").setup() diff --git a/pyprojects/secretnote/src/secretnote/display/__init__.py b/pyprojects/secretnote/src/secretnote/display/__init__.py new file mode 100644 index 00000000..ba73b4e3 --- /dev/null +++ b/pyprojects/secretnote/src/secretnote/display/__init__.py @@ -0,0 +1,3 @@ +from .app import visualize_run + +__all__ = ["visualize_run"] diff --git a/pyprojects/secretnote/src/secretnote/display/api.py b/pyprojects/secretnote/src/secretnote/display/api.py new file mode 100644 index 00000000..33d1dd00 --- /dev/null +++ b/pyprojects/secretnote/src/secretnote/display/api.py @@ -0,0 +1,10 @@ +from fastapi import FastAPI + +from .models import Visualization + +app = FastAPI() + + +@app.post("/visualize") +async def visualize(data: Visualization) -> None: + ... diff --git a/pyprojects/secretnote/src/secretnote/display/app.py b/pyprojects/secretnote/src/secretnote/display/app.py new file mode 100644 index 00000000..7d0497ad --- /dev/null +++ b/pyprojects/secretnote/src/secretnote/display/app.py @@ -0,0 +1,16 @@ +from secretnote.instrumentation.profiler import Profiler +from secretnote.utils.warnings import development_preview_warning + +from .core.renderer import render +from .models import Visualization +from .parsers.timeline import TimelineParser + + +def visualize_run(profiler: Profiler): + development_preview_warning() + + timeline = TimelineParser() + for span in profiler.exporter.iter_spans(): + timeline.feed(span) + timeline.digest() + return render(Visualization(timeline=timeline.export())) diff --git a/pyprojects/secretnote/src/secretnote/display/core/__init__.py b/pyprojects/secretnote/src/secretnote/display/core/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pyprojects/secretnote/src/secretnote/display/core/renderer.py b/pyprojects/secretnote/src/secretnote/display/core/renderer.py new file mode 100644 index 00000000..cec84206 --- /dev/null +++ b/pyprojects/secretnote/src/secretnote/display/core/renderer.py @@ -0,0 +1,36 @@ +from base64 import b64encode + +from IPython.core.display import HTML +from jinja2 import Template, select_autoescape +from pydantic import BaseModel + +from secretnote.utils.node import create_require + +require = create_require( + __package__, + "@secretflow/secretnote-ui/bundled", + "./templates/component.html", +) + + +def get_ui_bundle() -> str: + bundle = require.resolve("@secretflow/secretnote-ui/bundled").read_text() + bundle = b64encode(bundle.encode("utf-8")).decode("utf-8") + bundle_data_uri = f"data:text/javascript;base64,{bundle}" + return bundle_data_uri + + +def render(elem: BaseModel): + component_name = type(elem).__name__ + props = elem.json(by_alias=True, exclude_none=True) + encoded_props = b64encode(props.encode("utf-8")).decode("utf-8") + template = Template( + require.resolve("./templates/component.html").read_text(), + autoescape=select_autoescape(["html", "xml"], default_for_string=True), + ) + result = template.render( + component=component_name, + script_uri=get_ui_bundle(), + encoded_props=encoded_props, + ) + return HTML(result) diff --git a/pyprojects/secretnote/src/secretnote/display/core/templates/component.html b/pyprojects/secretnote/src/secretnote/display/core/templates/component.html new file mode 100644 index 00000000..d3170f92 --- /dev/null +++ b/pyprojects/secretnote/src/secretnote/display/core/templates/component.html @@ -0,0 +1,11 @@ + diff --git a/pyprojects/secretnote/src/secretnote/display/models.py b/pyprojects/secretnote/src/secretnote/display/models.py new file mode 100644 index 00000000..542f96e8 --- /dev/null +++ b/pyprojects/secretnote/src/secretnote/display/models.py @@ -0,0 +1,12 @@ +from pydantic import BaseModel + +from secretnote.utils.pydantic import ORJSONConfig + +from .parsers.timeline import Timeline + + +class Visualization(BaseModel): + timeline: Timeline + + class Config(ORJSONConfig): + pass diff --git a/pyprojects/secretnote/src/secretnote/display/parsers/__init__.py b/pyprojects/secretnote/src/secretnote/display/parsers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pyprojects/secretnote/src/secretnote/display/parsers/base.py b/pyprojects/secretnote/src/secretnote/display/parsers/base.py new file mode 100644 index 00000000..e0a62ba9 --- /dev/null +++ b/pyprojects/secretnote/src/secretnote/display/parsers/base.py @@ -0,0 +1,46 @@ +import abc +from typing import Callable, Generic, List, TypeVar + +from secretnote.utils.logging import log_dev_exception + +T = TypeVar("T", contravariant=True) +A = TypeVar("A") +R = TypeVar("R", covariant=True) + + +class Parser(abc.ABC, Generic[T, A, R]): + def __init__(self) -> None: + self.parsers: List[Callable[[T], R]] = [] + + @abc.abstractmethod + def data_name(self, data: T) -> str: + raise NotImplementedError + + @abc.abstractmethod + def rule_name(self, options: A) -> str: + raise NotImplementedError + + def parse(self, options: A): + rule_name = self.rule_name(options) + + def wrapper(fn: Callable[[T], R]) -> Callable[[T], R]: + def wrapped(data: T) -> R: + if self.data_name(data) != rule_name: + raise NotImplementedError + return fn(data) + + self.parsers.append(wrapped) + + return fn + + return wrapper + + def __call__(self, data: T): + for parser in self.parsers: + try: + return parser(data) + except NotImplementedError: + continue + except Exception as e: + log_dev_exception(e) + continue diff --git a/pyprojects/secretnote/src/secretnote/display/parsers/expression.py b/pyprojects/secretnote/src/secretnote/display/parsers/expression.py new file mode 100644 index 00000000..3831d07d --- /dev/null +++ b/pyprojects/secretnote/src/secretnote/display/parsers/expression.py @@ -0,0 +1,242 @@ +import inspect +from itertools import chain +from typing import ( + Callable, + Dict, + Iterable, + Optional, + Tuple, + cast, + overload, +) + +from more_itertools import first + +from secretnote.formal.symbols import ( + ExecExpression, + ExpressionType, + LocalObject, + LogicalLocation, + MoveExpression, + RemoteObject, + RevealExpression, +) +from secretnote.instrumentation.models import ( + DictSnapshot, + FunctionInfo, + ListSnapshot, + RemoteLocationSnapshot, + RemoteObjectSnapshot, + SnapshotType, + TracedFrame, +) +from secretnote.utils.pydantic import like_pytree + +from .base import Parser + +Options = Tuple[Callable, Tuple[int, ...]] + + +class ExpressionParser(Parser[TracedFrame, Options, ExpressionType]): + def rule_name(self, params: Options) -> str: + func, load_const = params + info = FunctionInfo.from_static(func, *load_const) + return info.function_name + + def data_name(self, data: TracedFrame) -> str: + return data.function_name + + def parse(self, func: Callable, *load_const: int): + return super().parse((func, load_const)) + + _did_setup = False + + def __call__(self, data: TracedFrame): + if not self._did_setup: + _setup() + self._did_setup = True + + return super().__call__(data) + + +def _get_parameters( + sig: inspect.Signature, + args: Iterable[SnapshotType], + kwargs: Iterable[Tuple[str, SnapshotType]], + defaults: Iterable[Tuple[str, SnapshotType]], +): + params: Dict[str, SnapshotType] = {k: v for k, v in defaults} + arguments = sig.bind_partial(*args, **{k: v for k, v in kwargs}).arguments + for name, item in arguments.items(): + param = sig.parameters[name] + if param.kind == param.VAR_POSITIONAL: + params.update( + { + f"{name}[{index}]{subkey}": subitem + for index, item in enumerate(item) + for subkey, subitem in like_pytree(item, SnapshotType) + } + ) + elif param.kind == param.VAR_KEYWORD: + params.update( + { + f"{name}{key}{subkey}": subitem + for key, item in item.items() + for subkey, subitem in like_pytree(item, SnapshotType) + } + ) + else: + params.update( + { + f"{name}{key}": subitem + for key, subitem in like_pytree(item, SnapshotType) + } + ) + return params + + +@overload +def _create_object(obj: SnapshotType, name: Optional[str] = None) -> LocalObject: + ... + + +@overload +def _create_object( + obj: RemoteObjectSnapshot, + name: Optional[str] = None, +) -> RemoteObject: + ... + + +def _create_object(obj, name=None): + if isinstance(obj, RemoteObjectSnapshot): + return RemoteObject(name=name, ref=obj.ref, location=obj.location) + return LocalObject(ref=obj.ref, name=name) + + +def _create_exec_expr( + func: SnapshotType, + location: LogicalLocation, + *, + args: Iterable[SnapshotType], + kwargs: Iterable[Tuple[str, SnapshotType]], + retvals: Iterable[Tuple[str, SnapshotType]], +): + if func.kind == "function": + name = func.name.split(".")[-1] + defaults = {k: v for k, v in func.default_args.of_type(SnapshotType)} + else: + name = str(func) + defaults = {} + + expr = ExecExpression(function=_create_object(func, name), location=location) + + try: + if func.kind != "function" or not func.signature: + raise TypeError + + signature = func.signature.reconstruct() + params = _get_parameters(signature, args, kwargs, defaults.items()).items() + + for key, item in params: + expr.boundvars.append(_create_object(item, key)) + + for key, item in chain( + func.closure_vars.of_type(SnapshotType), + func.global_vars.of_type(SnapshotType), + ): + expr.freevars.append(_create_object(item, key)) + + except TypeError: + for idx, item in enumerate(args): + expr.boundvars.append(_create_object(item, f"args[{idx}]")) + for key, item in kwargs: + expr.boundvars.append(_create_object(item, key)) + + for key, item in retvals: + expr.results.append(_create_object(item, key)) + + return expr + + +parser = ExpressionParser() + + +# FIXME: This is solely to avoid importing secretflow at the top level +def _setup(): + import secretflow + + @parser.parse(secretflow.PYU.__call__, 1) + def parse_pyu_call(frame: TracedFrame): + data = frame.get_frame() + func = data.local_vars[SnapshotType, "fn"] + + expr = _create_exec_expr( + func=func, + location=data.local_vars[RemoteLocationSnapshot, "self"].location, + args=data.local_vars[ListSnapshot, "args"].to_container(SnapshotType), + kwargs=data.local_vars[DictSnapshot, "kwargs"] + .to_container(SnapshotType) + .items(), + retvals=frame.iter_retvals(), + ) + + if func.bytecode_hash == frame.well_known.identity_function: + try: + source = first(expr.boundvars) + target = cast(RemoteObject, first(expr.results)) + return MoveExpression(source=source, target=target) + except ValueError: + pass + + return expr + + @parser.parse(secretflow.SPU.__call__, 1) + def parse_spu_call(frame: TracedFrame): + data = frame.get_frame() + return _create_exec_expr( + func=data.local_vars[SnapshotType, "func"], + location=data.local_vars[RemoteLocationSnapshot, "self"].location, + args=data.local_vars[ListSnapshot, "args"].to_container(SnapshotType), + kwargs=data.local_vars[DictSnapshot, "kwargs"] + .to_container(SnapshotType) + .items(), + retvals=frame.iter_retvals(), + ) + + @parser.parse(secretflow.device.kernels.pyu.pyu_to_pyu) + @parser.parse(secretflow.device.kernels.pyu.pyu_to_spu) + @parser.parse(secretflow.device.kernels.pyu.pyu_to_heu) + @parser.parse(secretflow.device.kernels.spu.spu_to_pyu) + @parser.parse(secretflow.device.kernels.spu.spu_to_spu) + @parser.parse(secretflow.device.kernels.spu.spu_to_heu) + @parser.parse(secretflow.device.kernels.heu.heu_to_pyu) + @parser.parse(secretflow.device.kernels.heu.heu_to_spu) + @parser.parse(secretflow.device.kernels.heu.heu_to_heu) + def parse_data_conversion(frame: TracedFrame): + data = frame.get_frame() + + source = data.local_vars[RemoteObjectSnapshot, "self"] + target_name, target = cast( + Tuple[str, RemoteObjectSnapshot], + first(frame.iter_retvals()), + ) + + return MoveExpression( + source=_create_object(source), + target=_create_object(target, target_name), + ) + + @parser.parse(secretflow.reveal) + def parse_reveal(frame: TracedFrame): + expr = RevealExpression(items=[], results=[]) + + inputs = frame.get_frame().local_vars[SnapshotType, "func_or_object"] + + for key, item in like_pytree(inputs, SnapshotType): + expr.items.append(_create_object(item, key)) + + for key, item in frame.iter_retvals(): + expr.results.append(_create_object(item, key)) + + return expr diff --git a/pyprojects/secretnote/src/secretnote/display/parsers/graph.py b/pyprojects/secretnote/src/secretnote/display/parsers/graph.py new file mode 100644 index 00000000..3c5e927d --- /dev/null +++ b/pyprojects/secretnote/src/secretnote/display/parsers/graph.py @@ -0,0 +1,267 @@ +from dataclasses import dataclass, field +from itertools import chain +from typing import ( + Generic, + Iterable, + List, + Literal, + Optional, + Tuple, + Type, + TypedDict, + TypeVar, + Union, + overload, +) + +import networkx as nx +from more_itertools import first +from pydantic import BaseModel, Field +from typing_extensions import Annotated + +from secretnote.formal.symbols import ( + ExecExpression, + ExpressionType, + LocalObject, + MoveExpression, + ObjectSymbolType, + RemoteObject, + RevealExpression, +) +from secretnote.instrumentation.models import LogicalLocation +from secretnote.utils.pydantic import Reference + +from .base import Parser + +M = TypeVar("M", bound=BaseModel) +T = TypeVar("T") + +V = TypeVar("V", bound="GraphNodeType") +E = TypeVar("E", bound="GraphEdgeType") + +TExpression = TypeVar("TExpression", bound=ExpressionType) +TObjectNode = TypeVar("TObjectNode", bound="ObjectNodeType") + + +class GraphNode(BaseModel): + id: str + epoch: int + order: int = 0 + + +class GraphEdge(BaseModel): + source: str + target: str + + +class LocalObjectNode(GraphNode): + kind: Literal["local"] = "local" + data: LocalObject + + +class RemoteObjectNode(GraphNode): + kind: Literal["remote"] = "remote" + data: RemoteObject + + +class FunctionNode(GraphNode): + kind: Literal["function"] = "function" + function: LocalObject + stackframe: Optional[LocalObject] + location: LogicalLocation + + +class RevealNode(GraphNode): + kind: Literal["reveal"] = "reveal" + + +class ArgumentEdge(GraphEdge): + kind: Literal["argument"] = "argument" + name: Optional[str] = None + + +class ReturnEdge(GraphEdge): + kind: Literal["return"] = "return" + assignment: Optional[str] = None + + +class TransformEdge(GraphEdge): + kind: Literal["transform"] = "transform" + destination: LogicalLocation + + +class ReferenceEdge(GraphEdge): + kind: Literal["reference"] = "reference" + + +class RevealEdge(GraphEdge): + kind: Literal["reveal"] = "reveal" + name: Optional[str] = None + + +ObjectNodeType = Union[LocalObjectNode, RemoteObjectNode] +GraphNodeType = Annotated[ + Union[ObjectNodeType, FunctionNode, RevealNode], + Field(discriminator="kind"), +] +GraphEdgeType = Annotated[ + Union[ArgumentEdge, ReturnEdge, TransformEdge, ReferenceEdge, RevealEdge], + Field(discriminator="kind"), +] + +T = Tuple[str, int] + + +class Graph(BaseModel): + nodes: List[GraphNodeType] = [] + edges: List[GraphEdgeType] = [] + + +class NodePosition(TypedDict): + id: str + epoch: int + order: int + + +@dataclass +class GraphState(Generic[TExpression]): + frame: Reference + state: nx.DiGraph + epoch: int + next_expr: TExpression + counter: int = 0 + changes: Graph = field(default_factory=Graph) + + def next_position(self, key: str) -> NodePosition: + self.counter += 1 + return { + "id": f"{key}@{self.epoch}:{self.counter}", + "epoch": self.epoch, + "order": self.counter, + } + + def add_node(self, node: V) -> V: + self.changes.nodes.append(node) + return node + + def add_edge(self, edge: GraphEdgeType) -> GraphEdgeType: + self.changes.edges.append(edge) + return edge + + @overload + def create_object_node(self, obj: LocalObject) -> LocalObjectNode: + ... + + @overload + def create_object_node(self, obj: RemoteObject) -> RemoteObjectNode: + ... + + def create_object_node(self, obj: ObjectSymbolType): + if isinstance(obj, LocalObject): + node = LocalObjectNode(data=obj, **self.next_position(obj.as_key())) + else: + node = RemoteObjectNode(data=obj, **self.next_position(obj.as_key())) + return node + + def all_nodes_of_type(self, type_: Type[M]) -> Iterable[M]: + for data in self.state.nodes.values(): + try: + yield type_.parse_obj(data) + except ValueError: + continue + + def most_recent_reference(self, reference: TObjectNode) -> TObjectNode: + filtered = filter( + lambda t: t.data.ref == reference.data.ref, + self.all_nodes_of_type(type(reference)), + ) + most_recent_first = sorted(filtered, key=lambda t: t.epoch, reverse=True) + return first(most_recent_first, default=reference) + + def reference_or_add_object(self, node: TObjectNode) -> TObjectNode: + last_node = self.most_recent_reference(node) + if last_node is node: + # This is the first time we've seen this object + self.changes.nodes.append(node) + return node + else: + return last_node + + +class GraphParser(Parser[GraphState, Type[ExpressionType], None]): + def rule_name(self, expr_type: Type[ExpressionType]) -> str: + return expr_type.__name__ + + def data_name(self, data: GraphState) -> str: + return type(data.next_expr).__name__ + + +parser = GraphParser() + + +@parser.parse(ExecExpression) +def parse_exec(self: GraphState[ExecExpression]): + expr = self.next_expr + + args: List[ObjectNodeType] = [] + arg_names: List[Union[str, None]] = [] + + for obj in chain(expr.boundvars, expr.freevars): + arg_names.append(obj.name) + if isinstance(obj, LocalObject): + args.append(self.add_node(self.create_object_node(obj))) + else: + args.append(self.reference_or_add_object(self.create_object_node(obj))) + + func = FunctionNode( + **self.next_position(expr.location.as_key()), + function=expr.function, + stackframe=LocalObject(ref=self.frame.ref) if self.frame else None, + location=expr.location, + ) + self.changes.nodes.append(func) + + for obj, name in zip(args, arg_names): + self.add_edge(ArgumentEdge(source=obj.id, target=func.id, name=name)) + + for obj in expr.results: + node = self.add_node(self.create_object_node(obj)) + self.add_edge(ReturnEdge(source=func.id, target=node.id, assignment=obj.name)) + + +@parser.parse(MoveExpression) +def parse_move(self: GraphState[MoveExpression]): + expr = self.next_expr + + if expr.source == expr.target: + return + + source = self.reference_or_add_object(self.create_object_node(expr.source)) + target = self.add_node(self.create_object_node(expr.target)) + + self.add_edge( + TransformEdge( + source=source.id, + target=target.id, + destination=expr.target.location, + ) + ) + + +@parser.parse(RevealExpression) +def parse_reveal(self: GraphState[RevealExpression]): + expr = self.next_expr + + args: List[ObjectNodeType] = [] + + for item in expr.items: + args.append(self.reference_or_add_object(self.create_object_node(item))) + + reveal = self.add_node(RevealNode(**self.next_position("reveal"))) + + for node in args: + self.add_edge(RevealEdge(source=node.id, target=reveal.id)) + + for result in expr.results: + node = self.add_node(self.create_object_node(result)) + self.add_edge(RevealEdge(source=reveal.id, target=node.id)) diff --git a/pyprojects/secretnote/src/secretnote/display/parsers/timeline.py b/pyprojects/secretnote/src/secretnote/display/parsers/timeline.py new file mode 100644 index 00000000..17f626b3 --- /dev/null +++ b/pyprojects/secretnote/src/secretnote/display/parsers/timeline.py @@ -0,0 +1,169 @@ +from typing import Dict, List, Optional, Union, cast + +import networkx as nx +from pydantic import BaseModel + +from secretnote.formal.symbols import ExpressionType, LocalObject, RemoteObject +from secretnote.instrumentation.models import ( + APILevel, + FunctionCheckpoint, + OTelSpanDict, + SnapshotType, + TracedFrame, +) +from secretnote.instrumentation.sdk import get_traced_frame +from secretnote.utils.pydantic import Reference + +from .expression import parser as parse_expression +from .graph import Graph, GraphState +from .graph import parser as parse_graph + + +class Frame(BaseModel): + span_id: str + parent_span_id: Optional[str] + start_time: str + end_time: str + + epoch: int = 0 + checkpoints: List[FunctionCheckpoint] = [] + function: Optional[Reference] = None + frame: Optional[Reference] = None + retval: Optional[Reference] = None + expression: Optional[ExpressionType] = None + + inner_frames: List["Frame"] = [] + + +class Timeline(BaseModel): + variables: Dict[str, SnapshotType] = {} + timeline: List[Frame] = [] + graph: Graph + + +class TimelineParser: + def __init__(self): + self.frames: Dict[str, Frame] = {} + self.object_refs: Dict[str, RemoteObject] = {} + self.trace_data: Dict[str, TracedFrame] = {} + self.variables: Dict[str, SnapshotType] = {} + self.call_graph = nx.DiGraph() + self.data_graph = nx.DiGraph() + self._pending_frames: Dict[str, Frame] = {} + + @property + def top_level_calls(self): + return [span_id for span_id, degree in self.call_graph.in_degree if degree == 0] + + def feed(self, raw_frame: OTelSpanDict): + span_id = raw_frame.context.span_id + parent_span_id = raw_frame.parent_id + start_time = raw_frame.start_time + end_time = raw_frame.end_time + self._pending_frames[span_id] = Frame( + span_id=span_id, + parent_span_id=parent_span_id, + start_time=start_time.isoformat(), + end_time=end_time.isoformat(), + ) + frame_data = get_traced_frame(raw_frame) + if frame_data is not None: + self.trace_data[span_id] = frame_data + + def digest(self): + for frame in sorted(self._pending_frames.values(), key=lambda f: f.end_time): + del self._pending_frames[frame.span_id] + self.frames[frame.span_id] = frame + self.digest_ordered(frame) + + def digest_ordered(self, frame: Frame): + if frame.parent_span_id: + self.call_graph.add_edge(frame.parent_span_id, frame.span_id) + else: + self.call_graph.add_node(frame.span_id) + + frame_data = self.trace_data.get(frame.span_id) + if frame_data is None: + return + + checkpoints = [ + f.checkpoint + for sid in self.call_stack(frame.span_id) + if (f := self.trace_data.get(sid)) is not None + ] + + frame.epoch = len(self.frames) + frame.checkpoints = checkpoints + frame.function = frame_data.function + frame.frame = frame_data.frame + frame.retval = frame_data.retval + + self.variables.update(frame_data.variables) + + frame.expression = parse_expression(frame_data) + + is_invariant = ( + len(checkpoints) == 1 + and checkpoints[0].semantics.api_level == APILevel.INVARIANT + ) + + if is_invariant and frame.expression: + for obj in frame.expression.objects(): + self.add_object(obj) + + state = GraphState( + frame=frame.frame, + state=self.data_graph, + epoch=frame.epoch, + next_expr=frame.expression, + ) + parse_graph(state) + for node in state.changes.nodes: + self.data_graph.add_node(node.id, **node.dict()) + for edge in state.changes.edges: + self.data_graph.add_edge(edge.source, edge.target, **edge.dict()) + + def export(self) -> Timeline: + timeline: List[Frame] = [] + all_frames: List[Frame] = [] + + for span_id, frame in self.frames.items(): + frame = frame.copy() + frame.inner_frames = [] + all_frames.append(frame) + if self.call_graph.in_degree(span_id) == 0: + timeline.append(frame) + for _source, target in self.call_graph.out_edges(span_id): + inner_frame = self.frames[target] + frame.inner_frames.append(inner_frame) + + for frame in all_frames: + frame.inner_frames = sorted(frame.inner_frames, key=lambda f: f.start_time) + + graph = Graph.parse_obj( + { + "nodes": [data for data in self.data_graph.nodes.values()], + "edges": [data for data in self.data_graph.edges.values()], + } + ) + + return Timeline(timeline=timeline, graph=graph, variables=self.variables) + + def add_object(self, obj: Union[LocalObject, RemoteObject]): + if not isinstance(obj, RemoteObject): + return + existing = self.object_refs.get(obj.ref) + if existing: + obj.numbering = existing.numbering + else: + self.object_refs[obj.ref] = obj + obj.numbering = len(self.object_refs) + + def call_stack(self, span_id: str) -> List[str]: + for top in self.top_level_calls: + try: + return cast(List[str], nx.shortest_path(self.call_graph, top, span_id)) + except nx.NetworkXNoPath: + continue + else: + raise ValueError(f"Cannot find path to {span_id}") diff --git a/pyprojects/secretnote/src/secretnote/formal/__init__.py b/pyprojects/secretnote/src/secretnote/formal/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pyprojects/secretnote/src/secretnote/formal/locations/__init__.py b/pyprojects/secretnote/src/secretnote/formal/locations/__init__.py new file mode 100644 index 00000000..758dd6c6 --- /dev/null +++ b/pyprojects/secretnote/src/secretnote/formal/locations/__init__.py @@ -0,0 +1,22 @@ +from .pyu import SymbolicPYU +from .spu import SPUFieldType, SPUProtocolKind, SymbolicSPU +from .utils import OnDemandDevice, PortBinding +from .world import ( + SFConfigNetworked, + SFConfigSimulation, + SFConfigSimulationWithExternalRay, + SymbolicWorld, +) + +__all__ = [ + "OnDemandDevice", + "PortBinding", + "SymbolicPYU", + "SymbolicSPU", + "SPUProtocolKind", + "SPUFieldType", + "SFConfigNetworked", + "SFConfigSimulation", + "SFConfigSimulationWithExternalRay", + "SymbolicWorld", +] diff --git a/pyprojects/secretnote/src/secretnote/formal/locations/pyu.py b/pyprojects/secretnote/src/secretnote/formal/locations/pyu.py new file mode 100644 index 00000000..b8c44ca7 --- /dev/null +++ b/pyprojects/secretnote/src/secretnote/formal/locations/pyu.py @@ -0,0 +1,8 @@ +class SymbolicPYU: + def __init__(self, name: str): + self.name = name + + def reify(self): + import secretflow + + return secretflow.PYU(self.name) diff --git a/pyprojects/secretnote/src/secretnote/formal/locations/spu.py b/pyprojects/secretnote/src/secretnote/formal/locations/spu.py new file mode 100644 index 00000000..af9f14b8 --- /dev/null +++ b/pyprojects/secretnote/src/secretnote/formal/locations/spu.py @@ -0,0 +1,50 @@ +from enum import IntEnum +from typing import FrozenSet + +from pydantic import BaseModel, validate_arguments + +from .utils import PortBinding + + +class SPUProtocolKind(IntEnum): + REF2K = 1 + SEMI2K = 2 + ABY3 = 3 + CHEETAH = 4 + + +class SPUFieldType(IntEnum): + FM32 = 1 + FM64 = 2 + FM128 = 3 + + +class SymbolicSPU(BaseModel): + world: FrozenSet[str] + protocol: SPUProtocolKind + field: SPUFieldType + fxp_fraction_bits: int + + @validate_arguments + def reify(self, **network: PortBinding): + from secretflow.device.device.spu import SPU + + nodes = [ + { + "party": party, + "address": network[party].announced_as, + "listen_addr": network[party].bind_to, + } + for party in self.world + ] + + return SPU( + cluster_def={ + "nodes": nodes, + "runtime_config": { + "protocol": int(self.protocol), + "field": int(self.field), + "fxp_fraction_bits": self.fxp_fraction_bits, + }, + }, + ) diff --git a/pyprojects/secretnote/src/secretnote/formal/locations/utils.py b/pyprojects/secretnote/src/secretnote/formal/locations/utils.py new file mode 100644 index 00000000..a7a64bee --- /dev/null +++ b/pyprojects/secretnote/src/secretnote/formal/locations/utils.py @@ -0,0 +1,53 @@ +from typing import Dict, Type, Union, overload + +from pydantic import BaseModel +from secretflow import HEU, PYU, SPU + +SupportedDevices = Union[Type[PYU], Type[SPU], Type[HEU]] + + +class OnDemandDevice: + def __init__(self, world: Dict): + self.world = world + + @overload + def __call__(self, kind: Type[PYU], party: str, *parties: str) -> PYU: + ... + + @overload + def __call__(self, kind: Type[SPU], party: str, *parties: str) -> SPU: + ... + + @overload + def __call__(self, kind: Type[HEU], party: str, *parties: str) -> HEU: + ... + + def __call__(self, kind: SupportedDevices, party: str, *parties: str): + if kind is PYU: + return PYU(party) + if kind is SPU: + expected_actors = {party, *parties} + for variable in self.world.values(): + if isinstance(variable, SPU): + if expected_actors == set(variable.actors): + return variable + raise ValueError(f"No suitable SPU found for {expected_actors}") + if kind == HEU: + private_key_owner = party + public_key_evaluators = parties + for variable in self.world.values(): + if isinstance(variable, HEU): + if ( + private_key_owner == variable.sk_keeper_name() + and public_key_evaluators == tuple(variable.evaluator_names()) + ): + return variable + raise ValueError( + "No suitable HEU found for " + f"{private_key_owner} and {public_key_evaluators}" + ) + + +class PortBinding(BaseModel): + announced_as: str + bind_to: str = "" diff --git a/pyprojects/secretnote/src/secretnote/formal/locations/world.py b/pyprojects/secretnote/src/secretnote/formal/locations/world.py new file mode 100644 index 00000000..32ea9b43 --- /dev/null +++ b/pyprojects/secretnote/src/secretnote/formal/locations/world.py @@ -0,0 +1,62 @@ +from typing import Dict, FrozenSet, Union + +from pydantic import BaseModel, validate_arguments + +from secretnote.formal.locations.utils import PortBinding + + +class SFConfigSimulation(BaseModel): + pass + + +class SFConfigSimulationWithExternalRay(BaseModel): + ray_address: str + + +class SFConfigNetworked(BaseModel): + self_party: str + ray_address: str + network: Dict[str, PortBinding] + + +class SymbolicWorld(BaseModel): + world: FrozenSet[str] + + @validate_arguments + def reify( + self, + config: Union[ + SFConfigNetworked, + SFConfigSimulationWithExternalRay, + SFConfigSimulation, + ], + ): + import secretflow + + if isinstance(config, SFConfigSimulation): + secretflow.init(parties=[*self.world], address="local") + return + + if isinstance(config, SFConfigSimulationWithExternalRay): + secretflow.init(parties=[*self.world], address=config.ray_address) + return + + parties = {} + + for k in self.world: + addr = {"address": config.network[k].announced_as} + if config.network[k].bind_to: + addr["listen_addr"] = config.network[k].bind_to + parties[k] = addr + + assert ( + config.self_party in parties + ), f"party {config.self_party} not in {parties}" + + secretflow.init( + address=config.ray_address, + cluster_config={ + "parties": parties, + "self_party": config.self_party, + }, + ) diff --git a/pyprojects/secretnote/src/secretnote/formal/symbols.py b/pyprojects/secretnote/src/secretnote/formal/symbols.py new file mode 100644 index 00000000..283acf25 --- /dev/null +++ b/pyprojects/secretnote/src/secretnote/formal/symbols.py @@ -0,0 +1,165 @@ +from typing import Any, Dict, List, Literal, Optional, Tuple, Union + +from pydantic import BaseModel, Field +from typing_extensions import Annotated + + +class LogicalLocation(BaseModel): + kind: Literal["location"] = "location" + + type: str + parties: Tuple[str, ...] + parameters: Dict[str, Any] = {} + + def __eq__(self, other: object) -> bool: + return ( + isinstance(other, type(self)) + and self.type == other.type + and self.parameters == other.parameters + and self.parties == other.parties + ) + + def __hash__(self) -> int: + return hash((self.type, self.parties, *self.parameters.items())) + + def __str__(self) -> str: + return f"{self.type}[{', '.join(self.parties)}]" + + def as_key(self) -> str: + args = [ + self.type, + *self.parties, + *[f"{k}={v}" for k, v in self.parameters.items()], + ] + return ":".join(args) + + +class RemoteObject(BaseModel): + kind: Literal["remote_object"] = "remote_object" + + numbering: int = -1 + ref: str + location: LogicalLocation + + name: Optional[str] = None + + def __eq__(self, other: object) -> bool: + return ( + isinstance(other, type(self)) + and self.ref == other.ref + and self.numbering == other.numbering + ) + + def __hash__(self) -> int: + return hash((self.numbering, self.ref)) + + def __str__(self) -> str: + if self.numbering == -1: + return f"{self.location.type[0]}({self.ref[-7:]})" + return f"{self.location.type[0]}({self.numbering})" + + def as_key(self): + return self.ref + + +class LocalObject(BaseModel): + kind: Literal["local_object"] = "local_object" + ref: str + name: Optional[str] = None + + def __eq__(self, other: object) -> bool: + return isinstance(other, type(self)) and self.ref == other.ref + + def __hash__(self) -> int: + return hash((self.ref,)) + + def __str__(self) -> str: + if not self.name: + return self.ref + return self.name + + def as_key(self): + return self.ref + + +class ExecExpression(BaseModel): + kind: Literal["exec"] = "exec" + function: LocalObject + location: LogicalLocation + boundvars: List[Union[LocalObject, RemoteObject]] = [] + freevars: List[Union[LocalObject, RemoteObject]] = [] + results: List[Union[LocalObject, RemoteObject]] = [] + + def __str__(self) -> str: + invariants = [] + static_args = [] + + for arg in self.boundvars: + if isinstance(arg, RemoteObject): + invariants.append(arg) + else: + static_args.append(arg) + + invariant_str = ", ".join(map(str, invariants)) or "()" + static_args_str = ", ".join(map(str, static_args)) + freevars_str = ", ".join(map(str, self.freevars)) + result_str = ", ".join(map(str, self.results)) + + location_str = str(self.location) + label = self.function.name if self.function else "?" + + return ( + f"{result_str} ::= {location_str}. let {invariant_str} in {label}" + f" | ({static_args_str}) + ({freevars_str})" + ) + + def objects(self): + if self.function: + yield self.function + yield from self.boundvars + yield from self.freevars + yield from self.results + + +class MoveExpression(BaseModel): + kind: Literal["move"] = "move" + source: Union[RemoteObject, LocalObject] + target: RemoteObject + + def __str__(self): + return f"{self.target} <~~ {self.source} / move to {self.target.location}" + + def objects(self): + yield self.source + yield self.target + + +class RevealExpression(BaseModel): + kind: Literal["reveal"] = "reveal" + items: List[Union[RemoteObject, LocalObject]] + results: List[LocalObject] + + def __str__(self) -> str: + output = ", ".join(map(str, self.results)) + inputs = ", ".join(map(str, self.items)) + return f"{output} <== reveal {inputs}" + + def objects(self): + yield from self.items + yield from self.results + + +ExpressionType = Annotated[ + Union[ExecExpression, MoveExpression, RevealExpression], + Field(discriminator="kind"), +] + +ObjectSymbolType = Annotated[ + Union[RemoteObject, LocalObject], + Field(discriminator="kind"), +] + +SymbolType = Annotated[ + Union[LogicalLocation, ObjectSymbolType], + Field(discriminator="kind"), +] diff --git a/pyprojects/secretnote/src/secretnote/instrumentation/__init__.py b/pyprojects/secretnote/src/secretnote/instrumentation/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pyprojects/secretnote/src/secretnote/instrumentation/checkpoint.py b/pyprojects/secretnote/src/secretnote/instrumentation/checkpoint.py new file mode 100644 index 00000000..b6b76dc3 --- /dev/null +++ b/pyprojects/secretnote/src/secretnote/instrumentation/checkpoint.py @@ -0,0 +1,28 @@ +from types import FrameType +from typing import Callable, Dict, Optional, TypeVar + +from .models import FunctionCheckpoint, FunctionInfo, Semantics +from .snapshot import hash_digest + +T = TypeVar("T", bound=Callable) + + +class CheckpointGroup: + def __init__(self): + self.checkpoints: Dict[str, FunctionCheckpoint] = {} + + def match_frame(self, frame: FrameType) -> Optional[FunctionCheckpoint]: + return self.checkpoints.get(hash_digest(frame.f_code)) + + def add_function(self, fn: Callable, *load_const: int, semantics: Semantics): + function = FunctionInfo.from_static(fn, *load_const) + ckpt = FunctionCheckpoint(function=function) + ckpt.semantics = semantics + self.checkpoints[ckpt.function.code_hash] = ckpt + + def tracing_checkpoint(self, semantics: Semantics): + def decorator(fn: T) -> T: + self.add_function(fn, semantics=semantics) + return fn + + return decorator diff --git a/pyprojects/secretnote/src/secretnote/instrumentation/envvars.py b/pyprojects/secretnote/src/secretnote/instrumentation/envvars.py new file mode 100644 index 00000000..2a63374b --- /dev/null +++ b/pyprojects/secretnote/src/secretnote/instrumentation/envvars.py @@ -0,0 +1,3 @@ +OTEL_PYTHON_SECRETNOTE_W3C_TRACE = "OTEL_PYTHON_SECRETNOTE_W3C_TRACE" + +OTEL_PYTHON_SECRETNOTE_PROFILER_FRAME = "secretnote.profiler.frame" diff --git a/pyprojects/secretnote/src/secretnote/instrumentation/exporters.py b/pyprojects/secretnote/src/secretnote/instrumentation/exporters.py new file mode 100644 index 00000000..bb9f8322 --- /dev/null +++ b/pyprojects/secretnote/src/secretnote/instrumentation/exporters.py @@ -0,0 +1,87 @@ +from typing import BinaryIO, Iterable, Protocol + +import orjson +from opentelemetry.sdk.trace import ReadableSpan +from opentelemetry.sdk.trace.export import SpanExportResult +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter as _InMemorySpanExporter, +) +from opentelemetry.sdk.util import ns_to_iso_str +from opentelemetry.trace import format_span_id + +from .models import OTelSpanDict + + +def parse_span(raw_span: ReadableSpan) -> OTelSpanDict: + parent_id = None + if raw_span.parent is not None: + parent_id = f"0x{format_span_id(raw_span.parent.span_id)}" + + start_time = None + if raw_span._start_time: + start_time = ns_to_iso_str(raw_span._start_time) + + end_time = None + if raw_span._end_time: + end_time = ns_to_iso_str(raw_span._end_time) + + if raw_span._status is not None: + status = {} + status["status_code"] = str(raw_span._status.status_code.name) + if raw_span._status.description: + status["description"] = raw_span._status.description + else: + status = None + + f_span = {} + + f_span["name"] = raw_span._name + f_span["context"] = raw_span._format_context(raw_span._context) + f_span["kind"] = str(raw_span.kind) + f_span["parent_id"] = parent_id + f_span["start_time"] = start_time + f_span["end_time"] = end_time + if status: + f_span["status"] = status + f_span["attributes"] = raw_span._format_attributes(raw_span._attributes) + f_span["events"] = raw_span._format_events(raw_span._events) + f_span["links"] = raw_span._format_links(raw_span._links) + f_span["resource"] = { + "attributes": dict(raw_span.resource._attributes), + "schema_url": raw_span.resource._schema_url, + } + + return OTelSpanDict.parse_obj(f_span) + + +class SpanReader(Protocol): + def iter_spans(self) -> Iterable[OTelSpanDict]: + ... + + +class InMemorySpanExporter(_InMemorySpanExporter, SpanReader): + def iter_spans(self) -> Iterable[OTelSpanDict]: + for span in self.get_finished_spans(): + yield parse_span(span) + + +class JSONLinesSpanExporter(_InMemorySpanExporter): + def __init__(self, file: BinaryIO) -> None: + self.file = file + + def flush(self) -> None: + for span in self._finished_spans: + span_dict = parse_span(span) + self.file.write(orjson.dumps(span_dict.dict(exclude_none=True))) + self.file.write(b"\n") + + def force_flush(self, timeout_millis: int = 30000) -> bool: + self.flush() + self.file.flush() + return True + + def export(self, spans): + result = super().export(spans) + if result is SpanExportResult.SUCCESS: + self.flush() + return result diff --git a/pyprojects/secretnote/src/secretnote/instrumentation/models.py b/pyprojects/secretnote/src/secretnote/instrumentation/models.py new file mode 100644 index 00000000..4cd80ac7 --- /dev/null +++ b/pyprojects/secretnote/src/secretnote/instrumentation/models.py @@ -0,0 +1,599 @@ +import abc +import inspect +from dataclasses import dataclass +from datetime import datetime +from enum import IntEnum +from textwrap import dedent +from types import CodeType, FrameType +from typing import ( + Any, + Callable, + Dict, + Iterable, + List, + Literal, + Mapping, + Optional, + Sequence, + Tuple, + Type, + TypeVar, + Union, +) + +import stack_data.core +from opentelemetry.util.types import Attributes +from pydantic import BaseModel, Field, PrivateAttr +from typing_extensions import Annotated, override + +from secretnote.formal.symbols import LogicalLocation +from secretnote.utils.pydantic import ( + ProxiedModel, + Reference, + ReferenceMap, + like_pytree, + update_forward_refs, +) + +from .snapshot import ( + bytecode_hash, + find_globals, + fingerprint, + hash_digest, + logical_location, + qualname, + qualname_tuple, + source_path, + to_string, + type_annotation, +) + +T = TypeVar("T", bound="SnapshotType") + + +class APILevel(IntEnum): + IMPLEMENTATION = 10 + INVARIANT = 20 + USERLAND = 90 + + +class ObjectTracer(abc.ABC): + @classmethod + @abc.abstractmethod + def typecheck(cls, x) -> bool: + ... + + @classmethod + @abc.abstractmethod + def trace(cls, x) -> "SnapshotType": + ... + + @classmethod + def tree(cls, x) -> Dict[str, Union[List, Dict]]: + return {} + + +class NoneSnapshot(ObjectTracer, Reference, ProxiedModel): + kind: Literal["none"] = "none" + + @classmethod + def typecheck(cls, x) -> bool: + return x is None + + @classmethod + def trace(cls, x) -> "SnapshotType": + return NoneSnapshot(ref=fingerprint(x)) + + +class ObjectSnapshot(ObjectTracer, Reference, ProxiedModel): + kind: Literal["object"] = "object" + type: str + snapshot: str + + @classmethod + def typecheck(cls, x) -> bool: + return isinstance(x, object) + + @classmethod + def trace(cls, x) -> "SnapshotType": + return ObjectSnapshot( + ref=fingerprint(x), + type=qualname(type(x)), + snapshot=to_string(x), + ) + + @classmethod + def none(cls): + return cls.trace(None) + + def __str__(self) -> str: + return f"{type(self)}[{self.snapshot}]" + + +class ListSnapshot(ObjectTracer, Reference, ProxiedModel): + kind: Literal["list"] = "list" + type: str + snapshot: str + values: ReferenceMap = ReferenceMap.empty_list() + + @classmethod + def typecheck(cls, x) -> bool: + if isinstance(x, (str, bytes, bytearray, memoryview)): + return False + return isinstance(x, Sequence) + + @classmethod + def trace(cls, x) -> "SnapshotType": + return ListSnapshot( + ref=fingerprint(x), + type=qualname(type(x)), + snapshot=to_string(x), + ) + + @classmethod + def tree(cls, x) -> Dict[str, Union[Dict, List]]: + try: + # namedtuple + values = x._asdict() + assert isinstance(values, dict) + return {"values": {**values}} + except Exception: + pass + return {"values": [*x]} + + @override + def to_container(self, of_type: Type[T] = Any): + return [v for k, v in self.values.of_type(of_type)] + + def __str__(self) -> str: + return f"{type(self)}[{self.snapshot}]" + + +class DictSnapshot(ObjectTracer, Reference, ProxiedModel): + kind: Literal["dict"] = "dict" + type: str + snapshot: str + values: ReferenceMap = ReferenceMap.empty_dict() + + @classmethod + def typecheck(cls, x): + return isinstance(x, Mapping) + + @classmethod + def trace(cls, x) -> "SnapshotType": + return DictSnapshot( + ref=fingerprint(x), + type=qualname(type(x)), + snapshot=to_string(x), + ) + + @classmethod + def tree(cls, x) -> Dict[str, Union[Dict, List]]: + return {"values": {**x}} + + @override + def to_container(self, of_type: Type[T] = Any): + return {k: v for k, v in self.values.of_type(of_type)} + + def __str__(self) -> str: + return f"{type(self)}[{self.snapshot}]" + + +class RemoteLocationSnapshot(ObjectTracer, Reference, ProxiedModel): + kind: Literal["remote_location"] = "remote_location" + type: str + location: LogicalLocation + + @classmethod + def typecheck(cls, x): + from secretflow.device.device import Device + + return isinstance(x, Device) + + @classmethod + def trace(cls, x) -> "SnapshotType": + from .snapshot import logical_location + + return RemoteLocationSnapshot( + ref=fingerprint(x), + type=qualname(type(x)), + location=logical_location(x), + ) + + def __str__(self) -> str: + return str(self.location) + + +class RemoteObjectSnapshot(ObjectTracer, Reference, ProxiedModel): + kind: Literal["remote_object"] = "remote_object" + type: str + location: LogicalLocation + refs: Tuple[str, ...] + + @classmethod + def typecheck(cls, x) -> bool: + from secretflow.device.device import DeviceObject + + return isinstance(x, DeviceObject) + + @classmethod + def trace(cls, x): + from secretflow.device.device import HEUObject, PYUObject, SPUObject, TEEUObject + + if isinstance(x, PYUObject): + refs = (fingerprint(x.data),) + elif isinstance(x, SPUObject): + refs = (fingerprint(x.meta), *map(fingerprint, x.shares_name)) + elif isinstance(x, HEUObject): + refs = (fingerprint(x.data),) + elif isinstance(x, TEEUObject): + refs = (fingerprint(x.data),) + else: + raise TypeError(f"Unknown device object type {type(x)}") + + return RemoteObjectSnapshot( + ref=fingerprint(x), + type=qualname(type(x)), + location=logical_location(x.device), + refs=refs, + ) + + def __str__(self) -> str: + return f"{self.ref} @ {self.location}" + + +class FunctionParameter(BaseModel): + name: str + kind: inspect._ParameterKind + annotation: Optional[str] = None + + def __str__(self) -> str: + if self.annotation is None: + return self.name + return f"{self.name}: {self.annotation}" + + +class FunctionSignature(BaseModel): + parameters: List[FunctionParameter] = [] + return_annotation: Optional[str] = None + + def reconstruct(self) -> inspect.Signature: + return inspect.Signature( + parameters=[ + inspect.Parameter( + name=p.name, + kind=p.kind, + annotation=p.annotation, + ) + for p in self.parameters + ], + return_annotation=self.return_annotation, + ) + + def __str__(self) -> str: + params = ", ".join(map(str, self.parameters)) + if self.return_annotation is None: + return f"({params})" + return f"({params}) -> {self.return_annotation}" + + +class FunctionSnapshot(ObjectTracer, Reference, ProxiedModel): + kind: Literal["function"] = "function" + type: str + bytecode_hash: Optional[str] = None + + module: Optional[str] = None + name: str + signature: Optional[FunctionSignature] = None + filename: Optional[str] = None + firstlineno: Optional[int] = None + source: Optional[str] = None + docstring: Optional[str] = None + + default_args: ReferenceMap = ReferenceMap.empty_dict() + closure_vars: ReferenceMap = ReferenceMap.empty_dict() + global_vars: ReferenceMap = ReferenceMap.empty_dict() + + @classmethod + def typecheck(cls, x) -> bool: + return inspect.isroutine(x) + + @classmethod + def trace(cls, func: Callable) -> "SnapshotType": + module, name = qualname_tuple(func) + + try: + signature = cls._signature(inspect.signature(func, follow_wrapped=False)) + except Exception: + signature = None + + try: + filename = source_path(inspect.getsourcefile(func)) + except Exception: + filename = None + + try: + sourcelines, firstlineno = inspect.getsourcelines(func) + sourcelines = dedent("".join(sourcelines)) + except Exception: + sourcelines = firstlineno = None + + try: + code_hash = bytecode_hash(func) + except TypeError: + code_hash = None + + return FunctionSnapshot( + ref=fingerprint(func), + type=qualname(type(func)), + bytecode_hash=code_hash, + name=name or "", + module=module, + filename=filename, + firstlineno=firstlineno, + source=sourcelines, + docstring=inspect.getdoc(func), + signature=signature, + ) + + @classmethod + def tree(cls, func: Callable) -> Dict[str, Union[Dict, List]]: + if not inspect.isfunction(func) or not inspect.ismethod(func): + return {} + return { + "default_args": { + k: p.default + for k, p in inspect.signature(func).parameters.items() + if p.default is not inspect.Parameter.empty + }, + "closure_vars": {**inspect.getclosurevars(func).nonlocals}, + "global_vars": find_globals(func, getattr(func, "__globals__", {})), + } + + @classmethod + def _signature(cls, sig: inspect.Signature): + params = [ + FunctionParameter( + name=p.name, + kind=p.kind, + annotation=type_annotation(p.annotation), + ) + for p in sig.parameters.values() + ] + return_type = type_annotation(sig.return_annotation) + return FunctionSignature(parameters=params, return_annotation=return_type) + + def __str__(self) -> str: + if not self.signature: + return f"{self.type} {self.name}, in {self.module}, file {self.filename}" + return ( + f"{self.type} {self.name}{self.signature}," + f" in {self.module}, file {self.filename}" + ) + + +class FrameInfoSnapshot(ObjectTracer, Reference, ProxiedModel): + kind: Literal["frame_info"] = "frame_info" + type: str + filename: str + lineno: int + func: str + code: Optional[str] + + @classmethod + def typecheck(cls, x) -> bool: + return isinstance(x, inspect.FrameInfo) + + @classmethod + def trace(cls, f: inspect.FrameInfo) -> "SnapshotType": + if f.code_context is None: + code = None + else: + code = "".join(f.code_context).strip() + return FrameInfoSnapshot( + ref=fingerprint(f), + type=qualname(type(f)), + filename=source_path(f.filename), + lineno=f.lineno, + func=f.function, + code=code, + ) + + +class FrameSnapshot(ObjectTracer, Reference, ProxiedModel): + kind: Literal["frame"] = "frame" + type: str + + local_vars: ReferenceMap = ReferenceMap.empty_dict() + global_vars: ReferenceMap = ReferenceMap.empty_dict() + outer_frames: ReferenceMap = ReferenceMap.empty_list() + + module: Optional[str] = None + func: str + + @classmethod + def typecheck(cls, x) -> bool: + return isinstance(x, FrameType) + + @classmethod + def trace(cls, f: FrameType) -> "SnapshotType": + info = stack_data.core.FrameInfo(f) + module = inspect.getmodule(f.f_code) + return FrameSnapshot( + ref=fingerprint(f), + type=qualname(type(f)), + module=qualname(module), + func=info.executing.code_qualname(), + ) + + @classmethod + def tree(cls, f: FrameType) -> Dict[str, Union[Dict, List]]: + return { + "local_vars": f.f_locals, + "global_vars": find_globals(f.f_code, f.f_globals), + "outer_frames": inspect.getouterframes(f), + } + + +SnapshotType = Annotated[ + Union[ + NoneSnapshot, + ObjectSnapshot, + ListSnapshot, + DictSnapshot, + RemoteObjectSnapshot, + RemoteLocationSnapshot, + FunctionSnapshot, + FrameInfoSnapshot, + FrameSnapshot, + ], + Field(discriminator="kind"), +] + + +class FunctionInfo(BaseModel): + code_hash: str + module: str + name: str + + _origin: Optional[Callable] = PrivateAttr(default=None) + + @property + def function_name(self) -> str: + return f"{self.module}.{self.name}" + + @classmethod + def from_static(cls, f: Callable, *load_const: int): + f = inspect.unwrap(f) + + module, name = qualname_tuple(f) + module = module or "" + name = name or "" + + try: + code = f.__code__ + for const in load_const: + code = code.co_consts[const] + assert isinstance(code, CodeType) + name += f"..{code.co_name}" + except IndexError as e: + raise TypeError( + f"unsupported callable {f}:" + f" index {load_const} out of range in co_consts" + ) from e + except (AttributeError, TypeError, AssertionError) as e: + raise TypeError( + f"unsupported callable {f}: cannot access code object" + ) from e + + info = FunctionInfo(code_hash=hash_digest(code), module=module, name=name) + + if not load_const: + info._origin = f + + return info + + +class Semantics(BaseModel): + api_level: Optional[int] = None + description: Optional[str] = None + + +class FunctionCheckpoint(BaseModel): + function: FunctionInfo + semantics: Semantics = Semantics() + + @property + def function_name(self) -> str: + return self.function.function_name + + +@dataclass +class WellKnownValues: + identity_function: str = bytecode_hash(lambda x: x) + + +class TracedFrame(BaseModel): + checkpoint: FunctionCheckpoint + + function: Reference + frame: Reference + retval: Reference + assignments: Reference + + variables: Dict[str, SnapshotType] + well_known: WellKnownValues = WellKnownValues() + + @property + def function_name(self) -> str: + try: + func = self.get_function() + module = func.module + name = func.name + except TypeError: + info = self.get_frame() + module = info.module + name = info.func + if not module: + return name + return f"{module}.{name}" + + def get_function(self): + return self.function.bind(FunctionSnapshot, self.variables) + + def get_frame(self): + return self.frame.bind(FrameSnapshot, self.variables) + + def iter_retvals(self) -> Iterable[Tuple[str, SnapshotType]]: + try: + assignments = self.assignments.bind(DictSnapshot, self.variables) + for key, value in assignments.values.of_type(SnapshotType): + yield key, value + except TypeError: + retval = self.retval.bind(SnapshotType, self.variables) + for key, value in like_pytree(retval, SnapshotType): + yield key, value + + +class OTelSpanContextDict(BaseModel): + span_id: str + trace_id: str + trace_state: str + + +class OTelSpanStatusDict(BaseModel): + status_code: str + description: Optional[str] = None + + +class OTelSpanEventDict(BaseModel): + name: str + attributes: Attributes + timestamp: datetime + + +class OTelSpanLinkDict(BaseModel): + attributes: Attributes + span_context: OTelSpanContextDict + + +class OTelSpanResourceDict(BaseModel): + attributes: Attributes + schema_url: str + + +class OTelSpanDict(BaseModel): + name: str + context: OTelSpanContextDict + kind: str + parent_id: Optional[str] = None + start_time: datetime + end_time: datetime + status: Optional[OTelSpanStatusDict] = None + attributes: Attributes + events: List[OTelSpanEventDict] + links: List[OTelSpanLinkDict] + resource: OTelSpanResourceDict + + +update_forward_refs(globals()) diff --git a/pyprojects/secretnote/src/secretnote/instrumentation/package.py b/pyprojects/secretnote/src/secretnote/instrumentation/package.py new file mode 100644 index 00000000..1981d3c9 --- /dev/null +++ b/pyprojects/secretnote/src/secretnote/instrumentation/package.py @@ -0,0 +1,3 @@ +_instruments = ("secretflow ~= 1.0",) + +_supports_metrics = False diff --git a/pyprojects/secretnote/src/secretnote/instrumentation/profiler.py b/pyprojects/secretnote/src/secretnote/instrumentation/profiler.py new file mode 100644 index 00000000..8c6c8a83 --- /dev/null +++ b/pyprojects/secretnote/src/secretnote/instrumentation/profiler.py @@ -0,0 +1,309 @@ +import ast +import contextvars +import sys +from types import FrameType +from typing import ( + Any, + Callable, + Dict, + List, + Mapping, + Optional, + Sequence, + Tuple, + Type, + cast, +) + +import stack_data.core +from astunparse import unparse +from more_itertools import first_true +from opentelemetry import trace +from opentelemetry.context import Context +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor + +from secretnote.utils.logging import log_dev_exception +from secretnote.utils.pydantic import Reference, ReferenceMap + +from .checkpoint import CheckpointGroup +from .envvars import OTEL_PYTHON_SECRETNOTE_PROFILER_FRAME +from .exporters import InMemorySpanExporter +from .models import ( + FunctionCheckpoint, + ObjectSnapshot, + ObjectTracer, + SnapshotType, + TracedFrame, +) +from .snapshot import fingerprint, json_key + +FinalizeSpan = Callable[[Optional[FrameType]], None] + + +class Profiler: + def __init__( + self, + checkpoints: CheckpointGroup, + recorders: List[Type[ObjectTracer]], + context: Optional[Context] = None, + ): + self._checkpoints = checkpoints + self._recorders = recorders + + self._tracer = trace.get_tracer(__name__) + self._exporter: InMemorySpanExporter + + self._parent_context = context + self._session_tokens: List[contextvars.Token] = [] + + # most recent on the right + self._recent_stacks: List[Tuple[Context, TracedFrame]] = [] + # most recent on the left + self._recent_returns: List[FinalizeSpan] = [] + + @property + def exporter(self): + try: + exporter = self._exporter + except AttributeError: + exporter = self._exporter = InMemorySpanExporter() + provider = cast(TracerProvider, trace.get_tracer_provider()) + processor = SimpleSpanProcessor(exporter) + provider.add_span_processor(processor) + return exporter + + def __call__(self, frame: FrameType, event: str, arg: Any): + if not (checkpoint := self._checkpoints.match_frame(frame)): + return + + if event == "call": + self._stack_push(frame, checkpoint) + return + + if event == "return": + self._stack_pop(frame, arg) + return + + def _trace_objects(self, *objects: Any): + refs: List[Reference] = [] + values: Dict[str, SnapshotType] = {} + for obj in objects: + ref, snapshots = trace_object(obj, self._recorders) + refs.append(ref) + values.update(snapshots) + return refs, values + + def _stack_push(self, frame: FrameType, checkpoint: FunctionCheckpoint): + if self._recent_stacks: + ctx = self._recent_stacks[-1][0] + else: + ctx = self._parent_context + + refs, values = self._trace_objects(checkpoint.function._origin, frame) + func_ref, frame_ref = refs + + result = TracedFrame( + checkpoint=checkpoint, + function=func_ref, + frame=frame_ref, + retval=Reference(ref=fingerprint(None)), + assignments=Reference(ref=fingerprint(None)), + variables=values, + ) + + span = self._tracer.start_span(result.function_name, ctx) + ctx = trace.set_span_in_context(span, ctx) + + self._recent_stacks.append((ctx, result)) + + def _stack_pop(self, frame: FrameType, retval: Any): + if not self._recent_stacks: + return + + ctx, call = self._recent_stacks.pop() + + (retval_ref,), values = self._trace_objects(retval) + call.retval = retval_ref + call.variables.update(values) + + def end_current_span(f_back: Optional[FrameType]): + try: + if f_back and (named_values := trace_named_return(f_back, retval)): + named_values = {k.strip(): v for k, v in named_values.items()} + (refs,), values = self._trace_objects(named_values) + call.assignments = refs + call.variables.update(values) + except Exception as e: + log_dev_exception(e) + span = trace.get_current_span(ctx) + payload = call.json(by_alias=True, exclude_none=True) + span.set_attribute(OTEL_PYTHON_SECRETNOTE_PROFILER_FRAME, payload) + span.end() + + self._recent_returns.append(end_current_span) + + def end_remaining_spans(f_back: Optional[FrameType]): + for fn in self._recent_returns: + fn(f_back) + self._recent_returns.clear() + frame.f_trace_lines = False + frame.f_trace = self._trace_noop + + def trace_line_in_outer_frame(frame: FrameType): + if frame.f_back: + frame.f_back.f_trace_lines = True + frame.f_back.f_trace = trace_next_assignments + else: + # no more outer frame, finalize spans + end_remaining_spans(None) + + def trace_next_assignments(f_back: FrameType, event: str, arg: None): + if event == "return": + # bubble up to outer frame + trace_line_in_outer_frame(f_back) + return None + + if event != "line": + # wait for next line trace + return trace_next_assignments + + end_remaining_spans(f_back) + + trace_line_in_outer_frame(frame) + + def _trace_noop(self, frame: FrameType, event: str, arg: Any): + frame.f_trace_lines = False + return self._trace_noop + + def start(self): + self.exporter.clear() + self._session_tokens.append(current_profiler.set(self)) + sys.settrace(self._trace_noop) + sys.setprofile(self) + + def stop(self): + sys.setprofile(None) + sys.settrace(None) + if self._session_tokens: + current_profiler.reset(self._session_tokens.pop()) + + def __enter__(self): + self.start() + return self + + def __exit__(self, *args): + self.stop() + return False + + def visualize(self): + from secretnote.display.app import visualize_run + + return visualize_run(self) + + +def trace_object(obj: Any, tracers: List[Type[ObjectTracer]]): + snapshots: Dict[str, SnapshotType] = { + Reference(ref=fingerprint(None)).ref: ObjectSnapshot.none() + } + + def snapshot_tree(root: Any) -> Optional[Reference]: + for rule in tracers: + if not rule.typecheck(root): + continue + + ref = Reference(ref=fingerprint(root)) + + if ref.ref in snapshots: + return ref + + try: + snapshot = rule.trace(root) + except NotImplementedError: + return None + + snapshots[ref.ref] = snapshot + + for key, items in rule.tree(root).items(): + if isinstance(items, Mapping): + refs = {k: snapshot_tree(v) for k, v in items.items()} + refs = {json_key(k): v for k, v in refs.items() if v is not None} + elif isinstance(items, Sequence): + refs = [snapshot_tree(x) for x in items] + refs = [x for x in refs if x is not None] + else: + raise TypeError(f"Cannot snapshot {type(items)}") + collection = ReferenceMap.from_container(refs) + setattr(snapshot, key, collection) + + return ref + return None + + result = snapshot_tree(obj) + + if result is None: + raise TypeError(f"Cannot snapshot {type(obj)}") + + return result, snapshots + + +def trace_named_return(frame: FrameType, retval: Any): + retval_type = type(retval) + + info = stack_data.core.FrameInfo(frame) + + if retval_type is tuple or retval_type is list: + # could be unpacking assignment, in which case there will be a different + # tuple in the parent frame, then we will have to compare by elements + # + # we limit the supporting types to vanilla tuple/list + # because we will need to iterate over it to determine item identity + # and we want to avoid side effects in custom iterables + + def resolve_variables() -> Optional[Dict]: + retval_len = len(retval) + + for expr, ast_nodes, value in cast( + List[stack_data.core.Variable], + info.variables, + ): + if value is retval: + # if it wasn't actually unpacked, return the entire iterable + return {expr: value} + + if ( + isinstance(value, tuple) + and len(value) == retval_len + and all(a is b for a, b in zip(value, retval)) + ): + tuple_ast = cast( + Optional[ast.Tuple], + first_true( + ast_nodes, + pred=lambda x: isinstance(x, ast.Tuple), + ), + ) + + if tuple_ast and len(tuple_ast.elts) == retval_len: + # map expressions to values + names = [unparse(x) for x in tuple_ast.elts] + return {name: value for name, value in zip(names, retval)} + + # can't find any corresponding value + # this can happen with unpacking assignment with stars + # (stack_data will refuse to parse the expression) + return None + + else: + # resolve by identity + + def resolve_variables() -> Optional[Dict]: + for expr, _, value in cast(List[stack_data.core.Variable], info.variables): + if value is retval: + return {expr: value} + return None + + return resolve_variables() + + +current_profiler = contextvars.ContextVar[Profiler]("current_profiler") diff --git a/pyprojects/secretnote/src/secretnote/instrumentation/sdk.py b/pyprojects/secretnote/src/secretnote/instrumentation/sdk.py new file mode 100644 index 00000000..4a6450cf --- /dev/null +++ b/pyprojects/secretnote/src/secretnote/instrumentation/sdk.py @@ -0,0 +1,258 @@ +import inspect +import json +import os +import sys +from datetime import datetime +from pathlib import Path +from types import ModuleType +from typing import ( + Callable, + List, + Optional, + Type, + cast, +) + +from opentelemetry import trace +from opentelemetry.sdk.environment_variables import OTEL_SERVICE_NAME +from opentelemetry.sdk.resources import SERVICE_NAME, Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator + +from secretnote.utils.warnings import development_preview_warning + +from .checkpoint import CheckpointGroup +from .envvars import ( + OTEL_PYTHON_SECRETNOTE_PROFILER_FRAME, + OTEL_PYTHON_SECRETNOTE_W3C_TRACE, +) +from .exporters import InMemorySpanExporter, JSONLinesSpanExporter +from .models import ( + APILevel, + DictSnapshot, + FrameInfoSnapshot, + FrameSnapshot, + FunctionSnapshot, + ListSnapshot, + ObjectSnapshot, + ObjectTracer, + OTelSpanDict, + RemoteLocationSnapshot, + RemoteObjectSnapshot, + Semantics, + TracedFrame, +) +from .profiler import Profiler +from .snapshot import ( + qualname, +) + + +def setup_tracing(service_name: Optional[str] = None): + current_provider = trace.get_tracer_provider() + + if not isinstance(current_provider, trace.ProxyTracerProvider): + # already initialized + return + + if service_name: + os.environ[OTEL_SERVICE_NAME] = name = service_name + else: + name = os.environ.get(OTEL_SERVICE_NAME, "unknown service") + + resource = Resource(attributes={SERVICE_NAME: name}) + provider = TracerProvider(resource=resource) + trace.set_tracer_provider(provider) + + +def setup_tracing_in_ray_worker(): + from ray.runtime_context import get_runtime_context + + runtime_ctx = get_runtime_context() + setup_tracing(runtime_ctx.get_worker_id()) + + +def setup_debug_exporter(): + from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter + + provider = cast(TracerProvider, trace.get_tracer_provider()) + processor = SimpleSpanProcessor( + OTLPSpanExporter(endpoint="localhost:4317", insecure=True) + ) + provider.add_span_processor(processor) + + +def setup_jsonlines_exporter(prefix: str): + output = open( + Path.cwd() / f"{prefix}.{datetime.now().timestamp():.0f}.jsonl", + "a+b", + ) + provider = cast(TracerProvider, trace.get_tracer_provider()) + processor = SimpleSpanProcessor(JSONLinesSpanExporter(output)) + provider.add_span_processor(processor) + + +def setup_memory_exporter(): + provider = cast(TracerProvider, trace.get_tracer_provider()) + exporter = InMemorySpanExporter() + processor = SimpleSpanProcessor(exporter) + provider.add_span_processor(processor) + return exporter + + +def inherit_tracing_context(): + try: + propagated_trace = json.loads(os.environ[OTEL_PYTHON_SECRETNOTE_W3C_TRACE]) + except Exception: + propagated_trace = {} + + return TraceContextTextMapPropagator().extract(propagated_trace) + + +def dump_tracing_context(): + carrier = {} + TraceContextTextMapPropagator().inject(carrier) + return {OTEL_PYTHON_SECRETNOTE_W3C_TRACE: json.dumps(carrier)} + + +def remote_trace(fn: Callable) -> Callable: + from .profiler import current_profiler + + try: + checkpoints = current_profiler.get()._checkpoints + recorders = current_profiler.get()._recorders + except LookupError: + return fn + + context = dump_tracing_context() + + def remote_task(*args, **kwargs): + from secretnote.instrumentation.profiler import Profiler + + os.environ.update(context) + + with Profiler(checkpoints, recorders, inherit_tracing_context()): + return fn(*args, **kwargs) + + return remote_task + + +def get_traced_frame(span: OTelSpanDict) -> Optional[TracedFrame]: + if not span.attributes: + return None + try: + raw = cast(str, span.attributes[OTEL_PYTHON_SECRETNOTE_PROFILER_FRAME]) + return TracedFrame.parse_raw(raw) + except Exception: + return None + + +def default_checkpoints(): + import fed + import fed._private.fed_call_holder + import ray + import ray.actor + import ray.remote_function + import secretflow + import secretflow.distributed + import secretflow.stats + from secretflow.device.proxy import _actor_wrapper + + checkpoints = CheckpointGroup() + add_function = checkpoints.add_function + + for fn in ( + ray.remote_function.RemoteFunction._remote, + ray.actor.ActorClass._remote, + ray.actor.ActorMethod._remote, + ray.get, + ray.wait, + fed.get, + fed.send, + fed.recv, + fed._private.fed_call_holder.FedCallHolder.internal_remote, + secretflow.SPU.infeed_shares, + secretflow.SPU.outfeed_shares, + ): + add_function(fn, semantics=Semantics(api_level=APILevel.IMPLEMENTATION)) + + for fn in ( + secretflow.device.kernels.pyu.pyu_to_pyu, + secretflow.device.kernels.pyu.pyu_to_spu, + secretflow.device.kernels.pyu.pyu_to_heu, + secretflow.device.kernels.spu.spu_to_pyu, + secretflow.device.kernels.spu.spu_to_spu, + secretflow.device.kernels.spu.spu_to_heu, + secretflow.device.kernels.heu.heu_to_pyu, + secretflow.device.kernels.heu.heu_to_spu, + secretflow.device.kernels.heu.heu_to_heu, + secretflow.reveal, + ): + add_function(fn, semantics=Semantics(api_level=APILevel.INVARIANT)) + + for fn, *load_const in ( + (secretflow.PYU.__call__, 1), + (secretflow.SPU.__call__, 1), + (_actor_wrapper, 1), + ): + add_function(fn, *load_const, semantics=Semantics(api_level=APILevel.INVARIANT)) + + return checkpoints + + +class ModuleTracer(ObjectTracer): + @classmethod + def typecheck(cls, x) -> bool: + return isinstance(x, ModuleType) + + @classmethod + def trace(cls, x): + raise NotImplementedError + + +class BuiltinSymbolTracer(ObjectTracer): + @classmethod + def typecheck(cls, x) -> bool: + module = inspect.getmodule(x) + if module is None: + return False + if sys.version_info >= (3, 10): + stdlib_names = sys.stdlib_module_names + else: + stdlib_names = sys.builtin_module_names + return module.__name__ in stdlib_names and qualname(type(x)) in ( + "abc.ABCMeta", + "builtins.type", + "builtins.builtin_function_or_method", + "builtins.builtin_function", + "builtins.method_descriptor", + "builtins.wrapper_descriptor", + ) + + @classmethod + def trace(cls, x): + raise NotImplementedError + + +def default_snapshot_rules() -> List[Type[ObjectTracer]]: + return [ + ModuleTracer, + BuiltinSymbolTracer, + FunctionSnapshot, + FrameInfoSnapshot, + FrameSnapshot, + RemoteObjectSnapshot, + RemoteLocationSnapshot, + ListSnapshot, + DictSnapshot, + ObjectSnapshot, + ] + + +def create_profiler(): + development_preview_warning() + setup_tracing() + checkpoints = default_checkpoints() + snapshot_rules = default_snapshot_rules() + return Profiler(checkpoints, snapshot_rules) diff --git a/pyprojects/secretnote/src/secretnote/instrumentation/snapshot.py b/pyprojects/secretnote/src/secretnote/instrumentation/snapshot.py new file mode 100644 index 00000000..105b08e3 --- /dev/null +++ b/pyprojects/secretnote/src/secretnote/instrumentation/snapshot.py @@ -0,0 +1,252 @@ +import builtins +import dis +import inspect +from collections import defaultdict +from contextlib import suppress +from pprint import pformat +from textwrap import dedent +from types import CodeType, FrameType, FunctionType, MethodType, ModuleType +from typing import ( + Any, + Callable, + Dict, + Hashable, + Optional, + Tuple, + Union, + overload, +) +from weakref import WeakValueDictionary + +from opentelemetry import trace + +from secretnote.formal.symbols import LogicalLocation + + +class LifetimeIdentityTracker: + def __init__(self) -> None: + self.generations: Dict[int, int] = defaultdict(int) + self.refs: WeakValueDictionary[int, Any] = WeakValueDictionary() + + def fingerprint(self, obj: Any) -> str: + obj_id = id(obj) + if obj_id in self.refs: + # still alive + return f"python/id/{hex(obj_id)}+{self.generations[obj_id]}" + # first object with this ID, or ID reused + # after a previous object was garbage collected + self.refs[obj_id] = obj + # will throw if object cannot be weakref'd + self.generations[obj_id] += 1 + return f"python/id/{hex(obj_id)}+{self.generations[obj_id]}" + + def reset(self) -> None: + self.generations.clear() + self.refs.clear() + + +id_tracker = LifetimeIdentityTracker() + + +def logical_location(device: Any) -> LogicalLocation: + from secretflow.device.device import HEU, PYU, SPU, TEEU + + if isinstance(device, PYU): + type_ = "PYU" + parties = (device.party,) + params = {} + + elif isinstance(device, SPU): + from libspu.spu_pb2 import FieldType, ProtocolKind + + type_ = "SPU" + parties = tuple(device.actors) + params = { + "protocol": ProtocolKind.Name(device.conf.protocol), + "field": FieldType.Name(device.conf.field), + "fxp_fraction_bits": device.conf.fxp_fraction_bits, + } + + elif isinstance(device, HEU): + type_ = "HEU" + parties = (device.sk_keeper_name(), *device.evaluator_names()) + params = {} + + elif isinstance(device, TEEU): + type_ = "TEEU" + parties = (device.party,) + params = {} + + else: + raise TypeError(f"Unknown device type {type(device)}") + + return LogicalLocation(type=type_, parties=parties, parameters=params) + + +def find_globals(fn: Union[FunctionType, MethodType, CodeType], ns: Dict): + global_vars = {} + # https://stackoverflow.com/a/61964607/22226623 + for inst in dis.get_instructions(fn): + if inst.opname == "LOAD_GLOBAL": + name = inst.argval + try: + value = ns[name] + except KeyError: + continue + if getattr(builtins, name, None) is value: + continue + global_vars[name] = value + return global_vars + + +def fingerprint(obj: Any) -> str: + from fed import FedObject + from ray import ObjectRef + from secretflow.device.device import ( + Device, + HEUObject, + PYUObject, + SPUObject, + TEEUObject, + ) + + if obj is None: + return "python/none" + + if isinstance(obj, Device): + return f"secretflow/location/{logical_location(obj).as_key()}" + if isinstance(obj, ObjectRef): + return f"ray/objectref/{obj}" + if isinstance(obj, FedObject): + return f"rayfed/{fingerprint(obj.get_ray_object_ref())}" + if isinstance(obj, PYUObject): + return f"secretflow/object/python/{fingerprint(obj.data)}" + if isinstance(obj, SPUObject): + return f"secretflow/object/mpc/{fingerprint(obj.meta)}" + if isinstance(obj, HEUObject): + return f"secretflow/object/homomorphic/{fingerprint(obj.data)}" + if isinstance(obj, TEEUObject): + return f"secretflow/object/tee/{fingerprint(obj.data)}" + + try: + return id_tracker.fingerprint(obj) + except TypeError: + pass + + span_id = hex(trace.get_current_span().get_span_context().span_id) + if isinstance(obj, FrameType): + obj_id = f"frame/{hex(id(obj))}/line/{obj.f_lineno}" + elif isinstance(obj, inspect.FrameInfo): + obj_id = f"frame/{hex(id(obj.frame))}/line/{obj.frame.f_lineno}" + else: + obj_id = f"id/{hex(id(obj))}" + return f"otel/span/{span_id}/transient/{obj_id}" + + +@overload +def hash_digest(obj: CodeType) -> str: + ... + + +@overload +def hash_digest(obj: Hashable) -> str: + ... + + +@overload +def hash_digest(obj: Any) -> Optional[str]: + ... + + +def hash_digest(obj): + try: + return f"python/hash/{hex(hash(obj))}" + except Exception: + return None + + +def bytecode_hash(f: Any) -> str: + if not inspect.isfunction(f): + raise TypeError(f"Expected Python function, got {type(f)}") + return hash_digest(f.__code__.co_code) + + +def json_key(obj: Any, key_fn: Callable[[Any], str] = fingerprint): + if obj is None: + return None + if isinstance(obj, (str, int, float, bool)): + return obj + return key_fn(obj) + + +def qualname_tuple(obj: Any) -> Tuple[Optional[str], Optional[str]]: + module_name = getattr(inspect.getmodule(obj), "__name__", None) + obj_name = ( + getattr(obj, "__qualname__", None) + or getattr(obj, "__name__", None) + or getattr(obj, "co_name", None) + or getattr(obj, "name", None) + ) + return module_name, obj_name + + +def qualname(obj: Any) -> str: + if isinstance(obj, ModuleType): + return obj.__name__ + module_name, obj_name = qualname_tuple(obj) + return f"{module_name or ''}.{obj_name or ''}" + + +def type_annotation(obj: Any) -> str: + if obj is inspect.Parameter.empty: + return type_annotation(Any) + if getattr(obj, "__module__", None) == "typing": + return str(obj) + module_name, obj_name = qualname_tuple(obj) + if module_name and obj_name: + return f"{module_name}.{obj_name}" + if obj_name: + return obj_name + return str(obj) + + +def source_code(obj): + try: + return dedent(inspect.getsource(obj)) + except Exception: + return None + + +@overload +def source_path(filename: str) -> str: + ... + + +@overload +def source_path(filename: None) -> None: + ... + + +def source_path(filename: Optional[str]) -> Optional[str]: + from IPython.core.getipython import get_ipython + from IPython.utils.path import compress_user + + if filename is None: + return None + + ipy = get_ipython() + + if ipy is not None and (data := ipy.compile.format_code_name(filename)) is not None: + label, name = data + return f"{label} {name}" + + return compress_user(filename) + + +def to_string(obj: Any) -> str: + for getter in (pformat, str, repr, fingerprint): + with suppress(Exception): + text = getter(obj) + assert isinstance(text, str) + return text + return "" diff --git a/pyprojects/secretnote/src/secretnote/instrumentation/version.py b/pyprojects/secretnote/src/secretnote/instrumentation/version.py new file mode 100644 index 00000000..b44315b1 --- /dev/null +++ b/pyprojects/secretnote/src/secretnote/instrumentation/version.py @@ -0,0 +1 @@ +__version__ = "0.0.0.dev0" diff --git a/pyprojects/secretnote/src/secretnote/py.typed b/pyprojects/secretnote/src/secretnote/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/pyprojects/secretnote/src/secretnote/server/__init__.py b/pyprojects/secretnote/src/secretnote/server/__init__.py new file mode 100644 index 00000000..0abd92cd --- /dev/null +++ b/pyprojects/secretnote/src/secretnote/server/__init__.py @@ -0,0 +1,9 @@ +JUPYTER_SERVER_EXTENSION_MODULE = __package__ + + +def _jupyter_server_extension_points(): + # See https://jupyter-server.readthedocs.io/en/latest/developers/extensions.html#making-an-extension-discoverable + + from .app import SecretNoteApp + + return [{"module": JUPYTER_SERVER_EXTENSION_MODULE, "app": SecretNoteApp}] diff --git a/pyprojects/secretnote/src/secretnote/server/__main__.py b/pyprojects/secretnote/src/secretnote/server/__main__.py new file mode 100644 index 00000000..ee1281fd --- /dev/null +++ b/pyprojects/secretnote/src/secretnote/server/__main__.py @@ -0,0 +1,4 @@ +if __name__ == "__main__": + from .app import SecretNoteApp + + SecretNoteApp.launch_instance() diff --git a/pyprojects/secretnote/src/secretnote/server/app.py b/pyprojects/secretnote/src/secretnote/server/app.py new file mode 100644 index 00000000..ce4ebf03 --- /dev/null +++ b/pyprojects/secretnote/src/secretnote/server/app.py @@ -0,0 +1,53 @@ +from jupyter_server.extension.application import ExtensionApp, ExtensionAppJinjaMixin + +from secretnote.utils.node import create_require + +from . import JUPYTER_SERVER_EXTENSION_MODULE +from .handlers import SinglePageApplicationHandler +from .node.handler import nodes_handlers + +require = create_require(__package__, "@secretflow/secretnote/index.html") + + +class SecretNoteApp(ExtensionAppJinjaMixin, ExtensionApp): + # -------------- Required traits -------------- + name = JUPYTER_SERVER_EXTENSION_MODULE + + load_other_extensions = True + + extension_url = "/secretnote/" + static_url_prefix = "/secretnote/" + + @property + def static_paths(self): + return [require.package("@secretflow/secretnote/index.html").joinpath("dist")] + + @property + def template_paths(self): + return self.static_paths + + def initialize_handlers(self): + routes = [ + *nodes_handlers, + ( + r"/secretnote/preview(.*)", + SinglePageApplicationHandler, + {"path": self.static_paths}, + ), + ( + r"/secretnote(.*)", + SinglePageApplicationHandler, + {"path": self.static_paths}, + ), + ] + self.handlers.extend(routes) + + @classmethod + def get_extension_package(cls): + """Returns the name of the Python package containing this extension. + + This allows the extension to be loaded during launch_instance even when not + explicitly enabled in Jupyter config. + """ + + return JUPYTER_SERVER_EXTENSION_MODULE diff --git a/pyprojects/secretnote/libro_server/db.py b/pyprojects/secretnote/src/secretnote/server/db.py similarity index 100% rename from pyprojects/secretnote/libro_server/db.py rename to pyprojects/secretnote/src/secretnote/server/db.py diff --git a/pyprojects/secretnote/src/secretnote/server/handlers.py b/pyprojects/secretnote/src/secretnote/server/handlers.py new file mode 100644 index 00000000..c36c4ef1 --- /dev/null +++ b/pyprojects/secretnote/src/secretnote/server/handlers.py @@ -0,0 +1,28 @@ +import tornado +from jupyter_server.base.handlers import FileFindHandler +from jupyter_server.extension.handler import ( + ExtensionHandlerJinjaMixin, + ExtensionHandlerMixin, +) +from tornado import web + + +class SinglePageApplicationHandler( + ExtensionHandlerJinjaMixin, + ExtensionHandlerMixin, + FileFindHandler, +): + @tornado.web.authenticated + async def get(self, path: str = "/"): + """Serve static content, emulating a typical single-page application. + + - Try to match path against a static file. If it exists, serve it + - Otherwise, serve the index.html file, which will load the application + and handle the routing. + """ + path = path.lstrip("/") + try: + await super().get(path) + except web.HTTPError: + self.clear() + self.write(self.render_template("index.html")) diff --git a/pyprojects/secretnote/libro_server/node/handler.py b/pyprojects/secretnote/src/secretnote/server/node/handler.py similarity index 92% rename from pyprojects/secretnote/libro_server/node/handler.py rename to pyprojects/secretnote/src/secretnote/server/node/handler.py index 563f033e..becbc485 100644 --- a/pyprojects/secretnote/libro_server/node/handler.py +++ b/pyprojects/secretnote/src/secretnote/server/node/handler.py @@ -1,9 +1,11 @@ import json +from typing import List, Tuple, Type -from jupyter_server.base.handlers import APIHandler -from libro_server.node.nodemanager import node_table +from jupyter_server.base.handlers import APIHandler, JupyterHandler from tornado import web +from secretnote.server.node.nodemanager import node_table + try: from jupyter_client.jsonutil import json_default except ImportError: @@ -15,7 +17,7 @@ class NodeRootHandler(APIHandler): def get(self): data = node_table.select_all("*") result = [] - print(data) + # print(data) for i in range(len(data)): result.append( { @@ -113,7 +115,7 @@ async def delete(self, node_id): _node_id_regex = r"(?P\w+-\w+-\w+-\w+-\w+)" -nodes_handlers = [ - (r"/api/nodes/%s" % _node_id_regex, NodeHandler), +nodes_handlers: List[Tuple[str, Type[JupyterHandler]]] = [ + (rf"/api/nodes/{_node_id_regex}", NodeHandler), (r"/api/nodes", NodeRootHandler), ] diff --git a/pyprojects/secretnote/libro_server/node/nodemanager.py b/pyprojects/secretnote/src/secretnote/server/node/nodemanager.py similarity index 96% rename from pyprojects/secretnote/libro_server/node/nodemanager.py rename to pyprojects/secretnote/src/secretnote/server/node/nodemanager.py index f3b1af54..3ae6cd2a 100644 --- a/pyprojects/secretnote/libro_server/node/nodemanager.py +++ b/pyprojects/secretnote/src/secretnote/server/node/nodemanager.py @@ -1,7 +1,8 @@ import os from jupyter_core import paths -from libro_server.db import Table + +from secretnote.server.db import Table class Node(Table): diff --git a/pyprojects/secretnote/src/secretnote/typing/__init__.py b/pyprojects/secretnote/src/secretnote/typing/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pyprojects/secretnote/src/secretnote/typing/tree_util.py b/pyprojects/secretnote/src/secretnote/typing/tree_util.py new file mode 100644 index 00000000..a7f8d3d3 --- /dev/null +++ b/pyprojects/secretnote/src/secretnote/typing/tree_util.py @@ -0,0 +1,22 @@ +from typing import Any, Iterable, Type, TypeVar + +import jax +from pydantic import BaseModel + +T = TypeVar("T", bound=BaseModel) + + +def supports_pytree(model_t: Type[T]): + keys = tuple(model_t.__fields__.keys()) + + def flatten(container: T): + return [ + (jax.tree_util.GetAttrKey(name), getattr(container, name)) for name in keys + ], None + + def unflatten(aux_data: Any, content: Iterable): + return model_t(**{name: value for name, value in zip(keys, content)}) + + jax.tree_util.register_pytree_with_keys(model_t, flatten, unflatten) + + return model_t diff --git a/pyprojects/secretnote/src/secretnote/utils/itertools.py b/pyprojects/secretnote/src/secretnote/utils/itertools.py new file mode 100644 index 00000000..d01ec0d5 --- /dev/null +++ b/pyprojects/secretnote/src/secretnote/utils/itertools.py @@ -0,0 +1,6 @@ +class NullIterable: + def __iter__(self): + return self + + def __next__(self): + raise StopIteration diff --git a/pyprojects/secretnote/src/secretnote/utils/logging.py b/pyprojects/secretnote/src/secretnote/utils/logging.py new file mode 100644 index 00000000..309ea9ed --- /dev/null +++ b/pyprojects/secretnote/src/secretnote/utils/logging.py @@ -0,0 +1,94 @@ +import asyncio +import logging +import os +import sys +from pathlib import Path +from typing import Optional, Union + +import loguru + +from .node import NODE_ENV + +IGNORED_ERRORS = (asyncio.TimeoutError,) + + +class InterceptHandler(logging.Handler): + def emit(self, record): + # Get corresponding Loguru level if it exists + try: + level = loguru.logger.level(record.levelname).name + except ValueError: + level = record.levelno + + # Find caller from where originated the logged message + frame, depth = logging.currentframe().f_back, 2 + while frame and frame.f_code.co_filename == logging.__file__: + frame = frame.f_back + depth += 1 + + loguru.logger.opt( + depth=depth, + exception=record.exc_info, + ).log(level, record.getMessage()) + + +def no_spinner(): + return os.environ.get("CI") or not sys.stderr.isatty() + + +def formatter_tty(record: "loguru.Record") -> str: + if record["extra"].get("raw_output"): + return "{message}\n" + prefix = "{level: <8}" + message = "{message}" + if record["level"].no >= logging.WARNING: + message = "{message}" + if record["exception"] and record["exception"].type not in IGNORED_ERRORS: + return f"{prefix} {message}\n{{exception}}\n" + return f"{prefix} {message}\n" + + +def formatter_ci(record: "loguru.Record") -> str: + if record["extra"].get("raw_output"): + return "{message}\n" + fmt = ( + "{time:YYYY-MM-DD HH:mm:ss.SSS ZZ} {level: <8}" + " {message} [{name}:{function}:{line}]\n" + ) + if record["exception"] and record["exception"].type not in IGNORED_ERRORS: + return f"{fmt}{{exception}}" + return fmt + + +def configure_logging( + *, + log_file: Optional[Union[str, Path]] = None, + level: Union[int, str] = logging.INFO, +): + if log_file or no_spinner(): + formatter = formatter_ci + else: + formatter = formatter_tty + loguru.logger.configure( + handlers=[ + { + "sink": log_file or sys.stderr, + "level": level, + "format": formatter, + }, + ], + levels=[ + {"name": "DEBUG", "color": ""}, + {"name": "INFO", "color": ""}, + {"name": "SUCCESS", "color": ""}, + {"name": "WARNING", "color": ""}, + {"name": "ERROR", "color": ""}, + {"name": "CRITICAL", "color": ""}, + ], + ) + logging.basicConfig(handlers=[InterceptHandler()], level=0, force=True) + + +def log_dev_exception(*args, **kwargs): + if NODE_ENV() == "development": + loguru.logger.exception(*args, **kwargs) diff --git a/pyprojects/secretnote/src/secretnote/utils/node.py b/pyprojects/secretnote/src/secretnote/utils/node.py new file mode 100644 index 00000000..e20de846 --- /dev/null +++ b/pyprojects/secretnote/src/secretnote/utils/node.py @@ -0,0 +1,197 @@ +import json +import os +import shutil +import subprocess +from pathlib import Path +from typing import Dict, Iterable, List, Optional, Tuple + +import importlib_resources +from loguru import logger +from pydantic import BaseModel + + +class PackageJSON(BaseModel): + name: str + version: str + files: List[str] = [] + dependencies: Dict[str, str] = {} + devDependencies: Dict[str, str] = {} + peerDependencies: Dict[str, str] = {} + + +class create_require: + def __init__(self, source_package: str, *specifiers: str): + self._source_package = source_package + self._specifiers = specifiers + self._resolved_files: Dict[str, Path] + + @property + def resolved_files(self): + try: + return self._resolved_files + except AttributeError: + self._resolve_all() + return self._resolved_files + + def resolve(self, specifier: str) -> Path: + try: + return self.resolved_files[specifier] + except KeyError: + raise KeyError(f"{specifier} is not available") from None + + def package(self, specifier: str) -> Path: + return find_package_json(self.resolve(specifier))[0] + + @property + def source_package(self): + return importlib_resources.files(self._source_package) + + @property + def dist_dir(self): + return self.source_package.joinpath("dist") + + def _resolve_all(self): + files_node_require: Dict[str, Path] = {} + files_find_static: Dict[str, Path] = {} + + err_node_require: Optional[str] = None + err_find_static: Optional[str] = None + + specifiers = self._specifiers + source_package = self.source_package + + # Method 1: for distribution + # + # We expect data files in `node_modules` to be packaged with the distribution. + # Further more, we expect an `resources.json` at the root of `node_modules`, + # which is a mapping from module specifiers to the path of the corresponding + # data files, relative to `node_modules`. + # + # Here, locate `node_modules` within `package`. + # If `package` is not specified, use the current package. + # Then, read `resources.json` and populate `resolved_files`. + # + # Example, if `package` is `foo.bar` and is on path `/path/to/foo/bar`, + # the we are looking for `/path/to/foo/bar/node_modules/resources.json`. + try: + dist_dir = self.dist_dir + with importlib_resources.as_file(dist_dir) as path: + if not path.is_dir(): + raise FileNotFoundError(f"{dist_dir} does not exist") + with open(path.joinpath("resources.json")) as f: + content = json.load(f) + files_find_static = { + k: path.joinpath(v).resolve() for k, v in content.items() + } + if unresolved := set(specifiers) - set(files_find_static.keys()): + raise KeyError(f"Failed to resolve these specifiers: {[*unresolved]}") + except Exception as e: + err_find_static = str(e) + + # Method 2: during development + # We expect `node` to be available on PATH. We will use `require.resolve`. + try: + node_bin = shutil.which("node") + if node_bin is None: + raise FileNotFoundError("node not found on PATH") + script: str = "" + for specifier in specifiers: + script += f"console.log(require.resolve({json.dumps(specifier)}));" + with importlib_resources.as_file(source_package) as cwd: + result = subprocess.run( + [node_bin, "-e", script], + cwd=cwd, + capture_output=True, + text=True, + check=True, + ) + files_node_require = { + k: Path(v).resolve() + for k, v in zip(specifiers, result.stdout.strip().splitlines()) + } + except subprocess.CalledProcessError as e: + err_node_require = e.stderr.strip() + except OSError as e: + err_node_require = str(e) + + if err_node_require is not None and err_find_static is not None: + error = ( + "Failed to resolve all specifiers:" + f"\nLooking for static files resulted in: {err_find_static}" + f"\nModule resolution using Node resulted in: {err_node_require}" + ) + raise RuntimeError(error) + + # prefer Node resolution + self._resolved_files = files_node_require or files_find_static + + +def find_package_json(path: Path) -> Tuple[Path, PackageJSON]: + while True: + package_json = path.joinpath("package.json") + if package_json.exists(): + return package_json.parent, PackageJSON.parse_file(package_json) + if path == path.parent: + raise FileNotFoundError("package.json not found") + path = path.parent + + +def find_all_files(path: Path) -> Iterable[Path]: + for item in path.iterdir(): + if item.is_dir(): + yield from find_all_files(item) + elif item.is_file() or item.is_symlink(): + yield item + + +def copy_static_files(require: create_require): + with importlib_resources.as_file(require.dist_dir) as root: + shutil.rmtree(root, ignore_errors=True) + + resources: Dict[str, str] = {} + packages: Dict[Path, PackageJSON] = {} + + for name, source in require.resolved_files.items(): + package_dir, package_json = find_package_json(source) + packages[package_dir] = package_json + + relpath = source.relative_to(package_dir) + target = root.joinpath(package_json.name).joinpath(relpath) + resources[name] = str(target.relative_to(root)) + + os.makedirs(target.parent, exist_ok=True) + shutil.copy(source, target) + + logger.info(f"Copied {source} to {target}") + + for package_dir, package_json in packages.items(): + target_root = root.joinpath(package_json.name) + + for file in [*package_json.files, "package.json"]: + source = package_dir.joinpath(file) + target = target_root.joinpath(file) + if not source.exists(): + continue + elif source.is_dir(): + shutil.copytree(source, target, dirs_exist_ok=True) + logger.info(f"Copied {source} to {target}") + elif source.is_file() or source.is_symlink(): + os.makedirs(target.parent, exist_ok=True) + shutil.copy(source, target, follow_symlinks=True) + logger.info(f"Copied {source} to {target}") + else: + continue + + for resource in find_all_files(target_root): + name = resource.relative_to(root) + resources[str(name)] = str(name) + + with open(root.joinpath("resources.json"), "w") as f: + json.dump(resources, f) + + +def NODE_ENV(): + try: + return os.environ["NODE_ENV"] + except KeyError: + return "production" diff --git a/pyprojects/secretnote/src/secretnote/utils/pydantic.py b/pyprojects/secretnote/src/secretnote/utils/pydantic.py new file mode 100644 index 00000000..494ebcdf --- /dev/null +++ b/pyprojects/secretnote/src/secretnote/utils/pydantic.py @@ -0,0 +1,187 @@ +from typing import ( + Any, + Dict, + Iterable, + List, + Mapping, + Tuple, + Type, + TypeVar, + Union, + cast, + get_args, + get_origin, + overload, +) + +import jax +import orjson +from pydantic import BaseModel, PrivateAttr +from typing_extensions import TypeGuard + +T = TypeVar("T") + +TypedKey = Tuple[Type[T], Any] + + +def update_forward_refs(global_ns: Dict): + models: Dict[str, Type[BaseModel]] = {} + + def collect_models(**items: Any): + for k, v in items.items(): + try: + is_model = issubclass(v, BaseModel) + except TypeError: + continue + if is_model: + models[k] = v + try: + collect_models(**vars(v)) + except TypeError: + continue + + collect_models(**global_ns) + + for v in models.values(): + v.update_forward_refs() + + +def orjson_dumps(v, *, default): + return orjson.dumps(v, default=default, option=orjson.OPT_NON_STR_KEYS).decode() + + +def is_of_type(obj: Any, annotation: Type[T]) -> TypeGuard[T]: + def extract_types(annotation) -> Tuple: + origin = annotation + while True: + args = get_args(origin) + origin = get_origin(origin) + if origin is Union: + return tuple(t for subtype in args for t in extract_types(subtype)) + if origin is None: + return args or (annotation,) + + types = extract_types(annotation) + + if any(t is Any for t in types): + return True + + return isinstance(obj, types) + + +def to_container(ref: "ProxiedModel", *, container_t=(list, dict, tuple)): + def reconstruct(root: ProxiedModel): + try: + container = root.to_container() + except (AttributeError, NotImplementedError): + return root + if isinstance(container, dict) and dict in container_t: + return {k: reconstruct(v) for k, v in container.items()} + if isinstance(container, list) and list in container_t: + return [reconstruct(v) for v in container] + if isinstance(container, tuple) and tuple in container_t: + return tuple(reconstruct(v) for v in container) + return container + + return reconstruct(ref) + + +def like_pytree( + ref: "ProxiedModel", + of_type: Type[T], + *, + container_t=(list, dict, tuple), +) -> Iterable[Tuple[str, T]]: + container = to_container(ref, container_t=container_t) + flattened, tree = jax.tree_util.tree_flatten_with_path(container) + for path, value in flattened: + if is_of_type(value, of_type): + yield (jax.tree_util.keystr(path), value) + + +class ORJSONConfig: + json_loads = orjson.loads + json_dumps = orjson_dumps + + +class ProxiedModel(BaseModel): + _lookup: Mapping[str, "ProxiedModel"] = PrivateAttr(default_factory=dict) + + def to_container(self) -> Union[List[T], Dict[Any, T], Tuple[T, ...]]: + raise NotImplementedError + + def __getattribute__(self, __name): + item = super().__getattribute__(__name) + if isinstance(item, ProxiedModel): + item._lookup = self._lookup + return item + + +class Reference(BaseModel): + ref: str + + def bind(self, types: Type[T], lookup: Mapping[str, "ProxiedModel"]) -> T: + item = lookup[self.ref] + + if not is_of_type(item, types): + raise TypeError(f"Expected {types}, got {type(item)}: {item}") + + if isinstance(item, ProxiedModel): + item._lookup = lookup + + return cast(types, item) + + +class ReferenceMap(ProxiedModel, Mapping): + __root__: Union[List[Reference], Dict[Any, Reference], Tuple[Reference, ...]] + + @overload + def __getitem__(self, item: TypedKey[T]) -> T: + ... + + @overload + def __getitem__(self, item: Any): + ... + + def __getitem__(self, item): + if not isinstance(item, tuple): + return self[Any, item] + types, key = item + try: + ref = self.__root__[key] + value = self._lookup[ref.ref] + value._lookup = self._lookup + if is_of_type(value, types): + return value + raise TypeError(f"Expected {types}, got {type(value)}: {value}") + except (LookupError, TypeError) as e: + raise KeyError(item) from e + + def __iter__(self): + if isinstance(self.__root__, dict): + yield from self.__root__ + else: + yield from range(len(self.__root__)) + + def of_type(self, types: Type[T]) -> Iterable[Tuple[Any, T]]: + for key, value in self.items(): + if is_of_type(value, types): + yield key, cast(T, value) + + def __len__(self): + return len(self.__root__) + + @classmethod + def empty_list(cls): + return ReferenceMap(__root__=[]) + + @classmethod + def empty_dict(cls): + return ReferenceMap(__root__={}) + + @classmethod + def from_container(cls, collection: Union[List, Dict]): + return ReferenceMap(__root__=collection) + + +update_forward_refs(globals()) diff --git a/pyprojects/secretnote/src/secretnote/utils/version.py b/pyprojects/secretnote/src/secretnote/utils/version.py new file mode 100644 index 00000000..f26c396c --- /dev/null +++ b/pyprojects/secretnote/src/secretnote/utils/version.py @@ -0,0 +1,31 @@ +from typing import Any + +from packaging.specifiers import SpecifierSet +from packaging.version import Version + + +def assert_version( + module: Any, + expected: str, + *, + version_attr: str = "__version__", +) -> None: + """Assert that a module's version satisfies a requirement. + + This is useful for ensuring for determining package compatibility at runtime, which + is necessary because it is difficult to guarantee package versions in Python + environments. + """ + current_version = Version(getattr(module, version_attr)) + acceptable_versions = SpecifierSet(expected) + + if current_version.is_prerelease: + satisfied = acceptable_versions.contains(current_version, prereleases=True) + else: + satisfied = current_version in acceptable_versions + + name = getattr(module, "__name__", str(module)) + + assert ( + satisfied + ), f"This program requires {name} {expected}, but you have {current_version}" diff --git a/pyprojects/secretnote/src/secretnote/utils/warnings.py b/pyprojects/secretnote/src/secretnote/utils/warnings.py new file mode 100644 index 00000000..96dca9f0 --- /dev/null +++ b/pyprojects/secretnote/src/secretnote/utils/warnings.py @@ -0,0 +1,11 @@ +import warnings + + +def development_preview_warning(): + warnings.warn( + "\n🟡 THIS IS A DEVELOPER PREVIEW 🧪🧪🧪" + "\nAPI may change without prior notice. No guarantee is made about the" + " security, correctness, performance, or usefulness of this feature.", + FutureWarning, + stacklevel=2, + ) diff --git a/pyprojects/secretnote/tests/secretnote/test_dummy.py b/pyprojects/secretnote/tests/secretnote/test_dummy.py deleted file mode 100644 index 63ae41d1..00000000 --- a/pyprojects/secretnote/tests/secretnote/test_dummy.py +++ /dev/null @@ -1,2 +0,0 @@ -def test_dummy(): - assert True is True diff --git a/pyprojects/secretnote/tests/secretnote/utils/test_version.py b/pyprojects/secretnote/tests/secretnote/utils/test_version.py new file mode 100644 index 00000000..f31377b7 --- /dev/null +++ b/pyprojects/secretnote/tests/secretnote/utils/test_version.py @@ -0,0 +1,49 @@ +import pytest + + +class MockModule: + def __init__(self, name: str, version: str) -> None: + self.__name__ = name + self.__version__ = version + + +def test_assert_version_equal(): + from secretnote.utils.version import assert_version + + module = MockModule("foo", "0.0.1") + + assert_version(module, "==0.0.1") + + with pytest.raises(AssertionError): + assert_version(module, "==0.0.2") + + +def test_assert_version_range(): + from secretnote.utils.version import assert_version + + module = MockModule("foo", "0.0.1") + + with pytest.raises(AssertionError): + assert_version(module, ">0.0.1") + + +def test_assert_version_compatible(): + from secretnote.utils.version import assert_version + + module = MockModule("foo", "1.1.3") + + with pytest.raises(AssertionError): + assert_version(module, "~=1.0.0") + + assert_version(module, "~=1.0") + + +def test_assert_version_prerelease(): + from secretnote.utils.version import assert_version + + module = MockModule("foo", "1.1.3rc1") + + with pytest.raises(AssertionError): + assert_version(module, ">=1.1.3") + + assert_version(module, ">=1.1.3rc1") diff --git a/pyprojects/spu-stubs/CHANGELOG.md b/pyprojects/spu-stubs/CHANGELOG.md index 2b4c3fef..e119bf61 100644 --- a/pyprojects/spu-stubs/CHANGELOG.md +++ b/pyprojects/spu-stubs/CHANGELOG.md @@ -1,4 +1,4 @@ -# @secretflow/notebook-pyproject-spu-stubs +# spu-stubs ## 0.0.1-dev.0 diff --git a/pyprojects/spu-stubs/package.json b/pyprojects/spu-stubs/package.json index ec6fd4fa..c1cecd73 100644 --- a/pyprojects/spu-stubs/package.json +++ b/pyprojects/spu-stubs/package.json @@ -2,13 +2,12 @@ "name": "spu-stubs", "private": true, "files": [], - "version": "0.0.1-dev.0", + "version": "0.0.0", "scripts": { - "build": "hatch clean && hatch build -t sdist -t wheel", - "test:pytest": "pytest", - "lint:black": "black --check src", - "lint:ruff": "ruff check src", - "lint:pyright": "pyright src", - "release": "hatch publish ./dist" + "format:black": "python -m black --check src tests", + "lint:ruff": "python -m ruff check src tests", + "publish": "python -m hatch publish ./dist", + "typecheck:pyright": "pyright --project ../.. src tests", + "build": "python -m hatch clean && python -m hatch build -t sdist -t wheel" } } diff --git a/pyprojects/spu-stubs/pyproject.toml b/pyprojects/spu-stubs/pyproject.toml index 52d5a429..5159f54e 100644 --- a/pyprojects/spu-stubs/pyproject.toml +++ b/pyprojects/spu-stubs/pyproject.toml @@ -7,9 +7,10 @@ classifiers = [ dependencies = [] description = "Type stubs for SPU" dynamic = ["version"] +license = "Apache-2.0" name = "spu-stubs" readme = "README.md" -requires-python = ">=3.8, <3.9" +requires-python = ">=3.8, <3.12" [project.scripts] diff --git a/requirements-dev.lock b/requirements-dev.lock index 6bfaa7f5..c0f18425 100644 --- a/requirements-dev.lock +++ b/requirements-dev.lock @@ -9,189 +9,246 @@ -e file:. -e file:pyprojects/secretnote -e file:pyprojects/spu-stubs -aiohttp==3.8.5 +aiohttp==3.9.0 aiohttp-cors==0.7.0 aiosignal==1.3.1 -anyio==4.0.0 +anyio==4.1.0 appnope==0.1.3 +argcomplete==3.1.6 argon2-cffi==23.1.0 argon2-cffi-bindings==21.2.0 -arrow==1.2.3 -asttokens==2.4.0 +arrow==1.3.0 +asttokens==2.4.1 +astunparse==1.6.3 async-lru==2.0.4 async-timeout==4.0.3 attrs==23.1.0 -babel==2.12.1 +babel==2.13.1 backcall==0.2.0 +backoff==2.2.1 beautifulsoup4==4.12.2 -black==23.7.0 -bleach==6.0.0 +black==23.11.0 +bleach==6.1.0 blessed==1.20.0 -cachetools==5.3.1 -certifi==2023.7.22 -cffi==1.15.1 -charset-normalizer==3.2.0 +cachetools==5.3.2 +certifi==2023.11.17 +cffi==1.16.0 +charset-normalizer==3.3.2 clean-text==0.6.0 click==8.1.7 -cloudpickle==2.2.1 +cloudpickle==3.0.0 colorful==0.5.5 -comm==0.1.4 -debugpy==1.6.7.post1 +comm==0.2.0 +contourpy==1.1.1 +cycler==0.12.1 +datamodel-code-generator==0.24.2 +debugpy==1.8.0 decorator==5.1.1 defusedxml==0.7.1 +deprecated==1.2.14 dill==0.3.7 distlib==0.3.7 +dnspython==2.4.2 editables==0.5 +email-validator==2.1.0.post1 emoji==1.7.0 -exceptiongroup==1.1.3 -executing==1.2.0 -fastjsonschema==2.18.0 -filelock==3.12.3 +exceptiongroup==1.2.0 +executing==2.0.1 +fastapi==0.99.1 +fastjsonschema==2.19.0 +filelock==3.13.1 +fonttools==4.45.0 fqdn==1.5.1 frozenlist==1.4.0 -ftfy==6.1.1 -google-api-core==2.11.1 -google-auth==2.23.0 -googleapis-common-protos==1.60.0 +fsspec==2023.10.0 +ftfy==6.1.3 +genson==1.2.2 +google-api-core==2.14.0 +google-auth==2.23.4 +googleapis-common-protos==1.61.0 gpustat==1.1.1 grpcio==1.56.2 h11==0.14.0 hatch==1.7.0 hatchling==1.18.0 -httpcore==0.17.3 -httpx==0.24.1 +httpcore==1.0.2 +httpx==0.25.1 +huggingface-hub==0.19.4 hyperlink==21.0.0 idna==3.4 importlib-metadata==6.8.0 -importlib-resources==6.0.1 +importlib-resources==6.1.1 +inflect==5.6.2 iniconfig==2.0.0 -ipykernel==6.25.2 -ipython==8.12.2 +ipykernel==6.27.0 +ipython==8.12.3 +ipython-genutils==0.2.0 +ipython-sql==0.5.0 +ipywidgets==8.1.1 isoduration==20.11.0 +isort==5.12.0 jaraco-classes==3.3.0 jax==0.4.12 jaxlib==0.4.12 -jedi==0.19.0 +jedi==0.19.1 jinja2==3.1.2 joblib==1.3.2 json5==0.9.14 jsonpointer==2.4 -jsonschema==4.19.0 -jsonschema-specifications==2023.7.1 -jupyter-client==8.3.1 -jupyter-core==5.3.1 -jupyter-events==0.7.0 +jsonschema==4.20.0 +jsonschema-specifications==2023.11.1 +jupyter-client==8.6.0 +jupyter-core==5.5.0 +jupyter-events==0.9.0 jupyter-lsp==2.2.0 -jupyter-server==2.7.3 +jupyter-resource-usage==1.0.1 +jupyter-server==2.10.1 jupyter-server-terminals==0.4.4 -jupyterlab==4.0.5 +jupyterlab==4.0.9 jupyterlab-pygments==0.2.2 -jupyterlab-server==2.24.0 -keyring==24.2.0 -kuscia==0.0.1b2 +jupyterlab-server==2.25.2 +jupyterlab-widgets==3.0.9 +keyring==24.3.0 +kiwisolver==1.4.5 +kuscia==0.0.2.dev231025 +lightning-utilities==0.10.0 llvmlite==0.40.1 logging-tree==1.9 loguru==0.7.2 markdown-it-py==3.0.0 markupsafe==2.1.3 +matplotlib==3.7.4 matplotlib-inline==0.1.6 mdurl==0.1.2 -mistune==3.0.1 +mistune==3.0.2 ml-dtypes==0.2.0 more-itertools==10.1.0 -msgpack==1.0.5 +mpmath==1.3.0 +msgpack==1.0.7 multidict==6.0.4 multiprocess==0.70.15 -mypy==1.5.1 +mypy==1.7.0 mypy-extensions==1.0.0 -nbclient==0.8.0 -nbconvert==7.8.0 +nbclient==0.9.0 +nbconvert==7.11.0 nbformat==5.9.2 -nest-asyncio==1.5.7 +nest-asyncio==1.5.8 +networkx==3.1 notebook-shim==0.2.3 numba==0.57.0 numpy==1.23.5 -nvidia-ml-py==12.535.108 -opencensus==0.11.2 +nvidia-ml-py==12.535.133 +opencensus==0.11.3 opencensus-context==0.1.3 +opentelemetry-api==1.21.0 +opentelemetry-exporter-otlp-proto-common==1.21.0 +opentelemetry-exporter-otlp-proto-grpc==1.21.0 +opentelemetry-proto==1.21.0 +opentelemetry-sdk==1.21.0 +opentelemetry-semantic-conventions==0.42b0 opt-einsum==3.3.0 +orjson==3.9.10 overrides==7.4.0 -packaging==23.1 +packaging==23.2 pandas==1.5.3 pandocfilters==1.5.0 parso==0.8.3 pathspec==0.11.2 +patsy==0.5.3 pexpect==4.8.0 pickleshare==0.7.5 +pillow==10.1.0 pkgutil-resolve-name==1.3.10 -platformdirs==3.10.0 +platformdirs==3.11.0 pluggy==1.3.0 -prometheus-client==0.13.1 -prompt-toolkit==3.0.39 +prettytable==3.9.0 +prometheus-client==0.19.0 +prompt-toolkit==3.0.41 protobuf==3.19.6 -psutil==5.9.5 +psutil==5.9.6 ptyprocess==0.7.0 pure-eval==0.2.2 py-spy==0.3.14 -pyarrow==11.0.0 -pyasn1==0.5.0 +pyarrow==13.0.0 +pyasn1==0.5.1 pyasn1-modules==0.3.0 pycparser==2.21 -pydantic==1.10.12 -pygments==2.16.1 +pydantic==1.10.13 +pygments==2.17.2 +pymysql==1.1.0 +pyparsing==3.1.1 pyperclip==1.8.2 -pytest==7.4.1 +pytest==7.4.3 python-dateutil==2.8.2 python-json-logger==2.0.7 pytz==2023.3.post1 pyyaml==6.0.1 pyzmq==25.1.1 -ray==2.2.0 -referencing==0.30.2 +ray==2.6.3 +referencing==0.31.0 +regex==2023.10.3 requests==2.31.0 rfc3339-validator==0.1.4 rfc3986-validator==0.1.1 -rich==13.5.3 -rpds-py==0.10.2 +rich==13.7.0 +rpds-py==0.13.1 rsa==4.9 -ruamel-yaml==0.17.32 -ruamel-yaml-clib==0.2.7 -ruff==0.0.287 +ruff==0.1.6 +safetensors==0.4.0 scikit-learn==1.1.3 scipy==1.10.1 -secretflow-lite==1.1.0b0 +secretflow-lite==1.3.0.dev20231122 secretflow-ray==2.2.0 secretflow-rayfed==0.2.0a7 send2trash==1.8.2 -sf-heu==0.4.4b0 -shellingham==1.5.3 +sf-heu==0.5.0.dev20231118 +shellingham==1.5.4 six==1.16.0 smart-open==6.4.0 +snakeviz==2.2.0 sniffio==1.3.0 soupsieve==2.5 -spu==0.5.0b0 -stack-data==0.6.2 +spu==0.6.0b0 +sqlalchemy==2.0.23 +sqlparse==0.4.4 +stack-data==0.6.3 +starlette==0.27.0 +statsmodels==0.14.0 +sympy==1.12 termcolor==2.3.0 -terminado==0.17.1 +terminado==0.18.0 threadpoolctl==3.2.0 tinycss2==1.2.1 +tokenize-rt==5.2.0 +tokenizers==0.15.0 +toml==0.10.2 tomli==2.0.1 tomli-w==1.0.0 -tomlkit==0.12.1 +tomlkit==0.12.3 +torch==2.1.1 +torchmetrics==1.2.0 +torchvision==0.16.1 tornado==6.3.3 -traitlets==5.9.0 -trove-classifiers==2023.8.7 -typing-extensions==4.7.1 +tqdm==4.66.1 +traitlets==5.13.0 +transformers==4.35.2 +trove-classifiers==2023.11.22 +types-python-dateutil==2.8.19.14 +typing-extensions==4.8.0 uri-template==1.3.0 -urllib3==1.26.16 +urllib3==2.1.0 userpath==1.9.1 -virtualenv==20.24.4 -wcwidth==0.2.6 +uvicorn==0.24.0.post1 +virtualenv==20.21.0 +wcwidth==0.2.12 webcolors==1.13 webencodings==0.5.1 -websocket-client==1.6.2 -yarl==1.9.2 -zipp==3.16.2 +websocket-client==1.6.4 +wheel==0.41.3 +widgetsnbextension==4.0.9 +wrapt==1.16.0 +yarl==1.9.3 +zipp==3.17.0 # The following packages are considered to be unsafe in a requirements file: -pip==23.2.1 -setuptools==68.2.2 +pip==23.3.1 +setuptools==69.0.2 diff --git a/requirements.lock b/requirements.lock index e9acb4b2..2039ea29 100644 --- a/requirements.lock +++ b/requirements.lock @@ -9,9 +9,144 @@ -e file:. -e file:pyprojects/secretnote -e file:pyprojects/spu-stubs +aiohttp==3.9.0 +aiohttp-cors==0.7.0 +aiosignal==1.3.1 +anyio==4.1.0 +appnope==0.1.3 +argon2-cffi==23.1.0 +argon2-cffi-bindings==21.2.0 +arrow==1.3.0 +asttokens==2.4.1 +astunparse==1.6.3 +async-timeout==4.0.3 +attrs==23.1.0 +backcall==0.2.0 +beautifulsoup4==4.12.2 +bleach==6.1.0 +blessed==1.20.0 +cachetools==5.3.2 +certifi==2023.11.17 +cffi==1.16.0 +charset-normalizer==3.3.2 +click==8.1.7 +colorful==0.5.5 +comm==0.2.0 +debugpy==1.8.0 +decorator==5.1.1 +defusedxml==0.7.1 +deprecated==1.2.14 +distlib==0.3.7 +exceptiongroup==1.2.0 +executing==2.0.1 +fastjsonschema==2.19.0 +filelock==3.13.1 +fqdn==1.5.1 +frozenlist==1.4.0 +google-api-core==2.14.0 +google-auth==2.23.4 +googleapis-common-protos==1.61.0 +gpustat==1.1.1 +grpcio==1.59.3 +idna==3.4 +importlib-metadata==6.8.0 +importlib-resources==6.1.1 +ipykernel==6.27.0 +ipython==8.12.3 +ipython-genutils==0.2.0 +ipython-sql==0.5.0 +ipywidgets==8.1.1 +isoduration==20.11.0 +jedi==0.19.1 +jinja2==3.1.2 +jsonpointer==2.4 +jsonschema==4.20.0 +jsonschema-specifications==2023.11.1 +jupyter-client==8.6.0 +jupyter-core==5.5.0 +jupyter-events==0.9.0 +jupyter-resource-usage==1.0.1 +jupyter-server==2.10.1 +jupyter-server-terminals==0.4.4 +jupyterlab-pygments==0.2.2 +jupyterlab-widgets==3.0.9 loguru==0.7.2 -packaging==23.1 -pydantic==1.10.12 -ruamel-yaml==0.17.32 -ruamel-yaml-clib==0.2.7 -typing-extensions==4.7.1 +markupsafe==2.1.3 +matplotlib-inline==0.1.6 +mistune==3.0.2 +more-itertools==10.1.0 +msgpack==1.0.7 +multidict==6.0.4 +nbclient==0.9.0 +nbconvert==7.11.0 +nbformat==5.9.2 +nest-asyncio==1.5.8 +networkx==3.1 +numpy==1.24.4 +nvidia-ml-py==12.535.133 +opencensus==0.11.3 +opencensus-context==0.1.3 +opentelemetry-api==1.21.0 +opentelemetry-sdk==1.21.0 +opentelemetry-semantic-conventions==0.42b0 +orjson==3.9.10 +overrides==7.4.0 +packaging==23.2 +pandocfilters==1.5.0 +parso==0.8.3 +pexpect==4.8.0 +pickleshare==0.7.5 +pkgutil-resolve-name==1.3.10 +platformdirs==3.11.0 +prettytable==3.9.0 +prometheus-client==0.19.0 +prompt-toolkit==3.0.41 +protobuf==4.25.1 +psutil==5.9.6 +ptyprocess==0.7.0 +pure-eval==0.2.2 +py-spy==0.3.14 +pyasn1==0.5.1 +pyasn1-modules==0.3.0 +pycparser==2.21 +pydantic==1.10.13 +pygments==2.17.2 +pymysql==1.1.0 +python-dateutil==2.8.2 +python-json-logger==2.0.7 +pyyaml==6.0.1 +pyzmq==25.1.1 +ray==2.6.3 +referencing==0.31.0 +requests==2.31.0 +rfc3339-validator==0.1.4 +rfc3986-validator==0.1.1 +rpds-py==0.13.1 +rsa==4.9 +send2trash==1.8.2 +six==1.16.0 +smart-open==6.4.0 +sniffio==1.3.0 +soupsieve==2.5 +sqlalchemy==2.0.23 +sqlparse==0.4.4 +stack-data==0.6.3 +terminado==0.18.0 +tinycss2==1.2.1 +tornado==6.3.3 +tqdm==4.66.1 +traitlets==5.13.0 +types-python-dateutil==2.8.19.14 +typing-extensions==4.8.0 +uri-template==1.3.0 +urllib3==2.1.0 +virtualenv==20.21.0 +wcwidth==0.2.12 +webcolors==1.13 +webencodings==0.5.1 +websocket-client==1.6.4 +wheel==0.41.3 +widgetsnbextension==4.0.9 +wrapt==1.16.0 +yarl==1.9.3 +zipp==3.17.0 diff --git a/scripts/setup_all.sh b/scripts/setup_all.sh index eb428456..c9f35d68 100755 --- a/scripts/setup_all.sh +++ b/scripts/setup_all.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +set -e + # Bootstrap this monorepo for development # Setup Node environment @@ -12,4 +14,4 @@ $(dirname $0)/setup_python.sh # Run setup tasks -pnpm exec nx run-many -t setup +pnpm run ci:setup diff --git a/scripts/setup_python.sh b/scripts/setup_python.sh index a5a61cd2..426a14c7 100755 --- a/scripts/setup_python.sh +++ b/scripts/setup_python.sh @@ -8,6 +8,9 @@ COLOR_RESET='\033[0m' if type "rye" &> /dev/null; then echo -e $COLOR_BLUE"Setting up Python environment using Rye"$COLOR_RESET + if [[ ! -z $PYTHON_VERSION ]]; then + rye pin $PYTHON_VERSION + fi rye sync --no-lock exit $? fi @@ -29,4 +32,4 @@ if test -z $CI && ! python -c "import sys; exit(int(sys.prefix == sys.base_prefi echo -e $COLOR_ORANGE"Not using a virtualenv. This is not recommended."$COLOR_RESET fi -python -m pip install -r requirements.lock -r requirements-dev.lock +python -m pip install -r requirements-dev.lock diff --git a/tsconfig.base.json b/tsconfig.base.json index adc8baae..34d950ac 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -1,6 +1,8 @@ { "$schema": "https://json.schemastore.org/tsconfig", "compilerOptions": { + "composite": true, + /* interop */ "allowJs": true, "allowSyntheticDefaultImports": true, @@ -12,7 +14,6 @@ "exactOptionalPropertyTypes": true, "noImplicitReturns": true, "noFallthroughCasesInSwitch": true, - // "noUncheckedIndexedAccess": true, "noUnusedLocals": true, "noUnusedParameters": true, "noPropertyAccessFromIndexSignature": true, diff --git a/tsconfig.node.json b/tsconfig.node.json index 381a159e..81b2bffc 100644 --- a/tsconfig.node.json +++ b/tsconfig.node.json @@ -2,9 +2,9 @@ "$schema": "https://json.schemastore.org/tsconfig", "extends": "./tsconfig.base.json", "compilerOptions": { + "composite": true, "module": "NodeNext", "moduleResolution": "NodeNext", - "experimentalDecorators": true, "lib": ["ESNext"], "types": ["node"] }