From a43aa0d77a02997d251a0945e108f4c1a29de004 Mon Sep 17 00:00:00 2001 From: Chong Shen Ng Date: Thu, 23 May 2024 17:17:33 +0100 Subject: [PATCH] ci(framework:skip) Add caching to speed up E2E CI tests (#3480) --- .github/workflows/e2e.yml | 20 +++++++++++++--- e2e/bare-client-auth/pyproject.toml | 21 +++++++++++------ e2e/bare-https/pyproject.toml | 21 +++++++++++------ e2e/bare/pyproject.toml | 21 +++++++++++------ e2e/fastai/pyproject.toml | 25 +++++++++++++------- e2e/jax/pyproject.toml | 31 ++++++++++++++---------- e2e/opacus/pyproject.toml | 27 +++++++++++++-------- e2e/pandas/pyproject.toml | 33 ++++++++++++++++---------- e2e/pytorch-lightning/pyproject.toml | 27 +++++++++++++-------- e2e/pytorch/pyproject.toml | 31 ++++++++++++++---------- e2e/scikit-learn/pyproject.toml | 29 +++++++++++++---------- e2e/tabnet/pyproject.toml | 35 +++++++++++++++++----------- e2e/tensorflow/pyproject.toml | 29 ++++++++++++++--------- 13 files changed, 224 insertions(+), 126 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 57802e598546..1802809bd2b0 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -118,12 +118,26 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Bootstrap - uses: ./.github/actions/bootstrap + - name: Set up Python + uses: actions/setup-python@v5 with: python-version: 3.8 + - name: Install build tools + run: | + python -m pip install -U pip==23.3.1 + shell: bash + # Using approach described here for Python location caching: + # https://blog.allenai.org/python-caching-in-github-actions-e9452698e98d + - name: Cache Python location + id: cache-python + uses: actions/cache@v4 + with: + path: ${{ env.pythonLocation }} + key: pythonloc-${{ runner.os }}-${{ matrix.directory }}-${{ env.pythonLocation }}-${{ hashFiles('**/pyproject.toml') }} + restore-keys: | + pythonloc-${{ runner.os }}-${{ matrix.directory }}-${{ env.pythonLocation }} - name: Install dependencies - run: python -m poetry install + run: python -m pip install --upgrade . - name: Install Flower wheel from artifact store if: ${{ github.repository == 'adap/flower' && !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]' }} run: | diff --git a/e2e/bare-client-auth/pyproject.toml b/e2e/bare-client-auth/pyproject.toml index 693fec815474..839f0779cc01 100644 --- a/e2e/bare-client-auth/pyproject.toml +++ b/e2e/bare-client-auth/pyproject.toml @@ -1,13 +1,20 @@ [build-system] -requires = ["poetry-core>=1.4.0"] -build-backend = "poetry.core.masonry.api" +requires = ["hatchling"] +build-backend = "hatchling.build" -[tool.poetry] +[project] name = "bare_client_auth_test" version = "0.1.0" description = "Client-auth-enabled bare Federated Learning test with Flower" -authors = ["The Flower Authors "] +authors = [ + { name = "The Flower Authors", email = "hello@flower.ai" }, +] +dependencies = [ + "flwr @ {root:parent:parent:uri}", +] -[tool.poetry.dependencies] -python = "^3.8" -flwr = { path = "../../", develop = true } +[tool.hatch.build.targets.wheel] +packages = ["."] + +[tool.hatch.metadata] +allow-direct-references = true diff --git a/e2e/bare-https/pyproject.toml b/e2e/bare-https/pyproject.toml index 3afb7b57a084..de8aa92cbd02 100644 --- a/e2e/bare-https/pyproject.toml +++ b/e2e/bare-https/pyproject.toml @@ -1,13 +1,20 @@ [build-system] -requires = ["poetry-core>=1.4.0"] -build-backend = "poetry.core.masonry.api" +requires = ["hatchling"] +build-backend = "hatchling.build" -[tool.poetry] +[project] name = "bare_https_test" version = "0.1.0" description = "HTTPS-enabled bare Federated Learning test with Flower" -authors = ["The Flower Authors "] +authors = [ + { name = "The Flower Authors", email = "hello@flower.ai" }, +] +dependencies = [ + "flwr @ {root:parent:parent:uri}", +] -[tool.poetry.dependencies] -python = "^3.8" -flwr = { path = "../../", develop = true } +[tool.hatch.build.targets.wheel] +packages = ["."] + +[tool.hatch.metadata] +allow-direct-references = true diff --git a/e2e/bare/pyproject.toml b/e2e/bare/pyproject.toml index 45ce7ea333af..ba8c1b2b2276 100644 --- a/e2e/bare/pyproject.toml +++ b/e2e/bare/pyproject.toml @@ -1,13 +1,20 @@ [build-system] -requires = ["poetry-core>=1.4.0"] -build-backend = "poetry.core.masonry.api" +requires = ["hatchling"] +build-backend = "hatchling.build" -[tool.poetry] +[project] name = "bare_test" version = "0.1.0" description = "Bare Federated Learning test with Flower" -authors = ["The Flower Authors "] +authors = [ + { name = "The Flower Authors", email = "hello@flower.ai" }, +] +dependencies = [ + "flwr[simulation,rest] @ {root:parent:parent:uri}", +] -[tool.poetry.dependencies] -python = "^3.8" -flwr = { path = "../../", develop = true, extras = ["simulation", "rest"] } +[tool.hatch.build.targets.wheel] +packages = ["."] + +[tool.hatch.metadata] +allow-direct-references = true diff --git a/e2e/fastai/pyproject.toml b/e2e/fastai/pyproject.toml index feed31f6d202..53d3b7e7baf1 100644 --- a/e2e/fastai/pyproject.toml +++ b/e2e/fastai/pyproject.toml @@ -1,15 +1,22 @@ [build-system] -requires = ["poetry-core>=1.4.0"] -build-backend = "poetry.core.masonry.api" +requires = ["hatchling"] +build-backend = "hatchling.build" -[tool.poetry] +[project] name = "quickstart-fastai" version = "0.1.0" description = "Fastai Federated Learning E2E test with Flower" -authors = ["The Flower Authors "] +authors = [ + { name = "The Flower Authors", email = "hello@flower.ai" }, +] +dependencies = [ + "flwr[simulation] @ {root:parent:parent:uri}", + "fastai>=2.7.12,<3.0.0", + "torch>=2.0.0,!=2.0.1,<2.1.0", +] -[tool.poetry.dependencies] -python = ">=3.8,<3.10" -flwr = { path = "../../", develop = true, extras = ["simulation"] } -fastai = "^2.7.12" -torch = ">=2.0.0, !=2.0.1, < 2.1.0" +[tool.hatch.build.targets.wheel] +packages = ["."] + +[tool.hatch.metadata] +allow-direct-references = true diff --git a/e2e/jax/pyproject.toml b/e2e/jax/pyproject.toml index 9a4af5dee59a..bb024ba14d23 100644 --- a/e2e/jax/pyproject.toml +++ b/e2e/jax/pyproject.toml @@ -1,17 +1,24 @@ -[tool.poetry] +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] name = "jax_example" version = "0.1.0" description = "JAX example training a linear regression model with federated learning" -authors = ["The Flower Authors "] +authors = [ + { name = "The Flower Authors", email = "hello@flower.ai" }, +] +dependencies = [ + "flwr[simulation] @ {root:parent:parent:uri}", + "jax==0.4.13", + "jaxlib==0.4.13", + "scikit-learn>=1.1.1,<2.0.0", + "numpy>=1.21.4,<2.0.0", +] -[tool.poetry.dependencies] -python = "^3.8" -flwr = { path = "../../", develop = true, extras = ["simulation"] } -jax = "==0.4.13" -jaxlib = "==0.4.13" -scikit-learn = "^1.1.1" -numpy = "^1.21.4" +[tool.hatch.build.targets.wheel] +packages = ["."] -[build-system] -requires = ["poetry-core>=1.4.0"] -build-backend = "poetry.core.masonry.api" +[tool.hatch.metadata] +allow-direct-references = true diff --git a/e2e/opacus/pyproject.toml b/e2e/opacus/pyproject.toml index ab4a727cc00b..8fd1056f43a3 100644 --- a/e2e/opacus/pyproject.toml +++ b/e2e/opacus/pyproject.toml @@ -1,16 +1,23 @@ [build-system] -requires = ["poetry-core>=1.4.0"] -build-backend = "poetry.core.masonry.api" +requires = ["hatchling"] +build-backend = "hatchling.build" -[tool.poetry] +[project] name = "opacus_e2e" version = "0.1.0" description = "Opacus E2E testing" -authors = ["The Flower Authors "] +authors = [ + { name = "The Flower Authors", email = "hello@flower.ai" }, +] +dependencies = [ + "flwr[simulation] @ {root:parent:parent:uri}", + "opacus>=1.4.0,<2.0.0", + "torch>=1.13.1,<2.0.0", + "torchvision>=0.14.0,<2.0.0", +] -[tool.poetry.dependencies] -python = "^3.8" -flwr = { path = "../../", develop = true, extras = ["simulation"] } -opacus = "^1.4.0" -torch = "^1.13.1" -torchvision = "^0.14.0" +[tool.hatch.build.targets.wheel] +packages = ["."] + +[tool.hatch.metadata] +allow-direct-references = true diff --git a/e2e/pandas/pyproject.toml b/e2e/pandas/pyproject.toml index 416dfeec3460..f8f8488a7006 100644 --- a/e2e/pandas/pyproject.toml +++ b/e2e/pandas/pyproject.toml @@ -1,17 +1,26 @@ [build-system] -requires = ["poetry-core>=1.4.0"] -build-backend = "poetry.core.masonry.api" +requires = ["hatchling"] +build-backend = "hatchling.build" -[tool.poetry] +[project] name = "quickstart-pandas" version = "0.1.0" -description = "Pandas Federated Analytics Quickstart with Flower" -authors = ["Ragy Haddad "] -maintainers = ["The Flower Authors "] +description = "Pandas E2E test with Flower" +authors = [ + { name = "Ragy Haddad", email = "ragy202@gmail.com" }, +] +maintainers = [ + { name = "The Flower Authors", email = "hello@flower.ai" }, +] +dependencies = [ + "flwr[simulation] @ {root:parent:parent:uri}", + "numpy>=1.21.0,<2.0.0", + "pandas>=2.0.0,<3.0.0", + "scikit-learn>=1.1.1,<2.0.0", +] -[tool.poetry.dependencies] -python = "^3.8" -flwr = { path = "../../", develop = true, extras = ["simulation"] } -numpy = "^1.21.0" -pandas = "^2.0.0" -scikit-learn = "^1.1.1" +[tool.hatch.build.targets.wheel] +packages = ["."] + +[tool.hatch.metadata] +allow-direct-references = true diff --git a/e2e/pytorch-lightning/pyproject.toml b/e2e/pytorch-lightning/pyproject.toml index 90d659813c28..8706ef098d8b 100644 --- a/e2e/pytorch-lightning/pyproject.toml +++ b/e2e/pytorch-lightning/pyproject.toml @@ -1,15 +1,22 @@ [build-system] -requires = ["poetry-core>=1.4.0"] -build-backend = "poetry.masonry.api" +requires = ["hatchling"] +build-backend = "hatchling.build" -[tool.poetry] -name = "quickstart-pytorch-lightning" +[project] +name = "quickstart-pytorch-lightning-test" version = "0.1.0" description = "Federated Learning E2E test with Flower and PyTorch Lightning" -authors = ["The Flower Authors "] +authors = [ + { name = "The Flower Authors", email = "hello@flower.ai" }, +] +dependencies = [ + "flwr[simulation] @ {root:parent:parent:uri}", + "pytorch-lightning==2.2.4", + "torchvision==0.14.1", +] -[tool.poetry.dependencies] -python = "^3.8" -flwr = { path = "../../", develop = true, extras = ["simulation"] } -pytorch-lightning = "2.2.4" -torchvision = "0.14.1" +[tool.hatch.build.targets.wheel] +packages = ["."] + +[tool.hatch.metadata] +allow-direct-references = true diff --git a/e2e/pytorch/pyproject.toml b/e2e/pytorch/pyproject.toml index e538f1437df6..8c59c43d50df 100644 --- a/e2e/pytorch/pyproject.toml +++ b/e2e/pytorch/pyproject.toml @@ -1,16 +1,23 @@ [build-system] -requires = ["poetry-core>=1.4.0"] -build-backend = "poetry.core.masonry.api" +requires = ["hatchling"] +build-backend = "hatchling.build" -[tool.poetry] -name = "quickstart-pytorch" +[project] +name = "pytorch_e2e" version = "0.1.0" -description = "PyTorch Federated Learning Quickstart with Flower" -authors = ["The Flower Authors "] +description = "PyTorch Federated Learning E2E test with Flower" +authors = [ + { name = "The Flower Authors", email = "hello@flower.ai" }, +] +dependencies = [ + "flwr[simulation] @ {root:parent:parent:uri}", + "torch>=1.12.0,<2.0.0", + "torchvision>=0.14.1,<0.15.0", + "tqdm>=4.63.0,<5.0.0", +] -[tool.poetry.dependencies] -python = "^3.8" -flwr = { path = "../../", develop = true, extras = ["simulation"] } -torch = "^1.12.0" -torchvision = "^0.14.1" -tqdm = "^4.63.0" +[tool.hatch.build.targets.wheel] +packages = ["."] + +[tool.hatch.metadata] +allow-direct-references = true diff --git a/e2e/scikit-learn/pyproject.toml b/e2e/scikit-learn/pyproject.toml index 50c07d31add7..caba2324d44f 100644 --- a/e2e/scikit-learn/pyproject.toml +++ b/e2e/scikit-learn/pyproject.toml @@ -1,18 +1,23 @@ [build-system] -requires = ["poetry-core>=1.4.0"] -build-backend = "poetry.core.masonry.api" +requires = ["hatchling"] +build-backend = "hatchling.build" -[tool.poetry] -name = "sklearn-mnist" +[project] +name = "sklearn-mnist-test" version = "0.1.0" -description = "Federated learning with scikit-learn and Flower" +description = "Federated learning E2E test with scikit-learn and Flower" authors = [ - "The Flower Authors ", - "Kaushik Amar Das ", + { name = "The Flower Authors", email = "hello@flower.ai" }, + { name = "Kaushik Amar Das", email = "kaushik.das@iiitg.ac.in"}, ] +dependencies = [ + "flwr[simulation,rest] @ {root:parent:parent:uri}", + "scikit-learn>=1.1.1,<2.0.0", + "openml>=0.14.0,<0.15.0" +] + +[tool.hatch.build.targets.wheel] +packages = ["."] -[tool.poetry.dependencies] -python = "^3.8" -flwr = { path = "../../", develop = true, extras = ["simulation"] } -scikit-learn = "^1.1.1" -openml = "^0.14.0" +[tool.hatch.metadata] +allow-direct-references = true diff --git a/e2e/tabnet/pyproject.toml b/e2e/tabnet/pyproject.toml index b1abf382a24a..99379ddb607e 100644 --- a/e2e/tabnet/pyproject.toml +++ b/e2e/tabnet/pyproject.toml @@ -1,18 +1,25 @@ [build-system] -requires = ["poetry-core>=1.4.0"] -build-backend = "poetry.core.masonry.api" +requires = ["hatchling"] +build-backend = "hatchling.build" -[tool.poetry] -name = "quickstart-tabnet" +[project] +name = "quickstart-tabnet-test" version = "0.1.0" -description = "Tabnet Federated Learning Quickstart with Flower" -authors = ["The Flower Authors "] +description = "Tabnet Federated Learning E2E test with Flower" +authors = [ + { name = "The Flower Authors", email = "hello@flower.ai" }, +] +dependencies = [ + "flwr[simulation] @ {root:parent:parent:uri}", + "tensorflow-cpu>=2.9.1,!=2.11.1; platform_machine == \"x86_64\"", + "tensorflow-macos>=2.9.1,!=2.11.1; sys_platform == \"darwin\" and platform_machine == \"arm64\"", + "tensorflow_datasets==4.9.2", + "tensorflow-io-gcs-filesystem<0.35.0", + "tabnet==0.1.6", +] -[tool.poetry.dependencies] -python = ">=3.8,<3.11" -flwr = { path = "../../", develop = true, extras = ["simulation"] } -tensorflow-cpu = { version = ">=2.9.1,<2.11.1 || >2.11.1", markers = "platform_machine == \"x86_64\"" } -tensorflow-macos = { version = ">=2.9.1,<2.11.1 || >2.11.1", markers = "sys_platform == \"darwin\" and platform_machine == \"arm64\"" } -tensorflow_datasets = "4.9.2" -tensorflow-io-gcs-filesystem = "<0.35.0" -tabnet = "0.1.6" +[tool.hatch.build.targets.wheel] +packages = ["."] + +[tool.hatch.metadata] +allow-direct-references = true diff --git a/e2e/tensorflow/pyproject.toml b/e2e/tensorflow/pyproject.toml index a7dbfe2305db..4b035873223c 100644 --- a/e2e/tensorflow/pyproject.toml +++ b/e2e/tensorflow/pyproject.toml @@ -1,15 +1,22 @@ [build-system] -requires = ["poetry-core>=1.4.0"] -build-backend = "poetry.core.masonry.api" +requires = ["hatchling"] +build-backend = "hatchling.build" -[tool.poetry] -name = "quickstart-tensorflow" +[project] +name = "quickstart-tensorflow-test" version = "0.1.0" -description = "Keras Federated Learning Quickstart with Flower" -authors = ["The Flower Authors "] +description = "Keras Federated Learning E2E test with Flower" +authors = [ + { name = "The Flower Authors", email = "hello@flower.ai" }, +] +dependencies = [ + "flwr[simulation] @ {root:parent:parent:uri}", + "tensorflow-cpu>=2.9.1,!=2.11.1", + "tensorflow-io-gcs-filesystem<0.35.0", +] -[tool.poetry.dependencies] -python = ">=3.8,<3.11" -flwr = { path = "../../", develop = true, extras = ["simulation"] } -tensorflow-cpu = "^2.9.1, !=2.11.1" -tensorflow-io-gcs-filesystem = "<0.35.0" +[tool.hatch.build.targets.wheel] +packages = ["."] + +[tool.hatch.metadata] +allow-direct-references = true