ci(framework:skip) Add caching to speed up E2E CI tests (#3480)

.github/workflows/e2e.yml

@@ -118,12 +118,26 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-      - name: Bootstrap
-        uses: ./.github/actions/bootstrap
+      - name: Set up Python
+        uses: actions/setup-python@v5
         with:
           python-version: 3.8
+      - name: Install build tools
+        run: |
+            python -m pip install -U pip==23.3.1
+        shell: bash
+      # Using approach described here for Python location caching:
+      # https://blog.allenai.org/python-caching-in-github-actions-e9452698e98d
+      - name: Cache Python location
+        id: cache-python
+        uses: actions/cache@v4
+        with:
+          path: ${{ env.pythonLocation }}
+          key: pythonloc-${{ runner.os }}-${{ matrix.directory }}-${{ env.pythonLocation }}-${{ hashFiles('**/pyproject.toml') }}
+          restore-keys: |
+            pythonloc-${{ runner.os }}-${{ matrix.directory }}-${{ env.pythonLocation }}
       - name: Install dependencies
-        run: python -m poetry install
+        run: python -m pip install --upgrade .
       - name: Install Flower wheel from artifact store
         if: ${{ github.repository == 'adap/flower' && !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]' }}
         run: |

e2e/bare-client-auth/pyproject.toml

@@ -1,13 +1,20 @@
 [build-system]
-requires = ["poetry-core>=1.4.0"]
-build-backend = "poetry.core.masonry.api"
+requires = ["hatchling"]
+build-backend = "hatchling.build"
 
-[tool.poetry]
+[project]
 name = "bare_client_auth_test"
 version = "0.1.0"
 description = "Client-auth-enabled bare Federated Learning test with Flower"
-authors = ["The Flower Authors <[email protected]>"]
+authors = [
+    { name = "The Flower Authors", email = "[email protected]" },
+]
+dependencies = [
+    "flwr @ {root:parent:parent:uri}",
+]
 
-[tool.poetry.dependencies]
-python = "^3.8"
-flwr = { path = "../../", develop = true }
+[tool.hatch.build.targets.wheel]
+packages = ["."]
+
+[tool.hatch.metadata]
+allow-direct-references = true

e2e/bare-https/pyproject.toml

@@ -1,13 +1,20 @@
 [build-system]
-requires = ["poetry-core>=1.4.0"]
-build-backend = "poetry.core.masonry.api"
+requires = ["hatchling"]
+build-backend = "hatchling.build"
 
-[tool.poetry]
+[project]
 name = "bare_https_test"
 version = "0.1.0"
 description = "HTTPS-enabled bare Federated Learning test with Flower"
-authors = ["The Flower Authors <[email protected]>"]
+authors = [
+    { name = "The Flower Authors", email = "[email protected]" },
+]
+dependencies = [
+    "flwr @ {root:parent:parent:uri}",
+]
 
-[tool.poetry.dependencies]
-python = "^3.8"
-flwr = { path = "../../", develop = true }
+[tool.hatch.build.targets.wheel]
+packages = ["."]
+
+[tool.hatch.metadata]
+allow-direct-references = true

e2e/bare/pyproject.toml

@@ -1,13 +1,20 @@
 [build-system]
-requires = ["poetry-core>=1.4.0"]
-build-backend = "poetry.core.masonry.api"
+requires = ["hatchling"]
+build-backend = "hatchling.build"
 
-[tool.poetry]
+[project]
 name = "bare_test"
 version = "0.1.0"
 description = "Bare Federated Learning test with Flower"
-authors = ["The Flower Authors <[email protected]>"]
+authors = [
+    { name = "The Flower Authors", email = "[email protected]" },
+]
+dependencies = [
+    "flwr[simulation,rest] @ {root:parent:parent:uri}",
+]
 
-[tool.poetry.dependencies]
-python = "^3.8"
-flwr = { path = "../../", develop = true, extras = ["simulation", "rest"] }
+[tool.hatch.build.targets.wheel]
+packages = ["."]
+
+[tool.hatch.metadata]
+allow-direct-references = true

e2e/fastai/pyproject.toml

@@ -1,15 +1,22 @@
 [build-system]
-requires = ["poetry-core>=1.4.0"]
-build-backend = "poetry.core.masonry.api"
+requires = ["hatchling"]
+build-backend = "hatchling.build"
 
-[tool.poetry]
+[project]
 name = "quickstart-fastai"
 version = "0.1.0"
 description = "Fastai Federated Learning E2E test with Flower"
-authors = ["The Flower Authors <[email protected]>"]
+authors = [
+    { name = "The Flower Authors", email = "[email protected]" },
+]
+dependencies = [
+    "flwr[simulation] @ {root:parent:parent:uri}",
+    "fastai>=2.7.12,<3.0.0",
+    "torch>=2.0.0,!=2.0.1,<2.1.0",
+]
 
-[tool.poetry.dependencies]
-python = ">=3.8,<3.10"
-flwr = { path = "../../", develop = true, extras = ["simulation"] }
-fastai = "^2.7.12"
-torch = ">=2.0.0, !=2.0.1, < 2.1.0"
+[tool.hatch.build.targets.wheel]
+packages = ["."]
+
+[tool.hatch.metadata]
+allow-direct-references = true

e2e/jax/pyproject.toml

@@ -1,17 +1,24 @@
-[tool.poetry]
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
 name = "jax_example"
 version = "0.1.0"
 description = "JAX example training a linear regression model with federated learning"
-authors = ["The Flower Authors <[email protected]>"]
+authors = [
+    { name = "The Flower Authors", email = "[email protected]" },
+]
+dependencies = [
+    "flwr[simulation] @ {root:parent:parent:uri}",
+    "jax==0.4.13",
+    "jaxlib==0.4.13",
+    "scikit-learn>=1.1.1,<2.0.0",
+    "numpy>=1.21.4,<2.0.0",
+]
 
-[tool.poetry.dependencies]
-python = "^3.8"
-flwr = { path = "../../", develop = true, extras = ["simulation"] }
-jax = "==0.4.13"
-jaxlib = "==0.4.13"
-scikit-learn = "^1.1.1"
-numpy = "^1.21.4"
+[tool.hatch.build.targets.wheel]
+packages = ["."]
 
-[build-system]
-requires = ["poetry-core>=1.4.0"]
-build-backend = "poetry.core.masonry.api"
+[tool.hatch.metadata]
+allow-direct-references = true

e2e/opacus/pyproject.toml

@@ -1,16 +1,23 @@
 [build-system]
-requires = ["poetry-core>=1.4.0"]
-build-backend = "poetry.core.masonry.api"
+requires = ["hatchling"]
+build-backend = "hatchling.build"
 
-[tool.poetry]
+[project]
 name = "opacus_e2e"
 version = "0.1.0"
 description = "Opacus E2E testing"
-authors = ["The Flower Authors <[email protected]>"]
+authors = [
+    { name = "The Flower Authors", email = "[email protected]" },
+]
+dependencies = [
+    "flwr[simulation] @ {root:parent:parent:uri}",
+    "opacus>=1.4.0,<2.0.0",
+    "torch>=1.13.1,<2.0.0",
+    "torchvision>=0.14.0,<2.0.0",
+]
 
-[tool.poetry.dependencies]
-python = "^3.8"
-flwr = { path = "../../", develop = true, extras = ["simulation"] }
-opacus = "^1.4.0"
-torch = "^1.13.1"
-torchvision = "^0.14.0"
+[tool.hatch.build.targets.wheel]
+packages = ["."]
+
+[tool.hatch.metadata]
+allow-direct-references = true

e2e/pandas/pyproject.toml

@@ -1,17 +1,26 @@
 [build-system]
-requires = ["poetry-core>=1.4.0"]
-build-backend = "poetry.core.masonry.api"
+requires = ["hatchling"]
+build-backend = "hatchling.build"
 
-[tool.poetry]
+[project]
 name = "quickstart-pandas"
 version = "0.1.0"
-description = "Pandas Federated Analytics Quickstart with Flower"
-authors = ["Ragy Haddad <[email protected]>"]
-maintainers = ["The Flower Authors <[email protected]>"]
+description = "Pandas E2E test with Flower"
+authors = [
+    { name = "Ragy Haddad", email = "[email protected]" },
+]
+maintainers = [
+    { name = "The Flower Authors", email = "[email protected]" },
+]
+dependencies = [
+    "flwr[simulation] @ {root:parent:parent:uri}",
+    "numpy>=1.21.0,<2.0.0",
+    "pandas>=2.0.0,<3.0.0",
+    "scikit-learn>=1.1.1,<2.0.0",
+]
 
-[tool.poetry.dependencies]
-python = "^3.8"
-flwr = { path = "../../", develop = true, extras = ["simulation"] }
-numpy = "^1.21.0"
-pandas = "^2.0.0"
-scikit-learn = "^1.1.1"
+[tool.hatch.build.targets.wheel]
+packages = ["."]
+
+[tool.hatch.metadata]
+allow-direct-references = true

e2e/pytorch-lightning/pyproject.toml

@@ -1,15 +1,22 @@
 [build-system]
-requires = ["poetry-core>=1.4.0"]
-build-backend = "poetry.masonry.api"
+requires = ["hatchling"]
+build-backend = "hatchling.build"
 
-[tool.poetry]
-name = "quickstart-pytorch-lightning"
+[project]
+name = "quickstart-pytorch-lightning-test"
 version = "0.1.0"
 description = "Federated Learning E2E test with Flower and PyTorch Lightning"
-authors = ["The Flower Authors <[email protected]>"]
+authors = [
+    { name = "The Flower Authors", email = "[email protected]" },
+]
+dependencies = [
+    "flwr[simulation] @ {root:parent:parent:uri}",
+    "pytorch-lightning==2.2.4",
+    "torchvision==0.14.1",
+]
 
-[tool.poetry.dependencies]
-python = "^3.8"
-flwr = { path = "../../", develop = true, extras = ["simulation"] }
-pytorch-lightning = "2.2.4"
-torchvision = "0.14.1"
+[tool.hatch.build.targets.wheel]
+packages = ["."]
+
+[tool.hatch.metadata]
+allow-direct-references = true

e2e/pytorch/pyproject.toml

@@ -1,16 +1,23 @@
 [build-system]
-requires = ["poetry-core>=1.4.0"]
-build-backend = "poetry.core.masonry.api"
+requires = ["hatchling"]
+build-backend = "hatchling.build"
 
-[tool.poetry]
-name = "quickstart-pytorch"
+[project]
+name = "pytorch_e2e"
 version = "0.1.0"
-description = "PyTorch Federated Learning Quickstart with Flower"
-authors = ["The Flower Authors <[email protected]>"]
+description = "PyTorch Federated Learning E2E test with Flower"
+authors = [
+    { name = "The Flower Authors", email = "[email protected]" },
+]
+dependencies = [
+    "flwr[simulation] @ {root:parent:parent:uri}",
+    "torch>=1.12.0,<2.0.0",
+    "torchvision>=0.14.1,<0.15.0",
+    "tqdm>=4.63.0,<5.0.0",
+]
 
-[tool.poetry.dependencies]
-python = "^3.8"
-flwr = { path = "../../", develop = true, extras = ["simulation"] }
-torch = "^1.12.0"
-torchvision = "^0.14.1"
-tqdm = "^4.63.0"
+[tool.hatch.build.targets.wheel]
+packages = ["."]
+
+[tool.hatch.metadata]
+allow-direct-references = true

e2e/scikit-learn/pyproject.toml

@@ -1,18 +1,23 @@
 [build-system]
-requires = ["poetry-core>=1.4.0"]
-build-backend = "poetry.core.masonry.api"
+requires = ["hatchling"]
+build-backend = "hatchling.build"
 
-[tool.poetry]
-name = "sklearn-mnist"
+[project]
+name = "sklearn-mnist-test"
 version = "0.1.0"
-description = "Federated learning with scikit-learn and Flower"
+description = "Federated learning E2E test with scikit-learn and Flower"
 authors = [
-    "The Flower Authors <[email protected]>",
-    "Kaushik Amar Das <[email protected]>",
+    { name = "The Flower Authors", email = "[email protected]" },
+    { name = "Kaushik Amar Das", email = "[email protected]"},
 ]
+dependencies = [
+    "flwr[simulation,rest] @ {root:parent:parent:uri}",
+    "scikit-learn>=1.1.1,<2.0.0",
+    "openml>=0.14.0,<0.15.0"
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]
 
-[tool.poetry.dependencies]
-python = "^3.8"
-flwr = { path = "../../", develop = true, extras = ["simulation"] }
-scikit-learn = "^1.1.1"
-openml = "^0.14.0"
+[tool.hatch.metadata]
+allow-direct-references = true