Merge pull request #522 from OpenMined/release_document

Added new release document

pyproject.toml

@@ -90,3 +90,9 @@ extend-select = ["I"]
 
 [tool.coverage.report]
 skip_empty = true
+
+[tool.setuptools]
+include-package-data = true
+
+[tool.setuptools.package-data]
+'syftbox' = ['syftbox/server2client_version.json']

scripts/upgrade_version_matrix.py

@@ -0,0 +1,59 @@
+import argparse
+import json
+
+from packaging.version import Version
+
+parser = argparse.ArgumentParser("upgrade_version_matrix")
+parser.add_argument("upgrade_type", choices=["major", "minor", "patch"])
+parser.add_argument("--breaking_changes", action="store_true")
+
+args = parser.parse_args()
+print(args.upgrade_type)
+print(args.breaking_changes)
+
+with open("../syftbox/server2client_version.json") as json_file:
+    version_matrix = json.load(json_file)
+
+versions = list(version_matrix.keys())
+versions.sort(key=Version)
+last_version = versions[-1]
+version_numbers = last_version.split(".")
+
+if args.upgrade_type == "patch":
+    if args.breaking_changes:
+        raise Exception(
+            "Patch upgrades imply no breaking changes. If you have breaking changes please consider a minor version upgrade"
+        )
+    version_numbers[2] = str(int(version_numbers[2]) + 1)
+    new_version = ".".join(version_numbers)
+    # new_version = last_version
+    version_matrix[new_version] = version_matrix[last_version]
+elif args.upgrade_type == "minor":
+    version_numbers[1] = str(int(version_numbers[1]) + 1)
+    version_numbers[2] = "0"
+    new_version = ".".join(version_numbers)
+    if args.breaking_changes:
+        version_matrix[new_version] = [new_version, ""]
+        for version in versions:
+            version_range = version_matrix[version]
+            if version_range[1] == "":
+                version_range[1] = new_version
+                version_matrix[version] = version_range
+    else:
+        version_matrix[new_version] = version_matrix[last_version]
+
+elif args.upgrade_type == "major":
+    raise NotImplementedError
+
+with open("../syftbox/server2client_version.json", "w") as json_file:
+    # json.dump(version_matrix, json_file, indent=4)
+    json_file.write("{\n")
+    json_file.write(
+        ",\n".join(
+            [
+                f"""\t"{key}": ["{version_range[0]}", "{version_range[1]}"]"""
+                for key, version_range in version_matrix.items()
+            ]
+        )
+    )
+    json_file.write("\n}\n")

syftbox/client/auth.py

@@ -5,14 +5,18 @@
 from rich import print as rprint
 from rich.prompt import Prompt
 
+from syftbox import __version__
 from syftbox.lib.client_config import SyftClientConfig
+from syftbox.lib.http import HEADER_SYFTBOX_VERSION
 
 
 def has_valid_access_token(conf: SyftClientConfig, auth_client: httpx.Client) -> bool:
     """Returns True if conf has a valid access token that matches the email in the config."""
     if not conf.access_token:
         return False
-    response = auth_client.post("/auth/whoami", headers={"Authorization": f"Bearer {conf.access_token}"})
+    response = auth_client.post(
+        "/auth/whoami", headers={"Authorization": f"Bearer {conf.access_token}", HEADER_SYFTBOX_VERSION: __version__}
+    )
     if response.status_code == 401:
         rprint("[red]Invalid access token, re-authenticating.[/red]")
         return False
@@ -41,7 +45,9 @@ def request_email_token(auth_client: httpx.Client, conf: SyftClientConfig) -> Op
     Returns:
         Optional[str]: email token if auth is disabled, None if auth is enabled
     """
-    response = auth_client.post("/auth/request_email_token", json={"email": conf.email})
+    response = auth_client.post(
+        "/auth/request_email_token", json={"email": conf.email}, headers={HEADER_SYFTBOX_VERSION: __version__}
+    )
     response.raise_for_status()
     return response.json().get("email_token", None)
 
@@ -69,7 +75,7 @@ def get_access_token(
 
     response = auth_client.post(
         "/auth/validate_email_token",
-        headers={"Authorization": f"Bearer {email_token}"},
+        headers={"Authorization": f"Bearer {email_token}", HEADER_SYFTBOX_VERSION: __version__},
         params={"email": conf.email},
     )
 

syftbox/client/base.py

@@ -4,11 +4,14 @@
 from typing import TYPE_CHECKING, Optional
 
 import httpx
+from packaging import version
 from typing_extensions import Protocol
 
-from syftbox.client.exceptions import SyftAuthenticationError, SyftPermissionError, SyftServerError
+from syftbox import __version__
+from syftbox.client.exceptions import SyftAuthenticationError, SyftPermissionError, SyftServerError, SyftServerTooOld
 from syftbox.lib.client_config import SyftClientConfig
-from syftbox.lib.http import HEADER_SYFTBOX_USER, SYFTBOX_HEADERS
+from syftbox.lib.http import HEADER_SYFTBOX_USER, HEADER_SYFTBOX_VERSION, SYFTBOX_HEADERS
+from syftbox.lib.version_utils import get_range_for_version
 from syftbox.lib.workspace import SyftWorkspace
 
 
@@ -90,6 +93,15 @@ def raise_for_status(self, response: httpx.Response) -> None:
             raise SyftPermissionError(f"No permission to access this resource: {response.text}")
         elif response.status_code != 200:
             raise SyftServerError(f"[{endpoint}] Server returned {response.status_code}: {response.text}")
+        server_version = response.headers.get(HEADER_SYFTBOX_VERSION)
+
+        version_range = get_range_for_version(server_version)
+        lower_bound_version = version_range[0]
+        upper_bound_version = version_range[1]
+
+        if len(upper_bound_version) > 0 and version.parse(upper_bound_version) < version.parse(__version__):
+            raise SyftServerTooOld(f"Server version is {server_version} and can only work with clients between \
+                                    {lower_bound_version} and {upper_bound_version}. Your client has version {__version__}.")
 
     @staticmethod
     def _make_headers(config: SyftClientConfig) -> dict:

syftbox/client/cli_setup.py

@@ -171,21 +171,6 @@ def verify_installation(conf: SyftClientConfig, client: httpx.Client) -> None:
             time.sleep(2)
             response = client.get("/info")
         response.raise_for_status()
-        server_info = response.json()
-        server_version = server_info["version"]
-        local_version = __version__
-
-        if server_version == local_version:
-            return
-
-        should_continue = Confirm.ask(
-            f"\n[yellow]Server version ({server_version}) does not match your client version ({local_version}).\n"
-            f"[bold](recommended)[/bold] To update, run:\n\n"
-            f"[bold]curl -LsSf https://syftbox.openmined.org/install.sh | sh[/bold][/yellow]\n\n"
-            f"Continue without updating?"
-        )
-        if not should_continue:
-            raise typer.Exit()
 
     except (httpx.HTTPError, KeyError):
         should_continue = Confirm.ask(

syftbox/client/exceptions.py

@@ -19,6 +19,10 @@ class SyftServerError(SyftBoxException):
     pass
 
 
+class SyftServerTooOld(SyftBoxException):
+    pass
+
+
 class SyftAuthenticationError(SyftServerError):
     default_message = "Authentication failed, please log in again."
 

syftbox/client/server_client.py

@@ -71,19 +71,13 @@ def get_datasite_states(self) -> dict[str, list[FileMetadata]]:
         return result
 
     def get_remote_state(self, relative_path: Path) -> list[FileMetadata]:
-        response = self.conn.post(
-            "/sync/dir_state",
-            params={"dir": relative_path.as_posix()},
-        )
+        response = self.conn.post("/sync/dir_state", params={"dir": relative_path.as_posix()})
         self.raise_for_status(response)
         data = response.json()
         return [FileMetadata(**item) for item in data]
 
     def get_metadata(self, path: Path) -> FileMetadata:
-        response = self.conn.post(
-            "/sync/get_metadata",
-            json={"path": path.as_posix()},
-        )
+        response = self.conn.post("/sync/get_metadata", json={"path": path.as_posix()})
         self.raise_for_status(response)
         return FileMetadata(**response.json())
 
@@ -138,10 +132,7 @@ def apply_diff(self, relative_path: Path, diff: Union[str, bytes], expected_hash
         return ApplyDiffResponse(**response.json())
 
     def delete(self, relative_path: Path) -> None:
-        response = self.conn.post(
-            "/sync/delete",
-            json={"path": relative_path.as_posix()},
-        )
+        response = self.conn.post("/sync/delete", json={"path": relative_path.as_posix()})
         self.raise_for_status(response)
 
     def create(self, relative_path: Path, data: bytes) -> None:
@@ -152,10 +143,7 @@ def create(self, relative_path: Path, data: bytes) -> None:
         self.raise_for_status(response)
 
     def download(self, relative_path: Path) -> bytes:
-        response = self.conn.post(
-            "/sync/download",
-            json={"path": relative_path.as_posix()},
-        )
+        response = self.conn.post("/sync/download", json={"path": relative_path.as_posix()})
         self.raise_for_status(response)
         return response.content
 

syftbox/lib/version_utils.py

@@ -0,0 +1,15 @@
+import json
+from os.path import dirname
+from typing import Dict, List
+
+
+def get_version_dict() -> Dict[str, List[str]]:
+    print(dirname(dirname(__file__)))
+    with open(dirname(dirname(__file__)) + "/server2client_version.json") as json_file:
+        version_matrix = json.load(json_file)
+    return version_matrix
+
+
+def get_range_for_version(version_string: str) -> List[str]:
+    print(f"{version_string=}")
+    return get_version_dict()[version_string]

syftbox/server/middleware.py

@@ -3,8 +3,15 @@
 
 from fastapi import Request, Response, status
 from loguru import logger
+from packaging import version
 from starlette.middleware.base import BaseHTTPMiddleware
 
+from syftbox import __version__
+from syftbox.lib.http import (
+    HEADER_SYFTBOX_VERSION,
+)
+from syftbox.lib.version_utils import get_range_for_version
+
 
 class LoguruMiddleware(BaseHTTPMiddleware):
     async def dispatch(self, request: Request, call_next: Callable) -> Response:
@@ -43,3 +50,30 @@ async def dispatch(self, request: Request, call_next: Callable) -> Response:
 
         response = await call_next(request)
         return response
+
+
+class VersionCheckMiddleware(BaseHTTPMiddleware):
+    async def dispatch(self, request: Request, call_next: Callable) -> Response:
+        logger.info(request.headers)
+        client_version = request.headers.get(HEADER_SYFTBOX_VERSION)
+        if not client_version:
+            return Response(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                content="Client version not provided. Please include the 'Version' header.",
+            )
+
+        version_range = get_range_for_version(client_version)
+        lower_bound_version = version_range[0]
+        # upper_bound_version = version_range[1]
+        print(client_version, lower_bound_version)
+
+        if version.parse(client_version) < version.parse(lower_bound_version):
+            return Response(
+                status_code=status.HTTP_426_UPGRADE_REQUIRED,
+                content=f"Client version is too old. Minimum version required is {lower_bound_version}",
+            )
+
+        response = await call_next(request)
+        response.headers[HEADER_SYFTBOX_VERSION] = __version__
+        logger.debug("server headers:", response.headers)
+        return response

syftbox/server/server.py

@@ -35,7 +35,7 @@
 )
 from syftbox.server.analytics import log_analytics_event
 from syftbox.server.logger import setup_logger
-from syftbox.server.middleware import LoguruMiddleware, RequestSizeLimitMiddleware
+from syftbox.server.middleware import LoguruMiddleware, RequestSizeLimitMiddleware, VersionCheckMiddleware
 from syftbox.server.settings import ServerSettings, get_server_settings
 from syftbox.server.telemetry import (
     OTEL_ATTR_CLIENT_OS_ARCH,
@@ -106,6 +106,7 @@ async def lifespan(app: FastAPI, settings: Optional[ServerSettings] = None) -> A
 app.add_middleware(GZipMiddleware, minimum_size=1000, compresslevel=5)
 app.add_middleware(LoguruMiddleware)
 app.add_middleware(RequestSizeLimitMiddleware)
+app.add_middleware(VersionCheckMiddleware)
 
 FastAPIInstrumentor.instrument_app(app, server_request_hook=server_request_hook)
 SQLite3Instrumentor().instrument()

syftbox/server2client_version.json

@@ -0,0 +1,39 @@
+{
+  "0.1.0": ["0.1.0", "0.2.0"],
+  "0.1.1": ["0.1.0", "0.2.0"],
+  "0.1.2": ["0.1.0", "0.2.0"],
+  "0.1.3": ["0.1.0", "0.2.0"],
+  "0.1.4": ["0.1.0", "0.2.0"],
+  "0.1.5": ["0.1.0", "0.2.0"],
+  "0.1.6": ["0.1.0", "0.2.0"],
+  "0.1.7": ["0.1.0", "0.2.0"],
+  "0.1.8": ["0.1.0", "0.2.0"],
+  "0.1.9": ["0.1.0", "0.2.0"],
+  "0.1.10": ["0.1.0", "0.2.0"],
+  "0.1.11": ["0.1.0", "0.2.0"],
+  "0.1.12": ["0.1.0", "0.2.0"],
+  "0.1.13": ["0.1.0", "0.2.0"],
+  "0.1.14": ["0.1.0", "0.2.0"],
+  "0.1.15": ["0.1.0", "0.2.0"],
+  "0.1.16": ["0.1.0", "0.2.0"],
+  "0.1.17": ["0.1.0", "0.2.0"],
+  "0.1.18": ["0.1.0", "0.2.0"],
+  "0.1.19": ["0.1.0", "0.2.0"],
+  "0.1.20": ["0.1.0", "0.2.0"],
+  "0.1.21": ["0.1.0", "0.2.0"],
+  "0.1.22": ["0.1.0", "0.2.0"],
+  "0.1.23": ["0.1.0", "0.2.0"],
+  "0.1.24": ["0.1.0", "0.2.0"],
+  "0.2.0": ["0.2.0", ""],
+  "0.2.1": ["0.2.0", ""],
+  "0.2.2": ["0.2.0", ""],
+  "0.2.3": ["0.2.0", ""],
+  "0.2.4": ["0.2.0", ""],
+  "0.2.5": ["0.2.0", ""],
+  "0.2.6": ["0.2.0", ""],
+  "0.2.7": ["0.2.0", ""],
+  "0.2.8": ["0.2.0", ""],
+  "0.2.9": ["0.2.0", ""],
+  "0.2.10": ["0.2.0", ""],
+  "0.2.11": ["0.2.0", ""]
+}

tests/integration/sync/conftest.py

@@ -6,11 +6,13 @@
 from fastapi import FastAPI
 from fastapi.testclient import TestClient
 
+from syftbox import __version__
 from syftbox.client.base import PluginManagerInterface, SyftBoxContextInterface
 from syftbox.client.core import SyftBoxContext
 from syftbox.client.server_client import SyftBoxClient
 from syftbox.lib.client_config import SyftClientConfig
 from syftbox.lib.datasite import create_datasite
+from syftbox.lib.http import HEADER_SYFTBOX_VERSION
 from syftbox.lib.workspace import SyftWorkspace
 from syftbox.server.migrations import run_migrations
 from syftbox.server.server import app as server_app
@@ -23,6 +25,7 @@ def authenticate_testclient(client: TestClient, email: str) -> None:
     access_token = get_access_token(client, email)
     client.headers["email"] = email
     client.headers["Authorization"] = f"Bearer {access_token}"
+    client.headers[HEADER_SYFTBOX_VERSION] = __version__
 
 
 class MockPluginManager(PluginManagerInterface):
@@ -74,17 +77,20 @@ def server_app_with_lifespan(tmp_path: Path) -> FastAPI:
 def datasite_1(tmp_path: Path, server_app_with_lifespan: FastAPI) -> SyftBoxContextInterface:
     email = "[email protected]"
     with TestClient(server_app_with_lifespan) as client:
+        client.headers[HEADER_SYFTBOX_VERSION] = __version__
         return setup_datasite(tmp_path, client, email)
 
 
 @pytest.fixture()
 def datasite_2(tmp_path: Path, server_app_with_lifespan: FastAPI) -> SyftBoxContextInterface:
     email = "[email protected]"
     with TestClient(server_app_with_lifespan) as client:
+        client.headers[HEADER_SYFTBOX_VERSION] = __version__
         return setup_datasite(tmp_path, client, email)
 
 
 @pytest.fixture(scope="function")
 def server_client(server_app_with_lifespan: FastAPI) -> Generator[TestClient, None, None]:
     with TestClient(server_app_with_lifespan) as client:
+        client.headers[HEADER_SYFTBOX_VERSION] = __version__
         yield client