From a45c4cad50095740fbb2cf633e1084e9b23ee608 Mon Sep 17 00:00:00 2001 From: kiukchung <43595115+kiukchung@users.noreply.github.com> Date: Tue, 19 Sep 2023 21:51:47 -0700 Subject: [PATCH] [torchx/specs] Use default_factory for the default value of Role.resource and mlflow_test.Config.model_config to support python 3.11 clients (#768) --- .github/workflows/python-unittests.yaml | 7 +++++-- dev-requirements.txt | 2 ++ requirements.txt | 3 ++- torchx/specs/api.py | 9 ++++++++- torchx/tracker/test/mlflow_test.py | 2 +- 5 files changed, 18 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-unittests.yaml b/.github/workflows/python-unittests.yaml index 5a8daa193..8403e2cee 100644 --- a/.github/workflows/python-unittests.yaml +++ b/.github/workflows/python-unittests.yaml @@ -10,7 +10,7 @@ jobs: unittest: strategy: matrix: - python-version: [3.8, 3.9, '3.10'] + python-version: [3.8, 3.9, "3.10", 3.11] platform: ["linux.20_04.4x"] include: - python-version: 3.9 @@ -32,7 +32,10 @@ jobs: run: | set -eux pip install pytest pytest-cov - pip install -e .[dev] + # use legacy resolver for python 3.11, otherwise pip will timeout trying to resolve deps + # TODO(kiukchung) long term we should narrowly scope dependency versions + # see: https://pip.pypa.io/en/latest/topics/dependency-resolution/ + pip install --use-deprecated=legacy-resolver -e .[dev] - name: Run tests run: pytest --cov=./ --cov-report=xml - name: Upload coverage to Codecov diff --git a/dev-requirements.txt b/dev-requirements.txt index a71aee081..36fe58345 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -14,6 +14,8 @@ ipython kfp==1.8.22 mlflow-skinny moto==4.1.6 +# kfp==1.8.22 needs protobuf < 4 +protobuf==3.20.3 pyre-extensions pyre-check pytest diff --git a/requirements.txt b/requirements.txt index 88856b6af..ee0407b9d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,8 @@ importlib-metadata pyyaml docker filelock -fsspec +# more recent versions of fsspec causes torchx/workspace/test/dir_workspace_test#test_torchcxignore to fail +fsspec==2023.1.0 # To resolve confliciting dependencies for urllib3: https://github.com/pytorch/torchx/actions/runs/3484190429/jobs/5828784263#step:4:552 urllib3<1.27,>=1.21.1 tabulate diff --git a/torchx/specs/api.py b/torchx/specs/api.py index 779e1b8de..61cf96081 100644 --- a/torchx/specs/api.py +++ b/torchx/specs/api.py @@ -104,6 +104,13 @@ def copy(original: "Resource", **capabilities: Any) -> "Resource": # sentinel value used for cases when resource does not matter (e.g. ignored) NULL_RESOURCE: Resource = Resource(cpu=-1, gpu=-1, memMB=-1) + +# no-arg static factory method to use with default_factory in @dataclass +# needed to support python 3.11 since mutable defaults for dataclasses are not allowed in 3.11 +def _null_resource() -> Resource: + return NULL_RESOURCE + + # used as "*" scheduler backend ALL: str = "all" @@ -333,7 +340,7 @@ class Role: num_replicas: int = 1 max_retries: int = 0 retry_policy: RetryPolicy = RetryPolicy.APPLICATION - resource: Resource = NULL_RESOURCE + resource: Resource = field(default_factory=_null_resource) port_map: Dict[str, int] = field(default_factory=dict) metadata: Dict[str, Any] = field(default_factory=dict) mounts: List[Union[BindMount, VolumeMount, DeviceMount]] = field( diff --git a/torchx/tracker/test/mlflow_test.py b/torchx/tracker/test/mlflow_test.py index d367218f4..791e388f0 100644 --- a/torchx/tracker/test/mlflow_test.py +++ b/torchx/tracker/test/mlflow_test.py @@ -44,7 +44,7 @@ class Config: locales: List[str] = field(default_factory=lambda: ["us", "eu", "fr"]) empty_list: List[str] = field(default_factory=list) empty_map: Dict[str, str] = field(default_factory=dict) - model_config: ModelConfig = ModelConfig() + model_config: ModelConfig = field(default_factory=ModelConfig) datasets: List[DatasetConfig] = field( default_factory=lambda: [ DatasetConfig(url="s3://dataset1"),