(torchx/schedulers) Retire gcp batch scheduler

kiukchung · web-flow · commit b72ba038140b · 2025-09-22T15:54:17.000-07:00
Differential Revision: D82797094 Pull Request resolved: #1122
diff --git a/.github/workflows/components-integration-tests.yaml b/.github/workflows/components-integration-tests.yaml
@@ -59,13 +59,6 @@ jobs:
           pip install -r dev-requirements.txt
           pip install -e .[kubernetes]
 
-      - name: Start Ray
-        if: ${{ matrix.scheduler == 'ray' }}
-        run: |
-          ray start --head
-          sleep 5 # wait till the node joins so that `status` command displays the head node info
-          ray status
-
       - name: Start Kubernetes
         if: ${{ matrix.scheduler == 'kubernetes' }}
         run: |
diff --git a/.github/workflows/gcp-batch-integration-tests.yaml b/.github/workflows/gcp-batch-integration-tests.yaml
diff --git a/README.md b/README.md
@@ -19,7 +19,6 @@ TorchX currently supports:
 * AWS Batch
 * Docker
 * Local
-* GCP Batch (prototype)
 
 Need a scheduler not listed? [Let us know!](https://github.com/pytorch/torchx/issues?q=is%3Aopen+is%3Aissue+label%3Ascheduler-request)
 
diff --git a/dev-requirements.txt b/dev-requirements.txt
@@ -7,9 +7,6 @@ kubernetes==25.3.0
 flake8==3.9.0
 fsspec==2024.3.1
 s3fs==2024.3.1
-google-cloud-batch==0.17.14
-google-cloud-logging==3.10.0
-google-cloud-runtimeconfig==0.34.0
 hydra-core
 ipython
 mlflow-skinny
@@ -34,10 +31,3 @@ wheel
 # lint (linter versions are managed by lintrunner)
 lintrunner
 lintrunner-adapters
-
-
-# reduce backtracking
-grpcio==1.62.1
-grpcio-status==1.48.1
-googleapis-common-protos==1.63.0
-google-api-core==2.18.0
diff --git a/docs/source/index.rst b/docs/source/index.rst
@@ -77,7 +77,6 @@ Works With
    schedulers/aws_batch
    schedulers/aws_sagemaker
    schedulers/lsf
-   schedulers/gcp_batch
 
 .. fbcode::
 
diff --git a/docs/source/schedulers/gcp_batch.rst b/docs/source/schedulers/gcp_batch.rst
diff --git a/requirements.txt b/requirements.txt
@@ -3,6 +3,4 @@ pyyaml
 docker
 filelock
 fsspec>=2023.10.0
-# To resolve confliciting dependencies for urllib3: https://github.com/pytorch/torchx/actions/runs/3484190429/jobs/5828784263#step:4:552
-urllib3<1.27,>=1.21.1
 tabulate
diff --git a/scripts/component_integration_tests.py b/scripts/component_integration_tests.py
@@ -65,7 +65,6 @@ def main() -> None:
         "local_docker",
         "aws_batch",
         "lsf",
-        "gcp_batch",
     ):
         build = build_and_push_image(args.container_repo)
         torchx_image = build.torchx_image
@@ -120,23 +119,6 @@ def main() -> None:
                 "queue": "torchx",
             },
         },
-        "gcp_batch": {
-            "providers": [
-                component_provider,
-            ],
-            "image": torchx_image,
-            "cfg": {},
-        },
-        "ray": {
-            "providers": [
-                component_provider,
-            ],
-            "image": torchx_image,
-            "cfg": {
-                "requirements": "",
-            },
-            "workspace": f"file://{os.getcwd()}",
-        },
         "lsf": {
             "providers": [
                 component_provider,
diff --git a/scripts/gcpbatchint.sh b/scripts/gcpbatchint.sh
diff --git a/scripts/slurmdockerint.sh b/scripts/slurmdockerint.sh
@@ -40,11 +40,11 @@ REMOTE_SCRIPT="$DIR/$(basename "$SCRIPT")"
 
 for host in c1 c2
 do
-  docker exec "$host" dnf install python39 -y
+  docker exec "$host" dnf install python3.11 -y
 done
 
 run_cmd mkdir -p "$DIR"
-run_cmd python3.9 -m venv "$VENV"
+run_cmd python3.11 -m venv "$VENV"
 run_scp "$WHEEL" "$REMOTE_WHEEL"
 run_scp "$SCRIPT" "$REMOTE_SCRIPT"
 run_cmd "$REMOTE_SCRIPT" "$REMOTE_WHEEL" "$VENV"
diff --git a/scripts/slurmtest.sh b/scripts/slurmtest.sh
@@ -32,7 +32,7 @@ python --version
 pip install "$REMOTE_WHEEL"
 pip install numpy
 pip install tabulate
-pip install torch==1.10.2+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html
+pip install torch
 
 PARTITION="$(sinfo --format=%R --noheader | head -n 1)"
 
diff --git a/setup.py b/setup.py
@@ -82,11 +82,6 @@ def get_nightly_version():
         },
         extras_require={
             "aws_batch": ["boto3"],
-            "gcp_batch": [
-                "google-cloud-batch>=0.5.0",
-                "google-cloud-logging>=3.0.0",
-                "google-cloud-runtimeconfig>=0.33.2",
-            ],
             "kubernetes": ["kubernetes>=11"],
             "dev": dev_reqs,
         },
diff --git a/torchx/cli/cmd_list.py b/torchx/cli/cmd_list.py
@@ -33,8 +33,7 @@ def add_arguments(self, subparser: argparse.ArgumentParser) -> None:
             type=str,
             default=get_default_scheduler_name(),
             choices=list(scheduler_names),
-            help=f"Name of the scheduler to use. One of: [{','.join(scheduler_names)}]."
-            " For listing app handles for ray scheduler, RAY_ADDRESS env variable should be set.",
+            help=f"Name of the scheduler to use. One of: [{','.join(scheduler_names)}].",
         )
 
     def run(self, args: argparse.Namespace) -> None:
diff --git a/torchx/components/integration_tests/component_provider.py b/torchx/components/integration_tests/component_provider.py
@@ -109,7 +109,7 @@ def __init__(self, image: str, scheduler: str) -> None:
         self._dst_path = "<None>"
 
     def setUp(self) -> None:
-        if self._scheduler in ["local_cwd", "ray"]:
+        if self._scheduler in ["local_cwd"]:
             fname = "torchx_copy_test.txt"
             self._src_path: str = os.path.join(tempfile.gettempdir(), fname)
             self._dst_path: str = os.path.join(tempfile.gettempdir(), f"{fname}.copy")
@@ -126,7 +126,7 @@ def _process_local_sched(self) -> None:
     def tearDown(self) -> None:
         if os.path.exists(self._dst_path):
             os.remove(self._dst_path)
-        if self._scheduler in ["local_cwd", "ray"] and os.path.exists(self._dst_path):
+        if self._scheduler in ["local_cwd"] and os.path.exists(self._dst_path):
             os.remove(self._dst_path)
 
     def get_app_def(self) -> AppDef:
diff --git a/torchx/schedulers/__init__.py b/torchx/schedulers/__init__.py
@@ -21,7 +21,6 @@
     "kubernetes_mcad": "torchx.schedulers.kubernetes_mcad_scheduler",
     "aws_batch": "torchx.schedulers.aws_batch_scheduler",
     "aws_sagemaker": "torchx.schedulers.aws_sagemaker_scheduler",
-    "gcp_batch": "torchx.schedulers.gcp_batch_scheduler",
     "lsf": "torchx.schedulers.lsf_scheduler",
 }
 
diff --git a/torchx/schedulers/gcp_batch_scheduler.py b/torchx/schedulers/gcp_batch_scheduler.py
diff --git a/torchx/schedulers/test/gcp_batch_scheduler_test.py b/torchx/schedulers/test/gcp_batch_scheduler_test.py
diff --git a/torchx/util/strings.py b/torchx/util/strings.py
