update AMI and pin dependency version (#1268) (#1274)

facebookresearch · Jan 6, 2021 · 9331ca7 · 9331ca7
1 parent 466f2ae
commit 9331ca7
Show file tree

Hide file tree

Showing 5 changed files with 51 additions and 58 deletions.
diff --git a/plugins/hydra_ray_launcher/integration_test_tools/create_integration_test_ami.py b/plugins/hydra_ray_launcher/integration_test_tools/create_integration_test_ami.py
@@ -49,11 +49,23 @@ def set_up_machine(cfg: DictConfig) -> None:
         _run_command(
             f"ray rsync_up {yaml} './setup_integration_test_ami.py' '/home/ubuntu/' "
         )
-
-        print(
-            "Installing dependencies now, this may take a while (very likely more than 20 mins) ..."
-        )
-        _run_command(f"ray exec {yaml} 'python ./setup_integration_test_ami.py' ")
+        _run_command("conda update --all -y")
+        output = _run_command("conda search python").split("\n")
+
+        # gather all the python versions and install conda envs
+        versions = set()
+        for o in output:
+            o = o.split()
+            if len(o) > 2 and o[0] == "python" and float(o[1][:3]) >= 3.6:
+
+                versions.add(o[1])
+        print(f"Found python versions {sorted( versions )}")
+
+        for v in versions:
+            print(f"Setting up conda env for py version {v}")
+            _run_command(
+                f"ray exec {yaml} 'python ./setup_integration_test_ami.py {v}' "
+            )
 
         # remove security group egress rules
         _run_command(

diff --git a/plugins/hydra_ray_launcher/integration_test_tools/create_integration_test_ami_config.yaml b/plugins/hydra_ray_launcher/integration_test_tools/create_integration_test_ami_config.yaml
@@ -1,4 +1,4 @@
-security_group_id: ${env:RAY_BUILD_AMI_SEC_GROUP}
+security_group_id: sg-095ac7c26aa0d33bb
 ray_yaml:
   cluster_name: ray_test_base_AMI
   min_workers: 0
@@ -25,19 +25,19 @@ ray_yaml:
   head_node:
     InstanceType: m5.large
     ImageId: ami-008d8ed4bd7dc2485
-    SubnetId: ${env:AWS_RAY_SUBNET}
+    SubnetId: subnet-acd2cfe7
     SecurityGroupIds:
-      - ${env:AWS_RAY_SECURITY_GROUP}
+      - sg-0a12b09a5ff961aee
     IamInstanceProfile:
-      Arn: ${env:INSTANCE_ROLE_ARN}
+      Arn: arn:aws:iam::135937774131:instance-profile/ray-autoscaler-v1
   worker_nodes:
     InstanceType: m5.large
     ImageId: ami-008d8ed4bd7dc2485
-    SubnetId: ${env:AWS_RAY_SUBNET}
+    SubnetId: subnet-acd2cfe7
     SecurityGroupIds:
-      - ${env:AWS_RAY_SECURITY_GROUP}
+      - sg-0a12b09a5ff961aee
     IamInstanceProfile:
-      Arn: ${env:INSTANCE_ROLE_ARN}
+      Arn: arn:aws:iam::135937774131:instance-profile/ray-autoscaler-v1
     InstanceMarketOptions:
       MarketType: spot
   file_mounts: {}

diff --git a/plugins/hydra_ray_launcher/integration_test_tools/setup_integration_test_ami.py b/plugins/hydra_ray_launcher/integration_test_tools/setup_integration_test_ami.py
@@ -1,9 +1,14 @@
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
 import subprocess
+import sys
 from datetime import datetime
 
 dependencies = [
-    "ray>=1.0.0",
+    "boto3==1.16.48",
+    "hydra-core>=1.0.0",
+    "ray==1.1.0",
+    "cloudpickle==1.6.0",
+    "pickle5==0.0.11",
 ]
 
 
@@ -14,28 +19,15 @@ def _run_command(command: str) -> str:
     return output
 
 
-def run():
-    _run_command("conda update --all -y")
-    output = _run_command("conda search python").split("\n")
-
-    # gather all the python versions and install conda envs
-    versions = set()
-    for o in output:
-        o = o.split()
-        if len(o) > 2 and o[0] == "python" and float(o[1][:3]) >= 3.6:
-            versions.add(o[1])
-    print(sorted(versions))
+def run(py_version):
 
     _run_command("rm /home/ubuntu/ray_bootstrap_config.yaml")
 
-    # prep conda env for all python versions
-
-    for v in sorted(versions):
-        _run_command(f"conda create -n hydra_{v} python={v} -y")
-        pip_path = f"/home/ubuntu/anaconda3/envs/hydra_{v}/bin/pip"
-        for d in dependencies:
-            _run_command(f"{pip_path} install {d}")
+    _run_command(f"conda create -n hydra_{py_version} python={py_version} -y")
+    pip_path = f"/home/ubuntu/anaconda3/envs/hydra_{py_version}/bin/pip"
+    for d in dependencies:
+        _run_command(f"{pip_path} install {d}")
 
 
 if __name__ == "__main__":
-    run()
+    run(sys.argv[1])
diff --git a/plugins/hydra_ray_launcher/setup.py b/plugins/hydra_ray_launcher/setup.py
@@ -22,11 +22,11 @@
             "Development Status :: 4 - Beta",
         ],
         install_requires=[
-            "boto3==1.15.6",
+            "boto3==1.16.48",
             "hydra-core>=1.0.0",
-            "ray>=1.0.0",
-            "cloudpickle>=1.6.0",
-            "pickle5",
+            "ray==1.1.0",
+            "cloudpickle==1.6.0",
+            "pickle5==0.0.11",
         ],
         include_package_data=True,
     )
diff --git a/plugins/hydra_ray_launcher/tests/test_ray_aws_launcher.py b/plugins/hydra_ray_launcher/tests/test_ray_aws_launcher.py
@@ -40,7 +40,6 @@
 cluster_name = "IntegrationTest-" + "".join(
     [random.choice(string.ascii_letters + string.digits) for n in range(5)]
 )
-
 win_msg = "Ray doesn't support Windows."
 cur_py_version = (
     f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"
@@ -55,7 +54,7 @@
     aws_not_configured = True
 
 
-ami = os.environ.get("AWS_RAY_AMI", "ami-099b85625184ce698")
+ami = os.environ.get("AWS_RAY_AMI", "ami-0e597fd7c6b402fce")
 security_group_id = os.environ.get("AWS_RAY_SECURITY_GROUP", "sg-0a12b09a5ff961aee")
 subnet_id = os.environ.get("AWS_RAY_SUBNET", "subnet-acd2cfe7")
 instance_role = os.environ.get(
@@ -164,27 +163,17 @@ def upload_and_install_wheels(
 def validate_lib_version(yaml: str) -> None:
     # a few lib versions that we care about
     libs = ["ray", "cloudpickle", "pickle5"]
-    local_versions = set()
     for lib in libs:
-        v = pkg_resources.get_distribution(lib).version
-        local_versions.add(f"{lib}=={v}")
-
-    log.info(f"Locally running {local_versions}")
-    out, _ = _run_command(
-        [
-            "ray",
-            "exec",
-            yaml,
-            "pip freeze",
-        ]
-    )
-    remote_versions = out.split()
-    log.info(f"Remotely running {remote_versions}")
-    for local in local_versions:
-        if local not in remote_versions:
-            raise ValueError(
-                f"lib version not matching, local_version: {local} \nremote_versions: {remote_versions}"
-            )
+        local_version = f"{pkg_resources.get_distribution(lib).version}"
+        out, _ = _run_command(
+            [
+                "ray",
+                "exec",
+                yaml,
+                f"pip freeze | grep {lib}",
+            ]
+        )
+        assert local_version in out, f"{lib} version mismatch"
 
     # validate python version
     info = sys.version_info