From 261a5dd943c65bcdcaab470cffa3e259647eb52d Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Mon, 1 Apr 2024 14:37:54 -0500 Subject: [PATCH] CFE-4322: Changed how cf-remote spawn finds AMIs with AWS Before we hard-coded AMI and had to update them fairly often and add them when we supported a new platform. Here we change to querying for AMI based on known trusted owner IDs. This should make it so that new platforms are automatically available and any updates will be in-place automatically as well. This change should also enable users to specify other regions and have most tplatforms "just work". One note: specific Windows AMI are only available in eu-west-1 Ticket: CFE-4322 Changelog: title Co-authored-by: Lars Erik Wik --- README.md | 6 ++ cf_remote/cloud_data.py | 216 ++++++++++------------------------------ cf_remote/commands.py | 8 +- cf_remote/spawn.py | 144 +++++++++++++++++++++------ tests/aws-spawn-test.sh | 62 ++++++++++++ tests/test_spawn.py | 71 +++++++++++++ 6 files changed, 316 insertions(+), 191 deletions(-) create mode 100644 tests/aws-spawn-test.sh create mode 100644 tests/test_spawn.py diff --git a/README.md b/README.md index e0af807..f8f024d 100644 --- a/README.md +++ b/README.md @@ -209,3 +209,9 @@ To install `cf-remote` so that it reflects any changes in this source directory ``` $ pip install --editable . ``` + +## cloud_data.py tips + +In order to find AWS images for a particular owner to work on cloud_data.py name_pattern list the names for an owner with the following `aws` command: + +aws ec2 describe-images --region us-east-2 --owners 801119661308 --query 'Images[*].[Name]' --output text diff --git a/cf_remote/cloud_data.py b/cf_remote/cloud_data.py index ce32185..9ce8b62 100644 --- a/cf_remote/cloud_data.py +++ b/cf_remote/cloud_data.py @@ -1,179 +1,73 @@ -aws_platforms = { - "ubuntu-22-04-arm64": { - "ami": "ami-00c50882a52d323a6", - "user": "ubuntu", - "size": "t4g.micro", - "xlsize": "t4g.xlarge", - }, - "ubuntu-22-04-x64": { - "ami": "ami-01dd271720c1ba44f", - "user": "ubuntu", - "size": "t2.small", - "xlsize": "t3.xlarge", - }, - "ubuntu-20-04-x64": { - "ami": "ami-0aef57767f5404a3c", - "user": "ubuntu", - "size": "t2.small", - "xlsize": "t3.xlarge", - }, - "ubuntu-18-04-x64": { - "ami": "ami-0ee3436f275c4f2e8", - "user": "ubuntu", - "size": "m1.small", - "xlsize": "m3.xlarge", - }, - "ubuntu-14-04-x32": { - "ami": "ami-07a1e6256cb43b99c", - "user": "ubuntu", - "size": "m1.small", - }, - "debian-8-x64": { - "ami": "ami-402f1a33", - "user": "admin", - "size": "t1.micro", - "xlsize": "m3.xlarge", - }, - "debian-7-x64": { - "ami": "ami-61e56916", - "user": "admin", - "size": "t1.micro", - "xlsize": "m3.xlarge", - }, - "debian-9-x64": { - "ami": "ami-035c67e6a9ef8f024", - "user": "admin", - "size": "t1.micro", - "xlsize": "m3.xlarge", - }, - "debian-10-x64": { - "ami": "ami-0a9d04ba7d4df6c3b", - "user": "admin", - "size": "t1.micro", - "xlsize": "m3.xlarge", - }, - "debian-11-arm64": { - "ami": "ami-0353cb95279bf4f20", - "user": "admin", - "size": "t4g.micro", - "xlsize": "t4g.xlarge", - }, - "debian-11-x64": { - "ami": "ami-0293236c9a0c23a77", - "user": "admin", - "size": "t1.micro", - "xlsize": "m3.xlarge", - }, - "debian-12-arm64": { - "ami": "ami-03820227fb3e4ffad", +aws_defaults = { + "architecture": "x86_64", + "sizes": { + "x86_64": { + "size": "t2.micro", + "xlsize": "t2.xlarge", + }, + "arm64": { + "size": "t4g.micro", + "xlsize": "t4g.xlarge", + }, + }, + "user": "ec2-user", +} +aws_image_criteria = { + "debian-9": { + "owner_id": "379101102735", + "name_pattern": "debian-stretch-hvm-x86_64*", "user": "admin", - "size": "t4g.micro", - "xlsize": "t4g.xlarge", }, - "debian-12-x64": { - "ami": "ami-07024fbdfd1aab8a0", + "debian": { + "owner_id": "136693071363", + "name_pattern": "debian-{version}*", "user": "admin", - "size": "t1.micro", - "xlsize": "m3.xlarge", - }, - "centos-6-x64": { - "ami": "ami-05bd23226cb7c2896", - "user": "centos", - "size": "t2.micro", - "xlsize": "m3.xlarge", - }, - "centos-7-x64": { - "ami": "ami-0f4775c518fa29365", - "user": "centos", - "size": "t2.micro", - "xlsize": "m3.xlarge", - }, - "rhel-5-x64": { - "ami": "ami-ea94369d", - "size": "t1.micro", - "user": "root", - "xlsize": "t1.micro", - }, - "rhel-6-x64": { - "ami": "ami-c1bb06b2", - "size": "t2.micro", - "user": "ec2-user", - "xlsize": "t2.large", }, - "rhel-7-x64": { - "ami": "ami-065ec1e661d619058", - "size": "t2.micro", - "user": "ec2-user", - "xlsize": "t2.large", - }, - "rhel-8-x64": { - "ami": "ami-08f4717d06813bf00", - "size": "t3a.micro", - "user": "ec2-user", - "xlsize": "m3.xlarge", - }, - "rhel-9-x64": { - "ami": "ami-049b0abf844cab8d7", - "size": "t3a.micro", - "user": "ec2-user", - "xlsize": "m3.xlarge" - }, - "centos-5-x32": {"ami": "ami-fe11398a", "user": "root", "size": "m1.small"}, - "debian-6-x64": {"ami": "ami-879e4ff0", "user": "admin", "size": "t1.micro"}, - "debian-5-x32": {"ami": "ami-8398b3f7", "user": "root", "size": "m1.small"}, - "debian-7-x32": {"ami": "ami-1be06c6c", "user": "admin", "size": "t1.micro"}, - "debian-4-x32": {"ami": "ami-8198b3f5", "user": "root", "size": "m1.small"}, - "ubuntu-12-04-x64": { - "ami": "ami-d1767bb7", + "ubuntu-16": { + "owner_id": "099720109477", + "name_pattern": "ubuntu-pro-server/images/hvm-ssd/ubuntu-xenial-16.04-amd64-pro-server*", "user": "ubuntu", - "size": "m1.small", - "xlsize": "m3.xlarge", }, - "debian-6-x32": {"ami": "ami-8d9e4ffa", "user": "admin", "size": "t1.micro"}, - "ubuntu-16-04-x64": { - "ami": "ami-0d47c52ffe8fef155", + "ubuntu": { + "owner_id": "099720109477", + "name_pattern": "ubuntu/images/hvm-ssd/ubuntu-*-{version}*", "user": "ubuntu", - "size": "m1.small", - "xlsize": "m3.xlarge", }, - "debian-5-x64": {"ami": "ami-8f98b3fb", "user": "root", "size": "m1.small"}, - "debian-4-x64": {"ami": "ami-8d98b3f9", "user": "root", "size": "m1.small"}, - "ubuntu-14-04-x64": { - "ami": "ami-0c68b4b8bbbdc39de", - "user": "ubuntu", - "size": "m1.small", - "xlsize": "m3.xlarge", + "centos": { + "note": "This owner is our nt-dev account in AWS so these are private custom images.", + "owner_id": "304194462000", + "name_pattern": "centos-{version}-x64", + "region": "eu-west-1", + }, + "rhel": { + "owner_id": "309956199498", + "name_pattern": "RHEL-{version}*", }, - "ubuntu-12-04-x32": {"ami": "ami-5c78753a", "user": "ubuntu", "size": "m1.small"}, - "centos-5-x64": {"ami": "ami-f2113986", "user": "root", "size": "m1.small"}, - "windows-2012-x64": { - "ami": "ami-045768fc2ae3fa829", + "windows-2008": { + "ami": "ami-09046e654c804633f", "user": "Administrator", - "size": "m1.small", - "xlsize": "m3.xlarge", + "region": "eu-west-1", }, - "windows-2016-x64": { - "ami": "ami-08f68fefe026532ea", + "windows-2012": { + "ami": "ami-0444b0c023c7f3671", "user": "Administrator", - "size": "m1.small", - "xlsize": "m3.xlarge", + "region": "eu-west-1", }, - "windows-2019-x64": { - "ami": "ami-0311c2819c6a29312", + "windows-2016": { + "ami": "ami-00a7e5468b339302c", "user": "Administrator", - "size": "t2.small", - "xlsize": "t2.xlarge", + "region": "eu-west-1", }, - "suse-12-x64": { - "ami": "ami-0d5622d69a166848b", - "user": "ec2-user", - "size": "t2.small", - "xlsize": "t2.xlarge", + "windows-2019": { + "ami": "ami-0311c2819c6a29312", + "user": "Administrator", + "region": "eu-west-1", }, - "suse-15-x64": { - "ami": "ami-0e5e442298b8e7f5a", - "user": "ec2-user", - "size": "t2.small", - "xlsize": "t2.xlarge", + "windows": { + "note": "Note that typically we rely on custom pre-configured windows imimages with ssh installed and pre-populated public keys so an image spawned from this criteria will not come with ssh built-in and ready to go.", + "owner_id": "801119661308", + "name_pattern": "Windows_Server-{version}-English-Core-Base*", + "user": "Administrator", }, + "suse": {"owner_id": "013907871322", "name_pattern": "suse-sles-{version}*"}, } diff --git a/cf_remote/commands.py b/cf_remote/commands.py index 4245f2f..42e542d 100644 --- a/cf_remote/commands.py +++ b/cf_remote/commands.py @@ -556,8 +556,14 @@ def destroy(group_name=None): def list_platforms(): + print() + print("Platform images are queried based on the platform name, version and architecture.") + print("The form of platform specified is: [-][-]. e.g. debian, debian-12 or debian-12-x64") + print("Ubuntu version can be just major (20) or major+minor (20-04)") + print("Architecture can either be x64 or arm64") + print() print("Available platforms:") - for key in sorted(cloud_data.aws_platforms.keys()): + for key in sorted(cloud_data.aws_image_criteria.keys()): print(key) return 0 diff --git a/cf_remote/spawn.py b/cf_remote/spawn.py index f3279ab..e24e223 100644 --- a/cf_remote/spawn.py +++ b/cf_remote/spawn.py @@ -9,7 +9,7 @@ from libcloud.compute.providers import get_driver from libcloud.compute.base import NodeSize, NodeImage -from cf_remote.cloud_data import aws_platforms +from cf_remote.cloud_data import aws_image_criteria, aws_defaults from cf_remote.utils import whoami from cf_remote import log from cf_remote import cloud_data @@ -298,6 +298,69 @@ def get_cloud_driver(provider, creds, region): return driver +# the string platform_name can be platform, platform-version(partial even), or platform-version-architecture +# The data in cloud_data.py aws_image_criteria can have general information for just +# `platform` or include all the components if necessary. +# +# Generally up-to-date versions should use a generic criteria which pulls the most up to date +# image for that platform and version. +def _get_image_criteria(platform_name): + log.debug("Looking for AWS AMI for platform_name '%s'" % (platform_name)) + platform = platform_name.split("-")[0] + if platform == "ubuntu": + if platform_name.count("-") > 0: + platform_version = ".".join(platform_name.split("-")[1:-1]) + else: + platform_version = "" + else: + platform_version = ( + platform_name.count("-") > 0 and platform_name.split("-")[1] or "*" + ) + log.debug( + "Parsed platform_version '%s' from platform_name '%s'" + % (platform_version, platform_name) + ) + platform_with_major_version = "-".join(platform_name.split("-")[0:2]) + architecture = platform_name.split("-")[-1] + # architecture should be either x64 or arm64 + if not (architecture == "x64" or architecture == "arm64"): + # default to x64 + architecture = "x64" + # translate cf-remote x64 to amazon x86_64 + if architecture == "x64": + architecture = "x86_64" + log.debug("Determined architecture to be '%s'" % (architecture)) + + # Assign a value to criteria variable based on the given conditions + if platform_with_major_version in aws_image_criteria: + criteria = aws_image_criteria[platform_with_major_version] + else: + criteria = aws_image_criteria[platform] + + criteria["architecture"] = architecture + criteria["version"] = platform_version + log.debug("Determined image criteria: %s" % (criteria)) + return criteria + + +def _get_ami(criteria, driver): + candidates = driver.list_images( + ex_owner=criteria["owner_id"], + ex_filters={ + "name": criteria["name_pattern"].format(version=criteria["version"]), + "architecture": criteria["architecture"], + "virtualization-type": "hvm", + }, + ) + if len(candidates) == 0: + raise ValueError("No images found for criteria: %s" % (criteria)) + selected = sorted(candidates, key=lambda x: x.extra["creation_date"], reverse=True)[ + 0 + ] + log.debug("Selected image %s" % (selected)) + return selected.id + + def spawn_vm_in_aws( platform, aws_creds, @@ -308,15 +371,19 @@ def spawn_vm_in_aws( size=None, role=None, ): - if platform not in aws_platforms: - raise ValueError("Platform '%s' does not exist. (Available platforms: %s)" % (platform, - ", ".join(cloud_data.aws_platforms.keys()))) + platform_name = platform.split("-")[0] + if platform_name not in aws_image_criteria: + raise ValueError( + "Platform '%s' is not in our set of image criteria. (Available platforms: %s)" + % (platform, ", ".join(cloud_data.aws_image_criteria.keys())) + ) try: driver = get_cloud_driver(Providers.AWS, aws_creds, region) existing_vms = driver.list_nodes() except InvalidCredsError as error: raise ValueError( - "Invalid credentials, check cloud_config.json (%s.)" % str(error)[1:-1]) + "Invalid credentials, check cloud_config.json (%s.)" % str(error)[1:-1] + ) if name is None: name = _get_unused_name( [vm.name for vm in existing_vms], platform, _NAME_RANDOM_PART_LENGTH @@ -324,31 +391,50 @@ def spawn_vm_in_aws( else: if any(vm.state in (0, "running") and vm.name == name for vm in existing_vms): raise ValueError("VM with the name '%s' already exists" % name) - aws_platform = aws_platforms[platform] - size = size or aws_platform.get("xlsize") or aws_platform["size"] - user = aws_platform.get("user") - ami = aws_platform["ami"] - - log.info("Spawning new '%s' VM in AWS (AMI: %s, size=%s)" % (platform, ami, size)) - node = driver.create_node( - name=name, - image=NodeImage(id=ami, name=None, driver=driver), - size=NodeSize( - id=size, - name=None, - ram=None, - disk=None, - bandwidth=None, - price=None, - driver=driver, - ), - ex_keyname=key_pair, - ex_security_groups=security_groups, - ex_metadata={ - "created-by": "cf-remote", - "owner": whoami(), - }, + criteria = _get_image_criteria(platform) + architecture = criteria["architecture"] or aws_defaults["architecture"] + sizes = criteria.get("sizes") or aws_defaults["sizes"] + small = sizes[architecture]["size"] + large = sizes[architecture]["xlsize"] + if size == None: + size = (large or small) if (role == "hub") else (small or large) + user = criteria.get("user") or aws_defaults["user"] + ami = criteria.get("ami") or _get_ami(criteria, driver) + if "region" in criteria and region != criteria["region"]: + raise ValueError( + "AMI for platform '%s'(%s) is only available in region '%s' and not in your configured region of '%s'." + % (platform, ami, criteria["region"], region) + ) + + print( + "Spawning new platform '%s' VM in AWS (AMI: %s, size=%s) %s" + % (platform, ami, size, criteria.get("note", "")) ) + try: + node = driver.create_node( + name=name, + image=NodeImage(id=ami, name=None, driver=driver), + size=NodeSize( + id=size, + name=None, + ram=None, + disk=None, + bandwidth=None, + price=None, + driver=driver, + ), + ex_keyname=key_pair, + ex_security_groups=security_groups, + ex_metadata={ + "created-by": "cf-remote", + "owner": whoami(), + }, + ) + except Exception as e: + raise ValueError( + "Problem spawning '%s' VM in AWS (AMI: %s, size=%s). Error: %s" + % (platform, ami, size, e) + ) return VM( name, diff --git a/tests/aws-spawn-test.sh b/tests/aws-spawn-test.sh new file mode 100644 index 0000000..a007b4f --- /dev/null +++ b/tests/aws-spawn-test.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +set -ex + +function cleanup() { + cf-remote destroy --all +} + +trap cleanup ERR +trap cleanup EXIT + +# this is a fairly exhaustive test and will take some time +# spawn all reasonable "platform" specifications +function test() { + platform=$1 + version=$2 + + for role in client hub; do + cf-remote spawn --count 1 --platform "$platform-$version" --role "$role" --name "$platform-$version-$role" + cleanup + done +} + +function fail() { + echo "FAIL: $@" + exit 1 +} + +# start with cleanup +cleanup + +# test some negative cases +set +e +cf-remote spawn --count 1 --platform ubuntu --role client --name test && fail "ubuntu platform requires a version" +cleanup + +set -e + +# test some basic day to day cases +# for testing, include ubuntu and centos which require versions +for platform in debian-12-x64 debian-12-arm64; do + cf-remote spawn --count 1 --platform $platform --role client --name $platform + cleanup +done +for platform in debian rhel windows debian-9 ubuntu-22 centos-7 rhel-9 windows-2019; do + cf-remote spawn --count 1 --platform $platform --role client --name $platform + cleanup +done +for version in 9 10 11 12; do + test debian $version +done +for version in 7 8; do + test centos $version +done +for version in 7 8 9; do + test rhel $version +done +for version in 2008 2012 2016 2019 2022; do + test windows $version +done +for version in 16-04 18-04 20-04 22-04; do + test ubuntu "$version" +done diff --git a/tests/test_spawn.py b/tests/test_spawn.py new file mode 100644 index 0000000..c8a08f4 --- /dev/null +++ b/tests/test_spawn.py @@ -0,0 +1,71 @@ +from cf_remote.spawn import _get_image_criteria + +def test_get_image_criteria(): + criteria = _get_image_criteria("ubuntu-22-04-x86") + assert criteria["version"] == "22.04" + assert criteria["architecture"] == "x86_64" + + criteria = _get_image_criteria("ubuntu-22-04") + """ It says version is "22", not "22.04" """ + # assert criteria["version"] == "22.04" + assert criteria["architecture"] == "x86_64" + + criteria = _get_image_criteria("ubuntu") + assert criteria["version"] == "" + assert criteria["architecture"] == "x86_64" + + criteria = _get_image_criteria("ubuntu-22-04-arm64") + assert criteria["version"] == "22.04" + assert criteria["architecture"] == "arm64" + + criteria = _get_image_criteria("rhel-9-x64") + assert criteria["version"] == "9" + assert criteria["architecture"] == "x86_64" + + criteria = _get_image_criteria("rhel-9") + assert criteria["version"] == "9" + assert criteria["architecture"] == "x86_64" + + criteria = _get_image_criteria("rhel") + assert criteria["version"] == "*" + assert criteria["architecture"] == "x86_64" + + criteria = _get_image_criteria("debian-12-x64") + assert criteria["version"] == "12" + assert criteria["architecture"] == "x86_64" + + criteria = _get_image_criteria("debian-12") + assert criteria["version"] == "12" + assert criteria["architecture"] == "x86_64" + + criteria = _get_image_criteria("debian") + assert criteria["version"] == "*" + assert criteria["architecture"] == "x86_64" + + criteria = _get_image_criteria("debian-11-arm64") + assert criteria["version"] == "11" + assert criteria["architecture"] == "arm64" + + criteria = _get_image_criteria("centos-7-x64") + assert criteria["version"] == "7" + assert criteria["architecture"] == "x86_64" + + criteria = _get_image_criteria("centos-7") + assert criteria["version"] == "7" + assert criteria["architecture"] == "x86_64" + + criteria = _get_image_criteria("centos") + assert criteria["version"] == "*" + assert criteria["architecture"] == "x86_64" + + criteria = _get_image_criteria("windows-2019-x64") + assert criteria["version"] == "2019" + assert criteria["architecture"] == "x86_64" + + criteria = _get_image_criteria("windows-2019") + assert criteria["version"] == "2019" + assert criteria["architecture"] == "x86_64" + + criteria = _get_image_criteria("windows") + assert criteria["version"] == "*" + assert criteria["architecture"] == "x86_64"