From 1d8cbd362e46e2e4e3602ae76f6dabfeed35b680 Mon Sep 17 00:00:00 2001 From: Alex Kennedy Date: Tue, 11 Feb 2025 16:38:35 -0800 Subject: [PATCH] chore: Remove tornado_rest_client based actors --- .github/workflows/main-release.yaml | 2 +- .github/workflows/publish-release.yaml | 2 +- .github/workflows/test.yaml | 2 +- kingpin/actors/packagecloud.py | 486 ------------- kingpin/actors/pingdom.py | 184 ----- kingpin/actors/slack.py | 186 ----- kingpin/actors/spotinst.py | 879 ----------------------- kingpin/actors/test/test_packagecloud.py | 397 ---------- kingpin/actors/test/test_pingdom.py | 119 --- kingpin/actors/test/test_slack.py | 183 ----- kingpin/actors/test/test_spotinst.py | 528 -------------- kingpin/version.py | 2 +- requirements.txt | 7 - 13 files changed, 4 insertions(+), 2973 deletions(-) delete mode 100644 kingpin/actors/packagecloud.py delete mode 100644 kingpin/actors/pingdom.py delete mode 100644 kingpin/actors/slack.py delete mode 100644 kingpin/actors/spotinst.py delete mode 100644 kingpin/actors/test/test_packagecloud.py delete mode 100644 kingpin/actors/test/test_pingdom.py delete mode 100644 kingpin/actors/test/test_slack.py delete mode 100644 kingpin/actors/test/test_spotinst.py diff --git a/.github/workflows/main-release.yaml b/.github/workflows/main-release.yaml index a7f77095..c2c463bf 100644 --- a/.github/workflows/main-release.yaml +++ b/.github/workflows/main-release.yaml @@ -11,7 +11,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.x' + python-version: '3.12' - run: make venv shell: bash - run: make lint diff --git a/.github/workflows/publish-release.yaml b/.github/workflows/publish-release.yaml index 91e08aac..4fd271dc 100644 --- a/.github/workflows/publish-release.yaml +++ b/.github/workflows/publish-release.yaml @@ -11,7 +11,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.x' + python-version: '3.12' - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 4d09bd93..5a383f98 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -9,7 +9,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.x' + python-version: '3.12' - run: make venv shell: bash - run: make lint diff --git a/kingpin/actors/packagecloud.py b/kingpin/actors/packagecloud.py deleted file mode 100644 index 1a2cd8be..00000000 --- a/kingpin/actors/packagecloud.py +++ /dev/null @@ -1,486 +0,0 @@ -""" -:mod:`kingpin.actors.packagecloud` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The packagecloud actor allows you to perform maintenance operations on -repositories hosted by packagecloud.io using their API: - -https://packagecloud.io/docs/api - -**Required Environment Variables** - -:PACKAGECLOUD_ACCOUNT: - packagecloud account name, i.e. https://packagecloud.io/PACKAGECLOUD_ACCOUNT - -:PACKAGECLOUD_TOKEN: - packagecloud API Token -""" - -import datetime -import logging -import os -import re -import sys - -from tornado import gen - -from tornado_rest_client import api - -from kingpin.actors import base -from kingpin.actors import exceptions -from kingpin.constants import REQUIRED - -log = logging.getLogger(__name__) - -__author__ = "Matt Wise " - -ACCOUNT = os.getenv("PACKAGECLOUD_ACCOUNT", None) -TOKEN = os.getenv("PACKAGECLOUD_TOKEN", None) - - -class PackagecloudAPI(api.RestConsumer): - - ENDPOINT = "https://packagecloud.io/api/v1/" - CONFIG = { - "attrs": { - "packages": { - "path": ( - "repos/%account%/%repo%/packages.json" - "?per_page={}".format(sys.maxsize) - ), - "http_methods": {"get": {}}, - }, - "delete": { - "path": "repos/%account%/%repo%/%distro_version%/%filename%", - "http_methods": {"delete": {}}, - }, - }, - "auth": {"user": TOKEN, "pass": ""}, - } - - -class PackagecloudBase(base.BaseActor): - """Simple packagecloud Abstract Base Object""" - - def __init__(self, *args, **kwargs): - """Check required environment variables.""" - super(PackagecloudBase, self).__init__(*args, **kwargs) - - if not ACCOUNT: - raise exceptions.InvalidCredentials( - 'Missing the "PACKAGECLOUD_ACCOUNT" environment variable.' - ) - - if not TOKEN: - raise exceptions.InvalidCredentials( - 'Missing the "PACKAGECLOUD_TOKEN" environment variable.' - ) - - rest_client = api.RestClient(timeout=120) - self._packagecloud_client = PackagecloudAPI(client=rest_client) - - @gen.coroutine - def _get_all_packages(self, repo): - """Simple method for fetching a dictionary of all packages in a repo - - Args: - repo: name of the packagecloud repo to fetch from - - Returns: - A hash of the packages. - """ - packages = yield self._packagecloud_client.packages( - token=TOKEN, account=ACCOUNT, repo=repo - ).http_get() - raise gen.Return(packages) - - def _get_package_versions(self, name, packages): - """Find all versions of a given package. - - Args: - name: name of the package to look for - packages: hash of all the packages, as returned by the API - - Returns: - A hash of package versions sorted by creation date - """ - versions = [ - { - "created_at": datetime.datetime.strptime( - package["created_at"], "%Y-%m-%dT%H:%M:%S.%fZ" - ), - "distro_version": package["distro_version"], - "filename": package["package_html_url"].split("/")[-1], - "name": package["name"], - } - for package in packages - if package["name"] == name - ] - - versions.sort(key=lambda x: x.get("created_at"), reverse=False) - return versions - - def _filter_packages(self, regex, packages): - """Extracts a list of unique package names to delete - - Args: - regex: regex of package names to delete - packages: hash of all the packages, as returned by the API - - Returns: - A list of unique package names that match the delete pattern. - """ - pattern = re.compile(regex) - packages_list_to_delete = { - package["name"] for package in packages if pattern.match(package["name"]) - } - - self.log.debug( - "List of packages matching regex (%s): %s" - % (regex, packages_list_to_delete) - ) - - return packages_list_to_delete - - @gen.coroutine - def _delete(self, regex, repo, older_than=0, number_to_keep=0): - """Generic packagecloud delete method, optionally supporting deleting - old packages by date and/or keeping a certain number of packages. - - Args: - regex: Regex of packages to delete, e.g. pkg1|pkg2 - repo: name of the packagecloud repo to delete from - older_than: Delete packages created before this number of seconds - number_to_keep: Keep at least this number of each package - - Returns: - A list of the packages that were deleted - """ - all_packages = yield self._get_all_packages(repo=repo) - packages_list_to_delete = self._filter_packages(regex, all_packages) - all_packages_deleted = [] - - # Loop through each unique package to delete - for name in packages_list_to_delete: - package_versions = self._get_package_versions(name, all_packages) - - # Create a tally of the packages we delete -- usd to give the user - # a final helpful log statement about the work we did. - packages_deleted = [] - - # Get a total count of the number of versions of this package in - # the repo -- this variable will then be counted down as we loop, - # to prevent us from deleting more than 'number_to_keep' packages. - number_in_repo = len(package_versions) - self.log.debug("Scanning %s versions (%s)" % (name, number_in_repo)) - - for package in package_versions: - # Safety check -- if there aren't more than the number_to_keep - # in the repo, then don't bother continuing through the loop - # for this package. Break out and move to the next name in - # packages_list_to_delete. - if number_in_repo <= number_to_keep: - self.log.debug( - "%s has only %s package versions left, skipping" - % (name, number_in_repo) - ) - break - - # If older_than (time in seconds) was supplied, figure out how - # old the package is. If the package_age (in seconds) is - # younger than allowed_age (in seconds), then skip to the next - # package version in the set. - if older_than: - package_age = datetime.datetime.now() - package["created_at"] - allowed_age = datetime.timedelta(seconds=older_than) - if package_age <= allowed_age: - self.log.debug( - "%s/%s is only %s old, skipping deletion" - % (package["distro_version"], package["name"], package_age) - ) - continue - - # Finally if we got here, then we have enough packages left in - # the repo, AND (optionally) this package is older than our - # cutoff age... so delete the package. - msg = "%s/%s/%s" % ( - repo, - package["distro_version"], - package["filename"], - ) - if self._dry: - self.log.info("Would have deleted %s" % msg) - else: - self.log.info("Deleting %s" % msg) - - yield self._packagecloud_client.delete( - token=TOKEN, - account=ACCOUNT, - repo=repo, - distro_version=package["distro_version"], - filename=package["filename"], - ).http_delete() - - # Decrement list of packages to track how many are left - number_in_repo = number_in_repo - 1 - - # Track *every* package we delete in one big dict -- this is - # used purely for the unit tests to validate which packages - # were deleted. - all_packages_deleted.append(package) - - # Track that this package was deleted -- used in the parent for - # loop to give the user a final tally of the packages that - # were kept, and that were deleted. - packages_deleted.append(package) - - # Print out the packages that were not deleted and left in the repo - all_files = [ - "%s/%s" % (package["distro_version"], package["filename"]) - for package in package_versions - ] - deleted_files = [ - "%s/%s" % (package["distro_version"], package["filename"]) - for package in packages_deleted - ] - files_left = list(set(all_files) - set(deleted_files)) - self.log.debug("%s remaining packages: %s" % (name, files_left)) - - raise gen.Return(all_packages_deleted) - - -class Delete(PackagecloudBase): - """Deletes packages from a PackageCloud repo. - - Searches for packages that match the `packages_to_delete` regex pattern and - deletes them. If `number_to_keep` is set, we always at least this number of - versions of the given package intact in the repo. Also if `number_to_keep` - is set, the older versions of a package (based on upload time) packages will - be deleted first effectively leaving newer packages in the repo. - - **Options** - - :number_to_keep: - Keep at least this number of each package - (defaults to *0*) - - :packages_to_delete: - Regex of packages to delete, e.g. pkg1|pkg2 - - :repo: - Which packagecloud repo to delete from - - **Examples** - - .. code-block:: json - - { - "desc": "packagecloud Delete example", - "actor": "packagecloud.Delete", - "options": { - "number_to_keep": 10, - "packages_to_delete": "deleteme", - "repo": "test" - } - } - """ - - all_options = { - "number_to_keep": (int, 0, "Keep at least this number of each package"), - "packages_to_delete": ( - str, - REQUIRED, - "Regex of packages to delete, e.g. pkg1|pkg2", - ), - "repo": (str, REQUIRED, "Which packagecloud repo to delete from"), - } - - desc = "Deleting {repo}/{packages_to_delete} (keeping {number_to_keep})" - - def __init__(self, *args, **kwargs): - """Check required environment variables.""" - super(Delete, self).__init__(*args, **kwargs) - - try: - re.compile(self.option("packages_to_delete")) - except re.error: - raise exceptions.InvalidOptions("packages_to_delete is an invalid regex") - - @gen.coroutine - def _execute(self): - """Deletes all packages that match the `packages_to_delete` pattern""" - yield self._delete( - regex=self.option("packages_to_delete"), - number_to_keep=self.option("number_to_keep"), - repo=self.option("repo"), - ) - - -class DeleteByDate(PackagecloudBase): - """Deletes packages from a PackageCloud repo older than X. - - Adds additional functionality to the `Delete` class with a `older_than` - option. Only packages older than that number of seconds will be deleted. - - **Options** - - :number_to_keep: - Keep at least this number of each package - (defaults to *0*) - - :older_than: - Delete packages created before this number of seconds - - :packages_to_delete: - Regex of packages to delete, e.g. pkg1|pkg2 - - :repo: - Which packagecloud repo to delete from - - **Examples** - - .. code-block:: json - - { - "desc": "packagecloud DeleteByDate example", - "actor": "packagecloud.DeleteByDate", - "options": { - "number_to_keep": 10, - "older_than": 600, - "packages_to_delete": "deleteme", - "repo": "test" - } - } - """ - - all_options = { - "number_to_keep": (int, 0, "Keep at least this number of each package"), - "older_than": ( - int, - REQUIRED, - "Delete packages created before this number of seconds", - ), - "packages_to_delete": ( - str, - REQUIRED, - "Regex of packages to delete, e.g. pkg1|pkg2", - ), - "repo": (str, REQUIRED, "Which packagecloud repo to delete from"), - } - - desc = "Deleting {repo}/{packages_to_delete} older than {older_than}" - - @gen.coroutine - def _execute(self): - yield self._delete( - regex=self.option("packages_to_delete"), - number_to_keep=self.option("number_to_keep"), - older_than=self.option("older_than"), - repo=self.option("repo"), - ) - - -class WaitForPackage(PackagecloudBase): - """Searches for a package that matches `name` and `version` until found or - a timeout occurs. - - **Options** - - :name: - Name of the package to search for as a regex - - :version: - Version of the package to search for as a regex - - :repo: - Which packagecloud repo to delete from - - :sleep: - Number of seconds to sleep for between each search - - **Examples** - - .. code-block:: json - - { "desc": "packagecloud WaitForPackage example", - "actor": "packagecloud.WaitForPackage", - "options": { - "name": "findme", - "version": "0.1", - "repo": "test", - "sleep": 10, - } - } - - """ - - all_options = { - "name": (str, REQUIRED, "Name of the package to search for as a regex"), - "version": (str, ".*", "Version of the package to search for as a regex"), - "repo": (str, REQUIRED, "Which packagecloud repo to search"), - "sleep": (int, 10, "Number of seconds to sleep for between each search"), - } - - desc = "Waiting for {repo}/{name}@{version} (up to {sleep}s)" - - def __init__(self, *args, **kwargs): - """Check required environment variables.""" - super(WaitForPackage, self).__init__(*args, **kwargs) - - try: - re.compile(self.option("name")) - except re.error: - raise exceptions.InvalidOptions("name is an invalid regex") - - try: - re.compile(self.option("version")) - except re.error: - raise exceptions.InvalidOptions("version is an invalid regex") - - @gen.coroutine - def _search(self, repo, name, version): - """Searches for a given package until found or a timeout occurs. - - Args: - repo: name of the repo to search - name: Name of the package to search for as a regex - version: Version of the package to search for as a regex - - Returns: - A list of the packages that were found - """ - - all_packages = yield self._get_all_packages(repo=repo) - self.log.debug("Found all packages: %s" % all_packages) - - name_pattern = re.compile(name) - version_pattern = re.compile(version) - - matched_packages = [ - p - for p in all_packages - if name_pattern.match(p["name"]) and version_pattern.match(p["version"]) - ] - - raise gen.Return(matched_packages) - - @gen.coroutine - def _execute(self): - """Execute method for the WaitForPackage actor""" - while True: - self.log.info( - "Searching for %s %s..." % (self.option("name"), self.option("version")) - ) - - matched_packages = yield self._search( - repo=self.option("repo"), - name=self.option("name"), - version=self.option("version"), - ) - - if len(matched_packages) > 0: - self.log.info("Found it!") - raise gen.Return() - - self.log.debug("Not found, sleeping for (%s)" % self.option("sleep")) - yield gen.sleep(self.option("sleep")) diff --git a/kingpin/actors/pingdom.py b/kingpin/actors/pingdom.py deleted file mode 100644 index 33804bdb..00000000 --- a/kingpin/actors/pingdom.py +++ /dev/null @@ -1,184 +0,0 @@ -""" -:mod:`kingpin.actors.pingdom` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Pingdom actors to pause and unpause checks. These are useful when you are aware -of an expected downtime and don't want to be alerted about it. Also known as -Maintenance mode. - -**Required Environment Variables** - -:PINGDOM_TOKEN: - Pingdom API Token - -:PINGDOM_USER: - Pingdom Username (email) - -:PINGDOM_PASS: - Pingdom Password -""" - -import logging -import os - -from tornado import gen -from tornado import httpclient - -from tornado_rest_client import api - -from kingpin.constants import REQUIRED -from kingpin.actors import base -from kingpin.actors import exceptions - -log = logging.getLogger(__name__) - -__author__ = "Mikhail Simin " - - -USER = os.getenv("PINGDOM_USER", None) -PASS = os.getenv("PINGDOM_PASS", None) -TOKEN = os.getenv("PINGDOM_TOKEN", None) - - -class PingdomAPI(api.RestConsumer): - - _ENDPOINT = "https://api.pingdom.com" - _CONFIG = { - "attrs": { - "checks": {"path": "/api/2.0/checks", "http_methods": {"get": {}}}, - "check": { - "path": "/api/2.0/checks/%check_id%", - "http_methods": {"put": {}}, - }, - }, - "auth": {"user": USER, "pass": PASS}, - } - - -class PingdomClient(api.RestClient): - - # The default exception handling is fine, but the Pingdom API uses a 599 to - # represent a timeout on the backend of their service. - _EXCEPTIONS = dict(api.RestClient.EXCEPTIONS) - _EXCEPTIONS[httpclient.HTTPError]["599"] = None - - -class PingdomBase(base.BaseActor): - """Simple Pingdom Abstract Base Object""" - - all_options = { - "name": (str, REQUIRED, "Name of the check"), - } - - def __init__(self, *args, **kwargs): - """Check required environment variables.""" - super(PingdomBase, self).__init__(*args, **kwargs) - - rest_client = PingdomClient(headers={"App-Key": TOKEN}) - self._pingdom_client = PingdomAPI(client=rest_client) - - @gen.coroutine - def _get_check(self): - """Get check data for actor's option "name". - - Pingdom returns an array of all checks. This method finds the check - with the exact name and returns its contents. - - Raises InvalidOptions if the check does not exist. - """ - resp = yield self._pingdom_client.checks().http_get() - all_checks = resp["checks"] - check = [c for c in all_checks if c["name"] == self.option("name")] - - if not check: - raise exceptions.InvalidOptions( - 'Check name "%s" was not found.' % self.option("name") - ) - - raise gen.Return(check[0]) - - -class Pause(PingdomBase): - """Start Pingdom Maintenance. - - Pause a particular "check" on Pingdom. - - **Options** - - :name: - (Str) Name of the check - - **Example** - - .. code-block:: json - - { "actor": "pingdom.Pause", - "desc": "Run Pause", - "options": { - "name": "fill-in" - } - } - - **Dry run** - - Will assert that the check name exists, but not take any action on it. - """ - - desc = "Pausing check {name}" - - @gen.coroutine - def _execute(self): - check = yield self._get_check() - - if self._dry: - self.log.info( - "Would pause %s (%s) pingdom check." - % (check["name"], check["hostname"]) - ) - raise gen.Return() - - self.log.info("Pausing %s" % check["name"]) - yield self._pingdom_client.check(check_id=check["id"]).http_put(paused="true") - - -class Unpause(PingdomBase): - """Stop Pingdom Maintenance. - - Unpause a particular "check" on Pingdom. - - **Options** - - :name: - (Str) Name of the check - - **Example** - - .. code-block:: json - - { "actor": "pingdom.Unpause", - "desc": "Run unpause", - "options": { - "name": "fill-in" - } - } - - **Dry run** - - Will assert that the check name exists, but not take any action on it. - """ - - desc = "Unpausing check {name}" - - @gen.coroutine - def _execute(self): - check = yield self._get_check() - - if self._dry: - self.log.info( - "Would unpause %s (%s) pingdom check." - % (check["name"], check["hostname"]) - ) - raise gen.Return() - - self.log.info("Unpausing %s" % check["name"]) - yield self._pingdom_client.check(check_id=check["id"]).http_put(paused="false") diff --git a/kingpin/actors/slack.py b/kingpin/actors/slack.py deleted file mode 100644 index 2ddef81a..00000000 --- a/kingpin/actors/slack.py +++ /dev/null @@ -1,186 +0,0 @@ -""" -:mod:`kingpin.actors.slack` -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The Slack Actors allow you to send messages to a Slack channel at stages during -your job execution. The actor supports dry mode by validating that the -configured API Token has access to execute the methods, without actually -sending the messages. - -**Required Environment Variables** - -:SLACK_TOKEN: - Slack API Token - -:SLACK_NAME: - Slack *message from* name - (defaults to *Kingpin*) -""" - -import logging -import os -import re - -from tornado import gen - -from tornado_rest_client import api - -from kingpin.constants import REQUIRED -from kingpin.actors import base -from kingpin.actors import exceptions - -log = logging.getLogger(__name__) - -__author__ = "Matt Wise " - - -TOKEN = os.getenv("SLACK_TOKEN", None) -NAME = os.getenv("SLACK_NAME", "Kingpin") - - -class SlackAPI(api.RestConsumer): - - _ENDPOINT = "https://api.slack.com" - _CONFIG = { - "attrs": { - "auth_test": { - "path": "/api/auth.test", - "http_methods": {"post": {}}, - }, - "chat_postMessage": { - "path": "/api/chat.postMessage", - "http_methods": {"post": {}}, - }, - } - } - - -class SlackBase(base.BaseActor): - """Simple Slack Abstract Base Object""" - - def __init__(self, *args, **kwargs): - """Check required environment variables.""" - super(SlackBase, self).__init__(*args, **kwargs) - - if not TOKEN: - raise exceptions.InvalidCredentials( - 'Missing the "SLACK_TOKEN" environment variable.' - ) - - rest_client = api.SimpleTokenRestClient(tokens={"token": TOKEN}) - self._slack_client = SlackAPI(client=rest_client) - - def _check_results(self, result): - """Returns True/False if the result was OK from Slack. - - The Slack API avoids using standard error codes, and instead embeds - error codes in the return results. This method returns True or False - based on those results. - - Args: - result: A return dict from Slack - - Raises: - InvalidCredentials if the creds are bad - RecoverableActorException on any other value - """ - try: - ok = result.get("ok", False) - except AttributeError: - raise exceptions.UnrecoverableActorFailure( - "An unexpected Slack API failure occured: %s" % result - ) - - if ok: - return - - # By default, our exception type is a RecoverableActorFailure. - exc = exceptions.RecoverableActorFailure - - # If we know what kind fo error it is, we'll return a more accurate - # exception type. - if result["error"] == "invalid_auth": - exc = exceptions.InvalidCredentials - - # Finally, raise our exception - raise exc("Slack API Error: %s" % result["error"]) - - -class Message(SlackBase): - """Sends a message to a channel in Slack. - - **Options** - - :channel: - The string-name of the channel to send a message to, or a list of - channels - - :message: - String of the message to send - - **Examples** - - .. code-block:: json - - { "desc": "Let the Engineers know things are happening", - "actor": "slack.Message", - "options": { - "channel": "#operations", - "message": "Beginning Deploy: %VER%" - } - } - - **Dry Mode** - - Fully supported -- does not actually send messages to a room, but validates - that the API credentials would have access to send the message using the - Slack `auth.test` API method. - """ - - all_options = { - "channel": ((str, list), REQUIRED, "Slack channel or a list of names"), - "message": (str, REQUIRED, "Message to send"), - } - - desc = "Sending Message to {channel}" - - @gen.coroutine - def _execute(self): - self.log.info( - 'Sending message "%s" to Slack channel "%s"' - % (self.option("message"), self.option("channel")) - ) - - if self._dry: - # Check if our authentication creds are valid - auth_ok = yield self._slack_client.auth_test().http_post() - self._check_results(auth_ok) - - self.log.info("API Credentials verified, skipping send.") - raise gen.Return() - - # If only one channel was supplied as string then prepare the list - if type(self.option("channel")) == list: - channels = self.option("channel") - else: - channels = re.split("[, ]+", self.option("channel")) - - posts = [] - for channel in channels: - self.log.debug("Posting to %s" % channel) - # Finally, send the message and check our return value - posts.append( - self._slack_client.chat_postMessage().http_post( - channel=channel, - text=self.option("message"), - username=NAME, - parse="none", - link_names=1, - unfurl_links=True, - unfurl_media=True, - ) - ) - - results = yield posts - for res in results: - self._check_results(res) diff --git a/kingpin/actors/spotinst.py b/kingpin/actors/spotinst.py deleted file mode 100644 index e50ca745..00000000 --- a/kingpin/actors/spotinst.py +++ /dev/null @@ -1,879 +0,0 @@ -""" -:mod:`kingpin.actors.spotinst` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The Spotinst package allows you to create, manage and destroy Spotinst -ElastiGroups. - -https://spotinst.atlassian.net/wiki/display/API/API+Semantics - -**Environment Variables** - -:SPOTINST_DEBUG: - If set, then every single response body from Spotinst will be printed out in - the debug logs for Kingpin. This can include credentials, and can be - extremely verbose, so use with caution. - -:SPOINST_TOKEN: - SpotInst API Token generated at - https://console.spotinst.com/#/settings/tokens - -:SPOTINST_ACCOUNT_ID: - SpotInst API Account ID - this is required unless you set the account_id - parameter on each individual actor call. - http://docs.spotinst.com/#page:api-semantic,header:header-organizations-with-a-single-account -""" - -import base64 -import copy -import logging -import os -import json - -from tornado import gen -from tornado import httpclient - -from tornado_rest_client import api - -from kingpin import utils -from kingpin import exceptions as kingpin_exceptions -from kingpin.actors import base -from kingpin.actors import exceptions -from kingpin.actors.utils import dry -from kingpin.constants import REQUIRED -from kingpin.constants import SchemaCompareBase - - -log = logging.getLogger(__name__) - -__author__ = "Matt Wise " - -DEBUG = os.getenv("SPOTINST_DEBUG", False) -TOKEN = os.getenv("SPOTINST_TOKEN", None) -ACCOUNT_ID = os.getenv("SPOTINST_ACCOUNT_ID", None) - - -class SpotinstAPI(api.RestConsumer): - - ENDPOINT = "https://api.spotinst.io/" - CONFIG = { - "attrs": { - "aws": { - "new": True, - "attrs": { - "ec2": { - "new": True, - "attrs": { - "list_groups": { - "new": True, - "path": "aws/ec2/group?accountId=%account_id%", - "http_methods": {"get": {}}, - }, - "create_group": { - "new": True, - "path": "aws/ec2/group?accountId=%account_id%", - "http_methods": {"post": {}}, - }, - "list_group": { - "path": "aws/ec2/group/%id%?accountId=%account_id%", # nopep8 - "http_methods": {"get": {}}, - }, - "update_group": { - "path": "aws/ec2/group/%id%?accountId=%account_id%", # nopep8 - "http_methods": {"put": {}}, - }, - "delete_group": { - "path": "aws/ec2/group/%id%?accountId=%account_id%", # nopep8 - "http_methods": {"delete": {}}, - }, - "group_status": { - "path": "aws/ec2/group/%id%/status?accountId=%account_id%", # nopep8 - "http_methods": {"get": {}}, - }, - "validate_group": { - "new": True, - "path": "aws/ec2/group/validation?accountId=%account_id%", # nopep8 - "http_methods": {"post": {}}, - }, - "roll": { - "path": "aws/ec2/group/%id%/roll?limit=50&accountId=%account_id%", # nopep8 - "http_methods": {"put": {}, "get": {}}, - }, - "roll_status": { - "path": "aws/ec2/group/%id%/roll/%roll_id%?accountId=%account_id%", # nopep8 - "http_methods": {"get": {}}, - }, - }, - } - }, - } - } - } - - -class SpotinstException(exceptions.RecoverableActorFailure): - """Base SpotInst exception handler. - - This exception handler parses the Spotinst returned messages when an error - is thrown. The error message comes back in the body in a JSON formatted - blob. This exception handler will parse out the exception message, print it - out in a semi-readable log form for the user, and then store it in the - Exception body. - - See https://spotinst.atlassian.net/wiki/display/API/API+Semantics - for more details. - """ - - def __init__(self, e): - msg = self._parse(e) - Exception.__init__(self, msg) - self.exc = e - - def _parse(self, e): - """Reads through a SpotInst error message body and parses it. - - This method looks for a proper error message(s) from Spotinst in the - response body, parses them into something more humanly readable, and - then logs them out. It also adds them to the exception message so that - you get something beyond '400 Bad Request'. - """ - log = logging.getLogger("%s.%s" % (self.__module__, self.__class__.__name__)) - try: - error = json.loads(e.response.body) - except AttributeError: - return "Unknown error: %s" % e - - msg_id = "Request ID (%s) %s %s" % ( - error["request"]["id"], - error["request"]["method"], - error["request"]["url"], - ) - log.error("Error on %s" % msg_id) - - if "error" in error["response"]: - return "Spotinst %s: %s" % (msg_id, error["response"]["error"]) - - if "errors" in error["response"]: - msgs = [] - for err in error["response"]["errors"]: - msg = "%s: %s" % (err["code"], err["message"]) - msgs.append(msg) - log.error(msg) - return "Spotinst %s: %s" % (msg_id, ", ".join(msgs)) - - # Fallback if we don't know what kind of error body this is - error_str = "Spotinst %s: %s" % (msg_id, error["response"]) - return error_str - - -class InvalidConfig(SpotinstException): - """Thrown when an invalid request was supplied to Spotinst""" - - -class SpotinstRestClient(api.RestClient): - - EXCEPTIONS = { - httpclient.HTTPError: { - "400": InvalidConfig, - "401": exceptions.InvalidCredentials, - "403": exceptions.InvalidCredentials, - "500": None, - "502": None, - "503": None, - "504": None, - # Represents a standard HTTP Timeout - "599": None, - "": exceptions.BadRequest, - } - } - - JSON_BODY = True - - TIMEOUT = 60 - - -class ElastiGroupSchema(SchemaCompareBase): - """Light validation against the Spotinst ElastiGroup schema. - - For full description of the JSON data format, please see: - https://spotinst.atlassian.net/wiki/display/API/Create+Group#CreateGroup-JF - - This schema handles the following validation cases: - - * Only allow a single `SubnetID` for each `availabilityZone` object. - * Disallow `t2|hc1` instance types for the `spot` instance section. - * Ensure that the `scaling.up` and `scaling.down` arrays are either `null` - or contain at least **1** record. - """ - - SCHEMA = { - "type": "object", - "additionalProperties": True, - "required": ["group"], - "properties": { - "group": { - "type": "object", - "properties": { - "compute": { - "type": "object", - "properties": { - "availabilityZones": { - "type": "array", - "uniqueItems": True, - "items": { - "type": "object", - "required": ["name", "subnetId"], - "additionalProperties": False, - "properties": { - "name": {"type": "string"}, - "subnetId": {"type": "string"}, - }, - }, - }, - "instanceTypes": { - "type": "object", - "properties": { - "spot": { - "type": "array", - "additionalItems": False, - "items": { - "type": "string", - "not": {"pattern": "^t2|hc1"}, - }, - } - }, - }, - }, - }, - "scaling": { - "type": ["object", "null"], - "additionalProperties": False, - "properties": { - "up": {"type": ["null", "array"], "minItems": 1}, - "down": {"type": ["null", "array"], "minItems": 1}, - }, - }, - }, - } - }, - } - - -class SpotinstBase(base.EnsurableBaseActor): - """Simple Spotinst Abstract Base Object""" - - def __init__(self, *args, **kwargs): - super(SpotinstBase, self).__init__(*args, **kwargs) - - if not TOKEN: - raise exceptions.InvalidCredentials( - 'Missing the "SPOTINST_TOKEN" environment variable.' - ) - - if not DEBUG: - logging.getLogger("tornado_rest_client.api").setLevel("INFO") - - # Figure out our account ID and set it.. Or this will end up falling - # back to None if neither are set. - account_id = self._options.get("account_id") - if account_id is None: - account_id = ACCOUNT_ID - - if account_id is None: - raise exceptions.InvalidCredentials( - "Missing SPOTINST_ACCOUNT_ID or account_id parameter" - ) - - rest_client = SpotinstRestClient( - headers={ - "Authorization": "Bearer %s" % TOKEN, - "Content-Type": "application/json", - } - ) - - self._client = SpotinstAPI(client=rest_client, account_id=account_id) - - -class ElastiGroup(SpotinstBase): - """Manages an ElastiGroup in Spotinst. - - `Spotinst ElastiGroups - `_ act as - smarter EC2 AutoScalingGroups that scale up and down leveraging Amazon Spot - instances wherever possible. These ElastiGroups are completely configurable - through a `JSON Blob - `_. - - For a fully functional example JSON config, see :download:`this one - <../examples/test/spotinst.elastigroup/unittest.json>`. You can also write - your files in YAML if you prefer -- Kingpin will handle the conversion. - - **UserData Startup Script** - - The Spotinst API wants the instances UserData script to be supplied as - a Base64-encoded string -- which you can do if you wish. However, there is - no need, as Kingpin will automatically convert your plain-text script into - a Base64 blob for you behind the scenes. - - **Rolling out Group Changes** - - We will trigger the "roll group" API if the `roll_on_change` parameter is - set to `True` after any change to an ElastiGroup. It is difficult to know - which changes may or may not require a replacement of your existing hosts, - so we leave this up to the user to decide on the behavior. - - **Known Limitations** - - * The Spotinst API does not allow you to change an ElastiGroup scaling - 'unit' (ie, CPU Count or Instance Count). You can also not change an - ElastiGroup's basic platform (ie, VPC Linux vs Non VPC Linux). We warn - about this on each change. - - **Options** - - :name: - The desired name of the ElastiGroup. Note that this will override - whatever value is inside your configuration JSON/YAML blob. - - :account_id: - The SpotInst Account ID that the action is taking place in - this - overrides the SPOTINST_ACCOUNT_ID environment variable (if its set). - - :config: - Path to the ElastiGroup configuration blob (JSON or YAML) file. - :ref:`token_replacement` can be used inside of your configuration files - allowing environment variables to replace `%VAR%` strings. - - This file will be checked against a light-schema defined in - :py:class:`ElastiGroupSchema` before any authentication is required. The - file will be further validated against the Spotinst API during the DRY - run, but this requires authentication. - - :tokens: - A dict of key/value pairs that can be used to swap in variables into a - common ElastiGroup template. These are added to (and override) the - Environment variables that Kingpin already uses for variables swapping - (as described in the :ref:`token_replacement` section. - - :roll_on_change: - Whether or not to forcefully roll out changes to the ElastiGroup. If - `True`, we will issue a 'roll call' to SpotInst and trigger all of the - instances to be replaced. Defaults to `False`. - - :roll_batch_size: - Indicates in percentage the amount of instances should be replaced in - each batch. Defaults to `20`. - - :roll_grace_period: - Indicates in seconds the timeout to wait until instance become healthy in - the ELB. Defaults to `600`. - - :wait_on_create: - If set to `True`, Kingpin will loop until the ElastiGroup has fully - launched -- this only applies if the group is being created from scratch. - On updates, see the `wait_on_roll` setting below. - Defaults to `False`. - - :wait_on_roll: - If set to `True`, Kingpin will loop until the rollout of any changes has - completed. This can take a long time, depending on your rollout settings. - Defaults to `False`. - - **Examples** - - .. code-block:: json - - { "actor": "spotinst.ElastiGroup", - "options": { - "name": "my-group", - "config": "./group_config.json", - } - } - - **Dry Mode** - - Will discover the current state of the ElastiGroup (*present*, *absent*), - and whether or not the current configuration is different than the desired - configuration. Will also validate the desired configuration against the - SpostInst API to give you a heads up about any potential failures up - front. - """ - - all_options = { - "name": (str, REQUIRED, "Name of the ElastiGroup to manage"), - "account_id": (str, None, "SpotInst Account ID"), - "config": (str, None, "Name of the file with the ElastiGroup config"), - "tokens": ( - dict, - {}, - ( - "A flat dictionary of Key/Value pairs that can be " - "swapped into the ElastiGroup template." - ), - ), - "roll_on_change": ( - bool, - False, - ("Roll out new instances upon any config change."), - ), - "roll_batch_size": ( - (str, int), - 20, - ( - "Indicates in percentage the amount of instances should be" - "replaced in each batch." - ), - ), - "roll_grace_period": ( - (str, int), - 600, - ( - "Indicates in seconds the timeout to wait until instance become" - "healthy in the ELB." - ), - ), - "wait_on_create": ( - bool, - False, - "Wait for the ElastiGroup to startup and stabalize", - ), - "wait_on_roll": (bool, False, "Wait on any changes to roll out to the nodes"), - } - unmanaged_options = [ - "name", - "account_id", - "wait_on_roll", - "wait_on_create", - "roll_on_change", - "roll_batch_size", - "roll_grace_period", - "tokens", - ] - - desc = "ElastiGroup {name}" - - def __init__(self, *args, **kwargs): - super(ElastiGroup, self).__init__(*args, **kwargs) - - # Quickly make sure that the roll_batch_size and roll_grace_period are - # integers... - for key in ("roll_batch_size", "roll_grace_period"): - try: - self._options[key] = int(self._options[key]) - except ValueError: - raise exceptions.InvalidOptions( - "%s (%s) must be an integer" % (key, self._options[key]) - ) - - # Parse the user-supplied ElastiGroup config, swap in any tokens, etc. - self._config = self._parse_group_config() - - # Filld in later by self._precache() - self._group = None - - def _parse_group_config(self): - """Parses the ElastiGroup config and replaces tokens. - - Reads through the supplied ElastiGroup configuration JSON blob (or - YAML!), replaces any tokens that need replacement, and then sanity - checks it against our schema. - - .. note:: - - Contextual tokens (which are evaluated at run time, not compilation - time) are not included here. Instead, those will be evaluated in the - ``self._precache()`` method. - """ - config = self.option("config") - - if config is None: - return None - - self.log.debug("Parsing and validating %s" % config) - - # Join the init_tokens the class was instantiated with and the explicit - # tokens that the user supplied. - tokens = dict(self._init_tokens) - tokens.update(self.option("tokens")) - - try: - parsed = utils.convert_script_to_dict(script_file=config, tokens=tokens) - except (kingpin_exceptions.InvalidScript, LookupError) as e: - raise exceptions.InvalidOptions("Error parsing %s: %s" % (config, e)) - - # The userData portion of the body data needs to be Base64 encoded if - # its not already. We will try to decode whatever is there, and if it - # fails, we assume its raw text and we encode it. - orig_data = parsed["group"]["compute"]["launchSpecification"]["userData"] - new = base64.b64encode(orig_data.encode("utf-8")) - parsed["group"]["compute"]["launchSpecification"]["userData"] = new - - # Ensure that the name of the ElastiGroup in the config file matches - # the name that was supplied to the actor -- or overwrite it. - parsed["group"]["name"] = self.option("name") - - # Now run the configuration through the schema validator - ElastiGroupSchema.validate(parsed) - - return parsed - - @gen.coroutine - def _precache(self): - """Pre-populate a bunch of data. - - Searches for the list of ElastiGroups and stores the existing - configuration for an ElastiGroup if it matches the name of the one - we're managing here. - - Attempts light schema-validation of the desired ElastiGroup config to - try to catch errors early in the Dry run. - """ - # Check if the desired ElastiGroup already exists or not -- if it does, - # store its configuration here for comparison purposes. - self._group = yield self._get_group() - - # Validate the desired ElastiGroup configuration against the - # schema-checker... light validation, but useful. - yield self._validate_group() - - # Note - we don't manage the ElastiGroup target size. If the group - # exists and has a target size set, we override the user-supplied - # target number with the value returned to us by Spotinst. - if self._group and "capacity" in self._group["group"]: - target = self._group["group"]["capacity"]["target"] - self.log.info( - "Using the Spotinst supplied [capacity][target] value: %s" % target - ) - self._config["group"]["capacity"]["target"] = target - - @gen.coroutine - def _list_groups(self): - """Returns a list of all ElastiGroups in your Spotinst acct. - - Returns: - [List of JSON ElastiGroup objects] - """ - raw = yield self._client.aws.ec2.list_groups.http_get() - resp = raw.get("response", {}) - items = resp.get("items", []) - raise gen.Return(items) - - @gen.coroutine - def _get_group(self): - """Finds and returns the existing ElastiGroup configuration. - - If the ElastiGroup exists, it returns the configuration for the group. - If the group is missing, it returns None. Used by the self._precache() - method to determine whether or not the desired ElastiGroup already - exists or not, and what its configuration looks like. - - Returns: - A dictionary with the ElastiGroup configuration returned by Spotinst - or None if no matching group is found. - - Raises: - exceptions.InvalidOptions: If too many groups are returned. - """ - all_groups = yield self._list_groups() - if not all_groups: - raise gen.Return(None) - - matching = [ - group for group in all_groups if group["name"] == self.option("name") - ] - - if len(matching) > 1: - raise exceptions.InvalidOptions( - "Found more than one ElastiGroup with the name %s - " - "this actor cannot manage multiple groups with the same" - "name, you must use a unique name for each group." % self.option("name") - ) - - if len(matching) < 1: - self.log.debug("Did not find an existing ElastiGroup") - raise gen.Return(None) - - match = matching[0] - self.log.debug("Found ElastiGroup %s" % match["id"]) - raise gen.Return({"group": match}) - - @gen.coroutine - def _validate_group(self): - """Basic Schema validation of the Elastigroup config. - - This endpoint is not documented, but it performs the most basic schema - validation of the supplied ElastiGroup config. It cannot verify that - instances will truly launch, but it can help catch obvious errors. - - It does require authentication, which is sad. - - Raises: - SpotinstException: If any known Spotinst style error comes back. - """ - yield self._client.aws.ec2.validate_group.http_post(group=self._config["group"]) - - @gen.coroutine - def _get_state(self): - """Validates whether or not a matching ElastiGroup already exists. - - Depends on the self._precache() method having been called. If it has, - then self._group should be populated if the group exists, or None if it - doesn't. - - Returns: - present: If the group exists - absent: If not - """ - if self._group: - raise gen.Return("present") - - raise gen.Return("absent") - - @gen.coroutine - def _set_state(self): - """Creates or Deletes the ElastiGroup - - If the desired state is absent adn the group exists, we trigger a - delete_group call. If the desired state is present and the group does - not exist, we trigger a group create call. In any other situation, we - do nothing because the desired and current states match. - """ - if self.option("state") == "absent" and self._group: - yield self._delete_group(id=self._group["group"]["id"]) - elif self.option("state") == "present": - yield self._create_group() - - # You'd think that we could store the returned group config from - # Spotinst .. but it turns out that the data returned in the - # create_group call above is not the same as what we've uploaded. - # Instead, we have to re-call the self._precache() method to make - # sure that we get an updated group config. - # self._group = {'group': ret['response']['items'][0]} - self._group = yield self._get_group() - - # Optionally, wait until the nodes have booted up before returning. - if self.option("wait_on_create"): - yield self._wait_until_stable() - - @gen.coroutine - @dry("Would have created ElastiGroup") - def _create_group(self): - self.log.info("Creating ElastiGroup %s" % self.option("name")) - yield self._client.aws.ec2.create_group.http_post(group=self._config["group"]) - - @gen.coroutine - @dry("Would have deleted ElastiGroup {id}") - def _delete_group(self, id): - self.log.info("Deleting ElastiGroup %s" % id) - yield self._client.aws.ec2.delete_group(id=id).http_delete() - - @gen.coroutine - def _get_group_status(self, id): - self.log.debug("Getting ElastiGroup %s status..." % id) - ret = yield self._client.aws.ec2.group_status(id=id).http_get() - raise gen.Return(ret) - - @gen.coroutine - def _get_config(self): - """Not really used, but a stub for correctness""" - raise gen.Return(self._group) - - @gen.coroutine - @dry("Would have updated ElastiGroup config") - def _set_config(self): - group_id = self._group["group"]["id"] - self.log.info("Updating ElastiGroup %s" % group_id) - - # There are certain fields that simply cannot be updated -- strip them - # out. We have a warning up in the above _compare_config() section that - # will tell the user about this in a dry run. - if "capacity" in self._config["group"]: - self.log.warning("Note: Ignoring the group[capacity][unit] setting.") - self._config["group"]["capacity"].pop("unit", None) - if "compute" in self._config["group"]: - self.log.warning("Note: Ignoring the group[compute][product] setting.") - self._config["group"]["compute"].pop("product", None) - - # Now do the update and capture the results. Once we have them, we'll - # store the updated group configuration. - ret = yield self._client.aws.ec2.update_group(id=group_id).http_put( - group=self._config["group"] - ) - self._group = {"group": ret["response"]["items"][0]} - - # If we're supposed to roll the group on any config changes, begin now - if self.option("roll_on_change"): - yield self._roll_group() - - @gen.coroutine - def _compare_config(self): - """Smart-ish comparison of Spotinst config to our own. - - This method is called by the EnsurableBaseClass to compare the desired - (local) config with the existing (remote) config of the ElastiGroup. - A simple == comparison will not work because there are additional - fields returned by the Spotinst API (id, createdAt, updatedAt, and - more) that will never be in the desired configuration object. - - This method makes copies of the configuration objects, strips out the - fields that we cannot compare against, and then diffs. If a diff is - detected, it logs out the diff for the end user, and then returns - False. - - Returns: - True: the configs match - False: the configs do not match - """ - # For the purpose of comparing the two configuration dicts, we need to - # modify them (below).. so first lets copy them so we don't modify the - # originals. - new = copy.deepcopy(self._config) - existing = copy.deepcopy(self._group) - - # If existing is none, then return .. there is no point in diffing the - # config if the group doesn't exist! Note, this really only happens in - # a dry run where we're creating the group because the group - if existing is None: - raise gen.Return(True) - - # Strip out some of the Spotinst generated and managed fields that - # should never end up in either our new or existing configs. - for field in ("id", "createdAt", "updatedAt", "userData"): - for g in (new, existing): - g["group"].pop(field, None) - - # Decode both of the userData fields so we can actually see the - # userdata differences. - for config in (new, existing): - config["group"]["compute"]["launchSpecification"]["userData"] = ( - base64.b64decode( - config["group"]["compute"]["launchSpecification"]["userData"] - ) - ) - - # We only allow a user to supply a single subnetId for each AZ (this is - # handled by the ElastiGroupSchema). Spotinst returns back though both - # the original setting, as well as a list of subnetIds. We purge that - # from our comparison here. - for az in existing["group"]["compute"]["availabilityZones"]: - az.pop("subnetIds", None) - - diff = utils.diff_dicts(existing, new) - - if diff: - self.log.warning("Group configurations do not match") - for line in diff.split("\n"): - self.log.info("Diff: %s" % line) - return False - - return True - - @gen.coroutine - @dry("Would have rolled the ElastiGroup..") - def _roll_group(self, delay=30): - """Triggers an ElastiGroup rolling operation and waits for completion. - - Sends a signal to Spotinst to "roll" (replace) the nodes in the - ElastiGroup based on the new configuration. This operation takes a - while based on the `roll_batch_size` and `roll_grace_period` options. - Depending on the `wait_on_roll` option, this method will wait until the - roll has completed before returning. - """ - group_id = self._group["group"]["id"] - - # You are not allowed to have two rolls happening at the same time -- - # so if there is already a roll in progress, we need to wait before we - # issue another one. This is a requirement regardless of whether the - # user has asked us to 'wait_on_roll' or not, because we'll get an - # exception back from the API if we try to issue a roll call during an - # existing roll operation. - yield self._wait_until_roll_complete(delay) - - # Now, try to do the roll... - self.log.info("Triggering an ElastiGroup roll") - yield self._client.aws.ec2.roll(id=group_id).http_put( - batchSizePercentage=self.option("roll_batch_size"), - gracePeriod=self.option("roll_grace_period"), - ) - - # Now, if the user wants us to wait, we will wait. - if self.option("wait_on_roll"): - yield self._wait_until_roll_complete(delay) - - @gen.coroutine - @dry("Would have waited for ElastiGroup changes to become active") - def _wait_until_roll_complete(self, delay): - """Poll and wait until an ElastiGroup roll is complete.""" - group_id = self._group["group"]["id"] - - # Note: We do not use the repeating_log because we only call this API - # every 30s or so. Rolling out group changes is almost guaranteed to be - # a very slow process, so there is no need to make frequent API calls - # to constantly check the status of the rollout. Instead, we make calls - # infrequently and thus we are able to simply log out the status after - # each call. - self.log.info("Checking if any ElastiGroup rolls are in progress..") - while True: - response = yield self._client.aws.ec2.roll(id=group_id).http_get() - - in_progress = [ - r for r in response["response"]["items"] if r["status"] != "finished" - ] - - if len(in_progress) < 1: - break - - status = in_progress[0]["status"] - unit = in_progress[0]["progress"]["unit"] - progress = in_progress[0]["progress"]["value"] - - self.log.info( - "Group roll is %s %s complete (%s)" % (progress, unit, status) - ) - - yield gen.sleep(delay) - - @gen.coroutine - @dry("Would have waited for all ElastiGroup nodes to launch") - def _wait_until_stable(self, delay=3): - """Poll and wait until an ElastiGroup has stabalized. - - Upon group creation, most of the instances will be in a "biding" state. - This method watches the list of instances and waits until they are all - in the 'fulfilled' state. - """ - group_id = self._group["group"]["id"] - - # We use the repeating_log to let the user know we're still monitoring - # things, while not flooding them every time we make an API call. We - # give them a message every 30s, but make an API call every 3 seconds - # to check the status. - repeating_log = utils.create_repeating_log( - self.log.info, "Waiting for ElastiGroup to become stable", seconds=30 - ) - - while True: - response = yield self._get_group_status(group_id) - - # Find any nodes that are waiting for spot instance requests to be - # fulfilled. - pending = [ - i - for i in response["response"]["items"] - if i["status"] == "pending-evaluation" - ] - fulfilled = [ - i["instanceId"] - for i in response["response"]["items"] - if i["status"] == "fulfilled" and i["instanceId"] is not None - ] - - if len(pending) < 1: - self.log.info( - "All instance requests fulfilled: %s" % ", ".join(fulfilled) - ) - break - - yield gen.sleep(delay) - - utils.clear_repeating_log(repeating_log) diff --git a/kingpin/actors/test/test_packagecloud.py b/kingpin/actors/test/test_packagecloud.py deleted file mode 100644 index 1b6f79d0..00000000 --- a/kingpin/actors/test/test_packagecloud.py +++ /dev/null @@ -1,397 +0,0 @@ -"""Tests for the actors.packagecloud package""" - -import datetime -import mock - -from tornado import testing - -from kingpin.actors import exceptions -from kingpin.actors import packagecloud -from kingpin.actors.test.helper import mock_tornado, tornado_value - -__author__ = "Charles McLaughlin " - -ALL_PACKAGES_MOCK_RESPONSE = [ - { - "name": "unittest", - "uploader_name": "unittest", - "created_at": "2015-07-07T20:27:18.000Z", - "distro_version": "ubuntu/trusty", - "filename": "unittest_0.2-1_all.deb", - "epoch": 0, - "version": "0.2", - "private": True, - "release": "1", - "package_url": ( - "/api/v1/repos/unittest/test/package/deb/ubuntu/" - "trusty/unittest/all/0.2-1.json" - ), - "type": "deb", - "package_html_url": ( - "/unittest/test/packages/ubuntu/trusty/unittest_0.2-1_all.deb" - ), - "repository_html_url": "/unittest/test", - }, - { - "name": "unittest", - "uploader_name": "unittest", - "created_at": "2014-07-07T20:27:18.000Z", - "distro_version": "ubuntu/trusty", - "filename": "unittest_0.1-1_all.deb", - "epoch": 0, - "version": "0.1", - "private": True, - "release": "1", - "package_url": ( - "/api/v1/repos/unittest/test/package/deb/ubuntu/" - "trusty/unittest/all/0.1-1.json" - ), - "type": "deb", - "package_html_url": ( - "/unittest/test/packages/ubuntu/trusty/unittest_0.1-1_all.deb" - ), - "repository_html_url": "/unittest/test", - }, -] - - -def _get_older_than(): - """Method for getting an `older_than` value which is used in a few - tests below. - - The tests specifically look for the second package in - ALL_PACKAGES_MOCK_RESPONSE, as it's older than the first""" - - created_at = datetime.datetime.strptime( - ALL_PACKAGES_MOCK_RESPONSE[0]["created_at"], "%Y-%m-%dT%H:%M:%S.%fZ" - ) - older_than = ( - datetime.datetime.now() - created_at + datetime.timedelta(seconds=600) - ) # Adding 5m in case the tests run long - return older_than - - -class TestPackagecloudBase(testing.AsyncTestCase): - """Unit tests for the packagecloud Base actor.""" - - def setUp(self, *args, **kwargs): - super(TestPackagecloudBase, self).setUp(*args, **kwargs) - packagecloud.TOKEN = "Unittest" - packagecloud.ACCOUNT = "Unittest" - self.maxDiff = None - - @testing.gen_test - def test_init_missing_token(self): - # Un-set the token and make sure the init fails - packagecloud.TOKEN = None - with self.assertRaises(exceptions.InvalidCredentials): - packagecloud.PackagecloudBase("Unit Test Action", {}) - - @testing.gen_test - def test_init_missing_account(self): - # Un-set the account and make sure the init fails - packagecloud.ACCOUNT = None - with self.assertRaises(exceptions.InvalidCredentials): - packagecloud.PackagecloudBase("Unit Test Action", {}) - - @testing.gen_test - def test_get_all_packages(self): - actor = packagecloud.PackagecloudBase("Unit test action", {}) - actor._packagecloud_client = mock.Mock() - actor._packagecloud_client.packages().http_get = mock_tornado( - ALL_PACKAGES_MOCK_RESPONSE - ) - all_packages = yield actor._get_all_packages(repo="unittest") - self.assertEqual(all_packages, ALL_PACKAGES_MOCK_RESPONSE) - - @testing.gen_test - def test_get_package_versions(self): - actor = packagecloud.PackagecloudBase("Unit test action", {}) - actor._packagecloud_client = mock.Mock() - actor._packagecloud_client.packages().http_get = mock_tornado( - ALL_PACKAGES_MOCK_RESPONSE - ) - packages = yield actor._get_all_packages(repo="unittest") - versions = actor._get_package_versions(name="unittest", packages=packages) - - self.assertEqual( - versions, - [ - { - "created_at": datetime.datetime(2014, 7, 7, 20, 27, 18), - "name": "unittest", - "distro_version": "ubuntu/trusty", - "filename": "unittest_0.1-1_all.deb", - }, - { - "created_at": datetime.datetime(2015, 7, 7, 20, 27, 18), - "name": "unittest", - "distro_version": "ubuntu/trusty", - "filename": "unittest_0.2-1_all.deb", - }, - ], - ) - - @testing.gen_test - def test_filter_packages(self): - actor = packagecloud.PackagecloudBase("Unit test action", {}) - actor._packagecloud_client = mock.Mock() - actor._packagecloud_client.packages().http_get = mock_tornado( - ALL_PACKAGES_MOCK_RESPONSE - ) - packages = yield actor._get_all_packages(repo="unittest") - packages_list_to_delete = actor._filter_packages( - regex="unittest", packages=packages - ) - self.assertEqual(packages_list_to_delete, set(["unittest"])) - - @testing.gen_test - def test_delete(self): - actor = packagecloud.PackagecloudBase("Unit test action", {}) - actor._packagecloud_client = mock.Mock() - actor._packagecloud_client.packages().http_get = mock_tornado( - ALL_PACKAGES_MOCK_RESPONSE - ) - actor._packagecloud_client.delete().http_delete = mock_tornado({}) - - deleted_packages = yield actor._delete(regex="unittest", repo="unittest") - - self.assertEqual( - deleted_packages, - [ - { - "created_at": datetime.datetime(2014, 7, 7, 20, 27, 18), - "name": "unittest", - "distro_version": "ubuntu/trusty", - "filename": "unittest_0.1-1_all.deb", - }, - { - "created_at": datetime.datetime(2015, 7, 7, 20, 27, 18), - "name": "unittest", - "distro_version": "ubuntu/trusty", - "filename": "unittest_0.2-1_all.deb", - }, - ], - ) - - @testing.gen_test - def test_delete_dry(self): - actor = packagecloud.PackagecloudBase("Unit test action", {}) - actor._packagecloud_client = mock.Mock() - actor._packagecloud_client.packages().http_get = mock_tornado( - ALL_PACKAGES_MOCK_RESPONSE - ) - actor._packagecloud_client.delete().http_delete = mock_tornado({}) - actor._dry = True - - deleted_packages = yield actor._delete(regex="unittest", repo="unittest") - - self.assertEqual( - deleted_packages, - [ - { - "created_at": datetime.datetime(2014, 7, 7, 20, 27, 18), - "name": "unittest", - "distro_version": "ubuntu/trusty", - "filename": "unittest_0.1-1_all.deb", - }, - { - "created_at": datetime.datetime(2015, 7, 7, 20, 27, 18), - "name": "unittest", - "distro_version": "ubuntu/trusty", - "filename": "unittest_0.2-1_all.deb", - }, - ], - ) - - @testing.gen_test - def test_delete_keep_one(self): - actor = packagecloud.PackagecloudBase("Unit test action", {}) - actor._packagecloud_client = mock.Mock() - actor._packagecloud_client.packages().http_get = mock_tornado( - ALL_PACKAGES_MOCK_RESPONSE - ) - actor._packagecloud_client.delete().http_delete = mock_tornado({}) - - deleted_packages = yield actor._delete( - regex="unittest", repo="unittest", number_to_keep=1 - ) - - self.assertEqual( - deleted_packages, - [ - { - "created_at": datetime.datetime(2014, 7, 7, 20, 27, 18), - "name": "unittest", - "distro_version": "ubuntu/trusty", - "filename": "unittest_0.1-1_all.deb", - } - ], - ) - - @testing.gen_test - def test_delete_older_than(self): - actor = packagecloud.PackagecloudBase("Unit test action", {}) - actor._packagecloud_client = mock.Mock() - actor._packagecloud_client.packages().http_get = mock_tornado( - ALL_PACKAGES_MOCK_RESPONSE - ) - actor._packagecloud_client.delete().http_delete = mock_tornado({}) - - older_than = _get_older_than() - - deleted_packages = yield actor._delete( - regex="unittest", repo="unittest", older_than=older_than.total_seconds() - ) - - self.assertEqual( - deleted_packages, - [ - { - "created_at": datetime.datetime(2014, 7, 7, 20, 27, 18), - "name": "unittest", - "distro_version": "ubuntu/trusty", - "filename": "unittest_0.1-1_all.deb", - } - ], - ) - - -class TestDelete(testing.AsyncTestCase): - """Unit tests for the packagecloud Delete actor.""" - - def setUp(self, *args, **kwargs): - super(TestDelete, self).setUp(*args, **kwargs) - packagecloud.TOKEN = "Unittest" - packagecloud.ACCOUNT = "Unittest" - self.maxDiff = None - - @testing.gen_test - def test_bad_regex_packages_to_delete(self): - with self.assertRaises(exceptions.InvalidOptions): - packagecloud.Delete( - "Unit test action", {"packages_to_delete": "[", "repo": "unittest"} - ) - - @testing.gen_test - def test_execute(self): - actor = packagecloud.Delete( - "Unit test action", {"packages_to_delete": "unittest", "repo": "unittest"} - ) - actor._packagecloud_client = mock.Mock() - actor._packagecloud_client.packages().http_get = mock_tornado( - ALL_PACKAGES_MOCK_RESPONSE - ) - actor._packagecloud_client.delete().http_delete = mock_tornado({}) - - deleted_packages = yield actor._execute() - - self.assertEqual(deleted_packages, None) - - -class TestDeleteByDate(testing.AsyncTestCase): - """Unit tests for the packagecloud DeleteByDate actor.""" - - def setUp(self, *args, **kwargs): - super(TestDeleteByDate, self).setUp(*args, **kwargs) - packagecloud.TOKEN = "Unittest" - packagecloud.ACCOUNT = "Unittest" - self.maxDiff = None - - @testing.gen_test - def test_execute(self): - older_than = _get_older_than() - actor = packagecloud.DeleteByDate( - "Unit test action", - { - "packages_to_delete": "unittest", - "repo": "unittest", - "older_than": int(older_than.total_seconds()), - }, - ) - - actor._packagecloud_client = mock.Mock() - actor._packagecloud_client.packages().http_get = mock_tornado( - ALL_PACKAGES_MOCK_RESPONSE - ) - actor._packagecloud_client.delete().http_delete = mock_tornado({}) - - deleted_packages = yield actor._execute() - - self.assertEqual(deleted_packages, None) - - -class TestWaitForPackage(testing.AsyncTestCase): - """Unit tests for the packagecloud WaitForPackage actor.""" - - def setUp(self, *args, **kwargs): - super(TestWaitForPackage, self).setUp(*args, **kwargs) - packagecloud.TOKEN = "Unittest" - packagecloud.ACCOUNT = "Unittest" - self.maxDiff = None - - @testing.gen_test - def test_bad_regex_name(self): - with self.assertRaises(exceptions.InvalidOptions): - packagecloud.WaitForPackage( - "Unit test action", {"name": "[", "version": "1", "repo": "unittest"} - ) - - @testing.gen_test - def test_bad_regex_version(self): - with self.assertRaises(exceptions.InvalidOptions): - packagecloud.WaitForPackage( - "Unit test action", - {"name": "unittest", "version": "[", "repo": "unittest"}, - ) - - @testing.gen_test - def test_execute(self): - actor = packagecloud.WaitForPackage( - "Unit test action", - {"name": "unittest", "repo": "unittest", "version": "0.2"}, - ) - - actor._packagecloud_client = mock.Mock() - actor._packagecloud_client.packages().http_get = mock_tornado( - ALL_PACKAGES_MOCK_RESPONSE - ) - actor._packagecloud_client.delete().http_delete = mock_tornado({}) - - matched_packages = yield actor._execute() - - self.assertEqual(matched_packages, None) - - @testing.gen_test - def test_execute_with_sleep(self): - actor = packagecloud.WaitForPackage( - "Unit test action", - {"name": "not_found", "repo": "unittest", "version": "0.2", "sleep": 1}, - ) - - actor._packagecloud_client = mock.Mock() - actor._packagecloud_client.packages().http_get = mock_tornado( - ALL_PACKAGES_MOCK_RESPONSE - ) - actor._search = mock.Mock( - side_effect=[tornado_value([]), tornado_value(["something"])] - ) - yield actor._execute() - self.assertEqual(actor._search.call_count, 2) - - @testing.gen_test - def test_search(self): - actor = packagecloud.WaitForPackage( - "Unit test action", - {"name": "unittest", "repo": "unittest", "version": "0.2"}, - ) - actor._packagecloud_client = mock.Mock() - actor._packagecloud_client.packages().http_get = mock_tornado( - ALL_PACKAGES_MOCK_RESPONSE - ) - - matched_packages = yield actor._search( - repo="unittest", name="unittest", version="0.2" - ) - - self.assertEqual(matched_packages, [ALL_PACKAGES_MOCK_RESPONSE[0]]) diff --git a/kingpin/actors/test/test_pingdom.py b/kingpin/actors/test/test_pingdom.py deleted file mode 100644 index 87f44cfc..00000000 --- a/kingpin/actors/test/test_pingdom.py +++ /dev/null @@ -1,119 +0,0 @@ -"""Tests for the pingdom actors""" - -import mock - -from tornado import testing - -from kingpin.actors import pingdom -from kingpin.actors import exceptions -from kingpin.actors.test.helper import mock_tornado, tornado_value - - -__author__ = "Mikhail Simin " - - -class TestPingdomBase(testing.AsyncTestCase): - def setUp(self, *args, **kwargs): - super(TestPingdomBase, self).setUp() - pingdom.TOKEN = "Unittest" - pingdom.USER = "Unittest" - pingdom.PASS = "Unittest" - - @testing.gen_test - def test_check_name(self): - actor = pingdom.PingdomBase("Unit Test Action", {"name": "lollipop"}) - - actor._pingdom_client = mock.Mock() - actor._pingdom_client.checks().http_get = mock_tornado( - {"checks": [{"name": "lollipop"}]} - ) - - check = yield actor._get_check() - - self.assertEqual(check["name"], "lollipop") - - @testing.gen_test - def test_check_name_fail(self): - actor = pingdom.PingdomBase("Unit Test Action", {"name": "lollipop"}) - - actor._pingdom_client = mock.Mock() - actor._pingdom_client.checks().http_get = mock_tornado({"checks": []}) - - with self.assertRaises(exceptions.InvalidOptions): - yield actor._get_check() - - -class TestPause(testing.AsyncTestCase): - def setUp(self, *args, **kwargs): - super(TestPause, self).setUp() - pingdom.TOKEN = "Unittest" - pingdom.USER = "Unittest" - pingdom.PASS = "Unittest" - - @testing.gen_test - def test_execute(self): - actor = pingdom.Pause("Unit Test Action", {"name": "lollipop"}) - - actor._pingdom_client = mock.Mock() - actor._get_check = mock_tornado( - {"name": "lollipop", "hostname": "http://lollipop.com", "id": "lol"} - ) - actor._pingdom_client.check().http_put.return_value = tornado_value() - - yield actor._execute() - - self.assertEqual(actor._get_check._call_count, 1) - actor._pingdom_client.check.assert_called_with(check_id="lol") - actor._pingdom_client.check().http_put.assert_called_with(paused="true") - - @testing.gen_test - def test_execute_dry(self): - actor = pingdom.Pause("Unit Test Action", {"name": "lollipop"}, dry=True) - - actor._pingdom_client = mock.Mock() - actor._get_check = mock_tornado( - {"name": "lollipop", "hostname": "http://lollipop.com", "id": "lol"} - ) - - yield actor._execute() - - self.assertEqual(actor._get_check._call_count, 1) - actor._pingdom_client.check().http_put.assert_not_called() - - -class TestUnpause(testing.AsyncTestCase): - def setUp(self, *args, **kwargs): - super(TestUnpause, self).setUp() - pingdom.TOKEN = "Unittest" - pingdom.USER = "Unittest" - pingdom.PASS = "Unittest" - - @testing.gen_test - def test_execute(self): - actor = pingdom.Unpause("Unit Test Action", {"name": "lollipop"}) - - actor._pingdom_client = mock.Mock() - actor._get_check = mock_tornado( - {"name": "lollipop", "hostname": "http://lollipop.com", "id": "lol"} - ) - actor._pingdom_client.check().http_put.return_value = tornado_value() - - yield actor._execute() - - self.assertEqual(actor._get_check._call_count, 1) - actor._pingdom_client.check.assert_called_with(check_id="lol") - actor._pingdom_client.check().http_put.assert_called_with(paused="false") - - @testing.gen_test - def test_execute_dry(self): - actor = pingdom.Unpause("Unit Test Action", {"name": "lollipop"}, dry=True) - - actor._pingdom_client = mock.Mock() - actor._get_check = mock_tornado( - {"name": "lollipop", "hostname": "http://lollipop.com", "id": "lol"} - ) - - yield actor._execute() - - self.assertEqual(actor._get_check._call_count, 1) - actor._pingdom_client.check().http_put.assert_not_called() diff --git a/kingpin/actors/test/test_slack.py b/kingpin/actors/test/test_slack.py deleted file mode 100644 index 7213cc98..00000000 --- a/kingpin/actors/test/test_slack.py +++ /dev/null @@ -1,183 +0,0 @@ -"""Tests for the actors.slack package""" - -import mock - -from tornado import testing - -from kingpin.actors import slack -from kingpin.actors import exceptions -from kingpin.actors.test.helper import mock_tornado -import importlib - - -__author__ = "Matt Wise " - - -class TestSlackBase(testing.AsyncTestCase): - """Unit tests for the Slack Base actor.""" - - def setUp(self, *args, **kwargs): - # For most tests, mock out the TOKEN - super(TestSlackBase, self).setUp(*args, **kwargs) - slack.TOKEN = "Unittest" - - def test_init(self): - actor = slack.SlackBase("Unit test action", {}) - - # Ensure that the actor._slack_client is configured with a dictionary - # that contains the token in it. - slack_client_tokens = actor._slack_client._client._tokens - self.assertEqual(slack_client_tokens, {"token": "Unittest"}) - - def test_init_missing_creds(self): - # Un-set the token now and make sure the init fails - slack.TOKEN = None - with self.assertRaises(exceptions.InvalidCredentials): - slack.SlackBase("Unit Test Action", {}) - # Reload the slack library to re-get the token - importlib.reload(slack) - - def test_check_results_with_ok_results(self): - actor = slack.SlackBase("Unit test action", {}) - results = { - "ok": True, - "channel": "C03H4GRDF", - "ts": "1423092527.000006", - "message": { - "text": "Hi, testing!", - "username": "Kingpin", - "type": "message", - "subtype": "bot_message", - "ts": "1423092527.000006", - }, - } - self.assertEqual(None, actor._check_results(results)) - - def test_check_results_with_invalid_creds(self): - actor = slack.SlackBase("Unit test action", {}) - results = {"ok": False, "error": "invalid_auth"} - with self.assertRaises(exceptions.InvalidCredentials): - actor._check_results(results) - - def test_check_results_with_unexpected_results(self): - actor = slack.SlackBase("Unit test action", {}) - results = "got some unexpected result" - with self.assertRaises(exceptions.UnrecoverableActorFailure): - actor._check_results(results) - - -class TestMessage(testing.AsyncTestCase): - """Unit tests for the Slack Message actor.""" - - def setUp(self, *args, **kwargs): - # For most cases, mock out the TOKEN - super(TestMessage, self).setUp(*args, **kwargs) - slack.TOKEN = "Unittest" - - self.actor = slack.Message( - "Unit test message", {"channel": "#testing", "message": "Unittest"} - ) - self._slack_mock = mock.MagicMock(name="SlackAPIMock") - self.actor._slack_client = self._slack_mock - - @testing.gen_test - def test_execute_dry(self): - # Mock out the calls to SlackAPI.auth_test().auth_test() - auth_test_mock = mock.MagicMock(name="auth_test") - auth_test_mock.http_post.side_effect = mock_tornado({"ok": "true"}) - self._slack_mock.auth_test.return_value = auth_test_mock - - # Ensure we're dry - self.actor._dry = True - ret = yield self.actor._execute() - self.assertEqual(None, ret) - - # Ensure the calls were made to the API - auth_test_mock.http_post.assert_has_calls([mock.call()]) - - @testing.gen_test - def test_execute(self): - # Mock out the calls to SlackAPI.chat_postMessage().http_post() - post_mock = mock.MagicMock() - post_mock.http_post.side_effect = mock_tornado({"ok": "true"}) - self._slack_mock.chat_postMessage.return_value = post_mock - - ret = yield self.actor._execute() - self.assertEqual(None, ret) - - # Ensure the calls were made to the API - post_mock.http_post.assert_has_calls( - [ - mock.call( - username="Kingpin", - unfurl_links=True, - text="Unittest", - unfurl_media=True, - parse="none", - link_names=1, - channel="#testing", - ) - ] - ) - - @testing.gen_test - def test_execute_list_rooms(self): - actor = slack.Message( - "Unit test message", - {"channel": ["#testing", "#testing2"], "message": "Unittest"}, - ) - actor._slack_client = self._slack_mock - - # Mock out the calls to SlackAPI.chat_postMessage().http_post() - post_mock = mock.MagicMock() - post_mock.http_post.side_effect = mock_tornado({"ok": "true"}) - self._slack_mock.chat_postMessage.return_value = post_mock - - ret = yield actor._execute() - self.assertEqual(None, ret) - - # Ensure the calls were made to the API - post_mock.http_post.assert_has_calls( - [ - mock.call( - username="Kingpin", - unfurl_links=True, - text="Unittest", - unfurl_media=True, - parse="none", - link_names=1, - channel="#testing", - ) - ] - ) - - @testing.gen_test - def test_execute_csv_rooms(self): - actor = slack.Message( - "Unit test message", - {"channel": "#testing, #testing2", "message": "Unittest"}, - ) - actor._slack_client = self._slack_mock - - # Mock out the calls to SlackAPI.chat_postMessage().http_post() - post_mock = mock.MagicMock() - post_mock.http_post.side_effect = mock_tornado({"ok": "true"}) - self._slack_mock.chat_postMessage.return_value = post_mock - - ret = yield actor._execute() - self.assertEqual(None, ret) - - # Ensure the calls were made to the API - post_mock.http_post.assert_has_calls( - [ - mock.call( - username="Kingpin", - unfurl_links=True, - text="Unittest", - unfurl_media=True, - parse="none", - link_names=1, - channel="#testing", - ) - ] - ) diff --git a/kingpin/actors/test/test_spotinst.py b/kingpin/actors/test/test_spotinst.py deleted file mode 100644 index e6a139e3..00000000 --- a/kingpin/actors/test/test_spotinst.py +++ /dev/null @@ -1,528 +0,0 @@ -import copy -import mock -import logging -import json - -from tornado import testing -from tornado import httpclient - -from kingpin.actors import exceptions -from kingpin.actors import spotinst -from kingpin.actors.test.helper import mock_tornado, tornado_value - -__author__ = "Matt Wise " - - -class TestSpotinstException(testing.AsyncTestCase): - def test_no_json_in_body(self): - fake_body = "400 Bad Mmmkay" - exc = spotinst.SpotinstException(fake_body) - self.assertEqual("Unknown error: 400 Bad Mmmkay", str(exc)) - - def test_invalid_auth_response(self): - fake_resp_body = mock.MagicMock(name="response_body") - fake_resp_body.body = json.dumps( - { - "request": { - "id": "fake_id", - "url": "/fake", - "method": "GET", - "timestamp": "2016-12-28T22:36:36.324Z", - }, - "response": {"error": "invalid auth", "error_id": "NOAUTH"}, - } - ) - source_exc = httpclient.HTTPError(400, "400 Bad Request", fake_resp_body) - fake_exc = spotinst.SpotinstException(source_exc) - self.assertEqual( - "Spotinst Request ID (fake_id) GET /fake: invalid auth", str(fake_exc) - ) - - def test_group_validation_errors(self): - fake_resp_body = mock.MagicMock(name="response_body") - fake_resp_body.body = json.dumps( - { - "request": { - "id": "fake_id", - "url": "/fake", - "method": "GET", - "timestamp": "2016-12-28T22:36:36.324Z", - }, - "response": { - "errors": [ - { - "message": "Cant create spot requests.", - "code": "GENERAL_ERROR", - }, - { - "message": "AMI ami-16fc4976 with an...", - "code": "UnsupportedOperation", - }, - ] - }, - } - ) - source_exc = httpclient.HTTPError(400, "400 Bad Request", fake_resp_body) - fake_exc = spotinst.SpotinstException(source_exc) - self.assertEqual( - ( - "Spotinst Request ID (fake_id) GET /fake: GENERAL_ERROR: Cant " - "create spot requests., UnsupportedOperation: AMI ami-16fc4976 " - "with an..." - ), - str(fake_exc), - ) - - def test_unknown_error_body(self): - fake_resp_body = mock.MagicMock(name="response_body") - fake_resp_body.body = json.dumps( - { - "request": { - "id": "fake_id", - "url": "/fake", - "method": "GET", - "timestamp": "2016-12-28T22:36:36.324Z", - }, - "response": {"something": "else"}, - } - ) - source_exc = httpclient.HTTPError(400, "400 Bad Request", fake_resp_body) - fake_exc = spotinst.SpotinstException(source_exc) - self.assertEqual( - ("Spotinst Request ID (fake_id) GET /fake: {'something': 'else'}"), - str(fake_exc), - ) - - -class TestSpotinstBase(testing.AsyncTestCase): - """Unit tests for the packagecloud Base actor.""" - - def setUp(self, *args, **kwargs): - super(TestSpotinstBase, self).setUp(*args, **kwargs) - spotinst.TOKEN = "Unittest" - spotinst.DEBUG = True - spotinst.ACCOUNT_ID = "act-test" - - def test_init_with_debug_disabled(self): - spotinst.DEBUG = False - spotinst.SpotinstBase("Unit Test Action", {}) - self.assertEqual(20, logging.getLogger("tornado_rest_client.api").level) - - def test_init_missing_token(self): - spotinst.TOKEN = None - with self.assertRaises(exceptions.InvalidCredentials): - spotinst.SpotinstBase("Unit Test Action", {}) - - def test_init_without_account_id(self): - spotinst.ACCOUNT_ID = None - with self.assertRaises(exceptions.InvalidCredentials): - spotinst.SpotinstBase("Unit Test Action", {}) - - -class TestElastiGroup(testing.AsyncTestCase): - """Unit tests for the ElastiGroup actor.""" - - def setUp(self, *args, **kwargs): - super(TestElastiGroup, self).setUp(*args, **kwargs) - file = "examples/test/spotinst.elastigroup/unittest.json" - spotinst.TOKEN = "Unittest" - spotinst.ACCOUNT_ID = "act-test" - - # Manually inject some fake values for the subnet/secgrp/zone - init_tokens = { - "SECGRP": "sg-123123", - "ZONE": "us-test-1a", - "SUBNET": "sn-123123", - } - - self.actor = spotinst.ElastiGroup( - "unittest", - { - "name": "unittest", - "config": file, - "wait_on_create": True, - "wait_on_roll": True, - }, - init_tokens=init_tokens, - ) - self.actor._client = mock.Mock() - - def test_init_with_string_roll_settings(self): - with self.assertRaises(exceptions.InvalidOptions): - self.actor = spotinst.ElastiGroup( - options={ - "name": "unittest", - "config": "junk", - "roll_batch_size": "some_number", - } - ) - - def test_parse_group_config(self): - self.assertEqual( - (self.actor._config["group"]["compute"]["availabilityZones"][0]["name"]), - "us-test-1a", - ) - self.assertEqual(self.actor._config["group"]["name"], "unittest") - - def test_parse_group_config_no_config(self): - self.actor._options["config"] = None - self.assertEqual(None, self.actor._parse_group_config()) - - def test_parse_group_config_missing_token(self): - del self.actor._init_tokens["ZONE"] - with self.assertRaises(exceptions.InvalidOptions): - self.actor._parse_group_config() - - @testing.gen_test - def test_list_groups(self): - list_of_groups = { - "request": { - "id": "fake_id", - "url": "/fake", - "method": "GET", - "timestamp": "2016-12-28T22:36:36.324Z", - }, - "response": {"items": [{"group": {"name": "test"}}]}, - } - - self.actor._client.aws.ec2.list_groups.http_get = mock_tornado(list_of_groups) - ret = yield self.actor._list_groups() - self.assertEqual(ret, [{"group": {"name": "test"}}]) - - @testing.gen_test - def test_get_group(self): - matching_group = { - "name": "unittest", - "id": "bogus", - } - self.actor._list_groups = mock_tornado([matching_group]) - - ret = yield self.actor._get_group() - self.assertEqual(ret, {"group": matching_group}) - - @testing.gen_test - def test_get_group_too_many_results(self): - matching_group = { - "name": "unittest", - "id": "bogus", - } - self.actor._list_groups = mock_tornado([matching_group, matching_group]) - - with self.assertRaises(exceptions.InvalidOptions): - yield self.actor._get_group() - - @testing.gen_test - def test_get_group_no_groups(self): - self.actor._list_groups = mock_tornado(None) - ret = yield self.actor._get_group() - self.assertEqual(ret, None) - - @testing.gen_test - def test_get_group_no_matching_groups(self): - unmatching_group = { - "name": "unittest-not-matching", - "id": "bogus", - } - self.actor._list_groups = mock_tornado([unmatching_group]) - ret = yield self.actor._get_group() - self.assertEqual(ret, None) - - @testing.gen_test - def test_precache(self): - fake_group = { - "name": "unittest", - "id": "bogus", - "group": {"capacity": {"target": 128}}, - } - self.actor._get_group = mock_tornado(fake_group) - self.actor._validate_group = mock_tornado(None) - yield self.actor._precache() - - # First make sure we stored the group - self.assertEqual(fake_group, self.actor._group) - - # Second, make sure we overwrote the user config's [capacity][target] - # setting with the spotinst value - self.assertEqual(self.actor._config["group"]["capacity"]["target"], 128) - - @testing.gen_test - def test_validate_group(self): - fake_ret = {"ok": True} - mock_client = mock.MagicMock() - mock_client.http_post.return_value = tornado_value(fake_ret) - self.actor._client.aws.ec2.validate_group = mock_client - - yield self.actor._validate_group() - mock_client.http_post.assert_called_with(group=self.actor._config["group"]) - - @testing.gen_test - def test_get_state(self): - self.actor._group = True - ret = yield self.actor._get_state() - self.assertEqual("present", ret) - - @testing.gen_test - def test_get_state_false(self): - self.actor._group = None - ret = yield self.actor._get_state() - self.assertEqual("absent", ret) - - @testing.gen_test - def test_set_state_present(self): - fake_ret = {"ok": True, "response": {"items": [{"desc": "new group"}]}} - mock_client = mock.MagicMock() - mock_client.http_post.return_value = tornado_value(fake_ret) - self.actor._client.aws.ec2.create_group = mock_client - - # Mock out the _get_group method which is called again after the - # create_group API call is made. - self.actor._get_group = mock_tornado(None) - - # Mock out the call to our wait_until_stable() call since thats not in - # the scope of this test. - self.actor._wait_until_stable = mock_tornado(None) - - yield self.actor._set_state() - mock_client.http_post.assert_called_with(group=self.actor._config["group"]) - - @testing.gen_test - def test_set_state_absent(self): - # First, lets copy the desired configuration blob. The first test, - # we'll copy the blob and we'll ensure that they are the same. - self.actor._group = copy.deepcopy(self.actor._config) - - # Insert some fake data that would normally have been returned in the - # included blob from Spotinst. - self.actor._group["group"]["id"] = "sig-1234123" - self.actor._group["group"]["createdAt"] = "timestamp" - self.actor._group["group"]["updatedAt"] = "timestamp" - - fake_ret = {"ok": True} - self.actor._options["state"] = "absent" - mock_client = mock.MagicMock() - mock_client.http_delete.return_value = tornado_value(fake_ret) - self.actor._client.aws.ec2.delete_group.return_value = mock_client - - yield self.actor._set_state() - mock_client.http_delete.assert_called_with() - - @testing.gen_test - def test_compare_config(self): - # First, lets copy the desired configuration blob. The first test, - # we'll copy the blob and we'll ensure that they are the same. - self.actor._group = copy.deepcopy(self.actor._config) - - # Insert some fake data that would normally have been returned in the - # included blob from Spotinst. - self.actor._group["group"]["id"] = "sig-1234123" - self.actor._group["group"]["createdAt"] = "timestamp" - self.actor._group["group"]["updatedAt"] = "timestamp" - - # This should return True because the configs are identical.. - ret = yield self.actor._compare_config() - self.assertEqual(True, ret) - - # Now, lets modify the ElastiGroup config a bit.. the diff should - # return false. - self.actor._group["group"]["description"] = "new description" - ret = yield self.actor._compare_config() - self.assertEqual(False, ret) - - @testing.gen_test - def test_compare_config_not_existing(self): - # Pretend that the group doesn't exist at all in Spotinst - self.actor._group = None - - # This should return True because the config simply doesnt exist in - # Spotinst. The _set_state() method will create it during a real run. - ret = yield self.actor._compare_config() - self.assertEqual(True, ret) - - @testing.gen_test - def test_get_config(self): - self.actor._group = 1 - ret = yield self.actor._get_config() - self.assertEqual(1, ret) - - @testing.gen_test - def test_set_config(self): - # First, lets copy the desired configuration blob. The first test, - # we'll copy the blob and we'll ensure that they are the same. - self.actor._group = copy.deepcopy(self.actor._config) - - # Insert some fake data that would normally have been returned in the - # included blob from Spotinst. - self.actor._group["group"]["id"] = "sig-1234123" - self.actor._group["group"]["createdAt"] = "timestamp" - self.actor._group["group"]["updatedAt"] = "timestamp" - - # Pretend to roll the group if a change is made - self.actor._options["roll_on_change"] = True - self.actor._roll_group = mock.MagicMock() - self.actor._roll_group.return_value = tornado_value(None) - - # Mock out the update_group call.. - fake_ret = {"response": {"items": [{"group": "object"}]}} - mock_client = mock.MagicMock() - mock_client.http_put.return_value = tornado_value(fake_ret) - self.actor._client.aws.ec2.update_group.return_value = mock_client - - yield self.actor._set_config() - - mock_client.http_put.assert_called_with(group=self.actor._config["group"]) - self.assertEqual(self.actor._group["group"], {"group": "object"}) - self.actor._roll_group.assert_has_calls([mock.call]) - - @testing.gen_test - def test_roll_group(self): - # First, lets copy the desired configuration blob. The first test, - # we'll copy the blob and we'll ensure that they are the same. - self.actor._group = copy.deepcopy(self.actor._config) - - # Insert some fake data that would normally have been returned in the - # included blob from Spotinst. - self.actor._group["group"]["id"] = "sig-1234123" - self.actor._group["group"]["createdAt"] = "timestamp" - self.actor._group["group"]["updatedAt"] = "timestamp" - - # Mock out the returned calls from Spotinst. The first time we call the - # roll status method, we'll return no rolls in progress. The second - # time, there will be a single roll in progress (pretending that the - # roll_group call was successful), and then the third time it will - # return no rolls in progress again (pretending that we're done). - in_progress = { - "id": "sbgd-44e6d801", - "status": "in_progress", - "progress": {"unit": "percent", "value": 0}, - "createdAt": "2017-01-05T15:48:28.000+0000", - "updatedAt": "2017-01-05T15:49:15.000+0000", - } - finished = { - "id": "sbgd-9f1aa4f6", - "status": "finished", - "progress": {"unit": "percent", "value": 100}, - "createdAt": "2017-01-05T15:06:25.000+0000", - "updatedAt": "2017-01-05T15:22:17.000+0000", - } - - roll_responses = [ - # First response, no rolls are in progress - tornado_value( - { - "request": { - "id": "request-1-no-in-progress", - "url": "/aws/ec2/group/sig-b55014f1/roll?limit=5", - "method": "GET", - "timestamp": "2017-01-05T15:50:44.215Z", - }, - "response": { - "status": {"code": 200, "message": "OK"}, - "kind": "spotinst:aws:ec2:group:roll", - "items": [finished], - }, - } - ), - # Ok now pretend that one roll is in progress, and one is finished - tornado_value( - { - "request": { - "id": "request-2-one-in-progress", - "url": "/aws/ec2/group/sig-b55014f1/roll?limit=5", - "method": "GET", - "timestamp": "2017-01-05T15:50:44.215Z", - }, - "response": { - "status": {"code": 200, "message": "OK"}, - "kind": "spotinst:aws:ec2:group:roll", - "items": [in_progress, finished], - }, - } - ), - # Finally, no rolls in progress. - tornado_value( - { - "request": { - "id": "request-3-none-in-progress", - "url": "/aws/ec2/group/sig-b55014f1/roll?limit=5", - "method": "GET", - "timestamp": "2017-01-05T15:50:44.215Z", - }, - "response": { - "status": {"code": 200, "message": "OK"}, - "kind": "spotinst:aws:ec2:group:roll", - "items": [finished, finished], - }, - } - ), - ] - - # Generate a basic mock object for the entire 'roll' RestConsumer. Mock - # out the http_get() method to return back the three fake responses - # above. Mock out the http_put() method to just return safely. - roll_mock = mock.MagicMock() - roll_mock.http_get.side_effect = roll_responses - roll_mock.http_put.side_effect = [tornado_value(None)] - self.actor._client.aws.ec2.roll.return_value = roll_mock - - # Make the call, and make sure we wait for all roll operations to - # finish - self.actor._options["wait_on_roll"] = True - yield self.actor._roll_group(delay=0.01) - - # Now verify that all the calls were made to the roll_mock - roll_mock.assert_has_calls( - [ - mock.call.http_get(), - mock.call.http_put(gracePeriod=600, batchSizePercentage=20), - mock.call.http_get(), - mock.call.http_get(), - ] - ) - - @testing.gen_test - def test_wait_until_stable(self): - # First, lets copy the desired configuration blob. The first test, - # we'll copy the blob and we'll ensure that they are the same. - self.actor._group = copy.deepcopy(self.actor._config) - - # Insert some fake data that would normally have been returned in the - # included blob from Spotinst. - self.actor._group["group"]["id"] = "sig-1234123" - self.actor._group["group"]["createdAt"] = "timestamp" - self.actor._group["group"]["updatedAt"] = "timestamp" - - # Mock out what a pending vs fulfilled instance looks like - pending = { - "spotInstanceRequestId": "sir-n8688grq", - "instanceId": None, - "instanceType": "t1.micro", - "product": "Linux/UNIX (Amazon VPC)", - "availabilityZone": "us-west-2a", - "createdAt": "2017-01-03T22:30:56.000Z", - "status": "pending-evaluation", - } - fullfilled = { - "spotInstanceRequestId": "sir-n8688grq", - "instanceId": "i-abcdefg", - "instanceType": "t1.micro", - "product": "Linux/UNIX (Amazon VPC)", - "availabilityZone": "us-west-2a", - "createdAt": "2017-01-03T22:30:56.000Z", - "status": "fullfilled", - } - - # Create a mock for the group_status API call that returns different - # results on the 3rd time its called. The first two times, the - # instances will be in a pending-evaluation state, the third call they - # will be in a fullfilled state. - group_status_mock = mock.MagicMock("group_status") - self.actor._client.aws.ec2.group_status().http_get = group_status_mock - self.actor._client.aws.ec2.group_status().http_get.side_effect = [ - tornado_value({"response": {"items": [pending, pending]}}), - tornado_value({"response": {"items": [pending, pending]}}), - tornado_value({"response": {"items": [fullfilled, fullfilled]}}), - ] - - # Now make the call - yield self.actor._wait_until_stable(delay=0.01) - group_status_mock.assert_has_calls([mock.call(), mock.call(), mock.call()]) diff --git a/kingpin/version.py b/kingpin/version.py index 5bb96bb1..4f828076 100644 --- a/kingpin/version.py +++ b/kingpin/version.py @@ -1,3 +1,3 @@ """Kingpin version number. You must bump this when creating a new release.""" -__version__ = "3.3.1" +__version__ = "4.0.0" diff --git a/requirements.txt b/requirements.txt index 4363b3b9..82d8ab68 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,10 +18,3 @@ boto3==1.9.46 inflection==0.5.1 zipp==3.21.0 configparser==7.1.0 - -# -# ND libraries -# - -# tornado rest client -tornado_rest_client==2.0.2