From 2cbabd70e4abfa224930b3ddf89bfcc6df55cb46 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Wed, 10 May 2017 16:38:10 +1000 Subject: [PATCH] Refactor behave environment definition * environment.py now just contains the behave testing hooks * helper definitions moved to `leapp_testing` support library --- integration-tests/README.md | 14 +- integration-tests/features/environment.py | 307 +----------------- .../features/leapp_testing/__init__.py | 298 +++++++++++++++++ 3 files changed, 319 insertions(+), 300 deletions(-) create mode 100644 integration-tests/features/leapp_testing/__init__.py diff --git a/integration-tests/README.md b/integration-tests/README.md index d1a4b6176..d5330932e 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -116,7 +116,7 @@ any new test steps required. ## Adding new steps to the steps catalog -New step definitions go in the ["features/steps"](./features.steps) +New step definitions go in the ["features/steps"](./features/steps) subdirectory, and use the ["hamcrest"](https://pyhamcrest.readthedocs.io/en/latest/tutorial/) library to define behavioural expectations. @@ -160,6 +160,18 @@ useful attributes for use in step implementations: * `http_helper`: a custom object for checking HTTP(S) responses (see `RequestsHelper` in the environment file for details) + +## Adding new helpers to the test context + +Helper functions and classes for a single set of steps can be included +directly in the Python file defining the steps. + +Helpers that are shared amongst multiple sets of steps should be defined in +the ["features/leapp_testing"](./features/leapp_testing) package, and then +added to the test context using one of the hooks in the +[environment file](./features/environment.py). + + ## Debugging the test VMs From the `integration-tests` directory, an instance of each of the integration diff --git a/integration-tests/features/environment.py b/integration-tests/features/environment.py index 79c1ff342..7d9144f34 100644 --- a/integration-tests/features/environment.py +++ b/integration-tests/features/environment.py @@ -1,302 +1,11 @@ import contextlib -import json -import os -import pathlib -import shutil import subprocess -import time - -from attr import attributes, attrib -from hamcrest import assert_that, equal_to, less_than_or_equal_to - -import requests - -############################## -# General utilities -############################## -_TEST_DIR = pathlib.Path(__file__).parent.parent -_REPO_DIR = _TEST_DIR.parent - -# Command execution helper -def _run_command(cmd, work_dir, ignore_errors): - print(" Running {} in {}".format(cmd, work_dir)) - output = None - try: - output = subprocess.check_output( - cmd, cwd=work_dir, stderr=subprocess.PIPE - ).decode() - except subprocess.CalledProcessError as exc: - output = exc.output.decode() - if not ignore_errors: - print("=== stdout for failed command ===") - print(output) - print("=== stderr for failed command ===") - print(exc.stderr.decode()) - raise - return output - - -############################## -# Local VM management -############################## - -_VM_HOSTNAME_PREFIX = "leapp-tests-" -_VM_DEFS = { - _VM_HOSTNAME_PREFIX + path.name: str(path) - for path in (_TEST_DIR / "vmdefs").iterdir() -} - -class VirtualMachineHelper(object): - """Test step helper to launch and manage VMs - - Currently based specifically on local Vagrant VMs - """ - - def __init__(self): - self.machines = {} - self._resource_manager = contextlib.ExitStack() - - def ensure_local_vm(self, name, definition, destroy=False): - """Ensure a local VM exists based on the given definition - - *name*: name used to refer to the VM in scenario steps - *definition*: directory name in integration-tests/vmdefs - *destroy*: whether or not to destroy any existing VM - """ - hostname = _VM_HOSTNAME_PREFIX + definition - if hostname not in _VM_DEFS: - raise ValueError("Unknown VM image: {}".format(definition)) - if destroy: - self._vm_destroy(hostname) - self._vm_up(name, hostname) - if destroy: - self._resource_manager.callback(self._vm_destroy, name) - else: - self._resource_manager.callback(self._vm_halt, name) - - def get_hostname(self, name): - """Return the expected hostname for the named machine""" - return self.machines[name] - - def close(self): - """Halt or destroy all created VMs""" - self._resource_manager.close() - - @staticmethod - def _run_vagrant(hostname, *args, ignore_errors=False): - # TODO: explore https://pypi.python.org/pypi/python-vagrant - vm_dir = _VM_DEFS[hostname] - cmd = ["vagrant"] - cmd.extend(args) - return _run_command(cmd, vm_dir, ignore_errors) - - def _vm_up(self, name, hostname): - result = self._run_vagrant(hostname, "up", "--provision") - print("Started {} VM instance".format(hostname)) - self.machines[name] = hostname - return result - - def _vm_halt(self, name): - hostname = self.machines.pop(name) - result = self._run_vagrant(hostname, "halt", ignore_errors=True) - print("Suspended {} VM instance".format(hostname)) - return result - - def _vm_destroy(self, name): - hostname = self.machines.pop(name) - result = self._run_vagrant(hostname, "destroy", ignore_errors=True) - print("Destroyed {} VM instance".format(hostname)) - return result - - -############################## -# Leapp commands -############################## - -_LEAPP_SRC_DIR = _REPO_DIR / "src" -_LEAPP_BIN_DIR = _REPO_DIR / "bin" -_LEAPP_TOOL = str(_LEAPP_BIN_DIR / "leapp-tool") - -_SSH_USER = "vagrant" -_SSH_IDENTITY = str(_REPO_DIR / "integration-tests/config/leappto_testing_key") -_DEFAULT_LEAPP_IDENTITY = ['--user', _SSH_USER, '--identity', _SSH_IDENTITY] - -def _install_client(): - """Install the CLI and its dependencies into a Python 2.7 environment""" - py27 = shutil.which("python2.7") - base_cmd = ["pipsi", "--bin-dir", str(_LEAPP_BIN_DIR)] - if pathlib.Path(_LEAPP_TOOL).exists(): - # For some reason, `upgrade` returns 1 even though it appears to work - # so we instead do a full uninstall/reinstall before the test run - uninstall = base_cmd + ["uninstall", "--yes", "leappto"] - _run_command(uninstall, work_dir=str(_REPO_DIR), ignore_errors=False) - install = base_cmd + ["install", "--python", py27, str(_LEAPP_SRC_DIR)] - print(_run_command(install, work_dir=str(_REPO_DIR), ignore_errors=False)) - # Ensure private SSH key is only readable by the current user - os.chmod(_SSH_IDENTITY, 0o600) - -@attributes -class MigrationInfo(object): - """Details of local hosts involved in an app migration command - - *local_vm_count*: Total number of local VMs found during migration - *source_ip*: host accessible IP address found for source VM - *target_ip*: host accessible IP address found for target VM - """ - local_vm_count = attrib() - source_ip = attrib() - target_ip = attrib() - - @classmethod - def from_vm_list(cls, machines, source_host, target_host): - """Build a result given a local VM listing and migration hostnames""" - vm_count = len(machines) - source_ip = target_ip = None - for machine in machines: - if machine["hostname"] == source_host: - source_ip = machine["ip"][0] - if machine["hostname"] == target_host: - target_ip = machine["ip"][0] - if source_ip is not None and target_ip is not None: - break - return cls(vm_count, source_ip, target_ip) - - -class ClientHelper(object): - """Test step helper to invoke the LeApp CLI - - Requires a VirtualMachineHelper instance - """ - - def __init__(self, vm_helper): - self._vm_helper = vm_helper - - def redeploy_as_macrocontainer(self, source_vm, target_vm): - """Recreate source VM as a macrocontainer on given target VM""" - vm_helper = self._vm_helper - source_host = vm_helper.get_hostname(source_vm) - target_host = vm_helper.get_hostname(target_vm) - self._convert_vm_to_macrocontainer(source_host, target_host) - return self._get_migration_host_info(source_host, target_host) - - def check_response_time(self, cmd_args, time_limit, complete_identity=False): - """Check given command completes within the specified time limit - - Returns the contents of stdout as a string. - """ - if complete_identity: - cmd_args.extend(_DEFAULT_LEAPP_IDENTITY) - start = time.monotonic() - cmd_output = self._run_leapp(cmd_args) - response_time = time.monotonic() - start - assert_that(response_time, less_than_or_equal_to(time_limit)) - return cmd_output - - @staticmethod - def _run_leapp(cmd_args): - cmd = [_LEAPP_TOOL] - cmd.extend(cmd_args) - return _run_command(cmd, work_dir=str(_LEAPP_BIN_DIR), ignore_errors=False) - - @classmethod - def _convert_vm_to_macrocontainer(cls, source_host, target_host): - cmd_args = ["migrate-machine"] - cmd_args.extend(_DEFAULT_LEAPP_IDENTITY) - cmd_args.extend(["-t", target_host, source_host]) - result = cls._run_leapp(cmd_args) - msg = "Redeployed {} as macrocontainer on {}" - print(msg.format(source_host, target_host)) - return result - - @classmethod - def _get_migration_host_info(cls, source_host, target_host): - leapp_output = cls._run_leapp(["list-machines", "--shallow"]) - machines = json.loads(leapp_output)["machines"] - return MigrationInfo.from_vm_list(machines, source_host, target_host) - - -############################## -# Service status checking -############################## - -class RequestsHelper(object): - """Test step helper to check HTTP responses""" - - @classmethod - def get_response(cls, service_url, wait_for_connection=None): - """Get HTTP response from given service URL - - Responses are returned as requests.Response objects - - *service_url*: the service URL to query - *wait_for_connection*: number of seconds to wait for a HTTP connection - to the service. `None` indicates that a response - is expected immediately. - """ - deadline = time.monotonic() - if wait_for_connection is None: - fail_msg = "No response from {}".format(service_url) - else: - fail_msg = "No response from {} within {} seconds".format( - service_url, - wait_for_connection - ) - deadline += wait_for_connection - while True: - try: - return requests.get(service_url) - except Exception: - pass - if time.monotonic() >= deadline: - break - raise AssertionError(fail_msg) - - @classmethod - def get_responses(cls, urls_to_check): - """Check responses from multiple given URLs - - Each URL can be either a string (which will be expected to return - a response immediately), or else a (service_url, wait_for_connection) - pair, which is interpreted as described for `get_response()`. - - Response are returned as a dictionary mapping from the service URLs - to requests.Response objects. - """ - # TODO: Use concurrent.futures to check the given URLs in parallel - responses = {} - for url_to_check in urls_to_check: - if isinstance(url_to_check, tuple): - url_to_check, wait_for_connection = url_to_check - else: - wait_for_connection = None - responses[url_to_check] = cls.get_response(url_to_check, - wait_for_connection) - return responses - - @classmethod - def compare_redeployed_response(cls, original_ip, redeployed_ip, *, - tcp_port, status, wait_for_target): - """Compare a pre-migration app response with a redeployed response - - Expects an immediate response from the original IP, and allows for - a delay before the redeployment target starts returning responses - """ - # Get response from source VM - original_url = "http://{}:{}".format(original_ip, tcp_port) - original_response = cls.get_response(original_url) - print("Response received from {}".format(original_url)) - original_status = original_response.status_code - assert_that(original_status, equal_to(status), "Original status") - # Get response from target VM - redeployed_url = "http://{}:{}".format(redeployed_ip, tcp_port) - redeployed_response = cls.get_response(redeployed_url, wait_for_target) - print("Response received from {}".format(redeployed_url)) - # Compare the responses - assert_that(redeployed_response.status_code, equal_to(original_status), "Redeployed status") - original_data = original_response.text - redeployed_data = redeployed_response.text - assert_that(redeployed_data, equal_to(original_data), "Same response") +# behave adds the features directory to sys.path, so import the testing helpers +from leapp_testing import ( + REPO_DIR, TEST_DIR, install_client, + VirtualMachineHelper, ClientHelper, RequestsHelper +) ############################## # Test execution hooks @@ -328,8 +37,8 @@ def _skip_test_group(context, test_group): def before_all(context): # Basic info about the test repository - context.BASE_REPO_DIR = _REPO_DIR - context.BASE_TEST_DIR = _TEST_DIR + context.BASE_REPO_DIR = REPO_DIR + context.BASE_TEST_DIR = TEST_DIR # Some steps require sudo, so for convenience in interactive use, # we ensure we prompt for elevated permissions immediately, @@ -337,7 +46,7 @@ def before_all(context): subprocess.check_output(["sudo", "echo", "Elevated permissions needed"]) # Install the CLI for use in the tests - _install_client() + install_client() # Use contextlib.ExitStack to manage global resources context._global_cleanup = contextlib.ExitStack() diff --git a/integration-tests/features/leapp_testing/__init__.py b/integration-tests/features/leapp_testing/__init__.py new file mode 100644 index 000000000..b50542454 --- /dev/null +++ b/integration-tests/features/leapp_testing/__init__.py @@ -0,0 +1,298 @@ +import contextlib +import json +import os +import pathlib +import shutil +import subprocess +import time + +from attr import attributes, attrib +from hamcrest import assert_that, equal_to, less_than_or_equal_to + +import requests + +############################## +# General utilities +############################## +TEST_DIR = pathlib.Path(__file__).parent.parent.parent +REPO_DIR = TEST_DIR.parent + +# Command execution helper +def _run_command(cmd, work_dir, ignore_errors): + print(" Running {} in {}".format(cmd, work_dir)) + output = None + try: + output = subprocess.check_output( + cmd, cwd=work_dir, stderr=subprocess.PIPE + ).decode() + except subprocess.CalledProcessError as exc: + output = exc.output.decode() + if not ignore_errors: + print("=== stdout for failed command ===") + print(output) + print("=== stderr for failed command ===") + print(exc.stderr.decode()) + raise + return output + + +############################## +# Local VM management +############################## + +_VM_HOSTNAME_PREFIX = "leapp-tests-" +_VM_DEFS = { + _VM_HOSTNAME_PREFIX + path.name: str(path) + for path in (TEST_DIR / "vmdefs").iterdir() +} + +class VirtualMachineHelper(object): + """Test step helper to launch and manage VMs + + Currently based specifically on local Vagrant VMs + """ + + def __init__(self): + self.machines = {} + self._resource_manager = contextlib.ExitStack() + + def ensure_local_vm(self, name, definition, destroy=False): + """Ensure a local VM exists based on the given definition + + *name*: name used to refer to the VM in scenario steps + *definition*: directory name in integration-tests/vmdefs + *destroy*: whether or not to destroy any existing VM + """ + hostname = _VM_HOSTNAME_PREFIX + definition + if hostname not in _VM_DEFS: + raise ValueError("Unknown VM image: {}".format(definition)) + if destroy: + self._vm_destroy(hostname) + self._vm_up(name, hostname) + if destroy: + self._resource_manager.callback(self._vm_destroy, name) + else: + self._resource_manager.callback(self._vm_halt, name) + + def get_hostname(self, name): + """Return the expected hostname for the named machine""" + return self.machines[name] + + def close(self): + """Halt or destroy all created VMs""" + self._resource_manager.close() + + @staticmethod + def _run_vagrant(hostname, *args, ignore_errors=False): + # TODO: explore https://pypi.python.org/pypi/python-vagrant + vm_dir = _VM_DEFS[hostname] + cmd = ["vagrant"] + cmd.extend(args) + return _run_command(cmd, vm_dir, ignore_errors) + + def _vm_up(self, name, hostname): + result = self._run_vagrant(hostname, "up", "--provision") + print("Started {} VM instance".format(hostname)) + self.machines[name] = hostname + return result + + def _vm_halt(self, name): + hostname = self.machines.pop(name) + result = self._run_vagrant(hostname, "halt", ignore_errors=True) + print("Suspended {} VM instance".format(hostname)) + return result + + def _vm_destroy(self, name): + hostname = self.machines.pop(name) + result = self._run_vagrant(hostname, "destroy", ignore_errors=True) + print("Destroyed {} VM instance".format(hostname)) + return result + + +############################## +# Leapp commands +############################## + +_LEAPP_SRC_DIR = REPO_DIR / "src" +_LEAPP_BIN_DIR = REPO_DIR / "bin" +_LEAPP_TOOL = str(_LEAPP_BIN_DIR / "leapp-tool") + +_SSH_USER = "vagrant" +_SSH_IDENTITY = str(REPO_DIR / "integration-tests/config/leappto_testing_key") +_DEFAULT_LEAPP_IDENTITY = ['--user', _SSH_USER, '--identity', _SSH_IDENTITY] + +def install_client(): + """Install the CLI and its dependencies into a Python 2.7 environment""" + py27 = shutil.which("python2.7") + base_cmd = ["pipsi", "--bin-dir", str(_LEAPP_BIN_DIR)] + if pathlib.Path(_LEAPP_TOOL).exists(): + # For some reason, `upgrade` returns 1 even though it appears to work + # so we instead do a full uninstall/reinstall before the test run + uninstall = base_cmd + ["uninstall", "--yes", "leappto"] + _run_command(uninstall, work_dir=str(REPO_DIR), ignore_errors=False) + install = base_cmd + ["install", "--python", py27, str(_LEAPP_SRC_DIR)] + print(_run_command(install, work_dir=str(REPO_DIR), ignore_errors=False)) + # Ensure private SSH key is only readable by the current user + os.chmod(_SSH_IDENTITY, 0o600) + +@attributes +class MigrationInfo(object): + """Details of local hosts involved in an app migration command + + *local_vm_count*: Total number of local VMs found during migration + *source_ip*: host accessible IP address found for source VM + *target_ip*: host accessible IP address found for target VM + """ + local_vm_count = attrib() + source_ip = attrib() + target_ip = attrib() + + @classmethod + def from_vm_list(cls, machines, source_host, target_host): + """Build a result given a local VM listing and migration hostnames""" + vm_count = len(machines) + source_ip = target_ip = None + for machine in machines: + if machine["hostname"] == source_host: + source_ip = machine["ip"][0] + if machine["hostname"] == target_host: + target_ip = machine["ip"][0] + if source_ip is not None and target_ip is not None: + break + return cls(vm_count, source_ip, target_ip) + + +class ClientHelper(object): + """Test step helper to invoke the LeApp CLI + + Requires a VirtualMachineHelper instance + """ + + def __init__(self, vm_helper): + self._vm_helper = vm_helper + + def redeploy_as_macrocontainer(self, source_vm, target_vm): + """Recreate source VM as a macrocontainer on given target VM""" + vm_helper = self._vm_helper + source_host = vm_helper.get_hostname(source_vm) + target_host = vm_helper.get_hostname(target_vm) + self._convert_vm_to_macrocontainer(source_host, target_host) + return self._get_migration_host_info(source_host, target_host) + + def check_response_time(self, cmd_args, time_limit, complete_identity=False): + """Check given command completes within the specified time limit + + Returns the contents of stdout as a string. + """ + if complete_identity: + cmd_args.extend(_DEFAULT_LEAPP_IDENTITY) + start = time.monotonic() + cmd_output = self._run_leapp(cmd_args) + response_time = time.monotonic() - start + assert_that(response_time, less_than_or_equal_to(time_limit)) + return cmd_output + + @staticmethod + def _run_leapp(cmd_args): + cmd = [_LEAPP_TOOL] + cmd.extend(cmd_args) + return _run_command(cmd, work_dir=str(_LEAPP_BIN_DIR), ignore_errors=False) + + @classmethod + def _convert_vm_to_macrocontainer(cls, source_host, target_host): + cmd_args = ["migrate-machine"] + cmd_args.extend(_DEFAULT_LEAPP_IDENTITY) + cmd_args.extend(["-t", target_host, source_host]) + result = cls._run_leapp(cmd_args) + msg = "Redeployed {} as macrocontainer on {}" + print(msg.format(source_host, target_host)) + return result + + @classmethod + def _get_migration_host_info(cls, source_host, target_host): + leapp_output = cls._run_leapp(["list-machines", "--shallow"]) + machines = json.loads(leapp_output)["machines"] + return MigrationInfo.from_vm_list(machines, source_host, target_host) + + +############################## +# Service status checking +############################## + +class RequestsHelper(object): + """Test step helper to check HTTP responses""" + + @classmethod + def get_response(cls, service_url, wait_for_connection=None): + """Get HTTP response from given service URL + + Responses are returned as requests.Response objects + + *service_url*: the service URL to query + *wait_for_connection*: number of seconds to wait for a HTTP connection + to the service. `None` indicates that a response + is expected immediately. + """ + deadline = time.monotonic() + if wait_for_connection is None: + fail_msg = "No response from {}".format(service_url) + else: + fail_msg = "No response from {} within {} seconds".format( + service_url, + wait_for_connection + ) + deadline += wait_for_connection + while True: + try: + return requests.get(service_url) + except Exception: + pass + if time.monotonic() >= deadline: + break + raise AssertionError(fail_msg) + + @classmethod + def get_responses(cls, urls_to_check): + """Check responses from multiple given URLs + + Each URL can be either a string (which will be expected to return + a response immediately), or else a (service_url, wait_for_connection) + pair, which is interpreted as described for `get_response()`. + + Response are returned as a dictionary mapping from the service URLs + to requests.Response objects. + """ + # TODO: Use concurrent.futures to check the given URLs in parallel + responses = {} + for url_to_check in urls_to_check: + if isinstance(url_to_check, tuple): + url_to_check, wait_for_connection = url_to_check + else: + wait_for_connection = None + responses[url_to_check] = cls.get_response(url_to_check, + wait_for_connection) + return responses + + @classmethod + def compare_redeployed_response(cls, original_ip, redeployed_ip, *, + tcp_port, status, wait_for_target): + """Compare a pre-migration app response with a redeployed response + + Expects an immediate response from the original IP, and allows for + a delay before the redeployment target starts returning responses + """ + # Get response from source VM + original_url = "http://{}:{}".format(original_ip, tcp_port) + original_response = cls.get_response(original_url) + print("Response received from {}".format(original_url)) + original_status = original_response.status_code + assert_that(original_status, equal_to(status), "Original status") + # Get response from target VM + redeployed_url = "http://{}:{}".format(redeployed_ip, tcp_port) + redeployed_response = cls.get_response(redeployed_url, wait_for_target) + print("Response received from {}".format(redeployed_url)) + # Compare the responses + assert_that(redeployed_response.status_code, equal_to(original_status), "Redeployed status") + original_data = original_response.text + redeployed_data = redeployed_response.text + assert_that(redeployed_data, equal_to(original_data), "Same response")