From 6de3af7facba170edac3308098d9b56d58b75a7b Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Wed, 3 Apr 2024 15:45:57 +0800 Subject: [PATCH 01/24] feat: support base images --- config.yaml | 7 +++ scripts/build-lxd-image.sh | 13 ++-- src-docs/charm_state.py.md | 63 +++++++++++++++---- src-docs/runner_manager.py.md | 4 +- src/charm.py | 2 +- src/charm_state.py | 100 ++++++++++++++++++++++++++---- src/runner_manager.py | 1 + tests/unit/test_runner_manager.py | 2 +- 8 files changed, 158 insertions(+), 34 deletions(-) diff --git a/config.yaml b/config.yaml index 670315ca8..9628df004 100644 --- a/config.yaml +++ b/config.yaml @@ -2,6 +2,13 @@ # See LICENSE file for licensing details. options: + base-image: + type: string + default: "jammy" + description: >- + The base ubuntu OS image to use for the runners. Codename (e.g. "jammy") or version tag + (e.g. 22.04) is supported as input. Currently only supports LTS versions of jammy and higher, + i.e. jammy, noble. denylist: type: string default: "" diff --git a/scripts/build-lxd-image.sh b/scripts/build-lxd-image.sh index 96caed895..2084bb2dc 100644 --- a/scripts/build-lxd-image.sh +++ b/scripts/build-lxd-image.sh @@ -51,7 +51,8 @@ cleanup() { HTTP_PROXY="$1" HTTPS_PROXY="$2" NO_PROXY="$3" -MODE="$4" +BASE_IMAGE="$4" +MODE="$5" if [[ -n "$HTTP_PROXY" ]]; then /snap/bin/lxc config set core.proxy_http "$HTTP_PROXY" @@ -63,9 +64,9 @@ fi cleanup '/snap/bin/lxc info builder &> /dev/null' '/snap/bin/lxc delete builder --force' 'Cleanup LXD VM of previous run' 10 if [[ "$MODE" == "test" ]]; then - retry '/snap/bin/lxc launch ubuntu-daily:jammy builder --device root,size=5GiB' 'Starting LXD container' + retry "/snap/bin/lxc launch ubuntu-daily:$BASE_IMAGE builder --device root,size=5GiB" 'Starting LXD container' else - retry '/snap/bin/lxc launch ubuntu-daily:jammy builder --vm --device root,size=8GiB' 'Starting LXD VM' + retry "/snap/bin/lxc launch ubuntu-daily:$BASE_IMAGE builder --vm --device root,size=8GiB" 'Starting LXD VM' fi retry '/snap/bin/lxc exec builder -- /usr/bin/who' 'Wait for lxd agent to be ready' 30 if [[ -n "$HTTP_PROXY" ]]; then @@ -146,9 +147,9 @@ fi /snap/bin/lxc publish builder --alias builder --reuse -f # Swap in the built image -/snap/bin/lxc image alias rename jammy old-jammy || true -/snap/bin/lxc image alias rename builder jammy -/snap/bin/lxc image delete old-jammy || true +/snap/bin/lxc image alias rename $BASE_IMAGE old-$BASE_IMAGE || true +/snap/bin/lxc image alias rename builder $BASE_IMAGE +/snap/bin/lxc image delete old-$BASE_IMAGE || true # Clean up LXD instance cleanup '/snap/bin/lxc info builder &> /dev/null' '/snap/bin/lxc delete builder --force' 'Cleanup LXD instance' 10 diff --git a/src-docs/charm_state.py.md b/src-docs/charm_state.py.md index 84577b6b4..7e2681a18 100644 --- a/src-docs/charm_state.py.md +++ b/src-docs/charm_state.py.md @@ -9,14 +9,16 @@ State of the Charm. --------------- - **ARCHITECTURES_ARM64** - **ARCHITECTURES_X86** +- **BASE_IMAGE_CONFIG_NAME** - **OPENSTACK_CLOUDS_YAML_CONFIG_NAME** - **LABELS_CONFIG_NAME** - **COS_AGENT_INTEGRATION_NAME** - **DEBUG_SSH_INTEGRATION_NAME** +- **LTS_IMAGE_VERSION_TAG_MAP** --- - + ## function `parse_github_path` @@ -46,6 +48,15 @@ Supported system architectures. +--- + +## class `BaseImage` +The ubuntu OS base image to build and deploy runners on. + + + + + --- ## class `CharmConfig` @@ -70,7 +81,7 @@ Some charm configurations are grouped into other configuration models. --- - + ### classmethod `check_fields` @@ -93,7 +104,7 @@ Validate the general charm configuration. --- - + ### classmethod `from_charm` @@ -126,7 +137,7 @@ Raised when charm config is invalid. - `msg`: Explanation of the error. - + ### function `__init__` @@ -166,7 +177,7 @@ The charm state. --- - + ### classmethod `from_charm` @@ -205,7 +216,7 @@ Represent GitHub organization. --- - + ### function `path` @@ -238,7 +249,7 @@ Represent GitHub repository. --- - + ### function `path` @@ -254,6 +265,31 @@ Return a string representing the path. Path to the GitHub entity. +--- + +## class `ImmutableConfigChangedError` +Represents an error when changing immutable charm state. + + + +### function `__init__` + +```python +__init__(msg: str) +``` + +Initialize a new instance of the ImmutableConfigChangedError exception. + + + +**Args:** + + - `msg`: Explanation of the error. + + + + + --- ## class `ProxyConfig` @@ -279,7 +315,7 @@ Return the aproxy address. --- - + ### classmethod `check_fields` @@ -302,7 +338,7 @@ Validate the proxy configuration. --- - + ### classmethod `from_charm` @@ -333,6 +369,7 @@ Runner configurations for the charm. **Attributes:** + - `base_image`: The ubuntu base image to run the runner viertual machines on. - `virtual_machines`: Number of virtual machine-based runner to spawn. - `virtual_machine_resources`: Hardware resource used by one virtual machine for a runner. - `runner_storage`: Storage to be used as disk for the runner. @@ -342,7 +379,7 @@ Runner configurations for the charm. --- - + ### classmethod `check_fields` @@ -365,7 +402,7 @@ Validate the runner configuration. --- - + ### classmethod `from_charm` @@ -415,7 +452,7 @@ SSH connection information for debug workflow. --- - + ### classmethod `from_charm` @@ -443,7 +480,7 @@ Raised when given machine charm architecture is unsupported. - `arch`: The current machine architecture. - + ### function `__init__` diff --git a/src-docs/runner_manager.py.md b/src-docs/runner_manager.py.md index 1b4c074bd..4dbbf705a 100644 --- a/src-docs/runner_manager.py.md +++ b/src-docs/runner_manager.py.md @@ -43,7 +43,7 @@ Construct RunnerManager object for creating and managing runners. --- - + ### function `build_runner_image` @@ -175,7 +175,7 @@ Bring runners in line with target. --- - + ### function `schedule_build_runner_image` diff --git a/src/charm.py b/src/charm.py index e595d195a..c668c4e71 100755 --- a/src/charm.py +++ b/src/charm.py @@ -333,7 +333,7 @@ def _get_runner_manager( RunnerManagerConfig( charm_state=state, dockerhub_mirror=state.charm_config.dockerhub_mirror, - image="jammy", + image=state.runner_config.base_image.value, lxd_storage_path=lxd_storage_path, path=path, service_token=self.service_token, diff --git a/src/charm_state.py b/src/charm_state.py index fd475f1b8..8f8d22ec1 100644 --- a/src/charm_state.py +++ b/src/charm_state.py @@ -30,6 +30,7 @@ CHARM_STATE_PATH = Path("charm_state.json") +BASE_IMAGE_CONFIG_NAME = "base-image" OPENSTACK_CLOUDS_YAML_CONFIG_NAME = "experimental-openstack-clouds-yaml" LABELS_CONFIG_NAME = "labels" @@ -341,15 +342,48 @@ def check_fields(cls, values: dict) -> dict: return values +LTS_IMAGE_VERSION_TAG_MAP = {"22.04": "jammy", "24.04": "noble"} + + +class BaseImage(str, Enum): + """The ubuntu OS base image to build and deploy runners on.""" + + JAMMY = "jammy" + NOBLE = "noble" + + @classmethod + def from_charm(cls, charm: CharmBase) -> "BaseImage": + """Retrieve the base image tag from charm. + + Args: + charm: The charm instance. + + Raises: + ValueError if an unsupporte base image is passed in. + + Returns: + The base image configuration of the charm. + """ + image_name = charm.config.get(BASE_IMAGE_CONFIG_NAME, "jammy").lower().strip() + try: + return cls(image_name) + except ValueError as exc: + if image_name in LTS_IMAGE_VERSION_TAG_MAP: + return cls(LTS_IMAGE_VERSION_TAG_MAP[image_name]) + raise ValueError(f"Unsupported base image: {image_name}") from exc + + class RunnerCharmConfig(BaseModel): """Runner configurations for the charm. Attributes: + base_image: The ubuntu base image to run the runner viertual machines on. virtual_machines: Number of virtual machine-based runner to spawn. virtual_machine_resources: Hardware resource used by one virtual machine for a runner. runner_storage: Storage to be used as disk for the runner. """ + base_image: BaseImage virtual_machines: int virtual_machine_resources: VirtualMachineResources runner_storage: RunnerStorage @@ -364,6 +398,11 @@ def from_charm(cls, charm: CharmBase) -> "RunnerCharmConfig": Returns: Current config of the charm. """ + try: + base_image = BaseImage.from_charm(charm) + except ValueError as err: + raise CharmConfigInvalidError("Invalid base image") from err + try: runner_storage = RunnerStorage(charm.config["runner-storage"]) except ValueError as err: @@ -386,6 +425,7 @@ def from_charm(cls, charm: CharmBase) -> "RunnerCharmConfig": ) return cls( + base_image=base_image, virtual_machines=virtual_machines, virtual_machine_resources=virtual_machine_resources, runner_storage=runner_storage, @@ -591,6 +631,18 @@ def from_charm(cls, charm: CharmBase) -> list["SSHDebugConnection"]: return ssh_debug_connections +class ImmutableConfigChangedError(Exception): + """Represents an error when changing immutable charm state.""" + + def __init__(self, msg: str): + """Initialize a new instance of the ImmutableConfigChangedError exception. + + Args: + msg: Explanation of the error. + """ + self.msg = msg + + @dataclasses.dataclass(frozen=True) class CharmState: """The charm state. @@ -610,6 +662,36 @@ class CharmState: runner_config: RunnerCharmConfig ssh_debug_connections: list[SSHDebugConnection] + @classmethod + def _check_immutable_config_change( + cls, prev_state: dict | None, runner_storage: RunnerStorage, base_image: BaseImage + ) -> None: + """Ensure immutable config has not changed. + + Raises: + ImmutableConfigChangedError: If an immutable configuration has changed. + """ + if not prev_state: + return + if prev_state["runner_config"]["runner_storage"] != runner_storage: + logger.warning( + "Storage option changed from %s to %s, blocking the charm", + prev_state["runner_config"]["runner_storage"], + runner_storage, + ) + raise ImmutableConfigChangedError( + msg="runner-storage config cannot be changed after deployment, redeploy if needed" + ) + if prev_state["runner_config"]["base_image"] != base_image.value: + logger.warning( + "Base image option changed from %s to %s, blocking the charm", + prev_state["runner_config"]["base_image"], + runner_storage, + ) + raise ImmutableConfigChangedError( + msg="base-image config cannot be changed after deployment, redeploy if needed" + ) + @classmethod def from_charm(cls, charm: CharmBase) -> "CharmState": """Initialize the state from charm. @@ -644,18 +726,14 @@ def from_charm(cls, charm: CharmBase) -> "CharmState": logger.error("Invalid charm config: %s", exc) raise CharmConfigInvalidError(f"Invalid configuration: {str(exc)}") from exc - if ( - prev_state is not None - and prev_state["runner_config"]["runner_storage"] != runner_config.runner_storage - ): - logger.warning( - "Storage option changed from %s to %s, blocking the charm", - prev_state["runner_config"]["runner_storage"], - runner_config.runner_storage, - ) - raise CharmConfigInvalidError( - "runner-storage config cannot be changed after deployment, redeploy if needed" + try: + cls._check_immutable_config_change( + prev_state=prev_state, + runner_storage=runner_config.runner_storage, + base_image=runner_config.base_image, ) + except ImmutableConfigChangedError as exc: + raise CharmConfigInvalidError(exc.msg) from exc try: arch = _get_supported_arch() diff --git a/src/runner_manager.py b/src/runner_manager.py index 8a8992b1c..82a6dcfe4 100644 --- a/src/runner_manager.py +++ b/src/runner_manager.py @@ -733,6 +733,7 @@ def _build_image_command(self) -> list[str]: http_proxy, https_proxy, no_proxy, + self.config.image, ] if LXD_PROFILE_YAML.exists(): cmd += ["test"] diff --git a/tests/unit/test_runner_manager.py b/tests/unit/test_runner_manager.py index ea6620ae6..0ee81c043 100644 --- a/tests/unit/test_runner_manager.py +++ b/tests/unit/test_runner_manager.py @@ -506,4 +506,4 @@ def test_schedule_build_runner_image( cmd = f"/usr/bin/bash {BUILD_IMAGE_SCRIPT_FILENAME.absolute()} {http} {https} {no_proxy}" assert cronfile.exists() - assert cronfile.read_text() == f"4 4,10,16,22 * * * ubuntu {cmd}\n" + assert cronfile.read_text() == f"4 4,10,16,22 * * * ubuntu {cmd} jammy\n" From c82b47ae4c5a6054841101e4ab97d8d7a120cf7e Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Mon, 8 Apr 2024 21:34:38 +0800 Subject: [PATCH 02/24] test: openstack base image --- .github/workflows/integration_test.yaml | 15 ++++-- scripts/build-openstack-image.sh | 11 ++-- src-docs/charm_state.py.md | 16 +++--- src-docs/openstack_manager.md | 44 +++++++++++++--- src/charm.py | 7 ++- src/charm_state.py | 4 ++ src/openstack_cloud/openstack_manager.py | 53 +++++++++++++++----- tests/integration/conftest.py | 4 +- tests/integration/test_charm_one_runner.py | 42 ++++++++++++++++ tests/integration/test_openstack.py | 44 +++++++++++++++- tests/unit/test_openstack_manager.py | 58 ++++++++++++++++------ 11 files changed, 242 insertions(+), 56 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 07a1ca231..73cda1468 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -3,6 +3,15 @@ name: integration-tests on: pull_request: +env: + LXD_TEST_MODULES: >- + '["test_charm_fork_repo", "test_charm_no_runner", "test_charm_scheduled_events", + "test_charm_one_runner", "test_charm_metrics_success", "test_charm_metrics_failure", + "test_self_hosted_runner", "test_charm_with_proxy", "test_charm_with_juju_storage", + "test_debug_ssh"]' + OPENSTACK_TEST_MODULES: >- + '["test_openstack"]' + jobs: # test option values defined at test/conftest.py are passed on via repository secret # INTEGRATION_TEST_ARGS to operator-workflows automatically. @@ -15,7 +24,7 @@ jobs: pre-run-script: scripts/pre-integration-test.sh provider: lxd test-tox-env: integration-juju2.9 - modules: '["test_charm_fork_repo", "test_charm_no_runner", "test_charm_scheduled_events", "test_charm_one_runner", "test_charm_metrics_success", "test_charm_metrics_failure", "test_self_hosted_runner", "test_charm_with_proxy", "test_charm_with_juju_storage", "test_debug_ssh"]' + modules: ${{ env.LXD_TEST_MODULES }} integration-tests: name: Integration test with juju 3.1 uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main @@ -25,7 +34,7 @@ jobs: pre-run-script: scripts/pre-integration-test.sh provider: lxd test-tox-env: integration-juju3.1 - modules: '["test_charm_fork_repo", "test_charm_no_runner", "test_charm_scheduled_events", "test_charm_one_runner", "test_charm_metrics_success", "test_charm_metrics_failure", "test_self_hosted_runner", "test_charm_with_proxy", "test_charm_with_juju_storage", "test_debug_ssh"]' + modules: ${{ env.LXD_TEST_MODULES }} # openstack tests use microstack, whose setup is kind of special # - due to the huge resource requirements, we use self-hosted runners for these tests # - microstack requires juju 3.2 and microk8s 1.26 @@ -42,6 +51,6 @@ jobs: channel: 1.26-strict/stable microk8s-addons: "dns ingress hostpath-storage" test-tox-env: integration-juju3.2 - modules: '["test_openstack"]' + modules: ${{ env.OPENSTACK_TEST_MODULES }} self-hosted-runner: true self-hosted-runner-label: two-xlarge diff --git a/scripts/build-openstack-image.sh b/scripts/build-openstack-image.sh index d3e41b615..5914a4aef 100755 --- a/scripts/build-openstack-image.sh +++ b/scripts/build-openstack-image.sh @@ -14,6 +14,7 @@ HTTPS_PROXY="$3" NO_PROXY="$4" DOCKER_PROXY_SERVICE_CONF="$5" DOCKER_PROXY_CONF="$6" +BASE_IMAGE="$7" # retry function retry() { @@ -108,15 +109,15 @@ sudo modprobe nbd # cleanup any existing mounts cleanup -retry "sudo wget https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-$BIN_ARCH.img \ - -O jammy-server-cloudimg-$BIN_ARCH.img" "Downloading cloud image" 3 +retry "sudo wget https://cloud-images.ubuntu.com/$BASE_IMAGE/current/$BASE_IMAGE-server-cloudimg-$BIN_ARCH.img \ + -O $BASE_IMAGE-server-cloudimg-$BIN_ARCH.img" "Downloading cloud image" 3 # resize image - installing dependencies requires more disk space -sudo qemu-img resize jammy-server-cloudimg-$BIN_ARCH.img +1.5G +sudo qemu-img resize $BASE_IMAGE-server-cloudimg-$BIN_ARCH.img +1.5G # mount nbd echo "Connecting network block device to image" -sudo qemu-nbd --connect=/dev/nbd0 jammy-server-cloudimg-$BIN_ARCH.img +sudo qemu-nbd --connect=/dev/nbd0 $BASE_IMAGE-server-cloudimg-$BIN_ARCH.img sudo mkdir -p /mnt/ubuntu-image retry "sudo mount -o rw /dev/nbd0p1 /mnt/ubuntu-image" "Mounting nbd0p1 device" 3 @@ -208,4 +209,4 @@ sudo sync cleanup # Reduce image size by removing sparse space & compressing -sudo virt-sparsify --compress jammy-server-cloudimg-$BIN_ARCH.img jammy-server-cloudimg-$BIN_ARCH-compressed.img +sudo virt-sparsify --compress $BASE_IMAGE-server-cloudimg-$BIN_ARCH.img $BASE_IMAGE-server-cloudimg-$BIN_ARCH-compressed.img diff --git a/src-docs/charm_state.py.md b/src-docs/charm_state.py.md index 7e2681a18..44d0e0d97 100644 --- a/src-docs/charm_state.py.md +++ b/src-docs/charm_state.py.md @@ -177,7 +177,7 @@ The charm state. --- - + ### classmethod `from_charm` @@ -270,7 +270,7 @@ Return a string representing the path. ## class `ImmutableConfigChangedError` Represents an error when changing immutable charm state. - + ### function `__init__` @@ -315,7 +315,7 @@ Return the aproxy address. --- - + ### classmethod `check_fields` @@ -338,7 +338,7 @@ Validate the proxy configuration. --- - + ### classmethod `from_charm` @@ -379,7 +379,7 @@ Runner configurations for the charm. --- - + ### classmethod `check_fields` @@ -402,7 +402,7 @@ Validate the runner configuration. --- - + ### classmethod `from_charm` @@ -452,7 +452,7 @@ SSH connection information for debug workflow. --- - + ### classmethod `from_charm` @@ -480,7 +480,7 @@ Raised when given machine charm architecture is unsupported. - `arch`: The current machine architecture. - + ### function `__init__` diff --git a/src-docs/openstack_manager.md b/src-docs/openstack_manager.md index ef228b256..36b8b8910 100644 --- a/src-docs/openstack_manager.md +++ b/src-docs/openstack_manager.md @@ -13,17 +13,16 @@ Module for handling interactions with OpenStack. --- - + ## function `build_image` ```python build_image( - arch: Arch, cloud_config: dict[str, dict], github_client: GithubClient, path: GithubOrg | GithubRepo, - proxies: Optional[ProxyConfig] = None + config: BuildImageConfig ) → str ``` @@ -36,7 +35,7 @@ Build and upload an image to OpenStack. - `cloud_config`: The cloud configuration to connect OpenStack with. - `github_client`: The Github client to interact with Github API. - `path`: Github organisation or repository path. - - `proxies`: HTTP proxy settings. + - `config`: The image build configuration values. @@ -52,7 +51,7 @@ Build and upload an image to OpenStack. --- - + ## function `create_instance_config` @@ -79,7 +78,7 @@ Create an instance config from charm data. --- - + ## function `create_instance` @@ -111,7 +110,7 @@ Create an OpenStack instance. --- - + ## class `ProxyStringValues` Wrapper class to proxy values to string. @@ -130,7 +129,7 @@ Wrapper class to proxy values to string. --- - + ## class `InstanceConfig` The configuration values for creating a single runner instance. @@ -167,3 +166,32 @@ __init__( +--- + + + +## class `BuildImageConfig` +The configuration values for building openstack image. + +Attrs: arch: The image architecture to build for. base_image: The ubuntu image to use as image build base. proxies: HTTP proxy settings. + + + +### method `__init__` + +```python +__init__( + arch: Arch, + base_image: BaseImage, + proxies: Optional[ProxyConfig] = None +) → None +``` + + + + + + + + + diff --git a/src/charm.py b/src/charm.py index c668c4e71..960be4da7 100755 --- a/src/charm.py +++ b/src/charm.py @@ -371,11 +371,14 @@ def _on_install(self, _event: InstallEvent) -> None: self.unit.status = MaintenanceStatus("Building Openstack image") github = GithubClient(token=state.charm_config.token) image = openstack_manager.build_image( - arch=state.arch, cloud_config=state.charm_config.openstack_clouds_yaml, github_client=github, path=state.charm_config.path, - proxies=state.proxy_config, + config=openstack_manager.BuildImageConfig( + arch=state.arch, + base_image=state.runner_config.base_image, + proxies=state.proxy_config, + ), ) instance_config = openstack_manager.create_instance_config( unit_name=self.unit.name, diff --git a/src/charm_state.py b/src/charm_state.py index 8f8d22ec1..ccc9630f3 100644 --- a/src/charm_state.py +++ b/src/charm_state.py @@ -351,6 +351,10 @@ class BaseImage(str, Enum): JAMMY = "jammy" NOBLE = "noble" + def __str__(self) -> str: + """Interpolate to string value.""" + return self.value + @classmethod def from_charm(cls, charm: CharmBase) -> "BaseImage": """Retrieve the base image tag from charm. diff --git a/src/openstack_cloud/openstack_manager.py b/src/openstack_cloud/openstack_manager.py index 58bd30ad8..02e487d5d 100644 --- a/src/openstack_cloud/openstack_manager.py +++ b/src/openstack_cloud/openstack_manager.py @@ -17,7 +17,13 @@ import openstack.image.v2.image from openstack.exceptions import OpenStackCloudException -from charm_state import Arch, ProxyConfig, SSHDebugConnection, UnsupportedArchitectureError +from charm_state import ( + Arch, + BaseImage, + ProxyConfig, + SSHDebugConnection, + UnsupportedArchitectureError, +) from errors import ( OpenstackImageBuildError, OpenstackInstanceLaunchError, @@ -148,13 +154,16 @@ def _generate_docker_client_proxy_config_json(http_proxy: str, https_proxy: str, def _build_image_command( - runner_info: RunnerApplication, proxies: Optional[ProxyConfig] = None + runner_info: RunnerApplication, + base_image: BaseImage, + proxies: Optional[ProxyConfig] = None, ) -> list[str]: """Get command for building runner image. Args: runner_info: The runner application to fetch runner tar download url. proxies: HTTP proxy settings. + base: The ubuntu base image to use. Returns: Command to execute to build runner image. @@ -178,6 +187,7 @@ def _build_image_command( proxy_values.no_proxy, docker_proxy_service_conf_content, docker_client_proxy_content, + str(base_image), ] return cmd @@ -202,7 +212,7 @@ class InstanceConfig: openstack_image: openstack.image.v2.image.Image -def _get_supported_runner_arch(arch: str) -> Literal["amd64", "arm64"]: +def _get_supported_runner_arch(arch: Arch) -> Literal["amd64", "arm64"]: """Validate and return supported runner architecture. The supported runner architecture takes in arch value from Github supported architecture and @@ -212,7 +222,7 @@ def _get_supported_runner_arch(arch: str) -> Literal["amd64", "arm64"]: and https://cloud-images.ubuntu.com/jammy/current/ Args: - arch: str + arch: The compute architecture to check support for. Raises: UnsupportedArchitectureError: If an unsupported architecture was passed. @@ -221,20 +231,34 @@ def _get_supported_runner_arch(arch: str) -> Literal["amd64", "arm64"]: The supported architecture. """ match arch: - case "x64": + case Arch.X64: return "amd64" - case "arm64": + case Arch.ARM64: return "arm64" case _: raise UnsupportedArchitectureError(arch) +@dataclass +class BuildImageConfig: + """The configuration values for building openstack image. + + Attrs: + arch: The image architecture to build for. + base_image: The ubuntu image to use as image build base. + proxies: HTTP proxy settings. + """ + + arch: Arch + base_image: BaseImage + proxies: Optional[ProxyConfig] = None + + def build_image( - arch: Arch, cloud_config: dict[str, dict], github_client: GithubClient, path: GithubPath, - proxies: Optional[ProxyConfig] = None, + config: BuildImageConfig, ) -> str: """Build and upload an image to OpenStack. @@ -242,7 +266,7 @@ def build_image( cloud_config: The cloud configuration to connect OpenStack with. github_client: The Github client to interact with Github API. path: Github organisation or repository path. - proxies: HTTP proxy settings. + config: The image build configuration values. Raises: ImageBuildError: If there were errors building/creating the image. @@ -251,18 +275,21 @@ def build_image( The created OpenStack image id. """ try: - runner_application = github_client.get_runner_application(path=path, arch=arch) + runner_application = github_client.get_runner_application(path=path, arch=config.arch) except RunnerBinaryError as exc: raise OpenstackImageBuildError("Failed to fetch runner application.") from exc try: - execute_command(_build_image_command(runner_application, proxies), check_exit=True) + execute_command( + _build_image_command(runner_application, config.base_image, config.proxies), + check_exit=True, + ) except SubprocessError as exc: raise OpenstackImageBuildError("Failed to build image.") from exc + runner_arch = runner_application["architecture"] try: - runner_arch = runner_application["architecture"] - image_arch = _get_supported_runner_arch(arch=runner_arch) + image_arch = _get_supported_runner_arch(arch=config.arch) except UnsupportedArchitectureError as exc: raise OpenstackImageBuildError(f"Unsupported architecture {runner_arch}") from exc diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index e9f631c94..8a81348c7 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -213,8 +213,8 @@ async def app_no_runner( return application -@pytest_asyncio.fixture(scope="module") -async def app_openstack_runner( +@pytest_asyncio.fixture(scope="function", name="app_openstack_runner") +async def app_openstack_runner_fixture( model: Model, charm_file: str, app_name: str, diff --git a/tests/integration/test_charm_one_runner.py b/tests/integration/test_charm_one_runner.py index eb305b25d..6e7a167d4 100644 --- a/tests/integration/test_charm_one_runner.py +++ b/tests/integration/test_charm_one_runner.py @@ -6,13 +6,18 @@ import pytest import pytest_asyncio +from github.Branch import Branch from github.Repository import Repository +from github.WorkflowRun import WorkflowRun from juju.application import Application from juju.model import Model from charm import GithubRunnerCharm +from charm_state import BASE_IMAGE_CONFIG_NAME from tests.integration.helpers import ( + DISPATCH_E2E_TEST_RUN_WORKFLOW_FILENAME, assert_resource_lxd_profile, + dispatch_workflow, ensure_charm_has_runner, get_runner_names, reconcile, @@ -291,3 +296,40 @@ async def test_token_config_changed_insufficient_perms( await model.wait_for_idle() await wait_till_num_of_runners(unit, num=0) + + +@pytest.mark.parametrize( + "image", + [ + pytest.param("noble", id="noble"), + ], +) +async def test_runner_base_image( + model: Model, + app_no_runner: Application, + image: str, + github_repository: Repository, + test_github_branch: Branch, +) -> None: + """ + arrange: A runner with noble as base image. + act: Dispatch a workflow. + assert: A runner should work with the different images. + """ + await app_no_runner.set_config( + { + BASE_IMAGE_CONFIG_NAME: image, + } + ) + await ensure_charm_has_runner(app_no_runner, model) + workflow = await dispatch_workflow( + app=app_no_runner, + branch=test_github_branch, + github_repository=github_repository, + conclusion="success", + workflow_id_or_name=DISPATCH_E2E_TEST_RUN_WORKFLOW_FILENAME, + dispatch_input={"runner-tag": app_no_runner.name}, + ) + + workflow_run: WorkflowRun = workflow.get_runs()[0] + assert workflow_run.status == "success" diff --git a/tests/integration/test_openstack.py b/tests/integration/test_openstack.py index a7b088ab0..ef3b395ea 100644 --- a/tests/integration/test_openstack.py +++ b/tests/integration/test_openstack.py @@ -12,7 +12,12 @@ from juju.model import Model from openstack.compute.v2.server import Server -from tests.integration.helpers import DISPATCH_E2E_TEST_RUN_WORKFLOW_FILENAME, dispatch_workflow +from charm_state import BASE_IMAGE_CONFIG_NAME +from tests.integration.helpers import ( + DISPATCH_E2E_TEST_RUN_WORKFLOW_FILENAME, + dispatch_workflow, + ensure_charm_has_runner, +) # 2024/03/19 - The firewall configuration on openstack will be implemented by follow up PR on @@ -55,3 +60,40 @@ async def test_openstack_integration( server: Server = servers[0] # 2. a server with image name jammy is created. assert server.image.name == "jammy" + + +@pytest.mark.parametrize( + "image", + [ + pytest.param("noble", id="noble"), + ], +) +async def test_runner_base_image( + model: Model, + app_openstack_runner: Application, + image: str, + github_repository: Repository, + test_github_branch: Branch, +) -> None: + """ + arrange: A runner with noble as base image. + act: Dispatch a workflow. + assert: A runner should work with the different images. + """ + await app_openstack_runner.set_config( + { + BASE_IMAGE_CONFIG_NAME: image, + } + ) + await ensure_charm_has_runner(app_openstack_runner, model) + workflow = await dispatch_workflow( + app=app_openstack_runner, + branch=test_github_branch, + github_repository=github_repository, + conclusion="success", + workflow_id_or_name=DISPATCH_E2E_TEST_RUN_WORKFLOW_FILENAME, + dispatch_input={"runner-tag": app_openstack_runner.name}, + ) + + workflow_run: WorkflowRun = workflow.get_runs()[0] + assert workflow_run.status == "success" diff --git a/tests/unit/test_openstack_manager.py b/tests/unit/test_openstack_manager.py index 88f26b436..0789818bc 100644 --- a/tests/unit/test_openstack_manager.py +++ b/tests/unit/test_openstack_manager.py @@ -8,6 +8,7 @@ import openstack.exceptions import pytest +from charm_state import Arch, BaseImage from errors import OpenStackUnauthorizedError from openstack_cloud import openstack_manager @@ -59,6 +60,20 @@ def patched_create_connection_context_fixture(monkeypatch: pytest.MonkeyPatch): return mock_connection.__enter__() +@pytest.fixture(name="build_image_config") +def build_image_config_fixture(): + """Return a test build image config.""" + return openstack_manager.BuildImageConfig( + arch=Arch.X64, + base_image=BaseImage.NOBLE, + proxies=openstack_manager.ProxyConfig( + http="http://test.internal", + https="https://test.internal", + no_proxy="http://no_proxy.internal", + ), + ) + + def test__create_connection_error(clouds_yaml: dict, openstack_connect_mock: MagicMock): """ arrange: given a monkeypatched connection.authorize() function that raises an error. @@ -311,9 +326,12 @@ def test__build_image_command(): no_proxy=(test_no_proxy := "http://no.proxy"), use_aproxy=False, ) + test_base_image = BaseImage.NOBLE command = openstack_manager._build_image_command( - runner_info=test_runner_info, proxies=test_proxy_config + runner_info=test_runner_info, + proxies=test_proxy_config, + base_image=test_base_image, ) assert command == [ "/usr/bin/bash", @@ -334,10 +352,11 @@ def test__build_image_command(): """, f"""{{"proxies": {{"default": {{"httpProxy": "{test_http_proxy}", \ "httpsProxy": "{test_https_proxy}", "noProxy": "{test_no_proxy}"}}}}}}""", + test_base_image.value, ], "Unexpected build image command." -def test_build_image_runner_binary_error(): +def test_build_image_runner_binary_error(build_image_config: openstack_manager.BuildImageConfig): """ arrange: given a mocked github client get_runner_application function that raises an error. act: when build_image is called. @@ -348,16 +367,18 @@ def test_build_image_runner_binary_error(): with pytest.raises(openstack_manager.OpenstackImageBuildError) as exc: openstack_manager.build_image( - arch=openstack_manager.Arch.X64, cloud_config=MagicMock(), github_client=mock_github_client, path=MagicMock(), + config=build_image_config, ) assert "Failed to fetch runner application." in str(exc) -def test_build_image_script_error(monkeypatch: pytest.MonkeyPatch): +def test_build_image_script_error( + monkeypatch: pytest.MonkeyPatch, build_image_config: openstack_manager.BuildImageConfig +): """ arrange: given a monkeypatched execute_command function that raises an error. act: when build_image is called. @@ -375,10 +396,10 @@ def test_build_image_script_error(monkeypatch: pytest.MonkeyPatch): with pytest.raises(openstack_manager.OpenstackImageBuildError) as exc: openstack_manager.build_image( - arch=openstack_manager.Arch.X64, cloud_config=MagicMock(), github_client=MagicMock(), path=MagicMock(), + config=build_image_config, ) assert "Failed to build image." in str(exc) @@ -386,7 +407,9 @@ def test_build_image_script_error(monkeypatch: pytest.MonkeyPatch): @pytest.mark.usefixtures("patch_execute_command") def test_build_image_runner_arch_error( - monkeypatch: pytest.MonkeyPatch, mock_github_client: MagicMock + monkeypatch: pytest.MonkeyPatch, + mock_github_client: MagicMock, + build_image_config: openstack_manager.BuildImageConfig, ): """ arrange: given _get_supported_runner_arch that raises unsupported architecture error. @@ -403,10 +426,10 @@ def test_build_image_runner_arch_error( with pytest.raises(openstack_manager.OpenstackImageBuildError) as exc: openstack_manager.build_image( - arch=openstack_manager.Arch.X64, cloud_config=MagicMock(), github_client=mock_github_client, path=MagicMock(), + config=build_image_config, ) assert "Unsupported architecture" in str(exc) @@ -414,7 +437,9 @@ def test_build_image_runner_arch_error( @pytest.mark.usefixtures("patch_execute_command") def test_build_image_delete_image_error( - mock_github_client: MagicMock, patched_create_connection_context: MagicMock + mock_github_client: MagicMock, + patched_create_connection_context: MagicMock, + build_image_config: openstack_manager.BuildImageConfig, ): """ arrange: given a mocked openstack connection that returns existing images and delete_image @@ -429,10 +454,10 @@ def test_build_image_delete_image_error( with pytest.raises(openstack_manager.OpenstackImageBuildError) as exc: openstack_manager.build_image( - arch=openstack_manager.Arch.X64, cloud_config=MagicMock(), github_client=mock_github_client, path=MagicMock(), + config=build_image_config, ) assert "Failed to delete duplicate image on Openstack." in str(exc) @@ -440,7 +465,9 @@ def test_build_image_delete_image_error( @pytest.mark.usefixtures("patch_execute_command") def test_build_image_create_image_error( - patched_create_connection_context: MagicMock, mock_github_client: MagicMock + patched_create_connection_context: MagicMock, + mock_github_client: MagicMock, + build_image_config: openstack_manager.BuildImageConfig, ): """ arrange: given a mocked connection that raises OpenStackCloudException on create_image. @@ -453,18 +480,21 @@ def test_build_image_create_image_error( with pytest.raises(openstack_manager.OpenstackImageBuildError) as exc: openstack_manager.build_image( - arch=openstack_manager.Arch.X64, cloud_config=MagicMock(), github_client=mock_github_client, path=MagicMock(), - proxies=None, + config=build_image_config, ) assert "Failed to upload image." in str(exc) @pytest.mark.usefixtures("patch_execute_command") -def test_build_image(patched_create_connection_context: MagicMock, mock_github_client: MagicMock): +def test_build_image( + patched_create_connection_context: MagicMock, + mock_github_client: MagicMock, + build_image_config: openstack_manager.BuildImageConfig, +): """ arrange: given monkeypatched execute_command and mocked openstack connection. act: when build_image is called. @@ -476,8 +506,8 @@ def test_build_image(patched_create_connection_context: MagicMock, mock_github_c ) openstack_manager.build_image( - arch=openstack_manager.Arch.X64, cloud_config=MagicMock(), github_client=mock_github_client, path=MagicMock(), + config=build_image_config, ) From f30f34d0f911f096abecf99daa437dd9a68600de Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Mon, 8 Apr 2024 21:41:35 +0800 Subject: [PATCH 03/24] docs: add docs for base image --- docs/how-to/set-base-image.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 docs/how-to/set-base-image.md diff --git a/docs/how-to/set-base-image.md b/docs/how-to/set-base-image.md new file mode 100644 index 000000000..7e958992d --- /dev/null +++ b/docs/how-to/set-base-image.md @@ -0,0 +1,14 @@ +# How to set base image + +This charm supports deploying the runners on different base images. + +By using [`juju config`](https://juju.is/docs/juju/juju-config) to change the +[charm configuration labels](https://charmhub.io/github-runner/configure#base-image), the runner +can be deployed with a different Ubuntu base image. The supported base images are limited to jammy +and noble to ensure guaranteed capabilities. + +```shell +juju config base-image= +``` + +An example of a BASE_IMAGE_TAG_OR_NAME value would be "jammy", "22.04", "noble", "24.04". From a53d59c25bc0f375b1f9a98610071427698ab23f Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Tue, 9 Apr 2024 10:08:57 +0800 Subject: [PATCH 04/24] fix: shellscript lint --- scripts/build-lxd-image.sh | 6 +++--- scripts/build-openstack-image.sh | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/build-lxd-image.sh b/scripts/build-lxd-image.sh index 2084bb2dc..c59a0fb25 100644 --- a/scripts/build-lxd-image.sh +++ b/scripts/build-lxd-image.sh @@ -147,9 +147,9 @@ fi /snap/bin/lxc publish builder --alias builder --reuse -f # Swap in the built image -/snap/bin/lxc image alias rename $BASE_IMAGE old-$BASE_IMAGE || true -/snap/bin/lxc image alias rename builder $BASE_IMAGE -/snap/bin/lxc image delete old-$BASE_IMAGE || true +/snap/bin/lxc image alias rename "$BASE_IMAGE" "old-$BASE_IMAGE" || true +/snap/bin/lxc image alias rename builder "$BASE_IMAGE" +/snap/bin/lxc image delete "old-$BASE_IMAGE" || true # Clean up LXD instance cleanup '/snap/bin/lxc info builder &> /dev/null' '/snap/bin/lxc delete builder --force' 'Cleanup LXD instance' 10 diff --git a/scripts/build-openstack-image.sh b/scripts/build-openstack-image.sh index 5914a4aef..dcd244590 100755 --- a/scripts/build-openstack-image.sh +++ b/scripts/build-openstack-image.sh @@ -113,11 +113,11 @@ retry "sudo wget https://cloud-images.ubuntu.com/$BASE_IMAGE/current/$BASE_IMAGE -O $BASE_IMAGE-server-cloudimg-$BIN_ARCH.img" "Downloading cloud image" 3 # resize image - installing dependencies requires more disk space -sudo qemu-img resize $BASE_IMAGE-server-cloudimg-$BIN_ARCH.img +1.5G +sudo qemu-img resize "$BASE_IMAGE-server-cloudimg-$BIN_ARCH.img" +1.5G # mount nbd echo "Connecting network block device to image" -sudo qemu-nbd --connect=/dev/nbd0 $BASE_IMAGE-server-cloudimg-$BIN_ARCH.img +sudo qemu-nbd --connect=/dev/nbd0 "$BASE_IMAGE-server-cloudimg-$BIN_ARCH.img" sudo mkdir -p /mnt/ubuntu-image retry "sudo mount -o rw /dev/nbd0p1 /mnt/ubuntu-image" "Mounting nbd0p1 device" 3 @@ -209,4 +209,4 @@ sudo sync cleanup # Reduce image size by removing sparse space & compressing -sudo virt-sparsify --compress $BASE_IMAGE-server-cloudimg-$BIN_ARCH.img $BASE_IMAGE-server-cloudimg-$BIN_ARCH-compressed.img +sudo virt-sparsify --compress "$BASE_IMAGE-server-cloudimg-$BIN_ARCH.img" "$BASE_IMAGE-server-cloudimg-$BIN_ARCH-compressed.img" From 1d6503c79346a476d18441aa0c80dff25f4c45e0 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Tue, 9 Apr 2024 14:05:23 +0800 Subject: [PATCH 05/24] remove unneeded hwe-generic patch --- scripts/build-lxd-image.sh | 1 - src-docs/charm_state.py.md | 20 ++++++++++---------- src-docs/openstack_manager.md | 8 +++++++- src-docs/runner_manager.py.md | 4 ++-- 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/scripts/build-lxd-image.sh b/scripts/build-lxd-image.sh index c59a0fb25..4feabc0a6 100644 --- a/scripts/build-lxd-image.sh +++ b/scripts/build-lxd-image.sh @@ -87,7 +87,6 @@ retry '/snap/bin/lxc exec builder -- /usr/bin/nslookup github.com' 'Wait for net /snap/bin/lxc exec builder -- /usr/bin/apt-get update /snap/bin/lxc exec builder --env DEBIAN_FRONTEND=noninteractive -- /usr/bin/apt-get upgrade -yq -/snap/bin/lxc exec builder --env DEBIAN_FRONTEND=noninteractive -- /usr/bin/apt-get install linux-generic-hwe-22.04 -yq # This will remove older version of kernel as HWE is installed now. /snap/bin/lxc exec builder -- /usr/bin/apt-get autoremove --purge diff --git a/src-docs/charm_state.py.md b/src-docs/charm_state.py.md index 59dcb80ca..54e4b7bdd 100644 --- a/src-docs/charm_state.py.md +++ b/src-docs/charm_state.py.md @@ -218,7 +218,7 @@ The charm state. --- - + ### classmethod `from_charm` @@ -362,7 +362,7 @@ Return a string representing the path. ## class `ImmutableConfigChangedError` Represents an error when changing immutable charm state. - + ### function `__init__` @@ -408,7 +408,7 @@ Return the aproxy address. --- - + ### classmethod `check_use_aproxy` @@ -438,7 +438,7 @@ Validate the proxy configuration. --- - + ### classmethod `from_charm` @@ -469,7 +469,7 @@ Runner configurations for the charm. **Attributes:** - - `base_image`: The ubuntu base image to run the runner viertual machines on. + - `base_image`: The ubuntu base image to run the runner virtual machines on. - `virtual_machines`: Number of virtual machine-based runner to spawn. - `virtual_machine_resources`: Hardware resource used by one virtual machine for a runner. - `runner_storage`: Storage to be used as disk for the runner. @@ -479,7 +479,7 @@ Runner configurations for the charm. --- - + ### classmethod `check_virtual_machine_resources` @@ -510,7 +510,7 @@ Validate the virtual_machine_resources field values. --- - + ### classmethod `check_virtual_machines` @@ -539,7 +539,7 @@ Validate the virtual machines configuration value. --- - + ### classmethod `from_charm` @@ -602,7 +602,7 @@ SSH connection information for debug workflow. --- - + ### classmethod `from_charm` @@ -635,7 +635,7 @@ Raised when given machine charm architecture is unsupported. - `arch`: The current machine architecture. - + ### function `__init__` diff --git a/src-docs/openstack_manager.md b/src-docs/openstack_manager.md index dba976b2a..bc5053a28 100644 --- a/src-docs/openstack_manager.md +++ b/src-docs/openstack_manager.md @@ -180,7 +180,13 @@ __init__( ## class `BuildImageConfig` The configuration values for building openstack image. -Attrs: arch: The image architecture to build for. base_image: The ubuntu image to use as image build base. proxies: HTTP proxy settings. + + +**Attributes:** + + - `arch`: The image architecture to build for. + - `base_image`: The ubuntu image to use as image build base. + - `proxies`: HTTP proxy settings. diff --git a/src-docs/runner_manager.py.md b/src-docs/runner_manager.py.md index a963762d9..57955bf4d 100644 --- a/src-docs/runner_manager.py.md +++ b/src-docs/runner_manager.py.md @@ -50,7 +50,7 @@ Construct RunnerManager object for creating and managing runners. --- - + ### function `build_runner_image` @@ -188,7 +188,7 @@ Bring runners in line with target. --- - + ### function `schedule_build_runner_image` From 2609a2df3fdf69cbd8175a26b68ea37a5d769230 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Tue, 9 Apr 2024 14:05:50 +0800 Subject: [PATCH 06/24] add retry (builder lxd not ready error) --- src/runner_manager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/runner_manager.py b/src/runner_manager.py index 4c639ad7c..d29d0da92 100644 --- a/src/runner_manager.py +++ b/src/runner_manager.py @@ -786,6 +786,7 @@ def _build_image_command(self) -> list[str]: cmd += ["test"] return cmd + @retry(tries=3, delay=30, local_logger=logger) def build_runner_image(self) -> None: """Build the LXD image for hosting runner. From 0bc98dac115e34b2681a5a9d6cee9598e91ca0a7 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Tue, 9 Apr 2024 14:06:02 +0800 Subject: [PATCH 07/24] improve doc --- docs/how-to/set-base-image.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/how-to/set-base-image.md b/docs/how-to/set-base-image.md index 7e958992d..5571a4110 100644 --- a/docs/how-to/set-base-image.md +++ b/docs/how-to/set-base-image.md @@ -4,8 +4,8 @@ This charm supports deploying the runners on different base images. By using [`juju config`](https://juju.is/docs/juju/juju-config) to change the [charm configuration labels](https://charmhub.io/github-runner/configure#base-image), the runner -can be deployed with a different Ubuntu base image. The supported base images are limited to jammy -and noble to ensure guaranteed capabilities. +can be deployed with a different Ubuntu base image. Latest two LTS images "jammy" and "noble" are +supported. ```shell juju config base-image= From a894576d9975d17dce1f4178a09c9a443b88fd20 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Tue, 9 Apr 2024 14:06:21 +0800 Subject: [PATCH 08/24] parse image improvements --- src/charm_state.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/charm_state.py b/src/charm_state.py index 203497e88..e3b76c796 100644 --- a/src/charm_state.py +++ b/src/charm_state.py @@ -470,20 +470,17 @@ def from_charm(cls, charm: CharmBase) -> "BaseImage": Returns: The base image configuration of the charm. """ - image_name = charm.config.get(BASE_IMAGE_CONFIG_NAME, "jammy").lower().strip() - try: - return cls(image_name) - except ValueError as exc: - if image_name in LTS_IMAGE_VERSION_TAG_MAP: - return cls(LTS_IMAGE_VERSION_TAG_MAP[image_name]) - raise ValueError(f"Unsupported base image: {image_name}") from exc + image_name = charm.config.get(BASE_IMAGE_CONFIG_NAME).lower().strip() + if image_name in LTS_IMAGE_VERSION_TAG_MAP: + return cls(LTS_IMAGE_VERSION_TAG_MAP[image_name]) + return cls(image_name) class RunnerCharmConfig(BaseModel): """Runner configurations for the charm. Attributes: - base_image: The ubuntu base image to run the runner viertual machines on. + base_image: The ubuntu base image to run the runner virtual machines on. virtual_machines: Number of virtual machine-based runner to spawn. virtual_machine_resources: Hardware resource used by one virtual machine for a runner. runner_storage: Storage to be used as disk for the runner. @@ -827,7 +824,7 @@ def _check_immutable_config_change( logger.info("Previous charm state: %s", prev_state) if prev_state["runner_config"]["runner_storage"] != runner_storage: - logger.warning( + logger.error( "Storage option changed from %s to %s, blocking the charm", prev_state["runner_config"]["runner_storage"], runner_storage, @@ -836,7 +833,7 @@ def _check_immutable_config_change( msg="runner-storage config cannot be changed after deployment, redeploy if needed" ) if prev_state["runner_config"]["base_image"] != base_image.value: - logger.warning( + logger.error( "Base image option changed from %s to %s, blocking the charm", prev_state["runner_config"]["base_image"], runner_storage, From 05ec43c8e04314db380c492c95c2de883effa300 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Tue, 9 Apr 2024 14:06:33 +0800 Subject: [PATCH 09/24] attributes docstring --- src/openstack_cloud/openstack_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/openstack_cloud/openstack_manager.py b/src/openstack_cloud/openstack_manager.py index 669d1fabe..f6bce9a4f 100644 --- a/src/openstack_cloud/openstack_manager.py +++ b/src/openstack_cloud/openstack_manager.py @@ -248,7 +248,7 @@ def _get_supported_runner_arch(arch: str) -> SupportedCloudImageArch: class BuildImageConfig: """The configuration values for building openstack image. - Attrs: + Attributes: arch: The image architecture to build for. base_image: The ubuntu image to use as image build base. proxies: HTTP proxy settings. From 3ec7e316a1727d62c08660fdb1a6be81df1b2659 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Tue, 9 Apr 2024 14:07:47 +0800 Subject: [PATCH 10/24] test: remove parametrization --- tests/integration/test_charm_one_runner.py | 9 +-------- tests/integration/test_openstack.py | 11 ++--------- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/tests/integration/test_charm_one_runner.py b/tests/integration/test_charm_one_runner.py index 481c973d1..74b65b4fb 100644 --- a/tests/integration/test_charm_one_runner.py +++ b/tests/integration/test_charm_one_runner.py @@ -308,16 +308,9 @@ async def test_token_config_changed_insufficient_perms( await wait_till_num_of_runners(unit, num=0) -@pytest.mark.parametrize( - "image", - [ - pytest.param("noble", id="noble"), - ], -) async def test_runner_base_image( model: Model, app_no_runner: Application, - image: str, github_repository: Repository, test_github_branch: Branch, ) -> None: @@ -328,7 +321,7 @@ async def test_runner_base_image( """ await app_no_runner.set_config( { - BASE_IMAGE_CONFIG_NAME: image, + BASE_IMAGE_CONFIG_NAME: "noble", } ) await ensure_charm_has_runner(app_no_runner, model) diff --git a/tests/integration/test_openstack.py b/tests/integration/test_openstack.py index ef3b395ea..1f82a1123 100644 --- a/tests/integration/test_openstack.py +++ b/tests/integration/test_openstack.py @@ -62,16 +62,9 @@ async def test_openstack_integration( assert server.image.name == "jammy" -@pytest.mark.parametrize( - "image", - [ - pytest.param("noble", id="noble"), - ], -) -async def test_runner_base_image( +async def test_noble_base_image( model: Model, app_openstack_runner: Application, - image: str, github_repository: Repository, test_github_branch: Branch, ) -> None: @@ -82,7 +75,7 @@ async def test_runner_base_image( """ await app_openstack_runner.set_config( { - BASE_IMAGE_CONFIG_NAME: image, + BASE_IMAGE_CONFIG_NAME: "noble", } ) await ensure_charm_has_runner(app_openstack_runner, model) From d96eb9f9c30003945ae5cfa690f78b8d4c310f70 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Tue, 9 Apr 2024 14:34:14 +0800 Subject: [PATCH 11/24] test: add unit tests --- src-docs/charm_state.py.md | 25 ++++--- src-docs/openstack_manager.md | 5 +- src/charm_state.py | 30 ++++++--- src/openstack_cloud/openstack_manager.py | 3 +- tests/unit/test_charm_state.py | 86 ++++++++++++++++++++++++ 5 files changed, 124 insertions(+), 25 deletions(-) diff --git a/src-docs/charm_state.py.md b/src-docs/charm_state.py.md index 54e4b7bdd..1aa256c29 100644 --- a/src-docs/charm_state.py.md +++ b/src-docs/charm_state.py.md @@ -83,6 +83,13 @@ The ubuntu OS base image to build and deploy runners on. +**Attributes:** + + - `JAMMY`: The jammy ubuntu LTS image. + - `NOBLE`: The noble ubuntu LTS image. + + + --- @@ -218,7 +225,7 @@ The charm state. --- - + ### classmethod `from_charm` @@ -362,7 +369,7 @@ Return a string representing the path. ## class `ImmutableConfigChangedError` Represents an error when changing immutable charm state. - + ### function `__init__` @@ -408,7 +415,7 @@ Return the aproxy address. --- - + ### classmethod `check_use_aproxy` @@ -438,7 +445,7 @@ Validate the proxy configuration. --- - + ### classmethod `from_charm` @@ -479,7 +486,7 @@ Runner configurations for the charm. --- - + ### classmethod `check_virtual_machine_resources` @@ -510,7 +517,7 @@ Validate the virtual_machine_resources field values. --- - + ### classmethod `check_virtual_machines` @@ -539,7 +546,7 @@ Validate the virtual machines configuration value. --- - + ### classmethod `from_charm` @@ -602,7 +609,7 @@ SSH connection information for debug workflow. --- - + ### classmethod `from_charm` @@ -635,7 +642,7 @@ Raised when given machine charm architecture is unsupported. - `arch`: The current machine architecture. - + ### function `__init__` diff --git a/src-docs/openstack_manager.md b/src-docs/openstack_manager.md index bc5053a28..f6b04c334 100644 --- a/src-docs/openstack_manager.md +++ b/src-docs/openstack_manager.md @@ -32,7 +32,6 @@ Build and upload an image to OpenStack. **Args:** - - `arch`: The system architecture to build the image for. - `cloud_config`: The cloud configuration to connect OpenStack with. - `github_client`: The Github client to interact with Github API. - `path`: Github organisation or repository path. @@ -52,7 +51,7 @@ Build and upload an image to OpenStack. --- - + ## function `create_instance_config` @@ -84,7 +83,7 @@ Create an instance config from charm data. --- - + ## function `create_instance` diff --git a/src/charm_state.py b/src/charm_state.py index e3b76c796..83f349bf8 100644 --- a/src/charm_state.py +++ b/src/charm_state.py @@ -114,7 +114,7 @@ def parse_github_path(path_str: str, runner_group: str) -> GithubPath: organization with runner group information. """ if "/" in path_str: - paths = path_str.split("/") + paths = tuple(segment for segment in path_str.split("/") if segment) if len(paths) != 2: raise CharmConfigInvalidError(f"Invalid path configuration {path_str}") owner, repo = paths @@ -448,13 +448,22 @@ def check_reconcile_interval(cls, reconcile_interval: int) -> int: class BaseImage(str, Enum): - """The ubuntu OS base image to build and deploy runners on.""" + """The ubuntu OS base image to build and deploy runners on. + + Attributes: + JAMMY: The jammy ubuntu LTS image. + NOBLE: The noble ubuntu LTS image. + """ JAMMY = "jammy" NOBLE = "noble" def __str__(self) -> str: - """Interpolate to string value.""" + """Interpolate to string value. + + Returns: + The enum string value. + """ return self.value @classmethod @@ -464,13 +473,10 @@ def from_charm(cls, charm: CharmBase) -> "BaseImage": Args: charm: The charm instance. - Raises: - ValueError if an unsupporte base image is passed in. - Returns: The base image configuration of the charm. """ - image_name = charm.config.get(BASE_IMAGE_CONFIG_NAME).lower().strip() + image_name = charm.config.get(BASE_IMAGE_CONFIG_NAME, "jammy").lower().strip() if image_name in LTS_IMAGE_VERSION_TAG_MAP: return cls(LTS_IMAGE_VERSION_TAG_MAP[image_name]) return cls(image_name) @@ -813,6 +819,10 @@ def _check_immutable_config_change( ) -> None: """Ensure immutable config has not changed. + Args: + runner_storage: The current runner_storage configuration. + base_image: The current base_image configuration. + Raises: ImmutableConfigChangedError: If an immutable configuration has changed. """ @@ -863,14 +873,12 @@ def from_charm(cls, charm: CharmBase) -> "CharmState": try: charm_config = CharmConfig.from_charm(charm) runner_config = RunnerCharmConfig.from_charm(charm) - except (ValidationError, ValueError) as exc: - raise CharmConfigInvalidError(f"Invalid configuration: {str(exc)}") from exc - - try: cls._check_immutable_config_change( runner_storage=runner_config.runner_storage, base_image=runner_config.base_image, ) + except (ValidationError, ValueError) as exc: + raise CharmConfigInvalidError(f"Invalid configuration: {str(exc)}") from exc except ImmutableConfigChangedError as exc: raise CharmConfigInvalidError(exc.msg) from exc diff --git a/src/openstack_cloud/openstack_manager.py b/src/openstack_cloud/openstack_manager.py index f6bce9a4f..ffb82da2c 100644 --- a/src/openstack_cloud/openstack_manager.py +++ b/src/openstack_cloud/openstack_manager.py @@ -164,8 +164,8 @@ def _build_image_command( Args: runner_info: The runner application to fetch runner tar download url. + base_image: The ubuntu base image to use. proxies: HTTP proxy settings. - base: The ubuntu base image to use. Returns: Command to execute to build runner image. @@ -305,7 +305,6 @@ def build_image( """Build and upload an image to OpenStack. Args: - arch: The system architecture to build the image for. cloud_config: The cloud configuration to connect OpenStack with. github_client: The Github client to interact with Github API. path: Github organisation or repository path. diff --git a/tests/unit/test_charm_state.py b/tests/unit/test_charm_state.py index c94280202..d3d9e6d04 100644 --- a/tests/unit/test_charm_state.py +++ b/tests/unit/test_charm_state.py @@ -12,8 +12,11 @@ import charm_state from charm_state import ( + BASE_IMAGE_CONFIG_NAME, COS_AGENT_INTEGRATION_NAME, DEBUG_SSH_INTEGRATION_NAME, + PATH_CONFIG_NAME, + TOKEN_CONFIG_NAME, USE_APROXY_CONFIG_NAME, Arch, CharmConfigInvalidError, @@ -47,6 +50,47 @@ def clouds_yaml() -> dict: } +@pytest.mark.parametrize( + "invalid_path", + [ + pytest.param("canonical/", id="org only"), + pytest.param("/github-runner-operator", id="repository only"), + ], +) +def test_parse_github_path_invalid_path(invalid_path: str): + """ + arrange: Given an invalid Github path. + act: when parse_github_path is called. + assert: CharmConfigInvalidError is raised. + """ + with pytest.raises(CharmConfigInvalidError) as exc: + charm_state.parse_github_path(invalid_path, MagicMock()) + + assert "Invalid path configuration" in str(exc) + + +@pytest.mark.parametrize( + "missing_config", + [ + pytest.param(PATH_CONFIG_NAME, id="missing path"), + pytest.param(TOKEN_CONFIG_NAME, id="missing token"), + ], +) +def test_github_config_from_charm_missing_token(missing_config: str): + """ + arrange: Given charm with missing token config. + act: when GithubConfig.from_charm is called. + assert: CharmConfigInvalidError is raised. + """ + mock_charm = MockGithubRunnerCharmFactory() + mock_charm.config[missing_config] = "" + + with pytest.raises(CharmConfigInvalidError) as exc: + charm_state.GithubConfig.from_charm(mock_charm) + + assert f"Missing {missing_config} configuration" in str(exc) + + def test_metrics_logging_available_true(): """ arrange: Setup mocked charm to return an integration. @@ -91,6 +135,48 @@ def test_aproxy_proxy_missing(): assert "Invalid proxy configuration" in str(exc.value) +@pytest.mark.parametrize( + "image", + [ + pytest.param("eagle", id="non existent image"), + pytest.param("bionic", id="unsupported image"), + ], +) +def test_invalid_base_image(image: str): + """ + arrange: Given an invalid base configuration. + act: Retrieve state from charm. + assert: CharmConfigInvalidError is raised. + """ + mock_charm = MockGithubRunnerCharmFactory() + mock_charm.config[BASE_IMAGE_CONFIG_NAME] = image + + with pytest.raises(CharmConfigInvalidError) as exc: + charm_state.RunnerCharmConfig.from_charm(mock_charm) + assert "Invalid base image" in str(exc.value) + + +@pytest.mark.parametrize( + "image, expected_base_image", + [ + pytest.param("jammy", charm_state.BaseImage.JAMMY, id="jammy"), + pytest.param("22.04", charm_state.BaseImage.JAMMY, id="jammy tag"), + pytest.param("noble", charm_state.BaseImage.NOBLE, id="noble"), + pytest.param("24.04", charm_state.BaseImage.NOBLE, id="noble tag"), + ], +) +def test_base_image(image: str, expected_base_image: charm_state.BaseImage): + """ + arrange: Given supported base image configuration. + act: Retrieve state from charm. + assert: CharmConfigInvalidError is raised. + """ + mock_charm = MockGithubRunnerCharmFactory() + mock_charm.config[BASE_IMAGE_CONFIG_NAME] = image + + assert charm_state.BaseImage.from_charm(mock_charm) == expected_base_image + + def test_proxy_invalid_format(): """ arrange: Setup mocked charm and invalid juju proxy settings. From d7883e05066e346fc352f5ba4ce4aae419329a50 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Tue, 9 Apr 2024 15:44:18 +0800 Subject: [PATCH 12/24] fix: workflow syntax --- .github/workflows/integration_test.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 73cda1468..8f7e05ee1 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -24,7 +24,7 @@ jobs: pre-run-script: scripts/pre-integration-test.sh provider: lxd test-tox-env: integration-juju2.9 - modules: ${{ env.LXD_TEST_MODULES }} + modules: ${{ LXD_TEST_MODULES }} integration-tests: name: Integration test with juju 3.1 uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main @@ -34,7 +34,7 @@ jobs: pre-run-script: scripts/pre-integration-test.sh provider: lxd test-tox-env: integration-juju3.1 - modules: ${{ env.LXD_TEST_MODULES }} + modules: ${{ LXD_TEST_MODULES }} # openstack tests use microstack, whose setup is kind of special # - due to the huge resource requirements, we use self-hosted runners for these tests # - microstack requires juju 3.2 and microk8s 1.26 @@ -51,6 +51,6 @@ jobs: channel: 1.26-strict/stable microk8s-addons: "dns ingress hostpath-storage" test-tox-env: integration-juju3.2 - modules: ${{ env.OPENSTACK_TEST_MODULES }} + modules: ${{ OPENSTACK_TEST_MODULES }} self-hosted-runner: true self-hosted-runner-label: two-xlarge From b4cf87c2a765d86740fab443ba815f9c667ae480 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Tue, 9 Apr 2024 15:52:16 +0800 Subject: [PATCH 13/24] revert env changes(env doesn't work w/ context) --- .github/workflows/integration_test.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 8f7e05ee1..73cda1468 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -24,7 +24,7 @@ jobs: pre-run-script: scripts/pre-integration-test.sh provider: lxd test-tox-env: integration-juju2.9 - modules: ${{ LXD_TEST_MODULES }} + modules: ${{ env.LXD_TEST_MODULES }} integration-tests: name: Integration test with juju 3.1 uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main @@ -34,7 +34,7 @@ jobs: pre-run-script: scripts/pre-integration-test.sh provider: lxd test-tox-env: integration-juju3.1 - modules: ${{ LXD_TEST_MODULES }} + modules: ${{ env.LXD_TEST_MODULES }} # openstack tests use microstack, whose setup is kind of special # - due to the huge resource requirements, we use self-hosted runners for these tests # - microstack requires juju 3.2 and microk8s 1.26 @@ -51,6 +51,6 @@ jobs: channel: 1.26-strict/stable microk8s-addons: "dns ingress hostpath-storage" test-tox-env: integration-juju3.2 - modules: ${{ OPENSTACK_TEST_MODULES }} + modules: ${{ env.OPENSTACK_TEST_MODULES }} self-hosted-runner: true self-hosted-runner-label: two-xlarge From d211427db6741cc2fa074014f24b35fccdaa84de Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Tue, 9 Apr 2024 15:52:31 +0800 Subject: [PATCH 14/24] revert env changes(env doesn't work w/ context) --- .github/workflows/integration_test.yaml | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 73cda1468..07a1ca231 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -3,15 +3,6 @@ name: integration-tests on: pull_request: -env: - LXD_TEST_MODULES: >- - '["test_charm_fork_repo", "test_charm_no_runner", "test_charm_scheduled_events", - "test_charm_one_runner", "test_charm_metrics_success", "test_charm_metrics_failure", - "test_self_hosted_runner", "test_charm_with_proxy", "test_charm_with_juju_storage", - "test_debug_ssh"]' - OPENSTACK_TEST_MODULES: >- - '["test_openstack"]' - jobs: # test option values defined at test/conftest.py are passed on via repository secret # INTEGRATION_TEST_ARGS to operator-workflows automatically. @@ -24,7 +15,7 @@ jobs: pre-run-script: scripts/pre-integration-test.sh provider: lxd test-tox-env: integration-juju2.9 - modules: ${{ env.LXD_TEST_MODULES }} + modules: '["test_charm_fork_repo", "test_charm_no_runner", "test_charm_scheduled_events", "test_charm_one_runner", "test_charm_metrics_success", "test_charm_metrics_failure", "test_self_hosted_runner", "test_charm_with_proxy", "test_charm_with_juju_storage", "test_debug_ssh"]' integration-tests: name: Integration test with juju 3.1 uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main @@ -34,7 +25,7 @@ jobs: pre-run-script: scripts/pre-integration-test.sh provider: lxd test-tox-env: integration-juju3.1 - modules: ${{ env.LXD_TEST_MODULES }} + modules: '["test_charm_fork_repo", "test_charm_no_runner", "test_charm_scheduled_events", "test_charm_one_runner", "test_charm_metrics_success", "test_charm_metrics_failure", "test_self_hosted_runner", "test_charm_with_proxy", "test_charm_with_juju_storage", "test_debug_ssh"]' # openstack tests use microstack, whose setup is kind of special # - due to the huge resource requirements, we use self-hosted runners for these tests # - microstack requires juju 3.2 and microk8s 1.26 @@ -51,6 +42,6 @@ jobs: channel: 1.26-strict/stable microk8s-addons: "dns ingress hostpath-storage" test-tox-env: integration-juju3.2 - modules: ${{ env.OPENSTACK_TEST_MODULES }} + modules: '["test_openstack"]' self-hosted-runner: true self-hosted-runner-label: two-xlarge From af07732b671251c1f3bf6d367fcbeda6c21af6d7 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Tue, 9 Apr 2024 20:27:01 +0800 Subject: [PATCH 15/24] test: generate app name for function scope --- tests/integration/conftest.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 44cde993d..400cff774 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -221,7 +221,6 @@ async def app_no_runner( async def app_openstack_runner_fixture( model: Model, charm_file: str, - app_name: str, path: str, token: str, http_proxy: str, @@ -232,8 +231,8 @@ async def app_openstack_runner_fixture( """Application launching VMs and no runners.""" application = await deploy_github_runner_charm( model=model, + app_name=f"integration-id{secrets.token_hex(2)}", charm_file=charm_file, - app_name=app_name, path=path, token=token, runner_storage="juju-storage", From 4838d22cef121f38dd00a6db6ee7707baddc58aa Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Wed, 10 Apr 2024 09:36:12 +0800 Subject: [PATCH 16/24] test: cleanup after test --- tests/integration/conftest.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 400cff774..24383fc1a 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -8,7 +8,7 @@ import zipfile from pathlib import Path from time import sleep -from typing import Any, AsyncIterator, Iterator, Optional +from typing import Any, AsyncGenerator, AsyncIterator, Generator, Iterator, Optional import openstack import openstack.connection @@ -22,6 +22,7 @@ from juju.application import Application from juju.client._definitions import FullStatus, UnitStatus from juju.model import Model +from openstack.compute.v2.server import Server from pytest_operator.plugin import OpsTest from charm_state import ( @@ -166,7 +167,7 @@ def openstack_clouds_yaml(pytestconfig: pytest.Config) -> Optional[str]: @pytest.fixture(scope="module", name="openstack_connection") def openstack_connection_fixture( openstack_clouds_yaml: Optional[str], -) -> openstack.connection.Connection: +) -> Generator[openstack.connection.Connection, None, None]: """The openstack connection instance.""" assert openstack_clouds_yaml, "Openstack clouds yaml was not provided." @@ -174,7 +175,12 @@ def openstack_connection_fixture( clouds_yaml_path = Path.cwd() / "clouds.yaml" clouds_yaml_path.write_text(data=openstack_clouds_yaml, encoding="utf-8") first_cloud = next(iter(openstack_clouds_yaml_yaml["clouds"].keys())) - return openstack.connect(first_cloud) + with openstack.connect(first_cloud) as conn: + yield conn + + server: Server + for server in conn.list_servers(): + conn.delete_server(server.name) @pytest.fixture(scope="module") @@ -227,7 +233,7 @@ async def app_openstack_runner_fixture( https_proxy: str, no_proxy: str, openstack_clouds_yaml: str, -) -> AsyncIterator[Application]: +) -> AsyncGenerator[Application, None]: """Application launching VMs and no runners.""" application = await deploy_github_runner_charm( model=model, @@ -249,7 +255,9 @@ async def app_openstack_runner_fixture( config={OPENSTACK_CLOUDS_YAML_CONFIG_NAME: openstack_clouds_yaml}, wait_idle=False, ) - return application + yield application + + model.remove_application(application.name) @pytest_asyncio.fixture(scope="module") From 88b756a7ee14e358c0749f6a475dba784e8d0ea0 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Wed, 10 Apr 2024 09:36:56 +0800 Subject: [PATCH 17/24] test: cleanup per function --- tests/integration/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 24383fc1a..b76bda84a 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -164,7 +164,7 @@ def openstack_clouds_yaml(pytestconfig: pytest.Config) -> Optional[str]: return Path(clouds_yaml).read_text(encoding="utf-8") if clouds_yaml else None -@pytest.fixture(scope="module", name="openstack_connection") +@pytest.fixture(scope="function", name="openstack_connection") def openstack_connection_fixture( openstack_clouds_yaml: Optional[str], ) -> Generator[openstack.connection.Connection, None, None]: From 69d22bff5ed853b0be079cab4f309fe4a5dd2715 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Wed, 10 Apr 2024 12:29:10 +0800 Subject: [PATCH 18/24] test: test fixes --- src-docs/openstack_manager.md | 16 ++++++++-------- src/openstack_cloud/openstack_manager.py | 24 +++++++++++++++++------- tests/integration/test_openstack.py | 20 +++++++++++++------- 3 files changed, 38 insertions(+), 22 deletions(-) diff --git a/src-docs/openstack_manager.md b/src-docs/openstack_manager.md index f6b04c334..a97034f7d 100644 --- a/src-docs/openstack_manager.md +++ b/src-docs/openstack_manager.md @@ -8,12 +8,12 @@ Module for handling interactions with OpenStack. **Global Variables** --------------- - **IMAGE_PATH_TMPL** -- **IMAGE_NAME** +- **IMAGE_NAME_TMPL** - **BUILD_OPENSTACK_IMAGE_SCRIPT_FILENAME** --- - + ## function `build_image` @@ -51,7 +51,7 @@ Build and upload an image to OpenStack. --- - + ## function `create_instance_config` @@ -83,7 +83,7 @@ Create an instance config from charm data. --- - + ## function `create_instance` @@ -116,7 +116,7 @@ Create an OpenStack instance. --- - + ## class `ProxyStringValues` Wrapper class to proxy values to string. @@ -135,7 +135,7 @@ Wrapper class to proxy values to string. --- - + ## class `InstanceConfig` The configuration values for creating a single runner instance. @@ -174,7 +174,7 @@ __init__( --- - + ## class `BuildImageConfig` The configuration values for building openstack image. @@ -209,7 +209,7 @@ __init__( --- - + ## class `ImageDeleteError` Represents an error while deleting existing openstack image. diff --git a/src/openstack_cloud/openstack_manager.py b/src/openstack_cloud/openstack_manager.py index ffb82da2c..2923b5669 100644 --- a/src/openstack_cloud/openstack_manager.py +++ b/src/openstack_cloud/openstack_manager.py @@ -38,8 +38,9 @@ logger = logging.getLogger(__name__) -IMAGE_PATH_TMPL = "jammy-server-cloudimg-{architecture}-compressed.img" -IMAGE_NAME = "jammy" +IMAGE_PATH_TMPL = "{base_image}-server-cloudimg-{architecture}-compressed.img" +# Update the version when the image are modified. +IMAGE_NAME_TMPL = "github-runner-{base_image}-v1" BUILD_OPENSTACK_IMAGE_SCRIPT_FILENAME = "scripts/build-openstack-image.sh" @@ -263,12 +264,15 @@ class ImageDeleteError(Exception): """Represents an error while deleting existing openstack image.""" -def _put_image(cloud_config: dict[str, dict], image_arch: SupportedCloudImageArch) -> str: +def _put_image( + cloud_config: dict[str, dict], image_arch: SupportedCloudImageArch, base_image: BaseImage +) -> str: """Create or replace the image with existing name. Args: cloud_config: The cloud configuration to connect OpenStack with. image_arch: Ubuntu cloud image architecture. + base_image: The ubuntu base image to use. Raises: ImageDeleteError: If there was an error deleting the image. @@ -280,14 +284,18 @@ def _put_image(cloud_config: dict[str, dict], image_arch: SupportedCloudImageArc try: with _create_connection(cloud_config) as conn: existing_image: openstack.image.v2.image.Image - for existing_image in conn.search_images(name_or_id=IMAGE_NAME): + for existing_image in conn.search_images( + name_or_id=IMAGE_NAME_TMPL.format(base_image=base_image) + ): # images with same name (different ID) can be created and will error during server # instantiation. if not conn.delete_image(name_or_id=existing_image.id, wait=True): raise ImageDeleteError("Failed to delete duplicate image on Openstack.") image: openstack.image.v2.image.Image = conn.create_image( - name=IMAGE_NAME, - filename=IMAGE_PATH_TMPL.format(architecture=image_arch), + name=IMAGE_NAME_TMPL.format(base_image=base_image.value), + filename=IMAGE_PATH_TMPL.format( + architecture=image_arch, base_image=base_image.value + ), wait=True, ) return image.id @@ -336,7 +344,9 @@ def build_image( raise OpenstackImageBuildError(f"Unsupported architecture {runner_arch}") from exc try: - return _put_image(cloud_config=cloud_config, image_arch=image_arch) + return _put_image( + cloud_config=cloud_config, image_arch=image_arch, base_image=config.base_image + ) except (ImageDeleteError, OpenStackCloudException) as exc: raise OpenstackImageBuildError(f"Failed to upload image: {str(exc)}") from exc diff --git a/tests/integration/test_openstack.py b/tests/integration/test_openstack.py index 1f82a1123..4c1e1f428 100644 --- a/tests/integration/test_openstack.py +++ b/tests/integration/test_openstack.py @@ -13,11 +13,7 @@ from openstack.compute.v2.server import Server from charm_state import BASE_IMAGE_CONFIG_NAME -from tests.integration.helpers import ( - DISPATCH_E2E_TEST_RUN_WORKFLOW_FILENAME, - dispatch_workflow, - ensure_charm_has_runner, -) +from tests.integration.helpers import DISPATCH_E2E_TEST_RUN_WORKFLOW_FILENAME, dispatch_workflow # 2024/03/19 - The firewall configuration on openstack will be implemented by follow up PR on @@ -65,6 +61,7 @@ async def test_openstack_integration( async def test_noble_base_image( model: Model, app_openstack_runner: Application, + openstack_connection: openstack.connection.Connection, github_repository: Repository, test_github_branch: Branch, ) -> None: @@ -78,7 +75,9 @@ async def test_noble_base_image( BASE_IMAGE_CONFIG_NAME: "noble", } ) - await ensure_charm_has_runner(app_openstack_runner, model) + await model.wait_for_idle(apps=[app_openstack_runner.name], status="blocked", timeout=40 * 60) + + # 1. when the e2e_test_run workflow is created. workflow = await dispatch_workflow( app=app_openstack_runner, branch=test_github_branch, @@ -87,6 +86,13 @@ async def test_noble_base_image( workflow_id_or_name=DISPATCH_E2E_TEST_RUN_WORKFLOW_FILENAME, dispatch_input={"runner-tag": app_openstack_runner.name}, ) - + # 1. the workflow run completes successfully. workflow_run: WorkflowRun = workflow.get_runs()[0] assert workflow_run.status == "success" + + # 2. when the servers are listed. + servers = openstack_connection.list_servers(detailed=True) + assert len(servers) == 1, f"Unexpected number of servers: {len(servers)}" + server: Server = servers[0] + # 2. a server with image name noble is created. + assert server.image.name == "noble" From a324e45cadfada75ecc51c7b8abd1bcf75aaca68 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Wed, 10 Apr 2024 12:41:22 +0800 Subject: [PATCH 19/24] test: fix & separate tests --- .github/workflows/integration_test.yaml | 6 +- tests/integration/test_charm_base_image.py | 56 +++++++++++++++++++ tests/integration/test_charm_one_runner.py | 35 ------------ .../integration/test_openstack_base_image.py | 55 ++++++++++++++++++ 4 files changed, 114 insertions(+), 38 deletions(-) create mode 100644 tests/integration/test_charm_base_image.py create mode 100644 tests/integration/test_openstack_base_image.py diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 07a1ca231..cfe5c5dc3 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -15,7 +15,7 @@ jobs: pre-run-script: scripts/pre-integration-test.sh provider: lxd test-tox-env: integration-juju2.9 - modules: '["test_charm_fork_repo", "test_charm_no_runner", "test_charm_scheduled_events", "test_charm_one_runner", "test_charm_metrics_success", "test_charm_metrics_failure", "test_self_hosted_runner", "test_charm_with_proxy", "test_charm_with_juju_storage", "test_debug_ssh"]' + modules: '["test_charm_base_image", "test_charm_fork_repo", "test_charm_no_runner", "test_charm_scheduled_events", "test_charm_one_runner", "test_charm_metrics_success", "test_charm_metrics_failure", "test_self_hosted_runner", "test_charm_with_proxy", "test_charm_with_juju_storage", "test_debug_ssh"]' integration-tests: name: Integration test with juju 3.1 uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main @@ -25,7 +25,7 @@ jobs: pre-run-script: scripts/pre-integration-test.sh provider: lxd test-tox-env: integration-juju3.1 - modules: '["test_charm_fork_repo", "test_charm_no_runner", "test_charm_scheduled_events", "test_charm_one_runner", "test_charm_metrics_success", "test_charm_metrics_failure", "test_self_hosted_runner", "test_charm_with_proxy", "test_charm_with_juju_storage", "test_debug_ssh"]' + modules: '["test_charm_base_image", "test_charm_fork_repo", "test_charm_no_runner", "test_charm_scheduled_events", "test_charm_one_runner", "test_charm_metrics_success", "test_charm_metrics_failure", "test_self_hosted_runner", "test_charm_with_proxy", "test_charm_with_juju_storage", "test_debug_ssh"]' # openstack tests use microstack, whose setup is kind of special # - due to the huge resource requirements, we use self-hosted runners for these tests # - microstack requires juju 3.2 and microk8s 1.26 @@ -42,6 +42,6 @@ jobs: channel: 1.26-strict/stable microk8s-addons: "dns ingress hostpath-storage" test-tox-env: integration-juju3.2 - modules: '["test_openstack"]' + modules: '["test_openstack_base_image", "test_openstack"]' self-hosted-runner: true self-hosted-runner-label: two-xlarge diff --git a/tests/integration/test_charm_base_image.py b/tests/integration/test_charm_base_image.py new file mode 100644 index 000000000..9994af124 --- /dev/null +++ b/tests/integration/test_charm_base_image.py @@ -0,0 +1,56 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Integration tests for github-runner charm containing one runner.""" + +from github.Branch import Branch +from github.Repository import Repository +from github.WorkflowRun import WorkflowRun +from juju.application import Application +from juju.model import Model + +from charm_state import BASE_IMAGE_CONFIG_NAME +from tests.integration.helpers import ( + DISPATCH_E2E_TEST_RUN_WORKFLOW_FILENAME, + dispatch_workflow, + ensure_charm_has_runner, + get_runner_name, + run_in_lxd_instance, +) + + +async def test_runner_base_image( + model: Model, + app_no_runner: Application, + github_repository: Repository, + test_github_branch: Branch, +) -> None: + """ + arrange: A runner with noble as base image. + act: Dispatch a workflow. + assert: A runner is created with noble OS base and the workflow job is successfully run. + """ + await app_no_runner.set_config( + { + BASE_IMAGE_CONFIG_NAME: "noble", + } + ) + await ensure_charm_has_runner(app_no_runner, model) + + unit = app_no_runner.units[0] + runner_name = await get_runner_name(unit) + code, stdout, stderr = await run_in_lxd_instance(unit, runner_name, "lsb_release -a") + assert code == 0, f"Unable to get release name, {stdout} {stderr}" + assert "noble" in str(stdout) + + workflow = await dispatch_workflow( + app=app_no_runner, + branch=test_github_branch, + github_repository=github_repository, + conclusion="success", + workflow_id_or_name=DISPATCH_E2E_TEST_RUN_WORKFLOW_FILENAME, + dispatch_input={"runner-tag": app_no_runner.name}, + ) + + workflow_run: WorkflowRun = workflow.get_runs()[0] + assert workflow_run.status == "success" diff --git a/tests/integration/test_charm_one_runner.py b/tests/integration/test_charm_one_runner.py index 3b0ff98d6..5d6b4d89f 100644 --- a/tests/integration/test_charm_one_runner.py +++ b/tests/integration/test_charm_one_runner.py @@ -6,15 +6,12 @@ import pytest import pytest_asyncio -from github.Branch import Branch from github.Repository import Repository -from github.WorkflowRun import WorkflowRun from juju.application import Application from juju.model import Model from charm import GithubRunnerCharm from charm_state import ( - BASE_IMAGE_CONFIG_NAME, RUNNER_STORAGE_CONFIG_NAME, TOKEN_CONFIG_NAME, VIRTUAL_MACHINES_CONFIG_NAME, @@ -23,9 +20,7 @@ VM_MEMORY_CONFIG_NAME, ) from tests.integration.helpers import ( - DISPATCH_E2E_TEST_RUN_WORKFLOW_FILENAME, assert_resource_lxd_profile, - dispatch_workflow, ensure_charm_has_runner, get_runner_names, reconcile, @@ -310,33 +305,3 @@ async def test_token_config_changed_insufficient_perms( await model.wait_for_idle() await wait_till_num_of_runners(unit, num=0) - - -async def test_runner_base_image( - model: Model, - app_no_runner: Application, - github_repository: Repository, - test_github_branch: Branch, -) -> None: - """ - arrange: A runner with noble as base image. - act: Dispatch a workflow. - assert: A runner should work with the different images. - """ - await app_no_runner.set_config( - { - BASE_IMAGE_CONFIG_NAME: "noble", - } - ) - await ensure_charm_has_runner(app_no_runner, model) - workflow = await dispatch_workflow( - app=app_no_runner, - branch=test_github_branch, - github_repository=github_repository, - conclusion="success", - workflow_id_or_name=DISPATCH_E2E_TEST_RUN_WORKFLOW_FILENAME, - dispatch_input={"runner-tag": app_no_runner.name}, - ) - - workflow_run: WorkflowRun = workflow.get_runs()[0] - assert workflow_run.status == "success" diff --git a/tests/integration/test_openstack_base_image.py b/tests/integration/test_openstack_base_image.py new file mode 100644 index 000000000..b800c7757 --- /dev/null +++ b/tests/integration/test_openstack_base_image.py @@ -0,0 +1,55 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Integration tests for OpenStack integration.""" + +import openstack.connection +from github.Branch import Branch +from github.Repository import Repository +from github.WorkflowRun import WorkflowRun +from juju.application import Application +from juju.model import Model +from openstack.compute.v2.server import Server + +from charm_state import BASE_IMAGE_CONFIG_NAME +from tests.integration.helpers import DISPATCH_E2E_TEST_RUN_WORKFLOW_FILENAME, dispatch_workflow + + +async def test_noble_base_image( + model: Model, + app_openstack_runner: Application, + openstack_connection: openstack.connection.Connection, + github_repository: Repository, + test_github_branch: Branch, +) -> None: + """ + arrange: A runner with noble as base image. + act: Dispatch a workflow. + assert: A runner should work with the different base image. + """ + await app_openstack_runner.set_config( + { + BASE_IMAGE_CONFIG_NAME: "noble", + } + ) + await model.wait_for_idle(apps=[app_openstack_runner.name], status="blocked", timeout=40 * 60) + + # 1. when the e2e_test_run workflow is created. + workflow = await dispatch_workflow( + app=app_openstack_runner, + branch=test_github_branch, + github_repository=github_repository, + conclusion="success", + workflow_id_or_name=DISPATCH_E2E_TEST_RUN_WORKFLOW_FILENAME, + dispatch_input={"runner-tag": app_openstack_runner.name}, + ) + # 1. the workflow run completes successfully. + workflow_run: WorkflowRun = workflow.get_runs()[0] + assert workflow_run.status == "success" + + # 2. when the servers are listed. + servers = openstack_connection.list_servers(detailed=True) + assert len(servers) == 1, f"Unexpected number of servers: {len(servers)}" + server: Server = servers[0] + # 2. a server with image name containing noble is created. + assert "noble" in server.image.name From 8e6602867536279af6a0cc122605d9ec1fe120b0 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Wed, 10 Apr 2024 12:45:22 +0800 Subject: [PATCH 20/24] test: revert function openstack fixture --- tests/integration/conftest.py | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index b76bda84a..1685323e9 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -8,7 +8,7 @@ import zipfile from pathlib import Path from time import sleep -from typing import Any, AsyncGenerator, AsyncIterator, Generator, Iterator, Optional +from typing import Any, AsyncIterator, Iterator, Optional import openstack import openstack.connection @@ -22,7 +22,6 @@ from juju.application import Application from juju.client._definitions import FullStatus, UnitStatus from juju.model import Model -from openstack.compute.v2.server import Server from pytest_operator.plugin import OpsTest from charm_state import ( @@ -164,10 +163,10 @@ def openstack_clouds_yaml(pytestconfig: pytest.Config) -> Optional[str]: return Path(clouds_yaml).read_text(encoding="utf-8") if clouds_yaml else None -@pytest.fixture(scope="function", name="openstack_connection") +@pytest.fixture(scope="module", name="openstack_connection") def openstack_connection_fixture( openstack_clouds_yaml: Optional[str], -) -> Generator[openstack.connection.Connection, None, None]: +) -> openstack.connection.Connection: """The openstack connection instance.""" assert openstack_clouds_yaml, "Openstack clouds yaml was not provided." @@ -175,12 +174,7 @@ def openstack_connection_fixture( clouds_yaml_path = Path.cwd() / "clouds.yaml" clouds_yaml_path.write_text(data=openstack_clouds_yaml, encoding="utf-8") first_cloud = next(iter(openstack_clouds_yaml_yaml["clouds"].keys())) - with openstack.connect(first_cloud) as conn: - yield conn - - server: Server - for server in conn.list_servers(): - conn.delete_server(server.name) + return openstack.connect(first_cloud) @pytest.fixture(scope="module") @@ -223,22 +217,23 @@ async def app_no_runner( return application -@pytest_asyncio.fixture(scope="function", name="app_openstack_runner") +@pytest_asyncio.fixture(scope="module", name="app_openstack_runner") async def app_openstack_runner_fixture( model: Model, charm_file: str, + app_name: str, path: str, token: str, http_proxy: str, https_proxy: str, no_proxy: str, openstack_clouds_yaml: str, -) -> AsyncGenerator[Application, None]: +) -> AsyncIterator[Application]: """Application launching VMs and no runners.""" application = await deploy_github_runner_charm( model=model, - app_name=f"integration-id{secrets.token_hex(2)}", charm_file=charm_file, + app_name=app_name, path=path, token=token, runner_storage="juju-storage", @@ -255,9 +250,7 @@ async def app_openstack_runner_fixture( config={OPENSTACK_CLOUDS_YAML_CONFIG_NAME: openstack_clouds_yaml}, wait_idle=False, ) - yield application - - model.remove_application(application.name) + return application @pytest_asyncio.fixture(scope="module") From 83326f90c8af868cefacd72f991c36f6329a7788 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Wed, 10 Apr 2024 16:31:12 +0800 Subject: [PATCH 21/24] test: increase timeout --- tests/integration/test_charm_base_image.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_charm_base_image.py b/tests/integration/test_charm_base_image.py index 9994af124..711771c10 100644 --- a/tests/integration/test_charm_base_image.py +++ b/tests/integration/test_charm_base_image.py @@ -21,7 +21,7 @@ async def test_runner_base_image( model: Model, - app_no_runner: Application, + app_no_wait: Application, github_repository: Repository, test_github_branch: Branch, ) -> None: @@ -30,26 +30,27 @@ async def test_runner_base_image( act: Dispatch a workflow. assert: A runner is created with noble OS base and the workflow job is successfully run. """ - await app_no_runner.set_config( + await app_no_wait.set_config( { BASE_IMAGE_CONFIG_NAME: "noble", } ) - await ensure_charm_has_runner(app_no_runner, model) + await model.wait_for_idle(apps=[app_no_wait.name], timeout=35 * 60) + await ensure_charm_has_runner(app_no_wait, model) - unit = app_no_runner.units[0] + unit = app_no_wait.units[0] runner_name = await get_runner_name(unit) code, stdout, stderr = await run_in_lxd_instance(unit, runner_name, "lsb_release -a") assert code == 0, f"Unable to get release name, {stdout} {stderr}" assert "noble" in str(stdout) workflow = await dispatch_workflow( - app=app_no_runner, + app=app_no_wait, branch=test_github_branch, github_repository=github_repository, conclusion="success", workflow_id_or_name=DISPATCH_E2E_TEST_RUN_WORKFLOW_FILENAME, - dispatch_input={"runner-tag": app_no_runner.name}, + dispatch_input={"runner-tag": app_no_wait.name}, ) workflow_run: WorkflowRun = workflow.get_runs()[0] From 6bdd4f1b033ebb54f74e13305378453ec5d65401 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Wed, 10 Apr 2024 18:42:05 +0800 Subject: [PATCH 22/24] test: increase timeout for openstack --- tests/integration/test_openstack_base_image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_openstack_base_image.py b/tests/integration/test_openstack_base_image.py index b800c7757..82418f3bf 100644 --- a/tests/integration/test_openstack_base_image.py +++ b/tests/integration/test_openstack_base_image.py @@ -32,7 +32,7 @@ async def test_noble_base_image( BASE_IMAGE_CONFIG_NAME: "noble", } ) - await model.wait_for_idle(apps=[app_openstack_runner.name], status="blocked", timeout=40 * 60) + await model.wait_for_idle(apps=[app_openstack_runner.name], status="blocked", timeout=50 * 60) # 1. when the e2e_test_run workflow is created. workflow = await dispatch_workflow( From bef451b562b1cf24ec9fe1ac394fa50fd845dca4 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Wed, 10 Apr 2024 18:42:46 +0800 Subject: [PATCH 23/24] test: specify file for openstack --- .github/workflows/integration_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index cfe5c5dc3..8e7b6cd53 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -42,6 +42,6 @@ jobs: channel: 1.26-strict/stable microk8s-addons: "dns ingress hostpath-storage" test-tox-env: integration-juju3.2 - modules: '["test_openstack_base_image", "test_openstack"]' + modules: '["test_openstack_base_image.py", "test_openstack.py"]' self-hosted-runner: true self-hosted-runner-label: two-xlarge From 8bd3cc2b54fbf32f005ebdc728c2584fdb782139 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Wed, 10 Apr 2024 18:43:43 +0800 Subject: [PATCH 24/24] test: rename tests --- .github/workflows/integration_test.yaml | 2 +- .../{test_openstack.py => test_openstack_one_runner.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename tests/integration/{test_openstack.py => test_openstack_one_runner.py} (100%) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 8e7b6cd53..0e8078536 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -42,6 +42,6 @@ jobs: channel: 1.26-strict/stable microk8s-addons: "dns ingress hostpath-storage" test-tox-env: integration-juju3.2 - modules: '["test_openstack_base_image.py", "test_openstack.py"]' + modules: '["test_openstack_base_image", "test_openstack_one_runner"]' self-hosted-runner: true self-hosted-runner-label: two-xlarge diff --git a/tests/integration/test_openstack.py b/tests/integration/test_openstack_one_runner.py similarity index 100% rename from tests/integration/test_openstack.py rename to tests/integration/test_openstack_one_runner.py