From a9dc05d1caf0759e76d35f4cd166ca26560d4a17 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 13 Sep 2024 17:15:19 +0800 Subject: [PATCH 01/34] Update to use openstack credential directly --- requirements.txt | 2 +- src/charm.py | 16 +++++++++---- .../test_runner_manager_openstack.py | 24 ++++++++++++++----- 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/requirements.txt b/requirements.txt index 6be16381d..439b4ee36 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,4 +14,4 @@ PyYAML ==6.0.* pyOpenSSL==24.2.1 kombu==5.4.1 pymongo==4.8.0 -github_runner_manager @ git+https://github.com/canonical/github-runner-manager.git@771530633cd8b3cb18cb95399c234c82118b77e2 +github_runner_manager @ git+https://github.com/canonical/github-runner-manager.git@fix/openstack-connection-creation diff --git a/src/charm.py b/src/charm.py index 9dae00059..7cff58b45 100755 --- a/src/charm.py +++ b/src/charm.py @@ -18,7 +18,7 @@ ) from github_runner_manager.manager.runner_scaler import RunnerScaler from github_runner_manager.openstack_cloud.openstack_runner_manager import ( - OpenStackCloudConfig, + OpenStackCredentials, OpenStackRunnerManager, OpenStackServerConfig, ) @@ -1251,9 +1251,15 @@ def _get_runner_scaler( logger.warning( "Multiple clouds defined in clouds.yaml. Using the first one to connect." ) - cloud_config = OpenStackCloudConfig( - clouds_config=state.charm_config.openstack_clouds_yaml, - cloud=clouds[0], + first_cloud_config = first_cloud_config[clouds[0]] + credentials = OpenStackCredentials( + auth_url=first_cloud_config["auth"]["auth_url"], + project_name=first_cloud_config["auth"]["project_name"], + username=first_cloud_config["auth"]["username"], + password=first_cloud_config["auth"]["password"], + user_domain_name=first_cloud_config["auth"]["user_domain_name"], + project_domain_name=first_cloud_config["auth"]["project_domain_name"], + region_name=first_cloud_config["region_name"], ) server_config = None image_labels = [] @@ -1280,7 +1286,7 @@ def _get_runner_scaler( openstack_runner_manager = OpenStackRunnerManager( manager_name=self.app.name, prefix=self.unit.name.replace("/", "-"), - cloud_config=cloud_config, + credentials=credentials, server_config=server_config, runner_config=runner_config, service_config=service_config, diff --git a/tests/integration/test_runner_manager_openstack.py b/tests/integration/test_runner_manager_openstack.py index cb88d84ba..fe4a93c25 100644 --- a/tests/integration/test_runner_manager_openstack.py +++ b/tests/integration/test_runner_manager_openstack.py @@ -29,7 +29,7 @@ from github_runner_manager.metrics import events, storage from github_runner_manager.openstack_cloud.openstack_cloud import _CLOUDS_YAML_PATH from github_runner_manager.openstack_cloud.openstack_runner_manager import ( - OpenStackCloudConfig, + OpenStackCredentials, OpenStackRunnerManager, OpenStackServerConfig, ) @@ -114,10 +114,22 @@ async def openstack_runner_manager_fixture( _CLOUDS_YAML_PATH.unlink(missing_ok=True) clouds_config = yaml.safe_load(private_endpoint_clouds_yaml) - cloud_config = OpenStackCloudConfig( - clouds_config=clouds_config, - cloud="testcloud", - ) + try: + # Pick the first cloud in the clouds.yaml + cloud = clouds_config["clouds"].values()[0] + + credentials = OpenStackCredentials( + auth_url=cloud["auth"]["auth_url"], + project_name=cloud["auth"]["project_name"], + username=cloud["auth"]["username"], + password=cloud["auth"]["password"], + user_domain_name=cloud["auth"]["user_domain_name"], + project_domain_name=cloud["auth"]["project_domain_name"], + region_name=cloud["region_name"], + ) + except KeyError as err: + raise AssertionError("Issue with the format of the clouds.yaml used in test") from err + server_config = OpenStackServerConfig( image=openstack_test_image, flavor=flavor_name, @@ -134,7 +146,7 @@ async def openstack_runner_manager_fixture( repo_policy_compliance=None, ) return OpenStackRunnerManager( - app_name, f"{app_name}-0", cloud_config, server_config, runner_config, service_config + app_name, f"{app_name}-0", credentials, server_config, runner_config, service_config ) From b989fc8efa33a06550a7ed49896da46c3a0f427a Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 18 Sep 2024 20:16:01 +0800 Subject: [PATCH 02/34] Remove test for removed code --- src/charm_state.py | 8 -------- .../test_runner_manager_openstack.py | 2 +- tests/unit/test_charm_state.py | 20 ------------------- 3 files changed, 1 insertion(+), 29 deletions(-) diff --git a/src/charm_state.py b/src/charm_state.py index 6ae46d386..7edf7c957 100644 --- a/src/charm_state.py +++ b/src/charm_state.py @@ -430,14 +430,6 @@ def _parse_openstack_clouds_config(cls, charm: CharmBase) -> OpenStackCloudsYAML f"Invalid {OPENSTACK_CLOUDS_YAML_CONFIG_NAME} config. Invalid yaml." ) from exc - try: - openstack_cloud.initialize(openstack_clouds_yaml) - except OpenStackInvalidConfigError as exc: - logger.error("Invalid openstack config, %s.", exc) - raise CharmConfigInvalidError( - "Invalid openstack config. Not able to initialize openstack integration." - ) from exc - return openstack_clouds_yaml @validator("reconcile_interval") diff --git a/tests/integration/test_runner_manager_openstack.py b/tests/integration/test_runner_manager_openstack.py index fe4a93c25..b957fb664 100644 --- a/tests/integration/test_runner_manager_openstack.py +++ b/tests/integration/test_runner_manager_openstack.py @@ -115,7 +115,7 @@ async def openstack_runner_manager_fixture( clouds_config = yaml.safe_load(private_endpoint_clouds_yaml) try: - # Pick the first cloud in the clouds.yaml + # Pick the first cloud in the clouds.yaml cloud = clouds_config["clouds"].values()[0] credentials = OpenStackCredentials( diff --git a/tests/unit/test_charm_state.py b/tests/unit/test_charm_state.py index b7df8a5dc..f1a56b404 100644 --- a/tests/unit/test_charm_state.py +++ b/tests/unit/test_charm_state.py @@ -357,26 +357,6 @@ def test_parse_openstack_clouds_config_invalid_yaml_list(): CharmConfig._parse_openstack_clouds_config(mock_charm) -def test_parse_openstack_clouds_initialize_fail( - valid_yaml_config: str, monkeypatch: pytest.MonkeyPatch -): - """ - arrange: Given monkeypatched openstack_cloud.initialize that raises an error. - act: Call _parse_openstack_clouds_config method with the mock CharmBase instance. - assert: Verify that the method raises CharmConfigInvalidError. - """ - mock_charm = MockGithubRunnerCharmFactory() - mock_charm.config[OPENSTACK_CLOUDS_YAML_CONFIG_NAME] = valid_yaml_config - monkeypatch.setattr( - github_runner_manager.openstack_cloud, - "initialize", - MagicMock(side_effect=github_runner_manager.openstack_cloud.OpenStackInvalidConfigError), - ) - - with pytest.raises(CharmConfigInvalidError): - CharmConfig._parse_openstack_clouds_config(mock_charm) - - def test_parse_openstack_clouds_config_valid(valid_yaml_config: str): """ arrange: Create a mock CharmBase instance with a valid OpenStack clouds YAML config. From 0c2679344fff861d3eee55239b24ebe97f813e5c Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 19 Sep 2024 14:37:02 +0800 Subject: [PATCH 03/34] Fix the unit tests --- src/charm.py | 2 +- src/charm_state.py | 2 -- tests/unit/test_charm_state.py | 1 - 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/charm.py b/src/charm.py index 7cff58b45..dd97c91ba 100755 --- a/src/charm.py +++ b/src/charm.py @@ -1251,7 +1251,7 @@ def _get_runner_scaler( logger.warning( "Multiple clouds defined in clouds.yaml. Using the first one to connect." ) - first_cloud_config = first_cloud_config[clouds[0]] + first_cloud_config = clouds[clouds[0]] credentials = OpenStackCredentials( auth_url=first_cloud_config["auth"]["auth_url"], project_name=first_cloud_config["auth"]["project_name"], diff --git a/src/charm_state.py b/src/charm_state.py index 7edf7c957..5aa6774e0 100644 --- a/src/charm_state.py +++ b/src/charm_state.py @@ -19,8 +19,6 @@ import yaml from charms.data_platform_libs.v0.data_interfaces import DatabaseRequires -from github_runner_manager import openstack_cloud -from github_runner_manager.errors import OpenStackInvalidConfigError from github_runner_manager.types_.github import GitHubPath, parse_github_path from ops import CharmBase from pydantic import ( diff --git a/tests/unit/test_charm_state.py b/tests/unit/test_charm_state.py index f1a56b404..b3a8376a6 100644 --- a/tests/unit/test_charm_state.py +++ b/tests/unit/test_charm_state.py @@ -8,7 +8,6 @@ from pathlib import Path from unittest.mock import MagicMock -import github_runner_manager.openstack_cloud import pytest import yaml from charms.data_platform_libs.v0.data_interfaces import DatabaseRequires From 9c3595a6e86260e5f8fa3fb8a89d8fd893b529b4 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 23 Sep 2024 09:05:54 +0800 Subject: [PATCH 04/34] Fix dict values type error --- tests/integration/test_runner_manager_openstack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_runner_manager_openstack.py b/tests/integration/test_runner_manager_openstack.py index b957fb664..1a7b0b7cd 100644 --- a/tests/integration/test_runner_manager_openstack.py +++ b/tests/integration/test_runner_manager_openstack.py @@ -116,7 +116,7 @@ async def openstack_runner_manager_fixture( try: # Pick the first cloud in the clouds.yaml - cloud = clouds_config["clouds"].values()[0] + cloud = tuple(clouds_config["clouds"].values())[0] credentials = OpenStackCredentials( auth_url=cloud["auth"]["auth_url"], From d9eb66a749b1f4a65ec10f760e1dcd6f83da97bb Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 23 Sep 2024 13:59:30 +0800 Subject: [PATCH 05/34] Fix dependency issues --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a492bdd04..b476649b9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ cosl ==0.0.15 # juju 3.1.2.0 depends on pyyaml<=6.0 and >=5.1.2 PyYAML ==6.0.* pyOpenSSL==24.2.1 -kombu==5.4.1 +kombu==5.4.2 pymongo==4.8.0 # TODO: Use commit after PR is merge. github_runner_manager @ git+https://github.com/canonical/github-runner-manager.git@fix/openstack-connection-creation From 1aa98193681706872458a393764e912442587366 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 23 Sep 2024 18:28:38 +0800 Subject: [PATCH 06/34] Try new deps --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b476649b9..874ce5dd2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ cosl ==0.0.15 # juju 3.1.2.0 depends on pyyaml<=6.0 and >=5.1.2 PyYAML ==6.0.* pyOpenSSL==24.2.1 -kombu==5.4.2 +kombu>=5.4 pymongo==4.8.0 # TODO: Use commit after PR is merge. github_runner_manager @ git+https://github.com/canonical/github-runner-manager.git@fix/openstack-connection-creation From c8edf13fa3876ef6afd906044c8976342319a899 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 23 Sep 2024 18:29:31 +0800 Subject: [PATCH 07/34] Try new deps --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 874ce5dd2..08ae8fc68 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,6 +13,6 @@ cosl ==0.0.15 PyYAML ==6.0.* pyOpenSSL==24.2.1 kombu>=5.4 -pymongo==4.8.0 +pymongo>=4.9 # TODO: Use commit after PR is merge. github_runner_manager @ git+https://github.com/canonical/github-runner-manager.git@fix/openstack-connection-creation From 20fbd203db258767ce1017a546e6eb5a1b792526 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 24 Sep 2024 10:19:22 +0800 Subject: [PATCH 08/34] Fix merge issue --- tests/integration/test_runner_manager_openstack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_runner_manager_openstack.py b/tests/integration/test_runner_manager_openstack.py index f0c234564..cdf394165 100644 --- a/tests/integration/test_runner_manager_openstack.py +++ b/tests/integration/test_runner_manager_openstack.py @@ -150,7 +150,7 @@ async def openstack_runner_manager_fixture( openstack_runner_manager_config = OpenStackRunnerManagerConfig( name=app_name, prefix=f"{app_name}-0", - cloud_config=cloud_config, + credentials=credentials, server_config=server_config, runner_config=runner_config, service_config=service_config, From 13da6e5bbd4ade42ede3a377cf96fae64ee878de Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 24 Sep 2024 13:52:14 +0800 Subject: [PATCH 09/34] Remove a todo comment --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e54dbd7e8..e2299262f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,5 +12,4 @@ cosl ==0.0.15 # juju 3.1.2.0 depends on pyyaml<=6.0 and >=5.1.2 PyYAML ==6.0.* pyOpenSSL==24.2.1 -# TODO: Use commit after PR is merge. github_runner_manager @ git+https://github.com/canonical/github-runner-manager.git@fix/openstack-connection-creation From 2c9eeefdbae5cdd2635d8cddc7372ce6e6ab086d Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 24 Sep 2024 16:02:15 +0800 Subject: [PATCH 10/34] Add debugging --- .github/workflows/e2e_test.yaml | 3 +++ .github/workflows/integration_test.yaml | 6 ++++++ requirements.txt | 1 + 3 files changed, 10 insertions(+) diff --git a/.github/workflows/e2e_test.yaml b/.github/workflows/e2e_test.yaml index 7d0383c12..443324bf5 100644 --- a/.github/workflows/e2e_test.yaml +++ b/.github/workflows/e2e_test.yaml @@ -20,6 +20,9 @@ jobs: extra-arguments: "-m openstack" self-hosted-runner: true self-hosted-runner-label: stg-private-endpoint + # TODO: debug only + tmate-debug: true + tmate-timeout: 600 required_status_checks: name: Required E2E Test Status Checks diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 8e0bc700a..d3af1ab14 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -36,6 +36,9 @@ jobs: modules: '["test_runner_manager_openstack"]' self-hosted-runner: true self-hosted-runner-label: stg-private-endpoint + # TODO: debug only + tmate-debug: true + tmate-timeout: 600 openstack-integration-tests-private-endpoint: name: Integration test using private-endpoint uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main @@ -50,3 +53,6 @@ jobs: extra-arguments: "-m openstack" self-hosted-runner: true self-hosted-runner-label: stg-private-endpoint + # TODO: debug only + tmate-debug: true + tmate-timeout: 600 diff --git a/requirements.txt b/requirements.txt index e2299262f..d5d3974e2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,4 +12,5 @@ cosl ==0.0.15 # juju 3.1.2.0 depends on pyyaml<=6.0 and >=5.1.2 PyYAML ==6.0.* pyOpenSSL==24.2.1 +# TODO: update the commit to use. github_runner_manager @ git+https://github.com/canonical/github-runner-manager.git@fix/openstack-connection-creation From 6100734920d4465f6c29d5fb19536796c4ed720b Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 24 Sep 2024 18:42:00 +0800 Subject: [PATCH 11/34] Fix merge issue --- src/charm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/charm.py b/src/charm.py index 3ba02e4dd..d3e3c1cf1 100755 --- a/src/charm.py +++ b/src/charm.py @@ -1311,7 +1311,7 @@ def _create_openstack_runner_manager_config( logger.warning( "Multiple clouds defined in clouds.yaml. Using the first one to connect." ) - first_cloud_config = clouds[clouds[0]] + first_cloud_config = clouds[0] credentials = OpenStackCredentials( auth_url=first_cloud_config["auth"]["auth_url"], project_name=first_cloud_config["auth"]["project_name"], From 89e827ae83203c2efc0986bec3572efb1d61aa11 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 25 Sep 2024 16:16:16 +0800 Subject: [PATCH 12/34] Remove debug --- .github/workflows/e2e_test.yaml | 3 --- .github/workflows/integration_test.yaml | 6 ------ 2 files changed, 9 deletions(-) diff --git a/.github/workflows/e2e_test.yaml b/.github/workflows/e2e_test.yaml index 443324bf5..7d0383c12 100644 --- a/.github/workflows/e2e_test.yaml +++ b/.github/workflows/e2e_test.yaml @@ -20,9 +20,6 @@ jobs: extra-arguments: "-m openstack" self-hosted-runner: true self-hosted-runner-label: stg-private-endpoint - # TODO: debug only - tmate-debug: true - tmate-timeout: 600 required_status_checks: name: Required E2E Test Status Checks diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index d3af1ab14..8e0bc700a 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -36,9 +36,6 @@ jobs: modules: '["test_runner_manager_openstack"]' self-hosted-runner: true self-hosted-runner-label: stg-private-endpoint - # TODO: debug only - tmate-debug: true - tmate-timeout: 600 openstack-integration-tests-private-endpoint: name: Integration test using private-endpoint uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main @@ -53,6 +50,3 @@ jobs: extra-arguments: "-m openstack" self-hosted-runner: true self-hosted-runner-label: stg-private-endpoint - # TODO: debug only - tmate-debug: true - tmate-timeout: 600 From 4d2ec320abab10e9be8a1933439029896b5f5246 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 26 Sep 2024 09:42:34 +0800 Subject: [PATCH 13/34] Fix clouds.yaml access --- src/charm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/charm.py b/src/charm.py index d3e3c1cf1..b7a7c6d86 100755 --- a/src/charm.py +++ b/src/charm.py @@ -1311,7 +1311,7 @@ def _create_openstack_runner_manager_config( logger.warning( "Multiple clouds defined in clouds.yaml. Using the first one to connect." ) - first_cloud_config = clouds[0] + first_cloud_config = state.charm_config.openstack_clouds_yaml[clouds[0]] credentials = OpenStackCredentials( auth_url=first_cloud_config["auth"]["auth_url"], project_name=first_cloud_config["auth"]["project_name"], From b4b0deacbebd7afd3ed4b77a7cde87b2771de1fc Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 26 Sep 2024 12:57:23 +0800 Subject: [PATCH 14/34] Fix clouds.yaml dict access --- src/charm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/charm.py b/src/charm.py index b7a7c6d86..3c78ac680 100755 --- a/src/charm.py +++ b/src/charm.py @@ -1311,7 +1311,7 @@ def _create_openstack_runner_manager_config( logger.warning( "Multiple clouds defined in clouds.yaml. Using the first one to connect." ) - first_cloud_config = state.charm_config.openstack_clouds_yaml[clouds[0]] + first_cloud_config = state.charm_config.openstack_clouds_yaml["clouds"][clouds[0]] credentials = OpenStackCredentials( auth_url=first_cloud_config["auth"]["auth_url"], project_name=first_cloud_config["auth"]["project_name"], From 6ee44e48eff07d3e635efd69bc9141d377019dd7 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 26 Sep 2024 14:16:13 +0800 Subject: [PATCH 15/34] Add comments on the auth fields of clouds.yaml --- src/charm.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/charm.py b/src/charm.py index 3c78ac680..68f416831 100755 --- a/src/charm.py +++ b/src/charm.py @@ -1311,6 +1311,10 @@ def _create_openstack_runner_manager_config( logger.warning( "Multiple clouds defined in clouds.yaml. Using the first one to connect." ) + # TODO: The fields will be replace with charm configuration, the clouds.yaml configuration + # will be removed. + # The clouds and the auth are assumed to be there as the `_parse_openstack_clouds_config` + # has use pydantic to verify the fields. first_cloud_config = state.charm_config.openstack_clouds_yaml["clouds"][clouds[0]] credentials = OpenStackCredentials( auth_url=first_cloud_config["auth"]["auth_url"], @@ -1319,7 +1323,7 @@ def _create_openstack_runner_manager_config( password=first_cloud_config["auth"]["password"], user_domain_name=first_cloud_config["auth"]["user_domain_name"], project_domain_name=first_cloud_config["auth"]["project_domain_name"], - region_name=first_cloud_config["region_name"], + region_name=first_cloud_config.get("region_name", None), ) server_config = None image_labels = [] From f9ad3196d08612a256f101788dbb41f0ee6046b7 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 26 Sep 2024 15:09:44 +0800 Subject: [PATCH 16/34] Fix fmt --- src/charm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/charm.py b/src/charm.py index 68f416831..6e58e2ad2 100755 --- a/src/charm.py +++ b/src/charm.py @@ -1311,9 +1311,9 @@ def _create_openstack_runner_manager_config( logger.warning( "Multiple clouds defined in clouds.yaml. Using the first one to connect." ) - # TODO: The fields will be replace with charm configuration, the clouds.yaml configuration + # TODO: The fields will be replace with charm configuration, the clouds.yaml configuration # will be removed. - # The clouds and the auth are assumed to be there as the `_parse_openstack_clouds_config` + # The clouds and the auth are assumed to be there as the `_parse_openstack_clouds_config` # has use pydantic to verify the fields. first_cloud_config = state.charm_config.openstack_clouds_yaml["clouds"][clouds[0]] credentials = OpenStackCredentials( From c1e91497b7e66dd17f5b8da2767e0aa626da1e42 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 27 Sep 2024 14:04:27 +0800 Subject: [PATCH 17/34] Add reactive debug test --- .github/workflows/integration_test.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 8e0bc700a..d07e05b19 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -24,6 +24,21 @@ jobs: # test_debug_ssh ensures tmate SSH actions works. # TODO: Add OpenStack integration versions of these tests. modules: '["test_charm_scheduled_events", "test_debug_ssh"]' + # TODO: debug + debug-reactive: + name: debug reactive + uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main + secrets: inherit + with: + juju-channel: 3.2/stable + pre-run-script: scripts/setup-lxd.sh + provider: lxd + test-tox-env: integration-juju3.2 + modules: '["test_reactive"]' + self-hosted-runner: true + self-hosted-runner-label: stg-private-endpoint + tmate-debug: true + tmate-timeout: 300 openstack-interface-tests-private-endpoint: name: openstack interface test using private-endpoint uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main From 4236ed4c83fc2caba23db6167ee1fcf8951e0ec7 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 27 Sep 2024 15:52:31 +0800 Subject: [PATCH 18/34] Debug reactive --- .github/workflows/integration_test.yaml | 81 +++++++++++++------------ 1 file changed, 41 insertions(+), 40 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index d07e05b19..aca8d8c51 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -10,20 +10,20 @@ concurrency: jobs: # test option values defined at test/conftest.py are passed on via repository secret # INTEGRATION_TEST_ARGS to operator-workflows automatically. - integration-tests: - name: Integration test with juju 3.1 - uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main - secrets: inherit - with: - juju-channel: 3.1/stable - pre-run-script: scripts/pre-integration-test.sh - provider: lxd - test-tox-env: integration-juju3.1 - # These important local LXD test has no OpenStack integration versions. - # test_charm_scheduled_events ensures reconcile events are fired on a schedule. - # test_debug_ssh ensures tmate SSH actions works. - # TODO: Add OpenStack integration versions of these tests. - modules: '["test_charm_scheduled_events", "test_debug_ssh"]' + # integration-tests: + # name: Integration test with juju 3.1 + # uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main + # secrets: inherit + # with: + # juju-channel: 3.1/stable + # pre-run-script: scripts/pre-integration-test.sh + # provider: lxd + # test-tox-env: integration-juju3.1 + # # These important local LXD test has no OpenStack integration versions. + # # test_charm_scheduled_events ensures reconcile events are fired on a schedule. + # # test_debug_ssh ensures tmate SSH actions works. + # # TODO: Add OpenStack integration versions of these tests. + # modules: '["test_charm_scheduled_events", "test_debug_ssh"]' # TODO: debug debug-reactive: name: debug reactive @@ -34,34 +34,35 @@ jobs: pre-run-script: scripts/setup-lxd.sh provider: lxd test-tox-env: integration-juju3.2 + extra-arguments: "-m openstack" modules: '["test_reactive"]' self-hosted-runner: true self-hosted-runner-label: stg-private-endpoint tmate-debug: true tmate-timeout: 300 - openstack-interface-tests-private-endpoint: - name: openstack interface test using private-endpoint - uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main - secrets: inherit - with: - juju-channel: 3.2/stable - pre-run-script: scripts/setup-lxd.sh - provider: lxd - test-tox-env: integration-juju3.2 - modules: '["test_runner_manager_openstack"]' - self-hosted-runner: true - self-hosted-runner-label: stg-private-endpoint - openstack-integration-tests-private-endpoint: - name: Integration test using private-endpoint - uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main - needs: openstack-interface-tests-private-endpoint - secrets: inherit - with: - juju-channel: 3.2/stable - pre-run-script: scripts/setup-lxd.sh - provider: lxd - test-tox-env: integration-juju3.2 - modules: '["test_charm_metrics_failure", "test_charm_metrics_success", "test_charm_fork_repo", "test_charm_runner", "test_reactive"]' - extra-arguments: "-m openstack" - self-hosted-runner: true - self-hosted-runner-label: stg-private-endpoint + # openstack-interface-tests-private-endpoint: + # name: openstack interface test using private-endpoint + # uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main + # secrets: inherit + # with: + # juju-channel: 3.2/stable + # pre-run-script: scripts/setup-lxd.sh + # provider: lxd + # test-tox-env: integration-juju3.2 + # modules: '["test_runner_manager_openstack"]' + # self-hosted-runner: true + # self-hosted-runner-label: stg-private-endpoint + # openstack-integration-tests-private-endpoint: + # name: Integration test using private-endpoint + # uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main + # needs: openstack-interface-tests-private-endpoint + # secrets: inherit + # with: + # juju-channel: 3.2/stable + # pre-run-script: scripts/setup-lxd.sh + # provider: lxd + # test-tox-env: integration-juju3.2 + # modules: '["test_charm_metrics_failure", "test_charm_metrics_success", "test_charm_fork_repo", "test_charm_runner", "test_reactive"]' + # extra-arguments: "-m openstack" + # self-hosted-runner: true + # self-hosted-runner-label: stg-private-endpoint From 8b2ce60b0cfa7bc79b459db1541d534a70a67445 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 30 Sep 2024 15:17:52 +0800 Subject: [PATCH 19/34] Try if region_name --- src/charm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/charm.py b/src/charm.py index 71f9decdf..561532893 100755 --- a/src/charm.py +++ b/src/charm.py @@ -1346,7 +1346,7 @@ def _create_openstack_runner_manager_config( password=first_cloud_config["auth"]["password"], user_domain_name=first_cloud_config["auth"]["user_domain_name"], project_domain_name=first_cloud_config["auth"]["project_domain_name"], - region_name=first_cloud_config.get("region_name", None), + region_name=first_cloud_config["region_name"], ) server_config = None image = state.runner_config.openstack_image From 8dde4fe398a0ec5cd0784995dd2dce221a6c2e69 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 30 Sep 2024 20:56:20 +0800 Subject: [PATCH 20/34] Print deubgging info --- tests/integration/test_runner_manager_openstack.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_runner_manager_openstack.py b/tests/integration/test_runner_manager_openstack.py index cdf394165..df5f11056 100644 --- a/tests/integration/test_runner_manager_openstack.py +++ b/tests/integration/test_runner_manager_openstack.py @@ -118,6 +118,9 @@ async def openstack_runner_manager_fixture( try: # Pick the first cloud in the clouds.yaml cloud = tuple(clouds_config["clouds"].values())[0] + print('============================================') + print(cloud) + print('============================================') credentials = OpenStackCredentials( auth_url=cloud["auth"]["auth_url"], @@ -126,7 +129,7 @@ async def openstack_runner_manager_fixture( password=cloud["auth"]["password"], user_domain_name=cloud["auth"]["user_domain_name"], project_domain_name=cloud["auth"]["project_domain_name"], - region_name=cloud["region_name"], + region_name=cloud.get(["region_name"], None), ) except KeyError as err: raise AssertionError("Issue with the format of the clouds.yaml used in test") from err From 4fef8557e906f573142d2c20bcbfbfbb9a1b6e31 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Wed, 16 Oct 2024 12:51:51 +0200 Subject: [PATCH 21/34] add region_name to _OpenStackAuth --- src-docs/charm_state.py.md | 28 ++++++++++++++-------------- src/charm_state.py | 2 ++ 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src-docs/charm_state.py.md b/src-docs/charm_state.py.md index 98647b26d..64ff18954 100644 --- a/src-docs/charm_state.py.md +++ b/src-docs/charm_state.py.md @@ -110,7 +110,7 @@ Some charm configurations are grouped into other configuration models. --- - + ### classmethod `check_reconcile_interval` @@ -139,7 +139,7 @@ Validate the general charm configuration. --- - + ### classmethod `from_charm` @@ -221,7 +221,7 @@ The charm state. --- - + ### classmethod `from_charm` @@ -300,7 +300,7 @@ Get github related charm configuration values from charm. ## class `ImmutableConfigChangedError` Represents an error when changing immutable charm state. - + ### function `__init__` @@ -355,7 +355,7 @@ Runner configurations for local LXD instances. --- - + ### classmethod `check_virtual_machine_resources` @@ -386,7 +386,7 @@ Validate the virtual_machine_resources field values. --- - + ### classmethod `check_virtual_machines` @@ -415,7 +415,7 @@ Validate the virtual machines configuration value. --- - + ### classmethod `from_charm` @@ -475,7 +475,7 @@ OpenstackImage from image builder relation data. --- - + ### classmethod `from_charm` @@ -518,7 +518,7 @@ Runner configuration for OpenStack Instances. --- - + ### classmethod `from_charm` @@ -572,7 +572,7 @@ Return the aproxy address. --- - + ### classmethod `check_use_aproxy` @@ -602,7 +602,7 @@ Validate the proxy configuration. --- - + ### classmethod `from_charm` @@ -640,7 +640,7 @@ Represents the configuration for reactive scheduling. --- - + ### classmethod `from_database` @@ -748,7 +748,7 @@ SSH connection information for debug workflow. --- - + ### classmethod `from_charm` @@ -781,7 +781,7 @@ Raised when given machine charm architecture is unsupported. - `arch`: The current machine architecture. - + ### function `__init__` diff --git a/src/charm_state.py b/src/charm_state.py index 2976c8b8a..ba1c242ca 100644 --- a/src/charm_state.py +++ b/src/charm_state.py @@ -296,6 +296,7 @@ class _OpenStackAuth(TypedDict): project_name: The OpenStack project to connect to. user_domain_name: The user domain in which the user belongs to. username: The user to authenticate as. + region_name: The OpenStack region to authenticate to. """ auth_url: str @@ -304,6 +305,7 @@ class _OpenStackAuth(TypedDict): project_name: str user_domain_name: str username: str + region_name: str class _OpenStackCloud(TypedDict): From 949cdfb5b7ba3e000672ae9b3140c78c0505113e Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Wed, 16 Oct 2024 13:31:31 +0200 Subject: [PATCH 22/34] add region_name to _OpenStackCloud --- src/charm_state.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/charm_state.py b/src/charm_state.py index ba1c242ca..ba4e8731b 100644 --- a/src/charm_state.py +++ b/src/charm_state.py @@ -296,7 +296,6 @@ class _OpenStackAuth(TypedDict): project_name: The OpenStack project to connect to. user_domain_name: The user domain in which the user belongs to. username: The user to authenticate as. - region_name: The OpenStack region to authenticate to. """ auth_url: str @@ -305,7 +304,6 @@ class _OpenStackAuth(TypedDict): project_name: str user_domain_name: str username: str - region_name: str class _OpenStackCloud(TypedDict): @@ -315,9 +313,11 @@ class _OpenStackCloud(TypedDict): Attributes: auth: The connection authentication info. + region_name: The OpenStack region to authenticate to. """ auth: _OpenStackAuth + region_name: str class OpenStackCloudsYAML(TypedDict): From 852aacad97685c9c6302bc063b147ab9f09d3831 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Wed, 16 Oct 2024 20:54:21 +0200 Subject: [PATCH 23/34] run other integration tests --- .github/workflows/integration_test.yaml | 46 +++++++++---------------- 1 file changed, 16 insertions(+), 30 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index ed9595267..ed8986ff6 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -24,9 +24,8 @@ jobs: # # test_debug_ssh ensures tmate SSH actions works. # # The test test_charm_upgrade needs to run to ensure the charm can be upgraded. # modules: '["test_charm_scheduled_events", "test_debug_ssh", "test_charm_upgrade"]' - # TODO: debug - debug-reactive: - name: debug reactive + openstack-interface-tests-private-endpoint: + name: openstack interface test using private-endpoint uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main secrets: inherit with: @@ -34,35 +33,22 @@ jobs: pre-run-script: scripts/setup-lxd.sh provider: lxd test-tox-env: integration-juju3.2 + modules: '["test_runner_manager_openstack"]' + self-hosted-runner: true + self-hosted-runner-label: stg-private-endpoint + openstack-integration-tests-private-endpoint: + name: Integration test using private-endpoint + uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main + needs: openstack-interface-tests-private-endpoint + secrets: inherit + with: + juju-channel: 3.2/stable + pre-run-script: scripts/setup-lxd.sh + provider: lxd + test-tox-env: integration-juju3.2 + modules: '["test_charm_metrics_failure", "test_charm_metrics_success", "test_charm_fork_repo", "test_charm_runner"]' extra-arguments: "-m openstack" - modules: '["test_reactive"]' self-hosted-runner: true self-hosted-runner-label: stg-private-endpoint tmate-debug: true tmate-timeout: 300 - # openstack-interface-tests-private-endpoint: - # name: openstack interface test using private-endpoint - # uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main - # secrets: inherit - # with: - # juju-channel: 3.2/stable - # pre-run-script: scripts/setup-lxd.sh - # provider: lxd - # test-tox-env: integration-juju3.2 - # modules: '["test_runner_manager_openstack"]' - # self-hosted-runner: true - # self-hosted-runner-label: stg-private-endpoint - # openstack-integration-tests-private-endpoint: - # name: Integration test using private-endpoint - # uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main - # needs: openstack-interface-tests-private-endpoint - # secrets: inherit - # with: - # juju-channel: 3.2/stable - # pre-run-script: scripts/setup-lxd.sh - # provider: lxd - # test-tox-env: integration-juju3.2 - # modules: '["test_charm_metrics_failure", "test_charm_metrics_success", "test_charm_fork_repo", "test_charm_runner", "test_reactive"]' - # extra-arguments: "-m openstack" - # self-hosted-runner: true - # self-hosted-runner-label: stg-private-endpoint From 76ba870dc66855f6c41fa0c7b4fcb861dfc0df59 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Wed, 16 Oct 2024 21:23:51 +0200 Subject: [PATCH 24/34] fix test_runner_manager_openstack --- .github/workflows/integration_test.yaml | 2 +- tests/integration/test_runner_manager_openstack.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index ed8986ff6..1de4ef856 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -39,7 +39,7 @@ jobs: openstack-integration-tests-private-endpoint: name: Integration test using private-endpoint uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main - needs: openstack-interface-tests-private-endpoint +# needs: openstack-interface-tests-private-endpoint secrets: inherit with: juju-channel: 3.2/stable diff --git a/tests/integration/test_runner_manager_openstack.py b/tests/integration/test_runner_manager_openstack.py index 0f176ddfa..144ec4324 100644 --- a/tests/integration/test_runner_manager_openstack.py +++ b/tests/integration/test_runner_manager_openstack.py @@ -31,7 +31,6 @@ ) from github_runner_manager.metrics import events from github_runner_manager.openstack_cloud import health_checks -from github_runner_manager.openstack_cloud.openstack_cloud import _CLOUDS_YAML_PATH from github_runner_manager.openstack_cloud.openstack_runner_manager import ( OpenStackCredentials, OpenStackRunnerManager, @@ -113,7 +112,6 @@ async def openstack_runner_manager_fixture( The prefix args of OpenstackRunnerManager set to app_name to let openstack_connection_fixture perform the cleanup of openstack resources. """ - _CLOUDS_YAML_PATH.unlink(missing_ok=True) clouds_config = yaml.safe_load(private_endpoint_clouds_yaml) try: @@ -130,7 +128,7 @@ async def openstack_runner_manager_fixture( password=cloud["auth"]["password"], user_domain_name=cloud["auth"]["user_domain_name"], project_domain_name=cloud["auth"]["project_domain_name"], - region_name=cloud.get(["region_name"], None), + region_name=cloud["region_name"], ) except KeyError as err: raise AssertionError("Issue with the format of the clouds.yaml used in test") from err From dce6256ad17b59b9402de1d5633263d1756aa5ef Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Wed, 16 Oct 2024 22:00:39 +0200 Subject: [PATCH 25/34] fix unit tests --- tests/integration/test_runner_manager_openstack.py | 4 ++-- tests/unit/test_charm.py | 3 ++- tests/unit/test_charm_state.py | 7 ++++--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_runner_manager_openstack.py b/tests/integration/test_runner_manager_openstack.py index 144ec4324..5bdecc9e8 100644 --- a/tests/integration/test_runner_manager_openstack.py +++ b/tests/integration/test_runner_manager_openstack.py @@ -117,9 +117,9 @@ async def openstack_runner_manager_fixture( try: # Pick the first cloud in the clouds.yaml cloud = tuple(clouds_config["clouds"].values())[0] - print('============================================') + print("============================================") print(cloud) - print('============================================') + print("============================================") credentials = OpenStackCredentials( auth_url=cloud["auth"]["auth_url"], diff --git a/tests/unit/test_charm.py b/tests/unit/test_charm.py index a28fc9743..f2451686f 100644 --- a/tests/unit/test_charm.py +++ b/tests/unit/test_charm.py @@ -681,7 +681,8 @@ def test_on_config_changed_openstack_clouds_yaml(self, run, wt, mkdir, orm, rm): "username": secrets.token_hex(16), "user_domain_name": secrets.token_hex(16), "password": secrets.token_hex(16), - } + }, + "region_name": secrets.token_hex(16), } } } diff --git a/tests/unit/test_charm_state.py b/tests/unit/test_charm_state.py index 0059538ef..f3d3564f7 100644 --- a/tests/unit/test_charm_state.py +++ b/tests/unit/test_charm_state.py @@ -296,7 +296,7 @@ def valid_yaml_config(): auth_url: 'http://keystone.openstack.svc.cluster.local:5000/v3' user_domain_name: 'Default' project_domain_name: 'Default' - region_name: 'RegionOne' + region_name: 'RegionOne' """ @@ -313,7 +313,7 @@ def invalid_yaml_config(): auth_url: 'http://keystone.openstack.svc.cluster.local:5000/v3' user_domain_name: 'Default' project_domain_name: 'Default' - region_name: 'RegionOne' + region_name: 'RegionOne' """ @@ -467,7 +467,8 @@ def test_charm_config_from_charm_valid(): "project_name": "test-project-name", "user_domain_name": "Default", "username": "test-user-name", - } + }, + "region_name": secrets.token_hex(16), } } } From f64b7a77a994848f9eef078ce7d010ee43145573 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Wed, 16 Oct 2024 23:30:34 +0200 Subject: [PATCH 26/34] revert integration_test.yaml --- .github/workflows/integration_test.yaml | 34 ++++++++++++------------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 1de4ef856..eb141a091 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -10,20 +10,20 @@ concurrency: jobs: # test option values defined at test/conftest.py are passed on via repository secret # INTEGRATION_TEST_ARGS to operator-workflows automatically. -# integration-tests: -# name: Integration test with juju 3.1 -# uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main -# secrets: inherit -# with: -# juju-channel: 3.1/stable -# pre-run-script: scripts/pre-integration-test.sh -# provider: lxd -# test-tox-env: integration-juju3.1 -# # These important local LXD test have no OpenStack integration versions. -# # test_charm_scheduled_events ensures reconcile events are fired on a schedule. -# # test_debug_ssh ensures tmate SSH actions works. -# # The test test_charm_upgrade needs to run to ensure the charm can be upgraded. -# modules: '["test_charm_scheduled_events", "test_debug_ssh", "test_charm_upgrade"]' + integration-tests: + name: Integration test with juju 3.1 + uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main + secrets: inherit + with: + juju-channel: 3.1/stable + pre-run-script: scripts/pre-integration-test.sh + provider: lxd + test-tox-env: integration-juju3.1 + # These important local LXD test have no OpenStack integration versions. + # test_charm_scheduled_events ensures reconcile events are fired on a schedule. + # test_debug_ssh ensures tmate SSH actions works. + # The test test_charm_upgrade needs to run to ensure the charm can be upgraded. + modules: '["test_charm_scheduled_events", "test_debug_ssh", "test_charm_upgrade"]' openstack-interface-tests-private-endpoint: name: openstack interface test using private-endpoint uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main @@ -39,16 +39,14 @@ jobs: openstack-integration-tests-private-endpoint: name: Integration test using private-endpoint uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main -# needs: openstack-interface-tests-private-endpoint + needs: openstack-interface-tests-private-endpoint secrets: inherit with: juju-channel: 3.2/stable pre-run-script: scripts/setup-lxd.sh provider: lxd test-tox-env: integration-juju3.2 - modules: '["test_charm_metrics_failure", "test_charm_metrics_success", "test_charm_fork_repo", "test_charm_runner"]' + modules: '["test_charm_metrics_failure", "test_charm_metrics_success", "test_charm_fork_repo", "test_charm_runner", "test_reactive"]' extra-arguments: "-m openstack" self-hosted-runner: true self-hosted-runner-label: stg-private-endpoint - tmate-debug: true - tmate-timeout: 300 From 742b650518035fbbe3af7990771775bbaa8fe60b Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Wed, 16 Oct 2024 23:39:24 +0200 Subject: [PATCH 27/34] fix unit test --- tests/unit/conftest.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index c0b760144..b1561cc03 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -101,19 +101,6 @@ def cloud_name_fixture() -> str: """The testing cloud name.""" return "microstack" - -@pytest.fixture(autouse=True, name="clouds_yaml_path") -def clouds_yaml_path(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path: - """Mocked clouds.yaml path. - - Returns: - Path: Mocked clouds.yaml path. - """ - clouds_yaml_path = tmp_path / "clouds.yaml" - monkeypatch.setattr("github_runner_manager.openstack_cloud.CLOUDS_YAML_PATH", clouds_yaml_path) - return clouds_yaml_path - - @pytest.fixture(name="clouds_yaml") def clouds_yaml_fixture(cloud_name: str) -> dict: """Testing clouds.yaml.""" From 229442f748346f4dc4c21b0933519b6b46924a9c Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Wed, 16 Oct 2024 23:44:07 +0200 Subject: [PATCH 28/34] lint --- tests/unit/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index b1561cc03..5a9182080 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -101,6 +101,7 @@ def cloud_name_fixture() -> str: """The testing cloud name.""" return "microstack" + @pytest.fixture(name="clouds_yaml") def clouds_yaml_fixture(cloud_name: str) -> dict: """Testing clouds.yaml.""" From 206c3ec6df2dbec6d7f597640f54aa60a3f035e6 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Thu, 17 Oct 2024 07:13:25 +0200 Subject: [PATCH 29/34] update changelog --- docs/changelog.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/changelog.md b/docs/changelog.md index acf0fc20b..aef5d9642 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,4 +1,12 @@ # Changelog + +### 2024-10-17 + +- Use in-memory authentication instead of clouds.yaml on disk for OpenStack. This prevents +the multi-processing fighting over the file handle for the clouds.yaml file in the github-runner-manager. + +- Fixed a bug where metrics storage for unmatched runners could not get cleaned up. + ### 2024-10-11 - Added support for COS integration with reactive runners. From 3d6cf9edb1adae96bd5cab9293eb62b7ffdd00d8 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Thu, 17 Oct 2024 11:06:57 +0200 Subject: [PATCH 30/34] increase image-builder deploy timeout --- tests/integration/conftest.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 1d52fd1ea..1fb03a35c 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -48,6 +48,8 @@ from tests.integration.helpers.openstack import OpenStackInstanceHelper, PrivateEndpointConfigs from tests.status_name import ACTIVE +IMAGE_BUILDER_DEPLOY_TIMEOUT_IN_SECONDS = 30 * 60 + # The following line is required because we are using request.getfixturevalue in conjunction # with pytest-asyncio. See https://github.com/pytest-dev/pytest-asyncio/issues/112 nest_asyncio.apply() @@ -397,7 +399,7 @@ async def image_builder_fixture( "openstack-user-name": private_endpoint_config["username"], }, ) - await model.wait_for_idle(apps=[app.name], wait_for_active=True, timeout=15 * 60) + await model.wait_for_idle(apps=[app.name], wait_for_active=True, timeout=IMAGE_BUILDER_DEPLOY_TIMEOUT_IN_SECONDS) else: app = model.applications["github-runner-image-builder"] return app From 7463e4fae564c3abe4118e8422b3c8ba4244a490 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Thu, 17 Oct 2024 11:38:39 +0200 Subject: [PATCH 31/34] pin correct commit --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b8acd8a80..d13a9f4d5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,4 +12,4 @@ cosl ==0.0.15 # juju 3.1.2.0 depends on pyyaml<=6.0 and >=5.1.2 PyYAML ==6.0.* pyOpenSSL==24.2.1 -github_runner_manager @ git+https://github.com/canonical/github-runner-manager.git@fix/openstack-connection-creation \ No newline at end of file +github_runner_manager @ git+https://github.com/canonical/github-runner-manager.git@6b136f4e915c7dfec22e801981bcc0a6af581df1 \ No newline at end of file From 4d6a2863afc0f665db757391278de70e245e6a2c Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Thu, 17 Oct 2024 11:40:38 +0200 Subject: [PATCH 32/34] final new line --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d13a9f4d5..083c189a5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,4 +12,4 @@ cosl ==0.0.15 # juju 3.1.2.0 depends on pyyaml<=6.0 and >=5.1.2 PyYAML ==6.0.* pyOpenSSL==24.2.1 -github_runner_manager @ git+https://github.com/canonical/github-runner-manager.git@6b136f4e915c7dfec22e801981bcc0a6af581df1 \ No newline at end of file +github_runner_manager @ git+https://github.com/canonical/github-runner-manager.git@6b136f4e915c7dfec22e801981bcc0a6af581df1 From 1c3aa06103b1b393585b8ee117cb4a8cce082e01 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Thu, 17 Oct 2024 11:42:19 +0200 Subject: [PATCH 33/34] remove todo --- src/charm.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/charm.py b/src/charm.py index 724d57981..f3c6858a3 100755 --- a/src/charm.py +++ b/src/charm.py @@ -1346,10 +1346,6 @@ def _create_openstack_runner_manager_config( logger.warning( "Multiple clouds defined in clouds.yaml. Using the first one to connect." ) - # TODO: The fields will be replace with charm configuration, the clouds.yaml configuration - # will be removed. - # The clouds and the auth are assumed to be there as the `_parse_openstack_clouds_config` - # has use pydantic to verify the fields. first_cloud_config = state.charm_config.openstack_clouds_yaml["clouds"][clouds[0]] credentials = OpenStackCredentials( auth_url=first_cloud_config["auth"]["auth_url"], From 459e754e46779ff07814adcebbeaacab57606368 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Thu, 17 Oct 2024 11:46:59 +0200 Subject: [PATCH 34/34] lint --- tests/integration/conftest.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 1fb03a35c..da3359250 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -399,7 +399,9 @@ async def image_builder_fixture( "openstack-user-name": private_endpoint_config["username"], }, ) - await model.wait_for_idle(apps=[app.name], wait_for_active=True, timeout=IMAGE_BUILDER_DEPLOY_TIMEOUT_IN_SECONDS) + await model.wait_for_idle( + apps=[app.name], wait_for_active=True, timeout=IMAGE_BUILDER_DEPLOY_TIMEOUT_IN_SECONDS + ) else: app = model.applications["github-runner-image-builder"] return app