Merge 6b389e6 into ce408e1

canonical · Sep 17, 2024 · 23a00e1 · 23a00e1
2 parents ce408e1 + 6b389e6
commit 23a00e1
Show file tree

Hide file tree

Showing 9 changed files with 106 additions and 155 deletions.
diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml
@@ -8,45 +8,50 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+
+# TODO: disabled tests to reduce resource usage and speedup testing, re-enable before merging
+
   # test option values defined at test/conftest.py are passed on via repository secret
   # INTEGRATION_TEST_ARGS to operator-workflows automatically.
-  integration-tests:
-    name: Integration test with juju 3.1
-    uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main
-    secrets: inherit
-    with:
-      juju-channel: 3.1/stable
-      pre-run-script: scripts/pre-integration-test.sh
-      provider: lxd
-      test-tox-env: integration-juju3.1
-      # These important local LXD test has no OpenStack integration versions.
-      # test_charm_scheduled_events ensures reconcile events are fired on a schedule.
-      # test_debug_ssh ensures tmate SSH actions works.
-      # TODO: Add OpenStack integration versions of these tests.
-      modules: '["test_charm_scheduled_events", "test_debug_ssh"]'
-  openstack-interface-tests-private-endpoint:
-    name: openstack interface test using private-endpoint
-    uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main
-    secrets: inherit
-    with:
-      juju-channel: 3.2/stable
-      pre-run-script: scripts/setup-lxd.sh
-      provider: lxd
-      test-tox-env: integration-juju3.2
-      modules: '["test_runner_manager_openstack"]'
-      self-hosted-runner: true
-      self-hosted-runner-label: stg-private-endpoint
+#  integration-tests:
+#    name: Integration test with juju 3.1
+#    uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main
+#    secrets: inherit
+#    with:
+#      juju-channel: 3.1/stable
+#      pre-run-script: scripts/pre-integration-test.sh
+#      provider: lxd
+#      test-tox-env: integration-juju3.1
+#      # These important local LXD test has no OpenStack integration versions.
+#      # test_charm_scheduled_events ensures reconcile events are fired on a schedule.
+#      # test_debug_ssh ensures tmate SSH actions works.
+#      # TODO: Add OpenStack integration versions of these tests.
+#      modules: '["test_charm_scheduled_events", "test_debug_ssh"]'
+#  openstack-interface-tests-private-endpoint:
+#    name: openstack interface test using private-endpoint
+#    uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main
+#    secrets: inherit
+#    with:
+#      juju-channel: 3.2/stable
+#      pre-run-script: scripts/setup-lxd.sh
+#      provider: lxd
+#      test-tox-env: integration-juju3.2
+#      modules: '["test_runner_manager_openstack"]'
+#      self-hosted-runner: true
+#      self-hosted-runner-label: stg-private-endpoint
   openstack-integration-tests-private-endpoint:
     name: Integration test using private-endpoint
     uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main
-    needs: openstack-interface-tests-private-endpoint
+#    needs: openstack-interface-tests-private-endpoint
     secrets: inherit
     with:
       juju-channel: 3.2/stable
       pre-run-script: scripts/setup-lxd.sh
       provider: lxd
       test-tox-env: integration-juju3.2
-      modules: '["test_charm_metrics_failure", "test_charm_metrics_success", "test_charm_fork_repo", "test_charm_runner", "test_reactive"]'
+      modules: '["test_reactive"]'
       extra-arguments: "-m openstack"
       self-hosted-runner: true
       self-hosted-runner-label: stg-private-endpoint
+      tmate-debug: true
+      tmate-timeout: 60
diff --git a/requirements.txt b/requirements.txt
@@ -12,6 +12,5 @@ cosl ==0.0.15
 # juju 3.1.2.0 depends on pyyaml<=6.0 and >=5.1.2
 PyYAML ==6.0.*
 pyOpenSSL==24.2.1
-kombu==5.4.1
 pymongo==4.8.0
-github_runner_manager @ git+https://github.com/canonical/github-runner-manager.git@6ee40e296f92f191dd12b1e24a613dfdbdc70f99
+github_runner_manager @ git+https://github.com/canonical/github-runner-manager.git@feat/reactive-spawning
diff --git a/src-docs/charm.md b/src-docs/charm.md
@@ -21,7 +21,7 @@ Charm for creating and managing GitHub self-hosted runner instances.
 
 ---
 
-<a href="../src/charm.py#L117"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
+<a href="../src/charm.py#L121"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
 
 ## <kbd>function</kbd> `catch_charm_errors`
 
@@ -47,7 +47,7 @@ Catch common errors in charm.
 
 ---
 
-<a href="../src/charm.py#L158"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
+<a href="../src/charm.py#L162"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
 
 ## <kbd>function</kbd> `catch_action_errors`
 
@@ -73,7 +73,7 @@ Catch common errors in actions.
 
 ---
 
-<a href="../src/charm.py#L110"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
+<a href="../src/charm.py#L114"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
 
 ## <kbd>class</kbd> `ReconcileRunnersEvent`
 Event representing a periodic check to ensure runners are ok. 
@@ -84,7 +84,7 @@ Event representing a periodic check to ensure runners are ok.
 
 ---
 
-<a href="../src/charm.py#L196"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
+<a href="../src/charm.py#L200"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
 
 ## <kbd>class</kbd> `GithubRunnerCharm`
 Charm for managing GitHub self-hosted runners. 
@@ -101,7 +101,7 @@ Charm for managing GitHub self-hosted runners.
  - <b>`ram_pool_path`</b>:  The path to memdisk storage. 
  - <b>`kernel_module_path`</b>:  The path to kernel modules. 
 
-<a href="../src/charm.py#L219"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
+<a href="../src/charm.py#L223"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
 
 ### <kbd>method</kbd> `__init__`
 

diff --git a/src-docs/runner_manager_type.md b/src-docs/runner_manager_type.md
@@ -121,54 +121,7 @@ Whether metrics for the runners should be collected.
 
 ---
 
-<a href="../src/runner_manager_type.py#L97"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
-
-## <kbd>class</kbd> `OpenstackRunnerManagerConfig`
-Configuration of runner manager. 
-
-
-
-**Attributes:**
-
- - <b>`charm_state`</b>:  The state of the charm. 
- - <b>`path`</b>:  GitHub repository path in the format '<owner>/<repo>', or the  GitHub organization name. 
- - <b>`labels`</b>:  Additional labels for the runners. 
- - <b>`token`</b>:  GitHub personal access token to register runner to the  repository or organization. 
- - <b>`flavor`</b>:  OpenStack flavor for defining the runner resources. 
- - <b>`image`</b>:  Openstack image id to boot the runner with. 
- - <b>`network`</b>:  OpenStack network for runner network access. 
- - <b>`dockerhub_mirror`</b>:  URL of dockerhub mirror to use. 
- - <b>`reactive_config`</b>:  The configuration to spawn runners reactively. 
-
-<a href="../<string>"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
-
-### <kbd>method</kbd> `__init__`
-
-```python
-__init__(
-    charm_state: CharmState,
-    path: GitHubOrg | GitHubRepo,
-    labels: Iterable[str],
-    token: str,
-    flavor: str,
-    image: str,
-    network: str,
-    dockerhub_mirror: str | None,
-    reactive_config: ReactiveConfig | None = None
-) → None
-```
-
-
-
-
-
-
-
-
-
----
-
-<a href="../src/runner_manager_type.py#L126"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
+<a href="../src/runner_manager_type.py#L96"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
 
 ## <kbd>class</kbd> `RunnerInfo`
 Information from GitHub of a runner. 

diff --git a/src/charm.py b/src/charm.py
@@ -7,6 +7,7 @@
 # pylint: disable=too-many-lines
 
 """Charm for creating and managing GitHub self-hosted runner instances."""
+from ghapi.actions import github_token
 from github_runner_manager.manager.cloud_runner_manager import (
     GitHubRunnerConfig,
     SupportServiceConfig,
@@ -20,8 +21,11 @@
 from github_runner_manager.openstack_cloud.openstack_runner_manager import (
     OpenStackCloudConfig,
     OpenStackRunnerManager,
+    OpenStackRunnerManagerConfig,
     OpenStackServerConfig,
 )
+from github_runner_manager.reactive.types_ import QueueConfig as ReactiveQueueConfig
+from github_runner_manager.reactive.types_ import RunnerConfig as ReactiveRunnerConfig
 from github_runner_manager.types_.github import GitHubPath, GitHubRunnerStatus, parse_github_path
 
 from utilities import bytes_with_unit_to_kib, execute_command, remove_residual_venv_dirs, retry
@@ -1276,25 +1280,41 @@ def _get_runner_scaler(
             ssh_debug_connections=state.ssh_debug_connections,
             repo_policy_compliance=state.charm_config.repo_policy_compliance,
         )
-        # The prefix is set to f"{application_name}-{unit number}"
-        openstack_runner_manager = OpenStackRunnerManager(
-            manager_name=self.app.name,
+
+        openstack_runner_manager_config = OpenStackRunnerManagerConfig(
+            manager_name=self.app.name,  # TODO: rename to name
+            # The prefix is set to f"{application_name}-{unit number}"
             prefix=self.unit.name.replace("/", "-"),
             cloud_config=cloud_config,
             server_config=server_config,
             runner_config=runner_config,
             service_config=service_config,
         )
+        openstack_runner_manager = OpenStackRunnerManager(
+            config=openstack_runner_manager_config,
+        )
         runner_manager_config = RunnerManagerConfig(
+            name=self.app.name,
             token=token,
             path=path,
         )
         runner_manager = RunnerManager(
-            manager_name=self.app.name,
             cloud_runner_manager=openstack_runner_manager,
             config=runner_manager_config,
         )
-        return RunnerScaler(runner_manager=runner_manager, reactive_config=state.reactive_config)
+        reactive_runner_config = None
+        if reactive_config := state.reactive_config:
+            reactive_runner_config = ReactiveRunnerConfig(
+                queue=ReactiveQueueConfig(
+                    mongodb_uri=reactive_config.mq_uri, queue_name=self.app.name
+                ),  # TODO think if queue_name should be really decided by charm
+                runner_manager=runner_manager_config,
+                cloud_runner_manager=openstack_runner_manager_config,
+                github_token=token
+            )
+        return RunnerScaler(
+            runner_manager=runner_manager, reactive_runner_config=reactive_runner_config
+        )
 
 
 if __name__ == "__main__":

diff --git a/src/runner_manager_type.py b/src/runner_manager_type.py
@@ -93,36 +93,6 @@ def are_metrics_enabled(self) -> bool:
         return self.charm_state.is_metrics_logging_available
 
 
-# This class is subject to refactor.
-@dataclass
-class OpenstackRunnerManagerConfig:  # pylint: disable=too-many-instance-attributes
-    """Configuration of runner manager.
-
-    Attributes:
-        charm_state: The state of the charm.
-        path: GitHub repository path in the format '<owner>/<repo>', or the
-            GitHub organization name.
-        labels: Additional labels for the runners.
-        token: GitHub personal access token to register runner to the
-            repository or organization.
-        flavor: OpenStack flavor for defining the runner resources.
-        image: Openstack image id to boot the runner with.
-        network: OpenStack network for runner network access.
-        dockerhub_mirror: URL of dockerhub mirror to use.
-        reactive_config: The configuration to spawn runners reactively.
-    """
-
-    charm_state: CharmState
-    path: GitHubPath
-    labels: Iterable[str]
-    token: str
-    flavor: str
-    image: str
-    network: str
-    dockerhub_mirror: str | None
-    reactive_config: ReactiveConfig | None = None
-
-
 @dataclass
 class RunnerInfo:
     """Information from GitHub of a runner.

diff --git a/tests/integration/helpers/common.py b/tests/integration/helpers/common.py
@@ -391,7 +391,7 @@ async def dispatch_workflow(
         app: The charm to dispatch the workflow for.
         branch: The branch to dispatch the workflow on.
         github_repository: The github repository to dispatch the workflow on.
-        conclusion: The expected workflow run conclusion.
+        conclusion: The expected workflow run conclusion. This argument is ignored if wait is False.
         workflow_id_or_name: The workflow filename in .github/workflows in main branch to run or
             its id.
         dispatch_input: Workflow input values.
@@ -420,14 +420,25 @@ async def dispatch_workflow(
 
     if not wait:
         return run
-    await wait_for(partial(_is_workflow_run_complete, run=run), timeout=60 * 30, check_interval=60)
-    # The run object is updated by _is_workflow_run_complete function above.
-    assert (
-        run.conclusion == conclusion
-    ), f"Unexpected run conclusion, expected: {conclusion}, got: {run.conclusion}"
+    await wait_for_completion(run=run, conclusion=conclusion)
 
     return run
 
+async def wait_for_completion(run: WorkflowRun, conclusion: str) -> None:
+    """Wait for the workflow run to complete.
+
+    Args:
+        run: The workflow run to wait for.
+        conclusion: The expected conclusion of the run.
+    """
+    await wait_for(
+        partial(_is_workflow_run_complete, run=run),
+        timeout=60 * 30,
+        check_interval=60,
+    )
+    # The run object is updated by _is_workflow_run_complete function above.
+    assert run.conclusion == conclusion, f"Unexpected run conclusion, expected: {conclusion}, got: {run.conclusion}"
+
 
 P = ParamSpec("P")
 R = TypeVar("R")

diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt
@@ -1,2 +1,3 @@
 GitPython>3,<4
 pygithub
+kombu==5.4.*