test: ssh-debug integration test

.github/workflows/integration_test.yaml

@@ -13,7 +13,7 @@ jobs:
       pre-run-script: scripts/pre-integration-test.sh
       provider: lxd
       test-tox-env: integration-juju2.9
-      modules: '["test_charm_fork_repo", "test_charm_no_runner", "test_charm_scheduled_events", "test_charm_one_runner", "test_charm_metrics_success", "test_charm_metrics_failure", "test_self_hosted_runner", "test_charm_with_proxy", "test_charm_with_juju_storage"]'
+      modules: '["test_charm_fork_repo", "test_charm_no_runner", "test_charm_scheduled_events", "test_charm_one_runner", "test_charm_metrics_success", "test_charm_metrics_failure", "test_self_hosted_runner", "test_charm_with_proxy", "test_charm_with_juju_storage", "test_debug_ssh"]'
   integration-tests-juju3:
     name: Integration test with juju 3.1
     uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main
@@ -23,4 +23,4 @@ jobs:
       pre-run-script: scripts/pre-integration-test.sh
       provider: lxd
       test-tox-env: integration-juju3.1
-      modules: '["test_charm_fork_repo", "test_charm_no_runner", "test_charm_scheduled_events", "test_charm_one_runner", "test_charm_metrics_success", "test_charm_metrics_failure", "test_self_hosted_runner", "test_charm_with_proxy", "test_charm_with_juju_storage"]'
+      modules: '["test_charm_fork_repo", "test_charm_no_runner", "test_charm_scheduled_events", "test_charm_one_runner", "test_charm_metrics_success", "test_charm_metrics_failure", "test_self_hosted_runner", "test_charm_with_proxy", "test_charm_with_juju_storage", "test_debug_ssh"]'

src/charm.py

@@ -932,6 +932,7 @@ def _on_debug_ssh_relation_changed(self, _: ops.RelationChangedEvent) -> None:
         """Handle debug ssh relation changed event."""
         runner_manager = self._get_runner_manager()
         runner_manager.flush(flush_busy=False)
+        self._reconcile_runners(runner_manager)
 
 
 if __name__ == "__main__":

src/charm_state.py

@@ -309,6 +309,9 @@ def from_charm(cls, charm: CharmBase) -> "State":
         # Convert pydantic object to python object serializable by json module.
         state_dict["proxy_config"] = json.loads(state_dict["proxy_config"].json())
         state_dict["charm_config"] = json.loads(state_dict["charm_config"].json())
+        state_dict["ssh_debug_info"] = (
+            json.loads(state_dict["ssh_debug_info"].json()) if ssh_debug_info else None
+        )
         json_data = json.dumps(state_dict, ensure_ascii=False)
         CHARM_STATE_PATH.write_text(json_data, encoding="utf-8")
 

tests/integration/conftest.py

@@ -12,17 +12,20 @@
 import pytest
 import pytest_asyncio
 import yaml
+from git import Repo
 from github import Github, GithubException
 from github.Branch import Branch
 from github.Repository import Repository
 from juju.application import Application
+from juju.client._definitions import FullStatus, UnitStatus
 from juju.model import Model
 from pytest_operator.plugin import OpsTest
 
 from tests.integration.helpers import (
     deploy_github_runner_charm,
     ensure_charm_has_runner,
     reconcile,
+    wait_for,
 )
 from tests.status_name import ACTIVE
 
@@ -246,6 +249,57 @@ async def app_runner(
     return application
 
 
+@pytest_asyncio.fixture(scope="module", name="tmate_ssh_server_app")
+async def tmate_ssh_server_app_fixture(
+    model: Model,
+    charm_file: str,
+    app_name: str,
+    path: str,
+    token: str,
+    http_proxy: str,
+    https_proxy: str,
+    no_proxy: str,
+) -> AsyncIterator[Application]:
+    """tmate-ssh-server charm application related to GitHub-Runner app charm."""
+    # 5 min reconcile period to register flushed runners after relation data changed event.
+    app: Application = await deploy_github_runner_charm(
+        model=model,
+        charm_file=charm_file,
+        app_name=app_name,
+        path=path,
+        token=token,
+        runner_storage="memory",
+        http_proxy=http_proxy,
+        https_proxy=https_proxy,
+        no_proxy=no_proxy,
+        reconcile_interval=60,
+        wait_idle=False,
+    )
+    await app.set_config({"virtual-machines": "1"})
+    tmate_app: Application = await model.deploy("tmate-ssh-server", channel="edge")
+    await app.relate("debug-ssh", f"{tmate_app.name}:debug-ssh")
+    await model.wait_for_idle(status=ACTIVE, timeout=60 * 30)
+
+    return tmate_app
+
+
+@pytest_asyncio.fixture(scope="module", name="tmate_ssh_server_unit_ip")
+async def tmate_ssh_server_unit_ip_fixture(
+    model: Model,
+    tmate_ssh_server_app: Application,
+) -> AsyncIterator[str]:
+    """tmate-ssh-server charm unit ip."""
+    status: FullStatus = await model.get_status([tmate_ssh_server_app.name])
+    try:
+        unit_status: UnitStatus = next(
+            iter(status.applications[tmate_ssh_server_app.name].units.values())
+        )
+        assert unit_status.public_address, "Invalid unit address"
+        return unit_status.public_address
+    except StopIteration as exc:
+        raise StopIteration("Invalid unit status") from exc
+
+
 @pytest.fixture(scope="module")
 def github_client(token: str) -> Github:
     """Returns the github client."""
@@ -357,6 +411,31 @@ async def app_juju_storage(
     return application
 
 
+@pytest_asyncio.fixture(scope="module", name="test_github_branch")
+async def test_github_branch_fixture(github_repository: Repository) -> AsyncIterator[Branch]:
+    """Create a new branch for testing, from latest commit in current branch."""
+    test_branch = f"test-{secrets.token_hex(4)}"
+    branch_ref = github_repository.create_git_ref(
+        ref=f"refs/heads/{test_branch}", sha=Repo().head.commit.hexsha
+    )
+
+    def get_branch():
+        """Get newly created branch."""
+        try:
+            branch = github_repository.get_branch(test_branch)
+        except GithubException as err:
+            if err.status == 404:
+                return False
+            raise
+        return branch
+
+    await wait_for(get_branch)
+
+    yield get_branch()
+
+    branch_ref.delete()
+
+
 @pytest_asyncio.fixture(scope="module", name="app_with_grafana_agent")
 async def app_with_grafana_agent_integrated_fixture(
     model: Model, app_no_runner: Application

tests/integration/helpers.py

@@ -3,10 +3,12 @@
 
 """Utilities for integration test."""
 
+import inspect
 import json
 import subprocess
+import time
 from asyncio import sleep
-from typing import Any
+from typing import Any, Awaitable, Callable, Union
 
 import juju.version
 import yaml
@@ -305,6 +307,7 @@ async def deploy_github_runner_charm(
     https_proxy: str,
     no_proxy: str,
     reconcile_interval: int,
+    wait_idle: bool = True,
 ) -> Application:
     """Deploy github-runner charm.
 
@@ -318,6 +321,7 @@ async def deploy_github_runner_charm(
         https_proxy: HTTPS proxy for the application to use.
         no_proxy: No proxy configuration for the application.
         reconcile_interval: Time between reconcile for the application.
+        wait_idle: wait for model to become idle.
     """
     subprocess.run(["sudo", "modprobe", "br_netfilter"])
 
@@ -351,5 +355,46 @@ async def deploy_github_runner_charm(
         storage=storage,
     )
 
-    await model.wait_for_idle(status=ACTIVE, timeout=60 * 30)
+    if wait_idle:
+        await model.wait_for_idle(status=ACTIVE, timeout=60 * 30)
+
     return application
+
+
+async def wait_for(
+    func: Callable[[], Union[Awaitable, Any]],
+    timeout: int = 300,
+    check_interval: int = 10,
+) -> Any:
+    """Wait for function execution to become truthy.
+
+    Args:
+        func: A callback function to wait to return a truthy value.
+        timeout: Time in seconds to wait for function result to become truthy.
+        check_interval: Time in seconds to wait between ready checks.
+
+    Raises:
+        TimeoutError: if the callback function did not return a truthy value within timeout.
+
+    Returns:
+        The result of the function if any.
+    """
+    deadline = time.time() + timeout
+    is_awaitable = inspect.iscoroutinefunction(func)
+    while time.time() < deadline:
+        if is_awaitable:
+            if result := await func():
+                return result
+        else:
+            if result := func():
+                return result
+        time.sleep(check_interval)
+
+    # final check before raising TimeoutError.
+    if is_awaitable:
+        if result := await func():
+            return result
+    else:
+        if result := func():
+            return result
+    raise TimeoutError()

tests/integration/requirements.txt

@@ -0,0 +1 @@
+GitPython>3,<4

tests/integration/test_debug_ssh.py

@@ -0,0 +1,87 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+"""Integration tests for github-runner charm with ssh-debug integration."""
+import logging
+import typing
+import zipfile
+from io import BytesIO
+
+import requests
+from github.Branch import Branch
+from github.Repository import Repository
+from github.Workflow import Workflow
+from github.WorkflowRun import WorkflowRun
+
+from tests.integration.helpers import wait_for
+
+logger = logging.getLogger(__name__)
+
+
+async def test_ssh_debug(
+    github_repository: Repository,
+    test_github_branch: Branch,
+    app_name: str,
+    token: str,
+    tmate_ssh_server_unit_ip: str,
+):
+    """
+    arrange: given an integrated GitHub-Runner charm and tmate-ssh-server charm.
+    act: when canonical/action-tmate is triggered.
+    assert: the ssh connection info from action-log and tmate-ssh-server matches.
+    """
+    # trigger tmate action
+    logger.info("Dispatching workflow_dispatch_ssh_debug.yaml workflow.")
+    workflow: Workflow = github_repository.get_workflow("workflow_dispatch_ssh_debug.yaml")
+    assert workflow.create_dispatch(
+        test_github_branch, inputs={"runner": app_name}
+    ), "Failed to dispatch workflow"
+
+    # get action logs
+    def latest_workflow_run() -> typing.Optional[WorkflowRun]:
+        """Get latest workflow run."""
+        try:
+            logger.info("Fetching latest workflow run on branch %s.", test_github_branch.name)
+            # The test branch is unique per test, hence there can only be one run per branch.
+            last_run: WorkflowRun = workflow.get_runs(branch=test_github_branch)[0]
+        except IndexError:
+            return None
+        return last_run
+
+    await wait_for(latest_workflow_run, timeout=60 * 10, check_interval=60)
+    lastest_run = typing.cast(WorkflowRun, latest_workflow_run())
+
+    def is_workflow_complete():
+        """Return if the workflow is complete."""
+        lastest_run.update()
+        logger.info("Fetched latest workflow status %s.", lastest_run.status)
+        return lastest_run.status == "completed"
+
+    await wait_for(is_workflow_complete, timeout=60 * 45, check_interval=60)
+
+    response = requests.get(
+        lastest_run.logs_url,
+        headers={
+            "Accept": "application/vnd.github+json",
+            "Authorization": f"Bearer {token}",
+            "X-GitHub-Api-Version": "2022-11-28",
+        },
+    )
+    zip_data = BytesIO(response.content)
+    with zipfile.ZipFile(zip_data, "r") as zip_ref:
+        logger.info("Files: %s", zip_ref.namelist())
+        tmate_log_filename = next(
+            iter(
+                [
+                    name
+                    for name in zip_ref.namelist()
+                    if "workflow-dispatch-tests/3_Setup tmate session.txt" == name
+                ]
+            )
+        )
+        logs = str(zip_ref.read(tmate_log_filename), encoding="utf-8")
+
+    # ensure ssh connection info printed in logs.
+    logger.info("Logs: %s", logs)
+    assert tmate_ssh_server_unit_ip in logs, "Tmate ssh server IP not found in action logs."
+    assert "10022" in logs, "Tmate ssh server connection port not found in action logs."

tox.ini

@@ -114,6 +114,7 @@ deps =
     # Type error problem with newer version of macaroonbakery
     macaroonbakery==1.3.2
     -r{toxinidir}/requirements.txt
+    -r{[vars]tst_path}integration/requirements.txt
 commands =
     pytest -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs}