Debug

.github/workflows/e2e_test.yaml

@@ -1,7 +1,9 @@
 name: End-to-End tests
 
 on:
-  pull_request:
+  # TODO: debug
+  workflow_dispatch:
+  # pull_request:
 
 
 jobs:

.github/workflows/manual_test_env.yaml

@@ -0,0 +1,22 @@
+name: Manual test env
+
+on:
+  pull_request:
+
+jobs:
+  manual-test-env:
+    name: manual-test-env
+    runs-on: ["self-hosted", "stg-private-endpoint", "x64"]
+    steps:
+      - run: sudo apt update -yq
+      - run: sudo apt install pipx -yq
+      - run: pipx ensurepath
+      - run: pipx install tox
+      - uses: actions/checkout@v4
+      - uses: charmed-kubernetes/actions-operator@main
+      - run: sudo snap install charmcraft --classic
+      - run: lxd init --auto
+      - run: charmcraft pack
+      - name: Tmate debugging session (self-hosted)
+        uses: canonical/action-tmate@main
+        timeout-minutes: 300

src/manager/runner_scaler.py

@@ -164,9 +164,8 @@ def reconcile(self, quantity: int) -> int:
             if runner.github_state == GitHubRunnerState.OFFLINE
             and runner.health == HealthState.HEALTHY
         ]
-        unhealthy_runners = [
-            runner for runner in runner_list if runner.health == HealthState.UNHEALTHY or runner.health == HealthState.UNKNOWN
-        ]
+        unhealthy_states = set(HealthState.UNHEALTHY, HealthState.UNKNOWN)
+        unhealthy_runners = [runner for runner in runner_list if runner.health in unhealthy_states]
         logger.info("Found %s busy runners: %s", len(busy_runners), busy_runners)
         logger.info("Found %s idle runners: %s", len(idle_runners), idle_runners)
         logger.info(

tests/integration/helpers/openstack.py

@@ -55,6 +55,11 @@ async def expose_to_instance(
                 break
         assert ip, f"Failed to get IP address for OpenStack server {runner.name}"
 
+        # TODO: debug
+        import pytest
+
+        pytest.set_trace()
+
         ssh_cmd = f'ssh -fNT -R {port}:localhost:{port} -i /home/ubuntu/.ssh/runner-{runner.name}.key -o "StrictHostKeyChecking no" -o "ControlPersist yes" ubuntu@{ip} &'
         exit_code, _, stderr = await run_in_unit(unit, ssh_cmd)
         assert exit_code == 0, f"Error in SSH remote forwarding of port {port}: {stderr}"