Fix units with stuck reconciliation (#367)

* add timeout command

* Check-in panel with average actual reconciliation interval

* introduce constant

* update dashboard

* rename panel using term "Application"

* update docs

* remove Optional

docs/reference/cos.md

@@ -19,6 +19,8 @@ The "GitHub Self-Hosted Runner Metrics" metrics dashboard presents the following
       - Runner idle duration
       - Charm reconciliation duration
       - Job queue duration - how long a job waits in the queue before a runner picks it up
+  - Max job queue duration by application: Similar to "Job queue duration" panel, but shows maximum durations by charm application.
+  - Average reconciliation interval: Shows the average time between reconciliation events, broken down by charm application.
 - Jobs: Displays certain metrics about the jobs executed by the runners. These metrics can be displayed per repository by specifying a
  regular expression on the `Repository` variable. The following metrics are displayed:
   - Proportion charts: Share of jobs by completion status, job conclusion, application, repo policy check failure http codes and github events over time.

src-docs/charm.md

@@ -15,12 +15,13 @@ Charm for creating and managing GitHub self-hosted runner instances.
 - **RECONCILE_INTERVAL_CONFIG_NAME**
 - **TEST_MODE_CONFIG_NAME**
 - **TOKEN_CONFIG_NAME**
+- **RECONCILIATION_INTERVAL_TIMEOUT_FACTOR**
 - **RECONCILE_RUNNERS_EVENT**
 - **REACTIVE_MQ_DB_NAME**
 
 ---
 
-<a href="../src/charm.py#L113"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
+<a href="../src/charm.py#L117"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
 
 ## <kbd>function</kbd> `catch_charm_errors`
 
@@ -46,7 +47,7 @@ Catch common errors in charm.
 
 ---
 
-<a href="../src/charm.py#L154"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
+<a href="../src/charm.py#L158"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
 
 ## <kbd>function</kbd> `catch_action_errors`
 
@@ -72,7 +73,7 @@ Catch common errors in actions.
 
 ---
 
-<a href="../src/charm.py#L106"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
+<a href="../src/charm.py#L110"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
 
 ## <kbd>class</kbd> `ReconcileRunnersEvent`
 Event representing a periodic check to ensure runners are ok. 
@@ -83,7 +84,7 @@ Event representing a periodic check to ensure runners are ok.
 
 ---
 
-<a href="../src/charm.py#L192"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
+<a href="../src/charm.py#L196"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
 
 ## <kbd>class</kbd> `GithubRunnerCharm`
 Charm for managing GitHub self-hosted runners. 
@@ -100,7 +101,7 @@ Charm for managing GitHub self-hosted runners.
  - <b>`ram_pool_path`</b>:  The path to memdisk storage. 
  - <b>`kernel_module_path`</b>:  The path to kernel modules. 
 
-<a href="../src/charm.py#L215"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
+<a href="../src/charm.py#L219"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
 
 ### <kbd>method</kbd> `__init__`
 

src-docs/event_timer.md

@@ -109,7 +109,7 @@ Construct the timer manager.
 
 ---
 
-<a href="../src/event_timer.py#L151"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
+<a href="../src/event_timer.py#L146"><img align="right" style="float:right;" src="https://img.shields.io/badge/-source-cccccc?style=flat-square"></a>
 
 ### <kbd>method</kbd> `disable_event_timer`
 
@@ -138,11 +138,7 @@ Disable the systemd timer for the given event.
 ### <kbd>method</kbd> `ensure_event_timer`
 
 ```python
-ensure_event_timer(
-    event_name: str,
-    interval: int,
-    timeout: Optional[int] = None
-) → None
+ensure_event_timer(event_name: str, interval: int, timeout: int) → None
 ```
 
 Ensure that a systemd service and timer are registered to dispatch the given event. 

src/charm.py

@@ -94,6 +94,10 @@
 from runner_manager import LXDRunnerManager, LXDRunnerManagerConfig
 from runner_manager_type import LXDFlushMode
 
+# We assume a stuck reconcile event when it takes longer
+# than 10 times a normal interval. Currently, we are only aware of
+# https://bugs.launchpad.net/juju/+bug/2055184 causing a stuck reconcile event.
+RECONCILIATION_INTERVAL_TIMEOUT_FACTOR = 10
 RECONCILE_RUNNERS_EVENT = "reconcile-runners"
 
 # This is currently hardcoded and may be moved to a config option in the future.
@@ -555,7 +559,8 @@ def _set_reconcile_timer(self) -> None:
         self._event_timer.ensure_event_timer(
             event_name="reconcile-runners",
             interval=int(self.config[RECONCILE_INTERVAL_CONFIG_NAME]),
-            timeout=int(self.config[RECONCILE_INTERVAL_CONFIG_NAME]) - 1,
+            timeout=RECONCILIATION_INTERVAL_TIMEOUT_FACTOR
+            * int(self.config[RECONCILE_INTERVAL_CONFIG_NAME]),
         )
 
     def _ensure_reconcile_timer_is_active(self) -> None:

src/event_timer.py

@@ -5,7 +5,7 @@
 import logging
 import subprocess  # nosec B404
 from pathlib import Path
-from typing import Optional, TypedDict
+from typing import TypedDict
 
 import jinja2
 
@@ -107,9 +107,7 @@ def is_active(self, event_name: str) -> bool:
 
         return ret_code == 0
 
-    def ensure_event_timer(
-        self, event_name: str, interval: int, timeout: Optional[int] = None
-    ) -> None:
+    def ensure_event_timer(self, event_name: str, interval: int, timeout: int) -> None:
         """Ensure that a systemd service and timer are registered to dispatch the given event.
 
         The interval is how frequently, in minutes, the event should be dispatched.
@@ -125,10 +123,7 @@ def ensure_event_timer(
         Raises:
             TimerEnableError: Timer cannot be started. Events will be not emitted.
         """
-        if timeout is not None:
-            timeout_in_secs = timeout * 60
-        else:
-            timeout_in_secs = interval * 30
+        timeout_in_secs = timeout * 60
 
         context: EventConfig = {
             "event": event_name,