From 9a2bb7f9a0addd5d5a112560b690d56e8196d93d Mon Sep 17 00:00:00 2001
From: Dominic Robinson <65237317+drobinson-moj@users.noreply.github.com>
Date: Thu, 14 Nov 2024 10:41:46 +0000
Subject: [PATCH] oem: tweak endpoint monitoring (#1112)

* skip monitoring reporting when tomcat servers are recycled

* fix for cert expiry

* Commit changes made by code formatters

---------

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
---
 .../environment_name_hmpps_oem_production.yml      |  1 +
 .../templates/collectd_endpoint_monitoring.sh.j2   | 14 +++++++-------
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/ansible/group_vars/environment_name_hmpps_oem_production.yml b/ansible/group_vars/environment_name_hmpps_oem_production.yml
index f8307f6d8..35e27c44a 100644
--- a/ansible/group_vars/environment_name_hmpps_oem_production.yml
+++ b/ansible/group_vars/environment_name_hmpps_oem_production.yml
@@ -96,6 +96,7 @@ collectd_endpoint_monitoring:
     url: https://c.nomis.az.justice.gov.uk/forms/frmservlet?config=tag
   - metric_dimension: reporting.nomis.az.justice.gov.uk
     url: https://reporting.nomis.az.justice.gov.uk/keepalive.htm
+    time_ranges: "0.0000-1.0200,1.0210-3.0200,3.0210-5.0200,5.0210-7.0000" # web servers refreshed Mon/Wed/Fri at 2am
   - metric_dimension: oasys.az.justice.gov.uk
     url: https://oasys.az.justice.gov.uk/eor/f?p=100
   - metric_dimension: training.oasys.az.justice.gov.uk
diff --git a/ansible/roles/collectd-endpoint-monitoring/templates/collectd_endpoint_monitoring.sh.j2 b/ansible/roles/collectd-endpoint-monitoring/templates/collectd_endpoint_monitoring.sh.j2
index 3fff6e272..3ef87ca79 100644
--- a/ansible/roles/collectd-endpoint-monitoring/templates/collectd_endpoint_monitoring.sh.j2
+++ b/ansible/roles/collectd-endpoint-monitoring/templates/collectd_endpoint_monitoring.sh.j2
@@ -101,16 +101,18 @@ check_endpoint() {
 n=${#ENDPOINTS[@]}
 
 last_error_log_timestamp=()
-last_days_to_expiry=()
-last_expiry_metric_timestamp=()
+last_expiry_metric_timestamp=0
+log_cert_expiry=0
 for ((i=0; i<n; i++)); do
-  last_days_to_expiry[i]=
-  last_expiry_metric_timestamp[i]=0
   last_error_log_timestamp[i]=0
 done
 
 while true; do
   now_epoch_secs=$(date +%s)
+  if [[ $((now_epoch_secs - last_expiry_metric_timestamp)) -gt $CERT_EXPIRY_METRIC_INTERVAL ]]; then
+    log_cert_expiry=1
+    last_expiry_metric_timestamp="$now_epoch_secs"
+  fi
   for ((i=0; i<n; i++)); do
     args=(${ENDPOINTS[$i]})
     timeranges="${args[4]}"
@@ -135,10 +137,8 @@ while true; do
     days_to_expiry=$(grep "^days_to_expiry=" <<< "$output" | cut -d= -f2)
     echo "PUTVAL $HOSTNAME/endpoint_status/exitcode-${args[3]} interval=$INTERVAL N:$exitcode"
     if [[ -n $days_to_expiry ]]; then
-      if [[ ${last_days_to_expiry[i]} != "$days_to_expiry" || $((now_epoch_secs - last_expiry_metric_timestamp[i])) -gt $CERT_EXPIRY_METRIC_INTERVAL ]]; then
+      if [[ $log_cert_expiry -eq 1 ]]; then
         echo "PUTVAL $HOSTNAME/endpoint_cert_expiry/gauge-${args[3]} interval=$INTERVAL N:$days_to_expiry"
-        last_expiry_metric_timestamp[i]="$now_epoch_secs"
-        last_days_to_expiry[i]="$days_to_expiry"
       fi
     fi
     if [[ $exitcode -ne 0 ]]; then