Nomis: DSOS-2359: fixngo connected refactor (#398)

* Renamed fixngo connectivity role

* fix

* fix

* Commit changes made by code formatters

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

ansible/group_vars/server_type_nomis_db.yml

@@ -6,12 +6,17 @@ server_type_roles_list:
   - set-ec2-hostname
   - domain-search
   - ansible-script
+  - epel
   - oracle-11g
   - oracle-secure-backup
   - oracle-db-backup
+  - collectd
+  - amazon-cloudwatch-agent
+  - amazon-cloudwatch-agent-collectd
   - collectd-service-metrics
-  #  - oracle-db-restore
-  - fixngo-connected-metric
+  - collectd-connectivity-test
+  - collectd-oracle-db-connected
+  - collectd-textfile-monitoring
   - nomis-misload
 
 # the below vars are defined in multiple groups.  Keep the values the same to avoid unexpected behaviour

ansible/group_vars/server_type_nomis_web.yml

@@ -6,8 +6,12 @@ server_type_roles_list:
   - set-ec2-hostname
   - domain-search
   - ansible-script
+  - epel
   - nomis-weblogic
   - nomis-release-deployment
+  - collectd
+  - amazon-cloudwatch-agent
+  - amazon-cloudwatch-agent-collectd
   - collectd-service-metrics
   - autoscale-group-hooks-state
 

ansible/group_vars/server_type_nomis_xtag.yml

@@ -7,7 +7,11 @@ server_type_roles_list:
   - set-ec2-hostname
   - domain-search
   - ansible-script
+  - epel
   - nomis-xtag-weblogic
+  - collectd
+  - amazon-cloudwatch-agent
+  - amazon-cloudwatch-agent-collectd
   - collectd-service-metrics
   - autoscale-group-hooks-state
 

ansible/roles/collectd-connectivity-test/README.md

@@ -0,0 +1,12 @@
+Check connectivity with a remote host/port using netcat.
+
+Define a `collectd-connectivity-tests` tag on the AWS instance. In format "hostname1:port1 hostname2:port2 ...".
+A collectd_connectivity_test.sh script is spawned to periodically check each hostname:port using netcat.
+
+The hostname:port will be used as a dimension in cloudwatch:
+
+Metric                                type     type_instance
+collectd_connectivity_test_value      exitcode hostname1:port1
+collectd_connectivity_test_value      exitcode hostname2:port2
+
+The metric value is the netcat exitcode, i.e. 0 if connected, non-zero if not.

ansible/roles/collectd-connectivity-test/defaults/main.yml

@@ -0,0 +1,6 @@
+---
+collectd_script_path: /usr/local/bin
+collectd_script_name: collectd_connectivity_test
+collectd_script_user: ec2-user
+collectd_script_interval: 60
+connectivity_test_connect_timeout: 2

ansible/roles/collectd-connectivity-test/handlers/main.yml

@@ -0,0 +1,10 @@
+---
+- name: restart collectd
+  ansible.builtin.service:
+    name: collectd
+    state: restarted
+
+- name: restart plugin script
+  ansible.builtin.shell: |
+    pkill -u {{ collectd_script_user }} -f {{ collectd_script_path }}/{{ collectd_script_name }}.sh
+  failed_when: false

ansible/roles/collectd-connectivity-test/meta/main.yml

@@ -0,0 +1,4 @@
+---
+dependencies:
+  - role: get-ec2-facts
+  - role: amazon-cloudwatch-agent-collectd

ansible/roles/collectd-connectivity-test/tasks/configure_collectd.yml

@@ -0,0 +1,18 @@
+---
+- name: copy collectd config
+  ansible.builtin.template:
+    src: "{{ collectd_script_name }}.conf.j2"
+    dest: "/etc/collectd.d/{{ collectd_script_name }}.conf"
+    owner: root
+    mode: 0644
+  notify:
+    - restart collectd
+
+- name: copy collectd plugin script
+  ansible.builtin.template:
+    src: "{{ collectd_script_name }}.sh.j2"
+    dest: "{{ collectd_script_path }}/{{ collectd_script_name }}.sh"
+    owner: root
+    mode: 0755
+  notify:
+    - restart plugin script

ansible/roles/collectd-connectivity-test/tasks/main.yml

@@ -0,0 +1,8 @@
+---
+- import_tasks: configure_collectd.yml
+  tags:
+    - ec2provision
+    - ec2patch
+  when:
+    - ansible_distribution in ['RedHat', 'OracleLinux']
+    - ec2.tags['connectivity-tests'] is defined

ansible/roles/collectd-connectivity-test/templates/collectd_connectivity_test.conf.j2

@@ -0,0 +1,4 @@
+LoadPlugin exec
+<Plugin exec>
+   Exec "{{ collectd_script_user }}" "{{ collectd_script_path }}/{{ collectd_script_name }}.sh"
+</Plugin>

ansible/roles/collectd-connectivity-test/templates/collectd_connectivity_test.sh.j2

@@ -0,0 +1,28 @@
+#!/bin/bash
+# Managed by collectd-connectivity-test ansible role
+# If manually editing, just kill script and collectd will respawn
+# e.g. pkill -u {{ collectd_script_user }} -f {{ collectd_script_path }}/{{ collectd_script_name }}.sh
+
+HOSTNAME="${HOSTNAME:-localhost}"
+INTERVAL="${INTERVAL:-{{ collectd_script_interval }}}"
+CONNECTIVITY_TIMEOUT={{ connectivity_test_connect_timeout }}
+
+get_targets() {
+  aws ec2 describe-tags --filters "Name=resource-id,Values={{ ansible_ec2_instance_id }}" "Name=key,Values=connectivity-tests" --query Tags[0].Value --output=text
+}
+
+connectivity_test() {
+  ip=$(echo $1 | cut -d: -f1)
+  port=$(echo $1 | cut -d: -f2)
+  ncat -vzw "$CONNECTIVITY_TIMEOUT" $ip $port
+}
+
+while sleep "$INTERVAL"; do
+  targets=$(get_targets)
+  if [[ "$targets" != "None" ]]; then
+    for target in $targets; do
+      connectivity_test $target >/dev/null 2>&1
+      echo "PUTVAL $HOSTNAME/connectivity_test/exitcode-$target interval=$INTERVAL N:$?"
+    done
+  fi
+done

ansible/roles/collectd-oracle-db-connected/templates/collectd_oracle_db_connected.sh.j2

@@ -57,7 +57,7 @@ ORAENV_ASK="NO"
 while sleep "$INTERVAL"; do
   SIDS=$(get_sids)
   if [[ "$SIDS" != "None" ]]; then
-    for SID in $(get_sids); do
+    for SID in $SIDS; do
       db_connected $SID >/dev/null 2>&1
       echo "PUTVAL $HOSTNAME/oracle_db_connected/exitcode-$SID interval=$INTERVAL N:$?"
     done

ansible/roles/collectd-textfile-monitoring/templates/collectd_textfile_monitoring.sh.j2

@@ -10,18 +10,20 @@ while sleep "$INTERVAL"; do
   now=$(date +%s)
   for file in {{ collectd_textfile_monitoring_paths }}; do
 {% raw %}
-    IFS=$'\n'
-    metrics=($(grep -E "^[[:alnum:]_]+[[:space:]]+[[:digit:]]+" $file))
-    unset IFS
-    file_last_modified=$(date -r $file +%s)
-    secs_since_last_modified=$((now - file_last_modified))
+    if [[ -e "$file" ]]; then
+      IFS=$'\n'
+      metrics=($(grep -E "^[[:alnum:]_]+[[:space:]]+[[:digit:]]+" "$file"))
+      unset IFS
+      file_last_modified=$(date -r "$file" +%s)
+      secs_since_last_modified=$((now - file_last_modified))
 
-    num_metrics=${#metrics[@]}
-    for ((i=0; i<num_metrics; i++)); do
-      metric=(${metrics[i]})
-      echo "PUTVAL $HOSTNAME/textfile_monitoring/gauge-${metric[0]} interval=$INTERVAL N:${metric[1]}"
-      echo "PUTVAL $HOSTNAME/textfile_monitoring/duration-${metric[0]} interval=$INTERVAL N:${secs_since_last_modified}"
-    done
+      num_metrics=${#metrics[@]}
+      for ((i=0; i<num_metrics; i++)); do
+        metric=(${metrics[i]})
+        echo "PUTVAL $HOSTNAME/textfile_monitoring/gauge-${metric[0]} interval=$INTERVAL N:${metric[1]}"
+        echo "PUTVAL $HOSTNAME/textfile_monitoring/duration-${metric[0]} interval=$INTERVAL N:${secs_since_last_modified}"
+      done
+    fi
 {% endraw %}
   done
 done