Merge pull request #16 from companieshouse/feature/cloudwatch

Implement CloudWatch logging for nGsrc, Tuxedo, and CloudWatch agent
companieshouse · Apr 22, 2021 · 50a9834 · 50a9834
2 parents a516ef9 + ab5b139
commit 50a9834
Show file tree

Hide file tree

Showing 5 changed files with 190 additions and 1 deletion.
diff --git a/roles/deploy/defaults/main.yml b/roles/deploy/defaults/main.yml
@@ -1,5 +1,123 @@
 ---
 
+cloudwatch_agent:
+  path: /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl
+  config_dir: /opt/aws/amazon-cloudwatch-agent/etc
+  log_file: /opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log
+  metrics_collection_interval: 60
+  user: cwagent
+
+tuxedo_log_default_parent: /var/log/tuxedo
+
+tuxedo_log_files:
+  ewf:
+    - file_pattern: CHG-stdout.txt
+      cloudwatch_log_group_name: frontend-tuxedo-ewf-chg-stdout
+    - file_pattern: CHG-stderr.txt
+      cloudwatch_log_group_name: frontend-tuxedo-ewf-chg-stderr
+    - file_pattern: BE-stdout.txt
+      cloudwatch_log_group_name: frontend-tuxedo-ewf-be-stdout
+    - file_pattern: BE-stderr.txt
+      cloudwatch_log_group_name: frontend-tuxedo-ewf-be-stderr
+    - file_pattern: DBG-stdout.txt
+      cloudwatch_log_group_name: frontend-tuxedo-ewf-dbg-stdout
+    - file_pattern: DBG-stderr.txt
+      cloudwatch_log_group_name: frontend-tuxedo-ewf-dbg-stderr
+    - file_pattern: CS-stdout.txt
+      cloudwatch_log_group_name: frontend-tuxedo-ewf-cs-stdout
+    - file_pattern: CS-stderr.txt
+      cloudwatch_log_group_name: frontend-tuxedo-ewf-cs-stderr
+    - file_pattern: Sys-stdout.txt
+      cloudwatch_log_group_name: frontend-tuxedo-ewf-sys-stdout
+    - file_pattern: Sys-stderr.txt
+      cloudwatch_log_group_name: frontend-tuxedo-ewf-sys-stderr
+    - file_pattern: VXBRL-stdout.txt
+      cloudwatch_log_group_name: frontend-tuxedo-ewf-vxbrl-stdout
+    - file_pattern: VXBRL-stderr.txt
+      cloudwatch_log_group_name: frontend-tuxedo-ewf-vxbrl-stderr
+    - file_pattern: VTNEP-stdout.txt
+      cloudwatch_log_group_name: frontend-tuxedo-ewf-vtnep-stdout
+    - file_pattern: VTNEP-stderr.txt
+      cloudwatch_log_group_name: frontend-tuxedo-ewf-vtnep-stderr
+    - file_pattern: TRXML-stdout.txt
+      cloudwatch_log_group_name: frontend-tuxedo-ewf-trxml-stdout
+    - file_pattern: TRXML-stderr.txt
+      cloudwatch_log_group_name: frontend-tuxedo-ewf-trxml-stderr
+    - file_pattern: ULOG*
+      cloudwatch_log_group_name: frontend-tuxedo-ewf-ulog
+    - file_pattern: ois-ngsrv.log
+      cloudwatch_log_group_name: frontend-tuxedo-ewf-ngsrv-ois
+    - file_pattern: search-ngsrv.log
+      cloudwatch_log_group_name: frontend-tuxedo-ewf-ngsrv-search
+    - file_pattern: chips-ngsrv.log
+      cloudwatch_log_group_name: frontend-tuxedo-ewf-ngsrv-chips
+    - file_pattern: ef-ngsrv.log
+      cloudwatch_log_group_name: frontend-tuxedo-ewf-ngsrv-ef
+    - file_pattern: ixbrl-ngsrv.log
+      cloudwatch_log_group_name: frontend-tuxedo-ewf-ngsrv-ixbrl
+    - file_pattern: tnep-ngsrv.log
+      cloudwatch_log_group_name: frontend-tuxedo-ewf-ngsrv-tnep
+    - file_pattern: trans-ngsrv.log
+      cloudwatch_log_group_name: frontend-tuxedo-ewf-ngsrv-trans
+    - file_pattern: gen-ngsrv.log
+      cloudwatch_log_group_name: frontend-tuxedo-ewf-ngsrv-gen
+
+  xml:
+    - file_pattern: CHG-stdout.txt
+      cloudwatch_log_group_name: frontend-tuxedo-xml-chg-stdout
+    - file_pattern: CHG-stderr.txt
+      cloudwatch_log_group_name: frontend-tuxedo-xml-chg-stderr
+    - file_pattern: BE-stdout.txt
+      cloudwatch_log_group_name: frontend-tuxedo-xml-be-stdout
+    - file_pattern: BE-stderr.txt
+      cloudwatch_log_group_name: frontend-tuxedo-xml-be-stderr
+    - file_pattern: DBG-stdout.txt
+      cloudwatch_log_group_name: frontend-tuxedo-xml-dbg-stdout
+    - file_pattern: DBG-stderr.txt
+      cloudwatch_log_group_name: frontend-tuxedo-xml-dbg-stderr
+    - file_pattern: CS-stdout.txt
+      cloudwatch_log_group_name: frontend-tuxedo-xml-cs-stdout
+    - file_pattern: CS-stderr.txt
+      cloudwatch_log_group_name: frontend-tuxedo-xml-cs-stderr
+    - file_pattern: Sys-stdout.txt
+      cloudwatch_log_group_name: frontend-tuxedo-xml-sys-stdout
+    - file_pattern: Sys-stderr.txt
+      cloudwatch_log_group_name: frontend-tuxedo-xml-sys-stderr
+    - file_pattern: VXBRL-stdout.txt
+      cloudwatch_log_group_name: frontend-tuxedo-xml-vxbrl-stdout
+    - file_pattern: VXBRL-stderr.txt
+      cloudwatch_log_group_name: frontend-tuxedo-xml-vxbrl-stderr
+    - file_pattern: VTNEP-stdout.txt
+      cloudwatch_log_group_name: frontend-tuxedo-xml-vtnep-stdout
+    - file_pattern: VTNEP-stderr.txt
+      cloudwatch_log_group_name: frontend-tuxedo-xml-vtnep-stderr
+    - file_pattern: TRXML-stdout.txt
+      cloudwatch_log_group_name: frontend-tuxedo-xml-trxml-stdout
+    - file_pattern: TRXML-stderr.txt
+      cloudwatch_log_group_name: frontend-tuxedo-xml-trxml-stderr
+    - file_pattern: IXBRL-stdout.txt
+      cloudwatch_log_group_name: frontend-tuxedo-xml-ixbrl-stdout
+    - file_pattern: IXBRL-stderr.txt
+      cloudwatch_log_group_name: frontend-tuxedo-xml-ixbrl-stderr
+    - file_pattern: ULOG*
+      cloudwatch_log_group_name: frontend-tuxedo-xml-ulog
+    - file_pattern: ois-ngsrv.log
+      cloudwatch_log_group_name: frontend-tuxedo-xml-ngsrv-ois
+    - file_pattern: search-ngsrv.log
+      cloudwatch_log_group_name: frontend-tuxedo-xml-ngsrv-search
+    - file_pattern: chips-ngsrv.log
+      cloudwatch_log_group_name: frontend-tuxedo-xml-ngsrv-chips
+    - file_pattern: ef-ngsrv.log
+      cloudwatch_log_group_name: frontend-tuxedo-xml-ngsrv-ef
+    - file_pattern: ixbrl-ngsrv.log
+      cloudwatch_log_group_name: frontend-tuxedo-xml-ngsrv-ixbrl
+    - file_pattern: tnep-ngsrv.log
+      cloudwatch_log_group_name: frontend-tuxedo-xml-ngsrv-tnep
+    - file_pattern: trans-ngsrv.log
+      cloudwatch_log_group_name: frontend-tuxedo-xml-ngsrv-trans
+    - file_pattern: gen-ngsrv.log
+      cloudwatch_log_group_name: frontend-tuxedo-xml-ngsrv-gen
+
 tuxedo_service_group: tuxedo
 tuxedo_service_users:
   - ceu

diff --git a/roles/deploy/tasks/deploy.yml b/roles/deploy/tasks/deploy.yml
@@ -168,6 +168,16 @@
     chdir: "/home/{{ tuxedo_user }}/{{ deployment_dir }}/config"
     executable: /bin/bash
 
+- name: "{{ tuxedo_user }} : Create CloudWatch agent configuration file for Tuxedo service group"
+  template:
+    src: templates/cloudwatch-config-service.json.j2
+    dest: "{{ cloudwatch_agent.config_dir }}//cloudwatch-config-{{ tuxedo_user }}.json"
+    trim_blocks: False
+
+- name: "{{ tuxedo_user }} : Add configuration for Tuxedo service group to CloudWatch agent"
+  command:
+    cmd: "{{ cloudwatch_agent.path }} -a append-config -m ec2 -s -c file:{{ cloudwatch_agent.config_dir }}//cloudwatch-config-{{ tuxedo_user }}.json"
+
 - name: "{{ tuxedo_user }} : Start Tuxedo services"
   become_user: "{{ tuxedo_user }}"
   shell: "source $HOME/deployment/config/envfile && tmboot -y"

diff --git a/roles/deploy/tasks/main.yml b/roles/deploy/tasks/main.yml
@@ -1,5 +1,8 @@
 ---
 
+- name: Set EC2 metadata facts
+  amazon.aws.ec2_metadata_facts:
+
 - name: Check required variables are set
   assert:
     that:
@@ -14,11 +17,27 @@
     msg: "Required variable(s) empty or undefined"
 
 # The hostname is assumed to be in the format: frontend-tuxedo-<environment>-<instance-index>
-- set_fact:
+- name: Set Tuxedo facts for config population
+  set_fact:
      tuxedo_domain_id_suffix: "{{ ansible_facts.hostname | regex_replace('^frontend-tuxedo-([A-Za-z].*)-(\\d+)$', 'INSTANCE_\\2_\\1_DOM') | upper }}"
      tuxedo_logical_machine_id_suffix: "{{ ansible_facts.hostname | regex_replace('^frontend-tuxedo-([A-Za-z].*)-(\\d+)$', 'INSTANCE_\\2_\\1_SRV') | upper }}"
      tuxedo_local_domain_suffix: "{{ ansible_facts.hostname | regex_replace('^frontend-tuxedo-([A-Za-z].*)-(\\d+)$', 'INSTANCE_\\2_\\1_LOD') | upper }}"
 
+- name: Set CloudWatch agent facts for config population
+  set_fact:
+    cloudwatch_log_stream_name: "{{ ansible_ec2_instance_id }}_{{ ansible_ec2_hostname }}"
+    region: "{{ ansible_ec2_instance_identity_document_region }}"
+
+- name: Create CloudWatch agent configuration file for instance
+  template:
+    src: templates/cloudwatch-config.json.j2
+    dest: "{{ cloudwatch_agent.config_dir }}/cloudwatch-config.json"
+    trim_blocks: False
+
+- name: Start CloudWatch agent using primary configuration file
+  command:
+    cmd: "{{ cloudwatch_agent.path }} -a fetch-config -m ec2 -s -c file:{{ cloudwatch_agent.config_dir }}//cloudwatch-config.json"
+
 - name: Using constructed variable suffixes
   ansible.builtin.debug:
     var: "{{ item }}"

diff --git a/roles/deploy/templates/cloudwatch-config-service.json.j2 b/roles/deploy/templates/cloudwatch-config-service.json.j2
@@ -0,0 +1,20 @@
+{
+  "logs": {
+    "logs_collected": {
+      "files": {
+        "collect_list": [
+          {% if tuxedo_user in tuxedo_log_files -%}
+          {%+ for log in tuxedo_log_files[tuxedo_user] -%}
+          {
+            "file_path": "{{ log.file_path_parent | default(tuxedo_log_default_parent) }}/{{ tuxedo_user }}/{{ log.file_pattern }}",
+            "log_group_name": "{{ log.cloudwatch_log_group_name }}",
+            "log_stream_name": "{{ cloudwatch_log_stream_name }}"
+          }{%- if not loop.last -%},{%- endif -%}
+          {% endfor -%}
+          {% endif %}
+        ]
+      }
+    },
+    "log_stream_name": "{{ cloudwatch_log_stream_name }}"
+  }
+}
diff --git a/roles/deploy/templates/cloudwatch-config.json.j2 b/roles/deploy/templates/cloudwatch-config.json.j2
@@ -0,0 +1,22 @@
+{
+  "agent": {
+   "metrics_collection_interval": {{ cloudwatch_agent.metrics_collection_interval }},
+   "region": "{{ region }}",
+   "logfile": "{{ cloudwatch_agent.log_file }}",
+   "debug": false,
+   "run_as_user": "{{ cloudwatch_agent.user }}"
+  },
+  "logs": {
+    "logs_collected": {
+      "files": {
+        "collect_list": [
+          {
+            "file_path": "{{ cloudwatch_agent.log_file }}",
+            "log_group_name": "frontend-tuxedo-cloudwatch",
+            "log_stream_name": "{{ cloudwatch_log_stream_name }}"
+          }
+        ]
+      }
+    }
+  }
+}