Recovery catalog service correction

ansible/group_vars/environment_name_corporate_staff_rostering_preproduction.yml

@@ -18,6 +18,9 @@ osbws_additional_configs:
   - name: osbws_prod
     s3_bucket_name: csr-db-backup-bucket20230822131807238100000001
 
+OMS_SERVER: preprod-oem-a.hmpps-oem.hmpps-preproduction.modernisation-platform.internal
+OEM_AGENT_VERSION: 13.5.0.0.0
+
 # rman details
 rman_backup_script: rman_backup.sh
 recovery_catalog: 0

ansible/group_vars/environment_name_nomis_preproduction.yml

@@ -9,6 +9,10 @@ dns_search_domains:
   - azure.hmpp.root
 PROD_SYSCON_WEB_RELEASE: DB_V11.2.1.1.219
 
+# OEM server
+OMS_SERVER: preprod-oem-a.hmpps-oem.hmpps-preproduction.modernisation-platform.internal
+OEM_AGENT_VERSION: 13.5.0.0.0
+
 db_configs:
   PPCNOM:
     db_name: PPCNOM
@@ -18,3 +22,21 @@ db_configs:
 osbws_additional_configs:
   - name: osbws_prod
     s3_bucket_name: nomis-db-backup-bucket20220427111226918600000001
+
+# rman details
+rman_backup_script: rman_backup.sh
+recovery_catalog: 1
+recovery_catalog_server: "{{ OMS_SERVER }}"
+rman_backup_cron:
+  backup_level_0:
+    - name: rman_backup_weekly
+      weekday: "0"
+      minute: "30"
+      hour: "04"
+      # job: command generated in rman-backup-setup
+  backup_level_1:
+    - name: rman_backup_daily
+      weekday: "1-6"
+      minute: "30"
+      hour: "04"
+      # job: command generated in rman-backup-setup

ansible/group_vars/environment_name_oasys_preproduction.yml

@@ -20,9 +20,8 @@ oracle_ru_patch: OCT2023
 ords_trusted_origins:
   preproduction: "https://pp-oasys.az.justice.gov.uk/eor,https://bridge-pp-oasys.az.justice.gov.uk/eor,https://pp.oasys.service.justice.gov.uk/eor,https://pp-int.oasys.service.justice.gov.uk/eor,http://localhost:8080/eor"
 
-# OEM server
-# OMS_SERVER: preproduction-oem-a.hmpps-oem.hmpps-preproduction.modernisation-platform.internal doesn't exist yet
-# OEM_AGENT_VERSION: 13.5.0.0.0
+OMS_SERVER: preprod-oem-a.hmpps-oem.hmpps-preproduction.modernisation-platform.internal
+OEM_AGENT_VERSION: 13.5.0.0.0
 
 osbws_additional_configs:
   - name: osbws_prod

ansible/roles/oracle-19c/defaults/main.yml

@@ -76,7 +76,6 @@ oracle_patch_details:
     psu_patch: p35742441_190000_Linux-x86-64.zip
     patch_string: "19.21.0.0.231017"
 
-
-oracle_ru_patch: OCT2023  # override this in group_vars when rolling out a new patch
+oracle_ru_patch: OCT2023 # override this in group_vars when rolling out a new patch
 oracle_patch: "{{ oracle_patch_details[ oracle_ru_patch ] }}"
 opatch: "p6880880_190000_Linux-x86-64-{{ oracle_patch.opatch_version }}.zip"

ansible/roles/oracle-oem-agent-setup/templates/add_targets.sh.j2

@@ -7,21 +7,30 @@ HOST_FQDN_NAME=$(hostname --fqdn)
 [[ `hostname` = t* ]] && LIFECYCLE_STATUS="Test" || LIFECYCLE_STATUS="Production"
 
 # For delius hosts, we cannot use the hostname to determine the lifecycle status, so we use the tags instead
-APPLICATION="{{ ec2.tags['application'] }}"
+APPLICATION="hmpps-oem"
 if [[ "${APPLICATION}" == "delius" ]]; then
    LIFECYCLE_STATUS=$(echo {{ ec2.tags['environment-name'].split('-')[-1] }})
    # First character must be made uppercase
    LIFECYCLE_STATUS="${LIFECYCLE_STATUS^}"
    # Oracle does not support a Preproduction status so use Staging instead
    [[ "$LIFECYCLE_STATUS" == "Preproduction" ]] && LIFECYCLE_STATUS="Staging"
+   if [[ "${LIFECYCLE_STATUS}" == "Production" || "${LIFECYCLE_STATUS}" == "MissionCritical" ]]; then
+      CONTACT="#${APPLICATION}-aws-oracle-prod-alerts"
+   else
+      CONTACT="#${APPLICATION}-aws-oracle-dev-alerts"
+   fi
+else 
+   # Oracle does not support a Preproduction status so use Staging instead
+   [[ "$LIFECYCLE_STATUS" == "Preproduction" ]] && LIFECYCLE_STATUS="Staging"
+   if [[ "${LIFECYCLE_STATUS}" == "Production" || "${LIFECYCLE_STATUS}" == "MissionCritical" ]]; then
+      CONTACT="#dba_alerts_prod"
+   else
+      CONTACT="#dba_alerts_devtest"
+   fi
 fi
 
 # We use CONTACT to determine which Slack channel to route incidents
-if [[ "${LIFECYCLE_STATUS}" == "Production" || "${LIFECYCLE_STATUS}" == "MissionCritical" ]]; then
-   CONTACT="#${APPLICATION}-aws-oracle-prod-alerts"
-else
-   CONTACT="#${APPLICATION}-aws-oracle-dev-alerts"
-fi
+
 
 TARGET_PROPERTIES="orcl_gtp_lifecycle_status:${LIFECYCLE_STATUS};orcl_gtp_line_of_bus:${APPLICATION};orcl_gtp_contact:${CONTACT};"
 

ansible/roles/oracle-oms-setup/defaults/main.yml

@@ -22,6 +22,9 @@ oracle_path: /usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/oracle
 oem_install_response_file: oem_install.rsp
 oem_configure_response_file: oem_configure.rsp
 weblogic_admin_username: weblogic
+emcli: "{{ oem_mw_home }}/bin/emcli"
+application: "{{ ec2.tags['application'] }}"
+ruleset_name: "AWS Incident management rule set for {{ application }} targets"
 
 # Variables for OMS RU patching
 oms_ru_patch: p35437906_135000_Generic.zip

ansible/roles/oracle-oms-setup/tasks/create_group.yml

@@ -0,0 +1,38 @@
+---
+- name: Create Dynamic Group for {{ application }} Targets
+  block:
+    # EMCLI Login script contains a password so ensure it is not readable by other users
+    - name: Copy group creation scripts
+      ansible.builtin.template:
+        src: "{{ item }}.j2"
+        dest: "{{ stage }}/{{ item }}"
+        mode: "0700"
+        owner: oracle
+        group: oinstall
+      loop:
+        - emcli_login.sh
+        - create_group.sh
+    - name: Run Dynamic Group Creation Script
+      ansible.builtin.shell: |
+        echo "To resolve - Error: Some required configuration is missing, corrupt, inaccessible, or insecure (access permissions are too liberal)."
+        echo "Resolve the problem and run setup."
+        chown -R oracle:oinstall /u01/app/oracle/product/gc_inst135/em/EMGC_OMS1/sysman/emcli/setup/.emcli
+
+    - name: Run Dynamic Group Creation Script
+      become_user: oracle
+      ansible.builtin.shell: |
+        echo "running emcli_login.sh"
+        {{ stage }}/emcli_login.sh
+        echo "running create_group.sh"
+        {{ stage }}/create_group.sh
+      register: run_group_creation
+      changed_when: run_group_creation.stdout is search('.*dynamic group created.*')
+
+  always:
+    - name: Remove Group Creation scripts from Staging Area
+      ansible.builtin.file:
+        path: "{{ stage }}/{{ item }}"
+        state: absent
+      loop:
+        - emcli_login.sh
+        - create_group.sh

ansible/roles/oracle-oms-setup/tasks/import_incident_rule_set.yml

@@ -0,0 +1,33 @@
+---
+- name: Import Incident Rule Set for {{ application }} Targets
+  block:
+    # EMCLI Login script contains a password so ensure it is not readable by other users
+    - name: Copy Incident Rule Set Import scripts
+      ansible.builtin.template:
+        src: "{{ item }}.j2"
+        dest: "{{ stage }}/{{ item }}"
+        mode: "0700"
+        owner: oracle
+        group: oinstall
+      loop:
+        - emcli_login.sh
+        - import_incident_rule_set.sh
+        - rule_set.xml
+
+    - name: Run Incident Rule Set Import
+      become_user: oracle
+      ansible.builtin.shell: |
+        echo "running emcli_login.sh"
+        {{ stage }}/emcli_login.sh
+        echo "running import_incident_rule_set.sh"
+        {{ stage }}/import_incident_rule_set.sh
+
+  always:
+    - name: Remove Incident Rule Set scripts from Staging Area
+      ansible.builtin.file:
+        path: "{{ stage }}/{{ item }}"
+        state: absent
+      loop:
+        - emcli_login.sh
+        - import_incident_rule_set.sh
+        - rule_set.xml

ansible/roles/oracle-oms-setup/tasks/main.yml

@@ -50,10 +50,26 @@
     - ec2provision
     - oracle_oem_patch_upgrade
 
+- import_tasks: create_group.yml
+  tags:
+    - amibuild
+    - ec2provision
+    - incident_rule_group
+
+- import_tasks: import_incident_rule_set.yml
+  tags:
+    - amibuild
+    - ec2provision
+    - incident_rule_set
+
 - import_tasks: create_slack_wallet.yml
   tags:
+    - amibuild
+    - ec2provision
     - create_slack_wallet
 
 - import_tasks: create_slack_notification_package.yml
   tags:
+    - amibuild
+    - ec2provision
     - create_slack_notification_package

ansible/roles/oracle-oms-setup/templates/create_group.sh.j2

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+. ~/.bash_profile
+export JAVA_HOME=$ORACLE_HOME/jdk/jre
+
+
+if [ $( {{ emcli }} get_groups -noheader -script | cut -f1 | grep -Ec {{ application }} ) -gt 0 ] ; then
+   echo "{{ application }} group already exists."
+else
+   {{ emcli }} create_dynamic_group -name={{ application }} -properties="orcl_gtp_line_of_bus:{{ application }}"
+   if [[ $? -eq 0 ]]; then
+      echo "New {{ application }} dynamic group created."
+   fi
+fi

ansible/roles/oracle-oms-setup/templates/create_slack_notification_package.sh.j2

@@ -49,7 +49,7 @@ create or replace PACKAGE BODY slack_notification AS
       highest priority channel to ensure messages are not lost).
     */
    l_slack_channel         VARCHAR2(100);
-   l_default_slack_channel CONSTANT VARCHAR2(100) DEFAULT '#delius-aws-oracle-prod-alerts';
+   l_default_slack_channel CONSTANT VARCHAR2(100) DEFAULT '#shef_dba;
    l_use_default_channel   BOOLEAN DEFAULT FALSE;
 
   BEGIN
@@ -82,7 +82,7 @@ create or replace PACKAGE BODY slack_notification AS
       END LOOP;
     END IF;
 
-    INSERT INTO z_timezone VALUES (l_target_timezone);
+    -- INSERT INTO z_timezone VALUES (l_target_timezone);
 
     COMMIT;
 
@@ -242,7 +242,7 @@ create or replace PACKAGE BODY slack_notification AS
                  'Event Reported Date: ' || to_char(event_msg.event_payload.reported_date, 'DD-MON-YY HH24:MI:SS')||chr(10)||
                  'Categories: '||l_categories_new||chr(10);
 
-    l_param := 'channel=#delius-aws-oracle-prod-alerts'||chr(38);
+    l_param := 'channel=#shef_dba'||chr(38);
     l_param := l_param||l_message;
 
     l_http_request  := utl_http.begin_request
@@ -284,4 +284,4 @@ create or replace PACKAGE BODY slack_notification AS
   END event_proc;
 
 END slack_notification;
-/
+/

ansible/roles/oracle-oms-setup/templates/emcli_login.sh.j2

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+. ~/.bash_profile
+export ORACLE_HOME=/u01/app/oracle/product/mw135
+export JAVA_HOME=$ORACLE_HOME/jdk/jre
+
+# An emcli session may already be active.  Check if we can run a sync
+# and only if this fails attempt to create a new session.
+{{ emcli }} sync
+if [[ $? -gt 0 ]]; then
+   echo "{{ db_sysman_password }}" | {{ emcli }} login -username=sysman
+   RC=$?
+   if [[ $RC -gt 0 ]]; then
+      exit $RC
+   fi
+   {{ emcli }} sync
+   echo "Logged in."
+fi

ansible/roles/oracle-oms-setup/templates/import_incident_rule_set.sh.j2

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+set -x
+
+. ~/.bash_profile
+
+export ORACLE_HOME=/u01/app/oracle/product/mw135
+export JAVA_HOME=$ORACLE_HOME/jdk/jre
+
+RULE_SET_BASE_NAME="Rule set for {{ ec2.tags['application'] }}"
+
+{{ emcli }} import_incident_rule_set -import_file={{ stage }}/rule_set.xml  -alt_rule_set_name="${RULE_SET_BASE_NAME}"
+RC=$?
+[[ $RC -eq 2 ]] && [[ '{{ replace_existing_rule_set | default('false')}}' == 'false' ]] && exit 2
+if [[ $RC -eq 2 ]]; then  
+  # The rule set already exists.  As of 13c Oracle do not provide EMCLI functionality
+  # to delete the rule set, so we instead disable the existing rule set and create
+  # a new one
+  RULE_SET_CREATE_FLAG=1
+  RULE_SET_NAME=${RULE_SET_BASE_NAME}
+  while [[ RULE_SET_CREATE_FLAG -ne 0 ]]; do
+     {{ emcli }} modify_incident_rule -action=disable -type=ruleset -rule_set_name="${RULE_SET_NAME}"
+     VERSION_NUMBER=${RULE_SET_NAME##*[^0-9]}
+     NEXT_VERSION=$((VERSION_NUMBER+1))
+     RULE_SET_NAME="${RULE_SET_BASE_NAME} VERSION ${NEXT_VERSION}"
+     {{ emcli }} import_incident_rule_set -import_file={{ stage }}/rule_set.xml  -alt_rule_set_name="${RULE_SET_NAME}"
+     RC=$?
+     # We loop again if the version already exists; exit the loop for any other return codes
+     [[ $RC -eq 2 ]] && RULE_SET_CREATE_FLAG=1 || RULE_SET_CREATE_FLAG=0
+  done
+  [[ $RC -ne 0 ]] && exit $RC
+  echo "Imported new version of existing rule set."
+else  
+  [[ $RC -ne 0 ]] && exit $RC
+  echo "Imported initial version of rule set."
+fi
+
+# Enable the newly imported rule set
+{{ emcli }} modify_incident_rule -action=enable -type=ruleset -rule_set_name="${RULE_SET_NAME}"