Add a rescue operation to handle network-related failures (#96)

Author: devanshjainms

docs/HIGH_AVAILABILITY.md

@@ -249,7 +249,7 @@ db_sid: "your-db-sid"
 
 # Boolean indicating if the SCS and database is configured as highly available.
 scs_high_availability: true
-db_high_availability: true
+database_high_availability: true
 
 # The high availability configuration of the SCS and DB instance. Supported values are:
 # - AFA (for Azure Fencing Agent)

requirements.txt

@@ -23,7 +23,7 @@ attrs==25.3.0
     #   referencing
 azure-common==1.1.28
     # via azure-mgmt-network
-azure-core==1.34.0
+azure-core==1.35.0
     # via
     #   azure-identity
     #   azure-kusto-data
@@ -40,7 +40,7 @@ azure-kusto-data==5.0.4
     #   azure-kusto-ingest
 azure-kusto-ingest==5.0.4
     # via -r requirements.in
-azure-mgmt-core==1.5.0
+azure-mgmt-core==1.6.0
     # via azure-mgmt-network
 azure-mgmt-network==29.0.0
     # via -r requirements.in
@@ -58,7 +58,7 @@ black==25.1.0
     #   ansible-lint
 bracex==2.6
     # via wcmatch
-certifi==2025.6.15
+certifi==2025.7.9
     # via requests
 cffi==1.17.1
     # via cryptography
@@ -68,11 +68,11 @@ click==8.2.1
     # via
     #   -r requirements.in
     #   black
-coverage[toml]==7.9.1
+coverage[toml]==7.9.2
     # via
     #   -r requirements.in
     #   pytest-cov
-cryptography==45.0.4
+cryptography==45.0.5
     # via
     #   ansible-core
     #   azure-identity
@@ -144,7 +144,7 @@ packaging==25.0
     #   ansible-runner
     #   black
     #   pytest
-pandas==2.3.0
+pandas==2.3.1
     # via -r requirements.in
 pathspec==0.12.1
     # via
@@ -241,7 +241,7 @@ tomli==2.2.1
     #   pytest
 tomlkit==0.13.3
     # via pylint
-typing-extensions==4.14.0
+typing-extensions==4.14.1
     # via
     #   astroid
     #   azure-core

scripts/sap_automation_qa.sh

@@ -301,7 +301,9 @@ run_ansible_playbook() {
         local filtered_config
         filtered_config=$(get_filtered_test_config)
         if [[ -n "$filtered_config" ]]; then
-            extra_vars="--extra-vars '$filtered_config'"
+            local temp_config_file=$(mktemp)
+            echo "$filtered_config" > "$temp_config_file"
+            extra_vars="--extra-vars @$temp_config_file"
         fi
     fi
 
@@ -384,14 +386,14 @@ run_ansible_playbook() {
                 check_file_exists "$temp_file" \
                     "Temporary password file not found. Please check the Key Vault secret ID."
                 command="ansible-playbook ${cmd_dir}/../src/$playbook_name.yml -i $system_hosts \
-                    --extra-vars \"ansible_ssh_pass=$(cat $temp_file)\" --extra-vars @$VARS_FILE -e @$system_params \
+                    --extra-vars 'ansible_ssh_pass=$(cat $temp_file)' --extra-vars @$VARS_FILE -e @$system_params \
                     -e '_workspace_directory=$system_config_folder' $extra_vars"
             else
                 local password_file="${cmd_dir}/../WORKSPACES/SYSTEM/$SYSTEM_CONFIG_NAME/password"
                 check_file_exists "$password_file" \
                     "password file not found in WORKSPACES/SYSTEM/$SYSTEM_CONFIG_NAME directory."
                 command="ansible-playbook ${cmd_dir}/../src/$playbook_name.yml -i $system_hosts \
-                    --extra-vars \"ansible_ssh_pass=$(cat $password_file)\" --extra-vars @$VARS_FILE -e @$system_params \
+                    --extra-vars 'ansible_ssh_pass=$(cat $password_file)' --extra-vars @$VARS_FILE -e @$system_params \
                     -e '_workspace_directory=$system_config_folder' $extra_vars"
             fi
 
@@ -411,11 +413,16 @@ run_ansible_playbook() {
     return_code=$?
     log "INFO" "Ansible playbook execution completed with return code: $return_code"
 
-    # Clean up temporary file if it exists
+    # Clean up temporary files if they exist
     if [[ -n "$temp_file" && -f "$temp_file" ]]; then
         rm -f "$temp_file"
         log "INFO" "Temporary file deleted: $temp_file"
     fi
+    
+    if [[ -n "$temp_config_file" && -f "$temp_config_file" ]]; then
+        rm -f "$temp_config_file"
+        log "INFO" "Temporary config file deleted: $temp_config_file"
+    fi
 
     exit $return_code
 }

src/module_utils/get_pcmk_properties.py

@@ -165,6 +165,8 @@ def _create_parameter(
 
         if isinstance(expected_value, list):
             expected_value = expected_value[0] if expected_value else ""
+        elif isinstance(expected_value, dict):
+            expected_value = list(expected_value.values())[0] if expected_value else ""
 
         return Parameters(
             category=f"{category}_{subcategory}" if subcategory else category,

src/modules/get_pcmk_properties_scs.py

@@ -324,7 +324,7 @@ def main() -> None:
         sid=module.params["sid"],
         scs_instance_number=module.params["ascs_instance_number"],
         ers_instance_number=module.params["ers_instance_number"],
-        os_type=OperatingSystemFamily(os_family),
+        os_type=OperatingSystemFamily(os_family.upper()),
         virtual_machine_name=module.params["virtual_machine_name"],
         constants=module.params["pcmk_constants"],
         fencing_mechanism=module.params["fencing_mechanism"],

src/roles/ha_db_hana/tasks/block-network.yml

@@ -203,6 +203,17 @@
       ansible.builtin.include_tasks:        "roles/misc/tasks/post-validations.yml"
 
   rescue:
+    - name:                                 "Test Execution Failure: Remove the firewall rule on primary node."
+      become:                               true
+      ansible.builtin.shell: |
+                                            iptables -D INPUT -s {{ secondary_node_ip }} -j DROP;
+                                            iptables -D OUTPUT -d {{ secondary_node_ip }} -j DROP
+      register:                             firewall_rule_deleted
+      changed_when:                         firewall_rule_deleted.rc == 0
+      failed_when:                          false
+      when:                                 ansible_hostname == cluster_status_pre.primary_node and
+                                            secondary_node_ip is defined
+
     - name:                                 "Rescue operation"
       ansible.builtin.include_tasks:        "roles/misc/tasks/rescue.yml"
 

src/roles/ha_scs/tasks/files/constants.yaml

@@ -334,7 +334,7 @@ OS_PARAMETERS:
   DEFAULTS:
     sysctl:
       net.ipv4.tcp_timestamps:          "net.ipv4.tcp_timestamps = 0"
-      vm.swappiness:                    "vm.swappiness = 60"
+      vm.swappiness:                    "vm.swappiness = 10"
     corosync-cmapctl:
       runtime.config.totem.token:       "runtime.config.totem.token (u32) = 30000"
       runtime.config.totem.consensus:   "runtime.config.totem.consensus (u32) = 36000"