diff --git a/generic_modules/salt_provisioner/main.tf b/generic_modules/salt_provisioner/main.tf index 883a0509a..f0035a738 100644 --- a/generic_modules/salt_provisioner/main.tf +++ b/generic_modules/salt_provisioner/main.tf @@ -64,7 +64,7 @@ resource "null_resource" "provision" { provisioner "remote-exec" { inline = [ - "sudo bash /tmp/salt/provision.sh -sol /var/log/salt-result.log", + "sudo bash /tmp/salt/provision.sh -socl /var/log/salt-result.log", ] } diff --git a/salt/cluster_node/ha/init.sls b/salt/cluster_node/ha/init.sls index 5076dbebe..e48462cdd 100644 --- a/salt/cluster_node/ha/init.sls +++ b/salt/cluster_node/ha/init.sls @@ -2,7 +2,6 @@ include: {% if grains['provider'] in ['aws', 'azure', 'gcp'] %} - cluster_node.ha.network {% endif %} - - cluster_node.ha.packages {% if grains['cluster_ssh_pub'] is defined and grains['cluster_ssh_key'] is defined %} - cluster_node.ha.ssh {% endif %} diff --git a/salt/cluster_node/ha/packages.sls b/salt/cluster_node/ha/packages.sls deleted file mode 100644 index a135cb1e9..000000000 --- a/salt/cluster_node/ha/packages.sls +++ /dev/null @@ -1,5 +0,0 @@ -habootstrap-formula: - pkg.installed: - - retry: - attempts: 3 - interval: 15 diff --git a/salt/count_states.py b/salt/count_states.py new file mode 100644 index 000000000..13a97da7c --- /dev/null +++ b/salt/count_states.py @@ -0,0 +1,103 @@ +""" +Count planned salt states that are executed during the os_setup, predeployment and deployment. + +The execution will run multiple salt show low state commands to render the planned states with the +provided pillar files. + +Find more information about highstate and lowstates: +https://docs.saltstack.com/en/latest/ref/states/layers.html + +The show prefix just runs the execution in a dry-run mode, without applying the real changes. + +Besides that, the code will run the lowstates command recursively if some of them have more +salt execution within them, like `set_grains_sbd_disk_device` in cluster_node.ha.iscsi_initiator. + +Find more about the used commands in: +Look for `show_low_sls` and `show_lowstate` +https://docs.saltstack.com/en/latest/ref/modules/all/salt.modules.state.html +""" + +import logging +import subprocess +import shlex +import json +import re + +LOWSTATE_CMD = "salt-call --local -l quiet --no-color state.show_lowstate saltenv={saltenv} --out=json" +LOWSTATE_SLS_CMD = "salt-call --local -l quiet --no-color state.show_low_sls {state_path} saltenv={saltenv} pillar='{pillar}' --out=json" + +LOW_STATES = ["os_setup"] +SALTENVS = ["predeployment", "base"] + +LOGGER = logging.getLogger(__name__) + + +def execute_command(cmd): + """ + Execute command and return output + """ + LOGGER.debug("Executing command: %s", cmd) + proc = subprocess.Popen( + shlex.split(cmd), + stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE) + + out, err = proc.communicate() + return json.loads(out) + +def run_lowstate(state_path, saltenv="base", pillar={}): + """ + Run state.show_low_sls command to the provided salt path and env + """ + state_count = 0 + pillar = json.dumps(pillar) + # Workaround to change to existing `/dev`. Real devs cause errors during the low_state + # sda2 is one of the partitions used for booting + pillar = re.sub('"/dev/.*?"', '"/dev/sda2"', pillar) + cmd = LOWSTATE_SLS_CMD.format(state_path=state_path, saltenv=saltenv, pillar=pillar) + state_data = execute_command(cmd) + state_count += len(state_data["local"]) + LOGGER.debug("States count: %d", state_count) + state_count += count_inner_states(saltenv, state_data) + return state_count + +def count_inner_states(saltenv, states): + """ + Count inner states recursively. + Some times, some salt code executes other salt code. This method will execute this + scenarios + """ + state_count = 0 + pillar = {} + for state in states["local"]: + if "state.sls" in state and state["state"] == "module": + LOGGER.debug("Inner state found: %s", state["name"]) + mods = state["state.sls"][0]["mods"] + if len(state["state.sls"]) > 1: + pillar = state["state.sls"][1].get("pillar", {}) + for mod in mods: + state_count += run_lowstate(state_path=mod, saltenv=saltenv, pillar=pillar) + return state_count + + +def main(): + """ + Main method. Check the script information at the top dostring entry + """ + state_count = 0 + for state in LOW_STATES: + state_count += run_lowstate(state_path=state) + for saltenv in SALTENVS: + cmd = LOWSTATE_CMD.format(saltenv=saltenv) + state_data = execute_command(cmd) + current_state_count = len(state_data["local"]) + LOGGER.debug("States count: %d", current_state_count) + state_count += current_state_count + state_count += count_inner_states(saltenv, state_data) + LOGGER.debug("Planned states count: %d", state_count) + return state_count + + +if __name__ == "__main__": + logging.basicConfig(level="INFO") + state_count = main() + print("Total planned states count:", state_count) diff --git a/salt/default/init.sls b/salt/default/init.sls index 580d2bfbc..186ae4eb9 100644 --- a/salt/default/init.sls +++ b/salt/default/init.sls @@ -1,5 +1,4 @@ include: - - default.hostname - default.pkgs {% if grains['provider'] == 'libvirt' %} - default.timezone diff --git a/salt/drbd_node/drbd_packages.sls b/salt/drbd_node/drbd_packages.sls index 5e9d213f6..84f559028 100644 --- a/salt/drbd_node/drbd_packages.sls +++ b/salt/drbd_node/drbd_packages.sls @@ -4,12 +4,6 @@ drbd-kmp-default: attempts: 3 interval: 15 -drbd-formula: - pkg.installed: - - retry: - attempts: 3 - interval: 15 - parted_package: pkg.installed: - name: parted diff --git a/salt/hana_node/hana_packages.sls b/salt/hana_node/hana_packages.sls index c04925536..73ea25c3d 100644 --- a/salt/hana_node/hana_packages.sls +++ b/salt/hana_node/hana_packages.sls @@ -10,9 +10,3 @@ python3-shaptools: attempts: 3 interval: 15 {% endif %} - -saphanabootstrap-formula: - pkg.installed: - - retry: - attempts: 3 - interval: 15 diff --git a/salt/netweaver_node/init.sls b/salt/netweaver_node/init.sls index 2540f8a20..fadbad51f 100644 --- a/salt/netweaver_node/init.sls +++ b/salt/netweaver_node/init.sls @@ -1,5 +1,4 @@ include: - netweaver_node.mount - netweaver_node.nfs - - netweaver_node.netweaver_packages - netweaver_node.installation_files diff --git a/salt/netweaver_node/netweaver_packages.sls b/salt/netweaver_node/netweaver_packages.sls deleted file mode 100644 index 1a19cc1fd..000000000 --- a/salt/netweaver_node/netweaver_packages.sls +++ /dev/null @@ -1,5 +0,0 @@ -sapnwbootstrap-formula: - pkg.installed: - - retry: - attempts: 3 - interval: 15 diff --git a/salt/default/hostname.sls b/salt/os_setup/hostname.sls similarity index 100% rename from salt/default/hostname.sls rename to salt/os_setup/hostname.sls diff --git a/salt/os_setup/init.sls b/salt/os_setup/init.sls index b3aa48fc5..afda3e93a 100644 --- a/salt/os_setup/init.sls +++ b/salt/os_setup/init.sls @@ -4,6 +4,9 @@ include: - os_setup.repos - os_setup.minion_configuration - os_setup.packages - {% if grains['provider'] == 'libvirt' %} + {%- if grains['hostname'] is defined %} + - os_setup.hostname + {%- endif %} + {%- if grains['provider'] == 'libvirt' %} - os_setup.ip_workaround - {% endif %} + {%- endif %} diff --git a/salt/os_setup/packages.sls b/salt/os_setup/packages.sls index bd6c28de3..40632cbf3 100644 --- a/salt/os_setup/packages.sls +++ b/salt/os_setup/packages.sls @@ -1,5 +1,20 @@ -iscsi-formula: +{%- if grains['role'] in ['iscsi_srv', 'drbd_node', 'hana_node', 'netweaver_node'] %} +install-salt-formulas: pkg.installed: + - pkgs: + {%- if grains['role'] == 'iscsi_srv' %} + - iscsi-formula + {%- elif grains['role'] == 'drbd_node' %} + - drbd-formula + - habootstrap-formula + {%- elif grains['role'] == 'hana_node' %} + - saphanabootstrap-formula + - habootstrap-formula + {%- elif grains['role'] == 'netweaver_node' %} + - sapnwbootstrap-formula + - habootstrap-formula + {%- endif %} - retry: attempts: 3 interval: 15 +{%- endif %} diff --git a/salt/provision.sh b/salt/provision.sh index 9059190f1..67dac4a71 100644 --- a/salt/provision.sh +++ b/salt/provision.sh @@ -33,7 +33,7 @@ log_ok () { log_error () { node=$(hostname) timestamp=$(date -u) - message="$timestamp::$node::[ERROR] $1" + message="$timestamp::$node::[ERROR] $1::Deployment failed" if [[ "$(get_grain provisioning_output_colored)" == "true" ]]; then red="\033[0;31m" nc="\033[0m" # No Color @@ -127,7 +127,7 @@ bootstrap_salt () { os_setup () { # Execute the states within /srv/salt/os_setup - # This first execution is done to configure the salt minion and install the iscsi formula + log_ok "Configuring operative system..." # shellcheck disable=SC2046 salt-call --local \ --log-level=$(get_grain provisioning_log_level) \ @@ -139,9 +139,17 @@ os_setup () { log_ok "os setup done" } +count_states () { + # Run count_state.py to print the planned states count + # This number is used to manage the blue-horizon progress bar + log_ok "running count states..." + python3 /srv/salt/count_states.py +} + predeploy () { # Execute the states defined in /srv/salt/top.sls # This execution is done to pre configure the cluster nodes, the support machines and install the formulas + log_ok "Provisioning system..." # shellcheck disable=SC2046 salt-call --local \ --log-level=$(get_grain provisioning_log_level) \ @@ -197,6 +205,7 @@ from top to bottom in this help text. Supported Options (if no options are provided (excluding -l) all the steps will be executed): -s Bootstrap salt installation and configuration. It will register to SCC channels if needed -o Execute OS setup operations. Register to SCC, updated the packages, etc + -c Count salt states that will be executed during predeployment and deployment -p Execute predeployment operations (update hosts and hostnames, install support packages, etc) -d Execute deployment operations (install sap, ha, drbd, etc) -q Execute qa tests @@ -206,7 +215,7 @@ EOF } argument_number=0 -while getopts ":hsopdql:" opt; do +while getopts ":hsocpdql:" opt; do argument_number=$((argument_number + 1)) case $opt in h) @@ -216,6 +225,9 @@ while getopts ":hsopdql:" opt; do s) excute_bootstrap_salt=1 ;; + c) + execute_count_states=1 + ;; o) excute_os_setup=1 ;; @@ -248,12 +260,14 @@ fi if [ $argument_number -eq 0 ]; then bootstrap_salt os_setup + count_states predeploy deploy run_tests else [[ -n $excute_bootstrap_salt ]] && bootstrap_salt [[ -n $excute_os_setup ]] && os_setup + [[ -n $execute_count_states ]] && count_states [[ -n $excute_predeploy ]] && predeploy [[ -n $excute_deploy ]] && deploy [[ -n $excute_run_tests ]] && run_tests