Merge pull request #609 from arbulu89/feature/count-states

Feature/count states

generic_modules/salt_provisioner/main.tf

@@ -64,7 +64,7 @@ resource "null_resource" "provision" {
 
   provisioner "remote-exec" {
     inline = [
-      "sudo bash /tmp/salt/provision.sh -sol /var/log/salt-result.log",
+      "sudo bash /tmp/salt/provision.sh -socl /var/log/salt-result.log",
     ]
   }
 

salt/cluster_node/ha/init.sls

@@ -2,7 +2,6 @@ include:
   {% if grains['provider'] in ['aws', 'azure', 'gcp'] %}
   - cluster_node.ha.network
   {% endif %}
-  - cluster_node.ha.packages
   {% if grains['cluster_ssh_pub'] is defined and grains['cluster_ssh_key'] is defined %}
   - cluster_node.ha.ssh
   {% endif %}

salt/count_states.py

@@ -0,0 +1,103 @@
+"""
+Count planned salt states that are executed during the os_setup, predeployment and deployment.
+
+The execution will run multiple salt show low state commands to render the planned states with the
+provided pillar files.
+
+Find more information about highstate and lowstates:
+https://docs.saltstack.com/en/latest/ref/states/layers.html
+
+The show prefix just runs the execution in a dry-run mode, without applying the real changes.
+
+Besides that, the code will run the lowstates command recursively if some of them have more
+salt execution within them, like `set_grains_sbd_disk_device` in cluster_node.ha.iscsi_initiator.
+
+Find more about the used commands in:
+Look for `show_low_sls` and `show_lowstate`
+https://docs.saltstack.com/en/latest/ref/modules/all/salt.modules.state.html
+"""
+
+import logging
+import subprocess
+import shlex
+import json
+import re
+
+LOWSTATE_CMD = "salt-call --local -l quiet --no-color state.show_lowstate saltenv={saltenv} --out=json"
+LOWSTATE_SLS_CMD = "salt-call --local -l quiet --no-color state.show_low_sls {state_path} saltenv={saltenv} pillar='{pillar}' --out=json"
+
+LOW_STATES = ["os_setup"]
+SALTENVS = ["predeployment", "base"]
+
+LOGGER = logging.getLogger(__name__)
+
+
+def execute_command(cmd):
+    """
+    Execute command and return output
+    """
+    LOGGER.debug("Executing command: %s", cmd)
+    proc = subprocess.Popen(
+        shlex.split(cmd),
+        stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
+
+    out, err = proc.communicate()
+    return json.loads(out)
+
+def run_lowstate(state_path, saltenv="base", pillar={}):
+    """
+    Run state.show_low_sls command to the provided salt path and env
+    """
+    state_count = 0
+    pillar = json.dumps(pillar)
+    # Workaround to change to existing `/dev`. Real devs cause errors during the low_state
+    # sda2 is one of the partitions used for booting
+    pillar = re.sub('"/dev/.*?"', '"/dev/sda2"', pillar)
+    cmd = LOWSTATE_SLS_CMD.format(state_path=state_path, saltenv=saltenv, pillar=pillar)
+    state_data = execute_command(cmd)
+    state_count += len(state_data["local"])
+    LOGGER.debug("States count: %d", state_count)
+    state_count += count_inner_states(saltenv, state_data)
+    return state_count
+
+def count_inner_states(saltenv, states):
+    """
+    Count inner states recursively.
+    Some times, some salt code executes other salt code. This method will execute this
+    scenarios
+    """
+    state_count = 0
+    pillar = {}
+    for state in states["local"]:
+        if "state.sls" in state and state["state"] == "module":
+            LOGGER.debug("Inner state found: %s", state["name"])
+            mods = state["state.sls"][0]["mods"]
+            if len(state["state.sls"]) > 1:
+                pillar = state["state.sls"][1].get("pillar", {})
+            for mod in mods:
+                state_count += run_lowstate(state_path=mod, saltenv=saltenv, pillar=pillar)
+    return state_count
+
+
+def main():
+    """
+    Main method. Check the script information at the top dostring entry
+    """
+    state_count = 0
+    for state in LOW_STATES:
+        state_count += run_lowstate(state_path=state)
+    for saltenv in SALTENVS:
+        cmd = LOWSTATE_CMD.format(saltenv=saltenv)
+        state_data = execute_command(cmd)
+        current_state_count = len(state_data["local"])
+        LOGGER.debug("States count: %d", current_state_count)
+        state_count += current_state_count
+        state_count += count_inner_states(saltenv, state_data)
+    LOGGER.debug("Planned states count: %d", state_count)
+    return state_count
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level="INFO")
+    state_count = main()
+    print("Total planned states count:", state_count)

salt/default/init.sls

@@ -1,5 +1,4 @@
 include:
-  - default.hostname
   - default.pkgs
 {% if grains['provider'] == 'libvirt' %}
   - default.timezone

salt/drbd_node/drbd_packages.sls

@@ -4,12 +4,6 @@ drbd-kmp-default:
         attempts: 3
         interval: 15
 
-drbd-formula:
-  pkg.installed:
-    - retry:
-        attempts: 3
-        interval: 15
-
 parted_package:
   pkg.installed:
     - name: parted

salt/hana_node/hana_packages.sls

@@ -10,9 +10,3 @@ python3-shaptools:
         attempts: 3
         interval: 15
 {% endif %}
-
-saphanabootstrap-formula:
-  pkg.installed:
-    - retry:
-        attempts: 3
-        interval: 15

salt/netweaver_node/init.sls

@@ -1,5 +1,4 @@
 include:
   - netweaver_node.mount
   - netweaver_node.nfs
-  - netweaver_node.netweaver_packages
   - netweaver_node.installation_files

salt/os_setup/init.sls

@@ -4,6 +4,9 @@ include:
   - os_setup.repos
   - os_setup.minion_configuration
   - os_setup.packages
-  {% if grains['provider'] == 'libvirt' %}
+  {%- if grains['hostname'] is defined %}
+  - os_setup.hostname
+  {%- endif %}
+  {%- if grains['provider'] == 'libvirt' %}
   - os_setup.ip_workaround
-  {% endif %}
+  {%- endif %}

salt/os_setup/packages.sls

@@ -1,5 +1,20 @@
-iscsi-formula:
+{%- if grains['role'] in ['iscsi_srv', 'drbd_node', 'hana_node', 'netweaver_node'] %}
+install-salt-formulas:
   pkg.installed:
+    - pkgs:
+      {%- if grains['role'] == 'iscsi_srv' %}
+      - iscsi-formula
+      {%- elif grains['role'] == 'drbd_node' %}
+      - drbd-formula
+      - habootstrap-formula
+      {%- elif grains['role'] == 'hana_node' %}
+      - saphanabootstrap-formula
+      - habootstrap-formula
+      {%- elif grains['role'] == 'netweaver_node' %}
+      - sapnwbootstrap-formula
+      - habootstrap-formula
+      {%- endif %}
     - retry:
         attempts: 3
         interval: 15
+{%- endif %}

salt/provision.sh

@@ -33,7 +33,7 @@ log_ok () {
 log_error () {
     node=$(hostname)
     timestamp=$(date -u)
-    message="$timestamp::$node::[ERROR] $1"
+    message="$timestamp::$node::[ERROR] $1::Deployment failed"
     if [[ "$(get_grain provisioning_output_colored)" == "true" ]]; then
       red="\033[0;31m"
       nc="\033[0m" # No Color
@@ -127,7 +127,7 @@ bootstrap_salt () {
 
 os_setup () {
     # Execute the states within /srv/salt/os_setup
-    # This first execution is done to configure the salt minion and install the iscsi formula
+    log_ok "Configuring operative system..."
     # shellcheck disable=SC2046
     salt-call --local \
         --log-level=$(get_grain provisioning_log_level) \
@@ -139,9 +139,17 @@ os_setup () {
     log_ok "os setup done"
 }
 
+count_states () {
+    # Run count_state.py to print the planned states count
+    # This number is used to manage the blue-horizon progress bar
+    log_ok "running count states..."
+    python3 /srv/salt/count_states.py
+}
+
 predeploy () {
     # Execute the states defined in /srv/salt/top.sls
     # This execution is done to pre configure the cluster nodes, the support machines and install the formulas
+    log_ok "Provisioning system..."
     # shellcheck disable=SC2046
     salt-call --local \
         --log-level=$(get_grain provisioning_log_level) \
@@ -197,6 +205,7 @@ from top to bottom in this help text.
 Supported Options (if no options are provided (excluding -l) all the steps will be executed):
   -s               Bootstrap salt installation and configuration. It will register to SCC channels if needed
   -o               Execute OS setup operations. Register to SCC, updated the packages, etc
+  -c               Count salt states that will be executed during predeployment and deployment
   -p               Execute predeployment operations (update hosts and hostnames, install support packages, etc)
   -d               Execute deployment operations (install sap, ha, drbd, etc)
   -q               Execute qa tests
@@ -206,7 +215,7 @@ EOF
 }
 
 argument_number=0
-while getopts ":hsopdql:" opt; do
+while getopts ":hsocpdql:" opt; do
     argument_number=$((argument_number + 1))
     case $opt in
         h)
@@ -216,6 +225,9 @@ while getopts ":hsopdql:" opt; do
         s)
             excute_bootstrap_salt=1
             ;;
+        c)
+            execute_count_states=1
+            ;;
         o)
             excute_os_setup=1
             ;;
@@ -248,12 +260,14 @@ fi
 if [ $argument_number -eq 0 ]; then
     bootstrap_salt
     os_setup
+    count_states
     predeploy
     deploy
     run_tests
 else
     [[ -n $excute_bootstrap_salt ]] && bootstrap_salt
     [[ -n $excute_os_setup ]] && os_setup
+    [[ -n $execute_count_states ]] && count_states
     [[ -n $excute_predeploy ]] && predeploy
     [[ -n $excute_deploy ]] && deploy
     [[ -n $excute_run_tests ]] && run_tests