Module to upgrade servers

enos/enos-modules.hcl

@@ -1,8 +1,6 @@
 // Copyright (c) HashiCorp, Inc.
 // SPDX-License-Identifier: BUSL-1.1
 
-// Find any released RPM or Deb in Artifactory. Requires the version, edition, distro, and distro
-// version.
 module "build_artifactory" {
   source = "./modules/fetch_artifactory"
 }
@@ -18,3 +16,7 @@ module "run_workloads" {
 module "test_cluster_health" {
   source = "./modules/test_cluster_health"
 }
+
+module "upgrade_servers" {
+  source = "./modules/upgrade_servers"
+}

enos/enos-scenario-upgrade.hcl

@@ -30,12 +30,14 @@ scenario "upgrade" {
     linux_count   = matrix.os == "linux" ? "4" : "0"
     windows_count = matrix.os == "windows" ? "4" : "0"
     arch          = matrix.arch
+    clients_count = local.linux_count + local.windows_count
   }
 
   step "copy_initial_binary" {
     description = <<-EOF
     Determine which Nomad artifact we want to use for the scenario, depending on the
-   'arch', 'edition' and 'os' and bring it from the artifactory to a local instance.
+    'arch', 'edition' and 'os' and bring it from the artifactory to the local instance
+    running enos.   
     EOF
 
     module = module.build_artifactory
@@ -52,9 +54,11 @@ scenario "upgrade" {
   }
 
   step "provision_cluster" {
-    depends_on  = [step.copy_initial_binary]
+    depends_on = [step.copy_initial_binary]
+
     description = <<-EOF
     Using the binary from the previous step, provision a Nomad cluster using the e2e
+    module.
     EOF
 
     module = module.provision_cluster
@@ -73,7 +77,8 @@ scenario "upgrade" {
   }
 
   step "run_initial_workloads" {
-    depends_on  = [step.provision_cluster]
+    depends_on = [step.provision_cluster]
+
     description = <<-EOF
     Verify the health of the cluster by running new workloads
     EOF
@@ -86,28 +91,34 @@ scenario "upgrade" {
       key_file    = step.provision_cluster.key_file
       nomad_token = step.provision_cluster.nomad_token
     }
+
     verifies = [
       quality.nomad_register_job,
     ]
   }
 
   step "initial_test_cluster_health" {
-    depends_on  = [step.run_initial_workloads]
+    depends_on = [step.run_initial_workloads]
+
     description = <<-EOF
-    Verify the health of the cluster by checking the status of all servers, nodes, jobs and allocs and stopping random allocs to check for correct reschedules"
+    Verify the health of the cluster by checking the status of all servers, nodes, 
+    jobs and allocs and stopping random allocs to check for correct reschedules"
     EOF
 
     module = module.test_cluster_health
     variables {
-      nomad_addr   = step.provision_cluster.nomad_addr
-      ca_file      = step.provision_cluster.ca_file
-      cert_file    = step.provision_cluster.cert_file
-      key_file     = step.provision_cluster.key_file
-      nomad_token  = step.provision_cluster.nomad_token
-      server_count = var.server_count
-      client_count = local.linux_count + local.windows_count
-      jobs_count   = step.run_initial_workloads.jobs_count
-      alloc_count  = step.run_initial_workloads.allocs_count
+      nomad_addr      = step.provision_cluster.nomad_addr
+      ca_file         = step.provision_cluster.ca_file
+      cert_file       = step.provision_cluster.cert_file
+      key_file        = step.provision_cluster.key_file
+      nomad_token     = step.provision_cluster.nomad_token
+      server_count    = var.server_count
+      client_count    = local.clients_count
+      jobs_count      = step.run_initial_workloads.jobs_count
+      alloc_count     = step.run_initial_workloads.allocs_count
+      servers         = step.provision_cluster.servers
+      clients_version = var.product_version
+      servers_version = var.product_version
     }
 
     verifies = [
@@ -120,10 +131,11 @@ scenario "upgrade" {
     ]
   }
 
-  step "copy_upgrade_binary" {
-    depends_on  = [step.provision_cluster]
+  step "fetch_upgrade_binary" {
+    depends_on = [step.provision_cluster]
+
     description = <<-EOF
-    Bring the new upgraded binary from the artifactory
+    Bring the new upgraded binary from the artifactory to the instance running enos.
     EOF
 
     module = module.build_artifactory
@@ -135,51 +147,71 @@ scenario "upgrade" {
       edition              = matrix.edition
       product_version      = var.upgrade_version
       os                   = matrix.os
-      binary_path          = "${var.nomad_local_binary}/${matrix.os}-${matrix.arch}-${matrix.edition}-${var.upgrade_version}"
+      download_binary      = false
     }
   }
-  /*
+
   step "upgrade_servers" {
+    depends_on = [step.fetch_upgrade_binary]
+
     description = <<-EOF
-    Upgrade the cluster's servers by invoking nomad-cc ...
-   EOF
+    Takes the servers one by one, makes a snapshot, updates the binary with the
+    new one previously fetched and restarts the servers.
 
-    module      = module.run_cc_nomad
+    Important: The path where the binary will be placed is hardcoded to match 
+    what the provision-cluster module does. It can be configurable in the future
+    but for now it is:
+
+     * "C:/opt/nomad.exe" for windows 
+     * "/usr/local/bin/nomad" for linux
+
+    To ensure the servers are upgraded one by one, they use the depends_on meta,
+    there are ONLY 3 SERVERS being upgraded in the module.
+   EOF
+    module      = module.upgrade_servers
 
     verifies = [
-        quality.nomad_agent_info,
-        quality.nomad_agent_info_self,
-        nomad_restore_snapshot
+      quality.nomad_agent_info,
+      quality.nomad_agent_info_self,
+      quality.nomad_restore_snapshot
     ]
 
     variables {
-        cc_update_type        = "server"  
-        nomad_upgraded_binary = step.copy_initial_binary.nomad_local_binary
-        // ...
+      nomad_addr           = step.provision_cluster.nomad_addr
+      ca_file              = step.provision_cluster.ca_file
+      cert_file            = step.provision_cluster.cert_file
+      key_file             = step.provision_cluster.key_file
+      nomad_token          = step.provision_cluster.nomad_token
+      servers              = step.provision_cluster.servers
+      ssh_key_path         = step.provision_cluster.ssh_key_file
+      artifactory_username = var.artifactory_username
+      artifactory_token    = var.artifactory_token
+      artifact_url         = step.fetch_upgrade_binary.artifact_url
+      artifact_sha         = step.fetch_upgrade_binary.artifact_sha
     }
   }
 
-  step "run_servers_workloads" {
-   // ...
-  }
-
   step "server_upgrade_test_cluster_health" {
-    depends_on  = [step.run_initial_workloads]
+    depends_on  = [step.upgrade_servers]
     description = <<-EOF
-    Verify the health of the cluster by checking the status of all servers, nodes, jobs and allocs and stopping random allocs to check for correct reschedules"
+    Verify the health of the cluster by checking the status of all servers, nodes, 
+    jobs and allocs and stopping random allocs to check for correct reschedules"
     EOF
 
     module = module.test_cluster_health
     variables {
-      nomad_addr   = step.provision_cluster.nomad_addr
-      ca_file      = step.provision_cluster.ca_file
-      cert_file    = step.provision_cluster.cert_file
-      key_file     = step.provision_cluster.key_file
-      nomad_token  = step.provision_cluster.nomad_token
-      server_count = var.server_count
-      client_count = local.linux_count + local.windows_count
-      jobs_count   = step.run_initial_workloads.jobs_count
-      alloc_count  = step.run_initial_workloads.allocs_count
+      nomad_addr      = step.provision_cluster.nomad_addr
+      ca_file         = step.provision_cluster.ca_file
+      cert_file       = step.provision_cluster.cert_file
+      key_file        = step.provision_cluster.key_file
+      nomad_token     = step.provision_cluster.nomad_token
+      server_count    = var.server_count
+      client_count    = local.linux_count + local.windows_count
+      jobs_count      = step.run_initial_workloads.jobs_count
+      alloc_count     = step.run_initial_workloads.allocs_count
+      servers         = step.provision_cluster.servers
+      clients_version = var.product_version
+      servers_version = var.upgrade_version
     }
 
     verifies = [
@@ -192,6 +224,11 @@ scenario "upgrade" {
     ]
   }
 
+  /*
+  step "run_servers_workloads" {
+   // ...
+  }
+
   step "upgrade_client" {
     description = <<-EOF
     Upgrade the cluster's clients by invoking nomad-cc ...
@@ -244,6 +281,7 @@ scenario "upgrade" {
     ]
   }
  */
+
   output "servers" {
     value = step.provision_cluster.servers
   }
@@ -280,5 +318,4 @@ scenario "upgrade" {
     value     = step.provision_cluster.nomad_token
     sensitive = true
   }
-
 }

enos/modules/fetch_artifactory/main.tf

@@ -23,6 +23,8 @@ data "enos_artifactory_item" "nomad" {
 }
 
 resource "enos_local_exec" "install_binary" {
+  count = var.download_binary ? 1 : 0
+
   environment = {
     URL         = data.enos_artifactory_item.nomad.results[0].url
     BINARY_PATH = var.binary_path

enos/modules/fetch_artifactory/outputs.tf

@@ -5,3 +5,13 @@ output "nomad_local_binary" {
   description = "Path where the binary will be placed"
   value       = var.os == "windows" ? "${var.binary_path}/nomad.exe" : "${var.binary_path}/nomad"
 }
+
+output "artifact_url" {
+  description = "URL to fetch the artifact"
+  value       = data.enos_artifactory_item.nomad.results[0].url
+}
+
+output "artifact_sha" {
+  description = "sha256 to fetch the artifact"
+  value       = data.enos_artifactory_item.nomad.results[0].sha256
+}

enos/modules/fetch_artifactory/variables.tf

@@ -53,3 +53,8 @@ variable "binary_path" {
   type        = string
   default     = "/home/ubuntu/nomad"
 }
+
+variable "download_binary" {
+  description = "Used to control if the artifact should be downloaded to the local instance or not"
+  default     = true
+}

enos/modules/test_cluster_health/main.tf

@@ -51,3 +51,4 @@ resource "enos_local_exec" "verify_versions" {
   ]
 }
 
+

enos/modules/test_cluster_health/scripts/allocs.sh

@@ -31,7 +31,8 @@ MAX_WAIT_TIME=30  # Maximum wait time in seconds
 POLL_INTERVAL=2    # Interval between status checks
 
 random_alloc_id=$(echo "$running_allocs" | jq -r ".[$((RANDOM % ($allocs_length + 1)))].ID")
-nomad alloc stop -detach "$random_alloc_id" || error_exit "Failed to stop allocation $random_alloc_id."
+nomad alloc stop "$random_alloc_id" || error_exit "Failed to stop allocation $random_alloc_id."
+
 
 echo "Waiting for allocation $random_alloc_id to reach 'complete' status..."
 elapsed_time=0

enos/modules/test_cluster_health/variables.tf

@@ -44,6 +44,7 @@ variable "jobs_count" {
 
 variable "alloc_count" {
   description = "Number of allocation that should be running in the cluster"
+  type        = number
 }
 
 variable "clients_version" {

enos/modules/upgrade_instance/.gitignore

@@ -0,0 +1,2 @@
+// Don't commit cluster snapshots
+*.snap

enos/modules/upgrade_instance/main.tf

@@ -0,0 +1,64 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+terraform {
+  required_providers {
+    enos = {
+      source = "registry.terraform.io/hashicorp-forge/enos"
+    }
+  }
+}
+
+locals {
+  binary_destination = var.platform == "windows" ? "C:/opt/" : "/usr/local/bin/"
+  ssh_user           = var.platform == "windows" ? "Administrator" : "ubuntu"
+}
+
+resource "enos_bundle_install" "nomad" {
+  destination = local.binary_destination
+
+  artifactory = var.artifactory_release
+
+  transport = {
+    ssh = {
+      host             = var.server_address
+      private_key_path = var.ssh_key_path
+      user             = local.ssh_user
+    }
+  }
+}
+
+resource "enos_remote_exec" "restart_linux_services" {
+  count      = var.platform == "linux" ? 1 : 0
+  depends_on = [enos_bundle_install.nomad]
+
+
+  transport = {
+    ssh = {
+      host             = var.server_address
+      private_key_path = var.ssh_key_path
+      user             = local.ssh_user
+    }
+  }
+
+  inline = [
+    "sudo systemctl restart nomad",
+  ]
+}
+
+resource "enos_remote_exec" "restart_windows_services" {
+  count      = var.platform == "windows" ? 1 : 0
+  depends_on = [enos_bundle_install.nomad]
+
+  transport = {
+    ssh = {
+      host             = var.server_address
+      private_key_path = var.ssh_key_path
+      user             = local.ssh_user
+    }
+  }
+
+  inline = [
+    "powershell Restart-Service Nomad"
+  ]
+}