microcloud/test: Add integration tests for Ceph networking

Signed-off-by: Gabriel Mougard <[email protected]>
canonical · Apr 18, 2024 · 88abe51 · 88abe51
1 parent 0cecf06
commit 88abe51
Show file tree

Hide file tree

Showing 3 changed files with 334 additions and 2 deletions.
diff --git a/microcloud/test/includes/microcloud.sh b/microcloud/test/includes/microcloud.sh
@@ -23,6 +23,8 @@ microcloud_interactive() {
   CEPH_WARNING=${CEPH_WARNING:-}                 # (yes/no) input for warning about eligible disk detection.
   CEPH_FILTER=${CEPH_FILTER:-}                   # filter string for CEPH disks.
   CEPH_WIPE=${CEPH_WIPE:-}                       # (yes/no) to wipe all disks.
+  CEPH_PUBLIC_NETWORK=${CEPH_PUBLIC_NETWORK:-}   # (default: MicroCloud internal subnet) input for setting up a public network.
+  CEPH_CLUSTER_NETWORK=${CEPH_CLUSTER_NETWORK:-} # (default: MicroCloud internal subnet or Ceph public network if specified previously) input for setting up a cluster network.
   SETUP_OVN=${SETUP_OVN:-}                       # (yes/no) input for initiating OVN network setup.
   OVN_WARNING=${OVN_WARNING:-}                   # (yes/no) input for warning about eligible interface detection.
   OVN_FILTER=${OVN_FILTER:-}                     # filter string for OVN interfaces.
@@ -74,6 +76,12 @@ EOF
 )
 fi
 
+setup=$(cat << EOF
+${setup}
+${CEPH_PUBLIC_NETWORK}                                  # setup public network
+${CEPH_CLUSTER_NETWORK}                                 # setup cluster network
+EOF
+)
 
 if [ -n "${SETUP_OVN}" ]; then
   setup=$(cat << EOF
@@ -152,6 +160,18 @@ validate_system_microceph() {
       shift 1
     fi
 
+    public_ceph_subnet=""
+    if echo "${1}" | grep -Pq '^([0-9]{1,3}\.){3}[0-9]{1,3}/([0-9]|[1-2][0-9]|3[0-2])$'; then
+      public_ceph_subnet="${1}"
+      shift 1
+    fi
+
+    cluster_ceph_subnet=""
+    if echo "${1}" | grep -Pq '^([0-9]{1,3}\.){3}[0-9]{1,3}/([0-9]|[1-2][0-9]|3[0-2])$'; then
+      cluster_ceph_subnet="${1}"
+      shift 1
+    fi
+
     disks="${*}"
 
     echo "==> ${name} Validating MicroCeph. Using disks: {${disks}}"
@@ -180,6 +200,18 @@ validate_system_microceph() {
         microceph.ceph osd pool get lxd_cephfs_data size
       " > /dev/null
     fi
+
+    if [ -n "${public_ceph_subnet}" ]; then
+      lxc exec "${name}" -- sh -ceu "
+        microceph.ceph config get mon public_network | grep -q ${public_ceph_subnet}
+      " > /dev/null
+    fi
+
+    if [ -n "${cluster_ceph_subnet}" ]; then
+      lxc exec "${name}" -- sh -ceu "
+        microceph.ceph config get mon cluster_network | grep -q ${cluster_ceph_subnet}
+      " > /dev/null
+    fi
 }
 
 # validate_system_microovn: Ensures the node with the given name has correctly set up MicroOVN with the given resources.
@@ -626,6 +658,26 @@ cluster_reset() {
   )
 }
 
+rand() {
+  od -An -N2 -i /dev/urandom | awk '{print $1}'
+}
+
+generate_ip() {
+  local num_hosts="$1"
+  local ip_num="$2"
+  local rand_host=$(( $(rand) % num_hosts + 1 ))
+  local new_ip_num=$(( ip_num + rand_host ))
+
+  local o1=$(( (new_ip_num / 256 / 256 / 256) % 256 ))
+  local o2=$(( (new_ip_num / 256 / 256) % 256 ))
+  local o3=$(( (new_ip_num / 256) % 256 ))
+  local o4=$(( new_ip_num % 256 ))
+  local new_ip="${o1}.${o2}.${o3}.${o4}"
+
+  echo "$new_ip"
+}
+
+
 # reset_systems: Concurrently or sequentially resets the specified number of systems.
 reset_systems() {
   if [ "${SNAPSHOT_RESTORE}" = 1 ]; then
@@ -652,6 +704,20 @@ reset_systems() {
     shift 1
   fi
 
+  public_ceph_subnet=""
+  if echo "${1}" | grep -Pq '^([0-9]{1,3}\.){3}[0-9]{1,3}/([0-9]|[1-2][0-9]|3[0-2])$'; then
+    public_ceph_subnet="${1}"
+    shift 1
+  fi
+
+  cluster_ceph_subnet=""
+  if echo "${1}" | grep -Pq '^([0-9]{1,3}\.){3}[0-9]{1,3}/([0-9]|[1-2][0-9]|3[0-2])$'; then
+    cluster_ceph_subnet="${1}"
+    shift 1
+  fi
+
+  used_ips=""
+
   for i in $(seq 1 "${num_vms}") ; do
     name=$(printf "micro%02d" "$i")
     if [ "$i" = 1 ]; then
@@ -663,6 +729,84 @@ reset_systems() {
     else
       reset_system "${name}" "${num_disks}" "${num_ifaces}"
     fi
+
+    if [ -n "${public_ceph_subnet}" ]; then
+      if [ "${num_ifaces}" -lt 3 ]; then
+        echo "Impossible to set a dedicated Ceph public network for less than 3 network interfaces."
+        false
+      fi
+
+      ip=$(echo "${public_ceph_subnet}" | cut -d '/' -f 1)
+      range=$(echo "${public_ceph_subnet}" | cut -d '/' -f 2)
+      IFS='.' read i1 i2 i3 i4 <<EOF
+$ip
+EOF
+
+      ip_num=$(( (i1 * 256 * 256 * 256) + (i2 * 256 * 256) + (i3 * 256) + i4 ))
+      host_bits=$(( 32 - range ))
+      pow=1
+      for _ in $(seq 1 "${host_bits}"); do
+        pow=$((pow * 2))
+      done
+
+      num_hosts=$(( pow - 2 ))
+
+      if [ "${num_vms}" -gt "${num_hosts}" ]; then
+        echo "Error: Not enough IPs available in the subnet."
+        false
+      fi
+
+      while :; do
+        new_ip=$(generate_ip "${num_hosts}" "${ip_num}")
+        if echo " ${used_ips} " | grep -q " ${new_ip} "; then
+          continue
+        fi
+
+        used_ips="${used_ips} ${new_ip} "
+        break
+      done
+
+      lxc exec "${name}" -- ip addr add "${new_ip}/${range}" dev enp7s0
+    fi
+
+    if [ -n "${cluster_ceph_subnet}" ] && [ "${cluster_ceph_subnet}" != "${public_ceph_subnet}" ]; then
+      if [ "${num_ifaces}" -lt 4 ]; then
+        echo "Impossible to set a dedicated Ceph cluster network for less than 3 network interfaces."
+        false
+      fi
+
+      ip=$(echo "${cluster_ceph_subnet}" | cut -d '/' -f 1)
+      range=$(echo "${cluster_ceph_subnet}" | cut -d '/' -f 2)
+      IFS='.' read i1 i2 i3 i4 <<EOF
+$ip
+EOF
+
+      ip_num=$(( (i1 * 256 * 256 * 256) + (i2 * 256 * 256) + (i3 * 256) + i4 ))
+      host_bits=$(( 32 - range ))
+      pow=1
+      for _ in $(seq 1 "${host_bits}"); do
+        pow=$((pow * 2))
+      done
+
+      num_hosts=$(( pow - 2 ))
+
+      if [ "${num_vms}" -gt "${num_hosts}" ]; then
+        echo "Error: Not enough IPs available in the subnet."
+        false
+      fi
+
+      while :; do
+        new_ip=$(generate_ip "${num_hosts}" "${ip_num}")
+        if echo " ${used_ips} " | grep -q " ${new_ip} "; then
+          continue
+        fi
+
+        used_ips="${used_ips} ${new_ip} "
+        break
+      done
+
+      lxc exec "${name}" -- ip addr add "${new_ip}/${range}" dev enp8s0
+    fi
   done
 
   # Pause any extra systems.

diff --git a/microcloud/test/suites/add.sh b/microcloud/test/suites/add.sh
@@ -99,7 +99,9 @@ test_add_auto() {
 }
 
 test_add_interactive() {
-  reset_systems 4 2 1
+  ceph_public_subnet="10.0.1.0/24"
+  ceph_cluster_subnet="10.0.2.0/24"
+  reset_systems 4 2 4 "${ceph_public_subnet}" "${ceph_cluster_subnet}"
 
   # Disable extra nodes so we don't add them yet.
   # shellcheck disable=SC2043
@@ -118,6 +120,8 @@ test_add_interactive() {
   export SETUP_CEPH="yes"
   export SETUP_CEPHFS="yes"
   export CEPH_WIPE="yes"
+  export CEPH_PUBLIC_NETWORK="${ceph_public_subnet}"
+  export CEPH_CLUSTER_NETWORK="${ceph_cluster_subnet}"
   export SETUP_OVN="yes"
   export OVN_FILTER="enp6s0"
   export IPV4_SUBNET="10.1.123.1/24"
@@ -150,7 +154,7 @@ test_add_interactive() {
 
   for m in micro01 micro02 micro03 micro04 ; do
     validate_system_lxd "${m}" 4 disk1 1 1 enp6s0 10.1.123.1/24 10.1.123.100-10.1.123.254 fd42:1:1234:1234::1/64  10.1.123.1,fd42:1:1234:1234::1
-    validate_system_microceph "${m}" 1 disk2
+    validate_system_microceph "${m}" 1 "${ceph_public_subnet}" "${ceph_cluster_subnet}" disk2
     validate_system_microovn "${m}"
   done