From ecdfb61d82037af4d2dc82fbd8dd03f6b34fe875 Mon Sep 17 00:00:00 2001 From: Gabriel Mougard Date: Thu, 1 Aug 2024 10:26:43 +0200 Subject: [PATCH] tests: Add OVN underlay tests Signed-off-by: Gabriel Mougard --- test/includes/microcloud.sh | 37 +++++++++++- test/suites/add.sh | 3 + test/suites/basic.sh | 109 ++++++++++++++++++++++++++++++++++++ 3 files changed, 146 insertions(+), 3 deletions(-) diff --git a/test/includes/microcloud.sh b/test/includes/microcloud.sh index f4a804e6e..38e396cb3 100644 --- a/test/includes/microcloud.sh +++ b/test/includes/microcloud.sh @@ -2,10 +2,10 @@ # unset_interactive_vars: Unsets all variables related to the test console. unset_interactive_vars() { - unset SKIP_LOOKUP LOOKUP_IFACE LIMIT_SUBNET SKIP_SERVICE EXPECT_PEERS REUSE_EXISTING REUSE_EXISTING_COUNT \ + unset SKIP_LOOKUP LOOKUP_IFACE LIMIT_SUBNET SKIP_SERVICE EXPECT_PEERS PEERS_FILTER REUSE_EXISTING REUSE_EXISTING_COUNT \ SETUP_ZFS ZFS_FILTER ZFS_WIPE \ SETUP_CEPH CEPH_MISSING_DISKS CEPH_FILTER CEPH_WIPE CEPH_ENCRYPT SETUP_CEPHFS CEPH_CLUSTER_NETWORK \ - PROCEED_WITH_NO_OVERLAY_NETWORKING SETUP_OVN OVN_WARNING OVN_FILTER IPV4_SUBNET IPV4_START IPV4_END DNS_ADDRESSES IPV6_SUBNET \ + PROCEED_WITH_NO_OVERLAY_NETWORKING SETUP_OVN OVN_UNDERLAY_NETWORK OVN_UNDERLAY_FILTER OVN_WARNING OVN_FILTER IPV4_SUBNET IPV4_START IPV4_END DNS_ADDRESSES IPV6_SUBNET \ REPLACE_PROFILE CEPH_RETRY_HA MULTI_NODE } @@ -27,6 +27,7 @@ microcloud_interactive() { LIMIT_SUBNET=${LIMIT_SUBNET:-} # (yes/no) input for limiting lookup of systems to the above subnet. SKIP_SERVICE=${SKIP_SERVICE:-} # (yes/no) input to skip any missing services. Should be unset if all services are installed. EXPECT_PEERS=${EXPECT_PEERS:-} # wait for this number of systems to be available to join the cluster. + PEERS_FILTER=${PEERS_FILTER:-} # filter string for the particular peer to init/add REUSE_EXISTING=${REUSE_EXISTING:-} # (yes/no) incorporate an existing clustered service. REUSE_EXISTING_COUNT=${REUSE_EXISTING_COUNT:-0} # (number) number of existing clusters to incorporate. SETUP_ZFS=${SETUP_ZFS:-} # (yes/no) input for initiating ZFS storage pool setup. @@ -48,6 +49,8 @@ microcloud_interactive() { IPV4_START=${IPV4_START:-} # OVN ipv4 range start. IPV4_END=${IPV4_END:-} # OVN ipv4 range end. DNS_ADDRESSES=${DNS_ADDRESSES:-} # OVN custom DNS addresses. + OVN_UNDERLAY_NETWORK=${OVN_UNDERLAY_NETWORK:-} # (yes/no) set up a custom OVN underlay network. + OVN_UNDERLAY_FILTER=${OVN_UNDERLAY_FILTER:-} # filter string for OVN underlay interfaces. IPV6_SUBNET=${IPV6_SUBNET:-} # OVN ipv6 range. REPLACE_PROFILE="${REPLACE_PROFILE:-}" # Replace default profile config and devices. @@ -67,6 +70,7 @@ $(true) ${LIMIT_SUBNET} # limit lookup subnet (yes/no) $([ "${SKIP_SERVICE}" = "yes" ] && printf "%s" "${SKIP_SERVICE}") # skip MicroOVN/MicroCeph (yes/no) expect ${EXPECT_PEERS} # wait until the systems show up +${PEERS_FILTER} # filter discovered peers select-all # select all the systems --- $(true) # workaround for set -e @@ -136,6 +140,17 @@ ${IPV6_SUBNET} ${DNS_ADDRESSES} $(true) # workaround for set -e " + + if [ -n "${OVN_UNDERLAY_NETWORK}" ]; then + setup="${setup} +${OVN_UNDERLAY_NETWORK} +$([ "${OVN_UNDERLAY_NETWORK}" = "yes" ] && printf "wait 300ms") +${OVN_UNDERLAY_FILTER} +$([ "${OVN_UNDERLAY_NETWORK}" = "yes" ] && printf "select-all") +$([ "${OVN_UNDERLAY_NETWORK}" = "yes" ] && printf -- "---") +$(true) # workaround for set -e +" + fi fi if [ -n "${REPLACE_PROFILE}" ] ; then @@ -297,13 +312,29 @@ validate_ceph_encrypt() { # validate_system_microovn: Ensures the node with the given name has correctly set up MicroOVN with the given resources. validate_system_microovn() { name=${1} + shift 1 echo "==> ${name} Validating MicroOVN" lxc remote switch local lxc exec "${name}" -- microovn cluster list | grep -q "${name}" -} + ovn_underlay_subnet_prefix="" + if [ $# -gt 0 ] && echo "${1}" | grep -Pq '^([0-9]{1,3}\.){2}[0-9]{1,3}$'; then + ovn_underlay_subnet_prefix="${1}" + shift 1 + fi + + # Instances are named micro01, micro02, etc. + # We need to extract the instance number to check the OVN Geneve tunnel IP. + instance_idx=$(echo "${name}" | grep -oE '[0-9]+$') + underlay_ip_idx=$((instance_idx + 1)) + if [ -n "${ovn_underlay_subnet_prefix}" ]; then + lxc exec "${name}" -- sh -ceu " + microovn.ovn-sbctl --columns=ip,type find Encap type=geneve | grep -q ${ovn_underlay_subnet_prefix}.${underlay_ip_idx} + " > /dev/null + fi +} # validate_system_lxd_zfs: Ensures the node with the given name has the given disk set up for ZFS storage. validate_system_lxd_zfs() { name=${1} diff --git a/test/suites/add.sh b/test/suites/add.sh index 9e57cecd6..2ad5d299d 100644 --- a/test/suites/add.sh +++ b/test/suites/add.sh @@ -138,6 +138,7 @@ test_add_interactive() { export IPV4_END="10.1.123.254" export IPV6_SUBNET="fd42:1:1234:1234::1/64" export DNS_ADDRESSES="10.1.123.1,fd42:1:1234:1234::1" + export OVN_UNDERLAY_NETWORK="no" microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q @@ -159,6 +160,7 @@ test_add_interactive() { export CEPH_ENCRYPT="no" export SETUP_OVN="yes" export OVN_FILTER="enp6s0" + export OVN_UNDERLAY_NETWORK="no" microcloud_interactive | lxc exec micro01 -- sh -c "microcloud add > out" lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q @@ -237,6 +239,7 @@ test_add_interactive() { export IPV6_SUBNET="fd42:1:1234:1234::1/64" export DNS_ADDRESSES="10.1.123.1,fd42:1:1234:1234::1" export REPLACE_PROFILE="yes" + export OVN_UNDERLAY_NETWORK="no" lxc exec micro04 -- snap enable microcloud lxc exec micro04 -- snap start microcloud diff --git a/test/suites/basic.sh b/test/suites/basic.sh index 55fa8d512..7da15316b 100644 --- a/test/suites/basic.sh +++ b/test/suites/basic.sh @@ -137,6 +137,7 @@ test_interactive() { export IPV6_SUBNET="fd42:1:1234:1234::1/64" export DNS_ADDRESSES="10.1.123.1,8.8.8.8" export CEPH_CLUSTER_NETWORK="${microcloud_internal_net_addr}" + export OVN_UNDERLAY_NETWORK="no" microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q @@ -207,6 +208,7 @@ test_interactive() { export IPV4_SUBNET="10.1.123.1/24" export IPV4_START="10.1.123.100" export IPV4_END="10.1.123.254" + export OVN_UNDERLAY_NETWORK="no" microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q @@ -215,6 +217,108 @@ test_interactive() { validate_system_microceph "${m}" 1 "${CEPH_CLUSTER_NETWORK}" disk2 validate_system_microovn "${m}" done + + reset_systems 3 3 3 + + ceph_cluster_subnet_prefix="10.2.123" + ceph_cluster_subnet_iface="enp7s0" + + for n in $(seq 2 4); do + cluster_ip="${ceph_cluster_subnet_prefix}.${n}/24" + lxc exec "micro0$((n-1))" -- ip addr add "${cluster_ip}" dev "${ceph_cluster_subnet_iface}" + done + + ovn_underlay_subnet_prefix="10.3.123" + ovn_underlay_subnet_iface="enp8s0" + + for n in $(seq 2 4); do + ovn_underlay_ip="${ovn_underlay_subnet_prefix}.${n}/24" + lxc exec "micro0$((n-1))" -- sh -c "ip addr add ${ovn_underlay_ip} dev ${ovn_underlay_subnet_iface} && ip link set ${ovn_underlay_subnet_iface} up" + done + + echo "Creating a MicroCloud with ZFS, Ceph storage with a fully disaggregated Ceph networking setup, OVN management network and OVN underlay network" + export CEPH_CLUSTER_NETWORK="${ceph_cluster_subnet_prefix}.0/24" + export OVN_UNDERLAY_NETWORK="yes" + export OVN_UNDERLAY_FILTER="${ovn_underlay_subnet_prefix}" + microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" + + lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q + for m in micro01 micro02 micro03 ; do + validate_system_lxd "${m}" 3 disk1 3 1 "${OVN_FILTER}" "${IPV4_SUBNET}" "${IPV4_START}"-"${IPV4_END}" "${IPV6_SUBNET}" + validate_system_microceph "${m}" 1 "${CEPH_CLUSTER_NETWORK}" disk2 + validate_system_microovn "${m}" "${ovn_underlay_subnet_prefix}" + done + + reset_systems 3 3 2 + echo "Add a MicroCloud node on a cluster of 2 nodes with MicroOVN not initialized" + + ovn_underlay_subnet_prefix="10.2.123" + ovn_underlay_subnet_iface="enp7s0" + + for n in $(seq 2 4); do + ovn_underlay_ip="${ovn_underlay_subnet_prefix}.${n}/24" + lxc exec "micro0$((n-1))" -- sh -c "ip addr add ${ovn_underlay_ip} dev ${ovn_underlay_subnet_iface} && ip link set ${ovn_underlay_subnet_iface} up" + done + + lxc exec micro03 -- snap disable microcloud || true + for m in micro01 micro02 ; do + lxc exec "${m}" -- snap disable microovn + done + + unset_interactive_vars + microcloud_internal_net_addr="$(ip_config_to_netaddr lxdbr0)" + export MULTI_NODE="yes" + export LOOKUP_IFACE="enp5s0" + export LIMIT_SUBNET="yes" + export EXPECT_PEERS=1 + export SKIP_SERVICE="yes" + export SETUP_ZFS="yes" + export ZFS_FILTER="lxd_disk1" + export ZFS_WIPE="yes" + export SETUP_CEPH="no" + export SETUP_OVN="no" + + # Run a 2 nodes MicroCloud without MicroOVN first. + microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" + + lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q + for m in micro01 micro02 ; do + lxc exec "${m}" -- snap enable microovn + done + + lxc exec micro03 -- microovn cluster bootstrap + lxc exec micro03 -- snap enable microcloud + lxc exec micro03 -- snap start microcloud + + unset_interactive_vars + export LIMIT_SUBNET="yes" + export EXPECT_PEERS=2 + export PEERS_FILTER="micro03" + export REUSE_EXISTING_COUNT=1 + export REUSE_EXISTING="add" + export SETUP_ZFS="yes" + export ZFS_FILTER="lxd_disk1" + export ZFS_WIPE="yes" + export SETUP_CEPH="no" + export SETUP_OVN="yes" + export OVN_FILTER="enp6s0" + export OVN_UNDERLAY_NETWORK="yes" + export OVN_UNDERLAY_FILTER="${ovn_underlay_subnet_prefix}" + export IPV4_SUBNET="10.1.123.1/24" + export IPV4_START="10.1.123.100" + export IPV4_END="10.1.123.254" + export DNS_ADDRESSES="10.1.123.1,8.8.8.8" + export IPV6_SUBNET="fd42:1:1234:1234::1/64" + export REPLACE_PROFILE="yes" + microcloud_interactive | lxc exec micro01 -- sh -c "microcloud add > out" + + for m in micro01 micro02 micro03 ; do + validate_system_lxd "${m}" 3 disk1 0 0 "${OVN_FILTER}" "${IPV4_SUBNET}" "${IPV4_START}"-"${IPV4_END}" "${IPV6_SUBNET}" + done + + for m in micro01 micro02 ; do + validate_system_microovn "${m}" "${ovn_underlay_subnet_prefix}" + done } test_instances_config() { @@ -666,6 +770,7 @@ _test_case() { export LOOKUP_IFACE="enp5s0" # filter string for the lookup interface table. export LIMIT_SUBNET="yes" # (yes/no) input for limiting lookup of systems to the above subnet. export CEPH_CLUSTER_NETWORK="${microcloud_internal_net_addr}" + export OVN_UNDERLAY_NETWORK="no" export EXPECT_PEERS="$((num_systems - 1))" @@ -1082,6 +1187,7 @@ test_reuse_cluster() { export IPV4_END="10.1.123.254" export DNS_ADDRESSES="10.1.123.1,8.8.8.8" export IPV6_SUBNET="fd42:1:1234:1234::1/64" + export OVN_UNDERLAY_NETWORK="no" reset_systems 3 3 3 echo "Create a MicroCloud that re-uses an existing service" @@ -1210,6 +1316,7 @@ test_remove_cluster_member() { export IPV4_END="10.1.123.254" export DNS_ADDRESSES="10.1.123.1,8.8.8.8" export IPV6_SUBNET="fd42:1:1234:1234::1/64" + export OVN_UNDERLAY_NETWORK="no" reset_systems 3 3 3 echo "Fail to remove member from MicroCeph and LXD until OSDs are removed" @@ -1358,6 +1465,7 @@ test_add_services() { export DNS_ADDRESSES="10.1.123.1,8.8.8.8" export IPV6_SUBNET="fd42:1:1234:1234::1/64" export CEPH_CLUSTER_NETWORK="${ceph_cluster_subnet_prefix}.0/24" + export OVN_UNDERLAY_NETWORK="no" reset_systems 3 3 3 set_cluster_subnet 3 "${ceph_cluster_subnet_iface}" "${ceph_cluster_subnet_prefix}" @@ -1495,6 +1603,7 @@ test_non_ha() { export CEPH_ENCRYPT="no" export CEPH_RETRY_HA="no" export SETUP_OVN="yes" + export OVN_UNDERLAY_NETWORK="no" export OVN_FILTER="enp6s0" export IPV4_SUBNET="10.1.123.1/24" export IPV4_START="10.1.123.100"