Skip to content

Commit

Permalink
tests: Add OVN underlay tests
Browse files Browse the repository at this point in the history 
Signed-off-by: Gabriel Mougard <[email protected]>
  • Loading branch information
@gabrielmougard
gabrielmougard committed Sep 5, 2024
1 parent 468b4a4 commit ecdfb61
Show file tree
Hide file tree
Showing 3 changed files with 146 additions and 3 deletions.
37 changes: 34 additions & 3 deletions test/includes/microcloud.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,10 @@

# unset_interactive_vars: Unsets all variables related to the test console.
unset_interactive_vars() {
unset SKIP_LOOKUP LOOKUP_IFACE LIMIT_SUBNET SKIP_SERVICE EXPECT_PEERS REUSE_EXISTING REUSE_EXISTING_COUNT \
unset SKIP_LOOKUP LOOKUP_IFACE LIMIT_SUBNET SKIP_SERVICE EXPECT_PEERS PEERS_FILTER REUSE_EXISTING REUSE_EXISTING_COUNT \
SETUP_ZFS ZFS_FILTER ZFS_WIPE \
SETUP_CEPH CEPH_MISSING_DISKS CEPH_FILTER CEPH_WIPE CEPH_ENCRYPT SETUP_CEPHFS CEPH_CLUSTER_NETWORK \
PROCEED_WITH_NO_OVERLAY_NETWORKING SETUP_OVN OVN_WARNING OVN_FILTER IPV4_SUBNET IPV4_START IPV4_END DNS_ADDRESSES IPV6_SUBNET \
PROCEED_WITH_NO_OVERLAY_NETWORKING SETUP_OVN OVN_UNDERLAY_NETWORK OVN_UNDERLAY_FILTER OVN_WARNING OVN_FILTER IPV4_SUBNET IPV4_START IPV4_END DNS_ADDRESSES IPV6_SUBNET \
REPLACE_PROFILE CEPH_RETRY_HA MULTI_NODE
}

Expand All @@ -27,6 +27,7 @@ microcloud_interactive() {
LIMIT_SUBNET=${LIMIT_SUBNET:-} # (yes/no) input for limiting lookup of systems to the above subnet.
SKIP_SERVICE=${SKIP_SERVICE:-} # (yes/no) input to skip any missing services. Should be unset if all services are installed.
EXPECT_PEERS=${EXPECT_PEERS:-} # wait for this number of systems to be available to join the cluster.
PEERS_FILTER=${PEERS_FILTER:-} # filter string for the particular peer to init/add
REUSE_EXISTING=${REUSE_EXISTING:-} # (yes/no) incorporate an existing clustered service.
REUSE_EXISTING_COUNT=${REUSE_EXISTING_COUNT:-0} # (number) number of existing clusters to incorporate.
SETUP_ZFS=${SETUP_ZFS:-} # (yes/no) input for initiating ZFS storage pool setup.
Expand All @@ -48,6 +49,8 @@ microcloud_interactive() {
IPV4_START=${IPV4_START:-} # OVN ipv4 range start.
IPV4_END=${IPV4_END:-} # OVN ipv4 range end.
DNS_ADDRESSES=${DNS_ADDRESSES:-} # OVN custom DNS addresses.
OVN_UNDERLAY_NETWORK=${OVN_UNDERLAY_NETWORK:-} # (yes/no) set up a custom OVN underlay network.
OVN_UNDERLAY_FILTER=${OVN_UNDERLAY_FILTER:-} # filter string for OVN underlay interfaces.
IPV6_SUBNET=${IPV6_SUBNET:-} # OVN ipv6 range.
REPLACE_PROFILE="${REPLACE_PROFILE:-}" # Replace default profile config and devices.

Expand All @@ -67,6 +70,7 @@ $(true)
${LIMIT_SUBNET} # limit lookup subnet (yes/no)
$([ "${SKIP_SERVICE}" = "yes" ] && printf "%s" "${SKIP_SERVICE}") # skip MicroOVN/MicroCeph (yes/no)
expect ${EXPECT_PEERS} # wait until the systems show up
${PEERS_FILTER} # filter discovered peers
select-all # select all the systems
---
$(true) # workaround for set -e
Expand Down Expand Up @@ -136,6 +140,17 @@ ${IPV6_SUBNET}
${DNS_ADDRESSES}
$(true) # workaround for set -e
"

if [ -n "${OVN_UNDERLAY_NETWORK}" ]; then
setup="${setup}
${OVN_UNDERLAY_NETWORK}
$([ "${OVN_UNDERLAY_NETWORK}" = "yes" ] && printf "wait 300ms")
${OVN_UNDERLAY_FILTER}
$([ "${OVN_UNDERLAY_NETWORK}" = "yes" ] && printf "select-all")
$([ "${OVN_UNDERLAY_NETWORK}" = "yes" ] && printf -- "---")
$(true) # workaround for set -e
"
fi
fi

if [ -n "${REPLACE_PROFILE}" ] ; then
Expand Down Expand Up @@ -297,13 +312,29 @@ validate_ceph_encrypt() {
# validate_system_microovn: Ensures the node with the given name has correctly set up MicroOVN with the given resources.
validate_system_microovn() {
name=${1}
shift 1

echo "==> ${name} Validating MicroOVN"

lxc remote switch local
lxc exec "${name}" -- microovn cluster list | grep -q "${name}"
}

ovn_underlay_subnet_prefix=""
if [ $# -gt 0 ] && echo "${1}" | grep -Pq '^([0-9]{1,3}\.){2}[0-9]{1,3}$'; then
ovn_underlay_subnet_prefix="${1}"
shift 1
fi

# Instances are named micro01, micro02, etc.
# We need to extract the instance number to check the OVN Geneve tunnel IP.
instance_idx=$(echo "${name}" | grep -oE '[0-9]+$')
underlay_ip_idx=$((instance_idx + 1))
if [ -n "${ovn_underlay_subnet_prefix}" ]; then
lxc exec "${name}" -- sh -ceu "
microovn.ovn-sbctl --columns=ip,type find Encap type=geneve | grep -q ${ovn_underlay_subnet_prefix}.${underlay_ip_idx}
" > /dev/null
fi
}
# validate_system_lxd_zfs: Ensures the node with the given name has the given disk set up for ZFS storage.
validate_system_lxd_zfs() {
name=${1}
Expand Down
3 changes: 3 additions & 0 deletions test/suites/add.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,7 @@ test_add_interactive() {
export IPV4_END="10.1.123.254"
export IPV6_SUBNET="fd42:1:1234:1234::1/64"
export DNS_ADDRESSES="10.1.123.1,fd42:1:1234:1234::1"
export OVN_UNDERLAY_NETWORK="no"
microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out"
lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q

Expand All @@ -159,6 +160,7 @@ test_add_interactive() {
export CEPH_ENCRYPT="no"
export SETUP_OVN="yes"
export OVN_FILTER="enp6s0"
export OVN_UNDERLAY_NETWORK="no"
microcloud_interactive | lxc exec micro01 -- sh -c "microcloud add > out"
lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q

Expand Down Expand Up @@ -237,6 +239,7 @@ test_add_interactive() {
export IPV6_SUBNET="fd42:1:1234:1234::1/64"
export DNS_ADDRESSES="10.1.123.1,fd42:1:1234:1234::1"
export REPLACE_PROFILE="yes"
export OVN_UNDERLAY_NETWORK="no"

lxc exec micro04 -- snap enable microcloud
lxc exec micro04 -- snap start microcloud
Expand Down
109 changes: 109 additions & 0 deletions test/suites/basic.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,7 @@ test_interactive() {
export IPV6_SUBNET="fd42:1:1234:1234::1/64"
export DNS_ADDRESSES="10.1.123.1,8.8.8.8"
export CEPH_CLUSTER_NETWORK="${microcloud_internal_net_addr}"
export OVN_UNDERLAY_NETWORK="no"
microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out"

lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q
Expand Down Expand Up @@ -207,6 +208,7 @@ test_interactive() {
export IPV4_SUBNET="10.1.123.1/24"
export IPV4_START="10.1.123.100"
export IPV4_END="10.1.123.254"
export OVN_UNDERLAY_NETWORK="no"
microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out"

lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q
Expand All @@ -215,6 +217,108 @@ test_interactive() {
validate_system_microceph "${m}" 1 "${CEPH_CLUSTER_NETWORK}" disk2
validate_system_microovn "${m}"
done

reset_systems 3 3 3

ceph_cluster_subnet_prefix="10.2.123"
ceph_cluster_subnet_iface="enp7s0"

for n in $(seq 2 4); do
cluster_ip="${ceph_cluster_subnet_prefix}.${n}/24"
lxc exec "micro0$((n-1))" -- ip addr add "${cluster_ip}" dev "${ceph_cluster_subnet_iface}"
done

ovn_underlay_subnet_prefix="10.3.123"
ovn_underlay_subnet_iface="enp8s0"

for n in $(seq 2 4); do
ovn_underlay_ip="${ovn_underlay_subnet_prefix}.${n}/24"
lxc exec "micro0$((n-1))" -- sh -c "ip addr add ${ovn_underlay_ip} dev ${ovn_underlay_subnet_iface} && ip link set ${ovn_underlay_subnet_iface} up"
done

echo "Creating a MicroCloud with ZFS, Ceph storage with a fully disaggregated Ceph networking setup, OVN management network and OVN underlay network"
export CEPH_CLUSTER_NETWORK="${ceph_cluster_subnet_prefix}.0/24"
export OVN_UNDERLAY_NETWORK="yes"
export OVN_UNDERLAY_FILTER="${ovn_underlay_subnet_prefix}"
microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out"

lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q
for m in micro01 micro02 micro03 ; do
validate_system_lxd "${m}" 3 disk1 3 1 "${OVN_FILTER}" "${IPV4_SUBNET}" "${IPV4_START}"-"${IPV4_END}" "${IPV6_SUBNET}"
validate_system_microceph "${m}" 1 "${CEPH_CLUSTER_NETWORK}" disk2
validate_system_microovn "${m}" "${ovn_underlay_subnet_prefix}"
done

reset_systems 3 3 2
echo "Add a MicroCloud node on a cluster of 2 nodes with MicroOVN not initialized"

ovn_underlay_subnet_prefix="10.2.123"
ovn_underlay_subnet_iface="enp7s0"

for n in $(seq 2 4); do
ovn_underlay_ip="${ovn_underlay_subnet_prefix}.${n}/24"
lxc exec "micro0$((n-1))" -- sh -c "ip addr add ${ovn_underlay_ip} dev ${ovn_underlay_subnet_iface} && ip link set ${ovn_underlay_subnet_iface} up"
done

lxc exec micro03 -- snap disable microcloud || true
for m in micro01 micro02 ; do
lxc exec "${m}" -- snap disable microovn
done

unset_interactive_vars
microcloud_internal_net_addr="$(ip_config_to_netaddr lxdbr0)"
export MULTI_NODE="yes"
export LOOKUP_IFACE="enp5s0"
export LIMIT_SUBNET="yes"
export EXPECT_PEERS=1
export SKIP_SERVICE="yes"
export SETUP_ZFS="yes"
export ZFS_FILTER="lxd_disk1"
export ZFS_WIPE="yes"
export SETUP_CEPH="no"
export SETUP_OVN="no"

# Run a 2 nodes MicroCloud without MicroOVN first.
microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out"

lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q
for m in micro01 micro02 ; do
lxc exec "${m}" -- snap enable microovn
done

lxc exec micro03 -- microovn cluster bootstrap
lxc exec micro03 -- snap enable microcloud
lxc exec micro03 -- snap start microcloud

unset_interactive_vars
export LIMIT_SUBNET="yes"
export EXPECT_PEERS=2
export PEERS_FILTER="micro03"
export REUSE_EXISTING_COUNT=1
export REUSE_EXISTING="add"
export SETUP_ZFS="yes"
export ZFS_FILTER="lxd_disk1"
export ZFS_WIPE="yes"
export SETUP_CEPH="no"
export SETUP_OVN="yes"
export OVN_FILTER="enp6s0"
export OVN_UNDERLAY_NETWORK="yes"
export OVN_UNDERLAY_FILTER="${ovn_underlay_subnet_prefix}"
export IPV4_SUBNET="10.1.123.1/24"
export IPV4_START="10.1.123.100"
export IPV4_END="10.1.123.254"
export DNS_ADDRESSES="10.1.123.1,8.8.8.8"
export IPV6_SUBNET="fd42:1:1234:1234::1/64"
export REPLACE_PROFILE="yes"
microcloud_interactive | lxc exec micro01 -- sh -c "microcloud add > out"

for m in micro01 micro02 micro03 ; do
validate_system_lxd "${m}" 3 disk1 0 0 "${OVN_FILTER}" "${IPV4_SUBNET}" "${IPV4_START}"-"${IPV4_END}" "${IPV6_SUBNET}"
done

for m in micro01 micro02 ; do
validate_system_microovn "${m}" "${ovn_underlay_subnet_prefix}"
done
}

test_instances_config() {
Expand Down Expand Up @@ -666,6 +770,7 @@ _test_case() {
export LOOKUP_IFACE="enp5s0" # filter string for the lookup interface table.
export LIMIT_SUBNET="yes" # (yes/no) input for limiting lookup of systems to the above subnet.
export CEPH_CLUSTER_NETWORK="${microcloud_internal_net_addr}"
export OVN_UNDERLAY_NETWORK="no"

export EXPECT_PEERS="$((num_systems - 1))"

Expand Down Expand Up @@ -1082,6 +1187,7 @@ test_reuse_cluster() {
export IPV4_END="10.1.123.254"
export DNS_ADDRESSES="10.1.123.1,8.8.8.8"
export IPV6_SUBNET="fd42:1:1234:1234::1/64"
export OVN_UNDERLAY_NETWORK="no"

reset_systems 3 3 3
echo "Create a MicroCloud that re-uses an existing service"
Expand Down Expand Up @@ -1210,6 +1316,7 @@ test_remove_cluster_member() {
export IPV4_END="10.1.123.254"
export DNS_ADDRESSES="10.1.123.1,8.8.8.8"
export IPV6_SUBNET="fd42:1:1234:1234::1/64"
export OVN_UNDERLAY_NETWORK="no"

reset_systems 3 3 3
echo "Fail to remove member from MicroCeph and LXD until OSDs are removed"
Expand Down Expand Up @@ -1358,6 +1465,7 @@ test_add_services() {
export DNS_ADDRESSES="10.1.123.1,8.8.8.8"
export IPV6_SUBNET="fd42:1:1234:1234::1/64"
export CEPH_CLUSTER_NETWORK="${ceph_cluster_subnet_prefix}.0/24"
export OVN_UNDERLAY_NETWORK="no"

reset_systems 3 3 3
set_cluster_subnet 3 "${ceph_cluster_subnet_iface}" "${ceph_cluster_subnet_prefix}"
Expand Down Expand Up @@ -1495,6 +1603,7 @@ test_non_ha() {
export CEPH_ENCRYPT="no"
export CEPH_RETRY_HA="no"
export SETUP_OVN="yes"
export OVN_UNDERLAY_NETWORK="no"
export OVN_FILTER="enp6s0"
export IPV4_SUBNET="10.1.123.1/24"
export IPV4_START="10.1.123.100"
Expand Down

0 comments on commit ecdfb61

Please sign in to comment.