From cdcb9956a1488ce55ee0f0342c1aede9faed7469 Mon Sep 17 00:00:00 2001 From: Gabriel Mougard Date: Mon, 2 Sep 2024 15:58:05 +0200 Subject: [PATCH] debug Signed-off-by: Gabriel Mougard --- cmd/microcloud/ask.go | 4 +- cmd/microcloud/asker.go | 231 ++++++++++++++++++++++++ cmd/microcloud/main.go | 7 +- cmd/microcloud/main_init.go | 3 +- cmd/microcloud/test_console.go | 5 +- test/suites/add.sh | 2 - test/suites/basic.sh | 316 ++++++++++++++++++--------------- 7 files changed, 417 insertions(+), 151 deletions(-) create mode 100644 cmd/microcloud/asker.go diff --git a/cmd/microcloud/ask.go b/cmd/microcloud/ask.go index a270a8026..2403b4165 100644 --- a/cmd/microcloud/ask.go +++ b/cmd/microcloud/ask.go @@ -1105,8 +1105,8 @@ func (c *initConfig) askOVNNetwork(sh *service.Handler) error { } canOVNUnderlay := true - for peer, state := range c.state { - if len(state.AvailableOVNInterfaces) == 0 { + for peer, system := range c.systems { + if len(c.state[system.ServerInfo.Name].AvailableOVNInterfaces) == 0 { fmt.Printf("Not enough interfaces available on %s to create an underlay network, skipping\n", peer) canOVNUnderlay = false break diff --git a/cmd/microcloud/asker.go b/cmd/microcloud/asker.go new file mode 100644 index 000000000..46fea8048 --- /dev/null +++ b/cmd/microcloud/asker.go @@ -0,0 +1,231 @@ +package main + +import ( + "bufio" + "fmt" + "log" + "os" + "strconv" + "strings" + + "golang.org/x/term" +) + +func ValueInSlice[T comparable](key T, list []T) bool { + for _, entry := range list { + if entry == key { + return true + } + } + + return false +} + +// Asker holds a reader for reading input into CLI questions. +type Asker struct { + reader *bufio.Reader + logger *log.Logger +} + +// NewAsker creates a new Asker instance that reads from the given reader. +// It can also be configured with a logger to help during the debug process. +func NewAsker(reader *bufio.Reader, logger *log.Logger) Asker { + return Asker{reader: reader, logger: logger} +} + +// AskBool asks a question and expect a yes/no answer. +func (a *Asker) AskBool(question string, defaultAnswer string) (bool, error) { + for { + answer, err := a.askQuestion(question, defaultAnswer) + if err != nil { + if a.logger != nil { + a.logger.Printf("Failed to read answer (%v) for question (%v). err : %v", answer, question, err) + } + + return false, err + } + + if ValueInSlice(strings.ToLower(answer), []string{"yes", "y"}) { + return true, nil + } else if ValueInSlice(strings.ToLower(answer), []string{"no", "n"}) { + return false, nil + } + + a.invalidInput(question, answer) + } +} + +// AskChoice asks the user to select one of multiple options. +func (a *Asker) AskChoice(question string, choices []string, defaultAnswer string) (string, error) { + for { + answer, err := a.askQuestion(question, defaultAnswer) + if err != nil { + if a.logger != nil { + a.logger.Printf("Failed to read answer (%v) for question (%v). err : %v", answer, question, err) + } + + return "", err + } + + if ValueInSlice(answer, choices) { + return answer, nil + } else if a.logger != nil { + a.logger.Printf("Answer (%v) not among available choices (%v))", answer, choices) + } + + a.invalidInput(question, answer) + } +} + +// AskInt asks the user to enter an integer between a min and max value. +func (a *Asker) AskInt(question string, min int64, max int64, defaultAnswer string, validate func(int64) error) (int64, error) { + for { + answer, err := a.askQuestion(question, defaultAnswer) + if err != nil { + if a.logger != nil { + a.logger.Printf("Failed to read answer (%v) for question (%v). err : %v", answer, question, err) + } + + return -1, err + } + + result, err := strconv.ParseInt(answer, 10, 64) + if err != nil { + if a.logger != nil { + a.logger.Printf("Invalid answer (%v) for question (%v). err : %v", answer, question, err) + } + + fmt.Fprintf(os.Stderr, "Invalid input: %v\n\n", err) + continue + } + + if !((min == -1 || result >= min) && (max == -1 || result <= max)) { + if a.logger != nil { + a.logger.Printf("Answer (%v) out of range for question (%v)", answer, question) + } + + fmt.Fprintf(os.Stderr, "Invalid input: out of range\n\n") + continue + } + + if validate != nil { + err = validate(result) + if err != nil { + if a.logger != nil { + a.logger.Printf("Invalid answer (%v) for question (%v). err : %v", answer, question, err) + } + + fmt.Fprintf(os.Stderr, "Invalid input: %v\n\n", err) + continue + } + } + + return result, err + } +} + +// AskString asks the user to enter a string, which optionally +// conforms to a validation function. +func (a *Asker) AskString(question string, defaultAnswer string, validate func(string) error) (string, error) { + for { + answer, err := a.askQuestion(question, defaultAnswer) + if err != nil { + if a.logger != nil { + a.logger.Printf("Failed to read answer (%v) for question (%v). err : %v", answer, question, err) + } + + return "", err + } + + if validate != nil { + err = validate(answer) + if err != nil { + if a.logger != nil { + a.logger.Printf("Invalid answer (%v) for question (%v). err : %v", answer, question, err) + } + + fmt.Fprintf(os.Stderr, "Invalid input: %v\n\n", err) + continue + } + + return answer, err + } + + if len(answer) != 0 { + return answer, err + } + + a.invalidInput(question, answer) + } +} + +// AskPassword asks the user to enter a password. +func (a *Asker) AskPassword(question string) string { + for { + fmt.Print(question) + + pwd, _ := term.ReadPassword(0) + fmt.Println("") + inFirst := string(pwd) + inFirst = strings.TrimSuffix(inFirst, "\n") + + fmt.Print("Again: ") + pwd, _ = term.ReadPassword(0) + fmt.Println("") + inSecond := string(pwd) + inSecond = strings.TrimSuffix(inSecond, "\n") + + // refuse empty password or if password inputs do not match + if len(inFirst) > 0 && inFirst == inSecond { + return inFirst + } + + a.invalidInput(question, "*****") + } +} + +// AskPasswordOnce asks the user to enter a password. +// +// It's the same as AskPassword, but it won't ask to enter it again. +func (a *Asker) AskPasswordOnce(question string) string { + for { + fmt.Print(question) + pwd, _ := term.ReadPassword(0) + fmt.Println("") + + // refuse empty password + spwd := string(pwd) + if len(spwd) > 0 { + return spwd + } + + a.invalidInput(question, "*****") + } +} + +// Ask a question on the output stream and read the answer from the input stream. +func (a *Asker) askQuestion(question, defaultAnswer string) (string, error) { + fmt.Print(question) + + return a.readAnswer(defaultAnswer) +} + +// Read the user's answer from the input stream, trimming newline and providing a default. +func (a *Asker) readAnswer(defaultAnswer string) (string, error) { + answer, err := a.reader.ReadString('\n') + answer = strings.TrimSpace(strings.TrimSuffix(answer, "\n")) + if answer == "" { + answer = defaultAnswer + } + + return answer, err +} + +// Print an invalid input message on the error stream. +func (a *Asker) invalidInput(question string, answer string) { + if a.logger != nil { + a.logger.Printf("Invalid answer (%v) for question (%v)", answer, question) + } + + fmt.Fprintf(os.Stderr, "Invalid input, try again.\n\n") +} diff --git a/cmd/microcloud/main.go b/cmd/microcloud/main.go index 4244e77a2..fdb620abd 100644 --- a/cmd/microcloud/main.go +++ b/cmd/microcloud/main.go @@ -4,9 +4,9 @@ package main import ( "bufio" "fmt" + "log" "os" - cli "github.com/canonical/lxd/shared/cmd" "github.com/spf13/cobra" "github.com/canonical/microcloud/microcloud/version" @@ -23,7 +23,7 @@ type CmdControl struct { FlagLogVerbose bool FlagMicroCloudDir string - asker cli.Asker + asker Asker } func main() { @@ -34,7 +34,8 @@ func main() { } // common flags. - commonCmd := CmdControl{asker: cli.NewAsker(bufio.NewReader(os.Stdin))} + logger := log.New(os.Stdout, "logger:", log.LstdFlags) + commonCmd := CmdControl{asker: NewAsker(bufio.NewReader(os.Stdin), logger)} useTestConsole := os.Getenv("TEST_CONSOLE") if useTestConsole == "1" { diff --git a/cmd/microcloud/main_init.go b/cmd/microcloud/main_init.go index 25d8d1613..8b2d13778 100644 --- a/cmd/microcloud/main_init.go +++ b/cmd/microcloud/main_init.go @@ -12,7 +12,6 @@ import ( "github.com/canonical/lxd/lxd/util" "github.com/canonical/lxd/shared" lxdAPI "github.com/canonical/lxd/shared/api" - cli "github.com/canonical/lxd/shared/cmd" "github.com/canonical/lxd/shared/logger" "github.com/canonical/lxd/shared/revert" "github.com/canonical/lxd/shared/validate" @@ -70,7 +69,7 @@ type initConfig struct { common *CmdControl // asker is the CLI user input helper. - asker *cli.Asker + asker *Asker // address is the cluster address of the local system. address string diff --git a/cmd/microcloud/test_console.go b/cmd/microcloud/test_console.go index 486b10444..25280a332 100644 --- a/cmd/microcloud/test_console.go +++ b/cmd/microcloud/test_console.go @@ -12,7 +12,6 @@ import ( "github.com/AlecAivazis/survey/v2/terminal" "github.com/Netflix/go-expect" - cli "github.com/canonical/lxd/shared/cmd" "github.com/creack/pty" "github.com/hinshun/vt10x" ) @@ -25,7 +24,7 @@ type testConsole struct { } // prepareTestAsker removes comments from the lines read from the given reader, and assigns them to the test console reader. -func prepareTestAsker(r io.Reader) cli.Asker { +func prepareTestAsker(r io.Reader) Asker { sc := bufio.NewScanner(r) b := bytes.Buffer{} for sc.Scan() { @@ -39,7 +38,7 @@ func prepareTestAsker(r io.Reader) cli.Asker { reader = bufio.NewReader(bytes.NewReader(b.Bytes())) - return cli.NewAsker(reader) + return NewAsker(reader, nil) } // NewTestConsole creates a new testConsole, with an underlying expect.Console and virtual terminal. diff --git a/test/suites/add.sh b/test/suites/add.sh index aeb1dc069..2ad5d299d 100644 --- a/test/suites/add.sh +++ b/test/suites/add.sh @@ -193,7 +193,6 @@ test_add_interactive() { export SETUP_ZFS="yes" export ZFS_FILTER="lxd_disk1" export ZFS_WIPE="yes" - export OVN_UNDERLAY_NETWORK="no" microcloud_interactive | lxc exec micro01 -- sh -c "microcloud add > out" lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q @@ -211,7 +210,6 @@ test_add_interactive() { export SETUP_ZFS="no" export SETUP_CEPH="no" export SETUP_OVN="no" - export OVN_UNDERLAY_NETWORK="no" lxc exec micro04 -- snap disable microcloud microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" diff --git a/test/suites/basic.sh b/test/suites/basic.sh index 27d61fc57..d6d03ad63 100644 --- a/test/suites/basic.sh +++ b/test/suites/basic.sh @@ -1,11 +1,11 @@ #!/bin/bash test_interactive() { - reset_systems 3 3 1 + # reset_systems 3 3 1 microcloud_internal_net_addr="$(ip_config_to_netaddr lxdbr0)" - echo "Creating a MicroCloud with all services but no devices" + # echo "Creating a MicroCloud with all services but no devices" export MULTI_NODE="yes" export LOOKUP_IFACE="enp5s0" export LIMIT_SUBNET="yes" @@ -14,119 +14,119 @@ test_interactive() { export SETUP_CEPH="no" export SETUP_OVN="no" export CEPH_CLUSTER_NETWORK="${microcloud_internal_net_addr}" - microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" + # microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" - lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q - for m in micro01 micro02 micro03 ; do - validate_system_lxd "${m}" 3 - validate_system_microceph "${m}" - validate_system_microovn "${m}" - done + # lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q + # for m in micro01 micro02 micro03 ; do + # validate_system_lxd "${m}" 3 + # validate_system_microceph "${m}" + # validate_system_microovn "${m}" + # done - # Reset the systems with just LXD. - reset_systems 3 3 1 + # # Reset the systems with just LXD. + # reset_systems 3 3 1 - for m in micro01 micro02 micro03 ; do - lxc exec "${m}" -- snap disable microceph || true - lxc exec "${m}" -- snap disable microovn || true - lxc exec "${m}" -- snap restart microcloud - done + # for m in micro01 micro02 micro03 ; do + # lxc exec "${m}" -- snap disable microceph || true + # lxc exec "${m}" -- snap disable microovn || true + # lxc exec "${m}" -- snap restart microcloud + # done - microcloud_internal_net_addr="$(ip_config_to_netaddr lxdbr0)" + # microcloud_internal_net_addr="$(ip_config_to_netaddr lxdbr0)" - echo "Creating a MicroCloud with ZFS storage" + # echo "Creating a MicroCloud with ZFS storage" export SKIP_SERVICE="yes" export SETUP_ZFS="yes" export ZFS_FILTER="lxd_disk1" export ZFS_WIPE="yes" export CEPH_CLUSTER_NETWORK="${microcloud_internal_net_addr}" unset SETUP_CEPH SETUP_OVN - microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" + # microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" - lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q - for m in micro01 micro02 micro03 ; do - validate_system_lxd "${m}" 3 disk1 - done + # lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q + # for m in micro01 micro02 micro03 ; do + # validate_system_lxd "${m}" 3 disk1 + # done # Reset the systems with just LXD and no IPv6 support. - reset_systems 3 3 1 - - for m in micro01 micro02 micro03 ; do - lxc exec "${m}" -- echo 1 > /proc/sys/net/ipv6/conf/all/disable_ipv6 - lxc exec "${m}" -- snap disable microceph || true - lxc exec "${m}" -- snap disable microovn || true - lxc exec "${m}" -- snap restart microcloud - done - - echo "Creating a MicroCloud with ZFS storage and no IPv6 support" - microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" - - lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q - for m in micro01 micro02 micro03 ; do - validate_system_lxd "${m}" 3 disk1 - done - - # Reset the systems with just LXD and no IPv4 support. - gw_net_addr=$(lxc network get lxdbr0 ipv4.address) - lxc network set lxdbr0 ipv4.address none - reset_systems 3 3 1 - - for m in micro01 micro02 micro03 ; do - lxc exec "${m}" -- snap disable microceph || true - lxc exec "${m}" -- snap disable microovn || true - lxc exec "${m}" -- snap restart microcloud - done + # reset_systems 3 3 1 + + # for m in micro01 micro02 micro03 ; do + # lxc exec "${m}" -- echo 1 > /proc/sys/net/ipv6/conf/all/disable_ipv6 + # lxc exec "${m}" -- snap disable microceph || true + # lxc exec "${m}" -- snap disable microovn || true + # lxc exec "${m}" -- snap restart microcloud + # done + + # echo "Creating a MicroCloud with ZFS storage and no IPv6 support" + # microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" + + # lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q + # for m in micro01 micro02 micro03 ; do + # validate_system_lxd "${m}" 3 disk1 + # done + + # # Reset the systems with just LXD and no IPv4 support. + # gw_net_addr=$(lxc network get lxdbr0 ipv4.address) + # lxc network set lxdbr0 ipv4.address none + # reset_systems 3 3 1 + + # for m in micro01 micro02 micro03 ; do + # lxc exec "${m}" -- snap disable microceph || true + # lxc exec "${m}" -- snap disable microovn || true + # lxc exec "${m}" -- snap restart microcloud + # done # Unset the lookup interface because we don't have multiple addresses to select from anymore. unset LOOKUP_IFACE export PROCEED_WITH_NO_OVERLAY_NETWORKING="no" # This will avoid to setup the cluster if no overlay networking is available. - echo "Creating a MicroCloud with ZFS storage and no IPv4 support" - ! microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init 2> err" || false + # echo "Creating a MicroCloud with ZFS storage and no IPv4 support" + # ! microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init 2> err" || false - # Ensure we error out due to a lack of usable overlay networking. - lxc exec micro01 -- cat err | grep "Cluster bootstrapping aborted due to lack of usable networking" -q + # # Ensure we error out due to a lack of usable overlay networking. + # lxc exec micro01 -- cat err | grep "Cluster bootstrapping aborted due to lack of usable networking" -q - # Set the IPv4 address back to the original value. - lxc network set lxdbr0 ipv4.address "${gw_net_addr}" + # # Set the IPv4 address back to the original value. + # lxc network set lxdbr0 ipv4.address "${gw_net_addr}" unset PROCEED_WITH_NO_OVERLAY_NETWORKING export LOOKUP_IFACE=enp5s0 # Reset the systems and install microceph. - reset_systems 3 3 1 + # reset_systems 3 3 1 - microcloud_internal_net_addr="$(ip_config_to_netaddr lxdbr0)" + # microcloud_internal_net_addr="$(ip_config_to_netaddr lxdbr0)" - for m in micro01 micro02 micro03 ; do - lxc exec "${m}" -- snap disable microovn || true - lxc exec "${m}" -- snap restart microcloud - done + # for m in micro01 micro02 micro03 ; do + # lxc exec "${m}" -- snap disable microovn || true + # lxc exec "${m}" -- snap restart microcloud + # done - echo "Creating a MicroCloud with ZFS and Ceph storage" + # echo "Creating a MicroCloud with ZFS and Ceph storage" export SETUP_CEPH="yes" export SETUP_CEPHFS="yes" export CEPH_FILTER="lxd_disk2" export CEPH_WIPE="yes" export CEPH_CLUSTER_NETWORK="${microcloud_internal_net_addr}" export CEPH_ENCRYPT="no" - microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" + # microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" - lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q - for m in micro01 micro02 micro03 ; do - validate_system_lxd "${m}" 3 disk1 1 1 - validate_system_microceph "${m}" 1 disk2 - done + # lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q + # for m in micro01 micro02 micro03 ; do + # validate_system_lxd "${m}" 3 disk1 1 1 + # validate_system_microceph "${m}" 1 disk2 + # done - # Reset the systems and install microovn. - reset_systems 3 3 1 + # # Reset the systems and install microovn. + # reset_systems 3 3 1 - microcloud_internal_net_addr="$(ip_config_to_netaddr lxdbr0)" + # microcloud_internal_net_addr="$(ip_config_to_netaddr lxdbr0)" - for m in micro01 micro02 micro03 ; do - lxc exec "${m}" -- snap disable microceph || true - lxc exec "${m}" -- snap restart microcloud - done + # for m in micro01 micro02 micro03 ; do + # lxc exec "${m}" -- snap disable microceph || true + # lxc exec "${m}" -- snap restart microcloud + # done - echo "Creating a MicroCloud with ZFS storage and OVN network" + # echo "Creating a MicroCloud with ZFS storage and OVN network" unset SETUP_CEPH CEPH_FILTER CEPH_WIPE SETUP_CEPHFS export SETUP_OVN="yes" @@ -138,20 +138,20 @@ test_interactive() { export DNS_ADDRESSES="10.1.123.1,8.8.8.8" export CEPH_CLUSTER_NETWORK="${microcloud_internal_net_addr}" export OVN_UNDERLAY_NETWORK="no" - microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" + # microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" - lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q - for m in micro01 micro02 micro03 ; do - validate_system_lxd "${m}" 3 disk1 0 0 "${OVN_FILTER}" "${IPV4_SUBNET}" "${IPV4_START}"-"${IPV4_END}" "${IPV6_SUBNET}" "${DNS_ADDRESSES}" - validate_system_microovn "${m}" - done + # lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q + # for m in micro01 micro02 micro03 ; do + # validate_system_lxd "${m}" 3 disk1 0 0 "${OVN_FILTER}" "${IPV4_SUBNET}" "${IPV4_START}"-"${IPV4_END}" "${IPV6_SUBNET}" "${DNS_ADDRESSES}" + # validate_system_microovn "${m}" + # done - # Reset the systems and install microovn and microceph. - reset_systems 3 3 1 + # # Reset the systems and install microovn and microceph. + # reset_systems 3 3 1 - microcloud_internal_net_addr="$(ip_config_to_netaddr lxdbr0)" + # microcloud_internal_net_addr="$(ip_config_to_netaddr lxdbr0)" - echo "Creating a MicroCloud with ZFS and Ceph storage, and OVN network" + # echo "Creating a MicroCloud with ZFS and Ceph storage, and OVN network" unset SKIP_SERVICE export SETUP_CEPH="yes" export SETUP_CEPHFS="yes" @@ -159,42 +159,42 @@ test_interactive() { export CEPH_WIPE="yes" export CEPH_CLUSTER_NETWORK="${microcloud_internal_net_addr}" export CEPH_ENCRYPT="no" - microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" + # microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" - lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q - for m in micro01 micro02 micro03 ; do - validate_system_lxd "${m}" 3 disk1 3 1 "${OVN_FILTER}" "${IPV4_SUBNET}" "${IPV4_START}"-"${IPV4_END}" "${IPV6_SUBNET}" "${DNS_ADDRESSES}" - validate_system_microceph "${m}" 1 disk2 - validate_system_microovn "${m}" - done + # lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q + # for m in micro01 micro02 micro03 ; do + # validate_system_lxd "${m}" 3 disk1 3 1 "${OVN_FILTER}" "${IPV4_SUBNET}" "${IPV4_START}"-"${IPV4_END}" "${IPV6_SUBNET}" "${DNS_ADDRESSES}" + # validate_system_microceph "${m}" 1 disk2 + # validate_system_microovn "${m}" + # done - # Reset the systems and install microovn and microceph (with Ceph encryption). - reset_systems 3 3 1 + # # Reset the systems and install microovn and microceph (with Ceph encryption). + # reset_systems 3 3 1 - echo "Creating a MicroCloud with ZFS and Ceph storage, and OVN network with Ceph encryption" + # echo "Creating a MicroCloud with ZFS and Ceph storage, and OVN network with Ceph encryption" export CEPH_ENCRYPT="yes" - microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" + # microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" - lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q - for m in micro01 micro02 micro03 ; do - validate_system_lxd "${m}" 3 disk1 3 1 "${OVN_FILTER}" "${IPV4_SUBNET}" "${IPV4_START}"-"${IPV4_END}" "${IPV6_SUBNET}" "${DNS_ADDRESSES}" - # Check ceph encryption for disk2 as part of the microceph validation. - validate_system_microceph "${m}" 1 1 disk2 disk2 - validate_system_microovn "${m}" - done + # lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q + # for m in micro01 micro02 micro03 ; do + # validate_system_lxd "${m}" 3 disk1 3 1 "${OVN_FILTER}" "${IPV4_SUBNET}" "${IPV4_START}"-"${IPV4_END}" "${IPV6_SUBNET}" "${DNS_ADDRESSES}" + # # Check ceph encryption for disk2 as part of the microceph validation. + # validate_system_microceph "${m}" 1 1 disk2 disk2 + # validate_system_microovn "${m}" + # done - # Reset the systems and install microovn and microceph with a partially disaggregated ceph network setup. - reset_systems 3 3 2 + # # Reset the systems and install microovn and microceph with a partially disaggregated ceph network setup. + # reset_systems 3 3 2 ceph_cluster_subnet_prefix="10.0.1" - ceph_cluster_subnet_iface="enp7s0" + # ceph_cluster_subnet_iface="enp7s0" - for n in $(seq 2 4); do - cluster_ip="${ceph_cluster_subnet_prefix}.${n}/24" - lxc exec "micro0$((n-1))" -- ip addr add "${cluster_ip}" dev "${ceph_cluster_subnet_iface}" - done + # for n in $(seq 2 4); do + # cluster_ip="${ceph_cluster_subnet_prefix}.${n}/24" + # lxc exec "micro0$((n-1))" -- ip addr add "${cluster_ip}" dev "${ceph_cluster_subnet_iface}" + # done - echo "Creating a MicroCloud with ZFS, Ceph storage with a fully disaggregated Ceph networking setup, and OVN network" + # echo "Creating a MicroCloud with ZFS, Ceph storage with a fully disaggregated Ceph networking setup, and OVN network" export SETUP_ZFS="yes" export ZFS_FILTER="lxd_disk1" export ZFS_WIPE="yes" @@ -209,45 +209,83 @@ test_interactive() { export IPV4_START="10.1.123.100" export IPV4_END="10.1.123.254" export OVN_UNDERLAY_NETWORK="no" - microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" + # microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" - lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q - for m in micro01 micro02 micro03 ; do - validate_system_lxd "${m}" 3 disk1 3 1 "${OVN_FILTER}" "${IPV4_SUBNET}" "${IPV4_START}"-"${IPV4_END}" "${IPV6_SUBNET}" - validate_system_microceph "${m}" 1 "${CEPH_CLUSTER_NETWORK}" disk2 - validate_system_microovn "${m}" - done + # lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q + # for m in micro01 micro02 micro03 ; do + # validate_system_lxd "${m}" 3 disk1 3 1 "${OVN_FILTER}" "${IPV4_SUBNET}" "${IPV4_START}"-"${IPV4_END}" "${IPV6_SUBNET}" + # validate_system_microceph "${m}" 1 "${CEPH_CLUSTER_NETWORK}" disk2 + # validate_system_microovn "${m}" + # done - reset_systems 3 3 3 + # reset_systems 3 3 3 - ceph_cluster_subnet_prefix="10.2.123" - ceph_cluster_subnet_iface="enp7s0" + # ceph_cluster_subnet_prefix="10.2.123" + # ceph_cluster_subnet_iface="enp7s0" - for n in $(seq 2 4); do - cluster_ip="${ceph_cluster_subnet_prefix}.${n}/24" - lxc exec "micro0$((n-1))" -- ip addr add "${cluster_ip}" dev "${ceph_cluster_subnet_iface}" - done + # for n in $(seq 2 4); do + # cluster_ip="${ceph_cluster_subnet_prefix}.${n}/24" + # lxc exec "micro0$((n-1))" -- ip addr add "${cluster_ip}" dev "${ceph_cluster_subnet_iface}" + # done ovn_underlay_subnet_prefix="10.3.123" - ovn_underlay_subnet_iface="enp8s0" + # ovn_underlay_subnet_iface="enp8s0" - for n in $(seq 2 4); do - ovn_underlay_ip="${ovn_underlay_subnet_prefix}.${n}/24" - lxc exec "micro0$((n-1))" -- sh -c "ip addr add ${ovn_underlay_ip} dev ${ovn_underlay_subnet_iface} && ip link set ${ovn_underlay_subnet_iface} up" - done + # for n in $(seq 2 4); do + # ovn_underlay_ip="${ovn_underlay_subnet_prefix}.${n}/24" + # lxc exec "micro0$((n-1))" -- sh -c "ip addr add ${ovn_underlay_ip} dev ${ovn_underlay_subnet_iface} && ip link set ${ovn_underlay_subnet_iface} up" + # done - echo "Creating a MicroCloud with ZFS, Ceph storage with a fully disaggregated Ceph networking setup, OVN management network and OVN underlay network" + # echo "Creating a MicroCloud with ZFS, Ceph storage with a fully disaggregated Ceph networking setup, OVN management network and OVN underlay network" export CEPH_CLUSTER_NETWORK="${ceph_cluster_subnet_prefix}.0/24" export OVN_UNDERLAY_NETWORK="yes" export OVN_UNDERLAY_FILTER="${ovn_underlay_subnet_prefix}" + # microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" + + # lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q + # for m in micro01 micro02 micro03 ; do + # validate_system_lxd "${m}" 3 disk1 3 1 "${OVN_FILTER}" "${IPV4_SUBNET}" "${IPV4_START}"-"${IPV4_END}" "${IPV6_SUBNET}" + # validate_system_microceph "${m}" 1 "${CEPH_CLUSTER_NETWORK}" disk2 + # validate_system_microovn "${m}" "${ovn_underlay_subnet_prefix}" + # done + + reset_systems 3 3 3 + echo "Add a MicroCloud node on a cluster of 2 nodes with MicroOVN not initialized" + lxc exec micro03 -- snap disable microcloud || true + unset_interactive_vars + microcloud_internal_net_addr="$(ip_config_to_netaddr lxdbr0)" + export MULTI_NODE="yes" + export LOOKUP_IFACE="enp5s0" + export LIMIT_SUBNET="yes" + export EXPECT_PEERS=1 + export SETUP_ZFS="yes" + export ZFS_FILTER="lxd_disk1" + export ZFS_WIPE="yes" + export SETUP_CEPH="yes" + export SETUP_CEPHFS="yes" + export CEPH_WIPE="yes" + export CEPH_ENCRYPT="no" + export SETUP_OVN="no" + export CEPH_CLUSTER_NETWORK="${microcloud_internal_net_addr}" + export IPV4_SUBNET="10.1.123.1/24" + export IPV4_START="10.1.123.100" + export IPV4_END="10.1.123.254" + export DNS_ADDRESSES="10.1.123.1,8.8.8.8" + export IPV6_SUBNET="fd42:1:1234:1234::1/64" + + # Run a 2 nodes MicroCloud without MicroOVN first. microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" - lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q - for m in micro01 micro02 micro03 ; do - validate_system_lxd "${m}" 3 disk1 3 1 "${OVN_FILTER}" "${IPV4_SUBNET}" "${IPV4_START}"-"${IPV4_END}" "${IPV6_SUBNET}" - validate_system_microceph "${m}" 1 "${CEPH_CLUSTER_NETWORK}" disk2 - validate_system_microovn "${m}" "${ovn_underlay_subnet_prefix}" - done + lxc exec micro03 -- microovn cluster bootstrap + lxc exec micro03 -- snap enable microcloud + + export EXPECT_PEERS=1 + export SETUP_OVN="yes" + unset SETUP_CEPH SETUP_CEPHFS CEPH_CLUSTER_NETWORK + unset IPV4_SUBNET IPV4_START IPV4_END DNS_ADDRESSES IPV6_SUBNET + microcloud_interactive | lxc exec micro01 -- sh -c "microcloud add > out" + + # TODO: we should verify that MicroCloud offers micro01 and micro02 to add their underlay configuration, but skips micro03 since it was already set up, even though technically micro03 is the one joining the overall MicroCloud cluster } test_instances_config() {