From 088ada58b422f3eb05be228a468171460653925f Mon Sep 17 00:00:00 2001 From: Gabriel Mougard Date: Mon, 2 Sep 2024 15:58:05 +0200 Subject: [PATCH] debug Signed-off-by: Gabriel Mougard --- cmd/microcloud/ask.go | 4 +- cmd/microcloud/asker.go | 231 ++++++++++++++++++++++ cmd/microcloud/main.go | 7 +- cmd/microcloud/main_init.go | 3 +- cmd/microcloud/test_console.go | 5 +- test/includes/microcloud.sh | 4 +- test/suites/add.sh | 2 - test/suites/basic.sh | 351 ++++++++++++++++++++------------- 8 files changed, 460 insertions(+), 147 deletions(-) create mode 100644 cmd/microcloud/asker.go diff --git a/cmd/microcloud/ask.go b/cmd/microcloud/ask.go index a270a8026..2403b4165 100644 --- a/cmd/microcloud/ask.go +++ b/cmd/microcloud/ask.go @@ -1105,8 +1105,8 @@ func (c *initConfig) askOVNNetwork(sh *service.Handler) error { } canOVNUnderlay := true - for peer, state := range c.state { - if len(state.AvailableOVNInterfaces) == 0 { + for peer, system := range c.systems { + if len(c.state[system.ServerInfo.Name].AvailableOVNInterfaces) == 0 { fmt.Printf("Not enough interfaces available on %s to create an underlay network, skipping\n", peer) canOVNUnderlay = false break diff --git a/cmd/microcloud/asker.go b/cmd/microcloud/asker.go new file mode 100644 index 000000000..46fea8048 --- /dev/null +++ b/cmd/microcloud/asker.go @@ -0,0 +1,231 @@ +package main + +import ( + "bufio" + "fmt" + "log" + "os" + "strconv" + "strings" + + "golang.org/x/term" +) + +func ValueInSlice[T comparable](key T, list []T) bool { + for _, entry := range list { + if entry == key { + return true + } + } + + return false +} + +// Asker holds a reader for reading input into CLI questions. +type Asker struct { + reader *bufio.Reader + logger *log.Logger +} + +// NewAsker creates a new Asker instance that reads from the given reader. +// It can also be configured with a logger to help during the debug process. +func NewAsker(reader *bufio.Reader, logger *log.Logger) Asker { + return Asker{reader: reader, logger: logger} +} + +// AskBool asks a question and expect a yes/no answer. +func (a *Asker) AskBool(question string, defaultAnswer string) (bool, error) { + for { + answer, err := a.askQuestion(question, defaultAnswer) + if err != nil { + if a.logger != nil { + a.logger.Printf("Failed to read answer (%v) for question (%v). err : %v", answer, question, err) + } + + return false, err + } + + if ValueInSlice(strings.ToLower(answer), []string{"yes", "y"}) { + return true, nil + } else if ValueInSlice(strings.ToLower(answer), []string{"no", "n"}) { + return false, nil + } + + a.invalidInput(question, answer) + } +} + +// AskChoice asks the user to select one of multiple options. +func (a *Asker) AskChoice(question string, choices []string, defaultAnswer string) (string, error) { + for { + answer, err := a.askQuestion(question, defaultAnswer) + if err != nil { + if a.logger != nil { + a.logger.Printf("Failed to read answer (%v) for question (%v). err : %v", answer, question, err) + } + + return "", err + } + + if ValueInSlice(answer, choices) { + return answer, nil + } else if a.logger != nil { + a.logger.Printf("Answer (%v) not among available choices (%v))", answer, choices) + } + + a.invalidInput(question, answer) + } +} + +// AskInt asks the user to enter an integer between a min and max value. +func (a *Asker) AskInt(question string, min int64, max int64, defaultAnswer string, validate func(int64) error) (int64, error) { + for { + answer, err := a.askQuestion(question, defaultAnswer) + if err != nil { + if a.logger != nil { + a.logger.Printf("Failed to read answer (%v) for question (%v). err : %v", answer, question, err) + } + + return -1, err + } + + result, err := strconv.ParseInt(answer, 10, 64) + if err != nil { + if a.logger != nil { + a.logger.Printf("Invalid answer (%v) for question (%v). err : %v", answer, question, err) + } + + fmt.Fprintf(os.Stderr, "Invalid input: %v\n\n", err) + continue + } + + if !((min == -1 || result >= min) && (max == -1 || result <= max)) { + if a.logger != nil { + a.logger.Printf("Answer (%v) out of range for question (%v)", answer, question) + } + + fmt.Fprintf(os.Stderr, "Invalid input: out of range\n\n") + continue + } + + if validate != nil { + err = validate(result) + if err != nil { + if a.logger != nil { + a.logger.Printf("Invalid answer (%v) for question (%v). err : %v", answer, question, err) + } + + fmt.Fprintf(os.Stderr, "Invalid input: %v\n\n", err) + continue + } + } + + return result, err + } +} + +// AskString asks the user to enter a string, which optionally +// conforms to a validation function. +func (a *Asker) AskString(question string, defaultAnswer string, validate func(string) error) (string, error) { + for { + answer, err := a.askQuestion(question, defaultAnswer) + if err != nil { + if a.logger != nil { + a.logger.Printf("Failed to read answer (%v) for question (%v). err : %v", answer, question, err) + } + + return "", err + } + + if validate != nil { + err = validate(answer) + if err != nil { + if a.logger != nil { + a.logger.Printf("Invalid answer (%v) for question (%v). err : %v", answer, question, err) + } + + fmt.Fprintf(os.Stderr, "Invalid input: %v\n\n", err) + continue + } + + return answer, err + } + + if len(answer) != 0 { + return answer, err + } + + a.invalidInput(question, answer) + } +} + +// AskPassword asks the user to enter a password. +func (a *Asker) AskPassword(question string) string { + for { + fmt.Print(question) + + pwd, _ := term.ReadPassword(0) + fmt.Println("") + inFirst := string(pwd) + inFirst = strings.TrimSuffix(inFirst, "\n") + + fmt.Print("Again: ") + pwd, _ = term.ReadPassword(0) + fmt.Println("") + inSecond := string(pwd) + inSecond = strings.TrimSuffix(inSecond, "\n") + + // refuse empty password or if password inputs do not match + if len(inFirst) > 0 && inFirst == inSecond { + return inFirst + } + + a.invalidInput(question, "*****") + } +} + +// AskPasswordOnce asks the user to enter a password. +// +// It's the same as AskPassword, but it won't ask to enter it again. +func (a *Asker) AskPasswordOnce(question string) string { + for { + fmt.Print(question) + pwd, _ := term.ReadPassword(0) + fmt.Println("") + + // refuse empty password + spwd := string(pwd) + if len(spwd) > 0 { + return spwd + } + + a.invalidInput(question, "*****") + } +} + +// Ask a question on the output stream and read the answer from the input stream. +func (a *Asker) askQuestion(question, defaultAnswer string) (string, error) { + fmt.Print(question) + + return a.readAnswer(defaultAnswer) +} + +// Read the user's answer from the input stream, trimming newline and providing a default. +func (a *Asker) readAnswer(defaultAnswer string) (string, error) { + answer, err := a.reader.ReadString('\n') + answer = strings.TrimSpace(strings.TrimSuffix(answer, "\n")) + if answer == "" { + answer = defaultAnswer + } + + return answer, err +} + +// Print an invalid input message on the error stream. +func (a *Asker) invalidInput(question string, answer string) { + if a.logger != nil { + a.logger.Printf("Invalid answer (%v) for question (%v)", answer, question) + } + + fmt.Fprintf(os.Stderr, "Invalid input, try again.\n\n") +} diff --git a/cmd/microcloud/main.go b/cmd/microcloud/main.go index 4244e77a2..fdb620abd 100644 --- a/cmd/microcloud/main.go +++ b/cmd/microcloud/main.go @@ -4,9 +4,9 @@ package main import ( "bufio" "fmt" + "log" "os" - cli "github.com/canonical/lxd/shared/cmd" "github.com/spf13/cobra" "github.com/canonical/microcloud/microcloud/version" @@ -23,7 +23,7 @@ type CmdControl struct { FlagLogVerbose bool FlagMicroCloudDir string - asker cli.Asker + asker Asker } func main() { @@ -34,7 +34,8 @@ func main() { } // common flags. - commonCmd := CmdControl{asker: cli.NewAsker(bufio.NewReader(os.Stdin))} + logger := log.New(os.Stdout, "logger:", log.LstdFlags) + commonCmd := CmdControl{asker: NewAsker(bufio.NewReader(os.Stdin), logger)} useTestConsole := os.Getenv("TEST_CONSOLE") if useTestConsole == "1" { diff --git a/cmd/microcloud/main_init.go b/cmd/microcloud/main_init.go index 25d8d1613..8b2d13778 100644 --- a/cmd/microcloud/main_init.go +++ b/cmd/microcloud/main_init.go @@ -12,7 +12,6 @@ import ( "github.com/canonical/lxd/lxd/util" "github.com/canonical/lxd/shared" lxdAPI "github.com/canonical/lxd/shared/api" - cli "github.com/canonical/lxd/shared/cmd" "github.com/canonical/lxd/shared/logger" "github.com/canonical/lxd/shared/revert" "github.com/canonical/lxd/shared/validate" @@ -70,7 +69,7 @@ type initConfig struct { common *CmdControl // asker is the CLI user input helper. - asker *cli.Asker + asker *Asker // address is the cluster address of the local system. address string diff --git a/cmd/microcloud/test_console.go b/cmd/microcloud/test_console.go index 486b10444..25280a332 100644 --- a/cmd/microcloud/test_console.go +++ b/cmd/microcloud/test_console.go @@ -12,7 +12,6 @@ import ( "github.com/AlecAivazis/survey/v2/terminal" "github.com/Netflix/go-expect" - cli "github.com/canonical/lxd/shared/cmd" "github.com/creack/pty" "github.com/hinshun/vt10x" ) @@ -25,7 +24,7 @@ type testConsole struct { } // prepareTestAsker removes comments from the lines read from the given reader, and assigns them to the test console reader. -func prepareTestAsker(r io.Reader) cli.Asker { +func prepareTestAsker(r io.Reader) Asker { sc := bufio.NewScanner(r) b := bytes.Buffer{} for sc.Scan() { @@ -39,7 +38,7 @@ func prepareTestAsker(r io.Reader) cli.Asker { reader = bufio.NewReader(bytes.NewReader(b.Bytes())) - return cli.NewAsker(reader) + return NewAsker(reader, nil) } // NewTestConsole creates a new testConsole, with an underlying expect.Console and virtual terminal. diff --git a/test/includes/microcloud.sh b/test/includes/microcloud.sh index 0c28ef44d..36fd3a125 100644 --- a/test/includes/microcloud.sh +++ b/test/includes/microcloud.sh @@ -2,7 +2,7 @@ # unset_interactive_vars: Unsets all variables related to the test console. unset_interactive_vars() { - unset SKIP_LOOKUP LOOKUP_IFACE LIMIT_SUBNET SKIP_SERVICE EXPECT_PEERS REUSE_EXISTING REUSE_EXISTING_COUNT \ + unset SKIP_LOOKUP LOOKUP_IFACE LIMIT_SUBNET SKIP_SERVICE EXPECT_PEERS PEERS_FILTER REUSE_EXISTING REUSE_EXISTING_COUNT \ SETUP_ZFS ZFS_FILTER ZFS_WIPE \ SETUP_CEPH CEPH_MISSING_DISKS CEPH_FILTER CEPH_WIPE CEPH_ENCRYPT SETUP_CEPHFS CEPH_CLUSTER_NETWORK \ PROCEED_WITH_NO_OVERLAY_NETWORKING SETUP_OVN OVN_UNDERLAY_NETWORK OVN_UNDERLAY_FILTEROVN_WARNING OVN_FILTER IPV4_SUBNET IPV4_START IPV4_END DNS_ADDRESSES IPV6_SUBNET \ @@ -27,6 +27,7 @@ microcloud_interactive() { LIMIT_SUBNET=${LIMIT_SUBNET:-} # (yes/no) input for limiting lookup of systems to the above subnet. SKIP_SERVICE=${SKIP_SERVICE:-} # (yes/no) input to skip any missing services. Should be unset if all services are installed. EXPECT_PEERS=${EXPECT_PEERS:-} # wait for this number of systems to be available to join the cluster. + PEERS_FILTER=${PEERS_FILTER:-} # filter string for the particular peer to init/add REUSE_EXISTING=${REUSE_EXISTING:-} # (yes/no) incorporate an existing clustered service. REUSE_EXISTING_COUNT=${REUSE_EXISTING_COUNT:-0} # (number) number of existing clusters to incorporate. SETUP_ZFS=${SETUP_ZFS:-} # (yes/no) input for initiating ZFS storage pool setup. @@ -69,6 +70,7 @@ $(true) ${LIMIT_SUBNET} # limit lookup subnet (yes/no) $([ "${SKIP_SERVICE}" = "yes" ] && printf "%s" "${SKIP_SERVICE}") # skip MicroOVN/MicroCeph (yes/no) expect ${EXPECT_PEERS} # wait until the systems show up +${PEERS_FILTER} # filter discovered peers select-all # select all the systems --- $(true) # workaround for set -e diff --git a/test/suites/add.sh b/test/suites/add.sh index aeb1dc069..2ad5d299d 100644 --- a/test/suites/add.sh +++ b/test/suites/add.sh @@ -193,7 +193,6 @@ test_add_interactive() { export SETUP_ZFS="yes" export ZFS_FILTER="lxd_disk1" export ZFS_WIPE="yes" - export OVN_UNDERLAY_NETWORK="no" microcloud_interactive | lxc exec micro01 -- sh -c "microcloud add > out" lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q @@ -211,7 +210,6 @@ test_add_interactive() { export SETUP_ZFS="no" export SETUP_CEPH="no" export SETUP_OVN="no" - export OVN_UNDERLAY_NETWORK="no" lxc exec micro04 -- snap disable microcloud microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" diff --git a/test/suites/basic.sh b/test/suites/basic.sh index 27d61fc57..4fa938cfc 100644 --- a/test/suites/basic.sh +++ b/test/suites/basic.sh @@ -1,11 +1,11 @@ #!/bin/bash test_interactive() { - reset_systems 3 3 1 + # reset_systems 3 3 1 microcloud_internal_net_addr="$(ip_config_to_netaddr lxdbr0)" - echo "Creating a MicroCloud with all services but no devices" + # echo "Creating a MicroCloud with all services but no devices" export MULTI_NODE="yes" export LOOKUP_IFACE="enp5s0" export LIMIT_SUBNET="yes" @@ -14,119 +14,119 @@ test_interactive() { export SETUP_CEPH="no" export SETUP_OVN="no" export CEPH_CLUSTER_NETWORK="${microcloud_internal_net_addr}" - microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" + # microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" - lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q - for m in micro01 micro02 micro03 ; do - validate_system_lxd "${m}" 3 - validate_system_microceph "${m}" - validate_system_microovn "${m}" - done + # lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q + # for m in micro01 micro02 micro03 ; do + # validate_system_lxd "${m}" 3 + # validate_system_microceph "${m}" + # validate_system_microovn "${m}" + # done - # Reset the systems with just LXD. - reset_systems 3 3 1 + # # Reset the systems with just LXD. + # reset_systems 3 3 1 - for m in micro01 micro02 micro03 ; do - lxc exec "${m}" -- snap disable microceph || true - lxc exec "${m}" -- snap disable microovn || true - lxc exec "${m}" -- snap restart microcloud - done + # for m in micro01 micro02 micro03 ; do + # lxc exec "${m}" -- snap disable microceph || true + # lxc exec "${m}" -- snap disable microovn || true + # lxc exec "${m}" -- snap restart microcloud + # done - microcloud_internal_net_addr="$(ip_config_to_netaddr lxdbr0)" + # microcloud_internal_net_addr="$(ip_config_to_netaddr lxdbr0)" - echo "Creating a MicroCloud with ZFS storage" + # echo "Creating a MicroCloud with ZFS storage" export SKIP_SERVICE="yes" export SETUP_ZFS="yes" export ZFS_FILTER="lxd_disk1" export ZFS_WIPE="yes" export CEPH_CLUSTER_NETWORK="${microcloud_internal_net_addr}" unset SETUP_CEPH SETUP_OVN - microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" + # microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" - lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q - for m in micro01 micro02 micro03 ; do - validate_system_lxd "${m}" 3 disk1 - done + # lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q + # for m in micro01 micro02 micro03 ; do + # validate_system_lxd "${m}" 3 disk1 + # done # Reset the systems with just LXD and no IPv6 support. - reset_systems 3 3 1 - - for m in micro01 micro02 micro03 ; do - lxc exec "${m}" -- echo 1 > /proc/sys/net/ipv6/conf/all/disable_ipv6 - lxc exec "${m}" -- snap disable microceph || true - lxc exec "${m}" -- snap disable microovn || true - lxc exec "${m}" -- snap restart microcloud - done - - echo "Creating a MicroCloud with ZFS storage and no IPv6 support" - microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" - - lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q - for m in micro01 micro02 micro03 ; do - validate_system_lxd "${m}" 3 disk1 - done - - # Reset the systems with just LXD and no IPv4 support. - gw_net_addr=$(lxc network get lxdbr0 ipv4.address) - lxc network set lxdbr0 ipv4.address none - reset_systems 3 3 1 - - for m in micro01 micro02 micro03 ; do - lxc exec "${m}" -- snap disable microceph || true - lxc exec "${m}" -- snap disable microovn || true - lxc exec "${m}" -- snap restart microcloud - done + # reset_systems 3 3 1 + + # for m in micro01 micro02 micro03 ; do + # lxc exec "${m}" -- echo 1 > /proc/sys/net/ipv6/conf/all/disable_ipv6 + # lxc exec "${m}" -- snap disable microceph || true + # lxc exec "${m}" -- snap disable microovn || true + # lxc exec "${m}" -- snap restart microcloud + # done + + # echo "Creating a MicroCloud with ZFS storage and no IPv6 support" + # microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" + + # lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q + # for m in micro01 micro02 micro03 ; do + # validate_system_lxd "${m}" 3 disk1 + # done + + # # Reset the systems with just LXD and no IPv4 support. + # gw_net_addr=$(lxc network get lxdbr0 ipv4.address) + # lxc network set lxdbr0 ipv4.address none + # reset_systems 3 3 1 + + # for m in micro01 micro02 micro03 ; do + # lxc exec "${m}" -- snap disable microceph || true + # lxc exec "${m}" -- snap disable microovn || true + # lxc exec "${m}" -- snap restart microcloud + # done # Unset the lookup interface because we don't have multiple addresses to select from anymore. unset LOOKUP_IFACE export PROCEED_WITH_NO_OVERLAY_NETWORKING="no" # This will avoid to setup the cluster if no overlay networking is available. - echo "Creating a MicroCloud with ZFS storage and no IPv4 support" - ! microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init 2> err" || false + # echo "Creating a MicroCloud with ZFS storage and no IPv4 support" + # ! microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init 2> err" || false - # Ensure we error out due to a lack of usable overlay networking. - lxc exec micro01 -- cat err | grep "Cluster bootstrapping aborted due to lack of usable networking" -q + # # Ensure we error out due to a lack of usable overlay networking. + # lxc exec micro01 -- cat err | grep "Cluster bootstrapping aborted due to lack of usable networking" -q - # Set the IPv4 address back to the original value. - lxc network set lxdbr0 ipv4.address "${gw_net_addr}" + # # Set the IPv4 address back to the original value. + # lxc network set lxdbr0 ipv4.address "${gw_net_addr}" unset PROCEED_WITH_NO_OVERLAY_NETWORKING export LOOKUP_IFACE=enp5s0 # Reset the systems and install microceph. - reset_systems 3 3 1 + # reset_systems 3 3 1 - microcloud_internal_net_addr="$(ip_config_to_netaddr lxdbr0)" + # microcloud_internal_net_addr="$(ip_config_to_netaddr lxdbr0)" - for m in micro01 micro02 micro03 ; do - lxc exec "${m}" -- snap disable microovn || true - lxc exec "${m}" -- snap restart microcloud - done + # for m in micro01 micro02 micro03 ; do + # lxc exec "${m}" -- snap disable microovn || true + # lxc exec "${m}" -- snap restart microcloud + # done - echo "Creating a MicroCloud with ZFS and Ceph storage" + # echo "Creating a MicroCloud with ZFS and Ceph storage" export SETUP_CEPH="yes" export SETUP_CEPHFS="yes" export CEPH_FILTER="lxd_disk2" export CEPH_WIPE="yes" export CEPH_CLUSTER_NETWORK="${microcloud_internal_net_addr}" export CEPH_ENCRYPT="no" - microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" + # microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" - lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q - for m in micro01 micro02 micro03 ; do - validate_system_lxd "${m}" 3 disk1 1 1 - validate_system_microceph "${m}" 1 disk2 - done + # lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q + # for m in micro01 micro02 micro03 ; do + # validate_system_lxd "${m}" 3 disk1 1 1 + # validate_system_microceph "${m}" 1 disk2 + # done - # Reset the systems and install microovn. - reset_systems 3 3 1 + # # Reset the systems and install microovn. + # reset_systems 3 3 1 - microcloud_internal_net_addr="$(ip_config_to_netaddr lxdbr0)" + # microcloud_internal_net_addr="$(ip_config_to_netaddr lxdbr0)" - for m in micro01 micro02 micro03 ; do - lxc exec "${m}" -- snap disable microceph || true - lxc exec "${m}" -- snap restart microcloud - done + # for m in micro01 micro02 micro03 ; do + # lxc exec "${m}" -- snap disable microceph || true + # lxc exec "${m}" -- snap restart microcloud + # done - echo "Creating a MicroCloud with ZFS storage and OVN network" + # echo "Creating a MicroCloud with ZFS storage and OVN network" unset SETUP_CEPH CEPH_FILTER CEPH_WIPE SETUP_CEPHFS export SETUP_OVN="yes" @@ -138,20 +138,20 @@ test_interactive() { export DNS_ADDRESSES="10.1.123.1,8.8.8.8" export CEPH_CLUSTER_NETWORK="${microcloud_internal_net_addr}" export OVN_UNDERLAY_NETWORK="no" - microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" + # microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" - lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q - for m in micro01 micro02 micro03 ; do - validate_system_lxd "${m}" 3 disk1 0 0 "${OVN_FILTER}" "${IPV4_SUBNET}" "${IPV4_START}"-"${IPV4_END}" "${IPV6_SUBNET}" "${DNS_ADDRESSES}" - validate_system_microovn "${m}" - done + # lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q + # for m in micro01 micro02 micro03 ; do + # validate_system_lxd "${m}" 3 disk1 0 0 "${OVN_FILTER}" "${IPV4_SUBNET}" "${IPV4_START}"-"${IPV4_END}" "${IPV6_SUBNET}" "${DNS_ADDRESSES}" + # validate_system_microovn "${m}" + # done - # Reset the systems and install microovn and microceph. - reset_systems 3 3 1 + # # Reset the systems and install microovn and microceph. + # reset_systems 3 3 1 - microcloud_internal_net_addr="$(ip_config_to_netaddr lxdbr0)" + # microcloud_internal_net_addr="$(ip_config_to_netaddr lxdbr0)" - echo "Creating a MicroCloud with ZFS and Ceph storage, and OVN network" + # echo "Creating a MicroCloud with ZFS and Ceph storage, and OVN network" unset SKIP_SERVICE export SETUP_CEPH="yes" export SETUP_CEPHFS="yes" @@ -159,42 +159,42 @@ test_interactive() { export CEPH_WIPE="yes" export CEPH_CLUSTER_NETWORK="${microcloud_internal_net_addr}" export CEPH_ENCRYPT="no" - microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" + # microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" - lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q - for m in micro01 micro02 micro03 ; do - validate_system_lxd "${m}" 3 disk1 3 1 "${OVN_FILTER}" "${IPV4_SUBNET}" "${IPV4_START}"-"${IPV4_END}" "${IPV6_SUBNET}" "${DNS_ADDRESSES}" - validate_system_microceph "${m}" 1 disk2 - validate_system_microovn "${m}" - done + # lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q + # for m in micro01 micro02 micro03 ; do + # validate_system_lxd "${m}" 3 disk1 3 1 "${OVN_FILTER}" "${IPV4_SUBNET}" "${IPV4_START}"-"${IPV4_END}" "${IPV6_SUBNET}" "${DNS_ADDRESSES}" + # validate_system_microceph "${m}" 1 disk2 + # validate_system_microovn "${m}" + # done - # Reset the systems and install microovn and microceph (with Ceph encryption). - reset_systems 3 3 1 + # # Reset the systems and install microovn and microceph (with Ceph encryption). + # reset_systems 3 3 1 - echo "Creating a MicroCloud with ZFS and Ceph storage, and OVN network with Ceph encryption" + # echo "Creating a MicroCloud with ZFS and Ceph storage, and OVN network with Ceph encryption" export CEPH_ENCRYPT="yes" - microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" + # microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" - lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q - for m in micro01 micro02 micro03 ; do - validate_system_lxd "${m}" 3 disk1 3 1 "${OVN_FILTER}" "${IPV4_SUBNET}" "${IPV4_START}"-"${IPV4_END}" "${IPV6_SUBNET}" "${DNS_ADDRESSES}" - # Check ceph encryption for disk2 as part of the microceph validation. - validate_system_microceph "${m}" 1 1 disk2 disk2 - validate_system_microovn "${m}" - done + # lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q + # for m in micro01 micro02 micro03 ; do + # validate_system_lxd "${m}" 3 disk1 3 1 "${OVN_FILTER}" "${IPV4_SUBNET}" "${IPV4_START}"-"${IPV4_END}" "${IPV6_SUBNET}" "${DNS_ADDRESSES}" + # # Check ceph encryption for disk2 as part of the microceph validation. + # validate_system_microceph "${m}" 1 1 disk2 disk2 + # validate_system_microovn "${m}" + # done - # Reset the systems and install microovn and microceph with a partially disaggregated ceph network setup. - reset_systems 3 3 2 + # # Reset the systems and install microovn and microceph with a partially disaggregated ceph network setup. + # reset_systems 3 3 2 ceph_cluster_subnet_prefix="10.0.1" - ceph_cluster_subnet_iface="enp7s0" + # ceph_cluster_subnet_iface="enp7s0" - for n in $(seq 2 4); do - cluster_ip="${ceph_cluster_subnet_prefix}.${n}/24" - lxc exec "micro0$((n-1))" -- ip addr add "${cluster_ip}" dev "${ceph_cluster_subnet_iface}" - done + # for n in $(seq 2 4); do + # cluster_ip="${ceph_cluster_subnet_prefix}.${n}/24" + # lxc exec "micro0$((n-1))" -- ip addr add "${cluster_ip}" dev "${ceph_cluster_subnet_iface}" + # done - echo "Creating a MicroCloud with ZFS, Ceph storage with a fully disaggregated Ceph networking setup, and OVN network" + # echo "Creating a MicroCloud with ZFS, Ceph storage with a fully disaggregated Ceph networking setup, and OVN network" export SETUP_ZFS="yes" export ZFS_FILTER="lxd_disk1" export ZFS_WIPE="yes" @@ -209,43 +209,126 @@ test_interactive() { export IPV4_START="10.1.123.100" export IPV4_END="10.1.123.254" export OVN_UNDERLAY_NETWORK="no" - microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" + # microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" - lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q - for m in micro01 micro02 micro03 ; do - validate_system_lxd "${m}" 3 disk1 3 1 "${OVN_FILTER}" "${IPV4_SUBNET}" "${IPV4_START}"-"${IPV4_END}" "${IPV6_SUBNET}" - validate_system_microceph "${m}" 1 "${CEPH_CLUSTER_NETWORK}" disk2 - validate_system_microovn "${m}" - done + # lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q + # for m in micro01 micro02 micro03 ; do + # validate_system_lxd "${m}" 3 disk1 3 1 "${OVN_FILTER}" "${IPV4_SUBNET}" "${IPV4_START}"-"${IPV4_END}" "${IPV6_SUBNET}" + # validate_system_microceph "${m}" 1 "${CEPH_CLUSTER_NETWORK}" disk2 + # validate_system_microovn "${m}" + # done - reset_systems 3 3 3 + # reset_systems 3 3 3 - ceph_cluster_subnet_prefix="10.2.123" - ceph_cluster_subnet_iface="enp7s0" + # ceph_cluster_subnet_prefix="10.2.123" + # ceph_cluster_subnet_iface="enp7s0" - for n in $(seq 2 4); do - cluster_ip="${ceph_cluster_subnet_prefix}.${n}/24" - lxc exec "micro0$((n-1))" -- ip addr add "${cluster_ip}" dev "${ceph_cluster_subnet_iface}" - done + # for n in $(seq 2 4); do + # cluster_ip="${ceph_cluster_subnet_prefix}.${n}/24" + # lxc exec "micro0$((n-1))" -- ip addr add "${cluster_ip}" dev "${ceph_cluster_subnet_iface}" + # done ovn_underlay_subnet_prefix="10.3.123" - ovn_underlay_subnet_iface="enp8s0" + # ovn_underlay_subnet_iface="enp8s0" + + # for n in $(seq 2 4); do + # ovn_underlay_ip="${ovn_underlay_subnet_prefix}.${n}/24" + # lxc exec "micro0$((n-1))" -- sh -c "ip addr add ${ovn_underlay_ip} dev ${ovn_underlay_subnet_iface} && ip link set ${ovn_underlay_subnet_iface} up" + # done + + # echo "Creating a MicroCloud with ZFS, Ceph storage with a fully disaggregated Ceph networking setup, OVN management network and OVN underlay network" + export CEPH_CLUSTER_NETWORK="${ceph_cluster_subnet_prefix}.0/24" + export OVN_UNDERLAY_NETWORK="yes" + export OVN_UNDERLAY_FILTER="${ovn_underlay_subnet_prefix}" + # microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" + + # lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q + # for m in micro01 micro02 micro03 ; do + # validate_system_lxd "${m}" 3 disk1 3 1 "${OVN_FILTER}" "${IPV4_SUBNET}" "${IPV4_START}"-"${IPV4_END}" "${IPV6_SUBNET}" + # validate_system_microceph "${m}" 1 "${CEPH_CLUSTER_NETWORK}" disk2 + # validate_system_microovn "${m}" "${ovn_underlay_subnet_prefix}" + # done + + reset_systems 3 3 2 + echo "Add a MicroCloud node on a cluster of 2 nodes with MicroOVN not initialized" + + ovn_underlay_subnet_prefix="10.2.123" + ovn_underlay_subnet_iface="enp7s0" for n in $(seq 2 4); do ovn_underlay_ip="${ovn_underlay_subnet_prefix}.${n}/24" lxc exec "micro0$((n-1))" -- sh -c "ip addr add ${ovn_underlay_ip} dev ${ovn_underlay_subnet_iface} && ip link set ${ovn_underlay_subnet_iface} up" done - echo "Creating a MicroCloud with ZFS, Ceph storage with a fully disaggregated Ceph networking setup, OVN management network and OVN underlay network" - export CEPH_CLUSTER_NETWORK="${ceph_cluster_subnet_prefix}.0/24" - export OVN_UNDERLAY_NETWORK="yes" - export OVN_UNDERLAY_FILTER="${ovn_underlay_subnet_prefix}" + lxc exec micro03 -- snap disable microcloud || true + for m in micro01 micro02 ; do + lxc exec "${m}" -- snap disable microovn + done + + unset_interactive_vars + microcloud_internal_net_addr="$(ip_config_to_netaddr lxdbr0)" + export MULTI_NODE="yes" + export LOOKUP_IFACE="enp5s0" + export LIMIT_SUBNET="yes" + export EXPECT_PEERS=1 + export SKIP_SERVICE="yes" + export SETUP_ZFS="yes" + export ZFS_FILTER="lxd_disk1" + export ZFS_WIPE="yes" + export SETUP_CEPH="no" + export SETUP_OVN="no" + + # Run a 2 nodes MicroCloud without MicroOVN first. microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q + for m in micro01 micro02 ; do + lxc exec "${m}" -- snap enable microovn + done + + lxc exec micro03 -- microovn cluster bootstrap + lxc exec micro03 -- snap enable microcloud + lxc exec micro03 -- snap start microcloud + + # export EXPECT_PEERS=1 + # export REUSE_EXISTING_COUNT=1 + # export REUSE_EXISTING="add" + # export SETUP_CEPH="no" + # export SETUP_OVN="yes" + # export OVN_FILTER="enp6s0" + # export OVN_UNDERLAY_NETWORK="yes" + # export OVN_UNDERLAY_FILTER="${ovn_underlay_subnet_prefix}" + # export IPV4_SUBNET="10.1.123.1/24" + # export IPV4_START="10.1.123.100" + # export IPV4_END="10.1.123.254" + # export DNS_ADDRESSES="10.1.123.1,8.8.8.8" + # export IPV6_SUBNET="fd42:1:1234:1234::1/64" + # export REPLACE_PROFILE="no" + unset_interactive_vars + export LIMIT_SUBNET="yes" + export EXPECT_PEERS=2 + export PEERS_FILTER="micro03" + export REUSE_EXISTING_COUNT=1 + export REUSE_EXISTING="add" + export SETUP_ZFS="yes" + export ZFS_FILTER="lxd_disk1" + export ZFS_WIPE="yes" + export SETUP_CEPH="no" + export SETUP_OVN="yes" + export OVN_FILTER="enp6s0" + export OVN_UNDERLAY_NETWORK="yes" + export OVN_UNDERLAY_FILTER="${ovn_underlay_subnet_prefix}" + export IPV4_SUBNET="10.1.123.1/24" + export IPV4_START="10.1.123.100" + export IPV4_END="10.1.123.254" + export DNS_ADDRESSES="10.1.123.1,8.8.8.8" + export IPV6_SUBNET="fd42:1:1234:1234::1/64" + export REPLACE_PROFILE="no" + microcloud_interactive | lxc exec micro01 -- sh -c "microcloud add > out 2>&1" + lxc exec micro01 -- cat out + for m in micro01 micro02 micro03 ; do - validate_system_lxd "${m}" 3 disk1 3 1 "${OVN_FILTER}" "${IPV4_SUBNET}" "${IPV4_START}"-"${IPV4_END}" "${IPV6_SUBNET}" - validate_system_microceph "${m}" 1 "${CEPH_CLUSTER_NETWORK}" disk2 + validate_system_lxd "${m}" 3 disk1 0 0 "${OVN_FILTER}" "${IPV4_SUBNET}" "${IPV4_START}"-"${IPV4_END}" "${IPV6_SUBNET}" validate_system_microovn "${m}" "${ovn_underlay_subnet_prefix}" done }