diff --git a/functional/platform/cluster.go b/functional/platform/cluster.go index ee47f59cc..244584d6c 100644 --- a/functional/platform/cluster.go +++ b/functional/platform/cluster.go @@ -35,7 +35,9 @@ type Cluster interface { // client operations Fleetctl(m Member, args ...string) (string, string, error) FleetctlWithInput(m Member, input string, args ...string) (string, string, error) + WaitForNUnits(Member, int) (map[string][]util.UnitState, error) WaitForNActiveUnits(Member, int) (map[string][]util.UnitState, error) + WaitForNUnitFiles(Member, int) (map[string][]util.UnitFileState, error) WaitForNMachines(Member, int) ([]string, error) } diff --git a/functional/platform/nspawn.go b/functional/platform/nspawn.go index 51fbbeee4..129c4302a 100644 --- a/functional/platform/nspawn.go +++ b/functional/platform/nspawn.go @@ -128,6 +128,47 @@ func (nc *nspawnCluster) FleetctlWithInput(m Member, input string, args ...strin return util.RunFleetctlWithInput(input, args...) } +// WaitForNUnits runs fleetctl list-units to verify the actual number of units +// matched with the given expected number. It periodically runs list-units +// waiting until list-units actually shows the expected units. +func (nc *nspawnCluster) WaitForNUnits(m Member, expectedUnits int) (map[string][]util.UnitState, error) { + var nUnits int + retStates := make(map[string][]util.UnitState) + checkListUnits := func() bool { + outListUnits, _, err := nc.Fleetctl(m, "list-units", "--no-legend", "--full", "--fields", "unit,active,machine") + if err != nil { + return false + } + // NOTE: There's no need to check if outListUnits is expected to be empty, + // because ParseUnitStates() implicitly filters out such cases. + // However, in case of ParseUnitStates() going away, we should not + // forget about such special cases. + units := strings.Split(strings.TrimSpace(outListUnits), "\n") + allStates := util.ParseUnitStates(units) + nUnits = len(allStates) + if nUnits != expectedUnits { + return false + } + + for _, state := range allStates { + name := state.Name + if _, ok := retStates[name]; !ok { + retStates[name] = []util.UnitState{} + } + retStates[name] = append(retStates[name], state) + } + return true + } + + timeout, err := util.WaitForState(checkListUnits) + if err != nil { + return nil, fmt.Errorf("failed to find %d units within %v (last found: %d)", + expectedUnits, timeout, nUnits) + } + + return retStates, nil +} + func (nc *nspawnCluster) WaitForNActiveUnits(m Member, count int) (map[string][]util.UnitState, error) { var nactive int states := make(map[string][]util.UnitState) @@ -165,6 +206,49 @@ func (nc *nspawnCluster) WaitForNActiveUnits(m Member, count int) (map[string][] return states, nil } +// WaitForNUnitFiles runs fleetctl list-unit-files to verify the actual number of units +// matched with the given expected number. It periodically runs list-unit-files +// waiting until list-unit-files actually shows the expected units. +func (nc *nspawnCluster) WaitForNUnitFiles(m Member, expectedUnits int) (map[string][]util.UnitFileState, error) { + var nUnits int + retStates := make(map[string][]util.UnitFileState) + + checkListUnitFiles := func() bool { + outListUnitFiles, _, err := nc.Fleetctl(m, "list-unit-files", "--no-legend", "--full", "--fields", "unit,dstate,state") + if err != nil { + return false + } + // NOTE: There's no need to check if outListUnits is expected to be empty, + // because ParseUnitFileStates() implicitly filters out such cases. + // However, in case of ParseUnitFileStates() going away, we should not + // forget about such special cases. + units := strings.Split(strings.TrimSpace(outListUnitFiles), "\n") + allStates := util.ParseUnitFileStates(units) + nUnits = len(allStates) + if nUnits != expectedUnits { + // retry until number of units matched + return false + } + + for _, state := range allStates { + name := state.Name + if _, ok := retStates[name]; !ok { + retStates[name] = []util.UnitFileState{} + } + retStates[name] = append(retStates[name], state) + } + return true + } + + timeout, err := util.WaitForState(checkListUnitFiles) + if err != nil { + return nil, fmt.Errorf("failed to find %d units within %v (last found: %d)", + expectedUnits, timeout, nUnits) + } + + return retStates, nil +} + func (nc *nspawnCluster) WaitForNMachines(m Member, count int) ([]string, error) { var machines []string timeout := 10 * time.Second diff --git a/functional/unit_action_test.go b/functional/unit_action_test.go index b9b997756..8d7dab729 100644 --- a/functional/unit_action_test.go +++ b/functional/unit_action_test.go @@ -15,6 +15,8 @@ package functional import ( + "io/ioutil" + "path" "strings" "testing" @@ -53,6 +55,9 @@ func TestUnitRunnable(t *testing.T) { } } +// TestUnitSubmit checks if a unit becomes submitted and destroyed successfully. +// First it submits a unit, and destroys the unit, verifies it's destroyed, +// finally submits the unit again. func TestUnitSubmit(t *testing.T) { cluster, err := platform.NewNspawnCluster("smoke") if err != nil { @@ -69,47 +74,133 @@ func TestUnitSubmit(t *testing.T) { t.Fatal(err) } + unitFile := "fixtures/units/hello.service" + // submit a unit and assert it shows up - if _, _, err := cluster.Fleetctl(m, "submit", "fixtures/units/hello.service"); err != nil { + if _, _, err := cluster.Fleetctl(m, "submit", unitFile); err != nil { t.Fatalf("Unable to submit fleet unit: %v", err) } - stdout, _, err := cluster.Fleetctl(m, "list-units", "--no-legend") + + // wait until the unit gets submitted up to 15 seconds + listUnitStates, err := cluster.WaitForNUnitFiles(m, 1) if err != nil { - t.Fatalf("Failed to run list-units: %v", err) + t.Fatalf("Failed to run list-unit-files: %v", err) } - units := strings.Split(strings.TrimSpace(stdout), "\n") - if len(units) != 1 { - t.Fatalf("Did not find 1 unit in cluster: \n%s", stdout) + + // given unit name must be there in list-unit-files + _, found := listUnitStates[path.Base(unitFile)] + if len(listUnitStates) != 1 || !found { + t.Fatalf("Expected %s to be unit file, got %v", path.Base(unitFile), listUnitStates) } // submitting the same unit should not fail - if _, _, err = cluster.Fleetctl(m, "submit", "fixtures/units/hello.service"); err != nil { + if _, _, err = cluster.Fleetctl(m, "submit", unitFile); err != nil { t.Fatalf("Expected no failure when double-submitting unit, got this: %v", err) } // destroy the unit and ensure it disappears from the unit list - if _, _, err := cluster.Fleetctl(m, "destroy", "fixtures/units/hello.service"); err != nil { + if _, _, err := cluster.Fleetctl(m, "destroy", unitFile); err != nil { t.Fatalf("Failed to destroy unit: %v", err) } - stdout, _, err = cluster.Fleetctl(m, "list-units", "--no-legend") + // wait until the unit gets destroyed up to 15 seconds + listUnitStates, err = cluster.WaitForNUnitFiles(m, 0) if err != nil { - t.Fatalf("Failed to run list-units: %v", err) + t.Fatalf("Failed to run list-unit-files: %v", err) } - if strings.TrimSpace(stdout) != "" { - t.Fatalf("Did not find 0 units in cluster: \n%s", stdout) + if len(listUnitStates) != 0 { + t.Fatalf("Expected nil unit file list, got %v", listUnitStates) } // submitting the unit after destruction should succeed - if _, _, err := cluster.Fleetctl(m, "submit", "fixtures/units/hello.service"); err != nil { + if _, _, err := cluster.Fleetctl(m, "submit", unitFile); err != nil { t.Fatalf("Unable to submit fleet unit: %v", err) } - stdout, _, err = cluster.Fleetctl(m, "list-units", "--no-legend") + + // wait until the unit gets submitted up to 15 seconds + listUnitStates, err = cluster.WaitForNUnitFiles(m, 1) + if err != nil { + t.Fatalf("Failed to run list-unit-files: %v", err) + } + + // given unit name must be there in list-unit-files + _, found = listUnitStates[path.Base(unitFile)] + if len(listUnitStates) != 1 || !found { + t.Fatalf("Expected %s to be unit file, got %v", path.Base(unitFile), listUnitStates) + } +} + +// TestUnitLoad checks if a unit becomes loaded and unloaded successfully. +// First it load a unit, and unloads the unit, verifies it's unloaded, +// finally loads the unit again. +func TestUnitLoad(t *testing.T) { + cluster, err := platform.NewNspawnCluster("smoke") + if err != nil { + t.Fatal(err) + } + defer cluster.Destroy() + + m, err := cluster.CreateMember() + if err != nil { + t.Fatal(err) + } + _, err = cluster.WaitForNMachines(m, 1) + if err != nil { + t.Fatal(err) + } + + unitFile := "fixtures/units/hello.service" + + // load a unit and assert it shows up + _, _, err = cluster.Fleetctl(m, "load", unitFile) + if err != nil { + t.Fatalf("Unable to load fleet unit: %v", err) + } + + // wait until the unit gets loaded up to 15 seconds + listUnitStates, err := cluster.WaitForNUnits(m, 1) + if err != nil { + t.Fatalf("Failed to run list-units: %v", err) + } + + // given unit name must be there in list-units + _, found := listUnitStates[path.Base(unitFile)] + if len(listUnitStates) != 1 || !found { + t.Fatalf("Expected %s to be unit, got %v", path.Base(unitFile), listUnitStates) + } + + // unload the unit and ensure it disappears from the unit list + _, _, err = cluster.Fleetctl(m, "unload", unitFile) + if err != nil { + t.Fatalf("Failed to unload unit: %v", err) + } + + // wait until the unit gets unloaded up to 15 seconds + listUnitStates, err = cluster.WaitForNUnits(m, 0) if err != nil { t.Fatalf("Failed to run list-units: %v", err) } - units = strings.Split(strings.TrimSpace(stdout), "\n") - if len(units) != 1 { - t.Fatalf("Did not find 1 unit in cluster: \n%s", stdout) + + // given unit name must be there in list-units + if len(listUnitStates) != 0 { + t.Fatalf("Expected nil unit list, got %v", listUnitStates) + } + + // loading the unit after destruction should succeed + _, _, err = cluster.Fleetctl(m, "load", unitFile) + if err != nil { + t.Fatalf("Unable to load fleet unit: %v", err) + } + + // wait until the unit gets loaded up to 15 seconds + listUnitStates, err = cluster.WaitForNUnits(m, 1) + if err != nil { + t.Fatalf("Failed to run list-units: %v", err) + } + + // given unit name must be there in list-units + _, found = listUnitStates[path.Base(unitFile)] + if len(listUnitStates) != 1 || !found { + t.Fatalf("Expected %s to be unit, got %v", path.Base(unitFile), listUnitStates) } } @@ -224,3 +315,92 @@ func TestUnitSSHActions(t *testing.T) { t.Errorf("Could not find expected string in journal output:\n%s", stdout) } } + +// TestUnitCat simply compares body of a unit file with that of a unit fetched +// from the remote cluster using "fleetctl cat". +func TestUnitCat(t *testing.T) { + cluster, err := platform.NewNspawnCluster("smoke") + if err != nil { + t.Fatal(err) + } + defer cluster.Destroy() + + m, err := cluster.CreateMember() + if err != nil { + t.Fatal(err) + } + _, err = cluster.WaitForNMachines(m, 1) + if err != nil { + t.Fatal(err) + } + + // read a sample unit file to a buffer + unitFile := "fixtures/units/hello.service" + fileBuf, err := ioutil.ReadFile(unitFile) + if err != nil { + t.Fatal(err) + } + fileBody := strings.TrimSpace(string(fileBuf)) + + // submit a unit and assert it shows up + _, _, err = cluster.Fleetctl(m, "submit", unitFile) + if err != nil { + t.Fatalf("Unable to submit fleet unit: %v", err) + } + // wait until the unit gets submitted up to 15 seconds + _, err = cluster.WaitForNUnitFiles(m, 1) + if err != nil { + t.Fatalf("Failed to run list-units: %v", err) + } + + // cat the unit file and compare it with the original unit body + stdout, _, err := cluster.Fleetctl(m, "cat", path.Base(unitFile)) + if err != nil { + t.Fatalf("Unable to submit fleet unit: %v", err) + } + catBody := strings.TrimSpace(stdout) + + if strings.Compare(catBody, fileBody) != 0 { + t.Fatalf("unit body changed across fleetctl cat: \noriginal:%s\nnew:%s", fileBody, catBody) + } +} + +// TestUnitStatus simply checks "fleetctl status hello.service" actually works. +func TestUnitStatus(t *testing.T) { + cluster, err := platform.NewNspawnCluster("smoke") + if err != nil { + t.Fatal(err) + } + defer cluster.Destroy() + + m, err := cluster.CreateMember() + if err != nil { + t.Fatal(err) + } + _, err = cluster.WaitForNMachines(m, 1) + if err != nil { + t.Fatal(err) + } + + unitFile := "fixtures/units/hello.service" + + // Load a unit and print out status. + // Without loading a unit, it's impossible to run fleetctl status + _, _, err = cluster.Fleetctl(m, "load", unitFile) + if err != nil { + t.Fatalf("Unable to load a fleet unit: %v", err) + } + + // wait until the unit gets loaded up to 15 seconds + _, err = cluster.WaitForNUnits(m, 1) + if err != nil { + t.Fatalf("Failed to run list-units: %v", err) + } + + stdout, stderr, err := cluster.Fleetctl(m, + "--strict-host-key-checking=false", "status", path.Base(unitFile)) + if !strings.Contains(stdout, "Loaded: loaded") { + t.Errorf("Could not find expected string in status output:\n%s\nstderr:\n%s", + stdout, stderr) + } +} diff --git a/functional/util/util.go b/functional/util/util.go index f4e4bdb2f..9439c56c7 100644 --- a/functional/util/util.go +++ b/functional/util/util.go @@ -97,6 +97,12 @@ type UnitState struct { Machine string } +type UnitFileState struct { + Name string + DesiredState string + State string +} + func ParseUnitStates(units []string) (states []UnitState) { for _, unit := range units { cols := strings.Fields(unit) @@ -108,6 +114,16 @@ func ParseUnitStates(units []string) (states []UnitState) { return states } +func ParseUnitFileStates(units []string) (states []UnitFileState) { + for _, unit := range units { + cols := strings.Fields(unit) + if len(cols) == 3 { + states = append(states, UnitFileState{cols[0], cols[1], cols[2]}) + } + } + return states +} + func FilterActiveUnits(states []UnitState) (filtered []UnitState) { for _, state := range states { if state.ActiveState == "active" {