-
Notifications
You must be signed in to change notification settings - Fork 110
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
RSDK-8701 Show configuring state through MachineStatus endpoint #4359
Merged
maximpertsov
merged 11 commits into
viamrobotics:main
from
maximpertsov:RSDK-8701-show-configuring-state
Sep 10, 2024
Merged
Changes from all commits
Commits
Show all changes
11 commits
Select commit
Hold shift + click to select a range
a66fa51
add mid-configure test
maximpertsov 69f70dc
refactor test helpers
maximpertsov 085588f
nolint
maximpertsov 8bfef27
lint
maximpertsov 3d3b4d4
break up tests
maximpertsov 053e482
WIP: release locks and try to read CONFIGURING during reconf
maximpertsov be8a42e
fix test and reinstate all locks to demonstrate bug
maximpertsov 085466e
remove resourceGraphLock
maximpertsov f4628a8
CR@cheukt: try using WaitForAssertion
maximpertsov 1b2eae9
Revert "CR@cheukt: try using WaitForAssertion"
maximpertsov e9adb6a
run ci
maximpertsov File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -6,11 +6,13 @@ import ( | |
"crypto/x509" | ||
"errors" | ||
"fmt" | ||
"log" | ||
"math" | ||
"os" | ||
"path" | ||
"path/filepath" | ||
"strings" | ||
"sync" | ||
"testing" | ||
"time" | ||
|
||
|
@@ -3329,8 +3331,23 @@ type mockResource struct { | |
} | ||
|
||
type mockConfig struct { | ||
Value int `json:"value"` | ||
Fail bool `json:"fail"` | ||
Value int `json:"value"` | ||
Fail bool `json:"fail"` | ||
Sleep string `json:"sleep"` | ||
} | ||
|
||
//nolint:unparam // the resource name is currently always "m" but this could easily change | ||
func newMockConfig(name string, val int, fail bool, sleep string) resource.Config { | ||
return resource.Config{ | ||
Name: name, | ||
Model: mockModel, | ||
API: mockAPI, | ||
// We need to specify both `Attributes` and `ConvertedAttributes`. | ||
// The former triggers a reconfiguration and the former is actually | ||
// used to reconfigure the component. | ||
Attributes: rutils.AttributeMap{"value": val, "fail": fail, "sleep": sleep}, | ||
ConvertedAttributes: &mockConfig{Value: val, Fail: fail, Sleep: sleep}, | ||
} | ||
} | ||
|
||
var errMockValidation = errors.New("whoops") | ||
|
@@ -3368,10 +3385,67 @@ func (m *mockResource) Reconfigure( | |
if err != nil { | ||
return err | ||
} | ||
if mConf.Sleep != "" { | ||
if d, err := time.ParseDuration(mConf.Sleep); err == nil { | ||
log.Printf("sleeping for %s\n", d) | ||
time.Sleep(d) | ||
} | ||
} | ||
m.value = mConf.Value | ||
return nil | ||
} | ||
|
||
// getExpectedDefaultStatuses returns a slice of default [resource.Status] with a given | ||
// revision set for motion and sensor services. | ||
func getExpectedDefaultStatuses(revision string) []resource.Status { | ||
return []resource.Status{ | ||
{ | ||
Name: resource.Name{ | ||
API: resource.APINamespaceRDKInternal.WithServiceType("framesystem"), | ||
Name: "builtin", | ||
}, | ||
State: resource.NodeStateReady, | ||
}, | ||
{ | ||
Name: resource.Name{ | ||
API: resource.APINamespaceRDKInternal.WithServiceType("cloud_connection"), | ||
Name: "builtin", | ||
}, | ||
State: resource.NodeStateReady, | ||
}, | ||
{ | ||
Name: resource.Name{ | ||
API: resource.APINamespaceRDKInternal.WithServiceType("packagemanager"), | ||
Name: "builtin", | ||
}, | ||
State: resource.NodeStateReady, | ||
}, | ||
{ | ||
Name: resource.Name{ | ||
API: resource.APINamespaceRDKInternal.WithServiceType("web"), | ||
Name: "builtin", | ||
}, | ||
State: resource.NodeStateReady, | ||
}, | ||
{ | ||
Name: resource.Name{ | ||
API: resource.APINamespaceRDK.WithServiceType("motion"), | ||
Name: "builtin", | ||
}, | ||
State: resource.NodeStateReady, | ||
Revision: revision, | ||
}, | ||
{ | ||
Name: resource.Name{ | ||
API: resource.APINamespaceRDK.WithServiceType("sensors"), | ||
Name: "builtin", | ||
}, | ||
State: resource.NodeStateReady, | ||
Revision: revision, | ||
}, | ||
} | ||
} | ||
|
||
func TestMachineStatus(t *testing.T) { | ||
logger := logging.NewTestLogger(t) | ||
ctx := context.Background() | ||
|
@@ -3383,93 +3457,32 @@ func TestMachineStatus(t *testing.T) { | |
) | ||
defer resource.Deregister(mockAPI, mockModel) | ||
|
||
rev1 := "rev1" | ||
builtinRev := rev1 | ||
|
||
getExpectedDefaultStatuses := func() []resource.Status { | ||
return []resource.Status{ | ||
{ | ||
Name: resource.Name{ | ||
API: resource.APINamespaceRDKInternal.WithServiceType("framesystem"), | ||
Name: "builtin", | ||
}, | ||
State: resource.NodeStateReady, | ||
}, | ||
{ | ||
Name: resource.Name{ | ||
API: resource.APINamespaceRDKInternal.WithServiceType("cloud_connection"), | ||
Name: "builtin", | ||
}, | ||
State: resource.NodeStateReady, | ||
}, | ||
{ | ||
Name: resource.Name{ | ||
API: resource.APINamespaceRDKInternal.WithServiceType("packagemanager"), | ||
Name: "builtin", | ||
}, | ||
State: resource.NodeStateReady, | ||
}, | ||
{ | ||
Name: resource.Name{ | ||
API: resource.APINamespaceRDKInternal.WithServiceType("web"), | ||
Name: "builtin", | ||
}, | ||
State: resource.NodeStateReady, | ||
}, | ||
{ | ||
Name: resource.Name{ | ||
API: resource.APINamespaceRDK.WithServiceType("motion"), | ||
Name: "builtin", | ||
}, | ||
State: resource.NodeStateReady, | ||
Revision: builtinRev, | ||
}, | ||
{ | ||
Name: resource.Name{ | ||
API: resource.APINamespaceRDK.WithServiceType("sensors"), | ||
Name: "builtin", | ||
}, | ||
State: resource.NodeStateReady, | ||
Revision: builtinRev, | ||
}, | ||
} | ||
} | ||
|
||
t.Run("default resources", func(t *testing.T) { | ||
rev1 := "rev1" | ||
lr := setupLocalRobot(t, ctx, &config.Config{Revision: rev1}, logger) | ||
|
||
mStatus, err := lr.MachineStatus(ctx) | ||
test.That(t, err, test.ShouldBeNil) | ||
test.That(t, mStatus.Config.Revision, test.ShouldEqual, rev1) | ||
|
||
expectedStatuses := getExpectedDefaultStatuses() | ||
expectedStatuses := getExpectedDefaultStatuses(rev1) | ||
rtestutils.VerifySameResourceStatuses(t, mStatus.Resources, expectedStatuses) | ||
}) | ||
|
||
t.Run("reconfigure", func(t *testing.T) { | ||
lr := setupLocalRobot(t, ctx, &config.Config{Revision: rev1}, logger) | ||
|
||
expectedConfigError := fmt.Errorf("resource config validation error: %w", errMockValidation) | ||
t.Run("poll after working and failing reconfigures", func(t *testing.T) { | ||
lr := setupLocalRobot(t, ctx, &config.Config{Revision: "rev1"}, logger) | ||
|
||
// Add a fake resource to the robot. | ||
rev2 := "rev2" | ||
builtinRev = rev2 | ||
lr.Reconfigure(ctx, &config.Config{ | ||
Revision: rev2, | ||
Components: []resource.Config{ | ||
{ | ||
Name: "m", | ||
Model: mockModel, | ||
API: mockAPI, | ||
ConvertedAttributes: &mockConfig{}, | ||
}, | ||
}, | ||
Revision: rev2, | ||
Components: []resource.Config{newMockConfig("m", 0, false, "")}, | ||
}) | ||
mStatus, err := lr.MachineStatus(ctx) | ||
test.That(t, err, test.ShouldBeNil) | ||
test.That(t, mStatus.Config.Revision, test.ShouldEqual, rev2) | ||
expectedStatuses := rtestutils.ConcatResourceStatuses( | ||
getExpectedDefaultStatuses(), | ||
getExpectedDefaultStatuses(rev2), | ||
[]resource.Status{ | ||
{ | ||
Name: mockNamed("m"), | ||
|
@@ -3482,27 +3495,17 @@ func TestMachineStatus(t *testing.T) { | |
|
||
// Update resource config to cause reconfiguration to fail. | ||
rev3 := "rev3" | ||
builtinRev = rev3 | ||
lr.Reconfigure(ctx, &config.Config{ | ||
Revision: rev3, | ||
Components: []resource.Config{ | ||
{ | ||
Name: "m", | ||
Model: mockModel, | ||
API: mockAPI, | ||
// We need to specify both `Attributes` and `ConvertedAttributes`. | ||
// The former triggers a reconfiguration and the former is actually | ||
// used to reconfigure the component. | ||
Attributes: rutils.AttributeMap{"fail": true}, | ||
ConvertedAttributes: &mockConfig{Fail: true}, | ||
}, | ||
}, | ||
Revision: rev3, | ||
Components: []resource.Config{newMockConfig("m", 0, true, "")}, | ||
}) | ||
mStatus, err = lr.MachineStatus(ctx) | ||
test.That(t, err, test.ShouldBeNil) | ||
test.That(t, mStatus.Config.Revision, test.ShouldEqual, rev3) | ||
|
||
expectedConfigError := fmt.Errorf("resource config validation error: %w", errMockValidation) | ||
expectedStatuses = rtestutils.ConcatResourceStatuses( | ||
getExpectedDefaultStatuses(), | ||
getExpectedDefaultStatuses(rev3), | ||
[]resource.Status{ | ||
{ | ||
Name: mockNamed("m"), | ||
|
@@ -3516,27 +3519,15 @@ func TestMachineStatus(t *testing.T) { | |
|
||
// Update resource with a working config. | ||
rev4 := "rev4" | ||
builtinRev = rev4 | ||
lr.Reconfigure(ctx, &config.Config{ | ||
Revision: rev4, | ||
Components: []resource.Config{ | ||
{ | ||
Name: "m", | ||
Model: mockModel, | ||
API: mockAPI, | ||
// We need to specify both `Attributes` and `ConvertedAttributes`. | ||
// The former triggers a reconfiguration and the former is actually | ||
// used to reconfigure the component. | ||
Attributes: rutils.AttributeMap{"value": 200}, | ||
ConvertedAttributes: &mockConfig{Value: 200}, | ||
}, | ||
}, | ||
Revision: rev4, | ||
Components: []resource.Config{newMockConfig("m", 200, false, "")}, | ||
}) | ||
mStatus, err = lr.MachineStatus(ctx) | ||
test.That(t, err, test.ShouldBeNil) | ||
test.That(t, mStatus.Config.Revision, test.ShouldEqual, rev4) | ||
expectedStatuses = rtestutils.ConcatResourceStatuses( | ||
getExpectedDefaultStatuses(), | ||
getExpectedDefaultStatuses(rev4), | ||
[]resource.Status{ | ||
{ | ||
Name: mockNamed("m"), | ||
|
@@ -3547,4 +3538,72 @@ func TestMachineStatus(t *testing.T) { | |
) | ||
rtestutils.VerifySameResourceStatuses(t, mStatus.Resources, expectedStatuses) | ||
}) | ||
|
||
t.Run("poll during reconfiguration", func(t *testing.T) { | ||
rev1 := "rev1" | ||
lr := setupLocalRobot(t, ctx, &config.Config{ | ||
Revision: rev1, | ||
Components: []resource.Config{newMockConfig("m", 200, false, "")}, | ||
}, logger) | ||
|
||
// update resource with a working config that is slow to reconfigure. | ||
rev2 := "rev2" | ||
var wg sync.WaitGroup | ||
wg.Add(1) | ||
go func() { | ||
defer wg.Done() | ||
lr.Reconfigure(ctx, &config.Config{ | ||
Revision: rev2, | ||
Components: []resource.Config{newMockConfig("m", 300, false, "1s")}, | ||
}) | ||
}() | ||
// sleep for a short amount of time to allow the machine to receive a new | ||
// revision. this sleep should be shorter than the resource update duration | ||
// defined above so that updated resource is still in a "configuring" state. | ||
time.Sleep(time.Millisecond * 100) | ||
|
||
// get status while reconfiguring | ||
mStatus, err := lr.MachineStatus(ctx) | ||
test.That(t, err, test.ShouldBeNil) | ||
test.That(t, mStatus.Config.Revision, test.ShouldEqual, rev2) | ||
|
||
// the component whose config changed should be the only component in a | ||
// "configuring" state and associated with the original revision. | ||
filterConfiguring := rtestutils.FilterByStatus(t, mStatus.Resources, resource.NodeStateConfiguring) | ||
expectedConfiguring := []resource.Status{ | ||
{ | ||
Name: mockNamed("m"), | ||
State: resource.NodeStateConfiguring, | ||
Revision: rev1, | ||
}, | ||
} | ||
rtestutils.VerifySameResourceStatuses(t, filterConfiguring, expectedConfiguring) | ||
|
||
// all other components should be in the "ready" state and associated with the | ||
// new revision. | ||
filterReady := rtestutils.FilterByStatus(t, mStatus.Resources, resource.NodeStateReady) | ||
expectedReady := getExpectedDefaultStatuses(rev2) | ||
rtestutils.VerifySameResourceStatuses(t, filterReady, expectedReady) | ||
|
||
wg.Wait() | ||
|
||
// get status after reconfigure finishes | ||
mStatus, err = lr.MachineStatus(ctx) | ||
test.That(t, err, test.ShouldBeNil) | ||
test.That(t, mStatus.Config.Revision, test.ShouldEqual, rev2) | ||
|
||
// now all components, including the one whose config changed, should all be in | ||
// the "ready" state and associated with the new revision. | ||
expectedStatuses := rtestutils.ConcatResourceStatuses( | ||
getExpectedDefaultStatuses(rev2), | ||
[]resource.Status{ | ||
{ | ||
Name: mockNamed("m"), | ||
State: resource.NodeStateReady, | ||
Revision: rev2, | ||
}, | ||
}, | ||
) | ||
rtestutils.VerifySameResourceStatuses(t, mStatus.Resources, expectedStatuses) | ||
}) | ||
Comment on lines
+3542
to
+3608
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. pardon the noisy diff - the test highlighted with this comment is the only new test. the remaining changes are refactors. this test correctly fails if the locked removed in commit 3556889 is reinstated. |
||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
relying on a sleep is a little janky - alternatively we could poll machine status until the revision on the machine is updated and fail after some attempts. any other suggestions are welcome here.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
can do a
WaitForAssertion