diff --git a/.gitignore b/.gitignore index fa0bd205553..433049f9e15 100644 --- a/.gitignore +++ b/.gitignore @@ -89,3 +89,6 @@ web/cmd/server/*.core # gomobile / android *.aar *.jar + +# direnv (optional dev tool) +.envrc diff --git a/cli/app.go b/cli/app.go index 9e14116ce7a..5c19a4bb446 100644 --- a/cli/app.go +++ b/cli/app.go @@ -51,6 +51,7 @@ const ( moduleFlagLocal = "local" moduleFlagHomeDir = "home" moduleCreateLocalOnly = "local-only" + moduleFlagID = "id" moduleBuildFlagPath = "module" moduleBuildFlagRef = "ref" @@ -1064,7 +1065,7 @@ var app = &cli.App{ { Name: "list", Usage: "list training jobs in Viam cloud based on organization ID", - UsageText: createUsageText("train list", []string{generalFlagOrgID, trainFlagJobStatus}, false), + UsageText: createUsageText("train list", []string{generalFlagOrgID}, true), Flags: []cli.Flag{ &cli.StringFlag{ Name: generalFlagOrgID, @@ -1074,7 +1075,8 @@ var app = &cli.App{ &cli.StringFlag{ Name: trainFlagJobStatus, Usage: "training status to filter for. can be one of " + allTrainingStatusValues(), - Required: true, + Required: false, + Value: defaultTrainingStatus(), }, }, Action: DataListTrainingJobs, @@ -1745,6 +1747,32 @@ This won't work unless you have an existing installation of our GitHub app on yo }, Action: ReloadModuleAction, }, + { + Name: "download", + Usage: "download a module package from the registry", + UsageText: createUsageText("module download", []string{}, false), + Flags: []cli.Flag{ + &cli.PathFlag{ + Name: packageFlagDestination, + Usage: "output directory for downloaded package", + Value: ".", + }, + &cli.StringFlag{ + Name: moduleFlagID, + Usage: "module ID as org-id:name or namespace:name. if missing, will try to read from meta.json", + }, + &cli.StringFlag{ + Name: packageFlagVersion, + Usage: "version of the requested package, can be `latest` to get the most recent version", + Value: "latest", + }, + &cli.StringFlag{ + Name: moduleFlagPlatform, + Usage: "platform like 'linux/amd64'. if missing, will use platform of the CLI binary", + }, + }, + Action: DownloadModuleAction, + }, }, }, { @@ -1756,30 +1784,25 @@ This won't work unless you have an existing installation of our GitHub app on yo Name: "export", Usage: "download a package from Viam cloud", UsageText: createUsageText("packages export", - []string{ - packageFlagDestination, generalFlagOrgID, packageFlagName, - packageFlagVersion, packageFlagType, - }, false), + []string{packageFlagType}, false), Flags: []cli.Flag{ &cli.PathFlag{ - Name: packageFlagDestination, - Required: true, - Usage: "output directory for downloaded package", + Name: packageFlagDestination, + Usage: "output directory for downloaded package", + Value: ".", }, &cli.StringFlag{ - Name: generalFlagOrgID, - Required: true, - Usage: "organization ID of the requested package", + Name: generalFlagOrgID, + Usage: "organization ID or namespace of the requested package. if missing, will try to read from meta.json", }, &cli.StringFlag{ - Name: packageFlagName, - Required: true, - Usage: "name of the requested package", + Name: packageFlagName, + Usage: "name of the requested package. if missing, will try to read from meta.json", }, &cli.StringFlag{ - Name: packageFlagVersion, - Required: true, - Usage: "version of the requested package, can be `latest` to get the most recent version", + Name: packageFlagVersion, + Usage: "version of the requested package, can be `latest` to get the most recent version", + Value: "latest", }, &cli.StringFlag{ Name: packageFlagType, diff --git a/cli/data.go b/cli/data.go index bf694cb7752..e887d8f3486 100644 --- a/cli/data.go +++ b/cli/data.go @@ -271,7 +271,7 @@ func (c *viamClient) binaryData(dst string, filter *datapb.Filter, parallelDownl return c.performActionOnBinaryDataFromFilter( func(id *datapb.BinaryID) error { - return downloadBinary(c.c.Context, c.dataClient, dst, id, c.authFlow.httpClient, c.conf.Auth) + return c.downloadBinary(dst, id) }, filter, parallelDownloads, func(i int32) { @@ -412,35 +412,39 @@ func getMatchingBinaryIDs(ctx context.Context, client datapb.DataServiceClient, } } -func downloadBinary(ctx context.Context, client datapb.DataServiceClient, dst string, id *datapb.BinaryID, - httpClient *http.Client, auth authMethod, -) error { +func (c *viamClient) downloadBinary(dst string, id *datapb.BinaryID) error { + debugf(c.c.App.Writer, c.c.Bool(debugFlag), "Attempting to download binary file %s", id.FileId) + var resp *datapb.BinaryDataByIDsResponse var err error largeFile := false // To begin, we assume the file is small and downloadable, so we try getting the binary directly for count := 0; count < maxRetryCount; count++ { - resp, err = client.BinaryDataByIDs(ctx, &datapb.BinaryDataByIDsRequest{ + resp, err = c.dataClient.BinaryDataByIDs(c.c.Context, &datapb.BinaryDataByIDsRequest{ BinaryIds: []*datapb.BinaryID{id}, IncludeBinary: !largeFile, }) // If the file is too large, we break and try a different pathway for downloading if err == nil || status.Code(err) == codes.ResourceExhausted { + debugf(c.c.App.Writer, c.c.Bool(debugFlag), "Small file download file %s: attempt %d/%d succeeded", id.FileId, count+1, maxRetryCount) break } + debugf(c.c.App.Writer, c.c.Bool(debugFlag), "Small file download for file %s: attempt %d/%d failed", id.FileId, count+1, maxRetryCount) } // For large files, we get the metadata but not the binary itself // Resource exhausted is returned when the message we're receiving exceeds the GRPC maximum limit if err != nil && status.Code(err) == codes.ResourceExhausted { largeFile = true for count := 0; count < maxRetryCount; count++ { - resp, err = client.BinaryDataByIDs(ctx, &datapb.BinaryDataByIDsRequest{ + resp, err = c.dataClient.BinaryDataByIDs(c.c.Context, &datapb.BinaryDataByIDsRequest{ BinaryIds: []*datapb.BinaryID{id}, IncludeBinary: !largeFile, }) if err == nil { + debugf(c.c.App.Writer, c.c.Bool(debugFlag), "Metadata fetch for file %s: attempt %d/%d succeeded", id.FileId, count+1, maxRetryCount) break } + debugf(c.c.App.Writer, c.c.Bool(debugFlag), "Metadata fetch for file %s: attempt %d/%d failed", id.FileId, count+1, maxRetryCount) } } if err != nil { @@ -477,29 +481,43 @@ func downloadBinary(ctx context.Context, client datapb.DataServiceClient, dst st var bin []byte if largeFile { + debugf(c.c.App.Writer, c.c.Bool(debugFlag), "Attempting file %s as a large file download", id.FileId) // Make request to the URI for large files since we exceed the message limit for gRPC - req, err := http.NewRequestWithContext(ctx, http.MethodGet, datum.GetMetadata().GetUri(), nil) + req, err := http.NewRequestWithContext(c.c.Context, http.MethodGet, datum.GetMetadata().GetUri(), nil) if err != nil { return errors.Wrapf(err, serverErrorMessage) } // Set the headers so HTTP requests that are not gRPC calls can still be authenticated in app // We can authenticate via token or API key, so we try both. - token, ok := auth.(*token) + token, ok := c.conf.Auth.(*token) if ok { req.Header.Add(rpc.MetadataFieldAuthorization, rpc.AuthorizationValuePrefixBearer+token.AccessToken) } - apiKey, ok := auth.(*apiKey) + apiKey, ok := c.conf.Auth.(*apiKey) if ok { req.Header.Add("key_id", apiKey.KeyID) req.Header.Add("key", apiKey.KeyCrypto) } - res, err := httpClient.Do(req) + var res *http.Response + for count := 0; count < maxRetryCount; count++ { + res, err = c.authFlow.httpClient.Do(req) + + if err == nil && res.StatusCode == http.StatusOK { + debugf(c.c.App.Writer, c.c.Bool(debugFlag), + "Large file download for file %s: attempt %d/%d succeeded", id.FileId, count+1, maxRetryCount) + break + } + debugf(c.c.App.Writer, c.c.Bool(debugFlag), "Large file download for file %s: attempt %d/%d failed", id.FileId, count+1, maxRetryCount) + } + if err != nil { + debugf(c.c.App.Writer, c.c.Bool(debugFlag), "Failed downloading large file %s: %s", id.FileId, err) return errors.Wrapf(err, serverErrorMessage) } if res.StatusCode != http.StatusOK { + debugf(c.c.App.Writer, c.c.Bool(debugFlag), "Failed downloading large file %s: Server returned %d response", id.FileId, res.StatusCode) return errors.New(serverErrorMessage) } defer func() { @@ -508,6 +526,7 @@ func downloadBinary(ctx context.Context, client datapb.DataServiceClient, dst st bin, err = io.ReadAll(res.Body) if err != nil { + debugf(c.c.App.Writer, c.c.Bool(debugFlag), "Failed downloading large file %s, error occurred while reading: %s", id.FileId, err) return errors.Wrapf(err, serverErrorMessage) } } else { @@ -525,6 +544,7 @@ func downloadBinary(ctx context.Context, client datapb.DataServiceClient, dst st if ext == gzFileExt { r, err = gzip.NewReader(r) if err != nil { + debugf(c.c.App.Writer, c.c.Bool(debugFlag), "Failed unzipping file %s: %s", id.FileId, err) return err } } else if filepath.Ext(dataPath) != ext { @@ -534,18 +554,22 @@ func downloadBinary(ctx context.Context, client datapb.DataServiceClient, dst st } if err := os.MkdirAll(filepath.Dir(dataPath), 0o700); err != nil { + debugf(c.c.App.Writer, c.c.Bool(debugFlag), "Failed creating data directory %s: %s", dataPath, err) return errors.Wrapf(err, "could not create data directory %s", filepath.Dir(dataPath)) } //nolint:gosec dataFile, err := os.Create(dataPath) if err != nil { + debugf(c.c.App.Writer, c.c.Bool(debugFlag), "Failed creating file %s: %s", id.FileId, err) return errors.Wrapf(err, fmt.Sprintf("could not create file for datum %s", datum.GetMetadata().GetId())) } //nolint:gosec if _, err := io.Copy(dataFile, r); err != nil { + debugf(c.c.App.Writer, c.c.Bool(debugFlag), "Failed writing data to file %s: %s", id.FileId, err) return err } if err := r.Close(); err != nil { + debugf(c.c.App.Writer, c.c.Bool(debugFlag), "Failed closing file %s: %s", id.FileId, err) return err } return nil diff --git a/cli/dataset.go b/cli/dataset.go index cd67c48fa1d..4e291043613 100644 --- a/cli/dataset.go +++ b/cli/dataset.go @@ -197,7 +197,7 @@ func (c *viamClient) downloadDataset(dst, datasetID string, includeJSONLines boo return c.performActionOnBinaryDataFromFilter( func(id *datapb.BinaryID) error { - downloadErr := downloadBinary(c.c.Context, c.dataClient, dst, id, c.authFlow.httpClient, c.conf.Auth) + downloadErr := c.downloadBinary(dst, id) var datasetErr error if includeJSONLines { datasetErr = binaryDataToJSONLines(c.c.Context, c.dataClient, dst, datasetFile, id) diff --git a/cli/ml_training.go b/cli/ml_training.go index fe3db077c2d..d3d4a1913d3 100644 --- a/cli/ml_training.go +++ b/cli/ml_training.go @@ -305,6 +305,10 @@ func allTrainingStatusValues() string { return "[" + strings.Join(formattedStatuses, ", ") + "]" } +func defaultTrainingStatus() string { + return strings.ToLower(strings.TrimPrefix(mltrainingpb.TrainingStatus_TRAINING_STATUS_UNSPECIFIED.String(), trainingStatusPrefix)) +} + // MLTrainingUploadAction uploads a new custom training script. func MLTrainingUploadAction(c *cli.Context) error { client, err := newViamClient(c) diff --git a/cli/module_registry.go b/cli/module_registry.go index 7ce7681fe9c..c31b85d1332 100644 --- a/cli/module_registry.go +++ b/cli/module_registry.go @@ -12,7 +12,10 @@ import ( "math" "os" "os/exec" + "path" "path/filepath" + "runtime" + "slices" "strings" "github.com/google/uuid" @@ -877,3 +880,76 @@ func getNextModuleUploadRequest(file *os.File) (*apppb.UploadModuleFileRequest, }, }, nil } + +// DownloadModuleAction downloads a module. +func DownloadModuleAction(c *cli.Context) error { + moduleID := c.String(moduleFlagID) + if moduleID == "" { + manifest, err := loadManifest(defaultManifestFilename) + if err != nil { + return errors.Wrap(err, "trying to get package ID from meta.json") + } + moduleID = manifest.ModuleID + } + client, err := newViamClient(c) + if err != nil { + return err + } + if err := client.ensureLoggedIn(); err != nil { + return err + } + req := &apppb.GetModuleRequest{ModuleId: moduleID} + res, err := client.client.GetModule(c.Context, req) + if err != nil { + return err + } + if len(res.Module.Versions) == 0 { + return errors.New("module has 0 uploaded versions, nothing to download") + } + requestedVersion := c.String(packageFlagVersion) + var ver *apppb.VersionHistory + if requestedVersion == "latest" { + ver = res.Module.Versions[len(res.Module.Versions)-1] + } else { + for _, iVer := range res.Module.Versions { + if iVer.Version == requestedVersion { + ver = iVer + break + } + } + if ver == nil { + return fmt.Errorf("version %s not found in versions for module", requestedVersion) + } + } + infof(c.App.ErrWriter, "found version %s", ver.Version) + if len(ver.Files) == 0 { + return fmt.Errorf("version %s has 0 files uploaded", ver.Version) + } + platform := c.String(moduleFlagPlatform) + if platform == "" { + platform = fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH) + infof(c.App.ErrWriter, "using default platform %s", platform) + } + if !slices.ContainsFunc(ver.Files, func(file *apppb.Uploads) bool { return file.Platform == platform }) { + return fmt.Errorf("platform %s not present for version %s", platform, ver.Version) + } + include := true + packageType := packagespb.PackageType_PACKAGE_TYPE_MODULE + // note: this is working around a GetPackage quirk where platform messes with version + fullVersion := fmt.Sprintf("%s-%s", ver.Version, strings.ReplaceAll(platform, "/", "-")) + pkg, err := client.packageClient.GetPackage(c.Context, &packagespb.GetPackageRequest{ + Id: strings.ReplaceAll(moduleID, ":", "/"), + Version: fullVersion, + IncludeUrl: &include, + Type: &packageType, + }) + if err != nil { + return err + } + destName := strings.ReplaceAll(moduleID, ":", "-") + infof(c.App.ErrWriter, "saving to %s", path.Join(c.String(packageFlagDestination), fullVersion, destName+".tar.gz")) + return downloadPackageFromURL(c.Context, client.authFlow.httpClient, + c.String(packageFlagDestination), destName, + fullVersion, pkg.Package.Url, client.conf.Auth, + ) +} diff --git a/cli/packages.go b/cli/packages.go index 53787264391..2b76e15ec6f 100644 --- a/cli/packages.go +++ b/cli/packages.go @@ -84,6 +84,16 @@ func (c *viamClient) packageExportAction(orgID, name, version, packageType, dest if err := c.ensureLoggedIn(); err != nil { return err } + if orgID == "" || name == "" { + if orgID != "" || name != "" { + return fmt.Errorf("if either of %s or %s is missing, both must be missing", generalFlagOrgID, packageFlagName) + } + manifest, err := loadManifest(defaultManifestFilename) + if err != nil { + return errors.Wrap(err, "trying to get package ID from meta.json") + } + orgID, name, _ = strings.Cut(manifest.ModuleID, ":") + } // Package ID is the / packageID := path.Join(orgID, name) packageTypeProto, err := convertPackageTypeToProto(packageType) diff --git a/components/arm/arm.go b/components/arm/arm.go index 38cc02dab8a..d01d6502a3f 100644 --- a/components/arm/arm.go +++ b/components/arm/arm.go @@ -1,6 +1,9 @@ //go:build !no_cgo // Package arm defines the arm that a robot uses to manipulate objects. +// For more information, see the [arm component docs]. +// +// [arm component docs]: https://docs.viam.com/components/arm/ package arm import ( @@ -48,6 +51,7 @@ func Named(name string) resource.Name { } // An Arm represents a physical robotic arm that exists in three-dimensional space. +// For more information, see the [arm component docs]. // // EndPosition example: // @@ -87,6 +91,8 @@ func Named(name string) resource.Name { // // // Get the current position of each joint on the arm as JointPositions. // pos, err := myArm.JointPositions(context.Background(), nil) +// +// [arm component docs]: https://docs.viam.com/components/arm/ type Arm interface { resource.Resource referenceframe.ModelFramer diff --git a/components/base/base.go b/components/base/base.go index 4a6cb3f17e4..7150bad73a3 100644 --- a/components/base/base.go +++ b/components/base/base.go @@ -1,4 +1,7 @@ // Package base defines the base that a robot uses to move around. +// For more information, see the [base component docs]. +// +// [base component docs]: https://docs.viam.com/components/base/ package base import ( @@ -34,6 +37,7 @@ func Named(name string) resource.Name { } // A Base represents a physical base of a robot. +// For more information, see the [base component docs]. // // MoveStraight example: // @@ -93,6 +97,8 @@ func Named(name string) resource.Name { // // // Get the wheel circumference // myBaseWheelCircumference := properties.WheelCircumferenceMeters +// +// [base component docs]: https://docs.viam.com/components/base/ type Base interface { resource.Resource resource.Actuator diff --git a/components/board/board.go b/components/board/board.go index 2b965259511..be6a8cffad9 100644 --- a/components/board/board.go +++ b/components/board/board.go @@ -2,6 +2,9 @@ // such as a Raspberry Pi. // // Besides the board itself, some other interfaces it defines are analog pins and digital interrupts. +// For more information, see the [board component docs]. +// +// [board component docs]: https://docs.viam.com/components/board/ package board import ( @@ -46,6 +49,7 @@ func Named(name string) resource.Name { // A Board represents a physical general purpose board that contains various // components such as analogs, and digital interrupts. +// For more information, see the [board component docs]. // // AnalogByName example: // @@ -93,6 +97,8 @@ func Named(name string) resource.Name { // } // // err = myBoard.StreamTicks(context.Background(), interrupts, ticksChan, nil) +// +// [board component docs]: https://docs.viam.com/components/board/ type Board interface { resource.Resource diff --git a/components/board/fake/board.go b/components/board/fake/board.go index ff7f6204f6d..95821b56bee 100644 --- a/components/board/fake/board.go +++ b/components/board/fake/board.go @@ -18,7 +18,6 @@ import ( "go.viam.com/rdk/grpc" "go.viam.com/rdk/logging" "go.viam.com/rdk/resource" - rdkutils "go.viam.com/rdk/utils" ) // In order to maintain test functionality, testPin will always return the analog value it is set @@ -83,7 +82,7 @@ func NewBoard(ctx context.Context, conf resource.Config, logger logging.Logger) Analogs: map[string]*Analog{}, Digitals: map[string]*DigitalInterrupt{}, GPIOPins: map[string]*GPIOPin{}, - workers: rdkutils.NewStoppableWorkers(), + workers: utils.NewBackgroundStoppableWorkers(), logger: logger, } @@ -167,7 +166,7 @@ type Board struct { logger logging.Logger CloseCount int - workers rdkutils.StoppableWorkers + workers *utils.StoppableWorkers } // AnalogByName returns the analog pin by the given name if it exists. @@ -247,7 +246,7 @@ func (b *Board) StreamTicks(ctx context.Context, interrupts []board.DigitalInter for _, di := range interrupts { // Don't need to check if interrupt exists, just did that above - b.workers.AddWorkers(func(workersContext context.Context) { + b.workers.Add(func(workersContext context.Context) { for { // sleep to avoid a busy loop if !utils.SelectContextOrWait(workersContext, 700*time.Millisecond) { diff --git a/components/board/genericlinux/board.go b/components/board/genericlinux/board.go index ada3b2c8b55..bf2e911f0c6 100644 --- a/components/board/genericlinux/board.go +++ b/components/board/genericlinux/board.go @@ -15,6 +15,7 @@ import ( "github.com/pkg/errors" "go.uber.org/multierr" pb "go.viam.com/api/component/board/v1" + "go.viam.com/utils" "go.viam.com/rdk/components/board" "go.viam.com/rdk/components/board/genericlinux/buses" @@ -23,7 +24,6 @@ import ( "go.viam.com/rdk/grpc" "go.viam.com/rdk/logging" "go.viam.com/rdk/resource" - "go.viam.com/rdk/utils" ) // RegisterBoard registers a sysfs based board of the given model. @@ -55,7 +55,7 @@ func NewBoard( convertConfig: convertConfig, logger: logger, - workers: utils.NewStoppableWorkers(), + workers: utils.NewBackgroundStoppableWorkers(), analogReaders: map[string]*wrappedAnalogReader{}, gpios: map[string]*gpioPin{}, @@ -343,7 +343,7 @@ func (b *Board) createGpioPin(mapping GPIOBoardMapping) *gpioPin { logger: b.logger, startSoftwarePWMChan: &startSoftwarePWMChan, } - pin.softwarePwm = utils.NewStoppableWorkers(pin.softwarePwmLoop) + pin.softwarePwm = utils.NewBackgroundStoppableWorkers(pin.softwarePwmLoop) if mapping.HWPWMSupported { pin.hwPwm = newPwmDevice(mapping.PWMSysFsDir, mapping.PWMID, b.logger) } @@ -363,7 +363,7 @@ type Board struct { gpios map[string]*gpioPin interrupts map[string]*digitalInterrupt - workers utils.StoppableWorkers + workers *utils.StoppableWorkers } // AnalogByName returns the analog pin by the given name if it exists. @@ -477,7 +477,7 @@ func (b *Board) StreamTicks(ctx context.Context, interrupts []board.DigitalInter i.AddChannel(ch) } - b.workers.AddWorkers(func(cancelCtx context.Context) { + b.workers.Add(func(cancelCtx context.Context) { // Wait until it's time to shut down then remove callbacks. select { case <-ctx.Done(): diff --git a/components/board/genericlinux/digital_interrupts.go b/components/board/genericlinux/digital_interrupts.go index ad19b4be207..9dfdfe693e9 100644 --- a/components/board/genericlinux/digital_interrupts.go +++ b/components/board/genericlinux/digital_interrupts.go @@ -15,11 +15,10 @@ import ( "go.viam.com/utils" "go.viam.com/rdk/components/board" - rdkutils "go.viam.com/rdk/utils" ) type digitalInterrupt struct { - workers rdkutils.StoppableWorkers + workers *utils.StoppableWorkers line *gpio.LineWithEvent mu sync.Mutex // Protects everything below here config board.DigitalInterruptConfig @@ -48,7 +47,7 @@ func newDigitalInterrupt( } di := digitalInterrupt{line: line, config: config} - di.workers = rdkutils.NewStoppableWorkers(di.monitor) + di.workers = utils.NewBackgroundStoppableWorkers(di.monitor) if oldInterrupt != nil { oldInterrupt.mu.Lock() diff --git a/components/board/genericlinux/gpio.go b/components/board/genericlinux/gpio.go index f8aa5f09644..680e34104e1 100644 --- a/components/board/genericlinux/gpio.go +++ b/components/board/genericlinux/gpio.go @@ -16,7 +16,6 @@ import ( "go.viam.com/rdk/components/board" "go.viam.com/rdk/logging" - rdkutils "go.viam.com/rdk/utils" ) const noPin = 0xFFFFFFFF // noPin is the uint32 version of -1. A pin with this offset has no GPIO @@ -35,7 +34,7 @@ type gpioPin struct { enableSoftwarePWM bool // Indicates whether a software PWM loop should continue running startSoftwarePWMChan *chan any // Close and reinitialize this to (re)start the SW PWM loop - softwarePwm rdkutils.StoppableWorkers + softwarePwm *utils.StoppableWorkers mu sync.Mutex logger logging.Logger diff --git a/components/board/numato/board.go b/components/board/numato/board.go index 2ae7f6665e9..b6d724f389c 100644 --- a/components/board/numato/board.go +++ b/components/board/numato/board.go @@ -26,7 +26,6 @@ import ( "go.viam.com/rdk/grpc" "go.viam.com/rdk/logging" "go.viam.com/rdk/resource" - rdkutils "go.viam.com/rdk/utils" ) var model = resource.DefaultModelFamily.WithModel("numato") @@ -116,7 +115,7 @@ type numatoBoard struct { sent map[string]bool sentMu sync.Mutex - workers rdkutils.StoppableWorkers + workers *utils.StoppableWorkers maxAnalogVoltage float32 stepSize float32 @@ -465,7 +464,7 @@ func connect(ctx context.Context, name resource.Name, conf *Config, logger loggi b.lines = make(chan string) - b.workers = rdkutils.NewStoppableWorkers(b.readThread) + b.workers = utils.NewBackgroundStoppableWorkers(b.readThread) ver, err := b.doSendReceive(ctx, "ver") if err != nil { diff --git a/components/board/pinwrappers/analog_smoother.go b/components/board/pinwrappers/analog_smoother.go index 046255e2344..17ea24f79b1 100644 --- a/components/board/pinwrappers/analog_smoother.go +++ b/components/board/pinwrappers/analog_smoother.go @@ -28,7 +28,7 @@ type AnalogSmoother struct { lastData atomic.Pointer[board.AnalogValue] lastError atomic.Pointer[errValue] logger logging.Logger - workers utils.StoppableWorkers + workers *goutils.StoppableWorkers } // SmoothAnalogReader wraps the given reader in a smoother. @@ -116,7 +116,7 @@ func (as *AnalogSmoother) Start() { as.data = nil } - as.workers = utils.NewStoppableWorkers(func(ctx context.Context) { + as.workers = goutils.NewBackgroundStoppableWorkers(func(ctx context.Context) { consecutiveErrors := 0 var lastError error diff --git a/components/camera/camera.go b/components/camera/camera.go index bf9c385dfde..ec1751d53b1 100644 --- a/components/camera/camera.go +++ b/components/camera/camera.go @@ -1,4 +1,7 @@ // Package camera defines an image capturing device. +// For more information, see the [camera component docs]. +// +// [camera component docs]: https://docs.viam.com/components/camera/ package camera import ( @@ -75,6 +78,7 @@ type Camera interface { } // A VideoSource represents anything that can capture frames. +// For more information, see the [camera component docs]. // // Images example: // @@ -105,6 +109,8 @@ type Camera interface { // myCamera, err := camera.FromRobot(machine, "my_camera") // // err = myCamera.Close(ctx) +// +// [camera component docs]: https://docs.viam.com/components/camera/ type VideoSource interface { // Images is used for getting simultaneous images from different imagers, // along with associated metadata (just timestamp for now). It's not for getting a time series of images from the same imager. diff --git a/components/camera/client.go b/components/camera/client.go index 69294a5b2a6..1f296876859 100644 --- a/components/camera/client.go +++ b/components/camera/client.go @@ -3,14 +3,16 @@ package camera import ( "bytes" "context" + "errors" "fmt" "image" "io" - "slices" + "os" "sync" + "sync/atomic" + "time" "github.com/pion/rtp" - "github.com/pkg/errors" "github.com/viamrobotics/webrtc/v3" "go.opencensus.io/trace" pb "go.viam.com/api/component/camera/v1" @@ -18,13 +20,14 @@ import ( goutils "go.viam.com/utils" goprotoutils "go.viam.com/utils/protoutils" "go.viam.com/utils/rpc" + "golang.org/x/exp/slices" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/structpb" "go.viam.com/rdk/components/camera/rtppassthrough" "go.viam.com/rdk/data" "go.viam.com/rdk/gostream" - rdkgrpc "go.viam.com/rdk/grpc" + "go.viam.com/rdk/grpc" "go.viam.com/rdk/logging" "go.viam.com/rdk/pointcloud" "go.viam.com/rdk/protoutils" @@ -36,28 +39,24 @@ import ( var ( // ErrNoPeerConnection indicates there was no peer connection. - ErrNoPeerConnection = errors.New("No PeerConnection") + ErrNoPeerConnection = errors.New("no PeerConnection") // ErrNoSharedPeerConnection indicates there was no shared peer connection. - ErrNoSharedPeerConnection = errors.New("No Shared PeerConnection") + ErrNoSharedPeerConnection = errors.New("no Shared PeerConnection") // ErrUnknownSubscriptionID indicates that a SubscriptionID is unknown. - ErrUnknownSubscriptionID = errors.New("SubscriptionID Unknown") + ErrUnknownSubscriptionID = errors.New("subscriptionID Unknown") + readRTPTimeout = time.Millisecond * 200 ) -type ( - singlePacketCallback func(*rtp.Packet) - bufAndCB struct { - cb singlePacketCallback - buf *rtppassthrough.Buffer - } - bufAndCBByID map[rtppassthrough.SubscriptionID]bufAndCB -) +type bufAndCB struct { + cb func(*rtp.Packet) + buf *rtppassthrough.Buffer +} // client implements CameraServiceClient. type client struct { resource.Named resource.TriviallyReconfigurable - ctx context.Context - cancelFn context.CancelFunc + remoteName string name string conn rpc.ClientConn client pb.CameraServiceClient @@ -65,12 +64,14 @@ type client struct { logger logging.Logger activeBackgroundWorkers sync.WaitGroup - mu sync.Mutex - healthyClientCh chan struct{} - bufAndCBByID bufAndCBByID - currentSubParentID rtppassthrough.SubscriptionID - subParentToChildren map[rtppassthrough.SubscriptionID][]rtppassthrough.SubscriptionID - trackClosed <-chan struct{} + healthyClientChMu sync.Mutex + healthyClientCh chan struct{} + + rtpPassthroughMu sync.Mutex + runningStreams map[rtppassthrough.SubscriptionID]bufAndCB + subGenerationID int + associatedSubs map[int][]rtppassthrough.SubscriptionID + trackClosed <-chan struct{} } // NewClientFromConn constructs a new Client from connection passed in. @@ -85,19 +86,17 @@ func NewClientFromConn( streamClient := streampb.NewStreamServiceClient(conn) trackClosed := make(chan struct{}) close(trackClosed) - closeCtx, cancelFn := context.WithCancel(context.Background()) return &client{ - ctx: closeCtx, - cancelFn: cancelFn, - Named: name.PrependRemote(remoteName).AsNamed(), - name: name.ShortName(), - conn: conn, - streamClient: streamClient, - client: c, - bufAndCBByID: map[rtppassthrough.SubscriptionID]bufAndCB{}, - trackClosed: trackClosed, - subParentToChildren: map[rtppassthrough.SubscriptionID][]rtppassthrough.SubscriptionID{}, - logger: logger, + remoteName: remoteName, + Named: name.PrependRemote(remoteName).AsNamed(), + name: name.ShortName(), + conn: conn, + streamClient: streamClient, + client: c, + runningStreams: map[rtppassthrough.SubscriptionID]bufAndCB{}, + trackClosed: trackClosed, + associatedSubs: map[int][]rtppassthrough.SubscriptionID{}, + logger: logger, }, nil } @@ -181,12 +180,7 @@ func (c *client) Stream( // New streams concurrent with closing cannot start until this drain completes. There // will never be stream goroutines from the old "generation" running concurrently // with those from the new "generation". - c.mu.Lock() - if c.healthyClientCh == nil { - c.healthyClientCh = make(chan struct{}) - } - healthyClientCh := c.healthyClientCh - c.mu.Unlock() + healthyClientCh := c.maybeResetHealthyClientCh() ctxWithMIME := gostream.WithMIMETypeHint(context.Background(), gostream.MIMETypeHint(ctx, "")) streamCtx, stream, frameCh := gostream.NewMediaStreamForChannel[image.Image](ctxWithMIME) @@ -240,7 +234,7 @@ func (c *client) Images(ctx context.Context) ([]NamedImage, resource.ResponseMet Name: c.name, }) if err != nil { - return nil, resource.ResponseMetadata{}, errors.Wrap(err, "camera client: could not gets images from the camera") + return nil, resource.ResponseMetadata{}, fmt.Errorf("camera client: could not gets images from the camera %w", err) } images := make([]NamedImage, 0, len(resp.Images)) @@ -300,21 +294,6 @@ func (c *client) NextPointCloud(ctx context.Context) (pointcloud.PointCloud, err }() } -func (c *client) Projector(ctx context.Context) (transform.Projector, error) { - var proj transform.Projector - props, err := c.Properties(ctx) - if err != nil { - return nil, err - } - intrinsics := props.IntrinsicParams - err = intrinsics.CheckValid() - if err != nil { - return nil, err - } - proj = intrinsics - return proj, nil -} - func (c *client) Properties(ctx context.Context) (Properties, error) { result := Properties{} resp, err := c.client.GetProperties(ctx, &pb.GetPropertiesRequest{ @@ -364,35 +343,38 @@ func (c *client) Close(ctx context.Context) error { _, span := trace.StartSpan(ctx, "camera::client::Close") defer span.End() - c.cancelFn() - c.mu.Lock() - + c.healthyClientChMu.Lock() if c.healthyClientCh != nil { close(c.healthyClientCh) } c.healthyClientCh = nil + c.healthyClientChMu.Unlock() + // unsubscribe from all video streams that have been established with modular cameras c.unsubscribeAll() // NOTE: (Nick S) we are intentionally releasing the lock before we wait for // background goroutines to terminate as some of them need to be able // to take the lock to terminate - c.mu.Unlock() c.activeBackgroundWorkers.Wait() return nil } -// SubscribeRTP begins a subscription to receive RTP packets. -// When the Subscription terminates the context in the returned Subscription -// is cancelled. -// It maintains the invariant that there is at most a single track between -// the client and WebRTC peer. -// It is strongly recommended to set a timeout on ctx as the underlying -// WebRTC peer connection's Track can be removed before sending an RTP packet which -// results in SubscribeRTP blocking until the provided context is cancelled or the client -// is closed. -// This rare condition may happen in cases like reconfiguring concurrently with -// a SubscribeRTP call. +func (c *client) maybeResetHealthyClientCh() chan struct{} { + c.healthyClientChMu.Lock() + defer c.healthyClientChMu.Unlock() + if c.healthyClientCh == nil { + c.healthyClientCh = make(chan struct{}) + } + return c.healthyClientCh +} + +// SubscribeRTP begins a subscription to receive RTP packets. SubscribeRTP waits for the VideoTrack +// to be available before returning. The subscription/video is valid until the +// `Subscription.Terminated` context is `Done`. +// +// SubscribeRTP maintains the invariant that there is at most a single track between the client and +// WebRTC peer. Concurrent callers will block and wait for the "winner" to complete. func (c *client) SubscribeRTP( ctx context.Context, bufferSize int, @@ -400,271 +382,355 @@ func (c *client) SubscribeRTP( ) (rtppassthrough.Subscription, error) { ctx, span := trace.StartSpan(ctx, "camera::client::SubscribeRTP") defer span.End() - c.mu.Lock() - defer c.mu.Unlock() - sub, buf, err := rtppassthrough.NewSubscription(bufferSize) + // RSDK-6340: The resource manager closes remote resources when the underlying + // connection goes bad. However, when the connection is re-established, the client + // objects these resources represent are not re-initialized/marked "healthy". + // `healthyClientCh` helps track these transitions between healthy and unhealthy + // states. + // + // When `client.SubscribeRTP()` is called we will either use the existing + // `healthyClientCh` or create a new one. + // + // The goroutine a `Tracker.AddOnTrackSub()` method spins off will listen to its version of the + // `healthyClientCh` to be notified when the connection has died so it can gracefully + // terminate. + // + // When a connection becomes unhealthy, the resource manager will call `Close` on the + // camera client object. Closing the client will: + // 1. close its `client.healthyClientCh` channel + // 2. wait for existing "SubscribeRTP" goroutines to drain + // 3. nil out the `client.healthyClientCh` member variable + // + // New streams concurrent with closing cannot start until this drain completes. There + // will never be stream goroutines from the old "generation" running concurrently + // with those from the new "generation". + healthyClientCh := c.maybeResetHealthyClientCh() + + c.rtpPassthroughMu.Lock() + defer c.rtpPassthroughMu.Unlock() + + // Create a Subscription object and allocate a ring buffer of RTP packets. + sub, rtpPacketBuffer, err := rtppassthrough.NewSubscription(bufferSize) if err != nil { return sub, err } g := utils.NewGuard(func() { - buf.Close() + c.logger.CInfo(ctx, "Error subscribing to RTP. Closing passthrough buffer.") + rtpPacketBuffer.Close() }) defer g.OnFail() + // RTP Passthrough only works over PeerConnection video tracks. if c.conn.PeerConn() == nil { return rtppassthrough.NilSubscription, ErrNoPeerConnection } - // check if we have established a connection that can be shared by multiple clients asking for cameras streams from viam server. - sc, ok := c.conn.(*rdkgrpc.SharedConn) + // Calling `AddStream(streamId)` on the remote viam-server/module will eventually result in + // invoking a local `PeerConnection.OnTrack(streamId, rtpReceiver)` callback. In this callback + // we set up the goroutine to read video data from the remote side and copy those packets + // upstream. + // + // webrtc.PeerConnection objects can only install a single `OnTrack` callback. Because there + // could be multiple video stream sources on the other side of a PeerConnection, we must play a + // game where we only create one `OnTrack` callback, but that callback object does a map lookup + // for which video stream. The `grpc.Tracker` API is for manipulating that map. + tracker, ok := c.conn.(grpc.Tracker) if !ok { + c.logger.CErrorw(ctx, "Client conn is not a `Tracker`", "connType", fmt.Sprintf("%T", c.conn)) return rtppassthrough.NilSubscription, ErrNoSharedPeerConnection } - c.logger.CDebugw(ctx, "SubscribeRTP", "subID", sub.ID.String(), "name", c.Name(), "bufAndCBByID", c.bufAndCBByID.String()) + c.logger.CDebugw(ctx, "SubscribeRTP", "subID", sub.ID.String(), "name", c.Name(), "bufAndCBByID", c.bufAndCBToString()) defer func() { c.logger.CDebugw(ctx, "SubscribeRTP after", "subID", sub.ID.String(), - "name", c.Name(), "bufAndCBByID", c.bufAndCBByID.String()) + "name", c.Name(), "bufAndCBByID", c.bufAndCBToString()) }() - // B/c there is only ever either 0 or 1 peer connections between a module & a viam-server - // once AddStream is called on the module for a given camera model instance & succeeds, we shouldn't - // call it again until the previous track is terminated (by calling RemoveStream) for a few reasons: - // 1. doing so would result in 2 webrtc tracks for the same camera sending the exact same RTP packets which would - // needlessly waste resources - // 2. b/c the signature of RemoveStream just takes the camera name, if there are 2 streams for the same camera - // & the module receives a call to RemoveStream, there is no way for the module to know which camera stream - // should be removed - if len(c.bufAndCBByID) == 0 { - // Wait for previous track to terminate or for the client to close + + // If we do not currently have a video stream open for this camera, we create one. Otherwise + // we'll piggy back on the existing stream. We piggy-back for two reasons: + // 1. Not doing so would result in two webrtc tracks for the same camera sending the exact same + // RTP packets. This would needlessly waste resources. + // 2. The signature of `RemoveStream` just takes the camera name. If we had two streams open for + // the same camera when the remote receives `RemoveStream`, it wouldn't know which to stop + // sending data for. + if len(c.runningStreams) == 0 { + c.logger.CInfow(ctx, "SubscribeRTP is creating a video track", + "client", fmt.Sprintf("%p", c), "peerConnection", fmt.Sprintf("%p", c.conn.PeerConn())) + // A previous subscriber/track may have exited, but its resources have not necessarily been + // cleaned up. We must wait for that to complete. We additionally select on other error + // conditions. select { - case <-ctx.Done(): - err := errors.Wrap(ctx.Err(), "Track not closed within SubscribeRTP provided context") - c.logger.Error(err) - return rtppassthrough.NilSubscription, err - case <-c.ctx.Done(): - err := errors.Wrap(c.ctx.Err(), "Track not closed before client closed") - c.logger.Error(err) - return rtppassthrough.NilSubscription, err case <-c.trackClosed: + case <-ctx.Done(): + return rtppassthrough.NilSubscription, fmt.Errorf("track not closed within SubscribeRTP provided context %w", ctx.Err()) + case <-healthyClientCh: + return rtppassthrough.NilSubscription, errors.New("camera client is closed") } + + // Create channels for two important events: when a video track is established an when a + // video track exits. I.e: When the `OnTrack` callback is invoked and when the `RTPReceiver` + // is no longer being read from. trackReceived, trackClosed := make(chan struct{}), make(chan struct{}) - // add the camera model's addOnTrackSubFunc to the shared peer connection's - // slice of OnTrack callbacks. This is what allows - // all the bufAndCBByID's callback functions to be called with the - // RTP packets from the module's peer connection's track - sc.AddOnTrackSub(c.Name(), c.addOnTrackSubFunc(trackReceived, trackClosed, sub.ID)) - // remove the OnTrackSub once we either fail or succeed - defer sc.RemoveOnTrackSub(c.Name()) - if _, err := c.streamClient.AddStream(ctx, &streampb.AddStreamRequest{Name: c.Name().String()}); err != nil { - c.logger.CDebugw(ctx, "SubscribeRTP AddStream hit error", "subID", sub.ID.String(), "name", c.Name(), "err", err) + + // We're creating a new video track. Bump the generation ID and associate all new + // subscriptions to this generation. + c.subGenerationID++ + c.associatedSubs[c.subGenerationID] = []rtppassthrough.SubscriptionID{} + + // Add the camera's addOnTrackSubFunc to the SharedConn's map of OnTrack callbacks. + tracker.AddOnTrackSub(c.trackName(), c.addOnTrackFunc(healthyClientCh, trackReceived, trackClosed, c.subGenerationID)) + // Remove the OnTrackSub once we either fail or succeed. + defer tracker.RemoveOnTrackSub(c.trackName()) + + // Call `AddStream` on the remote. In the successful case, this will result in a + // PeerConnection renegotiation to add this camera's video track and have the `OnTrack` + // callback invoked. + if _, err := c.streamClient.AddStream(ctx, &streampb.AddStreamRequest{Name: c.trackName()}); err != nil { + c.logger.CDebugw(ctx, "SubscribeRTP AddStream hit error", "subID", sub.ID.String(), "trackName", c.trackName(), "err", err) return rtppassthrough.NilSubscription, err } - // NOTE: (Nick S) This is a workaround to a Pion bug / missing feature. - // If the WebRTC peer on the other side of the PeerConnection calls pc.AddTrack followd by pc.RemoveTrack - // before the module writes RTP packets - // to the track, the client's PeerConnection.OnTrack callback is never called. - // That results in the client never receiving RTP packets on subscriptions which never terminate - // which is an unacceptable failure mode. - - // To prevent that failure mode, we exit with an error if a track is not received within - // the SubscribeRTP context. + c.logger.CDebugw(ctx, "SubscribeRTP waiting for track", "client", fmt.Sprintf("%p", c), "pc", fmt.Sprintf("%p", c.conn.PeerConn())) select { case <-ctx.Done(): - err := errors.Wrap(ctx.Err(), "Track not received within SubscribeRTP provided context") - c.logger.Error(err.Error()) - return rtppassthrough.NilSubscription, err - case <-c.ctx.Done(): - err := errors.Wrap(c.ctx.Err(), "Track not received before client closed") - c.logger.Error(err) - return rtppassthrough.NilSubscription, err + return rtppassthrough.NilSubscription, fmt.Errorf("track not received within SubscribeRTP provided context %w", ctx.Err()) + case <-healthyClientCh: + return rtppassthrough.NilSubscription, errors.New("track not received before client closed") case <-trackReceived: - c.logger.Debug("received track") + c.logger.CDebugw(ctx, "SubscribeRTP received track data", "client", fmt.Sprintf("%p", c), "pc", fmt.Sprintf("%p", c.conn.PeerConn())) } - // set up channel so we can detect when the track has closed (in response to an event / error internal to the - // peer or due to calling RemoveStream) + + // Set up channel to detect when the track has closed. This can happen in response to an + // event / error internal to the peer or due to calling `RemoveStream`. c.trackClosed = trackClosed - // the sub is the new parent of all subsequent subs until the number if subsriptions falls back to 0 - c.currentSubParentID = sub.ID - c.subParentToChildren[c.currentSubParentID] = []rtppassthrough.SubscriptionID{} - c.logger.CDebugw(ctx, "SubscribeRTP called AddStream and succeeded", "subID", sub.ID.String(), - "name", c.Name()) - } - c.subParentToChildren[c.currentSubParentID] = append(c.subParentToChildren[c.currentSubParentID], sub.ID) - // add the subscription to bufAndCBByID so the goroutine spawned by - // addOnTrackSubFunc can forward the packets it receives from the modular camera - // over WebRTC to the SubscribeRTP caller via the packetsCB callback - c.bufAndCBByID[sub.ID] = bufAndCB{ + c.logger.CDebugw(ctx, "SubscribeRTP called AddStream and succeeded", "subID", sub.ID.String()) + } + + // Associate this subscription with the current generation. + c.associatedSubs[c.subGenerationID] = append(c.associatedSubs[c.subGenerationID], sub.ID) + + // Add the subscription to runningStreams. The goroutine spawned by `addOnTrackFunc` will use + // this to know where to forward incoming RTP packets. + c.runningStreams[sub.ID] = bufAndCB{ cb: func(p *rtp.Packet) { packetsCB([]*rtp.Packet{p}) }, - buf: buf, + buf: rtpPacketBuffer, } - buf.Start() + + // Start the goroutine that copies RTP packets + rtpPacketBuffer.Start() g.Success() c.logger.CDebugw(ctx, "SubscribeRTP succeeded", "subID", sub.ID.String(), - "name", c.Name(), "bufAndCBByID", c.bufAndCBByID.String()) + "name", c.Name(), "bufAndCBByID", c.bufAndCBToString()) return sub, nil } -func (c *client) addOnTrackSubFunc( - trackReceived, trackClosed chan struct{}, - parentID rtppassthrough.SubscriptionID, -) rdkgrpc.OnTrackCB { +func (c *client) addOnTrackFunc( + healthyClientCh, trackReceived, trackClosed chan struct{}, + generationID int, +) grpc.OnTrackCB { + // This is invoked when `PeerConnection.OnTrack` is called for this camera's stream id. return func(tr *webrtc.TrackRemote, r *webrtc.RTPReceiver) { + // Our `OnTrack` was called. Inform `SubscribeRTP` that getting video data was successful. close(trackReceived) c.activeBackgroundWorkers.Add(1) goutils.ManagedGo(func() { + var count atomic.Uint64 + defer close(trackClosed) for { - if c.ctx.Err() != nil { - c.logger.Debugw("SubscribeRTP: camera client", "name ", c.Name(), "parentID", parentID.String(), - "OnTrack callback terminating as the client is closing") - close(trackClosed) + select { + case <-healthyClientCh: + c.logger.Debugw("OnTrack callback terminating as the client is closing", "parentID", generationID) + return + default: + } + + // We set a read deadline such that we don't block in I/O. This allows us to respond + // to events such as shutdown. Normally this would be done with a context on the + // `ReadRTP` method. + deadline := time.Now().Add(readRTPTimeout) + if err := tr.SetReadDeadline(deadline); err != nil { + c.logger.Errorw("SetReadDeadline error", "generationId", generationID, "err", err) return } pkt, _, err := tr.ReadRTP() + if os.IsTimeout(err) { + c.logger.Debugw("ReadRTP read timeout", "generationId", generationID, + "err", err, "timeout", readRTPTimeout.String()) + continue + } + if err != nil { - close(trackClosed) + if errors.Is(err, io.EOF) { + c.logger.Infow("ReadRTP received EOF", "generationId", generationID, "err", err) + } else { + c.logger.Warnw("ReadRTP error", "generationId", generationID, "err", err) + } // NOTE: (Nick S) We need to remember which subscriptions are consuming packets // from to which tr *webrtc.TrackRemote so that we can terminate the child subscriptions // when their track terminate. - c.unsubscribeChildrenSubs(parentID) - if errors.Is(err, io.EOF) { - c.logger.Debugw("SubscribeRTP: camera client", "name ", c.Name(), "parentID", parentID.String(), - "OnTrack callback terminating ReadRTP loop due to ", err.Error()) - return - } - c.logger.Errorw("SubscribeRTP: camera client", "name ", c.Name(), "parentID", parentID.String(), - "OnTrack callback hit unexpected error from ReadRTP err:", err.Error()) + c.unsubscribeChildrenSubs(generationID) return } - c.mu.Lock() - for _, tmp := range c.bufAndCBByID { + c.rtpPassthroughMu.Lock() + for _, tmp := range c.runningStreams { + if count.Add(1)%10000 == 0 { + c.logger.Debugw("ReadRTP called. Sampling 1/10000", "count", count.Load(), "packetTs", pkt.Header.Timestamp) + } + // This is needed to prevent the problem described here: // https://go.dev/blog/loopvar-preview bufAndCB := tmp err := bufAndCB.buf.Publish(func() { bufAndCB.cb(pkt) }) if err != nil { - c.logger.Debugw("SubscribeRTP: camera client", - "name", c.Name(), - "parentID", parentID.String(), - "dropped an RTP packet dropped due to", - "err", err.Error()) + c.logger.Debugw("rtp passthrough packet dropped", + "generationId", generationID, "err", err) } } - c.mu.Unlock() + c.rtpPassthroughMu.Unlock() } }, c.activeBackgroundWorkers.Done) } } // Unsubscribe terminates a subscription to receive RTP packets. -// It is strongly recommended to set a timeout on ctx as the underlying -// WebRTC peer connection's Track can be removed before sending an RTP packet which -// results in Unsubscribe blocking until the provided -// context is cancelled or the client is closed. -// This rare condition may happen in cases like reconfiguring concurrently with -// an Unsubscribe call. +// +// It is strongly recommended to set a timeout on ctx as the underlying WebRTC peer connection's +// Track can be removed before sending an RTP packet which results in Unsubscribe blocking until the +// provided context is cancelled or the client is closed. This rare condition may happen in cases +// like reconfiguring concurrently with an Unsubscribe call. func (c *client) Unsubscribe(ctx context.Context, id rtppassthrough.SubscriptionID) error { ctx, span := trace.StartSpan(ctx, "camera::client::Unsubscribe") defer span.End() - c.mu.Lock() if c.conn.PeerConn() == nil { - c.mu.Unlock() return ErrNoPeerConnection } - _, ok := c.conn.(*rdkgrpc.SharedConn) - if !ok { - c.mu.Unlock() - return ErrNoSharedPeerConnection - } - c.logger.CDebugw(ctx, "Unsubscribe called with", "name", c.Name(), "subID", id.String()) + c.healthyClientChMu.Lock() + healthyClientCh := c.healthyClientCh + c.healthyClientChMu.Unlock() - bufAndCB, ok := c.bufAndCBByID[id] + c.logger.CInfow(ctx, "Unsubscribe called", "subID", id.String()) + c.rtpPassthroughMu.Lock() + bufAndCB, ok := c.runningStreams[id] if !ok { - c.logger.CWarnw(ctx, "Unsubscribe called with unknown subID ", "name", c.Name(), "subID", id.String()) - c.mu.Unlock() + c.logger.CWarnw(ctx, "Unsubscribe called with unknown subID", "subID", id.String()) + c.rtpPassthroughMu.Unlock() return ErrUnknownSubscriptionID } - if len(c.bufAndCBByID) == 1 { - c.logger.CDebugw(ctx, "Unsubscribe calling RemoveStream", "name", c.Name(), "subID", id.String()) - if _, err := c.streamClient.RemoveStream(ctx, &streampb.RemoveStreamRequest{Name: c.Name().String()}); err != nil { - c.logger.CWarnw(ctx, "Unsubscribe RemoveStream returned err", "name", c.Name(), "subID", id.String(), "err", err) - c.mu.Unlock() - return err - } - - delete(c.bufAndCBByID, id) + if len(c.runningStreams) > 1 { + // There are still existing output streams for this camera. We close the resources + // associated with the input `SubscriptionID` and return. + delete(c.runningStreams, id) + c.rtpPassthroughMu.Unlock() bufAndCB.buf.Close() + return nil + } - // unlock so that the OnTrack callback can get the lock if it needs to before the ReadRTP method returns an error - // which will close `c.trackClosed`. - c.mu.Unlock() + // There is only one stream left. In addition to closing the input subscription's resources, we + // also close the stream from the underlying remote. + // + // Calling `RemoveStream` releases resources on the remote. We promise to keep retrying until + // we're successful. + c.logger.CInfow(ctx, "Last subscriber. calling RemoveStream", + "trackName", c.trackName(), "subID", id.String()) + request := &streampb.RemoveStreamRequest{Name: c.trackName()} + // We assume the server responds with a success if the requested `Name` is unknown/already + // removed. + if _, err := c.streamClient.RemoveStream(ctx, request); err != nil { + c.logger.CWarnw(ctx, "Unsubscribe RemoveStream returned err", "trackName", + c.trackName(), "subID", id.String(), "err", err) + c.rtpPassthroughMu.Unlock() + return err + } + + // We can only remove `runningStreams` when `RemoveStream` succeeds. If there was an error, the + // upper layers are responsible for retrying `Unsubscribe`. At this point we will not return an + // error to clearly inform the caller that `Unsubscribe` does not need to be retried. + delete(c.runningStreams, id) + bufAndCB.buf.Close() - select { - case <-ctx.Done(): - err := errors.Wrap(ctx.Err(), "track not closed within Unsubscribe provided context. Subscription may be left in broken state") - c.logger.Error(err) - return err - case <-c.ctx.Done(): - err := errors.Wrap(c.ctx.Err(), "track not closed before client closed") - c.logger.Error(err) - return err - case <-c.trackClosed: - } + // The goroutine we're stopping uses the `rtpPassthroughMu` to access the map of + // `runningStreams`. We must unlock while we wait for it to exit. + c.rtpPassthroughMu.Unlock() + + select { + case <-c.trackClosed: + return nil + case <-ctx.Done(): + c.logger.CWarnw(ctx, "RemoveStream successfully called, but errored waiting for stream to send an EOF", + "subID", id.String(), "err", ctx.Err()) + return nil + case <-healthyClientCh: + c.logger.CWarnw(ctx, "RemoveStream successfully called, but camera closed waiting for stream to send an EOF", + "subID", id.String()) return nil } - c.mu.Unlock() +} - delete(c.bufAndCBByID, id) - bufAndCB.buf.Close() +func (c *client) trackName() string { + // if c.conn is a *grpc.SharedConn then the client + // is talking to a module and we need to send the fully qualified name + if _, ok := c.conn.(*grpc.SharedConn); ok { + return c.Name().String() + } - return nil + // otherwise we know we are talking to either a main or remote robot part + if c.remoteName != "" { + // if c.remoteName != "" it indicates that we are talking to a remote part & we need to pop the remote name + // as the remote doesn't know it's own name from the perspective of the main part + return c.Name().PopRemote().SDPTrackName() + } + // in this case we are talking to a main part & the remote name (if it exists) needs to be preserved + return c.Name().SDPTrackName() } func (c *client) unsubscribeAll() { - if len(c.bufAndCBByID) > 0 { - for id, bufAndCB := range c.bufAndCBByID { - c.logger.Debugw("unsubscribeAll terminating sub", "name", c.Name(), "subID", id.String()) - delete(c.bufAndCBByID, id) + c.rtpPassthroughMu.Lock() + defer c.rtpPassthroughMu.Unlock() + if len(c.runningStreams) > 0 { + // shutdown & delete all *rtppassthrough.Buffer and callbacks + for subID, bufAndCB := range c.runningStreams { + c.logger.Debugw("unsubscribeAll terminating sub", "subID", subID.String()) + delete(c.runningStreams, subID) bufAndCB.buf.Close() } } } -func (c *client) unsubscribeChildrenSubs(parentID rtppassthrough.SubscriptionID) { - c.mu.Lock() - defer c.mu.Unlock() - c.logger.Debugw("client unsubscribeChildrenSubs called", "name", c.Name(), "parentID", parentID.String()) - defer c.logger.Debugw("client unsubscribeChildrenSubs done", "name", c.Name(), "parentID", parentID.String()) - for _, childID := range c.subParentToChildren[parentID] { - bufAndCB, ok := c.bufAndCBByID[childID] +func (c *client) unsubscribeChildrenSubs(generationID int) { + c.rtpPassthroughMu.Lock() + defer c.rtpPassthroughMu.Unlock() + c.logger.Debugw("client unsubscribeChildrenSubs called", "generationId", generationID, "numSubs", len(c.associatedSubs)) + defer c.logger.Debugw("client unsubscribeChildrenSubs done", "generationId", generationID) + for _, subID := range c.associatedSubs[generationID] { + bufAndCB, ok := c.runningStreams[subID] if !ok { continue } - c.logger.Debugw("stopping child subscription", "name", c.Name(), "parentID", parentID.String(), "childID", childID.String()) - delete(c.bufAndCBByID, childID) + c.logger.Debugw("stopping subscription", "generationId", generationID, "subId", subID.String()) + delete(c.runningStreams, subID) bufAndCB.buf.Close() } - delete(c.subParentToChildren, parentID) + delete(c.associatedSubs, generationID) } -func (s bufAndCBByID) String() string { - if len(s) == 0 { +func (c *client) bufAndCBToString() string { + if len(c.runningStreams) == 0 { return "len: 0" } strIds := []string{} strIdsToCB := map[string]bufAndCB{} - for id, cb := range s { + for id, cb := range c.runningStreams { strID := id.String() strIds = append(strIds, strID) strIdsToCB[strID] = cb } slices.Sort(strIds) - ret := fmt.Sprintf("len: %d, ", len(s)) + ret := fmt.Sprintf("len: %d, ", len(c.runningStreams)) for _, strID := range strIds { ret += fmt.Sprintf("%s: %v, ", strID, strIdsToCB[strID]) } diff --git a/components/camera/client_test.go b/components/camera/client_test.go index 60ad8b82730..11640bbd93e 100644 --- a/components/camera/client_test.go +++ b/components/camera/client_test.go @@ -18,17 +18,26 @@ import ( "google.golang.org/grpc/metadata" "go.viam.com/rdk/components/camera" + "go.viam.com/rdk/components/camera/fake" "go.viam.com/rdk/components/camera/rtppassthrough" + "go.viam.com/rdk/config" "go.viam.com/rdk/data" "go.viam.com/rdk/gostream" + "go.viam.com/rdk/gostream/codec/opus" + "go.viam.com/rdk/gostream/codec/x264" viamgrpc "go.viam.com/rdk/grpc" "go.viam.com/rdk/logging" "go.viam.com/rdk/pointcloud" "go.viam.com/rdk/resource" "go.viam.com/rdk/rimage" "go.viam.com/rdk/rimage/transform" + "go.viam.com/rdk/robot" + robotimpl "go.viam.com/rdk/robot/impl" + "go.viam.com/rdk/robot/web" + weboptions "go.viam.com/rdk/robot/web/options" "go.viam.com/rdk/testutils" "go.viam.com/rdk/testutils/inject" + "go.viam.com/rdk/testutils/robottestutils" rutils "go.viam.com/rdk/utils" "go.viam.com/rdk/utils/contextutils" ) @@ -669,3 +678,513 @@ func TestRTPPassthroughWithoutWebRTC(t *testing.T) { test.That(t, conn.Close(), test.ShouldBeNil) }) } + +func setupRealRobot( + t *testing.T, + robotConfig *config.Config, + logger logging.Logger, +) (context.Context, robot.LocalRobot, string, web.Service) { + t.Helper() + + ctx := context.Background() + robot, err := robotimpl.RobotFromConfig(ctx, robotConfig, logger) + test.That(t, err, test.ShouldBeNil) + + // We initialize with a stream config such that the stream server is capable of creating video stream and + // audio stream data. + webSvc := web.New(robot, logger, web.WithStreamConfig(gostream.StreamConfig{ + AudioEncoderFactory: opus.NewEncoderFactory(), + VideoEncoderFactory: x264.NewEncoderFactory(), + })) + options, _, addr := robottestutils.CreateBaseOptionsAndListener(t) + err = webSvc.Start(ctx, options) + test.That(t, err, test.ShouldBeNil) + + return ctx, robot, addr, webSvc +} + +func setupRealRobotWithOptions( + t *testing.T, + robotConfig *config.Config, + logger logging.Logger, + options weboptions.Options, +) (context.Context, robot.LocalRobot, web.Service) { + t.Helper() + + ctx := context.Background() + robot, err := robotimpl.RobotFromConfig(ctx, robotConfig, logger) + test.That(t, err, test.ShouldBeNil) + + // We initialize with a stream config such that the stream server is capable of creating video stream and + // audio stream data. + webSvc := web.New(robot, logger, web.WithStreamConfig(gostream.StreamConfig{ + AudioEncoderFactory: opus.NewEncoderFactory(), + VideoEncoderFactory: x264.NewEncoderFactory(), + })) + err = webSvc.Start(ctx, options) + test.That(t, err, test.ShouldBeNil) + + return ctx, robot, webSvc +} + +var ( + Green = "\033[32m" + Red = "\033[31m" + Magenta = "\033[35m" + Cyan = "\033[36m" + Yellow = "\033[33m" + Reset = "\033[0m" +) + +func TestMultiplexOverRemoteConnection(t *testing.T) { + logger := logging.NewTestLogger(t).Sublogger(t.Name()) + + remoteCfg := &config.Config{Components: []resource.Config{ + { + Name: "rtpPassthroughCamera", + API: resource.NewAPI("rdk", "component", "camera"), + Model: resource.DefaultModelFamily.WithModel("fake"), + ConvertedAttributes: &fake.Config{ + RTPPassthrough: true, + }, + }, + }} + + // Create a robot with a single fake camera. + remoteCtx, remoteRobot, addr, remoteWebSvc := setupRealRobot(t, remoteCfg, logger.Sublogger("remote")) + defer remoteRobot.Close(remoteCtx) + defer remoteWebSvc.Close(remoteCtx) + + mainCfg := &config.Config{Remotes: []config.Remote{ + { + Name: "remote", + Address: addr, + Insecure: true, + }, + }} + mainCtx, mainRobot, _, mainWebSvc := setupRealRobot(t, mainCfg, logger.Sublogger("main")) + logger.Info("robot setup") + defer mainRobot.Close(mainCtx) + defer mainWebSvc.Close(mainCtx) + + cameraClient, err := camera.FromRobot(mainRobot, "remote:rtpPassthroughCamera") + test.That(t, err, test.ShouldBeNil) + + image, _, err := cameraClient.Images(mainCtx) + test.That(t, err, test.ShouldBeNil) + test.That(t, image, test.ShouldNotBeNil) + logger.Info("got images") + + recvPktsCtx, recvPktsFn := context.WithCancel(context.Background()) + sub, err := cameraClient.(rtppassthrough.Source).SubscribeRTP(mainCtx, 2, func(pkts []*rtp.Packet) { + recvPktsFn() + }) + test.That(t, err, test.ShouldBeNil) + <-recvPktsCtx.Done() + logger.Info("got packets") + + err = cameraClient.(rtppassthrough.Source).Unsubscribe(mainCtx, sub.ID) + test.That(t, err, test.ShouldBeNil) + logger.Info("unsubscribe") +} + +func TestMultiplexOverMultiHopRemoteConnection(t *testing.T) { + logger := logging.NewTestLogger(t).Sublogger(t.Name()) + + remoteCfg2 := &config.Config{ + Network: config.NetworkConfig{NetworkConfigData: config.NetworkConfigData{Sessions: config.SessionsConfig{HeartbeatWindow: time.Hour}}}, + Components: []resource.Config{ + { + Name: "rtpPassthroughCamera", + API: resource.NewAPI("rdk", "component", "camera"), + Model: resource.DefaultModelFamily.WithModel("fake"), + ConvertedAttributes: &fake.Config{ + RTPPassthrough: true, + }, + }, + }, + } + + // Create a robot with a single fake camera. + remote2Ctx, remoteRobot2, addr2, remoteWebSvc2 := setupRealRobot(t, remoteCfg2, logger.Sublogger("remote-2")) + defer remoteRobot2.Close(remote2Ctx) + defer remoteWebSvc2.Close(remote2Ctx) + + remoteCfg1 := &config.Config{ + Network: config.NetworkConfig{NetworkConfigData: config.NetworkConfigData{Sessions: config.SessionsConfig{HeartbeatWindow: time.Hour}}}, + Remotes: []config.Remote{ + { + Name: "remote-2", + Address: addr2, + Insecure: true, + }, + }, + } + + remote1Ctx, remoteRobot1, addr1, remoteWebSvc1 := setupRealRobot(t, remoteCfg1, logger.Sublogger("remote-1")) + defer remoteRobot1.Close(remote1Ctx) + defer remoteWebSvc1.Close(remote1Ctx) + + mainCfg := &config.Config{ + Network: config.NetworkConfig{NetworkConfigData: config.NetworkConfigData{Sessions: config.SessionsConfig{HeartbeatWindow: time.Hour}}}, + Remotes: []config.Remote{ + { + Name: "remote-1", + Address: addr1, + Insecure: true, + }, + }, + } + + mainCtx, mainRobot, _, mainWebSvc := setupRealRobot(t, mainCfg, logger.Sublogger("main")) + defer mainRobot.Close(mainCtx) + defer mainWebSvc.Close(mainCtx) + + cameraClient, err := camera.FromRobot(mainRobot, "remote-1:remote-2:rtpPassthroughCamera") + test.That(t, err, test.ShouldBeNil) + + image, _, err := cameraClient.Images(mainCtx) + test.That(t, err, test.ShouldBeNil) + test.That(t, image, test.ShouldNotBeNil) + logger.Info("got images") + + time.Sleep(time.Second) + + recvPktsCtx, recvPktsFn := context.WithCancel(context.Background()) + sub, err := cameraClient.(rtppassthrough.Source).SubscribeRTP(mainCtx, 512, func(pkts []*rtp.Packet) { + recvPktsFn() + }) + test.That(t, err, test.ShouldBeNil) + <-recvPktsCtx.Done() + logger.Info("got packets") + + test.That(t, cameraClient.(rtppassthrough.Source).Unsubscribe(mainCtx, sub.ID), test.ShouldBeNil) +} + +//nolint +// NOTE: These tests fail when this condition occurs: +// +// logger.go:130: 2024-06-17T16:56:14.097-0400 DEBUG TestGrandRemoteRebooting.remote-1.rdk:remote:/remote-2.webrtc rpc/wrtc_client_channel.go:299 no stream for id; discarding {"ch": 0, "id": 11} +// +// https://github.com/viamrobotics/goutils/blob/main/rpc/wrtc_client_channel.go#L299 +// +// go test -race -v -run=TestWhyMustTimeoutOnReadRTP -timeout 10s +// TestWhyMustTimeoutOnReadRTP shows that if we don't timeout on ReadRTP (and also don't call RemoveStream) on close +// calling Close() on main's camera client blocks forever if there is a live SubscribeRTP subscription with a remote +// due to the fact that the TrackRemote.ReadRTP method blocking forever. +func TestWhyMustTimeoutOnReadRTP(t *testing.T) { + t.Skip("Depends on RSDK-7903") + logger := logging.NewTestLogger(t).Sublogger(t.Name()) + + remoteCfg2 := &config.Config{ + Network: config.NetworkConfig{NetworkConfigData: config.NetworkConfigData{Sessions: config.SessionsConfig{HeartbeatWindow: time.Hour}}}, + Components: []resource.Config{ + { + Name: "rtpPassthroughCamera", + API: resource.NewAPI("rdk", "component", "camera"), + Model: resource.DefaultModelFamily.WithModel("fake"), + ConvertedAttributes: &fake.Config{ + RTPPassthrough: true, + }, + }, + }, + } + // Create a robot with a single fake camera. + remote2Ctx, remoteRobot2, addr2, remoteWebSvc2 := setupRealRobot(t, remoteCfg2, logger.Sublogger("remote-2")) + defer remoteRobot2.Close(remote2Ctx) + defer remoteWebSvc2.Close(remote2Ctx) + + remoteCfg1 := &config.Config{ + Network: config.NetworkConfig{NetworkConfigData: config.NetworkConfigData{Sessions: config.SessionsConfig{HeartbeatWindow: time.Hour}}}, + Remotes: []config.Remote{ + { + Name: "remote-2", + Address: addr2, + Insecure: true, + }, + }, + } + + remote1Ctx, remoteRobot1, addr1, remoteWebSvc1 := setupRealRobot(t, remoteCfg1, logger.Sublogger("remote-1")) + defer remoteRobot1.Close(remote1Ctx) + defer remoteWebSvc1.Close(remote1Ctx) + + mainCfg := &config.Config{ + Network: config.NetworkConfig{NetworkConfigData: config.NetworkConfigData{Sessions: config.SessionsConfig{HeartbeatWindow: time.Hour}}}, + Remotes: []config.Remote{ + { + Name: "remote-1", + Address: addr1, + Insecure: true, + }, + }, + } + + mainCtx, mainRobot, _, mainWebSvc := setupRealRobot(t, mainCfg, logger.Sublogger("main")) + logger.Info("robot setup") + defer mainRobot.Close(mainCtx) + defer mainWebSvc.Close(mainCtx) + + cameraClient, err := camera.FromRobot(mainRobot, "remote-1:remote-2:rtpPassthroughCamera") + test.That(t, err, test.ShouldBeNil) + + image, _, err := cameraClient.Images(mainCtx) + test.That(t, err, test.ShouldBeNil) + test.That(t, image, test.ShouldNotBeNil) + logger.Info("got images") + + logger.Infof("calling SubscribeRTP on %T, %p", cameraClient, cameraClient) + time.Sleep(time.Second) + + pktsChan := make(chan []*rtp.Packet) + recvPktsCtx, recvPktsFn := context.WithCancel(context.Background()) + sub, err := cameraClient.(rtppassthrough.Source).SubscribeRTP(mainCtx, 512, func(pkts []*rtp.Packet) { + // first packet + recvPktsFn() + // at some point packets are no longer published + select { + case pktsChan <- pkts: + default: + } + }) + test.That(t, err, test.ShouldBeNil) + <-recvPktsCtx.Done() + logger.Info("got packets") + + logger.Info("calling close") + test.That(t, remoteRobot2.Close(context.Background()), test.ShouldBeNil) + test.That(t, remoteWebSvc2.Close(context.Background()), test.ShouldBeNil) + logger.Info("close called") + + logger.Info("waiting for SubscribeRTP to stop receiving packets") + + timeoutCtx, timeoutFn := context.WithTimeout(context.Background(), time.Second) + defer timeoutFn() + + var ( + pktTimeoutCtx context.Context + pktTimeoutFn context.CancelFunc + ) + + // Once we have not received packets for half a second we can assume that no more packets will be published + // by the first instance of remote2 +Loop: + for { + if pktTimeoutFn != nil { + pktTimeoutFn() + } + pktTimeout := time.Millisecond * 500 + pktTimeoutCtx, pktTimeoutFn = context.WithTimeout(context.Background(), pktTimeout) + select { + case <-pktsChan: + continue + case <-pktTimeoutCtx.Done(): + logger.Infof("it has been at least %s since SubscribeRTP has received a packet", pktTimeout) + pktTimeoutFn() + // https://go.dev/ref/spec#Break_statements + // The 'Loop' label is needed so that we break out of the loop + // rather than out of the select statement + break Loop + case <-timeoutCtx.Done(): + // Failure case. The following assertion always fails. We use this to get a failure line + // number + error message. + test.That(t, true, test.ShouldEqual, "timed out waiting for SubscribeRTP packets to drain") + } + } + + logger.Info("sleeping") + time.Sleep(time.Second) + + // sub should still be alive + test.That(t, sub.Terminated.Err(), test.ShouldBeNil) +} + +// Notes: +// - Ideally, we'd lower robot client reconnect timers down from 10 seconds. +// - We need to force robot client webrtc connections +// - WebRTC connections need to disable SRTP replay protection +// +// This tests the following scenario: +// 1. main-part (main) -> remote-part-1 (r1) -> remote-part-2 (r2) where r2 has a camera +// 2. the client in the main part makes an AddStream(r1:r2:rtpPassthroughCamera) request, starting a +// webrtc video track to be streamed from r2 -> r1 -> main -> client +// 3. r2 reboots +// 4. expect that r1 & main stop getting packets +// 5. when the new instance of r2 comes back online main gets new rtp packets from it's track with +// r1. +func TestGrandRemoteRebooting(t *testing.T) { + t.Skip("Depends on RSDK-7903") + logger := logging.NewTestLogger(t).Sublogger(t.Name()) + + remoteCfg2 := &config.Config{ + Components: []resource.Config{ + { + Name: "rtpPassthroughCamera", + API: resource.NewAPI("rdk", "component", "camera"), + Model: resource.DefaultModelFamily.WithModel("fake"), + ConvertedAttributes: &fake.Config{ + RTPPassthrough: true, + }, + }, + }, + } + + // Create a robot with a single fake camera. + options2, _, addr2 := robottestutils.CreateBaseOptionsAndListener(t) + remote2Ctx, remoteRobot2, remoteWebSvc2 := setupRealRobotWithOptions(t, remoteCfg2, logger.Sublogger("remote-2"), options2) + + remoteCfg1 := &config.Config{ + Remotes: []config.Remote{ + { + Name: "remote-2", + Address: addr2, + Insecure: true, + }, + }, + } + + remote1Ctx, remoteRobot1, addr1, remoteWebSvc1 := setupRealRobot(t, remoteCfg1, logger.Sublogger("remote-1")) + defer remoteRobot1.Close(remote1Ctx) + defer remoteWebSvc1.Close(remote1Ctx) + + mainCfg := &config.Config{ + Remotes: []config.Remote{ + { + Name: "remote-1", + Address: addr1, + Insecure: true, + }, + }, + } + + mainCtx, mainRobot, _, mainWebSvc := setupRealRobot(t, mainCfg, logger.Sublogger("main")) + logger.Info("robot setup") + defer mainRobot.Close(mainCtx) + defer mainWebSvc.Close(mainCtx) + + mainCameraClient, err := camera.FromRobot(mainRobot, "remote-1:remote-2:rtpPassthroughCamera") + test.That(t, err, test.ShouldBeNil) + + image, _, err := mainCameraClient.Images(mainCtx) + test.That(t, err, test.ShouldBeNil) + test.That(t, image, test.ShouldNotBeNil) + logger.Info("got images") + + logger.Infof("calling SubscribeRTP on %T, %p", mainCameraClient, mainCameraClient) + time.Sleep(time.Second) + + pktsChan := make(chan []*rtp.Packet, 10) + recvPktsCtx, recvPktsFn := context.WithCancel(context.Background()) + testDone := make(chan struct{}) + defer close(testDone) + sub, err := mainCameraClient.(rtppassthrough.Source).SubscribeRTP(mainCtx, 512, func(pkts []*rtp.Packet) { + // first packet + recvPktsFn() + // at some point packets are no longer published + lastPkt := pkts[len(pkts)-1] + logger.Info("Pushing packets: ", len(pkts), " TS:", lastPkt.Timestamp) + select { + case <-testDone: + case pktsChan <- pkts: + } + logger.Info("Pkt pushed. TS:", lastPkt.Timestamp) + }) + test.That(t, err, test.ShouldBeNil) + <-recvPktsCtx.Done() + logger.Info("got packets") + + logger.Info("calling close") + test.That(t, remoteRobot2.Close(remote2Ctx), test.ShouldBeNil) + test.That(t, remoteWebSvc2.Close(remote2Ctx), test.ShouldBeNil) + logger.Info("close called") + + logger.Info("waiting for SubscribeRTP to stop receiving packets") + + timeoutCtx, timeoutFn := context.WithTimeout(context.Background(), time.Second) + defer timeoutFn() + + var ( + pktTimeoutCtx context.Context + pktTimeoutFn context.CancelFunc + ) + + // Once we have not received packets for half a second we can assume that no more packets will be published + // by the first instance of remote2 +Loop: + for { + if pktTimeoutFn != nil { + pktTimeoutFn() + } + pktTimeout := time.Millisecond * 500 + pktTimeoutCtx, pktTimeoutFn = context.WithTimeout(context.Background(), pktTimeout) + select { + case pkts := <-pktsChan: + lastPkt := pkts[len(pkts)-1] + logger.Infof("First RTP packet received. TS: %v", lastPkt.Timestamp) + continue + case <-pktTimeoutCtx.Done(): + logger.Infow("SubscribeRTP timed out waiting for a packet. The remote is offline.", "pktTimeout", pktTimeout) + pktTimeoutFn() + // https://go.dev/ref/spec#Break_statements + // The 'Loop' label is needed so that we break out of the loop + // rather than out of the select statement + break Loop + case <-timeoutCtx.Done(): + // Failure case. The following assertion always fails. We use this to get a failure line + // number + error message. + test.That(t, true, test.ShouldEqual, "timed out waiting for SubscribeRTP packets to drain") + } + } + + // sub should still be alive + test.That(t, sub.Terminated.Err(), test.ShouldBeNil) + + // I'm trying to get the remote-2 to come back online at the same address under the hopes that remote-1 will + // treat it the same as it would if a real robot crasehed & came back online without changing its name. + // The expectation is that SubscribeRTP should start receiving packets from remote-1 when remote-1 starts + // receiving packets from the new remote-2 + // It is not working as remote 1 never detects remote 2 & as a result main calls Close() on it's client with + // remote-1 which can be detectd + // by the fact that sub.Terminated.Done() is always the path this test goes down + + logger.Infow("old robot address", "address", addr2) + tcpAddr, ok := options2.Network.Listener.Addr().(*net.TCPAddr) + test.That(t, ok, test.ShouldBeTrue) + newListener, err := net.ListenTCP("tcp", &net.TCPAddr{Port: tcpAddr.Port}) + test.That(t, err, test.ShouldBeNil) + options2.Network.Listener = newListener + + logger.Info("setting up new robot at address %s", newListener.Addr().String()) + + remote2CtxSecond, remoteRobot2Second, remoteWebSvc2Second := setupRealRobotWithOptions( + t, + remoteCfg2, + logger.Sublogger("remote-2SecondInstance"), + options2, + ) + defer remoteRobot2Second.Close(remote2CtxSecond) + defer remoteWebSvc2Second.Close(remote2CtxSecond) + sndPktTimeoutCtx, sndPktTimeoutFn := context.WithTimeout(context.Background(), time.Second*20) + defer sndPktTimeoutFn() + testPassed := false + for !testPassed { + select { + case <-sub.Terminated.Done(): + // Failure case. The following assertion always fails. We use this to get a failure line + // number + error message. + test.That(t, true, test.ShouldEqual, "main's sub terminated due to close") + case pkts := <-pktsChan: + lastPkt := pkts[len(pkts)-1] + logger.Info("Test finale RTP packet received. TS: %v", lastPkt.Timestamp) + // Right now we never go down this path as the test is not able to get remote1 to reconnect to the new remote 2 + logger.Info("SubscribeRTP got packets") + testPassed = true + case <-sndPktTimeoutCtx.Done(): + // Failure case. The following assertion always fails. We use this to get a failure line + // number + error message. + test.That(t, true, test.ShouldEqual, "timed out waiting for SubscribeRTP to receive packets") + case <-time.After(time.Second): + logger.Info("still waiting for RTP packets") + } + } +} diff --git a/components/camera/fake/camera.go b/components/camera/fake/camera.go index 2eef799b4c6..4e7d0134a43 100644 --- a/components/camera/fake/camera.go +++ b/components/camera/fake/camera.go @@ -5,6 +5,8 @@ import ( "bytes" "context" "encoding/base64" + "errors" + "fmt" "image" "image/color" "image/jpeg" @@ -17,7 +19,6 @@ import ( "github.com/bluenviron/gortsplib/v4/pkg/rtptime" "github.com/bluenviron/mediacommon/pkg/codecs/h264" "github.com/golang/geo/r3" - "github.com/pkg/errors" "go.viam.com/utils" "go.viam.com/rdk/components/camera" @@ -119,11 +120,11 @@ func (conf *Config) Validate(path string) ([]string, error) { } if conf.Height%2 != 0 { - return nil, errors.Errorf("odd-number resolutions cannot be rendered, cannot use a height of %d", conf.Height) + return nil, fmt.Errorf("odd-number resolutions cannot be rendered, cannot use a height of %d", conf.Height) } if conf.Width%2 != 0 { - return nil, errors.Errorf("odd-number resolutions cannot be rendered, cannot use a width of %d", conf.Width) + return nil, fmt.Errorf("odd-number resolutions cannot be rendered, cannot use a width of %d", conf.Width) } return nil, nil diff --git a/components/camera/videosource/static.go b/components/camera/fake/image_file.go similarity index 99% rename from components/camera/videosource/static.go rename to components/camera/fake/image_file.go index 4a7113492d8..458f3789d0f 100644 --- a/components/camera/videosource/static.go +++ b/components/camera/fake/image_file.go @@ -1,15 +1,14 @@ //go:build !no_cgo -package videosource +package fake import ( "context" + "errors" "fmt" "image" "time" - "github.com/pkg/errors" - "go.viam.com/rdk/components/camera" "go.viam.com/rdk/logging" "go.viam.com/rdk/pointcloud" diff --git a/components/camera/videosource/static_test.go b/components/camera/fake/image_file_test.go similarity index 99% rename from components/camera/videosource/static_test.go rename to components/camera/fake/image_file_test.go index 2c43c17cfa9..ab090f90a56 100644 --- a/components/camera/videosource/static_test.go +++ b/components/camera/fake/image_file_test.go @@ -1,4 +1,4 @@ -package videosource +package fake import ( "context" diff --git a/components/camera/transformpipeline/depth_edges_test.go b/components/camera/transformpipeline/depth_edges_test.go index 1db77683105..fdb193a4c7d 100644 --- a/components/camera/transformpipeline/depth_edges_test.go +++ b/components/camera/transformpipeline/depth_edges_test.go @@ -12,7 +12,7 @@ import ( "go.viam.com/utils/artifact" "go.viam.com/rdk/components/camera" - "go.viam.com/rdk/components/camera/videosource" + "go.viam.com/rdk/components/camera/fake" "go.viam.com/rdk/gostream" "go.viam.com/rdk/logging" "go.viam.com/rdk/rimage" @@ -25,7 +25,7 @@ func TestDepthSource(t *testing.T) { img, err := rimage.NewDepthMapFromFile( context.Background(), artifact.MustPath("rimage/board1_gray_small.png")) test.That(t, err, test.ShouldBeNil) - source := &videosource.StaticSource{DepthImg: img} + source := &fake.StaticSource{DepthImg: img} am := utils.AttributeMap{ "high_threshold": 0.85, "low_threshold": 0.40, @@ -57,7 +57,7 @@ func (h *depthSourceTestHelper) Process( pCtx.GotDebugImage(dm.ToPrettyPicture(0, rimage.MaxDepth), "aligned-depth") // create edge map - source := &videosource.StaticSource{DepthImg: dm} + source := &fake.StaticSource{DepthImg: dm} am := utils.AttributeMap{ "high_threshold": 0.85, "low_threshold": 0.40, @@ -78,7 +78,7 @@ func (h *depthSourceTestHelper) Process( pCtx.GotDebugPointCloud(fixedPointCloud, "aligned-pointcloud") // preprocess depth map - source = &videosource.StaticSource{DepthImg: dm} + source = &fake.StaticSource{DepthImg: dm} rs, stream, err := newDepthPreprocessTransform(context.Background(), gostream.NewVideoSource(source, prop.Video{})) test.That(t, err, test.ShouldBeNil) test.That(t, stream, test.ShouldEqual, camera.DepthStream) @@ -94,7 +94,7 @@ func (h *depthSourceTestHelper) Process( test.That(t, preprocessedPointCloud.MetaData().HasColor, test.ShouldBeFalse) pCtx.GotDebugPointCloud(preprocessedPointCloud, "preprocessed-aligned-pointcloud") - source = &videosource.StaticSource{DepthImg: preprocessed} + source = &fake.StaticSource{DepthImg: preprocessed} ds, stream, err = newDepthEdgesTransform(context.Background(), gostream.NewVideoSource(source, prop.Video{}), am) test.That(t, err, test.ShouldBeNil) test.That(t, stream, test.ShouldEqual, camera.DepthStream) diff --git a/components/camera/transformpipeline/mods_test.go b/components/camera/transformpipeline/mods_test.go index ee441e56538..721319a9bdf 100644 --- a/components/camera/transformpipeline/mods_test.go +++ b/components/camera/transformpipeline/mods_test.go @@ -10,7 +10,7 @@ import ( "go.viam.com/utils/artifact" "go.viam.com/rdk/components/camera" - "go.viam.com/rdk/components/camera/videosource" + "go.viam.com/rdk/components/camera/fake" "go.viam.com/rdk/gostream" "go.viam.com/rdk/rimage" "go.viam.com/rdk/utils" @@ -30,7 +30,7 @@ func TestCrop(t *testing.T) { test.That(t, err, test.ShouldBeNil) // test depth source - source := gostream.NewVideoSource(&videosource.StaticSource{DepthImg: dm}, prop.Video{}) + source := gostream.NewVideoSource(&fake.StaticSource{DepthImg: dm}, prop.Video{}) out, _, err := camera.ReadImage(context.Background(), source) test.That(t, err, test.ShouldBeNil) test.That(t, out.Bounds().Dx(), test.ShouldEqual, 128) @@ -48,7 +48,7 @@ func TestCrop(t *testing.T) { test.That(t, source.Close(context.Background()), test.ShouldBeNil) // test color source - source = gostream.NewVideoSource(&videosource.StaticSource{ColorImg: img}, prop.Video{}) + source = gostream.NewVideoSource(&fake.StaticSource{ColorImg: img}, prop.Video{}) out, _, err = camera.ReadImage(context.Background(), source) test.That(t, err, test.ShouldBeNil) test.That(t, out.Bounds().Dx(), test.ShouldEqual, 128) @@ -107,7 +107,7 @@ func TestResizeColor(t *testing.T) { "height_px": 20, "width_px": 30, } - source := gostream.NewVideoSource(&videosource.StaticSource{ColorImg: img}, prop.Video{}) + source := gostream.NewVideoSource(&fake.StaticSource{ColorImg: img}, prop.Video{}) out, _, err := camera.ReadImage(context.Background(), source) test.That(t, err, test.ShouldBeNil) test.That(t, out.Bounds().Dx(), test.ShouldEqual, 128) @@ -133,7 +133,7 @@ func TestResizeDepth(t *testing.T) { "height_px": 40, "width_px": 60, } - source := gostream.NewVideoSource(&videosource.StaticSource{DepthImg: img}, prop.Video{}) + source := gostream.NewVideoSource(&fake.StaticSource{DepthImg: img}, prop.Video{}) out, _, err := camera.ReadImage(context.Background(), source) test.That(t, err, test.ShouldBeNil) test.That(t, out.Bounds().Dx(), test.ShouldEqual, 128) @@ -154,7 +154,7 @@ func TestRotateColorSource(t *testing.T) { img, err := rimage.NewImageFromFile(artifact.MustPath("rimage/board1_small.png")) test.That(t, err, test.ShouldBeNil) - source := gostream.NewVideoSource(&videosource.StaticSource{ColorImg: img}, prop.Video{}) + source := gostream.NewVideoSource(&fake.StaticSource{ColorImg: img}, prop.Video{}) am := utils.AttributeMap{ "angle_degs": 180, } @@ -207,7 +207,7 @@ func TestRotateColorSource(t *testing.T) { test.That(t, rs.Close(context.Background()), test.ShouldBeNil) test.That(t, source.Close(context.Background()), test.ShouldBeNil) - source = gostream.NewVideoSource(&videosource.StaticSource{ColorImg: img}, prop.Video{}) + source = gostream.NewVideoSource(&fake.StaticSource{ColorImg: img}, prop.Video{}) am = utils.AttributeMap{ "angle_degs": 90, } @@ -238,7 +238,7 @@ func TestRotateColorSource(t *testing.T) { test.That(t, rs.Close(context.Background()), test.ShouldBeNil) test.That(t, source.Close(context.Background()), test.ShouldBeNil) - source = gostream.NewVideoSource(&videosource.StaticSource{ColorImg: img}, prop.Video{}) + source = gostream.NewVideoSource(&fake.StaticSource{ColorImg: img}, prop.Video{}) am = utils.AttributeMap{ "angle_degs": -90, } @@ -269,7 +269,7 @@ func TestRotateColorSource(t *testing.T) { test.That(t, rs.Close(context.Background()), test.ShouldBeNil) test.That(t, source.Close(context.Background()), test.ShouldBeNil) - source = gostream.NewVideoSource(&videosource.StaticSource{ColorImg: img}, prop.Video{}) + source = gostream.NewVideoSource(&fake.StaticSource{ColorImg: img}, prop.Video{}) am = utils.AttributeMap{ "angle_degs": 270, } @@ -300,7 +300,7 @@ func TestRotateColorSource(t *testing.T) { test.That(t, rs.Close(context.Background()), test.ShouldBeNil) test.That(t, source.Close(context.Background()), test.ShouldBeNil) - source = gostream.NewVideoSource(&videosource.StaticSource{ColorImg: img}, prop.Video{}) + source = gostream.NewVideoSource(&fake.StaticSource{ColorImg: img}, prop.Video{}) am = utils.AttributeMap{ "angle_degs": 0, // no-op } @@ -336,7 +336,7 @@ func TestRotateDepthSource(t *testing.T) { context.Background(), artifact.MustPath("rimage/board1_gray_small.png")) test.That(t, err, test.ShouldBeNil) - source := gostream.NewVideoSource(&videosource.StaticSource{DepthImg: pc}, prop.Video{}) + source := gostream.NewVideoSource(&fake.StaticSource{DepthImg: pc}, prop.Video{}) am := utils.AttributeMap{ "angle_degs": 180, } @@ -388,7 +388,7 @@ func TestRotateDepthSource(t *testing.T) { test.That(t, rs.Close(context.Background()), test.ShouldBeNil) test.That(t, source.Close(context.Background()), test.ShouldBeNil) - source = gostream.NewVideoSource(&videosource.StaticSource{DepthImg: pc}, prop.Video{}) + source = gostream.NewVideoSource(&fake.StaticSource{DepthImg: pc}, prop.Video{}) am = utils.AttributeMap{ "angle_degs": 90, } @@ -419,7 +419,7 @@ func TestRotateDepthSource(t *testing.T) { test.That(t, rs.Close(context.Background()), test.ShouldBeNil) test.That(t, source.Close(context.Background()), test.ShouldBeNil) - source = gostream.NewVideoSource(&videosource.StaticSource{DepthImg: pc}, prop.Video{}) + source = gostream.NewVideoSource(&fake.StaticSource{DepthImg: pc}, prop.Video{}) am = utils.AttributeMap{ "angle_degs": -90, } @@ -450,7 +450,7 @@ func TestRotateDepthSource(t *testing.T) { test.That(t, rs.Close(context.Background()), test.ShouldBeNil) test.That(t, source.Close(context.Background()), test.ShouldBeNil) - source = gostream.NewVideoSource(&videosource.StaticSource{DepthImg: pc}, prop.Video{}) + source = gostream.NewVideoSource(&fake.StaticSource{DepthImg: pc}, prop.Video{}) am = utils.AttributeMap{ "angle_degs": 270, } @@ -481,7 +481,7 @@ func TestRotateDepthSource(t *testing.T) { test.That(t, rs.Close(context.Background()), test.ShouldBeNil) test.That(t, source.Close(context.Background()), test.ShouldBeNil) - source = gostream.NewVideoSource(&videosource.StaticSource{DepthImg: pc}, prop.Video{}) + source = gostream.NewVideoSource(&fake.StaticSource{DepthImg: pc}, prop.Video{}) am = utils.AttributeMap{ "angle_degs": 0, // no-op } @@ -516,7 +516,7 @@ func BenchmarkColorRotate(b *testing.B) { img, err := rimage.NewImageFromFile(artifact.MustPath("rimage/board1.png")) test.That(b, err, test.ShouldBeNil) - source := gostream.NewVideoSource(&videosource.StaticSource{ColorImg: img}, prop.Video{}) + source := gostream.NewVideoSource(&fake.StaticSource{ColorImg: img}, prop.Video{}) src, err := camera.WrapVideoSourceWithProjector(context.Background(), source, nil, camera.ColorStream) test.That(b, err, test.ShouldBeNil) am := utils.AttributeMap{ @@ -540,7 +540,7 @@ func BenchmarkDepthRotate(b *testing.B) { context.Background(), artifact.MustPath("rimage/board1.dat.gz")) test.That(b, err, test.ShouldBeNil) - source := gostream.NewVideoSource(&videosource.StaticSource{DepthImg: img}, prop.Video{}) + source := gostream.NewVideoSource(&fake.StaticSource{DepthImg: img}, prop.Video{}) src, err := camera.WrapVideoSourceWithProjector(context.Background(), source, nil, camera.DepthStream) test.That(b, err, test.ShouldBeNil) am := utils.AttributeMap{ diff --git a/components/camera/transformpipeline/pipeline_test.go b/components/camera/transformpipeline/pipeline_test.go index 86e9bcf905b..b96ea0a9596 100644 --- a/components/camera/transformpipeline/pipeline_test.go +++ b/components/camera/transformpipeline/pipeline_test.go @@ -9,7 +9,7 @@ import ( "go.viam.com/utils/artifact" "go.viam.com/rdk/components/camera" - "go.viam.com/rdk/components/camera/videosource" + "go.viam.com/rdk/components/camera/fake" "go.viam.com/rdk/gostream" "go.viam.com/rdk/logging" "go.viam.com/rdk/resource" @@ -32,7 +32,7 @@ func TestTransformPipelineColor(t *testing.T) { img, err := rimage.NewImageFromFile(artifact.MustPath("rimage/board1_small.png")) test.That(t, err, test.ShouldBeNil) - source := gostream.NewVideoSource(&videosource.StaticSource{ColorImg: img}, prop.Video{}) + source := gostream.NewVideoSource(&fake.StaticSource{ColorImg: img}, prop.Video{}) src, err := camera.WrapVideoSourceWithProjector(context.Background(), source, nil, camera.ColorStream) test.That(t, err, test.ShouldBeNil) inImg, _, err := camera.ReadImage(context.Background(), src) @@ -78,7 +78,7 @@ func TestTransformPipelineDepth(t *testing.T) { dm, err := rimage.NewDepthMapFromFile(context.Background(), artifact.MustPath("rimage/board1_gray_small.png")) test.That(t, err, test.ShouldBeNil) - source := gostream.NewVideoSource(&videosource.StaticSource{DepthImg: dm}, prop.Video{}) + source := gostream.NewVideoSource(&fake.StaticSource{DepthImg: dm}, prop.Video{}) src, err := camera.WrapVideoSourceWithProjector(context.Background(), source, nil, camera.DepthStream) test.That(t, err, test.ShouldBeNil) inImg, _, err := camera.ReadImage(context.Background(), src) @@ -130,7 +130,7 @@ func TestTransformPipelineDepth2(t *testing.T) { dm, err := rimage.NewDepthMapFromFile( context.Background(), artifact.MustPath("rimage/board1_gray_small.png")) test.That(t, err, test.ShouldBeNil) - source := gostream.NewVideoSource(&videosource.StaticSource{DepthImg: dm}, prop.Video{}) + source := gostream.NewVideoSource(&fake.StaticSource{DepthImg: dm}, prop.Video{}) // first depth transform depth1, err := newTransformPipeline(context.Background(), source, transform1, r, logger) test.That(t, err, test.ShouldBeNil) @@ -160,7 +160,7 @@ func TestNullPipeline(t *testing.T) { img, err := rimage.NewImageFromFile(artifact.MustPath("rimage/board1_small.png")) test.That(t, err, test.ShouldBeNil) - source := gostream.NewVideoSource(&videosource.StaticSource{ColorImg: img}, prop.Video{}) + source := gostream.NewVideoSource(&fake.StaticSource{ColorImg: img}, prop.Video{}) _, err = newTransformPipeline(context.Background(), source, transform1, r, logger) test.That(t, err, test.ShouldNotBeNil) test.That(t, err.Error(), test.ShouldContainSubstring, "pipeline has no transforms") @@ -185,7 +185,7 @@ func TestPipeIntoPipe(t *testing.T) { img, err := rimage.NewImageFromFile(artifact.MustPath("rimage/board1_small.png")) test.That(t, err, test.ShouldBeNil) - source := gostream.NewVideoSource(&videosource.StaticSource{ColorImg: img}, prop.Video{}) + source := gostream.NewVideoSource(&fake.StaticSource{ColorImg: img}, prop.Video{}) intrinsics1 := &transform.PinholeCameraIntrinsics{Width: 128, Height: 72} transform1 := &transformConfig{ diff --git a/components/camera/transformpipeline/undistort_test.go b/components/camera/transformpipeline/undistort_test.go index abdf83dd2c3..41f6184bd84 100644 --- a/components/camera/transformpipeline/undistort_test.go +++ b/components/camera/transformpipeline/undistort_test.go @@ -10,7 +10,7 @@ import ( "go.viam.com/utils/artifact" "go.viam.com/rdk/components/camera" - "go.viam.com/rdk/components/camera/videosource" + "go.viam.com/rdk/components/camera/fake" "go.viam.com/rdk/gostream" "go.viam.com/rdk/rimage" "go.viam.com/rdk/rimage/transform" @@ -38,7 +38,7 @@ var undistortTestBC = &transform.BrownConrady{ func TestUndistortSetup(t *testing.T) { img, err := rimage.NewImageFromFile(artifact.MustPath("rimage/board1_small.png")) test.That(t, err, test.ShouldBeNil) - source := gostream.NewVideoSource(&videosource.StaticSource{ColorImg: img}, prop.Video{}) + source := gostream.NewVideoSource(&fake.StaticSource{ColorImg: img}, prop.Video{}) // no camera parameters am := utils.AttributeMap{} @@ -48,7 +48,7 @@ func TestUndistortSetup(t *testing.T) { test.That(t, source.Close(context.Background()), test.ShouldBeNil) // bad stream type - source = gostream.NewVideoSource(&videosource.StaticSource{ColorImg: img}, prop.Video{}) + source = gostream.NewVideoSource(&fake.StaticSource{ColorImg: img}, prop.Video{}) am = utils.AttributeMap{"intrinsic_parameters": undistortTestParams, "distortion_parameters": undistortTestBC} us, _, err := newUndistortTransform(context.Background(), source, camera.ImageType("fake"), am) test.That(t, err, test.ShouldBeNil) @@ -71,7 +71,7 @@ func TestUndistortSetup(t *testing.T) { func TestUndistortImage(t *testing.T) { img, err := rimage.NewImageFromFile(artifact.MustPath("rimage/board1_small.png")) test.That(t, err, test.ShouldBeNil) - source := gostream.NewVideoSource(&videosource.StaticSource{ColorImg: img}, prop.Video{}) + source := gostream.NewVideoSource(&fake.StaticSource{ColorImg: img}, prop.Video{}) // success am := utils.AttributeMap{"intrinsic_parameters": undistortTestParams, "distortion_parameters": undistortTestBC} @@ -92,7 +92,7 @@ func TestUndistortImage(t *testing.T) { test.That(t, result, test.ShouldResemble, expected) // bad source - source = gostream.NewVideoSource(&videosource.StaticSource{ColorImg: rimage.NewImage(10, 10)}, prop.Video{}) + source = gostream.NewVideoSource(&fake.StaticSource{ColorImg: rimage.NewImage(10, 10)}, prop.Video{}) us, stream, err = newUndistortTransform(context.Background(), source, camera.ColorStream, am) test.That(t, err, test.ShouldBeNil) test.That(t, stream, test.ShouldEqual, camera.ColorStream) @@ -105,7 +105,7 @@ func TestUndistortDepthMap(t *testing.T) { img, err := rimage.NewDepthMapFromFile( context.Background(), artifact.MustPath("rimage/board1_gray_small.png")) test.That(t, err, test.ShouldBeNil) - source := gostream.NewVideoSource(&videosource.StaticSource{DepthImg: img}, prop.Video{}) + source := gostream.NewVideoSource(&fake.StaticSource{DepthImg: img}, prop.Video{}) // success am := utils.AttributeMap{"intrinsic_parameters": undistortTestParams, "distortion_parameters": undistortTestBC} @@ -126,7 +126,7 @@ func TestUndistortDepthMap(t *testing.T) { test.That(t, result, test.ShouldResemble, expected) // bad source - source = gostream.NewVideoSource(&videosource.StaticSource{DepthImg: rimage.NewEmptyDepthMap(10, 10)}, prop.Video{}) + source = gostream.NewVideoSource(&fake.StaticSource{DepthImg: rimage.NewEmptyDepthMap(10, 10)}, prop.Video{}) us, stream, err = newUndistortTransform(context.Background(), source, camera.DepthStream, am) test.That(t, err, test.ShouldBeNil) test.That(t, stream, test.ShouldEqual, camera.DepthStream) @@ -135,7 +135,7 @@ func TestUndistortDepthMap(t *testing.T) { test.That(t, us.Close(context.Background()), test.ShouldBeNil) // can't convert image to depth map - source = gostream.NewVideoSource(&videosource.StaticSource{ColorImg: rimage.NewImage(10, 10)}, prop.Video{}) + source = gostream.NewVideoSource(&fake.StaticSource{ColorImg: rimage.NewImage(10, 10)}, prop.Video{}) us, stream, err = newUndistortTransform(context.Background(), source, camera.DepthStream, am) test.That(t, stream, test.ShouldEqual, camera.DepthStream) test.That(t, err, test.ShouldBeNil) diff --git a/components/encoder/ams/ams_as5048.go b/components/encoder/ams/ams_as5048.go index a262b401377..9e3811548e9 100644 --- a/components/encoder/ams/ams_as5048.go +++ b/components/encoder/ams/ams_as5048.go @@ -17,7 +17,6 @@ import ( "go.viam.com/rdk/components/encoder" "go.viam.com/rdk/logging" "go.viam.com/rdk/resource" - rdkutils "go.viam.com/rdk/utils" ) const ( @@ -113,7 +112,7 @@ type Encoder struct { i2cBus buses.I2C i2cAddr byte i2cBusName string // This is nessesary to check whether we need to create a new i2cBus during reconfigure. - workers rdkutils.StoppableWorkers + workers *utils.StoppableWorkers } func newAS5048Encoder( @@ -152,7 +151,7 @@ func makeAS5048Encoder( if err := res.ResetPosition(ctx, map[string]interface{}{}); err != nil { return nil, err } - res.workers = rdkutils.NewStoppableWorkers(res.positionLoop) + res.workers = utils.NewBackgroundStoppableWorkers(res.positionLoop) return res, nil } diff --git a/components/encoder/encoder.go b/components/encoder/encoder.go index aefb98f2f8d..95d5434b1db 100644 --- a/components/encoder/encoder.go +++ b/components/encoder/encoder.go @@ -1,4 +1,7 @@ -// Package encoder implements the encoder component +// Package encoder implements the encoder component. +// For more information, see the [encoder component docs]. +// +// [encoder component docs]: https://docs.viam.com/components/encoder/ package encoder import ( @@ -58,6 +61,7 @@ func (t PositionType) String() string { } // A Encoder turns a position into a signal. +// For more information, see the [encoder component docs]. // // Position example: // @@ -84,6 +88,8 @@ func (t PositionType) String() string { // // // Get whether the encoder returns position in ticks or degrees. // properties, err := myEncoder.Properties(context.Background(), nil) +// +// [encoder component docs]: https://docs.viam.com/components/encoder/ type Encoder interface { resource.Resource diff --git a/components/encoder/single/single_encoder.go b/components/encoder/single/single_encoder.go index 39ff03c8556..423ce9ff2a0 100644 --- a/components/encoder/single/single_encoder.go +++ b/components/encoder/single/single_encoder.go @@ -35,7 +35,6 @@ import ( "go.viam.com/rdk/components/encoder" "go.viam.com/rdk/logging" "go.viam.com/rdk/resource" - rdkutils "go.viam.com/rdk/utils" ) var singleModel = resource.DefaultModelFamily.WithModel("single") @@ -70,7 +69,7 @@ type Encoder struct { positionType encoder.PositionType logger logging.Logger - workers rdkutils.StoppableWorkers + workers *utils.StoppableWorkers } // Pin describes the configuration of Pins for a Single encoder. @@ -169,14 +168,14 @@ func (e *Encoder) Reconfigure( if e.workers != nil { e.workers.Stop() // Shut down the old interrupt stream } - e.start(ctx, board) // Start up the new interrupt stream + e.start(board) // Start up the new interrupt stream return nil } // start starts the Encoder background thread. It should only be called when the encoder's // background workers have been stopped (or never started). -func (e *Encoder) start(ctx context.Context, b board.Board) { - e.workers = rdkutils.NewStoppableWorkers() +func (e *Encoder) start(b board.Board) { + e.workers = utils.NewBackgroundStoppableWorkers() encoderChannel := make(chan board.Tick) err := b.StreamTicks(e.workers.Context(), []board.DigitalInterrupt{e.I}, encoderChannel, nil) @@ -185,7 +184,7 @@ func (e *Encoder) start(ctx context.Context, b board.Board) { return } - e.workers.AddWorkers(func(cancelCtx context.Context) { + e.workers.Add(func(cancelCtx context.Context) { for { select { case <-cancelCtx.Done(): @@ -208,7 +207,9 @@ func (e *Encoder) start(ctx context.Context, b board.Board) { atomic.AddInt64(&e.position, dir) } } else { - e.logger.CDebug(ctx, "received tick for encoder that isn't connected to a motor; ignoring") + // if no motor is attached to the encoder, increase in positive direction. + e.logger.Debug("no motor is attached to the encoder, increasing ticks count in the positive direction only") + atomic.AddInt64(&e.position, 1) } } }) diff --git a/components/encoder/single/single_encoder_test.go b/components/encoder/single/single_encoder_test.go index f0d9df88d4b..2e2693035d4 100644 --- a/components/encoder/single/single_encoder_test.go +++ b/components/encoder/single/single_encoder_test.go @@ -112,8 +112,7 @@ func TestEncoder(t *testing.T) { }) }) - // this test ensures that digital interrupts are ignored if AttachDirectionalAwareness - // is never called + // this test ensures that position goes forward if motor not attached. t.Run("run no direction", func(t *testing.T) { enc, err := NewSingleEncoder(ctx, deps, rawcfg, logging.NewTestLogger(t)) test.That(t, err, test.ShouldBeNil) @@ -128,6 +127,42 @@ func TestEncoder(t *testing.T) { // by the encoder worker time.Sleep(50 * time.Millisecond) + ticks, _, err := enc.Position(context.Background(), encoder.PositionTypeUnspecified, nil) + test.That(t, err, test.ShouldBeNil) + test.That(t, ticks, test.ShouldEqual, 1) + }) + + // Taking off directional awareness makes encoder tick forward. + t.Run("run motor then no motor", func(t *testing.T) { + enc, err := NewSingleEncoder(ctx, deps, rawcfg, logging.NewTestLogger(t)) + test.That(t, err, test.ShouldBeNil) + enc2 := enc.(*Encoder) + defer enc2.Close(context.Background()) + + m := &FakeDir{-1} // backward + enc2.AttachDirectionalAwareness(m) + + err = ii.Tick(context.Background(), true, uint64(time.Now().UnixNano())) + test.That(t, err, test.ShouldBeNil) + + testutils.WaitForAssertion(t, func(tb testing.TB) { + tb.Helper() + ticks, _, err := enc.Position(context.Background(), encoder.PositionTypeUnspecified, nil) + test.That(tb, err, test.ShouldBeNil) + test.That(tb, ticks, test.ShouldEqual, -1) + }) + + // take off directional awareness. + enc2.m = nil + + err = ii.Tick(context.Background(), true, uint64(time.Now().UnixNano())) + test.That(t, err, test.ShouldBeNil) + + // Give the tick time to propagate to encoder + // Warning: theres a race condition if the tick has not been processed + // by the encoder worker + time.Sleep(50 * time.Millisecond) + ticks, _, err := enc.Position(context.Background(), encoder.PositionTypeUnspecified, nil) test.That(t, err, test.ShouldBeNil) test.That(t, ticks, test.ShouldEqual, 0) diff --git a/components/gantry/gantry.go b/components/gantry/gantry.go index dc0a124f3b6..5d3e1a8ba29 100644 --- a/components/gantry/gantry.go +++ b/components/gantry/gantry.go @@ -1,3 +1,7 @@ +// Package gantry defines a robotic gantry with one or multiple axes. +// For more information, see the [gantry component docs]. +// +// [gantry component docs]: https://docs.viam.com/components/gantry/ package gantry import ( @@ -41,6 +45,7 @@ func Named(name string) resource.Name { } // Gantry is used for controlling gantries of N axis. +// For more information, see the [gantry component docs]. // // Position example: // @@ -74,6 +79,8 @@ func Named(name string) resource.Name { // myGantry, err := gantry.FromRobot(machine, "my_gantry") // // myGantry.Home(context.Background(), nil) +// +// [gantry component docs]: https://docs.viam.com/components/gantry/ type Gantry interface { resource.Resource resource.Actuator diff --git a/components/gantry/singleaxis/singleaxis.go b/components/gantry/singleaxis/singleaxis.go index 34f01ef5f7e..3c360f59265 100644 --- a/components/gantry/singleaxis/singleaxis.go +++ b/components/gantry/singleaxis/singleaxis.go @@ -30,9 +30,6 @@ var ( homingTimeout = time.Duration(15e9) ) -// limitErrorMargin is added or subtracted from the location of the limit switch to ensure the switch is not passed. -const limitErrorMargin = 0.25 - // Config is used for converting singleAxis config attributes. type Config struct { Board string `json:"board,omitempty"` // used to read limit switch pins and control motor with gpio pins @@ -372,7 +369,7 @@ func (g *singleAxis) homeLimSwitch(ctx context.Context) error { if g.positionRange == 0 { g.logger.CError(ctx, "positionRange is 0 or not a valid number") } else { - g.logger.CDebugf(ctx, "positionA: %0.2f positionB: %0.2f range: %0.2f", positionA, positionB, g.positionRange) + g.logger.CInfof(ctx, "positionA: %0.2f positionB: %0.2f range: %0.2f", positionA, positionB, g.positionRange) } // Go to start position at the middle of the axis. @@ -548,15 +545,15 @@ func (g *singleAxis) MoveToPosition(ctx context.Context, positions, speeds []flo // Currently needs to be moved by underlying gantry motor. if len(g.limitSwitchPins) > 0 { // Stops if position x is past the 0 limit switch - if x <= (g.positionLimits[0] + limitErrorMargin) { - g.logger.CError(ctx, "Cannot move past limit switch!") - return g.motor.Stop(ctx, extra) + if x < g.positionLimits[0] { + err := errors.New("cannot move past limit switch") + return multierr.Combine(err, g.motor.Stop(ctx, extra)) } // Stops if position x is past the at-length limit switch - if x >= (g.positionLimits[1] - limitErrorMargin) { - g.logger.CError(ctx, "Cannot move past limit switch!") - return g.motor.Stop(ctx, extra) + if x > g.positionLimits[1] { + err := errors.New("cannot move past limit switch") + return multierr.Combine(err, g.motor.Stop(ctx, extra)) } } diff --git a/components/gantry/singleaxis/singleaxis_test.go b/components/gantry/singleaxis/singleaxis_test.go index 1ffc6da2514..226f391e06f 100644 --- a/components/gantry/singleaxis/singleaxis_test.go +++ b/components/gantry/singleaxis/singleaxis_test.go @@ -666,12 +666,21 @@ func TestMoveToPosition(t *testing.T) { err = fakegantry.MoveToPosition(ctx, pos, speed, nil) test.That(t, err.Error(), test.ShouldContainSubstring, "out of range") + pos = []float64{0} fakegantry.lengthMm = float64(4) fakegantry.positionLimits = []float64{0, 4} fakegantry.limitSwitchPins = []string{"1", "2"} err = fakegantry.MoveToPosition(ctx, pos, speed, nil) test.That(t, err, test.ShouldBeNil) + pos = []float64{4} + fakegantry.lengthMm = float64(4) + fakegantry.positionLimits = []float64{0, 10} + fakegantry.limitSwitchPins = []string{"1", "2"} + err = fakegantry.MoveToPosition(ctx, pos, speed, nil) + test.That(t, err, test.ShouldBeNil) + + pos = []float64{1} fakegantry.lengthMm = float64(4) fakegantry.positionLimits = []float64{0.01, .01} fakegantry.limitSwitchPins = []string{"1", "2"} @@ -868,10 +877,10 @@ func TestGoToInputs(t *testing.T) { inputs = []referenceframe.Input{{Value: 1.0}} err = fakegantry.GoToInputs(ctx, inputs) - test.That(t, err, test.ShouldBeNil) + test.That(t, err.Error(), test.ShouldContainSubstring, "cannot move past limit switch") err = fakegantry.GoToInputs(ctx, inputs, inputs) - test.That(t, err, test.ShouldBeNil) + test.That(t, err.Error(), test.ShouldContainSubstring, "cannot move past limit switch") err = fakegantry.GoToInputs(ctx) test.That(t, err, test.ShouldBeNil) diff --git a/components/generic/generic.go b/components/generic/generic.go index e19bffff5a9..218b6fe5158 100644 --- a/components/generic/generic.go +++ b/components/generic/generic.go @@ -1,4 +1,7 @@ -// Package generic defines an abstract generic device and DoCommand() method +// Package generic defines an abstract generic device and DoCommand() method. +// For more information, see the [generic component docs]. +// +// [generic component docs]: https://docs.viam.com/components/generic/ package generic import ( diff --git a/components/gripper/gripper.go b/components/gripper/gripper.go index b34e6c0d66c..379e5186bdb 100644 --- a/components/gripper/gripper.go +++ b/components/gripper/gripper.go @@ -1,4 +1,7 @@ // Package gripper defines a robotic gripper. +// For more information, see the [gripper component docs]. +// +// [gripper component docs]: https://docs.viam.com/components/gripper/ package gripper import ( @@ -34,6 +37,7 @@ func Named(name string) resource.Name { } // A Gripper represents a physical robotic gripper. +// For more information, see the [gripper component docs]. // // Open example: // @@ -44,6 +48,8 @@ func Named(name string) resource.Name { // // // Grab with the gripper. // grabbed, err := myGripper.Grab(context.Background(), nil) +// +// [gripper component docs]: https://docs.viam.com/components/gripper/ type Gripper interface { resource.Resource resource.Shaped diff --git a/components/input/input.go b/components/input/input.go index 37c8eb74d40..cdfb8b19355 100644 --- a/components/input/input.go +++ b/components/input/input.go @@ -1,4 +1,7 @@ // Package input provides human input, such as buttons, switches, knobs, gamepads, joysticks, keyboards, mice, etc. +// For more information, see the [input controller component docs]. +// +// [input controller component docs]: https://docs.viam.com/components/input-controller/ package input import ( @@ -35,6 +38,7 @@ func Named(name string) resource.Name { // Controller is a logical "container" more than an actual device // Could be a single gamepad, or a collection of digitalInterrupts and analogReaders, a keyboard, etc. +// For more information, see the [input controller component docs]. // // Controls example: // @@ -64,6 +68,8 @@ func Named(name string) resource.Name { // return errors.New("button `ButtonStart` not found; controller may be disconnected") // } // Mycontroller.RegisterControlCallback(context.Background(), input.ButtonStart, triggers, printStartTime, nil) +// +// [input controller component docs]: https://docs.viam.com/components/input-controller/ type Controller interface { resource.Resource diff --git a/components/motor/motor.go b/components/motor/motor.go index 03f68937de5..ec8ed554706 100644 --- a/components/motor/motor.go +++ b/components/motor/motor.go @@ -1,3 +1,7 @@ +// Package motor defines machines that convert electricity into rotary motion. +// For more information, see the [motor component docs]. +// +// [motor component docs]: https://docs.viam.com/components/motor/ package motor import ( @@ -37,6 +41,7 @@ const SubtypeName = "motor" var API = resource.APINamespaceRDK.WithComponentType(SubtypeName) // A Motor represents a physical motor connected to a board. +// For more information, see the [motor component docs]. // // SetPower example: // @@ -92,6 +97,8 @@ var API = resource.APINamespaceRDK.WithComponentType(SubtypeName) // logger.Info(powered) // logger.Info("Power percent:") // logger.Info(pct) +// +// [motor component docs]: https://docs.viam.com/components/motor/ type Motor interface { resource.Resource resource.Actuator diff --git a/components/motor/ulnstepper/28byj-48.go b/components/motor/ulnstepper/28byj-48.go index 8c1768173d9..6a8a217678d 100644 --- a/components/motor/ulnstepper/28byj-48.go +++ b/components/motor/ulnstepper/28byj-48.go @@ -25,13 +25,13 @@ import ( "github.com/pkg/errors" "go.uber.org/multierr" + "go.viam.com/utils" "go.viam.com/rdk/components/board" "go.viam.com/rdk/components/motor" "go.viam.com/rdk/logging" "go.viam.com/rdk/operation" "go.viam.com/rdk/resource" - "go.viam.com/rdk/utils" ) var ( @@ -165,7 +165,7 @@ type uln28byj struct { motorName string // state - workers utils.StoppableWorkers + workers *utils.StoppableWorkers lock sync.Mutex opMgr *operation.SingleOperationManager doRunDone func() @@ -185,7 +185,7 @@ func (m *uln28byj) doRun() { // start a new doRun var doRunCtx context.Context doRunCtx, m.doRunDone = context.WithCancel(context.Background()) - m.workers = utils.NewStoppableWorkers(func(ctx context.Context) { + m.workers = utils.NewBackgroundStoppableWorkers(func(ctx context.Context) { for { select { case <-doRunCtx.Done(): diff --git a/components/movementsensor/adxl345/adxl345.go b/components/movementsensor/adxl345/adxl345.go index f449dd63001..30ad4b10b21 100644 --- a/components/movementsensor/adxl345/adxl345.go +++ b/components/movementsensor/adxl345/adxl345.go @@ -169,7 +169,7 @@ type adxl345 struct { linearAcceleration r3.Vector err movementsensor.LastError - workers rutils.StoppableWorkers + workers *utils.StoppableWorkers } // newAdxl345 is a constructor to create a new object representing an ADXL345 accelerometer. @@ -256,7 +256,7 @@ func makeAdxl345( // Now, turn on the background goroutine that constantly reads from the chip and stores data in // the object we created. - sensor.workers = rutils.NewStoppableWorkers(func(cancelContext context.Context) { + sensor.workers = utils.NewBackgroundStoppableWorkers(func(cancelContext context.Context) { // Reading data a thousand times per second is probably fast enough. timer := time.NewTicker(time.Millisecond) defer timer.Stop() @@ -337,7 +337,7 @@ func makeAdxl345( } func (adxl *adxl345) startInterruptMonitoring(ticksChan chan board.Tick) { - adxl.workers.AddWorkers(func(cancelContext context.Context) { + adxl.workers.Add(func(cancelContext context.Context) { for { select { case <-cancelContext.Done(): diff --git a/components/movementsensor/gpsrtk/gpsrtk.go b/components/movementsensor/gpsrtk/gpsrtk.go index 268e8d12218..439775dc346 100644 --- a/components/movementsensor/gpsrtk/gpsrtk.go +++ b/components/movementsensor/gpsrtk/gpsrtk.go @@ -26,13 +26,13 @@ import ( "github.com/go-gnss/rtcm/rtcm3" "github.com/golang/geo/r3" geo "github.com/kellydunn/golang-geo" + "go.viam.com/utils" "go.viam.com/rdk/components/movementsensor" "go.viam.com/rdk/components/movementsensor/gpsutils" "go.viam.com/rdk/logging" "go.viam.com/rdk/resource" "go.viam.com/rdk/spatialmath" - "go.viam.com/rdk/utils" ) // gpsrtk is an nmea movementsensor model that can intake RTK correction data. @@ -41,7 +41,7 @@ type gpsrtk struct { resource.AlwaysRebuild logger logging.Logger - workers utils.StoppableWorkers + workers *utils.StoppableWorkers err movementsensor.LastError isClosed bool @@ -62,7 +62,7 @@ func (g *gpsrtk) start() error { if g.workers != nil { return errors.New("do not call start() twice on the same object") } - g.workers = utils.NewStoppableWorkers(g.receiveAndWriteCorrectionData) + g.workers = utils.NewBackgroundStoppableWorkers(g.receiveAndWriteCorrectionData) return nil } diff --git a/components/movementsensor/gpsutils/cachedData.go b/components/movementsensor/gpsutils/cachedData.go index 4b449e7fb46..894d87c2bb1 100644 --- a/components/movementsensor/gpsutils/cachedData.go +++ b/components/movementsensor/gpsutils/cachedData.go @@ -8,6 +8,7 @@ import ( "github.com/golang/geo/r3" geo "github.com/kellydunn/golang-geo" "github.com/pkg/errors" + goutils "go.viam.com/utils" "go.viam.com/rdk/components/movementsensor" "go.viam.com/rdk/logging" @@ -36,7 +37,7 @@ type CachedData struct { dev DataReader logger logging.Logger - workers utils.StoppableWorkers + workers *goutils.StoppableWorkers } // CommonReadings struct contains the most common values we output in sensor reading. @@ -55,7 +56,7 @@ func NewCachedData(dev DataReader, logger logging.Logger) *CachedData { dev: dev, logger: logger, } - g.workers = utils.NewStoppableWorkers(g.start) + g.workers = goutils.NewBackgroundStoppableWorkers(g.start) return &g } diff --git a/components/movementsensor/gpsutils/i2c_data_reader.go b/components/movementsensor/gpsutils/i2c_data_reader.go index 9e991fa9414..0a67575fee0 100644 --- a/components/movementsensor/gpsutils/i2c_data_reader.go +++ b/components/movementsensor/gpsutils/i2c_data_reader.go @@ -14,7 +14,6 @@ import ( "go.viam.com/rdk/components/board/genericlinux/buses" "go.viam.com/rdk/components/movementsensor" "go.viam.com/rdk/logging" - rdkutils "go.viam.com/rdk/utils" ) // PmtkI2cDataReader implements the DataReader interface for a PMTK device by communicating with it @@ -22,7 +21,7 @@ import ( type PmtkI2cDataReader struct { data chan string - workers rdkutils.StoppableWorkers + workers *utils.StoppableWorkers logger logging.Logger bus buses.I2C @@ -67,7 +66,7 @@ func NewI2cDataReader( return nil, err } - reader.workers = rdkutils.NewStoppableWorkers(reader.backgroundWorker) + reader.workers = utils.NewBackgroundStoppableWorkers(reader.backgroundWorker) return &reader, nil } diff --git a/components/movementsensor/gpsutils/serial_data_reader.go b/components/movementsensor/gpsutils/serial_data_reader.go index b0db6b7a368..4083a944c84 100644 --- a/components/movementsensor/gpsutils/serial_data_reader.go +++ b/components/movementsensor/gpsutils/serial_data_reader.go @@ -10,9 +10,9 @@ import ( "io" "github.com/jacobsa/go-serial/serial" + "go.viam.com/utils" "go.viam.com/rdk/logging" - "go.viam.com/rdk/utils" ) // SerialDataReader implements the DataReader interface (defined in component.go) by interacting @@ -20,7 +20,7 @@ import ( type SerialDataReader struct { dev io.ReadWriteCloser data chan string - workers utils.StoppableWorkers + workers *utils.StoppableWorkers logger logging.Logger } @@ -61,7 +61,7 @@ func NewSerialDataReader( data: data, logger: logger, } - reader.workers = utils.NewStoppableWorkers(reader.backgroundWorker) + reader.workers = utils.NewBackgroundStoppableWorkers(reader.backgroundWorker) return &reader, nil } diff --git a/components/movementsensor/gpsutils/vrs.go b/components/movementsensor/gpsutils/vrs.go index 143daba4fbf..46a0512563b 100644 --- a/components/movementsensor/gpsutils/vrs.go +++ b/components/movementsensor/gpsutils/vrs.go @@ -16,7 +16,6 @@ import ( goutils "go.viam.com/utils" "go.viam.com/rdk/logging" - "go.viam.com/rdk/utils" ) const ( @@ -28,7 +27,7 @@ type VRS struct { ntripInfo *NtripInfo readerWriter *bufio.ReadWriter conn net.Conn - workers utils.StoppableWorkers + workers *goutils.StoppableWorkers logger logging.Logger } @@ -160,7 +159,7 @@ func (vrs *VRS) startGGAThread(getGGA func() (string, error)) { vrs.workers.Stop() } - vrs.workers = utils.NewStoppableWorkers(func(cancelCtx context.Context) { + vrs.workers = goutils.NewBackgroundStoppableWorkers(func(cancelCtx context.Context) { ticker := time.NewTicker(time.Duration(vrsGGARateSec) * time.Second) defer ticker.Stop() diff --git a/components/movementsensor/imuvectornav/imu.go b/components/movementsensor/imuvectornav/imu.go index 430017867f3..77b6d16b041 100644 --- a/components/movementsensor/imuvectornav/imu.go +++ b/components/movementsensor/imuvectornav/imu.go @@ -74,13 +74,12 @@ type vectornav struct { spiMu sync.Mutex polling int - cancelFunc func() - activeBackgroundWorkers sync.WaitGroup - bus buses.SPI - cs string - speed int - logger logging.Logger - busClosed bool + workers *goutils.StoppableWorkers + bus buses.SPI + cs string + speed int + logger logging.Logger + busClosed bool bdVX float64 bdVY float64 @@ -212,16 +211,13 @@ func newVectorNav( if err != nil { return nil, err } - var cancelCtx context.Context - cancelCtx, v.cancelFunc = context.WithCancel(context.Background()) + // optionally start a polling goroutine if pfreq > 0 { logger.CDebugf(ctx, "vecnav: will pool at %d Hz", pfreq) waitCh := make(chan struct{}) s := 1.0 / float64(pfreq) - v.activeBackgroundWorkers.Add(1) - goutils.PanicCapturingGo(func() { - defer v.activeBackgroundWorkers.Done() + v.workers = goutils.NewBackgroundStoppableWorkers(func(cancelCtx context.Context) { timer := time.NewTicker(time.Duration(s * float64(time.Second))) defer timer.Stop() close(waitCh) @@ -539,9 +535,8 @@ func (vn *vectornav) compensateDVBias(ctx context.Context, smpSize uint) error { func (vn *vectornav) Close(ctx context.Context) error { vn.logger.CDebug(ctx, "closing vecnav imu") - vn.cancelFunc() vn.busClosed = true - vn.activeBackgroundWorkers.Wait() + vn.workers.Stop() vn.logger.CDebug(ctx, "closed vecnav imu") return nil } diff --git a/components/movementsensor/imuwit/imu.go b/components/movementsensor/imuwit/imu.go index 9013bfb32eb..f3655438bb0 100644 --- a/components/movementsensor/imuwit/imu.go +++ b/components/movementsensor/imuwit/imu.go @@ -99,7 +99,7 @@ type wit struct { mu sync.Mutex reconfigMu sync.Mutex port io.ReadWriteCloser - workers rutils.StoppableWorkers + workers *utils.StoppableWorkers logger logging.Logger baudRate uint serialPath string @@ -298,7 +298,7 @@ func newWit( func (imu *wit) startUpdateLoop(portReader *bufio.Reader, logger logging.Logger) { imu.hasMagnetometer = false - imu.workers = rutils.NewStoppableWorkers(func(ctx context.Context) { + imu.workers = utils.NewBackgroundStoppableWorkers(func(ctx context.Context) { defer utils.UncheckedErrorFunc(func() error { if imu.port != nil { if err := imu.port.Close(); err != nil { diff --git a/components/movementsensor/imuwit/imuhwt905.go b/components/movementsensor/imuwit/imuhwt905.go index 7a53b5d33c5..9044e467a68 100644 --- a/components/movementsensor/imuwit/imuhwt905.go +++ b/components/movementsensor/imuwit/imuhwt905.go @@ -17,11 +17,11 @@ import ( "time" slib "github.com/jacobsa/go-serial/serial" + "go.viam.com/utils" "go.viam.com/rdk/components/movementsensor" "go.viam.com/rdk/logging" "go.viam.com/rdk/resource" - "go.viam.com/rdk/utils" ) var model905 = resource.DefaultModelFamily.WithModel("imu-wit-hwt905") @@ -77,7 +77,7 @@ func newWit905( func (imu *wit) start905UpdateLoop(portReader *bufio.Reader, logger logging.Logger) { imu.hasMagnetometer = false - imu.workers = utils.NewStoppableWorkers(func(cancelCtx context.Context) { + imu.workers = utils.NewBackgroundStoppableWorkers(func(cancelCtx context.Context) { for { if cancelCtx.Err() != nil { return diff --git a/components/movementsensor/movementsensor.go b/components/movementsensor/movementsensor.go index 78adeffe749..e458b29343f 100644 --- a/components/movementsensor/movementsensor.go +++ b/components/movementsensor/movementsensor.go @@ -1,4 +1,7 @@ -// Package movementsensor defines the interfaces of a MovementSensor +// Package movementsensor defines the interfaces of a MovementSensor. +// For more information, see the [movement sensor component docs]. +// +// [movement sensor component docs]: https://docs.viam.com/components/movement-sensor/ package movementsensor import ( @@ -65,6 +68,7 @@ func Named(name string) resource.Name { } // A MovementSensor reports information about the robot's direction, position and speed. +// For more information, see the [movement sensor component docs]. // // Position example: // @@ -117,6 +121,8 @@ func Named(name string) resource.Name { // // // Get the accuracy of the movement sensor. // accuracy, err := myMovementSensor.Accuracy(context.Background(), nil) +// +// [movement sensor component docs]: https://docs.viam.com/components/movement-sensor/ type MovementSensor interface { resource.Sensor resource.Resource diff --git a/components/movementsensor/mpu6050/mpu6050.go b/components/movementsensor/mpu6050/mpu6050.go index 8ef50a0b605..68068158e4b 100644 --- a/components/movementsensor/mpu6050/mpu6050.go +++ b/components/movementsensor/mpu6050/mpu6050.go @@ -29,6 +29,7 @@ import ( "github.com/golang/geo/r3" geo "github.com/kellydunn/golang-geo" "github.com/pkg/errors" + goutils "go.viam.com/utils" "go.viam.com/rdk/components/board/genericlinux/buses" "go.viam.com/rdk/components/movementsensor" @@ -83,7 +84,7 @@ type mpu6050 struct { // Stores the most recent error from the background goroutine err movementsensor.LastError - workers utils.StoppableWorkers + workers *goutils.StoppableWorkers logger logging.Logger } @@ -167,7 +168,7 @@ func makeMpu6050( // Now, turn on the background goroutine that constantly reads from the chip and stores data in // the object we created. - sensor.workers = utils.NewStoppableWorkers(func(cancelCtx context.Context) { + sensor.workers = goutils.NewBackgroundStoppableWorkers(func(cancelCtx context.Context) { // Reading data a thousand times per second is probably fast enough. timer := time.NewTicker(time.Millisecond) defer timer.Stop() diff --git a/components/movementsensor/wheeledodometry/wheeledodometry.go b/components/movementsensor/wheeledodometry/wheeledodometry.go index 4326836303b..c5095646926 100644 --- a/components/movementsensor/wheeledodometry/wheeledodometry.go +++ b/components/movementsensor/wheeledodometry/wheeledodometry.go @@ -12,6 +12,7 @@ import ( "github.com/golang/geo/r3" geo "github.com/kellydunn/golang-geo" + goutils "go.viam.com/utils" "go.viam.com/rdk/components/base" "go.viam.com/rdk/components/motor" @@ -76,7 +77,7 @@ type odometry struct { useCompass bool shiftPos bool - workers utils.StoppableWorkers + workers *goutils.StoppableWorkers mu sync.Mutex logger logging.Logger } @@ -368,7 +369,7 @@ func (o *odometry) checkBaseProps(ctx context.Context) { // https://stuff.mit.edu/afs/athena/course/6/6.186/OldFiles/2005/doc/odomtutorial/odomtutorial.pdf func (o *odometry) trackPosition() { // Spawn a new goroutine to do all the work in the background. - o.workers = utils.NewStoppableWorkers(func(ctx context.Context) { + o.workers = goutils.NewBackgroundStoppableWorkers(func(ctx context.Context) { ticker := time.NewTicker(time.Duration(o.timeIntervalMSecs) * time.Millisecond) for { select { diff --git a/components/posetracker/pose_tracker.go b/components/posetracker/pose_tracker.go index 7d3e9586d00..969f2c8a8c9 100644 --- a/components/posetracker/pose_tracker.go +++ b/components/posetracker/pose_tracker.go @@ -1,5 +1,5 @@ // Package posetracker contains the interface and gRPC infrastructure -// for a pose tracker component +// for a pose tracker component. package posetracker import ( diff --git a/components/powersensor/powersensor.go b/components/powersensor/powersensor.go index 5e703d6c9e2..bcf371bd39b 100644 --- a/components/powersensor/powersensor.go +++ b/components/powersensor/powersensor.go @@ -1,4 +1,7 @@ -// Package powersensor defines the interfaces of a powersensor +// Package powersensor defines the interfaces of a powersensor. +// For more information, see the [power sensor component docs]. +// +// [power sensor component docs]: https://docs.viam.com/components/power-sensor/ package powersensor import ( @@ -48,6 +51,7 @@ func Named(name string) resource.Name { } // A PowerSensor reports information about voltage, current and power. +// For more information, see the [power sensor component docs]. // // Voltage example: // @@ -63,6 +67,8 @@ func Named(name string) resource.Name { // // // Get the power measurement from device in watts. // power, err := myPowerSensor.Power(context.Background(), nil) +// +// [power sensor component docs]: https://docs.viam.com/components/power-sensor/ type PowerSensor interface { resource.Sensor resource.Resource diff --git a/components/sensor/sensor.go b/components/sensor/sensor.go index 2df2b048d2e..db8debd8d04 100644 --- a/components/sensor/sensor.go +++ b/components/sensor/sensor.go @@ -1,4 +1,7 @@ // Package sensor defines an abstract sensing device that can provide measurement readings. +// For more information, see the [sensor component docs]. +// +// [sensor component docs]: https://docs.viam.com/components/sensor/ package sensor import ( diff --git a/components/servo/servo.go b/components/servo/servo.go index 9846a11a315..392dc61fe28 100644 --- a/components/servo/servo.go +++ b/components/servo/servo.go @@ -1,3 +1,7 @@ +// Package servo supports “RC” or “hobby” servo motors. +// For more information, see the [servo component docs]. +// +// [servo component docs]: https://docs.viam.com/components/servo/ package servo import ( @@ -31,6 +35,7 @@ const SubtypeName = "servo" var API = resource.APINamespaceRDK.WithComponentType(SubtypeName) // A Servo represents a physical servo connected to a board. +// For more information, see the [servo component docs]. // // Move example: // @@ -50,6 +55,8 @@ var API = resource.APINamespaceRDK.WithComponentType(SubtypeName) // // logger.Info("Position 1: ", pos1) // logger.Info("Position 2: ", pos2) +// +// [servo component docs]: https://docs.viam.com/components/servo/ type Servo interface { resource.Resource resource.Actuator diff --git a/config/config.go b/config/config.go index 476ba07de64..ca708883684 100644 --- a/config/config.go +++ b/config/config.go @@ -2,6 +2,7 @@ package config import ( + "bytes" "crypto/tls" "encoding/json" "fmt" @@ -15,6 +16,7 @@ import ( "time" "github.com/pkg/errors" + "go.viam.com/utils/artifact" "go.viam.com/utils/jwks" "go.viam.com/utils/pexec" "go.viam.com/utils/rpc" @@ -68,6 +70,11 @@ type Config struct { // Revision contains the current revision of the config. Revision string + + // toCache stores the JSON marshalled version of the config to be cached. It should be a copy of + // the config pulled from cloud with minor changes. + // This version is kept because the config is changed as it moves through the system. + toCache []byte } // NOTE: This data must be maintained with what is in Config. @@ -238,6 +245,29 @@ func (c Config) FindComponent(name string) *resource.Config { return nil } +// SetToCache sets toCache with a marshalled copy of the config passed in. +func (c *Config) SetToCache(cfg *Config) error { + md, err := json.MarshalIndent(cfg, "", " ") + if err != nil { + return err + } + c.toCache = md + return nil +} + +// StoreToCache caches the toCache. +func (c *Config) StoreToCache() error { + if c.toCache == nil { + return errors.New("no unprocessed config to cache") + } + if err := os.MkdirAll(ViamDotDir, 0o700); err != nil { + return err + } + reader := bytes.NewReader(c.toCache) + path := getCloudCacheFilePath(c.Cloud.ID) + return artifact.AtomicStore(path, reader, c.Cloud.ID) +} + // UnmarshalJSON unmarshals JSON into the config and adjusts some // names if they are not fully filled in. func (c *Config) UnmarshalJSON(data []byte) error { diff --git a/config/reader.go b/config/reader.go index d0cedc7eeae..b0452a40b7a 100644 --- a/config/reader.go +++ b/config/reader.go @@ -17,7 +17,6 @@ import ( "github.com/pkg/errors" apppb "go.viam.com/api/app/v1" "go.viam.com/utils" - "go.viam.com/utils/artifact" "go.viam.com/utils/rpc" "golang.org/x/sys/cpu" @@ -114,22 +113,6 @@ func readFromCache(id string) (*Config, error) { return unprocessedConfig, nil } -func storeToCache(id string, cfg *Config) error { - if err := os.MkdirAll(ViamDotDir, 0o700); err != nil { - return err - } - - md, err := json.MarshalIndent(cfg, "", " ") - if err != nil { - return err - } - reader := bytes.NewReader(md) - - path := getCloudCacheFilePath(id) - - return artifact.AtomicStore(path, reader, id) -} - func clearCache(id string) { utils.UncheckedErrorFunc(func() error { return os.Remove(getCloudCacheFilePath(id)) @@ -318,10 +301,9 @@ func readFromCloud( unprocessedConfig.Cloud.TLSCertificate = tls.certificate unprocessedConfig.Cloud.TLSPrivateKey = tls.privateKey - if err := storeToCache(cloudCfg.ID, unprocessedConfig); err != nil { - logger.Errorw("failed to cache config", "error", err) + if err := cfg.SetToCache(unprocessedConfig); err != nil { + logger.Errorw("failed to set toCache on config", "error", err) } - return cfg, nil } diff --git a/config/reader_test.go b/config/reader_test.go index 495bda36e17..570d7fcb5d8 100644 --- a/config/reader_test.go +++ b/config/reader_test.go @@ -2,7 +2,6 @@ package config import ( "context" - "errors" "fmt" "io/fs" "os" @@ -11,6 +10,7 @@ import ( "time" "github.com/google/uuid" + "github.com/pkg/errors" pb "go.viam.com/api/app/v1" "go.viam.com/test" @@ -69,7 +69,6 @@ func TestFromReader(t *testing.T) { appAddress := fmt.Sprintf("http://%s", fakeServer.Addr().String()) cfgText := fmt.Sprintf(`{"cloud":{"id":%q,"app_address":%q,"secret":%q}}`, robotPartID, appAddress, secret) gotCfg, err := FromReader(ctx, "", strings.NewReader(cfgText), logger) - defer clearCache(robotPartID) test.That(t, err, test.ShouldBeNil) expectedCloud := *cloudResponse @@ -79,6 +78,8 @@ func TestFromReader(t *testing.T) { expectedCloud.RefreshInterval = time.Duration(10000000000) test.That(t, gotCfg.Cloud, test.ShouldResemble, &expectedCloud) + test.That(t, gotCfg.StoreToCache(), test.ShouldBeNil) + defer clearCache(robotPartID) cachedCfg, err := readFromCache(robotPartID) test.That(t, err, test.ShouldBeNil) expectedCloud.AppAddress = "" @@ -102,7 +103,10 @@ func TestFromReader(t *testing.T) { MachineID: "the-machine", } cachedConf := &Config{Cloud: cachedCloud} - err := storeToCache(robotPartID, cachedConf) + + cfgToCache := &Config{Cloud: &Cloud{ID: robotPartID}} + cfgToCache.SetToCache(cachedConf) + err := cfgToCache.StoreToCache() test.That(t, err, test.ShouldBeNil) defer clearCache(robotPartID) @@ -153,7 +157,6 @@ func TestFromReader(t *testing.T) { appAddress := fmt.Sprintf("http://%s", fakeServer.Addr().String()) cfgText := fmt.Sprintf(`{"cloud":{"id":%q,"app_address":%q,"secret":%q}}`, robotPartID, appAddress, secret) gotCfg, err := FromReader(ctx, "", strings.NewReader(cfgText), logger) - defer clearCache(robotPartID) test.That(t, err, test.ShouldBeNil) expectedCloud := *cloudResponse @@ -161,6 +164,9 @@ func TestFromReader(t *testing.T) { expectedCloud.RefreshInterval = time.Duration(10000000000) test.That(t, gotCfg.Cloud, test.ShouldResemble, &expectedCloud) + err = gotCfg.StoreToCache() + defer clearCache(robotPartID) + test.That(t, err, test.ShouldBeNil) cachedCfg, err := readFromCache(robotPartID) test.That(t, err, test.ShouldBeNil) expectedCloud.AppAddress = "" @@ -191,13 +197,20 @@ func TestStoreToCache(t *testing.T) { } cfg.Cloud = cloud - // store our config to the cloud - err = storeToCache(cfg.Cloud.ID, cfg) + // errors if no unprocessed config to cache + cfgToCache := &Config{Cloud: &Cloud{ID: "forCachingTest"}} + err = cfgToCache.StoreToCache() + test.That(t, err.Error(), test.ShouldContainSubstring, "no unprocessed config to cache") + + // store our config to the cache + cfgToCache.SetToCache(cfg) + err = cfgToCache.StoreToCache() test.That(t, err, test.ShouldBeNil) // read config from cloud, confirm consistency cloudCfg, err := readFromCloud(ctx, cfg, nil, true, false, logger) test.That(t, err, test.ShouldBeNil) + cloudCfg.toCache = nil test.That(t, cloudCfg, test.ShouldResemble, cfg) // Modify our config @@ -207,10 +220,12 @@ func TestStoreToCache(t *testing.T) { // read config from cloud again, confirm that the cached config differs from cfg cloudCfg2, err := readFromCloud(ctx, cfg, nil, true, false, logger) test.That(t, err, test.ShouldBeNil) - test.That(t, cloudCfg2, test.ShouldNotResemble, cfg) + cloudCfg2.toCache = nil + test.That(t, cloudCfg2, test.ShouldNotResemble, cfgToCache) // store the updated config to the cloud - err = storeToCache(cfg.Cloud.ID, cfg) + cfgToCache.SetToCache(cfg) + err = cfgToCache.StoreToCache() test.That(t, err, test.ShouldBeNil) test.That(t, cfg.Ensure(true, logger), test.ShouldBeNil) @@ -218,6 +233,7 @@ func TestStoreToCache(t *testing.T) { // read updated cloud config, confirm that it now matches our updated cfg cloudCfg3, err := readFromCloud(ctx, cfg, nil, true, false, logger) test.That(t, err, test.ShouldBeNil) + cloudCfg3.toCache = nil test.That(t, cloudCfg3, test.ShouldResemble, cfg) } @@ -304,7 +320,9 @@ func TestReadTLSFromCache(t *testing.T) { defer clearCache(robotPartID) cfg.Cloud = nil - err = storeToCache(robotPartID, cfg) + cfgToCache := &Config{Cloud: &Cloud{ID: robotPartID}} + cfgToCache.SetToCache(cfg) + err = cfgToCache.StoreToCache() test.That(t, err, test.ShouldBeNil) tls := tlsConfig{} @@ -315,11 +333,14 @@ func TestReadTLSFromCache(t *testing.T) { t.Run("invalid cached TLS", func(t *testing.T) { defer clearCache(robotPartID) cloud := &Cloud{ + ID: robotPartID, TLSPrivateKey: "key", } cfg.Cloud = cloud - err = storeToCache(robotPartID, cfg) + cfgToCache := &Config{Cloud: &Cloud{ID: robotPartID}} + cfgToCache.SetToCache(cfg) + err = cfgToCache.StoreToCache() test.That(t, err, test.ShouldBeNil) tls := tlsConfig{} @@ -333,12 +354,15 @@ func TestReadTLSFromCache(t *testing.T) { t.Run("invalid cached TLS but insecure signaling", func(t *testing.T) { defer clearCache(robotPartID) cloud := &Cloud{ + ID: robotPartID, TLSPrivateKey: "key", SignalingInsecure: true, } cfg.Cloud = cloud - err = storeToCache(robotPartID, cfg) + cfgToCache := &Config{Cloud: &Cloud{ID: robotPartID}} + cfgToCache.SetToCache(cfg) + err = cfgToCache.StoreToCache() test.That(t, err, test.ShouldBeNil) tls := tlsConfig{} @@ -352,12 +376,15 @@ func TestReadTLSFromCache(t *testing.T) { t.Run("valid cached TLS", func(t *testing.T) { defer clearCache(robotPartID) cloud := &Cloud{ + ID: robotPartID, TLSCertificate: "cert", TLSPrivateKey: "key", } cfg.Cloud = cloud - err = storeToCache(robotPartID, cfg) + cfgToCache := &Config{Cloud: &Cloud{ID: robotPartID}} + cfgToCache.SetToCache(cfg) + err = cfgToCache.StoreToCache() test.That(t, err, test.ShouldBeNil) // the config is missing several fields required to start the robot, but this diff --git a/config/watcher_test.go b/config/watcher_test.go index 3cb7df459a0..d1fcdf9c36c 100644 --- a/config/watcher_test.go +++ b/config/watcher_test.go @@ -259,6 +259,16 @@ func TestNewWatcherCloud(t *testing.T) { }}, } + unprocessedFromCfg := func(cfg config.Config) *config.Config { + // the unprocessed config uses the original config read from the cloud, + // and the cloud config is missing a few fields in the proto, meaning a few fields need to be cleared out. + unprocessed, err := cfg.CopyOnlyPublicFields() + test.That(t, err, test.ShouldBeNil) + unprocessed.Cloud.AppAddress = "" + unprocessed.Cloud.RefreshInterval = 10 * time.Second + return unprocessed + } + storeConfigInServer(confToReturn) watcher, err := config.NewWatcher(context.Background(), &config.Config{Cloud: newCloudConf()}, logger) @@ -268,6 +278,7 @@ func TestNewWatcherCloud(t *testing.T) { confToExpect.Cloud.TLSCertificate = certsToReturn.TLSCertificate confToExpect.Cloud.TLSPrivateKey = certsToReturn.TLSPrivateKey test.That(t, confToExpect.Ensure(true, logger), test.ShouldBeNil) + confToExpect.SetToCache(unprocessedFromCfg(confToExpect)) newConf := <-watcher.Config() test.That(t, newConf, test.ShouldResemble, &confToExpect) @@ -305,6 +316,7 @@ func TestNewWatcherCloud(t *testing.T) { confToExpect.Cloud.TLSCertificate = certsToReturn.TLSCertificate confToExpect.Cloud.TLSPrivateKey = certsToReturn.TLSPrivateKey test.That(t, confToExpect.Ensure(true, logger), test.ShouldBeNil) + confToExpect.SetToCache(unprocessedFromCfg(confToExpect)) newConf = <-watcher.Config() test.That(t, newConf, test.ShouldResemble, &confToExpect) @@ -356,6 +368,7 @@ func TestNewWatcherCloud(t *testing.T) { confToExpect.Cloud.TLSCertificate = certsToReturn.TLSCertificate confToExpect.Cloud.TLSPrivateKey = certsToReturn.TLSPrivateKey test.That(t, confToExpect.Ensure(true, logger), test.ShouldBeNil) + confToExpect.SetToCache(unprocessedFromCfg(confToExpect)) newConf = <-watcher.Config() test.That(t, newConf, test.ShouldResemble, &confToExpect) diff --git a/etc/setup.sh b/etc/setup.sh index eff80e9ce83..81ad693e71a 100755 --- a/etc/setup.sh +++ b/etc/setup.sh @@ -30,7 +30,7 @@ do_piOS(){ apt-get update && apt-get install -y build-essential nodejs libnlopt-dev libx264-dev libtensorflowlite-dev ffmpeg libjpeg62-turbo-dev # Install Gostream dependencies - sudo apt-get install -y --no-install-recommends libopus-dev libvpx-dev libx11-dev libxext-dev libopusfile-dev + sudo apt-get install -y --no-install-recommends libopus-dev libx11-dev libxext-dev libopusfile-dev # Install backports apt-get install -y -t $(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)-backports golang-go @@ -173,7 +173,6 @@ do_brew(){ brew "licensefinder" brew "opus" brew "opusfile" - brew "libvpx" brew "tensorflowlite" # Needs to be last EOS diff --git a/go.mod b/go.mod index fcca62f5252..1f8a4b88efb 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module go.viam.com/rdk -go 1.21 +go 1.21.13 require ( github.com/AlekSi/gocov-xml v1.0.0 @@ -44,7 +44,7 @@ require ( github.com/golang/protobuf v1.5.3 github.com/golangci/golangci-lint v1.54.0 github.com/google/flatbuffers v2.0.6+incompatible - github.com/google/go-cmp v0.5.9 + github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.6.0 github.com/gotesttools/gotestfmt/v2 v2.4.1 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 @@ -70,12 +70,12 @@ require ( github.com/pion/mediadevices v0.6.4 github.com/pion/rtp v1.8.7 github.com/rhysd/actionlint v1.6.24 - github.com/rs/cors v1.9.0 + github.com/rs/cors v1.11.1 github.com/sergi/go-diff v1.3.1 github.com/u2takey/ffmpeg-go v0.4.1 github.com/urfave/cli/v2 v2.10.3 github.com/viamrobotics/evdev v0.1.3 - github.com/viamrobotics/webrtc/v3 v3.99.9 + github.com/viamrobotics/webrtc/v3 v3.99.10 github.com/xfmoulet/qoi v0.2.0 go-hep.org/x/hep v0.32.1 go.mongodb.org/mongo-driver v1.11.6 @@ -85,15 +85,15 @@ require ( go.uber.org/zap v1.24.0 go.viam.com/api v0.1.336 go.viam.com/test v1.1.1-0.20220913152726-5da9916c08a2 - go.viam.com/utils v0.1.96 + go.viam.com/utils v0.1.98 goji.io v2.0.2+incompatible - golang.org/x/image v0.15.0 + golang.org/x/image v0.19.0 golang.org/x/mobile v0.0.0-20240112133503-c713f31d574b - golang.org/x/sync v0.6.0 + golang.org/x/sync v0.8.0 golang.org/x/sys v0.20.0 golang.org/x/term v0.20.0 golang.org/x/time v0.3.0 - golang.org/x/tools v0.17.0 + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d gonum.org/v1/gonum v0.12.0 gonum.org/v1/plot v0.12.0 google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 @@ -370,10 +370,10 @@ require ( go4.org/unsafe/assume-no-moving-gc v0.0.0-20230525183740-e7c30c78aeb2 // indirect golang.org/x/crypto v0.23.0 // indirect golang.org/x/exp/typeparams v0.0.0-20230224173230-c95f2b4c22f2 // indirect - golang.org/x/mod v0.14.0 // indirect + golang.org/x/mod v0.17.0 // indirect golang.org/x/net v0.25.0 // indirect golang.org/x/oauth2 v0.10.0 // indirect - golang.org/x/text v0.15.0 // indirect + golang.org/x/text v0.17.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/api v0.126.0 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/go.sum b/go.sum index 643e63beabc..4e547a1cc61 100644 --- a/go.sum +++ b/go.sum @@ -605,8 +605,8 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= @@ -1242,8 +1242,8 @@ github.com/rogpeppe/go-internal v1.6.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= -github.com/rs/cors v1.9.0 h1:l9HGsTsHJcvW14Nk7J9KFz8bzeAWXn3CG6bgt7LsrAE= -github.com/rs/cors v1.9.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= +github.com/rs/cors v1.11.1 h1:eU3gRzXLRK57F5rKMGMZURNdIG4EoAmX8k94r9wXWHA= +github.com/rs/cors v1.11.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= @@ -1439,8 +1439,8 @@ github.com/valyala/quicktemplate v1.6.3/go.mod h1:fwPzK2fHuYEODzJ9pkw0ipCPNHZ2tD github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/viamrobotics/evdev v0.1.3 h1:mR4HFafvbc5Wx4Vp1AUJp6/aITfVx9AKyXWx+rWjpfc= github.com/viamrobotics/evdev v0.1.3/go.mod h1:N6nuZmPz7HEIpM7esNWwLxbYzqWqLSZkfI/1Sccckqk= -github.com/viamrobotics/webrtc/v3 v3.99.9 h1:5FCctlMhO9lr4SJ1TC2WCFocBIriUMb3Sw7i9oDlz2o= -github.com/viamrobotics/webrtc/v3 v3.99.9/go.mod h1:ziH7/S52IyYAeDdwUUl5ZTbuyKe47fWorAz+0z5w6NA= +github.com/viamrobotics/webrtc/v3 v3.99.10 h1:ykE14wm+HkqMD5Ozq4rvhzzfvnXAu14ak/HzA1OCzfY= +github.com/viamrobotics/webrtc/v3 v3.99.10/go.mod h1:ziH7/S52IyYAeDdwUUl5ZTbuyKe47fWorAz+0z5w6NA= github.com/viki-org/dnscache v0.0.0-20130720023526-c70c1f23c5d8/go.mod h1:dniwbG03GafCjFohMDmz6Zc6oCuiqgH6tGNyXTkHzXE= github.com/wcharczuk/go-chart/v2 v2.1.0/go.mod h1:yx7MvAVNcP/kN9lKXM/NTce4au4DFN99j6i1OwDclNA= github.com/wlynxg/anet v0.0.3 h1:PvR53psxFXstc12jelG6f1Lv4MWqE0tI76/hHGjh9rg= @@ -1541,8 +1541,8 @@ go.viam.com/api v0.1.336 h1:mcz3Y5rivgXhsTu/bXkAVDw0/otarq3lCPIRcxhNnIY= go.viam.com/api v0.1.336/go.mod h1:msa4TPrMVeRDcG4YzKA/S6wLEUC7GyHQE973JklrQ10= go.viam.com/test v1.1.1-0.20220913152726-5da9916c08a2 h1:oBiK580EnEIzgFLU4lHOXmGAE3MxnVbeR7s1wp/F3Ps= go.viam.com/test v1.1.1-0.20220913152726-5da9916c08a2/go.mod h1:XM0tej6riszsiNLT16uoyq1YjuYPWlRBweTPRDanIts= -go.viam.com/utils v0.1.96 h1:lZ7m1jm6pkjn60e9OWeHyHcivFIRrFvS/5NGrPXFaDM= -go.viam.com/utils v0.1.96/go.mod h1:GCaRsDlW3p3tOnLo73swpPnfrmmqg6r+1oWhNVEHsDo= +go.viam.com/utils v0.1.98 h1:ZW8C4AcsbxM5FtFwOI9DyYMwQTVCt4pKYfNbdT5anAs= +go.viam.com/utils v0.1.98/go.mod h1:DnwL2Q5zzS9Z2ZlkNzLmormDO/ThxjnucxNXn6r6fM0= go4.org/unsafe/assume-no-moving-gc v0.0.0-20230525183740-e7c30c78aeb2 h1:WJhcL4p+YeDxmZWg141nRm7XC8IDmhz7lk5GpadO1Sg= go4.org/unsafe/assume-no-moving-gc v0.0.0-20230525183740-e7c30c78aeb2/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E= gocv.io/x/gocv v0.25.0/go.mod h1:Rar2PS6DV+T4FL+PM535EImD/h13hGVaHhnCu1xarBs= @@ -1602,8 +1602,8 @@ golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+o golang.org/x/image v0.0.0-20200927104501-e162460cd6b5/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= -golang.org/x/image v0.15.0 h1:kOELfmgrmJlw4Cdb7g/QGuB3CvDrXbqEIww/pNtNBm8= -golang.org/x/image v0.15.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE= +golang.org/x/image v0.19.0 h1:D9FX4QWkLfkeqaC62SonffIIuYdOk/UE2XKUBgRIBIQ= +golang.org/x/image v0.19.0/go.mod h1:y0zrRqlQRWQ5PXaYCOMLTW2fpsxZ8Qh9I/ohnInJEys= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1634,8 +1634,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= -golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1731,8 +1731,8 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1866,8 +1866,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1988,8 +1988,8 @@ golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/tools v0.5.0/go.mod h1:N+Kgy78s5I24c24dU8OfWNEotWjutIs8SnJvn5IDq+k= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc= -golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/gostream/codec/mmal/encoder.go b/gostream/codec/mmal/encoder.go deleted file mode 100644 index 0993a8b40ca..00000000000 --- a/gostream/codec/mmal/encoder.go +++ /dev/null @@ -1,68 +0,0 @@ -//go:build mmal - -// Package mmal contains the mmal video codec. -package mmal - -import ( - "context" - "image" - - ourcodec "go.viam.com/rdk/gostream/codec" - "go.viam.com/rdk/logging" - - "github.com/pion/mediadevices/pkg/codec" - "github.com/pion/mediadevices/pkg/codec/mmal" - "github.com/pion/mediadevices/pkg/prop" -) - -type encoder struct { - codec codec.ReadCloser - img image.Image - logger logging.Logger -} - -// Gives suitable results. Probably want to make this configurable this in the future. -const bitrate = 3_200_000 - -// NewEncoder returns an MMAL encoder that can encode images of the given width and height. It will -// also ensure that it produces key frames at the given interval. -func NewEncoder(width, height, keyFrameInterval int, logger logging.Logger) (ourcodec.VideoEncoder, error) { - enc := &encoder{logger: logger} - - var builder codec.VideoEncoderBuilder - params, err := mmal.NewParams() - if err != nil { - return nil, err - } - builder = ¶ms - params.BitRate = bitrate - params.KeyFrameInterval = keyFrameInterval - - codec, err := builder.BuildVideoEncoder(enc, prop.Media{ - Video: prop.Video{ - Width: width, - Height: height, - }, - }) - if err != nil { - return nil, err - } - enc.codec = codec - - return enc, nil -} - -// Read returns an image for codec to process. -func (v *encoder) Read() (img image.Image, release func(), err error) { - return v.img, nil, nil -} - -// Encode asks the codec to process the given image. -func (v *encoder) Encode(_ context.Context, img image.Image) ([]byte, error) { - v.img = img - data, release, err := v.codec.Read() - dataCopy := make([]byte, len(data)) - copy(dataCopy, data) - release() - return dataCopy, err -} diff --git a/gostream/codec/mmal/utils.go b/gostream/codec/mmal/utils.go deleted file mode 100644 index 5a328e4685d..00000000000 --- a/gostream/codec/mmal/utils.go +++ /dev/null @@ -1,32 +0,0 @@ -//go:build mmal - -package mmal - -import ( - "go.viam.com/rdk/gostream" - "go.viam.com/rdk/gostream/codec" - - "go.viam.com/rdk/logging" -) - -// DefaultStreamConfig configures MMAL as the encoder for a stream. -var DefaultStreamConfig gostream.StreamConfig - -func init() { - DefaultStreamConfig.VideoEncoderFactory = NewEncoderFactory() -} - -// NewEncoderFactory returns an MMAL encoder factory. -func NewEncoderFactory() codec.VideoEncoderFactory { - return &factory{} -} - -type factory struct{} - -func (f *factory) New(width, height, keyFrameInterval int, logger logging.Logger) (codec.VideoEncoder, error) { - return NewEncoder(width, height, keyFrameInterval, logger) -} - -func (f *factory) MIMEType() string { - return "video/H264" -} diff --git a/gostream/codec/vpx/encoder.go b/gostream/codec/vpx/encoder.go deleted file mode 100644 index 20e6612cb3c..00000000000 --- a/gostream/codec/vpx/encoder.go +++ /dev/null @@ -1,94 +0,0 @@ -// Package vpx contains the vpx video codec. -package vpx - -import ( - "context" - "fmt" - "image" - - "github.com/pion/mediadevices/pkg/codec" - "github.com/pion/mediadevices/pkg/codec/vpx" - "github.com/pion/mediadevices/pkg/prop" - - ourcodec "go.viam.com/rdk/gostream/codec" - "go.viam.com/rdk/logging" -) - -type encoder struct { - codec codec.ReadCloser - img image.Image - logger logging.Logger -} - -// Version determines the version of a vpx codec. -type Version string - -// The set of allowed vpx versions. -const ( - Version8 Version = "vp8" - Version9 Version = "vp9" -) - -// Gives suitable results. Probably want to make this configurable this in the future. -const bitrate = 3_200_000 - -// NewEncoder returns a vpx encoder of the given type that can encode images of the given width and height. It will -// also ensure that it produces key frames at the given interval. -func NewEncoder(codecVersion Version, width, height, keyFrameInterval int, logger logging.Logger) (ourcodec.VideoEncoder, error) { - enc := &encoder{logger: logger} - - var builder codec.VideoEncoderBuilder - switch codecVersion { - case Version8: - params, err := vpx.NewVP8Params() - if err != nil { - return nil, err - } - builder = ¶ms - params.BitRate = bitrate - params.KeyFrameInterval = keyFrameInterval - case Version9: - params, err := vpx.NewVP9Params() - if err != nil { - return nil, err - } - builder = ¶ms - params.BitRate = bitrate - params.KeyFrameInterval = keyFrameInterval - default: - return nil, fmt.Errorf("unsupported vpx version: %s", codecVersion) - } - - codec, err := builder.BuildVideoEncoder(enc, prop.Media{ - Video: prop.Video{ - Width: width, - Height: height, - }, - }) - if err != nil { - return nil, err - } - enc.codec = codec - - return enc, nil -} - -// Read returns an image for codec to process. -func (v *encoder) Read() (img image.Image, release func(), err error) { - return v.img, nil, nil -} - -// Encode asks the codec to process the given image. -func (v *encoder) Encode(_ context.Context, img image.Image) ([]byte, error) { - v.img = img - data, release, err := v.codec.Read() - dataCopy := make([]byte, len(data)) - copy(dataCopy, data) - release() - return dataCopy, err -} - -// Close closes the encoder. -func (v *encoder) Close() error { - return v.codec.Close() -} diff --git a/gostream/codec/vpx/utils.go b/gostream/codec/vpx/utils.go deleted file mode 100644 index 5df607217e8..00000000000 --- a/gostream/codec/vpx/utils.go +++ /dev/null @@ -1,40 +0,0 @@ -package vpx - -import ( - "fmt" - - "go.viam.com/rdk/gostream" - "go.viam.com/rdk/gostream/codec" - "go.viam.com/rdk/logging" -) - -// DefaultStreamConfig configures vpx as the encoder for a stream. -var DefaultStreamConfig gostream.StreamConfig - -func init() { - DefaultStreamConfig.VideoEncoderFactory = NewEncoderFactory(Version8) -} - -// NewEncoderFactory returns a vpx factory for the given vpx codec. -func NewEncoderFactory(codecVersion Version) codec.VideoEncoderFactory { - return &factory{codecVersion} -} - -type factory struct { - codecVersion Version -} - -func (f *factory) New(width, height, keyFrameInterval int, logger logging.Logger) (codec.VideoEncoder, error) { - return NewEncoder(f.codecVersion, width, height, keyFrameInterval, logger) -} - -func (f *factory) MIMEType() string { - switch f.codecVersion { - case Version8: - return "video/vp8" - case Version9: - return "video/vp9" - default: - panic(fmt.Errorf("unknown codec version %q", f.codecVersion)) - } -} diff --git a/gostream/media.go b/gostream/media.go index d5c33cc4a25..d5e34d7dc4e 100644 --- a/gostream/media.go +++ b/gostream/media.go @@ -360,6 +360,10 @@ func (pc *producerConsumer[T, U]) stop() { pc.consumerCond.L.Unlock() pc.activeBackgroundWorkers.Wait() + pc.currentMu.Lock() + defer pc.currentMu.Unlock() + pc.current = nil + // reset cancelCtx, cancel := context.WithCancel(WithMIMETypeHint(pc.rootCancelCtx, pc.mimeType)) pc.cancelCtxMu.Lock() diff --git a/gostream/media_test.go b/gostream/media_test.go index fe781543285..7b6c45fb5e9 100644 --- a/gostream/media_test.go +++ b/gostream/media_test.go @@ -191,3 +191,25 @@ func TestStreamMultipleConsumers(t *testing.T) { test.That(t, wrappedImg.released.Load(), test.ShouldBeTrue) } } + +func TestStreamWithoutNext(t *testing.T) { + colors := []*WrappedImage{createWrappedImage(t, rimage.Red)} + + imgSource := &imageSource{WrappedImages: colors} + videoSrc := NewVideoSource(imgSource, prop.Video{}) + + // Start stream + stream, err := videoSrc.Stream(context.Background()) + test.That(t, err, test.ShouldBeNil) + // Get one frame + _, release, err := stream.Next(context.Background()) + test.That(t, err, test.ShouldBeNil) + release() + // Close stream + stream.Close(context.Background()) + + // Spin up stream and close without calling Next + stream, err = videoSrc.Stream(context.Background()) + test.That(t, err, test.ShouldBeNil) + stream.Close(context.Background()) +} diff --git a/gostream/shell.nix b/gostream/shell.nix index 33621f0ee97..2c00f7a4662 100644 --- a/gostream/shell.nix +++ b/gostream/shell.nix @@ -4,7 +4,7 @@ pkgs.mkShell { buildInputs = - [ pkgs.which pkgs.htop pkgs.go pkgs.nodejs pkgs.pkg-config pkgs.libvpx pkgs.x264 pkgs.libopus ] + [ pkgs.which pkgs.htop pkgs.go pkgs.nodejs pkgs.pkg-config pkgs.x264 pkgs.libopus ] ++ pkgs.lib.optionals pkgs.stdenv.isDarwin [ pkgs.darwin.apple_sdk.frameworks.AVFoundation pkgs.darwin.apple_sdk.frameworks.CoreMedia diff --git a/gostream/stream.go b/gostream/stream.go index 687d72c0920..a270f8b2011 100644 --- a/gostream/stream.go +++ b/gostream/stream.go @@ -161,6 +161,9 @@ func (bs *basicStream) Start() { utils.ManagedGo(bs.processOutputAudioChunks, bs.activeBackgroundWorkers.Done) } +// NOTE: (Nick S) This only writes video RTP packets +// if we also need to support writing audio RTP packets, we should split +// this method into WriteVideoRTP and WriteAudioRTP. func (bs *basicStream) WriteRTP(pkt *rtp.Packet) error { return bs.videoTrackLocal.rtpTrack.WriteRTP(pkt) } diff --git a/grpc/conn.go b/grpc/conn.go index f94cb330a51..4f4413afd7c 100644 --- a/grpc/conn.go +++ b/grpc/conn.go @@ -7,13 +7,20 @@ import ( "github.com/viamrobotics/webrtc/v3" "go.viam.com/utils/rpc" + "golang.org/x/exp/maps" googlegrpc "google.golang.org/grpc" + + "go.viam.com/rdk/logging" ) -// ReconfigurableClientConn allows for the underlying client connections to be swapped under the hood. +// ReconfigurableClientConn allows for the underlying client connections to be swapped under the +// hood. A ReconfigurableClientConn may only be used for a connection to a single logical server. type ReconfigurableClientConn struct { connMu sync.RWMutex conn rpc.ClientConn + + onTrackCBByTrackNameMu sync.Mutex + onTrackCBByTrackName map[string]OnTrackCB } // Return this constant such that backoff error logging can compare consecutive errors and reliably @@ -59,6 +66,25 @@ func (c *ReconfigurableClientConn) NewStream( func (c *ReconfigurableClientConn) ReplaceConn(conn rpc.ClientConn) { c.connMu.Lock() c.conn = conn + // It is safe to access this without a mutex as it is only ever nil once at the beginning of the + // ReconfigurableClientConn's lifetime. Before it is shared with clients. + if c.onTrackCBByTrackName == nil { + c.onTrackCBByTrackName = make(map[string]OnTrackCB) + } + + if pc := conn.PeerConn(); pc != nil { + pc.OnTrack(func(trackRemote *webrtc.TrackRemote, rtpReceiver *webrtc.RTPReceiver) { + c.onTrackCBByTrackNameMu.Lock() + onTrackCB, ok := c.onTrackCBByTrackName[trackRemote.StreamID()] + c.onTrackCBByTrackNameMu.Unlock() + if !ok { + logging.Global().Errorf("Callback not found for StreamID (trackName): %s, keys(resOnTrackCBs): %#v", + trackRemote.StreamID(), maps.Keys(c.onTrackCBByTrackName)) + return + } + onTrackCB(trackRemote, rtpReceiver) + }) + } c.connMu.Unlock() } @@ -84,3 +110,17 @@ func (c *ReconfigurableClientConn) Close() error { c.conn = nil return conn.Close() } + +// AddOnTrackSub adds an OnTrack subscription for the track. +func (c *ReconfigurableClientConn) AddOnTrackSub(trackName string, onTrackCB OnTrackCB) { + c.onTrackCBByTrackNameMu.Lock() + defer c.onTrackCBByTrackNameMu.Unlock() + c.onTrackCBByTrackName[trackName] = onTrackCB +} + +// RemoveOnTrackSub removes an OnTrack subscription for the track. +func (c *ReconfigurableClientConn) RemoveOnTrackSub(trackName string) { + c.onTrackCBByTrackNameMu.Lock() + defer c.onTrackCBByTrackNameMu.Unlock() + delete(c.onTrackCBByTrackName, trackName) +} diff --git a/grpc/shared_conn.go b/grpc/shared_conn.go index 88ad2f82ff6..79ba0d0d85b 100644 --- a/grpc/shared_conn.go +++ b/grpc/shared_conn.go @@ -3,7 +3,6 @@ package grpc import ( "context" "errors" - "fmt" "net" "sync" "time" @@ -15,10 +14,10 @@ import ( "go.uber.org/zap" "go.viam.com/utils" "go.viam.com/utils/rpc" + "golang.org/x/exp/maps" googlegrpc "google.golang.org/grpc" "go.viam.com/rdk/logging" - "go.viam.com/rdk/resource" rutils "go.viam.com/rdk/utils" ) @@ -80,8 +79,8 @@ type SharedConn struct { // set to nil before this channel is closed. peerConnFailed chan struct{} - resOnTrackMu sync.Mutex - resOnTrackCBs map[resource.Name]OnTrackCB + onTrackCBByTrackNameMu sync.Mutex + onTrackCBByTrackName map[string]OnTrackCB logger logging.Logger } @@ -106,18 +105,18 @@ func (sc *SharedConn) NewStream( return sc.grpcConn.NewStream(ctx, desc, method, opts...) } -// AddOnTrackSub adds an OnTrack subscription for the resource. -func (sc *SharedConn) AddOnTrackSub(name resource.Name, onTrackCB OnTrackCB) { - sc.resOnTrackMu.Lock() - defer sc.resOnTrackMu.Unlock() - sc.resOnTrackCBs[name] = onTrackCB +// AddOnTrackSub adds an OnTrack subscription for the track. +func (sc *SharedConn) AddOnTrackSub(trackName string, onTrackCB OnTrackCB) { + sc.onTrackCBByTrackNameMu.Lock() + defer sc.onTrackCBByTrackNameMu.Unlock() + sc.onTrackCBByTrackName[trackName] = onTrackCB } -// RemoveOnTrackSub removes an OnTrack subscription for the resource. -func (sc *SharedConn) RemoveOnTrackSub(name resource.Name) { - sc.resOnTrackMu.Lock() - defer sc.resOnTrackMu.Unlock() - delete(sc.resOnTrackCBs, name) +// RemoveOnTrackSub removes an OnTrack subscription for the track. +func (sc *SharedConn) RemoveOnTrackSub(trackName string) { + sc.onTrackCBByTrackNameMu.Lock() + defer sc.onTrackCBByTrackNameMu.Unlock() + delete(sc.onTrackCBByTrackName, trackName) } // GrpcConn returns a gRPC capable client connection. @@ -159,9 +158,11 @@ func (sc *SharedConn) ResetConn(conn rpc.ClientConn, moduleLogger logging.Logger sc.logger = moduleLogger.Sublogger("networking.conn") } - if sc.resOnTrackCBs == nil { + // It is safe to access this without a mutex as it is only ever nil once at the beginning of the + // SharedConn's lifetime + if sc.onTrackCBByTrackName == nil { // Same initilization argument as above with the logger. - sc.resOnTrackCBs = make(map[resource.Name]OnTrackCB) + sc.onTrackCBByTrackName = make(map[string]OnTrackCB) } sc.peerConnMu.Lock() @@ -199,16 +200,12 @@ func (sc *SharedConn) ResetConn(conn rpc.ClientConn, moduleLogger logging.Logger } sc.peerConn.OnTrack(func(trackRemote *webrtc.TrackRemote, rtpReceiver *webrtc.RTPReceiver) { - name, err := resource.NewFromString(trackRemote.StreamID()) - if err != nil { - sc.logger.Errorw("StreamID did not parse as a ResourceName", "sharedConn", fmt.Sprintf("%p", sc), "streamID", trackRemote.StreamID()) - return - } - sc.resOnTrackMu.Lock() - onTrackCB, ok := sc.resOnTrackCBs[name] - sc.resOnTrackMu.Unlock() + sc.onTrackCBByTrackNameMu.Lock() + onTrackCB, ok := sc.onTrackCBByTrackName[trackRemote.StreamID()] + sc.onTrackCBByTrackNameMu.Unlock() if !ok { - sc.logger.Errorw("Callback not found for StreamID", "sharedConn", fmt.Sprintf("%p", sc), "streamID", trackRemote.StreamID()) + msg := "Callback not found for StreamID: %s, keys(resOnTrackCBs): %#v" + sc.logger.Errorf(msg, trackRemote.StreamID(), maps.Keys(sc.onTrackCBByTrackName)) return } onTrackCB(trackRemote, rtpReceiver) @@ -340,6 +337,14 @@ func NewLocalPeerConnection(logger logging.Logger) (*webrtc.PeerConnection, erro return false }) + // RSDK-8547: WebRTC video streams expect an "increasing" SRTP value. When we forward RTP + // packets through different hops, those values are copied as-is. If one connection in this + // chain of hops has a blip, it will reset its SRTP value. Other hops in the chain that were not + // disconnected will see these unexpected change in SRTP values and interpret it as a replay + // attack, dropping the data. We disable this "protection" as per the justification in the noted + // ticket. + settingEngine.DisableSRTPReplayProtection(true) + options := []func(a *webrtc.API){webrtc.WithMediaEngine(&m), webrtc.WithInterceptorRegistry(&i)} if utils.Debug { settingEngine.LoggerFactory = WebRTCLoggerFactory{logger} diff --git a/grpc/tracker.go b/grpc/tracker.go new file mode 100644 index 00000000000..6a2e1943d26 --- /dev/null +++ b/grpc/tracker.go @@ -0,0 +1,9 @@ +package grpc + +// Tracker allows callback functions to a WebRTC peer connection's OnTrack callback +// function by track name. +// Both grpc.SharedConn and grpc.ReconfigurableClientConn implement tracker. +type Tracker interface { + AddOnTrackSub(trackName string, onTrackCB OnTrackCB) + RemoveOnTrackSub(trackName string) +} diff --git a/grpc/tracker_test.go b/grpc/tracker_test.go new file mode 100644 index 00000000000..0c677ad3086 --- /dev/null +++ b/grpc/tracker_test.go @@ -0,0 +1,20 @@ +package grpc + +import ( + "reflect" + "testing" + + "go.viam.com/test" +) + +func TestTrackerImplementations(t *testing.T) { + tracker := reflect.TypeOf((*Tracker)(nil)).Elem() + + t.Run("*ReconfigurableClientConn should implement Tracker", func(t *testing.T) { + test.That(t, reflect.TypeOf(&ReconfigurableClientConn{}).Implements(tracker), test.ShouldBeTrue) + }) + + t.Run("*SharedConn should implement Tracker", func(t *testing.T) { + test.That(t, reflect.TypeOf(&SharedConn{}).Implements(tracker), test.ShouldBeTrue) + }) +} diff --git a/operation/web.go b/operation/web.go index d290312055b..aa63d1c0f7f 100644 --- a/operation/web.go +++ b/operation/web.go @@ -56,7 +56,16 @@ func (m *Manager) UnaryServerInterceptor( ctx, done := m.CreateFromIncomingContext(ctx, info.FullMethod) defer done() if op := Get(ctx); op != nil && op.ID.String() != "" { - utils.UncheckedError(grpc.SetHeader(ctx, metadata.MD{opidMetadataKey: []string{op.ID.String()}})) + // SetHeader will occasionally error because of a data race if the request has been cancelled from client side. + // The cancel signal (RST_STREAM) is processed on a separate goroutine and will close the existing gRPC stream, + // which will end up writing headers and returning a message to the server. If headers were sent before SetHeader + // is called here, SetHeader will error. + // Since the behavior is expected and part of the gRPC stream closing, only log the error if it is unexpected + // (the context error is nil, meaning request wasn't cancelled). + if err := grpc.SetHeader(ctx, metadata.MD{opidMetadataKey: []string{op.ID.String()}}); err != nil && + ctx.Err() == nil { + m.logger.CDebugw(ctx, "error while setting header", "err", err) + } } return handler(ctx, req) } diff --git a/resource/resource.go b/resource/resource.go index a910653b85b..9a7a9b4b267 100644 --- a/resource/resource.go +++ b/resource/resource.go @@ -150,11 +150,14 @@ func ContainsReservedCharacter(val string) error { // A Sensor represents a general purpose sensor that can give arbitrary readings // of all readings that it is sensing. +// For more information, see the [sensor component docs]. // // Readings example: // // // Get the readings provided by the sensor. // readings, err := mySensor.Readings(context.Background(), nil) +// +// [sensor component docs]: https://docs.viam.com/components/sensor/ type Sensor interface { // Readings return data specific to the type of sensor and can be of any type. Readings(ctx context.Context, extra map[string]interface{}) (map[string]interface{}, error) diff --git a/robot/client/client.go b/robot/client/client.go index 1bd33c7e728..f7d0920dbdc 100644 --- a/robot/client/client.go +++ b/robot/client/client.go @@ -393,6 +393,7 @@ func (rc *RobotClient) updateResourceClients(ctx context.Context) error { for resourceName, client := range rc.resourceClients { // check if no longer an active resource if !activeResources[resourceName] { + rc.logger.Infow("Removing resource from remote client", "resourceName", resourceName) if err := client.Close(ctx); err != nil { rc.Logger().CError(ctx, err) continue @@ -574,7 +575,8 @@ func (rc *RobotClient) createClient(name resource.Name) (resource.Resource, erro if !ok || apiInfo.RPCClient == nil { return grpc.NewForeignResource(name, &rc.conn), nil } - return apiInfo.RPCClient(rc.backgroundCtx, &rc.conn, rc.remoteName, name, rc.Logger()) + logger := rc.Logger().Sublogger(resource.RemoveRemoteName(name).ShortName()) + return apiInfo.RPCClient(rc.backgroundCtx, &rc.conn, rc.remoteName, name, logger) } func (rc *RobotClient) resources(ctx context.Context) ([]resource.Name, []resource.RPCAPI, error) { @@ -657,7 +659,7 @@ func (rc *RobotClient) updateResources(ctx context.Context) error { names, rpcAPIs, err := rc.resources(ctx) if err != nil && status.Code(err) != codes.Unimplemented { - return err + return fmt.Errorf("error updating resources: %w", err) } rc.resourceNames = make([]resource.Name, 0, len(names)) diff --git a/robot/impl/local_robot.go b/robot/impl/local_robot.go index cbc1408006c..6530b824e32 100644 --- a/robot/impl/local_robot.go +++ b/robot/impl/local_robot.go @@ -1190,14 +1190,26 @@ func (r *localRobot) reconfigure(ctx context.Context, newConfig *config.Config, // if anything has changed. err := r.packageManager.Sync(ctx, newConfig.Packages, newConfig.Modules) if err != nil { - allErrs = multierr.Combine(allErrs, err) + r.Logger().CErrorw(ctx, "reconfiguration aborted because cloud modules or packages download failed", "error", err) + return } // For local tarball modules, we create synthetic versions for package management. The `localRobot` keeps track of these because // config reader would overwrite if we just stored it in config. Here, we copy the synthetic version from the `localRobot` into the // appropriate `config.Module` object inside the `cfg.Modules` slice. Thus, when a local tarball module is reloaded, the viam-server // can unpack it into a fresh directory rather than reusing the previous one. r.applyLocalModuleVersions(newConfig) - allErrs = multierr.Combine(allErrs, r.localPackages.Sync(ctx, newConfig.Packages, newConfig.Modules)) + err = r.localPackages.Sync(ctx, newConfig.Packages, newConfig.Modules) + if err != nil { + r.Logger().CErrorw(ctx, "reconfiguration aborted because local modules or packages sync failed", "error", err) + return + } + + if newConfig.Cloud != nil { + r.Logger().CDebug(ctx, "updating cached config") + if err := newConfig.StoreToCache(); err != nil { + r.logger.CErrorw(ctx, "error storing the config", "error", err) + } + } // Add default services and process their dependencies. Dependencies may // already come from config validation so we check that here. @@ -1410,9 +1422,7 @@ func (r *localRobot) Shutdown(ctx context.Context) error { func (r *localRobot) MachineStatus(ctx context.Context) (robot.MachineStatus, error) { var result robot.MachineStatus - r.manager.resourceGraphLock.Lock() result.Resources = append(result.Resources, r.manager.resources.Status()...) - r.manager.resourceGraphLock.Unlock() r.configRevisionMu.RLock() result.Config = r.configRevision diff --git a/robot/impl/local_robot_test.go b/robot/impl/local_robot_test.go index 29ab1c10a69..12b97fa907f 100644 --- a/robot/impl/local_robot_test.go +++ b/robot/impl/local_robot_test.go @@ -6,12 +6,13 @@ import ( "crypto/x509" "errors" "fmt" + "log" "math" - "net" "os" "path" "path/filepath" "strings" + "sync" "testing" "time" @@ -56,7 +57,6 @@ import ( "go.viam.com/rdk/referenceframe" "go.viam.com/rdk/resource" "go.viam.com/rdk/robot" - "go.viam.com/rdk/robot/client" "go.viam.com/rdk/robot/framesystem" "go.viam.com/rdk/robot/packages" putils "go.viam.com/rdk/robot/packages/testutils" @@ -1912,232 +1912,6 @@ func TestConfigMethod(t *testing.T) { test.That(t, actualCfg, test.ShouldResemble, &expectedCfg) } -func TestReconnectRemote(t *testing.T) { - logger := logging.NewTestLogger(t) - options, _, addr := robottestutils.CreateBaseOptionsAndListener(t) - // start the first robot - ctx := context.Background() - armConfig := resource.Config{ - Name: "arm1", - API: arm.API, - Model: fakeModel, - ConvertedAttributes: &fake.Config{ - ModelFilePath: "../../components/arm/fake/fake_model.json", - }, - } - cfg := config.Config{ - Components: []resource.Config{armConfig}, - } - - robot := setupLocalRobot(t, ctx, &cfg, logger) - err := robot.StartWeb(ctx, options) - test.That(t, err, test.ShouldBeNil) - - // start the second robot - ctx1 := context.Background() - options1, _, addr1 := robottestutils.CreateBaseOptionsAndListener(t) - - remoteConf := config.Remote{ - Name: "remote", - Insecure: true, - Address: addr, - } - - cfg1 := config.Config{ - Remotes: []config.Remote{remoteConf}, - } - - robot1 := setupLocalRobot(t, ctx, &cfg1, logger) - - err = robot1.StartWeb(ctx1, options1) - test.That(t, err, test.ShouldBeNil) - - robotClient := robottestutils.NewRobotClient(t, logger, addr1, time.Second) - defer func() { - test.That(t, robotClient.Close(context.Background()), test.ShouldBeNil) - }() - - a1, err := arm.FromRobot(robot1, "arm1") - test.That(t, err, test.ShouldBeNil) - test.That(t, a1, test.ShouldNotBeNil) - - remoteRobot, ok := robot1.RemoteByName("remote") - test.That(t, ok, test.ShouldBeTrue) - test.That(t, remoteRobot, test.ShouldNotBeNil) - remoteRobotClient, ok := remoteRobot.(*client.RobotClient) - test.That(t, ok, test.ShouldBeTrue) - test.That(t, remoteRobotClient, test.ShouldNotBeNil) - - a, err := robotClient.ResourceByName(arm.Named("remote:arm1")) - test.That(t, err, test.ShouldBeNil) - test.That(t, a, test.ShouldNotBeNil) - anArm, ok := a.(arm.Arm) - test.That(t, ok, test.ShouldBeTrue) - _, err = anArm.EndPosition(context.Background(), map[string]interface{}{}) - test.That(t, err, test.ShouldBeNil) - - // close/disconnect the robot - robot.StopWeb() - test.That(t, <-remoteRobotClient.Changed(), test.ShouldBeTrue) - test.That(t, len(remoteRobotClient.ResourceNames()), test.ShouldEqual, 0) - testutils.WaitForAssertion(t, func(tb testing.TB) { - tb.Helper() - test.That(tb, len(robotClient.ResourceNames()), test.ShouldEqual, 2) - }) - test.That(t, len(robot1.ResourceNames()), test.ShouldEqual, 2) - _, err = anArm.EndPosition(context.Background(), map[string]interface{}{}) - test.That(t, err, test.ShouldBeError) - - // reconnect the first robot - ctx2 := context.Background() - listener, err := net.Listen("tcp", addr) - test.That(t, err, test.ShouldBeNil) - - options.Network.Listener = listener - err = robot.StartWeb(ctx2, options) - test.That(t, err, test.ShouldBeNil) - - // check if the original arm can still be called - test.That(t, <-remoteRobotClient.Changed(), test.ShouldBeTrue) - test.That(t, remoteRobotClient.Connected(), test.ShouldBeTrue) - test.That(t, len(remoteRobotClient.ResourceNames()), test.ShouldEqual, 3) - testutils.WaitForAssertion(t, func(tb testing.TB) { - tb.Helper() - test.That(tb, len(robotClient.ResourceNames()), test.ShouldEqual, 5) - }) - test.That(t, len(robot1.ResourceNames()), test.ShouldEqual, 5) - _, err = remoteRobotClient.ResourceByName(arm.Named("arm1")) - test.That(t, err, test.ShouldBeNil) - - _, err = robotClient.ResourceByName(arm.Named("arm1")) - test.That(t, err, test.ShouldBeNil) - _, err = anArm.EndPosition(context.Background(), map[string]interface{}{}) - test.That(t, err, test.ShouldBeNil) -} - -func TestReconnectRemoteChangeConfig(t *testing.T) { - logger := logging.NewTestLogger(t) - - // start the first robot - ctx := context.Background() - options, _, addr := robottestutils.CreateBaseOptionsAndListener(t) - armConfig := resource.Config{ - Name: "arm1", - API: arm.API, - Model: fakeModel, - ConvertedAttributes: &fake.Config{ - ModelFilePath: "../../components/arm/fake/fake_model.json", - }, - } - cfg := config.Config{ - Components: []resource.Config{armConfig}, - } - - robot := setupLocalRobot(t, ctx, &cfg, logger) - err := robot.StartWeb(ctx, options) - test.That(t, err, test.ShouldBeNil) - - // start the second robot - ctx1 := context.Background() - options1, _, addr1 := robottestutils.CreateBaseOptionsAndListener(t) - remoteConf := config.Remote{ - Name: "remote", - Insecure: true, - Address: addr, - } - - cfg1 := config.Config{ - Remotes: []config.Remote{remoteConf}, - } - - robot1 := setupLocalRobot(t, ctx, &cfg1, logger) - - err = robot1.StartWeb(ctx1, options1) - test.That(t, err, test.ShouldBeNil) - - robotClient := robottestutils.NewRobotClient(t, logger, addr1, time.Second) - defer func() { - test.That(t, robotClient.Close(context.Background()), test.ShouldBeNil) - }() - - a1, err := arm.FromRobot(robot1, "arm1") - test.That(t, err, test.ShouldBeNil) - test.That(t, a1, test.ShouldNotBeNil) - - remoteRobot, ok := robot1.RemoteByName("remote") - test.That(t, ok, test.ShouldBeTrue) - test.That(t, remoteRobot, test.ShouldNotBeNil) - remoteRobotClient, ok := remoteRobot.(*client.RobotClient) - test.That(t, ok, test.ShouldBeTrue) - test.That(t, remoteRobotClient, test.ShouldNotBeNil) - - a, err := robotClient.ResourceByName(arm.Named("remote:arm1")) - test.That(t, err, test.ShouldBeNil) - test.That(t, a, test.ShouldNotBeNil) - anArm, ok := a.(arm.Arm) - test.That(t, ok, test.ShouldBeTrue) - _, err = anArm.EndPosition(context.Background(), map[string]interface{}{}) - test.That(t, err, test.ShouldBeNil) - - // close/disconnect the robot - test.That(t, robot.Close(context.Background()), test.ShouldBeNil) - test.That(t, <-remoteRobotClient.Changed(), test.ShouldBeTrue) - test.That(t, len(remoteRobotClient.ResourceNames()), test.ShouldEqual, 0) - testutils.WaitForAssertion(t, func(tb testing.TB) { - tb.Helper() - test.That(tb, len(robotClient.ResourceNames()), test.ShouldEqual, 2) - }) - test.That(t, len(robot1.ResourceNames()), test.ShouldEqual, 2) - _, err = anArm.EndPosition(context.Background(), map[string]interface{}{}) - test.That(t, err, test.ShouldBeError) - - // reconnect the first robot - ctx2 := context.Background() - listener, err := net.Listen("tcp", addr) - test.That(t, err, test.ShouldBeNil) - baseConfig := resource.Config{ - Name: "base1", - API: base.API, - Model: fakeModel, - } - cfg = config.Config{ - Components: []resource.Config{baseConfig}, - } - - options = weboptions.New() - options.Network.BindAddress = "" - options.Network.Listener = listener - robot = setupLocalRobot(t, ctx, &cfg, logger) - err = robot.StartWeb(ctx2, options) - test.That(t, err, test.ShouldBeNil) - - // check if the original arm can't be called anymore - test.That(t, <-remoteRobotClient.Changed(), test.ShouldBeTrue) - test.That(t, remoteRobotClient.Connected(), test.ShouldBeTrue) - test.That(t, len(remoteRobotClient.ResourceNames()), test.ShouldEqual, 3) - testutils.WaitForAssertion(t, func(tb testing.TB) { - tb.Helper() - test.That(tb, len(robotClient.ResourceNames()), test.ShouldEqual, 5) - }) - test.That(t, len(robot1.ResourceNames()), test.ShouldEqual, 5) - _, err = anArm.EndPosition(context.Background(), map[string]interface{}{}) - test.That(t, err, test.ShouldBeError) - - // check that base is now instantiated - _, err = remoteRobotClient.ResourceByName(base.Named("base1")) - test.That(t, err, test.ShouldBeNil) - - b, err := robotClient.ResourceByName(base.Named("remote:base1")) - test.That(t, err, test.ShouldBeNil) - aBase, ok := b.(base.Base) - test.That(t, ok, test.ShouldBeTrue) - - err = aBase.Stop(ctx, map[string]interface{}{}) - test.That(t, err, test.ShouldBeNil) - - test.That(t, len(robotClient.ResourceNames()), test.ShouldEqual, 5) -} - func TestCheckMaxInstanceValid(t *testing.T) { logger := logging.NewTestLogger(t) cfg := &config.Config{ @@ -3557,8 +3331,23 @@ type mockResource struct { } type mockConfig struct { - Value int `json:"value"` - Fail bool `json:"fail"` + Value int `json:"value"` + Fail bool `json:"fail"` + Sleep string `json:"sleep"` +} + +//nolint:unparam // the resource name is currently always "m" but this could easily change +func newMockConfig(name string, val int, fail bool, sleep string) resource.Config { + return resource.Config{ + Name: name, + Model: mockModel, + API: mockAPI, + // We need to specify both `Attributes` and `ConvertedAttributes`. + // The former triggers a reconfiguration and the former is actually + // used to reconfigure the component. + Attributes: rutils.AttributeMap{"value": val, "fail": fail, "sleep": sleep}, + ConvertedAttributes: &mockConfig{Value: val, Fail: fail, Sleep: sleep}, + } } var errMockValidation = errors.New("whoops") @@ -3596,10 +3385,67 @@ func (m *mockResource) Reconfigure( if err != nil { return err } + if mConf.Sleep != "" { + if d, err := time.ParseDuration(mConf.Sleep); err == nil { + log.Printf("sleeping for %s\n", d) + time.Sleep(d) + } + } m.value = mConf.Value return nil } +// getExpectedDefaultStatuses returns a slice of default [resource.Status] with a given +// revision set for motion and sensor services. +func getExpectedDefaultStatuses(revision string) []resource.Status { + return []resource.Status{ + { + Name: resource.Name{ + API: resource.APINamespaceRDKInternal.WithServiceType("framesystem"), + Name: "builtin", + }, + State: resource.NodeStateReady, + }, + { + Name: resource.Name{ + API: resource.APINamespaceRDKInternal.WithServiceType("cloud_connection"), + Name: "builtin", + }, + State: resource.NodeStateReady, + }, + { + Name: resource.Name{ + API: resource.APINamespaceRDKInternal.WithServiceType("packagemanager"), + Name: "builtin", + }, + State: resource.NodeStateReady, + }, + { + Name: resource.Name{ + API: resource.APINamespaceRDKInternal.WithServiceType("web"), + Name: "builtin", + }, + State: resource.NodeStateReady, + }, + { + Name: resource.Name{ + API: resource.APINamespaceRDK.WithServiceType("motion"), + Name: "builtin", + }, + State: resource.NodeStateReady, + Revision: revision, + }, + { + Name: resource.Name{ + API: resource.APINamespaceRDK.WithServiceType("sensors"), + Name: "builtin", + }, + State: resource.NodeStateReady, + Revision: revision, + }, + } +} + func TestMachineStatus(t *testing.T) { logger := logging.NewTestLogger(t) ctx := context.Background() @@ -3611,93 +3457,32 @@ func TestMachineStatus(t *testing.T) { ) defer resource.Deregister(mockAPI, mockModel) - rev1 := "rev1" - builtinRev := rev1 - - getExpectedDefaultStatuses := func() []resource.Status { - return []resource.Status{ - { - Name: resource.Name{ - API: resource.APINamespaceRDKInternal.WithServiceType("framesystem"), - Name: "builtin", - }, - State: resource.NodeStateReady, - }, - { - Name: resource.Name{ - API: resource.APINamespaceRDKInternal.WithServiceType("cloud_connection"), - Name: "builtin", - }, - State: resource.NodeStateReady, - }, - { - Name: resource.Name{ - API: resource.APINamespaceRDKInternal.WithServiceType("packagemanager"), - Name: "builtin", - }, - State: resource.NodeStateReady, - }, - { - Name: resource.Name{ - API: resource.APINamespaceRDKInternal.WithServiceType("web"), - Name: "builtin", - }, - State: resource.NodeStateReady, - }, - { - Name: resource.Name{ - API: resource.APINamespaceRDK.WithServiceType("motion"), - Name: "builtin", - }, - State: resource.NodeStateReady, - Revision: builtinRev, - }, - { - Name: resource.Name{ - API: resource.APINamespaceRDK.WithServiceType("sensors"), - Name: "builtin", - }, - State: resource.NodeStateReady, - Revision: builtinRev, - }, - } - } - t.Run("default resources", func(t *testing.T) { + rev1 := "rev1" lr := setupLocalRobot(t, ctx, &config.Config{Revision: rev1}, logger) mStatus, err := lr.MachineStatus(ctx) test.That(t, err, test.ShouldBeNil) test.That(t, mStatus.Config.Revision, test.ShouldEqual, rev1) - expectedStatuses := getExpectedDefaultStatuses() + expectedStatuses := getExpectedDefaultStatuses(rev1) rtestutils.VerifySameResourceStatuses(t, mStatus.Resources, expectedStatuses) }) - t.Run("reconfigure", func(t *testing.T) { - lr := setupLocalRobot(t, ctx, &config.Config{Revision: rev1}, logger) - - expectedConfigError := fmt.Errorf("resource config validation error: %w", errMockValidation) + t.Run("poll after working and failing reconfigures", func(t *testing.T) { + lr := setupLocalRobot(t, ctx, &config.Config{Revision: "rev1"}, logger) // Add a fake resource to the robot. rev2 := "rev2" - builtinRev = rev2 lr.Reconfigure(ctx, &config.Config{ - Revision: rev2, - Components: []resource.Config{ - { - Name: "m", - Model: mockModel, - API: mockAPI, - ConvertedAttributes: &mockConfig{}, - }, - }, + Revision: rev2, + Components: []resource.Config{newMockConfig("m", 0, false, "")}, }) mStatus, err := lr.MachineStatus(ctx) test.That(t, err, test.ShouldBeNil) test.That(t, mStatus.Config.Revision, test.ShouldEqual, rev2) expectedStatuses := rtestutils.ConcatResourceStatuses( - getExpectedDefaultStatuses(), + getExpectedDefaultStatuses(rev2), []resource.Status{ { Name: mockNamed("m"), @@ -3710,27 +3495,17 @@ func TestMachineStatus(t *testing.T) { // Update resource config to cause reconfiguration to fail. rev3 := "rev3" - builtinRev = rev3 lr.Reconfigure(ctx, &config.Config{ - Revision: rev3, - Components: []resource.Config{ - { - Name: "m", - Model: mockModel, - API: mockAPI, - // We need to specify both `Attributes` and `ConvertedAttributes`. - // The former triggers a reconfiguration and the former is actually - // used to reconfigure the component. - Attributes: rutils.AttributeMap{"fail": true}, - ConvertedAttributes: &mockConfig{Fail: true}, - }, - }, + Revision: rev3, + Components: []resource.Config{newMockConfig("m", 0, true, "")}, }) mStatus, err = lr.MachineStatus(ctx) test.That(t, err, test.ShouldBeNil) test.That(t, mStatus.Config.Revision, test.ShouldEqual, rev3) + + expectedConfigError := fmt.Errorf("resource config validation error: %w", errMockValidation) expectedStatuses = rtestutils.ConcatResourceStatuses( - getExpectedDefaultStatuses(), + getExpectedDefaultStatuses(rev3), []resource.Status{ { Name: mockNamed("m"), @@ -3744,27 +3519,15 @@ func TestMachineStatus(t *testing.T) { // Update resource with a working config. rev4 := "rev4" - builtinRev = rev4 lr.Reconfigure(ctx, &config.Config{ - Revision: rev4, - Components: []resource.Config{ - { - Name: "m", - Model: mockModel, - API: mockAPI, - // We need to specify both `Attributes` and `ConvertedAttributes`. - // The former triggers a reconfiguration and the former is actually - // used to reconfigure the component. - Attributes: rutils.AttributeMap{"value": 200}, - ConvertedAttributes: &mockConfig{Value: 200}, - }, - }, + Revision: rev4, + Components: []resource.Config{newMockConfig("m", 200, false, "")}, }) mStatus, err = lr.MachineStatus(ctx) test.That(t, err, test.ShouldBeNil) test.That(t, mStatus.Config.Revision, test.ShouldEqual, rev4) expectedStatuses = rtestutils.ConcatResourceStatuses( - getExpectedDefaultStatuses(), + getExpectedDefaultStatuses(rev4), []resource.Status{ { Name: mockNamed("m"), @@ -3775,4 +3538,72 @@ func TestMachineStatus(t *testing.T) { ) rtestutils.VerifySameResourceStatuses(t, mStatus.Resources, expectedStatuses) }) + + t.Run("poll during reconfiguration", func(t *testing.T) { + rev1 := "rev1" + lr := setupLocalRobot(t, ctx, &config.Config{ + Revision: rev1, + Components: []resource.Config{newMockConfig("m", 200, false, "")}, + }, logger) + + // update resource with a working config that is slow to reconfigure. + rev2 := "rev2" + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + lr.Reconfigure(ctx, &config.Config{ + Revision: rev2, + Components: []resource.Config{newMockConfig("m", 300, false, "1s")}, + }) + }() + // sleep for a short amount of time to allow the machine to receive a new + // revision. this sleep should be shorter than the resource update duration + // defined above so that updated resource is still in a "configuring" state. + time.Sleep(time.Millisecond * 100) + + // get status while reconfiguring + mStatus, err := lr.MachineStatus(ctx) + test.That(t, err, test.ShouldBeNil) + test.That(t, mStatus.Config.Revision, test.ShouldEqual, rev2) + + // the component whose config changed should be the only component in a + // "configuring" state and associated with the original revision. + filterConfiguring := rtestutils.FilterByStatus(t, mStatus.Resources, resource.NodeStateConfiguring) + expectedConfiguring := []resource.Status{ + { + Name: mockNamed("m"), + State: resource.NodeStateConfiguring, + Revision: rev1, + }, + } + rtestutils.VerifySameResourceStatuses(t, filterConfiguring, expectedConfiguring) + + // all other components should be in the "ready" state and associated with the + // new revision. + filterReady := rtestutils.FilterByStatus(t, mStatus.Resources, resource.NodeStateReady) + expectedReady := getExpectedDefaultStatuses(rev2) + rtestutils.VerifySameResourceStatuses(t, filterReady, expectedReady) + + wg.Wait() + + // get status after reconfigure finishes + mStatus, err = lr.MachineStatus(ctx) + test.That(t, err, test.ShouldBeNil) + test.That(t, mStatus.Config.Revision, test.ShouldEqual, rev2) + + // now all components, including the one whose config changed, should all be in + // the "ready" state and associated with the new revision. + expectedStatuses := rtestutils.ConcatResourceStatuses( + getExpectedDefaultStatuses(rev2), + []resource.Status{ + { + Name: mockNamed("m"), + State: resource.NodeStateReady, + Revision: rev2, + }, + }, + ) + rtestutils.VerifySameResourceStatuses(t, mStatus.Resources, expectedStatuses) + }) } diff --git a/robot/impl/resource_manager.go b/robot/impl/resource_manager.go index 2f0554ab75d..ffafd47bb3d 100644 --- a/robot/impl/resource_manager.go +++ b/robot/impl/resource_manager.go @@ -3,6 +3,7 @@ package robotimpl import ( "context" "crypto/tls" + "errors" "fmt" "os" "reflect" @@ -11,7 +12,6 @@ import ( "time" "github.com/jhump/protoreflect/desc" - "github.com/pkg/errors" "go.uber.org/multierr" goutils "go.viam.com/utils" "go.viam.com/utils/pexec" @@ -182,7 +182,7 @@ func (manager *resourceManager) updateRemoteResourceNames( rr internalRemoteRobot, recreateAllClients bool, ) bool { - manager.logger.CDebugw(ctx, "updating remote resource names", "remote", remoteName) + manager.logger.CDebugw(ctx, "updating remote resource names", "remote", remoteName, "recreateAllClients", recreateAllClients) activeResourceNames := map[resource.Name]bool{} newResources := rr.ResourceNames() oldResources := manager.remoteResourceNames(remoteName) @@ -207,13 +207,11 @@ func (manager *resourceManager) updateRemoteResourceNames( } continue } - resName = resName.PrependRemote(remoteName.Name) - gNode, ok := manager.resources.Node(resName) - + gNode, nodeAlreadyExists := manager.resources.Node(resName) if _, alreadyCurrent := activeResourceNames[resName]; alreadyCurrent { activeResourceNames[resName] = true - if ok && !gNode.IsUninitialized() { + if nodeAlreadyExists && !gNode.IsUninitialized() { // resources that enter this block represent those with names that already exist in the resource graph. // it is possible that we are switching to a new remote with a identical resource name(s), so we may // need to create these resource clients. @@ -239,7 +237,7 @@ func (manager *resourceManager) updateRemoteResourceNames( } } - if ok { + if nodeAlreadyExists { gNode.SwapResource(res, unknownModel) } else { gNode = resource.NewConfiguredGraphNode(resource.Config{}, res, unknownModel) @@ -471,7 +469,7 @@ func (manager *resourceManager) closeResource(ctx context.Context, res resource. resName := res.Name() if manager.moduleManager != nil && manager.moduleManager.IsModularResource(resName) { if err := manager.moduleManager.RemoveResource(closeCtx, resName); err != nil { - allErrs = multierr.Combine(allErrs, errors.Wrap(err, "error removing modular resource for closure")) + allErrs = multierr.Combine(allErrs, fmt.Errorf("error removing modular resource for closure: %w", err)) } } @@ -526,7 +524,7 @@ func (manager *resourceManager) Close(ctx context.Context) error { var allErrs error if err := manager.processManager.Stop(); err != nil { - allErrs = multierr.Combine(allErrs, errors.Wrap(err, "error stopping process manager")) + allErrs = multierr.Combine(allErrs, fmt.Errorf("error stopping process manager: %w", err)) } // our caller will close web @@ -540,7 +538,7 @@ func (manager *resourceManager) Close(ctx context.Context) error { // moduleManager may be nil in tests, and must be closed last, after resources within have been closed properly above if manager.moduleManager != nil { if err := manager.moduleManager.Close(ctx); err != nil { - allErrs = multierr.Combine(allErrs, errors.Wrap(err, "error closing module manager")) + allErrs = multierr.Combine(allErrs, fmt.Errorf("error closing module manager: %w", err)) } } @@ -625,8 +623,7 @@ func (manager *resourceManager) completeConfig( if gNode.IsUninitialized() { verb = "configuring" gNode.InitializeLogger( - manager.logger, resName.String(), conf.LogConfiguration.Level, - ) + manager.logger, resName.String(), conf.LogConfiguration.Level) } else { verb = "reconfiguring" } @@ -756,8 +753,7 @@ func (manager *resourceManager) completeConfigForRemotes(ctx context.Context, lr } if gNode.IsUninitialized() { gNode.InitializeLogger( - manager.logger, fromRemoteNameToRemoteNodeName(remConf.Name).String(), manager.logger.GetLevel(), - ) + manager.logger, fromRemoteNameToRemoteNodeName(remConf.Name).String(), logging.INFO) } // this is done in config validation but partial start rules require us to check again if _, err := remConf.Validate(""); err != nil { @@ -877,7 +873,7 @@ func (manager *resourceManager) processRemote( err = errors.New("must use Config.AllowInsecureCreds to connect to a non-TLS secured robot") } } - return nil, errors.Errorf("couldn't connect to robot remote (%s): %s", config.Address, err) + return nil, fmt.Errorf("couldn't connect to robot remote (%s): %w", config.Address, err) } manager.logger.CInfow(ctx, "Connected now to remote", "remote", config.Name) return robotClient, nil @@ -1016,7 +1012,7 @@ func (manager *resourceManager) markResourceForUpdate(name resource.Name, conf r gNode = resource.NewUnconfiguredGraphNode(conf, deps) gNode.UpdatePendingRevision(revision) if err := manager.resources.AddNode(name, gNode); err != nil { - return errors.Errorf("failed to add new node for unconfigured resource %q: %v", name, err) + return fmt.Errorf("failed to add new node for unconfigured resource %q: %w", name, err) } return nil } diff --git a/robot/session_manager.go b/robot/session_manager.go index d70214e4f24..1e00cbb01f0 100644 --- a/robot/session_manager.go +++ b/robot/session_manager.go @@ -12,7 +12,6 @@ import ( "go.viam.com/rdk/logging" "go.viam.com/rdk/resource" "go.viam.com/rdk/session" - rdkutils "go.viam.com/rdk/utils" ) // NewSessionManager creates a new manager for holding sessions. @@ -24,7 +23,7 @@ func NewSessionManager(robot Robot, heartbeatWindow time.Duration) *SessionManag sessions: map[uuid.UUID]*session.Session{}, resourceToSession: map[resource.Name]uuid.UUID{}, } - m.workers = rdkutils.NewStoppableWorkers(m.expireLoop) + m.workers = utils.NewBackgroundStoppableWorkers(m.expireLoop) return m } @@ -40,7 +39,7 @@ type SessionManager struct { resourceToSession map[resource.Name]uuid.UUID - workers rdkutils.StoppableWorkers + workers *utils.StoppableWorkers } // All returns all active sessions. diff --git a/robot/web/stream/camera/camera.go b/robot/web/stream/camera/camera.go new file mode 100644 index 00000000000..948bfc112dc --- /dev/null +++ b/robot/web/stream/camera/camera.go @@ -0,0 +1,21 @@ +// Package camera provides functions for looking up a camera from a robot using a stream +package camera + +import ( + "go.viam.com/rdk/components/camera" + "go.viam.com/rdk/gostream" + "go.viam.com/rdk/resource" + "go.viam.com/rdk/robot" +) + +// Camera returns the camera from the robot (derived from the stream) or +// an error if it has no camera. +func Camera(robot robot.Robot, stream gostream.Stream) (camera.Camera, error) { + // Stream names are slightly modified versions of the resource short name + shortName := resource.SDPTrackNameToShortName(stream.Name()) + cam, err := camera.FromRobot(robot, shortName) + if err != nil { + return nil, err + } + return cam, nil +} diff --git a/robot/web/stream/server.go b/robot/web/stream/server.go index 3a69d2c0177..0baa47092e2 100644 --- a/robot/web/stream/server.go +++ b/robot/web/stream/server.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "sync" + "time" "github.com/pkg/errors" "github.com/viamrobotics/webrtc/v3" @@ -13,13 +14,18 @@ import ( "go.viam.com/utils" "go.viam.com/utils/rpc" + "go.viam.com/rdk/components/camera" "go.viam.com/rdk/gostream" "go.viam.com/rdk/logging" + "go.viam.com/rdk/resource" "go.viam.com/rdk/robot" + streamCamera "go.viam.com/rdk/robot/web/stream/camera" "go.viam.com/rdk/robot/web/stream/state" rutils "go.viam.com/rdk/utils" ) +var monitorCameraInterval = time.Second + type peerState struct { streamState *state.StreamState senders []*webrtc.RTPSender @@ -28,11 +34,12 @@ type peerState struct { // Server implements the gRPC audio/video streaming service. type Server struct { streampb.UnimplementedStreamServiceServer - logger logging.Logger - r robot.Robot + logger logging.Logger + robot robot.Robot + closedCtx context.Context + closedFn context.CancelFunc mu sync.RWMutex - streamNames []string nameToStreamState map[string]*state.StreamState activePeerStreams map[*webrtc.PeerConnection]map[string]*peerState activeBackgroundWorkers sync.WaitGroup @@ -43,11 +50,14 @@ type Server struct { // stream. func NewServer( streams []gostream.Stream, - r robot.Robot, + robot robot.Robot, logger logging.Logger, ) (*Server, error) { - ss := &Server{ - r: r, + closedCtx, closedFn := context.WithCancel(context.Background()) + server := &Server{ + closedCtx: closedCtx, + closedFn: closedFn, + robot: robot, logger: logger, nameToStreamState: map[string]*state.StreamState{}, activePeerStreams: map[*webrtc.PeerConnection]map[string]*peerState{}, @@ -55,11 +65,13 @@ func NewServer( } for _, stream := range streams { - if err := ss.add(stream); err != nil { + if err := server.add(stream); err != nil { return nil, err } } - return ss, nil + server.startMonitorCameraAvailable() + + return server, nil } // StreamAlreadyRegisteredError indicates that a stream has a name that is already registered on @@ -73,20 +85,20 @@ func (e *StreamAlreadyRegisteredError) Error() string { } // NewStream informs the stream server of new streams that are capable of being streamed. -func (ss *Server) NewStream(config gostream.StreamConfig) (gostream.Stream, error) { - ss.mu.Lock() - defer ss.mu.Unlock() +func (server *Server) NewStream(config gostream.StreamConfig) (gostream.Stream, error) { + server.mu.Lock() + defer server.mu.Unlock() - if _, ok := ss.nameToStreamState[config.Name]; ok { + if _, ok := server.nameToStreamState[config.Name]; ok { return nil, &StreamAlreadyRegisteredError{config.Name} } - stream, err := gostream.NewStream(config, ss.logger) + stream, err := gostream.NewStream(config, server.logger) if err != nil { return nil, err } - if err = ss.add(stream); err != nil { + if err = server.add(stream); err != nil { return nil, err } @@ -94,49 +106,62 @@ func (ss *Server) NewStream(config gostream.StreamConfig) (gostream.Stream, erro } // ListStreams implements part of the StreamServiceServer. -func (ss *Server) ListStreams(ctx context.Context, req *streampb.ListStreamsRequest) (*streampb.ListStreamsResponse, error) { +func (server *Server) ListStreams(ctx context.Context, req *streampb.ListStreamsRequest) (*streampb.ListStreamsResponse, error) { _, span := trace.StartSpan(ctx, "stream::server::ListStreams") defer span.End() - ss.mu.RLock() - defer ss.mu.RUnlock() + server.mu.RLock() + defer server.mu.RUnlock() - names := make([]string, 0, len(ss.streamNames)) - for _, name := range ss.streamNames { - streamState := ss.nameToStreamState[name] - names = append(names, streamState.Stream.Name()) + names := make([]string, 0, len(server.nameToStreamState)) + for name := range server.nameToStreamState { + names = append(names, name) } return &streampb.ListStreamsResponse{Names: names}, nil } // AddStream implements part of the StreamServiceServer. -func (ss *Server) AddStream(ctx context.Context, req *streampb.AddStreamRequest) (*streampb.AddStreamResponse, error) { +func (server *Server) AddStream(ctx context.Context, req *streampb.AddStreamRequest) (*streampb.AddStreamResponse, error) { ctx, span := trace.StartSpan(ctx, "stream::server::AddStream") defer span.End() - // Get the peer connection + // Get the peer connection to the caller. pc, ok := rpc.ContextPeerConnection(ctx) + server.logger.Infow("Adding video stream", "name", req.Name, "peerConn", pc) + defer server.logger.Warnf("AddStream END %s", req.Name) + if !ok { return nil, errors.New("can only add a stream over a WebRTC based connection") } - ss.mu.Lock() - defer ss.mu.Unlock() + server.mu.Lock() + defer server.mu.Unlock() - streamStateToAdd, ok := ss.nameToStreamState[req.Name] + streamStateToAdd, ok := server.nameToStreamState[req.Name] // return error if there is no stream for that camera if !ok { - err := fmt.Errorf("no stream for %q", req.Name) - ss.logger.Error(err.Error()) + var availableStreams string + for n := range server.nameToStreamState { + if availableStreams != "" { + availableStreams += ", " + } + availableStreams += fmt.Sprintf("%q", n) + } + err := fmt.Errorf("no stream for %q, available streams: %s", req.Name, availableStreams) + server.logger.Error(err.Error()) + return nil, err + } + // return error if camera is not in resource graph + if _, err := streamCamera.Camera(server.robot, streamStateToAdd.Stream); err != nil { return nil, err } // return error if the caller's peer connection is already being sent video data - if _, ok := ss.activePeerStreams[pc][req.Name]; ok { + if _, ok := server.activePeerStreams[pc][req.Name]; ok { err := errors.New("stream already active") - ss.logger.Error(err.Error()) + server.logger.Error(err.Error()) return nil, err } - nameToPeerState, ok := ss.activePeerStreams[pc] + nameToPeerState, ok := server.activePeerStreams[pc] // if there is no active video data being sent, set up a callback to remove the peer connection from // the active streams & stop the stream from doing h264 encode if this is the last peer connection // subcribed to the camera's video feed @@ -145,16 +170,16 @@ func (ss *Server) AddStream(ctx context.Context, req *streampb.AddStreamRequest) if !ok { nameToPeerState = map[string]*peerState{} pc.OnConnectionStateChange(func(peerConnectionState webrtc.PeerConnectionState) { - ss.logger.Debugf("%s pc.OnConnectionStateChange state: %s", req.Name, peerConnectionState) + server.logger.Infof("%s pc.OnConnectionStateChange state: %s", req.Name, peerConnectionState) switch peerConnectionState { case webrtc.PeerConnectionStateDisconnected, webrtc.PeerConnectionStateFailed, webrtc.PeerConnectionStateClosed: - ss.mu.Lock() - defer ss.mu.Unlock() + server.mu.Lock() + defer server.mu.Unlock() - if ss.isAlive { + if server.isAlive { // Dan: This conditional closing on `isAlive` is a hack to avoid a data // race. Shutting down a robot causes the PeerConnection to be closed // concurrently with this `stream.Server`. Thus, `stream.Server.Close` waiting @@ -162,25 +187,22 @@ func (ss *Server) AddStream(ctx context.Context, req *streampb.AddStreamRequest) // "worker". Given `Close` is expected to `Stop` remaining streams, we can elide // spinning off the below goroutine. // - // Given this is an existing race, I'm choosing to add to the tech - // debt rather than architect how shutdown should holistically work. Revert this - // change and run `TestRobotPeerConnect` (double check the test name at PR time) - // to reproduce the race. - ss.activeBackgroundWorkers.Add(1) + // Given this is an existing race, I'm choosing to add to the tech debt rather + // than architect how shutdown should holistically work. Revert this change and + // run `TestAudioTrackIsNotCreatedForVideoStream` to reproduce the race. + server.activeBackgroundWorkers.Add(1) utils.PanicCapturingGo(func() { - defer ss.activeBackgroundWorkers.Done() - ss.mu.Lock() - defer ss.mu.Unlock() - defer delete(ss.activePeerStreams, pc) + defer server.activeBackgroundWorkers.Done() + server.mu.Lock() + defer server.mu.Unlock() + defer delete(server.activePeerStreams, pc) var errs error - for _, ps := range ss.activePeerStreams[pc] { - ctx, cancel := context.WithTimeout(context.Background(), state.UnsubscribeTimeout) - errs = multierr.Combine(errs, ps.streamState.Decrement(ctx)) - cancel() + for _, ps := range server.activePeerStreams[pc] { + errs = multierr.Combine(errs, ps.streamState.Decrement()) } // We don't want to log this if the streamState was closed (as it only happens if viam-server is terminating) if errs != nil && !errors.Is(errs, state.ErrClosed) { - ss.logger.Errorw("error(s) stopping the streamState", "errs", errs) + server.logger.Errorw("error(s) stopping the streamState", "errs", errs) } }) } @@ -192,7 +214,7 @@ func (ss *Server) AddStream(ctx context.Context, req *streampb.AddStreamRequest) return } }) - ss.activePeerStreams[pc] = nameToPeerState + server.activePeerStreams[pc] = nameToPeerState } ps, ok := nameToPeerState[req.Name] @@ -221,19 +243,19 @@ func (ss *Server) AddStream(ctx context.Context, req *streampb.AddStreamRequest) // if the stream supports video, add the video track if trackLocal, haveTrackLocal := streamStateToAdd.Stream.VideoTrackLocal(); haveTrackLocal { if err := addTrack(trackLocal); err != nil { - ss.logger.Error(err.Error()) + server.logger.Error(err.Error()) return nil, err } } // if the stream supports audio, add the audio track if trackLocal, haveTrackLocal := streamStateToAdd.Stream.AudioTrackLocal(); haveTrackLocal { if err := addTrack(trackLocal); err != nil { - ss.logger.Error(err.Error()) + server.logger.Error(err.Error()) return nil, err } } - if err := streamStateToAdd.Increment(ctx); err != nil { - ss.logger.Error(err.Error()) + if err := streamStateToAdd.Increment(); err != nil { + server.logger.Error(err.Error()) return nil, err } @@ -242,70 +264,147 @@ func (ss *Server) AddStream(ctx context.Context, req *streampb.AddStreamRequest) } // RemoveStream implements part of the StreamServiceServer. -func (ss *Server) RemoveStream(ctx context.Context, req *streampb.RemoveStreamRequest) (*streampb.RemoveStreamResponse, error) { +func (server *Server) RemoveStream(ctx context.Context, req *streampb.RemoveStreamRequest) (*streampb.RemoveStreamResponse, error) { ctx, span := trace.StartSpan(ctx, "stream::server::RemoveStream") defer span.End() pc, ok := rpc.ContextPeerConnection(ctx) + server.logger.Infow("Removing video stream", "name", req.Name, "peerConn", pc) if !ok { return nil, errors.New("can only remove a stream over a WebRTC based connection") } - ss.mu.Lock() - defer ss.mu.Unlock() + server.mu.Lock() + defer server.mu.Unlock() - streamToRemove, ok := ss.nameToStreamState[req.Name] + streamToRemove, ok := server.nameToStreamState[req.Name] + // Callers of RemoveStream will continue calling RemoveStream until it succeeds. Retrying on the + // following "stream not found" errors is not helpful in this goal. Thus we return a success + // response. if !ok { - return nil, fmt.Errorf("no stream for %q", req.Name) + return &streampb.RemoveStreamResponse{}, nil } - if _, ok := ss.activePeerStreams[pc][req.Name]; !ok { - return nil, errors.New("stream already inactive") + //nolint:nilerr + if _, err := streamCamera.Camera(server.robot, streamToRemove.Stream); err != nil { + return &streampb.RemoveStreamResponse{}, nil + } + + if _, ok := server.activePeerStreams[pc][req.Name]; !ok { + return &streampb.RemoveStreamResponse{}, nil } var errs error - for _, sender := range ss.activePeerStreams[pc][req.Name].senders { + for _, sender := range server.activePeerStreams[pc][req.Name].senders { errs = multierr.Combine(errs, pc.RemoveTrack(sender)) } if errs != nil { - ss.logger.Error(errs.Error()) + server.logger.Error(errs.Error()) return nil, errs } - if err := streamToRemove.Decrement(ctx); err != nil { - ss.logger.Error(err.Error()) + if err := streamToRemove.Decrement(); err != nil { + server.logger.Error(err.Error()) return nil, err } - delete(ss.activePeerStreams[pc], req.Name) + delete(server.activePeerStreams[pc], req.Name) return &streampb.RemoveStreamResponse{}, nil } // Close closes the Server and waits for spun off goroutines to complete. -func (ss *Server) Close() error { - ss.mu.Lock() - ss.isAlive = false +func (server *Server) Close() error { + server.closedFn() + server.mu.Lock() + server.isAlive = false var errs error - for _, name := range ss.streamNames { - errs = multierr.Combine(errs, ss.nameToStreamState[name].Close()) + for _, streamState := range server.nameToStreamState { + errs = multierr.Combine(errs, streamState.Close()) } if errs != nil { - ss.logger.Errorf("Stream Server Close > StreamState.Close() errs: %s", errs) + server.logger.Errorf("Stream Server Close > StreamState.Close() errs: %s", errs) } - ss.mu.Unlock() - ss.activeBackgroundWorkers.Wait() + server.mu.Unlock() + server.activeBackgroundWorkers.Wait() return errs } -func (ss *Server) add(stream gostream.Stream) error { +func (server *Server) add(stream gostream.Stream) error { streamName := stream.Name() - if _, ok := ss.nameToStreamState[streamName]; ok { + if _, ok := server.nameToStreamState[streamName]; ok { return &StreamAlreadyRegisteredError{streamName} } - newStreamState := state.New(stream, ss.r, ss.logger) - newStreamState.Init() - ss.nameToStreamState[streamName] = newStreamState - ss.streamNames = append(ss.streamNames, streamName) + logger := server.logger.Sublogger(streamName) + newStreamState := state.New(stream, server.robot, logger) + server.nameToStreamState[streamName] = newStreamState return nil } + +// startMonitorCameraAvailable monitors whether or not the camera still exists +// If it no longer exists, it: +// 1. calls RemoveTrack on the senders of all peer connections that called AddTrack on the camera name. +// 2. decrements the number of active peers on the stream state (which should result in the +// stream state having no subscribers and calling gostream.Stop() or rtppaserverthrough.Unsubscribe) +// streaming tracks from it. +func (server *Server) startMonitorCameraAvailable() { + server.activeBackgroundWorkers.Add(1) + utils.ManagedGo(func() { + for utils.SelectContextOrWait(server.closedCtx, monitorCameraInterval) { + server.removeMissingStreams() + } + }, server.activeBackgroundWorkers.Done) +} + +func (server *Server) removeMissingStreams() { + server.mu.Lock() + defer server.mu.Unlock() + for key, streamState := range server.nameToStreamState { + // Stream names are slightly modified versions of the resource short name + camName := streamState.Stream.Name() + shortName := resource.SDPTrackNameToShortName(camName) + + _, err := camera.FromRobot(server.robot, shortName) + if !resource.IsNotFoundError(err) { + // Cameras can go through transient states during reconfigure that don't necessarily + // imply the camera is missing. E.g: *resource.notAvailableError. To double-check we + // have the right set of exceptions here, we log the error and ignore. + if err != nil { + server.logger.Warnw("Error getting camera from robot", + "camera", camName, "err", err, "errType", fmt.Sprintf("%T", err)) + } + continue + } + + // Best effort close any active peer streams. We'll remove from the known streams + // first. Such that we only try closing/unsubscribing once. + server.logger.Infow("Camera doesn't exist. Closing its streams", + "camera", camName, "err", err, "Type", fmt.Sprintf("%T", err)) + delete(server.nameToStreamState, key) + + for pc, peerStateByCamName := range server.activePeerStreams { + peerState, ok := peerStateByCamName[camName] + if !ok { + // There are no known peers for this camera. Do nothing. + server.logger.Infow("no entry in peer map", "camera", camName) + continue + } + + server.logger.Infow("unsubscribing", "camera", camName, "numSenders", len(peerState.senders)) + var errs error + for _, sender := range peerState.senders { + errs = multierr.Combine(errs, pc.RemoveTrack(sender)) + } + + if errs != nil { + server.logger.Warn(errs.Error()) + } + + if err := streamState.Decrement(); err != nil { + server.logger.Warn(err.Error()) + } + delete(server.activePeerStreams[pc], camName) + } + utils.UncheckedError(streamState.Close()) + } +} diff --git a/robot/web/stream/state/state.go b/robot/web/stream/state/state.go index 2f68699ff55..7a51600e72f 100644 --- a/robot/web/stream/state/state.go +++ b/robot/web/stream/state/state.go @@ -4,13 +4,13 @@ package state import ( "context" + "errors" "fmt" "sync" "sync/atomic" "time" "github.com/pion/rtp" - "github.com/pkg/errors" "go.uber.org/multierr" "go.viam.com/utils" @@ -19,37 +19,28 @@ import ( "go.viam.com/rdk/gostream" "go.viam.com/rdk/logging" "go.viam.com/rdk/robot" + streamCamera "go.viam.com/rdk/robot/web/stream/camera" ) -var ( - subscribeRTPTimeout = time.Second * 5 - // UnsubscribeTimeout is the timeout used when unsubscribing from an rtppassthrough subscription. - UnsubscribeTimeout = time.Second * 5 - // ErrRTPPassthroughNotSupported indicates that rtp_passthrough is not supported by the stream's camera. - ErrRTPPassthroughNotSupported = errors.New("RTP Passthrough Not Supported") - // ErrClosed indicates that the StreamState is already closed. - ErrClosed = errors.New("StreamState already closed") - // ErrUninitialized indicates that Init() has not been called on StreamState prior to Increment or Decrement being called. - ErrUninitialized = errors.New("uniniialized") -) +// ErrClosed indicates that the StreamState is already closed. +var ErrClosed = errors.New("StreamState already closed") // StreamState controls the source of the RTP packets being written to the stream's subscribers // and ensures there is only one active at a time while there are subsribers. type StreamState struct { // Stream is the StreamState's stream - Stream gostream.Stream - robot robot.Robot - closedCtx context.Context - closedFn context.CancelFunc - wg sync.WaitGroup - logger logging.Logger - initialized atomic.Bool - - msgChan chan msg - restartChan chan struct{} - - activePeers int - streamSource streamSource + Stream gostream.Stream + robot robot.Robot + closedCtx context.Context + closedFn context.CancelFunc + wg sync.WaitGroup + logger logging.Logger + + msgChan chan msg + tickChan chan struct{} + + activeClients int + streamSource streamSource // streamSourceSub is only non nil if streamSource == streamSourcePassthrough streamSourceSub rtppassthrough.Subscription } @@ -63,61 +54,47 @@ func New( logger logging.Logger, ) *StreamState { ctx, cancel := context.WithCancel(context.Background()) - return &StreamState{ - Stream: stream, - closedCtx: ctx, - closedFn: cancel, - robot: r, - msgChan: make(chan msg), - restartChan: make(chan struct{}), - logger: logger, - } -} - -// Init initializes the StreamState -// Init must be called before any other methods. -func (ss *StreamState) Init() { - ss.wg.Add(1) - utils.ManagedGo(ss.initEventHandler, ss.wg.Done) - ss.wg.Add(1) - utils.ManagedGo(ss.initStreamSourceMonitor, ss.wg.Done) - ss.initialized.Store(true) + ret := &StreamState{ + Stream: stream, + closedCtx: ctx, + closedFn: cancel, + robot: r, + msgChan: make(chan msg), + tickChan: make(chan struct{}), + logger: logger, + } + + ret.wg.Add(1) + // The event handler for a stream input manages the following events: + // - There's a new subscriber (bump ref counter) + // - A subscriber has left (dec ref counter) + // - Camera is remevod (dec for all subscribers) + // - Peer connection is closed (dec for all subscribers) + utils.ManagedGo(ret.sourceEventHandler, ret.wg.Done) + return ret } // Increment increments the peer connections subscribed to the stream. -func (ss *StreamState) Increment(ctx context.Context) error { - if !ss.initialized.Load() { - return ErrUninitialized - } - if err := ss.closedCtx.Err(); err != nil { +func (state *StreamState) Increment() error { + if err := state.closedCtx.Err(); err != nil { return multierr.Combine(ErrClosed, err) } - return ss.send(ctx, msgTypeIncrement) + return state.send(msgTypeIncrement) } // Decrement decrements the peer connections subscribed to the stream. -func (ss *StreamState) Decrement(ctx context.Context) error { - if !ss.initialized.Load() { - return ErrUninitialized - } - if err := ss.closedCtx.Err(); err != nil { +func (state *StreamState) Decrement() error { + if err := state.closedCtx.Err(); err != nil { return multierr.Combine(ErrClosed, err) } - return ss.send(ctx, msgTypeDecrement) -} - -// Restart restarts the stream source after it has terminated. -func (ss *StreamState) Restart(ctx context.Context) { - if err := ss.closedCtx.Err(); err != nil { - return - } - utils.UncheckedError(ss.send(ctx, msgTypeRestart)) + return state.send(msgTypeDecrement) } // Close closes the StreamState. -func (ss *StreamState) Close() error { - ss.closedFn() - ss.wg.Wait() +func (state *StreamState) Close() error { + state.logger.Info("Closing streamState") + state.closedFn() + state.wg.Wait() return nil } @@ -152,7 +129,6 @@ const ( msgTypeUnknown msgType = iota msgTypeIncrement msgTypeDecrement - msgTypeRestart ) func (mt msgType) String() string { @@ -161,8 +137,6 @@ func (mt msgType) String() string { return "Increment" case msgTypeDecrement: return "Decrement" - case msgTypeRestart: - return "Restart" case msgTypeUnknown: fallthrough default: @@ -171,296 +145,214 @@ func (mt msgType) String() string { } type msg struct { - msgType msgType - ctx context.Context - respChan chan error + msgType msgType } -// events (Inc Dec Restart). -func (ss *StreamState) initEventHandler() { - ss.logger.Debug("StreamState initEventHandler booted") - defer ss.logger.Debug("StreamState initEventHandler terminated") +// events (Inc Dec Tick). +func (state *StreamState) sourceEventHandler() { + state.logger.Debug("sourceEventHandler booted") defer func() { - ctx, cancel := context.WithTimeout(context.Background(), UnsubscribeTimeout) - defer cancel() - utils.UncheckedError(ss.stopBasedOnSub(ctx)) + state.logger.Debug("sourceEventHandler terminating") + state.stopInputStream() }() + ticker := time.NewTicker(time.Second) + defer ticker.Stop() for { - if ss.closedCtx.Err() != nil { - return - } - + // We wait for: + // - A message to be discovered on the `msgChan` queue. + // - The `tick` timer to be fired + // - The server/stream (i.e: camera) to be shutdown. + var msg msg select { - case <-ss.closedCtx.Done(): + case <-state.closedCtx.Done(): return - case msg := <-ss.msgChan: - ss.handleMsg(msg) + case msg = <-state.msgChan: + case <-ticker.C: + state.tick() + continue } - } -} -func (ss *StreamState) initStreamSourceMonitor() { - for { - select { - case <-ss.closedCtx.Done(): - return - case <-ss.restartChan: - ctx, cancel := context.WithTimeout(ss.closedCtx, subscribeRTPTimeout) - ss.Restart(ctx) - cancel() + switch msg.msgType { + case msgTypeIncrement: + state.activeClients++ + state.logger.Debugw("activeClients incremented", "activeClientCnt", state.activeClients) + if state.activeClients == 1 { + state.tick() + } + case msgTypeDecrement: + state.activeClients-- + state.logger.Debugw("activeClients decremented", "activeClientCnt", state.activeClients) + if state.activeClients == 0 { + state.tick() + } + case msgTypeUnknown: + fallthrough + default: + state.logger.Errorw("Invalid StreamState msg type received", "type", msg.msgType) } } } -func (ss *StreamState) monitorSubscription(sub rtppassthrough.Subscription) { - if ss.streamSource == streamSourceGoStream { - ss.logger.Debugf("monitorSubscription stopping gostream %s", ss.Stream.Name()) - // if we were streaming using gostream, stop streaming using gostream as we are now using passthrough - ss.Stream.Stop() - } - ss.streamSourceSub = sub - ss.streamSource = streamSourcePassthrough - monitorSubFunc := func() { - // if the stream state is shutting down, terminate - if ss.closedCtx.Err() != nil { - return - } - +func (state *StreamState) monitorSubscription(terminatedCtx context.Context) { + select { + case <-state.closedCtx.Done(): + return + case <-terminatedCtx.Done(): select { - case <-ss.closedCtx.Done(): - return - case <-sub.Terminated.Done(): - select { - case ss.restartChan <- struct{}{}: - case <-ss.closedCtx.Done(): - } - return + case state.tickChan <- struct{}{}: + state.logger.Info("monitorSubscription sent to tickChan") + default: } + return } - - ss.wg.Add(1) - utils.ManagedGo(monitorSubFunc, ss.wg.Done) } -// caller must be holding ss.mu. -func (ss *StreamState) stopBasedOnSub(ctx context.Context) error { - switch ss.streamSource { +func (state *StreamState) stopInputStream() { + switch state.streamSource { case streamSourceGoStream: - ss.logger.Debugf("%s stopBasedOnSub stopping GoStream", ss.Stream.Name()) - ss.Stream.Stop() - ss.streamSource = streamSourceUnknown - return nil + state.logger.Debug("stopping gostream stream") + defer state.logger.Debug("gostream stopped") + state.Stream.Stop() + state.streamSource = streamSourceUnknown + return case streamSourcePassthrough: - ss.logger.Debugf("%s stopBasedOnSub stopping passthrough", ss.Stream.Name()) - err := ss.unsubscribeH264Passthrough(ctx, ss.streamSourceSub.ID) - if err != nil { - return err + state.logger.Debug("stopping h264 passthrough stream") + defer state.logger.Debug("h264 passthrough stream stopped") + err := state.unsubscribeH264Passthrough(state.closedCtx, state.streamSourceSub.ID) + if err != nil && errors.Is(err, camera.ErrUnknownSubscriptionID) { + state.logger.Warnw("Error calling unsubscribe", "err", err) + return } - ss.streamSourceSub = rtppassthrough.NilSubscription - ss.streamSource = streamSourceUnknown - return nil - + state.streamSourceSub = rtppassthrough.NilSubscription + state.streamSource = streamSourceUnknown case streamSourceUnknown: - fallthrough default: - return nil } } -func (ss *StreamState) send(ctx context.Context, msgType msgType) error { - if err := ctx.Err(); err != nil { - return err - } - if err := ss.closedCtx.Err(); err != nil { - return err - } - msg := msg{ - ctx: ctx, - msgType: msgType, - respChan: make(chan error), - } +func (state *StreamState) send(msgType msgType) error { select { - case ss.msgChan <- msg: - select { - case err := <-msg.respChan: - return err - case <-ctx.Done(): - return ctx.Err() - case <-ss.closedCtx.Done(): - return ss.closedCtx.Err() - } - case <-ctx.Done(): - return ctx.Err() - case <-ss.closedCtx.Done(): - return ss.closedCtx.Err() - } -} - -func (ss *StreamState) handleMsg(msg msg) { - switch msg.msgType { - case msgTypeIncrement: - err := ss.inc(msg.ctx) - select { - case msg.respChan <- err: - case <-ss.closedCtx.Done(): - return - } - case msgTypeRestart: - ss.restart(msg.ctx) - select { - case msg.respChan <- nil: - case <-ss.closedCtx.Done(): - return - } - case msgTypeDecrement: - err := ss.dec(msg.ctx) - msg.respChan <- err - case msgTypeUnknown: - fallthrough - default: - ss.logger.Error("Invalid StreamState msg type received: %s", msg.msgType) + case state.msgChan <- msg{msgType: msgType}: + return nil + case <-state.closedCtx.Done(): + return state.closedCtx.Err() } } -func (ss *StreamState) inc(ctx context.Context) error { - ss.logger.Debugf("increment %s START activePeers: %d", ss.Stream.Name(), ss.activePeers) - defer func() { ss.logger.Debugf("increment %s END activePeers: %d", ss.Stream.Name(), ss.activePeers) }() - if ss.activePeers == 0 { - if ss.streamSource != streamSourceUnknown { - return fmt.Errorf("unexpected stream %s source %s", ss.Stream.Name(), ss.streamSource) - } +func (state *StreamState) tick() { + switch { + case state.activeClients < 0: + state.logger.Error("activeClients is less than 0") + case state.activeClients == 0: + // stop stream if there are no active clients + // noop if there is no stream source + state.stopInputStream() + case state.streamSource == streamSourceUnknown: // && state.activeClients > 0 // this is the first subscription, attempt passthrough - ss.logger.CDebugw(ctx, "attempting to subscribe to rtp_passthrough", "name", ss.Stream.Name()) - err := ss.streamH264Passthrough(ctx) + state.logger.Info("attempting to subscribe to rtp_passthrough") + err := state.streamH264Passthrough() if err != nil { - ss.logger.CDebugw(ctx, "rtp_passthrough not possible, falling back to GoStream", "err", err.Error(), "name", ss.Stream.Name()) + state.logger.Warnw("tick: rtp_passthrough not possible, falling back to GoStream", "err", err) // if passthrough failed, fall back to gostream based approach - ss.Stream.Start() - ss.streamSource = streamSourceGoStream + state.Stream.Start() + state.streamSource = streamSourceGoStream } - ss.activePeers++ - return nil - } + case state.streamSource == streamSourcePassthrough && state.streamSourceSub.Terminated.Err() != nil: + // restart stream if there we were using passthrough but the sub is terminated + state.logger.Info("previous subscription terminated attempting to subscribe to rtp_passthrough") - switch ss.streamSource { - case streamSourcePassthrough: - ss.logger.CDebugw(ctx, "continuing using rtp_passthrough", "name", ss.Stream.Name()) - // noop as we are already subscribed - case streamSourceGoStream: - ss.logger.CDebugw(ctx, "currently using gostream, trying upgrade to rtp_passthrough", "name", ss.Stream.Name()) - // attempt to cut over to passthrough - err := ss.streamH264Passthrough(ctx) + err := state.streamH264Passthrough() if err != nil { - ss.logger.CDebugw(ctx, "rtp_passthrough not possible, continuing with gostream", "err", err.Error(), "name", ss.Stream.Name()) - } - case streamSourceUnknown: - fallthrough - default: - err := fmt.Errorf("%s streamSource in unexpected state %s", ss.Stream.Name(), ss.streamSource) - ss.logger.Error(err.Error()) - return err - } - ss.activePeers++ - return nil -} - -func (ss *StreamState) dec(ctx context.Context) error { - ss.logger.Debugf("decrement START %s activePeers: %d", ss.Stream.Name(), ss.activePeers) - defer func() { ss.logger.Debugf("decrement END %s activePeers: %d", ss.Stream.Name(), ss.activePeers) }() - - var err error - defer func() { - if err != nil { - ss.logger.Errorf("decrement %s hit error: %s", ss.Stream.Name(), err.Error()) - return - } - ss.activePeers-- - if ss.activePeers <= 0 { - ss.activePeers = 0 + state.logger.Warn("rtp_passthrough not possible, falling back to GoStream", "err", err) + // if passthrough failed, fall back to gostream based approach + state.Stream.Start() + state.streamSource = streamSourceGoStream } - }() - if ss.activePeers == 1 { - ss.logger.Debugf("decrement %s calling stopBasedOnSub", ss.Stream.Name()) - err = ss.stopBasedOnSub(ctx) + case state.streamSource == streamSourcePassthrough: + // no op if we are using passthrough & are healthy + state.logger.Debug("still healthy and using h264 passthrough") + case state.streamSource == streamSourceGoStream: + // Try to upgrade to passthrough if we are using gostream. We leave logs these as debugs as + // we expect some components to not implement rtp passthrough. + state.logger.Debugw("currently using gostream, trying upgrade to rtp_passthrough") + // attempt to cut over to passthrough + err := state.streamH264Passthrough() if err != nil { - ss.logger.Error(err.Error()) - return err + state.logger.Debugw("rtp_passthrough upgrade failed, continuing with gostream", "err", err) } } - return nil -} - -func (ss *StreamState) restart(ctx context.Context) { - ss.logger.Debugf("restart %s START activePeers: %d", ss.Stream.Name(), ss.activePeers) - defer func() { ss.logger.Debugf("restart %s END activePeers: %d", ss.Stream.Name(), ss.activePeers) }() - - if ss.activePeers == 0 { - // nothing to do if we don't have any active peers - return - } - - if ss.streamSource == streamSourceGoStream { - // nothing to do if stream source is gostream - return - } - - if ss.streamSourceSub != rtppassthrough.NilSubscription && ss.streamSourceSub.Terminated.Err() == nil { - // if the stream is still healthy, do nothing - return - } - - err := ss.streamH264Passthrough(ctx) - if err != nil { - ss.logger.CDebugw(ctx, "rtp_passthrough not possible, falling back to GoStream", "err", err.Error(), "name", ss.Stream.Name()) - // if passthrough failed, fall back to gostream based approach - ss.Stream.Start() - ss.streamSource = streamSourceGoStream - - return - } - // passthrough succeeded, listen for when subscription end and call start again if so } -func (ss *StreamState) streamH264Passthrough(ctx context.Context) error { - cam, err := camera.FromRobot(ss.robot, ss.Stream.Name()) +func (state *StreamState) streamH264Passthrough() error { + cam, err := streamCamera.Camera(state.robot, state.Stream) if err != nil { return err } + // Get the camera and see if it implements the rtp passthrough API of SubscribeRTP + Unsubscribe rtpPassthroughSource, ok := cam.(rtppassthrough.Source) if !ok { - err := fmt.Errorf("expected %s to implement rtppassthrough.Source", ss.Stream.Name()) - return errors.Wrap(ErrRTPPassthroughNotSupported, err.Error()) + return errors.New("stream does not support RTP passthrough") } + var count atomic.Uint64 + + // We might be already sending video via gostream. In this case we: + // - First try and create an RTP passthrough subscription + // - If not successful, continue with gostream. + // - Otherwise if successful, stop gostream. + // - Once we're sure gostream is stopped, we close the `releasePackets` channel + // + // This ensures we only start sending passthrough packets after gostream has stopped sending + // video packets. + releasePackets := make(chan struct{}) + cb := func(pkts []*rtp.Packet) { + <-releasePackets for _, pkt := range pkts { - if err := ss.Stream.WriteRTP(pkt); err != nil { - ss.logger.Debugw("stream.WriteRTP", "name", ss.Stream.Name(), "err", err.Error()) + // Also, look at unsubscribe error logs. Definitely a bug. Probably benign. + if count.Add(1)%10000 == 0 { + state.logger.Debugw("WriteRTP called. Sampling 1/10000", + "count", count.Load(), "seqNumber", pkt.Header.SequenceNumber, "ts", pkt.Header.Timestamp) + } + if err := state.Stream.WriteRTP(pkt); err != nil { + state.logger.Debugw("stream.WriteRTP", "name", state.Stream.Name(), "err", err.Error()) } } } - sub, err := rtpPassthroughSource.SubscribeRTP(ctx, rtpBufferSize, cb) + sub, err := rtpPassthroughSource.SubscribeRTP(state.closedCtx, rtpBufferSize, cb) if err != nil { - return errors.Wrap(ErrRTPPassthroughNotSupported, err.Error()) + return fmt.Errorf("SubscribeRTP failed: %w", err) } - ss.logger.CWarnw(ctx, "Stream using experimental H264 passthrough", "name", ss.Stream.Name()) - ss.monitorSubscription(sub) + state.logger.Warnw("Stream using experimental H264 passthrough", "name", state.Stream.Name()) + + if state.streamSource == streamSourceGoStream { + state.logger.Debugf("monitorSubscription stopping gostream %s", state.Stream.Name()) + // We've succeeded creating a passthrough stream. If we were streaming using gostream, stop it. + state.Stream.Stop() + } + close(releasePackets) + state.streamSourceSub = sub + state.streamSource = streamSourcePassthrough + + state.wg.Add(1) + utils.ManagedGo(func() { + state.monitorSubscription(sub.Terminated) + }, state.wg.Done) return nil } -func (ss *StreamState) unsubscribeH264Passthrough(ctx context.Context, id rtppassthrough.SubscriptionID) error { - cam, err := camera.FromRobot(ss.robot, ss.Stream.Name()) +func (state *StreamState) unsubscribeH264Passthrough(ctx context.Context, id rtppassthrough.SubscriptionID) error { + cam, err := streamCamera.Camera(state.robot, state.Stream) if err != nil { return err } rtpPassthroughSource, ok := cam.(rtppassthrough.Source) if !ok { - err := fmt.Errorf("expected %s to implement rtppassthrough.Source", ss.Stream.Name()) - return errors.Wrap(ErrRTPPassthroughNotSupported, err.Error()) + return fmt.Errorf("subscription resource does not implement rtpPassthroughSource. CamType: %T", rtpPassthroughSource) } if err := rtpPassthroughSource.Unsubscribe(ctx, id); err != nil { diff --git a/robot/web/stream/state/state_test.go b/robot/web/stream/state/state_test.go index ed4c5b337b5..1c482482714 100644 --- a/robot/web/stream/state/state_test.go +++ b/robot/web/stream/state/state_test.go @@ -3,6 +3,7 @@ package state_test import ( "context" "errors" + "fmt" "image" "sync" "sync/atomic" @@ -16,6 +17,7 @@ import ( "github.com/viamrobotics/webrtc/v3" "go.viam.com/test" "go.viam.com/utils" + "go.viam.com/utils/testutils" "go.viam.com/rdk/components/camera" "go.viam.com/rdk/components/camera/rtppassthrough" @@ -39,10 +41,12 @@ func (mS *mockStream) Name() string { return mS.name } +// Start refers to starting gostream. func (mS *mockStream) Start() { mS.startFunc() } +// Stop refers to stopping gostream. func (mS *mockStream) Stop() { mS.stopFunc() } @@ -53,32 +57,27 @@ func (mS *mockStream) WriteRTP(pkt *rtp.Packet) error { // BEGIN Not tested gostream functions. func (mS *mockStream) StreamingReady() (<-chan struct{}, context.Context) { - mS.t.Log("unimplemented") - mS.t.FailNow() + test.That(mS.t, "should not be called", test.ShouldBeFalse) return nil, context.Background() } func (mS *mockStream) InputVideoFrames(props prop.Video) (chan<- gostream.MediaReleasePair[image.Image], error) { - mS.t.Log("unimplemented") - mS.t.FailNow() + test.That(mS.t, "should not be called", test.ShouldBeFalse) return nil, errors.New("unimplemented") } func (mS *mockStream) InputAudioChunks(props prop.Audio) (chan<- gostream.MediaReleasePair[wave.Audio], error) { - mS.t.Log("unimplemented") - mS.t.FailNow() + test.That(mS.t, "should not be called", test.ShouldBeFalse) return make(chan gostream.MediaReleasePair[wave.Audio]), nil } func (mS *mockStream) VideoTrackLocal() (webrtc.TrackLocal, bool) { - mS.t.Log("unimplemented") - mS.t.FailNow() + test.That(mS.t, "should not be called", test.ShouldBeFalse) return nil, false } func (mS *mockStream) AudioTrackLocal() (webrtc.TrackLocal, bool) { - mS.t.Log("unimplemented") - mS.t.FailNow() + test.That(mS.t, "should not be called", test.ShouldBeFalse) return nil, false } @@ -111,7 +110,6 @@ func (s *mockRTPPassthroughSource) Unsubscribe( var camName = "my-cam" -// END Not tested gostream functions. func mockRobot(s rtppassthrough.Source) robot.Robot { robot := &inject.Robot{} robot.MockResourcesFromMap(map[resource.Name]resource.Resource{ @@ -123,62 +121,17 @@ func mockRobot(s rtppassthrough.Source) robot.Robot { func TestStreamState(t *testing.T) { ctx := context.Background() logger := logging.NewTestLogger(t) + // we have to use sleep here as we are asserting the state doesn't change after a given period of time + sleepDuration := time.Millisecond * 200 - t.Run("Stream returns the provided stream", func(t *testing.T) { - mockRTPPassthroughSource := &mockRTPPassthroughSource{} - robot := mockRobot(mockRTPPassthroughSource) - streamMock := &mockStream{name: camName, t: t} - s := state.New(streamMock, robot, logger) - test.That(t, s.Stream, test.ShouldEqual, streamMock) - }) - - t.Run("close succeeds if no methods have been called", func(t *testing.T) { - mockRTPPassthroughSource := &mockRTPPassthroughSource{} - robot := mockRobot(mockRTPPassthroughSource) - streamMock := &mockStream{name: camName, t: t} - s := state.New(streamMock, robot, logger) - test.That(t, s.Close(), test.ShouldBeNil) - }) - - t.Run("Increment() returns an error if Init() is not called first", func(t *testing.T) { - mockRTPPassthroughSource := &mockRTPPassthroughSource{} - robot := mockRobot(mockRTPPassthroughSource) - streamMock := &mockStream{name: camName, t: t} - s := state.New(streamMock, robot, logger) - test.That(t, s.Increment(ctx), test.ShouldBeError, state.ErrUninitialized) - }) - - t.Run("Decrement() returns an error if Init() is not called first", func(t *testing.T) { - mockRTPPassthroughSource := &mockRTPPassthroughSource{} - robot := mockRobot(mockRTPPassthroughSource) - streamMock := &mockStream{name: camName, t: t} - s := state.New(streamMock, robot, logger) - test.That(t, s.Decrement(ctx), test.ShouldBeError, state.ErrUninitialized) - }) - - t.Run("Increment() returns an error if called after Close()", func(t *testing.T) { - mockRTPPassthroughSource := &mockRTPPassthroughSource{} - robot := mockRobot(mockRTPPassthroughSource) - streamMock := &mockStream{name: camName, t: t} - s := state.New(streamMock, robot, logger) - s.Init() - s.Close() - test.That(t, s.Increment(ctx), test.ShouldWrap, state.ErrClosed) - }) - - t.Run("Decrement() returns an error if called after Close()", func(t *testing.T) { - mockRTPPassthroughSource := &mockRTPPassthroughSource{} - robot := mockRobot(mockRTPPassthroughSource) - streamMock := &mockStream{name: camName, t: t} - s := state.New(streamMock, robot, logger) - s.Init() - s.Close() - test.That(t, s.Decrement(ctx), test.ShouldWrap, state.ErrClosed) - }) - - t.Run("when rtppassthrough.Souce is provided but SubscribeRTP always returns an error", func(t *testing.T) { + t.Run("when rtppassthrough.Source is provided but SubscribeRTP always returns an error", func(t *testing.T) { + // Define counters that are bumped every time the video stream is started or stopped in + // "gostream mode". var startCount atomic.Int64 var stopCount atomic.Int64 + + // Because SubscribeRTP will always return an error, it should be a test failure if the mock + // stream tries writing an RTP packet. streamMock := &mockStream{ name: camName, t: t, @@ -189,15 +142,15 @@ func TestStreamState(t *testing.T) { stopCount.Add(1) }, writeRTPFunc: func(pkt *rtp.Packet) error { - t.Log("should not happen") - t.FailNow() + test.That(t, "should not be called", test.ShouldBeFalse) return nil }, } + // Define an a function that matches the SubscribeRTP signature. It will fail on each + // SubscribeRTP call. We define a counter to know how often the function has been called. var subscribeRTPCount atomic.Int64 - - subscribeRTPFunc := func( + failingSubscribeRTPFunc := func( ctx context.Context, bufferSize int, packetsCB rtppassthrough.PacketCallback, @@ -206,118 +159,121 @@ func TestStreamState(t *testing.T) { return rtppassthrough.NilSubscription, errors.New("unimplemented") } + // Because SubscribeRTP will always fail, UnsubscribeRTP must not be called. unsubscribeFunc := func(ctx context.Context, id rtppassthrough.SubscriptionID) error { - t.Log("should not happen") - t.FailNow() + test.That(t, "should not be called", test.ShouldBeFalse) return errors.New("unimplemented") } mockRTPPassthroughSource := &mockRTPPassthroughSource{ - subscribeRTPFunc: subscribeRTPFunc, + subscribeRTPFunc: failingSubscribeRTPFunc, unsubscribeFunc: unsubscribeFunc, } robot := mockRobot(mockRTPPassthroughSource) s := state.New(streamMock, robot, logger) - defer func() { utils.UncheckedError(s.Close()) }() - s.Init() + defer func() { + utils.UncheckedError(s.Close()) + }() test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 0) test.That(t, startCount.Load(), test.ShouldEqual, 0) test.That(t, stopCount.Load(), test.ShouldEqual, 0) - t.Log("the first Increment() calls SubscribeRTP and then calls Start() when an error is reurned") - test.That(t, s.Increment(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 1) - test.That(t, startCount.Load(), test.ShouldEqual, 1) - test.That(t, stopCount.Load(), test.ShouldEqual, 0) - - t.Log("subsequent Increment() all calls call SubscribeRTP trying to determine " + - "if they can upgrade but don't call any other gostream methods as SubscribeRTP returns an error") - test.That(t, s.Increment(ctx), test.ShouldBeNil) - test.That(t, s.Increment(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 3) - test.That(t, startCount.Load(), test.ShouldEqual, 1) - test.That(t, stopCount.Load(), test.ShouldEqual, 0) - - t.Log("as long as the number of Decrement() calls is less than the number of Increment() calls, no gostream methods are called") - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 3) + // Now we "increment" the number of `AddStream` calls. This means there's a (single) + // consumer for this video stream. + logger.Info("the first Increment() eventually calls SubscribeRTP and then calls Start() when an error is returned") + test.That(t, s.Increment(), test.ShouldBeNil) + + // The post-loop invariant is that we've called SubscribeRTP (for passthrough) at least + // once. And gostream has been started exactly once and never stopped. In this state, the + // stream server will continue trying to upgrade to RTP passthrough. `Stop` will be only + // called on the state object if the SubscribeRTP method was a success. + var prevSubscribeRTPCount int64 + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, subscribeRTPCount.Load(), test.ShouldBeGreaterThan, 0) + prevSubscribeRTPCount = subscribeRTPCount.Load() + test.That(tb, startCount.Load(), test.ShouldEqual, 1) + test.That(tb, stopCount.Load(), test.ShouldEqual, 0) + }) + + logger.Info("as long as the number of Decrement() calls is less than the number of Increment() calls, no gostream methods are called") + test.That(t, s.Increment(), test.ShouldBeNil) + test.That(t, s.Increment(), test.ShouldBeNil) + test.That(t, s.Decrement(), test.ShouldBeNil) + test.That(t, s.Decrement(), test.ShouldBeNil) + + // Wait a bit and assert we've continue to try and upgrade to RTP passthrough. As measured + // by SubscribeRTP calls. + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, subscribeRTPCount.Load(), test.ShouldBeGreaterThan, prevSubscribeRTPCount) + }) + prevSubscribeRTPCount = subscribeRTPCount.Load() + // Double check we did not stop/restart gostream. test.That(t, startCount.Load(), test.ShouldEqual, 1) test.That(t, stopCount.Load(), test.ShouldEqual, 0) - t.Log("when the number of Decrement() calls is equal to the number of Increment() calls stop is called") - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 3) + // Decrement the stream state which should eventually stop gostream. + logger.Info("when the number of Decrement() calls is equal to the number of Increment() calls stop is called") + test.That(t, s.Decrement(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, stopCount.Load(), test.ShouldEqual, 1) + }) + + // An upgrade attempt may or may not have been made concurrently with decrementing. Update + // the running SubscribeRTP call counter. + prevSubscribeRTPCount = subscribeRTPCount.Load() + // We must not have tried restarting gostream. test.That(t, startCount.Load(), test.ShouldEqual, 1) - test.That(t, stopCount.Load(), test.ShouldEqual, 1) - t.Log("then when the number of Increment() calls exceeds Decrement(), both SubscribeRTP & Start are called again") - test.That(t, s.Increment(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 4) - test.That(t, startCount.Load(), test.ShouldEqual, 2) - test.That(t, stopCount.Load(), test.ShouldEqual, 1) - - t.Log("calling Decrement() more times than Increment() has a floor of zero") - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 4) - test.That(t, startCount.Load(), test.ShouldEqual, 2) - test.That(t, stopCount.Load(), test.ShouldEqual, 2) - - // multiple Decrement() calls when the count is already at zero doesn't call any methods - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 4) - test.That(t, startCount.Load(), test.ShouldEqual, 2) - test.That(t, stopCount.Load(), test.ShouldEqual, 2) - - // once the count is at zero , calling Increment() again calls SubscribeRTP and when it returns an error Start - test.That(t, s.Increment(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 5) - test.That(t, startCount.Load(), test.ShouldEqual, 3) - test.That(t, stopCount.Load(), test.ShouldEqual, 2) + logger.Info("then when the number of Increment() calls exceeds Decrement(), both SubscribeRTP & Start are eventually called again") + test.That(t, s.Increment(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + // An increment always tries SubscribeRTP before gostream. + test.That(tb, subscribeRTPCount.Load(), test.ShouldBeGreaterThan, prevSubscribeRTPCount) + test.That(tb, startCount.Load(), test.ShouldEqual, 2) + test.That(tb, stopCount.Load(), test.ShouldEqual, 1) + }) + prevSubscribeRTPCount = subscribeRTPCount.Load() + + // Wait some more to observe SubscribeRTP continuing to be called. Then decrement the stream + // state to observe gostream being `Stop`ed. + time.Sleep(time.Second) + test.That(t, s.Decrement(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, subscribeRTPCount.Load(), test.ShouldBeGreaterThan, prevSubscribeRTPCount) + test.That(tb, startCount.Load(), test.ShouldEqual, 2) + test.That(tb, stopCount.Load(), test.ShouldEqual, 2) + }) + prevSubscribeRTPCount = subscribeRTPCount.Load() + + // Once the count is at zero , calling Increment() again calls SubscribeRTP. SubscribeRTP + // will fail and `Start` gostream. + test.That(t, s.Increment(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, subscribeRTPCount.Load(), test.ShouldBeGreaterThan, prevSubscribeRTPCount) + test.That(tb, startCount.Load(), test.ShouldEqual, 3) + test.That(tb, stopCount.Load(), test.ShouldEqual, 2) + }) // set count back to zero - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 5) - test.That(t, startCount.Load(), test.ShouldEqual, 3) - test.That(t, stopCount.Load(), test.ShouldEqual, 3) - - t.Log("calling Increment() with a cancelled context returns an error & does not call any gostream or rtppassthrough.Source methods") - canceledCtx, cancelFn := context.WithCancel(context.Background()) - cancelFn() - test.That(t, s.Increment(canceledCtx), test.ShouldBeError, context.Canceled) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 5) - test.That(t, startCount.Load(), test.ShouldEqual, 3) - test.That(t, stopCount.Load(), test.ShouldEqual, 3) - - // make it so that non cancelled Decrement() would call stop to confirm that does not happen when context is cancelled - test.That(t, s.Increment(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 6) - test.That(t, startCount.Load(), test.ShouldEqual, 4) - test.That(t, stopCount.Load(), test.ShouldEqual, 3) - - t.Log("calling Decrement() with a cancelled context returns an error & does not call any gostream methods") - test.That(t, s.Decrement(canceledCtx), test.ShouldBeError, context.Canceled) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 6) - test.That(t, startCount.Load(), test.ShouldEqual, 4) - test.That(t, stopCount.Load(), test.ShouldEqual, 3) + test.That(t, s.Decrement(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, subscribeRTPCount.Load(), test.ShouldBeGreaterThanOrEqualTo, prevSubscribeRTPCount) + test.That(tb, startCount.Load(), test.ShouldEqual, 3) + test.That(tb, stopCount.Load(), test.ShouldEqual, 3) + }) }) - t.Run("when rtppassthrough.Souce is provided and SubscribeRTP doesn't return an error", func(t *testing.T) { + t.Run("when rtppassthrough.Source is provided and SubscribeRTP doesn't return an error", func(t *testing.T) { writeRTPCalledCtx, writeRTPCalledFunc := context.WithCancel(ctx) streamMock := &mockStream{ name: camName, t: t, startFunc: func() { - t.Logf("should not be called") - t.FailNow() + test.That(t, "should not be called", test.ShouldBeFalse) }, stopFunc: func() { - t.Logf("should not be called") - t.FailNow() + test.That(t, "should not be called", test.ShouldBeFalse) }, writeRTPFunc: func(pkt *rtp.Packet) error { // Test that WriteRTP is eventually called when SubscribeRTP is called @@ -365,8 +321,7 @@ func TestStreamState(t *testing.T) { defer unsubscribeCount.Add(1) subAndCancel, ok := subsAndCancelByID[id] if !ok { - t.Logf("Unsubscribe called with unknown id: %s", id.String()) - t.FailNow() + test.That(t, fmt.Sprintf("Unsubscribe called with unknown id: %s", id.String()), test.ShouldBeFalse) } subAndCancel.cancelFn() return nil @@ -378,85 +333,76 @@ func TestStreamState(t *testing.T) { } robot := mockRobot(mockRTPPassthroughSource) s := state.New(streamMock, robot, logger) - defer func() { utils.UncheckedError(s.Close()) }() - - s.Init() + defer func() { + utils.UncheckedError(s.Close()) + }() test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 0) test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 0) test.That(t, writeRTPCalledCtx.Err(), test.ShouldBeNil) - t.Log("the first Increment() call calls SubscribeRTP()") - test.That(t, s.Increment(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 1) - test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 0) + logger.Info("the first Increment() eventually call calls SubscribeRTP()") + test.That(t, s.Increment(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, subscribeRTPCount.Load(), test.ShouldEqual, 1) + test.That(tb, unsubscribeCount.Load(), test.ShouldEqual, 0) + }) // WriteRTP is called <-writeRTPCalledCtx.Done() - t.Log("subsequent Increment() calls don't call any other rtppassthrough.Source methods") - test.That(t, s.Increment(ctx), test.ShouldBeNil) - test.That(t, s.Increment(ctx), test.ShouldBeNil) + logger.Info("subsequent Increment() calls don't call any other rtppassthrough.Source methods") + test.That(t, s.Increment(), test.ShouldBeNil) + test.That(t, s.Increment(), test.ShouldBeNil) + time.Sleep(sleepDuration) test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 1) test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 0) - t.Log("as long as the number of Decrement() calls is less than the number " + + logger.Info("as long as the number of Decrement() calls is less than the number " + "of Increment() calls, no rtppassthrough.Source methods are called") - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, s.Decrement(ctx), test.ShouldBeNil) + test.That(t, s.Decrement(), test.ShouldBeNil) + test.That(t, s.Decrement(), test.ShouldBeNil) + time.Sleep(sleepDuration) test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 1) test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 0) - t.Log("when the number of Decrement() calls is equal to the number of Increment() calls stop is called") - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 1) - test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 1) - - t.Log("then when the number of Increment() calls exceeds Decrement(), SubscribeRTP is called again") - test.That(t, s.Increment(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 2) - test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 1) + logger.Info("when the number of Decrement() calls is equal to the number of Increment() calls stop is called") + test.That(t, s.Decrement(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, subscribeRTPCount.Load(), test.ShouldEqual, 1) + test.That(tb, unsubscribeCount.Load(), test.ShouldEqual, 1) + }) - t.Log("calling Decrement() more times than Increment() has a floor of zero") - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 2) - test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 2) + logger.Info("then when the number of Increment() calls exceeds Decrement(), SubscribeRTP is called again") + test.That(t, s.Increment(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, subscribeRTPCount.Load(), test.ShouldEqual, 2) + test.That(tb, unsubscribeCount.Load(), test.ShouldEqual, 1) + }) - // multiple Decrement() calls when the count is already at zero doesn't call any methods - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 2) - test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 2) + test.That(t, s.Decrement(), test.ShouldBeNil) // once the count is at zero , calling Increment() again calls start - test.That(t, s.Increment(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 3) - test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 2) + test.That(t, s.Increment(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, subscribeRTPCount.Load(), test.ShouldEqual, 3) + test.That(tb, unsubscribeCount.Load(), test.ShouldEqual, 2) + }) // set count back to zero - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 3) - test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 3) - - t.Log("calling Increment() with a cancelled context returns an error & does not call any rtppassthrough.Source methods") - canceledCtx, cancelFn := context.WithCancel(context.Background()) - cancelFn() - test.That(t, s.Increment(canceledCtx), test.ShouldBeError, context.Canceled) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 3) - test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 3) + test.That(t, s.Decrement(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, subscribeRTPCount.Load(), test.ShouldEqual, 3) + test.That(tb, unsubscribeCount.Load(), test.ShouldEqual, 3) + }) // make it so that non cancelled Decrement() would call stop to confirm that does not happen when context is cancelled - test.That(t, s.Increment(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 4) - test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 3) - - t.Log("calling Decrement() with a cancelled context returns an error & does not call any rtppassthrough.Source methods") - test.That(t, s.Decrement(canceledCtx), test.ShouldBeError, context.Canceled) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 4) - test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 3) + test.That(t, s.Increment(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, subscribeRTPCount.Load(), test.ShouldEqual, 4) + test.That(tb, unsubscribeCount.Load(), test.ShouldEqual, 3) + }) - t.Log("when the subscription terminates while there are still subscribers, SubscribeRTP is called again") + logger.Info("when the subscription terminates while there are still subscribers, SubscribeRTP is called again") var cancelledSubs int subsAndCancelByIDMu.Lock() for _, subAndCancel := range subsAndCancelByID { @@ -473,8 +419,9 @@ func TestStreamState(t *testing.T) { defer timeoutFn() for { if timeoutCtx.Err() != nil { - t.Log("timed out waiting for a new sub to be created after an in progress one terminated unexpectedly") - t.FailNow() + test.That(t, + "timed out waiting for a new sub to be created after an in progress one terminated unexpectedly", + test.ShouldBeFalse) } if subscribeRTPCount.Load() == 5 { break @@ -491,7 +438,7 @@ func TestStreamState(t *testing.T) { subsAndCancelByIDMu.Unlock() }) - t.Run("when rtppassthrough.Souce is provided and sometimes returns an errors "+ + t.Run("when rtppassthrough.Source is provided and sometimes returns an errors "+ "(test rtp_passthrough/gostream upgrade/downgrade path)", func(t *testing.T) { var startCount atomic.Int64 var stopCount atomic.Int64 @@ -543,8 +490,9 @@ func TestStreamState(t *testing.T) { defer unsubscribeCount.Add(1) subAndCancel, ok := subsAndCancelByID[id] if !ok { - t.Logf("Unsubscribe called with unknown id: %s", id.String()) - t.FailNow() + test.That(t, + fmt.Sprintf("Unsubscribe called with unknown id: %s", id.String()), + test.ShouldBeFalse) } subAndCancel.cancelFn() return nil @@ -556,32 +504,34 @@ func TestStreamState(t *testing.T) { } robot := mockRobot(mockRTPPassthroughSource) s := state.New(streamMock, robot, logger) - defer func() { utils.UncheckedError(s.Close()) }() - - // start with RTPPassthrough being supported - s.Init() + defer func() { + utils.UncheckedError(s.Close()) + }() test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 0) test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 0) test.That(t, startCount.Load(), test.ShouldEqual, 0) test.That(t, stopCount.Load(), test.ShouldEqual, 0) - t.Log("the first Increment() call calls SubscribeRTP() which returns a success") - test.That(t, s.Increment(ctx), test.ShouldBeNil) + logger.Info("the first Increment() call calls SubscribeRTP() which returns a success") + test.That(t, s.Increment(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, subscribeRTPCount.Load(), test.ShouldEqual, 1) + test.That(tb, unsubscribeCount.Load(), test.ShouldEqual, 0) + test.That(tb, startCount.Load(), test.ShouldEqual, 0) + test.That(tb, stopCount.Load(), test.ShouldEqual, 0) + }) + + logger.Info("subsequent Increment() calls don't call any other rtppassthrough.Source or gostream methods") + test.That(t, s.Increment(), test.ShouldBeNil) + test.That(t, s.Increment(), test.ShouldBeNil) + time.Sleep(sleepDuration) test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 1) test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 0) test.That(t, startCount.Load(), test.ShouldEqual, 0) test.That(t, stopCount.Load(), test.ShouldEqual, 0) - t.Log("subsequent Increment() calls don't call any other rtppassthrough.Source or gostream methods") - test.That(t, s.Increment(ctx), test.ShouldBeNil) - test.That(t, s.Increment(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 1) - test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 0) - test.That(t, startCount.Load(), test.ShouldEqual, 0) - test.That(t, stopCount.Load(), test.ShouldEqual, 0) - - t.Log("if the subscription terminates and SubscribeRTP() returns an error, starts gostream") + logger.Info("if the subscription terminates and SubscribeRTP() returns an error, starts gostream") subscribeRTPReturnError.Store(true) subsAndCancelByIDMu.Lock() test.That(t, len(subsAndCancelByID), test.ShouldEqual, 1) @@ -594,8 +544,9 @@ func TestStreamState(t *testing.T) { defer timeoutFn() for { if timeoutCtx.Err() != nil { - t.Log("timed out waiting for gostream start to be called on stream which terminated unexpectedly") - t.FailNow() + test.That(t, + "timed out waiting for gostream start to be called on stream which terminated unexpectedly", + test.ShouldBeFalse) } if subscribeRTPCount.Load() == 2 { break @@ -608,95 +559,87 @@ func TestStreamState(t *testing.T) { test.That(t, startCount.Load(), test.ShouldEqual, 1) test.That(t, stopCount.Load(), test.ShouldEqual, 0) - t.Log("when the number of Decrement() calls is less than the number of " + + logger.Info("when the number of Decrement() calls is less than the number of " + "Increment() calls no rtppassthrough.Source or gostream methods are called") - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, s.Decrement(ctx), test.ShouldBeNil) + test.That(t, s.Decrement(), test.ShouldBeNil) + test.That(t, s.Decrement(), test.ShouldBeNil) + time.Sleep(sleepDuration) test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 2) test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 0) test.That(t, startCount.Load(), test.ShouldEqual, 1) test.That(t, stopCount.Load(), test.ShouldEqual, 0) - t.Log("when the number of Decrement() calls is equal to the number of " + + logger.Info("when the number of Decrement() calls is equal to the number of " + "Increment() calls stop is called (as gostream is the data source)") - test.That(t, s.Decrement(ctx), test.ShouldBeNil) + test.That(t, s.Decrement(), test.ShouldBeNil) + time.Sleep(sleepDuration) test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 2) test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 0) test.That(t, startCount.Load(), test.ShouldEqual, 1) test.That(t, stopCount.Load(), test.ShouldEqual, 1) - t.Log("then when the number of Increment() calls exceeds Decrement(), " + + logger.Info("then when the number of Increment() calls exceeds Decrement(), " + "SubscribeRTP is called again followed by Start if it returns an error") - test.That(t, s.Increment(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 3) - test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 0) - test.That(t, startCount.Load(), test.ShouldEqual, 2) - test.That(t, stopCount.Load(), test.ShouldEqual, 1) - - t.Log("calling Decrement() more times than Increment() has a floor of zero and calls Stop()") - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 3) - test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 0) - test.That(t, startCount.Load(), test.ShouldEqual, 2) - test.That(t, stopCount.Load(), test.ShouldEqual, 2) - - // multiple Decrement() calls when the count is already at zero doesn't call any methods - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 3) - test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 0) - test.That(t, startCount.Load(), test.ShouldEqual, 2) - test.That(t, stopCount.Load(), test.ShouldEqual, 2) + test.That(t, s.Increment(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, subscribeRTPCount.Load(), test.ShouldEqual, 3) + test.That(tb, unsubscribeCount.Load(), test.ShouldEqual, 0) + test.That(tb, startCount.Load(), test.ShouldEqual, 2) + test.That(tb, stopCount.Load(), test.ShouldEqual, 1) + }) + + test.That(t, s.Decrement(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, subscribeRTPCount.Load(), test.ShouldEqual, 3) + test.That(tb, unsubscribeCount.Load(), test.ShouldEqual, 0) + test.That(tb, startCount.Load(), test.ShouldEqual, 2) + test.That(tb, stopCount.Load(), test.ShouldEqual, 2) + }) // once the count is at zero , calling Increment() again calls SubscribeRTP followed by Start - test.That(t, s.Increment(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 4) - test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 0) - test.That(t, startCount.Load(), test.ShouldEqual, 3) - test.That(t, stopCount.Load(), test.ShouldEqual, 2) - - t.Log("if while gostream is being used Increment is called and SubscribeRTP succeeds, Stop is called") + test.That(t, s.Increment(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, subscribeRTPCount.Load(), test.ShouldEqual, 4) + test.That(tb, unsubscribeCount.Load(), test.ShouldEqual, 0) + test.That(tb, startCount.Load(), test.ShouldEqual, 3) + test.That(tb, stopCount.Load(), test.ShouldEqual, 2) + }) + + logger.Info("if while gostream is being used Increment is called and SubscribeRTP succeeds, Stop is called") subscribeRTPReturnError.Store(false) - test.That(t, s.Increment(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 5) - test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 0) - test.That(t, startCount.Load(), test.ShouldEqual, 3) - test.That(t, stopCount.Load(), test.ShouldEqual, 3) + test.That(t, s.Increment(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, subscribeRTPCount.Load(), test.ShouldEqual, 5) + test.That(tb, unsubscribeCount.Load(), test.ShouldEqual, 0) + test.That(tb, startCount.Load(), test.ShouldEqual, 3) + test.That(tb, stopCount.Load(), test.ShouldEqual, 3) + }) // calling Decrement() fewer times than Increment() doesn't call any methods - test.That(t, s.Decrement(ctx), test.ShouldBeNil) + test.That(t, s.Decrement(), test.ShouldBeNil) + time.Sleep(sleepDuration) test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 5) test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 0) test.That(t, startCount.Load(), test.ShouldEqual, 3) test.That(t, stopCount.Load(), test.ShouldEqual, 3) - t.Log("calling Decrement() more times than Increment() has a floor of zero and calls Unsubscribe()") - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 5) - test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 1) - test.That(t, startCount.Load(), test.ShouldEqual, 3) - test.That(t, stopCount.Load(), test.ShouldEqual, 3) - - // multiple Decrement() calls when the count is already at zero doesn't call any methods - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 5) - test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 1) - test.That(t, startCount.Load(), test.ShouldEqual, 3) - test.That(t, stopCount.Load(), test.ShouldEqual, 3) + test.That(t, s.Decrement(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, subscribeRTPCount.Load(), test.ShouldEqual, 5) + test.That(tb, unsubscribeCount.Load(), test.ShouldEqual, 1) + test.That(tb, startCount.Load(), test.ShouldEqual, 3) + test.That(tb, stopCount.Load(), test.ShouldEqual, 3) + }) - t.Log("if while rtp_passthrough is being used the the subscription " + + logger.Info("if while rtp_passthrough is being used the the subscription " + "terminates & afterwards rtp_passthrough is no longer supported, Start is called") - test.That(t, s.Increment(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 6) - test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 1) - test.That(t, startCount.Load(), test.ShouldEqual, 3) - test.That(t, stopCount.Load(), test.ShouldEqual, 3) + test.That(t, s.Increment(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, subscribeRTPCount.Load(), test.ShouldEqual, 6) + test.That(tb, unsubscribeCount.Load(), test.ShouldEqual, 1) + test.That(tb, startCount.Load(), test.ShouldEqual, 3) + test.That(tb, stopCount.Load(), test.ShouldEqual, 3) + }) subscribeRTPReturnError.Store(true) @@ -709,8 +652,9 @@ func TestStreamState(t *testing.T) { defer timeoutFn() for { if timeoutCtx.Err() != nil { - t.Log("timed out waiting for Start() to be called after an in progress sub terminated unexpectedly") - t.FailNow() + test.That(t, + "timed out waiting for Start() to be called after an in progress sub terminated unexpectedly", + test.ShouldBeFalse) } if startCount.Load() == 4 { break @@ -723,14 +667,16 @@ func TestStreamState(t *testing.T) { test.That(t, stopCount.Load(), test.ShouldEqual, 3) // Decrement() calls Stop() as gostream is the data source - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 7) - test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 1) - test.That(t, startCount.Load(), test.ShouldEqual, 4) - test.That(t, stopCount.Load(), test.ShouldEqual, 4) + test.That(t, s.Decrement(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, subscribeRTPCount.Load(), test.ShouldEqual, 7) + test.That(tb, unsubscribeCount.Load(), test.ShouldEqual, 1) + test.That(tb, startCount.Load(), test.ShouldEqual, 4) + test.That(tb, stopCount.Load(), test.ShouldEqual, 4) + }) }) - t.Run("when the camera does not implement rtppassthrough.Souce", func(t *testing.T) { + t.Run("when the camera does not implement rtppassthrough.Source", func(t *testing.T) { var startCount atomic.Int64 var stopCount atomic.Int64 streamMock := &mockStream{ @@ -743,81 +689,77 @@ func TestStreamState(t *testing.T) { stopCount.Add(1) }, writeRTPFunc: func(pkt *rtp.Packet) error { - t.Log("should not happen") - t.FailNow() + test.That(t, + "should not happen", + test.ShouldBeFalse) return nil }, } robot := mockRobot(nil) s := state.New(streamMock, robot, logger) - defer func() { utils.UncheckedError(s.Close()) }() - s.Init() - - t.Log("the first Increment() -> Start()") - test.That(t, s.Increment(ctx), test.ShouldBeNil) + defer func() { + utils.UncheckedError(s.Close()) + }() + logger.Info("the first Increment() -> Start()") + test.That(t, s.Increment(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, startCount.Load(), test.ShouldEqual, 1) + test.That(tb, stopCount.Load(), test.ShouldEqual, 0) + }) + + logger.Info("subsequent Increment() all calls don't call any other gostream methods") + test.That(t, s.Increment(), test.ShouldBeNil) + test.That(t, s.Increment(), test.ShouldBeNil) + time.Sleep(sleepDuration) test.That(t, startCount.Load(), test.ShouldEqual, 1) test.That(t, stopCount.Load(), test.ShouldEqual, 0) - t.Log("subsequent Increment() all calls don't call any other gostream methods") - test.That(t, s.Increment(ctx), test.ShouldBeNil) - test.That(t, s.Increment(ctx), test.ShouldBeNil) + logger.Info("as long as the number of Decrement() calls is less than the number of Increment() calls, no gostream methods are called") + test.That(t, s.Decrement(), test.ShouldBeNil) + test.That(t, s.Decrement(), test.ShouldBeNil) + time.Sleep(sleepDuration) test.That(t, startCount.Load(), test.ShouldEqual, 1) test.That(t, stopCount.Load(), test.ShouldEqual, 0) - t.Log("as long as the number of Decrement() calls is less than the number of Increment() calls, no gostream methods are called") - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, startCount.Load(), test.ShouldEqual, 1) - test.That(t, stopCount.Load(), test.ShouldEqual, 0) - - t.Log("when the number of Decrement() calls is equal to the number of Increment() calls stop is called") - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, startCount.Load(), test.ShouldEqual, 1) - test.That(t, stopCount.Load(), test.ShouldEqual, 1) - test.That(t, s.Increment(ctx), test.ShouldBeNil) - - t.Log("then when the number of Increment() calls exceeds Decrement(), Start is called again") - test.That(t, startCount.Load(), test.ShouldEqual, 2) - test.That(t, stopCount.Load(), test.ShouldEqual, 1) - - t.Log("calling Decrement() more times than Increment() has a floor of zero") - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, startCount.Load(), test.ShouldEqual, 2) - test.That(t, stopCount.Load(), test.ShouldEqual, 2) - - // multiple Decrement() calls when the count is already at zero doesn't call any methods - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, startCount.Load(), test.ShouldEqual, 2) - test.That(t, stopCount.Load(), test.ShouldEqual, 2) + logger.Info("when the number of Decrement() calls is equal to the number of Increment() calls stop is called") + test.That(t, s.Decrement(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, startCount.Load(), test.ShouldEqual, 1) + test.That(tb, stopCount.Load(), test.ShouldEqual, 1) + }) + + logger.Info("then when the number of Increment() calls exceeds Decrement(), Start is called again") + test.That(t, s.Increment(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, startCount.Load(), test.ShouldEqual, 2) + test.That(tb, stopCount.Load(), test.ShouldEqual, 1) + }) + + test.That(t, s.Decrement(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, startCount.Load(), test.ShouldEqual, 2) + test.That(tb, stopCount.Load(), test.ShouldEqual, 2) + }) // once the count is at zero , calling Increment() again calls start - test.That(t, s.Increment(ctx), test.ShouldBeNil) - test.That(t, startCount.Load(), test.ShouldEqual, 3) - test.That(t, stopCount.Load(), test.ShouldEqual, 2) + test.That(t, s.Increment(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, startCount.Load(), test.ShouldEqual, 3) + test.That(tb, stopCount.Load(), test.ShouldEqual, 2) + }) // set count back to zero - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, startCount.Load(), test.ShouldEqual, 3) - test.That(t, stopCount.Load(), test.ShouldEqual, 3) - - t.Log("calling Increment() with a cancelled context returns an error & does not call any gostream methods") - canceledCtx, cancelFn := context.WithCancel(context.Background()) - cancelFn() - test.That(t, s.Increment(canceledCtx), test.ShouldBeError, context.Canceled) - test.That(t, startCount.Load(), test.ShouldEqual, 3) - test.That(t, stopCount.Load(), test.ShouldEqual, 3) + test.That(t, s.Decrement(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, startCount.Load(), test.ShouldEqual, 3) + test.That(tb, stopCount.Load(), test.ShouldEqual, 3) + }) // make it so that non cancelled Decrement() would call stop to confirm that does not happen when context is cancelled - test.That(t, s.Increment(ctx), test.ShouldBeNil) - test.That(t, startCount.Load(), test.ShouldEqual, 4) - test.That(t, stopCount.Load(), test.ShouldEqual, 3) - - t.Log("calling Decrement() with a cancelled context returns an error & does not call any gostream methods") - test.That(t, s.Decrement(canceledCtx), test.ShouldBeError, context.Canceled) - test.That(t, startCount.Load(), test.ShouldEqual, 4) - test.That(t, stopCount.Load(), test.ShouldEqual, 3) + test.That(t, s.Increment(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, startCount.Load(), test.ShouldEqual, 4) + test.That(tb, stopCount.Load(), test.ShouldEqual, 3) + }) }) } diff --git a/robot/web/stream/stream.go b/robot/web/stream/stream.go index da0f08ac469..7966dc78b57 100644 --- a/robot/web/stream/stream.go +++ b/robot/web/stream/stream.go @@ -6,7 +6,6 @@ package webstream import ( "context" "errors" - "math" "time" "go.viam.com/utils" @@ -23,7 +22,7 @@ func StreamVideoSource( backoffOpts *BackoffTuningOptions, logger logging.Logger, ) error { - return gostream.StreamVideoSourceWithErrorHandler(ctx, source, stream, backoffOpts.getErrorThrottledHandler(logger), logger) + return gostream.StreamVideoSourceWithErrorHandler(ctx, source, stream, backoffOpts.getErrorThrottledHandler(logger, stream.Name()), logger) } // StreamAudioSource starts a stream from an audio source with a throttled error handler. @@ -34,7 +33,7 @@ func StreamAudioSource( backoffOpts *BackoffTuningOptions, logger logging.Logger, ) error { - return gostream.StreamAudioSourceWithErrorHandler(ctx, source, stream, backoffOpts.getErrorThrottledHandler(logger), logger) + return gostream.StreamAudioSourceWithErrorHandler(ctx, source, stream, backoffOpts.getErrorThrottledHandler(logger, "audio"), logger) } // BackoffTuningOptions represents a set of parameters for determining exponential @@ -55,18 +54,25 @@ type BackoffTuningOptions struct { Cooldown time.Duration } +// Dan: This fixes the backoff bugs where we'd overflow the sleep to a negative value. But this no +// longer obeys the input tuning options. I'm considering that deprecated and to be removed. +var backoffSleeps = []time.Duration{500 * time.Millisecond, time.Second, 2 * time.Second, 5 * time.Second} + // GetSleepTimeFromErrorCount returns a sleep time from an error count. func (opts *BackoffTuningOptions) GetSleepTimeFromErrorCount(errorCount int) time.Duration { if errorCount < 1 || opts == nil { return 0 } - multiplier := math.Pow(2, float64(errorCount-1)) - uncappedSleep := opts.BaseSleep * time.Duration(multiplier) - sleep := math.Min(float64(uncappedSleep), float64(opts.MaxSleep)) - return time.Duration(sleep) + + errorCount-- + if errorCount >= len(backoffSleeps) { + return backoffSleeps[len(backoffSleeps)-1] + } + + return backoffSleeps[errorCount] } -func (opts *BackoffTuningOptions) getErrorThrottledHandler(logger logging.Logger) func(context.Context, error) { +func (opts *BackoffTuningOptions) getErrorThrottledHandler(logger logging.Logger, streamName string) func(context.Context, error) { var prevErr error var errorCount int lastErrTime := time.Now() @@ -78,15 +84,19 @@ func (opts *BackoffTuningOptions) getErrorThrottledHandler(logger logging.Logger } lastErrTime = now - if errors.Is(prevErr, err) { + switch { + case errors.Is(prevErr, err): + errorCount++ + case prevErr != nil && prevErr.Error() == err.Error(): errorCount++ - } else { + default: prevErr = err errorCount = 1 } sleep := opts.GetSleepTimeFromErrorCount(errorCount) - logger.Errorw("error getting media", "error", err, "count", errorCount, "sleep", sleep) + logger.Errorw("error getting media", "streamName", streamName, "error", err, "count", errorCount, "sleep", sleep) + utils.SelectContextOrWait(ctx, sleep) } } diff --git a/robot/web/stream/stream_test.go b/robot/web/stream/stream_test.go index d09eea7eb38..073e955c39e 100644 --- a/robot/web/stream/stream_test.go +++ b/robot/web/stream/stream_test.go @@ -4,17 +4,9 @@ package webstream_test import ( "context" - "errors" - "image" "strings" - "sync" "testing" - "time" - "github.com/pion/mediadevices/pkg/prop" - "github.com/pion/mediadevices/pkg/wave" - "github.com/pion/rtp" - "github.com/viamrobotics/webrtc/v3" streampb "go.viam.com/api/stream/v1" "go.viam.com/test" "go.viam.com/utils/rpc" @@ -32,117 +24,9 @@ import ( "go.viam.com/rdk/robot" robotimpl "go.viam.com/rdk/robot/impl" "go.viam.com/rdk/robot/web" - webstream "go.viam.com/rdk/robot/web/stream" "go.viam.com/rdk/testutils/robottestutils" ) -var errImageRetrieval = errors.New("image retrieval failed") - -type mockErrorVideoSource struct { - callsLeft int - wg sync.WaitGroup -} - -func newMockErrorVideoReader(expectedCalls int) *mockErrorVideoSource { - mock := &mockErrorVideoSource{callsLeft: expectedCalls} - mock.wg.Add(expectedCalls) - return mock -} - -func (videoSource *mockErrorVideoSource) Read(ctx context.Context) (image.Image, func(), error) { - if videoSource.callsLeft > 0 { - videoSource.wg.Done() - videoSource.callsLeft-- - } - return nil, nil, errImageRetrieval -} - -func (videoSource *mockErrorVideoSource) Close(ctx context.Context) error { - return nil -} - -type mockStream struct { - name string - streamingReadyFunc func() <-chan struct{} - inputFramesFunc func() (chan<- gostream.MediaReleasePair[image.Image], error) -} - -func (mS *mockStream) StreamingReady() (<-chan struct{}, context.Context) { - return mS.streamingReadyFunc(), context.Background() -} - -func (mS *mockStream) InputVideoFrames(props prop.Video) (chan<- gostream.MediaReleasePair[image.Image], error) { - return mS.inputFramesFunc() -} - -func (mS *mockStream) InputAudioChunks(props prop.Audio) (chan<- gostream.MediaReleasePair[wave.Audio], error) { - return make(chan gostream.MediaReleasePair[wave.Audio]), nil -} - -func (mS *mockStream) Name() string { - return mS.name -} - -func (mS *mockStream) Start() { -} - -func (mS *mockStream) Stop() { -} - -func (mS *mockStream) WriteRTP(*rtp.Packet) error { - return nil -} - -func (mS *mockStream) VideoTrackLocal() (webrtc.TrackLocal, bool) { - return nil, false -} - -func (mS *mockStream) AudioTrackLocal() (webrtc.TrackLocal, bool) { - return nil, false -} - -func TestStreamSourceErrorBackoff(t *testing.T) { - logger := logging.NewTestLogger(t) - ctx, cancel := context.WithCancel(context.Background()) - - backoffOpts := &webstream.BackoffTuningOptions{ - BaseSleep: 50 * time.Microsecond, - MaxSleep: 250 * time.Millisecond, - Cooldown: time.Second, - } - calls := 25 - videoReader := newMockErrorVideoReader(calls) - videoSrc := gostream.NewVideoSource(videoReader, prop.Video{}) - defer func() { - test.That(t, videoSrc.Close(context.Background()), test.ShouldBeNil) - }() - - totalExpectedSleep := int64(0) - // Note that we do not add the expected sleep duration for the last error since the - // streaming context will be cancelled during that error. - for i := 1; i < calls; i++ { - totalExpectedSleep += backoffOpts.GetSleepTimeFromErrorCount(i).Nanoseconds() - } - str := &mockStream{} - readyChan := make(chan struct{}) - inputChan := make(chan gostream.MediaReleasePair[image.Image]) - str.streamingReadyFunc = func() <-chan struct{} { - return readyChan - } - str.inputFramesFunc = func() (chan<- gostream.MediaReleasePair[image.Image], error) { - return inputChan, nil - } - - go webstream.StreamVideoSource(ctx, videoSrc, str, backoffOpts, logger) - start := time.Now() - readyChan <- struct{}{} - videoReader.wg.Wait() - cancel() - - duration := time.Since(start).Nanoseconds() - test.That(t, duration, test.ShouldBeGreaterThanOrEqualTo, totalExpectedSleep) -} - // setupRealRobot creates a robot from the input config and starts a WebRTC server with video // streaming capabilities. // diff --git a/robot/web/web_c.go b/robot/web/web_c.go index 6c393629e8f..26b25395ca5 100644 --- a/robot/web/web_c.go +++ b/robot/web/web_c.go @@ -10,7 +10,6 @@ import ( "runtime" "slices" "sync" - "time" "github.com/pkg/errors" streampb "go.viam.com/api/stream/v1" @@ -160,7 +159,7 @@ func (svc *webService) makeStreamServer(ctx context.Context) (*StreamServer, err if len(svc.videoSources) != 0 || len(svc.audioSources) != 0 { svc.logger.Debug("not starting streams due to no stream config being set") } - noopServer, err := webstream.NewServer(streams, svc.r, logging.GetOrNewLogger("rdk.networking")) + noopServer, err := webstream.NewServer(streams, svc.r, svc.logger.Sublogger("stream")) return &StreamServer{noopServer, false}, err } @@ -215,7 +214,7 @@ func (svc *webService) makeStreamServer(ctx context.Context) (*StreamServer, err streamTypes = append(streamTypes, false) } - streamServer, err := webstream.NewServer(streams, svc.r, logging.GetOrNewLogger("rdk.networking")) + streamServer, err := webstream.NewServer(streams, svc.r, svc.logger.Sublogger("stream")) if err != nil { return nil, err } @@ -237,12 +236,7 @@ func (svc *webService) startStream(streamFunc func(opts *webstream.BackoffTuning utils.PanicCapturingGo(func() { defer svc.webWorkers.Done() close(waitCh) - opts := &webstream.BackoffTuningOptions{ - BaseSleep: 50 * time.Microsecond, - MaxSleep: 2 * time.Second, - Cooldown: 5 * time.Second, - } - if err := streamFunc(opts); err != nil { + if err := streamFunc(&webstream.BackoffTuningOptions{}); err != nil { if utils.FilterOutError(err, context.Canceled) != nil { svc.logger.Errorw("error streaming", "error", err) } diff --git a/testutils/resource_utils.go b/testutils/resource_utils.go index eb8868ce8ed..9c87f9748e5 100644 --- a/testutils/resource_utils.go +++ b/testutils/resource_utils.go @@ -83,6 +83,20 @@ func VerifySameResourceStatuses(tb testing.TB, actual, expected []resource.Statu test.That(tb, sortedActual, test.ShouldResemble, sortedExpected) } +// FilterByStatus takes a slice of [resource.Status] and a [resource.NodeState] and +// returns a slice of [resource.Status] that are in the given [resource.NodeState]. +func FilterByStatus(tb testing.TB, resourceStatuses []resource.Status, state resource.NodeState) []resource.Status { + tb.Helper() + + var result []resource.Status + for _, rs := range resourceStatuses { + if rs.State == state { + result = append(result, rs) + } + } + return result +} + func newSortedResourceStatuses(resourceStatuses []resource.Status) []resource.Status { sorted := make([]resource.Status, len(resourceStatuses)) copy(sorted, resourceStatuses) diff --git a/utils/stoppable_workers.go b/utils/stoppable_workers.go deleted file mode 100644 index de6c626380e..00000000000 --- a/utils/stoppable_workers.go +++ /dev/null @@ -1,77 +0,0 @@ -package utils - -import ( - "context" - "sync" - - goutils "go.viam.com/utils" -) - -// TODO: When this struct is widely used and feature complete, move this to goutils instead of -// here. Until then, we cannot use this in any package imported by utils (e.g., the logging -// package) without introducing a circular import dependency. - -// StoppableWorkers is a collection of goroutines that can be stopped at a later time. -type StoppableWorkers interface { - AddWorkers(...func(context.Context)) - Stop() - Context() context.Context -} - -// stoppableWorkersImpl is the implementation of StoppableWorkers. The linter will complain if you -// try to make a copy of something that contains a sync.WaitGroup (and returning a value at the end -// of NewStoppableWorkers() would make a copy of it), so we do everything through the -// StoppableWorkers interface to avoid making copies (since interfaces do everything by pointer). -type stoppableWorkersImpl struct { - mu sync.Mutex - cancelCtx context.Context - cancelFunc func() - activeBackgroundWorkers sync.WaitGroup -} - -// NewStoppableWorkers runs the functions in separate goroutines. They can be stopped later. -func NewStoppableWorkers(funcs ...func(context.Context)) StoppableWorkers { - cancelCtx, cancelFunc := context.WithCancel(context.Background()) - workers := &stoppableWorkersImpl{cancelCtx: cancelCtx, cancelFunc: cancelFunc} - workers.AddWorkers(funcs...) - return workers -} - -// AddWorkers starts up additional goroutines for each function passed in. If you call this after -// calling Stop(), it will return immediately without starting any new goroutines. -func (sw *stoppableWorkersImpl) AddWorkers(funcs ...func(context.Context)) { - sw.mu.Lock() - defer sw.mu.Unlock() - - if sw.cancelCtx.Err() != nil { // We've already stopped everything. - return - } - - sw.activeBackgroundWorkers.Add(len(funcs)) - for _, f := range funcs { - // In Go 1.21 and earlier, variables created in a loop were reused from one iteration to - // the next. Make a "fresh" copy of it here so that, if we're on to the next iteration of - // the loop before the goroutine starts up, it starts this function instead of the next - // one. For details, see https://go.dev/blog/loopvar-preview - f := f - goutils.PanicCapturingGo(func() { - defer sw.activeBackgroundWorkers.Done() - f(sw.cancelCtx) - }) - } -} - -// Stop shuts down all the goroutines we started up. -func (sw *stoppableWorkersImpl) Stop() { - sw.mu.Lock() - defer sw.mu.Unlock() - - sw.cancelFunc() - sw.activeBackgroundWorkers.Wait() -} - -// Context gets the context the workers are checking on. Using this function is expected to be -// rare: usually you shouldn't need to interact with the context directly. -func (sw *stoppableWorkersImpl) Context() context.Context { - return sw.cancelCtx -} diff --git a/utils/value.go b/utils/value.go index be5a5343391..d8adc54f54c 100644 --- a/utils/value.go +++ b/utils/value.go @@ -1,10 +1,10 @@ package utils import ( - "flag" "math/rand" "os" "strings" + "testing" ) // AssertType attempts to assert that the given interface argument is @@ -46,9 +46,9 @@ type Rand interface { } // Testing returns true when you are running in test suite. +// Deprecated: this is in the standard library now. func Testing() bool { - // TODO switch to official testing.Testing method when we are on go 1.21 - return flag.Lookup("test.v") != nil + return testing.Testing() } // randWrapper is a pass-through to the shared math/rand functions. @@ -61,7 +61,7 @@ func (randWrapper) Float64() float64 { // SafeTestingRand returns a wrapper around the shared math/rand source in prod, // and a deterministic rand.Rand seeded with 0 in test. func SafeTestingRand() Rand { - if Testing() { + if testing.Testing() { return rand.New(rand.NewSource(0)) //nolint:gosec } return randWrapper{} diff --git a/vision/segmentation/color_objects_test.go b/vision/segmentation/color_objects_test.go index aeb7be47d3c..e42bf3188ce 100644 --- a/vision/segmentation/color_objects_test.go +++ b/vision/segmentation/color_objects_test.go @@ -8,7 +8,7 @@ import ( "go.viam.com/utils/artifact" "go.viam.com/rdk/components/camera" - "go.viam.com/rdk/components/camera/videosource" + "go.viam.com/rdk/components/camera/fake" "go.viam.com/rdk/rimage" "go.viam.com/rdk/rimage/transform" "go.viam.com/rdk/utils" @@ -24,7 +24,7 @@ func TestColorObjects(t *testing.T) { test.That(t, err, test.ShouldBeNil) params, err := transform.NewDepthColorIntrinsicsExtrinsicsFromJSONFile(intel515ParamsPath) test.That(t, err, test.ShouldBeNil) - c := &videosource.StaticSource{ColorImg: img, DepthImg: dm, Proj: ¶ms.ColorCamera} + c := &fake.StaticSource{ColorImg: img, DepthImg: dm, Proj: ¶ms.ColorCamera} src, err := camera.NewVideoSourceFromReader( context.Background(), c, diff --git a/web/frontend/package-lock.json b/web/frontend/package-lock.json index 3bfdf12a9a8..1283102e03c 100644 --- a/web/frontend/package-lock.json +++ b/web/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@viamrobotics/remote-control", - "version": "2.21.2", + "version": "2.22.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@viamrobotics/remote-control", - "version": "2.21.2", + "version": "2.22.0", "license": "Apache-2.0", "devDependencies": { "@improbable-eng/grpc-web": "0.15.0", @@ -25,7 +25,7 @@ "@viamrobotics/prime-blocks": "^0.1.3", "@viamrobotics/prime-core": "^0.0.87", "@viamrobotics/rpc": "0.2.3", - "@viamrobotics/sdk": "0.22.0", + "@viamrobotics/sdk": "0.23.0", "@viamrobotics/three": "^0.0.3", "@viamrobotics/typescript-config": "^0.1.0", "cypress": "12.17.3", @@ -60,7 +60,7 @@ "@improbable-eng/grpc-web": ">=0.15", "@viamrobotics/prime": ">=0.5", "@viamrobotics/rpc": ">=0.2", - "@viamrobotics/sdk": "0.22.0", + "@viamrobotics/sdk": "0.23.0", "google-protobuf": ">=3", "maplibre-gl": ">=4", "tailwindcss": ">=3.3", @@ -1546,9 +1546,9 @@ } }, "node_modules/@viamrobotics/sdk": { - "version": "0.22.0", - "resolved": "https://registry.npmjs.org/@viamrobotics/sdk/-/sdk-0.22.0.tgz", - "integrity": "sha512-AAobJxxQOZiiD5RxN/NtiLpg1SEk5vIO/wKhRxpH0J2uVu3p6llPu714nV/qMOE3A1uTYWw6ozCYnMo6an7Hvg==", + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/@viamrobotics/sdk/-/sdk-0.23.0.tgz", + "integrity": "sha512-3d50zGllR7ydzpFCe3e5Apih1PxNHOUK1iCnDaDFU5yalk0QzvPI4VCo/SWFqqoFb5CXcL+fCmr/FHFZDVzMag==", "dev": true, "dependencies": { "@viamrobotics/rpc": "^0.2.5", diff --git a/web/frontend/package.json b/web/frontend/package.json index d4e8758706b..b9cd97081f3 100644 --- a/web/frontend/package.json +++ b/web/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@viamrobotics/remote-control", - "version": "2.21.2", + "version": "2.22.0", "license": "Apache-2.0", "type": "module", "scripts": { @@ -37,7 +37,7 @@ "@improbable-eng/grpc-web": ">=0.15", "@viamrobotics/prime": ">=0.5", "@viamrobotics/rpc": ">=0.2", - "@viamrobotics/sdk": "0.22.0", + "@viamrobotics/sdk": "0.23.0", "google-protobuf": ">=3", "maplibre-gl": ">=4", "tailwindcss": ">=3.3", @@ -60,7 +60,7 @@ "@viamrobotics/prime-blocks": "^0.1.3", "@viamrobotics/prime-core": "^0.0.87", "@viamrobotics/rpc": "0.2.3", - "@viamrobotics/sdk": "0.22.0", + "@viamrobotics/sdk": "0.23.0", "@viamrobotics/three": "^0.0.3", "@viamrobotics/typescript-config": "^0.1.0", "cypress": "12.17.3", diff --git a/web/server/entrypoint.go b/web/server/entrypoint.go index 93709f277d0..a3fcb88c395 100644 --- a/web/server/entrypoint.go +++ b/web/server/entrypoint.go @@ -55,6 +55,21 @@ type robotServer struct { logger logging.Logger } +func logVersion(logger logging.Logger) { + var versionFields []interface{} + if config.Version != "" { + versionFields = append(versionFields, "version", config.Version) + } + if config.GitRevision != "" { + versionFields = append(versionFields, "git_rev", config.GitRevision) + } + if len(versionFields) != 0 { + logger.Infow("Viam RDK", versionFields...) + } else { + logger.Info("Viam RDK built from source; version unknown") + } +} + // RunServer is an entry point to starting the web server that can be called by main in a code // sample or otherwise be used to initialize the server. func RunServer(ctx context.Context, args []string, _ logging.Logger) (err error) { @@ -85,24 +100,21 @@ func RunServer(ctx context.Context, args []string, _ logging.Logger) (err error) config.InitLoggingSettings(logger, argsParsed.Debug) - // Always log the version, return early if the '-version' flag was provided - // fmt.Println would be better but fails linting. Good enough. - var versionFields []interface{} - if config.Version != "" { - versionFields = append(versionFields, "version", config.Version) - } - if config.GitRevision != "" { - versionFields = append(versionFields, "git_rev", config.GitRevision) - } - if len(versionFields) != 0 { - logger.Infow("Viam RDK", versionFields...) - } else { - logger.Info("Viam RDK built from source; version unknown") - } if argsParsed.Version { + // log version here and return if version flag. + logVersion(logger) return } + // log version locally if server fails and exits while attempting to start up + var versionLogged bool + defer func() { + if !versionLogged { + logger.CInfo(ctx, "error starting viam-server, logging version and exiting") + logVersion(logger) + } + }() + if argsParsed.ConfigFile == "" { logger.Error("please specify a config file through the -config parameter.") return @@ -155,6 +167,9 @@ func RunServer(ctx context.Context, args []string, _ logging.Logger) (err error) logger.AddAppender(netAppender) } + // log version after netlogger is initialized so it's captured in cloud machine logs. + logVersion(logger) + versionLogged = true server := robotServer{ logger: logger,