From 79346309c27232e44c460ff3c31c1cc765be19f0 Mon Sep 17 00:00:00 2001 From: sean yu <55464069+hexbabe@users.noreply.github.com> Date: Fri, 30 Aug 2024 12:16:02 -0400 Subject: [PATCH 01/26] Patch gostream release bug for modules (#4330) --- gostream/media.go | 4 ++++ gostream/media_test.go | 22 ++++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/gostream/media.go b/gostream/media.go index d5c33cc4a25..d5e34d7dc4e 100644 --- a/gostream/media.go +++ b/gostream/media.go @@ -360,6 +360,10 @@ func (pc *producerConsumer[T, U]) stop() { pc.consumerCond.L.Unlock() pc.activeBackgroundWorkers.Wait() + pc.currentMu.Lock() + defer pc.currentMu.Unlock() + pc.current = nil + // reset cancelCtx, cancel := context.WithCancel(WithMIMETypeHint(pc.rootCancelCtx, pc.mimeType)) pc.cancelCtxMu.Lock() diff --git a/gostream/media_test.go b/gostream/media_test.go index fe781543285..7b6c45fb5e9 100644 --- a/gostream/media_test.go +++ b/gostream/media_test.go @@ -191,3 +191,25 @@ func TestStreamMultipleConsumers(t *testing.T) { test.That(t, wrappedImg.released.Load(), test.ShouldBeTrue) } } + +func TestStreamWithoutNext(t *testing.T) { + colors := []*WrappedImage{createWrappedImage(t, rimage.Red)} + + imgSource := &imageSource{WrappedImages: colors} + videoSrc := NewVideoSource(imgSource, prop.Video{}) + + // Start stream + stream, err := videoSrc.Stream(context.Background()) + test.That(t, err, test.ShouldBeNil) + // Get one frame + _, release, err := stream.Next(context.Background()) + test.That(t, err, test.ShouldBeNil) + release() + // Close stream + stream.Close(context.Background()) + + // Spin up stream and close without calling Next + stream, err = videoSrc.Stream(context.Background()) + test.That(t, err, test.ShouldBeNil) + stream.Close(context.Background()) +} From 1f0a351b7b73c2392a1ff2fdbc550b160d3bfc3d Mon Sep 17 00:00:00 2001 From: Steve Briskin Date: Fri, 30 Aug 2024 13:34:03 -0400 Subject: [PATCH 02/26] Address cors and image dependabot alerts (#4336) --- go.mod | 14 +++++++------- go.sum | 28 ++++++++++++++-------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index fcca62f5252..b867cd657a9 100644 --- a/go.mod +++ b/go.mod @@ -44,7 +44,7 @@ require ( github.com/golang/protobuf v1.5.3 github.com/golangci/golangci-lint v1.54.0 github.com/google/flatbuffers v2.0.6+incompatible - github.com/google/go-cmp v0.5.9 + github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.6.0 github.com/gotesttools/gotestfmt/v2 v2.4.1 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 @@ -70,7 +70,7 @@ require ( github.com/pion/mediadevices v0.6.4 github.com/pion/rtp v1.8.7 github.com/rhysd/actionlint v1.6.24 - github.com/rs/cors v1.9.0 + github.com/rs/cors v1.11.1 github.com/sergi/go-diff v1.3.1 github.com/u2takey/ffmpeg-go v0.4.1 github.com/urfave/cli/v2 v2.10.3 @@ -87,13 +87,13 @@ require ( go.viam.com/test v1.1.1-0.20220913152726-5da9916c08a2 go.viam.com/utils v0.1.96 goji.io v2.0.2+incompatible - golang.org/x/image v0.15.0 + golang.org/x/image v0.19.0 golang.org/x/mobile v0.0.0-20240112133503-c713f31d574b - golang.org/x/sync v0.6.0 + golang.org/x/sync v0.8.0 golang.org/x/sys v0.20.0 golang.org/x/term v0.20.0 golang.org/x/time v0.3.0 - golang.org/x/tools v0.17.0 + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d gonum.org/v1/gonum v0.12.0 gonum.org/v1/plot v0.12.0 google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 @@ -370,10 +370,10 @@ require ( go4.org/unsafe/assume-no-moving-gc v0.0.0-20230525183740-e7c30c78aeb2 // indirect golang.org/x/crypto v0.23.0 // indirect golang.org/x/exp/typeparams v0.0.0-20230224173230-c95f2b4c22f2 // indirect - golang.org/x/mod v0.14.0 // indirect + golang.org/x/mod v0.17.0 // indirect golang.org/x/net v0.25.0 // indirect golang.org/x/oauth2 v0.10.0 // indirect - golang.org/x/text v0.15.0 // indirect + golang.org/x/text v0.17.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/api v0.126.0 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/go.sum b/go.sum index 643e63beabc..dd541719175 100644 --- a/go.sum +++ b/go.sum @@ -605,8 +605,8 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= @@ -1242,8 +1242,8 @@ github.com/rogpeppe/go-internal v1.6.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= -github.com/rs/cors v1.9.0 h1:l9HGsTsHJcvW14Nk7J9KFz8bzeAWXn3CG6bgt7LsrAE= -github.com/rs/cors v1.9.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= +github.com/rs/cors v1.11.1 h1:eU3gRzXLRK57F5rKMGMZURNdIG4EoAmX8k94r9wXWHA= +github.com/rs/cors v1.11.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= @@ -1602,8 +1602,8 @@ golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+o golang.org/x/image v0.0.0-20200927104501-e162460cd6b5/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= -golang.org/x/image v0.15.0 h1:kOELfmgrmJlw4Cdb7g/QGuB3CvDrXbqEIww/pNtNBm8= -golang.org/x/image v0.15.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE= +golang.org/x/image v0.19.0 h1:D9FX4QWkLfkeqaC62SonffIIuYdOk/UE2XKUBgRIBIQ= +golang.org/x/image v0.19.0/go.mod h1:y0zrRqlQRWQ5PXaYCOMLTW2fpsxZ8Qh9I/ohnInJEys= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1634,8 +1634,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= -golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1731,8 +1731,8 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1866,8 +1866,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1988,8 +1988,8 @@ golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/tools v0.5.0/go.mod h1:N+Kgy78s5I24c24dU8OfWNEotWjutIs8SnJvn5IDq+k= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc= -golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 593a6db90a7b88f14802d0e43a47b6041a0dfe8e Mon Sep 17 00:00:00 2001 From: Julie Krasnick <99275379+jckras@users.noreply.github.com> Date: Fri, 30 Aug 2024 14:51:22 -0400 Subject: [PATCH 03/26] RSDK-5528: Remove mmal from rdk/gostream (#4334) --- gostream/codec/mmal/encoder.go | 68 ---------------------------------- gostream/codec/mmal/utils.go | 32 ---------------- 2 files changed, 100 deletions(-) delete mode 100644 gostream/codec/mmal/encoder.go delete mode 100644 gostream/codec/mmal/utils.go diff --git a/gostream/codec/mmal/encoder.go b/gostream/codec/mmal/encoder.go deleted file mode 100644 index 0993a8b40ca..00000000000 --- a/gostream/codec/mmal/encoder.go +++ /dev/null @@ -1,68 +0,0 @@ -//go:build mmal - -// Package mmal contains the mmal video codec. -package mmal - -import ( - "context" - "image" - - ourcodec "go.viam.com/rdk/gostream/codec" - "go.viam.com/rdk/logging" - - "github.com/pion/mediadevices/pkg/codec" - "github.com/pion/mediadevices/pkg/codec/mmal" - "github.com/pion/mediadevices/pkg/prop" -) - -type encoder struct { - codec codec.ReadCloser - img image.Image - logger logging.Logger -} - -// Gives suitable results. Probably want to make this configurable this in the future. -const bitrate = 3_200_000 - -// NewEncoder returns an MMAL encoder that can encode images of the given width and height. It will -// also ensure that it produces key frames at the given interval. -func NewEncoder(width, height, keyFrameInterval int, logger logging.Logger) (ourcodec.VideoEncoder, error) { - enc := &encoder{logger: logger} - - var builder codec.VideoEncoderBuilder - params, err := mmal.NewParams() - if err != nil { - return nil, err - } - builder = ¶ms - params.BitRate = bitrate - params.KeyFrameInterval = keyFrameInterval - - codec, err := builder.BuildVideoEncoder(enc, prop.Media{ - Video: prop.Video{ - Width: width, - Height: height, - }, - }) - if err != nil { - return nil, err - } - enc.codec = codec - - return enc, nil -} - -// Read returns an image for codec to process. -func (v *encoder) Read() (img image.Image, release func(), err error) { - return v.img, nil, nil -} - -// Encode asks the codec to process the given image. -func (v *encoder) Encode(_ context.Context, img image.Image) ([]byte, error) { - v.img = img - data, release, err := v.codec.Read() - dataCopy := make([]byte, len(data)) - copy(dataCopy, data) - release() - return dataCopy, err -} diff --git a/gostream/codec/mmal/utils.go b/gostream/codec/mmal/utils.go deleted file mode 100644 index 5a328e4685d..00000000000 --- a/gostream/codec/mmal/utils.go +++ /dev/null @@ -1,32 +0,0 @@ -//go:build mmal - -package mmal - -import ( - "go.viam.com/rdk/gostream" - "go.viam.com/rdk/gostream/codec" - - "go.viam.com/rdk/logging" -) - -// DefaultStreamConfig configures MMAL as the encoder for a stream. -var DefaultStreamConfig gostream.StreamConfig - -func init() { - DefaultStreamConfig.VideoEncoderFactory = NewEncoderFactory() -} - -// NewEncoderFactory returns an MMAL encoder factory. -func NewEncoderFactory() codec.VideoEncoderFactory { - return &factory{} -} - -type factory struct{} - -func (f *factory) New(width, height, keyFrameInterval int, logger logging.Logger) (codec.VideoEncoder, error) { - return NewEncoder(width, height, keyFrameInterval, logger) -} - -func (f *factory) MIMEType() string { - return "video/H264" -} From 04921f1e84ab0535feea7b7a3beece5e09101fd8 Mon Sep 17 00:00:00 2001 From: Alan Davidson Date: Tue, 3 Sep 2024 10:47:51 -0400 Subject: [PATCH 04/26] use StoppableWorkers in vectornav/imu (#4318) This all compiles, but I don't know how to test it out on real hardware. I'm willing to do that if you think it's important, but my hope is the change is simple enough that we can skip it. I don't think this PR actually fixes any bugs or race conditions: the old code looked right to me. but I also think the new version is simpler to reason about. Github suggests that Dan G should review this; if you disagree, feel free to remove yourself and add someone else. --- components/movementsensor/imuvectornav/imu.go | 23 ++++++++----------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/components/movementsensor/imuvectornav/imu.go b/components/movementsensor/imuvectornav/imu.go index 430017867f3..2b07f7ca82e 100644 --- a/components/movementsensor/imuvectornav/imu.go +++ b/components/movementsensor/imuvectornav/imu.go @@ -74,13 +74,12 @@ type vectornav struct { spiMu sync.Mutex polling int - cancelFunc func() - activeBackgroundWorkers sync.WaitGroup - bus buses.SPI - cs string - speed int - logger logging.Logger - busClosed bool + workers utils.StoppableWorkers + bus buses.SPI + cs string + speed int + logger logging.Logger + busClosed bool bdVX float64 bdVY float64 @@ -212,16 +211,13 @@ func newVectorNav( if err != nil { return nil, err } - var cancelCtx context.Context - cancelCtx, v.cancelFunc = context.WithCancel(context.Background()) + // optionally start a polling goroutine if pfreq > 0 { logger.CDebugf(ctx, "vecnav: will pool at %d Hz", pfreq) waitCh := make(chan struct{}) s := 1.0 / float64(pfreq) - v.activeBackgroundWorkers.Add(1) - goutils.PanicCapturingGo(func() { - defer v.activeBackgroundWorkers.Done() + v.workers = utils.NewStoppableWorkers(func(cancelCtx context.Context) { timer := time.NewTicker(time.Duration(s * float64(time.Second))) defer timer.Stop() close(waitCh) @@ -539,9 +535,8 @@ func (vn *vectornav) compensateDVBias(ctx context.Context, smpSize uint) error { func (vn *vectornav) Close(ctx context.Context) error { vn.logger.CDebug(ctx, "closing vecnav imu") - vn.cancelFunc() vn.busClosed = true - vn.activeBackgroundWorkers.Wait() + vn.workers.Stop() vn.logger.CDebug(ctx, "closed vecnav imu") return nil } From cd371300ee923e690da5cd920bdaa5da132ab097 Mon Sep 17 00:00:00 2001 From: Benjamin Rewis <32186188+benjirewis@users.noreply.github.com> Date: Tue, 3 Sep 2024 11:56:30 -0400 Subject: [PATCH 05/26] RSDK-8570 Utilize `StoppableWorkers` from goutils (#4325) --- components/board/fake/board.go | 7 +- components/board/genericlinux/board.go | 10 +-- .../board/genericlinux/digital_interrupts.go | 5 +- components/board/genericlinux/gpio.go | 3 +- components/board/numato/board.go | 5 +- .../board/pinwrappers/analog_smoother.go | 4 +- components/encoder/ams/ams_as5048.go | 5 +- components/encoder/single/single_encoder.go | 7 +- components/motor/ulnstepper/28byj-48.go | 6 +- components/movementsensor/adxl345/adxl345.go | 6 +- components/movementsensor/gpsrtk/gpsrtk.go | 6 +- .../movementsensor/gpsutils/cachedData.go | 5 +- .../gpsutils/i2c_data_reader.go | 5 +- .../gpsutils/serial_data_reader.go | 6 +- components/movementsensor/gpsutils/vrs.go | 5 +- components/movementsensor/imuwit/imu.go | 4 +- components/movementsensor/imuwit/imuhwt905.go | 4 +- components/movementsensor/mpu6050/mpu6050.go | 5 +- .../wheeledodometry/wheeledodometry.go | 5 +- go.mod | 2 +- go.sum | 4 +- robot/session_manager.go | 5 +- utils/stoppable_workers.go | 77 ------------------- 23 files changed, 54 insertions(+), 137 deletions(-) delete mode 100644 utils/stoppable_workers.go diff --git a/components/board/fake/board.go b/components/board/fake/board.go index ff7f6204f6d..95821b56bee 100644 --- a/components/board/fake/board.go +++ b/components/board/fake/board.go @@ -18,7 +18,6 @@ import ( "go.viam.com/rdk/grpc" "go.viam.com/rdk/logging" "go.viam.com/rdk/resource" - rdkutils "go.viam.com/rdk/utils" ) // In order to maintain test functionality, testPin will always return the analog value it is set @@ -83,7 +82,7 @@ func NewBoard(ctx context.Context, conf resource.Config, logger logging.Logger) Analogs: map[string]*Analog{}, Digitals: map[string]*DigitalInterrupt{}, GPIOPins: map[string]*GPIOPin{}, - workers: rdkutils.NewStoppableWorkers(), + workers: utils.NewBackgroundStoppableWorkers(), logger: logger, } @@ -167,7 +166,7 @@ type Board struct { logger logging.Logger CloseCount int - workers rdkutils.StoppableWorkers + workers *utils.StoppableWorkers } // AnalogByName returns the analog pin by the given name if it exists. @@ -247,7 +246,7 @@ func (b *Board) StreamTicks(ctx context.Context, interrupts []board.DigitalInter for _, di := range interrupts { // Don't need to check if interrupt exists, just did that above - b.workers.AddWorkers(func(workersContext context.Context) { + b.workers.Add(func(workersContext context.Context) { for { // sleep to avoid a busy loop if !utils.SelectContextOrWait(workersContext, 700*time.Millisecond) { diff --git a/components/board/genericlinux/board.go b/components/board/genericlinux/board.go index ada3b2c8b55..bf2e911f0c6 100644 --- a/components/board/genericlinux/board.go +++ b/components/board/genericlinux/board.go @@ -15,6 +15,7 @@ import ( "github.com/pkg/errors" "go.uber.org/multierr" pb "go.viam.com/api/component/board/v1" + "go.viam.com/utils" "go.viam.com/rdk/components/board" "go.viam.com/rdk/components/board/genericlinux/buses" @@ -23,7 +24,6 @@ import ( "go.viam.com/rdk/grpc" "go.viam.com/rdk/logging" "go.viam.com/rdk/resource" - "go.viam.com/rdk/utils" ) // RegisterBoard registers a sysfs based board of the given model. @@ -55,7 +55,7 @@ func NewBoard( convertConfig: convertConfig, logger: logger, - workers: utils.NewStoppableWorkers(), + workers: utils.NewBackgroundStoppableWorkers(), analogReaders: map[string]*wrappedAnalogReader{}, gpios: map[string]*gpioPin{}, @@ -343,7 +343,7 @@ func (b *Board) createGpioPin(mapping GPIOBoardMapping) *gpioPin { logger: b.logger, startSoftwarePWMChan: &startSoftwarePWMChan, } - pin.softwarePwm = utils.NewStoppableWorkers(pin.softwarePwmLoop) + pin.softwarePwm = utils.NewBackgroundStoppableWorkers(pin.softwarePwmLoop) if mapping.HWPWMSupported { pin.hwPwm = newPwmDevice(mapping.PWMSysFsDir, mapping.PWMID, b.logger) } @@ -363,7 +363,7 @@ type Board struct { gpios map[string]*gpioPin interrupts map[string]*digitalInterrupt - workers utils.StoppableWorkers + workers *utils.StoppableWorkers } // AnalogByName returns the analog pin by the given name if it exists. @@ -477,7 +477,7 @@ func (b *Board) StreamTicks(ctx context.Context, interrupts []board.DigitalInter i.AddChannel(ch) } - b.workers.AddWorkers(func(cancelCtx context.Context) { + b.workers.Add(func(cancelCtx context.Context) { // Wait until it's time to shut down then remove callbacks. select { case <-ctx.Done(): diff --git a/components/board/genericlinux/digital_interrupts.go b/components/board/genericlinux/digital_interrupts.go index ad19b4be207..9dfdfe693e9 100644 --- a/components/board/genericlinux/digital_interrupts.go +++ b/components/board/genericlinux/digital_interrupts.go @@ -15,11 +15,10 @@ import ( "go.viam.com/utils" "go.viam.com/rdk/components/board" - rdkutils "go.viam.com/rdk/utils" ) type digitalInterrupt struct { - workers rdkutils.StoppableWorkers + workers *utils.StoppableWorkers line *gpio.LineWithEvent mu sync.Mutex // Protects everything below here config board.DigitalInterruptConfig @@ -48,7 +47,7 @@ func newDigitalInterrupt( } di := digitalInterrupt{line: line, config: config} - di.workers = rdkutils.NewStoppableWorkers(di.monitor) + di.workers = utils.NewBackgroundStoppableWorkers(di.monitor) if oldInterrupt != nil { oldInterrupt.mu.Lock() diff --git a/components/board/genericlinux/gpio.go b/components/board/genericlinux/gpio.go index f8aa5f09644..680e34104e1 100644 --- a/components/board/genericlinux/gpio.go +++ b/components/board/genericlinux/gpio.go @@ -16,7 +16,6 @@ import ( "go.viam.com/rdk/components/board" "go.viam.com/rdk/logging" - rdkutils "go.viam.com/rdk/utils" ) const noPin = 0xFFFFFFFF // noPin is the uint32 version of -1. A pin with this offset has no GPIO @@ -35,7 +34,7 @@ type gpioPin struct { enableSoftwarePWM bool // Indicates whether a software PWM loop should continue running startSoftwarePWMChan *chan any // Close and reinitialize this to (re)start the SW PWM loop - softwarePwm rdkutils.StoppableWorkers + softwarePwm *utils.StoppableWorkers mu sync.Mutex logger logging.Logger diff --git a/components/board/numato/board.go b/components/board/numato/board.go index 2ae7f6665e9..b6d724f389c 100644 --- a/components/board/numato/board.go +++ b/components/board/numato/board.go @@ -26,7 +26,6 @@ import ( "go.viam.com/rdk/grpc" "go.viam.com/rdk/logging" "go.viam.com/rdk/resource" - rdkutils "go.viam.com/rdk/utils" ) var model = resource.DefaultModelFamily.WithModel("numato") @@ -116,7 +115,7 @@ type numatoBoard struct { sent map[string]bool sentMu sync.Mutex - workers rdkutils.StoppableWorkers + workers *utils.StoppableWorkers maxAnalogVoltage float32 stepSize float32 @@ -465,7 +464,7 @@ func connect(ctx context.Context, name resource.Name, conf *Config, logger loggi b.lines = make(chan string) - b.workers = rdkutils.NewStoppableWorkers(b.readThread) + b.workers = utils.NewBackgroundStoppableWorkers(b.readThread) ver, err := b.doSendReceive(ctx, "ver") if err != nil { diff --git a/components/board/pinwrappers/analog_smoother.go b/components/board/pinwrappers/analog_smoother.go index 046255e2344..17ea24f79b1 100644 --- a/components/board/pinwrappers/analog_smoother.go +++ b/components/board/pinwrappers/analog_smoother.go @@ -28,7 +28,7 @@ type AnalogSmoother struct { lastData atomic.Pointer[board.AnalogValue] lastError atomic.Pointer[errValue] logger logging.Logger - workers utils.StoppableWorkers + workers *goutils.StoppableWorkers } // SmoothAnalogReader wraps the given reader in a smoother. @@ -116,7 +116,7 @@ func (as *AnalogSmoother) Start() { as.data = nil } - as.workers = utils.NewStoppableWorkers(func(ctx context.Context) { + as.workers = goutils.NewBackgroundStoppableWorkers(func(ctx context.Context) { consecutiveErrors := 0 var lastError error diff --git a/components/encoder/ams/ams_as5048.go b/components/encoder/ams/ams_as5048.go index a262b401377..9e3811548e9 100644 --- a/components/encoder/ams/ams_as5048.go +++ b/components/encoder/ams/ams_as5048.go @@ -17,7 +17,6 @@ import ( "go.viam.com/rdk/components/encoder" "go.viam.com/rdk/logging" "go.viam.com/rdk/resource" - rdkutils "go.viam.com/rdk/utils" ) const ( @@ -113,7 +112,7 @@ type Encoder struct { i2cBus buses.I2C i2cAddr byte i2cBusName string // This is nessesary to check whether we need to create a new i2cBus during reconfigure. - workers rdkutils.StoppableWorkers + workers *utils.StoppableWorkers } func newAS5048Encoder( @@ -152,7 +151,7 @@ func makeAS5048Encoder( if err := res.ResetPosition(ctx, map[string]interface{}{}); err != nil { return nil, err } - res.workers = rdkutils.NewStoppableWorkers(res.positionLoop) + res.workers = utils.NewBackgroundStoppableWorkers(res.positionLoop) return res, nil } diff --git a/components/encoder/single/single_encoder.go b/components/encoder/single/single_encoder.go index 39ff03c8556..ffb05f9a012 100644 --- a/components/encoder/single/single_encoder.go +++ b/components/encoder/single/single_encoder.go @@ -35,7 +35,6 @@ import ( "go.viam.com/rdk/components/encoder" "go.viam.com/rdk/logging" "go.viam.com/rdk/resource" - rdkutils "go.viam.com/rdk/utils" ) var singleModel = resource.DefaultModelFamily.WithModel("single") @@ -70,7 +69,7 @@ type Encoder struct { positionType encoder.PositionType logger logging.Logger - workers rdkutils.StoppableWorkers + workers *utils.StoppableWorkers } // Pin describes the configuration of Pins for a Single encoder. @@ -176,7 +175,7 @@ func (e *Encoder) Reconfigure( // start starts the Encoder background thread. It should only be called when the encoder's // background workers have been stopped (or never started). func (e *Encoder) start(ctx context.Context, b board.Board) { - e.workers = rdkutils.NewStoppableWorkers() + e.workers = utils.NewBackgroundStoppableWorkers() encoderChannel := make(chan board.Tick) err := b.StreamTicks(e.workers.Context(), []board.DigitalInterrupt{e.I}, encoderChannel, nil) @@ -185,7 +184,7 @@ func (e *Encoder) start(ctx context.Context, b board.Board) { return } - e.workers.AddWorkers(func(cancelCtx context.Context) { + e.workers.Add(func(cancelCtx context.Context) { for { select { case <-cancelCtx.Done(): diff --git a/components/motor/ulnstepper/28byj-48.go b/components/motor/ulnstepper/28byj-48.go index 8c1768173d9..6a8a217678d 100644 --- a/components/motor/ulnstepper/28byj-48.go +++ b/components/motor/ulnstepper/28byj-48.go @@ -25,13 +25,13 @@ import ( "github.com/pkg/errors" "go.uber.org/multierr" + "go.viam.com/utils" "go.viam.com/rdk/components/board" "go.viam.com/rdk/components/motor" "go.viam.com/rdk/logging" "go.viam.com/rdk/operation" "go.viam.com/rdk/resource" - "go.viam.com/rdk/utils" ) var ( @@ -165,7 +165,7 @@ type uln28byj struct { motorName string // state - workers utils.StoppableWorkers + workers *utils.StoppableWorkers lock sync.Mutex opMgr *operation.SingleOperationManager doRunDone func() @@ -185,7 +185,7 @@ func (m *uln28byj) doRun() { // start a new doRun var doRunCtx context.Context doRunCtx, m.doRunDone = context.WithCancel(context.Background()) - m.workers = utils.NewStoppableWorkers(func(ctx context.Context) { + m.workers = utils.NewBackgroundStoppableWorkers(func(ctx context.Context) { for { select { case <-doRunCtx.Done(): diff --git a/components/movementsensor/adxl345/adxl345.go b/components/movementsensor/adxl345/adxl345.go index f449dd63001..30ad4b10b21 100644 --- a/components/movementsensor/adxl345/adxl345.go +++ b/components/movementsensor/adxl345/adxl345.go @@ -169,7 +169,7 @@ type adxl345 struct { linearAcceleration r3.Vector err movementsensor.LastError - workers rutils.StoppableWorkers + workers *utils.StoppableWorkers } // newAdxl345 is a constructor to create a new object representing an ADXL345 accelerometer. @@ -256,7 +256,7 @@ func makeAdxl345( // Now, turn on the background goroutine that constantly reads from the chip and stores data in // the object we created. - sensor.workers = rutils.NewStoppableWorkers(func(cancelContext context.Context) { + sensor.workers = utils.NewBackgroundStoppableWorkers(func(cancelContext context.Context) { // Reading data a thousand times per second is probably fast enough. timer := time.NewTicker(time.Millisecond) defer timer.Stop() @@ -337,7 +337,7 @@ func makeAdxl345( } func (adxl *adxl345) startInterruptMonitoring(ticksChan chan board.Tick) { - adxl.workers.AddWorkers(func(cancelContext context.Context) { + adxl.workers.Add(func(cancelContext context.Context) { for { select { case <-cancelContext.Done(): diff --git a/components/movementsensor/gpsrtk/gpsrtk.go b/components/movementsensor/gpsrtk/gpsrtk.go index 268e8d12218..439775dc346 100644 --- a/components/movementsensor/gpsrtk/gpsrtk.go +++ b/components/movementsensor/gpsrtk/gpsrtk.go @@ -26,13 +26,13 @@ import ( "github.com/go-gnss/rtcm/rtcm3" "github.com/golang/geo/r3" geo "github.com/kellydunn/golang-geo" + "go.viam.com/utils" "go.viam.com/rdk/components/movementsensor" "go.viam.com/rdk/components/movementsensor/gpsutils" "go.viam.com/rdk/logging" "go.viam.com/rdk/resource" "go.viam.com/rdk/spatialmath" - "go.viam.com/rdk/utils" ) // gpsrtk is an nmea movementsensor model that can intake RTK correction data. @@ -41,7 +41,7 @@ type gpsrtk struct { resource.AlwaysRebuild logger logging.Logger - workers utils.StoppableWorkers + workers *utils.StoppableWorkers err movementsensor.LastError isClosed bool @@ -62,7 +62,7 @@ func (g *gpsrtk) start() error { if g.workers != nil { return errors.New("do not call start() twice on the same object") } - g.workers = utils.NewStoppableWorkers(g.receiveAndWriteCorrectionData) + g.workers = utils.NewBackgroundStoppableWorkers(g.receiveAndWriteCorrectionData) return nil } diff --git a/components/movementsensor/gpsutils/cachedData.go b/components/movementsensor/gpsutils/cachedData.go index 4b449e7fb46..894d87c2bb1 100644 --- a/components/movementsensor/gpsutils/cachedData.go +++ b/components/movementsensor/gpsutils/cachedData.go @@ -8,6 +8,7 @@ import ( "github.com/golang/geo/r3" geo "github.com/kellydunn/golang-geo" "github.com/pkg/errors" + goutils "go.viam.com/utils" "go.viam.com/rdk/components/movementsensor" "go.viam.com/rdk/logging" @@ -36,7 +37,7 @@ type CachedData struct { dev DataReader logger logging.Logger - workers utils.StoppableWorkers + workers *goutils.StoppableWorkers } // CommonReadings struct contains the most common values we output in sensor reading. @@ -55,7 +56,7 @@ func NewCachedData(dev DataReader, logger logging.Logger) *CachedData { dev: dev, logger: logger, } - g.workers = utils.NewStoppableWorkers(g.start) + g.workers = goutils.NewBackgroundStoppableWorkers(g.start) return &g } diff --git a/components/movementsensor/gpsutils/i2c_data_reader.go b/components/movementsensor/gpsutils/i2c_data_reader.go index 9e991fa9414..0a67575fee0 100644 --- a/components/movementsensor/gpsutils/i2c_data_reader.go +++ b/components/movementsensor/gpsutils/i2c_data_reader.go @@ -14,7 +14,6 @@ import ( "go.viam.com/rdk/components/board/genericlinux/buses" "go.viam.com/rdk/components/movementsensor" "go.viam.com/rdk/logging" - rdkutils "go.viam.com/rdk/utils" ) // PmtkI2cDataReader implements the DataReader interface for a PMTK device by communicating with it @@ -22,7 +21,7 @@ import ( type PmtkI2cDataReader struct { data chan string - workers rdkutils.StoppableWorkers + workers *utils.StoppableWorkers logger logging.Logger bus buses.I2C @@ -67,7 +66,7 @@ func NewI2cDataReader( return nil, err } - reader.workers = rdkutils.NewStoppableWorkers(reader.backgroundWorker) + reader.workers = utils.NewBackgroundStoppableWorkers(reader.backgroundWorker) return &reader, nil } diff --git a/components/movementsensor/gpsutils/serial_data_reader.go b/components/movementsensor/gpsutils/serial_data_reader.go index b0db6b7a368..4083a944c84 100644 --- a/components/movementsensor/gpsutils/serial_data_reader.go +++ b/components/movementsensor/gpsutils/serial_data_reader.go @@ -10,9 +10,9 @@ import ( "io" "github.com/jacobsa/go-serial/serial" + "go.viam.com/utils" "go.viam.com/rdk/logging" - "go.viam.com/rdk/utils" ) // SerialDataReader implements the DataReader interface (defined in component.go) by interacting @@ -20,7 +20,7 @@ import ( type SerialDataReader struct { dev io.ReadWriteCloser data chan string - workers utils.StoppableWorkers + workers *utils.StoppableWorkers logger logging.Logger } @@ -61,7 +61,7 @@ func NewSerialDataReader( data: data, logger: logger, } - reader.workers = utils.NewStoppableWorkers(reader.backgroundWorker) + reader.workers = utils.NewBackgroundStoppableWorkers(reader.backgroundWorker) return &reader, nil } diff --git a/components/movementsensor/gpsutils/vrs.go b/components/movementsensor/gpsutils/vrs.go index 143daba4fbf..46a0512563b 100644 --- a/components/movementsensor/gpsutils/vrs.go +++ b/components/movementsensor/gpsutils/vrs.go @@ -16,7 +16,6 @@ import ( goutils "go.viam.com/utils" "go.viam.com/rdk/logging" - "go.viam.com/rdk/utils" ) const ( @@ -28,7 +27,7 @@ type VRS struct { ntripInfo *NtripInfo readerWriter *bufio.ReadWriter conn net.Conn - workers utils.StoppableWorkers + workers *goutils.StoppableWorkers logger logging.Logger } @@ -160,7 +159,7 @@ func (vrs *VRS) startGGAThread(getGGA func() (string, error)) { vrs.workers.Stop() } - vrs.workers = utils.NewStoppableWorkers(func(cancelCtx context.Context) { + vrs.workers = goutils.NewBackgroundStoppableWorkers(func(cancelCtx context.Context) { ticker := time.NewTicker(time.Duration(vrsGGARateSec) * time.Second) defer ticker.Stop() diff --git a/components/movementsensor/imuwit/imu.go b/components/movementsensor/imuwit/imu.go index 9013bfb32eb..f3655438bb0 100644 --- a/components/movementsensor/imuwit/imu.go +++ b/components/movementsensor/imuwit/imu.go @@ -99,7 +99,7 @@ type wit struct { mu sync.Mutex reconfigMu sync.Mutex port io.ReadWriteCloser - workers rutils.StoppableWorkers + workers *utils.StoppableWorkers logger logging.Logger baudRate uint serialPath string @@ -298,7 +298,7 @@ func newWit( func (imu *wit) startUpdateLoop(portReader *bufio.Reader, logger logging.Logger) { imu.hasMagnetometer = false - imu.workers = rutils.NewStoppableWorkers(func(ctx context.Context) { + imu.workers = utils.NewBackgroundStoppableWorkers(func(ctx context.Context) { defer utils.UncheckedErrorFunc(func() error { if imu.port != nil { if err := imu.port.Close(); err != nil { diff --git a/components/movementsensor/imuwit/imuhwt905.go b/components/movementsensor/imuwit/imuhwt905.go index 7a53b5d33c5..9044e467a68 100644 --- a/components/movementsensor/imuwit/imuhwt905.go +++ b/components/movementsensor/imuwit/imuhwt905.go @@ -17,11 +17,11 @@ import ( "time" slib "github.com/jacobsa/go-serial/serial" + "go.viam.com/utils" "go.viam.com/rdk/components/movementsensor" "go.viam.com/rdk/logging" "go.viam.com/rdk/resource" - "go.viam.com/rdk/utils" ) var model905 = resource.DefaultModelFamily.WithModel("imu-wit-hwt905") @@ -77,7 +77,7 @@ func newWit905( func (imu *wit) start905UpdateLoop(portReader *bufio.Reader, logger logging.Logger) { imu.hasMagnetometer = false - imu.workers = utils.NewStoppableWorkers(func(cancelCtx context.Context) { + imu.workers = utils.NewBackgroundStoppableWorkers(func(cancelCtx context.Context) { for { if cancelCtx.Err() != nil { return diff --git a/components/movementsensor/mpu6050/mpu6050.go b/components/movementsensor/mpu6050/mpu6050.go index 8ef50a0b605..68068158e4b 100644 --- a/components/movementsensor/mpu6050/mpu6050.go +++ b/components/movementsensor/mpu6050/mpu6050.go @@ -29,6 +29,7 @@ import ( "github.com/golang/geo/r3" geo "github.com/kellydunn/golang-geo" "github.com/pkg/errors" + goutils "go.viam.com/utils" "go.viam.com/rdk/components/board/genericlinux/buses" "go.viam.com/rdk/components/movementsensor" @@ -83,7 +84,7 @@ type mpu6050 struct { // Stores the most recent error from the background goroutine err movementsensor.LastError - workers utils.StoppableWorkers + workers *goutils.StoppableWorkers logger logging.Logger } @@ -167,7 +168,7 @@ func makeMpu6050( // Now, turn on the background goroutine that constantly reads from the chip and stores data in // the object we created. - sensor.workers = utils.NewStoppableWorkers(func(cancelCtx context.Context) { + sensor.workers = goutils.NewBackgroundStoppableWorkers(func(cancelCtx context.Context) { // Reading data a thousand times per second is probably fast enough. timer := time.NewTicker(time.Millisecond) defer timer.Stop() diff --git a/components/movementsensor/wheeledodometry/wheeledodometry.go b/components/movementsensor/wheeledodometry/wheeledodometry.go index 4326836303b..c5095646926 100644 --- a/components/movementsensor/wheeledodometry/wheeledodometry.go +++ b/components/movementsensor/wheeledodometry/wheeledodometry.go @@ -12,6 +12,7 @@ import ( "github.com/golang/geo/r3" geo "github.com/kellydunn/golang-geo" + goutils "go.viam.com/utils" "go.viam.com/rdk/components/base" "go.viam.com/rdk/components/motor" @@ -76,7 +77,7 @@ type odometry struct { useCompass bool shiftPos bool - workers utils.StoppableWorkers + workers *goutils.StoppableWorkers mu sync.Mutex logger logging.Logger } @@ -368,7 +369,7 @@ func (o *odometry) checkBaseProps(ctx context.Context) { // https://stuff.mit.edu/afs/athena/course/6/6.186/OldFiles/2005/doc/odomtutorial/odomtutorial.pdf func (o *odometry) trackPosition() { // Spawn a new goroutine to do all the work in the background. - o.workers = utils.NewStoppableWorkers(func(ctx context.Context) { + o.workers = goutils.NewBackgroundStoppableWorkers(func(ctx context.Context) { ticker := time.NewTicker(time.Duration(o.timeIntervalMSecs) * time.Millisecond) for { select { diff --git a/go.mod b/go.mod index b867cd657a9..41ee05178fa 100644 --- a/go.mod +++ b/go.mod @@ -85,7 +85,7 @@ require ( go.uber.org/zap v1.24.0 go.viam.com/api v0.1.336 go.viam.com/test v1.1.1-0.20220913152726-5da9916c08a2 - go.viam.com/utils v0.1.96 + go.viam.com/utils v0.1.97 goji.io v2.0.2+incompatible golang.org/x/image v0.19.0 golang.org/x/mobile v0.0.0-20240112133503-c713f31d574b diff --git a/go.sum b/go.sum index dd541719175..faa94e5fc8e 100644 --- a/go.sum +++ b/go.sum @@ -1541,8 +1541,8 @@ go.viam.com/api v0.1.336 h1:mcz3Y5rivgXhsTu/bXkAVDw0/otarq3lCPIRcxhNnIY= go.viam.com/api v0.1.336/go.mod h1:msa4TPrMVeRDcG4YzKA/S6wLEUC7GyHQE973JklrQ10= go.viam.com/test v1.1.1-0.20220913152726-5da9916c08a2 h1:oBiK580EnEIzgFLU4lHOXmGAE3MxnVbeR7s1wp/F3Ps= go.viam.com/test v1.1.1-0.20220913152726-5da9916c08a2/go.mod h1:XM0tej6riszsiNLT16uoyq1YjuYPWlRBweTPRDanIts= -go.viam.com/utils v0.1.96 h1:lZ7m1jm6pkjn60e9OWeHyHcivFIRrFvS/5NGrPXFaDM= -go.viam.com/utils v0.1.96/go.mod h1:GCaRsDlW3p3tOnLo73swpPnfrmmqg6r+1oWhNVEHsDo= +go.viam.com/utils v0.1.97 h1:ZSmasJJkToHEbHJQ2xJbBjE5j+qlxFN9gDhad8Wex4M= +go.viam.com/utils v0.1.97/go.mod h1:GCaRsDlW3p3tOnLo73swpPnfrmmqg6r+1oWhNVEHsDo= go4.org/unsafe/assume-no-moving-gc v0.0.0-20230525183740-e7c30c78aeb2 h1:WJhcL4p+YeDxmZWg141nRm7XC8IDmhz7lk5GpadO1Sg= go4.org/unsafe/assume-no-moving-gc v0.0.0-20230525183740-e7c30c78aeb2/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E= gocv.io/x/gocv v0.25.0/go.mod h1:Rar2PS6DV+T4FL+PM535EImD/h13hGVaHhnCu1xarBs= diff --git a/robot/session_manager.go b/robot/session_manager.go index d70214e4f24..1e00cbb01f0 100644 --- a/robot/session_manager.go +++ b/robot/session_manager.go @@ -12,7 +12,6 @@ import ( "go.viam.com/rdk/logging" "go.viam.com/rdk/resource" "go.viam.com/rdk/session" - rdkutils "go.viam.com/rdk/utils" ) // NewSessionManager creates a new manager for holding sessions. @@ -24,7 +23,7 @@ func NewSessionManager(robot Robot, heartbeatWindow time.Duration) *SessionManag sessions: map[uuid.UUID]*session.Session{}, resourceToSession: map[resource.Name]uuid.UUID{}, } - m.workers = rdkutils.NewStoppableWorkers(m.expireLoop) + m.workers = utils.NewBackgroundStoppableWorkers(m.expireLoop) return m } @@ -40,7 +39,7 @@ type SessionManager struct { resourceToSession map[resource.Name]uuid.UUID - workers rdkutils.StoppableWorkers + workers *utils.StoppableWorkers } // All returns all active sessions. diff --git a/utils/stoppable_workers.go b/utils/stoppable_workers.go deleted file mode 100644 index de6c626380e..00000000000 --- a/utils/stoppable_workers.go +++ /dev/null @@ -1,77 +0,0 @@ -package utils - -import ( - "context" - "sync" - - goutils "go.viam.com/utils" -) - -// TODO: When this struct is widely used and feature complete, move this to goutils instead of -// here. Until then, we cannot use this in any package imported by utils (e.g., the logging -// package) without introducing a circular import dependency. - -// StoppableWorkers is a collection of goroutines that can be stopped at a later time. -type StoppableWorkers interface { - AddWorkers(...func(context.Context)) - Stop() - Context() context.Context -} - -// stoppableWorkersImpl is the implementation of StoppableWorkers. The linter will complain if you -// try to make a copy of something that contains a sync.WaitGroup (and returning a value at the end -// of NewStoppableWorkers() would make a copy of it), so we do everything through the -// StoppableWorkers interface to avoid making copies (since interfaces do everything by pointer). -type stoppableWorkersImpl struct { - mu sync.Mutex - cancelCtx context.Context - cancelFunc func() - activeBackgroundWorkers sync.WaitGroup -} - -// NewStoppableWorkers runs the functions in separate goroutines. They can be stopped later. -func NewStoppableWorkers(funcs ...func(context.Context)) StoppableWorkers { - cancelCtx, cancelFunc := context.WithCancel(context.Background()) - workers := &stoppableWorkersImpl{cancelCtx: cancelCtx, cancelFunc: cancelFunc} - workers.AddWorkers(funcs...) - return workers -} - -// AddWorkers starts up additional goroutines for each function passed in. If you call this after -// calling Stop(), it will return immediately without starting any new goroutines. -func (sw *stoppableWorkersImpl) AddWorkers(funcs ...func(context.Context)) { - sw.mu.Lock() - defer sw.mu.Unlock() - - if sw.cancelCtx.Err() != nil { // We've already stopped everything. - return - } - - sw.activeBackgroundWorkers.Add(len(funcs)) - for _, f := range funcs { - // In Go 1.21 and earlier, variables created in a loop were reused from one iteration to - // the next. Make a "fresh" copy of it here so that, if we're on to the next iteration of - // the loop before the goroutine starts up, it starts this function instead of the next - // one. For details, see https://go.dev/blog/loopvar-preview - f := f - goutils.PanicCapturingGo(func() { - defer sw.activeBackgroundWorkers.Done() - f(sw.cancelCtx) - }) - } -} - -// Stop shuts down all the goroutines we started up. -func (sw *stoppableWorkersImpl) Stop() { - sw.mu.Lock() - defer sw.mu.Unlock() - - sw.cancelFunc() - sw.activeBackgroundWorkers.Wait() -} - -// Context gets the context the workers are checking on. Using this function is expected to be -// rare: usually you shouldn't need to interact with the context directly. -func (sw *stoppableWorkersImpl) Context() context.Context { - return sw.cancelCtx -} From b808894269d52fc8feae2e7a784dcd9eb4f4ba31 Mon Sep 17 00:00:00 2001 From: Benjamin Rewis <32186188+benjirewis@users.noreply.github.com> Date: Tue, 3 Sep 2024 12:42:34 -0400 Subject: [PATCH 06/26] Use correct `StoppableWorkers` in imu code (#4340) --- components/movementsensor/imuvectornav/imu.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/movementsensor/imuvectornav/imu.go b/components/movementsensor/imuvectornav/imu.go index 2b07f7ca82e..77b6d16b041 100644 --- a/components/movementsensor/imuvectornav/imu.go +++ b/components/movementsensor/imuvectornav/imu.go @@ -74,7 +74,7 @@ type vectornav struct { spiMu sync.Mutex polling int - workers utils.StoppableWorkers + workers *goutils.StoppableWorkers bus buses.SPI cs string speed int @@ -217,7 +217,7 @@ func newVectorNav( logger.CDebugf(ctx, "vecnav: will pool at %d Hz", pfreq) waitCh := make(chan struct{}) s := 1.0 / float64(pfreq) - v.workers = utils.NewStoppableWorkers(func(cancelCtx context.Context) { + v.workers = goutils.NewBackgroundStoppableWorkers(func(cancelCtx context.Context) { timer := time.NewTicker(time.Duration(s * float64(time.Second))) defer timer.Stop() close(waitCh) From 4d3b913e4494ad7d319b6f5e5af4357ec6fa1367 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 3 Sep 2024 13:54:26 -0400 Subject: [PATCH 07/26] Automated remote-control Version Update (#4341) Co-authored-by: 10zingpd <10zingpd@users.noreply.github.com> --- web/frontend/package-lock.json | 14 +++++++------- web/frontend/package.json | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/web/frontend/package-lock.json b/web/frontend/package-lock.json index 3bfdf12a9a8..1283102e03c 100644 --- a/web/frontend/package-lock.json +++ b/web/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@viamrobotics/remote-control", - "version": "2.21.2", + "version": "2.22.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@viamrobotics/remote-control", - "version": "2.21.2", + "version": "2.22.0", "license": "Apache-2.0", "devDependencies": { "@improbable-eng/grpc-web": "0.15.0", @@ -25,7 +25,7 @@ "@viamrobotics/prime-blocks": "^0.1.3", "@viamrobotics/prime-core": "^0.0.87", "@viamrobotics/rpc": "0.2.3", - "@viamrobotics/sdk": "0.22.0", + "@viamrobotics/sdk": "0.23.0", "@viamrobotics/three": "^0.0.3", "@viamrobotics/typescript-config": "^0.1.0", "cypress": "12.17.3", @@ -60,7 +60,7 @@ "@improbable-eng/grpc-web": ">=0.15", "@viamrobotics/prime": ">=0.5", "@viamrobotics/rpc": ">=0.2", - "@viamrobotics/sdk": "0.22.0", + "@viamrobotics/sdk": "0.23.0", "google-protobuf": ">=3", "maplibre-gl": ">=4", "tailwindcss": ">=3.3", @@ -1546,9 +1546,9 @@ } }, "node_modules/@viamrobotics/sdk": { - "version": "0.22.0", - "resolved": "https://registry.npmjs.org/@viamrobotics/sdk/-/sdk-0.22.0.tgz", - "integrity": "sha512-AAobJxxQOZiiD5RxN/NtiLpg1SEk5vIO/wKhRxpH0J2uVu3p6llPu714nV/qMOE3A1uTYWw6ozCYnMo6an7Hvg==", + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/@viamrobotics/sdk/-/sdk-0.23.0.tgz", + "integrity": "sha512-3d50zGllR7ydzpFCe3e5Apih1PxNHOUK1iCnDaDFU5yalk0QzvPI4VCo/SWFqqoFb5CXcL+fCmr/FHFZDVzMag==", "dev": true, "dependencies": { "@viamrobotics/rpc": "^0.2.5", diff --git a/web/frontend/package.json b/web/frontend/package.json index d4e8758706b..b9cd97081f3 100644 --- a/web/frontend/package.json +++ b/web/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@viamrobotics/remote-control", - "version": "2.21.2", + "version": "2.22.0", "license": "Apache-2.0", "type": "module", "scripts": { @@ -37,7 +37,7 @@ "@improbable-eng/grpc-web": ">=0.15", "@viamrobotics/prime": ">=0.5", "@viamrobotics/rpc": ">=0.2", - "@viamrobotics/sdk": "0.22.0", + "@viamrobotics/sdk": "0.23.0", "google-protobuf": ">=3", "maplibre-gl": ">=4", "tailwindcss": ">=3.3", @@ -60,7 +60,7 @@ "@viamrobotics/prime-blocks": "^0.1.3", "@viamrobotics/prime-core": "^0.0.87", "@viamrobotics/rpc": "0.2.3", - "@viamrobotics/sdk": "0.22.0", + "@viamrobotics/sdk": "0.23.0", "@viamrobotics/three": "^0.0.3", "@viamrobotics/typescript-config": "^0.1.0", "cypress": "12.17.3", From 55ac5e3bc351b41ddc23b46df2016686e92b745a Mon Sep 17 00:00:00 2001 From: oliviamiller <106617921+oliviamiller@users.noreply.github.com> Date: Tue, 3 Sep 2024 15:20:05 -0400 Subject: [PATCH 08/26] RSDK-8592 Make single encoder tick even without motor (#4332) --- components/encoder/single/single_encoder.go | 8 ++-- .../encoder/single/single_encoder_test.go | 39 ++++++++++++++++++- 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/components/encoder/single/single_encoder.go b/components/encoder/single/single_encoder.go index ffb05f9a012..423ce9ff2a0 100644 --- a/components/encoder/single/single_encoder.go +++ b/components/encoder/single/single_encoder.go @@ -168,13 +168,13 @@ func (e *Encoder) Reconfigure( if e.workers != nil { e.workers.Stop() // Shut down the old interrupt stream } - e.start(ctx, board) // Start up the new interrupt stream + e.start(board) // Start up the new interrupt stream return nil } // start starts the Encoder background thread. It should only be called when the encoder's // background workers have been stopped (or never started). -func (e *Encoder) start(ctx context.Context, b board.Board) { +func (e *Encoder) start(b board.Board) { e.workers = utils.NewBackgroundStoppableWorkers() encoderChannel := make(chan board.Tick) @@ -207,7 +207,9 @@ func (e *Encoder) start(ctx context.Context, b board.Board) { atomic.AddInt64(&e.position, dir) } } else { - e.logger.CDebug(ctx, "received tick for encoder that isn't connected to a motor; ignoring") + // if no motor is attached to the encoder, increase in positive direction. + e.logger.Debug("no motor is attached to the encoder, increasing ticks count in the positive direction only") + atomic.AddInt64(&e.position, 1) } } }) diff --git a/components/encoder/single/single_encoder_test.go b/components/encoder/single/single_encoder_test.go index f0d9df88d4b..2e2693035d4 100644 --- a/components/encoder/single/single_encoder_test.go +++ b/components/encoder/single/single_encoder_test.go @@ -112,8 +112,7 @@ func TestEncoder(t *testing.T) { }) }) - // this test ensures that digital interrupts are ignored if AttachDirectionalAwareness - // is never called + // this test ensures that position goes forward if motor not attached. t.Run("run no direction", func(t *testing.T) { enc, err := NewSingleEncoder(ctx, deps, rawcfg, logging.NewTestLogger(t)) test.That(t, err, test.ShouldBeNil) @@ -128,6 +127,42 @@ func TestEncoder(t *testing.T) { // by the encoder worker time.Sleep(50 * time.Millisecond) + ticks, _, err := enc.Position(context.Background(), encoder.PositionTypeUnspecified, nil) + test.That(t, err, test.ShouldBeNil) + test.That(t, ticks, test.ShouldEqual, 1) + }) + + // Taking off directional awareness makes encoder tick forward. + t.Run("run motor then no motor", func(t *testing.T) { + enc, err := NewSingleEncoder(ctx, deps, rawcfg, logging.NewTestLogger(t)) + test.That(t, err, test.ShouldBeNil) + enc2 := enc.(*Encoder) + defer enc2.Close(context.Background()) + + m := &FakeDir{-1} // backward + enc2.AttachDirectionalAwareness(m) + + err = ii.Tick(context.Background(), true, uint64(time.Now().UnixNano())) + test.That(t, err, test.ShouldBeNil) + + testutils.WaitForAssertion(t, func(tb testing.TB) { + tb.Helper() + ticks, _, err := enc.Position(context.Background(), encoder.PositionTypeUnspecified, nil) + test.That(tb, err, test.ShouldBeNil) + test.That(tb, ticks, test.ShouldEqual, -1) + }) + + // take off directional awareness. + enc2.m = nil + + err = ii.Tick(context.Background(), true, uint64(time.Now().UnixNano())) + test.That(t, err, test.ShouldBeNil) + + // Give the tick time to propagate to encoder + // Warning: theres a race condition if the tick has not been processed + // by the encoder worker + time.Sleep(50 * time.Millisecond) + ticks, _, err := enc.Position(context.Background(), encoder.PositionTypeUnspecified, nil) test.That(t, err, test.ShouldBeNil) test.That(t, ticks, test.ShouldEqual, 0) From d596330a2a357ec2d57eb37030fbee0d84c2f531 Mon Sep 17 00:00:00 2001 From: Cheuk <90270663+cheukt@users.noreply.github.com> Date: Tue, 3 Sep 2024 16:36:53 -0400 Subject: [PATCH 09/26] RSDK-6985 - Filter out SetHeader errors (#4338) --- operation/web.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/operation/web.go b/operation/web.go index d290312055b..aa63d1c0f7f 100644 --- a/operation/web.go +++ b/operation/web.go @@ -56,7 +56,16 @@ func (m *Manager) UnaryServerInterceptor( ctx, done := m.CreateFromIncomingContext(ctx, info.FullMethod) defer done() if op := Get(ctx); op != nil && op.ID.String() != "" { - utils.UncheckedError(grpc.SetHeader(ctx, metadata.MD{opidMetadataKey: []string{op.ID.String()}})) + // SetHeader will occasionally error because of a data race if the request has been cancelled from client side. + // The cancel signal (RST_STREAM) is processed on a separate goroutine and will close the existing gRPC stream, + // which will end up writing headers and returning a message to the server. If headers were sent before SetHeader + // is called here, SetHeader will error. + // Since the behavior is expected and part of the gRPC stream closing, only log the error if it is unexpected + // (the context error is nil, meaning request wasn't cancelled). + if err := grpc.SetHeader(ctx, metadata.MD{opidMetadataKey: []string{op.ID.String()}}); err != nil && + ctx.Err() == nil { + m.logger.CDebugw(ctx, "error while setting header", "err", err) + } } return handler(ctx, req) } From 6cfb730763a32f84a964eea7dc17626f9cf683bb Mon Sep 17 00:00:00 2001 From: Maxim Pertsov Date: Wed, 4 Sep 2024 11:49:39 -0400 Subject: [PATCH 10/26] Ignore direnv (#4345) --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index fa0bd205553..433049f9e15 100644 --- a/.gitignore +++ b/.gitignore @@ -89,3 +89,6 @@ web/cmd/server/*.core # gomobile / android *.aar *.jar + +# direnv (optional dev tool) +.envrc From 337b63168f24c0c771620fcbdf783942d36dc635 Mon Sep 17 00:00:00 2001 From: randhid <35934754+randhid@users.noreply.github.com> Date: Wed, 4 Sep 2024 12:08:25 -0400 Subject: [PATCH 11/26] Move static source to fake (#4333) --- components/camera/client.go | 15 -------- components/camera/fake/camera.go | 7 ++-- .../static.go => fake/image_file.go} | 5 ++- .../image_file_test.go} | 2 +- .../transformpipeline/depth_edges_test.go | 10 +++--- .../camera/transformpipeline/mods_test.go | 34 +++++++++---------- .../camera/transformpipeline/pipeline_test.go | 12 +++---- .../transformpipeline/undistort_test.go | 16 ++++----- vision/segmentation/color_objects_test.go | 4 +-- 9 files changed, 45 insertions(+), 60 deletions(-) rename components/camera/{videosource/static.go => fake/image_file.go} (99%) rename components/camera/{videosource/static_test.go => fake/image_file_test.go} (99%) diff --git a/components/camera/client.go b/components/camera/client.go index 69294a5b2a6..c6f04a3ca48 100644 --- a/components/camera/client.go +++ b/components/camera/client.go @@ -300,21 +300,6 @@ func (c *client) NextPointCloud(ctx context.Context) (pointcloud.PointCloud, err }() } -func (c *client) Projector(ctx context.Context) (transform.Projector, error) { - var proj transform.Projector - props, err := c.Properties(ctx) - if err != nil { - return nil, err - } - intrinsics := props.IntrinsicParams - err = intrinsics.CheckValid() - if err != nil { - return nil, err - } - proj = intrinsics - return proj, nil -} - func (c *client) Properties(ctx context.Context) (Properties, error) { result := Properties{} resp, err := c.client.GetProperties(ctx, &pb.GetPropertiesRequest{ diff --git a/components/camera/fake/camera.go b/components/camera/fake/camera.go index 2eef799b4c6..4e7d0134a43 100644 --- a/components/camera/fake/camera.go +++ b/components/camera/fake/camera.go @@ -5,6 +5,8 @@ import ( "bytes" "context" "encoding/base64" + "errors" + "fmt" "image" "image/color" "image/jpeg" @@ -17,7 +19,6 @@ import ( "github.com/bluenviron/gortsplib/v4/pkg/rtptime" "github.com/bluenviron/mediacommon/pkg/codecs/h264" "github.com/golang/geo/r3" - "github.com/pkg/errors" "go.viam.com/utils" "go.viam.com/rdk/components/camera" @@ -119,11 +120,11 @@ func (conf *Config) Validate(path string) ([]string, error) { } if conf.Height%2 != 0 { - return nil, errors.Errorf("odd-number resolutions cannot be rendered, cannot use a height of %d", conf.Height) + return nil, fmt.Errorf("odd-number resolutions cannot be rendered, cannot use a height of %d", conf.Height) } if conf.Width%2 != 0 { - return nil, errors.Errorf("odd-number resolutions cannot be rendered, cannot use a width of %d", conf.Width) + return nil, fmt.Errorf("odd-number resolutions cannot be rendered, cannot use a width of %d", conf.Width) } return nil, nil diff --git a/components/camera/videosource/static.go b/components/camera/fake/image_file.go similarity index 99% rename from components/camera/videosource/static.go rename to components/camera/fake/image_file.go index 4a7113492d8..458f3789d0f 100644 --- a/components/camera/videosource/static.go +++ b/components/camera/fake/image_file.go @@ -1,15 +1,14 @@ //go:build !no_cgo -package videosource +package fake import ( "context" + "errors" "fmt" "image" "time" - "github.com/pkg/errors" - "go.viam.com/rdk/components/camera" "go.viam.com/rdk/logging" "go.viam.com/rdk/pointcloud" diff --git a/components/camera/videosource/static_test.go b/components/camera/fake/image_file_test.go similarity index 99% rename from components/camera/videosource/static_test.go rename to components/camera/fake/image_file_test.go index 2c43c17cfa9..ab090f90a56 100644 --- a/components/camera/videosource/static_test.go +++ b/components/camera/fake/image_file_test.go @@ -1,4 +1,4 @@ -package videosource +package fake import ( "context" diff --git a/components/camera/transformpipeline/depth_edges_test.go b/components/camera/transformpipeline/depth_edges_test.go index 1db77683105..fdb193a4c7d 100644 --- a/components/camera/transformpipeline/depth_edges_test.go +++ b/components/camera/transformpipeline/depth_edges_test.go @@ -12,7 +12,7 @@ import ( "go.viam.com/utils/artifact" "go.viam.com/rdk/components/camera" - "go.viam.com/rdk/components/camera/videosource" + "go.viam.com/rdk/components/camera/fake" "go.viam.com/rdk/gostream" "go.viam.com/rdk/logging" "go.viam.com/rdk/rimage" @@ -25,7 +25,7 @@ func TestDepthSource(t *testing.T) { img, err := rimage.NewDepthMapFromFile( context.Background(), artifact.MustPath("rimage/board1_gray_small.png")) test.That(t, err, test.ShouldBeNil) - source := &videosource.StaticSource{DepthImg: img} + source := &fake.StaticSource{DepthImg: img} am := utils.AttributeMap{ "high_threshold": 0.85, "low_threshold": 0.40, @@ -57,7 +57,7 @@ func (h *depthSourceTestHelper) Process( pCtx.GotDebugImage(dm.ToPrettyPicture(0, rimage.MaxDepth), "aligned-depth") // create edge map - source := &videosource.StaticSource{DepthImg: dm} + source := &fake.StaticSource{DepthImg: dm} am := utils.AttributeMap{ "high_threshold": 0.85, "low_threshold": 0.40, @@ -78,7 +78,7 @@ func (h *depthSourceTestHelper) Process( pCtx.GotDebugPointCloud(fixedPointCloud, "aligned-pointcloud") // preprocess depth map - source = &videosource.StaticSource{DepthImg: dm} + source = &fake.StaticSource{DepthImg: dm} rs, stream, err := newDepthPreprocessTransform(context.Background(), gostream.NewVideoSource(source, prop.Video{})) test.That(t, err, test.ShouldBeNil) test.That(t, stream, test.ShouldEqual, camera.DepthStream) @@ -94,7 +94,7 @@ func (h *depthSourceTestHelper) Process( test.That(t, preprocessedPointCloud.MetaData().HasColor, test.ShouldBeFalse) pCtx.GotDebugPointCloud(preprocessedPointCloud, "preprocessed-aligned-pointcloud") - source = &videosource.StaticSource{DepthImg: preprocessed} + source = &fake.StaticSource{DepthImg: preprocessed} ds, stream, err = newDepthEdgesTransform(context.Background(), gostream.NewVideoSource(source, prop.Video{}), am) test.That(t, err, test.ShouldBeNil) test.That(t, stream, test.ShouldEqual, camera.DepthStream) diff --git a/components/camera/transformpipeline/mods_test.go b/components/camera/transformpipeline/mods_test.go index ee441e56538..721319a9bdf 100644 --- a/components/camera/transformpipeline/mods_test.go +++ b/components/camera/transformpipeline/mods_test.go @@ -10,7 +10,7 @@ import ( "go.viam.com/utils/artifact" "go.viam.com/rdk/components/camera" - "go.viam.com/rdk/components/camera/videosource" + "go.viam.com/rdk/components/camera/fake" "go.viam.com/rdk/gostream" "go.viam.com/rdk/rimage" "go.viam.com/rdk/utils" @@ -30,7 +30,7 @@ func TestCrop(t *testing.T) { test.That(t, err, test.ShouldBeNil) // test depth source - source := gostream.NewVideoSource(&videosource.StaticSource{DepthImg: dm}, prop.Video{}) + source := gostream.NewVideoSource(&fake.StaticSource{DepthImg: dm}, prop.Video{}) out, _, err := camera.ReadImage(context.Background(), source) test.That(t, err, test.ShouldBeNil) test.That(t, out.Bounds().Dx(), test.ShouldEqual, 128) @@ -48,7 +48,7 @@ func TestCrop(t *testing.T) { test.That(t, source.Close(context.Background()), test.ShouldBeNil) // test color source - source = gostream.NewVideoSource(&videosource.StaticSource{ColorImg: img}, prop.Video{}) + source = gostream.NewVideoSource(&fake.StaticSource{ColorImg: img}, prop.Video{}) out, _, err = camera.ReadImage(context.Background(), source) test.That(t, err, test.ShouldBeNil) test.That(t, out.Bounds().Dx(), test.ShouldEqual, 128) @@ -107,7 +107,7 @@ func TestResizeColor(t *testing.T) { "height_px": 20, "width_px": 30, } - source := gostream.NewVideoSource(&videosource.StaticSource{ColorImg: img}, prop.Video{}) + source := gostream.NewVideoSource(&fake.StaticSource{ColorImg: img}, prop.Video{}) out, _, err := camera.ReadImage(context.Background(), source) test.That(t, err, test.ShouldBeNil) test.That(t, out.Bounds().Dx(), test.ShouldEqual, 128) @@ -133,7 +133,7 @@ func TestResizeDepth(t *testing.T) { "height_px": 40, "width_px": 60, } - source := gostream.NewVideoSource(&videosource.StaticSource{DepthImg: img}, prop.Video{}) + source := gostream.NewVideoSource(&fake.StaticSource{DepthImg: img}, prop.Video{}) out, _, err := camera.ReadImage(context.Background(), source) test.That(t, err, test.ShouldBeNil) test.That(t, out.Bounds().Dx(), test.ShouldEqual, 128) @@ -154,7 +154,7 @@ func TestRotateColorSource(t *testing.T) { img, err := rimage.NewImageFromFile(artifact.MustPath("rimage/board1_small.png")) test.That(t, err, test.ShouldBeNil) - source := gostream.NewVideoSource(&videosource.StaticSource{ColorImg: img}, prop.Video{}) + source := gostream.NewVideoSource(&fake.StaticSource{ColorImg: img}, prop.Video{}) am := utils.AttributeMap{ "angle_degs": 180, } @@ -207,7 +207,7 @@ func TestRotateColorSource(t *testing.T) { test.That(t, rs.Close(context.Background()), test.ShouldBeNil) test.That(t, source.Close(context.Background()), test.ShouldBeNil) - source = gostream.NewVideoSource(&videosource.StaticSource{ColorImg: img}, prop.Video{}) + source = gostream.NewVideoSource(&fake.StaticSource{ColorImg: img}, prop.Video{}) am = utils.AttributeMap{ "angle_degs": 90, } @@ -238,7 +238,7 @@ func TestRotateColorSource(t *testing.T) { test.That(t, rs.Close(context.Background()), test.ShouldBeNil) test.That(t, source.Close(context.Background()), test.ShouldBeNil) - source = gostream.NewVideoSource(&videosource.StaticSource{ColorImg: img}, prop.Video{}) + source = gostream.NewVideoSource(&fake.StaticSource{ColorImg: img}, prop.Video{}) am = utils.AttributeMap{ "angle_degs": -90, } @@ -269,7 +269,7 @@ func TestRotateColorSource(t *testing.T) { test.That(t, rs.Close(context.Background()), test.ShouldBeNil) test.That(t, source.Close(context.Background()), test.ShouldBeNil) - source = gostream.NewVideoSource(&videosource.StaticSource{ColorImg: img}, prop.Video{}) + source = gostream.NewVideoSource(&fake.StaticSource{ColorImg: img}, prop.Video{}) am = utils.AttributeMap{ "angle_degs": 270, } @@ -300,7 +300,7 @@ func TestRotateColorSource(t *testing.T) { test.That(t, rs.Close(context.Background()), test.ShouldBeNil) test.That(t, source.Close(context.Background()), test.ShouldBeNil) - source = gostream.NewVideoSource(&videosource.StaticSource{ColorImg: img}, prop.Video{}) + source = gostream.NewVideoSource(&fake.StaticSource{ColorImg: img}, prop.Video{}) am = utils.AttributeMap{ "angle_degs": 0, // no-op } @@ -336,7 +336,7 @@ func TestRotateDepthSource(t *testing.T) { context.Background(), artifact.MustPath("rimage/board1_gray_small.png")) test.That(t, err, test.ShouldBeNil) - source := gostream.NewVideoSource(&videosource.StaticSource{DepthImg: pc}, prop.Video{}) + source := gostream.NewVideoSource(&fake.StaticSource{DepthImg: pc}, prop.Video{}) am := utils.AttributeMap{ "angle_degs": 180, } @@ -388,7 +388,7 @@ func TestRotateDepthSource(t *testing.T) { test.That(t, rs.Close(context.Background()), test.ShouldBeNil) test.That(t, source.Close(context.Background()), test.ShouldBeNil) - source = gostream.NewVideoSource(&videosource.StaticSource{DepthImg: pc}, prop.Video{}) + source = gostream.NewVideoSource(&fake.StaticSource{DepthImg: pc}, prop.Video{}) am = utils.AttributeMap{ "angle_degs": 90, } @@ -419,7 +419,7 @@ func TestRotateDepthSource(t *testing.T) { test.That(t, rs.Close(context.Background()), test.ShouldBeNil) test.That(t, source.Close(context.Background()), test.ShouldBeNil) - source = gostream.NewVideoSource(&videosource.StaticSource{DepthImg: pc}, prop.Video{}) + source = gostream.NewVideoSource(&fake.StaticSource{DepthImg: pc}, prop.Video{}) am = utils.AttributeMap{ "angle_degs": -90, } @@ -450,7 +450,7 @@ func TestRotateDepthSource(t *testing.T) { test.That(t, rs.Close(context.Background()), test.ShouldBeNil) test.That(t, source.Close(context.Background()), test.ShouldBeNil) - source = gostream.NewVideoSource(&videosource.StaticSource{DepthImg: pc}, prop.Video{}) + source = gostream.NewVideoSource(&fake.StaticSource{DepthImg: pc}, prop.Video{}) am = utils.AttributeMap{ "angle_degs": 270, } @@ -481,7 +481,7 @@ func TestRotateDepthSource(t *testing.T) { test.That(t, rs.Close(context.Background()), test.ShouldBeNil) test.That(t, source.Close(context.Background()), test.ShouldBeNil) - source = gostream.NewVideoSource(&videosource.StaticSource{DepthImg: pc}, prop.Video{}) + source = gostream.NewVideoSource(&fake.StaticSource{DepthImg: pc}, prop.Video{}) am = utils.AttributeMap{ "angle_degs": 0, // no-op } @@ -516,7 +516,7 @@ func BenchmarkColorRotate(b *testing.B) { img, err := rimage.NewImageFromFile(artifact.MustPath("rimage/board1.png")) test.That(b, err, test.ShouldBeNil) - source := gostream.NewVideoSource(&videosource.StaticSource{ColorImg: img}, prop.Video{}) + source := gostream.NewVideoSource(&fake.StaticSource{ColorImg: img}, prop.Video{}) src, err := camera.WrapVideoSourceWithProjector(context.Background(), source, nil, camera.ColorStream) test.That(b, err, test.ShouldBeNil) am := utils.AttributeMap{ @@ -540,7 +540,7 @@ func BenchmarkDepthRotate(b *testing.B) { context.Background(), artifact.MustPath("rimage/board1.dat.gz")) test.That(b, err, test.ShouldBeNil) - source := gostream.NewVideoSource(&videosource.StaticSource{DepthImg: img}, prop.Video{}) + source := gostream.NewVideoSource(&fake.StaticSource{DepthImg: img}, prop.Video{}) src, err := camera.WrapVideoSourceWithProjector(context.Background(), source, nil, camera.DepthStream) test.That(b, err, test.ShouldBeNil) am := utils.AttributeMap{ diff --git a/components/camera/transformpipeline/pipeline_test.go b/components/camera/transformpipeline/pipeline_test.go index 86e9bcf905b..b96ea0a9596 100644 --- a/components/camera/transformpipeline/pipeline_test.go +++ b/components/camera/transformpipeline/pipeline_test.go @@ -9,7 +9,7 @@ import ( "go.viam.com/utils/artifact" "go.viam.com/rdk/components/camera" - "go.viam.com/rdk/components/camera/videosource" + "go.viam.com/rdk/components/camera/fake" "go.viam.com/rdk/gostream" "go.viam.com/rdk/logging" "go.viam.com/rdk/resource" @@ -32,7 +32,7 @@ func TestTransformPipelineColor(t *testing.T) { img, err := rimage.NewImageFromFile(artifact.MustPath("rimage/board1_small.png")) test.That(t, err, test.ShouldBeNil) - source := gostream.NewVideoSource(&videosource.StaticSource{ColorImg: img}, prop.Video{}) + source := gostream.NewVideoSource(&fake.StaticSource{ColorImg: img}, prop.Video{}) src, err := camera.WrapVideoSourceWithProjector(context.Background(), source, nil, camera.ColorStream) test.That(t, err, test.ShouldBeNil) inImg, _, err := camera.ReadImage(context.Background(), src) @@ -78,7 +78,7 @@ func TestTransformPipelineDepth(t *testing.T) { dm, err := rimage.NewDepthMapFromFile(context.Background(), artifact.MustPath("rimage/board1_gray_small.png")) test.That(t, err, test.ShouldBeNil) - source := gostream.NewVideoSource(&videosource.StaticSource{DepthImg: dm}, prop.Video{}) + source := gostream.NewVideoSource(&fake.StaticSource{DepthImg: dm}, prop.Video{}) src, err := camera.WrapVideoSourceWithProjector(context.Background(), source, nil, camera.DepthStream) test.That(t, err, test.ShouldBeNil) inImg, _, err := camera.ReadImage(context.Background(), src) @@ -130,7 +130,7 @@ func TestTransformPipelineDepth2(t *testing.T) { dm, err := rimage.NewDepthMapFromFile( context.Background(), artifact.MustPath("rimage/board1_gray_small.png")) test.That(t, err, test.ShouldBeNil) - source := gostream.NewVideoSource(&videosource.StaticSource{DepthImg: dm}, prop.Video{}) + source := gostream.NewVideoSource(&fake.StaticSource{DepthImg: dm}, prop.Video{}) // first depth transform depth1, err := newTransformPipeline(context.Background(), source, transform1, r, logger) test.That(t, err, test.ShouldBeNil) @@ -160,7 +160,7 @@ func TestNullPipeline(t *testing.T) { img, err := rimage.NewImageFromFile(artifact.MustPath("rimage/board1_small.png")) test.That(t, err, test.ShouldBeNil) - source := gostream.NewVideoSource(&videosource.StaticSource{ColorImg: img}, prop.Video{}) + source := gostream.NewVideoSource(&fake.StaticSource{ColorImg: img}, prop.Video{}) _, err = newTransformPipeline(context.Background(), source, transform1, r, logger) test.That(t, err, test.ShouldNotBeNil) test.That(t, err.Error(), test.ShouldContainSubstring, "pipeline has no transforms") @@ -185,7 +185,7 @@ func TestPipeIntoPipe(t *testing.T) { img, err := rimage.NewImageFromFile(artifact.MustPath("rimage/board1_small.png")) test.That(t, err, test.ShouldBeNil) - source := gostream.NewVideoSource(&videosource.StaticSource{ColorImg: img}, prop.Video{}) + source := gostream.NewVideoSource(&fake.StaticSource{ColorImg: img}, prop.Video{}) intrinsics1 := &transform.PinholeCameraIntrinsics{Width: 128, Height: 72} transform1 := &transformConfig{ diff --git a/components/camera/transformpipeline/undistort_test.go b/components/camera/transformpipeline/undistort_test.go index abdf83dd2c3..41f6184bd84 100644 --- a/components/camera/transformpipeline/undistort_test.go +++ b/components/camera/transformpipeline/undistort_test.go @@ -10,7 +10,7 @@ import ( "go.viam.com/utils/artifact" "go.viam.com/rdk/components/camera" - "go.viam.com/rdk/components/camera/videosource" + "go.viam.com/rdk/components/camera/fake" "go.viam.com/rdk/gostream" "go.viam.com/rdk/rimage" "go.viam.com/rdk/rimage/transform" @@ -38,7 +38,7 @@ var undistortTestBC = &transform.BrownConrady{ func TestUndistortSetup(t *testing.T) { img, err := rimage.NewImageFromFile(artifact.MustPath("rimage/board1_small.png")) test.That(t, err, test.ShouldBeNil) - source := gostream.NewVideoSource(&videosource.StaticSource{ColorImg: img}, prop.Video{}) + source := gostream.NewVideoSource(&fake.StaticSource{ColorImg: img}, prop.Video{}) // no camera parameters am := utils.AttributeMap{} @@ -48,7 +48,7 @@ func TestUndistortSetup(t *testing.T) { test.That(t, source.Close(context.Background()), test.ShouldBeNil) // bad stream type - source = gostream.NewVideoSource(&videosource.StaticSource{ColorImg: img}, prop.Video{}) + source = gostream.NewVideoSource(&fake.StaticSource{ColorImg: img}, prop.Video{}) am = utils.AttributeMap{"intrinsic_parameters": undistortTestParams, "distortion_parameters": undistortTestBC} us, _, err := newUndistortTransform(context.Background(), source, camera.ImageType("fake"), am) test.That(t, err, test.ShouldBeNil) @@ -71,7 +71,7 @@ func TestUndistortSetup(t *testing.T) { func TestUndistortImage(t *testing.T) { img, err := rimage.NewImageFromFile(artifact.MustPath("rimage/board1_small.png")) test.That(t, err, test.ShouldBeNil) - source := gostream.NewVideoSource(&videosource.StaticSource{ColorImg: img}, prop.Video{}) + source := gostream.NewVideoSource(&fake.StaticSource{ColorImg: img}, prop.Video{}) // success am := utils.AttributeMap{"intrinsic_parameters": undistortTestParams, "distortion_parameters": undistortTestBC} @@ -92,7 +92,7 @@ func TestUndistortImage(t *testing.T) { test.That(t, result, test.ShouldResemble, expected) // bad source - source = gostream.NewVideoSource(&videosource.StaticSource{ColorImg: rimage.NewImage(10, 10)}, prop.Video{}) + source = gostream.NewVideoSource(&fake.StaticSource{ColorImg: rimage.NewImage(10, 10)}, prop.Video{}) us, stream, err = newUndistortTransform(context.Background(), source, camera.ColorStream, am) test.That(t, err, test.ShouldBeNil) test.That(t, stream, test.ShouldEqual, camera.ColorStream) @@ -105,7 +105,7 @@ func TestUndistortDepthMap(t *testing.T) { img, err := rimage.NewDepthMapFromFile( context.Background(), artifact.MustPath("rimage/board1_gray_small.png")) test.That(t, err, test.ShouldBeNil) - source := gostream.NewVideoSource(&videosource.StaticSource{DepthImg: img}, prop.Video{}) + source := gostream.NewVideoSource(&fake.StaticSource{DepthImg: img}, prop.Video{}) // success am := utils.AttributeMap{"intrinsic_parameters": undistortTestParams, "distortion_parameters": undistortTestBC} @@ -126,7 +126,7 @@ func TestUndistortDepthMap(t *testing.T) { test.That(t, result, test.ShouldResemble, expected) // bad source - source = gostream.NewVideoSource(&videosource.StaticSource{DepthImg: rimage.NewEmptyDepthMap(10, 10)}, prop.Video{}) + source = gostream.NewVideoSource(&fake.StaticSource{DepthImg: rimage.NewEmptyDepthMap(10, 10)}, prop.Video{}) us, stream, err = newUndistortTransform(context.Background(), source, camera.DepthStream, am) test.That(t, err, test.ShouldBeNil) test.That(t, stream, test.ShouldEqual, camera.DepthStream) @@ -135,7 +135,7 @@ func TestUndistortDepthMap(t *testing.T) { test.That(t, us.Close(context.Background()), test.ShouldBeNil) // can't convert image to depth map - source = gostream.NewVideoSource(&videosource.StaticSource{ColorImg: rimage.NewImage(10, 10)}, prop.Video{}) + source = gostream.NewVideoSource(&fake.StaticSource{ColorImg: rimage.NewImage(10, 10)}, prop.Video{}) us, stream, err = newUndistortTransform(context.Background(), source, camera.DepthStream, am) test.That(t, stream, test.ShouldEqual, camera.DepthStream) test.That(t, err, test.ShouldBeNil) diff --git a/vision/segmentation/color_objects_test.go b/vision/segmentation/color_objects_test.go index aeb7be47d3c..e42bf3188ce 100644 --- a/vision/segmentation/color_objects_test.go +++ b/vision/segmentation/color_objects_test.go @@ -8,7 +8,7 @@ import ( "go.viam.com/utils/artifact" "go.viam.com/rdk/components/camera" - "go.viam.com/rdk/components/camera/videosource" + "go.viam.com/rdk/components/camera/fake" "go.viam.com/rdk/rimage" "go.viam.com/rdk/rimage/transform" "go.viam.com/rdk/utils" @@ -24,7 +24,7 @@ func TestColorObjects(t *testing.T) { test.That(t, err, test.ShouldBeNil) params, err := transform.NewDepthColorIntrinsicsExtrinsicsFromJSONFile(intel515ParamsPath) test.That(t, err, test.ShouldBeNil) - c := &videosource.StaticSource{ColorImg: img, DepthImg: dm, Proj: ¶ms.ColorCamera} + c := &fake.StaticSource{ColorImg: img, DepthImg: dm, Proj: ¶ms.ColorCamera} src, err := camera.NewVideoSourceFromReader( context.Background(), c, From 8228826ca2716d6e1543f83c5a14b25641a774b7 Mon Sep 17 00:00:00 2001 From: Sagie Maoz Date: Wed, 4 Sep 2024 15:01:15 -0400 Subject: [PATCH 12/26] DATA-3040: Add retries when downloading large files (#4314) --- cli/data.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cli/data.go b/cli/data.go index bf694cb7752..1635227f928 100644 --- a/cli/data.go +++ b/cli/data.go @@ -495,7 +495,15 @@ func downloadBinary(ctx context.Context, client datapb.DataServiceClient, dst st req.Header.Add("key", apiKey.KeyCrypto) } - res, err := httpClient.Do(req) + var res *http.Response + for count := 0; count < maxRetryCount; count++ { + res, err = httpClient.Do(req) + + if err == nil && res.StatusCode == http.StatusOK { + break + } + } + if err != nil { return errors.Wrapf(err, serverErrorMessage) } From 32e80bb545332ec7f6635b5eed4af93491e1fe0e Mon Sep 17 00:00:00 2001 From: Tahiya Date: Wed, 4 Sep 2024 15:38:07 -0400 Subject: [PATCH 13/26] DATA-3102 - Make the status optional for listing training jobs (#4346) --- cli/app.go | 5 +++-- cli/ml_training.go | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/cli/app.go b/cli/app.go index 9e14116ce7a..737ff3f93da 100644 --- a/cli/app.go +++ b/cli/app.go @@ -1064,7 +1064,7 @@ var app = &cli.App{ { Name: "list", Usage: "list training jobs in Viam cloud based on organization ID", - UsageText: createUsageText("train list", []string{generalFlagOrgID, trainFlagJobStatus}, false), + UsageText: createUsageText("train list", []string{generalFlagOrgID}, true), Flags: []cli.Flag{ &cli.StringFlag{ Name: generalFlagOrgID, @@ -1074,7 +1074,8 @@ var app = &cli.App{ &cli.StringFlag{ Name: trainFlagJobStatus, Usage: "training status to filter for. can be one of " + allTrainingStatusValues(), - Required: true, + Required: false, + Value: defaultTrainingStatus(), }, }, Action: DataListTrainingJobs, diff --git a/cli/ml_training.go b/cli/ml_training.go index fe3db077c2d..d3d4a1913d3 100644 --- a/cli/ml_training.go +++ b/cli/ml_training.go @@ -305,6 +305,10 @@ func allTrainingStatusValues() string { return "[" + strings.Join(formattedStatuses, ", ") + "]" } +func defaultTrainingStatus() string { + return strings.ToLower(strings.TrimPrefix(mltrainingpb.TrainingStatus_TRAINING_STATUS_UNSPECIFIED.String(), trainingStatusPrefix)) +} + // MLTrainingUploadAction uploads a new custom training script. func MLTrainingUploadAction(c *cli.Context) error { client, err := newViamClient(c) From 220164501af9cb178e3f81a3139469e01e2995fc Mon Sep 17 00:00:00 2001 From: Julie Krasnick <99275379+jckras@users.noreply.github.com> Date: Wed, 4 Sep 2024 19:52:58 -0400 Subject: [PATCH 14/26] RSDK-5835: Remove libvpx dependency (#4342) --- etc/setup.sh | 3 +- gostream/codec/vpx/encoder.go | 94 ----------------------------------- gostream/codec/vpx/utils.go | 40 --------------- gostream/shell.nix | 2 +- 4 files changed, 2 insertions(+), 137 deletions(-) delete mode 100644 gostream/codec/vpx/encoder.go delete mode 100644 gostream/codec/vpx/utils.go diff --git a/etc/setup.sh b/etc/setup.sh index eff80e9ce83..81ad693e71a 100755 --- a/etc/setup.sh +++ b/etc/setup.sh @@ -30,7 +30,7 @@ do_piOS(){ apt-get update && apt-get install -y build-essential nodejs libnlopt-dev libx264-dev libtensorflowlite-dev ffmpeg libjpeg62-turbo-dev # Install Gostream dependencies - sudo apt-get install -y --no-install-recommends libopus-dev libvpx-dev libx11-dev libxext-dev libopusfile-dev + sudo apt-get install -y --no-install-recommends libopus-dev libx11-dev libxext-dev libopusfile-dev # Install backports apt-get install -y -t $(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)-backports golang-go @@ -173,7 +173,6 @@ do_brew(){ brew "licensefinder" brew "opus" brew "opusfile" - brew "libvpx" brew "tensorflowlite" # Needs to be last EOS diff --git a/gostream/codec/vpx/encoder.go b/gostream/codec/vpx/encoder.go deleted file mode 100644 index 20e6612cb3c..00000000000 --- a/gostream/codec/vpx/encoder.go +++ /dev/null @@ -1,94 +0,0 @@ -// Package vpx contains the vpx video codec. -package vpx - -import ( - "context" - "fmt" - "image" - - "github.com/pion/mediadevices/pkg/codec" - "github.com/pion/mediadevices/pkg/codec/vpx" - "github.com/pion/mediadevices/pkg/prop" - - ourcodec "go.viam.com/rdk/gostream/codec" - "go.viam.com/rdk/logging" -) - -type encoder struct { - codec codec.ReadCloser - img image.Image - logger logging.Logger -} - -// Version determines the version of a vpx codec. -type Version string - -// The set of allowed vpx versions. -const ( - Version8 Version = "vp8" - Version9 Version = "vp9" -) - -// Gives suitable results. Probably want to make this configurable this in the future. -const bitrate = 3_200_000 - -// NewEncoder returns a vpx encoder of the given type that can encode images of the given width and height. It will -// also ensure that it produces key frames at the given interval. -func NewEncoder(codecVersion Version, width, height, keyFrameInterval int, logger logging.Logger) (ourcodec.VideoEncoder, error) { - enc := &encoder{logger: logger} - - var builder codec.VideoEncoderBuilder - switch codecVersion { - case Version8: - params, err := vpx.NewVP8Params() - if err != nil { - return nil, err - } - builder = ¶ms - params.BitRate = bitrate - params.KeyFrameInterval = keyFrameInterval - case Version9: - params, err := vpx.NewVP9Params() - if err != nil { - return nil, err - } - builder = ¶ms - params.BitRate = bitrate - params.KeyFrameInterval = keyFrameInterval - default: - return nil, fmt.Errorf("unsupported vpx version: %s", codecVersion) - } - - codec, err := builder.BuildVideoEncoder(enc, prop.Media{ - Video: prop.Video{ - Width: width, - Height: height, - }, - }) - if err != nil { - return nil, err - } - enc.codec = codec - - return enc, nil -} - -// Read returns an image for codec to process. -func (v *encoder) Read() (img image.Image, release func(), err error) { - return v.img, nil, nil -} - -// Encode asks the codec to process the given image. -func (v *encoder) Encode(_ context.Context, img image.Image) ([]byte, error) { - v.img = img - data, release, err := v.codec.Read() - dataCopy := make([]byte, len(data)) - copy(dataCopy, data) - release() - return dataCopy, err -} - -// Close closes the encoder. -func (v *encoder) Close() error { - return v.codec.Close() -} diff --git a/gostream/codec/vpx/utils.go b/gostream/codec/vpx/utils.go deleted file mode 100644 index 5df607217e8..00000000000 --- a/gostream/codec/vpx/utils.go +++ /dev/null @@ -1,40 +0,0 @@ -package vpx - -import ( - "fmt" - - "go.viam.com/rdk/gostream" - "go.viam.com/rdk/gostream/codec" - "go.viam.com/rdk/logging" -) - -// DefaultStreamConfig configures vpx as the encoder for a stream. -var DefaultStreamConfig gostream.StreamConfig - -func init() { - DefaultStreamConfig.VideoEncoderFactory = NewEncoderFactory(Version8) -} - -// NewEncoderFactory returns a vpx factory for the given vpx codec. -func NewEncoderFactory(codecVersion Version) codec.VideoEncoderFactory { - return &factory{codecVersion} -} - -type factory struct { - codecVersion Version -} - -func (f *factory) New(width, height, keyFrameInterval int, logger logging.Logger) (codec.VideoEncoder, error) { - return NewEncoder(f.codecVersion, width, height, keyFrameInterval, logger) -} - -func (f *factory) MIMEType() string { - switch f.codecVersion { - case Version8: - return "video/vp8" - case Version9: - return "video/vp9" - default: - panic(fmt.Errorf("unknown codec version %q", f.codecVersion)) - } -} diff --git a/gostream/shell.nix b/gostream/shell.nix index 33621f0ee97..2c00f7a4662 100644 --- a/gostream/shell.nix +++ b/gostream/shell.nix @@ -4,7 +4,7 @@ pkgs.mkShell { buildInputs = - [ pkgs.which pkgs.htop pkgs.go pkgs.nodejs pkgs.pkg-config pkgs.libvpx pkgs.x264 pkgs.libopus ] + [ pkgs.which pkgs.htop pkgs.go pkgs.nodejs pkgs.pkg-config pkgs.x264 pkgs.libopus ] ++ pkgs.lib.optionals pkgs.stdenv.isDarwin [ pkgs.darwin.apple_sdk.frameworks.AVFoundation pkgs.darwin.apple_sdk.frameworks.CoreMedia From c4e714296e23d2430d09ac2dacab9c5713ea120c Mon Sep 17 00:00:00 2001 From: Cheuk <90270663+cheukt@users.noreply.github.com> Date: Thu, 5 Sep 2024 10:54:34 -0400 Subject: [PATCH 15/26] RSDK-8598 - Replace cache after sync (#4343) --- config/config.go | 30 +++++++++++++++++++++++ config/reader.go | 22 ++--------------- config/reader_test.go | 51 ++++++++++++++++++++++++++++++--------- config/watcher_test.go | 13 ++++++++++ robot/impl/local_robot.go | 16 ++++++++++-- 5 files changed, 98 insertions(+), 34 deletions(-) diff --git a/config/config.go b/config/config.go index 476ba07de64..ca708883684 100644 --- a/config/config.go +++ b/config/config.go @@ -2,6 +2,7 @@ package config import ( + "bytes" "crypto/tls" "encoding/json" "fmt" @@ -15,6 +16,7 @@ import ( "time" "github.com/pkg/errors" + "go.viam.com/utils/artifact" "go.viam.com/utils/jwks" "go.viam.com/utils/pexec" "go.viam.com/utils/rpc" @@ -68,6 +70,11 @@ type Config struct { // Revision contains the current revision of the config. Revision string + + // toCache stores the JSON marshalled version of the config to be cached. It should be a copy of + // the config pulled from cloud with minor changes. + // This version is kept because the config is changed as it moves through the system. + toCache []byte } // NOTE: This data must be maintained with what is in Config. @@ -238,6 +245,29 @@ func (c Config) FindComponent(name string) *resource.Config { return nil } +// SetToCache sets toCache with a marshalled copy of the config passed in. +func (c *Config) SetToCache(cfg *Config) error { + md, err := json.MarshalIndent(cfg, "", " ") + if err != nil { + return err + } + c.toCache = md + return nil +} + +// StoreToCache caches the toCache. +func (c *Config) StoreToCache() error { + if c.toCache == nil { + return errors.New("no unprocessed config to cache") + } + if err := os.MkdirAll(ViamDotDir, 0o700); err != nil { + return err + } + reader := bytes.NewReader(c.toCache) + path := getCloudCacheFilePath(c.Cloud.ID) + return artifact.AtomicStore(path, reader, c.Cloud.ID) +} + // UnmarshalJSON unmarshals JSON into the config and adjusts some // names if they are not fully filled in. func (c *Config) UnmarshalJSON(data []byte) error { diff --git a/config/reader.go b/config/reader.go index d0cedc7eeae..b0452a40b7a 100644 --- a/config/reader.go +++ b/config/reader.go @@ -17,7 +17,6 @@ import ( "github.com/pkg/errors" apppb "go.viam.com/api/app/v1" "go.viam.com/utils" - "go.viam.com/utils/artifact" "go.viam.com/utils/rpc" "golang.org/x/sys/cpu" @@ -114,22 +113,6 @@ func readFromCache(id string) (*Config, error) { return unprocessedConfig, nil } -func storeToCache(id string, cfg *Config) error { - if err := os.MkdirAll(ViamDotDir, 0o700); err != nil { - return err - } - - md, err := json.MarshalIndent(cfg, "", " ") - if err != nil { - return err - } - reader := bytes.NewReader(md) - - path := getCloudCacheFilePath(id) - - return artifact.AtomicStore(path, reader, id) -} - func clearCache(id string) { utils.UncheckedErrorFunc(func() error { return os.Remove(getCloudCacheFilePath(id)) @@ -318,10 +301,9 @@ func readFromCloud( unprocessedConfig.Cloud.TLSCertificate = tls.certificate unprocessedConfig.Cloud.TLSPrivateKey = tls.privateKey - if err := storeToCache(cloudCfg.ID, unprocessedConfig); err != nil { - logger.Errorw("failed to cache config", "error", err) + if err := cfg.SetToCache(unprocessedConfig); err != nil { + logger.Errorw("failed to set toCache on config", "error", err) } - return cfg, nil } diff --git a/config/reader_test.go b/config/reader_test.go index 495bda36e17..570d7fcb5d8 100644 --- a/config/reader_test.go +++ b/config/reader_test.go @@ -2,7 +2,6 @@ package config import ( "context" - "errors" "fmt" "io/fs" "os" @@ -11,6 +10,7 @@ import ( "time" "github.com/google/uuid" + "github.com/pkg/errors" pb "go.viam.com/api/app/v1" "go.viam.com/test" @@ -69,7 +69,6 @@ func TestFromReader(t *testing.T) { appAddress := fmt.Sprintf("http://%s", fakeServer.Addr().String()) cfgText := fmt.Sprintf(`{"cloud":{"id":%q,"app_address":%q,"secret":%q}}`, robotPartID, appAddress, secret) gotCfg, err := FromReader(ctx, "", strings.NewReader(cfgText), logger) - defer clearCache(robotPartID) test.That(t, err, test.ShouldBeNil) expectedCloud := *cloudResponse @@ -79,6 +78,8 @@ func TestFromReader(t *testing.T) { expectedCloud.RefreshInterval = time.Duration(10000000000) test.That(t, gotCfg.Cloud, test.ShouldResemble, &expectedCloud) + test.That(t, gotCfg.StoreToCache(), test.ShouldBeNil) + defer clearCache(robotPartID) cachedCfg, err := readFromCache(robotPartID) test.That(t, err, test.ShouldBeNil) expectedCloud.AppAddress = "" @@ -102,7 +103,10 @@ func TestFromReader(t *testing.T) { MachineID: "the-machine", } cachedConf := &Config{Cloud: cachedCloud} - err := storeToCache(robotPartID, cachedConf) + + cfgToCache := &Config{Cloud: &Cloud{ID: robotPartID}} + cfgToCache.SetToCache(cachedConf) + err := cfgToCache.StoreToCache() test.That(t, err, test.ShouldBeNil) defer clearCache(robotPartID) @@ -153,7 +157,6 @@ func TestFromReader(t *testing.T) { appAddress := fmt.Sprintf("http://%s", fakeServer.Addr().String()) cfgText := fmt.Sprintf(`{"cloud":{"id":%q,"app_address":%q,"secret":%q}}`, robotPartID, appAddress, secret) gotCfg, err := FromReader(ctx, "", strings.NewReader(cfgText), logger) - defer clearCache(robotPartID) test.That(t, err, test.ShouldBeNil) expectedCloud := *cloudResponse @@ -161,6 +164,9 @@ func TestFromReader(t *testing.T) { expectedCloud.RefreshInterval = time.Duration(10000000000) test.That(t, gotCfg.Cloud, test.ShouldResemble, &expectedCloud) + err = gotCfg.StoreToCache() + defer clearCache(robotPartID) + test.That(t, err, test.ShouldBeNil) cachedCfg, err := readFromCache(robotPartID) test.That(t, err, test.ShouldBeNil) expectedCloud.AppAddress = "" @@ -191,13 +197,20 @@ func TestStoreToCache(t *testing.T) { } cfg.Cloud = cloud - // store our config to the cloud - err = storeToCache(cfg.Cloud.ID, cfg) + // errors if no unprocessed config to cache + cfgToCache := &Config{Cloud: &Cloud{ID: "forCachingTest"}} + err = cfgToCache.StoreToCache() + test.That(t, err.Error(), test.ShouldContainSubstring, "no unprocessed config to cache") + + // store our config to the cache + cfgToCache.SetToCache(cfg) + err = cfgToCache.StoreToCache() test.That(t, err, test.ShouldBeNil) // read config from cloud, confirm consistency cloudCfg, err := readFromCloud(ctx, cfg, nil, true, false, logger) test.That(t, err, test.ShouldBeNil) + cloudCfg.toCache = nil test.That(t, cloudCfg, test.ShouldResemble, cfg) // Modify our config @@ -207,10 +220,12 @@ func TestStoreToCache(t *testing.T) { // read config from cloud again, confirm that the cached config differs from cfg cloudCfg2, err := readFromCloud(ctx, cfg, nil, true, false, logger) test.That(t, err, test.ShouldBeNil) - test.That(t, cloudCfg2, test.ShouldNotResemble, cfg) + cloudCfg2.toCache = nil + test.That(t, cloudCfg2, test.ShouldNotResemble, cfgToCache) // store the updated config to the cloud - err = storeToCache(cfg.Cloud.ID, cfg) + cfgToCache.SetToCache(cfg) + err = cfgToCache.StoreToCache() test.That(t, err, test.ShouldBeNil) test.That(t, cfg.Ensure(true, logger), test.ShouldBeNil) @@ -218,6 +233,7 @@ func TestStoreToCache(t *testing.T) { // read updated cloud config, confirm that it now matches our updated cfg cloudCfg3, err := readFromCloud(ctx, cfg, nil, true, false, logger) test.That(t, err, test.ShouldBeNil) + cloudCfg3.toCache = nil test.That(t, cloudCfg3, test.ShouldResemble, cfg) } @@ -304,7 +320,9 @@ func TestReadTLSFromCache(t *testing.T) { defer clearCache(robotPartID) cfg.Cloud = nil - err = storeToCache(robotPartID, cfg) + cfgToCache := &Config{Cloud: &Cloud{ID: robotPartID}} + cfgToCache.SetToCache(cfg) + err = cfgToCache.StoreToCache() test.That(t, err, test.ShouldBeNil) tls := tlsConfig{} @@ -315,11 +333,14 @@ func TestReadTLSFromCache(t *testing.T) { t.Run("invalid cached TLS", func(t *testing.T) { defer clearCache(robotPartID) cloud := &Cloud{ + ID: robotPartID, TLSPrivateKey: "key", } cfg.Cloud = cloud - err = storeToCache(robotPartID, cfg) + cfgToCache := &Config{Cloud: &Cloud{ID: robotPartID}} + cfgToCache.SetToCache(cfg) + err = cfgToCache.StoreToCache() test.That(t, err, test.ShouldBeNil) tls := tlsConfig{} @@ -333,12 +354,15 @@ func TestReadTLSFromCache(t *testing.T) { t.Run("invalid cached TLS but insecure signaling", func(t *testing.T) { defer clearCache(robotPartID) cloud := &Cloud{ + ID: robotPartID, TLSPrivateKey: "key", SignalingInsecure: true, } cfg.Cloud = cloud - err = storeToCache(robotPartID, cfg) + cfgToCache := &Config{Cloud: &Cloud{ID: robotPartID}} + cfgToCache.SetToCache(cfg) + err = cfgToCache.StoreToCache() test.That(t, err, test.ShouldBeNil) tls := tlsConfig{} @@ -352,12 +376,15 @@ func TestReadTLSFromCache(t *testing.T) { t.Run("valid cached TLS", func(t *testing.T) { defer clearCache(robotPartID) cloud := &Cloud{ + ID: robotPartID, TLSCertificate: "cert", TLSPrivateKey: "key", } cfg.Cloud = cloud - err = storeToCache(robotPartID, cfg) + cfgToCache := &Config{Cloud: &Cloud{ID: robotPartID}} + cfgToCache.SetToCache(cfg) + err = cfgToCache.StoreToCache() test.That(t, err, test.ShouldBeNil) // the config is missing several fields required to start the robot, but this diff --git a/config/watcher_test.go b/config/watcher_test.go index 3cb7df459a0..d1fcdf9c36c 100644 --- a/config/watcher_test.go +++ b/config/watcher_test.go @@ -259,6 +259,16 @@ func TestNewWatcherCloud(t *testing.T) { }}, } + unprocessedFromCfg := func(cfg config.Config) *config.Config { + // the unprocessed config uses the original config read from the cloud, + // and the cloud config is missing a few fields in the proto, meaning a few fields need to be cleared out. + unprocessed, err := cfg.CopyOnlyPublicFields() + test.That(t, err, test.ShouldBeNil) + unprocessed.Cloud.AppAddress = "" + unprocessed.Cloud.RefreshInterval = 10 * time.Second + return unprocessed + } + storeConfigInServer(confToReturn) watcher, err := config.NewWatcher(context.Background(), &config.Config{Cloud: newCloudConf()}, logger) @@ -268,6 +278,7 @@ func TestNewWatcherCloud(t *testing.T) { confToExpect.Cloud.TLSCertificate = certsToReturn.TLSCertificate confToExpect.Cloud.TLSPrivateKey = certsToReturn.TLSPrivateKey test.That(t, confToExpect.Ensure(true, logger), test.ShouldBeNil) + confToExpect.SetToCache(unprocessedFromCfg(confToExpect)) newConf := <-watcher.Config() test.That(t, newConf, test.ShouldResemble, &confToExpect) @@ -305,6 +316,7 @@ func TestNewWatcherCloud(t *testing.T) { confToExpect.Cloud.TLSCertificate = certsToReturn.TLSCertificate confToExpect.Cloud.TLSPrivateKey = certsToReturn.TLSPrivateKey test.That(t, confToExpect.Ensure(true, logger), test.ShouldBeNil) + confToExpect.SetToCache(unprocessedFromCfg(confToExpect)) newConf = <-watcher.Config() test.That(t, newConf, test.ShouldResemble, &confToExpect) @@ -356,6 +368,7 @@ func TestNewWatcherCloud(t *testing.T) { confToExpect.Cloud.TLSCertificate = certsToReturn.TLSCertificate confToExpect.Cloud.TLSPrivateKey = certsToReturn.TLSPrivateKey test.That(t, confToExpect.Ensure(true, logger), test.ShouldBeNil) + confToExpect.SetToCache(unprocessedFromCfg(confToExpect)) newConf = <-watcher.Config() test.That(t, newConf, test.ShouldResemble, &confToExpect) diff --git a/robot/impl/local_robot.go b/robot/impl/local_robot.go index cbc1408006c..431aab4b6c2 100644 --- a/robot/impl/local_robot.go +++ b/robot/impl/local_robot.go @@ -1190,14 +1190,26 @@ func (r *localRobot) reconfigure(ctx context.Context, newConfig *config.Config, // if anything has changed. err := r.packageManager.Sync(ctx, newConfig.Packages, newConfig.Modules) if err != nil { - allErrs = multierr.Combine(allErrs, err) + r.Logger().CErrorw(ctx, "reconfiguration aborted because cloud modules or packages download failed", "error", err) + return } // For local tarball modules, we create synthetic versions for package management. The `localRobot` keeps track of these because // config reader would overwrite if we just stored it in config. Here, we copy the synthetic version from the `localRobot` into the // appropriate `config.Module` object inside the `cfg.Modules` slice. Thus, when a local tarball module is reloaded, the viam-server // can unpack it into a fresh directory rather than reusing the previous one. r.applyLocalModuleVersions(newConfig) - allErrs = multierr.Combine(allErrs, r.localPackages.Sync(ctx, newConfig.Packages, newConfig.Modules)) + err = r.localPackages.Sync(ctx, newConfig.Packages, newConfig.Modules) + if err != nil { + r.Logger().CErrorw(ctx, "reconfiguration aborted because local modules or packages sync failed", "error", err) + return + } + + if newConfig.Cloud != nil { + r.Logger().CDebug(ctx, "updating cached config") + if err := newConfig.StoreToCache(); err != nil { + r.logger.CErrorw(ctx, "error storing the config", "error", err) + } + } // Add default services and process their dependencies. Dependencies may // already come from config validation so we check that here. From 8b7eb000e4999ff6b85ccda0ecdd6315604cded8 Mon Sep 17 00:00:00 2001 From: Benjamin Rewis <32186188+benjirewis@users.noreply.github.com> Date: Thu, 5 Sep 2024 11:10:54 -0400 Subject: [PATCH 16/26] RSDK-8631 Bump goutils and webrtc versions (#4347) --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 41ee05178fa..e9c4f9c4d99 100644 --- a/go.mod +++ b/go.mod @@ -75,7 +75,7 @@ require ( github.com/u2takey/ffmpeg-go v0.4.1 github.com/urfave/cli/v2 v2.10.3 github.com/viamrobotics/evdev v0.1.3 - github.com/viamrobotics/webrtc/v3 v3.99.9 + github.com/viamrobotics/webrtc/v3 v3.99.10 github.com/xfmoulet/qoi v0.2.0 go-hep.org/x/hep v0.32.1 go.mongodb.org/mongo-driver v1.11.6 @@ -85,7 +85,7 @@ require ( go.uber.org/zap v1.24.0 go.viam.com/api v0.1.336 go.viam.com/test v1.1.1-0.20220913152726-5da9916c08a2 - go.viam.com/utils v0.1.97 + go.viam.com/utils v0.1.98 goji.io v2.0.2+incompatible golang.org/x/image v0.19.0 golang.org/x/mobile v0.0.0-20240112133503-c713f31d574b diff --git a/go.sum b/go.sum index faa94e5fc8e..4e547a1cc61 100644 --- a/go.sum +++ b/go.sum @@ -1439,8 +1439,8 @@ github.com/valyala/quicktemplate v1.6.3/go.mod h1:fwPzK2fHuYEODzJ9pkw0ipCPNHZ2tD github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/viamrobotics/evdev v0.1.3 h1:mR4HFafvbc5Wx4Vp1AUJp6/aITfVx9AKyXWx+rWjpfc= github.com/viamrobotics/evdev v0.1.3/go.mod h1:N6nuZmPz7HEIpM7esNWwLxbYzqWqLSZkfI/1Sccckqk= -github.com/viamrobotics/webrtc/v3 v3.99.9 h1:5FCctlMhO9lr4SJ1TC2WCFocBIriUMb3Sw7i9oDlz2o= -github.com/viamrobotics/webrtc/v3 v3.99.9/go.mod h1:ziH7/S52IyYAeDdwUUl5ZTbuyKe47fWorAz+0z5w6NA= +github.com/viamrobotics/webrtc/v3 v3.99.10 h1:ykE14wm+HkqMD5Ozq4rvhzzfvnXAu14ak/HzA1OCzfY= +github.com/viamrobotics/webrtc/v3 v3.99.10/go.mod h1:ziH7/S52IyYAeDdwUUl5ZTbuyKe47fWorAz+0z5w6NA= github.com/viki-org/dnscache v0.0.0-20130720023526-c70c1f23c5d8/go.mod h1:dniwbG03GafCjFohMDmz6Zc6oCuiqgH6tGNyXTkHzXE= github.com/wcharczuk/go-chart/v2 v2.1.0/go.mod h1:yx7MvAVNcP/kN9lKXM/NTce4au4DFN99j6i1OwDclNA= github.com/wlynxg/anet v0.0.3 h1:PvR53psxFXstc12jelG6f1Lv4MWqE0tI76/hHGjh9rg= @@ -1541,8 +1541,8 @@ go.viam.com/api v0.1.336 h1:mcz3Y5rivgXhsTu/bXkAVDw0/otarq3lCPIRcxhNnIY= go.viam.com/api v0.1.336/go.mod h1:msa4TPrMVeRDcG4YzKA/S6wLEUC7GyHQE973JklrQ10= go.viam.com/test v1.1.1-0.20220913152726-5da9916c08a2 h1:oBiK580EnEIzgFLU4lHOXmGAE3MxnVbeR7s1wp/F3Ps= go.viam.com/test v1.1.1-0.20220913152726-5da9916c08a2/go.mod h1:XM0tej6riszsiNLT16uoyq1YjuYPWlRBweTPRDanIts= -go.viam.com/utils v0.1.97 h1:ZSmasJJkToHEbHJQ2xJbBjE5j+qlxFN9gDhad8Wex4M= -go.viam.com/utils v0.1.97/go.mod h1:GCaRsDlW3p3tOnLo73swpPnfrmmqg6r+1oWhNVEHsDo= +go.viam.com/utils v0.1.98 h1:ZW8C4AcsbxM5FtFwOI9DyYMwQTVCt4pKYfNbdT5anAs= +go.viam.com/utils v0.1.98/go.mod h1:DnwL2Q5zzS9Z2ZlkNzLmormDO/ThxjnucxNXn6r6fM0= go4.org/unsafe/assume-no-moving-gc v0.0.0-20230525183740-e7c30c78aeb2 h1:WJhcL4p+YeDxmZWg141nRm7XC8IDmhz7lk5GpadO1Sg= go4.org/unsafe/assume-no-moving-gc v0.0.0-20230525183740-e7c30c78aeb2/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E= gocv.io/x/gocv v0.25.0/go.mod h1:Rar2PS6DV+T4FL+PM535EImD/h13hGVaHhnCu1xarBs= From a27b1cf3b416134bdf687f10ff59fc17ae7a7366 Mon Sep 17 00:00:00 2001 From: abe-winter Date: Thu, 5 Sep 2024 11:42:56 -0400 Subject: [PATCH 17/26] bump min golang (#4349) --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index e9c4f9c4d99..1f8a4b88efb 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module go.viam.com/rdk -go 1.21 +go 1.21.13 require ( github.com/AlekSi/gocov-xml v1.0.0 From 33d9f80ccf2290a4e0a68296106b96b34d7b7aa7 Mon Sep 17 00:00:00 2001 From: Dan Gottlieb Date: Fri, 6 Sep 2024 12:29:08 -0400 Subject: [PATCH 18/26] RSDK-7403: Improve remote camera clients. (#4294) Co-authored-by: nicksanford --- components/camera/client.go | 503 +++++++++++--------- components/camera/client_test.go | 519 +++++++++++++++++++++ gostream/stream.go | 3 + grpc/conn.go | 42 +- grpc/shared_conn.go | 55 ++- grpc/tracker.go | 9 + grpc/tracker_test.go | 20 + robot/client/client.go | 6 +- robot/impl/local_robot_test.go | 228 --------- robot/impl/resource_manager.go | 28 +- robot/web/stream/camera/camera.go | 21 + robot/web/stream/server.go | 259 +++++++---- robot/web/stream/state/state.go | 476 ++++++++----------- robot/web/stream/state/state_test.go | 662 ++++++++++++--------------- robot/web/stream/stream.go | 32 +- robot/web/stream/stream_test.go | 116 ----- robot/web/web_c.go | 12 +- 17 files changed, 1640 insertions(+), 1351 deletions(-) create mode 100644 grpc/tracker.go create mode 100644 grpc/tracker_test.go create mode 100644 robot/web/stream/camera/camera.go diff --git a/components/camera/client.go b/components/camera/client.go index c6f04a3ca48..1f296876859 100644 --- a/components/camera/client.go +++ b/components/camera/client.go @@ -3,14 +3,16 @@ package camera import ( "bytes" "context" + "errors" "fmt" "image" "io" - "slices" + "os" "sync" + "sync/atomic" + "time" "github.com/pion/rtp" - "github.com/pkg/errors" "github.com/viamrobotics/webrtc/v3" "go.opencensus.io/trace" pb "go.viam.com/api/component/camera/v1" @@ -18,13 +20,14 @@ import ( goutils "go.viam.com/utils" goprotoutils "go.viam.com/utils/protoutils" "go.viam.com/utils/rpc" + "golang.org/x/exp/slices" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/structpb" "go.viam.com/rdk/components/camera/rtppassthrough" "go.viam.com/rdk/data" "go.viam.com/rdk/gostream" - rdkgrpc "go.viam.com/rdk/grpc" + "go.viam.com/rdk/grpc" "go.viam.com/rdk/logging" "go.viam.com/rdk/pointcloud" "go.viam.com/rdk/protoutils" @@ -36,28 +39,24 @@ import ( var ( // ErrNoPeerConnection indicates there was no peer connection. - ErrNoPeerConnection = errors.New("No PeerConnection") + ErrNoPeerConnection = errors.New("no PeerConnection") // ErrNoSharedPeerConnection indicates there was no shared peer connection. - ErrNoSharedPeerConnection = errors.New("No Shared PeerConnection") + ErrNoSharedPeerConnection = errors.New("no Shared PeerConnection") // ErrUnknownSubscriptionID indicates that a SubscriptionID is unknown. - ErrUnknownSubscriptionID = errors.New("SubscriptionID Unknown") + ErrUnknownSubscriptionID = errors.New("subscriptionID Unknown") + readRTPTimeout = time.Millisecond * 200 ) -type ( - singlePacketCallback func(*rtp.Packet) - bufAndCB struct { - cb singlePacketCallback - buf *rtppassthrough.Buffer - } - bufAndCBByID map[rtppassthrough.SubscriptionID]bufAndCB -) +type bufAndCB struct { + cb func(*rtp.Packet) + buf *rtppassthrough.Buffer +} // client implements CameraServiceClient. type client struct { resource.Named resource.TriviallyReconfigurable - ctx context.Context - cancelFn context.CancelFunc + remoteName string name string conn rpc.ClientConn client pb.CameraServiceClient @@ -65,12 +64,14 @@ type client struct { logger logging.Logger activeBackgroundWorkers sync.WaitGroup - mu sync.Mutex - healthyClientCh chan struct{} - bufAndCBByID bufAndCBByID - currentSubParentID rtppassthrough.SubscriptionID - subParentToChildren map[rtppassthrough.SubscriptionID][]rtppassthrough.SubscriptionID - trackClosed <-chan struct{} + healthyClientChMu sync.Mutex + healthyClientCh chan struct{} + + rtpPassthroughMu sync.Mutex + runningStreams map[rtppassthrough.SubscriptionID]bufAndCB + subGenerationID int + associatedSubs map[int][]rtppassthrough.SubscriptionID + trackClosed <-chan struct{} } // NewClientFromConn constructs a new Client from connection passed in. @@ -85,19 +86,17 @@ func NewClientFromConn( streamClient := streampb.NewStreamServiceClient(conn) trackClosed := make(chan struct{}) close(trackClosed) - closeCtx, cancelFn := context.WithCancel(context.Background()) return &client{ - ctx: closeCtx, - cancelFn: cancelFn, - Named: name.PrependRemote(remoteName).AsNamed(), - name: name.ShortName(), - conn: conn, - streamClient: streamClient, - client: c, - bufAndCBByID: map[rtppassthrough.SubscriptionID]bufAndCB{}, - trackClosed: trackClosed, - subParentToChildren: map[rtppassthrough.SubscriptionID][]rtppassthrough.SubscriptionID{}, - logger: logger, + remoteName: remoteName, + Named: name.PrependRemote(remoteName).AsNamed(), + name: name.ShortName(), + conn: conn, + streamClient: streamClient, + client: c, + runningStreams: map[rtppassthrough.SubscriptionID]bufAndCB{}, + trackClosed: trackClosed, + associatedSubs: map[int][]rtppassthrough.SubscriptionID{}, + logger: logger, }, nil } @@ -181,12 +180,7 @@ func (c *client) Stream( // New streams concurrent with closing cannot start until this drain completes. There // will never be stream goroutines from the old "generation" running concurrently // with those from the new "generation". - c.mu.Lock() - if c.healthyClientCh == nil { - c.healthyClientCh = make(chan struct{}) - } - healthyClientCh := c.healthyClientCh - c.mu.Unlock() + healthyClientCh := c.maybeResetHealthyClientCh() ctxWithMIME := gostream.WithMIMETypeHint(context.Background(), gostream.MIMETypeHint(ctx, "")) streamCtx, stream, frameCh := gostream.NewMediaStreamForChannel[image.Image](ctxWithMIME) @@ -240,7 +234,7 @@ func (c *client) Images(ctx context.Context) ([]NamedImage, resource.ResponseMet Name: c.name, }) if err != nil { - return nil, resource.ResponseMetadata{}, errors.Wrap(err, "camera client: could not gets images from the camera") + return nil, resource.ResponseMetadata{}, fmt.Errorf("camera client: could not gets images from the camera %w", err) } images := make([]NamedImage, 0, len(resp.Images)) @@ -349,35 +343,38 @@ func (c *client) Close(ctx context.Context) error { _, span := trace.StartSpan(ctx, "camera::client::Close") defer span.End() - c.cancelFn() - c.mu.Lock() - + c.healthyClientChMu.Lock() if c.healthyClientCh != nil { close(c.healthyClientCh) } c.healthyClientCh = nil + c.healthyClientChMu.Unlock() + // unsubscribe from all video streams that have been established with modular cameras c.unsubscribeAll() // NOTE: (Nick S) we are intentionally releasing the lock before we wait for // background goroutines to terminate as some of them need to be able // to take the lock to terminate - c.mu.Unlock() c.activeBackgroundWorkers.Wait() return nil } -// SubscribeRTP begins a subscription to receive RTP packets. -// When the Subscription terminates the context in the returned Subscription -// is cancelled. -// It maintains the invariant that there is at most a single track between -// the client and WebRTC peer. -// It is strongly recommended to set a timeout on ctx as the underlying -// WebRTC peer connection's Track can be removed before sending an RTP packet which -// results in SubscribeRTP blocking until the provided context is cancelled or the client -// is closed. -// This rare condition may happen in cases like reconfiguring concurrently with -// a SubscribeRTP call. +func (c *client) maybeResetHealthyClientCh() chan struct{} { + c.healthyClientChMu.Lock() + defer c.healthyClientChMu.Unlock() + if c.healthyClientCh == nil { + c.healthyClientCh = make(chan struct{}) + } + return c.healthyClientCh +} + +// SubscribeRTP begins a subscription to receive RTP packets. SubscribeRTP waits for the VideoTrack +// to be available before returning. The subscription/video is valid until the +// `Subscription.Terminated` context is `Done`. +// +// SubscribeRTP maintains the invariant that there is at most a single track between the client and +// WebRTC peer. Concurrent callers will block and wait for the "winner" to complete. func (c *client) SubscribeRTP( ctx context.Context, bufferSize int, @@ -385,271 +382,355 @@ func (c *client) SubscribeRTP( ) (rtppassthrough.Subscription, error) { ctx, span := trace.StartSpan(ctx, "camera::client::SubscribeRTP") defer span.End() - c.mu.Lock() - defer c.mu.Unlock() - sub, buf, err := rtppassthrough.NewSubscription(bufferSize) + // RSDK-6340: The resource manager closes remote resources when the underlying + // connection goes bad. However, when the connection is re-established, the client + // objects these resources represent are not re-initialized/marked "healthy". + // `healthyClientCh` helps track these transitions between healthy and unhealthy + // states. + // + // When `client.SubscribeRTP()` is called we will either use the existing + // `healthyClientCh` or create a new one. + // + // The goroutine a `Tracker.AddOnTrackSub()` method spins off will listen to its version of the + // `healthyClientCh` to be notified when the connection has died so it can gracefully + // terminate. + // + // When a connection becomes unhealthy, the resource manager will call `Close` on the + // camera client object. Closing the client will: + // 1. close its `client.healthyClientCh` channel + // 2. wait for existing "SubscribeRTP" goroutines to drain + // 3. nil out the `client.healthyClientCh` member variable + // + // New streams concurrent with closing cannot start until this drain completes. There + // will never be stream goroutines from the old "generation" running concurrently + // with those from the new "generation". + healthyClientCh := c.maybeResetHealthyClientCh() + + c.rtpPassthroughMu.Lock() + defer c.rtpPassthroughMu.Unlock() + + // Create a Subscription object and allocate a ring buffer of RTP packets. + sub, rtpPacketBuffer, err := rtppassthrough.NewSubscription(bufferSize) if err != nil { return sub, err } g := utils.NewGuard(func() { - buf.Close() + c.logger.CInfo(ctx, "Error subscribing to RTP. Closing passthrough buffer.") + rtpPacketBuffer.Close() }) defer g.OnFail() + // RTP Passthrough only works over PeerConnection video tracks. if c.conn.PeerConn() == nil { return rtppassthrough.NilSubscription, ErrNoPeerConnection } - // check if we have established a connection that can be shared by multiple clients asking for cameras streams from viam server. - sc, ok := c.conn.(*rdkgrpc.SharedConn) + // Calling `AddStream(streamId)` on the remote viam-server/module will eventually result in + // invoking a local `PeerConnection.OnTrack(streamId, rtpReceiver)` callback. In this callback + // we set up the goroutine to read video data from the remote side and copy those packets + // upstream. + // + // webrtc.PeerConnection objects can only install a single `OnTrack` callback. Because there + // could be multiple video stream sources on the other side of a PeerConnection, we must play a + // game where we only create one `OnTrack` callback, but that callback object does a map lookup + // for which video stream. The `grpc.Tracker` API is for manipulating that map. + tracker, ok := c.conn.(grpc.Tracker) if !ok { + c.logger.CErrorw(ctx, "Client conn is not a `Tracker`", "connType", fmt.Sprintf("%T", c.conn)) return rtppassthrough.NilSubscription, ErrNoSharedPeerConnection } - c.logger.CDebugw(ctx, "SubscribeRTP", "subID", sub.ID.String(), "name", c.Name(), "bufAndCBByID", c.bufAndCBByID.String()) + c.logger.CDebugw(ctx, "SubscribeRTP", "subID", sub.ID.String(), "name", c.Name(), "bufAndCBByID", c.bufAndCBToString()) defer func() { c.logger.CDebugw(ctx, "SubscribeRTP after", "subID", sub.ID.String(), - "name", c.Name(), "bufAndCBByID", c.bufAndCBByID.String()) + "name", c.Name(), "bufAndCBByID", c.bufAndCBToString()) }() - // B/c there is only ever either 0 or 1 peer connections between a module & a viam-server - // once AddStream is called on the module for a given camera model instance & succeeds, we shouldn't - // call it again until the previous track is terminated (by calling RemoveStream) for a few reasons: - // 1. doing so would result in 2 webrtc tracks for the same camera sending the exact same RTP packets which would - // needlessly waste resources - // 2. b/c the signature of RemoveStream just takes the camera name, if there are 2 streams for the same camera - // & the module receives a call to RemoveStream, there is no way for the module to know which camera stream - // should be removed - if len(c.bufAndCBByID) == 0 { - // Wait for previous track to terminate or for the client to close + + // If we do not currently have a video stream open for this camera, we create one. Otherwise + // we'll piggy back on the existing stream. We piggy-back for two reasons: + // 1. Not doing so would result in two webrtc tracks for the same camera sending the exact same + // RTP packets. This would needlessly waste resources. + // 2. The signature of `RemoveStream` just takes the camera name. If we had two streams open for + // the same camera when the remote receives `RemoveStream`, it wouldn't know which to stop + // sending data for. + if len(c.runningStreams) == 0 { + c.logger.CInfow(ctx, "SubscribeRTP is creating a video track", + "client", fmt.Sprintf("%p", c), "peerConnection", fmt.Sprintf("%p", c.conn.PeerConn())) + // A previous subscriber/track may have exited, but its resources have not necessarily been + // cleaned up. We must wait for that to complete. We additionally select on other error + // conditions. select { - case <-ctx.Done(): - err := errors.Wrap(ctx.Err(), "Track not closed within SubscribeRTP provided context") - c.logger.Error(err) - return rtppassthrough.NilSubscription, err - case <-c.ctx.Done(): - err := errors.Wrap(c.ctx.Err(), "Track not closed before client closed") - c.logger.Error(err) - return rtppassthrough.NilSubscription, err case <-c.trackClosed: + case <-ctx.Done(): + return rtppassthrough.NilSubscription, fmt.Errorf("track not closed within SubscribeRTP provided context %w", ctx.Err()) + case <-healthyClientCh: + return rtppassthrough.NilSubscription, errors.New("camera client is closed") } + + // Create channels for two important events: when a video track is established an when a + // video track exits. I.e: When the `OnTrack` callback is invoked and when the `RTPReceiver` + // is no longer being read from. trackReceived, trackClosed := make(chan struct{}), make(chan struct{}) - // add the camera model's addOnTrackSubFunc to the shared peer connection's - // slice of OnTrack callbacks. This is what allows - // all the bufAndCBByID's callback functions to be called with the - // RTP packets from the module's peer connection's track - sc.AddOnTrackSub(c.Name(), c.addOnTrackSubFunc(trackReceived, trackClosed, sub.ID)) - // remove the OnTrackSub once we either fail or succeed - defer sc.RemoveOnTrackSub(c.Name()) - if _, err := c.streamClient.AddStream(ctx, &streampb.AddStreamRequest{Name: c.Name().String()}); err != nil { - c.logger.CDebugw(ctx, "SubscribeRTP AddStream hit error", "subID", sub.ID.String(), "name", c.Name(), "err", err) + + // We're creating a new video track. Bump the generation ID and associate all new + // subscriptions to this generation. + c.subGenerationID++ + c.associatedSubs[c.subGenerationID] = []rtppassthrough.SubscriptionID{} + + // Add the camera's addOnTrackSubFunc to the SharedConn's map of OnTrack callbacks. + tracker.AddOnTrackSub(c.trackName(), c.addOnTrackFunc(healthyClientCh, trackReceived, trackClosed, c.subGenerationID)) + // Remove the OnTrackSub once we either fail or succeed. + defer tracker.RemoveOnTrackSub(c.trackName()) + + // Call `AddStream` on the remote. In the successful case, this will result in a + // PeerConnection renegotiation to add this camera's video track and have the `OnTrack` + // callback invoked. + if _, err := c.streamClient.AddStream(ctx, &streampb.AddStreamRequest{Name: c.trackName()}); err != nil { + c.logger.CDebugw(ctx, "SubscribeRTP AddStream hit error", "subID", sub.ID.String(), "trackName", c.trackName(), "err", err) return rtppassthrough.NilSubscription, err } - // NOTE: (Nick S) This is a workaround to a Pion bug / missing feature. - - // If the WebRTC peer on the other side of the PeerConnection calls pc.AddTrack followd by pc.RemoveTrack - // before the module writes RTP packets - // to the track, the client's PeerConnection.OnTrack callback is never called. - // That results in the client never receiving RTP packets on subscriptions which never terminate - // which is an unacceptable failure mode. - // To prevent that failure mode, we exit with an error if a track is not received within - // the SubscribeRTP context. + c.logger.CDebugw(ctx, "SubscribeRTP waiting for track", "client", fmt.Sprintf("%p", c), "pc", fmt.Sprintf("%p", c.conn.PeerConn())) select { case <-ctx.Done(): - err := errors.Wrap(ctx.Err(), "Track not received within SubscribeRTP provided context") - c.logger.Error(err.Error()) - return rtppassthrough.NilSubscription, err - case <-c.ctx.Done(): - err := errors.Wrap(c.ctx.Err(), "Track not received before client closed") - c.logger.Error(err) - return rtppassthrough.NilSubscription, err + return rtppassthrough.NilSubscription, fmt.Errorf("track not received within SubscribeRTP provided context %w", ctx.Err()) + case <-healthyClientCh: + return rtppassthrough.NilSubscription, errors.New("track not received before client closed") case <-trackReceived: - c.logger.Debug("received track") + c.logger.CDebugw(ctx, "SubscribeRTP received track data", "client", fmt.Sprintf("%p", c), "pc", fmt.Sprintf("%p", c.conn.PeerConn())) } - // set up channel so we can detect when the track has closed (in response to an event / error internal to the - // peer or due to calling RemoveStream) + + // Set up channel to detect when the track has closed. This can happen in response to an + // event / error internal to the peer or due to calling `RemoveStream`. c.trackClosed = trackClosed - // the sub is the new parent of all subsequent subs until the number if subsriptions falls back to 0 - c.currentSubParentID = sub.ID - c.subParentToChildren[c.currentSubParentID] = []rtppassthrough.SubscriptionID{} - c.logger.CDebugw(ctx, "SubscribeRTP called AddStream and succeeded", "subID", sub.ID.String(), - "name", c.Name()) - } - c.subParentToChildren[c.currentSubParentID] = append(c.subParentToChildren[c.currentSubParentID], sub.ID) - // add the subscription to bufAndCBByID so the goroutine spawned by - // addOnTrackSubFunc can forward the packets it receives from the modular camera - // over WebRTC to the SubscribeRTP caller via the packetsCB callback - c.bufAndCBByID[sub.ID] = bufAndCB{ + c.logger.CDebugw(ctx, "SubscribeRTP called AddStream and succeeded", "subID", sub.ID.String()) + } + + // Associate this subscription with the current generation. + c.associatedSubs[c.subGenerationID] = append(c.associatedSubs[c.subGenerationID], sub.ID) + + // Add the subscription to runningStreams. The goroutine spawned by `addOnTrackFunc` will use + // this to know where to forward incoming RTP packets. + c.runningStreams[sub.ID] = bufAndCB{ cb: func(p *rtp.Packet) { packetsCB([]*rtp.Packet{p}) }, - buf: buf, + buf: rtpPacketBuffer, } - buf.Start() + + // Start the goroutine that copies RTP packets + rtpPacketBuffer.Start() g.Success() c.logger.CDebugw(ctx, "SubscribeRTP succeeded", "subID", sub.ID.String(), - "name", c.Name(), "bufAndCBByID", c.bufAndCBByID.String()) + "name", c.Name(), "bufAndCBByID", c.bufAndCBToString()) return sub, nil } -func (c *client) addOnTrackSubFunc( - trackReceived, trackClosed chan struct{}, - parentID rtppassthrough.SubscriptionID, -) rdkgrpc.OnTrackCB { +func (c *client) addOnTrackFunc( + healthyClientCh, trackReceived, trackClosed chan struct{}, + generationID int, +) grpc.OnTrackCB { + // This is invoked when `PeerConnection.OnTrack` is called for this camera's stream id. return func(tr *webrtc.TrackRemote, r *webrtc.RTPReceiver) { + // Our `OnTrack` was called. Inform `SubscribeRTP` that getting video data was successful. close(trackReceived) c.activeBackgroundWorkers.Add(1) goutils.ManagedGo(func() { + var count atomic.Uint64 + defer close(trackClosed) for { - if c.ctx.Err() != nil { - c.logger.Debugw("SubscribeRTP: camera client", "name ", c.Name(), "parentID", parentID.String(), - "OnTrack callback terminating as the client is closing") - close(trackClosed) + select { + case <-healthyClientCh: + c.logger.Debugw("OnTrack callback terminating as the client is closing", "parentID", generationID) + return + default: + } + + // We set a read deadline such that we don't block in I/O. This allows us to respond + // to events such as shutdown. Normally this would be done with a context on the + // `ReadRTP` method. + deadline := time.Now().Add(readRTPTimeout) + if err := tr.SetReadDeadline(deadline); err != nil { + c.logger.Errorw("SetReadDeadline error", "generationId", generationID, "err", err) return } pkt, _, err := tr.ReadRTP() + if os.IsTimeout(err) { + c.logger.Debugw("ReadRTP read timeout", "generationId", generationID, + "err", err, "timeout", readRTPTimeout.String()) + continue + } + if err != nil { - close(trackClosed) + if errors.Is(err, io.EOF) { + c.logger.Infow("ReadRTP received EOF", "generationId", generationID, "err", err) + } else { + c.logger.Warnw("ReadRTP error", "generationId", generationID, "err", err) + } // NOTE: (Nick S) We need to remember which subscriptions are consuming packets // from to which tr *webrtc.TrackRemote so that we can terminate the child subscriptions // when their track terminate. - c.unsubscribeChildrenSubs(parentID) - if errors.Is(err, io.EOF) { - c.logger.Debugw("SubscribeRTP: camera client", "name ", c.Name(), "parentID", parentID.String(), - "OnTrack callback terminating ReadRTP loop due to ", err.Error()) - return - } - c.logger.Errorw("SubscribeRTP: camera client", "name ", c.Name(), "parentID", parentID.String(), - "OnTrack callback hit unexpected error from ReadRTP err:", err.Error()) + c.unsubscribeChildrenSubs(generationID) return } - c.mu.Lock() - for _, tmp := range c.bufAndCBByID { + c.rtpPassthroughMu.Lock() + for _, tmp := range c.runningStreams { + if count.Add(1)%10000 == 0 { + c.logger.Debugw("ReadRTP called. Sampling 1/10000", "count", count.Load(), "packetTs", pkt.Header.Timestamp) + } + // This is needed to prevent the problem described here: // https://go.dev/blog/loopvar-preview bufAndCB := tmp err := bufAndCB.buf.Publish(func() { bufAndCB.cb(pkt) }) if err != nil { - c.logger.Debugw("SubscribeRTP: camera client", - "name", c.Name(), - "parentID", parentID.String(), - "dropped an RTP packet dropped due to", - "err", err.Error()) + c.logger.Debugw("rtp passthrough packet dropped", + "generationId", generationID, "err", err) } } - c.mu.Unlock() + c.rtpPassthroughMu.Unlock() } }, c.activeBackgroundWorkers.Done) } } // Unsubscribe terminates a subscription to receive RTP packets. -// It is strongly recommended to set a timeout on ctx as the underlying -// WebRTC peer connection's Track can be removed before sending an RTP packet which -// results in Unsubscribe blocking until the provided -// context is cancelled or the client is closed. -// This rare condition may happen in cases like reconfiguring concurrently with -// an Unsubscribe call. +// +// It is strongly recommended to set a timeout on ctx as the underlying WebRTC peer connection's +// Track can be removed before sending an RTP packet which results in Unsubscribe blocking until the +// provided context is cancelled or the client is closed. This rare condition may happen in cases +// like reconfiguring concurrently with an Unsubscribe call. func (c *client) Unsubscribe(ctx context.Context, id rtppassthrough.SubscriptionID) error { ctx, span := trace.StartSpan(ctx, "camera::client::Unsubscribe") defer span.End() - c.mu.Lock() if c.conn.PeerConn() == nil { - c.mu.Unlock() return ErrNoPeerConnection } - _, ok := c.conn.(*rdkgrpc.SharedConn) - if !ok { - c.mu.Unlock() - return ErrNoSharedPeerConnection - } - c.logger.CDebugw(ctx, "Unsubscribe called with", "name", c.Name(), "subID", id.String()) + c.healthyClientChMu.Lock() + healthyClientCh := c.healthyClientCh + c.healthyClientChMu.Unlock() - bufAndCB, ok := c.bufAndCBByID[id] + c.logger.CInfow(ctx, "Unsubscribe called", "subID", id.String()) + c.rtpPassthroughMu.Lock() + bufAndCB, ok := c.runningStreams[id] if !ok { - c.logger.CWarnw(ctx, "Unsubscribe called with unknown subID ", "name", c.Name(), "subID", id.String()) - c.mu.Unlock() + c.logger.CWarnw(ctx, "Unsubscribe called with unknown subID", "subID", id.String()) + c.rtpPassthroughMu.Unlock() return ErrUnknownSubscriptionID } - if len(c.bufAndCBByID) == 1 { - c.logger.CDebugw(ctx, "Unsubscribe calling RemoveStream", "name", c.Name(), "subID", id.String()) - if _, err := c.streamClient.RemoveStream(ctx, &streampb.RemoveStreamRequest{Name: c.Name().String()}); err != nil { - c.logger.CWarnw(ctx, "Unsubscribe RemoveStream returned err", "name", c.Name(), "subID", id.String(), "err", err) - c.mu.Unlock() - return err - } - - delete(c.bufAndCBByID, id) + if len(c.runningStreams) > 1 { + // There are still existing output streams for this camera. We close the resources + // associated with the input `SubscriptionID` and return. + delete(c.runningStreams, id) + c.rtpPassthroughMu.Unlock() bufAndCB.buf.Close() + return nil + } - // unlock so that the OnTrack callback can get the lock if it needs to before the ReadRTP method returns an error - // which will close `c.trackClosed`. - c.mu.Unlock() + // There is only one stream left. In addition to closing the input subscription's resources, we + // also close the stream from the underlying remote. + // + // Calling `RemoveStream` releases resources on the remote. We promise to keep retrying until + // we're successful. + c.logger.CInfow(ctx, "Last subscriber. calling RemoveStream", + "trackName", c.trackName(), "subID", id.String()) + request := &streampb.RemoveStreamRequest{Name: c.trackName()} + // We assume the server responds with a success if the requested `Name` is unknown/already + // removed. + if _, err := c.streamClient.RemoveStream(ctx, request); err != nil { + c.logger.CWarnw(ctx, "Unsubscribe RemoveStream returned err", "trackName", + c.trackName(), "subID", id.String(), "err", err) + c.rtpPassthroughMu.Unlock() + return err + } + + // We can only remove `runningStreams` when `RemoveStream` succeeds. If there was an error, the + // upper layers are responsible for retrying `Unsubscribe`. At this point we will not return an + // error to clearly inform the caller that `Unsubscribe` does not need to be retried. + delete(c.runningStreams, id) + bufAndCB.buf.Close() - select { - case <-ctx.Done(): - err := errors.Wrap(ctx.Err(), "track not closed within Unsubscribe provided context. Subscription may be left in broken state") - c.logger.Error(err) - return err - case <-c.ctx.Done(): - err := errors.Wrap(c.ctx.Err(), "track not closed before client closed") - c.logger.Error(err) - return err - case <-c.trackClosed: - } + // The goroutine we're stopping uses the `rtpPassthroughMu` to access the map of + // `runningStreams`. We must unlock while we wait for it to exit. + c.rtpPassthroughMu.Unlock() + + select { + case <-c.trackClosed: + return nil + case <-ctx.Done(): + c.logger.CWarnw(ctx, "RemoveStream successfully called, but errored waiting for stream to send an EOF", + "subID", id.String(), "err", ctx.Err()) + return nil + case <-healthyClientCh: + c.logger.CWarnw(ctx, "RemoveStream successfully called, but camera closed waiting for stream to send an EOF", + "subID", id.String()) return nil } - c.mu.Unlock() +} - delete(c.bufAndCBByID, id) - bufAndCB.buf.Close() +func (c *client) trackName() string { + // if c.conn is a *grpc.SharedConn then the client + // is talking to a module and we need to send the fully qualified name + if _, ok := c.conn.(*grpc.SharedConn); ok { + return c.Name().String() + } - return nil + // otherwise we know we are talking to either a main or remote robot part + if c.remoteName != "" { + // if c.remoteName != "" it indicates that we are talking to a remote part & we need to pop the remote name + // as the remote doesn't know it's own name from the perspective of the main part + return c.Name().PopRemote().SDPTrackName() + } + // in this case we are talking to a main part & the remote name (if it exists) needs to be preserved + return c.Name().SDPTrackName() } func (c *client) unsubscribeAll() { - if len(c.bufAndCBByID) > 0 { - for id, bufAndCB := range c.bufAndCBByID { - c.logger.Debugw("unsubscribeAll terminating sub", "name", c.Name(), "subID", id.String()) - delete(c.bufAndCBByID, id) + c.rtpPassthroughMu.Lock() + defer c.rtpPassthroughMu.Unlock() + if len(c.runningStreams) > 0 { + // shutdown & delete all *rtppassthrough.Buffer and callbacks + for subID, bufAndCB := range c.runningStreams { + c.logger.Debugw("unsubscribeAll terminating sub", "subID", subID.String()) + delete(c.runningStreams, subID) bufAndCB.buf.Close() } } } -func (c *client) unsubscribeChildrenSubs(parentID rtppassthrough.SubscriptionID) { - c.mu.Lock() - defer c.mu.Unlock() - c.logger.Debugw("client unsubscribeChildrenSubs called", "name", c.Name(), "parentID", parentID.String()) - defer c.logger.Debugw("client unsubscribeChildrenSubs done", "name", c.Name(), "parentID", parentID.String()) - for _, childID := range c.subParentToChildren[parentID] { - bufAndCB, ok := c.bufAndCBByID[childID] +func (c *client) unsubscribeChildrenSubs(generationID int) { + c.rtpPassthroughMu.Lock() + defer c.rtpPassthroughMu.Unlock() + c.logger.Debugw("client unsubscribeChildrenSubs called", "generationId", generationID, "numSubs", len(c.associatedSubs)) + defer c.logger.Debugw("client unsubscribeChildrenSubs done", "generationId", generationID) + for _, subID := range c.associatedSubs[generationID] { + bufAndCB, ok := c.runningStreams[subID] if !ok { continue } - c.logger.Debugw("stopping child subscription", "name", c.Name(), "parentID", parentID.String(), "childID", childID.String()) - delete(c.bufAndCBByID, childID) + c.logger.Debugw("stopping subscription", "generationId", generationID, "subId", subID.String()) + delete(c.runningStreams, subID) bufAndCB.buf.Close() } - delete(c.subParentToChildren, parentID) + delete(c.associatedSubs, generationID) } -func (s bufAndCBByID) String() string { - if len(s) == 0 { +func (c *client) bufAndCBToString() string { + if len(c.runningStreams) == 0 { return "len: 0" } strIds := []string{} strIdsToCB := map[string]bufAndCB{} - for id, cb := range s { + for id, cb := range c.runningStreams { strID := id.String() strIds = append(strIds, strID) strIdsToCB[strID] = cb } slices.Sort(strIds) - ret := fmt.Sprintf("len: %d, ", len(s)) + ret := fmt.Sprintf("len: %d, ", len(c.runningStreams)) for _, strID := range strIds { ret += fmt.Sprintf("%s: %v, ", strID, strIdsToCB[strID]) } diff --git a/components/camera/client_test.go b/components/camera/client_test.go index 60ad8b82730..11640bbd93e 100644 --- a/components/camera/client_test.go +++ b/components/camera/client_test.go @@ -18,17 +18,26 @@ import ( "google.golang.org/grpc/metadata" "go.viam.com/rdk/components/camera" + "go.viam.com/rdk/components/camera/fake" "go.viam.com/rdk/components/camera/rtppassthrough" + "go.viam.com/rdk/config" "go.viam.com/rdk/data" "go.viam.com/rdk/gostream" + "go.viam.com/rdk/gostream/codec/opus" + "go.viam.com/rdk/gostream/codec/x264" viamgrpc "go.viam.com/rdk/grpc" "go.viam.com/rdk/logging" "go.viam.com/rdk/pointcloud" "go.viam.com/rdk/resource" "go.viam.com/rdk/rimage" "go.viam.com/rdk/rimage/transform" + "go.viam.com/rdk/robot" + robotimpl "go.viam.com/rdk/robot/impl" + "go.viam.com/rdk/robot/web" + weboptions "go.viam.com/rdk/robot/web/options" "go.viam.com/rdk/testutils" "go.viam.com/rdk/testutils/inject" + "go.viam.com/rdk/testutils/robottestutils" rutils "go.viam.com/rdk/utils" "go.viam.com/rdk/utils/contextutils" ) @@ -669,3 +678,513 @@ func TestRTPPassthroughWithoutWebRTC(t *testing.T) { test.That(t, conn.Close(), test.ShouldBeNil) }) } + +func setupRealRobot( + t *testing.T, + robotConfig *config.Config, + logger logging.Logger, +) (context.Context, robot.LocalRobot, string, web.Service) { + t.Helper() + + ctx := context.Background() + robot, err := robotimpl.RobotFromConfig(ctx, robotConfig, logger) + test.That(t, err, test.ShouldBeNil) + + // We initialize with a stream config such that the stream server is capable of creating video stream and + // audio stream data. + webSvc := web.New(robot, logger, web.WithStreamConfig(gostream.StreamConfig{ + AudioEncoderFactory: opus.NewEncoderFactory(), + VideoEncoderFactory: x264.NewEncoderFactory(), + })) + options, _, addr := robottestutils.CreateBaseOptionsAndListener(t) + err = webSvc.Start(ctx, options) + test.That(t, err, test.ShouldBeNil) + + return ctx, robot, addr, webSvc +} + +func setupRealRobotWithOptions( + t *testing.T, + robotConfig *config.Config, + logger logging.Logger, + options weboptions.Options, +) (context.Context, robot.LocalRobot, web.Service) { + t.Helper() + + ctx := context.Background() + robot, err := robotimpl.RobotFromConfig(ctx, robotConfig, logger) + test.That(t, err, test.ShouldBeNil) + + // We initialize with a stream config such that the stream server is capable of creating video stream and + // audio stream data. + webSvc := web.New(robot, logger, web.WithStreamConfig(gostream.StreamConfig{ + AudioEncoderFactory: opus.NewEncoderFactory(), + VideoEncoderFactory: x264.NewEncoderFactory(), + })) + err = webSvc.Start(ctx, options) + test.That(t, err, test.ShouldBeNil) + + return ctx, robot, webSvc +} + +var ( + Green = "\033[32m" + Red = "\033[31m" + Magenta = "\033[35m" + Cyan = "\033[36m" + Yellow = "\033[33m" + Reset = "\033[0m" +) + +func TestMultiplexOverRemoteConnection(t *testing.T) { + logger := logging.NewTestLogger(t).Sublogger(t.Name()) + + remoteCfg := &config.Config{Components: []resource.Config{ + { + Name: "rtpPassthroughCamera", + API: resource.NewAPI("rdk", "component", "camera"), + Model: resource.DefaultModelFamily.WithModel("fake"), + ConvertedAttributes: &fake.Config{ + RTPPassthrough: true, + }, + }, + }} + + // Create a robot with a single fake camera. + remoteCtx, remoteRobot, addr, remoteWebSvc := setupRealRobot(t, remoteCfg, logger.Sublogger("remote")) + defer remoteRobot.Close(remoteCtx) + defer remoteWebSvc.Close(remoteCtx) + + mainCfg := &config.Config{Remotes: []config.Remote{ + { + Name: "remote", + Address: addr, + Insecure: true, + }, + }} + mainCtx, mainRobot, _, mainWebSvc := setupRealRobot(t, mainCfg, logger.Sublogger("main")) + logger.Info("robot setup") + defer mainRobot.Close(mainCtx) + defer mainWebSvc.Close(mainCtx) + + cameraClient, err := camera.FromRobot(mainRobot, "remote:rtpPassthroughCamera") + test.That(t, err, test.ShouldBeNil) + + image, _, err := cameraClient.Images(mainCtx) + test.That(t, err, test.ShouldBeNil) + test.That(t, image, test.ShouldNotBeNil) + logger.Info("got images") + + recvPktsCtx, recvPktsFn := context.WithCancel(context.Background()) + sub, err := cameraClient.(rtppassthrough.Source).SubscribeRTP(mainCtx, 2, func(pkts []*rtp.Packet) { + recvPktsFn() + }) + test.That(t, err, test.ShouldBeNil) + <-recvPktsCtx.Done() + logger.Info("got packets") + + err = cameraClient.(rtppassthrough.Source).Unsubscribe(mainCtx, sub.ID) + test.That(t, err, test.ShouldBeNil) + logger.Info("unsubscribe") +} + +func TestMultiplexOverMultiHopRemoteConnection(t *testing.T) { + logger := logging.NewTestLogger(t).Sublogger(t.Name()) + + remoteCfg2 := &config.Config{ + Network: config.NetworkConfig{NetworkConfigData: config.NetworkConfigData{Sessions: config.SessionsConfig{HeartbeatWindow: time.Hour}}}, + Components: []resource.Config{ + { + Name: "rtpPassthroughCamera", + API: resource.NewAPI("rdk", "component", "camera"), + Model: resource.DefaultModelFamily.WithModel("fake"), + ConvertedAttributes: &fake.Config{ + RTPPassthrough: true, + }, + }, + }, + } + + // Create a robot with a single fake camera. + remote2Ctx, remoteRobot2, addr2, remoteWebSvc2 := setupRealRobot(t, remoteCfg2, logger.Sublogger("remote-2")) + defer remoteRobot2.Close(remote2Ctx) + defer remoteWebSvc2.Close(remote2Ctx) + + remoteCfg1 := &config.Config{ + Network: config.NetworkConfig{NetworkConfigData: config.NetworkConfigData{Sessions: config.SessionsConfig{HeartbeatWindow: time.Hour}}}, + Remotes: []config.Remote{ + { + Name: "remote-2", + Address: addr2, + Insecure: true, + }, + }, + } + + remote1Ctx, remoteRobot1, addr1, remoteWebSvc1 := setupRealRobot(t, remoteCfg1, logger.Sublogger("remote-1")) + defer remoteRobot1.Close(remote1Ctx) + defer remoteWebSvc1.Close(remote1Ctx) + + mainCfg := &config.Config{ + Network: config.NetworkConfig{NetworkConfigData: config.NetworkConfigData{Sessions: config.SessionsConfig{HeartbeatWindow: time.Hour}}}, + Remotes: []config.Remote{ + { + Name: "remote-1", + Address: addr1, + Insecure: true, + }, + }, + } + + mainCtx, mainRobot, _, mainWebSvc := setupRealRobot(t, mainCfg, logger.Sublogger("main")) + defer mainRobot.Close(mainCtx) + defer mainWebSvc.Close(mainCtx) + + cameraClient, err := camera.FromRobot(mainRobot, "remote-1:remote-2:rtpPassthroughCamera") + test.That(t, err, test.ShouldBeNil) + + image, _, err := cameraClient.Images(mainCtx) + test.That(t, err, test.ShouldBeNil) + test.That(t, image, test.ShouldNotBeNil) + logger.Info("got images") + + time.Sleep(time.Second) + + recvPktsCtx, recvPktsFn := context.WithCancel(context.Background()) + sub, err := cameraClient.(rtppassthrough.Source).SubscribeRTP(mainCtx, 512, func(pkts []*rtp.Packet) { + recvPktsFn() + }) + test.That(t, err, test.ShouldBeNil) + <-recvPktsCtx.Done() + logger.Info("got packets") + + test.That(t, cameraClient.(rtppassthrough.Source).Unsubscribe(mainCtx, sub.ID), test.ShouldBeNil) +} + +//nolint +// NOTE: These tests fail when this condition occurs: +// +// logger.go:130: 2024-06-17T16:56:14.097-0400 DEBUG TestGrandRemoteRebooting.remote-1.rdk:remote:/remote-2.webrtc rpc/wrtc_client_channel.go:299 no stream for id; discarding {"ch": 0, "id": 11} +// +// https://github.com/viamrobotics/goutils/blob/main/rpc/wrtc_client_channel.go#L299 +// +// go test -race -v -run=TestWhyMustTimeoutOnReadRTP -timeout 10s +// TestWhyMustTimeoutOnReadRTP shows that if we don't timeout on ReadRTP (and also don't call RemoveStream) on close +// calling Close() on main's camera client blocks forever if there is a live SubscribeRTP subscription with a remote +// due to the fact that the TrackRemote.ReadRTP method blocking forever. +func TestWhyMustTimeoutOnReadRTP(t *testing.T) { + t.Skip("Depends on RSDK-7903") + logger := logging.NewTestLogger(t).Sublogger(t.Name()) + + remoteCfg2 := &config.Config{ + Network: config.NetworkConfig{NetworkConfigData: config.NetworkConfigData{Sessions: config.SessionsConfig{HeartbeatWindow: time.Hour}}}, + Components: []resource.Config{ + { + Name: "rtpPassthroughCamera", + API: resource.NewAPI("rdk", "component", "camera"), + Model: resource.DefaultModelFamily.WithModel("fake"), + ConvertedAttributes: &fake.Config{ + RTPPassthrough: true, + }, + }, + }, + } + // Create a robot with a single fake camera. + remote2Ctx, remoteRobot2, addr2, remoteWebSvc2 := setupRealRobot(t, remoteCfg2, logger.Sublogger("remote-2")) + defer remoteRobot2.Close(remote2Ctx) + defer remoteWebSvc2.Close(remote2Ctx) + + remoteCfg1 := &config.Config{ + Network: config.NetworkConfig{NetworkConfigData: config.NetworkConfigData{Sessions: config.SessionsConfig{HeartbeatWindow: time.Hour}}}, + Remotes: []config.Remote{ + { + Name: "remote-2", + Address: addr2, + Insecure: true, + }, + }, + } + + remote1Ctx, remoteRobot1, addr1, remoteWebSvc1 := setupRealRobot(t, remoteCfg1, logger.Sublogger("remote-1")) + defer remoteRobot1.Close(remote1Ctx) + defer remoteWebSvc1.Close(remote1Ctx) + + mainCfg := &config.Config{ + Network: config.NetworkConfig{NetworkConfigData: config.NetworkConfigData{Sessions: config.SessionsConfig{HeartbeatWindow: time.Hour}}}, + Remotes: []config.Remote{ + { + Name: "remote-1", + Address: addr1, + Insecure: true, + }, + }, + } + + mainCtx, mainRobot, _, mainWebSvc := setupRealRobot(t, mainCfg, logger.Sublogger("main")) + logger.Info("robot setup") + defer mainRobot.Close(mainCtx) + defer mainWebSvc.Close(mainCtx) + + cameraClient, err := camera.FromRobot(mainRobot, "remote-1:remote-2:rtpPassthroughCamera") + test.That(t, err, test.ShouldBeNil) + + image, _, err := cameraClient.Images(mainCtx) + test.That(t, err, test.ShouldBeNil) + test.That(t, image, test.ShouldNotBeNil) + logger.Info("got images") + + logger.Infof("calling SubscribeRTP on %T, %p", cameraClient, cameraClient) + time.Sleep(time.Second) + + pktsChan := make(chan []*rtp.Packet) + recvPktsCtx, recvPktsFn := context.WithCancel(context.Background()) + sub, err := cameraClient.(rtppassthrough.Source).SubscribeRTP(mainCtx, 512, func(pkts []*rtp.Packet) { + // first packet + recvPktsFn() + // at some point packets are no longer published + select { + case pktsChan <- pkts: + default: + } + }) + test.That(t, err, test.ShouldBeNil) + <-recvPktsCtx.Done() + logger.Info("got packets") + + logger.Info("calling close") + test.That(t, remoteRobot2.Close(context.Background()), test.ShouldBeNil) + test.That(t, remoteWebSvc2.Close(context.Background()), test.ShouldBeNil) + logger.Info("close called") + + logger.Info("waiting for SubscribeRTP to stop receiving packets") + + timeoutCtx, timeoutFn := context.WithTimeout(context.Background(), time.Second) + defer timeoutFn() + + var ( + pktTimeoutCtx context.Context + pktTimeoutFn context.CancelFunc + ) + + // Once we have not received packets for half a second we can assume that no more packets will be published + // by the first instance of remote2 +Loop: + for { + if pktTimeoutFn != nil { + pktTimeoutFn() + } + pktTimeout := time.Millisecond * 500 + pktTimeoutCtx, pktTimeoutFn = context.WithTimeout(context.Background(), pktTimeout) + select { + case <-pktsChan: + continue + case <-pktTimeoutCtx.Done(): + logger.Infof("it has been at least %s since SubscribeRTP has received a packet", pktTimeout) + pktTimeoutFn() + // https://go.dev/ref/spec#Break_statements + // The 'Loop' label is needed so that we break out of the loop + // rather than out of the select statement + break Loop + case <-timeoutCtx.Done(): + // Failure case. The following assertion always fails. We use this to get a failure line + // number + error message. + test.That(t, true, test.ShouldEqual, "timed out waiting for SubscribeRTP packets to drain") + } + } + + logger.Info("sleeping") + time.Sleep(time.Second) + + // sub should still be alive + test.That(t, sub.Terminated.Err(), test.ShouldBeNil) +} + +// Notes: +// - Ideally, we'd lower robot client reconnect timers down from 10 seconds. +// - We need to force robot client webrtc connections +// - WebRTC connections need to disable SRTP replay protection +// +// This tests the following scenario: +// 1. main-part (main) -> remote-part-1 (r1) -> remote-part-2 (r2) where r2 has a camera +// 2. the client in the main part makes an AddStream(r1:r2:rtpPassthroughCamera) request, starting a +// webrtc video track to be streamed from r2 -> r1 -> main -> client +// 3. r2 reboots +// 4. expect that r1 & main stop getting packets +// 5. when the new instance of r2 comes back online main gets new rtp packets from it's track with +// r1. +func TestGrandRemoteRebooting(t *testing.T) { + t.Skip("Depends on RSDK-7903") + logger := logging.NewTestLogger(t).Sublogger(t.Name()) + + remoteCfg2 := &config.Config{ + Components: []resource.Config{ + { + Name: "rtpPassthroughCamera", + API: resource.NewAPI("rdk", "component", "camera"), + Model: resource.DefaultModelFamily.WithModel("fake"), + ConvertedAttributes: &fake.Config{ + RTPPassthrough: true, + }, + }, + }, + } + + // Create a robot with a single fake camera. + options2, _, addr2 := robottestutils.CreateBaseOptionsAndListener(t) + remote2Ctx, remoteRobot2, remoteWebSvc2 := setupRealRobotWithOptions(t, remoteCfg2, logger.Sublogger("remote-2"), options2) + + remoteCfg1 := &config.Config{ + Remotes: []config.Remote{ + { + Name: "remote-2", + Address: addr2, + Insecure: true, + }, + }, + } + + remote1Ctx, remoteRobot1, addr1, remoteWebSvc1 := setupRealRobot(t, remoteCfg1, logger.Sublogger("remote-1")) + defer remoteRobot1.Close(remote1Ctx) + defer remoteWebSvc1.Close(remote1Ctx) + + mainCfg := &config.Config{ + Remotes: []config.Remote{ + { + Name: "remote-1", + Address: addr1, + Insecure: true, + }, + }, + } + + mainCtx, mainRobot, _, mainWebSvc := setupRealRobot(t, mainCfg, logger.Sublogger("main")) + logger.Info("robot setup") + defer mainRobot.Close(mainCtx) + defer mainWebSvc.Close(mainCtx) + + mainCameraClient, err := camera.FromRobot(mainRobot, "remote-1:remote-2:rtpPassthroughCamera") + test.That(t, err, test.ShouldBeNil) + + image, _, err := mainCameraClient.Images(mainCtx) + test.That(t, err, test.ShouldBeNil) + test.That(t, image, test.ShouldNotBeNil) + logger.Info("got images") + + logger.Infof("calling SubscribeRTP on %T, %p", mainCameraClient, mainCameraClient) + time.Sleep(time.Second) + + pktsChan := make(chan []*rtp.Packet, 10) + recvPktsCtx, recvPktsFn := context.WithCancel(context.Background()) + testDone := make(chan struct{}) + defer close(testDone) + sub, err := mainCameraClient.(rtppassthrough.Source).SubscribeRTP(mainCtx, 512, func(pkts []*rtp.Packet) { + // first packet + recvPktsFn() + // at some point packets are no longer published + lastPkt := pkts[len(pkts)-1] + logger.Info("Pushing packets: ", len(pkts), " TS:", lastPkt.Timestamp) + select { + case <-testDone: + case pktsChan <- pkts: + } + logger.Info("Pkt pushed. TS:", lastPkt.Timestamp) + }) + test.That(t, err, test.ShouldBeNil) + <-recvPktsCtx.Done() + logger.Info("got packets") + + logger.Info("calling close") + test.That(t, remoteRobot2.Close(remote2Ctx), test.ShouldBeNil) + test.That(t, remoteWebSvc2.Close(remote2Ctx), test.ShouldBeNil) + logger.Info("close called") + + logger.Info("waiting for SubscribeRTP to stop receiving packets") + + timeoutCtx, timeoutFn := context.WithTimeout(context.Background(), time.Second) + defer timeoutFn() + + var ( + pktTimeoutCtx context.Context + pktTimeoutFn context.CancelFunc + ) + + // Once we have not received packets for half a second we can assume that no more packets will be published + // by the first instance of remote2 +Loop: + for { + if pktTimeoutFn != nil { + pktTimeoutFn() + } + pktTimeout := time.Millisecond * 500 + pktTimeoutCtx, pktTimeoutFn = context.WithTimeout(context.Background(), pktTimeout) + select { + case pkts := <-pktsChan: + lastPkt := pkts[len(pkts)-1] + logger.Infof("First RTP packet received. TS: %v", lastPkt.Timestamp) + continue + case <-pktTimeoutCtx.Done(): + logger.Infow("SubscribeRTP timed out waiting for a packet. The remote is offline.", "pktTimeout", pktTimeout) + pktTimeoutFn() + // https://go.dev/ref/spec#Break_statements + // The 'Loop' label is needed so that we break out of the loop + // rather than out of the select statement + break Loop + case <-timeoutCtx.Done(): + // Failure case. The following assertion always fails. We use this to get a failure line + // number + error message. + test.That(t, true, test.ShouldEqual, "timed out waiting for SubscribeRTP packets to drain") + } + } + + // sub should still be alive + test.That(t, sub.Terminated.Err(), test.ShouldBeNil) + + // I'm trying to get the remote-2 to come back online at the same address under the hopes that remote-1 will + // treat it the same as it would if a real robot crasehed & came back online without changing its name. + // The expectation is that SubscribeRTP should start receiving packets from remote-1 when remote-1 starts + // receiving packets from the new remote-2 + // It is not working as remote 1 never detects remote 2 & as a result main calls Close() on it's client with + // remote-1 which can be detectd + // by the fact that sub.Terminated.Done() is always the path this test goes down + + logger.Infow("old robot address", "address", addr2) + tcpAddr, ok := options2.Network.Listener.Addr().(*net.TCPAddr) + test.That(t, ok, test.ShouldBeTrue) + newListener, err := net.ListenTCP("tcp", &net.TCPAddr{Port: tcpAddr.Port}) + test.That(t, err, test.ShouldBeNil) + options2.Network.Listener = newListener + + logger.Info("setting up new robot at address %s", newListener.Addr().String()) + + remote2CtxSecond, remoteRobot2Second, remoteWebSvc2Second := setupRealRobotWithOptions( + t, + remoteCfg2, + logger.Sublogger("remote-2SecondInstance"), + options2, + ) + defer remoteRobot2Second.Close(remote2CtxSecond) + defer remoteWebSvc2Second.Close(remote2CtxSecond) + sndPktTimeoutCtx, sndPktTimeoutFn := context.WithTimeout(context.Background(), time.Second*20) + defer sndPktTimeoutFn() + testPassed := false + for !testPassed { + select { + case <-sub.Terminated.Done(): + // Failure case. The following assertion always fails. We use this to get a failure line + // number + error message. + test.That(t, true, test.ShouldEqual, "main's sub terminated due to close") + case pkts := <-pktsChan: + lastPkt := pkts[len(pkts)-1] + logger.Info("Test finale RTP packet received. TS: %v", lastPkt.Timestamp) + // Right now we never go down this path as the test is not able to get remote1 to reconnect to the new remote 2 + logger.Info("SubscribeRTP got packets") + testPassed = true + case <-sndPktTimeoutCtx.Done(): + // Failure case. The following assertion always fails. We use this to get a failure line + // number + error message. + test.That(t, true, test.ShouldEqual, "timed out waiting for SubscribeRTP to receive packets") + case <-time.After(time.Second): + logger.Info("still waiting for RTP packets") + } + } +} diff --git a/gostream/stream.go b/gostream/stream.go index 687d72c0920..a270f8b2011 100644 --- a/gostream/stream.go +++ b/gostream/stream.go @@ -161,6 +161,9 @@ func (bs *basicStream) Start() { utils.ManagedGo(bs.processOutputAudioChunks, bs.activeBackgroundWorkers.Done) } +// NOTE: (Nick S) This only writes video RTP packets +// if we also need to support writing audio RTP packets, we should split +// this method into WriteVideoRTP and WriteAudioRTP. func (bs *basicStream) WriteRTP(pkt *rtp.Packet) error { return bs.videoTrackLocal.rtpTrack.WriteRTP(pkt) } diff --git a/grpc/conn.go b/grpc/conn.go index f94cb330a51..4f4413afd7c 100644 --- a/grpc/conn.go +++ b/grpc/conn.go @@ -7,13 +7,20 @@ import ( "github.com/viamrobotics/webrtc/v3" "go.viam.com/utils/rpc" + "golang.org/x/exp/maps" googlegrpc "google.golang.org/grpc" + + "go.viam.com/rdk/logging" ) -// ReconfigurableClientConn allows for the underlying client connections to be swapped under the hood. +// ReconfigurableClientConn allows for the underlying client connections to be swapped under the +// hood. A ReconfigurableClientConn may only be used for a connection to a single logical server. type ReconfigurableClientConn struct { connMu sync.RWMutex conn rpc.ClientConn + + onTrackCBByTrackNameMu sync.Mutex + onTrackCBByTrackName map[string]OnTrackCB } // Return this constant such that backoff error logging can compare consecutive errors and reliably @@ -59,6 +66,25 @@ func (c *ReconfigurableClientConn) NewStream( func (c *ReconfigurableClientConn) ReplaceConn(conn rpc.ClientConn) { c.connMu.Lock() c.conn = conn + // It is safe to access this without a mutex as it is only ever nil once at the beginning of the + // ReconfigurableClientConn's lifetime. Before it is shared with clients. + if c.onTrackCBByTrackName == nil { + c.onTrackCBByTrackName = make(map[string]OnTrackCB) + } + + if pc := conn.PeerConn(); pc != nil { + pc.OnTrack(func(trackRemote *webrtc.TrackRemote, rtpReceiver *webrtc.RTPReceiver) { + c.onTrackCBByTrackNameMu.Lock() + onTrackCB, ok := c.onTrackCBByTrackName[trackRemote.StreamID()] + c.onTrackCBByTrackNameMu.Unlock() + if !ok { + logging.Global().Errorf("Callback not found for StreamID (trackName): %s, keys(resOnTrackCBs): %#v", + trackRemote.StreamID(), maps.Keys(c.onTrackCBByTrackName)) + return + } + onTrackCB(trackRemote, rtpReceiver) + }) + } c.connMu.Unlock() } @@ -84,3 +110,17 @@ func (c *ReconfigurableClientConn) Close() error { c.conn = nil return conn.Close() } + +// AddOnTrackSub adds an OnTrack subscription for the track. +func (c *ReconfigurableClientConn) AddOnTrackSub(trackName string, onTrackCB OnTrackCB) { + c.onTrackCBByTrackNameMu.Lock() + defer c.onTrackCBByTrackNameMu.Unlock() + c.onTrackCBByTrackName[trackName] = onTrackCB +} + +// RemoveOnTrackSub removes an OnTrack subscription for the track. +func (c *ReconfigurableClientConn) RemoveOnTrackSub(trackName string) { + c.onTrackCBByTrackNameMu.Lock() + defer c.onTrackCBByTrackNameMu.Unlock() + delete(c.onTrackCBByTrackName, trackName) +} diff --git a/grpc/shared_conn.go b/grpc/shared_conn.go index 88ad2f82ff6..79ba0d0d85b 100644 --- a/grpc/shared_conn.go +++ b/grpc/shared_conn.go @@ -3,7 +3,6 @@ package grpc import ( "context" "errors" - "fmt" "net" "sync" "time" @@ -15,10 +14,10 @@ import ( "go.uber.org/zap" "go.viam.com/utils" "go.viam.com/utils/rpc" + "golang.org/x/exp/maps" googlegrpc "google.golang.org/grpc" "go.viam.com/rdk/logging" - "go.viam.com/rdk/resource" rutils "go.viam.com/rdk/utils" ) @@ -80,8 +79,8 @@ type SharedConn struct { // set to nil before this channel is closed. peerConnFailed chan struct{} - resOnTrackMu sync.Mutex - resOnTrackCBs map[resource.Name]OnTrackCB + onTrackCBByTrackNameMu sync.Mutex + onTrackCBByTrackName map[string]OnTrackCB logger logging.Logger } @@ -106,18 +105,18 @@ func (sc *SharedConn) NewStream( return sc.grpcConn.NewStream(ctx, desc, method, opts...) } -// AddOnTrackSub adds an OnTrack subscription for the resource. -func (sc *SharedConn) AddOnTrackSub(name resource.Name, onTrackCB OnTrackCB) { - sc.resOnTrackMu.Lock() - defer sc.resOnTrackMu.Unlock() - sc.resOnTrackCBs[name] = onTrackCB +// AddOnTrackSub adds an OnTrack subscription for the track. +func (sc *SharedConn) AddOnTrackSub(trackName string, onTrackCB OnTrackCB) { + sc.onTrackCBByTrackNameMu.Lock() + defer sc.onTrackCBByTrackNameMu.Unlock() + sc.onTrackCBByTrackName[trackName] = onTrackCB } -// RemoveOnTrackSub removes an OnTrack subscription for the resource. -func (sc *SharedConn) RemoveOnTrackSub(name resource.Name) { - sc.resOnTrackMu.Lock() - defer sc.resOnTrackMu.Unlock() - delete(sc.resOnTrackCBs, name) +// RemoveOnTrackSub removes an OnTrack subscription for the track. +func (sc *SharedConn) RemoveOnTrackSub(trackName string) { + sc.onTrackCBByTrackNameMu.Lock() + defer sc.onTrackCBByTrackNameMu.Unlock() + delete(sc.onTrackCBByTrackName, trackName) } // GrpcConn returns a gRPC capable client connection. @@ -159,9 +158,11 @@ func (sc *SharedConn) ResetConn(conn rpc.ClientConn, moduleLogger logging.Logger sc.logger = moduleLogger.Sublogger("networking.conn") } - if sc.resOnTrackCBs == nil { + // It is safe to access this without a mutex as it is only ever nil once at the beginning of the + // SharedConn's lifetime + if sc.onTrackCBByTrackName == nil { // Same initilization argument as above with the logger. - sc.resOnTrackCBs = make(map[resource.Name]OnTrackCB) + sc.onTrackCBByTrackName = make(map[string]OnTrackCB) } sc.peerConnMu.Lock() @@ -199,16 +200,12 @@ func (sc *SharedConn) ResetConn(conn rpc.ClientConn, moduleLogger logging.Logger } sc.peerConn.OnTrack(func(trackRemote *webrtc.TrackRemote, rtpReceiver *webrtc.RTPReceiver) { - name, err := resource.NewFromString(trackRemote.StreamID()) - if err != nil { - sc.logger.Errorw("StreamID did not parse as a ResourceName", "sharedConn", fmt.Sprintf("%p", sc), "streamID", trackRemote.StreamID()) - return - } - sc.resOnTrackMu.Lock() - onTrackCB, ok := sc.resOnTrackCBs[name] - sc.resOnTrackMu.Unlock() + sc.onTrackCBByTrackNameMu.Lock() + onTrackCB, ok := sc.onTrackCBByTrackName[trackRemote.StreamID()] + sc.onTrackCBByTrackNameMu.Unlock() if !ok { - sc.logger.Errorw("Callback not found for StreamID", "sharedConn", fmt.Sprintf("%p", sc), "streamID", trackRemote.StreamID()) + msg := "Callback not found for StreamID: %s, keys(resOnTrackCBs): %#v" + sc.logger.Errorf(msg, trackRemote.StreamID(), maps.Keys(sc.onTrackCBByTrackName)) return } onTrackCB(trackRemote, rtpReceiver) @@ -340,6 +337,14 @@ func NewLocalPeerConnection(logger logging.Logger) (*webrtc.PeerConnection, erro return false }) + // RSDK-8547: WebRTC video streams expect an "increasing" SRTP value. When we forward RTP + // packets through different hops, those values are copied as-is. If one connection in this + // chain of hops has a blip, it will reset its SRTP value. Other hops in the chain that were not + // disconnected will see these unexpected change in SRTP values and interpret it as a replay + // attack, dropping the data. We disable this "protection" as per the justification in the noted + // ticket. + settingEngine.DisableSRTPReplayProtection(true) + options := []func(a *webrtc.API){webrtc.WithMediaEngine(&m), webrtc.WithInterceptorRegistry(&i)} if utils.Debug { settingEngine.LoggerFactory = WebRTCLoggerFactory{logger} diff --git a/grpc/tracker.go b/grpc/tracker.go new file mode 100644 index 00000000000..6a2e1943d26 --- /dev/null +++ b/grpc/tracker.go @@ -0,0 +1,9 @@ +package grpc + +// Tracker allows callback functions to a WebRTC peer connection's OnTrack callback +// function by track name. +// Both grpc.SharedConn and grpc.ReconfigurableClientConn implement tracker. +type Tracker interface { + AddOnTrackSub(trackName string, onTrackCB OnTrackCB) + RemoveOnTrackSub(trackName string) +} diff --git a/grpc/tracker_test.go b/grpc/tracker_test.go new file mode 100644 index 00000000000..0c677ad3086 --- /dev/null +++ b/grpc/tracker_test.go @@ -0,0 +1,20 @@ +package grpc + +import ( + "reflect" + "testing" + + "go.viam.com/test" +) + +func TestTrackerImplementations(t *testing.T) { + tracker := reflect.TypeOf((*Tracker)(nil)).Elem() + + t.Run("*ReconfigurableClientConn should implement Tracker", func(t *testing.T) { + test.That(t, reflect.TypeOf(&ReconfigurableClientConn{}).Implements(tracker), test.ShouldBeTrue) + }) + + t.Run("*SharedConn should implement Tracker", func(t *testing.T) { + test.That(t, reflect.TypeOf(&SharedConn{}).Implements(tracker), test.ShouldBeTrue) + }) +} diff --git a/robot/client/client.go b/robot/client/client.go index 1bd33c7e728..f7d0920dbdc 100644 --- a/robot/client/client.go +++ b/robot/client/client.go @@ -393,6 +393,7 @@ func (rc *RobotClient) updateResourceClients(ctx context.Context) error { for resourceName, client := range rc.resourceClients { // check if no longer an active resource if !activeResources[resourceName] { + rc.logger.Infow("Removing resource from remote client", "resourceName", resourceName) if err := client.Close(ctx); err != nil { rc.Logger().CError(ctx, err) continue @@ -574,7 +575,8 @@ func (rc *RobotClient) createClient(name resource.Name) (resource.Resource, erro if !ok || apiInfo.RPCClient == nil { return grpc.NewForeignResource(name, &rc.conn), nil } - return apiInfo.RPCClient(rc.backgroundCtx, &rc.conn, rc.remoteName, name, rc.Logger()) + logger := rc.Logger().Sublogger(resource.RemoveRemoteName(name).ShortName()) + return apiInfo.RPCClient(rc.backgroundCtx, &rc.conn, rc.remoteName, name, logger) } func (rc *RobotClient) resources(ctx context.Context) ([]resource.Name, []resource.RPCAPI, error) { @@ -657,7 +659,7 @@ func (rc *RobotClient) updateResources(ctx context.Context) error { names, rpcAPIs, err := rc.resources(ctx) if err != nil && status.Code(err) != codes.Unimplemented { - return err + return fmt.Errorf("error updating resources: %w", err) } rc.resourceNames = make([]resource.Name, 0, len(names)) diff --git a/robot/impl/local_robot_test.go b/robot/impl/local_robot_test.go index 29ab1c10a69..de422e29695 100644 --- a/robot/impl/local_robot_test.go +++ b/robot/impl/local_robot_test.go @@ -7,7 +7,6 @@ import ( "errors" "fmt" "math" - "net" "os" "path" "path/filepath" @@ -56,7 +55,6 @@ import ( "go.viam.com/rdk/referenceframe" "go.viam.com/rdk/resource" "go.viam.com/rdk/robot" - "go.viam.com/rdk/robot/client" "go.viam.com/rdk/robot/framesystem" "go.viam.com/rdk/robot/packages" putils "go.viam.com/rdk/robot/packages/testutils" @@ -1912,232 +1910,6 @@ func TestConfigMethod(t *testing.T) { test.That(t, actualCfg, test.ShouldResemble, &expectedCfg) } -func TestReconnectRemote(t *testing.T) { - logger := logging.NewTestLogger(t) - options, _, addr := robottestutils.CreateBaseOptionsAndListener(t) - // start the first robot - ctx := context.Background() - armConfig := resource.Config{ - Name: "arm1", - API: arm.API, - Model: fakeModel, - ConvertedAttributes: &fake.Config{ - ModelFilePath: "../../components/arm/fake/fake_model.json", - }, - } - cfg := config.Config{ - Components: []resource.Config{armConfig}, - } - - robot := setupLocalRobot(t, ctx, &cfg, logger) - err := robot.StartWeb(ctx, options) - test.That(t, err, test.ShouldBeNil) - - // start the second robot - ctx1 := context.Background() - options1, _, addr1 := robottestutils.CreateBaseOptionsAndListener(t) - - remoteConf := config.Remote{ - Name: "remote", - Insecure: true, - Address: addr, - } - - cfg1 := config.Config{ - Remotes: []config.Remote{remoteConf}, - } - - robot1 := setupLocalRobot(t, ctx, &cfg1, logger) - - err = robot1.StartWeb(ctx1, options1) - test.That(t, err, test.ShouldBeNil) - - robotClient := robottestutils.NewRobotClient(t, logger, addr1, time.Second) - defer func() { - test.That(t, robotClient.Close(context.Background()), test.ShouldBeNil) - }() - - a1, err := arm.FromRobot(robot1, "arm1") - test.That(t, err, test.ShouldBeNil) - test.That(t, a1, test.ShouldNotBeNil) - - remoteRobot, ok := robot1.RemoteByName("remote") - test.That(t, ok, test.ShouldBeTrue) - test.That(t, remoteRobot, test.ShouldNotBeNil) - remoteRobotClient, ok := remoteRobot.(*client.RobotClient) - test.That(t, ok, test.ShouldBeTrue) - test.That(t, remoteRobotClient, test.ShouldNotBeNil) - - a, err := robotClient.ResourceByName(arm.Named("remote:arm1")) - test.That(t, err, test.ShouldBeNil) - test.That(t, a, test.ShouldNotBeNil) - anArm, ok := a.(arm.Arm) - test.That(t, ok, test.ShouldBeTrue) - _, err = anArm.EndPosition(context.Background(), map[string]interface{}{}) - test.That(t, err, test.ShouldBeNil) - - // close/disconnect the robot - robot.StopWeb() - test.That(t, <-remoteRobotClient.Changed(), test.ShouldBeTrue) - test.That(t, len(remoteRobotClient.ResourceNames()), test.ShouldEqual, 0) - testutils.WaitForAssertion(t, func(tb testing.TB) { - tb.Helper() - test.That(tb, len(robotClient.ResourceNames()), test.ShouldEqual, 2) - }) - test.That(t, len(robot1.ResourceNames()), test.ShouldEqual, 2) - _, err = anArm.EndPosition(context.Background(), map[string]interface{}{}) - test.That(t, err, test.ShouldBeError) - - // reconnect the first robot - ctx2 := context.Background() - listener, err := net.Listen("tcp", addr) - test.That(t, err, test.ShouldBeNil) - - options.Network.Listener = listener - err = robot.StartWeb(ctx2, options) - test.That(t, err, test.ShouldBeNil) - - // check if the original arm can still be called - test.That(t, <-remoteRobotClient.Changed(), test.ShouldBeTrue) - test.That(t, remoteRobotClient.Connected(), test.ShouldBeTrue) - test.That(t, len(remoteRobotClient.ResourceNames()), test.ShouldEqual, 3) - testutils.WaitForAssertion(t, func(tb testing.TB) { - tb.Helper() - test.That(tb, len(robotClient.ResourceNames()), test.ShouldEqual, 5) - }) - test.That(t, len(robot1.ResourceNames()), test.ShouldEqual, 5) - _, err = remoteRobotClient.ResourceByName(arm.Named("arm1")) - test.That(t, err, test.ShouldBeNil) - - _, err = robotClient.ResourceByName(arm.Named("arm1")) - test.That(t, err, test.ShouldBeNil) - _, err = anArm.EndPosition(context.Background(), map[string]interface{}{}) - test.That(t, err, test.ShouldBeNil) -} - -func TestReconnectRemoteChangeConfig(t *testing.T) { - logger := logging.NewTestLogger(t) - - // start the first robot - ctx := context.Background() - options, _, addr := robottestutils.CreateBaseOptionsAndListener(t) - armConfig := resource.Config{ - Name: "arm1", - API: arm.API, - Model: fakeModel, - ConvertedAttributes: &fake.Config{ - ModelFilePath: "../../components/arm/fake/fake_model.json", - }, - } - cfg := config.Config{ - Components: []resource.Config{armConfig}, - } - - robot := setupLocalRobot(t, ctx, &cfg, logger) - err := robot.StartWeb(ctx, options) - test.That(t, err, test.ShouldBeNil) - - // start the second robot - ctx1 := context.Background() - options1, _, addr1 := robottestutils.CreateBaseOptionsAndListener(t) - remoteConf := config.Remote{ - Name: "remote", - Insecure: true, - Address: addr, - } - - cfg1 := config.Config{ - Remotes: []config.Remote{remoteConf}, - } - - robot1 := setupLocalRobot(t, ctx, &cfg1, logger) - - err = robot1.StartWeb(ctx1, options1) - test.That(t, err, test.ShouldBeNil) - - robotClient := robottestutils.NewRobotClient(t, logger, addr1, time.Second) - defer func() { - test.That(t, robotClient.Close(context.Background()), test.ShouldBeNil) - }() - - a1, err := arm.FromRobot(robot1, "arm1") - test.That(t, err, test.ShouldBeNil) - test.That(t, a1, test.ShouldNotBeNil) - - remoteRobot, ok := robot1.RemoteByName("remote") - test.That(t, ok, test.ShouldBeTrue) - test.That(t, remoteRobot, test.ShouldNotBeNil) - remoteRobotClient, ok := remoteRobot.(*client.RobotClient) - test.That(t, ok, test.ShouldBeTrue) - test.That(t, remoteRobotClient, test.ShouldNotBeNil) - - a, err := robotClient.ResourceByName(arm.Named("remote:arm1")) - test.That(t, err, test.ShouldBeNil) - test.That(t, a, test.ShouldNotBeNil) - anArm, ok := a.(arm.Arm) - test.That(t, ok, test.ShouldBeTrue) - _, err = anArm.EndPosition(context.Background(), map[string]interface{}{}) - test.That(t, err, test.ShouldBeNil) - - // close/disconnect the robot - test.That(t, robot.Close(context.Background()), test.ShouldBeNil) - test.That(t, <-remoteRobotClient.Changed(), test.ShouldBeTrue) - test.That(t, len(remoteRobotClient.ResourceNames()), test.ShouldEqual, 0) - testutils.WaitForAssertion(t, func(tb testing.TB) { - tb.Helper() - test.That(tb, len(robotClient.ResourceNames()), test.ShouldEqual, 2) - }) - test.That(t, len(robot1.ResourceNames()), test.ShouldEqual, 2) - _, err = anArm.EndPosition(context.Background(), map[string]interface{}{}) - test.That(t, err, test.ShouldBeError) - - // reconnect the first robot - ctx2 := context.Background() - listener, err := net.Listen("tcp", addr) - test.That(t, err, test.ShouldBeNil) - baseConfig := resource.Config{ - Name: "base1", - API: base.API, - Model: fakeModel, - } - cfg = config.Config{ - Components: []resource.Config{baseConfig}, - } - - options = weboptions.New() - options.Network.BindAddress = "" - options.Network.Listener = listener - robot = setupLocalRobot(t, ctx, &cfg, logger) - err = robot.StartWeb(ctx2, options) - test.That(t, err, test.ShouldBeNil) - - // check if the original arm can't be called anymore - test.That(t, <-remoteRobotClient.Changed(), test.ShouldBeTrue) - test.That(t, remoteRobotClient.Connected(), test.ShouldBeTrue) - test.That(t, len(remoteRobotClient.ResourceNames()), test.ShouldEqual, 3) - testutils.WaitForAssertion(t, func(tb testing.TB) { - tb.Helper() - test.That(tb, len(robotClient.ResourceNames()), test.ShouldEqual, 5) - }) - test.That(t, len(robot1.ResourceNames()), test.ShouldEqual, 5) - _, err = anArm.EndPosition(context.Background(), map[string]interface{}{}) - test.That(t, err, test.ShouldBeError) - - // check that base is now instantiated - _, err = remoteRobotClient.ResourceByName(base.Named("base1")) - test.That(t, err, test.ShouldBeNil) - - b, err := robotClient.ResourceByName(base.Named("remote:base1")) - test.That(t, err, test.ShouldBeNil) - aBase, ok := b.(base.Base) - test.That(t, ok, test.ShouldBeTrue) - - err = aBase.Stop(ctx, map[string]interface{}{}) - test.That(t, err, test.ShouldBeNil) - - test.That(t, len(robotClient.ResourceNames()), test.ShouldEqual, 5) -} - func TestCheckMaxInstanceValid(t *testing.T) { logger := logging.NewTestLogger(t) cfg := &config.Config{ diff --git a/robot/impl/resource_manager.go b/robot/impl/resource_manager.go index 2f0554ab75d..ffafd47bb3d 100644 --- a/robot/impl/resource_manager.go +++ b/robot/impl/resource_manager.go @@ -3,6 +3,7 @@ package robotimpl import ( "context" "crypto/tls" + "errors" "fmt" "os" "reflect" @@ -11,7 +12,6 @@ import ( "time" "github.com/jhump/protoreflect/desc" - "github.com/pkg/errors" "go.uber.org/multierr" goutils "go.viam.com/utils" "go.viam.com/utils/pexec" @@ -182,7 +182,7 @@ func (manager *resourceManager) updateRemoteResourceNames( rr internalRemoteRobot, recreateAllClients bool, ) bool { - manager.logger.CDebugw(ctx, "updating remote resource names", "remote", remoteName) + manager.logger.CDebugw(ctx, "updating remote resource names", "remote", remoteName, "recreateAllClients", recreateAllClients) activeResourceNames := map[resource.Name]bool{} newResources := rr.ResourceNames() oldResources := manager.remoteResourceNames(remoteName) @@ -207,13 +207,11 @@ func (manager *resourceManager) updateRemoteResourceNames( } continue } - resName = resName.PrependRemote(remoteName.Name) - gNode, ok := manager.resources.Node(resName) - + gNode, nodeAlreadyExists := manager.resources.Node(resName) if _, alreadyCurrent := activeResourceNames[resName]; alreadyCurrent { activeResourceNames[resName] = true - if ok && !gNode.IsUninitialized() { + if nodeAlreadyExists && !gNode.IsUninitialized() { // resources that enter this block represent those with names that already exist in the resource graph. // it is possible that we are switching to a new remote with a identical resource name(s), so we may // need to create these resource clients. @@ -239,7 +237,7 @@ func (manager *resourceManager) updateRemoteResourceNames( } } - if ok { + if nodeAlreadyExists { gNode.SwapResource(res, unknownModel) } else { gNode = resource.NewConfiguredGraphNode(resource.Config{}, res, unknownModel) @@ -471,7 +469,7 @@ func (manager *resourceManager) closeResource(ctx context.Context, res resource. resName := res.Name() if manager.moduleManager != nil && manager.moduleManager.IsModularResource(resName) { if err := manager.moduleManager.RemoveResource(closeCtx, resName); err != nil { - allErrs = multierr.Combine(allErrs, errors.Wrap(err, "error removing modular resource for closure")) + allErrs = multierr.Combine(allErrs, fmt.Errorf("error removing modular resource for closure: %w", err)) } } @@ -526,7 +524,7 @@ func (manager *resourceManager) Close(ctx context.Context) error { var allErrs error if err := manager.processManager.Stop(); err != nil { - allErrs = multierr.Combine(allErrs, errors.Wrap(err, "error stopping process manager")) + allErrs = multierr.Combine(allErrs, fmt.Errorf("error stopping process manager: %w", err)) } // our caller will close web @@ -540,7 +538,7 @@ func (manager *resourceManager) Close(ctx context.Context) error { // moduleManager may be nil in tests, and must be closed last, after resources within have been closed properly above if manager.moduleManager != nil { if err := manager.moduleManager.Close(ctx); err != nil { - allErrs = multierr.Combine(allErrs, errors.Wrap(err, "error closing module manager")) + allErrs = multierr.Combine(allErrs, fmt.Errorf("error closing module manager: %w", err)) } } @@ -625,8 +623,7 @@ func (manager *resourceManager) completeConfig( if gNode.IsUninitialized() { verb = "configuring" gNode.InitializeLogger( - manager.logger, resName.String(), conf.LogConfiguration.Level, - ) + manager.logger, resName.String(), conf.LogConfiguration.Level) } else { verb = "reconfiguring" } @@ -756,8 +753,7 @@ func (manager *resourceManager) completeConfigForRemotes(ctx context.Context, lr } if gNode.IsUninitialized() { gNode.InitializeLogger( - manager.logger, fromRemoteNameToRemoteNodeName(remConf.Name).String(), manager.logger.GetLevel(), - ) + manager.logger, fromRemoteNameToRemoteNodeName(remConf.Name).String(), logging.INFO) } // this is done in config validation but partial start rules require us to check again if _, err := remConf.Validate(""); err != nil { @@ -877,7 +873,7 @@ func (manager *resourceManager) processRemote( err = errors.New("must use Config.AllowInsecureCreds to connect to a non-TLS secured robot") } } - return nil, errors.Errorf("couldn't connect to robot remote (%s): %s", config.Address, err) + return nil, fmt.Errorf("couldn't connect to robot remote (%s): %w", config.Address, err) } manager.logger.CInfow(ctx, "Connected now to remote", "remote", config.Name) return robotClient, nil @@ -1016,7 +1012,7 @@ func (manager *resourceManager) markResourceForUpdate(name resource.Name, conf r gNode = resource.NewUnconfiguredGraphNode(conf, deps) gNode.UpdatePendingRevision(revision) if err := manager.resources.AddNode(name, gNode); err != nil { - return errors.Errorf("failed to add new node for unconfigured resource %q: %v", name, err) + return fmt.Errorf("failed to add new node for unconfigured resource %q: %w", name, err) } return nil } diff --git a/robot/web/stream/camera/camera.go b/robot/web/stream/camera/camera.go new file mode 100644 index 00000000000..948bfc112dc --- /dev/null +++ b/robot/web/stream/camera/camera.go @@ -0,0 +1,21 @@ +// Package camera provides functions for looking up a camera from a robot using a stream +package camera + +import ( + "go.viam.com/rdk/components/camera" + "go.viam.com/rdk/gostream" + "go.viam.com/rdk/resource" + "go.viam.com/rdk/robot" +) + +// Camera returns the camera from the robot (derived from the stream) or +// an error if it has no camera. +func Camera(robot robot.Robot, stream gostream.Stream) (camera.Camera, error) { + // Stream names are slightly modified versions of the resource short name + shortName := resource.SDPTrackNameToShortName(stream.Name()) + cam, err := camera.FromRobot(robot, shortName) + if err != nil { + return nil, err + } + return cam, nil +} diff --git a/robot/web/stream/server.go b/robot/web/stream/server.go index 3a69d2c0177..0baa47092e2 100644 --- a/robot/web/stream/server.go +++ b/robot/web/stream/server.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "sync" + "time" "github.com/pkg/errors" "github.com/viamrobotics/webrtc/v3" @@ -13,13 +14,18 @@ import ( "go.viam.com/utils" "go.viam.com/utils/rpc" + "go.viam.com/rdk/components/camera" "go.viam.com/rdk/gostream" "go.viam.com/rdk/logging" + "go.viam.com/rdk/resource" "go.viam.com/rdk/robot" + streamCamera "go.viam.com/rdk/robot/web/stream/camera" "go.viam.com/rdk/robot/web/stream/state" rutils "go.viam.com/rdk/utils" ) +var monitorCameraInterval = time.Second + type peerState struct { streamState *state.StreamState senders []*webrtc.RTPSender @@ -28,11 +34,12 @@ type peerState struct { // Server implements the gRPC audio/video streaming service. type Server struct { streampb.UnimplementedStreamServiceServer - logger logging.Logger - r robot.Robot + logger logging.Logger + robot robot.Robot + closedCtx context.Context + closedFn context.CancelFunc mu sync.RWMutex - streamNames []string nameToStreamState map[string]*state.StreamState activePeerStreams map[*webrtc.PeerConnection]map[string]*peerState activeBackgroundWorkers sync.WaitGroup @@ -43,11 +50,14 @@ type Server struct { // stream. func NewServer( streams []gostream.Stream, - r robot.Robot, + robot robot.Robot, logger logging.Logger, ) (*Server, error) { - ss := &Server{ - r: r, + closedCtx, closedFn := context.WithCancel(context.Background()) + server := &Server{ + closedCtx: closedCtx, + closedFn: closedFn, + robot: robot, logger: logger, nameToStreamState: map[string]*state.StreamState{}, activePeerStreams: map[*webrtc.PeerConnection]map[string]*peerState{}, @@ -55,11 +65,13 @@ func NewServer( } for _, stream := range streams { - if err := ss.add(stream); err != nil { + if err := server.add(stream); err != nil { return nil, err } } - return ss, nil + server.startMonitorCameraAvailable() + + return server, nil } // StreamAlreadyRegisteredError indicates that a stream has a name that is already registered on @@ -73,20 +85,20 @@ func (e *StreamAlreadyRegisteredError) Error() string { } // NewStream informs the stream server of new streams that are capable of being streamed. -func (ss *Server) NewStream(config gostream.StreamConfig) (gostream.Stream, error) { - ss.mu.Lock() - defer ss.mu.Unlock() +func (server *Server) NewStream(config gostream.StreamConfig) (gostream.Stream, error) { + server.mu.Lock() + defer server.mu.Unlock() - if _, ok := ss.nameToStreamState[config.Name]; ok { + if _, ok := server.nameToStreamState[config.Name]; ok { return nil, &StreamAlreadyRegisteredError{config.Name} } - stream, err := gostream.NewStream(config, ss.logger) + stream, err := gostream.NewStream(config, server.logger) if err != nil { return nil, err } - if err = ss.add(stream); err != nil { + if err = server.add(stream); err != nil { return nil, err } @@ -94,49 +106,62 @@ func (ss *Server) NewStream(config gostream.StreamConfig) (gostream.Stream, erro } // ListStreams implements part of the StreamServiceServer. -func (ss *Server) ListStreams(ctx context.Context, req *streampb.ListStreamsRequest) (*streampb.ListStreamsResponse, error) { +func (server *Server) ListStreams(ctx context.Context, req *streampb.ListStreamsRequest) (*streampb.ListStreamsResponse, error) { _, span := trace.StartSpan(ctx, "stream::server::ListStreams") defer span.End() - ss.mu.RLock() - defer ss.mu.RUnlock() + server.mu.RLock() + defer server.mu.RUnlock() - names := make([]string, 0, len(ss.streamNames)) - for _, name := range ss.streamNames { - streamState := ss.nameToStreamState[name] - names = append(names, streamState.Stream.Name()) + names := make([]string, 0, len(server.nameToStreamState)) + for name := range server.nameToStreamState { + names = append(names, name) } return &streampb.ListStreamsResponse{Names: names}, nil } // AddStream implements part of the StreamServiceServer. -func (ss *Server) AddStream(ctx context.Context, req *streampb.AddStreamRequest) (*streampb.AddStreamResponse, error) { +func (server *Server) AddStream(ctx context.Context, req *streampb.AddStreamRequest) (*streampb.AddStreamResponse, error) { ctx, span := trace.StartSpan(ctx, "stream::server::AddStream") defer span.End() - // Get the peer connection + // Get the peer connection to the caller. pc, ok := rpc.ContextPeerConnection(ctx) + server.logger.Infow("Adding video stream", "name", req.Name, "peerConn", pc) + defer server.logger.Warnf("AddStream END %s", req.Name) + if !ok { return nil, errors.New("can only add a stream over a WebRTC based connection") } - ss.mu.Lock() - defer ss.mu.Unlock() + server.mu.Lock() + defer server.mu.Unlock() - streamStateToAdd, ok := ss.nameToStreamState[req.Name] + streamStateToAdd, ok := server.nameToStreamState[req.Name] // return error if there is no stream for that camera if !ok { - err := fmt.Errorf("no stream for %q", req.Name) - ss.logger.Error(err.Error()) + var availableStreams string + for n := range server.nameToStreamState { + if availableStreams != "" { + availableStreams += ", " + } + availableStreams += fmt.Sprintf("%q", n) + } + err := fmt.Errorf("no stream for %q, available streams: %s", req.Name, availableStreams) + server.logger.Error(err.Error()) + return nil, err + } + // return error if camera is not in resource graph + if _, err := streamCamera.Camera(server.robot, streamStateToAdd.Stream); err != nil { return nil, err } // return error if the caller's peer connection is already being sent video data - if _, ok := ss.activePeerStreams[pc][req.Name]; ok { + if _, ok := server.activePeerStreams[pc][req.Name]; ok { err := errors.New("stream already active") - ss.logger.Error(err.Error()) + server.logger.Error(err.Error()) return nil, err } - nameToPeerState, ok := ss.activePeerStreams[pc] + nameToPeerState, ok := server.activePeerStreams[pc] // if there is no active video data being sent, set up a callback to remove the peer connection from // the active streams & stop the stream from doing h264 encode if this is the last peer connection // subcribed to the camera's video feed @@ -145,16 +170,16 @@ func (ss *Server) AddStream(ctx context.Context, req *streampb.AddStreamRequest) if !ok { nameToPeerState = map[string]*peerState{} pc.OnConnectionStateChange(func(peerConnectionState webrtc.PeerConnectionState) { - ss.logger.Debugf("%s pc.OnConnectionStateChange state: %s", req.Name, peerConnectionState) + server.logger.Infof("%s pc.OnConnectionStateChange state: %s", req.Name, peerConnectionState) switch peerConnectionState { case webrtc.PeerConnectionStateDisconnected, webrtc.PeerConnectionStateFailed, webrtc.PeerConnectionStateClosed: - ss.mu.Lock() - defer ss.mu.Unlock() + server.mu.Lock() + defer server.mu.Unlock() - if ss.isAlive { + if server.isAlive { // Dan: This conditional closing on `isAlive` is a hack to avoid a data // race. Shutting down a robot causes the PeerConnection to be closed // concurrently with this `stream.Server`. Thus, `stream.Server.Close` waiting @@ -162,25 +187,22 @@ func (ss *Server) AddStream(ctx context.Context, req *streampb.AddStreamRequest) // "worker". Given `Close` is expected to `Stop` remaining streams, we can elide // spinning off the below goroutine. // - // Given this is an existing race, I'm choosing to add to the tech - // debt rather than architect how shutdown should holistically work. Revert this - // change and run `TestRobotPeerConnect` (double check the test name at PR time) - // to reproduce the race. - ss.activeBackgroundWorkers.Add(1) + // Given this is an existing race, I'm choosing to add to the tech debt rather + // than architect how shutdown should holistically work. Revert this change and + // run `TestAudioTrackIsNotCreatedForVideoStream` to reproduce the race. + server.activeBackgroundWorkers.Add(1) utils.PanicCapturingGo(func() { - defer ss.activeBackgroundWorkers.Done() - ss.mu.Lock() - defer ss.mu.Unlock() - defer delete(ss.activePeerStreams, pc) + defer server.activeBackgroundWorkers.Done() + server.mu.Lock() + defer server.mu.Unlock() + defer delete(server.activePeerStreams, pc) var errs error - for _, ps := range ss.activePeerStreams[pc] { - ctx, cancel := context.WithTimeout(context.Background(), state.UnsubscribeTimeout) - errs = multierr.Combine(errs, ps.streamState.Decrement(ctx)) - cancel() + for _, ps := range server.activePeerStreams[pc] { + errs = multierr.Combine(errs, ps.streamState.Decrement()) } // We don't want to log this if the streamState was closed (as it only happens if viam-server is terminating) if errs != nil && !errors.Is(errs, state.ErrClosed) { - ss.logger.Errorw("error(s) stopping the streamState", "errs", errs) + server.logger.Errorw("error(s) stopping the streamState", "errs", errs) } }) } @@ -192,7 +214,7 @@ func (ss *Server) AddStream(ctx context.Context, req *streampb.AddStreamRequest) return } }) - ss.activePeerStreams[pc] = nameToPeerState + server.activePeerStreams[pc] = nameToPeerState } ps, ok := nameToPeerState[req.Name] @@ -221,19 +243,19 @@ func (ss *Server) AddStream(ctx context.Context, req *streampb.AddStreamRequest) // if the stream supports video, add the video track if trackLocal, haveTrackLocal := streamStateToAdd.Stream.VideoTrackLocal(); haveTrackLocal { if err := addTrack(trackLocal); err != nil { - ss.logger.Error(err.Error()) + server.logger.Error(err.Error()) return nil, err } } // if the stream supports audio, add the audio track if trackLocal, haveTrackLocal := streamStateToAdd.Stream.AudioTrackLocal(); haveTrackLocal { if err := addTrack(trackLocal); err != nil { - ss.logger.Error(err.Error()) + server.logger.Error(err.Error()) return nil, err } } - if err := streamStateToAdd.Increment(ctx); err != nil { - ss.logger.Error(err.Error()) + if err := streamStateToAdd.Increment(); err != nil { + server.logger.Error(err.Error()) return nil, err } @@ -242,70 +264,147 @@ func (ss *Server) AddStream(ctx context.Context, req *streampb.AddStreamRequest) } // RemoveStream implements part of the StreamServiceServer. -func (ss *Server) RemoveStream(ctx context.Context, req *streampb.RemoveStreamRequest) (*streampb.RemoveStreamResponse, error) { +func (server *Server) RemoveStream(ctx context.Context, req *streampb.RemoveStreamRequest) (*streampb.RemoveStreamResponse, error) { ctx, span := trace.StartSpan(ctx, "stream::server::RemoveStream") defer span.End() pc, ok := rpc.ContextPeerConnection(ctx) + server.logger.Infow("Removing video stream", "name", req.Name, "peerConn", pc) if !ok { return nil, errors.New("can only remove a stream over a WebRTC based connection") } - ss.mu.Lock() - defer ss.mu.Unlock() + server.mu.Lock() + defer server.mu.Unlock() - streamToRemove, ok := ss.nameToStreamState[req.Name] + streamToRemove, ok := server.nameToStreamState[req.Name] + // Callers of RemoveStream will continue calling RemoveStream until it succeeds. Retrying on the + // following "stream not found" errors is not helpful in this goal. Thus we return a success + // response. if !ok { - return nil, fmt.Errorf("no stream for %q", req.Name) + return &streampb.RemoveStreamResponse{}, nil } - if _, ok := ss.activePeerStreams[pc][req.Name]; !ok { - return nil, errors.New("stream already inactive") + //nolint:nilerr + if _, err := streamCamera.Camera(server.robot, streamToRemove.Stream); err != nil { + return &streampb.RemoveStreamResponse{}, nil + } + + if _, ok := server.activePeerStreams[pc][req.Name]; !ok { + return &streampb.RemoveStreamResponse{}, nil } var errs error - for _, sender := range ss.activePeerStreams[pc][req.Name].senders { + for _, sender := range server.activePeerStreams[pc][req.Name].senders { errs = multierr.Combine(errs, pc.RemoveTrack(sender)) } if errs != nil { - ss.logger.Error(errs.Error()) + server.logger.Error(errs.Error()) return nil, errs } - if err := streamToRemove.Decrement(ctx); err != nil { - ss.logger.Error(err.Error()) + if err := streamToRemove.Decrement(); err != nil { + server.logger.Error(err.Error()) return nil, err } - delete(ss.activePeerStreams[pc], req.Name) + delete(server.activePeerStreams[pc], req.Name) return &streampb.RemoveStreamResponse{}, nil } // Close closes the Server and waits for spun off goroutines to complete. -func (ss *Server) Close() error { - ss.mu.Lock() - ss.isAlive = false +func (server *Server) Close() error { + server.closedFn() + server.mu.Lock() + server.isAlive = false var errs error - for _, name := range ss.streamNames { - errs = multierr.Combine(errs, ss.nameToStreamState[name].Close()) + for _, streamState := range server.nameToStreamState { + errs = multierr.Combine(errs, streamState.Close()) } if errs != nil { - ss.logger.Errorf("Stream Server Close > StreamState.Close() errs: %s", errs) + server.logger.Errorf("Stream Server Close > StreamState.Close() errs: %s", errs) } - ss.mu.Unlock() - ss.activeBackgroundWorkers.Wait() + server.mu.Unlock() + server.activeBackgroundWorkers.Wait() return errs } -func (ss *Server) add(stream gostream.Stream) error { +func (server *Server) add(stream gostream.Stream) error { streamName := stream.Name() - if _, ok := ss.nameToStreamState[streamName]; ok { + if _, ok := server.nameToStreamState[streamName]; ok { return &StreamAlreadyRegisteredError{streamName} } - newStreamState := state.New(stream, ss.r, ss.logger) - newStreamState.Init() - ss.nameToStreamState[streamName] = newStreamState - ss.streamNames = append(ss.streamNames, streamName) + logger := server.logger.Sublogger(streamName) + newStreamState := state.New(stream, server.robot, logger) + server.nameToStreamState[streamName] = newStreamState return nil } + +// startMonitorCameraAvailable monitors whether or not the camera still exists +// If it no longer exists, it: +// 1. calls RemoveTrack on the senders of all peer connections that called AddTrack on the camera name. +// 2. decrements the number of active peers on the stream state (which should result in the +// stream state having no subscribers and calling gostream.Stop() or rtppaserverthrough.Unsubscribe) +// streaming tracks from it. +func (server *Server) startMonitorCameraAvailable() { + server.activeBackgroundWorkers.Add(1) + utils.ManagedGo(func() { + for utils.SelectContextOrWait(server.closedCtx, monitorCameraInterval) { + server.removeMissingStreams() + } + }, server.activeBackgroundWorkers.Done) +} + +func (server *Server) removeMissingStreams() { + server.mu.Lock() + defer server.mu.Unlock() + for key, streamState := range server.nameToStreamState { + // Stream names are slightly modified versions of the resource short name + camName := streamState.Stream.Name() + shortName := resource.SDPTrackNameToShortName(camName) + + _, err := camera.FromRobot(server.robot, shortName) + if !resource.IsNotFoundError(err) { + // Cameras can go through transient states during reconfigure that don't necessarily + // imply the camera is missing. E.g: *resource.notAvailableError. To double-check we + // have the right set of exceptions here, we log the error and ignore. + if err != nil { + server.logger.Warnw("Error getting camera from robot", + "camera", camName, "err", err, "errType", fmt.Sprintf("%T", err)) + } + continue + } + + // Best effort close any active peer streams. We'll remove from the known streams + // first. Such that we only try closing/unsubscribing once. + server.logger.Infow("Camera doesn't exist. Closing its streams", + "camera", camName, "err", err, "Type", fmt.Sprintf("%T", err)) + delete(server.nameToStreamState, key) + + for pc, peerStateByCamName := range server.activePeerStreams { + peerState, ok := peerStateByCamName[camName] + if !ok { + // There are no known peers for this camera. Do nothing. + server.logger.Infow("no entry in peer map", "camera", camName) + continue + } + + server.logger.Infow("unsubscribing", "camera", camName, "numSenders", len(peerState.senders)) + var errs error + for _, sender := range peerState.senders { + errs = multierr.Combine(errs, pc.RemoveTrack(sender)) + } + + if errs != nil { + server.logger.Warn(errs.Error()) + } + + if err := streamState.Decrement(); err != nil { + server.logger.Warn(err.Error()) + } + delete(server.activePeerStreams[pc], camName) + } + utils.UncheckedError(streamState.Close()) + } +} diff --git a/robot/web/stream/state/state.go b/robot/web/stream/state/state.go index 2f68699ff55..7a51600e72f 100644 --- a/robot/web/stream/state/state.go +++ b/robot/web/stream/state/state.go @@ -4,13 +4,13 @@ package state import ( "context" + "errors" "fmt" "sync" "sync/atomic" "time" "github.com/pion/rtp" - "github.com/pkg/errors" "go.uber.org/multierr" "go.viam.com/utils" @@ -19,37 +19,28 @@ import ( "go.viam.com/rdk/gostream" "go.viam.com/rdk/logging" "go.viam.com/rdk/robot" + streamCamera "go.viam.com/rdk/robot/web/stream/camera" ) -var ( - subscribeRTPTimeout = time.Second * 5 - // UnsubscribeTimeout is the timeout used when unsubscribing from an rtppassthrough subscription. - UnsubscribeTimeout = time.Second * 5 - // ErrRTPPassthroughNotSupported indicates that rtp_passthrough is not supported by the stream's camera. - ErrRTPPassthroughNotSupported = errors.New("RTP Passthrough Not Supported") - // ErrClosed indicates that the StreamState is already closed. - ErrClosed = errors.New("StreamState already closed") - // ErrUninitialized indicates that Init() has not been called on StreamState prior to Increment or Decrement being called. - ErrUninitialized = errors.New("uniniialized") -) +// ErrClosed indicates that the StreamState is already closed. +var ErrClosed = errors.New("StreamState already closed") // StreamState controls the source of the RTP packets being written to the stream's subscribers // and ensures there is only one active at a time while there are subsribers. type StreamState struct { // Stream is the StreamState's stream - Stream gostream.Stream - robot robot.Robot - closedCtx context.Context - closedFn context.CancelFunc - wg sync.WaitGroup - logger logging.Logger - initialized atomic.Bool - - msgChan chan msg - restartChan chan struct{} - - activePeers int - streamSource streamSource + Stream gostream.Stream + robot robot.Robot + closedCtx context.Context + closedFn context.CancelFunc + wg sync.WaitGroup + logger logging.Logger + + msgChan chan msg + tickChan chan struct{} + + activeClients int + streamSource streamSource // streamSourceSub is only non nil if streamSource == streamSourcePassthrough streamSourceSub rtppassthrough.Subscription } @@ -63,61 +54,47 @@ func New( logger logging.Logger, ) *StreamState { ctx, cancel := context.WithCancel(context.Background()) - return &StreamState{ - Stream: stream, - closedCtx: ctx, - closedFn: cancel, - robot: r, - msgChan: make(chan msg), - restartChan: make(chan struct{}), - logger: logger, - } -} - -// Init initializes the StreamState -// Init must be called before any other methods. -func (ss *StreamState) Init() { - ss.wg.Add(1) - utils.ManagedGo(ss.initEventHandler, ss.wg.Done) - ss.wg.Add(1) - utils.ManagedGo(ss.initStreamSourceMonitor, ss.wg.Done) - ss.initialized.Store(true) + ret := &StreamState{ + Stream: stream, + closedCtx: ctx, + closedFn: cancel, + robot: r, + msgChan: make(chan msg), + tickChan: make(chan struct{}), + logger: logger, + } + + ret.wg.Add(1) + // The event handler for a stream input manages the following events: + // - There's a new subscriber (bump ref counter) + // - A subscriber has left (dec ref counter) + // - Camera is remevod (dec for all subscribers) + // - Peer connection is closed (dec for all subscribers) + utils.ManagedGo(ret.sourceEventHandler, ret.wg.Done) + return ret } // Increment increments the peer connections subscribed to the stream. -func (ss *StreamState) Increment(ctx context.Context) error { - if !ss.initialized.Load() { - return ErrUninitialized - } - if err := ss.closedCtx.Err(); err != nil { +func (state *StreamState) Increment() error { + if err := state.closedCtx.Err(); err != nil { return multierr.Combine(ErrClosed, err) } - return ss.send(ctx, msgTypeIncrement) + return state.send(msgTypeIncrement) } // Decrement decrements the peer connections subscribed to the stream. -func (ss *StreamState) Decrement(ctx context.Context) error { - if !ss.initialized.Load() { - return ErrUninitialized - } - if err := ss.closedCtx.Err(); err != nil { +func (state *StreamState) Decrement() error { + if err := state.closedCtx.Err(); err != nil { return multierr.Combine(ErrClosed, err) } - return ss.send(ctx, msgTypeDecrement) -} - -// Restart restarts the stream source after it has terminated. -func (ss *StreamState) Restart(ctx context.Context) { - if err := ss.closedCtx.Err(); err != nil { - return - } - utils.UncheckedError(ss.send(ctx, msgTypeRestart)) + return state.send(msgTypeDecrement) } // Close closes the StreamState. -func (ss *StreamState) Close() error { - ss.closedFn() - ss.wg.Wait() +func (state *StreamState) Close() error { + state.logger.Info("Closing streamState") + state.closedFn() + state.wg.Wait() return nil } @@ -152,7 +129,6 @@ const ( msgTypeUnknown msgType = iota msgTypeIncrement msgTypeDecrement - msgTypeRestart ) func (mt msgType) String() string { @@ -161,8 +137,6 @@ func (mt msgType) String() string { return "Increment" case msgTypeDecrement: return "Decrement" - case msgTypeRestart: - return "Restart" case msgTypeUnknown: fallthrough default: @@ -171,296 +145,214 @@ func (mt msgType) String() string { } type msg struct { - msgType msgType - ctx context.Context - respChan chan error + msgType msgType } -// events (Inc Dec Restart). -func (ss *StreamState) initEventHandler() { - ss.logger.Debug("StreamState initEventHandler booted") - defer ss.logger.Debug("StreamState initEventHandler terminated") +// events (Inc Dec Tick). +func (state *StreamState) sourceEventHandler() { + state.logger.Debug("sourceEventHandler booted") defer func() { - ctx, cancel := context.WithTimeout(context.Background(), UnsubscribeTimeout) - defer cancel() - utils.UncheckedError(ss.stopBasedOnSub(ctx)) + state.logger.Debug("sourceEventHandler terminating") + state.stopInputStream() }() + ticker := time.NewTicker(time.Second) + defer ticker.Stop() for { - if ss.closedCtx.Err() != nil { - return - } - + // We wait for: + // - A message to be discovered on the `msgChan` queue. + // - The `tick` timer to be fired + // - The server/stream (i.e: camera) to be shutdown. + var msg msg select { - case <-ss.closedCtx.Done(): + case <-state.closedCtx.Done(): return - case msg := <-ss.msgChan: - ss.handleMsg(msg) + case msg = <-state.msgChan: + case <-ticker.C: + state.tick() + continue } - } -} -func (ss *StreamState) initStreamSourceMonitor() { - for { - select { - case <-ss.closedCtx.Done(): - return - case <-ss.restartChan: - ctx, cancel := context.WithTimeout(ss.closedCtx, subscribeRTPTimeout) - ss.Restart(ctx) - cancel() + switch msg.msgType { + case msgTypeIncrement: + state.activeClients++ + state.logger.Debugw("activeClients incremented", "activeClientCnt", state.activeClients) + if state.activeClients == 1 { + state.tick() + } + case msgTypeDecrement: + state.activeClients-- + state.logger.Debugw("activeClients decremented", "activeClientCnt", state.activeClients) + if state.activeClients == 0 { + state.tick() + } + case msgTypeUnknown: + fallthrough + default: + state.logger.Errorw("Invalid StreamState msg type received", "type", msg.msgType) } } } -func (ss *StreamState) monitorSubscription(sub rtppassthrough.Subscription) { - if ss.streamSource == streamSourceGoStream { - ss.logger.Debugf("monitorSubscription stopping gostream %s", ss.Stream.Name()) - // if we were streaming using gostream, stop streaming using gostream as we are now using passthrough - ss.Stream.Stop() - } - ss.streamSourceSub = sub - ss.streamSource = streamSourcePassthrough - monitorSubFunc := func() { - // if the stream state is shutting down, terminate - if ss.closedCtx.Err() != nil { - return - } - +func (state *StreamState) monitorSubscription(terminatedCtx context.Context) { + select { + case <-state.closedCtx.Done(): + return + case <-terminatedCtx.Done(): select { - case <-ss.closedCtx.Done(): - return - case <-sub.Terminated.Done(): - select { - case ss.restartChan <- struct{}{}: - case <-ss.closedCtx.Done(): - } - return + case state.tickChan <- struct{}{}: + state.logger.Info("monitorSubscription sent to tickChan") + default: } + return } - - ss.wg.Add(1) - utils.ManagedGo(monitorSubFunc, ss.wg.Done) } -// caller must be holding ss.mu. -func (ss *StreamState) stopBasedOnSub(ctx context.Context) error { - switch ss.streamSource { +func (state *StreamState) stopInputStream() { + switch state.streamSource { case streamSourceGoStream: - ss.logger.Debugf("%s stopBasedOnSub stopping GoStream", ss.Stream.Name()) - ss.Stream.Stop() - ss.streamSource = streamSourceUnknown - return nil + state.logger.Debug("stopping gostream stream") + defer state.logger.Debug("gostream stopped") + state.Stream.Stop() + state.streamSource = streamSourceUnknown + return case streamSourcePassthrough: - ss.logger.Debugf("%s stopBasedOnSub stopping passthrough", ss.Stream.Name()) - err := ss.unsubscribeH264Passthrough(ctx, ss.streamSourceSub.ID) - if err != nil { - return err + state.logger.Debug("stopping h264 passthrough stream") + defer state.logger.Debug("h264 passthrough stream stopped") + err := state.unsubscribeH264Passthrough(state.closedCtx, state.streamSourceSub.ID) + if err != nil && errors.Is(err, camera.ErrUnknownSubscriptionID) { + state.logger.Warnw("Error calling unsubscribe", "err", err) + return } - ss.streamSourceSub = rtppassthrough.NilSubscription - ss.streamSource = streamSourceUnknown - return nil - + state.streamSourceSub = rtppassthrough.NilSubscription + state.streamSource = streamSourceUnknown case streamSourceUnknown: - fallthrough default: - return nil } } -func (ss *StreamState) send(ctx context.Context, msgType msgType) error { - if err := ctx.Err(); err != nil { - return err - } - if err := ss.closedCtx.Err(); err != nil { - return err - } - msg := msg{ - ctx: ctx, - msgType: msgType, - respChan: make(chan error), - } +func (state *StreamState) send(msgType msgType) error { select { - case ss.msgChan <- msg: - select { - case err := <-msg.respChan: - return err - case <-ctx.Done(): - return ctx.Err() - case <-ss.closedCtx.Done(): - return ss.closedCtx.Err() - } - case <-ctx.Done(): - return ctx.Err() - case <-ss.closedCtx.Done(): - return ss.closedCtx.Err() - } -} - -func (ss *StreamState) handleMsg(msg msg) { - switch msg.msgType { - case msgTypeIncrement: - err := ss.inc(msg.ctx) - select { - case msg.respChan <- err: - case <-ss.closedCtx.Done(): - return - } - case msgTypeRestart: - ss.restart(msg.ctx) - select { - case msg.respChan <- nil: - case <-ss.closedCtx.Done(): - return - } - case msgTypeDecrement: - err := ss.dec(msg.ctx) - msg.respChan <- err - case msgTypeUnknown: - fallthrough - default: - ss.logger.Error("Invalid StreamState msg type received: %s", msg.msgType) + case state.msgChan <- msg{msgType: msgType}: + return nil + case <-state.closedCtx.Done(): + return state.closedCtx.Err() } } -func (ss *StreamState) inc(ctx context.Context) error { - ss.logger.Debugf("increment %s START activePeers: %d", ss.Stream.Name(), ss.activePeers) - defer func() { ss.logger.Debugf("increment %s END activePeers: %d", ss.Stream.Name(), ss.activePeers) }() - if ss.activePeers == 0 { - if ss.streamSource != streamSourceUnknown { - return fmt.Errorf("unexpected stream %s source %s", ss.Stream.Name(), ss.streamSource) - } +func (state *StreamState) tick() { + switch { + case state.activeClients < 0: + state.logger.Error("activeClients is less than 0") + case state.activeClients == 0: + // stop stream if there are no active clients + // noop if there is no stream source + state.stopInputStream() + case state.streamSource == streamSourceUnknown: // && state.activeClients > 0 // this is the first subscription, attempt passthrough - ss.logger.CDebugw(ctx, "attempting to subscribe to rtp_passthrough", "name", ss.Stream.Name()) - err := ss.streamH264Passthrough(ctx) + state.logger.Info("attempting to subscribe to rtp_passthrough") + err := state.streamH264Passthrough() if err != nil { - ss.logger.CDebugw(ctx, "rtp_passthrough not possible, falling back to GoStream", "err", err.Error(), "name", ss.Stream.Name()) + state.logger.Warnw("tick: rtp_passthrough not possible, falling back to GoStream", "err", err) // if passthrough failed, fall back to gostream based approach - ss.Stream.Start() - ss.streamSource = streamSourceGoStream + state.Stream.Start() + state.streamSource = streamSourceGoStream } - ss.activePeers++ - return nil - } + case state.streamSource == streamSourcePassthrough && state.streamSourceSub.Terminated.Err() != nil: + // restart stream if there we were using passthrough but the sub is terminated + state.logger.Info("previous subscription terminated attempting to subscribe to rtp_passthrough") - switch ss.streamSource { - case streamSourcePassthrough: - ss.logger.CDebugw(ctx, "continuing using rtp_passthrough", "name", ss.Stream.Name()) - // noop as we are already subscribed - case streamSourceGoStream: - ss.logger.CDebugw(ctx, "currently using gostream, trying upgrade to rtp_passthrough", "name", ss.Stream.Name()) - // attempt to cut over to passthrough - err := ss.streamH264Passthrough(ctx) + err := state.streamH264Passthrough() if err != nil { - ss.logger.CDebugw(ctx, "rtp_passthrough not possible, continuing with gostream", "err", err.Error(), "name", ss.Stream.Name()) - } - case streamSourceUnknown: - fallthrough - default: - err := fmt.Errorf("%s streamSource in unexpected state %s", ss.Stream.Name(), ss.streamSource) - ss.logger.Error(err.Error()) - return err - } - ss.activePeers++ - return nil -} - -func (ss *StreamState) dec(ctx context.Context) error { - ss.logger.Debugf("decrement START %s activePeers: %d", ss.Stream.Name(), ss.activePeers) - defer func() { ss.logger.Debugf("decrement END %s activePeers: %d", ss.Stream.Name(), ss.activePeers) }() - - var err error - defer func() { - if err != nil { - ss.logger.Errorf("decrement %s hit error: %s", ss.Stream.Name(), err.Error()) - return - } - ss.activePeers-- - if ss.activePeers <= 0 { - ss.activePeers = 0 + state.logger.Warn("rtp_passthrough not possible, falling back to GoStream", "err", err) + // if passthrough failed, fall back to gostream based approach + state.Stream.Start() + state.streamSource = streamSourceGoStream } - }() - if ss.activePeers == 1 { - ss.logger.Debugf("decrement %s calling stopBasedOnSub", ss.Stream.Name()) - err = ss.stopBasedOnSub(ctx) + case state.streamSource == streamSourcePassthrough: + // no op if we are using passthrough & are healthy + state.logger.Debug("still healthy and using h264 passthrough") + case state.streamSource == streamSourceGoStream: + // Try to upgrade to passthrough if we are using gostream. We leave logs these as debugs as + // we expect some components to not implement rtp passthrough. + state.logger.Debugw("currently using gostream, trying upgrade to rtp_passthrough") + // attempt to cut over to passthrough + err := state.streamH264Passthrough() if err != nil { - ss.logger.Error(err.Error()) - return err + state.logger.Debugw("rtp_passthrough upgrade failed, continuing with gostream", "err", err) } } - return nil -} - -func (ss *StreamState) restart(ctx context.Context) { - ss.logger.Debugf("restart %s START activePeers: %d", ss.Stream.Name(), ss.activePeers) - defer func() { ss.logger.Debugf("restart %s END activePeers: %d", ss.Stream.Name(), ss.activePeers) }() - - if ss.activePeers == 0 { - // nothing to do if we don't have any active peers - return - } - - if ss.streamSource == streamSourceGoStream { - // nothing to do if stream source is gostream - return - } - - if ss.streamSourceSub != rtppassthrough.NilSubscription && ss.streamSourceSub.Terminated.Err() == nil { - // if the stream is still healthy, do nothing - return - } - - err := ss.streamH264Passthrough(ctx) - if err != nil { - ss.logger.CDebugw(ctx, "rtp_passthrough not possible, falling back to GoStream", "err", err.Error(), "name", ss.Stream.Name()) - // if passthrough failed, fall back to gostream based approach - ss.Stream.Start() - ss.streamSource = streamSourceGoStream - - return - } - // passthrough succeeded, listen for when subscription end and call start again if so } -func (ss *StreamState) streamH264Passthrough(ctx context.Context) error { - cam, err := camera.FromRobot(ss.robot, ss.Stream.Name()) +func (state *StreamState) streamH264Passthrough() error { + cam, err := streamCamera.Camera(state.robot, state.Stream) if err != nil { return err } + // Get the camera and see if it implements the rtp passthrough API of SubscribeRTP + Unsubscribe rtpPassthroughSource, ok := cam.(rtppassthrough.Source) if !ok { - err := fmt.Errorf("expected %s to implement rtppassthrough.Source", ss.Stream.Name()) - return errors.Wrap(ErrRTPPassthroughNotSupported, err.Error()) + return errors.New("stream does not support RTP passthrough") } + var count atomic.Uint64 + + // We might be already sending video via gostream. In this case we: + // - First try and create an RTP passthrough subscription + // - If not successful, continue with gostream. + // - Otherwise if successful, stop gostream. + // - Once we're sure gostream is stopped, we close the `releasePackets` channel + // + // This ensures we only start sending passthrough packets after gostream has stopped sending + // video packets. + releasePackets := make(chan struct{}) + cb := func(pkts []*rtp.Packet) { + <-releasePackets for _, pkt := range pkts { - if err := ss.Stream.WriteRTP(pkt); err != nil { - ss.logger.Debugw("stream.WriteRTP", "name", ss.Stream.Name(), "err", err.Error()) + // Also, look at unsubscribe error logs. Definitely a bug. Probably benign. + if count.Add(1)%10000 == 0 { + state.logger.Debugw("WriteRTP called. Sampling 1/10000", + "count", count.Load(), "seqNumber", pkt.Header.SequenceNumber, "ts", pkt.Header.Timestamp) + } + if err := state.Stream.WriteRTP(pkt); err != nil { + state.logger.Debugw("stream.WriteRTP", "name", state.Stream.Name(), "err", err.Error()) } } } - sub, err := rtpPassthroughSource.SubscribeRTP(ctx, rtpBufferSize, cb) + sub, err := rtpPassthroughSource.SubscribeRTP(state.closedCtx, rtpBufferSize, cb) if err != nil { - return errors.Wrap(ErrRTPPassthroughNotSupported, err.Error()) + return fmt.Errorf("SubscribeRTP failed: %w", err) } - ss.logger.CWarnw(ctx, "Stream using experimental H264 passthrough", "name", ss.Stream.Name()) - ss.monitorSubscription(sub) + state.logger.Warnw("Stream using experimental H264 passthrough", "name", state.Stream.Name()) + + if state.streamSource == streamSourceGoStream { + state.logger.Debugf("monitorSubscription stopping gostream %s", state.Stream.Name()) + // We've succeeded creating a passthrough stream. If we were streaming using gostream, stop it. + state.Stream.Stop() + } + close(releasePackets) + state.streamSourceSub = sub + state.streamSource = streamSourcePassthrough + + state.wg.Add(1) + utils.ManagedGo(func() { + state.monitorSubscription(sub.Terminated) + }, state.wg.Done) return nil } -func (ss *StreamState) unsubscribeH264Passthrough(ctx context.Context, id rtppassthrough.SubscriptionID) error { - cam, err := camera.FromRobot(ss.robot, ss.Stream.Name()) +func (state *StreamState) unsubscribeH264Passthrough(ctx context.Context, id rtppassthrough.SubscriptionID) error { + cam, err := streamCamera.Camera(state.robot, state.Stream) if err != nil { return err } rtpPassthroughSource, ok := cam.(rtppassthrough.Source) if !ok { - err := fmt.Errorf("expected %s to implement rtppassthrough.Source", ss.Stream.Name()) - return errors.Wrap(ErrRTPPassthroughNotSupported, err.Error()) + return fmt.Errorf("subscription resource does not implement rtpPassthroughSource. CamType: %T", rtpPassthroughSource) } if err := rtpPassthroughSource.Unsubscribe(ctx, id); err != nil { diff --git a/robot/web/stream/state/state_test.go b/robot/web/stream/state/state_test.go index ed4c5b337b5..1c482482714 100644 --- a/robot/web/stream/state/state_test.go +++ b/robot/web/stream/state/state_test.go @@ -3,6 +3,7 @@ package state_test import ( "context" "errors" + "fmt" "image" "sync" "sync/atomic" @@ -16,6 +17,7 @@ import ( "github.com/viamrobotics/webrtc/v3" "go.viam.com/test" "go.viam.com/utils" + "go.viam.com/utils/testutils" "go.viam.com/rdk/components/camera" "go.viam.com/rdk/components/camera/rtppassthrough" @@ -39,10 +41,12 @@ func (mS *mockStream) Name() string { return mS.name } +// Start refers to starting gostream. func (mS *mockStream) Start() { mS.startFunc() } +// Stop refers to stopping gostream. func (mS *mockStream) Stop() { mS.stopFunc() } @@ -53,32 +57,27 @@ func (mS *mockStream) WriteRTP(pkt *rtp.Packet) error { // BEGIN Not tested gostream functions. func (mS *mockStream) StreamingReady() (<-chan struct{}, context.Context) { - mS.t.Log("unimplemented") - mS.t.FailNow() + test.That(mS.t, "should not be called", test.ShouldBeFalse) return nil, context.Background() } func (mS *mockStream) InputVideoFrames(props prop.Video) (chan<- gostream.MediaReleasePair[image.Image], error) { - mS.t.Log("unimplemented") - mS.t.FailNow() + test.That(mS.t, "should not be called", test.ShouldBeFalse) return nil, errors.New("unimplemented") } func (mS *mockStream) InputAudioChunks(props prop.Audio) (chan<- gostream.MediaReleasePair[wave.Audio], error) { - mS.t.Log("unimplemented") - mS.t.FailNow() + test.That(mS.t, "should not be called", test.ShouldBeFalse) return make(chan gostream.MediaReleasePair[wave.Audio]), nil } func (mS *mockStream) VideoTrackLocal() (webrtc.TrackLocal, bool) { - mS.t.Log("unimplemented") - mS.t.FailNow() + test.That(mS.t, "should not be called", test.ShouldBeFalse) return nil, false } func (mS *mockStream) AudioTrackLocal() (webrtc.TrackLocal, bool) { - mS.t.Log("unimplemented") - mS.t.FailNow() + test.That(mS.t, "should not be called", test.ShouldBeFalse) return nil, false } @@ -111,7 +110,6 @@ func (s *mockRTPPassthroughSource) Unsubscribe( var camName = "my-cam" -// END Not tested gostream functions. func mockRobot(s rtppassthrough.Source) robot.Robot { robot := &inject.Robot{} robot.MockResourcesFromMap(map[resource.Name]resource.Resource{ @@ -123,62 +121,17 @@ func mockRobot(s rtppassthrough.Source) robot.Robot { func TestStreamState(t *testing.T) { ctx := context.Background() logger := logging.NewTestLogger(t) + // we have to use sleep here as we are asserting the state doesn't change after a given period of time + sleepDuration := time.Millisecond * 200 - t.Run("Stream returns the provided stream", func(t *testing.T) { - mockRTPPassthroughSource := &mockRTPPassthroughSource{} - robot := mockRobot(mockRTPPassthroughSource) - streamMock := &mockStream{name: camName, t: t} - s := state.New(streamMock, robot, logger) - test.That(t, s.Stream, test.ShouldEqual, streamMock) - }) - - t.Run("close succeeds if no methods have been called", func(t *testing.T) { - mockRTPPassthroughSource := &mockRTPPassthroughSource{} - robot := mockRobot(mockRTPPassthroughSource) - streamMock := &mockStream{name: camName, t: t} - s := state.New(streamMock, robot, logger) - test.That(t, s.Close(), test.ShouldBeNil) - }) - - t.Run("Increment() returns an error if Init() is not called first", func(t *testing.T) { - mockRTPPassthroughSource := &mockRTPPassthroughSource{} - robot := mockRobot(mockRTPPassthroughSource) - streamMock := &mockStream{name: camName, t: t} - s := state.New(streamMock, robot, logger) - test.That(t, s.Increment(ctx), test.ShouldBeError, state.ErrUninitialized) - }) - - t.Run("Decrement() returns an error if Init() is not called first", func(t *testing.T) { - mockRTPPassthroughSource := &mockRTPPassthroughSource{} - robot := mockRobot(mockRTPPassthroughSource) - streamMock := &mockStream{name: camName, t: t} - s := state.New(streamMock, robot, logger) - test.That(t, s.Decrement(ctx), test.ShouldBeError, state.ErrUninitialized) - }) - - t.Run("Increment() returns an error if called after Close()", func(t *testing.T) { - mockRTPPassthroughSource := &mockRTPPassthroughSource{} - robot := mockRobot(mockRTPPassthroughSource) - streamMock := &mockStream{name: camName, t: t} - s := state.New(streamMock, robot, logger) - s.Init() - s.Close() - test.That(t, s.Increment(ctx), test.ShouldWrap, state.ErrClosed) - }) - - t.Run("Decrement() returns an error if called after Close()", func(t *testing.T) { - mockRTPPassthroughSource := &mockRTPPassthroughSource{} - robot := mockRobot(mockRTPPassthroughSource) - streamMock := &mockStream{name: camName, t: t} - s := state.New(streamMock, robot, logger) - s.Init() - s.Close() - test.That(t, s.Decrement(ctx), test.ShouldWrap, state.ErrClosed) - }) - - t.Run("when rtppassthrough.Souce is provided but SubscribeRTP always returns an error", func(t *testing.T) { + t.Run("when rtppassthrough.Source is provided but SubscribeRTP always returns an error", func(t *testing.T) { + // Define counters that are bumped every time the video stream is started or stopped in + // "gostream mode". var startCount atomic.Int64 var stopCount atomic.Int64 + + // Because SubscribeRTP will always return an error, it should be a test failure if the mock + // stream tries writing an RTP packet. streamMock := &mockStream{ name: camName, t: t, @@ -189,15 +142,15 @@ func TestStreamState(t *testing.T) { stopCount.Add(1) }, writeRTPFunc: func(pkt *rtp.Packet) error { - t.Log("should not happen") - t.FailNow() + test.That(t, "should not be called", test.ShouldBeFalse) return nil }, } + // Define an a function that matches the SubscribeRTP signature. It will fail on each + // SubscribeRTP call. We define a counter to know how often the function has been called. var subscribeRTPCount atomic.Int64 - - subscribeRTPFunc := func( + failingSubscribeRTPFunc := func( ctx context.Context, bufferSize int, packetsCB rtppassthrough.PacketCallback, @@ -206,118 +159,121 @@ func TestStreamState(t *testing.T) { return rtppassthrough.NilSubscription, errors.New("unimplemented") } + // Because SubscribeRTP will always fail, UnsubscribeRTP must not be called. unsubscribeFunc := func(ctx context.Context, id rtppassthrough.SubscriptionID) error { - t.Log("should not happen") - t.FailNow() + test.That(t, "should not be called", test.ShouldBeFalse) return errors.New("unimplemented") } mockRTPPassthroughSource := &mockRTPPassthroughSource{ - subscribeRTPFunc: subscribeRTPFunc, + subscribeRTPFunc: failingSubscribeRTPFunc, unsubscribeFunc: unsubscribeFunc, } robot := mockRobot(mockRTPPassthroughSource) s := state.New(streamMock, robot, logger) - defer func() { utils.UncheckedError(s.Close()) }() - s.Init() + defer func() { + utils.UncheckedError(s.Close()) + }() test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 0) test.That(t, startCount.Load(), test.ShouldEqual, 0) test.That(t, stopCount.Load(), test.ShouldEqual, 0) - t.Log("the first Increment() calls SubscribeRTP and then calls Start() when an error is reurned") - test.That(t, s.Increment(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 1) - test.That(t, startCount.Load(), test.ShouldEqual, 1) - test.That(t, stopCount.Load(), test.ShouldEqual, 0) - - t.Log("subsequent Increment() all calls call SubscribeRTP trying to determine " + - "if they can upgrade but don't call any other gostream methods as SubscribeRTP returns an error") - test.That(t, s.Increment(ctx), test.ShouldBeNil) - test.That(t, s.Increment(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 3) - test.That(t, startCount.Load(), test.ShouldEqual, 1) - test.That(t, stopCount.Load(), test.ShouldEqual, 0) - - t.Log("as long as the number of Decrement() calls is less than the number of Increment() calls, no gostream methods are called") - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 3) + // Now we "increment" the number of `AddStream` calls. This means there's a (single) + // consumer for this video stream. + logger.Info("the first Increment() eventually calls SubscribeRTP and then calls Start() when an error is returned") + test.That(t, s.Increment(), test.ShouldBeNil) + + // The post-loop invariant is that we've called SubscribeRTP (for passthrough) at least + // once. And gostream has been started exactly once and never stopped. In this state, the + // stream server will continue trying to upgrade to RTP passthrough. `Stop` will be only + // called on the state object if the SubscribeRTP method was a success. + var prevSubscribeRTPCount int64 + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, subscribeRTPCount.Load(), test.ShouldBeGreaterThan, 0) + prevSubscribeRTPCount = subscribeRTPCount.Load() + test.That(tb, startCount.Load(), test.ShouldEqual, 1) + test.That(tb, stopCount.Load(), test.ShouldEqual, 0) + }) + + logger.Info("as long as the number of Decrement() calls is less than the number of Increment() calls, no gostream methods are called") + test.That(t, s.Increment(), test.ShouldBeNil) + test.That(t, s.Increment(), test.ShouldBeNil) + test.That(t, s.Decrement(), test.ShouldBeNil) + test.That(t, s.Decrement(), test.ShouldBeNil) + + // Wait a bit and assert we've continue to try and upgrade to RTP passthrough. As measured + // by SubscribeRTP calls. + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, subscribeRTPCount.Load(), test.ShouldBeGreaterThan, prevSubscribeRTPCount) + }) + prevSubscribeRTPCount = subscribeRTPCount.Load() + // Double check we did not stop/restart gostream. test.That(t, startCount.Load(), test.ShouldEqual, 1) test.That(t, stopCount.Load(), test.ShouldEqual, 0) - t.Log("when the number of Decrement() calls is equal to the number of Increment() calls stop is called") - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 3) + // Decrement the stream state which should eventually stop gostream. + logger.Info("when the number of Decrement() calls is equal to the number of Increment() calls stop is called") + test.That(t, s.Decrement(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, stopCount.Load(), test.ShouldEqual, 1) + }) + + // An upgrade attempt may or may not have been made concurrently with decrementing. Update + // the running SubscribeRTP call counter. + prevSubscribeRTPCount = subscribeRTPCount.Load() + // We must not have tried restarting gostream. test.That(t, startCount.Load(), test.ShouldEqual, 1) - test.That(t, stopCount.Load(), test.ShouldEqual, 1) - t.Log("then when the number of Increment() calls exceeds Decrement(), both SubscribeRTP & Start are called again") - test.That(t, s.Increment(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 4) - test.That(t, startCount.Load(), test.ShouldEqual, 2) - test.That(t, stopCount.Load(), test.ShouldEqual, 1) - - t.Log("calling Decrement() more times than Increment() has a floor of zero") - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 4) - test.That(t, startCount.Load(), test.ShouldEqual, 2) - test.That(t, stopCount.Load(), test.ShouldEqual, 2) - - // multiple Decrement() calls when the count is already at zero doesn't call any methods - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 4) - test.That(t, startCount.Load(), test.ShouldEqual, 2) - test.That(t, stopCount.Load(), test.ShouldEqual, 2) - - // once the count is at zero , calling Increment() again calls SubscribeRTP and when it returns an error Start - test.That(t, s.Increment(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 5) - test.That(t, startCount.Load(), test.ShouldEqual, 3) - test.That(t, stopCount.Load(), test.ShouldEqual, 2) + logger.Info("then when the number of Increment() calls exceeds Decrement(), both SubscribeRTP & Start are eventually called again") + test.That(t, s.Increment(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + // An increment always tries SubscribeRTP before gostream. + test.That(tb, subscribeRTPCount.Load(), test.ShouldBeGreaterThan, prevSubscribeRTPCount) + test.That(tb, startCount.Load(), test.ShouldEqual, 2) + test.That(tb, stopCount.Load(), test.ShouldEqual, 1) + }) + prevSubscribeRTPCount = subscribeRTPCount.Load() + + // Wait some more to observe SubscribeRTP continuing to be called. Then decrement the stream + // state to observe gostream being `Stop`ed. + time.Sleep(time.Second) + test.That(t, s.Decrement(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, subscribeRTPCount.Load(), test.ShouldBeGreaterThan, prevSubscribeRTPCount) + test.That(tb, startCount.Load(), test.ShouldEqual, 2) + test.That(tb, stopCount.Load(), test.ShouldEqual, 2) + }) + prevSubscribeRTPCount = subscribeRTPCount.Load() + + // Once the count is at zero , calling Increment() again calls SubscribeRTP. SubscribeRTP + // will fail and `Start` gostream. + test.That(t, s.Increment(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, subscribeRTPCount.Load(), test.ShouldBeGreaterThan, prevSubscribeRTPCount) + test.That(tb, startCount.Load(), test.ShouldEqual, 3) + test.That(tb, stopCount.Load(), test.ShouldEqual, 2) + }) // set count back to zero - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 5) - test.That(t, startCount.Load(), test.ShouldEqual, 3) - test.That(t, stopCount.Load(), test.ShouldEqual, 3) - - t.Log("calling Increment() with a cancelled context returns an error & does not call any gostream or rtppassthrough.Source methods") - canceledCtx, cancelFn := context.WithCancel(context.Background()) - cancelFn() - test.That(t, s.Increment(canceledCtx), test.ShouldBeError, context.Canceled) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 5) - test.That(t, startCount.Load(), test.ShouldEqual, 3) - test.That(t, stopCount.Load(), test.ShouldEqual, 3) - - // make it so that non cancelled Decrement() would call stop to confirm that does not happen when context is cancelled - test.That(t, s.Increment(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 6) - test.That(t, startCount.Load(), test.ShouldEqual, 4) - test.That(t, stopCount.Load(), test.ShouldEqual, 3) - - t.Log("calling Decrement() with a cancelled context returns an error & does not call any gostream methods") - test.That(t, s.Decrement(canceledCtx), test.ShouldBeError, context.Canceled) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 6) - test.That(t, startCount.Load(), test.ShouldEqual, 4) - test.That(t, stopCount.Load(), test.ShouldEqual, 3) + test.That(t, s.Decrement(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, subscribeRTPCount.Load(), test.ShouldBeGreaterThanOrEqualTo, prevSubscribeRTPCount) + test.That(tb, startCount.Load(), test.ShouldEqual, 3) + test.That(tb, stopCount.Load(), test.ShouldEqual, 3) + }) }) - t.Run("when rtppassthrough.Souce is provided and SubscribeRTP doesn't return an error", func(t *testing.T) { + t.Run("when rtppassthrough.Source is provided and SubscribeRTP doesn't return an error", func(t *testing.T) { writeRTPCalledCtx, writeRTPCalledFunc := context.WithCancel(ctx) streamMock := &mockStream{ name: camName, t: t, startFunc: func() { - t.Logf("should not be called") - t.FailNow() + test.That(t, "should not be called", test.ShouldBeFalse) }, stopFunc: func() { - t.Logf("should not be called") - t.FailNow() + test.That(t, "should not be called", test.ShouldBeFalse) }, writeRTPFunc: func(pkt *rtp.Packet) error { // Test that WriteRTP is eventually called when SubscribeRTP is called @@ -365,8 +321,7 @@ func TestStreamState(t *testing.T) { defer unsubscribeCount.Add(1) subAndCancel, ok := subsAndCancelByID[id] if !ok { - t.Logf("Unsubscribe called with unknown id: %s", id.String()) - t.FailNow() + test.That(t, fmt.Sprintf("Unsubscribe called with unknown id: %s", id.String()), test.ShouldBeFalse) } subAndCancel.cancelFn() return nil @@ -378,85 +333,76 @@ func TestStreamState(t *testing.T) { } robot := mockRobot(mockRTPPassthroughSource) s := state.New(streamMock, robot, logger) - defer func() { utils.UncheckedError(s.Close()) }() - - s.Init() + defer func() { + utils.UncheckedError(s.Close()) + }() test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 0) test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 0) test.That(t, writeRTPCalledCtx.Err(), test.ShouldBeNil) - t.Log("the first Increment() call calls SubscribeRTP()") - test.That(t, s.Increment(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 1) - test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 0) + logger.Info("the first Increment() eventually call calls SubscribeRTP()") + test.That(t, s.Increment(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, subscribeRTPCount.Load(), test.ShouldEqual, 1) + test.That(tb, unsubscribeCount.Load(), test.ShouldEqual, 0) + }) // WriteRTP is called <-writeRTPCalledCtx.Done() - t.Log("subsequent Increment() calls don't call any other rtppassthrough.Source methods") - test.That(t, s.Increment(ctx), test.ShouldBeNil) - test.That(t, s.Increment(ctx), test.ShouldBeNil) + logger.Info("subsequent Increment() calls don't call any other rtppassthrough.Source methods") + test.That(t, s.Increment(), test.ShouldBeNil) + test.That(t, s.Increment(), test.ShouldBeNil) + time.Sleep(sleepDuration) test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 1) test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 0) - t.Log("as long as the number of Decrement() calls is less than the number " + + logger.Info("as long as the number of Decrement() calls is less than the number " + "of Increment() calls, no rtppassthrough.Source methods are called") - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, s.Decrement(ctx), test.ShouldBeNil) + test.That(t, s.Decrement(), test.ShouldBeNil) + test.That(t, s.Decrement(), test.ShouldBeNil) + time.Sleep(sleepDuration) test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 1) test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 0) - t.Log("when the number of Decrement() calls is equal to the number of Increment() calls stop is called") - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 1) - test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 1) - - t.Log("then when the number of Increment() calls exceeds Decrement(), SubscribeRTP is called again") - test.That(t, s.Increment(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 2) - test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 1) + logger.Info("when the number of Decrement() calls is equal to the number of Increment() calls stop is called") + test.That(t, s.Decrement(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, subscribeRTPCount.Load(), test.ShouldEqual, 1) + test.That(tb, unsubscribeCount.Load(), test.ShouldEqual, 1) + }) - t.Log("calling Decrement() more times than Increment() has a floor of zero") - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 2) - test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 2) + logger.Info("then when the number of Increment() calls exceeds Decrement(), SubscribeRTP is called again") + test.That(t, s.Increment(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, subscribeRTPCount.Load(), test.ShouldEqual, 2) + test.That(tb, unsubscribeCount.Load(), test.ShouldEqual, 1) + }) - // multiple Decrement() calls when the count is already at zero doesn't call any methods - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 2) - test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 2) + test.That(t, s.Decrement(), test.ShouldBeNil) // once the count is at zero , calling Increment() again calls start - test.That(t, s.Increment(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 3) - test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 2) + test.That(t, s.Increment(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, subscribeRTPCount.Load(), test.ShouldEqual, 3) + test.That(tb, unsubscribeCount.Load(), test.ShouldEqual, 2) + }) // set count back to zero - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 3) - test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 3) - - t.Log("calling Increment() with a cancelled context returns an error & does not call any rtppassthrough.Source methods") - canceledCtx, cancelFn := context.WithCancel(context.Background()) - cancelFn() - test.That(t, s.Increment(canceledCtx), test.ShouldBeError, context.Canceled) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 3) - test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 3) + test.That(t, s.Decrement(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, subscribeRTPCount.Load(), test.ShouldEqual, 3) + test.That(tb, unsubscribeCount.Load(), test.ShouldEqual, 3) + }) // make it so that non cancelled Decrement() would call stop to confirm that does not happen when context is cancelled - test.That(t, s.Increment(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 4) - test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 3) - - t.Log("calling Decrement() with a cancelled context returns an error & does not call any rtppassthrough.Source methods") - test.That(t, s.Decrement(canceledCtx), test.ShouldBeError, context.Canceled) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 4) - test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 3) + test.That(t, s.Increment(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, subscribeRTPCount.Load(), test.ShouldEqual, 4) + test.That(tb, unsubscribeCount.Load(), test.ShouldEqual, 3) + }) - t.Log("when the subscription terminates while there are still subscribers, SubscribeRTP is called again") + logger.Info("when the subscription terminates while there are still subscribers, SubscribeRTP is called again") var cancelledSubs int subsAndCancelByIDMu.Lock() for _, subAndCancel := range subsAndCancelByID { @@ -473,8 +419,9 @@ func TestStreamState(t *testing.T) { defer timeoutFn() for { if timeoutCtx.Err() != nil { - t.Log("timed out waiting for a new sub to be created after an in progress one terminated unexpectedly") - t.FailNow() + test.That(t, + "timed out waiting for a new sub to be created after an in progress one terminated unexpectedly", + test.ShouldBeFalse) } if subscribeRTPCount.Load() == 5 { break @@ -491,7 +438,7 @@ func TestStreamState(t *testing.T) { subsAndCancelByIDMu.Unlock() }) - t.Run("when rtppassthrough.Souce is provided and sometimes returns an errors "+ + t.Run("when rtppassthrough.Source is provided and sometimes returns an errors "+ "(test rtp_passthrough/gostream upgrade/downgrade path)", func(t *testing.T) { var startCount atomic.Int64 var stopCount atomic.Int64 @@ -543,8 +490,9 @@ func TestStreamState(t *testing.T) { defer unsubscribeCount.Add(1) subAndCancel, ok := subsAndCancelByID[id] if !ok { - t.Logf("Unsubscribe called with unknown id: %s", id.String()) - t.FailNow() + test.That(t, + fmt.Sprintf("Unsubscribe called with unknown id: %s", id.String()), + test.ShouldBeFalse) } subAndCancel.cancelFn() return nil @@ -556,32 +504,34 @@ func TestStreamState(t *testing.T) { } robot := mockRobot(mockRTPPassthroughSource) s := state.New(streamMock, robot, logger) - defer func() { utils.UncheckedError(s.Close()) }() - - // start with RTPPassthrough being supported - s.Init() + defer func() { + utils.UncheckedError(s.Close()) + }() test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 0) test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 0) test.That(t, startCount.Load(), test.ShouldEqual, 0) test.That(t, stopCount.Load(), test.ShouldEqual, 0) - t.Log("the first Increment() call calls SubscribeRTP() which returns a success") - test.That(t, s.Increment(ctx), test.ShouldBeNil) + logger.Info("the first Increment() call calls SubscribeRTP() which returns a success") + test.That(t, s.Increment(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, subscribeRTPCount.Load(), test.ShouldEqual, 1) + test.That(tb, unsubscribeCount.Load(), test.ShouldEqual, 0) + test.That(tb, startCount.Load(), test.ShouldEqual, 0) + test.That(tb, stopCount.Load(), test.ShouldEqual, 0) + }) + + logger.Info("subsequent Increment() calls don't call any other rtppassthrough.Source or gostream methods") + test.That(t, s.Increment(), test.ShouldBeNil) + test.That(t, s.Increment(), test.ShouldBeNil) + time.Sleep(sleepDuration) test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 1) test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 0) test.That(t, startCount.Load(), test.ShouldEqual, 0) test.That(t, stopCount.Load(), test.ShouldEqual, 0) - t.Log("subsequent Increment() calls don't call any other rtppassthrough.Source or gostream methods") - test.That(t, s.Increment(ctx), test.ShouldBeNil) - test.That(t, s.Increment(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 1) - test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 0) - test.That(t, startCount.Load(), test.ShouldEqual, 0) - test.That(t, stopCount.Load(), test.ShouldEqual, 0) - - t.Log("if the subscription terminates and SubscribeRTP() returns an error, starts gostream") + logger.Info("if the subscription terminates and SubscribeRTP() returns an error, starts gostream") subscribeRTPReturnError.Store(true) subsAndCancelByIDMu.Lock() test.That(t, len(subsAndCancelByID), test.ShouldEqual, 1) @@ -594,8 +544,9 @@ func TestStreamState(t *testing.T) { defer timeoutFn() for { if timeoutCtx.Err() != nil { - t.Log("timed out waiting for gostream start to be called on stream which terminated unexpectedly") - t.FailNow() + test.That(t, + "timed out waiting for gostream start to be called on stream which terminated unexpectedly", + test.ShouldBeFalse) } if subscribeRTPCount.Load() == 2 { break @@ -608,95 +559,87 @@ func TestStreamState(t *testing.T) { test.That(t, startCount.Load(), test.ShouldEqual, 1) test.That(t, stopCount.Load(), test.ShouldEqual, 0) - t.Log("when the number of Decrement() calls is less than the number of " + + logger.Info("when the number of Decrement() calls is less than the number of " + "Increment() calls no rtppassthrough.Source or gostream methods are called") - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, s.Decrement(ctx), test.ShouldBeNil) + test.That(t, s.Decrement(), test.ShouldBeNil) + test.That(t, s.Decrement(), test.ShouldBeNil) + time.Sleep(sleepDuration) test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 2) test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 0) test.That(t, startCount.Load(), test.ShouldEqual, 1) test.That(t, stopCount.Load(), test.ShouldEqual, 0) - t.Log("when the number of Decrement() calls is equal to the number of " + + logger.Info("when the number of Decrement() calls is equal to the number of " + "Increment() calls stop is called (as gostream is the data source)") - test.That(t, s.Decrement(ctx), test.ShouldBeNil) + test.That(t, s.Decrement(), test.ShouldBeNil) + time.Sleep(sleepDuration) test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 2) test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 0) test.That(t, startCount.Load(), test.ShouldEqual, 1) test.That(t, stopCount.Load(), test.ShouldEqual, 1) - t.Log("then when the number of Increment() calls exceeds Decrement(), " + + logger.Info("then when the number of Increment() calls exceeds Decrement(), " + "SubscribeRTP is called again followed by Start if it returns an error") - test.That(t, s.Increment(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 3) - test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 0) - test.That(t, startCount.Load(), test.ShouldEqual, 2) - test.That(t, stopCount.Load(), test.ShouldEqual, 1) - - t.Log("calling Decrement() more times than Increment() has a floor of zero and calls Stop()") - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 3) - test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 0) - test.That(t, startCount.Load(), test.ShouldEqual, 2) - test.That(t, stopCount.Load(), test.ShouldEqual, 2) - - // multiple Decrement() calls when the count is already at zero doesn't call any methods - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 3) - test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 0) - test.That(t, startCount.Load(), test.ShouldEqual, 2) - test.That(t, stopCount.Load(), test.ShouldEqual, 2) + test.That(t, s.Increment(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, subscribeRTPCount.Load(), test.ShouldEqual, 3) + test.That(tb, unsubscribeCount.Load(), test.ShouldEqual, 0) + test.That(tb, startCount.Load(), test.ShouldEqual, 2) + test.That(tb, stopCount.Load(), test.ShouldEqual, 1) + }) + + test.That(t, s.Decrement(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, subscribeRTPCount.Load(), test.ShouldEqual, 3) + test.That(tb, unsubscribeCount.Load(), test.ShouldEqual, 0) + test.That(tb, startCount.Load(), test.ShouldEqual, 2) + test.That(tb, stopCount.Load(), test.ShouldEqual, 2) + }) // once the count is at zero , calling Increment() again calls SubscribeRTP followed by Start - test.That(t, s.Increment(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 4) - test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 0) - test.That(t, startCount.Load(), test.ShouldEqual, 3) - test.That(t, stopCount.Load(), test.ShouldEqual, 2) - - t.Log("if while gostream is being used Increment is called and SubscribeRTP succeeds, Stop is called") + test.That(t, s.Increment(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, subscribeRTPCount.Load(), test.ShouldEqual, 4) + test.That(tb, unsubscribeCount.Load(), test.ShouldEqual, 0) + test.That(tb, startCount.Load(), test.ShouldEqual, 3) + test.That(tb, stopCount.Load(), test.ShouldEqual, 2) + }) + + logger.Info("if while gostream is being used Increment is called and SubscribeRTP succeeds, Stop is called") subscribeRTPReturnError.Store(false) - test.That(t, s.Increment(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 5) - test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 0) - test.That(t, startCount.Load(), test.ShouldEqual, 3) - test.That(t, stopCount.Load(), test.ShouldEqual, 3) + test.That(t, s.Increment(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, subscribeRTPCount.Load(), test.ShouldEqual, 5) + test.That(tb, unsubscribeCount.Load(), test.ShouldEqual, 0) + test.That(tb, startCount.Load(), test.ShouldEqual, 3) + test.That(tb, stopCount.Load(), test.ShouldEqual, 3) + }) // calling Decrement() fewer times than Increment() doesn't call any methods - test.That(t, s.Decrement(ctx), test.ShouldBeNil) + test.That(t, s.Decrement(), test.ShouldBeNil) + time.Sleep(sleepDuration) test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 5) test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 0) test.That(t, startCount.Load(), test.ShouldEqual, 3) test.That(t, stopCount.Load(), test.ShouldEqual, 3) - t.Log("calling Decrement() more times than Increment() has a floor of zero and calls Unsubscribe()") - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 5) - test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 1) - test.That(t, startCount.Load(), test.ShouldEqual, 3) - test.That(t, stopCount.Load(), test.ShouldEqual, 3) - - // multiple Decrement() calls when the count is already at zero doesn't call any methods - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 5) - test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 1) - test.That(t, startCount.Load(), test.ShouldEqual, 3) - test.That(t, stopCount.Load(), test.ShouldEqual, 3) + test.That(t, s.Decrement(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, subscribeRTPCount.Load(), test.ShouldEqual, 5) + test.That(tb, unsubscribeCount.Load(), test.ShouldEqual, 1) + test.That(tb, startCount.Load(), test.ShouldEqual, 3) + test.That(tb, stopCount.Load(), test.ShouldEqual, 3) + }) - t.Log("if while rtp_passthrough is being used the the subscription " + + logger.Info("if while rtp_passthrough is being used the the subscription " + "terminates & afterwards rtp_passthrough is no longer supported, Start is called") - test.That(t, s.Increment(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 6) - test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 1) - test.That(t, startCount.Load(), test.ShouldEqual, 3) - test.That(t, stopCount.Load(), test.ShouldEqual, 3) + test.That(t, s.Increment(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, subscribeRTPCount.Load(), test.ShouldEqual, 6) + test.That(tb, unsubscribeCount.Load(), test.ShouldEqual, 1) + test.That(tb, startCount.Load(), test.ShouldEqual, 3) + test.That(tb, stopCount.Load(), test.ShouldEqual, 3) + }) subscribeRTPReturnError.Store(true) @@ -709,8 +652,9 @@ func TestStreamState(t *testing.T) { defer timeoutFn() for { if timeoutCtx.Err() != nil { - t.Log("timed out waiting for Start() to be called after an in progress sub terminated unexpectedly") - t.FailNow() + test.That(t, + "timed out waiting for Start() to be called after an in progress sub terminated unexpectedly", + test.ShouldBeFalse) } if startCount.Load() == 4 { break @@ -723,14 +667,16 @@ func TestStreamState(t *testing.T) { test.That(t, stopCount.Load(), test.ShouldEqual, 3) // Decrement() calls Stop() as gostream is the data source - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, subscribeRTPCount.Load(), test.ShouldEqual, 7) - test.That(t, unsubscribeCount.Load(), test.ShouldEqual, 1) - test.That(t, startCount.Load(), test.ShouldEqual, 4) - test.That(t, stopCount.Load(), test.ShouldEqual, 4) + test.That(t, s.Decrement(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, subscribeRTPCount.Load(), test.ShouldEqual, 7) + test.That(tb, unsubscribeCount.Load(), test.ShouldEqual, 1) + test.That(tb, startCount.Load(), test.ShouldEqual, 4) + test.That(tb, stopCount.Load(), test.ShouldEqual, 4) + }) }) - t.Run("when the camera does not implement rtppassthrough.Souce", func(t *testing.T) { + t.Run("when the camera does not implement rtppassthrough.Source", func(t *testing.T) { var startCount atomic.Int64 var stopCount atomic.Int64 streamMock := &mockStream{ @@ -743,81 +689,77 @@ func TestStreamState(t *testing.T) { stopCount.Add(1) }, writeRTPFunc: func(pkt *rtp.Packet) error { - t.Log("should not happen") - t.FailNow() + test.That(t, + "should not happen", + test.ShouldBeFalse) return nil }, } robot := mockRobot(nil) s := state.New(streamMock, robot, logger) - defer func() { utils.UncheckedError(s.Close()) }() - s.Init() - - t.Log("the first Increment() -> Start()") - test.That(t, s.Increment(ctx), test.ShouldBeNil) + defer func() { + utils.UncheckedError(s.Close()) + }() + logger.Info("the first Increment() -> Start()") + test.That(t, s.Increment(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, startCount.Load(), test.ShouldEqual, 1) + test.That(tb, stopCount.Load(), test.ShouldEqual, 0) + }) + + logger.Info("subsequent Increment() all calls don't call any other gostream methods") + test.That(t, s.Increment(), test.ShouldBeNil) + test.That(t, s.Increment(), test.ShouldBeNil) + time.Sleep(sleepDuration) test.That(t, startCount.Load(), test.ShouldEqual, 1) test.That(t, stopCount.Load(), test.ShouldEqual, 0) - t.Log("subsequent Increment() all calls don't call any other gostream methods") - test.That(t, s.Increment(ctx), test.ShouldBeNil) - test.That(t, s.Increment(ctx), test.ShouldBeNil) + logger.Info("as long as the number of Decrement() calls is less than the number of Increment() calls, no gostream methods are called") + test.That(t, s.Decrement(), test.ShouldBeNil) + test.That(t, s.Decrement(), test.ShouldBeNil) + time.Sleep(sleepDuration) test.That(t, startCount.Load(), test.ShouldEqual, 1) test.That(t, stopCount.Load(), test.ShouldEqual, 0) - t.Log("as long as the number of Decrement() calls is less than the number of Increment() calls, no gostream methods are called") - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, startCount.Load(), test.ShouldEqual, 1) - test.That(t, stopCount.Load(), test.ShouldEqual, 0) - - t.Log("when the number of Decrement() calls is equal to the number of Increment() calls stop is called") - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, startCount.Load(), test.ShouldEqual, 1) - test.That(t, stopCount.Load(), test.ShouldEqual, 1) - test.That(t, s.Increment(ctx), test.ShouldBeNil) - - t.Log("then when the number of Increment() calls exceeds Decrement(), Start is called again") - test.That(t, startCount.Load(), test.ShouldEqual, 2) - test.That(t, stopCount.Load(), test.ShouldEqual, 1) - - t.Log("calling Decrement() more times than Increment() has a floor of zero") - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, startCount.Load(), test.ShouldEqual, 2) - test.That(t, stopCount.Load(), test.ShouldEqual, 2) - - // multiple Decrement() calls when the count is already at zero doesn't call any methods - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, startCount.Load(), test.ShouldEqual, 2) - test.That(t, stopCount.Load(), test.ShouldEqual, 2) + logger.Info("when the number of Decrement() calls is equal to the number of Increment() calls stop is called") + test.That(t, s.Decrement(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, startCount.Load(), test.ShouldEqual, 1) + test.That(tb, stopCount.Load(), test.ShouldEqual, 1) + }) + + logger.Info("then when the number of Increment() calls exceeds Decrement(), Start is called again") + test.That(t, s.Increment(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, startCount.Load(), test.ShouldEqual, 2) + test.That(tb, stopCount.Load(), test.ShouldEqual, 1) + }) + + test.That(t, s.Decrement(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, startCount.Load(), test.ShouldEqual, 2) + test.That(tb, stopCount.Load(), test.ShouldEqual, 2) + }) // once the count is at zero , calling Increment() again calls start - test.That(t, s.Increment(ctx), test.ShouldBeNil) - test.That(t, startCount.Load(), test.ShouldEqual, 3) - test.That(t, stopCount.Load(), test.ShouldEqual, 2) + test.That(t, s.Increment(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, startCount.Load(), test.ShouldEqual, 3) + test.That(tb, stopCount.Load(), test.ShouldEqual, 2) + }) // set count back to zero - test.That(t, s.Decrement(ctx), test.ShouldBeNil) - test.That(t, startCount.Load(), test.ShouldEqual, 3) - test.That(t, stopCount.Load(), test.ShouldEqual, 3) - - t.Log("calling Increment() with a cancelled context returns an error & does not call any gostream methods") - canceledCtx, cancelFn := context.WithCancel(context.Background()) - cancelFn() - test.That(t, s.Increment(canceledCtx), test.ShouldBeError, context.Canceled) - test.That(t, startCount.Load(), test.ShouldEqual, 3) - test.That(t, stopCount.Load(), test.ShouldEqual, 3) + test.That(t, s.Decrement(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, startCount.Load(), test.ShouldEqual, 3) + test.That(tb, stopCount.Load(), test.ShouldEqual, 3) + }) // make it so that non cancelled Decrement() would call stop to confirm that does not happen when context is cancelled - test.That(t, s.Increment(ctx), test.ShouldBeNil) - test.That(t, startCount.Load(), test.ShouldEqual, 4) - test.That(t, stopCount.Load(), test.ShouldEqual, 3) - - t.Log("calling Decrement() with a cancelled context returns an error & does not call any gostream methods") - test.That(t, s.Decrement(canceledCtx), test.ShouldBeError, context.Canceled) - test.That(t, startCount.Load(), test.ShouldEqual, 4) - test.That(t, stopCount.Load(), test.ShouldEqual, 3) + test.That(t, s.Increment(), test.ShouldBeNil) + testutils.WaitForAssertion(t, func(tb testing.TB) { + test.That(tb, startCount.Load(), test.ShouldEqual, 4) + test.That(tb, stopCount.Load(), test.ShouldEqual, 3) + }) }) } diff --git a/robot/web/stream/stream.go b/robot/web/stream/stream.go index da0f08ac469..7966dc78b57 100644 --- a/robot/web/stream/stream.go +++ b/robot/web/stream/stream.go @@ -6,7 +6,6 @@ package webstream import ( "context" "errors" - "math" "time" "go.viam.com/utils" @@ -23,7 +22,7 @@ func StreamVideoSource( backoffOpts *BackoffTuningOptions, logger logging.Logger, ) error { - return gostream.StreamVideoSourceWithErrorHandler(ctx, source, stream, backoffOpts.getErrorThrottledHandler(logger), logger) + return gostream.StreamVideoSourceWithErrorHandler(ctx, source, stream, backoffOpts.getErrorThrottledHandler(logger, stream.Name()), logger) } // StreamAudioSource starts a stream from an audio source with a throttled error handler. @@ -34,7 +33,7 @@ func StreamAudioSource( backoffOpts *BackoffTuningOptions, logger logging.Logger, ) error { - return gostream.StreamAudioSourceWithErrorHandler(ctx, source, stream, backoffOpts.getErrorThrottledHandler(logger), logger) + return gostream.StreamAudioSourceWithErrorHandler(ctx, source, stream, backoffOpts.getErrorThrottledHandler(logger, "audio"), logger) } // BackoffTuningOptions represents a set of parameters for determining exponential @@ -55,18 +54,25 @@ type BackoffTuningOptions struct { Cooldown time.Duration } +// Dan: This fixes the backoff bugs where we'd overflow the sleep to a negative value. But this no +// longer obeys the input tuning options. I'm considering that deprecated and to be removed. +var backoffSleeps = []time.Duration{500 * time.Millisecond, time.Second, 2 * time.Second, 5 * time.Second} + // GetSleepTimeFromErrorCount returns a sleep time from an error count. func (opts *BackoffTuningOptions) GetSleepTimeFromErrorCount(errorCount int) time.Duration { if errorCount < 1 || opts == nil { return 0 } - multiplier := math.Pow(2, float64(errorCount-1)) - uncappedSleep := opts.BaseSleep * time.Duration(multiplier) - sleep := math.Min(float64(uncappedSleep), float64(opts.MaxSleep)) - return time.Duration(sleep) + + errorCount-- + if errorCount >= len(backoffSleeps) { + return backoffSleeps[len(backoffSleeps)-1] + } + + return backoffSleeps[errorCount] } -func (opts *BackoffTuningOptions) getErrorThrottledHandler(logger logging.Logger) func(context.Context, error) { +func (opts *BackoffTuningOptions) getErrorThrottledHandler(logger logging.Logger, streamName string) func(context.Context, error) { var prevErr error var errorCount int lastErrTime := time.Now() @@ -78,15 +84,19 @@ func (opts *BackoffTuningOptions) getErrorThrottledHandler(logger logging.Logger } lastErrTime = now - if errors.Is(prevErr, err) { + switch { + case errors.Is(prevErr, err): + errorCount++ + case prevErr != nil && prevErr.Error() == err.Error(): errorCount++ - } else { + default: prevErr = err errorCount = 1 } sleep := opts.GetSleepTimeFromErrorCount(errorCount) - logger.Errorw("error getting media", "error", err, "count", errorCount, "sleep", sleep) + logger.Errorw("error getting media", "streamName", streamName, "error", err, "count", errorCount, "sleep", sleep) + utils.SelectContextOrWait(ctx, sleep) } } diff --git a/robot/web/stream/stream_test.go b/robot/web/stream/stream_test.go index d09eea7eb38..073e955c39e 100644 --- a/robot/web/stream/stream_test.go +++ b/robot/web/stream/stream_test.go @@ -4,17 +4,9 @@ package webstream_test import ( "context" - "errors" - "image" "strings" - "sync" "testing" - "time" - "github.com/pion/mediadevices/pkg/prop" - "github.com/pion/mediadevices/pkg/wave" - "github.com/pion/rtp" - "github.com/viamrobotics/webrtc/v3" streampb "go.viam.com/api/stream/v1" "go.viam.com/test" "go.viam.com/utils/rpc" @@ -32,117 +24,9 @@ import ( "go.viam.com/rdk/robot" robotimpl "go.viam.com/rdk/robot/impl" "go.viam.com/rdk/robot/web" - webstream "go.viam.com/rdk/robot/web/stream" "go.viam.com/rdk/testutils/robottestutils" ) -var errImageRetrieval = errors.New("image retrieval failed") - -type mockErrorVideoSource struct { - callsLeft int - wg sync.WaitGroup -} - -func newMockErrorVideoReader(expectedCalls int) *mockErrorVideoSource { - mock := &mockErrorVideoSource{callsLeft: expectedCalls} - mock.wg.Add(expectedCalls) - return mock -} - -func (videoSource *mockErrorVideoSource) Read(ctx context.Context) (image.Image, func(), error) { - if videoSource.callsLeft > 0 { - videoSource.wg.Done() - videoSource.callsLeft-- - } - return nil, nil, errImageRetrieval -} - -func (videoSource *mockErrorVideoSource) Close(ctx context.Context) error { - return nil -} - -type mockStream struct { - name string - streamingReadyFunc func() <-chan struct{} - inputFramesFunc func() (chan<- gostream.MediaReleasePair[image.Image], error) -} - -func (mS *mockStream) StreamingReady() (<-chan struct{}, context.Context) { - return mS.streamingReadyFunc(), context.Background() -} - -func (mS *mockStream) InputVideoFrames(props prop.Video) (chan<- gostream.MediaReleasePair[image.Image], error) { - return mS.inputFramesFunc() -} - -func (mS *mockStream) InputAudioChunks(props prop.Audio) (chan<- gostream.MediaReleasePair[wave.Audio], error) { - return make(chan gostream.MediaReleasePair[wave.Audio]), nil -} - -func (mS *mockStream) Name() string { - return mS.name -} - -func (mS *mockStream) Start() { -} - -func (mS *mockStream) Stop() { -} - -func (mS *mockStream) WriteRTP(*rtp.Packet) error { - return nil -} - -func (mS *mockStream) VideoTrackLocal() (webrtc.TrackLocal, bool) { - return nil, false -} - -func (mS *mockStream) AudioTrackLocal() (webrtc.TrackLocal, bool) { - return nil, false -} - -func TestStreamSourceErrorBackoff(t *testing.T) { - logger := logging.NewTestLogger(t) - ctx, cancel := context.WithCancel(context.Background()) - - backoffOpts := &webstream.BackoffTuningOptions{ - BaseSleep: 50 * time.Microsecond, - MaxSleep: 250 * time.Millisecond, - Cooldown: time.Second, - } - calls := 25 - videoReader := newMockErrorVideoReader(calls) - videoSrc := gostream.NewVideoSource(videoReader, prop.Video{}) - defer func() { - test.That(t, videoSrc.Close(context.Background()), test.ShouldBeNil) - }() - - totalExpectedSleep := int64(0) - // Note that we do not add the expected sleep duration for the last error since the - // streaming context will be cancelled during that error. - for i := 1; i < calls; i++ { - totalExpectedSleep += backoffOpts.GetSleepTimeFromErrorCount(i).Nanoseconds() - } - str := &mockStream{} - readyChan := make(chan struct{}) - inputChan := make(chan gostream.MediaReleasePair[image.Image]) - str.streamingReadyFunc = func() <-chan struct{} { - return readyChan - } - str.inputFramesFunc = func() (chan<- gostream.MediaReleasePair[image.Image], error) { - return inputChan, nil - } - - go webstream.StreamVideoSource(ctx, videoSrc, str, backoffOpts, logger) - start := time.Now() - readyChan <- struct{}{} - videoReader.wg.Wait() - cancel() - - duration := time.Since(start).Nanoseconds() - test.That(t, duration, test.ShouldBeGreaterThanOrEqualTo, totalExpectedSleep) -} - // setupRealRobot creates a robot from the input config and starts a WebRTC server with video // streaming capabilities. // diff --git a/robot/web/web_c.go b/robot/web/web_c.go index 6c393629e8f..26b25395ca5 100644 --- a/robot/web/web_c.go +++ b/robot/web/web_c.go @@ -10,7 +10,6 @@ import ( "runtime" "slices" "sync" - "time" "github.com/pkg/errors" streampb "go.viam.com/api/stream/v1" @@ -160,7 +159,7 @@ func (svc *webService) makeStreamServer(ctx context.Context) (*StreamServer, err if len(svc.videoSources) != 0 || len(svc.audioSources) != 0 { svc.logger.Debug("not starting streams due to no stream config being set") } - noopServer, err := webstream.NewServer(streams, svc.r, logging.GetOrNewLogger("rdk.networking")) + noopServer, err := webstream.NewServer(streams, svc.r, svc.logger.Sublogger("stream")) return &StreamServer{noopServer, false}, err } @@ -215,7 +214,7 @@ func (svc *webService) makeStreamServer(ctx context.Context) (*StreamServer, err streamTypes = append(streamTypes, false) } - streamServer, err := webstream.NewServer(streams, svc.r, logging.GetOrNewLogger("rdk.networking")) + streamServer, err := webstream.NewServer(streams, svc.r, svc.logger.Sublogger("stream")) if err != nil { return nil, err } @@ -237,12 +236,7 @@ func (svc *webService) startStream(streamFunc func(opts *webstream.BackoffTuning utils.PanicCapturingGo(func() { defer svc.webWorkers.Done() close(waitCh) - opts := &webstream.BackoffTuningOptions{ - BaseSleep: 50 * time.Microsecond, - MaxSleep: 2 * time.Second, - Cooldown: 5 * time.Second, - } - if err := streamFunc(opts); err != nil { + if err := streamFunc(&webstream.BackoffTuningOptions{}); err != nil { if utils.FilterOutError(err, context.Canceled) != nil { svc.logger.Errorw("error streaming", "error", err) } From 23a998b2bd1830b8be95aea353f76c5477979831 Mon Sep 17 00:00:00 2001 From: Sagie Maoz Date: Fri, 6 Sep 2024 12:33:57 -0400 Subject: [PATCH 19/26] DATA-3040: Add debug log messages to binary data export to track retries (#4348) --- cli/data.go | 36 ++++++++++++++++++++++++++---------- cli/dataset.go | 2 +- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/cli/data.go b/cli/data.go index 1635227f928..e887d8f3486 100644 --- a/cli/data.go +++ b/cli/data.go @@ -271,7 +271,7 @@ func (c *viamClient) binaryData(dst string, filter *datapb.Filter, parallelDownl return c.performActionOnBinaryDataFromFilter( func(id *datapb.BinaryID) error { - return downloadBinary(c.c.Context, c.dataClient, dst, id, c.authFlow.httpClient, c.conf.Auth) + return c.downloadBinary(dst, id) }, filter, parallelDownloads, func(i int32) { @@ -412,35 +412,39 @@ func getMatchingBinaryIDs(ctx context.Context, client datapb.DataServiceClient, } } -func downloadBinary(ctx context.Context, client datapb.DataServiceClient, dst string, id *datapb.BinaryID, - httpClient *http.Client, auth authMethod, -) error { +func (c *viamClient) downloadBinary(dst string, id *datapb.BinaryID) error { + debugf(c.c.App.Writer, c.c.Bool(debugFlag), "Attempting to download binary file %s", id.FileId) + var resp *datapb.BinaryDataByIDsResponse var err error largeFile := false // To begin, we assume the file is small and downloadable, so we try getting the binary directly for count := 0; count < maxRetryCount; count++ { - resp, err = client.BinaryDataByIDs(ctx, &datapb.BinaryDataByIDsRequest{ + resp, err = c.dataClient.BinaryDataByIDs(c.c.Context, &datapb.BinaryDataByIDsRequest{ BinaryIds: []*datapb.BinaryID{id}, IncludeBinary: !largeFile, }) // If the file is too large, we break and try a different pathway for downloading if err == nil || status.Code(err) == codes.ResourceExhausted { + debugf(c.c.App.Writer, c.c.Bool(debugFlag), "Small file download file %s: attempt %d/%d succeeded", id.FileId, count+1, maxRetryCount) break } + debugf(c.c.App.Writer, c.c.Bool(debugFlag), "Small file download for file %s: attempt %d/%d failed", id.FileId, count+1, maxRetryCount) } // For large files, we get the metadata but not the binary itself // Resource exhausted is returned when the message we're receiving exceeds the GRPC maximum limit if err != nil && status.Code(err) == codes.ResourceExhausted { largeFile = true for count := 0; count < maxRetryCount; count++ { - resp, err = client.BinaryDataByIDs(ctx, &datapb.BinaryDataByIDsRequest{ + resp, err = c.dataClient.BinaryDataByIDs(c.c.Context, &datapb.BinaryDataByIDsRequest{ BinaryIds: []*datapb.BinaryID{id}, IncludeBinary: !largeFile, }) if err == nil { + debugf(c.c.App.Writer, c.c.Bool(debugFlag), "Metadata fetch for file %s: attempt %d/%d succeeded", id.FileId, count+1, maxRetryCount) break } + debugf(c.c.App.Writer, c.c.Bool(debugFlag), "Metadata fetch for file %s: attempt %d/%d failed", id.FileId, count+1, maxRetryCount) } } if err != nil { @@ -477,19 +481,20 @@ func downloadBinary(ctx context.Context, client datapb.DataServiceClient, dst st var bin []byte if largeFile { + debugf(c.c.App.Writer, c.c.Bool(debugFlag), "Attempting file %s as a large file download", id.FileId) // Make request to the URI for large files since we exceed the message limit for gRPC - req, err := http.NewRequestWithContext(ctx, http.MethodGet, datum.GetMetadata().GetUri(), nil) + req, err := http.NewRequestWithContext(c.c.Context, http.MethodGet, datum.GetMetadata().GetUri(), nil) if err != nil { return errors.Wrapf(err, serverErrorMessage) } // Set the headers so HTTP requests that are not gRPC calls can still be authenticated in app // We can authenticate via token or API key, so we try both. - token, ok := auth.(*token) + token, ok := c.conf.Auth.(*token) if ok { req.Header.Add(rpc.MetadataFieldAuthorization, rpc.AuthorizationValuePrefixBearer+token.AccessToken) } - apiKey, ok := auth.(*apiKey) + apiKey, ok := c.conf.Auth.(*apiKey) if ok { req.Header.Add("key_id", apiKey.KeyID) req.Header.Add("key", apiKey.KeyCrypto) @@ -497,17 +502,22 @@ func downloadBinary(ctx context.Context, client datapb.DataServiceClient, dst st var res *http.Response for count := 0; count < maxRetryCount; count++ { - res, err = httpClient.Do(req) + res, err = c.authFlow.httpClient.Do(req) if err == nil && res.StatusCode == http.StatusOK { + debugf(c.c.App.Writer, c.c.Bool(debugFlag), + "Large file download for file %s: attempt %d/%d succeeded", id.FileId, count+1, maxRetryCount) break } + debugf(c.c.App.Writer, c.c.Bool(debugFlag), "Large file download for file %s: attempt %d/%d failed", id.FileId, count+1, maxRetryCount) } if err != nil { + debugf(c.c.App.Writer, c.c.Bool(debugFlag), "Failed downloading large file %s: %s", id.FileId, err) return errors.Wrapf(err, serverErrorMessage) } if res.StatusCode != http.StatusOK { + debugf(c.c.App.Writer, c.c.Bool(debugFlag), "Failed downloading large file %s: Server returned %d response", id.FileId, res.StatusCode) return errors.New(serverErrorMessage) } defer func() { @@ -516,6 +526,7 @@ func downloadBinary(ctx context.Context, client datapb.DataServiceClient, dst st bin, err = io.ReadAll(res.Body) if err != nil { + debugf(c.c.App.Writer, c.c.Bool(debugFlag), "Failed downloading large file %s, error occurred while reading: %s", id.FileId, err) return errors.Wrapf(err, serverErrorMessage) } } else { @@ -533,6 +544,7 @@ func downloadBinary(ctx context.Context, client datapb.DataServiceClient, dst st if ext == gzFileExt { r, err = gzip.NewReader(r) if err != nil { + debugf(c.c.App.Writer, c.c.Bool(debugFlag), "Failed unzipping file %s: %s", id.FileId, err) return err } } else if filepath.Ext(dataPath) != ext { @@ -542,18 +554,22 @@ func downloadBinary(ctx context.Context, client datapb.DataServiceClient, dst st } if err := os.MkdirAll(filepath.Dir(dataPath), 0o700); err != nil { + debugf(c.c.App.Writer, c.c.Bool(debugFlag), "Failed creating data directory %s: %s", dataPath, err) return errors.Wrapf(err, "could not create data directory %s", filepath.Dir(dataPath)) } //nolint:gosec dataFile, err := os.Create(dataPath) if err != nil { + debugf(c.c.App.Writer, c.c.Bool(debugFlag), "Failed creating file %s: %s", id.FileId, err) return errors.Wrapf(err, fmt.Sprintf("could not create file for datum %s", datum.GetMetadata().GetId())) } //nolint:gosec if _, err := io.Copy(dataFile, r); err != nil { + debugf(c.c.App.Writer, c.c.Bool(debugFlag), "Failed writing data to file %s: %s", id.FileId, err) return err } if err := r.Close(); err != nil { + debugf(c.c.App.Writer, c.c.Bool(debugFlag), "Failed closing file %s: %s", id.FileId, err) return err } return nil diff --git a/cli/dataset.go b/cli/dataset.go index cd67c48fa1d..4e291043613 100644 --- a/cli/dataset.go +++ b/cli/dataset.go @@ -197,7 +197,7 @@ func (c *viamClient) downloadDataset(dst, datasetID string, includeJSONLines boo return c.performActionOnBinaryDataFromFilter( func(id *datapb.BinaryID) error { - downloadErr := downloadBinary(c.c.Context, c.dataClient, dst, id, c.authFlow.httpClient, c.conf.Auth) + downloadErr := c.downloadBinary(dst, id) var datasetErr error if includeJSONLines { datasetErr = binaryDataToJSONLines(c.c.Context, c.dataClient, dst, datasetFile, id) From 3b5bc06ce3680b65bf930dbdcee163868c470dde Mon Sep 17 00:00:00 2001 From: abe-winter Date: Fri, 6 Sep 2024 14:14:11 -0400 Subject: [PATCH 20/26] deprecate unneeded utils.Testing (#4350) --- utils/value.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/utils/value.go b/utils/value.go index be5a5343391..d8adc54f54c 100644 --- a/utils/value.go +++ b/utils/value.go @@ -1,10 +1,10 @@ package utils import ( - "flag" "math/rand" "os" "strings" + "testing" ) // AssertType attempts to assert that the given interface argument is @@ -46,9 +46,9 @@ type Rand interface { } // Testing returns true when you are running in test suite. +// Deprecated: this is in the standard library now. func Testing() bool { - // TODO switch to official testing.Testing method when we are on go 1.21 - return flag.Lookup("test.v") != nil + return testing.Testing() } // randWrapper is a pass-through to the shared math/rand functions. @@ -61,7 +61,7 @@ func (randWrapper) Float64() float64 { // SafeTestingRand returns a wrapper around the shared math/rand source in prod, // and a deterministic rand.Rand seeded with 0 in test. func SafeTestingRand() Rand { - if Testing() { + if testing.Testing() { return rand.New(rand.NewSource(0)) //nolint:gosec } return randWrapper{} From 60b08d6453eb6d8713bd177eff0e162d06e92da3 Mon Sep 17 00:00:00 2001 From: Sierra Guequierre Date: Mon, 9 Sep 2024 12:58:34 -0400 Subject: [PATCH 21/26] DOCS-2498: Link from RDK docs to main docs on component pages part 2 (#4352) --- components/input/input.go | 6 ++++++ components/motor/motor.go | 7 +++++++ components/movementsensor/movementsensor.go | 8 +++++++- components/posetracker/pose_tracker.go | 2 +- components/powersensor/powersensor.go | 8 +++++++- components/sensor/sensor.go | 3 +++ components/servo/servo.go | 7 +++++++ resource/resource.go | 3 +++ 8 files changed, 41 insertions(+), 3 deletions(-) diff --git a/components/input/input.go b/components/input/input.go index 37c8eb74d40..cdfb8b19355 100644 --- a/components/input/input.go +++ b/components/input/input.go @@ -1,4 +1,7 @@ // Package input provides human input, such as buttons, switches, knobs, gamepads, joysticks, keyboards, mice, etc. +// For more information, see the [input controller component docs]. +// +// [input controller component docs]: https://docs.viam.com/components/input-controller/ package input import ( @@ -35,6 +38,7 @@ func Named(name string) resource.Name { // Controller is a logical "container" more than an actual device // Could be a single gamepad, or a collection of digitalInterrupts and analogReaders, a keyboard, etc. +// For more information, see the [input controller component docs]. // // Controls example: // @@ -64,6 +68,8 @@ func Named(name string) resource.Name { // return errors.New("button `ButtonStart` not found; controller may be disconnected") // } // Mycontroller.RegisterControlCallback(context.Background(), input.ButtonStart, triggers, printStartTime, nil) +// +// [input controller component docs]: https://docs.viam.com/components/input-controller/ type Controller interface { resource.Resource diff --git a/components/motor/motor.go b/components/motor/motor.go index 78f8edaa78c..0a2796814fa 100644 --- a/components/motor/motor.go +++ b/components/motor/motor.go @@ -1,3 +1,7 @@ +// Package motor defines machines that convert electricity into rotary motion. +// For more information, see the [motor component docs]. +// +// [motor component docs]: https://docs.viam.com/components/motor/ package motor import ( @@ -37,6 +41,7 @@ const SubtypeName = "motor" var API = resource.APINamespaceRDK.WithComponentType(SubtypeName) // A Motor represents a physical motor connected to a board. +// For more information, see the [motor component docs]. // // SetPower example: // @@ -92,6 +97,8 @@ var API = resource.APINamespaceRDK.WithComponentType(SubtypeName) // logger.Info(powered) // logger.Info("Power percent:") // logger.Info(pct) +// +// [motor component docs]: https://docs.viam.com/components/motor/ type Motor interface { resource.Resource resource.Actuator diff --git a/components/movementsensor/movementsensor.go b/components/movementsensor/movementsensor.go index 78adeffe749..e458b29343f 100644 --- a/components/movementsensor/movementsensor.go +++ b/components/movementsensor/movementsensor.go @@ -1,4 +1,7 @@ -// Package movementsensor defines the interfaces of a MovementSensor +// Package movementsensor defines the interfaces of a MovementSensor. +// For more information, see the [movement sensor component docs]. +// +// [movement sensor component docs]: https://docs.viam.com/components/movement-sensor/ package movementsensor import ( @@ -65,6 +68,7 @@ func Named(name string) resource.Name { } // A MovementSensor reports information about the robot's direction, position and speed. +// For more information, see the [movement sensor component docs]. // // Position example: // @@ -117,6 +121,8 @@ func Named(name string) resource.Name { // // // Get the accuracy of the movement sensor. // accuracy, err := myMovementSensor.Accuracy(context.Background(), nil) +// +// [movement sensor component docs]: https://docs.viam.com/components/movement-sensor/ type MovementSensor interface { resource.Sensor resource.Resource diff --git a/components/posetracker/pose_tracker.go b/components/posetracker/pose_tracker.go index 7d3e9586d00..969f2c8a8c9 100644 --- a/components/posetracker/pose_tracker.go +++ b/components/posetracker/pose_tracker.go @@ -1,5 +1,5 @@ // Package posetracker contains the interface and gRPC infrastructure -// for a pose tracker component +// for a pose tracker component. package posetracker import ( diff --git a/components/powersensor/powersensor.go b/components/powersensor/powersensor.go index 5e703d6c9e2..bcf371bd39b 100644 --- a/components/powersensor/powersensor.go +++ b/components/powersensor/powersensor.go @@ -1,4 +1,7 @@ -// Package powersensor defines the interfaces of a powersensor +// Package powersensor defines the interfaces of a powersensor. +// For more information, see the [power sensor component docs]. +// +// [power sensor component docs]: https://docs.viam.com/components/power-sensor/ package powersensor import ( @@ -48,6 +51,7 @@ func Named(name string) resource.Name { } // A PowerSensor reports information about voltage, current and power. +// For more information, see the [power sensor component docs]. // // Voltage example: // @@ -63,6 +67,8 @@ func Named(name string) resource.Name { // // // Get the power measurement from device in watts. // power, err := myPowerSensor.Power(context.Background(), nil) +// +// [power sensor component docs]: https://docs.viam.com/components/power-sensor/ type PowerSensor interface { resource.Sensor resource.Resource diff --git a/components/sensor/sensor.go b/components/sensor/sensor.go index 2df2b048d2e..db8debd8d04 100644 --- a/components/sensor/sensor.go +++ b/components/sensor/sensor.go @@ -1,4 +1,7 @@ // Package sensor defines an abstract sensing device that can provide measurement readings. +// For more information, see the [sensor component docs]. +// +// [sensor component docs]: https://docs.viam.com/components/sensor/ package sensor import ( diff --git a/components/servo/servo.go b/components/servo/servo.go index 9846a11a315..392dc61fe28 100644 --- a/components/servo/servo.go +++ b/components/servo/servo.go @@ -1,3 +1,7 @@ +// Package servo supports “RC” or “hobby” servo motors. +// For more information, see the [servo component docs]. +// +// [servo component docs]: https://docs.viam.com/components/servo/ package servo import ( @@ -31,6 +35,7 @@ const SubtypeName = "servo" var API = resource.APINamespaceRDK.WithComponentType(SubtypeName) // A Servo represents a physical servo connected to a board. +// For more information, see the [servo component docs]. // // Move example: // @@ -50,6 +55,8 @@ var API = resource.APINamespaceRDK.WithComponentType(SubtypeName) // // logger.Info("Position 1: ", pos1) // logger.Info("Position 2: ", pos2) +// +// [servo component docs]: https://docs.viam.com/components/servo/ type Servo interface { resource.Resource resource.Actuator diff --git a/resource/resource.go b/resource/resource.go index a910653b85b..9a7a9b4b267 100644 --- a/resource/resource.go +++ b/resource/resource.go @@ -150,11 +150,14 @@ func ContainsReservedCharacter(val string) error { // A Sensor represents a general purpose sensor that can give arbitrary readings // of all readings that it is sensing. +// For more information, see the [sensor component docs]. // // Readings example: // // // Get the readings provided by the sensor. // readings, err := mySensor.Readings(context.Background(), nil) +// +// [sensor component docs]: https://docs.viam.com/components/sensor/ type Sensor interface { // Readings return data specific to the type of sensor and can be of any type. Readings(ctx context.Context, extra map[string]interface{}) (map[string]interface{}, error) From 1332e697ff455a6619e8bca3c6122e8487ddcf54 Mon Sep 17 00:00:00 2001 From: Sierra Guequierre Date: Mon, 9 Sep 2024 12:58:53 -0400 Subject: [PATCH 22/26] DOCS-2498: Link to main component documentation on RDK docs site (#4351) --- components/arm/arm.go | 6 ++++++ components/base/base.go | 6 ++++++ components/board/board.go | 6 ++++++ components/camera/camera.go | 6 ++++++ components/encoder/encoder.go | 8 +++++++- components/gantry/gantry.go | 7 +++++++ components/generic/generic.go | 5 ++++- components/gripper/gripper.go | 6 ++++++ 8 files changed, 48 insertions(+), 2 deletions(-) diff --git a/components/arm/arm.go b/components/arm/arm.go index 38cc02dab8a..d01d6502a3f 100644 --- a/components/arm/arm.go +++ b/components/arm/arm.go @@ -1,6 +1,9 @@ //go:build !no_cgo // Package arm defines the arm that a robot uses to manipulate objects. +// For more information, see the [arm component docs]. +// +// [arm component docs]: https://docs.viam.com/components/arm/ package arm import ( @@ -48,6 +51,7 @@ func Named(name string) resource.Name { } // An Arm represents a physical robotic arm that exists in three-dimensional space. +// For more information, see the [arm component docs]. // // EndPosition example: // @@ -87,6 +91,8 @@ func Named(name string) resource.Name { // // // Get the current position of each joint on the arm as JointPositions. // pos, err := myArm.JointPositions(context.Background(), nil) +// +// [arm component docs]: https://docs.viam.com/components/arm/ type Arm interface { resource.Resource referenceframe.ModelFramer diff --git a/components/base/base.go b/components/base/base.go index 4a6cb3f17e4..7150bad73a3 100644 --- a/components/base/base.go +++ b/components/base/base.go @@ -1,4 +1,7 @@ // Package base defines the base that a robot uses to move around. +// For more information, see the [base component docs]. +// +// [base component docs]: https://docs.viam.com/components/base/ package base import ( @@ -34,6 +37,7 @@ func Named(name string) resource.Name { } // A Base represents a physical base of a robot. +// For more information, see the [base component docs]. // // MoveStraight example: // @@ -93,6 +97,8 @@ func Named(name string) resource.Name { // // // Get the wheel circumference // myBaseWheelCircumference := properties.WheelCircumferenceMeters +// +// [base component docs]: https://docs.viam.com/components/base/ type Base interface { resource.Resource resource.Actuator diff --git a/components/board/board.go b/components/board/board.go index 2b965259511..be6a8cffad9 100644 --- a/components/board/board.go +++ b/components/board/board.go @@ -2,6 +2,9 @@ // such as a Raspberry Pi. // // Besides the board itself, some other interfaces it defines are analog pins and digital interrupts. +// For more information, see the [board component docs]. +// +// [board component docs]: https://docs.viam.com/components/board/ package board import ( @@ -46,6 +49,7 @@ func Named(name string) resource.Name { // A Board represents a physical general purpose board that contains various // components such as analogs, and digital interrupts. +// For more information, see the [board component docs]. // // AnalogByName example: // @@ -93,6 +97,8 @@ func Named(name string) resource.Name { // } // // err = myBoard.StreamTicks(context.Background(), interrupts, ticksChan, nil) +// +// [board component docs]: https://docs.viam.com/components/board/ type Board interface { resource.Resource diff --git a/components/camera/camera.go b/components/camera/camera.go index bf9c385dfde..ec1751d53b1 100644 --- a/components/camera/camera.go +++ b/components/camera/camera.go @@ -1,4 +1,7 @@ // Package camera defines an image capturing device. +// For more information, see the [camera component docs]. +// +// [camera component docs]: https://docs.viam.com/components/camera/ package camera import ( @@ -75,6 +78,7 @@ type Camera interface { } // A VideoSource represents anything that can capture frames. +// For more information, see the [camera component docs]. // // Images example: // @@ -105,6 +109,8 @@ type Camera interface { // myCamera, err := camera.FromRobot(machine, "my_camera") // // err = myCamera.Close(ctx) +// +// [camera component docs]: https://docs.viam.com/components/camera/ type VideoSource interface { // Images is used for getting simultaneous images from different imagers, // along with associated metadata (just timestamp for now). It's not for getting a time series of images from the same imager. diff --git a/components/encoder/encoder.go b/components/encoder/encoder.go index aefb98f2f8d..95d5434b1db 100644 --- a/components/encoder/encoder.go +++ b/components/encoder/encoder.go @@ -1,4 +1,7 @@ -// Package encoder implements the encoder component +// Package encoder implements the encoder component. +// For more information, see the [encoder component docs]. +// +// [encoder component docs]: https://docs.viam.com/components/encoder/ package encoder import ( @@ -58,6 +61,7 @@ func (t PositionType) String() string { } // A Encoder turns a position into a signal. +// For more information, see the [encoder component docs]. // // Position example: // @@ -84,6 +88,8 @@ func (t PositionType) String() string { // // // Get whether the encoder returns position in ticks or degrees. // properties, err := myEncoder.Properties(context.Background(), nil) +// +// [encoder component docs]: https://docs.viam.com/components/encoder/ type Encoder interface { resource.Resource diff --git a/components/gantry/gantry.go b/components/gantry/gantry.go index dc0a124f3b6..5d3e1a8ba29 100644 --- a/components/gantry/gantry.go +++ b/components/gantry/gantry.go @@ -1,3 +1,7 @@ +// Package gantry defines a robotic gantry with one or multiple axes. +// For more information, see the [gantry component docs]. +// +// [gantry component docs]: https://docs.viam.com/components/gantry/ package gantry import ( @@ -41,6 +45,7 @@ func Named(name string) resource.Name { } // Gantry is used for controlling gantries of N axis. +// For more information, see the [gantry component docs]. // // Position example: // @@ -74,6 +79,8 @@ func Named(name string) resource.Name { // myGantry, err := gantry.FromRobot(machine, "my_gantry") // // myGantry.Home(context.Background(), nil) +// +// [gantry component docs]: https://docs.viam.com/components/gantry/ type Gantry interface { resource.Resource resource.Actuator diff --git a/components/generic/generic.go b/components/generic/generic.go index e19bffff5a9..218b6fe5158 100644 --- a/components/generic/generic.go +++ b/components/generic/generic.go @@ -1,4 +1,7 @@ -// Package generic defines an abstract generic device and DoCommand() method +// Package generic defines an abstract generic device and DoCommand() method. +// For more information, see the [generic component docs]. +// +// [generic component docs]: https://docs.viam.com/components/generic/ package generic import ( diff --git a/components/gripper/gripper.go b/components/gripper/gripper.go index b34e6c0d66c..379e5186bdb 100644 --- a/components/gripper/gripper.go +++ b/components/gripper/gripper.go @@ -1,4 +1,7 @@ // Package gripper defines a robotic gripper. +// For more information, see the [gripper component docs]. +// +// [gripper component docs]: https://docs.viam.com/components/gripper/ package gripper import ( @@ -34,6 +37,7 @@ func Named(name string) resource.Name { } // A Gripper represents a physical robotic gripper. +// For more information, see the [gripper component docs]. // // Open example: // @@ -44,6 +48,8 @@ func Named(name string) resource.Name { // // // Grab with the gripper. // grabbed, err := myGripper.Grab(context.Background(), nil) +// +// [gripper component docs]: https://docs.viam.com/components/gripper/ type Gripper interface { resource.Resource resource.Shaped From a0cd8bd442208e08d480ee0b60d67a9338d825de Mon Sep 17 00:00:00 2001 From: martha-johnston <106617924+martha-johnston@users.noreply.github.com> Date: Mon, 9 Sep 2024 15:38:57 -0400 Subject: [PATCH 23/26] RSDK-8699: Cannot go to gantry position 0.0, only 0.2+. (#4354) --- components/gantry/singleaxis/singleaxis.go | 17 +++++++---------- components/gantry/singleaxis/singleaxis_test.go | 13 +++++++++++-- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/components/gantry/singleaxis/singleaxis.go b/components/gantry/singleaxis/singleaxis.go index 34f01ef5f7e..3c360f59265 100644 --- a/components/gantry/singleaxis/singleaxis.go +++ b/components/gantry/singleaxis/singleaxis.go @@ -30,9 +30,6 @@ var ( homingTimeout = time.Duration(15e9) ) -// limitErrorMargin is added or subtracted from the location of the limit switch to ensure the switch is not passed. -const limitErrorMargin = 0.25 - // Config is used for converting singleAxis config attributes. type Config struct { Board string `json:"board,omitempty"` // used to read limit switch pins and control motor with gpio pins @@ -372,7 +369,7 @@ func (g *singleAxis) homeLimSwitch(ctx context.Context) error { if g.positionRange == 0 { g.logger.CError(ctx, "positionRange is 0 or not a valid number") } else { - g.logger.CDebugf(ctx, "positionA: %0.2f positionB: %0.2f range: %0.2f", positionA, positionB, g.positionRange) + g.logger.CInfof(ctx, "positionA: %0.2f positionB: %0.2f range: %0.2f", positionA, positionB, g.positionRange) } // Go to start position at the middle of the axis. @@ -548,15 +545,15 @@ func (g *singleAxis) MoveToPosition(ctx context.Context, positions, speeds []flo // Currently needs to be moved by underlying gantry motor. if len(g.limitSwitchPins) > 0 { // Stops if position x is past the 0 limit switch - if x <= (g.positionLimits[0] + limitErrorMargin) { - g.logger.CError(ctx, "Cannot move past limit switch!") - return g.motor.Stop(ctx, extra) + if x < g.positionLimits[0] { + err := errors.New("cannot move past limit switch") + return multierr.Combine(err, g.motor.Stop(ctx, extra)) } // Stops if position x is past the at-length limit switch - if x >= (g.positionLimits[1] - limitErrorMargin) { - g.logger.CError(ctx, "Cannot move past limit switch!") - return g.motor.Stop(ctx, extra) + if x > g.positionLimits[1] { + err := errors.New("cannot move past limit switch") + return multierr.Combine(err, g.motor.Stop(ctx, extra)) } } diff --git a/components/gantry/singleaxis/singleaxis_test.go b/components/gantry/singleaxis/singleaxis_test.go index 1ffc6da2514..226f391e06f 100644 --- a/components/gantry/singleaxis/singleaxis_test.go +++ b/components/gantry/singleaxis/singleaxis_test.go @@ -666,12 +666,21 @@ func TestMoveToPosition(t *testing.T) { err = fakegantry.MoveToPosition(ctx, pos, speed, nil) test.That(t, err.Error(), test.ShouldContainSubstring, "out of range") + pos = []float64{0} fakegantry.lengthMm = float64(4) fakegantry.positionLimits = []float64{0, 4} fakegantry.limitSwitchPins = []string{"1", "2"} err = fakegantry.MoveToPosition(ctx, pos, speed, nil) test.That(t, err, test.ShouldBeNil) + pos = []float64{4} + fakegantry.lengthMm = float64(4) + fakegantry.positionLimits = []float64{0, 10} + fakegantry.limitSwitchPins = []string{"1", "2"} + err = fakegantry.MoveToPosition(ctx, pos, speed, nil) + test.That(t, err, test.ShouldBeNil) + + pos = []float64{1} fakegantry.lengthMm = float64(4) fakegantry.positionLimits = []float64{0.01, .01} fakegantry.limitSwitchPins = []string{"1", "2"} @@ -868,10 +877,10 @@ func TestGoToInputs(t *testing.T) { inputs = []referenceframe.Input{{Value: 1.0}} err = fakegantry.GoToInputs(ctx, inputs) - test.That(t, err, test.ShouldBeNil) + test.That(t, err.Error(), test.ShouldContainSubstring, "cannot move past limit switch") err = fakegantry.GoToInputs(ctx, inputs, inputs) - test.That(t, err, test.ShouldBeNil) + test.That(t, err.Error(), test.ShouldContainSubstring, "cannot move past limit switch") err = fakegantry.GoToInputs(ctx) test.That(t, err, test.ShouldBeNil) From 0146b5df49c6c921ba6cc886c0cba4a26e8ae9e4 Mon Sep 17 00:00:00 2001 From: Cheuk <90270663+cheukt@users.noreply.github.com> Date: Mon, 9 Sep 2024 17:20:56 -0400 Subject: [PATCH 24/26] RSDK-8722 - Log RDK version to cloud (#4358) --- web/server/entrypoint.go | 43 +++++++++++++++++++++++++++------------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/web/server/entrypoint.go b/web/server/entrypoint.go index 93709f277d0..a3fcb88c395 100644 --- a/web/server/entrypoint.go +++ b/web/server/entrypoint.go @@ -55,6 +55,21 @@ type robotServer struct { logger logging.Logger } +func logVersion(logger logging.Logger) { + var versionFields []interface{} + if config.Version != "" { + versionFields = append(versionFields, "version", config.Version) + } + if config.GitRevision != "" { + versionFields = append(versionFields, "git_rev", config.GitRevision) + } + if len(versionFields) != 0 { + logger.Infow("Viam RDK", versionFields...) + } else { + logger.Info("Viam RDK built from source; version unknown") + } +} + // RunServer is an entry point to starting the web server that can be called by main in a code // sample or otherwise be used to initialize the server. func RunServer(ctx context.Context, args []string, _ logging.Logger) (err error) { @@ -85,24 +100,21 @@ func RunServer(ctx context.Context, args []string, _ logging.Logger) (err error) config.InitLoggingSettings(logger, argsParsed.Debug) - // Always log the version, return early if the '-version' flag was provided - // fmt.Println would be better but fails linting. Good enough. - var versionFields []interface{} - if config.Version != "" { - versionFields = append(versionFields, "version", config.Version) - } - if config.GitRevision != "" { - versionFields = append(versionFields, "git_rev", config.GitRevision) - } - if len(versionFields) != 0 { - logger.Infow("Viam RDK", versionFields...) - } else { - logger.Info("Viam RDK built from source; version unknown") - } if argsParsed.Version { + // log version here and return if version flag. + logVersion(logger) return } + // log version locally if server fails and exits while attempting to start up + var versionLogged bool + defer func() { + if !versionLogged { + logger.CInfo(ctx, "error starting viam-server, logging version and exiting") + logVersion(logger) + } + }() + if argsParsed.ConfigFile == "" { logger.Error("please specify a config file through the -config parameter.") return @@ -155,6 +167,9 @@ func RunServer(ctx context.Context, args []string, _ logging.Logger) (err error) logger.AddAppender(netAppender) } + // log version after netlogger is initialized so it's captured in cloud machine logs. + logVersion(logger) + versionLogged = true server := robotServer{ logger: logger, From 2dc7ca798bf30da1aa4f887c9ce1cef02182aef4 Mon Sep 17 00:00:00 2001 From: abe-winter Date: Tue, 10 Sep 2024 14:22:16 -0400 Subject: [PATCH 25/26] `module download` command (#4353) --- cli/app.go | 54 +++++++++++++++++++++--------- cli/module_registry.go | 76 ++++++++++++++++++++++++++++++++++++++++++ cli/packages.go | 10 ++++++ 3 files changed, 124 insertions(+), 16 deletions(-) diff --git a/cli/app.go b/cli/app.go index 737ff3f93da..5c19a4bb446 100644 --- a/cli/app.go +++ b/cli/app.go @@ -51,6 +51,7 @@ const ( moduleFlagLocal = "local" moduleFlagHomeDir = "home" moduleCreateLocalOnly = "local-only" + moduleFlagID = "id" moduleBuildFlagPath = "module" moduleBuildFlagRef = "ref" @@ -1746,6 +1747,32 @@ This won't work unless you have an existing installation of our GitHub app on yo }, Action: ReloadModuleAction, }, + { + Name: "download", + Usage: "download a module package from the registry", + UsageText: createUsageText("module download", []string{}, false), + Flags: []cli.Flag{ + &cli.PathFlag{ + Name: packageFlagDestination, + Usage: "output directory for downloaded package", + Value: ".", + }, + &cli.StringFlag{ + Name: moduleFlagID, + Usage: "module ID as org-id:name or namespace:name. if missing, will try to read from meta.json", + }, + &cli.StringFlag{ + Name: packageFlagVersion, + Usage: "version of the requested package, can be `latest` to get the most recent version", + Value: "latest", + }, + &cli.StringFlag{ + Name: moduleFlagPlatform, + Usage: "platform like 'linux/amd64'. if missing, will use platform of the CLI binary", + }, + }, + Action: DownloadModuleAction, + }, }, }, { @@ -1757,30 +1784,25 @@ This won't work unless you have an existing installation of our GitHub app on yo Name: "export", Usage: "download a package from Viam cloud", UsageText: createUsageText("packages export", - []string{ - packageFlagDestination, generalFlagOrgID, packageFlagName, - packageFlagVersion, packageFlagType, - }, false), + []string{packageFlagType}, false), Flags: []cli.Flag{ &cli.PathFlag{ - Name: packageFlagDestination, - Required: true, - Usage: "output directory for downloaded package", + Name: packageFlagDestination, + Usage: "output directory for downloaded package", + Value: ".", }, &cli.StringFlag{ - Name: generalFlagOrgID, - Required: true, - Usage: "organization ID of the requested package", + Name: generalFlagOrgID, + Usage: "organization ID or namespace of the requested package. if missing, will try to read from meta.json", }, &cli.StringFlag{ - Name: packageFlagName, - Required: true, - Usage: "name of the requested package", + Name: packageFlagName, + Usage: "name of the requested package. if missing, will try to read from meta.json", }, &cli.StringFlag{ - Name: packageFlagVersion, - Required: true, - Usage: "version of the requested package, can be `latest` to get the most recent version", + Name: packageFlagVersion, + Usage: "version of the requested package, can be `latest` to get the most recent version", + Value: "latest", }, &cli.StringFlag{ Name: packageFlagType, diff --git a/cli/module_registry.go b/cli/module_registry.go index 7ce7681fe9c..c31b85d1332 100644 --- a/cli/module_registry.go +++ b/cli/module_registry.go @@ -12,7 +12,10 @@ import ( "math" "os" "os/exec" + "path" "path/filepath" + "runtime" + "slices" "strings" "github.com/google/uuid" @@ -877,3 +880,76 @@ func getNextModuleUploadRequest(file *os.File) (*apppb.UploadModuleFileRequest, }, }, nil } + +// DownloadModuleAction downloads a module. +func DownloadModuleAction(c *cli.Context) error { + moduleID := c.String(moduleFlagID) + if moduleID == "" { + manifest, err := loadManifest(defaultManifestFilename) + if err != nil { + return errors.Wrap(err, "trying to get package ID from meta.json") + } + moduleID = manifest.ModuleID + } + client, err := newViamClient(c) + if err != nil { + return err + } + if err := client.ensureLoggedIn(); err != nil { + return err + } + req := &apppb.GetModuleRequest{ModuleId: moduleID} + res, err := client.client.GetModule(c.Context, req) + if err != nil { + return err + } + if len(res.Module.Versions) == 0 { + return errors.New("module has 0 uploaded versions, nothing to download") + } + requestedVersion := c.String(packageFlagVersion) + var ver *apppb.VersionHistory + if requestedVersion == "latest" { + ver = res.Module.Versions[len(res.Module.Versions)-1] + } else { + for _, iVer := range res.Module.Versions { + if iVer.Version == requestedVersion { + ver = iVer + break + } + } + if ver == nil { + return fmt.Errorf("version %s not found in versions for module", requestedVersion) + } + } + infof(c.App.ErrWriter, "found version %s", ver.Version) + if len(ver.Files) == 0 { + return fmt.Errorf("version %s has 0 files uploaded", ver.Version) + } + platform := c.String(moduleFlagPlatform) + if platform == "" { + platform = fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH) + infof(c.App.ErrWriter, "using default platform %s", platform) + } + if !slices.ContainsFunc(ver.Files, func(file *apppb.Uploads) bool { return file.Platform == platform }) { + return fmt.Errorf("platform %s not present for version %s", platform, ver.Version) + } + include := true + packageType := packagespb.PackageType_PACKAGE_TYPE_MODULE + // note: this is working around a GetPackage quirk where platform messes with version + fullVersion := fmt.Sprintf("%s-%s", ver.Version, strings.ReplaceAll(platform, "/", "-")) + pkg, err := client.packageClient.GetPackage(c.Context, &packagespb.GetPackageRequest{ + Id: strings.ReplaceAll(moduleID, ":", "/"), + Version: fullVersion, + IncludeUrl: &include, + Type: &packageType, + }) + if err != nil { + return err + } + destName := strings.ReplaceAll(moduleID, ":", "-") + infof(c.App.ErrWriter, "saving to %s", path.Join(c.String(packageFlagDestination), fullVersion, destName+".tar.gz")) + return downloadPackageFromURL(c.Context, client.authFlow.httpClient, + c.String(packageFlagDestination), destName, + fullVersion, pkg.Package.Url, client.conf.Auth, + ) +} diff --git a/cli/packages.go b/cli/packages.go index 53787264391..2b76e15ec6f 100644 --- a/cli/packages.go +++ b/cli/packages.go @@ -84,6 +84,16 @@ func (c *viamClient) packageExportAction(orgID, name, version, packageType, dest if err := c.ensureLoggedIn(); err != nil { return err } + if orgID == "" || name == "" { + if orgID != "" || name != "" { + return fmt.Errorf("if either of %s or %s is missing, both must be missing", generalFlagOrgID, packageFlagName) + } + manifest, err := loadManifest(defaultManifestFilename) + if err != nil { + return errors.Wrap(err, "trying to get package ID from meta.json") + } + orgID, name, _ = strings.Cut(manifest.ModuleID, ":") + } // Package ID is the / packageID := path.Join(orgID, name) packageTypeProto, err := convertPackageTypeToProto(packageType) From 98d78c38fdee44edb60d145ff661c6896a2b7baf Mon Sep 17 00:00:00 2001 From: Maxim Pertsov Date: Tue, 10 Sep 2024 14:34:48 -0400 Subject: [PATCH 26/26] RSDK-8701 Show configuring state through MachineStatus endpoint (#4359) --- robot/impl/local_robot.go | 2 - robot/impl/local_robot_test.go | 259 ++++++++++++++++++++------------- testutils/resource_utils.go | 14 ++ 3 files changed, 173 insertions(+), 102 deletions(-) diff --git a/robot/impl/local_robot.go b/robot/impl/local_robot.go index 431aab4b6c2..6530b824e32 100644 --- a/robot/impl/local_robot.go +++ b/robot/impl/local_robot.go @@ -1422,9 +1422,7 @@ func (r *localRobot) Shutdown(ctx context.Context) error { func (r *localRobot) MachineStatus(ctx context.Context) (robot.MachineStatus, error) { var result robot.MachineStatus - r.manager.resourceGraphLock.Lock() result.Resources = append(result.Resources, r.manager.resources.Status()...) - r.manager.resourceGraphLock.Unlock() r.configRevisionMu.RLock() result.Config = r.configRevision diff --git a/robot/impl/local_robot_test.go b/robot/impl/local_robot_test.go index de422e29695..12b97fa907f 100644 --- a/robot/impl/local_robot_test.go +++ b/robot/impl/local_robot_test.go @@ -6,11 +6,13 @@ import ( "crypto/x509" "errors" "fmt" + "log" "math" "os" "path" "path/filepath" "strings" + "sync" "testing" "time" @@ -3329,8 +3331,23 @@ type mockResource struct { } type mockConfig struct { - Value int `json:"value"` - Fail bool `json:"fail"` + Value int `json:"value"` + Fail bool `json:"fail"` + Sleep string `json:"sleep"` +} + +//nolint:unparam // the resource name is currently always "m" but this could easily change +func newMockConfig(name string, val int, fail bool, sleep string) resource.Config { + return resource.Config{ + Name: name, + Model: mockModel, + API: mockAPI, + // We need to specify both `Attributes` and `ConvertedAttributes`. + // The former triggers a reconfiguration and the former is actually + // used to reconfigure the component. + Attributes: rutils.AttributeMap{"value": val, "fail": fail, "sleep": sleep}, + ConvertedAttributes: &mockConfig{Value: val, Fail: fail, Sleep: sleep}, + } } var errMockValidation = errors.New("whoops") @@ -3368,10 +3385,67 @@ func (m *mockResource) Reconfigure( if err != nil { return err } + if mConf.Sleep != "" { + if d, err := time.ParseDuration(mConf.Sleep); err == nil { + log.Printf("sleeping for %s\n", d) + time.Sleep(d) + } + } m.value = mConf.Value return nil } +// getExpectedDefaultStatuses returns a slice of default [resource.Status] with a given +// revision set for motion and sensor services. +func getExpectedDefaultStatuses(revision string) []resource.Status { + return []resource.Status{ + { + Name: resource.Name{ + API: resource.APINamespaceRDKInternal.WithServiceType("framesystem"), + Name: "builtin", + }, + State: resource.NodeStateReady, + }, + { + Name: resource.Name{ + API: resource.APINamespaceRDKInternal.WithServiceType("cloud_connection"), + Name: "builtin", + }, + State: resource.NodeStateReady, + }, + { + Name: resource.Name{ + API: resource.APINamespaceRDKInternal.WithServiceType("packagemanager"), + Name: "builtin", + }, + State: resource.NodeStateReady, + }, + { + Name: resource.Name{ + API: resource.APINamespaceRDKInternal.WithServiceType("web"), + Name: "builtin", + }, + State: resource.NodeStateReady, + }, + { + Name: resource.Name{ + API: resource.APINamespaceRDK.WithServiceType("motion"), + Name: "builtin", + }, + State: resource.NodeStateReady, + Revision: revision, + }, + { + Name: resource.Name{ + API: resource.APINamespaceRDK.WithServiceType("sensors"), + Name: "builtin", + }, + State: resource.NodeStateReady, + Revision: revision, + }, + } +} + func TestMachineStatus(t *testing.T) { logger := logging.NewTestLogger(t) ctx := context.Background() @@ -3383,93 +3457,32 @@ func TestMachineStatus(t *testing.T) { ) defer resource.Deregister(mockAPI, mockModel) - rev1 := "rev1" - builtinRev := rev1 - - getExpectedDefaultStatuses := func() []resource.Status { - return []resource.Status{ - { - Name: resource.Name{ - API: resource.APINamespaceRDKInternal.WithServiceType("framesystem"), - Name: "builtin", - }, - State: resource.NodeStateReady, - }, - { - Name: resource.Name{ - API: resource.APINamespaceRDKInternal.WithServiceType("cloud_connection"), - Name: "builtin", - }, - State: resource.NodeStateReady, - }, - { - Name: resource.Name{ - API: resource.APINamespaceRDKInternal.WithServiceType("packagemanager"), - Name: "builtin", - }, - State: resource.NodeStateReady, - }, - { - Name: resource.Name{ - API: resource.APINamespaceRDKInternal.WithServiceType("web"), - Name: "builtin", - }, - State: resource.NodeStateReady, - }, - { - Name: resource.Name{ - API: resource.APINamespaceRDK.WithServiceType("motion"), - Name: "builtin", - }, - State: resource.NodeStateReady, - Revision: builtinRev, - }, - { - Name: resource.Name{ - API: resource.APINamespaceRDK.WithServiceType("sensors"), - Name: "builtin", - }, - State: resource.NodeStateReady, - Revision: builtinRev, - }, - } - } - t.Run("default resources", func(t *testing.T) { + rev1 := "rev1" lr := setupLocalRobot(t, ctx, &config.Config{Revision: rev1}, logger) mStatus, err := lr.MachineStatus(ctx) test.That(t, err, test.ShouldBeNil) test.That(t, mStatus.Config.Revision, test.ShouldEqual, rev1) - expectedStatuses := getExpectedDefaultStatuses() + expectedStatuses := getExpectedDefaultStatuses(rev1) rtestutils.VerifySameResourceStatuses(t, mStatus.Resources, expectedStatuses) }) - t.Run("reconfigure", func(t *testing.T) { - lr := setupLocalRobot(t, ctx, &config.Config{Revision: rev1}, logger) - - expectedConfigError := fmt.Errorf("resource config validation error: %w", errMockValidation) + t.Run("poll after working and failing reconfigures", func(t *testing.T) { + lr := setupLocalRobot(t, ctx, &config.Config{Revision: "rev1"}, logger) // Add a fake resource to the robot. rev2 := "rev2" - builtinRev = rev2 lr.Reconfigure(ctx, &config.Config{ - Revision: rev2, - Components: []resource.Config{ - { - Name: "m", - Model: mockModel, - API: mockAPI, - ConvertedAttributes: &mockConfig{}, - }, - }, + Revision: rev2, + Components: []resource.Config{newMockConfig("m", 0, false, "")}, }) mStatus, err := lr.MachineStatus(ctx) test.That(t, err, test.ShouldBeNil) test.That(t, mStatus.Config.Revision, test.ShouldEqual, rev2) expectedStatuses := rtestutils.ConcatResourceStatuses( - getExpectedDefaultStatuses(), + getExpectedDefaultStatuses(rev2), []resource.Status{ { Name: mockNamed("m"), @@ -3482,27 +3495,17 @@ func TestMachineStatus(t *testing.T) { // Update resource config to cause reconfiguration to fail. rev3 := "rev3" - builtinRev = rev3 lr.Reconfigure(ctx, &config.Config{ - Revision: rev3, - Components: []resource.Config{ - { - Name: "m", - Model: mockModel, - API: mockAPI, - // We need to specify both `Attributes` and `ConvertedAttributes`. - // The former triggers a reconfiguration and the former is actually - // used to reconfigure the component. - Attributes: rutils.AttributeMap{"fail": true}, - ConvertedAttributes: &mockConfig{Fail: true}, - }, - }, + Revision: rev3, + Components: []resource.Config{newMockConfig("m", 0, true, "")}, }) mStatus, err = lr.MachineStatus(ctx) test.That(t, err, test.ShouldBeNil) test.That(t, mStatus.Config.Revision, test.ShouldEqual, rev3) + + expectedConfigError := fmt.Errorf("resource config validation error: %w", errMockValidation) expectedStatuses = rtestutils.ConcatResourceStatuses( - getExpectedDefaultStatuses(), + getExpectedDefaultStatuses(rev3), []resource.Status{ { Name: mockNamed("m"), @@ -3516,27 +3519,15 @@ func TestMachineStatus(t *testing.T) { // Update resource with a working config. rev4 := "rev4" - builtinRev = rev4 lr.Reconfigure(ctx, &config.Config{ - Revision: rev4, - Components: []resource.Config{ - { - Name: "m", - Model: mockModel, - API: mockAPI, - // We need to specify both `Attributes` and `ConvertedAttributes`. - // The former triggers a reconfiguration and the former is actually - // used to reconfigure the component. - Attributes: rutils.AttributeMap{"value": 200}, - ConvertedAttributes: &mockConfig{Value: 200}, - }, - }, + Revision: rev4, + Components: []resource.Config{newMockConfig("m", 200, false, "")}, }) mStatus, err = lr.MachineStatus(ctx) test.That(t, err, test.ShouldBeNil) test.That(t, mStatus.Config.Revision, test.ShouldEqual, rev4) expectedStatuses = rtestutils.ConcatResourceStatuses( - getExpectedDefaultStatuses(), + getExpectedDefaultStatuses(rev4), []resource.Status{ { Name: mockNamed("m"), @@ -3547,4 +3538,72 @@ func TestMachineStatus(t *testing.T) { ) rtestutils.VerifySameResourceStatuses(t, mStatus.Resources, expectedStatuses) }) + + t.Run("poll during reconfiguration", func(t *testing.T) { + rev1 := "rev1" + lr := setupLocalRobot(t, ctx, &config.Config{ + Revision: rev1, + Components: []resource.Config{newMockConfig("m", 200, false, "")}, + }, logger) + + // update resource with a working config that is slow to reconfigure. + rev2 := "rev2" + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + lr.Reconfigure(ctx, &config.Config{ + Revision: rev2, + Components: []resource.Config{newMockConfig("m", 300, false, "1s")}, + }) + }() + // sleep for a short amount of time to allow the machine to receive a new + // revision. this sleep should be shorter than the resource update duration + // defined above so that updated resource is still in a "configuring" state. + time.Sleep(time.Millisecond * 100) + + // get status while reconfiguring + mStatus, err := lr.MachineStatus(ctx) + test.That(t, err, test.ShouldBeNil) + test.That(t, mStatus.Config.Revision, test.ShouldEqual, rev2) + + // the component whose config changed should be the only component in a + // "configuring" state and associated with the original revision. + filterConfiguring := rtestutils.FilterByStatus(t, mStatus.Resources, resource.NodeStateConfiguring) + expectedConfiguring := []resource.Status{ + { + Name: mockNamed("m"), + State: resource.NodeStateConfiguring, + Revision: rev1, + }, + } + rtestutils.VerifySameResourceStatuses(t, filterConfiguring, expectedConfiguring) + + // all other components should be in the "ready" state and associated with the + // new revision. + filterReady := rtestutils.FilterByStatus(t, mStatus.Resources, resource.NodeStateReady) + expectedReady := getExpectedDefaultStatuses(rev2) + rtestutils.VerifySameResourceStatuses(t, filterReady, expectedReady) + + wg.Wait() + + // get status after reconfigure finishes + mStatus, err = lr.MachineStatus(ctx) + test.That(t, err, test.ShouldBeNil) + test.That(t, mStatus.Config.Revision, test.ShouldEqual, rev2) + + // now all components, including the one whose config changed, should all be in + // the "ready" state and associated with the new revision. + expectedStatuses := rtestutils.ConcatResourceStatuses( + getExpectedDefaultStatuses(rev2), + []resource.Status{ + { + Name: mockNamed("m"), + State: resource.NodeStateReady, + Revision: rev2, + }, + }, + ) + rtestutils.VerifySameResourceStatuses(t, mStatus.Resources, expectedStatuses) + }) } diff --git a/testutils/resource_utils.go b/testutils/resource_utils.go index eb8868ce8ed..9c87f9748e5 100644 --- a/testutils/resource_utils.go +++ b/testutils/resource_utils.go @@ -83,6 +83,20 @@ func VerifySameResourceStatuses(tb testing.TB, actual, expected []resource.Statu test.That(tb, sortedActual, test.ShouldResemble, sortedExpected) } +// FilterByStatus takes a slice of [resource.Status] and a [resource.NodeState] and +// returns a slice of [resource.Status] that are in the given [resource.NodeState]. +func FilterByStatus(tb testing.TB, resourceStatuses []resource.Status, state resource.NodeState) []resource.Status { + tb.Helper() + + var result []resource.Status + for _, rs := range resourceStatuses { + if rs.State == state { + result = append(result, rs) + } + } + return result +} + func newSortedResourceStatuses(resourceStatuses []resource.Status) []resource.Status { sorted := make([]resource.Status, len(resourceStatuses)) copy(sorted, resourceStatuses)