From 85038a0013e4d981650c3ca37093db67e4166664 Mon Sep 17 00:00:00 2001 From: Naveed Jooma Date: Mon, 23 Jun 2025 14:15:04 -0400 Subject: [PATCH 1/5] Add geometries to camera --- components/camera/camera.go | 1 + components/camera/client.go | 17 +++++++++++++++++ components/camera/client_test.go | 15 ++++++++++++++- components/camera/replaypcd/replaypcd.go | 5 +++++ components/camera/server.go | 13 +++++++++++++ components/camera/videosource/webcam.go | 5 +++++ components/camera/videosourcewrappers.go | 8 ++++++++ testutils/inject/camera.go | 10 ++++++++++ 8 files changed, 73 insertions(+), 1 deletion(-) diff --git a/components/camera/camera.go b/components/camera/camera.go index d61e0a88ea7..75f768e01d7 100644 --- a/components/camera/camera.go +++ b/components/camera/camera.go @@ -125,6 +125,7 @@ type ImageMetadata struct { // [Close method docs]: https://docs.viam.com/dev/reference/apis/components/camera/#close type Camera interface { resource.Resource + resource.Shaped // Image returns a byte slice representing an image that tries to adhere to the MIME type hint. // Image also may return metadata about the frame. diff --git a/components/camera/client.go b/components/camera/client.go index d04d5896113..f1ff9afadaf 100644 --- a/components/camera/client.go +++ b/components/camera/client.go @@ -15,6 +15,7 @@ import ( "github.com/pion/rtp" "github.com/viamrobotics/webrtc/v3" "go.opencensus.io/trace" + commonpb "go.viam.com/api/common/v1" pb "go.viam.com/api/component/camera/v1" streampb "go.viam.com/api/stream/v1" goutils "go.viam.com/utils" @@ -32,6 +33,7 @@ import ( "go.viam.com/rdk/resource" "go.viam.com/rdk/rimage" "go.viam.com/rdk/rimage/transform" + "go.viam.com/rdk/spatialmath" "go.viam.com/rdk/utils" ) @@ -324,6 +326,21 @@ func (c *client) DoCommand(ctx context.Context, cmd map[string]interface{}) (map return protoutils.DoFromResourceClient(ctx, c.client, c.name, cmd) } +func (c *client) Geometries(ctx context.Context, extra map[string]interface{}) ([]spatialmath.Geometry, error) { + ext, err := goprotoutils.StructToStructPb(extra) + if err != nil { + return nil, err + } + resp, err := c.client.GetGeometries(ctx, &commonpb.GetGeometriesRequest{ + Name: c.name, + Extra: ext, + }) + if err != nil { + return nil, err + } + return spatialmath.NewGeometriesFromProto(resp.GetGeometries()) +} + // TODO(RSDK-6433): This method can be called more than once during a client's lifecycle. // For example, consider a case where a remote camera goes offline and then back online. // We will call `Close` on the camera client when we detect the disconnection to remove diff --git a/components/camera/client_test.go b/components/camera/client_test.go index e3be957754a..8ab3bb8fec9 100644 --- a/components/camera/client_test.go +++ b/components/camera/client_test.go @@ -11,6 +11,7 @@ import ( "testing" "time" + "github.com/golang/geo/r3" "github.com/pion/rtp" "go.viam.com/test" "go.viam.com/utils/rpc" @@ -35,6 +36,7 @@ import ( robotimpl "go.viam.com/rdk/robot/impl" "go.viam.com/rdk/robot/web" weboptions "go.viam.com/rdk/robot/web/options" + "go.viam.com/rdk/spatialmath" "go.viam.com/rdk/testutils" "go.viam.com/rdk/testutils/inject" "go.viam.com/rdk/testutils/robottestutils" @@ -51,6 +53,7 @@ func TestClient(t *testing.T) { injectCamera := &inject.Camera{} img := image.NewNRGBA(image.Rect(0, 0, 4, 4)) + expectedGeometries := []spatialmath.Geometry{spatialmath.NewPoint(r3.Vector{1, 2, 3}, "")} var imgBuf bytes.Buffer test.That(t, png.Encode(&imgBuf, img), test.ShouldBeNil) @@ -105,6 +108,9 @@ func TestClient(t *testing.T) { test.That(t, err, test.ShouldBeNil) return resBytes, camera.ImageMetadata{MimeType: mimeType}, nil } + injectCamera.GeometriesFunc = func(context.Context, map[string]interface{}) ([]spatialmath.Geometry, error) { + return expectedGeometries, nil + } // depth camera injectCameraDepth := &inject.Camera{} depthImg := rimage.NewEmptyDepthMap(10, 20) @@ -221,6 +227,13 @@ func TestClient(t *testing.T) { test.That(t, camera1Client.Close(context.Background()), test.ShouldBeNil) test.That(t, conn.Close(), test.ShouldBeNil) + + // Geometries + geometries, err := camera1Client.Geometries(context.Background(), map[string]interface{}{"foo": "Geometries"}) + test.That(t, err, test.ShouldBeNil) + for i, geometry := range geometries { + test.That(t, spatialmath.GeometriesAlmostEqual(expectedGeometries[i], geometry), test.ShouldBeTrue) + } }) t.Run("camera client depth", func(t *testing.T) { conn, err := viamgrpc.Dial(context.Background(), listener1.Addr().String(), logger) @@ -801,7 +814,7 @@ func TestMultiplexOverMultiHopRemoteConnection(t *testing.T) { test.That(t, cameraClient.(rtppassthrough.Source).Unsubscribe(mainCtx, sub.ID), test.ShouldBeNil) } -//nolint +// nolint // NOTE: These tests fail when this condition occurs: // // logger.go:130: 2024-06-17T16:56:14.097-0400 DEBUG TestGrandRemoteRebooting.remote-1.rdk:remote:/remote-2.webrtc rpc/wrtc_client_channel.go:299 no stream for id; discarding {"ch": 0, "id": 11} diff --git a/components/camera/replaypcd/replaypcd.go b/components/camera/replaypcd/replaypcd.go index a873bf446f1..1df154e88be 100644 --- a/components/camera/replaypcd/replaypcd.go +++ b/components/camera/replaypcd/replaypcd.go @@ -23,6 +23,7 @@ import ( "go.viam.com/rdk/logging" "go.viam.com/rdk/pointcloud" "go.viam.com/rdk/resource" + "go.viam.com/rdk/spatialmath" "go.viam.com/rdk/utils/contextutils" ) @@ -350,6 +351,10 @@ func (replay *pcdCamera) Image(ctx context.Context, mimeType string, extra map[s return nil, camera.ImageMetadata{}, errors.New("Image is unimplemented") } +func (replay *pcdCamera) Geometries(ctx context.Context, extra map[string]interface{}) ([]spatialmath.Geometry, error) { + return make([]spatialmath.Geometry, 0), nil +} + // Close stops replay camera, closes the channels and its connections to the cloud. func (replay *pcdCamera) Close(ctx context.Context) error { replay.mu.Lock() diff --git a/components/camera/server.go b/components/camera/server.go index a27408866b9..ae9fcd382ed 100644 --- a/components/camera/server.go +++ b/components/camera/server.go @@ -17,6 +17,7 @@ import ( "go.viam.com/rdk/protoutils" "go.viam.com/rdk/resource" "go.viam.com/rdk/rimage" + "go.viam.com/rdk/spatialmath" "go.viam.com/rdk/utils" ) @@ -274,3 +275,15 @@ func (s *serviceServer) DoCommand(ctx context.Context, } return protoutils.DoFromResourceServer(ctx, camera, req) } + +func (s *serviceServer) GetGeometries(ctx context.Context, req *commonpb.GetGeometriesRequest) (*commonpb.GetGeometriesResponse, error) { + res, err := s.coll.Resource(req.GetName()) + if err != nil { + return nil, err + } + geometries, err := res.Geometries(ctx, req.Extra.AsMap()) + if err != nil { + return nil, err + } + return &commonpb.GetGeometriesResponse{Geometries: spatialmath.NewGeometriesToProto(geometries)}, nil +} diff --git a/components/camera/videosource/webcam.go b/components/camera/videosource/webcam.go index d6c83d8494d..c7e337bc200 100644 --- a/components/camera/videosource/webcam.go +++ b/components/camera/videosource/webcam.go @@ -28,6 +28,7 @@ import ( "go.viam.com/rdk/rimage" "go.viam.com/rdk/rimage/depthadapter" "go.viam.com/rdk/rimage/transform" + "go.viam.com/rdk/spatialmath" "go.viam.com/rdk/utils" ) @@ -477,6 +478,10 @@ func (c *webcam) Properties(ctx context.Context) (camera.Properties, error) { }, nil } +func (c *webcam) Geometries(ctx context.Context, extra map[string]interface{}) ([]spatialmath.Geometry, error) { + return make([]spatialmath.Geometry, 0), nil +} + func (c *webcam) Close(ctx context.Context) error { c.mu.Lock() if c.closed { diff --git a/components/camera/videosourcewrappers.go b/components/camera/videosourcewrappers.go index b0d06970549..7cd92950c18 100644 --- a/components/camera/videosourcewrappers.go +++ b/components/camera/videosourcewrappers.go @@ -16,6 +16,7 @@ import ( "go.viam.com/rdk/rimage" "go.viam.com/rdk/rimage/depthadapter" "go.viam.com/rdk/rimage/transform" + "go.viam.com/rdk/spatialmath" "go.viam.com/rdk/utils" ) @@ -317,3 +318,10 @@ func (vs *videoSource) Close(ctx context.Context) error { } return vs.videoSource.Close(ctx) } + +func (vs *videoSource) Geometries(ctx context.Context, extra map[string]interface{}) ([]spatialmath.Geometry, error) { + if res, ok := vs.actualSource.(resource.Shaped); ok { + return res.Geometries(ctx, extra) + } + return nil, errors.New("videoSource: geomtries unavailable") +} diff --git a/testutils/inject/camera.go b/testutils/inject/camera.go index c724e2cfd64..45ca8838664 100644 --- a/testutils/inject/camera.go +++ b/testutils/inject/camera.go @@ -10,6 +10,7 @@ import ( "go.viam.com/rdk/pointcloud" "go.viam.com/rdk/resource" "go.viam.com/rdk/rimage/transform" + "go.viam.com/rdk/spatialmath" ) // Camera is an injected camera. @@ -24,6 +25,7 @@ type Camera struct { ProjectorFunc func(ctx context.Context) (transform.Projector, error) PropertiesFunc func(ctx context.Context) (camera.Properties, error) CloseFunc func(ctx context.Context) error + GeometriesFunc func(context.Context, map[string]interface{}) ([]spatialmath.Geometry, error) } // NewCamera returns a new injected camera. @@ -98,6 +100,14 @@ func (c *Camera) DoCommand(ctx context.Context, cmd map[string]interface{}) (map return c.Camera.DoCommand(ctx, cmd) } +// Geometries calls the injected Geometries or the real version. +func (c *Camera) Geometries(ctx context.Context, cmd map[string]interface{}) ([]spatialmath.Geometry, error) { + if c.GeometriesFunc != nil { + return c.GeometriesFunc(ctx, cmd) + } + return c.Camera.Geometries(ctx, cmd) +} + // SubscribeRTP calls the injected RTPPassthroughSource or returns an error if unimplemented. func (c *Camera) SubscribeRTP( ctx context.Context, From 8079ccbcb961361bba1a96e3bc2ba46da8651d4e Mon Sep 17 00:00:00 2001 From: Naveed Jooma Date: Mon, 23 Jun 2025 14:28:21 -0400 Subject: [PATCH 2/5] Fix lint --- components/camera/client_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/camera/client_test.go b/components/camera/client_test.go index 8ab3bb8fec9..4b156a8e476 100644 --- a/components/camera/client_test.go +++ b/components/camera/client_test.go @@ -814,7 +814,7 @@ func TestMultiplexOverMultiHopRemoteConnection(t *testing.T) { test.That(t, cameraClient.(rtppassthrough.Source).Unsubscribe(mainCtx, sub.ID), test.ShouldBeNil) } -// nolint +//nolint // NOTE: These tests fail when this condition occurs: // // logger.go:130: 2024-06-17T16:56:14.097-0400 DEBUG TestGrandRemoteRebooting.remote-1.rdk:remote:/remote-2.webrtc rpc/wrtc_client_channel.go:299 no stream for id; discarding {"ch": 0, "id": 11} From 5837e610d831dacc11fed9668a83f7ecca47d572 Mon Sep 17 00:00:00 2001 From: Naveed Jooma Date: Mon, 23 Jun 2025 14:51:05 -0400 Subject: [PATCH 3/5] Fix tests --- components/camera/client_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/components/camera/client_test.go b/components/camera/client_test.go index 4b156a8e476..28e1cbfb025 100644 --- a/components/camera/client_test.go +++ b/components/camera/client_test.go @@ -225,15 +225,15 @@ func TestClient(t *testing.T) { test.That(t, resp["command"], test.ShouldEqual, testutils.TestCommand["command"]) test.That(t, resp["data"], test.ShouldEqual, testutils.TestCommand["data"]) - test.That(t, camera1Client.Close(context.Background()), test.ShouldBeNil) - test.That(t, conn.Close(), test.ShouldBeNil) - // Geometries geometries, err := camera1Client.Geometries(context.Background(), map[string]interface{}{"foo": "Geometries"}) test.That(t, err, test.ShouldBeNil) for i, geometry := range geometries { test.That(t, spatialmath.GeometriesAlmostEqual(expectedGeometries[i], geometry), test.ShouldBeTrue) } + + test.That(t, camera1Client.Close(context.Background()), test.ShouldBeNil) + test.That(t, conn.Close(), test.ShouldBeNil) }) t.Run("camera client depth", func(t *testing.T) { conn, err := viamgrpc.Dial(context.Background(), listener1.Addr().String(), logger) @@ -814,7 +814,7 @@ func TestMultiplexOverMultiHopRemoteConnection(t *testing.T) { test.That(t, cameraClient.(rtppassthrough.Source).Unsubscribe(mainCtx, sub.ID), test.ShouldBeNil) } -//nolint +// nolint // NOTE: These tests fail when this condition occurs: // // logger.go:130: 2024-06-17T16:56:14.097-0400 DEBUG TestGrandRemoteRebooting.remote-1.rdk:remote:/remote-2.webrtc rpc/wrtc_client_channel.go:299 no stream for id; discarding {"ch": 0, "id": 11} From bd67ac03011e5fda1dda5e677137f9f35eb6d369 Mon Sep 17 00:00:00 2001 From: Naveed Jooma Date: Mon, 23 Jun 2025 15:00:05 -0400 Subject: [PATCH 4/5] Fix lint --- components/camera/client_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/camera/client_test.go b/components/camera/client_test.go index 28e1cbfb025..d1d25d4558c 100644 --- a/components/camera/client_test.go +++ b/components/camera/client_test.go @@ -814,7 +814,7 @@ func TestMultiplexOverMultiHopRemoteConnection(t *testing.T) { test.That(t, cameraClient.(rtppassthrough.Source).Unsubscribe(mainCtx, sub.ID), test.ShouldBeNil) } -// nolint +//nolint // NOTE: These tests fail when this condition occurs: // // logger.go:130: 2024-06-17T16:56:14.097-0400 DEBUG TestGrandRemoteRebooting.remote-1.rdk:remote:/remote-2.webrtc rpc/wrtc_client_channel.go:299 no stream for id; discarding {"ch": 0, "id": 11} From 5343cf896fa006df56ba811907517f9e48cb6164 Mon Sep 17 00:00:00 2001 From: Naveed Jooma Date: Tue, 24 Jun 2025 10:48:44 -0400 Subject: [PATCH 5/5] Update components/camera/videosourcewrappers.go --- components/camera/videosourcewrappers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/camera/videosourcewrappers.go b/components/camera/videosourcewrappers.go index 7cd92950c18..be3d306620d 100644 --- a/components/camera/videosourcewrappers.go +++ b/components/camera/videosourcewrappers.go @@ -323,5 +323,5 @@ func (vs *videoSource) Geometries(ctx context.Context, extra map[string]interfac if res, ok := vs.actualSource.(resource.Shaped); ok { return res.Geometries(ctx, extra) } - return nil, errors.New("videoSource: geomtries unavailable") + return nil, errors.New("videoSource: geometries unavailable") }