Update GetImages method and use local proto replacement

hexbabe · hexbabe · commit d038ca2c081e · 2025-07-31T11:13:25.000-04:00
diff --git a/components/camera/camera.go b/components/camera/camera.go
@@ -133,6 +133,11 @@ func (ni *NamedImage) Bytes(ctx context.Context) ([]byte, error) {
 	return ni.data, nil
 }
 
+// MimeType returns the mime type of the NamedImage.
+func (ni *NamedImage) MimeType() string {
+	return ni.mimeType
+}
+
 // ImageMetadata contains useful information about returned image bytes such as its mimetype.
 type ImageMetadata struct {
 	MimeType string
@@ -193,7 +198,7 @@ type Camera interface {
 
 	// Images is used for getting simultaneous images from different imagers,
 	// along with associated metadata (just timestamp for now). It's not for getting a time series of images from the same imager.
-	Images(ctx context.Context) ([]NamedImage, resource.ResponseMetadata, error)
+	Images(ctx context.Context, filterSourceNames []string, extra map[string]interface{}) ([]NamedImage, resource.ResponseMetadata, error)
 
 	// NextPointCloud returns the next immediately available point cloud, not necessarily one
 	// a part of a sequence. In the future, there could be streaming of point clouds.
@@ -230,7 +235,7 @@ func DecodeImageFromCamera(ctx context.Context, mimeType string, extra map[strin
 // It uses the mimeType arg to specify how to encode the bytes returned from GetImages.
 func GetImageFromGetImages(ctx context.Context, sourceName *string, mimeType string, cam Camera) ([]byte, ImageMetadata, error) {
 	// TODO(RSDK-10991): pass through extra field when implemented
-	images, _, err := cam.Images(ctx)
+	images, _, err := cam.Images(ctx, nil, nil)
 	if err != nil {
 		return nil, ImageMetadata{}, fmt.Errorf("could not get images from camera: %w", err)
 	}
@@ -247,14 +252,14 @@ func GetImageFromGetImages(ctx context.Context, sourceName *string, mimeType str
 	if sourceName == nil {
 		img, err = images[0].Image(ctx)
 		if err != nil {
-			return nil, ImageMetadata{}, fmt.Errorf("could not get image from camera: %w", err)
+			return nil, ImageMetadata{}, fmt.Errorf("could not get image from named image: %w", err)
 		}
 	} else {
 		for _, i := range images {
 			if i.SourceName == *sourceName {
 				img, err = i.Image(ctx)
 				if err != nil {
-					return nil, ImageMetadata{}, fmt.Errorf("could not get image from camera: %w", err)
+					return nil, ImageMetadata{}, fmt.Errorf("could not get image from named image: %w", err)
 				}
 				break
 			}
@@ -328,7 +333,7 @@ type PointCloudSource interface {
 
 // A ImagesSource is a source that can return a list of images with timestamp.
 type ImagesSource interface {
-	Images(ctx context.Context) ([]NamedImage, resource.ResponseMetadata, error)
+	Images(ctx context.Context, filterSourceNames []string, extra map[string]interface{}) ([]NamedImage, resource.ResponseMetadata, error)
 }
 
 // NewPropertiesError returns an error specific to a failure in Properties.
diff --git a/components/camera/camera_test.go b/components/camera/camera_test.go
@@ -243,7 +243,7 @@ func TestCameraWithProjector(t *testing.T) {
 	test.That(t, img.Bounds().Dx(), test.ShouldEqual, 1280)
 	test.That(t, img.Bounds().Dy(), test.ShouldEqual, 720)
 	// cam2 should implement a default GetImages, that just returns the one image
-	images, _, err := videoSrc2.Images(context.Background())
+	images, _, err := videoSrc2.Images(context.Background(), nil, nil)
 	test.That(t, err, test.ShouldBeNil)
 	test.That(t, len(images), test.ShouldEqual, 1)
 	imgFromImages, err := images[0].Image(context.Background())
@@ -288,7 +288,11 @@ func TestGetImageFromGetImages(t *testing.T) {
 	testImg2 := image.NewRGBA(image.Rect(0, 0, 200, 200))
 
 	rgbaCam := inject.NewCamera("rgba_cam")
-	rgbaCam.ImagesFunc = func(ctx context.Context) ([]camera.NamedImage, resource.ResponseMetadata, error) {
+	rgbaCam.ImagesFunc = func(
+		ctx context.Context,
+		filterSourceNames []string,
+		extra map[string]interface{},
+	) ([]camera.NamedImage, resource.ResponseMetadata, error) {
 		namedImg1, err := camera.NamedImageFromImage(testImg1, source1Name, rutils.MimeTypeRawRGBA)
 		if err != nil {
 			return nil, resource.ResponseMetadata{}, err
@@ -305,7 +309,11 @@ func TestGetImageFromGetImages(t *testing.T) {
 
 	dm := rimage.NewEmptyDepthMap(100, 100)
 	depthCam := inject.NewCamera("depth_cam")
-	depthCam.ImagesFunc = func(ctx context.Context) ([]camera.NamedImage, resource.ResponseMetadata, error) {
+	depthCam.ImagesFunc = func(
+		ctx context.Context,
+		filterSourceNames []string,
+		extra map[string]interface{},
+	) ([]camera.NamedImage, resource.ResponseMetadata, error) {
 		namedImg, err := camera.NamedImageFromImage(dm, source1Name, rutils.MimeTypeRawDepth)
 		if err != nil {
 			return nil, resource.ResponseMetadata{}, err
@@ -357,7 +365,11 @@ func TestGetImageFromGetImages(t *testing.T) {
 
 	t.Run("error case", func(t *testing.T) {
 		errorCam := inject.NewCamera("error_cam")
-		errorCam.ImagesFunc = func(ctx context.Context) ([]camera.NamedImage, resource.ResponseMetadata, error) {
+		errorCam.ImagesFunc = func(
+			ctx context.Context,
+			filterSourceNames []string,
+			extra map[string]interface{},
+		) ([]camera.NamedImage, resource.ResponseMetadata, error) {
 			return nil, resource.ResponseMetadata{}, errors.New("test error")
 		}
 		_, _, err := camera.GetImageFromGetImages(context.Background(), nil, rutils.MimeTypePNG, errorCam)
@@ -366,7 +378,11 @@ func TestGetImageFromGetImages(t *testing.T) {
 
 	t.Run("empty images case", func(t *testing.T) {
 		emptyCam := inject.NewCamera("empty_cam")
-		emptyCam.ImagesFunc = func(ctx context.Context) ([]camera.NamedImage, resource.ResponseMetadata, error) {
+		emptyCam.ImagesFunc = func(
+			ctx context.Context,
+			filterSourceNames []string,
+			extra map[string]interface{},
+		) ([]camera.NamedImage, resource.ResponseMetadata, error) {
 			return []camera.NamedImage{}, resource.ResponseMetadata{CapturedAt: time.Now()}, nil
 		}
 		_, _, err := camera.GetImageFromGetImages(context.Background(), nil, rutils.MimeTypePNG, emptyCam)
@@ -375,7 +391,11 @@ func TestGetImageFromGetImages(t *testing.T) {
 
 	t.Run("nil image case", func(t *testing.T) {
 		nilImageCam := inject.NewCamera("nil_image_cam")
-		nilImageCam.ImagesFunc = func(ctx context.Context) ([]camera.NamedImage, resource.ResponseMetadata, error) {
+		nilImageCam.ImagesFunc = func(
+			ctx context.Context,
+			filterSourceNames []string,
+			extra map[string]interface{},
+		) ([]camera.NamedImage, resource.ResponseMetadata, error) {
 			namedImg, err := camera.NamedImageFromImage(nil, source1Name, rutils.MimeTypeRawRGBA)
 			if err != nil {
 				return nil, resource.ResponseMetadata{}, err
diff --git a/components/camera/client.go b/components/camera/client.go
@@ -208,12 +208,22 @@ func (c *client) Image(ctx context.Context, mimeType string, extra map[string]in
 	return resp.Image, ImageMetadata{MimeType: resp.MimeType}, nil
 }
 
-func (c *client) Images(ctx context.Context) ([]NamedImage, resource.ResponseMetadata, error) {
+func (c *client) Images(
+	ctx context.Context,
+	filterSourceNames []string,
+	extra map[string]interface{},
+) ([]NamedImage, resource.ResponseMetadata, error) {
 	ctx, span := trace.StartSpan(ctx, "camera::client::Images")
 	defer span.End()
 
+	convertedExtra, err := goprotoutils.StructToStructPb(extra)
+	if err != nil {
+		return nil, resource.ResponseMetadata{}, err
+	}
 	resp, err := c.client.GetImages(ctx, &pb.GetImagesRequest{
-		Name: c.name,
+		Name:              c.name,
+		FilterSourceNames: filterSourceNames,
+		Extra:             convertedExtra,
 	})
 	if err != nil {
 		return nil, resource.ResponseMetadata{}, fmt.Errorf("camera client: could not gets images from the camera %w", err)
diff --git a/components/camera/client_test.go b/components/camera/client_test.go
@@ -88,7 +88,11 @@ func TestClient(t *testing.T) {
 	injectCamera.ProjectorFunc = func(ctx context.Context) (transform.Projector, error) {
 		return projA, nil
 	}
-	injectCamera.ImagesFunc = func(ctx context.Context) ([]camera.NamedImage, resource.ResponseMetadata, error) {
+	injectCamera.ImagesFunc = func(
+		ctx context.Context,
+		filterSourceNames []string,
+		extra map[string]interface{},
+	) ([]camera.NamedImage, resource.ResponseMetadata, error) {
 		images := []camera.NamedImage{}
 		// one color image
 		color := rimage.NewImage(40, 50)
@@ -212,7 +216,7 @@ func TestClient(t *testing.T) {
 		test.That(t, propsB.SupportsPCD, test.ShouldBeTrue)
 		test.That(t, propsB.IntrinsicParams, test.ShouldResemble, intrinsics)
 
-		images, meta, err := camera1Client.Images(context.Background())
+		images, meta, err := camera1Client.Images(context.Background(), nil, nil)
 		test.That(t, err, test.ShouldBeNil)
 		test.That(t, meta.CapturedAt, test.ShouldEqual, time.UnixMilli(12345))
 		test.That(t, len(images), test.ShouldEqual, 2)
@@ -735,7 +739,7 @@ func TestMultiplexOverRemoteConnection(t *testing.T) {
 	cameraClient, err := camera.FromRobot(mainRobot, "remote:rtpPassthroughCamera")
 	test.That(t, err, test.ShouldBeNil)
 
-	image, _, err := cameraClient.Images(mainCtx)
+	image, _, err := cameraClient.Images(mainCtx, nil, nil)
 	test.That(t, err, test.ShouldBeNil)
 	test.That(t, image, test.ShouldNotBeNil)
 	logger.Info("got images")
@@ -808,7 +812,7 @@ func TestMultiplexOverMultiHopRemoteConnection(t *testing.T) {
 	cameraClient, err := camera.FromRobot(mainRobot, "remote-1:remote-2:rtpPassthroughCamera")
 	test.That(t, err, test.ShouldBeNil)
 
-	image, _, err := cameraClient.Images(mainCtx)
+	image, _, err := cameraClient.Images(mainCtx, nil, nil)
 	test.That(t, err, test.ShouldBeNil)
 	test.That(t, image, test.ShouldNotBeNil)
 	logger.Info("got images")
@@ -892,7 +896,7 @@ func TestWhyMustTimeoutOnReadRTP(t *testing.T) {
 	cameraClient, err := camera.FromRobot(mainRobot, "remote-1:remote-2:rtpPassthroughCamera")
 	test.That(t, err, test.ShouldBeNil)
 
-	image, _, err := cameraClient.Images(mainCtx)
+	image, _, err := cameraClient.Images(mainCtx, nil, nil)
 	test.That(t, err, test.ShouldBeNil)
 	test.That(t, image, test.ShouldNotBeNil)
 	logger.Info("got images")
@@ -1028,7 +1032,7 @@ func TestGrandRemoteRebooting(t *testing.T) {
 	mainCameraClient, err := camera.FromRobot(mainRobot, "remote-1:remote-2:rtpPassthroughCamera")
 	test.That(t, err, test.ShouldBeNil)
 
-	image, _, err := mainCameraClient.Images(mainCtx)
+	image, _, err := mainCameraClient.Images(mainCtx, nil, nil)
 	test.That(t, err, test.ShouldBeNil)
 	test.That(t, image, test.ShouldNotBeNil)
 	logger.Info("got images")
diff --git a/components/camera/collectors.go b/components/camera/collectors.go
@@ -148,7 +148,7 @@ func newGetImagesCollector(resource interface{}, params data.CollectorParams) (d
 		defer span.End()
 		ctx = context.WithValue(ctx, data.FromDMContextKey{}, true)
 
-		resImgs, resMetadata, err := camera.Images(ctx)
+		resImgs, resMetadata, err := camera.Images(ctx, nil, nil)
 		if err != nil {
 			if errors.Is(err, data.ErrNoCaptureToStore) {
 				return res, err
diff --git a/components/camera/collectors_test.go b/components/camera/collectors_test.go
@@ -197,7 +197,11 @@ func newCamera(
 		return pcd, nil
 	}
 
-	v.ImagesFunc = func(ctx context.Context) ([]camera.NamedImage, resource.ResponseMetadata, error) {
+	v.ImagesFunc = func(
+		ctx context.Context,
+		filterSourceNames []string,
+		extra map[string]interface{},
+	) ([]camera.NamedImage, resource.ResponseMetadata, error) {
 		leftImg, err := camera.NamedImageFromImage(left, "left", utils.MimeTypeJPEG)
 		if err != nil {
 			return nil, resource.ResponseMetadata{}, err
diff --git a/components/camera/fake/image_file_test.go b/components/camera/fake/image_file_test.go
@@ -134,7 +134,7 @@ func TestPreloadedImages(t *testing.T) {
 			test.That(t, bounds.Dx() > 0, test.ShouldBeTrue)
 			test.That(t, bounds.Dy() > 0, test.ShouldBeTrue)
 
-			namedImages, metadata, err := cam.Images(ctx)
+			namedImages, metadata, err := cam.Images(ctx, nil, nil)
 			test.That(t, err, test.ShouldBeNil)
 			test.That(t, len(namedImages), test.ShouldEqual, 1)
 			test.That(t, namedImages[0].SourceName, test.ShouldEqual, "preloaded")
@@ -159,7 +159,7 @@ func TestPreloadedImages(t *testing.T) {
 	test.That(t, err, test.ShouldBeNil)
 
 	// Should return both images
-	namedImages, _, err := cam.Images(ctx)
+	namedImages, _, err := cam.Images(ctx, nil, nil)
 	test.That(t, err, test.ShouldBeNil)
 	test.That(t, len(namedImages), test.ShouldEqual, 2)
 	test.That(t, namedImages[0].SourceName, test.ShouldEqual, "preloaded")
diff --git a/components/camera/replaypcd/replaypcd.go b/components/camera/replaypcd/replaypcd.go
@@ -329,7 +329,11 @@ func addGRPCMetadata(ctx context.Context, timeRequested, timeReceived *timestamp
 }
 
 // Images is a part of the camera interface but is not implemented for replay.
-func (replay *pcdCamera) Images(ctx context.Context) ([]camera.NamedImage, resource.ResponseMetadata, error) {
+func (replay *pcdCamera) Images(
+	ctx context.Context,
+	filterSourceNames []string,
+	extra map[string]interface{},
+) ([]camera.NamedImage, resource.ResponseMetadata, error) {
 	return nil, resource.ResponseMetadata{}, errors.New("Images is unimplemented")
 }
 
diff --git a/components/camera/server.go b/components/camera/server.go
@@ -108,7 +108,7 @@ func (s *serviceServer) GetImages(
 	}
 	// request the images, and then check to see what the underlying type is to determine
 	// what to encode as. If it's color, just encode as JPEG.
-	imgs, metadata, err := cam.Images(ctx)
+	imgs, metadata, err := cam.Images(ctx, req.FilterSourceNames, req.Extra.AsMap())
 	if err != nil {
 		return nil, errors.Wrap(err, "camera server GetImages could not call Images on the camera")
 	}
diff --git a/components/camera/server_test.go b/components/camera/server_test.go
@@ -93,7 +93,11 @@ func TestServer(t *testing.T) {
 			FrameRate:       float32(10.0),
 		}, nil
 	}
-	injectCamera.ImagesFunc = func(ctx context.Context) ([]camera.NamedImage, resource.ResponseMetadata, error) {
+	injectCamera.ImagesFunc = func(
+		ctx context.Context,
+		filterSourceNames []string,
+		extra map[string]interface{},
+	) ([]camera.NamedImage, resource.ResponseMetadata, error) {
 		images := []camera.NamedImage{}
 		// one color image
 		color := rimage.NewImage(40, 50)
diff --git a/components/camera/videosource/webcam.go b/components/camera/videosource/webcam.go
@@ -388,7 +388,11 @@ func (c *webcam) Monitor() {
 	})
 }
 
-func (c *webcam) Images(ctx context.Context) ([]camera.NamedImage, resource.ResponseMetadata, error) {
+func (c *webcam) Images(
+	ctx context.Context,
+	filterSourceNames []string,
+	extra map[string]interface{},
+) ([]camera.NamedImage, resource.ResponseMetadata, error) {
 	c.mu.RLock()
 	defer c.mu.RUnlock()
 	if err := c.ensureActive(); err != nil {
diff --git a/components/camera/videosourcewrappers.go b/components/camera/videosourcewrappers.go
@@ -235,11 +235,15 @@ func (vs *videoSource) Image(ctx context.Context, mimeType string, extra map[str
 // Images is for getting simultaneous images from different sensors
 // If the underlying source did not specify an Images function, a default is applied.
 // The default returns a list of 1 image from ReadImage, and the current time.
-func (vs *videoSource) Images(ctx context.Context) ([]NamedImage, resource.ResponseMetadata, error) {
+func (vs *videoSource) Images(
+	ctx context.Context,
+	filterSourceNames []string,
+	extra map[string]interface{},
+) ([]NamedImage, resource.ResponseMetadata, error) {
 	ctx, span := trace.StartSpan(ctx, "camera::videoSource::Images")
 	defer span.End()
 	if c, ok := vs.actualSource.(ImagesSource); ok {
-		return c.Images(ctx)
+		return c.Images(ctx, filterSourceNames, extra)
 	}
 	img, release, err := ReadImage(ctx, vs.videoSource)
 	if err != nil {
diff --git a/go.mod b/go.mod
@@ -447,3 +447,5 @@ require (
 	github.com/ziutek/mymysql v1.5.4 // indirect
 	golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e
 )
+
+replace go.viam.com/api => ../sean-api
diff --git a/go.sum b/go.sum
@@ -1518,8 +1518,6 @@ go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
 go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY=
 go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
 go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
-go.viam.com/api v0.1.455 h1:WY++sFAydRGcg/E1va85BFPi9U7IuEf2Dk0JrPZ1Wzs=
-go.viam.com/api v0.1.455/go.mod h1:gwJriv6EVWe97uFzzzWjzP3NPfpCrKtRAdWtYglUpqs=
 go.viam.com/test v1.2.4 h1:JYgZhsuGAQ8sL9jWkziAXN9VJJiKbjoi9BsO33TW3ug=
 go.viam.com/test v1.2.4/go.mod h1:zI2xzosHdqXAJ/kFqcN+OIF78kQuTV2nIhGZ8EzvaJI=
 go.viam.com/utils v0.1.153 h1:p775C6WmfOgf0gGRuvlcWDRuNURYiSzzv20I86tIhlE=
diff --git a/robot/web/stream/camera/camera.go b/robot/web/stream/camera/camera.go
diff --git a/robot/web/stream/server.go b/robot/web/stream/server.go
diff --git a/robot/web/stream/stream_test.go b/robot/web/stream/stream_test.go
diff --git a/testutils/inject/camera.go b/testutils/inject/camera.go

Original file line number	Diff line number	Diff line change
`@@ -329,7 +329,11 @@ func addGRPCMetadata(ctx context.Context, timeRequested, timeReceived *timestamp`
`329`	`329`	`}`
`330`	`330`
`331`	`331`	`// Images is a part of the camera interface but is not implemented for replay.`
`332`		`-func (replay *pcdCamera) Images(ctx context.Context) ([]camera.NamedImage, resource.ResponseMetadata, error) {`
	`332`	`+func (replay *pcdCamera) Images(`
	`333`	`+ ctx context.Context,`
	`334`	`+ filterSourceNames []string,`
	`335`	`+ extra map[string]interface{},`
	`336`	`+) ([]camera.NamedImage, resource.ResponseMetadata, error) {`
`333`	`337`	`return nil, resource.ResponseMetadata{}, errors.New("Images is unimplemented")`
`334`	`338`	`}`
`335`	`339`
Original file line number	Diff line number	Diff line change
`@@ -108,7 +108,7 @@ func (s *serviceServer) GetImages(`
`108`	`108`	`}`
`109`	`109`	`// request the images, and then check to see what the underlying type is to determine`
`110`	`110`	`// what to encode as. If it's color, just encode as JPEG.`
`111`		`- imgs, metadata, err := cam.Images(ctx)`
	`111`	`+ imgs, metadata, err := cam.Images(ctx, req.FilterSourceNames, req.Extra.AsMap())`
`112`	`112`	`if err != nil {`
`113`	`113`	`return nil, errors.Wrap(err, "camera server GetImages could not call Images on the camera")`
`114`	`114`	`}`