Skip to content

Commit 202cece

Browse files
hexbabehexbabe
committed
wip
1 parent ecc20fc commit 202cece

File tree

13 files changed

+211
-49
lines changed

13 files changed

+211
-49
lines changed

‎components/camera/camera.go‎

Lines changed: 62 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -75,8 +75,58 @@ type Properties struct {
7575

7676
// NamedImage is a struct that associates the source from where the image came from to the Image.
7777
type NamedImage struct {
78-
Image image.Image
78+
data []byte
79+
img image.Image
7980
SourceName string
81+
mimeType string
82+
}
83+
84+
func NamedImageFromBytes(data []byte, sourceName string, mimeType string) (NamedImage, error) {
85+
if data == nil {
86+
return NamedImage{}, fmt.Errorf("must provide image bytes to construct a named image from bytes")
87+
}
88+
if mimeType == "" {
89+
return NamedImage{}, fmt.Errorf("must provide a mime type to construct a named image")
90+
}
91+
return NamedImage{data: data, SourceName: sourceName, mimeType: mimeType}, nil
92+
}
93+
94+
func NamedImageFromImage(img image.Image, sourceName string, mimeType string) (NamedImage, error) {
95+
if img == nil {
96+
return NamedImage{}, fmt.Errorf("must provide image to construct a named image")
97+
}
98+
if mimeType == "" {
99+
return NamedImage{}, fmt.Errorf("must provide a mime type to construct a named image")
100+
}
101+
return NamedImage{img: img, SourceName: sourceName, mimeType: mimeType}, nil
102+
}
103+
104+
func (ni *NamedImage) Image(ctx context.Context) (image.Image, error) {
105+
if ni.img == nil {
106+
if ni.data == nil {
107+
return nil, fmt.Errorf("no image or image bytes available")
108+
}
109+
img, err := rimage.DecodeImage(ctx, ni.data, ni.mimeType)
110+
if err != nil {
111+
return nil, fmt.Errorf("could not decode into image.Image: %w", err)
112+
}
113+
ni.img = img
114+
}
115+
return ni.img, nil
116+
}
117+
118+
func (ni *NamedImage) Bytes(ctx context.Context) ([]byte, error) {
119+
if ni.data == nil {
120+
if ni.img == nil {
121+
return nil, fmt.Errorf("no image or image bytes available")
122+
}
123+
data, err := rimage.EncodeImage(ctx, ni.img, ni.mimeType)
124+
if err != nil {
125+
return nil, fmt.Errorf("could not encode image: %w", err)
126+
}
127+
ni.data = data
128+
}
129+
return ni.data, nil
80130
}
81131

82132
// ImageMetadata contains useful information about returned image bytes such as its mimetype.
@@ -191,11 +241,17 @@ func GetImageFromGetImages(ctx context.Context, sourceName *string, mimeType str
191241

192242
var img image.Image
193243
if sourceName == nil {
194-
img = images[0].Image
244+
img, err = images[0].Image(ctx)
245+
if err != nil {
246+
return nil, ImageMetadata{}, fmt.Errorf("could not get image from camera: %w", err)
247+
}
195248
} else {
196249
for _, i := range images {
197250
if i.SourceName == *sourceName {
198-
img = i.Image
251+
img, err = i.Image(ctx)
252+
if err != nil {
253+
return nil, ImageMetadata{}, fmt.Errorf("could not get image from camera: %w", err)
254+
}
199255
break
200256
}
201257
}
@@ -241,12 +297,12 @@ func GetImagesFromGetImage(
241297
logger.Warnf("requested mime type %s, but received %s", mimeType, resMimetype)
242298
}
243299

244-
img, err := rimage.DecodeImage(ctx, resBytes, utils.WithLazyMIMEType(resMetadata.MimeType))
300+
namedImg, err := NamedImageFromBytes(resBytes, "", resMetadata.MimeType)
245301
if err != nil {
246-
return nil, resource.ResponseMetadata{}, fmt.Errorf("could not decode into image.Image: %w", err)
302+
return nil, resource.ResponseMetadata{}, fmt.Errorf("could not create named image: %w", err)
247303
}
248304

249-
return []NamedImage{{Image: img, SourceName: ""}}, resource.ResponseMetadata{CapturedAt: time.Now()}, nil
305+
return []NamedImage{namedImg}, resource.ResponseMetadata{CapturedAt: time.Now()}, nil
250306
}
251307

252308
// VideoSource is a camera that has `Stream` embedded to directly integrate with gostream.

‎components/camera/camera_test.go‎

Lines changed: 35 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -248,9 +248,11 @@ func TestCameraWithProjector(t *testing.T) {
248248
images, _, err := videoSrc2.Images(context.Background())
249249
test.That(t, err, test.ShouldBeNil)
250250
test.That(t, len(images), test.ShouldEqual, 1)
251-
test.That(t, images[0].Image, test.ShouldHaveSameTypeAs, &rimage.DepthMap{})
252-
test.That(t, images[0].Image.Bounds().Dx(), test.ShouldEqual, 1280)
253-
test.That(t, images[0].Image.Bounds().Dy(), test.ShouldEqual, 720)
251+
imgFromImages, err := images[0].Image(context.Background())
252+
test.That(t, err, test.ShouldBeNil)
253+
test.That(t, imgFromImages, test.ShouldHaveSameTypeAs, &rimage.DepthMap{})
254+
test.That(t, imgFromImages.Bounds().Dx(), test.ShouldEqual, 1280)
255+
test.That(t, imgFromImages.Bounds().Dy(), test.ShouldEqual, 720)
254256

255257
test.That(t, cam2.Close(context.Background()), test.ShouldBeNil)
256258
}
@@ -289,16 +291,28 @@ func TestGetImageFromGetImages(t *testing.T) {
289291

290292
rgbaCam := inject.NewCamera("rgba_cam")
291293
rgbaCam.ImagesFunc = func(ctx context.Context) ([]camera.NamedImage, resource.ResponseMetadata, error) {
294+
namedImg1, err := camera.NamedImageFromImage(testImg1, source1Name, rutils.MimeTypeRawRGBA)
295+
if err != nil {
296+
return nil, resource.ResponseMetadata{}, err
297+
}
298+
namedImg2, err := camera.NamedImageFromImage(testImg2, source2Name, rutils.MimeTypeRawRGBA)
299+
if err != nil {
300+
return nil, resource.ResponseMetadata{}, err
301+
}
292302
return []camera.NamedImage{
293-
{Image: testImg1, SourceName: source1Name},
294-
{Image: testImg2, SourceName: source2Name},
303+
namedImg1,
304+
namedImg2,
295305
}, resource.ResponseMetadata{CapturedAt: time.Now()}, nil
296306
}
297307

298308
dm := rimage.NewEmptyDepthMap(100, 100)
299309
depthCam := inject.NewCamera("depth_cam")
300310
depthCam.ImagesFunc = func(ctx context.Context) ([]camera.NamedImage, resource.ResponseMetadata, error) {
301-
return []camera.NamedImage{{Image: dm, SourceName: source1Name}}, resource.ResponseMetadata{CapturedAt: time.Now()}, nil
311+
namedImg, err := camera.NamedImageFromImage(dm, source1Name, rutils.MimeTypeRawDepth)
312+
if err != nil {
313+
return nil, resource.ResponseMetadata{}, err
314+
}
315+
return []camera.NamedImage{namedImg}, resource.ResponseMetadata{CapturedAt: time.Now()}, nil
302316
}
303317

304318
t.Run("PNG mime type", func(t *testing.T) {
@@ -364,10 +378,14 @@ func TestGetImageFromGetImages(t *testing.T) {
364378
t.Run("nil image case", func(t *testing.T) {
365379
nilImageCam := inject.NewCamera("nil_image_cam")
366380
nilImageCam.ImagesFunc = func(ctx context.Context) ([]camera.NamedImage, resource.ResponseMetadata, error) {
367-
return []camera.NamedImage{{Image: nil, SourceName: source1Name}}, resource.ResponseMetadata{CapturedAt: time.Now()}, nil
381+
namedImg, err := camera.NamedImageFromImage(nil, source1Name, rutils.MimeTypeRawRGBA)
382+
if err != nil {
383+
return nil, resource.ResponseMetadata{}, err
384+
}
385+
return []camera.NamedImage{namedImg}, resource.ResponseMetadata{CapturedAt: time.Now()}, nil
368386
}
369387
_, _, err := camera.GetImageFromGetImages(context.Background(), nil, rutils.MimeTypePNG, nilImageCam)
370-
test.That(t, err, test.ShouldBeError, errors.New("image is nil"))
388+
test.That(t, err, test.ShouldBeError, errors.New("could not get images from camera: must provide image to construct a named image"))
371389
})
372390

373391
t.Run("multiple images, specify source name", func(t *testing.T) {
@@ -399,7 +417,9 @@ func TestGetImagesFromGetImage(t *testing.T) {
399417
test.That(t, err, test.ShouldBeNil)
400418
test.That(t, len(images), test.ShouldEqual, 1)
401419
test.That(t, images[0].SourceName, test.ShouldEqual, "")
402-
verifyImageEquality(t, images[0].Image, testImg)
420+
img, err := images[0].Image(context.Background())
421+
test.That(t, err, test.ShouldBeNil)
422+
verifyImageEquality(t, img, testImg)
403423
test.That(t, metadata.CapturedAt.IsZero(), test.ShouldBeFalse)
404424
test.That(t, metadata.CapturedAt.After(startTime), test.ShouldBeTrue)
405425
test.That(t, metadata.CapturedAt.Before(endTime), test.ShouldBeTrue)
@@ -412,7 +432,9 @@ func TestGetImagesFromGetImage(t *testing.T) {
412432
test.That(t, err, test.ShouldBeNil)
413433
test.That(t, len(images), test.ShouldEqual, 1)
414434
test.That(t, images[0].SourceName, test.ShouldEqual, "")
415-
imgBytes, err := rimage.EncodeImage(context.Background(), images[0].Image, rutils.MimeTypeJPEG)
435+
img, err := images[0].Image(context.Background())
436+
test.That(t, err, test.ShouldBeNil)
437+
imgBytes, err := rimage.EncodeImage(context.Background(), img, rutils.MimeTypeJPEG)
416438
test.That(t, err, test.ShouldBeNil)
417439
verifyDecodedImage(t, imgBytes, rutils.MimeTypeJPEG, testImg)
418440
test.That(t, metadata.CapturedAt.IsZero(), test.ShouldBeFalse)
@@ -439,7 +461,9 @@ func TestGetImagesFromGetImage(t *testing.T) {
439461
test.That(t, metadata.CapturedAt.IsZero(), test.ShouldBeFalse)
440462
test.That(t, metadata.CapturedAt.After(startTime), test.ShouldBeTrue)
441463
test.That(t, metadata.CapturedAt.Before(endTime), test.ShouldBeTrue)
442-
verifyImageEquality(t, images[0].Image, rgbaImg) // we should ignore the requested mime type and get back an RGBA image
464+
img, err := images[0].Image(context.Background())
465+
test.That(t, err, test.ShouldBeNil)
466+
verifyImageEquality(t, img, rgbaImg) // we should ignore the requested mime type and get back an RGBA image
443467
})
444468

445469
t.Run("error case", func(t *testing.T) {

‎components/camera/client.go‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -238,7 +238,12 @@ func (c *client) Images(ctx context.Context) ([]NamedImage, resource.ResponseMet
238238
return nil, resource.ResponseMetadata{}, err
239239
}
240240
}
241-
images = append(images, NamedImage{rdkImage, img.SourceName})
241+
// TODO(hexbabe): After updating proto we should have two logical branches to handle mime_type vs format
242+
namedImg, err := NamedImageFromImage(rdkImage, img.SourceName, img.Format.String())
243+
if err != nil {
244+
return nil, resource.ResponseMetadata{}, err
245+
}
246+
images = append(images, namedImg)
242247
}
243248
return images, resource.ResponseMetadataFromProto(resp.ResponseMetadata), nil
244249
}

‎components/camera/client_test.go‎

Lines changed: 23 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -92,10 +92,18 @@ func TestClient(t *testing.T) {
9292
images := []camera.NamedImage{}
9393
// one color image
9494
color := rimage.NewImage(40, 50)
95-
images = append(images, camera.NamedImage{color, "color"})
95+
namedImgColor, err := camera.NamedImageFromImage(color, "color", rutils.MimeTypeRawRGBA)
96+
if err != nil {
97+
return nil, resource.ResponseMetadata{}, err
98+
}
99+
images = append(images, namedImgColor)
96100
// one depth image
97101
depth := rimage.NewEmptyDepthMap(10, 20)
98-
images = append(images, camera.NamedImage{depth, "depth"})
102+
namedImgDepth, err := camera.NamedImageFromImage(depth, "depth", rutils.MimeTypeRawDepth)
103+
if err != nil {
104+
return nil, resource.ResponseMetadata{}, err
105+
}
106+
images = append(images, namedImgDepth)
99107
// a timestamp of 12345
100108
ts := time.UnixMilli(12345)
101109
return images, resource.ResponseMetadata{CapturedAt: ts}, nil
@@ -209,15 +217,19 @@ func TestClient(t *testing.T) {
209217
test.That(t, meta.CapturedAt, test.ShouldEqual, time.UnixMilli(12345))
210218
test.That(t, len(images), test.ShouldEqual, 2)
211219
test.That(t, images[0].SourceName, test.ShouldEqual, "color")
212-
test.That(t, images[0].Image.Bounds().Dx(), test.ShouldEqual, 40)
213-
test.That(t, images[0].Image.Bounds().Dy(), test.ShouldEqual, 50)
214-
test.That(t, images[0].Image, test.ShouldHaveSameTypeAs, &rimage.LazyEncodedImage{})
215-
test.That(t, images[0].Image.ColorModel(), test.ShouldHaveSameTypeAs, color.RGBAModel)
220+
img, err := images[0].Image(context.Background())
221+
test.That(t, err, test.ShouldBeNil)
222+
test.That(t, img.Bounds().Dx(), test.ShouldEqual, 40)
223+
test.That(t, img.Bounds().Dy(), test.ShouldEqual, 50)
224+
test.That(t, img, test.ShouldHaveSameTypeAs, &rimage.LazyEncodedImage{})
225+
test.That(t, img.ColorModel(), test.ShouldHaveSameTypeAs, color.RGBAModel)
216226
test.That(t, images[1].SourceName, test.ShouldEqual, "depth")
217-
test.That(t, images[1].Image.Bounds().Dx(), test.ShouldEqual, 10)
218-
test.That(t, images[1].Image.Bounds().Dy(), test.ShouldEqual, 20)
219-
test.That(t, images[1].Image, test.ShouldHaveSameTypeAs, &rimage.LazyEncodedImage{})
220-
test.That(t, images[1].Image.ColorModel(), test.ShouldHaveSameTypeAs, color.Gray16Model)
227+
img, err = images[1].Image(context.Background())
228+
test.That(t, err, test.ShouldBeNil)
229+
test.That(t, img.Bounds().Dx(), test.ShouldEqual, 10)
230+
test.That(t, img.Bounds().Dy(), test.ShouldEqual, 20)
231+
test.That(t, img, test.ShouldHaveSameTypeAs, &rimage.LazyEncodedImage{})
232+
test.That(t, img.ColorModel(), test.ShouldHaveSameTypeAs, color.Gray16Model)
221233

222234
// Do
223235
resp, err := camera1Client.DoCommand(context.Background(), testutils.TestCommand)
@@ -814,7 +826,7 @@ func TestMultiplexOverMultiHopRemoteConnection(t *testing.T) {
814826
test.That(t, cameraClient.(rtppassthrough.Source).Unsubscribe(mainCtx, sub.ID), test.ShouldBeNil)
815827
}
816828

817-
//nolint
829+
// nolint
818830
// NOTE: These tests fail when this condition occurs:
819831
//
820832
// logger.go:130: 2024-06-17T16:56:14.097-0400 DEBUG TestGrandRemoteRebooting.remote-1.rdk:remote:/remote-2.webrtc rpc/wrtc_client_channel.go:299 no stream for id; discarding {"ch": 0, "id": 11}

‎components/camera/collectors.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -158,7 +158,11 @@ func newGetImagesCollector(resource interface{}, params data.CollectorParams) (d
158158

159159
var binaries []data.Binary
160160
for _, img := range resImgs {
161-
format, imgBytes, err := encodeImageFromUnderlyingType(ctx, img.Image)
161+
namedImg, err := img.Image(ctx)
162+
if err != nil {
163+
return res, err
164+
}
165+
format, imgBytes, err := encodeImageFromUnderlyingType(ctx, namedImg)
162166
if err != nil {
163167
return res, err
164168
}

‎components/camera/collectors_test.go‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -198,10 +198,15 @@ func newCamera(
198198
}
199199

200200
v.ImagesFunc = func(ctx context.Context) ([]camera.NamedImage, resource.ResponseMetadata, error) {
201-
return []camera.NamedImage{
202-
{Image: left, SourceName: "left"},
203-
{Image: right, SourceName: "right"},
204-
},
201+
leftImg, err := camera.NamedImageFromImage(left, "left", utils.MimeTypeJPEG)
202+
if err != nil {
203+
return nil, resource.ResponseMetadata{}, err
204+
}
205+
rightImg, err := camera.NamedImageFromImage(right, "right", utils.MimeTypeJPEG)
206+
if err != nil {
207+
return nil, resource.ResponseMetadata{}, err
208+
}
209+
return []camera.NamedImage{leftImg, rightImg},
205210
resource.ResponseMetadata{CapturedAt: time.Now()},
206211
nil
207212
}

‎components/camera/fake/image_file.go‎

Lines changed: 26 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717
"go.viam.com/rdk/rimage"
1818
"go.viam.com/rdk/rimage/depthadapter"
1919
"go.viam.com/rdk/rimage/transform"
20+
"go.viam.com/rdk/utils"
2021
)
2122

2223
var fileModel = resource.DefaultModelFamily.WithModel("image_file")
@@ -161,23 +162,35 @@ func (fs *fileSource) Images(ctx context.Context) ([]camera.NamedImage, resource
161162
if err != nil {
162163
return nil, resource.ResponseMetadata{}, err
163164
}
164-
imgs = append(imgs, camera.NamedImage{img, "preloaded"})
165+
namedImg, err := camera.NamedImageFromImage(img, "preloaded", utils.MimeTypeJPEG)
166+
if err != nil {
167+
return nil, resource.ResponseMetadata{}, err
168+
}
169+
imgs = append(imgs, namedImg)
165170
}
166171

167172
if fs.ColorFN != "" {
168173
img, err := rimage.ReadImageFromFile(fs.ColorFN)
169174
if err != nil {
170175
return nil, resource.ResponseMetadata{}, err
171176
}
172-
imgs = append(imgs, camera.NamedImage{img, "color"})
177+
namedImg, err := camera.NamedImageFromImage(img, "color", utils.MimeTypeJPEG)
178+
if err != nil {
179+
return nil, resource.ResponseMetadata{}, err
180+
}
181+
imgs = append(imgs, namedImg)
173182
}
174183

175184
if fs.DepthFN != "" {
176185
dm, err := rimage.NewDepthMapFromFile(context.Background(), fs.DepthFN)
177186
if err != nil {
178187
return nil, resource.ResponseMetadata{}, err
179188
}
180-
imgs = append(imgs, camera.NamedImage{dm, "depth"})
189+
namedImg, err := camera.NamedImageFromImage(dm, "depth", utils.MimeTypeRawDepth)
190+
if err != nil {
191+
return nil, resource.ResponseMetadata{}, err
192+
}
193+
imgs = append(imgs, namedImg)
181194
}
182195

183196
ts := time.Now()
@@ -237,10 +250,18 @@ func (ss *StaticSource) Images(ctx context.Context) ([]camera.NamedImage, resour
237250
}
238251
imgs := []camera.NamedImage{}
239252
if ss.ColorImg != nil {
240-
imgs = append(imgs, camera.NamedImage{ss.ColorImg, "color"})
253+
namedImg, err := camera.NamedImageFromImage(ss.ColorImg, "color", utils.MimeTypeJPEG)
254+
if err != nil {
255+
return nil, resource.ResponseMetadata{}, err
256+
}
257+
imgs = append(imgs, namedImg)
241258
}
242259
if ss.DepthImg != nil {
243-
imgs = append(imgs, camera.NamedImage{ss.DepthImg, "depth"})
260+
namedImg, err := camera.NamedImageFromImage(ss.DepthImg, "depth", utils.MimeTypeRawDepth)
261+
if err != nil {
262+
return nil, resource.ResponseMetadata{}, err
263+
}
264+
imgs = append(imgs, namedImg)
244265
}
245266
ts := time.Now()
246267
return imgs, resource.ResponseMetadata{CapturedAt: ts}, nil

‎components/camera/server.go‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -114,13 +114,17 @@ func (s *serviceServer) GetImages(
114114
}
115115
imagesMessage := make([]*pb.Image, 0, len(imgs))
116116
for _, img := range imgs {
117-
format, outBytes, err := encodeImageFromUnderlyingType(ctx, img.Image)
117+
namedImg, err := img.Image(ctx)
118+
if err != nil {
119+
return nil, errors.Wrap(err, "camera server GetImages could not get the image")
120+
}
121+
format, outBytes, err := encodeImageFromUnderlyingType(ctx, namedImg)
118122
if err != nil {
119123
return nil, errors.Wrap(err, "camera server GetImages could not encode the images")
120124
}
121125
imgMes := &pb.Image{
122126
SourceName: img.SourceName,
123-
Format: format,
127+
Format: format, // TODO(hexbabe): After updating proto we should have two logical branches to handle mime_type vs format
124128
Image: outBytes,
125129
}
126130
imagesMessage = append(imagesMessage, imgMes)

0 commit comments

Comments
 (0)