RSDK-7532 - Adding getProperties to vision service (viamrobotics#3925)

go.mod

@@ -85,7 +85,7 @@ require (
 	go.uber.org/atomic v1.10.0
 	go.uber.org/multierr v1.11.0
 	go.uber.org/zap v1.24.0
-	go.viam.com/api v0.1.293
+	go.viam.com/api v0.1.296
 	go.viam.com/test v1.1.1-0.20220913152726-5da9916c08a2
 	go.viam.com/utils v0.1.77
 	goji.io v2.0.2+incompatible

go.sum

@@ -1540,8 +1540,8 @@ go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
 go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY=
 go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
 go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
-go.viam.com/api v0.1.293 h1:BO82qY1mWOZbIjK9kMUyoRv/apdy7O76RT7BNPH29Jo=
-go.viam.com/api v0.1.293/go.mod h1:msa4TPrMVeRDcG4YzKA/S6wLEUC7GyHQE973JklrQ10=
+go.viam.com/api v0.1.296 h1:CuGF7IVLUmVn5cvWvmGuF7TCviZ7iYJKBqABxcb8G4M=
+go.viam.com/api v0.1.296/go.mod h1:msa4TPrMVeRDcG4YzKA/S6wLEUC7GyHQE973JklrQ10=
 go.viam.com/test v1.1.1-0.20220913152726-5da9916c08a2 h1:oBiK580EnEIzgFLU4lHOXmGAE3MxnVbeR7s1wp/F3Ps=
 go.viam.com/test v1.1.1-0.20220913152726-5da9916c08a2/go.mod h1:XM0tej6riszsiNLT16uoyq1YjuYPWlRBweTPRDanIts=
 go.viam.com/utils v0.1.77 h1:eI2BzUxf2kILSqPT5GbWw605Z26gKAiJ7wGSqx8OfCw=

services/vision/client.go

@@ -242,6 +242,26 @@ func protoToObjects(pco []*commonpb.PointCloudObject) ([]*vision.Object, error)
 	return objects, nil
 }
 
+func (c *client) GetProperties(ctx context.Context, extra map[string]interface{}) (*Properties, error) {
+	ctx, span := trace.StartSpan(ctx, "service::vision::client::GetProperties")
+	defer span.End()
+
+	ext, err := protoutils.StructToStructPb(extra)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := c.client.GetProperties(ctx, &pb.GetPropertiesRequest{
+		Name:  c.name,
+		Extra: ext,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return &Properties{resp.ClassificationsSupported, resp.DetectionsSupported, resp.ObjectPointCloudsSupported}, nil
+}
+
 func (c *client) DoCommand(ctx context.Context, cmd map[string]interface{}) (map[string]interface{}, error) {
 	ctx, span := trace.StartSpan(ctx, "service::vision::client::DoCommand")
 	defer span.End()

services/vision/client_test.go

@@ -41,6 +41,9 @@ func TestClient(t *testing.T) {
 		det1 := objectdetection.NewDetection(image.Rect(0, 0, 10, 20), 0.8, "camera")
 		return []objectdetection.Detection{det1}, nil
 	}
+	srv.GetPropertiesFunc = func(ctx context.Context, extra map[string]interface{}) (*vision.Properties, error) {
+		return &vision.Properties{ClassificationSupported: false, DetectionSupported: true, ObjectPCDsSupported: false}, nil
+	}
 	test.That(t, err, test.ShouldBeNil)
 	m := map[resource.Name]vision.Service{
 		vision.Named(testVisionServiceName): srv,
@@ -100,6 +103,24 @@ func TestClient(t *testing.T) {
 		test.That(t, client.Close(context.Background()), test.ShouldBeNil)
 		test.That(t, conn.Close(), test.ShouldBeNil)
 	})
+
+	t.Run("get properties", func(t *testing.T) {
+		conn, err := viamgrpc.Dial(context.Background(), listener1.Addr().String(), logger)
+		test.That(t, err, test.ShouldBeNil)
+		client, err := vision.NewClientFromConn(context.Background(), conn, "", vision.Named(testVisionServiceName), logger)
+		test.That(t, err, test.ShouldBeNil)
+
+		props, err := client.GetProperties(context.Background(), nil)
+		test.That(t, err, test.ShouldBeNil)
+
+		test.That(t, props, test.ShouldNotBeNil)
+		test.That(t, props.ClassificationSupported, test.ShouldEqual, false)
+		test.That(t, props.DetectionSupported, test.ShouldEqual, true)
+		test.That(t, props.ObjectPCDsSupported, test.ShouldEqual, false)
+
+		test.That(t, client.Close(context.Background()), test.ShouldBeNil)
+		test.That(t, conn.Close(), test.ShouldBeNil)
+	})
 }
 
 func TestInjectedServiceClient(t *testing.T) {

services/vision/colordetector/color_detector_test.go

@@ -28,6 +28,13 @@ func TestColorDetector(t *testing.T) {
 	img, err := rimage.NewImageFromFile(artifact.MustPath("vision/objectdetection/detection_test.jpg"))
 	test.That(t, err, test.ShouldBeNil)
 
+	// Test properties. Should support detections and not classifications or object PCDs
+	props, err := srv.GetProperties(ctx, nil)
+	test.That(t, err, test.ShouldBeNil)
+	test.That(t, props.DetectionSupported, test.ShouldEqual, true)
+	test.That(t, props.ClassificationSupported, test.ShouldEqual, false)
+	test.That(t, props.ObjectPCDsSupported, test.ShouldEqual, false)
+
 	// Does implement Detections
 	det, err := srv.Detections(ctx, img, nil)
 	test.That(t, err, test.ShouldBeNil)

services/vision/obstaclesdepth/obstacles_depth_test.go

@@ -101,6 +101,13 @@ func TestObstacleDepth(t *testing.T) {
 	test.That(t, err, test.ShouldBeNil)
 	test.That(t, srv.Name(), test.ShouldResemble, name)
 
+	// Test properties. Should support object PCDs and not detections or classifications
+	props, err := srv.GetProperties(ctx, nil)
+	test.That(t, err, test.ShouldBeNil)
+	test.That(t, props.ObjectPCDsSupported, test.ShouldEqual, true)
+	test.That(t, props.DetectionSupported, test.ShouldEqual, false)
+	test.That(t, props.ClassificationSupported, test.ShouldEqual, false)
+
 	// Not a detector or classifier
 	img, err := rimage.NewImageFromFile(artifact.MustPath("vision/objectdetection/detection_test.jpg"))
 	test.That(t, err, test.ShouldBeNil)

services/vision/obstaclesdistance/obstacles_distance_test.go

@@ -55,6 +55,13 @@ func TestObstacleDist(t *testing.T) {
 	img, err := rimage.NewImageFromFile(artifact.MustPath("vision/objectdetection/detection_test.jpg"))
 	test.That(t, err, test.ShouldBeNil)
 
+	// Test properties. Should support object PCDs and not detections or classifications
+	props, err := srv.GetProperties(context.Background(), nil)
+	test.That(t, err, test.ShouldBeNil)
+	test.That(t, props.ObjectPCDsSupported, test.ShouldEqual, true)
+	test.That(t, props.DetectionSupported, test.ShouldEqual, false)
+	test.That(t, props.ClassificationSupported, test.ShouldEqual, false)
+
 	// Does not implement Detections
 	_, err = srv.Detections(ctx, img, nil)
 	test.That(t, err, test.ShouldNotBeNil)

services/vision/obstaclespointcloud/obstacles_pointcloud_test.go

@@ -59,6 +59,13 @@ func TestRadiusClusteringSegmentation(t *testing.T) {
 	test.That(t, err, test.ShouldBeNil)
 	test.That(t, seg.Name(), test.ShouldResemble, name)
 
+	// Test properties. Should support object PCDs and not detections or classifications
+	props, err := seg.GetProperties(context.Background(), nil)
+	test.That(t, err, test.ShouldBeNil)
+	test.That(t, props.ObjectPCDsSupported, test.ShouldEqual, true)
+	test.That(t, props.DetectionSupported, test.ShouldEqual, false)
+	test.That(t, props.ClassificationSupported, test.ShouldEqual, false)
+
 	// fails on not finding camera
 	_, err = seg.GetObjectPointClouds(context.Background(), "no_camera", map[string]interface{}{})
 	test.That(t, err, test.ShouldNotBeNil)

services/vision/server.go

@@ -217,6 +217,28 @@ func segmentsToProto(frame string, segs []*vision.Object) ([]*commonpb.PointClou
 	return protoSegs, nil
 }
 
+func (server *serviceServer) GetProperties(ctx context.Context,
+	req *pb.GetPropertiesRequest,
+) (*pb.GetPropertiesResponse, error) {
+	ctx, span := trace.StartSpan(ctx, "service::vision::server::GetProperties")
+	defer span.End()
+	svc, err := server.coll.Resource(req.Name)
+	if err != nil {
+		return nil, err
+	}
+	props, err := svc.GetProperties(ctx, req.Extra.AsMap())
+	if err != nil {
+		return nil, err
+	}
+
+	out := &pb.GetPropertiesResponse{
+		ClassificationsSupported:   props.ClassificationSupported,
+		DetectionsSupported:        props.DetectionSupported,
+		ObjectPointCloudsSupported: props.ObjectPCDsSupported,
+	}
+	return out, nil
+}
+
 // DoCommand receives arbitrary commands.
 func (server *serviceServer) DoCommand(ctx context.Context,
 	req *commonpb.DoCommandRequest,

services/vision/server_test.go

@@ -95,3 +95,29 @@ func TestServerGetDetections(t *testing.T) {
 	test.That(t, len(resp.GetDetections()), test.ShouldEqual, 1)
 	test.That(t, resp.GetDetections()[0].GetClassName(), test.ShouldEqual, "yes")
 }
+
+func TestServerGetProperties(t *testing.T) {
+	injectVS := &inject.VisionService{}
+	injectVS.GetPropertiesFunc = func(ctx context.Context, extra map[string]interface{}) (*vision.Properties, error) {
+		return &vision.Properties{ClassificationSupported: false, DetectionSupported: true, ObjectPCDsSupported: false}, nil
+	}
+	m := map[resource.Name]vision.Service{
+		visName1: injectVS,
+	}
+	server, err := newServer(m)
+	test.That(t, err, test.ShouldBeNil)
+
+	extra := map[string]interface{}{}
+	ext, err := protoutils.StructToStructPb(extra)
+	propsRequest := &pb.GetPropertiesRequest{
+		Name:  testVisionServiceName,
+		Extra: ext,
+	}
+	test.That(t, err, test.ShouldBeNil)
+
+	resp, err := server.GetProperties(context.Background(), propsRequest)
+	test.That(t, err, test.ShouldBeNil)
+	test.That(t, resp.ClassificationsSupported, test.ShouldEqual, false)
+	test.That(t, resp.DetectionsSupported, test.ShouldEqual, true)
+	test.That(t, resp.ObjectPointCloudsSupported, test.ShouldEqual, false)
+}

services/vision/vision.go

@@ -128,6 +128,8 @@ type Service interface {
 
 	// GetObjectPointClouds returns a list of 3D point cloud objects and metadata from the latest 3D camera image using a specified segmenter.
 	GetObjectPointClouds(ctx context.Context, cameraName string, extra map[string]interface{}) ([]*viz.Object, error)
+	// properties
+	GetProperties(ctx context.Context, extra map[string]interface{}) (*Properties, error)
 }
 
 // SubtypeName is the name of the type of service.
@@ -155,13 +157,22 @@ func FromDependencies(deps resource.Dependencies, name string) (Service, error)
 type vizModel struct {
 	resource.Named
 	resource.AlwaysRebuild
-	r               robot.Robot                     // in order to get access to all cameras
+	r               robot.Robot // in order to get access to all cameras
+	properties      Properties
 	closerFunc      func(ctx context.Context) error // close the underlying model
 	classifierFunc  classification.Classifier
 	detectorFunc    objectdetection.Detector
 	segmenter3DFunc segmentation.Segmenter
 }
 
+// Properties returns various information regarding the current vision service,
+// specifically, which vision tasks are supported by the resource.
+type Properties struct {
+	ClassificationSupported bool
+	DetectionSupported      bool
+	ObjectPCDsSupported     bool
+}
+
 // NewService wraps the vision model in the struct that fulfills the vision service interface.
 func NewService(
 	name resource.Name,
@@ -175,9 +186,22 @@ func NewService(
 		return nil, errors.Errorf(
 			"model %q does not fulfill any method of the vision service. It is neither a detector, nor classifier, nor 3D segmenter", name)
 	}
+
+	p := Properties{false, false, false}
+	if cf != nil {
+		p.ClassificationSupported = true
+	}
+	if df != nil {
+		p.DetectionSupported = true
+	}
+	if s3f != nil {
+		p.ObjectPCDsSupported = true
+	}
+
 	return &vizModel{
 		Named:           name.AsNamed(),
 		r:               r,
+		properties:      p,
 		closerFunc:      c,
 		classifierFunc:  cf,
 		detectorFunc:    df,
@@ -283,6 +307,14 @@ func (vm *vizModel) GetObjectPointClouds(ctx context.Context, cameraName string,
 	return vm.segmenter3DFunc(ctx, cam)
 }
 
+// GetProperties returns a Properties object that details the vision capabilities of the model.
+func (vm *vizModel) GetProperties(ctx context.Context, extra map[string]interface{}) (*Properties, error) {
+	_, span := trace.StartSpan(ctx, "service::vision::GetProperties::"+vm.Named.Name().String())
+	defer span.End()
+
+	return &vm.properties, nil
+}
+
 func (vm *vizModel) Close(ctx context.Context) error {
 	if vm.closerFunc == nil {
 		return nil

testutils/inject/vision_service.go

@@ -28,6 +28,7 @@ type VisionService struct {
 		n int, extra map[string]interface{}) (classification.Classifications, error)
 	// segmentation functions
 	GetObjectPointCloudsFunc func(ctx context.Context, cameraName string, extra map[string]interface{}) ([]*viz.Object, error)
+	GetPropertiesFunc        func(ctx context.Context, extra map[string]interface{}) (*vision.Properties, error)
 	DoCommandFunc            func(ctx context.Context,
 		cmd map[string]interface{}) (map[string]interface{}, error)
 	CloseFunc func(ctx context.Context) error
@@ -92,6 +93,17 @@ func (vs *VisionService) GetObjectPointClouds(
 	return vs.GetObjectPointCloudsFunc(ctx, cameraName, extra)
 }
 
+// GetProperties calls the injected GetProperties or the real variant.
+func (vs *VisionService) GetProperties(
+	ctx context.Context,
+	extra map[string]interface{},
+) (*vision.Properties, error) {
+	if vs.GetPropertiesFunc == nil {
+		return vs.Service.GetProperties(ctx, extra)
+	}
+	return vs.GetPropertiesFunc(ctx, extra)
+}
+
 // DoCommand calls the injected DoCommand or the real variant.
 func (vs *VisionService) DoCommand(ctx context.Context,
 	cmd map[string]interface{},