From ca8fe0b40ad1f99ea6644b68e5f1ca04f4c85910 Mon Sep 17 00:00:00 2001 From: Julie Krasnick Date: Mon, 4 Nov 2024 16:56:56 -0500 Subject: [PATCH 01/25] inital changes --- app/data_client.go | 168 +++++++++++++++++++++++++++++++++++++++++++++ go.mod | 6 +- 2 files changed, 171 insertions(+), 3 deletions(-) create mode 100644 app/data_client.go diff --git a/app/data_client.go b/app/data_client.go new file mode 100644 index 00000000000..44b4997f803 --- /dev/null +++ b/app/data_client.go @@ -0,0 +1,168 @@ +//go:build !no_cgo + +// Package arm contains a gRPC based arm client. +package arm + +import ( + "context" + "errors" + "net/url" + + "go.mongodb.org/mongo-driver/bson" + pb "go.viam.com/api/app/data/v1" + "go.viam.com/rdk/logging" + "go.viam.com/utils/rpc" + "google.golang.org/grpc" +) + +// type TabularData struct { +// // Data map[string]interface{} +// // Metadata *pb.CaptureMetadata +// // TimeRequested time.Time +// // TimeReceived time.Time +// //str +// //equals +// } + +// viamClient.dataClient. + +// i want to wrap NewDataServiceClient define a new dataclient struct and call the wrappers of the functions +// i want the user to call my dataClient struct w my wrappers and not the proto functions +type TabularData struct { + //actual requested data ==> Data map[string]interface{} + //metadata associated w/ actual data + //time data were requested + //time data were recieved +} +type DataClient struct { + client pb.DataServiceClient + TabularData TabularData +} + +// NewDataClient initializes the DataClient by calling getDataClient +func NewDataClient( + ctx context.Context, + viamBaseURL, + viamAPIKeyID, + viamAPIKey string, + logger logging.Logger, +) (*DataClient, error) { + u, err := url.Parse(viamBaseURL + ":443") + if err != nil { + return nil, err + } + opts := rpc.WithEntityCredentials( + viamAPIKeyID, + rpc.Credentials{ + Type: rpc.CredentialsTypeAPIKey, + Payload: viamAPIKey, + }, + ) + conn, err := rpc.DialDirectGRPC(ctx, u.Host, logger.AsZap(), opts) + if err != nil { + return nil, err + } + + d := pb.NewDataServiceClient(conn) + return &DataClient{ + client: d, + }, nil +} + +func (d *DataClient) TabularDataByFilter() error { + return errors.New("unimplemented") +} + +// returns an array of data objects +// interface{} is a special type in Go that represents any type. +// so map[string]interface{} is a map (aka a dictionary) where the keys are strings and the values are of any type +// a list of maps --> like we had in python a list of dictionaries +func (d *DataClient) TabularDataBySQL(ctx context.Context, organizationId string, sqlQuery string) ([]map[string]interface{}, error) { + //idk what the return type here is + resp, _ := d.client.TabularDataBySQL(ctx, &pb.TabularDataBySQLRequest{OrganizationId: organizationId, SqlQuery: sqlQuery}) + // return bson.Unmarshal(resp.RawData) + // var dataObjects []map[string]interface{} + // Initialize a slice to hold the data objects + dataObjects := make([]map[string]interface{}, len(resp.RawData)) + // for _, rawData := range resp.RawData { + // var obj map[string]interface{} + // if err := bson.Unmarshal(rawData, &obj); err != nil { + // return nil, fmt.Errorf("failed to unmarshal BSON: %w", err) + // } + // dataObjects = append(dataObjects, obj) + // } + // Loop over each BSON byte array in the response and unmarshal directly into the dataObjects slice + for i, rawData := range resp.RawData { + obj := make(map[string]interface{}) + bson.Unmarshal(rawData, &obj) + //do we want an error message for bson.Unmarshal...? + dataObjects[i] = obj + } + return dataObjects, nil + +} + +func (d *DataClient) TabularDataByMQL() error { + return errors.New("unimplemented") +} + +func (d *DataClient) BinaryDataByFilter() error { + return errors.New("unimplemented") +} + +func (d *DataClient) BinaryDataByIDs() error { + return errors.New("unimplemented") +} +func (d *DataClient) DeleteTabularData(ctx context.Context, organizationId string, deleteOlderThanDays uint32) (deletedCount uint64, err error) { + //so many things wrong w this - this does not feel like a safe wrapper? + // grpc.EmptyCallOption() feels wrong! + resp, _ := d.client.DeleteTabularData(ctx, &pb.DeleteTabularDataRequest{OrganizationId: organizationId, DeleteOlderThanDays: deleteOlderThanDays}, grpc.EmptyCallOption{}) + + return resp.DeletedCount, nil +} + +func (d *DataClient) DeleteBinaryDataByFilter() error { + return errors.New("unimplemented") +} +func (d *DataClient) DeleteBinaryDataByIDs() error { + return errors.New("unimplemented") +} +func (d *DataClient) AddTagsToBinaryDataByIDs() error { + return errors.New("unimplemented") +} +func (d *DataClient) AddTagsToBinaryDataByFilter() error { + return errors.New("unimplemented") +} +func (d *DataClient) RemoveTagsFromBinaryDataByIDs() error { + return errors.New("unimplemented") +} +func (d *DataClient) RemoveTagsFromBinaryDataByFilter() error { + return errors.New("unimplemented") +} +func (d *DataClient) TagsByFilter() error { + return errors.New("unimplemented") +} +func (d *DataClient) AddBoundingBoxToImageByID() error { + return errors.New("unimplemented") +} +func (d *DataClient) RemoveBoundingBoxFromImageByID() error { + return errors.New("unimplemented") +} +func (d *DataClient) BoundingBoxLabelsByFilter() error { + return errors.New("unimplemented") +} +func (d *DataClient) UpdateBoundingBox() error { + return errors.New("unimplemented") +} +func (d *DataClient) GetDatabaseConnection() error { + return errors.New("unimplemented") +} +func (d *DataClient) ConfigureDatabaseUser() error { + return errors.New("unimplemented") +} +func (d *DataClient) AddBinaryDataToDatasetByIDs() error { + return errors.New("unimplemented") +} +func (d *DataClient) RemoveBinaryDataFromDatasetByIDs() error { + return errors.New("unimplemented") +} diff --git a/go.mod b/go.mod index b97ebffb1b2..e989bbbe46c 100644 --- a/go.mod +++ b/go.mod @@ -59,7 +59,7 @@ require ( github.com/lucasb-eyer/go-colorful v1.2.0 github.com/matttproud/golang_protobuf_extensions v1.0.4 github.com/mkch/gpio v0.0.0-20190919032813-8327cd97d95e - github.com/montanaflynn/stats v0.7.0 + github.com/montanaflynn/stats v0.7.1 github.com/muesli/clusters v0.0.0-20200529215643-2700303c1762 github.com/muesli/kmeans v0.3.1 github.com/nathan-fiscaletti/consolesize-go v0.0.0-20220204101620-317176b6684d @@ -77,7 +77,7 @@ require ( github.com/viamrobotics/webrtc/v3 v3.99.10 github.com/xfmoulet/qoi v0.2.0 go-hep.org/x/hep v0.32.1 - go.mongodb.org/mongo-driver v1.12.2 + go.mongodb.org/mongo-driver v1.17.1 go.opencensus.io v0.24.0 go.uber.org/atomic v1.11.0 go.uber.org/multierr v1.11.0 @@ -401,7 +401,7 @@ require ( github.com/yagipy/maintidx v1.0.0 // indirect github.com/yeya24/promlinter v0.3.0 // indirect github.com/ykadowak/zerologlint v0.1.5 // indirect - github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect + github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect github.com/zitadel/oidc v1.13.4 // indirect gitlab.com/bosi/decorder v0.4.2 // indirect go-simpler.org/musttag v0.12.2 // indirect From 60d77ca17ba213cfb2bc6ff6dbe5d70799234027 Mon Sep 17 00:00:00 2001 From: Julie Krasnick Date: Tue, 5 Nov 2024 15:46:38 -0500 Subject: [PATCH 02/25] added alot of changes to wrapper --- app/data_client.go | 320 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 259 insertions(+), 61 deletions(-) diff --git a/app/data_client.go b/app/data_client.go index 44b4997f803..5e73f81ef90 100644 --- a/app/data_client.go +++ b/app/data_client.go @@ -6,71 +6,225 @@ package arm import ( "context" "errors" - "net/url" + "time" "go.mongodb.org/mongo-driver/bson" pb "go.viam.com/api/app/data/v1" "go.viam.com/rdk/logging" "go.viam.com/utils/rpc" - "google.golang.org/grpc" + "google.golang.org/protobuf/types/known/structpb" ) -// type TabularData struct { -// // Data map[string]interface{} -// // Metadata *pb.CaptureMetadata -// // TimeRequested time.Time -// // TimeReceived time.Time -// //str -// //equals -// } +//protobuf to type or type to protobuf (poseinframe to proto or proto to pose in frame) +//define the structs publically and a private function that does the conversion +//come back to "dest" path later to see if we wanna write to a file or not // viamClient.dataClient. // i want to wrap NewDataServiceClient define a new dataclient struct and call the wrappers of the functions -// i want the user to call my dataClient struct w my wrappers and not the proto functions +// // i want the user to call my dataClient struct w my wrappers and not the proto functions +type CaptureMetadata struct { + organization_id string + location_id2 string + robot_name string + robot_id string + part_name string + part_id string + component_type string + component_name string + method_name string + method_parameters map[string]interface{} + //^^ supposed to be: map method_parameters = 11; + tags []string + //^^ repeated string tags = 12; + mime_type string + //^^ string mime_type = 13; +} + +type BoundingBox struct { + id string + label string + x_min_normalized float64 //should be double but no doubles in go + y_min_normalized float64 + x_max_normalized float64 + y_max_normalized float64 +} + +// Annotations are data annotations used for machine learning. +type Annotations struct { + //supposed to be repeated bounding boxes + bboxes []BoundingBox +} type TabularData struct { - //actual requested data ==> Data map[string]interface{} - //metadata associated w/ actual data - //time data were requested - //time data were recieved + Data map[string]interface{} + // Metadata *pb.CaptureMetadata //idk why i put a star here -- if we aren't returning it is it okay? + Metadata CaptureMetadata //idk why i put a star here + TimeRequested time.Time + TimeReceived time.Time +} +type BinaryData struct { + Binary []byte + Metadata BinaryMetadata +} + +type BinaryMetadata struct { + ID string + //CaptureMetadata *pb.CaptureMetadata + CaptureMetadata CaptureMetadata + TimeRequested time.Time + TimeReceived time.Time + FileName string + FileExt string + URI string + Annotations Annotations + //Annotations *pb.Annotations + DatasetIDs []string +} + +func BoundingBoxFromProto(proto *pb.BoundingBox) BoundingBox { + return BoundingBox{ + id: proto.Id, + label: proto.Label, + x_min_normalized: proto.XMinNormalized, // cast if i want int, or use float64 for precision + y_min_normalized: proto.YMinNormalized, + x_max_normalized: proto.XMaxNormalized, + y_max_normalized: proto.YMaxNormalized, + } } +func AnnotationsFromProto(proto *pb.Annotations) Annotations { + if proto == nil { + return Annotations{} + } + // Convert each BoundingBox from proto to native type + bboxes := make([]BoundingBox, len(proto.Bboxes)) + for i, bboxProto := range proto.Bboxes { + bboxes[i] = BoundingBoxFromProto(bboxProto) + } + return Annotations{ + bboxes: bboxes, + } +} + +func CaptureMetadataFromProto(proto *pb.CaptureMetadata) CaptureMetadata { + if proto == nil { + return CaptureMetadata{} + } + // Convert method parameters from protobuf to native map + methodParameters := make(map[string]interface{}) + // Convert MethodParameters map[string]*anypb.Any to map[string]interface{} + for key, value := range proto.MethodParameters { + structValue := &structpb.Value{} + if err := value.UnmarshalTo(structValue); err != nil { + return CaptureMetadata{} // return error?? + } + methodParameters[key] = structValue.AsInterface() + } + return CaptureMetadata{ + organization_id: proto.OrganizationId, + location_id2: proto.LocationId, + robot_name: proto.RobotName, + robot_id: proto.RobotId, + part_name: proto.PartName, + part_id: proto.PartId, + component_type: proto.ComponentType, + component_name: proto.ComponentName, + method_name: proto.MethodName, + method_parameters: methodParameters, + tags: proto.Tags, // repeated string + mime_type: proto.MimeType, + } +} +func BinaryDataFromProto(proto *pb.BinaryData) BinaryData { + return BinaryData{ + Binary: proto.Binary, + Metadata: BinaryMetadataFromProto(proto.Metadata), + } +} + +func BinaryMetadataFromProto(proto *pb.BinaryMetadata) BinaryMetadata { + return BinaryMetadata{ + ID: proto.Id, + CaptureMetadata: CaptureMetadataFromProto(proto.CaptureMetadata), + TimeRequested: proto.TimeRequested.AsTime(), + TimeReceived: proto.TimeReceived.AsTime(), + FileName: proto.FileName, + FileExt: proto.FileExt, + URI: proto.Uri, + Annotations: AnnotationsFromProto(proto.Annotations), + DatasetIDs: proto.DatasetIds, + } +} + type DataClient struct { + //do we want this to be a public interface that defines the functions but does not include client and private details? + //would not include client and private details client pb.DataServiceClient TabularData TabularData } -// NewDataClient initializes the DataClient by calling getDataClient +// (private) dataClient implements DataServiceClient. **do we want this? +type dataClient interface { + // actual hold implementations of functions - how would the NewDataClient function work if we had private and public functions? + // client pb.DataServiceClient +} + +// NewDataClient constructs a new DataClient from connection passed in. func NewDataClient( ctx context.Context, - viamBaseURL, - viamAPIKeyID, - viamAPIKey string, + channel rpc.ClientConn, //this should just take a channek that the viamClient passes in logger logging.Logger, ) (*DataClient, error) { - u, err := url.Parse(viamBaseURL + ":443") - if err != nil { - return nil, err - } - opts := rpc.WithEntityCredentials( - viamAPIKeyID, - rpc.Credentials{ - Type: rpc.CredentialsTypeAPIKey, - Payload: viamAPIKey, - }, - ) - conn, err := rpc.DialDirectGRPC(ctx, u.Host, logger.AsZap(), opts) - if err != nil { - return nil, err - } - - d := pb.NewDataServiceClient(conn) + d := pb.NewDataServiceClient(channel) return &DataClient{ client: d, }, nil } -func (d *DataClient) TabularDataByFilter() error { - return errors.New("unimplemented") +// TabularDataByFilter queries tabular data and metadata based on given filters. +// ***I dont see anything about a file path here but python has something about it!! +// returns []TabularData, uint64, string, and error: returns multiple things containing the following: List[TabularData]: The tabular data, int: The count (number of entries), str: The last-returned page ID. +func (d *DataClient) TabularDataByFilter( + //include dest? + ctx context.Context, + filter *pb.Filter, + limit uint64, + last string, + sortOrder pb.Order, + countOnly bool, + includeInternalData bool) ([]TabularData, uint64, string, error) { + resp, err := d.client.TabularDataByFilter(ctx, &pb.TabularDataByFilterRequest{ + DataRequest: &pb.DataRequest{ //need to do checks to make sure it fits the constraints + Filter: filter, + Limit: limit, + Last: last, + SortOrder: sortOrder, + }, + CountOnly: countOnly, + IncludeInternalData: includeInternalData, + }) + //do we want this? + if err != nil { + return nil, 0, "", err + } + //need to undo the repeated tabularData in resp.Data and return it + dataArray := make([]TabularData, len(resp.Data)) + for i, data := range resp.Data { + mdIndex := data.MetadataIndex + // var metadata *pb.CaptureMetadata + var metadata CaptureMetadata + //is this check necessary?? + if len(resp.Metadata) != 0 && mdIndex < uint32(len(resp.Metadata)) { + metadata = CaptureMetadataFromProto(resp.Metadata[mdIndex]) + } + //creating a list of tabularData + dataArray[i] = TabularData{ + Data: data.Data.AsMap(), + Metadata: metadata, + TimeRequested: data.TimeRequested.AsTime(), + TimeReceived: data.TimeReceived.AsTime(), + } + } + return dataArray, resp.Count, resp.Last, nil } // returns an array of data objects @@ -78,19 +232,12 @@ func (d *DataClient) TabularDataByFilter() error { // so map[string]interface{} is a map (aka a dictionary) where the keys are strings and the values are of any type // a list of maps --> like we had in python a list of dictionaries func (d *DataClient) TabularDataBySQL(ctx context.Context, organizationId string, sqlQuery string) ([]map[string]interface{}, error) { - //idk what the return type here is - resp, _ := d.client.TabularDataBySQL(ctx, &pb.TabularDataBySQLRequest{OrganizationId: organizationId, SqlQuery: sqlQuery}) - // return bson.Unmarshal(resp.RawData) - // var dataObjects []map[string]interface{} - // Initialize a slice to hold the data objects + resp, err := d.client.TabularDataBySQL(ctx, &pb.TabularDataBySQLRequest{OrganizationId: organizationId, SqlQuery: sqlQuery}) + if err != nil { + return nil, err + } + // Initialize a an array of maps to hold the data objects (in python we had list of dicts) dataObjects := make([]map[string]interface{}, len(resp.RawData)) - // for _, rawData := range resp.RawData { - // var obj map[string]interface{} - // if err := bson.Unmarshal(rawData, &obj); err != nil { - // return nil, fmt.Errorf("failed to unmarshal BSON: %w", err) - // } - // dataObjects = append(dataObjects, obj) - // } // Loop over each BSON byte array in the response and unmarshal directly into the dataObjects slice for i, rawData := range resp.RawData { obj := make(map[string]interface{}) @@ -99,25 +246,76 @@ func (d *DataClient) TabularDataBySQL(ctx context.Context, organizationId string dataObjects[i] = obj } return dataObjects, nil - } -func (d *DataClient) TabularDataByMQL() error { - return errors.New("unimplemented") +func (d *DataClient) TabularDataByMQL(ctx context.Context, organizationId string, mqlbinary [][]byte) ([]map[string]interface{}, error) { + //need to double check this mqlbinary type***?? + resp, err := d.client.TabularDataByMQL(ctx, &pb.TabularDataByMQLRequest{OrganizationId: organizationId, MqlBinary: mqlbinary}) + if err != nil { + return nil, err + } + dataObjects := make([]map[string]interface{}, len(resp.RawData)) + for i, rawData := range resp.RawData { + obj := make(map[string]interface{}) + bson.Unmarshal(rawData, &obj) + dataObjects[i] = obj + } + return dataObjects, nil } -func (d *DataClient) BinaryDataByFilter() error { - return errors.New("unimplemented") +func (d *DataClient) BinaryDataByFilter( + //dest string?? + ctx context.Context, + filter *pb.Filter, + limit uint64, + last string, + sortOrder pb.Order, + includeBinary bool, + countOnly bool, + // includeInternalData bool) ([]*pb.BinaryData, uint64, string, error) { + includeInternalData bool) ([]BinaryData, uint64, string, error) { + resp, err := d.client.BinaryDataByFilter(ctx, &pb.BinaryDataByFilterRequest{ + DataRequest: &pb.DataRequest{ //need to do checks to make sure it fits the constraints + Filter: filter, + Limit: limit, + Last: last, + SortOrder: sortOrder, + }, + CountOnly: countOnly, + IncludeInternalData: includeInternalData, + }) + if err != nil { + return nil, 0, "", err + } + // Convert protobuf BinaryData to Go-native BinaryData + data := make([]BinaryData, len(resp.Data)) + for i, protoData := range resp.Data { + data[i] = BinaryDataFromProto(protoData) + } + // return resp.Data, resp.Count, resp.Last, nil + return data, resp.Count, resp.Last, nil + } -func (d *DataClient) BinaryDataByIDs() error { - return errors.New("unimplemented") +// do i need to be including error as a return type for all of these? +func (d *DataClient) BinaryDataByIDs(ctx context.Context, binaryIds []*pb.BinaryID) ([]BinaryData, error) { + resp, err := d.client.BinaryDataByIDs(ctx, &pb.BinaryDataByIDsRequest{ + IncludeBinary: true, + BinaryIds: binaryIds, + }) + if err != nil { + return nil, err + } + // Convert protobuf BinaryData to Go-native BinaryData + data := make([]BinaryData, len(resp.Data)) + for i, protoData := range resp.Data { + data[i] = BinaryDataFromProto(protoData) + } + // return resp.Data, nil + return data, nil } func (d *DataClient) DeleteTabularData(ctx context.Context, organizationId string, deleteOlderThanDays uint32) (deletedCount uint64, err error) { - //so many things wrong w this - this does not feel like a safe wrapper? - // grpc.EmptyCallOption() feels wrong! - resp, _ := d.client.DeleteTabularData(ctx, &pb.DeleteTabularDataRequest{OrganizationId: organizationId, DeleteOlderThanDays: deleteOlderThanDays}, grpc.EmptyCallOption{}) - + resp, _ := d.client.DeleteTabularData(ctx, &pb.DeleteTabularDataRequest{OrganizationId: organizationId, DeleteOlderThanDays: deleteOlderThanDays}) return resp.DeletedCount, nil } From c7068563d8152a432a320e36da7ab91032c5de72 Mon Sep 17 00:00:00 2001 From: Julie Krasnick Date: Tue, 5 Nov 2024 16:22:12 -0500 Subject: [PATCH 03/25] more changes --- app/data_client.go | 51 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 42 insertions(+), 9 deletions(-) diff --git a/app/data_client.go b/app/data_client.go index 5e73f81ef90..6417e3f9448 100644 --- a/app/data_client.go +++ b/app/data_client.go @@ -181,17 +181,24 @@ func NewDataClient( } // TabularDataByFilter queries tabular data and metadata based on given filters. -// ***I dont see anything about a file path here but python has something about it!! // returns []TabularData, uint64, string, and error: returns multiple things containing the following: List[TabularData]: The tabular data, int: The count (number of entries), str: The last-returned page ID. func (d *DataClient) TabularDataByFilter( //include dest? ctx context.Context, - filter *pb.Filter, - limit uint64, - last string, - sortOrder pb.Order, + filter *pb.Filter, //optional - no filter implies all tabular data + limit uint64, //optional - max defaults to 50 if unspecified + last string, //optional + sortOrder pb.Order, //optional countOnly bool, includeInternalData bool) ([]TabularData, uint64, string, error) { + // initialize limit if it's unspecified (zero value) + if limit == 0 { + limit = 50 + } + // ensure filter is not nil to represent a query for "all data" + if filter == nil { + filter = &pb.Filter{} + } resp, err := d.client.TabularDataByFilter(ctx, &pb.TabularDataByFilterRequest{ DataRequest: &pb.DataRequest{ //need to do checks to make sure it fits the constraints Filter: filter, @@ -213,7 +220,8 @@ func (d *DataClient) TabularDataByFilter( // var metadata *pb.CaptureMetadata var metadata CaptureMetadata //is this check necessary?? - if len(resp.Metadata) != 0 && mdIndex < uint32(len(resp.Metadata)) { + // Ensure the metadata index is within bounds + if len(resp.Metadata) != 0 && int(mdIndex) < len(resp.Metadata) { metadata = CaptureMetadataFromProto(resp.Metadata[mdIndex]) } //creating a list of tabularData @@ -274,6 +282,14 @@ func (d *DataClient) BinaryDataByFilter( countOnly bool, // includeInternalData bool) ([]*pb.BinaryData, uint64, string, error) { includeInternalData bool) ([]BinaryData, uint64, string, error) { + // initialize limit if it's unspecified (zero value) + if limit == 0 { + limit = 50 + } + // ensure filter is not nil to represent a query for "all data" + if filter == nil { + filter = &pb.Filter{} + } resp, err := d.client.BinaryDataByFilter(ctx, &pb.BinaryDataByFilterRequest{ DataRequest: &pb.DataRequest{ //need to do checks to make sure it fits the constraints Filter: filter, @@ -315,12 +331,29 @@ func (d *DataClient) BinaryDataByIDs(ctx context.Context, binaryIds []*pb.Binary return data, nil } func (d *DataClient) DeleteTabularData(ctx context.Context, organizationId string, deleteOlderThanDays uint32) (deletedCount uint64, err error) { - resp, _ := d.client.DeleteTabularData(ctx, &pb.DeleteTabularDataRequest{OrganizationId: organizationId, DeleteOlderThanDays: deleteOlderThanDays}) + resp, err := d.client.DeleteTabularData(ctx, &pb.DeleteTabularDataRequest{ + OrganizationId: organizationId, + DeleteOlderThanDays: deleteOlderThanDays, + }) + if err != nil { + return 0, err + } return resp.DeletedCount, nil } -func (d *DataClient) DeleteBinaryDataByFilter() error { - return errors.New("unimplemented") +func (d *DataClient) DeleteBinaryDataByFilter(ctx context.Context, filter *pb.Filter) (uint64, error) { + //should probably do some sort of check that filter isn't empty otherwise i need to do something + if filter == nil { + filter = &pb.Filter{} + } + resp, err := d.client.DeleteBinaryDataByFilter(ctx, &pb.DeleteBinaryDataByFilterRequest{ + Filter: filter, + IncludeInternalData: true, + }) + if err != nil { + return 0, err + } + return resp.DeletedCount, nil } func (d *DataClient) DeleteBinaryDataByIDs() error { return errors.New("unimplemented") From e6c25cffe9747035d29e3c14450590700a0f0218 Mon Sep 17 00:00:00 2001 From: Julie Krasnick Date: Tue, 5 Nov 2024 17:00:04 -0500 Subject: [PATCH 04/25] more --- app/data_client.go | 61 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 48 insertions(+), 13 deletions(-) diff --git a/app/data_client.go b/app/data_client.go index 6417e3f9448..7be45e38f2b 100644 --- a/app/data_client.go +++ b/app/data_client.go @@ -330,7 +330,7 @@ func (d *DataClient) BinaryDataByIDs(ctx context.Context, binaryIds []*pb.Binary // return resp.Data, nil return data, nil } -func (d *DataClient) DeleteTabularData(ctx context.Context, organizationId string, deleteOlderThanDays uint32) (deletedCount uint64, err error) { +func (d *DataClient) DeleteTabularData(ctx context.Context, organizationId string, deleteOlderThanDays uint32) (uint64, error) { resp, err := d.client.DeleteTabularData(ctx, &pb.DeleteTabularDataRequest{ OrganizationId: organizationId, DeleteOlderThanDays: deleteOlderThanDays, @@ -355,23 +355,58 @@ func (d *DataClient) DeleteBinaryDataByFilter(ctx context.Context, filter *pb.Fi } return resp.DeletedCount, nil } -func (d *DataClient) DeleteBinaryDataByIDs() error { - return errors.New("unimplemented") +func (d *DataClient) DeleteBinaryDataByIDs(ctx context.Context, binaryIds []*pb.BinaryID) (uint64, error) { + resp, err := d.client.DeleteBinaryDataByIDs(ctx, &pb.DeleteBinaryDataByIDsRequest{ + BinaryIds: binaryIds, + }) + if err != nil { + return 0, err + } + return resp.DeletedCount, nil } -func (d *DataClient) AddTagsToBinaryDataByIDs() error { - return errors.New("unimplemented") +func (d *DataClient) AddTagsToBinaryDataByIDs(ctx context.Context, tags []string, binaryIds []*pb.BinaryID) error { + _, err := d.client.AddTagsToBinaryDataByIDs(ctx, &pb.AddTagsToBinaryDataByIDsRequest{BinaryIds: binaryIds, Tags: tags}) + if err != nil { + return err + } + return nil } -func (d *DataClient) AddTagsToBinaryDataByFilter() error { - return errors.New("unimplemented") +func (d *DataClient) AddTagsToBinaryDataByFilter(ctx context.Context, tags []string, filter *pb.Filter) error { + if filter == nil { + filter = &pb.Filter{} + } + _, err := d.client.AddTagsToBinaryDataByFilter(ctx, &pb.AddTagsToBinaryDataByFilterRequest{Filter: filter, Tags: tags}) + if err != nil { + return err + } + return nil } -func (d *DataClient) RemoveTagsFromBinaryDataByIDs() error { - return errors.New("unimplemented") +func (d *DataClient) RemoveTagsFromBinaryDataByIDs(ctx context.Context, tags []string, binaryIds []*pb.BinaryID) (uint64, error) { + resp, err := d.client.RemoveTagsFromBinaryDataByIDs(ctx, &pb.RemoveTagsFromBinaryDataByIDsRequest{BinaryIds: binaryIds, Tags: tags}) + if err != nil { + return 0, err + } + return resp.DeletedCount, nil } -func (d *DataClient) RemoveTagsFromBinaryDataByFilter() error { - return errors.New("unimplemented") +func (d *DataClient) RemoveTagsFromBinaryDataByFilter(ctx context.Context, tags []string, filter *pb.Filter) (uint64, error) { + if filter == nil { + filter = &pb.Filter{} + } + resp, err := d.client.RemoveTagsFromBinaryDataByFilter(ctx, &pb.RemoveTagsFromBinaryDataByFilterRequest{Filter: filter, Tags: tags}) + if err != nil { + return 0, err + } + return resp.DeletedCount, nil } -func (d *DataClient) TagsByFilter() error { - return errors.New("unimplemented") +func (d *DataClient) TagsByFilter(ctx context.Context, filter *pb.Filter) ([]string, error) { + if filter == nil { + filter = &pb.Filter{} + } + resp, err := d.client.TagsByFilter(ctx, &pb.TagsByFilterRequest{Filter: filter}) + if err != nil { + return nil, err + } + return resp.Tags, nil } func (d *DataClient) AddBoundingBoxToImageByID() error { return errors.New("unimplemented") From 69d5f7606c7433d80b44f7eea0512204a6fb6ad5 Mon Sep 17 00:00:00 2001 From: Julie Krasnick Date: Tue, 5 Nov 2024 22:17:07 -0500 Subject: [PATCH 05/25] more work --- app/data_client.go | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/app/data_client.go b/app/data_client.go index 7be45e38f2b..965095d40cc 100644 --- a/app/data_client.go +++ b/app/data_client.go @@ -408,11 +408,24 @@ func (d *DataClient) TagsByFilter(ctx context.Context, filter *pb.Filter) ([]str } return resp.Tags, nil } -func (d *DataClient) AddBoundingBoxToImageByID() error { - return errors.New("unimplemented") +func (d *DataClient) AddBoundingBoxToImageByID(ctx context.Context, binaryIds []*pb.BinaryID, label string, + xMinNormalized float64, + yMinNormalized float64, + xMaxNormalized float64, + yMaxNormalized float64) (string, error) { + resp, err := d.client.AddBoundingBoxToImageByID(ctx, &pb.AddBoundingBoxToImageByIDRequest{BinaryId: &pb.BinaryID{}, Label: label, XMinNormalized: xMinNormalized, YMinNormalized: yMinNormalized, XMaxNormalized: xMaxNormalized, YMaxNormalized: yMaxNormalized}) + if err != nil { + return "", err + } + return resp.BboxId, nil + } -func (d *DataClient) RemoveBoundingBoxFromImageByID() error { - return errors.New("unimplemented") +func (d *DataClient) RemoveBoundingBoxFromImageByID(ctx context.Context, bboxId string, binaryId *pb.BinaryID) error { + _, err := d.client.RemoveBoundingBoxFromImageByID(ctx, &pb.RemoveBoundingBoxFromImageByIDRequest{BinaryId: binaryId, BboxId: bboxId}) + if err != nil { + return err + } + return nil } func (d *DataClient) BoundingBoxLabelsByFilter() error { return errors.New("unimplemented") From 51e247926e29fbff21f1d3c033321fa81ab84a40 Mon Sep 17 00:00:00 2001 From: Julie Krasnick Date: Wed, 6 Nov 2024 14:35:29 -0500 Subject: [PATCH 06/25] naveed pair program on tests --- app/data_client.go | 75 ++++++++--- testutils/inject/data_service_client.go | 166 +++++++++++++++++++++++- 2 files changed, 218 insertions(+), 23 deletions(-) diff --git a/app/data_client.go b/app/data_client.go index 965095d40cc..95ba98c9d5d 100644 --- a/app/data_client.go +++ b/app/data_client.go @@ -1,11 +1,9 @@ //go:build !no_cgo -// Package arm contains a gRPC based arm client. -package arm +package app import ( "context" - "errors" "time" "go.mongodb.org/mongo-driver/bson" @@ -158,8 +156,7 @@ func BinaryMetadataFromProto(proto *pb.BinaryMetadata) BinaryMetadata { type DataClient struct { //do we want this to be a public interface that defines the functions but does not include client and private details? //would not include client and private details - client pb.DataServiceClient - TabularData TabularData + client pb.DataServiceClient } // (private) dataClient implements DataServiceClient. **do we want this? @@ -408,7 +405,10 @@ func (d *DataClient) TagsByFilter(ctx context.Context, filter *pb.Filter) ([]str } return resp.Tags, nil } -func (d *DataClient) AddBoundingBoxToImageByID(ctx context.Context, binaryIds []*pb.BinaryID, label string, +func (d *DataClient) AddBoundingBoxToImageByID( + ctx context.Context, + binaryIds []*pb.BinaryID, + label string, xMinNormalized float64, yMinNormalized float64, xMaxNormalized float64, @@ -427,21 +427,60 @@ func (d *DataClient) RemoveBoundingBoxFromImageByID(ctx context.Context, bboxId } return nil } -func (d *DataClient) BoundingBoxLabelsByFilter() error { - return errors.New("unimplemented") +func (d *DataClient) BoundingBoxLabelsByFilter(ctx context.Context, filter *pb.Filter) ([]string, error) { + if filter == nil { + filter = &pb.Filter{} + } + resp, err := d.client.BoundingBoxLabelsByFilter(ctx, &pb.BoundingBoxLabelsByFilterRequest{Filter: filter}) + if err != nil { + return nil, err + } + return resp.Labels, nil } -func (d *DataClient) UpdateBoundingBox() error { - return errors.New("unimplemented") + +// ***python and typescript did not implement this one!!! +func (d *DataClient) UpdateBoundingBox(ctx context.Context, + binaryId *pb.BinaryID, + bboxId string, + label string, + xMinNormalized float64, + yMinNormalized float64, + xMaxNormalized float64, + yMaxNormalized float64) error { + + _, err := d.client.UpdateBoundingBox(ctx, &pb.UpdateBoundingBoxRequest{BinaryId: binaryId, BboxId: bboxId, Label: &label, XMinNormalized: &xMinNormalized, YMinNormalized: &yMinNormalized, XMaxNormalized: &xMaxNormalized, YMaxNormalized: &yMaxNormalized}) + if err != nil { + return err + } + return nil } -func (d *DataClient) GetDatabaseConnection() error { - return errors.New("unimplemented") + +// do we want to return more than a hostname?? +func (d *DataClient) GetDatabaseConnection(ctx context.Context, organizationId string) (string, error) { + resp, err := d.client.GetDatabaseConnection(ctx, &pb.GetDatabaseConnectionRequest{OrganizationId: organizationId}) + if err != nil { + return "", err + } + return resp.Hostname, nil } -func (d *DataClient) ConfigureDatabaseUser() error { - return errors.New("unimplemented") +func (d *DataClient) ConfigureDatabaseUser(ctx context.Context, organizationId string, password string) error { + _, err := d.client.ConfigureDatabaseUser(ctx, &pb.ConfigureDatabaseUserRequest{OrganizationId: organizationId, Password: password}) + if err != nil { + return err + } + return nil } -func (d *DataClient) AddBinaryDataToDatasetByIDs() error { - return errors.New("unimplemented") +func (d *DataClient) AddBinaryDataToDatasetByIDs(ctx context.Context, binaryIds []*pb.BinaryID, datasetId string) error { + _, err := d.client.AddBinaryDataToDatasetByIDs(ctx, &pb.AddBinaryDataToDatasetByIDsRequest{BinaryIds: binaryIds, DatasetId: datasetId}) + if err != nil { + return err + } + return nil } -func (d *DataClient) RemoveBinaryDataFromDatasetByIDs() error { - return errors.New("unimplemented") +func (d *DataClient) RemoveBinaryDataFromDatasetByIDs(ctx context.Context, binaryIds []*pb.BinaryID, datasetId string) error { + _, err := d.client.RemoveBinaryDataFromDatasetByIDs(ctx, &pb.RemoveBinaryDataFromDatasetByIDsRequest{BinaryIds: binaryIds, DatasetId: datasetId}) + if err != nil { + return err + } + return nil } diff --git a/testutils/inject/data_service_client.go b/testutils/inject/data_service_client.go index bba53dee407..a83e510b089 100644 --- a/testutils/inject/data_service_client.go +++ b/testutils/inject/data_service_client.go @@ -10,11 +10,27 @@ import ( // DataServiceClient represents a fake instance of a data service client. type DataServiceClient struct { datapb.DataServiceClient - TabularDataByFilterFunc func( - ctx context.Context, - in *datapb.TabularDataByFilterRequest, - opts ...grpc.CallOption, - ) (*datapb.TabularDataByFilterResponse, error) + TabularDataByFilterFunc func(ctx context.Context, in *datapb.TabularDataByFilterRequest, opts ...grpc.CallOption) (*datapb.TabularDataByFilterResponse, error) + TabularDataBySQLFunc func(ctx context.Context, in *datapb.TabularDataBySQLRequest, opts ...grpc.CallOption) (*datapb.TabularDataBySQLResponse, error) + TabularDataByMQLFunc func(ctx context.Context, in *datapb.TabularDataByMQLRequest, opts ...grpc.CallOption) (*datapb.TabularDataByMQLResponse, error) + BinaryDataByFilterFunc func(ctx context.Context, in *datapb.BinaryDataByFilterRequest, opts ...grpc.CallOption) (*datapb.BinaryDataByFilterResponse, error) + BinaryDataByIDsFunc func(ctx context.Context, in *datapb.BinaryDataByIDsRequest, opts ...grpc.CallOption) (*datapb.BinaryDataByIDsResponse, error) + DeleteTabularDataFunc func(ctx context.Context, in *datapb.DeleteTabularDataRequest, opts ...grpc.CallOption) (*datapb.DeleteTabularDataResponse, error) + DeleteBinaryDataByFilterFunc func(ctx context.Context, in *datapb.DeleteBinaryDataByFilterRequest, opts ...grpc.CallOption) (*datapb.DeleteBinaryDataByFilterResponse, error) + DeleteBinaryDataByIDsFunc func(ctx context.Context, in *datapb.DeleteBinaryDataByIDsRequest, opts ...grpc.CallOption) (*datapb.DeleteBinaryDataByIDsResponse, error) + AddTagsToBinaryDataByIDsFunc func(ctx context.Context, in *datapb.AddTagsToBinaryDataByIDsRequest, opts ...grpc.CallOption) (*datapb.AddTagsToBinaryDataByIDsResponse, error) + AddTagsToBinaryDataByFilterFunc func(ctx context.Context, in *datapb.AddTagsToBinaryDataByFilterRequest, opts ...grpc.CallOption) (*datapb.AddTagsToBinaryDataByFilterResponse, error) + RemoveTagsFromBinaryDataByIDsFunc func(ctx context.Context, in *datapb.RemoveTagsFromBinaryDataByIDsRequest, opts ...grpc.CallOption) (*datapb.RemoveTagsFromBinaryDataByIDsResponse, error) + RemoveTagsFromBinaryDataByFilterFunc func(ctx context.Context, in *datapb.RemoveTagsFromBinaryDataByFilterRequest, opts ...grpc.CallOption) (*datapb.RemoveTagsFromBinaryDataByFilterResponse, error) + TagsByFilterFunc func(ctx context.Context, in *datapb.TagsByFilterRequest, opts ...grpc.CallOption) (*datapb.TagsByFilterResponse, error) + AddBoundingBoxToImageByIDFunc func(ctx context.Context, in *datapb.AddBoundingBoxToImageByIDRequest, opts ...grpc.CallOption) (*datapb.AddBoundingBoxToImageByIDResponse, error) + RemoveBoundingBoxFromImageByIDFunc func(ctx context.Context, in *datapb.RemoveBoundingBoxFromImageByIDRequest, opts ...grpc.CallOption) (*datapb.RemoveBoundingBoxFromImageByIDResponse, error) + BoundingBoxLabelsByFilterFunc func(ctx context.Context, in *datapb.BoundingBoxLabelsByFilterRequest, opts ...grpc.CallOption) (*datapb.BoundingBoxLabelsByFilterResponse, error) + UpdateBoundingBoxFunc func(ctx context.Context, in *datapb.UpdateBoundingBoxRequest, opts ...grpc.CallOption) (*datapb.UpdateBoundingBoxResponse, error) + GetDatabaseConnectionFunc func(ctx context.Context, in *datapb.GetDatabaseConnectionRequest, opts ...grpc.CallOption) (*datapb.GetDatabaseConnectionResponse, error) + ConfigureDatabaseUserFunc func(ctx context.Context, in *datapb.ConfigureDatabaseUserRequest, opts ...grpc.CallOption) (*datapb.ConfigureDatabaseUserResponse, error) + AddBinaryDataToDatasetByIDsFunc func(ctx context.Context, in *datapb.AddBinaryDataToDatasetByIDsRequest, opts ...grpc.CallOption) (*datapb.AddBinaryDataToDatasetByIDsResponse, error) + RemoveBinaryDataFromDatasetByIDsFunc func(ctx context.Context, in *datapb.RemoveBinaryDataFromDatasetByIDsRequest, opts ...grpc.CallOption) (*datapb.RemoveBinaryDataFromDatasetByIDsResponse, error) } // TabularDataByFilter calls the injected TabularDataByFilter or the real version. @@ -25,3 +41,143 @@ func (client *DataServiceClient) TabularDataByFilter(ctx context.Context, in *da } return client.TabularDataByFilterFunc(ctx, in, opts...) } + +func (client *DataServiceClient) TabularDataBySQL(ctx context.Context, in *datapb.TabularDataBySQLRequest, opts ...grpc.CallOption) (*datapb.TabularDataBySQLResponse, error) { + if client.TabularDataBySQLFunc == nil { + return client.DataServiceClient.TabularDataBySQL(ctx, in, opts...) + } + return client.TabularDataBySQLFunc(ctx, in, opts...) +} + +func (client *DataServiceClient) TabularDataByMQL(ctx context.Context, in *datapb.TabularDataByMQLRequest, opts ...grpc.CallOption) (*datapb.TabularDataByMQLResponse, error) { + if client.TabularDataByMQLFunc == nil { + return client.DataServiceClient.TabularDataByMQL(ctx, in, opts...) + } + return client.TabularDataByMQLFunc(ctx, in, opts...) +} + +func (client *DataServiceClient) BinaryDataByFilter(ctx context.Context, in *datapb.BinaryDataByFilterRequest, opts ...grpc.CallOption) (*datapb.BinaryDataByFilterResponse, error) { + if client.BinaryDataByFilterFunc == nil { + return client.DataServiceClient.BinaryDataByFilter(ctx, in, opts...) + } + return client.BinaryDataByFilterFunc(ctx, in, opts...) +} + +func (client *DataServiceClient) BinaryDataByIDs(ctx context.Context, in *datapb.BinaryDataByIDsRequest, opts ...grpc.CallOption) (*datapb.BinaryDataByIDsResponse, error) { + if client.BinaryDataByIDsFunc == nil { + return client.DataServiceClient.BinaryDataByIDs(ctx, in, opts...) + } + return client.BinaryDataByIDsFunc(ctx, in, opts...) +} + +func (client *DataServiceClient) DeleteTabularData(ctx context.Context, in *datapb.DeleteTabularDataRequest, opts ...grpc.CallOption) (*datapb.DeleteTabularDataResponse, error) { + if client.DeleteTabularDataFunc == nil { + return client.DataServiceClient.DeleteTabularData(ctx, in, opts...) + } + return client.DeleteTabularDataFunc(ctx, in, opts...) +} + +func (client *DataServiceClient) DeleteBinaryDataByFilter(ctx context.Context, in *datapb.DeleteBinaryDataByFilterRequest, opts ...grpc.CallOption) (*datapb.DeleteBinaryDataByFilterResponse, error) { + if client.DeleteBinaryDataByFilterFunc == nil { + return client.DataServiceClient.DeleteBinaryDataByFilter(ctx, in, opts...) + } + return client.DeleteBinaryDataByFilterFunc(ctx, in, opts...) +} + +func (client *DataServiceClient) DeleteBinaryDataByIDs(ctx context.Context, in *datapb.DeleteBinaryDataByIDsRequest, opts ...grpc.CallOption) (*datapb.DeleteBinaryDataByIDsResponse, error) { + if client.DeleteBinaryDataByIDsFunc == nil { + return client.DataServiceClient.DeleteBinaryDataByIDs(ctx, in, opts...) + } + return client.DeleteBinaryDataByIDsFunc(ctx, in, opts...) +} + +func (client *DataServiceClient) AddTagsToBinaryDataByIDs(ctx context.Context, in *datapb.AddTagsToBinaryDataByIDsRequest, opts ...grpc.CallOption) (*datapb.AddTagsToBinaryDataByIDsResponse, error) { + if client.AddTagsToBinaryDataByIDsFunc == nil { + return client.DataServiceClient.AddTagsToBinaryDataByIDs(ctx, in, opts...) + } + return client.AddTagsToBinaryDataByIDsFunc(ctx, in, opts...) +} + +func (client *DataServiceClient) AddTagsToBinaryDataByFilter(ctx context.Context, in *datapb.AddTagsToBinaryDataByFilterRequest, opts ...grpc.CallOption) (*datapb.AddTagsToBinaryDataByFilterResponse, error) { + if client.AddTagsToBinaryDataByFilterFunc == nil { + return client.DataServiceClient.AddTagsToBinaryDataByFilter(ctx, in, opts...) + } + return client.AddTagsToBinaryDataByFilterFunc(ctx, in, opts...) +} + +func (client *DataServiceClient) RemoveTagsFromBinaryDataByIDs(ctx context.Context, in *datapb.RemoveTagsFromBinaryDataByIDsRequest, opts ...grpc.CallOption) (*datapb.RemoveTagsFromBinaryDataByIDsResponse, error) { + if client.RemoveTagsFromBinaryDataByIDsFunc == nil { + return client.DataServiceClient.RemoveTagsFromBinaryDataByIDs(ctx, in, opts...) + } + return client.RemoveTagsFromBinaryDataByIDsFunc(ctx, in, opts...) +} + +func (client *DataServiceClient) RemoveTagsFromBinaryDataByFilter(ctx context.Context, in *datapb.RemoveTagsFromBinaryDataByFilterRequest, opts ...grpc.CallOption) (*datapb.RemoveTagsFromBinaryDataByFilterResponse, error) { + if client.RemoveTagsFromBinaryDataByFilterFunc == nil { + return client.DataServiceClient.RemoveTagsFromBinaryDataByFilter(ctx, in, opts...) + } + return client.RemoveTagsFromBinaryDataByFilterFunc(ctx, in, opts...) +} + +func (client *DataServiceClient) TagsByFilter(ctx context.Context, in *datapb.TagsByFilterRequest, opts ...grpc.CallOption) (*datapb.TagsByFilterResponse, error) { + if client.TagsByFilterFunc == nil { + return client.DataServiceClient.TagsByFilter(ctx, in, opts...) + } + return client.TagsByFilterFunc(ctx, in, opts...) +} + +func (client *DataServiceClient) AddBoundingBoxToImageByID(ctx context.Context, in *datapb.AddBoundingBoxToImageByIDRequest, opts ...grpc.CallOption) (*datapb.AddBoundingBoxToImageByIDResponse, error) { + if client.AddBoundingBoxToImageByIDFunc == nil { + return client.DataServiceClient.AddBoundingBoxToImageByID(ctx, in, opts...) + } + return client.AddBoundingBoxToImageByIDFunc(ctx, in, opts...) +} + +func (client *DataServiceClient) RemoveBoundingBoxFromImageByID(ctx context.Context, in *datapb.RemoveBoundingBoxFromImageByIDRequest, opts ...grpc.CallOption) (*datapb.RemoveBoundingBoxFromImageByIDResponse, error) { + if client.RemoveBoundingBoxFromImageByIDFunc == nil { + return client.DataServiceClient.RemoveBoundingBoxFromImageByID(ctx, in, opts...) + } + return client.RemoveBoundingBoxFromImageByIDFunc(ctx, in, opts...) +} + +func (client *DataServiceClient) BoundingBoxLabelsByFilter(ctx context.Context, in *datapb.BoundingBoxLabelsByFilterRequest, opts ...grpc.CallOption) (*datapb.BoundingBoxLabelsByFilterResponse, error) { + if client.BoundingBoxLabelsByFilterFunc == nil { + return client.DataServiceClient.BoundingBoxLabelsByFilter(ctx, in, opts...) + } + return client.BoundingBoxLabelsByFilterFunc(ctx, in, opts...) +} + +func (client *DataServiceClient) UpdateBoundingBox(ctx context.Context, in *datapb.UpdateBoundingBoxRequest, opts ...grpc.CallOption) (*datapb.UpdateBoundingBoxResponse, error) { + if client.UpdateBoundingBoxFunc == nil { + return client.DataServiceClient.UpdateBoundingBox(ctx, in, opts...) + } + return client.UpdateBoundingBoxFunc(ctx, in, opts...) +} + +func (client *DataServiceClient) GetDatabaseConnection(ctx context.Context, in *datapb.GetDatabaseConnectionRequest, opts ...grpc.CallOption) (*datapb.GetDatabaseConnectionResponse, error) { + if client.GetDatabaseConnectionFunc == nil { + return client.DataServiceClient.GetDatabaseConnection(ctx, in, opts...) + } + return client.GetDatabaseConnectionFunc(ctx, in, opts...) +} + +func (client *DataServiceClient) ConfigureDatabaseUser(ctx context.Context, in *datapb.ConfigureDatabaseUserRequest, opts ...grpc.CallOption) (*datapb.ConfigureDatabaseUserResponse, error) { + if client.ConfigureDatabaseUserFunc == nil { + return client.DataServiceClient.ConfigureDatabaseUser(ctx, in, opts...) + } + return client.ConfigureDatabaseUserFunc(ctx, in, opts...) +} + +func (client *DataServiceClient) AddBinaryDataToDatasetByIDs(ctx context.Context, in *datapb.AddBinaryDataToDatasetByIDsRequest, opts ...grpc.CallOption) (*datapb.AddBinaryDataToDatasetByIDsResponse, error) { + if client.AddBinaryDataToDatasetByIDsFunc == nil { + return client.DataServiceClient.AddBinaryDataToDatasetByIDs(ctx, in, opts...) + } + return client.AddBinaryDataToDatasetByIDsFunc(ctx, in, opts...) +} + +func (client *DataServiceClient) RemoveBinaryDataFromDatasetByIDs(ctx context.Context, in *datapb.RemoveBinaryDataFromDatasetByIDsRequest, opts ...grpc.CallOption) (*datapb.RemoveBinaryDataFromDatasetByIDsResponse, error) { + if client.RemoveBinaryDataFromDatasetByIDsFunc == nil { + return client.DataServiceClient.RemoveBinaryDataFromDatasetByIDs(ctx, in, opts...) + } + return client.RemoveBinaryDataFromDatasetByIDsFunc(ctx, in, opts...) +} From 2d153107761b6c0f3d7cbeb987ee54cdd753db06 Mon Sep 17 00:00:00 2001 From: Julie Krasnick Date: Wed, 6 Nov 2024 16:40:46 -0500 Subject: [PATCH 07/25] more testing stuff --- app/data_client.go | 39 ++++++++- app/data_test.go | 191 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 228 insertions(+), 2 deletions(-) create mode 100644 app/data_test.go diff --git a/app/data_client.go b/app/data_client.go index 95ba98c9d5d..1eb47673c2d 100644 --- a/app/data_client.go +++ b/app/data_client.go @@ -4,6 +4,7 @@ package app import ( "context" + "fmt" "time" "go.mongodb.org/mongo-driver/bson" @@ -259,12 +260,46 @@ func (d *DataClient) TabularDataByMQL(ctx context.Context, organizationId string if err != nil { return nil, err } + // Debugging output to verify RawData content + fmt.Printf("Response RawData: %v\n", resp.RawData) + //loop thru rawData + //for each rawData byte slice you will need to unmarshall it into map[string]interface + //then add each unmarshalled map to a list and return it dataObjects := make([]map[string]interface{}, len(resp.RawData)) for i, rawData := range resp.RawData { - obj := make(map[string]interface{}) - bson.Unmarshal(rawData, &obj) + var obj map[string]interface{} + if err := bson.Unmarshal(rawData, &obj); err != nil { + fmt.Printf("(func) unmarshalling error %d: %v", i, err) + return nil, err + } dataObjects[i] = obj } + // dataObjects := make([]map[string]interface{}, len(resp.RawData)) + // for i, rawData := range resp.RawData { + // obj := make(map[string]interface{}) + // bson.Unmarshal(rawData, &obj) + // dataObjects[i] = obj + // } + // Unmarshal all raw data at once as an array of maps + // var dataObjects []map[string]interface{} + // for _, rawData := range resp.RawData { + // var singleData []map[string]interface{} // This should match your expected structure + // if err := bson.Unmarshal(rawData, &singleData); err != nil { + // return nil, err + // } + // dataObjects = append(dataObjects, singleData...) + // } + // Unmarshal each raw data entry as a separate map + // dataObjects := make([]map[string]interface{}, len(resp.RawData)) + // for i, rawData := range resp.RawData { + // var obj map[string]interface{} + // if err := bson.Unmarshal(rawData, &obj); err != nil { + // fmt.Printf("Unmarshalling error at index %d: %v\n", i, err) + // return nil, err + // } + // dataObjects[i] = obj + // } + fmt.Println("printing Deserialized dataObjects here", dataObjects) return dataObjects, nil } diff --git a/app/data_test.go b/app/data_test.go new file mode 100644 index 00000000000..4259f334a5b --- /dev/null +++ b/app/data_test.go @@ -0,0 +1,191 @@ +package app + +import ( + "context" + "fmt" + "testing" + + "go.mongodb.org/mongo-driver/bson" + datapb "go.viam.com/api/app/data/v1" + "go.viam.com/test" + "google.golang.org/grpc" + + "go.viam.com/rdk/testutils/inject" +) + +// var ( +// orgId = "e76d1b3b-0468-4efd-bb7f-fb1d2b352fcb" +// ) + +// // Helper function to create BSON documents for MongoDB queries +// func createMQLBSON() [][]byte { +// // create BSON documents for mongodb queries +// matchStage := bson.M{"$match": bson.M{"organization_id": "e76d1b3b-0468-4efd-bb7f-fb1d2b352fcb"}} +// limitStage := bson.M{"$limit": 1} +// // convert to BSON byte arrays +// matchBytes, _ := bson.Marshal(matchStage) +// limitBytes, _ := bson.Marshal(limitStage) +// mqlbinary := [][]byte{matchBytes, limitBytes} + +// return mqlbinary + +// } + +// set up gRPC client?? +func createGrpclient() datapb.DataServiceClient { + return &inject.DataServiceClient{} +} +func TestDataClient(t *testing.T) { + grpcClient := &inject.DataServiceClient{} + client := DataClient{client: grpcClient} + t.Run("TabularDataByFilter", func(t *testing.T) { + + }) + t.Run("TabularDataBySQL", func(t *testing.T) { + + }) + t.Run("TabularDataByMQL", func(t *testing.T) { + org_id := "MY_ORG" + input := map[string]interface{}{ + "foo": "bar", + "one": 1, + "list": []string{"a", "b", "c"}, + } + //serialize input into list of bytearrays, aka bson + bsonData, err := bson.Marshal(input) + if err != nil { + fmt.Printf("trying something out") + } + mql_binary := [][]byte{bsonData} //bson data type + + expected := []map[string]interface{}{ + {"foo": "bar"}, + {"one": 1}, + {"list": []string{"a", "b", "c"}}, + } + + grpcClient.TabularDataByMQLFunc = func(ctx context.Context, in *datapb.TabularDataByMQLRequest, opts ...grpc.CallOption) (*datapb.TabularDataByMQLResponse, error) { + test.That(t, in.OrganizationId, test.ShouldEqual, org_id) + test.That(t, in.MqlBinary, test.ShouldResemble, mql_binary) + return &datapb.TabularDataByMQLResponse{RawData: mql_binary}, nil //MQlResponse is created with BSON data + } + + response, _ := client.TabularDataByMQL(context.Background(), org_id, mql_binary) + // test.That(t, response, test.ShouldEqual, expected) + fmt.Printf("Expected: %#v\n", expected) + fmt.Printf("Actual : %#v\n", response) + test.That(t, response, test.ShouldResemble, expected) //*this test not working as expected yet! + }) + t.Run("TabularDataByFilter", func(t *testing.T) { + + }) + +} + +// func TestTabularDataByFilter(t *testing.T) { +// // Set up test logger +// logger := logging.NewTestLogger(t) +// // need this listener probs +// listener, err := net.Listen("tcp", "localhost:0") +// test.That(t, err, test.ShouldBeNil) +// client, conn, err := createGrpclient(t, logger, listener) // create client conn w helper +// test.That(t, err, test.ShouldBeNil) + +// dataRequest := &datapb.DataRequest{} //this doesn't seem right??? + +// //grpc requeset?? +// req := &datapb.TabularDataByFilterRequest{ +// DataRequest: dataRequest, //this doesn't seem right??? +// CountOnly: true, +// IncludeInternalData: true, +// } + +// // call the real method +// resp, err := client.TabularDataByFilter(context.Background(), req) +// test.That(t, err, test.ShouldBeNil) +// // check that the parameters being passed match the expected data?? +// test.That(t, req.DataRequest, test.ShouldResemble, dataRequest) + +// // check that the response matches expected data +// expectedData := &datapb.TabularDataByFilterResponse{ +// Data: resp.Data, //idk if it makes sense to be passing the resp.Data?? +// RawData: resp.RawData, //bc what are we actually testing? +// } +// // test.That(t, expectedData, test.ShouldResembleProto, &datapb.TabularDataByMQLResponse{}) +// test.That(t, expectedData, test.ShouldResemble, resp) +// // Close the connection +// require.NoError(t, conn.Close()) + +// } + +// func TestTabularDataBySQL(t *testing.T) {} +// func TestTabularDataByMQL(t *testing.T) { +// client := createGrpclient() + +// return +// // Set up test logger +// logger := logging.NewTestLogger(t) +// // need this listener probs +// listener, err := net.Listen("tcp", "localhost:0") +// test.That(t, err, test.ShouldBeNil) +// client, conn, err := createGrpclient(t, logger, listener) // create client conn w helper +// test.That(t, err, test.ShouldBeNil) + +// //call helper +// mqlbinary := createMQLBSON() +// // orgId := "e76d1b3b-0468-4efd-bb7f-fb1d2b352fcb" +// // make the actual call to the grpc dataserive function?? +// req := &datapb.TabularDataByMQLRequest{ +// OrganizationId: orgId, +// MqlBinary: mqlbinary, +// } +// // call the real method +// resp, err := client.TabularDataByMQL(context.Background(), req) +// test.That(t, err, test.ShouldBeNil) +// // check that the parameters being passed match the expected data?? +// test.That(t, req.OrganizationId, test.ShouldResemble, orgId) + +// // check that the response matches expected data +// expectedData := &datapb.TabularDataByMQLResponse{ +// Data: resp.Data, //idk if it makes sense to be passing the resp.Data?? +// RawData: resp.RawData, //bc what are we actually testing? +// } +// // test.That(t, expectedData, test.ShouldResembleProto, &datapb.TabularDataByMQLResponse{}) +// test.That(t, expectedData, test.ShouldResemble, resp) +// // Close the connection +// require.NoError(t, conn.Close()) +// } + +// func TestBinaryDataByFilter(t *testing.T) {} +// func TestBinaryDataByIDs(t *testing.T) {} +// func TestDeleteTabularData(t *testing.T) {} +// func TestDeleteBinaryDataByFilter(t *testing.T) {} +// func TestDeleteBinaryDataByIDs(t *testing.T) {} +// func TestAddTagsToBinaryDataByIDs(t *testing.T) {} +// func TestAddTagsToBinaryDataByFilter(t *testing.T) {} +// func TestRemoveTagsFromBinaryDataByIDs(t *testing.T) {} +// func TestRemoveTagsFromBinaryDataByFilter(t *testing.T) {} +// func TestTagsByFilter(t *testing.T) {} + +//notes: +// Set up gRPC server (should it be grpc.NewServer()??) +// logger := logging.NewTestLogger(t) +// listener, err := net.Listen("tcp", "localhost:0") +// test.That(t, err, test.ShouldBeNil) +// rpcServer, err := rpc.NewServer(logger, rpc.WithUnauthenticated()) +// test.That(t, err, test.ShouldBeNil) + +//****need to somehow register the DataService server here?? +// datapb.RegisterDataServiceHandlerFromEndpoint(context.Background(),runtime.NewServeMux(),) +// datapb.RegisterDataServiceServer(rpcServer,) +// datapb.RegisterDataServiceServer(rpcServer, &datapb.UnimplementedDataServiceServer{}) +// data = datapb.DataServiceServer{} + +// Start serving requests?? +// go rpcServer.Serve(listener) +// defer rpcServer.Stop() + +// Make client connection +// conn, err := viamgrpc.Dial(context.Background(), listener.Addr().String(), logger) +// test.That(t, err, test.ShouldBeNil) +// client := datapb.NewDataServiceClient(conn) From f64e97e0c649cbcaf620c458384f67236e0b7920 Mon Sep 17 00:00:00 2001 From: Julie Krasnick Date: Wed, 6 Nov 2024 17:15:57 -0500 Subject: [PATCH 08/25] more test stuff --- app/data_test.go | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/app/data_test.go b/app/data_test.go index 4259f334a5b..f09accce0dc 100644 --- a/app/data_test.go +++ b/app/data_test.go @@ -39,6 +39,36 @@ func TestDataClient(t *testing.T) { grpcClient := &inject.DataServiceClient{} client := DataClient{client: grpcClient} t.Run("TabularDataByFilter", func(t *testing.T) { + // expected := {} + // expectedData := "" //change - not done + // expectedCount := 5 //change - not done + // expectedLast := "" //change - not done + // filter := &datapb.Filter{} + // limit := uint64(5) + // last := "last" + // sortOrder := datapb.Order_ORDER_DESCENDING //i think this is correct + // countOnly := true + // includeInternalData := true + // myData := { + // Data: map[string]interface{} + // Metadata: CaptureMetadata + // } + // myFilter := create_filter(component_name="motor-1") + // myLast := nil + + // returns: List[TabularData]: The tabular data, int: The count (number of entries), str: The last-returned page ID. + + grpcClient.TabularDataByFilterFunc = func(ctx context.Context, in *datapb.TabularDataByFilterRequest, opts ...grpc.CallOption) (*datapb.TabularDataByFilterResponse, error) { + test.That(t, in.DataRequest, test.ShouldEqual) + test.That(t, in.CountOnly, test.ShouldBeTrue) + test.That(t, in.IncludeInternalData, test.ShouldBeTrue) + return &datapb.TabularDataByFilterResponse{Data: _, Count: _, Last: _}, nil + } + + respData, respCount, respLast, _ := client.TabularDataByFilter(context.Background(), filter, limit, last, sortOrder, countOnly, includeInternalData) + test.That(t, respData, test.ShouldEqual, expectedData) + test.That(t, respCount, test.ShouldEqual, expectedCount) + test.That(t, respLast, test.ShouldEqual, expectedLast) }) t.Run("TabularDataBySQL", func(t *testing.T) { From 8a6e7bbb3bd29db52e42f43ff9d23b90b901e82f Mon Sep 17 00:00:00 2001 From: Julie Krasnick Date: Thu, 7 Nov 2024 13:49:54 -0500 Subject: [PATCH 09/25] adding more stuff --- app/data_client.go | 192 +++++++++++++++++++++++++++++++++++---------- app/data_test.go | 71 ++++++++++++++++- 2 files changed, 220 insertions(+), 43 deletions(-) diff --git a/app/data_client.go b/app/data_client.go index 1eb47673c2d..353ae6372c8 100644 --- a/app/data_client.go +++ b/app/data_client.go @@ -12,6 +12,7 @@ import ( "go.viam.com/rdk/logging" "go.viam.com/utils/rpc" "google.golang.org/protobuf/types/known/structpb" + "google.golang.org/protobuf/types/known/timestamppb" ) //protobuf to type or type to protobuf (poseinframe to proto or proto to pose in frame) @@ -39,7 +40,11 @@ type CaptureMetadata struct { mime_type string //^^ string mime_type = 13; } - +type BinaryID struct { + FileId string + OrganizationId string + LocationId string +} type BoundingBox struct { id string label string @@ -66,6 +71,7 @@ type BinaryData struct { Metadata BinaryMetadata } +// can the return type be a struct called BinaryMetadata that I made up??? type BinaryMetadata struct { ID string //CaptureMetadata *pb.CaptureMetadata @@ -79,6 +85,48 @@ type BinaryMetadata struct { //Annotations *pb.Annotations DatasetIDs []string } +type Filter struct { + ComponentName string + ComponentType string + Method string + RobotName string + RobotId string + PartName string + PartId string + LocationIds []string + OrganizationIds []string + MimeType []string + Interval CaptureInterval //asterix or no?? + TagsFilter TagsFilter //asterix or no?? + BboxLabels []string + DatasetId string +} + +// func (f Filter) IsEmpty() bool { +// return reflect.DeepEqual(f, Filter{}) +// } + +//notes for above:: +// type TagsFilter struct { +// state protoimpl.MessageState +// sizeCache protoimpl.SizeCache +// unknownFields protoimpl.UnknownFields + +// Type TagsFilterType `protobuf:"varint,1,opt,name=type,proto3,enum=viam.app.data.v1.TagsFilterType" json:"type,omitempty"` +// // Tags are used to match documents if `type` is UNSPECIFIED or MATCH_BY_OR. +// Tags []string `protobuf:"bytes,2,rep,name=tags,proto3" json:"tags,omitempty"` +// } + +//type TagsFilterType int32 + +type TagsFilter struct { + Type int32 //type TagsFilterType int32 + Tags []string +} +type CaptureInterval struct { + Start time.Time + End time.Time +} func BoundingBoxFromProto(proto *pb.BoundingBox) BoundingBox { return BoundingBox{ @@ -154,6 +202,68 @@ func BinaryMetadataFromProto(proto *pb.BinaryMetadata) BinaryMetadata { } } +// PropertiesToProtoResponse takes a map of features to struct and converts it +// to a GetPropertiesResponse. +func FilterToProto(filter Filter) *pb.Filter { + return &pb.Filter{ + ComponentName: filter.ComponentName, + ComponentType: filter.ComponentType, + Method: filter.Method, + RobotName: filter.RobotName, + RobotId: filter.RobotId, + PartName: filter.PartName, + PartId: filter.PartId, + LocationIds: filter.LocationIds, + OrganizationIds: filter.OrganizationIds, + MimeType: filter.MimeType, + Interval: CaptureIntervalToProto(filter.Interval), //check this ?? + TagsFilter: TagsFilterToProto(filter.TagsFilter), //check this ?? + BboxLabels: filter.BboxLabels, + DatasetId: filter.DatasetId, + } +} +func CaptureIntervalToProto(interval CaptureInterval) *pb.CaptureInterval { + return &pb.CaptureInterval{ + Start: timestamppb.New(interval.Start), + End: timestamppb.New(interval.End), + } +} +func TagsFilterToProto(tagsFilter TagsFilter) *pb.TagsFilter { + return &pb.TagsFilter{ + Type: pb.TagsFilterType(tagsFilter.Type), + Tags: tagsFilter.Tags, + } +} + +// shadow type for Order +type Order int32 + +// do i even need these below??? +const ( + Unspecified Order = 0 + Descending Order = 1 + Ascending Order = 2 +) + +// Map SortOrder to corresponding values in pb.Order +func OrderToProto(sortOrder Order) pb.Order { + switch sortOrder { + case Ascending: + return pb.Order_ORDER_ASCENDING + case Descending: + return pb.Order_ORDER_DESCENDING + default: + return pb.Order_ORDER_UNSPECIFIED // default or error handling + } +} + +// LocationIds []string +// OrganizationIds []string +// MimeType []string +// Interval *CaptureInterval +// TagsFilter *TagsFilter +// BboxLabels []string + type DataClient struct { //do we want this to be a public interface that defines the functions but does not include client and private details? //would not include client and private details @@ -183,26 +293,28 @@ func NewDataClient( func (d *DataClient) TabularDataByFilter( //include dest? ctx context.Context, - filter *pb.Filter, //optional - no filter implies all tabular data + // filter *pb.Filter, //optional - no filter implies all tabular data + filter Filter, limit uint64, //optional - max defaults to 50 if unspecified last string, //optional - sortOrder pb.Order, //optional + sortOrder Order, //optional countOnly bool, includeInternalData bool) ([]TabularData, uint64, string, error) { // initialize limit if it's unspecified (zero value) if limit == 0 { limit = 50 } - // ensure filter is not nil to represent a query for "all data" - if filter == nil { - filter = &pb.Filter{} - } + + // // ensure filter is not nil to represent a query for "all data" + // if filter.IsEmpty(){ + // filter = Filter{} //i think if it is empty than it just implies that ALL tabular data?? + // } resp, err := d.client.TabularDataByFilter(ctx, &pb.TabularDataByFilterRequest{ DataRequest: &pb.DataRequest{ //need to do checks to make sure it fits the constraints - Filter: filter, + Filter: FilterToProto(filter), Limit: limit, Last: last, - SortOrder: sortOrder, + SortOrder: OrderToProto(sortOrder), }, CountOnly: countOnly, IncludeInternalData: includeInternalData, @@ -306,7 +418,7 @@ func (d *DataClient) TabularDataByMQL(ctx context.Context, organizationId string func (d *DataClient) BinaryDataByFilter( //dest string?? ctx context.Context, - filter *pb.Filter, + filter Filter, limit uint64, last string, sortOrder pb.Order, @@ -319,12 +431,12 @@ func (d *DataClient) BinaryDataByFilter( limit = 50 } // ensure filter is not nil to represent a query for "all data" - if filter == nil { - filter = &pb.Filter{} - } + // if filter == nil { + // filter = &pb.Filter{} + // } resp, err := d.client.BinaryDataByFilter(ctx, &pb.BinaryDataByFilterRequest{ DataRequest: &pb.DataRequest{ //need to do checks to make sure it fits the constraints - Filter: filter, + Filter: FilterToProto(filter), Limit: limit, Last: last, SortOrder: sortOrder, @@ -360,7 +472,7 @@ func (d *DataClient) BinaryDataByIDs(ctx context.Context, binaryIds []*pb.Binary data[i] = BinaryDataFromProto(protoData) } // return resp.Data, nil - return data, nil + return data, nil //the return type of this is: var data []BinaryData --> is that okay??? , do we only want go native types does this count? } func (d *DataClient) DeleteTabularData(ctx context.Context, organizationId string, deleteOlderThanDays uint32) (uint64, error) { resp, err := d.client.DeleteTabularData(ctx, &pb.DeleteTabularDataRequest{ @@ -373,13 +485,13 @@ func (d *DataClient) DeleteTabularData(ctx context.Context, organizationId strin return resp.DeletedCount, nil } -func (d *DataClient) DeleteBinaryDataByFilter(ctx context.Context, filter *pb.Filter) (uint64, error) { +func (d *DataClient) DeleteBinaryDataByFilter(ctx context.Context, filter Filter) (uint64, error) { //should probably do some sort of check that filter isn't empty otherwise i need to do something - if filter == nil { - filter = &pb.Filter{} - } + // if filter == nil { + // filter = &pb.Filter{} + // } resp, err := d.client.DeleteBinaryDataByFilter(ctx, &pb.DeleteBinaryDataByFilterRequest{ - Filter: filter, + Filter: FilterToProto(filter), IncludeInternalData: true, }) if err != nil { @@ -403,11 +515,11 @@ func (d *DataClient) AddTagsToBinaryDataByIDs(ctx context.Context, tags []string } return nil } -func (d *DataClient) AddTagsToBinaryDataByFilter(ctx context.Context, tags []string, filter *pb.Filter) error { - if filter == nil { - filter = &pb.Filter{} - } - _, err := d.client.AddTagsToBinaryDataByFilter(ctx, &pb.AddTagsToBinaryDataByFilterRequest{Filter: filter, Tags: tags}) +func (d *DataClient) AddTagsToBinaryDataByFilter(ctx context.Context, tags []string, filter Filter) error { + // if filter == nil { + // filter = &pb.Filter{} + // } + _, err := d.client.AddTagsToBinaryDataByFilter(ctx, &pb.AddTagsToBinaryDataByFilterRequest{Filter: FilterToProto(filter), Tags: tags}) if err != nil { return err } @@ -420,21 +532,21 @@ func (d *DataClient) RemoveTagsFromBinaryDataByIDs(ctx context.Context, tags []s } return resp.DeletedCount, nil } -func (d *DataClient) RemoveTagsFromBinaryDataByFilter(ctx context.Context, tags []string, filter *pb.Filter) (uint64, error) { - if filter == nil { - filter = &pb.Filter{} - } - resp, err := d.client.RemoveTagsFromBinaryDataByFilter(ctx, &pb.RemoveTagsFromBinaryDataByFilterRequest{Filter: filter, Tags: tags}) +func (d *DataClient) RemoveTagsFromBinaryDataByFilter(ctx context.Context, tags []string, filter Filter) (uint64, error) { + // if filter == nil { + // filter = &pb.Filter{} + // } + resp, err := d.client.RemoveTagsFromBinaryDataByFilter(ctx, &pb.RemoveTagsFromBinaryDataByFilterRequest{Filter: FilterToProto(filter), Tags: tags}) if err != nil { return 0, err } return resp.DeletedCount, nil } -func (d *DataClient) TagsByFilter(ctx context.Context, filter *pb.Filter) ([]string, error) { - if filter == nil { - filter = &pb.Filter{} - } - resp, err := d.client.TagsByFilter(ctx, &pb.TagsByFilterRequest{Filter: filter}) +func (d *DataClient) TagsByFilter(ctx context.Context, filter Filter) ([]string, error) { + // if filter == nil { + // filter = &pb.Filter{} + // } + resp, err := d.client.TagsByFilter(ctx, &pb.TagsByFilterRequest{Filter: FilterToProto(filter)}) if err != nil { return nil, err } @@ -462,11 +574,11 @@ func (d *DataClient) RemoveBoundingBoxFromImageByID(ctx context.Context, bboxId } return nil } -func (d *DataClient) BoundingBoxLabelsByFilter(ctx context.Context, filter *pb.Filter) ([]string, error) { - if filter == nil { - filter = &pb.Filter{} - } - resp, err := d.client.BoundingBoxLabelsByFilter(ctx, &pb.BoundingBoxLabelsByFilterRequest{Filter: filter}) +func (d *DataClient) BoundingBoxLabelsByFilter(ctx context.Context, filter Filter) ([]string, error) { + // if filter == nil { + // filter = &pb.Filter{} + // } + resp, err := d.client.BoundingBoxLabelsByFilter(ctx, &pb.BoundingBoxLabelsByFilterRequest{Filter: FilterToProto(filter)}) if err != nil { return nil, err } diff --git a/app/data_test.go b/app/data_test.go index f09accce0dc..998196491c1 100644 --- a/app/data_test.go +++ b/app/data_test.go @@ -4,11 +4,13 @@ import ( "context" "fmt" "testing" + "time" "go.mongodb.org/mongo-driver/bson" datapb "go.viam.com/api/app/data/v1" "go.viam.com/test" "google.golang.org/grpc" + "google.golang.org/protobuf/types/known/timestamppb" "go.viam.com/rdk/testutils/inject" ) @@ -35,6 +37,7 @@ import ( func createGrpclient() datapb.DataServiceClient { return &inject.DataServiceClient{} } + func TestDataClient(t *testing.T) { grpcClient := &inject.DataServiceClient{} client := DataClient{client: grpcClient} @@ -53,7 +56,6 @@ func TestDataClient(t *testing.T) { // Data: map[string]interface{} // Metadata: CaptureMetadata // } - // myFilter := create_filter(component_name="motor-1") // myLast := nil // returns: List[TabularData]: The tabular data, int: The count (number of entries), str: The last-returned page ID. @@ -65,7 +67,7 @@ func TestDataClient(t *testing.T) { return &datapb.TabularDataByFilterResponse{Data: _, Count: _, Last: _}, nil } - respData, respCount, respLast, _ := client.TabularDataByFilter(context.Background(), filter, limit, last, sortOrder, countOnly, includeInternalData) + respTabularData, respCount, respLast, _ := client.TabularDataByFilter(context.Background(), filter, limit, last, sortOrder, countOnly, includeInternalData) test.That(t, respData, test.ShouldEqual, expectedData) test.That(t, respCount, test.ShouldEqual, expectedCount) test.That(t, respLast, test.ShouldEqual, expectedLast) @@ -93,6 +95,7 @@ func TestDataClient(t *testing.T) { {"one": 1}, {"list": []string{"a", "b", "c"}}, } + grpcClient.TabularDataByMQLFunc = func(ctx context.Context, in *datapb.TabularDataByMQLRequest, opts ...grpc.CallOption) (*datapb.TabularDataByMQLResponse, error) { test.That(t, in.OrganizationId, test.ShouldEqual, org_id) @@ -106,9 +109,60 @@ func TestDataClient(t *testing.T) { fmt.Printf("Actual : %#v\n", response) test.That(t, response, test.ShouldResemble, expected) //*this test not working as expected yet! }) - t.Run("TabularDataByFilter", func(t *testing.T) { + t.Run("BinaryDataByFilter", func(t *testing.T) { }) + t.Run("BinaryDataByIDs", func(t *testing.T) { + binaryID1 := BinaryID{ //can this just be of type BinaryID??? why proto type? + FileId: "file1", + OrganizationId: "org1", + LocationId: "loc1", + } + // ==> why would it have to be of type datapb??? cant it jsut be of BinaryID + binaryID2 := BinaryID{ + FileId: "file2", + OrganizationId: "org1", + LocationId: "loc1", + } + binaryIDs := []BinaryID{binaryID1, binaryID2} + + // binaryMetadata that acts as input! + metadata := BinaryMetadata{ + ID: "id1", + CaptureMetadata: CaptureMetadata{ /* I probs need to fill these fields in???*/ }, + TimeRequested: time.Now(), + TimeReceived: time.Now(), + FileName: "file.txt", + FileExt: ".txt", + URI: "http://ex/file.txt", //can i just make this up?? + Annotations: Annotations{ /* this too???*/ }, + DatasetIDs: []string{"dataset1", "dataset2"}, + } + + binary := []byte("something") + // binaryData that acts as input + binaryData := BinaryData{ + Binary: binary, + Metadata: metadata, + } + binaryDataList := []BinaryData{binaryData} + + grpcClient.BinaryDataByIDsFunc = func(ctx context.Context, in *datapb.BinaryDataByIDsRequest, opts ...grpc.CallOption) (*datapb.BinaryDataByIDsResponse, error) { + test.That(t, in.BinaryIds, test.ShouldEqual, binaryIDs) + return &datapb.BinaryDataByIDsResponse{Data: binaryDataList, Count: uint64(len(binaryDataList))}, nil + } + response, _ := client.BinaryDataByIDs(context.Background(), binaryIDs) + + // expectedData := BinaryData{binary: binary, Metadata: metadata} + // var expectedData []BinaryData{BinaryDataFromProto(binaryDataList)} + var expectedData []BinaryData + for _, binaryDataItem := range binaryDataList { + convertedData := BinaryDataFromProto(binaryDataItem) + expectedData = append(expectedData, convertedData) + } + //loop thru binary dataDataList and convert binaryDataFromproto each one---> and then add that to a list...! + test.That(t, response, test.ShouldResemble, expectedData) + }) } @@ -219,3 +273,14 @@ func TestDataClient(t *testing.T) { // conn, err := viamgrpc.Dial(context.Background(), listener.Addr().String(), logger) // test.That(t, err, test.ShouldBeNil) // client := datapb.NewDataServiceClient(conn) + +//notes on the param chekcing stuff + // var request *datapb.TabularDataByMQLRequest + + // grpcClient.TabularDataByMQLFunc = func(ctx context.Context, in *datapb.TabularDataByMQLRequest, opts ...grpc.CallOption) (*datapb.TabularDataByMQLResponse, error) { + // request = in + // return &datapb.TabularDataByMQLResponse{RawData: mql_binary}, nil //MQlResponse is created with BSON data + // } + + // assert request.org + From b9c3e4f84141d633d7f5bbe6e5a5cc67a882b900 Mon Sep 17 00:00:00 2001 From: Julie Krasnick Date: Thu, 7 Nov 2024 14:10:09 -0500 Subject: [PATCH 10/25] added toproto funcs --- app/data_client.go | 56 +++++++++++++++++++++++++++++----------------- 1 file changed, 36 insertions(+), 20 deletions(-) diff --git a/app/data_client.go b/app/data_client.go index 353ae6372c8..e392b553f17 100644 --- a/app/data_client.go +++ b/app/data_client.go @@ -202,6 +202,22 @@ func BinaryMetadataFromProto(proto *pb.BinaryMetadata) BinaryMetadata { } } +func BinaryIdToProto(binaryId BinaryID) *pb.BinaryID { + return &pb.BinaryID{ + FileId: binaryId.FileId, + OrganizationId: binaryId.OrganizationId, + LocationId: binaryId.LocationId, + } +} + +func BinaryIdsToProto(binaryIds []BinaryID) []*pb.BinaryID { + var protoBinaryIds []*pb.BinaryID + for _, binaryId := range binaryIds { + protoBinaryIds = append(protoBinaryIds, BinaryIdToProto(binaryId)) + } + return protoBinaryIds +} + // PropertiesToProtoResponse takes a map of features to struct and converts it // to a GetPropertiesResponse. func FilterToProto(filter Filter) *pb.Filter { @@ -421,7 +437,7 @@ func (d *DataClient) BinaryDataByFilter( filter Filter, limit uint64, last string, - sortOrder pb.Order, + sortOrder Order, includeBinary bool, countOnly bool, // includeInternalData bool) ([]*pb.BinaryData, uint64, string, error) { @@ -439,7 +455,7 @@ func (d *DataClient) BinaryDataByFilter( Filter: FilterToProto(filter), Limit: limit, Last: last, - SortOrder: sortOrder, + SortOrder: OrderToProto(sortOrder), }, CountOnly: countOnly, IncludeInternalData: includeInternalData, @@ -458,10 +474,10 @@ func (d *DataClient) BinaryDataByFilter( } // do i need to be including error as a return type for all of these? -func (d *DataClient) BinaryDataByIDs(ctx context.Context, binaryIds []*pb.BinaryID) ([]BinaryData, error) { +func (d *DataClient) BinaryDataByIDs(ctx context.Context, binaryIds []BinaryID) ([]BinaryData, error) { resp, err := d.client.BinaryDataByIDs(ctx, &pb.BinaryDataByIDsRequest{ IncludeBinary: true, - BinaryIds: binaryIds, + BinaryIds: BinaryIdsToProto(binaryIds), }) if err != nil { return nil, err @@ -499,17 +515,17 @@ func (d *DataClient) DeleteBinaryDataByFilter(ctx context.Context, filter Filter } return resp.DeletedCount, nil } -func (d *DataClient) DeleteBinaryDataByIDs(ctx context.Context, binaryIds []*pb.BinaryID) (uint64, error) { +func (d *DataClient) DeleteBinaryDataByIDs(ctx context.Context, binaryIds []BinaryID) (uint64, error) { resp, err := d.client.DeleteBinaryDataByIDs(ctx, &pb.DeleteBinaryDataByIDsRequest{ - BinaryIds: binaryIds, + BinaryIds: BinaryIdsToProto(binaryIds), }) if err != nil { return 0, err } return resp.DeletedCount, nil } -func (d *DataClient) AddTagsToBinaryDataByIDs(ctx context.Context, tags []string, binaryIds []*pb.BinaryID) error { - _, err := d.client.AddTagsToBinaryDataByIDs(ctx, &pb.AddTagsToBinaryDataByIDsRequest{BinaryIds: binaryIds, Tags: tags}) +func (d *DataClient) AddTagsToBinaryDataByIDs(ctx context.Context, tags []string, binaryIds []BinaryID) error { + _, err := d.client.AddTagsToBinaryDataByIDs(ctx, &pb.AddTagsToBinaryDataByIDsRequest{BinaryIds: BinaryIdsToProto(binaryIds), Tags: tags}) if err != nil { return err } @@ -525,8 +541,8 @@ func (d *DataClient) AddTagsToBinaryDataByFilter(ctx context.Context, tags []str } return nil } -func (d *DataClient) RemoveTagsFromBinaryDataByIDs(ctx context.Context, tags []string, binaryIds []*pb.BinaryID) (uint64, error) { - resp, err := d.client.RemoveTagsFromBinaryDataByIDs(ctx, &pb.RemoveTagsFromBinaryDataByIDsRequest{BinaryIds: binaryIds, Tags: tags}) +func (d *DataClient) RemoveTagsFromBinaryDataByIDs(ctx context.Context, tags []string, binaryIds []BinaryID) (uint64, error) { + resp, err := d.client.RemoveTagsFromBinaryDataByIDs(ctx, &pb.RemoveTagsFromBinaryDataByIDsRequest{BinaryIds: BinaryIdsToProto(binaryIds), Tags: tags}) if err != nil { return 0, err } @@ -554,21 +570,21 @@ func (d *DataClient) TagsByFilter(ctx context.Context, filter Filter) ([]string, } func (d *DataClient) AddBoundingBoxToImageByID( ctx context.Context, - binaryIds []*pb.BinaryID, + binaryId BinaryID, label string, xMinNormalized float64, yMinNormalized float64, xMaxNormalized float64, yMaxNormalized float64) (string, error) { - resp, err := d.client.AddBoundingBoxToImageByID(ctx, &pb.AddBoundingBoxToImageByIDRequest{BinaryId: &pb.BinaryID{}, Label: label, XMinNormalized: xMinNormalized, YMinNormalized: yMinNormalized, XMaxNormalized: xMaxNormalized, YMaxNormalized: yMaxNormalized}) + resp, err := d.client.AddBoundingBoxToImageByID(ctx, &pb.AddBoundingBoxToImageByIDRequest{BinaryId: BinaryIdToProto(binaryId), Label: label, XMinNormalized: xMinNormalized, YMinNormalized: yMinNormalized, XMaxNormalized: xMaxNormalized, YMaxNormalized: yMaxNormalized}) if err != nil { return "", err } return resp.BboxId, nil } -func (d *DataClient) RemoveBoundingBoxFromImageByID(ctx context.Context, bboxId string, binaryId *pb.BinaryID) error { - _, err := d.client.RemoveBoundingBoxFromImageByID(ctx, &pb.RemoveBoundingBoxFromImageByIDRequest{BinaryId: binaryId, BboxId: bboxId}) +func (d *DataClient) RemoveBoundingBoxFromImageByID(ctx context.Context, bboxId string, binaryId BinaryID) error { + _, err := d.client.RemoveBoundingBoxFromImageByID(ctx, &pb.RemoveBoundingBoxFromImageByIDRequest{BinaryId: BinaryIdToProto(binaryId), BboxId: bboxId}) if err != nil { return err } @@ -587,7 +603,7 @@ func (d *DataClient) BoundingBoxLabelsByFilter(ctx context.Context, filter Filte // ***python and typescript did not implement this one!!! func (d *DataClient) UpdateBoundingBox(ctx context.Context, - binaryId *pb.BinaryID, + binaryId BinaryID, bboxId string, label string, xMinNormalized float64, @@ -595,7 +611,7 @@ func (d *DataClient) UpdateBoundingBox(ctx context.Context, xMaxNormalized float64, yMaxNormalized float64) error { - _, err := d.client.UpdateBoundingBox(ctx, &pb.UpdateBoundingBoxRequest{BinaryId: binaryId, BboxId: bboxId, Label: &label, XMinNormalized: &xMinNormalized, YMinNormalized: &yMinNormalized, XMaxNormalized: &xMaxNormalized, YMaxNormalized: &yMaxNormalized}) + _, err := d.client.UpdateBoundingBox(ctx, &pb.UpdateBoundingBoxRequest{BinaryId: BinaryIdToProto(binaryId), BboxId: bboxId, Label: &label, XMinNormalized: &xMinNormalized, YMinNormalized: &yMinNormalized, XMaxNormalized: &xMaxNormalized, YMaxNormalized: &yMaxNormalized}) if err != nil { return err } @@ -617,15 +633,15 @@ func (d *DataClient) ConfigureDatabaseUser(ctx context.Context, organizationId s } return nil } -func (d *DataClient) AddBinaryDataToDatasetByIDs(ctx context.Context, binaryIds []*pb.BinaryID, datasetId string) error { - _, err := d.client.AddBinaryDataToDatasetByIDs(ctx, &pb.AddBinaryDataToDatasetByIDsRequest{BinaryIds: binaryIds, DatasetId: datasetId}) +func (d *DataClient) AddBinaryDataToDatasetByIDs(ctx context.Context, binaryIds []BinaryID, datasetId string) error { + _, err := d.client.AddBinaryDataToDatasetByIDs(ctx, &pb.AddBinaryDataToDatasetByIDsRequest{BinaryIds: BinaryIdsToProto(binaryIds), DatasetId: datasetId}) if err != nil { return err } return nil } -func (d *DataClient) RemoveBinaryDataFromDatasetByIDs(ctx context.Context, binaryIds []*pb.BinaryID, datasetId string) error { - _, err := d.client.RemoveBinaryDataFromDatasetByIDs(ctx, &pb.RemoveBinaryDataFromDatasetByIDsRequest{BinaryIds: binaryIds, DatasetId: datasetId}) +func (d *DataClient) RemoveBinaryDataFromDatasetByIDs(ctx context.Context, binaryIds []BinaryID, datasetId string) error { + _, err := d.client.RemoveBinaryDataFromDatasetByIDs(ctx, &pb.RemoveBinaryDataFromDatasetByIDsRequest{BinaryIds: BinaryIdsToProto(binaryIds), DatasetId: datasetId}) if err != nil { return err } From 9a46735d16709ba5a92175e51bef82581aca4035 Mon Sep 17 00:00:00 2001 From: Julie Krasnick Date: Thu, 7 Nov 2024 15:33:41 -0500 Subject: [PATCH 11/25] lots of clean up --- app/data_client.go | 134 +++++++++++++++++++-------------------------- 1 file changed, 56 insertions(+), 78 deletions(-) diff --git a/app/data_client.go b/app/data_client.go index e392b553f17..02d831716c7 100644 --- a/app/data_client.go +++ b/app/data_client.go @@ -4,13 +4,13 @@ package app import ( "context" - "fmt" "time" "go.mongodb.org/mongo-driver/bson" pb "go.viam.com/api/app/data/v1" "go.viam.com/rdk/logging" "go.viam.com/utils/rpc" + "google.golang.org/protobuf/types/known/anypb" "google.golang.org/protobuf/types/known/structpb" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -62,6 +62,7 @@ type Annotations struct { type TabularData struct { Data map[string]interface{} // Metadata *pb.CaptureMetadata //idk why i put a star here -- if we aren't returning it is it okay? + MetadataIndex uint32 Metadata CaptureMetadata //idk why i put a star here TimeRequested time.Time TimeReceived time.Time @@ -156,16 +157,6 @@ func CaptureMetadataFromProto(proto *pb.CaptureMetadata) CaptureMetadata { if proto == nil { return CaptureMetadata{} } - // Convert method parameters from protobuf to native map - methodParameters := make(map[string]interface{}) - // Convert MethodParameters map[string]*anypb.Any to map[string]interface{} - for key, value := range proto.MethodParameters { - structValue := &structpb.Value{} - if err := value.UnmarshalTo(structValue); err != nil { - return CaptureMetadata{} // return error?? - } - methodParameters[key] = structValue.AsInterface() - } return CaptureMetadata{ organization_id: proto.OrganizationId, location_id2: proto.LocationId, @@ -176,11 +167,25 @@ func CaptureMetadataFromProto(proto *pb.CaptureMetadata) CaptureMetadata { component_type: proto.ComponentType, component_name: proto.ComponentName, method_name: proto.MethodName, - method_parameters: methodParameters, + method_parameters: methodParamsFromProto(proto.MethodParameters), tags: proto.Tags, // repeated string mime_type: proto.MimeType, } } + +func methodParamsFromProto(proto map[string]*anypb.Any) map[string]interface{} { + // Convert MethodParameters map[string]*anypb.Any to map[string]interface{} + methodParameters := make(map[string]interface{}) + for key, value := range proto { + structValue := &structpb.Value{} + if err := value.UnmarshalTo(structValue); err != nil { + return nil + } + methodParameters[key] = structValue.AsInterface() + } + return methodParameters +} + func BinaryDataFromProto(proto *pb.BinaryData) BinaryData { return BinaryData{ Binary: proto.Binary, @@ -202,6 +207,27 @@ func BinaryMetadataFromProto(proto *pb.BinaryMetadata) BinaryMetadata { } } +// returns tabular data and the associated metadata +func TabularDataFromProto(proto *pb.TabularData, metadata *pb.CaptureMetadata) TabularData { + return TabularData{ + Data: proto.Data.AsMap(), //we have data as this when it is is in non proto ==> map[string]interface{} + MetadataIndex: proto.MetadataIndex, + Metadata: CaptureMetadataFromProto(metadata), + TimeRequested: proto.TimeRequested.AsTime(), + TimeReceived: proto.TimeReceived.AsTime(), + } +} +func TabularDataBsonHelper(rawData [][]byte) ([]map[string]interface{}, error) { + dataObjects := make([]map[string]interface{}, len(rawData)) + // Loop over each BSON byte array in the response and unmarshal directly into the dataObjects slice + for _, rawData := range rawData { + obj := make(map[string]interface{}) + bson.Unmarshal(rawData, &obj) + dataObjects = append(dataObjects, obj) + } + return dataObjects, nil +} + func BinaryIdToProto(binaryId BinaryID) *pb.BinaryID { return &pb.BinaryID{ FileId: binaryId.FileId, @@ -307,57 +333,44 @@ func NewDataClient( // TabularDataByFilter queries tabular data and metadata based on given filters. // returns []TabularData, uint64, string, and error: returns multiple things containing the following: List[TabularData]: The tabular data, int: The count (number of entries), str: The last-returned page ID. func (d *DataClient) TabularDataByFilter( - //include dest? ctx context.Context, - // filter *pb.Filter, //optional - no filter implies all tabular data filter Filter, limit uint64, //optional - max defaults to 50 if unspecified last string, //optional sortOrder Order, //optional countOnly bool, includeInternalData bool) ([]TabularData, uint64, string, error) { - // initialize limit if it's unspecified (zero value) if limit == 0 { limit = 50 } - // // ensure filter is not nil to represent a query for "all data" // if filter.IsEmpty(){ // filter = Filter{} //i think if it is empty than it just implies that ALL tabular data?? // } resp, err := d.client.TabularDataByFilter(ctx, &pb.TabularDataByFilterRequest{ - DataRequest: &pb.DataRequest{ //need to do checks to make sure it fits the constraints + DataRequest: &pb.DataRequest{ Filter: FilterToProto(filter), Limit: limit, Last: last, - SortOrder: OrderToProto(sortOrder), - }, + SortOrder: OrderToProto(sortOrder)}, CountOnly: countOnly, IncludeInternalData: includeInternalData, }) - //do we want this? if err != nil { return nil, 0, "", err } - //need to undo the repeated tabularData in resp.Data and return it + //get the tabulardata --> get metadataIndex from tabularData, + //get metadata, --> use metadata[tabularData.MetdataIndex] to get the associated metadata with that tabular data!! + //then return tabularData with that metadata!! dataArray := make([]TabularData, len(resp.Data)) - for i, data := range resp.Data { - mdIndex := data.MetadataIndex - // var metadata *pb.CaptureMetadata - var metadata CaptureMetadata - //is this check necessary?? - // Ensure the metadata index is within bounds - if len(resp.Metadata) != 0 && int(mdIndex) < len(resp.Metadata) { - metadata = CaptureMetadataFromProto(resp.Metadata[mdIndex]) - } - //creating a list of tabularData - dataArray[i] = TabularData{ - Data: data.Data.AsMap(), - Metadata: metadata, - TimeRequested: data.TimeRequested.AsTime(), - TimeReceived: data.TimeReceived.AsTime(), + for _, data := range resp.Data { + var metadata *pb.CaptureMetadata + if int(data.MetadataIndex) < len(resp.Metadata) { + metadata = resp.Metadata[data.MetadataIndex] // Access the correct metadata using MetadataIndex } + dataArray = append(dataArray, TabularDataFromProto(data, metadata)) //the metadata that we pass has already been indexed } + //// TabularData contains data and metadata associated with tabular data. return dataArray, resp.Count, resp.Last, nil } @@ -370,15 +383,7 @@ func (d *DataClient) TabularDataBySQL(ctx context.Context, organizationId string if err != nil { return nil, err } - // Initialize a an array of maps to hold the data objects (in python we had list of dicts) - dataObjects := make([]map[string]interface{}, len(resp.RawData)) - // Loop over each BSON byte array in the response and unmarshal directly into the dataObjects slice - for i, rawData := range resp.RawData { - obj := make(map[string]interface{}) - bson.Unmarshal(rawData, &obj) - //do we want an error message for bson.Unmarshal...? - dataObjects[i] = obj - } + dataObjects, nil := TabularDataBsonHelper(resp.RawData) return dataObjects, nil } @@ -388,47 +393,20 @@ func (d *DataClient) TabularDataByMQL(ctx context.Context, organizationId string if err != nil { return nil, err } - // Debugging output to verify RawData content - fmt.Printf("Response RawData: %v\n", resp.RawData) - //loop thru rawData - //for each rawData byte slice you will need to unmarshall it into map[string]interface - //then add each unmarshalled map to a list and return it - dataObjects := make([]map[string]interface{}, len(resp.RawData)) - for i, rawData := range resp.RawData { - var obj map[string]interface{} - if err := bson.Unmarshal(rawData, &obj); err != nil { - fmt.Printf("(func) unmarshalling error %d: %v", i, err) - return nil, err - } - dataObjects[i] = obj - } - // dataObjects := make([]map[string]interface{}, len(resp.RawData)) - // for i, rawData := range resp.RawData { - // obj := make(map[string]interface{}) - // bson.Unmarshal(rawData, &obj) - // dataObjects[i] = obj - // } - // Unmarshal all raw data at once as an array of maps - // var dataObjects []map[string]interface{} - // for _, rawData := range resp.RawData { - // var singleData []map[string]interface{} // This should match your expected structure - // if err := bson.Unmarshal(rawData, &singleData); err != nil { - // return nil, err - // } - // dataObjects = append(dataObjects, singleData...) - // } - // Unmarshal each raw data entry as a separate map + dataObjects, nil := TabularDataBsonHelper(resp.RawData) + return dataObjects, nil + // dataObjects := make([]map[string]interface{}, len(resp.RawData)) // for i, rawData := range resp.RawData { // var obj map[string]interface{} // if err := bson.Unmarshal(rawData, &obj); err != nil { - // fmt.Printf("Unmarshalling error at index %d: %v\n", i, err) + // fmt.Printf("(func) unmarshalling error %d: %v", i, err) // return nil, err // } // dataObjects[i] = obj // } - fmt.Println("printing Deserialized dataObjects here", dataObjects) - return dataObjects, nil + + // fmt.Println("printing Deserialized dataObjects here", dataObjects) } func (d *DataClient) BinaryDataByFilter( From 93089dcac3488bb7d8cdfda31a90f4878a442970 Mon Sep 17 00:00:00 2001 From: Julie Krasnick Date: Thu, 7 Nov 2024 17:58:42 -0500 Subject: [PATCH 12/25] one test --- app/data_client.go | 90 +++++++++++++++++++---------- app/data_test.go | 141 +++++++++++++++++++++++++++++++++------------ 2 files changed, 162 insertions(+), 69 deletions(-) diff --git a/app/data_client.go b/app/data_client.go index 02d831716c7..261d1912fb0 100644 --- a/app/data_client.go +++ b/app/data_client.go @@ -9,6 +9,7 @@ import ( "go.mongodb.org/mongo-driver/bson" pb "go.viam.com/api/app/data/v1" "go.viam.com/rdk/logging" + utils "go.viam.com/utils/protoutils" "go.viam.com/utils/rpc" "google.golang.org/protobuf/types/known/anypb" "google.golang.org/protobuf/types/known/structpb" @@ -24,20 +25,20 @@ import ( // i want to wrap NewDataServiceClient define a new dataclient struct and call the wrappers of the functions // // i want the user to call my dataClient struct w my wrappers and not the proto functions type CaptureMetadata struct { - organization_id string - location_id2 string - robot_name string - robot_id string - part_name string - part_id string - component_type string - component_name string - method_name string - method_parameters map[string]interface{} + OrganizationId string + LocationId string + RobotName string + RobotId string + PartName string + PartId string + ComponentType string + ComponentName string + MethodName string + MethodParameters map[string]interface{} //^^ supposed to be: map method_parameters = 11; - tags []string + Tags []string //^^ repeated string tags = 12; - mime_type string + MimeType string //^^ string mime_type = 13; } type BinaryID struct { @@ -53,6 +54,12 @@ type BoundingBox struct { x_max_normalized float64 y_max_normalized float64 } +type DataRequest struct { + Filter Filter + Limit uint64 + Last string + SortOrder Order +} // Annotations are data annotations used for machine learning. type Annotations struct { @@ -158,18 +165,18 @@ func CaptureMetadataFromProto(proto *pb.CaptureMetadata) CaptureMetadata { return CaptureMetadata{} } return CaptureMetadata{ - organization_id: proto.OrganizationId, - location_id2: proto.LocationId, - robot_name: proto.RobotName, - robot_id: proto.RobotId, - part_name: proto.PartName, - part_id: proto.PartId, - component_type: proto.ComponentType, - component_name: proto.ComponentName, - method_name: proto.MethodName, - method_parameters: methodParamsFromProto(proto.MethodParameters), - tags: proto.Tags, // repeated string - mime_type: proto.MimeType, + OrganizationId: proto.OrganizationId, + LocationId: proto.LocationId, + RobotName: proto.RobotName, + RobotId: proto.RobotId, + PartName: proto.PartName, + PartId: proto.PartId, + ComponentType: proto.ComponentType, + ComponentName: proto.ComponentName, + MethodName: proto.MethodName, + MethodParameters: methodParamsFromProto(proto.MethodParameters), + Tags: proto.Tags, // repeated string + MimeType: proto.MimeType, } } @@ -217,6 +224,34 @@ func TabularDataFromProto(proto *pb.TabularData, metadata *pb.CaptureMetadata) T TimeReceived: proto.TimeReceived.AsTime(), } } +func TabularDataToProto(tabularData TabularData) *pb.TabularData { + structData, err := utils.StructToStructPb(tabularData.Data) + if err != nil { + return nil + } + timeRequested := timestamppb.New(tabularData.TimeRequested) + timeReceived := timestamppb.New(tabularData.TimeReceived) + return &pb.TabularData{ + Data: structData, + MetadataIndex: tabularData.MetadataIndex, + TimeRequested: timeRequested, + TimeReceived: timeReceived, + } +} +func TabularDataToProtoList(tabularDatas []TabularData) []*pb.TabularData { + var protoList []*pb.TabularData + // loop through each TabularData item + for _, tabularData := range tabularDatas { + // convert each TabularData to *pb.TabularData + protoData := TabularDataToProto(tabularData) + if protoData != nil { + // append the converted *pb.TabularData to the list + protoList = append(protoList, protoData) + } + } + // return the list of converted *pb.TabularData items + return protoList +} func TabularDataBsonHelper(rawData [][]byte) ([]map[string]interface{}, error) { dataObjects := make([]map[string]interface{}, len(rawData)) // Loop over each BSON byte array in the response and unmarshal directly into the dataObjects slice @@ -299,13 +334,6 @@ func OrderToProto(sortOrder Order) pb.Order { } } -// LocationIds []string -// OrganizationIds []string -// MimeType []string -// Interval *CaptureInterval -// TagsFilter *TagsFilter -// BboxLabels []string - type DataClient struct { //do we want this to be a public interface that defines the functions but does not include client and private details? //would not include client and private details diff --git a/app/data_test.go b/app/data_test.go index 998196491c1..ac5e5f00821 100644 --- a/app/data_test.go +++ b/app/data_test.go @@ -10,14 +10,36 @@ import ( datapb "go.viam.com/api/app/data/v1" "go.viam.com/test" "google.golang.org/grpc" - "google.golang.org/protobuf/types/known/timestamppb" "go.viam.com/rdk/testutils/inject" ) -// var ( -// orgId = "e76d1b3b-0468-4efd-bb7f-fb1d2b352fcb" -// ) +const ( + component_name = "component_name" + component_type = "component_type" + method = "method" + robot_name = "robot_name" + robot_id = "robot_id" + part_name = "part_name" + part_id = "part_id" + location_id = "location_id" + organization_id = "organization_id" + // password = "password" + mime_type = "mime_type" + uri = "some.robot.uri" + bbox_label = "bbox_label" + dataset_id = "dataset_id" + tag = "tag" +) + +var ( + location_IDs = []string{location_id} + org_IDs = []string{organization_id} + mime_Types = []string{mime_type} + Bbox_Labels = []string{bbox_label} + method_parameters = map[string]interface{}{} + tags = []string{tag} +) // // Helper function to create BSON documents for MongoDB queries // func createMQLBSON() [][]byte { @@ -41,38 +63,83 @@ func createGrpclient() datapb.DataServiceClient { func TestDataClient(t *testing.T) { grpcClient := &inject.DataServiceClient{} client := DataClient{client: grpcClient} + + capture_interval := CaptureInterval{ + Start: time.Now(), + End: time.Now(), + } + tags_filter := TagsFilter{ + Type: 2, + Tags: []string{"tag1", "tag2"}, + } + + myFilter := Filter{ + ComponentName: component_name, + ComponentType: component_type, + Method: method, + RobotName: robot_name, + RobotId: robot_id, + PartName: part_name, + PartId: part_id, + LocationIds: location_IDs, + OrganizationIds: org_IDs, + MimeType: mime_Types, + Interval: capture_interval, + TagsFilter: tags_filter, //asterix or no?? + BboxLabels: Bbox_Labels, + DatasetId: dataset_id, + } + myTabularMetadata := CaptureMetadata{ + OrganizationId: organization_id, + LocationId: location_id, + RobotName: robot_name, + RobotId: robot_id, + PartName: part_name, + PartId: part_id, + ComponentType: component_type, + ComponentName: component_name, + MethodName: method, + MethodParameters: method_parameters, + Tags: tags, + MimeType: mime_type, + } t.Run("TabularDataByFilter", func(t *testing.T) { - // expected := {} - // expectedData := "" //change - not done - // expectedCount := 5 //change - not done - // expectedLast := "" //change - not done - // filter := &datapb.Filter{} - // limit := uint64(5) - // last := "last" - // sortOrder := datapb.Order_ORDER_DESCENDING //i think this is correct - // countOnly := true - // includeInternalData := true - // myData := { - // Data: map[string]interface{} - // Metadata: CaptureMetadata - // } - // myLast := nil - - // returns: List[TabularData]: The tabular data, int: The count (number of entries), str: The last-returned page ID. + myDataRequest := DataRequest{ + Filter: myFilter, + Limit: 5, + Last: "last", + SortOrder: Unspecified, + } + myTabularData := TabularData{ + Data: map[string]interface{}{ + "key": "value", + }, + MetadataIndex: 0, + Metadata: myTabularMetadata, //not sure if i need to pass this + TimeRequested: time.Now(), + TimeReceived: time.Now(), + } + myTabularDatas := []TabularData{myTabularData} + myCount := uint64(5) + myLast := "last" + myLimit := uint64(100) + countOnly := true + includeInternalData := true grpcClient.TabularDataByFilterFunc = func(ctx context.Context, in *datapb.TabularDataByFilterRequest, opts ...grpc.CallOption) (*datapb.TabularDataByFilterResponse, error) { - test.That(t, in.DataRequest, test.ShouldEqual) + test.That(t, in.DataRequest, test.ShouldEqual, myDataRequest) test.That(t, in.CountOnly, test.ShouldBeTrue) test.That(t, in.IncludeInternalData, test.ShouldBeTrue) - return &datapb.TabularDataByFilterResponse{Data: _, Count: _, Last: _}, nil + return &datapb.TabularDataByFilterResponse{Data: TabularDataToProtoList(myTabularDatas), Count: myCount, Last: myLast}, nil + //the only returns we care about are TabularData, int coint, and str last rewturned pg id } - respTabularData, respCount, respLast, _ := client.TabularDataByFilter(context.Background(), filter, limit, last, sortOrder, countOnly, includeInternalData) - test.That(t, respData, test.ShouldEqual, expectedData) - test.That(t, respCount, test.ShouldEqual, expectedCount) - test.That(t, respLast, test.ShouldEqual, expectedLast) - + respTabularData, respCount, respLast, _ := client.TabularDataByFilter(context.Background(), myFilter, myLimit, myLast, myDataRequest.SortOrder, countOnly, includeInternalData) + test.That(t, respTabularData, test.ShouldEqual, myTabularData) + test.That(t, respCount, test.ShouldEqual, myCount) + test.That(t, respLast, test.ShouldEqual, myLast) }) + t.Run("TabularDataBySQL", func(t *testing.T) { }) @@ -95,7 +162,6 @@ func TestDataClient(t *testing.T) { {"one": 1}, {"list": []string{"a", "b", "c"}}, } - grpcClient.TabularDataByMQLFunc = func(ctx context.Context, in *datapb.TabularDataByMQLRequest, opts ...grpc.CallOption) (*datapb.TabularDataByMQLResponse, error) { test.That(t, in.OrganizationId, test.ShouldEqual, org_id) @@ -154,8 +220,8 @@ func TestDataClient(t *testing.T) { response, _ := client.BinaryDataByIDs(context.Background(), binaryIDs) // expectedData := BinaryData{binary: binary, Metadata: metadata} - // var expectedData []BinaryData{BinaryDataFromProto(binaryDataList)} - var expectedData []BinaryData + // var expectedData []BinaryData{BinaryDataFromProto(binaryDataList)} + var expectedData []BinaryData for _, binaryDataItem := range binaryDataList { convertedData := BinaryDataFromProto(binaryDataItem) expectedData = append(expectedData, convertedData) @@ -275,12 +341,11 @@ func TestDataClient(t *testing.T) { // client := datapb.NewDataServiceClient(conn) //notes on the param chekcing stuff - // var request *datapb.TabularDataByMQLRequest +// var request *datapb.TabularDataByMQLRequest - // grpcClient.TabularDataByMQLFunc = func(ctx context.Context, in *datapb.TabularDataByMQLRequest, opts ...grpc.CallOption) (*datapb.TabularDataByMQLResponse, error) { - // request = in - // return &datapb.TabularDataByMQLResponse{RawData: mql_binary}, nil //MQlResponse is created with BSON data - // } +// grpcClient.TabularDataByMQLFunc = func(ctx context.Context, in *datapb.TabularDataByMQLRequest, opts ...grpc.CallOption) (*datapb.TabularDataByMQLResponse, error) { +// request = in +// return &datapb.TabularDataByMQLResponse{RawData: mql_binary}, nil //MQlResponse is created with BSON data +// } - // assert request.org - +// assert request.org From e8380d8cbbc55104ed8a713a0efaf48dd1660ef8 Mon Sep 17 00:00:00 2001 From: Julie Krasnick Date: Mon, 11 Nov 2024 16:49:45 -0500 Subject: [PATCH 13/25] more changes --- app/data_client.go | 178 +++++++++---- app/data_test.go | 649 ++++++++++++++++++++++++++++++++++----------- 2 files changed, 628 insertions(+), 199 deletions(-) diff --git a/app/data_client.go b/app/data_client.go index 261d1912fb0..cbf93b877de 100644 --- a/app/data_client.go +++ b/app/data_client.go @@ -4,11 +4,13 @@ package app import ( "context" + "fmt" "time" "go.mongodb.org/mongo-driver/bson" pb "go.viam.com/api/app/data/v1" "go.viam.com/rdk/logging" + "go.viam.com/rdk/protoutils" utils "go.viam.com/utils/protoutils" "go.viam.com/utils/rpc" "google.golang.org/protobuf/types/known/anypb" @@ -34,7 +36,7 @@ type CaptureMetadata struct { ComponentType string ComponentName string MethodName string - MethodParameters map[string]interface{} + MethodParameters map[string]string //^^ supposed to be: map method_parameters = 11; Tags []string //^^ repeated string tags = 12; @@ -47,12 +49,12 @@ type BinaryID struct { LocationId string } type BoundingBox struct { - id string - label string - x_min_normalized float64 //should be double but no doubles in go - y_min_normalized float64 - x_max_normalized float64 - y_max_normalized float64 + id string + label string + xMinNormalized float64 //should be double but no doubles in go + yMinNormalized float64 + xMaxNormalized float64 + yMaxNormalized float64 } type DataRequest struct { Filter Filter @@ -138,12 +140,12 @@ type CaptureInterval struct { func BoundingBoxFromProto(proto *pb.BoundingBox) BoundingBox { return BoundingBox{ - id: proto.Id, - label: proto.Label, - x_min_normalized: proto.XMinNormalized, // cast if i want int, or use float64 for precision - y_min_normalized: proto.YMinNormalized, - x_max_normalized: proto.XMaxNormalized, - y_max_normalized: proto.YMaxNormalized, + id: proto.Id, + label: proto.Label, + xMinNormalized: proto.XMinNormalized, // cast if i want int, or use float64 for precision + yMinNormalized: proto.YMinNormalized, + xMaxNormalized: proto.XMaxNormalized, + yMaxNormalized: proto.YMaxNormalized, } } func AnnotationsFromProto(proto *pb.Annotations) Annotations { @@ -159,11 +161,31 @@ func AnnotationsFromProto(proto *pb.Annotations) Annotations { bboxes: bboxes, } } - +func AnnotationsToProto(annotations Annotations) *pb.Annotations { + // Convert each BoundingBox in annotations to its proto type + var protoBboxes []*pb.BoundingBox + for _, bbox := range annotations.bboxes { + protoBboxes = append(protoBboxes, &pb.BoundingBox{ + Id: bbox.id, + Label: bbox.label, + XMinNormalized: bbox.xMinNormalized, + YMinNormalized: bbox.yMinNormalized, + XMaxNormalized: bbox.xMaxNormalized, + YMaxNormalized: bbox.yMaxNormalized, + }) + } + // Return the proto Annotations with converted bounding boxes + return &pb.Annotations{ + Bboxes: protoBboxes, + } +} func CaptureMetadataFromProto(proto *pb.CaptureMetadata) CaptureMetadata { + // fmt.Printf("this is proto in capturemetadata proto: %+v\n", proto) + if proto == nil { return CaptureMetadata{} } + return CaptureMetadata{ OrganizationId: proto.OrganizationId, LocationId: proto.LocationId, @@ -178,17 +200,37 @@ func CaptureMetadataFromProto(proto *pb.CaptureMetadata) CaptureMetadata { Tags: proto.Tags, // repeated string MimeType: proto.MimeType, } + +} +func CaptureMetadataToProto(metadata CaptureMetadata) *pb.CaptureMetadata { + // MethodParameters map[string]interface{} + methodParms, _ := protoutils.ConvertStringMapToAnyPBMap(metadata.MethodParameters) + return &pb.CaptureMetadata{ + OrganizationId: metadata.OrganizationId, + LocationId: metadata.LocationId, + RobotName: metadata.RobotName, + RobotId: metadata.RobotId, + PartName: metadata.PartName, + PartId: metadata.PartId, + ComponentType: metadata.ComponentType, + ComponentName: metadata.ComponentName, + MethodName: metadata.MethodName, + MethodParameters: methodParms, + Tags: metadata.Tags, // repeated string + MimeType: metadata.MimeType, + } + } -func methodParamsFromProto(proto map[string]*anypb.Any) map[string]interface{} { +func methodParamsFromProto(proto map[string]*anypb.Any) map[string]string { // Convert MethodParameters map[string]*anypb.Any to map[string]interface{} - methodParameters := make(map[string]interface{}) + methodParameters := make(map[string]string) for key, value := range proto { structValue := &structpb.Value{} if err := value.UnmarshalTo(structValue); err != nil { return nil } - methodParameters[key] = structValue.AsInterface() + methodParameters[key] = structValue.String() } return methodParameters } @@ -199,6 +241,12 @@ func BinaryDataFromProto(proto *pb.BinaryData) BinaryData { Metadata: BinaryMetadataFromProto(proto.Metadata), } } +func BinaryDataToProto(binaryData BinaryData) *pb.BinaryData { + return &pb.BinaryData{ + Binary: binaryData.Binary, + Metadata: BinaryMetadataToProto(binaryData.Metadata), + } +} func BinaryMetadataFromProto(proto *pb.BinaryMetadata) BinaryMetadata { return BinaryMetadata{ @@ -213,9 +261,23 @@ func BinaryMetadataFromProto(proto *pb.BinaryMetadata) BinaryMetadata { DatasetIDs: proto.DatasetIds, } } +func BinaryMetadataToProto(binaryMetadata BinaryMetadata) *pb.BinaryMetadata { + return &pb.BinaryMetadata{ + Id: binaryMetadata.ID, + CaptureMetadata: CaptureMetadataToProto(binaryMetadata.CaptureMetadata), + TimeRequested: timestamppb.New(binaryMetadata.TimeRequested), + TimeReceived: timestamppb.New(binaryMetadata.TimeReceived), + FileName: binaryMetadata.FileName, + FileExt: binaryMetadata.FileExt, + Uri: binaryMetadata.URI, + Annotations: AnnotationsToProto(binaryMetadata.Annotations), + DatasetIds: binaryMetadata.DatasetIDs, + } +} // returns tabular data and the associated metadata func TabularDataFromProto(proto *pb.TabularData, metadata *pb.CaptureMetadata) TabularData { + fmt.Printf("this is proto in tabulardatafrom proto: %+v\n", metadata) return TabularData{ Data: proto.Data.AsMap(), //we have data as this when it is is in non proto ==> map[string]interface{} MetadataIndex: proto.MetadataIndex, @@ -229,13 +291,13 @@ func TabularDataToProto(tabularData TabularData) *pb.TabularData { if err != nil { return nil } - timeRequested := timestamppb.New(tabularData.TimeRequested) - timeReceived := timestamppb.New(tabularData.TimeReceived) + // timeRequested := timestamppb.New(tabularData.TimeRequested) + // timeReceived := timestamppb.New(tabularData.TimeReceived) return &pb.TabularData{ Data: structData, MetadataIndex: tabularData.MetadataIndex, - TimeRequested: timeRequested, - TimeReceived: timeReceived, + TimeRequested: timestamppb.New(tabularData.TimeRequested), + TimeReceived: timestamppb.New(tabularData.TimeReceived), } } func TabularDataToProtoList(tabularDatas []TabularData) []*pb.TabularData { @@ -252,12 +314,26 @@ func TabularDataToProtoList(tabularDatas []TabularData) []*pb.TabularData { // return the list of converted *pb.TabularData items return protoList } +func DataRequestToProto(dataRequest DataRequest) (*pb.DataRequest, error) { + return &pb.DataRequest{ + Filter: FilterToProto(dataRequest.Filter), + Limit: dataRequest.Limit, + Last: dataRequest.Last, + SortOrder: OrderToProto(dataRequest.SortOrder), + }, nil + +} func TabularDataBsonHelper(rawData [][]byte) ([]map[string]interface{}, error) { - dataObjects := make([]map[string]interface{}, len(rawData)) - // Loop over each BSON byte array in the response and unmarshal directly into the dataObjects slice - for _, rawData := range rawData { - obj := make(map[string]interface{}) - bson.Unmarshal(rawData, &obj) + dataObjects := []map[string]interface{}{} + for _, byteSlice := range rawData { + fmt.Printf("the byteslice is: %+v\n", byteSlice) + obj := map[string]interface{}{} + bson.Unmarshal(byteSlice, &obj) + for key, value := range obj { + if v, ok := value.(int32); ok { + obj[key] = int(v) + } + } dataObjects = append(dataObjects, obj) } return dataObjects, nil @@ -371,10 +447,6 @@ func (d *DataClient) TabularDataByFilter( if limit == 0 { limit = 50 } - // // ensure filter is not nil to represent a query for "all data" - // if filter.IsEmpty(){ - // filter = Filter{} //i think if it is empty than it just implies that ALL tabular data?? - // } resp, err := d.client.TabularDataByFilter(ctx, &pb.TabularDataByFilterRequest{ DataRequest: &pb.DataRequest{ Filter: FilterToProto(filter), @@ -387,18 +459,18 @@ func (d *DataClient) TabularDataByFilter( if err != nil { return nil, 0, "", err } - //get the tabulardata --> get metadataIndex from tabularData, - //get metadata, --> use metadata[tabularData.MetdataIndex] to get the associated metadata with that tabular data!! - //then return tabularData with that metadata!! - dataArray := make([]TabularData, len(resp.Data)) + dataArray := []TabularData{} + var metadata *pb.CaptureMetadata for _, data := range resp.Data { - var metadata *pb.CaptureMetadata - if int(data.MetadataIndex) < len(resp.Metadata) { - metadata = resp.Metadata[data.MetadataIndex] // Access the correct metadata using MetadataIndex + if len(resp.Metadata) != 0 && int(data.MetadataIndex) >= len(resp.Metadata) { + metadata = &pb.CaptureMetadata{} // Create new metadata if index is out of range + } else { + metadata = resp.Metadata[data.MetadataIndex] + } - dataArray = append(dataArray, TabularDataFromProto(data, metadata)) //the metadata that we pass has already been indexed + dataArray = append(dataArray, TabularDataFromProto(data, metadata)) + } - //// TabularData contains data and metadata associated with tabular data. return dataArray, resp.Count, resp.Last, nil } @@ -421,8 +493,21 @@ func (d *DataClient) TabularDataByMQL(ctx context.Context, organizationId string if err != nil { return nil, err } - dataObjects, nil := TabularDataBsonHelper(resp.RawData) - return dataObjects, nil + + // var result []map[string]interface{} + // for _, bsonBytes := range resp.RawData { + // var decodedData map[string]interface{} + // err := bson.Unmarshal(bsonBytes, &decodedData) + // if err != nil { + // return nil, fmt.Errorf("error decoding BSON: %v", err) + // } + // fmt.Printf("this is decoded data: %+v\n", decodedData) + // result = append(result, decodedData) + + // } + result, nil := TabularDataBsonHelper(resp.RawData) + fmt.Printf("this is result: %+v\n", result) + return result, nil // dataObjects := make([]map[string]interface{}, len(resp.RawData)) // for i, rawData := range resp.RawData { @@ -442,12 +527,14 @@ func (d *DataClient) BinaryDataByFilter( ctx context.Context, filter Filter, limit uint64, - last string, sortOrder Order, + last string, includeBinary bool, countOnly bool, // includeInternalData bool) ([]*pb.BinaryData, uint64, string, error) { includeInternalData bool) ([]BinaryData, uint64, string, error) { + fmt.Println("client.BinaryDataByFilter was called") + // initialize limit if it's unspecified (zero value) if limit == 0 { limit = 50 @@ -456,6 +543,7 @@ func (d *DataClient) BinaryDataByFilter( // if filter == nil { // filter = &pb.Filter{} // } + resp, err := d.client.BinaryDataByFilter(ctx, &pb.BinaryDataByFilterRequest{ DataRequest: &pb.DataRequest{ //need to do checks to make sure it fits the constraints Filter: FilterToProto(filter), @@ -463,6 +551,7 @@ func (d *DataClient) BinaryDataByFilter( Last: last, SortOrder: OrderToProto(sortOrder), }, + IncludeBinary: includeBinary, CountOnly: countOnly, IncludeInternalData: includeInternalData, }) @@ -493,7 +582,6 @@ func (d *DataClient) BinaryDataByIDs(ctx context.Context, binaryIds []BinaryID) for i, protoData := range resp.Data { data[i] = BinaryDataFromProto(protoData) } - // return resp.Data, nil return data, nil //the return type of this is: var data []BinaryData --> is that okay??? , do we only want go native types does this count? } func (d *DataClient) DeleteTabularData(ctx context.Context, organizationId string, deleteOlderThanDays uint32) (uint64, error) { @@ -597,9 +685,6 @@ func (d *DataClient) RemoveBoundingBoxFromImageByID(ctx context.Context, bboxId return nil } func (d *DataClient) BoundingBoxLabelsByFilter(ctx context.Context, filter Filter) ([]string, error) { - // if filter == nil { - // filter = &pb.Filter{} - // } resp, err := d.client.BoundingBoxLabelsByFilter(ctx, &pb.BoundingBoxLabelsByFilterRequest{Filter: FilterToProto(filter)}) if err != nil { return nil, err @@ -632,6 +717,7 @@ func (d *DataClient) GetDatabaseConnection(ctx context.Context, organizationId s } return resp.Hostname, nil } + func (d *DataClient) ConfigureDatabaseUser(ctx context.Context, organizationId string, password string) error { _, err := d.client.ConfigureDatabaseUser(ctx, &pb.ConfigureDatabaseUserRequest{OrganizationId: organizationId, Password: password}) if err != nil { diff --git a/app/data_test.go b/app/data_test.go index ac5e5f00821..c7d22674b8f 100644 --- a/app/data_test.go +++ b/app/data_test.go @@ -2,58 +2,112 @@ package app import ( "context" - "fmt" "testing" "time" "go.mongodb.org/mongo-driver/bson" datapb "go.viam.com/api/app/data/v1" + "go.viam.com/rdk/testutils/inject" "go.viam.com/test" + utils "go.viam.com/utils/protoutils" "google.golang.org/grpc" - - "go.viam.com/rdk/testutils/inject" + "google.golang.org/protobuf/types/known/timestamppb" ) +// ccamel case in go!!!! const ( - component_name = "component_name" - component_type = "component_type" - method = "method" - robot_name = "robot_name" - robot_id = "robot_id" - part_name = "part_name" - part_id = "part_id" - location_id = "location_id" - organization_id = "organization_id" - // password = "password" - mime_type = "mime_type" - uri = "some.robot.uri" - bbox_label = "bbox_label" - dataset_id = "dataset_id" - tag = "tag" + componentName = "component_name" + componentType = "component_type" + method = "method" + robotName = "robot_name" + robotId = "robot_id" + partName = "part_name" + partId = "part_id" + locationId = "location_id" + organizationId = "organization_id" + password = "password" + mimeType = "mime_type" + uri = "some.robot.uri" + bboxLabel = "bbox_label" + tag = "tag" + fileName = "file_name" + fileExt = "file_ext.ext" + datasetId = "dataset_id" + binaryMetaId = "binary_id" + mongodbUri = "mongo_uri" + hostName = "host_name" + + // deleteOlderThanDays = 1 + // deletedCount = 5 ) var ( - location_IDs = []string{location_id} - org_IDs = []string{organization_id} - mime_Types = []string{mime_type} - Bbox_Labels = []string{bbox_label} - method_parameters = map[string]interface{}{} - tags = []string{tag} + locationIds = []string{locationId} + orgIds = []string{organizationId} + mimeTypes = []string{mimeType} + BboxLabels = []string{bboxLabel} + methodParameters = map[string]string{} + tags = []string{tag} + // startTimePB = timestamppb.Timestamp{Seconds: 30, Nanos: 30} + // endTimePB = timestamppb.Timestamp{Seconds: 60, Nanos: 60} + startTime = time.Now().UTC() + endTime = time.Now().UTC() + data = map[string]interface{}{ + "key": "value", + } + binaryDataByte = []byte("BYTE") + mqlQuery = []map[string]interface{}{ + {"$match": map[string]interface{}{"organization_id": "e76d1b3b-0468-4efd-bb7f-fb1d2b352fcb"}}, + {"$limit": 1}, + } + sqlQuery = "SELECT * FROM readings WHERE organization_id='e76d1b3b-0468-4efd-bb7f-fb1d2b352fcb' LIMIT 1" + datasetIds = []string{datasetId} + annotations = Annotations{ + bboxes: []BoundingBox{ + { + id: "bbox1", + label: "label1", + xMinNormalized: 0.1, + yMinNormalized: 0.2, + xMaxNormalized: 0.8, + yMaxNormalized: 0.9, + }, + { + id: "bbox2", + label: "label2", + xMinNormalized: 0.15, + yMinNormalized: 0.25, + xMaxNormalized: 0.75, + yMaxNormalized: 0.85, + }, + }, + } + binaryId = BinaryID{ + FileId: "file1", + OrganizationId: organizationId, + LocationId: locationId, + } + binaryIds = []BinaryID{binaryId} ) -// // Helper function to create BSON documents for MongoDB queries -// func createMQLBSON() [][]byte { -// // create BSON documents for mongodb queries -// matchStage := bson.M{"$match": bson.M{"organization_id": "e76d1b3b-0468-4efd-bb7f-fb1d2b352fcb"}} -// limitStage := bson.M{"$limit": 1} -// // convert to BSON byte arrays -// matchBytes, _ := bson.Marshal(matchStage) -// limitBytes, _ := bson.Marshal(limitStage) -// mqlbinary := [][]byte{matchBytes, limitBytes} - -// return mqlbinary - -// } +// converts a slice of maps (representing MongoDB-like documents) into BSON byte arrays +func marshalToBSON(myBson []map[string]interface{}) [][]byte { + // create BSON documents for mongodb queries + var byteArray [][]byte + for _, item := range myBson { + input := bson.M(item) + bytes, _ := bson.Marshal(input) + byteArray = append(byteArray, bytes) + } + // matchStage := bson.M{"$match": bson.M{"organization_id": "e76d1b3b-0468-4efd-bb7f-fb1d2b352fcb"}} + // limitStage := bson.M{"$limit": 1} + // convert to BSON byte arrays + // matchBytes, _ := bson.Marshal(matchStage) + // limitBytes, _ := bson.Marshal(limitStage) + // mqlbinary := [][]byte{matchBytes, limitBytes} + // return mqlbinary + return byteArray +} // set up gRPC client?? func createGrpclient() datapb.DataServiceClient { @@ -64,170 +118,459 @@ func TestDataClient(t *testing.T) { grpcClient := &inject.DataServiceClient{} client := DataClient{client: grpcClient} - capture_interval := CaptureInterval{ + captureInterval := CaptureInterval{ Start: time.Now(), End: time.Now(), } - tags_filter := TagsFilter{ + tagsFilter := TagsFilter{ Type: 2, Tags: []string{"tag1", "tag2"}, } - myFilter := Filter{ - ComponentName: component_name, - ComponentType: component_type, + filter := Filter{ + ComponentName: componentName, + ComponentType: componentType, Method: method, - RobotName: robot_name, - RobotId: robot_id, - PartName: part_name, - PartId: part_id, - LocationIds: location_IDs, - OrganizationIds: org_IDs, - MimeType: mime_Types, - Interval: capture_interval, - TagsFilter: tags_filter, //asterix or no?? - BboxLabels: Bbox_Labels, - DatasetId: dataset_id, + RobotName: robotName, + RobotId: robotId, + PartName: partName, + PartId: partId, + LocationIds: locationIds, + OrganizationIds: orgIds, + MimeType: mimeTypes, + Interval: captureInterval, + TagsFilter: tagsFilter, //asterix or no?? + BboxLabels: BboxLabels, + DatasetId: datasetId, } - myTabularMetadata := CaptureMetadata{ - OrganizationId: organization_id, - LocationId: location_id, - RobotName: robot_name, - RobotId: robot_id, - PartName: part_name, - PartId: part_id, - ComponentType: component_type, - ComponentName: component_name, + tabularMetadata := CaptureMetadata{ + OrganizationId: organizationId, + LocationId: locationId, + RobotName: robotName, + RobotId: robotId, + PartName: partName, + PartId: partId, + ComponentType: componentType, + ComponentName: componentName, MethodName: method, - MethodParameters: method_parameters, + MethodParameters: methodParameters, Tags: tags, - MimeType: mime_type, + MimeType: mimeType, + } + binaryMetadata := BinaryMetadata{ + ID: binaryMetaId, + CaptureMetadata: tabularMetadata, + TimeRequested: startTime, + TimeReceived: endTime, + FileName: fileName, + FileExt: fileExt, + URI: uri, + Annotations: annotations, + DatasetIDs: datasetIds, } t.Run("TabularDataByFilter", func(t *testing.T) { - myDataRequest := DataRequest{ - Filter: myFilter, + //flags + countOnly := true + includeInternalData := true + + dataRequest := DataRequest{ + Filter: filter, Limit: 5, Last: "last", SortOrder: Unspecified, } - myTabularData := TabularData{ - Data: map[string]interface{}{ - "key": "value", - }, + expectedTabularData := TabularData{ + Data: data, MetadataIndex: 0, - Metadata: myTabularMetadata, //not sure if i need to pass this - TimeRequested: time.Now(), - TimeReceived: time.Now(), + Metadata: tabularMetadata, //not sure if i need to pass this + TimeRequested: startTime, + TimeReceived: endTime, + } + expectedCount := uint64(5) + expectedLast := "last" + expectedLimit := uint64(5) + + dataStruct, _ := utils.StructToStructPb(data) + tabularDataPb := &datapb.TabularData{ + Data: dataStruct, + MetadataIndex: 0, + TimeRequested: timestamppb.New(startTime), + TimeReceived: timestamppb.New(endTime), } - myTabularDatas := []TabularData{myTabularData} - myCount := uint64(5) - myLast := "last" - myLimit := uint64(100) - countOnly := true - includeInternalData := true grpcClient.TabularDataByFilterFunc = func(ctx context.Context, in *datapb.TabularDataByFilterRequest, opts ...grpc.CallOption) (*datapb.TabularDataByFilterResponse, error) { - test.That(t, in.DataRequest, test.ShouldEqual, myDataRequest) + expectedDataReqProto, _ := DataRequestToProto(dataRequest) + test.That(t, in.DataRequest, test.ShouldResemble, expectedDataReqProto) //toPRoto on myDataReq (bc a request only ever gets converted to proto!!) test.That(t, in.CountOnly, test.ShouldBeTrue) test.That(t, in.IncludeInternalData, test.ShouldBeTrue) - return &datapb.TabularDataByFilterResponse{Data: TabularDataToProtoList(myTabularDatas), Count: myCount, Last: myLast}, nil + return &datapb.TabularDataByFilterResponse{ + Data: []*datapb.TabularData{tabularDataPb}, + Count: expectedCount, + Last: expectedLast, + Metadata: []*datapb.CaptureMetadata{CaptureMetadataToProto(tabularMetadata)}}, nil + //bc we only ever see tabularData in a response, we dont need toProto, we need fromProto!! + //the only returns we care about are TabularData, int coint, and str last rewturned pg id } - respTabularData, respCount, respLast, _ := client.TabularDataByFilter(context.Background(), myFilter, myLimit, myLast, myDataRequest.SortOrder, countOnly, includeInternalData) - test.That(t, respTabularData, test.ShouldEqual, myTabularData) - test.That(t, respCount, test.ShouldEqual, myCount) - test.That(t, respLast, test.ShouldEqual, myLast) + respTabularData, respCount, respLast, _ := client.TabularDataByFilter(context.Background(), filter, expectedLimit, expectedLast, dataRequest.SortOrder, countOnly, includeInternalData) + test.That(t, respTabularData[0], test.ShouldResemble, expectedTabularData) + test.That(t, respCount, test.ShouldEqual, expectedCount) + test.That(t, respLast, test.ShouldEqual, expectedLast) }) - t.Run("TabularDataBySQL", func(t *testing.T) { + // t.Run("TabularDataBySQL", func(t *testing.T) { + // org_id := "MY_ORG" + // mySqlQuery := sqlQuery + // sqlBinary := []byte(sqlQuery) + // fmt.Printf("my sqlBinary : %#v\n", sqlBinary) + // expected := [][]byte{sqlBinary} //we expect the raw binary data? + // fmt.Printf("my expected : %#v\n", expected) + // // expected := myBsonMql + // // fmt.Printf("my expected : %#v\n", expected) + // grpcClient.TabularDataBySQLFunc = func(ctx context.Context, in *datapb.TabularDataBySQLRequest, opts ...grpc.CallOption) (*datapb.TabularDataBySQLResponse, error) { + // test.That(t, in.OrganizationId, test.ShouldEqual, org_id) //to proto + // test.That(t, in.SqlQuery, test.ShouldResemble, mySqlQuery) //to proto + // return &datapb.TabularDataBySQLResponse{RawData: [][]byte{sqlBinary}}, nil //MQlResponse is created with BSON data + // } + + // response, _ := client.TabularDataBySQL(context.Background(), org_id, sqlQuery) + + // // // fmt.Printf("expected type: %T, value: %+v\n", expected, expected) + // // // fmt.Printf("actual type: %T, value: %+v\n", response, response) + // test.That(t, response, test.ShouldResemble, expected) //*this test not working as expected yet! + + // }) + + // t.Run("TabularDataByMQL", func(t *testing.T) { + // org_id := "MY_ORG" + // mql_binary := marshalToBSON(mqlQuery) //this is my mql binary type! + // // fmt.Printf("my mql_binary : %#v\n", mql_binary) + // expected := mqlQuery //does this make sense??? + // // fmt.Printf("my expected : %#v\n", expected) + // grpcClient.TabularDataByMQLFunc = func(ctx context.Context, in *datapb.TabularDataByMQLRequest, opts ...grpc.CallOption) (*datapb.TabularDataByMQLResponse, error) { + // test.That(t, in.OrganizationId, test.ShouldEqual, org_id) //to proto + // test.That(t, in.MqlBinary, test.ShouldResemble, mql_binary) //to proto + // return &datapb.TabularDataByMQLResponse{RawData: mql_binary}, nil //MQlResponse is created with BSON data + // } + + // response, _ := client.TabularDataByMQL(context.Background(), org_id, mql_binary) + + // fmt.Printf("expected type: %T, value: %+v\n", expected, expected) + // fmt.Printf("actual type: %T, value: %+v\n", response, response) + // test.That(t, response, test.ShouldEqual, expected) //*this test not working as expected yet! + // }) + t.Run("BinaryDataByFilter", func(t *testing.T) { + includeBinary := true + countOnly := true + includeInternalData := true + + dataRequest := DataRequest{ + Filter: filter, + Limit: 5, + Last: "last", + SortOrder: Unspecified, + } + + expectedCount := uint64(5) + expectedLast := "last" + expectedBinaryData := BinaryData{ + Binary: binaryDataByte, + Metadata: binaryMetadata, + } + expectedBinaryDataPb := BinaryDataToProto(expectedBinaryData) + + grpcClient.BinaryDataByFilterFunc = func(ctx context.Context, in *datapb.BinaryDataByFilterRequest, opts ...grpc.CallOption) (*datapb.BinaryDataByFilterResponse, error) { + expectedDataReq, _ := DataRequestToProto(dataRequest) + test.That(t, in.DataRequest, test.ShouldResemble, expectedDataReq) //toPRoto on myDataReq (bc a request only ever gets converted to proto!!) + test.That(t, in.IncludeBinary, test.ShouldBeTrue) + test.That(t, in.CountOnly, test.ShouldBeTrue) + test.That(t, in.IncludeInternalData, test.ShouldBeTrue) + return &datapb.BinaryDataByFilterResponse{ + Data: []*datapb.BinaryData{expectedBinaryDataPb}, + Count: expectedCount, + Last: expectedLast, + }, nil + //bc we only ever see tabularData in a response, we dont need toProto, we need fromProto!! + //the only returns we care about are TabularData, int coint, and str last rewturned pg id + } + + // fmt.Printf("this is includeBinary type: %T, value: %+v\n", includeBinary, includeBinary) + + respBinaryData, respCount, respLast, _ := client.BinaryDataByFilter(context.Background(), filter, expectedCount, dataRequest.SortOrder, expectedLast, includeBinary, countOnly, includeInternalData) + // fmt.Printf("this is respBinaryData[0] type: %T, value: %+v\n", respBinaryData[0], respBinaryData[0]) + test.That(t, respBinaryData[0], test.ShouldResemble, expectedBinaryData) + test.That(t, respCount, test.ShouldEqual, expectedCount) + test.That(t, respLast, test.ShouldEqual, expectedLast) }) - t.Run("TabularDataByMQL", func(t *testing.T) { - org_id := "MY_ORG" - input := map[string]interface{}{ - "foo": "bar", - "one": 1, - "list": []string{"a", "b", "c"}, + t.Run("BinaryDataByIDs", func(t *testing.T) { + expectedBinaryData := BinaryData{ + Binary: binaryDataByte, + Metadata: binaryMetadata, } - //serialize input into list of bytearrays, aka bson - bsonData, err := bson.Marshal(input) - if err != nil { - fmt.Printf("trying something out") + grpcClient.BinaryDataByIDsFunc = func(ctx context.Context, in *datapb.BinaryDataByIDsRequest, opts ...grpc.CallOption) (*datapb.BinaryDataByIDsResponse, error) { + test.That(t, in.IncludeBinary, test.ShouldBeTrue) + test.That(t, in.BinaryIds, test.ShouldResemble, BinaryIdsToProto(binaryIds)) + //convert []binaryData to proto + expectedBinaryDataList := []*datapb.BinaryData{BinaryDataToProto(expectedBinaryData)} + + return &datapb.BinaryDataByIDsResponse{Data: expectedBinaryDataList, Count: uint64(len(expectedBinaryDataList))}, nil } - mql_binary := [][]byte{bsonData} //bson data type + respBinaryData, _ := client.BinaryDataByIDs(context.Background(), binaryIds) + test.That(t, respBinaryData[0], test.ShouldResemble, expectedBinaryData) + }) + t.Run("DeleteTabularData", func(t *testing.T) { + deleteOlderThanDays := uint32(1) + + expectedOrgId := organizationId + expectedDeleteOlderThanDays := deleteOlderThanDays + expectedDeletedCount := uint64(5) + + grpcClient.DeleteTabularDataFunc = func(ctx context.Context, in *datapb.DeleteTabularDataRequest, opts ...grpc.CallOption) (*datapb.DeleteTabularDataResponse, error) { + test.That(t, in.OrganizationId, test.ShouldEqual, expectedOrgId) + test.That(t, in.DeleteOlderThanDays, test.ShouldEqual, expectedDeleteOlderThanDays) - expected := []map[string]interface{}{ - {"foo": "bar"}, - {"one": 1}, - {"list": []string{"a", "b", "c"}}, + return &datapb.DeleteTabularDataResponse{ + DeletedCount: expectedDeletedCount, + }, nil } + resp, _ := client.DeleteTabularData(context.Background(), organizationId, deleteOlderThanDays) + test.That(t, resp, test.ShouldEqual, expectedDeletedCount) + }) + t.Run("DeleteBinaryDataByFilter", func(t *testing.T) { + expectedFilterPb := FilterToProto(filter) + expectedDeletedCount := uint64(5) - grpcClient.TabularDataByMQLFunc = func(ctx context.Context, in *datapb.TabularDataByMQLRequest, opts ...grpc.CallOption) (*datapb.TabularDataByMQLResponse, error) { - test.That(t, in.OrganizationId, test.ShouldEqual, org_id) - test.That(t, in.MqlBinary, test.ShouldResemble, mql_binary) - return &datapb.TabularDataByMQLResponse{RawData: mql_binary}, nil //MQlResponse is created with BSON data + grpcClient.DeleteBinaryDataByFilterFunc = func(ctx context.Context, in *datapb.DeleteBinaryDataByFilterRequest, opts ...grpc.CallOption) (*datapb.DeleteBinaryDataByFilterResponse, error) { + test.That(t, in.Filter, test.ShouldResemble, expectedFilterPb) + test.That(t, in.IncludeInternalData, test.ShouldBeTrue) + return &datapb.DeleteBinaryDataByFilterResponse{ + DeletedCount: expectedDeletedCount, + }, nil + } + resp, _ := client.DeleteBinaryDataByFilter(context.Background(), filter) + test.That(t, resp, test.ShouldEqual, expectedDeletedCount) + }) + t.Run("DeleteBinaryDataByIDs", func(t *testing.T) { + expectedDeletedCount := uint64(5) + expectedBinaryIds := BinaryIdsToProto(binaryIds) + grpcClient.DeleteBinaryDataByIDsFunc = func(ctx context.Context, in *datapb.DeleteBinaryDataByIDsRequest, opts ...grpc.CallOption) (*datapb.DeleteBinaryDataByIDsResponse, error) { + test.That(t, in.BinaryIds, test.ShouldResemble, expectedBinaryIds) + return &datapb.DeleteBinaryDataByIDsResponse{ + DeletedCount: expectedDeletedCount, + }, nil + } + resp, _ := client.DeleteBinaryDataByIDs(context.Background(), binaryIds) + test.That(t, resp, test.ShouldEqual, expectedDeletedCount) + }) + t.Run("AddTagsToBinaryDataByIDs", func(t *testing.T) { + expectedTags := tags + expectedBinaryIds := BinaryIdsToProto(binaryIds) + grpcClient.AddTagsToBinaryDataByIDsFunc = func(ctx context.Context, in *datapb.AddTagsToBinaryDataByIDsRequest, opts ...grpc.CallOption) (*datapb.AddTagsToBinaryDataByIDsResponse, error) { + test.That(t, in.BinaryIds, test.ShouldResemble, expectedBinaryIds) + test.That(t, in.Tags, test.ShouldResemble, expectedTags) + return &datapb.AddTagsToBinaryDataByIDsResponse{}, nil } + client.AddTagsToBinaryDataByIDs(context.Background(), tags, binaryIds) - response, _ := client.TabularDataByMQL(context.Background(), org_id, mql_binary) - // test.That(t, response, test.ShouldEqual, expected) - fmt.Printf("Expected: %#v\n", expected) - fmt.Printf("Actual : %#v\n", response) - test.That(t, response, test.ShouldResemble, expected) //*this test not working as expected yet! }) - t.Run("BinaryDataByFilter", func(t *testing.T) { + t.Run("AddTagsToBinaryDataByFilter", func(t *testing.T) { + expectedTags := tags + expectedFilterPb := FilterToProto(filter) + grpcClient.AddTagsToBinaryDataByFilterFunc = func(ctx context.Context, in *datapb.AddTagsToBinaryDataByFilterRequest, opts ...grpc.CallOption) (*datapb.AddTagsToBinaryDataByFilterResponse, error) { + test.That(t, in.Filter, test.ShouldResemble, expectedFilterPb) + test.That(t, in.Tags, test.ShouldResemble, expectedTags) + return &datapb.AddTagsToBinaryDataByFilterResponse{}, nil + } + client.AddTagsToBinaryDataByFilter(context.Background(), tags, filter) + }) + t.Run("RemoveTagsFromBinaryDataByIDs", func(t *testing.T) { + expectedTags := tags + expectedBinaryIds := BinaryIdsToProto(binaryIds) + expectedDeletedCount := uint64(5) + grpcClient.RemoveTagsFromBinaryDataByIDsFunc = func(ctx context.Context, in *datapb.RemoveTagsFromBinaryDataByIDsRequest, opts ...grpc.CallOption) (*datapb.RemoveTagsFromBinaryDataByIDsResponse, error) { + test.That(t, in.BinaryIds, test.ShouldResemble, expectedBinaryIds) + test.That(t, in.Tags, test.ShouldResemble, expectedTags) + return &datapb.RemoveTagsFromBinaryDataByIDsResponse{ + DeletedCount: expectedDeletedCount, + }, nil + } + resp, _ := client.RemoveTagsFromBinaryDataByIDs(context.Background(), tags, binaryIds) + test.That(t, resp, test.ShouldEqual, expectedDeletedCount) + }) + t.Run("RemoveTagsFromBinaryDataByFilter", func(t *testing.T) { + expectedTags := tags + expectedFilterPb := FilterToProto(filter) + expectedDeletedCount := uint64(5) + + grpcClient.RemoveTagsFromBinaryDataByFilterFunc = func(ctx context.Context, in *datapb.RemoveTagsFromBinaryDataByFilterRequest, opts ...grpc.CallOption) (*datapb.RemoveTagsFromBinaryDataByFilterResponse, error) { + test.That(t, in.Filter, test.ShouldResemble, expectedFilterPb) + test.That(t, in.Tags, test.ShouldResemble, expectedTags) + return &datapb.RemoveTagsFromBinaryDataByFilterResponse{ + DeletedCount: expectedDeletedCount, + }, nil + } + resp, _ := client.RemoveTagsFromBinaryDataByFilter(context.Background(), tags, filter) + test.That(t, resp, test.ShouldEqual, expectedDeletedCount) + }) + t.Run("TagsByFilter", func(t *testing.T) { + expectedTags := tags + expectedFilterPb := FilterToProto(filter) + + grpcClient.TagsByFilterFunc = func(ctx context.Context, in *datapb.TagsByFilterRequest, opts ...grpc.CallOption) (*datapb.TagsByFilterResponse, error) { + test.That(t, in.Filter, test.ShouldResemble, expectedFilterPb) + return &datapb.TagsByFilterResponse{ + Tags: tags, + }, nil + } + resp, _ := client.TagsByFilter(context.Background(), filter) + test.That(t, resp, test.ShouldResemble, expectedTags) + }) + t.Run("AddBoundingBoxToImageByID", func(t *testing.T) { + expectedBinaryIdPb := BinaryIdToProto(binaryId) + expectedLabel := bboxLabel + expectedXMin := annotations.bboxes[0].xMinNormalized + expectedYMin := annotations.bboxes[0].yMinNormalized + expectedXMax := annotations.bboxes[0].xMaxNormalized + expectedYMax := annotations.bboxes[0].yMaxNormalized + expectedBBoxId := annotations.bboxes[0].id + + grpcClient.AddBoundingBoxToImageByIDFunc = func(ctx context.Context, in *datapb.AddBoundingBoxToImageByIDRequest, opts ...grpc.CallOption) (*datapb.AddBoundingBoxToImageByIDResponse, error) { + test.That(t, in.BinaryId, test.ShouldResemble, expectedBinaryIdPb) + test.That(t, in.Label, test.ShouldEqual, expectedLabel) + test.That(t, in.XMinNormalized, test.ShouldEqual, expectedXMin) + test.That(t, in.YMinNormalized, test.ShouldEqual, expectedYMin) + test.That(t, in.XMaxNormalized, test.ShouldEqual, expectedXMax) + test.That(t, in.YMaxNormalized, test.ShouldEqual, expectedYMax) + + return &datapb.AddBoundingBoxToImageByIDResponse{ + BboxId: expectedBBoxId, + }, nil + } + resp, _ := client.AddBoundingBoxToImageByID(context.Background(), binaryId, bboxLabel, expectedXMin, expectedYMin, expectedXMax, expectedYMax) + test.That(t, resp, test.ShouldResemble, expectedBBoxId) + }) + + t.Run("RemoveBoundingBoxFromImageByID", func(t *testing.T) { + expectedBinaryIdPb := BinaryIdToProto(binaryId) + expectedBBoxId := annotations.bboxes[0].id + + grpcClient.RemoveBoundingBoxFromImageByIDFunc = func(ctx context.Context, in *datapb.RemoveBoundingBoxFromImageByIDRequest, opts ...grpc.CallOption) (*datapb.RemoveBoundingBoxFromImageByIDResponse, error) { + test.That(t, in.BinaryId, test.ShouldResemble, expectedBinaryIdPb) + test.That(t, in.BboxId, test.ShouldEqual, expectedBBoxId) + + return &datapb.RemoveBoundingBoxFromImageByIDResponse{}, nil + } + client.RemoveBoundingBoxFromImageByID(context.Background(), expectedBBoxId, binaryId) }) - t.Run("BinaryDataByIDs", func(t *testing.T) { - binaryID1 := BinaryID{ //can this just be of type BinaryID??? why proto type? - FileId: "file1", - OrganizationId: "org1", - LocationId: "loc1", + t.Run("BoundingBoxLabelsByFilter", func(t *testing.T) { + expectedFilterPb := FilterToProto(filter) + + expectedBBoxLabels := []string{ + annotations.bboxes[0].label, + annotations.bboxes[1].label, } - // ==> why would it have to be of type datapb??? cant it jsut be of BinaryID - binaryID2 := BinaryID{ - FileId: "file2", - OrganizationId: "org1", - LocationId: "loc1", + //we probs don't need to do this work below...but i like the explicity of it + annotationsPb := AnnotationsToProto(annotations) + expectedBBoxLabelsPb := []string{ + annotationsPb.Bboxes[0].Label, + annotationsPb.Bboxes[1].Label, } - binaryIDs := []BinaryID{binaryID1, binaryID2} - - // binaryMetadata that acts as input! - metadata := BinaryMetadata{ - ID: "id1", - CaptureMetadata: CaptureMetadata{ /* I probs need to fill these fields in???*/ }, - TimeRequested: time.Now(), - TimeReceived: time.Now(), - FileName: "file.txt", - FileExt: ".txt", - URI: "http://ex/file.txt", //can i just make this up?? - Annotations: Annotations{ /* this too???*/ }, - DatasetIDs: []string{"dataset1", "dataset2"}, + + grpcClient.BoundingBoxLabelsByFilterFunc = func(ctx context.Context, in *datapb.BoundingBoxLabelsByFilterRequest, opts ...grpc.CallOption) (*datapb.BoundingBoxLabelsByFilterResponse, error) { + test.That(t, in.Filter, test.ShouldResemble, expectedFilterPb) + return &datapb.BoundingBoxLabelsByFilterResponse{ + Labels: expectedBBoxLabelsPb, + }, nil + } + resp, _ := client.BoundingBoxLabelsByFilter(context.Background(), filter) + test.That(t, resp, test.ShouldResemble, expectedBBoxLabels) + }) + t.Run("UpdateBoundingBox", func(t *testing.T) { + bBoxId := annotations.bboxes[0].id + expectedBinaryIdPb := BinaryIdToProto(binaryId) + + annotationsPb := AnnotationsToProto(annotations) + expectedLabel := annotationsPb.Bboxes[0].Label + expectedBBoxIdPb := annotationsPb.Bboxes[0].Id + expectedXMin := annotationsPb.Bboxes[0].XMinNormalized + expectedYMin := annotationsPb.Bboxes[0].YMinNormalized + expectedXMax := annotationsPb.Bboxes[0].XMaxNormalized + expectedYMax := annotationsPb.Bboxes[0].YMaxNormalized + + grpcClient.UpdateBoundingBoxFunc = func(ctx context.Context, in *datapb.UpdateBoundingBoxRequest, opts ...grpc.CallOption) (*datapb.UpdateBoundingBoxResponse, error) { + test.That(t, in.BinaryId, test.ShouldResemble, expectedBinaryIdPb) + test.That(t, in.BboxId, test.ShouldResemble, expectedBBoxIdPb) + test.That(t, *in.Label, test.ShouldEqual, expectedLabel) + test.That(t, *in.XMinNormalized, test.ShouldEqual, expectedXMin) + test.That(t, *in.YMinNormalized, test.ShouldEqual, expectedYMin) + test.That(t, *in.XMaxNormalized, test.ShouldEqual, expectedXMax) + test.That(t, *in.YMaxNormalized, test.ShouldEqual, expectedYMax) + + return &datapb.UpdateBoundingBoxResponse{}, nil } + client.UpdateBoundingBox(context.Background(), binaryId, bBoxId, expectedLabel, expectedXMin, expectedYMin, expectedXMax, expectedYMax) - binary := []byte("something") - // binaryData that acts as input - binaryData := BinaryData{ - Binary: binary, - Metadata: metadata, + }) + + t.Run("GetDatabaseConnection", func(t *testing.T) { + expectedOrgId := organizationId + expectedHostName := hostName + expectedMongodbUri := mongodbUri + expectedDbUser := true + + grpcClient.GetDatabaseConnectionFunc = func(ctx context.Context, in *datapb.GetDatabaseConnectionRequest, opts ...grpc.CallOption) (*datapb.GetDatabaseConnectionResponse, error) { + test.That(t, in.OrganizationId, test.ShouldResemble, expectedOrgId) + return &datapb.GetDatabaseConnectionResponse{ + Hostname: expectedHostName, + MongodbUri: expectedMongodbUri, + HasDatabaseUser: expectedDbUser, + }, nil } - binaryDataList := []BinaryData{binaryData} + resp, _ := client.GetDatabaseConnection(context.Background(), organizationId) + test.That(t, resp, test.ShouldResemble, expectedHostName) - grpcClient.BinaryDataByIDsFunc = func(ctx context.Context, in *datapb.BinaryDataByIDsRequest, opts ...grpc.CallOption) (*datapb.BinaryDataByIDsResponse, error) { - test.That(t, in.BinaryIds, test.ShouldEqual, binaryIDs) - return &datapb.BinaryDataByIDsResponse{Data: binaryDataList, Count: uint64(len(binaryDataList))}, nil + }) + + t.Run("ConfigureDatabaseUser", func(t *testing.T) { + expectedOrgId := organizationId + expectedPassword := password + + grpcClient.ConfigureDatabaseUserFunc = func(ctx context.Context, in *datapb.ConfigureDatabaseUserRequest, opts ...grpc.CallOption) (*datapb.ConfigureDatabaseUserResponse, error) { + test.That(t, in.OrganizationId, test.ShouldResemble, expectedOrgId) + test.That(t, in.Password, test.ShouldResemble, expectedPassword) + return &datapb.ConfigureDatabaseUserResponse{}, nil } - response, _ := client.BinaryDataByIDs(context.Background(), binaryIDs) - - // expectedData := BinaryData{binary: binary, Metadata: metadata} - // var expectedData []BinaryData{BinaryDataFromProto(binaryDataList)} - var expectedData []BinaryData - for _, binaryDataItem := range binaryDataList { - convertedData := BinaryDataFromProto(binaryDataItem) - expectedData = append(expectedData, convertedData) + client.ConfigureDatabaseUser(context.Background(), organizationId, password) + }) + + t.Run("AddBinaryDataToDatasetByIDs", func(t *testing.T) { + expectedBinaryIds := BinaryIdsToProto(binaryIds) + expectedDataSetId := datasetId + + grpcClient.AddBinaryDataToDatasetByIDsFunc = func(ctx context.Context, in *datapb.AddBinaryDataToDatasetByIDsRequest, opts ...grpc.CallOption) (*datapb.AddBinaryDataToDatasetByIDsResponse, error) { + test.That(t, in.BinaryIds, test.ShouldResemble, expectedBinaryIds) + test.That(t, in.DatasetId, test.ShouldResemble, expectedDataSetId) + return &datapb.AddBinaryDataToDatasetByIDsResponse{}, nil } - //loop thru binary dataDataList and convert binaryDataFromproto each one---> and then add that to a list...! - test.That(t, response, test.ShouldResemble, expectedData) + client.AddBinaryDataToDatasetByIDs(context.Background(), binaryIds, datasetId) + + }) + + t.Run("RemoveBinaryDataFromDatasetByIDs", func(t *testing.T) { + expectedBinaryIds := BinaryIdsToProto(binaryIds) + expectedDataSetId := datasetId + + grpcClient.RemoveBinaryDataFromDatasetByIDsFunc = func(ctx context.Context, in *datapb.RemoveBinaryDataFromDatasetByIDsRequest, opts ...grpc.CallOption) (*datapb.RemoveBinaryDataFromDatasetByIDsResponse, error) { + test.That(t, in.BinaryIds, test.ShouldResemble, expectedBinaryIds) + test.That(t, in.DatasetId, test.ShouldResemble, expectedDataSetId) + + return &datapb.RemoveBinaryDataFromDatasetByIDsResponse{}, nil + } + client.RemoveBinaryDataFromDatasetByIDs(context.Background(), binaryIds, datasetId) + }) } From 6799d6a1a540e19183958049a91475e10bffbe56 Mon Sep 17 00:00:00 2001 From: Julie Krasnick Date: Mon, 11 Nov 2024 17:47:50 -0500 Subject: [PATCH 14/25] added almost all tests --- app/data_client.go | 89 +++++++-------------------------- app/data_test.go | 121 ++++++++++++++++++++++++++------------------- 2 files changed, 87 insertions(+), 123 deletions(-) diff --git a/app/data_client.go b/app/data_client.go index cbf93b877de..49d3e733f0e 100644 --- a/app/data_client.go +++ b/app/data_client.go @@ -37,11 +37,8 @@ type CaptureMetadata struct { ComponentName string MethodName string MethodParameters map[string]string - //^^ supposed to be: map method_parameters = 11; Tags []string - //^^ repeated string tags = 12; MimeType string - //^^ string mime_type = 13; } type BinaryID struct { FileId string @@ -51,7 +48,7 @@ type BinaryID struct { type BoundingBox struct { id string label string - xMinNormalized float64 //should be double but no doubles in go + xMinNormalized float64 yMinNormalized float64 xMaxNormalized float64 yMaxNormalized float64 @@ -62,17 +59,13 @@ type DataRequest struct { Last string SortOrder Order } - -// Annotations are data annotations used for machine learning. type Annotations struct { - //supposed to be repeated bounding boxes bboxes []BoundingBox } type TabularData struct { Data map[string]interface{} - // Metadata *pb.CaptureMetadata //idk why i put a star here -- if we aren't returning it is it okay? MetadataIndex uint32 - Metadata CaptureMetadata //idk why i put a star here + Metadata CaptureMetadata TimeRequested time.Time TimeReceived time.Time } @@ -81,10 +74,8 @@ type BinaryData struct { Metadata BinaryMetadata } -// can the return type be a struct called BinaryMetadata that I made up??? type BinaryMetadata struct { ID string - //CaptureMetadata *pb.CaptureMetadata CaptureMetadata CaptureMetadata TimeRequested time.Time TimeReceived time.Time @@ -92,7 +83,6 @@ type BinaryMetadata struct { FileExt string URI string Annotations Annotations - //Annotations *pb.Annotations DatasetIDs []string } type Filter struct { @@ -106,31 +96,14 @@ type Filter struct { LocationIds []string OrganizationIds []string MimeType []string - Interval CaptureInterval //asterix or no?? - TagsFilter TagsFilter //asterix or no?? + Interval CaptureInterval + TagsFilter TagsFilter BboxLabels []string DatasetId string } -// func (f Filter) IsEmpty() bool { -// return reflect.DeepEqual(f, Filter{}) -// } - -//notes for above:: -// type TagsFilter struct { -// state protoimpl.MessageState -// sizeCache protoimpl.SizeCache -// unknownFields protoimpl.UnknownFields - -// Type TagsFilterType `protobuf:"varint,1,opt,name=type,proto3,enum=viam.app.data.v1.TagsFilterType" json:"type,omitempty"` -// // Tags are used to match documents if `type` is UNSPECIFIED or MATCH_BY_OR. -// Tags []string `protobuf:"bytes,2,rep,name=tags,proto3" json:"tags,omitempty"` -// } - -//type TagsFilterType int32 - type TagsFilter struct { - Type int32 //type TagsFilterType int32 + Type int32 Tags []string } type CaptureInterval struct { @@ -142,7 +115,7 @@ func BoundingBoxFromProto(proto *pb.BoundingBox) BoundingBox { return BoundingBox{ id: proto.Id, label: proto.Label, - xMinNormalized: proto.XMinNormalized, // cast if i want int, or use float64 for precision + xMinNormalized: proto.XMinNormalized, yMinNormalized: proto.YMinNormalized, xMaxNormalized: proto.XMaxNormalized, yMaxNormalized: proto.YMaxNormalized, @@ -162,7 +135,6 @@ func AnnotationsFromProto(proto *pb.Annotations) Annotations { } } func AnnotationsToProto(annotations Annotations) *pb.Annotations { - // Convert each BoundingBox in annotations to its proto type var protoBboxes []*pb.BoundingBox for _, bbox := range annotations.bboxes { protoBboxes = append(protoBboxes, &pb.BoundingBox{ @@ -174,14 +146,11 @@ func AnnotationsToProto(annotations Annotations) *pb.Annotations { YMaxNormalized: bbox.yMaxNormalized, }) } - // Return the proto Annotations with converted bounding boxes return &pb.Annotations{ Bboxes: protoBboxes, } } func CaptureMetadataFromProto(proto *pb.CaptureMetadata) CaptureMetadata { - // fmt.Printf("this is proto in capturemetadata proto: %+v\n", proto) - if proto == nil { return CaptureMetadata{} } @@ -197,13 +166,12 @@ func CaptureMetadataFromProto(proto *pb.CaptureMetadata) CaptureMetadata { ComponentName: proto.ComponentName, MethodName: proto.MethodName, MethodParameters: methodParamsFromProto(proto.MethodParameters), - Tags: proto.Tags, // repeated string + Tags: proto.Tags, MimeType: proto.MimeType, } } func CaptureMetadataToProto(metadata CaptureMetadata) *pb.CaptureMetadata { - // MethodParameters map[string]interface{} methodParms, _ := protoutils.ConvertStringMapToAnyPBMap(metadata.MethodParameters) return &pb.CaptureMetadata{ OrganizationId: metadata.OrganizationId, @@ -216,14 +184,13 @@ func CaptureMetadataToProto(metadata CaptureMetadata) *pb.CaptureMetadata { ComponentName: metadata.ComponentName, MethodName: metadata.MethodName, MethodParameters: methodParms, - Tags: metadata.Tags, // repeated string + Tags: metadata.Tags, MimeType: metadata.MimeType, } } func methodParamsFromProto(proto map[string]*anypb.Any) map[string]string { - // Convert MethodParameters map[string]*anypb.Any to map[string]interface{} methodParameters := make(map[string]string) for key, value := range proto { structValue := &structpb.Value{} @@ -279,7 +246,7 @@ func BinaryMetadataToProto(binaryMetadata BinaryMetadata) *pb.BinaryMetadata { func TabularDataFromProto(proto *pb.TabularData, metadata *pb.CaptureMetadata) TabularData { fmt.Printf("this is proto in tabulardatafrom proto: %+v\n", metadata) return TabularData{ - Data: proto.Data.AsMap(), //we have data as this when it is is in non proto ==> map[string]interface{} + Data: proto.Data.AsMap(), MetadataIndex: proto.MetadataIndex, Metadata: CaptureMetadataFromProto(metadata), TimeRequested: proto.TimeRequested.AsTime(), @@ -291,8 +258,6 @@ func TabularDataToProto(tabularData TabularData) *pb.TabularData { if err != nil { return nil } - // timeRequested := timestamppb.New(tabularData.TimeRequested) - // timeReceived := timestamppb.New(tabularData.TimeReceived) return &pb.TabularData{ Data: structData, MetadataIndex: tabularData.MetadataIndex, @@ -302,16 +267,12 @@ func TabularDataToProto(tabularData TabularData) *pb.TabularData { } func TabularDataToProtoList(tabularDatas []TabularData) []*pb.TabularData { var protoList []*pb.TabularData - // loop through each TabularData item for _, tabularData := range tabularDatas { - // convert each TabularData to *pb.TabularData protoData := TabularDataToProto(tabularData) if protoData != nil { - // append the converted *pb.TabularData to the list protoList = append(protoList, protoData) } } - // return the list of converted *pb.TabularData items return protoList } func DataRequestToProto(dataRequest DataRequest) (*pb.DataRequest, error) { @@ -355,8 +316,6 @@ func BinaryIdsToProto(binaryIds []BinaryID) []*pb.BinaryID { return protoBinaryIds } -// PropertiesToProtoResponse takes a map of features to struct and converts it -// to a GetPropertiesResponse. func FilterToProto(filter Filter) *pb.Filter { return &pb.Filter{ ComponentName: filter.ComponentName, @@ -369,8 +328,8 @@ func FilterToProto(filter Filter) *pb.Filter { LocationIds: filter.LocationIds, OrganizationIds: filter.OrganizationIds, MimeType: filter.MimeType, - Interval: CaptureIntervalToProto(filter.Interval), //check this ?? - TagsFilter: TagsFilterToProto(filter.TagsFilter), //check this ?? + Interval: CaptureIntervalToProto(filter.Interval), + TagsFilter: TagsFilterToProto(filter.TagsFilter), BboxLabels: filter.BboxLabels, DatasetId: filter.DatasetId, } @@ -388,17 +347,14 @@ func TagsFilterToProto(tagsFilter TagsFilter) *pb.TagsFilter { } } -// shadow type for Order type Order int32 -// do i even need these below??? const ( Unspecified Order = 0 Descending Order = 1 Ascending Order = 2 ) -// Map SortOrder to corresponding values in pb.Order func OrderToProto(sortOrder Order) pb.Order { switch sortOrder { case Ascending: @@ -406,7 +362,7 @@ func OrderToProto(sortOrder Order) pb.Order { case Descending: return pb.Order_ORDER_DESCENDING default: - return pb.Order_ORDER_UNSPECIFIED // default or error handling + return pb.Order_ORDER_UNSPECIFIED } } @@ -417,10 +373,10 @@ type DataClient struct { } // (private) dataClient implements DataServiceClient. **do we want this? -type dataClient interface { - // actual hold implementations of functions - how would the NewDataClient function work if we had private and public functions? - // client pb.DataServiceClient -} +// type dataClient interface { +// // actual hold implementations of functions - how would the NewDataClient function work if we had private and public functions? +// // client pb.DataServiceClient +// } // NewDataClient constructs a new DataClient from connection passed in. func NewDataClient( @@ -439,7 +395,7 @@ func NewDataClient( func (d *DataClient) TabularDataByFilter( ctx context.Context, filter Filter, - limit uint64, //optional - max defaults to 50 if unspecified + limit uint64, //optional last string, //optional sortOrder Order, //optional countOnly bool, @@ -531,19 +487,11 @@ func (d *DataClient) BinaryDataByFilter( last string, includeBinary bool, countOnly bool, - // includeInternalData bool) ([]*pb.BinaryData, uint64, string, error) { includeInternalData bool) ([]BinaryData, uint64, string, error) { fmt.Println("client.BinaryDataByFilter was called") - - // initialize limit if it's unspecified (zero value) if limit == 0 { limit = 50 } - // ensure filter is not nil to represent a query for "all data" - // if filter == nil { - // filter = &pb.Filter{} - // } - resp, err := d.client.BinaryDataByFilter(ctx, &pb.BinaryDataByFilterRequest{ DataRequest: &pb.DataRequest{ //need to do checks to make sure it fits the constraints Filter: FilterToProto(filter), @@ -626,9 +574,6 @@ func (d *DataClient) AddTagsToBinaryDataByIDs(ctx context.Context, tags []string return nil } func (d *DataClient) AddTagsToBinaryDataByFilter(ctx context.Context, tags []string, filter Filter) error { - // if filter == nil { - // filter = &pb.Filter{} - // } _, err := d.client.AddTagsToBinaryDataByFilter(ctx, &pb.AddTagsToBinaryDataByFilterRequest{Filter: FilterToProto(filter), Tags: tags}) if err != nil { return err diff --git a/app/data_test.go b/app/data_test.go index c7d22674b8f..23b40ca6fbe 100644 --- a/app/data_test.go +++ b/app/data_test.go @@ -36,9 +36,6 @@ const ( binaryMetaId = "binary_id" mongodbUri = "mongo_uri" hostName = "host_name" - - // deleteOlderThanDays = 1 - // deletedCount = 5 ) var ( @@ -48,15 +45,23 @@ var ( BboxLabels = []string{bboxLabel} methodParameters = map[string]string{} tags = []string{tag} - // startTimePB = timestamppb.Timestamp{Seconds: 30, Nanos: 30} - // endTimePB = timestamppb.Timestamp{Seconds: 60, Nanos: 60} - startTime = time.Now().UTC() - endTime = time.Now().UTC() - data = map[string]interface{}{ + startTime = time.Now().UTC() + endTime = time.Now().UTC() + data = map[string]interface{}{ "key": "value", } binaryDataByte = []byte("BYTE") - mqlQuery = []map[string]interface{}{ + // expectedRawData = []map[string]interface{}{ + // { + // "key1": startTime, + // "key2": "2", + // "key3": []int{1, 2, 3}, + // "key4": map[string]interface{}{ + // "key4sub1": endTime, + // }, + // }, + // } + mqlQuery = []map[string]interface{}{ {"$match": map[string]interface{}{"organization_id": "e76d1b3b-0468-4efd-bb7f-fb1d2b352fcb"}}, {"$limit": 1}, } @@ -219,47 +224,61 @@ func TestDataClient(t *testing.T) { test.That(t, respLast, test.ShouldEqual, expectedLast) }) - // t.Run("TabularDataBySQL", func(t *testing.T) { - // org_id := "MY_ORG" - // mySqlQuery := sqlQuery - // sqlBinary := []byte(sqlQuery) - // fmt.Printf("my sqlBinary : %#v\n", sqlBinary) - // expected := [][]byte{sqlBinary} //we expect the raw binary data? - // fmt.Printf("my expected : %#v\n", expected) - // // expected := myBsonMql - // // fmt.Printf("my expected : %#v\n", expected) - // grpcClient.TabularDataBySQLFunc = func(ctx context.Context, in *datapb.TabularDataBySQLRequest, opts ...grpc.CallOption) (*datapb.TabularDataBySQLResponse, error) { - // test.That(t, in.OrganizationId, test.ShouldEqual, org_id) //to proto - // test.That(t, in.SqlQuery, test.ShouldResemble, mySqlQuery) //to proto - // return &datapb.TabularDataBySQLResponse{RawData: [][]byte{sqlBinary}}, nil //MQlResponse is created with BSON data - // } - - // response, _ := client.TabularDataBySQL(context.Background(), org_id, sqlQuery) - - // // // fmt.Printf("expected type: %T, value: %+v\n", expected, expected) - // // // fmt.Printf("actual type: %T, value: %+v\n", response, response) - // test.That(t, response, test.ShouldResemble, expected) //*this test not working as expected yet! - - // }) - - // t.Run("TabularDataByMQL", func(t *testing.T) { - // org_id := "MY_ORG" - // mql_binary := marshalToBSON(mqlQuery) //this is my mql binary type! - // // fmt.Printf("my mql_binary : %#v\n", mql_binary) - // expected := mqlQuery //does this make sense??? - // // fmt.Printf("my expected : %#v\n", expected) - // grpcClient.TabularDataByMQLFunc = func(ctx context.Context, in *datapb.TabularDataByMQLRequest, opts ...grpc.CallOption) (*datapb.TabularDataByMQLResponse, error) { - // test.That(t, in.OrganizationId, test.ShouldEqual, org_id) //to proto - // test.That(t, in.MqlBinary, test.ShouldResemble, mql_binary) //to proto - // return &datapb.TabularDataByMQLResponse{RawData: mql_binary}, nil //MQlResponse is created with BSON data - // } - - // response, _ := client.TabularDataByMQL(context.Background(), org_id, mql_binary) - - // fmt.Printf("expected type: %T, value: %+v\n", expected, expected) - // fmt.Printf("actual type: %T, value: %+v\n", response, response) - // test.That(t, response, test.ShouldEqual, expected) //*this test not working as expected yet! - // }) + t.Run("TabularDataBySQL", func(t *testing.T) { + expectedOrgId := organizationId + expectedSqlQuery := sqlQuery + expectedRawData := []map[string]interface{}{ + { + "key1": startTime, + "key2": "2", + "key3": []int{1, 2, 3}, + "key4": map[string]interface{}{ + "key4sub1": endTime, + }, + }, + } + expectedRawDataPb := marshalToBSON(expectedRawData) + grpcClient.TabularDataBySQLFunc = func(ctx context.Context, in *datapb.TabularDataBySQLRequest, opts ...grpc.CallOption) (*datapb.TabularDataBySQLResponse, error) { + test.That(t, in.OrganizationId, test.ShouldEqual, expectedOrgId) //to proto + test.That(t, in.SqlQuery, test.ShouldResemble, expectedSqlQuery) //to proto + return &datapb.TabularDataBySQLResponse{ + RawData: expectedRawDataPb, + }, nil + } + response, _ := client.TabularDataBySQL(context.Background(), expectedOrgId, expectedSqlQuery) + // // fmt.Printf("expected type: %T, value: %+v\n", expected, expected) + // // fmt.Printf("actual type: %T, value: %+v\n", response, response) + test.That(t, response, test.ShouldResemble, expectedRawData) + + }) + + t.Run("TabularDataByMQL", func(t *testing.T) { + expectedOrgId := organizationId + expectedMqlBinary := marshalToBSON(mqlQuery) //this is [][]byte type + //this is the format we want to get back... + expectedRawData := []map[string]interface{}{ + { + "key1": startTime, + "key2": "2", + "key3": []int{1, 2, 3}, + "key4": map[string]interface{}{ + "key4sub1": endTime, + }, + }, + } + //this is the rawData in protobuf type that we will expect to pass to the response + expectedRawDataPb := marshalToBSON(expectedRawData) + + grpcClient.TabularDataByMQLFunc = func(ctx context.Context, in *datapb.TabularDataByMQLRequest, opts ...grpc.CallOption) (*datapb.TabularDataByMQLResponse, error) { + test.That(t, in.OrganizationId, test.ShouldEqual, expectedOrgId) //to proto + test.That(t, in.MqlBinary, test.ShouldResemble, expectedMqlBinary) //to proto + return &datapb.TabularDataByMQLResponse{ + RawData: expectedRawDataPb, + }, nil + } + response, _ := client.TabularDataByMQL(context.Background(), expectedOrgId, expectedMqlBinary) + test.That(t, response, test.ShouldResemble, expectedRawData) + }) t.Run("BinaryDataByFilter", func(t *testing.T) { includeBinary := true countOnly := true @@ -491,7 +510,7 @@ func TestDataClient(t *testing.T) { t.Run("UpdateBoundingBox", func(t *testing.T) { bBoxId := annotations.bboxes[0].id expectedBinaryIdPb := BinaryIdToProto(binaryId) - + annotationsPb := AnnotationsToProto(annotations) expectedLabel := annotationsPb.Bboxes[0].Label expectedBBoxIdPb := annotationsPb.Bboxes[0].Id From 8dac3891f6f45890d3e2765f40fadd9ada44a60f Mon Sep 17 00:00:00 2001 From: Julie Krasnick Date: Wed, 13 Nov 2024 11:45:06 -0500 Subject: [PATCH 15/25] adding more helpers --- app/data_client.go | 298 ++++++++++++++++++++++++++++++++++++++++++--- app/data_test.go | 105 +++++++++++----- 2 files changed, 358 insertions(+), 45 deletions(-) diff --git a/app/data_client.go b/app/data_client.go index 49d3e733f0e..63b74523d13 100644 --- a/app/data_client.go +++ b/app/data_client.go @@ -8,6 +8,7 @@ import ( "time" "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" pb "go.viam.com/api/app/data/v1" "go.viam.com/rdk/logging" "go.viam.com/rdk/protoutils" @@ -37,8 +38,8 @@ type CaptureMetadata struct { ComponentName string MethodName string MethodParameters map[string]string - Tags []string - MimeType string + Tags []string + MimeType string } type BinaryID struct { FileId string @@ -63,7 +64,7 @@ type Annotations struct { bboxes []BoundingBox } type TabularData struct { - Data map[string]interface{} + Data map[string]interface{} MetadataIndex uint32 Metadata CaptureMetadata TimeRequested time.Time @@ -75,7 +76,7 @@ type BinaryData struct { } type BinaryMetadata struct { - ID string + ID string CaptureMetadata CaptureMetadata TimeRequested time.Time TimeReceived time.Time @@ -83,7 +84,7 @@ type BinaryMetadata struct { FileExt string URI string Annotations Annotations - DatasetIDs []string + DatasetIDs []string } type Filter struct { ComponentName string @@ -284,19 +285,244 @@ func DataRequestToProto(dataRequest DataRequest) (*pb.DataRequest, error) { }, nil } + +// func convertBsonToNative(data interface{}) interface{} { +// switch v := data.(type) { +// case primitive.DateTime: +// // Convert BSON DateTime to Go time.Time +// return v.Time().UTC() +// // case primitive.ObjectID: +// // // Convert BSON ObjectId to string (or another suitable representation) +// // return v.Hex() +// case int32, int64, float32, float64, bool, string: +// return v + +// case bson.A: +// // If it's a BSON array, convert each item inside it +// nativeArray := make([]interface{}, len(v)) +// for i, item := range v { +// nativeArray[i] = convertArrayToNative([]interface{}(item)) +// } +// // return nativeArray +// // return convertArrayToNative([]interface{}(data)) + +// case bson.M: +// convertedMap := make(map[string]interface{}) +// for key, value := range v { +// convertedMap[key] = convertBsonToNative(value) +// } +// return convertedMap + +// // case map[string]interface{}: +// // // If it's a BSON document (map), convert each value recursively +// // for k, val := range v { +// // v[k] = convertBsonToNative(val) +// // } +// // return v + +// default: +// // For all other types, return the value as is +// return v +// } + +// } +//RINMITVE.A W SLICE OF INTERFctx +//PRIMIVEDATAA WITH DTEIMTc +//dont do better than usinug any/interface for the containers of slices and maps + +func convertBsonToNative(data any) any { + // Check if the input is a slice (list) or a map, and process accordingly + fmt.Printf("this is data at the TOP %T, value: %+v\n", data, data) + switch v := data.(type) { + case primitive.DateTime: + return v.Time().UTC() + case primitive.A: //arrays/slices + nativeArray := make([]any, len(v)) + for i, item := range v { + nativeArray[i] = convertBsonToNative(item) + } + return nativeArray + case bson.M: //maps + convertedMap := make(map[string]any) + for key, value := range v { + convertedMap[key] = convertBsonToNative(value) + } + return convertedMap + case map[string]any: + for key, value := range v { + v[key] = convertBsonToNative(value) + } + return v + case int32: + return int(v) + case int64: + return int(v) + default: + return v + } + +} + +// case []map[string]interface{}: +// convertedArray := make([]map[string]interface{}, len(data)) +// for i, item := range data { +// fmt.Printf("this is value inside new []map[string]interface{}: %T, value: %+v\n", item, item) +// convertedArray[i] = convertBsonToNative(item).(map[string]interface{}) +// } +// return convertedArray +// case []interface{}: +// if len(data) > 0 { +// if _, ok := data[0].(map[string]interface{}); ok { +// // It’s a slice of maps: process accordingly +// convertedArray := make([]map[string]interface{}, len(data)) +// for i, item := range data { +// convertedArray[i] = convertBsonToNative(item).(map[string]interface{}) +// } +// return convertedArray +// } +// } +// fmt.Printf("this is data inside []interface{}: %T, value: %+v\n", data, data) +// return convertArrayToNative(data) +// case bson.A: // this is the same as []interface{} +// fmt.Printf("this is inside bson.A: %T, value: %+v\n", data, data) +// if len(data) > 0 { +// if _, ok := data[0].(map[string]interface{}); ok { +// // It’s a slice of maps: process accordingly +// convertedArray := make([]map[string]interface{}, len(data)) +// for i, item := range data { +// convertedArray[i] = convertBsonToNative(item).(map[string]interface{}) +// } +// return convertedArray +// } +// } +// return convertArrayToNative([]interface{}(data)) +// case map[string]interface{}: +// convertedMap := make(map[string]interface{}) +// for key, value := range data { +// fmt.Printf("this is value inside converted map inside map[string]interface{}: %T, value: %+v\n", value, value) +// convertedMap[key] = convertBsonToNative(value) +// } +// fmt.Printf("this is converted map inside map[string]interface{}: %T, value: %+v\n", convertedMap, convertedMap) +// return convertedMap + +// case bson.M: +// convertedMap := make(map[string]interface{}) +// for key, value := range data { +// convertedMap[key] = convertBsonToNative(value) +// } +// fmt.Printf("this is converted map inside bson.M: %T, value: %+v\n", convertedMap, convertedMap) +// return convertedMap +// default: +// return convertSingleValue(data) +// } +// } + +// Helper function to handle homogeneous arrays +// func convertArrayToNative(array []interface{}) interface{} { +// if len(array) == 0 { +// return array +// } +// // check if all elements are of type int +// allInts := true +// for _, item := range array { +// if _, ok := item.(int32); !ok { +// allInts = false +// break +// } +// } +// // convert to []int if all elements are integers +// if allInts { +// intArray := make([]int, len(array)) +// for i, item := range array { +// intArray[i] = int(item.(int32)) // assuming BSON uses int32 for integers +// } +// return intArray +// } +// // if not all integers then return []interface{} with recursive conversion +// nativeArray := make([]interface{}, len(array)) +// for i, item := range array { +// nativeArray[i] = convertBsonToNative(item) +// } +// return nativeArray +// } + +// // helper function to handle single BSON or primitive values +// func convertSingleValue(data interface{}) interface{} { +// switch v := data.(type) { +// case primitive.DateTime: +// return v.Time().UTC() +// case primitive.ObjectID: +// return v.Hex() +// case int32, int64, float32, float64, bool, string: +// return v +// default: +// return v +// } +// } + +// func convertBsonToNative(data interface{}) interface{} { +// // Check if the input is a slice (list) or a map, and process accordingly +// fmt.Printf("this is data type: %T, value: %+v\n", data, data) + +// switch data := data.(type) { +// case []interface{}: // If data is a generic list +// nativeArray := make([]interface{}, len(data)) +// for i, item := range data { +// nativeArray[i] = convertBsonToNative(item) // Recursively convert each item +// } +// fmt.Printf("this is nativeArray in interface{}: %T, value: %+v\n", nativeArray, nativeArray) +// return nativeArray + +// case bson.A: // If data is a BSON array +// //**we would need to change interface{} to be []int if we wanted that to work for us. and then our convertToBsonNative type would also have to not be data interface{} +// nativeArray := make([]interface{}, len(data)) +// for i, item := range data { +// nativeArray[i] = convertBsonToNative(item) // Recursively convert each item +// } +// fmt.Printf("this is nativeArray in bsonA: %T, value: %+v\n", nativeArray, nativeArray) +// return nativeArray + +// case map[string]interface{}: // If data is a generic map +// convertedMap := make(map[string]interface{}) +// for key, value := range data { +// convertedMap[key] = convertBsonToNative(value) // Recursively convert each value +// } +// return convertedMap + +// case bson.M: // If data is a BSON map +// convertedMap := make(map[string]interface{}) +// for key, value := range data { +// convertedMap[key] = convertBsonToNative(value) // Recursively convert each value +// } +// return convertedMap + +// default: +// // If data is not a list or map, handle BSON specific types and primitives +// return convertSingleValue(data) +// } +// } + func TabularDataBsonHelper(rawData [][]byte) ([]map[string]interface{}, error) { dataObjects := []map[string]interface{}{} + // fmt.Printf("this is rawData type: %T, value: %+v\n", rawData, rawData) for _, byteSlice := range rawData { - fmt.Printf("the byteslice is: %+v\n", byteSlice) + // fmt.Printf("the byteslice is: %+v\n", byteSlice) obj := map[string]interface{}{} - bson.Unmarshal(byteSlice, &obj) - for key, value := range obj { - if v, ok := value.(int32); ok { - obj[key] = int(v) - } + // bson.Unmarshal(byteSlice, &obj) //we are getting mongodb datetime objects, and al monogdb types + if err := bson.Unmarshal(byteSlice, &obj); err != nil { + return nil, err } - dataObjects = append(dataObjects, obj) - } + // for key, value := range obj { + // if v, ok := value.(int32); ok { + // obj[key] = int(v) + // } + // } + fmt.Printf("this is the object before conversion: %T, value: %+v\n", obj, obj) + convertedObj := convertBsonToNative(obj).(map[string]interface{}) + // fmt.Printf("this is object type: %T, value: %+v\n", convertedObj, convertedObj) + dataObjects = append(dataObjects, convertedObj) + } + fmt.Printf("this is dataObjectd type: %T, value: %+v\n", dataObjects, dataObjects) return dataObjects, nil } @@ -329,7 +555,7 @@ func FilterToProto(filter Filter) *pb.Filter { OrganizationIds: filter.OrganizationIds, MimeType: filter.MimeType, Interval: CaptureIntervalToProto(filter.Interval), - TagsFilter: TagsFilterToProto(filter.TagsFilter), + TagsFilter: TagsFilterToProto(filter.TagsFilter), BboxLabels: filter.BboxLabels, DatasetId: filter.DatasetId, } @@ -684,3 +910,47 @@ func (d *DataClient) RemoveBinaryDataFromDatasetByIDs(ctx context.Context, binar } return nil } + +//NOTES: +// func convertBsonTypes(obj map[string]interface{}) map[string]interface{} { +// for key, value := range obj { +// switch v := value.(type) { +// case string: +// //no conversion needed +// return value +// case float64, int32, int64: +// //no conversion needed +// return v +// case bool: +// //no conversion needed +// return v +// case nil: +// // Null - Represent as nil in Go +// return nil +// case bson.A: // BSON array +// result := []interface{}{} +// for _, item := range v { +// result = append(result, convertBsonTypes(item)) +// } +// return result +// case bson.M: // BSON object (map) +// result := map[string]interface{}{} +// for key, val := range v { +// result[key] = convertBsonTypes(val) +// } +// return result +// case time.Time: +// // Datetime - Convert BSON datetime to Go time.Time +// return v +// default: +// // Return other types as-is +// return v + +// // obj[key] = convertBsonTypes(v) +// case map[string]interface{}: +// // recursively convert nested maps +// obj[key] = convertBsonTypes(v) +// } +// } +// return obj +// } diff --git a/app/data_test.go b/app/data_test.go index 23b40ca6fbe..c0a6694e36e 100644 --- a/app/data_test.go +++ b/app/data_test.go @@ -2,6 +2,7 @@ package app import ( "context" + "fmt" "testing" "time" @@ -45,8 +46,8 @@ var ( BboxLabels = []string{bboxLabel} methodParameters = map[string]string{} tags = []string{tag} - startTime = time.Now().UTC() - endTime = time.Now().UTC() + startTime = time.Now().UTC().Round(time.Millisecond) + endTime = time.Now().UTC().Round(time.Millisecond) data = map[string]interface{}{ "key": "value", } @@ -95,25 +96,6 @@ var ( binaryIds = []BinaryID{binaryId} ) -// converts a slice of maps (representing MongoDB-like documents) into BSON byte arrays -func marshalToBSON(myBson []map[string]interface{}) [][]byte { - // create BSON documents for mongodb queries - var byteArray [][]byte - for _, item := range myBson { - input := bson.M(item) - bytes, _ := bson.Marshal(input) - byteArray = append(byteArray, bytes) - } - // matchStage := bson.M{"$match": bson.M{"organization_id": "e76d1b3b-0468-4efd-bb7f-fb1d2b352fcb"}} - // limitStage := bson.M{"$limit": 1} - // convert to BSON byte arrays - // matchBytes, _ := bson.Marshal(matchStage) - // limitBytes, _ := bson.Marshal(limitStage) - // mqlbinary := [][]byte{matchBytes, limitBytes} - // return mqlbinary - return byteArray -} - // set up gRPC client?? func createGrpclient() datapb.DataServiceClient { return &inject.DataServiceClient{} @@ -227,6 +209,7 @@ func TestDataClient(t *testing.T) { t.Run("TabularDataBySQL", func(t *testing.T) { expectedOrgId := organizationId expectedSqlQuery := sqlQuery + expectedRawData := []map[string]interface{}{ { "key1": startTime, @@ -237,7 +220,15 @@ func TestDataClient(t *testing.T) { }, }, } - expectedRawDataPb := marshalToBSON(expectedRawData) + //convert rawData to BSON + var expectedRawDataPb [][]byte + for _, byte := range expectedRawData { + bsonByte, err := bson.Marshal(byte) + if err != nil { + t.Fatalf("Failed to marshal expectedRawData: %v", err) + } + expectedRawDataPb = append(expectedRawDataPb, bsonByte) + } grpcClient.TabularDataBySQLFunc = func(ctx context.Context, in *datapb.TabularDataBySQLRequest, opts ...grpc.CallOption) (*datapb.TabularDataBySQLResponse, error) { test.That(t, in.OrganizationId, test.ShouldEqual, expectedOrgId) //to proto test.That(t, in.SqlQuery, test.ShouldResemble, expectedSqlQuery) //to proto @@ -251,33 +242,85 @@ func TestDataClient(t *testing.T) { test.That(t, response, test.ShouldResemble, expectedRawData) }) + //I want my response to look like expectedRawData --> good!! + //I want my input to TabularDataByMQLRespone to be what expectedRawData would look like in [][]byte form!!!!! + /* + + need to convert expectedRawData to BSON and then encode it as [][]byte to use it in your gRPC resp + + Convert expectedRawData to [][]byte --> expectedRawData to BSON. + Define TabularDataByMQLFunc ---> use the bson expectedRawData in TabularDataByMQLFunc + + + // Serialize expectedRawData to BSON and convert to [][]byte + var expectedRawDataPb [][]byte + for _, doc := range expectedRawData { + bsonDoc, err := bson.Marshal(doc) + if err != nil { + t.Fatalf("Failed to marshal expectedRawData: %v", err) + } + expectedRawDataPb = append(expectedRawDataPb, bsonDoc) + } + */ t.Run("TabularDataByMQL", func(t *testing.T) { expectedOrgId := organizationId - expectedMqlBinary := marshalToBSON(mqlQuery) //this is [][]byte type + //create expected mqlBinary + matchStage := bson.M{"$match": bson.M{"organization_id": "e76d1b3b-0468-4efd-bb7f-fb1d2b352fcb"}} + limitStage := bson.M{"$limit": 1} + // convert to BSON byte arrays + matchBytes, _ := bson.Marshal(matchStage) + limitBytes, _ := bson.Marshal(limitStage) + mqlbinary := [][]byte{matchBytes, limitBytes} + expectedMqlBinary := mqlbinary + + // expectedMqlBinary := marshalToBSON(mqlQuery) //this is [][]byte type //this is the format we want to get back... - expectedRawData := []map[string]interface{}{ + + //add a bool + expectedRawData := []map[string]any{ { "key1": startTime, "key2": "2", - "key3": []int{1, 2, 3}, - "key4": map[string]interface{}{ + "key3": []any{1, 2, 3}, //slice of integers + "key4": map[string]any{ // map and rep of JSON object "key4sub1": endTime, }, + "key5": 4.05, + "key6": []any{true, false, true}, //array of bools + "key7": []map[string]any{ // slice of maps + { + "nestedKey1": "simpleValue", + }, { + "nestedKey2": startTime, + }, + }, }, } - //this is the rawData in protobuf type that we will expect to pass to the response - expectedRawDataPb := marshalToBSON(expectedRawData) + //convert rawData to BSON + var expectedRawDataPb [][]byte + for _, byte := range expectedRawData { + bsonByte, err := bson.Marshal(byte) + if err != nil { + t.Fatalf("Failed to marshal expectedRawData: %v", err) + } + expectedRawDataPb = append(expectedRawDataPb, bsonByte) + } + + fmt.Printf("this is expectedBinaryDataPb type: %T, value: %+v\n", expectedRawDataPb, expectedRawDataPb) grpcClient.TabularDataByMQLFunc = func(ctx context.Context, in *datapb.TabularDataByMQLRequest, opts ...grpc.CallOption) (*datapb.TabularDataByMQLResponse, error) { test.That(t, in.OrganizationId, test.ShouldEqual, expectedOrgId) //to proto - test.That(t, in.MqlBinary, test.ShouldResemble, expectedMqlBinary) //to proto + test.That(t, in.MqlBinary, test.ShouldResemble, expectedMqlBinary) //to proto //we also want this to just be [][]byte ...? return &datapb.TabularDataByMQLResponse{ - RawData: expectedRawDataPb, + RawData: expectedRawDataPb, //we want this to be expectedRawData but in proto form (so [][]byte) }, nil } response, _ := client.TabularDataByMQL(context.Background(), expectedOrgId, expectedMqlBinary) - test.That(t, response, test.ShouldResemble, expectedRawData) + // fmt.Printf("this is actual type: %T, value: %+v\n", response, response) + // fmt.Printf("this is expected type: %T, value: %+v\n", expectedRawData, expectedRawData) + test.That(t, response, test.ShouldResemble, expectedRawData) //we want this to be expectedRawData + }) t.Run("BinaryDataByFilter", func(t *testing.T) { includeBinary := true From 9d60f98f3bb8a07906b1654971270bff39e2fd88 Mon Sep 17 00:00:00 2001 From: Julie Krasnick Date: Wed, 13 Nov 2024 12:01:34 -0500 Subject: [PATCH 16/25] fixed conversions --- app/data_client.go | 185 +-------------------------------------------- app/data_test.go | 23 ++++-- 2 files changed, 20 insertions(+), 188 deletions(-) diff --git a/app/data_client.go b/app/data_client.go index 63b74523d13..f1f12246dec 100644 --- a/app/data_client.go +++ b/app/data_client.go @@ -286,46 +286,6 @@ func DataRequestToProto(dataRequest DataRequest) (*pb.DataRequest, error) { } -// func convertBsonToNative(data interface{}) interface{} { -// switch v := data.(type) { -// case primitive.DateTime: -// // Convert BSON DateTime to Go time.Time -// return v.Time().UTC() -// // case primitive.ObjectID: -// // // Convert BSON ObjectId to string (or another suitable representation) -// // return v.Hex() -// case int32, int64, float32, float64, bool, string: -// return v - -// case bson.A: -// // If it's a BSON array, convert each item inside it -// nativeArray := make([]interface{}, len(v)) -// for i, item := range v { -// nativeArray[i] = convertArrayToNative([]interface{}(item)) -// } -// // return nativeArray -// // return convertArrayToNative([]interface{}(data)) - -// case bson.M: -// convertedMap := make(map[string]interface{}) -// for key, value := range v { -// convertedMap[key] = convertBsonToNative(value) -// } -// return convertedMap - -// // case map[string]interface{}: -// // // If it's a BSON document (map), convert each value recursively -// // for k, val := range v { -// // v[k] = convertBsonToNative(val) -// // } -// // return v - -// default: -// // For all other types, return the value as is -// return v -// } - -// } //RINMITVE.A W SLICE OF INTERFctx //PRIMIVEDATAA WITH DTEIMTc //dont do better than usinug any/interface for the containers of slices and maps @@ -342,13 +302,13 @@ func convertBsonToNative(data any) any { nativeArray[i] = convertBsonToNative(item) } return nativeArray - case bson.M: //maps + case bson.M: //maps convertedMap := make(map[string]any) for key, value := range v { convertedMap[key] = convertBsonToNative(value) } return convertedMap - case map[string]any: + case map[string]any: //handle go maps for key, value := range v { v[key] = convertBsonToNative(value) } @@ -360,147 +320,8 @@ func convertBsonToNative(data any) any { default: return v } - -} - -// case []map[string]interface{}: -// convertedArray := make([]map[string]interface{}, len(data)) -// for i, item := range data { -// fmt.Printf("this is value inside new []map[string]interface{}: %T, value: %+v\n", item, item) -// convertedArray[i] = convertBsonToNative(item).(map[string]interface{}) -// } -// return convertedArray -// case []interface{}: -// if len(data) > 0 { -// if _, ok := data[0].(map[string]interface{}); ok { -// // It’s a slice of maps: process accordingly -// convertedArray := make([]map[string]interface{}, len(data)) -// for i, item := range data { -// convertedArray[i] = convertBsonToNative(item).(map[string]interface{}) -// } -// return convertedArray -// } -// } -// fmt.Printf("this is data inside []interface{}: %T, value: %+v\n", data, data) -// return convertArrayToNative(data) -// case bson.A: // this is the same as []interface{} -// fmt.Printf("this is inside bson.A: %T, value: %+v\n", data, data) -// if len(data) > 0 { -// if _, ok := data[0].(map[string]interface{}); ok { -// // It’s a slice of maps: process accordingly -// convertedArray := make([]map[string]interface{}, len(data)) -// for i, item := range data { -// convertedArray[i] = convertBsonToNative(item).(map[string]interface{}) -// } -// return convertedArray -// } -// } -// return convertArrayToNative([]interface{}(data)) -// case map[string]interface{}: -// convertedMap := make(map[string]interface{}) -// for key, value := range data { -// fmt.Printf("this is value inside converted map inside map[string]interface{}: %T, value: %+v\n", value, value) -// convertedMap[key] = convertBsonToNative(value) -// } -// fmt.Printf("this is converted map inside map[string]interface{}: %T, value: %+v\n", convertedMap, convertedMap) -// return convertedMap - -// case bson.M: -// convertedMap := make(map[string]interface{}) -// for key, value := range data { -// convertedMap[key] = convertBsonToNative(value) -// } -// fmt.Printf("this is converted map inside bson.M: %T, value: %+v\n", convertedMap, convertedMap) -// return convertedMap -// default: -// return convertSingleValue(data) -// } -// } -// Helper function to handle homogeneous arrays -// func convertArrayToNative(array []interface{}) interface{} { -// if len(array) == 0 { -// return array -// } -// // check if all elements are of type int -// allInts := true -// for _, item := range array { -// if _, ok := item.(int32); !ok { -// allInts = false -// break -// } -// } -// // convert to []int if all elements are integers -// if allInts { -// intArray := make([]int, len(array)) -// for i, item := range array { -// intArray[i] = int(item.(int32)) // assuming BSON uses int32 for integers -// } -// return intArray -// } -// // if not all integers then return []interface{} with recursive conversion -// nativeArray := make([]interface{}, len(array)) -// for i, item := range array { -// nativeArray[i] = convertBsonToNative(item) -// } -// return nativeArray -// } - -// // helper function to handle single BSON or primitive values -// func convertSingleValue(data interface{}) interface{} { -// switch v := data.(type) { -// case primitive.DateTime: -// return v.Time().UTC() -// case primitive.ObjectID: -// return v.Hex() -// case int32, int64, float32, float64, bool, string: -// return v -// default: -// return v -// } -// } - -// func convertBsonToNative(data interface{}) interface{} { -// // Check if the input is a slice (list) or a map, and process accordingly -// fmt.Printf("this is data type: %T, value: %+v\n", data, data) - -// switch data := data.(type) { -// case []interface{}: // If data is a generic list -// nativeArray := make([]interface{}, len(data)) -// for i, item := range data { -// nativeArray[i] = convertBsonToNative(item) // Recursively convert each item -// } -// fmt.Printf("this is nativeArray in interface{}: %T, value: %+v\n", nativeArray, nativeArray) -// return nativeArray - -// case bson.A: // If data is a BSON array -// //**we would need to change interface{} to be []int if we wanted that to work for us. and then our convertToBsonNative type would also have to not be data interface{} -// nativeArray := make([]interface{}, len(data)) -// for i, item := range data { -// nativeArray[i] = convertBsonToNative(item) // Recursively convert each item -// } -// fmt.Printf("this is nativeArray in bsonA: %T, value: %+v\n", nativeArray, nativeArray) -// return nativeArray - -// case map[string]interface{}: // If data is a generic map -// convertedMap := make(map[string]interface{}) -// for key, value := range data { -// convertedMap[key] = convertBsonToNative(value) // Recursively convert each value -// } -// return convertedMap - -// case bson.M: // If data is a BSON map -// convertedMap := make(map[string]interface{}) -// for key, value := range data { -// convertedMap[key] = convertBsonToNative(value) // Recursively convert each value -// } -// return convertedMap - -// default: -// // If data is not a list or map, handle BSON specific types and primitives -// return convertSingleValue(data) -// } -// } +} func TabularDataBsonHelper(rawData [][]byte) ([]map[string]interface{}, error) { dataObjects := []map[string]interface{}{} diff --git a/app/data_test.go b/app/data_test.go index c0a6694e36e..173fb17ab81 100644 --- a/app/data_test.go +++ b/app/data_test.go @@ -210,14 +210,24 @@ func TestDataClient(t *testing.T) { expectedOrgId := organizationId expectedSqlQuery := sqlQuery - expectedRawData := []map[string]interface{}{ + expectedRawData := []map[string]any{ { "key1": startTime, "key2": "2", - "key3": []int{1, 2, 3}, - "key4": map[string]interface{}{ + "key3": []any{1, 2, 3}, //slice of integers + "key4": map[string]any{ // map and rep of JSON object "key4sub1": endTime, }, + "key5": 4.05, + "key6": []any{true, false, true}, //array of bools + "key7": []any{ // slice of maps + map[string]any{ + "nestedKey1": "simpleValue", + }, + map[string]any{ + "nestedKey2": startTime, + }, + }, }, } //convert rawData to BSON @@ -288,10 +298,11 @@ func TestDataClient(t *testing.T) { }, "key5": 4.05, "key6": []any{true, false, true}, //array of bools - "key7": []map[string]any{ // slice of maps - { + "key7": []any{ // slice of maps + map[string]any{ "nestedKey1": "simpleValue", - }, { + }, + map[string]any{ "nestedKey2": startTime, }, }, From 19cb64e6ec38629601c2d3687229f73a10190f3a Mon Sep 17 00:00:00 2001 From: Julie Krasnick Date: Wed, 13 Nov 2024 13:49:18 -0500 Subject: [PATCH 17/25] clean up data_client --- app/data_client.go | 498 ++++++++++++++++++++------------------------- 1 file changed, 221 insertions(+), 277 deletions(-) diff --git a/app/data_client.go b/app/data_client.go index f1f12246dec..c6e701b1eca 100644 --- a/app/data_client.go +++ b/app/data_client.go @@ -4,7 +4,6 @@ package app import ( "context" - "fmt" "time" "go.mongodb.org/mongo-driver/bson" @@ -27,6 +26,63 @@ import ( // i want to wrap NewDataServiceClient define a new dataclient struct and call the wrappers of the functions // // i want the user to call my dataClient struct w my wrappers and not the proto functions + +type DataClient struct { + client pb.DataServiceClient +} + +// Order specifies the order in which data is returned. +type Order int32 + +const ( + Unspecified Order = 0 + Descending Order = 1 + Ascending Order = 2 +) + +// DataRequest encapsulates the filter for the data, a limit on the maximum results returned, +// a last string associated with the last returned document, and the sorting order by time. +type DataRequest struct { + Filter Filter + Limit uint64 + Last string + SortOrder Order +} + +// Filter defines the fields over which we can filter data using a logic AND. +type Filter struct { + ComponentName string + ComponentType string + Method string + RobotName string + RobotId string + PartName string + PartId string + LocationIds []string + OrganizationIds []string + MimeType []string + Interval CaptureInterval + TagsFilter TagsFilter + BboxLabels []string + DatasetId string +} + +// TagsFilter defines the type of filtering and, if applicable, over which tags to perform a logical OR. +type TagsFilterType int32 + +const ( + TagsFilterTypeUnspecified TagsFilterType = 0 + TagsFilterTypeMatchByOr TagsFilterType = 1 + TagsFilterTypeTagged TagsFilterType = 2 + TagsFilterTypeUntagged TagsFilterType = 3 +) + +type TagsFilter struct { + Type TagsFilterType + Tags []string +} + +// CaptureMetadata contains information on the settings used for the data capture. type CaptureMetadata struct { OrganizationId string LocationId string @@ -41,27 +97,11 @@ type CaptureMetadata struct { Tags []string MimeType string } -type BinaryID struct { - FileId string - OrganizationId string - LocationId string -} -type BoundingBox struct { - id string - label string - xMinNormalized float64 - yMinNormalized float64 - xMaxNormalized float64 - yMaxNormalized float64 -} -type DataRequest struct { - Filter Filter - Limit uint64 - Last string - SortOrder Order -} -type Annotations struct { - bboxes []BoundingBox + +// CaptureInterval describes the start and end time of the capture in this file. +type CaptureInterval struct { + Start time.Time + End time.Time } type TabularData struct { Data map[string]interface{} @@ -70,6 +110,7 @@ type TabularData struct { TimeRequested time.Time TimeReceived time.Time } + type BinaryData struct { Binary []byte Metadata BinaryMetadata @@ -86,42 +127,48 @@ type BinaryMetadata struct { Annotations Annotations DatasetIDs []string } -type Filter struct { - ComponentName string - ComponentType string - Method string - RobotName string - RobotId string - PartName string - PartId string - LocationIds []string - OrganizationIds []string - MimeType []string - Interval CaptureInterval - TagsFilter TagsFilter - BboxLabels []string - DatasetId string + +type BinaryID struct { + FileId string + OrganizationId string + LocationId string +} +type BoundingBox struct { + Id string + Label string + XMinNormalized float64 + YMinNormalized float64 + XMaxNormalized float64 + YMaxNormalized float64 } -type TagsFilter struct { - Type int32 - Tags []string +type Annotations struct { + Bboxes []BoundingBox } -type CaptureInterval struct { - Start time.Time - End time.Time + +// NewDataClient constructs a new DataClient from connection passed in. +func NewDataClient( + ctx context.Context, + channel rpc.ClientConn, //this should just take a channel that the viamClient passes in + logger logging.Logger, +) (*DataClient, error) { + d := pb.NewDataServiceClient(channel) + return &DataClient{ + client: d, + }, nil } func BoundingBoxFromProto(proto *pb.BoundingBox) BoundingBox { return BoundingBox{ - id: proto.Id, - label: proto.Label, - xMinNormalized: proto.XMinNormalized, - yMinNormalized: proto.YMinNormalized, - xMaxNormalized: proto.XMaxNormalized, - yMaxNormalized: proto.YMaxNormalized, + Id: proto.Id, + Label: proto.Label, + XMinNormalized: proto.XMinNormalized, + YMinNormalized: proto.YMinNormalized, + XMaxNormalized: proto.XMaxNormalized, + YMaxNormalized: proto.YMaxNormalized, } } + func AnnotationsFromProto(proto *pb.Annotations) Annotations { if proto == nil { return Annotations{} @@ -132,30 +179,43 @@ func AnnotationsFromProto(proto *pb.Annotations) Annotations { bboxes[i] = BoundingBoxFromProto(bboxProto) } return Annotations{ - bboxes: bboxes, + Bboxes: bboxes, } } + func AnnotationsToProto(annotations Annotations) *pb.Annotations { var protoBboxes []*pb.BoundingBox - for _, bbox := range annotations.bboxes { + for _, bbox := range annotations.Bboxes { protoBboxes = append(protoBboxes, &pb.BoundingBox{ - Id: bbox.id, - Label: bbox.label, - XMinNormalized: bbox.xMinNormalized, - YMinNormalized: bbox.yMinNormalized, - XMaxNormalized: bbox.xMaxNormalized, - YMaxNormalized: bbox.yMaxNormalized, + Id: bbox.Id, + Label: bbox.Label, + XMinNormalized: bbox.XMinNormalized, + YMinNormalized: bbox.YMinNormalized, + XMaxNormalized: bbox.XMaxNormalized, + YMaxNormalized: bbox.YMaxNormalized, }) } return &pb.Annotations{ Bboxes: protoBboxes, } } + +func methodParamsFromProto(proto map[string]*anypb.Any) map[string]string { + methodParameters := make(map[string]string) + for key, value := range proto { + structValue := &structpb.Value{} + if err := value.UnmarshalTo(structValue); err != nil { + return nil + } + methodParameters[key] = structValue.String() + } + return methodParameters +} + func CaptureMetadataFromProto(proto *pb.CaptureMetadata) CaptureMetadata { if proto == nil { return CaptureMetadata{} } - return CaptureMetadata{ OrganizationId: proto.OrganizationId, LocationId: proto.LocationId, @@ -170,8 +230,8 @@ func CaptureMetadataFromProto(proto *pb.CaptureMetadata) CaptureMetadata { Tags: proto.Tags, MimeType: proto.MimeType, } - } + func CaptureMetadataToProto(metadata CaptureMetadata) *pb.CaptureMetadata { methodParms, _ := protoutils.ConvertStringMapToAnyPBMap(metadata.MethodParameters) return &pb.CaptureMetadata{ @@ -191,24 +251,13 @@ func CaptureMetadataToProto(metadata CaptureMetadata) *pb.CaptureMetadata { } -func methodParamsFromProto(proto map[string]*anypb.Any) map[string]string { - methodParameters := make(map[string]string) - for key, value := range proto { - structValue := &structpb.Value{} - if err := value.UnmarshalTo(structValue); err != nil { - return nil - } - methodParameters[key] = structValue.String() - } - return methodParameters -} - func BinaryDataFromProto(proto *pb.BinaryData) BinaryData { return BinaryData{ Binary: proto.Binary, Metadata: BinaryMetadataFromProto(proto.Metadata), } } + func BinaryDataToProto(binaryData BinaryData) *pb.BinaryData { return &pb.BinaryData{ Binary: binaryData.Binary, @@ -229,6 +278,7 @@ func BinaryMetadataFromProto(proto *pb.BinaryMetadata) BinaryMetadata { DatasetIDs: proto.DatasetIds, } } + func BinaryMetadataToProto(binaryMetadata BinaryMetadata) *pb.BinaryMetadata { return &pb.BinaryMetadata{ Id: binaryMetadata.ID, @@ -243,9 +293,7 @@ func BinaryMetadataToProto(binaryMetadata BinaryMetadata) *pb.BinaryMetadata { } } -// returns tabular data and the associated metadata func TabularDataFromProto(proto *pb.TabularData, metadata *pb.CaptureMetadata) TabularData { - fmt.Printf("this is proto in tabulardatafrom proto: %+v\n", metadata) return TabularData{ Data: proto.Data.AsMap(), MetadataIndex: proto.MetadataIndex, @@ -254,6 +302,7 @@ func TabularDataFromProto(proto *pb.TabularData, metadata *pb.CaptureMetadata) T TimeReceived: proto.TimeReceived.AsTime(), } } + func TabularDataToProto(tabularData TabularData) *pb.TabularData { structData, err := utils.StructToStructPb(tabularData.Data) if err != nil { @@ -266,6 +315,7 @@ func TabularDataToProto(tabularData TabularData) *pb.TabularData { TimeReceived: timestamppb.New(tabularData.TimeReceived), } } + func TabularDataToProtoList(tabularDatas []TabularData) []*pb.TabularData { var protoList []*pb.TabularData for _, tabularData := range tabularDatas { @@ -276,6 +326,7 @@ func TabularDataToProtoList(tabularDatas []TabularData) []*pb.TabularData { } return protoList } + func DataRequestToProto(dataRequest DataRequest) (*pb.DataRequest, error) { return &pb.DataRequest{ Filter: FilterToProto(dataRequest.Filter), @@ -286,67 +337,6 @@ func DataRequestToProto(dataRequest DataRequest) (*pb.DataRequest, error) { } -//RINMITVE.A W SLICE OF INTERFctx -//PRIMIVEDATAA WITH DTEIMTc -//dont do better than usinug any/interface for the containers of slices and maps - -func convertBsonToNative(data any) any { - // Check if the input is a slice (list) or a map, and process accordingly - fmt.Printf("this is data at the TOP %T, value: %+v\n", data, data) - switch v := data.(type) { - case primitive.DateTime: - return v.Time().UTC() - case primitive.A: //arrays/slices - nativeArray := make([]any, len(v)) - for i, item := range v { - nativeArray[i] = convertBsonToNative(item) - } - return nativeArray - case bson.M: //maps - convertedMap := make(map[string]any) - for key, value := range v { - convertedMap[key] = convertBsonToNative(value) - } - return convertedMap - case map[string]any: //handle go maps - for key, value := range v { - v[key] = convertBsonToNative(value) - } - return v - case int32: - return int(v) - case int64: - return int(v) - default: - return v - } - -} - -func TabularDataBsonHelper(rawData [][]byte) ([]map[string]interface{}, error) { - dataObjects := []map[string]interface{}{} - // fmt.Printf("this is rawData type: %T, value: %+v\n", rawData, rawData) - for _, byteSlice := range rawData { - // fmt.Printf("the byteslice is: %+v\n", byteSlice) - obj := map[string]interface{}{} - // bson.Unmarshal(byteSlice, &obj) //we are getting mongodb datetime objects, and al monogdb types - if err := bson.Unmarshal(byteSlice, &obj); err != nil { - return nil, err - } - // for key, value := range obj { - // if v, ok := value.(int32); ok { - // obj[key] = int(v) - // } - // } - fmt.Printf("this is the object before conversion: %T, value: %+v\n", obj, obj) - convertedObj := convertBsonToNative(obj).(map[string]interface{}) - // fmt.Printf("this is object type: %T, value: %+v\n", convertedObj, convertedObj) - dataObjects = append(dataObjects, convertedObj) - } - fmt.Printf("this is dataObjectd type: %T, value: %+v\n", dataObjects, dataObjects) - return dataObjects, nil -} - func BinaryIdToProto(binaryId BinaryID) *pb.BinaryID { return &pb.BinaryID{ FileId: binaryId.FileId, @@ -381,12 +371,14 @@ func FilterToProto(filter Filter) *pb.Filter { DatasetId: filter.DatasetId, } } + func CaptureIntervalToProto(interval CaptureInterval) *pb.CaptureInterval { return &pb.CaptureInterval{ Start: timestamppb.New(interval.Start), End: timestamppb.New(interval.End), } } + func TagsFilterToProto(tagsFilter TagsFilter) *pb.TagsFilter { return &pb.TagsFilter{ Type: pb.TagsFilterType(tagsFilter.Type), @@ -394,14 +386,6 @@ func TagsFilterToProto(tagsFilter TagsFilter) *pb.TagsFilter { } } -type Order int32 - -const ( - Unspecified Order = 0 - Descending Order = 1 - Ascending Order = 2 -) - func OrderToProto(sortOrder Order) pb.Order { switch sortOrder { case Ascending: @@ -413,43 +397,65 @@ func OrderToProto(sortOrder Order) pb.Order { } } -type DataClient struct { - //do we want this to be a public interface that defines the functions but does not include client and private details? - //would not include client and private details - client pb.DataServiceClient +// convertBsonToNative recursively converts BSON types (e.g., DateTime, arrays, maps) +// into their native Go equivalents. This ensures all BSON data types are converted +// to the appropriate Go types like time.Time, slices, and maps. +func convertBsonToNative(data any) any { + switch v := data.(type) { + case primitive.DateTime: + return v.Time().UTC() + case primitive.A: // Handle BSON arrays/slices + nativeArray := make([]any, len(v)) + for i, item := range v { + nativeArray[i] = convertBsonToNative(item) + } + return nativeArray + case bson.M: // Handle BSON maps + convertedMap := make(map[string]any) + for key, value := range v { + convertedMap[key] = convertBsonToNative(value) + } + return convertedMap + case map[string]any: // Handle Go maps + for key, value := range v { + v[key] = convertBsonToNative(value) + } + return v + case int32: + return int(v) + case int64: + return int(v) + default: + return v + } } -// (private) dataClient implements DataServiceClient. **do we want this? -// type dataClient interface { -// // actual hold implementations of functions - how would the NewDataClient function work if we had private and public functions? -// // client pb.DataServiceClient -// } - -// NewDataClient constructs a new DataClient from connection passed in. -func NewDataClient( - ctx context.Context, - channel rpc.ClientConn, //this should just take a channek that the viamClient passes in - logger logging.Logger, -) (*DataClient, error) { - d := pb.NewDataServiceClient(channel) - return &DataClient{ - client: d, - }, nil +// BsonToGo converts raw BSON data (as [][]byte) into native Go types and interfaces. +// Returns a slice of maps representing the data objects. +func BsonToGo(rawData [][]byte) ([]map[string]any, error) { + dataObjects := []map[string]any{} + for _, byteSlice := range rawData { + // Unmarshal each BSON byte slice into a Go map + obj := map[string]any{} + if err := bson.Unmarshal(byteSlice, &obj); err != nil { + return nil, err + } + // Convert the unmarshalled map to native Go types + convertedObj := convertBsonToNative(obj).(map[string]any) + dataObjects = append(dataObjects, convertedObj) + } + return dataObjects, nil } // TabularDataByFilter queries tabular data and metadata based on given filters. -// returns []TabularData, uint64, string, and error: returns multiple things containing the following: List[TabularData]: The tabular data, int: The count (number of entries), str: The last-returned page ID. func (d *DataClient) TabularDataByFilter( ctx context.Context, filter Filter, - limit uint64, //optional - last string, //optional - sortOrder Order, //optional + limit uint64, + last string, + sortOrder Order, countOnly bool, includeInternalData bool) ([]TabularData, uint64, string, error) { - if limit == 0 { - limit = 50 - } resp, err := d.client.TabularDataByFilter(ctx, &pb.TabularDataByFilterRequest{ DataRequest: &pb.DataRequest{ Filter: FilterToProto(filter), @@ -466,67 +472,38 @@ func (d *DataClient) TabularDataByFilter( var metadata *pb.CaptureMetadata for _, data := range resp.Data { if len(resp.Metadata) != 0 && int(data.MetadataIndex) >= len(resp.Metadata) { - metadata = &pb.CaptureMetadata{} // Create new metadata if index is out of range + metadata = &pb.CaptureMetadata{} } else { metadata = resp.Metadata[data.MetadataIndex] } dataArray = append(dataArray, TabularDataFromProto(data, metadata)) - } return dataArray, resp.Count, resp.Last, nil } -// returns an array of data objects -// interface{} is a special type in Go that represents any type. -// so map[string]interface{} is a map (aka a dictionary) where the keys are strings and the values are of any type -// a list of maps --> like we had in python a list of dictionaries +// TabularDataBySQL queries tabular data with a SQL query. func (d *DataClient) TabularDataBySQL(ctx context.Context, organizationId string, sqlQuery string) ([]map[string]interface{}, error) { resp, err := d.client.TabularDataBySQL(ctx, &pb.TabularDataBySQLRequest{OrganizationId: organizationId, SqlQuery: sqlQuery}) if err != nil { return nil, err } - dataObjects, nil := TabularDataBsonHelper(resp.RawData) + dataObjects, nil := BsonToGo(resp.RawData) return dataObjects, nil } +// TabularDataByMQL queries tabular data with an MQL (MongoDB Query Language) query. func (d *DataClient) TabularDataByMQL(ctx context.Context, organizationId string, mqlbinary [][]byte) ([]map[string]interface{}, error) { - //need to double check this mqlbinary type***?? resp, err := d.client.TabularDataByMQL(ctx, &pb.TabularDataByMQLRequest{OrganizationId: organizationId, MqlBinary: mqlbinary}) if err != nil { return nil, err } - - // var result []map[string]interface{} - // for _, bsonBytes := range resp.RawData { - // var decodedData map[string]interface{} - // err := bson.Unmarshal(bsonBytes, &decodedData) - // if err != nil { - // return nil, fmt.Errorf("error decoding BSON: %v", err) - // } - // fmt.Printf("this is decoded data: %+v\n", decodedData) - // result = append(result, decodedData) - - // } - result, nil := TabularDataBsonHelper(resp.RawData) - fmt.Printf("this is result: %+v\n", result) + result, nil := BsonToGo(resp.RawData) return result, nil - - // dataObjects := make([]map[string]interface{}, len(resp.RawData)) - // for i, rawData := range resp.RawData { - // var obj map[string]interface{} - // if err := bson.Unmarshal(rawData, &obj); err != nil { - // fmt.Printf("(func) unmarshalling error %d: %v", i, err) - // return nil, err - // } - // dataObjects[i] = obj - // } - - // fmt.Println("printing Deserialized dataObjects here", dataObjects) } +// BinaryDataByFilter queries binary data and metadata based on given filters. func (d *DataClient) BinaryDataByFilter( - //dest string?? ctx context.Context, filter Filter, limit uint64, @@ -535,12 +512,8 @@ func (d *DataClient) BinaryDataByFilter( includeBinary bool, countOnly bool, includeInternalData bool) ([]BinaryData, uint64, string, error) { - fmt.Println("client.BinaryDataByFilter was called") - if limit == 0 { - limit = 50 - } resp, err := d.client.BinaryDataByFilter(ctx, &pb.BinaryDataByFilterRequest{ - DataRequest: &pb.DataRequest{ //need to do checks to make sure it fits the constraints + DataRequest: &pb.DataRequest{ Filter: FilterToProto(filter), Limit: limit, Last: last, @@ -553,17 +526,15 @@ func (d *DataClient) BinaryDataByFilter( if err != nil { return nil, 0, "", err } - // Convert protobuf BinaryData to Go-native BinaryData data := make([]BinaryData, len(resp.Data)) for i, protoData := range resp.Data { data[i] = BinaryDataFromProto(protoData) } - // return resp.Data, resp.Count, resp.Last, nil return data, resp.Count, resp.Last, nil } -// do i need to be including error as a return type for all of these? +// BinaryDataByIDs queries binary data and metadata based on given IDs. func (d *DataClient) BinaryDataByIDs(ctx context.Context, binaryIds []BinaryID) ([]BinaryData, error) { resp, err := d.client.BinaryDataByIDs(ctx, &pb.BinaryDataByIDsRequest{ IncludeBinary: true, @@ -572,13 +543,14 @@ func (d *DataClient) BinaryDataByIDs(ctx context.Context, binaryIds []BinaryID) if err != nil { return nil, err } - // Convert protobuf BinaryData to Go-native BinaryData data := make([]BinaryData, len(resp.Data)) for i, protoData := range resp.Data { data[i] = BinaryDataFromProto(protoData) } - return data, nil //the return type of this is: var data []BinaryData --> is that okay??? , do we only want go native types does this count? + return data, nil } + +// DeleteTabularData deletes tabular data older than a number of days, based on the given organization ID. func (d *DataClient) DeleteTabularData(ctx context.Context, organizationId string, deleteOlderThanDays uint32) (uint64, error) { resp, err := d.client.DeleteTabularData(ctx, &pb.DeleteTabularDataRequest{ OrganizationId: organizationId, @@ -590,11 +562,8 @@ func (d *DataClient) DeleteTabularData(ctx context.Context, organizationId strin return resp.DeletedCount, nil } +// DeleteBinaryDataByFilter deletes binary data based on given filters. func (d *DataClient) DeleteBinaryDataByFilter(ctx context.Context, filter Filter) (uint64, error) { - //should probably do some sort of check that filter isn't empty otherwise i need to do something - // if filter == nil { - // filter = &pb.Filter{} - // } resp, err := d.client.DeleteBinaryDataByFilter(ctx, &pb.DeleteBinaryDataByFilterRequest{ Filter: FilterToProto(filter), IncludeInternalData: true, @@ -604,6 +573,8 @@ func (d *DataClient) DeleteBinaryDataByFilter(ctx context.Context, filter Filter } return resp.DeletedCount, nil } + +// DeleteBinaryDataByIDs deletes binary data based on given IDs. func (d *DataClient) DeleteBinaryDataByIDs(ctx context.Context, binaryIds []BinaryID) (uint64, error) { resp, err := d.client.DeleteBinaryDataByIDs(ctx, &pb.DeleteBinaryDataByIDsRequest{ BinaryIds: BinaryIdsToProto(binaryIds), @@ -613,6 +584,8 @@ func (d *DataClient) DeleteBinaryDataByIDs(ctx context.Context, binaryIds []Bina } return resp.DeletedCount, nil } + +// AddTagsToBinaryDataByIDs adds string tags, unless the tags are already present, to binary data based on given IDs. func (d *DataClient) AddTagsToBinaryDataByIDs(ctx context.Context, tags []string, binaryIds []BinaryID) error { _, err := d.client.AddTagsToBinaryDataByIDs(ctx, &pb.AddTagsToBinaryDataByIDsRequest{BinaryIds: BinaryIdsToProto(binaryIds), Tags: tags}) if err != nil { @@ -620,6 +593,8 @@ func (d *DataClient) AddTagsToBinaryDataByIDs(ctx context.Context, tags []string } return nil } + +// AddTagsToBinaryDataByFilter adds string tags, unless the tags are already present, to binary data based on the given filter. func (d *DataClient) AddTagsToBinaryDataByFilter(ctx context.Context, tags []string, filter Filter) error { _, err := d.client.AddTagsToBinaryDataByFilter(ctx, &pb.AddTagsToBinaryDataByFilterRequest{Filter: FilterToProto(filter), Tags: tags}) if err != nil { @@ -627,6 +602,8 @@ func (d *DataClient) AddTagsToBinaryDataByFilter(ctx context.Context, tags []str } return nil } + +// RemoveTagsToBinaryDataByIDs removes string tags from binary data based on given IDs. func (d *DataClient) RemoveTagsFromBinaryDataByIDs(ctx context.Context, tags []string, binaryIds []BinaryID) (uint64, error) { resp, err := d.client.RemoveTagsFromBinaryDataByIDs(ctx, &pb.RemoveTagsFromBinaryDataByIDsRequest{BinaryIds: BinaryIdsToProto(binaryIds), Tags: tags}) if err != nil { @@ -634,26 +611,26 @@ func (d *DataClient) RemoveTagsFromBinaryDataByIDs(ctx context.Context, tags []s } return resp.DeletedCount, nil } + +// RemoveTagsToBinaryDataByFilter removes string tags from binary data based on the given filter. func (d *DataClient) RemoveTagsFromBinaryDataByFilter(ctx context.Context, tags []string, filter Filter) (uint64, error) { - // if filter == nil { - // filter = &pb.Filter{} - // } resp, err := d.client.RemoveTagsFromBinaryDataByFilter(ctx, &pb.RemoveTagsFromBinaryDataByFilterRequest{Filter: FilterToProto(filter), Tags: tags}) if err != nil { return 0, err } return resp.DeletedCount, nil } + +// TagsByFilter gets all unique tags from data based on given filter. func (d *DataClient) TagsByFilter(ctx context.Context, filter Filter) ([]string, error) { - // if filter == nil { - // filter = &pb.Filter{} - // } resp, err := d.client.TagsByFilter(ctx, &pb.TagsByFilterRequest{Filter: FilterToProto(filter)}) if err != nil { return nil, err } return resp.Tags, nil } + +// AddBoundingBoxToImageByID adds a bounding box to an image with the given ID. func (d *DataClient) AddBoundingBoxToImageByID( ctx context.Context, binaryId BinaryID, @@ -669,6 +646,8 @@ func (d *DataClient) AddBoundingBoxToImageByID( return resp.BboxId, nil } + +// RemoveBoundingBoxFromImageByID removes a bounding box from an image with the given ID. func (d *DataClient) RemoveBoundingBoxFromImageByID(ctx context.Context, bboxId string, binaryId BinaryID) error { _, err := d.client.RemoveBoundingBoxFromImageByID(ctx, &pb.RemoveBoundingBoxFromImageByIDRequest{BinaryId: BinaryIdToProto(binaryId), BboxId: bboxId}) if err != nil { @@ -676,6 +655,8 @@ func (d *DataClient) RemoveBoundingBoxFromImageByID(ctx context.Context, bboxId } return nil } + +// BoundingBoxLabelsByFilter gets all string labels for bounding boxes from data based on given filter. func (d *DataClient) BoundingBoxLabelsByFilter(ctx context.Context, filter Filter) ([]string, error) { resp, err := d.client.BoundingBoxLabelsByFilter(ctx, &pb.BoundingBoxLabelsByFilterRequest{Filter: FilterToProto(filter)}) if err != nil { @@ -684,24 +665,25 @@ func (d *DataClient) BoundingBoxLabelsByFilter(ctx context.Context, filter Filte return resp.Labels, nil } -// ***python and typescript did not implement this one!!! +// UpdateBoundingBox updates the bounding box associated with a given binary ID and bounding box ID. func (d *DataClient) UpdateBoundingBox(ctx context.Context, binaryId BinaryID, bboxId string, - label string, - xMinNormalized float64, - yMinNormalized float64, - xMaxNormalized float64, - yMaxNormalized float64) error { - - _, err := d.client.UpdateBoundingBox(ctx, &pb.UpdateBoundingBoxRequest{BinaryId: BinaryIdToProto(binaryId), BboxId: bboxId, Label: &label, XMinNormalized: &xMinNormalized, YMinNormalized: &yMinNormalized, XMaxNormalized: &xMaxNormalized, YMaxNormalized: &yMaxNormalized}) + label *string, // optional + xMinNormalized *float64, // optional + yMinNormalized *float64, // optional + xMaxNormalized *float64, // optional + yMaxNormalized *float64, // optional +) error { + _, err := d.client.UpdateBoundingBox(ctx, &pb.UpdateBoundingBoxRequest{BinaryId: BinaryIdToProto(binaryId), BboxId: bboxId, Label: label, XMinNormalized: xMinNormalized, YMinNormalized: yMinNormalized, XMaxNormalized: xMaxNormalized, YMaxNormalized: yMaxNormalized}) if err != nil { return err } return nil } -// do we want to return more than a hostname?? +// GetDatabaseConnection gets a connection to access a MongoDB Atlas Data Federation instance. It +// returns the hostname of the federated database. func (d *DataClient) GetDatabaseConnection(ctx context.Context, organizationId string) (string, error) { resp, err := d.client.GetDatabaseConnection(ctx, &pb.GetDatabaseConnectionRequest{OrganizationId: organizationId}) if err != nil { @@ -710,6 +692,8 @@ func (d *DataClient) GetDatabaseConnection(ctx context.Context, organizationId s return resp.Hostname, nil } +// ConfigureDatabaseUser configures a database user for the Viam organization's MongoDB Atlas Data +// Federation instance. It can also be used to reset the password of the existing database user. func (d *DataClient) ConfigureDatabaseUser(ctx context.Context, organizationId string, password string) error { _, err := d.client.ConfigureDatabaseUser(ctx, &pb.ConfigureDatabaseUserRequest{OrganizationId: organizationId, Password: password}) if err != nil { @@ -717,6 +701,8 @@ func (d *DataClient) ConfigureDatabaseUser(ctx context.Context, organizationId s } return nil } + +// AddBinaryDataToDatasetByIDs adds the binary data with the given binary IDs to the dataset. func (d *DataClient) AddBinaryDataToDatasetByIDs(ctx context.Context, binaryIds []BinaryID, datasetId string) error { _, err := d.client.AddBinaryDataToDatasetByIDs(ctx, &pb.AddBinaryDataToDatasetByIDsRequest{BinaryIds: BinaryIdsToProto(binaryIds), DatasetId: datasetId}) if err != nil { @@ -724,6 +710,8 @@ func (d *DataClient) AddBinaryDataToDatasetByIDs(ctx context.Context, binaryIds } return nil } + +// RemoveBinaryDataFromDatasetByIDs removes the binary data with the given binary IDs from the dataset. func (d *DataClient) RemoveBinaryDataFromDatasetByIDs(ctx context.Context, binaryIds []BinaryID, datasetId string) error { _, err := d.client.RemoveBinaryDataFromDatasetByIDs(ctx, &pb.RemoveBinaryDataFromDatasetByIDsRequest{BinaryIds: BinaryIdsToProto(binaryIds), DatasetId: datasetId}) if err != nil { @@ -731,47 +719,3 @@ func (d *DataClient) RemoveBinaryDataFromDatasetByIDs(ctx context.Context, binar } return nil } - -//NOTES: -// func convertBsonTypes(obj map[string]interface{}) map[string]interface{} { -// for key, value := range obj { -// switch v := value.(type) { -// case string: -// //no conversion needed -// return value -// case float64, int32, int64: -// //no conversion needed -// return v -// case bool: -// //no conversion needed -// return v -// case nil: -// // Null - Represent as nil in Go -// return nil -// case bson.A: // BSON array -// result := []interface{}{} -// for _, item := range v { -// result = append(result, convertBsonTypes(item)) -// } -// return result -// case bson.M: // BSON object (map) -// result := map[string]interface{}{} -// for key, val := range v { -// result[key] = convertBsonTypes(val) -// } -// return result -// case time.Time: -// // Datetime - Convert BSON datetime to Go time.Time -// return v -// default: -// // Return other types as-is -// return v - -// // obj[key] = convertBsonTypes(v) -// case map[string]interface{}: -// // recursively convert nested maps -// obj[key] = convertBsonTypes(v) -// } -// } -// return obj -// } From aa3aaf419e030bd82a425e2d3c5b288cbb5514d1 Mon Sep 17 00:00:00 2001 From: Julie Krasnick Date: Wed, 13 Nov 2024 14:14:27 -0500 Subject: [PATCH 18/25] cleaned up data_test --- app/data_test.go | 355 ++++++++++------------------------------------- 1 file changed, 77 insertions(+), 278 deletions(-) diff --git a/app/data_test.go b/app/data_test.go index 173fb17ab81..1190b7440b0 100644 --- a/app/data_test.go +++ b/app/data_test.go @@ -2,7 +2,6 @@ package app import ( "context" - "fmt" "testing" "time" @@ -15,7 +14,6 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" ) -// ccamel case in go!!!! const ( componentName = "component_name" componentType = "component_type" @@ -43,7 +41,7 @@ var ( locationIds = []string{locationId} orgIds = []string{organizationId} mimeTypes = []string{mimeType} - BboxLabels = []string{bboxLabel} + bboxLabels = []string{bboxLabel} methodParameters = map[string]string{} tags = []string{tag} startTime = time.Now().UTC().Round(time.Millisecond) @@ -51,58 +49,62 @@ var ( data = map[string]interface{}{ "key": "value", } + binaryId = BinaryID{ + FileId: "file1", + OrganizationId: organizationId, + LocationId: locationId, + } + binaryIds = []BinaryID{binaryId} binaryDataByte = []byte("BYTE") - // expectedRawData = []map[string]interface{}{ - // { - // "key1": startTime, - // "key2": "2", - // "key3": []int{1, 2, 3}, - // "key4": map[string]interface{}{ - // "key4sub1": endTime, - // }, - // }, - // } - mqlQuery = []map[string]interface{}{ - {"$match": map[string]interface{}{"organization_id": "e76d1b3b-0468-4efd-bb7f-fb1d2b352fcb"}}, - {"$limit": 1}, + sqlQuery = "SELECT * FROM readings WHERE organization_id='e76d1b3b-0468-4efd-bb7f-fb1d2b352fcb' LIMIT 1" + rawData = []map[string]any{ + { + "key1": startTime, + "key2": "2", + "key3": []any{1, 2, 3}, + "key4": map[string]any{ + "key4sub1": endTime, + }, + "key5": 4.05, + "key6": []any{true, false, true}, + "key7": []any{ + map[string]any{ + "nestedKey1": "simpleValue", + }, + map[string]any{ + "nestedKey2": startTime, + }, + }, + }, } - sqlQuery = "SELECT * FROM readings WHERE organization_id='e76d1b3b-0468-4efd-bb7f-fb1d2b352fcb' LIMIT 1" datasetIds = []string{datasetId} annotations = Annotations{ - bboxes: []BoundingBox{ + Bboxes: []BoundingBox{ { - id: "bbox1", - label: "label1", - xMinNormalized: 0.1, - yMinNormalized: 0.2, - xMaxNormalized: 0.8, - yMaxNormalized: 0.9, + Id: "bbox1", + Label: "label1", + XMinNormalized: 0.1, + YMinNormalized: 0.2, + XMaxNormalized: 0.8, + YMaxNormalized: 0.9, }, { - id: "bbox2", - label: "label2", - xMinNormalized: 0.15, - yMinNormalized: 0.25, - xMaxNormalized: 0.75, - yMaxNormalized: 0.85, + Id: "bbox2", + Label: "label2", + XMinNormalized: 0.15, + YMinNormalized: 0.25, + XMaxNormalized: 0.75, + YMaxNormalized: 0.85, }, }, } - binaryId = BinaryID{ - FileId: "file1", - OrganizationId: organizationId, - LocationId: locationId, - } - binaryIds = []BinaryID{binaryId} ) -// set up gRPC client?? -func createGrpclient() datapb.DataServiceClient { +func createGrpcClient() *inject.DataServiceClient { return &inject.DataServiceClient{} } - func TestDataClient(t *testing.T) { - grpcClient := &inject.DataServiceClient{} + grpcClient := createGrpcClient() client := DataClient{client: grpcClient} captureInterval := CaptureInterval{ @@ -110,7 +112,7 @@ func TestDataClient(t *testing.T) { End: time.Now(), } tagsFilter := TagsFilter{ - Type: 2, + Type: TagsFilterTypeUnspecified, Tags: []string{"tag1", "tag2"}, } @@ -127,7 +129,7 @@ func TestDataClient(t *testing.T) { MimeType: mimeTypes, Interval: captureInterval, TagsFilter: tagsFilter, //asterix or no?? - BboxLabels: BboxLabels, + BboxLabels: bboxLabels, DatasetId: datasetId, } tabularMetadata := CaptureMetadata{ @@ -156,10 +158,8 @@ func TestDataClient(t *testing.T) { DatasetIDs: datasetIds, } t.Run("TabularDataByFilter", func(t *testing.T) { - //flags countOnly := true includeInternalData := true - dataRequest := DataRequest{ Filter: filter, Limit: 5, @@ -169,7 +169,7 @@ func TestDataClient(t *testing.T) { expectedTabularData := TabularData{ Data: data, MetadataIndex: 0, - Metadata: tabularMetadata, //not sure if i need to pass this + Metadata: tabularMetadata, TimeRequested: startTime, TimeReceived: endTime, } @@ -187,7 +187,7 @@ func TestDataClient(t *testing.T) { grpcClient.TabularDataByFilterFunc = func(ctx context.Context, in *datapb.TabularDataByFilterRequest, opts ...grpc.CallOption) (*datapb.TabularDataByFilterResponse, error) { expectedDataReqProto, _ := DataRequestToProto(dataRequest) - test.That(t, in.DataRequest, test.ShouldResemble, expectedDataReqProto) //toPRoto on myDataReq (bc a request only ever gets converted to proto!!) + test.That(t, in.DataRequest, test.ShouldResemble, expectedDataReqProto) test.That(t, in.CountOnly, test.ShouldBeTrue) test.That(t, in.IncludeInternalData, test.ShouldBeTrue) return &datapb.TabularDataByFilterResponse{ @@ -195,11 +195,7 @@ func TestDataClient(t *testing.T) { Count: expectedCount, Last: expectedLast, Metadata: []*datapb.CaptureMetadata{CaptureMetadataToProto(tabularMetadata)}}, nil - //bc we only ever see tabularData in a response, we dont need toProto, we need fromProto!! - - //the only returns we care about are TabularData, int coint, and str last rewturned pg id } - respTabularData, respCount, respLast, _ := client.TabularDataByFilter(context.Background(), filter, expectedLimit, expectedLast, dataRequest.SortOrder, countOnly, includeInternalData) test.That(t, respTabularData[0], test.ShouldResemble, expectedTabularData) test.That(t, respCount, test.ShouldEqual, expectedCount) @@ -209,27 +205,8 @@ func TestDataClient(t *testing.T) { t.Run("TabularDataBySQL", func(t *testing.T) { expectedOrgId := organizationId expectedSqlQuery := sqlQuery + expectedRawData := rawData - expectedRawData := []map[string]any{ - { - "key1": startTime, - "key2": "2", - "key3": []any{1, 2, 3}, //slice of integers - "key4": map[string]any{ // map and rep of JSON object - "key4sub1": endTime, - }, - "key5": 4.05, - "key6": []any{true, false, true}, //array of bools - "key7": []any{ // slice of maps - map[string]any{ - "nestedKey1": "simpleValue", - }, - map[string]any{ - "nestedKey2": startTime, - }, - }, - }, - } //convert rawData to BSON var expectedRawDataPb [][]byte for _, byte := range expectedRawData { @@ -240,74 +217,28 @@ func TestDataClient(t *testing.T) { expectedRawDataPb = append(expectedRawDataPb, bsonByte) } grpcClient.TabularDataBySQLFunc = func(ctx context.Context, in *datapb.TabularDataBySQLRequest, opts ...grpc.CallOption) (*datapb.TabularDataBySQLResponse, error) { - test.That(t, in.OrganizationId, test.ShouldEqual, expectedOrgId) //to proto - test.That(t, in.SqlQuery, test.ShouldResemble, expectedSqlQuery) //to proto + test.That(t, in.OrganizationId, test.ShouldEqual, expectedOrgId) + test.That(t, in.SqlQuery, test.ShouldResemble, expectedSqlQuery) return &datapb.TabularDataBySQLResponse{ RawData: expectedRawDataPb, }, nil } response, _ := client.TabularDataBySQL(context.Background(), expectedOrgId, expectedSqlQuery) - // // fmt.Printf("expected type: %T, value: %+v\n", expected, expected) - // // fmt.Printf("actual type: %T, value: %+v\n", response, response) test.That(t, response, test.ShouldResemble, expectedRawData) - }) - //I want my response to look like expectedRawData --> good!! - //I want my input to TabularDataByMQLRespone to be what expectedRawData would look like in [][]byte form!!!!! - /* - - need to convert expectedRawData to BSON and then encode it as [][]byte to use it in your gRPC resp - - Convert expectedRawData to [][]byte --> expectedRawData to BSON. - Define TabularDataByMQLFunc ---> use the bson expectedRawData in TabularDataByMQLFunc - - // Serialize expectedRawData to BSON and convert to [][]byte - var expectedRawDataPb [][]byte - for _, doc := range expectedRawData { - bsonDoc, err := bson.Marshal(doc) - if err != nil { - t.Fatalf("Failed to marshal expectedRawData: %v", err) - } - expectedRawDataPb = append(expectedRawDataPb, bsonDoc) - } - - */ t.Run("TabularDataByMQL", func(t *testing.T) { expectedOrgId := organizationId - //create expected mqlBinary matchStage := bson.M{"$match": bson.M{"organization_id": "e76d1b3b-0468-4efd-bb7f-fb1d2b352fcb"}} limitStage := bson.M{"$limit": 1} + // convert to BSON byte arrays matchBytes, _ := bson.Marshal(matchStage) limitBytes, _ := bson.Marshal(limitStage) mqlbinary := [][]byte{matchBytes, limitBytes} expectedMqlBinary := mqlbinary + expectedRawData := rawData - // expectedMqlBinary := marshalToBSON(mqlQuery) //this is [][]byte type - //this is the format we want to get back... - - //add a bool - expectedRawData := []map[string]any{ - { - "key1": startTime, - "key2": "2", - "key3": []any{1, 2, 3}, //slice of integers - "key4": map[string]any{ // map and rep of JSON object - "key4sub1": endTime, - }, - "key5": 4.05, - "key6": []any{true, false, true}, //array of bools - "key7": []any{ // slice of maps - map[string]any{ - "nestedKey1": "simpleValue", - }, - map[string]any{ - "nestedKey2": startTime, - }, - }, - }, - } //convert rawData to BSON var expectedRawDataPb [][]byte for _, byte := range expectedRawData { @@ -317,34 +248,27 @@ func TestDataClient(t *testing.T) { } expectedRawDataPb = append(expectedRawDataPb, bsonByte) } - - fmt.Printf("this is expectedBinaryDataPb type: %T, value: %+v\n", expectedRawDataPb, expectedRawDataPb) - grpcClient.TabularDataByMQLFunc = func(ctx context.Context, in *datapb.TabularDataByMQLRequest, opts ...grpc.CallOption) (*datapb.TabularDataByMQLResponse, error) { - test.That(t, in.OrganizationId, test.ShouldEqual, expectedOrgId) //to proto - test.That(t, in.MqlBinary, test.ShouldResemble, expectedMqlBinary) //to proto //we also want this to just be [][]byte ...? + test.That(t, in.OrganizationId, test.ShouldEqual, expectedOrgId) + test.That(t, in.MqlBinary, test.ShouldResemble, expectedMqlBinary) return &datapb.TabularDataByMQLResponse{ - RawData: expectedRawDataPb, //we want this to be expectedRawData but in proto form (so [][]byte) + RawData: expectedRawDataPb, }, nil } response, _ := client.TabularDataByMQL(context.Background(), expectedOrgId, expectedMqlBinary) - // fmt.Printf("this is actual type: %T, value: %+v\n", response, response) - // fmt.Printf("this is expected type: %T, value: %+v\n", expectedRawData, expectedRawData) - test.That(t, response, test.ShouldResemble, expectedRawData) //we want this to be expectedRawData - + test.That(t, response, test.ShouldResemble, expectedRawData) }) + t.Run("BinaryDataByFilter", func(t *testing.T) { includeBinary := true countOnly := true includeInternalData := true - dataRequest := DataRequest{ Filter: filter, Limit: 5, Last: "last", SortOrder: Unspecified, } - expectedCount := uint64(5) expectedLast := "last" expectedBinaryData := BinaryData{ @@ -355,7 +279,7 @@ func TestDataClient(t *testing.T) { grpcClient.BinaryDataByFilterFunc = func(ctx context.Context, in *datapb.BinaryDataByFilterRequest, opts ...grpc.CallOption) (*datapb.BinaryDataByFilterResponse, error) { expectedDataReq, _ := DataRequestToProto(dataRequest) - test.That(t, in.DataRequest, test.ShouldResemble, expectedDataReq) //toPRoto on myDataReq (bc a request only ever gets converted to proto!!) + test.That(t, in.DataRequest, test.ShouldResemble, expectedDataReq) test.That(t, in.IncludeBinary, test.ShouldBeTrue) test.That(t, in.CountOnly, test.ShouldBeTrue) test.That(t, in.IncludeInternalData, test.ShouldBeTrue) @@ -364,14 +288,8 @@ func TestDataClient(t *testing.T) { Count: expectedCount, Last: expectedLast, }, nil - //bc we only ever see tabularData in a response, we dont need toProto, we need fromProto!! - //the only returns we care about are TabularData, int coint, and str last rewturned pg id } - - // fmt.Printf("this is includeBinary type: %T, value: %+v\n", includeBinary, includeBinary) - respBinaryData, respCount, respLast, _ := client.BinaryDataByFilter(context.Background(), filter, expectedCount, dataRequest.SortOrder, expectedLast, includeBinary, countOnly, includeInternalData) - // fmt.Printf("this is respBinaryData[0] type: %T, value: %+v\n", respBinaryData[0], respBinaryData[0]) test.That(t, respBinaryData[0], test.ShouldResemble, expectedBinaryData) test.That(t, respCount, test.ShouldEqual, expectedCount) test.That(t, respLast, test.ShouldEqual, expectedLast) @@ -385,7 +303,6 @@ func TestDataClient(t *testing.T) { grpcClient.BinaryDataByIDsFunc = func(ctx context.Context, in *datapb.BinaryDataByIDsRequest, opts ...grpc.CallOption) (*datapb.BinaryDataByIDsResponse, error) { test.That(t, in.IncludeBinary, test.ShouldBeTrue) test.That(t, in.BinaryIds, test.ShouldResemble, BinaryIdsToProto(binaryIds)) - //convert []binaryData to proto expectedBinaryDataList := []*datapb.BinaryData{BinaryDataToProto(expectedBinaryData)} return &datapb.BinaryDataByIDsResponse{Data: expectedBinaryDataList, Count: uint64(len(expectedBinaryDataList))}, nil @@ -393,9 +310,9 @@ func TestDataClient(t *testing.T) { respBinaryData, _ := client.BinaryDataByIDs(context.Background(), binaryIds) test.That(t, respBinaryData[0], test.ShouldResemble, expectedBinaryData) }) + t.Run("DeleteTabularData", func(t *testing.T) { deleteOlderThanDays := uint32(1) - expectedOrgId := organizationId expectedDeleteOlderThanDays := deleteOlderThanDays expectedDeletedCount := uint64(5) @@ -411,6 +328,7 @@ func TestDataClient(t *testing.T) { resp, _ := client.DeleteTabularData(context.Background(), organizationId, deleteOlderThanDays) test.That(t, resp, test.ShouldEqual, expectedDeletedCount) }) + t.Run("DeleteBinaryDataByFilter", func(t *testing.T) { expectedFilterPb := FilterToProto(filter) expectedDeletedCount := uint64(5) @@ -425,6 +343,7 @@ func TestDataClient(t *testing.T) { resp, _ := client.DeleteBinaryDataByFilter(context.Background(), filter) test.That(t, resp, test.ShouldEqual, expectedDeletedCount) }) + t.Run("DeleteBinaryDataByIDs", func(t *testing.T) { expectedDeletedCount := uint64(5) expectedBinaryIds := BinaryIdsToProto(binaryIds) @@ -437,6 +356,7 @@ func TestDataClient(t *testing.T) { resp, _ := client.DeleteBinaryDataByIDs(context.Background(), binaryIds) test.That(t, resp, test.ShouldEqual, expectedDeletedCount) }) + t.Run("AddTagsToBinaryDataByIDs", func(t *testing.T) { expectedTags := tags expectedBinaryIds := BinaryIdsToProto(binaryIds) @@ -448,6 +368,7 @@ func TestDataClient(t *testing.T) { client.AddTagsToBinaryDataByIDs(context.Background(), tags, binaryIds) }) + t.Run("AddTagsToBinaryDataByFilter", func(t *testing.T) { expectedTags := tags expectedFilterPb := FilterToProto(filter) @@ -458,6 +379,7 @@ func TestDataClient(t *testing.T) { } client.AddTagsToBinaryDataByFilter(context.Background(), tags, filter) }) + t.Run("RemoveTagsFromBinaryDataByIDs", func(t *testing.T) { expectedTags := tags expectedBinaryIds := BinaryIdsToProto(binaryIds) @@ -472,6 +394,7 @@ func TestDataClient(t *testing.T) { resp, _ := client.RemoveTagsFromBinaryDataByIDs(context.Background(), tags, binaryIds) test.That(t, resp, test.ShouldEqual, expectedDeletedCount) }) + t.Run("RemoveTagsFromBinaryDataByFilter", func(t *testing.T) { expectedTags := tags expectedFilterPb := FilterToProto(filter) @@ -487,6 +410,7 @@ func TestDataClient(t *testing.T) { resp, _ := client.RemoveTagsFromBinaryDataByFilter(context.Background(), tags, filter) test.That(t, resp, test.ShouldEqual, expectedDeletedCount) }) + t.Run("TagsByFilter", func(t *testing.T) { expectedTags := tags expectedFilterPb := FilterToProto(filter) @@ -500,14 +424,15 @@ func TestDataClient(t *testing.T) { resp, _ := client.TagsByFilter(context.Background(), filter) test.That(t, resp, test.ShouldResemble, expectedTags) }) + t.Run("AddBoundingBoxToImageByID", func(t *testing.T) { expectedBinaryIdPb := BinaryIdToProto(binaryId) expectedLabel := bboxLabel - expectedXMin := annotations.bboxes[0].xMinNormalized - expectedYMin := annotations.bboxes[0].yMinNormalized - expectedXMax := annotations.bboxes[0].xMaxNormalized - expectedYMax := annotations.bboxes[0].yMaxNormalized - expectedBBoxId := annotations.bboxes[0].id + expectedXMin := annotations.Bboxes[0].XMinNormalized + expectedYMin := annotations.Bboxes[0].YMinNormalized + expectedXMax := annotations.Bboxes[0].XMaxNormalized + expectedYMax := annotations.Bboxes[0].YMaxNormalized + expectedBBoxId := annotations.Bboxes[0].Id grpcClient.AddBoundingBoxToImageByIDFunc = func(ctx context.Context, in *datapb.AddBoundingBoxToImageByIDRequest, opts ...grpc.CallOption) (*datapb.AddBoundingBoxToImageByIDResponse, error) { test.That(t, in.BinaryId, test.ShouldResemble, expectedBinaryIdPb) @@ -527,7 +452,7 @@ func TestDataClient(t *testing.T) { t.Run("RemoveBoundingBoxFromImageByID", func(t *testing.T) { expectedBinaryIdPb := BinaryIdToProto(binaryId) - expectedBBoxId := annotations.bboxes[0].id + expectedBBoxId := annotations.Bboxes[0].Id grpcClient.RemoveBoundingBoxFromImageByIDFunc = func(ctx context.Context, in *datapb.RemoveBoundingBoxFromImageByIDRequest, opts ...grpc.CallOption) (*datapb.RemoveBoundingBoxFromImageByIDResponse, error) { test.That(t, in.BinaryId, test.ShouldResemble, expectedBinaryIdPb) @@ -538,14 +463,13 @@ func TestDataClient(t *testing.T) { client.RemoveBoundingBoxFromImageByID(context.Background(), expectedBBoxId, binaryId) }) + t.Run("BoundingBoxLabelsByFilter", func(t *testing.T) { expectedFilterPb := FilterToProto(filter) - expectedBBoxLabels := []string{ - annotations.bboxes[0].label, - annotations.bboxes[1].label, + annotations.Bboxes[0].Label, + annotations.Bboxes[1].Label, } - //we probs don't need to do this work below...but i like the explicity of it annotationsPb := AnnotationsToProto(annotations) expectedBBoxLabelsPb := []string{ annotationsPb.Bboxes[0].Label, @@ -562,7 +486,7 @@ func TestDataClient(t *testing.T) { test.That(t, resp, test.ShouldResemble, expectedBBoxLabels) }) t.Run("UpdateBoundingBox", func(t *testing.T) { - bBoxId := annotations.bboxes[0].id + bBoxId := annotations.Bboxes[0].Id expectedBinaryIdPb := BinaryIdToProto(binaryId) annotationsPb := AnnotationsToProto(annotations) @@ -581,11 +505,9 @@ func TestDataClient(t *testing.T) { test.That(t, *in.YMinNormalized, test.ShouldEqual, expectedYMin) test.That(t, *in.XMaxNormalized, test.ShouldEqual, expectedXMax) test.That(t, *in.YMaxNormalized, test.ShouldEqual, expectedYMax) - return &datapb.UpdateBoundingBoxResponse{}, nil } - client.UpdateBoundingBox(context.Background(), binaryId, bBoxId, expectedLabel, expectedXMin, expectedYMin, expectedXMax, expectedYMax) - + client.UpdateBoundingBox(context.Background(), binaryId, bBoxId, &expectedLabel, &expectedXMin, &expectedYMin, &expectedXMax, &expectedYMax) }) t.Run("GetDatabaseConnection", func(t *testing.T) { @@ -604,7 +526,6 @@ func TestDataClient(t *testing.T) { } resp, _ := client.GetDatabaseConnection(context.Background(), organizationId) test.That(t, resp, test.ShouldResemble, expectedHostName) - }) t.Run("ConfigureDatabaseUser", func(t *testing.T) { @@ -629,7 +550,6 @@ func TestDataClient(t *testing.T) { return &datapb.AddBinaryDataToDatasetByIDsResponse{}, nil } client.AddBinaryDataToDatasetByIDs(context.Background(), binaryIds, datasetId) - }) t.Run("RemoveBinaryDataFromDatasetByIDs", func(t *testing.T) { @@ -639,129 +559,8 @@ func TestDataClient(t *testing.T) { grpcClient.RemoveBinaryDataFromDatasetByIDsFunc = func(ctx context.Context, in *datapb.RemoveBinaryDataFromDatasetByIDsRequest, opts ...grpc.CallOption) (*datapb.RemoveBinaryDataFromDatasetByIDsResponse, error) { test.That(t, in.BinaryIds, test.ShouldResemble, expectedBinaryIds) test.That(t, in.DatasetId, test.ShouldResemble, expectedDataSetId) - return &datapb.RemoveBinaryDataFromDatasetByIDsResponse{}, nil } client.RemoveBinaryDataFromDatasetByIDs(context.Background(), binaryIds, datasetId) - }) - } - -// func TestTabularDataByFilter(t *testing.T) { -// // Set up test logger -// logger := logging.NewTestLogger(t) -// // need this listener probs -// listener, err := net.Listen("tcp", "localhost:0") -// test.That(t, err, test.ShouldBeNil) -// client, conn, err := createGrpclient(t, logger, listener) // create client conn w helper -// test.That(t, err, test.ShouldBeNil) - -// dataRequest := &datapb.DataRequest{} //this doesn't seem right??? - -// //grpc requeset?? -// req := &datapb.TabularDataByFilterRequest{ -// DataRequest: dataRequest, //this doesn't seem right??? -// CountOnly: true, -// IncludeInternalData: true, -// } - -// // call the real method -// resp, err := client.TabularDataByFilter(context.Background(), req) -// test.That(t, err, test.ShouldBeNil) -// // check that the parameters being passed match the expected data?? -// test.That(t, req.DataRequest, test.ShouldResemble, dataRequest) - -// // check that the response matches expected data -// expectedData := &datapb.TabularDataByFilterResponse{ -// Data: resp.Data, //idk if it makes sense to be passing the resp.Data?? -// RawData: resp.RawData, //bc what are we actually testing? -// } -// // test.That(t, expectedData, test.ShouldResembleProto, &datapb.TabularDataByMQLResponse{}) -// test.That(t, expectedData, test.ShouldResemble, resp) -// // Close the connection -// require.NoError(t, conn.Close()) - -// } - -// func TestTabularDataBySQL(t *testing.T) {} -// func TestTabularDataByMQL(t *testing.T) { -// client := createGrpclient() - -// return -// // Set up test logger -// logger := logging.NewTestLogger(t) -// // need this listener probs -// listener, err := net.Listen("tcp", "localhost:0") -// test.That(t, err, test.ShouldBeNil) -// client, conn, err := createGrpclient(t, logger, listener) // create client conn w helper -// test.That(t, err, test.ShouldBeNil) - -// //call helper -// mqlbinary := createMQLBSON() -// // orgId := "e76d1b3b-0468-4efd-bb7f-fb1d2b352fcb" -// // make the actual call to the grpc dataserive function?? -// req := &datapb.TabularDataByMQLRequest{ -// OrganizationId: orgId, -// MqlBinary: mqlbinary, -// } -// // call the real method -// resp, err := client.TabularDataByMQL(context.Background(), req) -// test.That(t, err, test.ShouldBeNil) -// // check that the parameters being passed match the expected data?? -// test.That(t, req.OrganizationId, test.ShouldResemble, orgId) - -// // check that the response matches expected data -// expectedData := &datapb.TabularDataByMQLResponse{ -// Data: resp.Data, //idk if it makes sense to be passing the resp.Data?? -// RawData: resp.RawData, //bc what are we actually testing? -// } -// // test.That(t, expectedData, test.ShouldResembleProto, &datapb.TabularDataByMQLResponse{}) -// test.That(t, expectedData, test.ShouldResemble, resp) -// // Close the connection -// require.NoError(t, conn.Close()) -// } - -// func TestBinaryDataByFilter(t *testing.T) {} -// func TestBinaryDataByIDs(t *testing.T) {} -// func TestDeleteTabularData(t *testing.T) {} -// func TestDeleteBinaryDataByFilter(t *testing.T) {} -// func TestDeleteBinaryDataByIDs(t *testing.T) {} -// func TestAddTagsToBinaryDataByIDs(t *testing.T) {} -// func TestAddTagsToBinaryDataByFilter(t *testing.T) {} -// func TestRemoveTagsFromBinaryDataByIDs(t *testing.T) {} -// func TestRemoveTagsFromBinaryDataByFilter(t *testing.T) {} -// func TestTagsByFilter(t *testing.T) {} - -//notes: -// Set up gRPC server (should it be grpc.NewServer()??) -// logger := logging.NewTestLogger(t) -// listener, err := net.Listen("tcp", "localhost:0") -// test.That(t, err, test.ShouldBeNil) -// rpcServer, err := rpc.NewServer(logger, rpc.WithUnauthenticated()) -// test.That(t, err, test.ShouldBeNil) - -//****need to somehow register the DataService server here?? -// datapb.RegisterDataServiceHandlerFromEndpoint(context.Background(),runtime.NewServeMux(),) -// datapb.RegisterDataServiceServer(rpcServer,) -// datapb.RegisterDataServiceServer(rpcServer, &datapb.UnimplementedDataServiceServer{}) -// data = datapb.DataServiceServer{} - -// Start serving requests?? -// go rpcServer.Serve(listener) -// defer rpcServer.Stop() - -// Make client connection -// conn, err := viamgrpc.Dial(context.Background(), listener.Addr().String(), logger) -// test.That(t, err, test.ShouldBeNil) -// client := datapb.NewDataServiceClient(conn) - -//notes on the param chekcing stuff -// var request *datapb.TabularDataByMQLRequest - -// grpcClient.TabularDataByMQLFunc = func(ctx context.Context, in *datapb.TabularDataByMQLRequest, opts ...grpc.CallOption) (*datapb.TabularDataByMQLResponse, error) { -// request = in -// return &datapb.TabularDataByMQLResponse{RawData: mql_binary}, nil //MQlResponse is created with BSON data -// } - -// assert request.org From 53dc8e4cab5f51e8c65df7753cd4fc4ae32cda18 Mon Sep 17 00:00:00 2001 From: Julie Krasnick Date: Wed, 13 Nov 2024 14:37:12 -0500 Subject: [PATCH 19/25] Clean up comments --- app/data_client.go | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/app/data_client.go b/app/data_client.go index c6e701b1eca..44c85c4d599 100644 --- a/app/data_client.go +++ b/app/data_client.go @@ -1,5 +1,3 @@ -//go:build !no_cgo - package app import ( @@ -18,17 +16,9 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" ) -//protobuf to type or type to protobuf (poseinframe to proto or proto to pose in frame) -//define the structs publically and a private function that does the conversion -//come back to "dest" path later to see if we wanna write to a file or not - -// viamClient.dataClient. - -// i want to wrap NewDataServiceClient define a new dataclient struct and call the wrappers of the functions -// // i want the user to call my dataClient struct w my wrappers and not the proto functions - type DataClient struct { client pb.DataServiceClient + logger logging.Logger } // Order specifies the order in which data is returned. @@ -146,15 +136,16 @@ type Annotations struct { Bboxes []BoundingBox } -// NewDataClient constructs a new DataClient from connection passed in. +// NewDataClient constructs a new DataClient using the connection passed in by the viamClient and the provided logger. func NewDataClient( ctx context.Context, - channel rpc.ClientConn, //this should just take a channel that the viamClient passes in + channel rpc.ClientConn, logger logging.Logger, ) (*DataClient, error) { d := pb.NewDataServiceClient(channel) return &DataClient{ client: d, + logger: logger, }, nil } @@ -173,7 +164,6 @@ func AnnotationsFromProto(proto *pb.Annotations) Annotations { if proto == nil { return Annotations{} } - // Convert each BoundingBox from proto to native type bboxes := make([]BoundingBox, len(proto.Bboxes)) for i, bboxProto := range proto.Bboxes { bboxes[i] = BoundingBoxFromProto(bboxProto) @@ -397,9 +387,8 @@ func OrderToProto(sortOrder Order) pb.Order { } } -// convertBsonToNative recursively converts BSON types (e.g., DateTime, arrays, maps) -// into their native Go equivalents. This ensures all BSON data types are converted -// to the appropriate Go types like time.Time, slices, and maps. +// convertBsonToNative recursively converts BSON datetime objects to Go DateTime and BSON arrays to slices of interface{}. +// For slices and maps of specific types, the best we can do is use interface{} as the container type. func convertBsonToNative(data any) any { switch v := data.(type) { case primitive.DateTime: From 4a623f10599b2997e223024de96143868909c2af Mon Sep 17 00:00:00 2001 From: Julie Krasnick Date: Wed, 13 Nov 2024 16:23:52 -0500 Subject: [PATCH 20/25] lint fixes --- app/data_client.go | 392 ++++++++++++++---------- app/data_test.go | 316 +++++++++++-------- go.sum | 22 +- testutils/inject/data_service_client.go | 168 +++++++--- 4 files changed, 553 insertions(+), 345 deletions(-) diff --git a/app/data_client.go b/app/data_client.go index 44c85c4d599..e8f46bb0906 100644 --- a/app/data_client.go +++ b/app/data_client.go @@ -1,4 +1,5 @@ -package app +// Package data contains a gRPC based data client. +package data import ( "context" @@ -7,16 +8,17 @@ import ( "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" pb "go.viam.com/api/app/data/v1" - "go.viam.com/rdk/logging" - "go.viam.com/rdk/protoutils" - utils "go.viam.com/utils/protoutils" "go.viam.com/utils/rpc" "google.golang.org/protobuf/types/known/anypb" "google.golang.org/protobuf/types/known/structpb" "google.golang.org/protobuf/types/known/timestamppb" + + "go.viam.com/rdk/logging" + "go.viam.com/rdk/protoutils" ) -type DataClient struct { +// Client implements the DataServiceClient interface. +type Client struct { client pb.DataServiceClient logger logging.Logger } @@ -24,6 +26,7 @@ type DataClient struct { // Order specifies the order in which data is returned. type Order int32 +// Order constants define the possible ordering options. const ( Unspecified Order = 0 Descending Order = 1 @@ -45,21 +48,22 @@ type Filter struct { ComponentType string Method string RobotName string - RobotId string + RobotID string PartName string - PartId string - LocationIds []string - OrganizationIds []string + PartID string + LocationIDs []string + OrganizationIDs []string MimeType []string Interval CaptureInterval TagsFilter TagsFilter BboxLabels []string - DatasetId string + DatasetID string } -// TagsFilter defines the type of filtering and, if applicable, over which tags to perform a logical OR. +// TagsFilterType specifies how data can be filtered based on tags. type TagsFilterType int32 +// TagsFilterType constants define the ways data can be filtered based on tag matching criteria. const ( TagsFilterTypeUnspecified TagsFilterType = 0 TagsFilterTypeMatchByOr TagsFilterType = 1 @@ -67,6 +71,7 @@ const ( TagsFilterTypeUntagged TagsFilterType = 3 ) +// TagsFilter defines the type of filtering and, if applicable, over which tags to perform a logical OR. type TagsFilter struct { Type TagsFilterType Tags []string @@ -74,12 +79,12 @@ type TagsFilter struct { // CaptureMetadata contains information on the settings used for the data capture. type CaptureMetadata struct { - OrganizationId string - LocationId string + OrganizationID string + LocationID string RobotName string - RobotId string + RobotID string PartName string - PartId string + PartID string ComponentType string ComponentName string MethodName string @@ -93,6 +98,8 @@ type CaptureInterval struct { Start time.Time End time.Time } + +// TabularData contains data and metadata associated with tabular data. type TabularData struct { Data map[string]interface{} MetadataIndex uint32 @@ -101,11 +108,13 @@ type TabularData struct { TimeReceived time.Time } +// BinaryData contains data and metadata associated with binary data. type BinaryData struct { Binary []byte Metadata BinaryMetadata } +// BinaryMetadata is the metadata associated with binary data. type BinaryMetadata struct { ID string CaptureMetadata CaptureMetadata @@ -118,13 +127,17 @@ type BinaryMetadata struct { DatasetIDs []string } +// BinaryID is the unique identifier for a file that one can request to be retrieved or modified. type BinaryID struct { - FileId string - OrganizationId string - LocationId string + FileID string + OrganizationID string + LocationID string } + +// BoundingBox represents a labeled bounding box on an image. +// x and y values are normalized ratios between 0 and 1. type BoundingBox struct { - Id string + ID string Label string XMinNormalized float64 YMinNormalized float64 @@ -132,6 +145,7 @@ type BoundingBox struct { YMaxNormalized float64 } +// Annotations are data annotations used for machine learning. type Annotations struct { Bboxes []BoundingBox } @@ -141,17 +155,17 @@ func NewDataClient( ctx context.Context, channel rpc.ClientConn, logger logging.Logger, -) (*DataClient, error) { +) (*Client, error) { d := pb.NewDataServiceClient(channel) - return &DataClient{ + return &Client{ client: d, logger: logger, }, nil } -func BoundingBoxFromProto(proto *pb.BoundingBox) BoundingBox { +func boundingBoxFromProto(proto *pb.BoundingBox) BoundingBox { return BoundingBox{ - Id: proto.Id, + ID: proto.Id, Label: proto.Label, XMinNormalized: proto.XMinNormalized, YMinNormalized: proto.YMinNormalized, @@ -160,24 +174,24 @@ func BoundingBoxFromProto(proto *pb.BoundingBox) BoundingBox { } } -func AnnotationsFromProto(proto *pb.Annotations) Annotations { +func annotationsFromProto(proto *pb.Annotations) Annotations { if proto == nil { return Annotations{} } bboxes := make([]BoundingBox, len(proto.Bboxes)) for i, bboxProto := range proto.Bboxes { - bboxes[i] = BoundingBoxFromProto(bboxProto) + bboxes[i] = boundingBoxFromProto(bboxProto) } return Annotations{ Bboxes: bboxes, } } -func AnnotationsToProto(annotations Annotations) *pb.Annotations { +func annotationsToProto(annotations Annotations) *pb.Annotations { var protoBboxes []*pb.BoundingBox for _, bbox := range annotations.Bboxes { protoBboxes = append(protoBboxes, &pb.BoundingBox{ - Id: bbox.Id, + Id: bbox.ID, Label: bbox.Label, XMinNormalized: bbox.XMinNormalized, YMinNormalized: bbox.YMinNormalized, @@ -202,17 +216,17 @@ func methodParamsFromProto(proto map[string]*anypb.Any) map[string]string { return methodParameters } -func CaptureMetadataFromProto(proto *pb.CaptureMetadata) CaptureMetadata { +func captureMetadataFromProto(proto *pb.CaptureMetadata) CaptureMetadata { if proto == nil { return CaptureMetadata{} } return CaptureMetadata{ - OrganizationId: proto.OrganizationId, - LocationId: proto.LocationId, + OrganizationID: proto.OrganizationId, + LocationID: proto.LocationId, RobotName: proto.RobotName, - RobotId: proto.RobotId, + RobotID: proto.RobotId, PartName: proto.PartName, - PartId: proto.PartId, + PartID: proto.PartId, ComponentType: proto.ComponentType, ComponentName: proto.ComponentName, MethodName: proto.MethodName, @@ -222,15 +236,18 @@ func CaptureMetadataFromProto(proto *pb.CaptureMetadata) CaptureMetadata { } } -func CaptureMetadataToProto(metadata CaptureMetadata) *pb.CaptureMetadata { - methodParms, _ := protoutils.ConvertStringMapToAnyPBMap(metadata.MethodParameters) +func captureMetadataToProto(metadata CaptureMetadata) *pb.CaptureMetadata { + methodParms, err := protoutils.ConvertStringMapToAnyPBMap(metadata.MethodParameters) + if err != nil { + return nil + } return &pb.CaptureMetadata{ - OrganizationId: metadata.OrganizationId, - LocationId: metadata.LocationId, + OrganizationId: metadata.OrganizationID, + LocationId: metadata.LocationID, RobotName: metadata.RobotName, - RobotId: metadata.RobotId, + RobotId: metadata.RobotID, PartName: metadata.PartName, - PartId: metadata.PartId, + PartId: metadata.PartID, ComponentType: metadata.ComponentType, ComponentName: metadata.ComponentName, MethodName: metadata.MethodName, @@ -238,150 +255,126 @@ func CaptureMetadataToProto(metadata CaptureMetadata) *pb.CaptureMetadata { Tags: metadata.Tags, MimeType: metadata.MimeType, } - } -func BinaryDataFromProto(proto *pb.BinaryData) BinaryData { +func binaryDataFromProto(proto *pb.BinaryData) BinaryData { return BinaryData{ Binary: proto.Binary, - Metadata: BinaryMetadataFromProto(proto.Metadata), + Metadata: binaryMetadataFromProto(proto.Metadata), } } -func BinaryDataToProto(binaryData BinaryData) *pb.BinaryData { +func binaryDataToProto(binaryData BinaryData) *pb.BinaryData { return &pb.BinaryData{ Binary: binaryData.Binary, - Metadata: BinaryMetadataToProto(binaryData.Metadata), + Metadata: binaryMetadataToProto(binaryData.Metadata), } } -func BinaryMetadataFromProto(proto *pb.BinaryMetadata) BinaryMetadata { +func binaryMetadataFromProto(proto *pb.BinaryMetadata) BinaryMetadata { return BinaryMetadata{ ID: proto.Id, - CaptureMetadata: CaptureMetadataFromProto(proto.CaptureMetadata), + CaptureMetadata: captureMetadataFromProto(proto.CaptureMetadata), TimeRequested: proto.TimeRequested.AsTime(), TimeReceived: proto.TimeReceived.AsTime(), FileName: proto.FileName, FileExt: proto.FileExt, URI: proto.Uri, - Annotations: AnnotationsFromProto(proto.Annotations), + Annotations: annotationsFromProto(proto.Annotations), DatasetIDs: proto.DatasetIds, } } -func BinaryMetadataToProto(binaryMetadata BinaryMetadata) *pb.BinaryMetadata { +func binaryMetadataToProto(binaryMetadata BinaryMetadata) *pb.BinaryMetadata { return &pb.BinaryMetadata{ Id: binaryMetadata.ID, - CaptureMetadata: CaptureMetadataToProto(binaryMetadata.CaptureMetadata), + CaptureMetadata: captureMetadataToProto(binaryMetadata.CaptureMetadata), TimeRequested: timestamppb.New(binaryMetadata.TimeRequested), TimeReceived: timestamppb.New(binaryMetadata.TimeReceived), FileName: binaryMetadata.FileName, FileExt: binaryMetadata.FileExt, Uri: binaryMetadata.URI, - Annotations: AnnotationsToProto(binaryMetadata.Annotations), + Annotations: annotationsToProto(binaryMetadata.Annotations), DatasetIds: binaryMetadata.DatasetIDs, } } -func TabularDataFromProto(proto *pb.TabularData, metadata *pb.CaptureMetadata) TabularData { +func tabularDataFromProto(proto *pb.TabularData, metadata *pb.CaptureMetadata) TabularData { return TabularData{ Data: proto.Data.AsMap(), MetadataIndex: proto.MetadataIndex, - Metadata: CaptureMetadataFromProto(metadata), + Metadata: captureMetadataFromProto(metadata), TimeRequested: proto.TimeRequested.AsTime(), TimeReceived: proto.TimeReceived.AsTime(), } } -func TabularDataToProto(tabularData TabularData) *pb.TabularData { - structData, err := utils.StructToStructPb(tabularData.Data) - if err != nil { - return nil - } - return &pb.TabularData{ - Data: structData, - MetadataIndex: tabularData.MetadataIndex, - TimeRequested: timestamppb.New(tabularData.TimeRequested), - TimeReceived: timestamppb.New(tabularData.TimeReceived), - } -} - -func TabularDataToProtoList(tabularDatas []TabularData) []*pb.TabularData { - var protoList []*pb.TabularData - for _, tabularData := range tabularDatas { - protoData := TabularDataToProto(tabularData) - if protoData != nil { - protoList = append(protoList, protoData) - } - } - return protoList -} - -func DataRequestToProto(dataRequest DataRequest) (*pb.DataRequest, error) { +func dataRequestToProto(dataRequest DataRequest) *pb.DataRequest { return &pb.DataRequest{ - Filter: FilterToProto(dataRequest.Filter), + Filter: filterToProto(dataRequest.Filter), Limit: dataRequest.Limit, Last: dataRequest.Last, - SortOrder: OrderToProto(dataRequest.SortOrder), - }, nil - + SortOrder: orderToProto(dataRequest.SortOrder), + } } -func BinaryIdToProto(binaryId BinaryID) *pb.BinaryID { +func binaryIDToProto(binaryID BinaryID) *pb.BinaryID { return &pb.BinaryID{ - FileId: binaryId.FileId, - OrganizationId: binaryId.OrganizationId, - LocationId: binaryId.LocationId, + FileId: binaryID.FileID, + OrganizationId: binaryID.OrganizationID, + LocationId: binaryID.LocationID, } } -func BinaryIdsToProto(binaryIds []BinaryID) []*pb.BinaryID { - var protoBinaryIds []*pb.BinaryID - for _, binaryId := range binaryIds { - protoBinaryIds = append(protoBinaryIds, BinaryIdToProto(binaryId)) +func binaryIDsToProto(binaryIDs []BinaryID) []*pb.BinaryID { + var protoBinaryIDs []*pb.BinaryID + for _, binaryID := range binaryIDs { + protoBinaryIDs = append(protoBinaryIDs, binaryIDToProto(binaryID)) } - return protoBinaryIds + return protoBinaryIDs } -func FilterToProto(filter Filter) *pb.Filter { +func filterToProto(filter Filter) *pb.Filter { return &pb.Filter{ ComponentName: filter.ComponentName, ComponentType: filter.ComponentType, Method: filter.Method, RobotName: filter.RobotName, - RobotId: filter.RobotId, + RobotId: filter.RobotID, PartName: filter.PartName, - PartId: filter.PartId, - LocationIds: filter.LocationIds, - OrganizationIds: filter.OrganizationIds, + PartId: filter.PartID, + LocationIds: filter.LocationIDs, + OrganizationIds: filter.OrganizationIDs, MimeType: filter.MimeType, - Interval: CaptureIntervalToProto(filter.Interval), - TagsFilter: TagsFilterToProto(filter.TagsFilter), + Interval: captureIntervalToProto(filter.Interval), + TagsFilter: tagsFilterToProto(filter.TagsFilter), BboxLabels: filter.BboxLabels, - DatasetId: filter.DatasetId, + DatasetId: filter.DatasetID, } } -func CaptureIntervalToProto(interval CaptureInterval) *pb.CaptureInterval { +func captureIntervalToProto(interval CaptureInterval) *pb.CaptureInterval { return &pb.CaptureInterval{ Start: timestamppb.New(interval.Start), End: timestamppb.New(interval.End), } } -func TagsFilterToProto(tagsFilter TagsFilter) *pb.TagsFilter { +func tagsFilterToProto(tagsFilter TagsFilter) *pb.TagsFilter { return &pb.TagsFilter{ Type: pb.TagsFilterType(tagsFilter.Type), Tags: tagsFilter.Tags, } } -func OrderToProto(sortOrder Order) pb.Order { +func orderToProto(sortOrder Order) pb.Order { switch sortOrder { case Ascending: return pb.Order_ORDER_ASCENDING case Descending: return pb.Order_ORDER_DESCENDING + case Unspecified: + return pb.Order_ORDER_UNSPECIFIED default: return pb.Order_ORDER_UNSPECIFIED } @@ -437,20 +430,22 @@ func BsonToGo(rawData [][]byte) ([]map[string]any, error) { } // TabularDataByFilter queries tabular data and metadata based on given filters. -func (d *DataClient) TabularDataByFilter( +func (d *Client) TabularDataByFilter( ctx context.Context, filter Filter, limit uint64, last string, sortOrder Order, countOnly bool, - includeInternalData bool) ([]TabularData, uint64, string, error) { + includeInternalData bool, +) ([]TabularData, uint64, string, error) { resp, err := d.client.TabularDataByFilter(ctx, &pb.TabularDataByFilterRequest{ DataRequest: &pb.DataRequest{ - Filter: FilterToProto(filter), + Filter: filterToProto(filter), Limit: limit, Last: last, - SortOrder: OrderToProto(sortOrder)}, + SortOrder: orderToProto(sortOrder), + }, CountOnly: countOnly, IncludeInternalData: includeInternalData, }) @@ -464,35 +459,47 @@ func (d *DataClient) TabularDataByFilter( metadata = &pb.CaptureMetadata{} } else { metadata = resp.Metadata[data.MetadataIndex] - } - dataArray = append(dataArray, TabularDataFromProto(data, metadata)) + dataArray = append(dataArray, tabularDataFromProto(data, metadata)) } return dataArray, resp.Count, resp.Last, nil } // TabularDataBySQL queries tabular data with a SQL query. -func (d *DataClient) TabularDataBySQL(ctx context.Context, organizationId string, sqlQuery string) ([]map[string]interface{}, error) { - resp, err := d.client.TabularDataBySQL(ctx, &pb.TabularDataBySQLRequest{OrganizationId: organizationId, SqlQuery: sqlQuery}) +func (d *Client) TabularDataBySQL(ctx context.Context, organizationID, sqlQuery string) ([]map[string]interface{}, error) { + resp, err := d.client.TabularDataBySQL(ctx, &pb.TabularDataBySQLRequest{ + OrganizationId: organizationID, + SqlQuery: sqlQuery, + }) + if err != nil { + return nil, err + } + dataObjects, err := BsonToGo(resp.RawData) if err != nil { return nil, err } - dataObjects, nil := BsonToGo(resp.RawData) return dataObjects, nil } // TabularDataByMQL queries tabular data with an MQL (MongoDB Query Language) query. -func (d *DataClient) TabularDataByMQL(ctx context.Context, organizationId string, mqlbinary [][]byte) ([]map[string]interface{}, error) { - resp, err := d.client.TabularDataByMQL(ctx, &pb.TabularDataByMQLRequest{OrganizationId: organizationId, MqlBinary: mqlbinary}) +func (d *Client) TabularDataByMQL(ctx context.Context, organizationID string, mqlbinary [][]byte) ([]map[string]interface{}, error) { + resp, err := d.client.TabularDataByMQL(ctx, &pb.TabularDataByMQLRequest{ + OrganizationId: organizationID, + MqlBinary: mqlbinary, + }) + if err != nil { + return nil, err + } + + result, err := BsonToGo(resp.RawData) if err != nil { return nil, err } - result, nil := BsonToGo(resp.RawData) return result, nil } // BinaryDataByFilter queries binary data and metadata based on given filters. -func (d *DataClient) BinaryDataByFilter( +func (d *Client) BinaryDataByFilter( ctx context.Context, filter Filter, limit uint64, @@ -500,13 +507,14 @@ func (d *DataClient) BinaryDataByFilter( last string, includeBinary bool, countOnly bool, - includeInternalData bool) ([]BinaryData, uint64, string, error) { + includeInternalData bool, +) ([]BinaryData, uint64, string, error) { resp, err := d.client.BinaryDataByFilter(ctx, &pb.BinaryDataByFilterRequest{ DataRequest: &pb.DataRequest{ - Filter: FilterToProto(filter), + Filter: filterToProto(filter), Limit: limit, Last: last, - SortOrder: OrderToProto(sortOrder), + SortOrder: orderToProto(sortOrder), }, IncludeBinary: includeBinary, CountOnly: countOnly, @@ -517,32 +525,31 @@ func (d *DataClient) BinaryDataByFilter( } data := make([]BinaryData, len(resp.Data)) for i, protoData := range resp.Data { - data[i] = BinaryDataFromProto(protoData) + data[i] = binaryDataFromProto(protoData) } return data, resp.Count, resp.Last, nil - } // BinaryDataByIDs queries binary data and metadata based on given IDs. -func (d *DataClient) BinaryDataByIDs(ctx context.Context, binaryIds []BinaryID) ([]BinaryData, error) { +func (d *Client) BinaryDataByIDs(ctx context.Context, binaryIDs []BinaryID) ([]BinaryData, error) { resp, err := d.client.BinaryDataByIDs(ctx, &pb.BinaryDataByIDsRequest{ IncludeBinary: true, - BinaryIds: BinaryIdsToProto(binaryIds), + BinaryIds: binaryIDsToProto(binaryIDs), }) if err != nil { return nil, err } data := make([]BinaryData, len(resp.Data)) for i, protoData := range resp.Data { - data[i] = BinaryDataFromProto(protoData) + data[i] = binaryDataFromProto(protoData) } return data, nil } // DeleteTabularData deletes tabular data older than a number of days, based on the given organization ID. -func (d *DataClient) DeleteTabularData(ctx context.Context, organizationId string, deleteOlderThanDays uint32) (uint64, error) { +func (d *Client) DeleteTabularData(ctx context.Context, organizationID string, deleteOlderThanDays uint32) (uint64, error) { resp, err := d.client.DeleteTabularData(ctx, &pb.DeleteTabularDataRequest{ - OrganizationId: organizationId, + OrganizationId: organizationID, DeleteOlderThanDays: deleteOlderThanDays, }) if err != nil { @@ -552,9 +559,9 @@ func (d *DataClient) DeleteTabularData(ctx context.Context, organizationId strin } // DeleteBinaryDataByFilter deletes binary data based on given filters. -func (d *DataClient) DeleteBinaryDataByFilter(ctx context.Context, filter Filter) (uint64, error) { +func (d *Client) DeleteBinaryDataByFilter(ctx context.Context, filter Filter) (uint64, error) { resp, err := d.client.DeleteBinaryDataByFilter(ctx, &pb.DeleteBinaryDataByFilterRequest{ - Filter: FilterToProto(filter), + Filter: filterToProto(filter), IncludeInternalData: true, }) if err != nil { @@ -564,9 +571,9 @@ func (d *DataClient) DeleteBinaryDataByFilter(ctx context.Context, filter Filter } // DeleteBinaryDataByIDs deletes binary data based on given IDs. -func (d *DataClient) DeleteBinaryDataByIDs(ctx context.Context, binaryIds []BinaryID) (uint64, error) { +func (d *Client) DeleteBinaryDataByIDs(ctx context.Context, binaryIDs []BinaryID) (uint64, error) { resp, err := d.client.DeleteBinaryDataByIDs(ctx, &pb.DeleteBinaryDataByIDsRequest{ - BinaryIds: BinaryIdsToProto(binaryIds), + BinaryIds: binaryIDsToProto(binaryIDs), }) if err != nil { return 0, err @@ -575,8 +582,11 @@ func (d *DataClient) DeleteBinaryDataByIDs(ctx context.Context, binaryIds []Bina } // AddTagsToBinaryDataByIDs adds string tags, unless the tags are already present, to binary data based on given IDs. -func (d *DataClient) AddTagsToBinaryDataByIDs(ctx context.Context, tags []string, binaryIds []BinaryID) error { - _, err := d.client.AddTagsToBinaryDataByIDs(ctx, &pb.AddTagsToBinaryDataByIDsRequest{BinaryIds: BinaryIdsToProto(binaryIds), Tags: tags}) +func (d *Client) AddTagsToBinaryDataByIDs(ctx context.Context, tags []string, binaryIDs []BinaryID) error { + _, err := d.client.AddTagsToBinaryDataByIDs(ctx, &pb.AddTagsToBinaryDataByIDsRequest{ + BinaryIds: binaryIDsToProto(binaryIDs), + Tags: tags, + }) if err != nil { return err } @@ -584,26 +594,39 @@ func (d *DataClient) AddTagsToBinaryDataByIDs(ctx context.Context, tags []string } // AddTagsToBinaryDataByFilter adds string tags, unless the tags are already present, to binary data based on the given filter. -func (d *DataClient) AddTagsToBinaryDataByFilter(ctx context.Context, tags []string, filter Filter) error { - _, err := d.client.AddTagsToBinaryDataByFilter(ctx, &pb.AddTagsToBinaryDataByFilterRequest{Filter: FilterToProto(filter), Tags: tags}) +func (d *Client) AddTagsToBinaryDataByFilter(ctx context.Context, tags []string, filter Filter) error { + _, err := d.client.AddTagsToBinaryDataByFilter(ctx, &pb.AddTagsToBinaryDataByFilterRequest{ + Filter: filterToProto(filter), + Tags: tags, + }) if err != nil { return err } return nil } -// RemoveTagsToBinaryDataByIDs removes string tags from binary data based on given IDs. -func (d *DataClient) RemoveTagsFromBinaryDataByIDs(ctx context.Context, tags []string, binaryIds []BinaryID) (uint64, error) { - resp, err := d.client.RemoveTagsFromBinaryDataByIDs(ctx, &pb.RemoveTagsFromBinaryDataByIDsRequest{BinaryIds: BinaryIdsToProto(binaryIds), Tags: tags}) +// RemoveTagsFromBinaryDataByIDs removes string tags from binary data based on given IDs. +func (d *Client) RemoveTagsFromBinaryDataByIDs(ctx context.Context, + tags []string, binaryIDs []BinaryID, +) (uint64, error) { + resp, err := d.client.RemoveTagsFromBinaryDataByIDs(ctx, &pb.RemoveTagsFromBinaryDataByIDsRequest{ + BinaryIds: binaryIDsToProto(binaryIDs), + Tags: tags, + }) if err != nil { return 0, err } return resp.DeletedCount, nil } -// RemoveTagsToBinaryDataByFilter removes string tags from binary data based on the given filter. -func (d *DataClient) RemoveTagsFromBinaryDataByFilter(ctx context.Context, tags []string, filter Filter) (uint64, error) { - resp, err := d.client.RemoveTagsFromBinaryDataByFilter(ctx, &pb.RemoveTagsFromBinaryDataByFilterRequest{Filter: FilterToProto(filter), Tags: tags}) +// RemoveTagsFromBinaryDataByFilter removes string tags from binary data based on the given filter. +func (d *Client) RemoveTagsFromBinaryDataByFilter(ctx context.Context, + tags []string, filter Filter, +) (uint64, error) { + resp, err := d.client.RemoveTagsFromBinaryDataByFilter(ctx, &pb.RemoveTagsFromBinaryDataByFilterRequest{ + Filter: filterToProto(filter), + Tags: tags, + }) if err != nil { return 0, err } @@ -611,8 +634,10 @@ func (d *DataClient) RemoveTagsFromBinaryDataByFilter(ctx context.Context, tags } // TagsByFilter gets all unique tags from data based on given filter. -func (d *DataClient) TagsByFilter(ctx context.Context, filter Filter) ([]string, error) { - resp, err := d.client.TagsByFilter(ctx, &pb.TagsByFilterRequest{Filter: FilterToProto(filter)}) +func (d *Client) TagsByFilter(ctx context.Context, filter Filter) ([]string, error) { + resp, err := d.client.TagsByFilter(ctx, &pb.TagsByFilterRequest{ + Filter: filterToProto(filter), + }) if err != nil { return nil, err } @@ -620,25 +645,39 @@ func (d *DataClient) TagsByFilter(ctx context.Context, filter Filter) ([]string, } // AddBoundingBoxToImageByID adds a bounding box to an image with the given ID. -func (d *DataClient) AddBoundingBoxToImageByID( +func (d *Client) AddBoundingBoxToImageByID( ctx context.Context, - binaryId BinaryID, + binaryID BinaryID, label string, xMinNormalized float64, yMinNormalized float64, xMaxNormalized float64, - yMaxNormalized float64) (string, error) { - resp, err := d.client.AddBoundingBoxToImageByID(ctx, &pb.AddBoundingBoxToImageByIDRequest{BinaryId: BinaryIdToProto(binaryId), Label: label, XMinNormalized: xMinNormalized, YMinNormalized: yMinNormalized, XMaxNormalized: xMaxNormalized, YMaxNormalized: yMaxNormalized}) + yMaxNormalized float64, +) (string, error) { + resp, err := d.client.AddBoundingBoxToImageByID(ctx, &pb.AddBoundingBoxToImageByIDRequest{ + BinaryId: binaryIDToProto(binaryID), + Label: label, + XMinNormalized: xMinNormalized, + YMinNormalized: yMinNormalized, + XMaxNormalized: xMaxNormalized, + YMaxNormalized: yMaxNormalized, + }) if err != nil { return "", err } return resp.BboxId, nil - } // RemoveBoundingBoxFromImageByID removes a bounding box from an image with the given ID. -func (d *DataClient) RemoveBoundingBoxFromImageByID(ctx context.Context, bboxId string, binaryId BinaryID) error { - _, err := d.client.RemoveBoundingBoxFromImageByID(ctx, &pb.RemoveBoundingBoxFromImageByIDRequest{BinaryId: BinaryIdToProto(binaryId), BboxId: bboxId}) +func (d *Client) RemoveBoundingBoxFromImageByID( + ctx context.Context, + bboxID string, + binaryID BinaryID, +) error { + _, err := d.client.RemoveBoundingBoxFromImageByID(ctx, &pb.RemoveBoundingBoxFromImageByIDRequest{ + BinaryId: binaryIDToProto(binaryID), + BboxId: bboxID, + }) if err != nil { return err } @@ -646,8 +685,10 @@ func (d *DataClient) RemoveBoundingBoxFromImageByID(ctx context.Context, bboxId } // BoundingBoxLabelsByFilter gets all string labels for bounding boxes from data based on given filter. -func (d *DataClient) BoundingBoxLabelsByFilter(ctx context.Context, filter Filter) ([]string, error) { - resp, err := d.client.BoundingBoxLabelsByFilter(ctx, &pb.BoundingBoxLabelsByFilterRequest{Filter: FilterToProto(filter)}) +func (d *Client) BoundingBoxLabelsByFilter(ctx context.Context, filter Filter) ([]string, error) { + resp, err := d.client.BoundingBoxLabelsByFilter(ctx, &pb.BoundingBoxLabelsByFilterRequest{ + Filter: filterToProto(filter), + }) if err != nil { return nil, err } @@ -655,16 +696,24 @@ func (d *DataClient) BoundingBoxLabelsByFilter(ctx context.Context, filter Filte } // UpdateBoundingBox updates the bounding box associated with a given binary ID and bounding box ID. -func (d *DataClient) UpdateBoundingBox(ctx context.Context, - binaryId BinaryID, - bboxId string, +func (d *Client) UpdateBoundingBox(ctx context.Context, + binaryID BinaryID, + bboxID string, label *string, // optional xMinNormalized *float64, // optional yMinNormalized *float64, // optional xMaxNormalized *float64, // optional yMaxNormalized *float64, // optional ) error { - _, err := d.client.UpdateBoundingBox(ctx, &pb.UpdateBoundingBoxRequest{BinaryId: BinaryIdToProto(binaryId), BboxId: bboxId, Label: label, XMinNormalized: xMinNormalized, YMinNormalized: yMinNormalized, XMaxNormalized: xMaxNormalized, YMaxNormalized: yMaxNormalized}) + _, err := d.client.UpdateBoundingBox(ctx, &pb.UpdateBoundingBoxRequest{ + BinaryId: binaryIDToProto(binaryID), + BboxId: bboxID, + Label: label, + XMinNormalized: xMinNormalized, + YMinNormalized: yMinNormalized, + XMaxNormalized: xMaxNormalized, + YMaxNormalized: yMaxNormalized, + }) if err != nil { return err } @@ -673,8 +722,10 @@ func (d *DataClient) UpdateBoundingBox(ctx context.Context, // GetDatabaseConnection gets a connection to access a MongoDB Atlas Data Federation instance. It // returns the hostname of the federated database. -func (d *DataClient) GetDatabaseConnection(ctx context.Context, organizationId string) (string, error) { - resp, err := d.client.GetDatabaseConnection(ctx, &pb.GetDatabaseConnectionRequest{OrganizationId: organizationId}) +func (d *Client) GetDatabaseConnection(ctx context.Context, organizationID string) (string, error) { + resp, err := d.client.GetDatabaseConnection(ctx, &pb.GetDatabaseConnectionRequest{ + OrganizationId: organizationID, + }) if err != nil { return "", err } @@ -683,8 +734,15 @@ func (d *DataClient) GetDatabaseConnection(ctx context.Context, organizationId s // ConfigureDatabaseUser configures a database user for the Viam organization's MongoDB Atlas Data // Federation instance. It can also be used to reset the password of the existing database user. -func (d *DataClient) ConfigureDatabaseUser(ctx context.Context, organizationId string, password string) error { - _, err := d.client.ConfigureDatabaseUser(ctx, &pb.ConfigureDatabaseUserRequest{OrganizationId: organizationId, Password: password}) +func (d *Client) ConfigureDatabaseUser( + ctx context.Context, + organizationID string, + password string, +) error { + _, err := d.client.ConfigureDatabaseUser(ctx, &pb.ConfigureDatabaseUserRequest{ + OrganizationId: organizationID, + Password: password, + }) if err != nil { return err } @@ -692,8 +750,15 @@ func (d *DataClient) ConfigureDatabaseUser(ctx context.Context, organizationId s } // AddBinaryDataToDatasetByIDs adds the binary data with the given binary IDs to the dataset. -func (d *DataClient) AddBinaryDataToDatasetByIDs(ctx context.Context, binaryIds []BinaryID, datasetId string) error { - _, err := d.client.AddBinaryDataToDatasetByIDs(ctx, &pb.AddBinaryDataToDatasetByIDsRequest{BinaryIds: BinaryIdsToProto(binaryIds), DatasetId: datasetId}) +func (d *Client) AddBinaryDataToDatasetByIDs( + ctx context.Context, + binaryIDs []BinaryID, + datasetID string, +) error { + _, err := d.client.AddBinaryDataToDatasetByIDs(ctx, &pb.AddBinaryDataToDatasetByIDsRequest{ + BinaryIds: binaryIDsToProto(binaryIDs), + DatasetId: datasetID, + }) if err != nil { return err } @@ -701,8 +766,15 @@ func (d *DataClient) AddBinaryDataToDatasetByIDs(ctx context.Context, binaryIds } // RemoveBinaryDataFromDatasetByIDs removes the binary data with the given binary IDs from the dataset. -func (d *DataClient) RemoveBinaryDataFromDatasetByIDs(ctx context.Context, binaryIds []BinaryID, datasetId string) error { - _, err := d.client.RemoveBinaryDataFromDatasetByIDs(ctx, &pb.RemoveBinaryDataFromDatasetByIDsRequest{BinaryIds: BinaryIdsToProto(binaryIds), DatasetId: datasetId}) +func (d *Client) RemoveBinaryDataFromDatasetByIDs( + ctx context.Context, + binaryIDs []BinaryID, + datasetID string, +) error { + _, err := d.client.RemoveBinaryDataFromDatasetByIDs(ctx, &pb.RemoveBinaryDataFromDatasetByIDsRequest{ + BinaryIds: binaryIDsToProto(binaryIDs), + DatasetId: datasetID, + }) if err != nil { return err } diff --git a/app/data_test.go b/app/data_test.go index 1190b7440b0..ef4c945b076 100644 --- a/app/data_test.go +++ b/app/data_test.go @@ -1,4 +1,4 @@ -package app +package data import ( "context" @@ -7,11 +7,12 @@ import ( "go.mongodb.org/mongo-driver/bson" datapb "go.viam.com/api/app/data/v1" - "go.viam.com/rdk/testutils/inject" "go.viam.com/test" utils "go.viam.com/utils/protoutils" "google.golang.org/grpc" "google.golang.org/protobuf/types/known/timestamppb" + + "go.viam.com/rdk/testutils/inject" ) const ( @@ -19,11 +20,11 @@ const ( componentType = "component_type" method = "method" robotName = "robot_name" - robotId = "robot_id" + robotID = "robot_id" partName = "part_name" - partId = "part_id" - locationId = "location_id" - organizationId = "organization_id" + partID = "part_id" + locationID = "location_id" + organizationID = "organization_id" password = "password" mimeType = "mime_type" uri = "some.robot.uri" @@ -31,15 +32,15 @@ const ( tag = "tag" fileName = "file_name" fileExt = "file_ext.ext" - datasetId = "dataset_id" - binaryMetaId = "binary_id" - mongodbUri = "mongo_uri" + datasetID = "dataset_id" + binaryMetaID = "binary_id" + mongodbURI = "mongo_uri" hostName = "host_name" ) var ( - locationIds = []string{locationId} - orgIds = []string{organizationId} + locationIDs = []string{locationID} + orgIDs = []string{organizationID} mimeTypes = []string{mimeType} bboxLabels = []string{bboxLabel} methodParameters = map[string]string{} @@ -49,12 +50,12 @@ var ( data = map[string]interface{}{ "key": "value", } - binaryId = BinaryID{ - FileId: "file1", - OrganizationId: organizationId, - LocationId: locationId, + binaryID = BinaryID{ + FileID: "file1", + OrganizationID: organizationID, + LocationID: locationID, } - binaryIds = []BinaryID{binaryId} + binaryIDs = []BinaryID{binaryID} binaryDataByte = []byte("BYTE") sqlQuery = "SELECT * FROM readings WHERE organization_id='e76d1b3b-0468-4efd-bb7f-fb1d2b352fcb' LIMIT 1" rawData = []map[string]any{ @@ -77,11 +78,11 @@ var ( }, }, } - datasetIds = []string{datasetId} + datasetIDs = []string{datasetID} annotations = Annotations{ Bboxes: []BoundingBox{ { - Id: "bbox1", + ID: "bbox1", Label: "label1", XMinNormalized: 0.1, YMinNormalized: 0.2, @@ -89,7 +90,7 @@ var ( YMaxNormalized: 0.9, }, { - Id: "bbox2", + ID: "bbox2", Label: "label2", XMinNormalized: 0.15, YMinNormalized: 0.25, @@ -103,9 +104,10 @@ var ( func createGrpcClient() *inject.DataServiceClient { return &inject.DataServiceClient{} } + func TestDataClient(t *testing.T) { grpcClient := createGrpcClient() - client := DataClient{client: grpcClient} + client := Client{client: grpcClient} captureInterval := CaptureInterval{ Start: time.Now(), @@ -121,24 +123,24 @@ func TestDataClient(t *testing.T) { ComponentType: componentType, Method: method, RobotName: robotName, - RobotId: robotId, + RobotID: robotID, PartName: partName, - PartId: partId, - LocationIds: locationIds, - OrganizationIds: orgIds, + PartID: partID, + LocationIDs: locationIDs, + OrganizationIDs: orgIDs, MimeType: mimeTypes, Interval: captureInterval, - TagsFilter: tagsFilter, //asterix or no?? + TagsFilter: tagsFilter, // asterix or no?? BboxLabels: bboxLabels, - DatasetId: datasetId, + DatasetID: datasetID, } tabularMetadata := CaptureMetadata{ - OrganizationId: organizationId, - LocationId: locationId, + OrganizationID: organizationID, + LocationID: locationID, RobotName: robotName, - RobotId: robotId, + RobotID: robotID, PartName: partName, - PartId: partId, + PartID: partID, ComponentType: componentType, ComponentName: componentName, MethodName: method, @@ -147,7 +149,7 @@ func TestDataClient(t *testing.T) { MimeType: mimeType, } binaryMetadata := BinaryMetadata{ - ID: binaryMetaId, + ID: binaryMetaID, CaptureMetadata: tabularMetadata, TimeRequested: startTime, TimeReceived: endTime, @@ -155,7 +157,7 @@ func TestDataClient(t *testing.T) { FileExt: fileExt, URI: uri, Annotations: annotations, - DatasetIDs: datasetIds, + DatasetIDs: datasetIDs, } t.Run("TabularDataByFilter", func(t *testing.T) { countOnly := true @@ -185,8 +187,10 @@ func TestDataClient(t *testing.T) { TimeReceived: timestamppb.New(endTime), } - grpcClient.TabularDataByFilterFunc = func(ctx context.Context, in *datapb.TabularDataByFilterRequest, opts ...grpc.CallOption) (*datapb.TabularDataByFilterResponse, error) { - expectedDataReqProto, _ := DataRequestToProto(dataRequest) + grpcClient.TabularDataByFilterFunc = func(ctx context.Context, in *datapb.TabularDataByFilterRequest, + opts ...grpc.CallOption, + ) (*datapb.TabularDataByFilterResponse, error) { + expectedDataReqProto := dataRequestToProto(dataRequest) test.That(t, in.DataRequest, test.ShouldResemble, expectedDataReqProto) test.That(t, in.CountOnly, test.ShouldBeTrue) test.That(t, in.IncludeInternalData, test.ShouldBeTrue) @@ -194,20 +198,23 @@ func TestDataClient(t *testing.T) { Data: []*datapb.TabularData{tabularDataPb}, Count: expectedCount, Last: expectedLast, - Metadata: []*datapb.CaptureMetadata{CaptureMetadataToProto(tabularMetadata)}}, nil + Metadata: []*datapb.CaptureMetadata{captureMetadataToProto(tabularMetadata)}, + }, nil } - respTabularData, respCount, respLast, _ := client.TabularDataByFilter(context.Background(), filter, expectedLimit, expectedLast, dataRequest.SortOrder, countOnly, includeInternalData) + respTabularData, respCount, respLast, _ := client.TabularDataByFilter( + context.Background(), filter, expectedLimit, expectedLast, + dataRequest.SortOrder, countOnly, includeInternalData) test.That(t, respTabularData[0], test.ShouldResemble, expectedTabularData) test.That(t, respCount, test.ShouldEqual, expectedCount) test.That(t, respLast, test.ShouldEqual, expectedLast) }) t.Run("TabularDataBySQL", func(t *testing.T) { - expectedOrgId := organizationId - expectedSqlQuery := sqlQuery + expectedOrgID := organizationID + expectedSQLQuery := sqlQuery expectedRawData := rawData - //convert rawData to BSON + // convert rawData to BSON var expectedRawDataPb [][]byte for _, byte := range expectedRawData { bsonByte, err := bson.Marshal(byte) @@ -216,19 +223,21 @@ func TestDataClient(t *testing.T) { } expectedRawDataPb = append(expectedRawDataPb, bsonByte) } - grpcClient.TabularDataBySQLFunc = func(ctx context.Context, in *datapb.TabularDataBySQLRequest, opts ...grpc.CallOption) (*datapb.TabularDataBySQLResponse, error) { - test.That(t, in.OrganizationId, test.ShouldEqual, expectedOrgId) - test.That(t, in.SqlQuery, test.ShouldResemble, expectedSqlQuery) + grpcClient.TabularDataBySQLFunc = func(ctx context.Context, in *datapb.TabularDataBySQLRequest, + opts ...grpc.CallOption, + ) (*datapb.TabularDataBySQLResponse, error) { + test.That(t, in.OrganizationId, test.ShouldEqual, expectedOrgID) + test.That(t, in.SqlQuery, test.ShouldResemble, expectedSQLQuery) return &datapb.TabularDataBySQLResponse{ RawData: expectedRawDataPb, }, nil } - response, _ := client.TabularDataBySQL(context.Background(), expectedOrgId, expectedSqlQuery) + response, _ := client.TabularDataBySQL(context.Background(), expectedOrgID, expectedSQLQuery) test.That(t, response, test.ShouldResemble, expectedRawData) }) t.Run("TabularDataByMQL", func(t *testing.T) { - expectedOrgId := organizationId + expectedOrgID := organizationID matchStage := bson.M{"$match": bson.M{"organization_id": "e76d1b3b-0468-4efd-bb7f-fb1d2b352fcb"}} limitStage := bson.M{"$limit": 1} @@ -239,7 +248,7 @@ func TestDataClient(t *testing.T) { expectedMqlBinary := mqlbinary expectedRawData := rawData - //convert rawData to BSON + // convert rawData to BSON var expectedRawDataPb [][]byte for _, byte := range expectedRawData { bsonByte, err := bson.Marshal(byte) @@ -248,14 +257,16 @@ func TestDataClient(t *testing.T) { } expectedRawDataPb = append(expectedRawDataPb, bsonByte) } - grpcClient.TabularDataByMQLFunc = func(ctx context.Context, in *datapb.TabularDataByMQLRequest, opts ...grpc.CallOption) (*datapb.TabularDataByMQLResponse, error) { - test.That(t, in.OrganizationId, test.ShouldEqual, expectedOrgId) + grpcClient.TabularDataByMQLFunc = func(ctx context.Context, in *datapb.TabularDataByMQLRequest, + opts ...grpc.CallOption, + ) (*datapb.TabularDataByMQLResponse, error) { + test.That(t, in.OrganizationId, test.ShouldEqual, expectedOrgID) test.That(t, in.MqlBinary, test.ShouldResemble, expectedMqlBinary) return &datapb.TabularDataByMQLResponse{ RawData: expectedRawDataPb, }, nil } - response, _ := client.TabularDataByMQL(context.Background(), expectedOrgId, expectedMqlBinary) + response, _ := client.TabularDataByMQL(context.Background(), expectedOrgID, expectedMqlBinary) test.That(t, response, test.ShouldResemble, expectedRawData) }) @@ -275,10 +286,12 @@ func TestDataClient(t *testing.T) { Binary: binaryDataByte, Metadata: binaryMetadata, } - expectedBinaryDataPb := BinaryDataToProto(expectedBinaryData) + expectedBinaryDataPb := binaryDataToProto(expectedBinaryData) - grpcClient.BinaryDataByFilterFunc = func(ctx context.Context, in *datapb.BinaryDataByFilterRequest, opts ...grpc.CallOption) (*datapb.BinaryDataByFilterResponse, error) { - expectedDataReq, _ := DataRequestToProto(dataRequest) + grpcClient.BinaryDataByFilterFunc = func(ctx context.Context, in *datapb.BinaryDataByFilterRequest, + opts ...grpc.CallOption, + ) (*datapb.BinaryDataByFilterResponse, error) { + expectedDataReq := dataRequestToProto(dataRequest) test.That(t, in.DataRequest, test.ShouldResemble, expectedDataReq) test.That(t, in.IncludeBinary, test.ShouldBeTrue) test.That(t, in.CountOnly, test.ShouldBeTrue) @@ -289,51 +302,58 @@ func TestDataClient(t *testing.T) { Last: expectedLast, }, nil } - respBinaryData, respCount, respLast, _ := client.BinaryDataByFilter(context.Background(), filter, expectedCount, dataRequest.SortOrder, expectedLast, includeBinary, countOnly, includeInternalData) + respBinaryData, respCount, respLast, _ := client.BinaryDataByFilter( + context.Background(), filter, expectedCount, dataRequest.SortOrder, + expectedLast, includeBinary, countOnly, includeInternalData) test.That(t, respBinaryData[0], test.ShouldResemble, expectedBinaryData) test.That(t, respCount, test.ShouldEqual, expectedCount) test.That(t, respLast, test.ShouldEqual, expectedLast) - }) t.Run("BinaryDataByIDs", func(t *testing.T) { expectedBinaryData := BinaryData{ Binary: binaryDataByte, Metadata: binaryMetadata, } - grpcClient.BinaryDataByIDsFunc = func(ctx context.Context, in *datapb.BinaryDataByIDsRequest, opts ...grpc.CallOption) (*datapb.BinaryDataByIDsResponse, error) { + grpcClient.BinaryDataByIDsFunc = func(ctx context.Context, in *datapb.BinaryDataByIDsRequest, + opts ...grpc.CallOption, + ) (*datapb.BinaryDataByIDsResponse, error) { test.That(t, in.IncludeBinary, test.ShouldBeTrue) - test.That(t, in.BinaryIds, test.ShouldResemble, BinaryIdsToProto(binaryIds)) - expectedBinaryDataList := []*datapb.BinaryData{BinaryDataToProto(expectedBinaryData)} + test.That(t, in.BinaryIds, test.ShouldResemble, binaryIDsToProto(binaryIDs)) + expectedBinaryDataList := []*datapb.BinaryData{binaryDataToProto(expectedBinaryData)} return &datapb.BinaryDataByIDsResponse{Data: expectedBinaryDataList, Count: uint64(len(expectedBinaryDataList))}, nil } - respBinaryData, _ := client.BinaryDataByIDs(context.Background(), binaryIds) + respBinaryData, _ := client.BinaryDataByIDs(context.Background(), binaryIDs) test.That(t, respBinaryData[0], test.ShouldResemble, expectedBinaryData) }) t.Run("DeleteTabularData", func(t *testing.T) { deleteOlderThanDays := uint32(1) - expectedOrgId := organizationId + expectedOrgID := organizationID expectedDeleteOlderThanDays := deleteOlderThanDays expectedDeletedCount := uint64(5) - grpcClient.DeleteTabularDataFunc = func(ctx context.Context, in *datapb.DeleteTabularDataRequest, opts ...grpc.CallOption) (*datapb.DeleteTabularDataResponse, error) { - test.That(t, in.OrganizationId, test.ShouldEqual, expectedOrgId) + grpcClient.DeleteTabularDataFunc = func(ctx context.Context, in *datapb.DeleteTabularDataRequest, + opts ...grpc.CallOption, + ) (*datapb.DeleteTabularDataResponse, error) { + test.That(t, in.OrganizationId, test.ShouldEqual, expectedOrgID) test.That(t, in.DeleteOlderThanDays, test.ShouldEqual, expectedDeleteOlderThanDays) return &datapb.DeleteTabularDataResponse{ DeletedCount: expectedDeletedCount, }, nil } - resp, _ := client.DeleteTabularData(context.Background(), organizationId, deleteOlderThanDays) + resp, _ := client.DeleteTabularData(context.Background(), organizationID, deleteOlderThanDays) test.That(t, resp, test.ShouldEqual, expectedDeletedCount) }) t.Run("DeleteBinaryDataByFilter", func(t *testing.T) { - expectedFilterPb := FilterToProto(filter) + expectedFilterPb := filterToProto(filter) expectedDeletedCount := uint64(5) - grpcClient.DeleteBinaryDataByFilterFunc = func(ctx context.Context, in *datapb.DeleteBinaryDataByFilterRequest, opts ...grpc.CallOption) (*datapb.DeleteBinaryDataByFilterResponse, error) { + grpcClient.DeleteBinaryDataByFilterFunc = func(ctx context.Context, in *datapb.DeleteBinaryDataByFilterRequest, + opts ...grpc.CallOption, + ) (*datapb.DeleteBinaryDataByFilterResponse, error) { test.That(t, in.Filter, test.ShouldResemble, expectedFilterPb) test.That(t, in.IncludeInternalData, test.ShouldBeTrue) return &datapb.DeleteBinaryDataByFilterResponse{ @@ -346,33 +366,38 @@ func TestDataClient(t *testing.T) { t.Run("DeleteBinaryDataByIDs", func(t *testing.T) { expectedDeletedCount := uint64(5) - expectedBinaryIds := BinaryIdsToProto(binaryIds) - grpcClient.DeleteBinaryDataByIDsFunc = func(ctx context.Context, in *datapb.DeleteBinaryDataByIDsRequest, opts ...grpc.CallOption) (*datapb.DeleteBinaryDataByIDsResponse, error) { - test.That(t, in.BinaryIds, test.ShouldResemble, expectedBinaryIds) + expectedBinaryIDs := binaryIDsToProto(binaryIDs) + grpcClient.DeleteBinaryDataByIDsFunc = func(ctx context.Context, in *datapb.DeleteBinaryDataByIDsRequest, + opts ...grpc.CallOption, + ) (*datapb.DeleteBinaryDataByIDsResponse, error) { + test.That(t, in.BinaryIds, test.ShouldResemble, expectedBinaryIDs) return &datapb.DeleteBinaryDataByIDsResponse{ DeletedCount: expectedDeletedCount, }, nil } - resp, _ := client.DeleteBinaryDataByIDs(context.Background(), binaryIds) + resp, _ := client.DeleteBinaryDataByIDs(context.Background(), binaryIDs) test.That(t, resp, test.ShouldEqual, expectedDeletedCount) }) t.Run("AddTagsToBinaryDataByIDs", func(t *testing.T) { expectedTags := tags - expectedBinaryIds := BinaryIdsToProto(binaryIds) - grpcClient.AddTagsToBinaryDataByIDsFunc = func(ctx context.Context, in *datapb.AddTagsToBinaryDataByIDsRequest, opts ...grpc.CallOption) (*datapb.AddTagsToBinaryDataByIDsResponse, error) { - test.That(t, in.BinaryIds, test.ShouldResemble, expectedBinaryIds) + expectedBinaryIDs := binaryIDsToProto(binaryIDs) + grpcClient.AddTagsToBinaryDataByIDsFunc = func(ctx context.Context, in *datapb.AddTagsToBinaryDataByIDsRequest, + opts ...grpc.CallOption, + ) (*datapb.AddTagsToBinaryDataByIDsResponse, error) { + test.That(t, in.BinaryIds, test.ShouldResemble, expectedBinaryIDs) test.That(t, in.Tags, test.ShouldResemble, expectedTags) return &datapb.AddTagsToBinaryDataByIDsResponse{}, nil } - client.AddTagsToBinaryDataByIDs(context.Background(), tags, binaryIds) - + client.AddTagsToBinaryDataByIDs(context.Background(), tags, binaryIDs) }) t.Run("AddTagsToBinaryDataByFilter", func(t *testing.T) { expectedTags := tags - expectedFilterPb := FilterToProto(filter) - grpcClient.AddTagsToBinaryDataByFilterFunc = func(ctx context.Context, in *datapb.AddTagsToBinaryDataByFilterRequest, opts ...grpc.CallOption) (*datapb.AddTagsToBinaryDataByFilterResponse, error) { + expectedFilterPb := filterToProto(filter) + grpcClient.AddTagsToBinaryDataByFilterFunc = func(ctx context.Context, in *datapb.AddTagsToBinaryDataByFilterRequest, + opts ...grpc.CallOption, + ) (*datapb.AddTagsToBinaryDataByFilterResponse, error) { test.That(t, in.Filter, test.ShouldResemble, expectedFilterPb) test.That(t, in.Tags, test.ShouldResemble, expectedTags) return &datapb.AddTagsToBinaryDataByFilterResponse{}, nil @@ -382,25 +407,29 @@ func TestDataClient(t *testing.T) { t.Run("RemoveTagsFromBinaryDataByIDs", func(t *testing.T) { expectedTags := tags - expectedBinaryIds := BinaryIdsToProto(binaryIds) + expectedBinaryIDs := binaryIDsToProto(binaryIDs) expectedDeletedCount := uint64(5) - grpcClient.RemoveTagsFromBinaryDataByIDsFunc = func(ctx context.Context, in *datapb.RemoveTagsFromBinaryDataByIDsRequest, opts ...grpc.CallOption) (*datapb.RemoveTagsFromBinaryDataByIDsResponse, error) { - test.That(t, in.BinaryIds, test.ShouldResemble, expectedBinaryIds) + grpcClient.RemoveTagsFromBinaryDataByIDsFunc = func(ctx context.Context, in *datapb.RemoveTagsFromBinaryDataByIDsRequest, + opts ...grpc.CallOption, + ) (*datapb.RemoveTagsFromBinaryDataByIDsResponse, error) { + test.That(t, in.BinaryIds, test.ShouldResemble, expectedBinaryIDs) test.That(t, in.Tags, test.ShouldResemble, expectedTags) return &datapb.RemoveTagsFromBinaryDataByIDsResponse{ DeletedCount: expectedDeletedCount, }, nil } - resp, _ := client.RemoveTagsFromBinaryDataByIDs(context.Background(), tags, binaryIds) + resp, _ := client.RemoveTagsFromBinaryDataByIDs(context.Background(), tags, binaryIDs) test.That(t, resp, test.ShouldEqual, expectedDeletedCount) }) t.Run("RemoveTagsFromBinaryDataByFilter", func(t *testing.T) { expectedTags := tags - expectedFilterPb := FilterToProto(filter) + expectedFilterPb := filterToProto(filter) expectedDeletedCount := uint64(5) - grpcClient.RemoveTagsFromBinaryDataByFilterFunc = func(ctx context.Context, in *datapb.RemoveTagsFromBinaryDataByFilterRequest, opts ...grpc.CallOption) (*datapb.RemoveTagsFromBinaryDataByFilterResponse, error) { + grpcClient.RemoveTagsFromBinaryDataByFilterFunc = func(ctx context.Context, in *datapb.RemoveTagsFromBinaryDataByFilterRequest, + opts ...grpc.CallOption, + ) (*datapb.RemoveTagsFromBinaryDataByFilterResponse, error) { test.That(t, in.Filter, test.ShouldResemble, expectedFilterPb) test.That(t, in.Tags, test.ShouldResemble, expectedTags) return &datapb.RemoveTagsFromBinaryDataByFilterResponse{ @@ -413,9 +442,11 @@ func TestDataClient(t *testing.T) { t.Run("TagsByFilter", func(t *testing.T) { expectedTags := tags - expectedFilterPb := FilterToProto(filter) + expectedFilterPb := filterToProto(filter) - grpcClient.TagsByFilterFunc = func(ctx context.Context, in *datapb.TagsByFilterRequest, opts ...grpc.CallOption) (*datapb.TagsByFilterResponse, error) { + grpcClient.TagsByFilterFunc = func(ctx context.Context, in *datapb.TagsByFilterRequest, + opts ...grpc.CallOption, + ) (*datapb.TagsByFilterResponse, error) { test.That(t, in.Filter, test.ShouldResemble, expectedFilterPb) return &datapb.TagsByFilterResponse{ Tags: tags, @@ -426,16 +457,19 @@ func TestDataClient(t *testing.T) { }) t.Run("AddBoundingBoxToImageByID", func(t *testing.T) { - expectedBinaryIdPb := BinaryIdToProto(binaryId) + expectedBinaryIDPb := binaryIDToProto(binaryID) expectedLabel := bboxLabel expectedXMin := annotations.Bboxes[0].XMinNormalized expectedYMin := annotations.Bboxes[0].YMinNormalized expectedXMax := annotations.Bboxes[0].XMaxNormalized expectedYMax := annotations.Bboxes[0].YMaxNormalized - expectedBBoxId := annotations.Bboxes[0].Id + expectedBBoxID := annotations.Bboxes[0].ID - grpcClient.AddBoundingBoxToImageByIDFunc = func(ctx context.Context, in *datapb.AddBoundingBoxToImageByIDRequest, opts ...grpc.CallOption) (*datapb.AddBoundingBoxToImageByIDResponse, error) { - test.That(t, in.BinaryId, test.ShouldResemble, expectedBinaryIdPb) + grpcClient.AddBoundingBoxToImageByIDFunc = func(ctx context.Context, + in *datapb.AddBoundingBoxToImageByIDRequest, + opts ...grpc.CallOption, + ) (*datapb.AddBoundingBoxToImageByIDResponse, error) { + test.That(t, in.BinaryId, test.ShouldResemble, expectedBinaryIDPb) test.That(t, in.Label, test.ShouldEqual, expectedLabel) test.That(t, in.XMinNormalized, test.ShouldEqual, expectedXMin) test.That(t, in.YMinNormalized, test.ShouldEqual, expectedYMin) @@ -443,40 +477,45 @@ func TestDataClient(t *testing.T) { test.That(t, in.YMaxNormalized, test.ShouldEqual, expectedYMax) return &datapb.AddBoundingBoxToImageByIDResponse{ - BboxId: expectedBBoxId, + BboxId: expectedBBoxID, }, nil } - resp, _ := client.AddBoundingBoxToImageByID(context.Background(), binaryId, bboxLabel, expectedXMin, expectedYMin, expectedXMax, expectedYMax) - test.That(t, resp, test.ShouldResemble, expectedBBoxId) + resp, _ := client.AddBoundingBoxToImageByID( + context.Background(), binaryID, bboxLabel, expectedXMin, + expectedYMin, expectedXMax, expectedYMax) + test.That(t, resp, test.ShouldResemble, expectedBBoxID) }) t.Run("RemoveBoundingBoxFromImageByID", func(t *testing.T) { - expectedBinaryIdPb := BinaryIdToProto(binaryId) - expectedBBoxId := annotations.Bboxes[0].Id + expectedBinaryIDPb := binaryIDToProto(binaryID) + expectedBBoxID := annotations.Bboxes[0].ID - grpcClient.RemoveBoundingBoxFromImageByIDFunc = func(ctx context.Context, in *datapb.RemoveBoundingBoxFromImageByIDRequest, opts ...grpc.CallOption) (*datapb.RemoveBoundingBoxFromImageByIDResponse, error) { - test.That(t, in.BinaryId, test.ShouldResemble, expectedBinaryIdPb) - test.That(t, in.BboxId, test.ShouldEqual, expectedBBoxId) + grpcClient.RemoveBoundingBoxFromImageByIDFunc = func(ctx context.Context, in *datapb.RemoveBoundingBoxFromImageByIDRequest, + opts ...grpc.CallOption, + ) (*datapb.RemoveBoundingBoxFromImageByIDResponse, error) { + test.That(t, in.BinaryId, test.ShouldResemble, expectedBinaryIDPb) + test.That(t, in.BboxId, test.ShouldEqual, expectedBBoxID) return &datapb.RemoveBoundingBoxFromImageByIDResponse{}, nil } - client.RemoveBoundingBoxFromImageByID(context.Background(), expectedBBoxId, binaryId) - + client.RemoveBoundingBoxFromImageByID(context.Background(), expectedBBoxID, binaryID) }) t.Run("BoundingBoxLabelsByFilter", func(t *testing.T) { - expectedFilterPb := FilterToProto(filter) + expectedFilterPb := filterToProto(filter) expectedBBoxLabels := []string{ annotations.Bboxes[0].Label, annotations.Bboxes[1].Label, } - annotationsPb := AnnotationsToProto(annotations) + annotationsPb := annotationsToProto(annotations) expectedBBoxLabelsPb := []string{ annotationsPb.Bboxes[0].Label, annotationsPb.Bboxes[1].Label, } - grpcClient.BoundingBoxLabelsByFilterFunc = func(ctx context.Context, in *datapb.BoundingBoxLabelsByFilterRequest, opts ...grpc.CallOption) (*datapb.BoundingBoxLabelsByFilterResponse, error) { + grpcClient.BoundingBoxLabelsByFilterFunc = func(ctx context.Context, in *datapb.BoundingBoxLabelsByFilterRequest, + opts ...grpc.CallOption, + ) (*datapb.BoundingBoxLabelsByFilterResponse, error) { test.That(t, in.Filter, test.ShouldResemble, expectedFilterPb) return &datapb.BoundingBoxLabelsByFilterResponse{ Labels: expectedBBoxLabelsPb, @@ -486,20 +525,22 @@ func TestDataClient(t *testing.T) { test.That(t, resp, test.ShouldResemble, expectedBBoxLabels) }) t.Run("UpdateBoundingBox", func(t *testing.T) { - bBoxId := annotations.Bboxes[0].Id - expectedBinaryIdPb := BinaryIdToProto(binaryId) + bBoxID := annotations.Bboxes[0].ID + expectedBinaryIDPb := binaryIDToProto(binaryID) - annotationsPb := AnnotationsToProto(annotations) + annotationsPb := annotationsToProto(annotations) expectedLabel := annotationsPb.Bboxes[0].Label - expectedBBoxIdPb := annotationsPb.Bboxes[0].Id + expectedBBoxIDPb := annotationsPb.Bboxes[0].Id expectedXMin := annotationsPb.Bboxes[0].XMinNormalized expectedYMin := annotationsPb.Bboxes[0].YMinNormalized expectedXMax := annotationsPb.Bboxes[0].XMaxNormalized expectedYMax := annotationsPb.Bboxes[0].YMaxNormalized - grpcClient.UpdateBoundingBoxFunc = func(ctx context.Context, in *datapb.UpdateBoundingBoxRequest, opts ...grpc.CallOption) (*datapb.UpdateBoundingBoxResponse, error) { - test.That(t, in.BinaryId, test.ShouldResemble, expectedBinaryIdPb) - test.That(t, in.BboxId, test.ShouldResemble, expectedBBoxIdPb) + grpcClient.UpdateBoundingBoxFunc = func(ctx context.Context, in *datapb.UpdateBoundingBoxRequest, + opts ...grpc.CallOption, + ) (*datapb.UpdateBoundingBoxResponse, error) { + test.That(t, in.BinaryId, test.ShouldResemble, expectedBinaryIDPb) + test.That(t, in.BboxId, test.ShouldResemble, expectedBBoxIDPb) test.That(t, *in.Label, test.ShouldEqual, expectedLabel) test.That(t, *in.XMinNormalized, test.ShouldEqual, expectedXMin) test.That(t, *in.YMinNormalized, test.ShouldEqual, expectedYMin) @@ -507,60 +548,69 @@ func TestDataClient(t *testing.T) { test.That(t, *in.YMaxNormalized, test.ShouldEqual, expectedYMax) return &datapb.UpdateBoundingBoxResponse{}, nil } - client.UpdateBoundingBox(context.Background(), binaryId, bBoxId, &expectedLabel, &expectedXMin, &expectedYMin, &expectedXMax, &expectedYMax) + client.UpdateBoundingBox(context.Background(), binaryID, bBoxID, &expectedLabel, + &expectedXMin, &expectedYMin, &expectedXMax, &expectedYMax) }) t.Run("GetDatabaseConnection", func(t *testing.T) { - expectedOrgId := organizationId + expectedOrgID := organizationID expectedHostName := hostName - expectedMongodbUri := mongodbUri - expectedDbUser := true + expectedMongodbURI := mongodbURI + expectedDBUser := true - grpcClient.GetDatabaseConnectionFunc = func(ctx context.Context, in *datapb.GetDatabaseConnectionRequest, opts ...grpc.CallOption) (*datapb.GetDatabaseConnectionResponse, error) { - test.That(t, in.OrganizationId, test.ShouldResemble, expectedOrgId) + grpcClient.GetDatabaseConnectionFunc = func(ctx context.Context, in *datapb.GetDatabaseConnectionRequest, + opts ...grpc.CallOption, + ) (*datapb.GetDatabaseConnectionResponse, error) { + test.That(t, in.OrganizationId, test.ShouldResemble, expectedOrgID) return &datapb.GetDatabaseConnectionResponse{ Hostname: expectedHostName, - MongodbUri: expectedMongodbUri, - HasDatabaseUser: expectedDbUser, + MongodbUri: expectedMongodbURI, + HasDatabaseUser: expectedDBUser, }, nil } - resp, _ := client.GetDatabaseConnection(context.Background(), organizationId) + resp, _ := client.GetDatabaseConnection(context.Background(), organizationID) test.That(t, resp, test.ShouldResemble, expectedHostName) }) t.Run("ConfigureDatabaseUser", func(t *testing.T) { - expectedOrgId := organizationId + expectedOrgID := organizationID expectedPassword := password - grpcClient.ConfigureDatabaseUserFunc = func(ctx context.Context, in *datapb.ConfigureDatabaseUserRequest, opts ...grpc.CallOption) (*datapb.ConfigureDatabaseUserResponse, error) { - test.That(t, in.OrganizationId, test.ShouldResemble, expectedOrgId) + grpcClient.ConfigureDatabaseUserFunc = func(ctx context.Context, in *datapb.ConfigureDatabaseUserRequest, + opts ...grpc.CallOption, + ) (*datapb.ConfigureDatabaseUserResponse, error) { + test.That(t, in.OrganizationId, test.ShouldResemble, expectedOrgID) test.That(t, in.Password, test.ShouldResemble, expectedPassword) return &datapb.ConfigureDatabaseUserResponse{}, nil } - client.ConfigureDatabaseUser(context.Background(), organizationId, password) + client.ConfigureDatabaseUser(context.Background(), organizationID, password) }) t.Run("AddBinaryDataToDatasetByIDs", func(t *testing.T) { - expectedBinaryIds := BinaryIdsToProto(binaryIds) - expectedDataSetId := datasetId - - grpcClient.AddBinaryDataToDatasetByIDsFunc = func(ctx context.Context, in *datapb.AddBinaryDataToDatasetByIDsRequest, opts ...grpc.CallOption) (*datapb.AddBinaryDataToDatasetByIDsResponse, error) { - test.That(t, in.BinaryIds, test.ShouldResemble, expectedBinaryIds) - test.That(t, in.DatasetId, test.ShouldResemble, expectedDataSetId) + expectedBinaryIDs := binaryIDsToProto(binaryIDs) + expectedDataSetID := datasetID + + grpcClient.AddBinaryDataToDatasetByIDsFunc = func(ctx context.Context, in *datapb.AddBinaryDataToDatasetByIDsRequest, + opts ...grpc.CallOption, + ) (*datapb.AddBinaryDataToDatasetByIDsResponse, error) { + test.That(t, in.BinaryIds, test.ShouldResemble, expectedBinaryIDs) + test.That(t, in.DatasetId, test.ShouldResemble, expectedDataSetID) return &datapb.AddBinaryDataToDatasetByIDsResponse{}, nil } - client.AddBinaryDataToDatasetByIDs(context.Background(), binaryIds, datasetId) + client.AddBinaryDataToDatasetByIDs(context.Background(), binaryIDs, datasetID) }) t.Run("RemoveBinaryDataFromDatasetByIDs", func(t *testing.T) { - expectedBinaryIds := BinaryIdsToProto(binaryIds) - expectedDataSetId := datasetId - - grpcClient.RemoveBinaryDataFromDatasetByIDsFunc = func(ctx context.Context, in *datapb.RemoveBinaryDataFromDatasetByIDsRequest, opts ...grpc.CallOption) (*datapb.RemoveBinaryDataFromDatasetByIDsResponse, error) { - test.That(t, in.BinaryIds, test.ShouldResemble, expectedBinaryIds) - test.That(t, in.DatasetId, test.ShouldResemble, expectedDataSetId) + expectedBinaryIDs := binaryIDsToProto(binaryIDs) + expectedDataSetID := datasetID + + grpcClient.RemoveBinaryDataFromDatasetByIDsFunc = func(ctx context.Context, in *datapb.RemoveBinaryDataFromDatasetByIDsRequest, + opts ...grpc.CallOption, + ) (*datapb.RemoveBinaryDataFromDatasetByIDsResponse, error) { + test.That(t, in.BinaryIds, test.ShouldResemble, expectedBinaryIDs) + test.That(t, in.DatasetId, test.ShouldResemble, expectedDataSetID) return &datapb.RemoveBinaryDataFromDatasetByIDsResponse{}, nil } - client.RemoveBinaryDataFromDatasetByIDs(context.Background(), binaryIds, datasetId) + client.RemoveBinaryDataFromDatasetByIDs(context.Background(), binaryIDs, datasetID) }) } diff --git a/go.sum b/go.sum index 2b1bd15d364..d761c2480e5 100644 --- a/go.sum +++ b/go.sum @@ -596,7 +596,6 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4= @@ -839,7 +838,6 @@ github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYs github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.0/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e h1:+lIPJOWl+jSiJOc70QXJ07+2eg2Jy2EC7Mi11BWujeM= @@ -997,9 +995,9 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= -github.com/montanaflynn/stats v0.7.0 h1:r3y12KyNxj/Sb/iOE46ws+3mS1+MZca1wlHQFPsY/JU= -github.com/montanaflynn/stats v0.7.0/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= +github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= +github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/moricho/tparallel v0.2.1/go.mod h1:fXEIZxG2vdfl0ZF8b42f5a78EhjjD5mX8qUplsoSU4k= github.com/moricho/tparallel v0.3.2 h1:odr8aZVFA3NZrNybggMkYO3rgPRcqjeQUlBBFVxKHTI= github.com/moricho/tparallel v0.3.2/go.mod h1:OQ+K3b4Ln3l2TZveGCywybl68glfLEwFGqvnjok8b+U= @@ -1448,6 +1446,11 @@ github.com/ykadowak/zerologlint v0.1.5/go.mod h1:KaUskqF3e/v59oPmdq1U1DnKcuHokl2 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a h1:fZHgsYlfvtyqToslyjUt3VOPF4J7aK/3MPcK7xp3PDk= github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a/go.mod h1:ul22v+Nro/R083muKhosV54bj5niojjWZvU8xrevuH4= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= +github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= +github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= +github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -1472,8 +1475,10 @@ go-simpler.org/sloglint v0.7.2/go.mod h1:US+9C80ppl7VsThQclkM7BkCHQAzuz8kHLsW3pp go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= -go.mongodb.org/mongo-driver v1.12.2 h1:gbWY1bJkkmUB9jjZzcdhOL8O85N9H+Vvsf2yFN0RDws= -go.mongodb.org/mongo-driver v1.12.2/go.mod h1:/rGBTebI3XYboVmgz+Wv3Bcbl3aD0QF9zl6kDDw18rQ= +go.etcd.io/etcd v0.0.0-20200513171258-e048e166ab9c/go.mod h1:xCI7ZzBfRuGgBXyXO6yfWfDmlWd35khcWpUa4L0xI/k= +go.mongodb.org/mongo-driver v1.17.1 h1:Wic5cJIwJgSpBhe3lx3+/RybR5PiYRMpVFgO7cOHyIM= +go.mongodb.org/mongo-driver v1.17.1/go.mod h1:wwWm/+BuOddhcq3n68LKRmgk2wXzmF6s0SFOa0GINL4= +go.mozilla.org/mozlog v0.0.0-20170222151521-4bb13139d403/go.mod h1:jHoPAGnDrCy6kaI2tAze5Prf0Nr0w/oNkROt2lw3n3o= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= @@ -1542,10 +1547,8 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= @@ -1660,7 +1663,6 @@ golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6/go.mod h1:OJAsFXCWl8Ukc7SiCT golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= diff --git a/testutils/inject/data_service_client.go b/testutils/inject/data_service_client.go index a83e510b089..f1c21f5ba19 100644 --- a/testutils/inject/data_service_client.go +++ b/testutils/inject/data_service_client.go @@ -10,31 +10,53 @@ import ( // DataServiceClient represents a fake instance of a data service client. type DataServiceClient struct { datapb.DataServiceClient - TabularDataByFilterFunc func(ctx context.Context, in *datapb.TabularDataByFilterRequest, opts ...grpc.CallOption) (*datapb.TabularDataByFilterResponse, error) - TabularDataBySQLFunc func(ctx context.Context, in *datapb.TabularDataBySQLRequest, opts ...grpc.CallOption) (*datapb.TabularDataBySQLResponse, error) - TabularDataByMQLFunc func(ctx context.Context, in *datapb.TabularDataByMQLRequest, opts ...grpc.CallOption) (*datapb.TabularDataByMQLResponse, error) - BinaryDataByFilterFunc func(ctx context.Context, in *datapb.BinaryDataByFilterRequest, opts ...grpc.CallOption) (*datapb.BinaryDataByFilterResponse, error) - BinaryDataByIDsFunc func(ctx context.Context, in *datapb.BinaryDataByIDsRequest, opts ...grpc.CallOption) (*datapb.BinaryDataByIDsResponse, error) - DeleteTabularDataFunc func(ctx context.Context, in *datapb.DeleteTabularDataRequest, opts ...grpc.CallOption) (*datapb.DeleteTabularDataResponse, error) - DeleteBinaryDataByFilterFunc func(ctx context.Context, in *datapb.DeleteBinaryDataByFilterRequest, opts ...grpc.CallOption) (*datapb.DeleteBinaryDataByFilterResponse, error) - DeleteBinaryDataByIDsFunc func(ctx context.Context, in *datapb.DeleteBinaryDataByIDsRequest, opts ...grpc.CallOption) (*datapb.DeleteBinaryDataByIDsResponse, error) - AddTagsToBinaryDataByIDsFunc func(ctx context.Context, in *datapb.AddTagsToBinaryDataByIDsRequest, opts ...grpc.CallOption) (*datapb.AddTagsToBinaryDataByIDsResponse, error) - AddTagsToBinaryDataByFilterFunc func(ctx context.Context, in *datapb.AddTagsToBinaryDataByFilterRequest, opts ...grpc.CallOption) (*datapb.AddTagsToBinaryDataByFilterResponse, error) - RemoveTagsFromBinaryDataByIDsFunc func(ctx context.Context, in *datapb.RemoveTagsFromBinaryDataByIDsRequest, opts ...grpc.CallOption) (*datapb.RemoveTagsFromBinaryDataByIDsResponse, error) - RemoveTagsFromBinaryDataByFilterFunc func(ctx context.Context, in *datapb.RemoveTagsFromBinaryDataByFilterRequest, opts ...grpc.CallOption) (*datapb.RemoveTagsFromBinaryDataByFilterResponse, error) - TagsByFilterFunc func(ctx context.Context, in *datapb.TagsByFilterRequest, opts ...grpc.CallOption) (*datapb.TagsByFilterResponse, error) - AddBoundingBoxToImageByIDFunc func(ctx context.Context, in *datapb.AddBoundingBoxToImageByIDRequest, opts ...grpc.CallOption) (*datapb.AddBoundingBoxToImageByIDResponse, error) - RemoveBoundingBoxFromImageByIDFunc func(ctx context.Context, in *datapb.RemoveBoundingBoxFromImageByIDRequest, opts ...grpc.CallOption) (*datapb.RemoveBoundingBoxFromImageByIDResponse, error) - BoundingBoxLabelsByFilterFunc func(ctx context.Context, in *datapb.BoundingBoxLabelsByFilterRequest, opts ...grpc.CallOption) (*datapb.BoundingBoxLabelsByFilterResponse, error) - UpdateBoundingBoxFunc func(ctx context.Context, in *datapb.UpdateBoundingBoxRequest, opts ...grpc.CallOption) (*datapb.UpdateBoundingBoxResponse, error) - GetDatabaseConnectionFunc func(ctx context.Context, in *datapb.GetDatabaseConnectionRequest, opts ...grpc.CallOption) (*datapb.GetDatabaseConnectionResponse, error) - ConfigureDatabaseUserFunc func(ctx context.Context, in *datapb.ConfigureDatabaseUserRequest, opts ...grpc.CallOption) (*datapb.ConfigureDatabaseUserResponse, error) - AddBinaryDataToDatasetByIDsFunc func(ctx context.Context, in *datapb.AddBinaryDataToDatasetByIDsRequest, opts ...grpc.CallOption) (*datapb.AddBinaryDataToDatasetByIDsResponse, error) - RemoveBinaryDataFromDatasetByIDsFunc func(ctx context.Context, in *datapb.RemoveBinaryDataFromDatasetByIDsRequest, opts ...grpc.CallOption) (*datapb.RemoveBinaryDataFromDatasetByIDsResponse, error) + TabularDataByFilterFunc func(ctx context.Context, in *datapb.TabularDataByFilterRequest, + opts ...grpc.CallOption) (*datapb.TabularDataByFilterResponse, error) + TabularDataBySQLFunc func(ctx context.Context, in *datapb.TabularDataBySQLRequest, + opts ...grpc.CallOption) (*datapb.TabularDataBySQLResponse, error) + TabularDataByMQLFunc func(ctx context.Context, in *datapb.TabularDataByMQLRequest, + opts ...grpc.CallOption) (*datapb.TabularDataByMQLResponse, error) + BinaryDataByFilterFunc func(ctx context.Context, in *datapb.BinaryDataByFilterRequest, + opts ...grpc.CallOption) (*datapb.BinaryDataByFilterResponse, error) + BinaryDataByIDsFunc func(ctx context.Context, in *datapb.BinaryDataByIDsRequest, + opts ...grpc.CallOption) (*datapb.BinaryDataByIDsResponse, error) + DeleteTabularDataFunc func(ctx context.Context, in *datapb.DeleteTabularDataRequest, + opts ...grpc.CallOption) (*datapb.DeleteTabularDataResponse, error) + DeleteBinaryDataByFilterFunc func(ctx context.Context, in *datapb.DeleteBinaryDataByFilterRequest, + opts ...grpc.CallOption) (*datapb.DeleteBinaryDataByFilterResponse, error) + DeleteBinaryDataByIDsFunc func(ctx context.Context, in *datapb.DeleteBinaryDataByIDsRequest, + opts ...grpc.CallOption) (*datapb.DeleteBinaryDataByIDsResponse, error) + AddTagsToBinaryDataByIDsFunc func(ctx context.Context, in *datapb.AddTagsToBinaryDataByIDsRequest, + opts ...grpc.CallOption) (*datapb.AddTagsToBinaryDataByIDsResponse, error) + AddTagsToBinaryDataByFilterFunc func(ctx context.Context, in *datapb.AddTagsToBinaryDataByFilterRequest, + opts ...grpc.CallOption) (*datapb.AddTagsToBinaryDataByFilterResponse, error) + RemoveTagsFromBinaryDataByIDsFunc func(ctx context.Context, in *datapb.RemoveTagsFromBinaryDataByIDsRequest, + opts ...grpc.CallOption) (*datapb.RemoveTagsFromBinaryDataByIDsResponse, error) + RemoveTagsFromBinaryDataByFilterFunc func(ctx context.Context, in *datapb.RemoveTagsFromBinaryDataByFilterRequest, + opts ...grpc.CallOption) (*datapb.RemoveTagsFromBinaryDataByFilterResponse, error) + TagsByFilterFunc func(ctx context.Context, in *datapb.TagsByFilterRequest, + opts ...grpc.CallOption) (*datapb.TagsByFilterResponse, error) + AddBoundingBoxToImageByIDFunc func(ctx context.Context, in *datapb.AddBoundingBoxToImageByIDRequest, + opts ...grpc.CallOption) (*datapb.AddBoundingBoxToImageByIDResponse, error) + RemoveBoundingBoxFromImageByIDFunc func(ctx context.Context, in *datapb.RemoveBoundingBoxFromImageByIDRequest, + opts ...grpc.CallOption) (*datapb.RemoveBoundingBoxFromImageByIDResponse, error) + BoundingBoxLabelsByFilterFunc func(ctx context.Context, in *datapb.BoundingBoxLabelsByFilterRequest, + opts ...grpc.CallOption) (*datapb.BoundingBoxLabelsByFilterResponse, error) + UpdateBoundingBoxFunc func(ctx context.Context, in *datapb.UpdateBoundingBoxRequest, + opts ...grpc.CallOption) (*datapb.UpdateBoundingBoxResponse, error) + GetDatabaseConnectionFunc func(ctx context.Context, in *datapb.GetDatabaseConnectionRequest, + opts ...grpc.CallOption) (*datapb.GetDatabaseConnectionResponse, error) + ConfigureDatabaseUserFunc func(ctx context.Context, in *datapb.ConfigureDatabaseUserRequest, + opts ...grpc.CallOption) (*datapb.ConfigureDatabaseUserResponse, error) + AddBinaryDataToDatasetByIDsFunc func(ctx context.Context, in *datapb.AddBinaryDataToDatasetByIDsRequest, + opts ...grpc.CallOption) (*datapb.AddBinaryDataToDatasetByIDsResponse, error) + RemoveBinaryDataFromDatasetByIDsFunc func(ctx context.Context, in *datapb.RemoveBinaryDataFromDatasetByIDsRequest, + opts ...grpc.CallOption) (*datapb.RemoveBinaryDataFromDatasetByIDsResponse, error) } // TabularDataByFilter calls the injected TabularDataByFilter or the real version. -func (client *DataServiceClient) TabularDataByFilter(ctx context.Context, in *datapb.TabularDataByFilterRequest, opts ...grpc.CallOption, +func (client *DataServiceClient) TabularDataByFilter(ctx context.Context, in *datapb.TabularDataByFilterRequest, + opts ...grpc.CallOption, ) (*datapb.TabularDataByFilterResponse, error) { if client.TabularDataByFilterFunc == nil { return client.DataServiceClient.TabularDataByFilter(ctx, in, opts...) @@ -42,140 +64,202 @@ func (client *DataServiceClient) TabularDataByFilter(ctx context.Context, in *da return client.TabularDataByFilterFunc(ctx, in, opts...) } -func (client *DataServiceClient) TabularDataBySQL(ctx context.Context, in *datapb.TabularDataBySQLRequest, opts ...grpc.CallOption) (*datapb.TabularDataBySQLResponse, error) { +// TabularDataBySQL calls the injected TabularDataBySQL or the real version. +func (client *DataServiceClient) TabularDataBySQL(ctx context.Context, in *datapb.TabularDataBySQLRequest, + opts ...grpc.CallOption, +) (*datapb.TabularDataBySQLResponse, error) { if client.TabularDataBySQLFunc == nil { return client.DataServiceClient.TabularDataBySQL(ctx, in, opts...) } return client.TabularDataBySQLFunc(ctx, in, opts...) } -func (client *DataServiceClient) TabularDataByMQL(ctx context.Context, in *datapb.TabularDataByMQLRequest, opts ...grpc.CallOption) (*datapb.TabularDataByMQLResponse, error) { +// TabularDataByMQL calls the injected TabularDataByMQL or the real version. +func (client *DataServiceClient) TabularDataByMQL(ctx context.Context, in *datapb.TabularDataByMQLRequest, + opts ...grpc.CallOption, +) (*datapb.TabularDataByMQLResponse, error) { if client.TabularDataByMQLFunc == nil { return client.DataServiceClient.TabularDataByMQL(ctx, in, opts...) } return client.TabularDataByMQLFunc(ctx, in, opts...) } -func (client *DataServiceClient) BinaryDataByFilter(ctx context.Context, in *datapb.BinaryDataByFilterRequest, opts ...grpc.CallOption) (*datapb.BinaryDataByFilterResponse, error) { +// BinaryDataByFilter calls the injected BinaryDataByFilter or the real version. +func (client *DataServiceClient) BinaryDataByFilter(ctx context.Context, in *datapb.BinaryDataByFilterRequest, + opts ...grpc.CallOption, +) (*datapb.BinaryDataByFilterResponse, error) { if client.BinaryDataByFilterFunc == nil { return client.DataServiceClient.BinaryDataByFilter(ctx, in, opts...) } return client.BinaryDataByFilterFunc(ctx, in, opts...) } -func (client *DataServiceClient) BinaryDataByIDs(ctx context.Context, in *datapb.BinaryDataByIDsRequest, opts ...grpc.CallOption) (*datapb.BinaryDataByIDsResponse, error) { +// BinaryDataByIDs calls the injected BinaryDataByIDs or the real version. +func (client *DataServiceClient) BinaryDataByIDs(ctx context.Context, in *datapb.BinaryDataByIDsRequest, + opts ...grpc.CallOption, +) (*datapb.BinaryDataByIDsResponse, error) { if client.BinaryDataByIDsFunc == nil { return client.DataServiceClient.BinaryDataByIDs(ctx, in, opts...) } return client.BinaryDataByIDsFunc(ctx, in, opts...) } -func (client *DataServiceClient) DeleteTabularData(ctx context.Context, in *datapb.DeleteTabularDataRequest, opts ...grpc.CallOption) (*datapb.DeleteTabularDataResponse, error) { +// DeleteTabularData calls the injected DeleteTabularData or the real version. +func (client *DataServiceClient) DeleteTabularData(ctx context.Context, in *datapb.DeleteTabularDataRequest, + opts ...grpc.CallOption, +) (*datapb.DeleteTabularDataResponse, error) { if client.DeleteTabularDataFunc == nil { return client.DataServiceClient.DeleteTabularData(ctx, in, opts...) } return client.DeleteTabularDataFunc(ctx, in, opts...) } -func (client *DataServiceClient) DeleteBinaryDataByFilter(ctx context.Context, in *datapb.DeleteBinaryDataByFilterRequest, opts ...grpc.CallOption) (*datapb.DeleteBinaryDataByFilterResponse, error) { +// DeleteBinaryDataByFilter calls the injected DeleteBinaryDataByFilter or the real version. +func (client *DataServiceClient) DeleteBinaryDataByFilter(ctx context.Context, in *datapb.DeleteBinaryDataByFilterRequest, + opts ...grpc.CallOption, +) (*datapb.DeleteBinaryDataByFilterResponse, error) { if client.DeleteBinaryDataByFilterFunc == nil { return client.DataServiceClient.DeleteBinaryDataByFilter(ctx, in, opts...) } return client.DeleteBinaryDataByFilterFunc(ctx, in, opts...) } -func (client *DataServiceClient) DeleteBinaryDataByIDs(ctx context.Context, in *datapb.DeleteBinaryDataByIDsRequest, opts ...grpc.CallOption) (*datapb.DeleteBinaryDataByIDsResponse, error) { +// DeleteBinaryDataByIDs calls the injected DeleteBinaryDataByIDs or the real version. +func (client *DataServiceClient) DeleteBinaryDataByIDs(ctx context.Context, in *datapb.DeleteBinaryDataByIDsRequest, + opts ...grpc.CallOption, +) (*datapb.DeleteBinaryDataByIDsResponse, error) { if client.DeleteBinaryDataByIDsFunc == nil { return client.DataServiceClient.DeleteBinaryDataByIDs(ctx, in, opts...) } return client.DeleteBinaryDataByIDsFunc(ctx, in, opts...) } -func (client *DataServiceClient) AddTagsToBinaryDataByIDs(ctx context.Context, in *datapb.AddTagsToBinaryDataByIDsRequest, opts ...grpc.CallOption) (*datapb.AddTagsToBinaryDataByIDsResponse, error) { +// AddTagsToBinaryDataByIDs calls the injected AddTagsToBinaryDataByIDs or the real version. +func (client *DataServiceClient) AddTagsToBinaryDataByIDs(ctx context.Context, in *datapb.AddTagsToBinaryDataByIDsRequest, + opts ...grpc.CallOption, +) (*datapb.AddTagsToBinaryDataByIDsResponse, error) { if client.AddTagsToBinaryDataByIDsFunc == nil { return client.DataServiceClient.AddTagsToBinaryDataByIDs(ctx, in, opts...) } return client.AddTagsToBinaryDataByIDsFunc(ctx, in, opts...) } -func (client *DataServiceClient) AddTagsToBinaryDataByFilter(ctx context.Context, in *datapb.AddTagsToBinaryDataByFilterRequest, opts ...grpc.CallOption) (*datapb.AddTagsToBinaryDataByFilterResponse, error) { +// AddTagsToBinaryDataByFilter calls the injected AddTagsToBinaryDataByFilter or the real version. +func (client *DataServiceClient) AddTagsToBinaryDataByFilter(ctx context.Context, in *datapb.AddTagsToBinaryDataByFilterRequest, + opts ...grpc.CallOption, +) (*datapb.AddTagsToBinaryDataByFilterResponse, error) { if client.AddTagsToBinaryDataByFilterFunc == nil { return client.DataServiceClient.AddTagsToBinaryDataByFilter(ctx, in, opts...) } return client.AddTagsToBinaryDataByFilterFunc(ctx, in, opts...) } -func (client *DataServiceClient) RemoveTagsFromBinaryDataByIDs(ctx context.Context, in *datapb.RemoveTagsFromBinaryDataByIDsRequest, opts ...grpc.CallOption) (*datapb.RemoveTagsFromBinaryDataByIDsResponse, error) { +// RemoveTagsFromBinaryDataByIDs calls the injected RemoveTagsFromBinaryDataByIDs or the real version. +func (client *DataServiceClient) RemoveTagsFromBinaryDataByIDs(ctx context.Context, + in *datapb.RemoveTagsFromBinaryDataByIDsRequest, + opts ...grpc.CallOption, +) (*datapb.RemoveTagsFromBinaryDataByIDsResponse, error) { if client.RemoveTagsFromBinaryDataByIDsFunc == nil { return client.DataServiceClient.RemoveTagsFromBinaryDataByIDs(ctx, in, opts...) } return client.RemoveTagsFromBinaryDataByIDsFunc(ctx, in, opts...) } -func (client *DataServiceClient) RemoveTagsFromBinaryDataByFilter(ctx context.Context, in *datapb.RemoveTagsFromBinaryDataByFilterRequest, opts ...grpc.CallOption) (*datapb.RemoveTagsFromBinaryDataByFilterResponse, error) { +// RemoveTagsFromBinaryDataByFilter calls the injected RemoveTagsFromBinaryDataByFilter or the real version. +func (client *DataServiceClient) RemoveTagsFromBinaryDataByFilter(ctx context.Context, + in *datapb.RemoveTagsFromBinaryDataByFilterRequest, + opts ...grpc.CallOption, +) (*datapb.RemoveTagsFromBinaryDataByFilterResponse, error) { if client.RemoveTagsFromBinaryDataByFilterFunc == nil { return client.DataServiceClient.RemoveTagsFromBinaryDataByFilter(ctx, in, opts...) } return client.RemoveTagsFromBinaryDataByFilterFunc(ctx, in, opts...) } -func (client *DataServiceClient) TagsByFilter(ctx context.Context, in *datapb.TagsByFilterRequest, opts ...grpc.CallOption) (*datapb.TagsByFilterResponse, error) { +// TagsByFilter calls the injected TagsByFilter or the real version. +func (client *DataServiceClient) TagsByFilter(ctx context.Context, in *datapb.TagsByFilterRequest, + opts ...grpc.CallOption, +) (*datapb.TagsByFilterResponse, error) { if client.TagsByFilterFunc == nil { return client.DataServiceClient.TagsByFilter(ctx, in, opts...) } return client.TagsByFilterFunc(ctx, in, opts...) } -func (client *DataServiceClient) AddBoundingBoxToImageByID(ctx context.Context, in *datapb.AddBoundingBoxToImageByIDRequest, opts ...grpc.CallOption) (*datapb.AddBoundingBoxToImageByIDResponse, error) { +// AddBoundingBoxToImageByID calls the injected AddBoundingBoxToImageByID or the real version. +func (client *DataServiceClient) AddBoundingBoxToImageByID(ctx context.Context, in *datapb.AddBoundingBoxToImageByIDRequest, + opts ...grpc.CallOption, +) (*datapb.AddBoundingBoxToImageByIDResponse, error) { if client.AddBoundingBoxToImageByIDFunc == nil { return client.DataServiceClient.AddBoundingBoxToImageByID(ctx, in, opts...) } return client.AddBoundingBoxToImageByIDFunc(ctx, in, opts...) } -func (client *DataServiceClient) RemoveBoundingBoxFromImageByID(ctx context.Context, in *datapb.RemoveBoundingBoxFromImageByIDRequest, opts ...grpc.CallOption) (*datapb.RemoveBoundingBoxFromImageByIDResponse, error) { +// RemoveBoundingBoxFromImageByID calls the injected RemoveBoundingBoxFromImageByID or the real version. +func (client *DataServiceClient) RemoveBoundingBoxFromImageByID(ctx context.Context, + in *datapb.RemoveBoundingBoxFromImageByIDRequest, opts ...grpc.CallOption, +) (*datapb.RemoveBoundingBoxFromImageByIDResponse, error) { if client.RemoveBoundingBoxFromImageByIDFunc == nil { return client.DataServiceClient.RemoveBoundingBoxFromImageByID(ctx, in, opts...) } return client.RemoveBoundingBoxFromImageByIDFunc(ctx, in, opts...) } -func (client *DataServiceClient) BoundingBoxLabelsByFilter(ctx context.Context, in *datapb.BoundingBoxLabelsByFilterRequest, opts ...grpc.CallOption) (*datapb.BoundingBoxLabelsByFilterResponse, error) { +// BoundingBoxLabelsByFilter calls the injected BoundingBoxLabelsByFilter or the real version. +func (client *DataServiceClient) BoundingBoxLabelsByFilter(ctx context.Context, in *datapb.BoundingBoxLabelsByFilterRequest, + opts ...grpc.CallOption, +) (*datapb.BoundingBoxLabelsByFilterResponse, error) { if client.BoundingBoxLabelsByFilterFunc == nil { return client.DataServiceClient.BoundingBoxLabelsByFilter(ctx, in, opts...) } return client.BoundingBoxLabelsByFilterFunc(ctx, in, opts...) } -func (client *DataServiceClient) UpdateBoundingBox(ctx context.Context, in *datapb.UpdateBoundingBoxRequest, opts ...grpc.CallOption) (*datapb.UpdateBoundingBoxResponse, error) { +// UpdateBoundingBox calls the injected UpdateBoundingBox or the real version. +func (client *DataServiceClient) UpdateBoundingBox(ctx context.Context, in *datapb.UpdateBoundingBoxRequest, + opts ...grpc.CallOption, +) (*datapb.UpdateBoundingBoxResponse, error) { if client.UpdateBoundingBoxFunc == nil { return client.DataServiceClient.UpdateBoundingBox(ctx, in, opts...) } return client.UpdateBoundingBoxFunc(ctx, in, opts...) } -func (client *DataServiceClient) GetDatabaseConnection(ctx context.Context, in *datapb.GetDatabaseConnectionRequest, opts ...grpc.CallOption) (*datapb.GetDatabaseConnectionResponse, error) { +// GetDatabaseConnection calls the injected GetDatabaseConnection or the real version. +func (client *DataServiceClient) GetDatabaseConnection(ctx context.Context, in *datapb.GetDatabaseConnectionRequest, + opts ...grpc.CallOption, +) (*datapb.GetDatabaseConnectionResponse, error) { if client.GetDatabaseConnectionFunc == nil { return client.DataServiceClient.GetDatabaseConnection(ctx, in, opts...) } return client.GetDatabaseConnectionFunc(ctx, in, opts...) } -func (client *DataServiceClient) ConfigureDatabaseUser(ctx context.Context, in *datapb.ConfigureDatabaseUserRequest, opts ...grpc.CallOption) (*datapb.ConfigureDatabaseUserResponse, error) { +// ConfigureDatabaseUser calls the injected ConfigureDatabaseUser or the real version. +func (client *DataServiceClient) ConfigureDatabaseUser(ctx context.Context, in *datapb.ConfigureDatabaseUserRequest, + opts ...grpc.CallOption, +) (*datapb.ConfigureDatabaseUserResponse, error) { if client.ConfigureDatabaseUserFunc == nil { return client.DataServiceClient.ConfigureDatabaseUser(ctx, in, opts...) } return client.ConfigureDatabaseUserFunc(ctx, in, opts...) } -func (client *DataServiceClient) AddBinaryDataToDatasetByIDs(ctx context.Context, in *datapb.AddBinaryDataToDatasetByIDsRequest, opts ...grpc.CallOption) (*datapb.AddBinaryDataToDatasetByIDsResponse, error) { +// AddBinaryDataToDatasetByIDs calls the injected AddBinaryDataToDatasetByIDs or the real version. +func (client *DataServiceClient) AddBinaryDataToDatasetByIDs(ctx context.Context, in *datapb.AddBinaryDataToDatasetByIDsRequest, + opts ...grpc.CallOption, +) (*datapb.AddBinaryDataToDatasetByIDsResponse, error) { if client.AddBinaryDataToDatasetByIDsFunc == nil { return client.DataServiceClient.AddBinaryDataToDatasetByIDs(ctx, in, opts...) } return client.AddBinaryDataToDatasetByIDsFunc(ctx, in, opts...) } -func (client *DataServiceClient) RemoveBinaryDataFromDatasetByIDs(ctx context.Context, in *datapb.RemoveBinaryDataFromDatasetByIDsRequest, opts ...grpc.CallOption) (*datapb.RemoveBinaryDataFromDatasetByIDsResponse, error) { +// RemoveBinaryDataFromDatasetByIDs calls the injected RemoveBinaryDataFromDatasetByIDs or the real version. +func (client *DataServiceClient) RemoveBinaryDataFromDatasetByIDs(ctx context.Context, in *datapb.RemoveBinaryDataFromDatasetByIDsRequest, + opts ...grpc.CallOption, +) (*datapb.RemoveBinaryDataFromDatasetByIDsResponse, error) { if client.RemoveBinaryDataFromDatasetByIDsFunc == nil { return client.DataServiceClient.RemoveBinaryDataFromDatasetByIDs(ctx, in, opts...) } From 9da952280fa656bd77bd422aad63de2760b12311 Mon Sep 17 00:00:00 2001 From: Julie Krasnick Date: Wed, 13 Nov 2024 16:50:07 -0500 Subject: [PATCH 21/25] fix comments --- app/data_client.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/app/data_client.go b/app/data_client.go index e8f46bb0906..990350f81c6 100644 --- a/app/data_client.go +++ b/app/data_client.go @@ -720,8 +720,7 @@ func (d *Client) UpdateBoundingBox(ctx context.Context, return nil } -// GetDatabaseConnection gets a connection to access a MongoDB Atlas Data Federation instance. It -// returns the hostname of the federated database. +// GetDatabaseConnection gets a connection to access a MongoDB Atlas Data Federation instance. func (d *Client) GetDatabaseConnection(ctx context.Context, organizationID string) (string, error) { resp, err := d.client.GetDatabaseConnection(ctx, &pb.GetDatabaseConnectionRequest{ OrganizationId: organizationID, @@ -732,8 +731,7 @@ func (d *Client) GetDatabaseConnection(ctx context.Context, organizationID strin return resp.Hostname, nil } -// ConfigureDatabaseUser configures a database user for the Viam organization's MongoDB Atlas Data -// Federation instance. It can also be used to reset the password of the existing database user. +// ConfigureDatabaseUser configures a database user for the Viam organization's MongoDB Atlas Data Federation instance. func (d *Client) ConfigureDatabaseUser( ctx context.Context, organizationID string, From d79cb15faf66b028cca6cab9a28217912e460f9f Mon Sep 17 00:00:00 2001 From: Julie Krasnick Date: Wed, 13 Nov 2024 16:53:21 -0500 Subject: [PATCH 22/25] ignore linter --- app/data_client.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/data_client.go b/app/data_client.go index 990350f81c6..7b9be35236d 100644 --- a/app/data_client.go +++ b/app/data_client.go @@ -33,8 +33,10 @@ const ( Ascending Order = 2 ) -// DataRequest encapsulates the filter for the data, a limit on the maximum results returned, +// DataRequest encapsulates the filter for the data, a limit on the max results returned, // a last string associated with the last returned document, and the sorting order by time. + +//nolint:revive // stutter: Ignore the "stuttering" warning for this type name type DataRequest struct { Filter Filter Limit uint64 From 1e5ef0f66058f89057a0c097b4c18c77804f6cc6 Mon Sep 17 00:00:00 2001 From: Julie Krasnick Date: Wed, 13 Nov 2024 17:04:45 -0500 Subject: [PATCH 23/25] changes in go.sum --- go.sum | 9 --------- 1 file changed, 9 deletions(-) diff --git a/go.sum b/go.sum index d761c2480e5..584a73eb7a7 100644 --- a/go.sum +++ b/go.sum @@ -995,7 +995,6 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/moricho/tparallel v0.2.1/go.mod h1:fXEIZxG2vdfl0ZF8b42f5a78EhjjD5mX8qUplsoSU4k= @@ -1443,14 +1442,8 @@ github.com/yeya24/promlinter v0.3.0 h1:JVDbMp08lVCP7Y6NP3qHroGAO6z2yGKQtS5Jsjqto github.com/yeya24/promlinter v0.3.0/go.mod h1:cDfJQQYv9uYciW60QT0eeHlFodotkYZlL+YcPQN+mW4= github.com/ykadowak/zerologlint v0.1.5 h1:Gy/fMz1dFQN9JZTPjv1hxEk+sRWm05row04Yoolgdiw= github.com/ykadowak/zerologlint v0.1.5/go.mod h1:KaUskqF3e/v59oPmdq1U1DnKcuHokl2/K1U4pmIELKg= -github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= -github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a h1:fZHgsYlfvtyqToslyjUt3VOPF4J7aK/3MPcK7xp3PDk= -github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a/go.mod h1:ul22v+Nro/R083muKhosV54bj5niojjWZvU8xrevuH4= github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= -github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= -github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= -github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -1475,10 +1468,8 @@ go-simpler.org/sloglint v0.7.2/go.mod h1:US+9C80ppl7VsThQclkM7BkCHQAzuz8kHLsW3pp go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= -go.etcd.io/etcd v0.0.0-20200513171258-e048e166ab9c/go.mod h1:xCI7ZzBfRuGgBXyXO6yfWfDmlWd35khcWpUa4L0xI/k= go.mongodb.org/mongo-driver v1.17.1 h1:Wic5cJIwJgSpBhe3lx3+/RybR5PiYRMpVFgO7cOHyIM= go.mongodb.org/mongo-driver v1.17.1/go.mod h1:wwWm/+BuOddhcq3n68LKRmgk2wXzmF6s0SFOa0GINL4= -go.mozilla.org/mozlog v0.0.0-20170222151521-4bb13139d403/go.mod h1:jHoPAGnDrCy6kaI2tAze5Prf0Nr0w/oNkROt2lw3n3o= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= From 23f75174dba4d9d93d7f6e9cdae0cc916e8ff3a9 Mon Sep 17 00:00:00 2001 From: Julie Krasnick Date: Thu, 14 Nov 2024 11:40:21 -0500 Subject: [PATCH 24/25] remove unused context --- app/data_client.go | 1 - 1 file changed, 1 deletion(-) diff --git a/app/data_client.go b/app/data_client.go index 7b9be35236d..ae95193b0ef 100644 --- a/app/data_client.go +++ b/app/data_client.go @@ -154,7 +154,6 @@ type Annotations struct { // NewDataClient constructs a new DataClient using the connection passed in by the viamClient and the provided logger. func NewDataClient( - ctx context.Context, channel rpc.ClientConn, logger logging.Logger, ) (*Client, error) { From 67c2888e7449b9717cf30d882d495b6d8b0cd79f Mon Sep 17 00:00:00 2001 From: Julie Krasnick Date: Thu, 14 Nov 2024 14:23:44 -0500 Subject: [PATCH 25/25] nicoles changes --- app/data_client.go | 47 --- app/data_client_test.go | 579 +++++++++++++++++++++++++++++++++++++ app/data_test.go | 616 ---------------------------------------- 3 files changed, 579 insertions(+), 663 deletions(-) create mode 100644 app/data_client_test.go delete mode 100644 app/data_test.go diff --git a/app/data_client.go b/app/data_client.go index ae95193b0ef..828ee1956d9 100644 --- a/app/data_client.go +++ b/app/data_client.go @@ -188,23 +188,6 @@ func annotationsFromProto(proto *pb.Annotations) Annotations { } } -func annotationsToProto(annotations Annotations) *pb.Annotations { - var protoBboxes []*pb.BoundingBox - for _, bbox := range annotations.Bboxes { - protoBboxes = append(protoBboxes, &pb.BoundingBox{ - Id: bbox.ID, - Label: bbox.Label, - XMinNormalized: bbox.XMinNormalized, - YMinNormalized: bbox.YMinNormalized, - XMaxNormalized: bbox.XMaxNormalized, - YMaxNormalized: bbox.YMaxNormalized, - }) - } - return &pb.Annotations{ - Bboxes: protoBboxes, - } -} - func methodParamsFromProto(proto map[string]*anypb.Any) map[string]string { methodParameters := make(map[string]string) for key, value := range proto { @@ -265,13 +248,6 @@ func binaryDataFromProto(proto *pb.BinaryData) BinaryData { } } -func binaryDataToProto(binaryData BinaryData) *pb.BinaryData { - return &pb.BinaryData{ - Binary: binaryData.Binary, - Metadata: binaryMetadataToProto(binaryData.Metadata), - } -} - func binaryMetadataFromProto(proto *pb.BinaryMetadata) BinaryMetadata { return BinaryMetadata{ ID: proto.Id, @@ -286,20 +262,6 @@ func binaryMetadataFromProto(proto *pb.BinaryMetadata) BinaryMetadata { } } -func binaryMetadataToProto(binaryMetadata BinaryMetadata) *pb.BinaryMetadata { - return &pb.BinaryMetadata{ - Id: binaryMetadata.ID, - CaptureMetadata: captureMetadataToProto(binaryMetadata.CaptureMetadata), - TimeRequested: timestamppb.New(binaryMetadata.TimeRequested), - TimeReceived: timestamppb.New(binaryMetadata.TimeReceived), - FileName: binaryMetadata.FileName, - FileExt: binaryMetadata.FileExt, - Uri: binaryMetadata.URI, - Annotations: annotationsToProto(binaryMetadata.Annotations), - DatasetIds: binaryMetadata.DatasetIDs, - } -} - func tabularDataFromProto(proto *pb.TabularData, metadata *pb.CaptureMetadata) TabularData { return TabularData{ Data: proto.Data.AsMap(), @@ -310,15 +272,6 @@ func tabularDataFromProto(proto *pb.TabularData, metadata *pb.CaptureMetadata) T } } -func dataRequestToProto(dataRequest DataRequest) *pb.DataRequest { - return &pb.DataRequest{ - Filter: filterToProto(dataRequest.Filter), - Limit: dataRequest.Limit, - Last: dataRequest.Last, - SortOrder: orderToProto(dataRequest.SortOrder), - } -} - func binaryIDToProto(binaryID BinaryID) *pb.BinaryID { return &pb.BinaryID{ FileId: binaryID.FileID, diff --git a/app/data_client_test.go b/app/data_client_test.go new file mode 100644 index 00000000000..8b95a5edbec --- /dev/null +++ b/app/data_client_test.go @@ -0,0 +1,579 @@ +package data + +import ( + "context" + "testing" + "time" + + "go.mongodb.org/mongo-driver/bson" + pb "go.viam.com/api/app/data/v1" + "go.viam.com/test" + utils "go.viam.com/utils/protoutils" + "google.golang.org/grpc" + "google.golang.org/protobuf/types/known/timestamppb" + + "go.viam.com/rdk/testutils/inject" +) + +const ( + componentName = "component_name" + componentType = "component_type" + method = "method" + robotName = "robot_name" + robotID = "robot_id" + partName = "part_name" + partID = "part_id" + locationID = "location_id" + organizationID = "organization_id" + password = "password" + mimeType = "mime_type" + uri = "some.robot.uri" + bboxLabel = "bbox_label" + tag = "tag" + fileName = "file_name" + fileExt = "file_ext.ext" + datasetID = "dataset_id" + binaryMetaID = "binary_id" + mongodbURI = "mongo_uri" + hostName = "host_name" + last = "last" +) + +var ( + locationIDs = []string{locationID} + orgIDs = []string{organizationID} + mimeTypes = []string{mimeType} + bboxLabels = []string{bboxLabel} + methodParameters = map[string]string{} + tags = []string{tag} + startTime = time.Now().UTC().Round(time.Millisecond) + endTime = time.Now().UTC().Round(time.Millisecond) + count = uint64(5) + limit = uint64(5) + countOnly = true + includeInternalData = true + data = map[string]interface{}{ + "key": "value", + } + binaryID = BinaryID{ + FileID: "file1", + OrganizationID: organizationID, + LocationID: locationID, + } + binaryIDs = []BinaryID{binaryID} + binaryDataByte = []byte("BYTE") + sqlQuery = "SELECT * FROM readings WHERE organization_id='e76d1b3b-0468-4efd-bb7f-fb1d2b352fcb' LIMIT 1" + rawData = []map[string]any{ + { + "key1": startTime, + "key2": "2", + "key3": []any{1, 2, 3}, + "key4": map[string]any{ + "key4sub1": endTime, + }, + "key5": 4.05, + "key6": []any{true, false, true}, + "key7": []any{ + map[string]any{ + "nestedKey1": "simpleValue", + }, + map[string]any{ + "nestedKey2": startTime, + }, + }, + }, + } + datasetIDs = []string{datasetID} + annotations = Annotations{ + Bboxes: []BoundingBox{ + { + ID: "bbox1", + Label: "label1", + XMinNormalized: 0.1, + YMinNormalized: 0.2, + XMaxNormalized: 0.8, + YMaxNormalized: 0.9, + }, + { + ID: "bbox2", + Label: "label2", + XMinNormalized: 0.15, + YMinNormalized: 0.25, + XMaxNormalized: 0.75, + YMaxNormalized: 0.85, + }, + }, + } +) + +func annotationsToProto(annotations Annotations) *pb.Annotations { + var protoBboxes []*pb.BoundingBox + for _, bbox := range annotations.Bboxes { + protoBboxes = append(protoBboxes, &pb.BoundingBox{ + Id: bbox.ID, + Label: bbox.Label, + XMinNormalized: bbox.XMinNormalized, + YMinNormalized: bbox.YMinNormalized, + XMaxNormalized: bbox.XMaxNormalized, + YMaxNormalized: bbox.YMaxNormalized, + }) + } + return &pb.Annotations{ + Bboxes: protoBboxes, + } +} + +func binaryDataToProto(binaryData BinaryData) *pb.BinaryData { + return &pb.BinaryData{ + Binary: binaryData.Binary, + Metadata: binaryMetadataToProto(binaryData.Metadata), + } +} + +func binaryMetadataToProto(binaryMetadata BinaryMetadata) *pb.BinaryMetadata { + return &pb.BinaryMetadata{ + Id: binaryMetadata.ID, + CaptureMetadata: captureMetadataToProto(binaryMetadata.CaptureMetadata), + TimeRequested: timestamppb.New(binaryMetadata.TimeRequested), + TimeReceived: timestamppb.New(binaryMetadata.TimeReceived), + FileName: binaryMetadata.FileName, + FileExt: binaryMetadata.FileExt, + Uri: binaryMetadata.URI, + Annotations: annotationsToProto(binaryMetadata.Annotations), + DatasetIds: binaryMetadata.DatasetIDs, + } +} + +func dataRequestToProto(dataRequest DataRequest) *pb.DataRequest { + return &pb.DataRequest{ + Filter: filterToProto(dataRequest.Filter), + Limit: dataRequest.Limit, + Last: dataRequest.Last, + SortOrder: orderToProto(dataRequest.SortOrder), + } +} + +func createGrpcClient() *inject.DataServiceClient { + return &inject.DataServiceClient{} +} + +func TestDataClient(t *testing.T) { + grpcClient := createGrpcClient() + client := Client{client: grpcClient} + + captureInterval := CaptureInterval{ + Start: time.Now(), + End: time.Now(), + } + tagsFilter := TagsFilter{ + Type: TagsFilterTypeUnspecified, + Tags: []string{"tag1", "tag2"}, + } + + filter := Filter{ + ComponentName: componentName, + ComponentType: componentType, + Method: method, + RobotName: robotName, + RobotID: robotID, + PartName: partName, + PartID: partID, + LocationIDs: locationIDs, + OrganizationIDs: orgIDs, + MimeType: mimeTypes, + Interval: captureInterval, + TagsFilter: tagsFilter, + BboxLabels: bboxLabels, + DatasetID: datasetID, + } + + tabularMetadata := CaptureMetadata{ + OrganizationID: organizationID, + LocationID: locationID, + RobotName: robotName, + RobotID: robotID, + PartName: partName, + PartID: partID, + ComponentType: componentType, + ComponentName: componentName, + MethodName: method, + MethodParameters: methodParameters, + Tags: tags, + MimeType: mimeType, + } + + binaryMetadata := BinaryMetadata{ + ID: binaryMetaID, + CaptureMetadata: tabularMetadata, + TimeRequested: startTime, + TimeReceived: endTime, + FileName: fileName, + FileExt: fileExt, + URI: uri, + Annotations: annotations, + DatasetIDs: datasetIDs, + } + + dataRequest := DataRequest{ + Filter: filter, + Limit: count, + Last: last, + SortOrder: Unspecified, + } + + binaryData := BinaryData{ + Binary: binaryDataByte, + Metadata: binaryMetadata, + } + + t.Run("TabularDataByFilter", func(t *testing.T) { + tabularData := TabularData{ + Data: data, + MetadataIndex: 0, + Metadata: tabularMetadata, + TimeRequested: startTime, + TimeReceived: endTime, + } + dataStruct, _ := utils.StructToStructPb(data) + tabularDataPb := &pb.TabularData{ + Data: dataStruct, + MetadataIndex: 0, + TimeRequested: timestamppb.New(startTime), + TimeReceived: timestamppb.New(endTime), + } + grpcClient.TabularDataByFilterFunc = func(ctx context.Context, in *pb.TabularDataByFilterRequest, + opts ...grpc.CallOption, + ) (*pb.TabularDataByFilterResponse, error) { + test.That(t, in.DataRequest, test.ShouldResemble, dataRequestToProto(dataRequest)) + test.That(t, in.CountOnly, test.ShouldBeTrue) + test.That(t, in.IncludeInternalData, test.ShouldBeTrue) + return &pb.TabularDataByFilterResponse{ + Data: []*pb.TabularData{tabularDataPb}, + Count: count, + Last: last, + Metadata: []*pb.CaptureMetadata{captureMetadataToProto(tabularMetadata)}, + }, nil + } + respTabularData, respCount, respLast, _ := client.TabularDataByFilter( + context.Background(), filter, limit, last, + dataRequest.SortOrder, countOnly, includeInternalData) + test.That(t, respTabularData[0], test.ShouldResemble, tabularData) + test.That(t, respCount, test.ShouldEqual, count) + test.That(t, respLast, test.ShouldEqual, last) + }) + + t.Run("TabularDataBySQL", func(t *testing.T) { + // convert rawData to BSON + var expectedRawDataPb [][]byte + for _, byte := range rawData { + bsonByte, err := bson.Marshal(byte) + if err != nil { + t.Fatalf("Failed to marshal expectedRawData: %v", err) + } + expectedRawDataPb = append(expectedRawDataPb, bsonByte) + } + grpcClient.TabularDataBySQLFunc = func(ctx context.Context, in *pb.TabularDataBySQLRequest, + opts ...grpc.CallOption, + ) (*pb.TabularDataBySQLResponse, error) { + test.That(t, in.OrganizationId, test.ShouldEqual, organizationID) + test.That(t, in.SqlQuery, test.ShouldResemble, sqlQuery) + return &pb.TabularDataBySQLResponse{ + RawData: expectedRawDataPb, + }, nil + } + response, _ := client.TabularDataBySQL(context.Background(), organizationID, sqlQuery) + test.That(t, response, test.ShouldResemble, rawData) + }) + + t.Run("TabularDataByMQL", func(t *testing.T) { + // convert to BSON byte arrays + matchBytes, _ := bson.Marshal(bson.M{"$match": bson.M{"organization_id": "e76d1b3b-0468-4efd-bb7f-fb1d2b352fcb"}}) + limitBytes, _ := bson.Marshal(bson.M{"$limit": 1}) + mqlbinary := [][]byte{matchBytes, limitBytes} + + // convert rawData to BSON + var expectedRawDataPb [][]byte + for _, byte := range rawData { + bsonByte, err := bson.Marshal(byte) + if err != nil { + t.Fatalf("Failed to marshal expectedRawData: %v", err) + } + expectedRawDataPb = append(expectedRawDataPb, bsonByte) + } + grpcClient.TabularDataByMQLFunc = func(ctx context.Context, in *pb.TabularDataByMQLRequest, + opts ...grpc.CallOption, + ) (*pb.TabularDataByMQLResponse, error) { + test.That(t, in.OrganizationId, test.ShouldEqual, organizationID) + test.That(t, in.MqlBinary, test.ShouldResemble, mqlbinary) + return &pb.TabularDataByMQLResponse{ + RawData: expectedRawDataPb, + }, nil + } + response, _ := client.TabularDataByMQL(context.Background(), organizationID, mqlbinary) + test.That(t, response, test.ShouldResemble, rawData) + }) + + t.Run("BinaryDataByFilter", func(t *testing.T) { + includeBinary := true + grpcClient.BinaryDataByFilterFunc = func(ctx context.Context, in *pb.BinaryDataByFilterRequest, + opts ...grpc.CallOption, + ) (*pb.BinaryDataByFilterResponse, error) { + expectedDataReq := dataRequestToProto(dataRequest) + test.That(t, in.DataRequest, test.ShouldResemble, expectedDataReq) + test.That(t, in.IncludeBinary, test.ShouldBeTrue) + test.That(t, in.CountOnly, test.ShouldBeTrue) + test.That(t, in.IncludeInternalData, test.ShouldBeTrue) + return &pb.BinaryDataByFilterResponse{ + Data: []*pb.BinaryData{binaryDataToProto(binaryData)}, + Count: count, + Last: last, + }, nil + } + respBinaryData, respCount, respLast, _ := client.BinaryDataByFilter( + context.Background(), filter, count, dataRequest.SortOrder, + last, includeBinary, countOnly, includeInternalData) + test.That(t, respBinaryData[0], test.ShouldResemble, binaryData) + test.That(t, respCount, test.ShouldEqual, count) + test.That(t, respLast, test.ShouldEqual, last) + }) + t.Run("BinaryDataByIDs", func(t *testing.T) { + grpcClient.BinaryDataByIDsFunc = func(ctx context.Context, in *pb.BinaryDataByIDsRequest, + opts ...grpc.CallOption, + ) (*pb.BinaryDataByIDsResponse, error) { + test.That(t, in.IncludeBinary, test.ShouldBeTrue) + test.That(t, in.BinaryIds, test.ShouldResemble, binaryIDsToProto(binaryIDs)) + expectedBinaryDataList := []*pb.BinaryData{binaryDataToProto(binaryData)} + + return &pb.BinaryDataByIDsResponse{Data: expectedBinaryDataList, Count: uint64(len(expectedBinaryDataList))}, nil + } + respBinaryData, _ := client.BinaryDataByIDs(context.Background(), binaryIDs) + test.That(t, respBinaryData[0], test.ShouldResemble, binaryData) + }) + + t.Run("DeleteTabularData", func(t *testing.T) { + deleteOlderThanDays := uint32(1) + grpcClient.DeleteTabularDataFunc = func(ctx context.Context, in *pb.DeleteTabularDataRequest, + opts ...grpc.CallOption, + ) (*pb.DeleteTabularDataResponse, error) { + test.That(t, in.OrganizationId, test.ShouldEqual, organizationID) + test.That(t, in.DeleteOlderThanDays, test.ShouldEqual, deleteOlderThanDays) + + return &pb.DeleteTabularDataResponse{ + DeletedCount: count, + }, nil + } + resp, _ := client.DeleteTabularData(context.Background(), organizationID, deleteOlderThanDays) + test.That(t, resp, test.ShouldEqual, count) + }) + + t.Run("DeleteBinaryDataByFilter", func(t *testing.T) { + grpcClient.DeleteBinaryDataByFilterFunc = func(ctx context.Context, in *pb.DeleteBinaryDataByFilterRequest, + opts ...grpc.CallOption, + ) (*pb.DeleteBinaryDataByFilterResponse, error) { + test.That(t, in.Filter, test.ShouldResemble, filterToProto(filter)) + test.That(t, in.IncludeInternalData, test.ShouldBeTrue) + return &pb.DeleteBinaryDataByFilterResponse{ + DeletedCount: count, + }, nil + } + resp, _ := client.DeleteBinaryDataByFilter(context.Background(), filter) + test.That(t, resp, test.ShouldEqual, count) + }) + + t.Run("DeleteBinaryDataByIDs", func(t *testing.T) { + grpcClient.DeleteBinaryDataByIDsFunc = func(ctx context.Context, in *pb.DeleteBinaryDataByIDsRequest, + opts ...grpc.CallOption, + ) (*pb.DeleteBinaryDataByIDsResponse, error) { + test.That(t, in.BinaryIds, test.ShouldResemble, binaryIDsToProto(binaryIDs)) + return &pb.DeleteBinaryDataByIDsResponse{ + DeletedCount: count, + }, nil + } + resp, _ := client.DeleteBinaryDataByIDs(context.Background(), binaryIDs) + test.That(t, resp, test.ShouldEqual, count) + }) + + t.Run("AddTagsToBinaryDataByIDs", func(t *testing.T) { + grpcClient.AddTagsToBinaryDataByIDsFunc = func(ctx context.Context, in *pb.AddTagsToBinaryDataByIDsRequest, + opts ...grpc.CallOption, + ) (*pb.AddTagsToBinaryDataByIDsResponse, error) { + test.That(t, in.BinaryIds, test.ShouldResemble, binaryIDsToProto(binaryIDs)) + test.That(t, in.Tags, test.ShouldResemble, tags) + return &pb.AddTagsToBinaryDataByIDsResponse{}, nil + } + client.AddTagsToBinaryDataByIDs(context.Background(), tags, binaryIDs) + }) + + t.Run("AddTagsToBinaryDataByFilter", func(t *testing.T) { + grpcClient.AddTagsToBinaryDataByFilterFunc = func(ctx context.Context, in *pb.AddTagsToBinaryDataByFilterRequest, + opts ...grpc.CallOption, + ) (*pb.AddTagsToBinaryDataByFilterResponse, error) { + test.That(t, in.Filter, test.ShouldResemble, filterToProto(filter)) + test.That(t, in.Tags, test.ShouldResemble, tags) + return &pb.AddTagsToBinaryDataByFilterResponse{}, nil + } + client.AddTagsToBinaryDataByFilter(context.Background(), tags, filter) + }) + + t.Run("RemoveTagsFromBinaryDataByIDs", func(t *testing.T) { + grpcClient.RemoveTagsFromBinaryDataByIDsFunc = func(ctx context.Context, in *pb.RemoveTagsFromBinaryDataByIDsRequest, + opts ...grpc.CallOption, + ) (*pb.RemoveTagsFromBinaryDataByIDsResponse, error) { + test.That(t, in.BinaryIds, test.ShouldResemble, binaryIDsToProto(binaryIDs)) + test.That(t, in.Tags, test.ShouldResemble, tags) + return &pb.RemoveTagsFromBinaryDataByIDsResponse{ + DeletedCount: count, + }, nil + } + resp, _ := client.RemoveTagsFromBinaryDataByIDs(context.Background(), tags, binaryIDs) + test.That(t, resp, test.ShouldEqual, count) + }) + + t.Run("RemoveTagsFromBinaryDataByFilter", func(t *testing.T) { + grpcClient.RemoveTagsFromBinaryDataByFilterFunc = func(ctx context.Context, in *pb.RemoveTagsFromBinaryDataByFilterRequest, + opts ...grpc.CallOption, + ) (*pb.RemoveTagsFromBinaryDataByFilterResponse, error) { + test.That(t, in.Filter, test.ShouldResemble, filterToProto(filter)) + test.That(t, in.Tags, test.ShouldResemble, tags) + return &pb.RemoveTagsFromBinaryDataByFilterResponse{ + DeletedCount: count, + }, nil + } + resp, _ := client.RemoveTagsFromBinaryDataByFilter(context.Background(), tags, filter) + test.That(t, resp, test.ShouldEqual, count) + }) + + t.Run("TagsByFilter", func(t *testing.T) { + grpcClient.TagsByFilterFunc = func(ctx context.Context, in *pb.TagsByFilterRequest, + opts ...grpc.CallOption, + ) (*pb.TagsByFilterResponse, error) { + test.That(t, in.Filter, test.ShouldResemble, filterToProto(filter)) + return &pb.TagsByFilterResponse{ + Tags: tags, + }, nil + } + resp, _ := client.TagsByFilter(context.Background(), filter) + test.That(t, resp, test.ShouldResemble, tags) + }) + + t.Run("AddBoundingBoxToImageByID", func(t *testing.T) { + grpcClient.AddBoundingBoxToImageByIDFunc = func(ctx context.Context, + in *pb.AddBoundingBoxToImageByIDRequest, + opts ...grpc.CallOption, + ) (*pb.AddBoundingBoxToImageByIDResponse, error) { + test.That(t, in.BinaryId, test.ShouldResemble, binaryIDToProto(binaryID)) + test.That(t, in.Label, test.ShouldEqual, bboxLabel) + test.That(t, in.XMinNormalized, test.ShouldEqual, annotations.Bboxes[0].XMinNormalized) + test.That(t, in.YMinNormalized, test.ShouldEqual, annotations.Bboxes[0].YMinNormalized) + test.That(t, in.XMaxNormalized, test.ShouldEqual, annotations.Bboxes[0].XMaxNormalized) + test.That(t, in.YMaxNormalized, test.ShouldEqual, annotations.Bboxes[0].YMaxNormalized) + + return &pb.AddBoundingBoxToImageByIDResponse{ + BboxId: annotations.Bboxes[0].ID, + }, nil + } + resp, _ := client.AddBoundingBoxToImageByID( + context.Background(), binaryID, bboxLabel, annotations.Bboxes[0].XMinNormalized, + annotations.Bboxes[0].YMinNormalized, annotations.Bboxes[0].XMaxNormalized, annotations.Bboxes[0].YMaxNormalized) + test.That(t, resp, test.ShouldResemble, annotations.Bboxes[0].ID) + }) + + t.Run("RemoveBoundingBoxFromImageByID", func(t *testing.T) { + grpcClient.RemoveBoundingBoxFromImageByIDFunc = func(ctx context.Context, in *pb.RemoveBoundingBoxFromImageByIDRequest, + opts ...grpc.CallOption, + ) (*pb.RemoveBoundingBoxFromImageByIDResponse, error) { + test.That(t, in.BinaryId, test.ShouldResemble, binaryIDToProto(binaryID)) + test.That(t, in.BboxId, test.ShouldEqual, annotations.Bboxes[0].ID) + + return &pb.RemoveBoundingBoxFromImageByIDResponse{}, nil + } + client.RemoveBoundingBoxFromImageByID(context.Background(), annotations.Bboxes[0].ID, binaryID) + }) + + t.Run("BoundingBoxLabelsByFilter", func(t *testing.T) { + expectedBBoxLabels := []string{ + annotations.Bboxes[0].Label, + annotations.Bboxes[1].Label, + } + expectedBBoxLabelsPb := []string{ + annotationsToProto(annotations).Bboxes[0].Label, + annotationsToProto(annotations).Bboxes[1].Label, + } + grpcClient.BoundingBoxLabelsByFilterFunc = func(ctx context.Context, in *pb.BoundingBoxLabelsByFilterRequest, + opts ...grpc.CallOption, + ) (*pb.BoundingBoxLabelsByFilterResponse, error) { + test.That(t, in.Filter, test.ShouldResemble, filterToProto(filter)) + return &pb.BoundingBoxLabelsByFilterResponse{ + Labels: expectedBBoxLabelsPb, + }, nil + } + resp, _ := client.BoundingBoxLabelsByFilter(context.Background(), filter) + test.That(t, resp, test.ShouldResemble, expectedBBoxLabels) + }) + t.Run("UpdateBoundingBox", func(t *testing.T) { + annotationsPb := annotationsToProto(annotations) + grpcClient.UpdateBoundingBoxFunc = func(ctx context.Context, in *pb.UpdateBoundingBoxRequest, + opts ...grpc.CallOption, + ) (*pb.UpdateBoundingBoxResponse, error) { + test.That(t, in.BinaryId, test.ShouldResemble, binaryIDToProto(binaryID)) + test.That(t, in.BboxId, test.ShouldResemble, annotationsPb.Bboxes[0].Id) + test.That(t, *in.Label, test.ShouldEqual, annotationsPb.Bboxes[0].Label) + test.That(t, *in.XMinNormalized, test.ShouldEqual, annotationsPb.Bboxes[0].XMinNormalized) + test.That(t, *in.YMinNormalized, test.ShouldEqual, annotationsPb.Bboxes[0].YMinNormalized) + test.That(t, *in.XMaxNormalized, test.ShouldEqual, annotationsPb.Bboxes[0].XMaxNormalized) + test.That(t, *in.YMaxNormalized, test.ShouldEqual, annotationsPb.Bboxes[0].YMaxNormalized) + return &pb.UpdateBoundingBoxResponse{}, nil + } + client.UpdateBoundingBox(context.Background(), binaryID, annotations.Bboxes[0].ID, &annotationsPb.Bboxes[0].Label, + &annotationsPb.Bboxes[0].XMinNormalized, &annotationsPb.Bboxes[0].YMinNormalized, + &annotationsPb.Bboxes[0].XMaxNormalized, &annotationsPb.Bboxes[0].YMaxNormalized) + }) + + t.Run("GetDatabaseConnection", func(t *testing.T) { + grpcClient.GetDatabaseConnectionFunc = func(ctx context.Context, in *pb.GetDatabaseConnectionRequest, + opts ...grpc.CallOption, + ) (*pb.GetDatabaseConnectionResponse, error) { + test.That(t, in.OrganizationId, test.ShouldResemble, organizationID) + return &pb.GetDatabaseConnectionResponse{ + Hostname: hostName, + MongodbUri: mongodbURI, + HasDatabaseUser: true, + }, nil + } + resp, _ := client.GetDatabaseConnection(context.Background(), organizationID) + test.That(t, resp, test.ShouldResemble, hostName) + }) + + t.Run("ConfigureDatabaseUser", func(t *testing.T) { + grpcClient.ConfigureDatabaseUserFunc = func(ctx context.Context, in *pb.ConfigureDatabaseUserRequest, + opts ...grpc.CallOption, + ) (*pb.ConfigureDatabaseUserResponse, error) { + test.That(t, in.OrganizationId, test.ShouldResemble, organizationID) + test.That(t, in.Password, test.ShouldResemble, password) + return &pb.ConfigureDatabaseUserResponse{}, nil + } + client.ConfigureDatabaseUser(context.Background(), organizationID, password) + }) + + t.Run("AddBinaryDataToDatasetByIDs", func(t *testing.T) { + grpcClient.AddBinaryDataToDatasetByIDsFunc = func(ctx context.Context, in *pb.AddBinaryDataToDatasetByIDsRequest, + opts ...grpc.CallOption, + ) (*pb.AddBinaryDataToDatasetByIDsResponse, error) { + test.That(t, in.BinaryIds, test.ShouldResemble, binaryIDsToProto(binaryIDs)) + test.That(t, in.DatasetId, test.ShouldResemble, datasetID) + return &pb.AddBinaryDataToDatasetByIDsResponse{}, nil + } + client.AddBinaryDataToDatasetByIDs(context.Background(), binaryIDs, datasetID) + }) + + t.Run("RemoveBinaryDataFromDatasetByIDs", func(t *testing.T) { + grpcClient.RemoveBinaryDataFromDatasetByIDsFunc = func(ctx context.Context, in *pb.RemoveBinaryDataFromDatasetByIDsRequest, + opts ...grpc.CallOption, + ) (*pb.RemoveBinaryDataFromDatasetByIDsResponse, error) { + test.That(t, in.BinaryIds, test.ShouldResemble, binaryIDsToProto(binaryIDs)) + test.That(t, in.DatasetId, test.ShouldResemble, datasetID) + return &pb.RemoveBinaryDataFromDatasetByIDsResponse{}, nil + } + client.RemoveBinaryDataFromDatasetByIDs(context.Background(), binaryIDs, datasetID) + }) +} diff --git a/app/data_test.go b/app/data_test.go deleted file mode 100644 index ef4c945b076..00000000000 --- a/app/data_test.go +++ /dev/null @@ -1,616 +0,0 @@ -package data - -import ( - "context" - "testing" - "time" - - "go.mongodb.org/mongo-driver/bson" - datapb "go.viam.com/api/app/data/v1" - "go.viam.com/test" - utils "go.viam.com/utils/protoutils" - "google.golang.org/grpc" - "google.golang.org/protobuf/types/known/timestamppb" - - "go.viam.com/rdk/testutils/inject" -) - -const ( - componentName = "component_name" - componentType = "component_type" - method = "method" - robotName = "robot_name" - robotID = "robot_id" - partName = "part_name" - partID = "part_id" - locationID = "location_id" - organizationID = "organization_id" - password = "password" - mimeType = "mime_type" - uri = "some.robot.uri" - bboxLabel = "bbox_label" - tag = "tag" - fileName = "file_name" - fileExt = "file_ext.ext" - datasetID = "dataset_id" - binaryMetaID = "binary_id" - mongodbURI = "mongo_uri" - hostName = "host_name" -) - -var ( - locationIDs = []string{locationID} - orgIDs = []string{organizationID} - mimeTypes = []string{mimeType} - bboxLabels = []string{bboxLabel} - methodParameters = map[string]string{} - tags = []string{tag} - startTime = time.Now().UTC().Round(time.Millisecond) - endTime = time.Now().UTC().Round(time.Millisecond) - data = map[string]interface{}{ - "key": "value", - } - binaryID = BinaryID{ - FileID: "file1", - OrganizationID: organizationID, - LocationID: locationID, - } - binaryIDs = []BinaryID{binaryID} - binaryDataByte = []byte("BYTE") - sqlQuery = "SELECT * FROM readings WHERE organization_id='e76d1b3b-0468-4efd-bb7f-fb1d2b352fcb' LIMIT 1" - rawData = []map[string]any{ - { - "key1": startTime, - "key2": "2", - "key3": []any{1, 2, 3}, - "key4": map[string]any{ - "key4sub1": endTime, - }, - "key5": 4.05, - "key6": []any{true, false, true}, - "key7": []any{ - map[string]any{ - "nestedKey1": "simpleValue", - }, - map[string]any{ - "nestedKey2": startTime, - }, - }, - }, - } - datasetIDs = []string{datasetID} - annotations = Annotations{ - Bboxes: []BoundingBox{ - { - ID: "bbox1", - Label: "label1", - XMinNormalized: 0.1, - YMinNormalized: 0.2, - XMaxNormalized: 0.8, - YMaxNormalized: 0.9, - }, - { - ID: "bbox2", - Label: "label2", - XMinNormalized: 0.15, - YMinNormalized: 0.25, - XMaxNormalized: 0.75, - YMaxNormalized: 0.85, - }, - }, - } -) - -func createGrpcClient() *inject.DataServiceClient { - return &inject.DataServiceClient{} -} - -func TestDataClient(t *testing.T) { - grpcClient := createGrpcClient() - client := Client{client: grpcClient} - - captureInterval := CaptureInterval{ - Start: time.Now(), - End: time.Now(), - } - tagsFilter := TagsFilter{ - Type: TagsFilterTypeUnspecified, - Tags: []string{"tag1", "tag2"}, - } - - filter := Filter{ - ComponentName: componentName, - ComponentType: componentType, - Method: method, - RobotName: robotName, - RobotID: robotID, - PartName: partName, - PartID: partID, - LocationIDs: locationIDs, - OrganizationIDs: orgIDs, - MimeType: mimeTypes, - Interval: captureInterval, - TagsFilter: tagsFilter, // asterix or no?? - BboxLabels: bboxLabels, - DatasetID: datasetID, - } - tabularMetadata := CaptureMetadata{ - OrganizationID: organizationID, - LocationID: locationID, - RobotName: robotName, - RobotID: robotID, - PartName: partName, - PartID: partID, - ComponentType: componentType, - ComponentName: componentName, - MethodName: method, - MethodParameters: methodParameters, - Tags: tags, - MimeType: mimeType, - } - binaryMetadata := BinaryMetadata{ - ID: binaryMetaID, - CaptureMetadata: tabularMetadata, - TimeRequested: startTime, - TimeReceived: endTime, - FileName: fileName, - FileExt: fileExt, - URI: uri, - Annotations: annotations, - DatasetIDs: datasetIDs, - } - t.Run("TabularDataByFilter", func(t *testing.T) { - countOnly := true - includeInternalData := true - dataRequest := DataRequest{ - Filter: filter, - Limit: 5, - Last: "last", - SortOrder: Unspecified, - } - expectedTabularData := TabularData{ - Data: data, - MetadataIndex: 0, - Metadata: tabularMetadata, - TimeRequested: startTime, - TimeReceived: endTime, - } - expectedCount := uint64(5) - expectedLast := "last" - expectedLimit := uint64(5) - - dataStruct, _ := utils.StructToStructPb(data) - tabularDataPb := &datapb.TabularData{ - Data: dataStruct, - MetadataIndex: 0, - TimeRequested: timestamppb.New(startTime), - TimeReceived: timestamppb.New(endTime), - } - - grpcClient.TabularDataByFilterFunc = func(ctx context.Context, in *datapb.TabularDataByFilterRequest, - opts ...grpc.CallOption, - ) (*datapb.TabularDataByFilterResponse, error) { - expectedDataReqProto := dataRequestToProto(dataRequest) - test.That(t, in.DataRequest, test.ShouldResemble, expectedDataReqProto) - test.That(t, in.CountOnly, test.ShouldBeTrue) - test.That(t, in.IncludeInternalData, test.ShouldBeTrue) - return &datapb.TabularDataByFilterResponse{ - Data: []*datapb.TabularData{tabularDataPb}, - Count: expectedCount, - Last: expectedLast, - Metadata: []*datapb.CaptureMetadata{captureMetadataToProto(tabularMetadata)}, - }, nil - } - respTabularData, respCount, respLast, _ := client.TabularDataByFilter( - context.Background(), filter, expectedLimit, expectedLast, - dataRequest.SortOrder, countOnly, includeInternalData) - test.That(t, respTabularData[0], test.ShouldResemble, expectedTabularData) - test.That(t, respCount, test.ShouldEqual, expectedCount) - test.That(t, respLast, test.ShouldEqual, expectedLast) - }) - - t.Run("TabularDataBySQL", func(t *testing.T) { - expectedOrgID := organizationID - expectedSQLQuery := sqlQuery - expectedRawData := rawData - - // convert rawData to BSON - var expectedRawDataPb [][]byte - for _, byte := range expectedRawData { - bsonByte, err := bson.Marshal(byte) - if err != nil { - t.Fatalf("Failed to marshal expectedRawData: %v", err) - } - expectedRawDataPb = append(expectedRawDataPb, bsonByte) - } - grpcClient.TabularDataBySQLFunc = func(ctx context.Context, in *datapb.TabularDataBySQLRequest, - opts ...grpc.CallOption, - ) (*datapb.TabularDataBySQLResponse, error) { - test.That(t, in.OrganizationId, test.ShouldEqual, expectedOrgID) - test.That(t, in.SqlQuery, test.ShouldResemble, expectedSQLQuery) - return &datapb.TabularDataBySQLResponse{ - RawData: expectedRawDataPb, - }, nil - } - response, _ := client.TabularDataBySQL(context.Background(), expectedOrgID, expectedSQLQuery) - test.That(t, response, test.ShouldResemble, expectedRawData) - }) - - t.Run("TabularDataByMQL", func(t *testing.T) { - expectedOrgID := organizationID - matchStage := bson.M{"$match": bson.M{"organization_id": "e76d1b3b-0468-4efd-bb7f-fb1d2b352fcb"}} - limitStage := bson.M{"$limit": 1} - - // convert to BSON byte arrays - matchBytes, _ := bson.Marshal(matchStage) - limitBytes, _ := bson.Marshal(limitStage) - mqlbinary := [][]byte{matchBytes, limitBytes} - expectedMqlBinary := mqlbinary - expectedRawData := rawData - - // convert rawData to BSON - var expectedRawDataPb [][]byte - for _, byte := range expectedRawData { - bsonByte, err := bson.Marshal(byte) - if err != nil { - t.Fatalf("Failed to marshal expectedRawData: %v", err) - } - expectedRawDataPb = append(expectedRawDataPb, bsonByte) - } - grpcClient.TabularDataByMQLFunc = func(ctx context.Context, in *datapb.TabularDataByMQLRequest, - opts ...grpc.CallOption, - ) (*datapb.TabularDataByMQLResponse, error) { - test.That(t, in.OrganizationId, test.ShouldEqual, expectedOrgID) - test.That(t, in.MqlBinary, test.ShouldResemble, expectedMqlBinary) - return &datapb.TabularDataByMQLResponse{ - RawData: expectedRawDataPb, - }, nil - } - response, _ := client.TabularDataByMQL(context.Background(), expectedOrgID, expectedMqlBinary) - test.That(t, response, test.ShouldResemble, expectedRawData) - }) - - t.Run("BinaryDataByFilter", func(t *testing.T) { - includeBinary := true - countOnly := true - includeInternalData := true - dataRequest := DataRequest{ - Filter: filter, - Limit: 5, - Last: "last", - SortOrder: Unspecified, - } - expectedCount := uint64(5) - expectedLast := "last" - expectedBinaryData := BinaryData{ - Binary: binaryDataByte, - Metadata: binaryMetadata, - } - expectedBinaryDataPb := binaryDataToProto(expectedBinaryData) - - grpcClient.BinaryDataByFilterFunc = func(ctx context.Context, in *datapb.BinaryDataByFilterRequest, - opts ...grpc.CallOption, - ) (*datapb.BinaryDataByFilterResponse, error) { - expectedDataReq := dataRequestToProto(dataRequest) - test.That(t, in.DataRequest, test.ShouldResemble, expectedDataReq) - test.That(t, in.IncludeBinary, test.ShouldBeTrue) - test.That(t, in.CountOnly, test.ShouldBeTrue) - test.That(t, in.IncludeInternalData, test.ShouldBeTrue) - return &datapb.BinaryDataByFilterResponse{ - Data: []*datapb.BinaryData{expectedBinaryDataPb}, - Count: expectedCount, - Last: expectedLast, - }, nil - } - respBinaryData, respCount, respLast, _ := client.BinaryDataByFilter( - context.Background(), filter, expectedCount, dataRequest.SortOrder, - expectedLast, includeBinary, countOnly, includeInternalData) - test.That(t, respBinaryData[0], test.ShouldResemble, expectedBinaryData) - test.That(t, respCount, test.ShouldEqual, expectedCount) - test.That(t, respLast, test.ShouldEqual, expectedLast) - }) - t.Run("BinaryDataByIDs", func(t *testing.T) { - expectedBinaryData := BinaryData{ - Binary: binaryDataByte, - Metadata: binaryMetadata, - } - grpcClient.BinaryDataByIDsFunc = func(ctx context.Context, in *datapb.BinaryDataByIDsRequest, - opts ...grpc.CallOption, - ) (*datapb.BinaryDataByIDsResponse, error) { - test.That(t, in.IncludeBinary, test.ShouldBeTrue) - test.That(t, in.BinaryIds, test.ShouldResemble, binaryIDsToProto(binaryIDs)) - expectedBinaryDataList := []*datapb.BinaryData{binaryDataToProto(expectedBinaryData)} - - return &datapb.BinaryDataByIDsResponse{Data: expectedBinaryDataList, Count: uint64(len(expectedBinaryDataList))}, nil - } - respBinaryData, _ := client.BinaryDataByIDs(context.Background(), binaryIDs) - test.That(t, respBinaryData[0], test.ShouldResemble, expectedBinaryData) - }) - - t.Run("DeleteTabularData", func(t *testing.T) { - deleteOlderThanDays := uint32(1) - expectedOrgID := organizationID - expectedDeleteOlderThanDays := deleteOlderThanDays - expectedDeletedCount := uint64(5) - - grpcClient.DeleteTabularDataFunc = func(ctx context.Context, in *datapb.DeleteTabularDataRequest, - opts ...grpc.CallOption, - ) (*datapb.DeleteTabularDataResponse, error) { - test.That(t, in.OrganizationId, test.ShouldEqual, expectedOrgID) - test.That(t, in.DeleteOlderThanDays, test.ShouldEqual, expectedDeleteOlderThanDays) - - return &datapb.DeleteTabularDataResponse{ - DeletedCount: expectedDeletedCount, - }, nil - } - resp, _ := client.DeleteTabularData(context.Background(), organizationID, deleteOlderThanDays) - test.That(t, resp, test.ShouldEqual, expectedDeletedCount) - }) - - t.Run("DeleteBinaryDataByFilter", func(t *testing.T) { - expectedFilterPb := filterToProto(filter) - expectedDeletedCount := uint64(5) - - grpcClient.DeleteBinaryDataByFilterFunc = func(ctx context.Context, in *datapb.DeleteBinaryDataByFilterRequest, - opts ...grpc.CallOption, - ) (*datapb.DeleteBinaryDataByFilterResponse, error) { - test.That(t, in.Filter, test.ShouldResemble, expectedFilterPb) - test.That(t, in.IncludeInternalData, test.ShouldBeTrue) - return &datapb.DeleteBinaryDataByFilterResponse{ - DeletedCount: expectedDeletedCount, - }, nil - } - resp, _ := client.DeleteBinaryDataByFilter(context.Background(), filter) - test.That(t, resp, test.ShouldEqual, expectedDeletedCount) - }) - - t.Run("DeleteBinaryDataByIDs", func(t *testing.T) { - expectedDeletedCount := uint64(5) - expectedBinaryIDs := binaryIDsToProto(binaryIDs) - grpcClient.DeleteBinaryDataByIDsFunc = func(ctx context.Context, in *datapb.DeleteBinaryDataByIDsRequest, - opts ...grpc.CallOption, - ) (*datapb.DeleteBinaryDataByIDsResponse, error) { - test.That(t, in.BinaryIds, test.ShouldResemble, expectedBinaryIDs) - return &datapb.DeleteBinaryDataByIDsResponse{ - DeletedCount: expectedDeletedCount, - }, nil - } - resp, _ := client.DeleteBinaryDataByIDs(context.Background(), binaryIDs) - test.That(t, resp, test.ShouldEqual, expectedDeletedCount) - }) - - t.Run("AddTagsToBinaryDataByIDs", func(t *testing.T) { - expectedTags := tags - expectedBinaryIDs := binaryIDsToProto(binaryIDs) - grpcClient.AddTagsToBinaryDataByIDsFunc = func(ctx context.Context, in *datapb.AddTagsToBinaryDataByIDsRequest, - opts ...grpc.CallOption, - ) (*datapb.AddTagsToBinaryDataByIDsResponse, error) { - test.That(t, in.BinaryIds, test.ShouldResemble, expectedBinaryIDs) - test.That(t, in.Tags, test.ShouldResemble, expectedTags) - return &datapb.AddTagsToBinaryDataByIDsResponse{}, nil - } - client.AddTagsToBinaryDataByIDs(context.Background(), tags, binaryIDs) - }) - - t.Run("AddTagsToBinaryDataByFilter", func(t *testing.T) { - expectedTags := tags - expectedFilterPb := filterToProto(filter) - grpcClient.AddTagsToBinaryDataByFilterFunc = func(ctx context.Context, in *datapb.AddTagsToBinaryDataByFilterRequest, - opts ...grpc.CallOption, - ) (*datapb.AddTagsToBinaryDataByFilterResponse, error) { - test.That(t, in.Filter, test.ShouldResemble, expectedFilterPb) - test.That(t, in.Tags, test.ShouldResemble, expectedTags) - return &datapb.AddTagsToBinaryDataByFilterResponse{}, nil - } - client.AddTagsToBinaryDataByFilter(context.Background(), tags, filter) - }) - - t.Run("RemoveTagsFromBinaryDataByIDs", func(t *testing.T) { - expectedTags := tags - expectedBinaryIDs := binaryIDsToProto(binaryIDs) - expectedDeletedCount := uint64(5) - grpcClient.RemoveTagsFromBinaryDataByIDsFunc = func(ctx context.Context, in *datapb.RemoveTagsFromBinaryDataByIDsRequest, - opts ...grpc.CallOption, - ) (*datapb.RemoveTagsFromBinaryDataByIDsResponse, error) { - test.That(t, in.BinaryIds, test.ShouldResemble, expectedBinaryIDs) - test.That(t, in.Tags, test.ShouldResemble, expectedTags) - return &datapb.RemoveTagsFromBinaryDataByIDsResponse{ - DeletedCount: expectedDeletedCount, - }, nil - } - resp, _ := client.RemoveTagsFromBinaryDataByIDs(context.Background(), tags, binaryIDs) - test.That(t, resp, test.ShouldEqual, expectedDeletedCount) - }) - - t.Run("RemoveTagsFromBinaryDataByFilter", func(t *testing.T) { - expectedTags := tags - expectedFilterPb := filterToProto(filter) - expectedDeletedCount := uint64(5) - - grpcClient.RemoveTagsFromBinaryDataByFilterFunc = func(ctx context.Context, in *datapb.RemoveTagsFromBinaryDataByFilterRequest, - opts ...grpc.CallOption, - ) (*datapb.RemoveTagsFromBinaryDataByFilterResponse, error) { - test.That(t, in.Filter, test.ShouldResemble, expectedFilterPb) - test.That(t, in.Tags, test.ShouldResemble, expectedTags) - return &datapb.RemoveTagsFromBinaryDataByFilterResponse{ - DeletedCount: expectedDeletedCount, - }, nil - } - resp, _ := client.RemoveTagsFromBinaryDataByFilter(context.Background(), tags, filter) - test.That(t, resp, test.ShouldEqual, expectedDeletedCount) - }) - - t.Run("TagsByFilter", func(t *testing.T) { - expectedTags := tags - expectedFilterPb := filterToProto(filter) - - grpcClient.TagsByFilterFunc = func(ctx context.Context, in *datapb.TagsByFilterRequest, - opts ...grpc.CallOption, - ) (*datapb.TagsByFilterResponse, error) { - test.That(t, in.Filter, test.ShouldResemble, expectedFilterPb) - return &datapb.TagsByFilterResponse{ - Tags: tags, - }, nil - } - resp, _ := client.TagsByFilter(context.Background(), filter) - test.That(t, resp, test.ShouldResemble, expectedTags) - }) - - t.Run("AddBoundingBoxToImageByID", func(t *testing.T) { - expectedBinaryIDPb := binaryIDToProto(binaryID) - expectedLabel := bboxLabel - expectedXMin := annotations.Bboxes[0].XMinNormalized - expectedYMin := annotations.Bboxes[0].YMinNormalized - expectedXMax := annotations.Bboxes[0].XMaxNormalized - expectedYMax := annotations.Bboxes[0].YMaxNormalized - expectedBBoxID := annotations.Bboxes[0].ID - - grpcClient.AddBoundingBoxToImageByIDFunc = func(ctx context.Context, - in *datapb.AddBoundingBoxToImageByIDRequest, - opts ...grpc.CallOption, - ) (*datapb.AddBoundingBoxToImageByIDResponse, error) { - test.That(t, in.BinaryId, test.ShouldResemble, expectedBinaryIDPb) - test.That(t, in.Label, test.ShouldEqual, expectedLabel) - test.That(t, in.XMinNormalized, test.ShouldEqual, expectedXMin) - test.That(t, in.YMinNormalized, test.ShouldEqual, expectedYMin) - test.That(t, in.XMaxNormalized, test.ShouldEqual, expectedXMax) - test.That(t, in.YMaxNormalized, test.ShouldEqual, expectedYMax) - - return &datapb.AddBoundingBoxToImageByIDResponse{ - BboxId: expectedBBoxID, - }, nil - } - resp, _ := client.AddBoundingBoxToImageByID( - context.Background(), binaryID, bboxLabel, expectedXMin, - expectedYMin, expectedXMax, expectedYMax) - test.That(t, resp, test.ShouldResemble, expectedBBoxID) - }) - - t.Run("RemoveBoundingBoxFromImageByID", func(t *testing.T) { - expectedBinaryIDPb := binaryIDToProto(binaryID) - expectedBBoxID := annotations.Bboxes[0].ID - - grpcClient.RemoveBoundingBoxFromImageByIDFunc = func(ctx context.Context, in *datapb.RemoveBoundingBoxFromImageByIDRequest, - opts ...grpc.CallOption, - ) (*datapb.RemoveBoundingBoxFromImageByIDResponse, error) { - test.That(t, in.BinaryId, test.ShouldResemble, expectedBinaryIDPb) - test.That(t, in.BboxId, test.ShouldEqual, expectedBBoxID) - - return &datapb.RemoveBoundingBoxFromImageByIDResponse{}, nil - } - client.RemoveBoundingBoxFromImageByID(context.Background(), expectedBBoxID, binaryID) - }) - - t.Run("BoundingBoxLabelsByFilter", func(t *testing.T) { - expectedFilterPb := filterToProto(filter) - expectedBBoxLabels := []string{ - annotations.Bboxes[0].Label, - annotations.Bboxes[1].Label, - } - annotationsPb := annotationsToProto(annotations) - expectedBBoxLabelsPb := []string{ - annotationsPb.Bboxes[0].Label, - annotationsPb.Bboxes[1].Label, - } - - grpcClient.BoundingBoxLabelsByFilterFunc = func(ctx context.Context, in *datapb.BoundingBoxLabelsByFilterRequest, - opts ...grpc.CallOption, - ) (*datapb.BoundingBoxLabelsByFilterResponse, error) { - test.That(t, in.Filter, test.ShouldResemble, expectedFilterPb) - return &datapb.BoundingBoxLabelsByFilterResponse{ - Labels: expectedBBoxLabelsPb, - }, nil - } - resp, _ := client.BoundingBoxLabelsByFilter(context.Background(), filter) - test.That(t, resp, test.ShouldResemble, expectedBBoxLabels) - }) - t.Run("UpdateBoundingBox", func(t *testing.T) { - bBoxID := annotations.Bboxes[0].ID - expectedBinaryIDPb := binaryIDToProto(binaryID) - - annotationsPb := annotationsToProto(annotations) - expectedLabel := annotationsPb.Bboxes[0].Label - expectedBBoxIDPb := annotationsPb.Bboxes[0].Id - expectedXMin := annotationsPb.Bboxes[0].XMinNormalized - expectedYMin := annotationsPb.Bboxes[0].YMinNormalized - expectedXMax := annotationsPb.Bboxes[0].XMaxNormalized - expectedYMax := annotationsPb.Bboxes[0].YMaxNormalized - - grpcClient.UpdateBoundingBoxFunc = func(ctx context.Context, in *datapb.UpdateBoundingBoxRequest, - opts ...grpc.CallOption, - ) (*datapb.UpdateBoundingBoxResponse, error) { - test.That(t, in.BinaryId, test.ShouldResemble, expectedBinaryIDPb) - test.That(t, in.BboxId, test.ShouldResemble, expectedBBoxIDPb) - test.That(t, *in.Label, test.ShouldEqual, expectedLabel) - test.That(t, *in.XMinNormalized, test.ShouldEqual, expectedXMin) - test.That(t, *in.YMinNormalized, test.ShouldEqual, expectedYMin) - test.That(t, *in.XMaxNormalized, test.ShouldEqual, expectedXMax) - test.That(t, *in.YMaxNormalized, test.ShouldEqual, expectedYMax) - return &datapb.UpdateBoundingBoxResponse{}, nil - } - client.UpdateBoundingBox(context.Background(), binaryID, bBoxID, &expectedLabel, - &expectedXMin, &expectedYMin, &expectedXMax, &expectedYMax) - }) - - t.Run("GetDatabaseConnection", func(t *testing.T) { - expectedOrgID := organizationID - expectedHostName := hostName - expectedMongodbURI := mongodbURI - expectedDBUser := true - - grpcClient.GetDatabaseConnectionFunc = func(ctx context.Context, in *datapb.GetDatabaseConnectionRequest, - opts ...grpc.CallOption, - ) (*datapb.GetDatabaseConnectionResponse, error) { - test.That(t, in.OrganizationId, test.ShouldResemble, expectedOrgID) - return &datapb.GetDatabaseConnectionResponse{ - Hostname: expectedHostName, - MongodbUri: expectedMongodbURI, - HasDatabaseUser: expectedDBUser, - }, nil - } - resp, _ := client.GetDatabaseConnection(context.Background(), organizationID) - test.That(t, resp, test.ShouldResemble, expectedHostName) - }) - - t.Run("ConfigureDatabaseUser", func(t *testing.T) { - expectedOrgID := organizationID - expectedPassword := password - - grpcClient.ConfigureDatabaseUserFunc = func(ctx context.Context, in *datapb.ConfigureDatabaseUserRequest, - opts ...grpc.CallOption, - ) (*datapb.ConfigureDatabaseUserResponse, error) { - test.That(t, in.OrganizationId, test.ShouldResemble, expectedOrgID) - test.That(t, in.Password, test.ShouldResemble, expectedPassword) - return &datapb.ConfigureDatabaseUserResponse{}, nil - } - client.ConfigureDatabaseUser(context.Background(), organizationID, password) - }) - - t.Run("AddBinaryDataToDatasetByIDs", func(t *testing.T) { - expectedBinaryIDs := binaryIDsToProto(binaryIDs) - expectedDataSetID := datasetID - - grpcClient.AddBinaryDataToDatasetByIDsFunc = func(ctx context.Context, in *datapb.AddBinaryDataToDatasetByIDsRequest, - opts ...grpc.CallOption, - ) (*datapb.AddBinaryDataToDatasetByIDsResponse, error) { - test.That(t, in.BinaryIds, test.ShouldResemble, expectedBinaryIDs) - test.That(t, in.DatasetId, test.ShouldResemble, expectedDataSetID) - return &datapb.AddBinaryDataToDatasetByIDsResponse{}, nil - } - client.AddBinaryDataToDatasetByIDs(context.Background(), binaryIDs, datasetID) - }) - - t.Run("RemoveBinaryDataFromDatasetByIDs", func(t *testing.T) { - expectedBinaryIDs := binaryIDsToProto(binaryIDs) - expectedDataSetID := datasetID - - grpcClient.RemoveBinaryDataFromDatasetByIDsFunc = func(ctx context.Context, in *datapb.RemoveBinaryDataFromDatasetByIDsRequest, - opts ...grpc.CallOption, - ) (*datapb.RemoveBinaryDataFromDatasetByIDsResponse, error) { - test.That(t, in.BinaryIds, test.ShouldResemble, expectedBinaryIDs) - test.That(t, in.DatasetId, test.ShouldResemble, expectedDataSetID) - return &datapb.RemoveBinaryDataFromDatasetByIDsResponse{}, nil - } - client.RemoveBinaryDataFromDatasetByIDs(context.Background(), binaryIDs, datasetID) - }) -}