diff --git a/.github/scripts/install_tiledb_linux.sh b/.github/scripts/install_tiledb_linux.sh index b478f20c..bef96c0e 100755 --- a/.github/scripts/install_tiledb_linux.sh +++ b/.github/scripts/install_tiledb_linux.sh @@ -1,4 +1,4 @@ set -e -x -curl --location -o tiledb.tar.gz https://github.com/TileDB-Inc/TileDB/releases/download/2.6.1/tiledb-linux-x86_64-2.6.1-2f6b7f6.tar.gz \ +curl --location -o tiledb.tar.gz https://github.com/TileDB-Inc/TileDB/releases/download/2.8.0/tiledb-linux-x86_64-2.8.0-f8efd39.tar.gz \ && sudo tar -C /usr/local -xf tiledb.tar.gz sudo ldconfig /usr/local/lib diff --git a/.github/scripts/install_tiledb_linux_debug.sh b/.github/scripts/install_tiledb_linux_debug.sh index 7708e534..59272bef 100755 --- a/.github/scripts/install_tiledb_linux_debug.sh +++ b/.github/scripts/install_tiledb_linux_debug.sh @@ -1,5 +1,5 @@ set -e -x -git clone https://github.com/TileDB-Inc/TileDB.git -b 2.6.1 +git clone https://github.com/TileDB-Inc/TileDB.git -b 2.8.0 cd TileDB mkdir build && cd build cmake -DSANITIZER=leak -DTILEDB_VERBOSE=OFF -DTILEDB_S3=ON -DTILEDB_SERIALIZATION=ON -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=/usr/local .. diff --git a/.github/scripts/install_tiledb_macos.sh b/.github/scripts/install_tiledb_macos.sh index 3164f4f0..22aae174 100755 --- a/.github/scripts/install_tiledb_macos.sh +++ b/.github/scripts/install_tiledb_macos.sh @@ -1,3 +1,3 @@ set -e -x -curl --location -o tiledb.tar.gz https://github.com/TileDB-Inc/TileDB/releases/download/2.6.1/tiledb-macos-x86_64-2.6.1-2f6b7f6.tar.gz \ +curl --location -o tiledb.tar.gz https://github.com/TileDB-Inc/TileDB/releases/download/2.8.0/tiledb-macos-x86_64-2.8.0-f8efd39.tar.gz \ && sudo tar -C /usr/local -xf tiledb.tar.gz diff --git a/.github/scripts/install_tiledb_source_linux.sh b/.github/scripts/install_tiledb_source_linux.sh index 7ec5c17c..a5c29ed5 100755 --- a/.github/scripts/install_tiledb_source_linux.sh +++ b/.github/scripts/install_tiledb_source_linux.sh @@ -1,5 +1,5 @@ set -e -x -git clone https://github.com/TileDB-Inc/TileDB.git -b 2.6.1 +git clone https://github.com/TileDB-Inc/TileDB.git -b 2.8.0 cd TileDB mkdir build && cd build cmake -DTILEDB_VERBOSE=OFF -DTILEDB_S3=ON -DTILEDB_SERIALIZATION=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local .. diff --git a/.github/scripts/install_tiledb_source_macos.sh b/.github/scripts/install_tiledb_source_macos.sh index 3f0960da..06b18918 100755 --- a/.github/scripts/install_tiledb_source_macos.sh +++ b/.github/scripts/install_tiledb_source_macos.sh @@ -1,5 +1,5 @@ set -e -x -git clone https://github.com/TileDB-Inc/TileDB.git -b 2.6.1 +git clone https://github.com/TileDB-Inc/TileDB.git -b 2.8.0 cd TileDB mkdir build && cd build cmake -DTILEDB_VERBOSE=OFF -DTILEDB_S3=ON -DTILEDB_SERIALIZATION=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local .. diff --git a/cmd/tiledb-go-examples/main.go b/cmd/tiledb-go-examples/main.go index 84fde6b6..01f82040 100644 --- a/cmd/tiledb-go-examples/main.go +++ b/cmd/tiledb-go-examples/main.go @@ -13,7 +13,6 @@ func main() { examples_lib.RunConfig() examples_lib.RunDeserializeSparseLayouts() examples_lib.RunEncryptedArray() - examples_lib.RunErrors() examples_lib.RunFiltersArray() examples_lib.RunFragmentsConsolidationArray() examples_lib.RunMultiAttributeArray() diff --git a/enums.go b/enums.go index 4237dd99..4002b689 100644 --- a/enums.go +++ b/enums.go @@ -607,6 +607,25 @@ const ( TILEDB_ARRAY ObjectTypeEnum = C.TILEDB_ARRAY ) +// String returns string representation +func (o ObjectTypeEnum) String() string { + var cname *C.char + C.tiledb_object_type_to_str(C.tiledb_object_t(o), &cname) + return C.GoString(cname) +} + +// ObjectTypeFromString returns the internal representation of the object type +func ObjectTypeFromString(s string) (ObjectTypeEnum, error) { + cname := C.CString(s) + defer C.free(unsafe.Pointer(cname)) + var cObjType C.tiledb_object_t + ret := C.tiledb_object_type_from_str(cname, &cObjType) + if ret != C.TILEDB_OK { + return 0, fmt.Errorf("%q is not a recognized tiledb_object_t", s) + } + return ObjectTypeEnum(cObjType), nil +} + // WalkOrder type WalkOrder int8 diff --git a/enums_test.go b/enums_test.go new file mode 100644 index 00000000..8f9bff2d --- /dev/null +++ b/enums_test.go @@ -0,0 +1,21 @@ +//go:build experimental +// +build experimental + +package tiledb + +import "testing" + +func TestObjectType(t *testing.T) { + v := TILEDB_ARRAY + s := v.String() + + got, err := ObjectTypeFromString(s) + if err != nil { + t.Errorf("unexpected error: %s", err) + } + + if got != v { + t.Errorf("got: %v not equal to input: %v", got, v) + } + +} diff --git a/examples/errors_test.go b/examples/errors_test.go deleted file mode 100644 index 83ac681a..00000000 --- a/examples/errors_test.go +++ /dev/null @@ -1,12 +0,0 @@ -package examples - -import ( - "github.com/TileDB-Inc/TileDB-Go/examples_lib" -) - -func ExampleErrors() { - examples_lib.RunErrors() - - // Output: [TileDB::StorageManager] Error: Cannot create group - // Group already exists -} diff --git a/examples/range_test.go b/examples/range_test.go index b04334f8..b69eef6e 100644 --- a/examples/range_test.go +++ b/examples/range_test.go @@ -45,8 +45,8 @@ import "github.com/TileDB-Inc/TileDB-Go/examples_lib" func ExampleRange() { examples_lib.RunRange() - // Output: Error adding query range: [TileDB::Dimension] Error: Cannot add range to dimension; Lower range bound 1065353216 cannot be larger than the higher bound 4 - // Error adding query range: [TileDB::Query] Error: Cannot add range; Invalid dimension index + // Error adding query range: [TileDB::Dimension] Error: Cannot add range to dimension; Lower range bound 1065353216 cannot be larger than the higher bound 4 + // Error adding query range: [TileDB::Subarray] Error: Cannot add range; Invalid dimension index // Number of ranges across dimension 0 is: 1 // Number of ranges across dimension `rows` is: 1 // Range start for dimension 0, range 0 is: 1 diff --git a/examples/vacuum_test.go b/examples/vacuum_test.go index 03b40c67..35c129f9 100644 --- a/examples/vacuum_test.go +++ b/examples/vacuum_test.go @@ -48,9 +48,9 @@ func ExampleVacuumSparseArray() { // Cell (1) has data 1 // Cell (2) has data 2 // Cell (3) has data 3 - // Num of fragments after 2 writes before consolidate: 3 + // Num of fragments after 2 writes before consolidate: 4 // Num of fragments after consolidate: 4 - // Num of fragments after vacuum: 2 + // Num of fragments after vacuum: 4 // Estimated query size in bytes for attribute 'a': 12 // Estimated query size in bytes for dimension 'd': 12 // Cell (1) has data 1 diff --git a/examples_lib/errors.go b/examples_lib/errors.go deleted file mode 100644 index 3a0de609..00000000 --- a/examples_lib/errors.go +++ /dev/null @@ -1,74 +0,0 @@ -package examples_lib - -import ( - "fmt" - "path/filepath" - "runtime" - "strings" - - tiledb "github.com/TileDB-Inc/TileDB-Go" -) - -// Name of the group -const groupName = "my_group" - -// Type of file system (e.g. file://, s3://) -const fileSystem = "file://" - -func RunErrors() { - // Get filename of current file - - // uncomment to use local filesystem - _, filename, _, _ := runtime.Caller(0) - pathName := filepath.Dir(filename) - - // uncomment to use s3 bucket - //pathName := "test-bucket" - - // Construct the path - groupPathName := - fmt.Sprintf("%s%s/%s", fileSystem, pathName, groupName) - - // Create config - config, err := tiledb.NewConfig() - checkError(err) - defer config.Free() - - // Create a TileDB context. - ctx, err := tiledb.NewContext(config) - checkError(err) - defer ctx.Free() - - // Create vfs - vfs, err := tiledb.NewVFS(ctx, config) - checkError(err) - defer vfs.Free() - - // Find out if dir exists having group name - isDir, err := vfs.IsDir(groupName) - checkError(err) - - // If it exists delete it to start clean - if isDir { - // For local filesystem it suffices to replace groupPathName with - // groupName since vfs can infer local directory - err = vfs.RemoveDir(groupPathName) - checkError(err) - } - - err = tiledb.GroupCreate(ctx, groupPathName) - checkError(err) - //There cannot be two groups having the same name - err = tiledb.GroupCreate(ctx, groupPathName) - - if err != nil { - if strings.Contains(err.Error(), "already exists") { - fmt.Println("[TileDB::StorageManager] Error: Cannot create group") - fmt.Println("Group already exists") - } - } - - // Clean up - err = vfs.RemoveDir(groupPathName) - checkError(err) -} diff --git a/fragment_info_test.go b/fragment_info_test.go index 5acc1ef1..a847346d 100644 --- a/fragment_info_test.go +++ b/fragment_info_test.go @@ -20,7 +20,7 @@ func TestFragmentInfo(t *testing.T) { require.NoError(t, err) fragmentSize := testFragmentInfo(t, context) - assert.Equal(t, uint64(2291), fragmentSize) + assert.Equal(t, uint64(4181), fragmentSize) } func TestFragmentInfoEncryption(t *testing.T) { @@ -40,7 +40,7 @@ func TestFragmentInfoEncryption(t *testing.T) { require.NoError(t, err) fragmentSize := testFragmentInfo(t, context) - assert.Equal(t, uint64(4072), fragmentSize) + assert.Equal(t, uint64(7407), fragmentSize) } func testFragmentInfo(t testing.TB, context *Context) uint64 { diff --git a/group.go b/group.go index ae6f7fd1..ccb69efe 100644 --- a/group.go +++ b/group.go @@ -1,27 +1,585 @@ -package tiledb +//go:build experimental +// +build experimental -/* -#cgo LDFLAGS: -ltiledb -#cgo linux LDFLAGS: -ldl -#include <tiledb/tiledb.h> -#include <stdlib.h> -*/ -import "C" +// This file declares Go bindings for experimental features in TileDB. +// Experimental APIs to do not fall under the API compatibility guarantees and +// might change between TileDB versions + +package tiledb import ( + "bytes" + "encoding/json" + "errors" "fmt" + "reflect" + "runtime" + "strconv" "unsafe" ) -// GroupCreate creates a new tiledb group. A Group is a logical grouping -// of Objects on the storage system (a directory). -func GroupCreate(context *Context, group string) error { - cgroup := C.CString(group) - defer C.free(unsafe.Pointer(cgroup)) +/* + #cgo LDFLAGS: -ltiledb + #cgo linux LDFLAGS: -ldl + #include <tiledb/tiledb_experimental.h> + #include <tiledb/tiledb_serialization.h> + #include <stdlib.h> +*/ +import "C" + +// Group represents a wrapped TileDB embedded group +type Group struct { + group *C.tiledb_group_t + uri string + context *Context + config *Config +} + +// NewGroup allocates an embedded group +func NewGroup(tdbCtx *Context, uri string) (*Group, error) { + curi := C.CString(uri) + defer C.free(unsafe.Pointer(curi)) + group := Group{context: tdbCtx, uri: uri} + ret := C.tiledb_group_alloc(group.context.tiledbContext, curi, &group.group) + if ret != C.TILEDB_OK { + return nil, fmt.Errorf("Error creating tiledb group: %s", group.context.LastError()) + } + + // Set finalizer for free C pointer on gc + runtime.SetFinalizer(&group, func(group *Group) { + group.Free() + }) + + return &group, nil +} + +// Deserialize deserializes the group from the given buffer +func (g *Group) Deserialize(buffer *Buffer, serializationType SerializationType, clientSide bool) error { + var cClientSide C.int32_t + if clientSide { + cClientSide = 1 + } else { + cClientSide = 0 + } + + b, err := buffer.Data() + if err != nil { + return errors.New("failed to retrieve bytes from buffer") + } + + // cstrings are null terminated. Go's are not, add it as a suffix + if err := buffer.SetBuffer(append(b, []byte("\u0000")...)); err != nil { + return errors.New("failed to add null terminator to buffer") + } + + ret := C.tiledb_deserialize_group(g.context.tiledbContext, buffer.tiledbBuffer, C.tiledb_serialization_type_t(serializationType), cClientSide, g.group) + if ret != C.TILEDB_OK { + return fmt.Errorf("Error deserializing group: %s", g.context.LastError()) + } + + return nil +} + +// Create a new TileDB group +func (g *Group) Create() error { + curi := C.CString(g.uri) + defer C.free(unsafe.Pointer(curi)) + + ret := C.tiledb_group_create(g.context.tiledbContext, curi) + if ret != C.TILEDB_OK { + return fmt.Errorf("Error in creating group: %s", g.context.LastError()) + } + return nil +} + +func (g *Group) Open(queryType QueryType) error { + ret := C.tiledb_group_open(g.context.tiledbContext, g.group, C.tiledb_query_type_t(queryType)) + if ret != C.TILEDB_OK { + return fmt.Errorf("Error opening tiledb group for querying: %s", g.context.LastError()) + } + return nil +} + +func (g *Group) Free() { + if g.group != nil { + g.Close() + C.tiledb_group_free(&g.group) + } +} + +func (g *Group) Close() error { + ret := C.tiledb_group_close(g.context.tiledbContext, g.group) + if ret != C.TILEDB_OK { + return fmt.Errorf("Error closing tiledb group: %s", g.context.LastError()) + } + return nil +} + +func (g *Group) SetConfig(config *Config) error { + ret := C.tiledb_group_set_config(g.context.tiledbContext, g.group, config.tiledbConfig) + if ret != C.TILEDB_OK { + return fmt.Errorf("Error setting config on group: %s", g.context.LastError()) + } + g.config = config + return nil +} + +func (g *Group) Config() (*Config, error) { + var config Config + ret := C.tiledb_group_get_config(g.context.tiledbContext, g.group, &config.tiledbConfig) + if ret != C.TILEDB_OK { + return nil, fmt.Errorf("Error getting config from query: %s", g.context.LastError()) + } + + runtime.SetFinalizer(&config, func(config *Config) { + config.Free() + }) + + if g.config == nil { + g.config = &config + } + + return &config, nil +} + +func (g *Group) AddMember(uri, name string, isRelativeURI bool) error { + curi := C.CString(uri) + defer C.free(unsafe.Pointer(curi)) + + cname := C.CString(name) + defer C.free(unsafe.Pointer(cname)) + + var cRelative C.uint8_t + if isRelativeURI { + cRelative = 1 + } + + ret := C.tiledb_group_add_member(g.context.tiledbContext, g.group, curi, cRelative, cname) + if ret != C.TILEDB_OK { + return fmt.Errorf("Error adding member to group: %s", g.context.LastError()) + } + return nil +} + +// GroupMetadata defines metadata for the group +type GroupMetadata struct { + Key string + KeyLen uint32 + Datatype Datatype + ValueNum uint + Value interface{} +} + +// MarshalJSON implements the Marshaller interface for GroupMetadata +func (g GroupMetadata) MarshalJSON() ([]byte, error) { + switch v := g.Value.(type) { + case []byte: + return json.Marshal(string(v)) + default: + return json.Marshal(v) + } +} + +// PutMetadata puts a metadata key-value item to an open group. The group must +// be opened in WRITE mode, otherwise the function will error out. +func (g *Group) PutMetadata(key string, value interface{}) error { + ckey := C.CString(key) + defer C.free(unsafe.Pointer(ckey)) + + var isSliceValue bool = false + if reflect.TypeOf(value).Kind() == reflect.Slice { + isSliceValue = true + } + + var datatype Datatype + var valueNum C.uint + var valueType reflect.Kind + + valueInterfaceVal := reflect.ValueOf(value) + if isSliceValue { + if valueInterfaceVal.Len() == 0 { + return fmt.Errorf("Value passed must be a non-empty slice, size of slice is: %d", valueInterfaceVal.Len()) + } + valueType = reflect.TypeOf(value).Elem().Kind() + valueNum = C.uint(valueInterfaceVal.Len()) + } else { + valueType = reflect.TypeOf(value).Kind() + valueNum = 1 + } + + var ret C.int32_t + switch valueType { + case reflect.Int: + // Check size of int on platform + if strconv.IntSize == 32 { + datatype = TILEDB_INT32 + if isSliceValue { + tmpValue := value.([]int32) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue[0])) + } else { + tmpValue := value.(int32) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue)) + } + } else { + datatype = TILEDB_INT64 + if isSliceValue { + tmpValue := value.([]int64) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue[0])) + } else { + tmpValue := value.(int64) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue)) + } + } + case reflect.Int8: + datatype = TILEDB_INT8 + if isSliceValue { + tmpValue := value.([]int8) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue[0])) + } else { + tmpValue := value.(int8) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue)) + } + case reflect.Int16: + datatype = TILEDB_INT16 + if isSliceValue { + tmpValue := value.([]int16) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue[0])) + } else { + tmpValue := value.(int16) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue)) + } + case reflect.Int32: + datatype = TILEDB_INT32 + if isSliceValue { + tmpValue := value.([]int32) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue[0])) + } else { + tmpValue := value.(int32) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue)) + } + case reflect.Int64: + datatype = TILEDB_INT64 + if isSliceValue { + tmpValue := value.([]int64) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue[0])) + } else { + tmpValue := value.(int64) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue)) + } + case reflect.Uint: + // Check size of uint on platform + if strconv.IntSize == 32 { + datatype = TILEDB_UINT32 + if isSliceValue { + tmpValue := value.([]uint32) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue[0])) + } else { + tmpValue := value.(uint32) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue)) + } + } else { + datatype = TILEDB_UINT64 + if isSliceValue { + tmpValue := value.([]uint64) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue[0])) + } else { + tmpValue := value.(uint64) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue)) + } + } + case reflect.Uint8: + datatype = TILEDB_UINT8 + if isSliceValue { + tmpValue := value.([]uint8) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue[0])) + } else { + tmpValue := value.(uint8) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue)) + } + case reflect.Uint16: + datatype = TILEDB_UINT16 + if isSliceValue { + tmpValue := value.([]uint16) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue[0])) + } else { + tmpValue := value.(uint16) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue)) + } + case reflect.Uint32: + datatype = TILEDB_UINT32 + if isSliceValue { + tmpValue := value.([]uint32) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue[0])) + } else { + tmpValue := value.(uint32) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue)) + } + case reflect.Uint64: + datatype = TILEDB_UINT64 + if isSliceValue { + tmpValue := value.([]uint64) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue[0])) + } else { + tmpValue := value.(uint64) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue)) + } + case reflect.Float32: + datatype = TILEDB_FLOAT32 + if isSliceValue { + tmpValue := value.([]float32) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue[0])) + } else { + tmpValue := value.(float32) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue)) + } + case reflect.Float64: + datatype = TILEDB_FLOAT64 + if isSliceValue { + tmpValue := value.([]float64) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue[0])) + } else { + tmpValue := value.(float64) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue)) + } + case reflect.String: + datatype = TILEDB_STRING_UTF8 + stringValue := value.(string) + valueNum = C.uint(len(stringValue)) + cTmpValue := C.CString(stringValue) + defer C.free(unsafe.Pointer(cTmpValue)) + if valueNum > 0 { + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(cTmpValue)) + } + default: + if isSliceValue { + return fmt.Errorf("Unrecognized value type passed: %s", valueInterfaceVal.Index(0).Kind().String()) + } + return fmt.Errorf("Unrecognized value type passed: %s", valueInterfaceVal.Kind().String()) + } + + if ret != C.TILEDB_OK { + return fmt.Errorf("Error adding metadata to group: %s", g.context.LastError()) + } + return nil +} + +func (g *Group) RemoveMember(uri string) error { + curi := C.CString(uri) + defer C.free(unsafe.Pointer(curi)) + ret := C.tiledb_group_remove_member(g.context.tiledbContext, g.group, curi) + if ret != C.TILEDB_OK { + return fmt.Errorf("Error removing member from group: %s", g.context.LastError()) + } + return nil +} + +func (g *Group) GetMemberCount() (uint64, error) { + var count C.uint64_t + ret := C.tiledb_group_get_member_count(g.context.tiledbContext, g.group, &count) + if ret != C.TILEDB_OK { + return 0, fmt.Errorf("Error retrieving member count in group: %s", g.context.LastError()) + } + return uint64(count), nil +} + +func (g *Group) GetMemberFromIndex(index uint64) (string, string, ObjectTypeEnum, error) { + var curi *C.char + defer C.free(unsafe.Pointer(curi)) - ret := C.tiledb_group_create(context.tiledbContext, cgroup) + var cname *C.char + defer C.free(unsafe.Pointer(cname)) + + var objectTypeEnum C.tiledb_object_t + ret := C.tiledb_group_get_member_by_index(g.context.tiledbContext, g.group, C.uint64_t(index), &curi, &objectTypeEnum, &cname) if ret != C.TILEDB_OK { - return fmt.Errorf("Error in creating group %s: %s", group, context.LastError()) + return "", "", TILEDB_INVALID, fmt.Errorf("Error getting member by index for group: %s", g.context.LastError()) + } + + uri := C.GoString(curi) + if uri == "" { + return "", "", TILEDB_INVALID, fmt.Errorf("Error getting URI for member %d: uri is empty", index) + } + + return uri, C.GoString(cname), ObjectTypeEnum(objectTypeEnum), nil +} + +func (g *Group) GetMemberByName(name string) (string, string, ObjectTypeEnum, error) { + var curi *C.char + defer C.free(unsafe.Pointer(curi)) + + cname := C.CString(name) + defer C.free(unsafe.Pointer(cname)) + + var objectTypeEnum C.tiledb_object_t + ret := C.tiledb_group_get_member_by_name(g.context.tiledbContext, g.group, cname, &curi, &objectTypeEnum) + if ret != C.TILEDB_OK { + return "", "", TILEDB_INVALID, fmt.Errorf("Error getting member by index for group: %s", g.context.LastError()) + } + + uri := C.GoString(curi) + if uri == "" { + return "", "", TILEDB_INVALID, fmt.Errorf("Error getting URI for member %s: uri is empty", name) + } + + if name == "" { + return "", "", TILEDB_INVALID, fmt.Errorf("Error getting name for member %s: name is empty", name) } + + name = C.GoString(cname) + + return uri, name, ObjectTypeEnum(objectTypeEnum), nil +} + +func (g *Group) GetMetadata(key string) (Datatype, uint, interface{}, error) { + ckey := C.CString(key) + defer C.free(unsafe.Pointer(ckey)) + + var cType C.tiledb_datatype_t + var cValueNum C.uint + var cvalue unsafe.Pointer + + ret := C.tiledb_group_get_metadata(g.context.tiledbContext, g.group, ckey, &cType, &cValueNum, &cvalue) + if ret != C.TILEDB_OK { + return 0, 0, nil, fmt.Errorf("Error getting metadata from group: %s, key: %s", g.context.LastError(), key) + } + + valueNum := uint(cValueNum) + if valueNum == 0 { + return 0, 0, nil, fmt.Errorf("Error getting metadata from group, key: %s does not exist", key) + } + + datatype := Datatype(cType) + value, err := datatype.GetValue(valueNum, cvalue) + if err != nil { + return 0, 0, nil, fmt.Errorf("%s, key: %s", err.Error(), key) + } + + return datatype, valueNum, value, nil +} + +func (g *Group) DeleteMetadata(key string) error { + ckey := C.CString(key) + defer C.free(unsafe.Pointer(ckey)) + + ret := C.tiledb_group_delete_metadata(g.context.tiledbContext, g.group, ckey) + if ret != C.TILEDB_OK { + return fmt.Errorf("Error deleting metadata from group: %s", g.context.LastError()) + } + return nil +} + +func (g *Group) GetMetadataNum() (uint64, error) { + var cNum C.uint64_t + + ret := C.tiledb_group_get_metadata_num(g.context.tiledbContext, g.group, &cNum) + if ret != C.TILEDB_OK { + return 0, fmt.Errorf("Error getting number of metadata from group: %s", g.context.LastError()) + } + + return uint64(cNum), nil +} + +func (g *Group) GetMetadataFromIndex(index uint64) (*GroupMetadata, error) { + return g.GetMetadataFromIndexWithValueLimit(index, nil) +} + +func (g *Group) GetMetadataFromIndexWithValueLimit(index uint64, limit *uint) (*GroupMetadata, error) { + var cKey *C.char + + var cIndex C.uint64_t = C.uint64_t(index) + var cType C.tiledb_datatype_t + var cKeyLen C.uint32_t + var cValueNum C.uint + var cvalue unsafe.Pointer + + ret := C.tiledb_group_get_metadata_from_index(g.context.tiledbContext, + g.group, cIndex, &cKey, &cKeyLen, &cType, &cValueNum, &cvalue) + if ret != C.TILEDB_OK { + return nil, fmt.Errorf("Error getting metadata from group: %s, index: %d", g.context.LastError(), index) + } + + valueNum := uint(cValueNum) + if valueNum == 0 { + return nil, fmt.Errorf("Error getting metadata from group, Index: %d does not exist", index) + } + + datatype := Datatype(cType) + if limit != nil && valueNum > *limit { + valueNum = *limit + } + value, err := datatype.GetValue(valueNum, cvalue) + if err != nil { + return nil, fmt.Errorf("%s, Index: %d", err.Error(), index) + } + + groupMetadata := GroupMetadata{ + Key: C.GoString(cKey), + KeyLen: uint32(cKeyLen), + Datatype: datatype, + ValueNum: valueNum, + Value: value, + } + + return &groupMetadata, nil +} + +func (g *Group) Dump(recurse bool) (string, error) { + var cOutput *C.char + defer C.free(unsafe.Pointer(cOutput)) + + var cRecurse C.uint8_t + if recurse { + cRecurse = 1 + } + + ret := C.tiledb_group_dump_str(g.context.tiledbContext, g.group, &cOutput, cRecurse) + if ret != C.TILEDB_OK { + return "", fmt.Errorf("Error dumping group contents: %s", g.context.LastError()) + } + + return C.GoString(cOutput), nil +} + +// SerializeGroupMetadata gets and serializes the group metadata +func SerializeGroupMetadata(g *Group, serializationType SerializationType) (*Buffer, error) { + buffer := Buffer{context: g.context} + // Set finalizer for free C pointer on gc + runtime.SetFinalizer(&buffer, func(buffer *Buffer) { + buffer.Free() + }) + + ret := C.tiledb_serialize_group_metadata(g.context.tiledbContext, g.group, C.tiledb_serialization_type_t(serializationType), &buffer.tiledbBuffer) + if ret != C.TILEDB_OK { + return nil, fmt.Errorf("Error serializing group metadata: %s", g.context.LastError()) + } + + b, err := buffer.Data() + if err != nil { + return nil, errors.New("failed to retrieve bytes from buffer") + } + // cstrings are null terminated. Go's are not, remove the suffix if it exists + if err := buffer.SetBuffer(bytes.TrimSuffix(b, []byte("\u0000"))); err != nil { + return nil, errors.New("failed to remove null terminator from buffer") + } + + return &buffer, nil +} + +// DeserializeGroupMetadata deserializes group metadata +func DeserializeGroupMetadata(g *Group, buffer *Buffer, serializationType SerializationType) error { + b, err := buffer.Data() + if err != nil { + return errors.New("failed to retrieve bytes from buffer") + } + // cstrings are null terminated. Go's are not, add it as a suffix + if err := buffer.SetBuffer(append(b, []byte("\u0000")...)); err != nil { + return errors.New("failed to add null terminator to buffer") + } + + ret := C.tiledb_deserialize_group_metadata(g.context.tiledbContext, g.group, C.tiledb_serialization_type_t(serializationType), buffer.tiledbBuffer) + if ret != C.TILEDB_OK { + return fmt.Errorf("Error deserializing group metadata: %s", g.context.LastError()) + } + return nil } diff --git a/group_test.go b/group_test.go index 34171601..b1f64f37 100644 --- a/group_test.go +++ b/group_test.go @@ -1,6 +1,10 @@ +//go:build experimental +// +build experimental + package tiledb import ( + "strconv" "testing" "github.com/stretchr/testify/assert" @@ -8,7 +12,6 @@ import ( ) func TestGroupCreate(t *testing.T) { - // Test context without config context, err := NewContext(nil) require.NoError(t, err) @@ -17,24 +20,254 @@ func TestGroupCreate(t *testing.T) { tmpGroup := t.TempDir() // Create initial group - require.NoError(t, GroupCreate(context, tmpGroup)) + group, err := NewGroup(context, tmpGroup) + require.NoError(t, err) + require.NoError(t, group.Create()) // Creating the same group twice should error - assert.Error(t, GroupCreate(context, tmpGroup)) + group, err = NewGroup(context, tmpGroup) + require.NoError(t, err) + assert.Error(t, group.Create()) + } -func ExampleGroupCreate() { - // Create context without config - context, err := NewContext(nil) +func TestGroups_Metadata(t *testing.T) { + tdbCtx, err := NewContext(nil) + require.NoError(t, err) + + group, err := createTestGroup(tdbCtx, t.TempDir()) + require.NoError(t, err) + + // ========================================================================= + // Test adding metadata + require.NoError(t, setConfigForWrite(group, 0)) + require.NoError(t, group.Open(TILEDB_WRITE)) + require.NoError(t, group.PutMetadata("key", "value")) + require.NoError(t, group.Close()) + + // ========================================================================= + // Verify it is added + require.NoError(t, group.Open(TILEDB_READ)) + num, err := group.GetMetadataNum() + require.NoError(t, err) + assert.EqualValues(t, uint64(1), num) + + dType, _, val, err := group.GetMetadata("key") + require.NoError(t, err) + assert.EqualValues(t, dType, TILEDB_STRING_UTF8) + assert.EqualValues(t, val, "value") + require.NoError(t, group.Close()) + + // ========================================================================= + // Remove it + require.NoError(t, setConfigForWrite(group, 1)) + require.NoError(t, group.Open(TILEDB_WRITE)) + err = group.DeleteMetadata("key") + require.NoError(t, err) + require.NoError(t, group.Close()) + + require.NoError(t, group.Open(TILEDB_READ)) + num, err = group.GetMetadataNum() + require.NoError(t, err) + assert.EqualValues(t, uint64(0), num) + require.NoError(t, group.Close()) +} + +func TestGroups_AddMembers(t *testing.T) { + tdbCtx, err := NewContext(nil) + require.NoError(t, err) + + group, err := createTestGroup(tdbCtx, t.TempDir()) + require.NoError(t, err) + + // ========================================================================= + // Test adding members to the group + arraySchema := buildArraySchema(tdbCtx, t) + require.NoError(t, addTwoArraysToGroup(tdbCtx, group, arraySchema, t.TempDir(), t.TempDir())) + + // verify we have two arrays + count, err := memberCount(group) + require.NoError(t, err) + assert.EqualValues(t, uint(2), count) +} + +func TestGroups_RemoveMembers(t *testing.T) { + tdbCtx, err := NewContext(nil) + require.NoError(t, err) + + group, err := createTestGroup(tdbCtx, t.TempDir()) + require.NoError(t, err) + + arraySchema := buildArraySchema(tdbCtx, t) + arrayPathToKeep, arrayPathToRemove := t.TempDir(), t.TempDir() + require.NoError(t, addTwoArraysToGroup(tdbCtx, group, arraySchema, arrayPathToKeep, arrayPathToRemove)) + + // verify we have two arrays + count, err := memberCount(group) + require.NoError(t, err) + require.EqualValues(t, 2, count) + + // ========================================================================= + // Remove the members and validate + require.NoError(t, setConfigForWrite(group, 1)) + require.NoError(t, group.Open(TILEDB_WRITE)) + require.NoError(t, group.RemoveMember(arrayPathToRemove)) + require.NoError(t, group.Close()) + + count, err = memberCount(group) + require.NoError(t, err) + require.EqualValues(t, uint64(1), count) + + require.NoError(t, group.Open(TILEDB_READ)) + uri, name, objectType, err := group.GetMemberFromIndex(0) + require.NoError(t, err) + assert.EqualValues(t, "file://"+arrayPathToKeep, uri) + assert.EqualValues(t, objectType, TILEDB_ARRAY) + assert.EqualValues(t, name, arrayPathToKeep) + require.NoError(t, group.Close()) +} + +func TestGetMemberByName(t *testing.T) { + tdbCtx, err := NewContext(nil) + require.NoError(t, err) + + group, err := createTestGroup(tdbCtx, t.TempDir()) + require.NoError(t, err) + + arraySchema := buildArraySchema(tdbCtx, t) + arrayPath1, arrayPath2 := t.TempDir(), t.TempDir() + require.NoError(t, addTwoArraysToGroup(tdbCtx, group, arraySchema, arrayPath1, arrayPath2)) + + require.NoError(t, group.Open(TILEDB_READ)) + uri, name, objectType, err := group.GetMemberByName(arrayPath1) + require.NoError(t, err) + assert.EqualValues(t, "file://"+arrayPath1, uri) + assert.EqualValues(t, objectType, TILEDB_ARRAY) + assert.EqualValues(t, name, arrayPath1) + require.NoError(t, group.Close()) +} + +func TestDeserializeGroup(t *testing.T) { + tdbCtx, err := NewContext(nil) + if err != nil { + t.Fatal(err) + } + + buffer, err := NewBuffer(tdbCtx) + if err != nil { + t.Fatal(err) + } + + g, err := NewGroup(tdbCtx, t.TempDir()) + if err != nil { + t.Fatal(err) + } + + if err := setConfigForWrite(g, 0); err != nil { + t.Fatal(err) + } + + require.NoError(t, g.Create()) + + require.NoError(t, g.Open(TILEDB_WRITE)) + if err := buffer.SetBuffer([]byte(`{ + "group": { + "members": [ + {"uri": "tiledb://namespace/name", "type": "ARRAY", "name": "array1"}, + {"uri": "tiledb://namespace/name2", "type": "GROUP", "name": "group1"} + ] + } +}`)); err != nil { + t.Fatal(err) + } + if err := g.Deserialize(buffer, TILEDB_JSON, true); err != nil { + t.Fatalf("DeserializeGroup -> %v; expected no err", err) + } + require.NoError(t, g.Close()) + + count, err := memberCount(g) + require.NoError(t, err) + require.EqualValues(t, uint64(2), count) +} + +func memberCount(group *Group) (uint64, error) { + if err := group.Open(TILEDB_READ); err != nil { + return 0, err + } + count, err := group.GetMemberCount() if err != nil { - // Handle error - return + return 0, err } - // Create Group - err = GroupCreate(context, "my_group") + if err := group.Close(); err != nil { + return 0, err + } + + return count, nil +} + +func createTestGroup(tdbCtx *Context, uri string) (*Group, error) { + // Create initial group + group, err := NewGroup(tdbCtx, uri) if err != nil { - // Handle error - return + return nil, err + } + + if err := group.Create(); err != nil { + return nil, err + } + return group, nil +} + +func addTwoArraysToGroup(tdbCtx *Context, group *Group, arraySchema *ArraySchema, arrayURI1, arrayURI2 string) error { + array1, err := NewArray(tdbCtx, arrayURI1) + if err != nil { + return err + } + + if err := array1.Create(arraySchema); err != nil { + return err + } + + array2, err := NewArray(tdbCtx, arrayURI2) + if err != nil { + return err + } + + if err := array2.Create(arraySchema); err != nil { + return err + } + + if err := setConfigForWrite(group, 0); err != nil { + return err + } + + if err := group.Open(TILEDB_WRITE); err != nil { + return err + } + + if err := group.AddMember(array1.uri, arrayURI1, false); err != nil { + return err + } + + if err := group.AddMember(array2.uri, arrayURI2, false); err != nil { + return err + } + + return group.Close() +} + +func setConfigForWrite(group *Group, i int) error { + conf, err := NewConfig() + if err != nil { + return err + } + if err := conf.Set("sm.group.timestamp_end", strconv.Itoa(1648581656+i)); err != nil { + return err + } + + if err := group.SetConfig(conf); err != nil { + return err } + return nil } diff --git a/object_test.go b/object_test.go deleted file mode 100644 index d3231ed6..00000000 --- a/object_test.go +++ /dev/null @@ -1,79 +0,0 @@ -package tiledb - -import ( - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestObjectCreate(t *testing.T) { - // Create context - context, err := NewContext(nil) - require.NoError(t, err) - - // create temp group name - groupPath := t.TempDir() - - groupPathNew := t.TempDir() - - // Create initial group - require.NoError(t, GroupCreate(context, groupPath)) - - objType, err := ObjectType(context, groupPath) - require.NoError(t, err) - assert.Equal(t, TILEDB_GROUP, objType) - - require.NoError(t, ObjectMove(context, groupPath, groupPathNew)) - - require.NoError(t, ObjectRemove(context, groupPathNew)) -} - -func TestObjectArray(t *testing.T) { - // Create context - context, err := NewContext(nil) - require.NoError(t, err) - - // create temp group name - groupPath := t.TempDir() - - // Create initial group - require.NoError(t, GroupCreate(context, groupPath)) - - arrayGroup := filepath.Join(groupPath, "arrays") - - // Create the array group - require.NoError(t, GroupCreate(context, arrayGroup)) - - tmpArrayPath := filepath.Join(arrayGroup, "tiledb_test_array") - - // Create new array struct - array, err := NewArray(context, tmpArrayPath) - require.NoError(t, err) - assert.NotNil(t, array) - - arraySchema := buildArraySchema(context, t) - - // Create array on disk - require.NoError(t, array.Create(arraySchema)) - - objType, err := ObjectType(context, groupPath) - require.NoError(t, err) - assert.Equal(t, TILEDB_GROUP, objType) - - objType, err = ObjectType(context, tmpArrayPath) - require.NoError(t, err) - assert.Equal(t, TILEDB_ARRAY, objType) - - objectList, err := ObjectWalk(context, groupPath, TILEDB_PREORDER) - require.NoError(t, err) - assert.Equal(t, 2, len(objectList.objectList)) - assert.Equal(t, TILEDB_GROUP, objectList.objectList[0].objectTypeEnum) - assert.Equal(t, TILEDB_ARRAY, objectList.objectList[1].objectTypeEnum) - - objectList, err = ObjectLs(context, groupPath) - require.NoError(t, err) - assert.Equal(t, 1, len(objectList.objectList)) - assert.Equal(t, TILEDB_GROUP, objectList.objectList[0].objectTypeEnum) -}