From 990f2db1eccee0f5080bb319b8af5c40eabacfa1 Mon Sep 17 00:00:00 2001 From: Stefan Naglee Date: Mon, 28 Mar 2022 18:20:26 -0400 Subject: [PATCH 01/15] Use tiledb 2.8 --- .github/scripts/install_tiledb_linux.sh | 2 +- .github/scripts/install_tiledb_linux_debug.sh | 2 +- .github/scripts/install_tiledb_macos.sh | 2 +- .github/scripts/install_tiledb_source_linux.sh | 2 +- .github/scripts/install_tiledb_source_macos.sh | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/scripts/install_tiledb_linux.sh b/.github/scripts/install_tiledb_linux.sh index b478f20c..bef96c0e 100755 --- a/.github/scripts/install_tiledb_linux.sh +++ b/.github/scripts/install_tiledb_linux.sh @@ -1,4 +1,4 @@ set -e -x -curl --location -o tiledb.tar.gz https://github.com/TileDB-Inc/TileDB/releases/download/2.6.1/tiledb-linux-x86_64-2.6.1-2f6b7f6.tar.gz \ +curl --location -o tiledb.tar.gz https://github.com/TileDB-Inc/TileDB/releases/download/2.8.0/tiledb-linux-x86_64-2.8.0-f8efd39.tar.gz \ && sudo tar -C /usr/local -xf tiledb.tar.gz sudo ldconfig /usr/local/lib diff --git a/.github/scripts/install_tiledb_linux_debug.sh b/.github/scripts/install_tiledb_linux_debug.sh index 7708e534..59272bef 100755 --- a/.github/scripts/install_tiledb_linux_debug.sh +++ b/.github/scripts/install_tiledb_linux_debug.sh @@ -1,5 +1,5 @@ set -e -x -git clone https://github.com/TileDB-Inc/TileDB.git -b 2.6.1 +git clone https://github.com/TileDB-Inc/TileDB.git -b 2.8.0 cd TileDB mkdir build && cd build cmake -DSANITIZER=leak -DTILEDB_VERBOSE=OFF -DTILEDB_S3=ON -DTILEDB_SERIALIZATION=ON -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=/usr/local .. diff --git a/.github/scripts/install_tiledb_macos.sh b/.github/scripts/install_tiledb_macos.sh index 3164f4f0..22aae174 100755 --- a/.github/scripts/install_tiledb_macos.sh +++ b/.github/scripts/install_tiledb_macos.sh @@ -1,3 +1,3 @@ set -e -x -curl --location -o tiledb.tar.gz https://github.com/TileDB-Inc/TileDB/releases/download/2.6.1/tiledb-macos-x86_64-2.6.1-2f6b7f6.tar.gz \ +curl --location -o tiledb.tar.gz https://github.com/TileDB-Inc/TileDB/releases/download/2.8.0/tiledb-macos-x86_64-2.8.0-f8efd39.tar.gz \ && sudo tar -C /usr/local -xf tiledb.tar.gz diff --git a/.github/scripts/install_tiledb_source_linux.sh b/.github/scripts/install_tiledb_source_linux.sh index 7ec5c17c..a5c29ed5 100755 --- a/.github/scripts/install_tiledb_source_linux.sh +++ b/.github/scripts/install_tiledb_source_linux.sh @@ -1,5 +1,5 @@ set -e -x -git clone https://github.com/TileDB-Inc/TileDB.git -b 2.6.1 +git clone https://github.com/TileDB-Inc/TileDB.git -b 2.8.0 cd TileDB mkdir build && cd build cmake -DTILEDB_VERBOSE=OFF -DTILEDB_S3=ON -DTILEDB_SERIALIZATION=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local .. diff --git a/.github/scripts/install_tiledb_source_macos.sh b/.github/scripts/install_tiledb_source_macos.sh index 3f0960da..06b18918 100755 --- a/.github/scripts/install_tiledb_source_macos.sh +++ b/.github/scripts/install_tiledb_source_macos.sh @@ -1,5 +1,5 @@ set -e -x -git clone https://github.com/TileDB-Inc/TileDB.git -b 2.6.1 +git clone https://github.com/TileDB-Inc/TileDB.git -b 2.8.0 cd TileDB mkdir build && cd build cmake -DTILEDB_VERBOSE=OFF -DTILEDB_S3=ON -DTILEDB_SERIALIZATION=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local .. From 7273a22dee12142c34cda833967bfe024eddc612 Mon Sep 17 00:00:00 2001 From: Stefan Naglee Date: Mon, 28 Mar 2022 16:26:13 -0400 Subject: [PATCH 02/15] Add group allocation, creation, frees, close, open, and config functions --- examples_lib/errors.go | 74 --------------------------- group.go | 113 +++++++++++++++++++++++++++++++++++------ group_test.go | 28 ++++------ object_test.go | 33 +++--------- 4 files changed, 115 insertions(+), 133 deletions(-) delete mode 100644 examples_lib/errors.go diff --git a/examples_lib/errors.go b/examples_lib/errors.go deleted file mode 100644 index 3a0de609..00000000 --- a/examples_lib/errors.go +++ /dev/null @@ -1,74 +0,0 @@ -package examples_lib - -import ( - "fmt" - "path/filepath" - "runtime" - "strings" - - tiledb "github.com/TileDB-Inc/TileDB-Go" -) - -// Name of the group -const groupName = "my_group" - -// Type of file system (e.g. file://, s3://) -const fileSystem = "file://" - -func RunErrors() { - // Get filename of current file - - // uncomment to use local filesystem - _, filename, _, _ := runtime.Caller(0) - pathName := filepath.Dir(filename) - - // uncomment to use s3 bucket - //pathName := "test-bucket" - - // Construct the path - groupPathName := - fmt.Sprintf("%s%s/%s", fileSystem, pathName, groupName) - - // Create config - config, err := tiledb.NewConfig() - checkError(err) - defer config.Free() - - // Create a TileDB context. - ctx, err := tiledb.NewContext(config) - checkError(err) - defer ctx.Free() - - // Create vfs - vfs, err := tiledb.NewVFS(ctx, config) - checkError(err) - defer vfs.Free() - - // Find out if dir exists having group name - isDir, err := vfs.IsDir(groupName) - checkError(err) - - // If it exists delete it to start clean - if isDir { - // For local filesystem it suffices to replace groupPathName with - // groupName since vfs can infer local directory - err = vfs.RemoveDir(groupPathName) - checkError(err) - } - - err = tiledb.GroupCreate(ctx, groupPathName) - checkError(err) - //There cannot be two groups having the same name - err = tiledb.GroupCreate(ctx, groupPathName) - - if err != nil { - if strings.Contains(err.Error(), "already exists") { - fmt.Println("[TileDB::StorageManager] Error: Cannot create group") - fmt.Println("Group already exists") - } - } - - // Clean up - err = vfs.RemoveDir(groupPathName) - checkError(err) -} diff --git a/group.go b/group.go index ae6f7fd1..79db9505 100644 --- a/group.go +++ b/group.go @@ -1,27 +1,110 @@ -package tiledb +//go:build experimental +// +build experimental -/* -#cgo LDFLAGS: -ltiledb -#cgo linux LDFLAGS: -ldl -#include -#include -*/ -import "C" +// This file declares Go bindings for experimental features in TileDB. +// Experimental APIs to do not fall under the API compatibility guarantees and +// might change between TileDB versions + +package tiledb import ( "fmt" + "runtime" "unsafe" ) -// GroupCreate creates a new tiledb group. A Group is a logical grouping -// of Objects on the storage system (a directory). -func GroupCreate(context *Context, group string) error { - cgroup := C.CString(group) - defer C.free(unsafe.Pointer(cgroup)) +/* + #cgo LDFLAGS: -ltiledb + #cgo linux LDFLAGS: -ldl + #include + #include +*/ +import "C" + +// Group represents a wrapped TileDB embedded group +type Group struct { + group *C.tiledb_group_t + uri string + context *Context + config *Config +} + +// NewGroup allocates an embedded group +func NewGroup(tdbCtx *Context, uri string) (*Group, error) { + curi := C.CString(uri) + defer C.free(unsafe.Pointer(curi)) + group := Group{context: tdbCtx, uri: uri} + ret := C.tiledb_group_alloc(group.context.tiledbContext, curi, &group.group) + if ret != C.TILEDB_OK { + return nil, fmt.Errorf("Error creating tiledb group: %s", group.context.LastError()) + } + + // Set finalizer for free C pointer on gc + runtime.SetFinalizer(&group, func(group *Group) { + group.Free() + }) + + return &group, nil +} + +// Create a new TileDB group +func (g *Group) Create() error { + curi := C.CString(g.uri) + defer C.free(unsafe.Pointer(curi)) + + ret := C.tiledb_group_create(g.context.tiledbContext, curi) + if ret != C.TILEDB_OK { + return fmt.Errorf("Error in creating group: %s", g.context.LastError()) + } + return nil +} + +func (g *Group) Open(queryType QueryType) error { + ret := C.tiledb_group_open(g.context.tiledbContext, g.group, C.tiledb_query_type_t(queryType)) + if ret != C.TILEDB_OK { + return fmt.Errorf("Error opening tiledb group for querying: %s", g.context.LastError()) + } + return nil +} + +func (g *Group) Free() { + if g.group != nil { + g.Close() + C.tiledb_group_free(&g.group) + } +} + +func (g *Group) Close() error { + ret := C.tiledb_group_close(g.context.tiledbContext, g.group) + if ret != C.TILEDB_OK { + return fmt.Errorf("Error closing tiledb group: %s", g.context.LastError()) + } + return nil +} - ret := C.tiledb_group_create(context.tiledbContext, cgroup) +func (g *Group) SetConfig(config *Config) error { + ret := C.tiledb_group_set_config(g.context.tiledbContext, g.group, g.config.tiledbConfig) if ret != C.TILEDB_OK { - return fmt.Errorf("Error in creating group %s: %s", group, context.LastError()) + return fmt.Errorf("Error setting config on group: %s", g.context.LastError()) } + g.config = config return nil } + +func (g *Group) Config() (*Config, error) { + var config Config + ret := C.tiledb_group_get_config(g.context.tiledbContext, g.group, &config.tiledbConfig) + if ret != C.TILEDB_OK { + return nil, fmt.Errorf("Error getting config from query: %s", g.context.LastError()) + } + + runtime.SetFinalizer(&config, func(config *Config) { + config.Free() + }) + + if g.config == nil { + g.config = &config + } + + return &config, nil +} diff --git a/group_test.go b/group_test.go index 34171601..68232b44 100644 --- a/group_test.go +++ b/group_test.go @@ -1,3 +1,6 @@ +//go:build experimental +// +build experimental + package tiledb import ( @@ -8,7 +11,6 @@ import ( ) func TestGroupCreate(t *testing.T) { - // Test context without config context, err := NewContext(nil) require.NoError(t, err) @@ -17,24 +19,12 @@ func TestGroupCreate(t *testing.T) { tmpGroup := t.TempDir() // Create initial group - require.NoError(t, GroupCreate(context, tmpGroup)) + group, err := NewGroup(context, tmpGroup) + require.NoError(t, err) + require.NoError(t, group.Create()) // Creating the same group twice should error - assert.Error(t, GroupCreate(context, tmpGroup)) -} - -func ExampleGroupCreate() { - // Create context without config - context, err := NewContext(nil) - if err != nil { - // Handle error - return - } - - // Create Group - err = GroupCreate(context, "my_group") - if err != nil { - // Handle error - return - } + group, err = NewGroup(context, tmpGroup) + require.NoError(t, err) + assert.Error(t, group.Create()) } diff --git a/object_test.go b/object_test.go index d3231ed6..b18de890 100644 --- a/object_test.go +++ b/object_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestObjectCreate(t *testing.T) { +func TestObjectArray(t *testing.T) { // Create context context, err := NewContext(nil) require.NoError(t, err) @@ -16,37 +16,20 @@ func TestObjectCreate(t *testing.T) { // create temp group name groupPath := t.TempDir() - groupPathNew := t.TempDir() - - // Create initial group - require.NoError(t, GroupCreate(context, groupPath)) - - objType, err := ObjectType(context, groupPath) - require.NoError(t, err) - assert.Equal(t, TILEDB_GROUP, objType) - - require.NoError(t, ObjectMove(context, groupPath, groupPathNew)) - - require.NoError(t, ObjectRemove(context, groupPathNew)) -} - -func TestObjectArray(t *testing.T) { - // Create context - context, err := NewContext(nil) + group, err := NewGroup(context, groupPath) require.NoError(t, err) - // create temp group name - groupPath := t.TempDir() - // Create initial group - require.NoError(t, GroupCreate(context, groupPath)) + require.NoError(t, group.Create()) - arrayGroup := filepath.Join(groupPath, "arrays") + arrayGroupPath := filepath.Join(groupPath, "arrays") + arrayGroup, err := NewGroup(context, groupPath) + require.NoError(t, err) // Create the array group - require.NoError(t, GroupCreate(context, arrayGroup)) + require.NoError(t, arrayGroup.Create()) - tmpArrayPath := filepath.Join(arrayGroup, "tiledb_test_array") + tmpArrayPath := filepath.Join(arrayGroupPath, "tiledb_test_array") // Create new array struct array, err := NewArray(context, tmpArrayPath) From 56107e05eda66e33fc2859ca30badf08a679ec9f Mon Sep 17 00:00:00 2001 From: Stefan Naglee Date: Mon, 28 Mar 2022 17:59:54 -0400 Subject: [PATCH 03/15] Add member functions --- group.go | 54 ++++++++++++++++++++- group_test.go | 128 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 181 insertions(+), 1 deletion(-) diff --git a/group.go b/group.go index 79db9505..af01ac7b 100644 --- a/group.go +++ b/group.go @@ -83,7 +83,7 @@ func (g *Group) Close() error { } func (g *Group) SetConfig(config *Config) error { - ret := C.tiledb_group_set_config(g.context.tiledbContext, g.group, g.config.tiledbConfig) + ret := C.tiledb_group_set_config(g.context.tiledbContext, g.group, config.tiledbConfig) if ret != C.TILEDB_OK { return fmt.Errorf("Error setting config on group: %s", g.context.LastError()) } @@ -108,3 +108,55 @@ func (g *Group) Config() (*Config, error) { return &config, nil } + +func (g *Group) AddMember(uri string, isRelativeURI bool) error { + curi := C.CString(uri) + defer C.free(unsafe.Pointer(curi)) + var cRelative C.uint8_t + if isRelativeURI { + cRelative = 1 + } + + ret := C.tiledb_group_add_member(g.context.tiledbContext, g.group, curi, cRelative) + if ret != C.TILEDB_OK { + return fmt.Errorf("Error adding member to group: %s", g.context.LastError()) + } + return nil +} + +func (g *Group) RemoveMember(uri string) error { + curi := C.CString(uri) + defer C.free(unsafe.Pointer(curi)) + ret := C.tiledb_group_remove_member(g.context.tiledbContext, g.group, curi) + if ret != C.TILEDB_OK { + return fmt.Errorf("Error removing member from group: %s", g.context.LastError()) + } + return nil +} + +func (g *Group) GetMemberCount() (uint64, error) { + var count C.uint64_t + ret := C.tiledb_group_get_member_count(g.context.tiledbContext, g.group, &count) + if ret != C.TILEDB_OK { + return 0, fmt.Errorf("Error retrieving member count in group: %s", g.context.LastError()) + } + return count, nil +} + +func (g *Group) GetMemberFromIndex(index uint64) (string, ObjectTypeEnum, error) { + var curi *C.char + defer C.free(unsafe.Pointer(curi)) + + var objectTypeEnum C.tiledb_object_t + ret := C.tiledb_group_get_member_by_index(g.context.tiledbContext, g.group, C.uint64_t(index), curi, &objectTypeEnum) + if ret != C.TILEDB_OK { + return "", TILEDB_INVALID, fmt.Errorf("Error getting member by index for group: %s", g.context.LastError()) + } + + uri := C.GoString(curi) + if uri == "" { + return "", TILEDB_INVALID, fmt.Errorf("Error getting URI for member %d: uri is empty", index) + } + + return uri, ObjectTypeEnum(objectTypeEnum), nil +} diff --git a/group_test.go b/group_test.go index 68232b44..f79ed7aa 100644 --- a/group_test.go +++ b/group_test.go @@ -4,7 +4,9 @@ package tiledb import ( + "strconv" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -28,3 +30,129 @@ func TestGroupCreate(t *testing.T) { require.NoError(t, err) assert.Error(t, group.Create()) } + +func TestGroups_AddMembers(t *testing.T) { + tdbCtx, err := NewContext(nil) + require.NoError(t, err) + + group, err := createTestGroup(tdbCtx, t.TempDir()) + require.NoError(t, err) + + // ========================================================================= + // Test adding members to the group + arraySchema := buildArraySchema(tdbCtx, t) + require.NoError(t, addTwoArraysToGroup(tdbCtx, group, arraySchema, t.TempDir(), t.TempDir())) + + // verify we have two arrays + count, err := memberCount(group) + require.NoError(t, err) + assert.EqualValues(t, uint(2), count) + require.NoError(t, group.Close()) +} + +func TestGroups_RemoveMembers(t *testing.T) { + tdbCtx, err := NewContext(nil) + require.NoError(t, err) + + group, err := createTestGroup(tdbCtx, t.TempDir()) + require.NoError(t, err) + + arraySchema := buildArraySchema(tdbCtx, t) + arrayPathToKeep, arrayPathToRemove := t.TempDir(), t.TempDir() + require.NoError(t, addTwoArraysToGroup(tdbCtx, group, arraySchema, arrayPathToKeep, arrayPathToRemove)) + + // verify we have two arrays + count, err := memberCount(group) + require.NoError(t, err) + require.EqualValues(t, 2, count) + + // ========================================================================= + // Remove the members and validate + require.NoError(t, group.Open(TILEDB_WRITE)) + require.NoError(t, group.RemoveMember(arrayPathToRemove)) + require.NoError(t, group.Close()) + + count, err = memberCount(group) + require.NoError(t, err) + require.EqualValues(t, uint64(1), count) + + uri, objectType, err := group.GetMemberFromIndex(1) + require.NoError(t, err) + assert.EqualValues(t, uri, arrayPathToKeep) + assert.EqualValues(t, objectType, TILEDB_ARRAY) +} + +func memberCount(group *Group) (uint64, error) { + conf, err := NewConfig() + if err != nil { + return 0, err + } + if err := conf.Set("sm.group.timestamp_start", strconv.Itoa(int(time.Now().UnixMilli()))); err != nil { + return 0, err + } + + if err := group.SetConfig(conf); err != nil { + return 0, err + } + + if err := group.Open(TILEDB_READ); err != nil { + return 0, err + } + count, err := group.GetMemberCount() + if err != nil { + return 0, err + } + + if err := group.Close(); err != nil { + return 0, err + } + + return count, nil +} + +func createTestGroup(tdbCtx *Context, uri string) (*Group, error) { + // Create initial group + group, err := NewGroup(tdbCtx, uri) + if err != nil { + return nil, err + } + + if err := group.Create(); err != nil { + return nil, err + } + return group, nil +} + +func addTwoArraysToGroup(tdbCtx *Context, group *Group, arraySchema *ArraySchema, arrayURI1, arrayURI2 string) error { + array1, err := NewArray(tdbCtx, arrayURI1) + if err != nil { + return err + } + + if err := array1.Create(arraySchema); err != nil { + return err + } + + array2, err := NewArray(tdbCtx, arrayURI2) + if err != nil { + return err + } + + if err := array2.Create(arraySchema); err != nil { + return err + } + + if err := group.Open(TILEDB_WRITE); err != nil { + return err + } + + if err := group.AddMember(array1.uri, false); err != nil { + return err + } + + if err := group.AddMember(array2.uri, false); err != nil { + return err + } + + return group.Close() +} From 3e5f797bc620dd1cdf1c4fd9395557918fd78231 Mon Sep 17 00:00:00 2001 From: Stefan Naglee Date: Mon, 28 Mar 2022 18:05:48 -0400 Subject: [PATCH 04/15] Add dump helper functions --- group.go | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/group.go b/group.go index af01ac7b..6e4fc00e 100644 --- a/group.go +++ b/group.go @@ -140,7 +140,7 @@ func (g *Group) GetMemberCount() (uint64, error) { if ret != C.TILEDB_OK { return 0, fmt.Errorf("Error retrieving member count in group: %s", g.context.LastError()) } - return count, nil + return uint64(count), nil } func (g *Group) GetMemberFromIndex(index uint64) (string, ObjectTypeEnum, error) { @@ -148,7 +148,7 @@ func (g *Group) GetMemberFromIndex(index uint64) (string, ObjectTypeEnum, error) defer C.free(unsafe.Pointer(curi)) var objectTypeEnum C.tiledb_object_t - ret := C.tiledb_group_get_member_by_index(g.context.tiledbContext, g.group, C.uint64_t(index), curi, &objectTypeEnum) + ret := C.tiledb_group_get_member_by_index(g.context.tiledbContext, g.group, C.uint64_t(index), &curi, &objectTypeEnum) if ret != C.TILEDB_OK { return "", TILEDB_INVALID, fmt.Errorf("Error getting member by index for group: %s", g.context.LastError()) } @@ -160,3 +160,20 @@ func (g *Group) GetMemberFromIndex(index uint64) (string, ObjectTypeEnum, error) return uri, ObjectTypeEnum(objectTypeEnum), nil } + +func (g *Group) Dump(recurse bool) (string, error) { + var cOutput *C.char + defer C.free(unsafe.Pointer(cOutput)) + + var cRecurse C.uint8_t + if recurse { + cRecurse = 1 + } + + ret := C.tiledb_group_dump_str(g.context.tiledbContext, g.group, &cOutput, cRecurse) + if ret != C.TILEDB_OK { + return "", fmt.Errorf("Error dumping group contents: %s", g.context.LastError()) + } + + return C.GoString(cOutput), nil +} From 60fe794bd46c569fb385ba835e43f4d24d1ef2aa Mon Sep 17 00:00:00 2001 From: Stefan Naglee Date: Mon, 28 Mar 2022 17:21:36 -0400 Subject: [PATCH 05/15] Add group metadata APIs --- group.go | 298 ++++++++++++++++++++++++++++++++++++++++++++++++++ group_test.go | 35 ++++++ 2 files changed, 333 insertions(+) diff --git a/group.go b/group.go index 6e4fc00e..c6d50bd3 100644 --- a/group.go +++ b/group.go @@ -8,8 +8,11 @@ package tiledb import ( + "encoding/json" "fmt" + "reflect" "runtime" + "strconv" "unsafe" ) @@ -124,6 +127,208 @@ func (g *Group) AddMember(uri string, isRelativeURI bool) error { return nil } +// GroupMetadata defines metadata for the group +type GroupMetadata struct { + Key string + KeyLen uint32 + Datatype Datatype + ValueNum uint + Value interface{} +} + +// MarshalJSON implements the Marshaller interface for GroupMetadata +func (g GroupMetadata) MarshalJSON() ([]byte, error) { + switch v := g.Value.(type) { + case []byte: + return json.Marshal(string(v)) + default: + return json.Marshal(v) + } +} + +// PutMetadata puts a metadata key-value item to an open group. The group must +// be opened in WRITE mode, otherwise the function will error out. +func (g *Group) PutMetadata(key string, value interface{}) error { + ckey := C.CString(key) + defer C.free(unsafe.Pointer(ckey)) + + var isSliceValue bool = false + if reflect.TypeOf(value).Kind() == reflect.Slice { + isSliceValue = true + } + + var datatype Datatype + var valueNum C.uint + var valueType reflect.Kind + + valueInterfaceVal := reflect.ValueOf(value) + if isSliceValue { + if valueInterfaceVal.Len() == 0 { + return fmt.Errorf("Value passed must be a non-empty slice, size of slice is: %d", valueInterfaceVal.Len()) + } + valueType = reflect.TypeOf(value).Elem().Kind() + valueNum = C.uint(valueInterfaceVal.Len()) + } else { + valueType = reflect.TypeOf(value).Kind() + valueNum = 1 + } + + var ret C.int32_t + switch valueType { + case reflect.Int: + // Check size of int on platform + if strconv.IntSize == 32 { + datatype = TILEDB_INT32 + if isSliceValue { + tmpValue := value.([]int32) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue[0])) + } else { + tmpValue := value.(int32) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue)) + } + } else { + datatype = TILEDB_INT64 + if isSliceValue { + tmpValue := value.([]int64) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue[0])) + } else { + tmpValue := value.(int64) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue)) + } + } + case reflect.Int8: + datatype = TILEDB_INT8 + if isSliceValue { + tmpValue := value.([]int8) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue[0])) + } else { + tmpValue := value.(int8) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue)) + } + case reflect.Int16: + datatype = TILEDB_INT16 + if isSliceValue { + tmpValue := value.([]int16) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue[0])) + } else { + tmpValue := value.(int16) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue)) + } + case reflect.Int32: + datatype = TILEDB_INT32 + if isSliceValue { + tmpValue := value.([]int32) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue[0])) + } else { + tmpValue := value.(int32) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue)) + } + case reflect.Int64: + datatype = TILEDB_INT64 + if isSliceValue { + tmpValue := value.([]int64) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue[0])) + } else { + tmpValue := value.(int64) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue)) + } + case reflect.Uint: + // Check size of uint on platform + if strconv.IntSize == 32 { + datatype = TILEDB_UINT32 + if isSliceValue { + tmpValue := value.([]uint32) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue[0])) + } else { + tmpValue := value.(uint32) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue)) + } + } else { + datatype = TILEDB_UINT64 + if isSliceValue { + tmpValue := value.([]uint64) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue[0])) + } else { + tmpValue := value.(uint64) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue)) + } + } + case reflect.Uint8: + datatype = TILEDB_UINT8 + if isSliceValue { + tmpValue := value.([]uint8) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue[0])) + } else { + tmpValue := value.(uint8) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue)) + } + case reflect.Uint16: + datatype = TILEDB_UINT16 + if isSliceValue { + tmpValue := value.([]uint16) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue[0])) + } else { + tmpValue := value.(uint16) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue)) + } + case reflect.Uint32: + datatype = TILEDB_UINT32 + if isSliceValue { + tmpValue := value.([]uint32) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue[0])) + } else { + tmpValue := value.(uint32) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue)) + } + case reflect.Uint64: + datatype = TILEDB_UINT64 + if isSliceValue { + tmpValue := value.([]uint64) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue[0])) + } else { + tmpValue := value.(uint64) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue)) + } + case reflect.Float32: + datatype = TILEDB_FLOAT32 + if isSliceValue { + tmpValue := value.([]float32) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue[0])) + } else { + tmpValue := value.(float32) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue)) + } + case reflect.Float64: + datatype = TILEDB_FLOAT64 + if isSliceValue { + tmpValue := value.([]float64) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue[0])) + } else { + tmpValue := value.(float64) + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(&tmpValue)) + } + case reflect.String: + datatype = TILEDB_STRING_UTF8 + stringValue := value.(string) + valueNum = C.uint(len(stringValue)) + cTmpValue := C.CString(stringValue) + defer C.free(unsafe.Pointer(cTmpValue)) + if valueNum > 0 { + ret = C.tiledb_group_put_metadata(g.context.tiledbContext, g.group, ckey, C.tiledb_datatype_t(datatype), valueNum, unsafe.Pointer(cTmpValue)) + } + default: + if isSliceValue { + return fmt.Errorf("Unrecognized value type passed: %s", valueInterfaceVal.Index(0).Kind().String()) + } + return fmt.Errorf("Unrecognized value type passed: %s", valueInterfaceVal.Kind().String()) + } + + if ret != C.TILEDB_OK { + return fmt.Errorf("Error adding metadata to group: %s", g.context.LastError()) + } + return nil +} + func (g *Group) RemoveMember(uri string) error { curi := C.CString(uri) defer C.free(unsafe.Pointer(curi)) @@ -161,6 +366,99 @@ func (g *Group) GetMemberFromIndex(index uint64) (string, ObjectTypeEnum, error) return uri, ObjectTypeEnum(objectTypeEnum), nil } +func (g *Group) GetMetadata(key string) (Datatype, uint, interface{}, error) { + ckey := C.CString(key) + defer C.free(unsafe.Pointer(ckey)) + + var cType C.tiledb_datatype_t + var cValueNum C.uint + var cvalue unsafe.Pointer + + ret := C.tiledb_group_get_metadata(g.context.tiledbContext, g.group, ckey, &cType, &cValueNum, &cvalue) + if ret != C.TILEDB_OK { + return 0, 0, nil, fmt.Errorf("Error getting metadata from group: %s, key: %s", g.context.LastError(), key) + } + + valueNum := uint(cValueNum) + if valueNum == 0 { + return 0, 0, nil, fmt.Errorf("Error getting metadata from group, key: %s does not exist", key) + } + + datatype := Datatype(cType) + value, err := datatype.GetValue(valueNum, cvalue) + if err != nil { + return 0, 0, nil, fmt.Errorf("%s, key: %s", err.Error(), key) + } + + return datatype, valueNum, value, nil +} + +func (g *Group) DeleteMetadata(key string) error { + ckey := C.CString(key) + defer C.free(unsafe.Pointer(ckey)) + + ret := C.tiledb_group_delete_metadata(g.context.tiledbContext, g.group, ckey) + if ret != C.TILEDB_OK { + return fmt.Errorf("Error deleting metadata from group: %s", g.context.LastError()) + } + return nil +} + +func (g *Group) GetMetadataNum() (uint64, error) { + var cNum C.uint64_t + + ret := C.tiledb_group_get_metadata_num(g.context.tiledbContext, g.group, &cNum) + if ret != C.TILEDB_OK { + return 0, fmt.Errorf("Error getting number of metadata from group: %s", g.context.LastError()) + } + + return uint64(cNum), nil +} + +func (g *Group) GetMetadataFromIndex(index uint64) (*GroupMetadata, error) { + return g.GetMetadataFromIndexWithValueLimit(index, nil) +} + +func (g *Group) GetMetadataFromIndexWithValueLimit(index uint64, limit *uint) (*GroupMetadata, error) { + var cKey *C.char + + var cIndex C.uint64_t = C.uint64_t(index) + var cType C.tiledb_datatype_t + var cKeyLen C.uint32_t + var cValueNum C.uint + var cvalue unsafe.Pointer + + ret := C.tiledb_group_get_metadata_from_index(g.context.tiledbContext, + g.group, cIndex, &cKey, &cKeyLen, &cType, &cValueNum, &cvalue) + if ret != C.TILEDB_OK { + return nil, fmt.Errorf("Error getting metadata from group: %s, index: %d", g.context.LastError(), index) + } + + valueNum := uint(cValueNum) + if valueNum == 0 { + return nil, fmt.Errorf("Error getting metadata from group, Index: %d does not exist", index) + } + + datatype := Datatype(cType) + if limit != nil && valueNum > *limit { + valueNum = *limit + } + value, err := datatype.GetValue(valueNum, cvalue) + if err != nil { + return nil, fmt.Errorf("%s, Index: %d", err.Error(), index) + } + + groupMetadata := GroupMetadata{ + Key: C.GoString(cKey), + KeyLen: uint32(cKeyLen), + Datatype: datatype, + ValueNum: valueNum, + Value: value, + } + + return &groupMetadata, nil +} + func (g *Group) Dump(recurse bool) (string, error) { var cOutput *C.char defer C.free(unsafe.Pointer(cOutput)) diff --git a/group_test.go b/group_test.go index f79ed7aa..3fd1b5f8 100644 --- a/group_test.go +++ b/group_test.go @@ -29,6 +29,41 @@ func TestGroupCreate(t *testing.T) { group, err = NewGroup(context, tmpGroup) require.NoError(t, err) assert.Error(t, group.Create()) + +} + +func TestGroups_Metadata(t *testing.T) { + tdbCtx, err := NewContext(nil) + require.NoError(t, err) + + group, err := createTestGroup(tdbCtx, t.TempDir()) + require.NoError(t, err) + + // ========================================================================= + // Test adding metadata + require.NoError(t, group.Open(TILEDB_WRITE)) + require.NoError(t, group.PutMetadata("key", "value")) + require.NoError(t, group.Close()) + + // ========================================================================= + // Verify it is added + require.NoError(t, group.Open(TILEDB_READ)) + num, err := group.GetMetadataNum() + require.NoError(t, err) + assert.EqualValues(t, uint64(1), num) + + dType, _, val, err := group.GetMetadata("key") + require.NoError(t, err) + assert.EqualValues(t, dType, TILEDB_STRING_UTF8) + assert.EqualValues(t, val, "value") + require.NoError(t, group.Close()) + + // ========================================================================= + // Remove it + require.NoError(t, group.Open(TILEDB_WRITE)) + err = group.DeleteMetadata("key") + require.NoError(t, err) + assert.EqualValues(t, uint64(0), num) } func TestGroups_AddMembers(t *testing.T) { From bcefbfa7089aa302661848ea09897dc686a9ad66 Mon Sep 17 00:00:00 2001 From: Stefan Naglee Date: Tue, 29 Mar 2022 16:32:14 -0400 Subject: [PATCH 06/15] Always write at specific incremented timestamps to prevent write/read race --- group_test.go | 45 +++++++++++++++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/group_test.go b/group_test.go index 3fd1b5f8..ee2870b8 100644 --- a/group_test.go +++ b/group_test.go @@ -6,7 +6,6 @@ package tiledb import ( "strconv" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -41,6 +40,7 @@ func TestGroups_Metadata(t *testing.T) { // ========================================================================= // Test adding metadata + require.NoError(t, setConfigForWrite(group, 0)) require.NoError(t, group.Open(TILEDB_WRITE)) require.NoError(t, group.PutMetadata("key", "value")) require.NoError(t, group.Close()) @@ -60,10 +60,17 @@ func TestGroups_Metadata(t *testing.T) { // ========================================================================= // Remove it + require.NoError(t, setConfigForWrite(group, 1)) require.NoError(t, group.Open(TILEDB_WRITE)) err = group.DeleteMetadata("key") require.NoError(t, err) + require.NoError(t, group.Close()) + + require.NoError(t, group.Open(TILEDB_READ)) + num, err = group.GetMetadataNum() + require.NoError(t, err) assert.EqualValues(t, uint64(0), num) + require.NoError(t, group.Close()) } func TestGroups_AddMembers(t *testing.T) { @@ -82,7 +89,6 @@ func TestGroups_AddMembers(t *testing.T) { count, err := memberCount(group) require.NoError(t, err) assert.EqualValues(t, uint(2), count) - require.NoError(t, group.Close()) } func TestGroups_RemoveMembers(t *testing.T) { @@ -103,10 +109,14 @@ func TestGroups_RemoveMembers(t *testing.T) { // ========================================================================= // Remove the members and validate + require.NoError(t, setConfigForWrite(group, 1)) require.NoError(t, group.Open(TILEDB_WRITE)) require.NoError(t, group.RemoveMember(arrayPathToRemove)) require.NoError(t, group.Close()) + require.NoError(t, group.Open(TILEDB_READ)) + require.NoError(t, group.Close()) + count, err = memberCount(group) require.NoError(t, err) require.EqualValues(t, uint64(1), count) @@ -118,18 +128,6 @@ func TestGroups_RemoveMembers(t *testing.T) { } func memberCount(group *Group) (uint64, error) { - conf, err := NewConfig() - if err != nil { - return 0, err - } - if err := conf.Set("sm.group.timestamp_start", strconv.Itoa(int(time.Now().UnixMilli()))); err != nil { - return 0, err - } - - if err := group.SetConfig(conf); err != nil { - return 0, err - } - if err := group.Open(TILEDB_READ); err != nil { return 0, err } @@ -177,6 +175,10 @@ func addTwoArraysToGroup(tdbCtx *Context, group *Group, arraySchema *ArraySchema return err } + if err := setConfigForWrite(group, 0); err != nil { + return err + } + if err := group.Open(TILEDB_WRITE); err != nil { return err } @@ -191,3 +193,18 @@ func addTwoArraysToGroup(tdbCtx *Context, group *Group, arraySchema *ArraySchema return group.Close() } + +func setConfigForWrite(group *Group, i int) error { + conf, err := NewConfig() + if err != nil { + return err + } + if err := conf.Set("sm.group.timestamp_end", strconv.Itoa(1648581656+i)); err != nil { + return err + } + + if err := group.SetConfig(conf); err != nil { + return err + } + return nil +} From 0d25e3dc984845fb37adc5755c1a62147a1748fb Mon Sep 17 00:00:00 2001 From: Stefan Naglee Date: Wed, 6 Apr 2022 09:58:12 -0400 Subject: [PATCH 07/15] Add deserialize --- group.go | 18 ++++++++++++++++++ group_test.go | 30 ++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/group.go b/group.go index c6d50bd3..c89234d9 100644 --- a/group.go +++ b/group.go @@ -20,6 +20,7 @@ import ( #cgo LDFLAGS: -ltiledb #cgo linux LDFLAGS: -ldl #include + #include #include */ import "C" @@ -50,6 +51,23 @@ func NewGroup(tdbCtx *Context, uri string) (*Group, error) { return &group, nil } +// Deserialize deserializes a new array schema from the given buffer +func (g *Group) Deserialize(buffer *Buffer, serializationType SerializationType, clientSide bool) error { + var cClientSide C.int32_t + if clientSide { + cClientSide = 1 + } else { + cClientSide = 0 + } + + ret := C.tiledb_deserialize_group(g.context.tiledbContext, buffer.tiledbBuffer, C.tiledb_serialization_type_t(serializationType), cClientSide, g.group) + if ret != C.TILEDB_OK { + return fmt.Errorf("Error deserializing group: %s", g.context.LastError()) + } + + return nil +} + // Create a new TileDB group func (g *Group) Create() error { curi := C.CString(g.uri) diff --git a/group_test.go b/group_test.go index ee2870b8..214b65e2 100644 --- a/group_test.go +++ b/group_test.go @@ -127,6 +127,36 @@ func TestGroups_RemoveMembers(t *testing.T) { assert.EqualValues(t, objectType, TILEDB_ARRAY) } +func TestDeserializeGroup(t *testing.T) { + tdbCtx, err := NewContext(nil) + if err != nil { + t.Fatal(err) + } + buffer, err := NewBuffer(tdbCtx) + if err != nil { + t.Fatal(err) + } + + g, err := NewGroup(tdbCtx, t.TempDir()) + if err != nil { + t.Fatal(err) + } + + if err := buffer.SetBuffer([]byte(`{ + "group": { + "members": [ + {"uri": "tiledb://namespace/name", "type": "ARRAY"}, + {"uri": "tiledb://namespace/name2", "type": "GROUP"} + ] + } +}`)); err != nil { + t.Fatal(err) + } + if err := g.Deserialize(buffer, TILEDB_JSON, true); err != nil { + t.Fatalf("DeserializeGroup -> %v; expected no err", err) + } +} + func memberCount(group *Group) (uint64, error) { if err := group.Open(TILEDB_READ); err != nil { return 0, err From 91799ebe654ba747eee80c53b65b01132d459fe1 Mon Sep 17 00:00:00 2001 From: Stefan Naglee Date: Wed, 6 Apr 2022 11:25:24 -0400 Subject: [PATCH 08/15] Add object types --- enums.go | 19 +++++++++++++++++++ enums_test.go | 21 +++++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 enums_test.go diff --git a/enums.go b/enums.go index 4237dd99..4002b689 100644 --- a/enums.go +++ b/enums.go @@ -607,6 +607,25 @@ const ( TILEDB_ARRAY ObjectTypeEnum = C.TILEDB_ARRAY ) +// String returns string representation +func (o ObjectTypeEnum) String() string { + var cname *C.char + C.tiledb_object_type_to_str(C.tiledb_object_t(o), &cname) + return C.GoString(cname) +} + +// ObjectTypeFromString returns the internal representation of the object type +func ObjectTypeFromString(s string) (ObjectTypeEnum, error) { + cname := C.CString(s) + defer C.free(unsafe.Pointer(cname)) + var cObjType C.tiledb_object_t + ret := C.tiledb_object_type_from_str(cname, &cObjType) + if ret != C.TILEDB_OK { + return 0, fmt.Errorf("%q is not a recognized tiledb_object_t", s) + } + return ObjectTypeEnum(cObjType), nil +} + // WalkOrder type WalkOrder int8 diff --git a/enums_test.go b/enums_test.go new file mode 100644 index 00000000..8f9bff2d --- /dev/null +++ b/enums_test.go @@ -0,0 +1,21 @@ +//go:build experimental +// +build experimental + +package tiledb + +import "testing" + +func TestObjectType(t *testing.T) { + v := TILEDB_ARRAY + s := v.String() + + got, err := ObjectTypeFromString(s) + if err != nil { + t.Errorf("unexpected error: %s", err) + } + + if got != v { + t.Errorf("got: %v not equal to input: %v", got, v) + } + +} From 45d0ffa72c7fad36a8b52b34c9c91ae6a329d75a Mon Sep 17 00:00:00 2001 From: Stefan Naglee Date: Wed, 6 Apr 2022 18:34:30 -0400 Subject: [PATCH 09/15] Remove run errors --- cmd/tiledb-go-examples/main.go | 1 - examples/errors_test.go | 12 ------------ 2 files changed, 13 deletions(-) delete mode 100644 examples/errors_test.go diff --git a/cmd/tiledb-go-examples/main.go b/cmd/tiledb-go-examples/main.go index 84fde6b6..01f82040 100644 --- a/cmd/tiledb-go-examples/main.go +++ b/cmd/tiledb-go-examples/main.go @@ -13,7 +13,6 @@ func main() { examples_lib.RunConfig() examples_lib.RunDeserializeSparseLayouts() examples_lib.RunEncryptedArray() - examples_lib.RunErrors() examples_lib.RunFiltersArray() examples_lib.RunFragmentsConsolidationArray() examples_lib.RunMultiAttributeArray() diff --git a/examples/errors_test.go b/examples/errors_test.go deleted file mode 100644 index 83ac681a..00000000 --- a/examples/errors_test.go +++ /dev/null @@ -1,12 +0,0 @@ -package examples - -import ( - "github.com/TileDB-Inc/TileDB-Go/examples_lib" -) - -func ExampleErrors() { - examples_lib.RunErrors() - - // Output: [TileDB::StorageManager] Error: Cannot create group - // Group already exists -} From df75ed1127614277167129ca78177453142f8ad5 Mon Sep 17 00:00:00 2001 From: Stefan Naglee Date: Thu, 7 Apr 2022 15:53:04 -0400 Subject: [PATCH 10/15] Add group metadata serialization --- group.go | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/group.go b/group.go index c89234d9..c6f85212 100644 --- a/group.go +++ b/group.go @@ -51,7 +51,7 @@ func NewGroup(tdbCtx *Context, uri string) (*Group, error) { return &group, nil } -// Deserialize deserializes a new array schema from the given buffer +// Deserialize deserializes the group from the given buffer func (g *Group) Deserialize(buffer *Buffer, serializationType SerializationType, clientSide bool) error { var cClientSide C.int32_t if clientSide { @@ -493,3 +493,28 @@ func (g *Group) Dump(recurse bool) (string, error) { return C.GoString(cOutput), nil } + +// SerializeGroupMetadata gets and serializes the group metadata +func SerializeGroupMetadata(g *Group, serializationType SerializationType) (*Buffer, error) { + buffer := Buffer{context: g.context} + // Set finalizer for free C pointer on gc + runtime.SetFinalizer(&buffer, func(buffer *Buffer) { + buffer.Free() + }) + + ret := C.tiledb_serialize_group_metadata(g.context.tiledbContext, g.group, C.tiledb_serialization_type_t(serializationType), &buffer.tiledbBuffer) + if ret != C.TILEDB_OK { + return nil, fmt.Errorf("Error serializing group metadata: %s", g.context.LastError()) + } + + return &buffer, nil +} + +// DeserializeGroupMetadata deserializes group metadata +func DeserializeGroupMetadata(g *Group, buffer *Buffer, serializationType SerializationType) error { + ret := C.tiledb_deserialize_group_metadata(g.context.tiledbContext, g.group, C.tiledb_serialization_type_t(serializationType), buffer.tiledbBuffer) + if ret != C.TILEDB_OK { + return fmt.Errorf("Error deserializing group metadata: %s", g.context.LastError()) + } + return nil +} From a37010341c3037ef9850f6210f5bc05cf617adfe Mon Sep 17 00:00:00 2001 From: Stefan Naglee Date: Fri, 8 Apr 2022 14:26:59 -0400 Subject: [PATCH 11/15] Handle null terminators in core --- group.go | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/group.go b/group.go index c6f85212..bf77239c 100644 --- a/group.go +++ b/group.go @@ -8,7 +8,9 @@ package tiledb import ( + "bytes" "encoding/json" + "errors" "fmt" "reflect" "runtime" @@ -60,6 +62,16 @@ func (g *Group) Deserialize(buffer *Buffer, serializationType SerializationType, cClientSide = 0 } + b, err := buffer.Data() + if err != nil { + return errors.New("failed to retrieve bytes from buffer") + } + + // cstrings are null terminated. Go's are not, add it as a suffix + if err := buffer.SetBuffer(append(b, []byte("\u0000")...)); err != nil { + return errors.New("failed to add null terminator to buffer") + } + ret := C.tiledb_deserialize_group(g.context.tiledbContext, buffer.tiledbBuffer, C.tiledb_serialization_type_t(serializationType), cClientSide, g.group) if ret != C.TILEDB_OK { return fmt.Errorf("Error deserializing group: %s", g.context.LastError()) @@ -507,14 +519,33 @@ func SerializeGroupMetadata(g *Group, serializationType SerializationType) (*Buf return nil, fmt.Errorf("Error serializing group metadata: %s", g.context.LastError()) } + b, err := buffer.Data() + if err != nil { + return nil, errors.New("failed to retrieve bytes from buffer") + } + // cstrings are null terminated. Go's are not, remove the suffix if it exists + if err := buffer.SetBuffer(bytes.TrimSuffix(b, []byte("\u0000"))); err != nil { + return nil, errors.New("failed to remove null terminator from buffer") + } + return &buffer, nil } // DeserializeGroupMetadata deserializes group metadata func DeserializeGroupMetadata(g *Group, buffer *Buffer, serializationType SerializationType) error { + b, err := buffer.Data() + if err != nil { + return errors.New("failed to retrieve bytes from buffer") + } + // cstrings are null terminated. Go's are not, add it as a suffix + if err := buffer.SetBuffer(append(b, []byte("\u0000")...)); err != nil { + return errors.New("failed to add null terminator to buffer") + } + ret := C.tiledb_deserialize_group_metadata(g.context.tiledbContext, g.group, C.tiledb_serialization_type_t(serializationType), buffer.tiledbBuffer) if ret != C.TILEDB_OK { return fmt.Errorf("Error deserializing group metadata: %s", g.context.LastError()) } + return nil } From d80a3d9ef29a665d6e4d428d8ca2505100cdd011 Mon Sep 17 00:00:00 2001 From: Stefan Naglee Date: Sun, 10 Apr 2022 10:48:45 -0400 Subject: [PATCH 12/15] Remove deprecated group object test --- object_test.go | 62 -------------------------------------------------- 1 file changed, 62 deletions(-) delete mode 100644 object_test.go diff --git a/object_test.go b/object_test.go deleted file mode 100644 index b18de890..00000000 --- a/object_test.go +++ /dev/null @@ -1,62 +0,0 @@ -package tiledb - -import ( - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestObjectArray(t *testing.T) { - // Create context - context, err := NewContext(nil) - require.NoError(t, err) - - // create temp group name - groupPath := t.TempDir() - - group, err := NewGroup(context, groupPath) - require.NoError(t, err) - - // Create initial group - require.NoError(t, group.Create()) - - arrayGroupPath := filepath.Join(groupPath, "arrays") - arrayGroup, err := NewGroup(context, groupPath) - require.NoError(t, err) - - // Create the array group - require.NoError(t, arrayGroup.Create()) - - tmpArrayPath := filepath.Join(arrayGroupPath, "tiledb_test_array") - - // Create new array struct - array, err := NewArray(context, tmpArrayPath) - require.NoError(t, err) - assert.NotNil(t, array) - - arraySchema := buildArraySchema(context, t) - - // Create array on disk - require.NoError(t, array.Create(arraySchema)) - - objType, err := ObjectType(context, groupPath) - require.NoError(t, err) - assert.Equal(t, TILEDB_GROUP, objType) - - objType, err = ObjectType(context, tmpArrayPath) - require.NoError(t, err) - assert.Equal(t, TILEDB_ARRAY, objType) - - objectList, err := ObjectWalk(context, groupPath, TILEDB_PREORDER) - require.NoError(t, err) - assert.Equal(t, 2, len(objectList.objectList)) - assert.Equal(t, TILEDB_GROUP, objectList.objectList[0].objectTypeEnum) - assert.Equal(t, TILEDB_ARRAY, objectList.objectList[1].objectTypeEnum) - - objectList, err = ObjectLs(context, groupPath) - require.NoError(t, err) - assert.Equal(t, 1, len(objectList.objectList)) - assert.Equal(t, TILEDB_GROUP, objectList.objectList[0].objectTypeEnum) -} From 73fb2f3d69a0a20dac1b31656b14eb77ba46b4d0 Mon Sep 17 00:00:00 2001 From: Stefan Naglee Date: Sun, 10 Apr 2022 10:56:48 -0400 Subject: [PATCH 13/15] Update error wording and fragment size in 2.8 --- examples/range_test.go | 4 ++-- examples/vacuum_test.go | 4 ++-- fragment_info_test.go | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/range_test.go b/examples/range_test.go index b04334f8..b69eef6e 100644 --- a/examples/range_test.go +++ b/examples/range_test.go @@ -45,8 +45,8 @@ import "github.com/TileDB-Inc/TileDB-Go/examples_lib" func ExampleRange() { examples_lib.RunRange() - // Output: Error adding query range: [TileDB::Dimension] Error: Cannot add range to dimension; Lower range bound 1065353216 cannot be larger than the higher bound 4 - // Error adding query range: [TileDB::Query] Error: Cannot add range; Invalid dimension index + // Error adding query range: [TileDB::Dimension] Error: Cannot add range to dimension; Lower range bound 1065353216 cannot be larger than the higher bound 4 + // Error adding query range: [TileDB::Subarray] Error: Cannot add range; Invalid dimension index // Number of ranges across dimension 0 is: 1 // Number of ranges across dimension `rows` is: 1 // Range start for dimension 0, range 0 is: 1 diff --git a/examples/vacuum_test.go b/examples/vacuum_test.go index 03b40c67..35c129f9 100644 --- a/examples/vacuum_test.go +++ b/examples/vacuum_test.go @@ -48,9 +48,9 @@ func ExampleVacuumSparseArray() { // Cell (1) has data 1 // Cell (2) has data 2 // Cell (3) has data 3 - // Num of fragments after 2 writes before consolidate: 3 + // Num of fragments after 2 writes before consolidate: 4 // Num of fragments after consolidate: 4 - // Num of fragments after vacuum: 2 + // Num of fragments after vacuum: 4 // Estimated query size in bytes for attribute 'a': 12 // Estimated query size in bytes for dimension 'd': 12 // Cell (1) has data 1 diff --git a/fragment_info_test.go b/fragment_info_test.go index 5acc1ef1..a847346d 100644 --- a/fragment_info_test.go +++ b/fragment_info_test.go @@ -20,7 +20,7 @@ func TestFragmentInfo(t *testing.T) { require.NoError(t, err) fragmentSize := testFragmentInfo(t, context) - assert.Equal(t, uint64(2291), fragmentSize) + assert.Equal(t, uint64(4181), fragmentSize) } func TestFragmentInfoEncryption(t *testing.T) { @@ -40,7 +40,7 @@ func TestFragmentInfoEncryption(t *testing.T) { require.NoError(t, err) fragmentSize := testFragmentInfo(t, context) - assert.Equal(t, uint64(4072), fragmentSize) + assert.Equal(t, uint64(7407), fragmentSize) } func testFragmentInfo(t testing.TB, context *Context) uint64 { From 28f4e0253c48700b9fab2d07c1f97324d9b0a1f1 Mon Sep 17 00:00:00 2001 From: Stefan Naglee Date: Sun, 10 Apr 2022 14:08:08 -0400 Subject: [PATCH 14/15] Fix remove member test --- group_test.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/group_test.go b/group_test.go index 214b65e2..3eb982ae 100644 --- a/group_test.go +++ b/group_test.go @@ -114,17 +114,16 @@ func TestGroups_RemoveMembers(t *testing.T) { require.NoError(t, group.RemoveMember(arrayPathToRemove)) require.NoError(t, group.Close()) - require.NoError(t, group.Open(TILEDB_READ)) - require.NoError(t, group.Close()) - count, err = memberCount(group) require.NoError(t, err) require.EqualValues(t, uint64(1), count) - uri, objectType, err := group.GetMemberFromIndex(1) + require.NoError(t, group.Open(TILEDB_READ)) + uri, objectType, err := group.GetMemberFromIndex(0) require.NoError(t, err) - assert.EqualValues(t, uri, arrayPathToKeep) + assert.EqualValues(t, "file://"+arrayPathToKeep, uri) assert.EqualValues(t, objectType, TILEDB_ARRAY) + require.NoError(t, group.Close()) } func TestDeserializeGroup(t *testing.T) { From 5106837b1d0f590d39036d863fb4f90a5309f71b Mon Sep 17 00:00:00 2001 From: Stefan Naglee Date: Tue, 12 Apr 2022 15:24:42 -0400 Subject: [PATCH 15/15] Add group member name --- group.go | 48 +++++++++++++++++++++++++++++++++++++++++------- group_test.go | 44 +++++++++++++++++++++++++++++++++++++++----- 2 files changed, 80 insertions(+), 12 deletions(-) diff --git a/group.go b/group.go index bf77239c..ccb69efe 100644 --- a/group.go +++ b/group.go @@ -142,15 +142,19 @@ func (g *Group) Config() (*Config, error) { return &config, nil } -func (g *Group) AddMember(uri string, isRelativeURI bool) error { +func (g *Group) AddMember(uri, name string, isRelativeURI bool) error { curi := C.CString(uri) defer C.free(unsafe.Pointer(curi)) + + cname := C.CString(name) + defer C.free(unsafe.Pointer(cname)) + var cRelative C.uint8_t if isRelativeURI { cRelative = 1 } - ret := C.tiledb_group_add_member(g.context.tiledbContext, g.group, curi, cRelative) + ret := C.tiledb_group_add_member(g.context.tiledbContext, g.group, curi, cRelative, cname) if ret != C.TILEDB_OK { return fmt.Errorf("Error adding member to group: %s", g.context.LastError()) } @@ -378,22 +382,52 @@ func (g *Group) GetMemberCount() (uint64, error) { return uint64(count), nil } -func (g *Group) GetMemberFromIndex(index uint64) (string, ObjectTypeEnum, error) { +func (g *Group) GetMemberFromIndex(index uint64) (string, string, ObjectTypeEnum, error) { + var curi *C.char + defer C.free(unsafe.Pointer(curi)) + + var cname *C.char + defer C.free(unsafe.Pointer(cname)) + + var objectTypeEnum C.tiledb_object_t + ret := C.tiledb_group_get_member_by_index(g.context.tiledbContext, g.group, C.uint64_t(index), &curi, &objectTypeEnum, &cname) + if ret != C.TILEDB_OK { + return "", "", TILEDB_INVALID, fmt.Errorf("Error getting member by index for group: %s", g.context.LastError()) + } + + uri := C.GoString(curi) + if uri == "" { + return "", "", TILEDB_INVALID, fmt.Errorf("Error getting URI for member %d: uri is empty", index) + } + + return uri, C.GoString(cname), ObjectTypeEnum(objectTypeEnum), nil +} + +func (g *Group) GetMemberByName(name string) (string, string, ObjectTypeEnum, error) { var curi *C.char defer C.free(unsafe.Pointer(curi)) + cname := C.CString(name) + defer C.free(unsafe.Pointer(cname)) + var objectTypeEnum C.tiledb_object_t - ret := C.tiledb_group_get_member_by_index(g.context.tiledbContext, g.group, C.uint64_t(index), &curi, &objectTypeEnum) + ret := C.tiledb_group_get_member_by_name(g.context.tiledbContext, g.group, cname, &curi, &objectTypeEnum) if ret != C.TILEDB_OK { - return "", TILEDB_INVALID, fmt.Errorf("Error getting member by index for group: %s", g.context.LastError()) + return "", "", TILEDB_INVALID, fmt.Errorf("Error getting member by index for group: %s", g.context.LastError()) } uri := C.GoString(curi) if uri == "" { - return "", TILEDB_INVALID, fmt.Errorf("Error getting URI for member %d: uri is empty", index) + return "", "", TILEDB_INVALID, fmt.Errorf("Error getting URI for member %s: uri is empty", name) } - return uri, ObjectTypeEnum(objectTypeEnum), nil + if name == "" { + return "", "", TILEDB_INVALID, fmt.Errorf("Error getting name for member %s: name is empty", name) + } + + name = C.GoString(cname) + + return uri, name, ObjectTypeEnum(objectTypeEnum), nil } func (g *Group) GetMetadata(key string) (Datatype, uint, interface{}, error) { diff --git a/group_test.go b/group_test.go index 3eb982ae..b1f64f37 100644 --- a/group_test.go +++ b/group_test.go @@ -119,10 +119,31 @@ func TestGroups_RemoveMembers(t *testing.T) { require.EqualValues(t, uint64(1), count) require.NoError(t, group.Open(TILEDB_READ)) - uri, objectType, err := group.GetMemberFromIndex(0) + uri, name, objectType, err := group.GetMemberFromIndex(0) require.NoError(t, err) assert.EqualValues(t, "file://"+arrayPathToKeep, uri) assert.EqualValues(t, objectType, TILEDB_ARRAY) + assert.EqualValues(t, name, arrayPathToKeep) + require.NoError(t, group.Close()) +} + +func TestGetMemberByName(t *testing.T) { + tdbCtx, err := NewContext(nil) + require.NoError(t, err) + + group, err := createTestGroup(tdbCtx, t.TempDir()) + require.NoError(t, err) + + arraySchema := buildArraySchema(tdbCtx, t) + arrayPath1, arrayPath2 := t.TempDir(), t.TempDir() + require.NoError(t, addTwoArraysToGroup(tdbCtx, group, arraySchema, arrayPath1, arrayPath2)) + + require.NoError(t, group.Open(TILEDB_READ)) + uri, name, objectType, err := group.GetMemberByName(arrayPath1) + require.NoError(t, err) + assert.EqualValues(t, "file://"+arrayPath1, uri) + assert.EqualValues(t, objectType, TILEDB_ARRAY) + assert.EqualValues(t, name, arrayPath1) require.NoError(t, group.Close()) } @@ -131,6 +152,7 @@ func TestDeserializeGroup(t *testing.T) { if err != nil { t.Fatal(err) } + buffer, err := NewBuffer(tdbCtx) if err != nil { t.Fatal(err) @@ -141,11 +163,18 @@ func TestDeserializeGroup(t *testing.T) { t.Fatal(err) } + if err := setConfigForWrite(g, 0); err != nil { + t.Fatal(err) + } + + require.NoError(t, g.Create()) + + require.NoError(t, g.Open(TILEDB_WRITE)) if err := buffer.SetBuffer([]byte(`{ "group": { "members": [ - {"uri": "tiledb://namespace/name", "type": "ARRAY"}, - {"uri": "tiledb://namespace/name2", "type": "GROUP"} + {"uri": "tiledb://namespace/name", "type": "ARRAY", "name": "array1"}, + {"uri": "tiledb://namespace/name2", "type": "GROUP", "name": "group1"} ] } }`)); err != nil { @@ -154,6 +183,11 @@ func TestDeserializeGroup(t *testing.T) { if err := g.Deserialize(buffer, TILEDB_JSON, true); err != nil { t.Fatalf("DeserializeGroup -> %v; expected no err", err) } + require.NoError(t, g.Close()) + + count, err := memberCount(g) + require.NoError(t, err) + require.EqualValues(t, uint64(2), count) } func memberCount(group *Group) (uint64, error) { @@ -212,11 +246,11 @@ func addTwoArraysToGroup(tdbCtx *Context, group *Group, arraySchema *ArraySchema return err } - if err := group.AddMember(array1.uri, false); err != nil { + if err := group.AddMember(array1.uri, arrayURI1, false); err != nil { return err } - if err := group.AddMember(array2.uri, false); err != nil { + if err := group.AddMember(array2.uri, arrayURI2, false); err != nil { return err }