From ade824860d97d24d582bbf584cb02015dfe7b662 Mon Sep 17 00:00:00 2001 From: matthiasmatt Date: Mon, 28 Oct 2024 11:18:47 +0100 Subject: [PATCH 1/2] feat: Update TimeKeyEncoder to Preserve Lexicographical Order --- keys.go | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/keys.go b/keys.go index 79675af..b381df4 100644 --- a/keys.go +++ b/keys.go @@ -1,6 +1,7 @@ package collections import ( + "encoding/binary" "fmt" "strconv" "time" @@ -59,13 +60,20 @@ func (uint64Key) Decode(b []byte) (int, uint64) { return 8, sdk.BigEndianToUint6 type timeKey struct{} func (timeKey) Stringify(t time.Time) string { return t.String() } -func (timeKey) Encode(t time.Time) []byte { return sdk.FormatTimeBytes(t) } + +func (timeKey) Encode(t time.Time) []byte { + // Use Unix milliseconds to reduce the size (8 bytes) + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, uint64(t.UnixMilli())) + return b +} + func (timeKey) Decode(b []byte) (int, time.Time) { - t, err := sdk.ParseTimeBytes(b) - if err != nil { - panic(fmt.Errorf("%w %s", err, HumanizeBytes(b))) + if len(b) < 8 { + panic("invalid time key") } - return len(b), t + ts := binary.BigEndian.Uint64(b[:8]) + return 8, time.UnixMilli(int64(ts)) } type accAddressKey struct{} From f912b743f69edac10f23cdb76273034bd053ae24 Mon Sep 17 00:00:00 2001 From: matthiasmatt Date: Mon, 28 Oct 2024 11:25:58 +0100 Subject: [PATCH 2/2] feat: Maintain Ns Precision --- go.mod | 2 +- keys.go | 7 +- keyset_test.go | 220 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 224 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 781f9f7..2984110 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( cosmossdk.io/store v1.1.0 github.com/cosmos/cosmos-db v1.0.2 github.com/cosmos/cosmos-sdk v0.50.6 + github.com/cosmos/gogoproto v1.4.12 github.com/gogo/protobuf v1.3.3 github.com/stretchr/testify v1.9.0 ) @@ -43,7 +44,6 @@ require ( github.com/cosmos/cosmos-proto v1.0.0-beta.5 // indirect github.com/cosmos/go-bip39 v1.0.0 // indirect github.com/cosmos/gogogateway v1.2.0 // indirect - github.com/cosmos/gogoproto v1.4.12 // indirect github.com/cosmos/iavl v1.1.2 // indirect github.com/cosmos/ics23/go v0.10.0 // indirect github.com/cosmos/ledger-cosmos-go v0.13.3 // indirect diff --git a/keys.go b/keys.go index b381df4..aec2ae1 100644 --- a/keys.go +++ b/keys.go @@ -62,9 +62,8 @@ type timeKey struct{} func (timeKey) Stringify(t time.Time) string { return t.String() } func (timeKey) Encode(t time.Time) []byte { - // Use Unix milliseconds to reduce the size (8 bytes) b := make([]byte, 8) - binary.BigEndian.PutUint64(b, uint64(t.UnixMilli())) + binary.BigEndian.PutUint64(b, uint64(t.UnixNano())) return b } @@ -72,8 +71,8 @@ func (timeKey) Decode(b []byte) (int, time.Time) { if len(b) < 8 { panic("invalid time key") } - ts := binary.BigEndian.Uint64(b[:8]) - return 8, time.UnixMilli(int64(ts)) + ts := int64(binary.BigEndian.Uint64(b[:8])) + return 8, time.Unix(0, ts).UTC() } type accAddressKey struct{} diff --git a/keyset_test.go b/keyset_test.go index ac20952..1efd94a 100644 --- a/keyset_test.go +++ b/keyset_test.go @@ -1,7 +1,9 @@ package collections import ( + "sort" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -53,3 +55,221 @@ func TestKeysetIterator(t *testing.T) { iter.Next() assert.False(t, iter.Valid()) } + +func TestTimeKeySet(t *testing.T) { + sk, ctx, _ := deps() + keyset := NewKeySet[time.Time](sk, 0, TimeKeyEncoder) + + // Use a fixed time + now := time.Date(2021, time.January, 1, 0, 0, 0, 0, time.UTC) + keyset.Insert(ctx, now) + require.True(t, keyset.Has(ctx, now)) + + // Test delete and get + keyset.Delete(ctx, now) + require.False(t, keyset.Has(ctx, now)) +} + +func TestTimeKeySet_IterateAscending(t *testing.T) { + sk, ctx, _ := deps() + keyset := NewKeySet[time.Time](sk, 0, TimeKeyEncoder) + + now := time.Date(2021, time.January, 1, 0, 0, 0, 0, time.UTC) + times := []time.Time{ + now.Add(2 * time.Second), + now.Add(1 * time.Second), + now.Add(3 * time.Second), + now, + } + + // Insert times into the keyset + for _, t := range times { + keyset.Insert(ctx, t) + } + + // Sort times in ascending order + sort.Slice(times, func(i, j int) bool { + return times[i].Before(times[j]) + }) + + // Iterate over the keyset in ascending order + iter := keyset.Iterate(ctx, Range[time.Time]{}) + defer iter.Close() + + keys := iter.Keys() + require.Equal(t, len(times), len(keys)) + + for i, k := range keys { + // Strip monotonic clock readings + expectedTime := times[i].Round(0) + actualTime := k.Round(0) + + // Compare UnixNano timestamps + require.Equal(t, expectedTime.UnixNano(), actualTime.UnixNano()) + } +} + +func TestTimeKeySet_IterateDescending(t *testing.T) { + sk, ctx, _ := deps() + keyset := NewKeySet[time.Time](sk, 0, TimeKeyEncoder) + + now := time.Date(2021, time.January, 1, 0, 0, 0, 0, time.UTC) + times := []time.Time{ + now.Add(2 * time.Second), + now.Add(1 * time.Second), + now.Add(3 * time.Second), + now, + } + + // Insert times into the keyset + for _, t := range times { + keyset.Insert(ctx, t) + } + + // Sort times in descending order + sort.Slice(times, func(i, j int) bool { + return times[i].After(times[j]) + }) + + // Iterate over the keyset in descending order + iter := keyset.Iterate(ctx, Range[time.Time]{}.Descending()) + defer iter.Close() + + keys := iter.Keys() + require.Equal(t, len(times), len(keys)) + + for i, k := range keys { + expectedTime := times[i].Round(0) + actualTime := k.Round(0) + require.Equal(t, expectedTime.UnixNano(), actualTime.UnixNano()) + } +} + +func TestTimeKeyEncoder_EncodeDecode(t *testing.T) { + now := time.Date(2021, time.January, 1, 0, 0, 0, 0, time.UTC) + encoded := TimeKeyEncoder.Encode(now) + _, decoded := TimeKeyEncoder.Decode(encoded) + + // Compare UnixNano timestamps + require.Equal(t, now.UnixNano(), decoded.UnixNano()) +} + +func TestTimeKeySet_OrderConsistency(t *testing.T) { + sk, ctx, _ := deps() + keyset := NewKeySet[time.Time](sk, 0, TimeKeyEncoder) + + now := time.Date(2021, time.January, 1, 0, 0, 0, 0, time.UTC) + times := []time.Time{ + now.Add(-1 * time.Hour), + now, + now.Add(1 * time.Hour), + now.Add(2 * time.Hour), + now.Add(-2 * time.Hour), + } + + // Insert times into the keyset + for _, t := range times { + keyset.Insert(ctx, t) + } + + // Sort times in ascending order + sortedTimesAsc := make([]time.Time, len(times)) + copy(sortedTimesAsc, times) + sort.Slice(sortedTimesAsc, func(i, j int) bool { + return sortedTimesAsc[i].Before(sortedTimesAsc[j]) + }) + + // Iterate over the keyset in ascending order + iterAsc := keyset.Iterate(ctx, Range[time.Time]{}) + defer iterAsc.Close() + + keysAsc := iterAsc.Keys() + require.Equal(t, len(times), len(keysAsc)) + + for i, k := range keysAsc { + expectedTime := sortedTimesAsc[i].Round(0) + actualTime := k.Round(0) + require.Equal(t, expectedTime.UnixNano(), actualTime.UnixNano()) + } + + // Sort times in descending order + sortedTimesDesc := make([]time.Time, len(times)) + copy(sortedTimesDesc, times) + sort.Slice(sortedTimesDesc, func(i, j int) bool { + return sortedTimesDesc[i].After(sortedTimesDesc[j]) + }) + + // Iterate over the keyset in descending order + iterDesc := keyset.Iterate(ctx, Range[time.Time]{}.Descending()) + defer iterDesc.Close() + + keysDesc := iterDesc.Keys() + require.Equal(t, len(times), len(keysDesc)) + + for i, k := range keysDesc { + expectedTime := sortedTimesDesc[i].Round(0) + actualTime := k.Round(0) + require.Equal(t, expectedTime.UnixNano(), actualTime.UnixNano()) + } +} + +func TestTimeKeySet_IterateRange(t *testing.T) { + sk, ctx, _ := deps() + keyset := NewKeySet[time.Time](sk, 0, TimeKeyEncoder) + + now := time.Date(2021, time.January, 1, 0, 0, 0, 0, time.UTC) + times := []time.Time{ + now.Add(1 * time.Second), + now.Add(2 * time.Second), + now.Add(3 * time.Second), + now.Add(4 * time.Second), + now.Add(5 * time.Second), + } + + // Insert times into the keyset + for _, t := range times { + keyset.Insert(ctx, t) + } + + // Define range from now.Add(2s) inclusive to now.Add(4s) exclusive + iter := keyset.Iterate(ctx, Range[time.Time]{}. + StartInclusive(now.Add(2*time.Second)). + EndExclusive(now.Add(4*time.Second))) + defer iter.Close() + + expectedTimes := []time.Time{ + now.Add(2 * time.Second), + now.Add(3 * time.Second), + } + + keys := iter.Keys() + require.Equal(t, len(expectedTimes), len(keys)) + + for i, k := range keys { + expectedTime := expectedTimes[i].Round(0) + actualTime := k.Round(0) + require.Equal(t, expectedTime.UnixNano(), actualTime.UnixNano()) + } +} + +func TestTimeKeySet_SameTimeKeys(t *testing.T) { + sk, ctx, _ := deps() + keyset := NewKeySet[time.Time](sk, 0, TimeKeyEncoder) + + now := time.Date(2021, time.January, 1, 0, 0, 0, 0, time.UTC) + + // Insert the same time multiple times (should only be stored once in a set) + keyset.Insert(ctx, now) + keyset.Insert(ctx, now) + keyset.Insert(ctx, now) + + iter := keyset.Iterate(ctx, Range[time.Time]{}) + defer iter.Close() + + keys := iter.Keys() + require.Equal(t, 1, len(keys)) + + expectedTime := now.Round(0) + actualTime := keys[0].Round(0) + require.Equal(t, expectedTime.UnixNano(), actualTime.UnixNano()) +}