From 0851701f09c738a0d9e78b86d304197cff2df2ac Mon Sep 17 00:00:00 2001 From: Miroslav Balaz Date: Sat, 3 Feb 2024 11:24:07 +0100 Subject: [PATCH 1/4] Update update_test.go Added UpdateSameKeyLinerity --- test/update_test.go | 60 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/test/update_test.go b/test/update_test.go index 1daaf7a2..3b0b941a 100644 --- a/test/update_test.go +++ b/test/update_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "testing" + "time" . "github.com/onsi/gomega" clientv3 "go.etcd.io/etcd/client/v3" @@ -77,6 +78,24 @@ func TestUpdate(t *testing.T) { g.Expect(resp.Kvs[0].ModRevision).To(BeNumerically(">", resp.Kvs[0].CreateRevision)) } }) + t.Run("UpdateSameKeyLinerity", func(t *testing.T) { + g := NewWithT(t) + + // Add a large number of entries, two times and + // it should take approximately same amout of time + numAddEntries := 1000 + + startFirst := time.Now() + addSameEntries(ctx, g, client, numAddEntries, true) + durationFirstBatch := time.Since(startFirst) + + startSecond := time.Now() + addSameEntries(ctx, g, client, numAddEntries, false) + durationSecondBatch := time.Since(startSecond) + + g.Expect(durationSecondBatch <= durationFirstBatch*2).To(BeTrue()) + + }) // Trying to update an old revision(in compare) should fail t.Run("UpdateOldRevisionFails", func(t *testing.T) { @@ -125,6 +144,47 @@ func TestUpdate(t *testing.T) { } +func addSameEntries(ctx context.Context, g Gomega, client *clientv3.Client, numEntries int, create_first bool) { + for i := 0; i < numEntries; i++ { + key := "testkey-same" + value := fmt.Sprintf("value-%d", i) + + if i != 0 || !create_first { + updateEntry(ctx, g, client, key, value) + } else { + addEntry(ctx, g, client, key, value) + } + } +} + +func addEntry(ctx context.Context, g Gomega, client *clientv3.Client, key string, value string) { + resp, err := client.Txn(ctx). + If(clientv3.Compare(clientv3.ModRevision(key), "=", 0)). + Then(clientv3.OpPut(key, value)). + Commit() + + g.Expect(err).To(BeNil()) + g.Expect(resp.Succeeded).To(BeTrue()) +} + +func updateEntry(ctx context.Context, g Gomega, client *clientv3.Client, key string, value string) { + + resp, err := client.Get(ctx, key, clientv3.WithRange("")) + + g.Expect(err).To(BeNil()) + g.Expect(resp.Kvs).To(HaveLen(1)) + + resp2, err2 := client.Txn(ctx). + If(clientv3.Compare(clientv3.ModRevision(key), "=", resp.Kvs[0].ModRevision)). + Then(clientv3.OpPut(key, value)). + Else(clientv3.OpGet(key, clientv3.WithRange(""))). + Commit() + + g.Expect(err2).To(BeNil()) + g.Expect(resp2.Succeeded).To(BeTrue()) + +} + // BenchmarkUpdate is a benchmark for the Update operation. func BenchmarkUpdate(b *testing.B) { ctx := context.Background() From 175d5609e1782c5088e637fa4976fafc8872ada1 Mon Sep 17 00:00:00 2001 From: Miroslav Balaz Date: Sat, 3 Feb 2024 11:26:42 +0100 Subject: [PATCH 2/4] Update update_test.go --- test/update_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/update_test.go b/test/update_test.go index 3b0b941a..fc4dcf2a 100644 --- a/test/update_test.go +++ b/test/update_test.go @@ -78,7 +78,7 @@ func TestUpdate(t *testing.T) { g.Expect(resp.Kvs[0].ModRevision).To(BeNumerically(">", resp.Kvs[0].CreateRevision)) } }) - t.Run("UpdateSameKeyLinerity", func(t *testing.T) { + t.Run("UpdateSameKeyLinearity", func(t *testing.T) { g := NewWithT(t) // Add a large number of entries, two times and From 79a9d2d3b755f590f0fce22f8693bfc47d36f14f Mon Sep 17 00:00:00 2001 From: Miroslav Balaz Date: Sat, 3 Feb 2024 11:32:04 +0100 Subject: [PATCH 3/4] Update update_test.go Removed addEntry --- test/update_test.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/test/update_test.go b/test/update_test.go index fc4dcf2a..5190e549 100644 --- a/test/update_test.go +++ b/test/update_test.go @@ -157,16 +157,6 @@ func addSameEntries(ctx context.Context, g Gomega, client *clientv3.Client, numE } } -func addEntry(ctx context.Context, g Gomega, client *clientv3.Client, key string, value string) { - resp, err := client.Txn(ctx). - If(clientv3.Compare(clientv3.ModRevision(key), "=", 0)). - Then(clientv3.OpPut(key, value)). - Commit() - - g.Expect(err).To(BeNil()) - g.Expect(resp.Succeeded).To(BeTrue()) -} - func updateEntry(ctx context.Context, g Gomega, client *clientv3.Client, key string, value string) { resp, err := client.Get(ctx, key, clientv3.WithRange("")) From 550618127673e261148a2c6af93babaae9a1b359 Mon Sep 17 00:00:00 2001 From: Miroslav Balaz Date: Sat, 3 Feb 2024 11:36:58 +0100 Subject: [PATCH 4/4] Update generic.go - fixed slow query --- pkg/kine/drivers/generic/generic.go | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/pkg/kine/drivers/generic/generic.go b/pkg/kine/drivers/generic/generic.go index 6f2e6c4a..27c9b6d1 100644 --- a/pkg/kine/drivers/generic/generic.go +++ b/pkg/kine/drivers/generic/generic.go @@ -24,14 +24,17 @@ var ( listSQL = fmt.Sprintf(` SELECT %s - FROM kine AS kv - LEFT JOIN kine kv2 - ON kv.name = kv2.name - AND kv.id < kv2.id - WHERE kv2.name IS NULL - AND kv.name >= ? AND kv.name < ? - AND (? OR kv.deleted = 0) - %%s + FROM kine kv + JOIN ( + SELECT MAX(mkv.id) as id + FROM kine mkv + WHERE + mkv.name >= ? AND mkv.name < ? + %%s + GROUP BY mkv.name) maxkv + ON maxkv.id = kv.id + WHERE + (kv.deleted = 0 OR ?) ORDER BY kv.id ASC `, columns) @@ -200,7 +203,7 @@ func Open(ctx context.Context, driverName, dataSourceName string, paramCharacter WHERE kv.id = ?`, columns), paramCharacter, numbered), GetCurrentSQL: q(fmt.Sprintf(listSQL, ""), paramCharacter, numbered), - ListRevisionStartSQL: q(fmt.Sprintf(listSQL, "AND kv.id <= ?"), paramCharacter, numbered), + ListRevisionStartSQL: q(fmt.Sprintf(listSQL, "AND mkv.id <= ?"), paramCharacter, numbered), GetRevisionAfterSQL: q(revisionAfterSQL, paramCharacter, numbered), CountSQL: q(fmt.Sprintf(`